From f0f17b9b6346d617477ee02e4ee3685ea88c1292 Mon Sep 17 00:00:00 2001 From: The412Banner Date: Sun, 25 Oct 2026 22:21:05 +0000 Subject: [PATCH] Vendor OpenNOW's core and native streamer (MIT, by Zortos) From OpenCloudGaming/OpenNOW a49b124c (2026-10-18): native/opennow-core and the native/opennow-streamer crates DroidDeck's GeForce NOW store builds on, unmodified, with OpenNOW's MIT license and a NOTICE crediting Zortos and the OpenNOW contributors. --- external/opennow/CHANGES.md | 3 + external/opennow/LICENSE | 21 + external/opennow/NOTICE.md | 17 + external/opennow/docs/core-protocol.md | 1857 ++ .../opennow/docs/qt-native-streamer-ffi.md | 69 + .../opennow/native/opennow-core/Cargo.lock | 3128 ++++ .../opennow/native/opennow-core/Cargo.toml | 47 + .../contracts/legacy-open-now-api.json | 116 + .../contracts/legacy-open-now-api.schema.json | 36 + .../opennow-core/src/account_connections.rs | 1459 ++ .../native/opennow-core/src/analytics.rs | 863 + .../native/opennow-core/src/artwork_cache.rs | 1033 ++ .../src/bin/opennow-acceptance-verify.rs | 1105 ++ .../src/bin/opennow-license-report.rs | 211 + .../src/bin/opennow-update-helper.rs | 15 + .../src/bin/opennow-update-manifest.rs | 231 + .../native/opennow-core/src/bug_reports.rs | 340 + .../native/opennow-core/src/catalog_types.rs | 367 + .../native/opennow-core/src/cloudmatch.rs | 7398 ++++++++ .../native/opennow-core/src/community.rs | 268 + .../opennow-core/src/console_profiles.rs | 506 + .../opennow-core/src/credential_vault.rs | 1242 ++ .../encrypted_secret_store.rs | 451 + .../src/credential_vault/json_secret_store.rs | 721 + .../opennow-core/src/device_identity.rs | 104 + .../native/opennow-core/src/diagnostics.rs | 1255 ++ .../native/opennow-core/src/discord.rs | 2113 +++ .../opennow/native/opennow-core/src/dlss.rs | 292 + .../native/opennow-core/src/frame_rate.rs | 403 + .../opennow/native/opennow-core/src/gfn.rs | 5252 ++++++ .../native/opennow-core/src/gfn/catalog.rs | 1216 ++ .../opennow-core/src/gfn/catalog_actions.rs | 607 + .../src/gfn/catalog_actions/tests.rs | 914 + .../opennow-core/src/gfn/catalog_tests.rs | 396 + .../opennow-core/src/gfn/dlss_support.rs | 763 + .../opennow-core/src/gfn/routing_tests.rs | 2050 +++ .../opennow-core/src/gfn/store_launch.rs | 262 + .../src/gfn/store_launch_tests.rs | 736 + .../native/opennow-core/src/language.rs | 161 + .../opennow/native/opennow-core/src/lib.rs | 2 + .../opennow/native/opennow-core/src/main.rs | 1735 ++ .../opennow/native/opennow-core/src/media.rs | 552 + .../native/opennow-core/src/network.rs | 321 + .../native/opennow-core/src/network_test.rs | 1425 ++ .../opennow-core/src/persistent_storage.rs | 193 + .../opennow/native/opennow-core/src/proxy.rs | 126 + .../native/opennow-core/src/push/config.rs | 347 + .../native/opennow-core/src/push/decrypt.rs | 368 + .../native/opennow-core/src/push/event.rs | 102 + .../native/opennow-core/src/push/mod.rs | 163 + .../native/opennow-core/src/push/owner.rs | 1179 ++ .../native/opennow-core/src/push/protocol.rs | 480 + .../opennow-core/src/push/registration.rs | 595 + .../native/opennow-core/src/push/store.rs | 78 + .../native/opennow-core/src/push/tests.rs | 2881 +++ .../native/opennow-core/src/push/transport.rs | 320 + .../opennow-core/src/push/vendor_default.rs | 103 + .../native/opennow-core/src/push_registry.rs | 943 + .../native/opennow-core/src/queue_servers.rs | 523 + .../native/opennow-core/src/reports.rs | 529 + .../native/opennow-core/src/requests.rs | 226 + .../opennow-core/src/server_vpc_cache.rs | 284 + .../native/opennow-core/src/settings.rs | 3467 ++++ .../native/opennow-core/src/store_cache.rs | 676 + .../opennow-core/src/store_catalog_page.rs | 246 + .../native/opennow-core/src/store_index.rs | 469 + .../native/opennow-core/src/store_requests.rs | 252 + .../native/opennow-core/src/streamer.rs | 4313 +++++ .../opennow/native/opennow-core/src/thanks.rs | 278 + .../opennow-core/src/update_apply/archive.rs | 170 + .../opennow-core/src/update_apply/bundle.rs | 267 + .../src/update_apply/integration_tests.rs | 490 + .../opennow-core/src/update_apply/managed.rs | 756 + .../opennow-core/src/update_apply/mod.rs | 1569 ++ .../opennow-core/src/update_apply/process.rs | 544 + .../opennow-core/src/update_apply/recovery.rs | 313 + .../opennow-core/src/update_apply/security.rs | 355 + .../opennow-core/src/update_apply/tests.rs | 1178 ++ .../src/update_apply/verification.rs | 134 + .../native/opennow-core/src/updater.rs | 2009 +++ .../native/opennow-core/src/version.rs | 40 + .../opennow-core/tests/core_protocol_cli.rs | 390 + .../tests/fixtures/profile_fsync_gate.c | 61 + .../tests/graphics_preferences_cli.rs | 48 + .../opennow-core/tests/legacy_contract.rs | 112 + .../opennow-core/tests/media_env_cli.rs | 132 + .../tests/settings_event_order_cli.rs | 126 + .../native/opennow-streamer/Cargo.lock | 3176 ++++ .../native/opennow-streamer/Cargo.toml | 75 + .../opennow/native/opennow-streamer/README.md | 315 + .../crates/opennow-streamer-core/Cargo.toml | 23 + .../src/decode_progress.rs | 707 + .../crates/opennow-streamer-core/src/lib.rs | 6858 ++++++++ .../opennow-streamer-core/src/microphone.rs | 262 + .../opennow-streamer-core/src/nvst_rtsp.rs | 3921 +++++ .../src/nvst_rtsp_color.rs | 34 + .../src/nvst_rtsp_color_tests.rs | 77 + .../src/nvst_rtsp_control_ping_tests.rs | 330 + .../src/nvst_rtsp_negotiation.rs | 295 + .../src/nvst_rtsp_setup_tests.rs | 441 + .../src/nvst_rtsp_tls_tests.rs | 136 + .../src/nvst_rtsp_transport_diagnostics.rs | 176 + .../src/nvst_server_control_tests.rs | 239 + .../opennow-streamer-core/src/queue_drops.rs | 228 + .../src/recording_tests.rs | 246 + .../opennow-streamer-core/src/telemetry.rs | 30 + .../opennow-streamer-core/src/webrtc/input.rs | 185 + .../opennow-streamer-core/src/webrtc/mod.rs | 818 + .../opennow-streamer-core/src/webrtc/sdp.rs | 210 + .../src/webrtc/signaling.rs | 470 + .../src/webrtc/signaling/tests.rs | 284 + .../opennow-streamer-core/src/webrtc/tests.rs | 1288 ++ .../src/webrtc/video_attributes.sdp | 74 + .../crates/opennow-streamer-ffi/Cargo.toml | 25 + .../crates/opennow-streamer-ffi/README.md | 301 + .../crates/opennow-streamer-ffi/build.rs | 15 + .../include/opennow_streamer_ffi.h | 415 + .../crates/opennow-streamer-ffi/src/lib.rs | 4230 +++++ .../crates/opennow-streamer-hid/Cargo.toml | 12 + .../opennow-streamer-hid/src/allowlist.rs | 62 + .../opennow-streamer-hid/src/descriptor.rs | 772 + .../crates/opennow-streamer-hid/src/ds4.rs | 440 + .../opennow-streamer-hid/src/generic.rs | 1136 ++ .../src/generic_runtime.rs | 761 + .../crates/opennow-streamer-hid/src/lib.rs | 443 + .../opennow-streamer-hid/src/runtime.rs | 738 + .../opennow-streamer-hid/src/session.rs | 673 + .../Cargo.toml | 34 + .../README.md | 226 + .../examples/probe.rs | 7 + .../fixtures/probe/README.md | 111 + .../fixtures/probe/av1-p010-pq.obu | Bin 0 -> 77 bytes .../fixtures/probe/av1-p010-sdr.obu | Bin 0 -> 77 bytes .../fixtures/probe/hevc-ayuv-sdr.hevc | Bin 0 -> 666 bytes .../probe/hevc-p010-pq-precision-luma.bin | Bin 0 -> 1754 bytes .../probe/hevc-p010-pq-precision.hevc | Bin 0 -> 2173 bytes .../fixtures/probe/hevc-p010-pq.hevc | Bin 0 -> 665 bytes .../fixtures/probe/hevc-p010-sdr.hevc | Bin 0 -> 665 bytes .../probe/hevc-y410-pq-precision.hevc | Bin 0 -> 94032 bytes .../fixtures/probe/hevc-y410-pq.hevc | Bin 0 -> 667 bytes .../fixtures/probe/hevc-y410-precision.hevc | Bin 0 -> 94032 bytes .../fixtures/probe/hevc-y410-sdr.hevc | Bin 0 -> 667 bytes .../src/adapter_decode.rs | 203 + .../src/aperture.rs | 110 + .../src/decoder_color.rs | 226 + .../src/decoder_order.rs | 62 + .../src/format.rs | 658 + .../src/lib.rs | 916 + .../src/queue.rs | 493 + .../src/windows/adapter_index.rs | 233 + .../src/windows/audio.rs | 371 + .../src/windows/color.rs | 73 + .../src/windows/decoder.rs | 2085 +++ .../src/windows/embedded.rs | 2945 ++++ .../src/windows/graphics.rs | 1565 ++ .../src/windows/mod.rs | 1104 ++ .../src/windows/y410.hlsl | 14 + .../src/windows/y410.rs | 283 + .../src/y410_color.rs | 118 + .../opennow-streamer-platform/Cargo.toml | 61 + .../crates/opennow-streamer-platform/build.rs | 29 + .../src/audio_playout.rs | 160 + .../src/embedded_input.rs | 442 + .../opennow-streamer-platform/src/graphics.rs | 1149 ++ .../opennow-streamer-platform/src/h264.rs | 138 + .../opennow-streamer-platform/src/lib.rs | 1108 ++ .../src/linux_backend.rs | 1136 ++ .../src/linux_frame_pacing.rs | 333 + .../src/linux_xinput.rs | 380 + .../src/macos_backend.rs | 881 + .../opennow-streamer-platform/src/media.rs | 5784 ++++++ .../src/media_recording_tests.rs | 553 + .../src/microphone.rs | 1114 ++ .../src/native_surface.rs | 569 + .../opennow-streamer-platform/src/output.rs | 3455 ++++ .../opennow-streamer-platform/src/queue.rs | 107 + .../src/recording.rs | 1578 ++ .../opennow-streamer-platform/src/replay.rs | 448 + .../opennow-streamer-platform/src/runtime.rs | 2103 +++ .../src/video_queue.rs | 360 + .../src/windows_graphics.rs | 74 + .../src/windows_raw_input.rs | 472 + .../opennow-streamer-protocol/Cargo.toml | 10 + .../opennow-streamer-protocol/src/lib.rs | 606 + .../opennow-streamer-protocol/src/log.rs | 729 + .../src/text_input.rs | 134 + .../opennow-streamer-transport/Cargo.toml | 47 + .../src/bin/nvst_peer_probe.rs | 360 + .../src/frame_stage_timing.rs | 468 + .../opennow-streamer-transport/src/lib.rs | 138 + .../opennow-streamer-transport/src/nvst.rs | 14695 ++++++++++++++++ .../src/nvst_audio_jitter.rs | 473 + .../src/nvst_audio_tests.rs | 575 + .../src/nvst_bandwidth.rs | 417 + .../src/nvst_budget.rs | 146 + .../src/nvst_budget_tests.rs | 632 + .../src/nvst_control.rs | 414 + .../src/nvst_cursor.rs | 297 + .../src/nvst_haptics.rs | 231 + .../src/nvst_input.rs | 2442 +++ .../src/nvst_input_delivery.rs | 607 + .../src/nvst_invalidation.rs | 275 + .../src/nvst_microphone.rs | 362 + .../src/nvst_nack_tests.rs | 450 + .../src/nvst_network.rs | 112 + .../src/nvst_ping_tests.rs | 530 + .../src/nvst_progress_tests.rs | 939 + .../src/nvst_qos_tests.rs | 449 + .../src/nvst_recovery_policy.rs | 48 + .../src/nvst_recovery_tests.rs | 955 + .../src/nvst_server_control.rs | 385 + .../src/nvst_server_control_tests.rs | 132 + .../src/webrtc/diagnostics.rs | 185 + .../src/webrtc/input.rs | 258 + .../src/webrtc/mod.rs | 1155 ++ .../src/webrtc/stats.rs | 77 + .../src/webrtc/tests.rs | 1385 ++ .../audiopus-sys/.github/workflows/ci.yml | 72 + .../vendor/audiopus-sys/.gitignore | 3 + .../vendor/audiopus-sys/.gitmodules | 3 + .../vendor/audiopus-sys/CHANGELOG.md | 60 + .../vendor/audiopus-sys/CONTRIBUTING.md | 62 + .../vendor/audiopus-sys/Cargo.toml | 44 + .../vendor/audiopus-sys/Cargo.toml.orig | 30 + .../vendor/audiopus-sys/LICENSE.md | 15 + .../vendor/audiopus-sys/README.md | 81 + .../vendor/audiopus-sys/build.rs | 167 + .../vendor/audiopus-sys/opus/.appveyor.yml | 37 + .../vendor/audiopus-sys/opus/.gitlab-ci.yml | 61 + .../vendor/audiopus-sys/opus/.travis.yml | 21 + .../vendor/audiopus-sys/opus/AUTHORS | 6 + .../vendor/audiopus-sys/opus/CMakeLists.txt | 618 + .../vendor/audiopus-sys/opus/COPYING | 44 + .../vendor/audiopus-sys/opus/ChangeLog | 0 .../audiopus-sys/opus/LICENSE_PLEASE_READ.txt | 22 + .../vendor/audiopus-sys/opus/Makefile.am | 371 + .../vendor/audiopus-sys/opus/Makefile.mips | 161 + .../vendor/audiopus-sys/opus/Makefile.unix | 159 + .../vendor/audiopus-sys/opus/NEWS | 0 .../vendor/audiopus-sys/opus/README | 161 + .../vendor/audiopus-sys/opus/README.draft | 54 + .../vendor/audiopus-sys/opus/autogen.sh | 14 + .../audiopus-sys/opus/celt/_kiss_fft_guts.h | 182 + .../vendor/audiopus-sys/opus/celt/arch.h | 291 + .../audiopus-sys/opus/celt/arm/arm2gnu.pl | 353 + .../audiopus-sys/opus/celt/arm/arm_celt_map.c | 160 + .../audiopus-sys/opus/celt/arm/armcpu.c | 187 + .../audiopus-sys/opus/celt/arm/armcpu.h | 77 + .../audiopus-sys/opus/celt/arm/armopts.s.in | 37 + .../opus/celt/arm/celt_fft_ne10.c | 173 + .../opus/celt/arm/celt_mdct_ne10.c | 258 + .../opus/celt/arm/celt_neon_intr.c | 211 + .../opus/celt/arm/celt_pitch_xcorr_arm.s | 551 + .../audiopus-sys/opus/celt/arm/fft_arm.h | 71 + .../audiopus-sys/opus/celt/arm/fixed_arm64.h | 35 + .../audiopus-sys/opus/celt/arm/fixed_armv4.h | 80 + .../audiopus-sys/opus/celt/arm/fixed_armv5e.h | 151 + .../opus/celt/arm/kiss_fft_armv4.h | 121 + .../opus/celt/arm/kiss_fft_armv5e.h | 118 + .../audiopus-sys/opus/celt/arm/mdct_arm.h | 59 + .../audiopus-sys/opus/celt/arm/pitch_arm.h | 160 + .../opus/celt/arm/pitch_neon_intr.c | 290 + .../vendor/audiopus-sys/opus/celt/bands.c | 1672 ++ .../vendor/audiopus-sys/opus/celt/bands.h | 123 + .../vendor/audiopus-sys/opus/celt/celt.c | 316 + .../vendor/audiopus-sys/opus/celt/celt.h | 251 + .../audiopus-sys/opus/celt/celt_decoder.c | 1378 ++ .../audiopus-sys/opus/celt/celt_encoder.c | 2607 +++ .../vendor/audiopus-sys/opus/celt/celt_lpc.c | 340 + .../vendor/audiopus-sys/opus/celt/celt_lpc.h | 66 + .../audiopus-sys/opus/celt/cpu_support.h | 70 + .../vendor/audiopus-sys/opus/celt/cwrs.c | 715 + .../vendor/audiopus-sys/opus/celt/cwrs.h | 48 + .../opus/celt/dump_modes/dump_modes.c | 353 + .../opus/celt/dump_modes/dump_modes_arch.h | 45 + .../celt/dump_modes/dump_modes_arm_ne10.c | 152 + .../vendor/audiopus-sys/opus/celt/ecintrin.h | 91 + .../vendor/audiopus-sys/opus/celt/entcode.c | 153 + .../vendor/audiopus-sys/opus/celt/entcode.h | 152 + .../vendor/audiopus-sys/opus/celt/entdec.c | 245 + .../vendor/audiopus-sys/opus/celt/entdec.h | 100 + .../vendor/audiopus-sys/opus/celt/entenc.c | 294 + .../vendor/audiopus-sys/opus/celt/entenc.h | 110 + .../vendor/audiopus-sys/opus/celt/fixed_c5x.h | 79 + .../vendor/audiopus-sys/opus/celt/fixed_c6x.h | 70 + .../audiopus-sys/opus/celt/fixed_debug.h | 836 + .../audiopus-sys/opus/celt/fixed_generic.h | 188 + .../audiopus-sys/opus/celt/float_cast.h | 152 + .../vendor/audiopus-sys/opus/celt/kiss_fft.c | 604 + .../vendor/audiopus-sys/opus/celt/kiss_fft.h | 200 + .../vendor/audiopus-sys/opus/celt/laplace.c | 134 + .../vendor/audiopus-sys/opus/celt/laplace.h | 48 + .../vendor/audiopus-sys/opus/celt/mathops.c | 209 + .../vendor/audiopus-sys/opus/celt/mathops.h | 290 + .../vendor/audiopus-sys/opus/celt/mdct.c | 343 + .../vendor/audiopus-sys/opus/celt/mdct.h | 112 + .../vendor/audiopus-sys/opus/celt/meson.build | 63 + .../vendor/audiopus-sys/opus/celt/mfrngcod.h | 48 + .../audiopus-sys/opus/celt/mips/celt_mipsr1.h | 152 + .../opus/celt/mips/fixed_generic_mipsr1.h | 126 + .../opus/celt/mips/kiss_fft_mipsr1.h | 167 + .../audiopus-sys/opus/celt/mips/mdct_mipsr1.h | 288 + .../opus/celt/mips/pitch_mipsr1.h | 161 + .../audiopus-sys/opus/celt/mips/vq_mipsr1.h | 116 + .../vendor/audiopus-sys/opus/celt/modes.c | 442 + .../vendor/audiopus-sys/opus/celt/modes.h | 75 + .../audiopus-sys/opus/celt/opus_custom_demo.c | 210 + .../audiopus-sys/opus/celt/os_support.h | 91 + .../vendor/audiopus-sys/opus/celt/pitch.c | 537 + .../vendor/audiopus-sys/opus/celt/pitch.h | 192 + .../audiopus-sys/opus/celt/quant_bands.c | 563 + .../audiopus-sys/opus/celt/quant_bands.h | 66 + .../vendor/audiopus-sys/opus/celt/rate.c | 644 + .../vendor/audiopus-sys/opus/celt/rate.h | 101 + .../audiopus-sys/opus/celt/stack_alloc.h | 184 + .../opus/celt/static_modes_fixed.h | 892 + .../opus/celt/static_modes_fixed_arm_ne10.h | 388 + .../opus/celt/static_modes_float.h | 888 + .../opus/celt/static_modes_float_arm_ne10.h | 404 + .../audiopus-sys/opus/celt/tests/meson.build | 19 + .../opus/celt/tests/test_unit_cwrs32.c | 161 + .../opus/celt/tests/test_unit_dft.c | 179 + .../opus/celt/tests/test_unit_entropy.c | 383 + .../opus/celt/tests/test_unit_laplace.c | 93 + .../opus/celt/tests/test_unit_mathops.c | 266 + .../opus/celt/tests/test_unit_mdct.c | 227 + .../opus/celt/tests/test_unit_rotation.c | 86 + .../opus/celt/tests/test_unit_types.c | 50 + .../vendor/audiopus-sys/opus/celt/vq.c | 442 + .../vendor/audiopus-sys/opus/celt/vq.h | 79 + .../audiopus-sys/opus/celt/x86/celt_lpc_sse.h | 66 + .../opus/celt/x86/celt_lpc_sse4_1.c | 89 + .../audiopus-sys/opus/celt/x86/pitch_sse.c | 185 + .../audiopus-sys/opus/celt/x86/pitch_sse.h | 192 + .../audiopus-sys/opus/celt/x86/pitch_sse2.c | 95 + .../audiopus-sys/opus/celt/x86/pitch_sse4_1.c | 195 + .../audiopus-sys/opus/celt/x86/vq_sse.h | 50 + .../audiopus-sys/opus/celt/x86/vq_sse2.c | 217 + .../audiopus-sys/opus/celt/x86/x86_celt_map.c | 167 + .../audiopus-sys/opus/celt/x86/x86cpu.c | 157 + .../audiopus-sys/opus/celt/x86/x86cpu.h | 95 + .../vendor/audiopus-sys/opus/celt_headers.mk | 53 + .../vendor/audiopus-sys/opus/celt_sources.mk | 50 + .../opus/cmake/CFeatureCheck.cmake | 39 + .../opus/cmake/OpusBuildtype.cmake | 27 + .../audiopus-sys/opus/cmake/OpusConfig.cmake | 104 + .../opus/cmake/OpusConfig.cmake.in | 20 + .../opus/cmake/OpusFunctions.cmake | 215 + .../opus/cmake/OpusPackageVersion.cmake | 70 + .../audiopus-sys/opus/cmake/OpusSources.cmake | 46 + .../audiopus-sys/opus/cmake/config.h.cmake.in | 1 + .../vendor/audiopus-sys/opus/cmake/vla.c | 7 + .../vendor/audiopus-sys/opus/configure.ac | 946 + .../vendor/audiopus-sys/opus/doc/Doxyfile.in | 340 + .../vendor/audiopus-sys/opus/doc/Makefile.am | 45 + .../audiopus-sys/opus/doc/build_draft.sh | 113 + .../audiopus-sys/opus/doc/build_isobmff.sh | 50 + .../audiopus-sys/opus/doc/build_oggdraft.sh | 52 + .../audiopus-sys/opus/doc/customdoxygen.css | 1011 ++ .../opus/doc/draft-ietf-codec-oggopus.xml | 1873 ++ .../opus/doc/draft-ietf-codec-opus-update.xml | 513 + .../opus/doc/draft-ietf-codec-opus.xml | 8276 +++++++++ .../opus/doc/draft-ietf-payload-rtp-opus.xml | 960 + .../vendor/audiopus-sys/opus/doc/footer.html | 32 + .../vendor/audiopus-sys/opus/doc/header.html | 61 + .../vendor/audiopus-sys/opus/doc/meson.build | 33 + .../audiopus-sys/opus/doc/opus_in_isobmff.css | 60 + .../opus/doc/opus_in_isobmff.html | 692 + .../audiopus-sys/opus/doc/opus_logo.svg | 157 + .../audiopus-sys/opus/doc/opus_update.patch | 244 + .../vendor/audiopus-sys/opus/doc/release.txt | 43 + .../audiopus-sys/opus/doc/trivial_example.c | 171 + .../audiopus-sys/opus/include/meson.build | 13 + .../vendor/audiopus-sys/opus/include/opus.h | 981 ++ .../audiopus-sys/opus/include/opus_custom.h | 342 + .../audiopus-sys/opus/include/opus_defines.h | 799 + .../opus/include/opus_multistream.h | 660 + .../opus/include/opus_projection.h | 568 + .../audiopus-sys/opus/include/opus_types.h | 166 + .../opus/m4/as-gcc-inline-assembly.m4 | 98 + .../opus/m4/ax_add_fortify_source.m4 | 53 + .../audiopus-sys/opus/m4/opus-intrinsics.m4 | 29 + .../vendor/audiopus-sys/opus/meson.build | 675 + .../audiopus-sys/opus/meson/get-version.py | 86 + .../opus/meson/read-sources-list.py | 28 + .../audiopus-sys/opus/meson_options.txt | 22 + .../audiopus-sys/opus/opus-uninstalled.pc.in | 12 + .../vendor/audiopus-sys/opus/opus.m4 | 117 + .../vendor/audiopus-sys/opus/opus.pc.in | 16 + .../vendor/audiopus-sys/opus/opus_headers.mk | 9 + .../vendor/audiopus-sys/opus/opus_sources.mk | 16 + .../vendor/audiopus-sys/opus/releases.sha2 | 79 + .../audiopus-sys/opus/scripts/dump_rnn.py | 57 + .../audiopus-sys/opus/scripts/rnn_train.py | 67 + .../vendor/audiopus-sys/opus/silk/A2NLSF.c | 267 + .../vendor/audiopus-sys/opus/silk/API.h | 135 + .../vendor/audiopus-sys/opus/silk/CNG.c | 188 + .../opus/silk/HP_variable_cutoff.c | 77 + .../vendor/audiopus-sys/opus/silk/Inlines.h | 188 + .../opus/silk/LPC_analysis_filter.c | 111 + .../vendor/audiopus-sys/opus/silk/LPC_fit.c | 82 + .../opus/silk/LPC_inv_pred_gain.c | 141 + .../opus/silk/LP_variable_cutoff.c | 135 + .../audiopus-sys/opus/silk/MacroCount.h | 710 + .../audiopus-sys/opus/silk/MacroDebug.h | 951 + .../vendor/audiopus-sys/opus/silk/NLSF2A.c | 141 + .../vendor/audiopus-sys/opus/silk/NLSF_VQ.c | 76 + .../opus/silk/NLSF_VQ_weights_laroia.c | 80 + .../audiopus-sys/opus/silk/NLSF_decode.c | 93 + .../opus/silk/NLSF_del_dec_quant.c | 215 + .../audiopus-sys/opus/silk/NLSF_encode.c | 124 + .../audiopus-sys/opus/silk/NLSF_stabilize.c | 142 + .../audiopus-sys/opus/silk/NLSF_unpack.c | 55 + .../vendor/audiopus-sys/opus/silk/NSQ.c | 437 + .../vendor/audiopus-sys/opus/silk/NSQ.h | 101 + .../audiopus-sys/opus/silk/NSQ_del_dec.c | 733 + .../vendor/audiopus-sys/opus/silk/PLC.c | 446 + .../vendor/audiopus-sys/opus/silk/PLC.h | 62 + .../audiopus-sys/opus/silk/SigProc_FIX.h | 641 + .../vendor/audiopus-sys/opus/silk/VAD.c | 360 + .../audiopus-sys/opus/silk/VQ_WMat_EC.c | 131 + .../audiopus-sys/opus/silk/ana_filt_bank_1.c | 74 + .../opus/silk/arm/LPC_inv_pred_gain_arm.h | 57 + .../silk/arm/LPC_inv_pred_gain_neon_intr.c | 288 + .../opus/silk/arm/NSQ_del_dec_arm.h | 100 + .../opus/silk/arm/NSQ_del_dec_neon_intr.c | 1124 ++ .../audiopus-sys/opus/silk/arm/NSQ_neon.c | 112 + .../audiopus-sys/opus/silk/arm/NSQ_neon.h | 114 + .../opus/silk/arm/SigProc_FIX_armv4.h | 47 + .../opus/silk/arm/SigProc_FIX_armv5e.h | 61 + .../audiopus-sys/opus/silk/arm/arm_silk_map.c | 123 + .../opus/silk/arm/biquad_alt_arm.h | 68 + .../opus/silk/arm/biquad_alt_neon_intr.c | 156 + .../audiopus-sys/opus/silk/arm/macros_arm64.h | 39 + .../audiopus-sys/opus/silk/arm/macros_armv4.h | 110 + .../opus/silk/arm/macros_armv5e.h | 220 + .../audiopus-sys/opus/silk/biquad_alt.c | 121 + .../audiopus-sys/opus/silk/bwexpander.c | 51 + .../audiopus-sys/opus/silk/bwexpander_32.c | 51 + .../opus/silk/check_control_input.c | 106 + .../audiopus-sys/opus/silk/code_signs.c | 115 + .../vendor/audiopus-sys/opus/silk/control.h | 150 + .../audiopus-sys/opus/silk/control_SNR.c | 113 + .../opus/silk/control_audio_bandwidth.c | 132 + .../audiopus-sys/opus/silk/control_codec.c | 423 + .../vendor/audiopus-sys/opus/silk/debug.c | 172 + .../vendor/audiopus-sys/opus/silk/debug.h | 267 + .../vendor/audiopus-sys/opus/silk/dec_API.c | 419 + .../audiopus-sys/opus/silk/decode_core.c | 237 + .../audiopus-sys/opus/silk/decode_frame.c | 129 + .../audiopus-sys/opus/silk/decode_indices.c | 151 + .../opus/silk/decode_parameters.c | 115 + .../audiopus-sys/opus/silk/decode_pitch.c | 77 + .../audiopus-sys/opus/silk/decode_pulses.c | 115 + .../audiopus-sys/opus/silk/decoder_set_fs.c | 108 + .../vendor/audiopus-sys/opus/silk/define.h | 235 + .../vendor/audiopus-sys/opus/silk/enc_API.c | 576 + .../audiopus-sys/opus/silk/encode_indices.c | 181 + .../audiopus-sys/opus/silk/encode_pulses.c | 206 + .../vendor/audiopus-sys/opus/silk/errors.h | 98 + .../opus/silk/fixed/LTP_analysis_filter_FIX.c | 90 + .../opus/silk/fixed/LTP_scale_ctrl_FIX.c | 53 + .../opus/silk/fixed/apply_sine_window_FIX.c | 101 + .../arm/warped_autocorrelation_FIX_arm.h | 68 + .../warped_autocorrelation_FIX_neon_intr.c | 265 + .../opus/silk/fixed/autocorr_FIX.c | 48 + .../opus/silk/fixed/burg_modified_FIX.c | 280 + .../opus/silk/fixed/corrMatrix_FIX.c | 150 + .../opus/silk/fixed/encode_frame_FIX.c | 448 + .../opus/silk/fixed/find_LPC_FIX.c | 151 + .../opus/silk/fixed/find_LTP_FIX.c | 99 + .../opus/silk/fixed/find_pitch_lags_FIX.c | 143 + .../opus/silk/fixed/find_pred_coefs_FIX.c | 145 + .../audiopus-sys/opus/silk/fixed/k2a_FIX.c | 54 + .../opus/silk/fixed/k2a_Q16_FIX.c | 54 + .../audiopus-sys/opus/silk/fixed/main_FIX.h | 244 + .../mips/noise_shape_analysis_FIX_mipsr1.h | 336 + .../silk/fixed/mips/prefilter_FIX_mipsr1.h | 184 + .../mips/warped_autocorrelation_FIX_mipsr1.h | 165 + .../silk/fixed/noise_shape_analysis_FIX.c | 407 + .../opus/silk/fixed/pitch_analysis_core_FIX.c | 721 + .../opus/silk/fixed/process_gains_FIX.c | 117 + .../silk/fixed/regularize_correlations_FIX.c | 47 + .../opus/silk/fixed/residual_energy16_FIX.c | 103 + .../opus/silk/fixed/residual_energy_FIX.c | 98 + .../opus/silk/fixed/schur64_FIX.c | 93 + .../audiopus-sys/opus/silk/fixed/schur_FIX.c | 107 + .../opus/silk/fixed/structs_FIX.h | 116 + .../opus/silk/fixed/vector_ops_FIX.c | 102 + .../silk/fixed/warped_autocorrelation_FIX.c | 92 + .../silk/fixed/x86/burg_modified_FIX_sse4_1.c | 377 + .../opus/silk/fixed/x86/prefilter_FIX_sse.c | 160 + .../silk/fixed/x86/vector_ops_FIX_sse4_1.c | 88 + .../opus/silk/float/LPC_analysis_filter_FLP.c | 249 + .../opus/silk/float/LPC_inv_pred_gain_FLP.c | 73 + .../opus/silk/float/LTP_analysis_filter_FLP.c | 75 + .../opus/silk/float/LTP_scale_ctrl_FLP.c | 52 + .../opus/silk/float/SigProc_FLP.h | 197 + .../opus/silk/float/apply_sine_window_FLP.c | 81 + .../opus/silk/float/autocorrelation_FLP.c | 52 + .../opus/silk/float/burg_modified_FLP.c | 186 + .../opus/silk/float/bwexpander_FLP.c | 49 + .../opus/silk/float/corrMatrix_FLP.c | 93 + .../opus/silk/float/encode_frame_FLP.c | 435 + .../audiopus-sys/opus/silk/float/energy_FLP.c | 59 + .../opus/silk/float/find_LPC_FLP.c | 104 + .../opus/silk/float/find_LTP_FLP.c | 64 + .../opus/silk/float/find_pitch_lags_FLP.c | 132 + .../opus/silk/float/find_pred_coefs_FLP.c | 116 + .../opus/silk/float/inner_product_FLP.c | 59 + .../audiopus-sys/opus/silk/float/k2a_FLP.c | 54 + .../audiopus-sys/opus/silk/float/main_FLP.h | 286 + .../silk/float/noise_shape_analysis_FLP.c | 350 + .../opus/silk/float/pitch_analysis_core_FLP.c | 630 + .../opus/silk/float/process_gains_FLP.c | 103 + .../silk/float/regularize_correlations_FLP.c | 48 + .../opus/silk/float/residual_energy_FLP.c | 117 + .../opus/silk/float/scale_copy_vector_FLP.c | 57 + .../opus/silk/float/scale_vector_FLP.c | 56 + .../audiopus-sys/opus/silk/float/schur_FLP.c | 70 + .../audiopus-sys/opus/silk/float/sort_FLP.c | 83 + .../opus/silk/float/structs_FLP.h | 112 + .../silk/float/warped_autocorrelation_FLP.c | 73 + .../opus/silk/float/wrappers_FLP.c | 207 + .../audiopus-sys/opus/silk/gain_quant.c | 142 + .../audiopus-sys/opus/silk/init_decoder.c | 57 + .../audiopus-sys/opus/silk/init_encoder.c | 64 + .../opus/silk/inner_prod_aligned.c | 47 + .../audiopus-sys/opus/silk/interpolate.c | 51 + .../vendor/audiopus-sys/opus/silk/lin2log.c | 46 + .../vendor/audiopus-sys/opus/silk/log2lin.c | 58 + .../vendor/audiopus-sys/opus/silk/macros.h | 151 + .../vendor/audiopus-sys/opus/silk/main.h | 476 + .../vendor/audiopus-sys/opus/silk/meson.build | 53 + .../opus/silk/mips/NSQ_del_dec_mipsr1.h | 410 + .../opus/silk/mips/macros_mipsr1.h | 92 + .../opus/silk/mips/sigproc_fix_mipsr1.h | 60 + .../opus/silk/pitch_est_defines.h | 88 + .../audiopus-sys/opus/silk/pitch_est_tables.c | 99 + .../audiopus-sys/opus/silk/process_NLSFs.c | 107 + .../audiopus-sys/opus/silk/quant_LTP_gains.c | 132 + .../vendor/audiopus-sys/opus/silk/resampler.c | 215 + .../audiopus-sys/opus/silk/resampler_down2.c | 74 + .../opus/silk/resampler_down2_3.c | 103 + .../opus/silk/resampler_private.h | 88 + .../opus/silk/resampler_private_AR2.c | 55 + .../opus/silk/resampler_private_IIR_FIR.c | 107 + .../opus/silk/resampler_private_down_FIR.c | 194 + .../opus/silk/resampler_private_up2_HQ.c | 113 + .../audiopus-sys/opus/silk/resampler_rom.c | 96 + .../audiopus-sys/opus/silk/resampler_rom.h | 68 + .../opus/silk/resampler_structs.h | 60 + .../audiopus-sys/opus/silk/shell_coder.c | 151 + .../vendor/audiopus-sys/opus/silk/sigm_Q15.c | 76 + .../vendor/audiopus-sys/opus/silk/sort.c | 154 + .../audiopus-sys/opus/silk/stereo_LR_to_MS.c | 229 + .../audiopus-sys/opus/silk/stereo_MS_to_LR.c | 85 + .../opus/silk/stereo_decode_pred.c | 73 + .../opus/silk/stereo_encode_pred.c | 62 + .../opus/silk/stereo_find_predictor.c | 79 + .../opus/silk/stereo_quant_pred.c | 73 + .../vendor/audiopus-sys/opus/silk/structs.h | 329 + .../audiopus-sys/opus/silk/sum_sqr_shift.c | 83 + .../audiopus-sys/opus/silk/table_LSF_cos.c | 70 + .../vendor/audiopus-sys/opus/silk/tables.h | 114 + .../audiopus-sys/opus/silk/tables_LTP.c | 294 + .../opus/silk/tables_NLSF_CB_NB_MB.c | 195 + .../opus/silk/tables_NLSF_CB_WB.c | 234 + .../audiopus-sys/opus/silk/tables_gain.c | 63 + .../audiopus-sys/opus/silk/tables_other.c | 124 + .../audiopus-sys/opus/silk/tables_pitch_lag.c | 69 + .../opus/silk/tables_pulses_per_block.c | 264 + .../audiopus-sys/opus/silk/tests/meson.build | 8 + .../silk/tests/test_unit_LPC_inv_pred_gain.c | 129 + .../opus/silk/tuning_parameters.h | 155 + .../vendor/audiopus-sys/opus/silk/typedef.h | 81 + .../opus/silk/x86/NSQ_del_dec_sse4_1.c | 859 + .../audiopus-sys/opus/silk/x86/NSQ_sse4_1.c | 719 + .../opus/silk/x86/SigProc_FIX_sse.h | 94 + .../audiopus-sys/opus/silk/x86/VAD_sse4_1.c | 277 + .../opus/silk/x86/VQ_WMat_EC_sse4_1.c | 142 + .../audiopus-sys/opus/silk/x86/main_sse.h | 248 + .../audiopus-sys/opus/silk/x86/x86_silk_map.c | 164 + .../vendor/audiopus-sys/opus/silk_headers.mk | 44 + .../vendor/audiopus-sys/opus/silk_sources.mk | 154 + .../vendor/audiopus-sys/opus/src/analysis.c | 983 ++ .../vendor/audiopus-sys/opus/src/analysis.h | 103 + .../audiopus-sys/opus/src/mapping_matrix.c | 378 + .../audiopus-sys/opus/src/mapping_matrix.h | 133 + .../vendor/audiopus-sys/opus/src/meson.build | 45 + .../vendor/audiopus-sys/opus/src/mlp.c | 144 + .../vendor/audiopus-sys/opus/src/mlp.h | 60 + .../vendor/audiopus-sys/opus/src/mlp_data.c | 672 + .../vendor/audiopus-sys/opus/src/opus.c | 356 + .../audiopus-sys/opus/src/opus_compare.c | 382 + .../audiopus-sys/opus/src/opus_decoder.c | 1032 ++ .../vendor/audiopus-sys/opus/src/opus_demo.c | 892 + .../audiopus-sys/opus/src/opus_encoder.c | 2773 +++ .../audiopus-sys/opus/src/opus_multistream.c | 92 + .../opus/src/opus_multistream_decoder.c | 552 + .../opus/src/opus_multistream_encoder.c | 1328 ++ .../audiopus-sys/opus/src/opus_private.h | 201 + .../opus/src/opus_projection_decoder.c | 258 + .../opus/src/opus_projection_encoder.c | 468 + .../audiopus-sys/opus/src/repacketizer.c | 349 + .../audiopus-sys/opus/src/repacketizer_demo.c | 247 + .../audiopus-sys/opus/src/tansig_table.h | 45 + .../audiopus-sys/opus/tests/meson.build | 34 + .../opus/tests/opus_decode_fuzzer.c | 124 + .../opus/tests/opus_decode_fuzzer.options | 2 + .../opus/tests/opus_encode_regressions.c | 1035 ++ .../audiopus-sys/opus/tests/run_vectors.sh | 143 + .../audiopus-sys/opus/tests/test_opus_api.c | 1904 ++ .../opus/tests/test_opus_common.h | 85 + .../opus/tests/test_opus_decode.c | 463 + .../opus/tests/test_opus_encode.c | 703 + .../opus/tests/test_opus_padding.c | 93 + .../opus/tests/test_opus_projection.c | 394 + .../audiopus-sys/opus/training/rnn_dump.py | 66 + .../audiopus-sys/opus/training/rnn_train.py | 177 + .../audiopus-sys/opus/training/txt2hdf5.py | 12 + .../opus/win32/VS2015/common.props | 82 + .../audiopus-sys/opus/win32/VS2015/opus.sln | 168 + .../opus/win32/VS2015/opus.vcxproj | 399 + .../opus/win32/VS2015/opus.vcxproj.filters | 585 + .../opus/win32/VS2015/opus_demo.vcxproj | 171 + .../win32/VS2015/opus_demo.vcxproj.filters | 22 + .../opus/win32/VS2015/test_opus_api.vcxproj | 171 + .../VS2015/test_opus_api.vcxproj.filters | 14 + .../win32/VS2015/test_opus_decode.vcxproj | 171 + .../VS2015/test_opus_decode.vcxproj.filters | 14 + .../win32/VS2015/test_opus_encode.vcxproj | 172 + .../VS2015/test_opus_encode.vcxproj.filters | 17 + .../audiopus-sys/opus/win32/genversion.bat | 37 + .../vendor/audiopus-sys/src/lib.rs | 1251 ++ .../vendor/audiopus-sys/src/wrapper.h | 1 + .../vendor/str0m/.cargo_vcs_info.json | 6 + .../vendor/str0m/CHANGELOG.md | 433 + .../opennow-streamer/vendor/str0m/Cargo.toml | 187 + .../vendor/str0m/Cargo.toml.orig | 184 + .../vendor/str0m/LICENSE-MIT.txt | 7 + .../vendor/str0m/OPENNOW-PATCH.md | 52 + .../opennow-streamer/vendor/str0m/README.md | 656 + .../str0m/src/_internal_test_exports/fuzz.rs | 246 + .../str0m/src/_internal_test_exports/mod.rs | 48 + .../str0m/src/_internal_test_exports/rng.rs | 55 + .../str0m/src/_internal_test_exports/setup.rs | 67 + .../str0m/src/bwe/acked_bitrate_estimator.rs | 253 + .../str0m/src/bwe/alr_detector/budget.rs | 281 + .../str0m/src/bwe/alr_detector/detector.rs | 344 + .../vendor/str0m/src/bwe/alr_detector/mod.rs | 6 + .../vendor/str0m/src/bwe/api.rs | 57 + .../str0m/src/bwe/delay/arrival_group.rs | 437 + .../vendor/str0m/src/bwe/delay/control.rs | 226 + .../vendor/str0m/src/bwe/delay/mod.rs | 6 + .../str0m/src/bwe/delay/rate_control.rs | 524 + .../vendor/str0m/src/bwe/delay/trendline.rs | 454 + .../str0m/src/bwe/link_capacity_estimator.rs | 181 + .../vendor/str0m/src/bwe/loss_controller.rs | 1551 ++ .../vendor/str0m/src/bwe/macros.rs | 98 + .../vendor/str0m/src/bwe/mod.rs | 486 + .../vendor/str0m/src/bwe/probe/cluster.rs | 431 + .../vendor/str0m/src/bwe/probe/control.rs | 1000 ++ .../vendor/str0m/src/bwe/probe/estimator.rs | 696 + .../vendor/str0m/src/bwe/probe/mod.rs | 7 + .../vendor/str0m/src/bwe/smoother.rs | 96 + .../vendor/str0m/src/bwe/time.rs | 760 + .../vendor/str0m/src/change/direct.rs | 450 + .../vendor/str0m/src/change/mod.rs | 20 + .../vendor/str0m/src/change/sdp.rs | 2699 +++ .../vendor/str0m/src/channel.rs | 664 + .../vendor/str0m/src/config.rs | 847 + .../vendor/str0m/src/crypto/finger.rs | 89 + .../vendor/str0m/src/crypto/mod.rs | 66 + .../vendor/str0m/src/crypto/srtp.rs | 529 + .../opennow-streamer/vendor/str0m/src/dtls.rs | 187 + .../vendor/str0m/src/error.rs | 182 + .../vendor/str0m/src/format/codec.rs | 211 + .../vendor/str0m/src/format/codec_config.rs | 1633 ++ .../vendor/str0m/src/format/format_params.rs | 316 + .../vendor/str0m/src/format/mod.rs | 29 + .../vendor/str0m/src/format/payload_params.rs | 1882 ++ .../vendor/str0m/src/io/error.rs | 45 + .../vendor/str0m/src/io/mod.rs | 182 + .../opennow-streamer/vendor/str0m/src/lib.rs | 2260 +++ .../vendor/str0m/src/media/event.rs | 370 + .../vendor/str0m/src/media/mod.rs | 686 + .../vendor/str0m/src/media/writer.rs | 249 + .../vendor/str0m/src/pacer/control.rs | 91 + .../vendor/str0m/src/pacer/leaky.rs | 1370 ++ .../vendor/str0m/src/pacer/mod.rs | 164 + .../vendor/str0m/src/pacer/null.rs | 84 + .../vendor/str0m/src/pacer/queue.rs | 108 + .../str0m/src/packet/LICENSE-APACHE.txt | 201 + .../vendor/str0m/src/packet/av1.rs | 1278 ++ .../vendor/str0m/src/packet/buffer_rx.rs | 1197 ++ .../vendor/str0m/src/packet/comfort_noise.rs | 89 + .../vendor/str0m/src/packet/contiguity.rs | 104 + .../vendor/str0m/src/packet/contiguity_vp8.rs | 185 + .../vendor/str0m/src/packet/contiguity_vp9.rs | 159 + .../vendor/str0m/src/packet/error.rs | 52 + .../vendor/str0m/src/packet/g7xx.rs | 164 + .../vendor/str0m/src/packet/h264.rs | 927 + .../vendor/str0m/src/packet/h264_profile.rs | 374 + .../vendor/str0m/src/packet/h265.rs | 5203 ++++++ .../vendor/str0m/src/packet/h265_profile.rs | 374 + .../vendor/str0m/src/packet/h266.rs | 2580 +++ .../vendor/str0m/src/packet/h266_profile.rs | 405 + .../vendor/str0m/src/packet/mod.rs | 517 + .../vendor/str0m/src/packet/null.rs | 43 + .../vendor/str0m/src/packet/opus.rs | 149 + .../vendor/str0m/src/packet/payload.rs | 105 + .../vendor/str0m/src/packet/vp8.rs | 1264 ++ .../vendor/str0m/src/packet/vp9.rs | 1586 ++ .../vendor/str0m/src/rtp/dir.rs | 85 + .../vendor/str0m/src/rtp/error.rs | 49 + .../vendor/str0m/src/rtp/ext.rs | 1581 ++ .../vendor/str0m/src/rtp/header.rs | 926 + .../vendor/str0m/src/rtp/id.rs | 374 + .../vendor/str0m/src/rtp/mod.rs | 34 + .../vendor/str0m/src/rtp/mtime.rs | 450 + .../vendor/str0m/src/rtp/rtcp/bb.rs | 62 + .../vendor/str0m/src/rtp/rtcp/fir.rs | 94 + .../vendor/str0m/src/rtp/rtcp/fmt.rs | 132 + .../vendor/str0m/src/rtp/rtcp/header.rs | 198 + .../vendor/str0m/src/rtp/rtcp/list.rs | 252 + .../vendor/str0m/src/rtp/rtcp/mod.rs | 726 + .../vendor/str0m/src/rtp/rtcp/nack.rs | 156 + .../vendor/str0m/src/rtp/rtcp/pli.rs | 52 + .../vendor/str0m/src/rtp/rtcp/psfbapp.rs | 167 + .../vendor/str0m/src/rtp/rtcp/remb.rs | 304 + .../vendor/str0m/src/rtp/rtcp/rr.rs | 133 + .../vendor/str0m/src/rtp/rtcp/rtcpfb.rs | 175 + .../vendor/str0m/src/rtp/rtcp/sdes.rs | 325 + .../vendor/str0m/src/rtp/rtcp/sr.rs | 150 + .../vendor/str0m/src/rtp/rtcp/twcc.rs | 2324 +++ .../vendor/str0m/src/rtp/rtcp/xr.rs | 285 + .../vendor/str0m/src/rtp/srtp.rs | 1542 ++ .../vendor/str0m/src/rtp/vla.rs | 1165 ++ .../vendor/str0m/src/sctp/dcep.rs | 122 + .../vendor/str0m/src/sctp/error.rs | 55 + .../vendor/str0m/src/sctp/mod.rs | 1664 ++ .../vendor/str0m/src/sctp/snap.rs | 170 + .../vendor/str0m/src/sdp/data.rs | 3720 ++++ .../vendor/str0m/src/sdp/error.rs | 21 + .../vendor/str0m/src/sdp/mod.rs | 201 + .../vendor/str0m/src/sdp/parser.rs | 1818 ++ .../vendor/str0m/src/session.rs | 1329 ++ .../vendor/str0m/src/stats.rs | 337 + .../vendor/str0m/src/streams/mod.rs | 725 + .../vendor/str0m/src/streams/receive.rs | 911 + .../vendor/str0m/src/streams/register.rs | 426 + .../vendor/str0m/src/streams/register_nack.rs | 649 + .../vendor/str0m/src/streams/rtx_cache.rs | 192 + .../vendor/str0m/src/streams/rtx_cache_buf.rs | 386 + .../vendor/str0m/src/streams/send.rs | 1330 ++ .../vendor/str0m/src/streams/send_queue.rs | 346 + .../vendor/str0m/src/streams/send_stats.rs | 190 + .../vendor/str0m/src/util/average.rs | 80 + .../vendor/str0m/src/util/bit_pattern.rs | 107 + .../vendor/str0m/src/util/mod.rs | 87 + .../vendor/str0m/src/util/time_tricks.rs | 210 + .../vendor/str0m/src/util/value_history.rs | 109 + 763 files changed, 338298 insertions(+) create mode 100644 external/opennow/CHANGES.md create mode 100644 external/opennow/LICENSE create mode 100644 external/opennow/NOTICE.md create mode 100644 external/opennow/docs/core-protocol.md create mode 100644 external/opennow/docs/qt-native-streamer-ffi.md create mode 100644 external/opennow/native/opennow-core/Cargo.lock create mode 100644 external/opennow/native/opennow-core/Cargo.toml create mode 100644 external/opennow/native/opennow-core/contracts/legacy-open-now-api.json create mode 100644 external/opennow/native/opennow-core/contracts/legacy-open-now-api.schema.json create mode 100644 external/opennow/native/opennow-core/src/account_connections.rs create mode 100644 external/opennow/native/opennow-core/src/analytics.rs create mode 100644 external/opennow/native/opennow-core/src/artwork_cache.rs create mode 100644 external/opennow/native/opennow-core/src/bin/opennow-acceptance-verify.rs create mode 100644 external/opennow/native/opennow-core/src/bin/opennow-license-report.rs create mode 100644 external/opennow/native/opennow-core/src/bin/opennow-update-helper.rs create mode 100644 external/opennow/native/opennow-core/src/bin/opennow-update-manifest.rs create mode 100644 external/opennow/native/opennow-core/src/bug_reports.rs create mode 100644 external/opennow/native/opennow-core/src/catalog_types.rs create mode 100644 external/opennow/native/opennow-core/src/cloudmatch.rs create mode 100644 external/opennow/native/opennow-core/src/community.rs create mode 100644 external/opennow/native/opennow-core/src/console_profiles.rs create mode 100644 external/opennow/native/opennow-core/src/credential_vault.rs create mode 100644 external/opennow/native/opennow-core/src/credential_vault/encrypted_secret_store.rs create mode 100644 external/opennow/native/opennow-core/src/credential_vault/json_secret_store.rs create mode 100644 external/opennow/native/opennow-core/src/device_identity.rs create mode 100644 external/opennow/native/opennow-core/src/diagnostics.rs create mode 100644 external/opennow/native/opennow-core/src/discord.rs create mode 100644 external/opennow/native/opennow-core/src/dlss.rs create mode 100644 external/opennow/native/opennow-core/src/frame_rate.rs create mode 100644 external/opennow/native/opennow-core/src/gfn.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/catalog.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/catalog_actions.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/catalog_actions/tests.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/catalog_tests.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/dlss_support.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/routing_tests.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/store_launch.rs create mode 100644 external/opennow/native/opennow-core/src/gfn/store_launch_tests.rs create mode 100644 external/opennow/native/opennow-core/src/language.rs create mode 100644 external/opennow/native/opennow-core/src/lib.rs create mode 100644 external/opennow/native/opennow-core/src/main.rs create mode 100644 external/opennow/native/opennow-core/src/media.rs create mode 100644 external/opennow/native/opennow-core/src/network.rs create mode 100644 external/opennow/native/opennow-core/src/network_test.rs create mode 100644 external/opennow/native/opennow-core/src/persistent_storage.rs create mode 100644 external/opennow/native/opennow-core/src/proxy.rs create mode 100644 external/opennow/native/opennow-core/src/push/config.rs create mode 100644 external/opennow/native/opennow-core/src/push/decrypt.rs create mode 100644 external/opennow/native/opennow-core/src/push/event.rs create mode 100644 external/opennow/native/opennow-core/src/push/mod.rs create mode 100644 external/opennow/native/opennow-core/src/push/owner.rs create mode 100644 external/opennow/native/opennow-core/src/push/protocol.rs create mode 100644 external/opennow/native/opennow-core/src/push/registration.rs create mode 100644 external/opennow/native/opennow-core/src/push/store.rs create mode 100644 external/opennow/native/opennow-core/src/push/tests.rs create mode 100644 external/opennow/native/opennow-core/src/push/transport.rs create mode 100644 external/opennow/native/opennow-core/src/push/vendor_default.rs create mode 100644 external/opennow/native/opennow-core/src/push_registry.rs create mode 100644 external/opennow/native/opennow-core/src/queue_servers.rs create mode 100644 external/opennow/native/opennow-core/src/reports.rs create mode 100644 external/opennow/native/opennow-core/src/requests.rs create mode 100644 external/opennow/native/opennow-core/src/server_vpc_cache.rs create mode 100644 external/opennow/native/opennow-core/src/settings.rs create mode 100644 external/opennow/native/opennow-core/src/store_cache.rs create mode 100644 external/opennow/native/opennow-core/src/store_catalog_page.rs create mode 100644 external/opennow/native/opennow-core/src/store_index.rs create mode 100644 external/opennow/native/opennow-core/src/store_requests.rs create mode 100644 external/opennow/native/opennow-core/src/streamer.rs create mode 100644 external/opennow/native/opennow-core/src/thanks.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/archive.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/bundle.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/integration_tests.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/managed.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/mod.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/process.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/recovery.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/security.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/tests.rs create mode 100644 external/opennow/native/opennow-core/src/update_apply/verification.rs create mode 100644 external/opennow/native/opennow-core/src/updater.rs create mode 100644 external/opennow/native/opennow-core/src/version.rs create mode 100644 external/opennow/native/opennow-core/tests/core_protocol_cli.rs create mode 100644 external/opennow/native/opennow-core/tests/fixtures/profile_fsync_gate.c create mode 100644 external/opennow/native/opennow-core/tests/graphics_preferences_cli.rs create mode 100644 external/opennow/native/opennow-core/tests/legacy_contract.rs create mode 100644 external/opennow/native/opennow-core/tests/media_env_cli.rs create mode 100644 external/opennow/native/opennow-core/tests/settings_event_order_cli.rs create mode 100644 external/opennow/native/opennow-streamer/Cargo.lock create mode 100644 external/opennow/native/opennow-streamer/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/README.md create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/decode_progress.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/microphone.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_control_ping_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_negotiation.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_setup_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_tls_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_transport_diagnostics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_server_control_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/queue_drops.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/recording_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/telemetry.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/mod.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/sdp.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling/tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/video_attributes.sdp create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/README.md create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/build.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/include/opennow_streamer_ffi.h create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/allowlist.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/descriptor.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/ds4.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic_runtime.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/runtime.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/session.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/README.md create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/examples/probe.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/README.md create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-pq.obu create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-sdr.obu create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-ayuv-sdr.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision-luma.bin create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-sdr.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-pq-precision.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-pq.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-precision.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-sdr.hevc create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/adapter_decode.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/aperture.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_color.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_order.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/format.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/queue.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/adapter_index.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/audio.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/color.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/decoder.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/embedded.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/graphics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/mod.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.hlsl create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/y410_color.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/build.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/audio_playout.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/embedded_input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/graphics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/h264.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_backend.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_frame_pacing.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_xinput.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/macos_backend.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media_recording_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/microphone.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/native_surface.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/output.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/queue.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/recording.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/replay.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/runtime.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/video_queue.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_graphics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_raw_input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/log.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/text_input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/bin/nvst_peer_probe.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/frame_stage_timing.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_jitter.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_bandwidth.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_control.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_cursor.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_haptics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input_delivery.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_microphone.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_network.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_ping_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_progress_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_qos_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_policy.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control_tests.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/diagnostics.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/input.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/mod.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/stats.rs create mode 100644 external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/tests.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/.github/workflows/ci.yml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitignore create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitmodules create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/CHANGELOG.md create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/CONTRIBUTING.md create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml.orig create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/LICENSE.md create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/README.md create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/build.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.appveyor.yml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.gitlab-ci.yml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.travis.yml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/AUTHORS create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/CMakeLists.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/COPYING create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/ChangeLog create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/LICENSE_PLEASE_READ.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.am create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.mips create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.unix create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/NEWS create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README.draft create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/autogen.sh create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/_kiss_fft_guts.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arch.h create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm2gnu.pl create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm_celt_map.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armopts.s.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_fft_ne10.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_mdct_ne10.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_pitch_xcorr_arm.s create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fft_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_arm64.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv4.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv5e.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv4.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv5e.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/mdct_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/bands.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/bands.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_decoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_encoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cpu_support.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes_arch.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes_arm_ne10.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/ecintrin.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c5x.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c6x.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_debug.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_generic.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/float_cast.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/laplace.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/laplace.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mathops.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mathops.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mfrngcod.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/celt_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/fixed_generic_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/kiss_fft_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/mdct_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/pitch_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/vq_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/modes.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/modes.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/opus_custom_demo.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/os_support.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/pitch.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/pitch.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/quant_bands.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/quant_bands.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/stack_alloc.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed_arm_ne10.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float_arm_ne10.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_cwrs32.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_dft.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_entropy.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_laplace.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mathops.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mdct.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_rotation.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_types.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/vq.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/vq.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/celt_lpc_sse.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/celt_lpc_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/pitch_sse.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/pitch_sse.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/pitch_sse2.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/pitch_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/vq_sse.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/vq_sse2.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86_celt_map.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86cpu.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86cpu.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_headers.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_sources.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/CFeatureCheck.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusBuildtype.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusFunctions.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusPackageVersion.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusSources.cmake create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/config.h.cmake.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/vla.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/configure.ac create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Doxyfile.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Makefile.am create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_draft.sh create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_isobmff.sh create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_oggdraft.sh create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/customdoxygen.css create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-oggopus.xml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus-update.xml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus.xml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-payload-rtp-opus.xml create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/footer.html create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/header.html create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.css create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.html create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_logo.svg create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_update.patch create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/release.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/trivial_example.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_custom.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_defines.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_multistream.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_projection.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_types.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/as-gcc-inline-assembly.m4 create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/ax_add_fortify_source.m4 create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/opus-intrinsics.m4 create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson.build create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/get-version.py create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/read-sources-list.py create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson_options.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus-uninstalled.pc.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.m4 create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.pc.in create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_headers.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_sources.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/releases.sha2 create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/dump_rnn.py create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/rnn_train.py create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/A2NLSF.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/API.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/CNG.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/HP_variable_cutoff.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/Inlines.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_analysis_filter.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_fit.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_inv_pred_gain.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LP_variable_cutoff.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroCount.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroDebug.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF2A.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ_weights_laroia.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_decode.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_del_dec_quant.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_encode.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_stabilize.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_unpack.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ_del_dec.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/SigProc_FIX.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VAD.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VQ_WMat_EC.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/ana_filt_bank_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv4.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv5e.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/arm_silk_map.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_arm64.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv4.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv5e.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/biquad_alt.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander_32.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/check_control_input.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/code_signs.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_SNR.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_audio_bandwidth.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_codec.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/dec_API.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_core.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_frame.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_indices.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_parameters.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pitch.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pulses.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decoder_set_fs.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/define.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/enc_API.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_indices.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_pulses.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/errors.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_analysis_filter_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_scale_ctrl_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/apply_sine_window_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_arm.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/autocorr_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/burg_modified_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/corrMatrix_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/encode_frame_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LPC_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LTP_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pitch_lags_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pred_coefs_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_Q16_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/main_FIX.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/noise_shape_analysis_FIX_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/prefilter_FIX_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/warped_autocorrelation_FIX_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/noise_shape_analysis_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/pitch_analysis_core_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/process_gains_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/regularize_correlations_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy16_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur64_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/structs_FIX.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/vector_ops_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/warped_autocorrelation_FIX.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/burg_modified_FIX_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/prefilter_FIX_sse.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/vector_ops_FIX_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_analysis_filter_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_inv_pred_gain_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_analysis_filter_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_scale_ctrl_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/SigProc_FLP.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/apply_sine_window_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/autocorrelation_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/burg_modified_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/bwexpander_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/corrMatrix_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/encode_frame_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/energy_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LPC_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LTP_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pitch_lags_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pred_coefs_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/inner_product_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/k2a_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/main_FLP.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/noise_shape_analysis_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/pitch_analysis_core_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/process_gains_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/regularize_correlations_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/residual_energy_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_copy_vector_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_vector_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/schur_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/sort_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/structs_FLP.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/warped_autocorrelation_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/wrappers_FLP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/gain_quant.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_decoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_encoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/inner_prod_aligned.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/interpolate.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/lin2log.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/log2lin.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/macros.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/main.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/NSQ_del_dec_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/macros_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/sigproc_fix_mipsr1.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_defines.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_tables.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/process_NLSFs.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/quant_LTP_gains.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2_3.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_AR2.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_IIR_FIR.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_down_FIR.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_up2_HQ.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_structs.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/shell_coder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sigm_Q15.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sort.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_LR_to_MS.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_MS_to_LR.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_decode_pred.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_encode_pred.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_find_predictor.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_quant_pred.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/structs.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sum_sqr_shift.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/table_LSF_cos.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_LTP.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_NB_MB.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_WB.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_gain.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_other.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pitch_lag.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pulses_per_block.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/test_unit_LPC_inv_pred_gain.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tuning_parameters.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/typedef.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_del_dec_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/SigProc_FIX_sse.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VAD_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VQ_WMat_EC_sse4_1.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/main_sse.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/x86_silk_map.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_headers.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_sources.mk create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp_data.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_compare.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_decoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_demo.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_encoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_decoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_encoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_private.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_decoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_encoder.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/repacketizer.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/repacketizer_demo.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/tansig_table.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/meson.build create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.options create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_encode_regressions.c create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/run_vectors.sh create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_api.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_common.h create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_decode.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_encode.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_padding.c create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_projection.c create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_dump.py create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_train.py create mode 100755 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/txt2hdf5.py create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/common.props create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.sln create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj.filters create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj.filters create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj.filters create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj.filters create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj.filters create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/genversion.bat create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/wrapper.h create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/.cargo_vcs_info.json create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/CHANGELOG.md create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml.orig create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/LICENSE-MIT.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/OPENNOW-PATCH.md create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/README.md create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/fuzz.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/rng.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/setup.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/acked_bitrate_estimator.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/budget.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/detector.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/api.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/arrival_group.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/control.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/rate_control.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/trendline.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/link_capacity_estimator.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/loss_controller.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/macros.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/cluster.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/control.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/estimator.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/smoother.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/time.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/change/direct.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/change/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/change/sdp.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/channel.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/config.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/finger.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/srtp.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/dtls.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec_config.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/format/format_params.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/format/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/format/payload_params.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/io/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/io/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/lib.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/media/event.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/media/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/media/writer.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/control.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/leaky.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/null.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/queue.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/LICENSE-APACHE.txt create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/av1.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/buffer_rx.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/comfort_noise.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp8.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp9.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/g7xx.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264_profile.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265_profile.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266_profile.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/null.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/opus.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/payload.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp8.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp9.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/dir.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/ext.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/header.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/id.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mtime.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/bb.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fir.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fmt.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/header.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/list.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/nack.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/pli.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/psfbapp.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/remb.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rr.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rtcpfb.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sdes.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sr.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/twcc.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/xr.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/srtp.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/vla.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/dcep.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/snap.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/data.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/error.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/parser.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/session.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/stats.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/receive.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register_nack.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache_buf.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_queue.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_stats.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/util/average.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/util/bit_pattern.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/util/mod.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/util/time_tricks.rs create mode 100644 external/opennow/native/opennow-streamer/vendor/str0m/src/util/value_history.rs diff --git a/external/opennow/CHANGES.md b/external/opennow/CHANGES.md new file mode 100644 index 000000000..4f135bcf7 --- /dev/null +++ b/external/opennow/CHANGES.md @@ -0,0 +1,3 @@ +# DroidDeck changes to the vendored OpenNOW code + +None yet (pristine import of a49b124c). diff --git a/external/opennow/LICENSE b/external/opennow/LICENSE new file mode 100644 index 000000000..3ca06cd13 --- /dev/null +++ b/external/opennow/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Zortos + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/external/opennow/NOTICE.md b/external/opennow/NOTICE.md new file mode 100644 index 000000000..22286496d --- /dev/null +++ b/external/opennow/NOTICE.md @@ -0,0 +1,17 @@ +# OpenNOW (vendored) + +This folder holds parts of **OpenNOW**, an open-source GeForce NOW client by +**Zortos** ([zortos293](https://github.com/zortos293)) and the OpenNOW contributors: +, MIT License (see `LICENSE`). + +Imported from OpenCloudGaming/OpenNOW commit `a49b124ce31e87113fd4961f6a321fbf0ad1fdc5` +(2026-10-18). Only the parts DroidDeck's GeForce NOW store uses are kept: + +- `native/opennow-core`: accounts, sign-in, catalog and CloudMatch session services + (newline-delimited JSON over stdin/stdout, `docs/core-protocol.md`). +- `native/opennow-streamer`: the native NVST streamer (protocol, transport, core, + platform, FFI) plus its vendored `str0m` and `audiopus-sys`. + +DroidDeck's own changes to these files are listed in `CHANGES.md`. NVIDIA and +GeForce NOW are trademarks of NVIDIA Corporation; neither OpenNOW nor DroidDeck is +affiliated with or endorsed by NVIDIA. diff --git a/external/opennow/docs/core-protocol.md b/external/opennow/docs/core-protocol.md new file mode 100644 index 000000000..b4acf2704 --- /dev/null +++ b/external/opennow/docs/core-protocol.md @@ -0,0 +1,1857 @@ +# OpenNOW shell/core protocol + +The Qt shell and Rust application core communicate over newline-delimited UTF-8 +JSON on inherited standard input/output. Standard error is reserved for redacted +diagnostics. A line is limited to 1 MiB. Unknown, malformed or oversized protocol +messages terminate the core connection instead of leaving the shell in an +ambiguous state. + +## Handshake + +The first shell request is always: + +```json +{"type":"request","id":"1","method":"core.hello","params":{"protocolVersion":5,"shell":"qt","shellVersion":"0.5.4"}} +``` + +Protocol 5 requires an exact selected catalog variant and account scope for fresh +session allocation. It retains the bounded library-page contract introduced in +protocol 4. Older shells are rejected with `incompatible_protocol` during +`core.hello`, and protocol-5 shells reject older cores. The native streamer +protocol remains 7. + +The core must return the same protocol version and its capabilities. The shell +does not send product requests before this succeeds. Version mismatches, a +five-second handshake deadline, process exit and invalid data all transition the +transport to `failed` with a credential-free diagnostic. + +The desktop queue selector requires the `queue.servers.v1` capability in this +handshake. The Qt client and relocated package probes reject cores that omit it, +including older protocol-5 binaries, before sending any product requests. This +additive capability leaves the JSON envelope and native streaming ABI unchanged. + +## Saved accounts (`auth.accountPicker.v1`) + +The "Who's playing?" picker requires the `auth.accountPicker.v1` capability. The +Qt client rejects cores that omit it during `core.hello`, before any product +request. The capability is additive: the protocol version, the envelope and the +native streaming ABI are unchanged. + +`auth.accounts.list` returns `{accounts, activeUserId, generation}`. Each account +contains only public identity fields; tokens never cross the boundary: + +| Field | Type | Meaning | +|---|---|---| +| `userId`, `displayName` | string | Saved identity. | +| `email`, `avatarUrl` | string or null | From the last sign-in or token refresh. | +| `membershipTier` | string | `FREE`, `PERFORMANCE` or `ULTIMATE` today. Shells must render other values without failing. | +| `providerCode` | string | Login provider code. `NVIDIA` is NVIDIA; any other code is an alliance partner. | +| `providerName` | string | Display name for the provider. The core owns the code-to-name mapping (`gfn::provider_display_name`): the name saved at sign-in, then the discovered provider name for that code, then the code itself. `BPC` is always `bro.game`. | +| `providerIdpId` | string, optional | Provider IdP for "Sign in again". Absent for accounts saved before this field existed until their next sign-in or refresh. | +| `alliance` | bool | `providerCode != "NVIDIA"`. | +| `hasPin` | bool | A local profile PIN guards `auth.accounts.switch`. | +| `needsSignIn` | bool | The last restore or switch for this account ended with refresh outcome `expired` after the provider answered (not after a network failure). Cleared when the account signs in or refreshes again. | + +`auth.accounts.switch` never ends a running cloud session; the seat keeps its +original owner. The Qt shell ends the session through its end-session +confirmation and waits for `session.stop` before it sends the switch. + +The `startAsLastUsedAccount` setting (bool, default `false`) skips the launch +picker when two or more accounts are saved. Loading settings migrates the +retired `consoleProfilePickerOnLaunch: false` to `startAsLastUsedAccount: true`. + +## Cloud library actions and launch decisions + +`catalog.launch.inspect({appId, variantId})` always resolves the exact parent and +store variant. Its response contains `appId`, `variantId`, `game`, `scope`, +`catalogRevision`, `fetchedAt`, `freshness`, and `decision: {status, message}`. +Statuses are `ready`, `ownership_required`, `selection_required`, `link_required`, +`subscription_required`, `patching`, `maintenance`, `unavailable`, and +`metadata_unconfirmed`. Only the selected variant's `MANUAL` or `PLATFORM_SYNC` +library status records ownership. App-wide library flags, Home pins, favorites, +free-game labels, store subscriptions, and ownership of another variant do not. +Store-link requirements use the store definitions and fresh account state. +Recorded store-subscription IDs must match the account's active subscriptions. +App playability, variant readiness, patch metadata, and membership restrictions +remain independent checks. Unknown metadata is not a positive authorization. + +`catalog.launch.store.inspect({store?})` resolves the platform-client apps for the +current streaming region with a `PLATFORM_CLIENT` type filter and returns +`{store, appId, variantId, game, scope, decision, catalogRevision, fetchedAt, +freshness}`. The resolved `appId`, `variantId`, and `game` are the values the +platform-client query returned for this account. `store` defaults to `STEAM`, +the only supported store-client launch, and any other value is `invalid_params`. +The eligible target is the first game exposing a variant whose exact `appStore` +value matches the store and whose `id` is a bounded identifier; extra variants and +extra platform-client apps are searched, not rejected. + +The store-launch decision reuses the shared decision vocabulary. It requires the +exact parent identity and variant identity, the exact store on the resolved +variant, a present `gfn.status` of `AVAILABLE`, and any patching or maintenance +metadata the server returns. It then requires the persistent-storage entitlement: +the subscription `addons` entry whose `type` is `STORAGE` and whose `subType` is +`PERMANENT_STORAGE` and whose `status` is `OK`, which the core publishes as +`subscription.storageAddon`. An ephemeral storage add-on, including +`EPHEMERAL_STORAGE`, does not authorize a store launch. Store-account linking +requirements, recorded store subscriptions, membership tier, and +`isGamePlayAllowed` apply from the same account metadata the ordinary launch path +uses. + +Cloud DLSS overrides follow the official client. The `dlssOverrides` setting stores +the exact `requestedStreamingFeatures.dlssOverrides` wire object (`srMode`, `srModel`, +`srScale`, `rrModel`, `fgMode`, `fgModel`, `fgMultiplier`), where `0` leaves a feature +to the game. The core clamps unknown enum values to `0`, leaves a mode to the game +when its model is, fixes `srScale` for every SR mode except Custom (33–100), and sends +`fgMultiplier` only for Dynamic frame generation. Every create request carries the +object. `subscription.dlssOverrides` reports `entitled` and `upsell` for +`superResolution`, `frameGeneration`, and `rayReconstruction` from the MES +`SUPPORTED_DLSS_OVERRIDE_*` features and `SUPPORT_DLSS_OVERRIDE_*` upsells, plus the +entitled `frameGenerationMultipliers` from `SUPPORTED_DLSS_FG_MULTIPLIERS`. + +A platform client that the ordinary catalog resolution also returns is judged by +that resolution's metadata, so catalog `playabilityState` and patch state are +enforced there. When the ordinary resolution does not return the app, the store +query's own `gfn.status` is the readiness field, because that query selects no +catalog playability. When the ordinary resolution returns the app but not the +discovered variant, the two sources disagree and the decision is +`metadata_unconfirmed` rather than a guess. + +`catalog.launch.inspect` and `session.create` accept `storeLaunch: true`, a +boolean that selects the platform-client resolution source and the +persistent-storage requirement. The flag grants nothing: the same mutation +admission, scope, catalog revision, active-seat, and exact-variant checks run, and +a store intent that names a target the platform-client resolution does not return +is not ready. Missing, non-boolean, or mismatched values do not fall back to +another target. + +All five mutation RPCs require a bounded nonempty parent `appId` and the current +`scope: {generation, userId, providerIdpId}`. Ownership mutations also require the +exact `variantId`. `catalog.ownership.add` requires +`confirmedExistingLicense: true`, the user's assertion that they already own the +selected store license. This operation neither buys nor grants a license. +`catalog.ownership.select` accepts only an owned variant and does not add ownership. + +Favorites map to the GraphQL `AddFavoriteApp` and `RemoveFavoriteApp` operations +with `appId` and `locale` variables. Ownership maps to `AddOwnedVariant`, +`RemoveOwnedVariant`, and `SelectOwnedVariant`, with `cmsId` and `locale` +variables. `cmsId` supplies the `variantId` argument. Every mutation selects +`app { id }` and checks the returned parent identity. Mutations share catalog +scheduling and serialize conflicting actions for the same account and parent +app. Fresh allocation holds that same app admission while checking and creating. + +Mutation results distinguish `outcome: acknowledged|unconfirmed` from +`reconciliation: confirmed|unconfirmed`. A fresh exact read confirms the desired +current state; acknowledgement alone does not. Results include captured target +IDs, `operation`, `scope`, `catalogRevision`, nullable refreshed `game`, a visible +`message`, and sanitized `error.code`, `error.httpStatus`, and `error.graphql` +code/path details. `reconciliationCode` and `invalidationCode` distinguish read +or cache-invalidation failures from the mutation outcome. +Neither ambiguous failures nor HTTP 401 cause mutation replay. Catalog pages are +invalidated before sending and again after the attempt. Cancellation may prevent +delivery of the outcome without preventing an already-sent upstream mutation. +The shell marks that outcome unconfirmed and refreshes metadata without resending. + +`catalog.favorites.list` reads the `FAVORITES` panel using `GetGameSection` with +`panelNames`, `vpcId`, and `locale`; it has no cursor. It preserves all parsed game +items up to the 1,000-game and 768-KiB response bounds, rather than the Store +panel's 24-item display limit. The response contains `games`, section identity +and `seeMoreInfo`, `scope`, `catalogRevision`, and `fetchedAt`. `coverage` is +`unknown` and `complete` is false because the panel does not establish global +completeness. No pagination or see-more request is invented. A failed read keeps +the shell's last usable list, and a specific favorite is reconciled through the +exact app's `library.favorited` value. Existing `favoriteGameIds` remain local +Home pins; pinning, collections, hiding, and reordering do not upload favorites. + +## Messages + +### Authentication boundary + +Protocol 5 auth responses and `auth.session.changed` events use the same envelope. +`session` is either null or an allowlisted object containing `user` and `provider`. +Credentials and issuing-client details remain private to the core. The envelope +includes the core-process account `generation`, `persistence`, `refresh`, `warnings`, +and `deviceIdentity`. Consumers reject older generations and reset their generation +when a new core process completes its handshake. A token refresh does not change +account generation. + +`auth.device.start` returns `attemptId`, `userCode`, verification URLs, `qrRows`, +`expiresAt`, and `intervalSeconds`. Poll, complete, and cancel use `attemptId` only. +The core retains the device grant and enforces the deadline, one in-flight poll, +and cumulative five-second `slow_down` increments. Pending poll replies include +`retryAfterMs`; the shell schedules one subsequent poll from that reply. +Cancellation before the completion commit fence prevents persistence and publication. +Once the fence is entered, completion is committed under auth ownership; cancelling +its response does not undo that login. The shell reconciles such cancellation with +`auth.session.get`. Logout or account replacement invalidates pending login/link work. + +Persistence intent is independent of outcome. Persistent sessions prefer the OS +credential store on every desktop platform, including Windows. `secure-store` means +the session was written to and verified in that store and its nonsecret index committed. +`local-file` means session tokens are saved in unencrypted JSON because the OS store +is unavailable. The per-account file is under +`data_dir/fallback-sessions/.json`, with private file access permissions. +Those permissions do not encrypt the tokens. The shell warns that anyone who can read +the file can access the account. Fallback reads take priority over stale OS-store entries. +The next successful secure save verifies the OS-store entry before removing the JSON +fallback. +If the account-index commit fails after JSON is written, persistence remains +`local-file` and warnings report that account restoration may fail. Interrupted +JSON temporary files are removed on startup and during account cleanup. + +`memory-only` never writes a plaintext credential fallback. `migration-pending` means +a recoverable legacy source remains because migration could not finish. `unavailable` +means restoration failed. Warnings identify deferred cleanup without containing +credentials, and the auth envelope never includes tokens. An explicitly temporary +login suppresses automatic restoration of older saved grants for that identity. +Legacy sources are removed only after verified persistence and metadata commit, or +explicit account removal. This is file cleanup, not guaranteed secure erasure of +backups or snapshots. + +Logout always ends local auth ownership and reports `remoteRevoke` separately from +`localCleanup`. Revocation is best-effort DELETE of the selected client grant with an +access bearer; all-account revocation has a five-second total network budget. Failed +local deletion remains suppressed in the nonsecret account index and is retried on +startup. Logout removes the JSON fallback even if the OS store remains inaccessible; +deferred OS-store cleanup keeps the existing suppression semantics. A metadata write +failure reports pending cleanup and cannot guarantee that suppression survives restart. +Logout selects the next saved account only when it does not require a profile PIN. +Automatic restoration, including a new core process, never activates a PIN-protected +profile. Its credentials remain saved for an explicit PIN-verified switch. An already +unlocked session remains active within the current core process. + +Qt loads saved profiles independently of signed-in account services. After a signed-out +startup completes, saved profiles open in the existing account picker. An active stream +does not trigger automatic navigation. Its user can open saved accounts explicitly from +sign-in and unlock a profile without replacing the native video item. + +Logout-all also scans local fallback files independently of the account index. Missing +or corrupt metadata cannot prevent best-effort plaintext cleanup. Recoverable identities +are suppressed and removed from the OS store too. Unreadable identities or failed cleanup +remain `pending`; corrupt metadata is not silently overwritten. + +The core persists one versioned 64-hex device identity. Upgrades freeze the previously +derived identity using the current environment; if that environment changed before the +first upgraded launch, the former hash cannot be recovered. Corrupt or unwritable identity +storage is not replaced, new login is blocked, and restoration reports unavailable device +identity rather than claiming durable compatibility. + +```json +{"type":"request","id":"42","method":"settings.get","params":{}} +{"type":"response","id":"42","ok":true,"result":{"settings":{}}} +{"type":"response","id":"42","ok":false,"error":{"code":"invalid_setting","message":"…"}} +{"type":"event","name":"settings.changed","payload":{"key":"fps","value":120}} +{"type":"cancel","id":"42"} +``` + +Request IDs are unique within a core process. Every request has a bounded +deadline (100 ms to five minutes). The shell sends cancellation after a timeout +or explicit cancellation. Events are delivered in batches through a queue of at +most 512 items; overflow drops the oldest event and emits a diagnostic counter. + +The core admits at most eight RPC workers, with at most four background workers +(`catalog.*`, `artwork.*`, and `network.regions.ping`). The remaining capacity +is reserved for other methods, including session/control operations. Excess +requests receive `busy`; duplicate active IDs are also rejected. Cancellation +only tracks active IDs and never frees a worker slot before that worker exits. +The Qt client keeps requests rejected with `busy` pending and retries the same +ID and payload after 100 ms, doubling the delay up to one second. Retries do not +extend the original deadline. Cancellation, shutdown, and process failure discard +pending retries. Other errors are delivered to the caller without retrying. +Cancelled requests suppress their response. Store page retries/cache traversal +and region measurement loops stop at cooperative checkpoints. An already-running +blocking HTTP, DNS, or TCP operation is not forcibly interrupted; its existing +timeout still applies. Other mutating operations already dispatched are not rolled back. + +Protocol 5 retains the requirement for the Qt client to acknowledge an accepted successful `session.create` +response with `{"type":"ack","id":"42"}` before delivering that response to QML. +Cancelled or timed-out requests do not acknowledge late responses. The create worker +retains its admission slot for at most ten seconds awaiting acceptance, then the +CloudMatch owner deletes an unaccepted fresh allocation with an eight-second HTTP +deadline. Fresh creation sends a CloudMatch POST without a follow-up PUT; +cancellation after POST uses the same cleanup. PUT RESUME applies only to claims +of existing sessions. +The receipt does not apply to claims of existing sessions. A create response is not +also broadcast as `session.changed`, preventing a late event from reviving a cancelled +launch. Cleanup failure retains the seat and its scoped discovery route for explicit +retry, blocks another fresh allocation, and reports `session_cleanup_pending`. +The failed-cleanup record persists only the seat identity, app/status, trusted control +route, original provider/account identity, and error code in `pending-session-cleanup.json`; +it contains no tokens or signaling secrets. Discovery surfaces it only to that account. +Explicit successful DELETE or not-found clears that exact record, never a different seat. +HTTP-success DELETE responses containing an explicit vendor rejection retain ownership. +The CloudMatch owner reserves fresh-create admission before endpoint resolution or POST. +Concurrent creation, pending handoff, and in-progress cleanup lock contention fail promptly with +`session_update_busy`, not the automatically retried `busy` response. Admission is +released on every pre-ID error; after allocation the exact seat remains reserved until +handoff or compensation completes. Cleanup records are read through a 16-KiB limit plus +one overflow-detection byte before parsing, including for corrupt or oversized files. + +The `session.terminalCleanup.v1` capability adds `session.ended`. The Qt client +requires this capability during the handshake so an older packaged core cannot +silently retain an ended seat. The JSON protocol version remains 5. + +After an authoritative, non-resumable `nvst-server` termination, the shell sends +`session.ended` with `{sessionId, ownerScope: {userId, providerIdpId}}` +from the ended session. This local acknowledgement makes no network request and +requires no valid authentication token. It releases only the exact accepted seat +whose account and provider match. An extra `ownerScope.generation` is ignored: +control reconciliation can change the core generation before the shell consumes +its response, but it does not change the remote seat's identity. +Pending allocation handoffs and cleanup records are not released. +A matching pending handoff returns +`session_update_busy` so the shell retries after handoff. The response is +`{sessionId, cleared}`; duplicate or stale notifications return `cleared: false` +without changing another seat. +No `session.changed` event is emitted, so a late acknowledgement cannot clear a +replacement session in the shell. The shell blocks another launch until the +acknowledgement arrives and retries after transient RPC failure or core restart. +Late session snapshots cannot revive the natively terminated seat. +Transport EOF and other failures with unknown resumability do not send this request. + +Raw CloudMatch status 7 is `phase: "finished"`, with a `termination` object containing +`source: "cloudmatch-session-status"`, `status: 7`, and `resumable: false`. This proves +the seat is terminal, not whether the game exited successfully; no vendor reason is +invented. A targeted GET returning HTTP 404 yields `session: null` and a top-level +termination with `source: "cloudmatch-http"`, `httpStatus: 404`, `sessionId`, and +`resumable: false`. Authentication, invalid payloads, other HTTP failures, empty lists, +and native transport closure are not terminal evidence. Recovery first polls the exact +seat before claiming it. `session.poll` with `recoveryMode: true` does not broadcast a +session change while the recovery owner is deciding whether to claim. + +Negotiated profiles retain normalized `bitDepth` (8 or 10), CloudMatch `chromaFormat` +(0 or 1), and per-component `*Source` fields. `request`, `finalized`, `server`, and +`unreported` distinguish present values from omissions. Same-seat partial updates retain +previously known components; explicit finalized invalid values are not replaced with +saved preferences. Incomplete or unsupported accepted color is rejected before native +attachment. Transport terminal +events add `termination: {source: "nvst-transport", code, resumable: null}`, preserving +unknown cloud-session disposition rather than assigning a normal-exit reason to EOF. + +Native ABI 13 adds `session-timer` events through the existing JSON event callback, +with `startId`, `timerType`, `secondsLeft` and `active`. Idle-warning type 4 shows +stream chrome without stopping media; type 5 or successfully delivered gameplay +input clears it. Server termination events preserve `wireCode` separately from +optional application `reason` and `extendedCode` in `termination`, use source +`nvst-server`, and carry a boolean `resumable` only for verified classifications. +Nonresumable server termination prevents automatic restart and resume. Unknown +classification remains null and is reconciled against CloudMatch. The JSON protocol +version is unchanged; the native library and Qt shell must be rebuilt together. +Expected, authoritative server endings do not create stream-error incidents or +session-error telemetry. Resumable, unknown, and local transport/decode failures +retain their existing reporting. + +Deferred input also preserves the ordering of one bounded, cancellation-aware paste +batch. Accepted prefixes are not replayed, later discrete input waits behind text, +and cancellation removes only unsent text. Rejected paste admission emits +`text-input-error` with `startId` and a fixed `reason`, through the existing ABI 13 +JSON callback. It is a nonterminal retry notice, not a stream failure; Qt discards +stale generations and shows feedback over the current video item. + +After a decoder-stall recovery attempt, the native engine allows one further video-timeout +interval for decoded output to resume. Output progress or a decoder-epoch change clears +that deadline. Otherwise the engine emits `nvst-recovery-exhausted` and a stopped status +once, allowing Qt to recover the session instead of leaving silent media connected forever. +This does not classify the remote cloud session as ended. + +## Implemented core methods + +### Game artwork + +GraphQL-backed game objects expose nullable artwork URL strings: `imageUrl` prefers +`GAME_BOX_ART`, `heroImageUrl` prefers hero/banner imagery, and `keyArtUrl` prefers +NVIDIA `KEY_ART` with `KEY_IMAGE` as a fallback. Console Home uses `keyArtUrl` for +square tiles while retaining hero imagery for wide tiles. If key art is absent, +the shell falls back to `imageUrl`, then `heroImageUrl`. Older cached objects and +public-catalog games may omit `keyArtUrl`; it is an optional, additive field in +protocol 1 and does not change the handshake or existing artwork fields. + +### Store pagination (`catalog.storePages.v1`) + +The protocol-1 envelope and 1 MiB limit are unchanged. This additive capability +advertises cursor pagination and separate storefront presentation. The existing +`catalog.store.list` method now caps `limit` at 100 (default 100), returning one +complete upstream page instead of aggregating thousands of games. + +Fresh upstream pages request up to 100 games. Existing cached pages may contain +fewer games; always follow the returned cursor rather than assuming a full page +has the requested count. + +Request: `{ "limit":100, "cursor":"", "searchQuery":"" }`. Cursor is an opaque +string (at most 4096 UTF-8 bytes); search is at most 512 UTF-8 bytes. Response: +`{ "games":[], "count":0, "totalCount":0, "hasNextPage":false, +"nextCursor":"", "source":"store-browse", "fetchedAt":0 }`. +Pass `nextCursor` unchanged to the next call with the same search. A final page +has `hasNextPage:false`; empty non-final pages may advance only when the upstream supplies an advancing cursor. Invalid mapped identities are errors. +Missing, repeated or oversized continuation cursors are errors, not completion. + +Each result is limited to 768 KiB after JSON encoding, leaving room for the +envelope. Oversized pages are refetched with a smaller count at the same cursor; +they are never truncated while advancing the cursor. A single oversized game +returns `catalog_response_too_large` without disrupting the core connection. + +`catalog.store.presentation` accepts `{ "section":"marquee" }` (also `panels` +or `filters`) and returns `{ "section":"marquee", "items":[] }`, independently +bounded to 768 KiB. Failed/oversized optional sections do not fail game pages. + +Store pages and presentation results include `cacheHit` (boolean). Successful +responses are persisted under the core data directory in `store-cache-v1`, with +hashed account/provider/membership/proxy/locale and request keys. Credentials +are not stored. Reads remain bounded to 768 KiB; the cache is limited to 64 MiB +and 512 entries. Missing or corrupt entries refetch normally. There is no timed +catalog invalidation: an optional `refresh:true` on a first-page request clears +that account/context's pages and presentation before fetching. Continuations +must omit it or send false. Other accounts' entries are unaffected. + +Concurrent misses for the same cache key and refresh epoch share one successful +fetch. Store network requests (including server metadata, presentation fallbacks, +and oversized-page retries) are serialized with at least 50 ms between them. +HTTP 429 starts a core-wide Store cooldown using `Retry-After` (seconds or HTTP +date), or 60 seconds when it is absent or invalid. Uncached requests during the +cooldown return `rate_limited` without network traffic; cached responses remain +available. The core does not automatically retry a rate-limited request. + +Server metadata lookups share one bounded, in-memory cache across Store, library, +and subscription requests. A successful lookup is reused for five minutes, scoped +by provider endpoint, account, and token fingerprint; concurrent callers share +the lookup. Ordinary failures use the same context's last known value (or +`GFN-PC` when none exists) for 30 seconds before retrying. Rate-limit and +cancellation errors propagate instead of becoming cached fallback values. + +The shell serializes game requests, merges by stable game identity, and retains +loaded games and the failed cursor on error. Retries resume that page. +Search/account changes cancel requests before clearing state; late responses +are ignored. + +### Local Store browsing (`catalog.storeLocal.v1`) + +`catalog.store.local` pages and searches the saved catalog, without replaying +every cached page into Qt. It accepts `limit` (capped at 60), `cursor`, +`searchQuery`, and optional `genre`, `store`, and `categoryId` strings (at most +256 UTF-8 bytes each). A cursor belongs to its search/filter context and must +be returned unchanged. The existing 768 KiB result budget still applies. + +The result includes `games`, `count`, `totalCount` (matching games), +`catalogTotalCount`, `hasNextPage`, `nextCursor`, `source:"store-local"`, +`cacheHit:true`, and `cacheComplete`. First pages also include `facets` with +all indexed genres, stores, and official categories (`id`, `label`, `count`), +not just those present on the visible page. Subsequent pages omit facet data +by returning `facets:null`. `categoryId:"all"` selects the entire catalog. + +The core builds one bounded, account-scoped metadata index from saved pages; +full records remain on disk and only selected results are materialized. +Search ranks exact titles, acronyms, prefixes, reordered words and single +typing errors, with owned games preferred on close matches. Store requests +40 results per page; Ctrl+K uses the same algorithm with a six-result limit +and debounced, cancellable requests. + +There is no automatic catalog crawl. Scroll/navigation demand or Load more +requests the next page. A missing/partial cache fetches bounded upstream pages +on demand; `cacheComplete:false` distinguishes this from a complete index. +On this method, `refresh:true` rebuilds the local index without deleting saved +pages. The upstream `catalog.store.list` refresh behavior remains unchanged. + +`catalog.store.presentation` also accepts `metadataOnly:true`. For `panels`, +each section returns its title and `totalCount` with an empty `games` array; +the complete panel remains cached in the core. Qt materializes shelf games +and artwork only near the viewport, using the section's local category ID +(`shelf::
`). See all opens that category in Store. + +### Method list + +- `core.hello` +- `app.status` +- `settings.get` +- `settings.choices.get` +- `settings.set` +- `settings.shortcuts.update` +- `settings.reset` +- `auth.providers.list` +- `auth.session.get` +- `auth.device.start` +- `auth.device.poll` +- `auth.device.complete` +- `auth.device.cancel` +- `auth.logout` +- `auth.accounts.logoutAll` +- `auth.accounts.list` +- `auth.accounts.switch` +- `auth.accounts.remove` +- `auth.pin.status`, `auth.pin.set`, `auth.pin.clear`, `auth.pin.verify` +- `catalog.public.list` +- `catalog.library.list` returns one bounded upstream page, not an aggregate library. +- `catalog.game.get`, `catalog.definitions.get`, `catalog.languages.get` +- `catalog.launch.inspect`, `catalog.favorites.list` +- `catalog.launch.store.inspect` resolves the eligible Steam store-client launch target. +- `catalog.favorites.add`, `catalog.favorites.remove` +- `catalog.ownership.add`, `catalog.ownership.remove`, `catalog.ownership.select` +- `catalog.store.list`, `catalog.store.local`, `catalog.store.presentation` +- `network.regions.list` +- `network.regions.ping` +- `queue.servers.list` +- `account.subscription.get` +- `account.connections.list`, `account.connections.sync`, `account.connections.unlink` +- `account.connections.sync.status`, `account.connections.sync.cancel` +- `account.connections.link.start`, `account.connections.link.poll` + +- `account.storage.locations`, `account.storage.reset` +- `session.create` +- `session.poll` +- `session.stop` +- `session.ended` +- `session.active.get` +- `session.remote.list`, `session.claim`, `session.ad.report` +- `streamer.detect` +- `streamer.prepare` +- `streamer.start` +- `streamer.status.get` +- `streamer.stop` +- `streamer.input.pause`, `streamer.control`, `streamer.surface.update` +- `streamer.recording.start`, `streamer.recording.stop` +- `diagnostics.snapshot`, `diagnostics.export`, `acceptance.export` +- `media.root.get`, `media.recording.target`, `media.list`, `media.delete` +- `cache.delete` +- `queue.status.get`, `queue.serverMapping.get` +- `thanks.data.get`, `communityProxy.provision` +- `updater.state.get`, `updater.check`, `updater.download`, `updater.install`, `updater.startup.ack` +- `updater.highlights.get`, `updater.highlights.ack` +- `social.capabilities.get` +- `discord.activity.sync`, `discord.activity.clear` +- `feedback.submit`, `bug_report.submit`, `bug_report.incident`, `analytics.track` + +### Usage & bug reports (`automaticBugReports.v2`) + +`automaticBugReports.v2` replaces `automaticBugReports.v1` and removes +`telemetry.sync` and the `optInTelemetry` capability. The core owns every +network call: reports go to `POST {base}/v1/reports` and usage events to +`POST {base}/v1/events`, where `base` is `https://opennow-reports-production.up.railway.app` unless the +`OPENNOW_REPORTS_API` environment variable names another `http(s)` origin when +the core starts. + +One setting, `automaticBugReports` (`unset`, `enabled`, `disabled`), labelled +"Usage & bug reports", gates automatic reports and usage events. `unset` means +the notice has not been answered and is treated as enabled while OpenNOW is +experimental. Loading settings migrates the legacy `errorReportingConsent`: +`denied` together with `unset` becomes `disabled`; the legacy key is otherwise +inert. `settings.set` for `automaticBugReports` accepts an optional `source` +(`signin_notice`, `first_run_sheet`, or `settings`, default `settings`). Every +`enabled` or `disabled` write queues one anonymous `consent_changed` event +(`account` is `null`) even when the new value is `disabled`. A `disabled` write +first deletes the queued events, the event spool, and pending reports; the core +does the same at startup when the setting is already `disabled`. + +#### Reports + +Every report is multipart form data with a `report` part (JSON, at most 64 KB) +and an optional `log` file part (`application/gzip`, at most 5 MB compressed) +holding the redacted diagnostics export: + +```json +{"schemaVersion":1, "automatic":true, "installId":"32 hex", + "account":{"userId":"…", "providerIdpId":"…", "providerCode":"NVIDIA", + "providerName":"NVIDIA", "alliancePartner":false, "reporter":"Zortos", + "reporterBasis":"username", "membershipTier":"ULTIMATE"}, + "trigger":{"kind":"frame_drops", "code":"sustained_frame_drops", "message":"…", + "metrics":{"droppedFrames":412, "windowSeconds":60}}, + "activity":{"currentGame":{"title":"Cyberpunk 2077", "appId":"100"}, + "recentGames":[{"title":"…", "appId":"…"}]}, + "app":{"version":"1.0.3", "os":"windows", "arch":"x86_64", "gpu":"NVIDIA RTX 4070", + "decoderBackend":"d3d11", "channel":"stable"}} +``` + +Automatic reports carry `trigger`; manual reports set `automatic:false` and carry +`manual:{title, description}` instead. `account` is `null` when no account is +signed in. `reporterBasis` is `username`, `email`, or `none`. `gpu` and +`decoderBackend` come from the latest runtime capabilities the shell sent and may +be `null`. + +The core triggers an automatic report when `session.create`, `session.poll`, +`session.claim`, `streamer.prepare`, `streamer.start`, or `catalog.library.list` +fails with a code that is not a user action or account state (for example +`cancelled`, `stale_account`, or `session_conflict` are ignored). +`session.create` also records the launched title in the diagnostics log and the +recent-games list attached to every report. + +The Qt shell reports in-process media problems with `bug_report.incident`: +`kind` is `stream_error` or `frame_drops`, `code` is a lowercase identifier, +`message` is redacted and bounded, and `metrics` copies only the numeric fields +`droppedFrames`, `windowSeconds`, `framesPerSecond`, `packetLossPercent`, +`pingMs`, and `decodeTimeMs`. The result is `{accepted:true}` or +`{accepted:false, reason}` (`disabled`, `signed_out`, `already_reported`). +Each kind and code pair is reported once per core run, at most five reports per +run, at least 30 seconds apart. Only a signed-in account triggers automatic +reports. + +Accepted automatic reports upload in the background and publish +`bug_report.changed` events with `state` `sending`, then `sent` (with +`reportId` and `issueStatus`, for example `investigating`) or `failed` (with +`message` and `queued`), plus `kind`, `code`, and the current `game` title. + +`bug_report.submit` takes `{title (8–120 characters), description (40–12,000), +includeDiagnostics}` and returns `{submitted:true, reportId, issueStatus}`. +`feedback.submit` takes `{category: bug|idea|other, message (8–4,000), +includeSystemInfo}` and sends a manual report titled `Feedback: ` +without an account; without `includeSystemInfo` it omits the game activity, GPU, +and decoder backend. Both return `{submitted:false, queued:true}` when the upload +was queued for retry. + +A network failure, or an error response whose `error.retryable` is true (or a +408, 429, or 5xx without that field), writes the report and its compressed log +to `/reports/pending/`. The core keeps at most five pending reports, no +older than seven days, and retries them oldest first after the first sign-in of +the next run, stopping at the first retryable failure. A non-retryable error +drops the report. + +#### Usage events + +`analytics.track` takes `{event, props, uiSurface?}`. `uiSurface` is `desktop` +or `console`; the core remembers it for events it emits itself. The shell may +send these events, and the core keeps only the listed properties that pass +validation (unknown events are `invalid_params`; invalid properties are dropped): + +| Event | Properties | +| --- | --- | +| `session_started` | `game_id`, `game_title`, `codec` (`h264`/`h265`/`av1`), `resolution` (`x`), `fps_target`, `hdr`, `decoder_backend`, `first_frame_ms`, `queue_wait_s` | +| `session_ended` | `game_id`, `game_title`, `duration_s`, `outcome`, `error_code`, `recoveries`, `video_drop_count`, `avg_fps`, `avg_ping_ms`, `avg_packet_loss_pct`, `decoder_errors` | +| `session_error` | `stage` (`create`, `poll`, `claim`, `prepare`, `start`, `stream`, `presentation`), `code`, `game_id` | +| `frame_drops_detected` | `dropped`, `window_s`, `game_id`, `decoder_backend` | + +`outcome` is `clean` when the user ends a session that presented video, `user_stopped` +when the user ends it before the first frame, `remote_ended` when the cloud +session ends, and `error` when the stream had failed. Codes are +`[a-z0-9_]{1,64}`; identifiers are `[A-Za-z0-9_.:-]{1,64}`; numbers are finite +and non-negative. + +`{event:"app_opened", runtimeCapabilities}` is the one exception: the shell sends +it once the native runtime handshake finishes, and the core sends `app_opened` +once per run with `gpu_vendor` and `package_kind` derived from those +capabilities and the installation. The core also emits `signed_in` +(`auth.device.complete` with `restored:false`; a restored or switched account +with `restored:true`), `game_launch_requested` (`session.create`, from its +`appId`, `title`, `store`, and `zone`), `session_error` and `library_load_failed` +for the failures that trigger automatic reports, `bug_report_sent` after a +report is accepted, and `consent_changed`. Each event gets `app_version`, `os`, +`arch`, `ui_surface`, and `channel`. + +The result is `{accepted:true}` or `{accepted:false, reason}`. Events are +queued in memory (at most 1,000; overflow drops the oldest and is logged in +diagnostics) and sent as `{installId, account:{userId, providerIdpId}|null, +events:[{event, ts, props}]}` batches of at most 100 events every 30 seconds, +at 50 queued events, and at shutdown within one second. Failed batches are +appended to `/analytics/spool.jsonl` (at most 1 MB, oldest batches +dropped first) and resent after the next successful send or at the next start. + +`diagnostics.export` optionally accepts `embeddedStream.drops` and +`lastSessionReport.drops` from the Qt session owner. Each contains the cumulative +`videoDropCount` (frames), `audioDiscardedMs` (decoded audio duration), +`audioPacketDropCount` (audio packets/PCM blocks with unknown duration), +`callbackDropCount` (Qt callbacks), and `otherQueueDropCount` (unclassified items). +The core copies only bounded, non-negative numeric counters into the export's +`shell` section; it does not export arbitrary caller-provided fields. These +counters survive embedded-runtime stop and remain separate from the core's +process-streamer snapshot and its legacy mixed-unit `queueDropCount`. + +`diagnostics.export` also optionally accepts `runtimeCapabilities` from the +in-process Qt streamer. The export's separate `nativeRuntime` section preserves +allowlisted backend and codec availability, HDR support, and bounded, redacted +failure reasons, even before a stream starts. It does not copy arbitrary runtime +fields or treat an empty process-streamer snapshot as the embedded capabilities. + +### Session resume and reconnect + +`session.active.get` without hints remains a local-only lookup. After a core +restart, Qt can reconcile an existing native stream by supplying +`{sessionId, ownerScope: {userId, providerIdpId}}` after authentication is restored. +Both hints are required together. The session ID must contain 1–256 ASCII letters, +digits, hyphens, or underscores; each owner identity must contain 1–1024 bytes and +match the authenticated selected account. An optional old `ownerScope.generation` +is ignored because generations are process-local. Caller-supplied endpoints are +ignored. + +Hinted reconciliation discovers the exact seat through authenticated, core-owned +provider routes and reads its current state with GET. It restores local control +context and ownership without RESUME/PUT, allocation, or native transport changes. +An already-owned matching seat returns its current local state. Another seat or +owner fails with `session_owner_mismatch`. Successful results use the existing +`{session, scope}` shape and publish the current generation in `scope` and +`session.ownerScope`, preserving the remote phase without adding `resumePending`. + +A missing discovery match fails with `session_discovery_failed`; authentication, +network, cancellation, and stale-account failures remain errors. Local absence is +not termination. Only an exact-seat GET reporting status 7 or HTTP 404 confirms +termination. Status 7 includes `session.termination`; HTTP 404 returns +`{session: null, termination: {source: "cloudmatch-http", httpStatus: 404, +sessionId, resumable: false}, scope}`. Qt keeps the native stream alive while +retrying reconciliation errors and resumes ordinary session polling after local +ownership is restored. +Scope-checked discovery also authorizes exact-seat cleanup if reconciliation is +cancelled before active ownership is adopted. A later account generation cannot +reuse that discovery to stop a seat. + +`session.remote.list` checks the selected endpoint and the regions advertised by +CloudMatch instead of treating the first empty regional response as authoritative. +Discovery deduplicates session IDs and uses at most four concurrent requests, a +three-second per-request timeout, a 12-second total budget (including region +discovery), and at most 32 regional endpoints. If no sessions were found but a +region failed, returned invalid data, or could not be checked within these bounds, +the request fails with `session_discovery_failed` rather than returning an empty +list. HTTP 401 remains `http_unauthorized` after at most one safe-read renewal; +HTTP 403 remains `authentication_required`. A found session +can be returned even when another region fails; an empty successful list means +all discovered regions were checked successfully. Callers must not create a new +session after a discovery failure. + +### Provider routing and account scope + +`auth.providers.list` returns `providers`, `defaultProviderIdpId`, `generation`, and +`discovery: {state, message, retryAfterMs}`. Successful discovery is fresh for +15 minutes. A failed refresh retains the last known providers and reports +`state: "degraded"`. Without a discovered list, the response includes the restored +provider, when present, and an explicit NVIDIA fallback. This fallback is not +authoritative discovery. Retries wait at least 30 seconds; HTTP 429 `Retry-After` +can extend the wait up to one hour. Qt performs at most three automatic retries. +An explicit unknown `providerIdpId` fails with `provider_unavailable` rather than +selecting another provider. Endpoint updates reconcile by exact IDP identity. + +Authenticated catalog and account reads include +`scope: {generation, providerIdpId, userId}`. Core reads capture private ServiceId +credentials and reject obsolete account/provider generations before publishing. +HTTP 401 permits one renewal and one replay of a safe read for the same owner. +HTTP 403 does not trigger renewal. Create, claim, account sync/unlink, and storage +reset do not replay mutations after an ambiguous response. + +Authenticated VPC lookup requires a successful server-info response with a +nonempty `requestStatus.serverId`. Missing or rejected metadata is an error, +not a successful `GFN-PC` library. VPC cache entries are scoped by provider route, +account, credential, generation, and proxy, with coalescing within each scope. +Library and VPC requests use the same selected proxy. The core does not silently +retry a selected proxy directly. Authenticated HTTP clients do not follow redirects. +LCARS retains its configured shared endpoint; an Alliance provider ID does not +select `GFNPartnerJWT` or invent a GraphQL hostname. + +GFN service transport failures, including server-info lookup, keep the +`network_error` code. When the HTTP client identifies the failed stage, the +message ends with `(dns)`, `(connect)`, `(tls)`, or `(timeout)`; the message +never includes the request URL or host. When an Alliance (non-NVIDIA) provider's +server-info lookup for VPC or region discovery fails at the `dns` stage, the +`network_error` message instead says that the provider display name can only be +reached from inside its home country and network, asks the user to turn off any +VPN, proxy or custom DNS, and keeps the original failure after `Details:`. Library, +favorites, store, launch inspection, subscription and region reads share this +message; the shell displays it as received. + +`settings.set` for `region` also requires the current `providerIdpId`. The core +atomically updates `region`, `regionProviderIdpId`, and the `providerRegions` map. +The metadata fields cannot be written separately. Existing unscoped preferences +remain eligible only for NVIDIA. Region overrides must occur in the current +provider's server-info list. An unavailable or incompatible override falls back +once to that provider's base without deleting the saved preference. Provider and +region bases must be HTTPS NVIDIA-grid names without userinfo or nonstandard ports. +Arbitrary partner domains require a separate evidenced trust policy. + +### Free-tier queue locations + +`queue.servers.list({})` returns `{locations, recommendedZoneId}`. Each location +contains `zoneId`, `title`, `region`, `queuePosition` (nonnegative integer), +`etaMs` (milliseconds or null), `lastUpdated` (Unix seconds), `pingMs` +(milliseconds or null), `streamingBaseUrl`, and `alternateCount` (the number of +other fresh zones folded into this location). Missing mapping entries retain +their raw zone ID as the title and a friendly continent name as the region. +Mapping failure does not prevent use of valid queue data. Mapping entries marked +`nuked: true`, malformed rows, non-NVIDIA zones, timestamps over 15 minutes old, +and timestamps more than 60 seconds in the future are excluded. No usable fresh +rows, invalid queue payloads, and HTTP failures return `queue_servers_failed`; +cancellation returns `cancelled`. + +The core requests PrintedWaste's public queue and mapping endpoints without +credentials, account identifiers, cookies, proxies, or redirects. Each fetch has +a five-second deadline and a 256 KiB streamed body limit, with at most 256 input +entries and 128 usable zones. Only strict `NP-<3–8 uppercase alphanumeric +location beginning with a letter>-<2 digits>` identifiers generate a probe or +launch URL: `https://.cloudmatchbeta.nvidiagrid.net/`. Neither +provider payload can supply an arbitrary host. Latency uses a TCP warm-up and two +TCP connection samples, each bounded to 750 ms across all resolved addresses, +with 50 ms between samples. Probes run in batches of at most 32 and check +cancellation between samples and batches. DNS resolution has a 750 ms caller +deadline, uses at most 32 resolver workers and 32 queued jobs, and returns at most +eight addresses per host. Stalled system DNS calls cannot block the caller or +spawn additional workers; expired/cancelled jobs are skipped before resolution. +`pingMs` is the rounded-up mean of successful TCP samples, excluding DNS, TLS, +and server processing. All-failed probes produce null latency, not fabricated +ping values. + +Zones sharing a title and region are grouped. Each primary and the final +recommendation prefer measured zones when available, then minimize a normalized +75% latency / 25% queue score. A score winner above 100 ms is replaced by the +lowest-latency candidate. Without measurements the shortest queue wins. Ties +are deterministic. `recommendedZoneId` always names a returned location primary; +that row sorts first, with remaining rows ordered by region and title. + +The desktop shell presents this selector only for an explicitly known `FREE` +NVIDIA membership, not for unknown membership or alliance accounts. +`hideQueueSelector` is a persisted boolean preference, default `false`, exposed +through the standard settings API and boolean normalization. It does not change +the separate region-selector preference. The public queue RPC is unauthenticated +and does not itself authorize gameplay or infer membership. + +For a fresh NVIDIA launch, the shell may pass a selected location's exact `zone` +and `streamingBaseUrl` to `session.create`. The core accepts a strict zone and +its exact derived URL for the NVIDIA IDP and NVIDIA provider code even when that +zone is absent from server-info discovery. This exception does not apply to +alliance providers, does not alter persisted region preferences, and does not +bypass account scope, catalog eligibility, or subscription checks. All other +region overrides retain the provider-discovery rules above. + +`session.create` requires `catalogAppId` as the parent LCARS identifier, +`variantId` as the selected positive GraphQL Int identifier encoded as a string, +and `appId` equal to that exact variant ID. `scope` must contain the current +`generation`, `userId`, and `providerIdpId`. The core resolves fresh catalog and +account metadata and applies the same decision as `catalog.launch.inspect` +immediately before allocation. Only `ready` admits a fresh allocation. A decision +from an earlier inspection is not a transferable permission. Missing, foreign, +or mismatched identifiers do not fall back to another variant or a title. +Existing validated seat claims remain separate and never write ownership. + +`session.create` acquires a single typed CloudMatch admission guard before provider +discovery, authentication, or route locks. A concurrent create returns +`session_update_busy` immediately instead of waiting to allocate after the first +request fails or is cancelled. Pre-allocation failure releases the guard; after +an allocation exists, the existing receipt and exact-seat cleanup state continue +to prevent another allocation. + +CloudMatch requests serialize with account changes. An active seat belongs to +its exact session ID, user ID, and provider IDP, independently of the account +generation. Returning from account A to B to A, clearing caches, or signing in +again as A does not revoke A's ownership. New owned-seat operations publish the +current generation and retain the core-captured control and media endpoints. +An active seat retains its original account for polling and exact-seat cleanup +after a switch or logout. +If that retained credential expires, `session_owner_authentication_required` +asks the user to return to the original account without restarting media. +New-account credentials never authorize requests to the retained seat. When the +selected account matches that owner, explicit cleanup renews expired ServiceId +credentials before issuing DELETE. A rejected DELETE is not automatically replayed. +Session results include `ownerScope` inside `session`; Qt distinguishes updates +for its existing native session from unrelated old-account responses. Ordinary +asynchronous results, discovered seats, and catalog results remain generation +fenced. A new allocation's receipt retains its original generation even when +the active seat is republished under a newer generation; an obsolete receipt +still triggers exact-seat compensation. + +Qt accepts authoritative terminal status 7 or HTTP 404 for the exact displayed +session and its user/provider identity even when the result's generation is old. +This exception applies before ordinary response and `session.changed` scope +filters. It does not admit stale nonterminal updates, another account/provider, +another session ID, or a resumable or non-authoritative termination. Duplicate +terminal delivery cannot end a replacement seat. + +`streamer.prepare` uses the caller's session ID to resolve the current core-owned +active seat. Caller-supplied connection endpoints are ignored. The response also +returns the owned `session` with its current `ownerScope`, which Qt retains for +later session events. Foreign ownership or an unknown seat fails with +`session_owner_mismatch`, a non-ready seat with +`session_not_ready`, and missing RTSPS endpoints with `session_endpoint_missing`. +The existing negotiated profile checks still run. Preparation does not expose +OAuth tokens to Qt or alter the native `/rtsp` websocket session-ID authentication. +These fields are additive to protocol 5; allocation receipts and exact-seat +compensation remain unchanged. + +`session.create` reports an existing-session limit as `session_conflict`, including +CloudMatch status `11`, `SESSION_LIMIT` descriptions, and unified error +`4AF1201E`, including structured session-limit responses with HTTP 403. An +unrecognized HTTP 403 remains `authentication_required`; HTTP 401 is +`http_unauthorized`. Session creation is never automatically replayed. The +rejected request never becomes an active local session. The shell +should call `session.remote.list` once and offer to resume or end the existing +session instead of displaying the raw vendor error or retrying creation. When the +create response supplies usable `otherUserSessions` or `session` details, the core +normalizes them to the ordinary discovery descriptors and hands them to the next +list request without another network lookup. This handoff is account-scoped, +consumed once, and expires after 30 seconds; it is not a persistent discovery +fallback. The code/message error envelope and protocol version are unchanged. + +`session.claim` first queries the trusted seat host returned by discovery, including +conflict-response handoffs whose create region may not own the existing session. +If that host fails for a reason other than authentication, it falls back to the regional +endpoint. The shell refreshes the chosen session's descriptor before retrying a failed +claim, without substituting another game or silently creating a new session. + +`session.claim` discovers the session's actual control server, sends the minimal +`action: 2, data: "RESUME"` request for a ready, streaming, or paused seat, and returns a session +with `resumePending: true` and `phase: "resuming"`. It preserves the stable device +identity and existing launch mode; it does not renegotiate codec, resolution, FPS, +or bitrate. An initializing (`1`) or resuming (`6`) seat is polled without repeating +the mutation. Discovery includes paused (`4`/`5`) and resuming seats so they remain +available to resume and reconnect. Finished or unknown states reject the claim. + +The PUT acknowledgement is not stream readiness. Call `session.poll` until a fresh +GET has a successful CloudMatch request status (`statusCode: 1`), session status +`2` or `3`, and nonempty native RTSPS endpoints. Only then does the core clear +`resumePending` and expose the ready/streaming phase. Transient status `6` continues +to report `resuming`; finished or unknown poll states clear `resumePending` and +report `failed`. A RESUME response of `SESSION_NOT_PAUSED` (`statusCode: 34`) +also proceeds to polling, including on an HTTP error response. Authentication +failures and other transport/API failures remain typed RPC errors. + +Qt polls every 1.5 seconds with only one request outstanding, bounded by 60 polls +and a 90-second deadline checked between requests. Native connection recovery first +stops the old media transport, discovers the same session ID through +`session.remote.list`, then claims/polls it and calls `streamer.prepare` for fresh +connection context. It never creates a replacement cloud game or claims a different +session. During recovery, `session.remote.list` accepts the active `sessionId` so the +core can use its remembered regional service rather than the native server IP. +Failed recovery attempts back off up to eight seconds and stop after eight +attempts; only a presented first frame resets this budget. Ending the session cancels +recovery. A native stop stalled for 30 seconds reports an error without launching +another transport over the still-owned resources. + +An exhausted video SETUP negotiation (`missing-video-peer`) or an explicitly +unsupported legacy transport (`nvst-legacy-transport-unsupported`) stops automatic +recovery and preserves the original error and owned seat for an explicit retry or +stop. These compatibility failures do not trigger another claim of the same seat. + +For the embedded Qt client, `session.create` also accepts an optional numeric `maxEntitledFps`: +the highest frame rate the signed-in membership entitles at the requested resolution, or `0` +or absent when that is not confirmed. Qt derives it from the normalized `entitledResolutions` +entries it already displays; the core never invents it. The requested frame rate is bounded by +the documented resolution ceiling (1920x1080 and 1920x1200 top out at 360 FPS, every other +resolution at 240 FPS) and, above 240 FPS, by both a hardware decoder for the selected codec +confirmed in `runtimeCapabilities` and an entitlement limit that covers the requested rate. +An absent or software-only capability probe and an absent or lower entitlement limit both bound +the request to the unconditional 240 FPS ceiling, so the saved preference cannot request the +conditional tier without affirmative evidence. This is a necessary condition, not a throughput +qualification: the client does not measure decoder throughput, and CloudMatch remains the +authority on the finalized profile. +Rates snap to 30, 60, 90, 120, 144, 165, 240, or 360 FPS before allocation. +Embedded codec limits below can reduce the request further. + +For the embedded Qt client, `session.create` and `streamer.prepare` accept an optional +`runtimeCapabilities` object copied from the in-process streamer's protocol-7 `hello` response. +The core filters its available `videoBackends` by the persisted `nativeVideoBackend` preference +and resolves codec `auto` to AV1, HEVC, then H.264 for both SDR and HDR (subject to requested color mode) before +CloudMatch allocation. This resolution is session-local: the saved preference stays `auto`. +Each codec entry may include `colorQualities`, an authoritative array of supported settings +values such as `8bit_420` and `10bit_420`. An empty array means no supported color modes. +The embedded Linux runtime publishes this field for every codec: Vulkan values come from the +attached device's actual decode profiles, while non-Vulkan Linux paths report only 8-bit 4:2:0. +The core filters codecs by the requested color mode before both Auto and manual selection, so +an HEVC decoder supporting Main but not Main10 cannot allocate a 10-bit session, and the shared +Vulkan path cannot allocate 4:4:4. Missing or malformed advanced-color profile information on +Linux/Vulkan fails closed; other platforms retain their existing behavior when this optional +field is absent. The selected codec and color remain session-local and never downgrade the +user's requested color mode. +Manual unavailable codecs/backends return `streamer_codec_unavailable` or +`streamer_backend_unavailable`, respectively. `streamer.prepare` also validates the negotiated +codec on resume; it never changes the codec of an already allocated stream. Older callers without +this optional object retain the external-streamer probe path. The additive fields do not change +the JSON protocol version or native FFI ABI. + +### Windows graphics preference + +`settings.get`, `settings.set`, and `settings.reset` expose `windowsGpuDeviceId`. +The default empty string selects Automatic. Explicit values are opaque device +identities, limited to 1024 UTF-8 bytes with no NUL characters. Invalid persisted +values normalize to Automatic; invalid writes are rejected without changing the +saved preference. + +Before creating the Qt graphics device, the shell may run: + +```text +opennow-core --graphics-preferences +``` + +This mode emits one JSON line, then exits without initializing account, network, +catalog, telemetry, or streaming services: + +```json +{"version":1,"windowsGpuDeviceId":""} +``` + +It uses the normal data-directory resolution, including `--data-dir`, +`OPENNOW_DATA_DIR`, and legacy-directory discovery. The read is limited to 1 MiB +and never saves migrations, creates directories, or renames corrupt settings. +Missing, corrupt, or oversized input selects Automatic. Qt bounds the subprocess +and its output and also uses Automatic if bootstrap fails. + +Qt resolves the saved identity to a current-boot Windows adapter LUID. The same +LUID selects Qt's D3D11 adapter and the native runtime's capability probes; LUIDs +are not persisted. A missing saved GPU falls back for that launch without +erasing the preference. The settings selector appears only with at least two +detected hardware adapters, excluding software adapters. It is on the Stream +settings page. Saving a different GPU affects the next application launch, not +the active graphics device or session. + +Adapters are enumerated high-performance first. Before that launch, each adapter +is indexed for H.264, HEVC, and AV1 hardware decode profiles at 1920×1080 or +1280×720. Automatic uses the first adapter in that order that exposes one of +those profiles. A hybrid laptop whose discrete GPU has no decoder, such as a +GeForce MX110 beside Intel HD Graphics 620, therefore uses the integrated GPU +instead of failing the session. An explicit saved GPU is still used even when +its index is empty. Each settings choice lists the codecs that index found. + +The embedded streamer's protocol-7 `hello` may include `graphicsAdapters`. The +field is omitted when no adapters were indexed. Each entry has `name`, `active`, +`codecs` (`h264`, `h265`, `av1`), `h265Main10`, and an optional `reason`. It does +not include the adapter LUID. `active` marks the adapter selected for that +process. When no hardware backend is available and another indexed adapter can +decode, the session error names that GPU and points to Settings → Stream → +Graphics processor. Diagnostics copy the same allowlisted fields. + +### Recording and replay capture + +The Qt/native recorder and replay exporter preserve the negotiated source video and +Opus game audio in Matroska without decoding or re-encoding. Recording resolution, +frame rate and bitrate follow the stream; the retained legacy `recordingResolution`, +`recordingFps` and `recordingBitrateMbps` preferences do not configure native capture. + +`settings.get` / `settings.set` expose `replayBufferEnabled` (default `false`), +`replayBufferSeconds` (default 30, clamped to 15–120), `replayBufferMemoryMiB` +(default 256, clamped to 64–512), and `shortcutSaveClip` (default `Ctrl+F12`). +`streamer.prepare` includes these settings in the session context and maps +`shortcutSaveClip` to `shortcuts.saveClip`. Enabling replay and changing its limits +apply to the next native session. Disabling it sends `replay-stop` immediately, +clears buffered media and cancels an in-progress clip export. + +These commands use the embedded streamer's protocol-7 JSON payload without +changing the C ABI: + +- Manual `recording-start` uses its command `id` as the recording identity. A + `recording-state` terminal event carries that identity as `requestId` and a + `state` of `saved` or `failed`. A saved result includes `path`, `videoPackets`, + `audioPackets`, and `completion`. Normal completion is `{"kind":"complete"}`. + A preserved valid prefix is `{"kind":"cut","reason":"discontinuity"}`, + `{"kind":"cut","reason":"queue-overflow"}`, or + `{"kind":"cut","reason":"interrupted"}`. A cut stops manual recording and + produces a visible early-save notice; it does not start another file or change + replay capture. +- `recording-stopped` repeats the terminal result and its originating `requestId`; + its `id` still identifies the stop command. Qt applies a correlated terminal + result once, whether the event or stop response arrives first. A repeated stop + returns `recording-not-active` without replacing an earlier cut or failure. + Packet-write, trailer, publication, and no-valid-keyframe failures remain + failures, not saved prefixes. +- Completion metadata and event correlation are additive protocol-7 fields. Qt + ignores uncorrelated terminal events and stale recording identities. A missing + completion field means ordinary saved output only on an authoritative, + correlated stop response. Historical producers without correlated terminal + events do not provide the new unsolicited-completion behavior. These fields do + not guarantee callback delivery or preservation after a process crash. +- The `start` response includes `replayEnabled` for the actual session. +- `clip-save` accepts `id` and an absolute `.mkv` `outputPath` allocated by + `media.recording.target`. It returns `clip-saving` promptly or a typed error if + replay is disabled, not yet decodable, or another export is still running. +- A `clip-state` event reports `state` (`saved` or `failed`), `requestId`, `path` + and `message`. The shell correlates `requestId` with the outstanding export and + ignores completion from a previous session. Cancelled exports do not publish a + completed file or a stale success event. +- `replay-stop` returns `replay-stopped`. A new session is required to enable + buffering again. + +Replay retains encoded packets with bounded memory, frame count and duration. +Clips start on a retained video keyframe, so their length may be shorter than the +requested duration. Export transfers the retained buffer to a single worker; +buffering rebuilds from a subsequent keyframe within the remaining memory budget. +There is no assumed periodic-keyframe guarantee: when a whole GOP exceeds the +configured limit, replay returns `replay-not-ready` until a new source keyframe. +Capture does not issue periodic keyframe requests that would change stream traffic. +Discontinuity or producer contention clears replay history rather than blocking +playback or publishing a clip with missing references. There is no additional +video encoder or GPU readback. Packet bookkeeping, container muxing and disk I/O +still consume CPU and bandwidth; zero CPU usage or zero performance impact is not +a supported guarantee. + +### HDR session contract + +`settings.enableHdr` is a persisted boolean with default `false`; HDR requires explicit user +opt-in. Qt adds `nativeHdrSupported: boolean` to `params.runtimeCapabilities` on each +`session.create` and `streamer.prepare` request. It must describe the actual stream window's +current HDR output, including the active monitor, compositor/OS HDR state, and presentation +surface support, not merely a GPU or decoder capability. Missing, false, or malformed values +deny HDR. This value is transient: `settings.set` rejects `nativeHdrSupported`, the settings +loader discards legacy copies, and the core never saves runtime capability results. + +With HDR enabled, an HDR session needs an available hardware HEVC or AV1 decoder whose +`colorQualities` explicitly includes the requested ten-bit profile. The optional per-codec +`hdrSupported: true` permits `10bit_420` when `colorQualities` is absent, but does not establish +4:4:4 support. The optional per-codec `hdrColorQualities` array lists verified HDR profiles. +When present, it must contain the requested profile; empty or malformed values deny HDR. +HDR 4:4:4 requires an explicit `10bit_444` entry in both color-quality arrays. +Missing `hdrColorQualities` retains the existing HDR 4:2:0 checks only. Windows reports +explicit SDR color-quality lists and HDR support from actual decoded profile fixtures and GPU conversion; +it no longer infers advanced formats from an eight-bit codec probe. macOS HEVC HDR and +ten-bit 4:4:4 similarly require hardware-required fixture decode and Metal conversion. +Linux uses exact attached-device profiles for Vulkan Video. An explicit false or +malformed `hdrSupported` denies HDR even if 10-bit profiles exist. A true value never +overrides an explicit empty or incompatible `colorQualities` array. Neither setting +changes SDR capability filtering. HDR constrains the session-local color quality to ten +bits while preserving the selected chroma format and saved SDR color preference. HEVC +supports `10bit_420` and `10bit_444`; AV1 remains limited to `10bit_420`. + +Unsupported color and HDR requests degrade before a seat is allocated instead of failing. +The core resolves them in this order and returns the effective `codec`, `colorQuality`, +`enableHdr`, `resolution`, and `fps` that +CloudMatch and the streamer use: + +1. Missing HDR output (`nativeHdrSupported` not `true`) clears HDR. +2. Software decoding resolves to 8-bit 4:2:0 SDR. +3. A 4:4:4 request selects HEVC, replacing Auto or any explicit codec, when the selected + backend decodes HEVC at that 4:4:4 profile (or HEVC HDR 10-bit 4:4:4 with HDR). + Otherwise the chroma becomes 4:2:0 and the bit depth is kept. +4. HDR that the selected codec cannot decode is cleared. Explicit H.264 is SDR-only. + Auto prefers AV1, then HEVC, then H.264 for both SDR and HDR, independent of bitrate + and platform. When neither AV1 nor HEVC decodes HDR, Auto continues as SDR. +5. Ten-bit SDR that the selected codec, or every Auto candidate, cannot decode falls back + to 8-bit 4:2:0 BT.709. +6. H.264 caps the frame rate at 120 FPS and the image area below 4,953,600 pixels, + including explicit requests and Auto with only H.264 decode. Resolution reduction + selects the closest aspect ratio among supported settings modes that fit the limits, + then the largest image at that ratio. For example, 3440x1440 becomes 2560x1080. +7. AV1 remains eligible for Auto at up to 120 FPS and 5120x2880 pixels. No runtime probe + currently confirms higher AV1 frame rates, so explicit AV1 above 120 FPS also caps at 120. +8. Each codec may report `maxDecodeDimensions:{width,height}` in `runtimeCapabilities`. + Both values are positive uint32 dimensions. Windows reports the decoder's enforced + 4096x2304 envelope; the attached Linux Vulkan device reports the conservative intersection + of its advertised color-profile limits; the opt-in MediaCodec bridge reports 2560x1440. + Other producers omit unknown limits rather than estimating them. Core reduces oversized + modes before allocation using the same closest-aspect rule. It rejects malformed limits + or a request with no supported mode within the limits. + +`session.create` records each applied fallback as a `streamer`/`stream_profile_fallback` +diagnostics event. An explicitly selected codec that the selected backend cannot decode at +all, an unavailable backend, or a backend that cannot decode 8-bit 4:2:0 with any candidate +still fails with `streamer_codec_unavailable` or `streamer_backend_unavailable`. The core +never substitutes another codec for an unavailable explicit codec except for the HEVC 4:4:4 +rule above. `streamer.prepare` repeats the same resolution against the accepted session; when +the accepted codec, color quality, or HDR mode differs from what the current decoder can +present, it fails with `streamer_color_unavailable` instead of attaching a mismatched decoder. +Accepted native resolution and FPS override current saved preferences on attachment. +An accepted mode requiring resolution or FPS normalization fails with +`streamer_mode_unavailable`; preflight never changes an already allocated stream's mode. +Without HDR, the saved color preference is used, subject to exact hardware profiles and the +GFN codec restrictions below. Callers without embedded runtime +capabilities cannot request HDR through the external-streamer probe path. + +CloudMatch receives `sessionRequestData.sdrHdrMode=1` and monitor `sdrHdrMode=1` +only for this validated HDR request. `requestedStreamingFeatures.trueHdr=false` remains +separate because TrueHDR is the server's AI SDR-to-HDR filter, not native HDR. CloudMatch +uses bit-depth/chroma enums `1/0` for 10-bit 4:2:0 and `1/1` for 10-bit 4:4:4. +For HDR, monitor `displayData` carries validated output luminance +when the current output reports it. `desiredContentMaxLuminance` is the peak in nits; +`desiredContentMinLuminance` is the minimum in 0.0001-nit units, rounded after +multiplication by 10000. Wayland color-management supplies only this luminance pair. +`desiredContentMaxFrameAverageLuminance` is omitted for a pair-only snapshot because +Wayland exposes no comparable sustained full-frame value. When the runtime supplies +a complete validated optional group, the core also writes its full-frame nits into +that field and writes the red, green, blue, and white chromaticities as rounded +`xy * 50000` integers in `displayPrimaryX0/Y0`, `displayPrimaryX1/Y1`, +`displayPrimaryX2/Y2`, and `displayWhitePointX/Y`, respectively. The official PC +client's Bifrost serializer confirms the field order and scales; the mapping from +the display's maximum full-frame luminance to the frame-average field is inferred +from native struct offsets, not an observed HDR session. + +Without a validated output snapshot, HDR requests keep the fixed requested-content +defaults of maximum luminance 1000 nits, minimum luminance 0, and maximum frame-average +luminance 400 nits, matching the Mac native session payload. Those defaults are requested +content characteristics rather than measurements of the physical display and are not +presented as calibration. SDR sends `displayData:null`. HDR does not invent display +primaries or a white point; those fields require validated chromaticities from the +current runtime output. + +The validated snapshot travels from Qt in `runtimeCapabilities.nativeHdrDisplay` as +`minimumNits` and `maximumNits` in cd/m², omitting either value the output does not +report. Optionally it also carries `maximumFullFrameNits` and normalized floating-point +`redX/redY`, `greenX/greenY`, `blueX/blueY`, and `whiteX/whiteY`. The core validates +the pair again and drops it when the bounds are not finite, negative, above 10000 cd/m², +or not strictly increasing. It accepts the optional group only when every coordinate +is finite, within [0, 1], has a valid xy sum and positive y, the primaries form a +nondegenerate triangle, and `minimumNits < maximumFullFrameNits <= maximumNits`. +Missing or invalid optional fields discard the entire group but retain a valid pair. +The snapshot is transient: +`settings.set` rejects it, the settings loader discards persisted copies, and the core +never saves runtime capability results. + +Stale snapshots are discarded rather than reused. Qt injects the snapshot only while the +current output reports a validated range and removes any caller-supplied or previously +injected value when it does not; the core strips any earlier snapshot from the resolved +settings before carrying the current capability, so a display change that invalidates the +output falls back to the requested-content defaults instead of replaying the previous +display's luminance. Claim and resume requests do not carry monitor settings at all, so +they cannot replay one either. + +The normalized server response carries `negotiatedStreamProfile.enableHdr: boolean`. It +comes from the returned session's `sdrHdrMode`, then the returned monitor's mode, then the +returned session-request mode; missing or unsupported modes mean SDR. An explicit server +SDR response wins over saved HDR intent. `trueHdr` is not used to infer accepted dynamic +range. Resume preserves that returned mode rather than renegotiating from current settings. +The claim request intentionally omits monitor settings and requested streaming features, so +its only copied dynamic-range field is the accepted session `sdrHdrMode`. Fresh creation +sends the full request in its POST, with the requested session mode, monitor mode, +requested-content luminance, and `trueHdr`; only an existing-session claim uses PUT RESUME. +Attachment revalidates the session's HDR codec/color profile and current window output, so +moving to an SDR display cannot silently resume an HDR stream as SDR. + +Fresh creation also applies the launched store version's catalog HDR support. When +`enableHdr` is on, the window output reports native HDR, and the exact variant that +`catalog.launch.inspect` validated for this launch has a `features` array without `hdr`, +the core requests that one session as SDR before resolving the embedded profile: the +request's `enableHdr` is false, so `sdrHdrMode` is `0`, `displayData` is null, and +`trueHdr` stays false. Codec and color selection then run as for any SDR request. The saved +`enableHdr` preference is not changed. A variant whose `features` is null (the catalog +returned no feature list), a Steam store launch, and every case where HDR was not effectively +requested keep the request unchanged; missing metadata never downgrades HDR. The normalized +session carries `hdrDowngrade`: `"game-unsupported"` when this core downgraded the request, +otherwise null. Polls, resume, and reconnects preserve it only for the same session ID; +sessions this core did not create report null. Shells use it to explain the SDR stream and +must not repeat the catalog lookup. The session also carries `catalogAppId`, the parent catalog +game ID of the launch (`appId` stays the launched store version's CloudMatch ID), with the same +same-session preservation; it is null for sessions this core did not create. Shells key +once-per-game state on it, so every store version of a game shares one identity. + +The normalized session also carries `keyboardLayout` when this core created or resumed +the session. It records the exact layout sent in the CloudMatch request, not the current +saved preference. Polls and ad updates preserve it only for the same session ID; a successful +resume replaces it with the newly requested layout. Qt uses this value for physical key +mapping, so changing settings during a stream does not change input before the remote +layout changes. An unclaimed session whose layout is unknown omits this field. + +Color negotiation overlays each returned `finalizedStreamingFeatures` field on the server's +returned `sessionRequestData.requestedStreamingFeatures`. An empty or partial finalized object +must not erase the echoed codec, bit depth, or chroma. Explicit finalized values, including +invalid values, take precedence; missing values never come from current saved preferences. +CloudMatch chroma enums are `0` for 4:2:0 and `1` for 4:4:4; NVST chroma-format IDs `2` and +`3` are not accepted as CloudMatch 4:4:4 values. + +When present, `session.negotiatedStreamProfile.codec` takes precedence over the numeric +feature-map codec. H.264/AVC and H.265/HEVC names normalize to `H264` and `H265`; +`AV1` remains unchanged. An explicit null or unsupported codec stays unknown rather +than falling back to a requested codec. The request itself names no codec: like the +official client, codec selection stays client-side and reaches the seat in the RTSP +ANNOUNCE (`x-nv-vqos` bit-stream format), so a negotiated codec is only ever +server-reported (echo, finalized features, or a direct profile). When the server omits +every codec field, downstream stages resolve the codec from current saved preferences +instead. Polling, direct-server +responses, claims, and ad updates preserve that evidence only for the same session ID. +`codecSource` distinguishes `request`, `server`, and `unreported`; a reported codec supersedes +the request and remains authoritative in later partial responses. Unknown discovered sessions +never borrow a codec from current saved preferences. Preparation failures log bounded +codec/color evidence and a redacted reason, never the full session or credentials. + +Embedded session preflight rejects H.264 with advanced color and AV1 with 4:4:4 before +allocation, even if a decoder capability lists those formats. These combinations are not +requested by the supported GFN wire policy. Auto selects HEVC for 4:4:4 rather than silently +reducing chroma; an explicit incompatible codec remains an error. + +The native context preserves the resolved profile and codec provenance. Its accepted +`enableHdr` initializes the native HDR mode (missing means false). Invalid accepted HDR +profiles are rejected before stream startup. NVST ANNOUNCE states color explicitly on +every session using literal bit depth `8` or `10` with `chroma_format_idc` +(`chromaFormat=1` for 4:2:0, `3` for 4:4:4); `dynamicRangeMode=1` is sent for HDR +only, and SDR omits the line. This matches the vendor capture of a `10bit_420` +session (`bitDepth:10 chromaFormat:1`) and the working third-party reference +clients: a lone `bitDepth` line is never sent, because seats cannot initialize +an encoder from an incomplete color spec. The internal 0/1 chroma value from the +client's app-to-NVST conversion is mapped back to idc at SDP emission and never +reaches the wire. CloudMatch keeps its own bit-depth enum (`0` or `1`) with +chroma `0`/`1`. + +The ANNOUNCE also carries the encoder identity the seat reads before +initializing (`maxCodecProfile`/`maxCodecLevel` and the `maxH264` pair, profile +3 / level 61, as captured). One resolved configuration feeds the wire values, +the media runtime, and the decoder, so a server color choice cannot leave the +decoder expecting a different format. The stored CloudMatch profile and user +preferences are not rewritten. Full SDP must not be logged because it contains +ICE credentials and encryption material. + +After an update check, `updater.highlights.get` returns the latest published +release notes for the selected channel, even when that release is equal to or +older than the installed app. Its `version` and `title` identify that published +release, independently of `updater.state.get.availableVersion`. Historical notes +do not emit `updater.highlights.show` or enable downloading a downgrade. Missing +release bodies and empty channels return an explanatory note instead of the +pre-check placeholder. + +Updater preferences are independent. `autoCheckForUpdates` defaults to `true`; +`autoDownloadUpdates` defaults to `false` and requires an explicit opt-in. Neither +preference authorizes installation or application shutdown. New profiles use the +nightly update channel when the embedded application version has a `nightly` +prerelease identifier; existing persisted channel choices are preserved. + +`updater.install` requires the boolean parameter `confirmed: true`. The core +serializes update preparation against session creation, claiming, polling, and +streamer startup. It rejects installation while a CloudMatch session exists or +the streamer is starting, negotiating, streaming, or recovering. The shell must +also check its local native-runtime state before requesting installation. + +`updater.state.get` is authoritative after an error or request timeout. The core +emits `updater.changed` after update operations even if the original request was +cancelled, because cancellation does not undo an installation already prepared +by the native helper. + +While `updater.download` runs, the updater state carries the additive field +`downloadProgress: {"receivedBytes": n, "totalBytes": n}`, where `totalBytes` is +the signed asset size. It is `null` in every other state. The core emits +`updater.changed` when the download starts and each time the received bytes cross +another whole percent of the signed size, so a download produces at most about +101 progress events before its response. Progress is advisory: hash and size +verification still decide whether the download succeeds. Clients must not synthesize `canDownload`, `canInstall`, or +`canCheck` from an error message. A failed check or a later release check does not +discard an already verified download. + +Inside Flatpak, the updater reports `status: "unsupported"` and +`updateSource: "flatpak"`. `canCheck`, `canDownload`, `canInstall`, and +`exitRequired` are false. `updater.check` returns that state without contacting +GitHub. Download and install requests fail with the package-manager instruction +in `message`. The core does not recover native update transactions inside the +sandbox, and the native update helper rejects execution there. Flatpak owns +package replacement and updates. + +The additive updater state fields `exitRequired` and `installVersion` describe +the prepared installation, separately from `availableVersion` and +`downloadedVersion`. The shell may quit for an update only after a confirmed +install request in that same shell process, with authoritative +`status: "awaiting-exit"` and `exitRequired: true`, and while the local session +remains inactive. `preparing`, `applying`, and `restarting` are not permission to +quit. Terminal outcomes are `succeeded`, `rolled-back`, and `failed`; +`managed-pending` and `reboot-required` require native package-manager or operating +system completion. Spawning an installer or the replacement application never +counts as success. + +On a later launch, the core reconciles these managed outcomes against the native +package registration and its running version. A known live installer keeps +`managed-pending` active. A completed installer resolves to `succeeded` or +`failed`, and the core clears the persisted active transaction. Terminal helper +outcomes (`succeeded`, `rolled-back`, `failed`) are likewise reported for that +launch only; the core clears the persisted transaction after the first +reconciliation so the same failure dialog does not reappear on every restart. +An MSI reboot +warning remains until Windows' per-boot sequence number changes. Sessions +remain available, but another update must wait for the required reboot so it +cannot overlap pending Windows file replacements. Reopening the app before +reboot does not count as successful installation. + +If an installer process cannot be identified, the core keeps the transaction +pending until a recorded boot change proves that the previous installer has +stopped. Legacy transactions without a boot marker establish a baseline and may +require one additional restart rather than guessing that installation finished. + +Windows portable replacement requires a volume with persistent ACL support; +FAT/exFAT installations are refused before shutdown because private staging +cannot be enforced there. Preserved portable profile data retains its ownership +and effective access permissions. Preparation also fails before shutdown if the +replacement overlaps user data or exceeds the bounded copy limits. MSI packages +use Windows Installer registration and preserve the registered installation root; +they are not treated as portable directories. + +`updater.startup.ack` accepts no parameters and returns `{ "acknowledged": true }` +only for a valid helper-launched update attempt. A normal launch returns +`acknowledged: false`. The shell calls it after its UI and core connection are +ready. The core validates the prepared version, per-attempt nonce, and running +Qt application identity before acknowledging startup to the waiting helper. +These changes are additive within protocol version 1; they do not change the +native streamer ABI. + +`updater.highlights.show` announces unread release notes, not a navigation +command. The shell keeps the current stream and its video item alive, defers the +announcement during sessions, and acknowledges the notes only when the user +opens them. + +Setting `themePack` applies its default appearance (`light` for Bone/Cobalt, `dark` +for the other built-in packs) and clears `themeAccentOverride` in the same save. +Setting `appAccentColor` enables `themeAccentOverride` in the same save. The +`settings.set` response and `settings.changed` event include these coupled values +in `changes`; clients can still override appearance or restore the pack accent +by setting `appTheme` or `themeAccentOverride` independently. + +`settings.shortcuts.update` writes stream shortcut bindings as one transaction: +`{"bindings":{"shortcutToggleStats":"Ctrl+F11","shortcutScreenshot":""}}`. Only the +existing `shortcut*` setting keys are accepted, each value is a string of at most 80 +bytes (empty means unbound), and `Ctrl+G` and `Shift+F3` stay reserved. After the +change is merged, no two non-empty bindings may share a chord (case and modifier +order are ignored). Any rejection leaves every binding unchanged. The core persists +the whole map in one save and rolls it back in memory if the save fails. The response is +`{"bindings":{...applied}}`; the `settings.changed` event names the first applied +key and carries the full map in `changes`. Qt uses this for moving a chord from one +command to another, clearing, resetting one binding, and resetting all bindings. + +Each successful settings write publishes `settings.changed` before its own +response. A client that starts its next per-key write from that response has +already consumed the previous event. Other response/event pairs, including +`settings.reset`, retain their existing response-first order. + +Qt delivers `settings.changed` before the corresponding response rather than deferring +it through the general event queue. The settings owner serializes writes per key, keeps +ordinary controls optimistic, and retains a confirmed snapshot for failure rollback. +Later edits use the latest intended value; a failed queued tail restores the last confirmed +value and its coupled changes. Confirmation-sensitive settings and collection callbacks +retain their existing behavior. + +Settings writes use a temporary file plus recoverable backup and normalize +compatibility-sensitive values. `audioOutputDevice` is an opaque native output identifier +(at most 1024 UTF-8 bytes, without NUL characters); an empty string follows the +system default. It is persisted and passed unchanged in the prepared stream +context for the next session. A missing fixed output fails playback startup +instead of falling back to another device. Setting `launchInConsoleMode=false` atomically +sets `switchToConsoleOnPad=false` too, so a manual desktop choice survives restart. +That response and `settings.changed` event additionally contain +`"changes":{"switchToConsoleOnPad":false}`; consumers apply these coupled values +before the primary key. Automatic console switching defaults off. Existing +pre-opt-in settings receive a one-time reset of automatic switching only; +explicit subsequent opt-ins and the independent startup preference are preserved. +Existing microphone device selections are cleared only when explicitly selecting Open +microphone; that write likewise reports `"changes":{"microphoneDeviceId":""}` so the +shell follows the system-default capture selection. Setting `colorQuality` repairs an +explicitly saved `codec` (or `fallbackCodec`) the new color mode cannot use toward +`auto` in the same save — H.264 is 8-bit 4:2:0 only, AV1 is 4:2:0 only — and reports +the repair in `changes`. Explicit `codec`/`fallbackCodec` selections the saved color +cannot use are rejected as `invalid_setting` instead; unknown spellings still clamp +to Auto. + +`onboardingCompleted` is a persisted boolean, defaulting to `false` for a new +profile or an unreadable or malformed settings file. A valid existing settings +JSON object without the key migrates to `true` and is saved during loading, so +upgrades do not trigger first-run setup. Explicit `false` and `true` values survive +reloads and unrelated writes, including startup preferences and window geometry. +Non-boolean values normalize to `false` under the standard settings type rules. +`settings.reset` preserves the current completion value while resetting preferences; +it does not replay onboarding for an existing user. Failed saves leave the previous +in-memory value and persisted settings unchanged. + +`gameCollections` defaults to `[]`. `settings.set` replaces the complete ordered +array with at most 100 objects of the form +`{"id":"collection-id","name":"Collection name","gameIds":["game-id"]}`. +Collection IDs are caller-owned stable strings, unique across the array, and +must not be regenerated when renaming a collection. IDs are preserved verbatim, +must contain a non-whitespace character, and are limited to 128 Unicode characters. +Names are trimmed and must contain 1–80 Unicode characters after trimming; names +need not be unique. Each `gameIds` array contains at most 10,000 unique strings +with the same nonempty/128-character ID constraints. Game IDs may appear in +multiple collections, and empty collections are allowed. Array order is preserved. +Malformed types, missing or extra fields, duplicate IDs, or exceeded limits return +an error without modifying the in-memory settings or persisted file. Save failures +also restore the previous in-memory settings, including when resetting settings. +Reloads trim valid persisted names and preserve collections across unrelated writes; +invalid persisted collections cause an `InvalidData` load error before any write, +rather than silently dropping collections or truncating identifiers. A successful +`settings.reset` clears collections along with other preferences. + +`controllerLeftStickDeadzone`, `controllerRightStickDeadzone` (integers 0–50, +default 5) and `controllerVibrationIntensity` (integer 0–100, default 100) stay the +global controller tuning and the default for every pad. `controllerProfiles` +defaults to `{}` and holds per-pad overrides keyed by pad identity. Qt builds the +key as `serial:::` (four-digit lowercase hex vendor and +product IDs) when SDL reports a serial number, and as `guid:` +otherwise. Two identical pads without a serial therefore share one key and one +profile. Each value has the form +`{"name":"DualSense","leftStickDeadzone":8,"rightStickDeadzone":5,"vibrationIntensity":70,"lastSeen":1760440000000}`. +Every tuning field is optional; a missing field means the global setting applies. +`settings.set` replaces the whole map and normalizes it rather than rejecting it: +keys are trimmed and must hold 1–160 characters; tuning fields that are not +integers are dropped and integers are clamped to the global ranges; `name` is +trimmed to at most 80 characters and dropped when empty; `lastSeen` (Unix +milliseconds) defaults to 0. Entries that are not objects or keep no tuning field +are dropped, so deleting a pad's fields resets it to the globals. At most 32 +profiles are kept, preferring the highest `lastSeen`. Non-object values normalize +to `{}`. Settings saved before this key existed load with `{}`, and +`settings.reset` clears it. These values apply live in Qt before input reaches the +streamer; they never require a new session. + +Provider discovery falls back to NVIDIA's +default service when discovery is unavailable. Device-login tokens are stored +through the OS credential store (DPAPI/Credential Manager, Keychain or Secret +Service), with an explicit memory-only fallback when that facility is +unavailable; the shell never receives a password. Public catalog results are +cached in the core process and bounded per response. + +CloudMatch session methods preserve one client/device identity through create, +poll and stop, retain pending queue responses before signaling is available, +and return the complete ordered connection, ICE and negotiated-feature payload +needed by the native streamer. `streamer.prepare` returns the normalized session +context used by the native runtime linked into the Qt shell. The in-process runtime +owns secure NVIDIA signaling, ICE/DTLS/SCTP, RTSPS, Mjolnir, RTCP and native +gameplay input, while Qt owns the graphics device, scene graph, video item and +all top-level windows. Legacy streamer lifecycle methods remain protocol +compatibility routes and are not used by the Qt shell. + +`webrtcCompatibilityMode` selects `auto`, `on`, or `off` for new allocations. +The default `auto` uses WebRTC for a detected, authenticated alliance provider and +NVST for NVIDIA. `on` explicitly selects WebRTC for either provider; `off` selects +NVST. Provider selection is resolved in the core before codec validation and +allocation, not inferred by Qt or changed after allocation. The UI calls it +**WebRTC compatibility mode** and warns about lower performance than NVST. +An older `allianceWebrtcCompatibility: true` preference migrates to `on`; the old +default `false` migrates to `auto`. The historical boolean remains a compatibility +alias for explicit on/off writes. Session ownership and authentication remain +required. Legacy persisted +`transportMode` values still normalize to `nvst`; only the core-owned session's +transport selects the embedded WebRTC path. An explicit H.265 selection preserves +the requested resolution and frame rate, subject to the normal allocation and +decoder checks, and supports 8-bit or 10-bit 4:2:0 SDR. Both WebRTC and NVST +CloudMatch requests use `requestedStreamingFeatures.bitDepth: 1` for 10-bit +and `0` for 8-bit, rather than the literal media bit depth. The owned session +profile carries the selected codec through polling, restart, and attachment; +changed preferences cannot reinterpret an existing stream. H.264, Auto, and AV1 +selections retain the H.264 compatibility profile, at most 1920×1080 at 60 FPS +with 8-bit 4:2:0. Both WebRTC codecs use SDR, stereo, and no microphone. +WebRTC startup rejects profiles outside the native runtime's dimensions or frame +rate bounds rather than substituting a different viewport in its SDP answer. +WebRTC signaling endpoints remain separate +from RTSPS endpoints, including their server-provided paths and ports. The native +runtime owns the WebSocket offer, answer, NVIDIA quality SDP, and ICE exchange. +It delivers compressed media to the existing decoder and GPU presentation path. +The setting does not authorize a live NVST seat to be converted, deleted, or replaced. +For an owned WebRTC allocation that supplies only legacy RTSP descriptors, +the core derives `wss:///nvst/` on port 443, matching Android's +WebRTC signaling contract. It does not reuse the RTSP port as a WebSocket or UDP +media port. Explicit WebSocket descriptors take priority and keep their paths +and ports. `prepare_endpoints` diagnostics record bounded descriptor types and +numeric protocol/port fields, never hosts, URLs, session IDs, or credentials. +WebRTC offers may contain additional microphone or inactive media sections. +Negotiation keeps their order and MIDs, disables RTP upload, and accepts one +incoming video track matching the selected H.264, H.265 Main, or H.265 Main10 +profile and at most one incoming Opus audio track. Unsupported +microphone codecs are rejected in place with valid SDP format identifiers rather +than rejecting the whole session. Offers remain bounded to eight media sections. +The `webrtc-offer` diagnostic records section kinds and directions without SDP +credentials. Keyframe feedback uses the transport's negotiated video MID. +For an `actpass` offer, the native client takes the active DTLS role and sends +ClientHello, matching browser behavior against NVIDIA's ICE-lite endpoint. +Explicit active or passive remote roles retain their complementary local role. +Connection diagnostics distinguish ICE completion from DTLS establishment and +record redacted candidate classes, ports, and packet-family counters without +addresses, credentials, or payloads. +The Qt video item remains visible while the native backend is connecting, +including reconnect attempts, so it can create the graphics context before the +first frame. Gameplay focus/input still waits for streaming; startup overlays +may cover the same video item without preventing presentation initialization. + +`acceptance.export` is available only through the Qt shell's Diagnostics screen. It rejects +headless window systems and writes an atomic, redacted `opennow.live-acceptance` JSON file. The +file records machine-observed ten-minute streaming, first-frame, guide/input ownership, surface, +microphone, recording, hashed media, bounded recovery and terminal-error checks. It contains no +account identifiers, session identifiers, URLs, process identifiers, executable paths or local +media paths. A false check is retained as evidence of an incomplete run; it is never promoted to +a pass by the release verifier. + +Network work runs outside the protocol reader with a fixed concurrency ceiling, +connect/request deadlines, one serialized output writer and best-effort response +suppression after cancellation. The full Electron API inventory remains tracked +in [the machine-readable parity manifest](../native/opennow-core/contracts/legacy-open-now-api.json), +validated against its [JSON schema](../native/opennow-core/contracts/legacy-open-now-api.schema.json) +and executable golden-fixture tests. A method is not considered ported until its +owner, wire shape, fixtures and replacement disposition are recorded there. + +### Push invalidation capability + +Protocol 5 advertises `account.pushInvalidation.v1` when the core ships the native +push subscriber. The capability only describes the accelerator below; the shell +must keep working when it is absent. + +The core may open a native FCM/PNS subscription for the signed-in account. The subscription is +provider-scoped and bounded: it exists only while an account is signed in, and it is torn down or +paused on sign-out, account switch, or shutdown. + +`push.json` in the data directory is an optional override, never required. When it is genuinely +absent, the core uses a bundled default built from the vendor's public client identifiers (see +provenance below). When it is present, it always wins: the core never falls back to the bundled +default for an existing override. The override is either a single object or an array of objects, +each with `providerIdpId`, `projectId`, `apiKey`, `senderId`, `appId`, `firebaseAppId`, optional +`vapidKey`, `pnsServer`, optional `pnsVersion`, and `pnsClientId`, plus an optional `enabled` +boolean. The entry whose `providerIdpId` matches the signed-in provider is the only one used. A +present override suppresses the default when it disables push (`"enabled": false`, whole-file or +per entry), is unreadable, malformed, or larger than 64 KiB, or carries no complete entry for the +signed-in provider. The override file is read-only deployment input: the core never writes it, +never logs it, never returns it from `settings.get`, and never stores it in the user settings +schema. + +The bundled default contains only public client identifiers: a Firebase web API key, app, +project, and sender identifiers, the VAPID public key, and the PNS client id and server. It +carries no credentials and the core never writes it to the data directory. Per-account and +per-device secrets — the ECE key pair, the auth secret, and the GCM/FCM tokens — are generated +locally and stay in the OS credential store, and the PNS bearer token is the user's own sign-in +token. The default is bound to the authenticated provider and generation snapshot by the same +owner guards as an override, so it is re-evaluated on account and provider changes. Whether the +PNS backend accepts registrations for providers other than the vendor's own is unverified; the +bundled default mirrors the vendor client's authenticated-account gate and does not claim +guaranteed acceptance. + +The bundled default derives from the vendor's public client configuration asset +(`shared/assets/config/config.json` inside the official package, SHA256 +8e7db09026e5b48b0eabb364395c726543fcab9778b3561529fcb6a96208db65; official package archive +SHA256 47ddbe0425b9ab560f64fa42a0052794c9de335ded0fd59637f082dd7a161ad4), mapping +`firebase.pns` to the Firebase identity and `pnsServerConfig` to the PNS endpoint. A subscription +delivers invalidation hints only. Each hint is emitted as an ordinary event envelope: + +```json +{"type":"event","name":"account.push.changed","payload":{"generation":7,"kind":"library","changedIds":["app-id"]}} +``` + +`kind` is one of `library`, `favorites`, `subscription`, `linked-account`, or +`platform-sync`. `generation` is the GFN state generation the hint belongs to; +a shell drops hints whose generation does not match its current account +generation. `changedIds` is bounded and advisory. + +Hints never assert a result. `platform-sync` carries the store's reported +`platformCode`, `syncState`, `syncDate`, and `syncGameCount` and only makes the +shell observe the existing sync earlier; completion still requires a fresh +`syncDate` with `syncState` `SYNC_SUCCESS` from +`account.connections.sync.status`. A hint that arrives while its account is no +longer current is discarded, and a subscription never serves another account's +registration. + +Push registration state is per account and stored in the OS credential store. No token, key, +or message body is written to settings, diagnostics, or logs. The subscriber +uses only deadline-bounded network operations, and a configuration change, +sign-out, or account switch retires the current subscription before another one +starts. Every registration stage is guarded between requests, so a retired scope +or shutdown stops the remaining stages after at most one in-flight request +deadline. + +A refused MCS login (a non-zero `LoginResponse.error.code`) is a session failure, not a +credential verdict: it triggers one bounded registration refresh that keeps the stored device +identity and replaces only the registration keys and tokens. A `LoginResponse` that carries a +zero error code is accepted as a successful login. The device identity is replaced only when the +check-in authority rejects it with HTTP 400 or 401. + +Every received frame counts toward the `last_stream_id_received` the session advertises. The +session acknowledges incoming heartbeat pings, answers an immediate-ack data message or every +ten unacknowledged persistent messages with an IQ stream acknowledgement (`extension id` 13), +and honours the heartbeat interval the server negotiates through `LoginResponse.heartbeat_config` +within bounded limits. A heartbeat that stays unanswered past the acknowledgement deadline, or a +login request without a response within its deadline, ends the session so the owner reconnects +with the stored registration rather than holding a half-open connection. + +Encrypted bodies are selected the way the maintained client selects them: a declared +`content-encoding` of `aes128gcm` uses the RFC 8291 body header (salt, record size, key id), +`aesgcm` uses the legacy `Crypto-Key`/`Encryption` headers, any other declared value is +rejected, and an absent declaration falls back to the legacy headers when both are present and +to the RFC 8291 body otherwise. This follows Chromium's +`components/gcm_driver/crypto/gcm_encryption_provider.cc`, where `content-encoding` is the +`kContentEncodingProperty` discriminator and the legacy path is keyed on the presence of both +`Encryption` and `Crypto-Key`. No ciphertext shape is guessed: an unsupported declaration is an +error, not a heuristic. + +Live delivery remains unverified. The subscriber ships complete in code with the bundled default +and is covered by fixtures, but no live Google check-in, c2dm, FIS, FCM, PNS, or end-to-end +encrypted delivery has been exercised, and hardware and account-backed validation are out of +scope for the fixture path. + +### Catalog page and metadata capabilities + +Protocol 4 requires `catalog.libraryPages.v1`, `catalog.metadata.v1`, +`account.syncObservation.v1`, and `catalog.languages.v1` in the core handshake. +The Qt client and both relocated package probes check these capabilities. +The JSON envelope and native streaming ABI do not change. + +`catalog.library.list` accepts `limit` from 1 to 100, an opaque `cursor` up to +4,096 bytes, and a `traversalId` up to 256 bytes. Continuations must carry the +`catalogRevision` and opaque `catalogContext` returned by the first page. The context binds the cursor to the provider, account, generation, endpoint, resolved VPC, proxy route and locale. Results contain `games`, `count`, +nullable `totalCount`, `hasNextPage`, `nextCursor`, `fetchedAt`, `freshness`, +`traversalId`, `catalogRevision`, `catalogContext`, and the authenticated `scope`. One result is +at most 768 KiB. Oversized pages are retried at the same cursor with a smaller +count; a record that cannot fit is an error. Invalid identities or missing +pagination state are errors, not an empty complete library. + +CatalogState stages these pages and detects cursor cycles. A complete library +means the traversal reached a validated end; it does not imply a transactional +snapshot of concurrently changing vendor data. Refresh preserves the previous +complete snapshot until the new traversal ends. The foreground slice is 30 +seconds, with an explicit Continue action. The hard aggregate bounds are 20,000 +games and 32 MiB of conservative serialized-size accounting. Partial/error +results retain usable data, the failed cursor where safe, and the error text. +`catalogLastCompleteAt` changes only when the aggregate traversal completes. + +`catalog.game.get` accepts exactly one `appId` or `variantId`. The former is a +parent LCARS string; the latter must fit a positive GraphQL `Int`. The request +selects library-aware metadata and verifies the returned identity. It always +revalidates rather than authorizing from cached browse cards. The result has +`game`, `catalogRevision`, `scope`, `fetchedAt`, and `freshness`. + +Game variants retain nullable `librarySelected`, `libraryStatus`, `playStatus`, +`installed`, `subscription`, `gfnStatus`, `stateDetails`, `paymentModels`, +`subscriptions`, and `supportedLanguages`. Patch details distinguish automatic +patches, manual patches, maintenance, and unknown types. Games retain app-level +`favorited`, `catalogSkuStrings`, `campaignIds`, and fallback `paymentModels`. +App availability, favorites, payment models, and GFN membership do not establish +ownership of another variant. Visible detail metadata refreshes every 30 seconds; +patch duration history is an estimate, not a promised completion time. + +Every game variant also carries `features`, an array of lowercase strings +parsed from the variant's `gfn.features` key/value list: `rtx` from +`RTX_ENABLED`, `hdr` from `HDR_ENABLED` or a non-empty +`SUPPORTED_HDR_VERSION`, and `reflex` from `REFLEX_ENABLED`. Disabled, null, +and unknown keys produce nothing, so the array is empty when the catalog does +not declare a feature. `features` is null when the catalog response carried no +feature list for that variant, so consumers can tell unknown from none. + +The catalog has no per-game DLSS key, so the core, and only the core, derives a +game-level `dlss` field from NVIDIA's published DLSS games list (the JSON behind +NVIDIA's RTX games page) by title. Shells still must not infer features from +titles or genres themselves, and a `"dlss"` string in a variant's `features` +means nothing. While the list is loaded, every game in every `catalog.*` result carries `dlss`, including +`catalog.public.list`, `catalog.library.list`, `catalog.game.get`, +`catalog.launch.inspect`, `catalog.launch.store.inspect`, +`catalog.favorites.list`, the favorites and ownership mutation results, +`catalog.store.local`, `catalog.store.list`, and `catalog.store.presentation`. +The value is either null (not on NVIDIA's list) or: + +```json +{"superResolution":"native","frameGeneration":"override", + "multiFrameGeneration":{"maxMultiplier":4},"rayReconstruction":null, + "dlaa":"override","source":"nvidia-rtx-games-list"} +``` + +`superResolution`, `frameGeneration`, `rayReconstruction`, and `dlaa` are +`native` when the game supports the technology itself, `override` when a DLSS +override acts on it (the list's "NV, …" cells: an upgraded SR/RR/DLAA or frame +generation model, or DLAA enabled through the SR mode override), and null when +the list does not show it. `multiFrameGeneration` is null or the highest +multiplier an override enables. `dlss` is null when the title has no match and +when the matched row lists no DLSS technology. When the list is unavailable, games +have no `dlss` key at all: an absent key means DLSS support is unknown, and shells +must not present it as "not on NVIDIA's list". + +Matching removes ™, ®, ©, and zero-width characters, folds the title to ASCII +with NFKD, lowercases it, spells `&` as "and", and collapses every other +non-alphanumeric run to one space. An exact match wins; otherwise a loose match +also drops the words "the" and "tm" and trailing edition words (game of the +year, goty, definitive, enhanced, complete, deluxe, standard, ultimate, +director's cut, anniversary, edition, pc). "Remastered" is never dropped, +because remasters are listed separately. Only rows of type "Game" count. + +The core keeps the list in the store cache for 24 hours (stale copies are used +when a refresh fails), rejects documents over 4 MiB or 5,000 rows, and applies +it when a catalog response is sent, so cached catalog pages are unaffected and +the scope stays `catalog-v3`. Fetching never fails or holds a catalog response: +only one request waits for the core's first attempt, for at most 1.5 seconds; a failed attempt +retries on a catalog request at least 60 seconds later. The field is additive +within protocol version 5. + +Every game also carries `requiresPaidMembership`, true when the catalog's +`minimumMembershipTierLabel` names a tier other than Free (the catalog reports +`Premium` today). It is the rule the core applies before a launch, so shells +compare it with the account's `membershipTier` instead of parsing the label. +Both fields are additive within protocol version 5; store-catalog cache scopes moved from +`catalog-v2` to `catalog-v3` so cached cards are refetched with it. + +`catalog.definitions.get` returns independently fetched `stores`, `genres`, and +`subscriptions` sections. Each section has `items`, `source`, `status`, +`freshness`, `fetchedAt`, `expiresAt`, and nullable `error`. Static definitions +expire after 24 hours. Store definitions retain the feature union and per-variant +linking metadata. They drive account rows and allowed actions. Unknown stores +are display-only; unavailable definitions use labelled, action-disabled fallback +rows. A current server `supported:false` overrides older capability data. +Store subscription IDs are separate from MES/GFN membership. + +`catalog.store.presentation` retains parsed filter expressions and `sortOrders`. +`catalog.store.list` can receive a `filterId` or `sortId`; the core resolves these +to the returned server expressions rather than sending an ID as a filter object. +`revalidate:true` invalidates only the addressed page key; use it for an explicit +search submission rather than `refresh:true`, which resets a browse chain. +Local Store facets remain local. `catalog.store.local` returns +`localHasNextPage`, `cacheComplete`, `upstreamCoverage`, and `facetsSource` +alongside its existing demand-driven cursor. Local exhaustion does not prove +upstream coverage. Browse disk pages expire after 15 minutes, and explicit local +refresh first revalidates the upstream first page before rebuilding the index. +Confirmed account changes persist an increasing catalog revision, so old pages cannot join +a new traversal or reappear after a core restart. Failure to persist the revision is an explicit cache-invalidation error, not completion. Cache identities include provider/account, generation, +endpoint, resolved VPC, proxy route, locale and schema revision. + +### Store synchronization observation + +`account.connections.sync` sends one ALS POST and accepts only HTTP 202. It +returns `operationId`, `provider`, and a local `phase`. Repeated starts for the +same active provider return the existing operation instead of repeating POST. +There are at most eight account-scoped operations. Mutating requests are never +automatically replayed after a 401, timeout, cancellation, or ambiguous send. + +`account.connections.sync.status` accepts `operationId`. Each poll reads fresh +`userAccount` at most once. Calls are spaced at two seconds initially and five +seconds after 20 seconds, with a 120-second observation deadline. The operation +records the baseline date and state. Only a changed valid RFC3339 `syncDate` +with `SYNC_SUCCESS` enters `refreshing_library`; unchanged old success, an +unchanged old failure, and changed counts alone do not prove completion. New +failure, disconnection, and expiry enter `failed`. Missing completion evidence +ends as `timed_out`, meaning completion is unconfirmed. + +After `refreshing_library`, Qt starts the final library traversal. Only a +complete traversal displays completion and acknowledges it with +`sync.status({operationId, libraryRefreshed:true})`. Partial refresh retains a +separate incomplete notice. `account.connections.sync.cancel` stops observation, +not accepted remote work. Account/provider generation changes clear pending +work; late results cannot complete another account's operation. Polls do not +sleep across their observation lifetime or hold authentication or scheduler +locks across HTTP. Link callbacks and unlink acknowledgements require a fresh +account read before claiming the connection changed. + +### Overall supported game languages + +`catalog.languages.get({refresh:false})` works without sign-in. Its anonymous +HTTP document is exactly `{ overallGfnSupportedLanguages { language } }`. +There are no variables and no Authorization header. It returns exact wire IDs +in `languages`, `status` (`success`, `stale`, or `error`), `source`, cache timing, +nullable `error`, and `scopeGeneration`. The bounded cache is shared with the +existing disk-cache owner, keyed separately by public endpoint, provider context, +and proxy route. Its TTL is 14 days. At most 512 safe IDs of 64 bytes are accepted; +duplicate IDs are removed without normalization. Invalid or empty metadata does +not replace a previous good list. Failure with a previous list is explicitly +stale; failure without one has an empty list and error status. This reference +data does not select the interface locale, keyboard map, or a game's preferences. + +### Independent language preferences and settings choices + +`appLanguage` defaults to `system` and selects a bundled Qt interface locale. +`gameLanguage` defaults to `en_US`; `keyboardLayout` defaults to `en-US`. +Changing one does not change either of the others. Game language identifiers +retain exact case, separators, script, and numeric-region subtags. They have an +ASCII alphabetic first subtag of 2–8 bytes, subsequent alphanumeric subtags of +1–8 bytes separated by `_` or `-`, and a total limit of 64 bytes. `auto` and +`system` are rejected, case-insensitively. Safe future IDs need not appear in +current metadata to remain saved. + +`settings.get` returns `keyboardLayouts` beside `settings`, never inside the +persisted values. Each descriptor has `value`, `label`, and `aliases`. The table +in `language.rs` is the authority for the supported Windows-rig keyboard subset +on all Qt platforms. Historical `ja-JP` and `Japanese106` request `ja-106`; +historical `es-ES` requests `es-ES_tradnl`. Valid saved aliases retain their +original spelling on disk. This does not select proprietary Mac keyboard IDs. +New unrecognized keyboard values and malformed game IDs return `invalid_setting`. +Restored strings remain visible, but a shared request resolver replaces corrupt +game or keyboard values with the existing respective defaults before create, +immediate resume, or claim requests. The interface locale never enters these +query parameters. + +`settings.choices.get({runtimeCapabilities})` also returns `frameRates` for the documented +frame-rate tiers, using the same `value`, `disabled`, and nullable `reason` descriptor shape. +The conditional top tier reports a reason for each unconfirmed condition: a resolution outside +full HD, an unreported capability probe, or a reported probe without a hardware decoder for the +selected codec. Missing or unreported capabilities never imply support, and only rates at or +below 240 FPS are unconditional. The entitlement limit stays a Qt-side decision because the +core holds no subscription data. + +`settings.choices.get({runtimeCapabilities})` returns `colorQualities` for the current +persisted settings and embedded streamer capability snapshot. Each of the four +descriptors contains `value`, `disabled`, and nullable `reason`. The operation +calls the same profile resolver as final launch. A color quality is disabled when +resolution fails or when the launch would fall back to a lower bit depth or a +different chroma format; the reason names the delivered profile. Codec descriptors +are disabled only when the codec itself cannot decode 8-bit 4:2:0 SDR on the selected +backend. Missing or incompatible capabilities do not imply support. This read does not persist or coerce any setting. Protocol 5 and native +streamer protocol 7 are unchanged. + +CoreClient injects the current native window's `nativeHdrSupported` into +`runtimeCapabilities`, overriding any caller-supplied flag, just as it does for +`session.create` and `streamer.prepare`. It injects `nativeHdrDisplay` alongside it only +when the output reported a validated luminance range, so an unavailable snapshot is +omitted rather than sent as zero. SettingsState observes +`HdrOutput.supported` only to cancel and refetch choices when the display changes; +it does not author the wire capability or expose platform handles. + +Qt `SettingsState` owns shared desktop/console choices and the single lazy +language request. Settings entry loads idle or expired metadata. Explicit retry +sets `refresh:true`; there is no retry loop. Requests have a 15-second deadline. +Readiness, account/provider generation, or proxy changes clear request ownership +before cancellation and discard old metadata, but preserve saved preferences. +Only a response for the current request and generation is accepted. `cacheHit`, +stale/error status, saved IDs absent from metadata, and local fallback choices +remain distinguishable. Language and profile edits serialize per key and become +selected only after persistence succeeds; older failures cannot roll back a +newer successful edit. These local settings operations do not restart media. diff --git a/external/opennow/docs/qt-native-streamer-ffi.md b/external/opennow/docs/qt-native-streamer-ffi.md new file mode 100644 index 000000000..fa4547602 --- /dev/null +++ b/external/opennow/docs/qt-native-streamer-ffi.md @@ -0,0 +1,69 @@ +# Qt native streamer lifecycle events + +ABI version 13 delivers these events through the generic JSON event callback +without a new C ABI layout. +The native streamer owns wire parsing and termination classification. Qt does not +infer resumability from a wire code or reason. + +## Session timers + +```json +{"type":"session-timer","startId":"start-42","timerType":4,"secondsLeft":60,"active":true} +``` + +`timerType` is 4 or 5, `secondsLeft` is an unsigned 32-bit integer, and `active` +is a boolean. `NativeStreamRuntime` validates these fields and emits +`sessionTimerReceived(quint8, quint32, bool)` on the Qt thread. Malformed timer +events do not reach QML. `startId` must match the current start-command generation. +Timer events from a replaced or stopped session, or a previous runtime instance, +are discarded. + +`ShellStore.acceptSessionTimer` owns warning state for the current stream. Type 4 +with `active:true` starts a display-only countdown from the server-provided seconds. +The countdown clamps at zero and never clears the warning or ends the session. +Type 4 with `active:false` clears the warning after the transport confirms successful input. Type 5 clears +the warning. Qt neither clears on local input submission nor ends a session when +a timer arrives. Session end, replacement, native restart, and terminal native +status also clear the warning. + +Both desktop and console compose `StreamIdleWarning` above the same +`StreamVideoItem`. It has no input handlers, focus requests, or overlay route, so +it does not pause gameplay input or recreate the session. A refreshed server timer +resets the display-only countdown. + +## Server termination + +Terminal error and status events can include this object: + +```json +{"source":"nvst-server","code":"server-terminated","startId":"start-42","wireCode":2147680291,"reason":56,"extendedCode":2147680263,"resumable":false} +``` + +`reason` and `extendedCode` can be null. `resumable` can be true, false, or null. +The native streamer retains the numeric values through the existing JSON callback. +Qt rejects server termination events whose `startId` does not match the current +start-command generation, before invalidating input or presentation. The matching +error and stopped pair can both arrive after presentation has been invalidated. +Qt treats `nvst-server` with `code:"server-terminated"` and `resumable:false` as an +authoritative session end, cancels pending recovery, and never auto-resumes that +session. It retains the server message in `lastError` and the accessibility +announcement, and the full termination object in the streamer snapshot. +True or unknown classification follows the existing bounded +CloudMatch reconciliation path. Existing authoritative CloudMatch status 7 and +HTTP 404 handling remains unchanged. + +The evidence baseline is `reports/opennow-gfn-parity/audio-input-lifecycle.md`, +rows L3 through L5, and Windows `transport-feedback.md`, section 8. Offline Qt +callback, orchestration, and smoke tests cover delivery and passive composition; +they do not connect to GFN or execute GFN binaries. + +## Text retry feedback + +`text-input-error` carries the current `startId` and a fixed `reason`, with no +paste contents. C++ rejects stale generations before forwarding it through the +existing `eventReceived` signal. ShellStore announces localized retry feedback, +without changing streamer state, input ownership, error telemetry, or incidents. +Desktop and console display it for five seconds in the existing passive input +notice, including when the Guide is closed. The normal stream video item and +gameplay focus remain unchanged. Temporary SCTP deferral queues one bounded paste +behind older input; only rejected local paste admission needs this notice. diff --git a/external/opennow/native/opennow-core/Cargo.lock b/external/opennow/native/opennow-core/Cargo.lock new file mode 100644 index 000000000..3c887a3d6 --- /dev/null +++ b/external/opennow/native/opennow-core/Cargo.lock @@ -0,0 +1,3128 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "aes", + "block-padding", + "cbc", + "dbus", + "fastrand", + "hkdf", + "num", + "once_cell", + "sha2", + "zeroize", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "byteorder", + "dbus-secret-service", + "log", + "secret-service", + "security-framework 2.11.1", + "security-framework 3.7.0", + "windows-sys 0.60.2", + "zeroize", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opennow-core" +version = "1.0.0" +dependencies = [ + "aes-gcm", + "base64", + "chrono", + "ed25519-dalek", + "flate2", + "fs2", + "hkdf", + "hmac", + "httpdate", + "icu_normalizer", + "keyring", + "libc", + "md5", + "p256", + "qrcode", + "rand 0.9.5", + "regex", + "reqwest", + "rustls", + "scrypt", + "semver", + "serde", + "serde_json", + "sha2", + "subtle", + "tempfile", + "url", + "webpki-roots", + "windows-sys 0.59.0", + "zip", +] + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "qrcode" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68782463e408eb1e668cf6152704bd856c78c5b6417adaee3203d8f4c1fc9ec" + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "password-hash", + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "secret-service" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" +dependencies = [ + "aes", + "cbc", + "futures-util", + "generic-array", + "hkdf", + "num", + "once_cell", + "rand 0.8.8", + "serde", + "sha2", + "zbus", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-process", + "async-recursion", + "async-trait", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix", + "ordered-stream", + "rand 0.8.8", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "zvariant_derive", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/external/opennow/native/opennow-core/Cargo.toml b/external/opennow/native/opennow-core/Cargo.toml new file mode 100644 index 000000000..8eec93eda --- /dev/null +++ b/external/opennow/native/opennow-core/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "opennow-core" +version = "1.0.0" +edition = "2024" +license = "MIT" +rust-version = "1.85" + +[dependencies] +aes-gcm = "0.10" +base64 = "0.22" +chrono = { version = "0.4", default-features = false, features = ["clock"] } +ed25519-dalek = "2.2" +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +hkdf = "0.12" +httpdate = "1" +md5 = "0.7" +keyring = { version = "3.6", features = ["apple-native", "windows-native", "sync-secret-service", "crypto-rust"] } +p256 = { version = "0.13", features = ["ecdh", "pkcs8"] } +qrcode = { version = "0.14", default-features = false } +rand = "0.9" +regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "socks"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +hmac = "0.12" +icu_normalizer = { version = "2.1", default-features = false, features = ["compiled_data"] } +scrypt = "0.11" +semver = "1" +subtle = "2" +url = "2" +webpki-roots = "1" +zip = { version = "2.4", default-features = false, features = ["deflate"] } +fs2 = "0.4" +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_System_JobObjects", "Win32_System_ProcessStatus", "Win32_System_ApplicationInstallationAndServicing", "Win32_System_SystemServices", "Win32_System_IO", "Win32_Storage_FileSystem", "Wdk_System_Threading"] } + +[dev-dependencies] +tempfile = "3" + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = true diff --git a/external/opennow/native/opennow-core/contracts/legacy-open-now-api.json b/external/opennow/native/opennow-core/contracts/legacy-open-now-api.json new file mode 100644 index 000000000..ef0d26e5b --- /dev/null +++ b/external/opennow/native/opennow-core/contracts/legacy-open-now-api.json @@ -0,0 +1,116 @@ +{ + "version": 1, + "source": "opennow-stable/src/shared/gfn/api.ts:OpenNowApi", + "columns": ["legacy", "kind", "requestSchema", "resultSchema", "owner", "replacement", "status", "requestFixture", "resultFixture"], + "operations": [ + ["getAuthSession","request","AuthSessionRequest | undefined","AuthSessionResult","core","auth.session.get","ported",{}, {"session":null}], + ["getLoginProviders","request","void","LoginProvider[]","core","auth.providers.list","ported",{}, []], + ["getRegions","request","RegionsFetchRequest | undefined","StreamRegion[]","core","network.regions.list","ported",{}, []], + ["login","request","AuthLoginRequest","AuthSession","core","auth.device.start + auth.device.poll + auth.device.complete","superseded",{"provider":"nvidia"},{"session":null}], + ["startDeviceLogin","request","AuthDeviceLoginStartRequest","AuthDeviceLoginChallenge","core","auth.device.start","ported",{"provider":"nvidia"},{"attemptId":"fixture","userCode":"ABCD-EFGH"}], + ["pollDeviceLogin","request","AuthDeviceLoginPollRequest","AuthDeviceLoginPollResult","core","auth.device.poll","ported",{"attemptId":"fixture","deviceCode":"redacted-fixture"},{"status":"pending"}], + ["completeDeviceLogin","request","AuthDeviceLoginAttemptRequest","AuthSession","core","auth.device.complete","ported",{"attemptId":"fixture"},{"session":null}], + ["cancelDeviceLogin","request","AuthDeviceLoginAttemptRequest","void","core","auth.device.cancel","ported",{"attemptId":"fixture"},null], + ["logout","request","void","void","core","auth.logout","ported",{},null], + ["logoutAll","request","void","void","core","auth.accounts.logoutAll","ported",{},null], + ["getSavedAccounts","request","void","SavedAccount[]","core","auth.accounts.list","ported",{},[]], + ["switchAccount","request","{userId:string,pin?:string}","AuthSession","core","auth.accounts.switch","ported",{"userId":"fixture-user"},{"session":null}], + ["removeAccount","request","{userId:string}","void","core","auth.accounts.remove","ported",{"userId":"fixture-user"},null], + ["getConsolePinStatus","request","{userId:string}","ConsolePinStatus","core","auth.pin.status","ported",{"userId":"fixture-user"},{"configured":false}], + ["setConsolePin","request","ConsolePinSetRequest","ConsolePinMutationResult","core","auth.pin.set","ported",{"userId":"fixture-user","pin":"0000"},{"ok":true}], + ["clearConsolePin","request","ConsolePinClearRequest","ConsolePinMutationResult","core","auth.pin.clear","ported",{"userId":"fixture-user","pin":"0000"},{"ok":true}], + ["verifyConsolePin","request","ConsolePinVerifyRequest","ConsolePinVerifyResult","core","auth.pin.verify","ported",{"userId":"fixture-user","pin":"0000"},{"verified":true}], + ["fetchSubscription","request","SubscriptionFetchRequest","SubscriptionInfo","core","account.subscription.get","ported",{},{}], + ["fetchPersistentStorageLocations","request","PersistentStorageLocationsFetchRequest | undefined","PersistentStorageLocationsResult","core","account.storage.locations","ported",{}, {"locations":[]}], + ["resetPersistentStorage","request","PersistentStorageResetRequest | undefined","PersistentStorageResetResult","core","account.storage.reset","ported",{}, {"ok":true}], + ["fetchGameAccountConnections","request","void","GameAccountConnectionsResult","core","account.connections.list","ported",{}, {"connections":[]}], + ["linkGameAccount","request","GameAccountOperationRequest","GameAccountOperationResult","core","account.connections.link.start + account.connections.link.poll","superseded",{"provider":"steam"},{"status":"pending"}], + ["unlinkGameAccount","request","GameAccountOperationRequest","GameAccountOperationResult","core","account.connections.unlink","ported",{"provider":"steam"},{"ok":true}], + ["resyncGameAccount","request","GameAccountOperationRequest","GameAccountOperationResult","core","account.connections.sync","ported",{"provider":"steam"},{"ok":true}], + ["fetchMainGames","request","GamesFetchRequest","GameInfo[]","core","catalog.public.list","superseded",{},[]], + ["fetchStorePanels","request","GamesFetchRequest","GamePanelResult[]","core","catalog.public.list + Qt catalog projection","superseded",{},[]], + ["fetchFeaturedGames","request","GamesFetchRequest","GameInfo[]","core","catalog.public.list + Qt featured projection","superseded",{},[]], + ["fetchLibraryGames","request","GamesFetchRequest","GameInfo[]","core","catalog.library.list","ported",{},[]], + ["browseCatalog","request","CatalogBrowseRequest","CatalogBrowseResult","core","catalog.public.list + Qt filters","superseded",{}, {"games":[]}], + ["fetchPublicGames","request","void","GameInfo[]","core","catalog.public.list","ported",{},[]], + ["resolveLaunchAppId","request","ResolveLaunchIdRequest","string | null","core","catalog public/library normalized launchAppId","superseded",{"appId":"123"},"123"], + ["resolveStoreUrl","request","ResolveStoreUrlRequest","string | null","core","catalog normalized storeUrl","superseded",{"appId":"123"},null], + ["markGameOwned","request","MarkGameOwnedRequest","MarkGameOwnedResult","core","account.connections.sync + catalog.library.list","superseded",{"appId":"123"},{"ok":true}], + ["getPendingDirectLaunchRequest","request","void","DirectLaunchRequest | null","qt","AppController.handleArguments","superseded",{},null], + ["onDirectLaunchRequest","event","void","DirectLaunchRequest","qt","AppController.directLaunchRequested","ported",{}, {"appId":"123","title":"Fixture Game"}], + ["createSession","request","SessionCreateRequest","SessionInfo","core","session.create","ported",{"appId":"123"},{"status":1}], + ["pollSession","request","SessionPollRequest","SessionInfo","core","session.poll","ported",{"sessionId":"fixture-session"},{"status":1}], + ["reportSessionAd","request","SessionAdReportRequest","SessionInfo","core","session.ad.report","ported",{"sessionId":"fixture-session"},{"status":1}], + ["stopSession","request","SessionStopRequest","void","core","session.stop","ported",{"sessionId":"fixture-session"},null], + ["getActiveSessions","request","{token?:string,streamingBaseUrl?:string}","ActiveSessionInfo[]","core","session.remote.list","ported",{},[]], + ["claimSession","request","SessionClaimRequest","SessionInfo","core","session.claim","ported",{"sessionId":"fixture-session"},{"status":2}], + ["getNativeStreamerStatus","request","void","NativeStreamerStatus","core","streamer.status.get","ported",{}, {"state":"idle"}], + ["getNativeCloudGsyncCapabilities","request","void","NativeCloudGsyncCapabilities","streamer","capabilities event surfaced by streamer.status.get","superseded",{}, {"supported":false}], + ["showSessionConflictDialog","request","void","SessionConflictChoice","qt","SessionConflictScreen","ported",{},"cancel"], + ["connectSignaling","request","SignalingConnectRequest","void","core","streamer.start owns signaling lifecycle","superseded",{},null], + ["disconnectSignaling","request","void","void","core","streamer.stop owns signaling lifecycle","superseded",{},null], + ["sendAnswer","request","SendAnswerRequest","void","core","streamer.start child protocol relay","superseded",{},null], + ["sendIceCandidate","request","IceCandidatePayload","void","core","streamer.start child protocol relay","superseded",{},null], + ["sendNativeInput","send","NativeInputPacket","void","streamer","native input transport","superseded",{"kind":"keyboard"},null], + ["setNativeInputPaused","send","boolean","void","core","streamer.input.pause","ported",{"paused":true},null], + ["updateNativeRenderSurface","send","NativeRenderSurfaceUpdate","void","core","streamer.surface.update","ported",{"visible":true},null], + ["updateNativeShortcuts","send","NativeStreamerShortcutBindings","void","streamer","streamer.start shortcut bindings","superseded",{},null], + ["requestKeyframe","request","KeyframeRequest","void","core","streamer.control request-keyframe","superseded",{},null], + ["onSignalingEvent","event","void","MainToRendererSignalingEvent","core","streamer.changed event","superseded",{}, {"type":"status"}], + ["onToggleFullscreen","event","void","void","qt","streamer shortcut routed through ShellStore","superseded",{},null], + ["onExitFullscreen","event","void","void","qt","deterministic overlay/back handling","superseded",{},null], + ["quitApp","request","void","void","qt","QCoreApplication.quit","ported",{},null], + ["getUpdaterState","request","void","AppUpdaterState","core","updater.state.get","ported",{}, {"status":"idle"}], + ["checkForUpdates","request","void","AppUpdaterState","core","updater.check","ported",{}, {"status":"not-available"}], + ["downloadUpdate","request","void","AppUpdaterState","core","updater.download","ported",{}, {"status":"downloaded"}], + ["installUpdateAndRestart","request","void","AppUpdaterState","core","updater.install","ported",{}, {"status":"installing"}], + ["onUpdaterStateChanged","event","void","AppUpdaterState","core","updater.changed event","ported",{}, {"status":"idle"}], + ["setFullscreen","request","boolean","void","qt","Window visibility/fullscreen state","ported",{"value":true},null], + ["toggleFullscreen","request","void","void","qt","Window visibility/fullscreen state","ported",{},null], + ["togglePointerLock","request","void","void","streamer","native input ownership contract","superseded",{},null], + ["notifyPointerLockChange","send","{active:boolean,suppressEscapeFullscreenGrace?:boolean}","void","streamer","native input ownership contract","superseded",{"active":true},null], + ["notifyNativeInputModeChange","send","{active:boolean,rawInputOwnsEscape:boolean}","void","streamer","streamer.input.pause + native ownership","superseded",{"active":true,"rawInputOwnsEscape":true},null], + ["readClipboardText","request","void","string","qt","AppController.readClipboardText","ported",{},"fixture text"], + ["getSettings","request","void","Settings","core","settings.get","ported",{},{}], + ["setSetting","request","{key:keyof Settings,value:Settings[key]}","void","core","settings.set","ported",{"key":"resolution","value":"1920x1080"},null], + ["getGpuInfo","request","void","GpuBackendInfo","core","streamer.detect","superseded",{}, {"gpuName":null,"hardwareDecodeCodecs":[]}], + ["resetSettings","request","void","Settings","core","settings.reset","ported",{},{}], + ["selectNativeStreamerExecutable","request","void","string | null","qt","AppController.normalizeNativeStreamerExecutable + FileDialog","ported",{},null], + ["getMicrophonePermission","request","void","MicrophonePermissionResult","streamer","native microphone permission/start status","superseded",{}, {"status":"unknown"}], + ["exportLogs","request","'text' | 'json' | undefined","string","core","diagnostics.export","ported",{"format":"text"},"/tmp/opennow-diagnostics.txt"], + ["submitBugReport","request","DesktopBugReportRequest","DesktopBugReportReceipt","core","bug_report.submit","ported",{"summary":"Fixture report"},{"ok":true}], + ["pingRegions","request","StreamRegion[]","PingResult[]","core","network.regions.ping","ported",{"regions":[]},[]], + ["saveScreenshot","request","ScreenshotSaveRequest","ScreenshotEntry","qt","AppController.captureScreenRegion + media.list","superseded",{}, {"id":"fixture.png","kind":"screenshot"}], + ["listScreenshots","request","void","ScreenshotEntry[]","core","media.list screenshot projection","superseded",{},[]], + ["deleteScreenshot","request","ScreenshotDeleteRequest","void","core","media.delete","superseded",{"id":"fixture.png"},null], + ["saveScreenshotAs","request","ScreenshotSaveAsRequest","ScreenshotSaveAsResult","qt","AppController.copyScreenshotTo + FileDialog","ported",{"id":"fixture.png"},{"cancelled":true}], + ["onTriggerScreenshot","event","void","void","streamer","screenshotRequestGeneration in streamer.changed","superseded",{},null], + ["onExternalEscape","event","void","void","qt","deterministic back/overlay state","superseded",{},null], + ["onStreamShortcutAction","event","void","NativeStreamerShortcutAction","streamer","shortcut action in streamer.changed","superseded",{},"screenshot"], + ["setStreamShortcutInterceptionGate","send","StreamShortcutInterceptionGate","void","streamer","streamer.input.pause ownership gate","superseded",{"enabled":true},null], + ["openExternalUrl","request","string","void","qt","AppController.openExternalUrl","ported",{"url":"https://opennow.app"},null], + ["beginRecording","request","RecordingBeginRequest","RecordingBeginResult","core","media.recording.target + streamer.recording.start","superseded",{}, {"recordingId":"fixture"}], + ["sendRecordingChunk","request","RecordingChunkRequest","void","streamer","source-stream recorder tap","superseded",{},null], + ["finishRecording","request","RecordingFinishRequest","RecordingEntry","core","streamer.recording.stop","superseded",{}, {"id":"fixture.mkv","kind":"recording"}], + ["abortRecording","request","RecordingAbortRequest","void","streamer","recording disconnect/fail-closed cleanup","superseded",{},null], + ["listRecordings","request","void","RecordingEntry[]","core","media.list recording projection","superseded",{},[]], + ["deleteRecording","request","RecordingDeleteRequest","void","core","media.delete","superseded",{"id":"fixture.mkv"},null], + ["showRecordingInFolder","request","string","void","qt","AppController.openLocalPath(reveal)","ported",{"id":"fixture.mkv"},null], + ["listMediaByGame","request","{gameTitle?:string} | undefined","MediaListingResult","core","media.list","ported",{}, {"entries":[]}], + ["getMediaThumbnail","request","{filePath:string}","string | null","core","media.list thumbnailPath","superseded",{"filePath":"fixture.mkv"},null], + ["showMediaInFolder","request","{filePath:string}","void","qt","AppController.openLocalPath(reveal)","ported",{"filePath":"fixture.mkv"},null], + ["getMediaPlaybackUrl","request","{filePath:string}","string | null","qt","trusted local media URL in QML","superseded",{"filePath":"fixture.mkv"},null], + ["deleteMediaFile","request","{filePath:string}","{ok:boolean}","core","media.delete","superseded",{"filePath":"fixture.mkv"},{"ok":true}], + ["regenMediaThumbnail","request","{filePath:string}","{ok:boolean,thumbnailDataUrl:string|null}","qt","ThumbnailGenerator.regenerate","superseded",{"filePath":"fixture.mkv"},{"ok":true,"thumbnailDataUrl":null}], + ["deleteCache","request","void","void","core","cache.delete","ported",{},null], + ["fetchPrintedWasteQueue","request","void","PrintedWasteQueueData","core","queue.status.get","ported",{}, {"zone-fixture":{"QueuePosition":1,"Last Updated":1,"Region":"EU"}}], + ["fetchPrintedWasteServerMapping","request","void","PrintedWasteServerMapping","core","queue.serverMapping.get","ported",{}, {"zone-fixture":{"title":"Fixture","region":"EU"}}], + ["getThanksData","request","void","ThankYouDataResult","core","thanks.data.get","ported",{}, {"contributors":[],"supporters":[]}], + ["provisionZortosCommunityProxy","request","void","CommunityProxyProvisionResult","core","communityProxy.provision","ported",{}, {"proxyUrl":"http://fixture:fixture@altaria.proxy.rlwy.net:51545/"}], + ["setDiscordActivity","request","DiscordActivityUpdate","void","core","discord.activity.sync","ported",{},null], + ["clearDiscordActivity","request","void","void","core","discord.activity.clear","ported",{},null], + ["getReleaseHighlights","request","string | undefined","ReleaseHighlightsPayload","core","updater.highlights.get","ported",{"version":"0.5.4"},{"version":"0.5.4"}], + ["ackReleaseHighlights","request","void","void","core","updater.highlights.ack","ported",{},null], + ["onReleaseHighlightsShow","event","void","ReleaseHighlightsPayload","core","updater.highlights.show event","ported",{}, {"version":"0.5.4"}] + ] +} diff --git a/external/opennow/native/opennow-core/contracts/legacy-open-now-api.schema.json b/external/opennow/native/opennow-core/contracts/legacy-open-now-api.schema.json new file mode 100644 index 000000000..cbe98f618 --- /dev/null +++ b/external/opennow/native/opennow-core/contracts/legacy-open-now-api.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://opennow.app/contracts/legacy-open-now-api.schema.json", + "title": "OpenNOW Electron bridge parity inventory", + "type": "object", + "required": ["version", "source", "columns", "operations"], + "additionalProperties": false, + "properties": { + "version": { "const": 1 }, + "source": { "type": "string", "minLength": 1 }, + "columns": { + "const": ["legacy", "kind", "requestSchema", "resultSchema", "owner", "replacement", "status", "requestFixture", "resultFixture"] + }, + "operations": { + "type": "array", + "minItems": 109, + "maxItems": 109, + "items": { + "type": "array", + "prefixItems": [ + { "type": "string", "pattern": "^[A-Za-z][A-Za-z0-9]*$" }, + { "enum": ["request", "send", "event"] }, + { "type": "string", "minLength": 1 }, + { "type": "string", "minLength": 1 }, + { "enum": ["core", "qt", "streamer"] }, + { "type": "string", "minLength": 1 }, + { "enum": ["ported", "superseded", "pending"] }, + {}, + {} + ], + "minItems": 9, + "maxItems": 9 + } + } + } +} diff --git a/external/opennow/native/opennow-core/src/account_connections.rs b/external/opennow/native/opennow-core/src/account_connections.rs new file mode 100644 index 000000000..f5b79dcfc --- /dev/null +++ b/external/opennow/native/opennow-core/src/account_connections.rs @@ -0,0 +1,1459 @@ +use crate::gfn::{AuthSession, ServiceError}; +use rand::RngCore as _; +use reqwest::blocking::Client; +use reqwest::header::{ + ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue, ORIGIN, REFERER, USER_AGENT, +}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use url::Url; + +const USER_ACCOUNT_QUERY: &str = "{ userAccount { subscriptions { id } storesData { store accountLinkingData { userDisplayName expiresIn userIdentifier accountSyncingData { totalNumberOfSyncedGfnGames syncState syncDate } } } } }"; +const CALLBACK_PORTS: [u16; 5] = [2259, 6460, 7119, 8870, 9096]; +const LCARS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; +const CLIENT_VERSION: &str = "2.0.87.131"; +const USER_AGENT_VALUE: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/128.0.0.0 Safari/537.36 GFN-PC/2.0.87.131"; + +struct LinkAttempt { + provider: String, + scope: String, + receiver: mpsc::Receiver>, + expires: Instant, + cancelled: Arc, + callback: Option, + next_check: Instant, +} + +impl Drop for LinkAttempt { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::Release); + } +} + +pub struct AccountConnectionsService { + attempts: Mutex>, + link_generation: AtomicU64, + syncs: Mutex>, +} + +pub struct AccountContext<'a> { + pub client: &'a Client, + pub auth: &'a AuthSession, + pub generation: u64, + pub graphql: &'a str, + pub als: &'a str, + pub definitions: &'a Value, + pub requests: &'a crate::store_requests::StoreRequests, + pub check: &'a dyn Fn() -> Result<(), ServiceError>, +} + +impl AccountContext<'_> { + fn scope(&self) -> String { + json!([ + self.auth.user.user_id, + self.auth.provider.idp_id, + self.generation + ]) + .to_string() + } + + fn providers(&self) -> Vec { + if let Some(items) = self.definitions["stores"]["items"].as_array() { + return items + .iter() + .filter_map(|item| { + serde_json::from_value::(item.clone()) + .ok() + }) + .map(|definition| { + let mut value = definition.connection_definition(); + value["capabilitySource"] = self.definitions["stores"]["status"].clone(); + value + }) + .collect(); + } + provider_definitions() + .into_iter() + .map(|mut definition| { + definition["capabilitySource"] = json!("fallback"); + definition["supportsLinking"] = json!(false); + definition["supportsSync"] = json!(false); + definition + }) + .collect() + } +} + +#[derive(Clone)] +struct SyncOperation { + id: String, + provider: String, + scope: String, + baseline: Value, + phase: &'static str, + started: Instant, + next_poll: Instant, + message: String, + invalidated: bool, +} + +impl SyncOperation { + fn result(&self) -> Value { + json!({"operationId":self.id,"provider":self.provider,"phase":self.phase, + "retryAfterMs":self.next_poll.saturating_duration_since(Instant::now()).as_millis() as u64, + "message":self.message}) + } +} + +impl AccountConnectionsService { + pub fn cancel_pending(&self) { + { + let mut attempts = self.attempts.lock().expect("account-link state poisoned"); + self.link_generation.fetch_add(1, Ordering::Relaxed); + attempts.clear(); + } + self.syncs + .lock() + .expect("account-sync state poisoned") + .clear(); + } + pub fn new() -> Self { + Self { + attempts: Mutex::new(HashMap::new()), + link_generation: AtomicU64::new(0), + syncs: Mutex::new(HashMap::new()), + } + } + + pub fn list(&self, context: &AccountContext<'_>) -> Result { + (context.check)()?; + let token = session_token(context.auth)?; + let response = context.requests.send( + context + .client + .post(context.graphql) + .headers(graphql_headers(token)?) + .json(&json!({"query":USER_ACCOUNT_QUERY})), + "Game-account discovery failed", + )?; + let payload = crate::gfn::catalog::catalog_payload(response, Some(token))?; + (context.check)()?; + let account = &payload["data"]["userAccount"]; + let stores = account["storesData"] + .as_array() + .filter(|items| items.len() <= 128) + .ok_or_else(crate::catalog_types::invalid_metadata)?; + let subscriptions = account["subscriptions"] + .as_array() + .filter(|items| items.len() <= 128) + .ok_or_else(crate::catalog_types::invalid_metadata)?; + let fetched_at = unix_millis(); + let accounts = context + .providers() + .iter() + .map(|definition| { + let provider = definition["provider"].as_str().unwrap_or_default(); + let store = stores.iter().find(|store| { + normalize_provider(store["store"].as_str().unwrap_or_default()) == provider + }); + connection_from_store(definition, store, fetched_at) + }) + .collect::>(); + crate::store_catalog_page::bounded_result( + json!({"accounts":accounts,"subscriptions":subscriptions,"definitions":context.definitions,"fetchedAt":fetched_at}), + ) + } + + pub fn sync( + &self, + params: &Value, + context: &AccountContext<'_>, + ) -> Result { + let provider = required_provider(params)?; + ensure_provider_feature(&context.providers(), &provider, "supportsSync")?; + let id = random_id(); + { + let mut operations = self.syncs.lock().expect("account-sync state poisoned"); + operations.retain(|_, operation| { + operation.scope == context.scope() + && operation.started.elapsed() < Duration::from_secs(180) + }); + if let Some(operation) = operations.values().find(|operation| { + operation.provider == provider + && matches!( + operation.phase, + "starting" | "waiting_remote" | "refreshing_library" + ) + }) { + return Ok(operation.result()); + } + if operations.len() >= 8 { + return Err(invalid("Too many account-sync operations")); + } + operations.insert( + id.clone(), + SyncOperation { + id: id.clone(), + provider: provider.clone(), + scope: context.scope(), + baseline: Value::Null, + phase: "starting", + started: Instant::now(), + next_poll: Instant::now(), + message: String::new(), + invalidated: false, + }, + ); + } + let outcome = (|| { + let baseline = self.list(context)?; + let account = baseline["accounts"] + .as_array() + .and_then(|items| items.iter().find(|account| account["provider"] == provider)) + .cloned() + .ok_or_else(|| invalid("Game-account provider is unavailable"))?; + (context.check)()?; + let response = context.requests.send( + context + .client + .post(format!("{}/sync/{provider}", context.als)) + .headers(als_headers(session_token(context.auth)?, true)?) + .json(&json!({})), + "Game-account sync failed", + )?; + if response.status().as_u16() != 202 { + return Err(response_error("Game-account sync failed", response)); + } + (context.check)()?; + Ok(account) + })(); + let mut operations = self.syncs.lock().expect("account-sync state poisoned"); + let operation = operations + .get_mut(&id) + .ok_or_else(|| invalid("Account-sync observation was cancelled"))?; + match outcome { + Ok(baseline) => { + operation.baseline = baseline; + operation.phase = "waiting_remote"; + operation.next_poll = Instant::now() + Duration::from_secs(2); + } + Err(error) => { + operation.phase = "failed"; + operation.message = + "Could not confirm sync acceptance. Refresh before retrying.".into(); + return Err(error); + } + } + Ok(operation.result()) + } + + pub fn sync_status( + &self, + params: &Value, + context: &AccountContext<'_>, + ) -> Result { + let id = params["operationId"] + .as_str() + .ok_or_else(|| invalid("operationId is required"))?; + let mut operation = { + let mut operations = self.syncs.lock().expect("account-sync state poisoned"); + let operation = operations + .get_mut(id) + .ok_or_else(|| invalid("Account-sync observation is no longer active"))?; + if operation.scope != context.scope() { + return Err(ServiceError { + code: "stale_account", + message: "The sync belongs to a previous account context".into(), + }); + } + if params["cancelObservation"] == true { + operation.phase = "cancelled_observation"; + } + if params["libraryRefreshed"] == true && operation.phase == "refreshing_library" { + operation.phase = "complete"; + } + if operation.phase == "waiting_remote" + && operation.started.elapsed() >= Duration::from_secs(120) + { + operation.phase = "timed_out"; + operation.message = + "Sync completion could not be confirmed. The remote sync may still finish." + .into(); + } + if operation.phase != "waiting_remote" || Instant::now() < operation.next_poll { + return Ok(operation.result()); + } + operation.next_poll = Instant::now() + + Duration::from_secs(if operation.started.elapsed() < Duration::from_secs(20) { + 2 + } else { + 5 + }); + operation.clone() + }; + let account = match self.list(context) { + Ok(account) => account, + Err(error) => { + if error.code == "http_unauthorized" + && let Some(operation) = self + .syncs + .lock() + .expect("account-sync state poisoned") + .get_mut(id) + { + operation.next_poll = Instant::now(); + } + return Err(error); + } + }; + let current = account["accounts"].as_array().and_then(|items| { + items + .iter() + .find(|item| item["provider"] == operation.provider) + }); + operation.phase = observed_sync_phase(&operation.baseline, current); + (context.check)()?; + let mut operations = self.syncs.lock().expect("account-sync state poisoned"); + let current = operations + .get_mut(id) + .ok_or_else(|| invalid("Account-sync observation was cancelled"))?; + if current.phase == "waiting_remote" { + current.phase = operation.phase; + if operation.phase == "failed" { + current.message = + "The store reported a sync or connection failure. Check the account status." + .into(); + } + } + Ok(current.result()) + } + + pub fn invalidate_sync( + &self, + id: &str, + invalidate: impl FnOnce() -> Result<(), ServiceError>, + ) -> Result { + let mut operations = self.syncs.lock().expect("account-sync state poisoned"); + let Some(operation) = operations.get_mut(id) else { + return Ok(false); + }; + if operation.invalidated { + return Ok(false); + } + invalidate()?; + operation.invalidated = true; + Ok(true) + } + + pub fn unlink( + &self, + params: &Value, + context: &AccountContext<'_>, + ) -> Result { + let provider = required_provider(params)?; + ensure_provider_feature(&context.providers(), &provider, "supportsLinking")?; + let url = format!("{}/linking/{provider}", context.als); + (context.check)()?; + let response = context + .client + .delete(url) + .headers(als_headers(session_token(context.auth)?, false)?) + .send() + .map_err(|error| network("Game-account unlink failed", error))?; + if !response.status().is_success() && response.status().as_u16() != 404 { + return Err(response_error("Game-account unlink failed", response)); + } + let account = self.list(context)?; + let disconnected = account["accounts"].as_array().is_some_and(|items| { + items + .iter() + .any(|item| item["provider"] == provider && item["isConnected"] == false) + }); + Ok( + json!({"ok":disconnected,"provider":provider,"phase":if disconnected {"complete"} else {"unconfirmed"}, + "message":if disconnected {"Account disconnected"} else {"Disconnect accepted; account update could not yet be confirmed"}}), + ) + } + + pub fn start_link( + &self, + params: &Value, + context: &AccountContext<'_>, + ) -> Result { + let provider = required_provider(params)?; + ensure_provider_feature(&context.providers(), &provider, "supportsLinking")?; + (context.check)()?; + let generation = { + let mut attempts = self.attempts.lock().expect("account-link state poisoned"); + attempts.retain(|_, attempt| attempt.expires > Instant::now()); + if attempts.len() >= 8 { + return Err(invalid("Too many account-link attempts")); + } + self.link_generation.load(Ordering::Relaxed) + }; + let (listener, port) = bind_callback()?; + let redirect_uri = format!("http://localhost:{port}/"); + let mut url = + Url::parse(&format!("{}/login_url", context.als)).expect("constant ALS URL is valid"); + url.query_pairs_mut() + .append_pair("platform", &provider) + .append_pair("redirect_uri", &redirect_uri) + .append_pair("client_id", "gfn-pc"); + (context.check)()?; + let response = context + .client + .get(url) + .headers(als_headers(session_token(context.auth)?, false)?) + .send() + .map_err(|error| network("Account-linking URL failed", error))?; + if !response.status().is_success() { + return Err(response_error("Account-linking URL failed", response)); + } + let login_url = response + .json::() + .ok() + .and_then(|payload| payload["login_url"].as_str().map(ToOwned::to_owned)) + .ok_or_else(|| upstream("Account-linking URL response was incomplete"))?; + crate::requests::check()?; + (context.check)()?; + let mut attempts = self.attempts.lock().expect("account-link state poisoned"); + if self.link_generation.load(Ordering::Relaxed) != generation { + return Err(invalid("Account-link attempt was cancelled")); + } + if attempts.len() >= 8 { + return Err(invalid("Too many account-link attempts")); + } + let attempt_id = random_id(); + let (sender, receiver) = mpsc::channel(); + let worker_provider = provider.clone(); + let cancelled = Arc::new(AtomicBool::new(false)); + let worker_cancelled = cancelled.clone(); + thread::Builder::new() + .name(format!("opennow-account-link-{attempt_id}")) + .spawn(move || { + let result = wait_for_callback(listener, &worker_provider, &worker_cancelled); + let _ = sender.send(result); + }) + .map_err(|error| network("Could not start account-link callback", error))?; + attempts.insert( + attempt_id.clone(), + LinkAttempt { + provider: provider.clone(), + scope: context.scope(), + receiver, + expires: Instant::now() + Duration::from_secs(300), + cancelled, + callback: None, + next_check: Instant::now(), + }, + ); + Ok( + json!({"attemptId":attempt_id,"provider":provider,"loginUrl":login_url,"expiresInSeconds":300}), + ) + } + + pub fn poll_link( + &self, + params: &Value, + context: &AccountContext<'_>, + ) -> Result { + let attempt_id = params["attemptId"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("account.connections.link.poll requires attemptId"))?; + let mut attempts = self.attempts.lock().expect("account-link state poisoned"); + let attempt = attempts.get_mut(attempt_id).ok_or_else(|| ServiceError { + code: "link_attempt_not_found", + message: "Account-link attempt is no longer active".to_owned(), + })?; + if attempt.scope != context.scope() { + return Err(ServiceError { + code: "stale_account", + message: "The link belongs to a previous account context".into(), + }); + } + if Instant::now() >= attempt.expires { + attempts.remove(attempt_id); + return Ok(json!({"status":"expired"})); + } + if attempt.callback.is_none() { + match attempt.receiver.try_recv() { + Ok(Ok(result)) => attempt.callback = Some(result), + Ok(Err(message)) => { + attempts.remove(attempt_id); + return Ok(json!({"status":"error","message":message})); + } + Err(mpsc::TryRecvError::Empty) => return Ok(json!({"status":"pending"})), + Err(mpsc::TryRecvError::Disconnected) => { + attempts.remove(attempt_id); + return Ok( + json!({"status":"error","message":"Account-link callback stopped unexpectedly"}), + ); + } + } + } + if Instant::now() < attempt.next_check { + return Ok(json!({"status":"pending","phase":"verifying_account"})); + } + attempt.next_check = Instant::now() + Duration::from_secs(2); + let provider = attempt.provider.clone(); + drop(attempts); + let account = match self.list(context) { + Ok(account) => account, + Err(error) => { + if error.code == "http_unauthorized" + && let Some(attempt) = self + .attempts + .lock() + .expect("account-link state poisoned") + .get_mut(attempt_id) + { + attempt.next_check = Instant::now(); + } + return Err(error); + } + }; + let connected = account["accounts"].as_array().is_some_and(|items| { + items.iter().any(|item| { + item["provider"] == provider + && item["isConnected"] == true + && item["status"] != "expired" + }) + }); + if connected { + self.attempts + .lock() + .expect("account-link state poisoned") + .remove(attempt_id); + } + Ok( + json!({"status":if connected {"complete"} else {"pending"},"phase":"verifying_account","provider":provider,"account":account}), + ) + } +} + +fn wait_for_callback( + listener: TcpListener, + provider: &str, + cancelled: &AtomicBool, +) -> Result { + listener + .set_nonblocking(true) + .map_err(|error| error.to_string())?; + let deadline = Instant::now() + Duration::from_secs(300); + while Instant::now() < deadline && !cancelled.load(Ordering::Acquire) { + match listener.accept() { + Ok((mut stream, _)) => { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 8192]; + let count = stream + .read(&mut buffer) + .map_err(|error| error.to_string())?; + let request = String::from_utf8_lossy(&buffer[..count]); + let target = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/"); + let callback = Url::parse(&format!("http://localhost{target}")) + .map_err(|_| "Invalid account-link callback".to_owned())?; + if !callback.query_pairs().any(|(key, _)| { + matches!( + key.as_ref(), + "platform" | "error" | "display_name" | "expires_in" + ) + }) { + let _ = + stream.write_all(b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n"); + continue; + } + let error = callback + .query_pairs() + .find(|(key, _)| key == "error") + .map(|(_, value)| value.into_owned()); + if let Some(error) = error { + redirect_callback(&mut stream, provider, true); + return Err(error); + } + let actual = callback + .query_pairs() + .find(|(key, _)| key == "platform") + .map(|(_, value)| normalize_provider(&value)) + .ok_or_else(|| "Account-link callback did not include a provider".to_owned())?; + if actual != provider { + redirect_callback(&mut stream, provider, true); + return Err("Account-link callback provider mismatch".to_owned()); + } + let display_name = callback + .query_pairs() + .find(|(key, _)| key == "display_name") + .map(|(_, value)| value.into_owned()); + let expires_in = callback + .query_pairs() + .find(|(key, _)| key == "expires_in") + .map(|(_, value)| value.into_owned()); + redirect_callback(&mut stream, provider, false); + return Ok( + json!({"platform":provider,"displayName":display_name,"expiresIn":expires_in}), + ); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(100)) + } + Err(error) => return Err(error.to_string()), + } + } + Err("Timed out waiting for account-linking callback".to_owned()) +} + +fn redirect_callback(stream: &mut impl Write, provider: &str, failed: bool) { + let location = if failed { + format!( + "https://static-als.nvidia.com/result?platform={provider}&ui_locales=en_US&error=accountlink_fail" + ) + } else { + format!("https://static-als.nvidia.com/result?platform={provider}&ui_locales=en_US") + }; + let response = format!( + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + ); + let _ = stream.write_all(response.as_bytes()); +} + +fn bind_callback() -> Result<(TcpListener, u16), ServiceError> { + for port in CALLBACK_PORTS { + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + return Ok((listener, port)); + } + } + Err(upstream("No account-linking callback port is available")) +} + +fn connection_from_store(definition: &Value, store: Option<&Value>, fetched_at: i64) -> Value { + let linking = store.map(|value| &value["accountLinkingData"]); + let sync = linking.map(|value| &value["accountSyncingData"]); + let connected = linking.is_some_and(|value| value.is_object()); + let expires_in = linking + .and_then(|value| value["expiresIn"].as_str()) + .and_then(|value| value.parse::().ok()); + let expires_at = expires_in + .filter(|value| *value >= 0) + .map(|value| fetched_at + value * 1000); + let sync_state = sync.and_then(|value| value["syncState"].as_str()); + let expired = connected + && definition["supportsLinking"].as_bool() == Some(true) + && expires_at.is_some_and(|value| value <= fetched_at); + let sync_error = connected + && definition["supportsSync"].as_bool() == Some(true) + && sync_state.is_some_and(|value| value != "SYNC_SUCCESS"); + json!({ + "provider":definition["provider"],"label":definition["label"],"sortOrder":definition["sortOrder"], + "capabilitySource":definition["capabilitySource"],"features":definition["features"],"accountLinkingMetadata":definition["accountLinkingMetadata"], + "supportsLinking":definition["supportsLinking"],"supportsSync":definition["supportsSync"],"isRequired":definition["isRequired"], + "isConnected":connected,"status":if !connected { "not_connected" } else if expired { "expired" } else if sync_error { "sync_error" } else { "connected" }, + "displayName":linking.and_then(|value| value["userDisplayName"].as_str()), + "userIdentifier":linking.and_then(|value| value["userIdentifier"].as_str()), + "expiresIn":linking.and_then(|value| value["expiresIn"].as_str()),"expiresAt":expires_at, + "syncState":sync_state,"syncDate":sync.and_then(|value| value["syncDate"].as_str()), + "syncedGames":sync.and_then(|value| value["totalNumberOfSyncedGfnGames"].as_i64()).unwrap_or_default() + }) +} + +fn provider_definitions() -> Vec { + vec![ + json!({"provider":"UPLAY","label":"Ubisoft","sortOrder":100,"supportsLinking":true,"supportsSync":true,"isRequired":true}), + json!({"provider":"BATTLENET","label":"Battle.net","sortOrder":101,"supportsLinking":true,"supportsSync":true,"isRequired":true}), + json!({"provider":"EPIC","label":"Epic Games","sortOrder":104,"supportsLinking":true,"supportsSync":false,"isRequired":true}), + json!({"provider":"GAIJIN","label":"Gaijin.net","sortOrder":105,"supportsLinking":true,"supportsSync":true,"isRequired":true}), + json!({"provider":"GOG","label":"GOG","sortOrder":106,"supportsLinking":true,"supportsSync":true,"isRequired":true}), + json!({"provider":"STEAM","label":"Steam","sortOrder":108,"supportsLinking":false,"supportsSync":true,"isRequired":false}), + json!({"provider":"XBOX","label":"Xbox","sortOrder":120,"supportsLinking":true,"supportsSync":true,"isRequired":true}), + ] +} + +fn ensure_provider_feature( + definitions: &[Value], + provider: &str, + feature: &str, +) -> Result<(), ServiceError> { + let supported = definitions.iter().any(|definition| { + definition["provider"] == provider && definition[feature].as_bool() == Some(true) + }); + if supported { + Ok(()) + } else { + Err(invalid( + "This game-account provider does not support that action", + )) + } +} + +fn required_provider(params: &Value) -> Result { + let provider = params["provider"] + .as_str() + .map(normalize_provider) + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("A game-account provider is required"))?; + if provider_definitions() + .iter() + .any(|definition| definition["provider"] == provider) + { + Ok(provider) + } else { + Err(invalid("Unsupported game-account provider")) + } +} + +fn normalize_provider(value: &str) -> String { + crate::catalog_types::normalize_store(value) +} + +fn observed_sync_phase(baseline: &Value, current: Option<&Value>) -> &'static str { + let Some(current) = current else { + return "failed"; + }; + if current["isConnected"] != true || current["status"] == "expired" { + return "failed"; + } + let state = current["syncState"].as_str().unwrap_or(""); + let date = current["syncDate"].as_str(); + let changed_date = date + .and_then(|date| chrono::DateTime::parse_from_rfc3339(date).ok()) + .is_some_and(|date| { + baseline["syncDate"] + .as_str() + .and_then(|baseline| chrono::DateTime::parse_from_rfc3339(baseline).ok()) + .is_none_or(|baseline| date > baseline) + }); + if state == "SYNC_SUCCESS" && changed_date { + return "refreshing_library"; + } + if !state.is_empty() + && state != "SYNC_SUCCESS" + && (changed_date || current["syncState"] != baseline["syncState"]) + { + return "failed"; + } + "waiting_remote" +} + +fn graphql_headers(token: &str) -> Result { + let mut headers = base_headers()?; + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + ORIGIN, + HeaderValue::from_static("https://play.geforcenow.com"), + ); + headers.insert( + REFERER, + HeaderValue::from_static("https://play.geforcenow.com/"), + ); + headers.insert("nv-browser-type", HeaderValue::from_static("CHROME")); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("GFNJWT {token}")) + .map_err(|_| invalid("Invalid authentication token"))?, + ); + Ok(headers) +} + +fn als_headers(token: &str, json_body: bool) -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + ACCEPT, + HeaderValue::from_static("application/json, text/plain, */*"), + ); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|_| invalid("Invalid authentication token"))?, + ); + headers.insert( + ORIGIN, + HeaderValue::from_static("https://play.geforcenow.com"), + ); + headers.insert( + REFERER, + HeaderValue::from_static("https://play.geforcenow.com/"), + ); + headers.insert(USER_AGENT, HeaderValue::from_static(USER_AGENT_VALUE)); + if json_body { + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + } + Ok(headers) +} + +fn base_headers() -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + ACCEPT, + HeaderValue::from_static("application/json, text/plain, */*"), + ); + headers.insert("nv-client-id", HeaderValue::from_static(LCARS_CLIENT_ID)); + headers.insert("nv-client-type", HeaderValue::from_static("NATIVE")); + headers.insert( + "nv-client-version", + HeaderValue::from_static(CLIENT_VERSION), + ); + headers.insert( + "nv-client-streamer", + HeaderValue::from_static("NVIDIA-CLASSIC"), + ); + headers.insert( + "nv-device-os", + HeaderValue::from_static(if cfg!(target_os = "windows") { + "WINDOWS" + } else if cfg!(target_os = "macos") { + "MACOS" + } else { + "LINUX" + }), + ); + headers.insert("nv-device-type", HeaderValue::from_static("DESKTOP")); + headers.insert("nv-device-make", HeaderValue::from_static("GENERIC")); + headers.insert("nv-device-model", HeaderValue::from_static("PC")); + headers.insert(USER_AGENT, HeaderValue::from_static(USER_AGENT_VALUE)); + Ok(headers) +} + +fn session_token(auth: &AuthSession) -> Result<&str, ServiceError> { + auth.tokens + .id_token + .as_deref() + .or(Some(auth.tokens.access_token.as_str())) + .filter(|value| !value.is_empty()) + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "No authenticated token is available".to_owned(), + }) +} + +fn random_id() -> String { + let mut bytes = [0_u8; 16]; + rand::rng().fill_bytes(&mut bytes); + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} +fn unix_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64 +} +fn invalid(message: impl Into) -> ServiceError { + ServiceError { + code: "invalid_params", + message: message.into(), + } +} +fn upstream(message: impl Into) -> ServiceError { + ServiceError { + code: "upstream_error", + message: message.into(), + } +} +fn network(context: &str, error: impl std::fmt::Display) -> ServiceError { + ServiceError { + code: "network_error", + message: format!("{context}: {error}"), + } +} +fn response_error(context: &str, response: reqwest::blocking::Response) -> ServiceError { + let status = response.status(); + ServiceError { + code: if status.as_u16() == 401 { + "http_unauthorized" + } else if status.as_u16() == 403 { + "authentication_required" + } else { + "upstream_error" + }, + message: format!("{context} ({status})"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn provider_aliases_are_stable() { + assert_eq!(normalize_provider("Ubisoft Connect"), "UPLAY"); + assert_eq!(normalize_provider("battle-net"), "BATTLENET"); + } + #[test] + fn unsupported_actions_are_rejected() { + assert!( + ensure_provider_feature(&provider_definitions(), "STEAM", "supportsLinking").is_err() + ); + assert!(ensure_provider_feature(&provider_definitions(), "STEAM", "supportsSync").is_ok()); + } + + #[test] + fn gog_link_uses_als_and_confirms_connection_through_account_discovery() { + let (callback_sender, callback_receiver) = mpsc::channel(); + let (url, worker) = crate::gfn::tests::mock_requests( + vec![ + (200, json!({"login_url":"https://example.com/gog-sign-in"})), + ( + 200, + json!({"data":{"userAccount":{"subscriptions":[],"storesData":[{ + "store":"GOG","accountLinkingData":{"userDisplayName":"Fixture"} + }]}}}), + ), + ], + move |index, request| { + if index == 0 { + let target = request.split_whitespace().nth(1).unwrap(); + let url = Url::parse(&format!("http://localhost{target}")).unwrap(); + assert_eq!(url.path(), "/login_url"); + let query: HashMap<_, _> = url.query_pairs().into_owned().collect(); + assert_eq!(query["platform"], "GOG"); + assert_eq!(query["client_id"], "gfn-pc"); + let callback = Url::parse(&query["redirect_uri"]).unwrap(); + assert_eq!(callback.host_str(), Some("localhost")); + assert!(CALLBACK_PORTS.contains(&callback.port().unwrap())); + callback_sender.send(callback).unwrap(); + } else { + assert!(request.starts_with("POST / HTTP/1.1\r\n")); + } + }, + ); + let service = AccountConnectionsService::new(); + let client = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"status":"success","items":[{ + "store":"GOG","label":"GOG","features":[ + {"__typename":"AccountLinkingSso","supported":true}, + {"__typename":"AccountGamesSyncing","supported":true} + ] + }]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 1, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let result = service + .start_link(&json!({"provider":"gog"}), &context) + .unwrap(); + assert_eq!(result["provider"], "GOG"); + assert_eq!(result["loginUrl"], "https://example.com/gog-sign-in"); + let attempt_id = result["attemptId"].as_str().unwrap(); + assert_eq!( + service + .poll_link(&json!({"attemptId":attempt_id}), &context) + .unwrap()["status"], + "pending" + ); + let callback = client + .get(callback_receiver.recv().unwrap()) + .query(&[("platform", "GOG")]) + .send() + .unwrap(); + assert_eq!(callback.status().as_u16(), 302); + let deadline = Instant::now() + Duration::from_secs(3); + let result = loop { + let result = service + .poll_link(&json!({"attemptId":attempt_id}), &context) + .unwrap(); + if result["status"] != "pending" { + break result; + } + assert!(Instant::now() < deadline, "GOG callback was not observed"); + thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(result["status"], "complete"); + assert_eq!(result["account"]["accounts"][0]["supportsSync"], true); + assert!(service.attempts.lock().unwrap().is_empty()); + worker.join().unwrap(); + + let mut disabled = definitions.clone(); + disabled["stores"]["items"][0]["features"] = json!([]); + let context = AccountContext { + definitions: &disabled, + ..context + }; + let params = json!({"provider":"GOG"}); + assert!(service.start_link(¶ms, &context).is_err()); + assert!(service.sync(¶ms, &context).is_err()); + assert!(service.unlink(¶ms, &context).is_err()); + let context = AccountContext { + definitions: &Value::Null, + ..context + }; + assert!(service.start_link(¶ms, &context).is_err()); + assert!(service.sync(¶ms, &context).is_err()); + } + + fn account(date: &str, state: &str) -> Value { + json!({"data":{"userAccount":{"subscriptions":[{"id":"store-subscription"}],"storesData":[{ + "store":"STEAM","accountLinkingData":{"userDisplayName":"Fixture","accountSyncingData":{ + "syncDate":date,"syncState":state,"totalNumberOfSyncedGfnGames":5}}}]}}}) + } + + #[test] + fn account_discovery_posts_a_json_graphql_document() { + let (url, worker) = crate::gfn::tests::mock_requests( + vec![(200, account("2026-09-14T00:00:00Z", "SYNC_SUCCESS"))], + |_, request| { + assert!(request.starts_with("POST / HTTP/1.1\r\n")); + let (headers, body) = request.split_once("\r\n\r\n").unwrap(); + assert!( + headers + .to_ascii_lowercase() + .contains("\r\ncontent-type: application/json\r\n") + ); + assert_eq!( + serde_json::from_str::(body).unwrap(), + json!({"query":USER_ACCOUNT_QUERY}) + ); + }, + ); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"status":"success","items":[{"store":"STEAM","label":"Steam", + "features":[{"__typename":"AccountGamesSyncing","supported":true}]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 1, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let result = AccountConnectionsService::new().list(&context).unwrap(); + assert_eq!( + result["subscriptions"], + json!([{"id":"store-subscription"}]) + ); + worker.join().unwrap(); + } + + #[test] + fn sync_acceptance_requires_new_remote_success_then_library_acknowledgement() { + let old = "2026-09-14T00:00:00Z"; + let fresh = "2026-09-14T01:00:00Z"; + let (url, worker) = crate::gfn::tests::mock_requests( + vec![ + (200, account(old, "SYNC_SUCCESS")), + (202, json!({})), + (200, account(old, "SYNC_SUCCESS")), + (200, account(fresh, "SYNC_SUCCESS")), + ], + |index, request| { + if index == 1 { + assert!(request.starts_with("POST /sync/STEAM ")); + } else { + assert!(request.contains("userAccount")); + } + }, + ); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"status":"success","items":[{"store":"STEAM","label":"Steam", + "features":[{"__typename":"AccountGamesSyncing","supported":true}]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 4, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let service = AccountConnectionsService::new(); + let started = service + .sync(&json!({"provider":"STEAM"}), &context) + .unwrap(); + assert_eq!(started["phase"], "waiting_remote"); + let id = started["operationId"].as_str().unwrap(); + assert_eq!( + service + .sync(&json!({"provider":"STEAM"}), &context) + .unwrap()["operationId"], + id + ); + let params = json!({"operationId":id}); + assert_eq!( + service.sync_status(¶ms, &context).unwrap()["phase"], + "waiting_remote" + ); + for phase in ["waiting_remote", "refreshing_library"] { + service.syncs.lock().unwrap().get_mut(id).unwrap().next_poll = Instant::now(); + assert_eq!( + service.sync_status(¶ms, &context).unwrap()["phase"], + phase + ); + } + assert!(service.invalidate_sync(id, || Ok(())).unwrap()); + assert!( + !service + .invalidate_sync(id, || panic!("duplicate invalidation")) + .unwrap() + ); + assert_eq!( + service + .sync_status(&json!({"operationId":id,"libraryRefreshed":true}), &context) + .unwrap()["phase"], + "complete" + ); + worker.join().unwrap(); + } + + #[test] + fn sync_failure_baselines_unknown_dates_and_disconnect_are_distinct() { + let baseline = json!({"isConnected":true,"status":"connected","syncDate":"2026-09-14T00:00:00Z","syncState":"SYNC_SUCCESS"}); + for (change, expected) in [ + (json!({"syncDate":"not-a-date"}), "waiting_remote"), + (json!({"syncState":"SYNC_DENIED"}), "failed"), + (json!({"syncState":"NEW_UNKNOWN_FAILURE"}), "failed"), + (json!({"status":"expired"}), "failed"), + (json!({"isConnected":false}), "failed"), + (json!({"syncedGames":900}), "waiting_remote"), + ] { + let mut current = baseline.clone(); + for (key, value) in change.as_object().unwrap() { + current[key] = value.clone(); + } + assert_eq!(observed_sync_phase(&baseline, Some(¤t)), expected); + } + let failed = + json!({"isConnected":true,"syncState":"SYNC_DENIED","syncDate":"2026-09-14T00:00:00Z"}); + assert_eq!( + observed_sync_phase(&failed, Some(&failed)), + "waiting_remote" + ); + assert_eq!(observed_sync_phase(&baseline, None), "failed"); + } + + #[test] + fn server_false_and_unknown_stores_never_inherit_fallback_actions() { + for (store, supported, expected) in [ + ("STEAM", false, false), + ("STEAM", true, true), + ("NEW_STORE", true, false), + ] { + let definition: crate::catalog_types::StoreDefinition = serde_json::from_value(json!({ + "store":store,"label":"Fixture","features":[{"__typename":"AccountGamesSyncing","supported":supported}] + })).unwrap(); + assert_eq!(definition.connection_definition()["supportsSync"], expected); + } + } + + #[test] + fn observation_timeout_cancel_and_scope_rejection_do_not_send_network_requests() { + let client = Client::builder().no_proxy().build().unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({}); + let requests = crate::store_requests::StoreRequests::default(); + let mut context = AccountContext { + client: &client, + auth: &auth, + generation: 4, + graphql: "http://127.0.0.1:1", + als: "http://127.0.0.1:1", + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let service = AccountConnectionsService::new(); + service.syncs.lock().unwrap().insert( + "fixture".into(), + SyncOperation { + id: "fixture".into(), + provider: "STEAM".into(), + scope: context.scope(), + baseline: json!({}), + phase: "waiting_remote", + started: Instant::now() - Duration::from_secs(121), + next_poll: Instant::now(), + message: String::new(), + invalidated: false, + }, + ); + let params = json!({"operationId":"fixture"}); + assert_eq!( + service.sync_status(¶ms, &context).unwrap()["phase"], + "timed_out" + ); + assert_eq!( + service + .sync_status( + &json!({"operationId":"fixture","cancelObservation":true}), + &context + ) + .unwrap()["phase"], + "cancelled_observation" + ); + context.generation += 1; + assert_eq!( + service.sync_status(¶ms, &context).unwrap_err().code, + "stale_account" + ); + service.cancel_pending(); + assert!(service.sync_status(¶ms, &context).is_err()); + } + + #[test] + fn abandoned_link_attempts_are_pruned_before_admission() { + let (url, worker) = crate::gfn::tests::mock_requests( + vec![(200, json!({"login_url":"https://example.com/link"}))], + |_, _| {}, + ); + let service = AccountConnectionsService::new(); + let cancellations = seed_link_attempts(&service, 8); + let client = Client::builder().no_proxy().build().unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"items":[{"store":"STEAM","label":"Steam", + "features":[{"__typename":"AccountLinkingSso","supported":true}]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 1, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let params = json!({"provider":"STEAM"}); + assert_eq!( + service.start_link(¶ms, &context).unwrap_err().message, + "Too many account-link attempts" + ); + for attempt in service.attempts.lock().unwrap().values_mut() { + attempt.expires = Instant::now() - Duration::from_secs(1); + } + let result = service.start_link(¶ms, &context).unwrap(); + assert!(result["attemptId"].is_string()); + assert_eq!(service.attempts.lock().unwrap().len(), 1); + assert!( + cancellations + .iter() + .all(|cancelled| cancelled.load(Ordering::Acquire)) + ); + service.cancel_pending(); + worker.join().unwrap(); + } + + #[test] + fn cancellation_during_link_url_fetch_cannot_install_an_attempt() { + let service = Arc::new(AccountConnectionsService::new()); + let cancelling = Arc::clone(&service); + let (url, worker) = crate::gfn::tests::mock_requests( + vec![(200, json!({"login_url":"https://example.com/link"}))], + move |_, _| cancelling.cancel_pending(), + ); + let client = Client::builder().no_proxy().build().unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"items":[{"store":"STEAM","label":"Steam", + "features":[{"__typename":"AccountLinkingSso","supported":true}]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 1, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + assert_eq!( + service + .start_link(&json!({"provider":"STEAM"}), &context) + .unwrap_err() + .message, + "Account-link attempt was cancelled" + ); + assert!(service.attempts.lock().unwrap().is_empty()); + worker.join().unwrap(); + } + + #[test] + fn overlapping_link_admission_never_exceeds_the_limit() { + let (url, worker) = crate::gfn::tests::mock_requests( + vec![(200, json!({"login_url":"https://example.com/link"})); 2], + |_, _| {}, + ); + let service = AccountConnectionsService::new(); + seed_link_attempts(&service, 7); + let barrier = std::sync::Barrier::new(2); + let client = Client::builder().no_proxy().build().unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"items":[{"store":"STEAM","label":"Steam", + "features":[{"__typename":"AccountLinkingSso","supported":true}]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..2) + .map(|_| { + scope.spawn(|| { + let checks = std::cell::Cell::new(0); + let check = || { + checks.set(checks.get() + 1); + if checks.get() == 2 { + barrier.wait(); + } + Ok(()) + }; + let context = AccountContext { + client: &client, + auth: &auth, + generation: 1, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &check, + }; + service.start_link(&json!({"provider":"STEAM"}), &context) + }) + }) + .collect(); + let results: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect(); + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .find_map(|result| result.as_ref().err()) + .unwrap() + .message, + "Too many account-link attempts" + ); + }); + assert_eq!(service.attempts.lock().unwrap().len(), 8); + service.cancel_pending(); + worker.join().unwrap(); + } + + fn seed_link_attempts( + service: &AccountConnectionsService, + count: usize, + ) -> Vec> { + (0..count) + .map(|index| { + let cancelled = Arc::new(AtomicBool::new(false)); + let (_, receiver) = mpsc::channel(); + service.attempts.lock().unwrap().insert( + format!("abandoned-{index}"), + LinkAttempt { + provider: "STEAM".into(), + scope: "fixture".into(), + receiver, + expires: Instant::now() + Duration::from_secs(300), + cancelled: cancelled.clone(), + callback: None, + next_check: Instant::now(), + }, + ); + cancelled + }) + .collect() + } + + #[test] + fn link_callback_survives_unauthorized_and_waits_for_authoritative_connection() { + let (url, worker) = crate::gfn::tests::mock_requests( + vec![ + (401, json!({})), + ( + 200, + json!({"data":{"userAccount":{"subscriptions":[],"storesData":[]}}}), + ), + (200, account("2026-09-14T01:00:00Z", "SYNC_SUCCESS")), + ], + |_, request| assert!(request.contains("userAccount")), + ); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let auth = crate::gfn::tests::auth_fixture("account-a"); + let definitions = json!({"stores":{"status":"success","items":[{"store":"STEAM","label":"Steam","features":[]}]}}); + let requests = crate::store_requests::StoreRequests::default(); + let context = AccountContext { + client: &client, + auth: &auth, + generation: 4, + graphql: &url, + als: &url, + definitions: &definitions, + requests: &requests, + check: &|| Ok(()), + }; + let service = AccountConnectionsService::new(); + let (sender, receiver) = mpsc::channel(); + sender.send(Ok(json!({"platform":"STEAM"}))).unwrap(); + drop(sender); + service.attempts.lock().unwrap().insert( + "fixture".into(), + LinkAttempt { + provider: "STEAM".into(), + scope: context.scope(), + receiver, + expires: Instant::now() + Duration::from_secs(300), + cancelled: Arc::new(AtomicBool::new(false)), + callback: None, + next_check: Instant::now(), + }, + ); + let params = json!({"attemptId":"fixture"}); + assert_eq!( + service.poll_link(¶ms, &context).unwrap_err().code, + "http_unauthorized" + ); + let pending = service.poll_link(¶ms, &context).unwrap(); + assert_eq!(pending["status"], "pending"); + assert_eq!(pending["phase"], "verifying_account"); + assert_eq!( + service.poll_link(¶ms, &context).unwrap()["status"], + "pending" + ); + service + .attempts + .lock() + .unwrap() + .get_mut("fixture") + .unwrap() + .next_check = Instant::now(); + assert_eq!( + service.poll_link(¶ms, &context).unwrap()["status"], + "complete" + ); + assert!(service.attempts.lock().unwrap().is_empty()); + worker.join().unwrap(); + } +} diff --git a/external/opennow/native/opennow-core/src/analytics.rs b/external/opennow/native/opennow-core/src/analytics.rs new file mode 100644 index 000000000..31594f18e --- /dev/null +++ b/external/opennow/native/opennow-core/src/analytics.rs @@ -0,0 +1,863 @@ +use reqwest::blocking::Client; +use serde_json::{Map, Value, json}; +use std::collections::VecDeque; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +const MAX_QUEUED_EVENTS: usize = 1_000; +const FLUSH_INTERVAL: Duration = Duration::from_secs(30); +const FLUSH_THRESHOLD: usize = 50; +const MAX_BATCH_EVENTS: usize = 100; +const MAX_SPOOL_BYTES: usize = 1024 * 1024; +const SHUTDOWN_BUDGET: Duration = Duration::from_millis(1_000); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Emitter { + Core, + Shell, +} + +#[derive(Clone, Copy)] +enum Prop { + Text(usize), + Id, + Code, + Number(f64), + Bool, + Choice(&'static [&'static str]), + Resolution, +} + +const STAGES: &[&str] = &[ + "create", + "poll", + "claim", + "prepare", + "start", + "stream", + "presentation", +]; +const REPORT_KINDS: &[&str] = &[ + "session_error", + "stream_error", + "frame_drops", + "library_error", + "manual", +]; + +type EventSpec = (&'static str, Emitter, &'static [(&'static str, Prop)]); + +const EVENTS: &[EventSpec] = &[ + ( + "app_opened", + Emitter::Core, + &[ + ("gpu_vendor", Prop::Code), + ( + "package_kind", + Prop::Choice(&["msi", "appimage", "flatpak", "deb", "dmg", "zip"]), + ), + ], + ), + ( + "signed_in", + Emitter::Core, + &[ + ("provider_code", Prop::Id), + ("alliance_partner", Prop::Bool), + ("membership_tier", Prop::Id), + ("restored", Prop::Bool), + ], + ), + ( + "game_launch_requested", + Emitter::Core, + &[ + ("game_id", Prop::Id), + ("game_title", Prop::Text(120)), + ("store", Prop::Text(64)), + ("zone", Prop::Id), + ], + ), + ( + "session_started", + Emitter::Shell, + &[ + ("game_id", Prop::Id), + ("game_title", Prop::Text(120)), + ("codec", Prop::Choice(&["h264", "h265", "av1"])), + ("resolution", Prop::Resolution), + ("fps_target", Prop::Number(1_000.0)), + ("hdr", Prop::Bool), + ("decoder_backend", Prop::Code), + ("first_frame_ms", Prop::Number(3_600_000.0)), + ("queue_wait_s", Prop::Number(86_400.0)), + ], + ), + ( + "session_ended", + Emitter::Shell, + &[ + ("game_id", Prop::Id), + ("game_title", Prop::Text(120)), + ("duration_s", Prop::Number(604_800.0)), + ( + "outcome", + Prop::Choice(&["clean", "user_stopped", "remote_ended", "error"]), + ), + ("error_code", Prop::Code), + ("recoveries", Prop::Number(100_000.0)), + ("video_drop_count", Prop::Number(1e12)), + ("avg_fps", Prop::Number(1_000.0)), + ("avg_ping_ms", Prop::Number(100_000.0)), + ("avg_packet_loss_pct", Prop::Number(100.0)), + ("decoder_errors", Prop::Number(1e12)), + ], + ), + ( + "session_error", + Emitter::Shell, + &[ + ("stage", Prop::Choice(STAGES)), + ("code", Prop::Code), + ("game_id", Prop::Id), + ], + ), + ( + "frame_drops_detected", + Emitter::Shell, + &[ + ("dropped", Prop::Number(1e12)), + ("window_s", Prop::Number(3_600.0)), + ("game_id", Prop::Id), + ("decoder_backend", Prop::Code), + ], + ), + ( + "library_load_failed", + Emitter::Core, + &[("code", Prop::Code)], + ), + ( + "bug_report_sent", + Emitter::Core, + &[ + ("report_id", Prop::Id), + ("issue_id", Prop::Id), + ("kind", Prop::Choice(REPORT_KINDS)), + ("code", Prop::Code), + ("automatic", Prop::Bool), + ], + ), + ( + "consent_changed", + Emitter::Core, + &[ + ("value", Prop::Choice(&["enabled", "disabled"])), + ( + "source", + Prop::Choice(&["signin_notice", "first_run_sheet", "settings"]), + ), + ], + ), +]; + +pub struct ShellEvent { + pub name: &'static str, + pub props: Map, +} + +pub fn ui_surface(params: &Value) -> Result, String> { + match params["uiSurface"].as_str() { + None => Ok(None), + Some("desktop") => Ok(Some("desktop")), + Some("console") => Ok(Some("console")), + Some(_) => Err("Unsupported analytics surface".to_owned()), + } +} + +pub fn shell_event(params: &Value) -> Result { + let name = params["event"].as_str().unwrap_or_default(); + let (name, emitter, _) = EVENTS + .iter() + .find(|(event, _, _)| *event == name) + .ok_or_else(|| "Unsupported analytics event".to_owned())?; + if *emitter != Emitter::Shell { + return Err("Unsupported analytics event".to_owned()); + } + Ok(ShellEvent { + name, + props: sanitize(name, ¶ms["props"]), + }) +} + +pub fn sanitize(event: &str, props: &Value) -> Map { + let allowed = EVENTS + .iter() + .find(|(name, _, _)| *name == event) + .map_or(&[][..], |(_, _, props)| *props); + let mut clean = Map::new(); + for (key, prop) in allowed { + let value = &props[*key]; + let accepted = match prop { + Prop::Text(limit) => value + .as_str() + .map(|text| bounded_text(text, *limit)) + .filter(|text| !text.is_empty()) + .map(Value::from), + Prop::Id => value + .as_str() + .filter(|text| { + (1..=64).contains(&text.len()) + && text.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!(byte, b'_' | b'-' | b'.' | b':') + }) + }) + .map(Value::from), + Prop::Code => value + .as_str() + .filter(|text| valid_code(text)) + .map(Value::from), + Prop::Number(maximum) => value + .as_f64() + .filter(|number| number.is_finite() && (0.0..=*maximum).contains(number)) + .map(|number| (number * 100.0).round() / 100.0) + .map(|number| { + if number.fract() == 0.0 && number < 9_007_199_254_740_992.0 { + json!(number as u64) + } else { + json!(number) + } + }), + Prop::Bool => value.as_bool().map(Value::from), + Prop::Choice(choices) => value + .as_str() + .filter(|text| choices.contains(text)) + .map(Value::from), + Prop::Resolution => value + .as_str() + .filter(|text| { + text.split_once('x').is_some_and(|(width, height)| { + [width, height].iter().all(|part| { + (1..=5).contains(&part.len()) + && part.bytes().all(|b| b.is_ascii_digit()) + }) + }) + }) + .map(Value::from), + }; + if let Some(accepted) = accepted { + clean.insert((*key).to_owned(), accepted); + } + } + clean +} + +pub fn valid_code(value: &str) -> bool { + (1..=64).contains(&value.len()) + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} + +pub fn gpu_vendor(name: Option<&str>) -> &'static str { + let Some(name) = name.map(str::to_ascii_lowercase) else { + return "unknown"; + }; + [ + ("nvidia", &["nvidia", "geforce", "quadro", "rtx", "gtx"][..]), + ("amd", &["amd", "radeon", "ati "]), + ("intel", &["intel", "iris", "arc "]), + ("apple", &["apple"]), + ("qualcomm", &["qualcomm", "adreno"]), + ] + .iter() + .find(|(_, needles)| needles.iter().any(|needle| name.contains(needle))) + .map_or("other", |(vendor, _)| vendor) +} + +fn bounded_text(value: &str, limit: usize) -> String { + value + .chars() + .map(|character| { + if character.is_control() { + ' ' + } else { + character + } + }) + .take(limit) + .collect::() + .trim() + .to_owned() +} + +struct Queued { + install_id: String, + account: Value, + event: Value, +} + +struct State { + queue: VecDeque, + dropped: u64, + generation: u64, + ui_surface: &'static str, + signed_in_user: Option, + app_opened: bool, + device: Value, + shutdown: bool, +} + +pub struct AnalyticsService { + client: Client, + endpoint: String, + spool_path: PathBuf, + state: Mutex, + wake: Condvar, + spool_lock: Mutex<()>, +} + +impl AnalyticsService { + pub fn new(base: &str, data_dir: &Path) -> Result { + Ok(Self { + client: Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .build() + .map_err(|error| error.to_string())?, + endpoint: format!("{base}/v1/events"), + spool_path: data_dir.join("analytics").join("spool.jsonl"), + state: Mutex::new(State { + queue: VecDeque::new(), + dropped: 0, + generation: 0, + ui_surface: "desktop", + signed_in_user: None, + app_opened: false, + device: json!({"gpu":null, "decoderBackend":null}), + shutdown: false, + }), + wake: Condvar::new(), + spool_lock: Mutex::new(()), + }) + } + + pub fn start(self: &Arc) -> Result, String> { + let (done, finished) = mpsc::channel(); + let service = Arc::clone(self); + thread::Builder::new() + .name("opennow-analytics".to_owned()) + .spawn(move || { + service.run(); + let _ = done.send(()); + }) + .map_err(|error| error.to_string())?; + Ok(finished) + } + + pub fn shutdown(&self, finished: &mpsc::Receiver<()>) { + self.state.lock().expect("analytics poisoned").shutdown = true; + self.wake.notify_all(); + let _ = finished.recv_timeout(SHUTDOWN_BUDGET + Duration::from_millis(200)); + } + + pub fn enqueue( + &self, + install_id: &str, + account: Value, + event: &str, + props: Map, + ) -> u64 { + let mut state = self.state.lock().expect("analytics poisoned"); + let mut props = props; + props.insert( + "app_version".to_owned(), + json!(crate::version::APPLICATION_VERSION), + ); + props.insert("os".to_owned(), json!(std::env::consts::OS)); + props.insert("arch".to_owned(), json!(std::env::consts::ARCH)); + props.insert("ui_surface".to_owned(), json!(state.ui_surface)); + props.insert( + "channel".to_owned(), + json!(crate::version::update_channel( + crate::version::APPLICATION_VERSION + )), + ); + if state.queue.len() >= MAX_QUEUED_EVENTS { + state.queue.pop_front(); + state.dropped += 1; + } + state.queue.push_back(Queued { + install_id: install_id.to_owned(), + account, + event: json!({"event":event, "ts":now_ms(), "props":props}), + }); + if state.queue.len() >= FLUSH_THRESHOLD { + self.wake.notify_all(); + } + state.dropped + } + + pub fn wipe(&self) { + let _spool = self.spool_lock.lock().expect("analytics spool poisoned"); + let mut state = self.state.lock().expect("analytics poisoned"); + state.queue.clear(); + state.generation += 1; + drop(state); + let _ = fs::remove_file(&self.spool_path); + } + + pub fn set_ui_surface(&self, surface: &'static str) { + self.state.lock().expect("analytics poisoned").ui_surface = surface; + } + + pub fn observe_sign_in(&self, user_id: &str) -> bool { + let mut state = self.state.lock().expect("analytics poisoned"); + if state.signed_in_user.as_deref() == Some(user_id) { + return false; + } + state.signed_in_user = Some(user_id.to_owned()); + true + } + + pub fn observe_sign_out(&self) { + self.state + .lock() + .expect("analytics poisoned") + .signed_in_user = None; + } + + pub fn observe_runtime(&self, capabilities: &Value) { + let gpu = crate::streamer::active_gpu_label(capabilities); + let backend = capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool() == Some(true)) + .find_map(|backend| backend["backend"].as_str()) + .filter(|backend| valid_code(backend)); + if gpu.is_some() || backend.is_some() { + self.state.lock().expect("analytics poisoned").device = + json!({"gpu":gpu, "decoderBackend":backend}); + } + } + + pub fn open_app(&self, capabilities: &Value) -> Option> { + self.observe_runtime(capabilities); + let mut state = self.state.lock().expect("analytics poisoned"); + if std::mem::replace(&mut state.app_opened, true) { + return None; + } + let mut props = Map::new(); + props.insert( + "gpu_vendor".to_owned(), + json!(gpu_vendor(state.device["gpu"].as_str())), + ); + if let Some(kind) = package_kind() { + props.insert("package_kind".to_owned(), json!(kind)); + } + Some(props) + } + + pub fn device(&self) -> Value { + self.state + .lock() + .expect("analytics poisoned") + .device + .clone() + } + + fn run(&self) { + let generation = self.state.lock().expect("analytics poisoned").generation; + self.deliver(Vec::new(), generation, None); + loop { + let (batches, generation, deadline) = { + let mut state = self.state.lock().expect("analytics poisoned"); + let wake_at = Instant::now() + FLUSH_INTERVAL; + while !state.shutdown && state.queue.len() < FLUSH_THRESHOLD { + let now = Instant::now(); + if now >= wake_at { + break; + } + state = self + .wake + .wait_timeout(state, wake_at - now) + .expect("analytics poisoned") + .0; + } + let deadline = state.shutdown.then(|| Instant::now() + SHUTDOWN_BUDGET); + (batches(state.queue.drain(..)), state.generation, deadline) + }; + if !batches.is_empty() || deadline.is_none() { + self.deliver(batches, generation, deadline); + } + if deadline.is_some() { + return; + } + } + } + + fn deliver(&self, batches: Vec, generation: u64, deadline: Option) { + let mut failed = Vec::new(); + for batch in batches { + if !failed.is_empty() || self.post(&batch, deadline) == Delivery::Retry { + failed.push(batch); + } + } + if !failed.is_empty() { + self.append_spool(&failed, generation); + } else if deadline.is_none() { + self.resend_spool(generation); + } + } + + fn post(&self, batch: &Value, deadline: Option) -> Delivery { + let mut request = self.client.post(&self.endpoint).json(batch); + if let Some(deadline) = deadline { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Delivery::Retry; + } + request = request.timeout(remaining); + } + match request.send() { + Ok(response) if response.status().is_success() => Delivery::Sent, + Ok(response) + if response.status().is_client_error() + && !matches!(response.status().as_u16(), 408 | 429) => + { + Delivery::Rejected + } + _ => Delivery::Retry, + } + } + + fn append_spool(&self, batches: &[Value], generation: u64) { + let _spool = self.spool_lock.lock().expect("analytics spool poisoned"); + if self.state.lock().expect("analytics poisoned").generation != generation { + return; + } + let mut lines = read_lines(&self.spool_path); + lines.extend(batches.iter().map(Value::to_string)); + self.write_spool(lines); + } + + fn resend_spool(&self, generation: u64) { + let lines = { + let _spool = self.spool_lock.lock().expect("analytics spool poisoned"); + read_lines(&self.spool_path) + }; + if lines.is_empty() { + return; + } + let mut sent = 0; + for line in &lines { + let Ok(batch) = serde_json::from_str::(line) else { + sent += 1; + continue; + }; + if self.post(&batch, None) == Delivery::Retry { + break; + } + sent += 1; + } + let _spool = self.spool_lock.lock().expect("analytics spool poisoned"); + if self.state.lock().expect("analytics poisoned").generation != generation { + return; + } + let mut current = read_lines(&self.spool_path); + current.drain(..sent.min(current.len())); + self.write_spool(current); + } + + fn write_spool(&self, mut lines: Vec) { + let mut total = lines.iter().map(|line| line.len() + 1).sum::(); + let mut skip = 0; + while total > MAX_SPOOL_BYTES && skip < lines.len() { + total -= lines[skip].len() + 1; + skip += 1; + } + lines.drain(..skip); + if lines.is_empty() { + let _ = fs::remove_file(&self.spool_path); + return; + } + let Some(parent) = self.spool_path.parent() else { + return; + }; + let temporary = self.spool_path.with_extension("jsonl.tmp"); + let mut text = lines.join("\n"); + text.push('\n'); + if fs::create_dir_all(parent).is_ok() && fs::write(&temporary, text).is_ok() { + let _ = fs::rename(&temporary, &self.spool_path); + } + } +} + +#[derive(PartialEq, Eq)] +enum Delivery { + Sent, + Rejected, + Retry, +} + +fn batches(queue: impl Iterator) -> Vec { + let mut batches: Vec<(String, Value, Vec)> = Vec::new(); + for item in queue { + match batches.last_mut() { + Some((install_id, account, events)) + if *install_id == item.install_id + && *account == item.account + && events.len() < MAX_BATCH_EVENTS => + { + events.push(item.event) + } + _ => batches.push((item.install_id, item.account, vec![item.event])), + } + } + batches + .into_iter() + .map(|(install_id, account, events)| { + json!({"installId":install_id, "account":account, "events":events}) + }) + .collect() +} + +fn read_lines(path: &Path) -> Vec { + fs::read_to_string(path) + .map(|text| { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default() +} + +fn package_kind() -> Option<&'static str> { + if opennow_core::update_apply::external_update_message().is_some() { + return Some("flatpak"); + } + opennow_core::update_apply::compatible_package_extension().ok() +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpListener; + + const INSTALL: &str = "0123456789abcdef0123456789abcdef"; + + fn service(base: &str) -> (tempfile::TempDir, AnalyticsService) { + let directory = tempfile::tempdir().unwrap(); + let service = AnalyticsService::new(base, directory.path()).unwrap(); + (directory, service) + } + + fn server(statuses: Vec) -> (String, thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let worker = thread::spawn(move || { + let mut bodies = Vec::new(); + for status in statuses { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(&stream); + let mut length = 0; + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert_eq!(line.trim(), "POST /v1/events HTTP/1.1"); + loop { + line.clear(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + bodies.push(serde_json::from_slice(&body).unwrap()); + write!(stream, "HTTP/1.1 {status} Fixture\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{{}}").unwrap(); + } + bodies + }); + (base, worker) + } + + fn drain(service: &AnalyticsService) -> (Vec, u64) { + let mut state = service.state.lock().unwrap(); + (batches(state.queue.drain(..)), state.generation) + } + + #[test] + fn shell_events_are_allowlisted_and_validated() { + let event = shell_event(&json!({"event":"session_ended", "uiSurface":"console", "props":{ + "game_id":"100", "game_title":"Portal\u{0007} 2", "duration_s":61.256, "outcome":"clean", + "error_code":"Bad Code", "avg_fps":-1, "avg_ping_ms":23.4, "token":"secret", "recoveries":0.0, + "avg_packet_loss_pct":140 + }})) + .unwrap(); + assert_eq!(event.name, "session_ended"); + assert_eq!( + ui_surface(&json!({"uiSurface":"console"})), + Ok(Some("console")) + ); + assert!(ui_surface(&json!({"uiSurface":"web"})).is_err()); + assert_eq!( + Value::Object(event.props), + json!({"game_id":"100", "game_title":"Portal 2", "duration_s":61.26, + "outcome":"clean", "avg_ping_ms":23.4, "recoveries":0}) + ); + let started = shell_event(&json!({"event":"session_started", "props":{ + "resolution":"1920x1080", "codec":"h265", "hdr":true}})) + .unwrap(); + assert_eq!( + Value::Object(started.props), + json!({"codec":"h265", "resolution":"1920x1080", "hdr":true}) + ); + assert!( + shell_event(&json!({"event":"session_started", "props":{"resolution":"1920*1080"}})) + .unwrap() + .props + .is_empty() + ); + for rejected in [ + json!({"event":"app_opened"}), + json!({"event":"consent_changed"}), + json!({"event":"purchase"}), + ] { + assert!(shell_event(&rejected).is_err()); + } + assert_eq!( + sanitize( + "bug_report_sent", + &json!({"report_id":"br-7F3A21", "kind":"manual", + "code":"manual", "automatic":false, "issue_id":"iss_01J"}) + ), + json!({"report_id":"br-7F3A21", "issue_id":"iss_01J", "kind":"manual", + "code":"manual", "automatic":false}) + .as_object() + .unwrap() + .clone() + ); + } + + #[test] + fn gpu_vendors_are_coarse() { + assert_eq!(gpu_vendor(Some("NVIDIA GeForce RTX 4070")), "nvidia"); + assert_eq!(gpu_vendor(Some("AMD Radeon RX 7900")), "amd"); + assert_eq!(gpu_vendor(Some("Intel(R) Iris(R) Xe Graphics")), "intel"); + assert_eq!(gpu_vendor(Some("Virtual Display")), "other"); + assert_eq!(gpu_vendor(None), "unknown"); + } + + #[test] + fn the_queue_is_bounded_and_drops_the_oldest_events() { + let (_directory, service) = service("http://127.0.0.1:9"); + for index in 0..MAX_QUEUED_EVENTS + 3 { + let mut props = Map::new(); + props.insert("code".to_owned(), json!(format!("e{index}"))); + service.enqueue(INSTALL, Value::Null, "library_load_failed", props); + } + let state = service.state.lock().unwrap(); + assert_eq!(state.queue.len(), MAX_QUEUED_EVENTS); + assert_eq!(state.dropped, 3); + assert_eq!(state.queue[0].event["props"]["code"], "e3"); + assert_eq!(state.queue[0].event["props"]["ui_surface"], "desktop"); + } + + #[test] + fn batches_group_by_account_and_cap_at_one_hundred_events() { + let (_directory, service) = service("http://127.0.0.1:9"); + let account = json!({"userId":"sub", "providerIdpId":"idp"}); + for _ in 0..150 { + service.enqueue(INSTALL, account.clone(), "library_load_failed", Map::new()); + } + service.enqueue(INSTALL, Value::Null, "consent_changed", Map::new()); + let (batches, _) = drain(&service); + assert_eq!(batches.len(), 3); + assert_eq!(batches[0]["events"].as_array().unwrap().len(), 100); + assert_eq!(batches[1]["events"].as_array().unwrap().len(), 50); + assert_eq!(batches[2]["account"], Value::Null); + assert_eq!(batches[2]["installId"], INSTALL); + assert_eq!(batches[2]["events"][0]["event"], "consent_changed"); + assert!(batches[2]["events"][0]["ts"].as_u64().unwrap() > 0); + } + + #[test] + fn failed_batches_spool_and_resend_once_the_service_recovers() { + let (base, worker) = server(vec![503, 202, 202]); + let (_directory, service) = service(&base); + service.enqueue(INSTALL, Value::Null, "library_load_failed", Map::new()); + let (batches, generation) = drain(&service); + service.deliver(batches, generation, None); + assert_eq!(read_lines(&service.spool_path).len(), 1); + service.enqueue(INSTALL, Value::Null, "app_opened", Map::new()); + let (batches, generation) = drain(&service); + service.deliver(batches, generation, None); + assert!(!service.spool_path.exists()); + let bodies = worker.join().unwrap(); + assert_eq!(bodies[0]["events"][0]["event"], "library_load_failed"); + assert_eq!(bodies[1]["events"][0]["event"], "app_opened"); + assert_eq!(bodies[2], bodies[0]); + } + + #[test] + fn the_spool_is_capped_at_one_megabyte_keeping_the_newest_batches() { + let (_directory, service) = service("http://127.0.0.1:9"); + let line = "x".repeat(300 * 1024); + service.write_spool((0..5).map(|index| format!("{index}{line}")).collect()); + let lines = read_lines(&service.spool_path); + assert_eq!(lines.len(), 3); + assert!(lines[0].starts_with('2')); + assert!(fs::metadata(&service.spool_path).unwrap().len() <= MAX_SPOOL_BYTES as u64); + } + + #[test] + fn withdrawing_consent_wipes_the_queue_spool_and_in_flight_batches() { + let (_directory, service) = service("http://127.0.0.1:9"); + service.enqueue(INSTALL, Value::Null, "library_load_failed", Map::new()); + service.write_spool(vec![json!({"events":[]}).to_string()]); + let (batches, generation) = drain(&service); + service.enqueue(INSTALL, Value::Null, "library_load_failed", Map::new()); + service.wipe(); + assert!(service.state.lock().unwrap().queue.is_empty()); + assert!(!service.spool_path.exists()); + service.append_spool(&batches, generation); + assert!(!service.spool_path.exists()); + } + + #[test] + fn runtime_capabilities_open_the_app_once_and_describe_the_device() { + let (_directory, service) = service("http://127.0.0.1:9"); + let capabilities = json!({ + "graphicsAdapters":[{"name":"NVIDIA GeForce RTX 4070", "active":true}], + "videoBackends":[{"backend":"software","available":false},{"backend":"d3d11","available":true}] + }); + let opened = service.open_app(&capabilities).unwrap(); + assert_eq!(opened["gpu_vendor"], "nvidia"); + assert!(service.open_app(&capabilities).is_none()); + assert_eq!( + service.device(), + json!({"gpu":"NVIDIA GeForce RTX 4070", "decoderBackend":"d3d11"}) + ); + assert!(service.observe_sign_in("sub")); + assert!(!service.observe_sign_in("sub")); + service.observe_sign_out(); + assert!(service.observe_sign_in("sub")); + } +} diff --git a/external/opennow/native/opennow-core/src/artwork_cache.rs b/external/opennow/native/opennow-core/src/artwork_cache.rs new file mode 100644 index 000000000..70251c2fa --- /dev/null +++ b/external/opennow/native/opennow-core/src/artwork_cache.rs @@ -0,0 +1,1033 @@ +use crate::proxy::client_for_settings_with; +use rand::RngCore as _; +use reqwest::blocking::Client; +use reqwest::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs::{self, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::{Ipv4Addr, Ipv6Addr}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use url::Url; + +const CACHE_VERSION: u32 = 1; +const DEFAULT_MAX_BYTES: u64 = 512 * 1024 * 1024; +const DEFAULT_MAX_ENTRIES: usize = 4096; +const DEFAULT_MAX_OBJECT_BYTES: u64 = 8 * 1024 * 1024; +const DEFAULT_WORKERS: usize = 3; +const DEFAULT_QUEUE_CAPACITY: usize = 256; +const MAX_SOURCE_URL_BYTES: usize = 4096; +const FAILURE_RETRY_BASE: Duration = Duration::from_secs(30); +const FAILURE_RETRY_MAX: Duration = Duration::from_secs(30 * 60); +const INDEX_FLUSH_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Clone, Copy)] +struct CacheLimits { + max_bytes: u64, + max_entries: usize, + max_object_bytes: u64, + workers: usize, + queue_capacity: usize, + allow_private_network: bool, +} + +impl Default for CacheLimits { + fn default() -> Self { + Self { + max_bytes: DEFAULT_MAX_BYTES, + max_entries: DEFAULT_MAX_ENTRIES, + max_object_bytes: DEFAULT_MAX_OBJECT_BYTES, + workers: DEFAULT_WORKERS, + queue_capacity: DEFAULT_QUEUE_CAPACITY, + allow_private_network: false, + } + } +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct CacheIndex { + version: u32, + entries: BTreeMap, +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct CacheEntry { + extension: String, + size_bytes: u64, + last_access_ms: u64, +} + +struct IndexState { + index: CacheIndex, + dirty: bool, + last_flush: Instant, +} + +struct FailureState { + attempts: u32, + retry_at: Instant, +} + +struct SharedState { + objects_dir: Option, + index_path: Option, + limits: CacheLimits, + index: Mutex, + queued: Mutex>, + failures: Mutex>, + event_tx: mpsc::Sender, +} + +struct DownloadJob { + key: String, + source_url: String, + client: Client, +} + +struct CachedObject { + file_url: String, + file_path: String, +} + +enum ScheduleResult { + Pending, + Saturated, + RetryLater, + Unavailable, +} + +pub struct ArtworkCache { + base_client: Option, + state: Arc, + sender: Option>, + _workers: Vec>, +} + +impl ArtworkCache { + pub fn new(data_dir: &Path, event_tx: mpsc::Sender) -> Self { + Self::with_limits(data_dir, event_tx, CacheLimits::default()) + } + + fn with_limits(data_dir: &Path, event_tx: mpsc::Sender, limits: CacheLimits) -> Self { + let base_client = Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(12)) + .pool_idle_timeout(Duration::from_secs(30)) + .redirect(artwork_redirect_policy(limits.allow_private_network)) + .build() + .map_err(|error| { + eprintln!("artwork cache: HTTP client initialization failed: {error}"); + error + }) + .ok(); + let root = data_dir.join("artwork-cache").join("v1"); + let objects_dir = root.join("objects"); + let index_path = root.join("index.json"); + let enabled = fs::create_dir_all(&objects_dir) + .map_err(|error| { + eprintln!("artwork cache: storage initialization failed: {error}"); + error + }) + .is_ok(); + let state = Arc::new(SharedState { + objects_dir: enabled.then_some(objects_dir), + index_path: enabled.then_some(index_path), + limits, + index: Mutex::new(IndexState { + index: CacheIndex::default(), + dirty: false, + last_flush: Instant::now(), + }), + queued: Mutex::new(HashSet::new()), + failures: Mutex::new(HashMap::new()), + event_tx, + }); + if enabled { + state.load_and_reconcile(); + } + let (sender, receiver) = mpsc::sync_channel(limits.queue_capacity.max(1)); + let receiver = Arc::new(Mutex::new(receiver)); + let mut workers = Vec::new(); + if enabled { + for index in 0..limits.workers.max(1) { + let worker_state = Arc::clone(&state); + let worker_receiver = Arc::clone(&receiver); + match thread::Builder::new() + .name(format!("opennow-artwork-{index}")) + .spawn(move || worker_loop(worker_state, worker_receiver)) + { + Ok(worker) => workers.push(worker), + Err(error) => eprintln!("artwork cache: worker initialization failed: {error}"), + } + } + } + Self { + base_client, + state, + sender: enabled.then_some(sender), + _workers: workers, + } + } + + pub fn resolve(&self, params: &Value, settings: &Value) -> Result { + let source_url = validate_source_url( + params["sourceUrl"] + .as_str() + .or_else(|| params["url"].as_str()) + .unwrap_or_default(), + self.state.limits.allow_private_network, + )?; + let key = cache_key(&source_url); + if let Some(cached) = self.state.lookup(&key) { + return Ok(json!({ + "version":CACHE_VERSION, + "status":"ready", + "sourceUrl":source_url, + "artworkUrl":cached.file_url, + "filePath":cached.file_path, + "cached":true + })); + } + let Some(base_client) = self.base_client.as_ref() else { + return Ok(fallback(&source_url, "cache-unavailable")); + }; + let client = client_for_settings_with(base_client, settings, |builder| { + builder.redirect(artwork_redirect_policy( + self.state.limits.allow_private_network, + )) + })?; + let status = self.schedule(DownloadJob { + key, + source_url: source_url.clone(), + client, + }); + Ok(match status { + ScheduleResult::Pending => json!({ + "version":CACHE_VERSION, + "status":"pending", + "sourceUrl":source_url + }), + ScheduleResult::Saturated => fallback(&source_url, "queue-full"), + ScheduleResult::RetryLater => fallback(&source_url, "retry-later"), + ScheduleResult::Unavailable => fallback(&source_url, "cache-unavailable"), + }) + } + + fn schedule(&self, job: DownloadJob) -> ScheduleResult { + let Some(sender) = self.sender.as_ref() else { + return ScheduleResult::Unavailable; + }; + { + let failures = self + .state + .failures + .lock() + .expect("artwork failures poisoned"); + if failures + .get(&job.key) + .is_some_and(|failure| failure.retry_at > Instant::now()) + { + return ScheduleResult::RetryLater; + } + } + { + let mut queued = self.state.queued.lock().expect("artwork queue poisoned"); + if !queued.insert(job.key.clone()) { + return ScheduleResult::Pending; + } + } + let key = job.key.clone(); + match sender.try_send(job) { + Ok(()) => ScheduleResult::Pending, + Err(mpsc::TrySendError::Full(_)) => { + self.state + .queued + .lock() + .expect("artwork queue poisoned") + .remove(&key); + ScheduleResult::Saturated + } + Err(mpsc::TrySendError::Disconnected(_)) => { + self.state + .queued + .lock() + .expect("artwork queue poisoned") + .remove(&key); + ScheduleResult::Unavailable + } + } + } +} + +impl Drop for ArtworkCache { + fn drop(&mut self) { + self.sender.take(); + self.state.flush_index(); + } +} + +impl SharedState { + fn load_and_reconcile(&self) { + let Some(objects_dir) = self.objects_dir.as_ref() else { + return; + }; + let mut index_state = self.index.lock().expect("artwork index poisoned"); + index_state.index = self + .index_path + .as_ref() + .and_then(|path| load_index(path)) + .unwrap_or_default(); + index_state.index.version = CACHE_VERSION; + let mut discovered = BTreeMap::new(); + if let Ok(files) = fs::read_dir(objects_dir) { + for file in files.flatten() { + let path = file.path(); + let Some((key, extension)) = parse_object_name(&file.file_name().to_string_lossy()) + else { + let _ = fs::remove_file(path); + continue; + }; + let Ok(metadata) = file.metadata() else { + continue; + }; + if !metadata.is_file() || metadata.len() > self.limits.max_object_bytes { + let _ = fs::remove_file(path); + continue; + } + let previous = index_state.index.entries.get(&key); + discovered.insert( + key, + CacheEntry { + extension, + size_bytes: metadata.len(), + last_access_ms: previous + .map(|entry| entry.last_access_ms) + .unwrap_or_else(|| modified_ms(&metadata)), + }, + ); + } + } + index_state.index.entries = discovered; + prune_to_limits(self, &mut index_state.index, 0, 0); + index_state.dirty = true; + self.persist_locked(&mut index_state); + } + + fn lookup(&self, key: &str) -> Option { + let objects_dir = self.objects_dir.as_ref()?; + let mut index_state = self.index.lock().expect("artwork index poisoned"); + let entry = index_state.index.entries.get(key)?.clone(); + let path = objects_dir.join(object_name(key, &entry.extension)); + let metadata = fs::metadata(&path).ok(); + if !metadata.as_ref().is_some_and(|value| { + value.is_file() + && value.len() == entry.size_bytes + && value.len() <= self.limits.max_object_bytes + }) { + index_state.index.entries.remove(key); + index_state.dirty = true; + return None; + } + if let Some(entry) = index_state.index.entries.get_mut(key) { + entry.last_access_ms = now_ms(); + } + index_state.dirty = true; + if index_state.last_flush.elapsed() >= INDEX_FLUSH_INTERVAL { + self.persist_locked(&mut index_state); + } + Some(CachedObject { + file_url: Url::from_file_path(&path).ok()?.to_string(), + file_path: path.to_string_lossy().into_owned(), + }) + } + + fn store_download( + &self, + key: &str, + extension: &str, + temporary: &Path, + size_bytes: u64, + ) -> io::Result { + let objects_dir = self + .objects_dir + .as_ref() + .ok_or_else(|| io::Error::other("cache unavailable"))?; + let mut index_state = self.index.lock().expect("artwork index poisoned"); + if !prune_to_limits(self, &mut index_state.index, size_bytes, 1) { + return Err(io::Error::other("cache limit could not be enforced")); + } + let path = objects_dir.join(object_name(key, extension)); + fs::rename(temporary, &path)?; + index_state.index.entries.insert( + key.to_owned(), + CacheEntry { + extension: extension.to_owned(), + size_bytes, + last_access_ms: now_ms(), + }, + ); + index_state.dirty = true; + self.persist_locked(&mut index_state); + Ok(CachedObject { + file_url: Url::from_file_path(&path) + .map_err(|_| io::Error::other("cache path is not a file URL"))? + .to_string(), + file_path: path.to_string_lossy().into_owned(), + }) + } + + fn persist_locked(&self, index_state: &mut IndexState) { + if !index_state.dirty { + return; + } + let Some(path) = self.index_path.as_ref() else { + return; + }; + match write_index_atomic(path, &index_state.index) { + Ok(()) => { + index_state.dirty = false; + index_state.last_flush = Instant::now(); + } + Err(error) => eprintln!("artwork cache: index write failed: {error}"), + } + } + + fn flush_index(&self) { + let mut index_state = self.index.lock().expect("artwork index poisoned"); + self.persist_locked(&mut index_state); + } + + fn record_failure(&self, key: &str) { + let mut failures = self.failures.lock().expect("artwork failures poisoned"); + let attempts = failures + .get(key) + .map(|failure| failure.attempts.saturating_add(1)) + .unwrap_or(1); + let factor = 1_u32 << attempts.saturating_sub(1).min(6); + let delay = FAILURE_RETRY_BASE + .checked_mul(factor) + .unwrap_or(FAILURE_RETRY_MAX) + .min(FAILURE_RETRY_MAX); + failures.insert( + key.to_owned(), + FailureState { + attempts, + retry_at: Instant::now() + delay, + }, + ); + } +} + +fn worker_loop(state: Arc, receiver: Arc>>) { + loop { + let job = receiver.lock().expect("artwork receiver poisoned").recv(); + let Ok(job) = job else { + break; + }; + let result = download(&state, &job); + let payload = match result { + Ok(cached) => { + state + .failures + .lock() + .expect("artwork failures poisoned") + .remove(&job.key); + json!({ + "version":CACHE_VERSION, + "sourceUrl":job.source_url, + "artworkUrl":cached.file_url, + "filePath":cached.file_path, + "cached":true + }) + } + Err(reason) => { + state.record_failure(&job.key); + json!({ + "version":CACHE_VERSION, + "sourceUrl":job.source_url, + "artworkUrl":job.source_url, + "cached":false, + "fallbackReason":reason + }) + } + }; + state + .queued + .lock() + .expect("artwork queue poisoned") + .remove(&job.key); + let _ = state + .event_tx + .send(json!({"type":"event", "name":"artwork.ready", "payload":payload})); + } +} + +fn download(state: &SharedState, job: &DownloadJob) -> Result { + let objects_dir = state.objects_dir.as_ref().ok_or("cache-unavailable")?; + let response = job + .client + .get(&job.source_url) + .header( + ACCEPT, + "image/avif,image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1", + ) + .send() + .map_err(|_| "network-error")?; + if !response.status().is_success() { + return Err("http-error"); + } + if validate_response_url(response.url(), state.limits.allow_private_network).is_err() { + return Err("unsafe-redirect"); + } + if response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|size| size > state.limits.max_object_bytes) + { + return Err("too-large"); + } + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_ascii_lowercase()); + if content_type.as_deref().is_some_and(|value| { + !value.starts_with("image/") && !value.starts_with("application/octet-stream") + }) { + return Err("unsupported-content"); + } + let temporary = temporary_path(objects_dir, &job.key); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + .map_err(|_| "storage-error")?; + let mut reader = response; + let mut buffer = [0_u8; 32 * 1024]; + let mut prefix = Vec::with_capacity(16); + let mut total = 0_u64; + let write_result = (|| { + loop { + let read = reader.read(&mut buffer).map_err(|_| "network-error")?; + if read == 0 { + break; + } + total = total.saturating_add(read as u64); + if total > state.limits.max_object_bytes { + return Err("too-large"); + } + if prefix.len() < 16 { + let needed = 16 - prefix.len(); + prefix.extend_from_slice(&buffer[..read.min(needed)]); + } + file.write_all(&buffer[..read]) + .map_err(|_| "storage-error")?; + } + if total == 0 { + return Err("empty-response"); + } + file.sync_all().map_err(|_| "storage-error")?; + Ok(()) + })(); + if let Err(error) = write_result { + drop(file); + let _ = fs::remove_file(&temporary); + return Err(error); + } + drop(file); + let extension = sniff_extension(&prefix).ok_or_else(|| { + let _ = fs::remove_file(&temporary); + "unsupported-content" + })?; + let result = state + .store_download(&job.key, extension, &temporary, total) + .map_err(|_| "storage-error"); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn validate_source_url(raw: &str, allow_private_network: bool) -> Result { + let source_url = raw.trim(); + if source_url.is_empty() || source_url.len() > MAX_SOURCE_URL_BYTES { + return Err("Artwork sourceUrl must be a non-empty HTTP(S) URL".to_owned()); + } + let parsed = Url::parse(source_url) + .map_err(|_| "Artwork sourceUrl must be a valid HTTP(S) URL".to_owned())?; + validate_response_url(&parsed, allow_private_network)?; + Ok(source_url.to_owned()) +} + +fn artwork_redirect_policy(allow_private_network: bool) -> reqwest::redirect::Policy { + reqwest::redirect::Policy::custom(move |attempt| { + if attempt.previous().len() >= 5 { + return attempt.error("too many artwork redirects"); + } + if validate_response_url(attempt.url(), allow_private_network).is_err() { + return attempt.error("unsafe artwork redirect"); + } + attempt.follow() + }) +} + +fn validate_response_url(url: &Url, allow_private_network: bool) -> Result<(), String> { + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + { + return Err("Artwork URLs must use HTTP(S) without embedded credentials".to_owned()); + } + if !allow_private_network && !public_artwork_host(url) { + return Err("Artwork URLs must not target a local or private network".to_owned()); + } + Ok(()) +} + +fn public_artwork_host(url: &Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(address)) => public_ipv4(address), + Some(url::Host::Ipv6(address)) => public_ipv6(address), + Some(url::Host::Domain(domain)) => { + let domain = domain.trim_end_matches('.').to_ascii_lowercase(); + domain != "localhost" && !domain.ends_with(".localhost") && !domain.ends_with(".local") + } + None => false, + } +} + +fn public_ipv4(address: Ipv4Addr) -> bool { + let octets = address.octets(); + !address.is_private() + && !address.is_loopback() + && !address.is_link_local() + && !address.is_broadcast() + && !address.is_documentation() + && !address.is_unspecified() + && !address.is_multicast() + && octets[0] != 0 + && !(octets[0] == 100 && (64..=127).contains(&octets[1])) + && !(octets[0] == 198 && (octets[1] == 18 || octets[1] == 19)) +} + +fn public_ipv6(address: Ipv6Addr) -> bool { + if let Some(mapped) = address.to_ipv4_mapped() { + return public_ipv4(mapped); + } + let segments = address.segments(); + !(address.is_loopback() + || address.is_unspecified() + || address.is_multicast() + || address.is_unique_local() + || address.is_unicast_link_local() + || (segments[0] == 0x2001 && segments[1] == 0x0db8)) +} + +fn fallback(source_url: &str, reason: &str) -> Value { + json!({ + "version":CACHE_VERSION, + "status":"fallback", + "sourceUrl":source_url, + "artworkUrl":source_url, + "cached":false, + "fallbackReason":reason + }) +} + +fn cache_key(source_url: &str) -> String { + format!("{:x}", Sha256::digest(source_url.as_bytes())) +} + +fn object_name(key: &str, extension: &str) -> String { + format!("{key}.{extension}") +} + +fn parse_object_name(name: &str) -> Option<(String, String)> { + let (key, extension) = name.rsplit_once('.')?; + if key.len() != 64 + || !key.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !matches!(extension, "jpg" | "png" | "webp" | "gif" | "avif") + { + return None; + } + Some((key.to_owned(), extension.to_owned())) +} + +fn temporary_path(objects_dir: &Path, key: &str) -> PathBuf { + let mut nonce = [0_u8; 8]; + rand::rng().fill_bytes(&mut nonce); + let nonce = nonce + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + objects_dir.join(format!(".{key}-{nonce}.part")) +} + +fn sniff_extension(bytes: &[u8]) -> Option<&'static str> { + if bytes.starts_with(&[0xff, 0xd8, 0xff]) { + Some("jpg") + } else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + Some("png") + } else if bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + Some("webp") + } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + Some("gif") + } else if bytes.len() >= 12 + && &bytes[4..8] == b"ftyp" + && matches!(&bytes[8..12], b"avif" | b"avis") + { + Some("avif") + } else { + None + } +} + +fn load_index(path: &Path) -> Option { + [path.to_path_buf(), path.with_extension("json.backup")] + .into_iter() + .find_map(|candidate| { + fs::read(&candidate) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter(|index| index.version == CACHE_VERSION) + }) +} + +fn write_index_atomic(path: &Path, index: &CacheIndex) -> io::Result<()> { + let bytes = serde_json::to_vec(index).map_err(io::Error::other)?; + let temporary = path.with_extension("json.tmp"); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + match fs::rename(&temporary, path) { + Ok(()) => Ok(()), + Err(_first_error) if path.exists() => { + let backup = path.with_extension("json.backup"); + let _ = fs::remove_file(&backup); + fs::rename(path, &backup)?; + if let Err(error) = fs::rename(&temporary, path) { + let _ = fs::rename(&backup, path); + return Err(error); + } + let _ = fs::remove_file(backup); + Ok(()) + } + Err(error) => Err(error), + } +} + +fn prune_to_limits( + state: &SharedState, + index: &mut CacheIndex, + incoming_bytes: u64, + incoming_entries: usize, +) -> bool { + let mut total_bytes = index + .entries + .values() + .map(|entry| entry.size_bytes) + .sum::(); + let mut entries = index.entries.len(); + let mut oldest = index + .entries + .iter() + .map(|(key, entry)| (entry.last_access_ms, key.clone())) + .collect::>(); + oldest.sort(); + for (_, key) in oldest { + if total_bytes.saturating_add(incoming_bytes) <= state.limits.max_bytes + && entries.saturating_add(incoming_entries) <= state.limits.max_entries + { + break; + } + let Some(entry) = index.entries.get(&key).cloned() else { + continue; + }; + let Some(objects_dir) = state.objects_dir.as_ref() else { + return false; + }; + let path = objects_dir.join(object_name(&key, &entry.extension)); + if fs::remove_file(&path).is_ok() || !path.exists() { + index.entries.remove(&key); + total_bytes = total_bytes.saturating_sub(entry.size_bytes); + entries = entries.saturating_sub(1); + } + } + total_bytes.saturating_add(incoming_bytes) <= state.limits.max_bytes + && entries.saturating_add(incoming_entries) <= state.limits.max_entries +} + +fn modified_ms(metadata: &fs::Metadata) -> u64 { + metadata + .modified() + .ok() + .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) + .map(|value| value.as_millis() as u64) + .unwrap_or_else(now_ms) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn temporary_directory(name: &str) -> PathBuf { + let mut nonce = [0_u8; 8]; + rand::rng().fill_bytes(&mut nonce); + let suffix = nonce + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let path = std::env::temp_dir().join(format!("opennow-artwork-{name}-{suffix}")); + fs::create_dir_all(&path).unwrap(); + path + } + + fn test_limits(max_bytes: u64, max_object_bytes: u64) -> CacheLimits { + CacheLimits { + max_bytes, + max_entries: 16, + max_object_bytes, + workers: 1, + queue_capacity: 4, + allow_private_network: true, + } + } + + fn serve_once(body: Vec, content_type: &str) -> (String, Arc) { + serve_once_after(body, content_type, Duration::ZERO) + } + + fn serve_once_after( + body: Vec, + content_type: &str, + delay: Duration, + ) -> (String, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let server_count = Arc::clone(&count); + let content_type = content_type.to_owned(); + thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request); + server_count.fetch_add(1, Ordering::SeqCst); + thread::sleep(delay); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .unwrap(); + stream.write_all(&body).unwrap(); + }); + ( + format!("http://{address}/artwork/../image.jpg?width=900"), + count, + ) + } + + fn jpeg(size: usize) -> Vec { + let mut bytes = vec![0_u8; size.max(4)]; + bytes[..4].copy_from_slice(&[0xff, 0xd8, 0xff, 0xd9]); + bytes + } + + fn wait_for_event(receiver: &mpsc::Receiver) -> Value { + receiver.recv_timeout(Duration::from_secs(3)).unwrap() + } + + #[test] + fn only_safe_http_urls_are_accepted_and_paths_are_hashed() { + for value in [ + "file:///tmp/image.jpg", + "data:image/png;base64,AAAA", + "https://user:secret@example.com/image.jpg", + "../image.jpg", + ] { + assert!( + validate_source_url(value, false).is_err(), + "accepted {value}" + ); + } + for value in [ + "http://127.0.0.1/image.jpg", + "http://[::1]/image.jpg", + "http://[::ffff:127.0.0.1]/image.jpg", + "http://[::ffff:10.0.0.1]/image.jpg", + "http://[2001:db8::1]/image.jpg", + "http://169.254.169.254/latest/meta-data", + "https://artwork.local/image.jpg", + ] { + assert!( + validate_source_url(value, false).is_err(), + "accepted {value}" + ); + } + let source = "https://img.nvidiagrid.net/../../escape.jpg;f=webp;w=900"; + assert_eq!(validate_source_url(source, false).unwrap(), source); + let key = cache_key(source); + assert_eq!(key.len(), 64); + assert!(!object_name(&key, "webp").contains("..")); + } + + #[test] + fn resolve_does_not_wait_for_the_network_worker() { + let root = temporary_directory("nonblocking"); + let (source_url, _) = serve_once_after(jpeg(64), "image/jpeg", Duration::from_millis(300)); + let (event_tx, event_rx) = mpsc::channel(); + let cache = ArtworkCache::with_limits(&root, event_tx, test_limits(1024, 128)); + let started = Instant::now(); + let result = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + assert_eq!(result["status"], "pending"); + assert!(started.elapsed() < Duration::from_millis(100)); + assert_eq!(wait_for_event(&event_rx)["payload"]["cached"], true); + drop(cache); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn miss_returns_pending_then_persists_an_atomic_file() { + let root = temporary_directory("persistent"); + let (source_url, requests) = serve_once(jpeg(128), "image/jpeg"); + let (event_tx, event_rx) = mpsc::channel(); + { + let cache = ArtworkCache::with_limits(&root, event_tx.clone(), test_limits(1024, 512)); + let result = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + assert_eq!(result["status"], "pending"); + assert!(result.get("artworkUrl").is_none()); + let event = wait_for_event(&event_rx); + assert_eq!(event["name"], "artwork.ready"); + assert_eq!(event["payload"]["cached"], true); + assert!( + event["payload"]["artworkUrl"] + .as_str() + .unwrap() + .starts_with("file:") + ); + } + let cache = ArtworkCache::with_limits(&root, event_tx, test_limits(1024, 512)); + let hit = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + assert_eq!(hit["status"], "ready"); + assert_eq!(hit["cached"], true); + assert_eq!(requests.load(Ordering::SeqCst), 1); + let objects = fs::read_dir(root.join("artwork-cache/v1/objects")) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(objects.len(), 1); + assert!(objects[0].ends_with(".jpg")); + assert!(!objects[0].contains(".part")); + drop(cache); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn oversized_download_emits_safe_remote_fallback() { + let root = temporary_directory("oversized"); + let (source_url, _) = serve_once(jpeg(256), "image/jpeg"); + let (event_tx, event_rx) = mpsc::channel(); + let cache = ArtworkCache::with_limits(&root, event_tx, test_limits(1024, 64)); + let result = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + assert_eq!(result["status"], "pending"); + let event = wait_for_event(&event_rx); + assert_eq!(event["payload"]["cached"], false); + assert_eq!(event["payload"]["artworkUrl"], source_url); + assert_eq!(event["payload"]["fallbackReason"], "too-large"); + assert!( + fs::read_dir(root.join("artwork-cache/v1/objects")) + .unwrap() + .next() + .is_none() + ); + drop(cache); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn pruning_removes_the_least_recently_resolved_entry() { + let root = temporary_directory("pruning"); + let (first_url, _) = serve_once(jpeg(80), "image/jpeg"); + let (event_tx, event_rx) = mpsc::channel(); + let cache = ArtworkCache::with_limits(&root, event_tx, test_limits(120, 100)); + cache + .resolve(&json!({"sourceUrl":first_url}), &json!({})) + .unwrap(); + wait_for_event(&event_rx); + thread::sleep(Duration::from_millis(2)); + let (second_url, _) = serve_once(jpeg(80), "image/jpeg"); + cache + .resolve(&json!({"sourceUrl":second_url}), &json!({})) + .unwrap(); + wait_for_event(&event_rx); + assert!(cache.state.lookup(&cache_key(&first_url)).is_none()); + assert!(cache.state.lookup(&cache_key(&second_url)).is_some()); + let index = cache.state.index.lock().unwrap(); + assert_eq!(index.index.entries.len(), 1); + assert!( + index + .index + .entries + .values() + .map(|entry| entry.size_bytes) + .sum::() + <= 120 + ); + drop(index); + drop(cache); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn duplicate_resolves_share_one_background_download() { + let root = temporary_directory("dedupe"); + let (source_url, requests) = serve_once(jpeg(64), "image/jpeg"); + let (event_tx, event_rx) = mpsc::channel(); + let cache = ArtworkCache::with_limits(&root, event_tx, test_limits(1024, 128)); + let first = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + let second = cache + .resolve(&json!({"sourceUrl":source_url}), &json!({})) + .unwrap(); + assert_eq!(first["status"], "pending"); + assert_eq!(second["status"], "pending"); + wait_for_event(&event_rx); + assert_eq!(requests.load(Ordering::SeqCst), 1); + assert!(event_rx.try_recv().is_err()); + drop(cache); + let _ = fs::remove_dir_all(root); + } +} diff --git a/external/opennow/native/opennow-core/src/bin/opennow-acceptance-verify.rs b/external/opennow/native/opennow-core/src/bin/opennow-acceptance-verify.rs new file mode 100644 index 000000000..532ee528c --- /dev/null +++ b/external/opennow/native/opennow-core/src/bin/opennow-acceptance-verify.rs @@ -0,0 +1,1105 @@ +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::env; +use std::fs::{self, OpenOptions}; +use std::io::{BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::time::{SystemTime, UNIX_EPOCH}; + +const MAX_JSON_BYTES: u64 = 4 * 1024 * 1024; +const REQUIRED_ATTESTATIONS: &[&str] = &[ + "deviceLogin", + "accountSwitching", + "subscriptionRefresh", + "regionRefresh", + "guideVideoOrdering", + "keyboardInput", + "relativeMouseInput", + "controllerInput", + "controllerNeutralState", + "windowResize", + "fullscreen", + "displayMigration", + "highestRefreshRate", + "screenshotVisible", + "recordingPlayback", + "thumbnailGenerated", + "mediaReveal", + "networkRecovery", + "streamerRecovery", + "terminalErrorUsable", + "installUpgradeUninstall", + "desktopAndProtocolAssociations", +]; + +#[derive(Debug)] +struct Arguments { + live: PathBuf, + performance_1080p: PathBuf, + performance_4k: PathBuf, + attestations: PathBuf, + packages: Vec, + output: PathBuf, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MatrixOs { + Windows, + Macos, + Linux, +} + +#[derive(Clone, Copy, Debug)] +struct MatrixRow { + os: MatrixOs, + architecture: &'static str, + window_system: &'static [&'static str], +} + +fn main() -> ExitCode { + let raw_arguments = env::args_os().skip(1).collect::>(); + if raw_arguments + .iter() + .any(|value| value == "--help" || value == "-h") + { + println!("{}", usage()); + return ExitCode::SUCCESS; + } + let arguments = match parse_arguments(raw_arguments.into_iter()) { + Ok(arguments) => arguments, + Err(message) => { + eprintln!("{message}\n\n{}", usage()); + return ExitCode::from(2); + } + }; + match verify(&arguments) { + Ok(passed) => ExitCode::from(if passed { 0 } else { 1 }), + Err(message) => { + eprintln!("Acceptance verification could not run: {message}"); + ExitCode::from(2) + } + } +} + +fn usage() -> &'static str { + concat!( + "Usage: opennow-acceptance-verify \\\n", + " --live --performance-1080p <1080p.json> \\\n", + " --performance-4k <4k.json> --attestations \\\n", + " --package [--package ...] --output " + ) +} + +fn parse_arguments( + arguments: impl Iterator, +) -> Result { + let mut values: BTreeMap> = BTreeMap::new(); + let mut arguments = arguments.peekable(); + while let Some(flag) = arguments.next() { + let flag = flag + .into_string() + .map_err(|_| "Arguments must be valid Unicode".to_owned())?; + if !matches!( + flag.as_str(), + "--live" + | "--performance-1080p" + | "--performance-4k" + | "--attestations" + | "--package" + | "--output" + ) { + return Err(format!("Unknown argument: {flag}")); + } + let value = arguments + .next() + .ok_or_else(|| format!("Missing value for {flag}"))?; + values.entry(flag).or_default().push(PathBuf::from(value)); + } + let one = |flag: &str| -> Result { + let found = values.get(flag).cloned().unwrap_or_default(); + if found.len() != 1 { + return Err(format!("{flag} must be supplied exactly once")); + } + Ok(found[0].clone()) + }; + let packages = values.get("--package").cloned().unwrap_or_default(); + if packages.is_empty() { + return Err("At least one --package is required".to_owned()); + } + Ok(Arguments { + live: one("--live")?, + performance_1080p: one("--performance-1080p")?, + performance_4k: one("--performance-4k")?, + attestations: one("--attestations")?, + packages, + output: one("--output")?, + }) +} + +fn verify(arguments: &Arguments) -> Result { + if !arguments.output.is_absolute() { + return Err("--output must be an absolute path".to_owned()); + } + let live = read_json(&arguments.live)?; + let performance_1080p = read_json(&arguments.performance_1080p)?; + let performance_4k = read_json(&arguments.performance_4k)?; + let attestations = read_json(&arguments.attestations)?; + + let mut failures = Vec::new(); + validate_live(&live, &mut failures); + validate_performance(&performance_1080p, 1920, 1080, &mut failures); + validate_performance(&performance_4k, 3840, 2160, &mut failures); + + let matrix_name = attestations["matrixRow"].as_str().unwrap_or_default(); + let matrix = matrix_row(matrix_name); + if matrix.is_none() { + failures.push("unsupported matrixRow".to_owned()); + } + validate_attestations(&attestations, matrix, &mut failures); + validate_cross_report( + &live, + &performance_1080p, + &performance_4k, + &attestations, + matrix, + &mut failures, + ); + + let package_evidence = hash_packages(&arguments.packages)?; + validate_packages(&attestations, matrix, &package_evidence, &mut failures); + failures.sort(); + failures.dedup(); + let passed = failures.is_empty(); + let report = json!({ + "schemaVersion": 1, + "kind": "opennow.acceptance-verification", + "pass": passed, + "matrixRow": matrix.map(|_| matrix_name), + "releaseCandidate": safe_output_label(&attestations["releaseCandidate"]), + "applicationVersion": safe_output_label(&live["applicationVersion"]), + "inputs": { + "live": file_evidence(&arguments.live)?, + "performance1080p": file_evidence(&arguments.performance_1080p)?, + "performance4k": file_evidence(&arguments.performance_4k)?, + "attestations": file_evidence(&arguments.attestations)?, + "packages": package_evidence.values().cloned().collect::>() + }, + "failures": failures, + "policy": "all-machine-observations-performance-reports-manual-attestations-packages" + }); + write_new_json(&arguments.output, &report)?; + println!( + "Acceptance verification {}: {}", + if passed { "passed" } else { "failed" }, + arguments.output.display() + ); + Ok(passed) +} + +fn read_json(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("Could not inspect {}: {error}", path.display()))?; + if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "JSON input must be a regular non-symlink file: {}", + path.display() + )); + } + if metadata.len() == 0 || metadata.len() > MAX_JSON_BYTES { + return Err(format!("JSON input size is invalid: {}", path.display())); + } + let bytes = + fs::read(path).map_err(|error| format!("Could not read {}: {error}", path.display()))?; + serde_json::from_slice(&bytes) + .map_err(|error| format!("Invalid JSON in {}: {error}", path.display())) +} + +fn validate_live(value: &Value, failures: &mut Vec) { + require( + value["schemaVersion"] == 1, + "live schemaVersion must be 1", + failures, + ); + require( + value["kind"] == "opennow.live-acceptance", + "live kind is invalid", + failures, + ); + require( + value["observedPass"] == true, + "live observedPass is false", + failures, + ); + require( + value["scope"] == "machine-observed-live-runtime", + "live scope is invalid", + failures, + ); + require( + safe_label(value["applicationVersion"].as_str().unwrap_or_default()), + "live applicationVersion is missing or invalid", + failures, + ); + require( + !contains_sensitive(value), + "live evidence contains a sensitive key or value", + failures, + ); + let checks = value["checks"].as_object(); + require(checks.is_some(), "live checks object is missing", failures); + if let Some(checks) = checks { + for name in [ + "streamingTenMinutes", + "firstFramePresented", + "nvstTransportActive", + "nativeInputReady", + "inputOwnershipExercised", + "allGuidePagesVisited", + "surfaceReconfigured", + "fullscreenControlExercised", + "statsControlExercised", + "recordingRoundTrip", + "mediaArtifactsComplete", + "networkRecoveryExercised", + "streamerRecoveryExercised", + "noTerminalMediaError", + "deviceRecoveryBalanced", + ] { + require( + checks.get(name) == Some(&Value::Bool(true)), + &format!("live check did not pass: {name}"), + failures, + ); + } + } + require( + value["stream"]["status"] == "streaming", + "live stream was not active", + failures, + ); + require( + value["stream"]["transport"] == "nvst", + "live stream transport is not NVST", + failures, + ); + require( + value["stream"]["sessionUptimeMs"] + .as_u64() + .unwrap_or_default() + >= 600_000, + "live stream duration is under ten minutes", + failures, + ); + require_media_item(&value["media"]["screenshot"], false, failures); + require_media_item(&value["media"]["recording"], true, failures); +} + +fn require_media_item(value: &Value, thumbnail: bool, failures: &mut Vec) { + require( + value.is_object(), + "live media artifact is missing", + failures, + ); + require( + safe_file_name(value["fileName"].as_str().unwrap_or_default()), + "live media filename is invalid", + failures, + ); + require( + value["sizeBytes"].as_u64().unwrap_or_default() > 0, + "live media artifact is empty", + failures, + ); + require( + valid_sha(value["sha256"].as_str().unwrap_or_default()), + "live media SHA-256 is invalid", + failures, + ); + if thumbnail { + require_media_item(&value["thumbnail"], false, failures); + } +} + +fn validate_performance(value: &Value, width: u64, height: u64, failures: &mut Vec) { + let prefix = format!("{width}x{height} performance"); + require( + value["schemaVersion"] == 1, + &format!("{prefix} schemaVersion must be 1"), + failures, + ); + require( + value["kind"] == "opennow.qt.performance", + &format!("{prefix} kind is invalid"), + failures, + ); + require( + value["pass"] == true, + &format!("{prefix} report did not pass"), + failures, + ); + require( + value["failures"].as_array().is_some_and(Vec::is_empty), + &format!("{prefix} contains failures"), + failures, + ); + let environment = &value["environment"]; + for field in ["requestedPhysicalWidth", "actualPhysicalWidth"] { + require( + environment[field].as_u64() == Some(width), + &format!("{prefix} {field} is wrong"), + failures, + ); + } + for field in ["requestedPhysicalHeight", "actualPhysicalHeight"] { + require( + environment[field].as_u64() == Some(height), + &format!("{prefix} {field} is wrong"), + failures, + ); + } + require( + environment["refreshRateOverrideHz"].is_null(), + &format!("{prefix} used a refresh override"), + failures, + ); + require( + safe_label(environment["machineLabel"].as_str().unwrap_or_default()), + &format!("{prefix} machineLabel is missing or invalid"), + failures, + ); + require( + !environment["screen"] + .as_str() + .unwrap_or_default() + .is_empty(), + &format!("{prefix} screen is missing"), + failures, + ); + let platform = environment["qtPlatform"] + .as_str() + .unwrap_or_default() + .to_ascii_lowercase(); + require( + !matches!(platform.as_str(), "" | "offscreen" | "minimal"), + &format!("{prefix} used a non-representative Qt platform"), + failures, + ); + let graphics = environment["graphicsApi"] + .as_str() + .unwrap_or_default() + .to_ascii_lowercase(); + require( + !matches!(graphics.as_str(), "" | "software" | "null" | "unknown"), + &format!("{prefix} used a non-hardware renderer"), + failures, + ); + require( + value["workload"]["cycles"].as_u64().unwrap_or_default() >= 3, + &format!("{prefix} used fewer than three cycles"), + failures, + ); + let steps = value["steps"].as_array(); + require( + steps.is_some_and(|steps| steps.len() >= 27), + &format!("{prefix} workload is incomplete"), + failures, + ); + if let Some(steps) = steps { + for step in steps { + let raw_name = step["name"].as_str().unwrap_or_default(); + let name = if safe_label(raw_name) { + raw_name + } else { + "" + }; + require( + safe_label(raw_name), + &format!("{prefix} contains an invalid step name"), + failures, + ); + require( + step["accepted"] == true, + &format!("{prefix} rejected step {name}"), + failures, + ); + require( + step["focusValid"] == true, + &format!("{prefix} invalid focus at {name}"), + failures, + ); + require( + step["firstFrameMs"] + .as_f64() + .is_some_and(|number| number.is_finite() && number >= 0.0), + &format!("{prefix} missing frame at {name}"), + failures, + ); + } + } +} + +fn validate_attestations(value: &Value, matrix: Option, failures: &mut Vec) { + require( + value["schemaVersion"] == 1, + "attestations schemaVersion must be 1", + failures, + ); + require( + value["kind"] == "opennow.acceptance-attestations", + "attestations kind is invalid", + failures, + ); + require( + safe_label(value["releaseCandidate"].as_str().unwrap_or_default()), + "releaseCandidate is missing or invalid", + failures, + ); + require( + safe_label(value["machineLabel"].as_str().unwrap_or_default()), + "attestation machineLabel is missing or invalid", + failures, + ); + require( + safe_label(value["applicationVersion"].as_str().unwrap_or_default()), + "attestation applicationVersion is missing or invalid", + failures, + ); + require( + !contains_sensitive(value), + "attestations contain a sensitive key or value", + failures, + ); + let checks = value["checks"].as_object(); + require( + checks.is_some(), + "attestation checks object is missing", + failures, + ); + if let Some(checks) = checks { + for name in REQUIRED_ATTESTATIONS { + require( + checks.get(*name) == Some(&Value::Bool(true)), + &format!("manual attestation did not pass: {name}"), + failures, + ); + } + } + for name in ["hdr", "vrr"] { + let capability = &value["displayCapabilities"][name]; + let status = capability["status"].as_str().unwrap_or_default(); + require( + matches!(status, "passed" | "not-supported"), + &format!("display capability {name} has no valid status"), + failures, + ); + require( + !capability["evidence"] + .as_str() + .unwrap_or_default() + .is_empty(), + &format!("display capability {name} evidence is missing"), + failures, + ); + } + if matrix.is_some() { + require( + value["matrixRow"] + .as_str() + .is_some_and(|name| matrix_row(name).is_some()), + "matrix row is invalid", + failures, + ); + } +} + +fn validate_cross_report( + live: &Value, + first: &Value, + second: &Value, + attestations: &Value, + matrix: Option, + failures: &mut Vec, +) { + for value in [first, second] { + require( + value["environment"]["machineLabel"] == attestations["machineLabel"], + "performance machineLabel differs from attestations", + failures, + ); + require( + value["environment"]["applicationVersion"] == live["applicationVersion"], + "application versions differ between live and performance evidence", + failures, + ); + } + require( + attestations["applicationVersion"] == live["applicationVersion"], + "attestation applicationVersion differs from live evidence", + failures, + ); + require( + first["environment"]["cpuArchitecture"] == second["environment"]["cpuArchitecture"], + "performance CPU architectures differ", + failures, + ); + require( + first["environment"]["qtPlatform"] == second["environment"]["qtPlatform"], + "performance Qt platforms differ", + failures, + ); + if let Some(matrix) = matrix { + require( + normalize_arch( + live["platform"]["cpuArchitecture"] + .as_str() + .unwrap_or_default(), + ) == matrix.architecture, + "live architecture does not match matrix row", + failures, + ); + require( + normalize_arch( + first["environment"]["cpuArchitecture"] + .as_str() + .unwrap_or_default(), + ) == matrix.architecture, + "performance architecture does not match matrix row", + failures, + ); + let live_os = live["platform"]["os"].as_str().unwrap_or_default(); + require( + os_matches(matrix.os, live_os), + "live OS does not match matrix row", + failures, + ); + let qt_platform = first["environment"]["qtPlatform"] + .as_str() + .unwrap_or_default() + .to_ascii_lowercase(); + require( + matrix + .window_system + .iter() + .any(|allowed| qt_platform.contains(allowed)), + "Qt platform does not match matrix row", + failures, + ); + let live_window = live["platform"]["windowSystem"] + .as_str() + .unwrap_or_default() + .to_ascii_lowercase(); + require( + matrix + .window_system + .iter() + .any(|allowed| live_window.contains(allowed)), + "live window system does not match matrix row", + failures, + ); + } +} + +fn validate_packages( + value: &Value, + matrix: Option, + actual: &BTreeMap, + failures: &mut Vec, +) { + let declared = value["packages"].as_array(); + require( + declared.is_some(), + "attestation packages array is missing", + failures, + ); + let mut declared_names = BTreeSet::new(); + if let Some(declared) = declared { + for package in declared { + let name = package["fileName"].as_str().unwrap_or_default(); + let display_name = if safe_file_name(name) { + name + } else { + "" + }; + declared_names.insert(name.to_owned()); + require( + safe_file_name(name), + "declared package filename is invalid", + failures, + ); + require( + actual.get(name).is_some(), + &format!("declared package was not supplied: {display_name}"), + failures, + ); + if let Some(actual) = actual.get(name) { + require( + package["sha256"] == actual["sha256"], + &format!("package SHA-256 mismatch: {display_name}"), + failures, + ); + require( + package["sizeBytes"] == actual["sizeBytes"], + &format!("package size mismatch: {display_name}"), + failures, + ); + } + require( + package["signingVerified"] == true, + &format!("package signing was not verified: {display_name}"), + failures, + ); + require( + package["updateManifestVerified"] == true, + &format!("update manifest was not verified: {display_name}"), + failures, + ); + } + } + require( + declared_names == actual.keys().cloned().collect(), + "supplied and declared package sets differ", + failures, + ); + if let Some(matrix) = matrix { + let lower = declared_names + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect::>(); + let has = |suffix: &str| lower.iter().any(|name| name.ends_with(suffix)); + match matrix.os { + MatrixOs::Windows => { + require( + has(".msi") || has(".exe"), + "Windows installer package is missing", + failures, + ); + require(has(".zip"), "Windows portable ZIP is missing", failures); + } + MatrixOs::Macos => { + require(has(".dmg"), "macOS DMG is missing", failures); + require(has(".zip"), "macOS ZIP is missing", failures); + } + MatrixOs::Linux => { + require(has(".appimage"), "Linux AppImage is missing", failures); + require(has(".deb"), "Linux DEB is missing", failures); + } + } + } +} + +fn hash_packages(paths: &[PathBuf]) -> Result, String> { + let mut output = BTreeMap::new(); + for path in paths { + let evidence = file_evidence(path)?; + let name = evidence["fileName"].as_str().unwrap_or_default().to_owned(); + if output.insert(name.clone(), evidence).is_some() { + return Err(format!("Duplicate package filename: {name}")); + } + } + Ok(output) +} + +fn file_evidence(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("Could not inspect {}: {error}", path.display()))?; + if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "Evidence must be a regular non-symlink file: {}", + path.display() + )); + } + let name = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + if !safe_file_name(name) { + return Err(format!("Unsafe evidence filename: {}", path.display())); + } + let file = fs::File::open(path) + .map_err(|error| format!("Could not open {}: {error}", path.display()))?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 128 * 1024]; + loop { + let read = reader + .read(&mut buffer) + .map_err(|error| format!("Could not hash {}: {error}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok( + json!({"fileName": name, "sizeBytes": metadata.len(), "sha256": format!("{:x}", hasher.finalize())}), + ) +} + +fn write_new_json(path: &Path, value: &Value) -> Result<(), String> { + if path.exists() { + return Err(format!( + "Output already exists; choose a new path: {}", + path.display() + )); + } + let parent = path + .parent() + .ok_or_else(|| "Output has no parent directory".to_owned())?; + fs::create_dir_all(parent) + .map_err(|error| format!("Could not create output directory: {error}"))?; + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let temporary = parent.join(format!( + ".{}.{}.{}.tmp", + path.file_name() + .and_then(|value| value.to_str()) + .unwrap_or("acceptance"), + std::process::id(), + nonce + )); + let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| format!("Could not create temporary output: {error}"))?; + if let Err(error) = file.write_all(&bytes).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(&temporary); + return Err(format!("Could not write output: {error}")); + } + drop(file); + if let Err(error) = fs::rename(&temporary, path) { + let _ = fs::remove_file(&temporary); + return Err(format!("Could not commit output: {error}")); + } + Ok(()) +} + +fn matrix_row(name: &str) -> Option { + match name { + "windows-x64" => Some(MatrixRow { + os: MatrixOs::Windows, + architecture: "x64", + window_system: &["windows"], + }), + "windows-arm64" => Some(MatrixRow { + os: MatrixOs::Windows, + architecture: "arm64", + window_system: &["windows"], + }), + "macos-apple-silicon" => Some(MatrixRow { + os: MatrixOs::Macos, + architecture: "arm64", + window_system: &["cocoa"], + }), + "macos-intel" => Some(MatrixRow { + os: MatrixOs::Macos, + architecture: "x64", + window_system: &["cocoa"], + }), + "linux-x64-x11" => Some(MatrixRow { + os: MatrixOs::Linux, + architecture: "x64", + window_system: &["xcb"], + }), + "linux-x64-wayland" => Some(MatrixRow { + os: MatrixOs::Linux, + architecture: "x64", + window_system: &["wayland"], + }), + "linux-arm64-native" => Some(MatrixRow { + os: MatrixOs::Linux, + architecture: "arm64", + window_system: &["xcb", "wayland"], + }), + _ => None, + } +} + +fn normalize_arch(value: &str) -> &'static str { + match value.to_ascii_lowercase().as_str() { + "x86_64" | "amd64" | "x64" => "x64", + "aarch64" | "arm64" => "arm64", + _ => "unknown", + } +} + +fn os_matches(expected: MatrixOs, value: &str) -> bool { + let value = value.to_ascii_lowercase(); + match expected { + MatrixOs::Windows => value.contains("windows"), + MatrixOs::Macos => value.contains("macos") || value.contains("darwin"), + MatrixOs::Linux => value.contains("linux"), + } +} + +fn contains_sensitive(value: &Value) -> bool { + match value { + Value::Object(values) => values.iter().any(|(key, value)| { + let key = key.to_ascii_lowercase(); + [ + "token", + "authorization", + "password", + "secret", + "sessionid", + "processid", + "executable", + "filepath", + "url", + ] + .iter() + .any(|needle| key.contains(needle)) + || contains_sensitive(value) + }), + Value::Array(values) => values.iter().any(contains_sensitive), + Value::String(value) => { + let lower = value.to_ascii_lowercase(); + value.contains('@') + || lower.starts_with("http://") + || lower.starts_with("https://") + || lower.starts_with("wss://") + || lower.starts_with("/home/") + || lower.starts_with("/users/") + || lower.starts_with("c:\\users\\") + } + _ => false, + } +} + +fn safe_file_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 255 + && value != "." + && value != ".." + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_' | b'+')) +} + +fn safe_label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_')) +} + +fn safe_output_label(value: &Value) -> Value { + value + .as_str() + .filter(|value| safe_label(value)) + .map(|value| Value::String(value.to_owned())) + .unwrap_or(Value::Null) +} + +fn valid_sha(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +fn require(condition: bool, message: &str, failures: &mut Vec) { + if !condition { + failures.push(message.to_owned()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Map; + + fn write_value(path: &Path, value: &Value) { + fs::write(path, serde_json::to_vec(value).unwrap()).unwrap(); + } + + fn performance_report(width: u64, height: u64) -> Value { + let steps = (0..27) + .map(|index| { + json!({"name":format!("step-{index}"),"accepted":true, + "focusValid":true,"firstFrameMs":1.0}) + }) + .collect::>(); + json!({ + "schemaVersion":1,"kind":"opennow.qt.performance","pass":true,"failures":[], + "environment":{"applicationVersion":"1.0.0","machineLabel":"machine-a", + "cpuArchitecture":"x86_64","qtPlatform":"wayland","graphicsApi":"vulkan", + "screen":"Display-1","refreshRateOverrideHz":null, + "requestedPhysicalWidth":width,"actualPhysicalWidth":width, + "requestedPhysicalHeight":height,"actualPhysicalHeight":height}, + "workload":{"cycles":3},"steps":steps + }) + } + + fn live_report() -> Value { + let mut checks = Map::new(); + for name in [ + "streamingTenMinutes", + "firstFramePresented", + "nvstTransportActive", + "nativeInputReady", + "inputOwnershipExercised", + "allGuidePagesVisited", + "surfaceReconfigured", + "fullscreenControlExercised", + "statsControlExercised", + "recordingRoundTrip", + "mediaArtifactsComplete", + "networkRecoveryExercised", + "streamerRecoveryExercised", + "noTerminalMediaError", + "deviceRecoveryBalanced", + ] { + checks.insert(name.to_owned(), Value::Bool(true)); + } + let media = |name: &str| json!({"fileName":name,"sizeBytes":1,"sha256":"0".repeat(64)}); + json!({ + "schemaVersion":1,"kind":"opennow.live-acceptance","observedPass":true, + "scope":"machine-observed-live-runtime","applicationVersion":"1.0.0", + "platform":{"os":"linux","cpuArchitecture":"x86_64","windowSystem":"wayland"}, + "checks":checks,"stream":{"status":"streaming","transport":"nvst","sessionUptimeMs":600000}, + "media":{"screenshot":media("shot.png"), + "recording":{"fileName":"clip.mkv","sizeBytes":1,"sha256":"0".repeat(64), + "thumbnail":media("thumb.jpg")}} + }) + } + + #[test] + fn matrix_rows_are_explicit_and_architectures_normalize() { + assert_eq!(matrix_row("linux-x64-wayland").unwrap().os, MatrixOs::Linux); + assert!(matrix_row("linux-auto").is_none()); + assert_eq!(normalize_arch("x86_64"), "x64"); + assert_eq!(normalize_arch("aarch64"), "arm64"); + } + + #[test] + fn live_validation_rejects_false_checks_sensitive_values_and_bad_hashes() { + let mut value = json!({ + "schemaVersion":1, "kind":"opennow.live-acceptance", "observedPass":true, + "scope":"machine-observed-live-runtime", "applicationVersion":"1.0.0", + "checks":{}, "stream":{"status":"streaming","transport":"nvst","sessionUptimeMs":600000}, + "media":{"screenshot":{"fileName":"shot.png","sizeBytes":1,"sha256":"0".repeat(64)}, + "recording":{"fileName":"clip.mkv","sizeBytes":1,"sha256":"0".repeat(64), + "thumbnail":{"fileName":"thumb.jpg","sizeBytes":1,"sha256":"0".repeat(64)}}} + }); + for name in [ + "streamingTenMinutes", + "firstFramePresented", + "nvstTransportActive", + "nativeInputReady", + "inputOwnershipExercised", + "allGuidePagesVisited", + "surfaceReconfigured", + "fullscreenControlExercised", + "statsControlExercised", + "recordingRoundTrip", + "mediaArtifactsComplete", + "networkRecoveryExercised", + "streamerRecoveryExercised", + "noTerminalMediaError", + "deviceRecoveryBalanced", + ] { + value["checks"][name] = Value::Bool(true); + } + let mut failures = Vec::new(); + validate_live(&value, &mut failures); + assert!(failures.is_empty(), "{failures:?}"); + value["checks"]["nativeInputReady"] = Value::Bool(false); + value["stream"]["transport"] = Value::String("webrtc".to_owned()); + value["unsafeUrl"] = Value::String("https://example.invalid".to_owned()); + value["media"]["screenshot"]["sha256"] = Value::String("BAD".to_owned()); + validate_live(&value, &mut failures); + assert!( + failures + .iter() + .any(|value| value.contains("nativeInputReady")) + ); + assert!(failures.iter().any(|value| value.contains("sensitive"))); + assert!(failures.iter().any(|value| value.contains("SHA-256"))); + assert!(failures.iter().any(|value| value.contains("not NVST"))); + } + + #[test] + fn required_manual_attestation_names_are_unique() { + let unique = REQUIRED_ATTESTATIONS + .iter() + .copied() + .collect::>(); + assert_eq!(unique.len(), REQUIRED_ATTESTATIONS.len()); + } + + #[test] + fn complete_bundle_passes_and_package_tampering_fails_closed() { + let root = env::temp_dir().join(format!( + "opennow-acceptance-verifier-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&root).unwrap(); + let live = root.join("live.json"); + let perf_1080p = root.join("1080p.json"); + let perf_4k = root.join("4k.json"); + let attestations = root.join("attestations.json"); + let appimage = root.join("OpenNOW.AppImage"); + let deb = root.join("opennow.deb"); + fs::write(&appimage, b"appimage").unwrap(); + fs::write(&deb, b"deb").unwrap(); + write_value(&live, &live_report()); + write_value(&perf_1080p, &performance_report(1920, 1080)); + write_value(&perf_4k, &performance_report(3840, 2160)); + let mut checks = Map::new(); + for name in REQUIRED_ATTESTATIONS { + checks.insert((*name).to_owned(), Value::Bool(true)); + } + let package = |path: &Path| { + let mut evidence = file_evidence(path).unwrap(); + evidence["signingVerified"] = Value::Bool(true); + evidence["updateManifestVerified"] = Value::Bool(true); + evidence + }; + write_value( + &attestations, + &json!({ + "schemaVersion":1,"kind":"opennow.acceptance-attestations", + "releaseCandidate":"v1.0.0-rc1","applicationVersion":"1.0.0", + "matrixRow":"linux-x64-wayland","machineLabel":"machine-a","checks":checks, + "displayCapabilities":{"hdr":{"status":"not-supported","evidence":"display probe"}, + "vrr":{"status":"passed","evidence":"capture hash"}}, + "packages":[package(&appimage),package(&deb)] + }), + ); + let arguments = Arguments { + live, + performance_1080p: perf_1080p, + performance_4k: perf_4k, + attestations, + packages: vec![appimage.clone(), deb], + output: root.join("result.json"), + }; + assert!(verify(&arguments).unwrap()); + assert_eq!(read_json(&arguments.output).unwrap()["pass"], true); + fs::write(&appimage, b"tampered").unwrap(); + let failed_arguments = Arguments { + output: root.join("failed.json"), + ..arguments + }; + assert!(!verify(&failed_arguments).unwrap()); + let failure = read_json(&failed_arguments.output).unwrap(); + assert_eq!(failure["pass"], false); + assert!(failure["failures"].as_array().unwrap().iter().any(|value| { + value + .as_str() + .is_some_and(|value| value.contains("SHA-256 mismatch")) + })); + let _ = fs::remove_dir_all(root); + } +} diff --git a/external/opennow/native/opennow-core/src/bin/opennow-license-report.rs b/external/opennow/native/opennow-core/src/bin/opennow-license-report.rs new file mode 100644 index 000000000..b60ac3b97 --- /dev/null +++ b/external/opennow/native/opennow-core/src/bin/opennow-license-report.rs @@ -0,0 +1,211 @@ +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const MAX_LICENSE_BYTES: u64 = 2 * 1024 * 1024; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct Package { + name: String, + version: String, + license: String, + source: String, + directory: PathBuf, + license_file: Option, +} + +fn main() { + if let Err(error) = run() { + eprintln!("opennow-license-report: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut arguments = env::args_os().skip(1); + let base = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| usage("missing base notice path"))?; + let output = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| usage("missing output path"))?; + let manifests = arguments.map(PathBuf::from).collect::>(); + if manifests.is_empty() { + return Err(usage("at least one Cargo manifest is required")); + } + + let mut packages = BTreeSet::new(); + for manifest in manifests { + packages.extend(packages_for_manifest(&manifest)?); + } + + let mut texts: BTreeMap> = BTreeMap::new(); + let mut metadata_only = Vec::new(); + for package in packages { + let label = format!( + "{} {} — {} — {}", + package.name, package.version, package.license, package.source + ); + let license_files = find_license_files(&package)?; + if license_files.is_empty() { + metadata_only.push(label); + continue; + } + for path in license_files { + let contents = fs::read_to_string(&path) + .map_err(|error| format!("could not read {}: {error}", path.display()))?; + texts.entry(contents).or_default().insert(label.clone()); + } + } + + let mut notice = fs::read_to_string(&base) + .map_err(|error| format!("could not read {}: {error}", base.display()))?; + notice.push_str("\n\nGenerated Rust dependency notices\n=================================\n\n"); + notice.push_str( + "This section is generated from the exact Cargo dependency graphs used by the OpenNOW application core and native streamer. Packages sharing identical license text are grouped together.\n", + ); + for (index, (license_text, labels)) in texts.into_iter().enumerate() { + notice.push_str(&format!( + "\nLicense text group {}\n---------------------\n", + index + 1 + )); + for label in labels { + notice.push_str("- "); + notice.push_str(&label); + notice.push('\n'); + } + notice.push('\n'); + notice.push_str(license_text.trim()); + notice.push('\n'); + } + if !metadata_only.is_empty() { + notice.push_str("\nSPDX metadata-only packages\n---------------------------\n"); + notice.push_str("These packages declared an SPDX expression but shipped no standalone license file in the resolved source archive:\n"); + for label in metadata_only { + notice.push_str("- "); + notice.push_str(&label); + notice.push('\n'); + } + } + + let parent = output + .parent() + .ok_or_else(|| "output path has no parent directory".to_owned())?; + fs::create_dir_all(parent) + .map_err(|error| format!("could not create {}: {error}", parent.display()))?; + let temporary = output.with_extension("tmp"); + fs::write(&temporary, notice) + .map_err(|error| format!("could not write {}: {error}", temporary.display()))?; + if output.exists() { + fs::remove_file(&output) + .map_err(|error| format!("could not replace {}: {error}", output.display()))?; + } + fs::rename(&temporary, &output) + .map_err(|error| format!("could not publish {}: {error}", output.display()))?; + Ok(()) +} + +fn packages_for_manifest(manifest: &Path) -> Result, String> { + let cargo = env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let output = Command::new(cargo) + .args(["metadata", "--format-version", "1", "--manifest-path"]) + .arg(manifest) + .output() + .map_err(|error| format!("could not execute cargo metadata: {error}"))?; + if !output.status.success() { + return Err(format!( + "cargo metadata failed for {}: {}", + manifest.display(), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let metadata: Value = serde_json::from_slice(&output.stdout) + .map_err(|error| format!("cargo metadata returned invalid JSON: {error}"))?; + let mut packages = BTreeSet::new(); + for value in metadata["packages"].as_array().into_iter().flatten() { + let name = value["name"].as_str().unwrap_or_default(); + if name == "opennow-core" || name.starts_with("opennow-streamer") { + continue; + } + let manifest_path = value["manifest_path"] + .as_str() + .map(PathBuf::from) + .ok_or_else(|| format!("package {name} has no manifest path"))?; + let directory = manifest_path + .parent() + .ok_or_else(|| format!("package {name} has an invalid manifest path"))? + .to_path_buf(); + let source = value["repository"] + .as_str() + .or_else(|| value["homepage"].as_str()) + .or_else(|| value["source"].as_str()) + .unwrap_or("resolved local source") + .to_owned(); + packages.insert(Package { + name: name.to_owned(), + version: value["version"].as_str().unwrap_or("unknown").to_owned(), + license: value["license"] + .as_str() + .unwrap_or("NOASSERTION") + .to_owned(), + source, + directory, + license_file: value["license_file"].as_str().map(PathBuf::from), + }); + } + Ok(packages) +} + +fn find_license_files(package: &Package) -> Result, String> { + if let Some(path) = package.license_file.as_ref() { + let path = if path.is_absolute() { + path.clone() + } else { + package.directory.join(path) + }; + validate_license_file(&path)?; + return Ok(vec![path]); + } + + let mut paths = fs::read_dir(&package.directory) + .map_err(|error| format!("could not inspect {}: {error}", package.directory.display()))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() + && path + .file_name() + .and_then(|name| name.to_str()) + .map(str::to_ascii_lowercase) + .is_some_and(|name| { + name.starts_with("license") + || name.starts_with("copying") + || name.starts_with("notice") + || name.starts_with("copyright") + }) + }) + .collect::>(); + paths.sort(); + for path in &paths { + validate_license_file(path)?; + } + Ok(paths) +} + +fn validate_license_file(path: &Path) -> Result<(), String> { + let metadata = fs::metadata(path) + .map_err(|error| format!("could not inspect {}: {error}", path.display()))?; + if !metadata.is_file() || metadata.len() > MAX_LICENSE_BYTES { + return Err(format!("unusable license file: {}", path.display())); + } + Ok(()) +} + +fn usage(message: &str) -> String { + format!("{message}; usage: opennow-license-report BASE_NOTICE OUTPUT MANIFEST [MANIFEST ...]") +} diff --git a/external/opennow/native/opennow-core/src/bin/opennow-update-helper.rs b/external/opennow/native/opennow-core/src/bin/opennow-update-helper.rs new file mode 100644 index 000000000..84f236a73 --- /dev/null +++ b/external/opennow/native/opennow-core/src/bin/opennow-update-helper.rs @@ -0,0 +1,15 @@ +use opennow_core::update_apply; + +fn main() { + let mut arguments = std::env::args_os().skip(1); + let result = match (arguments.next(), arguments.next(), arguments.next()) { + (Some(flag), Some(path), None) if flag == "--apply" => { + update_apply::run_helper(std::path::Path::new(&path)) + } + _ => Err("Usage: opennow-update-helper --apply ".to_owned()), + }; + if let Err(error) = result { + eprintln!("opennow-update-helper: {error}"); + std::process::exit(1); + } +} diff --git a/external/opennow/native/opennow-core/src/bin/opennow-update-manifest.rs b/external/opennow/native/opennow-core/src/bin/opennow-update-manifest.rs new file mode 100644 index 000000000..12344e6af --- /dev/null +++ b/external/opennow/native/opennow-core/src/bin/opennow-update-manifest.rs @@ -0,0 +1,231 @@ +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::env; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Manifest { + schema_version: u32, + version: String, + asset: String, + size: u64, + sha256: String, + signature: String, +} + +fn main() { + if let Err(error) = run() { + eprintln!("opennow-update-manifest: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut arguments = env::args_os().skip(1); + let asset = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| "usage: opennow-update-manifest [output]".to_owned())?; + let version = arguments + .next() + .and_then(|value| value.into_string().ok()) + .ok_or_else(|| "update version is required".to_owned())?; + let version = semver::Version::parse(version.strip_prefix('v').unwrap_or(&version)) + .map_err(|_| "update version must be valid semver".to_owned())? + .to_string(); + let output = arguments.next().map(PathBuf::from).unwrap_or_else(|| { + let name = asset.file_name().unwrap_or_default().to_string_lossy(); + asset.with_file_name(format!("{name}.manifest.json")) + }); + if arguments.next().is_some() { + return Err("update manifest arguments are invalid".to_owned()); + } + let asset_name = asset + .file_name() + .and_then(|value| value.to_str()) + .filter(|value| safe_asset_name(value)) + .ok_or_else(|| "update asset name is unsafe".to_owned())? + .to_owned(); + let key = signing_key()?; + let verifying_key = configured_verifying_key()?; + if key.verifying_key() != verifying_key { + return Err( + "OPENNOW_UPDATE_ED25519_PUBLIC_KEY does not match the protected signing key".to_owned(), + ); + } + let (size, sha256) = hash_file(&asset)?; + let mut manifest = Manifest { + schema_version: 1, + version, + asset: asset_name, + size, + sha256, + signature: String::new(), + }; + let signature = key.sign(signature_payload(&manifest).as_bytes()); + verifying_key + .verify_strict(signature_payload(&manifest).as_bytes(), &signature) + .map_err(|_| "generated update signature did not verify".to_owned())?; + manifest.signature = BASE64.encode(signature.to_bytes()); + let bytes = serde_json::to_vec_pretty(&manifest).map_err(|error| error.to_string())?; + let temporary = output.with_extension("json.tmp"); + let mut file = fs::File::create(&temporary) + .map_err(|error| format!("could not create manifest: {error}"))?; + file.write_all(&bytes) + .and_then(|()| file.write_all(b"\n")) + .and_then(|()| file.sync_all()) + .map_err(|error| format!("could not write manifest: {error}"))?; + fs::rename(&temporary, &output) + .map_err(|error| format!("could not finalize manifest: {error}"))?; + println!("{}", output.display()); + Ok(()) +} + +fn signing_key() -> Result { + let encoded = env::var("OPENNOW_UPDATE_ED25519_PRIVATE_KEY") + .map_err(|_| "OPENNOW_UPDATE_ED25519_PRIVATE_KEY is required".to_owned())?; + let bytes = BASE64 + .decode(encoded.trim()) + .map_err(|_| "update signing key is not valid base64".to_owned())?; + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| "update signing key must be a 32-byte Ed25519 seed".to_owned())?; + Ok(SigningKey::from_bytes(&bytes)) +} + +fn configured_verifying_key() -> Result { + let encoded = env::var("OPENNOW_UPDATE_ED25519_PUBLIC_KEY") + .map_err(|_| "OPENNOW_UPDATE_ED25519_PUBLIC_KEY is required".to_owned())?; + decode_verifying_key(&encoded) +} + +fn decode_verifying_key(encoded: &str) -> Result { + let bytes = BASE64 + .decode(encoded.trim()) + .map_err(|_| "update verifying key is not valid base64".to_owned())?; + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| "update verifying key must be 32 bytes".to_owned())?; + VerifyingKey::from_bytes(&bytes).map_err(|_| "update verifying key is invalid".to_owned()) +} + +fn hash_file(path: &Path) -> Result<(u64, String), String> { + let mut file = + fs::File::open(path).map_err(|error| format!("could not read update asset: {error}"))?; + let mut hasher = Sha256::new(); + let mut size = 0_u64; + let mut buffer = [0_u8; 128 * 1024]; + loop { + let count = file + .read(&mut buffer) + .map_err(|error| format!("could not hash update asset: {error}"))?; + if count == 0 { + break; + } + size = size + .checked_add(count as u64) + .ok_or_else(|| "update asset size overflowed".to_owned())?; + hasher.update(&buffer[..count]); + } + if size == 0 { + return Err("update asset is empty".to_owned()); + } + Ok((size, format!("{:x}", hasher.finalize()))) +} + +fn signature_payload(manifest: &Manifest) -> String { + format!( + "OpenNOW update manifest v1\nversion={}\nasset={}\nsize={}\nsha256={}\n", + manifest.version, manifest.asset, manifest.size, manifest.sha256 + ) +} + +fn safe_asset_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 240 + && !value.contains(['/', '\\']) + && !value.contains("..") + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn public_key_decoder_rejects_malformed_and_accepts_the_signing_pair() { + assert!(decode_verifying_key("not-base64").is_err()); + assert!(decode_verifying_key(&BASE64.encode([0_u8; 31])).is_err()); + let signing = SigningKey::from_bytes(&[7_u8; 32]); + let encoded = BASE64.encode(signing.verifying_key().as_bytes()); + assert_eq!( + decode_verifying_key(&encoded).unwrap(), + signing.verifying_key() + ); + } + + #[test] + fn canonical_manifest_signature_verifies_and_detects_tampering() { + let signing = SigningKey::from_bytes(&[9_u8; 32]); + let manifest = Manifest { + schema_version: 1, + version: "1.2.3".to_owned(), + asset: "OpenNOW-Qt-1.2.3.deb".to_owned(), + size: 42, + sha256: "0".repeat(64), + signature: String::new(), + }; + let signature = signing.sign(signature_payload(&manifest).as_bytes()); + assert!( + signing + .verifying_key() + .verify_strict(signature_payload(&manifest).as_bytes(), &signature) + .is_ok() + ); + let mut tampered = manifest; + tampered.size += 1; + assert!( + signing + .verifying_key() + .verify_strict(signature_payload(&tampered).as_bytes(), &signature) + .is_err() + ); + } + + #[test] + fn nightly_signature_binds_the_full_version() { + let signing = SigningKey::from_bytes(&[9_u8; 32]); + let mut manifest = Manifest { + schema_version: 1, + version: semver::Version::parse("1.0.0-nightly.123456.2") + .unwrap() + .to_string(), + asset: "OpenNOW-Qt-1.0.0-nightly.123456.2-Linux-x64.deb".to_owned(), + size: 42, + sha256: "0".repeat(64), + signature: String::new(), + }; + let signature = signing.sign(signature_payload(&manifest).as_bytes()); + signing + .verifying_key() + .verify_strict(signature_payload(&manifest).as_bytes(), &signature) + .unwrap(); + for version in ["1.0.0-nightly.123456.3", "1.0.0"] { + manifest.version = version.to_owned(); + assert!( + signing + .verifying_key() + .verify_strict(signature_payload(&manifest).as_bytes(), &signature) + .is_err() + ); + } + } +} diff --git a/external/opennow/native/opennow-core/src/bug_reports.rs b/external/opennow/native/opennow-core/src/bug_reports.rs new file mode 100644 index 000000000..2a692f32a --- /dev/null +++ b/external/opennow/native/opennow-core/src/bug_reports.rs @@ -0,0 +1,340 @@ +use serde_json::{Map, Value, json}; +use std::collections::{HashSet, VecDeque}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +const ENABLED_WHEN_UNSET: bool = true; +const MAXIMUM_REPORTS_PER_RUN: usize = 5; +const MINIMUM_REPORT_INTERVAL: Duration = Duration::from_secs(30); +const RECENT_GAME_LIMIT: usize = 5; + +const REPORTED_METHODS: &[(&str, Kind)] = &[ + ("session.create", Kind::SessionError), + ("session.poll", Kind::SessionError), + ("session.claim", Kind::SessionError), + ("streamer.prepare", Kind::StreamError), + ("streamer.start", Kind::StreamError), + ("catalog.library.list", Kind::LibraryError), +]; + +const EXPECTED_CODES: &[&str] = &[ + "authentication_required", + "busy", + "cancelled", + "catalog_changed", + "catalog_mutation_busy", + "confirmation_required", + "launch_not_ready", + "rate_limited", + "routing_busy", + "session_cleanup_pending", + "session_conflict", + "session_update_busy", + "stale_account", + "update_pending", +]; + +const SHELL_METRICS: &[&str] = &[ + "droppedFrames", + "windowSeconds", + "framesPerSecond", + "packetLossPercent", + "pingMs", + "decodeTimeMs", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + SessionError, + StreamError, + FrameDrops, + LibraryError, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Self::SessionError => "session_error", + Self::StreamError => "stream_error", + Self::FrameDrops => "frame_drops", + Self::LibraryError => "library_error", + } + } +} + +#[derive(Clone, Debug)] +pub struct Incident { + pub kind: Kind, + pub code: String, + pub message: String, + pub details: Value, +} + +impl Incident { + pub fn trigger(&self) -> Value { + json!({ + "kind": self.kind.as_str(), + "code": self.code, + "message": self.message, + "metrics": self.details.get("metrics").cloned().unwrap_or_else(|| json!({})) + }) + } +} + +#[derive(Clone, Debug)] +struct Game { + title: String, + app_id: String, +} + +impl Game { + fn value(&self) -> Value { + json!({"title":self.title, "appId":self.app_id}) + } +} + +#[derive(Default)] +struct State { + current: Option, + recent: VecDeque, + reported: HashSet, + admitted: usize, + last_admitted: Option, +} + +#[derive(Default)] +pub struct BugReporter { + state: Mutex, +} + +pub fn enabled(settings: &Value) -> bool { + match settings["automaticBugReports"].as_str() { + Some("enabled") => true, + Some("disabled") => false, + _ => ENABLED_WHEN_UNSET, + } +} + +impl BugReporter { + pub fn observe_launch(&self, params: &Value) -> String { + let game = Game { + title: bounded_text(params["title"].as_str().unwrap_or("Unknown game"), 120), + app_id: bounded_text(params["appId"].as_str().unwrap_or_default(), 32), + }; + let detail = format!("title={} appId={}", game.title, game.app_id); + let mut state = self.state.lock().expect("bug reporter poisoned"); + state.recent.retain(|recent| recent.app_id != game.app_id); + state.recent.push_front(game.clone()); + state.recent.truncate(RECENT_GAME_LIMIT); + state.current = Some(game); + detail + } + + pub fn observe_stop(&self) { + self.state.lock().expect("bug reporter poisoned").current = None; + } + + pub fn activity(&self) -> Value { + let state = self.state.lock().expect("bug reporter poisoned"); + json!({ + "currentGame": state.current.as_ref().map(Game::value), + "recentGames": state.recent.iter().map(Game::value).collect::>() + }) + } + + pub fn rpc_incident(method: &str, code: &str, message: &str) -> Option { + let kind = REPORTED_METHODS + .iter() + .find(|(reported, _)| *reported == method) + .map(|(_, kind)| *kind)?; + if EXPECTED_CODES.contains(&code) { + return None; + } + Some(Incident { + kind, + code: bounded_code(code), + message: crate::diagnostics::runtime_failure_reason(message), + details: json!({"method":method}), + }) + } + + pub fn shell_incident(params: &Value) -> Result { + let kind = match params["kind"].as_str() { + Some("stream_error") => Kind::StreamError, + Some("frame_drops") => Kind::FrameDrops, + _ => return Err("Unsupported automatic report kind".to_owned()), + }; + let code = params["code"] + .as_str() + .filter(|code| { + (1..=64).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) + }) + .ok_or_else(|| "Automatic report code is invalid".to_owned())?; + let mut metrics = Map::new(); + for key in SHELL_METRICS { + if let Some(value) = params["metrics"][*key] + .as_f64() + .filter(|value| value.is_finite() && (0.0..=1_000_000.0).contains(value)) + { + metrics.insert((*key).to_owned(), json!(value)); + } + } + Ok(Incident { + kind, + code: code.to_owned(), + message: crate::diagnostics::runtime_failure_reason( + params["message"].as_str().unwrap_or_default(), + ), + details: json!({"source":"shell", "metrics":metrics}), + }) + } + + pub fn admit(&self, incident: &Incident, now: Instant) -> Option { + let mut state = self.state.lock().expect("bug reporter poisoned"); + let signature = format!("{}:{}", incident.kind.as_str(), incident.code); + if state.reported.contains(&signature) + || state.admitted >= MAXIMUM_REPORTS_PER_RUN + || state + .last_admitted + .is_some_and(|last| now.duration_since(last) < MINIMUM_REPORT_INTERVAL) + { + return None; + } + state.reported.insert(signature); + state.admitted += 1; + state.last_admitted = Some(now); + drop(state); + Some(self.activity()) + } +} + +fn bounded_text(value: &str, limit: usize) -> String { + value + .chars() + .map(|character| { + if character.is_control() && character != '\n' { + ' ' + } else { + character + } + }) + .take(limit) + .collect::() + .trim() + .to_owned() +} + +fn bounded_code(value: &str) -> String { + let code = value + .chars() + .filter(|character| character.is_ascii_alphanumeric() || *character == '_') + .map(|character| character.to_ascii_lowercase()) + .take(64) + .collect::(); + if code.is_empty() { + "unknown".to_owned() + } else { + code + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unset_consent_is_opt_out_and_explicit_choices_win() { + assert!(enabled(&json!({"automaticBugReports":"unset"}))); + assert!(enabled(&json!({}))); + assert!(enabled(&json!({"automaticBugReports":"enabled"}))); + assert!(!enabled(&json!({"automaticBugReports":"disabled"}))); + } + + #[test] + fn only_unexpected_failures_of_reported_methods_become_incidents() { + let incident = BugReporter::rpc_incident("catalog.library.list", "network_error", "boom") + .expect("library failures are reported"); + assert_eq!(incident.kind, Kind::LibraryError); + assert!(BugReporter::rpc_incident("session.create", "cancelled", "user").is_none()); + assert!(BugReporter::rpc_incident("settings.set", "invalid_setting", "x").is_none()); + let incident = + BugReporter::rpc_incident("session.poll", "session_error", "Bearer abc.def failed") + .unwrap(); + assert!(!incident.message.contains("abc.def")); + } + + #[test] + fn shell_incidents_accept_only_known_kinds_codes_and_metrics() { + let incident = BugReporter::shell_incident(&json!({ + "kind":"frame_drops", "code":"sustained_frame_drops", + "metrics":{"droppedFrames":412, "windowSeconds":60, "secret":"x", "pingMs":-1} + })) + .unwrap(); + assert_eq!(incident.kind, Kind::FrameDrops); + assert_eq!( + incident.details["metrics"], + json!({"droppedFrames":412.0, "windowSeconds":60.0}) + ); + assert!(BugReporter::shell_incident(&json!({"kind":"library_error","code":"x"})).is_err()); + assert!( + BugReporter::shell_incident(&json!({"kind":"stream_error","code":"Bad Code"})).is_err() + ); + } + + #[test] + fn reports_are_deduplicated_spaced_and_capped_per_run() { + let reporter = BugReporter::default(); + let start = Instant::now(); + let incident = |code: &str| Incident { + kind: Kind::StreamError, + code: code.to_owned(), + message: String::new(), + details: Value::Null, + }; + assert!(reporter.admit(&incident("a"), start).is_some()); + assert!( + reporter + .admit(&incident("a"), start + Duration::from_secs(60)) + .is_none() + ); + assert!( + reporter + .admit(&incident("b"), start + Duration::from_secs(5)) + .is_none() + ); + for (index, code) in ["b", "c", "d", "e"].iter().enumerate() { + let at = start + MINIMUM_REPORT_INTERVAL * (index as u32 + 1); + assert!(reporter.admit(&incident(code), at).is_some()); + } + assert!( + reporter + .admit(&incident("f"), start + Duration::from_secs(3600)) + .is_none() + ); + } + + #[test] + fn reports_carry_the_current_game_recent_games_and_the_trigger() { + let reporter = BugReporter::default(); + reporter.observe_launch(&json!({"title":"Portal 2", "appId":"100"})); + reporter.observe_launch(&json!({"title":"Cyberpunk 2077", "appId":"200"})); + reporter.observe_launch(&json!({"title":"Portal 2", "appId":"100"})); + let incident = + BugReporter::rpc_incident("session.create", "session_error", "seat lost").unwrap(); + let activity = reporter.admit(&incident, Instant::now()).unwrap(); + assert_eq!( + activity, + json!({"currentGame":{"title":"Portal 2","appId":"100"}, + "recentGames":[{"title":"Portal 2","appId":"100"},{"title":"Cyberpunk 2077","appId":"200"}]}) + ); + assert_eq!( + incident.trigger(), + json!({"kind":"session_error", "code":"session_error", "message":"seat lost", "metrics":{}}) + ); + reporter.observe_stop(); + assert_eq!(reporter.activity()["currentGame"], Value::Null); + } +} diff --git a/external/opennow/native/opennow-core/src/catalog_types.rs b/external/opennow/native/opennow-core/src/catalog_types.rs new file mode 100644 index 000000000..3723f2274 --- /dev/null +++ b/external/opennow/native/opennow-core/src/catalog_types.rs @@ -0,0 +1,367 @@ +use crate::gfn::ServiceError; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "__typename")] +pub enum StoreFeature { + AccountLinkingSso { + supported: Option, + #[serde(rename = "displayProposition")] + display_proposition: Option, + }, + AccountGamesSyncing { + supported: Option, + #[serde(rename = "displayProposition")] + display_proposition: Option, + }, + AccountSubscriptions { + #[serde(rename = "displayProposition")] + display_proposition: Option, + }, + #[serde(other)] + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GameFeature { + Rtx, + Hdr, + Reflex, +} + +impl GameFeature { + pub fn from_variant_features(features: &Value) -> Vec { + let entries: Vec<&Value> = match features { + Value::Array(items) => items.iter().collect(), + Value::Object(_) => vec![features], + _ => Vec::new(), + }; + let enabled = |key: &str| { + entries.iter().any(|feature| { + feature["key"].as_str() == Some(key) + && (feature["value"].as_bool() == Some(true) + || feature["value"] + .as_str() + .is_some_and(|value| value.eq_ignore_ascii_case("true"))) + }) + }; + let hdr_version = entries.iter().any(|feature| { + feature["key"].as_str() == Some("SUPPORTED_HDR_VERSION") + && [&feature["value"], &feature["values"]] + .into_iter() + .any(|value| match value { + Value::String(text) => !text.is_empty(), + Value::Array(items) => items + .iter() + .any(|item| item.as_str().is_some_and(|text| !text.is_empty())), + _ => false, + }) + }); + let mut result = Vec::new(); + if enabled("RTX_ENABLED") { + result.push(GameFeature::Rtx); + } + if enabled("HDR_ENABLED") || hdr_version { + result.push(GameFeature::Hdr); + } + if enabled("REFLEX_ENABLED") { + result.push(GameFeature::Reflex); + } + result + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LinkingMetadata { + pub supported_variant_ids: Option>, + pub is_supported: Option, + pub is_required: Option, + pub label: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StoreDefinition { + pub store: String, + pub label: String, + pub sort_order: Option, + pub small_image_url: Option, + #[serde(default)] + pub features: Vec, + pub account_linking_metadata: Option, +} + +impl StoreDefinition { + pub fn connection_definition(&self) -> Value { + let provider = normalize_store(&self.store); + let known = matches!( + provider.as_str(), + "UPLAY" | "BATTLENET" | "EPIC" | "GAIJIN" | "GOG" | "STEAM" | "XBOX" + ); + json!({ + "provider":provider,"store":self.store,"label":self.label,"sortOrder":self.sort_order, + "supportsLinking":known && self.features.iter().any(|feature| matches!(feature, StoreFeature::AccountLinkingSso { supported:Some(true), .. })), + "supportsSync":known && self.features.iter().any(|feature| matches!(feature, StoreFeature::AccountGamesSyncing { supported:Some(true), .. })), + "isRequired":self.account_linking_metadata.as_ref().and_then(|metadata| metadata.is_required), + "accountLinkingMetadata":self.account_linking_metadata,"features":self.features, + "capabilitySource":"server" + }) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct GenreDefinition { + pub genre: String, + pub label: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionDefinition { + pub subscription: String, + pub label: String, + #[serde(rename = "logoURL")] + pub logo_url: Option, + #[serde(rename = "buySubscriptionURL")] + pub buy_subscription_url: Option, + pub primary_store: Option, + pub additional_stores: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionStore { + pub store: String, + pub third_party_linking_redirect_keyword: Option, + #[serde(rename = "thirdPartyLinkingShortURL")] + pub third_party_linking_short_url: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "__typename")] +pub enum PatchDetails { + VariantGfnAutoPatchingMetadata { + #[serde(rename = "subType")] + sub_type: Option, + #[serde(rename = "startTime")] + start_time: Option, + #[serde(rename = "endTime")] + end_time: Option, + #[serde(rename = "historicalEtaMins")] + historical_eta_mins: Option, + #[serde(rename = "etaPredictionType")] + eta_prediction_type: Option, + }, + VariantGfnManualPatchingMetadata { + #[serde(rename = "subType")] + sub_type: Option, + #[serde(rename = "startTime")] + start_time: Option, + #[serde(rename = "endTime")] + end_time: Option, + }, + VariantGfnMaintenanceMetadata { + #[serde(rename = "subType")] + sub_type: Option, + }, + #[serde(other)] + Unknown, +} + +pub fn normalize_store(value: &str) -> String { + match value + .trim() + .to_ascii_uppercase() + .replace([' ', '-'], "_") + .as_str() + { + "UBISOFT" | "UBISOFT_CONNECT" => "UPLAY".into(), + "BATTLE_NET" | "BLIZZARD" => "BATTLENET".into(), + "EPIC_GAMES" | "EPIC_GAMES_STORE" => "EPIC".into(), + value => value.into(), + } +} + +pub fn parse_list( + value: &Value, +) -> Result { + let values = value + .as_array() + .filter(|items| items.len() <= 512) + .ok_or_else(invalid_metadata)?; + if serde_json::to_vec(value).map_or(true, |bytes| bytes.len() > 256 * 1024) { + return Err(invalid_metadata()); + } + let parsed: Vec = values + .iter() + .map(|item| serde_json::from_value(item.clone())) + .collect::>() + .map_err(|_| invalid_metadata())?; + serde_json::to_value(parsed).map_err(|_| invalid_metadata()) +} + +pub fn invalid_metadata() -> ServiceError { + ServiceError { + code: "invalid_upstream_response", + message: "Catalog metadata is missing, invalid, or exceeds its size limit".into(), + } +} + +pub fn filter_expression(value: &Value) -> Option { + fn merge(target: &mut Value, source: Value, depth: usize) -> Option<()> { + if depth > 8 { + return None; + } + match (target, source) { + (Value::Object(target), Value::Object(source)) => { + if source.len() > 64 { + return None; + } + for (key, value) in source { + if let Some(target) = target.get_mut(&key) { + merge(target, value, depth + 1)?; + } else { + let mut field = if value.is_object() { + json!({}) + } else if value.is_array() { + json!([]) + } else { + Value::Null + }; + merge(&mut field, value, depth + 1)?; + target.insert(key, field); + } + } + } + (Value::Array(target), Value::Array(source)) => { + if target.len() + source.len() > 128 { + return None; + } + target.extend(source); + } + (target, source) => *target = source, + } + Some(()) + } + let inputs = value.as_array().filter(|items| items.len() <= 32)?; + let mut output = json!({}); + for input in inputs { + let text = input.as_str().filter(|text| text.len() <= 8192)?; + let parsed: Value = serde_json::from_str(text).ok()?; + if !parsed.is_object() { + return None; + } + merge(&mut output, parsed, 0)?; + } + Some(output) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn game_features_follow_catalog_keys_and_ignore_unbacked_ones() { + let features = json!([ + {"__typename":"GfnSubscriptionFeatureValue","key":"RTX_ENABLED","value":"true"}, + {"__typename":"GfnSubscriptionFeatureValue","key":"SUPPORTED_HDR_VERSION","value":null}, + {"__typename":"GfnSubscriptionFeatureValueList","key":"SUPPORTED_HDR_VERSION","values":["HDR"]}, + {"__typename":"GfnSubscriptionFeatureValue","key":"REFLEX_ENABLED","value":"true"}, + {"__typename":"GfnSubscriptionFeatureValue","key":"DLSS_ENABLED","value":"true"}, + {"__typename":"GfnSubscriptionFeatureValue","key":"IN_GAME_SETTINGS_PERSISTENCE_ENABLED","value":"true"} + ]); + assert_eq!( + GameFeature::from_variant_features(&features), + vec![GameFeature::Rtx, GameFeature::Hdr, GameFeature::Reflex] + ); + assert_eq!( + serde_json::to_value(GameFeature::from_variant_features(&features)).unwrap(), + json!(["rtx", "hdr", "reflex"]) + ); + let disabled = json!([ + {"key":"RTX_ENABLED","value":"false"}, + {"key":"SUPPORTED_HDR_VERSION","value":null}, + {"key":"HDR_ENABLED","value":null} + ]); + assert!(GameFeature::from_variant_features(&disabled).is_empty()); + assert_eq!( + GameFeature::from_variant_features(&json!({"key":"HDR_ENABLED","value":true})), + vec![GameFeature::Hdr] + ); + assert!(GameFeature::from_variant_features(&Value::Null).is_empty()); + } + + #[test] + fn gog_connection_capabilities_follow_server_features() { + for supported in [false, true] { + let store: StoreDefinition = serde_json::from_value(json!({ + "store":"GOG","label":"GOG","features":[ + {"__typename":"AccountLinkingSso","supported":supported}, + {"__typename":"AccountGamesSyncing","supported":supported} + ] + })) + .unwrap(); + let connection = store.connection_definition(); + assert_eq!(connection["provider"], "GOG"); + assert_eq!(connection["supportsLinking"], supported); + assert_eq!(connection["supportsSync"], supported); + } + } + + #[test] + fn store_definitions_preserve_nullable_variant_scope_and_link_requirements() { + for variants in [Value::Null, json!([]), json!(["123"])] { + let input = json!([{"store":"XBOX","label":"Xbox","features":[], + "accountLinkingMetadata":{"supportedVariantIds":variants,"isRequired":true}}]); + let parsed = parse_list::(&input).unwrap(); + let store: StoreDefinition = serde_json::from_value(parsed[0].clone()).unwrap(); + let connection = store.connection_definition(); + assert_eq!( + connection["accountLinkingMetadata"]["supportedVariantIds"], + variants + ); + assert_eq!(connection["isRequired"], true); + } + for invalid in [json!("123"), json!([123]), json!([null])] { + assert!( + parse_list::(&json!([{"store":"XBOX","label":"Xbox", + "accountLinkingMetadata":{"supportedVariantIds":invalid}}])) + .is_err() + ); + } + } + + #[test] + fn subscription_definitions_accept_null_but_reject_invalid_additional_stores() { + for stores in [Value::Null, json!([]), json!([{"store":"UPLAY"}])] { + let parsed = parse_list::(&json!([{ + "subscription":"STORE_PASS","label":"Store Pass","additionalStores":stores + }])) + .unwrap(); + assert_eq!(parsed[0]["additionalStores"].is_null(), stores.is_null()); + if let Some(stores) = stores.as_array() { + assert_eq!( + parsed[0]["additionalStores"].as_array().unwrap().len(), + stores.len() + ); + for (index, store) in stores.iter().enumerate() { + assert_eq!( + parsed[0]["additionalStores"][index]["store"], + store["store"] + ); + } + } + } + for invalid in [json!("UPLAY"), json!(["UPLAY"]), json!([null])] { + assert!( + parse_list::(&json!([{ + "subscription":"STORE_PASS","label":"Store Pass","additionalStores":invalid + }])) + .is_err() + ); + } + } +} diff --git a/external/opennow/native/opennow-core/src/cloudmatch.rs b/external/opennow/native/opennow-core/src/cloudmatch.rs new file mode 100644 index 000000000..74cc7946c --- /dev/null +++ b/external/opennow/native/opennow-core/src/cloudmatch.rs @@ -0,0 +1,7398 @@ +use crate::gfn::{AuthSession, ServiceError}; +use crate::proxy::client_for_settings; +use rand::RngCore as _; +use reqwest::blocking::{Client, Response}; +use reqwest::header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue, USER_AGENT}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::io::Read; +use std::net::{IpAddr, UdpSocket}; +use std::path::PathBuf; +use std::sync::{Mutex, MutexGuard}; +use std::thread; +use std::time::{Duration, Instant}; +use url::Url; + +const LCARS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; +const GFN_CLIENT_VERSION: &str = "2.0.87.131"; +const DEFAULT_STREAMING_BASE: &str = "https://prod.cloudmatchbeta.nvidiagrid.net/"; +const DEFAULT_STUN_SERVER: &str = "stun:s1.stun.gamestream.nvidia.com:19308"; +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(12); +const DISCOVERY_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); +const NETWORK_TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +const MAXIMUM_NETWORK_TEST_RESPONSE_BYTES: u64 = 4 * 1024 * 1024; +const MAX_DISCOVERY_REGIONS: usize = 32; +const DISCOVERY_CONCURRENCY: usize = 4; +const MAX_CLEANUP_RECORD_BYTES: usize = 16 * 1024; + +fn webrtc_compatibility_enabled(settings: &Value, auth: &AuthSession) -> bool { + let provider = auth.provider.code.trim(); + let automatic = !auth.provider.idp_id.is_empty() + && !provider.is_empty() + && !provider.eq_ignore_ascii_case("NVIDIA"); + match settings["webrtcCompatibilityMode"].as_str() { + Some("on") => true, + Some("off") => false, + Some("auto") => automatic, + _ => settings["allianceWebrtcCompatibility"] == true, + } +} + +pub(crate) fn allocation_settings(settings: &Value, auth: &AuthSession) -> Value { + let mut settings = settings.clone(); + let enabled = webrtc_compatibility_enabled(&settings, auth); + settings["allianceWebrtcCompatibility"] = json!(enabled); + if enabled { + let hevc = codec_wire(&setting_string(&settings, "codec", "auto")) == 2; + settings["codec"] = json!(if hevc { "h265" } else { "h264" }); + settings["colorQuality"] = json!(if hevc + && matches!( + settings["colorQuality"].as_str(), + Some("10bit_420" | "10bit_444") + ) { + "10bit_420" + } else { + "8bit_420" + }); + settings["enableHdr"] = json!(false); + settings["nativeHdrSupported"] = json!(false); + settings["microphoneMode"] = json!("disabled"); + settings["enableCloudGsync"] = json!(false); + settings["nativeCloudGsyncMode"] = json!("disabled"); + let (width, height) = + parse_resolution(&setting_string(&settings, "resolution", "1920x1080")); + if (!hevc && (width > 1920 || height > 1080)) || width == 0 || height == 0 { + settings["resolution"] = json!("1920x1080"); + } + settings["fps"] = json!( + settings["fps"] + .as_u64() + .filter(|fps| *fps > 0) + .unwrap_or(60) + .clamp(30, if hevc { 360 } else { 60 }) + ); + } + settings +} + +pub(crate) fn catalog_hdr_settings(settings: &Value, runtime: &Value, variant: &Value) -> Value { + let mut settings = settings.clone(); + let display_hdr = runtime + .get("nativeHdrSupported") + .unwrap_or(&settings["nativeHdrSupported"]) + == true; + let declares_hdr = variant["features"] + .as_array() + .map(|features| features.iter().any(|feature| feature == "hdr")); + if setting_bool(&settings, "enableHdr", false) && display_hdr && declares_hdr == Some(false) { + settings["enableHdr"] = json!(false); + settings["hdrDowngrade"] = json!("game-unsupported"); + } + settings +} + +fn reported_webrtc_session(session: &Value) -> bool { + session["sessionRequestData"]["secureRTSPSupported"] != true + && session["sessionRequestData"]["metaData"] + .as_array() + .is_some_and(|metadata| { + metadata + .iter() + .any(|entry| entry["key"] == "GSStreamerType" && entry["value"] == "WebRTC") + }) +} + +fn reported_native_session(session: &Value) -> bool { + session["sessionRequestData"]["secureRTSPSupported"] == true + || value_i64(&session["sessionRequestData"]["streamerVersion"]) == Some(14) +} + +#[derive(Clone)] +struct ActiveSession { + session_id: String, + control_base: String, + zone: String, + app_id: String, + info: Value, + client: Client, +} + +struct SessionConflict { + owner: (String, String), + received: Instant, + sessions: Vec, +} + +struct FreshAllocation { + info: Value, + base: Url, + client: Client, + headers: HeaderMap, + owner: (String, String), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AttachMode { + Claim, + Reconcile, +} + +pub struct CloudMatchService { + client: Client, + active: Mutex>, + discovered: Mutex>, + conflict: Mutex>, + allocation_admission: Mutex<()>, + fresh: Mutex>, + cleanup_path: Option, + retained_cleanup: Mutex>, + webrtc_allocation: Mutex>, + #[cfg(test)] + test_control_base: Option, +} + +pub(crate) struct CreateAdmission<'a> { + service: &'a CloudMatchService, + _guard: MutexGuard<'a, ()>, +} + +impl CreateAdmission<'_> { + pub(crate) fn create( + self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + let service = self.service; + self.create_at(params, settings, auth, device_id, || { + let client = client_for_settings(&service.client, settings).map_err(invalid)?; + #[cfg(test)] + if let Some(base) = &service.test_control_base { + return Ok((client, base.clone())); + } + let requested_base = requested_streaming_base(params, settings, auth)?; + let base = service.create_base( + &client, + &requested_base, + params, + settings, + session_token(auth), + device_id, + )?; + Ok((client, base)) + }) + } + + fn create_at( + self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + connection: impl FnOnce() -> Result<(Client, Url), ServiceError>, + ) -> Result { + self.service + .create_admitted(params, settings, auth, device_id, connection) + } +} + +impl CloudMatchService { + pub fn new(client: Client) -> Self { + Self { + client, + active: Mutex::new(None), + discovered: Mutex::new(HashMap::new()), + conflict: Mutex::new(None), + allocation_admission: Mutex::new(()), + fresh: Mutex::new(None), + cleanup_path: None, + retained_cleanup: Mutex::new(None), + webrtc_allocation: Mutex::new(None), + #[cfg(test)] + test_control_base: None, + } + } + + pub fn with_cleanup_path(client: Client, path: PathBuf) -> Self { + let mut service = Self::new(client); + if let Ok(file) = std::fs::File::open(path.with_file_name("webrtc-session.json")) + && let Some(record) = read_cleanup_record(file) + && record["sessionId"] + .as_str() + .is_some_and(|id| !id.is_empty() && id.len() <= 256) + && record["transportMode"] == "webrtc" + && record["owner"].as_array().is_some_and(|owner| { + owner.len() == 2 + && owner.iter().all(|value| { + value + .as_str() + .is_some_and(|value| !value.is_empty() && value.len() <= 256) + }) + }) + { + *service + .webrtc_allocation + .lock() + .expect("WebRTC allocation state poisoned") = Some(record); + } + if let Ok(file) = std::fs::File::open(&path) + && let Some(record) = read_cleanup_record(file) + && record["sessionId"] + .as_str() + .is_some_and(|id| !id.is_empty()) + && record["streamingBaseUrl"] + .as_str() + .is_some_and(|base| trusted_cloudmatch_base(base).is_ok()) + && record["owner"].as_array().is_some_and(|owner| { + owner.len() == 2 + && owner.iter().all(|part| { + part.as_str() + .is_some_and(|value| !value.is_empty() && value.len() <= 256) + }) + }) + { + *service + .retained_cleanup + .lock() + .expect("CloudMatch cleanup state poisoned") = Some(record); + } + service.cleanup_path = Some(path); + service + } + + #[cfg(test)] + fn create( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + self.admit_create()? + .create(params, settings, auth, device_id) + } + + #[cfg(test)] + fn create_at( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + connection: impl FnOnce() -> Result<(Client, Url), ServiceError>, + ) -> Result { + self.admit_create()? + .create_at(params, settings, auth, device_id, connection) + } + + pub(crate) fn admit_create(&self) -> Result, ServiceError> { + let guard = self + .allocation_admission + .try_lock() + .map_err(|_| allocation_in_progress())?; + crate::requests::check()?; + if self + .retained_cleanup + .try_lock() + .map_err(|_| allocation_in_progress())? + .is_some() + { + return Err(ServiceError {code:"session_cleanup_pending", message:"A cancelled allocation still needs cleanup. End that session before starting another game.".to_owned()}); + } + if let Some(allocation) = self + .fresh + .try_lock() + .map_err(|_| allocation_in_progress())? + .as_ref() + { + return Err(ServiceError { + code: if allocation.info["cleanupPending"] == true { "session_cleanup_pending" } else { "session_update_busy" }, + message: "The previous allocation is awaiting confirmation or cleanup. End that session before starting another game.".to_owned(), + }); + } + Ok(CreateAdmission { + service: self, + _guard: guard, + }) + } + + fn create_admitted( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + connection: impl FnOnce() -> Result<(Client, Url), ServiceError>, + ) -> Result { + let settings = allocation_settings(settings, auth); + let settings = &settings; + let (client, base) = connection()?; + crate::requests::check()?; + let app_id = launch_app_id(params)?; + *self + .conflict + .lock() + .expect("CloudMatch conflict state poisoned") = None; + let token = session_token(auth); + let mut session_params = params.clone(); + let network_test = if requests_network_test(params, settings) { + acquire_network_test_session(&client, &base, token, device_id, params, settings) + } else { + json!({"status":"not_requested"}) + }; + match network_test["status"].as_str() { + Some("measured") => eprintln!( + "Network test measured path datagram {} bytes in {} probes", + network_test["measuredDatagramBytes"], network_test["probes"] + ), + Some("unmeasured") => eprintln!( + "Network test confirmed no probe datagram in {} probes", + network_test["probes"] + ), + Some("unavailable") => eprintln!( + "Network test session unavailable: {}", + network_test["error"].as_str().unwrap_or_default() + ), + _ => {} + } + crate::requests::check()?; + session_params["networkTestSessionId"] = json!(network_test["sessionId"].as_str()); + let body = build_create_body(&app_id, &session_params, settings, device_id); + let mut url = base + .join("v2/session") + .map_err(|_| invalid("Invalid CloudMatch session URL"))?; + crate::language::append_session_preferences(&mut url, settings); + let response = client + .post(url) + .headers(cloudmatch_headers(token, device_id)?) + .json(&body) + .send() + .map_err(|error| network("Session creation failed", error))?; + let status = response.status(); + let payload = response.json::(); + if let Ok(payload) = &payload + && let Some(error) = self.capture_session_conflict(status, payload, &base, auth) + { + return Err(error); + } + let payload = + validate_cloudmatch_response("Session creation failed", status, payload, false)?; + let zone = params["zone"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .or_else(|| base.host_str().map(ToOwned::to_owned)) + .unwrap_or_default(); + let mut info = session_info(&payload, &base, &zone, &app_id, device_id)?; + info["hdrDowngrade"] = settings["hdrDowngrade"].clone(); + info["catalogAppId"] = json!(params["catalogAppId"].as_str()); + info["transportMode"] = json!(if settings["allianceWebrtcCompatibility"] == true { + "webrtc" + } else { + "nvst" + }); + select_webrtc_endpoint(&mut info); + info["keyboardLayout"] = json!(crate::language::session_keyboard_layout(settings)); + if let Some(session_id) = network_test["sessionId"].as_str() { + info["networkTestSessionId"] = json!(session_id); + } + info["networkTest"] = network_test; + let control_base = info["streamingBaseUrl"] + .as_str() + .and_then(|raw| trusted_cloudmatch_base(raw).ok()) + .unwrap_or_else(|| base.clone()); + *self + .fresh + .lock() + .expect("CloudMatch allocation state poisoned") = Some(FreshAllocation { + info: info.clone(), + base: control_base, + client: client.clone(), + headers: cloudmatch_headers(token, device_id)?, + owner: (auth.provider.idp_id.clone(), auth.user.user_id.clone()), + }); + if crate::requests::current().cancelled() { + self.finish_create(info["sessionId"].as_str().unwrap_or_default(), false)?; + return Err(cancelled_allocation()); + } + let mut request_profile = negotiated_profile( + &body["sessionRequestData"]["clientRequestMonitorSettings"][0], + &body["sessionRequestData"]["requestedStreamingFeatures"], + ); + // The request no longer names a codec (the official client resolves it + // locally and announces it at RTSP time), so only claim request + // provenance when a codec value is actually present. + if request_profile["codec"].is_null() { + request_profile["codecSource"] = json!("unreported"); + } else { + request_profile["codecSource"] = json!("request"); + } + if info["transportMode"] == "webrtc" { + request_profile["codec"] = + json!(setting_string(settings, "codec", "h264").to_ascii_uppercase()); + request_profile["codecSource"] = json!("request"); + request_profile["enableHdr"] = json!(false); + request_profile["enableHdrSource"] = json!("request"); + } + let request_codec = json!({ + "sessionId":info["sessionId"], + "transportMode":info["transportMode"], + "negotiatedStreamProfile":request_profile + }); + preserve_session_profile(&mut info, &request_codec); + + if crate::requests::current().cancelled() { + self.finish_create(info["sessionId"].as_str().unwrap_or_default(), false)?; + return Err(cancelled_allocation()); + } + self.store_active(&mut info, &base, &zone, &app_id, client)?; + self.remember_webrtc_allocation(&info, auth)?; + info["phase"] = + Value::String(session_phase(info["status"].as_i64().unwrap_or_default()).to_owned()); + Ok(json!({"session":info})) + } + + fn remember_webrtc_allocation( + &self, + info: &Value, + auth: &AuthSession, + ) -> Result<(), ServiceError> { + if info["transportMode"] != "webrtc" || info["status"] == 7 { + return Ok(()); + } + let record = json!({"sessionId":info["sessionId"],"transportMode":"webrtc", + "owner":[auth.provider.idp_id, auth.user.user_id], + "negotiatedStreamProfile":info["negotiatedStreamProfile"]}); + if self + .webrtc_allocation + .lock() + .expect("WebRTC allocation state poisoned") + .as_ref() + == Some(&record) + { + return Ok(()); + } + if let Some(path) = &self.cleanup_path { + let path = path.with_file_name("webrtc-session.json"); + let save = || -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary = path.with_extension("tmp"); + std::fs::write(&temporary, serde_json::to_vec(&record)?)?; + std::fs::rename(temporary, path) + }; + save().map_err(|_| ServiceError { + code: "session_provenance_unavailable", + message: + "The WebRTC session is retained, but its resume profile could not be saved." + .into(), + })?; + } + *self + .webrtc_allocation + .lock() + .expect("WebRTC allocation state poisoned") = Some(record); + Ok(()) + } + + fn retained_webrtc_allocation(&self, session_id: &str, auth: &AuthSession) -> Option { + self.webrtc_allocation + .lock() + .expect("WebRTC allocation state poisoned") + .as_ref() + .filter(|record| { + record["sessionId"] == session_id + && record["owner"] == json!([auth.provider.idp_id, auth.user.user_id]) + }) + .cloned() + } + + pub fn finish_create( + &self, + expected_session_id: &str, + accepted: bool, + ) -> Result<(), ServiceError> { + let mut fresh = self + .fresh + .lock() + .expect("CloudMatch allocation state poisoned"); + let Some(allocation) = fresh.as_mut() else { + return Ok(()); + }; + let session_id = allocation.info["sessionId"] + .as_str() + .ok_or_else(|| upstream("Allocated session has no ID"))? + .to_owned(); + if session_id != expected_session_id { + return Ok(()); + } + if !accepted { + let url = allocation + .base + .join(&format!("v2/session/{session_id}")) + .map_err(|_| invalid("Invalid allocation cleanup URL"))?; + let result = delete_session(&allocation.client, url, allocation.headers.clone()); + self.clear_active(&session_id); + if let Err(error) = result { + allocation.info["cleanupPending"] = json!(true); + allocation.info["cleanupErrorCode"] = json!(error.code); + let mut pending = allocation.info.clone(); + pending["cleanupPending"] = json!(true); + self.discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .insert(session_id.to_owned(), pending); + let record = json!({"sessionId":session_id,"appId":allocation.info["appId"], + "status":allocation.info["status"],"phase":allocation.info["phase"], + "streamingBaseUrl":allocation.base.origin().ascii_serialization(), + "cleanupPending":true,"cleanupErrorCode":error.code,"owner":allocation.owner}); + *self + .retained_cleanup + .lock() + .expect("CloudMatch cleanup state poisoned") = Some(record.clone()); + if let Some(path) = &self.cleanup_path { + let saved = (|| -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary = path.with_extension("tmp"); + std::fs::write(&temporary, serde_json::to_vec(&record)?)?; + std::fs::rename(temporary, path) + })(); + if saved.is_err() { + eprintln!("CloudMatch could not persist pending allocation cleanup"); + } + } + return Err(ServiceError {code:"session_cleanup_pending", message:"The cancelled cloud session could not be closed. End it before starting another game.".to_owned()}); + } + self.discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .remove(&session_id); + self.clear_cleanup(&session_id); + } + *fresh = None; + Ok(()) + } + + fn pending_cleanup(&self, auth: &AuthSession) -> Option { + let provider_base = + trusted_cloudmatch_base(&crate::gfn::effective_provider_url(&auth.provider)).ok()?; + self.retained_cleanup + .lock() + .expect("CloudMatch cleanup state poisoned") + .as_ref() + .filter(|record| record["owner"] == json!([auth.provider.idp_id, auth.user.user_id])) + .map(|record| { + let mut public = record.clone(); + if !record["streamingBaseUrl"] + .as_str() + .and_then(|raw| trusted_cloudmatch_base(raw).ok()) + .is_some_and(|base| { + base == provider_base + || (auth.provider.code.eq_ignore_ascii_case("NVIDIA") + && base.host_str().is_some_and(|host| { + host.ends_with(".cloudmatchbeta.nvidiagrid.net") + })) + }) + { + public["streamingBaseUrl"] = Value::Null; + public["cleanupEndpointUnverified"] = json!(true); + } + if let Some(object) = public.as_object_mut() { + object.remove("owner"); + } + public + }) + } + + fn clear_cleanup(&self, session_id: &str) { + let mut retained = self + .retained_cleanup + .lock() + .expect("CloudMatch cleanup state poisoned"); + if retained + .as_ref() + .is_some_and(|record| record["sessionId"] == session_id) + { + if let Some(path) = &self.cleanup_path + && let Err(error) = std::fs::remove_file(path) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!("CloudMatch could not remove completed allocation cleanup record"); + return; + } + *retained = None; + } + } + + pub fn poll( + &self, + params: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + let current = self + .active + .lock() + .expect("CloudMatch state poisoned") + .clone(); + let client = current + .as_ref() + .map(|state| state.client.clone()) + .unwrap_or_else(|| self.client.clone()); + let session_id = params["sessionId"] + .as_str() + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| current.as_ref().map(|state| state.session_id.clone())) + .ok_or_else(|| invalid("session.poll requires sessionId"))?; + let control_base = current + .as_ref() + .map(|state| state.control_base.clone()) + .ok_or_else(|| invalid("No active session control endpoint"))?; + let base = trusted_cloudmatch_base(&control_base)?; + let token = session_token(auth); + let headers = cloudmatch_headers(token, device_id)?; + let payload = match self.get_session(&client, &base, &session_id, &headers) { + Ok(payload) => payload, + Err(error) if error.code == "session_not_found" => { + self.clear_active(&session_id); + return Ok(json!({"session":null,"termination":{ + "source":"cloudmatch-http","httpStatus":404, + "sessionId":session_id,"resumable":false + }})); + } + Err(error) => return Err(error), + }; + let zone = current + .as_ref() + .map(|state| state.zone.clone()) + .unwrap_or_default(); + let app_id = current + .as_ref() + .map(|state| state.app_id.clone()) + .unwrap_or_default(); + let mut info = session_info(&payload, &base, &zone, &app_id, device_id)?; + if let Some(current) = ¤t { + preserve_session_profile(&mut info, ¤t.info); + } + info["phase"] = + Value::String(session_phase(info["status"].as_i64().unwrap_or_default()).to_owned()); + if current + .as_ref() + .is_some_and(|state| state.info["resumePending"] == true) + { + mark_resume_progress(&mut info); + } + self.store_active(&mut info, &base, &zone, &app_id, client)?; + self.remember_webrtc_allocation(&info, auth)?; + Ok(json!({"session":info})) + } + + pub fn stop( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + let current = self + .active + .lock() + .expect("CloudMatch state poisoned") + .clone() + .filter(|state| { + params["sessionId"] + .as_str() + .is_none_or(|id| id == state.session_id) + }); + let session_id = params["sessionId"] + .as_str() + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| current.as_ref().map(|state| state.session_id.clone())); + let Some(session_id) = session_id else { + return Ok(json!({"session":null,"stopped":false})); + }; + if self + .retained_cleanup + .lock() + .expect("CloudMatch cleanup state poisoned") + .as_ref() + .is_some_and(|record| record["sessionId"] == session_id) + { + return Err(ServiceError { + code: "session_cleanup_pending", + message: "The pending session cannot be safely verified after restart. Cleanup remains pending." + .to_owned(), + }); + } + let client = current + .as_ref() + .map(|state| state.client.clone()) + .or_else(|| { + self.fresh + .lock() + .expect("CloudMatch allocation state poisoned") + .as_ref() + .filter(|allocation| allocation.info["sessionId"] == session_id) + .map(|allocation| allocation.client.clone()) + }) + .map(Ok) + .unwrap_or_else(|| client_for_settings(&self.client, settings).map_err(invalid))?; + let discovered = self + .discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .get(&session_id) + .cloned(); + let base_value = current + .as_ref() + .map(|state| state.control_base.clone()) + .or_else(|| { + discovered + .as_ref() + .and_then(|session| session["serverIp"].as_str()) + .and_then(|host| trusted_learned_server_base(host).ok()) + .map(|base| base.origin().ascii_serialization()) + }) + .or_else(|| { + discovered + .as_ref() + .and_then(|session| session["streamingBaseUrl"].as_str()) + .map(ToOwned::to_owned) + }) + .ok_or_else(|| invalid("No active session control endpoint"))?; + let base = trusted_cloudmatch_base(&base_value)?; + let url = base + .join(&format!("v2/session/{session_id}")) + .map_err(|_| invalid("Invalid CloudMatch stop URL"))?; + #[cfg(test)] + let url = self + .test_control_base + .as_ref() + .map_or(Ok(url), |base| { + base.join(&format!("v2/session/{session_id}")) + }) + .map_err(|_| invalid("Invalid test stop URL"))?; + self.stop_at( + &session_id, + &client, + url, + cloudmatch_headers(session_token(auth), device_id)?, + ) + } + + fn stop_at( + &self, + session_id: &str, + client: &Client, + url: Url, + headers: HeaderMap, + ) -> Result { + let response = client + .delete(url) + .headers(headers) + .send() + .map_err(|error| network("Session stop failed", error))?; + validate_delete_response("Session stop failed", response)?; + self.clear_active(session_id); + self.clear_cleanup(session_id); + let mut fresh = self + .fresh + .lock() + .expect("CloudMatch allocation state poisoned"); + if fresh + .as_ref() + .is_some_and(|allocation| allocation.info["sessionId"] == session_id) + { + *fresh = None; + } + drop(fresh); + self.discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .remove(session_id); + if let Some(conflict) = self + .conflict + .lock() + .expect("CloudMatch conflict state poisoned") + .as_mut() + { + conflict + .sessions + .retain(|session| session["sessionId"] != session_id); + } + Ok(json!({"session":self.active()["session"],"stopped":true,"sessionId":session_id})) + } + + pub fn active(&self) -> Value { + let state = self.active.lock().expect("CloudMatch state poisoned"); + json!({"session":state.as_ref().map(|session| session.info.clone())}) + } + + pub(crate) fn discovered_session(&self, session_id: &str) -> Option { + self.discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .get(session_id) + .cloned() + } + + pub(crate) fn cleanup_session(&self, auth: &AuthSession, session_id: &str) -> Option { + self.pending_cleanup(auth) + .filter(|session| session["sessionId"] == session_id) + } + + #[cfg(test)] + pub(crate) fn seed_owned_session(&self, mut info: Value) { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + self.store_active(&mut info, &base, "", "fixture", self.client.clone()) + .unwrap(); + } + + #[cfg(test)] + pub(crate) fn set_test_control_base(&mut self, base: Url) { + self.test_control_base = Some(base); + } + + #[cfg(test)] + pub(crate) fn seed_discovered_sessions(&self, sessions: &[Value]) { + self.store_discovered(sessions); + } + + #[cfg(test)] + fn fixture_url(&self, url: Url) -> Url { + self.test_control_base.as_ref().map_or_else( + || url.clone(), + |base| { + let mut fixture = base.join(url.path()).expect("valid fixture URL"); + fixture.set_query(url.query()); + fixture + }, + ) + } + + fn capture_session_conflict( + &self, + status: reqwest::StatusCode, + payload: &Value, + base: &Url, + auth: &AuthSession, + ) -> Option { + if status == reqwest::StatusCode::UNAUTHORIZED || !is_session_conflict(payload) { + return None; + } + let sessions = payload["otherUserSessions"] + .as_array() + .into_iter() + .flatten() + .chain(payload.get("session")) + .filter_map(|session| remote_session_info(session, base)) + .filter(|session| value_i64(&session["appId"]).is_some_and(|id| id > 0)) + .filter(|session| { + session["serverIp"] + .as_str() + .is_some_and(|host| trusted_learned_server_base(host).is_ok()) + }) + .take(32) + .collect(); + *self + .conflict + .lock() + .expect("CloudMatch conflict state poisoned") = Some(SessionConflict { + owner: (auth.provider.idp_id.clone(), auth.user.user_id.clone()), + received: Instant::now(), + sessions, + }); + Some(ServiceError { + code: "session_conflict", + message: "A GeForce NOW session is already active. Resume it or end it before starting another game.".to_owned(), + }) + } + + fn take_conflict_sessions(&self, auth: &AuthSession) -> Option> { + self.conflict + .lock() + .expect("CloudMatch conflict state poisoned") + .take() + .filter(|conflict| { + conflict.owner == (auth.provider.idp_id.clone(), auth.user.user_id.clone()) + && conflict.received.elapsed() < Duration::from_secs(30) + && !conflict.sessions.is_empty() + }) + .map(|conflict| conflict.sessions) + } + + pub fn remote_sessions( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + let client = client_for_settings(&self.client, settings).map_err(invalid)?; + crate::requests::check()?; + if let Some(sessions) = self.take_conflict_sessions(auth) { + self.store_discovered(&sessions); + return Ok(json!({"sessions":sessions})); + } + let deadline = Instant::now() + DISCOVERY_TIMEOUT; + let current = self + .active + .lock() + .expect("CloudMatch state poisoned") + .clone(); + let recovery_region = current + .as_ref() + .filter(|state| params["sessionId"].as_str() == Some(state.session_id.as_str())) + .and_then(|state| { + trusted_cloudmatch_base(&state.control_base) + .ok() + .or_else(|| trusted_cloudmatch_base(&format!("https://{}", state.zone)).ok()) + }); + let requested = + recovery_region.map_or_else(|| requested_streaming_base(params, settings, auth), Ok)?; + let headers = cloudmatch_headers(session_token(auth), device_id)?; + let mut bases = vec![requested.clone()]; + let server_info_url = requested + .join("v2/serverInfo") + .map_err(|_| invalid("Invalid server-info URL"))?; + #[cfg(test)] + let server_info_url = self.fixture_url(server_info_url); + let server_info = client + .get(server_info_url) + .headers(headers.clone()) + .timeout(DISCOVERY_REQUEST_TIMEOUT) + .send() + .map_err(|error| network("Region discovery failed", error)) + .and_then(|response| { + if !response.status().is_success() { + return Err(response_error("Region discovery failed", response)); + } + let payload = response + .json::() + .map_err(|error| network("Invalid region response", error))?; + if payload["metaData"].as_array().is_none() + || (payload.get("requestStatus").is_some() + && value_i64(&payload["requestStatus"]["statusCode"]) != Some(1)) + { + return Err(upstream("Invalid region response")); + } + Ok(payload) + }); + if let Err(error) = &server_info + && matches!(error.code, "authentication_required" | "http_unauthorized") + { + return Err(error.clone()); + } + if let Ok(payload) = &server_info { + for base in regional_bases(payload) { + if !bases.contains(&base) { + bases.push(base); + } + } + } + let incomplete = server_info.is_err() || bases.len() > MAX_DISCOVERY_REGIONS; + bases.truncate(MAX_DISCOVERY_REGIONS); + let mut sessions = discover_sessions(&bases, deadline, incomplete, |base, timeout| { + let url = base + .join("v2/session") + .map_err(|_| invalid("Invalid active-session URL"))?; + #[cfg(test)] + let url = self.fixture_url(url); + let response = client + .get(url) + .headers(headers.clone()) + .timeout(timeout) + .send() + .map_err(|error| network("Active-session discovery failed", error))?; + let payload = + read_cloudmatch_response("Active-session discovery failed", response, false)?; + let sessions = payload["sessions"] + .as_array() + .ok_or_else(|| upstream("Invalid active-session response"))?; + Ok(sessions + .iter() + .filter_map(|session| remote_session_info(session, base)) + .collect()) + })?; + if let Some(pending) = self.pending_cleanup(auth) + && !sessions + .iter() + .any(|session| session["sessionId"] == pending["sessionId"]) + { + sessions.push(pending); + } + self.store_discovered(&sessions); + Ok(json!({"sessions":sessions})) + } + + fn store_discovered(&self, sessions: &[Value]) { + let mut discovered = self + .discovered + .lock() + .expect("CloudMatch discovery state poisoned"); + discovered.clear(); + for session in sessions { + if let Some(session_id) = session["sessionId"].as_str() { + discovered.insert(session_id.to_owned(), session.clone()); + } + } + } + + pub fn claim( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + self.attach(params, settings, auth, device_id, AttachMode::Claim, || { + Ok(()) + }) + } + + pub(crate) fn reconcile( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + check: impl Fn() -> Result<(), ServiceError>, + ) -> Result { + self.attach( + params, + settings, + auth, + device_id, + AttachMode::Reconcile, + check, + ) + } + + fn attach( + &self, + params: &Value, + settings: &Value, + auth: &AuthSession, + device_id: &str, + mode: AttachMode, + check: impl Fn() -> Result<(), ServiceError>, + ) -> Result { + let client = client_for_settings(&self.client, settings).map_err(invalid)?; + let session_id = params["sessionId"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("session.claim requires sessionId"))?; + let requested = requested_streaming_base(params, settings, auth)?; + let headers = cloudmatch_headers(session_token(auth), device_id)?; + let discovered = self + .discovered + .lock() + .expect("CloudMatch discovery state poisoned") + .get(session_id) + .cloned(); + let zone_base = discovered + .as_ref() + .and_then(|session| session["streamingBaseUrl"].as_str()) + .or_else(|| params["streamingBaseUrl"].as_str()) + .map(trusted_cloudmatch_base) + .transpose()? + .unwrap_or(requested); + let mut initial_base = claim_lookup_base(discovered.as_ref(), &zone_base); + let initial_payload = self + .get_session(&client, &initial_base, session_id, &headers) + .or_else(|error| { + if initial_base == zone_base + || matches!(error.code, "authentication_required" | "http_unauthorized") + { + Err(error) + } else { + let payload = self.get_session(&client, &zone_base, session_id, &headers)?; + initial_base = zone_base.clone(); + Ok(payload) + } + }); + let initial_payload = match initial_payload { + Err(error) if mode == AttachMode::Reconcile && error.code == "session_not_found" => { + check()?; + return Ok(json!({"session":null,"termination":{ + "source":"cloudmatch-http","httpStatus":404, + "sessionId":session_id,"resumable":false + }})); + } + result => result?, + }; + let session = &initial_payload["session"]; + if session["sessionId"] != session_id { + return Err(upstream("CloudMatch returned a different session ID")); + } + let retained_webrtc = self.retained_webrtc_allocation(session_id, auth); + if retained_webrtc.is_some() && reported_native_session(session) { + return Err(ServiceError { + code: "session_transport_mismatch", + message: "The saved WebRTC session now reports a native allocation. The session has been retained without changing its transport.".into(), + }); + } + let webrtc = reported_webrtc_session(session) || retained_webrtc.is_some(); + if webrtc && retained_webrtc.is_none() && !webrtc_compatibility_enabled(settings, auth) { + return Err(ServiceError { + code: "session_transport_not_enabled", + message: "This is a WebRTC session. Enable WebRTC compatibility before attaching; the cloud session has been retained.".into(), + }); + } + let initial_status = value_i64(&session["status"]).unwrap_or_default(); + let learned_server = first_string(&session["sessionControlInfo"]["ip"]); + let control_base = learned_server + .as_deref() + .and_then(|server| trusted_learned_server_base(server).ok()) + .unwrap_or(initial_base); + + let app_id = first_string(&session["sessionRequestData"]["appId"]) + .or_else(|| first_string(¶ms["appId"])) + .unwrap_or_else(|| "0".to_owned()); + if initial_status == 7 { + check()?; + self.clear_active(session_id); + let info = session_info(&initial_payload, &control_base, "", &app_id, device_id)?; + return Ok(json!({"session":info})); + } + if mode == AttachMode::Reconcile { + session_requires_resume(initial_status)?; + let zone = zone_base.host_str().unwrap_or_default(); + let mut info = session_info(&initial_payload, &control_base, zone, &app_id, device_id)?; + if let Some(retained) = &retained_webrtc { + preserve_session_profile(&mut info, retained); + } + check()?; + self.store_active(&mut info, &control_base, zone, &app_id, client)?; + self.remember_webrtc_allocation(&info, auth)?; + return Ok(json!({"session":info})); + } + let resumed = session_requires_resume(initial_status)?; + if resumed { + let mut url = control_base + .join(&format!("v2/session/{session_id}")) + .map_err(|_| invalid("Invalid CloudMatch claim URL"))?; + crate::language::append_session_preferences(&mut url, settings); + let mut resume_session = session.clone(); + if webrtc { + resume_session["transportMode"] = json!("webrtc"); + } + if let Some(active) = self + .active + .lock() + .expect("CloudMatch state poisoned") + .as_ref() + && active.session_id == session_id + { + resume_session["transportMode"] = active.info["transportMode"].clone(); + } + let body = build_resume_body(&app_id, &resume_session, settings, device_id); + #[cfg(test)] + let url = self.fixture_url(url); + let response = client + .put(url) + .headers(headers.clone()) + .json(&body) + .send() + .map_err(|error| network("Session claim failed", error))?; + let _ = read_cloudmatch_response("Session claim failed", response, true)?; + eprintln!( + "CloudMatch RESUME handover completed; awaiting fresh ready status and stream endpoints" + ); + } + + let zone = zone_base.host_str().unwrap_or_default(); + // Do not treat the pre-claim status or PUT acknowledgement as readiness. + // Qt schedules cancellable session.poll requests until the fresh GET + // reports status 2/3 with native endpoints. No long blocking RPC loop. + let mut info = session_info(&initial_payload, &control_base, zone, &app_id, device_id)?; + if let Some(retained) = &retained_webrtc { + preserve_session_profile(&mut info, retained); + } + if resumed { + info["keyboardLayout"] = json!(crate::language::session_keyboard_layout(settings)); + } + info["resumePending"] = json!(true); + info["phase"] = json!("resuming"); + self.store_active(&mut info, &control_base, zone, &app_id, client)?; + self.remember_webrtc_allocation(&info, auth)?; + Ok(json!({"session":info})) + } + + pub fn report_ad( + &self, + params: &Value, + auth: &AuthSession, + device_id: &str, + ) -> Result { + let current = self + .active + .lock() + .expect("CloudMatch state poisoned") + .clone(); + let client = current + .as_ref() + .map(|state| state.client.clone()) + .unwrap_or_else(|| self.client.clone()); + let session_id = params["sessionId"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| current.as_ref().map(|session| session.session_id.clone())) + .ok_or_else(|| invalid("session.ad.report requires sessionId"))?; + let action = match params["action"].as_str() { + Some("start") => 1, + Some("pause") => 2, + Some("resume") => 3, + Some("finish") => 4, + Some("cancel") => 5, + _ => return Err(invalid("Unknown session ad action")), + }; + let ad_id = params["adId"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("session.ad.report requires adId"))?; + let base = current + .as_ref() + .and_then(|session| trusted_cloudmatch_base(&session.control_base).ok()) + .ok_or_else(|| invalid("No active session control endpoint"))?; + let url = base + .join(&format!("v2/session/{session_id}")) + .map_err(|_| invalid("Invalid session ad update URL"))?; + #[cfg(test)] + let url = self.fixture_url(url); + let mut update = json!({ + "adId": ad_id, + "adAction": action, + "clientTimestamp": params["clientTimestamp"].as_i64().unwrap_or_else(unix_seconds) + }); + for key in ["watchedTimeInMs", "pausedTimeInMs"] { + if let Some(value) = params[key].as_i64() { + update[key] = json!(value.max(0)); + } + } + if let Some(reason) = params["cancelReason"].as_str() { + update["cancelReason"] = json!(reason); + } + let response = client + .put(url) + .headers(cloudmatch_headers(session_token(auth), device_id)?) + .json(&json!({"action":6,"adUpdates":[update]})) + .send() + .map_err(|error| network("Session ad update failed", error))?; + let payload = read_cloudmatch_response("Session ad update failed", response, false)?; + let app_id = current + .as_ref() + .map(|session| session.app_id.as_str()) + .unwrap_or("0"); + let zone = current + .as_ref() + .map(|session| session.zone.as_str()) + .unwrap_or(""); + let mut info = session_info(&payload, &base, zone, app_id, device_id)?; + info["phase"] = + Value::String(session_phase(info["status"].as_i64().unwrap_or_default()).to_owned()); + self.store_active(&mut info, &base, zone, app_id, client)?; + Ok(json!({"session":info})) + } + + fn store_active( + &self, + info: &mut Value, + fallback_base: &Url, + zone: &str, + app_id: &str, + client: Client, + ) -> Result<(), ServiceError> { + let session_id = info["sessionId"] + .as_str() + .ok_or_else(|| upstream("Session result did not include an ID"))? + .to_owned(); + let mut active = self.active.lock().expect("CloudMatch state poisoned"); + if info["status"] == 7 { + if active + .as_ref() + .is_some_and(|state| state.session_id == session_id) + { + *active = None; + } + return Ok(()); + } + if let Some(previous) = active.as_ref() { + preserve_session_profile(info, &previous.info); + if previous.session_id == session_id && info["keyboardLayout"].is_null() { + info["keyboardLayout"] = previous.info["keyboardLayout"].clone(); + } + } + *active = Some(ActiveSession { + session_id, + control_base: info["streamingBaseUrl"] + .as_str() + .unwrap_or_else(|| fallback_base.as_str()) + .to_owned(), + zone: zone.to_owned(), + app_id: app_id.to_owned(), + info: info.clone(), + client, + }); + Ok(()) + } + + pub(crate) fn clear_active(&self, session_id: &str) { + let mut active = self.active.lock().expect("CloudMatch state poisoned"); + if active + .as_ref() + .is_some_and(|state| state.session_id == session_id) + { + *active = None; + } + } + + fn get_session( + &self, + client: &Client, + base: &Url, + session_id: &str, + headers: &HeaderMap, + ) -> Result { + let url = base + .join(&format!("v2/session/{session_id}")) + .map_err(|_| invalid("Invalid CloudMatch polling URL"))?; + #[cfg(test)] + let url = self.fixture_url(url); + let mut last_error = None; + for attempt in 0..=2 { + match client.get(url.clone()).headers(headers.clone()).send() { + Ok(response) + if attempt < 2 + && matches!( + response.status().as_u16(), + 408 | 425 | 429 | 500 | 502 | 503 | 504 + ) => + { + thread::sleep(Duration::from_millis(if attempt == 0 { 250 } else { 750 })); + } + Ok(response) => { + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Err(ServiceError { + code: "session_not_found", + message: "The requested GeForce NOW session no longer exists." + .to_owned(), + }); + } + return read_cloudmatch_response("Session polling failed", response, false); + } + Err(error) => { + last_error = Some(error); + if attempt < 2 { + thread::sleep(Duration::from_millis(if attempt == 0 { 250 } else { 750 })); + } + } + } + } + Err(network( + "Session polling failed", + last_error.expect("polling loop records its final error"), + )) + } + + fn create_base( + &self, + client: &Client, + requested: &Url, + params: &Value, + settings: &Value, + token: &str, + device_id: &str, + ) -> Result { + if explicit_create_base(params, settings) { + return Ok(requested.clone()); + } + self.resolve_create_base(client, requested, token, device_id, true) + } + + fn resolve_create_base( + &self, + client: &Client, + requested: &Url, + token: &str, + device_id: &str, + prefer_regional: bool, + ) -> Result { + let Ok(url) = requested.join("v2/serverInfo") else { + return Ok(requested.clone()); + }; + let headers = cloudmatch_headers(token, device_id)?; + let Ok(response) = client.get(url).headers(headers).send() else { + return Ok(requested.clone()); + }; + if matches!(response.status().as_u16(), 401 | 403) { + return Err(response_error("Session region discovery failed", response)); + } + if !response.status().is_success() { + return Ok(requested.clone()); + } + let Ok(payload) = response.json::() else { + return Ok(requested.clone()); + }; + Ok(regional_bases(&payload) + .into_iter() + .find(|base| { + !prefer_regional || !base.host_str().unwrap_or_default().starts_with("np-") + }) + .unwrap_or_else(|| requested.clone())) + } +} + +fn launch_app_id(params: &Value) -> Result { + let value = params["appId"] + .as_str() + .or_else(|| params["launchAppId"].as_str()) + .or_else(|| params["variantId"].as_str()) + .ok_or_else(|| invalid("The selected game does not have a launch app ID"))?; + if value.is_empty() || !value.bytes().all(|character| character.is_ascii_digit()) { + return Err(invalid("The selected game launch app ID must be numeric")); + } + Ok(value.to_owned()) +} + +fn requested_streaming_base( + params: &Value, + settings: &Value, + auth: &AuthSession, +) -> Result { + let raw: String = params["streamingBaseUrl"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .or_else(|| { + settings["region"] + .as_str() + .filter(|value| value.starts_with("https://")) + .map(ToOwned::to_owned) + }) + .unwrap_or_else(|| { + let effective = crate::gfn::effective_provider_url(&auth.provider); + if effective.trim().is_empty() { + DEFAULT_STREAMING_BASE.to_owned() + } else { + effective + } + }); + trusted_cloudmatch_base(&raw) +} + +fn explicit_create_base(params: &Value, settings: &Value) -> bool { + params["streamingBaseUrl"] + .as_str() + .is_some_and(|value| !value.trim().is_empty()) + || settings["region"] + .as_str() + .is_some_and(|value| value.starts_with("https://")) +} + +fn claim_lookup_base(discovered: Option<&Value>, zone_base: &Url) -> Url { + discovered + .and_then(|session| session["serverIp"].as_str()) + .and_then(|server| trusted_learned_server_base(server).ok()) + .unwrap_or_else(|| zone_base.clone()) +} + +fn session_requires_resume(status: i64) -> Result { + match status { + 2..=5 => Ok(true), + 1 | 6 => Ok(false), + _ => Err(upstream( + "This GeForce NOW session is no longer resumable. End it and launch again.", + )), + } +} + +fn mark_resume_progress(info: &mut Value) { + if !matches!(info["status"].as_i64(), Some(1..=6)) { + info["resumePending"] = json!(false); + info["phase"] = json!(if info["status"] == 7 { + "finished" + } else { + "failed" + }); + return; + } + let ready = matches!(info["status"].as_i64(), Some(2 | 3)) + && if info["transportMode"] == "webrtc" { + has_webrtc_endpoint(info) + } else { + info["rtspsEndpoints"] + .as_array() + .is_some_and(|endpoints| !endpoints.is_empty()) + }; + info["resumePending"] = json!(!ready); + if !ready { + info["phase"] = json!("resuming"); + } +} + +fn build_resume_body(app_id: &str, session: &Value, settings: &Value, device_id: &str) -> Value { + let mut settings = settings.clone(); + let webrtc = reported_webrtc_session(session) || session["transportMode"] == "webrtc"; + settings["allianceWebrtcCompatibility"] = json!(webrtc); + let created = build_create_body(app_id, &json!({}), &settings, device_id); + let source = &created["sessionRequestData"]; + let mut request = serde_json::Map::new(); + // Resume must not renegotiate codec, monitor geometry, FPS or bitrate. + for key in [ + "appId", + "audioMode", + "remoteControllersBitmap", + "sdrHdrMode", + "networkTestSessionId", + "availableSupportedControllers", + "preferredController", + "clientVersion", + "deviceHashId", + "internalTitle", + "clientPlatformName", + "surroundAudioInfo", + "clientTimezoneOffset", + "clientIdentification", + "parentSessionId", + "streamerVersion", + "secureRTSPSupported", + ] { + request.insert(key.to_owned(), source[key].clone()); + } + request.insert( + "sdrHdrMode".to_owned(), + json!(accepted_hdr_mode(session).unwrap_or(0)), + ); + if webrtc { + request.insert("sdkVersion".to_owned(), json!("1.0")); + request.insert("enhancedStreamMode".to_owned(), json!(1)); + } + request.insert( + "metaData".to_owned(), + json!( + source["metaData"] + .as_array() + .into_iter() + .flatten() + .filter(|entry| entry["key"] != "clientPhysicalResolution") + .cloned() + .collect::>() + ), + ); + for key in [ + "appLaunchMode", + "enablePersistingInGameSettings", + "clientPlatformName", + ] { + if !session["sessionRequestData"][key].is_null() { + request.insert(key.to_owned(), session["sessionRequestData"][key].clone()); + } + } + json!({"action":2, "data":"RESUME", "sessionRequestData":request, + "metaData":null, "adUpdates":null}) +} + +fn monitor_display_data(hdr: bool, settings: &Value) -> Value { + if !hdr { + return Value::Null; + } + let mut data = json!({ + "desiredContentMaxLuminance":1000, + "desiredContentMinLuminance":0, + "desiredContentMaxFrameAverageLuminance":400 + }); + if let Some(display) = + crate::streamer::validated_native_hdr_display(&settings["nativeHdrDisplay"]) + { + data["desiredContentMaxLuminance"] = json!(display.maximum_nits); + data["desiredContentMinLuminance"] = + json!((display.minimum_nits * 10_000.0).round() as u64); + if let Some(object) = data.as_object_mut() { + object.remove("desiredContentMaxFrameAverageLuminance"); + } + if let Some(metadata) = display.metadata { + data["desiredContentMaxFrameAverageLuminance"] = + json!(metadata.maximum_full_frame_nits); + for (key, coordinate) in [ + "displayPrimaryX0", + "displayPrimaryY0", + "displayPrimaryX1", + "displayPrimaryY1", + "displayPrimaryX2", + "displayPrimaryY2", + "displayWhitePointX", + "displayWhitePointY", + ] + .iter() + .zip(metadata.coordinates) + { + data[*key] = json!((coordinate * 50_000.0).round() as u64); + } + } + } + data +} + +fn build_create_body(app_id: &str, params: &Value, settings: &Value, device_id: &str) -> Value { + let (width, height) = parse_resolution(&setting_string(settings, "resolution", "1920x1080")); + let fps = crate::frame_rate::request_frame_rate(settings, params, width, height); + // The codec stays client-selected: the official Bifrost client resolves it from local + // preferences ("Selected %s codec from client preferences") and announces the choice + // at RTSP time (x-nv-vqos bitStreamFormat). It is only read here to constrain the + // requested color envelope, never sent to CloudMatch. + let codec = codec_wire(&setting_string(settings, "codec", "auto")); + let hdr = setting_bool(settings, "enableHdr", false) + && setting_bool(settings, "nativeHdrSupported", false) + && matches!(codec, 2 | 3) + && setting_string(settings, "decoderPreference", "auto") != "software" + && !matches!( + setting_string(settings, "nativeVideoBackend", "auto").as_str(), + "software" | "ffmpeg" + ); + let requested_color = color_quality_wire(&setting_string(settings, "colorQuality", "8bit_420")); + // Keep a manually selected codec fixed. H.264 supports only 8-bit 4:2:0, + // while the official Windows client exposes AV1 at 4:2:0 only. Constrain + // color instead of silently switching an explicit codec back to Auto/HEVC. + let (bit_depth, chroma) = match codec { + 2 if hdr => (1, requested_color.1), + 3 if hdr => (1, 0), + 1 => (0, 0), + 3 => (requested_color.0, 0), + _ => requested_color, + }; + let cloud_gsync = resolved_cloud_gsync(settings); + let reflex = cloud_gsync || fps >= 120; + let persistence = setting_bool(settings, "enablePersistingInGameSettings", true) + && params["supportsInGameSettingsPersistence"].as_bool() == Some(true); + let metadata = vec![ + json!({"key":"wssignaling","value":"1"}), + json!({"key":"SubSessionId","value":random_uuid()}), + json!({"key":"surroundAudioInfo","value":"2"}), + ]; + // requestedStreamingFeatures, field-for-field with the official Bifrost request + // builder: reflex, bitDepth, cloudGsync, enabledL4S, mouseMovementFlags, trueHdr, + // supportedHidDevices, profile, fallbackToLogicalResolution, hidDevices, + // chromaFormat, prefilterMode/Sharpness/NoiseReduction, hudStreamingMode, dlssOverrides. + // Codec, bitrate ceiling, vsync, channel count, QoS policy, touch support, and the + // dynamic quality policy are deliberately absent: the official client resolves the + // codec locally and carries bitrate/policy purely in the RTSP ANNOUNCE. + let mut features = json!({ + "reflex":reflex, + "bitDepth":bit_depth, + "cloudGsync":cloud_gsync, + "enabledL4S":setting_bool(settings, "enableL4S", false), + "supportedHidDevices":0, + "profile":0, + "fallbackToLogicalResolution":false, + "chromaFormat":chroma, + "prefilterMode":0, + "prefilterSharpness":0, + "prefilterNoiseReduction":0, + "hudStreamingMode":0, + "dlssOverrides":crate::dlss::session_request(settings) + }); + features["mouseMovementFlags"] = json!(0); + features["trueHdr"] = json!(false); + features["hidDevices"] = Value::Null; + let mut body = json!({"sessionRequestData":{ + "appId":app_id.parse::().unwrap_or_default(), + "internalTitle":params["title"].as_str(), + "availableSupportedControllers":[2], + "preferredController":2, + "networkTestSessionId":params["networkTestSessionId"].as_str(), + "parentSessionId":null, + "clientIdentification":"GFN-PC", + "deviceHashId":device_id, + "clientVersion":"30.0", + "sdkVersion":"2.0", + "streamerVersion":"14", + "clientPlatformName":platform_name(settings), + "clientRequestMonitorSettings":[{ + "monitorId":0,"positionX":0,"positionY":0, + "widthInPixels":width,"heightInPixels":height,"framesPerSecond":fps, + "sdrHdrMode":if hdr { 1 } else { 0 }, + "displayData":monitor_display_data(hdr, settings), + "hdr10PlusGamingData":null, + "dpi":if cfg!(target_os = "macos") { 144 } else { 96 } + }], + "useOps":true, + "audioMode":2, + "metaData":metadata, + "sdrHdrMode":if hdr { 1 } else { 0 }, + "clientDisplayHdrCapabilities":null, + "surroundAudioInfo":0, + "remoteControllersBitmap":0, + "clientTimezoneOffset":client_timezone_offset_ms(*chrono::Local::now().offset()), + "enhancedStreamMode":0, + "appLaunchMode":app_launch_mode(params), + "secureRTSPSupported":true, + "partnerCustomData":null, + "accountLinked":params["accountLinked"].as_bool().unwrap_or(false), + "enablePersistingInGameSettings":persistence, + "requestedAudioFormat":0, + "userAge":25, + "requestedStreamingFeatures":features, + "transport":null + }}); + if settings["allianceWebrtcCompatibility"] == true { + let request = &mut body["sessionRequestData"]; + request["sdkVersion"] = json!("1.0"); + request["streamerVersion"] = json!(1); + request["enhancedStreamMode"] = json!(1); + request["secureRTSPSupported"] = json!(false); + request["remoteControllersBitmap"] = json!(1); + request["metaData"] + .as_array_mut() + .unwrap() + .push(json!({"key":"GSStreamerType","value":"WebRTC"})); + request + .as_object_mut() + .unwrap() + .remove("requestedAudioFormat"); + request.as_object_mut().unwrap().remove("transport"); + } + body +} + +fn valid_webrtc_endpoint(value: &Value) -> bool { + value + .as_str() + .and_then(|raw| Url::parse(raw).ok()) + .is_some_and(|url| { + url.scheme() == "wss" + && url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() + }) +} + +pub(crate) fn has_webrtc_endpoint(session: &Value) -> bool { + session["connectionInfo"] + .as_array() + .is_some_and(|connections| { + connections + .iter() + .filter_map(webrtc_signaling_endpoint) + .any(|url| session["signalingUrl"] == url.as_str()) + }) +} + +fn select_webrtc_endpoint(session: &mut Value) { + if session["transportMode"] != "webrtc" { + return; + } + let endpoint = session["connectionInfo"] + .as_array() + .and_then(|connections| { + connections + .iter() + .filter(|connection| { + let resource = connection["resourcePath"].as_str().unwrap_or_default(); + resource.starts_with("wss://") + || resource.starts_with("https://") + || (value_i64(&connection["usage"]) == Some(14) + && !matches!(value_i64(&connection["appLevelProtocol"]), Some(1 | 6)) + && !resource.starts_with("rtsp://") + && !resource.starts_with("rtsps://")) + }) + .chain( + connections + .iter() + .filter(|connection| value_i64(&connection["usage"]) == Some(14)), + ) + .chain( + connections + .iter() + .filter(|connection| value_i64(&connection["usage"]) == Some(16)), + ) + .find_map(|connection| { + webrtc_signaling_endpoint(connection) + .map(|url| (url, first_string(&connection["ip"]))) + }) + }); + let media = session["connectionInfo"] + .as_array() + .and_then(|connections| { + [2, 17] + .into_iter() + .find_map(|usage| { + connections + .iter() + .filter(|connection| value_i64(&connection["usage"]) == Some(usage)) + .find_map(media_connection_info) + }) + .or_else(|| { + connections + .iter() + .filter(|connection| { + value_i64(&connection["usage"]) == Some(14) + && !matches!( + value_i64(&connection["appLevelProtocol"]), + Some(1 | 6) + ) + && !connection["resourcePath"].as_str().is_some_and(|resource| { + resource.starts_with("rtsp://") + || resource.starts_with("rtsps://") + }) + }) + .filter_map(media_connection_info) + .max_by_key(|media| media["port"].as_i64()) + }) + }); + session["mediaConnectionInfo"] = json!(media); + session["signalingUrl"] = json!(endpoint.as_ref().map(|(url, _)| url.as_str())); + if let Some((endpoint, server_ip)) = endpoint { + session["serverIp"] = json!(server_ip.or_else(|| { + endpoint + .host_str() + .map(|host| host.trim_matches(['[', ']']).to_owned()) + })); + session["signalingServer"] = json!(format!( + "{}:{}", + endpoint.host_str().unwrap_or_default(), + endpoint.port_or_known_default().unwrap_or(443) + )); + } +} + +fn media_connection_info(connection: &Value) -> Option { + let ip = first_string(&connection["ip"]).or_else(|| { + connection["resourcePath"] + .as_str() + .and_then(host_from_resource) + })?; + let port = value_i64(&connection["port"]).or_else(|| { + Url::parse(connection["resourcePath"].as_str()?) + .ok()? + .port() + .map(i64::from) + })?; + (1..=65535) + .contains(&port) + .then(|| json!({"ip":ip,"port":port,"usage":connection["usage"]})) +} + +fn webrtc_signaling_endpoint(connection: &Value) -> Option { + if !matches!(value_i64(&connection["usage"]), Some(14 | 16)) { + return None; + } + let resource = connection["resourcePath"] + .as_str() + .filter(|path| !path.is_empty()) + .unwrap_or("/nvst/"); + let rtsp = resource.starts_with("rtsp://") || resource.starts_with("rtsps://"); + let native_descriptor = value_i64(&connection["usage"]) == Some(16) + || matches!(value_i64(&connection["appLevelProtocol"]), Some(1 | 6)); + let absolute = resource.starts_with("wss://") || resource.starts_with("https://") || rtsp; + let mut url = if absolute { + let mut url = if rtsp { + Url::parse(&format!("wss://{}", resource.split_once("://")?.1)).ok()? + } else { + Url::parse(resource).ok()? + }; + url.set_scheme("wss").ok()?; + if rtsp { + url.set_port(None).ok()?; + url.set_path("/nvst/"); + url.set_query(None); + } + url + } else { + if resource.contains("://") || !resource.starts_with('/') { + return None; + } + let host = connection["ip"].as_str()?; + let mut url = Url::parse("wss://endpoint.invalid/").ok()?; + if let Ok(ip) = host.parse::() { + url.set_ip_host(ip).ok()?; + } else { + url.set_host(Some(host)).ok()?; + } + url.set_path(resource); + url + }; + if !absolute && let Some(host) = connection["ip"].as_str().filter(|host| !host.is_empty()) { + if let Ok(ip) = host.parse::() { + if ip.is_unspecified() { + return None; + } + url.set_ip_host(ip).ok()?; + } else { + url.set_host(Some(host)).ok()?; + } + } + if !absolute + && !native_descriptor + && let Some(port) = value_i64(&connection["port"]) + { + if !(1..=65535).contains(&port) { + return None; + } + url.set_port(Some(port as u16)).ok()?; + } + valid_webrtc_endpoint(&json!(url.as_str())).then_some(url) +} + +fn session_info( + payload: &Value, + fallback_base: &Url, + zone: &str, + fallback_app_id: &str, + device_id: &str, +) -> Result { + let session = &payload["session"]; + let session_id = session["sessionId"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| upstream("CloudMatch response did not include a session ID"))?; + let status = value_i64(&session["status"]).unwrap_or_default(); + let connections = session["connectionInfo"] + .as_array() + .cloned() + .unwrap_or_default(); + let signaling_connection = connections.iter().find(|connection| { + matches!(value_i64(&connection["usage"]), Some(16)) + || (value_i64(&connection["usage"]) == Some(14) + && matches!(value_i64(&connection["appLevelProtocol"]), Some(1 | 6))) + }); + let control_host = first_string(&session["sessionControlInfo"]["ip"]); + let server_ip = signaling_connection + .and_then(|connection| first_string(&connection["ip"])) + .or_else(|| { + signaling_connection + .and_then(|connection| connection["resourcePath"].as_str()) + .and_then(host_from_resource) + }) + .or_else(|| control_host.clone()) + .or_else(|| fallback_base.host_str().map(ToOwned::to_owned)) + .unwrap_or_default(); + let control_base = control_host + .as_deref() + .filter(|_| { + value_i64(&session["sessionControlInfo"]["port"]).is_none_or(|port| port == 443) + }) + .and_then(|host| trusted_learned_server_base(host).ok()) + .map(|base| base.origin().ascii_serialization()) + .unwrap_or_else(|| fallback_base.origin().ascii_serialization()); + let queue_position = queue_position(session); + let seat_setup_step = value_i64(&session["seatSetupInfo"]["seatSetupStep"]); + let app_id = first_string(&session["sessionRequestData"]["appId"]) + .unwrap_or_else(|| fallback_app_id.to_owned()); + let rtsps_endpoints = connections + .iter() + .filter(|connection| { + value_i64(&connection["usage"]) == Some(16) + || (value_i64(&connection["usage"]) == Some(14) + && matches!(value_i64(&connection["appLevelProtocol"]), Some(1 | 6))) + }) + .filter_map(|connection| { + let address = first_string(&connection["ip"]) + .or_else(|| first_string(&connection["resourcePath"]))?; + if value_i64(&connection["port"]).is_some_and(|port| !(1..=65535).contains(&port)) { + return None; + } + if address.starts_with("rtsps://") { + let url = Url::parse(&address).ok()?; + return (url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.port().is_some()) + .then_some(address); + } + if address.contains(&['/', '@', '?', '#'][..]) { + return None; + } + let port = value_i64(&connection["port"])?; + let host = if address.parse::().is_ok_and(|ip| ip.is_ipv6()) { + format!("[{address}]") + } else { + address + }; + let endpoint = format!("rtsps://{host}:{port}"); + Url::parse(&endpoint).ok()?.host_str().map(|_| endpoint) + }) + .collect::>(); + let signaling_url = rtsps_endpoints + .first() + .map(|endpoint| endpoint.replacen("rtsps://", "wss://", 1)); + let ice_servers = normalize_ice_servers(session); + let media = connections + .iter() + .find(|connection| matches!(value_i64(&connection["usage"]), Some(2 | 17))) + .and_then(media_connection_info); + let monitor = &session["sessionRequestData"]["clientRequestMonitorSettings"][0]; + let mut features = session["sessionRequestData"]["requestedStreamingFeatures"] + .as_object() + .cloned() + .unwrap_or_default(); + if let Some(finalized) = session["finalizedStreamingFeatures"].as_object() { + features.extend(finalized.clone()); + } + let codec_reported = + session["negotiatedStreamProfile"].get("codec").is_some() || features.contains_key("codec"); + let mut negotiated = negotiated_profile(monitor, &Value::Object(features)); + for key in ["bitDepth", "chromaFormat"] { + if session["finalizedStreamingFeatures"].get(key).is_some() { + negotiated[format!("{key}Source")] = json!("finalized"); + } + } + if let Some(codec) = session["negotiatedStreamProfile"].get("codec") { + negotiated["codec"] = json!(codec.as_str().and_then(|value| { + match value.trim().to_ascii_uppercase().as_str() { + "H264" | "AVC" => Some("H264"), + "H265" | "HEVC" => Some("H265"), + "AV1" => Some("AV1"), + _ => None, + } + })); + } + negotiated["codecSource"] = json!(if codec_reported { + "server" + } else { + "unreported" + }); + let hdr = accepted_hdr_mode(session); + negotiated["enableHdr"] = json!(hdr.map(|mode| mode == 1)); + negotiated["enableHdrSource"] = json!(if hdr_mode_value(session).is_some() { + "server" + } else { + "unreported" + }); + let ad_state = normalize_ad_state(session); + let mut info = json!({ + "sessionId":session_id, + "transportMode":if reported_webrtc_session(session) { "webrtc" } else { "nvst" }, + "reportedNativeTransport":reported_native_session(session), + "subSessionId":session["subSessionId"], + "appId":app_id, + "status":status, + "phase":session_phase(status), + "termination":if status == 7 { json!({"source":"cloudmatch-session-status","status":7,"sessionId":session_id,"resumable":false}) } else { Value::Null }, + "queuePosition":queue_position, + "seatSetupStep":seat_setup_step, + "adState":ad_state, + "zone":zone, + "streamingBaseUrl":control_base, + "serverIp":server_ip, + "signalingServer":if server_ip.contains(':') { server_ip.clone() } else { format!("{server_ip}:443") }, + "signalingUrl":signaling_url, + "serverLocation":session["serverLocation"], + "gpuType":session["gpuType"], + "appLaunchMode":session["sessionRequestData"]["appLaunchMode"], + "enablePersistingInGameSettings":session["sessionRequestData"]["enablePersistingInGameSettings"], + "connectionInfo":connections, + "rtspsEndpoints":rtsps_endpoints, + "iceServers":ice_servers, + "mediaConnectionInfo":media, + "negotiatedStreamProfile":negotiated, + "hdrDowngrade":null, + "catalogAppId":null, + "requestedStreamingFeatures":session["sessionRequestData"]["requestedStreamingFeatures"], + "finalizedStreamingFeatures":session["finalizedStreamingFeatures"], + "clientId":LCARS_CLIENT_ID, + "deviceId":device_id + }); + select_webrtc_endpoint(&mut info); + Ok(info) +} + +fn normalize_ad_state(session: &Value) -> Value { + let ads = session["sessionAds"] + .as_array() + .cloned() + .unwrap_or_default(); + let required = session["sessionAdsRequired"] + .as_bool() + .or_else(|| session["isAdsRequired"].as_bool()) + .or_else(|| session["sessionProgress"]["isAdsRequired"].as_bool()) + .unwrap_or(!ads.is_empty()); + let opportunity = session["opportunity"].clone(); + if !required && ads.is_empty() && opportunity.is_null() { + Value::Null + } else { + json!({ + "isAdsRequired":required, + "sessionAdsRequired":required, + "isQueuePaused":opportunity["queuePaused"].as_bool().unwrap_or(false), + "gracePeriodSeconds":opportunity["gracePeriodSeconds"], + "message":opportunity["message"], + "sessionAds":ads, + "ads":ads, + "opportunity":opportunity + }) + } +} + +fn accepted_hdr_mode(session: &Value) -> Option { + hdr_mode_value(session) + .and_then(value_i64) + .map(|mode| i64::from(mode == 1)) +} + +fn hdr_mode_value(session: &Value) -> Option<&Value> { + session + .get("sdrHdrMode") + .or_else(|| { + session["sessionRequestData"]["clientRequestMonitorSettings"][0].get("sdrHdrMode") + }) + .or_else(|| session["sessionRequestData"].get("sdrHdrMode")) +} + +fn preserve_session_profile(info: &mut Value, previous: &Value) { + let Some(session_id) = info["sessionId"].as_str().filter(|id| !id.is_empty()) else { + return; + }; + let profile = &previous["negotiatedStreamProfile"]; + if previous["sessionId"].as_str() != Some(session_id) { + return; + } + for key in ["hdrDowngrade", "catalogAppId"] { + if info[key].is_null() { + info[key] = previous[key].clone(); + } + } + if previous["transportMode"] == "webrtc" { + info["transportMode"] = json!("webrtc"); + select_webrtc_endpoint(info); + for key in ["resolution", "width", "height", "fps"] { + if info["negotiatedStreamProfile"][key].is_null() && !profile[key].is_null() { + info["negotiatedStreamProfile"][key] = profile[key].clone(); + } + } + } + if (info["negotiatedStreamProfile"]["codecSource"] == "unreported" + || (info["negotiatedStreamProfile"]["codecSource"] == "request" + && profile["codecSource"] == "server")) + && matches!(profile["codec"].as_str(), Some("H264" | "H265" | "AV1")) + && matches!(profile["codecSource"].as_str(), Some("request" | "server")) + { + info["negotiatedStreamProfile"]["codec"] = profile["codec"].clone(); + info["negotiatedStreamProfile"]["codecSource"] = profile["codecSource"].clone(); + } + for key in ["bitDepth", "chromaFormat", "enableHdr"] { + let source = format!("{key}Source"); + let current = &info["negotiatedStreamProfile"][&source]; + if (current == "unreported" || (current == "request" && profile[&source] == "finalized")) + && matches!( + profile[&source].as_str(), + Some("request" | "server" | "finalized") + ) + { + info["negotiatedStreamProfile"][key] = profile[key].clone(); + info["negotiatedStreamProfile"][&source] = profile[&source].clone(); + } + } + let updated = &mut info["negotiatedStreamProfile"]; + if updated.get("bitDepthSource").is_some() || updated.get("chromaFormatSource").is_some() { + updated["colorQuality"] = json!(profile_color( + &updated["bitDepth"], + &updated["chromaFormat"] + )); + } +} + +fn profile_color(depth: &Value, chroma: &Value) -> Option<&'static str> { + match (depth.as_i64(), chroma.as_i64()) { + (Some(8), Some(0)) => Some("8bit_420"), + (Some(8), Some(1)) => Some("8bit_444"), + (Some(10), Some(0)) => Some("10bit_420"), + (Some(10), Some(1)) => Some("10bit_444"), + _ => None, + } +} + +fn codec_from_wire(value: &Value) -> Option<&'static str> { + match value_i64(value) { + Some(1) => Some("H264"), + Some(2) => Some("H265"), + Some(3) => Some("AV1"), + _ => None, + } +} + +fn negotiated_profile(monitor: &Value, features: &Value) -> Value { + let width = value_i64(&monitor["widthInPixels"]); + let height = value_i64(&monitor["heightInPixels"]); + let resolution = width + .zip(height) + .map(|(width, height)| format!("{width}x{height}")); + let codec = codec_from_wire(&features["codec"]); + let bit_depth = value_i64(&features["bitDepth"]).and_then(|value| match value { + 0 | 8 => Some(8), + 1 | 10 => Some(10), + _ => None, + }); + let chroma = value_i64(&features["chromaFormat"]).and_then(|value| match value { + 0 => Some(0), + 1 => Some(1), + _ => None, + }); + let color = profile_color(&json!(bit_depth), &json!(chroma)); + let dynamic_streaming_mode = + value_i64(&features["dynamicStreamingMode"]).filter(|value| (0..=3).contains(value)); + json!({ + "resolution":resolution, + // Carried alongside `resolution` so consumers do not have to re-parse + // the string. The stream view sizes its video item from these, and a + // missing size makes it stretch the picture to the window instead of + // preserving the aspect ratio. + "width":width, + "height":height, + "fps":value_i64(&monitor["framesPerSecond"]), + "codec":codec, + "colorQuality":color, + "bitDepth":bit_depth, + "chromaFormat":chroma, + "bitDepthSource":if features.get("bitDepth").is_some() { "request" } else { "unreported" }, + "chromaFormatSource":if features.get("chromaFormat").is_some() { "request" } else { "unreported" }, + "dynamicStreamingMode":dynamic_streaming_mode, + "enableL4S":features["enabledL4S"], + "enableCloudGsync":features["cloudGsync"], + "enableReflex":features["reflex"] + }) +} + +fn cloudmatch_headers(token: &str, device_id: &str) -> Result { + let mut headers = HeaderMap::new(); + let user_agent = bifrost_user_agent(); + insert_header(&mut headers, USER_AGENT, &user_agent)?; + insert_header(&mut headers, AUTHORIZATION, &format!("GFNJWT {token}"))?; + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + headers.insert("nv-client-id", HeaderValue::from_static(LCARS_CLIENT_ID)); + headers.insert( + "nv-client-streamer", + HeaderValue::from_static("NVIDIA-CLASSIC"), + ); + headers.insert("nv-client-type", HeaderValue::from_static("NATIVE")); + headers.insert( + "nv-client-version", + HeaderValue::from_static(GFN_CLIENT_VERSION), + ); + headers.insert("nv-device-os", HeaderValue::from_static(device_os())); + headers.insert("nv-device-type", HeaderValue::from_static(device_type())); + headers.insert("nv-device-make", HeaderValue::from_static(device_make())); + headers.insert("nv-device-model", HeaderValue::from_static(device_model())); + insert_header(&mut headers, "x-device-id", device_id)?; + insert_header(&mut headers, "x-nv-client-identity", &user_agent)?; + Ok(headers) +} + +fn insert_header( + headers: &mut HeaderMap, + name: impl reqwest::header::IntoHeaderName, + value: &str, +) -> Result<(), ServiceError> { + headers.insert( + name, + HeaderValue::from_str(value).map_err(|_| invalid("Invalid CloudMatch header value"))?, + ); + Ok(()) +} + +fn read_cloudmatch_response( + context: &str, + response: Response, + allow_not_paused: bool, +) -> Result { + let status = response.status(); + let payload = response.json::(); + validate_cloudmatch_response(context, status, payload, allow_not_paused) +} + +fn validate_cloudmatch_response( + context: &str, + status: reqwest::StatusCode, + payload: Result, + allow_not_paused: bool, +) -> Result { + if allow_not_paused + && status != reqwest::StatusCode::UNAUTHORIZED + && status != reqwest::StatusCode::FORBIDDEN + && payload.as_ref().is_ok_and(|payload| { + value_i64(&payload["requestStatus"]["statusCode"]) == Some(34) + || payload["requestStatus"]["statusDescription"] + .as_str() + .is_some_and(|description| description.contains("SESSION_NOT_PAUSED")) + }) + { + return payload.map_err(|error| network("CloudMatch returned invalid JSON", error)); + } + if !status.is_success() { + return Err(cloudmatch_http_error( + context, + status, + payload.ok().as_ref(), + )); + } + let payload = payload.map_err(|error| network("CloudMatch returned invalid JSON", error))?; + if value_i64(&payload["requestStatus"]["statusCode"]) != Some(1) { + let description = payload["requestStatus"]["statusDescription"] + .as_str() + .unwrap_or("CloudMatch rejected the request"); + let code = value_i64(&payload["requestStatus"]["unifiedErrorCode"]) + .or_else(|| value_i64(&payload["session"]["errorCode"])); + return Err(ServiceError { + code: "session_error", + message: code.map_or_else( + || description.to_owned(), + |code| format!("{description} ({code})"), + ), + }); + } + Ok(payload) +} + +fn is_session_conflict(payload: &Value) -> bool { + value_i64(&payload["requestStatus"]["statusCode"]) == Some(11) + || payload["requestStatus"]["statusDescription"] + .as_str() + .is_some_and(|description| description.to_ascii_uppercase().contains("SESSION_LIMIT")) + || [ + &payload["requestStatus"]["unifiedErrorCode"], + &payload["session"]["errorCode"], + ] + .iter() + .any(|code| { + value_i64(code) == Some(0x4AF1201E) + || code.as_str().is_some_and(|code| { + code.trim_start_matches("0x") + .eq_ignore_ascii_case("4AF1201E") + }) + }) +} + +fn response_error(context: &str, response: Response) -> ServiceError { + let status = response.status(); + cloudmatch_http_error(context, status, response.json::().ok().as_ref()) +} + +fn cloudmatch_http_error( + context: &str, + status: reqwest::StatusCode, + payload: Option<&Value>, +) -> ServiceError { + let detail = payload.and_then(|payload| payload["requestStatus"]["statusDescription"].as_str()); + ServiceError { + code: if status.as_u16() == 401 { + "http_unauthorized" + } else if status.as_u16() == 403 { + "authentication_required" + } else { + "upstream_error" + }, + message: detail.map_or_else( + || format!("{context} ({status})"), + |detail| format!("{context} ({status}): {detail}"), + ), + } +} + +pub(crate) fn trusted_cloudmatch_base(raw: &str) -> Result { + let mut url = Url::parse(raw.trim()).map_err(|_| invalid("Invalid CloudMatch endpoint"))?; + if url.scheme() != "https" + || !url.username().is_empty() + || url.password().is_some() + || url.port().is_some_and(|port| port != 443) + || !url.host().is_some_and(public_session_host) + { + return Err(invalid("Untrusted CloudMatch endpoint")); + } + url.set_path("/"); + url.set_query(None); + url.set_fragment(None); + Ok(url) +} + +fn trusted_learned_server_base(server: &str) -> Result { + let raw = if server.starts_with("https://") { + server.to_owned() + } else if server.contains(':') && server.parse::().is_ok() { + format!("https://[{server}]") + } else { + format!("https://{server}") + }; + trusted_cloudmatch_base(&raw) +} + +fn public_session_host(host: url::Host<&str>) -> bool { + match host { + url::Host::Domain(name) => name.contains('.') && !name.ends_with(".local"), + url::Host::Ipv4(address) => public_session_ipv4(address), + url::Host::Ipv6(address) => address.to_ipv4_mapped().map_or_else( + || { + !address.is_loopback() + && !address.is_unicast_link_local() + && !address.is_unspecified() + && !address.is_unique_local() + && !address.is_multicast() + }, + public_session_ipv4, + ), + } +} + +fn public_session_ipv4(address: std::net::Ipv4Addr) -> bool { + let octets = address.octets(); + !address.is_private() + && !address.is_loopback() + && !address.is_link_local() + && !address.is_unspecified() + && !address.is_broadcast() + && !address.is_multicast() + && octets[0] != 0 + && octets[0] < 240 + && !(octets[0] == 100 && (64..=127).contains(&octets[1])) +} + +fn session_server_ip(session: &Value) -> Option { + first_string(&session["sessionControlInfo"]["ip"]).or_else(|| { + session["connectionInfo"] + .as_array() + .into_iter() + .flatten() + .find(|connection| { + value_i64(&connection["usage"]) == Some(14) + && first_string(&connection["ip"]).is_some() + }) + .and_then(|connection| first_string(&connection["ip"])) + }) +} + +fn remote_session_info(session: &Value, base: &Url) -> Option { + let session_id = session["sessionId"] + .as_str() + .filter(|id| !id.trim().is_empty())? + .to_owned(); + let status = value_i64(&session["status"])?; + if !matches!(status, 1..=6) { + return None; + } + let app_id = value_i64(&session["sessionRequestData"]["appId"]).unwrap_or_default(); + let server_ip = first_string(&session["sessionControlInfo"]["ip"]) + .or_else(|| session_server_ip(session)) + .or_else(|| { + session["connectionInfo"] + .as_array()? + .iter() + .filter(|connection| value_i64(&connection["usage"]) == Some(14)) + .find_map(|connection| { + let url = Url::parse(connection["resourcePath"].as_str()?).ok()?; + url.host_str().map(ToOwned::to_owned) + }) + }); + let monitor = session["monitorSettings"] + .as_array() + .and_then(|values| values.first()) + .unwrap_or(&session["sessionRequestData"]["clientRequestMonitorSettings"][0]); + let resolution = value_i64(&monitor["widthInPixels"]) + .zip(value_i64(&monitor["heightInPixels"])) + .map(|(width, height)| format!("{width}x{height}")); + Some(json!({ + "sessionId":session_id, + "subSessionId":session["subSessionId"], + "appId":app_id, + "appLaunchMode":session["sessionRequestData"]["appLaunchMode"], + "enablePersistingInGameSettings":session["sessionRequestData"]["enablePersistingInGameSettings"], + "gpuType":session["gpuType"], + "status":status, + "phase":session_phase(status), + "queuePosition":queue_position(session), + "seatSetupStep":value_i64(&session["seatSetupInfo"]["seatSetupStep"]), + "streamingBaseUrl":base.origin().ascii_serialization(), + "serverIp":server_ip, + "signalingUrl":null, + "resolution":resolution, + "fps":value_i64(&monitor["framesPerSecond"]) + })) +} + +fn unix_seconds() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +fn discover_sessions( + bases: &[Url], + deadline: Instant, + mut incomplete: bool, + fetch: impl Fn(&Url, Duration) -> Result, ServiceError> + Sync, +) -> Result, ServiceError> { + let fetch = &fetch; + let cancellation = crate::requests::current(); + let mut results = thread::scope(|scope| { + let workers = (0..DISCOVERY_CONCURRENCY.min(bases.len())) + .map(|worker| { + let cancellation = cancellation.clone(); + scope.spawn(move || { + bases + .iter() + .enumerate() + .skip(worker) + .step_by(DISCOVERY_CONCURRENCY) + .map(|(index, base)| { + let remaining = deadline.saturating_duration_since(Instant::now()); + let result = cancellation.check().and_then(|()| { + if remaining.is_zero() { + Err(discovery_failed()) + } else { + fetch(base, remaining.min(DISCOVERY_REQUEST_TIMEOUT)) + } + }); + (index, result) + }) + .collect::>() + }) + }) + .collect::>(); + workers + .into_iter() + .flat_map(|worker| worker.join().expect("CloudMatch discovery worker panicked")) + .collect::>() + }); + cancellation.check()?; + results.sort_by_key(|(index, _)| *index); + let mut sessions = Vec::new(); + for (_, result) in results { + match result { + Ok(found) => { + for session in found { + if !sessions + .iter() + .any(|known: &Value| known["sessionId"] == session["sessionId"]) + { + sessions.push(session); + } + } + } + Err(error) if matches!(error.code, "authentication_required" | "http_unauthorized") => { + return Err(error); + } + Err(_) => incomplete = true, + } + } + if sessions.is_empty() && (incomplete || bases.is_empty()) { + return Err(discovery_failed()); + } + Ok(sessions) +} + +fn discovery_failed() -> ServiceError { + ServiceError { + code: "session_discovery_failed", + message: "Could not check all GeForce NOW regions for an existing session. Try again." + .to_owned(), + } +} + +fn regional_bases(payload: &Value) -> Vec { + let metadata = payload["metaData"].as_array().cloned().unwrap_or_default(); + let value_for = |key: &str| { + metadata.iter().find_map(|entry| { + (entry["key"].as_str() == Some(key)) + .then(|| entry["value"].as_str().map(ToOwned::to_owned)) + .flatten() + }) + }; + let mut names = Vec::new(); + if let Some(local) = value_for("local-region") { + names.push(local); + } + names.extend( + value_for("gfn-regions") + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned), + ); + let mut result = Vec::new(); + for name in names { + if let Some(raw) = value_for(&name) + && let Ok(base) = trusted_cloudmatch_base(&raw) + && !result.iter().any(|existing: &Url| existing == &base) + { + result.push(base); + } + } + result +} + +fn normalize_ice_servers(session: &Value) -> Vec { + let mut servers = session["iceServerConfiguration"]["iceServers"] + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .filter_map(|entry| { + let urls = if let Some(values) = entry["urls"].as_array() { + values.clone() + } else { + entry["urls"].as_str().map(|value| vec![json!(value)])? + }; + (!urls.is_empty()).then(|| { + json!({"urls":urls,"username":entry["username"],"credential":entry["credential"]}) + }) + }) + .collect::>(); + if servers.is_empty() { + servers.push(json!({"urls":[DEFAULT_STUN_SERVER]})); + servers.push(json!({"urls":["stun:stun.l.google.com:19302"]})); + servers.push(json!({"urls":["stun:stun1.l.google.com:19302"]})); + } + servers +} + +fn queue_position(session: &Value) -> Option { + [ + &session["queuePosition"], + &session["seatSetupInfo"]["queuePosition"], + &session["sessionProgress"]["queuePosition"], + &session["progressInfo"]["queuePosition"], + ] + .into_iter() + .find_map(value_i64) + .filter(|value| *value > 0) +} + +fn first_string(value: &Value) -> Option { + value + .as_str() + .map(ToOwned::to_owned) + .or_else(|| value.as_i64().map(|value| value.to_string())) + .or_else(|| { + value + .as_array() + .and_then(|values| values.first()) + .and_then(first_string) + }) + .filter(|value| !value.trim().is_empty()) +} + +fn value_i64(value: &Value) -> Option { + value + .as_i64() + .or_else(|| value.as_u64().and_then(|value| i64::try_from(value).ok())) + .or_else(|| value.as_str()?.parse().ok()) +} + +fn host_from_resource(resource: &str) -> Option { + let translated = resource + .replacen("rtsps://", "https://", 1) + .replacen("rtsp://", "http://", 1); + Url::parse(&translated) + .ok()? + .host_str() + .map(ToOwned::to_owned) +} + +fn session_phase(status: i64) -> &'static str { + match status { + 1 => "preparing", + 2 => "ready", + 3 => "streaming", + 4 | 5 => "paused", + 6 => "resuming", + 7 => "finished", + status if status > 3 => "failed", + _ => "requesting", + } +} + +fn cancelled_allocation() -> ServiceError { + ServiceError { + code: "cancelled", + message: "Fresh allocation cancelled and cleaned up".to_owned(), + } +} + +fn delete_session(client: &Client, url: Url, headers: HeaderMap) -> Result<(), ServiceError> { + let response = client + .delete(url) + .headers(headers) + .timeout(Duration::from_secs(8)) + .send() + .map_err(|error| network("Allocation cleanup failed", error))?; + validate_delete_response("Allocation cleanup failed", response) +} + +fn validate_delete_response(context: &str, response: Response) -> Result<(), ServiceError> { + let status = response.status(); + if matches!( + status, + reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::NO_CONTENT + ) { + return Ok(()); + } + if !status.is_success() { + return Err(response_error(context, response)); + } + let bytes = response.bytes().map_err(|error| network(context, error))?; + if bytes.is_empty() { + return Ok(()); + } + let payload: Value = serde_json::from_slice(&bytes) + .map_err(|_| upstream("Invalid session deletion response"))?; + if payload.get("requestStatus").is_some() { + validate_cloudmatch_response(context, status, Ok(payload), false)?; + } + Ok(()) +} + +fn requests_network_test(params: &Value, settings: &Value) -> bool { + params["networkTest"] + .as_bool() + .or_else(|| settings["networkTest"].as_bool()) + .unwrap_or(false) +} + +fn network_test_key_unavailable() -> ServiceError { + ServiceError { + code: "network-test-key-unavailable", + message: "The session did not provision a network test HMAC key; refusing to probe without verified key material".to_owned(), + } +} + +fn acquire_network_test_session( + client: &Client, + base: &Url, + token: &str, + device_id: &str, + params: &Value, + settings: &Value, +) -> Value { + match try_network_test_session(client, base, token, device_id, params, settings) { + Ok(value) => value, + Err(error) => json!({ + "status":"unavailable", + "code":error.code, + "error":error.message, + }), + } +} + +fn network_test_display_profile( + settings: &Value, + params: &Value, +) -> crate::network_test::DisplayProfile { + let (width, height) = parse_resolution(&setting_string(settings, "resolution", "1920x1080")); + crate::network_test::DisplayProfile { + width: u32::try_from(width).unwrap_or(1920), + height: u32::try_from(height).unwrap_or(1080), + fps: u32::try_from(crate::frame_rate::request_frame_rate( + settings, params, width, height, + )) + .unwrap_or(60), + } +} + +fn try_network_test_session( + client: &Client, + base: &Url, + token: &str, + device_id: &str, + params: &Value, + settings: &Value, +) -> Result { + let profile = network_test_display_profile(settings, params); + let url = crate::network_test::nettest_url(base)?; + let body = crate::network_test::allocation_body("GFN-PC", profile); + let mut headers = cloudmatch_headers(token, device_id)?; + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + crate::requests::check()?; + let response = client + .post(url) + .headers(headers) + .timeout(NETWORK_TEST_REQUEST_TIMEOUT) + .json(&body) + .send() + .map_err(|error| network("Network test session failed", error))?; + let status = response.status(); + if response.content_length().unwrap_or(0) > MAXIMUM_NETWORK_TEST_RESPONSE_BYTES { + return Err(ServiceError { + code: "network-test-rejected", + message: "Network test session response exceeded the size limit".to_owned(), + }); + } + let mut response = response; + let mut body_bytes = Vec::new(); + response + .by_ref() + .take(MAXIMUM_NETWORK_TEST_RESPONSE_BYTES + 1) + .read_to_end(&mut body_bytes) + .map_err(|error| network("Network test session failed", error))?; + if body_bytes.len() as u64 > MAXIMUM_NETWORK_TEST_RESPONSE_BYTES { + return Err(ServiceError { + code: "network-test-rejected", + message: "Network test session response exceeded the size limit".to_owned(), + }); + } + let payload = serde_json::from_slice::(&body_bytes); + if !status.is_success() { + let detail = payload + .ok() + .and_then(|payload| { + payload["requestStatus"]["statusDescription"] + .as_str() + .map(ToOwned::to_owned) + }) + .unwrap_or_default(); + return Err(ServiceError { + code: if matches!(status.as_u16(), 401 | 403) { + "network-test-unauthorized" + } else { + "network-test-rejected" + }, + message: format!( + "Network test session returned HTTP {} {detail}", + status.as_u16() + ), + }); + } + let payload = payload.map_err(|_| invalid("Network test session returned invalid JSON"))?; + let session = crate::network_test::parse_allocation(&payload)?; + let Some(key) = session.hmac_key.as_deref() else { + return Err(network_test_key_unavailable()); + }; + crate::requests::check()?; + let outcome = probe_network_test_path(&session, key)?; + crate::requests::check()?; + let Some(measured_datagram_bytes) = outcome.measured_datagram_bytes else { + return Ok(json!({ + "status":"unmeasured", + "probes":outcome.probes, + "error":"No probe datagram was confirmed on the measured path", + })); + }; + Ok(json!({ + "status":"measured", + "sessionId":session.session_id, + "serverId":session.server_id, + "zone":base.host_str().unwrap_or_default(), + "address":session.address, + "port":session.port, + "secure":session.secure, + "measuredDatagramBytes":measured_datagram_bytes, + "probes":outcome.probes, + "thresholds":{ + "bandwidthRecommendedMbps":session.thresholds.bandwidth_recommended_mbps, + "bandwidthLimitMbps":session.thresholds.bandwidth_limit_mbps, + "latencyRecommendedMs":session.thresholds.latency_recommended_ms, + "latencyLimitMs":session.thresholds.latency_limit_ms, + "packetLossRecommendedPct":session.thresholds.packet_loss_recommended_pct, + "packetLossLimitPct":session.thresholds.packet_loss_limit_pct, + }, + })) +} + +fn probe_network_test_path( + session: &crate::network_test::NetworkTestSession, + key: &[u8], +) -> Result { + let peer = std::net::SocketAddr::new(session.address, session.port); + let socket = UdpSocket::bind(if peer.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }) + .map_err(|_| invalid("Network test probe could not bind a UDP socket"))?; + crate::network_test::probe_mtu( + &socket, + peer, + key, + session.session_id.as_bytes(), + crate::network_test::PROBE_FLOOR_BYTES, + crate::network_test::PROBE_CEILING_BYTES, + ) +} + +fn session_token(auth: &AuthSession) -> &str { + auth.tokens + .id_token + .as_deref() + .unwrap_or(&auth.tokens.access_token) +} + +fn parse_resolution(value: &str) -> (i64, i64) { + value + .split_once('x') + .and_then(|(width, height)| Some((width.parse().ok()?, height.parse().ok()?))) + .filter(|(width, height)| *width > 0 && *height > 0) + .unwrap_or((1920, 1080)) +} + +fn setting_string(settings: &Value, key: &str, fallback: &str) -> String { + settings[key] + .as_str() + .filter(|value| !value.is_empty()) + .unwrap_or(fallback) + .to_owned() +} + +fn setting_bool(settings: &Value, key: &str, fallback: bool) -> bool { + settings[key].as_bool().unwrap_or(fallback) +} + +fn resolved_cloud_gsync(settings: &Value) -> bool { + match settings["nativeCloudGsyncMode"].as_str().unwrap_or("auto") { + "disabled" => false, + "forced" => true, + _ => setting_bool(settings, "enableCloudGsync", false), + } +} + +fn codec_wire(value: &str) -> i64 { + match value.to_ascii_lowercase().as_str() { + "h264" => 1, + "h265" | "hevc" => 2, + "av1" => 3, + // Zero delegates the final choice to CloudMatch, matching the + // official native client. Explicit user choices remain pinned. + _ => 0, + } +} + +fn color_quality_wire(value: &str) -> (i64, i64) { + match value { + "10bit_420" => (1, 0), + "8bit_444" => (0, 1), + "10bit_444" => (1, 1), + _ => (0, 0), + } +} + +/// CloudMatch applies `clientTimezoneOffset` as local minus UTC in milliseconds, so +/// UTC-5 is sent as -18,000,000. The opposite sign puts the VM clock at UTC+5. +fn client_timezone_offset_ms(offset: chrono::FixedOffset) -> i64 { + i64::from(offset.local_minus_utc()) * 1000 +} + +fn app_launch_mode(params: &Value) -> i64 { + match params["appLaunchMode"].as_str() { + Some("gamepadFriendly") => 2, + Some("touchFriendly") => 3, + _ => 1, + } +} + +fn platform_name(settings: &Value) -> &'static str { + if setting_bool(settings, "identifyAsSteamDeck", false) { + "SteamOS" + } else if cfg!(target_os = "windows") { + "Windows" + } else if cfg!(target_os = "macos") { + "MacOSX" + } else { + "Linux" + } +} + +fn device_os() -> &'static str { + if cfg!(target_os = "windows") { + "WINDOWS" + } else if cfg!(target_os = "macos") { + "MACOS" + } else { + "LINUX" + } +} + +fn device_type() -> &'static str { + "DESKTOP" +} + +fn device_make() -> &'static str { + if cfg!(target_os = "macos") { + "Apple" + } else { + "UNKNOWN" + } +} + +fn device_model() -> &'static str { + "UNKNOWN" +} + +fn bifrost_user_agent() -> String { + let platform = if cfg!(target_os = "windows") { + "Windows NT 10.0" + } else if cfg!(target_os = "macos") { + "MacOSX" + } else { + "Linux" + }; + format!("GFN-PC/30.0 ({platform}) BifrostClientSDK/4.9 (38495286)") +} + +fn random_uuid() -> String { + let mut bytes = [0_u8; 16]; + rand::rng().fill_bytes(&mut bytes); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + format!( + "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + bytes[0], + bytes[1], + bytes[2], + bytes[3], + bytes[4], + bytes[5], + bytes[6], + bytes[7], + bytes[8], + bytes[9], + bytes[10], + bytes[11], + bytes[12], + bytes[13], + bytes[14], + bytes[15] + ) +} + +fn read_cleanup_record(reader: impl Read) -> Option { + let mut bytes = Vec::new(); + reader + .take((MAX_CLEANUP_RECORD_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() > MAX_CLEANUP_RECORD_BYTES { + return None; + } + serde_json::from_slice(&bytes).ok() +} + +fn allocation_in_progress() -> ServiceError { + ServiceError { + code: "session_update_busy", + message: "A fresh allocation is already in progress. Wait for it to finish before starting another game.".to_owned(), + } +} + +fn invalid(message: impl Into) -> ServiceError { + ServiceError { + code: "invalid_params", + message: message.into(), + } +} + +fn upstream(message: impl Into) -> ServiceError { + ServiceError { + code: "upstream_error", + message: message.into(), + } +} + +fn network(context: &str, error: impl std::fmt::Display) -> ServiceError { + ServiceError { + code: "network_error", + message: format!("{context}: {error}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn session_server( + replies: Vec<(u16, Value)>, + on_request: impl Fn(usize) + Send + 'static, + ) -> (Url, thread::JoinHandle>) { + use std::io::{BufRead, BufReader, Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let worker = thread::spawn(move || { + let mut requests = Vec::new(); + for (index, (status, body)) in replies.into_iter().enumerate() { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + requests.push(line.trim().to_owned()); + let mut length = 0; + loop { + line.clear(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + reader.read_exact(&mut vec![0; length]).unwrap(); + on_request(index); + let body = body.to_string(); + write!(stream, "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap(); + } + requests + }); + (base, worker) + } + + #[test] + fn create_and_claim_use_exact_independent_language_query_pairs() { + for (game, keyboard, expected_game, expected_keyboard) in [ + ("es_419", "de-DE", "es_419", "de-DE"), + ("zh_Hant_TW", "ja-JP", "zh_Hant_TW", "ja-106"), + ("system", "es-ES", "en_US", "es-ES_tradnl"), + ("uk_UA", "uk-UA", "uk_UA", "uk-UA"), + ] { + let settings = + json!({"appLanguage":"fr", "gameLanguage":game, "keyboardLayout":keyboard}); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + ), + (204, json!({})), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"B","status":2}}), + ), + (200, json!({"requestStatus":{"statusCode":1}})), + ], + |_| {}, + ); + let client = Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let mut service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123"}), + &settings, + &conflict_auth(), + "device", + || Ok((client, base.clone())), + ) + .unwrap(); + assert_eq!(created["session"]["keyboardLayout"], expected_keyboard); + service.finish_create("A", false).unwrap(); + service.set_test_control_base(base); + let claimed = service + .claim( + &json!({"sessionId":"B"}), + &settings, + &conflict_auth(), + "device", + ) + .unwrap(); + assert_eq!(claimed["session"]["keyboardLayout"], expected_keyboard); + let received = server.join().unwrap(); + assert_eq!(received.len(), 4); + for index in [0, 3] { + let target = received[index].split_whitespace().nth(1).unwrap(); + let url = Url::parse(&format!("https://fixture.invalid{target}")).unwrap(); + let pairs: std::collections::HashMap<_, _> = url.query_pairs().collect(); + assert_eq!(pairs["languageCode"], expected_game); + assert_eq!(pairs["keyboardLayout"], expected_keyboard); + assert_eq!(pairs.len(), 2); + } + assert!(received[0].starts_with("POST ")); + assert!(received[1].starts_with("DELETE /v2/session/A")); + assert!(received[3].starts_with("PUT /v2/session/B?")); + } + } + + #[test] + fn concurrent_create_is_rejected_before_network_and_cannot_replace_handoff_owner() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }; + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + ), + (204, json!({})), + ], + move |index| { + if index == 0 { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ); + let client = Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let connection_calls = AtomicUsize::new(0); + let (result_tx, result_rx) = mpsc::channel(); + thread::scope(|scope| { + let first = scope.spawn(|| { + service.create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || { + connection_calls.fetch_add(1, Ordering::SeqCst); + Ok((client.clone(), base.clone())) + }, + ) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + scope.spawn(|| { + result_tx + .send(service.create_at( + &json!({"appId":"456"}), + &json!({}), + &conflict_auth(), + "device", + || { + connection_calls.fetch_add(1, Ordering::SeqCst); + Ok((client.clone(), base.clone())) + }, + )) + .unwrap(); + }); + let second = result_rx.recv_timeout(Duration::from_secs(2)); + release_tx.send(()).unwrap(); + assert_eq!( + second + .expect("another create must not wait for the delayed POST") + .unwrap_err() + .code, + "session_update_busy" + ); + assert_eq!(first.join().unwrap().unwrap()["session"]["sessionId"], "A"); + }); + assert_eq!(connection_calls.load(Ordering::SeqCst), 1); + let second = service.create_at( + &json!({"appId":"456"}), + &json!({}), + &conflict_auth(), + "device", + || panic!("an unaccepted allocation must keep its reservation"), + ); + assert_eq!(second.unwrap_err().code, "session_update_busy"); + service.finish_create("B", false).unwrap(); + assert_eq!( + service.fresh.lock().unwrap().as_ref().unwrap().info["sessionId"], + "A" + ); + { + let _cleanup = service.fresh.lock().unwrap(); + assert_eq!( + service + .create_at( + &json!({"appId":"456"}), + &json!({}), + &conflict_auth(), + "device", + || { panic!("allocation admission must not wait for a cleanup lock") } + ) + .unwrap_err() + .code, + "session_update_busy" + ); + } + service.finish_create("A", false).unwrap(); + assert!(service.fresh.lock().unwrap().is_none()); + let received = server.join().unwrap(); + assert_eq!(received.len(), 2); + assert!(received[0].starts_with("POST /v2/session?")); + assert_eq!(received[1], "DELETE /v2/session/A HTTP/1.1"); + } + + #[test] + fn pre_id_failures_release_allocation_admission_for_the_next_attempt() { + let (base, server) = session_server( + vec![ + (503, json!({})), + (200, json!({"requestStatus":{"statusCode":4}})), + (200, json!({"requestStatus":{"statusCode":1},"session":{}})), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + ), + ], + |_| {}, + ); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("cancelled", "session.create").unwrap(); + requests.cancel("cancelled"); + let cancelled = crate::requests::scope(permit.token.clone(), || { + service.create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || panic!("cancelled admission cannot resolve an endpoint"), + ) + }); + assert_eq!(cancelled.unwrap_err().code, "cancelled"); + assert!(service.allocation_admission.try_lock().is_ok()); + let preparing = requests.admit("preparing", "session.create").unwrap(); + let cancelled = crate::requests::scope(preparing.token.clone(), || { + service.create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || { + requests.cancel("preparing"); + Ok((client.clone(), base.clone())) + }, + ) + }); + assert_eq!(cancelled.unwrap_err().code, "cancelled"); + assert!(service.allocation_admission.try_lock().is_ok()); + let unavailable = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let unavailable_base = + Url::parse(&format!("http://{}/", unavailable.local_addr().unwrap())).unwrap(); + let timed_client = Client::builder() + .timeout(Duration::from_millis(20)) + .build() + .unwrap(); + assert_eq!( + service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || { Ok((timed_client, unavailable_base)) } + ) + .unwrap_err() + .code, + "network_error" + ); + assert!(service.allocation_admission.try_lock().is_ok()); + assert!(service.fresh.lock().unwrap().is_none()); + assert!( + service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || { Err(invalid("fixture connection preparation failure")) } + ) + .is_err() + ); + assert!( + service + .create_at(&json!({}), &json!({}), &conflict_auth(), "device", || { + Ok((client.clone(), base.clone())) + }) + .is_err() + ); + for code in ["upstream_error", "session_error", "upstream_error"] { + assert_eq!( + service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || { Ok((client.clone(), base.clone())) } + ) + .unwrap_err() + .code, + code + ); + assert!(service.allocation_admission.try_lock().is_ok()); + assert!(service.fresh.lock().unwrap().is_none()); + } + let result = service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client.clone(), base.clone())), + ) + .unwrap(); + assert_eq!(result["session"]["sessionId"], "A"); + service.finish_create("A", true).unwrap(); + let received = server.join().unwrap(); + assert_eq!(received.len(), 4); + assert!( + received[..4] + .iter() + .all(|request| request.starts_with("POST /v2/session?")) + ); + } + + #[test] + fn cleanup_record_reader_bounds_consumption_before_parsing() { + struct EndlessReader(usize); + impl Read for EndlessReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.0 += buffer.len(); + buffer.fill(b' '); + Ok(buffer.len()) + } + } + let mut endless = EndlessReader(0); + assert!(read_cleanup_record(&mut endless).is_none()); + assert_eq!(endless.0, MAX_CLEANUP_RECORD_BYTES + 1); + let mut boundary = vec![b' '; MAX_CLEANUP_RECORD_BYTES]; + boundary[..2].copy_from_slice(b"{}"); + assert_eq!(read_cleanup_record(boundary.as_slice()), Some(json!({}))); + assert!(read_cleanup_record(b"not JSON".as_slice()).is_none()); + } + + #[test] + fn fresh_allocation_uses_validated_response_control_host() { + for (control, expected) in [ + ( + json!({"ip":"control.partner.example","port":443}), + "https://control.partner.example/", + ), + ( + json!({"ip":"forwarded.partner.example"}), + "https://forwarded.partner.example/", + ), + (json!({"ip":"localhost","port":443}), ""), + (json!({"ip":"control.partner.example","port":8443}), ""), + (json!({"ip":"https://user@control.partner.example"}), ""), + (Value::Null, ""), + ] { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"fresh-seat","status":1,"sessionControlInfo":control + }}), + )], + |_| {}, + ); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let result = service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base.clone())), + ) + .unwrap(); + assert_eq!( + result["session"]["streamingBaseUrl"], + if expected.is_empty() { + base.origin().ascii_serialization() + } else { + expected.trim_end_matches('/').to_owned() + } + ); + assert_eq!( + service + .fresh + .lock() + .unwrap() + .as_ref() + .unwrap() + .base + .as_str(), + if expected.is_empty() { + base.as_str() + } else { + expected + } + ); + service.finish_create("fresh-seat", true).unwrap(); + assert_eq!(server.join().unwrap().len(), 1); + } + } + + #[test] + fn cancelled_allocation_retries_response_control_host_and_persists_guarded_cleanup() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("create", "session.create").unwrap(); + let (base, post_server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"fresh-seat","status":1, + "sessionControlInfo":{"ip":"203.0.113.20","port":443} + }}), + )], + move |_| requests.cancel("create"), + ); + let proxy = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let proxy_base = format!("http://{}", proxy.local_addr().unwrap()); + let proxy_server = thread::spawn(move || { + use std::io::{BufRead, BufReader, Write}; + let mut targets = Vec::new(); + for _ in 0..2 { + let (mut stream, _) = proxy.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + targets.push(line.trim().to_owned()); + loop { + line.clear(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + assert!(!line.to_ascii_lowercase().starts_with("authorization:")); + } + let _ = stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + } + targets + }); + let client = Client::builder() + .proxy(reqwest::Proxy::https(&proxy_base).unwrap()) + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let service = CloudMatchService::with_cleanup_path(client.clone(), path.clone()); + let result = crate::requests::scope(permit.token.clone(), || { + service.create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + }); + assert_eq!(result.unwrap_err().code, "session_cleanup_pending"); + assert_eq!( + service + .fresh + .lock() + .unwrap() + .as_ref() + .unwrap() + .base + .as_str(), + "https://203.0.113.20/" + ); + assert!(service.active()["session"].is_null()); + assert_eq!( + service.discovered.lock().unwrap()["fresh-seat"]["cleanupPending"], + true + ); + assert_eq!( + service.finish_create("fresh-seat", false).unwrap_err().code, + "session_cleanup_pending" + ); + assert_eq!( + service.retained_cleanup.lock().unwrap().as_ref().unwrap()["streamingBaseUrl"], + "https://203.0.113.20" + ); + let persisted: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(persisted["streamingBaseUrl"], "https://203.0.113.20"); + let restarted = CloudMatchService::with_cleanup_path(Client::new(), path); + let pending = restarted.pending_cleanup(&conflict_auth()).unwrap(); + assert_eq!(pending["sessionId"], "fresh-seat"); + assert!(pending["streamingBaseUrl"].is_null()); + assert_eq!(pending["cleanupEndpointUnverified"], true); + assert_eq!( + restarted.admit_create().err().unwrap().code, + "session_cleanup_pending" + ); + assert_eq!(post_server.join().unwrap().len(), 1); + assert_eq!( + proxy_server.join().unwrap(), + [ + "CONNECT 203.0.113.20:443 HTTP/1.1", + "CONNECT 203.0.113.20:443 HTTP/1.1", + ] + ); + } + + #[test] + fn cancelled_fresh_post_is_compensated_without_resume() { + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("create", "session.create").unwrap(); + let mut replies = vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"fresh-seat","status":1}}), + )]; + replies.push((204, json!({}))); + let (base, server) = session_server(replies, move |index| { + if index == 0 { + requests.cancel("create"); + } + }); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let result = crate::requests::scope(permit.token.clone(), || { + service.create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + }); + assert_eq!(result.unwrap_err().code, "cancelled"); + let received = server.join().unwrap(); + assert!(received[0].starts_with("POST /v2/session?")); + assert_eq!(received.len(), 2); + assert_eq!( + received.last().unwrap(), + "DELETE /v2/session/fresh-seat HTTP/1.1" + ); + assert!(service.active()["session"].is_null()); + assert!(service.fresh.lock().unwrap().is_none()); + } + + #[test] + fn unaccepted_allocation_retains_failed_cleanup_and_retries_exact_seat() { + let directory = tempfile::tempdir().unwrap(); + let cleanup_path = directory.path().join("pending-session-cleanup.json"); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"fresh-seat","status":1}}), + ), + (503, json!({})), + (404, json!({})), + ], + |_| {}, + ); + let client = Client::new(); + let service = CloudMatchService::with_cleanup_path(client.clone(), cleanup_path.clone()); + let result = service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client.clone(), base.clone())), + ) + .unwrap(); + assert_eq!(result["session"]["sessionId"], "fresh-seat"); + assert_eq!( + service.finish_create("fresh-seat", false).unwrap_err().code, + "session_cleanup_pending" + ); + assert!(service.active()["session"].is_null()); + assert_eq!( + service.discovered.lock().unwrap()["fresh-seat"]["cleanupPending"], + true + ); + assert!(cleanup_path.exists()); + assert_eq!( + service + .stop( + &json!({"sessionId":"fresh-seat"}), + &json!({}), + &conflict_auth(), + "device" + ) + .unwrap_err() + .code, + "session_cleanup_pending" + ); + assert_eq!( + service + .create_at( + &json!({"appId":"456"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)) + ) + .unwrap_err() + .code, + "session_cleanup_pending" + ); + service.finish_create("other-seat", false).unwrap(); + assert!(service.fresh.lock().unwrap().is_some()); + service.finish_create("fresh-seat", false).unwrap(); + assert!(!cleanup_path.exists()); + assert!(service.fresh.lock().unwrap().is_none()); + assert!(service.discovered.lock().unwrap().is_empty()); + let received = server.join().unwrap(); + assert_eq!(received.len(), 3); + assert_eq!(received[1], received[2]); + } + + #[test] + fn accepted_allocation_is_not_deleted_and_unrelated_terminal_keeps_active_slot() { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + )], + |_| {}, + ); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client.clone(), base.clone())), + ) + .unwrap(); + service.finish_create("A", true).unwrap(); + assert!(service.fresh.lock().unwrap().is_none()); + assert_eq!(server.join().unwrap().len(), 1); + service.clear_active("B"); + assert_eq!(service.active()["session"]["sessionId"], "A"); + let mut finished = session_info( + &json!({"session":{"sessionId":"B","status":7}}), + &base, + "", + "", + "device", + ) + .unwrap(); + assert_eq!(finished["phase"], "finished"); + assert_eq!( + finished["termination"]["source"], + "cloudmatch-session-status" + ); + service + .store_active(&mut finished, &base, "", "", client) + .unwrap(); + assert_eq!(service.active()["session"]["sessionId"], "A"); + service.clear_active("A"); + assert!(service.active()["session"].is_null()); + } + + #[test] + fn stopping_discovered_b_keeps_active_a_and_other_discovered_sessions() { + let (base, server) = session_server(vec![(204, json!({}))], |_| {}); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let mut active = json!({"sessionId":"A","status":3}); + service + .store_active(&mut active, &base, "", "123", client.clone()) + .unwrap(); + service.store_discovered(&[json!({"sessionId":"B"}), json!({"sessionId":"C"})]); + let result = service + .stop_at( + "B", + &client, + base.join("v2/session/B").unwrap(), + HeaderMap::new(), + ) + .unwrap(); + assert_eq!(result["session"]["sessionId"], "A"); + assert_eq!(service.active()["session"]["sessionId"], "A"); + assert!(!service.discovered.lock().unwrap().contains_key("B")); + assert!(service.discovered.lock().unwrap().contains_key("C")); + assert_eq!(server.join().unwrap(), ["DELETE /v2/session/B HTTP/1.1"]); + } + + #[test] + fn deletion_rejection_does_not_forget_the_active_seat() { + let (base, server) = session_server( + vec![(200, json!({"requestStatus":{"statusCode":4}}))], + |_| {}, + ); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let mut active = json!({"sessionId":"A","status":3}); + service + .store_active(&mut active, &base, "", "123", client.clone()) + .unwrap(); + assert_eq!( + service + .stop_at( + "A", + &client, + base.join("v2/session/A").unwrap(), + HeaderMap::new() + ) + .unwrap_err() + .code, + "session_error" + ); + assert_eq!(service.active()["session"]["sessionId"], "A"); + assert_eq!(server.join().unwrap(), ["DELETE /v2/session/A HTTP/1.1"]); + } + + #[test] + fn pending_cleanup_survives_restart_and_is_scoped_to_original_account() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let auth = conflict_auth(); + let record = json!({"sessionId":"cancelled","appId":"123","status":1,"phase":"preparing", + "streamingBaseUrl":DEFAULT_STREAMING_BASE,"cleanupPending":true, + "owner":[auth.provider.idp_id,auth.user.user_id]}); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + let service = CloudMatchService::with_cleanup_path(Client::new(), path.clone()); + let pending = service.pending_cleanup(&auth).unwrap(); + assert_eq!(pending["sessionId"], "cancelled"); + assert!(pending.get("owner").is_none()); + let mut other = auth.clone(); + other.user.user_id = "other".to_owned(); + assert!(service.pending_cleanup(&other).is_none()); + assert_eq!( + service + .create(&json!({"appId":"123"}), &json!({}), &auth, "device") + .unwrap_err() + .code, + "session_cleanup_pending" + ); + service.clear_cleanup("other-seat"); + assert!(path.exists()); + service.clear_cleanup("cancelled"); + assert!(!path.exists()); + assert!(service.pending_cleanup(&auth).is_none()); + } + + #[test] + fn persisted_cleanup_cannot_redirect_token_to_unverified_partner_host() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let auth = conflict_auth(); + let record = json!({"sessionId":"seat","streamingBaseUrl":"https://attacker.example/", + "owner":[auth.provider.idp_id,auth.user.user_id]}); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + let service = CloudMatchService::with_cleanup_path(Client::new(), path); + let pending = service.pending_cleanup(&auth).unwrap(); + assert!(pending["streamingBaseUrl"].is_null()); + assert_eq!(pending["cleanupEndpointUnverified"], true); + assert_eq!( + service.admit_create().err().unwrap().code, + "session_cleanup_pending" + ); + } + + #[test] + fn persisted_cleanup_stop_never_uses_marker_or_synthetic_endpoint() { + let auth = conflict_auth(); + for (id, marker_url) in [ + ("seat", DEFAULT_STREAMING_BASE), + ("seat/other?forged=1", "https://attacker.example/"), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let record = json!({"sessionId":id,"streamingBaseUrl":marker_url, + "owner":[auth.provider.idp_id,auth.user.user_id]}); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + let service = CloudMatchService::with_cleanup_path(Client::new(), path.clone()); + service.store_discovered(&[json!({"sessionId":id, + "serverIp":"seat.nvidiagrid.net","streamingBaseUrl":marker_url})]); + assert_eq!( + service + .stop(&json!({"sessionId":id}), &json!({}), &auth, "device") + .unwrap_err() + .code, + "session_cleanup_pending" + ); + assert!(path.exists()); + assert_eq!( + service.admit_create().err().unwrap().code, + "session_cleanup_pending" + ); + } + } + + #[test] + fn targeted_not_found_is_distinct_from_authentication_and_invalid_payloads() { + for (status, body, code) in [ + (404, json!({}), "session_not_found"), + (401, json!({}), "http_unauthorized"), + (403, json!({}), "authentication_required"), + ( + 200, + json!({"requestStatus":{"statusCode":32}}), + "session_error", + ), + ] { + let (base, server) = session_server(vec![(status, body)], |_| {}); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + assert_eq!( + service + .get_session(&client, &base, "seat", &HeaderMap::new()) + .unwrap_err() + .code, + code + ); + assert_eq!(server.join().unwrap(), ["GET /v2/session/seat HTTP/1.1"]); + } + } + + #[test] + fn same_seat_partial_finalized_color_preserves_components_not_preferences() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let parse = |session| { + session_info(&json!({"session":session}), &base, "", "123", "device").unwrap() + }; + for (color, depth, chroma) in [("8bit_420", 0, 0), ("10bit_420", 1, 0), ("10bit_444", 1, 1)] + { + let body = build_create_body( + "123", + &json!({}), + &json!({"codec":"h265", "colorQuality":color}), + "device", + ); + assert_eq!( + body["sessionRequestData"]["requestedStreamingFeatures"]["bitDepth"], + depth + ); + assert_eq!( + body["sessionRequestData"]["requestedStreamingFeatures"]["chromaFormat"], + chroma + ); + let previous = parse(json!({"sessionId":"seat","status":2,"sdrHdrMode":1, + "finalizedStreamingFeatures":{"codec":2,"bitDepth":depth,"chromaFormat":chroma}})); + let mut partial = parse(json!({"sessionId":"seat","status":2})); + preserve_session_profile(&mut partial, &previous); + assert_eq!(partial["negotiatedStreamProfile"]["colorQuality"], color); + assert_eq!(partial["negotiatedStreamProfile"]["enableHdr"], true); + let mut sdr = partial.clone(); + sdr["negotiatedStreamProfile"]["enableHdr"] = json!(false); + let prepared = crate::streamer::StreamerService::new().prepare_embedded( + &json!({"session":sdr,"runtimeCapabilities":{"protocolVersion":7,"videoBackends":[{ + "backend":"vaapi","platform":"linux","available":true,"codecs":[{ + "codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420","10bit_444"] + }] + }]}}), &json!({"codec":"h265","colorQuality":"8bit_420"}), + ).unwrap(); + assert_eq!(prepared["context"]["settings"]["colorQuality"], color); + assert_eq!( + prepared["context"]["session"]["negotiatedStreamProfile"]["colorQuality"], + color + ); + let mut downgrade = parse(json!({"sessionId":"seat","status":2,"sdrHdrMode":0, + "finalizedStreamingFeatures":{"bitDepth":0}})); + preserve_session_profile(&mut downgrade, &previous); + assert_eq!(downgrade["negotiatedStreamProfile"]["bitDepth"], 8); + assert_eq!(downgrade["negotiatedStreamProfile"]["chromaFormat"], chroma); + assert_eq!(downgrade["negotiatedStreamProfile"]["enableHdr"], false); + let mut chroma_only = parse(json!({"sessionId":"seat","status":2, + "finalizedStreamingFeatures":{"chromaFormat":0}})); + preserve_session_profile(&mut chroma_only, &previous); + assert_eq!(chroma_only["negotiatedStreamProfile"]["chromaFormat"], 0); + assert_eq!( + chroma_only["negotiatedStreamProfile"]["bitDepth"], + if depth == 1 { 10 } else { 8 } + ); + let mut invalid = parse(json!({"sessionId":"seat","status":2, + "finalizedStreamingFeatures":{"bitDepth":99}})); + preserve_session_profile(&mut invalid, &previous); + assert!(invalid["negotiatedStreamProfile"]["colorQuality"].is_null()); + let mut other = parse(json!({"sessionId":"other","status":2})); + preserve_session_profile(&mut other, &previous); + assert!(other["negotiatedStreamProfile"]["colorQuality"].is_null()); + } + } + + fn conflict_auth() -> AuthSession { + serde_json::from_value(json!({ + "provider":{"idpId":"provider", "code":"NVIDIA", "displayName":"NVIDIA", "streamingServiceUrl":DEFAULT_STREAMING_BASE, "priority":0}, + "tokens":{"accessToken":"test-token", "expiresAt":0, "authClientId":"test"}, + "user":{"userId":"test-user", "displayName":"Test", "membershipTier":""} + })).unwrap() + } + + #[test] + fn allocation_uses_cloudmatch_bit_depth_enum_for_both_transports() { + let mut auth = conflict_auth(); + auth.provider.code = "YES".into(); + for mode in ["auto", "on", "off"] { + for (color, bit_depth) in [("8bit_420", 0), ("10bit_420", 1)] { + let requested = json!({"webrtcCompatibilityMode":mode,"codec":"h265", + "colorQuality":color,"resolution":"1920x1200","fps":60}); + let settings = allocation_settings(&requested, &auth); + let body = build_create_body("123", &json!({}), &settings, "device"); + assert_eq!( + body["sessionRequestData"]["requestedStreamingFeatures"]["bitDepth"], bit_depth, + "{mode} {color}" + ); + assert_eq!(settings["colorQuality"], color); + } + } + } + + #[test] + fn webrtc_hevc_allocation_preserves_requested_output() { + let mut auth = conflict_auth(); + auth.provider.code = "YES".into(); + for codec in ["h265", "HEVC"] { + for color in ["8bit_420", "10bit_420"] { + let requested = json!({"webrtcCompatibilityMode":"auto", "codec":codec, + "colorQuality":color,"resolution":"1920x1200","fps":60, + "maxBitrateMbps":200,"enableHdr":false}); + let settings = allocation_settings(&requested, &auth); + assert_eq!(settings["codec"], "h265"); + assert_eq!(settings["colorQuality"], color); + assert_eq!(settings["resolution"], "1920x1200"); + assert_eq!(settings["fps"], 60); + assert_eq!(settings["maxBitrateMbps"], 200); + assert_eq!(requested["codec"], codec); + let body = build_create_body("123", &json!({}), &settings, "device"); + let request = &body["sessionRequestData"]; + assert!(reported_webrtc_session(&body)); + assert_eq!( + request["clientRequestMonitorSettings"][0]["heightInPixels"], + 1200 + ); + assert_eq!( + request["requestedStreamingFeatures"]["bitDepth"], + if color == "10bit_420" { 1 } else { 0 } + ); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 0); + assert_eq!(request["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!(request["sdrHdrMode"], 0); + } + } + } + + #[test] + fn webrtc_hevc_profile_survives_allocation_poll_and_restart() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let ready = json!({"sessionId":"hevc-seat","status":3,"connectionInfo":[ + {"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","port":443,"resourcePath":"/nvst/"}]}); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"hevc-seat","status":1, + "sessionControlInfo":{"ip":"seat.nvidiagrid.net","port":443}}}), + ), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":ready}), + ), + ], + |_| {}, + ); + let mut auth = conflict_auth(); + auth.provider.code = "YES".into(); + let client = Client::new(); + let mut service = CloudMatchService::with_cleanup_path(client.clone(), path.clone()); + service.set_test_control_base(base.clone()); + let settings = json!({"webrtcCompatibilityMode":"auto","codec":"h265", + "colorQuality":"10bit_420","resolution":"1920x1200","fps":60}); + let created = service + .create_at(&json!({"appId":"123"}), &settings, &auth, "device", || { + Ok((client, base)) + }) + .unwrap(); + service.finish_create("hevc-seat", true).unwrap(); + let polled = service + .poll(&json!({"sessionId":"hevc-seat"}), &auth, "device") + .unwrap(); + assert_eq!(server.join().unwrap().len(), 2); + drop(service); + let restored = CloudMatchService::with_cleanup_path(Client::new(), path); + let retained = restored + .retained_webrtc_allocation("hevc-seat", &auth) + .unwrap(); + for session in [&created["session"], &polled["session"], &retained] { + assert_eq!(session["transportMode"], "webrtc"); + let profile = &session["negotiatedStreamProfile"]; + assert_eq!(profile["codec"], "H265"); + assert_eq!(profile["codecSource"], "request"); + assert_eq!(profile["colorQuality"], "10bit_420"); + assert_eq!(profile["bitDepth"], 10); + assert_eq!(profile["resolution"], "1920x1200"); + assert_eq!(profile["fps"], 60); + } + } + + #[test] + fn webrtc_profile_requires_explicit_opt_in() { + let requested = json!({"allianceWebrtcCompatibility":true,"transportMode":"webrtc", + "codec":"av1","colorQuality":"10bit_444","enableHdr":true,"nativeHdrSupported":true, + "resolution":"3840x2160","fps":240,"microphoneMode":"open"}); + let compatible = allocation_settings(&requested, &conflict_auth()); + assert_eq!(compatible["codec"], "h264"); + assert_eq!(compatible["colorQuality"], "8bit_420"); + assert_eq!(compatible["enableHdr"], false); + assert_eq!(compatible["microphoneMode"], "disabled"); + assert_eq!(compatible["resolution"], "1920x1080"); + assert_eq!(compatible["fps"], 60); + assert_eq!(requested["codec"], "av1"); + for (requested_fps, expected) in [(0, 60), (30, 30), (60, 60), (120, 60)] { + let mut settings = requested.clone(); + settings["fps"] = json!(requested_fps); + assert_eq!( + allocation_settings(&settings, &conflict_auth())["fps"], + expected + ); + } + for enabled in [json!(false), json!("true"), Value::Null] { + let mut settings = requested.clone(); + settings["allianceWebrtcCompatibility"] = enabled; + assert_eq!( + allocation_settings(&settings, &conflict_auth())["codec"], + "av1" + ); + assert_eq!( + allocation_settings(&settings, &conflict_auth())["allianceWebrtcCompatibility"], + false + ); + } + } + + #[test] + fn create_request_carries_dlss_overrides_like_the_official_client() { + let default_body = build_create_body("123", &json!({}), &json!({}), "device"); + assert_eq!( + default_body["sessionRequestData"]["requestedStreamingFeatures"]["dlssOverrides"], + json!({"srMode":0,"srModel":0,"srScale":100,"rrModel":0,"fgMode":0,"fgModel":0}) + ); + let settings = json!({"dlssOverrides":{"srMode":2,"srModel":1,"srScale":58,"rrModel":1,"fgMode":1,"fgModel":1,"fgMultiplier":4}}); + let body = build_create_body("123", &json!({}), &settings, "device"); + assert_eq!( + body["sessionRequestData"]["requestedStreamingFeatures"]["dlssOverrides"], + json!({"srMode":2,"srModel":1,"srScale":58,"rrModel":1,"fgMode":1,"fgModel":1,"fgMultiplier":4}) + ); + } + + #[test] + fn alliance_webrtc_request_changes_allocation_without_changing_native_headers() { + let settings = allocation_settings( + &json!({"allianceWebrtcCompatibility":true, + "codec":"h265","enableHdr":true,"nativeHdrSupported":true}), + &conflict_auth(), + ); + let body = build_create_body("123", &json!({}), &settings, "device"); + let request = &body["sessionRequestData"]; + assert_eq!(request["streamerVersion"], 1); + assert_eq!(request["sdkVersion"], "1.0"); + assert_eq!(request["enhancedStreamMode"], 1); + assert_eq!(request["secureRTSPSupported"], false); + assert_eq!(request["sdrHdrMode"], 0); + assert_eq!(request["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!(request["requestedStreamingFeatures"]["bitDepth"], 0); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 0); + assert_eq!(request["remoteControllersBitmap"], 1); + assert!(reported_webrtc_session(&body)); + let headers = cloudmatch_headers("test", "device").unwrap(); + assert_eq!(headers["nv-client-streamer"], "NVIDIA-CLASSIC"); + assert_eq!(headers["nv-client-type"], "NATIVE"); + assert!(request.get("requestedAudioFormat").is_none()); + assert!(request.get("transport").is_none()); + } + + #[test] + fn alliance_webrtc_endpoints_preserve_authoritative_address_port_and_path() { + for (connection, expected) in [ + ( + json!({"usage":14,"appLevelProtocol":4,"ip":"203.0.113.8","port":8443,"resourcePath":"wss://signaling.example:9443/nvst/custom/"}), + "wss://signaling.example:9443/nvst/custom/", + ), + ( + json!({"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","resourcePath":"/nvst/"}), + "wss://seat.partner.example/nvst/", + ), + ( + json!({"usage":14,"resourcePath":"wss://seat.partner.example:9443/custom/"}), + "wss://seat.partner.example:9443/custom/", + ), + ( + json!({"usage":14,"appLevelProtocol":4,"ip":"2001:db8::1","port":8443}), + "wss://[2001:db8::1]:8443/nvst/", + ), + ] { + assert_eq!( + webrtc_signaling_endpoint(&connection).unwrap().as_str(), + expected + ); + } + for connection in [ + json!({"usage":2,"ip":"seat.example","port":322}), + json!({"usage":15,"ip":"seat.example","port":322}), + json!({"usage":14,"resourcePath":"rtsps://user:password@seat.example:322"}), + json!({"usage":14,"resourcePath":"wss://user:password@seat.example/nvst/"}), + json!({"usage":14,"ip":"seat.example","port":65536}), + json!({"usage":14,"ip":"0.0.0.0","port":443}), + ] { + assert!( + webrtc_signaling_endpoint(&connection).is_none(), + "{connection}" + ); + } + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let mut native = session_info(&json!({"session":{"sessionId":"A","status":3,"connectionInfo":[ + {"usage":14,"appLevelProtocol":4,"ip":"webrtc.example","port":8443,"resourcePath":"/nvst/"}, + {"usage":16,"ip":"native.example","port":322}, + {"usage":15,"ip":"203.0.113.15","port":49015}, + {"usage":17,"ip":"203.0.113.17","port":49017}]}}), &base, "", "123", "device").unwrap(); + assert_eq!(native["serverIp"], "native.example"); + assert_eq!(native["signalingUrl"], "wss://native.example:322"); + assert_eq!(native["mediaConnectionInfo"]["port"], 49017); + native["transportMode"] = json!("webrtc"); + select_webrtc_endpoint(&mut native); + assert_eq!(native["serverIp"], "webrtc.example"); + assert_eq!(native["signalingUrl"], "wss://webrtc.example:8443/nvst/"); + assert_eq!(native["mediaConnectionInfo"]["port"], 49017); + native["connectionInfo"] + .as_array_mut() + .unwrap() + .retain(|connection| connection["usage"] != 17); + select_webrtc_endpoint(&mut native); + assert_eq!(native["mediaConnectionInfo"]["port"], 8443); + native["connectionInfo"][0]["appLevelProtocol"] = json!(6); + select_webrtc_endpoint(&mut native); + assert!(native["mediaConnectionInfo"].is_null()); + } + + #[test] + fn owned_webrtc_sessions_accept_legacy_rtsp_connection_descriptors() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + for connection in [ + json!({"usage":14,"appLevelProtocol":6,"resourcePath":"rtsps://seat.nvidiagrid.net:48322"}), + json!({"usage":14,"appLevelProtocol":1,"ip":"seat.nvidiagrid.net","port":48010}), + json!({"usage":16,"ip":"seat.nvidiagrid.net","port":322}), + ] { + let mut session = session_info( + &json!({"session":{"sessionId":"owned","status":3, + "connectionInfo":[connection]}}), + &base, + "", + "123", + "device", + ) + .unwrap(); + session["transportMode"] = json!("webrtc"); + select_webrtc_endpoint(&mut session); + assert_eq!(session["signalingUrl"], "wss://seat.nvidiagrid.net/nvst/"); + assert!(has_webrtc_endpoint(&session)); + assert!(session["mediaConnectionInfo"].is_null()); + session["signalingUrl"] = json!("wss://forged.invalid/nvst/"); + assert!(!has_webrtc_endpoint(&session)); + } + } + + #[test] + fn explicit_webrtc_endpoint_wins_over_legacy_descriptors() { + let mut session = json!({"transportMode":"webrtc", "connectionInfo":[ + {"usage":16,"ip":"legacy.nvidiagrid.net","port":48322}, + {"usage":14,"appLevelProtocol":6,"resourcePath":"rtsps://legacy.nvidiagrid.net:322"}, + {"usage":14,"appLevelProtocol":6,"resourcePath":"wss://browser.nvidiagrid.net:9443/custom/"} + ]}); + select_webrtc_endpoint(&mut session); + assert_eq!( + session["signalingUrl"], + "wss://browser.nvidiagrid.net:9443/custom/" + ); + assert!(has_webrtc_endpoint(&session)); + session["connectionInfo"].as_array_mut().unwrap().pop(); + session["connectionInfo"][1]["resourcePath"] = + json!("rtsps://browser-alias.nvidiagrid.net:322"); + select_webrtc_endpoint(&mut session); + assert_eq!( + session["signalingUrl"], + "wss://browser-alias.nvidiagrid.net/nvst/" + ); + } + + #[test] + fn automatic_alliance_transport_and_manual_overrides_select_the_allocation() { + for (provider, mode, webrtc) in [ + ("ALLIANCE", "auto", true), + ("NVIDIA", "auto", false), + ("nvidia", "auto", false), + ("", "auto", false), + ("ALLIANCE", "off", false), + ("NVIDIA", "on", true), + ("ALLIANCE", "on", true), + ("NVIDIA", "off", false), + ] { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"A","status":1, + "sessionControlInfo":{"ip":"seat.nvidiagrid.net","port":443} + }}), + )], + |_| {}, + ); + let client = Client::new(); + let mut service = CloudMatchService::new(client.clone()); + service.set_test_control_base(base.clone()); + let mut auth = conflict_auth(); + auth.provider.code = provider.into(); + let settings = json!({"webrtcCompatibilityMode":mode, + "codec":"av1","resolution":"2560x1440","fps":120}); + let created = service + .create_at(&json!({"appId":"123"}), &settings, &auth, "device", || { + Ok((client, base)) + }) + .unwrap(); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST ")); + assert_eq!( + created["session"]["transportMode"], + if webrtc { "webrtc" } else { "nvst" }, + "{provider} {mode}" + ); + if webrtc { + assert_eq!( + created["session"]["negotiatedStreamProfile"]["codec"], + "H264" + ); + assert_eq!( + created["session"]["negotiatedStreamProfile"]["resolution"], + "1920x1080" + ); + assert_eq!(created["session"]["negotiatedStreamProfile"]["fps"], 60); + } + assert_eq!(settings["codec"], "av1"); + } + } + + #[test] + fn alliance_webrtc_allocation_and_poll_keep_owned_transport_and_requested_profile() { + for provider in ["NVIDIA", "ALLIANCE"] { + let ready = json!({"sessionId":"A","status":3,"connectionInfo":[ + {"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","port":8443,"resourcePath":"/nvst/"}, + {"usage":2,"ip":"203.0.113.9","port":49000}]}); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1, + "sessionControlInfo":{"ip":"seat.nvidiagrid.net","port":443}}}), + ), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":ready}), + ), + ], + |_| {}, + ); + let client = Client::new(); + let mut service = CloudMatchService::new(client.clone()); + service.set_test_control_base(base.clone()); + let mut auth = conflict_auth(); + auth.provider.code = provider.into(); + let created = service.create_at(&json!({"appId":"123"}), + &json!({"allianceWebrtcCompatibility":true,"codec":"av1","resolution":"2560x1440","fps":120}), + &auth, "device", || Ok((client, base))).unwrap(); + assert_eq!(created["session"]["transportMode"], "webrtc"); + assert_eq!( + created["session"]["negotiatedStreamProfile"]["codec"], + "H264" + ); + assert_eq!( + created["session"]["negotiatedStreamProfile"]["codecSource"], + "request" + ); + service.finish_create("A", true).unwrap(); + let polled = service + .poll( + &json!({"sessionId":"A","transportMode":"nvst"}), + &auth, + "device", + ) + .unwrap(); + assert_eq!(polled["session"]["transportMode"], "webrtc"); + assert_eq!( + polled["session"]["signalingUrl"], + "wss://seat.partner.example:8443/nvst/" + ); + assert_eq!(polled["session"]["mediaConnectionInfo"]["port"], 49000); + assert_eq!( + polled["session"]["negotiatedStreamProfile"]["resolution"], + "1920x1080" + ); + assert_eq!(polled["session"]["negotiatedStreamProfile"]["fps"], 60); + assert_eq!( + polled["session"]["negotiatedStreamProfile"]["codec"], + "H264" + ); + assert!(has_webrtc_endpoint(&polled["session"])); + assert_eq!(server.join().unwrap().len(), 2); + } + } + + #[test] + fn automatic_and_explicit_modes_never_convert_an_existing_session() { + for mode in ["auto", "on", "off"] { + for webrtc in [true, false] { + let session = json!({"sessionRequestData":{ + "secureRTSPSupported":!webrtc, + "metaData":[{"key":"GSStreamerType","value":if webrtc {"WebRTC"} else {"native"}}] + }}); + let settings = + json!({"webrtcCompatibilityMode":mode,"codec":"av1","enableHdr":true}); + let resumed = build_resume_body("123", &session, &settings, "device"); + assert_eq!(reported_webrtc_session(&resumed), webrtc, "{mode}"); + assert_eq!( + resumed["sessionRequestData"]["secureRTSPSupported"], + !webrtc + ); + } + } + } + + #[test] + fn alliance_webrtc_resume_uses_existing_allocation_not_current_preference() { + let native = json!({"sessionRequestData":{"secureRTSPSupported":true}}); + let native_resume = build_resume_body( + "123", + &native, + &json!({"allianceWebrtcCompatibility":true,"transportMode":"webrtc"}), + "device", + ); + assert_eq!( + native_resume["sessionRequestData"]["secureRTSPSupported"], + true + ); + assert!(!reported_webrtc_session(&native_resume)); + let webrtc = json!({"sessionRequestData":{"secureRTSPSupported":false, + "metaData":[{"key":"GSStreamerType","value":"WebRTC"}]}}); + let resumed = build_resume_body( + "123", + &webrtc, + &json!({"allianceWebrtcCompatibility":false,"codec":"av1","enableHdr":true}), + "device", + ); + assert!(reported_webrtc_session(&resumed)); + assert_eq!(resumed["sessionRequestData"]["sdkVersion"], "1.0"); + assert_eq!(resumed["sessionRequestData"]["enhancedStreamMode"], 1); + for key in ["clientRequestMonitorSettings", "requestedStreamingFeatures"] { + assert!(resumed["sessionRequestData"].get(key).is_none()); + } + } + + #[test] + fn alliance_webrtc_profile_preservation_never_overwrites_server_acceptance() { + let previous = json!({"sessionId":"A","transportMode":"webrtc","negotiatedStreamProfile":{ + "codec":"H264","codecSource":"request","resolution":"1920x1080","fps":60, + "enableHdr":false,"enableHdrSource":"request"}}); + let mut next = json!({"sessionId":"A","transportMode":"nvst","negotiatedStreamProfile":{ + "codec":"AV1","codecSource":"server","resolution":"1280x720","fps":30, + "enableHdr":true,"enableHdrSource":"server"}}); + preserve_session_profile(&mut next, &previous); + assert_eq!(next["transportMode"], "webrtc"); + assert_eq!(next["negotiatedStreamProfile"]["codec"], "AV1"); + assert_eq!(next["negotiatedStreamProfile"]["resolution"], "1280x720"); + assert_eq!(next["negotiatedStreamProfile"]["fps"], 30); + assert_eq!(next["negotiatedStreamProfile"]["enableHdr"], true); + next["sessionId"] = json!("B"); + next["transportMode"] = json!("nvst"); + preserve_session_profile(&mut next, &previous); + assert_eq!(next["transportMode"], "nvst"); + } + + #[test] + fn alliance_webrtc_restart_retains_owned_transport_when_toggle_and_metadata_are_absent() { + for provider in ["NVIDIA", "ALLIANCE"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pending-session-cleanup.json"); + let mut auth = conflict_auth(); + auth.provider.code = provider.into(); + let original = CloudMatchService::with_cleanup_path(Client::new(), path.clone()); + original.remember_webrtc_allocation(&json!({"sessionId":"A","status":3,"transportMode":"webrtc", + "negotiatedStreamProfile":{"codec":"H264","codecSource":"request","colorQuality":"8bit_420", + "bitDepth":8,"bitDepthSource":"request","chromaFormat":0,"chromaFormatSource":"request", + "enableHdr":false,"enableHdrSource":"request","resolution":"1280x720","fps":30}}), &auth).unwrap(); + drop(original); + let ready = json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":3, + "connectionInfo":[{"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","port":443,"resourcePath":"/nvst/"}]}}); + let (base, server) = session_server( + vec![ + (200, ready.clone()), + (200, json!({"requestStatus":{"statusCode":1}})), + (200, ready), + ], + |_| {}, + ); + let mut restored = CloudMatchService::with_cleanup_path(Client::new(), path); + restored.set_test_control_base(base); + let claimed = restored + .claim( + &json!({"sessionId":"A"}), + &json!({"allianceWebrtcCompatibility":false,"codec":"av1"}), + &auth, + "device", + ) + .unwrap(); + assert_eq!(claimed["session"]["transportMode"], "webrtc"); + assert_eq!( + claimed["session"]["negotiatedStreamProfile"]["codec"], + "H264" + ); + let polled = restored + .poll(&json!({"sessionId":"A"}), &auth, "device") + .unwrap(); + assert_eq!(polled["session"]["resumePending"], false); + assert_eq!(polled["session"]["transportMode"], "webrtc"); + assert_eq!( + polled["session"]["negotiatedStreamProfile"]["resolution"], + "1280x720" + ); + let mut other = auth.clone(); + other.user.user_id = "someone-else".into(); + assert!(restored.retained_webrtc_allocation("A", &other).is_none()); + other = auth.clone(); + other.provider.idp_id = "other-partner".into(); + assert!(restored.retained_webrtc_allocation("A", &other).is_none()); + assert!(restored.retained_webrtc_allocation("B", &auth).is_none()); + assert_eq!(server.join().unwrap().len(), 3); + } + } + + #[test] + fn claiming_webrtc_uses_the_authenticated_provider_and_explicit_override() { + for (provider, mode, allowed) in [ + ("ALLIANCE", "auto", true), + ("NVIDIA", "auto", false), + ("ALLIANCE", "off", false), + ("NVIDIA", "on", true), + ] { + let mut replies = vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"existing","status":3, + "sessionRequestData":{"appId":"123","secureRTSPSupported":false, + "metaData":[{"key":"GSStreamerType","value":"WebRTC"}]}, + "connectionInfo":[{"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example", + "port":443,"resourcePath":"/nvst/"}] + }}), + )]; + if allowed { + replies.push((200, json!({"requestStatus":{"statusCode":1}}))); + } + let (base, server) = session_server(replies, |_| {}); + let mut service = CloudMatchService::new(Client::new()); + service.set_test_control_base(base); + let mut auth = conflict_auth(); + auth.provider.code = provider.into(); + let result = service.claim( + &json!({"sessionId":"existing"}), + &json!({"webrtcCompatibilityMode":mode}), + &auth, + "device", + ); + if allowed { + let claimed = result.unwrap(); + assert_eq!(claimed["session"]["transportMode"], "webrtc"); + assert_eq!(claimed["session"]["resumePending"], true); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[1].starts_with("PUT ")); + } else { + assert_eq!(result.unwrap_err().code, "session_transport_not_enabled"); + assert!(service.active()["session"].is_null()); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + } + } + } + + #[test] + fn alliance_webrtc_foreign_session_without_opt_in_is_retained_without_resume() { + for provider in ["NVIDIA", "ALLIANCE"] { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"foreign","status":3,"sessionRequestData":{"secureRTSPSupported":false, + "metaData":[{"key":"GSStreamerType","value":"WebRTC"}]}}}), + )], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::new()); + service.set_test_control_base(base); + let mut auth = conflict_auth(); + auth.provider.code = provider.into(); + let failure = service + .claim( + &json!({"sessionId":"foreign","transportMode":"webrtc"}), + &json!({"allianceWebrtcCompatibility":false}), + &auth, + "device", + ) + .unwrap_err(); + assert_eq!(failure.code, "session_transport_not_enabled"); + assert!(service.active()["session"].is_null()); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET ")); + } + } + + #[test] + fn alliance_webrtc_saved_seat_never_converts_a_reported_native_allocation() { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"A","status":3,"sessionRequestData":{"secureRTSPSupported":true,"streamerVersion":"14"}}}), + )], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::new()); + service.set_test_control_base(base); + let mut auth = conflict_auth(); + auth.provider.code = "ALLIANCE".into(); + service + .remember_webrtc_allocation( + &json!({"sessionId":"A","status":3,"transportMode":"webrtc"}), + &auth, + ) + .unwrap(); + let failure = service + .claim( + &json!({"sessionId":"A"}), + &json!({"allianceWebrtcCompatibility":true}), + &auth, + "device", + ) + .unwrap_err(); + assert_eq!(failure.code, "session_transport_mismatch"); + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET ")); + } + + fn conflict_payload() -> Value { + json!({ + "requestStatus":{"statusCode":11,"statusDescription":"SESSION_LIMIT_PER_DEVICE_EXCEEDED_STATUS 4AF1201E"}, + "otherUserSessions":[{ + "sessionId":"existing-seat", "status":5, + "sessionRequestData":{"appId":456}, + "sessionControlInfo":{"ip":"seat.nvidiagrid.net"} + }] + }) + } + + #[test] + fn create_conflicts_preserve_resumable_details_for_one_discovery() { + for status in [200, 400, 403, 409, 500] { + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let error = service + .capture_session_conflict( + reqwest::StatusCode::from_u16(status).unwrap(), + &conflict_payload(), + &base, + &auth, + ) + .unwrap(); + assert_eq!(error.code, "session_conflict"); + assert!(!error.message.contains("4AF1201E")); + assert!(service.active()["session"].is_null()); + let response = service + .remote_sessions(&json!({}), &json!({}), &auth, "device") + .unwrap(); + assert_eq!(response["sessions"][0]["sessionId"], "existing-seat"); + assert_eq!(response["sessions"][0]["appId"], 456); + assert_eq!(response["sessions"][0]["status"], 5); + assert_eq!(response["sessions"][0]["serverIp"], "seat.nvidiagrid.net"); + assert_eq!( + response["sessions"][0]["streamingBaseUrl"], + base.origin().ascii_serialization() + ); + assert!( + service + .discovered + .lock() + .unwrap() + .contains_key("existing-seat") + ); + assert!(service.take_conflict_sessions(&auth).is_none()); + } + } + + #[test] + fn conflict_handoff_claims_the_existing_host_instead_of_the_create_region() { + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let create_region = trusted_cloudmatch_base("https://create.nvidiagrid.net").unwrap(); + for host in ["other-region-seat.nvidiagrid.net", "80.84.160.10"] { + let mut payload = conflict_payload(); + payload["otherUserSessions"][0]["sessionControlInfo"]["ip"] = json!(host); + service + .capture_session_conflict( + reqwest::StatusCode::FORBIDDEN, + &payload, + &create_region, + &auth, + ) + .unwrap(); + service + .remote_sessions(&json!({}), &json!({}), &auth, "device") + .unwrap(); + let discovered = service.discovered.lock().unwrap(); + let session = discovered.get("existing-seat"); + assert_eq!( + claim_lookup_base(session, &create_region), + trusted_learned_server_base(host).unwrap() + ); + } + for session in [ + None, + Some(json!({})), + Some(json!({"serverIp":"localhost"})), + Some(json!({"serverIp":"https://127.0.0.1"})), + ] { + assert_eq!( + claim_lookup_base(session.as_ref(), &create_region), + create_region + ); + } + assert_eq!( + claim_lookup_base( + Some(&json!({"serverIp":"https://partner.example.com"})), + &create_region + ), + trusted_cloudmatch_base("https://partner.example.com").unwrap() + ); + } + + #[test] + fn conflict_handoff_expires_and_is_scoped_to_the_account() { + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + for different_account in [false, true] { + service + .capture_session_conflict( + reqwest::StatusCode::BAD_REQUEST, + &conflict_payload(), + &base, + &auth, + ) + .unwrap(); + let mut next_auth = auth.clone(); + if different_account { + next_auth.user.user_id = "other-user".to_owned(); + } else { + service.conflict.lock().unwrap().as_mut().unwrap().received = + Instant::now() - Duration::from_secs(31); + } + assert!(service.take_conflict_sessions(&next_auth).is_none()); + assert!(service.take_conflict_sessions(&auth).is_none()); + } + } + + #[test] + fn conflict_detection_supports_vendor_codes_and_preserves_unauthorized_responses() { + for payload in [ + json!({"requestStatus":{"statusCode":"11"}}), + json!({"requestStatus":{"statusDescription":"SESSION_LIMIT_PER_DEVICE_EXCEEDED_STATUS"}}), + json!({"requestStatus":{"unifiedErrorCode":"4AF1201E"}}), + json!({"session":{"errorCode":0x4AF1201E_i64}}), + ] { + assert!(is_session_conflict(&payload)); + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let status = reqwest::StatusCode::UNAUTHORIZED; + assert!( + service + .capture_session_conflict(status, &payload, &base, &auth) + .is_none() + ); + let error = validate_cloudmatch_response("create", status, Ok(payload.clone()), false) + .unwrap_err(); + assert_eq!(error.code, "http_unauthorized"); + } + assert!(!is_session_conflict( + &json!({"requestStatus":{"statusCode":4,"statusDescription":"INTERNAL_ERROR_STATUS"}}) + )); + } + + #[test] + fn forbidden_session_limit_is_a_conflict_but_unrecognized_forbidden_is_authentication() { + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let status = reqwest::StatusCode::FORBIDDEN; + let payload = json!({"requestStatus":{"statusDescription":"SESSION_LIMIT_PER_DEVICE_EXCEEDED_STATUS 4AF1201E"}}); + assert_eq!( + service + .capture_session_conflict(status, &payload, &base, &auth) + .unwrap() + .code, + "session_conflict" + ); + for payload in [ + json!({"requestStatus":{"statusDescription":"Forbidden"}}), + json!({}), + json!("SESSION_LIMIT_PER_DEVICE_EXCEEDED_STATUS 4AF1201E"), + ] { + assert!( + service + .capture_session_conflict(status, &payload, &base, &auth) + .is_none() + ); + assert_eq!( + validate_cloudmatch_response("create", status, Ok(payload), false) + .unwrap_err() + .code, + "authentication_required" + ); + } + assert_eq!( + read_cloudmatch_response("create", cloudmatch_response(403, "Forbidden"), false) + .unwrap_err() + .code, + "authentication_required" + ); + } + + #[test] + fn conflict_handoff_rejects_unusable_seats_and_accepts_signaling_resource_paths() { + let service = CloudMatchService::new(Client::new()); + let auth = conflict_auth(); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + for patch in [ + json!({"sessionId":""}), + json!({"status":7}), + json!({"sessionRequestData":{"appId":0}}), + json!({"sessionControlInfo":{"ip":"localhost"}}), + ] { + let mut payload = conflict_payload(); + for (key, value) in patch.as_object().unwrap() { + payload["otherUserSessions"][0][key] = value.clone(); + } + assert!( + service + .capture_session_conflict( + reqwest::StatusCode::BAD_REQUEST, + &payload, + &base, + &auth + ) + .is_some() + ); + assert!(service.take_conflict_sessions(&auth).is_none()); + } + let mut payload = conflict_payload(); + payload["otherUserSessions"][0]["sessionControlInfo"] = Value::Null; + payload["otherUserSessions"][0]["connectionInfo"] = + json!([{"usage":14,"resourcePath":"wss://signal.nvidiagrid.net/nvst/"}]); + service + .capture_session_conflict(reqwest::StatusCode::BAD_REQUEST, &payload, &base, &auth) + .unwrap(); + assert_eq!( + service.take_conflict_sessions(&auth).unwrap()[0]["serverIp"], + "signal.nvidiagrid.net" + ); + } + + #[test] + fn discovery_continues_after_empty_regions_and_deduplicates_sessions() { + let bases = [ + "https://first.nvidiagrid.net", + "https://second.nvidiagrid.net", + "https://third.nvidiagrid.net", + ] + .map(|url| trusted_cloudmatch_base(url).unwrap()); + let calls = std::sync::atomic::AtomicUsize::new(0); + let sessions = discover_sessions( + &bases, + Instant::now() + DISCOVERY_TIMEOUT, + false, + |base, timeout| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + assert!(timeout <= DISCOVERY_REQUEST_TIMEOUT); + if base == &bases[0] { + Ok(vec![]) + } else { + Ok(vec![ + json!({"sessionId":"seat", "streamingBaseUrl":base.as_str()}), + ]) + } + }, + ) + .unwrap(); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 3); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0]["streamingBaseUrl"], bases[1].as_str()); + } + + #[test] + fn discovery_reports_incomplete_absence_but_keeps_found_sessions() { + let bases = [ + trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(), + trusted_cloudmatch_base("https://region.nvidiagrid.net").unwrap(), + ]; + for found in [false, true] { + let result = discover_sessions( + &bases, + Instant::now() + DISCOVERY_TIMEOUT, + false, + |base, _| { + if base == &bases[0] { + Err(upstream("failed region")) + } else { + Ok(if found { + vec![json!({"sessionId":"seat"})] + } else { + vec![] + }) + } + }, + ); + if found { + assert_eq!(result.unwrap().len(), 1); + } else { + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + } + } + assert!( + discover_sessions( + &bases, + Instant::now() + DISCOVERY_TIMEOUT, + false, + |_, _| Ok(vec![]) + ) + .unwrap() + .is_empty() + ); + assert_eq!( + discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, true, |_, _| Ok( + vec![] + )) + .unwrap_err() + .code, + "session_discovery_failed" + ); + assert_eq!( + discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { + Err(upstream("failed")) + }) + .unwrap_err() + .code, + "session_discovery_failed" + ); + } + + #[test] + fn discovery_respects_deadline_and_concurrency_bound() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + assert_eq!( + discover_sessions( + std::slice::from_ref(&base), + Instant::now(), + false, + |_, _| panic!("expired search must not send requests") + ) + .unwrap_err() + .code, + "session_discovery_failed" + ); + let active = std::sync::atomic::AtomicUsize::new(0); + let peak = std::sync::atomic::AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(DISCOVERY_CONCURRENCY); + discover_sessions( + &vec![base; 8], + Instant::now() + DISCOVERY_TIMEOUT, + false, + |_, _| { + let current = active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + peak.fetch_max(current, std::sync::atomic::Ordering::SeqCst); + barrier.wait(); + active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + Ok(vec![]) + }, + ) + .unwrap(); + assert_eq!( + peak.load(std::sync::atomic::Ordering::SeqCst), + DISCOVERY_CONCURRENCY + ); + } + + #[test] + fn discovery_preserves_authentication_failures_and_cancellation() { + let bases = [trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap()]; + let error = discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { + Err(ServiceError { + code: "authentication_required", + message: "Expired credentials".to_owned(), + }) + }) + .unwrap_err(); + assert_eq!(error.code, "authentication_required"); + + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("discovery", "session.remote.list").unwrap(); + requests.cancel("discovery"); + let error = crate::requests::scope(permit.token.clone(), || { + discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { + panic!("cancelled discovery must not send requests") + }) + }) + .unwrap_err(); + assert_eq!(error.code, "cancelled"); + } + + fn cloudmatch_response(status: u16, body: &str) -> Response { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let body = body.to_owned(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&stream); + loop { + let mut line = String::new(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + } + write!(stream, + "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len()).unwrap(); + }); + let response = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap() + .get(format!("http://{address}")) + .send() + .unwrap(); + server.join().unwrap(); + response + } + + #[test] + fn resume_claims_paused_and_live_seats_but_only_polls_transitions() { + for status in [2, 3, 4, 5] { + assert!(session_requires_resume(status).unwrap(), "status {status}"); + } + for status in [1, 6] { + assert!(!session_requires_resume(status).unwrap(), "status {status}"); + } + for status in [0, 7, 8, -1] { + assert!(session_requires_resume(status).is_err(), "status {status}"); + } + assert_eq!(session_phase(4), "paused"); + assert_eq!(session_phase(5), "paused"); + assert_eq!(session_phase(6), "resuming"); + } + + #[test] + fn resume_discovery_keeps_paused_and_resuming_seats() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + for status in 0..=8 { + let session = json!({"sessionId":"seat", "status":status}); + let info = remote_session_info(&session, &base); + assert_eq!(info.is_some(), (1..=6).contains(&status)); + if let Some(info) = info { + assert_eq!(info["status"], status); + assert_eq!(info["phase"], session_phase(status)); + } + } + } + + #[test] + fn resume_not_paused_response_continues_polling_for_http_and_api_rejections() { + for status in [200, 400, 409, 500] { + for request_status in [ + json!({"statusCode":34}), + json!({"statusCode":"34"}), + json!({"statusCode":0,"statusDescription":"SESSION_NOT_PAUSED"}), + ] { + let body = json!({"requestStatus":request_status}).to_string(); + let result = read_cloudmatch_response( + "Session claim failed", + cloudmatch_response(status, &body), + true, + ); + assert!(result.is_ok(), "HTTP {status}: {body}"); + let result = read_cloudmatch_response( + "Session polling failed", + cloudmatch_response(status, &body), + false, + ); + assert!( + result.is_err(), + "poll must not accept HTTP {status}: {body}" + ); + } + } + } + + #[test] + fn resume_response_preserves_other_failures_and_success() { + for status in [401, 403] { + let error = read_cloudmatch_response( + "Session claim failed", + cloudmatch_response(status, r#"{"requestStatus":{"statusCode":34}}"#), + true, + ) + .unwrap_err(); + assert_eq!( + error.code, + if status == 401 { + "http_unauthorized" + } else { + "authentication_required" + } + ); + } + for (status, body, code) in [ + ( + 200, + r#"{"requestStatus":{"statusCode":32,"statusDescription":"SESSION_EXPIRED"}}"#, + "session_error", + ), + ( + 409, + r#"{"requestStatus":{"statusCode":32,"statusDescription":"SESSION_EXPIRED"}}"#, + "upstream_error", + ), + (502, "not JSON", "upstream_error"), + ] { + let error = read_cloudmatch_response( + "Session claim failed", + cloudmatch_response(status, body), + true, + ) + .unwrap_err(); + assert_eq!(error.code, code); + } + let response = read_cloudmatch_response( + "Session claim failed", + cloudmatch_response(200, r#"{"requestStatus":{"statusCode":1}}"#), + true, + ) + .unwrap(); + assert_eq!(response["requestStatus"]["statusCode"], 1); + } + + #[test] + fn resume_poll_preserves_paused_progress_and_stops_on_terminal_states() { + for status in [4, 5] { + let mut info = json!({"status":status, "phase":session_phase(status), + "rtspsEndpoints":["rtsps://example.invalid:322"]}); + mark_resume_progress(&mut info); + assert_eq!(info["resumePending"], true); + assert_eq!(info["phase"], "resuming"); + } + for status in [0, 7, 8] { + let mut info = json!({"status":status, "resumePending":true, + "rtspsEndpoints":["rtsps://example.invalid:322"]}); + mark_resume_progress(&mut info); + assert_eq!(info["resumePending"], false); + assert_eq!( + info["phase"], + if status == 7 { "finished" } else { "failed" } + ); + } + } + + #[test] + fn direct_resume_poll_retains_regional_discovery_endpoint() { + let direct = trusted_learned_server_base("80.84.160.10").unwrap(); + let payload = json!({"session": {"sessionId":"resumed-seat", "status":2, + "connectionInfo":[{"usage":14, "ip":"80.84.160.10"}, + {"usage":16, "ip":"80.84.160.10", "port":322}]}}); + let info = session_info( + &payload, + &direct, + "np-sof-01.cloudmatchbeta.nvidiagrid.net", + "123", + "device", + ) + .unwrap(); + assert_eq!( + info["streamingBaseUrl"], + direct.origin().ascii_serialization() + ); + assert_eq!(info["serverIp"], "80.84.160.10"); + assert_eq!(info["rtspsEndpoints"][0], "rtsps://80.84.160.10:322"); + assert!(trusted_cloudmatch_base(direct.as_str()).is_ok()); + } + + #[test] + fn launched_variant_without_hdr_requests_sdr_for_that_session_only() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420"],"hdrColorQualities":["10bit_420"]} + ] + }]}); + let hdr_on = json!({"codec":"auto","enableHdr":true,"colorQuality":"8bit_420"}); + let persisted = hdr_on.clone(); + let request = |settings: &Value, variant: &Value| { + let settings = catalog_hdr_settings(settings, &capabilities, variant); + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &settings, + &capabilities, + ) + .unwrap(); + let body = build_create_body("123", &json!({}), &resolved, "device"); + (body["sessionRequestData"].clone(), resolved) + }; + + let (sdr, resolved) = request(&hdr_on, &json!({"features":["rtx","reflex"]})); + assert_eq!(resolved["hdrDowngrade"], "game-unsupported"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "8bit_420"); + assert_eq!(sdr["sdrHdrMode"], 0); + assert_eq!(sdr["clientRequestMonitorSettings"][0]["sdrHdrMode"], 0); + assert!(sdr["clientRequestMonitorSettings"][0]["displayData"].is_null()); + assert_eq!(sdr["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!(sdr["requestedStreamingFeatures"]["bitDepth"], 0); + + let (hdr, resolved) = request(&hdr_on, &json!({"features":["hdr"]})); + assert!(resolved["hdrDowngrade"].is_null()); + assert_eq!(hdr["sdrHdrMode"], 1); + assert!(hdr["clientRequestMonitorSettings"][0]["displayData"].is_object()); + + for unknown in [json!({"features":null}), json!({"id":"123"}), Value::Null] { + let (request, resolved) = request(&hdr_on, &unknown); + assert!(resolved["hdrDowngrade"].is_null(), "{unknown}"); + assert_eq!(request["sdrHdrMode"], 1, "{unknown}"); + } + + let hdr_off = json!({"codec":"auto","enableHdr":false,"colorQuality":"8bit_420"}); + let (request, resolved) = request(&hdr_off, &json!({"features":[]})); + assert!(resolved["hdrDowngrade"].is_null()); + assert_eq!(request["sdrHdrMode"], 0); + assert!( + catalog_hdr_settings( + &hdr_on, + &json!({"nativeHdrSupported":false}), + &json!({"features":[]}) + )["hdrDowngrade"] + .is_null(), + "a display without HDR output is already SDR" + ); + assert_eq!( + hdr_on, persisted, + "the saved HDR preference is never rewritten" + ); + } + + #[test] + fn hdr_downgrade_follows_the_same_session_through_polls_and_resume() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let payload = |id: &str| json!({"session":{"sessionId":id,"status":2,"sdrHdrMode":0}}); + let mut created = session_info(&payload("seat"), &base, "auto", "123", "device").unwrap(); + assert!(created["hdrDowngrade"].is_null() && created["catalogAppId"].is_null()); + created["hdrDowngrade"] = json!("game-unsupported"); + created["catalogAppId"] = json!("doom-eternal"); + let mut polled = session_info(&payload("seat"), &base, "auto", "123", "device").unwrap(); + preserve_session_profile(&mut polled, &created); + assert_eq!(polled["hdrDowngrade"], "game-unsupported"); + assert_eq!(polled["catalogAppId"], "doom-eternal"); + assert_eq!(polled["negotiatedStreamProfile"]["enableHdr"], false); + let mut other = session_info(&payload("other"), &base, "auto", "123", "device").unwrap(); + preserve_session_profile(&mut other, &created); + assert!(other["hdrDowngrade"].is_null() && other["catalogAppId"].is_null()); + } + + #[test] + fn hdr_444_request_and_accepted_session_preserve_wire_chroma() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["10bit_444"],"hdrColorQualities":["10bit_444"]} + ] + }]}); + for color in ["8bit_444", "10bit_444"] { + let settings = crate::streamer::StreamerService::embedded_session_settings( + &json!({"codec":"h265","enableHdr":true,"colorQuality":color}), + &capabilities, + ) + .unwrap(); + let body = build_create_body("123", &json!({}), &settings, "device"); + let request = &body["sessionRequestData"]; + assert_eq!(request["sdrHdrMode"], 1); + assert_eq!(request["clientRequestMonitorSettings"][0]["sdrHdrMode"], 1); + assert_eq!(request["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!(request["requestedStreamingFeatures"]["bitDepth"], 1); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 1); + } + let payload = json!({"session":{"sessionId":"hdr-444","status":2,"sdrHdrMode":1, + "finalizedStreamingFeatures":{"codec":2,"bitDepth":1,"chromaFormat":1}}}); + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["colorQuality"], "10bit_444"); + assert_eq!(info["negotiatedStreamProfile"]["enableHdr"], true); + let prepared = crate::streamer::StreamerService::new() + .prepare_embedded( + &json!({"session":info,"runtimeCapabilities":capabilities}), + &json!({"codec":"auto","colorQuality":"8bit_420","enableHdr":false}), + ) + .unwrap(); + assert_eq!(prepared["context"]["settings"]["colorQuality"], "10bit_444"); + assert_eq!(prepared["context"]["settings"]["enableHdr"], true); + } + + #[test] + fn hdr_request_requires_resolved_runtime_opt_in_and_uses_cloudmatch_enums() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + let settings = crate::streamer::StreamerService::embedded_session_settings( + &json!({"enableHdr":true}), + &capabilities, + ) + .unwrap(); + let body = build_create_body("123", &json!({}), &settings, "device"); + let request = &body["sessionRequestData"]; + assert_eq!(request["sdrHdrMode"], 1); + assert_eq!(request["clientRequestMonitorSettings"][0]["sdrHdrMode"], 1); + assert_eq!(request["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!(request["requestedStreamingFeatures"]["bitDepth"], 1); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 0); + let display_data = &request["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 1000); + assert_eq!(display_data["desiredContentMaxFrameAverageLuminance"], 400); + assert_eq!(display_data["desiredContentMinLuminance"], 0); + for settings in [ + json!({}), + json!({"codec":"h265","enableHdr":true}), + json!({"codec":"h265","nativeHdrSupported":true}), + json!({"codec":"h264","enableHdr":true,"nativeHdrSupported":true}), + json!({"codec":"h265","enableHdr":true,"nativeHdrSupported":true,"decoderPreference":"software"}), + ] { + let body = build_create_body("123", &json!({}), &settings, "device"); + assert_eq!(body["sessionRequestData"]["sdrHdrMode"], 0); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert!(display_data.is_null()); + assert_eq!( + body["sessionRequestData"]["requestedStreamingFeatures"]["trueHdr"], + false + ); + } + } + + #[test] + fn validated_display_luminance_replaces_requested_content_defaults() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }],"nativeHdrDisplay":{"minimumNits":0.005,"maximumNits":620}}); + let settings = crate::streamer::StreamerService::embedded_session_settings( + &json!({"enableHdr":true}), + &capabilities, + ) + .unwrap(); + let body = build_create_body("123", &json!({}), &settings, "device"); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 620.0); + assert_eq!(display_data["desiredContentMinLuminance"], 50); + assert!( + display_data + .get("desiredContentMaxFrameAverageLuminance") + .is_none() + ); + assert!(display_data.get("displayPrimaryX0").is_none()); + assert!(display_data.get("displayWhitePointY").is_none()); + assert_eq!(body["sessionRequestData"]["sdrHdrMode"], 1); + let sdr = build_create_body( + "123", + &json!({}), + &json!({"enableHdr":false,"nativeHdrDisplay":{"minimumNits":0.005,"maximumNits":620}}), + "device", + ); + let sdr_data = &sdr["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert!(sdr_data.is_null()); + } + + #[test] + fn measured_monitor_metadata_survives_resolution_and_uses_wire_units() { + for peak in [400, 620, 1068] { + let full_frame = if peak == 620 { peak } else { peak - 80 }; + let display = json!({"minimumNits":0.005,"maximumNits":peak, + "maximumFullFrameNits":full_frame, + "redX":0.64,"redY":0.33,"greenX":0.30,"greenY":0.60, + "blueX":0.15,"blueY":0.06,"whiteX":0.3127,"whiteY":0.329}); + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ]}],"nativeHdrDisplay":display}); + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &json!({"codec":"h265","enableHdr":true,"nativeHdrDisplay":{ + "minimumNits":0.0,"maximumNits":1000,"maximumFullFrameNits":400, + "redX":0.7}}), + &capabilities, + ) + .unwrap(); + for (key, value) in display.as_object().unwrap() { + assert_eq!( + resolved["nativeHdrDisplay"][key].as_f64(), + value.as_f64(), + "{key}" + ); + } + let body = build_create_body("123", &json!({}), &resolved, "device"); + let request = &body["sessionRequestData"]; + let data = &request["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(request["requestedStreamingFeatures"]["trueHdr"], false); + assert_eq!( + data["desiredContentMaxLuminance"].as_f64(), + Some(peak as f64) + ); + assert_eq!(data["desiredContentMinLuminance"], 50); + assert_eq!( + data["desiredContentMaxFrameAverageLuminance"].as_f64(), + Some(full_frame as f64) + ); + for (key, expected) in [ + ("displayPrimaryX0", 32000), + ("displayPrimaryY0", 16500), + ("displayPrimaryX1", 15000), + ("displayPrimaryY1", 30000), + ("displayPrimaryX2", 7500), + ("displayPrimaryY2", 3000), + ("displayWhitePointX", 15635), + ("displayWhitePointY", 16450), + ] { + assert_eq!(data[key], expected, "{key} at {peak} nits"); + } + let sdr = build_create_body( + "123", + &json!({}), + &json!({"codec":"h265", + "enableHdr":false,"nativeHdrSupported":true,"nativeHdrDisplay":display}), + "device", + ); + assert!( + sdr["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"] + .is_null() + ); + assert_eq!( + sdr["sessionRequestData"]["requestedStreamingFeatures"]["trueHdr"], + false + ); + } + } + + #[test] + fn incomplete_or_invalid_monitor_metadata_keeps_only_valid_luminance() { + let valid = json!({"minimumNits":0.005,"maximumNits":620, + "maximumFullFrameNits":400, + "redX":0.64,"redY":0.33,"greenX":0.30,"greenY":0.60, + "blueX":0.15,"blueY":0.06,"whiteX":0.3127,"whiteY":0.329}); + let capabilities = |display: Value| { + json!({"protocolVersion":7,"nativeHdrSupported":true, + "videoBackends":[{"backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ]}],"nativeHdrDisplay":display}) + }; + for (key, value) in [ + ("maximumFullFrameNits", json!(0.005)), + ("maximumFullFrameNits", json!(621)), + ("maximumFullFrameNits", json!(null)), + ("redX", json!(1.1)), + ("redY", json!(-0.1)), + ("greenX", json!("0.3")), + ("blueY", json!(null)), + ("whiteX", json!(0.9)), + ("whiteY", json!(0.0)), + ("greenX", json!(0.64)), + ("greenX", json!(0.640001)), + ] { + let mut display = valid.clone(); + display[key] = value; + if key == "blueY" { + display.as_object_mut().unwrap().remove(key); + } + if key == "greenX" && display[key] == json!(0.64) { + display["greenY"] = json!(0.33); + } + if key == "greenX" && display[key] == json!(0.640001) { + display["greenY"] = json!(0.330001); + } + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &json!({"codec":"h265","enableHdr":true,"nativeHdrDisplay":valid}), + &capabilities(display), + ) + .unwrap(); + assert_eq!( + resolved["nativeHdrDisplay"], + json!({"minimumNits":0.005,"maximumNits":620.0}), + "{key}" + ); + let body = build_create_body("123", &json!({}), &resolved, "device"); + let data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(data["desiredContentMaxLuminance"].as_f64(), Some(620.0)); + assert_eq!(data["desiredContentMinLuminance"], 50); + assert!(data.get("desiredContentMaxFrameAverageLuminance").is_none()); + assert!(data.get("displayPrimaryX0").is_none()); + assert!(data.get("displayWhitePointY").is_none()); + } + let no_display = crate::streamer::StreamerService::embedded_session_settings( + &json!({"codec":"h265","enableHdr":true,"nativeHdrDisplay":valid}), + &json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ]}]}), + ) + .unwrap(); + assert!(no_display.get("nativeHdrDisplay").is_none()); + let fallback = build_create_body("123", &json!({}), &no_display, "device"); + let data = + &fallback["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(data["desiredContentMaxLuminance"], 1000); + assert_eq!(data["desiredContentMinLuminance"], 0); + assert_eq!(data["desiredContentMaxFrameAverageLuminance"], 400); + } + + #[test] + fn malformed_display_luminance_keeps_documented_defaults() { + for display in [ + json!({"minimumNits":600.0,"maximumNits":400.0}), + json!({"minimumNits":-1.0,"maximumNits":400.0}), + json!({"minimumNits":0.0,"maximumNits":10001.0}), + json!({"minimumNits":0.0}), + json!({"minimumNits":"0","maximumNits":400.0}), + json!([0.0, 400.0]), + ] { + let settings = json!({"enableHdr":true,"nativeHdrSupported":true, + "codec":"h265","nativeHdrDisplay":display}); + let body = build_create_body("123", &json!({}), &settings, "device"); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 1000); + assert_eq!(display_data["desiredContentMinLuminance"], 0); + assert_eq!(display_data["desiredContentMaxFrameAverageLuminance"], 400); + } + } + + #[test] + fn native_hdr_display_capability_requires_a_validated_pair() { + let capabilities = |display: Value| { + json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }],"nativeHdrDisplay":display}) + }; + for (display, minimum, maximum) in [ + (json!({"minimumNits":0.005,"maximumNits":620}), 0.005, 620.0), + (json!({"minimumNits":0,"maximumNits":400}), 0.0, 400.0), + ] { + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &json!({"enableHdr":true,"nativeHdrSupported":true}), + &capabilities(display), + ) + .unwrap(); + assert_eq!(resolved["nativeHdrDisplay"]["minimumNits"], minimum); + assert_eq!(resolved["nativeHdrDisplay"]["maximumNits"], maximum); + } + for display in [ + json!({"minimumNits":400.0,"maximumNits":400.0}), + json!({"minimumNits":0.0,"maximumNits":10001.0}), + json!({"maximumNits":620}), + json!({"minimumNits":0.005}), + json!("620"), + ] { + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &json!({"enableHdr":true,"nativeHdrSupported":true}), + &capabilities(display), + ) + .unwrap(); + assert!(resolved.get("nativeHdrDisplay").is_none()); + } + } + + #[test] + fn stale_display_snapshot_is_dropped_across_output_transitions() { + let capabilities = |display: Value| { + json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }],"nativeHdrDisplay":display}) + }; + let previous = json!({"enableHdr":true,"nativeHdrSupported":true,"codec":"h265", + "nativeHdrDisplay":{"minimumNits":0.005,"maximumNits":620}}); + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &previous, + &json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"vaapi","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}), + ) + .unwrap(); + assert!(resolved.get("nativeHdrDisplay").is_none()); + let body = build_create_body("123", &json!({}), &resolved, "device"); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 1000); + assert_eq!(display_data["desiredContentMinLuminance"], 0); + assert_eq!(display_data["desiredContentMaxFrameAverageLuminance"], 400); + for invalid in [ + json!({"minimumNits":620,"maximumNits":620}), + json!({"minimumNits":0.005,"maximumNits":10001}), + json!("unavailable"), + ] { + let stale = crate::streamer::StreamerService::embedded_session_settings( + &previous, + &capabilities(invalid), + ) + .unwrap(); + assert!(stale.get("nativeHdrDisplay").is_none()); + let body = build_create_body("123", &json!({}), &stale, "device"); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 1000); + assert_eq!(display_data["desiredContentMinLuminance"], 0); + assert_eq!(display_data["desiredContentMaxFrameAverageLuminance"], 400); + } + let migrated = crate::streamer::StreamerService::embedded_session_settings( + &previous, + &capabilities(json!({"minimumNits":0.0005,"maximumNits":400})), + ) + .unwrap(); + let body = build_create_body("123", &json!({}), &migrated, "device"); + let display_data = + &body["sessionRequestData"]["clientRequestMonitorSettings"][0]["displayData"]; + assert_eq!(display_data["desiredContentMaxLuminance"], 400.0); + assert_eq!(display_data["desiredContentMinLuminance"], 5); + assert!( + display_data + .get("desiredContentMaxFrameAverageLuminance") + .is_none() + ); + let resume = build_resume_body( + "123", + &json!({"sessionId":"s","status":2, + "sessionRequestData":{"sdrHdrMode":1,"clientRequestMonitorSettings":[{"sdrHdrMode":1}]}}), + &resolved, + "device", + ); + assert!(resume.to_string().find("desiredContent").is_none()); + } + + #[test] + fn accepted_hdr_mode_survives_resume_and_explicit_sdr_fallback_wins() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let mut payload = json!({"session":{"sessionId":"hdr-seat","status":2, + "sessionRequestData":{"sdrHdrMode":1,"clientRequestMonitorSettings":[{"sdrHdrMode":1}]}, + "finalizedStreamingFeatures":{"codec":2,"bitDepth":1,"chromaFormat":0}}}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["enableHdr"], true); + let resumed = build_resume_body( + "123", + &payload["session"], + &json!({"enableHdr":false}), + "device", + ); + assert_eq!(resumed["sessionRequestData"]["sdrHdrMode"], 1); + assert!( + resumed["sessionRequestData"] + .get("clientRequestMonitorSettings") + .is_none() + ); + assert!( + resumed["sessionRequestData"] + .get("requestedStreamingFeatures") + .is_none() + ); + payload["session"]["sdrHdrMode"] = json!(0); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["enableHdr"], false); + let resumed = build_resume_body( + "123", + &payload["session"], + &json!({"enableHdr":true,"nativeHdrSupported":true}), + "device", + ); + assert_eq!(resumed["sessionRequestData"]["sdrHdrMode"], 0); + assert!( + resumed["sessionRequestData"] + .get("clientRequestMonitorSettings") + .is_none() + ); + assert!( + resumed["sessionRequestData"] + .get("requestedStreamingFeatures") + .is_none() + ); + assert_eq!(accepted_hdr_mode(&json!({})), None); + assert_eq!(accepted_hdr_mode(&json!({"sdrHdrMode":2})), Some(0)); + } + + #[test] + fn in_game_settings_persistence_defaults_on_and_requires_game_support() { + for preference in [Value::Null, json!(false), json!(true)] { + for support in [Value::Null, json!(false), json!(true)] { + let mut params = json!({}); + let mut settings = json!({}); + if !support.is_null() { + params["supportsInGameSettingsPersistence"] = support.clone(); + } + if !preference.is_null() { + settings["enablePersistingInGameSettings"] = preference.clone(); + } + let body = build_create_body("123", ¶ms, &settings, "stable-device"); + assert_eq!( + body["sessionRequestData"]["enablePersistingInGameSettings"], + preference != false && support == true + ); + } + } + } + + #[test] + fn resume_preserves_in_game_settings_persistence_despite_preference_changes() { + for enabled in [false, true] { + let original = json!({"sessionRequestData": { + "enablePersistingInGameSettings": enabled + }}); + let body = build_resume_body( + "123", + &original, + &json!({"enablePersistingInGameSettings": !enabled}), + "stable-device", + ); + assert_eq!( + body["sessionRequestData"]["enablePersistingInGameSettings"], + enabled + ); + } + } + + #[test] + fn resume_does_not_renegotiate_allocated_video_parameters() { + let original = json!({"sessionRequestData": { + "appLaunchMode":2, "enablePersistingInGameSettings":true, + "clientPlatformName":"windows"}}); + let body = build_resume_body( + "123", + &original, + &json!({"resolution":"3840x2160", "fps":240, "codec":"av1"}), + "stable-device", + ); + assert_eq!(body["action"], 2); + assert_eq!(body["data"], "RESUME"); + let request = &body["sessionRequestData"]; + assert_eq!(request["deviceHashId"], "stable-device"); + assert_eq!(request["appLaunchMode"], 2); + assert_eq!(request["enablePersistingInGameSettings"], true); + assert!(request.get("clientRequestMonitorSettings").is_none()); + assert!(request.get("requestedStreamingFeatures").is_none()); + assert!( + request["metaData"] + .as_array() + .unwrap() + .iter() + .all(|entry| entry["key"] != "clientPhysicalResolution") + ); + } + + #[test] + fn resume_waits_for_fresh_ready_status_and_native_endpoints() { + for status in [1, 6, 2, 3] { + let mut info = + json!({"status":status, "phase":session_phase(status), "rtspsEndpoints":[]}); + mark_resume_progress(&mut info); + assert_eq!(info["resumePending"], true); + assert_eq!(info["phase"], "resuming"); + } + for status in [2, 3] { + let mut info = json!({"status":status, "phase":session_phase(status), + "rtspsEndpoints":["rtsps://example.invalid:48010"]}); + mark_resume_progress(&mut info); + assert_eq!(info["resumePending"], false); + assert_eq!(info["phase"], session_phase(status)); + } + } + + #[test] + fn session_requests_keep_the_classic_nvst_client_identity() { + let headers = cloudmatch_headers("token", "device-id").unwrap(); + assert_eq!(headers["nv-client-type"], "NATIVE"); + assert_eq!(headers["nv-client-streamer"], "NVIDIA-CLASSIC"); + } + + #[test] + fn builds_a_stable_official_session_shape() { + let body = build_create_body( + "12345", + &json!({ + "supportsInGameSettingsPersistence":true, + "title":"Portal 2", + "accountLinked":true, + "appLaunchMode":"gamepadFriendly" + }), + &json!({ + "resolution":"2560x1440","fps":120,"maxBitrateMbps":80, + "codec":"av1","colorQuality":"10bit_420","transportMode":"webrtc", + "enableCloudGsync":true,"enableL4S":true,"enablePersistingInGameSettings":true + }), + "device-id", + ); + let request = &body["sessionRequestData"]; + assert_eq!(request["appId"], 12345); + assert_eq!(request["deviceHashId"], "device-id"); + assert_eq!( + request["clientRequestMonitorSettings"][0]["widthInPixels"], + 2560 + ); + // Codec, bitrate ceiling, and the dynamic quality policy stay + // client-side: the official request carries neither, resolving the codec + // locally and the policy at RTSP ANNOUNCE time. + assert!(request["requestedStreamingFeatures"].get("codec").is_none()); + assert!( + request["requestedStreamingFeatures"] + .get("maxBitrateKbps") + .is_none() + ); + assert!(request.get("externalAppId").is_none()); + assert_eq!(request["enablePersistingInGameSettings"], true); + assert_eq!(request["internalTitle"], "Portal 2"); + assert_eq!(request["accountLinked"], true); + assert_eq!(request["appLaunchMode"], 2); + assert_eq!(request["secureRTSPSupported"], true); + assert!( + request["metaData"] + .as_array() + .expect("metadata") + .iter() + .all(|entry| entry["key"] != "GSStreamerType") + ); + assert!( + request["requestedStreamingFeatures"] + .get("dynamicStreamingMode") + .is_none() + ); + } + + #[test] + fn launch_mode_defaults_to_normal_and_maps_explicit_requests() { + for (mode, expected) in [ + (Value::Null, 1), + (json!("default"), 1), + (json!("gamepadFriendly"), 2), + (json!("touchFriendly"), 3), + (json!("unknown"), 1), + ] { + let body = build_create_body( + "12345", + &json!({"appLaunchMode": mode}), + &json!({"controllerMode":true,"launchInConsoleMode":true}), + "device-id", + ); + assert_eq!(body["sessionRequestData"]["appLaunchMode"], expected); + } + } + + #[test] + fn native_session_requests_use_the_current_local_timezone_in_milliseconds() { + let offset_before = client_timezone_offset_ms(*chrono::Local::now().offset()); + let created = build_create_body("12345", &json!({}), &json!({}), "device-id"); + let resumed = build_resume_body("12345", &json!({}), &json!({}), "device-id"); + let offset_after = client_timezone_offset_ms(*chrono::Local::now().offset()); + for body in [created, resumed] { + let offset = body["sessionRequestData"]["clientTimezoneOffset"] + .as_i64() + .unwrap(); + assert!(offset == offset_before || offset == offset_after); + } + } + + #[test] + fn client_timezone_offset_is_local_minus_utc_in_milliseconds() { + let hours = |h: i32| chrono::FixedOffset::east_opt(h * 3600).unwrap(); + assert_eq!(client_timezone_offset_ms(hours(-5)), -18_000_000); + assert_eq!(client_timezone_offset_ms(hours(0)), 0); + assert_eq!(client_timezone_offset_ms(hours(1)), 3_600_000); + let kathmandu = chrono::FixedOffset::east_opt(5 * 3600 + 45 * 60).unwrap(); + assert_eq!(client_timezone_offset_ms(kathmandu), 20_700_000); + } + + #[test] + fn session_metadata_matches_the_official_pair_set() { + let body = build_create_body("12345", &json!({}), &json!({}), "device-id"); + let metadata = body["sessionRequestData"]["metaData"].as_array().unwrap(); + assert_eq!(metadata[1]["key"], "SubSessionId"); + assert_eq!(metadata[1]["value"].as_str().unwrap().len(), 36); + assert_eq!(metadata[2], json!({"key":"surroundAudioInfo","value":"2"})); + } + + #[test] + fn session_metadata_contains_only_owned_values() { + let body = build_create_body( + "12345", + &json!({"zone":"eu-netherlands-north.cloudmatchbeta.nvidiagrid.net", + "networkType":"Ethernet", "ClientImeSupport":"1", + "latency@eu-netherlands-north.cloudmatchbeta.nvidiagrid.net":12}), + &json!({"resolution":"2560x1440", "windowWidth":2560, "windowHeight":1440, + "nativeHdrDisplay":{"minimumNits":0.005,"maximumNits":620}}), + "device-id", + ); + let metadata = body["sessionRequestData"]["metaData"].as_array().unwrap(); + assert_eq!(metadata.len(), 3); + assert_eq!(metadata[0], json!({"key":"wssignaling","value":"1"})); + assert_eq!(metadata[1]["key"], "SubSessionId"); + assert_eq!(metadata[2]["key"], "surroundAudioInfo"); + assert!(metadata.iter().all(|entry| { + !matches!( + entry["key"].as_str(), + Some("networkType" | "ClientImeSupport" | "clientPhysicalResolution") + ) && !entry["key"].as_str().unwrap().starts_with("latency@") + })); + } + + #[test] + fn concrete_create_zones_bypass_local_region_but_auto_discovers_it() { + let selected = json!({"metaData":[ + {"key":"local-region","value":"nearest"}, + {"key":"nearest","value":"https://nearest.partner.example/"}, + {"key":"gfn-regions","value":"queue"}, + {"key":"queue","value":"https://np-queue.nvidiagrid.net/"} + ]}); + let (requested, server) = session_server(vec![(200, selected)], |_| {}); + let client = Client::builder().no_proxy().build().unwrap(); + let service = CloudMatchService::new(client.clone()); + let auth = conflict_auth(); + for (params, settings, expected) in [ + ( + json!({"streamingBaseUrl":"https://chosen.partner.example/", "zone":"chosen"}), + json!({}), + "https://chosen.partner.example/", + ), + ( + json!({}), + json!({"region":"https://chosen.partner.example/"}), + "https://chosen.partner.example/", + ), + ( + json!({"streamingBaseUrl":"https://np-queue.nvidiagrid.net/", "zone":"np-queue"}), + json!({}), + "https://np-queue.nvidiagrid.net/", + ), + ] { + let chosen = requested_streaming_base(¶ms, &settings, &auth).unwrap(); + assert_eq!(chosen.as_str(), expected); + assert_eq!( + service + .create_base(&client, &chosen, ¶ms, &settings, "token", "device") + .unwrap(), + chosen + ); + } + assert_eq!( + service + .create_base( + &client, + &requested, + &json!({}), + &json!({}), + "token", + "device" + ) + .unwrap() + .as_str(), + "https://nearest.partner.example/" + ); + assert_eq!(server.join().unwrap(), ["GET /v2/serverInfo HTTP/1.1"]); + } + + #[test] + fn automatic_codec_delegates_selection_to_cloudmatch() { + assert_eq!(codec_wire("auto"), 0); + assert_eq!(codec_wire("unknown"), 0); + assert_eq!(codec_wire("h264"), 1); + assert_eq!(codec_wire("h265"), 2); + assert_eq!(codec_wire("av1"), 3); + } + + #[test] + fn native_request_preserves_stream_quality_and_bandwidth() { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &json!({ + "resolution":"2560x1440", + "fps":120, + "maxBitrateMbps":75, + "codec":"auto", + "colorQuality":"10bit_444", + "transportMode":"nvst" + }), + "device-id", + ); + let features = &body["sessionRequestData"]["requestedStreamingFeatures"]; + assert!(features.get("codec").is_none()); + assert_eq!(features["bitDepth"], 1); + assert_eq!(features["chromaFormat"], 1); + assert!(features.get("maxBitrateKbps").is_none()); + assert!(features.get("dynamicStreamingMode").is_none()); + assert!(features.get("audioChannelCount").is_none()); + assert!(features.get("vsync").is_none()); + assert!(features.get("qosPolicy").is_none()); + assert!(features.get("touchSupport").is_none()); + } + + #[test] + fn session_create_requests_the_documented_frame_rate_ceiling() { + let hardware = json!({"protocolVersion":7, "videoBackends":[{"backend":"vaapi", + "available":true, "codecs":[{"codec":"h265", "available":true, + "colorQualities":["8bit_420"]}]}]}); + let software = json!({"protocolVersion":7, "videoBackends":[{"backend":"software", + "available":true, "codecs":[{"codec":"h265", "available":true, + "colorQualities":["8bit_420"]}]}]}); + let request = |resolution: &str, fps: i64, capabilities: &Value, entitled: i64| { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2", "runtimeCapabilities":capabilities, + "maxEntitledFps":entitled}), + &json!({"resolution":resolution, "fps":fps, "codec":"h265"}), + "device-id", + ); + body["sessionRequestData"]["clientRequestMonitorSettings"][0]["framesPerSecond"].clone() + }; + assert_eq!(request("1920x1080", 360, &hardware, 360), json!(360)); + assert_eq!(request("1920x1200", 360, &hardware, 360), json!(360)); + for resolution in [ + "2560x1440", + "2560x1600", + "3440x1440", + "3840x2160", + "3840x1080", + ] { + assert_eq!( + request(resolution, 360, &hardware, 360), + json!(240), + "{resolution} must not request the full-HD-only tier" + ); + assert_eq!(request(resolution, 240, &hardware, 360), json!(240)); + } + assert_eq!(request("1920x1080", 999, &hardware, 360), json!(360)); + assert_eq!(request("1920x1080", 1, &hardware, 360), json!(30)); + assert_eq!( + request("1920x1080", 360, &software, 360), + json!(240), + "a software-only decode path cannot request the top tier" + ); + assert_eq!( + request("1920x1080", 360, &json!({}), 360), + json!(240), + "an unreported capability probe is not affirmative support" + ); + assert_eq!( + request("1920x1080", 360, &hardware, 0), + json!(240), + "unconfirmed entitlement cannot request the top tier" + ); + assert_eq!( + request("1920x1080", 360, &hardware, 240), + json!(240), + "a 240 FPS entitlement cannot request the top tier" + ); + assert_eq!( + request("1920x1080", 360, &hardware, 120), + json!(120), + "a lower entitlement bounds the request" + ); + assert_eq!( + request("1920x1080", 240, &software, 0), + json!(240), + "base rates stay unaffected by the capability verdict" + ); + } + + #[test] + fn the_network_test_profile_matches_the_session_profile() { + let hardware = json!({"protocolVersion":7, "videoBackends":[{"backend":"vaapi", + "available":true, "codecs":[{"codec":"h265", "available":true, + "colorQualities":["8bit_420"]}]}]}); + let software = json!({"protocolVersion":7, "videoBackends":[{"backend":"software", + "available":true, "codecs":[{"codec":"h265", "available":true, + "colorQualities":["8bit_420"]}]}]}); + let settings = json!({"resolution":"1920x1080", "fps":360, "codec":"h265"}); + for (capabilities, entitled, expected) in [ + (&hardware, 360_i64, 360_i64), + (&software, 360, 240), + (&json!({}), 360, 240), + (&hardware, 0, 240), + (&hardware, 120, 120), + ] { + let params = json!({"runtimeCapabilities":capabilities, "maxEntitledFps":entitled}); + let session = build_create_body("12345", ¶ms, &settings, "device-id"); + let session_fps = + session["sessionRequestData"]["clientRequestMonitorSettings"][0]["framesPerSecond"] + .clone(); + let profile = network_test_display_profile(&settings, ¶ms); + let allocation = crate::network_test::allocation_body("GFN-PC", profile); + assert_eq!(profile.width, 1920); + assert_eq!(profile.height, 1080); + assert_eq!( + allocation["netTestRequestData"]["netTestProfile"]["framesPerSecond"], session_fps, + "the allocation profile must match the session profile for {capabilities}" + ); + assert_eq!(session_fps, json!(expected), "{capabilities}"); + } + } + + #[test] + fn manual_av1_uses_native_nvst_even_with_a_legacy_transport_value() { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &json!({ + "codec":"av1", + "colorQuality":"10bit_420", + "transportMode":"webrtc" + }), + "device-id", + ); + let request = &body["sessionRequestData"]; + assert!(request["requestedStreamingFeatures"].get("codec").is_none()); + assert_eq!(request["requestedStreamingFeatures"]["bitDepth"], 1); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 0); + assert!( + request["requestedStreamingFeatures"] + .get("dynamicStreamingMode") + .is_none() + ); + assert_eq!(request["secureRTSPSupported"], true); + } + + #[test] + fn manual_av1_preserves_ten_bit_but_constrains_unsupported_444_chroma() { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &json!({"codec":"av1", "colorQuality":"10bit_444"}), + "device-id", + ); + let features = &body["sessionRequestData"]["requestedStreamingFeatures"]; + assert!(features.get("codec").is_none()); + assert_eq!(features["bitDepth"], 1); + assert_eq!(features["chromaFormat"], 0); + } + + #[test] + fn resolved_capability_fallbacks_reach_the_request_unchanged() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":false, + "colorQualities":["8bit_420","10bit_420","10bit_444"]}, + {"codec":"av1","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + for (requested, codec, depth, chroma, hdr) in [ + ( + json!({"codec":"av1","colorQuality":"10bit_444"}), + "h265", + 1, + 1, + 0, + ), + ( + json!({"codec":"h264","colorQuality":"8bit_444"}), + "h264", + 0, + 0, + 0, + ), + ( + json!({"codec":"h264","colorQuality":"10bit_420"}), + "h264", + 0, + 0, + 0, + ), + ( + json!({"codec":"h265","colorQuality":"10bit_420","enableHdr":true}), + "h265", + 1, + 0, + 0, + ), + (json!({"codec":"h264","enableHdr":true}), "h264", 0, 0, 0), + ] { + let resolved = crate::streamer::StreamerService::embedded_session_settings( + &requested, + &capabilities, + ) + .unwrap(); + assert_eq!(resolved["codec"], codec, "{requested}"); + let body = build_create_body("123", &json!({}), &resolved, "device"); + let request = &body["sessionRequestData"]; + let features = &request["requestedStreamingFeatures"]; + assert_eq!(features["bitDepth"], depth, "{requested}"); + assert_eq!(features["chromaFormat"], chroma, "{requested}"); + assert_eq!(request["sdrHdrMode"], hdr, "{requested}"); + let profile = negotiated_profile(&request["clientRequestMonitorSettings"][0], features); + assert_eq!( + profile["colorQuality"], resolved["colorQuality"], + "{requested}" + ); + } + } + + #[test] + fn bandwidth_saving_stays_client_side_and_out_of_the_cloudmatch_request() { + // The official client never sends dynamicStreamingMode to CloudMatch: the + // bandwidth policy is an RTSP ANNOUNCE value resolved from live settings. + for saved in [None, Some(false), Some(true)] { + let mut settings = json!({ + "resolution":"1920x1080", + "fps":60, + "maxBitrateMbps":75 + }); + if let Some(saved) = saved { + settings["saveBandwidth"] = json!(saved); + } + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &settings, + "device-id", + ); + assert!( + body["sessionRequestData"]["requestedStreamingFeatures"] + .get("dynamicStreamingMode") + .is_none() + ); + } + } + + #[test] + fn negotiated_profile_carries_the_session_dynamic_quality_policy() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let parse = |session| { + session_info(&json!({"session":session}), &base, "", "123", "device").unwrap() + }; + for (echoed, finalized, mode) in [ + (None, None, Value::Null), + (Some(1), None, json!(1)), + (Some(1), Some(0), json!(0)), + (None, Some(3), json!(3)), + (Some(7), None, Value::Null), + ] { + let mut session = json!({"sessionId":"seat","status":2}); + if let Some(echoed) = echoed { + session["sessionRequestData"]["requestedStreamingFeatures"]["dynamicStreamingMode"] = + json!(echoed); + } + if let Some(finalized) = finalized { + session["finalizedStreamingFeatures"]["dynamicStreamingMode"] = json!(finalized); + } + assert_eq!( + parse(session)["negotiatedStreamProfile"]["dynamicStreamingMode"], + mode + ); + } + } + + #[test] + fn manual_h265_preserves_ten_bit_color_on_native_nvst() { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &json!({ + "codec":"h265", + "colorQuality":"10bit_420", + "transportMode":"nvst" + }), + "device-id", + ); + let request = &body["sessionRequestData"]; + assert!(request["requestedStreamingFeatures"].get("codec").is_none()); + assert_eq!(request["requestedStreamingFeatures"]["bitDepth"], 1); + assert_eq!(request["requestedStreamingFeatures"]["chromaFormat"], 0); + assert!( + request["requestedStreamingFeatures"] + .get("dynamicStreamingMode") + .is_none() + ); + assert_eq!(request["secureRTSPSupported"], true); + } + + #[test] + fn manual_h264_stays_fixed_and_constrains_unsupported_color() { + let body = build_create_body( + "12345", + &json!({"title":"Portal 2"}), + &json!({ + "codec":"h264", + "colorQuality":"10bit_444", + "transportMode":"nvst" + }), + "device-id", + ); + let features = &body["sessionRequestData"]["requestedStreamingFeatures"]; + assert!(features.get("codec").is_none()); + assert_eq!(features["bitDepth"], 0); + assert_eq!(features["chromaFormat"], 0); + } + + #[test] + fn native_cloud_gsync_policy_overrides_the_general_toggle() { + assert!(!resolved_cloud_gsync(&json!({ + "enableCloudGsync": true, + "nativeCloudGsyncMode": "disabled" + }))); + assert!(resolved_cloud_gsync(&json!({ + "enableCloudGsync": false, + "nativeCloudGsyncMode": "forced" + }))); + assert!(resolved_cloud_gsync(&json!({ + "enableCloudGsync": true, + "nativeCloudGsyncMode": "auto" + }))); + } + + #[test] + fn parses_pending_and_ready_session_responses() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let pending = json!({ + "requestStatus":{"statusCode":1}, + "session":{"sessionId":"one","status":1,"queuePosition":42, + "sessionControlInfo":{"ip":"np-ams-01.cloudmatchbeta.nvidiagrid.net"}} + }); + let info = session_info(&pending, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["phase"], "preparing"); + assert_eq!(info["queuePosition"], 42); + assert_eq!( + info["streamingBaseUrl"], + "https://np-ams-01.cloudmatchbeta.nvidiagrid.net" + ); + + let ready = json!({ + "requestStatus":{"statusCode":1}, + "session":{"sessionId":"one","status":2, + "connectionInfo":[{"usage":14,"appLevelProtocol":6,"ip":"80.1.2.3","port":443,"resourcePath":"/nvst/"}]} + }); + let info = session_info(&ready, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["signalingUrl"], "wss://80.1.2.3:443"); + assert_eq!(info["serverIp"], "80.1.2.3"); + } + + #[test] + fn omitted_codec_keeps_the_client_selected_codec_through_hdr_preparation() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let requested = json!({"codec":"h265","colorQuality":"10bit_420","enableHdr":true,"nativeHdrSupported":true}); + let body = build_create_body("123", &json!({}), &requested, "device"); + // The official request names no codec: color is negotiated, the codec + // stays a client selection until the RTSP ANNOUNCE. + assert!( + body["sessionRequestData"]["requestedStreamingFeatures"] + .get("codec") + .is_none() + ); + let initial = session_info( + &json!({"session":{"sessionId":"omitted-codec","status":1,"sdrHdrMode":1}}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + assert_eq!(initial["negotiatedStreamProfile"]["codec"], Value::Null); + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[{ + "codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["10bit_420"],"hdrColorQualities":["10bit_420"] + }] + }]}); + for status in [2, 3] { + let mut ready = session_info( + &json!({"session":{ + "sessionId":"omitted-codec","status":status,"sdrHdrMode":1, + "finalizedStreamingFeatures":{"bitDepth":1,"chromaFormat":0} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + assert_eq!(ready["negotiatedStreamProfile"]["codec"], Value::Null); + preserve_session_profile(&mut ready, &initial); + assert_eq!(ready["negotiatedStreamProfile"]["codec"], Value::Null); + let prepared = crate::streamer::StreamerService::new() + .prepare_embedded( + &json!({"session":ready,"runtimeCapabilities":capabilities}), + &json!({"codec":"h265","colorQuality":"10bit_420","enableHdr":true}), + ) + .unwrap(); + assert_eq!(prepared["context"]["settings"]["codec"], "H265"); + assert_eq!(prepared["context"]["settings"]["enableHdr"], true); + } + } + + #[test] + fn codec_inheritance_never_crosses_sessions_or_overrides_reported_values() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let previous = json!({"sessionId":"same-seat","negotiatedStreamProfile":{ + "codec":"H265","codecSource":"request" + }}); + let mut different = session_info( + &json!({"session":{ + "sessionId":"other-seat","status":2 + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + preserve_session_profile(&mut different, &previous); + assert_eq!(different["negotiatedStreamProfile"]["codec"], Value::Null); + for reported in [ + Value::Null, + json!("unsupported"), + json!("H264"), + json!("AV1"), + ] { + let mut info = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2,"negotiatedStreamProfile":{"codec":reported} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + let before = info.clone(); + preserve_session_profile(&mut info, &previous); + assert_eq!(info, before); + assert_eq!(info["negotiatedStreamProfile"]["codecSource"], "server"); + } + for value in [Value::Null, json!(0), json!(99)] { + let mut info = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2,"finalizedStreamingFeatures":{"codec":value} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + preserve_session_profile(&mut info, &previous); + assert_eq!(info["negotiatedStreamProfile"]["codec"], Value::Null); + assert_eq!(info["negotiatedStreamProfile"]["codecSource"], "server"); + } + } + + #[test] + fn reported_codec_survives_later_partial_responses_without_reverting_to_request() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let request = json!({"sessionId":"same-seat","negotiatedStreamProfile":{ + "codec":"H265","codecSource":"request" + }}); + let mut regional = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2,"negotiatedStreamProfile":{"codec":"AV1"} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + preserve_session_profile(&mut regional, &request); + let mut direct = session_info(&json!({"session":{ + "sessionId":"same-seat","status":2,"finalizedStreamingFeatures":{"bitDepth":1,"chromaFormat":0} + }}), &base, "auto", "123", "device").unwrap(); + preserve_session_profile(&mut direct, ®ional); + preserve_session_profile(&mut direct, &request); + assert_eq!(direct["negotiatedStreamProfile"]["codec"], "AV1"); + assert_eq!(direct["negotiatedStreamProfile"]["codecSource"], "server"); + assert_eq!( + direct["negotiatedStreamProfile"]["colorQuality"], + "10bit_420" + ); + } + + #[test] + fn active_updates_use_latest_codec_evidence_and_keep_returned_info_in_sync() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let mut initial = json!({"sessionId":"same-seat","negotiatedStreamProfile":{ + "codec":"H265","codecSource":"request" + }}); + service + .store_active(&mut initial, &base, "auto", "123", client.clone()) + .unwrap(); + let mut stale = initial.clone(); + let mut reported = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2,"negotiatedStreamProfile":{"codec":"AV1"} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + service + .store_active(&mut reported, &base, "auto", "123", client.clone()) + .unwrap(); + service + .store_active(&mut stale, &base, "auto", "123", client.clone()) + .unwrap(); + assert_eq!(stale["negotiatedStreamProfile"]["codec"], "AV1"); + assert_eq!(stale["negotiatedStreamProfile"]["codecSource"], "server"); + assert_eq!(service.active.lock().unwrap().as_ref().unwrap().info, stale); + let mut invalid = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2,"negotiatedStreamProfile":{"codec":null} + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + service + .store_active(&mut invalid, &base, "auto", "123", client.clone()) + .unwrap(); + let mut omitted = session_info( + &json!({"session":{ + "sessionId":"same-seat","status":2 + }}), + &base, + "auto", + "123", + "device", + ) + .unwrap(); + service + .store_active(&mut omitted, &base, "auto", "123", client) + .unwrap(); + assert_eq!(omitted["negotiatedStreamProfile"]["codec"], Value::Null); + } + + #[test] + fn active_keyboard_layout_survives_polling_but_not_a_different_session() { + let client = Client::new(); + let service = CloudMatchService::new(client.clone()); + let base = Url::parse("https://prod.cloudmatchbeta.nvidiagrid.net/").unwrap(); + let mut created = json!({"sessionId":"seat-one", "keyboardLayout":"fr-FR"}); + service + .store_active(&mut created, &base, "auto", "123", client.clone()) + .unwrap(); + + let mut polled = json!({"sessionId":"seat-one", "status":2}); + service + .store_active(&mut polled, &base, "auto", "123", client.clone()) + .unwrap(); + assert_eq!(polled["keyboardLayout"], "fr-FR"); + + let mut resumed = json!({"sessionId":"seat-one", "keyboardLayout":"de-DE"}); + service + .store_active(&mut resumed, &base, "auto", "123", client.clone()) + .unwrap(); + let mut polled = json!({"sessionId":"seat-one", "status":3}); + service + .store_active(&mut polled, &base, "auto", "123", client.clone()) + .unwrap(); + assert_eq!(polled["keyboardLayout"], "de-DE"); + + let mut different = json!({"sessionId":"seat-two"}); + service + .store_active(&mut different, &base, "auto", "123", client) + .unwrap(); + assert!(different["keyboardLayout"].is_null()); + } + + #[test] + fn nested_negotiated_codec_reaches_hdr_preparation() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[{ + "codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["10bit_420"],"hdrColorQualities":["10bit_420"] + }] + }]}); + let settings = json!({"codec":"h264","colorQuality":"8bit_420","enableHdr":false}); + for status in [2, 3] { + for codec in ["H265", "HEVC", "hevc"] { + let payload = json!({"session":{ + "sessionId":"nested-codec", "status":status, "sdrHdrMode":1, + "negotiatedStreamProfile":{"codec":codec}, + "finalizedStreamingFeatures":{"bitDepth":1,"chromaFormat":0} + }}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["codec"], "H265"); + assert_eq!(info["negotiatedStreamProfile"]["colorQuality"], "10bit_420"); + let prepared = crate::streamer::StreamerService::new() + .prepare_embedded( + &json!({"session":info,"runtimeCapabilities":capabilities}), + &settings, + ) + .unwrap(); + assert_eq!(prepared["context"]["settings"]["codec"], "H265"); + assert_eq!(prepared["context"]["settings"]["enableHdr"], true); + } + } + } + + #[test] + fn nested_negotiated_codec_overrides_feature_hints_without_guessing() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + for (codec, expected) in [ + (json!("H264"), json!("H264")), + (json!("avc"), json!("H264")), + (json!("AV1"), json!("AV1")), + (json!("unsupported"), Value::Null), + (json!(2), Value::Null), + (json!(""), Value::Null), + (Value::Null, Value::Null), + ] { + let payload = json!({"session":{ + "sessionId":"nested-codec", "status":2, "sdrHdrMode":1, + "negotiatedStreamProfile":{"codec":codec}, + "sessionRequestData":{"requestedStreamingFeatures":{"codec":2}}, + "finalizedStreamingFeatures":{"codec":2,"bitDepth":1,"chromaFormat":0} + }}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["codec"], expected); + } + } + + #[test] + fn partial_finalized_features_preserve_returned_session_color_fields() { + let base = Url::parse(DEFAULT_STREAMING_BASE).unwrap(); + let mut payload = json!({"session":{ + "sessionId":"color-seat", "status":2, "sdrHdrMode":1, + "sessionRequestData":{ + "clientRequestMonitorSettings":[{"widthInPixels":2560,"heightInPixels":1440}], + "requestedStreamingFeatures":{"codec":2,"bitDepth":1,"chromaFormat":0} + }, + "finalizedStreamingFeatures":{} + }}); + for finalized in [ + json!({}), + json!({"maxBitrateKbps":50000}), + json!({"codec":2}), + ] { + payload["session"]["finalizedStreamingFeatures"] = finalized; + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["codec"], "H265"); + assert_eq!(info["negotiatedStreamProfile"]["colorQuality"], "10bit_420"); + assert_eq!(info["negotiatedStreamProfile"]["enableHdr"], true); + } + payload["session"]["sdrHdrMode"] = json!(0); + payload["session"]["finalizedStreamingFeatures"] = json!({"chromaFormat":1}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["colorQuality"], "10bit_444"); + assert_eq!(info["negotiatedStreamProfile"]["enableHdr"], false); + payload["session"]["finalizedStreamingFeatures"] = json!({"bitDepth":0}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert_eq!(info["negotiatedStreamProfile"]["colorQuality"], "8bit_420"); + payload["session"]["finalizedStreamingFeatures"] = json!({"bitDepth":null}); + let info = session_info(&payload, &base, "auto", "123", "device").unwrap(); + assert!(info["negotiatedStreamProfile"]["colorQuality"].is_null()); + } + + #[test] + fn cloudmatch_chroma_enums_are_not_nvst_chroma_format_ids() { + for (chroma, expected) in [ + (0, json!("10bit_420")), + (1, json!("10bit_444")), + (2, Value::Null), + (3, Value::Null), + ] { + let profile = + negotiated_profile(&json!({}), &json!({"bitDepth":1,"chromaFormat":chroma})); + assert_eq!(profile["colorQuality"], expected); + } + } + + #[test] + fn rejects_untrusted_session_endpoints() { + assert!(trusted_cloudmatch_base("http://prod.cloudmatchbeta.nvidiagrid.net").is_err()); + assert_eq!( + trusted_cloudmatch_base("https://partner.example.com/v2/serverInfo?q=1") + .unwrap() + .as_str(), + "https://partner.example.com/" + ); + assert!(trusted_cloudmatch_base("https://partner.example.com:8443").is_err()); + assert!(trusted_cloudmatch_base("https://user@partner.example.com").is_err()); + assert!(trusted_cloudmatch_base("https://127.0.0.1").is_err()); + assert!(trusted_cloudmatch_base("https://seat.local").is_err()); + for address in [ + "10.0.0.1", + "127.0.0.1", + "169.254.1.1", + "100.64.0.1", + "224.0.0.1", + "255.255.255.255", + "::1", + "fc00::1", + "fe80::1", + "ff02::1", + "::ffff:127.0.0.1", + ] { + assert!(trusted_learned_server_base(address).is_err(), "{address}"); + } + assert!(trusted_learned_server_base("203.0.113.20").is_ok()); + assert!(trusted_learned_server_base("2001:db8::20").is_ok()); + } + + #[test] + fn partner_control_host_and_status_take_precedence_over_early_signaling() { + let zone = trusted_cloudmatch_base("https://zone.partner.example").unwrap(); + let payload = json!({"session":{"sessionId":"seat","status":1, + "sessionControlInfo":{"ip":"control.partner.example","port":443}, + "connectionInfo":[ + {"usage":15,"appLevelProtocol":6,"ip":"media.partner.example","port":322}, + {"usage":14,"appLevelProtocol":6,"ip":"signaling.partner.example","port":322, + "resourcePath":"rtsps://other.partner.example:48322"}]}}); + let info = session_info(&payload, &zone, "zone.partner.example", "123", "device").unwrap(); + assert_eq!(info["streamingBaseUrl"], "https://control.partner.example"); + assert_eq!(info["phase"], "preparing"); + assert_eq!( + info["rtspsEndpoints"], + json!(["rtsps://signaling.partner.example:322"]) + ); + let mut ready = payload; + ready["session"]["status"] = json!(2); + ready["session"]["sessionControlInfo"]["ip"] = json!("forwarded.partner.example"); + let next = session_info(&ready, &zone, "zone.partner.example", "123", "device").unwrap(); + assert_eq!( + next["streamingBaseUrl"], + "https://forwarded.partner.example" + ); + assert_eq!(next["phase"], "ready"); + ready["session"]["sessionControlInfo"]["port"] = json!(8443); + assert_eq!( + session_info(&ready, &zone, "zone.partner.example", "123", "device").unwrap()["streamingBaseUrl"], + "https://zone.partner.example" + ); + } + + #[test] + fn signaling_entries_keep_cloudmatch_order_and_ignore_irrelevant_ports() { + let base = trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap(); + let payload = json!({"session":{"sessionId":"seat","status":2,"connectionInfo":[ + {"usage":14,"appLevelProtocol":5,"ip":"not-native.example","port":322}, + {"usage":15,"appLevelProtocol":6,"ip":"media.example","port":322}, + {"usage":14,"appLevelProtocol":1,"ip":"first.example","port":322}, + {"usage":16,"ip":"second.example","port":48322,"resourcePath":"rtsps://wrong.example:322"}, + {"usage":16,"ip":"bad.example","port":0}, + {"usage":16,"resourcePath":"rtsps://third.example:443","port":443}]}}); + let info = session_info(&payload, &base, "", "123", "device").unwrap(); + assert_eq!( + info["rtspsEndpoints"], + json!([ + "rtsps://first.example:322", + "rtsps://second.example:48322", + "rtsps://third.example:443" + ]) + ); + assert_eq!(info["signalingUrl"], "wss://first.example:322"); + } + + fn network_test_udp_server( + cap: u32, + key: &[u8], + session_id: &str, + ) -> (std::net::SocketAddr, thread::JoinHandle) { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let address = socket.local_addr().unwrap(); + let key = key.to_vec(); + let session_id = session_id.as_bytes().to_vec(); + let worker = thread::spawn(move || { + let mut served = 0_usize; + let mut buffer = vec![0_u8; 4096]; + while let Ok((length, peer)) = socket.recv_from(&mut buffer) { + let Ok(request) = + crate::network_test::NetworkTestMessage::decode(&buffer[..length]) + else { + continue; + }; + let Some(size) = request.payload_size() else { + continue; + }; + if !request.verify(&key).unwrap_or(false) || size > cap { + continue; + } + let mut reply = crate::network_test::NetworkTestMessage::default(); + reply.set_message_type(crate::network_test::MESSAGE_TYPE_MTU_RESPONSE); + reply.set_session_id(session_id.clone()); + reply.set_payload_size(size); + let datagram = reply.encode_response(size as usize); + let _ = socket.send_to(&datagram, peer); + served += 1; + } + served + }); + (address, worker) + } + + #[test] + fn measured_network_test_session_reaches_allocation_and_the_session_context() { + use std::io::{BufRead, BufReader, Read, Write}; + use std::sync::{Arc, Mutex}; + + let key: [u8; 32] = [0x7e; 32]; + let (udp_address, udp_server) = network_test_udp_server(1_200, &key, "nt-1"); + + let allocation = json!({ + "requestStatus":{"requestId":"req-1","serverId":"zone-1","statusCode":0}, + "netTestSession":{ + "sessionId":"nt-1", + "serverId":"zone-1", + "hmacKey":"~".repeat(32), + "connectionInfo":[{ + "ip":udp_address.ip().to_string(), + "port":udp_address.port(), + "appLevelProtocol":5 + }], + "netTestThresholds":{ + "recommendedBandwidthMBPS":50.0, + "requiredBandwidthMBPS":25.0, + "recommendedLatencyMS":40.0, + "requiredLatencyMS":80.0, + "recommendedPacketLossPct":1.0, + "requiredPacketLossPct":3.0 + } + } + }); + let create_reply = json!({ + "requestStatus":{"statusCode":1}, + "session":{ + "sessionId":"seat-1", + "status":2, + "connectionInfo":[{ + "ip":"127.0.0.1","port":49_100,"usage":14,"resourcePath":"/nvst/" + }] + } + }); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); + let server_records = Arc::clone(&recorded); + let server = thread::spawn(move || { + for (status, body) in [ + (200_u16, allocation.to_string()), + (200, create_reply.to_string()), + ] { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&stream); + let mut request_line = String::new(); + reader.read_line(&mut request_line).unwrap(); + let mut length = 0_usize; + let mut line = String::new(); + loop { + line.clear(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + let mut payload = vec![0_u8; length]; + reader.read_exact(&mut payload).unwrap(); + server_records.lock().unwrap().push(( + request_line.trim().to_owned(), + String::from_utf8_lossy(&payload).into_owned(), + )); + write!( + stream, + "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + } + }); + + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123", "networkTest":true}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + + let info = &created["session"]; + assert_eq!(info["sessionId"], "seat-1"); + assert_eq!(info["networkTest"]["sessionId"], "nt-1"); + assert_eq!(info["networkTest"]["status"], "measured"); + assert_eq!(info["networkTestSessionId"], "nt-1"); + assert_eq!(info["networkTest"]["zone"], "127.0.0.1"); + let measured = info["networkTest"]["measuredDatagramBytes"] + .as_u64() + .expect("measured datagram size"); + assert!(measured <= 1_200, "measured {measured}"); + assert!(measured + 32 >= 1_200, "measured {measured}"); + assert!(info["networkTest"]["probes"].as_u64().unwrap_or_default() > 0); + + server.join().unwrap(); + let received = recorded.lock().unwrap().clone(); + assert_eq!(received.len(), 2); + assert!(received[0].0.starts_with("POST /v2/nettestsession")); + assert!(received[0].1.contains("\"clientPlatformName\"")); + assert!(received[1].0.starts_with("POST /v2/session")); + assert!( + received[1].1.contains("\"networkTestSessionId\":\"nt-1\""), + "allocation body carries the measured session: {}", + received[1].1 + ); + + assert!( + udp_server.join().unwrap() > 0, + "the probe never reached the authenticated server" + ); + } + + #[test] + fn a_session_without_a_response_key_refuses_to_probe() { + let (base, server) = session_server( + vec![ + ( + 200, + json!({"netTestSession":{ + "sessionId":"nt-nokey", + "connectionInfo":[{ + "ip":"127.0.0.1","port":49_100,"appLevelProtocol":5 + }], + "netTestThresholds":{ + "recommendedBandwidthMBPS":50.0,"requiredBandwidthMBPS":25.0, + "recommendedLatencyMS":40.0,"requiredLatencyMS":80.0, + "recommendedPacketLossPct":1.0,"requiredPacketLossPct":3.0 + } + }}), + ), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"B","status":2}}), + ), + ], + |_| {}, + ); + let client = Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123", "networkTest":true}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + assert_eq!(created["session"]["networkTest"]["status"], "unavailable"); + assert_eq!( + created["session"]["networkTest"]["code"], + "network-test-key-unavailable" + ); + assert!( + created["session"]["networkTestSessionId"].is_null(), + "no session is advertised without a verified measurement" + ); + let received = server.join().unwrap(); + assert_eq!(received.len(), 2); + assert!( + received[0].starts_with("POST /v2/nettestsession"), + "{}", + received[0] + ); + assert!(received[1].starts_with("POST /v2/session")); + } + + #[test] + fn a_stalled_network_test_allocation_times_out() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let worker = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(12)); + drop(stream); + }); + let client = Client::builder().no_proxy().build().unwrap(); + let error = + try_network_test_session(&client, &base, "token", "device", &json!({}), &json!({})) + .unwrap_err(); + assert_eq!(error.code, "network_error"); + worker.join().unwrap(); + } + + #[test] + fn a_cancelled_request_never_reaches_the_network_test_allocation() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + listener.set_nonblocking(true).unwrap(); + let client = Client::builder().no_proxy().build().unwrap(); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("nettest", "session.create").unwrap(); + requests.cancel("nettest"); + let result = crate::requests::scope(permit.token.clone(), || { + try_network_test_session(&client, &base, "token", "device", &json!({}), &json!({})) + }); + assert_eq!(result.unwrap_err().code, "cancelled"); + assert!( + matches!(listener.accept(), Err(ref error) if error.kind() == std::io::ErrorKind::WouldBlock), + "a cancelled request must not open a connection" + ); + } + + #[test] + fn the_network_test_setting_enables_the_probe() { + use std::io::Write; + use std::sync::{Arc, Mutex}; + + let key: [u8; 32] = [0x7e; 32]; + let (udp_address, udp_server) = network_test_udp_server(1_200, &key, "nt-1"); + let allocation = json!({ + "netTestSession":{ + "sessionId":"nt-1", + "serverId":"zone-1", + "hmacKey":"~".repeat(32), + "connectionInfo":[{ + "ip":udp_address.ip().to_string(), + "port":udp_address.port(), + "appLevelProtocol":5 + }], + "netTestThresholds":{ + "recommendedBandwidthMBPS":50.0,"requiredBandwidthMBPS":25.0, + "recommendedLatencyMS":40.0,"requiredLatencyMS":80.0, + "recommendedPacketLossPct":1.0,"requiredPacketLossPct":3.0 + } + } + }); + let create_reply = json!({ + "requestStatus":{"statusCode":1}, + "session":{"sessionId":"seat-1","status":2} + }); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); + let server_records = Arc::clone(&recorded); + let server = thread::spawn(move || { + for (status, body) in [ + (200_u16, allocation.to_string()), + (200, create_reply.to_string()), + ] { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = std::io::BufReader::new(&stream); + let mut request_line = String::new(); + std::io::BufRead::read_line(&mut reader, &mut request_line).unwrap(); + let mut length = 0_usize; + let mut line = String::new(); + loop { + line.clear(); + assert!(std::io::BufRead::read_line(&mut reader, &mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + server_records + .lock() + .unwrap() + .push(request_line.trim().to_owned()); + let mut payload = vec![0_u8; length]; + std::io::Read::read_exact(&mut reader, &mut payload).unwrap(); + write!( + stream, + "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + } + }); + + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123"}), + &json!({"networkTest":true}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + assert_eq!(created["session"]["networkTest"]["status"], "measured"); + assert_eq!(created["session"]["networkTestSessionId"], "nt-1"); + server.join().unwrap(); + let received = recorded.lock().unwrap().clone(); + assert_eq!(received.len(), 2); + assert!( + received[0].starts_with("POST /v2/nettestsession"), + "{}", + received[0] + ); + assert!(udp_server.join().unwrap() > 0, "the setting never probed"); + } + + #[test] + fn a_zone_without_network_test_keeps_the_previous_allocation_body() { + let (base, server) = session_server( + vec![( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + )], + |_| {}, + ); + let client = Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123"}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + assert_eq!(created["session"]["networkTest"]["status"], "not_requested"); + assert!(created["session"]["networkTestSessionId"].is_null()); + server.join().unwrap(); + } + + #[test] + fn an_oversized_chunked_allocation_response_is_rejected() { + use std::io::Write; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let worker = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = vec![0_u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + let _ = stream.write_all( + b"HTTP/1.1 200 Fixture\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + ); + let chunk = vec![b'x'; 64 * 1024]; + let mut written = 0_usize; + while written <= MAXIMUM_NETWORK_TEST_RESPONSE_BYTES as usize { + let _ = stream.write_all(format!("{:x}\r\n", chunk.len()).as_bytes()); + let _ = stream.write_all(&chunk); + let _ = stream.write_all(b"\r\n"); + written += chunk.len(); + } + }); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let error = + try_network_test_session(&client, &base, "token", "device", &json!({}), &json!({})) + .unwrap_err(); + assert_eq!(error.code, "network-test-rejected", "{}", error.message); + assert!(error.message.contains("size limit"), "{}", error.message); + worker.join().unwrap(); + } + + #[test] + fn the_persisted_network_test_setting_reaches_session_creation() { + let directory = tempfile::tempdir().unwrap(); + let mut store = + crate::settings::SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(store.all()["networkTest"], false, "the setting ships off"); + + let key: [u8; 32] = [0x7e; 32]; + let (udp_address, udp_server) = network_test_udp_server(1_200, &key, "nt-1"); + let allocation = json!({ + "netTestSession":{ + "sessionId":"nt-1", + "serverId":"zone-1", + "hmacKey":"~".repeat(32), + "connectionInfo":[{ + "ip":udp_address.ip().to_string(), + "port":udp_address.port(), + "appLevelProtocol":5 + }], + "netTestThresholds":{ + "recommendedBandwidthMBPS":50.0,"requiredBandwidthMBPS":25.0, + "recommendedLatencyMS":40.0,"requiredLatencyMS":80.0, + "recommendedPacketLossPct":1.0,"requiredPacketLossPct":3.0 + } + } + }); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":1}}), + ), + (200, json!({})), + (200, allocation), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"B","status":2}}), + ), + ], + |_| {}, + ); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + + let created = service + .create_at( + &json!({"appId":"123"}), + &store.all(), + &conflict_auth(), + "device", + || Ok((client.clone(), base.clone())), + ) + .unwrap(); + assert_eq!(created["session"]["networkTest"]["status"], "not_requested"); + service.finish_create("A", false).unwrap(); + + store.set("networkTest", json!(true)).unwrap(); + let restored = + crate::settings::SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(restored.all()["networkTest"], true, "the setting persists"); + + let created = service + .create_at( + &json!({"appId":"123"}), + &restored.all(), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + assert_eq!(created["session"]["networkTest"]["status"], "measured"); + assert_eq!(created["session"]["networkTestSessionId"], "nt-1"); + + let received = server.join().unwrap(); + assert_eq!(received.len(), 4, "{received:?}"); + assert!(received[0].starts_with("POST /v2/session"), "{received:?}"); + let probe = received + .iter() + .position(|line| line.starts_with("POST /v2/nettestsession")) + .expect("the opt-in probe runs"); + assert_eq!( + probe, 2, + "the default-off create must not probe: {received:?}" + ); + assert!(udp_server.join().unwrap() > 0); + } + + #[test] + fn a_path_without_a_confirmed_datagram_is_reported_unmeasured() { + let silent = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let silent_address = silent.local_addr().unwrap(); + let (base, server) = session_server( + vec![ + ( + 200, + json!({"netTestSession":{ + "sessionId":"nt-silent", + "serverId":"zone-1", + "hmacKey":"~".repeat(32), + "connectionInfo":[{ + "ip":silent_address.ip().to_string(), + "port":silent_address.port(), + "appLevelProtocol":5 + }], + "netTestThresholds":{ + "recommendedBandwidthMBPS":50.0,"requiredBandwidthMBPS":25.0, + "recommendedLatencyMS":40.0,"requiredLatencyMS":80.0, + "recommendedPacketLossPct":1.0,"requiredPacketLossPct":3.0 + } + }}), + ), + ( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"A","status":2}}), + ), + ], + |_| {}, + ); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let service = CloudMatchService::new(client.clone()); + let created = service + .create_at( + &json!({"appId":"123", "networkTest":true}), + &json!({}), + &conflict_auth(), + "device", + || Ok((client, base)), + ) + .unwrap(); + let measured = &created["session"]["networkTest"]; + assert_eq!(measured["status"], "unmeasured"); + assert!(measured["measuredDatagramBytes"].is_null()); + assert!(measured["probes"].as_u64().unwrap_or_default() > 0); + assert!( + created["session"]["networkTestSessionId"].is_null(), + "an unconfirmed path must not advertise an unmeasured session" + ); + assert_eq!(created["session"]["sessionId"], "A"); + server.join().unwrap(); + } +} diff --git a/external/opennow/native/opennow-core/src/community.rs b/external/opennow/native/opennow-core/src/community.rs new file mode 100644 index 000000000..f7bf562bd --- /dev/null +++ b/external/opennow/native/opennow-core/src/community.rs @@ -0,0 +1,268 @@ +use reqwest::blocking::Client; +use serde::Deserialize; +use serde_json::{Map, Value, json}; +use std::thread; +use std::time::Duration; +use url::Url; + +const QUEUE_URL: &str = "https://api.printedwaste.com/gfn/queue/"; +const SERVER_MAPPING_URL: &str = + "https://remote.printedwaste.com/config/GFN_SERVERID_TO_REGION_MAPPING"; +const PROVISION_URL: &str = "https://opennow-proxy-production.up.railway.app/api/public/proxy"; +const COMMUNITY_PROXY_HOST: &str = "altaria.proxy.rlwy.net"; +const COMMUNITY_PROXY_PORT: u16 = 51_545; +const MAXIMUM_RESPONSE_BYTES: u64 = 4 * 1024 * 1024; + +pub struct CommunityService { + client: Client, +} + +impl CommunityService { + pub fn new() -> Result { + let client = Client::builder() + .connect_timeout(Duration::from_secs(8)) + .timeout(Duration::from_secs(45)) + .pool_idle_timeout(Duration::from_secs(30)) + .build() + .map_err(|error| error.to_string())?; + Ok(Self { client }) + } + + pub fn queue(&self) -> Result { + let payload = self.get_json(QUEUE_URL, Duration::from_secs(7))?; + normalize_queue(&payload) + } + + pub fn server_mapping(&self) -> Result { + let payload = self.get_json(SERVER_MAPPING_URL, Duration::from_secs(7))?; + normalize_server_mapping(&payload) + } + + pub fn provision_proxy(&self, client_id: &str) -> Result { + if client_id.len() < 16 || client_id.len() > 128 { + return Err("Stable device identity is invalid".to_owned()); + } + let mut response = self.provision_request(client_id)?; + if matches!( + response.status().as_u16(), + 408 | 425 | 429 | 500 | 502 | 503 | 504 + ) { + thread::sleep(Duration::from_millis(1_500)); + response = self.provision_request(client_id)?; + } + if !response.status().is_success() { + return Err(format!( + "Community proxy provision failed ({})", + response.status().as_u16() + )); + } + let length = response.content_length().unwrap_or(0); + if length > MAXIMUM_RESPONSE_BYTES { + return Err("Community proxy response exceeded the size limit".to_owned()); + } + let payload = response + .json::() + .map_err(|_| "Community proxy provision returned invalid JSON".to_owned())?; + let proxy_url = if let Some(value) = payload.proxy_url { + normalize_community_proxy_url(&value)? + } else { + build_community_proxy_url( + payload + .username + .as_deref() + .ok_or_else(|| "Community proxy response omitted credentials".to_owned())?, + payload + .password + .as_deref() + .ok_or_else(|| "Community proxy response omitted credentials".to_owned())?, + )? + }; + Ok(json!({"proxyUrl":proxy_url})) + } + + fn get_json(&self, url: &str, timeout: Duration) -> Result { + let response = self + .client + .get(url) + .timeout(timeout) + .header( + "User-Agent", + format!("opennow/{}", crate::version::APPLICATION_VERSION), + ) + .header("Accept", "application/json") + .send() + .map_err(|_| "Community service is unavailable right now".to_owned())?; + if !response.status().is_success() { + return Err(format!( + "Community service returned HTTP {}", + response.status().as_u16() + )); + } + if response.content_length().unwrap_or(0) > MAXIMUM_RESPONSE_BYTES { + return Err("Community response exceeded the size limit".to_owned()); + } + response + .json::() + .map_err(|_| "Community service returned invalid JSON".to_owned()) + } + + fn provision_request(&self, client_id: &str) -> Result { + self.client + .post(PROVISION_URL) + .timeout(Duration::from_secs(45)) + .header( + "User-Agent", + format!( + "OpenNOW-DesktopClient/{}", + crate::version::APPLICATION_VERSION + ), + ) + .header("Accept", "application/json") + .json(&json!({"clientId":client_id})) + .send() + .map_err(|_| "Community proxy provisioning is unavailable right now".to_owned()) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ProvisionResponse { + proxy_url: Option, + username: Option, + password: Option, +} + +fn data_object<'a>(payload: &'a Value, context: &str) -> Result<&'a Map, String> { + if payload["status"].as_bool() != Some(true) { + return Err(format!("{context} returned status:false")); + } + payload["data"] + .as_object() + .ok_or_else(|| format!("{context} response omitted its data object")) +} + +fn normalize_queue(payload: &Value) -> Result { + let data = data_object(payload, "PrintedWaste queue")?; + let mut output = Map::new(); + for (zone_id, raw) in data { + let Some(zone) = raw.as_object() else { + continue; + }; + let (Some(position), Some(updated), Some(region)) = ( + zone.get("QueuePosition").and_then(Value::as_f64), + zone.get("Last Updated").and_then(Value::as_f64), + zone.get("Region") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()), + ) else { + continue; + }; + if !position.is_finite() || !updated.is_finite() { + continue; + } + let mut normalized = json!({ + "QueuePosition":position, + "Last Updated":updated, + "Region":region + }); + if let Some(eta) = zone + .get("eta") + .and_then(Value::as_f64) + .filter(|value| value.is_finite()) + { + normalized["eta"] = json!(eta); + } + output.insert(zone_id.clone(), normalized); + } + if output.is_empty() { + return Err("PrintedWaste queue returned no valid zones".to_owned()); + } + Ok(Value::Object(output)) +} + +fn normalize_server_mapping(payload: &Value) -> Result { + let data = data_object(payload, "PrintedWaste server mapping")?; + let mut output = Map::new(); + for (zone_id, raw) in data { + let Some(zone) = raw.as_object() else { + continue; + }; + let mut normalized = Map::new(); + for key in ["title", "region"] { + if let Some(value) = zone.get(key).and_then(Value::as_str) { + normalized.insert(key.to_owned(), json!(value)); + } + } + for key in ["is4080Server", "is5080Server", "nuked"] { + if let Some(value) = zone.get(key).and_then(Value::as_bool) { + normalized.insert(key.to_owned(), json!(value)); + } + } + output.insert(zone_id.clone(), Value::Object(normalized)); + } + Ok(Value::Object(output)) +} + +fn build_community_proxy_url(username: &str, password: &str) -> Result { + if username.is_empty() { + return Err("Community proxy response omitted credentials".to_owned()); + } + let mut url = Url::parse(&format!( + "http://{COMMUNITY_PROXY_HOST}:{COMMUNITY_PROXY_PORT}" + )) + .map_err(|_| "Community proxy endpoint is invalid".to_owned())?; + url.set_username(username) + .map_err(|_| "Community proxy username is invalid".to_owned())?; + url.set_password(Some(password)) + .map_err(|_| "Community proxy password is invalid".to_owned())?; + Ok(url.to_string()) +} + +fn normalize_community_proxy_url(raw: &str) -> Result { + let url = Url::parse(raw).map_err(|_| "Community proxy returned an invalid URL".to_owned())?; + if url.scheme() != "http" + || url.host_str() != Some(COMMUNITY_PROXY_HOST) + || url.port() != Some(COMMUNITY_PROXY_PORT) + || url.username().is_empty() + || url.password().is_none() + || (url.path() != "" && url.path() != "/") + || url.query().is_some() + || url.fragment().is_some() + { + return Err("Community proxy returned an invalid URL".to_owned()); + } + Ok(url.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn printed_waste_payloads_are_strictly_normalized() { + let queue = normalize_queue(&json!({"status":true,"data":{ + "zone-a":{"QueuePosition":42,"Last Updated":123,"Region":"EU","eta":9}, + "bad":{"QueuePosition":"many"} + }})) + .unwrap(); + assert_eq!(queue.as_object().unwrap().len(), 1); + assert_eq!(queue["zone-a"]["Region"], "EU"); + let mapping = normalize_server_mapping(&json!({"status":true,"data":{ + "zone-a":{"title":"Paris","region":"EU","nuked":false,"ignored":"x"} + }})) + .unwrap(); + assert_eq!(mapping["zone-a"]["title"], "Paris"); + assert!(mapping["zone-a"].get("ignored").is_none()); + } + + #[test] + fn community_proxy_credentials_are_encoded_and_endpoint_scoped() { + let value = build_community_proxy_url("player name", "p@ss/word").unwrap(); + let parsed = Url::parse(&value).unwrap(); + assert_eq!(parsed.host_str(), Some(COMMUNITY_PROXY_HOST)); + assert_eq!(parsed.port(), Some(COMMUNITY_PROXY_PORT)); + assert_eq!(parsed.username(), "player%20name"); + assert!(normalize_community_proxy_url(&value).is_ok()); + assert!(normalize_community_proxy_url("http://evil.example:51545/u").is_err()); + } +} diff --git a/external/opennow/native/opennow-core/src/console_profiles.rs b/external/opennow/native/opennow-core/src/console_profiles.rs new file mode 100644 index 000000000..d63d9a47b --- /dev/null +++ b/external/opennow/native/opennow-core/src/console_profiles.rs @@ -0,0 +1,506 @@ +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use rand::RngCore as _; +use scrypt::{Params, scrypt}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; +use subtle::ConstantTimeEq as _; + +const PIN_MAX_ATTEMPTS: u32 = 5; +const LOCKOUT_STEPS_MS: [u64; 4] = [30_000, 60_000, 300_000, 900_000]; +const SCRYPT_LOG_N: u8 = 15; +const SCRYPT_R: u32 = 8; +const SCRYPT_P: u32 = 1; +const KEY_LENGTH: usize = 32; +const SALT_LENGTH: usize = 16; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PinRecord { + version: u32, + algorithm: String, + log_n: u8, + r: u32, + p: u32, + key_length: usize, + salt: String, + hash: String, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AttemptState { + failed_attempts: u32, + lockout_level: usize, + locked_until_ms: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Profile { + user_id: String, + pin: Option, + attempts: AttemptState, + updated_at_ms: u64, +} + +#[derive(Default, Serialize, Deserialize)] +struct Document { + version: u32, + profiles: Vec, +} + +pub struct ConsoleProfiles { + path: PathBuf, + mutation: Mutex<()>, + profiles: Mutex>, +} + +impl ConsoleProfiles { + pub fn load(data_dir: &Path) -> Self { + // Electron owns console-profiles.json and wraps it with safeStorage. + // Keep the Qt/Rust format separate so a migration trial never makes + // the shipping rollback client unable to read its profile locks. + let path = data_dir.join("console-profiles-qt.json"); + let profiles = fs::read_to_string(&path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .map(|document| { + document + .profiles + .into_iter() + .filter(|profile| !profile.user_id.is_empty()) + .map(|profile| (profile.user_id.clone(), profile)) + .collect() + }) + .unwrap_or_default(); + Self { + path, + mutation: Mutex::new(()), + profiles: Mutex::new(profiles), + } + } + + pub fn has_pin(&self, user_id: &str) -> bool { + self.profiles + .lock() + .expect("console profiles poisoned") + .get(user_id) + .is_some_and(|profile| profile.pin.is_some()) + } + + pub fn status(&self, user_id: &str) -> Value { + let profiles = self.profiles.lock().expect("console profiles poisoned"); + let Some(profile) = profiles + .get(user_id) + .filter(|profile| profile.pin.is_some()) + else { + return json!({ + "userId":user_id,"hasPin":false,"lockedUntilMs":null, + "remainingAttempts":PIN_MAX_ATTEMPTS + }); + }; + let gate = gate(&profile.attempts, now_ms()); + json!({ + "userId":user_id,"hasPin":true,"lockedUntilMs":gate.locked_until_ms, + "remainingAttempts":gate.remaining_attempts + }) + } + + pub fn set_pin( + &self, + user_id: &str, + pin: &str, + current_pin: Option<&str>, + ) -> Result { + let _mutation = self + .mutation + .lock() + .expect("console profile mutation poisoned"); + if !valid_pin(pin) { + return Ok( + json!({"ok":false,"reason":"invalid_format","hasPin":self.has_pin(user_id)}), + ); + } + let mut next = self + .profiles + .lock() + .expect("console profiles poisoned") + .clone(); + let now = now_ms(); + let profile = next.entry(user_id.to_owned()).or_insert_with(|| Profile { + user_id: user_id.to_owned(), + pin: None, + attempts: AttemptState::default(), + updated_at_ms: 0, + }); + if let Some(existing) = profile.pin.as_ref() { + let gate = gate(&profile.attempts, now); + if !gate.allowed { + return Ok( + json!({"ok":false,"reason":"locked_out","hasPin":true,"lockedUntilMs":gate.locked_until_ms}), + ); + } + let Some(current_pin) = current_pin.filter(|pin| valid_pin(pin)) else { + return Ok(json!({"ok":false,"reason":"invalid_format","hasPin":true})); + }; + if !verify_hash(current_pin, existing) { + register_failure(&mut profile.attempts, now); + self.persist_and_replace(next)?; + return Ok(json!({"ok":false,"reason":"invalid_pin","hasPin":true})); + } + } + profile.pin = Some(hash_pin(pin)?); + profile.attempts = AttemptState::default(); + profile.updated_at_ms = now; + self.persist_and_replace(next)?; + Ok(json!({"ok":true,"hasPin":true})) + } + + pub fn clear_pin(&self, user_id: &str, current_pin: &str) -> Result { + let _mutation = self + .mutation + .lock() + .expect("console profile mutation poisoned"); + let mut next = self + .profiles + .lock() + .expect("console profiles poisoned") + .clone(); + let Some(profile) = next.get_mut(user_id) else { + return Ok(json!({"ok":false,"reason":"no_pin_set","hasPin":false})); + }; + let Some(existing) = profile.pin.as_ref() else { + return Ok(json!({"ok":false,"reason":"no_pin_set","hasPin":false})); + }; + let now = now_ms(); + let gate = gate(&profile.attempts, now); + if !gate.allowed { + return Ok( + json!({"ok":false,"reason":"locked_out","hasPin":true,"lockedUntilMs":gate.locked_until_ms}), + ); + } + if !valid_pin(current_pin) { + return Ok(json!({"ok":false,"reason":"invalid_format","hasPin":true})); + } + if !verify_hash(current_pin, existing) { + register_failure(&mut profile.attempts, now); + self.persist_and_replace(next)?; + return Ok(json!({"ok":false,"reason":"invalid_pin","hasPin":true})); + } + profile.pin = None; + profile.attempts = AttemptState::default(); + profile.updated_at_ms = now; + self.persist_and_replace(next)?; + Ok(json!({"ok":true,"hasPin":false})) + } + + pub fn verify(&self, user_id: &str, pin: &str) -> Result { + let _mutation = self + .mutation + .lock() + .expect("console profile mutation poisoned"); + let mut next = self + .profiles + .lock() + .expect("console profiles poisoned") + .clone(); + let Some(profile) = next.get_mut(user_id) else { + return Ok( + json!({"ok":true,"reason":"no_pin_set","remainingAttempts":PIN_MAX_ATTEMPTS,"lockedUntilMs":null}), + ); + }; + let Some(record) = profile.pin.as_ref() else { + return Ok( + json!({"ok":true,"reason":"no_pin_set","remainingAttempts":PIN_MAX_ATTEMPTS,"lockedUntilMs":null}), + ); + }; + let now = now_ms(); + let current_gate = gate(&profile.attempts, now); + if !current_gate.allowed { + return Ok( + json!({"ok":false,"reason":"locked_out","remainingAttempts":0,"lockedUntilMs":current_gate.locked_until_ms}), + ); + } + if !valid_pin(pin) { + return Ok( + json!({"ok":false,"reason":"invalid_format","remainingAttempts":current_gate.remaining_attempts,"lockedUntilMs":null}), + ); + } + if verify_hash(pin, record) { + profile.attempts = AttemptState::default(); + self.persist_and_replace(next)?; + return Ok( + json!({"ok":true,"remainingAttempts":PIN_MAX_ATTEMPTS,"lockedUntilMs":null}), + ); + } + register_failure(&mut profile.attempts, now); + let next_gate = gate(&profile.attempts, now); + self.persist_and_replace(next)?; + Ok(json!({ + "ok":false, + "reason":if next_gate.allowed { "invalid_pin" } else { "locked_out" }, + "remainingAttempts":next_gate.remaining_attempts, + "lockedUntilMs":next_gate.locked_until_ms + })) + } + + pub fn forget(&self, user_id: &str) -> Result<(), String> { + let _mutation = self + .mutation + .lock() + .expect("console profile mutation poisoned"); + let mut next = self + .profiles + .lock() + .expect("console profiles poisoned") + .clone(); + if next.remove(user_id).is_some() { + self.persist_and_replace(next)?; + } + Ok(()) + } + + pub fn forget_all(&self) -> Result<(), String> { + let _mutation = self + .mutation + .lock() + .expect("console profile mutation poisoned"); + self.persist_and_replace(HashMap::new()) + } + + fn persist_and_replace(&self, profiles: HashMap) -> Result<(), String> { + let mut values = profiles.values().cloned().collect::>(); + values.sort_by(|left, right| left.user_id.cmp(&right.user_id)); + let document = Document { + version: 1, + profiles: values, + }; + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + let temporary = self.path.with_extension("json.tmp"); + let bytes = serde_json::to_vec_pretty(&document).map_err(|error| error.to_string())?; + write_private_file(&temporary, &bytes).map_err(|error| error.to_string())?; + fs::rename(&temporary, &self.path).map_err(|error| error.to_string())?; + *self.profiles.lock().expect("console profiles poisoned") = profiles; + Ok(()) + } +} + +#[derive(Clone, Copy)] +struct Gate { + allowed: bool, + locked_until_ms: Option, + remaining_attempts: u32, +} + +fn gate(state: &AttemptState, now: u64) -> Gate { + let locked = state.locked_until_ms.is_some_and(|until| now < until); + Gate { + allowed: !locked, + locked_until_ms: locked.then_some(state.locked_until_ms).flatten(), + remaining_attempts: if locked { + 0 + } else { + PIN_MAX_ATTEMPTS.saturating_sub(state.failed_attempts) + }, + } +} + +fn register_failure(state: &mut AttemptState, now: u64) { + state.failed_attempts += 1; + if state.failed_attempts < PIN_MAX_ATTEMPTS { + state.locked_until_ms = None; + return; + } + let duration = LOCKOUT_STEPS_MS[state.lockout_level.min(LOCKOUT_STEPS_MS.len() - 1)]; + state.failed_attempts = 0; + state.lockout_level = (state.lockout_level + 1).min(LOCKOUT_STEPS_MS.len()); + state.locked_until_ms = Some(now.saturating_add(duration)); +} + +fn hash_pin(pin: &str) -> Result { + let mut salt = [0_u8; SALT_LENGTH]; + rand::rng().fill_bytes(&mut salt); + let params = Params::new(SCRYPT_LOG_N, SCRYPT_R, SCRYPT_P, KEY_LENGTH) + .map_err(|error| error.to_string())?; + let mut hash = [0_u8; KEY_LENGTH]; + scrypt(pin.as_bytes(), &salt, ¶ms, &mut hash).map_err(|error| error.to_string())?; + Ok(PinRecord { + version: 1, + algorithm: "scrypt".to_owned(), + log_n: SCRYPT_LOG_N, + r: SCRYPT_R, + p: SCRYPT_P, + key_length: KEY_LENGTH, + salt: BASE64.encode(salt), + hash: BASE64.encode(hash), + }) +} + +fn verify_hash(pin: &str, record: &PinRecord) -> bool { + if record.version != 1 + || record.algorithm != "scrypt" + || record.key_length == 0 + || record.key_length > 64 + { + return false; + } + let Ok(salt) = BASE64.decode(&record.salt) else { + return false; + }; + let Ok(expected) = BASE64.decode(&record.hash) else { + return false; + }; + if expected.len() != record.key_length { + return false; + } + let Ok(params) = Params::new(record.log_n, record.r, record.p, record.key_length) else { + return false; + }; + let mut actual = vec![0_u8; record.key_length]; + if scrypt(pin.as_bytes(), &salt, ¶ms, &mut actual).is_err() { + return false; + } + bool::from(actual.ct_eq(&expected)) +} + +fn valid_pin(pin: &str) -> bool { + pin.len() == 4 && pin.bytes().all(|character| character.is_ascii_digit()) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +#[cfg(unix)] +fn write_private_file(path: &Path, data: &[u8]) -> io::Result<()> { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let mut file = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path)?; + file.write_all(data)?; + file.sync_all() +} + +#[cfg(not(unix))] +fn write_private_file(path: &Path, data: &[u8]) -> io::Result<()> { + fs::write(path, data) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temporary_store() -> (ConsoleProfiles, PathBuf) { + let path = std::env::temp_dir().join(format!( + "opennow-profiles-{}-{}-{}", + std::process::id(), + now_ms(), + rand::random::() + )); + (ConsoleProfiles::load(&path), path) + } + + #[test] + fn overlapping_pin_writes_preserve_every_acknowledged_account() { + let directory = tempfile::tempdir().unwrap(); + let profiles = ConsoleProfiles::load(directory.path()); + let barrier = std::sync::Barrier::new(4); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..4) + .map(|index| { + let profiles = &profiles; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + profiles.set_pin(&format!("user-{index}"), "1234", None) + }) + }) + .collect(); + for worker in workers { + assert_eq!(worker.join().unwrap().unwrap()["ok"], true); + } + }); + let loaded = ConsoleProfiles::load(directory.path()); + for index in 0..4 { + assert!(profiles.has_pin(&format!("user-{index}"))); + assert!(loaded.has_pin(&format!("user-{index}"))); + } + } + + #[test] + fn overlapping_pin_failures_all_count_toward_lockout() { + let directory = tempfile::tempdir().unwrap(); + let profiles = ConsoleProfiles::load(directory.path()); + profiles.set_pin("user", "1234", None).unwrap(); + let barrier = std::sync::Barrier::new(PIN_MAX_ATTEMPTS as usize); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..PIN_MAX_ATTEMPTS) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + profiles.verify("user", "0000") + }) + }) + .collect(); + for worker in workers { + assert_eq!(worker.join().unwrap().unwrap()["ok"], false); + } + }); + assert_eq!(profiles.status("user")["remainingAttempts"], 0); + let loaded = ConsoleProfiles::load(directory.path()); + assert_eq!(loaded.status("user")["remainingAttempts"], 0); + } + + #[test] + fn profile_persistence_failure_preserves_committed_state() { + let directory = tempfile::tempdir().unwrap(); + let profiles = ConsoleProfiles::load(directory.path()); + profiles.set_pin("user", "1234", None).unwrap(); + let persisted = fs::read(&profiles.path).unwrap(); + fs::create_dir(profiles.path.with_extension("json.tmp")).unwrap(); + assert!(profiles.clear_pin("user", "1234").is_err()); + assert!(profiles.forget_all().is_err()); + assert!(profiles.has_pin("user")); + assert_eq!(fs::read(&profiles.path).unwrap(), persisted); + } + + #[test] + fn pin_round_trip_and_lockout_are_persisted() { + let (profiles, path) = temporary_store(); + assert_eq!(profiles.set_pin("user", "1234", None).unwrap()["ok"], true); + assert!(profiles.has_pin("user")); + assert_eq!(profiles.verify("user", "1234").unwrap()["ok"], true); + for _ in 0..PIN_MAX_ATTEMPTS { + let _ = profiles.verify("user", "0000").unwrap(); + } + assert_eq!(profiles.status("user")["remainingAttempts"], 0); + let loaded = ConsoleProfiles::load(&path); + assert_eq!(loaded.status("user")["remainingAttempts"], 0); + let _ = fs::remove_dir_all(path); + } + + #[test] + fn malformed_pin_does_not_consume_an_attempt() { + let (profiles, path) = temporary_store(); + profiles.set_pin("user", "1234", None).unwrap(); + let result = profiles.verify("user", "1234").unwrap(); + assert_eq!(result["reason"], "invalid_format"); + assert_eq!(result["remainingAttempts"], PIN_MAX_ATTEMPTS); + let _ = fs::remove_dir_all(path); + } +} diff --git a/external/opennow/native/opennow-core/src/credential_vault.rs b/external/opennow/native/opennow-core/src/credential_vault.rs new file mode 100644 index 000000000..8fe920fad --- /dev/null +++ b/external/opennow/native/opennow-core/src/credential_vault.rs @@ -0,0 +1,1242 @@ +use crate::gfn::AuthSession; +use keyring::Entry; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +#[cfg(any(windows, test))] +mod encrypted_secret_store; +mod json_secret_store; + +const SERVICE_NAME: &str = "app.opennow.auth"; +#[cfg(windows)] +const KEY_SERVICE_NAME: &str = "app.opennow.auth.session-keys"; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SavedIdentity { + user_id: String, + display_name: String, + email: Option, + avatar_url: Option, + membership_tier: String, + provider_code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + provider_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + provider_idp_id: Option, + #[serde(default)] + needs_sign_in: bool, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Metadata { + active_user_id: Option, + accounts: Vec, + #[serde(default)] + suppressed: Vec, + #[serde(default)] + legacy_discarded: Vec, +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacyAuthState { + #[serde(default)] + sessions: Vec, + session: Option, + active_user_id: Option, +} + +pub struct CredentialVault { + metadata_path: PathBuf, + store: Box, + warnings: Mutex>, + suppressed: Mutex>, +} + +trait SecretStore: Send + Sync { + fn get(&self, user_id: &str) -> Result, String>; + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String>; + fn delete(&self, user_id: &str) -> Result<(), String>; + fn remove_all_local( + &self, + _remove_account: &dyn Fn(&str) -> Result<(), String>, + ) -> Result<(), String> { + Ok(()) + } + fn is_plaintext(&self, _user_id: &str) -> bool { + false + } +} + +struct OsSecretStore { + service: &'static str, +} + +impl SecretStore for OsSecretStore { + fn get(&self, user_id: &str) -> Result, String> { + match credential(self.service, user_id)?.get_password() { + Ok(value) => Ok(Some(value)), + Err(keyring::Error::NoEntry) => Ok(None), + Err(_) => Err("OS credential store is unavailable or locked".into()), + } + } + + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + credential(self.service, user_id)? + .set_password(encoded) + .map_err(|_| "OS credential store could not save the session".into()) + } + + fn delete(&self, user_id: &str) -> Result<(), String> { + match credential(self.service, user_id)?.delete_credential() { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(_) => Err("OS credential store could not remove the session".into()), + } + } +} + +impl CredentialVault { + #[cfg(test)] + pub(crate) fn memory(data_dir: PathBuf) -> Self { + let mut vault = Self::new(data_dir); + vault.store = Box::::default(); + vault + } + + #[cfg(test)] + pub(crate) fn without_os_store(data_dir: PathBuf) -> Self { + Self::with_store( + data_dir, + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }), + ) + } + pub fn new(data_dir: PathBuf) -> Self { + #[cfg(windows)] + let store = Box::new(encrypted_secret_store::EncryptedSecretStore::new( + data_dir.join("secure-sessions"), + Box::new(OsSecretStore { + service: KEY_SERVICE_NAME, + }), + Box::new(OsSecretStore { + service: SERVICE_NAME, + }), + )); + #[cfg(not(windows))] + let store = Box::new(OsSecretStore { + service: SERVICE_NAME, + }); + Self::with_store(data_dir, store) + } + + fn with_store(data_dir: PathBuf, store: Box) -> Self { + let store = + json_secret_store::JsonFallbackStore::new(data_dir.join("fallback-sessions"), store); + let mut warnings = std::collections::BTreeMap::new(); + if let Err(error) = store.cleanup_temporary_files() { + warnings.insert("cleanup:local-temporary-files".into(), error); + } + Self { + metadata_path: data_dir.join("accounts.json"), + store: Box::new(store), + warnings: Mutex::new(warnings), + suppressed: Mutex::new(Vec::new()), + } + } + + pub fn save(&self, session: &AuthSession) -> Result<(), String> { + let encoded = serde_json::to_string(session).map_err(|error| error.to_string())?; + let mut metadata = self.read_metadata()?; + self.store.set(&session.user.user_id, &encoded)?; + if self.store.get(&session.user.user_id)?.as_deref() != Some(&encoded) { + return Err("OS credential store verification failed".into()); + } + metadata.suppressed.retain(|id| id != &session.user.user_id); + metadata.active_user_id = Some(session.user.user_id.clone()); + let identity = SavedIdentity { + user_id: session.user.user_id.clone(), + display_name: session.user.display_name.clone(), + email: session.user.email.clone(), + avatar_url: session.user.avatar_url.clone(), + membership_tier: session.user.membership_tier.clone(), + provider_code: session.provider.code.clone(), + provider_name: Some(session.provider.display_name.clone()) + .filter(|name| !name.trim().is_empty()), + provider_idp_id: Some(session.provider.idp_id.clone()) + .filter(|idp_id| !idp_id.is_empty()), + needs_sign_in: false, + }; + if let Some(existing) = metadata + .accounts + .iter_mut() + .find(|item| item.user_id == identity.user_id) + { + *existing = identity; + } else { + metadata.accounts.push(identity); + } + if let Err(error) = self.write_metadata(&metadata) { + self.warn(format!("persistence:{}", session.user.user_id), + "Session credentials were saved, but the account list could not be updated. Restoring this account may fail.".into()); + return Err(format!("Could not save account metadata: {error}")); + } + self.clear_warning(&format!("persistence:{}", session.user.user_id)); + self.suppressed + .lock() + .expect("vault suppression poisoned") + .retain(|id| id != &session.user.user_id); + if let Err(error) = self.remove_session_file(&session.user.user_id) { + self.warn(format!("cleanup:{}", session.user.user_id), error); + } else { + self.clear_warning(&format!("cleanup:{}", session.user.user_id)); + } + self.clear_warning(&format!("migration:{}", session.user.user_id)); + Ok(()) + } + + pub fn migrate_legacy_electron_sessions(&self) -> Result { + let metadata = self.read_metadata()?; + let mut imported = 0; + let mut failures = Vec::new(); + for user_id in &metadata.suppressed { + if let Err(error) = self.remove(user_id) { + failures.push(error); + } + } + for user_id in self.discovered_user_ids()? { + match self.load(&user_id) { + Ok(Some(session)) if self.durable(&session) => imported += 1, + Ok(Some(_)) => failures.push("Legacy credential migration is pending".into()), + Ok(None) => {} + Err(error) => failures.push(error), + } + } + if self.has_discovery_warning() { + failures.push("Some legacy credential sources need recovery".into()); + } + if let Some(active) = metadata.active_user_id.as_deref() { + if !metadata.suppressed.iter().any(|id| id == active) { + self.set_active(active)?; + } + } + let Some(parent) = self.metadata_path.parent() else { + return Ok(0); + }; + let legacy_path = parent.join("auth-state.json"); + let bytes = match read_bounded(&legacy_path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return if failures.is_empty() { + self.clear_warning("migration"); + Ok(imported) + } else { + Err(failures.join("; ")) + }; + } + Err(error) => { + return Err(format!( + "Could not read the Electron account state: {error}" + )); + } + }; + if bytes.len() > 4 * 1024 * 1024 { + return Err("Electron account state exceeds the migration size limit".to_owned()); + } + let legacy = parse_legacy_auth_state(&bytes)?; + if legacy.sessions.is_empty() { + return Ok(0); + } + + for session in &legacy.sessions { + if metadata.suppressed.contains(&session.user.user_id) + || metadata.legacy_discarded.contains(&session.user.user_id) + { + continue; + } + let result = self.store.get(&session.user.user_id).and_then(|stored| { + let selected = match stored { + Some(encoded) => decode_session(&encoded, &session.user.user_id)?, + None => session.clone(), + }; + self.save(&selected) + }); + match result { + Ok(()) => imported += 1, + Err(error) => failures.push(error), + } + } + if !failures.is_empty() { + self.warn( + "migration".into(), + "Legacy credential migration is pending; recoverable source data was retained" + .into(), + ); + return Err(failures.join("; ")); + } + if let Some(active) = metadata.active_user_id.or(legacy.active_user_id) { + if !metadata.suppressed.contains(&active) { + self.set_active(&active)?; + } + } + fs::remove_file(&legacy_path) + .map_err(|_| "Legacy credential cleanup is pending".to_owned())?; + self.clear_warning("migration"); + Ok(imported) + } + + pub fn load_active(&self) -> Result, String> { + let metadata = self.read_metadata()?; + let candidates = self.discovered_user_ids()?; + let mut last_error = self + .has_discovery_warning() + .then(|| "Legacy credential sources need recovery".to_owned()); + for user_id in &candidates { + match self.load(user_id) { + Ok(Some(session)) => { + if metadata.active_user_id.as_deref() != Some(user_id.as_str()) { + if let Err(error) = self.set_active(user_id) { + eprintln!( + "auth: recovered account could not be marked active: {error}" + ); + } + } + return Ok(Some(session)); + } + Ok(None) => {} + Err(error) => last_error = Some(error), + } + } + match last_error { + Some(error) => Err(error), + None => Ok(None), + } + } + + pub fn load(&self, user_id: &str) -> Result, String> { + if self + .suppressed + .lock() + .expect("vault suppression poisoned") + .iter() + .any(|id| id == user_id) + { + return Ok(None); + } + if self + .read_metadata()? + .suppressed + .iter() + .any(|id| id == user_id) + { + return Ok(None); + } + match self.store.get(user_id) { + Ok(Some(encoded)) => { + let session = decode_session(&encoded, user_id)?; + if self.session_file(user_id).exists() { + if let Err(error) = self.save(&session) { + self.warn( + format!("cleanup:{user_id}"), + format!("Legacy credential cleanup is pending: {error}"), + ); + } + } + self.clear_warning(&format!("migration:{user_id}")); + return Ok(Some(session)); + } + Err(error) => { + if let Some(session) = self.load_legacy_session(user_id)? { + self.warn(format!("migration:{user_id}"), "Legacy credential migration is pending; this session is not securely persisted".into()); + return Ok(Some(session)); + } + return Err(error); + } + Ok(None) => {} + } + let session = self.load_legacy_session(user_id)?; + if let Some(session) = &session { + if let Err(error) = self.save(session) { + self.warn( + format!("migration:{user_id}"), + format!("Legacy credential migration is pending: {error}"), + ); + } + } + Ok(session) + } + + pub fn durable(&self, session: &AuthSession) -> bool { + self.store + .get(&session.user.user_id) + .ok() + .flatten() + .is_some_and(|encoded| serde_json::to_string(session).ok().as_deref() == Some(&encoded)) + } + + pub fn persistence_state(&self, session: &AuthSession) -> &'static str { + if !self.durable(session) { + "migration-pending" + } else if self.store.is_plaintext(&session.user.user_id) { + "local-file" + } else { + "secure-store" + } + } + + pub fn failed_save_state(&self, session: &AuthSession) -> &'static str { + if self.store.is_plaintext(&session.user.user_id) { + "local-file" + } else { + "memory-only" + } + } + + pub fn warnings(&self) -> Vec { + self.warnings + .lock() + .expect("vault warnings poisoned") + .values() + .cloned() + .collect() + } + + fn warn(&self, key: String, warning: String) { + let mut warnings = self.warnings.lock().expect("vault warnings poisoned"); + if warnings.len() < 16 || warnings.contains_key(&key) { + warnings.insert(key, warning); + } + } + + fn clear_warning(&self, key: &str) { + self.warnings + .lock() + .expect("vault warnings poisoned") + .remove(key); + } + + fn has_discovery_warning(&self) -> bool { + self.warnings + .lock() + .expect("vault warnings poisoned") + .keys() + .any(|key| key.starts_with("source:")) + } + + pub fn list(&self) -> Result, String> { + self.read_metadata()? + .accounts + .into_iter() + .map(|identity| serde_json::to_value(identity).map_err(|error| error.to_string())) + .collect() + } + + pub fn mark_needs_sign_in(&self, user_id: &str) -> Result<(), String> { + let mut metadata = self.read_metadata()?; + let Some(identity) = metadata + .accounts + .iter_mut() + .find(|identity| identity.user_id == user_id) + else { + return Ok(()); + }; + if identity.needs_sign_in { + return Ok(()); + } + identity.needs_sign_in = true; + self.write_metadata(&metadata) + .map_err(|error| error.to_string()) + } + + pub fn set_active(&self, user_id: &str) -> Result<(), String> { + let mut metadata = self.read_metadata()?; + if !metadata + .accounts + .iter() + .any(|identity| identity.user_id == user_id) + { + return Err("Saved account not found".to_owned()); + } + metadata.active_user_id = Some(user_id.to_owned()); + self.write_metadata(&metadata) + .map_err(|error| error.to_string()) + } + + pub fn remove(&self, user_id: &str) -> Result<(), String> { + self.suppressed + .lock() + .expect("vault suppression poisoned") + .push(user_id.to_owned()); + let suppression = self.read_metadata().and_then(|mut metadata| { + if !metadata.suppressed.iter().any(|id| id == user_id) { + metadata.suppressed.push(user_id.to_owned()); + } + if !metadata.legacy_discarded.iter().any(|id| id == user_id) { + metadata.legacy_discarded.push(user_id.to_owned()); + } + metadata.accounts.retain(|item| item.user_id != user_id); + if metadata.active_user_id.as_deref() == Some(user_id) { + metadata.active_user_id = + metadata.accounts.first().map(|item| item.user_id.clone()); + } + self.write_metadata(&metadata) + .map_err(|error| error.to_string()) + }); + let secret = self.store.delete(user_id); + let legacy = self.remove_session_file(user_id); + let result = suppression.and(secret).and(legacy); + self.clear_warning(&format!("persistence:{user_id}")); + if result.is_err() { + self.warn( + format!("cleanup:{user_id}"), + "Removed account credential cleanup is pending".into(), + ); + } else { + self.clear_warning(&format!("cleanup:{user_id}")); + self.clear_warning(&format!("migration:{user_id}")); + } + result + } + + pub fn remove_all(&self) -> Result<(), String> { + let mut failures = Vec::new(); + let discovered = self.read_metadata().and_then(|metadata| { + let mut user_ids = self.discovered_user_ids()?; + user_ids.extend(metadata.suppressed); + Ok(user_ids) + }); + let user_ids = match discovered { + Ok(ids) => ids, + Err(error) => { + failures.push(error); + Vec::new() + } + }; + for user_id in user_ids { + if let Err(error) = self.remove(&user_id) { + failures.push(error); + } + } + if let Err(error) = self.store.remove_all_local(&|user_id| self.remove(user_id)) { + failures.push(error); + } + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } + } + + fn data_dir(&self) -> &Path { + self.metadata_path + .parent() + .unwrap_or_else(|| Path::new(".")) + } + + fn sessions_dir(&self) -> PathBuf { + self.data_dir().join("sessions") + } + + fn session_file(&self, user_id: &str) -> PathBuf { + self.sessions_dir() + .join(format!("{}.json", sanitize_user_id(user_id))) + } + + fn remove_session_file(&self, user_id: &str) -> Result<(), String> { + if self.load_session_file(user_id)?.is_none() { + return Ok(()); + } + fs::remove_file(self.session_file(user_id)) + .map_err(|_| "Legacy credential cleanup is pending".into()) + } + + fn load_session_file(&self, user_id: &str) -> Result, String> { + let bytes = match read_bounded(&self.session_file(user_id)) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(format!("Could not read the local session store: {error}")); + } + }; + let encoded = String::from_utf8(bytes) + .map_err(|error| format!("Saved session is invalid: {error}"))?; + Ok(Some(decode_session(&encoded, user_id)?)) + } + + fn load_legacy_session(&self, user_id: &str) -> Result, String> { + if self + .read_metadata()? + .legacy_discarded + .iter() + .any(|id| id == user_id) + { + return Ok(None); + } + if let Some(session) = self.load_session_file(user_id)? { + return Ok(Some(session)); + } + match read_bounded(&self.data_dir().join("auth-state.json")) { + Ok(bytes) => Ok(parse_legacy_auth_state(&bytes)? + .sessions + .into_iter() + .find(|session| session.user.user_id == user_id)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(_) => Err("Legacy account state could not be read".into()), + } + } + + fn discovered_user_ids(&self) -> Result, String> { + let metadata = self.read_metadata()?; + let mut ids = candidate_user_ids(&metadata); + self.warnings + .lock() + .expect("vault warnings poisoned") + .retain(|key, _| !key.starts_with("source:")); + match read_bounded(&self.data_dir().join("auth-state.json")) { + Ok(bytes) => match parse_legacy_auth_state(&bytes) { + Ok(legacy) => { + for session in legacy.sessions { + if !ids.contains(&session.user.user_id) { + ids.push(session.user.user_id); + } + } + } + Err(error) => self.warn("source:legacy".into(), error), + }, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => self.warn( + "source:legacy".into(), + "Legacy account state could not be read".into(), + ), + } + ids.retain(|id| !metadata.suppressed.contains(id)); + let entries = match fs::read_dir(self.sessions_dir()) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(ids), + Err(_) => { + self.warn( + "source:directory".into(), + "Legacy session directory could not be read".into(), + ); + return Ok(ids); + } + }; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(_) => { + self.warn( + "source:directory".into(), + "Legacy session entry could not be read".into(), + ); + continue; + } + }; + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let session = match read_bounded(&path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + Some(session) => session, + None => { + self.warn(format!("source:{}", path.display()), "A legacy credential file is invalid or unavailable; it was retained for recovery".into()); + continue; + } + }; + if session.user.user_id.is_empty() || self.session_file(&session.user.user_id) != path { + self.warn( + format!("source:{}", path.display()), + "Legacy credential filename does not match its identity".into(), + ); + continue; + } + if !ids.iter().any(|id| id == &session.user.user_id) { + ids.push(session.user.user_id); + } + } + ids.retain(|id| !metadata.suppressed.contains(id)); + Ok(ids) + } + + fn read_metadata(&self) -> Result { + match read_bounded(&self.metadata_path) { + Ok(bytes) => serde_json::from_slice(&bytes) + .map_err(|_| "Account metadata is invalid; recovery is required".into()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Metadata::default()), + Err(_) => Err("Account metadata could not be read; recovery is required".into()), + } + } + + fn write_metadata(&self, metadata: &Metadata) -> io::Result<()> { + let parent = self + .metadata_path + .parent() + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let temporary = self.metadata_path.with_extension("json.tmp"); + let data = serde_json::to_vec_pretty(metadata).map_err(io::Error::other)?; + write_private_file(&temporary, &data)?; + fs::rename(temporary, &self.metadata_path)?; + #[cfg(unix)] + fs::File::open(parent)?.sync_all()?; + Ok(()) + } +} + +#[cfg(test)] +#[derive(Default)] +struct MemorySecretStore { + values: Mutex>, + unavailable: bool, + fail_user: Option, +} + +#[cfg(test)] +impl SecretStore for MemorySecretStore { + fn get(&self, user_id: &str) -> Result, String> { + if self.unavailable || self.fail_user.as_deref() == Some(user_id) { + return Err("locked".into()); + } + Ok(self.values.lock().unwrap().get(user_id).cloned()) + } + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + if self.unavailable || self.fail_user.as_deref() == Some(user_id) { + return Err("locked".into()); + } + self.values + .lock() + .unwrap() + .insert(user_id.into(), encoded.into()); + Ok(()) + } + fn delete(&self, user_id: &str) -> Result<(), String> { + if self.unavailable || self.fail_user.as_deref() == Some(user_id) { + return Err("locked".into()); + } + self.values.lock().unwrap().remove(user_id); + Ok(()) + } +} + +fn read_bounded(path: &Path) -> io::Result> { + use std::io::Read as _; + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(4 * 1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 4 * 1024 * 1024 { + return Err(io::Error::other("Account document exceeds size limit")); + } + Ok(bytes) +} + +fn sanitize_user_id(user_id: &str) -> String { + user_id + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' || character == '_' { + character + } else { + '_' + } + }) + .collect() +} + +fn decode_session(encoded: &str, user_id: &str) -> Result { + if encoded.len() > 4 * 1024 * 1024 || user_id.trim().is_empty() { + return Err("Saved credential exceeds size or identity limits".into()); + } + let session = serde_json::from_str::(encoded) + .map_err(|_| "Saved session is invalid".to_owned())?; + if session.user.user_id != user_id { + return Err("Saved credential identity does not match account metadata".to_owned()); + } + Ok(session) +} + +fn candidate_user_ids(metadata: &Metadata) -> Vec { + let mut ids = Vec::new(); + if let Some(active) = metadata + .active_user_id + .as_deref() + .filter(|user_id| !user_id.is_empty()) + { + ids.push(active.to_owned()); + } + for account in &metadata.accounts { + if account.user_id.is_empty() { + continue; + } + if !ids.iter().any(|id| id == &account.user_id) { + ids.push(account.user_id.clone()); + } + } + ids +} + +fn parse_legacy_auth_state(bytes: &[u8]) -> Result { + let mut legacy = serde_json::from_slice::(bytes) + .map_err(|_| "Legacy account state is invalid".to_owned())?; + if let Some(session) = legacy.session.take() { + legacy.sessions.push(session); + } + if legacy + .sessions + .iter() + .any(|session| session.user.user_id.trim().is_empty()) + { + return Err("Legacy account identity is invalid".into()); + } + legacy + .sessions + .sort_by(|left, right| left.user.user_id.cmp(&right.user.user_id)); + for pair in legacy.sessions.windows(2) { + if pair[0].user.user_id == pair[1].user.user_id + && serde_json::to_value(&pair[0]).ok() != serde_json::to_value(&pair[1]).ok() + { + return Err( + "Legacy account state contains conflicting grants; recovery is required".into(), + ); + } + } + legacy + .sessions + .dedup_by(|left, right| left.user.user_id == right.user.user_id); + Ok(legacy) +} + +fn credential(service: &str, user_id: &str) -> Result { + Entry::new(service, &format!("session:{user_id}")) + .map_err(|error| format!("OS credential store is unavailable: {error}")) +} + +#[cfg(unix)] +fn write_private_file(path: &Path, data: &[u8]) -> io::Result<()> { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let mut file = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path)?; + file.write_all(data)?; + file.sync_all() +} + +#[cfg(not(unix))] +fn write_private_file(path: &Path, data: &[u8]) -> io::Result<()> { + fs::write(path, data) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn unavailable_vault_persists_and_restores_json_after_restart() { + let directory = tempfile::tempdir().unwrap(); + let unavailable = || { + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }) as Box + }; + let vault = CredentialVault::with_store(directory.path().into(), unavailable()); + let session = sample_session("fallback-user"); + vault.save(&session).unwrap(); + drop(vault); + let restored = CredentialVault::with_store(directory.path().into(), unavailable()); + assert_eq!( + restored.load_active().unwrap().unwrap().user.user_id, + "fallback-user" + ); + assert!(restored.durable(&session)); + assert!(restored.remove("fallback-user").is_err()); + drop(restored); + assert!( + CredentialVault::with_store(directory.path().into(), unavailable()) + .load_active() + .unwrap() + .is_none() + ); + } + + #[test] + fn secure_restore_survives_failed_legacy_cleanup_and_retries() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + let session = sample_session("selected"); + vault.save(&session).unwrap(); + fs::create_dir(directory.path().join("sessions")).unwrap(); + let mut legacy = session.clone(); + legacy.tokens.access_token = "superseded-legacy-grant".into(); + let legacy_bytes = serde_json::to_vec(&legacy).unwrap(); + fs::write(vault.session_file("selected"), &legacy_bytes).unwrap(); + let blocked_metadata = directory.path().join("accounts.json.tmp"); + fs::create_dir(&blocked_metadata).unwrap(); + + let restored = vault.load_active().unwrap().unwrap(); + assert_eq!(restored.tokens.access_token, session.tokens.access_token); + assert_eq!( + fs::read(vault.session_file("selected")).unwrap(), + legacy_bytes + ); + assert!(!vault.warnings().is_empty()); + + fs::remove_dir(blocked_metadata).unwrap(); + let restored = vault.load_active().unwrap().unwrap(); + assert_eq!(restored.tokens.access_token, session.tokens.access_token); + assert!(!vault.session_file("selected").exists()); + assert!(vault.warnings().is_empty()); + } + + #[test] + fn migration_preserves_selection_and_conflicting_sources() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + vault.save(&sample_session("selected")).unwrap(); + fs::create_dir(directory.path().join("sessions")).unwrap(); + fs::write( + vault.session_file("later"), + serde_json::to_vec(&sample_session("later")).unwrap(), + ) + .unwrap(); + vault.migrate_legacy_electron_sessions().unwrap(); + assert_eq!( + vault.load_active().unwrap().unwrap().user.user_id, + "selected" + ); + let mut other = sample_session("selected"); + other.tokens.access_token = "other-grant".into(); + let bytes = + serde_json::to_vec(&json!({"sessions":[sample_session("selected")],"session":other})) + .unwrap(); + let source = directory.path().join("auth-state.json"); + fs::write(&source, &bytes).unwrap(); + assert!(vault.migrate_legacy_electron_sessions().is_err()); + assert_eq!(fs::read(source).unwrap(), bytes); + } + + #[test] + fn damaged_legacy_sources_do_not_strand_other_restorable_accounts() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + vault.save(&sample_session("selected")).unwrap(); + fs::create_dir(directory.path().join("sessions")).unwrap(); + fs::write(vault.session_file("damaged"), "invalid credential").unwrap(); + fs::write( + vault.session_file("recoverable"), + serde_json::to_vec(&sample_session("recoverable")).unwrap(), + ) + .unwrap(); + assert!(vault.migrate_legacy_electron_sessions().is_err()); + assert_eq!( + vault.load_active().unwrap().unwrap().user.user_id, + "selected" + ); + assert!(vault.durable(&sample_session("recoverable"))); + assert!(vault.session_file("damaged").exists()); + fs::write( + directory.path().join("auth-state.json"), + "damaged legacy document", + ) + .unwrap(); + assert_eq!( + vault.load_active().unwrap().unwrap().user.user_id, + "selected" + ); + assert!(vault.has_discovery_warning()); + } + + #[test] + fn unavailable_store_retains_legacy_data_and_new_saves_write_no_credentials() { + let directory = tempfile::tempdir().unwrap(); + let mut vault = CredentialVault::memory(directory.path().into()); + vault.store = Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }); + let session = sample_session("legacy"); + let legacy = serde_json::to_vec(&json!({"sessions":[session]})).unwrap(); + let source = directory.path().join("auth-state.json"); + fs::write(&source, &legacy).unwrap(); + assert!(vault.migrate_legacy_electron_sessions().is_err()); + assert_eq!(vault.load_active().unwrap().unwrap().user.user_id, "legacy"); + assert_eq!(fs::read(&source).unwrap(), legacy); + assert!(vault.save(&sample_session("new")).is_err()); + assert!(!directory.path().join("sessions").exists()); + assert!(!directory.path().join("accounts.json").exists()); + vault.store = Box::::default(); + assert!(vault.migrate_legacy_electron_sessions().unwrap() > 0); + assert!(!source.exists()); + assert!(vault.warnings().is_empty()); + assert_eq!(vault.load_active().unwrap().unwrap().user.user_id, "legacy"); + assert_eq!(vault.migrate_legacy_electron_sessions().unwrap(), 1); + } + + #[test] + fn partial_migration_keeps_source_and_does_not_skip_later_accounts() { + let directory = tempfile::tempdir().unwrap(); + let mut vault = CredentialVault::memory(directory.path().into()); + vault.store = Box::new(MemorySecretStore { + fail_user: Some("b".into()), + ..Default::default() + }); + vault.save(&sample_session("existing")).unwrap(); + let source = directory.path().join("auth-state.json"); + let bytes = serde_json::to_vec( + &json!({"sessions":[sample_session("a"),sample_session("b"),sample_session("c")]}), + ) + .unwrap(); + fs::write(&source, &bytes).unwrap(); + assert!(vault.migrate_legacy_electron_sessions().is_err()); + assert!(vault.durable(&sample_session("a"))); + assert!(vault.durable(&sample_session("c"))); + assert_eq!(fs::read(source).unwrap(), bytes); + } + + #[test] + fn valid_secure_entry_wins_and_metadata_failure_preserves_source() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + let session = sample_session("user"); + vault.save(&session).unwrap(); + fs::create_dir(directory.path().join("sessions")).unwrap(); + let mut stale = session.clone(); + stale.tokens.access_token = "obsolete".into(); + let source = vault.session_file("user"); + fs::write(&source, serde_json::to_vec(&stale).unwrap()).unwrap(); + assert_eq!( + vault.load("user").unwrap().unwrap().tokens.access_token, + "access" + ); + assert!(!source.exists()); + fs::write(&source, serde_json::to_vec(&stale).unwrap()).unwrap(); + fs::write(directory.path().join("accounts.json"), "corrupt metadata").unwrap(); + assert!(vault.save(&session).is_err()); + assert!(vault.list().is_err()); + assert!(source.exists()); + assert_eq!( + fs::read_to_string(directory.path().join("accounts.json")).unwrap(), + "corrupt metadata" + ); + } + + #[test] + fn failed_deletion_is_suppressed_and_old_legacy_grants_never_return() { + let directory = tempfile::tempdir().unwrap(); + let mut vault = CredentialVault::memory(directory.path().into()); + let session = sample_session("user"); + vault.save(&session).unwrap(); + fs::write( + directory.path().join("auth-state.json"), + serde_json::to_vec(&json!({"sessions":[session]})).unwrap(), + ) + .unwrap(); + vault.store = Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }); + assert!(vault.remove("user").is_err()); + assert!(vault.load_active().unwrap().is_none()); + assert!(!vault.warnings().is_empty()); + vault.store = Box::::default(); + vault.remove("user").unwrap(); + assert!(vault.warnings().is_empty()); + let mut restarted = CredentialVault::memory(directory.path().into()); + assert!(restarted.load("user").unwrap().is_none()); + restarted.save(&sample_session("user")).unwrap(); + restarted.store = Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }); + assert!(restarted.load("user").is_err()); + } + + #[test] + fn mismatched_legacy_identity_is_not_imported_or_deleted() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + fs::create_dir(directory.path().join("sessions")).unwrap(); + let source = vault.session_file("other-user"); + fs::write( + &source, + serde_json::to_vec(&sample_session("real-user")).unwrap(), + ) + .unwrap(); + assert!(vault.migrate_legacy_electron_sessions().is_err()); + assert!(source.exists()); + assert!(vault.load("other-user").is_err()); + } + + pub(super) fn sample_identity(user_id: &str) -> SavedIdentity { + SavedIdentity { + user_id: user_id.to_owned(), + display_name: user_id.to_owned(), + email: None, + avatar_url: None, + membership_tier: "FREE".to_owned(), + provider_code: "NVIDIA".to_owned(), + provider_name: None, + provider_idp_id: None, + needs_sign_in: false, + } + } + + #[test] + fn expired_flag_persists_until_the_account_signs_in_again() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + vault.save(&sample_session("expired-user")).unwrap(); + let saved = &vault.list().unwrap()[0]; + assert_eq!(saved["needsSignIn"], false); + assert_eq!(saved["providerName"], "NVIDIA"); + assert_eq!(saved["providerIdpId"], "idp"); + vault.mark_needs_sign_in("expired-user").unwrap(); + vault.mark_needs_sign_in("unknown-user").unwrap(); + let reopened = CredentialVault::memory(directory.path().into()); + assert_eq!(reopened.list().unwrap()[0]["needsSignIn"], true); + vault.save(&sample_session("expired-user")).unwrap(); + assert_eq!(vault.list().unwrap()[0]["needsSignIn"], false); + } + + #[test] + fn metadata_saved_before_provider_names_still_loads() { + let directory = tempfile::tempdir().unwrap(); + fs::write( + directory.path().join("accounts.json"), + br#"{"activeUserId":"old","accounts":[{"userId":"old","displayName":"Old","email":null,"avatarUrl":null,"membershipTier":"ULTIMATE","providerCode":"BPC"}]}"#, + ) + .unwrap(); + let vault = CredentialVault::memory(directory.path().into()); + let account = &vault.list().unwrap()[0]; + assert_eq!(account["providerCode"], "BPC"); + assert!(account.get("providerName").is_none()); + assert_eq!(account["needsSignIn"], false); + } + + #[test] + fn missing_metadata_has_no_active_session() { + let path = std::env::temp_dir().join(format!("opennow-vault-{}", std::process::id())); + let vault = CredentialVault::memory(path.clone()); + assert!(vault.load_active().unwrap().is_none()); + let _ = fs::remove_dir_all(path); + } + + pub(super) fn sample_session(user_id: &str) -> AuthSession { + serde_json::from_value(serde_json::json!({ + "provider": { + "idpId": "idp", "code": "NVIDIA", "displayName": "NVIDIA", + "streamingServiceUrl": "https://example.invalid/", "priority": 0 + }, + "tokens": { + "accessToken": "access", "refreshToken": "refresh", + "expiresAt": 9_999_999_999_999u64, "authClientId": "client" + }, + "user": { + "userId": user_id, "displayName": "Player", + "membershipTier": "free" + } + })) + .unwrap() + } + + #[test] + fn save_and_load_roundtrip_uses_secure_store_without_plaintext_mirror() { + let path = std::env::temp_dir().join(format!( + "opennow-vault-file-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&path).unwrap(); + let vault = CredentialVault::memory(path.clone()); + vault.save(&sample_session("user-file")).unwrap(); + let loaded = vault + .load_active() + .unwrap() + .expect("secure session should restore"); + assert_eq!(loaded.user.user_id, "user-file"); + assert!(!path.join("sessions").join("user-file.json").exists()); + assert!( + !fs::read_to_string(path.join("accounts.json")) + .unwrap() + .contains("accessToken") + ); + let _ = fs::remove_dir_all(path); + } + + #[test] + fn load_active_falls_back_to_first_account_when_active_user_id_is_missing() { + let path = std::env::temp_dir().join(format!( + "opennow-vault-fallback-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&path).unwrap(); + let vault = CredentialVault::memory(path.clone()); + vault + .write_metadata(&Metadata { + active_user_id: None, + accounts: vec![sample_identity("user-a"), sample_identity("user-b")], + suppressed: Vec::new(), + legacy_discarded: Vec::new(), + }) + .unwrap(); + let metadata = vault.read_metadata().unwrap(); + assert_eq!(metadata.active_user_id, None); + assert_eq!(metadata.accounts.len(), 2); + assert_eq!( + candidate_user_ids(&metadata), + vec!["user-a".to_string(), "user-b".to_string()] + ); + let _ = fs::remove_dir_all(path); + } + + #[test] + fn parses_current_and_legacy_electron_account_documents() { + let session = json!({ + "provider": { + "idpId": "idp", "code": "NVIDIA", "displayName": "NVIDIA", + "streamingServiceUrl": "https://example.invalid/", "priority": 0 + }, + "tokens": { + "accessToken": "access", "refreshToken": "refresh", + "expiresAt": 1234, "authClientId": "client" + }, + "user": { + "userId": "user-1", "displayName": "Player", + "membershipTier": "free" + } + }); + let current = serde_json::to_vec(&json!({ + "sessions": [session.clone(), session.clone()], + "activeUserId": "user-1" + })) + .unwrap(); + let parsed = parse_legacy_auth_state(¤t).unwrap(); + assert_eq!(parsed.sessions.len(), 1); + assert_eq!(parsed.active_user_id.as_deref(), Some("user-1")); + + let legacy = serde_json::to_vec(&json!({"session": session})).unwrap(); + assert_eq!(parse_legacy_auth_state(&legacy).unwrap().sessions.len(), 1); + } +} diff --git a/external/opennow/native/opennow-core/src/credential_vault/encrypted_secret_store.rs b/external/opennow/native/opennow-core/src/credential_vault/encrypted_secret_store.rs new file mode 100644 index 000000000..2253118b4 --- /dev/null +++ b/external/opennow/native/opennow-core/src/credential_vault/encrypted_secret_store.rs @@ -0,0 +1,451 @@ +use super::SecretStore; +use aes_gcm::aead::{Aead, Payload}; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::{self, Read as _, Write as _}; +use std::path::{Path, PathBuf}; + +const MAX_SESSION_BYTES: usize = 4 * 1024 * 1024; +const NONCE_BYTES: usize = 12; +const TAG_BYTES: usize = 16; + +pub(super) struct EncryptedSecretStore { + root: PathBuf, + keys: Box, + legacy: Box, +} + +impl EncryptedSecretStore { + pub(super) fn new( + root: PathBuf, + keys: Box, + legacy: Box, + ) -> Self { + Self { root, keys, legacy } + } + + fn path(&self, user_id: &str) -> PathBuf { + let digest = Sha256::digest(user_id.as_bytes()); + self.root.join(format!("{digest:x}.bin")) + } + + fn read_key(&self, user_id: &str) -> Result, String> { + let Some(encoded) = self.keys.get(user_id)? else { + return Ok(None); + }; + if encoded.len() != 44 { + return Err("Saved session encryption key is invalid".into()); + } + let bytes = STANDARD + .decode(encoded) + .map_err(|_| "Saved session encryption key is invalid")?; + let key = bytes + .try_into() + .map_err(|_| "Saved session encryption key is invalid")?; + Ok(Some(key)) + } + + fn write_encrypted(&self, path: &Path, user_id: &str, encoded: &str) -> Result<(), String> { + let key = match self.read_key(user_id)? { + Some(key) => key, + None => { + let key = rand::random::<[u8; 32]>(); + if path.exists() { + return Err("Saved session encryption key is missing".into()); + } + self.keys.set(user_id, &STANDARD.encode(key))?; + key + } + }; + let nonce = rand::random::<[u8; NONCE_BYTES]>(); + let cipher = Aes256Gcm::new_from_slice(&key) + .map_err(|_| "Saved session encryption key is invalid")?; + let ciphertext = cipher + .encrypt( + Nonce::from_slice(&nonce), + Payload { + msg: encoded.as_bytes(), + aad: user_id.as_bytes(), + }, + ) + .map_err(|_| "Could not encrypt saved session")?; + fs::create_dir_all(&self.root).map_err(|_| "Could not create secure session directory")?; + let temporary = self + .root + .join(format!("{:032x}.tmp", rand::random::())); + let result = (|| -> io::Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + file.write_all(&nonce)?; + file.write_all(&ciphertext)?; + file.sync_all()?; + fs::rename(&temporary, path)?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result.map_err(|_| "Could not write encrypted session".into()) + } +} + +impl SecretStore for EncryptedSecretStore { + fn get(&self, user_id: &str) -> Result, String> { + let file = match fs::File::open(self.path(user_id)) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let legacy = self.legacy.get(user_id)?; + if let Some(encoded) = &legacy { + let _ = self.set(user_id, encoded); + } + return Ok(legacy); + } + Err(_) => return Err("Encrypted session could not be read".into()), + }; + let mut bytes = Vec::new(); + file.take((MAX_SESSION_BYTES + NONCE_BYTES + TAG_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "Encrypted session could not be read")?; + if bytes.len() < NONCE_BYTES + TAG_BYTES + || bytes.len() > MAX_SESSION_BYTES + NONCE_BYTES + TAG_BYTES + { + return Err("Encrypted session exceeds size or format limits".into()); + } + let key = self + .read_key(user_id)? + .ok_or("Saved session encryption key is missing")?; + let cipher = Aes256Gcm::new_from_slice(&key) + .map_err(|_| "Saved session encryption key is invalid")?; + let plaintext = cipher + .decrypt( + Nonce::from_slice(&bytes[..NONCE_BYTES]), + Payload { + msg: &bytes[NONCE_BYTES..], + aad: user_id.as_bytes(), + }, + ) + .map_err(|_| "Encrypted session authentication failed")?; + let encoded = + String::from_utf8(plaintext).map_err(|_| "Encrypted session contains invalid text")?; + let _ = self.legacy.delete(user_id); + Ok(Some(encoded)) + } + + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + if encoded.len() > MAX_SESSION_BYTES || user_id.trim().is_empty() { + return Err("Saved credential exceeds size or identity limits".into()); + } + self.write_encrypted(&self.path(user_id), user_id, encoded) + } + + fn delete(&self, user_id: &str) -> Result<(), String> { + let path = self.path(user_id); + let file = match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(_) => Err("Encrypted session could not be removed".into()), + }; + let key = self.keys.delete(user_id); + let legacy = self.legacy.delete(user_id); + file.and(key).and(legacy) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + #[derive(Clone, Default)] + struct TestStore { + entries: Arc>>, + limit: Option, + fail_set: bool, + } + + impl SecretStore for TestStore { + fn get(&self, user_id: &str) -> Result, String> { + Ok(self.entries.lock().unwrap().get(user_id).cloned()) + } + + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + if self.fail_set + || self + .limit + .is_some_and(|limit| encoded.encode_utf16().count() * 2 > limit) + { + return Err("credential write failed".into()); + } + self.entries + .lock() + .unwrap() + .insert(user_id.into(), encoded.into()); + Ok(()) + } + + fn delete(&self, user_id: &str) -> Result<(), String> { + self.entries.lock().unwrap().remove(user_id); + Ok(()) + } + } + + fn store(root: &Path, keys: TestStore, legacy: TestStore) -> EncryptedSecretStore { + EncryptedSecretStore::new(root.into(), Box::new(keys), Box::new(legacy)) + } + + #[test] + fn large_session_reopens_without_plaintext_or_credential_blob_overflow() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("secure-sessions"); + let keys = TestStore { + limit: Some(2560), + ..Default::default() + }; + let legacy = TestStore { + limit: Some(2560), + ..Default::default() + }; + let session = "opaque-payload".repeat(300); + assert!(session.encode_utf16().count() * 2 > 2560); + let first = store(&root, keys.clone(), legacy.clone()); + first.set("account", &session).unwrap(); + assert!( + !fs::read(first.path("account")) + .unwrap() + .windows(session.len()) + .any(|window| window == session.as_bytes()) + ); + assert_eq!(keys.get("account").unwrap().unwrap().len(), 44); + assert_eq!( + store(&root, keys, legacy).get("account").unwrap(), + Some(session) + ); + } + + #[test] + fn legacy_migration_survives_failed_write_and_removes_old_entry_after_reopen() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("secure-sessions"); + let keys = TestStore::default(); + let legacy = TestStore::default(); + legacy.set("account", "legacy-session").unwrap(); + let blocked = store( + &root, + TestStore { + fail_set: true, + ..keys.clone() + }, + legacy.clone(), + ); + assert_eq!( + blocked.get("account").unwrap().as_deref(), + Some("legacy-session") + ); + assert_eq!( + legacy.get("account").unwrap().as_deref(), + Some("legacy-session") + ); + assert!(keys.get("account").unwrap().is_none()); + keys.set("account", &STANDARD.encode(rand::random::<[u8; 32]>())) + .unwrap(); + assert_eq!( + store(&root, keys.clone(), legacy.clone()) + .get("account") + .unwrap() + .as_deref(), + Some("legacy-session") + ); + assert_eq!( + store(&root, keys, legacy.clone()) + .get("account") + .unwrap() + .as_deref(), + Some("legacy-session") + ); + assert!(legacy.get("account").unwrap().is_none()); + } + + #[test] + fn corrupted_or_missing_key_does_not_fall_back_or_rekey_ciphertext() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("secure-sessions"); + let keys = TestStore::default(); + let legacy = TestStore::default(); + let first = store(&root, keys.clone(), legacy.clone()); + first.set("account", "current-session").unwrap(); + legacy.set("account", "stale-session").unwrap(); + let path = first.path("account"); + let original = fs::read(&path).unwrap(); + keys.delete("account").unwrap(); + assert!(first.get("account").unwrap_err().contains("key is missing")); + assert!( + first + .set("account", "replacement") + .unwrap_err() + .contains("key is missing") + ); + assert_eq!(fs::read(&path).unwrap(), original); + keys.set("account", &STANDARD.encode(rand::random::<[u8; 32]>())) + .unwrap(); + assert!( + first + .get("account") + .unwrap_err() + .contains("authentication failed") + ); + assert_eq!( + legacy.get("account").unwrap().as_deref(), + Some("stale-session") + ); + first.delete("account").unwrap(); + assert!(!path.exists()); + assert!(keys.get("account").unwrap().is_none()); + assert!(legacy.get("account").unwrap().is_none()); + } + + #[test] + fn oversized_input_and_ciphertext_are_rejected_without_migration() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("secure-sessions"); + let keys = TestStore::default(); + let legacy = TestStore::default(); + let first = store(&root, keys.clone(), legacy.clone()); + assert!( + first + .set("account", &"x".repeat(MAX_SESSION_BYTES + 1)) + .is_err() + ); + assert!(keys.get("account").unwrap().is_none()); + fs::create_dir_all(&root).unwrap(); + fs::write( + first.path("account"), + vec![0; MAX_SESSION_BYTES + NONCE_BYTES + TAG_BYTES + 1], + ) + .unwrap(); + legacy.set("account", "old-session").unwrap(); + assert!(first.get("account").unwrap_err().contains("size or format")); + assert_eq!( + legacy.get("account").unwrap().as_deref(), + Some("old-session") + ); + } + + #[test] + fn sign_out_deletes_both_stores_and_next_sign_in_uses_new_key() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("secure-sessions"); + let keys = TestStore::default(); + let legacy = TestStore::default(); + let vault = store(&root, keys.clone(), legacy.clone()); + vault.set("account", "old-session").unwrap(); + let old_key = keys.get("account").unwrap().unwrap(); + vault.delete("account").unwrap(); + assert!(vault.get("account").unwrap().is_none()); + vault.set("account", "new-session").unwrap(); + assert_ne!(keys.get("account").unwrap().unwrap(), old_key); + assert_eq!( + store(&root, keys, legacy) + .get("account") + .unwrap() + .as_deref(), + Some("new-session") + ); + } + + #[cfg(windows)] + #[test] + fn windows_credential_manager_migrates_and_restores_after_restart() { + use crate::credential_vault::{ + CredentialVault, KEY_SERVICE_NAME, SERVICE_NAME, credential, + }; + + struct Cleanup(String); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = credential(SERVICE_NAME, &self.0) + .and_then(|entry| entry.delete_credential().map_err(|error| error.to_string())); + let _ = credential(KEY_SERVICE_NAME, &self.0) + .and_then(|entry| entry.delete_credential().map_err(|error| error.to_string())); + } + } + + let directory = tempfile::tempdir().unwrap(); + let user_id = format!("opennow-vault-test-{:032x}", rand::random::()); + let _cleanup = Cleanup(user_id.clone()); + let mut session = super::super::tests::sample_session(&user_id); + let legacy = serde_json::to_string(&session).unwrap(); + credential(SERVICE_NAME, &user_id) + .unwrap() + .set_password(&legacy) + .unwrap(); + let vault = CredentialVault::new(directory.path().into()); + vault + .write_metadata(&super::super::Metadata { + active_user_id: Some(user_id.clone()), + accounts: vec![super::super::tests::sample_identity(&user_id)], + ..Default::default() + }) + .unwrap(); + assert_eq!(vault.load_active().unwrap().unwrap().user.user_id, user_id); + drop(vault); + let reopened = CredentialVault::new(directory.path().into()); + assert_eq!( + reopened.load_active().unwrap().unwrap().user.user_id, + user_id + ); + assert!(matches!( + credential(SERVICE_NAME, &user_id).unwrap().get_password(), + Err(keyring::Error::NoEntry) + )); + + session.tokens.access_token = "x".repeat(1800); + let size = serde_json::to_string(&session).unwrap().len(); + assert!((1944..2560).contains(&size), "fixture size: {size}"); + reopened.save(&session).unwrap(); + drop(reopened); + let restored = CredentialVault::new(directory.path().into()); + assert_eq!( + restored.load_active().unwrap().unwrap().tokens.access_token, + session.tokens.access_token + ); + let encrypted = directory + .path() + .join("secure-sessions") + .join(format!("{:x}.bin", Sha256::digest(user_id.as_bytes()))); + assert!( + !fs::read(encrypted) + .unwrap() + .windows(1800) + .any(|window| window == session.tokens.access_token.as_bytes()) + ); + session.tokens.access_token = "x".repeat(3200); + assert!( + serde_json::to_string(&session) + .unwrap() + .encode_utf16() + .count() + * 2 + > 2560 + ); + restored.save(&session).unwrap(); + drop(restored); + let restored = CredentialVault::new(directory.path().into()); + assert_eq!( + restored.load_active().unwrap().unwrap().tokens.access_token, + session.tokens.access_token + ); + restored.remove(&user_id).unwrap(); + assert!( + CredentialVault::new(directory.path().into()) + .load(&user_id) + .unwrap() + .is_none() + ); + } +} diff --git a/external/opennow/native/opennow-core/src/credential_vault/json_secret_store.rs b/external/opennow/native/opennow-core/src/credential_vault/json_secret_store.rs new file mode 100644 index 000000000..16ec79295 --- /dev/null +++ b/external/opennow/native/opennow-core/src/credential_vault/json_secret_store.rs @@ -0,0 +1,721 @@ +use super::{SecretStore, decode_session, read_bounded}; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::{self, Write as _}; +use std::path::{Path, PathBuf}; + +pub(super) struct JsonFallbackStore { + root: PathBuf, + secure: Box, +} + +impl JsonFallbackStore { + pub(super) fn new(root: PathBuf, secure: Box) -> Self { + Self { root, secure } + } + + fn path(&self, user_id: &str) -> PathBuf { + let digest = Sha256::digest(user_id.as_bytes()); + self.root.join(format!("{digest:x}.json")) + } + + pub(super) fn cleanup_temporary_files(&self) -> Result<(), String> { + let entries = match fs::read_dir(&self.root) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(_) => return Err("Local session temporary-file cleanup is pending".into()), + }; + for entry in entries { + let entry = entry.map_err(|_| "Local session temporary-file cleanup is pending")?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.strip_suffix(".tmp").is_some_and(|stem| { + stem.len() == 64 && stem.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) { + fs::remove_file(entry.path()) + .map_err(|_| "Local session temporary-file cleanup is pending")?; + } + } + Ok(()) + } + + fn write(&self, user_id: &str, encoded: &str) -> Result<(), String> { + let result = (|| -> io::Result<()> { + fs::create_dir_all(&self.root)?; + if !fs::symlink_metadata(&self.root)?.is_dir() { + return Err(io::Error::other("Invalid credential directory")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&self.root, fs::Permissions::from_mode(0o700))?; + } + let temporary = self.path(user_id).with_extension("tmp"); + match fs::remove_file(&temporary) { + Ok(()) => (), + Err(error) if error.kind() == io::ErrorKind::NotFound => (), + Err(error) => return Err(error), + } + let result = (|| -> io::Result<()> { + let mut file = create_private_file(&temporary)?; + file.write_all(encoded.as_bytes())?; + file.sync_all()?; + drop(file); + fs::rename(&temporary, self.path(user_id))?; + #[cfg(unix)] + fs::File::open(&self.root)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result + })(); + result.map_err(|_| "Could not save the local session file".into()) + } + + fn remove_file(&self, user_id: &str) -> Result<(), String> { + let mut result = Ok(()); + for path in [self.path(user_id), self.path(user_id).with_extension("tmp")] { + match fs::remove_file(path) { + Ok(()) => (), + Err(error) if error.kind() == io::ErrorKind::NotFound => (), + Err(_) => result = Err("Could not remove the local session file".into()), + } + } + result + } +} + +impl SecretStore for JsonFallbackStore { + fn get(&self, user_id: &str) -> Result, String> { + match read_bounded(&self.path(user_id)) { + Ok(bytes) => { + let encoded = String::from_utf8(bytes).map_err(|_| "Local session is invalid")?; + decode_session(&encoded, user_id)?; + Ok(Some(encoded)) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => self.secure.get(user_id), + Err(_) => Err("Could not read the local session file".into()), + } + } + + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + decode_session(encoded, user_id)?; + let local = self + .path(user_id) + .try_exists() + .map_err(|_| "Could not inspect local session")?; + if local { + self.write(user_id, encoded)?; + } + if self.secure.set(user_id, encoded).is_ok() + && self.secure.get(user_id).ok().flatten().as_deref() == Some(encoded) + { + self.remove_file(user_id)?; + return Ok(()); + } + if !local { + self.write(user_id, encoded)?; + } + Ok(()) + } + + fn delete(&self, user_id: &str) -> Result<(), String> { + let local = self.remove_file(user_id); + let secure = self.secure.delete(user_id); + local.and(secure) + } + + fn remove_all_local( + &self, + remove_account: &dyn Fn(&str) -> Result<(), String>, + ) -> Result<(), String> { + match fs::symlink_metadata(&self.root) { + Ok(metadata) if metadata.is_dir() => (), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + _ => return Err("Local session directory cleanup is pending".into()), + } + let entries = + fs::read_dir(&self.root).map_err(|_| "Local session directory cleanup is pending")?; + let mut failure = None; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(_) => { + failure.get_or_insert_with(|| "Local session entry cleanup is pending".into()); + continue; + } + }; + let path = entry.path(); + let Some(extension @ ("json" | "tmp")) = + path.extension().and_then(|value| value.to_str()) + else { + continue; + }; + if !path + .file_stem() + .and_then(|value| value.to_str()) + .is_some_and(|stem| { + stem.len() == 64 && stem.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + { + continue; + } + let session = if entry.file_type().is_ok_and(|kind| kind.is_file()) { + match read_bounded(&path) { + Ok(bytes) => serde_json::from_slice::(&bytes).ok(), + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(_) => None, + } + } else { + None + }; + if let Some(session) = session.filter(|session| { + !session.user.user_id.trim().is_empty() + && self.path(&session.user.user_id).with_extension(extension) == path + }) { + if let Err(error) = remove_account(&session.user.user_id) { + failure.get_or_insert(error); + } + } else { + failure.get_or_insert_with(|| { + "Local session identity could not be recovered; secure cleanup is pending" + .into() + }); + } + match fs::remove_file(&path) { + Ok(()) => (), + Err(error) if error.kind() == io::ErrorKind::NotFound => (), + Err(_) => { + failure.get_or_insert_with(|| "Could not remove the local session file".into()); + } + } + } + failure.map_or(Ok(()), Err) + } + + fn is_plaintext(&self, user_id: &str) -> bool { + self.path(user_id).exists() + } +} + +#[cfg(unix)] +fn create_private_file(path: &Path) -> io::Result { + use std::os::unix::fs::OpenOptionsExt; + fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) +} + +#[cfg(windows)] +fn create_private_file(path: &Path) -> io::Result { + use std::os::windows::ffi::OsStrExt; + use std::os::windows::io::FromRawHandle; + use windows_sys::Win32::Foundation::{INVALID_HANDLE_VALUE, LocalFree}; + use windows_sys::Win32::Security::Authorization::ConvertStringSecurityDescriptorToSecurityDescriptorW; + use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; + use windows_sys::Win32::Storage::FileSystem::{ + CREATE_NEW, CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_GENERIC_WRITE, + }; + + let text: Vec = "D:P(A;;FA;;;OW)(A;;FA;;;SY)" + .encode_utf16() + .chain(Some(0)) + .collect(); + let mut descriptor = std::ptr::null_mut(); + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + text.as_ptr(), + 1, + &mut descriptor, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + let attributes = SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: descriptor, + bInheritHandle: 0, + }; + let wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + FILE_GENERIC_WRITE, + 0, + &attributes, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + std::ptr::null_mut(), + ) + }; + let error = io::Error::last_os_error(); + unsafe { + LocalFree(descriptor); + } + if handle == INVALID_HANDLE_VALUE { + return Err(error); + } + Ok(unsafe { fs::File::from_raw_handle(handle) }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::credential_vault::{CredentialVault, MemorySecretStore, tests::sample_session}; + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + #[derive(Clone, Default)] + struct Store { + entries: Arc>>, + unavailable: Arc, + discard_writes: Arc, + } + + impl SecretStore for Store { + fn get(&self, user_id: &str) -> Result, String> { + if self.unavailable.load(Ordering::SeqCst) { + return Err("unavailable".into()); + } + Ok(self.entries.lock().unwrap().get(user_id).cloned()) + } + + fn set(&self, user_id: &str, encoded: &str) -> Result<(), String> { + if self.unavailable.load(Ordering::SeqCst) { + return Err("unavailable".into()); + } + if !self.discard_writes.load(Ordering::SeqCst) { + self.entries + .lock() + .unwrap() + .insert(user_id.into(), encoded.into()); + } + Ok(()) + } + + fn delete(&self, user_id: &str) -> Result<(), String> { + if self.unavailable.load(Ordering::SeqCst) { + return Err("unavailable".into()); + } + self.entries.lock().unwrap().remove(user_id); + Ok(()) + } + } + + #[test] + fn secure_store_is_preferred_without_creating_json() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::with_store(directory.path().into(), Box::::default()); + let session = sample_session("user"); + vault.save(&session).unwrap(); + assert_eq!(vault.persistence_state(&session), "secure-store"); + assert!(!directory.path().join("fallback-sessions").exists()); + } + + #[test] + fn logout_all_cleans_orphans_without_readable_metadata() { + for corrupt in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let secure = Store::default(); + let vault = + CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + let session = sample_session("orphan"); + vault.save(&session).unwrap(); + secure.unavailable.store(true, Ordering::SeqCst); + vault.save(&session).unwrap(); + secure.unavailable.store(false, Ordering::SeqCst); + if corrupt { + fs::write(directory.path().join("accounts.json"), "invalid metadata").unwrap(); + } else { + fs::remove_file(directory.path().join("accounts.json")).unwrap(); + } + let result = vault.remove_all(); + assert_eq!(result.is_err(), corrupt); + assert_eq!( + fs::read_dir(directory.path().join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + assert!(secure.get("orphan").unwrap().is_none()); + if corrupt { + assert_eq!( + fs::read_to_string(directory.path().join("accounts.json")).unwrap(), + "invalid metadata" + ); + } else { + assert!(vault.load("orphan").unwrap().is_none()); + } + } + } + + #[test] + fn logout_all_suppresses_orphans_when_secure_delete_fails() { + let directory = tempfile::tempdir().unwrap(); + let secure = Store::default(); + let vault = CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + let session = sample_session("orphan"); + vault.save(&session).unwrap(); + secure.unavailable.store(true, Ordering::SeqCst); + vault.save(&session).unwrap(); + fs::remove_file(directory.path().join("accounts.json")).unwrap(); + assert!(vault.remove_all().is_err()); + assert_eq!( + fs::read_dir(directory.path().join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + secure.unavailable.store(false, Ordering::SeqCst); + assert!(secure.get("orphan").unwrap().is_some()); + let reopened = + CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + assert!(reopened.load_active().unwrap().is_none()); + assert!(reopened.load("orphan").unwrap().is_none()); + reopened.remove_all().unwrap(); + assert!(secure.get("orphan").unwrap().is_none()); + } + + #[test] + fn logout_all_removes_invalid_and_partial_local_files_without_skipping_valid_accounts() { + let directory = tempfile::tempdir().unwrap(); + let secure = Store::default(); + let vault = CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + let root = directory.path().join("fallback-sessions"); + let store = JsonFallbackStore::new(root.clone(), Box::new(secure.clone())); + fs::create_dir_all(&root).unwrap(); + let invalid = store.path("invalid"); + fs::write(&invalid, "invalid-json").unwrap(); + let encoded = serde_json::to_string(&sample_session("partial")).unwrap(); + secure.set("partial", &encoded).unwrap(); + let partial = store.path("partial").with_extension("tmp"); + fs::write(&partial, encoded).unwrap(); + let other = root.join("unrelated.json"); + fs::write(&other, "unrelated").unwrap(); + assert!(vault.remove_all().is_err()); + assert!(!invalid.exists()); + assert!(!partial.exists()); + assert!(other.exists()); + assert!(secure.get("partial").unwrap().is_none()); + assert!(vault.load("partial").unwrap().is_none()); + } + + #[test] + fn metadata_failure_does_not_hide_plaintext_storage() { + let directory = tempfile::tempdir().unwrap(); + let vault = CredentialVault::without_os_store(directory.path().into()); + let mut session = sample_session("user"); + vault.save(&session).unwrap(); + fs::create_dir(directory.path().join("accounts.json.tmp")).unwrap(); + session.tokens.refresh_token = Some("new-test-token".into()); + assert!(vault.save(&session).is_err()); + assert_eq!(vault.failed_save_state(&session), "local-file"); + assert!(!vault.warnings().is_empty()); + drop(vault); + let restored = CredentialVault::without_os_store(directory.path().into()); + assert_eq!( + restored + .load_active() + .unwrap() + .unwrap() + .tokens + .refresh_token, + session.tokens.refresh_token + ); + fs::remove_dir(directory.path().join("accounts.json.tmp")).unwrap(); + restored.save(&session).unwrap(); + assert!(restored.warnings().is_empty()); + } + + #[test] + fn restart_cleans_interrupted_plaintext_writes_without_account_metadata() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("fallback-sessions"); + fs::create_dir(&root).unwrap(); + let store = JsonFallbackStore::new(root.clone(), Box::::default()); + let session = serde_json::to_string(&sample_session("unindexed")).unwrap(); + let pending = store.path("unindexed").with_extension("tmp"); + fs::write(&pending, &session).unwrap(); + let committed = store.path("other"); + fs::write(&committed, "not-temporary").unwrap(); + CredentialVault::without_os_store(directory.path().into()); + assert!(!pending.exists()); + assert!(committed.exists()); + assert!(!directory.path().join("accounts.json").exists()); + } + + #[test] + fn logout_and_save_remove_only_the_accounts_interrupted_write() { + let directory = tempfile::tempdir().unwrap(); + let store = JsonFallbackStore::new( + directory.path().into(), + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }), + ); + let encoded = serde_json::to_string(&sample_session("user")).unwrap(); + let pending = store.path("user").with_extension("tmp"); + let other = store.path("other").with_extension("tmp"); + fs::write(&pending, &encoded).unwrap(); + fs::write(&other, &encoded).unwrap(); + assert!(store.delete("user").is_err()); + assert!(!pending.exists()); + assert!(other.exists()); + fs::write(&pending, &encoded).unwrap(); + store.set("user", &encoded).unwrap(); + assert!(!pending.exists()); + assert!(other.exists()); + assert_eq!( + store.get("user").unwrap().as_deref(), + Some(encoded.as_str()) + ); + } + + #[test] + fn latest_json_wins_until_a_verified_secure_save() { + let directory = tempfile::tempdir().unwrap(); + let secure = Store::default(); + let vault = CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + let mut session = sample_session("user"); + vault.save(&session).unwrap(); + secure.unavailable.store(true, Ordering::SeqCst); + session.tokens.refresh_token = Some("refreshed-test-token".into()); + vault.save(&session).unwrap(); + assert_eq!(vault.persistence_state(&session), "local-file"); + drop(vault); + secure.unavailable.store(false, Ordering::SeqCst); + let restored = + CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + assert_eq!( + restored + .load_active() + .unwrap() + .unwrap() + .tokens + .refresh_token, + Some("refreshed-test-token".into()) + ); + assert_eq!(restored.persistence_state(&session), "local-file"); + secure.discard_writes.store(true, Ordering::SeqCst); + restored.save(&session).unwrap(); + assert_eq!(restored.persistence_state(&session), "local-file"); + secure.discard_writes.store(false, Ordering::SeqCst); + restored.save(&session).unwrap(); + assert_eq!(restored.persistence_state(&session), "secure-store"); + assert_eq!( + fs::read_dir(directory.path().join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + assert_eq!( + restored + .load_active() + .unwrap() + .unwrap() + .tokens + .refresh_token, + Some("refreshed-test-token".into()) + ); + restored.remove("user").unwrap(); + assert!(secure.get("user").unwrap().is_none()); + } + + #[test] + fn logout_removes_json_and_suppresses_old_secure_grant() { + let directory = tempfile::tempdir().unwrap(); + let secure = Store::default(); + let vault = CredentialVault::with_store(directory.path().into(), Box::new(secure.clone())); + let mut session = sample_session("user"); + vault.save(&session).unwrap(); + secure.unavailable.store(true, Ordering::SeqCst); + session.tokens.refresh_token = Some("new-test-token".into()); + vault.save(&session).unwrap(); + assert!(vault.remove("user").is_err()); + assert_eq!( + fs::read_dir(directory.path().join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + secure.unavailable.store(false, Ordering::SeqCst); + let restored = CredentialVault::with_store(directory.path().into(), Box::new(secure)); + assert!(restored.load("user").unwrap().is_none()); + assert!(restored.load_active().unwrap().is_none()); + restored.remove("user").unwrap(); + assert!(restored.warnings().is_empty()); + } + + #[test] + fn windows_encryption_store_without_vault_keys_uses_json() { + let directory = tempfile::tempdir().unwrap(); + let unavailable = || { + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }) as Box + }; + let secure = super::super::encrypted_secret_store::EncryptedSecretStore::new( + directory.path().join("secure-sessions"), + unavailable(), + unavailable(), + ); + let vault = CredentialVault::with_store(directory.path().into(), Box::new(secure)); + let session = sample_session("user"); + vault.save(&session).unwrap(); + assert_eq!(vault.persistence_state(&session), "local-file"); + assert!(vault.durable(&session)); + assert!(!directory.path().join("secure-sessions").exists()); + } + + #[test] + fn malformed_or_mismatched_local_json_does_not_restore_stale_secure_tokens() { + let directory = tempfile::tempdir().unwrap(); + let store = JsonFallbackStore::new(directory.path().into(), Box::::default()); + let encoded = serde_json::to_string(&sample_session("user")).unwrap(); + store.secure.set("user", &encoded).unwrap(); + fs::write(store.path("user"), "invalid-json").unwrap(); + assert!(store.get("user").is_err()); + fs::write( + store.path("user"), + serde_json::to_string(&sample_session("other")).unwrap(), + ) + .unwrap(); + assert!(store.get("user").is_err()); + assert!(store.set("other", &encoded).is_err()); + assert!(store.set("user", &"x".repeat(4 * 1024 * 1024 + 1)).is_err()); + assert_ne!(store.path("a/b"), store.path("a_b")); + assert_eq!(store.path("../../outside").parent(), Some(directory.path())); + } + + #[test] + fn unwritable_fallback_reports_failure_without_temporary_files() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("not-a-directory"); + fs::write(&root, "blocked").unwrap(); + let store = JsonFallbackStore::new( + root, + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }), + ); + assert!( + store + .set( + "user", + &serde_json::to_string(&sample_session("user")).unwrap() + ) + .is_err() + ); + assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1); + } + + #[cfg(unix)] + #[test] + fn json_permissions_remain_private_across_atomic_replacements() { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().unwrap(); + let store = JsonFallbackStore::new( + directory.path().join("fallback"), + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }), + ); + let mut session = sample_session("user"); + for token in ["first-test-token", "second-test-token"] { + session.tokens.refresh_token = Some(token.into()); + let encoded = serde_json::to_string(&session).unwrap(); + store.set("user", &encoded).unwrap(); + assert_eq!( + fs::metadata(store.path("user")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + fs::metadata(&store.root).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + store.get("user").unwrap().as_deref(), + Some(encoded.as_str()) + ); + assert_eq!(fs::read_dir(&store.root).unwrap().count(), 1); + } + } + + #[cfg(windows)] + #[test] + fn windows_json_has_a_protected_owner_and_system_acl_after_replacement() { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::{GetNamedSecurityInfoW, SE_FILE_OBJECT}; + use windows_sys::Win32::Security::{ + DACL_SECURITY_INFORMATION, GetSecurityDescriptorControl, SE_DACL_PROTECTED, + }; + let directory = tempfile::tempdir().unwrap(); + let store = JsonFallbackStore::new( + directory.path().join("fallback"), + Box::new(MemorySecretStore { + unavailable: true, + ..Default::default() + }), + ); + let mut session = sample_session("user"); + for token in ["first-test-token", "second-test-token"] { + session.tokens.refresh_token = Some(token.into()); + let encoded = serde_json::to_string(&session).unwrap(); + store.set("user", &encoded).unwrap(); + assert_eq!( + store.get("user").unwrap().as_deref(), + Some(encoded.as_str()) + ); + let path: Vec = store + .path("user") + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + let mut descriptor = std::ptr::null_mut(); + let mut acl = std::ptr::null_mut(); + assert_eq!( + unsafe { + GetNamedSecurityInfoW( + path.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + std::ptr::null_mut(), + std::ptr::null_mut(), + &mut acl, + std::ptr::null_mut(), + &mut descriptor, + ) + }, + 0 + ); + let mut control = 0; + let mut revision = 0; + let success = + unsafe { GetSecurityDescriptorControl(descriptor, &mut control, &mut revision) }; + let entries = unsafe { (*acl).AceCount }; + unsafe { + LocalFree(descriptor); + } + assert_ne!(success, 0); + assert_ne!(control & SE_DACL_PROTECTED, 0); + assert_eq!(entries, 2); + } + } +} diff --git a/external/opennow/native/opennow-core/src/device_identity.rs b/external/opennow/native/opennow-core/src/device_identity.rs new file mode 100644 index 000000000..cf72e3369 --- /dev/null +++ b/external/opennow/native/opennow-core/src/device_identity.rs @@ -0,0 +1,104 @@ +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::fs::{self, OpenOptions}; +use std::io::{Read, Write}; +use std::path::Path; + +#[derive(Serialize, Deserialize)] +struct Identity { + version: u8, + id: String, +} + +pub fn load(data_dir: &Path, legacy_id: impl FnOnce() -> String) -> Result { + let load = || -> Result> { + fs::create_dir_all(data_dir)?; + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(data_dir.join("device-identity.lock"))?; + lock.lock_exclusive()?; + let path = data_dir.join("device-identity.json"); + match fs::File::open(&path) { + Ok(file) => { + let mut bytes = Vec::new(); + file.take(4097).read_to_end(&mut bytes)?; + let identity: Identity = serde_json::from_slice(&bytes)?; + if identity.version != 1 + || identity.id.len() != 64 + || !identity.id.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err("Device identity is invalid".into()); + } + Ok(identity.id) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let upgraded = [ + "accounts.json", + "auth-state.json", + "sessions", + "pending-session-cleanup.json", + ] + .iter() + .any(|name| data_dir.join(name).exists()); + let id = if upgraded { + legacy_id() + } else { + use rand::RngCore as _; + let mut bytes = [0u8; 32]; + rand::rng().fill_bytes(&mut bytes); + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + }; + let temporary = path.with_extension("json.tmp"); + let mut file = fs::File::create(&temporary)?; + file.write_all(&serde_json::to_vec(&Identity { + version: 1, + id: id.clone(), + })?)?; + file.sync_all()?; + fs::rename(temporary, path)?; + #[cfg(unix)] + fs::File::open(data_dir)?.sync_all()?; + Ok(id) + } + Err(error) => Err(error.into()), + } + }; + load().map_err(|_| "Persistent device identity is unavailable or invalid; restore access to the data directory before signing in".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn freezes_legacy_identity_and_refuses_corruption() { + let directory = tempfile::tempdir().unwrap(); + fs::write(directory.path().join("accounts.json"), "{}").unwrap(); + let original = "a".repeat(64); + assert_eq!( + load(directory.path(), || original.clone()).unwrap(), + original + ); + assert_eq!(load(directory.path(), || "b".repeat(64)).unwrap(), original); + fs::write(directory.path().join("device-identity.json"), "broken").unwrap(); + assert!(load(directory.path(), || "c".repeat(64)).is_err()); + } + + #[test] + fn concurrent_fresh_initialization_chooses_one_identity() { + let directory = tempfile::tempdir().unwrap(); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..8) + .map(|_| scope.spawn(|| load(directory.path(), || panic!("fresh installation")))) + .collect(); + let ids: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap().unwrap()) + .collect(); + assert!(ids.iter().all(|id| id == &ids[0] && id.len() == 64)); + }); + } +} diff --git a/external/opennow/native/opennow-core/src/diagnostics.rs b/external/opennow/native/opennow-core/src/diagnostics.rs new file mode 100644 index 000000000..25f2c39cb --- /dev/null +++ b/external/opennow/native/opennow-core/src/diagnostics.rs @@ -0,0 +1,1255 @@ +use serde_json::{Value, json}; +use std::collections::VecDeque; +use std::fs::{self, OpenOptions}; +use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +const ENTRY_LIMIT: usize = 1_200; +const LOG_LIMIT_BYTES: u64 = 5 * 1024 * 1024; + +pub fn stream_endpoint_evidence(session: &Value) -> Value { + let number = |value: &Value| { + value + .as_u64() + .or_else(|| value.as_str().and_then(|text| text.parse::().ok())) + .filter(|number| *number <= u64::from(u16::MAX)) + }; + let connections = session["connectionInfo"].as_array(); + let descriptors = connections + .into_iter() + .flatten() + .take(16) + .map(|connection| { + let resource = connection["resourcePath"].as_str().unwrap_or_default(); + let kind = if resource.is_empty() { + "missing" + } else if resource.starts_with('/') { + "relative" + } else { + match url::Url::parse(resource) + .ok() + .as_ref() + .map(|url| url.scheme()) + { + Some("wss") => "wss", + Some("https") => "https", + Some("rtsps") => "rtsps", + Some("rtsp") => "rtsp", + _ => "other", + } + }; + json!({"usage":number(&connection["usage"]), + "appLevelProtocol":number(&connection["appLevelProtocol"]), + "protocol":number(&connection["protocol"]), + "port":number(&connection["port"]), "resourceKind":kind, + "hasAddress":connection["ip"].as_str().is_some_and(|value| !value.is_empty())}) + }) + .collect::>(); + json!({"transportMode":session["transportMode"].as_str().filter(|mode| matches!(*mode, "nvst" | "webrtc")), + "connectionCount":connections.map_or(0, Vec::len), "connections":descriptors, + "hasSignalingUrl":session["signalingUrl"].as_str().is_some_and(|value| !value.is_empty()), + "signalingPort":session["signalingUrl"].as_str().and_then(|raw| url::Url::parse(raw).ok()).and_then(|url| url.port_or_known_default()), + "reportedNativeTransport":session["reportedNativeTransport"].as_bool()}) +} + +pub fn stream_profile_evidence(session: &Value) -> Value { + let profile = &session["negotiatedStreamProfile"]; + let mut evidence = json!({ + "codec": profile["codec"].as_str().filter(|value| matches!(*value, "H264" | "H265" | "HEVC" | "AV1")), + "codecSource": profile["codecSource"].as_str().filter(|value| matches!(*value, "request" | "server" | "unreported")), + "colorQuality": profile["colorQuality"].as_str().filter(|value| matches!(*value, "8bit_420" | "8bit_444" | "10bit_420" | "10bit_444")), + "enableHdr": profile["enableHdr"].as_bool() + }); + for section in ["requestedStreamingFeatures", "finalizedStreamingFeatures"] { + let mut fields = serde_json::Map::new(); + for field in ["codec", "bitDepth", "chromaFormat"] { + if let Some(value) = session[section].get(field) { + let number = value + .as_i64() + .or_else(|| value.as_str().and_then(|text| text.parse::().ok())); + fields.insert(field.to_owned(), json!(number)); + } + } + evidence[section] = Value::Object(fields); + } + evidence +} + +pub fn native_runtime_evidence(capabilities: &Value) -> Value { + let mut evidence = serde_json::Map::new(); + for field in [ + "supportsVideoDecode", + "supportsVideoPresent", + "nativeHdrSupported", + ] { + if let Some(value) = capabilities[field].as_bool() { + evidence.insert(field.to_owned(), json!(value)); + } + } + if let Some(version) = capabilities["protocolVersion"] + .as_u64() + .filter(|v| *v <= u32::MAX as u64) + { + evidence.insert("protocolVersion".to_owned(), json!(version)); + } + let mut backends = Vec::new(); + for backend in capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .take(16) + { + let Some(name) = backend["backend"].as_str().filter(|name| { + matches!( + *name, + "vulkan" + | "cuda" + | "vaapi" + | "v4l2" + | "d3d11" + | "d3d12" + | "videotoolbox" + | "software" + | "ffmpeg" + ) + }) else { + continue; + }; + let mut entry = serde_json::Map::from_iter([("backend".to_owned(), json!(name))]); + if let Some(platform) = backend["platform"] + .as_str() + .filter(|name| matches!(*name, "linux" | "windows" | "macos" | "cross-platform")) + { + entry.insert("platform".to_owned(), json!(platform)); + } + if let Some(value) = backend["available"].as_bool() { + entry.insert("available".to_owned(), json!(value)); + } + if let Some(reason) = backend["reason"].as_str() { + entry.insert("reason".to_owned(), json!(runtime_failure_reason(reason))); + } + let mut codecs = Vec::new(); + for codec in backend["codecs"].as_array().into_iter().flatten().take(8) { + let Some(name) = codec["codec"] + .as_str() + .filter(|name| matches!(*name, "h264" | "h265" | "av1")) + else { + continue; + }; + let mut item = serde_json::Map::from_iter([("codec".to_owned(), json!(name))]); + for field in ["available", "hdrSupported"] { + if let Some(value) = codec[field].as_bool() { + item.insert(field.to_owned(), json!(value)); + } + } + if let Some(reason) = codec["reason"].as_str() { + item.insert("reason".to_owned(), json!(runtime_failure_reason(reason))); + } + codecs.push(Value::Object(item)); + } + entry.insert("codecs".to_owned(), json!(codecs)); + backends.push(Value::Object(entry)); + } + evidence.insert("videoBackends".to_owned(), json!(backends)); + if let Some(adapters) = graphics_adapter_evidence(capabilities) { + evidence.insert("graphicsAdapters".to_owned(), Value::Array(adapters)); + } + Value::Object(evidence) +} + +fn graphics_adapter_evidence(capabilities: &Value) -> Option> { + let source = capabilities.get("graphicsAdapters")?.as_array()?; + if source.is_empty() { + return None; + } + let mut adapters = Vec::new(); + for adapter in source.iter().take(8) { + let mut entry = serde_json::Map::new(); + if let Some(name) = adapter["name"].as_str() { + entry.insert("name".to_owned(), json!(runtime_failure_reason(name))); + } + if let Some(active) = adapter["active"].as_bool() { + entry.insert("active".to_owned(), json!(active)); + } + if let Some(main10) = adapter["h265Main10"].as_bool() { + entry.insert("h265Main10".to_owned(), json!(main10)); + } + let codecs = adapter["codecs"] + .as_array() + .into_iter() + .flatten() + .filter_map(|codec| { + codec + .as_str() + .filter(|name| matches!(*name, "h264" | "h265" | "av1")) + .map(|name| json!(name)) + }) + .take(4) + .collect::>(); + entry.insert("codecs".to_owned(), json!(codecs)); + if let Some(reason) = adapter["reason"].as_str() { + entry.insert("reason".to_owned(), json!(runtime_failure_reason(reason))); + } + adapters.push(Value::Object(entry)); + } + Some(adapters) +} + +pub fn runtime_failure_reason(value: &str) -> String { + static SENSITIVE: std::sync::OnceLock = std::sync::OnceLock::new(); + let pattern = SENSITIVE.get_or_init(|| { + regex::Regex::new(r#"(?i)(?:https?://|wss://|/home/|/users/|[a-z]:\\+users\\+)\S+|\bbearer\s+[^\s,;]+|\b[a-z_]*(?:token|authorization|password|secret)[a-z_]*\s*[\"']?\s*[:=]?\s*[\"']?\s*(?:bearer\s+)?[^\s,;]+"#).unwrap() + }); + let bounded: String = value + .chars() + .take(4096) + .map(|character| { + if character.is_control() { + ' ' + } else { + character + } + }) + .collect(); + redact(&pattern.replace_all(&bounded, "[redacted]"), 480) +} + +pub fn embedded_drop_evidence(params: &Value) -> Value { + let mut evidence = serde_json::Map::new(); + for section in ["embeddedStream", "lastSessionReport"] { + let Some(source) = params[section]["drops"].as_object() else { + continue; + }; + let mut drops = serde_json::Map::new(); + for field in [ + "videoDropCount", + "audioDiscardedMs", + "audioPacketDropCount", + "callbackDropCount", + "otherQueueDropCount", + ] { + if let Some(value) = source.get(field).filter(|value| { + value.as_f64().is_some_and(|number| { + (0.0..=9_007_199_254_740_991.0).contains(&number) + && (field == "audioDiscardedMs" || number.fract() == 0.0) + }) + }) { + drops.insert(field.to_owned(), value.clone()); + } + } + evidence.insert(section.to_owned(), json!({"drops": drops})); + } + Value::Object(evidence) +} + +#[derive(Clone)] +struct Entry { + at_ms: u128, + area: String, + event: String, + detail: String, +} + +pub struct DiagnosticsService { + directory: PathBuf, + current_path: PathBuf, + previous_path: PathBuf, + entries: Mutex>, +} + +impl DiagnosticsService { + pub fn new(data_dir: &Path) -> io::Result { + let directory = data_dir.join("diagnostics"); + fs::create_dir_all(&directory)?; + let current_path = directory.join("current.log"); + let previous_path = directory.join("previous.log"); + let service = Self { + directory, + current_path, + previous_path, + entries: Mutex::new(VecDeque::with_capacity(ENTRY_LIMIT)), + }; + service.rotate_log(0)?; + Ok(service) + } + + fn rotate_log(&self, incoming_bytes: u64) -> io::Result<()> { + let size = match self.current_path.metadata() { + Ok(metadata) => metadata.len(), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + if size.saturating_add(incoming_bytes) <= LOG_LIMIT_BYTES { + return Ok(()); + } + match fs::remove_file(&self.previous_path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if size <= LOG_LIMIT_BYTES { + return fs::rename(&self.current_path, &self.previous_path); + } + let mut current = fs::File::open(&self.current_path)?; + current.seek(SeekFrom::Start(size.saturating_sub(LOG_LIMIT_BYTES)))?; + let mut tail = BufReader::new(current.take(LOG_LIMIT_BYTES)); + tail.skip_until(b'\n')?; + let mut previous = fs::File::create(&self.previous_path)?; + io::copy(&mut tail, &mut previous)?; + OpenOptions::new() + .write(true) + .open(&self.current_path)? + .set_len(0) + } + + pub fn record(&self, area: &str, event: &str, detail: impl AsRef) { + let entry = Entry { + at_ms: now_ms(), + area: clean(area, 48), + event: clean(event, 72), + detail: redact(detail.as_ref(), 480), + }; + let mut entries = self.entries.lock().expect("diagnostics poisoned"); + if entries.len() == ENTRY_LIMIT { + entries.pop_front(); + } + entries.push_back(entry.clone()); + let line = format_entry(&entry); + if self.rotate_log(line.len() as u64).is_err() { + return; + } + if let Ok(mut file) = OpenOptions::new() + .create(true) + .append(true) + .open(&self.current_path) + { + let _ = file.write_all(line.as_bytes()); + } + } + + pub fn snapshot(&self) -> Value { + let entries = self.entries.lock().expect("diagnostics poisoned"); + let mut values = entries + .iter() + .rev() + .take(200) + .map(|entry| { + json!({ + "atMs": entry.at_ms.to_string(), + "area": entry.area, + "event": entry.event, + "detail": entry.detail + }) + }) + .collect::>(); + drop(entries); + // The in-app diagnostics screen uses this same bounded entry contract. + // Read adjacent embedded-runtime traces only on an explicit snapshot request. + for name in ["qt-native.log", "native-streamer.log"] { + if let Ok(tail) = native_log_tail(&self.directory.join(name)) { + for line in tail.lines().rev().take(60) { + let Some((timestamp, detail)) = line.split_once(' ') else { + continue; + }; + let Ok(at_ms) = timestamp.parse::() else { + continue; + }; + values.push(json!({ + "atMs":at_ms.to_string(), "area":name, + "event":"trace", "detail":redact(detail, 480) + })); + } + } + } + values.sort_by_key(|entry| { + std::cmp::Reverse( + entry["atMs"] + .as_str() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0), + ) + }); + values.truncate(200); + json!({ + "entries": values, + "persistent": true, + "redacted": true, + "currentBytes": self.current_path.metadata().map(|value| value.len()).unwrap_or(0), + "previousRunAvailable": self.previous_path.is_file() + }) + } + + pub fn export(&self) -> io::Result { + self.export_with_runtime(None) + } + + pub fn export_with_runtime(&self, runtime: Option<&Value>) -> io::Result { + fs::create_dir_all(&self.directory)?; + let path = self + .directory + .join(format!("opennow-diagnostics-{}.txt", now_ms())); + let temporary = path.with_extension("txt.tmp"); + let mut output = String::from( + "OpenNOW Qt/Rust diagnostics\nSecrets, URLs, tokens, e-mail addresses and local user paths are redacted.\n\n", + ); + if let Ok(previous) = log_tail(&self.previous_path, 500_000) { + output.push_str("Previous run\n------------\n"); + output.push_str(&redact_log_tail(&previous, 500_000)); + output.push_str("\n\n"); + } + output.push_str("Current run\n-----------\n"); + if let Ok(current) = log_tail(&self.current_path, 900_000) { + output.push_str(&redact_log_tail(¤t, 900_000)); + } else { + let entries = self.entries.lock().expect("diagnostics poisoned"); + for entry in entries.iter() { + output.push_str(&format_entry(entry)); + } + } + if let Some(runtime) = runtime { + output.push_str("\n\nStructured runtime snapshot\n---------------------------\n"); + let rendered = + serde_json::to_string_pretty(runtime).unwrap_or_else(|_| "{}".to_owned()); + output.push_str(&redact_lines(&rendered, 200_000)); + output.push('\n'); + } + // The embedded streamer does not run as a child of the core. Include its + // adjacent file sink explicitly, bounded and redacted like the RPC log. + for name in [ + "native-streamer.log", + "native-streamer.log.previous", + "native-streamer.previous.log", + "qt-native.log", + "qt-native.log.previous", + ] { + if let Ok(tail) = native_log_tail(&self.directory.join(name)) { + output.push_str(&format!( + "\n\nNative media: {name}\n---------------------------\n" + )); + output.push_str(&redact_lines(&tail, 262_144)); + } + } + fs::write(&temporary, output.as_bytes())?; + fs::rename(&temporary, &path)?; + Ok(json!({ + "path": path.to_string_lossy(), + "sizeBytes": output.len(), + "redacted": true, + "runtimeSnapshotIncluded": runtime.is_some() + })) + } + + pub fn export_acceptance(&self, manifest: &Value) -> io::Result { + if manifest["schemaVersion"].as_u64() != Some(1) + || manifest["kind"].as_str() != Some("opennow.live-acceptance") + || !acceptance_value_is_safe(manifest) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Acceptance manifest is invalid or contains sensitive values", + )); + } + fs::create_dir_all(&self.directory)?; + let path = self + .directory + .join(format!("opennow-live-acceptance-{}.json", now_ms())); + let temporary = path.with_extension("json.tmp"); + let mut bytes = serde_json::to_vec_pretty(manifest).map_err(io::Error::other)?; + bytes.push(b'\n'); + fs::write(&temporary, &bytes)?; + fs::rename(&temporary, &path)?; + Ok(json!({ + "path": path.to_string_lossy(), + "sizeBytes": bytes.len(), + "redacted": true, + "schemaVersion": 1 + })) + } +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +fn clean(value: &str, limit: usize) -> String { + value + .chars() + .filter(|value| !value.is_control()) + .take(limit) + .collect() +} + +fn redact(value: &str, limit: usize) -> String { + let mut result = String::with_capacity(value.len().min(limit)); + for token in value.split_whitespace() { + let normalized = token.trim_start_matches(['"', '\'', '{', '[', '(', ',', ':']); + let lower = normalized.to_ascii_lowercase(); + let sensitive = lower.contains("token") + || lower.contains("authorization") + || lower.contains("password") + || lower.contains("secret") + || lower.starts_with("http://") + || lower.starts_with("https://") + || lower.starts_with("wss://") + || normalized.contains('@') + || lower.starts_with("/home/") + || lower.starts_with("/users/") + || lower.starts_with("c:\\users\\") + || lower.starts_with("c:\\\\users\\\\"); + let rendered = if sensitive { "[redacted]" } else { token }; + if !result.is_empty() { + result.push(' '); + } + if result.len() + rendered.len() > limit { + result.push('…'); + break; + } + result.push_str(rendered); + } + result +} + +fn format_entry(entry: &Entry) -> String { + format!( + "{} [{}] {}: {}\n", + entry.at_ms, entry.area, entry.event, entry.detail + ) +} + +fn acceptance_value_is_safe(value: &Value) -> bool { + match value { + Value::Object(values) => values.iter().all(|(key, value)| { + let key = key.to_ascii_lowercase(); + ![ + "token", + "authorization", + "password", + "secret", + "sessionid", + "processid", + "executable", + "filepath", + "url", + ] + .iter() + .any(|needle| key.contains(needle)) + && acceptance_value_is_safe(value) + }), + Value::Array(values) => values.iter().all(acceptance_value_is_safe), + Value::String(value) => { + let lower = value.to_ascii_lowercase(); + !value.contains('@') + && !lower.starts_with("http://") + && !lower.starts_with("https://") + && !lower.starts_with("wss://") + && !lower.starts_with("/home/") + && !lower.starts_with("/users/") + && !lower.starts_with("c:\\users\\") + && !lower.contains("c:\\\\users\\\\") + } + _ => true, + } +} + +fn native_log_tail(path: &Path) -> io::Result { + log_tail(path, 262_144) +} + +fn log_tail(path: &Path, limit: u64) -> io::Result { + let mut file = fs::File::open(path)?; + let start = file.metadata()?.len().saturating_sub(limit); + file.seek(SeekFrom::Start(start.saturating_sub(1)))?; + let mut bytes = Vec::new(); + file.take(limit + u64::from(start > 0)) + .read_to_end(&mut bytes)?; + let tail = if start > 0 { + bytes + .iter() + .position(|byte| *byte == b'\n') + .map_or(&[][..], |newline| &bytes[newline + 1..]) + } else { + &bytes + }; + Ok(String::from_utf8_lossy(tail).into_owned()) +} + +fn redact_log_tail(value: &str, limit: usize) -> String { + let complete = value.rsplit_once('\n').map_or("", |(lines, _)| lines); + let mut lines = Vec::new(); + let mut remaining = limit; + for line in complete.lines().rev() { + let mut rendered = redact(line, usize::MAX); + rendered.push('\n'); + if rendered.len() > remaining { + break; + } + remaining -= rendered.len(); + lines.push(rendered); + } + lines.into_iter().rev().collect() +} + +fn redact_lines(value: &str, limit: usize) -> String { + let mut output = String::new(); + for line in value.lines() { + let remaining = limit.saturating_sub(output.len()); + if remaining < 4 { + break; + } + let rendered = redact(line, remaining - 4); + output.push_str(&rendered); + output.push('\n'); + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + fn exported_rpc_sections(service: &DiagnosticsService) -> (String, String) { + let exported = service.export().unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert_eq!(exported["sizeBytes"].as_u64().unwrap(), text.len() as u64); + let (_, logs) = text.split_once("Previous run\n------------\n").unwrap(); + let (previous, current) = logs.split_once("\n\nCurrent run\n-----------\n").unwrap(); + (previous.to_owned(), current.to_owned()) + } + + #[test] + fn diagnostics_export_preserves_newest_entries_in_both_oversized_rpc_logs() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + let history = "1 [rpc] old: historical entry\n".repeat(80_000); + fs::write(&service.current_path, &history).unwrap(); + fs::write( + &service.previous_path, + format!("{history}2 [streamer] prepare: previous incident\n"), + ) + .unwrap(); + service.record("streamer", "prepare", "latest incident"); + let newest = fs::read_to_string(&service.current_path) + .unwrap() + .lines() + .last() + .unwrap() + .to_owned(); + + let (previous, current) = exported_rpc_sections(&service); + + assert!( + current.ends_with(&format!("{newest}\n")), + "newest current entry missing" + ); + assert!(previous.ends_with("2 [streamer] prepare: previous incident\n")); + assert!(current.len() <= 900_000); + assert!(previous.len() <= 500_000); + } + + #[test] + fn diagnostics_export_preserves_newest_rpc_entries_when_redaction_expands() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + let newest = "9 [streamer] prepare: incident-é token=x https://example.com user@example.com /home/alice/file\n"; + for (path, limit) in [ + (&service.current_path, 900_000), + (&service.previous_path, 500_000), + ] { + fs::write(path, format!("{}{newest}", "@\n".repeat(limit / 2))).unwrap(); + } + + let (previous, current) = exported_rpc_sections(&service); + + for (section, limit) in [(current, 900_000), (previous, 500_000)] { + assert!(section.ends_with( + "9 [streamer] prepare: incident-é [redacted] [redacted] [redacted] [redacted]\n" + )); + assert!(section.len() <= limit); + assert!(section.starts_with("[redacted]\n")); + assert!(!section.contains('…')); + for sensitive in ["token=x", "example.com", "alice", "@"] { + assert!(!section.contains(sensitive)); + } + } + } + + #[test] + fn diagnostics_export_keeps_small_rpc_logs_redacted_and_in_order() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + let log = + "1 [rpc] first: café\n2 [rpc] last: token=x user@example.com C:\\Users\\Alice\\file\n"; + fs::write(&service.current_path, log).unwrap(); + fs::write(&service.previous_path, log).unwrap(); + + let (previous, current) = exported_rpc_sections(&service); + + let expected = "1 [rpc] first: café\n2 [rpc] last: [redacted] [redacted] [redacted]\n"; + assert_eq!(current, expected); + assert_eq!(previous, expected); + } + + #[test] + fn diagnostics_export_drops_rpc_tail_fragments_without_corrupting_utf8() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + for (path, limit) in [ + (&service.current_path, 900_000), + (&service.previous_path, 500_000), + ] { + let log = format!( + "discarded-fragment {}\n7 [rpc] complete: café 🎮\n8 [rpc] partial: unfinished", + "é".repeat(limit / 2) + ); + let start = log.len() - limit; + assert!(!log.is_char_boundary(start)); + fs::write(path, log).unwrap(); + } + + let (previous, current) = exported_rpc_sections(&service); + + assert_eq!(current, "7 [rpc] complete: café 🎮\n"); + assert_eq!(previous, current); + } + + #[test] + fn diagnostics_export_keeps_rpc_lines_at_the_exact_tail_boundary() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + let first = "2 [rpc] boundary: first retained\n"; + let last = "3 [rpc] incident: newest\n"; + for (path, limit) in [ + (&service.current_path, 900_000), + (&service.previous_path, 500_000), + ] { + let tail = format!( + "{first}{}\n{last}", + "x".repeat(limit - first.len() - last.len() - 1) + ); + assert_eq!(tail.len(), limit); + fs::write(path, format!("1 [rpc] discarded: old\n{tail}")).unwrap(); + } + + let (previous, current) = exported_rpc_sections(&service); + + for (section, limit) in [(current, 900_000), (previous, 500_000)] { + assert!(section.starts_with(first)); + assert!(section.ends_with(last)); + assert_eq!(section.len(), limit); + assert_eq!(section.lines().count(), 3); + } + } + + #[test] + fn diagnostics_export_does_not_truncate_an_oversized_rpc_line() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + for (path, limit) in [ + (&service.current_path, 900_000), + (&service.previous_path, 500_000), + ] { + fs::write(path, format!("{}\n", "é".repeat(limit))).unwrap(); + } + + let (previous, current) = exported_rpc_sections(&service); + + assert!(current.is_empty()); + assert!(previous.is_empty()); + } + + #[test] + fn diagnostics_export_keeps_structured_runtime_prefix_budgeting() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + let entries = [ + vec!["early-runtime-entry"; 20_000], + vec!["late-runtime-entry"], + ] + .concat(); + let runtime = json!({"entries": entries}); + let rendered = serde_json::to_string_pretty(&runtime).unwrap(); + assert!(rendered.len() > 200_000); + + let exported = service.export_with_runtime(Some(&runtime)).unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + let (_, section) = text + .split_once("Structured runtime snapshot\n---------------------------\n") + .unwrap(); + + assert_eq!(section, format!("{}\n", redact_lines(&rendered, 200_000))); + assert!(section.contains("early-runtime-entry")); + assert!(!section.contains("late-runtime-entry")); + assert!(section.len() <= 200_001); + } + + #[test] + fn diagnostics_rotate_during_a_session_and_replace_the_previous_log() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + for marker in *b"abc" { + let mut current = fs::File::create(&service.current_path).unwrap(); + current.set_len(LOG_LIMIT_BYTES).unwrap(); + current.seek(SeekFrom::End(-1)).unwrap(); + current.write_all(&[marker]).unwrap(); + drop(current); + + service.record("test", "rotation", "new entry"); + + let previous = fs::read(&service.previous_path).unwrap(); + assert_eq!(previous.len() as u64, LOG_LIMIT_BYTES); + assert_eq!(previous.last(), Some(&marker)); + let current = fs::read_to_string(&service.current_path).unwrap(); + assert!(current.contains("new entry")); + assert!((current.len() as u64) < LOG_LIMIT_BYTES); + assert_eq!(fs::read_dir(&service.directory).unwrap().count(), 2); + } + } + + #[test] + fn diagnostics_startup_retains_a_bounded_tail_of_an_oversized_log() { + let directory = tempfile::tempdir().unwrap(); + let diagnostics = directory.path().join("diagnostics"); + fs::create_dir_all(&diagnostics).unwrap(); + let mut current = fs::File::create(diagnostics.join("current.log")).unwrap(); + current.set_len(LOG_LIMIT_BYTES * 2).unwrap(); + current.seek(SeekFrom::Start(LOG_LIMIT_BYTES - 1)).unwrap(); + current.write_all("é\n".as_bytes()).unwrap(); + current.seek(SeekFrom::End(-5)).unwrap(); + current.write_all(b"tail\n").unwrap(); + drop(current); + + let service = DiagnosticsService::new(directory.path()).unwrap(); + + let previous = fs::read_to_string(&service.previous_path).unwrap(); + assert_eq!(previous.len() as u64, LOG_LIMIT_BYTES - 2); + assert!(previous.ends_with("tail\n")); + assert_eq!(service.current_path.metadata().unwrap().len(), 0); + } + + #[test] + fn diagnostics_rotation_failure_does_not_grow_the_log() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + fs::File::create(&service.current_path) + .unwrap() + .set_len(LOG_LIMIT_BYTES) + .unwrap(); + fs::create_dir(&service.previous_path).unwrap(); + + service.record("test", "rotation", "still available in memory"); + + assert_eq!( + service.current_path.metadata().unwrap().len(), + LOG_LIMIT_BYTES + ); + assert_eq!(service.entries.lock().unwrap().len(), 1); + fs::remove_dir(&service.previous_path).unwrap(); + service.record("test", "recovered", "disk logging resumed"); + assert!( + fs::read_to_string(&service.current_path) + .unwrap() + .contains("disk logging resumed") + ); + } + + #[test] + fn diagnostics_concurrent_writers_rotate_without_losing_entries() { + let directory = tempfile::tempdir().unwrap(); + let service = DiagnosticsService::new(directory.path()).unwrap(); + fs::File::create(&service.current_path) + .unwrap() + .set_len(LOG_LIMIT_BYTES - 1) + .unwrap(); + + std::thread::scope(|scope| { + for worker in 0..8 { + let service = &service; + scope.spawn(move || { + for entry in 0..100 { + service.record( + "test", + "concurrent", + format!("worker-{worker}-entry-{entry}"), + ); + } + }); + } + }); + + let current = fs::read_to_string(&service.current_path).unwrap(); + assert_eq!(current.lines().count(), 800); + for worker in 0..8 { + for entry in 0..100 { + assert!(current.contains(&format!("worker-{worker}-entry-{entry}\n"))); + } + } + assert!((current.len() as u64) <= LOG_LIMIT_BYTES); + assert_eq!( + service.previous_path.metadata().unwrap().len(), + LOG_LIMIT_BYTES - 1 + ); + } + + #[test] + fn diagnostics_export_includes_bounded_redacted_native_log() { + let directory = env::temp_dir().join(format!("opennow-native-diagnostics-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + let path = directory.join("diagnostics/native-streamer.log"); + fs::write( + &path, + format!( + "{}\ndecoder initialization failed token=secret https://example.com\n", + "x".repeat(300_000) + ), + ) + .unwrap(); + let tail = native_log_tail(&path).unwrap(); + assert!(tail.len() < 262_144); + let exported = service.export().unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert!(text.contains("Native media: native-streamer.log")); + assert!(text.contains("decoder initialization failed")); + assert!(!text.contains("secret")); + assert!(!text.contains("example.com")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn diagnostics_redact_and_export_atomically() { + let directory = env::temp_dir().join(format!("opennow-diagnostics-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + service.record( + "auth", + "failure", + "token=abc user@example.com https://example.com /home/alice/file", + ); + let exported = service.export().unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert!(!text.contains("abc")); + assert!(!text.contains("user@example.com")); + assert!(!text.contains("example.com")); + assert!(!text.contains("alice")); + assert!(text.contains("[redacted]")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn diagnostics_export_embeds_a_redacted_structured_runtime_snapshot() { + let directory = env::temp_dir().join(format!("opennow-runtime-diagnostics-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + fs::write( + directory.join("diagnostics/qt-native.log"), + "first handshake\nsecond user@example.com\n", + ) + .unwrap(); + let exported = service + .export_with_runtime(Some(&json!({ + "kind":"opennow.acceptance", + "streamer":{"mediaBackend":"ffmpeg","queueDropCount":2}, + "unsafe":"user@example.com /home/alice/recording.mkv C:\\Users\\Alice\\capture.mkv /Users/alice/capture.mkv" + }))) + .unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert_eq!(exported["runtimeSnapshotIncluded"], true); + assert!(text.contains("Structured runtime snapshot")); + assert!(text.contains("opennow.acceptance")); + assert!(text.contains("queueDropCount")); + assert!(text.contains("qt-native.log")); + assert!(text.contains("first handshake\nsecond [redacted]\n")); + assert!(!text.contains("user@example.com")); + assert!(!text.contains("/home/alice")); + assert!(!text.contains("Alice")); + assert!(!text.contains("/Users/alice")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn native_runtime_evidence_allowlists_bounds_and_redacts_probe_results() { + let capabilities = json!({ + "protocolVersion": 7, "supportsVideoDecode": false, "supportsVideoPresent": "yes", + "sessionId": "private-session", "accessToken": "private-access", + "videoBackends": [{ + "backend": "v4l2", "platform": "linux", "available": false, + "devicePath": "/home/alice/device", "unknown": "private-extra", + "reason": "HEVC topology probe failed path=/home/alice/private Authorization: Bearer abc123 token=xyz password: hunter2 https://example.com user@example.com", + "codecs": [{"codec": "h265", "available": false, + "reason": "MEDIA_IOC_G_TOPOLOGY failed", "secret": "private-codec"}, + {"codec": "private-unknown-codec", "available": true}] + }, {"backend": "private-unknown-backend", "available": true}] + }); + let evidence = native_runtime_evidence(&capabilities); + assert_eq!(evidence["supportsVideoDecode"], false); + assert!(evidence.get("supportsVideoPresent").is_none()); + assert_eq!(evidence["videoBackends"].as_array().unwrap().len(), 1); + let backend = &evidence["videoBackends"][0]; + assert_eq!(backend["available"], false); + assert_eq!( + backend["codecs"], + json!([{"codec":"h265", "available":false, "reason":"MEDIA_IOC_G_TOPOLOGY failed"}]) + ); + let rendered = evidence.to_string(); + assert!(rendered.contains("HEVC topology probe failed")); + for sensitive in [ + "private", + "alice", + "abc123", + "xyz", + "hunter2", + "example.com", + ] { + assert!( + !rendered.contains(sensitive), + "{sensitive} leaked: {rendered}" + ); + } + let oversized = json!({"protocolVersion": u64::MAX, "videoBackends": vec![json!({ + "backend":"v4l2", "reason":"x".repeat(10000), "codecs":vec![json!({ + "codec":"h265", "available":"true", "reason":"y".repeat(10000) + }); 100] + }); 100]}); + let evidence = native_runtime_evidence(&oversized); + assert!(evidence.get("protocolVersion").is_none()); + let backends = evidence["videoBackends"].as_array().unwrap(); + assert_eq!(backends.len(), 16); + assert!(backends[0]["reason"].as_str().unwrap().len() <= 483); + assert_eq!(backends[0]["codecs"].as_array().unwrap().len(), 8); + assert!(backends[0]["codecs"][0].get("available").is_none()); + assert_eq!( + native_runtime_evidence(&Value::Null), + json!({"videoBackends":[]}) + ); + let indexed = native_runtime_evidence(&json!({ + "videoBackends":[], + "graphicsAdapters":[ + {"name":"NVIDIA GeForce MX110","active":true,"codecs":[],"h265Main10":false, + "reason":"no supported hardware decoder profile","luid":"private-luid"}, + {"name":"Intel(R) HD Graphics 620 path=/home/alice/gpu","active":false, + "codecs":["h264","h265","private-codec"],"h265Main10":true} + ] + })); + assert!(indexed.get("graphicsAdapters").is_some()); + assert_eq!(indexed["graphicsAdapters"][0]["codecs"], json!([])); + assert_eq!( + indexed["graphicsAdapters"][1]["codecs"], + json!(["h264", "h265"]) + ); + assert_eq!(indexed["graphicsAdapters"][1]["h265Main10"], true); + let rendered = indexed.to_string(); + assert!(rendered.contains("NVIDIA GeForce MX110")); + assert!(!rendered.contains("private-luid")); + assert!(!rendered.contains("alice")); + assert!(!rendered.contains("private-codec")); + } + + #[test] + fn stream_endpoint_evidence_excludes_endpoint_and_credential_values() { + let descriptor = json!({"usage":"14","appLevelProtocol":6,"protocol":1,"port":48322, + "ip":"private-host", "resourcePath":"rtsps://user:private-password@private-host:48322/private-path?token=private-token"}); + let evidence = stream_endpoint_evidence( + &json!({"sessionId":"private-session", "transportMode":"webrtc", + "signalingUrl":"wss://private-host/nvst/?token=private-token", "connectionInfo":vec![descriptor; 20]}), + ); + assert_eq!(evidence["connectionCount"], 20); + assert_eq!(evidence["connections"].as_array().unwrap().len(), 16); + assert_eq!(evidence["connections"][0]["resourceKind"], "rtsps"); + assert_eq!(evidence["connections"][0]["appLevelProtocol"], 6); + assert_eq!(evidence["connections"][0]["usage"], 14); + assert_eq!(evidence["signalingPort"], 443); + assert!(!evidence.to_string().contains("private")); + let invalid = stream_endpoint_evidence(&json!({"transportMode":"private-token", + "connectionInfo":[{"usage":"private-token","port":99999999,"resourcePath":"private-token"}]})); + assert_eq!(invalid["connections"][0]["port"], Value::Null); + assert!(!invalid.to_string().contains("private")); + } + + #[test] + fn stream_profile_evidence_preserves_only_codec_and_color_fields() { + let session = json!({ + "sessionId":"private-session", "accessToken":"private-token", + "negotiatedStreamProfile":{"codec":"H265", "codecSource":"request", "colorQuality":"10bit_420", "enableHdr":true}, + "requestedStreamingFeatures":{"codec":"2", "bitDepth":1, "chromaFormat":0, "token":"private-token"}, + "finalizedStreamingFeatures":{"bitDepth":null, "chromaFormat":1, "password":"private-password"} + }); + let evidence = stream_profile_evidence(&session); + assert_eq!(evidence["codec"], "H265"); + assert_eq!(evidence["codecSource"], "request"); + assert_eq!(evidence["colorQuality"], "10bit_420"); + assert_eq!(evidence["enableHdr"], true); + assert_eq!( + evidence["requestedStreamingFeatures"], + json!({"codec":2,"bitDepth":1,"chromaFormat":0}) + ); + assert_eq!( + evidence["finalizedStreamingFeatures"], + json!({"bitDepth":null,"chromaFormat":1}) + ); + assert!(!evidence.to_string().contains("private")); + let invalid = stream_profile_evidence(&json!({ + "negotiatedStreamProfile":{"codec":"private-token", "codecSource":"private-token", "colorQuality":"private-token", "enableHdr":"private-token"}, + "requestedStreamingFeatures":{"codec":"private-token","bitDepth":{},"chromaFormat":["private-token"]} + })); + assert_eq!(invalid["codec"], Value::Null); + assert_eq!(invalid["codecSource"], Value::Null); + assert_eq!(invalid["colorQuality"], Value::Null); + assert_eq!(invalid["enableHdr"], Value::Null); + assert!(!invalid.to_string().contains("private")); + } + + #[test] + fn runtime_failure_reasons_redact_bearer_values_and_quoted_credentials() { + for reason in [ + "probe failed Bearer private-value", + r#"probe failed {"Authorization": "Bearer private-value"}"#, + r#"probe failed access_token: "private-value""#, + r"probe failed device=C:\Users\Alice\video path=/Users/Alice/video", + ] { + let redacted = runtime_failure_reason(reason); + assert!(redacted.starts_with("probe failed")); + assert!(!redacted.contains("private-value")); + assert!(!redacted.contains("Alice")); + } + let evidence = native_runtime_evidence(&json!({"videoBackends":[{ + "backend":"software", "platform":"cross-platform", "available":true, + "codecs":[{"codec":"h265","available":true},{"codec":"hevc","available":true}] + }]})); + assert_eq!(evidence["videoBackends"][0]["platform"], "cross-platform"); + assert_eq!( + evidence["videoBackends"][0]["codecs"], + json!([{"codec":"h265","available":true}]) + ); + } + + #[test] + fn diagnostics_export_includes_native_probe_failures_before_stream_start() { + let directory = env::temp_dir().join(format!("opennow-probe-diagnostics-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + let streamer = crate::streamer::StreamerService::new(); + let runtime = json!({ + "streamer": streamer.acceptance_snapshot(), + "nativeRuntime": native_runtime_evidence(&json!({ + "protocolVersion": 7, "supportsVideoDecode": false, + "videoBackends": [{"backend":"v4l2", "available":false, + "reason":"HEVC probe failed", "codecs":[{"codec":"h265", "available":false, + "reason":"MEDIA_IOC_G_TOPOLOGY failed"}]}] + })) + }); + assert_eq!(runtime["streamer"]["status"], "stopped"); + let exported = service.export_with_runtime(Some(&runtime)).unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert!(text.contains("\"nativeRuntime\"")); + assert!(text.contains("\"backend\": \"v4l2\"")); + assert!(text.contains("\"available\": false")); + assert!(text.contains("MEDIA_IOC_G_TOPOLOGY failed")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn embedded_drop_evidence_preserves_only_bounded_typed_counters() { + let evidence = embedded_drop_evidence(&json!({ + "embeddedStream": {"drops": { + "videoDropCount": 7, "audioDiscardedMs": 50.5, + "audioPacketDropCount": 2, "callbackDropCount": 3, + "otherQueueDropCount": 1, "token": "must-not-be-exported" + }}, + "lastSessionReport": {"drops": { + "videoDropCount": -1, "audioDiscardedMs": "50", + "audioPacketDropCount": 1.5, "callbackDropCount": 1e100, + "otherQueueDropCount": null + }, "sessionId": "must-not-be-exported"} + })); + assert_eq!( + evidence["embeddedStream"]["drops"], + json!({ + "videoDropCount": 7, "audioDiscardedMs": 50.5, + "audioPacketDropCount": 2, "callbackDropCount": 3, + "otherQueueDropCount": 1 + }) + ); + assert_eq!(evidence["lastSessionReport"], json!({"drops": {}})); + assert_eq!(embedded_drop_evidence(&json!({})), json!({})); + } + + #[test] + fn diagnostics_export_retains_embedded_drops_after_native_stop() { + let directory = env::temp_dir().join(format!("opennow-drop-diagnostics-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + let runtime = json!({ + "streamer": {"status": "stopped", "queueDropCount": 0}, + "shell": embedded_drop_evidence(&json!({ + "embeddedStream": {"drops": {"videoDropCount": 7, "audioDiscardedMs": 50}}, + "lastSessionReport": {"drops": {"videoDropCount": 7, "audioDiscardedMs": 50}} + })) + }); + let exported = service.export_with_runtime(Some(&runtime)).unwrap(); + let text = fs::read_to_string(exported["path"].as_str().unwrap()).unwrap(); + assert!(text.contains("\"videoDropCount\": 7")); + assert!(text.contains("\"audioDiscardedMs\": 50")); + assert!(text.contains("\"lastSessionReport\"")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn snapshot_includes_bounded_native_and_qt_handshake_entries() { + let directory = env::temp_dir().join(format!("opennow-handshake-snapshot-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + let log = (1..=250) + .map(|i| format!("{i} qt-native delivered type=ready user@example.com\n")) + .collect::(); + fs::write(directory.join("diagnostics/qt-native.log"), log).unwrap(); + let snapshot = service.snapshot(); + let entries = snapshot["entries"].as_array().unwrap(); + assert_eq!(entries.len(), 60); + assert_eq!(entries[0]["atMs"], "250"); + assert!( + entries[0]["detail"] + .as_str() + .unwrap() + .contains("type=ready") + ); + assert!(!snapshot.to_string().contains("user@example.com")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn live_acceptance_export_is_atomic_machine_readable_and_secret_free() { + let directory = env::temp_dir().join(format!("opennow-live-acceptance-{}", now_ms())); + let service = DiagnosticsService::new(&directory).unwrap(); + let manifest = json!({ + "schemaVersion":1, + "kind":"opennow.live-acceptance", + "platform":{"os":"linux","cpuArchitecture":"x86_64","windowSystem":"wayland"}, + "stream":{"status":"streaming","firstFrameLatencyMs":1420}, + "media":{"complete":true} + }); + let exported = service.export_acceptance(&manifest).unwrap(); + let bytes = fs::read(exported["path"].as_str().unwrap()).unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(parsed, manifest); + assert_eq!(exported["redacted"], true); + assert!( + service + .export_acceptance(&json!({ + "schemaVersion":1, + "kind":"opennow.live-acceptance", + "sessionId":"secret-session" + })) + .is_err() + ); + assert!( + service + .export_acceptance(&json!({ + "schemaVersion":1, + "kind":"opennow.live-acceptance", + "message":"/home/alice/private" + })) + .is_err() + ); + let _ = fs::remove_dir_all(directory); + } +} diff --git a/external/opennow/native/opennow-core/src/discord.rs b/external/opennow/native/opennow-core/src/discord.rs new file mode 100644 index 000000000..25d6186a0 --- /dev/null +++ b/external/opennow/native/opennow-core/src/discord.rs @@ -0,0 +1,2113 @@ +use rand::RngCore; +use serde_json::{Value, json}; +use std::io; +#[cfg(unix)] +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender, TryRecvError}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError}; +use std::thread; +use std::time::{Duration, Instant}; + +const CLIENT_ID: &str = "1479944467112001669"; +const MAX_FRAME_BYTES: usize = 1024 * 1024; +const MAX_READ_CHUNK: usize = 64 * 1024; +const MAX_WAITERS: usize = 8; +const OPCODE_HANDSHAKE: u32 = 0; +const OPCODE_FRAME: u32 = 1; +const OPCODE_CLOSE: u32 = 2; +const OPCODE_PING: u32 = 3; +const OPCODE_PONG: u32 = 4; +const REPLY_TIMEOUT: Duration = Duration::from_secs(2); +const SERVICE_TICK: Duration = Duration::from_millis(500); +const SERVICE_FRAME_BUDGET: usize = 32; +const READ_SLICE: Duration = Duration::from_millis(100); +const ACKNOWLEDGE_TIMEOUT: Duration = Duration::from_millis(1000); +const WRITE_DEADLINE: Duration = Duration::from_secs(3); +#[cfg(windows)] +const CANCEL_REAP_TIMEOUT: Duration = Duration::from_secs(2); +const RETRY_SCHEDULE: [Duration; 7] = [ + Duration::from_millis(250), + Duration::from_millis(500), + Duration::from_secs(1), + Duration::from_secs(2), + Duration::from_secs(4), + Duration::from_secs(8), + Duration::from_secs(15), +]; +const UNAVAILABLE: &str = "Discord presence worker is unavailable"; +const WORKER_STOPPED: &str = "Discord presence worker stopped"; + +pub struct DiscordService { + directories: Vec, + mailbox: OnceLock>>, + worker_alive: Arc, + abandoned: Arc, + #[cfg(test)] + iterations: Arc, +} + +impl DiscordService { + pub fn new() -> Self { + Self::with_directories(candidate_directories()) + } + + fn with_directories(directories: Vec) -> Self { + Self { + directories, + mailbox: OnceLock::new(), + worker_alive: Arc::new(AtomicBool::new(true)), + abandoned: Arc::new(AtomicBool::new(false)), + #[cfg(test)] + iterations: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + } + } + + pub fn sync(&self, params: &Value) -> Result { + if params["enabled"].as_bool() != Some(true) { + return self.clear(); + } + let activity = activity_payload(params)?; + let signature = serde_json::to_string(&activity).map_err(|error| error.to_string())?; + self.request(Intent::Set(Desired { + activity, + signature, + })) + } + + pub fn clear(&self) -> Result { + self.request(Intent::Clear) + } + + fn request(&self, intent: Intent) -> Result { + let Some(mailbox) = self.mailbox() else { + return Err(UNAVAILABLE.to_owned()); + }; + if !self.worker_alive.load(Ordering::SeqCst) { + return Err(UNAVAILABLE.to_owned()); + } + if self.abandoned.load(Ordering::SeqCst) { + return Ok(terminal_status()); + } + let (ack, response) = mpsc::channel(); + { + let mut state = mailbox.state(); + match intent { + Intent::Set(desired) => state.desired = Some(desired), + Intent::Clear => state.desired = None, + } + if state.waiters.len() >= MAX_WAITERS { + state.waiters.remove(0); + } + state.waiters.push(ack); + } + let _ = mailbox.wake.try_send(()); + match response.recv_timeout(ACKNOWLEDGE_TIMEOUT) { + Ok(status) => Ok(status), + Err(_) => Ok(pending_status()), + } + } + + fn mailbox(&self) -> &Option> { + self.mailbox.get_or_init(|| { + let (wake, woken) = mpsc::sync_channel(1); + let state = Arc::new(Mutex::new(MailState::default())); + let mailbox = Arc::new(Mailbox { + state: Arc::clone(&state), + wake, + }); + let directories = self.directories.clone(); + let alive = Arc::clone(&self.worker_alive); + let abandoned = Arc::clone(&self.abandoned); + #[cfg(test)] + let iterations = Arc::clone(&self.iterations); + match thread::Builder::new() + .name("opennow-discord".to_owned()) + .spawn(move || { + let _guard = WorkerGuard(alive); + #[cfg(test)] + Worker::new(directories, state, woken, abandoned, iterations).run(); + #[cfg(not(test))] + Worker::new(directories, state, woken, abandoned).run(); + }) { + Ok(_) => Some(mailbox), + Err(error) => { + eprintln!("opennow-core: discord presence worker failed to start: {error}"); + self.worker_alive.store(false, Ordering::SeqCst); + None + } + } + }) + } +} + +struct WorkerGuard(Arc); + +impl Drop for WorkerGuard { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +enum Intent { + Set(Desired), + Clear, +} + +#[derive(Clone)] +struct Desired { + activity: Value, + signature: String, +} + +struct Mailbox { + state: Arc>, + wake: SyncSender<()>, +} + +#[derive(Default)] +struct MailState { + desired: Option, + waiters: Vec>, +} + +impl Mailbox { + fn state(&self) -> MutexGuard<'_, MailState> { + mail_state(&self.state) + } + + #[cfg(test)] + fn desired(&self) -> Option { + self.state().desired.clone() + } +} + +fn mail_state(state: &Arc>) -> MutexGuard<'_, MailState> { + state.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[derive(Debug)] +enum Failure { + NotRunning, + TimedOut, + HandshakeRejected(String), + Rejected(String), + Closed(String), + Corrupt(String), + Terminal, +} + +impl Failure { + fn status(&self) -> Value { + match self { + Self::NotRunning => json!({ + "connected":false,"message":"Discord is not running","pending":true + }), + Self::TimedOut => pending_status(), + Self::HandshakeRejected(message) => { + json!({"connected":false,"applied":false,"pending":true,"message":message}) + } + Self::Rejected(message) => { + json!({"connected":true,"applied":false,"pending":true,"message":message}) + } + Self::Closed(message) | Self::Corrupt(message) => { + json!({"connected":false,"pending":true,"message":message}) + } + Self::Terminal => terminal_status(), + } + } +} + +fn terminal_status() -> Value { + json!({ + "connected":false,"applied":false,"terminal":true,"pending":false, + "message":"Discord IPC was disabled after the transport could not cancel outstanding I/O" + }) +} + +fn pending_status() -> Value { + json!({"connected":false,"timedOut":true,"pending":true}) +} + +enum ServiceOutcome { + Continue, + Shutdown, +} + +struct Snapshot { + desired: Option, + waiters: Vec>, +} + +struct Worker { + directories: Vec, + state: Arc>, + wake: Receiver<()>, + abandoned: Arc, + connection: Option, + retry_at: Option, + retry_index: usize, + #[cfg(test)] + iterations: Arc, +} + +impl Worker { + fn new( + directories: Vec, + state: Arc>, + wake: Receiver<()>, + abandoned: Arc, + #[cfg(test)] iterations: Arc, + ) -> Self { + Self { + directories, + state, + wake, + abandoned, + connection: None, + retry_at: None, + retry_index: 0, + #[cfg(test)] + iterations, + } + } + + fn run(mut self) { + loop { + #[cfg(test)] + self.iterations.fetch_add(1, Ordering::Relaxed); + let tick = Instant::now() + SERVICE_TICK; + let snapshot = self.snapshot(); + if !snapshot.waiters.is_empty() || self.retry_due(snapshot.desired.is_some()) { + let status = self.converge(snapshot.desired); + for waiter in snapshot.waiters { + let _ = waiter.send(status.clone()); + } + } + if self.abandoned.load(Ordering::SeqCst) { + self.connection = None; + self.terminate(Failure::Terminal); + } + let outcome = self.service_connection(tick); + let desired_pending = mail_state(&self.state).desired.is_some(); + if matches!(outcome, ServiceOutcome::Shutdown) || !self.wait(tick, desired_pending) { + break; + } + } + self.shutdown(); + } + + fn snapshot(&self) -> Snapshot { + let mut state = mail_state(&self.state); + Snapshot { + desired: state.desired.clone(), + waiters: std::mem::take(&mut state.waiters), + } + } + + fn shutdown(&mut self) { + self.connection = None; + let message = json!({"connected":false,"pending":true,"message":WORKER_STOPPED}); + for waiter in std::mem::take(&mut mail_state(&self.state).waiters) { + let _ = waiter.send(message.clone()); + } + } + + fn converged(&self, desired: &Desired) -> bool { + self.connection + .as_ref() + .and_then(|connection| connection.applied.as_deref()) + == Some(desired.signature.as_str()) + } + + fn converge(&mut self, desired: Option) -> Value { + let Some(desired) = desired else { + return self.release(); + }; + if self.converged(&desired) { + return json!({"connected":true,"unchanged":true}); + } + if self.abandoned.load(Ordering::SeqCst) { + return Failure::Terminal.status(); + } + if self.connection.is_none() { + match Connection::connect(&self.directories, &self.abandoned) { + Ok(connection) => self.connection = Some(connection), + Err(failure) => { + if matches!(failure, Failure::Terminal) { + return self.terminate(failure); + } + self.schedule_retry(); + return failure.status(); + } + } + } + let outcome = match self.connection.as_mut() { + Some(connection) => connection.set_activity(&desired.activity), + None => return Failure::NotRunning.status(), + }; + if outcome.is_err() && self.abandoned.load(Ordering::SeqCst) { + self.connection = None; + return self.terminate(Failure::Terminal); + } + match outcome { + Ok(()) => { + if let Some(connection) = self.connection.as_mut() { + connection.applied = Some(desired.signature); + } + self.retry_at = None; + self.retry_index = 0; + json!({"connected":true,"unchanged":false}) + } + Err(failure) => { + self.connection = None; + if matches!(failure, Failure::Terminal) { + return self.terminate(failure); + } + self.schedule_retry(); + failure.status() + } + } + } + + fn terminate(&mut self, failure: Failure) -> Value { + self.abandoned.store(true, Ordering::SeqCst); + self.retry_at = None; + self.retry_index = 0; + let mut state = mail_state(&self.state); + state.desired = None; + failure.status() + } + + fn release(&mut self) -> Value { + let Some(mut connection) = self.connection.take() else { + return json!({"connected":false,"cleared":true}); + }; + if connection.applied.is_some() { + let _ = connection.set_activity(&Value::Null); + } + drop(connection); + self.retry_at = None; + self.retry_index = 0; + json!({"connected":true,"cleared":true}) + } + + fn retry_due(&self, desired_pending: bool) -> bool { + if self.connection.is_some() || !desired_pending || self.abandoned.load(Ordering::SeqCst) { + return false; + } + match self.retry_at { + Some(at) => Instant::now() >= at, + None => true, + } + } + + fn schedule_retry(&mut self) { + let delay = RETRY_SCHEDULE[self.retry_index.min(RETRY_SCHEDULE.len() - 1)]; + self.retry_index = self.retry_index.saturating_add(1); + self.retry_at = Some(Instant::now() + delay); + } + + fn service_connection(&mut self, tick: Instant) -> ServiceOutcome { + let Some(mut connection) = self.connection.take() else { + return ServiceOutcome::Continue; + }; + let mut failure = None; + let mut shutdown = false; + for _ in 0..SERVICE_FRAME_BUDGET { + if Instant::now() >= tick { + break; + } + match self.wake.try_recv() { + Ok(()) => break, + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + shutdown = true; + break; + } + } + let slice = (Instant::now() + READ_SLICE).min(tick); + match connection.poll(slice) { + Ok(Some((opcode, value))) => match opcode { + OPCODE_FRAME => {} + OPCODE_PING => { + if let Err(error) = connection.send(OPCODE_PONG, &value) { + failure = Some(error); + break; + } + } + OPCODE_PONG => {} + OPCODE_CLOSE => { + failure = Some(close_failure(&value)); + break; + } + _ => { + failure = Some(Failure::Corrupt( + "Discord sent an unexpected RPC opcode".to_owned(), + )); + break; + } + }, + Ok(None) => break, + Err(error) => { + failure = Some(error); + break; + } + } + } + if failure.is_none() { + self.connection = Some(connection); + } else if mail_state(&self.state).desired.is_some() { + self.schedule_retry(); + } + if shutdown { + ServiceOutcome::Shutdown + } else { + ServiceOutcome::Continue + } + } + + fn wait(&mut self, tick: Instant, desired_pending: bool) -> bool { + let bound = if self.connection.is_some() { + Some(tick.saturating_duration_since(Instant::now())) + } else if desired_pending && !self.abandoned.load(Ordering::SeqCst) { + Some( + self.retry_at + .unwrap_or_else(Instant::now) + .saturating_duration_since(Instant::now()), + ) + } else { + None + }; + match bound { + Some(bound) => match self.wake.recv_timeout(bound) { + Ok(()) | Err(RecvTimeoutError::Timeout) => true, + Err(RecvTimeoutError::Disconnected) => false, + }, + None => self.wake.recv().is_ok(), + } + } +} + +struct Connection { + stream: IpcStream, + scratch: Vec, + buffered: Vec, + applied: Option, +} + +impl Connection { + fn connect(directories: &[PathBuf], abandoned: &Arc) -> Result { + let stream = connect(directories, abandoned).map_err(|_| Failure::NotRunning)?; + let mut connection = Self { + stream, + scratch: Vec::new(), + buffered: Vec::new(), + applied: None, + }; + connection.send(OPCODE_HANDSHAKE, &json!({"v":1,"client_id":CLIENT_ID}))?; + let deadline = Instant::now() + REPLY_TIMEOUT; + loop { + match connection.poll(deadline)? { + Some((OPCODE_FRAME, value)) => match value["evt"].as_str() { + Some("READY") => return Ok(connection), + Some("ERROR") => { + return Err(Failure::HandshakeRejected(protocol_error( + &value, + "Discord rejected the RPC handshake", + ))); + } + _ => {} + }, + Some((OPCODE_PING, value)) => connection.send(OPCODE_PONG, &value)?, + Some((OPCODE_PONG, _)) => {} + Some((OPCODE_CLOSE, value)) => return Err(close_failure(&value)), + Some(_) => { + return Err(Failure::Corrupt( + "Discord sent an unexpected RPC opcode".to_owned(), + )); + } + None => return Err(Failure::TimedOut), + } + } + } + + fn set_activity(&mut self, activity: &Value) -> Result<(), Failure> { + let activity = if activity.is_null() { + Value::Null + } else { + activity.clone() + }; + let mut nonce = [0_u8; 12]; + rand::rng().fill_bytes(&mut nonce); + let nonce = nonce + .iter() + .map(|value| format!("{value:02x}")) + .collect::(); + self.send( + OPCODE_FRAME, + &json!({ + "cmd":"SET_ACTIVITY", + "args":{"pid":std::process::id(),"activity":activity}, + "nonce":nonce + }), + )?; + let deadline = Instant::now() + REPLY_TIMEOUT; + loop { + match self.poll(deadline)? { + Some((OPCODE_FRAME, value)) => { + if value["evt"] == "ERROR" { + let error_nonce = value["nonce"].as_str(); + if error_nonce.is_none() || error_nonce == Some(nonce.as_str()) { + return Err(Failure::Rejected(protocol_error( + &value, + "Discord rejected the activity update", + ))); + } + continue; + } + if value["cmd"] == "SET_ACTIVITY" + && value["nonce"].as_str() == Some(nonce.as_str()) + { + return Ok(()); + } + } + Some((OPCODE_PING, value)) => self.send(OPCODE_PONG, &value)?, + Some((OPCODE_PONG, _)) => {} + Some((OPCODE_CLOSE, value)) => return Err(close_failure(&value)), + Some(_) => { + return Err(Failure::Corrupt( + "Discord sent an unexpected RPC opcode".to_owned(), + )); + } + None => return Err(Failure::TimedOut), + } + } + } + + fn send(&mut self, opcode: u32, value: &Value) -> Result<(), Failure> { + let body = + serde_json::to_vec(value).map_err(|error| Failure::Corrupt(error.to_string()))?; + if body.len() > MAX_FRAME_BYTES { + return Err(Failure::Corrupt( + "Discord RPC frame is too large".to_owned(), + )); + } + let mut frame = Vec::with_capacity(8 + body.len()); + frame.extend_from_slice(&opcode.to_le_bytes()); + frame.extend_from_slice(&(body.len() as u32).to_le_bytes()); + frame.extend_from_slice(&body); + write_frame(&mut self.stream, &frame, Instant::now() + WRITE_DEADLINE) + .map_err(|error| Failure::Closed(error.to_string())) + } + + fn poll(&mut self, deadline: Instant) -> Result, Failure> { + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + if let Some(frame) = take_frame(&mut self.buffered)? { + return Ok(Some(frame)); + } + match read_some(&mut self.stream, &mut self.scratch, remaining) + .map_err(|error| Failure::Closed(error.to_string()))? + { + ReadOutcome::Data(count) => self.buffered.extend_from_slice(&self.scratch[..count]), + ReadOutcome::TimedOut => {} + ReadOutcome::Eof => return Err(Failure::Closed(CLOSED_BY_DISCORD.to_owned())), + } + } + } +} + +const CLOSED_BY_DISCORD: &str = "Discord closed the connection"; + +fn close_failure(value: &Value) -> Failure { + let code = value["code"].as_u64().unwrap_or_default(); + let message = value["message"].as_str().unwrap_or_default(); + Failure::Closed(if message.is_empty() { + format!("Discord closed the connection (code {code})") + } else { + format!("Discord closed the connection (code {code}: {message})") + }) +} + +fn protocol_error(value: &Value, fallback: &str) -> String { + let message = value["data"]["message"].as_str().unwrap_or(fallback); + match value["data"]["code"].as_u64() { + Some(code) => format!("{message} (code {code})"), + None => message.to_owned(), + } +} + +fn take_frame(buffered: &mut Vec) -> Result, Failure> { + if buffered.len() < 8 { + return Ok(None); + } + let opcode = u32::from_le_bytes( + buffered[..4] + .try_into() + .expect("four-byte Discord RPC opcode"), + ); + let length = u32::from_le_bytes( + buffered[4..8] + .try_into() + .expect("four-byte Discord RPC length"), + ) as usize; + if length > MAX_FRAME_BYTES { + return Err(Failure::Corrupt( + "Discord RPC response is too large".to_owned(), + )); + } + if buffered.len() < 8 + length { + return Ok(None); + } + let value: Value = serde_json::from_slice(&buffered[8..8 + length]) + .map_err(|_| Failure::Corrupt("Invalid Discord RPC response".to_owned()))?; + buffered.drain(..8 + length); + Ok(Some((opcode, value))) +} + +fn activity_payload(params: &Value) -> Result { + let game_name = params["gameName"] + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| "Discord activity requires a game name".to_owned())?; + let kind = params["kind"].as_str().unwrap_or("streaming"); + let state = match kind { + "queued" => params["queuePosition"] + .as_u64() + .filter(|value| *value > 0) + .map(|value| format!("In queue (#{value})")) + .unwrap_or_else(|| "In queue".to_owned()), + "starting" => "Starting stream".to_owned(), + "streaming" => "Streaming via OpenNOW".to_owned(), + _ => return Err("Unsupported Discord activity kind".to_owned()), + }; + let mut activity = json!({ + "details": bounded(game_name, 128), + "state": state, + "instance": false + }); + if kind == "streaming" + && let Some(timestamp) = params["startTimestampMs"].as_u64() + && timestamp > 0 + { + activity["timestamps"] = json!({"start":timestamp / 1000}); + } + if let Some(image) = params["gameImageUrl"].as_str().filter(|value| { + value.starts_with("https://") && value.len() <= 2_048 && !value.contains(['\r', '\n']) + }) { + activity["assets"] = json!({ + "large_image": image, + "large_text": bounded(game_name, 128) + }); + } + Ok(activity) +} + +fn bounded(value: &str, limit: usize) -> String { + value.chars().take(limit).collect() +} + +fn candidate_directories() -> Vec { + #[cfg(unix)] + { + use std::env; + let mut directories = Vec::new(); + for name in ["XDG_RUNTIME_DIR", "TMPDIR", "TMP", "TEMP"] { + if let Some(path) = env::var_os(name).map(PathBuf::from) + && !path.as_os_str().is_empty() + && !directories.contains(&path) + { + directories.push(path); + } + } + let fallback = PathBuf::from("/tmp"); + if !directories.contains(&fallback) { + directories.push(fallback); + } + directories + } + #[cfg(windows)] + { + Vec::new() + } +} + +#[cfg(unix)] +type IpcStream = std::os::unix::net::UnixStream; + +#[cfg(unix)] +fn connect(directories: &[PathBuf], _abandoned: &Arc) -> io::Result { + for directory in directories { + for index in 0..10 { + let path = directory.join(format!("discord-ipc-{index}")); + if let Ok(stream) = IpcStream::connect(path) { + return Ok(stream); + } + } + } + Err(io::Error::new( + io::ErrorKind::NotFound, + "Discord IPC unavailable", + )) +} + +#[cfg(unix)] +fn write_frame(stream: &mut IpcStream, frame: &[u8], deadline: Instant) -> io::Result<()> { + let remaining = deadline + .saturating_duration_since(Instant::now()) + .max(Duration::from_millis(1)); + stream.set_write_timeout(Some(remaining))?; + stream.write_all(frame) +} + +#[cfg(unix)] +fn read_some( + stream: &mut IpcStream, + scratch: &mut Vec, + timeout: Duration, +) -> io::Result { + if scratch.len() < MAX_READ_CHUNK { + scratch.resize(MAX_READ_CHUNK, 0); + } + stream.set_read_timeout(Some(timeout))?; + match std::io::Read::read(stream, scratch) { + Ok(0) => Ok(ReadOutcome::Eof), + Ok(count) => Ok(ReadOutcome::Data(count)), + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => + { + Ok(ReadOutcome::TimedOut) + } + Err(error) => Err(error), + } +} + +enum ReadOutcome { + Data(usize), + TimedOut, + Eof, +} + +#[cfg(windows)] +struct IpcStream { + handle: windows_sys::Win32::Foundation::HANDLE, + pending_read: Option, + pending_write: Option, + abandoned: Arc, +} + +#[cfg(windows)] +struct PendingIo { + overlapped: Box, + buffer: Vec, + event: windows_sys::Win32::Foundation::HANDLE, +} + +#[cfg(windows)] +fn close_pending(pending: PendingIo) -> Vec { + use windows_sys::Win32::Foundation::CloseHandle; + + let PendingIo { + overlapped, + buffer, + event, + } = pending; + unsafe { CloseHandle(event) }; + drop(overlapped); + buffer +} + +#[cfg(windows)] +fn abandon_pending(pending: PendingIo) { + std::mem::forget(pending); +} + +#[cfg(windows)] +fn finish_overlapped( + handle: windows_sys::Win32::Foundation::HANDLE, + pending: &PendingIo, +) -> (bool, u32, Option) { + use windows_sys::Win32::System::IO::GetOverlappedResult; + + let mut transferred = 0_u32; + let success = + unsafe { GetOverlappedResult(handle, &*pending.overlapped, &mut transferred, 0) } != 0; + let raw_error = if success { + None + } else { + io::Error::last_os_error().raw_os_error() + }; + (success, transferred, raw_error) +} + +#[cfg(windows)] +fn reap_cancelled( + handle: windows_sys::Win32::Foundation::HANDLE, + pending: PendingIo, +) -> Result { + use windows_sys::Win32::Foundation::WAIT_OBJECT_0; + use windows_sys::Win32::System::Threading::WaitForSingleObject; + + let milliseconds = CANCEL_REAP_TIMEOUT.as_millis().min(u32::MAX as u128) as u32; + if unsafe { WaitForSingleObject(pending.event, milliseconds) } != WAIT_OBJECT_0 { + return Err(pending); + } + let (success, transferred, raw_error) = finish_overlapped(handle, &pending); + close_pending(pending); + Ok(classify_overlapped(success, transferred, raw_error)) +} + +#[cfg(any(windows, test))] +const ERROR_OPERATION_ABORTED_CODE: i32 = 995; + +#[cfg(any(windows, test))] +#[derive(Debug, PartialEq)] +enum OverlappedStatus { + Completed(usize), + Aborted, + Failed(i32), +} + +#[cfg(any(windows, test))] +fn classify_overlapped( + success: bool, + transferred: u32, + raw_error: Option, +) -> OverlappedStatus { + if success { + return OverlappedStatus::Completed(transferred as usize); + } + match raw_error { + Some(code) if code == ERROR_OPERATION_ABORTED_CODE => OverlappedStatus::Aborted, + Some(code) => OverlappedStatus::Failed(code), + None => OverlappedStatus::Failed(0), + } +} + +#[cfg(windows)] +impl IpcStream { + fn start(&mut self, read: bool, mut buffer: Vec) -> io::Result, usize)>> { + use windows_sys::Win32::Foundation::ERROR_IO_PENDING; + use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; + use windows_sys::Win32::System::IO::OVERLAPPED; + use windows_sys::Win32::System::Threading::{CreateEventW, ResetEvent}; + + let event = unsafe { CreateEventW(std::ptr::null(), 1, 0, std::ptr::null()) }; + if event.is_null() { + return Err(io::Error::last_os_error()); + } + let mut overlapped: Box = Box::new(unsafe { std::mem::zeroed() }); + overlapped.hEvent = event; + unsafe { ResetEvent(event) }; + + let length = buffer.len() as u32; + let mut transferred = 0_u32; + let started = unsafe { + let overlapped_ptr = &mut *overlapped; + if read { + ReadFile( + self.handle, + buffer.as_mut_ptr(), + length, + &mut transferred, + overlapped_ptr, + ) + } else { + WriteFile( + self.handle, + buffer.as_ptr(), + length, + &mut transferred, + overlapped_ptr, + ) + } + }; + if started != 0 { + unsafe { windows_sys::Win32::Foundation::CloseHandle(event) }; + drop(overlapped); + return Ok(Some((buffer, transferred as usize))); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(ERROR_IO_PENDING as i32) { + unsafe { windows_sys::Win32::Foundation::CloseHandle(event) }; + drop((overlapped, buffer)); + return Err(error); + } + let pending = PendingIo { + overlapped, + buffer, + event, + }; + if read { + self.pending_read = Some(pending); + } else { + self.pending_write = Some(pending); + } + Ok(None) + } + + fn take_pending(&mut self, read: bool) -> io::Result { + let pending = if read { + self.pending_read.take() + } else { + self.pending_write.take() + }; + pending.ok_or_else(|| io::Error::other("no pending Discord IPC operation")) + } + + fn restore_pending(&mut self, read: bool, pending: PendingIo) { + if read { + self.pending_read = Some(pending); + } else { + self.pending_write = Some(pending); + } + } + + fn poll_pending( + &mut self, + read: bool, + deadline: Instant, + ) -> io::Result, usize)>> { + use windows_sys::Win32::Foundation::WAIT_OBJECT_0; + use windows_sys::Win32::System::Threading::WaitForSingleObject; + + let pending = self.take_pending(read)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + let milliseconds = remaining.as_millis().min(u32::MAX as u128) as u32; + if unsafe { WaitForSingleObject(pending.event, milliseconds) } != WAIT_OBJECT_0 { + self.restore_pending(read, pending); + return Ok(None); + } + let (success, transferred, raw_error) = finish_overlapped(self.handle, &pending); + match classify_overlapped(success, transferred, raw_error) { + OverlappedStatus::Completed(count) => Ok(Some((close_pending(pending), count))), + OverlappedStatus::Aborted => { + close_pending(pending); + Err(io::Error::new( + io::ErrorKind::TimedOut, + "Discord IPC operation was cancelled", + )) + } + OverlappedStatus::Failed(code) => { + close_pending(pending); + Err(io::Error::from_raw_os_error(code)) + } + } + } + + fn cancel_pending(&mut self, read: bool) -> io::Result { + use windows_sys::Win32::System::IO::CancelIoEx; + + let pending = self.take_pending(read)?; + unsafe { CancelIoEx(self.handle, &*pending.overlapped) }; + match reap_cancelled(self.handle, pending) { + Ok(status) => Ok(status), + Err(pending) => { + self.abandoned.store(true, Ordering::SeqCst); + abandon_pending(pending); + Err(io::Error::new( + io::ErrorKind::TimedOut, + "Discord IPC operation could not be cancelled", + )) + } + } + } +} + +#[cfg(windows)] +impl Drop for IpcStream { + fn drop(&mut self) { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::System::IO::CancelIoEx; + + for pending in [self.pending_read.take(), self.pending_write.take()] + .into_iter() + .flatten() + { + unsafe { CancelIoEx(self.handle, &*pending.overlapped) }; + if let Err(pending) = reap_cancelled(self.handle, pending) { + self.abandoned.store(true, Ordering::SeqCst); + abandon_pending(pending); + } + } + unsafe { CloseHandle(self.handle) }; + } +} + +#[cfg(windows)] +fn connect(_directories: &[PathBuf], abandoned: &Arc) -> io::Result { + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::IntoRawHandle; + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; + + for index in 0..10 { + let path = format!(r"\\?\pipe\discord-ipc-{index}"); + let opened = std::fs::OpenOptions::new() + .read(true) + .write(true) + .custom_flags(FILE_FLAG_OVERLAPPED) + .open(path); + let Ok(file) = opened else { + continue; + }; + let handle = file.into_raw_handle() as isize as _; + if handle == INVALID_HANDLE_VALUE { + continue; + } + return Ok(IpcStream { + handle, + pending_read: None, + pending_write: None, + abandoned: Arc::clone(abandoned), + }); + } + Err(io::Error::new( + io::ErrorKind::NotFound, + "Discord IPC unavailable", + )) +} + +#[cfg(windows)] +fn deliver_read(scratch: &mut Vec, buffer: Vec, count: usize) -> io::Result { + if count == 0 { + return Ok(ReadOutcome::Eof); + } + if scratch.len() < count { + scratch.resize(count, 0); + } + scratch[..count].copy_from_slice(&buffer[..count]); + Ok(ReadOutcome::Data(count)) +} + +#[cfg(windows)] +fn read_some( + stream: &mut IpcStream, + scratch: &mut Vec, + timeout: Duration, +) -> io::Result { + use windows_sys::Win32::Foundation::{ERROR_BROKEN_PIPE, ERROR_NO_DATA}; + + let deadline = Instant::now() + timeout; + if stream.pending_read.is_none() + && let Some((buffer, count)) = stream.start(true, vec![0_u8; MAX_READ_CHUNK])? + { + return deliver_read(scratch, buffer, count); + } + match stream.poll_pending(true, deadline) { + Ok(Some((buffer, count))) => deliver_read(scratch, buffer, count), + Ok(None) => Ok(ReadOutcome::TimedOut), + Err(error) => match error.raw_os_error() { + Some(code) if code == ERROR_BROKEN_PIPE as i32 || code == ERROR_NO_DATA as i32 => { + Ok(ReadOutcome::Eof) + } + _ => Err(error), + }, + } +} + +#[cfg(windows)] +fn write_frame(stream: &mut IpcStream, frame: &[u8], deadline: Instant) -> io::Result<()> { + let mut written = 0; + while written < frame.len() { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "Discord IPC write timed out", + )); + } + if stream.pending_write.is_none() { + let chunk = frame[written..].to_vec(); + if let Some((_buffer, count)) = stream.start(false, chunk)? { + if count == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "Discord IPC write made no progress", + )); + } + written += count; + continue; + } + } + match stream.poll_pending(false, deadline)? { + Some((_buffer, count)) => { + if count == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "Discord IPC write made no progress", + )); + } + written += count; + } + None => match stream.cancel_pending(false)? { + OverlappedStatus::Completed(count) => { + written += count; + } + OverlappedStatus::Aborted => { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "Discord IPC write timed out", + )); + } + OverlappedStatus::Failed(code) => { + return Err(io::Error::from_raw_os_error(code)); + } + }, + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn activity_payload_is_bounded_and_privacy_scoped() { + let payload = activity_payload(&json!({ + "gameName":"Example", + "kind":"queued", + "queuePosition":14, + "gameImageUrl":"http://unsafe.example/image" + })) + .unwrap(); + assert_eq!(payload["state"], "In queue (#14)"); + assert!(payload.get("assets").is_none()); + assert!(activity_payload(&json!({"gameName":"X","kind":"unknown"})).is_err()); + } + + #[test] + fn streaming_activity_reports_second_timestamps() { + let payload = activity_payload(&json!({ + "gameName":"Example", + "kind":"streaming", + "startTimestampMs":1_528_245_984_000_u64 + })) + .unwrap(); + assert_eq!(payload["timestamps"]["start"], 1_528_245_984_u64); + let without_start = activity_payload(&json!({ + "gameName":"Example", + "kind":"streaming", + "startTimestampMs":0 + })) + .unwrap(); + assert!(without_start.get("timestamps").is_none()); + } + + fn frame_bytes(opcode: u32, value: &Value) -> Vec { + let body = serde_json::to_vec(value).unwrap(); + let mut frame = Vec::new(); + frame.extend_from_slice(&opcode.to_le_bytes()); + frame.extend_from_slice(&(body.len() as u32).to_le_bytes()); + frame.extend_from_slice(&body); + frame + } + + #[test] + fn discord_frames_are_little_endian_and_bounded() { + let frame = frame_bytes(OPCODE_FRAME, &json!({"evt":"READY"})); + assert_eq!( + u32::from_le_bytes(frame[..4].try_into().unwrap()), + OPCODE_FRAME + ); + let body = serde_json::to_vec(&json!({"evt":"READY"})).unwrap(); + assert_eq!( + u32::from_le_bytes(frame[4..8].try_into().unwrap()) as usize, + body.len() + ); + let mut buffered = frame; + let (opcode, value) = take_frame(&mut buffered).unwrap().unwrap(); + assert_eq!(opcode, OPCODE_FRAME); + assert_eq!(value["evt"], "READY"); + + let oversized = MAX_FRAME_BYTES + 1; + assert!(oversized <= u32::MAX as usize); + } + + #[test] + fn partial_frames_are_buffered_until_complete() { + let first = frame_bytes(OPCODE_FRAME, &json!({"evt":"READY"})); + let second = frame_bytes(OPCODE_FRAME, &json!({"cmd":"SET_ACTIVITY"})); + + let mut buffered = Vec::new(); + buffered.extend_from_slice(&first[..3]); + assert!(take_frame(&mut buffered).unwrap().is_none()); + buffered.extend_from_slice(&first[3..first.len() - 2]); + assert!(take_frame(&mut buffered).unwrap().is_none()); + buffered.extend_from_slice(&first[first.len() - 2..]); + buffered.extend_from_slice(&second); + + let (opcode, value) = take_frame(&mut buffered).unwrap().unwrap(); + assert_eq!(opcode, OPCODE_FRAME); + assert_eq!(value["evt"], "READY"); + let (opcode, value) = take_frame(&mut buffered).unwrap().unwrap(); + assert_eq!(opcode, OPCODE_FRAME); + assert_eq!(value["cmd"], "SET_ACTIVITY"); + assert!(buffered.is_empty()); + } + + #[test] + fn oversized_frames_are_refused() { + let mut buffered = Vec::new(); + buffered.extend_from_slice(&OPCODE_FRAME.to_le_bytes()); + buffered.extend_from_slice(&((MAX_FRAME_BYTES + 1) as u32).to_le_bytes()); + assert!(matches!( + take_frame(&mut buffered), + Err(Failure::Corrupt(_)) + )); + } + + #[test] + fn a_panicking_exchange_still_releases_the_in_flight_marker() { + let marker = Arc::new(AtomicBool::new(true)); + drop(WorkerGuard(Arc::clone(&marker))); + assert!(!marker.load(Ordering::SeqCst)); + } + + #[test] + fn overlapped_completion_is_classified_by_documented_outcome() { + assert_eq!( + classify_overlapped(true, 42, None), + OverlappedStatus::Completed(42) + ); + assert_eq!( + classify_overlapped(false, 0, Some(ERROR_OPERATION_ABORTED_CODE)), + OverlappedStatus::Aborted + ); + assert_eq!( + classify_overlapped(false, 0, Some(6)), + OverlappedStatus::Failed(6) + ); + assert_eq!( + classify_overlapped(false, 0, Some(ERROR_OPERATION_ABORTED_CODE)), + OverlappedStatus::Aborted + ); + } + + #[test] + fn a_cancelled_operation_that_completed_still_delivers_its_bytes() { + assert_eq!( + classify_overlapped(true, 7, None), + OverlappedStatus::Completed(7) + ); + } + + #[test] + fn the_mailbox_keeps_one_intent_and_bounds_its_waiters() { + let (wake, _woken) = mpsc::sync_channel(1); + let mailbox = Mailbox { + state: Arc::new(Mutex::new(MailState::default())), + wake, + }; + for position in 0..(MAX_WAITERS as u64 * 3) { + let mut state = mailbox.state(); + state.desired = Some(Desired { + activity: json!({"state":position}), + signature: position.to_string(), + }); + if state.waiters.len() >= MAX_WAITERS { + state.waiters.remove(0); + } + state.waiters.push(mpsc::channel().0); + } + assert_eq!(mail_state(&mailbox.state).waiters.len(), MAX_WAITERS); + assert_eq!( + mailbox.desired().unwrap().signature, + (MAX_WAITERS as u64 * 3 - 1).to_string() + ); + } + + #[cfg(unix)] + mod fake_ipc { + use super::*; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::sync::Mutex; + use tempfile::TempDir; + + struct FakeDiscord { + directory: TempDir, + state: Arc>, + streams: Arc>>, + running: Arc, + accept: Option>, + } + + #[derive(Default)] + struct FakeState { + connections: usize, + open: usize, + peak_open: usize, + handshakes: usize, + applied: Option, + pongs: usize, + reject_handshake: bool, + reject_activity: bool, + silent: bool, + split_replies: bool, + interleave_events: bool, + foreign_error: bool, + delay_ready: bool, + } + + impl FakeDiscord { + fn start(directory: TempDir) -> Self { + let mut fake = Self { + directory, + state: Arc::new(Mutex::new(FakeState::default())), + streams: Arc::new(Mutex::new(Vec::new())), + running: Arc::new(AtomicBool::new(true)), + accept: None, + }; + fake.bind(); + fake + } + + fn bind(&mut self) { + let listener = + UnixListener::bind(self.directory.path().join("discord-ipc-0")).unwrap(); + listener.set_nonblocking(true).unwrap(); + let state = Arc::clone(&self.state); + let streams = Arc::clone(&self.streams); + let running = Arc::clone(&self.running); + self.accept = Some(thread::spawn(move || { + while running.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + let state = Arc::clone(&state); + let streams = Arc::clone(&streams); + let running = Arc::clone(&running); + thread::spawn(move || serve(stream, state, streams, running)); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(2)); + } + Err(_) => break, + } + } + })); + } + + fn directories(&self) -> Vec { + vec![self.directory.path().to_path_buf()] + } + + fn state(&self) -> std::sync::MutexGuard<'_, FakeState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn send_frame(&self, opcode: u32, value: &Value) { + if let Some(stream) = self.streams.lock().unwrap().last_mut() { + let _ = stream.write_all(&frame_bytes(opcode, value)); + } + } + + fn restart(&mut self) { + self.shutdown(); + self.state = Arc::new(Mutex::new(FakeState::default())); + self.streams = Arc::new(Mutex::new(Vec::new())); + self.running = Arc::new(AtomicBool::new(true)); + self.bind(); + } + + fn shutdown(&mut self) { + self.running.store(false, Ordering::SeqCst); + for stream in self.streams.lock().unwrap().drain(..) { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + if let Some(accept) = self.accept.take() { + let _ = accept.join(); + } + let _ = std::fs::remove_file(self.directory.path().join("discord-ipc-0")); + } + } + + impl Drop for FakeDiscord { + fn drop(&mut self) { + self.shutdown(); + } + } + + fn serve( + mut stream: UnixStream, + state: Arc>, + streams: Arc>>, + running: Arc, + ) { + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + streams.lock().unwrap().push(stream.try_clone().unwrap()); + { + let mut state = state.lock().unwrap(); + state.connections += 1; + state.open += 1; + state.peak_open = state.peak_open.max(state.open); + } + if let Some((OPCODE_HANDSHAKE, _)) = read_frame(&mut stream) { + let (reject, split, delay_ready) = { + let mut state = state.lock().unwrap(); + state.handshakes += 1; + ( + state.reject_handshake, + state.split_replies, + state.delay_ready, + ) + }; + if reject { + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"DISPATCH","evt":"ERROR", + "data":{"code":4000,"message":"Invalid Client ID"} + }), + split, + ); + } else { + if delay_ready { + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"DISPATCH","evt":"CURRENT_USER_UPDATE", + "data":{"id":"1"} + }), + split, + ); + } + if write_frame( + &mut stream, + OPCODE_FRAME, + &json!({"cmd":"DISPATCH","evt":"READY","data":{"v":1}}), + split, + ) + .is_ok() + { + while running.load(Ordering::SeqCst) { + let Some((opcode, value)) = read_frame(&mut stream) else { + break; + }; + match opcode { + OPCODE_FRAME => { + let (silent, reject, split, interleave, foreign) = { + let state = state.lock().unwrap(); + ( + state.silent, + state.reject_activity, + state.split_replies, + state.interleave_events, + state.foreign_error, + ) + }; + if silent { + continue; + } + if value["cmd"] == "SET_ACTIVITY" { + if foreign { + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"SET_ACTIVITY","evt":"ERROR", + "data":{"code":4002,"message":"Other command"}, + "nonce":"foreign-nonce" + }), + split, + ); + } + if interleave { + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"DISPATCH","evt":"CURRENT_USER_UPDATE", + "data":{"id":"1"} + }), + split, + ); + } + if reject { + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"SET_ACTIVITY","evt":"ERROR", + "data":{"code":4002,"message":"Invalid activity"}, + "nonce":value["nonce"] + }), + split, + ); + continue; + } + { + let mut state = state.lock().unwrap(); + let activity = value["args"]["activity"].clone(); + state.applied = if activity.is_null() { + None + } else { + Some(activity) + }; + } + let _ = write_frame( + &mut stream, + OPCODE_FRAME, + &json!({ + "cmd":"SET_ACTIVITY", + "data":value["args"]["activity"], + "evt":Value::Null, + "nonce":value["nonce"] + }), + split, + ); + } + } + OPCODE_PING => { + let split = state.lock().unwrap().split_replies; + let _ = + write_frame(&mut stream, OPCODE_PONG, &json!({}), split); + } + OPCODE_PONG => state.lock().unwrap().pongs += 1, + OPCODE_CLOSE => break, + _ => {} + } + } + } + } + } + let mut state = state.lock().unwrap_or_else(PoisonError::into_inner); + state.open -= 1; + state.applied = None; + } + + fn read_frame(stream: &mut UnixStream) -> Option<(u32, Value)> { + let mut header = [0_u8; 8]; + let mut filled = 0; + while filled < header.len() { + match std::io::Read::read(stream, &mut header[filled..]) { + Ok(0) => return None, + Ok(count) => filled += count, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => + { + continue; + } + Err(_) => return None, + } + } + let opcode = u32::from_le_bytes(header[..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..].try_into().unwrap()) as usize; + if length > MAX_FRAME_BYTES { + return None; + } + let mut body = vec![0_u8; length]; + std::io::Read::read_exact(stream, &mut body).ok()?; + serde_json::from_slice(&body) + .ok() + .map(|value| (opcode, value)) + } + + fn write_frame( + stream: &mut UnixStream, + opcode: u32, + value: &Value, + split: bool, + ) -> io::Result<()> { + let frame = frame_bytes(opcode, value); + if split { + stream.write_all(&frame[..3])?; + stream.flush()?; + thread::sleep(Duration::from_millis(20)); + } + stream.write_all(&frame[if split { 3 } else { 0 }..]) + } + + fn wait_until(condition: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if condition() { + return; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("timed out waiting for the fake Discord state to settle"); + } + + fn streaming_params() -> Value { + json!({ + "enabled":true, + "gameName":"Example Game", + "kind":"streaming", + "startTimestampMs":1_528_245_984_000_u64 + }) + } + + #[test] + fn presence_stays_visible_while_opennow_holds_the_session() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], true); + assert_eq!(status["unchanged"], false); + + wait_until(|| { + let state = fake.state(); + state.open == 1 && state.applied.is_some() + }); + + let repeated = service.sync(&streaming_params()).unwrap(); + assert_eq!(repeated["unchanged"], true); + let state = fake.state(); + assert_eq!(state.connections, 1); + assert_eq!(state.handshakes, 1); + assert_eq!(state.open, 1); + let applied = state.applied.as_ref().unwrap(); + assert_eq!(applied["details"], "Example Game"); + assert_eq!(applied["timestamps"]["start"], 1_528_245_984_u64); + } + + #[test] + fn a_discord_restart_reapplies_the_activity_without_a_new_request() { + let mut fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + + fake.restart(); + wait_until(|| { + let state = fake.state(); + state.open == 1 + && state + .applied + .as_ref() + .map(|activity| activity["details"] == "Example Game") + .unwrap_or(false) + }); + } + + #[test] + fn a_close_frame_is_recovered_from_and_the_activity_reapplied() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + + fake.send_frame(OPCODE_CLOSE, &json!({"code":4000,"message":"Overtaken"})); + wait_until(|| fake.state().connections >= 2); + wait_until(|| { + fake.state() + .applied + .as_ref() + .map(|activity| activity["details"] == "Example Game") + .unwrap_or(false) + }); + } + + #[test] + fn a_wedged_discord_does_not_block_the_caller_and_recovers() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().silent = true; + let service = DiscordService::with_directories(fake.directories()); + + let started = Instant::now(); + let status = service.sync(&streaming_params()).unwrap(); + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(4), + "sync blocked a caller for {elapsed:?}" + ); + assert_eq!(status["pending"], true); + assert!(fake.state().applied.is_none()); + + fake.state().silent = false; + wait_until(|| fake.state().applied.is_some()); + } + + #[test] + fn clearing_removes_the_activity_and_releases_the_connection() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + + let status = service.clear().unwrap(); + assert_eq!(status["cleared"], true); + wait_until(|| { + let state = fake.state(); + state.open == 0 && state.applied.is_none() + }); + assert_eq!(fake.state().connections, 1); + } + + #[test] + fn a_disabled_presence_never_opens_a_socket() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + let status = service + .sync(&json!({"enabled":false,"gameName":"Example"})) + .unwrap(); + assert_eq!(status["cleared"], true); + let status = service.clear().unwrap(); + assert_eq!(status["cleared"], true); + thread::sleep(Duration::from_millis(100)); + assert_eq!(fake.state().connections, 0); + } + + #[test] + fn a_rejected_activity_is_not_cached_as_applied_and_is_retried() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().reject_activity = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["applied"], false); + assert!(fake.state().applied.is_none()); + + fake.state().reject_activity = false; + wait_until(|| fake.state().applied.is_some()); + } + + #[test] + fn a_rejected_handshake_is_reported_and_retried() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().reject_handshake = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], false); + assert_eq!(status["applied"], false); + assert!( + status["message"] + .as_str() + .unwrap_or_default() + .contains("Invalid Client ID") + ); + + fake.state().reject_handshake = false; + wait_until(|| fake.state().applied.is_some()); + } + + #[test] + fn a_handshake_must_deliver_ready() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().delay_ready = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], true); + wait_until(|| fake.state().applied.is_some()); + } + + #[test] + fn unrelated_and_foreign_error_frames_are_ignored() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().interleave_events = true; + fake.state().foreign_error = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], true); + assert_eq!(status["unchanged"], false); + wait_until(|| fake.state().applied.is_some()); + } + + #[test] + fn fragmented_replies_are_reassembled() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().split_replies = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], true); + wait_until(|| fake.state().applied.is_some()); + assert_eq!( + fake.state().applied.as_ref().unwrap()["details"], + "Example Game" + ); + } + + #[test] + fn a_discord_ping_is_answered_with_a_pong() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + + fake.send_frame(OPCODE_PING, &json!({"cmd":"PING"})); + wait_until(|| fake.state().pongs >= 1); + assert!(fake.state().applied.is_some()); + } + + #[test] + fn an_exchange_is_refused_while_another_is_in_flight() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().silent = true; + let service = DiscordService::with_directories(fake.directories()); + + let first = service.sync(&streaming_params()).unwrap(); + assert_eq!(first["pending"], true); + assert_eq!(first["unchanged"], Value::Null); + + let started = Instant::now(); + let second = service.sync(&streaming_params()).unwrap(); + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(4), + "a request behind an in-flight exchange was not answered: {elapsed:?}" + ); + assert_eq!(second["pending"], true); + assert_eq!(second["unchanged"], Value::Null); + assert_eq!(fake.state().peak_open, 1); + + fake.state().silent = false; + wait_until(|| fake.state().applied.is_some()); + let state = fake.state(); + assert_eq!(state.peak_open, 1); + assert_eq!(state.open, 1); + } + + #[test] + fn abandonment_after_a_failed_exchange_stops_the_worker_without_spinning() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().reject_activity = true; + let service = DiscordService::with_directories(fake.directories()); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["applied"], false); + wait_until(|| fake.state().open == 0); + + service.abandoned.store(true, Ordering::SeqCst); + let observed = { + let before = service.iterations.load(std::sync::atomic::Ordering::SeqCst); + thread::sleep(Duration::from_millis(300)); + service + .iterations + .load(std::sync::atomic::Ordering::SeqCst) + .saturating_sub(before) + }; + assert!( + observed < 50, + "worker spun {observed} times after abandonment with no live connection" + ); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["terminal"], true); + assert_eq!(status["pending"], false); + } + + #[test] + fn abandonment_after_a_failed_initial_connect_stops_the_worker_without_spinning() { + let directory = TempDir::new().unwrap(); + let service = DiscordService::with_directories(vec![directory.path().to_path_buf()]); + + let status = service.sync(&streaming_params()).unwrap(); + assert_eq!(status["connected"], false); + + service.abandoned.store(true, Ordering::SeqCst); + let observed = { + let before = service.iterations.load(std::sync::atomic::Ordering::SeqCst); + thread::sleep(Duration::from_millis(300)); + service + .iterations + .load(std::sync::atomic::Ordering::SeqCst) + .saturating_sub(before) + }; + assert!( + observed < 50, + "worker spun {observed} times after abandonment with no connection" + ); + + let status = service.clear().unwrap(); + assert_eq!(status["terminal"], true); + } + + #[test] + fn buffered_frames_do_not_bypass_the_poll_deadline() { + let (stream, _peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let mut connection = Connection { + stream, + scratch: Vec::new(), + buffered: Vec::new(), + applied: None, + }; + for _ in 0..5 { + connection.buffered.extend_from_slice(&frame_bytes( + OPCODE_FRAME, + &json!({"cmd":"DISPATCH","evt":"CURRENT_USER_UPDATE","data":{"id":"1"}}), + )); + } + let buffered_before = connection.buffered.len(); + + let expired = Instant::now() + .checked_sub(Duration::from_millis(1)) + .expect("monotonic clock has advanced past one millisecond"); + assert!(connection.poll(expired).unwrap().is_none()); + assert_eq!(connection.buffered.len(), buffered_before); + + let first = connection + .poll(Instant::now() + Duration::from_secs(1)) + .unwrap(); + assert!(first.is_some()); + assert!(connection.buffered.len() < buffered_before); + } + + #[test] + fn abandoning_the_transport_stops_further_connections_and_bounds_abandoned_work() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + let alive = Arc::clone(&service.worker_alive); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + let connections_before = fake.state().connections; + + service.abandoned.store(true, Ordering::SeqCst); + + let started = Instant::now(); + for _ in 0..10 { + let synced = service.sync(&streaming_params()).unwrap(); + assert_eq!(synced["terminal"], true); + assert_eq!(synced["applied"], false); + let cleared = service.clear().unwrap(); + assert_eq!(cleared["terminal"], true); + } + assert!( + started.elapsed() < Duration::from_secs(1), + "callers were not answered immediately after abandonment" + ); + + wait_until(|| fake.state().open == 0); + thread::sleep(Duration::from_millis(700)); + assert_eq!( + fake.state().connections, + connections_before, + "a new connection was attempted after the transport was abandoned" + ); + assert!(alive.load(Ordering::SeqCst)); + + drop(service); + wait_until(|| !alive.load(Ordering::SeqCst)); + } + + #[test] + fn a_newer_intent_is_never_lost_behind_an_older_exchange() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + fake.state().silent = true; + let service = DiscordService::with_directories(fake.directories()); + + let _older = service.sync(&json!({ + "enabled":true, + "gameName":"Example Game", + "kind":"queued", + "queuePosition":1 + })); + assert!(fake.state().applied.is_none()); + + fake.state().silent = false; + let _newer = service.sync(&json!({ + "enabled":true, + "gameName":"Example Game", + "kind":"queued", + "queuePosition":2 + })); + + wait_until(|| { + fake.state() + .applied + .as_ref() + .map(|activity| activity["state"] == "In queue (#2)") + .unwrap_or(false) + }); + } + + #[test] + fn a_flood_of_updates_coalesces_and_a_clear_supersedes_them() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + + for position in 1..=40_u64 { + let _ = service.sync(&json!({ + "enabled":true, + "gameName":"Example Game", + "kind":"queued", + "queuePosition":position + })); + } + let status = service.clear().unwrap(); + assert_eq!(status["cleared"], true); + wait_until(|| { + let state = fake.state(); + state.applied.is_none() + }); + let state = fake.state(); + assert_eq!(state.peak_open, 1); + assert_eq!(state.connections, 1); + } + + #[test] + fn dropping_the_service_stops_the_worker_and_releases_the_connection() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + let alive = Arc::clone(&service.worker_alive); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + + drop(service); + wait_until(|| !alive.load(Ordering::SeqCst)); + wait_until(|| { + let state = fake.state(); + state.open == 0 && state.applied.is_none() + }); + } + + #[test] + fn dropping_an_idle_service_stops_the_worker() { + let fake = FakeDiscord::start(TempDir::new().unwrap()); + let service = DiscordService::with_directories(fake.directories()); + let alive = Arc::clone(&service.worker_alive); + + service.sync(&streaming_params()).unwrap(); + wait_until(|| fake.state().applied.is_some()); + service.clear().unwrap(); + wait_until(|| { + let state = fake.state(); + state.open == 0 + }); + + drop(service); + wait_until(|| !alive.load(Ordering::SeqCst)); + assert_eq!(fake.state().open, 0); + } + + #[test] + fn candidate_directories_are_deduplicated_and_end_with_the_tmp_fallback() { + let directories = candidate_directories(); + assert_eq!(directories.last().unwrap(), std::path::Path::new("/tmp")); + let mut sorted = directories.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), directories.len()); + assert!( + directories + .iter() + .all(|directory| !directory.as_os_str().is_empty()) + ); + } + } +} diff --git a/external/opennow/native/opennow-core/src/dlss.rs b/external/opennow/native/opennow-core/src/dlss.rs new file mode 100644 index 000000000..4ac5a0031 --- /dev/null +++ b/external/opennow/native/opennow-core/src/dlss.rs @@ -0,0 +1,292 @@ +//! Cloud DLSS overrides, mirroring the official GeForce NOW client. +//! +//! The setting is stored as the exact wire object the official client sends in +//! `requestedStreamingFeatures.dlssOverrides`. Every field is an enum value from the +//! client's `DlssSrMode`, `DlssSrModel`, `DlssRrModel`, `DlssFgMode`, and `DlssFgModel`; +//! zero means "use the game's own setting". + +use serde_json::{Value, json}; + +pub const SETTING: &str = "dlssOverrides"; + +const SR_MODE_CUSTOM: i64 = 6; +const SR_CUSTOM_DEFAULT_SCALE: i64 = 75; +const SR_MINIMUM_SCALE: i64 = 33; +const SR_MAXIMUM_SCALE: i64 = 100; +const SR_MAXIMUM_MODEL: i64 = 14; +const RR_MAXIMUM_MODEL: i64 = 7; +const FG_MAXIMUM_MODEL: i64 = 3; +const FG_MODE_DYNAMIC: i64 = 1; +const FG_MULTIPLIERS: [i64; 4] = [3, 4, 5, 6]; + +pub fn default_overrides() -> Value { + json!({"srMode":0,"srModel":0,"srScale":100,"rrModel":0,"fgMode":0,"fgModel":0,"fgMultiplier":0}) +} + +/// Input resolution percentage the official client pairs with each fixed SR mode. +fn fixed_sr_scale(sr_mode: i64) -> Option { + match sr_mode { + 0 | 5 => Some(100), + 3 => Some(67), + 2 => Some(58), + 1 => Some(50), + 4 => Some(33), + _ => None, + } +} + +fn integer(value: &Value, key: &str) -> Option { + value[key] + .as_f64() + .filter(|number| number.is_finite() && number.fract() == 0.0) + .map(|number| number as i64) +} + +fn bounded(value: &Value, key: &str, maximum: i64) -> i64 { + integer(value, key) + .filter(|number| (0..=maximum).contains(number)) + .unwrap_or(0) +} + +/// Clamp every field to a known enum value and apply the official client's coupling: +/// a model left to the game also leaves its mode to the game, fixed SR modes carry their +/// fixed input resolution, and the multiplier only exists for Dynamic frame generation. +pub fn normalize(value: &Value) -> Value { + let sr_model = bounded(value, "srModel", SR_MAXIMUM_MODEL); + let sr_mode = if sr_model == 0 { + 0 + } else { + bounded(value, "srMode", SR_MODE_CUSTOM) + }; + let sr_scale = fixed_sr_scale(sr_mode).unwrap_or_else(|| { + integer(value, "srScale") + .unwrap_or(SR_CUSTOM_DEFAULT_SCALE) + .clamp(SR_MINIMUM_SCALE, SR_MAXIMUM_SCALE) + }); + let fg_model = bounded(value, "fgModel", FG_MAXIMUM_MODEL); + let fg_mode = if fg_model == 0 { + 0 + } else { + bounded(value, "fgMode", FG_MODE_DYNAMIC) + }; + let fg_multiplier = if fg_mode == FG_MODE_DYNAMIC { + integer(value, "fgMultiplier") + .filter(|multiplier| FG_MULTIPLIERS.contains(multiplier)) + .unwrap_or(FG_MULTIPLIERS[0]) + } else { + 0 + }; + json!({ + "srMode":sr_mode, + "srModel":sr_model, + "srScale":sr_scale, + "rrModel":bounded(value, "rrModel", RR_MAXIMUM_MODEL), + "fgMode":fg_mode, + "fgModel":fg_model, + "fgMultiplier":fg_multiplier + }) +} + +/// The `requestedStreamingFeatures.dlssOverrides` object. Like the official client, the +/// multiplier is omitted unless frame generation is Dynamic. +pub fn session_request(settings: &Value) -> Value { + let mut overrides = normalize(&settings[SETTING]); + if overrides["fgMode"] != FG_MODE_DYNAMIC { + overrides + .as_object_mut() + .expect("normalized overrides are an object") + .remove("fgMultiplier"); + } + overrides +} + +/// Which overrides the MES subscription response entitles or upsells, read the way the +/// official client's ecommerce service reads `features.features` and `upsells`. +pub fn entitlements(subscription: &Value) -> Value { + let features = subscription["features"]["features"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default(); + let feature = |key: &str| { + features + .iter() + .find(|feature| feature["key"].as_str() == Some(key)) + }; + let entitled = |key: &str| { + feature(key) + .and_then(|feature| feature["textValue"].as_str()) + .and_then(|text| serde_json::from_str::(text).ok()) + .is_some_and(|value| match value { + Value::Null => false, + Value::Bool(enabled) => enabled, + Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(_) => true, + }) + }; + let upsell = |key: &str| { + subscription["upsells"] + .as_array() + .into_iter() + .flatten() + .any(|upsell| upsell["key"].as_str() == Some(key) && upsell["booleanValue"] == true) + }; + let capability = |name: &str| { + json!({ + "entitled":entitled(&format!("SUPPORTED_DLSS_OVERRIDE_{name}")), + "upsell":upsell(&format!("SUPPORT_DLSS_OVERRIDE_{name}")) + }) + }; + let multiplier_entitled = |multiplier: i64| { + let label = format!("{multiplier}X"); + let Some(feature) = feature("SUPPORTED_DLSS_FG_MULTIPLIERS") else { + return false; + }; + if let Some(text) = feature["textValue"].as_str() { + return match serde_json::from_str::(text) { + Ok(Value::Array(items)) => items.iter().any(|item| { + item["value"].as_str() == Some(label.as_str()) && item["isEntitled"] == true + }), + Ok(Value::Bool(enabled)) => enabled, + _ => false, + }; + } + feature["setValue"] + .as_array() + .into_iter() + .flatten() + .any(|value| value.as_str() == Some(label.as_str())) + }; + json!({ + "superResolution":capability("SR"), + "frameGeneration":capability("FG"), + "rayReconstruction":capability("RR"), + "frameGenerationMultipliers":FG_MULTIPLIERS + .into_iter() + .filter(|multiplier| multiplier_entitled(*multiplier)) + .collect::>() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_leave_every_feature_to_the_game() { + assert_eq!(normalize(&Value::Null), default_overrides()); + assert_eq!( + session_request(&json!({})), + json!({"srMode":0,"srModel":0,"srScale":100,"rrModel":0,"fgMode":0,"fgModel":0}) + ); + } + + #[test] + fn fixed_sr_modes_carry_the_official_input_resolution() { + for (mode, scale) in [(0, 100), (1, 50), (2, 58), (3, 67), (4, 33), (5, 100)] { + let normalized = normalize(&json!({"srModel":1,"srMode":mode,"srScale":90})); + assert_eq!(normalized["srMode"], mode); + assert_eq!(normalized["srScale"], scale, "mode {mode}"); + } + } + + #[test] + fn custom_sr_scale_is_clamped_and_defaults_to_75() { + assert_eq!( + normalize(&json!({"srModel":1,"srMode":6,"srScale":12}))["srScale"], + 33 + ); + assert_eq!( + normalize(&json!({"srModel":1,"srMode":6,"srScale":140}))["srScale"], + 100 + ); + assert_eq!( + normalize(&json!({"srModel":1,"srMode":6,"srScale":64.0}))["srScale"], + 64 + ); + assert_eq!(normalize(&json!({"srModel":1,"srMode":6}))["srScale"], 75); + } + + #[test] + fn a_model_left_to_the_game_leaves_its_mode_to_the_game() { + let normalized = + normalize(&json!({"srModel":0,"srMode":2,"fgModel":0,"fgMode":1,"fgMultiplier":4})); + assert_eq!(normalized, default_overrides()); + } + + #[test] + fn multiplier_exists_only_for_dynamic_frame_generation() { + let dynamic = json!({"dlssOverrides":{"fgModel":1,"fgMode":1,"fgMultiplier":4}}); + assert_eq!(session_request(&dynamic)["fgMultiplier"], 4); + let invalid = normalize(&json!({"fgModel":1,"fgMode":1,"fgMultiplier":2})); + assert_eq!(invalid["fgMultiplier"], 3); + let fixed = normalize(&json!({"fgModel":1,"fgMode":2,"fgMultiplier":4})); + assert_eq!(fixed["fgMode"], 0); + assert_eq!(fixed["fgMultiplier"], 0); + assert!( + session_request(&json!({"dlssOverrides":fixed})) + .get("fgMultiplier") + .is_none() + ); + } + + #[test] + fn unknown_enum_values_fall_back_to_the_game() { + let normalized = normalize( + &json!({"srModel":15,"srMode":7,"rrModel":8,"fgModel":4,"fgMode":"1","srScale":"58"}), + ); + assert_eq!(normalized, default_overrides()); + } + + #[test] + fn the_screenshot_configuration_reaches_the_wire() { + let settings = json!({"dlssOverrides":{"srMode":2,"srModel":1,"srScale":10,"rrModel":1,"fgMode":1,"fgModel":1,"fgMultiplier":3}}); + assert_eq!( + session_request(&settings), + json!({"srMode":2,"srModel":1,"srScale":58,"rrModel":1,"fgMode":1,"fgModel":1,"fgMultiplier":3}) + ); + } + + #[test] + fn entitlements_follow_the_official_feature_and_upsell_readers() { + let subscription = json!({ + "features":{"features":[ + {"key":"SUPPORTED_DLSS_OVERRIDE_SR","textValue":"true"}, + {"key":"SUPPORTED_DLSS_OVERRIDE_FG","textValue":"[\"DYNAMIC\"]"}, + {"key":"SUPPORTED_DLSS_OVERRIDE_RR","textValue":"false"}, + {"key":"SUPPORTED_DLSS_FG_MULTIPLIERS","textValue":"[{\"value\":\"3X\",\"isEntitled\":true},{\"value\":\"4X\",\"isEntitled\":true},{\"value\":\"5X\",\"isEntitled\":false}]"} + ]}, + "upsells":[ + {"key":"SUPPORT_DLSS_OVERRIDE_RR","booleanValue":true}, + {"key":"SUPPORT_DLSS_OVERRIDE_FG","booleanValue":false} + ] + }); + assert_eq!( + entitlements(&subscription), + json!({ + "superResolution":{"entitled":true,"upsell":false}, + "frameGeneration":{"entitled":true,"upsell":false}, + "rayReconstruction":{"entitled":false,"upsell":true}, + "frameGenerationMultipliers":[3,4] + }) + ); + } + + #[test] + fn multiplier_set_values_and_missing_features_are_read() { + let subscription = json!({"features":{"features":[ + {"key":"SUPPORTED_DLSS_FG_MULTIPLIERS","setValue":["3X","4X","5X","6X"]} + ]}}); + let read = entitlements(&subscription); + assert_eq!(read["frameGenerationMultipliers"], json!([3, 4, 5, 6])); + assert_eq!( + read["superResolution"], + json!({"entitled":false,"upsell":false}) + ); + assert_eq!( + entitlements(&json!({}))["frameGenerationMultipliers"], + json!([]) + ); + } +} diff --git a/external/opennow/native/opennow-core/src/frame_rate.rs b/external/opennow/native/opennow-core/src/frame_rate.rs new file mode 100644 index 000000000..8f4a1c4f2 --- /dev/null +++ b/external/opennow/native/opennow-core/src/frame_rate.rs @@ -0,0 +1,403 @@ +use serde_json::{Value, json}; + +pub const SUPPORTED_FRAME_RATES: [i64; 8] = [30, 60, 90, 120, 144, 165, 240, 360]; + +pub fn snap_frame_rate(requested: i64) -> i64 { + if requested == 59 { + return 60; + } + SUPPORTED_FRAME_RATES + .into_iter() + .rev() + .find(|rate| *rate <= requested) + .unwrap_or(30) +} + +pub fn resolution_ceiling(width: i64, height: i64) -> i64 { + if width == 1920 && matches!(height, 1080 | 1200) { + 360 + } else { + 240 + } +} + +fn is_full_hd(width: i64, height: i64) -> bool { + resolution_ceiling(width, height) > 240 +} + +fn resolution(settings: &Value) -> (i64, i64) { + settings["resolution"] + .as_str() + .and_then(|value| value.split_once(['x', 'X'])) + .and_then(|(width, height)| Some((width.parse().ok()?, height.parse().ok()?))) + .unwrap_or((1920, 1080)) +} + +fn hardware_decode_available(settings: &Value, capabilities: &Value) -> bool { + let requested_backend = crate::streamer::requested_embedded_backend(settings); + let requested_codec = settings["codec"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase(); + let explicit_codec = match requested_codec.as_str() { + "" | "auto" => None, + value => Some(value), + }; + capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool().unwrap_or(false)) + .filter(|backend| { + let name = backend["backend"].as_str().unwrap_or(""); + !matches!(name, "software" | "ffmpeg") + && (requested_backend == "auto" + || requested_backend == name + || (requested_backend == "nvdec" && name == "cuda")) + }) + .flat_map(|backend| backend["codecs"].as_array().into_iter().flatten()) + .any(|entry| { + if entry["available"].as_bool() != Some(true) { + return false; + } + let name = entry["codec"].as_str().unwrap_or(""); + match &explicit_codec { + Some(codec) => name.eq_ignore_ascii_case(codec), + None => ["h264", "h265", "av1"] + .iter() + .any(|codec| name.eq_ignore_ascii_case(codec)), + } + }) +} + +const BASE_FRAME_RATE_CEILING: i64 = 240; + +pub fn request_frame_rate(settings: &Value, params: &Value, width: i64, height: i64) -> i64 { + let requested = settings["fps"].as_i64().unwrap_or(60).clamp(30, 360); + let mut rate = requested.min(resolution_ceiling(width, height)); + if rate > BASE_FRAME_RATE_CEILING { + let capabilities = ¶ms["runtimeCapabilities"]; + let entitled = params["maxEntitledFps"].as_i64().unwrap_or(0).clamp(0, 360); + let cap = if entitled > 0 { + entitled.min(BASE_FRAME_RATE_CEILING) + } else { + BASE_FRAME_RATE_CEILING + }; + if !hardware_decode_available(settings, capabilities) || entitled < rate { + rate = cap; + } + } + snap_frame_rate(rate) +} + +pub fn frame_rate_choices(settings: &Value, capabilities: &Value) -> Value { + let (width, height) = resolution(settings); + let full_hd = is_full_hd(width, height); + let hardware = hardware_decode_available(settings, capabilities); + json!( + SUPPORTED_FRAME_RATES + .into_iter() + .map(|value| { + let reason = if value <= BASE_FRAME_RATE_CEILING { + None + } else if !full_hd { + Some( + "360 FPS is offered at full HD (1920x1080 or 1920x1200) only. Choose a full HD resolution first." + .to_owned(), + ) + } else if capabilities["videoBackends"].as_array().is_none() { + Some( + "360 FPS needs a confirmed hardware video decoder for the selected codec. This device has not reported one." + .to_owned(), + ) + } else if !hardware { + Some( + "360 FPS needs a hardware video decoder for the selected codec. This device has none available." + .to_owned(), + ) + } else { + None + }; + json!({"value":value, "disabled":reason.is_some(), "reason":reason}) + }) + .collect::>() + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn capabilities(backend: &str, codec: &str) -> Value { + json!({"protocolVersion":7, "videoBackends":[{"backend":backend, "platform":"linux", + "available":true, "codecs":[{"codec":codec, "available":true, + "colorQualities":["8bit_420"]}]}]}) + } + + fn descriptor(choices: &Value, value: i64) -> Value { + choices + .as_array() + .unwrap() + .iter() + .find(|choice| choice["value"] == value) + .cloned() + .unwrap() + } + + #[test] + fn supported_frame_rates_end_at_the_documented_top_tier() { + assert_eq!(SUPPORTED_FRAME_RATES, [30, 60, 90, 120, 144, 165, 240, 360]); + assert_eq!(SUPPORTED_FRAME_RATES.last(), Some(&360)); + } + + #[test] + fn frame_rates_snap_to_supported_modes_before_requesting_allocation() { + for (requested, expected) in [ + (0, 30), + (58, 30), + (59, 60), + (89, 60), + (90, 90), + (119, 90), + (120, 120), + (143, 120), + (144, 144), + (164, 144), + (165, 165), + (239, 165), + (240, 240), + (359, 240), + (360, 360), + (999, 360), + ] { + assert_eq!(snap_frame_rate(requested), expected); + } + assert_eq!( + request_frame_rate(&json!({"fps":164,"codec":"h265"}), &json!({}), 1920, 1080), + 144 + ); + } + + #[test] + fn resolution_ceiling_is_documented_full_hd_only() { + assert_eq!(resolution_ceiling(1920, 1080), 360); + assert_eq!(resolution_ceiling(1920, 1200), 360); + for (width, height) in [ + (1280, 720), + (1600, 900), + (1920, 1440), + (2560, 1440), + (2560, 1600), + (3440, 1440), + (3840, 2160), + (5120, 1440), + (3840, 1080), + (2560, 1080), + ] { + assert_eq!( + resolution_ceiling(width, height), + 240, + "{width}x{height} must not request the full-HD-only tier" + ); + } + } + + #[test] + fn only_the_top_tier_is_conditional_and_every_rate_reports_a_reason() { + let settings = + json!({"resolution":"1920x1080", "codec":"h265", "nativeVideoBackend":"auto"}); + let choices = frame_rate_choices(&settings, &capabilities("vaapi", "h265")); + assert_eq!( + choices.as_array().unwrap().len(), + SUPPORTED_FRAME_RATES.len() + ); + for rate in [30, 60, 90, 120, 144, 165, 240] { + let entry = descriptor(&choices, rate); + assert_eq!(entry["disabled"], false, "{rate} stays selectable"); + assert_eq!(entry["reason"], Value::Null, "{rate} needs no reason"); + } + assert_eq!(descriptor(&choices, 360)["disabled"], false); + + let non_full_hd = frame_rate_choices( + &json!({"resolution":"2560x1440", "codec":"h265", "nativeVideoBackend":"auto"}), + &capabilities("vaapi", "h265"), + ); + let entry = descriptor(&non_full_hd, 360); + assert_eq!(entry["disabled"], true); + assert!( + entry["reason"] + .as_str() + .unwrap() + .contains("full HD (1920x1080 or 1920x1200) only") + ); + assert_eq!(descriptor(&non_full_hd, 240)["disabled"], false); + } + + #[test] + fn top_tier_requires_a_hardware_decoder_for_the_selected_codec() { + let full_hd = json!({"resolution":"1920x1200", "codec":"av1", "nativeVideoBackend":"auto"}); + let software_only = json!({"videoBackends":[{"backend":"software", "available":true, + "codecs":[{"codec":"av1", "available":true}]}]}); + let entry = descriptor(&frame_rate_choices(&full_hd, &software_only), 360); + assert_eq!(entry["disabled"], true); + assert!( + entry["reason"] + .as_str() + .unwrap() + .contains("needs a hardware video decoder") + ); + + let mismatched_codec = capabilities("vaapi", "h265"); + assert_eq!( + descriptor(&frame_rate_choices(&full_hd, &mismatched_codec), 360)["disabled"], + true, + "an unrelated hardware decoder must not unlock the top tier" + ); + + let explicit_backend = + json!({"resolution":"1920x1200", "codec":"h264", "nativeVideoBackend":"cuda"}); + assert_eq!( + descriptor( + &frame_rate_choices(&explicit_backend, &capabilities("vaapi", "h264")), + 360 + )["disabled"], + true, + "a backend the preference excludes cannot unlock the top tier" + ); + let nvdec_alias = + json!({"resolution":"1920x1200", "codec":"av1", "nativeVideoBackend":"nvdec"}); + assert_eq!( + descriptor( + &frame_rate_choices(&nvdec_alias, &capabilities("cuda", "av1")), + 360 + )["disabled"], + false, + "the nvdec alias still matches the CUDA backend" + ); + + let unavailable = json!({"videoBackends":[{"backend":"vaapi", "available":false, + "codecs":[{"codec":"h265", "available":true}]}]}); + assert_eq!( + descriptor(&frame_rate_choices(&full_hd, &unavailable), 360)["disabled"], + true, + "an unavailable backend cannot unlock the top tier" + ); + assert_eq!( + descriptor(&frame_rate_choices(&full_hd, &json!({})), 360)["disabled"], + true, + "unreported capabilities never unlock the top tier" + ); + } + + #[test] + fn request_rate_applies_the_resolution_device_and_entitlement_ceiling() { + let hardware = capabilities("vaapi", "h265"); + let software = json!({"videoBackends":[{"backend":"software", "available":true, + "codecs":[{"codec":"h265", "available":true}]}]}); + let settings = json!({"resolution":"1920x1080", "fps":360, "codec":"h265", + "nativeVideoBackend":"auto"}); + let request = |capabilities: &Value, entitled: i64, width: i64, height: i64| { + request_frame_rate( + &settings, + &json!({"runtimeCapabilities":capabilities, "maxEntitledFps":entitled}), + width, + height, + ) + }; + assert_eq!(request(&hardware, 360, 1920, 1080), 360); + assert_eq!( + request(&software, 360, 1920, 1080), + 240, + "a reported software-only probe cannot request the top tier" + ); + assert_eq!( + request(&json!({}), 360, 1920, 1080), + 240, + "an unreported probe is not affirmative capability" + ); + assert_eq!( + request(&json!({"videoBackends":[]}), 360, 1920, 1080), + 240, + "a probe that reported no backend cannot request the top tier" + ); + assert_eq!( + request(&hardware, 0, 1920, 1080), + 240, + "unconfirmed entitlement cannot request the top tier" + ); + assert_eq!( + request(&hardware, 240, 1920, 1080), + 240, + "a 240 FPS entitlement cannot request the top tier" + ); + assert_eq!( + request(&hardware, 120, 1920, 1080), + 120, + "a lower entitlement bounds the request" + ); + assert_eq!( + request(&hardware, 360, 2560, 1440), + 240, + "the resolution ceiling applies regardless of entitlement" + ); + assert_eq!( + request_frame_rate( + &json!({"resolution":"1920x1080", "fps":999, "codec":"h265"}), + &json!({"runtimeCapabilities":hardware, "maxEntitledFps":360}), + 1920, + 1080 + ), + 360, + "a stored value above the ceiling is still bounded" + ); + assert_eq!( + request_frame_rate( + &json!({"resolution":"1920x1080", "fps":120, "codec":"h265"}), + &json!({"runtimeCapabilities":software, "maxEntitledFps":0}), + 1920, + 1080 + ), + 120, + "base rates ignore the capability and entitlement verdicts" + ); + } + + #[test] + fn legacy_software_preference_never_qualifies_for_the_top_tier() { + let hardware = capabilities("vaapi", "h264"); + let settings = json!({"resolution":"1920x1080", "fps":360, "codec":"h264", + "nativeVideoBackend":"auto", "decoderPreference":"software"}); + let params = json!({"runtimeCapabilities":hardware, "maxEntitledFps":360}); + assert_eq!( + request_frame_rate(&settings, ¶ms, 1920, 1080), + 240, + "the legacy software preference is a software decode path" + ); + assert_eq!( + request_frame_rate( + &json!({"resolution":"1920x1080", "fps":360, "codec":"h264", + "nativeVideoBackend":"auto", "decoderPreference":"auto"}), + ¶ms, + 1920, + 1080 + ), + 360, + "the same hardware profile stays eligible without the software preference" + ); + } + + #[test] + fn auto_selection_accepts_any_hardware_codec_without_mutating_inputs() { + let settings = + json!({"resolution":"1920x1080", "codec":"auto", "nativeVideoBackend":"auto"}); + let original = settings.clone(); + let choices = frame_rate_choices(&settings, &capabilities("vulkan", "h265")); + assert_eq!(descriptor(&choices, 360)["disabled"], false); + assert_eq!(settings, original); + + let missing = frame_rate_choices(&settings, &json!({})); + assert_eq!(descriptor(&missing, 240)["disabled"], false); + assert_eq!(descriptor(&missing, 360)["disabled"], true); + } +} diff --git a/external/opennow/native/opennow-core/src/gfn.rs b/external/opennow/native/opennow-core/src/gfn.rs new file mode 100644 index 000000000..8fed7e15c --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn.rs @@ -0,0 +1,5252 @@ +use crate::account_connections::AccountConnectionsService; +use crate::cloudmatch::CloudMatchService; +use crate::console_profiles::ConsoleProfiles; +use crate::credential_vault::CredentialVault; +use crate::persistent_storage::PersistentStorageService; +use crate::proxy::{client_for_settings, config_from_settings}; +use base64::Engine as _; +use qrcode::QrCode; +use reqwest::blocking::{Client, Response}; +use reqwest::header::{ + ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue, ORIGIN, REFERER, USER_AGENT, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::env; +use std::io::Read; +use std::net::ToSocketAddrs; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub(crate) mod catalog; +mod catalog_actions; +mod dlss_support; +mod store_launch; +use catalog::*; + +#[cfg(test)] +mod catalog_tests; +#[cfg(test)] +mod routing_tests; +#[cfg(test)] +mod store_launch_tests; + +const DEFAULT_IDP_ID: &str = "PDiAhv2kJTFeQ7WOPqiQ2tRZ7lGhR2X11dXvM4TZSxg"; +const DEFAULT_STREAMING_URL: &str = "https://prod.cloudmatchbeta.nvidiagrid.net/"; +// Alliance `prod.*` discovery endpoints are geo-steered: they resolve to the +// nearest regional PoP from inside the partner footprint (via VPN or local +// presence) and return NODATA/NXDOMAIN from outside it. For partners below, +// discovery still advertises only the `prod.*` name, so out-of-footprint +// users get no DNS at all. Each entry maps that stale name to a globally +// reachable regional endpoint in the same `nvidiagrid.net` trust policy. The +// fallback engages only when the advertised host fails DNS resolution, so +// in-footprint users keep native geo-steering untouched. +struct ProviderFallback { + idp_id: &'static str, + stale_host: &'static str, + fallback_url: &'static str, +} + +const PROVIDER_FALLBACKS: &[ProviderFallback] = &[ + // Verified live Sep 2026: serverId NPA-DIG-SCL-01, region "LATAM West", + // session create and first video frame at 1080p60 H264. Digevo also + // serves LATAM North (Bogota) via geo-steered `prod.dig`. + ProviderFallback { + idp_id: "IsvVBA3Aj8KZ7gwwuRUhB6-tOF2o2F1wncD-XjYv100", + stale_host: "prod.dig.geforcenow.nvidiagrid.net", + fallback_url: "https://latam-west.dig.geforcenow.nvidiagrid.net/", + }, +]; +const STEAM_DECK_CLIENT_ID: &str = "q61ddeJrVt7O90Nl-P-N7I36yctih4Ml6FyXLrb6j-U"; +const SCOPES: &str = "openid consent email tk_client age"; +const STEAM_DECK_USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; Steam Deck) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"; +const GFN_USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 NVIDIACEFClient/HEAD/7b92719716 GFN-PC/2.0.87.131"; +const NVIDIA_FILE_ORIGIN: &str = "https://nvfile"; +const TOKEN_REFRESH_WINDOW_MS: u64 = 10 * 60 * 1000; +const CLIENT_TOKEN_REFRESH_WINDOW_MS: u64 = 5 * 60 * 1000; +const LCARS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; +const GFN_CLIENT_VERSION: &str = "2.0.87.131"; +const GRAPHQL_URL: &str = "https://games.geforce.com/graphql"; +const MES_URL: &str = "https://mes.geforcenow.com/v4/subscriptions"; + +#[derive(Clone)] +pub struct Endpoints { + pub service_urls: String, + pub device_authorize: String, + pub token: String, + pub client_token: String, + pub userinfo: String, + pub revoke: String, + pub public_catalog: String, + pub graphql: String, + pub public_graphql: String, + pub account_linking: String, + pub subscription: String, + pub dlss_games: String, + #[cfg(test)] + server_info: Option, +} + +impl Default for Endpoints { + fn default() -> Self { + Self { + service_urls: "https://pcs.geforcenow.com/v1/serviceUrls".to_owned(), + device_authorize: "https://login.nvidia.com/device/authorize".to_owned(), + token: "https://login.nvidia.com/token".to_owned(), + client_token: "https://login.nvidia.com/client_token".to_owned(), + userinfo: "https://login.nvidia.com/userinfo".to_owned(), + revoke: "https://login.nvidia.com/assets/v2/Tokens?level=client".to_owned(), + public_catalog: + "https://static.nvidiagrid.net/supported-public-game-list/locales/gfnpc-en-US.json" + .to_owned(), + graphql: GRAPHQL_URL.into(), + public_graphql: "https://public.games.geforce.com/graphql".into(), + account_linking: "https://als.geforcenow.com/v1".into(), + subscription: MES_URL.into(), + dlss_games: dlss_support::LIST_URL.into(), + #[cfg(test)] + server_info: None, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LoginProvider { + pub idp_id: String, + pub code: String, + pub display_name: String, + pub streaming_service_url: String, + pub priority: i64, +} + +impl LoginProvider { + fn default_nvidia() -> Self { + Self { + idp_id: DEFAULT_IDP_ID.to_owned(), + code: "NVIDIA".to_owned(), + display_name: "NVIDIA".to_owned(), + streaming_service_url: DEFAULT_STREAMING_URL.to_owned(), + priority: 0, + } + } + + fn normalize(mut self) -> Self { + self.streaming_service_url = effective_provider_url(&self); + if !self.streaming_service_url.ends_with('/') { + self.streaming_service_url.push('/'); + } + self + } +} + +pub(crate) fn effective_provider_url(provider: &LoginProvider) -> String { + effective_provider_url_with(provider, host_resolves) +} + +fn host_resolves(host: &str) -> bool { + // The port is irrelevant; this only exercises DNS resolution. NXDOMAIN + // answers fast, and successes are OS-cached, so the probe stays cheap + // next to the HTTPS calls every caller issues afterwards. + format!("{host}:443") + .to_socket_addrs() + .is_ok_and(|mut addresses| addresses.next().is_some()) +} + +fn effective_provider_url_with( + provider: &LoginProvider, + resolves: impl Fn(&str) -> bool, +) -> String { + let raw = provider.streaming_service_url.trim(); + let host = url::Url::parse(raw) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)) + .unwrap_or_default(); + let fallback = PROVIDER_FALLBACKS.iter().find(|entry| { + provider.idp_id == entry.idp_id && host.eq_ignore_ascii_case(entry.stale_host) + }); + match fallback { + Some(entry) if !resolves(&host) => { + eprintln!( + "provider: {} discovery endpoint is unreachable; using fallback {}", + provider.code, entry.fallback_url + ); + entry.fallback_url.to_owned() + } + _ => raw.to_owned(), + } +} + +pub(crate) fn provider_streaming_base(provider: &LoginProvider) -> Result { + trusted_streaming_base(&effective_provider_url(provider)) +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthTokens { + pub access_token: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub refresh_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub id_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id_token_expires_at: Option, + pub expires_at: u64, + pub auth_client_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_token_expires_at: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_token_lifetime_ms: Option, +} + +impl std::fmt::Debug for AuthTokens { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("AuthTokens([redacted])") + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthUser { + pub user_id: String, + pub display_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub avatar_url: Option, + pub membership_tier: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AuthSession { + pub provider: LoginProvider, + pub tokens: AuthTokens, + pub user: AuthUser, +} + +#[derive(Serialize)] +struct PublicAuthSession<'a> { + user: &'a AuthUser, + provider: &'a LoginProvider, +} + +impl AuthSession { + fn public(&self) -> PublicAuthSession<'_> { + PublicAuthSession { + user: &self.user, + provider: &self.provider, + } + } +} + +#[derive(Clone, Copy)] +pub(crate) enum TokenPurpose { + StarfleetAccess, + ServiceId, +} + +impl AuthTokens { + fn service_token(&self) -> &str { + self.id_token.as_deref().unwrap_or(&self.access_token) + } + + fn expiry(&self, purpose: TokenPurpose) -> u64 { + match purpose { + TokenPurpose::StarfleetAccess => self.expires_at, + TokenPurpose::ServiceId => self.id_token.as_deref().map_or(self.expires_at, |token| { + self.id_token_expires_at + .or_else(|| jwt_expiry(token)) + .unwrap_or(0) + }), + } + } +} + +#[derive(Clone, Copy, Default, PartialEq)] +enum PersistenceIntent { + #[default] + MemoryOnly, + SecureStore, +} + +#[derive(Clone)] +struct DeviceAttempt { + provider: LoginProvider, + device_code: String, + expires_at: u64, + deadline: Instant, + interval_seconds: u64, + next_poll: Instant, + in_flight: bool, + pending_session: Option, +} + +#[derive(Default)] +struct ServiceState { + providers: Vec, + providers_expires: Option, + providers_retry: Option, + providers_error: Option, + providers_default: Option, + attempts: HashMap, + session: Option, + public_games: Vec, + public_games_proxy_scope: String, + restore_attempted: bool, + persistence_state: String, + persistence_intent: PersistenceIntent, + generation: u64, + login_generation: u64, + refresh_retry_at: Option, +} + +#[derive(Clone, Debug)] +pub struct ServiceError { + pub code: &'static str, + pub message: String, +} + +impl ServiceError { + pub(crate) fn network(context: &str, error: reqwest::Error) -> Self { + let error = error.without_url(); + Self { + code: "network_error", + message: match transport_failure(&error) { + Some(stage) => format!("{context}: {error} ({stage})"), + None => format!("{context}: {error}"), + }, + } + } + + fn server_info(provider: &LoginProvider, context: &str, error: reqwest::Error) -> Self { + let home_network_only = + provider.idp_id != DEFAULT_IDP_ID && transport_failure(&error) == Some("dns"); + let error = Self::network(context, error); + if !home_network_only { + return error; + } + let name = Some(provider.display_name.trim()) + .filter(|name| !name.is_empty()) + .unwrap_or(provider.code.trim()); + Self { + code: error.code, + message: format!( + "{name} can only be reached from inside its home country and network. Turn off any VPN, proxy or custom DNS (such as 1.1.1.1 or 8.8.8.8) and try again. Details: {}", + error.message + ), + } + } + + fn response(context: &str, response: Response) -> Self { + let status = response.status(); + let mut body = Vec::new(); + let payload = response + .take(16 * 1024 + 1) + .read_to_end(&mut body) + .ok() + .filter(|_| body.len() <= 16 * 1024) + .and_then(|_| serde_json::from_slice::(&body).ok()); + let detail = payload + .as_ref() + .and_then(|payload| payload["error"].as_str()) + .filter(|_| matches!(status.as_u16(), 400 | 401)) + .filter(|error| matches!(*error, "invalid_grant" | "invalid_token" | "token_revoked")); + Self { + code: if status.as_u16() == 401 { + "http_unauthorized" + } else { + "upstream_error" + }, + message: match detail { + Some(detail) => format!("{context} ({status}): {detail}"), + None => format!("{context} ({status})"), + }, + } + } + + fn invalid(message: impl Into) -> Self { + Self { + code: "invalid_params", + message: message.into(), + } + } +} + +fn transport_failure(error: &reqwest::Error) -> Option<&'static str> { + if error.is_timeout() { + return Some("timeout"); + } + let mut source = std::error::Error::source(error); + while let Some(cause) = source { + if cause.to_string() == "dns error" { + return Some("dns"); + } + if cause.is::() { + return Some("tls"); + } + source = match cause.downcast_ref::() { + Some(io) => io.get_ref().map(|inner| inner as _), + None => cause.source(), + }; + } + error.is_connect().then_some("connect") +} + +#[derive(Default)] +struct SessionRouting { + active_owner: Option, + discovery_owner: Option<(String, String, u64)>, +} + +struct ActiveSeatOwner { + auth: AuthSession, + session_id: String, + last_published_generation: u64, + allocation_generation: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SessionReconciliationHint { + session_id: String, + owner_scope: SessionOwnerIdentity, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SessionOwnerIdentity { + user_id: String, + provider_idp_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SessionEndNotification { + session_id: String, + owner_scope: SessionOwnerIdentity, +} + +impl ActiveSeatOwner { + fn capture( + auth: AuthSession, + generation: u64, + session: &Value, + allocation_generation: Option, + ) -> Result { + Ok(Self { + auth, + session_id: session["sessionId"] + .as_str() + .filter(|id| !id.is_empty()) + .ok_or_else(session_owner_error)? + .to_owned(), + last_published_generation: generation, + allocation_generation, + }) + } + + fn matches(&self, auth: &AuthSession, session_id: &str) -> bool { + self.session_id == session_id + && self.auth.user.user_id == auth.user.user_id + && self.auth.provider.idp_id == auth.provider.idp_id + } +} + +pub struct GfnService { + client: Client, + endpoints: Endpoints, + device_id: String, + device_identity_error: Option, + vault: CredentialVault, + profiles: ConsoleProfiles, + cloudmatch: CloudMatchService, + account_connections: AccountConnectionsService, + persistent_storage: PersistentStorageService, + store_cache: std::sync::Arc, + dlss_games: dlss_support::DlssCatalog, + catalog_revision: std::sync::atomic::AtomicU64, + catalog_mutations: Mutex>, + server_vpc_cache: crate::server_vpc_cache::ServerVpcCache, + auth_operation: Mutex<()>, + discovery_operation: Mutex<()>, + session_routing: Mutex, + state: Mutex, +} + +impl GfnService { + pub fn new(data_dir: PathBuf) -> Result { + let client = Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(8)) + .timeout(Duration::from_secs(20)) + .pool_idle_timeout(Duration::from_secs(60)) + .build() + .map_err(|error| error.to_string())?; + Ok(Self::with_client(client, Endpoints::default(), data_dir)) + } + + fn with_client(client: Client, endpoints: Endpoints, data_dir: PathBuf) -> Self { + let identity = crate::device_identity::load(&data_dir, stable_device_id); + let (device_id, device_identity_error) = match identity { + Ok(id) => (id, None), + Err(error) => (stable_device_id(), Some(error)), + }; + let vault = CredentialVault::new(data_dir.clone()); + match vault.migrate_legacy_electron_sessions() { + Ok(count) if count > 0 => { + eprintln!( + "auth: migrated {count} Electron account session(s) into the OS credential store" + ) + } + Ok(_) => {} + Err(error) => eprintln!("auth: Electron account migration was deferred: {error}"), + } + let store_cache = + std::sync::Arc::new(crate::store_cache::StoreCache::new(data_dir.clone())); + let catalog_revision = store_cache.catalog_revision(); + let dlss_games = dlss_support::DlssCatalog::new( + client.clone(), + endpoints.dlss_games.clone(), + store_cache.clone(), + ); + Self { + cloudmatch: CloudMatchService::with_cleanup_path( + client.clone(), + data_dir.join("pending-session-cleanup.json"), + ), + account_connections: AccountConnectionsService::new(), + persistent_storage: PersistentStorageService::new(client.clone()), + client, + endpoints, + device_id, + device_identity_error, + vault, + profiles: ConsoleProfiles::load(&data_dir), + store_cache, + dlss_games, + catalog_revision: std::sync::atomic::AtomicU64::new(catalog_revision), + catalog_mutations: Mutex::new(std::collections::HashSet::new()), + server_vpc_cache: crate::server_vpc_cache::ServerVpcCache::default(), + auth_operation: Mutex::new(()), + discovery_operation: Mutex::new(()), + session_routing: Mutex::new(SessionRouting::default()), + state: Mutex::new(ServiceState::default()), + } + } + + pub fn providers(&self) -> Result { + let _discovery = crate::store_requests::lock(&self.discovery_operation)?; + { + let state = self.state.lock().expect("GFN state poisoned"); + if state + .providers_expires + .is_some_and(|time| time > Instant::now()) + || state + .providers_retry + .is_some_and(|time| time > Instant::now()) + { + return Ok(provider_result(&state)); + } + } + let mut retry_delay = Duration::from_secs(30); + let result = self + .client + .get(&self.endpoints.service_urls) + .timeout(Duration::from_secs(5)) + .header(ACCEPT, "application/json") + .header(USER_AGENT, GFN_USER_AGENT) + .send() + .map_err(|error| ServiceError::network("Provider discovery failed", error)) + .and_then(|response| { + if !response.status().is_success() { + if response.status().as_u16() == 429 { + retry_delay = + response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + value.parse::().ok().map(Duration::from_secs).or_else( + || { + httpdate::parse_http_date(value).ok().and_then(|time| { + time.duration_since(SystemTime::now()).ok() + }) + }, + ) + }) + .unwrap_or(retry_delay) + .clamp(Duration::from_secs(30), Duration::from_secs(3600)); + } + return Err(ServiceError::response( + "Provider discovery failed", + response, + )); + } + let payload = response + .json::() + .map_err(|error| ServiceError::network("Invalid provider discovery", error))?; + let providers = parse_providers(&payload) + .into_iter() + .filter(|provider| { + trusted_streaming_base(&provider.streaming_service_url).is_ok() + }) + .collect::>(); + if providers.is_empty() { + return Err(ServiceError::invalid( + "Provider discovery returned no usable providers", + )); + } + let default = payload["gfnServiceInfo"]["defaultProvider"] + .as_str() + .and_then(|default| { + payload["gfnServiceInfo"]["gfnServiceEndpoints"] + .as_array()? + .iter() + .find(|entry| { + entry["loginProvider"] == default + || entry["loginProviderCode"] == default + }) + .and_then(|entry| entry["idpId"].as_str()) + .filter(|id| providers.iter().any(|provider| provider.idp_id == *id)) + .map(ToOwned::to_owned) + }); + Ok((providers, default)) + }); + crate::requests::check()?; + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + let mut state = self.state.lock().expect("GFN state poisoned"); + match result { + Ok((providers, default)) => { + if let Some(session) = &mut state.session { + match providers + .iter() + .find(|provider| provider.idp_id == session.provider.idp_id) + { + Some(provider) + if session.provider.streaming_service_url + != provider.streaming_service_url => + { + session.provider = provider.clone(); + state.generation += 1; + } + None => state.generation += 1, + _ => {} + } + } + state.providers = providers; + state.providers_default = default; + state.providers_expires = Some(Instant::now() + Duration::from_secs(15 * 60)); + state.providers_retry = None; + state.providers_error = None; + } + Err(error) => { + state.providers_retry = Some(Instant::now() + retry_delay); + state.providers_error = Some(error.message); + } + } + Ok(provider_result(&state)) + } + + pub fn start_device_login(&self, params: &Value) -> Result { + crate::requests::check()?; + if let Some(message) = &self.device_identity_error { + return Err(ServiceError { + code: "device_identity_unavailable", + message: message.clone(), + }); + } + let generation = { + let mut state = self.state.lock().expect("GFN state poisoned"); + state.login_generation += 1; + state.attempts.clear(); + state.login_generation + }; + eprintln!("auth: starting device authorization"); + let discovery = self.providers()?; + let providers = discovery["providers"] + .as_array() + .cloned() + .unwrap_or_default(); + let provider_id = params["providerIdpId"] + .as_str() + .or_else(|| discovery["defaultProviderIdpId"].as_str()); + let provider = providers + .iter() + .filter_map(|value| serde_json::from_value::(value.clone()).ok()) + .find(|item| provider_id.is_some_and(|wanted| item.idp_id == wanted)) + .or_else(|| provider_id.is_none().then(|| { + providers + .first() + .and_then(|value| serde_json::from_value(value.clone()).ok()) + }).flatten()) + .ok_or_else(|| ServiceError { code: "provider_unavailable", message: "The selected provider is unavailable. Refresh the provider list and select it again.".into() })? + .normalize(); + + let form = [ + ("client_id", STEAM_DECK_CLIENT_ID), + ("scope", SCOPES), + ("device_id", self.device_id.as_str()), + ("display_name", "OpenNOW"), + ("idp_id", provider.idp_id.as_str()), + ]; + let response = self + .client + .post(&self.endpoints.device_authorize) + .header(ACCEPT, "application/json, text/plain, */*") + .header( + CONTENT_TYPE, + "application/x-www-form-urlencoded; charset=UTF-8", + ) + .header(ORIGIN, "https://play.geforcenow.com") + .header(REFERER, "https://play.geforcenow.com/") + .header(USER_AGENT, STEAM_DECK_USER_AGENT) + .header("x-device-id", &self.device_id) + .header("nv-client-id", STEAM_DECK_CLIENT_ID) + .header("nv-client-streamer", "WEBRTC") + .header("nv-client-type", "BROWSER") + .header("nv-client-platform-name", "browser") + .header("nv-browser-type", "CHROME") + .header("nv-device-os", "STEAMOS") + .header("nv-device-type", "CONSOLE") + .header("nv-device-model", "STEAMDECK") + .header("nv-device-make", "VALVE") + .form(&form) + .send() + .map_err(|error| ServiceError::network("Device authorization failed", error))?; + eprintln!("auth: device authorization response {}", response.status()); + if !response.status().is_success() { + return Err(ServiceError::response( + "Device authorization failed", + response, + )); + } + let payload = bounded_auth_response(response)?; + let device_code = required_string(&payload, "device_code")?; + let user_code = required_string(&payload, "user_code")?; + let verification_uri = required_string(&payload, "verification_uri")?; + let verification_uri_complete = required_string(&payload, "verification_uri_complete")?; + let lifetime = payload["expires_in"] + .as_u64() + .filter(|value| *value > 0 && *value <= 3600) + .unwrap_or(600); + let expires_at = now_ms().saturating_add(lifetime * 1000); + let interval_seconds = payload["interval"] + .as_u64() + .filter(|value| *value > 0) + .unwrap_or(5) + .min(3600); + let attempt_id = random_attempt_id(); + eprintln!("auth: prepared device authorization challenge"); + self.prune_attempts(); + crate::requests::check()?; + let mut state = self.state.lock().expect("GFN state poisoned"); + if state.login_generation != generation { + return Err(ServiceError::invalid("QR login was replaced or cancelled")); + } + state.attempts.insert( + attempt_id.clone(), + DeviceAttempt { + provider, + device_code: device_code.clone(), + expires_at, + deadline: Instant::now() + Duration::from_secs(lifetime), + interval_seconds, + next_poll: Instant::now() + Duration::from_secs(interval_seconds), + in_flight: false, + pending_session: None, + }, + ); + Ok(json!({ + "attemptId": attempt_id, + "userCode": user_code, + "verificationUri": verification_uri, + "verificationUriComplete": verification_uri_complete, + "expiresAt": expires_at, + "intervalSeconds": interval_seconds, + "qrRows": qr_rows(&verification_uri_complete), + })) + } + + pub fn poll_device_login(&self, params: &Value) -> Result { + crate::requests::check()?; + let attempt_id = required_param(params, "attemptId")?; + self.prune_attempts(); + let attempt = { + let mut state = self.state.lock().expect("GFN state poisoned"); + let Some(attempt) = state.attempts.get_mut(attempt_id) else { + return Ok( + json!({"status":"expired", "error":"QR login was cancelled or expired"}), + ); + }; + if attempt.pending_session.is_some() { + return Ok(json!({"status":"authorized"})); + } + if attempt.in_flight || Instant::now() < attempt.next_poll { + return Ok( + json!({"status":"pending", "retryAfterMs": attempt.next_poll.saturating_duration_since(Instant::now()).as_millis().max(1000), "intervalSeconds":attempt.interval_seconds}), + ); + } + attempt.in_flight = true; + attempt.clone() + }; + let _poll = DevicePoll { + service: self, + attempt_id, + }; + let form = [ + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("device_code", attempt.device_code.as_str()), + ("client_id", STEAM_DECK_CLIENT_ID), + ]; + let response = self + .client + .post(&self.endpoints.token) + .header(ACCEPT, "application/json, text/plain, */*") + .header( + CONTENT_TYPE, + "application/x-www-form-urlencoded; charset=UTF-8", + ) + .header(ORIGIN, "https://play.geforcenow.com") + .header(REFERER, "https://play.geforcenow.com/") + .header(USER_AGENT, STEAM_DECK_USER_AGENT) + .form(&form) + .send() + .map_err(|error| ServiceError::network("Device token exchange failed", error))?; + let status = response.status(); + let payload = bounded_auth_response(response)?; + self.check_device_attempt(attempt_id)?; + if !status.is_success() { + let error = payload["error"] + .as_str() + .unwrap_or("device_token_exchange_failed"); + return Ok(match error { + "authorization_pending" => { + json!({"status":"pending", "intervalSeconds":attempt.interval_seconds, "retryAfterMs":attempt.interval_seconds * 1000}) + } + "slow_down" => { + let mut state = self.state.lock().expect("GFN state poisoned"); + let stored = state + .attempts + .get_mut(attempt_id) + .ok_or_else(|| ServiceError::invalid("QR login was cancelled"))?; + stored.interval_seconds = stored.interval_seconds.saturating_add(5).min(3600); + json!({"status":"slow_down", "intervalSeconds":stored.interval_seconds, "retryAfterMs":stored.interval_seconds * 1000}) + } + "expired_token" => { + self.cancel_device_login(params)?; + json!({"status":"expired", "error":"QR login expired"}) + } + "access_denied" => { + self.cancel_device_login(params)?; + json!({"status":"access_denied", "error":"QR login was declined"}) + } + _ => { + self.cancel_device_login(params)?; + json!({"status":"error", "error":"QR login token exchange failed"}) + } + }); + } + + let access_token = required_string(&payload, "access_token")?; + let tokens = AuthTokens { + access_token, + refresh_token: payload["refresh_token"].as_str().map(ToOwned::to_owned), + id_token: payload["id_token"].as_str().map(ToOwned::to_owned), + id_token_expires_at: payload["id_token"].as_str().and_then(jwt_expiry), + expires_at: token_expiry(&payload), + auth_client_id: STEAM_DECK_CLIENT_ID.to_owned(), + client_token: payload["client_token"].as_str().map(ToOwned::to_owned), + client_token_expires_at: None, + client_token_lifetime_ms: None, + }; + let tokens = self + .ensure_client_token(tokens.clone()) + .unwrap_or_else(|error| { + eprintln!("auth: client-token bootstrap deferred: {}", error.message); + tokens + }); + self.check_device_attempt(attempt_id)?; + let user = self.fetch_user_info(&tokens)?; + self.check_device_attempt(attempt_id)?; + let session = AuthSession { + provider: attempt.provider, + tokens, + user, + }; + if let Some(stored) = self + .state + .lock() + .expect("GFN state poisoned") + .attempts + .get_mut(attempt_id) + { + stored.pending_session = Some(session); + } else { + return Ok(json!({"status":"expired", "error":"QR login was cancelled"})); + } + Ok(json!({"status":"authorized"})) + } + + pub fn complete_device_login(&self, params: &Value) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let attempt_id = required_param(params, "attemptId")?; + crate::requests::current().commit(|| { + let mut state = self.state.lock().expect("GFN state poisoned"); + let attempt = state + .attempts + .get(attempt_id) + .filter(|attempt| { + attempt.deadline > Instant::now() && attempt.expires_at > now_ms() + }) + .ok_or_else(|| ServiceError::invalid("QR login is no longer active"))?; + let session = attempt + .pending_session + .clone() + .ok_or_else(|| ServiceError::invalid("QR login has not been authorized yet"))?; + let persist = params["staySignedIn"].as_bool().unwrap_or(true); + state.persistence_intent = if persist { + PersistenceIntent::SecureStore + } else { + PersistenceIntent::MemoryOnly + }; + state.persistence_state = if persist { + match self.vault.save(&session) { + Ok(()) => self.vault.persistence_state(&session), + Err(_) => self.vault.failed_save_state(&session), + } + } else { + let _ = self.vault.remove(&session.user.user_id); + "memory-only" + } + .into(); + state.attempts.clear(); + self.account_connections.cancel_pending(); + state.session = Some(session); + state.restore_attempted = true; + state.generation += 1; + state.login_generation += 1; + state.refresh_retry_at = None; + Ok(self.auth_envelope( + &state, + state.session.as_ref(), + json!({"attempted":false,"outcome":"not_attempted"}), + )) + }) + } + + pub fn cancel_device_login(&self, params: &Value) -> Result { + let attempt_id = required_param(params, "attemptId")?; + self.state + .lock() + .expect("GFN state poisoned") + .attempts + .remove(attempt_id); + Ok(json!({"cancelled":true})) + } + + fn check_device_attempt(&self, attempt_id: &str) -> Result<(), ServiceError> { + crate::requests::check()?; + if self + .state + .lock() + .expect("GFN state poisoned") + .attempts + .get(attempt_id) + .is_none_or(|attempt| { + attempt.deadline <= Instant::now() || attempt.expires_at <= now_ms() + }) + { + return Err(ServiceError::invalid("QR login was cancelled or expired")); + } + Ok(()) + } + + fn auth_envelope( + &self, + state: &ServiceState, + session: Option<&AuthSession>, + refresh: Value, + ) -> Value { + json!({"session":session.map(AuthSession::public), "generation":state.generation, + "persistence":state.persistence_state, "refresh":refresh, "warnings":self.vault.warnings(), + "deviceIdentity": if self.device_identity_error.is_some() { "unavailable" } else { "durable" }}) + } + + pub fn session(&self) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + self.session_locked() + } + + fn session_locked(&self) -> Result { + let (session, refresh) = + self.resolve_session_locked(TokenPurpose::StarfleetAccess, false)?; + let state = self.state.lock().expect("GFN state poisoned"); + Ok(self.auth_envelope(&state, session.as_ref(), refresh)) + } + + fn resolve_session_locked( + &self, + purpose: TokenPurpose, + force: bool, + ) -> Result<(Option, Value), ServiceError> { + { + let mut state = self.state.lock().expect("GFN state poisoned"); + if state.session.is_none() && !state.restore_attempted { + state.restore_attempted = true; + drop(state); + match self.vault.load_active() { + Ok(session) => { + let session = + session.filter(|session| !self.profiles.has_pin(&session.user.user_id)); + let mut state = self.state.lock().expect("GFN state poisoned"); + state.persistence_state = session + .as_ref() + .map(|session| self.vault.persistence_state(session)) + .unwrap_or("none") + .to_owned(); + state.persistence_intent = PersistenceIntent::SecureStore; + state.generation += 1; + state.session = session; + } + Err(error) => { + eprintln!("auth: saved session unavailable: {error}"); + self.state + .lock() + .expect("GFN state poisoned") + .persistence_state = "unavailable".to_owned(); + } + } + } + } + let current = self + .state + .lock() + .expect("GFN state poisoned") + .session + .clone(); + let Some(current) = current else { + return Ok(( + None, + json!({"attempted":false,"outcome":"not_attempted","message":"No saved session found."}), + )); + }; + + let needs_refresh = force + || current.tokens.expiry(purpose) <= now_ms().saturating_add(TOKEN_REFRESH_WINDOW_MS); + let needs_client_token = current + .tokens + .client_token + .as_deref() + .unwrap_or("") + .is_empty() + || current + .tokens + .client_token_expires_at + .is_none_or(|expiry| expiry <= now_ms() + CLIENT_TOKEN_REFRESH_WINDOW_MS); + if self + .state + .lock() + .expect("GFN state poisoned") + .refresh_retry_at + .is_some_and(|deadline| deadline > Instant::now()) + { + return Ok(( + if !force && current.tokens.expiry(purpose) > now_ms() { + Some(current) + } else { + None + }, + json!({"attempted":false,"outcome":"deferred","message":"Authentication renewal is temporarily deferred."}), + )); + } + let (session, refresh) = if needs_refresh { + match self.refresh_session(¤t) { + Ok(session) if session.tokens.expiry(purpose) > now_ms() => ( + Some(session), + json!({"attempted":true,"outcome":"refreshed","message":"Saved session token refreshed."}), + ), + Ok(_) => { + self.defer_refresh(); + self.mark_needs_sign_in(¤t.user.user_id); + ( + None, + json!({"attempted":true,"outcome":"expired","message":"The required authentication token was not renewed. Sign in again."}), + ) + } + Err(error) + if !force + && current.tokens.expiry(purpose) > now_ms() + && !is_definitive_auth_revocation(&error) => + { + self.defer_refresh(); + eprintln!( + "auth: refresh failed; using unexpired token: {}", + error.message + ); + ( + Some(current), + json!({"attempted":true,"outcome":"failed","message":"Refresh failed; using the unexpired saved token."}), + ) + } + Err(error) if is_definitive_auth_revocation(&error) => { + eprintln!("auth: saved session was revoked: {}", error.message); + let _ = self.vault.remove(¤t.user.user_id); + self.account_connections.cancel_pending(); + let mut state = self.state.lock().expect("GFN state poisoned"); + state.session = None; + state.persistence_state = "none".into(); + state.persistence_intent = PersistenceIntent::MemoryOnly; + state.restore_attempted = true; + state.generation += 1; + ( + None, + json!({"attempted":true,"outcome":"revoked","message":"Saved session is no longer valid. Sign in again."}), + ) + } + Err(error) => { + self.defer_refresh(); + eprintln!("auth: expired session could not refresh: {}", error.message); + if error.code == "http_unauthorized" { + self.mark_needs_sign_in(¤t.user.user_id); + } + ( + None, + json!({"attempted":true,"outcome":"expired","message":"Saved session expired. Sign in again if this continues."}), + ) + } + } + } else if needs_client_token { + match self.update_client_token(¤t) { + Ok(session) => ( + Some(session), + json!({"attempted":true,"outcome":"refreshed","message":"Client token refreshed."}), + ), + Err(error) => { + self.defer_refresh(); + eprintln!("auth: client-token bootstrap deferred: {}", error.message); + ( + Some(current), + json!({"attempted":true,"outcome":"failed","message":"Session is valid; client-token refresh was deferred."}), + ) + } + } + } else { + ( + Some(current), + json!({"attempted":false,"outcome":"not_attempted","message":"Session token is still valid."}), + ) + }; + Ok((session, refresh)) + } + + fn mark_needs_sign_in(&self, user_id: &str) { + if let Err(error) = self.vault.mark_needs_sign_in(user_id) { + eprintln!("auth: expired account state could not be saved: {error}"); + } + } + + fn defer_refresh(&self) { + self.state + .lock() + .expect("GFN state poisoned") + .refresh_retry_at = Some(Instant::now() + Duration::from_secs(30)); + } + + pub fn logout(&self) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let session = self + .state + .lock() + .expect("GFN state poisoned") + .session + .clone(); + self.invalidate_auth_work(true); + let cleanup = session.as_ref().map(|session| { + self.cleanup_account( + &session.user.user_id, + Some(session), + Instant::now() + Duration::from_secs(5), + ) + }); + self.restore_next_account(); + let state = self.state.lock().expect("GFN state poisoned"); + let mut result = self.auth_envelope(&state, state.session.as_ref(), Value::Null); + result["ok"] = json!(true); + result["cleanup"] = json!(cleanup); + Ok(result) + } + + pub fn logout_all(&self) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let active = self + .state + .lock() + .expect("GFN state poisoned") + .session + .clone(); + self.invalidate_auth_work(true); + let deadline = Instant::now() + Duration::from_secs(5); + let mut ids: Vec = self + .vault + .list() + .unwrap_or_default() + .iter() + .filter_map(|account| account["userId"].as_str().map(str::to_owned)) + .collect(); + if let Some(session) = &active { + if !ids.contains(&session.user.user_id) { + ids.push(session.user.user_id.clone()); + } + } + let mut cleanup = Vec::new(); + for id in ids { + let session = active + .as_ref() + .filter(|session| session.user.user_id == id) + .cloned() + .or_else(|| self.vault.load(&id).ok().flatten()); + cleanup.push(self.cleanup_account(&id, session.as_ref(), deadline)); + } + let local = self.vault.remove_all(); + let profiles = self.profiles.forget_all(); + let state = self.state.lock().expect("GFN state poisoned"); + let mut result = self.auth_envelope(&state, None, Value::Null); + result["ok"] = json!(true); + result["cleanup"] = json!(cleanup); + result["localCleanup"] = json!(if local.is_ok() && profiles.is_ok() { + "complete" + } else { + "pending" + }); + Ok(result) + } + + fn invalidate_auth_work(&self, clear_session: bool) { + self.account_connections.cancel_pending(); + let mut state = self.state.lock().expect("GFN state poisoned"); + state.attempts.clear(); + state.login_generation += 1; + state.generation += 1; + state.refresh_retry_at = None; + state.restore_attempted = true; + if clear_session { + state.session = None; + state.persistence_state = "none".into(); + state.persistence_intent = PersistenceIntent::MemoryOnly; + } + } + + fn cleanup_account( + &self, + user_id: &str, + session: Option<&AuthSession>, + deadline: Instant, + ) -> Value { + let remote = if let Some(session) = session.filter(|_| Instant::now() < deadline) { + match self + .client + .delete(&self.endpoints.revoke) + .timeout( + deadline + .saturating_duration_since(Instant::now()) + .max(Duration::from_millis(1)), + ) + .bearer_auth(&session.tokens.access_token) + .header(ACCEPT, "application/json") + .send() + { + Ok(response) if response.status().is_success() => "complete", + _ => "failed", + } + } else { + "not_attempted" + }; + let local = self.vault.remove(user_id); + let profile = self.profiles.forget(user_id); + json!({"remoteRevoke":remote, "localCleanup":if local.is_ok() && profile.is_ok() { "complete" } else { "pending" }}) + } + + fn restore_next_account(&self) { + let next = self + .vault + .load_active() + .ok() + .flatten() + .filter(|session| !self.profiles.has_pin(&session.user.user_id)); + let mut state = self.state.lock().expect("GFN state poisoned"); + state.persistence_state = match &next { + Some(session) => self.vault.persistence_state(session), + None => "none", + } + .into(); + state.persistence_intent = if next.is_some() { + PersistenceIntent::SecureStore + } else { + PersistenceIntent::MemoryOnly + }; + state.session = next; + } + + pub fn clear_cache(&self) -> Value { + let mut state = self.state.lock().expect("GFN state poisoned"); + let catalog_entries = state.public_games.len(); + let provider_entries = state.providers.len(); + state.public_games.clear(); + state.providers.clear(); + state.providers_expires = None; + state.providers_retry = None; + state.providers_error = None; + state.generation += 1; + json!({"ok":true,"catalogEntries":catalog_entries,"providerEntries":provider_entries}) + } + + pub fn device_id(&self) -> &str { + &self.device_id + } + + pub fn with_region_provider( + &self, + provider: &str, + write: impl FnOnce() -> Result, + ) -> Result { + let _operation = crate::store_requests::lock(&self.auth_operation)?; + if self + .state + .lock() + .expect("GFN state poisoned") + .session + .as_ref() + .is_none_or(|session| session.provider.idp_id != provider) + { + return Err(ServiceError { + code: "stale_account", + message: "The provider changed before saving the region".into(), + }); + } + write() + } + + pub fn saved_accounts(&self) -> Result { + let mut accounts = self.vault.list().map_err(|message| ServiceError { + code: "credential_store_error", + message, + })?; + let (active_user_id, providers) = { + let state = self.state.lock().expect("GFN state poisoned"); + ( + state + .session + .as_ref() + .map(|session| session.user.user_id.clone()), + state.providers.clone(), + ) + }; + for account in &mut accounts { + let user_id = account["userId"].as_str().unwrap_or_default(); + account["hasPin"] = Value::Bool(self.profiles.has_pin(user_id)); + let code = account["providerCode"] + .as_str() + .filter(|code| !code.is_empty()) + .unwrap_or("NVIDIA") + .to_owned(); + let discovered = providers + .iter() + .find(|provider| provider.code == code) + .map(|provider| provider.display_name.as_str()); + let name = + provider_display_name(&code, account["providerName"].as_str().or(discovered)) + .unwrap_or_else(|| code.clone()); + account["providerName"] = Value::String(name); + account["alliance"] = Value::Bool(is_alliance_provider(&code)); + } + Ok( + json!({"accounts":accounts,"activeUserId":active_user_id,"generation":self.auth_generation()}), + ) + } + + pub fn switch_account(&self, params: &Value) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let user_id = required_param(params, "userId")?; + if self.profiles.has_pin(user_id) { + let verification = self + .profiles + .verify(user_id, params["pin"].as_str().unwrap_or("")) + .map_err(|message| ServiceError { + code: "profile_storage_error", + message, + })?; + if verification["ok"].as_bool() != Some(true) { + return Err(ServiceError { + code: if verification["reason"] == "locked_out" { + "profile_pin_locked" + } else { + "profile_pin_required" + }, + message: if verification["reason"] == "locked_out" { + "Profile PIN is temporarily locked".to_owned() + } else { + "Profile PIN is required or incorrect".to_owned() + }, + }); + } + } + let session = self + .vault + .load(user_id) + .map_err(|message| ServiceError { + code: "credential_store_error", + message, + })? + .ok_or_else(|| ServiceError { + code: "saved_account_not_found", + message: "Saved account not found".to_owned(), + })?; + if session.user.user_id != user_id { + return Err(ServiceError { + code: "session_identity_mismatch", + message: "Saved session did not match the selected account".to_owned(), + }); + } + self.vault + .set_active(user_id) + .map_err(|message| ServiceError { + code: "credential_store_error", + message, + })?; + { + let mut state = self.state.lock().expect("GFN state poisoned"); + state.persistence_state = self.vault.persistence_state(&session).to_owned(); + state.persistence_intent = PersistenceIntent::SecureStore; + state.session = Some(session); + } + self.invalidate_auth_work(false); + let result = self.session_locked()?; + Ok(result) + } + + pub fn remove_account(&self, params: &Value) -> Result { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let user_id = required_param(params, "userId")?; + let active = self + .state + .lock() + .expect("GFN state poisoned") + .session + .clone(); + let was_active = active + .as_ref() + .is_some_and(|session| session.user.user_id == user_id); + let selected = active + .filter(|session| session.user.user_id == user_id) + .or_else(|| self.vault.load(user_id).ok().flatten()); + self.invalidate_auth_work(was_active); + let cleanup = self.cleanup_account( + user_id, + selected.as_ref(), + Instant::now() + Duration::from_secs(5), + ); + if was_active { + self.restore_next_account(); + } + let state = self.state.lock().expect("GFN state poisoned"); + let mut result = self.auth_envelope(&state, state.session.as_ref(), Value::Null); + result["ok"] = json!(true); + result["cleanup"] = cleanup; + Ok(result) + } + + pub fn pin_status(&self, params: &Value) -> Result { + let user_id = self.profile_user_id(params)?; + Ok(self.profiles.status(&user_id)) + } + + pub fn set_pin(&self, params: &Value) -> Result { + let user_id = self.profile_user_id(params)?; + let pin = required_param(params, "pin")?; + self.profiles + .set_pin(&user_id, pin, params["currentPin"].as_str()) + .map_err(|message| ServiceError { + code: "profile_storage_error", + message, + }) + } + + pub fn clear_pin(&self, params: &Value) -> Result { + let user_id = self.profile_user_id(params)?; + let pin = required_param(params, "currentPin")?; + self.profiles + .clear_pin(&user_id, pin) + .map_err(|message| ServiceError { + code: "profile_storage_error", + message, + }) + } + + pub fn verify_pin(&self, params: &Value) -> Result { + let user_id = self.profile_user_id(params)?; + let pin = params["pin"].as_str().unwrap_or(""); + self.profiles + .verify(&user_id, pin) + .map_err(|message| ServiceError { + code: "profile_storage_error", + message, + }) + } + + fn profile_user_id(&self, params: &Value) -> Result { + if let Some(user_id) = params["userId"].as_str().filter(|value| !value.is_empty()) { + return Ok(user_id.to_owned()); + } + self.state + .lock() + .expect("GFN state poisoned") + .session + .as_ref() + .map(|session| session.user.user_id.clone()) + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "Sign in to manage a profile PIN".to_owned(), + }) + } + + pub fn public_catalog(&self, params: &Value, settings: &Value) -> Result { + let limit = params["limit"].as_u64().unwrap_or(240).clamp(1, 1000) as usize; + let query = params["searchQuery"] + .as_str() + .unwrap_or("") + .trim() + .to_lowercase(); + let proxy = config_from_settings(settings).map_err(ServiceError::invalid)?; + let proxy_scope = proxy + .as_ref() + .map(|value| value.cache_scope.clone()) + .unwrap_or_else(|| "direct".to_owned()); + let bypass_cache = proxy.as_ref().is_some_and(|value| value.has_credentials); + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let refresh = params["refresh"].as_bool().unwrap_or(false) || bypass_cache; + let mut cached = { + let state = self.state.lock().expect("GFN state poisoned"); + if state.public_games_proxy_scope == proxy_scope { + state.public_games.clone() + } else { + Vec::new() + } + }; + if cached.is_empty() || refresh { + let response = client + .get(&self.endpoints.public_catalog) + .header(ACCEPT, "application/json") + .header(USER_AGENT, GFN_USER_AGENT) + .send() + .map_err(|error| ServiceError::network("Public games fetch failed", error))?; + if !response.status().is_success() { + return Err(ServiceError::response( + "Public games fetch failed", + response, + )); + } + let raw = response + .json::>() + .map_err(|error| ServiceError::network("Invalid public games response", error))?; + cached = raw.iter().filter_map(public_game_to_info).collect(); + cached.sort_by(|left, right| { + left["title"] + .as_str() + .unwrap_or("") + .to_lowercase() + .cmp(&right["title"].as_str().unwrap_or("").to_lowercase()) + }); + if !bypass_cache { + let mut state = self.state.lock().expect("GFN state poisoned"); + state.public_games = cached.clone(); + state.public_games_proxy_scope = proxy_scope; + } + } + let filtered = cached + .iter() + .filter(|game| { + query.is_empty() + || game["searchText"] + .as_str() + .is_some_and(|text| text.contains(&query)) + }) + .take(limit) + .cloned() + .collect::>(); + Ok( + json!({"games":filtered, "count":filtered.len(), "totalCount":cached.len(), "fetchedAt":now_ms()}), + ) + } + + pub fn regions(&self, settings: &Value) -> Result { + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + self.authenticated_read(|session, generation| { + self.check_scope(session, generation)?; + self.provider_regions(&client, session) + }) + } + + fn provider_regions( + &self, + client: &Client, + session: &AuthSession, + ) -> Result { + let token = session.tokens.service_token(); + let base = provider_streaming_base(&session.provider)?; + let url = self.server_info_url(&base)?; + let response = client + .get(url) + .headers(lcars_headers(token, "BROWSER", "WEBRTC", false)?) + .send() + .map_err(|error| { + ServiceError::server_info(&session.provider, "Region discovery failed", error) + })?; + if !response.status().is_success() { + return Err(ServiceError::response("Region discovery failed", response)); + } + let payload = response + .json::() + .map_err(|error| ServiceError::network("Invalid region response", error))?; + let vpc_id = verified_vpc(&payload)?; + if payload["metaData"].as_array().is_none() { + return Err(ServiceError { + code: "invalid_upstream_response", + message: "Provider server info has no regions or verified VPC".into(), + }); + } + let regions = provider_region_entries(&payload); + Ok(json!({"regions":regions,"vpcId":vpc_id,"providerIdpId":session.provider.idp_id})) + } + + pub fn subscription(&self, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session.tokens.service_token(); + let vpc_id = self.vpc_id(&client, session, generation, settings, token, None)?; + let steam_deck = settings["identifyAsSteamDeck"].as_bool().unwrap_or(false); + let mut url = url::Url::parse(&self.endpoints.subscription).expect("MES URL is valid"); + url.query_pairs_mut() + .append_pair("serviceName", "gfn_pc") + .append_pair("languageCode", "en_US") + .append_pair("vpcId", &vpc_id) + .append_pair("userId", &session.user.user_id); + let response = client + .get(url) + .headers(lcars_headers( + token, + "NATIVE", + "NVIDIA-CLASSIC", + steam_deck, + )?) + .send() + .map_err(|error| ServiceError::network("Subscription request failed", error))?; + if !response.status().is_success() { + return Err(ServiceError::response( + "Subscription request failed", + response, + )); + } + let data = response + .json::() + .map_err(|error| ServiceError::network("Invalid subscription response", error))?; + let allotted = number_value(&data["allottedTimeInMinutes"]).unwrap_or(0.0); + let purchased = number_value(&data["purchasedTimeInMinutes"]).unwrap_or(0.0); + let rolled = number_value(&data["rolledOverTimeInMinutes"]).unwrap_or(0.0); + let total = + number_value(&data["totalTimeInMinutes"]).unwrap_or(allotted + purchased + rolled); + let remaining = number_value(&data["remainingTimeInMinutes"]); + let mut resolutions = data["features"]["resolutions"].as_array().into_iter().flatten() + .filter(|resolution| resolution["isEntitled"].as_bool() == Some(true)) + .map(|resolution| json!({"width":resolution["widthInPixels"],"height":resolution["heightInPixels"],"fps":resolution["framesPerSecond"]})) + .collect::>(); + resolutions.sort_by(|left, right| { + right["width"] + .as_i64() + .cmp(&left["width"].as_i64()) + .then_with(|| right["height"].as_i64().cmp(&left["height"].as_i64())) + .then_with(|| right["fps"].as_i64().cmp(&left["fps"].as_i64())) + }); + let membership = data["membershipTier"].as_str().unwrap_or("FREE"); + let storage_addon = data["addons"].as_array().into_iter().flatten().find(|addon| { + addon["type"].as_str() == Some("STORAGE") + && addon["subType"].as_str() == Some("PERMANENT_STORAGE") + && addon["status"].as_str() == Some("OK") + }).map(|addon| { + let attribute = |key: &str| addon["attributes"].as_array().into_iter().flatten() + .find(|attribute| attribute["key"].as_str() == Some(key)) + .and_then(|attribute| attribute["textValue"].as_str()); + json!({ + "type":"PERMANENT_STORAGE", + "sizeGb":attribute("TOTAL_STORAGE_SIZE_IN_GB").and_then(|value| value.parse::().ok()), + "usedGb":attribute("USED_STORAGE_SIZE_IN_GB").and_then(|value| value.parse::().ok()), + "regionName":attribute("STORAGE_METRO_REGION_NAME"), + "regionCode":attribute("STORAGE_METRO_REGION") + }) + }); + self.check_scope(session, generation)?; + Ok(json!({"subscription":{ + "membershipTier":membership,"subscriptionType":data["type"],"subscriptionSubType":data["subType"], + "allottedHours":allotted/60.0,"purchasedHours":purchased/60.0,"rolledOverHours":rolled/60.0, + "usedHours":remaining.map(|remaining| (total-remaining).max(0.0)/60.0), + "remainingHours":remaining.map(|remaining| remaining/60.0),"totalHours":total/60.0, + "firstEntitlementStartDateTime":data["firstEntitlementStartDateTime"],"serverRegionId":vpc_id, + "currentSpanStartDateTime":data["currentSpanStartDateTime"],"currentSpanEndDateTime":data["currentSpanEndDateTime"], + "notifyUserWhenTimeRemainingInMinutes":data["notifications"]["notifyUserWhenTimeRemainingInMinutes"], + "notifyUserOnSessionWhenRemainingTimeInMinutes":data["notifications"]["notifyUserOnSessionWhenRemainingTimeInMinutes"], + "state":data["currentSubscriptionState"]["state"],"isGamePlayAllowed":data["currentSubscriptionState"]["isGamePlayAllowed"], + "isUnlimited":data["subType"] == "UNLIMITED","entitledResolutions":resolutions,"storageAddon":storage_addon, + "dlssOverrides":crate::dlss::entitlements(&data) + }})) + }) + } + + pub fn create_session( + &self, + params: &Value, + settings: &Value, + resolve_profile: impl FnOnce(&Value) -> Result, + ) -> Result { + let admission = self.cloudmatch.admit_create()?; + let app_id = params["catalogAppId"].as_str().unwrap_or_default(); + let variant_id = params["variantId"].as_str().unwrap_or_default(); + if params["appId"].as_str() != Some(variant_id) { + return Err(ServiceError::invalid( + "The launch ID must match the selected variant", + )); + } + self.providers()?; + let (intent_session, intent_generation) = + self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + if params["scope"] != scoped_result(json!({}), &intent_session, intent_generation)["scope"] + { + return Err(ServiceError { + code: "stale_account", + message: "This launch belongs to a different account context.".into(), + }); + } + let _catalog_action = self.admit_catalog_action(&intent_session, app_id)?; + let store_launch = store_launch::store_launch_intent(params)?; + let inspection = self.catalog_launch_inspect( + &json!({"appId":app_id,"variantId":variant_id,"storeLaunch":store_launch}), + settings, + )?; + if inspection["decision"]["status"] != "ready" { + return Err(ServiceError { + code: "launch_not_ready", + message: inspection["decision"]["message"] + .as_str() + .unwrap_or("The selected store version cannot be launched") + .into(), + }); + } + self.providers()?; + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (session, generation) = self.session_snapshot_locked()?; + if inspection["scope"] != params["scope"] + || inspection["scope"] != scoped_result(json!({}), &session, generation)["scope"] + || inspection["catalogRevision"].as_u64() + != Some( + self.catalog_revision + .load(std::sync::atomic::Ordering::Acquire), + ) + { + return Err(ServiceError { + code: "stale_account", + message: "The launch context changed. Try again.".into(), + }); + } + if !self.cloudmatch.active()["session"].is_null() { + return Err(ServiceError { + code: "session_update_busy", + message: "End the active session before starting another game".into(), + }); + } + let variant = catalog_actions::selected_variant(&inspection["game"], variant_id) + .expect("ready decision validated the exact variant"); + let settings = resolve_profile(&if store_launch { + settings.clone() + } else { + crate::cloudmatch::catalog_hdr_settings( + settings, + ¶ms["runtimeCapabilities"], + variant, + ) + })?; + let (mut params, settings) = self.scoped_session_route(params, &settings, &session)?; + params["accountLinked"] = variant["inLibrary"].clone(); + params["supportsInGameSettingsPersistence"] = + variant["supportsInGameSettingsPersistence"].clone(); + params["title"] = inspection["game"]["title"].clone(); + let result = admission + .create(¶ms, &settings, &session, &self.device_id) + .map(|result| scoped_result(result, &session, generation)); + if !self.cloudmatch.active()["session"].is_null() { + routing.active_owner = Some(ActiveSeatOwner::capture( + session, + generation, + &self.cloudmatch.active()["session"], + Some(generation), + )?); + } + result + } + + pub fn poll_session(&self, params: &Value) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + if let Some(owner) = &routing.active_owner + && !self + .state + .lock() + .expect("GFN state poisoned") + .session + .as_ref() + .is_some_and(|current| owner.matches(current, &owner.session_id)) + { + if owner.auth.tokens.expiry(TokenPurpose::ServiceId) <= now_ms() { + return Err(ServiceError { + code: "session_owner_authentication_required", + message: + "Sign in to the active session's original account to continue managing it" + .into(), + }); + } + let active = self.cloudmatch.active()["session"].clone(); + if params["sessionId"] + .as_str() + .is_some_and(|id| active["sessionId"] != id) + || active["sessionId"] != owner.session_id + { + return Err(session_owner_error()); + } + let result = self + .cloudmatch + .poll(&active, &owner.auth, &self.device_id) + .map(|result| scoped_result(result, &owner.auth, owner.last_published_generation)) + .map_err(|mut error| { + if error.code == "http_unauthorized" { + error.code = "session_owner_authentication_required"; + } + error + }); + if self.cloudmatch.active()["session"].is_null() { + routing.active_owner = None; + } + return result; + } + let result = self.session_read_locked(|session, generation| { + let params = self.owned_session_params(params, &routing, session, generation, false)?; + self.cloudmatch.poll(¶ms, session, &self.device_id) + }); + result.and_then(|(result, session, generation)| { + self.publish_active_result(&mut routing, &session, generation, result) + }) + } + + pub fn finish_session_create( + &self, + session_id: &str, + accepted: bool, + ) -> Result<(), ServiceError> { + let mut routing = self + .session_routing + .lock() + .expect("Session routing poisoned"); + let accepted = accepted + && routing.active_owner.as_ref().is_some_and(|owner| { + owner.session_id == session_id + && owner + .allocation_generation + .is_some_and(|generation| self.check_scope(&owner.auth, generation).is_ok()) + }); + let result = self.cloudmatch.finish_create(session_id, accepted); + if self.cloudmatch.active()["session"].is_null() { + routing.active_owner = None; + } else if result.is_ok() + && let Some(owner) = &mut routing.active_owner + && owner.session_id == session_id + { + owner.allocation_generation = None; + } + result + } + + pub fn stop_session(&self, params: &Value, settings: &Value) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let active = self.cloudmatch.active()["session"].clone(); + let requested = params["sessionId"].as_str().unwrap_or(""); + if !active.is_null() && (requested.is_empty() || active["sessionId"] == requested) { + let owner = routing + .active_owner + .as_ref() + .ok_or_else(session_owner_error)?; + let selected_owner = self + .state + .lock() + .expect("GFN state poisoned") + .session + .as_ref() + .is_some_and(|current| owner.matches(current, &owner.session_id)); + let (session, generation) = if selected_owner { + self.session_snapshot_locked()? + } else { + (owner.auth.clone(), owner.last_published_generation) + }; + if session.tokens.expiry(TokenPurpose::ServiceId) <= now_ms() { + return Err(ServiceError { + code: "authentication_required", + message: "Sign in to the session's original account to end it".into(), + }); + } + if active["sessionId"] != owner.session_id { + return Err(session_owner_error()); + } + let result = self + .cloudmatch + .stop(&active, settings, &session, &self.device_id) + .and_then(|result| { + self.publish_active_result(&mut routing, &session, generation, result) + }); + return result; + } + let (session, generation) = self.session_snapshot_locked()?; + let params = self.owned_session_params(params, &routing, &session, generation, true)?; + self.cloudmatch + .stop(¶ms, settings, &session, &self.device_id) + .map(|result| scoped_result(result, &session, generation)) + } + + pub fn acknowledge_session_end(&self, params: &Value) -> Result { + let notification: SessionEndNotification = serde_json::from_value(params.clone()) + .map_err(|_| ServiceError::invalid("Session end requires sessionId and ownerScope"))?; + if notification.session_id.is_empty() + || notification.session_id.len() > 256 + || notification.owner_scope.user_id.is_empty() + || notification.owner_scope.user_id.len() > 1024 + || notification.owner_scope.provider_idp_id.is_empty() + || notification.owner_scope.provider_idp_id.len() > 1024 + { + return Err(ServiceError::invalid("Invalid ended session identity")); + } + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let Some(owner) = routing.active_owner.as_ref().filter(|owner| { + owner.session_id == notification.session_id + && owner.auth.user.user_id == notification.owner_scope.user_id + && owner.auth.provider.idp_id == notification.owner_scope.provider_idp_id + }) else { + return Ok(json!({"sessionId":notification.session_id,"cleared":false})); + }; + if owner.allocation_generation.is_some() { + return Err(ServiceError { + code: "session_update_busy", + message: "The session allocation handoff is still completing. Retry cleanup." + .into(), + }); + } + self.cloudmatch.clear_active(¬ification.session_id); + routing.active_owner = None; + Ok(json!({"sessionId":notification.session_id,"cleared":true})) + } + + pub fn reconcile_active_session( + &self, + params: &Value, + settings: &Value, + ) -> Result { + if params.get("sessionId").is_none() && params.get("ownerScope").is_none() { + return self.active_session(); + } + let hint: SessionReconciliationHint = + serde_json::from_value(params.clone()).map_err(|_| { + ServiceError::invalid("Session reconciliation requires sessionId and ownerScope") + })?; + if hint.session_id.is_empty() + || hint.session_id.len() > 256 + || !hint + .session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + || hint.owner_scope.user_id.is_empty() + || hint.owner_scope.user_id.len() > 1024 + || hint.owner_scope.provider_idp_id.is_empty() + || hint.owner_scope.provider_idp_id.len() > 1024 + { + return Err(ServiceError::invalid( + "Invalid session reconciliation identity", + )); + } + self.providers()?; + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (result, session, generation) = self.session_read_locked(|session, generation| { + if hint.owner_scope.user_id != session.user.user_id + || hint.owner_scope.provider_idp_id != session.provider.idp_id + || routing + .active_owner + .as_ref() + .is_some_and(|owner| !owner.matches(session, &hint.session_id)) + { + return Err(session_owner_error()); + } + let active = self.cloudmatch.active(); + if !active["session"].is_null() { + if active["session"]["sessionId"] != hint.session_id + || !routing + .active_owner + .as_ref() + .is_some_and(|owner| owner.matches(session, &hint.session_id)) + { + return Err(session_owner_error()); + } + self.check_scope(session, generation)?; + return Ok(active); + } + let (params, settings) = self.scoped_session_route(&json!({}), settings, session)?; + self.cloudmatch + .remote_sessions(¶ms, &settings, session, &self.device_id)?; + self.check_scope(session, generation)?; + routing.discovery_owner = Some(( + session.provider.idp_id.clone(), + session.user.user_id.clone(), + generation, + )); + let discovered = self + .cloudmatch + .discovered_session(&hint.session_id) + .ok_or_else(|| ServiceError { + code: "session_discovery_failed", + message: "The existing stream's session was not found. Retry reconciliation." + .into(), + })?; + self.cloudmatch + .reconcile(&discovered, &settings, session, &self.device_id, || { + self.check_scope(session, generation) + }) + })?; + if routing.active_owner.is_none() && !self.cloudmatch.active()["session"].is_null() { + routing.active_owner = Some(ActiveSeatOwner::capture( + session.clone(), + generation, + &self.cloudmatch.active()["session"], + None, + )?); + } + self.publish_active_result(&mut routing, &session, generation, result) + } + + pub fn active_session(&self) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let state = self.state.lock().expect("GFN state poisoned"); + let active = self.cloudmatch.active(); + let Some(session) = state.session.as_ref().filter(|session| { + routing.active_owner.as_ref().is_some_and(|owner| { + owner.matches( + session, + active["session"]["sessionId"].as_str().unwrap_or(""), + ) + }) + }) else { + return Ok(json!({"session":null})); + }; + let session = session.clone(); + let generation = state.generation; + drop(state); + self.publish_active_result(&mut routing, &session, generation, active) + } + + pub fn remote_sessions(&self, params: &Value, settings: &Value) -> Result { + self.providers()?; + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (result, session, generation) = self.session_read_locked(|session, _| { + let (mut params, settings) = self.scoped_session_route(params, settings, session)?; + if routing.active_owner.as_ref().is_none_or(|owner| { + !owner.matches(session, params["sessionId"].as_str().unwrap_or("")) + }) { + if let Some(params) = params.as_object_mut() { + params.remove("sessionId"); + } + } + self.cloudmatch + .remote_sessions(¶ms, &settings, session, &self.device_id) + })?; + routing.discovery_owner = Some((session.provider.idp_id, session.user.user_id, generation)); + Ok(result) + } + + pub fn claim_session(&self, params: &Value, settings: &Value) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (session, generation) = self.session_snapshot_locked()?; + let params = self.owned_session_params(params, &routing, &session, generation, true)?; + let active = self.cloudmatch.active()["session"].clone(); + if !active.is_null() && active["sessionId"] != params["sessionId"] { + return Err(session_owner_error()); + } + let result = scoped_result( + self.cloudmatch + .claim(¶ms, settings, &session, &self.device_id)?, + &session, + generation, + ); + if routing.active_owner.is_none() && !self.cloudmatch.active()["session"].is_null() { + routing.active_owner = Some(ActiveSeatOwner::capture( + session.clone(), + generation, + &self.cloudmatch.active()["session"], + None, + )?); + } + self.publish_active_result(&mut routing, &session, generation, result) + } + + pub fn report_session_ad(&self, params: &Value) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (session, generation) = self.session_snapshot_locked()?; + let mut owned = self.owned_session_params(params, &routing, &session, generation, false)?; + for key in [ + "action", + "adId", + "watchedTimeInMs", + "pausedTimeInMs", + "cancelReason", + "clientTimestamp", + ] { + if let Some(value) = params.get(key) { + owned[key] = value.clone(); + } + } + self.cloudmatch + .report_ad(&owned, &session, &self.device_id) + .and_then(|result| { + self.publish_active_result(&mut routing, &session, generation, result) + }) + } + + fn publish_active_result( + &self, + routing: &mut SessionRouting, + session: &AuthSession, + generation: u64, + result: Value, + ) -> Result { + let active = self.cloudmatch.active(); + if active["session"].is_null() { + routing.active_owner = None; + } else { + self.check_scope(session, generation)?; + let owner = routing + .active_owner + .as_mut() + .filter(|owner| { + owner.matches( + session, + active["session"]["sessionId"].as_str().unwrap_or(""), + ) + }) + .ok_or_else(session_owner_error)?; + owner.auth = session.clone(); + owner.last_published_generation = generation; + } + Ok(scoped_result(result, session, generation)) + } + + fn session_snapshot_locked(&self) -> Result<(AuthSession, u64), ServiceError> { + let session = self + .resolve_session_locked(TokenPurpose::ServiceId, false)? + .0 + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "Sign in to manage streaming sessions".into(), + })?; + let state = self.state.lock().expect("GFN state poisoned"); + let mut session = session; + if let Some(provider) = state + .providers + .iter() + .find(|provider| provider.idp_id == session.provider.idp_id) + { + session.provider = provider.clone(); + } + provider_streaming_base(&session.provider)?; + Ok((session, state.generation)) + } + + fn session_read_locked( + &self, + mut read: impl FnMut(&AuthSession, u64) -> Result, + ) -> Result<(Value, AuthSession, u64), ServiceError> { + let (session, generation) = self.session_snapshot_locked()?; + let result = read(&session, generation); + if result + .as_ref() + .is_err_and(|error| error.code == "http_unauthorized") + { + let renewed = self + .resolve_session_locked(TokenPurpose::ServiceId, true)? + .0 + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "Sign in to renew the session credential".into(), + })?; + self.check_scope(&session, generation)?; + if renewed.provider.idp_id != session.provider.idp_id + || renewed.user.user_id != session.user.user_id + { + return Err(session_owner_error()); + } + let result = read(&renewed, generation)?; + return Ok(( + scoped_result(result, &renewed, generation), + renewed, + generation, + )); + } + Ok(( + scoped_result(result?, &session, generation), + session, + generation, + )) + } + + fn owned_session_params( + &self, + params: &Value, + routing: &SessionRouting, + session: &AuthSession, + generation: u64, + allow_discovered: bool, + ) -> Result { + let active = self.cloudmatch.active()["session"].clone(); + let id = params["sessionId"] + .as_str() + .or_else(|| active["sessionId"].as_str()) + .ok_or_else(session_owner_error)?; + if active["sessionId"] == id { + if routing + .active_owner + .as_ref() + .is_some_and(|owner| owner.matches(session, id)) + { + return Ok(active); + } + return Err(session_owner_error()); + } + if allow_discovered + && routing + .discovery_owner + .as_ref() + .is_some_and(|(provider, user, scope)| { + provider == &session.provider.idp_id + && user == &session.user.user_id + && *scope == generation + }) + && let Some(discovered) = self.cloudmatch.discovered_session(id) + { + return Ok(discovered); + } + if allow_discovered && let Some(cleanup) = self.cloudmatch.cleanup_session(session, id) { + return Ok(cleanup); + } + Err(session_owner_error()) + } + + pub fn prepare_owned_stream( + &self, + params: &Value, + prepare: impl FnOnce(&Value) -> Result, + ) -> Result { + let mut routing = crate::store_requests::lock(&self.session_routing)?; + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (session, generation) = self.session_snapshot_locked()?; + let owned = + self.owned_session_params(¶ms["session"], &routing, &session, generation, false)?; + if !matches!(owned["status"].as_i64(), Some(2 | 3)) { + return Err(ServiceError { + code: "session_not_ready", + message: "The owned session is not ready for media attachment".into(), + }); + } + let webrtc = owned["transportMode"] == "webrtc"; + let endpoint_ready = if webrtc { + crate::cloudmatch::has_webrtc_endpoint(&owned) + } else { + owned["rtspsEndpoints"].as_array().is_some_and(|endpoints| { + endpoints.iter().any(|endpoint| { + endpoint + .as_str() + .is_some_and(|endpoint| endpoint.starts_with("rtsps://")) + }) + }) + }; + if !endpoint_ready { + return Err(ServiceError { + code: "session_endpoint_missing", + message: if webrtc { + "The owned session has no WebRTC signaling endpoint" + } else { + "The owned session has no RTSPS media endpoint" + } + .into(), + }); + } + let mut params = params.clone(); + params["session"] = + scoped_result(json!({"session":owned}), &session, generation)["session"].clone(); + let mut result = prepare(¶ms)?; + result["session"] = params["session"].clone(); + self.publish_active_result(&mut routing, &session, generation, result) + } + + fn scoped_session_route( + &self, + params: &Value, + settings: &Value, + session: &AuthSession, + ) -> Result<(Value, Value), ServiceError> { + let state = self.state.lock().expect("GFN state poisoned"); + if state.providers_expires.is_some() + && !state + .providers + .iter() + .any(|provider| provider.idp_id == session.provider.idp_id) + { + return Err(ServiceError { + code: "provider_unavailable", + message: "The selected provider is unavailable".into(), + }); + } + drop(state); + if session.provider.idp_id == DEFAULT_IDP_ID + && session.provider.code.eq_ignore_ascii_case("NVIDIA") + && let Some(zone) = params["zone"].as_str() + && let Some(expected_url) = crate::queue_servers::zone_url(zone) + && params["streamingBaseUrl"].as_str() == Some(expected_url.as_str()) + { + let mut settings = settings.clone(); + settings["region"] = json!(""); + return Ok((params.clone(), settings)); + } + let mut params = params.clone(); + let mut settings = settings.clone(); + let selected = params["streamingBaseUrl"] + .as_str() + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| { + settings["providerRegions"][&session.provider.idp_id] + .as_str() + .map(ToOwned::to_owned) + }) + .or_else(|| { + (settings["regionProviderIdpId"] == session.provider.idp_id + || (settings["regionProviderIdpId"] + .as_str() + .unwrap_or("") + .is_empty() + && session.provider.idp_id == DEFAULT_IDP_ID)) + .then(|| settings["region"].as_str().unwrap_or("").to_owned()) + }); + if let Some(params) = params.as_object_mut() { + params.remove("streamingBaseUrl"); + params.remove("zone"); + } + settings["region"] = json!(""); + if let Some(selected) = selected.filter(|value| !value.is_empty()) { + let client = + client_for_settings(&self.client, &settings).map_err(ServiceError::invalid)?; + match self.provider_regions(&client, session) { + Ok(result) => { + if let Some(region) = result["regions"] + .as_array() + .into_iter() + .flatten() + .find(|region| region["url"] == selected || region["name"] == selected) + { + params["streamingBaseUrl"] = region["url"].clone(); + params["zone"] = region["name"].clone(); + } + } + Err(error) if matches!(error.code, "http_unauthorized" | "cancelled") => { + return Err(error); + } + Err(_) => {} + } + } + Ok((params, settings)) + } + + pub fn account_connections(&self, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + self.with_account_context(session, generation, settings, |context| { + self.account_connections.list(context) + }) + }) + } + + fn with_account_context( + &self, + session: &AuthSession, + generation: u64, + settings: &Value, + operation: impl FnOnce( + &crate::account_connections::AccountContext<'_>, + ) -> Result, + ) -> Result { + let definitions = self.definitions_for(session, generation, &json!({}), settings)?; + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let check = || self.check_scope(session, generation); + check()?; + operation(&crate::account_connections::AccountContext { + client: &client, + auth: session, + generation, + graphql: &self.endpoints.graphql, + als: &self.endpoints.account_linking, + definitions: &definitions, + requests: &self.store_cache.requests, + check: &check, + }) + } + + pub fn sync_account_connection( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.providers()?; + let (session, generation) = self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + let result = self.with_account_context(&session, generation, settings, |context| { + self.account_connections.sync(params, context) + })?; + self.check_scope(&session, generation)?; + Ok(scoped_result(result, &session, generation)) + } + + pub fn account_sync_status( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.authenticated_read(|session, generation| { + let mut result = + self.with_account_context(session, generation, settings, |context| { + self.account_connections.sync_status(params, context) + })?; + if result["phase"] == "refreshing_library" { + let id = result["operationId"].as_str().unwrap_or(""); + self.account_connections + .invalidate_sync(id, || self.invalidate_catalog())?; + result["catalogRevision"] = json!( + self.catalog_revision + .load(std::sync::atomic::Ordering::Acquire) + ); + } + Ok(result) + }) + } + + pub fn unlink_account_connection( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.providers()?; + let (session, generation) = self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + let result = self.with_account_context(&session, generation, settings, |context| { + self.account_connections.unlink(params, context) + })?; + self.check_scope(&session, generation)?; + self.invalidate_catalog()?; + Ok(scoped_result(result, &session, generation)) + } + + pub fn start_account_link( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.providers()?; + let (session, generation) = self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + let result = self.with_account_context(&session, generation, settings, |context| { + self.account_connections.start_link(params, context) + })?; + self.check_scope(&session, generation)?; + Ok(scoped_result(result, &session, generation)) + } + + pub fn poll_account_link( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.authenticated_read(|session, generation| { + let result = self.with_account_context(session, generation, settings, |context| { + self.account_connections.poll_link(params, context) + })?; + if result["status"] == "complete" { + self.invalidate_catalog()?; + } + Ok(result) + }) + } + + pub fn persistent_storage_locations(&self, params: &Value) -> Result { + self.authenticated_read(|session, _| self.persistent_storage.locations(params, session)) + } + + pub fn reset_persistent_storage(&self, params: &Value) -> Result { + let _operation = crate::store_requests::lock(&self.auth_operation)?; + let (session, generation) = self.session_snapshot_locked()?; + self.persistent_storage + .reset(params, &session) + .map(|result| scoped_result(result, &session, generation)) + } + + pub(crate) fn authenticated_snapshot( + &self, + purpose: TokenPurpose, + force: bool, + ) -> Result<(AuthSession, u64), ServiceError> { + let _operation = self + .auth_operation + .lock() + .expect("GFN auth operation poisoned"); + crate::requests::check()?; + let (session, _) = self.resolve_session_locked(purpose, force)?; + session + .map(|mut session| { + let state = self.state.lock().expect("GFN state poisoned"); + if state.providers_expires.is_some() { + let provider = state + .providers + .iter() + .find(|provider| provider.idp_id == session.provider.idp_id) + .ok_or_else(|| ServiceError { + code: "provider_unavailable", + message: + "The signed-in provider is no longer in the provider directory" + .into(), + })?; + session.provider = provider.clone(); + } + provider_streaming_base(&session.provider)?; + Ok((session, state.generation)) + }) + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "Sign in to renew authentication".into(), + })? + } + + fn check_scope(&self, session: &AuthSession, generation: u64) -> Result<(), ServiceError> { + crate::requests::check()?; + let state = self.state.lock().expect("GFN state poisoned"); + if state.generation != generation + || !state.session.as_ref().is_some_and(|current| { + current.user.user_id == session.user.user_id + && current.provider.idp_id == session.provider.idp_id + }) + { + return Err(ServiceError { + code: "stale_account", + message: "The account or provider changed. Retry this request.".into(), + }); + } + Ok(()) + } + + fn authenticated_snapshot_for( + &self, + owner: &AuthSession, + generation: u64, + purpose: TokenPurpose, + ) -> Result { + self.check_scope(owner, generation)?; + let (current, current_generation) = self.authenticated_snapshot(purpose, false)?; + if current_generation != generation + || current.user.user_id != owner.user.user_id + || current.provider.idp_id != owner.provider.idp_id + { + return Err(ServiceError { + code: "stale_account", + message: "The account or provider changed. Retry this request.".into(), + }); + } + self.check_scope(owner, generation)?; + Ok(current) + } + + fn authenticated_read( + &self, + mut read: impl FnMut(&AuthSession, u64) -> Result, + ) -> Result { + self.providers()?; + let (session, generation) = self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + self.check_scope(&session, generation)?; + let result = read(&session, generation); + self.check_scope(&session, generation)?; + if result + .as_ref() + .is_err_and(|error| error.code == "http_unauthorized") + { + let (renewed, renewed_generation) = + self.authenticated_snapshot(TokenPurpose::ServiceId, true)?; + self.check_scope(&session, generation)?; + if renewed_generation != generation + || renewed.provider.idp_id != session.provider.idp_id + || renewed.user.user_id != session.user.user_id + { + return Err(ServiceError { + code: "stale_account", + message: "The account changed during authentication renewal".into(), + }); + } + let result = read(&renewed, generation); + self.check_scope(&renewed, generation)?; + return result.map(|result| scoped_result(result, &renewed, generation)); + } + result.map(|result| scoped_result(result, &session, generation)) + } + + pub(crate) fn auth_generation(&self) -> u64 { + self.state.lock().expect("GFN state poisoned").generation + } + + pub(crate) fn bug_report_identity(&self) -> Value { + let state = self.state.lock().expect("GFN state poisoned"); + state.session.as_ref().map_or(Value::Null, |session| { + bug_report_identity(&session.user, &session.provider) + }) + } + + pub(crate) fn push_scope(&self) -> Option { + let state = self.state.lock().expect("GFN state poisoned"); + let session = state.session.as_ref()?; + Some(opennow_core::push::PushScope { + user_id: session.user.user_id.clone(), + provider_id: session.provider.idp_id.clone(), + generation: state.generation, + }) + } + + pub(crate) fn push_token_for_scope( + &self, + expected: &opennow_core::push::PushScope, + ) -> Option { + let (session, generation) = self + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .ok()?; + if session.user.user_id != expected.user_id + || session.provider.idp_id != expected.provider_id + || generation != expected.generation + { + return None; + } + Some(session.tokens.service_token().to_owned()) + } + + fn vpc_id( + &self, + client: &Client, + session: &AuthSession, + generation: u64, + settings: &Value, + token: &str, + requests: Option<&crate::store_requests::StoreRequests>, + ) -> Result { + self.check_scope(session, generation)?; + let base = provider_streaming_base(&session.provider)?; + let url = self.server_info_url(&base)?; + let headers = lcars_headers(token, "NATIVE", "NVIDIA-CLASSIC", false)?; + self.server_vpc_cache.resolve( + &json!([ + base.as_str(), + generation, + config_from_settings(settings) + .map_err(ServiceError::invalid)? + .map(|proxy| proxy.cache_scope) + ]) + .to_string(), + &session.user.user_id, + token, + || { + self.check_scope(session, generation)?; + let request = client.get(url).headers(headers); + let response = match requests { + Some(requests) => requests.send_mapped(request, |error| { + ServiceError::server_info( + &session.provider, + "Store server info failed", + error, + ) + }), + None => request.send().map_err(|error| { + ServiceError::server_info(&session.provider, "Server info failed", error) + }), + }; + let response = response?; + if !response.status().is_success() { + return Err(ServiceError::response("Server info failed", response)); + } + let payload = response + .json::() + .map_err(|error| ServiceError::network("Invalid server info", error))?; + self.check_scope(session, generation)?; + verified_vpc(&payload).map(Some) + }, + ) + } + + fn server_info_url(&self, base: &url::Url) -> Result { + #[cfg(test)] + if let Some(url) = &self.endpoints.server_info { + return url::Url::parse(url).map_err(|_| ServiceError::invalid("Invalid fixture URL")); + } + base.join("v2/serverInfo") + .map_err(|_| ServiceError::invalid("Invalid provider URL")) + } + + fn fetch_user_info(&self, tokens: &AuthTokens) -> Result { + if let Some(user) = tokens + .id_token + .as_deref() + .or(Some(tokens.access_token.as_str())) + .and_then(user_from_jwt) + { + if user.email.is_some() || user.avatar_url.is_some() { + return Ok(user); + } + } + let response = self + .client + .get(&self.endpoints.userinfo) + .timeout(Duration::from_secs(5)) + .header(ACCEPT, "application/json") + .header(AUTHORIZATION, format!("Bearer {}", tokens.access_token)) + .header(ORIGIN, NVIDIA_FILE_ORIGIN) + .header(USER_AGENT, STEAM_DECK_USER_AGENT) + .send() + .map_err(|error| ServiceError::network("User info failed", error))?; + if !response.status().is_success() { + return Err(ServiceError::response("User info failed", response)); + } + let payload = bounded_auth_response(response)?; + let user_id = required_string(&payload, "sub")?; + let email = payload["email"].as_str().map(ToOwned::to_owned); + let display_name = payload["preferred_username"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| { + email + .as_ref() + .and_then(|value| value.split('@').next().map(ToOwned::to_owned)) + }) + .unwrap_or_else(|| "User".to_owned()); + let avatar_url = payload["picture"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| email.as_deref().map(|value| gravatar_url(value, 80))); + Ok(AuthUser { + user_id, + display_name, + email, + avatar_url, + membership_tier: "FREE".to_owned(), + }) + } + + fn ensure_client_token(&self, mut tokens: AuthTokens) -> Result { + if tokens.expires_at <= now_ms() { + return Ok(tokens); + } + if tokens.client_token.is_some() + && tokens + .client_token_expires_at + .is_some_and(|expiry| expiry > now_ms() + CLIENT_TOKEN_REFRESH_WINDOW_MS) + { + return Ok(tokens); + } + let response = self + .client + .get(&self.endpoints.client_token) + .timeout(Duration::from_secs(5)) + .header(ACCEPT, "application/json, text/plain, */*") + .header(AUTHORIZATION, format!("Bearer {}", tokens.access_token)) + .header(ORIGIN, "https://play.geforcenow.com") + .header(REFERER, "https://play.geforcenow.com/") + .header(USER_AGENT, STEAM_DECK_USER_AGENT) + .send() + .map_err(|error| ServiceError::network("Client token request failed", error))?; + if !response.status().is_success() { + return Err(ServiceError::response( + "Client token request failed", + response, + )); + } + let payload = bounded_auth_response(response)?; + let client_token = required_string(&payload, "client_token")?; + let lifetime = token_lifetime(&payload).unwrap_or(0); + tokens.client_token = Some(client_token); + tokens.client_token_expires_at = Some(now_ms() + lifetime); + tokens.client_token_lifetime_ms = Some(lifetime); + Ok(tokens) + } + + fn update_client_token(&self, session: &AuthSession) -> Result { + let tokens = self.ensure_client_token(session.tokens.clone())?; + let updated = AuthSession { + provider: session.provider.clone(), + tokens, + user: session.user.clone(), + }; + self.store_refreshed_session(updated) + } + + fn refresh_session(&self, session: &AuthSession) -> Result { + let mut errors = Vec::new(); + if let Some(client_token) = session.tokens.client_token.as_deref().filter(|_| { + session + .tokens + .client_token_expires_at + .is_some_and(|expiry| expiry > now_ms()) + }) { + let form = [ + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:client_token", + ), + ("client_token", client_token), + ("client_id", session.tokens.auth_client_id.as_str()), + ("sub", session.user.user_id.as_str()), + ]; + match self.token_refresh_request(&form, "Client-token refresh failed") { + Ok(payload) => return self.finish_token_refresh(session, &payload), + Err(error) => errors.push(error), + } + } + if let Some(refresh_token) = session.tokens.refresh_token.as_deref() { + let form = [ + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("client_id", session.tokens.auth_client_id.as_str()), + ]; + match self.token_refresh_request(&form, "Refresh-token exchange failed") { + Ok(payload) => return self.finish_token_refresh(session, &payload), + Err(error) => errors.push(error), + } + } + Err(ServiceError { + code: if !errors.is_empty() && errors.iter().all(is_definitive_auth_revocation) { + "session_revoked" + } else { + "session_refresh_failed" + }, + message: if errors.is_empty() { + "Session has no refresh mechanism".to_owned() + } else { + errors + .into_iter() + .map(|error| error.message) + .collect::>() + .join(" | ") + }, + }) + } + + fn token_refresh_request( + &self, + form: &[(&str, &str)], + context: &str, + ) -> Result { + let response = self + .client + .post(&self.endpoints.token) + .timeout(Duration::from_secs(5)) + .header(ACCEPT, "application/json, text/plain, */*") + .header( + CONTENT_TYPE, + "application/x-www-form-urlencoded; charset=UTF-8", + ) + .header(ORIGIN, "https://play.geforcenow.com") + .header(REFERER, "https://play.geforcenow.com/") + .header( + USER_AGENT, + if form + .iter() + .any(|(key, value)| *key == "client_id" && *value == STEAM_DECK_CLIENT_ID) + { + STEAM_DECK_USER_AGENT + } else { + GFN_USER_AGENT + }, + ) + .form(form) + .send() + .map_err(|error| ServiceError::network(context, error))?; + if !response.status().is_success() { + return Err(ServiceError::response(context, response)); + } + bounded_auth_response(response) + } + + fn finish_token_refresh( + &self, + session: &AuthSession, + payload: &Value, + ) -> Result { + let access_token = required_string(payload, "access_token")?; + let mut tokens = AuthTokens { + access_token, + refresh_token: payload["refresh_token"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| session.tokens.refresh_token.clone()), + id_token: payload["id_token"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| session.tokens.id_token.clone()), + id_token_expires_at: if let Some(token) = payload["id_token"].as_str() { + jwt_expiry(token) + } else { + session + .tokens + .id_token_expires_at + .or_else(|| session.tokens.id_token.as_deref().and_then(jwt_expiry)) + }, + expires_at: token_expiry(payload), + auth_client_id: session.tokens.auth_client_id.clone(), + client_token: payload["client_token"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| session.tokens.client_token.clone()), + client_token_expires_at: session.tokens.client_token_expires_at, + client_token_lifetime_ms: session.tokens.client_token_lifetime_ms, + }; + if payload["client_token"] + .as_str() + .is_some_and(|value| Some(value) != session.tokens.client_token.as_deref()) + { + tokens.client_token_expires_at = None; + tokens.client_token_lifetime_ms = None; + } + tokens = self.ensure_client_token(tokens.clone()).unwrap_or(tokens); + let user = self + .fetch_user_info(&tokens) + .unwrap_or_else(|_| session.user.clone()); + if user.user_id != session.user.user_id { + return Err(ServiceError { + code: "session_identity_mismatch", + message: "Refreshed token belongs to a different account".to_owned(), + }); + } + self.store_refreshed_session(AuthSession { + provider: session.provider.clone(), + tokens, + user, + }) + } + + fn store_refreshed_session(&self, session: AuthSession) -> Result { + let persist = { + let state = self.state.lock().expect("GFN state poisoned"); + state.persistence_intent == PersistenceIntent::SecureStore + && self.device_identity_error.is_none() + }; + let persistence = if persist { + match self.vault.save(&session) { + Ok(()) => self.vault.persistence_state(&session), + Err(error) => { + eprintln!("auth: refreshed session persistence failed: {error}"); + self.vault.failed_save_state(&session) + } + } + } else { + "memory-only" + }; + let mut state = self.state.lock().expect("GFN state poisoned"); + state.session = Some(session.clone()); + state.persistence_state = persistence.to_owned(); + state.refresh_retry_at = None; + Ok(session) + } + + fn prune_attempts(&self) { + let now = now_ms(); + self.state + .lock() + .expect("GFN state poisoned") + .attempts + .retain(|_, attempt| attempt.expires_at > now && attempt.deadline > Instant::now()); + } +} + +struct DevicePoll<'a> { + service: &'a GfnService, + attempt_id: &'a str, +} + +impl Drop for DevicePoll<'_> { + fn drop(&mut self) { + if let Some(attempt) = self + .service + .state + .lock() + .expect("GFN state poisoned") + .attempts + .get_mut(self.attempt_id) + { + attempt.in_flight = false; + attempt.next_poll = Instant::now() + Duration::from_secs(attempt.interval_seconds); + } + } +} + +fn bounded_auth_response(response: Response) -> Result { + let mut bytes = Vec::new(); + response + .take(256 * 1024 + 1) + .read_to_end(&mut bytes) + .map_err(|_| ServiceError::invalid("Authentication response could not be read"))?; + if bytes.len() > 256 * 1024 { + return Err(ServiceError::invalid( + "Authentication response exceeds size limit", + )); + } + serde_json::from_slice(&bytes) + .map_err(|_| ServiceError::invalid("Authentication response is invalid")) +} + +fn token_lifetime(payload: &Value) -> Option { + payload["expires_in"] + .as_u64() + .filter(|seconds| *seconds > 0 && *seconds <= 366 * 24 * 3600)? + .checked_mul(1000) +} + +fn token_expiry(payload: &Value) -> u64 { + token_lifetime(payload) + .and_then(|duration| now_ms().checked_add(duration)) + .unwrap_or(0) +} + +fn jwt_expiry(token: &str) -> Option { + if token.len() > 64 * 1024 { + return None; + } + let mut parts = token.split('.'); + parts.next()?; + let payload = parts.next()?; + parts.next()?; + if parts.next().is_some() { + return None; + } + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .ok()?; + serde_json::from_slice::(&decoded).ok()?["exp"] + .as_u64()? + .checked_mul(1000) +} + +fn is_definitive_auth_revocation(error: &ServiceError) -> bool { + if error.code == "session_revoked" { + return true; + } + if !matches!(error.code, "upstream_error" | "http_unauthorized") { + return false; + } + let message = error.message.to_ascii_lowercase(); + message.contains("invalid_grant") + || message.contains("invalid_token") + || message.contains("token_revoked") + || message.contains("revoked") +} + +/// Partners whose discovery label is not the brand their members know. +const PROVIDER_DISPLAY_OVERRIDES: &[(&str, &str)] = &[("BPC", "bro.game")]; + +pub(crate) fn is_alliance_provider(code: &str) -> bool { + code != "NVIDIA" +} + +/// The one place a provider code becomes the name shown to people. +pub(crate) fn provider_display_name(code: &str, discovered: Option<&str>) -> Option { + PROVIDER_DISPLAY_OVERRIDES + .iter() + .find(|(known, _)| *known == code) + .map(|(_, name)| (*name).to_owned()) + .or_else(|| { + discovered + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + }) +} + +fn parse_providers(payload: &Value) -> Vec { + let mut providers = payload["gfnServiceInfo"]["gfnServiceEndpoints"] + .as_array() + .into_iter() + .flatten() + .filter_map(|entry| { + let code = entry["loginProviderCode"].as_str()?; + Some( + LoginProvider { + idp_id: entry["idpId"].as_str()?.to_owned(), + code: code.to_owned(), + display_name: provider_display_name( + code, + entry["loginProviderDisplayName"].as_str(), + )?, + streaming_service_url: entry["streamingServiceUrl"].as_str()?.to_owned(), + priority: entry["loginProviderPriority"].as_i64().unwrap_or(0), + } + .normalize(), + ) + }) + .collect::>(); + providers.sort_by_key(|provider| provider.priority); + providers +} + +fn public_game_to_info(item: &Value) -> Option { + if item["status"].as_str()? != "AVAILABLE" { + return None; + } + let title = item["title"].as_str()?.trim(); + if title.is_empty() { + return None; + } + let source_id = item["id"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| item["id"].as_i64().map(|value| value.to_string())) + .unwrap_or_else(|| title.to_owned()); + let steam_id = item["steamUrl"] + .as_str() + .and_then(|url| url.split("/app/").nth(1)) + .and_then(|tail| tail.split('/').next()) + .filter(|value| value.chars().all(|character| character.is_ascii_digit())) + .map(ToOwned::to_owned); + let id = steam_id.clone().unwrap_or_else(|| source_id.clone()); + let store = item["store"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .or_else(|| { + item["publisher"] + .as_str() + .filter(|value| value.to_lowercase().contains("ncsoft")) + .map(|_| "NCSoft".to_owned()) + }) + .unwrap_or_else(|| "Unknown".to_owned()); + let image_url = steam_id.as_ref().map(|value| { + format!("https://cdn.cloudflare.steamstatic.com/steam/apps/{value}/header.jpg") + }); + let hero_image_url = steam_id.as_ref().map(|value| { + format!("https://cdn.cloudflare.steamstatic.com/steam/apps/{value}/library_hero.jpg") + }); + let publisher = item["publisher"].as_str().unwrap_or(""); + Some(json!({ + "id":id, + "uuid":source_id, + "launchAppId":if id.chars().all(|character| character.is_ascii_digit()) { Some(id.clone()) } else { None }, + "title":title, + "searchText":format!("{title} {store} {publisher}").to_lowercase(), + "selectedVariantIndex":0, + "variants":[{"id":id, "store":store, "supportedControls":[]}], + "imageUrl":image_url, + "heroImageUrl":hero_image_url, + "availableStores":[store], + "isInLibrary":false, + })) +} + +fn gfn_feature_enabled(features: &Value, expected_key: &str) -> bool { + let matches = |feature: &Value| { + feature["key"].as_str() == Some(expected_key) + && (feature["value"].as_bool() == Some(true) + || feature["value"] + .as_str() + .is_some_and(|value| value.eq_ignore_ascii_case("true"))) + }; + features + .as_array() + .is_some_and(|features| features.iter().any(matches)) + || features.as_object().is_some_and(|_| matches(features)) +} + +fn image_values(value: &Value, width: u32) -> Vec { + let values = if let Some(items) = value.as_array() { + items.iter().filter_map(Value::as_str).collect::>() + } else { + value.as_str().into_iter().collect() + }; + values + .into_iter() + .filter_map(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else if trimmed.contains("img.nvidiagrid.net") { + Some(format!("{trimmed};f=jpg;w={width}")) + } else { + Some(trimmed.to_owned()) + } + }) + .collect() +} + +fn first_image(images: &Value, keys: &[&str], width: u32) -> Option { + keys.iter() + .find_map(|key| image_values(&images[*key], width).into_iter().next()) +} + +fn string_array(value: &Value) -> Vec { + value + .as_array() + .into_iter() + .flatten() + .filter_map(|item| { + item.as_str().map(ToOwned::to_owned).or_else(|| { + ["name", "label", "title", "displayName"] + .iter() + .find_map(|key| item[*key].as_str().map(ToOwned::to_owned)) + }) + }) + .collect() +} + +fn graphql_error_message(payload: &Value) -> Option { + let messages = payload["errors"] + .as_array() + .into_iter() + .flatten() + .filter_map(|error| error["message"].as_str()) + .collect::>(); + (!messages.is_empty()).then(|| messages.join(", ")) +} + +fn game_identity(value: &Value) -> Option { + for key in ["uuid", "id", "launchAppId"] { + if let Some(id) = value[key].as_str().filter(|id| !id.is_empty()) { + return Some(id.to_owned()); + } + } + None +} + +/// GETs a CMS panels document (persisted query with full-text fallback, +/// mirroring Electron's fetchLcarsGraphQl). Used for the storefront +/// marquee hero and the official Main shelves. +fn fetch_panels_document( + requests: &crate::store_requests::StoreRequests, + client: &Client, + token: &str, + variables: Value, + request_type: &str, + sha: &str, + fallback_query: &str, +) -> Result { + let context = "GFN storefront query"; + crate::requests::check()?; + let extensions = json!({"persistedQuery":{"sha256Hash":sha}}).to_string(); + let variables_text = variables.to_string(); + let hu_id = random_attempt_id(); + let mut url = url::Url::parse(GRAPHQL_URL).expect("GraphQL URL is valid"); + url.query_pairs_mut() + .append_pair("extensions", &extensions) + .append_pair("huId", &hu_id) + .append_pair("variables", &variables_text) + .append_pair("requestType", request_type); + let mut headers = graphql_headers(token)?; + headers.insert( + reqwest::header::CONTENT_TYPE, + HeaderValue::from_static("application/graphql"), + ); + let response = requests.send( + client.get(url.clone()).headers(headers), + &format!("{context} failed"), + )?; + let payload = if response.status().as_u16() == 400 { + crate::requests::check()?; + url.query_pairs_mut().append_pair("query", fallback_query); + let mut retry_headers = graphql_headers(token)?; + retry_headers.insert( + reqwest::header::CONTENT_TYPE, + HeaderValue::from_static("application/graphql"), + ); + let response = requests.send( + client.get(url).headers(retry_headers), + &format!("{context} failed"), + )?; + if !response.status().is_success() { + return Err(ServiceError::response( + &format!("{context} failed"), + response, + )); + } + response + .json::() + .map_err(|error| ServiceError::network(&format!("Invalid {context} response"), error))? + } else { + if !response.status().is_success() { + return Err(ServiceError::response( + &format!("{context} failed"), + response, + )); + } + response + .json::() + .map_err(|error| ServiceError::network(&format!("Invalid {context} response"), error))? + }; + if let Some(message) = graphql_error_message(&payload) { + return Err(ServiceError { + code: "graphql_error", + message, + }); + } + Ok(payload) +} + +fn marquee_hero_image(item: &Value) -> Option { + first_image(&item["images"], &["MARQUEE_HERO_IMAGE", "HERO_IMAGE"], 1600) +} + +fn parse_store_marquee(payload: &Value, browse_by_id: &HashMap) -> Vec { + let mut slides = Vec::new(); + let panels = payload["data"]["panels"].as_array().into_iter().flatten(); + for panel in panels { + let sections = panel["sections"].as_array().into_iter().flatten(); + for section in sections { + let items = section["items"].as_array().into_iter().flatten(); + for item in items { + if slides.len() >= 8 { + return slides; + } + match item["__typename"].as_str().unwrap_or("") { + "MarketingItem" => { + let title = item["title"].as_str().unwrap_or("").trim(); + if title.is_empty() { + continue; + } + slides.push(json!({ + "kind":"marketing", + "title":title, + "body":item["body"].as_str().unwrap_or(""), + "image":marquee_hero_image(item), + "actionLabel":item["action"]["label"].as_str().unwrap_or(""), + "actionUri":item["action"]["uri"].as_str().unwrap_or(""), + })); + } + "GameItem" => { + let Some(game) = app_to_game(&item["app"]).map(|mut game| { + if game["heroImageUrl"].is_null() { + if let Some(art) = marquee_hero_image(&item["app"]) { + game["heroImageUrl"] = Value::String(art); + } + } + game + }) else { + continue; + }; + let identity = game_identity(&game); + let resolved = identity + .as_ref() + .and_then(|id| browse_by_id.get(id)) + .cloned() + .unwrap_or(game); + let title = resolved["title"].as_str().unwrap_or("").to_owned(); + if title.is_empty() { + continue; + } + slides.push(json!({ + "kind":"game", + "title":title, + "body":resolved["publisherName"].as_str().unwrap_or(""), + "image":marquee_hero_image(&item["app"]), + "game":resolved, + })); + } + _ => {} + } + } + } + } + slides +} + +fn parse_store_panels(payload: &Value, browse_by_id: &HashMap) -> Vec { + let mut panels = Vec::new(); + let incoming = payload["data"]["panels"].as_array().into_iter().flatten(); + for panel in incoming { + let mut sections = Vec::new(); + let panel_sections = panel["sections"].as_array().into_iter().flatten(); + for section in panel_sections { + let title = section["title"].as_str().unwrap_or("").trim().to_owned(); + let mut games = Vec::new(); + let items = section["items"].as_array().into_iter().flatten(); + for item in items { + if item["__typename"].as_str() != Some("GameItem") { + continue; + } + let Some(game) = app_to_game(&item["app"]) else { + continue; + }; + if game["id"].as_str().unwrap_or("").is_empty() + || game["title"].as_str().unwrap_or("").is_empty() + || game["variants"] + .as_array() + .is_none_or(|variants| variants.is_empty()) + { + continue; + } + let resolved = game_identity(&game) + .as_ref() + .and_then(|id| browse_by_id.get(id)) + .cloned() + .unwrap_or(game); + if games.len() < 24 + && !games + .iter() + .any(|existing: &Value| game_identity(existing) == game_identity(&resolved)) + { + games.push(resolved); + } + } + if title.is_empty() || games.is_empty() { + continue; + } + sections.push(json!({ + "id":section["id"].as_str().unwrap_or(&title), + "title":title, + "games":games, + })); + } + if sections.is_empty() { + continue; + } + panels.push(json!({ + "id":panel["id"].as_str().or_else(|| panel["name"].as_str()).unwrap_or(""), + "title":panel["name"].as_str().unwrap_or(""), + "sections":sections, + })); + } + panels +} + +fn parse_store_definitions(payload: &Value) -> Vec { + payload["data"]["filterGroupDefinitions"] + .as_array() + .into_iter() + .flatten() + .filter_map(|group| { + let options = group["filters"] + .as_array() + .into_iter() + .flatten() + .filter_map(|entry| { + Some(json!({ + "id":entry["id"].as_str()?, + "label":entry["label"].as_str().unwrap_or(entry["id"].as_str()?), + "filters":entry["filters"], + "expression":crate::catalog_types::filter_expression(&entry["filters"]), + })) + }) + .collect::>(); + if options.is_empty() { + return None; + } + Some(json!({ + "id":group["id"].as_str()?, + "label":group["label"].as_str().unwrap_or(group["id"].as_str()?), + "options":options, + })) + }) + .collect() +} + +fn trusted_streaming_base(value: &str) -> Result { + crate::cloudmatch::trusted_cloudmatch_base(value) +} + +fn provider_region_entries(payload: &Value) -> Vec { + let Some(metadata) = payload["metaData"].as_array() else { + return Vec::new(); + }; + let names = metadata + .iter() + .find(|entry| entry["key"] == "gfn-regions") + .and_then(|entry| entry["value"].as_str()) + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + .collect::>(); + let mut regions = metadata + .iter() + .filter_map(|entry| { + let name = entry["key"].as_str()?; + let value = entry["value"].as_str()?; + if !names.contains(&name) { + return None; + } + Some(json!({"name":name,"url":trusted_streaming_base(value).ok()?.as_str()})) + }) + .collect::>(); + regions.sort_by(|left, right| { + left["name"] + .as_str() + .unwrap_or("") + .cmp(right["name"].as_str().unwrap_or("")) + }); + regions +} + +fn provider_result(state: &ServiceState) -> Value { + let mut providers = state.providers.clone(); + if providers.is_empty() { + if let Some(session) = &state.session { + providers.push(session.provider.clone()); + } + if !providers + .iter() + .any(|provider| provider.idp_id == DEFAULT_IDP_ID) + { + providers.push(LoginProvider::default_nvidia()); + } + } + json!({"providers":providers,"defaultProviderIdpId":state.providers_default,"generation":state.generation,"discovery":{ + "state":if state.providers_error.is_some() {"degraded"} else {"ready"}, + "message":state.providers_error, + "retryAfterMs":state.providers_retry.map(|time| time.saturating_duration_since(Instant::now()).as_millis() as u64).unwrap_or(0) + }}) +} + +fn session_owner_error() -> ServiceError { + ServiceError { code: "session_owner_mismatch", message: "This session is not owned by the current account and provider. Refresh active sessions.".into() } +} + +fn verified_vpc(payload: &Value) -> Result { + let status = &payload["requestStatus"]; + if status.get("statusCode").is_some() && number_value(&status["statusCode"]) != Some(1.0) { + return Err(ServiceError { + code: "invalid_upstream_response", + message: "Provider server info reported an unsuccessful status".into(), + }); + } + status["serverId"] + .as_str() + .filter(|value| !value.trim().is_empty() && value.len() <= 256) + .map(ToOwned::to_owned) + .ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: "Provider server info did not include a verified VPC".into(), + }) +} + +fn scoped_result(mut result: Value, session: &AuthSession, generation: u64) -> Value { + result["scope"] = json!({"generation":generation,"providerIdpId":session.provider.idp_id,"userId":session.user.user_id}); + if result["session"].is_object() { + result["session"]["ownerScope"] = result["scope"].clone(); + } + result +} + +fn number_value(value: &Value) -> Option { + value + .as_f64() + .or_else(|| value.as_str()?.trim().parse().ok()) +} + +fn graphql_headers(token: &str) -> Result { + let mut headers = lcars_headers(token, "NATIVE", "NVIDIA-CLASSIC", false)?; + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + ORIGIN, + HeaderValue::from_static("https://play.geforcenow.com"), + ); + headers.insert( + REFERER, + HeaderValue::from_static("https://play.geforcenow.com/"), + ); + headers.insert("nv-browser-type", HeaderValue::from_static("CHROME")); + Ok(headers) +} + +fn lcars_headers( + token: &str, + client_type: &str, + streamer: &str, + steam_deck: bool, +) -> Result { + let mut headers = HeaderMap::new(); + let authorization = HeaderValue::from_str(&format!("GFNJWT {token}")) + .map_err(|_| ServiceError::invalid("Session token contains invalid header bytes"))?; + headers.insert( + ACCEPT, + HeaderValue::from_static("application/json, text/plain, */*"), + ); + headers.insert(AUTHORIZATION, authorization); + headers.insert("nv-client-id", HeaderValue::from_static(LCARS_CLIENT_ID)); + headers.insert( + "nv-client-type", + HeaderValue::from_str(client_type) + .map_err(|_| ServiceError::invalid("Invalid client type"))?, + ); + headers.insert( + "nv-client-version", + HeaderValue::from_static(GFN_CLIENT_VERSION), + ); + headers.insert( + "nv-client-streamer", + HeaderValue::from_str(streamer) + .map_err(|_| ServiceError::invalid("Invalid streamer type"))?, + ); + headers.insert( + "nv-device-os", + HeaderValue::from_static(if steam_deck { + "STEAMOS" + } else if cfg!(target_os = "windows") { + "WINDOWS" + } else if cfg!(target_os = "macos") { + "MACOS" + } else { + "LINUX" + }), + ); + // Mirrors Electron's Steam Deck device profile: MES returns the Deck + // resolution catalog (including 90 FPS tuples) under these headers. + headers.insert( + "nv-device-type", + HeaderValue::from_static(if steam_deck { "CONSOLE" } else { "DESKTOP" }), + ); + headers.insert( + "nv-device-make", + HeaderValue::from_static(if steam_deck { "VALVE" } else { "GENERIC" }), + ); + headers.insert( + "nv-device-model", + HeaderValue::from_static(if steam_deck { "STEAMDECK" } else { "PC" }), + ); + headers.insert("x-nv-client-identity", HeaderValue::from_static("GFN-PC")); + headers.insert(USER_AGENT, HeaderValue::from_static(GFN_USER_AGENT)); + Ok(headers) +} + +fn bug_report_identity(user: &AuthUser, provider: &LoginProvider) -> Value { + let email = user + .email + .as_deref() + .map(str::trim) + .filter(|email| !email.is_empty()); + let derived = email.and_then(|email| email.split('@').next()); + let username = Some(user.display_name.trim()) + .filter(|name| !name.is_empty() && *name != "User" && Some(*name) != derived); + let (reporter, basis) = match (username, email) { + (Some(username), _) => (Some(username), "username"), + (None, Some(email)) => (Some(email), "email"), + (None, None) => (None, "none"), + }; + json!({ + "userId": user.user_id, + "providerIdpId": provider.idp_id, + "providerCode": provider.code, + "providerName": provider.display_name, + "reporter": reporter, + "reporterBasis": basis, + "alliancePartner": is_alliance_provider(&provider.code), + "membershipTier": user.membership_tier + }) +} + +fn user_from_jwt(token: &str) -> Option { + let encoded = token.split('.').nth(1)?; + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded) + .ok()?; + let payload = serde_json::from_slice::(&decoded).ok()?; + let user_id = payload["sub"].as_str()?.to_owned(); + let email = payload["email"].as_str().map(ToOwned::to_owned); + let avatar_url = payload["picture"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| email.as_deref().map(|value| gravatar_url(value, 80))); + let display_name = payload["preferred_username"] + .as_str() + .map(ToOwned::to_owned) + .or_else(|| { + email + .as_ref() + .and_then(|value| value.split('@').next().map(ToOwned::to_owned)) + }) + .unwrap_or_else(|| "User".to_owned()); + Some(AuthUser { + user_id, + display_name, + email, + avatar_url, + membership_tier: payload["gfn_tier"].as_str().unwrap_or("FREE").to_owned(), + }) +} + +fn qr_rows(value: &str) -> Vec { + QrCode::new(value.as_bytes()) + .map(|code| { + let width = code.width(); + code.to_colors() + .chunks(width) + .map(|row| { + row.iter() + .map(|color| { + if matches!(color, qrcode::Color::Dark) { + '1' + } else { + '0' + } + }) + .collect() + }) + .collect() + }) + .unwrap_or_default() +} + +fn stable_device_id() -> String { + let host = env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_owned()); + let user = env::var("USER") + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "unknown-user".to_owned()); + let mut hasher = Sha256::new(); + hasher.update(format!("{host}:{user}:opennow-stable")); + format!("{:x}", hasher.finalize()) +} + +fn random_attempt_id() -> String { + use rand::RngCore as _; + let mut bytes = [0_u8; 16]; + rand::rng().fill_bytes(&mut bytes); + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn gravatar_url(email: &str, size: u32) -> String { + let normalized = email.trim().to_lowercase(); + format!( + "https://www.gravatar.com/avatar/{:x}?s={size}&d=identicon", + md5::compute(normalized) + ) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +fn required_param<'a>(params: &'a Value, key: &str) -> Result<&'a str, ServiceError> { + params[key] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| ServiceError::invalid(format!("Missing {key}"))) +} + +fn required_string(payload: &Value, key: &str) -> Result { + payload[key] + .as_str() + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: format!("Response did not include {key}"), + }) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + fn tokens(id_token: Option<&str>) -> AuthTokens { + AuthTokens { + access_token: "access-token".to_owned(), + refresh_token: None, + id_token: id_token.map(str::to_owned), + id_token_expires_at: None, + expires_at: 0, + auth_client_id: "client".to_owned(), + client_token: None, + client_token_expires_at: None, + client_token_lifetime_ms: None, + } + } + + #[test] + fn the_service_token_prefers_the_id_token_and_falls_back_to_access() { + assert_eq!(tokens(Some("id-token")).service_token(), "id-token"); + assert_eq!(tokens(None).service_token(), "access-token"); + } + + fn mock_responses( + responses: Vec<(u16, Value)>, + before_response: impl Fn(usize) + Send + 'static, + ) -> (String, std::thread::JoinHandle<()>) { + mock_requests(responses, move |index, _| before_response(index)) + } + + pub(crate) fn mock_requests( + responses: Vec<(u16, Value)>, + before_response: impl Fn(usize, &str) + Send + 'static, + ) -> (String, std::thread::JoinHandle<()>) { + use std::io::{BufRead, BufReader, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let worker = std::thread::spawn(move || { + for (index, (status, body)) in responses.into_iter().enumerate() { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(std::time::Instant::now() < deadline, "missing HTTP request"); + std::thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("{error}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut length = 0; + let mut request = String::new(); + loop { + let mut line = String::new(); + assert!(reader.read_line(&mut line).unwrap() > 0); + request.push_str(&line); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse::().unwrap(); + } + } + let mut body_bytes = vec![0; length]; + reader.read_exact(&mut body_bytes).unwrap(); + request.push_str(std::str::from_utf8(&body_bytes).unwrap()); + before_response(index, &request); + let body = body.to_string(); + write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap(); + } + }); + (url, worker) + } + + pub(crate) fn auth_fixture(user: &str) -> AuthSession { + serde_json::from_value(json!({ + "provider": LoginProvider::default_nvidia(), + "tokens": {"accessToken":"test-access", "refreshToken":"test-refresh", + "clientToken":"test-client", "expiresAt":now_ms() + 3_600_000, + "clientTokenExpiresAt":now_ms() + 3_600_000, "authClientId":"test-client-id"}, + "user":{"userId":user,"displayName":"Test","membershipTier":"FREE"} + })) + .unwrap() + } + + pub(super) fn jwt(user: &str, expiry: u64) -> String { + format!( + "header.{}.signature", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( + serde_json::to_vec( + &json!({"sub":user,"email":"fixture@example.invalid","exp":expiry / 1000}) + ) + .unwrap() + ) + ) + } + + pub(super) fn pending_attempt(session: Option) -> DeviceAttempt { + DeviceAttempt { + provider: LoginProvider::default_nvidia(), + device_code: "private-device-sentinel".into(), + expires_at: now_ms() + 60_000, + deadline: Instant::now() + Duration::from_secs(60), + interval_seconds: 7, + next_poll: Instant::now(), + in_flight: false, + pending_session: session, + } + } + + fn assert_public_auth(value: &Value) { + let encoded = value.to_string(); + for private in [ + "accessToken", + "idToken", + "refreshToken", + "clientToken", + "authClientId", + "deviceCode", + "test-access", + "test-refresh", + "test-client", + "private-device-sentinel", + ] { + assert!( + !encoded.contains(private), + "private field escaped: {private}" + ); + } + } + + #[test] + fn cancelled_authorization_does_not_install_a_late_challenge() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let challenge = json!({"device_code":"private-device-sentinel","user_code":"ABCD", "verification_uri":"https://example.invalid", "verification_uri_complete":"https://example.invalid/code", "expires_in":600,"interval":0}); + let (url, server) = mock_responses(vec![(200, challenge)], move |_| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + let (service, path) = test_service(&url); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("start", "auth.device.start").unwrap(); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + crate::requests::scope(permit.token.clone(), || { + service.start_device_login(&json!({})) + }) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + requests.cancel("start"); + release_tx.send(()).unwrap(); + assert_eq!(worker.join().unwrap().unwrap_err().code, "cancelled"); + }); + assert!(service.state.lock().unwrap().attempts.is_empty()); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn challenge_hides_device_grant_and_defaults_zero_interval() { + let challenge = json!({"device_code":"private-device-sentinel","user_code":"ABCD", "verification_uri":"https://example.invalid", "verification_uri_complete":"https://example.invalid/code", "expires_in":600,"interval":0}); + let (url, server) = mock_requests(vec![(200, challenge)], |_, request| { + assert!(request.contains("device_id=")); + assert!(request.contains("client_id=")); + }); + let (service, path) = test_service(&url); + let result = service.start_device_login(&json!({})).unwrap(); + assert_eq!(result["intervalSeconds"], 5); + assert_public_auth(&result); + let state = service.state.lock().unwrap(); + assert_eq!( + state.attempts[result["attemptId"].as_str().unwrap()].device_code, + "private-device-sentinel" + ); + drop(state); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn expiration_during_profile_io_prevents_completion() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, server) = mock_responses( + vec![ + (200, json!({"access_token":"test-access","expires_in":3600})), + (200, json!({"client_token":"test-client","expires_in":3600})), + (200, json!({"sub":"user","email":"fixture@example.invalid"})), + ], + move |index| { + if index == 2 { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ); + let (service, path) = test_service(&url); + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(None)); + std::thread::scope(|scope| { + let worker = scope.spawn(|| service.poll_device_login(&json!({"attemptId":"login"}))); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + service + .state + .lock() + .unwrap() + .attempts + .get_mut("login") + .unwrap() + .deadline = Instant::now(); + release_tx.send(()).unwrap(); + assert!(worker.join().unwrap().is_err()); + }); + assert!( + service + .complete_device_login(&json!({"attemptId":"login"})) + .is_err() + ); + assert!(service.state.lock().unwrap().session.is_none()); + assert!(!path.join("accounts.json").exists()); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn public_auth_is_token_free_and_temporary_intent_survives_refresh() { + let (service, path) = test_service("http://127.0.0.1:1"); + service.vault.save(&auth_fixture("user")).unwrap(); + assert!(!format!("{:?}", auth_fixture("user")).contains("test-access")); + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(Some(auth_fixture("user")))); + let response = service + .complete_device_login(&json!({"attemptId":"login","staySignedIn":false})) + .unwrap(); + assert_public_auth(&response); + assert_eq!(response["persistence"], "memory-only"); + assert_eq!(response["generation"], 1); + service + .store_refreshed_session(auth_fixture("user")) + .unwrap(); + assert!(service.vault.load("user").unwrap().is_none()); + let refreshed = service.session().unwrap(); + assert_eq!(refreshed["persistence"], "memory-only"); + assert_public_auth(&refreshed); + assert!( + service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .map(|(session, _)| session) + .is_ok() + ); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn automatic_account_restoration_requires_profile_pin() { + for remove in [false, true] { + let (mut service, path) = test_service("http://127.0.0.1:1"); + service.vault = CredentialVault::without_os_store(path.clone()); + service.vault.save(&auth_fixture("protected")).unwrap(); + service.vault.save(&auth_fixture("current")).unwrap(); + service.profiles.set_pin("protected", "1234", None).unwrap(); + service.state.lock().unwrap().session = Some(auth_fixture("current")); + service.state.lock().unwrap().restore_attempted = true; + assert_eq!( + service + .switch_account(&json!({"userId":"protected"})) + .unwrap_err() + .code, + "profile_pin_required" + ); + let result = if remove { + service + .remove_account(&json!({"userId":"current"})) + .unwrap() + } else { + service.logout().unwrap() + }; + assert!(result["session"].is_null()); + assert!( + service + .authenticated_snapshot(TokenPurpose::StarfleetAccess, false) + .is_err() + ); + assert!(service.vault.load("protected").unwrap().is_some()); + assert!(service.profiles.has_pin("protected")); + drop(service); + let (mut restarted, unused_path) = test_service("http://127.0.0.1:1"); + restarted.vault = CredentialVault::without_os_store(path.clone()); + restarted.profiles = ConsoleProfiles::load(&path); + assert!(restarted.session().unwrap()["session"].is_null()); + assert!(restarted.vault.load("protected").unwrap().is_some()); + assert_eq!( + restarted + .switch_account(&json!({"userId":"protected"})) + .unwrap_err() + .code, + "profile_pin_required" + ); + assert_eq!( + restarted + .switch_account(&json!({"userId":"protected","pin":"1234"})) + .unwrap()["session"]["user"]["userId"], + "protected" + ); + assert_eq!( + restarted.session().unwrap()["session"]["user"]["userId"], + "protected" + ); + std::fs::remove_dir_all(path).unwrap(); + std::fs::remove_dir_all(unused_path).unwrap(); + } + } + + #[test] + fn logout_all_cleans_partial_json_saves_after_restart() { + let (mut service, path) = test_service("http://127.0.0.1:1"); + service.vault = CredentialVault::without_os_store(path.clone()); + std::fs::create_dir_all(path.join("accounts.json.tmp")).unwrap(); + service.state.lock().unwrap().attempts.insert( + "login".into(), + pending_attempt(Some(auth_fixture("orphan"))), + ); + let login = service + .complete_device_login(&json!({"attemptId":"login","staySignedIn":true})) + .unwrap(); + assert_eq!(login["persistence"], "local-file"); + assert!(!path.join("accounts.json").exists()); + drop(service); + std::fs::remove_dir(path.join("accounts.json.tmp")).unwrap(); + let (mut reopened, unused_path) = test_service("http://127.0.0.1:1"); + reopened.vault = CredentialVault::without_os_store(path.clone()); + assert!(reopened.session().unwrap()["session"].is_null()); + let logout = reopened.logout_all().unwrap(); + assert_eq!(logout["localCleanup"], "pending"); + assert_eq!( + std::fs::read_dir(path.join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + assert!(reopened.vault.load("orphan").unwrap().is_none()); + let restarted = CredentialVault::without_os_store(path.clone()); + assert!(restarted.load("orphan").unwrap().is_none()); + std::fs::remove_dir_all(path).unwrap(); + std::fs::remove_dir_all(unused_path).unwrap(); + } + + #[test] + fn partial_json_login_and_refresh_keep_plaintext_warning_and_hide_tokens() { + let (mut service, path) = test_service("http://127.0.0.1:1"); + service.vault = CredentialVault::without_os_store(path.clone()); + std::fs::create_dir_all(path.join("accounts.json.tmp")).unwrap(); + let session = auth_fixture("user"); + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(Some(session.clone()))); + let response = service + .complete_device_login(&json!({"attemptId":"login","staySignedIn":true})) + .unwrap(); + assert_eq!(response["persistence"], "local-file"); + assert!(!response["warnings"].as_array().unwrap().is_empty()); + assert_public_auth(&response); + assert!(service.vault.durable(&session)); + assert!(!path.join("accounts.json").exists()); + + let mut renewed = auth_fixture("user"); + renewed.tokens.refresh_token = Some("renewed-private-refresh-sentinel".into()); + service.store_refreshed_session(renewed.clone()).unwrap(); + let refreshed = service.session().unwrap(); + assert_eq!(refreshed["persistence"], "local-file"); + assert!(!refreshed["warnings"].as_array().unwrap().is_empty()); + assert_public_auth(&refreshed); + assert!( + !refreshed + .to_string() + .contains("renewed-private-refresh-sentinel") + ); + assert!(service.vault.durable(&renewed)); + assert_eq!( + service + .vault + .load("user") + .unwrap() + .unwrap() + .tokens + .refresh_token, + renewed.tokens.refresh_token + ); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn unavailable_vault_keeps_temporary_login_and_refresh_out_of_json() { + for previously_saved in [false, true] { + let (mut service, path) = test_service("http://127.0.0.1:1"); + service.vault = CredentialVault::without_os_store(path.clone()); + if previously_saved { + service.vault.save(&auth_fixture("user")).unwrap(); + assert_eq!( + std::fs::read_dir(path.join("fallback-sessions")) + .unwrap() + .count(), + 1 + ); + } + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(Some(auth_fixture("user")))); + let response = service + .complete_device_login(&json!({"attemptId":"login","staySignedIn":false})) + .unwrap(); + assert_eq!(response["persistence"], "memory-only"); + assert_public_auth(&response); + service + .store_refreshed_session(auth_fixture("user")) + .unwrap(); + let refreshed = service.session().unwrap(); + assert_eq!(refreshed["persistence"], "memory-only"); + assert_public_auth(&refreshed); + if previously_saved { + assert_eq!( + std::fs::read_dir(path.join("fallback-sessions")) + .unwrap() + .count(), + 0 + ); + } else { + assert!(!path.join("fallback-sessions").exists()); + } + let reopened = CredentialVault::without_os_store(path.clone()); + assert!(reopened.load("user").unwrap().is_none()); + assert!(reopened.load_active().unwrap().is_none()); + std::fs::remove_dir_all(path).unwrap(); + } + } + + #[test] + fn token_expiries_are_independent_and_do_not_invent_lifetimes() { + assert_eq!(token_expiry(&json!({})), 0); + assert_eq!(token_expiry(&json!({"expires_in":u64::MAX})), 0); + assert_eq!(token_expiry(&json!({"expires_in":-1})), 0); + assert_eq!(jwt_expiry("malformed"), None); + let mut session = auth_fixture("user"); + session.tokens.id_token = Some(jwt("user", 1000)); + assert_eq!(session.tokens.expiry(TokenPurpose::ServiceId), 1000); + assert!(session.tokens.expiry(TokenPurpose::StarfleetAccess) > now_ms()); + session.tokens.id_token = Some(jwt("user", now_ms() + 3_600_000)); + session.tokens.expires_at = 0; + assert_eq!(session.tokens.expiry(TokenPurpose::StarfleetAccess), 0); + assert!(session.tokens.expiry(TokenPurpose::ServiceId) > now_ms()); + } + + #[test] + fn expired_id_renews_with_issuing_client_even_when_access_is_valid() { + let token = jwt("user", now_ms() + 3_600_000); + let (url, server) = mock_requests( + vec![( + 200, + json!({"access_token":"renewed","id_token":token,"expires_in":3600}), + )], + |_, request| { + assert!(request.contains( + "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Aclient_token" + )); + assert!(request.contains("client_id=test-client-id")); + assert!(request.contains("sub=user")); + }, + ); + let (service, path) = test_service(&url); + let mut session = auth_fixture("user"); + session.tokens.id_token = Some(jwt("user", 1000)); + service.state.lock().unwrap().session = Some(session); + let session = service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .map(|(session, _)| session) + .unwrap(); + assert_eq!(session.tokens.access_token, "renewed"); + assert!(session.tokens.expiry(TokenPurpose::ServiceId) > now_ms()); + assert_eq!(session.tokens.auth_client_id, "test-client-id"); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn omitted_id_refresh_does_not_extend_old_id_or_hammer_auth() { + let (url, server) = mock_responses( + vec![(200, json!({"access_token":"renewed","expires_in":3600}))], + |_| {}, + ); + let (service, path) = test_service(&url); + let mut session = auth_fixture("user"); + session.tokens.id_token = Some(jwt("user", 1000)); + service.state.lock().unwrap().session = Some(session); + assert!( + service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .map(|(session, _)| session) + .is_err() + ); + assert!( + service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .map(|(session, _)| session) + .is_err() + ); + assert!( + service + .authenticated_snapshot(TokenPurpose::StarfleetAccess, false) + .map(|(session, _)| session) + .is_ok() + ); + assert_eq!( + service + .state + .lock() + .unwrap() + .session + .as_ref() + .unwrap() + .tokens + .id_token_expires_at, + Some(1000) + ); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn expired_client_grant_uses_refresh_token_and_transient_errors_are_paced() { + let (url, server) = mock_requests( + vec![(503, json!({"error":"temporarily_unavailable"}))], + |_, request| { + assert!(request.contains("grant_type=refresh_token")); + assert!(!request.contains("client_token=")); + }, + ); + let (service, path) = test_service(&url); + let mut session = auth_fixture("user"); + session.tokens.expires_at = 0; + session.tokens.client_token_expires_at = Some(1); + service.vault.save(&session).unwrap(); + service.state.lock().unwrap().session = Some(session); + assert!( + service + .authenticated_snapshot(TokenPurpose::StarfleetAccess, false) + .map(|(session, _)| session) + .is_err() + ); + assert!( + service + .authenticated_snapshot(TokenPurpose::StarfleetAccess, false) + .map(|(session, _)| session) + .is_err() + ); + assert!(service.vault.load("user").unwrap().is_some()); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn device_slow_down_is_cumulative_and_early_polls_stay_local() { + let (url, server) = mock_requests( + vec![ + ( + 400, + json!({"error":"slow_down","error_description":"private-device-sentinel"}), + ), + (400, json!({"error":"slow_down"})), + ], + |_, request| { + assert!(request.contains("device_code=private-device-sentinel")); + }, + ); + let (service, path) = test_service(&url); + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(None)); + let params = json!({"attemptId":"login"}); + let first = service.poll_device_login(¶ms).unwrap(); + assert_eq!(first["intervalSeconds"], 12); + assert_public_auth(&first); + let early = service.poll_device_login(¶ms).unwrap(); + assert_eq!(early["status"], "pending"); + assert!(early["retryAfterMs"].as_u64().unwrap() >= 11_000); + service + .state + .lock() + .unwrap() + .attempts + .get_mut("login") + .unwrap() + .next_poll = Instant::now(); + assert_eq!( + service.poll_device_login(¶ms).unwrap()["intervalSeconds"], + 17 + ); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn cancellation_during_device_token_io_cannot_authorize_or_publish() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, server) = mock_responses( + vec![(200, json!({"access_token":"test-access","expires_in":3600}))], + move |_| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }, + ); + let (service, path) = test_service(&url); + service + .state + .lock() + .unwrap() + .attempts + .insert("login".into(), pending_attempt(None)); + std::thread::scope(|scope| { + let worker = scope.spawn(|| service.poll_device_login(&json!({"attemptId":"login"}))); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + service + .poll_device_login(&json!({"attemptId":"login"})) + .unwrap()["status"], + "pending" + ); + service + .cancel_device_login(&json!({"attemptId":"login"})) + .unwrap(); + release_tx.send(()).unwrap(); + assert!(worker.join().unwrap().is_err()); + }); + assert!(service.state.lock().unwrap().session.is_none()); + assert!( + service + .complete_device_login(&json!({"attemptId":"login"})) + .is_err() + ); + assert!(!path.join("accounts.json").exists()); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn revoke_uses_selected_access_grant_and_local_logout_survives_rejection() { + let (url, server) = mock_requests( + vec![(401, json!({"error":"invalid_token"}))], + |_, request| { + assert!(request.starts_with("DELETE /assets/v2/Tokens?level=client HTTP/1.1")); + assert!( + request + .to_ascii_lowercase() + .contains("authorization: bearer test-access") + ); + assert!(!request.contains("test-client")); + }, + ); + let (service, path) = test_service(&url); + service.vault.save(&auth_fixture("next")).unwrap(); + service.vault.save(&auth_fixture("current")).unwrap(); + { + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("current")); + state + .attempts + .insert("stale".into(), pending_attempt(Some(auth_fixture("late")))); + } + let result = service.logout().unwrap(); + assert_eq!(result["session"]["user"]["userId"], "next"); + assert_eq!(result["cleanup"]["remoteRevoke"], "failed"); + assert_eq!(result["cleanup"]["localCleanup"], "complete"); + assert!(service.vault.load("current").unwrap().is_none()); + assert!(service.state.lock().unwrap().attempts.is_empty()); + assert_public_auth(&result); + server.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + pub(super) fn test_service(url: &str) -> (GfnService, PathBuf) { + let path = + std::env::temp_dir().join(format!("opennow-auth-test-{}", rand::random::())); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let mut service = GfnService::with_client( + client, + Endpoints { + token: url.to_owned(), + client_token: url.to_owned(), + userinfo: url.to_owned(), + device_authorize: url.to_owned(), + revoke: format!("{url}/assets/v2/Tokens?level=client"), + ..Endpoints::default() + }, + path.clone(), + ); + service.vault = CredentialVault::memory(path.clone()); + service.state.lock().unwrap().providers = vec![LoginProvider::default_nvidia()]; + service.state.lock().unwrap().providers_expires = + Some(Instant::now() + Duration::from_secs(900)); + (service, path) + } + + #[test] + fn refresh_cannot_overwrite_a_new_login() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, server) = mock_responses( + vec![(200, json!({"client_token":"updated", "expires_in":3600}))], + move |_| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }, + ); + let (service, path) = test_service(&url); + let service = std::sync::Arc::new(service); + { + let mut state = service.state.lock().unwrap(); + let mut old = auth_fixture("old-account"); + old.tokens.client_token_expires_at = None; + state.session = Some(old); + state.persistence_state = "none".into(); + state.attempts.insert( + "new-login".into(), + DeviceAttempt { + provider: LoginProvider::default_nvidia(), + device_code: "test-device".into(), + expires_at: now_ms() + 60_000, + deadline: Instant::now() + Duration::from_secs(60), + interval_seconds: 5, + next_poll: Instant::now(), + in_flight: false, + pending_session: Some(auth_fixture("new-account")), + }, + ); + } + let refreshing = service.clone(); + let refresh = std::thread::spawn(move || refreshing.session().unwrap()); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let signing_in = service.clone(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let login = std::thread::spawn(move || { + let result = signing_in + .complete_device_login(&json!({"attemptId":"new-login", "staySignedIn":false})); + done_tx.send(()).unwrap(); + result.unwrap() + }); + let completed_during_refresh = done_rx.recv_timeout(Duration::from_millis(100)).is_ok(); + release_tx.send(()).unwrap(); + refresh.join().unwrap(); + login.join().unwrap(); + server.join().unwrap(); + assert_eq!( + service + .state + .lock() + .unwrap() + .session + .as_ref() + .unwrap() + .user + .user_id, + "new-account" + ); + assert!(!completed_during_refresh); + let _ = std::fs::remove_dir_all(path); + } + + #[test] + fn cancelled_auth_mutations_do_not_run_after_waiting_for_refresh() { + type Mutation = fn(&GfnService) -> Result; + let mutations: [Mutation; 5] = [ + |service| { + service.complete_device_login(&json!({"attemptId":"pending", "staySignedIn":false})) + }, + GfnService::logout, + GfnService::logout_all, + |service| service.switch_account(&json!({"userId":"new-account"})), + |service| service.remove_account(&json!({"userId":"old-account"})), + ]; + let (service, path) = test_service("http://127.0.0.1:1"); + { + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("old-account")); + state.persistence_state = "none".into(); + state.attempts.insert( + "pending".into(), + DeviceAttempt { + provider: LoginProvider::default_nvidia(), + device_code: "test-device".into(), + expires_at: now_ms() + 60_000, + deadline: Instant::now() + Duration::from_secs(60), + interval_seconds: 5, + next_poll: Instant::now(), + in_flight: false, + pending_session: Some(auth_fixture("new-account")), + }, + ); + } + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + for mutation in mutations { + let operation = service.auth_operation.lock().unwrap(); + let permit = requests.admit("mutation", "auth.mutation").unwrap(); + let token = permit.token.clone(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + crate::requests::scope(token, || { + crate::requests::check().unwrap(); + started_tx.send(()).unwrap(); + mutation(&service) + }) + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + requests.cancel("mutation"); + drop(operation); + assert_eq!(worker.join().unwrap().unwrap_err().code, "cancelled"); + }); + let state = service.state.lock().unwrap(); + assert_eq!(state.session.as_ref().unwrap().user.user_id, "old-account"); + assert!(state.attempts.contains_key("pending")); + assert!(!path.join("accounts.json").exists()); + drop(permit); + } + let _ = std::fs::remove_dir_all(path); + } + + #[test] + fn transient_fallback_failure_does_not_revoke_saved_session() { + let (url, server) = mock_responses( + vec![ + (400, json!({"error":"invalid_grant"})), + (503, json!({"error":"temporarily_unavailable"})), + ], + |_| {}, + ); + let (service, path) = test_service(&url); + let error = service + .refresh_session(&auth_fixture("test-account")) + .unwrap_err(); + server.join().unwrap(); + assert_eq!(error.code, "session_refresh_failed"); + assert!(!is_definitive_auth_revocation(&error)); + let _ = std::fs::remove_dir_all(path); + } + + #[test] + fn all_revoked_refresh_mechanisms_are_definitive() { + let (url, server) = mock_responses( + vec![ + (400, json!({"error":"invalid_grant"})), + (401, json!({"error":"invalid_token"})), + ], + |_| {}, + ); + let (service, path) = test_service(&url); + let error = service + .refresh_session(&auth_fixture("test-account")) + .unwrap_err(); + server.join().unwrap(); + assert!(is_definitive_auth_revocation(&error)); + let _ = std::fs::remove_dir_all(path); + } + + #[test] + fn upstream_errors_do_not_echo_credentials_or_trust_server_error_revocation() { + let (url, server) = mock_responses( + vec![ + ( + 400, + json!({"error":"invalid_grant", "error_description":"test-secret-token"}), + ), + ( + 503, + json!({"error":"invalid_grant", "access_token":"test-secret-token"}), + ), + ], + |_| {}, + ); + let client = Client::builder().no_proxy().build().unwrap(); + let error = ServiceError::response("Test", client.get(&url).send().unwrap()); + assert!(!error.message.contains("test-secret-token")); + assert!(is_definitive_auth_revocation(&error)); + let error = ServiceError::response("Test", client.get(&url).send().unwrap()); + assert!(!error.message.contains("test-secret-token")); + assert!(!is_definitive_auth_revocation(&error)); + server.join().unwrap(); + } + + #[test] + fn network_errors_do_not_include_request_urls() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let error = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(1)) + .build() + .unwrap() + .get(format!("http://{address}/?access_token=test-secret-token")) + .send() + .unwrap_err(); + let error = ServiceError::network("Test request", error); + assert_eq!(error.code, "network_error"); + assert!(!error.message.contains("test-secret-token")); + assert!(!error.message.contains(&address.to_string())); + } + + #[test] + fn network_errors_name_the_failed_transport_stage_without_the_host() { + struct UnresolvedHost; + impl reqwest::dns::Resolve for UnresolvedHost { + fn resolve(&self, _: reqwest::dns::Name) -> reqwest::dns::Resolving { + Box::pin(std::future::ready(Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "no such host", + ) + .into()))) + } + } + let client = |builder: reqwest::blocking::ClientBuilder, timeout: Duration| { + builder.no_proxy().timeout(timeout).build().unwrap() + }; + let failure = |client: Client, url: String| { + ServiceError::network("Server info failed", client.get(url).send().unwrap_err()) + }; + + let unresolved = failure( + client( + Client::builder().dns_resolver(std::sync::Arc::new(UnresolvedHost)), + Duration::from_secs(10), + ), + "https://prod.partner.geforcenow.nvidiagrid.net/v2/serverInfo".into(), + ); + assert_eq!(unresolved.code, "network_error"); + assert_eq!( + unresolved.message, + "Server info failed: error sending request (dns)" + ); + + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = closed.local_addr().unwrap(); + drop(closed); + let refused = failure( + client(Client::builder(), Duration::from_secs(10)), + format!("http://{address}/v2/serverInfo"), + ); + assert_eq!( + refused.message, + "Server info failed: error sending request (connect)" + ); + + let plaintext = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = plaintext.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = plaintext.accept().unwrap(); + let _ = std::io::Write::write_all( + &mut stream, + b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\n\r\n", + ); + let _ = stream.read(&mut [0; 1024]); + }); + let handshake = failure( + client(Client::builder(), Duration::from_secs(10)), + format!("https://{address}/v2/serverInfo"), + ); + server.join().unwrap(); + assert_eq!( + handshake.message, + "Server info failed: error sending request (tls)" + ); + + let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = silent.local_addr().unwrap(); + let stalled = failure( + client(Client::builder(), Duration::from_millis(500)), + format!("http://{address}/v2/serverInfo"), + ); + drop(silent); + assert_eq!( + stalled.message, + "Server info failed: error sending request (timeout)" + ); + } + + #[test] + fn browser_service_identity_keeps_nvidia_required_webrtc_label() { + let headers = lcars_headers("token", "BROWSER", "WEBRTC", false).unwrap(); + assert_eq!(headers["nv-client-type"], "BROWSER"); + assert_eq!(headers["nv-client-streamer"], "WEBRTC"); + } + + #[test] + fn steam_deck_identity_advertises_valve_console_profile() { + let headers = lcars_headers("token", "NATIVE", "NVIDIA-CLASSIC", true).unwrap(); + assert_eq!(headers["nv-device-os"], "STEAMOS"); + assert_eq!(headers["nv-device-type"], "CONSOLE"); + assert_eq!(headers["nv-device-make"], "VALVE"); + assert_eq!(headers["nv-device-model"], "STEAMDECK"); + } + + #[test] + fn desktop_identity_keeps_generic_pc_profile() { + let headers = lcars_headers("token", "NATIVE", "NVIDIA-CLASSIC", false).unwrap(); + assert_eq!(headers["nv-device-type"], "DESKTOP"); + assert_eq!(headers["nv-device-make"], "GENERIC"); + assert_eq!(headers["nv-device-model"], "PC"); + } + + #[test] + fn store_marquee_parses_marketing_and_game_slides() { + let payload: Value = serde_json::from_str( + r#"{"data":{"panels":[{ + "id":"marquee","name":"Marquee","sections":[{ + "id":"s1","title":"Hero","items":[ + {"__typename":"MarketingItem","title":"GFN Thursday","body":"New drops", + "images":{"MARQUEE_HERO_IMAGE":"https://img.example/hero.jpg"}, + "action":{"label":"View details","uri":"gfn://x"}}, + {"__typename":"GameItem","app":{ + "id":"123","title":"Doom","publisherName":"Bethesda", + "images":{"MARQUEE_HERO_IMAGE":"https://img.example/doom.jpg"}, + "variants":[{"id":"123","appStore":"Steam", + "gfn":{"library":{"status":"NOT_OWNED"}}}], + "gfn":{"playabilityState":"PLAYABLE"}}}, + {"__typename":"FilterItem","id":"f","title":"Shop"} + ]}]}]}}"#, + ) + .unwrap(); + let slides = parse_store_marquee(&payload, &HashMap::new()); + assert_eq!(slides.len(), 2); + assert_eq!(slides[0]["kind"], "marketing"); + assert_eq!(slides[0]["title"], "GFN Thursday"); + assert_eq!(slides[0]["image"], "https://img.example/hero.jpg"); + assert_eq!(slides[0]["actionLabel"], "View details"); + assert_eq!(slides[1]["kind"], "game"); + assert_eq!(slides[1]["game"]["title"], "Doom"); + } + + #[test] + fn store_panels_keep_titled_sections_with_valid_games() { + let payload: Value = serde_json::from_str( + r#"{"data":{"panels":[{ + "id":"main","name":"Main","sections":[ + {"id":"gfn-thu","title":"GFN Thursday","items":[ + {"__typename":"GameItem","app":{ + "id":"7","title":"Hades", + "variants":[{"id":"7","appStore":"Steam", + "gfn":{"library":{"status":"NOT_OWNED"}}}], + "gfn":{"playabilityState":"PLAYABLE"}}}, + {"__typename":"GameItem","app":{"id":"8","title":"","variants":[]}} + ]}, + {"id":"empty","title":"","items":[]} + ]}]}}"#, + ) + .unwrap(); + let panels = parse_store_panels(&payload, &HashMap::new()); + assert_eq!(panels.len(), 1); + assert_eq!(panels[0]["sections"].as_array().unwrap().len(), 1); + let games = panels[0]["sections"][0]["games"].as_array().unwrap(); + assert_eq!(games.len(), 1); + assert_eq!(games[0]["title"], "Hades"); + } + + #[test] + fn store_shelves_prefer_posters_without_changing_hero_art() { + let payload = json!({"data":{"panels":[{"id":"main","name":"Main","sections":[{ + "id":"featured","title":"Featured","items":[{"__typename":"GameItem","app":{ + "id":"7","title":"Game","variants":[{"id":"7","appStore":"STEAM"}], + "images":{"GAME_BOX_ART":"https://img.example/poster.jpg","HERO_IMAGE":"https://img.example/hero.jpg"} + }}] + }]}]}}); + let panels = parse_store_panels(&payload, &HashMap::new()); + let game = &panels[0]["sections"][0]["games"][0]; + assert_eq!(game["imageUrl"], "https://img.example/poster.jpg"); + assert_eq!(game["heroImageUrl"], "https://img.example/hero.jpg"); + assert!(STORE_PANELS_QUERY.contains("GAME_BOX_ART")); + assert!(STORE_PANELS_QUERY.contains("... on GfnSubscriptionFeatureValue { key value }")); + } + + #[test] + fn home_key_art_is_separate_from_posters_and_heroes() { + let game = app_to_game(&json!({ + "id":"7","title":"Game","variants":[{"id":"7","appStore":"STEAM"}], + "images":{ + "GAME_BOX_ART":"https://img.example/poster.jpg", + "KEY_IMAGE":"https://img.example/key-image.jpg", + "KEY_ART":"https://img.nvidiagrid.net/apps/game/ZZ/KEY_ART.jpg", + "HERO_IMAGE":"https://img.example/hero.jpg" + } + })) + .unwrap(); + assert_eq!(game["imageUrl"], "https://img.example/poster.jpg"); + assert_eq!(game["heroImageUrl"], "https://img.example/hero.jpg"); + assert_eq!( + game["keyArtUrl"], + "https://img.nvidiagrid.net/apps/game/ZZ/KEY_ART.jpg;f=jpg;w=900" + ); + } + + #[test] + fn home_key_art_handles_missing_and_empty_images() { + for (images, expected) in [ + ( + json!({"KEY_ART":" ", "KEY_IMAGE":"https://img.example/key.jpg"}), + json!("https://img.example/key.jpg"), + ), + ( + json!({"KEY_ART":["", "https://img.example/key-art.jpg"]}), + json!("https://img.example/key-art.jpg"), + ), + ( + json!({"GAME_BOX_ART":"https://img.example/poster.jpg"}), + Value::Null, + ), + (Value::Null, Value::Null), + ] { + let game = app_to_game(&json!({ + "id":"7","title":"Game","variants":[{"id":"7","appStore":"STEAM"}], + "images":images + })) + .unwrap(); + assert_eq!(game["keyArtUrl"], expected); + } + } + + #[test] + fn store_definitions_keep_groups_with_options() { + let payload = json!({"data":{"filterGroupDefinitions":[ + {"id":"digital_store","label":"Stores","filters":[ + {"id":"steam","label":"Steam"}, + {"id":"epic","label":"Epic Games"}]}, + {"id":"empty","label":"Empty","filters":[]}, + ]}}); + let groups = parse_store_definitions(&payload); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0]["id"], "digital_store"); + assert_eq!(groups[0]["options"].as_array().unwrap().len(), 2); + } + + #[test] + fn provider_codes_map_to_one_display_name() { + assert_eq!( + provider_display_name("BPC", Some("BPC")).as_deref(), + Some("bro.game") + ); + assert_eq!( + provider_display_name("BPC", None).as_deref(), + Some("bro.game") + ); + assert_eq!( + provider_display_name("DIG", Some(" Digevo ")).as_deref(), + Some("Digevo") + ); + assert_eq!(provider_display_name("NEW", Some(" ")), None); + assert_eq!(provider_display_name("NEW", None), None); + assert!(!is_alliance_provider("NVIDIA")); + assert!(is_alliance_provider("BPC")); + assert!(is_alliance_provider("UNKNOWN")); + } + + #[test] + fn saved_accounts_name_partners_and_report_expired_sign_ins() { + let (service, path) = test_service("http://127.0.0.1:9"); + let mut partner = auth_fixture("partner"); + partner.provider.code = "BPC".into(); + partner.provider.idp_id = "bpc-idp".into(); + partner.provider.display_name = "bro.game".into(); + service.vault.save(&partner).unwrap(); + service.vault.save(&auth_fixture("nvidia")).unwrap(); + service.vault.mark_needs_sign_in("partner").unwrap(); + service.state.lock().unwrap().providers.push(LoginProvider { + idp_id: "dig-idp".into(), + code: "DIG".into(), + display_name: "Digevo".into(), + streaming_service_url: "https://dig.example/".into(), + priority: 3, + }); + let mut legacy = auth_fixture("legacy"); + legacy.provider.code = "DIG".into(); + legacy.provider.display_name = String::new(); + service.vault.save(&legacy).unwrap(); + let result = service.saved_accounts().unwrap(); + let accounts = result["accounts"].as_array().unwrap(); + let by_id = |id: &str| { + accounts + .iter() + .find(|item| item["userId"] == id) + .unwrap() + .clone() + }; + assert_eq!(by_id("partner")["providerName"], "bro.game"); + assert_eq!(by_id("partner")["providerIdpId"], "bpc-idp"); + assert_eq!(by_id("partner")["alliance"], true); + assert_eq!(by_id("partner")["needsSignIn"], true); + assert_eq!(by_id("nvidia")["providerName"], "NVIDIA"); + assert_eq!(by_id("nvidia")["alliance"], false); + assert_eq!(by_id("nvidia")["needsSignIn"], false); + assert_eq!(by_id("legacy")["providerName"], "Digevo"); + assert!(accounts.iter().all(|item| item.get("tokens").is_none())); + assert!(!result.to_string().contains("test-access")); + std::fs::remove_dir_all(path).ok(); + } + + #[test] + fn provider_discovery_is_normalized_and_sorted() { + let payload = json!({"gfnServiceInfo":{"gfnServiceEndpoints":[ + {"idpId":"two","loginProviderCode":"BPC","loginProviderDisplayName":"BPC","streamingServiceUrl":"https://two.example","loginProviderPriority":20}, + {"idpId":"one","loginProviderCode":"NVIDIA","loginProviderDisplayName":"NVIDIA","streamingServiceUrl":"https://one.example/","loginProviderPriority":1} + ]}}); + let providers = parse_providers(&payload); + assert_eq!(providers[0].idp_id, "one"); + assert_eq!(providers[1].display_name, "bro.game"); + assert!(providers[1].streaming_service_url.ends_with('/')); + } + + #[test] + fn cached_provider_lookup_does_not_reenter_state_lock() { + let client = Client::builder().build().unwrap(); + let service = GfnService::with_client(client, Endpoints::default(), std::env::temp_dir()); + service.state.lock().unwrap().providers = vec![LoginProvider::default_nvidia()]; + service.state.lock().unwrap().providers_expires = + Some(Instant::now() + Duration::from_secs(900)); + let result = service.providers().unwrap(); + assert_eq!(result["providers"][0]["code"], "NVIDIA"); + } + + #[test] + fn public_catalog_mapping_matches_electron_contract() { + let game = public_game_to_info(&json!({ + "id":"ignored", "title":"Portal 2", "status":"AVAILABLE", + "steamUrl":"https://store.steampowered.com/app/620/Portal_2/", "store":"Steam" + })) + .unwrap(); + assert_eq!(game["id"], "620"); + assert_eq!(game["launchAppId"], "620"); + assert_eq!(game["variants"][0]["store"], "Steam"); + assert!( + game["imageUrl"] + .as_str() + .unwrap() + .contains("/620/header.jpg") + ); + assert!(public_game_to_info(&json!({"title":"Gone", "status":"MAINTENANCE"})).is_none()); + } + + #[test] + fn account_library_mapping_preserves_launch_and_ownership() { + let game = app_to_game(&json!({ + "id":"cms-portal", + "title":"Portal 2", + "publisherName":"Valve", + "genres":["Puzzle"], + "images":{"GAME_BOX_ART":"https://img.nvidiagrid.net/apps/portal"}, + "variants":[{ + "id":"620", + "appStore":"STEAM", + "supportedControls":["GAMEPAD"], + "gfn":{"status":"AVAILABLE","features":[{"key":"IN_GAME_SETTINGS_PERSISTENCE_ENABLED","value":"true"}],"library":{"status":"PLATFORM_SYNC","selected":true,"lastPlayedDate":"2026-01-01"}} + }], + "gfn":{"playabilityState":"PLAYABLE"} + })).unwrap(); + assert_eq!(game["launchAppId"], "620"); + assert_eq!(game["selectedVariantIndex"], 0); + assert_eq!(game["isInLibrary"], true); + assert_eq!(game["variants"][0]["inLibrary"], true); + assert_eq!( + game["variants"][0]["supportsInGameSettingsPersistence"], + true + ); + assert_eq!(game["variants"][0]["features"], json!([])); + assert_eq!(game["requiresPaidMembership"], false); + assert!(game["imageUrl"].as_str().unwrap().ends_with(";f=jpg;w=900")); + assert!(game["searchText"].as_str().unwrap().contains("valve")); + assert!(!gfn_feature_enabled( + &json!({"key":"IN_GAME_SETTINGS_PERSISTENCE_ENABLED","value":"false"}), + "IN_GAME_SETTINGS_PERSISTENCE_ENABLED" + )); + } + + #[test] + fn account_library_mapping_preserves_every_platform_ownership_state() { + let game = app_to_game(&json!({ + "id":"cms-multi-store", + "title":"Multi Store Game", + "variants":[ + {"id":"1001","appStore":"Steam","gfn":{"library":{"status":"PLATFORM_SYNC","selected":true},"features":[{"key":"RTX_ENABLED","value":"true"},{"key":"REFLEX_ENABLED","value":"true"}]}}, + {"id":"1002","appStore":"Epic Games Store","gfn":{"library":{"status":"NOT_OWNED","selected":false}}}, + {"id":"1003","appStore":"Xbox","gfn":{"library":{"status":"MANUAL","selected":false}}} + ], + "gfn":{"playabilityState":"PLAYABLE","minimumMembershipTierLabel":"Premium"} + })) + .unwrap(); + + assert_eq!(game["requiresPaidMembership"], true); + assert_eq!( + game["availableStores"], + json!(["Steam", "Epic Games Store", "Xbox"]) + ); + assert_eq!(game["selectedVariantIndex"], 0); + assert_eq!(game["variants"][0]["inLibrary"], true); + assert_eq!(game["variants"][1]["inLibrary"], false); + assert_eq!(game["variants"][2]["inLibrary"], true); + assert_eq!(game["variants"][0]["features"], json!(["rtx", "reflex"])); + assert!(game["variants"][1]["features"].is_null()); + } + + #[test] + fn account_library_mapping_prefers_owned_variant_without_saved_selection() { + for status in ["MANUAL", "PLATFORM_SYNC", "IN_LIBRARY"] { + let game = app_to_game(&json!({ + "id":"cms-multi-store", "title":"Multi Store Game", + "variants":[ + {"id":"1001","appStore":"Steam","gfn":{"library":{"status":"NOT_OWNED"}}}, + {"id":"1003","appStore":"Xbox","gfn":{"library":{"status":status}}} + ] + })) + .unwrap(); + assert_eq!(game["selectedVariantIndex"], 1); + assert_eq!(game["launchAppId"], "1003"); + } + } + + #[test] + fn account_library_mapping_preserves_saved_selection_over_ownership() { + for selected in [false, true] { + let game = app_to_game(&json!({ + "id":"cms-multi-store", "title":"Multi Store Game", + "variants":[ + {"id":"1001","appStore":"Steam","gfn":{"library":{"status":"NOT_OWNED","selected":selected}}}, + {"id":"1003","appStore":"Xbox","gfn":{"library":{"status":"MANUAL"}}} + ] + })).unwrap(); + assert_eq!(game["selectedVariantIndex"], if selected { 0 } else { 1 }); + assert_eq!(game["launchAppId"], if selected { "1001" } else { "1003" }); + } + } + + #[test] + fn account_library_mapping_keeps_first_variant_when_none_owned() { + let game = app_to_game(&json!({ + "id":"cms-multi-store", "title":"Multi Store Game", + "variants":[ + {"id":"1001","appStore":"Steam"}, + {"id":"1003","appStore":"Xbox"} + ] + })) + .unwrap(); + assert_eq!(game["selectedVariantIndex"], 0); + assert_eq!(game["launchAppId"], "1001"); + } + + #[test] + fn creates_real_qr_matrix() { + let rows = qr_rows("https://login.nvidia.com/device?user_code=ABCD"); + assert!(rows.len() >= 21); + assert!(rows.iter().all(|row| row.len() == rows.len())); + assert!(rows.iter().any(|row| row.contains('1'))); + } + + #[test] + fn jwt_user_info_uses_claims_without_network() { + let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( + br#"{"sub":"42","email":"player@example.com","preferred_username":"Player","gfn_tier":"ULTIMATE"}"#, + ); + let user = user_from_jwt(&format!("header.{payload}.signature")).unwrap(); + assert_eq!(user.user_id, "42"); + assert_eq!(user.display_name, "Player"); + assert_eq!(user.membership_tier, "ULTIMATE"); + assert!(user.avatar_url.unwrap().contains("gravatar.com/avatar/")); + } + + #[test] + fn bug_reports_use_the_username_then_the_email_and_name_alliance_partners() { + let jwt_user = |claims: &str| { + let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(claims); + user_from_jwt(&format!("header.{payload}.signature")).unwrap() + }; + let nvidia = LoginProvider::default_nvidia(); + let named = + jwt_user(r#"{"sub":"1","email":"player@example.com","preferred_username":"Zortos"}"#); + let identity = bug_report_identity(&named, &nvidia); + assert_eq!(identity["reporter"], "Zortos"); + assert_eq!(identity["reporterBasis"], "username"); + assert_eq!(identity["alliancePartner"], false); + + let unnamed = jwt_user(r#"{"sub":"2","email":"player@example.com"}"#); + let partner = LoginProvider { + code: "BPC".to_owned(), + display_name: "Boosteroid".to_owned(), + ..LoginProvider::default_nvidia() + }; + let identity = bug_report_identity(&unnamed, &partner); + assert_eq!(identity["reporter"], "player@example.com"); + assert_eq!(identity["reporterBasis"], "email"); + assert_eq!(identity["providerName"], "Boosteroid"); + assert_eq!(identity["providerIdpId"], partner.idp_id); + assert_eq!(identity["userId"], "2"); + assert_eq!(identity["alliancePartner"], true); + + let anonymous = jwt_user(r#"{"sub":"3"}"#); + assert_eq!( + bug_report_identity(&anonymous, &nvidia)["reporter"], + Value::Null + ); + } + + #[test] + fn refresh_revocation_is_detected_from_oauth_errors() { + assert!(is_definitive_auth_revocation(&ServiceError { + code: "upstream_error", + message: "Refresh-token exchange failed (400): {\"error\":\"invalid_grant\"}" + .to_owned(), + })); + assert!(is_definitive_auth_revocation(&ServiceError { + code: "upstream_error", + message: "token has been revoked by the user".to_owned(), + })); + assert!(!is_definitive_auth_revocation(&ServiceError { + code: "network_error", + message: "Refresh-token exchange failed: connection reset".to_owned(), + })); + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/catalog.rs b/external/opennow/native/opennow-core/src/gfn/catalog.rs new file mode 100644 index 000000000..53bdef03b --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/catalog.rs @@ -0,0 +1,1216 @@ +use super::*; + +pub(super) const STORE_PANELS_QUERY: &str = r#"query GetStorePanels($vpcId: String!, $locale: String!, $panelNames: [String]!) { + panels(vpcId: $vpcId, language: $locale, names: $panelNames) { + id + name + sections { + id + title + items { + __typename + ... on GameItem { + app { + id + title + publisherName + images { GAME_BOX_ART KEY_IMAGE KEY_ART HERO_IMAGE TV_BANNER MARQUEE_HERO_IMAGE } + itemMetadata { campaignIds } + variants { + id + appStore + storeUrl + supportedControls + gfn { + status + features { + __typename + ... on GfnSubscriptionFeatureValue { key value } + ... on GfnSubscriptionFeatureValueList { key values } + } + library { status selected } + } + } + gfn { playType playabilityState minimumMembershipTierLabel } + } + } + } + } + } +}"#; + +const STORE_MARQUEE_QUERY: &str = r#"query GetStoreMarquee($vpcId: String!, $locale: String!, $panelNames: [String]!) { + panels(vpcId: $vpcId, language: $locale, names: $panelNames) { + id + name + sections { + id + title + items { + __typename + ... on MarketingItem { + id + title + body + images { MARQUEE_HERO_IMAGE HERO_IMAGE } + action { uri label } + } + ... on GameItem { + app { + id + title + publisherName + images { GAME_BOX_ART KEY_IMAGE KEY_ART HERO_IMAGE TV_BANNER MARQUEE_HERO_IMAGE } + itemMetadata { campaignIds } + variants { + id + appStore + storeUrl + supportedControls + gfn { + status + library { status selected } + } + } + gfn { playType playabilityState minimumMembershipTierLabel } + } + } + } + } + } +}"#; + +const STORE_DEFINITIONS_QUERY: &str = r#"query GetStoreFilterDefinitions($locale: String!) { + filterGroupDefinitions(language: $locale) { + id + label + filters { + id + label + filters + } + } + sortOrderDefinitions(language: $locale) { + id + label + orderBy + } +}"#; + +const STORE_MARQUEE_SHA: &str = "dd4bddfdef4707dfe340cc2040d6bb9c4c45f706976fca15b2ef33221c385d7f"; +const STORE_PANELS_SHA: &str = "46ec15f267a056e7d5e46e629efa929529e5e7542a4850faece90b9f8fa5f810"; + +const STORE_BROWSE_QUERY: &str = r#"query GetStoreBrowseApps( + $vpcId: String!, $locale: String!, $sortString: String!, + $fetchCount: Int!, $cursor: String!, $filters: AppFilterFields! +) { + apps(vpcId: $vpcId, language: $locale, orderBy: $sortString, first: $fetchCount, after: $cursor, filters: $filters) { + numberReturned numberSupported pageInfo { hasNextPage endCursor totalCount } + items { + id title developerName publisherName genres supportedControls + library { favorited } + images { KEY_ART KEY_IMAGE GAME_BOX_ART TV_BANNER HERO_IMAGE MARQUEE_HERO_IMAGE FEATURE_IMAGE GAME_LOGO SCREENSHOTS } + variants { + id appStore storeUrl supportedControls paymentModels { __typename } subscriptions + gfn { + status + features { + __typename + ... on GfnSubscriptionFeatureValue { key value } + ... on GfnSubscriptionFeatureValueList { key values } + } + library { status selected lastPlayedDate playStatus installed subscription } + stateDetails { __typename ... on VariantGfnAutoPatchingMetadata { subType startTime endTime historicalEtaMins etaPredictionType } ... on VariantGfnManualPatchingMetadata { subType startTime endTime } ... on VariantGfnMaintenanceMetadata { subType } } + } + } + gfn { playType playabilityState minimumMembershipTierLabel catalogSkuStrings { SKU_BASED_TAG SKU_BASED_PLAYABILITY_TEXT } } + itemMetadata { campaignIds } + } + } +}"#; + +const STORE_SEARCH_QUERY: &str = r#"query GetStoreSearchApps( + $vpcId: String!, $locale: String!, $sortString: String!, + $fetchCount: Int!, $cursor: String!, $searchString: String!, $filters: AppFilterFields! +) { + apps(vpcId: $vpcId, language: $locale, orderBy: $sortString, first: $fetchCount, after: $cursor, searchQuery: $searchString, filters: $filters) { + numberReturned numberSupported pageInfo { hasNextPage endCursor totalCount } + items { + id title developerName publisherName genres supportedControls + library { favorited } + images { KEY_ART KEY_IMAGE GAME_BOX_ART TV_BANNER HERO_IMAGE MARQUEE_HERO_IMAGE FEATURE_IMAGE GAME_LOGO SCREENSHOTS } + variants { + id appStore storeUrl supportedControls paymentModels { __typename } subscriptions + gfn { + status + features { + __typename + ... on GfnSubscriptionFeatureValue { key value } + ... on GfnSubscriptionFeatureValueList { key values } + } + library { status selected lastPlayedDate playStatus installed subscription } + stateDetails { __typename ... on VariantGfnAutoPatchingMetadata { subType startTime endTime historicalEtaMins etaPredictionType } ... on VariantGfnManualPatchingMetadata { subType startTime endTime } ... on VariantGfnMaintenanceMetadata { subType } } + } + } + gfn { playType playabilityState minimumMembershipTierLabel catalogSkuStrings { SKU_BASED_TAG SKU_BASED_PLAYABILITY_TEXT } } + itemMetadata { campaignIds } + } + } +}"#; + +const LIBRARY_QUERY: &str = r#"query GetLibraryApps( + $vpcId: String!, $locale: String!, $sortString: String!, + $fetchCount: Int!, $cursor: String!, $filters: AppFilterFields! +) { + apps(vpcId: $vpcId, language: $locale, orderBy: $sortString, first: $fetchCount, after: $cursor, filters: $filters) { + numberReturned numberSupported pageInfo { hasNextPage endCursor totalCount } + items { + id title developerName publisherName genres supportedControls + library { favorited } + images { KEY_ART KEY_IMAGE GAME_BOX_ART TV_BANNER HERO_IMAGE MARQUEE_HERO_IMAGE FEATURE_IMAGE GAME_LOGO SCREENSHOTS } + variants { + id appStore storeUrl supportedControls paymentModels { __typename } subscriptions + gfn { + status + features { + __typename + ... on GfnSubscriptionFeatureValue { key value } + ... on GfnSubscriptionFeatureValueList { key values } + } + library { status selected lastPlayedDate playStatus installed subscription } + stateDetails { __typename ... on VariantGfnAutoPatchingMetadata { subType startTime endTime historicalEtaMins etaPredictionType } ... on VariantGfnManualPatchingMetadata { subType startTime endTime } ... on VariantGfnMaintenanceMetadata { subType } } + } + } + gfn { playType playabilityState minimumMembershipTierLabel catalogSkuStrings { SKU_BASED_TAG SKU_BASED_PLAYABILITY_TEXT } } + itemMetadata { campaignIds } + } + } +}"#; + +impl GfnService { + pub fn catalog_languages( + &self, + params: &Value, + settings: &Value, + ) -> Result { + let (generation, provider) = { + let state = self.state.lock().expect("GFN state poisoned"); + ( + state.generation, + state + .session + .as_ref() + .map(|session| session.provider.clone()), + ) + }; + let proxy = config_from_settings(settings).map_err(ServiceError::invalid)?; + let scope = json!([ + "languages-v1", + self.endpoints.public_graphql, + provider, + proxy.map(|config| config.cache_scope) + ]); + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let mut result = self.store_cache.load_with_policy(&scope, &json!(["languages"]), params["refresh"] == true, crate::store_cache::CachePolicy {ttl_ms:14 * 24 * 60 * 60 * 1000,allow_stale:true}, || { + let response = self.store_cache.requests.send(client.post(&self.endpoints.public_graphql) + .header(ACCEPT, "application/json") + .json(&json!({"query":"{ overallGfnSupportedLanguages { language } }"})), "Supported-language query failed")?; + let payload = catalog_payload(response, None)?; + let items = payload["data"]["overallGfnSupportedLanguages"].as_array() + .filter(|items| !items.is_empty() && items.len() <= 512).ok_or_else(crate::catalog_types::invalid_metadata)?; + let mut languages = Vec::new(); + for item in items { + let language = item["language"].as_str().filter(|value| crate::language::valid_game_language(value)) + .ok_or_else(crate::catalog_types::invalid_metadata)?; + if !languages.contains(&language) { languages.push(language); } + } + if self.auth_generation() != generation { return Err(stale_catalog_scope()); } + Ok(json!({"metadataKind":"languages","languages":languages,"source":"overallGfnSupportedLanguages"})) + })?; + if self.auth_generation() != generation { + return Err(stale_catalog_scope()); + } + result["scopeGeneration"] = json!(generation); + result["source"] = json!("overallGfnSupportedLanguages"); + if !result["languages"].is_array() { + result["languages"] = json!([]); + } + if result["languages"].as_array().is_some_and(|languages| { + languages.len() > 512 + || languages.iter().any(|language| { + !language + .as_str() + .is_some_and(crate::language::valid_game_language) + }) + }) { + return Err(crate::catalog_types::invalid_metadata()); + } + Ok(result) + } + + pub fn catalog_definitions( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.authenticated_read(|session, generation| { + self.definitions_for(session, generation, params, settings) + }) + } + + pub(super) fn definitions_for( + &self, + session: &AuthSession, + generation: u64, + params: &Value, + settings: &Value, + ) -> Result { + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session + .tokens + .id_token + .as_deref() + .unwrap_or(&session.tokens.access_token); + let scope = json!([ + "definitions-v1", + session.user.user_id, + session.provider, + generation, + self.endpoints.graphql, + config_from_settings(settings) + .map_err(ServiceError::invalid)? + .map(|proxy| proxy.cache_scope), + "en_US" + ]); + let mut sections = json!({}); + for (name, fields) in [ + ( + "stores", + "appStoreDefinitions(language:$locale) { store label sortOrder smallImageUrl features { __typename ... on AccountLinkingSso { supported displayProposition } ... on AccountGamesSyncing { supported displayProposition } ... on AccountSubscriptions { displayProposition } } accountLinkingMetadata { supportedVariantIds isSupported isRequired label } }", + ), + ( + "genres", + "genreDefinitions(language:$locale) { genre label }", + ), + ( + "subscriptions", + "subscriptionDefinitions(language:$locale) { subscription label logoURL buySubscriptionURL primaryStore additionalStores { store thirdPartyLinkingRedirectKeyword thirdPartyLinkingShortURL } }", + ), + ] { + self.check_scope(session, generation)?; + let section = + self.store_cache.load_with_policy( + &scope, + &json!(["definitions", name]), + params["refresh"] == true, + crate::store_cache::CachePolicy { + ttl_ms: 24 * 60 * 60 * 1000, + allow_stale: true, + }, + || { + let payload = self.catalog_document( + &client, + token, + &format!("query OpenNowDefinitions($locale:String!) {{ {fields} }}"), + json!({"locale":"en_US"}), + )?; + let items = match name { + "stores" => crate::catalog_types::parse_list::< + crate::catalog_types::StoreDefinition, + >( + &payload["data"]["appStoreDefinitions"] + ), + "genres" => crate::catalog_types::parse_list::< + crate::catalog_types::GenreDefinition, + >( + &payload["data"]["genreDefinitions"] + ), + _ => crate::catalog_types::parse_list::< + crate::catalog_types::SubscriptionDefinition, + >( + &payload["data"]["subscriptionDefinitions"] + ), + }?; + self.check_scope(session, generation)?; + Ok(json!({"metadataKind":name,"items":items,"source":"server"})) + }, + )?; + sections[name] = section; + } + Ok(sections) + } + + pub fn catalog_game(&self, params: &Value, settings: &Value) -> Result { + let app_id = params + .get("appId") + .map(|_| bounded_id(params, "appId", true)) + .transpose()?; + let variant_id = params + .get("variantId") + .map(|_| bounded_id(params, "variantId", true)) + .transpose()?; + if app_id.is_some() == variant_id.is_some() { + return Err(ServiceError::invalid( + "Provide exactly one appId or variantId", + )); + } + let cms_id = variant_id + .as_ref() + .map(|value| { + value + .parse::() + .ok() + .filter(|id| *id > 0) + .ok_or_else(|| { + ServiceError::invalid("variantId must be a positive GraphQL Int") + }) + }) + .transpose()?; + self.authenticated_read(|session,generation| { + let revision = self.catalog_revision.load(std::sync::atomic::Ordering::Acquire); + let client = client_for_settings(&self.client,settings).map_err(ServiceError::invalid)?; + let token = session.tokens.id_token.as_deref().unwrap_or(&session.tokens.access_token); + let vpc = self.vpc_id(&client,session,generation,settings,token,Some(&self.store_cache.requests))?; + let (kind, ty, ids) = if let Some(id) = &app_id { ("appIds","String",json!([id])) } else { ("variantIds","Int",json!([cms_id])) }; + let fields = STORE_BROWSE_QUERY.split(" items {").nth(1).expect("catalog item selection"); + let fields = fields.strip_suffix(" }\n}").expect("catalog query suffix"); + let fields = fields.replace("id title developerName", "shortDescription computedValues { paymentModels { __typename } } id title developerName") + .replace("itemMetadata { campaignIds }", "") + .replace("status\n features", "status\n supportedLanguages { language ... on GfnLanguageSettings { availableFeatures setMethod } }\n features"); + let query = format!("query OpenNowGame($vpcId:String!,$locale:String!,$ids:[{ty}]!) {{ apps(vpcId:$vpcId,language:$locale,{kind}:$ids) {{ items {{ {fields} }} }}"); + let payload = self.catalog_document(&client,token,&query,json!({"vpcId":vpc,"locale":"en_US","ids":ids}))?; + let games = complete_games(&payload["data"]["apps"]["items"])?; + let game = games.into_iter().find(|game| { + app_id.as_ref().is_some_and(|id| game["id"] == *id) + || variant_id.as_ref().is_some_and(|id| game["variants"].as_array().is_some_and(|variants| variants.iter().any(|variant| variant["id"] == *id))) + }).ok_or_else(|| ServiceError { code:"catalog_game_not_found", message:"The exact catalog game was not returned".into() })?; + self.check_scope(session,generation)?; + self.check_catalog_revision(revision)?; + crate::store_catalog_page::bounded_result(json!({"game":game,"catalogRevision":revision,"freshness":"fresh","fetchedAt":now_ms()})) + }) + } + + pub fn library_catalog(&self, params: &Value, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + let page = crate::store_catalog_page::PageRequest::parse(params)?; + let traversal = bounded_id(params, "traversalId", false)?; + let revision = self.catalog_revision.load(std::sync::atomic::Ordering::Acquire); + if !page.cursor.is_empty() && params["catalogRevision"].as_u64() != Some(revision) { + return Err(ServiceError { code: "catalog_changed", message: "The catalog changed. Restart the library refresh.".into() }); + } + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session.tokens.id_token.as_deref().unwrap_or(&session.tokens.access_token); + let vpc = self.vpc_id(&client, session, generation, settings, token, None)?; + let context = format!("{:x}", Sha256::digest(json!([session.user.user_id,session.provider,generation, + self.endpoints.graphql,vpc,config_from_settings(settings).map_err(ServiceError::invalid)?.map(|proxy| proxy.cache_scope),revision,"en_US"]).to_string().as_bytes())); + if !page.cursor.is_empty() && params["catalogContext"] != context { + return Err(ServiceError { code:"catalog_changed", message:"The catalog route changed. Restart the library refresh.".into() }); + } + crate::store_catalog_page::fetch_bounded_page(page.limit, |count| { + self.check_scope(session,generation)?; + self.check_catalog_revision(revision)?; + let payload = self.catalog_document(&client, token, LIBRARY_QUERY, json!({ + "vpcId":vpc, "locale":"en_US", "sortString":"variants.gfn.library.lastPlayedDate:DESC,computedValues.libraryAddedDate:DESC,sortName:ASC", + "fetchCount":count, "cursor":page.cursor, + "filters":{"variants":{"gfn":{"library":{"status":{"notEquals":"NOT_OWNED"}}}}} + }))?; + self.check_scope(session, generation)?; + self.check_catalog_revision(revision)?; + let apps = &payload["data"]["apps"]; + let games = complete_games(&apps["items"])?; + let mut result = crate::store_catalog_page::page_result(&page.cursor, games, &apps["pageInfo"], now_ms())?; + result["source"] = json!("account-library"); + result["freshness"] = json!("fresh"); + result["catalogRevision"] = json!(revision); + result["catalogContext"] = json!(context); + result["traversalId"] = json!(traversal); + Ok(result) + }) + }) + } + + /// Sets `dlss` on every catalog game in a response from NVIDIA's DLSS games list. + /// An unavailable list omits the `dlss` key, a loaded list sets `dlss: null` for a + /// game it does not list, and neither ever fails the response. + pub fn annotate_dlss(&self, response: &mut Value, settings: &Value) { + let index = self.dlss_games.index(settings); + super::dlss_support::annotate(response, index.as_deref()); + } + + pub(super) fn check_catalog_revision(&self, revision: u64) -> Result<(), ServiceError> { + if self + .catalog_revision + .load(std::sync::atomic::Ordering::Acquire) + != revision + { + return Err(ServiceError { + code: "catalog_changed", + message: "The catalog changed. Restart this request.".into(), + }); + } + Ok(()) + } + + pub(super) fn invalidate_catalog(&self) -> Result<(), ServiceError> { + let revision = self + .catalog_revision + .fetch_add(1, std::sync::atomic::Ordering::AcqRel) + + 1; + self.store_cache.invalidate(revision) + } + + pub(super) fn catalog_document( + &self, + client: &Client, + token: &str, + query: &str, + variables: Value, + ) -> Result { + let response = self.store_cache.requests.send( + client + .post(&self.endpoints.graphql) + .headers(graphql_headers(token)?) + .json(&json!({"query":query,"variables":variables})), + "Catalog query failed", + )?; + catalog_payload(response, Some(token)) + } + + pub fn store_local_catalog( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.authenticated_read(|session, generation| { + let revision = self + .catalog_revision + .load(std::sync::atomic::Ordering::Acquire); + let mut scope = self.store_cache_scope(session, generation, settings)?; + let refreshed_scope = || { + let current = + self.authenticated_snapshot_for(session, generation, TokenPurpose::ServiceId)?; + self.store_cache_scope(¤t, generation, settings) + }; + if params["refresh"] == true { + self.store_catalog(&json!({"limit":100,"refresh":true}), settings)?; + scope = refreshed_scope()?; + } + let mut result = match self.store_cache.local_query(&scope, params) { + Err(error) if error.code == "store_cache_missing" => { + self.store_catalog( + &json!({"limit":100,"cursor":"","searchQuery":""}), + settings, + )?; + scope = refreshed_scope()?; + self.store_cache.local_query(&scope, params)? + } + result => result?, + }; + // A cold/partial cache grows by at most one upstream page per explicit + // demand. Never crawl the entire catalog on startup or a search keypress. + if result["count"] == 0 && result["cacheComplete"] == false { + if let Some(cursor) = result["upstreamCursor"].as_str().filter(|s| !s.is_empty()) { + self.store_catalog( + &json!({"limit":100,"cursor":cursor,"searchQuery":""}), + settings, + )?; + scope = refreshed_scope()?; + result = self.store_cache.local_query(&scope, params)?; + } + } + result + .as_object_mut() + .expect("Local Store response") + .remove("upstreamCursor"); + self.check_scope(session, generation)?; + self.check_catalog_revision(revision)?; + result["catalogRevision"] = json!(revision); + Ok(result) + }) + } + + pub fn store_catalog(&self, params: &Value, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + let revision = self + .catalog_revision + .load(std::sync::atomic::Ordering::Acquire); + self.check_scope(session, generation)?; + let page = crate::store_catalog_page::PageRequest::parse(params)?; + let client = + client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session + .tokens + .id_token + .as_deref() + .unwrap_or(&session.tokens.access_token); + let scope = self.store_cache_scope(session, generation, settings)?; + let mut filters = json!({}); + let mut sort = "itemMetadata.relevance:DESC,sortName:ASC".to_owned(); + if params.get("filterId").is_some() || params.get("sortId").is_some() { + let definitions = + self.store_presentation(&json!({"section":"filters"}), settings)?; + if params.get("filterId").is_some() { + let id = bounded_id(params, "filterId", true)?; + filters = definitions["items"] + .as_array() + .into_iter() + .flatten() + .flat_map(|group| group["options"].as_array().into_iter().flatten()) + .find(|option| option["id"] == id) + .map(|option| option["expression"].clone()) + .filter(Value::is_object) + .ok_or_else(|| { + ServiceError::invalid("The server filter expression is unavailable") + })?; + } + if params.get("sortId").is_some() { + let id = bounded_id(params, "sortId", true)?; + sort = definitions["sortOrders"] + .as_array() + .into_iter() + .flatten() + .find(|option| option["id"] == id) + .and_then(|option| option["orderBy"].as_str()) + .filter(|value| !value.is_empty() && value.len() <= 1024) + .ok_or_else(|| { + ServiceError::invalid("The server sort expression is unavailable") + })? + .to_owned(); + } + } + let key = if filters == json!({}) && sort == "itemMetadata.relevance:DESC,sortName:ASC" + { + json!(["page", page.limit, page.cursor, page.search]) + } else { + json!(["page", page.limit, page.cursor, page.search, filters, sort]) + }; + let refresh = params["refresh"].as_bool() == Some(true) && page.cursor.is_empty(); + if params["revalidate"] == true { + self.store_cache.invalidate_key(&scope, &key); + } + let mut result = self.store_cache.load_or_fetch(&scope, &key, refresh, || { + let vpc_id = self.vpc_id( + &client, + session, + generation, + settings, + token, + Some(&self.store_cache.requests), + )?; + // Each retry starts at the SAME cursor. Never truncate a fetched page: + // doing so would skip games when returning NVIDIA's end cursor. + crate::store_catalog_page::fetch_bounded_page(page.limit, |fetch_count| { + let searching = !page.search.is_empty(); + let query = if searching { + STORE_SEARCH_QUERY + } else { + STORE_BROWSE_QUERY + }; + let mut variables = json!({ + "vpcId":vpc_id, "locale":"en_US", + "sortString":sort, + "fetchCount":fetch_count, "cursor":page.cursor, "filters":filters + }); + if searching { + variables["searchString"] = Value::String(page.search.clone()); + } + let response = self.store_cache.requests.send( + client + .post(&self.endpoints.graphql) + .headers(graphql_headers(token)?) + .json(&json!({"query":query,"variables":variables})), + "GFN store query failed", + )?; + let payload = catalog_payload(response, Some(token))?; + let apps = &payload["data"]["apps"]; + let items = apps["items"].as_array().ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: "Store response has no games array".to_owned(), + })?; + let games = complete_games(&Value::Array(items.clone()))?; + self.check_scope(session, generation)?; + self.check_catalog_revision(revision)?; + crate::store_catalog_page::page_result( + &page.cursor, + games, + &apps["pageInfo"], + now_ms(), + ) + }) + })?; + self.check_catalog_revision(revision)?; + result["catalogRevision"] = json!(revision); + Ok(result) + }) + } + + fn store_cache_scope( + &self, + session: &AuthSession, + generation: u64, + settings: &Value, + ) -> Result { + let proxy = config_from_settings(settings).map_err(ServiceError::invalid)?; + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session + .tokens + .id_token + .as_deref() + .unwrap_or(&session.tokens.access_token); + let vpc = self.vpc_id( + &client, + session, + generation, + settings, + token, + Some(&self.store_cache.requests), + )?; + Ok(json!([ + session.user.user_id, + session.provider, + generation, + session.user.membership_tier, + proxy + .map(|config| config.cache_scope) + .unwrap_or_else(|| "direct".into()), + "en_US", + self.endpoints.graphql, + vpc, + self.catalog_revision + .load(std::sync::atomic::Ordering::Acquire), + "catalog-v3" + ])) + } + + pub fn store_presentation( + &self, + params: &Value, + settings: &Value, + ) -> Result { + self.authenticated_read(|session, generation| { + let revision = self.catalog_revision.load(std::sync::atomic::Ordering::Acquire); + let section = params["section"].as_str().unwrap_or(""); + if !matches!(section, "marquee" | "panels" | "filters") { + return Err(ServiceError::invalid( + "Store presentation requires marquee, panels or filters", + )); + } + let client = + client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let scope = self.store_cache_scope(session, generation, settings)?; + let mut result = self.store_cache.load_or_fetch( + &scope, + &json!(["presentation", section]), + false, + || { + let token = session + .tokens + .id_token + .as_deref() + .unwrap_or(&session.tokens.access_token); + let vpc_id = self.vpc_id( + &client, + session, + generation, + settings, + token, + Some(&self.store_cache.requests), + )?; + let (variables, request_type, sha, query) = match section { + "panels" => ( + json!({"vpcId":vpc_id,"locale":"en_US","panelNames":["MAIN"]}), + "panels/MainV2", + STORE_PANELS_SHA, + STORE_PANELS_QUERY, + ), + "marquee" => ( + json!({"vpcId":vpc_id,"locale":"en_US","panelNames":["MARQUEE"]}), + "panels/Marquee", + STORE_MARQUEE_SHA, + STORE_MARQUEE_QUERY, + ), + _ => ( + json!({"locale":"en_US"}), + "filterGroupAndSortOrderDefinitions", + "ef725de5e93b093de1ac7418fed0ffb4f6ae2b9c14f743ab274a791521488eb9", + STORE_DEFINITIONS_QUERY, + ), + }; + let payload = if section == "panels" { + // The persisted MainV2 document only supplies landscape hero art. + // Execute our document so GAME_BOX_ART is actually requested, just + // like Library/browse, rather than silently ignoring these fields. + self.check_scope(session, generation)?; + let response = self.store_cache.requests.send( + client + .post(&self.endpoints.graphql) + .headers(graphql_headers(token)?) + .json(&json!({"query":query,"variables":variables})), + "GFN storefront query failed", + )?; + if !response.status().is_success() { + return Err(ServiceError::response( + "GFN storefront query failed", + response, + )); + } + let payload = response.json::().map_err(|error| { + ServiceError::network("Invalid GFN storefront response", error) + })?; + if let Some(message) = graphql_error_message(&payload) { + return Err(ServiceError { + code: "graphql_error", + message, + }); + } + payload + } else { + fetch_panels_document( + &self.store_cache.requests, + &client, + token, + variables, + request_type, + sha, + query, + )? + }; + self.check_scope(session, generation)?; + let empty_index = HashMap::new(); + let items = match section { + "panels" => parse_store_panels(&payload, &empty_index), + "marquee" => parse_store_marquee(&payload, &empty_index), + _ => parse_store_definitions(&payload), + }; + // Optional chrome must not enlarge the games response or restart the core. + // An oversized/failed section is reported independently by the shell. + crate::store_catalog_page::bounded_result( + json!({"section":section,"items":items,"sortOrders":payload["data"]["sortOrderDefinitions"]}), + ) + }, + )?; + if section == "panels" && params["metadataOnly"] == true { + for panel in result["items"].as_array_mut().into_iter().flatten() { + for section in panel["sections"].as_array_mut().into_iter().flatten() { + let count = section["games"].as_array().map_or(0, Vec::len); + section["totalCount"] = json!(count); + section["games"] = json!([]); + } + } + } + self.check_catalog_revision(revision)?; + result["catalogRevision"] = json!(revision); + Ok(result) + }) + } +} + +pub(super) fn app_to_game(app: &Value) -> Option { + let id = app["id"] + .as_str() + .filter(|id| !id.is_empty() && id.len() <= 256)? + .to_owned(); + let title = app["title"].as_str()?.trim().to_owned(); + if title.is_empty() { + return None; + } + let mut variants = app["variants"].as_array().into_iter().flatten().filter_map(|variant| { + let variant_id = variant["id"].as_str()?.to_owned(); + let store = variant["appStore"].as_str().unwrap_or("Unknown").to_owned(); + let library_status = variant["gfn"]["library"]["status"].as_str().map(ToOwned::to_owned); + let in_library = library_status.as_deref().is_some_and(|status| matches!(status, "MANUAL" | "PLATFORM_SYNC" | "IN_LIBRARY")); + let supports_persistence = gfn_feature_enabled( + &variant["gfn"]["features"], + "IN_GAME_SETTINGS_PERSISTENCE_ENABLED", + ); + let subscription = &variant["gfn"]["library"]["subscription"]; + Some(json!({ + "id":variant_id, + "store":store, + "storeUrl":variant["storeUrl"], + "supportedControls":variant["supportedControls"].as_array().cloned().unwrap_or_default(), + "librarySelected":variant["gfn"]["library"]["selected"], + "inLibrary":in_library, + "libraryStatus":library_status, + "lastPlayedDate":variant["gfn"]["library"]["lastPlayedDate"], + "playStatus":variant["gfn"]["library"]["playStatus"], + "installed":variant["gfn"]["library"]["installed"], + "subscription":if subscription == "NONE" { &Value::Null } else { subscription }, + "paymentModels":variant["paymentModels"], + "subscriptions":variant["subscriptions"], + "supportedLanguages":variant["gfn"]["supportedLanguages"], + "stateDetails":serde_json::from_value::(variant["gfn"]["stateDetails"].clone()).ok(), + "gfnStatus":variant["gfn"]["status"], + "supportsInGameSettingsPersistence":supports_persistence, + "features":matches!(variant["gfn"]["features"], Value::Array(_) | Value::Object(_)) + .then(|| crate::catalog_types::GameFeature::from_variant_features(&variant["gfn"]["features"])), + })) + }).collect::>(); + let mut variant_ids = std::collections::HashSet::new(); + variants.retain(|variant| variant_ids.insert(variant["id"].as_str().unwrap().to_owned())); + if variants.is_empty() { + return None; + } + let selected_index = variants + .iter() + .position(|variant| variant["librarySelected"].as_bool() == Some(true)) + .or_else(|| { + variants + .iter() + .position(|variant| variant["inLibrary"].as_bool() == Some(true)) + }) + .unwrap_or(0); + let launch_id = variants + .get(selected_index) + .and_then(|variant| variant["id"].as_str()) + .filter(|value| value.chars().all(|character| character.is_ascii_digit())) + .or_else(|| { + variants + .iter() + .filter_map(|variant| variant["id"].as_str()) + .find(|value| value.chars().all(|character| character.is_ascii_digit())) + }) + .or_else(|| { + id.chars() + .all(|character| character.is_ascii_digit()) + .then_some(id.as_str()) + }) + .map(ToOwned::to_owned); + let available_stores = variants + .iter() + .filter_map(|variant| variant["store"].as_str().map(ToOwned::to_owned)) + .collect::>(); + let genres = string_array(&app["genres"]); + let controls = string_array(&app["supportedControls"]); + let image_url = first_image( + &app["images"], + &[ + "GAME_BOX_ART", + "KEY_IMAGE", + "KEY_ART", + "HERO_IMAGE", + "TV_BANNER", + ], + 900, + ); + let hero_image_url = first_image( + &app["images"], + &[ + "MARQUEE_HERO_IMAGE", + "HERO_IMAGE", + "TV_BANNER", + "FEATURE_IMAGE", + "KEY_IMAGE", + "KEY_ART", + ], + 1200, + ); + let key_art_url = first_image(&app["images"], &["KEY_ART", "KEY_IMAGE"], 900); + let screenshots = image_values(&app["images"]["SCREENSHOTS"], 1200); + let publisher = app["publisherName"].as_str().map(ToOwned::to_owned); + let developer = app["developerName"].as_str().map(ToOwned::to_owned); + let search_text = [ + vec![title.clone()], + publisher.clone().into_iter().collect(), + developer.clone().into_iter().collect(), + available_stores.clone(), + genres.clone(), + ] + .concat() + .join(" ") + .to_lowercase(); + let is_in_library = variants + .iter() + .any(|variant| variant["inLibrary"].as_bool() == Some(true)); + let last_played = variants + .iter() + .filter_map(|variant| variant["lastPlayedDate"].as_str()) + .next() + .map(ToOwned::to_owned); + let mut game = json!({ + "id":id, + "uuid":id, + "launchAppId":launch_id, + "title":title, + "developerName":developer, + "publisherName":publisher, + "genres":genres, + "supportedControls":controls, + "imageUrl":image_url, + "heroImageUrl":hero_image_url, + "keyArtUrl":key_art_url, + "screenshotUrl":screenshots.first(), + "screenshotUrls":screenshots, + "playType":app["gfn"]["playType"], + "favorited":app["library"]["favorited"], + "catalogSkuStrings":app["gfn"]["catalogSkuStrings"], + "campaignIds":app["itemMetadata"]["campaignIds"], + "paymentModels":app["computedValues"]["paymentModels"], + "description":app["shortDescription"], + "membershipTierLabel":app["gfn"]["minimumMembershipTierLabel"], + "playabilityState":app["gfn"]["playabilityState"], + "availableStores":available_stores, + "searchText":search_text, + "lastPlayed":last_played, + "isInLibrary":is_in_library, + "selectedVariantIndex":selected_index, + "variants":variants, + }); + game["requiresPaidMembership"] = json!(super::catalog_actions::requires_paid_membership(&game)); + Some(game) +} + +pub(super) fn bounded_id( + params: &Value, + key: &str, + required: bool, +) -> Result { + match params.get(key) { + None if !required => Ok(String::new()), + Some(Value::String(value)) + if !value.is_empty() && value.len() <= 256 && !value.chars().any(char::is_control) => + { + Ok(value.clone()) + } + _ => Err(ServiceError::invalid(format!( + "{key} must be a bounded nonempty identifier" + ))), + } +} + +fn stale_catalog_scope() -> ServiceError { + ServiceError { + code: "stale_account", + message: "The catalog context changed. Retry this request.".into(), + } +} + +pub(crate) fn catalog_payload( + response: Response, + token: Option<&str>, +) -> Result { + if !response.status().is_success() { + let status = response.status(); + let mut bytes = Vec::new(); + let payload = response + .take(16 * 1024 + 1) + .read_to_end(&mut bytes) + .ok() + .filter(|_| bytes.len() <= 16 * 1024) + .and_then(|_| serde_json::from_slice::(&bytes).ok()); + let messages = payload + .as_ref() + .into_iter() + .flat_map(|value| value["errors"].as_array().into_iter().flatten()) + .take(4) + .filter_map(|error| error["message"].as_str()) + .map(|message| { + let message = match token.filter(|token| !token.is_empty()) { + Some(token) => message.replace(token, "[redacted]"), + None => message.to_owned(), + }; + crate::diagnostics::runtime_failure_reason(&message) + }) + .collect::>(); + let context = format!("Catalog query failed ({status})"); + let auth_error = payload + .as_ref() + .and_then(|payload| payload["error"].as_str()) + .filter(|_| matches!(status.as_u16(), 400 | 401)) + .filter(|error| matches!(*error, "invalid_grant" | "invalid_token" | "token_revoked")); + return Err(ServiceError { + code: if status.as_u16() == 401 { + "http_unauthorized" + } else { + "upstream_error" + }, + message: if let Some(auth_error) = auth_error { + format!("{context}: {auth_error}") + } else if messages.is_empty() { + context + } else { + format!("{context}: {}", messages.join("; ")) + }, + }); + } + let mut bytes = Vec::new(); + response + .take(4 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .map_err(|_| ServiceError { + code: "network_error", + message: "Catalog response could not be read".into(), + })?; + if bytes.len() > 4 * 1024 * 1024 { + return Err(ServiceError { + code: "catalog_response_too_large", + message: "Catalog response exceeds the receive budget".into(), + }); + } + let payload: Value = serde_json::from_slice(&bytes).map_err(|_| ServiceError { + code: "invalid_upstream_response", + message: "Catalog response is not valid JSON".into(), + })?; + if payload["errors"] + .as_array() + .is_some_and(|errors| !errors.is_empty()) + { + return Err(ServiceError { + code: "graphql_error", + message: "The catalog query returned GraphQL errors".into(), + }); + } + Ok(payload) +} + +pub(super) fn complete_games(items: &Value) -> Result, ServiceError> { + let items = items.as_array().ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: "Catalog response has no games array".into(), + })?; + let mut games: Vec = Vec::new(); + let mut ids = HashMap::new(); + for item in items { + let variants = item["variants"] + .as_array() + .filter(|variants| !variants.is_empty() && variants.len() <= 128) + .ok_or_else(crate::catalog_types::invalid_metadata)?; + if variants.iter().any(|variant| { + variant["id"] + .as_str() + .is_none_or(|id| id.is_empty() || id.len() > 256) + }) { + return Err(crate::catalog_types::invalid_metadata()); + } + let mut game = app_to_game(item).ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: "Catalog page contains an invalid game identity".into(), + })?; + let id = game["id"].as_str().unwrap().to_owned(); + if let Some(index) = ids.get(&id).copied() { + let existing: &Value = &games[index]; + let mut variants = existing["variants"].as_array().unwrap().clone(); + let selected = game["variants"] + [game["selectedVariantIndex"].as_u64().unwrap_or(0) as usize]["id"] + .clone(); + for variant in game["variants"].as_array().unwrap() { + if let Some(index) = variants + .iter() + .position(|existing| existing["id"] == variant["id"]) + { + variants[index] = variant.clone(); + } else { + variants.push(variant.clone()); + } + } + game["selectedVariantIndex"] = json!( + variants + .iter() + .position(|variant| variant["id"] == selected) + .unwrap_or(0) + ); + game["availableStores"] = json!( + variants + .iter() + .map(|variant| variant["store"].clone()) + .collect::>() + ); + game["isInLibrary"] = + json!(variants.iter().any(|variant| variant["inLibrary"] == true)); + game["variants"] = json!(variants); + games[index] = game; + } else { + ids.insert(id, games.len()); + games.push(game); + } + } + Ok(games) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalog_http_error_details_are_redacted_and_bounded() { + let (url, worker) = crate::gfn::tests::mock_requests( + vec![( + 400, + json!({"errors":[ + {"message":"Syntax error: fixture-access-value Bearer fixture-bearer https://example.invalid/private fixture@example.invalid\nnext"}, + {"message":"x".repeat(2_000)}, + {"message":"third"}, + {"message":"fourth"}, + {"message":"fifth-must-not-appear"} + ]}), + )], + |_, _| {}, + ); + let response = Client::builder() + .no_proxy() + .build() + .unwrap() + .get(url) + .send() + .unwrap(); + let error = catalog_payload(response, Some("fixture-access-value")).unwrap_err(); + assert_eq!(error.code, "upstream_error"); + assert!(error.message.contains("Syntax error:")); + assert!(error.message.contains("fourth")); + for private in [ + "fixture-access-value", + "fixture-bearer", + "example.invalid", + "fifth-must-not-appear", + "\n", + ] { + assert!(!error.message.contains(private)); + } + assert!(error.message.chars().count() <= 4 * 480 + 100); + worker.join().unwrap(); + } + + #[test] + fn catalog_http_error_fallback_preserves_status_without_raw_bodies() { + for (status, body, code, suffix) in [ + ( + 401, + json!({"error":"invalid_token"}), + "http_unauthorized", + ": invalid_token", + ), + ( + 400, + json!({"error":"invalid_grant"}), + "upstream_error", + ": invalid_grant", + ), + (400, json!("private-raw-body"), "upstream_error", ""), + ( + 400, + json!({"errors":[{"message":"x".repeat(16 * 1024)}]}), + "upstream_error", + "", + ), + ] { + let (url, worker) = crate::gfn::tests::mock_requests(vec![(status, body)], |_, _| {}); + let response = Client::builder() + .no_proxy() + .build() + .unwrap() + .get(url) + .send() + .unwrap(); + let error = catalog_payload(response, None).unwrap_err(); + assert_eq!(error.code, code); + assert_eq!( + error.message, + format!( + "Catalog query failed ({}){suffix}", + reqwest::StatusCode::from_u16(status).unwrap() + ) + ); + worker.join().unwrap(); + } + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/catalog_actions.rs b/external/opennow/native/opennow-core/src/gfn/catalog_actions.rs new file mode 100644 index 000000000..86f13d9a7 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/catalog_actions.rs @@ -0,0 +1,607 @@ +use super::*; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(super) enum LaunchStatus { + Ready, + OwnershipRequired, + SelectionRequired, + LinkRequired, + SubscriptionRequired, + Patching, + Maintenance, + Unavailable, + MetadataUnconfirmed, +} + +#[derive(Serialize)] +pub(super) struct LaunchDecision { + pub(super) status: LaunchStatus, + pub(super) message: &'static str, +} + +pub(super) enum PlayabilityMetadata { + Required, + WhenPresent, +} + +pub(super) fn selected_variant<'a>(game: &'a Value, variant_id: &str) -> Option<&'a Value> { + game["variants"] + .as_array()? + .iter() + .find(|variant| variant["id"] == variant_id) +} + +pub(super) fn readiness_decision( + game: &Value, + variant: &Value, + playability: PlayabilityMetadata, +) -> Option { + use LaunchStatus::*; + let decide = |status, message| LaunchDecision { status, message }; + match game["playabilityState"].as_str() { + Some("PLAYABLE") => (), + Some("UNPLAYABLE_DUE_TO_UPGRADE" | "UNPLAYABLE_DUE_TO_TIME_CAPPED_LIMIT") => { + return Some(decide( + SubscriptionRequired, + "Your GeForce NOW membership does not currently allow this game. Check your membership and available playtime.", + )); + } + Some("UNKNOWN") => { + return Some(decide( + MetadataUnconfirmed, + "Game availability could not be confirmed. Refresh and try again.", + )); + } + None if matches!(playability, PlayabilityMetadata::Required) => { + return Some(decide( + MetadataUnconfirmed, + "Game availability could not be confirmed. Refresh and try again.", + )); + } + None => (), + _ => { + return Some(decide( + Unavailable, + "This game is currently unavailable on GeForce NOW.", + )); + } + } + match variant["stateDetails"]["__typename"].as_str() { + Some("VariantGfnAutoPatchingMetadata" | "VariantGfnManualPatchingMetadata") => { + return Some(decide( + Patching, + "This store version is being patched. Try again after the patch finishes.", + )); + } + Some("VariantGfnMaintenanceMetadata") => { + return Some(decide( + Maintenance, + "This store version is under maintenance.", + )); + } + _ => (), + } + match variant["gfnStatus"].as_str() { + Some("AVAILABLE") => (), + Some("PATCHING") => { + return Some(decide( + Patching, + "This store version is being patched. Try again later.", + )); + } + Some("SERVER_MAINTENANCE") => { + return Some(decide( + Maintenance, + "This store version is under maintenance.", + )); + } + None | Some("UNKNOWN") => { + return Some(decide( + MetadataUnconfirmed, + "Store version readiness could not be confirmed. Refresh and try again.", + )); + } + _ => { + return Some(decide( + Unavailable, + "This store version is currently unavailable.", + )); + } + } + if variant["playStatus"] == "NOT_PLAYABLE" { + return Some(decide( + Unavailable, + "This store version cannot currently be played. Check your store account, subscription, and library sync.", + )); + } + None +} + +pub(super) fn account_decision( + variant: &Value, + variant_id: &str, + access: &Value, +) -> Option { + use LaunchStatus::*; + let decide = |status, message| LaunchDecision { status, message }; + if access["definitions"]["stores"]["status"] != "success" { + return Some(decide( + MetadataUnconfirmed, + "Store account requirements could not be confirmed. Refresh and try again.", + )); + } + let store = + crate::catalog_types::normalize_store(variant["store"].as_str().unwrap_or_default()); + let Some(account) = access["accounts"] + .as_array() + .and_then(|accounts| accounts.iter().find(|account| account["provider"] == store)) + else { + return Some(decide( + MetadataUnconfirmed, + "The selected store's account requirements are unavailable. Refresh and try again.", + )); + }; + let linking = &account["accountLinkingMetadata"]; + let applies = linking["supportedVariantIds"] + .as_array() + .is_none_or(|ids| ids.is_empty() || ids.iter().any(|id| id == variant_id)); + if applies + && account["isRequired"] == true + && (account["isConnected"] != true || account["status"] == "expired") + { + return Some(decide( + LinkRequired, + "Link or reconnect the selected store account in Settings before launching this version.", + )); + } + if applies && account["supportsLinking"] == true && !account["isRequired"].is_boolean() { + return Some(decide( + MetadataUnconfirmed, + "The selected store's linking requirement could not be confirmed.", + )); + } + if let Some(id) = variant["subscription"].as_str().filter(|id| !id.is_empty()) { + if !access["subscriptions"] + .as_array() + .is_some_and(|subscriptions| { + subscriptions + .iter() + .any(|subscription| subscription["id"] == id) + }) + { + return Some(decide( + SubscriptionRequired, + "The store subscription recorded for this version is not active. Check your store account and sync your library.", + )); + } + } + None +} + +pub(super) fn requires_paid_membership(game: &Value) -> bool { + game["membershipTierLabel"].as_str().is_some_and(|tier| { + !tier.trim().is_empty() && !tier.trim().to_ascii_lowercase().starts_with("free") + }) +} + +pub(super) fn paid_membership_decision(subscription: &Value) -> Option { + use LaunchStatus::*; + let decide = |status, message| LaunchDecision { status, message }; + match subscription["membershipTier"] + .as_str() + .filter(|tier| !tier.trim().is_empty()) + { + None => Some(decide( + MetadataUnconfirmed, + "Membership details could not be confirmed. Refresh and try again.", + )), + Some(tier) if tier.trim().to_ascii_lowercase().starts_with("free") => Some(decide( + SubscriptionRequired, + "This game requires a paid GeForce NOW membership.", + )), + _ => game_play_denial(subscription), + } +} + +pub(super) fn game_play_denial(subscription: &Value) -> Option { + (subscription["isGamePlayAllowed"] == false).then_some(LaunchDecision { + status: LaunchStatus::SubscriptionRequired, + message: "Your membership does not currently allow gameplay. Check your available playtime.", + }) +} + +fn launch_decision( + game: &Value, + app_id: &str, + variant_id: &str, + access: &Value, + membership: impl FnOnce() -> Result, +) -> Result { + use LaunchStatus::*; + let decide = |status, message| LaunchDecision { status, message }; + if game["id"] != app_id || variant_id.parse::().ok().is_none_or(|id| id <= 0) { + return Ok(decide( + MetadataUnconfirmed, + "The exact store version could not be confirmed. Refresh and try again.", + )); + } + let Some(variant) = selected_variant(game, variant_id) else { + return Ok(decide( + MetadataUnconfirmed, + "The selected store version is no longer available. Choose a store version again.", + )); + }; + if let Some(decision) = readiness_decision(game, variant, PlayabilityMetadata::Required) { + return Ok(decision); + } + match variant["libraryStatus"].as_str() { + Some("MANUAL" | "PLATFORM_SYNC") => (), + Some("NOT_OWNED") => { + return Ok(decide( + OwnershipRequired, + "Confirm that you already own this game on the selected store before adding it to your GeForce NOW library.", + )); + } + _ => { + return Ok(decide( + MetadataUnconfirmed, + "Ownership of this store version could not be confirmed. Refresh or sync your library.", + )); + } + } + if let Some(decision) = account_decision(variant, variant_id, access) { + return Ok(decision); + } + if requires_paid_membership(game) + && let Some(decision) = paid_membership_decision(&membership()?) + { + return Ok(decision); + } + if variant["librarySelected"] == false { + return Ok(decide( + SelectionRequired, + "Choose this owned store version for play before launching.", + )); + } + if variant["librarySelected"] != true { + return Ok(decide( + MetadataUnconfirmed, + "Your preferred store version could not be confirmed. Refresh and try again.", + )); + } + Ok(decide(Ready, "Ready to play this store version.")) +} + +#[derive(Clone, Copy)] +enum Mutation { + AddFavorite, + RemoveFavorite, + AddOwned, + RemoveOwned, + SelectOwned, +} + +impl Mutation { + fn parse(method: &str) -> Result { + match method { + "catalog.favorites.add" => Ok(Self::AddFavorite), + "catalog.favorites.remove" => Ok(Self::RemoveFavorite), + "catalog.ownership.add" => Ok(Self::AddOwned), + "catalog.ownership.remove" => Ok(Self::RemoveOwned), + "catalog.ownership.select" => Ok(Self::SelectOwned), + _ => Err(ServiceError::invalid("Unknown catalog mutation")), + } + } + + fn root(self) -> &'static str { + match self { + Self::AddFavorite => "addFavoriteApp", + Self::RemoveFavorite => "removeFavoriteApp", + Self::AddOwned => "addOwnedVariant", + Self::RemoveOwned => "removeOwnedVariant", + Self::SelectOwned => "selectOwnedVariant", + } + } + + fn favorite(self) -> bool { + matches!(self, Self::AddFavorite | Self::RemoveFavorite) + } + + fn confirmed(self, game: &Value, variant_id: &str) -> bool { + let variant = selected_variant(game, variant_id); + match self { + Self::AddFavorite => game["favorited"] == true, + Self::RemoveFavorite => game["favorited"] == false, + Self::AddOwned => variant.is_some_and(|variant| { + matches!( + variant["libraryStatus"].as_str(), + Some("MANUAL" | "PLATFORM_SYNC") + ) + }), + Self::RemoveOwned => { + variant.is_some_and(|variant| variant["libraryStatus"] == "NOT_OWNED") + } + Self::SelectOwned => variant.is_some_and(|variant| { + variant["librarySelected"] == true + && matches!( + variant["libraryStatus"].as_str(), + Some("MANUAL" | "PLATFORM_SYNC") + ) + }), + } + } +} + +#[derive(Clone, Eq, Hash, PartialEq)] +pub(super) struct CatalogActionKey { + provider_id: String, + user_id: String, + app_id: String, +} + +pub(super) struct MutationPermit<'a> { + pending: &'a Mutex>, + key: CatalogActionKey, +} + +impl Drop for MutationPermit<'_> { + fn drop(&mut self) { + self.pending + .lock() + .expect("catalog mutations poisoned") + .remove(&self.key); + } +} + +impl GfnService { + pub(super) fn admit_catalog_action( + &self, + session: &AuthSession, + app_id: &str, + ) -> Result, ServiceError> { + let key = CatalogActionKey { + provider_id: session.provider.idp_id.clone(), + user_id: session.user.user_id.clone(), + app_id: app_id.to_owned(), + }; + let mut pending = crate::store_requests::lock(&self.catalog_mutations)?; + if !pending.insert(key.clone()) { + return Err(ServiceError { + code: "catalog_mutation_busy", + message: "An update or launch for this game is already in progress.".into(), + }); + } + Ok(MutationPermit { + pending: &self.catalog_mutations, + key, + }) + } + pub fn catalog_launch_inspect( + &self, + params: &Value, + settings: &Value, + ) -> Result { + let app_id = bounded_id(params, "appId", true)?; + let variant_id = bounded_id(params, "variantId", true)?; + if super::store_launch::store_launch_intent(params)? { + return self.store_launch_inspect(params, Some((&app_id, &variant_id)), settings); + } + let mut result = self.catalog_game(&json!({"appId":app_id}), settings)?; + let access = if selected_variant(&result["game"], &variant_id).is_some_and(|variant| { + matches!( + variant["libraryStatus"].as_str(), + Some("MANUAL" | "PLATFORM_SYNC") + ) + }) { + let access = self.account_connections(settings)?; + if access["scope"] != result["scope"] { + return Err(ServiceError { + code: "stale_account", + message: "The store account changed.".into(), + }); + } + access + } else { + Value::Null + }; + let decision = launch_decision(&result["game"], &app_id, &variant_id, &access, || { + let subscription = self.subscription(settings)?; + if subscription["scope"] != result["scope"] { + return Err(ServiceError { + code: "stale_account", + message: "The membership account changed.".into(), + }); + } + Ok(subscription["subscription"].clone()) + })?; + result["decision"] = serde_json::to_value(decision).expect("launch decision serializable"); + result["appId"] = json!(app_id); + result["variantId"] = json!(variant_id); + Ok(result) + } + + pub fn catalog_mutate( + &self, + method: &str, + params: &Value, + settings: &Value, + ) -> Result { + let mutation = Mutation::parse(method)?; + let app_id = bounded_id(params, "appId", true)?; + let variant_id = if mutation.favorite() { + String::new() + } else { + bounded_id(params, "variantId", true)? + }; + if matches!(mutation, Mutation::AddOwned) && params["confirmedExistingLicense"] != true { + return Err(ServiceError::invalid( + "Confirm existing ownership of the selected store license first", + )); + } + let (session, generation) = self.authenticated_snapshot(TokenPurpose::ServiceId, false)?; + if params["scope"] != scoped_result(json!({}), &session, generation)["scope"] { + return Err(ServiceError { + code: "stale_account", + message: "The catalog update belongs to a different account context.".into(), + }); + } + let _permit = self.admit_catalog_action(&session, &app_id)?; + let before = self.catalog_game(&json!({"appId":app_id}), settings)?; + self.check_scope(&session, generation)?; + if !mutation.favorite() && selected_variant(&before["game"], &variant_id).is_none() { + return Err(ServiceError::invalid( + "The selected variant does not belong to this game", + )); + } + if matches!(mutation, Mutation::SelectOwned) + && !Mutation::AddOwned.confirmed(&before["game"], &variant_id) + { + return Err(ServiceError::invalid( + "Only an owned store version can be selected for play", + )); + } + let root = mutation.root(); + let operation = format!("{}{}", root[..1].to_uppercase(), &root[1..]); + let (variable, argument, id) = if mutation.favorite() { + ("appId", "appId", &app_id) + } else { + ("cmsId", "variantId", &variant_id) + }; + let query = format!( + "mutation {operation}(${variable}: String!, $locale: String!) {{ {root}(language: $locale, {argument}: ${variable}) {{ app {{ id }} }} }}" + ); + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let current = + self.authenticated_snapshot_for(&session, generation, TokenPurpose::ServiceId)?; + let token = current + .tokens + .id_token + .as_deref() + .unwrap_or(¤t.tokens.access_token); + self.check_scope(&session, generation)?; + self.invalidate_catalog()?; + let sent = self.store_cache.requests.send( + client + .post(&self.endpoints.graphql) + .headers(graphql_headers(token)?) + .json(&json!({"query":query,"variables":{variable:id,"locale":"en_US"}})), + "Catalog update could not be confirmed", + ); + let mut http_status = None; + let mut graphql_errors = Vec::new(); + let error_code = match sent { + Ok(response) => { + http_status = Some(response.status().as_u16()); + let success = response.status().is_success(); + let mut bytes = Vec::new(); + let payload = response + .take(64 * 1024 + 1) + .read_to_end(&mut bytes) + .ok() + .filter(|_| bytes.len() <= 64 * 1024) + .and_then(|_| serde_json::from_slice::(&bytes).ok()); + if let Some(payload) = &payload { + for error in payload["errors"].as_array().into_iter().flatten().take(8) { + let safe = |value: &Value| { + value + .as_str() + .filter(|text| { + text.len() <= 128 + && text + .chars() + .all(|c| c.is_ascii_alphanumeric() || "_.-".contains(c)) + }) + .map(ToOwned::to_owned) + }; + graphql_errors.push(json!({"code":safe(&error["extensions"]["code"]),"path":error["path"].as_array().map(|path| path.iter().take(8).filter_map(safe).collect::>())})); + } + } + if !success { + Some(if http_status == Some(401) { + "http_unauthorized" + } else { + "upstream_error" + }) + } else if payload.as_ref().is_some_and(|payload| { + !payload["errors"].is_null() + && payload["errors"] + .as_array() + .is_none_or(|errors| !errors.is_empty()) + }) { + Some("graphql_error") + } else if payload.is_none() { + Some("invalid_upstream_response") + } else if payload + .as_ref() + .is_none_or(|payload| payload["data"][root]["app"]["id"] != app_id) + { + Some("mutation_identity_mismatch") + } else { + None + } + } + Err(error) => Some(error.code), + }; + let invalidation = self.invalidate_catalog(); + self.check_scope(&session, generation)?; + let fresh = self.catalog_game(&json!({"appId":app_id}), settings); + self.check_scope(&session, generation)?; + let confirmed = invalidation.is_ok() + && fresh + .as_ref() + .is_ok_and(|result| mutation.confirmed(&result["game"], &variant_id)); + let message = if confirmed { + "The current cloud library state is confirmed." + } else { + "Could not confirm the update. Refresh this game before trying again; the update may already have reached GeForce NOW." + }; + Ok(scoped_result( + json!({"appId":app_id,"variantId":variant_id,"operation":method, + "outcome":if error_code.is_none() {"acknowledged"} else {"unconfirmed"}, + "reconciliation":if confirmed {"confirmed"} else {"unconfirmed"},"message":message, + "error":{"code":error_code,"httpStatus":http_status,"graphql":graphql_errors, + "reconciliationCode":fresh.as_ref().err().map(|error| error.code),"invalidationCode":invalidation.as_ref().err().map(|error| error.code)}, + "game":fresh.as_ref().ok().map(|result| &result["game"]), + "catalogRevision":self.catalog_revision.load(std::sync::atomic::Ordering::Acquire)}), + &session, + generation, + )) + } + + pub fn catalog_favorites(&self, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + let revision = self.catalog_revision.load(std::sync::atomic::Ordering::Acquire); + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session.tokens.id_token.as_deref().unwrap_or(&session.tokens.access_token); + let vpc = self.vpc_id(&client, session, generation, settings, token, Some(&self.store_cache.requests))?; + let query = STORE_PANELS_QUERY.replace("GetStorePanels", "GetGameSection") + .replace(" title\n items", " title\n seeMoreInfo { filterTileId title filterIds minTiles sortOrderId }\n items"); + let payload = self.catalog_document(&client, token, &query, json!({"vpcId":vpc,"locale":"en_US","panelNames":["FAVORITES"]}))?; + let panels = payload["data"]["panels"].as_array().ok_or_else(|| ServiceError::invalid("Favorites panels were not returned"))?; + let mut games = Vec::new(); + let mut sections = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for panel in panels { + if panel["name"] != "FAVORITES" { + return Err(ServiceError::invalid("The returned panel is not the requested favorites panel")); + } + for section in panel["sections"].as_array().ok_or_else(|| ServiceError::invalid("Favorites sections were not returned"))? { + sections.push(json!({"id":section["id"],"title":section["title"],"seeMoreInfo":section["seeMoreInfo"]})); + for item in section["items"].as_array().ok_or_else(|| ServiceError::invalid("Favorites items were not returned"))? { + if item["__typename"] != "GameItem" { continue; } + let game = app_to_game(&item["app"]).ok_or_else(|| ServiceError::invalid("A favorites game could not be parsed"))?; + if seen.insert(game["id"].as_str().unwrap().to_owned()) { games.push(game); } + if games.len() > 1000 { return Err(ServiceError::invalid("Favorites exceed the bounded panel budget")); } + } + } + } + self.check_scope(session, generation)?; + if revision != self.catalog_revision.load(std::sync::atomic::Ordering::Acquire) { return Err(ServiceError {code:"stale_account",message:"Favorites changed while loading. Refresh again.".into()}); } + crate::store_catalog_page::bounded_result(json!({"games":games,"sections":sections,"coverage":"unknown","complete":false,"fetchedAt":now_ms(),"catalogRevision":revision})) + }) + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/catalog_actions/tests.rs b/external/opennow/native/opennow-core/src/gfn/catalog_actions/tests.rs new file mode 100644 index 000000000..5ff4c3f16 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/catalog_actions/tests.rs @@ -0,0 +1,914 @@ +use super::*; +use crate::gfn::tests::{auth_fixture, jwt, mock_requests, test_service}; + +fn service(url: &str) -> (GfnService, PathBuf) { + let (mut service, path) = test_service(url); + service.endpoints.graphql = format!("{url}/graphql"); + service.endpoints.server_info = Some(format!("{url}/vpc")); + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("fixture-account")); + state.restore_attempted = true; + drop(state); + (service, path) +} + +fn app(status: &str, selected: bool, favorite: bool) -> Value { + json!({"id":"parent-app","title":"Fixture game","library":{"favorited":favorite}, + "gfn":{"playabilityState":"PLAYABLE"},"variants":[ + {"id":"123","appStore":"STEAM","gfn":{"status":"AVAILABLE","library":{"status":status,"selected":selected,"playStatus":"PLAYABLE"}}}, + {"id":"456","appStore":"EPIC","gfn":{"status":"AVAILABLE","library":{"status":"NOT_OWNED","selected":false,"playStatus":"UNKNOWN"}}} + ]}) +} + +fn read(app: Value) -> (u16, Value) { + (200, json!({"data":{"apps":{"items":[app]}}})) +} + +fn vpc() -> (u16, Value) { + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})) +} + +fn scoped(mut params: Value) -> Value { + params["scope"] = + scoped_result(json!({}), &auth_fixture("fixture-account"), 0)["scope"].clone(); + params +} + +fn access() -> Value { + json!({"definitions":{"stores":{"status":"success"}},"accounts":[{"provider":"STEAM","isRequired":false,"supportsLinking":true,"isConnected":false}],"subscriptions":[]}) +} + +fn decision(game: &Value, app_id: &str, variant_id: &str, subscription: &Value) -> LaunchDecision { + super::launch_decision(game, app_id, variant_id, &access(), || { + Ok(subscription.clone()) + }) + .unwrap() +} + +#[test] +fn selected_store_link_and_subscription_requirements_use_current_account_metadata() { + let mut game = app_to_game(&app("MANUAL", true, false)).unwrap(); + let mut account = access(); + account["accounts"][0]["isRequired"] = json!(true); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::LinkRequired + ); + account["accounts"][0]["isConnected"] = json!(true); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::Ready + ); + account["accounts"][0]["status"] = json!("expired"); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::LinkRequired + ); + account["accounts"][0]["status"] = json!("connected"); + game["variants"][0]["subscription"] = json!("STORE_PASS"); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::SubscriptionRequired + ); + account["subscriptions"] = json!([{"id":"STORE_PASS"}]); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::Ready + ); + account["definitions"]["stores"]["status"] = json!("stale"); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + LaunchStatus::MetadataUnconfirmed + ); +} + +#[test] +fn nullable_variant_scope_preserves_required_account_linking() { + let game = app_to_game(&app("PLATFORM_SYNC", true, false)).unwrap(); + for required in [false, true] { + let definition: crate::catalog_types::StoreDefinition = serde_json::from_value(json!({ + "store":"STEAM","label":"Steam","features":[], + "accountLinkingMetadata":{"supportedVariantIds":null,"isRequired":required} + })) + .unwrap(); + let mut account = access(); + account["accounts"][0] = definition.connection_definition(); + assert_eq!( + super::launch_decision(&game, "parent-app", "123", &account, || Ok(Value::Null)) + .unwrap() + .status, + if required { + LaunchStatus::LinkRequired + } else { + LaunchStatus::Ready + } + ); + } +} + +#[test] +fn no_subscription_sentinel_is_normalized_without_bypassing_real_subscriptions() { + for subscription in [ + Value::Null, + json!("NONE"), + json!("STORE_PASS"), + json!("UNKNOWN_PASS"), + ] { + let mut metadata = app("PLATFORM_SYNC", true, false); + metadata["variants"][0]["gfn"]["library"]["subscription"] = subscription.clone(); + let game = app_to_game(&metadata).unwrap(); + let requires_subscription = subscription.is_string() && subscription != "NONE"; + assert_eq!( + game["variants"][0]["subscription"], + if requires_subscription { + subscription + } else { + Value::Null + } + ); + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + if requires_subscription { + LaunchStatus::SubscriptionRequired + } else { + LaunchStatus::Ready + } + ); + } +} + +#[test] +fn exact_selected_variant_policy_never_infers_ownership_from_aggregate_or_labels() { + let mut game = app_to_game(&app("MANUAL", true, true)).unwrap(); + for key in ["isInLibrary", "accountLinked", "favorited"] { + game[key] = json!(true); + } + game["paymentModels"] = json!([{"__typename":"FreeToPlayPaymentModel"}]); + game["variants"][1]["subscription"] = json!("GAME_PASS"); + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + LaunchStatus::Ready + ); + assert_eq!( + decision(&game, "parent-app", "456", &Value::Null).status, + LaunchStatus::OwnershipRequired + ); + for id in ["789", "0", "2147483648", "parent-app"] { + assert_eq!( + decision(&game, "parent-app", id, &Value::Null).status, + LaunchStatus::MetadataUnconfirmed + ); + } + assert_eq!( + decision(&game, "other-app", "123", &Value::Null).status, + LaunchStatus::MetadataUnconfirmed + ); +} + +#[test] +fn library_status_and_missing_readiness_are_explicit() { + for (status, expected) in [ + ("MANUAL", LaunchStatus::Ready), + ("PLATFORM_SYNC", LaunchStatus::Ready), + ("NOT_OWNED", LaunchStatus::OwnershipRequired), + ("IN_LIBRARY", LaunchStatus::MetadataUnconfirmed), + ("UNKNOWN", LaunchStatus::MetadataUnconfirmed), + ] { + let mut game = app_to_game(&app(status, true, false)).unwrap(); + for play in [Value::Null, json!("UNKNOWN"), json!("PLAYABLE")] { + game["variants"][0]["playStatus"] = play; + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + expected + ); + } + game["variants"][0]["playStatus"] = json!("NOT_PLAYABLE"); + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + LaunchStatus::Unavailable + ); + } + for key in ["libraryStatus", "gfnStatus", "librarySelected"] { + let mut game = app_to_game(&app("MANUAL", true, false)).unwrap(); + game["variants"][0][key] = Value::Null; + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + LaunchStatus::MetadataUnconfirmed, + "{key}" + ); + } + let game = app_to_game(&app("MANUAL", false, false)).unwrap(); + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + LaunchStatus::SelectionRequired + ); +} + +#[test] +fn patch_maintenance_membership_and_missing_app_metadata_block_launch() { + let baseline = app_to_game(&app("MANUAL", true, false)).unwrap(); + for (status, expected) in [ + ("PATCHING", LaunchStatus::Patching), + ("SERVER_MAINTENANCE", LaunchStatus::Maintenance), + ("UNAVAILABLE", LaunchStatus::Unavailable), + ("UNKNOWN", LaunchStatus::MetadataUnconfirmed), + ] { + let mut game = baseline.clone(); + game["variants"][0]["gfnStatus"] = json!(status); + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + expected + ); + } + for (state, expected) in [ + (Value::Null, LaunchStatus::MetadataUnconfirmed), + (json!("UNKNOWN"), LaunchStatus::MetadataUnconfirmed), + ( + json!("UNPLAYABLE_DUE_TO_UPGRADE"), + LaunchStatus::SubscriptionRequired, + ), + ( + json!("UNPLAYABLE_DUE_TO_TIME_CAPPED_LIMIT"), + LaunchStatus::SubscriptionRequired, + ), + ] { + let mut game = baseline.clone(); + game["playabilityState"] = state; + assert_eq!( + decision(&game, "parent-app", "123", &Value::Null).status, + expected + ); + } + let mut game = baseline; + game["membershipTierLabel"] = json!("Performance"); + for (subscription, expected) in [ + (Value::Null, LaunchStatus::MetadataUnconfirmed), + ( + json!({"membershipTier":"FREE"}), + LaunchStatus::SubscriptionRequired, + ), + ( + json!({"membershipTier":"ULTIMATE","isGamePlayAllowed":true}), + LaunchStatus::Ready, + ), + ( + json!({"membershipTier":"ULTIMATE","isGamePlayAllowed":false}), + LaunchStatus::SubscriptionRequired, + ), + ] { + assert_eq!( + decision(&game, "parent-app", "123", &subscription).status, + expected + ); + } +} + +#[test] +fn every_mutation_uses_exact_wire_identity_and_fresh_reconciliation() { + for (method, mutation, desired) in [ + ( + "catalog.favorites.add", + Mutation::AddFavorite, + app("MANUAL", true, true), + ), + ( + "catalog.favorites.remove", + Mutation::RemoveFavorite, + app("MANUAL", true, false), + ), + ( + "catalog.ownership.add", + Mutation::AddOwned, + app("MANUAL", true, false), + ), + ( + "catalog.ownership.remove", + Mutation::RemoveOwned, + app("NOT_OWNED", false, false), + ), + ( + "catalog.ownership.select", + Mutation::SelectOwned, + app("MANUAL", true, false), + ), + ] { + let root = mutation.root(); + let (url, worker) = mock_requests( + vec![ + vpc(), + read(app("MANUAL", false, false)), + (200, json!({"data":{root:{"app":{"id":"parent-app"}}}})), + read(desired), + ], + move |index, request| { + if index != 2 { + return; + } + let body: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + let query = body["query"].as_str().unwrap(); + assert!(query.starts_with("mutation ")); + assert!(query.contains(&format!("{root}(language: $locale,"))); + assert_eq!(body["variables"]["locale"], "en_US"); + assert_eq!( + body["variables"][if mutation.favorite() { + "appId" + } else { + "cmsId" + }], + if mutation.favorite() { + "parent-app" + } else { + "123" + } + ); + assert_eq!(body["variables"].as_object().unwrap().len(), 2); + }, + ); + let (service, path) = service(&url); + let result = service + .catalog_mutate( + method, + &scoped( + json!({"appId":"parent-app","variantId":"123","confirmedExistingLicense":true}), + ), + &json!({}), + ) + .unwrap(); + assert_eq!(result["outcome"], "acknowledged"); + assert_eq!(result["reconciliation"], "confirmed"); + assert_eq!(result["catalogRevision"], 2); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn ambiguous_or_invalid_acknowledgements_are_never_replayed_or_mislabeled() { + for (status, payload) in [ + (401, json!({})), + (404, json!({})), + (503, json!({})), + ( + 200, + json!({"data":{"addFavoriteApp":{"app":{"id":"other-app"}}}}), + ), + (200, json!({"data":{"addFavoriteApp":null}})), + ( + 200, + json!({"data":{"addFavoriteApp":{"app":{"id":"parent-app"}}},"errors":{}}), + ), + ( + 200, + json!({"errors":[{"message":"not public","extensions":{"code":"DENIED"},"path":["addFavoriteApp"]}]}), + ), + ] { + let (url, worker) = mock_requests( + vec![ + vpc(), + read(app("MANUAL", true, false)), + (status, payload), + read(app("MANUAL", true, true)), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .catalog_mutate( + "catalog.favorites.add", + &scoped(json!({"appId":"parent-app"})), + &json!({}), + ) + .unwrap(); + assert_eq!(result["outcome"], "unconfirmed"); + assert_eq!(result["reconciliation"], "confirmed"); + assert_eq!(result["error"]["httpStatus"], status); + assert!(result["error"]["code"].is_string()); + assert!(!result.to_string().contains("not public")); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn acknowledgement_without_convergence_or_failed_read_remains_unconfirmed() { + for last in [read(app("MANUAL", true, false)), (503, json!({}))] { + let (url, worker) = mock_requests( + vec![ + vpc(), + read(app("MANUAL", true, false)), + ( + 200, + json!({"data":{"addFavoriteApp":{"app":{"id":"parent-app"}}}}), + ), + last, + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .catalog_mutate( + "catalog.favorites.add", + &scoped(json!({"appId":"parent-app"})), + &json!({}), + ) + .unwrap(); + assert_eq!(result["outcome"], "acknowledged"); + assert_eq!(result["reconciliation"], "unconfirmed"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn ownership_requires_explicit_confirmation_and_parent_variant_match() { + let (url, worker) = mock_requests(vec![vpc(), read(app("MANUAL", true, false))], |_, _| {}); + let (service, path) = service(&url); + assert!( + service + .catalog_mutate( + "catalog.ownership.add", + &scoped(json!({"appId":"parent-app","variantId":"123"})), + &json!({}) + ) + .is_err() + ); + assert!( + service + .catalog_mutate( + "catalog.ownership.add", + &scoped( + json!({"appId":"parent-app","variantId":"999","confirmedExistingLicense":true}) + ), + &json!({}) + ) + .is_err() + ); + assert_eq!( + service + .catalog_revision + .load(std::sync::atomic::Ordering::Acquire), + 0 + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn favorites_preserve_more_than_twenty_four_and_unknown_coverage() { + let items: Vec<_> = (0..75) + .map(|index| { + let mut game = app("MANUAL", true, true); + game["id"] = json!(format!("parent-{index}")); + json!({"__typename":"GameItem","app":game}) + }) + .collect(); + let (url, worker) = mock_requests( + vec![ + vpc(), + ( + 200, + json!({"data":{"panels":[{"id":"favorites","name":"FAVORITES","sections":[{"id":"section","seeMoreInfo":{"filterIds":["favorites"]},"items":items}]}]}}), + ), + ], + |index, request| { + if index == 1 { + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!(payload["variables"]["panelNames"], json!(["FAVORITES"])); + assert!(payload["variables"].get("cursor").is_none()); + assert!(payload["query"].as_str().unwrap().contains("seeMoreInfo")); + } + }, + ); + let (service, path) = service(&url); + let result = service.catalog_favorites(&json!({})).unwrap(); + assert_eq!(result["games"].as_array().unwrap().len(), 75); + assert_eq!(result["coverage"], "unknown"); + assert_eq!(result["complete"], false); + assert_eq!( + result["sections"][0]["seeMoreInfo"]["filterIds"], + json!(["favorites"]) + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn fresh_create_rejects_unowned_variant_before_allocation() { + let (url, worker) = mock_requests(vec![vpc(), read(app("MANUAL", true, false))], |_, _| {}); + let (service, path) = service(&url); + let result = service.create_session( + &scoped(json!({"catalogAppId":"parent-app","variantId":"456","appId":"456"})), + &json!({}), + |settings| Ok(settings.clone()), + ); + assert_eq!(result.unwrap_err().code, "launch_not_ready"); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!(service.cloudmatch.admit_create().is_ok()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn create_rejects_mismatched_launch_identity_without_network() { + let (service, path) = service("http://127.0.0.1:1"); + assert!( + service + .create_session( + &json!({"catalogAppId":"parent-app","variantId":"456","appId":"123"}), + &json!({}), + |settings| Ok(settings.clone()) + ) + .is_err() + ); + assert!(service.cloudmatch.admit_create().is_ok()); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn old_account_mutations_are_rejected_before_network() { + let (service, path) = service("http://127.0.0.1:1"); + let mut params = scoped(json!({"appId":"parent-app"})); + params["scope"]["generation"] = json!(99); + assert_eq!( + service + .catalog_mutate("catalog.favorites.add", ¶ms, &json!({})) + .unwrap_err() + .code, + "stale_account" + ); + assert_eq!( + service + .catalog_revision + .load(std::sync::atomic::Ordering::Acquire), + 0 + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn mutation_and_create_admission_serialize_only_the_same_account_app() { + let (service, path) = service("http://127.0.0.1:1"); + let auth = auth_fixture("fixture-account"); + let permit = service.admit_catalog_action(&auth, "parent-app").unwrap(); + assert!(service.admit_catalog_action(&auth, "parent-app").is_err()); + assert!(service.admit_catalog_action(&auth, "different-app").is_ok()); + drop(permit); + assert!(service.admit_catalog_action(&auth, "parent-app").is_ok()); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn account_change_during_mutation_cannot_reconcile_under_the_new_account() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = mock_requests( + vec![ + vpc(), + read(app("MANUAL", true, false)), + ( + 200, + json!({"data":{"addFavoriteApp":{"app":{"id":"parent-app"}}}}), + ), + ], + move |index, _| { + if index == 2 { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ); + let (service, path) = service(&url); + std::thread::scope(|threads| { + let mutation = threads.spawn(|| { + service.catalog_mutate( + "catalog.favorites.add", + &scoped(json!({"appId":"parent-app"})), + &json!({}), + ) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + service.state.lock().unwrap().generation += 1; + release_tx.send(()).unwrap(); + assert_eq!(mutation.join().unwrap().unwrap_err().code, "stale_account"); + }); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +fn held_mutation_remains_admitted_after_generation_change(switch_accounts: bool) { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = mock_requests( + vec![ + vpc(), + read(app("MANUAL", true, false)), + ( + 200, + json!({"data":{"addFavoriteApp":{"app":{"id":"parent-app"}}}}), + ), + ], + move |index, request| { + if index == 2 { + assert!(request.contains("mutation AddFavoriteApp")); + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ); + let (service, path) = service(&url); + let owner = auth_fixture("fixture-account"); + std::thread::scope(|threads| { + let mutation = threads.spawn(|| { + service.catalog_mutate( + "catalog.favorites.add", + &scoped(json!({"appId":"parent-app"})), + &json!({}), + ) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + if switch_accounts { + let other = auth_fixture("other-account"); + { + let mut state = service.state.lock().unwrap(); + state.session = Some(other.clone()); + state.generation += 1; + } + assert!(service.admit_catalog_action(&other, "parent-app").is_ok()); + let mut state = service.state.lock().unwrap(); + state.session = Some(owner.clone()); + state.generation += 1; + } else { + service.clear_cache(); + let mut state = service.state.lock().unwrap(); + state.providers = vec![owner.provider.clone()]; + state.providers_expires = Some(Instant::now() + Duration::from_secs(60)); + } + let generation = service.auth_generation(); + let blocked = service + .admit_catalog_action(&owner, "parent-app") + .err() + .map(|error| error.code); + assert!(service.admit_catalog_action(&owner, "another-app").is_ok()); + assert!( + service + .admit_catalog_action(&auth_fixture("another-account"), "parent-app") + .is_ok() + ); + let mut partner = owner.clone(); + partner.provider.idp_id = "another-provider".into(); + assert!(service.admit_catalog_action(&partner, "parent-app").is_ok()); + if blocked.is_some() { + let scope = scoped_result(json!({}), &owner, generation)["scope"].clone(); + assert_eq!( + service + .catalog_mutate( + "catalog.favorites.remove", + &json!({"appId":"parent-app","scope":scope}), + &json!({}) + ) + .unwrap_err() + .code, + "catalog_mutation_busy" + ); + assert_eq!(service.create_session(&json!({"catalogAppId":"parent-app","variantId":"123","appId":"123","scope":scope}), &json!({}), |settings| Ok(settings.clone())).unwrap_err().code, "catalog_mutation_busy"); + } + release_tx.send(()).unwrap(); + assert_eq!(mutation.join().unwrap().unwrap_err().code, "stale_account"); + assert_eq!(blocked, Some("catalog_mutation_busy")); + assert!(service.admit_catalog_action(&owner, "parent-app").is_ok()); + }); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn held_mutation_blocks_same_app_after_account_switch_back() { + held_mutation_remains_admitted_after_generation_change(true); +} + +#[test] +fn held_mutation_blocks_same_app_after_cache_generation_bump() { + held_mutation_remains_admitted_after_generation_change(false); +} + +#[test] +fn mutation_preflight_401_renews_before_exactly_one_send() { + for mutation_status in [200, 401] { + let old_id = jwt("fixture-account", now_ms() + 3_600_000); + let renewed_id = jwt("fixture-account", now_ms() + 7_200_000); + let requests = std::sync::Arc::new(Mutex::new(Vec::new())); + let observed = requests.clone(); + let (url, worker) = mock_requests( + vec![ + vpc(), + (401, json!({})), + ( + 200, + json!({"access_token":"renewed-access","id_token":renewed_id,"expires_in":7200}), + ), + vpc(), + read(app("MANUAL", true, false)), + ( + mutation_status, + json!({"data":{"addFavoriteApp":{"app":{"id":"parent-app"}}}}), + ), + read(app("MANUAL", true, true)), + ], + move |_, request| observed.lock().unwrap().push(request.to_owned()), + ); + let (service, path) = service(&url); + { + let mut state = service.state.lock().unwrap(); + let session = state.session.as_mut().unwrap(); + session.tokens.id_token = Some(old_id.clone()); + session.tokens.id_token_expires_at = Some(now_ms() + 3_600_000); + } + let result = service + .catalog_mutate( + "catalog.favorites.add", + &scoped(json!({"appId":"parent-app"})), + &json!({}), + ) + .unwrap(); + worker.join().unwrap(); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 7); + assert!(requests[0].contains(&format!("GFNJWT {old_id}"))); + assert!(requests[1].contains(&format!("GFNJWT {old_id}"))); + assert!(requests[2].starts_with("POST / ")); + let refresh: HashMap<_, _> = + url::form_urlencoded::parse(requests[2].split("\r\n\r\n").nth(1).unwrap().as_bytes()) + .into_owned() + .collect(); + assert_eq!( + refresh["grant_type"], + "urn:ietf:params:oauth:grant-type:client_token" + ); + assert_eq!(refresh["client_id"], "test-client-id"); + assert!(requests[4].contains(&format!("GFNJWT {renewed_id}"))); + let sends: Vec<_> = requests + .iter() + .filter(|request| request.contains("mutation AddFavoriteApp")) + .collect(); + assert_eq!(sends.len(), 1); + assert!( + sends[0].contains(&format!("GFNJWT {renewed_id}")), + "mutation retained the preflight credential after renewal" + ); + assert!(!sends[0].contains(&format!("GFNJWT {old_id}"))); + assert!(requests[6].contains(&format!("GFNJWT {renewed_id}"))); + assert_eq!(result["scope"]["generation"], 0); + assert_eq!( + result["outcome"], + if mutation_status == 200 { + "acknowledged" + } else { + "unconfirmed" + } + ); + assert_eq!(result["reconciliation"], "confirmed"); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn membership_requirements_preserve_all_existing_tier_cases_in_the_shared_policy() { + for (required, membership, expected) in [ + ("Premium", "FREE", LaunchStatus::SubscriptionRequired), + ( + "Performance", + " Free Tier ", + LaunchStatus::SubscriptionRequired, + ), + ("Ultimate", "free-tier", LaunchStatus::SubscriptionRequired), + ("Premium", "PERFORMANCE", LaunchStatus::Ready), + ("Premium", "Priority", LaunchStatus::Ready), + ("", "FREE", LaunchStatus::Ready), + (" ", "", LaunchStatus::Ready), + (" FREE ", "FREE", LaunchStatus::Ready), + ("Free Tier", "", LaunchStatus::Ready), + ("free-tier", "", LaunchStatus::Ready), + ("Premium", "", LaunchStatus::MetadataUnconfirmed), + ("Premium", " ", LaunchStatus::MetadataUnconfirmed), + ] { + let mut game = app_to_game(&app("MANUAL", true, false)).unwrap(); + game["membershipTierLabel"] = json!(required); + assert_eq!( + decision( + &game, + "parent-app", + "123", + &json!({"membershipTier":membership}) + ) + .status, + expected, + "{required}/{membership}" + ); + } +} + +fn premium_service(url: &str) -> (GfnService, PathBuf) { + let (mut service, path) = service(url); + service.endpoints.account_linking = format!("{url}/graphql"); + service.endpoints.subscription = format!("{url}/subscription"); + (service, path) +} + +fn premium_app(playability: &str) -> Value { + let mut game = app("MANUAL", true, false); + game["gfn"] = json!({"playabilityState":playability,"minimumMembershipTierLabel":"Premium"}); + game +} + +fn store_access() -> Vec<(u16, Value)> { + vec![ + ( + 200, + json!({"data":{"appStoreDefinitions":[{"store":"STEAM","label":"Steam","features":[], + "accountLinkingMetadata":{"isRequired":false,"isSupported":true,"supportedVariantIds":[]}}]}}), + ), + (200, json!({"data":{"genreDefinitions":[]}})), + (200, json!({"data":{"subscriptionDefinitions":[]}})), + ( + 200, + json!({"data":{"userAccount":{"storesData":[{"store":"STEAM","isConnected":true,"status":"connected"}],"subscriptions":[]}}}), + ), + ] +} + +#[test] +fn account_playability_decides_premium_launches_without_the_membership_service() { + let mut responses = vec![vpc(), read(premium_app("UNPLAYABLE_DUE_TO_UPGRADE"))]; + responses.extend(store_access()); + let (url, worker) = mock_requests(responses, |_, request| { + assert!( + !request.starts_with("GET /subscription"), + "membership was requested" + ); + }); + let (service, path) = premium_service(&url); + let result = service + .catalog_launch_inspect(&json!({"appId":"parent-app","variantId":"123"}), &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "subscription_required"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn playable_premium_launches_still_require_a_confirmed_membership() { + for (membership, expected) in [ + ((404, json!({})), Err("upstream_error")), + ((500, json!({})), Err("upstream_error")), + ( + (200, json!({"membershipTier":"FREE"})), + Ok("subscription_required"), + ), + ( + ( + 200, + json!({"membershipTier":"ULTIMATE","currentSubscriptionState":{"isGamePlayAllowed":true}}), + ), + Ok("ready"), + ), + ] { + let status = membership.0; + let mut responses = vec![vpc(), read(premium_app("PLAYABLE"))]; + responses.extend(store_access()); + responses.push(membership); + let (url, worker) = mock_requests(responses, |index, request| { + assert_eq!(index == 6, request.starts_with("GET /subscription?")); + }); + let (service, path) = premium_service(&url); + let result = service + .catalog_launch_inspect(&json!({"appId":"parent-app","variantId":"123"}), &json!({})); + match expected { + Ok(decision) => assert_eq!(result.unwrap()["decision"]["status"], decision), + Err(code) => { + let error = result.unwrap_err(); + assert_eq!(error.code, code); + assert!( + error.message.contains(&status.to_string()), + "{}", + error.message + ); + } + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/catalog_tests.rs b/external/opennow/native/opennow-core/src/gfn/catalog_tests.rs new file mode 100644 index 000000000..98e71fd24 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/catalog_tests.rs @@ -0,0 +1,396 @@ +use super::tests::{auth_fixture, mock_requests, test_service}; +use super::*; + +fn service(url: &str) -> (GfnService, PathBuf) { + let (mut service, path) = test_service(url); + service.endpoints.graphql = format!("{url}/graphql"); + service.endpoints.public_graphql = format!("{url}/public"); + service.endpoints.server_info = Some(format!("{url}/vpc")); + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("fixture-account")); + state.restore_attempted = true; + drop(state); + (service, path) +} + +fn app(id: usize) -> Value { + json!({"id":format!("app-{id}"),"title":format!("Game {id}"),"library":{"favorited":true}, + "gfn":{"playabilityState":"PLAYABLE","catalogSkuStrings":{"SKU_BASED_TAG":["Fixture"]}}, + "variants":[{"id":"123","appStore":"STEAM","paymentModels":[{"__typename":"Paid"}], + "gfn":{"status":"PATCHING","library":{"status":"MANUAL","selected":true,"playStatus":"NOT_PLAYABLE"}, + "stateDetails":{"__typename":"VariantGfnAutoPatchingMetadata","historicalEtaMins":12.5}}}]}) +} + +#[test] +fn library_traverses_thirteen_complete_pages_without_a_thousand_game_ceiling() { + let mut responses = vec![(200, json!({"requestStatus":{"serverId":"fixture-vpc"}}))]; + for page in 0..13 { + responses.push((200,json!({"data":{"apps":{"items":(page*100..page*100+100).map(app).collect::>(), + "pageInfo":{"hasNextPage":page < 12,"endCursor":format!("cursor-{}",page+1),"totalCount":1300}}}}))); + } + let (url, worker) = mock_requests(responses, |index, request| { + if index == 0 { + assert!(request.starts_with("GET /vpc ")); + return; + } + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!(payload["variables"]["fetchCount"], 100); + assert_eq!( + payload["variables"]["cursor"], + if index == 1 { + String::new() + } else { + format!("cursor-{}", index - 1) + } + ); + assert!(payload["query"].as_str().unwrap().contains("playStatus")); + }); + let (service, path) = service(&url); + let mut cursor = String::new(); + let mut context = Value::Null; + let mut count = 0; + for _ in 0..13 { + let result = service + .library_catalog( + &json!({"limit":100,"cursor":cursor,"catalogRevision":0,"catalogContext":context,"traversalId":"fixture"}), + &json!({}), + ) + .unwrap(); + count += result["games"].as_array().unwrap().len(); + cursor = result["nextCursor"].as_str().unwrap().into(); + context = result["catalogContext"].clone(); + assert_eq!(result["traversalId"], "fixture"); + } + assert_eq!(count, 1300); + assert!(cursor.is_empty()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn exact_game_lookup_preserves_selected_variant_readiness_and_rejects_wrong_identity() { + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})), + (200, json!({"data":{"apps":{"items":[app(1)]}}})), + (200, json!({"data":{"apps":{"items":[app(2)]}}})), + ], + |index, request| { + if index == 0 { + return; + } + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + let query = payload["query"].as_str().unwrap(); + assert_eq!(query.matches('{').count(), query.matches('}').count()); + assert!(query.contains("appIds:$ids")); + assert!(query.contains("supportedLanguages")); + assert_eq!(payload["variables"]["ids"], json!(["app-1"])); + }, + ); + let (service, path) = service(&url); + let result = service + .catalog_game(&json!({"appId":"app-1"}), &json!({})) + .unwrap(); + assert_eq!(result["game"]["favorited"], true); + assert_eq!(result["game"]["variants"][0]["playStatus"], "NOT_PLAYABLE"); + assert_eq!( + result["game"]["variants"][0]["stateDetails"]["historicalEtaMins"], + 12.5 + ); + assert_eq!( + service + .catalog_game(&json!({"appId":"app-1"}), &json!({})) + .unwrap_err() + .code, + "catalog_game_not_found" + ); + assert!( + service + .catalog_game(&json!({"appId":"app-1","variantId":"123"}), &json!({})) + .is_err() + ); + assert!( + service + .catalog_game(&json!({"variantId":"2147483648"}), &json!({})) + .is_err() + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn targeted_game_lookups_omit_campaign_dependencies_and_keep_launch_metadata() { + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})), + (200, json!({"data":{"apps":{"items":[app(1)]}}})), + (200, json!({"data":{"apps":{"items":[app(1)]}}})), + ], + |index, request| { + if index == 0 { + return; + } + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + let query = payload["query"].as_str().unwrap(); + assert!(!query.contains("itemMetadata")); + assert!(!query.contains("campaignIds")); + for field in [ + "favorited", + "playStatus", + "selected", + "installed", + "subscription", + "stateDetails", + "supportedLanguages", + "paymentModels", + "minimumMembershipTierLabel", + "playabilityState", + ] { + assert!(query.contains(field), "missing {field}"); + } + assert_eq!(payload["variables"]["vpcId"], "fixture-vpc"); + assert_eq!(payload["variables"]["locale"], "en_US"); + if index == 1 { + assert!(query.contains("appIds:$ids")); + assert_eq!(payload["variables"]["ids"], json!(["app-1"])); + } else { + assert!(query.contains("variantIds:$ids")); + assert_eq!(payload["variables"]["ids"], json!([123])); + } + }, + ); + let (service, path) = service(&url); + for params in [json!({"appId":"app-1"}), json!({"variantId":"123"})] { + let result = service.catalog_game(¶ms, &json!({})).unwrap(); + assert_eq!(result["game"]["id"], "app-1"); + assert_eq!(result["game"]["variants"][0]["libraryStatus"], "MANUAL"); + assert_eq!(result["game"]["variants"][0]["playStatus"], "NOT_PLAYABLE"); + assert_eq!(result["game"]["variants"][0]["gfnStatus"], "PATCHING"); + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn overall_languages_are_anonymous_variable_free_cached_and_truthfully_stale() { + let (url, worker) = mock_requests( + vec![ + ( + 200, + json!({"data":{"overallGfnSupportedLanguages":[{"language":"en_US"},{"language":"es_419"},{"language":"zh_Hant_TW"},{"language":"en_US"}]}}), + ), + ( + 200, + json!({"data":{"overallGfnSupportedLanguages":[]},"errors":[{"message":"fixture failure"}]}), + ), + ], + |_, request| { + assert!(request.starts_with("POST /public ")); + assert!(!request.to_ascii_lowercase().contains("authorization:")); + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!( + payload, + json!({"query":"{ overallGfnSupportedLanguages { language } }"}) + ); + }, + ); + let (service, path) = service(&url); + service.state.lock().unwrap().session = None; + let result = service.catalog_languages(&json!({}), &json!({})).unwrap(); + assert_eq!( + result["languages"], + json!(["en_US", "es_419", "zh_Hant_TW"]) + ); + assert_eq!(result["status"], "success"); + assert_eq!( + result["expiresAt"].as_u64().unwrap() - result["fetchedAt"].as_u64().unwrap(), + 14 * 24 * 60 * 60 * 1000 + ); + assert_eq!( + service.catalog_languages(&json!({}), &json!({})).unwrap()["cacheHit"], + true + ); + let stale = service + .catalog_languages(&json!({"refresh":true}), &json!({})) + .unwrap(); + assert_eq!(stale["status"], "stale"); + assert_eq!(stale["languages"], result["languages"]); + assert_eq!(stale["error"]["code"], "graphql_error"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn invalid_language_metadata_is_not_cached_as_success() { + for value in [ + Value::Null, + json!([]), + json!([{"language":""}]), + json!([{"language":"auto"}]), + json!([{"language":"system"}]), + json!([{"language":"en__US"}]), + json!([{"language":"en_US/../../"}]), + json!([{"language":"x".repeat(65)}]), + json!([{"language":42}]), + ] { + let (url, worker) = mock_requests( + vec![(200, json!({"data":{"overallGfnSupportedLanguages":value}}))], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service.catalog_languages(&json!({}), &json!({})).unwrap(); + assert_eq!(result["status"], "error"); + assert_eq!(result["languages"], json!([])); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn revised_catalog_rejects_continuation_before_network_dispatch() { + let (service, path) = service("http://127.0.0.1:1"); + service.invalidate_catalog().unwrap(); + assert_eq!( + service + .library_catalog(&json!({"cursor":"old","catalogRevision":0}), &json!({})) + .unwrap_err() + .code, + "catalog_changed" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn local_store_refresh_revalidates_a_warm_disk_chain_and_reports_coverage() { + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})), + ( + 200, + json!({"data":{"apps":{"items":[app(1)],"pageInfo":{"hasNextPage":true,"endCursor":"more","totalCount":50}}}}), + ), + ( + 200, + json!({"data":{"apps":{"items":[app(2)],"pageInfo":{"hasNextPage":false,"totalCount":1}}}}), + ), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let first = service.store_local_catalog(&json!({}), &json!({})).unwrap(); + assert_eq!(first["games"][0]["id"], "app-1"); + assert_eq!(first["cacheComplete"], false); + assert_eq!(first["upstreamCoverage"], "partial"); + assert_eq!(first["localHasNextPage"], false); + let refreshed = service + .store_local_catalog(&json!({"refresh":true}), &json!({})) + .unwrap(); + assert_eq!(refreshed["games"][0]["id"], "app-2"); + assert_eq!(refreshed["cacheComplete"], true); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn local_store_bootstrap_uses_scope_refreshed_during_nested_read() { + for params in [json!({}), json!({"refresh":true})] { + let renewed_id = super::tests::jwt("fixture-account", now_ms() + 7_200_000); + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"old-vpc"}})), + (401, json!({})), + ( + 200, + json!({"access_token":"renewed-access","id_token":renewed_id,"expires_in":7200}), + ), + (200, json!({"requestStatus":{"serverId":"new-vpc"}})), + ( + 200, + json!({"data":{"apps":{"items":[app(1)],"pageInfo":{"hasNextPage":false,"totalCount":1}}}}), + ), + ], + |index, request| { + if index == 1 { + assert!(request.contains("old-vpc")); + } + if index == 4 { + assert!(request.contains("new-vpc")); + } + }, + ); + let (service, path) = service(&url); + let result = service.store_local_catalog(¶ms, &json!({})); + worker.join().unwrap(); + assert!( + result.is_ok(), + "nested renewal left the local index in the old catalog scope: {:?}", + result.err() + ); + let result = result.unwrap(); + assert_eq!(result["games"][0]["id"], "app-1"); + assert_eq!(result["scope"]["generation"], 0); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn optional_definition_failure_does_not_hide_server_disabled_store_actions() { + let (url, worker) = mock_requests( + vec![ + ( + 200, + json!({"data":{"appStoreDefinitions":[{"store":"STEAM","label":"Steam","features":[ + {"__typename":"AccountGamesSyncing","supported":false}]}]}}), + ), + (503, json!({})), + ( + 200, + json!({"data":{"subscriptionDefinitions":[{"subscription":"GAMEPASS","label":"Game Pass","primaryStore":"XBOX"}]}}), + ), + ( + 200, + json!({"data":{"userAccount":{"subscriptions":[{"id":"GAMEPASS"}],"storesData":[]}}}), + ), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service.account_connections(&json!({})).unwrap(); + assert_eq!(result["accounts"][0]["supportsSync"], false); + assert_eq!(result["accounts"][0]["capabilitySource"], "success"); + assert_eq!(result["definitions"]["genres"]["status"], "error"); + assert_eq!(result["subscriptions"][0]["id"], "GAMEPASS"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn delayed_anonymous_metadata_is_rejected_after_a_context_change() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = mock_requests( + vec![( + 200, + json!({"data":{"overallGfnSupportedLanguages":[{"language":"en_US"}]}}), + )], + move |_, _| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }, + ); + let (service, path) = service(&url); + std::thread::scope(|threads| { + let pending = threads.spawn(|| service.catalog_languages(&json!({}), &json!({}))); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + service.state.lock().unwrap().generation += 1; + release_tx.send(()).unwrap(); + assert_eq!(pending.join().unwrap().unwrap_err().code, "stale_account"); + }); + worker.join().unwrap(); + assert!(!path.join("store-cache-v2").exists()); + std::fs::remove_dir_all(path).unwrap(); +} diff --git a/external/opennow/native/opennow-core/src/gfn/dlss_support.rs b/external/opennow/native/opennow-core/src/gfn/dlss_support.rs new file mode 100644 index 000000000..bf1f1cb36 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/dlss_support.rs @@ -0,0 +1,763 @@ +//! Per-game DLSS support derived from NVIDIA's published DLSS games list. +//! +//! The GeForce NOW catalog has no per-game DLSS key. NVIDIA's RTX games page loads its +//! table from a public JSON list, so the core matches catalog titles against that list. +//! The list only decorates catalog responses: when it is missing, oversized, or invalid, +//! every game carries `dlss: null` and the next catalog request retries. + +use super::{ServiceError, now_ms}; +use crate::proxy::{client_for_settings, config_from_settings}; +use crate::store_cache::{CachePolicy, StoreCache}; +use icu_normalizer::DecomposingNormalizerBorrowed; +use reqwest::blocking::Client; +use reqwest::header::ACCEPT; +use serde::Serialize; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::io::Read; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +pub const LIST_URL: &str = "https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/news/nvidia-rtx-games-engines-apps/dlss-rt-games-apps-overrides.json"; +const SOURCE: &str = "nvidia-rtx-games-list"; +const MAX_LIST_BYTES: usize = 4 * 1024 * 1024; +const MAX_ROWS: usize = 5_000; +const MAX_NAME_BYTES: usize = 512; +const MAX_CELL_BYTES: usize = 64; +const TTL_MS: u64 = 24 * 60 * 60 * 1000; +const RETRY_MS: u64 = 60 * 1000; +const FIRST_LOAD_WAIT: Duration = Duration::from_millis(1500); +const FETCH_TIMEOUT: Duration = Duration::from_secs(15); +const MAX_ANNOTATION_DEPTH: usize = 10; + +/// Column order of a cached row; the cache stores only these cells. +const COLUMNS: [&str; 5] = [ + "dlss super resolution", + "dlss frame generation", + "dlss multi frame generation", + "dlss ray reconstruction", + "dlaa", +]; + +/// Trailing title words that name an edition of the same game. "Remastered" is +/// deliberately absent: remasters are listed separately and differ in support. +const EDITION_SUFFIXES: [&[&str]; 11] = [ + &["game", "of", "year"], + &["goty"], + &["definitive"], + &["enhanced"], + &["complete"], + &["deluxe"], + &["standard"], + &["ultimate"], + &["directors", "cut"], + &["director", "s", "cut"], + &["anniversary"], +]; + +type Row = (String, [String; 5]); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Support { + Native, + Override, +} + +impl Support { + fn from_cell(cell: &str) -> Option { + if cell.eq_ignore_ascii_case("yes") { + Some(Self::Native) + } else if cell.starts_with("NV") { + Some(Self::Override) + } else { + None + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MultiFrameGeneration { + max_multiplier: u8, +} + +impl MultiFrameGeneration { + fn from_cell(cell: &str) -> Option { + cell.split(|character: char| !character.is_ascii_alphanumeric()) + .find_map(|token| token.strip_suffix(['X', 'x'])?.parse::().ok()) + .filter(|multiplier| (2..=16).contains(multiplier)) + .map(|max_multiplier| Self { max_multiplier }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DlssSupport { + super_resolution: Option, + frame_generation: Option, + multi_frame_generation: Option, + ray_reconstruction: Option, + dlaa: Option, +} + +impl DlssSupport { + /// None when the row lists no DLSS technology (for example ray tracing only). + fn from_cells([sr, fg, mfg, rr, dlaa]: &[String; 5]) -> Option { + let support = Self { + super_resolution: Support::from_cell(sr), + frame_generation: Support::from_cell(fg), + multi_frame_generation: MultiFrameGeneration::from_cell(mfg), + ray_reconstruction: Support::from_cell(rr), + dlaa: Support::from_cell(dlaa), + }; + let listed = support.super_resolution.is_some() + || support.frame_generation.is_some() + || support.multi_frame_generation.is_some() + || support.ray_reconstruction.is_some() + || support.dlaa.is_some(); + listed.then_some(support) + } + + fn to_json(self) -> Value { + let mut value = serde_json::to_value(self).expect("DLSS support serializes"); + value["source"] = json!(SOURCE); + value + } +} + +fn invalid_list(message: &str) -> ServiceError { + ServiceError { + code: "invalid_upstream_response", + message: format!("DLSS games list rejected: {message}"), + } +} + +/// Parses NVIDIA's list into bounded Game rows. Apps and malformed rows are skipped; +/// an oversized or structurally invalid document is rejected as a whole. +pub fn parse_list(bytes: &[u8]) -> Result, ServiceError> { + if bytes.len() > MAX_LIST_BYTES { + return Err(invalid_list("larger than 4 MiB")); + } + let document: Value = + serde_json::from_slice(bytes).map_err(|_| invalid_list("not valid JSON"))?; + let data = document["data"] + .as_array() + .ok_or_else(|| invalid_list("no data array"))?; + if data.len() > MAX_ROWS { + return Err(invalid_list("more than 5,000 rows")); + } + Ok(data + .iter() + .filter(|row| row["type"].as_str() == Some("Game")) + .filter_map(|row| { + let name = match &row["name"] { + Value::String(name) => name.trim().to_owned(), + Value::Number(number) => number.to_string(), + _ => return None, + }; + if name.is_empty() || name.len() > MAX_NAME_BYTES { + return None; + } + let cells = COLUMNS.map(|column| { + row[column] + .as_str() + .map(str::trim) + .filter(|cell| cell.len() <= MAX_CELL_BYTES) + .unwrap_or("") + .to_owned() + }); + Some((name, cells)) + }) + .collect()) +} + +/// Lowercase ASCII words: trademark and zero-width characters removed, NFKD folded, +/// "&" spelled "and", and every other non-alphanumeric run collapsed to one space. +pub fn normalize_title(title: &str) -> String { + let stripped: String = title + .chars() + .filter(|character| { + !matches!( + character, + '\u{2122}' + | '\u{00AE}' + | '\u{00A9}' + | '\u{200B}' + | '\u{200C}' + | '\u{200D}' + | '\u{2060}' + | '\u{FEFF}' + ) + }) + .collect(); + let ascii: String = DecomposingNormalizerBorrowed::new_nfkd() + .normalize(&stripped) + .chars() + .filter(char::is_ascii) + .collect::() + .to_ascii_lowercase() + .replace('&', " and "); + ascii + .split(|character: char| !character.is_ascii_alphanumeric()) + .filter(|word| !word.is_empty()) + .collect::>() + .join(" ") +} + +/// A normalized title without "the"/"tm" and without trailing edition words. +fn loose_title(normalized: &str) -> String { + let mut words: Vec<&str> = normalized + .split(' ') + .filter(|word| !word.is_empty() && *word != "the" && *word != "tm") + .collect(); + loop { + let before = words.len(); + if words.len() > 1 && matches!(words.last(), Some(&"edition" | &"pc")) { + words.pop(); + } + if let Some(suffix) = EDITION_SUFFIXES + .iter() + .find(|suffix| words.len() > suffix.len() && words.ends_with(suffix)) + { + words.truncate(words.len() - suffix.len()); + } + if words.len() == before { + return words.join(" "); + } + } +} + +#[derive(Default)] +pub struct DlssIndex { + exact: HashMap>, + loose: HashMap>, +} + +impl DlssIndex { + /// Every Game row is indexed, including rows without DLSS, so an exact title + /// without DLSS never falls through to a loosely matching edition that has it. + pub fn from_rows(rows: &[Row]) -> Self { + let mut index = Self::default(); + for (name, cells) in rows { + let exact = normalize_title(name); + if exact.is_empty() { + continue; + } + let support = DlssSupport::from_cells(cells); + index.loose.entry(loose_title(&exact)).or_insert(support); + index.exact.entry(exact).or_insert(support); + } + index + } + + pub fn lookup(&self, title: &str) -> Option { + let exact = normalize_title(title); + if exact.is_empty() { + return None; + } + match self.exact.get(&exact) { + Some(support) => *support, + None => self.loose.get(&loose_title(&exact)).copied().flatten(), + } + } +} + +/// Sets `dlss` on every catalog game in a response. A game is an object with a +/// string `title`, a `variants` array, and a `launchAppId` key, the shape both +/// catalog mappers produce. Games are never nested inside games. +pub fn annotate(value: &mut Value, index: Option<&DlssIndex>) { + fn visit(value: &mut Value, index: Option<&DlssIndex>, depth: usize) { + if depth > MAX_ANNOTATION_DEPTH { + return; + } + match value { + Value::Array(items) => { + for item in items { + visit(item, index, depth + 1); + } + } + Value::Object(object) => { + if object.get("variants").is_some_and(Value::is_array) + && object.contains_key("launchAppId") + && let Some(title) = object.get("title").and_then(Value::as_str) + { + match index { + Some(index) => { + let dlss = index + .lookup(title) + .map_or(Value::Null, DlssSupport::to_json); + object.insert("dlss".to_owned(), dlss); + } + None => { + object.remove("dlss"); + } + } + return; + } + for child in object.values_mut() { + visit(child, index, depth + 1); + } + } + _ => {} + } + } + visit(value, index, 0); +} + +fn fetch_list(client: &Client, url: &str) -> Result { + let response = client + .get(url) + .header(ACCEPT, "application/json") + .timeout(FETCH_TIMEOUT) + .send() + .map_err(|error| ServiceError::network("DLSS games list fetch failed", error))?; + if !response.status().is_success() { + return Err(ServiceError::response( + "DLSS games list fetch failed", + response, + )); + } + if response + .content_length() + .is_some_and(|length| length > MAX_LIST_BYTES as u64) + { + return Err(invalid_list("larger than 4 MiB")); + } + let mut bytes = Vec::new(); + response + .take(MAX_LIST_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| invalid_list("the download was interrupted"))?; + let rows = parse_list(&bytes)?; + Ok(json!({"metadataKind":"dlssGames","source":SOURCE,"rows":rows})) +} + +#[derive(Default)] +struct LoadState { + index: Option>, + expires_at: u64, + retry_at: u64, + loading: bool, + first_attempt_done: bool, + first_wait_claimed: bool, +} + +#[derive(Default)] +struct Shared { + state: Mutex, + settled: Condvar, +} + +/// Owns the in-memory list and its background refresh. Catalog requests never wait on +/// a download except the process's first attempt, and only for `FIRST_LOAD_WAIT`. +pub struct DlssCatalog { + client: Client, + url: String, + cache: Arc, + shared: Arc, +} + +impl DlssCatalog { + pub fn new(client: Client, url: String, cache: Arc) -> Self { + Self { + client, + url, + cache, + shared: Arc::default(), + } + } + + pub fn index(&self, settings: &Value) -> Option> { + let mut state = self.shared.state.lock().expect("DLSS list poisoned"); + let now = now_ms(); + if !state.loading + && now >= state.retry_at + && (state.index.is_none() || now >= state.expires_at) + { + state.loading = self.spawn_load(settings); + if !state.loading { + state.first_attempt_done = true; + state.retry_at = now.saturating_add(RETRY_MS); + } + } + if state.index.is_none() && !state.first_attempt_done && !state.first_wait_claimed { + state.first_wait_claimed = true; + state = self + .shared + .settled + .wait_timeout_while(state, FIRST_LOAD_WAIT, |state| !state.first_attempt_done) + .expect("DLSS list poisoned") + .0; + } + state.index.clone() + } + + fn spawn_load(&self, settings: &Value) -> bool { + let (Ok(client), Ok(proxy)) = ( + client_for_settings(&self.client, settings), + config_from_settings(settings), + ) else { + return false; + }; + let scope = json!([ + "dlss-games-v1", + self.url, + proxy.map_or_else(|| "direct".to_owned(), |config| config.cache_scope) + ]); + let (url, cache, shared) = (self.url.clone(), self.cache.clone(), self.shared.clone()); + std::thread::Builder::new() + .name("dlss-games-list".into()) + .spawn(move || { + let loaded = cache + .load_with_policy( + &scope, + &json!(["dlss-games"]), + false, + CachePolicy { + ttl_ms: TTL_MS, + allow_stale: true, + }, + || fetch_list(&client, &url), + ) + .and_then(|value| { + if let Some(code) = value["error"]["code"].as_str() { + eprintln!("dlss games list: refresh failed ({code})"); + } + let rows: Vec = serde_json::from_value(value["rows"].clone()) + .map_err(|_| invalid_list("no cached rows"))?; + if rows.len() > MAX_ROWS { + return Err(invalid_list("more than 5,000 rows")); + } + Ok(( + DlssIndex::from_rows(&rows), + value["expiresAt"].as_u64().unwrap_or(0), + )) + }); + let mut state = shared.state.lock().expect("DLSS list poisoned"); + let now = now_ms(); + match loaded { + Ok((index, expires_at)) => { + state.index = Some(Arc::new(index)); + state.expires_at = expires_at; + } + Err(error) => eprintln!("dlss games list: unavailable ({})", error.code), + } + if state.expires_at <= now { + state.retry_at = now.saturating_add(RETRY_MS); + } + state.loading = false; + state.first_attempt_done = true; + shared.settled.notify_all(); + }) + .is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn list(rows: Value) -> Vec { + serde_json::to_vec(&json!({"columns":[],"footer":"","data":rows})).unwrap() + } + + fn row(name: Value, cells: [&str; 5]) -> Value { + let mut row = json!({"sno":1,"name":name,"type":"Game","ray tracing":"","ai":""}); + for (column, cell) in COLUMNS.iter().zip(cells) { + row[*column] = json!(cell); + } + row + } + + fn index(rows: Value) -> DlssIndex { + DlssIndex::from_rows(&parse_list(&list(rows)).unwrap()) + } + + #[test] + fn parses_string_and_numeric_names_and_skips_apps() { + let mut app = row(json!("Broadcast"), ["Yes", "", "", "", ""]); + app["type"] = json!("App"); + let rows = parse_list(&list(json!([ + row(json!("007 First Light"), ["NV, T", "NV, U", "NV, 6X", "NV, T", "NV, T"]), + row(json!(83), ["NV, T", "", "", "", ""]), + app, + {"name":null,"type":"Game"}, + "not a row", + ]))) + .unwrap(); + assert_eq!( + rows.iter() + .map(|(name, _)| name.as_str()) + .collect::>(), + ["007 First Light", "83"] + ); + assert_eq!(rows[0].1[2], "NV, 6X"); + } + + #[test] + fn maps_cells_to_native_override_and_multiplier() { + let index = index(json!([ + row( + json!("Cyberpunk 2077"), + ["Yes", "Yes", "NV, 6X", "NV, T", "Yes"] + ), + row( + json!("Black Myth: Wukong"), + ["NV, T", "Yes", "NV, 4X", "", ""] + ), + row(json!("Ray Traced Only"), ["", "", "", "", ""]), + ])); + assert_eq!( + index.lookup("Cyberpunk 2077").unwrap().to_json(), + json!({"superResolution":"native","frameGeneration":"native", + "multiFrameGeneration":{"maxMultiplier":6},"rayReconstruction":"override", + "dlaa":"native","source":"nvidia-rtx-games-list"}) + ); + let wukong = index.lookup("Black Myth: Wukong").unwrap(); + assert_eq!(wukong.super_resolution, Some(Support::Override)); + assert_eq!( + wukong.multi_frame_generation, + Some(MultiFrameGeneration { max_multiplier: 4 }) + ); + assert_eq!(wukong.ray_reconstruction, None); + assert_eq!(index.lookup("Ray Traced Only"), None); + assert_eq!(MultiFrameGeneration::from_cell("Yes"), None); + assert_eq!(Support::from_cell("Maybe"), None); + } + + #[test] + fn rejects_oversized_or_invalid_lists() { + assert!(parse_list(&vec![b' '; MAX_LIST_BYTES + 1]).is_err()); + assert!(parse_list(b"not json").is_err()); + assert!(parse_list(br#"{"data":{}}"#).is_err()); + assert!(parse_list(br#"[]"#).is_err()); + let rows = vec![json!({"name":"x","type":"App"}); MAX_ROWS + 1]; + assert!(parse_list(&list(Value::Array(rows))).is_err()); + let rows = vec![json!({"name":"x","type":"App"}); MAX_ROWS]; + assert!(parse_list(&list(Value::Array(rows))).unwrap().is_empty()); + } + + #[test] + fn normalizes_marks_accents_and_punctuation() { + assert_eq!(normalize_title("Cyberpunk 2077®"), "cyberpunk 2077"); + assert_eq!(normalize_title("Fortnite™"), "fortnite"); + assert_eq!( + normalize_title("Shadow of the Tomb Raider\u{200B}™ Definitive Edition"), + "shadow of the tomb raider definitive edition" + ); + assert_eq!(normalize_title("Pokémon & Friends©"), "pokemon and friends"); + assert_eq!( + normalize_title(" S.T.A.L.K.E.R. 2: Heart of Chornobyl "), + "s t a l k e r 2 heart of chornobyl" + ); + } + + #[test] + fn matches_exact_then_loose_editions_but_not_remasters() { + let index = index(json!([ + row( + json!("Cyberpunk 2077"), + ["Yes", "Yes", "NV, 6X", "NV, T", "Yes"] + ), + row(json!("Fortnite"), ["NV, T", "", "", "", "NV, T"]), + row(json!("Control"), ["Yes", "", "", "", ""]), + row(json!("Alan Wake Remastered"), ["Yes", "", "", "", ""]), + row(json!("Goat Simulator: Remastered"), ["Yes", "", "", "", ""]), + row(json!("Battlefield V"), ["Yes", "", "", "", ""]), + row(json!("The Witcher 3: Wild Hunt"), ["Yes", "", "", "", ""]), + ])); + assert!(index.lookup("Cyberpunk 2077®").is_some()); + let fortnite = index.lookup("Fortnite®").unwrap(); + assert_eq!(fortnite.super_resolution, Some(Support::Override)); + assert_eq!(fortnite.frame_generation, None); + assert!(index.lookup("Control Ultimate Edition").is_some()); + assert!(index.lookup("Battlefield™ V: Definitive").is_some()); + assert!( + index + .lookup("The Witcher 3: Wild Hunt - Game of the Year Edition") + .is_some() + ); + assert!( + index + .lookup("Witcher 3: Wild Hunt Complete Edition") + .is_some() + ); + assert_eq!(index.lookup("Alan Wake"), None); + assert_eq!(index.lookup("Goat Simulator"), None); + assert_eq!(index.lookup("Control 2"), None); + assert_eq!(index.lookup("Ultimate"), None); + assert_eq!(index.lookup("™"), None); + } + + #[test] + fn exact_rows_without_dlss_do_not_fall_back_to_editions() { + let index = index(json!([ + row(json!("Puzzle Quest"), ["", "", "", "", ""]), + row(json!("Puzzle Quest Deluxe"), ["Yes", "", "", "", ""]), + ])); + assert_eq!(index.lookup("Puzzle Quest"), None); + assert!(index.lookup("Puzzle Quest™ Deluxe").is_some()); + } + + #[test] + fn catalog_games_gain_dlss_or_null() { + let index = index(json!([row( + json!("Cyberpunk 2077"), + ["Yes", "Yes", "NV, 6X", "NV, T", "Yes"] + )])); + let app = |id: &str, title: &str| { + json!({"id":id,"title":title,"variants":[{"id":format!("{id}1"),"appStore":"Steam", + "gfn":{"features":[{"key":"RTX_ENABLED","value":"true"}]}}],"gfn":{"playabilityState":"PLAYABLE"}}) + }; + let matched = super::super::catalog::app_to_game(&app("cms-1", "Cyberpunk 2077®")).unwrap(); + let unmatched = super::super::catalog::app_to_game(&app("cms-2", "Alan Wake")).unwrap(); + let mut response = json!({ + "games":[matched.clone(), unmatched], + "items":[{"sections":[{"games":[matched.clone()]}]},{"kind":"game","game":matched}], + "catalogRevision":3 + }); + annotate(&mut response, Some(&index)); + assert_eq!( + response["games"][0]["dlss"]["multiFrameGeneration"], + json!({"maxMultiplier":6}) + ); + assert_eq!( + response["games"][0]["dlss"]["source"], + "nvidia-rtx-games-list" + ); + assert_eq!( + response["games"][0]["variants"][0]["features"], + json!(["rtx"]) + ); + assert!(response["games"][0]["variants"][0].get("dlss").is_none()); + assert!(response["games"][1]["dlss"].is_null()); + assert!(response["games"][1].get("dlss").is_some()); + assert_eq!( + response["items"][0]["sections"][0]["games"][0]["dlss"]["superResolution"], + "native" + ); + assert_eq!( + response["items"][1]["game"]["dlss"]["rayReconstruction"], + "override" + ); + assert!(response.get("dlss").is_none()); + + let mut unavailable = json!({"game":response["games"][0].clone()}); + annotate(&mut unavailable, None); + assert!(unavailable["game"].get("dlss").is_none()); + } + + fn serve(body: &'static [u8], requests: usize) -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/dlss.json", listener.local_addr().unwrap()); + std::thread::spawn(move || { + use std::io::Write; + for stream in listener.incoming().take(requests) { + let mut stream = stream.unwrap(); + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(body); + } + }); + url + } + + #[test] + fn catalog_loads_once_then_serves_from_the_disk_cache() { + let body: &'static [u8] = Box::leak( + list(json!([row( + json!("Fortnite"), + ["NV, T", "", "", "", "NV, T"] + )])) + .into_boxed_slice(), + ); + let directory = tempfile::tempdir().unwrap(); + let url = serve(body, 1); + let cache = Arc::new(StoreCache::new(directory.path().to_path_buf())); + let catalog = DlssCatalog::new(Client::new(), url.clone(), cache.clone()); + let index = catalog + .index(&json!({})) + .expect("first load within the bound"); + assert!(index.lookup("Fortnite®").is_some()); + + let restarted = DlssCatalog::new(Client::new(), url, cache); + assert!( + restarted + .index(&json!({})) + .unwrap() + .lookup("Fortnite") + .is_some() + ); + } + + #[test] + fn only_one_request_waits_for_a_slow_first_download() { + let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/dlss.json", silent.local_addr().unwrap()); + let directory = tempfile::tempdir().unwrap(); + let catalog = Arc::new(DlssCatalog::new( + Client::new(), + url, + Arc::new(StoreCache::new(directory.path().to_path_buf())), + )); + let started = std::time::Instant::now(); + let concurrent = { + let catalog = catalog.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(200)); + let started = std::time::Instant::now(); + assert!(catalog.index(&json!({})).is_none()); + started.elapsed() + }) + }; + assert!(catalog.index(&json!({})).is_none()); + assert!(started.elapsed() >= FIRST_LOAD_WAIT); + assert!(concurrent.join().unwrap() < Duration::from_millis(100)); + let started = std::time::Instant::now(); + assert!(catalog.index(&json!({})).is_none()); + assert!(started.elapsed() < Duration::from_millis(100)); + assert!(catalog.shared.state.lock().unwrap().loading); + drop(silent); + } + + #[test] + fn unavailable_list_yields_no_index_without_failing() { + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/dlss.json", closed.local_addr().unwrap()); + drop(closed); + let directory = tempfile::tempdir().unwrap(); + let catalog = DlssCatalog::new( + Client::new(), + url, + Arc::new(StoreCache::new(directory.path().to_path_buf())), + ); + let started = std::time::Instant::now(); + assert!(catalog.index(&json!({})).is_none()); + assert!(started.elapsed() <= FIRST_LOAD_WAIT + Duration::from_millis(500)); + let started = std::time::Instant::now(); + assert!(catalog.index(&json!({})).is_none()); + assert!(started.elapsed() < Duration::from_millis(100)); + let state = catalog + .shared + .settled + .wait_timeout_while( + catalog.shared.state.lock().unwrap(), + FETCH_TIMEOUT * 2, + |state| state.loading, + ) + .unwrap() + .0; + assert!(!state.loading && state.index.is_none() && state.retry_at > now_ms()); + drop(state); + let started = std::time::Instant::now(); + assert!(catalog.index(&json!({})).is_none()); + assert!(started.elapsed() < Duration::from_millis(100)); + assert!(!catalog.shared.state.lock().unwrap().loading); + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/routing_tests.rs b/external/opennow/native/opennow-core/src/gfn/routing_tests.rs new file mode 100644 index 000000000..d3a3db544 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/routing_tests.rs @@ -0,0 +1,2050 @@ +use super::tests::{auth_fixture, jwt, mock_requests, pending_attempt, test_service}; +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn service(url: &str) -> (GfnService, PathBuf) { + let (mut service, path) = test_service(url); + service.endpoints.service_urls = format!("{url}/providers"); + service.endpoints.server_info = Some(format!("{url}/v2/serverInfo")); + service.endpoints.graphql = format!("{url}/graphql"); + service.endpoints.subscription = format!("{url}/subscription"); + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("account-a")); + state.generation = 7; + state.restore_attempted = true; + drop(state); + (service, path) +} + +fn directory(id: &str, host: &str) -> Value { + json!({"gfnServiceInfo":{"gfnServiceEndpoints":[{ + "idpId":id,"loginProviderCode":"ALLIANCE","loginProviderDisplayName":"Alliance fixture", + "streamingServiceUrl":format!("https://{host}/"),"loginProviderPriority":1 + }]}}) +} + +fn reconciliation_hint() -> Value { + json!({"sessionId":"seat-a","ownerScope":{ + "userId":"account-a","providerIdpId":DEFAULT_IDP_ID,"generation":999 + },"streamingBaseUrl":"https://forged.nvidiagrid.net/"}) +} + +#[test] +fn hinted_active_get_restores_control_without_resuming_media() { + let payload = json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"seat-a","status":3,"sessionRequestData":{"appId":"123"}, + "sessionControlInfo":{"ip":"owned.nvidiagrid.net","port":443}, + "connectionInfo":[{"usage":16,"ip":"owned.nvidiagrid.net","port":443}] + }}); + let (url, worker) = mock_requests( + vec![ + (200, json!({"metaData":[]})), + ( + 200, + json!({"requestStatus":{"statusCode":1},"sessions":[payload["session"].clone()]}), + ), + (200, payload.clone()), + (200, payload), + ], + |index, request| { + let path = match index { + 0 => "/v2/serverInfo", + 1 => "/v2/session", + _ => "/v2/session/seat-a", + }; + assert!(request.starts_with(&format!("GET {path} ")), "{request}"); + assert!(request.contains("GFNJWT test-access")); + assert!(!request.contains("forged.nvidiagrid.net")); + }, + ); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + assert!(service.active_session().unwrap()["session"].is_null()); + let restored = service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap(); + assert_eq!(restored["session"]["sessionId"], "seat-a"); + assert_eq!(restored["session"]["status"], 3); + assert_ne!(restored["session"]["resumePending"], true); + assert_ne!(restored["session"]["phase"], "resuming"); + assert_eq!(restored["session"]["ownerScope"]["generation"], 7); + assert_eq!( + service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap(), + restored + ); + assert_eq!( + service + .poll_session(&json!({"sessionId":"seat-a"})) + .unwrap()["session"]["sessionId"], + "seat-a" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +fn reconciliation_discovery() -> Vec<(u16, Value)> { + vec![ + (200, json!({"metaData":[]})), + ( + 200, + json!({"requestStatus":{"statusCode":1},"sessions":[{ + "sessionId":"seat-a","status":3,"sessionRequestData":{"appId":"123"} + }]}), + ), + ] +} + +#[test] +fn hinted_active_get_validates_identity_and_keeps_unhinted_lookup_local() { + let (service, path) = service("http://127.0.0.1:1"); + assert_eq!( + service + .reconcile_active_session(&json!({}), &json!({})) + .unwrap(), + json!({"session":null}) + ); + for hint in [ + json!({"sessionId":"seat-a"}), + json!({"ownerScope":reconciliation_hint()["ownerScope"]}), + json!({"sessionId":null,"ownerScope":reconciliation_hint()["ownerScope"]}), + json!({"sessionId":"seat-a","ownerScope":{"userId":"","providerIdpId":DEFAULT_IDP_ID}}), + ] { + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "invalid_params" + ); + } + for id in [ + "", + "../other", + "seat/a", + "seat?other", + "seat#other", + "seat%2fother", + " ", + &"x".repeat(257), + ] { + let mut hint = reconciliation_hint(); + hint["sessionId"] = json!(id); + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "invalid_params" + ); + } + for key in ["userId", "providerIdpId"] { + let mut hint = reconciliation_hint(); + hint["ownerScope"][key] = json!("other"); + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + hint["ownerScope"][key] = json!("x".repeat(1025)); + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "invalid_params" + ); + } + service.state.lock().unwrap().session = None; + assert_eq!( + service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap_err() + .code, + "authentication_required" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn hinted_active_get_rebinds_existing_owner_but_rejects_other_seats() { + let (service, path) = service("http://127.0.0.1:1"); + let seat = json!({"sessionId":"seat-a","status":3}); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(auth_fixture("account-a"), 7, &seat, None).unwrap()); + service.state.lock().unwrap().generation = 8; + let result = service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap(); + assert_eq!(result["session"]["ownerScope"]["generation"], 8); + assert_eq!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .as_ref() + .unwrap() + .last_published_generation, + 8 + ); + let mut hint = reconciliation_hint(); + hint["sessionId"] = json!("seat-b"); + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + service.state.lock().unwrap().session = Some(auth_fixture("account-b")); + hint = reconciliation_hint(); + hint["ownerScope"]["userId"] = json!("account-b"); + assert_eq!( + service + .reconcile_active_session(&hint, &json!({})) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + assert_eq!(service.cloudmatch.active()["session"], seat); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn hinted_active_get_preserves_phases_and_requires_authoritative_end() { + for status in [1, 2, 3, 4, 5, 6, 7, 404] { + let mut responses = reconciliation_discovery(); + responses.push(( + if status == 404 { 404 } else { 200 }, + json!({ + "requestStatus":{"statusCode":1},"session":{"sessionId":"seat-a","status":status} + }), + )); + let (url, worker) = + mock_requests(responses, |_, request| assert!(request.starts_with("GET "))); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let result = service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap(); + assert_eq!(result["scope"]["generation"], 7); + if status == 404 { + assert!(result["session"].is_null()); + assert_eq!( + result["termination"], + json!({"source":"cloudmatch-http","httpStatus":404,"sessionId":"seat-a","resumable":false}) + ); + } else { + let phase = match status { + 1 => "preparing", + 2 => "ready", + 3 => "streaming", + 4 | 5 => "paused", + 6 => "resuming", + _ => "finished", + }; + assert_eq!(result["session"]["status"], status); + assert_eq!(result["session"]["phase"], phase); + assert_eq!( + result["session"]["streamingBaseUrl"], + DEFAULT_STREAMING_URL.trim_end_matches('/') + ); + assert_ne!(result["session"]["resumePending"], true); + if status == 7 { + assert_eq!( + result["session"]["termination"], + json!({"source":"cloudmatch-session-status","status":7,"sessionId":"seat-a","resumable":false}) + ); + } + } + assert_eq!( + service.cloudmatch.active()["session"].is_null(), + status >= 7 + ); + assert_eq!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none(), + status >= 7 + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn hinted_active_get_never_treats_discovery_or_read_errors_as_termination() { + for scenario in [ + "missing", + "other-seat", + "forbidden", + "discovery-forbidden", + "network-error", + "server-error", + "wrong-get-seat", + "invalid-status", + ] { + let mut responses = reconciliation_discovery(); + let expected = match scenario { + "missing" | "other-seat" => { + responses[1].1["sessions"] = if scenario == "missing" { + json!([]) + } else { + json!([{"sessionId":"seat-b","status":3}]) + }; + "session_discovery_failed" + } + "forbidden" => { + responses.push((403, json!({}))); + "authentication_required" + } + "discovery-forbidden" => { + responses = vec![(403, json!({}))]; + "authentication_required" + } + "network-error" => "network_error", + "server-error" => { + responses.extend(vec![(503, json!({})); 3]); + "upstream_error" + } + _ => { + responses.push(( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":if scenario == "wrong-get-seat" {"seat-b"} else {"seat-a"}, + "status":if scenario == "invalid-status" {0} else {3} + }}), + )); + "upstream_error" + } + }; + let (url, worker) = + mock_requests(responses, |_, request| assert!(request.starts_with("GET "))); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let error = service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap_err(); + assert_eq!(error.code, expected, "{scenario}: {error:?}"); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn hinted_active_get_fences_cancellation_and_account_changes_before_adoption() { + for cancel in [false, true] { + for status in [3, 7, 404] { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let mut responses = reconciliation_discovery(); + responses.push((if status == 404 {404} else {200}, json!({ + "requestStatus":{"statusCode":1},"session":{"sessionId":"seat-a","status":status} + }))); + if cancel && status == 3 { + responses.push((204, json!({}))); + } + let (url, worker) = mock_requests(responses, move |index, request| { + assert!(request.starts_with(if index == 3 { + "DELETE /v2/session/seat-a " + } else { + "GET " + })); + if index == 2 { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("restore", "session.active.get").unwrap(); + std::thread::scope(|threads| { + let restoring = threads.spawn(|| { + crate::requests::scope(permit.token.clone(), || { + service.reconcile_active_session(&reconciliation_hint(), &json!({})) + }) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + if cancel { + requests.cancel("restore"); + } else { + service.clear_cache(); + } + release_tx.send(()).unwrap(); + assert_eq!( + restoring.join().unwrap().unwrap_err().code, + if cancel { "cancelled" } else { "stale_account" } + ); + }); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + if cancel && status == 3 { + assert_eq!( + service + .stop_session(&reconciliation_hint(), &json!({})) + .unwrap()["stopped"], + true + ); + } else if !cancel { + assert_eq!( + service + .stop_session(&reconciliation_hint(), &json!({})) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + } +} + +#[test] +fn provider_regions_only_include_named_authenticated_zones() { + let payload = json!({"metaData":[ + {"key":"gfn-regions","value":"North, South"}, + {"key":"local-region","value":"North"}, + {"key":"North","value":"https://north.partner.example"}, + {"key":"South","value":"https://south.partner.example"}, + {"key":"unlisted","value":"https://unlisted.partner.example"}, + {"key":"unsafe","value":"http://unsafe.partner.example"} + ]}); + assert_eq!( + json!(provider_region_entries(&payload)), + json!([ + {"name":"North","url":"https://north.partner.example/"}, + {"name":"South","url":"https://south.partner.example/"} + ]) + ); +} + +fn expire_discovery(service: &GfnService) { + let mut state = service.state.lock().unwrap(); + state.providers_expires = None; + state.providers_retry = None; +} + +fn launch_params(id: &str) -> Value { + json!({"appId":id,"variantId":id,"catalogAppId":"launch-fixture", + "scope":scoped_result(json!({}), &auth_fixture("account-a"), 7)["scope"]}) +} + +fn launch_metadata() -> Vec<(u16, Value)> { + vec![ + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})), + ( + 200, + json!({"data":{"apps":{"items":[{"id":"launch-fixture","title":"Launch fixture", + "gfn":{"playabilityState":"PLAYABLE"},"variants":[{"id":"123","appStore":"STEAM", + "gfn":{"status":"AVAILABLE","library":{"status":"MANUAL","selected":true,"playStatus":"PLAYABLE"}}}]}]}}}), + ), + ( + 200, + json!({"data":{"appStoreDefinitions":[{"store":"STEAM","label":"Steam","features":[],"accountLinkingMetadata":{"isRequired":false}}]}}), + ), + (200, json!({"data":{"genreDefinitions":[]}})), + (200, json!({"data":{"subscriptionDefinitions":[]}})), + ( + 200, + json!({"data":{"userAccount":{"storesData":[],"subscriptions":[]}}}), + ), + ] +} + +#[test] +fn create_uses_the_launched_variant_hdr_metadata_without_saving_settings() { + let directory = tempfile::tempdir().unwrap(); + let mut store = + crate::settings::SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + store.set("enableHdr", json!(true)).unwrap(); + store.set("webrtcCompatibilityMode", json!("off")).unwrap(); + let saved = store.all(); + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[{"codec":"h265","available":true, + "hdrSupported":true,"colorQualities":["8bit_420","10bit_420"], + "hdrColorQualities":["10bit_420"]}] + }]}); + for (features, expected_mode, expected_downgrade) in [ + ( + json!([{"key":"RTX_ENABLED","value":"true"}]), + 0, + json!("game-unsupported"), + ), + ( + json!([{"key":"SUPPORTED_HDR_VERSION","values":["HDR"]}]), + 1, + Value::Null, + ), + (Value::Null, 1, Value::Null), + ] { + let mut responses = launch_metadata(); + responses[1].1["data"]["apps"]["items"][0]["variants"][0]["gfn"]["features"] = features; + responses.push(( + 200, + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"hdr-seat","status":1}}), + )); + let (url, worker) = mock_requests(responses, move |index, request| { + if index == 6 { + let body: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!(body["sessionRequestData"]["sdrHdrMode"], expected_mode); + assert_eq!( + body["sessionRequestData"]["clientRequestMonitorSettings"][0]["sdrHdrMode"], + expected_mode + ); + } + }); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let mut params = launch_params("123"); + params["runtimeCapabilities"] = capabilities.clone(); + let settings = + crate::cloudmatch::allocation_settings(&store.all(), &auth_fixture("account-a")); + let created = service + .create_session(¶ms, &settings, |settings| { + crate::streamer::StreamerService::embedded_session_settings(settings, &capabilities) + .map_err(|error| ServiceError { + code: error.code, + message: error.message, + }) + }) + .unwrap(); + assert_eq!(created["session"]["hdrDowngrade"], expected_downgrade); + assert_eq!(created["session"]["catalogAppId"], "launch-fixture"); + assert_eq!(created["session"]["appId"], "123"); + assert_eq!( + service.cloudmatch.active()["session"]["catalogAppId"], + "launch-fixture" + ); + assert_eq!( + service.cloudmatch.active()["session"]["hdrDowngrade"], + expected_downgrade + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + assert_eq!(store.all(), saved); + assert_eq!( + crate::settings::SettingsStore::load(Some(directory.path().to_owned())) + .unwrap() + .all(), + saved + ); +} + +#[test] +fn native_session_end_releases_owned_seat_before_next_allocation() { + let mut responses = launch_metadata(); + responses.push(( + 200, + json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"replacement-seat","status":1 + }}), + )); + let (url, worker) = mock_requests(responses, |index, request| { + if index == 6 { + assert!(request.starts_with("POST /v2/session?"), "{request}"); + } + assert!(!request.starts_with("DELETE ")); + }); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let seat = json!({"sessionId":"ended-seat","status":3}); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(auth_fixture("account-a"), 7, &seat, None).unwrap()); + let notification = json!({"sessionId":"ended-seat","ownerScope":{ + "userId":"account-a","providerIdpId":DEFAULT_IDP_ID,"generation":7 + }}); + assert_eq!( + service.acknowledge_session_end(¬ification).unwrap()["cleared"], + true + ); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + assert_eq!( + service.acknowledge_session_end(¬ification).unwrap()["cleared"], + false + ); + let created = service + .create_session(&launch_params("123"), &json!({}), |settings| { + Ok(settings.clone()) + }) + .unwrap(); + assert_eq!(created["session"]["sessionId"], "replacement-seat"); + service + .finish_session_create("replacement-seat", true) + .unwrap(); + assert_eq!( + service.acknowledge_session_end(¬ification).unwrap()["cleared"], + false + ); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "replacement-seat" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn native_session_end_clears_the_same_seat_after_control_generation_changes() { + let (service, path) = service("http://127.0.0.1:1"); + let seat = json!({"sessionId":"seat-a","status":3}); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(auth_fixture("account-a"), 7, &seat, None).unwrap()); + let old_snapshot = service.active_session().unwrap()["session"].clone(); + service.state.lock().unwrap().generation = 8; + let restored = service + .reconcile_active_session(&reconciliation_hint(), &json!({})) + .unwrap(); + assert_eq!(restored["session"]["ownerScope"]["generation"], 8); + assert_eq!(old_snapshot["ownerScope"]["generation"], 7); + let notification = + json!({"sessionId":old_snapshot["sessionId"],"ownerScope":old_snapshot["ownerScope"]}); + assert_eq!( + service.acknowledge_session_end(¬ification).unwrap()["cleared"], + true + ); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn native_session_end_rejects_stale_owners_and_preserves_pending_handoffs() { + let (service, path) = service("http://127.0.0.1:1"); + let mut owner = auth_fixture("account-a"); + owner.tokens.expires_at = 0; + owner.tokens.id_token_expires_at = Some(0); + let seat = json!({"sessionId":"ended-seat","status":3}); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner, 7, &seat, None).unwrap()); + let notification = json!({"sessionId":"ended-seat","ownerScope":{ + "userId":"account-a","providerIdpId":DEFAULT_IDP_ID,"generation":7 + }}); + for (key, value) in [ + ("userId", json!("other")), + ("providerIdpId", json!("other")), + ] { + let mut stale = notification.clone(); + stale["ownerScope"][key] = value; + assert_eq!( + service.acknowledge_session_end(&stale).unwrap()["cleared"], + false + ); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "ended-seat" + ); + } + for invalid in [ + json!({}), + json!({"sessionId":"ended-seat"}), + json!({"sessionId":"","ownerScope":notification["ownerScope"]}), + json!({"sessionId":"ended-seat","ownerScope":{"userId":"account-a","providerIdpId":""}}), + ] { + assert_eq!( + service.acknowledge_session_end(&invalid).unwrap_err().code, + "invalid_params" + ); + } + service + .session_routing + .lock() + .unwrap() + .active_owner + .as_mut() + .unwrap() + .allocation_generation = Some(7); + assert_eq!( + service + .acknowledge_session_end(¬ification) + .unwrap_err() + .code, + "session_update_busy" + ); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "ended-seat" + ); + service + .session_routing + .lock() + .unwrap() + .active_owner + .as_mut() + .unwrap() + .allocation_generation = None; + service.state.lock().unwrap().session = None; + assert_eq!( + service.acknowledge_session_end(¬ification).unwrap()["cleared"], + true + ); + assert!(service.cloudmatch.active()["session"].is_null()); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn owned_ad_reports_preserve_event_fields_without_accepting_foreign_session_routing() { + let response = + json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"seat-a","status":2}}); + let (url, worker) = mock_requests(vec![(200, response); 2], |index, request| { + assert!(request.starts_with("PUT /v2/session/seat-a ")); + assert!(request.contains("GFNJWT test-access")); + assert!(!request.contains("forged.nvidiagrid.net")); + let body: Value = serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + let update = &body["adUpdates"][0]; + assert_eq!(update["adId"], "ad-a"); + assert_eq!(update["clientTimestamp"], 123456); + assert_eq!(update["adAction"], if index == 0 { 4 } else { 5 }); + assert_eq!( + update["watchedTimeInMs"], + if index == 0 { 30000 } else { 0 } + ); + assert_eq!(update["pausedTimeInMs"], 250); + assert_eq!( + update["cancelReason"], + if index == 0 { + Value::Null + } else { + json!("error") + } + ); + }); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let seat = + json!({"sessionId":"seat-a","status":3,"streamingBaseUrl":"https://owned.nvidiagrid.net/"}); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(auth_fixture("account-a"), 7, &seat, None).unwrap()); + for action in ["finish", "cancel"] { + service.report_session_ad(&json!({ + "sessionId":"seat-a","streamingBaseUrl":"https://forged.nvidiagrid.net/", + "action":action,"adId":"ad-a","clientTimestamp":123456, + "watchedTimeInMs":if action == "finish" {30000} else {-10}, + "pausedTimeInMs":250,"cancelReason":if action == "cancel" {json!("error")} else {Value::Null} + })).unwrap(); + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +fn reject_concurrent_create_during(discovery: bool) { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let mut responses = if discovery { vec![] } else { launch_metadata() }; + responses.push((503, json!({}))); + let (url, worker) = mock_requests(responses, move |index, request| { + if !discovery && index < 6 { + return; + } + if discovery { + assert!(request.starts_with("GET /providers ")); + } else { + assert!(request.starts_with("POST /v2/session?")); + let body: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!(body["sessionRequestData"]["accountLinked"], true); + } + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + let (mut service, path) = service(&url); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + if discovery { + expire_discovery(&service); + } + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("first-create", "session.create").unwrap(); + std::thread::scope(|threads| { + let first = threads.spawn(|| { + crate::requests::scope(permit.token.clone(), || { + service.create_session(&launch_params("123"), &json!({}), |settings| { + Ok(settings.clone()) + }) + }) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let service = &service; + let second = threads.spawn(move || { + result_tx + .send( + service.create_session(&launch_params("456"), &json!({}), |settings| { + Ok(settings.clone()) + }), + ) + .unwrap() + }); + let rejected = result_rx.recv_timeout(Duration::from_secs(1)); + if discovery { + requests.cancel("first-create"); + } + release_tx.send(()).unwrap(); + assert_eq!( + first.join().unwrap().unwrap_err().code, + if discovery { + "cancelled" + } else { + "upstream_error" + } + ); + assert_eq!( + rejected + .expect("concurrent RPC queued behind the first create") + .unwrap_err() + .code, + "session_update_busy" + ); + second.join().unwrap(); + }); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!(service.cloudmatch.admit_create().is_ok()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn concurrent_rpc_create_is_rejected_before_cancelled_provider_preflight_finishes() { + reject_concurrent_create_during(true); +} + +#[test] +fn concurrent_rpc_create_is_rejected_before_failed_allocation_post_finishes() { + reject_concurrent_create_during(false); +} + +#[test] +fn stopping_same_owner_renews_expired_service_id_before_exactly_one_delete() { + for status in [204, 401, 503] { + let renewed_id = jwt("account-a", now_ms() + 3_600_000); + let expected_id = renewed_id.clone(); + let delete_count = std::sync::Arc::new(AtomicUsize::new(0)); + let observed = delete_count.clone(); + let (url, worker) = mock_requests( + vec![ + ( + 200, + json!({"access_token":"renewed-access","id_token":renewed_id,"expires_in":3600}), + ), + (status, json!({})), + ], + move |index, request| { + if index == 0 { + assert!(request.starts_with("POST / ")); + assert!(request.contains("client_id=test-client-id")); + } else { + assert!(request.starts_with("DELETE /v2/session/owned-seat HTTP/1.1")); + assert!(request.contains(&format!("GFNJWT {expected_id}"))); + assert!(!request.contains("expired-service-id")); + observed.fetch_add(1, Ordering::SeqCst); + } + }, + ); + let (mut service, path) = service(&url); + let mut owner = auth_fixture("account-a"); + owner.tokens.id_token = Some("expired-service-id".into()); + owner.tokens.id_token_expires_at = Some(now_ms() - 1); + service.state.lock().unwrap().session = Some(owner.clone()); + service.session_routing.lock().unwrap().active_owner = Some( + ActiveSeatOwner::capture(owner, 7, &json!({"sessionId":"owned-seat"}), None).unwrap(), + ); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + service + .cloudmatch + .seed_owned_session(json!({"sessionId":"owned-seat","status":3})); + let result = service.stop_session( + &json!({"sessionId":"owned-seat","streamingBaseUrl":"https://foreign.nvidiagrid.net/"}), + &json!({}), + ); + if status == 204 { + assert_eq!(result.unwrap()["scope"]["userId"], "account-a"); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + } else { + assert_eq!( + result.unwrap_err().code, + if status == 401 { + "http_unauthorized" + } else { + "upstream_error" + } + ); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "owned-seat" + ); + } + worker.join().unwrap(); + assert_eq!(delete_count.load(Ordering::SeqCst), 1); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn stopping_expired_foreign_owner_never_renews_or_deletes_with_selected_credentials() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (mut service, path) = service(&url); + let mut owner = auth_fixture("account-a"); + owner.tokens.id_token = Some("expired-service-id".into()); + owner.tokens.id_token_expires_at = Some(now_ms() - 1); + let mut selected = auth_fixture("account-b"); + selected.tokens.access_token = "foreign-access".into(); + service.state.lock().unwrap().session = Some(selected); + service.state.lock().unwrap().generation = 8; + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner, 7, &json!({"sessionId":"owned-seat"}), None).unwrap()); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + service + .cloudmatch + .seed_owned_session(json!({"sessionId":"owned-seat","status":3})); + assert_eq!( + service + .stop_session(&json!({"sessionId":"owned-seat"}), &json!({})) + .unwrap_err() + .code, + "authentication_required" + ); + assert_eq!( + service + .stop_session(&json!({"sessionId":"foreign-seat"}), &json!({})) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + assert_eq!( + listener.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "owned-seat" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn discovery_transport_schema_and_untrusted_endpoints_remain_retryable_failures() { + for payload in [ + json!("invalid discovery"), + json!({"gfnServiceInfo":{"gfnServiceEndpoints":[]}}), + directory("local", "localhost"), + ] { + let (url, worker) = mock_requests(vec![(200, payload)], |_, _| {}); + let (service, path) = service(&url); + expire_discovery(&service); + let result = service.providers().unwrap(); + assert_eq!(result["discovery"]["state"], "degraded"); + assert!(service.state.lock().unwrap().providers_expires.is_none()); + assert!(service.state.lock().unwrap().providers_retry.is_some()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + let (url, worker) = mock_requests( + vec![(200, directory("outside", "outside.invalid"))], + |_, _| {}, + ); + let (external_service, path) = service(&url); + expire_discovery(&external_service); + assert_eq!( + external_service.providers().unwrap()["discovery"]["state"], + "ready" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + let (service, path) = service("http://127.0.0.1:1"); + expire_discovery(&service); + assert_eq!( + service.providers().unwrap()["discovery"]["state"], + "degraded" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn discovery_honors_bounded_retry_after_and_rejects_malformed_json() { + use std::io::{BufRead, BufReader, Write}; + for (status, body, retry) in [ + (429, "{}", "120".to_owned()), + ( + 429, + "{}", + httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(125)), + ), + (200, "{", String::new()), + ] { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let worker = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&mut stream); + loop { + let mut line = String::new(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + } + write!(stream,"HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nRetry-After: {retry}\r\nConnection: close\r\n\r\n{body}",body.len()).unwrap(); + }); + let (service, path) = service(&url); + expire_discovery(&service); + let result = service.providers().unwrap(); + assert_eq!(result["discovery"]["state"], "degraded"); + let delay = result["discovery"]["retryAfterMs"].as_u64().unwrap(); + assert!(delay <= 3_600_000); + if status == 429 { + assert!(delay > 115_000); + } else { + assert!(delay <= 30_000); + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn library_and_vpc_use_the_same_explicit_proxy_without_direct_fallback() { + let (proxy, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"proxy-vpc"}})), + ( + 200, + json!({"data":{"apps":{"items":[],"pageInfo":{"hasNextPage":false}}}}), + ), + ], + |index, request| { + if index == 0 { + assert!(request.starts_with("GET http://metadata.fixture.invalid/v2/serverInfo ")); + } else { + assert!(request.starts_with("POST http://catalog.fixture.invalid/graphql ")); + assert!(request.contains("proxy-vpc")); + } + assert!(request.contains("GFNJWT test-access")); + }, + ); + let (mut service, path) = service("http://127.0.0.1:1"); + service.endpoints.server_info = Some("http://metadata.fixture.invalid/v2/serverInfo".into()); + service.endpoints.graphql = "http://catalog.fixture.invalid/graphql".into(); + let result = service + .library_catalog( + &json!({}), + &json!({"sessionProxyEnabled":true,"sessionProxyUrl":proxy}), + ) + .unwrap(); + assert_eq!(result["source"], "account-library"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn advertised_default_provider_does_not_replace_an_explicit_selection() { + let mut payload = directory("alliance", "alliance.nvidiagrid.net"); + payload["gfnServiceInfo"]["defaultProvider"] = json!("ALLIANCE"); + let (url, worker) = mock_requests(vec![(200, payload)], |_, _| {}); + let (service, path) = service(&url); + expire_discovery(&service); + let result = service.providers().unwrap(); + assert_eq!(result["defaultProviderIdpId"], "alliance"); + assert_eq!( + service + .start_device_login(&json!({"providerIdpId":"deleted"})) + .unwrap_err() + .code, + "provider_unavailable" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn failed_discovery_is_degraded_and_retries_without_remapping_explicit_identity() { + let (url, worker) = mock_requests( + vec![ + (503, json!({})), + (200, directory("alliance", "alliance.nvidiagrid.net")), + ], + |_, request| { + assert!(request.starts_with("GET /providers ")); + assert!(!request.to_ascii_lowercase().contains("authorization:")); + }, + ); + let (service, path) = service(&url); + expire_discovery(&service); + service.state.lock().unwrap().providers.clear(); + let first = service.providers().unwrap(); + assert_eq!(first["discovery"]["state"], "degraded"); + assert!(service.state.lock().unwrap().providers.is_empty()); + assert!(first["discovery"]["retryAfterMs"].as_u64().unwrap() > 0); + assert_eq!( + service.providers().unwrap()["discovery"]["state"], + "degraded" + ); + assert_eq!( + service + .start_device_login(&json!({"providerIdpId":"missing-alliance"})) + .unwrap_err() + .code, + "provider_unavailable" + ); + expire_discovery(&service); + let recovered = service.providers().unwrap(); + assert_eq!(recovered["discovery"]["state"], "ready"); + assert_eq!(recovered["providers"][0]["idpId"], "alliance"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn discovery_reconciles_only_matching_provider_and_fences_changed_or_removed_routes() { + let (url, worker) = mock_requests( + vec![ + (503, json!({})), + (200, directory("alliance", "new.nvidiagrid.net")), + (200, directory("another", "other.nvidiagrid.net")), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + { + let mut state = service.state.lock().unwrap(); + let session = state.session.as_mut().unwrap(); + session.provider.idp_id = "alliance".into(); + session.provider.streaming_service_url = "https://old.nvidiagrid.net/".into(); + state.providers.clear(); + } + expire_discovery(&service); + assert_eq!( + service.providers().unwrap()["providers"][0]["idpId"], + "alliance" + ); + let (old, generation) = service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .unwrap(); + assert_eq!( + old.provider.streaming_service_url, + "https://old.nvidiagrid.net/" + ); + expire_discovery(&service); + service.providers().unwrap(); + assert_eq!( + service.check_scope(&old, generation).unwrap_err().code, + "stale_account" + ); + let (new, generation) = service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .unwrap(); + assert_eq!( + new.provider.streaming_service_url, + "https://new.nvidiagrid.net/" + ); + expire_discovery(&service); + service.providers().unwrap(); + assert_eq!( + service.check_scope(&new, generation).unwrap_err().code, + "stale_account" + ); + assert_eq!( + service + .authenticated_snapshot(TokenPurpose::ServiceId, false) + .unwrap_err() + .code, + "provider_unavailable" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn server_info_failures_never_dispatch_a_generic_library_query() { + for (status, body, expected) in [ + (403, json!({}), "upstream_error"), + (429, json!({}), "upstream_error"), + (200, json!({}), "invalid_upstream_response"), + (503, json!({}), "upstream_error"), + ] { + let (url, worker) = mock_requests(vec![(status, body)], |_, request| { + assert!(request.starts_with("GET /v2/serverInfo ")); + assert!(request.contains("GFNJWT test-access")); + assert!(!request.contains("GFNPartnerJWT")); + }); + let (service, path) = service(&url); + assert_eq!( + service + .library_catalog(&json!({}), &json!({})) + .unwrap_err() + .code, + expected + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } +} + +#[test] +fn library_401_renews_once_and_replays_with_the_same_owner_and_new_vpc() { + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"vpc-old"}})), + (401, json!({})), + ( + 200, + json!({"access_token":"renewed-access","expires_in":3600}), + ), + ( + 200, + json!({"sub":"account-a","email":"fixture@example.invalid"}), + ), + (200, json!({"requestStatus":{"serverId":"vpc-renewed"}})), + ( + 200, + json!({"data":{"apps":{"items":[],"pageInfo":{"totalCount":0,"hasNextPage":false}}}}), + ), + ], + |index, request| { + if matches!(index, 0 | 1 | 4 | 5) { + let expected = if index < 2 { + "test-access" + } else { + "renewed-access" + }; + assert!(request.contains(&format!("GFNJWT {expected}"))); + } + if index == 1 { + assert!(request.starts_with("POST /graphql ")); + assert!(request.contains("vpc-old")); + } + if index == 5 { + assert!(request.starts_with("POST /graphql ")); + assert!(request.contains("vpc-renewed")); + } + if index == 2 { + assert!(request.contains("client_id=test-client-id")); + } + }, + ); + let (service, path) = service(&url); + let result = service.library_catalog(&json!({}), &json!({})).unwrap(); + assert_eq!(result["scope"]["generation"], 7); + assert_eq!(result["scope"]["userId"], "account-a"); + assert!(!result.to_string().contains("renewed-access")); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn second_401_is_terminal_and_never_loops_renewal() { + let (url, worker) = mock_requests( + vec![ + ( + 200, + json!({"access_token":"renewed-access","expires_in":3600}), + ), + ( + 200, + json!({"sub":"account-a","email":"fixture@example.invalid"}), + ), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let calls = AtomicUsize::new(0); + let error = service + .authenticated_read(|_, _| { + calls.fetch_add(1, Ordering::SeqCst); + Err(ServiceError { + code: "http_unauthorized", + message: "Unauthorized fixture".into(), + }) + }) + .unwrap_err(); + assert_eq!(error.code, "http_unauthorized"); + assert_eq!(calls.load(Ordering::SeqCst), 2); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn delayed_library_result_cannot_publish_after_account_replacement() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = mock_requests( + vec![ + (200, json!({"requestStatus":{"serverId":"vpc-a"}})), + ( + 200, + json!({"data":{"apps":{"items":[],"pageInfo":{"hasNextPage":false}}}}), + ), + ], + move |index, _| { + if index == 1 { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ); + let (service, path) = service(&url); + std::thread::scope(|threads| { + let read = threads.spawn(|| service.library_catalog(&json!({}), &json!({}))); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + { + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("account-b")); + state.generation += 1; + } + release_tx.send(()).unwrap(); + assert_eq!(read.join().unwrap().unwrap_err().code, "stale_account"); + }); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn dynamic_nvidia_queue_route_does_not_require_discovery() { + let (service, path) = service("http://127.0.0.1:1"); + let session = auth_fixture("account-a"); + let params = json!({"zone":"NP-NEW9-01","streamingBaseUrl":"https://np-new9-01.cloudmatchbeta.nvidiagrid.net/"}); + let settings = json!({"region":"https://saved.nvidiagrid.net/"}); + let (routed, effective) = service + .scoped_session_route(¶ms, &settings, &session) + .unwrap(); + assert_eq!(routed, params); + assert_eq!(effective["region"], ""); + assert_eq!(settings["region"], "https://saved.nvidiagrid.net/"); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn dynamic_queue_route_rejects_mismatched_urls_malformed_ids_and_alliance() { + let (url, worker) = mock_requests( + vec![( + 200, + json!({"requestStatus":{"serverId":"fixture"},"metaData":[]}), + )], + |_, request| { + assert!(request.starts_with("GET /v2/serverInfo ")); + }, + ); + let (service, path) = service(&url); + let mut session = auth_fixture("account-a"); + for (zone, base) in [ + ( + "NP-NEW9-01", + "https://np-new9-02.cloudmatchbeta.nvidiagrid.net/", + ), + ( + "NP-NEW9-01", + "https://np-new9-01.cloudmatchbeta.nvidiagrid.net.evil/", + ), + ( + "NP-NEW9-01", + "https://user@np-new9-01.cloudmatchbeta.nvidiagrid.net/", + ), + ( + "NP-NEW9-01", + "https://np-new9-01.cloudmatchbeta.nvidiagrid.net/path", + ), + ( + "NP-NEW9-01", + "http://np-new9-01.cloudmatchbeta.nvidiagrid.net/", + ), + ( + "NP-NEW9-01.evil", + "https://np-new9-01.evil.cloudmatchbeta.nvidiagrid.net/", + ), + ( + "NPA-NEW9-01", + "https://npa-new9-01.cloudmatchbeta.nvidiagrid.net/", + ), + ] { + let (routed, _) = service + .scoped_session_route( + &json!({"zone":zone,"streamingBaseUrl":base}), + &json!({}), + &session, + ) + .unwrap(); + assert!(routed["streamingBaseUrl"].is_null(), "{base}"); + } + session.provider.idp_id = "alliance".into(); + session.provider.code = "ALLIANCE".into(); + service.state.lock().unwrap().providers = vec![session.provider.clone()]; + let (routed, _) = service.scoped_session_route(&json!({"zone":"NP-NEW9-01","streamingBaseUrl":"https://np-new9-01.cloudmatchbeta.nvidiagrid.net/"}), &json!({}), &session).unwrap(); + assert!(routed["streamingBaseUrl"].is_null()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn region_overrides_require_current_provider_membership_and_preserve_saved_preferences() { + let (url, worker) = mock_requests( + vec![( + 200, + json!({"requestStatus":{"serverId":"alliance-vpc"},"metaData":[ + {"key":"gfn-regions","value":"Alliance region"}, + {"key":"Alliance region","value":"https://alliance-region.nvidiagrid.net/"}, + {"key":"Untrusted","value":"https://outside.invalid/"} + ]}), + )], + |_, request| assert!(request.starts_with("GET /v2/serverInfo ")), + ); + let (service, path) = service(&url); + let mut session = auth_fixture("account-a"); + session.provider.idp_id = "alliance".into(); + session.provider.streaming_service_url = "https://alliance.nvidiagrid.net/".into(); + service.state.lock().unwrap().providers = vec![session.provider.clone()]; + let settings = json!({"region":"https://nvidia-region.nvidiagrid.net/","regionProviderIdpId":DEFAULT_IDP_ID}); + let (params, effective) = service + .scoped_session_route(&json!({}), &settings, &session) + .unwrap(); + assert!(params["streamingBaseUrl"].is_null()); + assert_eq!(effective["region"], ""); + let (params, _) = service + .scoped_session_route( + &json!({"streamingBaseUrl":"https://outside.invalid/"}), + &settings, + &session, + ) + .unwrap(); + assert!(params["streamingBaseUrl"].is_null()); + assert_eq!(settings["region"], "https://nvidia-region.nvidiagrid.net/"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn endpoint_validation_preserves_https_authority_constraints_for_partner_hosts() { + for url in [ + "https://prod.cloudmatchbeta.nvidiagrid.net/", + "https://provider.nvidiagrid.net/", + "https://region.geforcenow.nvidiagrid.net/", + "https://partner.example.com/", + ] { + assert!(trusted_streaming_base(url).is_ok()); + } + for url in [ + "http://provider.nvidiagrid.net/", + "https://user@provider.nvidiagrid.net/", + "https://provider.nvidiagrid.net:8443/", + "https://127.0.0.1/", + "https://partner.example.com:8443/", + ] { + assert!(trusted_streaming_base(url).is_err(), "{url}"); + } +} + +#[test] +fn authenticated_client_does_not_follow_redirects_with_credentials() { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let directory = tempfile::tempdir().unwrap(); + let service = GfnService::new(directory.path().to_path_buf()).unwrap(); + let source = TcpListener::bind("127.0.0.1:0").unwrap(); + let target = TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + let source_url = format!("http://{}/original", source.local_addr().unwrap()); + let target_url = format!("http://{}/stolen", target.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = source.accept().unwrap(); + let mut buffer = [0_u8; 4096]; + let length = stream.read(&mut buffer).unwrap(); + assert!(String::from_utf8_lossy(&buffer[..length]).contains("GFNJWT secret-for-test")); + write!(stream, "HTTP/1.1 302 Found\r\nLocation: {target_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); + }); + let response = service + .client + .get(source_url) + .header(AUTHORIZATION, "GFNJWT secret-for-test") + .send() + .unwrap(); + server.join().unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::FOUND); + assert!(target.accept().is_err()); +} + +#[test] +fn preparation_uses_owned_context_and_rejects_old_generations_or_other_seats() { + let (service, path) = service("http://127.0.0.1:1"); + let owner = auth_fixture("account-a"); + service.cloudmatch.seed_owned_session(json!({"sessionId":"seat-a","subSessionId":"sub-a","status":3,"rtspsEndpoints":["rtsps://owned.nvidiagrid.net:443"],"connectionInfo":[{"protocol":"RTSPS","host":"owned.nvidiagrid.net","port":443}]})); + service.session_routing.lock().unwrap().active_owner = Some( + ActiveSeatOwner::capture(owner.clone(), 7, &json!({"sessionId":"seat-a"}), None).unwrap(), + ); + let params = json!({"session":{"sessionId":"seat-a","status":3,"connectionInfo":[{"host":"evil.invalid"}]}}); + let calls = AtomicUsize::new(0); + let result = service + .prepare_owned_stream(¶ms, |params| { + calls.fetch_add(1, Ordering::SeqCst); + assert_eq!( + params["session"]["connectionInfo"][0]["host"], + "owned.nvidiagrid.net" + ); + Ok(params.clone()) + }) + .unwrap(); + assert_eq!(result["session"]["sessionId"], "seat-a"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!( + service + .prepare_owned_stream(&json!({"session":{"sessionId":"other"}}), |_| panic!( + "foreign seat prepared" + )) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + service.state.lock().unwrap().generation += 1; + let regained = service + .prepare_owned_stream(¶ms, |params| Ok(params.clone())) + .unwrap(); + assert_eq!(regained["scope"]["generation"], 8); + assert_eq!( + regained["session"]["connectionInfo"][0]["host"], + "owned.nvidiagrid.net" + ); + assert_eq!( + service.check_scope(&owner, 7).unwrap_err().code, + "stale_account" + ); + service.state.lock().unwrap().session = Some(auth_fixture("account-b")); + assert_eq!( + service + .prepare_owned_stream(¶ms, |_| panic!("foreign owner prepared")) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn alliance_webrtc_preparation_uses_only_the_owned_transport_and_endpoint() { + let (service, path) = service("http://127.0.0.1:1"); + let mut owner = auth_fixture("account-a"); + owner.provider.idp_id = "alliance-provider".into(); + owner.provider.code = "ALLIANCE".into(); + service.state.lock().unwrap().session = Some(owner.clone()); + let owned = json!({"sessionId":"seat-a","status":3,"transportMode":"webrtc", + "signalingUrl":"wss://owned.nvidiagrid.net/nvst/", + "connectionInfo":[{"usage":14,"appLevelProtocol":4,"ip":"owned.nvidiagrid.net","port":443,"resourcePath":"/nvst/"}]}); + service.cloudmatch.seed_owned_session(owned.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner.clone(), 7, &owned, None).unwrap()); + let forged = json!({"session":{"sessionId":"seat-a","transportMode":"nvst","signalingUrl":"wss://evil.invalid/"}}); + let prepared = service + .prepare_owned_stream(&forged, |params| Ok(params.clone())) + .unwrap(); + assert_eq!(prepared["session"]["transportMode"], "webrtc"); + assert_eq!(prepared["session"]["signalingUrl"], owned["signalingUrl"]); + let mut missing = owned.clone(); + missing["connectionInfo"] = json!([{ "usage":2,"ip":"owned.nvidiagrid.net","port":322 }]); + service.cloudmatch.seed_owned_session(missing); + assert_eq!( + service + .prepare_owned_stream(&forged, |_| panic!("invalid endpoint prepared")) + .unwrap_err() + .code, + "session_endpoint_missing" + ); + service.cloudmatch.seed_owned_session(owned.clone()); + owner.provider = LoginProvider::default_nvidia(); + service.state.lock().unwrap().session = Some(owner.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner, 7, &owned, None).unwrap()); + let prepared = service + .prepare_owned_stream(&forged, |params| Ok(params.clone())) + .unwrap(); + assert_eq!(prepared["session"]["transportMode"], "webrtc"); + assert_eq!(prepared["session"]["signalingUrl"], owned["signalingUrl"]); + let _ = std::fs::remove_dir_all(path); +} + +#[test] +fn original_identity_regains_each_exact_seat_operation_after_generation_changes() { + for transition in ["switch-back", "clear-cache", "relogin"] { + for operation in ["get", "poll", "prepare", "claim", "ad", "stop"] { + let payload = json!({"requestStatus":{"statusCode":1},"session":{ + "sessionId":"seat-a","subSessionId":"sub-a","status":2, + "connectionInfo":[{"usage":16,"ip":"owned.nvidiagrid.net","port":443}] + }}); + let responses = match operation { + "poll" | "ad" => vec![(200, payload.clone())], + "claim" => vec![(200, payload.clone()), (200, payload)], + "stop" => vec![(204, json!({}))], + _ => vec![], + }; + let (url, worker) = mock_requests(responses, |_, request| { + assert!(request.contains(" /v2/session/seat-a")); + assert!(request.contains("GFNJWT test-access")); + assert!(!request.contains("foreign-access")); + assert!(!request.contains("forged.nvidiagrid.net")); + }); + let (mut service, path) = service(&url); + let owner = auth_fixture("account-a"); + service.vault.save(&owner).unwrap(); + let mut foreign = auth_fixture("account-b"); + foreign.tokens.access_token = "foreign-access".into(); + service.vault.save(&foreign).unwrap(); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + let seat = json!({"sessionId":"seat-a","subSessionId":"sub-a","status":3, + "streamingBaseUrl":"https://owned.nvidiagrid.net/","zone":"owned.nvidiagrid.net", + "rtspsEndpoints":["rtsps://owned.nvidiagrid.net:443"], + "connectionInfo":[{"protocol":"RTSPS","host":"owned.nvidiagrid.net","port":443}] + }); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner.clone(), 7, &seat, None).unwrap()); + match transition { + "switch-back" => { + service + .switch_account(&json!({"userId":"account-b"})) + .unwrap(); + assert!(service.active_session().unwrap()["session"].is_null()); + let params = json!({"sessionId":"seat-a"}); + assert_eq!( + service.claim_session(¶ms, &json!({})).unwrap_err().code, + "session_owner_mismatch" + ); + assert_eq!( + service.report_session_ad(¶ms).unwrap_err().code, + "session_owner_mismatch" + ); + assert_eq!( + service + .prepare_owned_stream(&json!({"session":params}), |_| panic!( + "foreign owner prepared" + )) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + service + .switch_account(&json!({"userId":"account-a"})) + .unwrap(); + } + "clear-cache" => { + service.clear_cache(); + } + "relogin" => { + service + .state + .lock() + .unwrap() + .attempts + .insert("relogin".into(), pending_attempt(Some(owner.clone()))); + service + .complete_device_login(&json!({"attemptId":"relogin"})) + .unwrap(); + } + _ => unreachable!(), + } + let generation = service.state.lock().unwrap().generation; + assert!(generation > 7); + assert_eq!( + service.check_scope(&owner, 7).unwrap_err().code, + "stale_account" + ); + let params = json!({"sessionId":"seat-a","streamingBaseUrl":"https://forged.nvidiagrid.net/","action":"start","adId":"fixture"}); + let result = match operation { + "get" => service.active_session(), + "poll" => service.poll_session(¶ms), + "prepare" => service.prepare_owned_stream(&json!({"session":params}), |params| { + assert_eq!( + params["session"]["connectionInfo"][0]["host"], + "owned.nvidiagrid.net" + ); + assert_eq!(params["session"]["ownerScope"]["generation"], generation); + Ok(json!({"context":{}})) + }), + "claim" => service.claim_session(¶ms, &json!({})), + "ad" => service.report_session_ad(¶ms), + "stop" => service.stop_session(¶ms, &json!({})), + _ => unreachable!(), + } + .unwrap_or_else(|error| panic!("{transition}/{operation}: {error:?}")); + assert_eq!( + result["scope"]["generation"], generation, + "{transition}/{operation}" + ); + assert_eq!(result["scope"]["userId"], "account-a"); + if operation != "stop" { + assert_eq!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .as_ref() + .unwrap() + .last_published_generation, + generation + ); + } + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + } +} + +#[test] +fn durable_seat_republication_does_not_renew_allocation_receipt_authority() { + let mut responses = launch_metadata(); + responses.extend([ + (200, json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"fresh-seat","status":1}})), + (204, json!({})), + ]); + let (url, worker) = mock_requests(responses, |index, request| { + if index == 7 { + assert!(request.starts_with("DELETE /v2/session/fresh-seat ")); + } + }); + let (mut service, path) = service(&url); + { + let mut state = service.state.lock().unwrap(); + state.providers = vec![auth_fixture("account-a").provider]; + state.providers_expires = Some(Instant::now() + Duration::from_secs(60)); + } + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + service + .create_session(&launch_params("123"), &json!({}), |settings| { + Ok(settings.clone()) + }) + .unwrap(); + service.clear_cache(); + assert_eq!(service.active_session().unwrap()["scope"]["generation"], 8); + assert_eq!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .as_ref() + .unwrap() + .allocation_generation, + Some(7) + ); + service.finish_session_create("fresh-seat", true).unwrap(); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn rediscovery_stays_generation_fenced_and_same_user_other_provider_cannot_manage_seat() { + let (service, path) = service("http://127.0.0.1:1"); + let owner = auth_fixture("account-a"); + let seat = json!({"sessionId":"seat-a","status":3}); + service + .cloudmatch + .seed_discovered_sessions(std::slice::from_ref(&seat)); + service.session_routing.lock().unwrap().discovery_owner = + Some((owner.provider.idp_id.clone(), owner.user.user_id.clone(), 7)); + service.clear_cache(); + assert_eq!( + service.claim_session(&seat, &json!({})).unwrap_err().code, + "session_owner_mismatch" + ); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner.clone(), 7, &seat, None).unwrap()); + service + .state + .lock() + .unwrap() + .session + .as_mut() + .unwrap() + .provider + .idp_id = "other-provider".into(); + assert!(service.active_session().unwrap()["session"].is_null()); + assert_eq!( + service.claim_session(&seat, &json!({})).unwrap_err().code, + "session_owner_mismatch" + ); + assert_eq!( + service.report_session_ad(&seat).unwrap_err().code, + "session_owner_mismatch" + ); + assert_eq!( + service + .prepare_owned_stream(&json!({"session":seat}), |_| panic!( + "foreign provider prepared" + )) + .unwrap_err() + .code, + "session_owner_mismatch" + ); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn delayed_poll_fences_ordinary_results_but_preserves_exact_seat_termination() { + for foreign_selected in [false, true] { + for status in [2, 7, 404] { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let payload = json!({"requestStatus":{"statusCode":1},"session":{"sessionId":"seat-a","status":status}}); + let (url, worker) = mock_requests( + vec![(if status == 404 { 404 } else { 200 }, payload)], + move |_, request| { + assert!(request.contains(" /v2/session/seat-a ")); + assert!(request.contains("GFNJWT test-access")); + assert!(!request.contains("foreign-access")); + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }, + ); + let (mut service, path) = service(&url); + let owner = auth_fixture("account-a"); + let seat = json!({"sessionId":"seat-a","status":3}); + service + .cloudmatch + .set_test_control_base(url::Url::parse(&url).unwrap()); + service.cloudmatch.seed_owned_session(seat.clone()); + service.session_routing.lock().unwrap().active_owner = + Some(ActiveSeatOwner::capture(owner, 7, &seat, None).unwrap()); + if foreign_selected { + let mut foreign = auth_fixture("account-b"); + foreign.tokens.access_token = "foreign-access".into(); + service.vault.save(&foreign).unwrap(); + service + .switch_account(&json!({"userId":"account-b"})) + .unwrap(); + } + std::thread::scope(|threads| { + let poll = threads.spawn(|| service.poll_session(&json!({"sessionId":"seat-a"}))); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + service.clear_cache(); + release_tx.send(()).unwrap(); + let result = poll.join().unwrap(); + if status == 2 && !foreign_selected { + assert_eq!(result.unwrap_err().code, "stale_account"); + } else { + let result = result.unwrap(); + assert_eq!(result["scope"]["generation"], 7); + assert_eq!(result["scope"]["userId"], "account-a"); + if status != 2 { + let termination = if status == 7 { + &result["session"]["termination"] + } else { + &result["termination"] + }; + assert_eq!(termination["sessionId"], "seat-a"); + assert_eq!(termination["resumable"], false); + assert!(service.cloudmatch.active()["session"].is_null()); + assert!( + service + .session_routing + .lock() + .unwrap() + .active_owner + .is_none() + ); + } + } + }); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + } +} + +#[test] +fn digevo_unreachable_discovery_endpoint_falls_back_to_latam_west() { + let stale = LoginProvider { + idp_id: PROVIDER_FALLBACKS[0].idp_id.to_owned(), + code: "DIG".to_owned(), + display_name: "Digevo".to_owned(), + streaming_service_url: "https://prod.DIG.geforcenow.nvidiagrid.net/".to_owned(), + priority: 10, + }; + // NXDOMAIN (out of footprint, no VPN): use the verified regional fallback. + assert_eq!( + effective_provider_url_with(&stale, |_| false), + "https://latam-west.dig.geforcenow.nvidiagrid.net/" + ); + // Reachable (VPN / in footprint): keep the geo-steered discovery endpoint. + assert_eq!( + effective_provider_url_with(&stale, |_| true), + "https://prod.DIG.geforcenow.nvidiagrid.net/" + ); + // Non-Digevo providers and already-regional Digevo URLs are untouched. + let nvidia = LoginProvider::default_nvidia(); + assert_eq!( + effective_provider_url_with(&nvidia, |_| false), + "https://prod.cloudmatchbeta.nvidiagrid.net/" + ); + let current = LoginProvider { + streaming_service_url: "https://latam-west.dig.geforcenow.nvidiagrid.net/".to_owned(), + ..stale + }; + assert_eq!( + effective_provider_url_with(¤t, |_| false), + "https://latam-west.dig.geforcenow.nvidiagrid.net/" + ); +} + +#[test] +fn only_alliance_server_info_dns_failures_explain_the_home_network_requirement() { + struct Unresolved; + impl reqwest::dns::Resolve for Unresolved { + fn resolve(&self, _: reqwest::dns::Name) -> reqwest::dns::Resolving { + Box::pin(std::future::ready(Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "no such host", + ) + .into()))) + } + } + let unresolved = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(10)) + .dns_resolver(std::sync::Arc::new(Unresolved)) + .build() + .unwrap(); + let direct = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let refused = format!("http://{}/v2/serverInfo", closed.local_addr().unwrap()); + drop(closed); + let partner = "https://prod.partner.geforcenow.nvidiagrid.net/v2/serverInfo".to_owned(); + let home_network = "Zain can only be reached from inside its home country and network. Turn off any VPN, proxy or custom DNS (such as 1.1.1.1 or 8.8.8.8) and try again. Details: "; + type Read = fn(&GfnService) -> Result; + let reads: [(Read, &str); 4] = [ + ( + |service| service.library_catalog(&json!({}), &json!({})), + "Server info failed", + ), + ( + |service| service.subscription(&json!({})), + "Server info failed", + ), + ( + |service| service.catalog_favorites(&json!({})), + "Store server info failed", + ), + ( + |service| service.regions(&json!({})), + "Region discovery failed", + ), + ]; + for (idp_id, client, server_info, stage, prefix) in [ + ("zain-idp", &unresolved, &partner, "dns", home_network), + (DEFAULT_IDP_ID, &unresolved, &partner, "dns", ""), + ("zain-idp", &direct, &refused, "connect", ""), + ] { + for (read, context) in reads { + let (mut service, path) = test_service("http://unused.invalid"); + service.client = client.clone(); + service.endpoints.server_info = Some(server_info.clone()); + let mut session = auth_fixture("account-a"); + session.provider.idp_id = idp_id.into(); + session.provider.code = "ZAI".into(); + session.provider.display_name = "Zain".into(); + let mut state = service.state.lock().unwrap(); + state.providers = vec![session.provider.clone()]; + state.session = Some(session); + state.generation = 7; + state.restore_attempted = true; + drop(state); + let error = read(&service).unwrap_err(); + assert_eq!(error.code, "network_error"); + assert_eq!( + error.message, + format!("{prefix}{context}: error sending request ({stage})") + ); + std::fs::remove_dir_all(path).ok(); + } + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/store_launch.rs b/external/opennow/native/opennow-core/src/gfn/store_launch.rs new file mode 100644 index 000000000..512bf82c7 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/store_launch.rs @@ -0,0 +1,262 @@ +use super::catalog_actions::{ + LaunchDecision, LaunchStatus, PlayabilityMetadata, account_decision, game_play_denial, + paid_membership_decision, readiness_decision, requires_paid_membership, selected_variant, +}; +use super::*; + +const STORE_GAME_QUERY: &str = r#"query OpenNowStoreClientApps($vpcId: String!, $locale: String!) { + apps(vpcId: $vpcId, language: $locale, filters: { + type: { + in: ["PLATFORM_CLIENT"] + } + }) { + items { + id + title + images { + HERO_IMAGE + } + variants { + appStore + id + gfn { + status + } + } + gfn { + playType + } + } + } +}"#; + +const SUPPORTED_STORES: [&str; 1] = ["STEAM"]; + +pub(super) fn store_launch_intent(params: &Value) -> Result { + match params.get("storeLaunch") { + None | Some(Value::Bool(false)) => Ok(false), + Some(Value::Bool(true)) => Ok(true), + Some(_) => Err(ServiceError::invalid("storeLaunch must be a boolean")), + } +} + +fn requested_store(params: &Value) -> Result<&'static str, ServiceError> { + match params.get("store") { + None => Ok(SUPPORTED_STORES[0]), + Some(Value::String(store)) if store == SUPPORTED_STORES[0] => Ok(SUPPORTED_STORES[0]), + Some(Value::String(_)) => Err(ServiceError::invalid( + "Only Steam store launches are supported", + )), + Some(_) => Err(ServiceError::invalid("store must be a string")), + } +} + +fn launch_id(variant: &Value) -> Option<&str> { + variant["id"] + .as_str() + .filter(|id| !id.is_empty() && id.len() <= 256) +} + +pub(super) fn eligible_store_target<'a>( + games: &'a [Value], + store: &str, +) -> Option<(&'a Value, &'a Value)> { + games.iter().find_map(|game| { + let variant = game["variants"] + .as_array()? + .iter() + .find(|variant| variant["store"] == store && launch_id(variant).is_some())?; + Some((game, variant)) + }) +} + +fn resolved_store_target( + candidates: &[Value], + store: &str, + target: Option<(&str, &str)>, +) -> Option<(Value, Value)> { + match target { + Some((app_id, variant_id)) => { + let game = candidates.iter().find(|game| game["id"] == app_id)?; + let variant = selected_variant(game, variant_id)?; + Some((game.clone(), variant.clone())) + } + None => eligible_store_target(candidates, store) + .map(|(game, variant)| (game.clone(), variant.clone())), + } +} + +pub(super) fn store_launch_decision( + game: &Value, + app_id: &str, + variant_id: &str, + store: &str, + subscription: &Value, + access: &Value, + exact: bool, +) -> LaunchDecision { + use LaunchStatus::*; + let decide = |status, message| LaunchDecision { status, message }; + if game["id"] != app_id || variant_id.parse::().ok().is_none_or(|id| id <= 0) { + return decide( + MetadataUnconfirmed, + "The exact store version could not be confirmed. Refresh and try again.", + ); + } + let Some(variant) = selected_variant(game, variant_id) else { + return decide( + MetadataUnconfirmed, + "The selected store version is no longer available. Refresh and try again.", + ); + }; + if variant["store"] != store { + return decide( + MetadataUnconfirmed, + "The selected store version changed. Refresh and try again.", + ); + } + let playability = if exact { + PlayabilityMetadata::Required + } else { + PlayabilityMetadata::WhenPresent + }; + if let Some(decision) = readiness_decision(game, variant, playability) { + return decision; + } + if !subscription["storageAddon"].is_object() { + return decide( + SubscriptionRequired, + "Launching the Steam store for persistent game management requires the GeForce NOW persistent storage add-on.", + ); + } + if let Some(decision) = game_play_denial(subscription) { + return decision; + } + if let Some(decision) = account_decision(variant, variant_id, access) { + return decision; + } + if requires_paid_membership(game) + && let Some(decision) = paid_membership_decision(subscription) + { + return decision; + } + decide( + Ready, + "Ready to open the Steam store for persistent game management.", + ) +} + +impl GfnService { + fn store_games(&self, settings: &Value) -> Result { + self.authenticated_read(|session, generation| { + let revision = self.catalog_revision.load(std::sync::atomic::Ordering::Acquire); + let client = client_for_settings(&self.client, settings).map_err(ServiceError::invalid)?; + let token = session + .tokens + .id_token + .as_deref() + .unwrap_or(&session.tokens.access_token); + let vpc = self.vpc_id( + &client, + session, + generation, + settings, + token, + Some(&self.store_cache.requests), + )?; + let payload = self.catalog_document( + &client, + token, + STORE_GAME_QUERY, + json!({"vpcId":vpc,"locale":"en_US"}), + )?; + let games = complete_games(&payload["data"]["apps"]["items"])?; + self.check_scope(session, generation)?; + self.check_catalog_revision(revision)?; + Ok(json!({"games":games,"catalogRevision":revision,"freshness":"fresh","fetchedAt":now_ms()})) + }) + } + + pub(crate) fn store_launch_inspect( + &self, + params: &Value, + target: Option<(&str, &str)>, + settings: &Value, + ) -> Result { + let store = requested_store(params)?; + let games = self.store_games(settings)?; + let subscription = self.subscription(settings)?; + if subscription["scope"] != games["scope"] { + return Err(ServiceError { + code: "stale_account", + message: "The membership account changed.".into(), + }); + } + let candidates = games["games"].as_array().cloned().unwrap_or_default(); + let (app_id, variant_id, decision, game) = match resolved_store_target( + &candidates, + store, + target, + ) { + Some((game, variant)) => { + let app_id = game["id"].as_str().unwrap_or_default().to_owned(); + let variant_id = variant["id"].as_str().unwrap_or_default().to_owned(); + let access = self.account_connections(settings)?; + if access["scope"] != games["scope"] { + return Err(ServiceError { + code: "stale_account", + message: "The store account changed.".into(), + }); + } + let detail = self.catalog_game(&json!({"appId":app_id}), settings); + let (metadata, exact) = match detail { + Ok(detail) => (detail["game"].clone(), true), + Err(error) if error.code == "catalog_game_not_found" => (game, false), + Err(error) => return Err(error), + }; + let decision = store_launch_decision( + &metadata, + &app_id, + &variant_id, + store, + &subscription["subscription"], + &access, + exact, + ); + (app_id, variant_id, decision, metadata) + } + None => match target { + Some((app_id, variant_id)) => ( + app_id.to_owned(), + variant_id.to_owned(), + LaunchDecision { + status: LaunchStatus::MetadataUnconfirmed, + message: "The exact store version could not be confirmed. Refresh and try again.", + }, + Value::Null, + ), + None => ( + String::new(), + String::new(), + LaunchDecision { + status: LaunchStatus::Unavailable, + message: "No Steam store launch is available for this account.", + }, + Value::Null, + ), + }, + }; + let mut result = json!({ + "store":store, + "appId":if app_id.is_empty() { Value::Null } else { json!(app_id) }, + "variantId":if variant_id.is_empty() { Value::Null } else { json!(variant_id) }, + "game":game, + "decision":decision, + "catalogRevision":games["catalogRevision"], + "fetchedAt":games["fetchedAt"], + "freshness":games["freshness"], + }); + result["scope"] = games["scope"].clone(); + Ok(result) + } +} diff --git a/external/opennow/native/opennow-core/src/gfn/store_launch_tests.rs b/external/opennow/native/opennow-core/src/gfn/store_launch_tests.rs new file mode 100644 index 000000000..38495db32 --- /dev/null +++ b/external/opennow/native/opennow-core/src/gfn/store_launch_tests.rs @@ -0,0 +1,736 @@ +use super::catalog_actions::LaunchStatus; +use super::tests::{auth_fixture, mock_requests, test_service}; +use super::*; + +fn service(url: &str) -> (GfnService, PathBuf) { + let (mut service, path) = test_service(url); + service.endpoints.graphql = format!("{url}/graphql"); + service.endpoints.account_linking = format!("{url}/graphql"); + service.endpoints.subscription = format!("{url}/subscription"); + service.endpoints.server_info = Some(format!("{url}/vpc")); + let mut state = service.state.lock().unwrap(); + state.session = Some(auth_fixture("fixture-account")); + state.restore_attempted = true; + drop(state); + (service, path) +} + +fn vpc() -> (u16, Value) { + (200, json!({"requestStatus":{"serverId":"fixture-vpc"}})) +} + +fn platform_client( + id: &str, + title: &str, + app_store: &str, + variant_id: &str, + status: Value, +) -> Value { + json!({"id":id,"title":title, + "images":{"HERO_IMAGE":[{"url":format!("https://cdn.example.invalid/{id}.jpg"),"width":1920,"height":1080}]}, + "variants":[{"appStore":app_store,"id":variant_id,"gfn":{"status":status}}], + "gfn":{"playType":"PLATFORM_CLIENT"}}) +} + +fn store_apps(items: Vec) -> (u16, Value) { + (200, json!({"data":{"apps":{"items":items}}})) +} + +fn subscription(addon: bool) -> (u16, Value) { + let addons = if addon { + json!([{"type":"STORAGE","subType":"PERMANENT_STORAGE","status":"OK", + "attributes":[{"key":"TOTAL_STORAGE_SIZE_IN_GB","textValue":"512"}]}]) + } else { + json!([]) + }; + ( + 200, + json!({"allottedTimeInMinutes":600.0,"purchasedTimeInMinutes":0.0,"rolledOverTimeInMinutes":0.0, + "totalTimeInMinutes":600.0,"remainingTimeInMinutes":500.0,"membershipTier":"ULTIMATE", + "type":"SUBSCRIPTION","subType":"UNLIMITED","addons":addons, + "features":{"resolutions":[{"isEntitled":true,"widthInPixels":1920,"heightInPixels":1080,"framesPerSecond":60}]}, + "currentSubscriptionState":{"state":"ACTIVE","isGamePlayAllowed":true}, + "notifications":{"notifyUserWhenTimeRemainingInMinutes":15,"notifyUserOnSessionWhenRemainingTimeInMinutes":5}, + "firstEntitlementStartDateTime":"2026-01-01T00:00:00Z", + "currentSpanStartDateTime":"2026-01-01T00:00:00Z", + "currentSpanEndDateTime":"2026-02-01T00:00:00Z"}), + ) +} + +fn store_definitions() -> Vec<(u16, Value)> { + vec![ + ( + 200, + json!({"data":{"appStoreDefinitions":[{"store":"STEAM","label":"Steam","features":[], + "accountLinkingMetadata":{"isRequired":false,"isSupported":true,"supportedVariantIds":[]}}]}}), + ), + (200, json!({"data":{"genreDefinitions":[]}})), + (200, json!({"data":{"subscriptionDefinitions":[]}})), + ( + 200, + json!({"data":{"userAccount":{"storesData":[{"store":"STEAM","isConnected":true,"status":"connected"}],"subscriptions":[]}}}), + ), + ] +} + +fn detail_game(app_id: &str, variant_id: &str, playability: &str) -> (u16, Value) { + ( + 200, + json!({"data":{"apps":{"items":[{"id":app_id,"title":"Steam", + "gfn":{"playabilityState":playability},"variants":[{"id":variant_id,"appStore":"STEAM", + "gfn":{"status":"AVAILABLE","library":{"status":"MANUAL","selected":true,"playStatus":"PLAYABLE"}}}]}]}}}), + ) +} + +fn no_game() -> (u16, Value) { + (200, json!({"data":{"apps":{"items":[]}}})) +} + +#[test] +fn store_launch_discovers_the_eligible_steam_client_and_carries_the_exact_target() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![ + platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + ), + platform_client("platform-epic", "Epic", "EPIC", "999", json!("AVAILABLE")), + ]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + detail_game("platform-steam", "12345", "PLAYABLE"), + ], + |index, request| match index { + 0 => assert!(request.starts_with("GET /vpc ")), + 1 => { + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + let query = payload["query"].as_str().unwrap(); + assert!(query.contains("PLATFORM_CLIENT")); + assert!(query.contains("appStore")); + assert!(query.contains("HERO_IMAGE")); + assert!(query.contains("filters")); + assert!(!query.contains("appIds")); + } + 2 => assert!(request.starts_with("GET /subscription")), + 7 => { + let payload: Value = + serde_json::from_str(request.split("\r\n\r\n").nth(1).unwrap()).unwrap(); + assert_eq!(payload["variables"]["ids"], json!(["platform-steam"])); + } + _ => {} + }, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["store"], "STEAM"); + assert_eq!(result["appId"], "platform-steam"); + assert_eq!(result["variantId"], "12345"); + assert_eq!(result["game"]["id"], "platform-steam"); + assert_eq!(result["game"]["title"], "Steam"); + assert_eq!(result["decision"]["status"], "ready"); + assert_eq!(result["catalogRevision"], 0); + assert_eq!(result["freshness"], "fresh"); + assert_eq!(result["scope"]["userId"], "fixture-account"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_requires_the_persistent_storage_addon() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(false), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + detail_game("platform-steam", "12345", "PLAYABLE"), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "subscription_required"); + assert_eq!(result["appId"], "platform-steam"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_ephemeral_only_storage_does_not_authorize_the_store_launch() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + ( + 200, + json!({"allottedTimeInMinutes":600.0,"membershipTier":"ULTIMATE","type":"SUBSCRIPTION", + "subType":"UNLIMITED","addons":[{"type":"STORAGE","subType":"EPHEMERAL_STORAGE","status":"OK"}], + "features":{"resolutions":[]},"currentSubscriptionState":{"state":"ACTIVE","isGamePlayAllowed":true}, + "notifications":{}}), + ), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + detail_game("platform-steam", "12345", "PLAYABLE"), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "subscription_required"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_reports_unavailable_without_an_eligible_steam_variant() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-epic", + "Epic", + "EPIC", + "999", + json!("AVAILABLE"), + )]), + subscription(true), + ], + |index, _| assert!(index < 3), + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "unavailable"); + assert!(result["appId"].is_null()); + assert!(result["variantId"].is_null()); + assert!(result["game"].is_null()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_reports_unavailable_without_platform_clients() { + let (url, worker) = mock_requests( + vec![vpc(), store_apps(vec![]), subscription(true)], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "unavailable"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_rejects_unsupported_stores_and_non_boolean_intent() { + let (url, worker) = mock_requests(vec![], |_, _| {}); + let (service, path) = service(&url); + assert_eq!( + service + .store_launch_inspect(&json!({"store":"EPIC"}), None, &json!({})) + .unwrap_err() + .code, + "invalid_params" + ); + assert_eq!( + service + .store_launch_inspect(&json!({"store":7}), None, &json!({})) + .unwrap_err() + .code, + "invalid_params" + ); + assert_eq!( + store_launch::store_launch_intent(&json!({"storeLaunch":"yes"})) + .unwrap_err() + .code, + "invalid_params" + ); + assert!(!store_launch::store_launch_intent(&json!({})).unwrap()); + assert!(!store_launch::store_launch_intent(&json!({"storeLaunch":false})).unwrap()); + assert!(store_launch::store_launch_intent(&json!({"storeLaunch":true})).unwrap()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_enforces_variant_readiness_from_the_discovery_source() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("PATCHING"), + )]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + no_game(), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "patching"); + assert_eq!(result["variantId"], "12345"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_reports_unconfirmed_readiness_when_the_status_is_missing() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![ + json!({"id":"platform-steam","title":"Steam","images":{}, + "variants":[{"appStore":"STEAM","id":"12345","gfn":{}}],"gfn":{"playType":"PLATFORM_CLIENT"}}), + ]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + no_game(), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "metadata_unconfirmed"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_falls_back_to_discovery_metadata_when_the_exact_game_is_not_browsable() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + no_game(), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "ready"); + assert_eq!(result["appId"], "platform-steam"); + assert_eq!(result["variantId"], "12345"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_blocks_explicit_unplayability_from_the_exact_resolution() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + detail_game("platform-steam", "12345", "UNPLAYABLE_DUE_TO_UPGRADE"), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "subscription_required"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_requires_a_linked_store_account_when_the_store_requires_it() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + ( + 200, + json!({"data":{"appStoreDefinitions":[{"store":"STEAM","label":"Steam","features":[], + "accountLinkingMetadata":{"isRequired":true,"isSupported":true,"supportedVariantIds":["12345"]}}]}}), + ), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + ( + 200, + json!({"data":{"userAccount":{"storesData":[{"store":"STEAM","isConnected":false,"status":"disconnected"}],"subscriptions":[]}}}), + ), + detail_game("platform-steam", "12345", "PLAYABLE"), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "link_required"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn targeted_store_launch_requires_the_exact_platform_client_and_variant() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), Some(("platform-epic", "999")), &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "metadata_unconfirmed"); + assert_eq!(result["appId"], "platform-epic"); + assert_eq!(result["variantId"], "999"); + assert!(result["game"].is_null()); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_requires_an_identifier_and_the_supported_store_for_the_exact_variant() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![ + json!({"id":"platform-steam","title":"Steam","images":{}, + "variants":[{"appStore":"STEAM","id":"not-numeric","gfn":{"status":"AVAILABLE"}}], + "gfn":{"playType":"PLATFORM_CLIENT"}}), + ]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + no_game(), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "metadata_unconfirmed"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_rejects_a_changed_membership_context() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + ], + move |index, _| { + if index != 2 { + return; + } + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + }, + ); + let (service, path) = service(&url); + std::thread::scope(|threads| { + let pending = threads.spawn(|| service.store_launch_inspect(&json!({}), None, &json!({}))); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + service.state.lock().unwrap().generation += 1; + release_tx.send(()).unwrap(); + assert_eq!(pending.join().unwrap().unwrap_err().code, "stale_account"); + }); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_never_borrows_a_registered_launch_target() { + let game = json!({"id":"platform-steam","title":"Steam", + "variants":[{"id":"12345","store":"STEAM","gfnStatus":"AVAILABLE"}]}); + let subscription = + json!({"storageAddon":{"type":"PERMANENT_STORAGE"},"isGamePlayAllowed":true}); + let access = json!({"definitions":{"stores":{"status":"success"}}, + "accounts":[{"provider":"STEAM","isRequired":false,"supportsLinking":true,"isConnected":true}], + "subscriptions":[]}); + assert_eq!( + store_launch::store_launch_decision( + &game, + "platform-steam", + "999", + "STEAM", + &subscription, + &access, + false, + ) + .status, + LaunchStatus::MetadataUnconfirmed + ); + assert_eq!( + store_launch::store_launch_decision( + &game, + "other-app", + "12345", + "STEAM", + &subscription, + &access, + false, + ) + .status, + LaunchStatus::MetadataUnconfirmed + ); + let epic_variant = json!({"id":"999","store":"EPIC","gfnStatus":"AVAILABLE"}); + let games = vec![ + json!({"id":"platform-epic","variants":[epic_variant.clone()]}), + json!({"id":"platform-steam","variants":game["variants"].clone()}), + ]; + let (owner, variant) = + store_launch::eligible_store_target(&games, "STEAM").expect("steam client present"); + assert_eq!(owner["id"], "platform-steam"); + assert_eq!(variant["id"], "12345"); + assert!(store_launch::eligible_store_target(&games, "XBOX").is_none()); + assert_eq!( + store_launch::store_launch_decision( + &game, + "platform-steam", + "12345", + "EPIC", + &subscription, + &access, + false, + ) + .status, + LaunchStatus::MetadataUnconfirmed + ); + assert_eq!( + store_launch::store_launch_decision( + &game, + "platform-steam", + "12345", + "STEAM", + &subscription, + &access, + false, + ) + .status, + LaunchStatus::Ready + ); +} + +#[test] +fn store_launch_preserves_explicit_gameplay_denial_without_a_membership_tier() { + let (_, mut subscription) = subscription(true); + subscription["currentSubscriptionState"]["isGamePlayAllowed"] = json!(false); + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + (200, subscription), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + no_game(), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service + .store_launch_inspect(&json!({}), None, &json!({})) + .unwrap(); + assert_eq!(result["decision"]["status"], "subscription_required"); + assert_eq!(result["appId"], "platform-steam"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_propagates_detail_failures_that_are_not_missing_games() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + (503, json!({})), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + let result = service.store_launch_inspect(&json!({}), None, &json!({})); + assert_eq!(result.unwrap_err().code, "upstream_error"); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn store_launch_preserves_the_active_seat_guard() { + let (url, worker) = mock_requests( + vec![ + vpc(), + store_apps(vec![platform_client( + "platform-steam", + "Steam", + "STEAM", + "12345", + json!("AVAILABLE"), + )]), + subscription(true), + store_definitions()[0].clone(), + store_definitions()[1].clone(), + store_definitions()[2].clone(), + store_definitions()[3].clone(), + detail_game("platform-steam", "12345", "PLAYABLE"), + ], + |_, _| {}, + ); + let (service, path) = service(&url); + service + .cloudmatch + .seed_owned_session(json!({"sessionId":"owned-seat","status":3})); + let result = service.create_session( + &json!({"appId":"12345","variantId":"12345","catalogAppId":"platform-steam", + "storeLaunch":true, + "scope":scoped_result(json!({}), &auth_fixture("fixture-account"), 0)["scope"]}), + &json!({}), + |settings| Ok(settings.clone()), + ); + assert_eq!(result.unwrap_err().code, "session_update_busy"); + assert_eq!( + service.cloudmatch.active()["session"]["sessionId"], + "owned-seat" + ); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} + +#[test] +fn subscription_reports_missing_remaining_playtime_as_unknown() { + let (_, mut missing) = subscription(false); + missing + .as_object_mut() + .unwrap() + .remove("remainingTimeInMinutes"); + let (url, worker) = mock_requests(vec![vpc(), subscription(false), (200, missing)], |_, _| {}); + let (service, path) = service(&url); + let known = service.subscription(&json!({})).unwrap(); + assert_eq!(known["subscription"]["remainingHours"], json!(500.0 / 60.0)); + assert_eq!(known["subscription"]["usedHours"], json!(100.0 / 60.0)); + let unknown = service.subscription(&json!({})).unwrap(); + assert_eq!(unknown["subscription"]["remainingHours"], Value::Null); + assert_eq!(unknown["subscription"]["usedHours"], Value::Null); + assert_eq!(unknown["subscription"]["totalHours"], json!(10.0)); + worker.join().unwrap(); + std::fs::remove_dir_all(path).unwrap(); +} diff --git a/external/opennow/native/opennow-core/src/language.rs b/external/opennow/native/opennow-core/src/language.rs new file mode 100644 index 000000000..9bcb4a908 --- /dev/null +++ b/external/opennow/native/opennow-core/src/language.rs @@ -0,0 +1,161 @@ +use serde_json::{Value, json}; + +const KEYBOARDS: &[(&str, &str, &[&str])] = &[ + ("en-US", "English (US)", &[]), + ("en-GB", "English (UK)", &[]), + ("tr-TR", "Turkish Q", &[]), + ("de-DE", "German", &[]), + ("fr-FR", "French", &[]), + ("es-ES_tradnl", "Spanish (traditional)", &["es-ES"]), + ("es-MX", "Spanish (Latin America)", &[]), + ("it-IT", "Italian", &[]), + ("pt-PT", "Portuguese (Portugal)", &[]), + ("pt-BR", "Portuguese (Brazil)", &[]), + ("pl-PL", "Polish", &[]), + ("da-DK", "Danish", &[]), + ("nb-NO", "Norwegian", &[]), + ("sv-SE", "Swedish", &[]), + ("fi-FI", "Finnish", &[]), + ("ru-RU", "Russian", &[]), + ("uk-UA", "Ukrainian", &[]), + ("ja-106", "Japanese 106", &["ja-JP", "Japanese106"]), + ("ko-KR", "Korean", &[]), + ("zh-CN", "Chinese (Simplified)", &[]), + ("zh-TW", "Chinese (Traditional)", &[]), +]; + +pub fn valid_game_language(value: &str) -> bool { + if value.len() > 64 || matches!(value.to_ascii_lowercase().as_str(), "auto" | "system") { + return false; + } + let mut parts = value.split(['-', '_']); + let first = parts.next().unwrap_or_default(); + (2..=8).contains(&first.len()) + && first.bytes().all(|byte| byte.is_ascii_alphabetic()) + && parts.all(|part| { + (1..=8).contains(&part.len()) && part.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }) +} + +fn keyboard_wire_id(value: &str) -> Option<&'static str> { + KEYBOARDS + .iter() + .find_map(|(id, _, aliases)| (*id == value || aliases.contains(&value)).then_some(*id)) +} + +pub fn validate_setting(key: &str, value: &Value) -> Result<(), String> { + let valid = value.as_str().is_some_and(|value| match key { + "gameLanguage" => valid_game_language(value), + "keyboardLayout" => keyboard_wire_id(value).is_some(), + _ => true, + }); + if valid { + Ok(()) + } else { + Err(format!("Invalid {key} identifier")) + } +} + +pub fn keyboard_choices() -> Value { + json!( + KEYBOARDS + .iter() + .map(|(id, label, aliases)| { json!({"value":id, "label":label, "aliases":aliases}) }) + .collect::>() + ) +} + +pub fn session_keyboard_layout(settings: &Value) -> &'static str { + settings["keyboardLayout"] + .as_str() + .and_then(keyboard_wire_id) + .unwrap_or("en-US") +} + +pub fn append_session_preferences(url: &mut url::Url, settings: &Value) { + let language = settings["gameLanguage"] + .as_str() + .filter(|value| valid_game_language(value)) + .unwrap_or("en_US"); + url.query_pairs_mut() + .append_pair("keyboardLayout", session_keyboard_layout(settings)) + .append_pair("languageCode", language); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn game_identifiers_preserve_wire_spelling_and_allow_safe_future_values() { + for value in [ + "en_US", + "es_419", + "zh_Hant_TW", + "sr-Latn-RS", + "future_001", + "EN-us", + ] { + assert!(valid_game_language(value), "{value}"); + } + for value in [ + "", "auto", "AUTO", "system", "en US", "en\nUS", "en__US", "en/US", "1en", "en_", "éé", + ] { + assert!(!valid_game_language(value), "{value:?}"); + } + assert!(!valid_game_language(&format!( + "en{}", + "_abcdefgh".repeat(8) + ))); + } + + #[test] + fn keyboard_table_and_legacy_aliases_have_one_wire_identity() { + for (id, _, aliases) in KEYBOARDS { + assert_eq!(keyboard_wire_id(id), Some(*id)); + for alias in *aliases { + assert_eq!(keyboard_wire_id(alias), Some(*id)); + } + } + assert_eq!(keyboard_wire_id("ja-JP"), Some("ja-106")); + assert_eq!(keyboard_wire_id("Japanese106"), Some("ja-106")); + assert_eq!(keyboard_wire_id("es-ES"), Some("es-ES_tradnl")); + for value in ["auto", "system", "en_US", "m-us", "unknown"] { + assert!(keyboard_wire_id(value).is_none()); + } + } + + #[test] + fn request_preferences_are_independent_and_corrupt_restore_is_safe() { + for (settings, language, keyboard) in [ + ( + json!({"appLanguage":"ja", "gameLanguage":"es_419", "keyboardLayout":"de-DE"}), + "es_419", + "de-DE", + ), + ( + json!({"gameLanguage":"zh_Hant_TW", "keyboardLayout":"ja-JP"}), + "zh_Hant_TW", + "ja-106", + ), + ( + json!({"gameLanguage":"system", "keyboardLayout":"m-us"}), + "en_US", + "en-US", + ), + ( + json!({"gameLanguage":"auto", "keyboardLayout":"es-ES"}), + "en_US", + "es-ES_tradnl", + ), + ] { + let mut url = url::Url::parse("https://fixture.invalid/session?existing=1").unwrap(); + append_session_preferences(&mut url, &settings); + let pairs: std::collections::HashMap<_, _> = url.query_pairs().collect(); + assert_eq!(pairs["languageCode"], language); + assert_eq!(pairs["keyboardLayout"], keyboard); + assert_eq!(pairs["existing"], "1"); + assert_eq!(pairs.len(), 3); + } + } +} diff --git a/external/opennow/native/opennow-core/src/lib.rs b/external/opennow/native/opennow-core/src/lib.rs new file mode 100644 index 000000000..1b928ad36 --- /dev/null +++ b/external/opennow/native/opennow-core/src/lib.rs @@ -0,0 +1,2 @@ +pub mod push; +pub mod update_apply; diff --git a/external/opennow/native/opennow-core/src/main.rs b/external/opennow/native/opennow-core/src/main.rs new file mode 100644 index 000000000..eec276816 --- /dev/null +++ b/external/opennow/native/opennow-core/src/main.rs @@ -0,0 +1,1735 @@ +#![recursion_limit = "512"] + +mod account_connections; +mod analytics; +mod artwork_cache; +mod bug_reports; +mod catalog_types; +mod cloudmatch; +mod community; +mod console_profiles; +mod credential_vault; +mod device_identity; +mod diagnostics; +mod discord; +mod dlss; +mod frame_rate; +mod gfn; +mod language; +mod media; +mod network; +mod network_test; +mod persistent_storage; +mod proxy; +mod push_registry; +mod queue_servers; +mod reports; +mod requests; +mod server_vpc_cache; +mod settings; +mod store_cache; +mod store_catalog_page; +mod store_index; +mod store_requests; +mod streamer; +mod thanks; +mod updater; +mod version; + +use fs2::FileExt; +use gfn::GfnService; +use opennow_core::update_apply; +use rand::RngCore; +use serde_json::{Map, Value, json}; +use settings::{SettingsStore, resolve_data_dir}; +use std::env; +use std::io::{self, BufRead, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, mpsc}; +use std::thread; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use streamer::StreamerService; + +const PROTOCOL_VERSION: i64 = 5; +const MAXIMUM_LINE_BYTES: usize = 1024 * 1024; +static PROFILE_LOCK: OnceLock = OnceLock::new(); + +struct AppCore { + session_update_gate: Mutex<()>, + artwork: artwork_cache::ArtworkCache, + settings: Mutex, + gfn: Arc, + streamer: StreamerService, + diagnostics: diagnostics::DiagnosticsService, + media: media::MediaService, + updater: updater::UpdaterService, + push: Mutex, + community: community::CommunityService, + thanks: thanks::ThanksService, + discord: discord::DiscordService, + reports: reports::ReportsClient, + analytics: Arc, + pending_reports_retried: AtomicBool, + bug_reports: bug_reports::BugReporter, +} + +fn main() { + if let Err(error) = run() { + eprintln!("opennow-core: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let data_dir = resolve_data_dir(argument_value("--data-dir").map(PathBuf::from)); + if env::args_os().any(|argument| argument == "--graphics-preferences") { + let windows_gpu_device_id = SettingsStore::windows_gpu_device_id_read_only(Some(data_dir)) + .map_err(|error| error.to_string())?; + let stdout = io::stdout(); + return write_json( + &mut stdout.lock(), + &json!({"version":1,"windowsGpuDeviceId":windows_gpu_device_id}), + ); + } + std::fs::create_dir_all(&data_dir) + .map_err(|error| format!("Could not initialize the data directory: {error}"))?; + let profile_lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(data_dir.join("core.lock")) + .map_err(|error| format!("Could not open the data directory lock: {error}"))?; + profile_lock.try_lock_exclusive().map_err(|error| { + if error.raw_os_error() == fs2::lock_contended_error().raw_os_error() { + "The OpenNOW data directory is already in use".to_owned() + } else { + format!("Could not lock the data directory: {error}") + } + })?; + PROFILE_LOCK.get_or_init(|| profile_lock); + let (output_tx, output_rx) = mpsc::channel::(); + thread::Builder::new() + .name("opennow-core-writer".to_owned()) + .spawn(move || { + let stdout = io::stdout(); + let mut output = stdout.lock(); + for value in output_rx { + if let Err(error) = write_json(&mut output, &value) { + eprintln!("opennow-core: output failed: {error}"); + break; + } + } + }) + .map_err(|error| error.to_string())?; + let reports_api = reports::api_base_from_env(); + let gfn = Arc::new(GfnService::new(data_dir.clone())?); + let push = Mutex::new(push_registry::PushRegistry::new( + Arc::clone(&gfn), + output_tx.clone(), + data_dir.clone(), + )); + let core = Arc::new(AppCore { + session_update_gate: Mutex::new(()), + artwork: artwork_cache::ArtworkCache::new(&data_dir, output_tx.clone()), + settings: Mutex::new( + SettingsStore::load(Some(data_dir.clone())).map_err(|error| error.to_string())?, + ), + gfn, + streamer: StreamerService::new(), + diagnostics: diagnostics::DiagnosticsService::new(&data_dir) + .map_err(|error| format!("Could not initialize diagnostics: {error}"))?, + media: media::MediaService::new() + .map_err(|error| format!("Could not initialize media library: {error}"))?, + updater: updater::UpdaterService::new(&data_dir) + .map_err(|error| format!("Could not initialize updater: {error}"))?, + push, + community: community::CommunityService::new() + .map_err(|error| format!("Could not initialize community services: {error}"))?, + thanks: thanks::ThanksService::new() + .map_err(|error| format!("Could not initialize acknowledgements: {error}"))?, + discord: discord::DiscordService::new(), + reports: reports::ReportsClient::new(&reports_api, &data_dir) + .map_err(|error| format!("Could not initialize reporting services: {error}"))?, + analytics: Arc::new( + analytics::AnalyticsService::new(&reports_api, &data_dir) + .map_err(|error| format!("Could not initialize analytics: {error}"))?, + ), + pending_reports_retried: AtomicBool::new(false), + bug_reports: bug_reports::BugReporter::default(), + }); + if !reporting_enabled(&core) { + core.analytics.wipe(); + core.reports.clear_pending(); + } + let analytics_finished = core.analytics.start()?; + reconcile_push(&core); + let requests = Arc::new(requests::Requests::default()); + let stdin = io::stdin(); + + for line in stdin.lock().lines() { + let line = line.map_err(|error| error.to_string())?; + if line.len() > MAXIMUM_LINE_BYTES { + return Err("protocol line exceeds the size limit".to_owned()); + } + let message: Value = match serde_json::from_str(&line) { + Ok(value) => value, + Err(_) => return Err("malformed JSON protocol message".to_owned()), + }; + if message["type"] == "cancel" { + if let Some(id) = message["id"].as_str() { + requests.cancel(id); + } + continue; + } + if message["type"] == "ack" { + if let Some(id) = message["id"].as_str() { + requests.acknowledge(id); + } + continue; + } + if message["type"] != "request" { + return Err("unknown protocol message".to_owned()); + } + let id = message["id"].as_str().unwrap_or_default().to_owned(); + let method = message["method"].as_str().unwrap_or_default().to_owned(); + let params = message.get("params").cloned().unwrap_or_else(|| json!({})); + if id.is_empty() || method.is_empty() { + output_tx.send(json!({"type":"response", "id":id, "ok":false, "error":{"code":"invalid_request", "message":"Request requires string id and method"}})) + .map_err(|error| error.to_string())?; + continue; + } + let Some(permit) = requests.admit(&id, &method) else { + output_tx.send(json!({"type":"response", "id":id, "ok":false, "error":{"code":"busy", "message":"Core request limit reached"}})) + .map_err(|error| error.to_string())?; + continue; + }; + + let worker_core = Arc::clone(&core); + let worker_output = output_tx.clone(); + thread::Builder::new().name(format!("opennow-rpc-{id}")).spawn(move || { + let started = Instant::now(); + let result = requests::scope(permit.token.clone(), || { + requests::check().map_err(|error| (error.code.to_owned(), error.message))?; + dispatch(&method, ¶ms, &worker_core, &worker_output).map(|(mut value, event)| { + if method.starts_with("catalog.") { + let settings = worker_core.settings.lock().expect("settings poisoned").all(); + worker_core.gfn.annotate_dlss(&mut value, &settings); + } + (value, event) + }) + }); + let outcome = match &result { + Ok(_) => "outcome=ok".to_owned(), + Err((code, message)) => match upstream_http_status(message) { + Some(status) => format!("outcome={code} httpStatus={status}"), + None => format!("outcome={code}"), + }, + }; + worker_core.diagnostics.record( + "rpc", + &method, + format!("{outcome} durationMs={}", started.elapsed().as_millis()), + ); + observe_reporting(&worker_core, &worker_output, &method, ¶ms, &result); + let was_cancelled = permit.token.cancelled(); + if method == "session.create" + && let Err((code, message)) = &result + && code == "session_cleanup_pending" + { + let _ = worker_output.send(json!({"type":"event","name":"session.cleanup.pending", + "payload":{"code":code,"message":message}})); + } + if method == "session.create" + && let Ok((value, _)) = &result + && let Some(session_id) = value["session"]["sessionId"].as_str() + { + let delivered = !was_cancelled && worker_output.send(json!({"type":"response", "id":id, "ok":true, "result":value})).is_ok(); + let accepted = delivered && permit.token.await_acceptance(std::time::Duration::from_secs(10)); + let cleanup = worker_core.gfn.finish_session_create(session_id, accepted); + worker_core.diagnostics.record("session", "allocation-handoff", format!( + "accepted={accepted} cleanup={}", cleanup.as_ref().map_or_else(|error| error.code, |()| "ok") + )); + if let Err(error) = cleanup { + let _ = worker_output.send(json!({"type":"event","name":"session.cleanup.pending","payload":{ + "sessionId":session_id,"code":error.code,"message":"The cancelled cloud session could not be closed. End it before starting another game." + }})); + } + return; + } + if matches!(method.as_str(), "updater.check" | "updater.download" | "updater.install") { + if let Err((_, message)) = &result { + worker_core.updater.request_failed(message); + } + let _ = worker_output.send(json!({"type":"event", "name":"updater.changed", "payload":worker_core.updater.state()})); + } + if matches!( + method.as_str(), + "auth.device.complete" + | "auth.logout" + | "auth.accounts.logoutAll" + | "auth.accounts.switch" + | "auth.accounts.remove" + | "settings.set" + ) { + reconcile_push(&worker_core); + } + if !was_cancelled { + match result { + Ok((value, event)) => { + if let Some(("settings.changed", payload)) = &event { + let _ = worker_output.send(json!({"type":"event", "name":"settings.changed", "payload":payload})); + } + let _ = worker_output.send(json!({"type":"response", "id":id, "ok":true, "result":value})); + if let Some((name, payload)) = event && name != "settings.changed" { + let _ = worker_output.send(json!({"type":"event", "name":name, "payload":payload})); + } + } + Err((code, message)) => { + let _ = worker_output.send(json!({"type":"response", "id":id, "ok":false, "error":{"code":code, "message":message}})); + } + } + } + drop(permit); + }).map_err(|error| error.to_string())?; + } + core.analytics.shutdown(&analytics_finished); + Ok(()) +} + +type DispatchResult = Result<(Value, Option<(&'static str, Value)>), (String, String)>; + +fn update_session_idle(session: &Value, streamer: &Value) -> bool { + session.get("session") == Some(&Value::Null) + && matches!( + streamer["streamer"]["status"].as_str(), + Some("stopped" | "error") + ) +} + +fn reconcile_push(core: &AppCore) { + let _ = core + .push + .lock() + .expect("push registry poisoned") + .reconcile(); +} + +fn unix_time_millis() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +fn acceptance_window_system(params: &Value) -> Result { + let value = params["windowSystem"] + .as_str() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + let valid = match std::env::consts::OS { + "linux" => matches!(value.as_str(), "xcb" | "wayland"), + "windows" => value == "windows", + "macos" => value == "cocoa", + _ => false, + }; + if !valid { + return Err(( + "acceptance_platform_invalid".to_owned(), + "Live acceptance requires the native X11, Wayland, Win32, or AppKit platform" + .to_owned(), + )); + } + Ok(value) +} + +fn acceptance_shell_evidence(params: &Value) -> Result { + let source = params["shell"].as_object().ok_or_else(|| { + ( + "acceptance_shell_invalid".to_owned(), + "Live acceptance requires bounded shell recovery and guide evidence".to_owned(), + ) + })?; + let counter = |key: &str| { + source + .get(key) + .and_then(Value::as_u64) + .filter(|value| *value <= 100) + .ok_or_else(|| { + ( + "acceptance_shell_invalid".to_owned(), + format!("Live acceptance shell field {key} is missing or out of range"), + ) + }) + }; + let streamer_recovery_count = counter("streamerRecoveryCount")?; + let session_recovery_count = counter("sessionRecoveryCount")?; + let pages = source + .get("guidePagesVisited") + .and_then(Value::as_array) + .ok_or_else(|| { + ( + "acceptance_shell_invalid".to_owned(), + "Live acceptance requires the visited guide page list".to_owned(), + ) + })?; + let allowed = [ + "guide-session", + "guide-controls", + "guide-media", + "guide-shortcuts", + ]; + let mut visited = pages + .iter() + .filter_map(Value::as_str) + .filter(|page| allowed.contains(page)) + .map(ToOwned::to_owned) + .collect::>(); + visited.sort(); + visited.dedup(); + Ok(json!({ + "streamerRecoveryCount": streamer_recovery_count, + "sessionRecoveryCount": session_recovery_count, + "guidePagesVisited": visited, + "allGuidePagesVisited": visited.len() == allowed.len() + })) +} + +fn dispatch( + method: &str, + params: &Value, + core: &Arc, + output: &mpsc::Sender, +) -> DispatchResult { + let session_transition = matches!( + method, + "session.create" | "session.claim" | "session.poll" | "streamer.start" | "streamer.prepare" + ); + let _session_update_guard = if session_transition || method == "updater.install" { + Some(core.session_update_gate.try_lock().map_err(|_| { + ( + "session_update_busy".to_owned(), + "A session transition or update preparation is in progress".to_owned(), + ) + })?) + } else { + None + }; + if session_transition && core.updater.installation_pending() { + return Err(( + "update_pending".to_owned(), + "An update is waiting for OpenNOW to exit".to_owned(), + )); + } + match method { + "core.hello" => { + if params["protocolVersion"].as_i64() != Some(PROTOCOL_VERSION) { + return Err(( + "incompatible_protocol".to_owned(), + "Shell and core protocol versions differ".to_owned(), + )); + } + Ok(( + json!({"protocolVersion":PROTOCOL_VERSION, "coreVersion":version::APPLICATION_VERSION, "capabilities":["settings", "gfn.deviceAuth", "gfn.providers", "gfn.publicCatalog", "catalog.storePages.v1", "catalog.libraryPages.v1", "catalog.metadata.v1", "account.syncObservation.v1", "account.pushInvalidation.v1", "catalog.languages.v1", "queue.servers.v1", "auth.accountPicker.v1", "catalog.storeLocal.v1", "gfn.accountLibrary", "gfn.regions", "gfn.subscription", "gfn.cloudmatch", "sessionProxy", "session.terminalCleanup.v1", "catalogArtworkCache.v1", "nativeStreamer.v7", "nativeStreamer.ownedNvstNegotiation", "nativeStreamer.dynamicSurface", "nativeStreamer.acceptanceEvidence", "liveAcceptance.v1", "osCredentialStore", "electronAccountMigration", "redactedDiagnostics", "mediaLibrary", "githubUpdateDiscovery", "discordRpc", "feedback", "bugReports", "automaticBugReports.v2", "social.capabilitySurface"]}), + None, + )) + } + "app.status" => Ok(( + json!({"status":"ready", "version":version::APPLICATION_VERSION}), + None, + )), + "social.capabilities.get" => Ok(( + json!({ + "friendsAvailable":false, + "presenceAvailable":false, + "invitesAvailable":false, + "localControllerJoin":true, + "reason":"NVIDIA does not expose the GeForce NOW friends, presence, or invitation service to third-party clients. OpenNOW will not display invented contacts or claim invitations were sent." + }), + None, + )), + "settings.get" => Ok(( + json!({"settings":core.settings.lock().expect("settings poisoned").all(), + "keyboardLayouts":language::keyboard_choices()}), + None, + )), + "settings.choices.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + Ok(( + json!({"colorQualities":streamer::StreamerService::color_quality_choices( + &settings, ¶ms["runtimeCapabilities"]), + "codecs":streamer::StreamerService::codec_choices( + &settings, ¶ms["runtimeCapabilities"]), + "frameRates":frame_rate::frame_rate_choices( + &settings, ¶ms["runtimeCapabilities"])}), + None, + )) + } + "settings.set" => { + let key = params["key"].as_str().ok_or(( + "invalid_params".to_owned(), + "settings.set requires a key".to_owned(), + ))?; + let value = params.get("value").cloned().ok_or(( + "invalid_params".to_owned(), + "settings.set requires a value".to_owned(), + ))?; + if key == "region" { + let provider = params["providerIdpId"].as_str().unwrap_or(""); + let event = core.gfn.with_region_provider(provider, || { + let mut settings = core.settings.lock().expect("settings poisoned"); + let applied = settings.set_provider_region(provider, value).map_err(|message| gfn::ServiceError { code: "invalid_setting", message })?; + Ok(json!({"key":key,"value":applied,"changes":{ + "regionProviderIdpId":provider,"providerRegions":settings.all()["providerRegions"] + }})) + }).map_err(gfn_error)?; + return Ok((event.clone(), Some(("settings.changed", event)))); + } + let mut settings = core.settings.lock().expect("settings poisoned"); + let codec_before = settings.all()["codec"].clone(); + let fallback_before = settings.all()["fallbackCodec"].clone(); + let applied = settings + .set(key, value) + .map_err(|message| ("invalid_setting".to_owned(), message))?; + let mut event = json!({"key":key, "value":applied}); + if key == "colorQuality" { + // An explicitly saved codec the new color mode cannot use is + // healed toward Auto in the same save; report the repair so the + // shell follows it without re-reading all settings. + let current = settings.all(); + let mut changes = Map::new(); + if current["codec"] != codec_before { + changes.insert("codec".to_owned(), current["codec"].clone()); + } + if current["fallbackCodec"] != fallback_before { + changes.insert("fallbackCodec".to_owned(), current["fallbackCodec"].clone()); + } + if !changes.is_empty() { + event["changes"] = Value::Object(changes); + } + } + if key == "launchInConsoleMode" && applied == json!(false) { + event["changes"] = json!({"switchToConsoleOnPad": false}); + } else if key == "themePack" { + let values = settings.all(); + event["changes"] = json!({ + "appTheme": values["appTheme"], + "themeAccentOverride": values["themeAccentOverride"] + }); + } else if key == "appAccentColor" { + event["changes"] = json!({"themeAccentOverride": true}); + } else if key == "webrtcCompatibilityMode" { + event["changes"] = json!({"allianceWebrtcCompatibility": settings.all()["allianceWebrtcCompatibility"]}); + } else if key == "allianceWebrtcCompatibility" { + event["changes"] = + json!({"webrtcCompatibilityMode": settings.all()["webrtcCompatibilityMode"]}); + } + if key == "microphoneMode" && applied == json!("voice-activity") { + event["changes"] = json!({"microphoneDeviceId": ""}); + } + Ok((event.clone(), Some(("settings.changed", event)))) + } + "settings.shortcuts.update" => { + let bindings = core + .settings + .lock() + .expect("settings poisoned") + .set_shortcuts(¶ms["bindings"]) + .map_err(|message| ("invalid_setting".to_owned(), message))?; + let (key, value) = bindings + .iter() + .next() + .map(|(key, value)| (key.clone(), value.clone())) + .expect("shortcut transaction applies at least one binding"); + let event = json!({"key":key, "value":value, "changes":bindings}); + Ok(( + json!({"bindings":bindings}), + Some(("settings.changed", event)), + )) + } + "settings.reset" => { + let values = core + .settings + .lock() + .expect("settings poisoned") + .reset() + .map_err(|message| ("settings_write_failed".to_owned(), message))?; + Ok(( + json!({"settings":values}), + Some(("settings.reset", json!({}))), + )) + } + "auth.providers.list" => core + .gfn + .providers() + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.device.start" => core + .gfn + .start_device_login(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.device.poll" => core + .gfn + .poll_device_login(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.device.complete" => core + .gfn + .complete_device_login(params) + .map(|value| (value.clone(), Some(("auth.session.changed", value)))) + .map_err(gfn_error), + "auth.device.cancel" => core + .gfn + .cancel_device_login(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.session.get" => core + .gfn + .session() + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.logout" => { + let value = core.gfn.logout().map_err(gfn_error)?; + Ok((value.clone(), Some(("auth.session.changed", value)))) + } + "auth.accounts.logoutAll" => { + let value = core.gfn.logout_all().map_err(gfn_error)?; + Ok((value.clone(), Some(("auth.session.changed", value)))) + } + "auth.accounts.list" => core + .gfn + .saved_accounts() + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.accounts.switch" => core + .gfn + .switch_account(params) + .map(|value| (value.clone(), Some(("auth.session.changed", value)))) + .map_err(gfn_error), + "auth.accounts.remove" => core + .gfn + .remove_account(params) + .map(|value| (value.clone(), Some(("auth.session.changed", value)))) + .map_err(gfn_error), + "auth.pin.status" => core + .gfn + .pin_status(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.pin.set" => core + .gfn + .set_pin(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.pin.clear" => core + .gfn + .clear_pin(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "auth.pin.verify" => core + .gfn + .verify_pin(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "catalog.public.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .public_catalog(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.library.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .library_catalog(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.game.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_game(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.launch.inspect" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_launch_inspect(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.launch.store.inspect" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .store_launch_inspect(params, None, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.favorites.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_favorites(&settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.favorites.add" + | "catalog.favorites.remove" + | "catalog.ownership.add" + | "catalog.ownership.remove" + | "catalog.ownership.select" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_mutate(method, params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.definitions.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_definitions(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.languages.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .catalog_languages(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.store.local" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .store_local_catalog(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.store.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .store_catalog(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "catalog.store.presentation" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .store_presentation(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "artwork.resolve" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.artwork + .resolve(params, &settings) + .map(|value| (value, None)) + .map_err(|message| ("invalid_params".to_owned(), message)) + } + "network.regions.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .regions(&settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "network.regions.ping" => network::ping_regions(params) + .map(|value| (value, None)) + .map_err(|message| ("region_ping_failed".to_owned(), message)), + "queue.servers.list" => queue_servers::list() + .map(|value| (value, None)) + .map_err(gfn_error), + "account.subscription.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .subscription(&settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .account_connections(&settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.sync" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .sync_account_connection(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.unlink" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .unlink_account_connection(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.link.start" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .start_account_link(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.link.poll" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .poll_account_link(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.sync.status" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .account_sync_status(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.connections.sync.cancel" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .account_sync_status( + &json!({"operationId":params["operationId"],"cancelObservation":true}), + &settings, + ) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "account.storage.locations" => core + .gfn + .persistent_storage_locations(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "account.storage.reset" => core + .gfn + .reset_persistent_storage(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "session.create" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + let (auth, _) = core + .gfn + .authenticated_snapshot(gfn::TokenPurpose::ServiceId, false) + .map_err(gfn_error)?; + let settings = cloudmatch::allocation_settings(&settings, &auth); + let streamer_service_error = |error: streamer::StreamerError| gfn::ServiceError { + code: error.code, + message: error.message, + }; + core.gfn + .create_session(params, &settings, |settings| { + if params["runtimeCapabilities"].is_null() { + core.streamer + .validate_codec(settings) + .map_err(streamer_service_error)?; + return Ok(settings.clone()); + } + let (resolved, fallbacks) = StreamerService::resolve_embedded_session( + settings, + ¶ms["runtimeCapabilities"], + ) + .map_err(streamer_service_error)?; + for fallback in fallbacks { + core.diagnostics + .record("streamer", "stream_profile_fallback", fallback); + } + Ok(resolved) + }) + .map(|value| (value.clone(), Some(("session.changed", value)))) + .map_err(gfn_error) + } + "session.poll" => core + .gfn + .poll_session(params) + .map(|value| { + ( + value.clone(), + (params["recoveryMode"] != true).then_some(("session.changed", value)), + ) + }) + .map_err(gfn_error), + "session.stop" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .stop_session(params, &settings) + .map(|value| (value.clone(), Some(("session.changed", value)))) + .map_err(gfn_error) + } + "session.ended" => core + .gfn + .acknowledge_session_end(params) + .map(|value| (value, None)) + .map_err(gfn_error), + "session.active.get" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .reconcile_active_session(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "session.remote.list" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .remote_sessions(params, &settings) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "session.claim" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.gfn + .claim_session(params, &settings) + .map(|value| (value.clone(), Some(("session.changed", value)))) + .map_err(gfn_error) + } + "session.ad.report" => core + .gfn + .report_session_ad(params) + .map(|value| (value.clone(), Some(("session.changed", value)))) + .map_err(gfn_error), + "streamer.detect" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.streamer + .detect(&settings) + .map(|value| (value, None)) + .map_err(streamer_error) + } + "streamer.start" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + core.streamer + .start(params, &settings) + .map(|value| (value.clone(), Some(("streamer.changed", value)))) + .map_err(streamer_error) + } + "streamer.prepare" => { + let settings = core.settings.lock().expect("settings poisoned").all(); + if params["session"]["transportMode"] == "webrtc" { + core.diagnostics.record( + "streamer", + "prepare_endpoints", + diagnostics::stream_endpoint_evidence(¶ms["session"]).to_string(), + ); + } + core.gfn + .prepare_owned_stream(params, |owned| { + core.streamer + .prepare_embedded(owned, &settings) + .map_err(|error| gfn::ServiceError { + code: error.code, + message: error.message, + }) + }) + .inspect(|prepared| { + core.diagnostics.record( + "streamer", + "prepare_profile", + diagnostics::stream_profile_evidence(&prepared["context"]["session"]) + .to_string(), + ); + }) + .inspect_err(|error| { + core.diagnostics.record( + "streamer", + "prepare_profile", + diagnostics::stream_profile_evidence(¶ms["session"]).to_string(), + ); + core.diagnostics.record( + "streamer", + "prepare_rejected", + diagnostics::runtime_failure_reason(&error.message), + ); + }) + .map(|value| (value, None)) + .map_err(gfn_error) + } + "streamer.status.get" => Ok((core.streamer.status(), None)), + "streamer.stop" => core + .streamer + .stop(params["reason"].as_str().unwrap_or("session stopped")) + .map(|value| (value.clone(), Some(("streamer.changed", value)))) + .map_err(streamer_error), + "streamer.input.pause" => core + .streamer + .set_input_paused(params["paused"].as_bool().unwrap_or(true)) + .map(|value| (value, None)) + .map_err(streamer_error), + "streamer.control" => core + .streamer + .control(params["action"].as_str().unwrap_or_default()) + .map(|value| (value, None)) + .map_err(streamer_error), + "streamer.recording.start" => { + let validated = core + .media + .validate_recording_target(params) + .map_err(|message| ("media_recording_target_invalid".to_owned(), message))?; + core.streamer + .recording(&validated, true) + .map(|value| (value, None)) + .map_err(streamer_error) + } + "streamer.recording.stop" => core + .streamer + .recording(params, false) + .map(|value| (value.clone(), Some(("media.changed", value)))) + .map_err(streamer_error), + "streamer.surface.update" => core + .streamer + .update_surface(params) + .map(|value| (value, None)) + .map_err(streamer_error), + "diagnostics.snapshot" => Ok((core.diagnostics.snapshot(), None)), + "diagnostics.export" => { + let runtime = json!({ + "schemaVersion": 1, + "kind": "opennow.acceptance", + "generatedAtMs": unix_time_millis().to_string(), + "applicationVersion": version::APPLICATION_VERSION, + "os": std::env::consts::OS, + "cpuArchitecture": std::env::consts::ARCH, + "streamer": core.streamer.acceptance_snapshot(), + "nativeRuntime": diagnostics::native_runtime_evidence(¶ms["runtimeCapabilities"]), + "shell": diagnostics::embedded_drop_evidence(params) + }); + core.diagnostics + .export_with_runtime(Some(&runtime)) + .map(|value| (value, None)) + .map_err(|error| ("diagnostics_export_failed".to_owned(), error.to_string())) + } + "acceptance.export" => { + let window_system = acceptance_window_system(params)?; + let streamer = core.streamer.acceptance_snapshot(); + let session_started_at_ms = streamer["sessionStartedAtMs"] + .as_str() + .and_then(|value| value.parse::().ok()); + let media = core + .media + .acceptance_evidence(session_started_at_ms) + .map_err(|message| ("acceptance_media_failed".to_owned(), message))?; + let shell = acceptance_shell_evidence(params)?; + let checks = json!({ + "streamingTenMinutes": streamer["status"] == "streaming" + && streamer["sessionUptimeMs"].as_u64().unwrap_or_default() >= 600_000, + "firstFramePresented": streamer["firstFrameLatencyMs"].is_number() + && streamer["mediaBackend"].is_string(), + "nvstTransportActive": streamer["transport"] == "nvst", + "nativeInputReady": streamer["inputReady"].as_bool().unwrap_or(false), + "inputOwnershipExercised": streamer["inputPauseCount"].as_u64().unwrap_or_default() > 0 + && streamer["inputResumeCount"].as_u64().unwrap_or_default() > 0, + "allGuidePagesVisited": shell["allGuidePagesVisited"].as_bool().unwrap_or(false), + "surfaceReconfigured": streamer["surfaceUpdateCount"].as_u64().unwrap_or_default() > 0, + "fullscreenControlExercised": streamer["fullscreenToggleCount"].as_u64().unwrap_or_default() > 0, + "statsControlExercised": streamer["statsToggleCount"].as_u64().unwrap_or_default() > 0, + "recordingRoundTrip": streamer["recordingStartCount"].as_u64().unwrap_or_default() > 0 + && streamer["recordingStopCount"].as_u64().unwrap_or_default() > 0, + "mediaArtifactsComplete": media["complete"].as_bool().unwrap_or(false), + "networkRecoveryExercised": shell["sessionRecoveryCount"].as_u64().unwrap_or_default() > 0, + "streamerRecoveryExercised": shell["streamerRecoveryCount"].as_u64().unwrap_or_default() > 0, + "noTerminalMediaError": streamer["errorCode"].is_null() + && streamer["decoderErrorCount"].as_u64().unwrap_or_default() == 0 + && streamer["outputErrorCount"].as_u64().unwrap_or_default() == 0, + "deviceRecoveryBalanced": streamer["deviceLossCount"].as_u64().unwrap_or_default() + <= streamer["deviceRecoveryCount"].as_u64().unwrap_or_default() + }); + let observed_pass = checks + .as_object() + .is_some_and(|values| values.values().all(|value| value.as_bool() == Some(true))); + let manifest = json!({ + "schemaVersion": 1, + "kind": "opennow.live-acceptance", + "generatedAtMs": unix_time_millis().to_string(), + "applicationVersion": version::APPLICATION_VERSION, + "platform": { + "os": std::env::consts::OS, + "cpuArchitecture": std::env::consts::ARCH, + "windowSystem": window_system + }, + "stream": streamer, + "shell": shell, + "media": media, + "checks": checks, + "observedPass": observed_pass, + "scope": "machine-observed-live-runtime" + }); + core.diagnostics + .export_acceptance(&manifest) + .map(|value| (value, None)) + .map_err(|error| ("acceptance_export_failed".to_owned(), error.to_string())) + } + "media.root.get" => core + .media + .root() + .map(|value| (value, None)) + .map_err(|message| ("media_unavailable".to_owned(), message)), + "media.recording.target" => core + .media + .recording_target(params) + .map(|value| (value, None)) + .map_err(|message| ("media_recording_target_failed".to_owned(), message)), + "media.list" => core + .media + .list(params) + .map(|value| (value, None)) + .map_err(|message| ("media_list_failed".to_owned(), message)), + "media.delete" => core + .media + .delete(params) + .map(|value| (value.clone(), Some(("media.changed", value)))) + .map_err(|message| ("media_delete_failed".to_owned(), message)), + "cache.delete" => Ok((core.gfn.clear_cache(), Some(("cache.changed", json!({}))))), + "queue.status.get" => core + .community + .queue() + .map(|value| (value, None)) + .map_err(|message| ("queue_fetch_failed".to_owned(), message)), + "queue.serverMapping.get" => core + .community + .server_mapping() + .map(|value| (value, None)) + .map_err(|message| ("server_mapping_fetch_failed".to_owned(), message)), + "thanks.data.get" => Ok((core.thanks.data(), None)), + "communityProxy.provision" => core + .community + .provision_proxy(core.gfn.device_id()) + .map(|value| (value, None)) + .map_err(|message| ("community_proxy_failed".to_owned(), message)), + "updater.state.get" => Ok((core.updater.state(), None)), + "updater.startup.ack" => { + update_apply::acknowledge_startup_from_env(version::APPLICATION_VERSION) + .map(|acknowledged| (json!({"acknowledged":acknowledged}), None)) + .map_err(|message| ("update_startup_ack_failed".to_owned(), message)) + } + "updater.check" => { + let value = core + .updater + .check(params) + .map_err(|message| ("update_check_failed".to_owned(), message))?; + Ok((value, None)) + } + "updater.highlights.get" => { + let highlights = core.updater.highlights(); + let seen = core + .settings + .lock() + .expect("settings poisoned") + .all()["lastSeenReleaseHighlightsVersion"] + .as_str() + .unwrap_or_default() + .to_owned(); + // Historical notes remain readable without announcing an older + // release as an available update or navigating away from About. + let unseen = core.updater.state()["availableVersion"] + .as_str() + .is_some_and(|version| { + !version.is_empty() + && version != seen + && highlights["version"].as_str() == Some(version) + }); + Ok(( + highlights.clone(), + unseen.then_some(("updater.highlights.show", highlights)), + )) + } + "updater.highlights.ack" => { + let highlights = core.updater.highlights(); + let version = params["version"] + .as_str() + .or_else(|| highlights["version"].as_str()) + .unwrap_or(version::APPLICATION_VERSION) + .trim_start_matches('v') + .to_owned(); + if version.is_empty() || version.len() > 128 { + return Err(( + "invalid_version".to_owned(), + "Release version is invalid".to_owned(), + )); + } + core.settings + .lock() + .expect("settings poisoned") + .set("lastSeenReleaseHighlightsVersion", json!(version.clone())) + .map_err(|message| ("settings_write_failed".to_owned(), message))?; + Ok((json!({"acknowledged":true,"version":version}), None)) + } + "updater.download" => core + .updater + .download(&|state| { + let _ = + output.send(json!({"type":"event", "name":"updater.changed", "payload":state})); + }) + .map(|value| (value, None)) + .map_err(|message| ("update_download_failed".to_owned(), message)), + "updater.install" => { + let session = core.gfn.active_session().map_err(gfn_error)?; + if !update_session_idle(&session, &core.streamer.status()) { + return Err(( + "update_session_active".to_owned(), + "End the active session before installing an update".to_owned(), + )); + } + core.updater + .install(params) + .map(|value| (value, None)) + .map_err(|message| ("update_install_failed".to_owned(), message)) + } + "discord.activity.sync" => core + .discord + .sync(params) + .map(|value| (value, None)) + .map_err(|message| ("discord_rpc_failed".to_owned(), message)), + "discord.activity.clear" => core + .discord + .clear() + .map(|value| (value, None)) + .map_err(|message| ("discord_rpc_failed".to_owned(), message)), + "feedback.submit" => { + let install_id = ensure_install_id(core)?; + let category = params["category"].as_str().unwrap_or("other"); + if !matches!(category, "bug" | "idea" | "other") { + return Err(( + "feedback_failed".to_owned(), + "Unsupported feedback category".to_owned(), + )); + } + let message = reports::required_text(¶ms["message"], 8, 4_000, "Feedback") + .map_err(|message| ("feedback_failed".to_owned(), message))?; + let include = params["includeSystemInfo"].as_bool() == Some(true); + let report = reports::Document { + install_id: &install_id, + account: &Value::Null, + activity: if include { + core.bug_reports.activity() + } else { + json!({"currentGame":null, "recentGames":[]}) + }, + app: report_app(core, include), + } + .manual(&format!("Feedback: {category}"), &message); + match upload_report(core, &report, None) { + Ok(accepted) => Ok(( + json!({"submitted":true, "reportId":accepted.report_id, + "message":"Thanks — your feedback was sent."}), + None, + )), + Err(failure) if failure.retryable => Ok(( + json!({"submitted":false, "queued":true, + "message":"Your feedback will be sent the next time OpenNOW starts."}), + None, + )), + Err(failure) => Err(("feedback_failed".to_owned(), failure.message)), + } + } + "bug_report.submit" => { + let install_id = ensure_install_id(core)?; + let title = reports::required_text(¶ms["title"], 8, 120, "Bug report title") + .map_err(|message| ("bug_report_failed".to_owned(), message))?; + let description = reports::required_text( + ¶ms["description"], + 40, + 12_000, + "Bug report description", + ) + .map_err(|message| ("bug_report_failed".to_owned(), message))?; + let log = if params["includeDiagnostics"].as_bool() == Some(true) { + let export = core + .diagnostics + .export() + .map_err(|error| ("diagnostics_export_failed".to_owned(), error.to_string()))?; + export["path"] + .as_str() + .and_then(|path| reports::gzip_log(Path::new(path))) + } else { + None + }; + let report = reports::Document { + install_id: &install_id, + account: &core.gfn.bug_report_identity(), + activity: core.bug_reports.activity(), + app: report_app(core, true), + } + .manual(&title, &description); + match upload_report(core, &report, log) { + Ok(accepted) => Ok(( + json!({"submitted":true, "reportId":accepted.report_id, + "issueStatus":accepted.issue_status}), + None, + )), + Err(failure) if failure.retryable => { + Ok((json!({"submitted":false, "queued":true}), None)) + } + Err(failure) => Err(("bug_report_failed".to_owned(), failure.message)), + } + } + "analytics.track" => { + if let Some(surface) = analytics::ui_surface(params) + .map_err(|message| ("invalid_params".to_owned(), message))? + { + core.analytics.set_ui_surface(surface); + } + if params["event"] == "app_opened" { + return Ok(( + match core.analytics.open_app(¶ms["runtimeCapabilities"]) { + Some(props) => track(core, "app_opened", Value::Object(props)), + None => json!({"accepted":false,"reason":"already_sent"}), + }, + None, + )); + } + let event = analytics::shell_event(params) + .map_err(|message| ("invalid_params".to_owned(), message))?; + Ok((track(core, event.name, Value::Object(event.props)), None)) + } + "bug_report.incident" => { + let incident = bug_reports::BugReporter::shell_incident(params) + .map_err(|message| ("invalid_params".to_owned(), message))?; + Ok((submit_automatic_report(core, output, incident), None)) + } + _ => Err(( + "method_not_found".to_owned(), + format!("Unknown core method: {method}"), + )), + } +} + +fn observe_reporting( + core: &Arc, + output: &mpsc::Sender, + method: &str, + params: &Value, + result: &DispatchResult, +) { + match (method, result) { + ("session.create", _) => { + let detail = core.bug_reports.observe_launch(params); + core.diagnostics.record("activity", "game_launch", detail); + track( + core, + "game_launch_requested", + json!({"game_id":params["appId"], "game_title":params["title"], + "store":params["store"], "zone":params["zone"]}), + ); + } + ("session.stop", Ok(_)) => core.bug_reports.observe_stop(), + ("streamer.prepare", _) => core + .analytics + .observe_runtime(¶ms["runtimeCapabilities"]), + ("auth.device.complete", Ok(_)) => observe_sign_in(core, false), + ("auth.session.get" | "auth.accounts.switch", Ok(_)) => observe_sign_in(core, true), + ("auth.logout" | "auth.accounts.logoutAll", Ok(_)) => core.analytics.observe_sign_out(), + ("settings.set", Ok((value, _))) if params["key"] == "automaticBugReports" => { + observe_consent(core, &value["value"], ¶ms["source"]) + } + _ => {} + } + if let Err((code, message)) = result + && let Some(incident) = bug_reports::BugReporter::rpc_incident(method, code, message) + { + if incident.kind == bug_reports::Kind::LibraryError { + track(core, "library_load_failed", json!({"code":incident.code})); + } else { + let game_id = core.bug_reports.activity()["currentGame"]["appId"].clone(); + track( + core, + "session_error", + json!({"stage":method.split('.').nth(1), "code":incident.code, "game_id":game_id}), + ); + } + submit_automatic_report(core, output, incident); + } +} + +fn reporting_enabled(core: &AppCore) -> bool { + bug_reports::enabled(&core.settings.lock().expect("settings poisoned").all()) +} + +fn track(core: &AppCore, event: &str, props: Value) -> Value { + if !reporting_enabled(core) { + return json!({"accepted":false,"reason":"disabled"}); + } + let install_id = match ensure_install_id(core) { + Ok(install_id) => install_id, + Err((_, message)) => return json!({"accepted":false,"reason":message}), + }; + let identity = core.gfn.bug_report_identity(); + let account = if identity.is_null() { + Value::Null + } else { + json!({"userId":identity["userId"], "providerIdpId":identity["providerIdpId"]}) + }; + let dropped = core.analytics.enqueue( + &install_id, + account, + event, + analytics::sanitize(event, &props), + ); + if dropped % 100 == 1 { + core.diagnostics + .record("analytics", "queue-overflow", format!("dropped={dropped}")); + } + json!({"accepted":true}) +} + +fn observe_consent(core: &AppCore, value: &Value, source: &Value) { + let Some(value) = value.as_str().filter(|value| *value != "unset") else { + return; + }; + if value == "disabled" { + core.analytics.wipe(); + core.reports.clear_pending(); + } + let Ok(install_id) = ensure_install_id(core) else { + return; + }; + core.analytics.enqueue( + &install_id, + Value::Null, + "consent_changed", + analytics::sanitize( + "consent_changed", + &json!({"value":value, "source":source.as_str().unwrap_or("settings")}), + ), + ); +} + +fn observe_sign_in(core: &Arc, restored: bool) { + let identity = core.gfn.bug_report_identity(); + let Some(user_id) = identity["userId"].as_str() else { + return; + }; + if !core.analytics.observe_sign_in(user_id) { + return; + } + track( + core, + "signed_in", + json!({"provider_code":identity["providerCode"], "alliance_partner":identity["alliancePartner"], + "membership_tier":identity["membershipTier"], "restored":restored}), + ); + if !reporting_enabled(core) || core.pending_reports_retried.swap(true, Ordering::AcqRel) { + return; + } + let worker_core = Arc::clone(core); + let _ = thread::Builder::new() + .name("opennow-report-retry".to_owned()) + .spawn(move || { + for (report, accepted) in worker_core.reports.retry_pending(unix_time_millis()) { + record_sent(&worker_core, &report, &accepted); + } + }); +} + +fn report_app(core: &AppCore, include_device: bool) -> Value { + let device = if include_device { + core.analytics.device() + } else { + json!({"gpu":null, "decoderBackend":null}) + }; + let backend = core.streamer.acceptance_snapshot()["mediaBackend"] + .as_str() + .filter(|backend| include_device && analytics::valid_code(backend)) + .map_or_else(|| device["decoderBackend"].clone(), Value::from); + json!({ + "version": version::APPLICATION_VERSION, + "os": std::env::consts::OS, + "arch": std::env::consts::ARCH, + "gpu": device["gpu"], + "decoderBackend": backend, + "channel": version::update_channel(version::APPLICATION_VERSION) + }) +} + +fn upload_report( + core: &AppCore, + report: &Value, + log: Option>, +) -> Result { + let result = core.reports.submit(report, log.as_deref()); + match &result { + Ok(accepted) => record_sent(core, report, accepted), + Err(failure) => { + let spooled = failure.retryable + && core + .reports + .spool(report, log.as_deref(), unix_time_millis()) + .is_ok(); + core.diagnostics.record( + "bug-report", + "upload-failed", + format!("retryable={} spooled={spooled}", failure.retryable), + ); + } + } + result +} + +fn record_sent(core: &AppCore, report: &Value, accepted: &reports::Accepted) { + core.diagnostics.record( + "bug-report", + "sent", + format!("reportId={}", accepted.report_id), + ); + let automatic = report["automatic"] == true; + let (kind, code) = if automatic { + ( + report["trigger"]["kind"].clone(), + report["trigger"]["code"].clone(), + ) + } else { + (json!("manual"), json!("manual")) + }; + track( + core, + "bug_report_sent", + json!({"report_id":accepted.report_id, "issue_id":accepted.issue_id, + "kind":kind, "code":code, "automatic":automatic}), + ); +} + +fn submit_automatic_report( + core: &Arc, + output: &mpsc::Sender, + incident: bug_reports::Incident, +) -> Value { + if !reporting_enabled(core) { + return json!({"accepted":false,"reason":"disabled"}); + } + let account = core.gfn.bug_report_identity(); + if account.is_null() { + return json!({"accepted":false,"reason":"signed_out"}); + } + let Some(activity) = core.bug_reports.admit(&incident, Instant::now()) else { + return json!({"accepted":false,"reason":"already_reported"}); + }; + let install_id = match ensure_install_id(core) { + Ok(install_id) => install_id, + Err((_, message)) => return json!({"accepted":false,"reason":message}), + }; + core.diagnostics.record( + "bug-report", + "automatic", + format!("kind={} code={}", incident.kind.as_str(), incident.code), + ); + let report = reports::Document { + install_id: &install_id, + account: &account, + activity: activity.clone(), + app: report_app(core, true), + } + .automatic(incident.trigger()); + let mut payload = json!({"state":"sending", "kind":incident.kind.as_str(), + "code":incident.code, "game":activity["currentGame"]["title"]}); + let _ = + output.send(json!({"type":"event","name":"bug_report.changed","payload":payload.clone()})); + let worker_core = Arc::clone(core); + let worker_output = output.clone(); + let spawned = thread::Builder::new() + .name("opennow-bug-report".to_owned()) + .spawn(move || { + let core = worker_core; + let runtime = json!({ + "schemaVersion": 1, + "kind": "opennow.automatic-bug-report", + "generatedAtMs": unix_time_millis().to_string(), + "applicationVersion": version::APPLICATION_VERSION, + "os": std::env::consts::OS, + "cpuArchitecture": std::env::consts::ARCH, + "trigger": incident.trigger(), + "activity": activity, + "streamer": core.streamer.acceptance_snapshot() + }); + let log = core + .diagnostics + .export_with_runtime(Some(&runtime)) + .ok() + .and_then(|export| { + let path = PathBuf::from(export["path"].as_str()?); + let log = reports::gzip_log(&path); + let _ = std::fs::remove_file(path); + log + }); + match upload_report(&core, &report, log) { + Ok(accepted) => { + payload["state"] = json!("sent"); + payload["reportId"] = json!(accepted.report_id); + payload["issueStatus"] = json!(accepted.issue_status); + } + Err(failure) => { + payload["state"] = json!("failed"); + payload["message"] = json!(failure.message); + payload["queued"] = json!(failure.retryable); + } + } + core.diagnostics.record( + "bug-report", + "automatic-result", + format!("state={}", payload["state"].as_str().unwrap_or_default()), + ); + let _ = worker_output + .send(json!({"type":"event","name":"bug_report.changed","payload":payload})); + }); + if spawned.is_err() { + return json!({"accepted":false,"reason":"Could not start the report upload"}); + } + json!({"accepted":true}) +} + +fn ensure_install_id(core: &AppCore) -> Result { + let mut settings = core.settings.lock().expect("settings poisoned"); + let current = settings.all()["telemetryInstallId"] + .as_str() + .unwrap_or_default() + .replace('-', ""); + let install_id = if reports::valid_install_id(¤t) { + current + } else { + let mut bytes = [0_u8; 16]; + rand::rng().fill_bytes(&mut bytes); + bytes.iter().map(|value| format!("{value:02x}")).collect() + }; + settings + .set("telemetryInstallId", json!(install_id.clone())) + .map_err(|message| ("settings_write_failed".to_owned(), message))?; + Ok(install_id) +} + +fn upstream_http_status(message: &str) -> Option { + let (_, detail) = message.split_once(" (")?; + let status = detail.get(..3)?.parse::().ok()?; + (matches!(detail.as_bytes().get(3), Some(b' ' | b')')) && (100..=599).contains(&status)) + .then_some(status) +} + +fn gfn_error(error: gfn::ServiceError) -> (String, String) { + (error.code.to_owned(), error.message) +} + +fn streamer_error(error: streamer::StreamerError) -> (String, String) { + (error.code.to_owned(), error.message) +} + +fn write_json(output: &mut impl Write, value: &Value) -> Result<(), String> { + serde_json::to_writer(&mut *output, value).map_err(|error| error.to_string())?; + output + .write_all(b"\n") + .and_then(|_| output.flush()) + .map_err(|error| error.to_string()) +} + +fn argument_value(name: &str) -> Option { + let arguments: Vec = env::args().collect(); + arguments + .windows(2) + .find(|pair| pair[0] == name) + .map(|pair| pair[1].clone()) +} + +#[cfg(test)] +mod acceptance_tests { + use super::*; + + #[test] + fn rpc_diagnostics_keep_only_the_upstream_http_status() { + for (message, expected) in [ + ( + format!( + "Subscription request failed ({})", + reqwest::StatusCode::NOT_FOUND + ), + Some(404), + ), + ( + format!( + "Session polling failed ({}): INTERNAL_ERROR_STATUS C2EC1025", + reqwest::StatusCode::INTERNAL_SERVER_ERROR + ), + Some(500), + ), + ("Server info failed (408)".to_owned(), Some(408)), + ( + "Server info failed: error sending request (dns)".to_owned(), + None, + ), + ("Subscription request failed (4040 x)".to_owned(), None), + ("Subscription request failed (099 x)".to_owned(), None), + ( + "Membership details could not be confirmed.".to_owned(), + None, + ), + ] { + assert_eq!(upstream_http_status(&message), expected, "{message}"); + } + } + + #[test] + fn updates_require_no_session_and_a_terminal_streamer() { + for status in ["stopped", "error"] { + assert!(update_session_idle( + &json!({"session":null}), + &json!({"streamer":{"status":status}}) + )); + } + for status in [ + "starting", + "streaming", + "recovering", + "negotiating", + "unknown", + ] { + assert!(!update_session_idle( + &json!({"session":null}), + &json!({"streamer":{"status":status}}) + )); + } + for session in [ + json!({}), + json!({"session":{"phase":"queued"}}), + json!({"session":{"phase":"ready"}}), + ] { + assert!(!update_session_idle( + &session, + &json!({"streamer":{"status":"stopped"}}) + )); + } + } + + #[test] + fn shell_acceptance_evidence_is_bounded_normalized_and_complete() { + let evidence = acceptance_shell_evidence(&json!({"shell":{ + "streamerRecoveryCount":1, + "sessionRecoveryCount":2, + "guidePagesVisited":["guide-shortcuts","unknown","guide-session", + "guide-controls","guide-media","guide-session"] + }})) + .unwrap(); + assert_eq!(evidence["allGuidePagesVisited"], true); + assert_eq!(evidence["guidePagesVisited"].as_array().unwrap().len(), 4); + assert!( + acceptance_shell_evidence(&json!({"shell":{ + "streamerRecoveryCount":101, + "sessionRecoveryCount":0, + "guidePagesVisited":[] + }})) + .is_err() + ); + } + + #[test] + fn acceptance_platform_rejects_headless_and_accepts_only_the_native_plugin() { + assert!(acceptance_window_system(&json!({"windowSystem":"offscreen"})).is_err()); + let expected = match std::env::consts::OS { + "linux" => "wayland", + "windows" => "windows", + "macos" => "cocoa", + _ => return, + }; + assert_eq!( + acceptance_window_system(&json!({"windowSystem":expected})).unwrap(), + expected + ); + } +} diff --git a/external/opennow/native/opennow-core/src/media.rs b/external/opennow/native/opennow-core/src/media.rs new file mode 100644 index 000000000..f687609be --- /dev/null +++ b/external/opennow/native/opennow-core/src/media.rs @@ -0,0 +1,552 @@ +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::env; +use std::ffi::OsString; +use std::fs; +use std::io::{self, BufReader, Read}; +use std::path::{Component, Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; +use url::Url; + +const LIST_LIMIT: usize = 100; + +const MEDIA_UNAVAILABLE: &str = "No pictures directory is available for the media library"; + +pub struct MediaService { + root: Option, +} + +impl MediaService { + pub fn new() -> io::Result { + let Some(root) = pictures_directory() else { + return Ok(Self { root: None }); + }; + let root = root.join("OpenNOW"); + fs::create_dir_all(root.join("Screenshots"))?; + fs::create_dir_all(root.join("Recordings"))?; + Ok(Self { root: Some(root) }) + } + + fn available_root(&self) -> Result<&PathBuf, String> { + self.root + .as_ref() + .ok_or_else(|| MEDIA_UNAVAILABLE.to_owned()) + } + + pub fn root(&self) -> Result { + Ok(json!({ "path": self.available_root()?.to_string_lossy() })) + } + + pub fn recording_target(&self, params: &Value) -> Result { + let title = params["gameTitle"] + .as_str() + .map(sanitized_title) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "opennow".to_owned()); + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let directory = self.directory_for_kind("recording")?; + for suffix in 0_u16..=999 { + let suffix = if suffix == 0 { + String::new() + } else { + format!("-{suffix}") + }; + let stem = format!("OpenNOW-{title}-{timestamp}{suffix}"); + let path = directory.join(format!("{stem}.mkv")); + let part = directory.join(format!(".{stem}.mkv.part")); + if !path.exists() && !part.exists() { + return Ok(json!({ + "path":path.to_string_lossy(), + "thumbnailPath":directory.join(format!("{stem}-thumb.jpg")).to_string_lossy(), + })); + } + } + Err("Could not allocate a unique recording file name".to_owned()) + } + + pub fn validate_recording_target(&self, params: &Value) -> Result { + let raw = params["outputPath"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| "Native recording requires an outputPath".to_owned())?; + let path = PathBuf::from(raw); + let file_name = path + .file_name() + .and_then(|value| value.to_str()) + .filter(|value| value.starts_with("OpenNOW-") && !value.contains("..")) + .ok_or_else(|| "Recording output file name is invalid".to_owned())?; + if path.extension().and_then(|value| value.to_str()) != Some("mkv") || path.exists() { + return Err("Recording output must be a new .mkv file".to_owned()); + } + let recordings = self.directory_for_kind("recording")?; + let expected_parent = recordings + .canonicalize() + .map_err(|error| format!("Could not validate recordings directory: {error}"))?; + let actual_parent = path + .parent() + .ok_or_else(|| "Recording output has no parent directory".to_owned())? + .canonicalize() + .map_err(|error| format!("Could not validate recording output directory: {error}"))?; + if actual_parent != expected_parent { + return Err("Recording output is outside the OpenNOW recordings directory".to_owned()); + } + let part = recordings.join(format!(".{file_name}.part")); + if part.exists() { + return Err("Recording output is already being written".to_owned()); + } + Ok(json!({"outputPath":path.to_string_lossy()})) + } + + pub fn list(&self, params: &Value) -> Result { + let filter = params["gameTitle"] + .as_str() + .map(sanitized_title) + .filter(|value| !value.is_empty()); + let mut entries = Vec::new(); + self.collect("screenshot", "Screenshots", &mut entries)?; + self.collect("recording", "Recordings", &mut entries)?; + if let Some(filter) = filter { + entries.retain(|entry| { + entry["fileName"] + .as_str() + .is_some_and(|name| name.to_ascii_lowercase().contains(&filter)) + }); + } + entries.sort_by_key(|entry| std::cmp::Reverse(entry["createdAtMs"].as_u64().unwrap_or(0))); + entries.truncate(LIST_LIMIT); + let screenshot_count = entries + .iter() + .filter(|entry| entry["kind"] == "screenshot") + .count(); + let recording_count = entries.len() - screenshot_count; + Ok(json!({ + "items": entries, + "screenshots": screenshot_count, + "recordings": recording_count, + "rootPath": self.available_root()?.to_string_lossy() + })) + } + + pub fn acceptance_evidence(&self, since_ms: Option) -> Result { + let listed = self.list(&json!({}))?; + let items = listed["items"] + .as_array() + .ok_or_else(|| "Media listing did not contain items".to_owned())?; + let after_start = |item: &&Value| { + since_ms.is_none_or(|minimum| { + u128::from(item["createdAtMs"].as_u64().unwrap_or_default()) >= minimum + }) + }; + let screenshot = items + .iter() + .filter(|item| item["kind"] == "screenshot") + .find(after_start) + .map(|item| self.evidence_for_item(item)) + .transpose()?; + let recording = items + .iter() + .filter(|item| item["kind"] == "recording") + .find(after_start) + .map(|item| self.evidence_for_item(item)) + .transpose()?; + let recording_has_thumbnail = recording + .as_ref() + .is_some_and(|item| item["thumbnail"].is_object()); + Ok(json!({ + "schemaVersion": 1, + "sinceMs": since_ms.map(|value| value.to_string()), + "screenshot": screenshot, + "recording": recording, + "complete": screenshot.is_some() && recording.is_some() && recording_has_thumbnail + })) + } + + pub fn delete(&self, params: &Value) -> Result { + if params["confirmed"].as_bool() != Some(true) { + return Err("Media deletion requires explicit confirmation".to_owned()); + } + let kind = params["kind"] + .as_str() + .ok_or_else(|| "media.delete requires kind".to_owned())?; + let id = safe_id(¶ms["id"])?; + let directory = self.directory_for_kind(kind)?; + let path = directory.join(id); + if !path.is_file() { + return Err("Media item was not found".to_owned()); + } + fs::remove_file(&path).map_err(|error| format!("Could not delete media: {error}"))?; + if kind == "recording" { + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + let thumbnail = directory.join(format!("{stem}-thumb.jpg")); + let _ = fs::remove_file(thumbnail); + } + Ok(json!({ "deleted": true, "id": id, "kind": kind })) + } + + fn collect(&self, kind: &str, folder: &str, output: &mut Vec) -> Result<(), String> { + let directory = self.available_root()?.join(folder); + let items = fs::read_dir(&directory) + .map_err(|error| format!("Could not read media directory: {error}"))?; + for item in items.flatten() { + let path = item.path(); + if !path.is_file() || !allowed_extension(kind, &path) || is_thumbnail(&path) { + continue; + } + let Ok(metadata) = item.metadata() else { + continue; + }; + let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + let created = metadata + .created() + .or_else(|_| metadata.modified()) + .unwrap_or(UNIX_EPOCH) + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let media_url = Url::from_file_path(&path) + .ok() + .map(|value| value.to_string()) + .unwrap_or_default(); + let thumbnail_url = if kind == "recording" { + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + let thumbnail = directory.join(format!("{stem}-thumb.jpg")); + thumbnail + .is_file() + .then(|| { + Url::from_file_path(thumbnail) + .ok() + .map(|value| value.to_string()) + }) + .flatten() + .unwrap_or_default() + } else { + media_url.clone() + }; + output.push(json!({ + "id": file_name, + "kind": kind, + "fileName": file_name, + "filePath": path.to_string_lossy(), + "mediaUrl": media_url, + "thumbnailUrl": thumbnail_url, + "createdAtMs": created, + "sizeBytes": metadata.len() + })); + } + Ok(()) + } + + fn evidence_for_item(&self, item: &Value) -> Result { + let path = PathBuf::from( + item["filePath"] + .as_str() + .ok_or_else(|| "Media evidence path is unavailable".to_owned())?, + ); + let (size, digest) = digest_regular_file(&path)?; + let thumbnail = if item["kind"] == "recording" { + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + let path = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(format!("{stem}-thumb.jpg")); + if path.exists() { + let (thumbnail_size, thumbnail_digest) = digest_regular_file(&path)?; + Some(json!({ + "fileName": path.file_name().and_then(|value| value.to_str()).unwrap_or_default(), + "sizeBytes": thumbnail_size, + "sha256": thumbnail_digest + })) + } else { + None + } + } else { + None + }; + Ok(json!({ + "kind": item["kind"], + "fileName": item["fileName"], + "createdAtMs": item["createdAtMs"], + "sizeBytes": size, + "sha256": digest, + "thumbnail": thumbnail + })) + } + + fn directory_for_kind(&self, kind: &str) -> Result { + let root = self.available_root()?; + match kind { + "screenshot" => Ok(root.join("Screenshots")), + "recording" => Ok(root.join("Recordings")), + _ => Err("Unsupported media kind".to_owned()), + } + } +} + +fn digest_regular_file(path: &Path) -> Result<(u64, String), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("Could not inspect media evidence: {error}"))?; + if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { + return Err("Media evidence must be a regular, non-symlink file".to_owned()); + } + let file = + fs::File::open(path).map_err(|error| format!("Could not open media evidence: {error}"))?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = reader + .read(&mut buffer) + .map_err(|error| format!("Could not hash media evidence: {error}"))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok((metadata.len(), format!("{:x}", hasher.finalize()))) +} + +fn pictures_directory() -> Option { + #[cfg(target_os = "windows")] + let profile_root = env::var_os("USERPROFILE"); + #[cfg(not(target_os = "windows"))] + let profile_root = env::var_os("HOME"); + resolve_pictures_directory(env::var_os("OPENNOW_PICTURES_DIR"), profile_root) +} + +fn resolve_pictures_directory( + override_value: Option, + profile_root: Option, +) -> Option { + match override_value { + Some(value) if !value.is_empty() => normalize_root(PathBuf::from(value)), + Some(_) => None, + None => profile_root + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .filter(|path| path.is_absolute()) + .map(|path| path.join("Pictures")), + } +} + +fn normalize_root(path: PathBuf) -> Option { + std::path::absolute(&path) + .ok() + .filter(|resolved| resolved.is_absolute()) +} + +fn safe_id(value: &Value) -> Result<&str, String> { + let id = value + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| "media.delete requires an id".to_owned())?; + let path = Path::new(id); + if path.components().count() != 1 + || !matches!(path.components().next(), Some(Component::Normal(_))) + || id.contains("..") + { + return Err("Invalid media id".to_owned()); + } + Ok(id) +} + +fn allowed_extension(kind: &str, path: &Path) -> bool { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + match kind { + "screenshot" => matches!(extension.as_str(), "png" | "jpg" | "jpeg" | "webp"), + "recording" => matches!(extension.as_str(), "mp4" | "webm" | "mkv" | "mov" | "avi"), + _ => false, + } +} + +fn is_thumbnail(path: &Path) -> bool { + path.file_stem() + .and_then(|value| value.to_str()) + .is_some_and(|value| value.ends_with("-thumb")) +} + +fn sanitized_title(value: &str) -> String { + value + .to_ascii_lowercase() + .chars() + .map(|value| { + if value.is_ascii_alphanumeric() { + value + } else { + '-' + } + }) + .collect::() + .trim_matches('-') + .to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + use std::time::SystemTime; + + static ENVIRONMENT_LOCK: Mutex<()> = Mutex::new(()); + + fn lock_environment() -> std::sync::MutexGuard<'static, ()> { + ENVIRONMENT_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()) + } + + #[test] + fn pictures_root_prefers_an_explicit_override_and_rejects_blank_values() { + let captures = env::temp_dir().join("opennow-media-captures"); + assert_eq!( + resolve_pictures_directory(Some(captures.clone().into_os_string()), None), + Some(captures) + ); + let relative = resolve_pictures_directory(Some(OsString::from("relative/captures")), None) + .expect("a relative override must resolve to an absolute path"); + assert!(relative.is_absolute()); + assert!(relative.ends_with("relative/captures")); + + let home = env::temp_dir().join("opennow-media-home"); + assert_eq!( + resolve_pictures_directory(None, Some(home.clone().into_os_string())), + Some(home.join("Pictures")) + ); + assert_eq!( + resolve_pictures_directory(Some(OsString::new()), Some(home.clone().into_os_string())), + None + ); + assert_eq!( + resolve_pictures_directory(None, Some(OsString::from("home/user"))), + None + ); + assert_eq!( + resolve_pictures_directory(None, Some(OsString::new())), + None + ); + assert_eq!(resolve_pictures_directory(None, None), None); + } + + #[test] + fn override_normalization_is_always_absolute_or_unavailable() { + let absolute = env::temp_dir().join("opennow-media-absolute-root"); + assert_eq!(normalize_root(absolute.clone()), Some(absolute.clone())); + let relative = normalize_root(PathBuf::from("relative/root")) + .expect("a relative override must resolve to an absolute path"); + assert!(relative.is_absolute()); + assert!(relative.ends_with("relative/root")); + assert_eq!(normalize_root(PathBuf::new()), None); + } + + #[cfg(target_os = "windows")] + #[test] + fn windows_partial_overrides_resolve_to_fully_qualified_paths() { + for value in [r"C:Pictures", r"\Pictures"] { + let resolved = normalize_root(PathBuf::from(value)) + .expect("a partially rooted override must resolve on Windows"); + assert!(resolved.is_absolute()); + assert!(resolved.ends_with("Pictures")); + assert_ne!(resolved, PathBuf::from(value)); + } + } + + #[test] + fn media_service_reports_unavailable_root_without_failing_construction() { + let _guard = lock_environment(); + let previous = env::var_os("OPENNOW_PICTURES_DIR"); + unsafe { env::set_var("OPENNOW_PICTURES_DIR", "") }; + let service = MediaService::new().expect("an unavailable root must not fail construction"); + assert!(service.root().is_err()); + assert!(service.list(&json!({})).is_err()); + assert!( + service + .recording_target(&json!({"gameTitle":"Test"})) + .is_err() + ); + assert!( + service + .delete(&json!({"kind":"screenshot","id":"shot.png","confirmed":true})) + .is_err() + ); + match previous { + Some(value) => unsafe { env::set_var("OPENNOW_PICTURES_DIR", value) }, + None => unsafe { env::remove_var("OPENNOW_PICTURES_DIR") }, + } + } + + #[test] + fn media_listing_is_bounded_and_deletion_is_scoped() { + let _guard = lock_environment(); + let directory = env::temp_dir().join(format!( + "opennow-media-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + unsafe { env::set_var("OPENNOW_PICTURES_DIR", &directory) }; + let service = MediaService::new().unwrap(); + let root = service.available_root().unwrap().clone(); + let target = service + .recording_target(&json!({"gameTitle":"Test / Game"})) + .unwrap(); + let validated = service + .validate_recording_target(&json!({"outputPath":target["path"]})) + .unwrap(); + assert_eq!(validated["outputPath"], target["path"]); + assert!( + service + .validate_recording_target(&json!({"outputPath":directory.join("escape.mkv")})) + .is_err() + ); + fs::write(root.join("Screenshots/shot.png"), b"png").unwrap(); + fs::write(root.join("Recordings/OpenNOW-test.mkv"), b"mkv").unwrap(); + fs::write(root.join("Recordings/OpenNOW-test-thumb.jpg"), b"jpg").unwrap(); + fs::write(root.join("Screenshots/nope.txt"), b"no").unwrap(); + let listed = service.list(&json!({})).unwrap(); + assert_eq!(listed["items"].as_array().unwrap().len(), 2); + let evidence = service.acceptance_evidence(Some(0)).unwrap(); + assert_eq!(evidence["complete"], true); + assert_eq!(evidence["screenshot"]["sizeBytes"], 3); + assert_eq!( + evidence["screenshot"]["sha256"], + "8f8cbb7dcf46e0bc7d53265749a6c17d116093a6ba95e442764060c76fd4a86c" + ); + assert_eq!(evidence["recording"]["thumbnail"]["sizeBytes"], 3); + let rendered = evidence.to_string(); + assert!(!rendered.contains(directory.to_string_lossy().as_ref())); + assert!( + service + .delete(&json!({"kind":"screenshot","id":"../shot.png","confirmed":true})) + .is_err() + ); + assert!( + service + .delete(&json!({"kind":"screenshot","id":"shot.png"})) + .is_err() + ); + service + .delete(&json!({"kind":"screenshot","id":"shot.png","confirmed":true})) + .unwrap(); + assert!(!root.join("Screenshots/shot.png").exists()); + let _ = fs::remove_dir_all(directory); + unsafe { env::remove_var("OPENNOW_PICTURES_DIR") }; + } +} diff --git a/external/opennow/native/opennow-core/src/network.rs b/external/opennow/native/opennow-core/src/network.rs new file mode 100644 index 000000000..6358aa2bb --- /dev/null +++ b/external/opennow/native/opennow-core/src/network.rs @@ -0,0 +1,321 @@ +use serde_json::{Value, json}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::sync::{Arc, Mutex, OnceLock, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; +use url::Url; + +const MAX_REGIONS: usize = 32; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +const QUEUE_PROBE_TIMEOUT: Duration = Duration::from_millis(750); +const RESOLVER_WORKERS: usize = 32; + +struct ResolveJob { + host: String, + port: u16, + deadline: Instant, + cancellation: crate::requests::Cancellation, + result: mpsc::SyncSender>, +} + +fn resolver() -> Option<&'static mpsc::SyncSender> { + static RESOLVER: OnceLock>> = OnceLock::new(); + RESOLVER + .get_or_init(|| { + let (sender, receiver) = mpsc::sync_channel::(RESOLVER_WORKERS); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..RESOLVER_WORKERS { + let receiver = receiver.clone(); + thread::Builder::new() + .name(format!("queue-dns-{index}")) + .spawn(move || { + loop { + let job = match receiver.lock() { + Ok(receiver) => receiver.recv(), + Err(_) => return, + }; + let Ok(job) = job else { return }; + if job.cancellation.cancelled() || Instant::now() >= job.deadline { + continue; + } + let addresses = (job.host.as_str(), job.port) + .to_socket_addrs() + .map(|addresses| addresses.take(8).collect()) + .unwrap_or_default(); + let _ = job.result.try_send(addresses); + } + }) + .ok()?; + } + Some(sender) + }) + .as_ref() +} + +pub(super) fn resolve_bounded(host: String, port: u16) -> Option> { + let cancellation = crate::requests::current(); + if cancellation.cancelled() { + return None; + } + let deadline = Instant::now() + QUEUE_PROBE_TIMEOUT; + let (result, receive) = mpsc::sync_channel(1); + resolver()? + .try_send(ResolveJob { + host, + port, + deadline, + cancellation: cancellation.clone(), + result, + }) + .ok()?; + receive_addresses(&receive, deadline, &cancellation) +} + +fn receive_addresses( + receive: &mpsc::Receiver>, + deadline: Instant, + cancellation: &crate::requests::Cancellation, +) -> Option> { + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() || cancellation.cancelled() { + return None; + } + match receive.recv_timeout(remaining.min(Duration::from_millis(50))) { + Ok(addresses) => return Some(addresses), + Err(mpsc::RecvTimeoutError::Disconnected) => return None, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +pub(super) fn measure_queue_region(region_url: &str) -> Option { + if crate::requests::check().is_err() { + return None; + } + let parsed = Url::parse(region_url).ok()?; + let addresses = resolve_bounded( + parsed.host_str()?.to_owned(), + parsed.port_or_known_default()?, + )?; + let _ = bounded_tcp_ping(&addresses); + let mut samples = Vec::with_capacity(2); + for attempt in 0..2 { + if crate::requests::check().is_err() { + return None; + } + if attempt > 0 { + thread::sleep(Duration::from_millis(50)); + } + if let Some(sample) = bounded_tcp_ping(&addresses) { + samples.push(sample); + } + } + (!samples.is_empty()).then(|| samples.iter().sum::().div_ceil(samples.len() as u64)) +} + +fn bounded_tcp_ping(addresses: &[SocketAddr]) -> Option { + let started = Instant::now(); + for address in addresses { + let remaining = QUEUE_PROBE_TIMEOUT.saturating_sub(started.elapsed()); + if remaining.is_zero() || crate::requests::check().is_err() { + return None; + } + if TcpStream::connect_timeout(address, remaining).is_ok() { + return Some(started.elapsed().as_millis() as u64); + } + } + None +} + +pub fn ping_regions(params: &Value) -> Result { + let cancellation = crate::requests::current(); + cancellation.check().map_err(|error| error.message)?; + let regions = params["regions"] + .as_array() + .ok_or_else(|| "network.regions.ping requires a regions array".to_owned())?; + if regions.len() > MAX_REGIONS { + return Err(format!( + "At most {MAX_REGIONS} regions can be measured at once" + )); + } + let inputs = regions + .iter() + .map(|region| region["url"].as_str().unwrap_or_default().to_owned()) + .collect::>(); + let results = Arc::new(Mutex::new(vec![Value::Null; inputs.len()])); + thread::scope(|scope| { + for (index, region_url) in inputs.into_iter().enumerate() { + let output = Arc::clone(&results); + let cancellation = cancellation.clone(); + scope.spawn(move || { + let result = crate::requests::scope(cancellation, || measure_region(®ion_url)); + output.lock().expect("region ping output poisoned")[index] = result; + }); + } + }); + cancellation.check().map_err(|error| error.message)?; + let results = Arc::try_unwrap(results) + .map_err(|_| "Region measurement workers did not finish".to_owned())? + .into_inner() + .map_err(|_| "Region measurement output was poisoned".to_owned())?; + Ok(json!({"results":results})) +} + +fn measure_region(region_url: &str) -> Value { + if crate::requests::check().is_err() { + return Value::Null; + } + let endpoint = match region_endpoint(region_url) { + Ok(endpoint) => endpoint, + Err(error) => return json!({"url":region_url,"pingMs":null,"error":error}), + }; + let _ = tcp_ping(&endpoint); + let mut samples = Vec::with_capacity(3); + for attempt in 0..3 { + if crate::requests::check().is_err() { + return Value::Null; + } + if attempt > 0 { + thread::sleep(Duration::from_millis(100)); + } + if let Some(sample) = tcp_ping(&endpoint) { + samples.push(sample); + } + } + if samples.is_empty() { + return json!({"url":region_url,"pingMs":null,"error":"All TCP measurements failed"}); + } + let total = samples.iter().copied().sum::(); + let average = total.div_ceil(samples.len() as u128); + json!({"url":region_url,"pingMs":u64::try_from(average).unwrap_or(u64::MAX)}) +} + +fn region_endpoint(region_url: &str) -> Result, String> { + let parsed = Url::parse(region_url).map_err(|_| "Invalid region URL".to_owned())?; + if !matches!(parsed.scheme(), "https" | "http") { + return Err("Unsupported region URL scheme".to_owned()); + } + let host = parsed + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| "Region URL has no host".to_owned())?; + let port = parsed + .port_or_known_default() + .ok_or_else(|| "Region URL has no TCP port".to_owned())?; + let addresses = (host, port) + .to_socket_addrs() + .map_err(|_| "Region host could not be resolved".to_owned())? + .collect::>(); + if addresses.is_empty() { + return Err("Region host has no routable address".to_owned()); + } + Ok(addresses) +} + +fn tcp_ping(addresses: &[SocketAddr]) -> Option { + let started = Instant::now(); + for address in addresses { + if crate::requests::check().is_err() { + return None; + } + if TcpStream::connect_timeout(address, CONNECT_TIMEOUT).is_ok() { + return Some(started.elapsed().as_millis()); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + #[test] + fn queue_tcp_measurement_warms_up_then_takes_two_samples_without_http() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let worker = thread::spawn(move || { + use std::io::Read; + for _ in 0..3 { + let (mut connection, _) = listener.accept().unwrap(); + connection + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + assert_eq!(connection.read(&mut [0]).unwrap(), 0); + } + }); + assert!(measure_queue_region(&format!("http://127.0.0.1:{port}")).is_some()); + worker.join().unwrap(); + } + + #[test] + fn stalled_resolution_obeys_deadline_and_cancellation() { + let (_sender, receive) = mpsc::sync_channel(1); + let cancellation = crate::requests::Cancellation::default(); + let started = Instant::now(); + assert!( + receive_addresses(&receive, started + Duration::from_millis(20), &cancellation) + .is_none() + ); + assert!(started.elapsed() < Duration::from_secs(1)); + let requests = Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("dns", "queue.servers.list").unwrap(); + requests.cancel("dns"); + assert!( + receive_addresses( + &receive, + Instant::now() + Duration::from_secs(60), + &permit.token + ) + .is_none() + ); + crate::requests::scope(permit.token.clone(), || { + assert!(resolve_bounded("unresolvable.invalid".into(), 443).is_none()); + }); + } + + #[test] + fn cancelled_region_measurement_does_not_start_network_work() { + let requests = Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("ping", "network.regions.ping").unwrap(); + requests.cancel("ping"); + crate::requests::scope(permit.token.clone(), || { + assert_eq!( + ping_regions(&json!({"regions":[]})).unwrap_err(), + "Request cancelled" + ); + assert_eq!(measure_region("https://unresolvable.invalid"), Value::Null); + }); + } + + #[test] + fn region_ping_rejects_invalid_and_unbounded_input() { + let invalid = ping_regions(&json!({"regions":[{"url":"file:///tmp/nope"}]})) + .expect("invalid regions return per-item results"); + assert_eq!(invalid["results"][0]["pingMs"], Value::Null); + assert_eq!( + invalid["results"][0]["error"], + "Unsupported region URL scheme" + ); + + let regions = (0..=MAX_REGIONS) + .map(|_| json!({"url":"https://example.invalid"})) + .collect::>(); + assert!(ping_regions(&json!({"regions":regions})).is_err()); + } + + #[test] + fn offline_region_is_a_scoped_result_instead_of_a_request_failure() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral port"); + let port = listener.local_addr().expect("listener address").port(); + drop(listener); + + let result = ping_regions(&json!({ + "regions":[{"url":format!("http://127.0.0.1:{port}")}] + })) + .expect("offline measurements remain serializable"); + assert_eq!(result["results"][0]["pingMs"], Value::Null); + assert_eq!(result["results"][0]["error"], "All TCP measurements failed"); + } +} diff --git a/external/opennow/native/opennow-core/src/network_test.rs b/external/opennow/native/opennow-core/src/network_test.rs new file mode 100644 index 000000000..aa60d6ce0 --- /dev/null +++ b/external/opennow/native/opennow-core/src/network_test.rs @@ -0,0 +1,1425 @@ +use hmac::{Hmac, Mac}; +use sha2::Sha256; +#[cfg(test)] +use subtle::ConstantTimeEq; + +use crate::gfn::ServiceError; + +pub const MESSAGE_TYPE_MTU_PROBE: u32 = 7; +pub const MESSAGE_TYPE_MTU_RESPONSE: u32 = 8; +pub const MAC_LEN: usize = 32; + +pub const REPLY_PREFIX_THRESHOLD: usize = 484; + +const MAX_MESSAGE_BYTES: usize = 64 * 1024; +const MAX_FIELDS: usize = 64; +const MAX_STRING_BYTES: usize = 16 * 1024; +const MAX_SESSION_KEY_BYTES: usize = 4096; +const MAX_SESSION_ID_BYTES: usize = 256; + +#[derive(Debug, Clone, PartialEq)] +pub enum WireValue { + Varint(u64), + Fixed64(u64), + Fixed32(u32), + Bytes(Vec), +} + +pub const TYPE_FIELD: u32 = 1; +pub const SESSION_ID_FIELD: u32 = 2; +pub const MAC_FIELD: u32 = 10; +pub const SEQUENCE_FIELD: u32 = 12; +pub const PAYLOAD_SIZE_FIELD: u32 = 13; +pub const KNOWN_FIELD_MAX: u32 = 18; + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NetworkTestMessage { + known: [Option; (KNOWN_FIELD_MAX + 1) as usize], + unknown: Vec<(u32, WireValue)>, +} + +impl NetworkTestMessage { + pub fn mtu_probe(payload_size: u32, session_id: &[u8], sequence: u32) -> Self { + let mut message = Self::default(); + message.set_message_type(MESSAGE_TYPE_MTU_PROBE); + message.set_session_id(session_id.to_vec()); + message.set_sequence(sequence); + message.set_payload_size(payload_size); + message + } + + pub fn message_type(&self) -> u32 { + match self.known[TYPE_FIELD as usize] { + Some(WireValue::Varint(raw)) => raw as u32, + _ => 0, + } + } + + pub fn set_message_type(&mut self, value: u32) { + self.known[TYPE_FIELD as usize] = Some(WireValue::Varint(u64::from(value))); + } + + pub fn payload_size(&self) -> Option { + match self.known[PAYLOAD_SIZE_FIELD as usize] { + Some(WireValue::Varint(raw)) => u32::try_from(raw).ok(), + _ => None, + } + } + + pub fn set_payload_size(&mut self, value: u32) { + self.known[PAYLOAD_SIZE_FIELD as usize] = Some(WireValue::Varint(u64::from(value))); + } + + pub fn session_id(&self) -> &[u8] { + match &self.known[SESSION_ID_FIELD as usize] { + Some(WireValue::Bytes(raw)) => raw, + _ => &[], + } + } + + pub fn set_session_id(&mut self, value: Vec) { + self.known[SESSION_ID_FIELD as usize] = Some(WireValue::Bytes(value)); + } + + #[cfg(test)] + pub fn sequence(&self) -> Option { + match self.known[SEQUENCE_FIELD as usize] { + Some(WireValue::Varint(raw)) => u32::try_from(raw).ok(), + _ => None, + } + } + + pub fn set_sequence(&mut self, value: u32) { + self.known[SEQUENCE_FIELD as usize] = Some(WireValue::Varint(u64::from(value))); + } + + #[cfg(test)] + pub fn mac(&self) -> &[u8] { + match &self.known[MAC_FIELD as usize] { + Some(WireValue::Bytes(raw)) => raw, + _ => &[], + } + } + + pub fn set_mac(&mut self, value: Vec) { + self.known[MAC_FIELD as usize] = Some(WireValue::Bytes(value)); + } + + #[cfg(test)] + pub fn unknown(&self) -> &[(u32, WireValue)] { + &self.unknown + } + + #[cfg(test)] + pub fn push_unknown(&mut self, number: u32, value: WireValue) { + self.unknown.push((number, value)); + } + + pub fn encode(&self) -> Vec { + let mut body = Vec::new(); + self.encode_body(&mut body, true); + body + } + + #[cfg(test)] + pub fn encode_response(&self, datagram_bytes: usize) -> Vec { + let body = self.encode(); + let prefix = u32::try_from(body.len()).unwrap_or(u32::MAX).to_le_bytes(); + let mut datagram = Vec::with_capacity(datagram_bytes.max(body.len() + 4)); + datagram.extend_from_slice(&prefix); + datagram.extend_from_slice(&body); + datagram.resize(datagram_bytes.max(datagram.len()), 0x5a); + datagram + } + + fn encode_body(&self, out: &mut Vec, include_mac: bool) { + for number in 1..=KNOWN_FIELD_MAX { + let Some(value) = &self.known[number as usize] else { + continue; + }; + if number == MAC_FIELD && !include_mac { + continue; + } + write_tag(out, number, wire_type(value)); + write_value(out, value); + } + for (number, value) in &self.unknown { + write_tag(out, *number, wire_type(value)); + write_value(out, value); + } + } + + pub fn authenticated(&self, key: &[u8]) -> Result, ServiceError> { + let mut body = Vec::new(); + self.encode_body(&mut body, false); + Ok(compute_mac(key, &body)?.to_vec()) + } + + pub fn seal(&mut self, key: &[u8]) -> Result<(), ServiceError> { + let mac = self.authenticated(key)?; + self.set_mac(mac); + Ok(()) + } + + #[cfg(test)] + pub fn verify(&self, key: &[u8]) -> Result { + let mac = self.mac(); + if mac.len() != MAC_LEN { + return Ok(false); + } + let expected = self.authenticated(key)?; + Ok(bool::from(expected.as_slice().ct_eq(mac))) + } + + pub fn decode_reply(datagram: &[u8]) -> Result { + if datagram.len() >= REPLY_PREFIX_THRESHOLD { + Self::decode_response(datagram) + } else { + Self::decode(datagram) + } + } + + pub fn decode_response(datagram: &[u8]) -> Result { + if datagram.len() > MAX_MESSAGE_BYTES { + return Err(invalid("network test message is too large")); + } + let prefix = datagram + .get(..4) + .ok_or_else(|| invalid("network test message is truncated"))?; + let body_len = usize::try_from(u32::from_le_bytes(prefix.try_into().expect("four bytes"))) + .map_err(|_| invalid("network test length"))?; + let end = 4_usize + .checked_add(body_len) + .filter(|end| *end <= datagram.len()) + .ok_or_else(|| invalid("network test message is truncated"))?; + Self::decode(&datagram[4..end]) + } + + pub fn decode(body: &[u8]) -> Result { + if body.len() > MAX_MESSAGE_BYTES { + return Err(invalid("network test message is too large")); + } + let end = body.len(); + let mut offset = 0_usize; + let mut message = Self::default(); + let mut fields = 0_usize; + while offset < end { + fields += 1; + if fields > MAX_FIELDS { + return Err(invalid("network test message has too many fields")); + } + let (tag, next) = read_varint(body.get(offset..).unwrap_or_default())?; + offset += next; + let number = + u32::try_from(tag >> 3).map_err(|_| invalid("network test field number"))?; + if number == 0 { + return Err(invalid("network test field number is zero")); + } + let value = decode_value(body, &mut offset, tag & 0x7)?; + match known_wire_type(number) { + Some(expected) if expected == tag & 0x7 => { + message.known[number as usize] = Some(value); + } + _ => message.unknown.push((number, value)), + } + } + Ok(message) + } +} + +fn known_wire_type(number: u32) -> Option { + match number { + 1 | 6 | 8 | 9 | 12 | 13 | 14 | 16 | 17 | 18 => Some(0), + 2 | 3 | 5 | 7 | 10 => Some(2), + 11 | 15 => Some(5), + _ => None, + } +} + +fn decode_value(body: &[u8], offset: &mut usize, wire: u64) -> Result { + Ok(match wire { + 0 => { + let (value, next) = read_varint(body.get(*offset..).unwrap_or_default())?; + *offset += next; + WireValue::Varint(value) + } + 1 => { + let raw = take(body, *offset, 8)?; + *offset += 8; + WireValue::Fixed64(u64::from_le_bytes(raw.try_into().expect("eight bytes"))) + } + 2 => { + let (len, next) = read_varint(body.get(*offset..).unwrap_or_default())?; + *offset += next; + let len = usize::try_from(len).map_err(|_| invalid("network test length"))?; + if len > MAX_STRING_BYTES { + return Err(invalid("network test field is too large")); + } + let raw = take(body, *offset, len)?; + *offset += len; + WireValue::Bytes(raw.to_vec()) + } + 5 => { + let raw = take(body, *offset, 4)?; + *offset += 4; + WireValue::Fixed32(u32::from_le_bytes(raw.try_into().expect("four bytes"))) + } + _ => return Err(invalid("network test field type is unsupported")), + }) +} + +fn wire_type(value: &WireValue) -> u64 { + match value { + WireValue::Varint(_) => 0, + WireValue::Fixed64(_) => 1, + WireValue::Bytes(_) => 2, + WireValue::Fixed32(_) => 5, + } +} + +fn write_tag(out: &mut Vec, number: u32, wire: u64) { + write_varint(out, (u64::from(number) << 3) | wire); +} + +fn write_value(out: &mut Vec, value: &WireValue) { + match value { + WireValue::Varint(raw) => write_varint(out, *raw), + WireValue::Fixed64(raw) => out.extend_from_slice(&raw.to_le_bytes()), + WireValue::Fixed32(raw) => out.extend_from_slice(&raw.to_le_bytes()), + WireValue::Bytes(raw) => { + write_varint(out, raw.len() as u64); + out.extend_from_slice(raw); + } + } +} + +fn write_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn read_varint(input: &[u8]) -> Result<(u64, usize), ServiceError> { + let mut value = 0_u64; + let mut shift = 0_u32; + for (index, byte) in input.iter().enumerate().take(10) { + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok((value, index + 1)); + } + shift += 7; + } + Err(invalid("network test varint is malformed")) +} + +fn take(input: &[u8], offset: usize, len: usize) -> Result<&[u8], ServiceError> { + input + .get(offset..offset + len) + .ok_or_else(|| invalid("network test message is truncated")) +} + +fn compute_mac(key: &[u8], body: &[u8]) -> Result<[u8; MAC_LEN], ServiceError> { + if key.is_empty() || key.len() > 4096 { + return Err(invalid("network test key length is invalid")); + } + let mut mac = as Mac>::new_from_slice(key) + .map_err(|_| invalid("network test key is invalid"))?; + mac.update(body); + let mut tag = [0_u8; MAC_LEN]; + tag.copy_from_slice(&mac.finalize().into_bytes()); + Ok(tag) +} + +fn invalid(message: &str) -> ServiceError { + ServiceError { + code: "network-test-invalid", + message: message.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: [u8; 32] = [0x5a; 32]; + + #[test] + fn mtu_probe_encodes_the_verified_field_numbers() { + let message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + let encoded = message.encode(); + assert_eq!( + encoded, + vec![ + 0x08, 0x07, 0x12, 0x04, b'n', b't', b'-', b'1', 0x60, 0x01, 0x68, 0x80, 0x0a, + ], + "bare protobuf: type 7, session id, sequence and payload size" + ); + assert_eq!(message.session_id(), b"nt-1"); + assert_eq!(message.sequence(), Some(1)); + } + + #[test] + fn mac_is_field_ten_and_covers_the_body_without_it() { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + let unsealed_body = message.encode(); + message.seal(&KEY).unwrap(); + let body = message.encode(); + + assert_eq!(&body[..2], &[0x08, 0x07], "field 1 carries the type"); + assert_eq!(message.mac().len(), MAC_LEN); + let expected = compute_mac(&KEY, &unsealed_body).unwrap(); + assert_eq!(message.mac(), expected.as_slice()); + + let mac_at = body + .windows(2) + .position(|pair| pair == [0x52, 0x20]) + .expect("field 10 tag"); + let mut without_mac = body.clone(); + without_mac.drain(mac_at..mac_at + 2 + MAC_LEN); + assert_eq!( + without_mac, unsealed_body, + "sealing only inserts field 10 and leaves the covered body intact" + ); + assert_eq!( + &body[mac_at + 2..mac_at + 2 + MAC_LEN], + expected.as_slice(), + "mac covers the body without itself" + ); + } + + #[test] + fn response_framing_is_a_length_prefix_then_body_then_padding() { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + message.seal(&KEY).unwrap(); + let body = message.encode(); + let datagram = message.encode_response(1_300); + assert_eq!( + datagram.len(), + 1_300, + "datagram is exactly the promised size" + ); + assert_eq!( + u32::from_le_bytes(datagram[..4].try_into().unwrap()) as usize, + body.len(), + "prefix is the little-endian protobuf body length" + ); + assert_eq!(&datagram[4..4 + body.len()], body.as_slice()); + + let decoded = NetworkTestMessage::decode_response(&datagram).unwrap(); + assert_eq!(decoded, message); + assert!(decoded.verify(&KEY).unwrap()); + assert_eq!(decoded.payload_size(), Some(1_280)); + } + + #[test] + fn a_sealed_message_verifies_and_rejects_tampering() { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + message.seal(&KEY).unwrap(); + assert!(message.verify(&KEY).unwrap()); + + let mut changed = message.clone(); + changed.set_payload_size(1_279); + assert!(!changed.verify(&KEY).unwrap()); + + let mut rekeyed = message.clone(); + assert!(!rekeyed.verify(&[0x11; 32]).unwrap()); + + let mut tampered = rekeyed.mac().to_vec(); + tampered[0] ^= 0x01; + rekeyed.set_mac(tampered); + assert!(!rekeyed.verify(&KEY).unwrap()); + } + + #[test] + fn decoding_round_trips_unknown_fields() { + let mut message = NetworkTestMessage::mtu_probe(1_500, b"nt-1", 1); + message.push_unknown(4, WireValue::Bytes(b"session".to_vec())); + message.push_unknown(19, WireValue::Fixed32(1.5_f32.to_bits())); + message.push_unknown(20, WireValue::Varint(9)); + message.seal(&KEY).unwrap(); + + let decoded = NetworkTestMessage::decode(&message.encode()).unwrap(); + assert_eq!(decoded, message); + assert!(decoded.verify(&KEY).unwrap()); + } + + #[test] + fn decode_rejects_malformed_and_oversized_input() { + assert!(NetworkTestMessage::decode(&[0x7f, 0x08, 0x07]).is_err()); + let oversized = vec![0xff; MAX_MESSAGE_BYTES + 1]; + assert!(NetworkTestMessage::decode(&oversized).is_err()); + + assert!(NetworkTestMessage::decode_response(&[0x05, 0x00]).is_err()); + assert!( + NetworkTestMessage::decode_response(&[0xff, 0xff, 0xff, 0xff, 0x08]).is_err(), + "declared body length must fit in the datagram" + ); + let truncated = { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + message.seal(&KEY).unwrap(); + let mut datagram = message.encode_response(1_300); + datagram.truncate(datagram.len() - 30); + datagram + }; + assert!( + NetworkTestMessage::decode_response(&truncated).is_ok(), + "padding is outside the prefixed body" + ); + } + + #[test] + fn sealing_fails_closed_without_usable_key_material() { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + assert!(message.seal(&[]).is_err()); + assert!(message.authenticated(&[]).is_err()); + assert_eq!(message.mac(), &[] as &[u8]); + assert!(!message.verify(&[]).unwrap()); + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct NetworkTestThresholds { + pub bandwidth_recommended_mbps: f64, + pub bandwidth_limit_mbps: f64, + pub latency_recommended_ms: f64, + pub latency_limit_ms: f64, + pub packet_loss_recommended_pct: f64, + pub packet_loss_limit_pct: f64, +} + +#[derive(Clone, PartialEq)] +pub struct NetworkTestSession { + pub session_id: String, + pub server_id: String, + pub address: std::net::IpAddr, + pub port: u16, + pub secure: bool, + pub hmac_key: Option>, + pub thresholds: NetworkTestThresholds, +} + +impl std::fmt::Debug for NetworkTestSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NetworkTestSession") + .field("session_id", &self.session_id) + .field("server_id", &self.server_id) + .field("address", &self.address) + .field("port", &self.port) + .field("secure", &self.secure) + .field("hmac_key", &self.hmac_key.as_ref().map(|_| "[redacted]")) + .field("thresholds", &self.thresholds) + .finish() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProbeOutcome { + pub measured_datagram_bytes: Option, + pub probes: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DisplayProfile { + pub width: u32, + pub height: u32, + pub fps: u32, +} + +pub fn allocation_body(platform: &str, profile: DisplayProfile) -> serde_json::Value { + let DisplayProfile { width, height, fps } = profile; + serde_json::json!({ + "netTestRequestData": { + "clientPlatformName": platform, + "netTestProfile": { + "widthInPixels": width, + "heightInPixels": height, + "framesPerSecond": fps, + }, + } + }) +} + +pub fn parse_allocation(response: &serde_json::Value) -> Result { + let session = response + .get("netTestSession") + .ok_or_else(|| invalid("network test response has no session"))?; + let session_id = session + .get("sessionId") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= MAX_SESSION_ID_BYTES) + .ok_or_else(|| invalid("network test session has no id"))? + .to_owned(); + let server_id = session + .get("serverId") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_owned(); + let connection = session + .get("connectionInfo") + .and_then(serde_json::Value::as_array) + .and_then(|entries| entries.first()) + .ok_or_else(|| invalid("network test session has no connection info"))?; + let address = connection + .get("ip") + .and_then(serde_json::Value::as_str) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| invalid("network test session has no literal address"))?; + let port = connection + .get("port") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u16::try_from(value).ok()) + .filter(|value| *value != 0) + .ok_or_else(|| invalid("network test session has no port"))?; + let secure = connection + .get("appLevelProtocol") + .and_then(serde_json::Value::as_u64) + == Some(5); + let hmac_key = match session.get("hmacKey") { + None | Some(serde_json::Value::Null) => None, + Some(value) => Some(parse_session_key(value)?), + }; + let thresholds = session + .get("netTestThresholds") + .ok_or_else(|| invalid("network test session has no thresholds"))?; + let number = |key: &str| { + thresholds + .get(key) + .and_then(serde_json::Value::as_f64) + .unwrap_or_default() + }; + Ok(NetworkTestSession { + session_id, + server_id, + address, + port, + secure, + hmac_key, + thresholds: NetworkTestThresholds { + bandwidth_recommended_mbps: number("recommendedBandwidthMBPS"), + bandwidth_limit_mbps: number("requiredBandwidthMBPS"), + latency_recommended_ms: number("recommendedLatencyMS"), + latency_limit_ms: number("requiredLatencyMS"), + packet_loss_recommended_pct: number("recommendedPacketLossPct"), + packet_loss_limit_pct: number("requiredPacketLossPct"), + }, + }) +} + +fn parse_session_key(value: &serde_json::Value) -> Result, ServiceError> { + let raw = value + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("network test session key is not a string"))?; + let bytes = raw.as_bytes(); + if bytes.len() > MAX_SESSION_KEY_BYTES { + return Err(invalid("network test session key is too large")); + } + if bytes.is_empty() || bytes.contains(&0) { + return Err(invalid("network test session key is not usable")); + } + Ok(bytes.to_vec()) +} + +pub fn nettest_url(base: &url::Url) -> Result { + let host = base.host_str().unwrap_or_default(); + if host.is_empty() { + return Err(invalid("network test zone has no host")); + } + let loopback = matches!(host, "127.0.0.1" | "::1" | "localhost"); + if base.scheme() != "https" && !(loopback && base.scheme() == "http") { + return Err(invalid("network test zone must use https")); + } + base.join("v2/nettestsession") + .map_err(|_| invalid("invalid network test session URL")) +} + +pub const PROBE_BUDGET: std::time::Duration = std::time::Duration::from_millis(400); +pub const PROBE_FLOOR_BYTES: u32 = 548; +pub const PROBE_CEILING_BYTES: u32 = 1_472; +const PROBE_ATTEMPT_WAIT: std::time::Duration = std::time::Duration::from_millis(40); +const MAX_PROBES: usize = 12; +const PROBE_STEP: u32 = 16; + +fn reply_is_accepted(reply: &NetworkTestMessage, session_id: &[u8], size: u32) -> bool { + reply.message_type() == MESSAGE_TYPE_MTU_RESPONSE + && reply.session_id() == session_id + && reply.payload_size() == Some(size) +} + +fn confirm_datagram( + socket: &std::net::UdpSocket, + peer: std::net::SocketAddr, + key: &[u8], + session_id: &[u8], + sequence: u32, + size: u32, + deadline: std::time::Instant, +) -> bool { + let deadline = deadline.min(std::time::Instant::now() + PROBE_ATTEMPT_WAIT); + let mut message = NetworkTestMessage::mtu_probe(size, session_id, sequence); + if message.seal(key).is_err() { + return false; + } + if socket.send_to(&message.encode(), peer).is_err() { + return false; + } + let mut buffer = vec![0_u8; MAX_MESSAGE_BYTES]; + while std::time::Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() || socket.set_read_timeout(Some(remaining)).is_err() { + return false; + } + let Ok((length, source)) = socket.recv_from(&mut buffer) else { + return false; + }; + if std::time::Instant::now() >= deadline { + return false; + } + if source != peer || length < size as usize { + continue; + } + let Ok(reply) = NetworkTestMessage::decode_reply(&buffer[..length]) else { + continue; + }; + if reply_is_accepted(&reply, session_id, size) { + return true; + } + } + false +} + +pub fn probe_mtu( + socket: &std::net::UdpSocket, + peer: std::net::SocketAddr, + key: &[u8], + session_id: &[u8], + floor: u32, + ceiling: u32, +) -> Result { + if ceiling <= floor { + return Err(invalid("network test probe range is empty")); + } + let deadline = std::time::Instant::now() + PROBE_BUDGET; + let mut probes = 0_usize; + let confirmed = |size: u32, probes: &mut usize| -> bool { + if *probes >= MAX_PROBES || std::time::Instant::now() >= deadline { + return false; + } + *probes += 1; + confirm_datagram( + socket, + peer, + key, + session_id, + u32::try_from(*probes).unwrap_or(u32::MAX), + size, + deadline, + ) + }; + if confirmed(ceiling, &mut probes) { + return Ok(ProbeOutcome { + measured_datagram_bytes: Some(ceiling), + probes, + }); + } + if !confirmed(floor, &mut probes) { + return Ok(ProbeOutcome { + measured_datagram_bytes: None, + probes, + }); + } + let mut low = floor; + let mut high = ceiling; + while high - low > PROBE_STEP && probes < MAX_PROBES { + let middle = low + (high - low) / 2; + if confirmed(middle, &mut probes) { + low = middle; + } else { + high = middle; + } + } + Ok(ProbeOutcome { + measured_datagram_bytes: Some(low), + probes, + }) +} + +#[cfg(test)] +mod probe_tests { + use super::*; + use std::net::{SocketAddr, UdpSocket}; + use std::time::Duration; + + const KEY: [u8; 32] = [0x33; 32]; + const SESSION: &[u8] = b"nt-1"; + + fn vendor_reply_body(session: &[u8], size: u32, message_type: u32) -> Vec { + let mut body = Vec::new(); + assert!(message_type < 0x80 && session.len() < 0x80); + body.push(0x08); + body.push(message_type as u8); + body.push(0x12); + body.push(session.len() as u8); + body.extend_from_slice(session); + body.push(0x68); + let mut value = size; + while value >= 0x80 { + body.push((value as u8) | 0x80); + value >>= 7; + } + body.push(value as u8); + body + } + + fn vendor_datagram(session: &[u8], size: u32, message_type: u32) -> Vec { + let body = vendor_reply_body(session, size, message_type); + let mut datagram = Vec::with_capacity(size as usize); + datagram.extend_from_slice(&(body.len() as u32).to_le_bytes()); + datagram.extend_from_slice(&body); + datagram.resize(size as usize, 0x5a); + datagram + } + + fn vendor_server(cap: u32, respond: F) -> (SocketAddr, std::thread::JoinHandle) + where + F: Fn(u32) -> Vec> + Send + 'static, + { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(250))) + .unwrap(); + let address = socket.local_addr().unwrap(); + let handle = std::thread::spawn(move || { + let mut served = 0_usize; + let mut buffer = vec![0_u8; 4096]; + while let Ok((length, peer)) = socket.recv_from(&mut buffer) { + let Ok(request) = NetworkTestMessage::decode(&buffer[..length]) else { + continue; + }; + let Some(size) = request.payload_size() else { + continue; + }; + if size > cap { + continue; + } + for datagram in respond(size) { + let _ = socket.send_to(&datagram, peer); + } + served += 1; + } + served + }); + (address, handle) + } + + fn foreign_server(respond: F) -> (SocketAddr, std::thread::JoinHandle) + where + F: Fn(u32) -> Vec + Send + 'static, + { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let sender = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(250))) + .unwrap(); + let address = socket.local_addr().unwrap(); + let handle = std::thread::spawn(move || { + let mut served = 0_usize; + let mut buffer = vec![0_u8; 4096]; + while let Ok((length, peer)) = socket.recv_from(&mut buffer) { + let Ok(request) = NetworkTestMessage::decode(&buffer[..length]) else { + continue; + }; + let Some(size) = request.payload_size() else { + continue; + }; + let datagram = respond(size); + let deadline = std::time::Instant::now() + Duration::from_millis(150); + while std::time::Instant::now() < deadline { + let _ = sender.send_to(&datagram, peer); + std::thread::sleep(Duration::from_millis(5)); + } + served += 1; + } + served + }); + (address, handle) + } + + fn client_socket() -> UdpSocket { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(10))) + .unwrap(); + socket + } + + #[test] + fn a_path_that_carries_the_ceiling_needs_no_adjustment() { + let (peer, server) = + vendor_server(u32::MAX, |size| vec![vendor_datagram(SESSION, size, 8)]); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, Some(1_340)); + drop(socket); + let _ = server.join(); + } + + #[test] + fn a_narrower_path_reports_the_largest_confirmed_size() { + let (peer, server) = vendor_server(1_280, |size| vec![vendor_datagram(SESSION, size, 8)]); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + let measured = outcome.measured_datagram_bytes.expect("measured"); + assert!(measured <= 1_280, "measured {measured}"); + assert!(measured + PROBE_STEP * 2 >= 1_280, "measured {measured}"); + assert!(outcome.probes <= MAX_PROBES); + drop(socket); + let _ = server.join(); + } + + #[test] + fn a_silent_path_yields_no_measurement() { + let (peer, server) = vendor_server(1_280, |_| Vec::new()); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn replies_with_a_foreign_type_are_ignored() { + let (peer, server) = + vendor_server(u32::MAX, |size| vec![vendor_datagram(SESSION, size, 7)]); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn replies_for_another_session_are_ignored() { + let (peer, server) = + vendor_server(u32::MAX, |size| vec![vendor_datagram(b"other", size, 8)]); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn stale_replies_for_another_size_are_ignored() { + let (peer, server) = vendor_server(u32::MAX, |size| { + vec![ + vendor_datagram(SESSION, size + 64, 8), + vendor_datagram(SESSION, size, 8), + ] + }); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, Some(1_340)); + drop(socket); + let _ = server.join(); + } + + #[test] + fn duplicate_replies_do_not_change_the_measurement() { + let (peer, server) = vendor_server(u32::MAX, |size| { + let datagram = vendor_datagram(SESSION, size, 8); + vec![datagram.clone(), datagram] + }); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, Some(1_340)); + drop(socket); + let _ = server.join(); + } + + #[test] + fn replies_from_a_foreign_source_are_ignored() { + let (peer, server) = foreign_server(|size| vendor_datagram(SESSION, size, 8)); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn truncated_and_malformed_replies_are_ignored() { + let (peer, server) = vendor_server(u32::MAX, |size| { + let mut overclaimed = vec![0_u8; size as usize]; + overclaimed[..4].copy_from_slice(&(size + 1_000).to_le_bytes()); + overclaimed[4] = 0x08; + overclaimed[5] = 0x08; + vec![overclaimed, vec![0xff; size as usize]] + }); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn short_replies_are_ignored() { + let (peer, server) = vendor_server(u32::MAX, |size| { + let datagram = vendor_datagram(SESSION, size, 8); + vec![datagram[..datagram.len() - 16].to_vec()] + }); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn a_floor_below_usefulness_yields_no_measurement() { + let (peer, server) = vendor_server(100, |size| vec![vendor_datagram(SESSION, size, 8)]); + let socket = client_socket(); + let outcome = probe_mtu(&socket, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + assert_eq!(outcome.measured_datagram_bytes, None); + drop(socket); + let _ = server.join(); + } + + #[test] + fn an_empty_probe_range_is_rejected() { + let socket = client_socket(); + let peer: SocketAddr = "127.0.0.1:1".parse().unwrap(); + assert!(probe_mtu(&socket, peer, &KEY, SESSION, 1_340, PROBE_FLOOR_BYTES).is_err()); + } + + #[test] + fn unsigned_replies_are_accepted_by_the_correlation_rules() { + let datagram = vendor_datagram(SESSION, 1_280, 8); + assert_eq!( + datagram.len(), + 1_280, + "vendor datagram is the requested size" + ); + let reply = NetworkTestMessage::decode_reply(&datagram).unwrap(); + assert!(reply_is_accepted(&reply, SESSION, 1_280)); + assert!(!reply_is_accepted(&reply, b"other", 1_280), "session"); + assert!(!reply_is_accepted(&reply, SESSION, 1_279), "size"); + assert!(reply.mac().is_empty(), "vendor replies are unsigned"); + } + + #[test] + fn bare_replies_below_the_prefix_threshold_are_parsed() { + let body = vendor_reply_body(SESSION, 300, 8); + assert!(body.len() < REPLY_PREFIX_THRESHOLD); + let reply = NetworkTestMessage::decode_reply(&body).unwrap(); + assert!(reply_is_accepted(&reply, SESSION, 300)); + } + + #[test] + fn unrelated_replies_do_not_renew_a_probe_attempt() { + use std::time::Instant; + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let peer = socket.local_addr().unwrap(); + let client = client_socket(); + let client_address = client.local_addr().unwrap(); + let deadline = Instant::now() + PROBE_BUDGET; + let server = std::thread::spawn(move || { + while Instant::now() < deadline { + let _ = socket.send_to(&vendor_datagram(b"other", 1_340, 8), client_address); + std::thread::sleep(Duration::from_millis(1)); + } + }); + let started = Instant::now(); + let confirmed = confirm_datagram(&client, peer, &KEY, SESSION, 1, 1_340, deadline); + let elapsed = started.elapsed(); + server.join().unwrap(); + assert!(!confirmed); + assert!( + elapsed < PROBE_BUDGET / 2, + "unrelated replies extended one probe attempt to {elapsed:?}" + ); + } + + #[test] + fn a_hostile_flood_cannot_stretch_the_probe_deadline() { + use std::time::Instant; + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let peer = socket.local_addr().unwrap(); + let client = client_socket(); + let client_address = client.local_addr().unwrap(); + let started = Instant::now(); + let flood_until = started + PROBE_BUDGET - Duration::from_millis(10); + let server = std::thread::spawn(move || { + let mut sent = 0_usize; + while Instant::now() < flood_until { + let _ = socket.send_to(&vendor_datagram(b"other", 1_340, 8), client_address); + sent += 1; + std::thread::sleep(Duration::from_millis(1)); + } + sent + }); + let outcome = probe_mtu(&client, peer, &KEY, SESSION, PROBE_FLOOR_BYTES, 1_340).unwrap(); + let elapsed = Instant::now() - started; + assert_eq!(outcome.measured_datagram_bytes, None); + assert!( + elapsed < PROBE_BUDGET + PROBE_ATTEMPT_WAIT / 2, + "the hostile flood stretched the probe to {elapsed:?}" + ); + let sent = server.join().unwrap(); + assert!(sent > 20, "the flood never reached the client: {sent}"); + } +} +#[cfg(test)] +mod allocation_tests { + use super::*; + + fn response(app_level_protocol: u64) -> serde_json::Value { + serde_json::json!({ + "requestStatus": {"requestId": "req-1", "serverId": "zone-1", "statusCode": "200"}, + "netTestSession": { + "sessionId": "nt-42", + "serverId": "srv-9", + "connectionInfo": [{ + "ip": "192.0.2.44", + "port": 49100, + "appLevelProtocol": app_level_protocol, + }], + "netTestThresholds": { + "recommendedBandwidthMBPS": 50.0, + "requiredBandwidthMBPS": 25.0, + "recommendedLatencyMS": 40.0, + "requiredLatencyMS": 80.0, + "recommendedPacketLossPct": 1.0, + "requiredPacketLossPct": 3.0, + } + } + }) + } + + #[test] + fn debug_output_never_contains_the_session_key() { + let mut payload = response(5); + payload["netTestSession"]["hmacKey"] = serde_json::json!("super-secret-key-material"); + let session = parse_allocation(&payload).unwrap(); + assert_eq!( + session.hmac_key.as_deref(), + Some("super-secret-key-material".as_bytes()) + ); + let debug = format!("{session:?}"); + assert!( + !debug.contains("super-secret-key-material"), + "debug leaked the key: {debug}" + ); + assert!(debug.contains("[redacted]")); + } + + #[test] + fn non_literal_addresses_and_oversized_ids_are_rejected() { + let mut hostname = response(5); + hostname["netTestSession"]["connectionInfo"][0]["ip"] = + serde_json::json!("net-test.example.com"); + assert!( + parse_allocation(&hostname).is_err(), + "hostnames must not reach a resolver" + ); + + let mut long_id = response(5); + long_id["netTestSession"]["sessionId"] = serde_json::json!("n".repeat(257)); + assert!(parse_allocation(&long_id).is_err()); + } + + #[test] + fn unusable_session_keys_are_rejected() { + let mut nul_key = String::from("nul"); + nul_key.push('\0'); + nul_key.push_str("byte"); + for key in [ + serde_json::json!(""), + serde_json::json!(7), + serde_json::json!(nul_key), + ] { + let mut payload = response(5); + payload["netTestSession"]["hmacKey"] = key.clone(); + assert!(parse_allocation(&payload).is_err(), "accepted {key:?}"); + } + let mut payload = response(5); + payload["netTestSession"]["hmacKey"] = serde_json::Value::Null; + assert_eq!(parse_allocation(&payload).unwrap().hmac_key, None); + } + + #[test] + fn allocation_body_matches_the_recovered_schema() { + let body = allocation_body( + "GFN-PC", + DisplayProfile { + width: 1920, + height: 1080, + fps: 60, + }, + ); + assert_eq!(body["netTestRequestData"]["clientPlatformName"], "GFN-PC"); + assert_eq!( + body["netTestRequestData"]["netTestProfile"]["widthInPixels"], + 1920 + ); + assert_eq!( + body["netTestRequestData"]["netTestProfile"]["heightInPixels"], + 1080 + ); + assert_eq!( + body["netTestRequestData"]["netTestProfile"]["framesPerSecond"], + 60 + ); + } + + #[test] + fn allocation_response_is_parsed_with_thresholds() { + let session = parse_allocation(&response(5)).unwrap(); + assert_eq!(session.session_id, "nt-42"); + assert_eq!(session.server_id, "srv-9"); + assert_eq!( + session.address, + "192.0.2.44".parse::().unwrap() + ); + assert_eq!(session.port, 49_100); + assert!(session.secure); + assert_eq!(session.thresholds.bandwidth_limit_mbps, 25.0); + assert_eq!(session.thresholds.latency_limit_ms, 80.0); + assert_eq!(session.thresholds.packet_loss_limit_pct, 3.0); + assert!(!parse_allocation(&response(4)).unwrap().secure); + } + + #[test] + fn malformed_allocation_responses_are_rejected() { + assert!(parse_allocation(&serde_json::json!({})).is_err()); + let mut missing_port = response(5); + missing_port["netTestSession"]["connectionInfo"][0] + .as_object_mut() + .unwrap() + .remove("port"); + assert!(parse_allocation(&missing_port).is_err()); + let mut empty_session = response(5); + empty_session["netTestSession"]["sessionId"] = serde_json::json!(""); + assert!(parse_allocation(&empty_session).is_err()); + } + + #[test] + fn allocation_requires_https_and_a_token() { + assert!(nettest_url(&url::Url::parse("http://example.test").unwrap()).is_err()); + assert!(nettest_url(&url::Url::parse("http://127.0.0.1:1").unwrap()).is_ok()); + assert!(nettest_url(&url::Url::parse("https://example.test").unwrap()).is_ok()); + assert_eq!( + nettest_url(&url::Url::parse("https://example.test").unwrap()) + .unwrap() + .as_str(), + "https://example.test/v2/nettestsession" + ); + } +} + +#[cfg(test)] +mod canonical_tests { + use super::*; + + const KEY: [u8; 32] = [0x77; 32]; + + #[test] + fn known_fields_encode_ascending_and_unknown_fields_follow() { + let mut message = NetworkTestMessage::default(); + message.set_message_type(7); + message.set_payload_size(1_280); + message.push_unknown(19, WireValue::Varint(3)); + message.push_unknown(4, WireValue::Bytes(b"b".to_vec())); + message.push_unknown(4, WireValue::Varint(1)); + let encoded = message.encode(); + assert_eq!( + decode_field_numbers(&encoded), + vec![1, 13, 19, 4, 4], + "known fields ascending, then unknown in received order" + ); + } + + #[test] + fn duplicate_scalar_fields_take_the_last_value() { + let mut body = Vec::new(); + write_varint(&mut body, 0x08); + write_varint(&mut body, 1); + write_varint(&mut body, 0x08); + write_varint(&mut body, 7); + let decoded = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(decoded.message_type(), 7); + let numbers = decode_field_numbers(&decoded.encode()); + assert_eq!(numbers, vec![1], "re-encoding collapses the duplicate"); + } + + #[test] + fn unknown_fields_are_authenticated_and_survive_re_encoding() { + let mut sender = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + sender.push_unknown(4, WireValue::Varint(11)); + sender.seal(&KEY).unwrap(); + + let received = NetworkTestMessage::decode(&sender.encode()).unwrap(); + assert!(received.verify(&KEY).unwrap()); + assert_eq!(received.unknown(), &[(4, WireValue::Varint(11))]); + assert_eq!(received.encode(), sender.encode()); + } + + #[test] + fn non_canonical_input_order_still_authenticates() { + let mut sender = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + sender.push_unknown(19, WireValue::Varint(5)); + sender.seal(&KEY).unwrap(); + let canonical = sender.encode(); + + let mut reordered = Vec::new(); + write_varint(&mut reordered, 19 << 3); + write_varint(&mut reordered, 5); + reordered.extend_from_slice(&canonical[..canonical.len() - 3]); + + let received = NetworkTestMessage::decode(&reordered).unwrap(); + assert!( + received.verify(&KEY).unwrap(), + "official order-independent parse" + ); + assert_eq!(received.encode(), canonical); + } + + fn decode_field_numbers(body: &[u8]) -> Vec { + let mut numbers = Vec::new(); + let mut offset = 0_usize; + let end = body.len(); + while offset < end { + let (tag, next) = read_varint(&body[offset..]).unwrap(); + offset += next; + numbers.push((tag >> 3) as u32); + offset += match tag & 7 { + 0 => read_varint(&body[offset..]).unwrap().1, + 1 => 8, + 2 => { + let (len, next) = read_varint(&body[offset..]).unwrap(); + next + len as usize + } + 5 => 4, + other => panic!("unexpected wire type {other}"), + }; + } + numbers + } +} + +#[cfg(test)] +mod golden_tests { + use super::*; + + const KEY: [u8; 32] = [0x33; 32]; + const PROBE_BODY: &[u8] = &[ + 0x08, 0x07, 0x12, 0x04, b'n', b't', b'-', b'1', 0x60, 0x01, 0x68, 0x80, 0x0a, + ]; + const PROBE_HMAC: &str = "e891c98a2b313e55ebb343cf4f5aa46db2054fb419e1c0c4f51c23f1ffdfe987"; + const PROBE_SEALED: &str = "080712046e742d315220e891c98a2b313e55ebb343cf4f5aa46db2054fb419e1c0c4f51c23f1ffdfe987600168800a"; + const UNKNOWN_HMAC: &str = "770f593149064511de33d7dc0aaeebd71510eabcd591cfd5fd3b64a159c9f84d"; + + fn hex(text: &str) -> Vec { + (0..text.len() / 2) + .map(|index| u8::from_str_radix(&text[index * 2..index * 2 + 2], 16).unwrap()) + .collect() + } + + fn expected_body() -> Vec { + PROBE_BODY.to_vec() + } + + #[test] + fn golden_probe_matches_an_independent_protobuf_encoder() { + let message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + assert_eq!(message.encode(), expected_body()); + assert_eq!(message.authenticated(&KEY).unwrap(), hex(PROBE_HMAC)); + } + + #[test] + fn golden_sealed_bytes_match_the_independent_encoder() { + let mut message = NetworkTestMessage::mtu_probe(1_280, b"nt-1", 1); + message.seal(&KEY).unwrap(); + assert_eq!(message.encode(), hex(PROBE_SEALED)); + assert!(message.verify(&KEY).unwrap()); + assert_eq!( + NetworkTestMessage::decode(&message.encode()).unwrap(), + message, + "bare request body round-trips" + ); + } + + #[test] + fn golden_unknown_duplicates_authenticate_like_the_independent_encoder() { + let body = hex("080768800a200b200c"); + assert_eq!(body.len(), 9); + let received = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(received.unknown().len(), 2, "both duplicates preserved"); + assert_eq!( + received.unknown(), + &[(4, WireValue::Varint(11)), (4, WireValue::Varint(12))] + ); + assert_eq!(received.encode(), body, "unknown re-encodes in order"); + assert_eq!(received.authenticated(&KEY).unwrap(), hex(UNKNOWN_HMAC)); + assert!(!received.verify(&KEY).unwrap(), "no mac field present"); + } + + #[test] + fn golden_duplicate_known_field_takes_the_last_value() { + let body = hex("080768800a68800b"); + let decoded = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(decoded.payload_size(), Some(1_408)); + assert_eq!( + decoded.encode(), + hex("080768800b"), + "re-encoding collapses the duplicate" + ); + } + + #[test] + fn golden_unknown_before_known_moves_after_known() { + let body = hex("200b080768800a"); + let decoded = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(decoded.encode(), expected_body_with_unknown()); + } + + fn expected_body_with_unknown() -> Vec { + hex("080768800a200b") + } +} + +#[cfg(test)] +mod wire_semantics_tests { + use super::*; + + #[test] + fn wrong_wire_type_on_a_known_field_is_preserved_as_unknown() { + let body = hex_bytes("6a02aabb"); + let decoded = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(decoded.payload_size(), None); + assert_eq!( + decoded.unknown(), + &[(13, WireValue::Bytes(vec![0xAA, 0xBB]))] + ); + assert_eq!(decoded.encode(), body); + } + + #[test] + fn wrong_wire_type_on_the_type_field_is_preserved_as_unknown() { + let body = hex_bytes("0a020700"); + let decoded = NetworkTestMessage::decode(&body).unwrap(); + assert_eq!(decoded.message_type(), 0); + assert_eq!(decoded.encode(), body); + } + + fn hex_bytes(text: &str) -> Vec { + (0..text.len() / 2) + .map(|index| u8::from_str_radix(&text[index * 2..index * 2 + 2], 16).unwrap()) + .collect() + } +} diff --git a/external/opennow/native/opennow-core/src/persistent_storage.rs b/external/opennow/native/opennow-core/src/persistent_storage.rs new file mode 100644 index 000000000..505499d80 --- /dev/null +++ b/external/opennow/native/opennow-core/src/persistent_storage.rs @@ -0,0 +1,193 @@ +use crate::gfn::{AuthSession, ServiceError}; +use reqwest::blocking::Client; +use reqwest::header::{ACCEPT, CONTENT_TYPE, HeaderMap, HeaderValue}; +use serde_json::{Value, json}; +use url::Url; + +const PAYWALL_BASE: &str = "https://api-prod.nvidia.com/gfn-paywall-api/api/v2"; + +pub struct PersistentStorageService { + client: Client, +} + +impl PersistentStorageService { + pub fn new(client: Client) -> Self { + Self { client } + } + + pub fn locations(&self, params: &Value, auth: &AuthSession) -> Result { + let current_code = params["currentRegionCode"] + .as_str() + .filter(|value| !value.trim().is_empty()); + let current_name = params["currentRegionName"] + .as_str() + .filter(|value| !value.trim().is_empty()); + let mut url = + Url::parse(&format!("{PAYWALL_BASE}/products")).expect("constant paywall URL is valid"); + url.query_pairs_mut() + .append_pair("locale", params["locale"].as_str().unwrap_or("en_US")); + if let Some(vpc_id) = params["serverRegionId"] + .as_str() + .filter(|value| !value.trim().is_empty()) + { + url.query_pairs_mut().append_pair("vpcId", vpc_id); + } + let token = session_token(auth)?; + let response = self + .client + .get(url) + .headers(paywall_headers(token)?) + .send() + .map_err(|error| ServiceError::network("Storage region discovery failed", error))?; + let status = response.status(); + if !status.is_success() { + return Err(ServiceError { + code: if status.as_u16() == 401 { + "http_unauthorized" + } else { + "upstream_error" + }, + message: format!("Storage region discovery failed ({status})"), + }); + } + let payload = response + .json::() + .map_err(|error| ServiceError::network("Invalid storage region response", error))?; + if !payload["products"].is_array() { + return Err(ServiceError { + code: "invalid_upstream_response", + message: "Storage region response has no products".into(), + }); + } + let mut locations = locations_from_products(&payload, current_code); + if let Some(code) = current_code + && !locations + .iter() + .any(|location| location["code"].as_str() == Some(code)) + { + locations.insert(0, json!({"code":code,"name":current_name.unwrap_or(code),"isAvailable":false,"isCurrent":true})); + } + Ok( + json!({"locations":locations,"currentRegionCode":current_code,"currentRegionName":current_name}), + ) + } + + pub fn reset(&self, params: &Value, auth: &AuthSession) -> Result { + if params["confirmed"].as_bool() != Some(true) { + return Err(ServiceError { + code: "confirmation_required", + message: "Persistent storage reset requires explicit confirmation".to_owned(), + }); + } + let region = params["storageRegion"] + .as_str() + .filter(|value| !value.trim().is_empty()); + let mut url = Url::parse(&format!("{PAYWALL_BASE}/reset/storage")) + .expect("constant paywall URL is valid"); + url.query_pairs_mut() + .append_pair("storageRegion", region.unwrap_or("null")); + let response = self + .client + .post(url) + .headers(paywall_headers(session_token(auth)?)?) + .send() + .map_err(|error| ServiceError { + code: "network_error", + message: format!("Persistent storage reset failed: {error}"), + })?; + let status = response.status(); + let payload = response.json::().unwrap_or(Value::Null); + if !status.is_success() { + let message = payload["message"] + .as_str() + .or_else(|| payload["errors"]["errorMessage"].as_str()) + .unwrap_or("Persistent storage reset failed"); + let message = if message.contains("idToken") { + "NVIDIA requires a web-account session for this storage action. Open NVIDIA Storage Manager in your browser." + } else { + message + }; + return Err(ServiceError { + code: if matches!(status.as_u16(), 401 | 403) { + "authentication_required" + } else { + "upstream_error" + }, + message: message.to_owned(), + }); + } + Ok(json!({"ok":true,"storageRegion":region,"message":payload["message"]})) + } +} + +fn locations_from_products(payload: &Value, current_code: Option<&str>) -> Vec { + let mut products = payload["products"].as_array().cloned().unwrap_or_default(); + let addons = products + .iter() + .flat_map(|product| product["add_on"].as_array().cloned().unwrap_or_default()) + .collect::>(); + products.extend(addons); + let regions = products + .iter() + .find_map(|product| { + let kind = product["productType"].as_str().unwrap_or_default(); + (kind.eq_ignore_ascii_case("STORAGE") + || kind.eq_ignore_ascii_case("PAID") + || product["regions"].is_array()) + .then(|| product["regions"].as_array().cloned()) + .flatten() + .filter(|values| !values.is_empty()) + }) + .unwrap_or_default(); + let mut seen = std::collections::HashSet::new(); + regions.into_iter().filter_map(|region| { + let code = region["metroRegion"].as_str()?.trim(); + if code.is_empty() || !seen.insert(code.to_owned()) { return None; } + Some(json!({ + "code":code,"name":region["metroRegionName"].as_str().unwrap_or(code), + "isAvailable":region["isAvailable"].as_bool().unwrap_or(true), + "isCurrent":current_code == Some(code),"isRecommended":region["isRecommendedRegion"].as_bool().unwrap_or(false) + })) + }).collect() +} + +fn paywall_headers(token: &str) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + "idtoken", + HeaderValue::from_str(token).map_err(|_| ServiceError { + code: "invalid_params", + message: "Invalid authentication token".to_owned(), + })?, + ); + Ok(headers) +} + +fn session_token(auth: &AuthSession) -> Result<&str, ServiceError> { + auth.tokens + .id_token + .as_deref() + .or(Some(auth.tokens.access_token.as_str())) + .filter(|value| !value.is_empty()) + .ok_or_else(|| ServiceError { + code: "authentication_required", + message: "No authenticated token is available".to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn product_regions_are_deduplicated() { + let payload = json!({"products":[{"productType":"STORAGE","regions":[ + {"metroRegion":"NP-AMS-07","metroRegionName":"Amsterdam","isAvailable":true}, + {"metroRegion":"NP-AMS-07","metroRegionName":"Duplicate"} + ]}]}); + let result = locations_from_products(&payload, Some("NP-AMS-07")); + assert_eq!(result.len(), 1); + assert_eq!(result[0]["isCurrent"], true); + } +} diff --git a/external/opennow/native/opennow-core/src/proxy.rs b/external/opennow/native/opennow-core/src/proxy.rs new file mode 100644 index 000000000..3e71b7b0f --- /dev/null +++ b/external/opennow/native/opennow-core/src/proxy.rs @@ -0,0 +1,126 @@ +use reqwest::Proxy; +use reqwest::blocking::{Client, ClientBuilder}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::time::Duration; +use url::Url; + +const INVALID_PROXY: &str = "Invalid session proxy URL. Use http://host:port, https://host:port, socks4://host:port, or socks5://host:port."; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProxyConfig { + pub normalized_url: String, + pub cache_scope: String, + pub has_credentials: bool, +} + +pub fn config_from_settings(settings: &Value) -> Result, String> { + if settings["sessionProxyEnabled"].as_bool() != Some(true) { + return Ok(None); + } + let raw = settings["sessionProxyUrl"].as_str().unwrap_or(""); + normalize_proxy_url(raw).map(Some) +} + +pub fn client_for_settings(base: &Client, settings: &Value) -> Result { + client_for_settings_with(base, settings, |builder| builder) +} + +pub fn client_for_settings_with( + base: &Client, + settings: &Value, + configure: impl FnOnce(ClientBuilder) -> ClientBuilder, +) -> Result { + let Some(config) = config_from_settings(settings)? else { + return Ok(base.clone()); + }; + let proxy = Proxy::all(&config.normalized_url).map_err(|_| INVALID_PROXY.to_owned())?; + configure( + Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(8)) + .timeout(Duration::from_secs(20)) + .pool_idle_timeout(Duration::from_secs(60)) + .proxy(proxy), + ) + .build() + .map_err(|_| "Could not initialize the configured session proxy".to_owned()) +} + +pub fn normalize_proxy_url(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(INVALID_PROXY.to_owned()); + } + let candidate = if trimmed.contains("://") { + trimmed.to_owned() + } else { + format!("http://{trimmed}") + }; + let parsed = Url::parse(&candidate).map_err(|_| INVALID_PROXY.to_owned())?; + if !matches!(parsed.scheme(), "http" | "https" | "socks4" | "socks5") + || parsed.host_str().is_none() + || parsed.port().is_none() + || (parsed.password().is_some() && parsed.username().is_empty()) + || (parsed.path() != "" && parsed.path() != "/") + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(INVALID_PROXY.to_owned()); + } + let endpoint = format!( + "{}://{}:{}", + parsed.scheme(), + parsed.host_str().unwrap_or_default().to_ascii_lowercase(), + parsed.port().unwrap_or_default() + ); + let digest = Sha256::digest(endpoint.as_bytes()); + let digest = format!("{digest:x}"); + Ok(ProxyConfig { + normalized_url: parsed.to_string(), + cache_scope: format!("proxy:{}", &digest[..16]), + has_credentials: !parsed.username().is_empty(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn proxy_configuration_is_strict_and_credential_safe() { + let direct = json!({"sessionProxyEnabled":false,"sessionProxyUrl":"ignored:80"}); + assert_eq!(config_from_settings(&direct).unwrap(), None); + + let first = config_from_settings(&json!({ + "sessionProxyEnabled":true, + "sessionProxyUrl":"user:secret@Proxy.Example:8080" + })) + .unwrap() + .unwrap(); + let second = config_from_settings(&json!({ + "sessionProxyEnabled":true, + "sessionProxyUrl":"http://other:password@proxy.example:8080" + })) + .unwrap() + .unwrap(); + assert!(first.has_credentials); + assert_eq!(first.cache_scope, second.cache_scope); + assert!(!first.cache_scope.contains("secret")); + assert!( + config_from_settings(&json!({ + "sessionProxyEnabled":true, + "sessionProxyUrl":"ftp://proxy.example:21" + })) + .is_err() + ); + assert!( + config_from_settings(&json!({ + "sessionProxyEnabled":true, + "sessionProxyUrl":"proxy.example" + })) + .is_err() + ); + } +} diff --git a/external/opennow/native/opennow-core/src/push/config.rs b/external/opennow/native/opennow-core/src/push/config.rs new file mode 100644 index 000000000..99b4b05d7 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/config.rs @@ -0,0 +1,347 @@ +use crate::push::PushOwnerConfig; +use crate::push::registration::{PushEndpoints, PushIdentity}; +use serde_json::Value; +use std::io::Read; +use std::path::Path; + +pub const FILE_NAME: &str = "push.json"; +pub const MAXIMUM_CONFIG_BYTES: u64 = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PushConfig { + Absent, + Configured(Box), + Suppressed, +} + +pub fn load_for_provider(data_dir: &Path, provider_idp_id: &str, device_id: String) -> PushConfig { + let path = data_dir.join(FILE_NAME); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return PushConfig::Absent, + Err(_) => return PushConfig::Suppressed, + Ok(_) => {} + } + match std::fs::metadata(&path) { + Ok(metadata) if metadata.is_file() => {} + _ => return PushConfig::Suppressed, + } + let Some(bytes) = read_bounded(&path) else { + return PushConfig::Suppressed; + }; + let Ok(raw) = serde_json::from_slice::(&bytes) else { + return PushConfig::Suppressed; + }; + let Some(entry) = select_entry(&raw, provider_idp_id) else { + return PushConfig::Suppressed; + }; + from_json(entry, provider_idp_id, device_id) + .map(|config| PushConfig::Configured(Box::new(config))) + .unwrap_or(PushConfig::Suppressed) +} + +fn read_bounded(path: &Path) -> Option> { + let file = std::fs::File::open(path).ok()?; + let mut bytes = Vec::new(); + file.take(MAXIMUM_CONFIG_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + (bytes.len() as u64 <= MAXIMUM_CONFIG_BYTES).then_some(bytes) +} + +fn select_entry<'a>(raw: &'a Value, provider_idp_id: &str) -> Option<&'a Value> { + match raw { + Value::Array(entries) => entries.iter().find(|entry| { + entry + .get("providerIdpId") + .and_then(Value::as_str) + .is_some_and(|provider| provider == provider_idp_id) + }), + Value::Object(_) => Some(raw), + _ => None, + } +} + +pub(crate) fn pns_base_is_usable(base: &str) -> bool { + let Ok(url) = url::Url::parse(base) else { + return false; + }; + url.scheme() == "https" + && url.host_str().is_some_and(|host| !host.is_empty()) + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() +} + +pub fn from_json(raw: &Value, provider_idp_id: &str, device_id: String) -> Option { + let object = raw.as_object()?; + if let Some(enabled) = object.get("enabled") { + if enabled.as_bool() != Some(true) { + return None; + } + } + let string = |key: &str| -> Option { + object + .get(key)? + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + }; + let configured_provider = string("providerIdpId")?; + if configured_provider != provider_idp_id { + return None; + } + let project_id = string("projectId")?; + let api_key = string("apiKey")?; + let sender_id = string("senderId")?; + let app_id = string("appId")?; + let firebase_app_id = string("firebaseAppId")?; + let pns_server = string("pnsServer")?; + let pns_version = string("pnsVersion").unwrap_or_else(|| "v1".to_owned()); + let pns_client_id = string("pnsClientId")?; + let pns_base = format!( + "{}/{}", + pns_server.trim_end_matches('/'), + pns_version.trim_matches('/') + ); + if !pns_base_is_usable(&pns_server) || !pns_base_is_usable(&pns_base) { + return None; + } + let endpoints = PushEndpoints { + pns: pns_base, + pns_client_id, + ..PushEndpoints::default() + }; + let identity = PushIdentity { + project_id, + api_key, + sender_id, + app_id, + firebase_app_id, + vapid_key: string("vapidKey"), + }; + Some(PushOwnerConfig::bounded( + endpoints, + identity, + configured_provider, + device_id, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn raw() -> Value { + json!({ + "providerIdpId": "nvidia", + "projectId": "project", + "apiKey": "api-key", + "senderId": "123456789012", + "appId": "app", + "firebaseAppId": "1:1:web:abc", + "vapidKey": "vapid", + "pnsServer": "https://pns.example/", + "pnsVersion": "v1", + "pnsClientId": "client" + }) + } + + fn data_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("opennow-push-config-{name}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn load(dir: &Path, provider: &str) -> PushConfig { + load_for_provider(dir, provider, "device".into()) + } + + #[test] + fn valid_config_produces_a_bounded_owner_config() { + let config = from_json(&raw(), "nvidia", "device".into()).unwrap(); + assert_eq!(config.identity.project_id, "project"); + assert_eq!(config.identity.sender_id, "123456789012"); + assert_eq!(config.endpoints.pns, "https://pns.example/v1"); + assert_eq!(config.endpoints.pns_client_id, "client"); + assert_eq!(config.device_id, "device"); + assert_eq!(config.endpoints.mcs_host, "mtalk.google.com"); + assert!(config.maximum_frame_bytes <= 64 * 1024); + } + + #[test] + fn provider_mismatch_never_starts_the_vendor_config() { + assert!(from_json(&raw(), "alliance", "device".into()).is_none()); + } + + #[test] + fn incomplete_or_blank_config_is_rejected() { + let mut value = raw(); + value["projectId"] = json!(" "); + assert!(from_json(&value, "nvidia", "device".into()).is_none()); + let mut value = raw(); + value.as_object_mut().unwrap().remove("pnsServer"); + assert!(from_json(&value, "nvidia", "device".into()).is_none()); + assert!(from_json(&json!("not an object"), "nvidia", "device".into()).is_none()); + } + + #[test] + fn missing_vapid_key_is_allowed_and_version_defaults() { + let mut value = raw(); + value.as_object_mut().unwrap().remove("vapidKey"); + value.as_object_mut().unwrap().remove("pnsVersion"); + let config = from_json(&value, "nvidia", "device".into()).unwrap(); + assert!(config.identity.vapid_key.is_none()); + assert_eq!(config.endpoints.pns, "https://pns.example/v1"); + } + + #[test] + fn an_explicit_disable_suppresses_the_override() { + assert!(from_json(&json!({"enabled": false}), "nvidia", "device".into()).is_none()); + let mut value = raw(); + value["enabled"] = json!(false); + assert!(from_json(&value, "nvidia", "device".into()).is_none()); + assert!(from_json(&json!({"enabled": "no"}), "nvidia", "device".into()).is_none()); + let mut value = raw(); + value["enabled"] = json!(true); + assert!(from_json(&value, "nvidia", "device".into()).is_some()); + } + + #[test] + fn the_config_file_is_absent_without_an_override() { + let dir = data_dir("absent"); + assert_eq!(load(&dir, "nvidia"), PushConfig::Absent); + } + + #[test] + fn the_config_file_is_configured_for_a_complete_matching_entry() { + let dir = data_dir("configured"); + std::fs::write(dir.join(FILE_NAME), raw().to_string()).unwrap(); + let PushConfig::Configured(config) = load(&dir, "nvidia") else { + panic!("a complete entry must be configured"); + }; + assert_eq!(config.identity.project_id, "project"); + assert_eq!(config.provider_id, "nvidia"); + } + + #[test] + fn an_existing_override_never_falls_back_to_a_default() { + let dir = data_dir("suppressed"); + std::fs::write(dir.join(FILE_NAME), raw().to_string()).unwrap(); + assert_eq!(load(&dir, "alliance"), PushConfig::Suppressed); + std::fs::write(dir.join(FILE_NAME), b"not json").unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Suppressed); + std::fs::write(dir.join(FILE_NAME), json!({"enabled": false}).to_string()).unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Suppressed); + } + + #[test] + fn the_config_file_selects_the_matching_provider_entry() { + let dir = data_dir("array"); + let entries = json!([ + {"providerIdpId": "alliance", "projectId": "other"}, + raw() + ]); + std::fs::write(dir.join(FILE_NAME), entries.to_string()).unwrap(); + let PushConfig::Configured(config) = load(&dir, "nvidia") else { + panic!("the matching entry must be configured"); + }; + assert_eq!(config.identity.project_id, "project"); + assert_eq!(load(&dir, "alliance"), PushConfig::Suppressed); + } + + #[test] + fn oversized_config_files_are_suppressed() { + let dir = data_dir("oversized"); + std::fs::write( + dir.join(FILE_NAME), + vec![b' '; MAXIMUM_CONFIG_BYTES as usize + 1], + ) + .unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Suppressed); + } + + #[test] + fn an_empty_config_file_is_suppressed() { + let dir = data_dir("empty"); + std::fs::write(dir.join(FILE_NAME), b"").unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Suppressed); + } + + #[test] + fn an_insecure_or_ambiguous_pns_base_is_rejected() { + for server in [ + "http://pns.example", + "https://", + "https://user@pns.example", + "https://user:pw@pns.example", + "https://pns.example/?x=1", + "https://pns.example/#f", + "pns.example", + "not a url", + ] { + let mut value = raw(); + value["pnsServer"] = json!(server); + assert!( + from_json(&value, "nvidia", "device".into()).is_none(), + "the pns base {server} must be rejected" + ); + } + } + + #[test] + fn a_valid_pns_base_keeps_the_documented_shape() { + let mut value = raw(); + value["pnsServer"] = json!("https://pns.example/api"); + value["pnsVersion"] = json!("v2"); + let config = from_json(&value, "nvidia", "device".into()).unwrap(); + assert_eq!(config.endpoints.pns, "https://pns.example/api/v2"); + } + + #[cfg(unix)] + #[test] + fn a_dangling_config_link_is_suppressed() { + let dir = data_dir("dangling"); + let link = dir.join(FILE_NAME); + std::os::unix::fs::symlink(dir.join("missing-push.json"), &link).unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Suppressed); + std::fs::remove_file(&link).unwrap(); + assert_eq!(load(&dir, "nvidia"), PushConfig::Absent); + } + + #[cfg(unix)] + #[test] + fn a_config_link_to_a_valid_file_is_configured() { + let dir = data_dir("link-valid"); + let target = dir.join("target.json"); + std::fs::write(&target, raw().to_string()).unwrap(); + std::os::unix::fs::symlink(&target, dir.join(FILE_NAME)).unwrap(); + let PushConfig::Configured(config) = load(&dir, "nvidia") else { + panic!("a valid link target must stay configured"); + }; + assert_eq!(config.identity.project_id, "project"); + } + + #[cfg(unix)] + #[test] + fn a_non_regular_override_entry_is_suppressed_without_blocking() { + use std::os::unix::ffi::OsStrExt as _; + let dir = data_dir("fifo"); + let path = dir.join(FILE_NAME); + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(load(&dir, "nvidia")); + }); + assert_eq!( + receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("a non-regular override entry must not block the loader"), + PushConfig::Suppressed + ); + } +} diff --git a/external/opennow/native/opennow-core/src/push/decrypt.rs b/external/opennow/native/opennow-core/src/push/decrypt.rs new file mode 100644 index 000000000..eb4885843 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/decrypt.rs @@ -0,0 +1,368 @@ +use crate::push::PushError; +use aes_gcm::aead::{Aead, KeyInit}; +use aes_gcm::{Aes128Gcm, Nonce}; +use hkdf::Hkdf; +use p256::ecdh; +use p256::elliptic_curve::sec1::ToEncodedPoint; +use p256::{PublicKey, SecretKey}; +use rand::RngCore; +use sha2::Sha256; + +const AUTH_SECRET_BYTES: usize = 16; +const PUBLIC_KEY_BYTES: usize = 65; +const SALT_BYTES: usize = 16; +const NONCE_BYTES: usize = 12; +const TAG_BYTES: usize = 16; +const MAXIMUM_RECORD_SIZE: u32 = 16 * 1024; +const MINIMUM_RECORD_SIZE: u32 = 18; +const AES128GCM_HEADER_BYTES: usize = 21; + +pub struct EceKeyPair { + secret: SecretKey, + public: Vec, + auth_secret: Vec, +} + +impl EceKeyPair { + pub fn generate() -> Result { + let mut scalar = [0_u8; 32]; + rand::rng().fill_bytes(&mut scalar); + let secret = SecretKey::from_slice(&scalar) + .map_err(|_| PushError::new("push_key_generation", "Could not create a push key"))?; + let mut auth_secret = vec![0_u8; AUTH_SECRET_BYTES]; + rand::rng().fill_bytes(&mut auth_secret); + Ok(Self { + public: secret + .public_key() + .to_encoded_point(false) + .as_bytes() + .to_vec(), + secret, + auth_secret, + }) + } + + pub fn from_parts( + private: &[u8], + public: &[u8], + auth_secret: &[u8], + ) -> Result { + if public.len() != PUBLIC_KEY_BYTES || auth_secret.len() != AUTH_SECRET_BYTES { + return Err(PushError::new( + "push_key_invalid", + "The stored push key material is malformed", + )); + } + let secret = SecretKey::from_slice(private) + .map_err(|_| PushError::new("push_key_invalid", "The stored push key is invalid"))?; + let derived = secret.public_key().to_encoded_point(false); + if derived.as_bytes() != public { + return Err(PushError::new( + "push_key_mismatch", + "The stored push key does not match its public key", + )); + } + Ok(Self { + secret, + public: public.to_vec(), + auth_secret: auth_secret.to_vec(), + }) + } + + pub fn public_key(&self) -> &[u8] { + &self.public + } + + pub fn auth_secret(&self) -> &[u8] { + &self.auth_secret + } + + pub fn private_key(&self) -> Vec { + self.secret.to_bytes().to_vec() + } + + fn shared_secret(&self, server_public: &[u8]) -> Result, PushError> { + let key = PublicKey::from_sec1_bytes(server_public).map_err(|_| { + PushError::new( + "push_decrypt_invalid", + "The message key header is not a valid public key", + ) + })?; + let shared = ecdh::diffie_hellman(self.secret.to_nonzero_scalar(), key.as_affine()); + Ok(shared.raw_secret_bytes().to_vec()) + } +} + +pub fn decrypt_web_push( + keys: &EceKeyPair, + content_encoding: Option<&str>, + crypto_key: Option<&str>, + encryption: Option<&str>, + body: &[u8], +) -> Result, PushError> { + match content_encoding { + Some(encoding) if encoding.eq_ignore_ascii_case("aes128gcm") => { + decrypt_aes128gcm_body(keys, body) + } + Some(encoding) if encoding.eq_ignore_ascii_case("aesgcm") => { + decrypt_legacy_body(keys, crypto_key, encryption, body) + } + Some(_) => Err(PushError::new( + "push_decrypt_unsupported", + "The message declares an unsupported content encoding", + )), + None if crypto_key.is_some() && encryption.is_some() => { + decrypt_legacy_body(keys, crypto_key, encryption, body) + } + None => decrypt_aes128gcm_body(keys, body), + } +} + +fn decrypt_legacy_body( + keys: &EceKeyPair, + crypto_key: Option<&str>, + encryption: Option<&str>, + body: &[u8], +) -> Result, PushError> { + let crypto_key = crypto_key.ok_or_else(|| { + PushError::new( + "push_decrypt_invalid", + "The message is missing its key header", + ) + })?; + let encryption = encryption.ok_or_else(|| { + PushError::new( + "push_decrypt_invalid", + "The message is missing its salt header", + ) + })?; + let dh = parameter(crypto_key, "dh").ok_or_else(|| { + PushError::new( + "push_decrypt_invalid", + "The message is missing its key header", + ) + })?; + let salt = parameter(encryption, "salt").ok_or_else(|| { + PushError::new( + "push_decrypt_invalid", + "The message is missing its salt header", + ) + })?; + decrypt_aesgcm(keys, &dh, &salt, body) +} + +fn parameter(value: &str, name: &str) -> Option> { + value.split(';').find_map(|part| { + part.trim() + .strip_prefix(name) + .and_then(|rest| rest.strip_prefix('=')) + .and_then(base64_url_decode) + }) +} + +fn base64_url_decode(value: &str) -> Option> { + use base64::Engine as _; + base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value.trim()) + .ok() +} + +pub fn decrypt_aes128gcm_body(keys: &EceKeyPair, body: &[u8]) -> Result, PushError> { + if body.len() < AES128GCM_HEADER_BYTES + TAG_BYTES { + return Err(PushError::new( + "push_decrypt_invalid", + "The encrypted message is too short", + )); + } + let salt = &body[0..16]; + let record_size = u32::from_be_bytes([body[16], body[17], body[18], body[19]]); + let id_length = body[20] as usize; + if body.len() < AES128GCM_HEADER_BYTES + id_length + TAG_BYTES { + return Err(PushError::new( + "push_decrypt_invalid", + "The encrypted message header is malformed", + )); + } + let server_public = &body[AES128GCM_HEADER_BYTES..AES128GCM_HEADER_BYTES + id_length]; + let records = &body[AES128GCM_HEADER_BYTES + id_length..]; + let shared = keys.shared_secret(server_public)?; + + let mut key_info = Vec::with_capacity(19 + PUBLIC_KEY_BYTES * 2); + key_info.extend_from_slice(b"WebPush: info\0"); + key_info.extend_from_slice(&keys.public); + key_info.extend_from_slice(server_public); + + let mut input_key = [0_u8; 32]; + Hkdf::::new(Some(&keys.auth_secret), &shared) + .expand(&key_info, &mut input_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message key derivation failed"))?; + let (content_key, base_nonce) = derive_content_keys(&input_key, salt)?; + decode_aes128gcm_records(&content_key, &base_nonce, record_size, records) +} + +pub fn derive_content_keys( + input_key: &[u8], + salt: &[u8], +) -> Result<([u8; 16], [u8; 12]), PushError> { + let prk = Hkdf::::new(Some(salt), input_key); + let mut content_key = [0_u8; 16]; + let mut nonce = [0_u8; 12]; + prk.expand(b"Content-Encoding: aes128gcm\0", &mut content_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message key derivation failed"))?; + prk.expand(b"Content-Encoding: nonce\0", &mut nonce) + .map_err(|_| { + PushError::new("push_decrypt_failed", "The message nonce derivation failed") + })?; + Ok((content_key, nonce)) +} + +pub fn decode_aes128gcm_records( + content_key: &[u8; 16], + base_nonce: &[u8; 12], + record_size: u32, + records: &[u8], +) -> Result, PushError> { + if !(MINIMUM_RECORD_SIZE..=MAXIMUM_RECORD_SIZE).contains(&record_size) { + return Err(PushError::new( + "push_decrypt_invalid", + "The message record size is outside the accepted range", + )); + } + if records.is_empty() { + return Err(PushError::new( + "push_decrypt_invalid", + "The encrypted message carries no record", + )); + } + let cipher = Aes128Gcm::new_from_slice(content_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message cipher could not start"))?; + let mut plaintext = Vec::new(); + let mut sequence = 0_u64; + let mut remaining = records; + while !remaining.is_empty() { + let take = remaining.len().min(record_size as usize); + let (record, rest) = remaining.split_at(take); + remaining = rest; + if record.len() < TAG_BYTES + 1 { + return Err(PushError::new( + "push_decrypt_invalid", + "A message record is too short", + )); + } + let last = remaining.is_empty(); + plaintext.extend_from_slice(&decrypt_record( + &cipher, base_nonce, sequence, record, last, + )?); + sequence += 1; + } + Ok(plaintext) +} + +fn decrypt_record( + cipher: &Aes128Gcm, + base_nonce: &[u8; 12], + sequence: u64, + record: &[u8], + last: bool, +) -> Result, PushError> { + let sequence_bytes = sequence.to_be_bytes(); + let mut nonce = *base_nonce; + for (target, byte) in nonce[4..].iter_mut().zip(sequence_bytes.iter()) { + *target ^= *byte; + } + let decrypted = cipher + .decrypt(Nonce::from_slice(&nonce), record) + .map_err(|_| PushError::new("push_decrypt_failed", "The message could not be decrypted"))?; + let end = decrypted + .iter() + .rposition(|byte| *byte != 0) + .ok_or_else(|| { + PushError::new( + "push_decrypt_invalid", + "The decrypted record carries no padding delimiter", + ) + })?; + let delimiter = decrypted[end]; + let expected = if last { 2 } else { 1 }; + if delimiter != expected { + return Err(PushError::new( + "push_decrypt_invalid", + "The decrypted record carries an invalid padding delimiter", + )); + } + Ok(decrypted[..end].to_vec()) +} + +pub fn decrypt_aesgcm( + keys: &EceKeyPair, + server_public: &[u8], + salt: &[u8], + payload: &[u8], +) -> Result, PushError> { + if salt.len() != SALT_BYTES { + return Err(PushError::new( + "push_decrypt_invalid", + "The message salt is outside the accepted range", + )); + } + if payload.len() <= TAG_BYTES + 2 { + return Err(PushError::new( + "push_decrypt_invalid", + "The encrypted message is too short", + )); + } + if server_public.len() != PUBLIC_KEY_BYTES { + return Err(PushError::new( + "push_decrypt_invalid", + "The message key header is malformed", + )); + } + let shared = keys.shared_secret(server_public)?; + let mut input_key = [0_u8; 32]; + Hkdf::::new(Some(&keys.auth_secret), &shared) + .expand(b"Content-Encoding: auth\0", &mut input_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message key derivation failed"))?; + let mut context = Vec::with_capacity(134); + context.extend_from_slice(&[0, PUBLIC_KEY_BYTES as u8]); + context.extend_from_slice(&keys.public); + context.extend_from_slice(&[0, PUBLIC_KEY_BYTES as u8]); + context.extend_from_slice(server_public); + let mut key_info = b"Content-Encoding: aesgcm\0P-256\0".to_vec(); + key_info.extend_from_slice(&context); + let mut nonce_info = b"Content-Encoding: nonce\0P-256\0".to_vec(); + nonce_info.extend_from_slice(&context); + let prk = Hkdf::::new(Some(salt), &input_key); + let mut content_key = [0_u8; 16]; + let mut nonce = [0_u8; NONCE_BYTES]; + prk.expand(&key_info, &mut content_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message key derivation failed"))?; + prk.expand(&nonce_info, &mut nonce).map_err(|_| { + PushError::new("push_decrypt_failed", "The message nonce derivation failed") + })?; + let cipher = Aes128Gcm::new_from_slice(&content_key) + .map_err(|_| PushError::new("push_decrypt_failed", "The message cipher could not start"))?; + let mut plaintext = cipher + .decrypt(Nonce::from_slice(&nonce), payload) + .map_err(|_| PushError::new("push_decrypt_failed", "The message could not be decrypted"))?; + if plaintext.len() <= 2 { + return Err(PushError::new( + "push_decrypt_invalid", + "The decrypted message is too short", + )); + } + let padding = ((plaintext[0] as usize) << 8) | plaintext[1] as usize; + let start = 2 + padding; + if start >= plaintext.len() { + return Err(PushError::new( + "push_decrypt_invalid", + "The decrypted message carries invalid padding", + )); + } + if plaintext[2..start].iter().any(|byte| *byte != 0) { + return Err(PushError::new( + "push_decrypt_invalid", + "The decrypted message carries invalid padding", + )); + } + Ok(plaintext.split_off(start)) +} diff --git a/external/opennow/native/opennow-core/src/push/event.rs b/external/opennow/native/opennow-core/src/push/event.rs new file mode 100644 index 000000000..05a9bceec --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/event.rs @@ -0,0 +1,102 @@ +use crate::push::PushEvent; +use serde_json::{Value, json}; + +pub const EVENT_NAME: &str = "account.push.changed"; + +pub fn payload(event: &PushEvent, generation: u64) -> Value { + match event { + PushEvent::LibraryChanged { changed_ids } => json!({ + "generation": generation, + "kind": "library", + "changedIds": changed_ids, + }), + PushEvent::FavoritesChanged { changed_ids } => json!({ + "generation": generation, + "kind": "favorites", + "changedIds": changed_ids, + }), + PushEvent::SubscriptionChanged { changed_ids } => json!({ + "generation": generation, + "kind": "subscription", + "changedIds": changed_ids, + }), + PushEvent::LinkedAccountChanged { + account_type, + linked, + } => json!({ + "generation": generation, + "kind": "linked-account", + "accountType": account_type, + "linked": linked, + }), + PushEvent::PlatformSyncChanged { + platform_code, + sync_state, + sync_date, + sync_game_count, + } => json!({ + "generation": generation, + "kind": "platform-sync", + "platformCode": platform_code, + "syncState": sync_state, + "syncDate": sync_date, + "syncGameCount": sync_game_count, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn library_events_carry_the_generation_and_changed_ids() { + let value = payload( + &PushEvent::LibraryChanged { + changed_ids: vec!["app-1".into()], + }, + 7, + ); + assert_eq!(value["kind"], "library"); + assert_eq!(value["generation"], 7); + assert_eq!(value["changedIds"][0], "app-1"); + assert_eq!(EVENT_NAME, "account.push.changed"); + } + + #[test] + fn platform_sync_events_never_claim_completion() { + let value = payload( + &PushEvent::PlatformSyncChanged { + platform_code: Some("STEAM".into()), + sync_state: Some("SYNCING".into()), + sync_date: Some("2026-01-01T00:00:00Z".into()), + sync_game_count: Some(12), + }, + 9, + ); + assert_eq!(value["kind"], "platform-sync"); + assert_eq!(value["syncState"], "SYNCING"); + assert!(value.get("completed").is_none()); + } + + #[test] + fn linked_account_and_favorites_payloads_are_explicit() { + let linked = payload( + &PushEvent::LinkedAccountChanged { + account_type: Some("STEAM".into()), + linked: Some(true), + }, + 1, + ); + assert_eq!(linked["accountType"], "STEAM"); + assert_eq!(linked["linked"], true); + let favorites = payload( + &PushEvent::FavoritesChanged { + changed_ids: vec![], + }, + 2, + ); + assert_eq!(favorites["kind"], "favorites"); + assert_eq!(favorites["changedIds"].as_array().unwrap().len(), 0); + } +} diff --git a/external/opennow/native/opennow-core/src/push/mod.rs b/external/opennow/native/opennow-core/src/push/mod.rs new file mode 100644 index 000000000..b5685818a --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/mod.rs @@ -0,0 +1,163 @@ +mod config; +mod decrypt; +mod event; +mod owner; +mod protocol; +mod registration; +mod store; +mod transport; +mod vendor_default; + +pub use config::{ + FILE_NAME as PUSH_CONFIG_FILE, PushConfig, load_for_provider as push_config_for_provider, +}; +pub use decrypt::{EceKeyPair, decrypt_web_push}; +pub use event::{EVENT_NAME as PUSH_EVENT_NAME, payload as push_event_payload}; +pub use owner::{ + GfnTokenSource, PushEvent, PushOwner, PushOwnerConfig, PushOwnerDeps, PushScope, PushSink, + ScopeSource, +}; +pub use registration::{ + DEVICE_IDENTITY_REJECTED, HttpMethod, HttpRequest, HttpResponse, PushEndpoints, PushHttp, + PushIdentity, Registration, RegistrationClient, ReqwestPushHttp, checkin_and_register, + pns_unregister, +}; +pub use store::{PushStateStore, RegistrationStore}; +pub use vendor_default::config_for_provider as bundled_push_config; + +pub const EVENT_NAME: &str = crate::push::event::EVENT_NAME; +pub use transport::{ + PushTransport, PushTransportFactory, TlsPushTransport, TlsPushTransportFactory, +}; + +#[derive(Debug, Clone)] +pub struct PushError { + pub code: &'static str, + pub message: String, +} + +impl PushError { + pub fn new(code: &'static str, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum MessageType { + ProductChange, + AssetChange, + SessionChange, + LayoutChange, + GswsSync, + SectionChange, + LinkedAccountChange, + AppChange, + ServerInfoSync, + ConfigurationChange, + SubscriptionChange, + PatchingEvent, + KvStoreChange, + LibraryChange, + PlatformSyncChange, + CampaignChange, + FavoritesChange, +} + +impl MessageType { + pub fn parse(value: &str) -> Option { + Some(match value { + "PRODUCT_CHANGE" => Self::ProductChange, + "ASSET_CHANGE" => Self::AssetChange, + "SESSION_CHANGE" => Self::SessionChange, + "LAYOUT_CHANGE" => Self::LayoutChange, + "GSWS_SYNC" => Self::GswsSync, + "SECTION_CHANGE" => Self::SectionChange, + "LINKEDACCOUNT_CHANGE" => Self::LinkedAccountChange, + "APP_CHANGE" => Self::AppChange, + "SERVER_INFO_SYNC" => Self::ServerInfoSync, + "CONFIGURATION_CHANGE" => Self::ConfigurationChange, + "SUBSCRIPTION_CHANGE" => Self::SubscriptionChange, + "PATCHING_EVENT" => Self::PatchingEvent, + "KV_STORE_CHANGE" => Self::KvStoreChange, + "LIBRARY_CHANGE" => Self::LibraryChange, + "PLATFORM_SYNC_CHANGE" => Self::PlatformSyncChange, + "CAMPAIGN_CHANGE" => Self::CampaignChange, + "FAVORITES_CHANGE" => Self::FavoritesChange, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PushMessage { + pub message_type: Option, + pub region_ids: Vec, + pub changed_ids: Vec, + pub change_id_type: Option, + pub platform_code: Option, + pub sync_state: Option, + pub sync_date: Option, + pub sync_game_count: Option, + pub account_type: Option, + pub account_linked: Option, +} + +impl PushMessage { + pub fn parse(data: &serde_json::Map) -> Option { + let mut message = PushMessage::default(); + let mut saw_type = false; + for (key, value) in data { + match key.as_str() { + "messageType" => { + let text = json_string(value)?; + message.message_type = MessageType::parse(&text); + message.message_type?; + saw_type = true; + } + "changedIds" => message.changed_ids = json_string_list(value), + "changeIdType" => message.change_id_type = json_string(value), + "regionIds" => message.region_ids = json_string_list(value), + "platformSyncInfo" => { + let text = json_string(value)?; + let info: serde_json::Value = serde_json::from_str(&text).ok()?; + message.platform_code = info["platformCode"].as_str().map(str::to_owned); + message.sync_state = info["syncState"].as_str().map(str::to_owned); + message.sync_date = info["syncDate"].as_str().map(str::to_owned); + message.sync_game_count = info["syncGameCount"].as_u64(); + } + "accountLinkedInfo" => { + let text = json_string(value)?; + let info: serde_json::Value = serde_json::from_str(&text).ok()?; + message.account_type = info["accountType"].as_str().map(str::to_owned); + message.account_linked = info["linked"].as_bool(); + } + _ => {} + } + } + saw_type.then_some(message) + } +} + +fn json_string(value: &serde_json::Value) -> Option { + if let Some(text) = value.as_str() { + if let Ok(inner) = serde_json::from_str::(text) { + return Some(inner); + } + return Some(text.to_owned()); + } + Some(value.to_string()) +} + +fn json_string_list(value: &serde_json::Value) -> Vec { + let text = json_string(value).unwrap_or_default(); + serde_json::from_str::>(&text).unwrap_or_default() +} + +pub const MAXIMUM_FRAME_BYTES: usize = 64 * 1024; +pub const MAXIMUM_CHANGED_IDS: usize = 64; + +#[cfg(test)] +mod tests; diff --git a/external/opennow/native/opennow-core/src/push/owner.rs b/external/opennow/native/opennow-core/src/push/owner.rs new file mode 100644 index 000000000..5939a10b2 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/owner.rs @@ -0,0 +1,1179 @@ +use crate::push::MessageType; +use crate::push::PushError; +use crate::push::decrypt::{EceKeyPair, decrypt_web_push}; +use crate::push::protocol::{ + Frame, FrameReader, decode_frame, encode_close, encode_heartbeat_ack, encode_heartbeat_ping, + encode_login_request, encode_stream_ack, +}; +use crate::push::registration::{ + DEVICE_IDENTITY_REJECTED, PushEndpoints, PushHttp, PushIdentity, Registration, + RegistrationClient, checkin_and_register, +}; +use crate::push::store::PushStateStore; +use crate::push::transport::PushTransportFactory; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +const SESSION_SLICE: Duration = Duration::from_millis(250); +const PAUSED_SLICE: Duration = Duration::from_secs(1); +const STORED_REGISTRATION_RETIRED: &str = "push_registration_retired"; +const REGISTRATION_MAXIMUM_AGE_SECONDS: u64 = 7 * 24 * 60 * 60; +const UNACKNOWLEDGED_PERSISTENT_STREAM_ACK: usize = 10; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushScope { + pub user_id: String, + pub provider_id: String, + pub generation: u64, +} + +impl PushScope { + pub fn account_key(&self) -> String { + format!("{}:{}", self.provider_id, self.user_id) + } + + pub(crate) fn same_account(&self, other: &PushScope) -> bool { + self.user_id == other.user_id && self.provider_id == other.provider_id + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PushEvent { + LibraryChanged { + changed_ids: Vec, + }, + FavoritesChanged { + changed_ids: Vec, + }, + SubscriptionChanged { + changed_ids: Vec, + }, + LinkedAccountChanged { + account_type: Option, + linked: Option, + }, + PlatformSyncChanged { + platform_code: Option, + sync_state: Option, + sync_date: Option, + sync_game_count: Option, + }, +} + +pub type PushSink = Arc; +pub type ScopeSource = Arc Option + Send + Sync>; +pub type GfnTokenSource = Arc Option + Send + Sync>; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushOwnerConfig { + pub endpoints: PushEndpoints, + pub identity: PushIdentity, + pub provider_id: String, + pub device_id: String, + pub connect_timeout: Duration, + pub read_timeout: Duration, + pub login_timeout: Duration, + pub heartbeat_interval: Duration, + pub heartbeat_ack_timeout: Duration, + pub minimum_heartbeat_interval: Duration, + pub maximum_heartbeat_interval: Duration, + pub coalescing_window: Duration, + pub backoff_initial: Duration, + pub backoff_maximum: Duration, + pub stop_deadline: Duration, + pub maximum_frame_bytes: usize, + pub maximum_changed_ids: usize, + pub maximum_persistent_ids: usize, + pub registration_maximum_age: Duration, + pub registration_timeout: Duration, +} + +impl PushOwnerConfig { + pub fn bounded( + endpoints: PushEndpoints, + identity: PushIdentity, + provider_id: String, + device_id: String, + ) -> Self { + Self { + endpoints, + identity, + provider_id, + device_id, + connect_timeout: Duration::from_secs(20), + read_timeout: Duration::from_secs(5), + login_timeout: Duration::from_secs(20), + heartbeat_interval: Duration::from_secs(600), + heartbeat_ack_timeout: Duration::from_secs(60), + minimum_heartbeat_interval: Duration::from_secs(30), + maximum_heartbeat_interval: Duration::from_secs(28 * 60), + coalescing_window: Duration::from_secs(2), + backoff_initial: Duration::from_secs(2), + backoff_maximum: Duration::from_secs(60), + stop_deadline: Duration::from_secs(5), + maximum_frame_bytes: crate::push::MAXIMUM_FRAME_BYTES, + maximum_changed_ids: crate::push::MAXIMUM_CHANGED_IDS, + maximum_persistent_ids: 32, + registration_maximum_age: Duration::from_secs(REGISTRATION_MAXIMUM_AGE_SECONDS), + registration_timeout: Duration::from_secs(10), + } + } + + pub(crate) fn registration_fingerprint(&self, scope: &PushScope) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(b"opennow-push-registration\0"); + hasher.update(scope.account_key().as_bytes()); + hasher.update(b"\0"); + hasher.update(self.device_id.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.identity.project_id.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.identity.api_key.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.identity.sender_id.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.identity.app_id.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.identity.firebase_app_id.as_bytes()); + hasher.update(b"\0"); + hasher.update( + self.identity + .vapid_key + .as_deref() + .unwrap_or_default() + .as_bytes(), + ); + hasher.update(b"\0"); + hasher.update(self.endpoints.pns.as_bytes()); + hasher.update(b"\0"); + hasher.update(self.endpoints.pns_client_id.as_bytes()); + format!("{:x}", hasher.finalize()) + } +} + +pub struct PushOwnerDeps { + pub http: Arc, + pub transport: Arc, + pub store: Arc, + pub sink: PushSink, + pub scope_source: ScopeSource, + pub token_source: GfnTokenSource, +} + +struct IntentState { + active: Option, + supervised: bool, +} + +struct ConfigState { + config: PushOwnerConfig, + generation: u64, +} + +struct Intent { + state: Mutex, + wake: Condvar, + shutdown: AtomicBool, + paused: AtomicBool, + workers: AtomicUsize, + generation: AtomicU64, + refresh_pending: AtomicBool, + config: Mutex, +} + +impl Intent { + fn new(config: PushOwnerConfig) -> Self { + Self { + state: Mutex::new(IntentState { + active: None, + supervised: false, + }), + wake: Condvar::new(), + shutdown: AtomicBool::new(false), + paused: AtomicBool::new(false), + workers: AtomicUsize::new(0), + generation: AtomicU64::new(0), + refresh_pending: AtomicBool::new(false), + config: Mutex::new(ConfigState { + config, + generation: 1, + }), + } + } + + fn config(&self) -> (PushOwnerConfig, u64) { + let state = self.config.lock().expect("push config poisoned"); + (state.config.clone(), state.generation) + } + + fn replace_config(&self, config: PushOwnerConfig) { + { + let mut state = self.config.lock().expect("push config poisoned"); + state.config = config; + state.generation += 1; + } + self.wake.notify_all(); + } + + fn accepts_provider(&self, provider_id: &str) -> bool { + self.config + .lock() + .expect("push config poisoned") + .config + .provider_id + == provider_id + } + + fn paused(&self) -> bool { + self.paused.load(Ordering::SeqCst) + } + + fn request_registration_refresh(&self) { + self.refresh_pending.store(true, Ordering::SeqCst); + } + + fn take_registration_refresh(&self) -> bool { + self.refresh_pending.swap(false, Ordering::SeqCst) + } + + fn set_paused(&self, paused: bool) { + self.paused.store(paused, Ordering::SeqCst); + self.wake.notify_all(); + } + + fn desired(&self, deps: &PushOwnerDeps) -> Option { + if self.paused() { + return None; + } + let scope = (deps.scope_source)()?; + if scope.user_id.is_empty() || scope.provider_id.is_empty() { + return None; + } + if !self.accepts_provider(&scope.provider_id) { + return None; + } + self.generation.store(scope.generation, Ordering::SeqCst); + Some(scope) + } + + fn shutdown(&self) -> bool { + self.shutdown.load(Ordering::SeqCst) + } + + fn set_active(&self, active: Option) { + self.state.lock().expect("push intent poisoned").active = active; + } + + fn wait(&self, duration: Duration) -> bool { + let state = self.state.lock().expect("push intent poisoned"); + if self.shutdown() { + return false; + } + let (state, _) = self + .wake + .wait_timeout(state, duration) + .expect("push intent poisoned"); + drop(state); + !self.shutdown() + } +} + +pub struct PushOwner { + deps: PushOwnerDeps, + intent: Arc, + supervisor: Option>, +} + +impl PushOwner { + pub fn new(config: PushOwnerConfig, deps: PushOwnerDeps) -> Self { + Self { + deps, + intent: Arc::new(Intent::new(config)), + supervisor: None, + } + } + + pub fn update_config(&self, config: PushOwnerConfig) { + self.intent.replace_config(config); + } + + pub fn config(&self) -> (PushOwnerConfig, u64) { + self.intent.config() + } + + pub fn start(&mut self) -> Result<(), PushError> { + if self.supervisor.is_some() { + if self.intent.shutdown() { + return Err(PushError::new( + "push_worker_stopped", + "The push worker has been shut down", + )); + } + self.intent.set_paused(false); + return Ok(()); + } + let deps = PushOwnerDeps { + http: Arc::clone(&self.deps.http), + transport: Arc::clone(&self.deps.transport), + store: Arc::clone(&self.deps.store), + sink: Arc::clone(&self.deps.sink), + scope_source: Arc::clone(&self.deps.scope_source), + token_source: Arc::clone(&self.deps.token_source), + }; + let intent = Arc::clone(&self.intent); + intent.shutdown.store(false, Ordering::SeqCst); + intent.paused.store(false, Ordering::SeqCst); + intent.workers.store(1, Ordering::SeqCst); + intent + .state + .lock() + .expect("push intent poisoned") + .supervised = true; + let worker_intent = Arc::clone(&intent); + let handle = thread::Builder::new() + .name("opennow-push".to_owned()) + .spawn(move || { + let _guard = WorkerGuard(Arc::clone(&worker_intent)); + run_supervisor(&deps, &worker_intent); + }) + .map_err(|_| { + intent.workers.store(0, Ordering::SeqCst); + intent + .state + .lock() + .expect("push intent poisoned") + .supervised = false; + PushError::new("push_worker_failed", "The push worker could not start") + })?; + self.supervisor = Some(handle); + Ok(()) + } + + pub fn stop(&mut self) { + self.intent.set_paused(true); + self.intent.set_active(None); + } + + pub fn shutdown(&mut self, timeout: Duration) -> bool { + self.intent.shutdown.store(true, Ordering::SeqCst); + self.intent.set_active(None); + self.intent.wake.notify_all(); + let Some(handle) = self.supervisor.take() else { + return true; + }; + let deadline = Instant::now() + timeout; + while !handle.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + if handle.is_finished() { + let _ = handle.join(); + true + } else { + false + } + } + + pub fn wait_idle(&self, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + while self.is_running() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + !self.is_running() + } + + pub fn is_supervised(&self) -> bool { + self.intent + .state + .lock() + .expect("push intent poisoned") + .supervised + } + + pub fn is_running(&self) -> bool { + self.is_supervised() && self.active_scope().is_some() + } + + pub fn active_scope(&self) -> Option { + self.intent + .state + .lock() + .expect("push intent poisoned") + .active + .clone() + } + + pub fn worker_count(&self) -> usize { + self.intent.workers.load(Ordering::SeqCst) + } + + pub fn wait_stopped(&self, timeout: Duration) -> bool { + self.wait_idle(timeout) + } +} + +impl Drop for PushOwner { + fn drop(&mut self) { + let deadline = self.intent.config().0.stop_deadline; + let _ = self.shutdown(deadline); + } +} + +struct WorkerGuard(Arc); + +impl Drop for WorkerGuard { + fn drop(&mut self) { + self.0.workers.store(0, Ordering::SeqCst); + let mut state = self.0.state.lock().expect("push intent poisoned"); + state.supervised = false; + state.active = None; + drop(state); + self.0.wake.notify_all(); + } +} + +enum SessionOutcome { + ScopeChanged, + Completed, + Failed, + RegistrationRetired, +} + +fn run_supervisor(deps: &PushOwnerDeps, intent: &Arc) { + let mut backoff = intent.config().0.backoff_initial; + loop { + if intent.shutdown() { + return; + } + let (config, config_generation) = intent.config(); + let Some(scope) = intent.desired(deps) else { + let idle = if intent.paused() { + PAUSED_SLICE + } else { + SESSION_SLICE + }; + if !intent.wait(idle) { + return; + } + continue; + }; + match run_scoped_session(&config, config_generation, deps, intent, &scope) { + SessionOutcome::ScopeChanged => { + backoff = config.backoff_initial; + } + SessionOutcome::Completed => { + backoff = config.backoff_initial; + if intent.shutdown() || !intent.wait(SESSION_SLICE) { + return; + } + } + SessionOutcome::Failed | SessionOutcome::RegistrationRetired => { + if intent.shutdown() || !intent.wait(backoff) { + return; + } + backoff = (backoff * 2).min(config.backoff_maximum); + } + } + } +} + +fn run_scoped_session( + config: &PushOwnerConfig, + config_generation: u64, + deps: &PushOwnerDeps, + intent: &Arc, + scope: &PushScope, +) -> SessionOutcome { + let registration = match ensure_registration(config, config_generation, deps, intent, scope) { + Ok(Some(registration)) => registration, + Ok(None) => return SessionOutcome::ScopeChanged, + Err(error) => return session_failure_outcome(&error), + }; + match run_session( + config, + config_generation, + deps, + intent, + scope, + ®istration, + ) { + Ok(true) => SessionOutcome::ScopeChanged, + Ok(false) => SessionOutcome::Completed, + Err(error) => session_failure_outcome(&error), + } +} + +fn session_failure_outcome(error: &PushError) -> SessionOutcome { + if error.code == "push_scope_stale" { + SessionOutcome::ScopeChanged + } else if error.code == STORED_REGISTRATION_RETIRED { + SessionOutcome::RegistrationRetired + } else { + SessionOutcome::Failed + } +} + +fn clamped_heartbeat_interval(config: &PushOwnerConfig, interval: Duration) -> Duration { + interval.clamp( + config.minimum_heartbeat_interval, + config.maximum_heartbeat_interval, + ) +} + +fn advance_advertised_stream_id( + stream_id_in: u64, + advertised: &mut u64, + unacknowledged: &mut usize, +) { + if stream_id_in > *advertised { + *advertised = stream_id_in; + *unacknowledged = 0; + } +} + +fn session_is_current( + config: &PushOwnerConfig, + config_generation: u64, + deps: &PushOwnerDeps, + intent: &Arc, + scope: &PushScope, +) -> bool { + if intent.shutdown() { + return false; + } + let (current, current_generation) = intent.config(); + if current_generation != config_generation + || current.provider_id != config.provider_id + || current.provider_id != scope.provider_id + { + return false; + } + intent + .desired(deps) + .is_some_and(|desired| desired.same_account(scope)) +} + +fn ensure_registration( + config: &PushOwnerConfig, + config_generation: u64, + deps: &PushOwnerDeps, + intent: &Arc, + scope: &PushScope, +) -> Result, PushError> { + if !session_is_current(config, config_generation, deps, intent, scope) { + return Ok(None); + } + let account = scope.account_key(); + let fingerprint = config.registration_fingerprint(scope); + let refresh = intent.take_registration_refresh(); + if !refresh + && let Some(registration) = deps.store.load(&account)? + && registration_is_usable(®istration, &fingerprint, config) + { + return Ok(Some(registration)); + } + let Some(token) = (deps.token_source)(scope) else { + return Err(PushError::new( + "push_token_unavailable", + "The account token is not available for the push registration", + )); + }; + let existing = deps.store.load(&account)?; + let client = RegistrationClient { + http: deps.http.as_ref(), + endpoints: &config.endpoints, + timeout: config.registration_timeout, + }; + let mut guard = || session_is_current(config, config_generation, deps, intent, scope); + let mut registration = checkin_and_register( + &client, + &config.identity, + existing.as_ref(), + &token, + &config.device_id, + &mut guard, + ) + .map_err(|error| { + if error.code == DEVICE_IDENTITY_REJECTED { + retire_registration(deps, scope, error.message.clone()) + } else { + error + } + })?; + registration.fingerprint = fingerprint; + if !session_is_current(config, config_generation, deps, intent, scope) { + return Ok(None); + } + deps.store.save(&account, ®istration)?; + Ok(Some(registration)) +} + +fn registration_is_usable( + registration: &Registration, + fingerprint: &str, + config: &PushOwnerConfig, +) -> bool { + if registration.fcm_token.is_empty() + || registration.public_key.is_empty() + || registration.private_key.is_empty() + || registration.auth_secret.is_empty() + { + return false; + } + if EceKeyPair::from_parts( + ®istration.private_key, + ®istration.public_key, + ®istration.auth_secret, + ) + .is_err() + { + return false; + } + if registration.fingerprint != fingerprint { + return false; + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default(); + now.saturating_sub(registration.created_at_seconds) <= config.registration_maximum_age.as_secs() +} + +fn run_session( + config: &PushOwnerConfig, + config_generation: u64, + deps: &PushOwnerDeps, + intent: &Arc, + scope: &PushScope, + registration: &Registration, +) -> Result { + if !session_is_current(config, config_generation, deps, intent, scope) { + return Ok(true); + } + let mut transport = deps.transport.connect( + &config.endpoints.mcs_host, + config.endpoints.mcs_port, + config.connect_timeout, + )?; + let login = encode_login_request( + registration.android_id, + registration.security_token, + ®istration.received_persistent_ids, + ); + transport.send(&login, config.connect_timeout)?; + intent.set_active(Some(scope.clone())); + + let mut reader = FrameReader::new(); + let mut pending: BTreeMap = BTreeMap::new(); + let mut persistent_ids = registration.received_persistent_ids.clone(); + let mut stream_id_in: u64 = 0; + let mut advertised_stream_id: u64 = 0; + let mut unacknowledged_persistent: usize = 0; + let mut heartbeat_interval = clamped_heartbeat_interval(config, config.heartbeat_interval); + let mut next_heartbeat = Instant::now() + heartbeat_interval; + let mut heartbeat_ack_deadline: Option = None; + let mut login_deadline = Some(Instant::now() + config.login_timeout); + let keys = match EceKeyPair::from_parts( + ®istration.private_key, + ®istration.public_key, + ®istration.auth_secret, + ) { + Ok(keys) => keys, + Err(_) => { + return Err(retire_registration( + deps, + scope, + "push_key_invalid".to_owned(), + )); + } + }; + + let result = (|| -> Result { + loop { + if !session_is_current(config, config_generation, deps, intent, scope) { + return Ok(true); + } + flush_due_events( + config, + config_generation, + deps, + intent, + scope, + &mut pending, + Instant::now(), + ); + let now = Instant::now(); + if login_deadline.is_some_and(|deadline| now >= deadline) { + return Err(PushError::new( + "push_login_timeout", + "The push service did not answer the login request", + )); + } + if heartbeat_ack_deadline.is_some_and(|deadline| now >= deadline) { + return Err(PushError::new( + "push_heartbeat_timeout", + "The push service stopped acknowledging heartbeats", + )); + } + let until_heartbeat = if heartbeat_ack_deadline.is_some() { + config.read_timeout + } else { + next_heartbeat.saturating_duration_since(now) + }; + let until_flush = pending + .values() + .map(|event| event.deadline.saturating_duration_since(now)) + .min() + .unwrap_or(config.read_timeout); + let until_login = login_deadline + .map(|deadline| deadline.saturating_duration_since(now)) + .unwrap_or(config.read_timeout); + let slice = until_heartbeat + .min(until_flush) + .min(until_login) + .min(config.read_timeout) + .min(SESSION_SLICE) + .max(Duration::from_millis(20)); + let bytes = transport.recv(slice)?; + if !bytes.is_empty() { + reader.extend(&bytes); + while let Some((tag, payload)) = reader.next_frame(config.maximum_frame_bytes)? { + stream_id_in += 1; + heartbeat_ack_deadline = None; + next_heartbeat = Instant::now() + heartbeat_interval; + match decode_frame(tag, &payload)? { + Frame::HeartbeatPing => { + transport.send( + &encode_heartbeat_ack(Some(stream_id_in)), + config.connect_timeout, + )?; + advance_advertised_stream_id( + stream_id_in, + &mut advertised_stream_id, + &mut unacknowledged_persistent, + ); + } + Frame::DataMessage(message) => { + if message + .persistent_id + .as_deref() + .is_some_and(|id| !id.is_empty()) + { + unacknowledged_persistent += 1; + } + let immediate_ack = message.immediate_ack; + handle_data_message( + &message, + &keys, + &mut pending, + &mut persistent_ids, + config, + ); + if immediate_ack + || (unacknowledged_persistent > 0 + && unacknowledged_persistent + % UNACKNOWLEDGED_PERSISTENT_STREAM_ACK + == 0) + { + transport.send( + &encode_stream_ack(Some(stream_id_in)), + config.connect_timeout, + )?; + advance_advertised_stream_id( + stream_id_in, + &mut advertised_stream_id, + &mut unacknowledged_persistent, + ); + } + } + Frame::LoginResponse(response) => { + if let Some(code) = response.error_code.filter(|code| *code != 0) { + intent.request_registration_refresh(); + return Err(PushError::new( + "push_login_refused", + format!("push_login_refused ({code})"), + )); + } + login_deadline = None; + if let Some(millis) = + response.heartbeat_interval_ms.filter(|millis| *millis > 0) + { + heartbeat_interval = clamped_heartbeat_interval( + config, + Duration::from_millis(millis as u64), + ); + next_heartbeat = Instant::now() + heartbeat_interval; + } + } + Frame::Close => return Ok(false), + _ => {} + } + } + } + if heartbeat_ack_deadline.is_none() && Instant::now() >= next_heartbeat { + transport.send( + &encode_heartbeat_ping(Some(stream_id_in)), + config.connect_timeout, + )?; + heartbeat_ack_deadline = Some(Instant::now() + config.heartbeat_ack_timeout); + advance_advertised_stream_id( + stream_id_in, + &mut advertised_stream_id, + &mut unacknowledged_persistent, + ); + } + } + })(); + + if result.is_ok() { + let _ = transport.send(&encode_close(), config.connect_timeout); + } + transport.close(); + intent.set_active(None); + if session_is_current(config, config_generation, deps, intent, scope) { + let deadline = Instant::now() + .checked_sub(config.coalescing_window) + .unwrap_or_else(Instant::now); + flush_due_events( + config, + config_generation, + deps, + intent, + scope, + &mut pending, + deadline, + ); + } + result +} + +struct PendingEvent { + deadline: Instant, + changed_ids: Vec, + account_type: Option, + linked: Option, + platform_code: Option, + sync_state: Option, + sync_date: Option, + sync_game_count: Option, +} + +fn handle_data_message( + message: &crate::push::protocol::DataMessage, + keys: &EceKeyPair, + pending: &mut BTreeMap, + persistent_ids: &mut Vec, + config: &PushOwnerConfig, +) { + if let Some(id) = &message.persistent_id { + if persistent_ids.len() >= config.maximum_persistent_ids { + persistent_ids.remove(0); + } + persistent_ids.push(id.clone()); + } + let Some(parsed) = parse_message(message, keys) else { + return; + }; + let Some(message_type) = parsed.message_type else { + return; + }; + if !matches!( + message_type, + MessageType::LibraryChange + | MessageType::FavoritesChange + | MessageType::SubscriptionChange + | MessageType::LinkedAccountChange + | MessageType::PlatformSyncChange + ) { + return; + } + let deadline = Instant::now() + config.coalescing_window; + let entry = pending.entry(message_type).or_insert_with(|| PendingEvent { + deadline, + changed_ids: Vec::new(), + account_type: None, + linked: None, + platform_code: None, + sync_state: None, + sync_date: None, + sync_game_count: None, + }); + for id in parsed.changed_ids { + if entry.changed_ids.len() >= config.maximum_changed_ids { + break; + } + if !entry.changed_ids.contains(&id) { + entry.changed_ids.push(id); + } + } + if parsed.account_type.is_some() { + entry.account_type = parsed.account_type; + } + if parsed.account_linked.is_some() { + entry.linked = parsed.account_linked; + } + if parsed.platform_code.is_some() { + entry.platform_code = parsed.platform_code; + } + if parsed.sync_state.is_some() { + entry.sync_state = parsed.sync_state; + } + if parsed.sync_date.is_some() { + entry.sync_date = parsed.sync_date; + } + if parsed.sync_game_count.is_some() { + entry.sync_game_count = parsed.sync_game_count; + } +} + +fn parse_message( + message: &crate::push::protocol::DataMessage, + keys: &EceKeyPair, +) -> Option { + let raw_data = message.raw_data.as_ref()?; + let property = |name: &str| { + message + .app_data + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + let body = decrypt_web_push( + keys, + property("content-encoding"), + property("crypto-key"), + property("encryption"), + raw_data, + ) + .ok()?; + let value: serde_json::Value = serde_json::from_slice(&body).ok()?; + let data = value.get("data")?.as_object()?; + crate::push::PushMessage::parse(data) +} + +fn retire_registration(deps: &PushOwnerDeps, scope: &PushScope, detail: String) -> PushError { + let _ = deps.store.clear(&scope.account_key()); + PushError::new( + STORED_REGISTRATION_RETIRED, + format!("The stored push registration was retired ({detail})"), + ) +} + +fn flush_due_events( + config: &PushOwnerConfig, + config_generation: u64, + deps: &PushOwnerDeps, + intent: &Arc, + scope: &PushScope, + pending: &mut BTreeMap, + now: Instant, +) { + let mut ready: Vec = pending + .iter() + .filter(|(_, event)| event.deadline <= now) + .map(|(message_type, _)| *message_type) + .collect(); + ready.sort(); + for message_type in ready { + if !session_is_current(config, config_generation, deps, intent, scope) { + pending.clear(); + return; + } + let Some(event) = pending.remove(&message_type) else { + continue; + }; + let event = match message_type { + MessageType::LibraryChange => Some(PushEvent::LibraryChanged { + changed_ids: event.changed_ids, + }), + MessageType::FavoritesChange => Some(PushEvent::FavoritesChanged { + changed_ids: event.changed_ids, + }), + MessageType::SubscriptionChange => Some(PushEvent::SubscriptionChanged { + changed_ids: event.changed_ids, + }), + MessageType::LinkedAccountChange => Some(PushEvent::LinkedAccountChanged { + account_type: event.account_type, + linked: event.linked, + }), + MessageType::PlatformSyncChange => Some(PushEvent::PlatformSyncChanged { + platform_code: event.platform_code, + sync_state: event.sync_state, + sync_date: event.sync_date, + sync_game_count: event.sync_game_count, + }), + _ => None, + }; + if let Some(event) = event { + let generation = intent.generation.load(Ordering::SeqCst); + (deps.sink)(event, generation); + } + } +} + +#[cfg(test)] +mod guard_tests { + use super::*; + use crate::push::registration::{ + HttpRequest, HttpResponse, PushEndpoints, PushHttp, PushIdentity, + }; + use crate::push::store::PushStateStore; + use crate::push::transport::{PushTransport, PushTransportFactory}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CountingHttp(Arc); + + impl PushHttp for CountingHttp { + fn execute(&self, _request: &HttpRequest) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(PushError::new("push_http_failed", "unexpected request")) + } + } + + struct EmptyStore; + + impl PushStateStore for EmptyStore { + fn load(&self, _account: &str) -> Result, PushError> { + Ok(None) + } + + fn save(&self, _account: &str, _registration: &Registration) -> Result<(), PushError> { + Ok(()) + } + + fn clear(&self, _account: &str) -> Result<(), PushError> { + Ok(()) + } + } + + struct NoTransport; + + impl PushTransport for NoTransport { + fn send(&mut self, _frame: &[u8], _timeout: Duration) -> Result<(), PushError> { + Ok(()) + } + + fn recv(&mut self, _timeout: Duration) -> Result, PushError> { + Ok(Vec::new()) + } + + fn close(&mut self) {} + } + + struct NoFactory; + + impl PushTransportFactory for NoFactory { + fn connect( + &self, + _host: &str, + _port: u16, + _timeout: Duration, + ) -> Result, PushError> { + Ok(Box::new(NoTransport)) + } + } + + fn config(provider_id: &str) -> PushOwnerConfig { + PushOwnerConfig::bounded( + PushEndpoints::default(), + PushIdentity { + project_id: "project".into(), + api_key: "api-key".into(), + sender_id: "123456789012".into(), + app_id: "app".into(), + firebase_app_id: "1:1:web:abc".into(), + vapid_key: None, + }, + provider_id.into(), + "device".into(), + ) + } + + fn scope(provider_id: &str) -> PushScope { + PushScope { + user_id: "user-1".into(), + provider_id: provider_id.into(), + generation: 4, + } + } + + #[test] + fn a_stale_config_pair_never_reaches_registration_http() { + let stale_config = config("nvidia"); + let intent = Arc::new(Intent::new(stale_config.clone())); + let desired = Arc::new(Mutex::new(Some(scope("alliance")))); + let token_calls = Arc::new(AtomicUsize::new(0)); + let http_calls = Arc::new(AtomicUsize::new(0)); + let deps = PushOwnerDeps { + http: Arc::new(CountingHttp(Arc::clone(&http_calls))), + transport: Arc::new(NoFactory), + store: Arc::new(EmptyStore), + sink: Arc::new(|_: PushEvent, _: u64| {}), + scope_source: { + let desired = Arc::clone(&desired); + Arc::new(move || desired.lock().unwrap().clone()) + }, + token_source: { + let token_calls = Arc::clone(&token_calls); + Arc::new(move |_: &PushScope| { + token_calls.fetch_add(1, Ordering::SeqCst); + Some("token".to_owned()) + }) + }, + }; + let (before, before_generation) = intent.config(); + assert_eq!(before.provider_id, "nvidia"); + intent.replace_config(config("alliance")); + let (current, current_generation) = intent.config(); + assert_eq!(current.provider_id, "alliance"); + assert!(current_generation > before_generation); + let outcome = run_scoped_session( + &stale_config, + before_generation, + &deps, + &intent, + &scope("alliance"), + ); + assert!(matches!(outcome, SessionOutcome::ScopeChanged)); + assert_eq!( + token_calls.load(Ordering::SeqCst), + 0, + "no token may be requested" + ); + assert_eq!( + http_calls.load(Ordering::SeqCst), + 0, + "no registration HTTP may run" + ); + assert!(!session_is_current( + &stale_config, + before_generation, + &deps, + &intent, + &scope("alliance") + )); + assert!( + !session_is_current( + &stale_config, + current_generation, + &deps, + &intent, + &scope("alliance") + ), + "a stale provider must not be accepted" + ); + assert!( + session_is_current( + ¤t, + current_generation, + &deps, + &intent, + &scope("alliance") + ), + "the current pair must be accepted" + ); + } +} diff --git a/external/opennow/native/opennow-core/src/push/protocol.rs b/external/opennow/native/opennow-core/src/push/protocol.rs new file mode 100644 index 000000000..657f00fdf --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/protocol.rs @@ -0,0 +1,480 @@ +use crate::push::PushError; + +pub const MCS_VERSION: u8 = 41; + +pub const TAG_HEARTBEAT_PING: u8 = 0; +pub const TAG_HEARTBEAT_ACK: u8 = 1; +pub const TAG_LOGIN_REQUEST: u8 = 2; +pub const TAG_LOGIN_RESPONSE: u8 = 3; +pub const TAG_CLOSE: u8 = 4; +pub const TAG_IQ_STANZA: u8 = 7; +pub const TAG_DATA_MESSAGE_STANZA: u8 = 8; + +pub const STREAM_ACK_EXTENSION_ID: u64 = 13; +pub const IQ_TYPE_SET: u64 = 1; + +pub const AUTH_SERVICE_ANDROID_ID: u64 = 2; + +pub struct CheckinRequest { + pub android_id: Option, + pub security_token: Option, +} + +pub struct CheckinResponse { + pub android_id: Option, + pub security_token: Option, +} + +pub struct LoginResponse { + pub error_code: Option, + pub heartbeat_interval_ms: Option, +} + +pub struct DataMessage { + pub from: String, + pub category: String, + pub app_data: Vec<(String, String)>, + pub persistent_id: Option, + pub raw_data: Option>, + pub immediate_ack: bool, +} + +pub enum Frame { + HeartbeatPing, + HeartbeatAck, + LoginResponse(LoginResponse), + DataMessage(Box), + Close, + Other, +} + +fn write_frame(versioned: bool, tag: u8, payload: &[u8]) -> Vec { + let mut frame = Vec::with_capacity(payload.len() + 8); + if versioned { + frame.push(MCS_VERSION); + } + frame.push(tag); + write_varint(payload.len() as u64, &mut frame); + frame.extend_from_slice(payload); + frame +} + +fn heartbeat(last_stream_id_received: Option) -> Vec { + let mut payload = Vec::with_capacity(12); + if let Some(stream_id) = last_stream_id_received { + write_varint_field(2, stream_id, &mut payload); + } + payload +} + +pub fn encode_heartbeat_ack(last_stream_id_received: Option) -> Vec { + write_frame( + false, + TAG_HEARTBEAT_ACK, + &heartbeat(last_stream_id_received), + ) +} + +pub fn encode_heartbeat_ping(last_stream_id_received: Option) -> Vec { + write_frame( + false, + TAG_HEARTBEAT_PING, + &heartbeat(last_stream_id_received), + ) +} + +pub fn encode_close() -> Vec { + write_frame(false, TAG_CLOSE, &[]) +} + +pub fn encode_stream_ack(last_stream_id_received: Option) -> Vec { + let mut payload = Vec::with_capacity(24); + write_varint_field(2, IQ_TYPE_SET, &mut payload); + write_string(3, "", &mut payload); + let mut extension = Vec::with_capacity(8); + write_varint_field(1, STREAM_ACK_EXTENSION_ID, &mut extension); + write_bytes(2, &[], &mut extension); + write_bytes(7, &extension, &mut payload); + if let Some(stream_id) = last_stream_id_received { + write_varint_field(10, stream_id, &mut payload); + } + write_frame(false, TAG_IQ_STANZA, &payload) +} + +pub fn encode_login_request( + android_id: i64, + security_token: u64, + received_persistent_ids: &[String], +) -> Vec { + let mut payload = Vec::with_capacity(256 + received_persistent_ids.len() * 32); + write_string( + 1, + &format!("chrome-{}", env!("CARGO_PKG_VERSION")), + &mut payload, + ); + write_string(2, "mcs.android.com", &mut payload); + write_string(3, &android_id.to_string(), &mut payload); + write_string(4, &android_id.to_string(), &mut payload); + write_string(5, &security_token.to_string(), &mut payload); + write_string(6, &format!("android-{android_id:x}"), &mut payload); + write_setting("new_vc", "1", &mut payload); + for id in received_persistent_ids.iter().take(64) { + write_string(10, id, &mut payload); + } + write_bool(12, false, &mut payload); + write_bool(14, true, &mut payload); + write_varint_field(16, AUTH_SERVICE_ANDROID_ID, &mut payload); + write_varint_field(17, 1, &mut payload); + write_frame(true, TAG_LOGIN_REQUEST, &payload) +} + +pub fn decode_frame(tag: u8, payload: &[u8]) -> Result { + Ok(match tag { + TAG_HEARTBEAT_PING => Frame::HeartbeatPing, + TAG_HEARTBEAT_ACK => Frame::HeartbeatAck, + TAG_CLOSE => Frame::Close, + TAG_LOGIN_RESPONSE => Frame::LoginResponse(decode_login_response(payload)?), + TAG_DATA_MESSAGE_STANZA => Frame::DataMessage(Box::new(decode_data_message(payload)?)), + _ => Frame::Other, + }) +} + +fn decode_login_response(payload: &[u8]) -> Result { + let mut reader = Reader::new(payload); + let mut response = LoginResponse { + error_code: None, + heartbeat_interval_ms: None, + }; + while let Some((field, wire)) = reader.next_tag()? { + match (field, wire) { + (3, WIRE_LENGTH) => { + let nested = reader.take_length()?; + let mut error = Reader::new(nested); + while let Some((inner, inner_wire)) = error.next_tag()? { + if inner == 1 && inner_wire == WIRE_VARINT { + response.error_code = Some(error.read_varint()? as i32); + } else { + error.skip(inner_wire)?; + } + } + } + (7, WIRE_LENGTH) => { + let nested = reader.take_length()?; + let mut config = Reader::new(nested); + while let Some((inner, inner_wire)) = config.next_tag()? { + if inner == 3 && inner_wire == WIRE_VARINT { + response.heartbeat_interval_ms = Some(config.read_varint()? as i32); + } else { + config.skip(inner_wire)?; + } + } + } + _ => reader.skip(wire)?, + } + } + Ok(response) +} + +fn decode_data_message(payload: &[u8]) -> Result { + let mut reader = Reader::new(payload); + let mut message = DataMessage { + from: String::new(), + category: String::new(), + app_data: Vec::new(), + persistent_id: None, + raw_data: None, + immediate_ack: false, + }; + while let Some((field, wire)) = reader.next_tag()? { + match (field, wire) { + (3, WIRE_LENGTH) => message.from = reader.read_string()?, + (5, WIRE_LENGTH) => message.category = reader.read_string()?, + (7, WIRE_LENGTH) => { + let nested = reader.take_length()?; + let mut app = Reader::new(nested); + let mut key = String::new(); + let mut value = String::new(); + while let Some((inner, inner_wire)) = app.next_tag()? { + match (inner, inner_wire) { + (1, WIRE_LENGTH) => key = app.read_string()?, + (2, WIRE_LENGTH) => value = app.read_string()?, + _ => app.skip(inner_wire)?, + } + } + if !key.is_empty() && message.app_data.len() < 128 { + message.app_data.push((key, value)); + } + } + (9, WIRE_LENGTH) => message.persistent_id = Some(reader.read_string()?), + (21, WIRE_LENGTH) => message.raw_data = Some(reader.take_length()?.to_vec()), + (24, WIRE_VARINT) => message.immediate_ack = reader.read_varint()? != 0, + _ => reader.skip(wire)?, + } + } + Ok(message) +} + +pub fn encode_checkin_request(request: &CheckinRequest) -> Vec { + let mut payload = Vec::with_capacity(128); + if let Some(id) = request.android_id { + write_varint_field(2, id as u64, &mut payload); + } + let mut checkin = Vec::with_capacity(48); + write_varint_field(12, 3, &mut checkin); + let mut chrome_build = Vec::with_capacity(32); + write_varint_field(1, 3, &mut chrome_build); + write_string(2, "63.0.3234.0", &mut chrome_build); + write_varint_field(3, 1, &mut chrome_build); + write_bytes(13, &chrome_build, &mut checkin); + write_bytes(4, &checkin, &mut payload); + write_varint_field(14, 3, &mut payload); + write_varint_field(22, 0, &mut payload); + if let Some(token) = request.security_token { + write_fixed64(13, token, &mut payload); + } + payload +} + +pub fn decode_checkin_response(payload: &[u8]) -> Result { + let mut reader = Reader::new(payload); + let mut response = CheckinResponse { + android_id: None, + security_token: None, + }; + while let Some((field, wire)) = reader.next_tag()? { + match (field, wire) { + (7, WIRE_FIXED64) => response.android_id = Some(reader.read_fixed64()?), + (8, WIRE_FIXED64) => response.security_token = Some(reader.read_fixed64()?), + _ => reader.skip(wire)?, + } + } + Ok(response) +} + +pub struct FrameReader { + buffer: Vec, + expects_version: bool, +} + +impl Default for FrameReader { + fn default() -> Self { + Self::new() + } +} + +impl FrameReader { + pub fn new() -> Self { + Self { + buffer: Vec::with_capacity(4096), + expects_version: true, + } + } + + pub fn extend(&mut self, bytes: &[u8]) { + self.buffer.extend_from_slice(bytes); + } + + pub fn next_frame(&mut self, maximum: usize) -> Result)>, PushError> { + let mut position = 0; + if self.expects_version { + if self.buffer.len() < 2 { + return Ok(None); + } + let version = self.buffer[0]; + if version < MCS_VERSION && version != 38 { + return Err(PushError::new( + "push_protocol_version", + "The push service reported an unsupported protocol version", + )); + } + position = 1; + } + if self.buffer.len() <= position { + return Ok(None); + } + let tag = self.buffer[position]; + position += 1; + let mut size = 0_u64; + let mut shift = 0; + let mut header = 0; + loop { + let Some(byte) = self.buffer.get(position + header) else { + return Ok(None); + }; + if header >= 5 { + return Err(PushError::new( + "push_protocol_frame", + "The push service sent an invalid frame size", + )); + } + size |= u64::from(byte & 0x7f) << shift; + header += 1; + if byte & 0x80 == 0 { + break; + } + shift += 7; + } + if size as usize > maximum { + return Err(PushError::new( + "push_protocol_frame", + "The push service sent an oversized frame", + )); + } + let start = position + header; + let end = start + size as usize; + if self.buffer.len() < end { + return Ok(None); + } + let payload = self.buffer[start..end].to_vec(); + self.buffer.drain(..end); + self.expects_version = false; + Ok(Some((tag, payload))) + } +} + +const WIRE_VARINT: u64 = 0; +const WIRE_FIXED64: u64 = 1; +const WIRE_LENGTH: u64 = 2; + +struct Reader<'a> { + data: &'a [u8], + position: usize, +} + +impl<'a> Reader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, position: 0 } + } + + fn next_tag(&mut self) -> Result, PushError> { + if self.position >= self.data.len() { + return Ok(None); + } + let tag = self.read_varint()?; + Ok(Some((tag >> 3, tag & 0x7))) + } + + fn read_varint(&mut self) -> Result { + let mut value = 0_u64; + let mut shift = 0; + loop { + let byte = *self.data.get(self.position).ok_or_else(|| { + PushError::new("push_protocol_frame", "The frame ended unexpectedly") + })?; + self.position += 1; + if shift >= 64 { + return Err(PushError::new( + "push_protocol_frame", + "The frame carried an oversized integer", + )); + } + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + } + } + + fn read_fixed64(&mut self) -> Result { + let end = self.position + 8; + let bytes = self + .data + .get(self.position..end) + .ok_or_else(|| PushError::new("push_protocol_frame", "The frame ended unexpectedly"))?; + self.position = end; + Ok(u64::from_le_bytes(bytes.try_into().map_err(|_| { + PushError::new( + "push_protocol_frame", + "The frame carried an invalid integer", + ) + })?)) + } + + fn take_length(&mut self) -> Result<&'a [u8], PushError> { + let length = self.read_varint()? as usize; + if length > self.data.len().saturating_sub(self.position) { + return Err(PushError::new( + "push_protocol_frame", + "The frame carried an oversized field", + )); + } + let end = self.position + length; + let slice = &self.data[self.position..end]; + self.position = end; + Ok(slice) + } + + fn read_string(&mut self) -> Result { + String::from_utf8(self.take_length()?.to_vec()) + .map_err(|_| PushError::new("push_protocol_frame", "The frame carried invalid text")) + } + + fn skip(&mut self, wire: u64) -> Result<(), PushError> { + match wire { + WIRE_VARINT => { + self.read_varint()?; + } + WIRE_FIXED64 => { + self.read_fixed64()?; + } + WIRE_LENGTH => { + self.take_length()?; + } + _ => { + return Err(PushError::new( + "push_protocol_frame", + "The frame used an unsupported field type", + )); + } + } + Ok(()) + } +} + +fn write_varint(mut value: u64, out: &mut Vec) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn write_tag(field: u64, wire: u64, out: &mut Vec) { + write_varint((field << 3) | wire, out); +} + +fn write_varint_field(field: u64, value: u64, out: &mut Vec) { + write_tag(field, WIRE_VARINT, out); + write_varint(value, out); +} + +fn write_bool(field: u64, value: bool, out: &mut Vec) { + write_varint_field(field, u64::from(value), out); +} + +fn write_bytes(field: u64, value: &[u8], out: &mut Vec) { + write_tag(field, WIRE_LENGTH, out); + write_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +fn write_string(field: u64, value: &str, out: &mut Vec) { + write_bytes(field, value.as_bytes(), out); +} + +fn write_fixed64(field: u64, value: u64, out: &mut Vec) { + write_tag(field, WIRE_FIXED64, out); + out.extend_from_slice(&value.to_le_bytes()); +} + +fn write_setting(name: &str, value: &str, out: &mut Vec) { + let mut setting = Vec::with_capacity(name.len() + value.len() + 8); + write_string(1, name, &mut setting); + write_string(2, value, &mut setting); + write_bytes(8, &setting, out); +} diff --git a/external/opennow/native/opennow-core/src/push/registration.rs b/external/opennow/native/opennow-core/src/push/registration.rs new file mode 100644 index 000000000..04649d5d0 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/registration.rs @@ -0,0 +1,595 @@ +use crate::push::PushError; +use crate::push::decrypt::EceKeyPair; +use crate::push::protocol::{CheckinRequest, decode_checkin_response, encode_checkin_request}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde::{Deserialize, Serialize}; +use std::io::Read as _; +use std::time::Duration; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushIdentity { + pub project_id: String, + pub api_key: String, + pub sender_id: String, + pub app_id: String, + pub firebase_app_id: String, + pub vapid_key: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushEndpoints { + pub checkin: String, + pub c2dm: String, + pub fis: String, + pub fcm_registration: String, + pub fcm_send: String, + pub mcs_host: String, + pub mcs_port: u16, + pub pns: String, + pub pns_client_id: String, +} + +impl Default for PushEndpoints { + fn default() -> Self { + Self { + checkin: "https://android.clients.google.com/checkin".into(), + c2dm: "https://android.clients.google.com/c2dm/register3".into(), + fis: "https://firebaseinstallations.googleapis.com/v1".into(), + fcm_registration: "https://fcmregistrations.googleapis.com/v1".into(), + fcm_send: "https://fcm.googleapis.com/fcm/send".into(), + mcs_host: "mtalk.google.com".into(), + mcs_port: 5228, + pns: String::new(), + pns_client_id: String::new(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpMethod { + Get, + Post, +} + +pub struct HttpRequest { + pub url: String, + pub method: HttpMethod, + pub headers: Vec<(String, String)>, + pub body: Vec, + pub timeout: Duration, +} + +pub struct HttpResponse { + pub status: u16, + pub body: Vec, +} + +pub trait PushHttp: Send + Sync { + fn execute(&self, request: &HttpRequest) -> Result; +} + +pub struct RegistrationClient<'a> { + pub http: &'a dyn PushHttp, + pub endpoints: &'a PushEndpoints, + pub timeout: Duration, +} + +pub struct ReqwestPushHttp { + client: reqwest::blocking::Client, +} + +impl ReqwestPushHttp { + pub fn new() -> Result { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .map_err(|_| PushError::new("push_http_failed", "Could not start the push requests"))?; + Ok(Self { client }) + } +} + +pub const MAXIMUM_RESPONSE_BYTES: u64 = 64 * 1024; +pub const DEVICE_IDENTITY_REJECTED: &str = "push_device_identity_rejected"; + +pub(crate) fn read_bounded(reader: impl std::io::Read, limit: u64) -> Result, PushError> { + let mut bytes = Vec::new(); + reader + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| PushError::new("push_http_failed", "The push response could not be read"))?; + if bytes.len() as u64 > limit { + return Err(PushError::new( + "push_http_failed", + "The push response exceeded the accepted size", + )); + } + Ok(bytes) +} + +impl PushHttp for ReqwestPushHttp { + fn execute(&self, request: &HttpRequest) -> Result { + let mut builder = match request.method { + HttpMethod::Get => self.client.get(&request.url), + HttpMethod::Post => self.client.post(&request.url), + } + .timeout(request.timeout); + for (name, value) in &request.headers { + builder = builder.header(name, value); + } + let response = builder + .body(request.body.clone()) + .send() + .map_err(|_| PushError::new("push_http_failed", "The push request failed"))?; + let status = response.status().as_u16(); + let body = read_bounded(response, MAXIMUM_RESPONSE_BYTES)?; + Ok(HttpResponse { status, body }) + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct Registration { + pub android_id: i64, + pub security_token: u64, + pub gcm_token: String, + pub fcm_token: String, + pub endpoint: String, + pub private_key: Vec, + pub public_key: Vec, + pub auth_secret: Vec, + #[serde(default)] + pub received_persistent_ids: Vec, + #[serde(default)] + pub created_at_seconds: u64, + #[serde(default)] + pub fingerprint: String, +} + +impl Registration { + pub fn public_key(&self) -> &[u8] { + &self.public_key + } + + pub fn auth_secret(&self) -> &[u8] { + &self.auth_secret + } +} + +pub struct CheckinResult { + pub android_id: i64, + pub security_token: u64, +} + +pub fn checkin( + client: &RegistrationClient<'_>, + android_id: Option, + security_token: Option, +) -> Result { + let body = encode_checkin_request(&CheckinRequest { + android_id, + security_token, + }); + let response = client.http.execute(&HttpRequest { + url: client.endpoints.checkin.clone(), + method: HttpMethod::Post, + headers: vec![("Content-Type".into(), "application/x-protobuf".into())], + body, + timeout: client.timeout, + })?; + if response.status == 401 || response.status == 400 { + return Err(PushError::new( + DEVICE_IDENTITY_REJECTED, + "The device registration service rejected the stored device identity", + )); + } + if response.status != 200 { + return Err(PushError::new( + "push_checkin_refused", + "The device registration service refused the check-in", + )); + } + let decoded = decode_checkin_response(&response.body)?; + let android_id = decoded.android_id.ok_or_else(|| { + PushError::new( + "push_checkin_incomplete", + "The check-in response did not carry a device identity", + ) + })?; + let security_token = decoded.security_token.ok_or_else(|| { + PushError::new( + "push_checkin_incomplete", + "The check-in response did not carry a device token", + ) + })?; + if android_id == 0 || security_token == 0 { + return Err(PushError::new( + "push_checkin_incomplete", + "The check-in response carried an empty device identity", + )); + } + Ok(CheckinResult { + android_id: i64::try_from(android_id).map_err(|_| { + PushError::new( + "push_checkin_incomplete", + "The device identity is outside the accepted range", + ) + })?, + security_token, + }) +} + +pub fn gcm_register( + client: &RegistrationClient<'_>, + android_id: i64, + security_token: u64, + identity: &PushIdentity, +) -> Result { + let mut form = format!( + "app=org.chromium.linux&X-subtype={}&device={android_id}", + urlencode(&identity.app_id) + ); + if !identity.sender_id.is_empty() { + form.push_str(&format!("&sender={}", urlencode(&identity.sender_id))); + } + let response = client.http.execute(&HttpRequest { + url: client.endpoints.c2dm.clone(), + method: HttpMethod::Post, + headers: vec![ + ( + "Content-Type".into(), + "application/x-www-form-urlencoded".into(), + ), + ( + "Authorization".into(), + format!("AidLogin {android_id}:{security_token}"), + ), + ], + body: form.into_bytes(), + timeout: client.timeout, + })?; + if response.status != 200 { + return Err(PushError::new( + "push_gcm_refused", + "The messaging service refused the registration", + )); + } + let text = String::from_utf8_lossy(&response.body); + let mut parts = text.split('='); + let name = parts.next().unwrap_or_default(); + let value = parts.next().unwrap_or_default(); + if name != "token" || value.is_empty() { + return Err(PushError::new( + "push_gcm_refused", + "The messaging service did not return a token", + )); + } + Ok(value.to_owned()) +} + +pub fn firebase_installation_token( + client: &RegistrationClient<'_>, + identity: &PushIdentity, + fid: &str, +) -> Result { + if identity.project_id.is_empty() || identity.api_key.is_empty() { + return Err(PushError::new( + "push_config_incomplete", + "The push identity is incomplete", + )); + } + let body = serde_json::json!({ + "fid": fid, + "authVersion": "FIS_v2", + "appId": identity.firebase_app_id, + "sdkVersion": "w:0.1.0", + }) + .to_string(); + let response = client.http.execute(&HttpRequest { + url: format!( + "{}/projects/{}/installations", + client.endpoints.fis.trim_end_matches('/'), + identity.project_id + ), + method: HttpMethod::Post, + headers: vec![ + ("Content-Type".into(), "application/json".into()), + ("Accept".into(), "application/json".into()), + ("x-goog-api-key".into(), identity.api_key.clone()), + ( + "x-firebase-client".into(), + URL_SAFE_NO_PAD.encode(b"{\"heartbeats\": [], \"version\": 2}"), + ), + ], + body: body.into_bytes(), + timeout: client.timeout, + })?; + if response.status != 200 { + return Err(PushError::new( + "push_fis_refused", + "The installation service refused the request", + )); + } + let value: serde_json::Value = serde_json::from_slice(&response.body).map_err(|_| { + PushError::new( + "push_fis_incomplete", + "The installation response could not be read", + ) + })?; + value["authToken"]["token"] + .as_str() + .map(str::to_owned) + .ok_or_else(|| { + PushError::new( + "push_fis_incomplete", + "The installation response did not carry a token", + ) + }) +} + +pub fn fcm_register( + client: &RegistrationClient<'_>, + identity: &PushIdentity, + installation_token: &str, + gcm_token: &str, + public_key: &[u8], + auth_secret: &[u8], +) -> Result<(String, String), PushError> { + let endpoint = format!( + "{}/{}", + client.endpoints.fcm_send.trim_end_matches('/'), + gcm_token + ); + let mut web = serde_json::json!({ + "endpoint": endpoint, + "auth": URL_SAFE_NO_PAD.encode(auth_secret), + "p256dh": URL_SAFE_NO_PAD.encode(public_key), + }); + if let Some(vapid) = identity.vapid_key.as_deref() { + if !vapid.is_empty() { + web["application_pub_key"] = serde_json::Value::String(vapid.to_owned()); + } + } + let body = serde_json::json!({ "web": web }).to_string(); + let response = client.http.execute(&HttpRequest { + url: format!( + "{}/projects/{}/registrations", + client.endpoints.fcm_registration.trim_end_matches('/'), + identity.project_id + ), + method: HttpMethod::Post, + headers: vec![ + ("Content-Type".into(), "application/json".into()), + ("Accept".into(), "application/json".into()), + ("x-goog-api-key".into(), identity.api_key.clone()), + ( + "x-goog-firebase-installations-auth".into(), + format!("FIS {installation_token}"), + ), + ], + body: body.into_bytes(), + timeout: client.timeout, + })?; + if response.status != 200 { + return Err(PushError::new( + "push_fcm_refused", + "The messaging service refused the client registration", + )); + } + let value: serde_json::Value = serde_json::from_slice(&response.body).map_err(|_| { + PushError::new( + "push_fcm_incomplete", + "The registration response could not be read", + ) + })?; + let token = value["token"].as_str().ok_or_else(|| { + PushError::new( + "push_fcm_incomplete", + "The registration response did not carry a token", + ) + })?; + if token.is_empty() { + return Err(PushError::new( + "push_fcm_incomplete", + "The registration response carried an empty token", + )); + } + Ok((token.to_owned(), endpoint)) +} + +pub fn pns_register( + client: &RegistrationClient<'_>, + registration_token: &str, + device_id: &str, + previous: &[String], + gfnjwt: &str, +) -> Result<(), PushError> { + if client.endpoints.pns.is_empty() || client.endpoints.pns_client_id.is_empty() { + return Err(PushError::new( + "push_config_incomplete", + "The notification service is not configured", + )); + } + let body = serde_json::json!({ + "registrationToken": registration_token, + "deviceId": device_id, + "previousRegistrationTokens": previous, + }) + .to_string(); + let response = client.http.execute(&HttpRequest { + url: client.endpoints.pns_registrations_url(), + method: HttpMethod::Post, + headers: vec![ + ("Content-Type".into(), "application/json".into()), + ("Accept".into(), "application/json".into()), + ( + "NV-Client-ID".into(), + client.endpoints.pns_client_id.clone(), + ), + ("Authorization".into(), format!("GFNJWT {gfnjwt}")), + ], + body: body.into_bytes(), + timeout: client.timeout, + })?; + match response.status { + 200 | 201 | 202 | 204 => Ok(()), + _ => Err(PushError::new( + "push_pns_refused", + "The notification service refused the registration", + )), + } +} + +pub fn pns_unregister( + client: &RegistrationClient<'_>, + previous_tokens: &[String], + gfnjwt: &str, +) -> Result<(), PushError> { + if client.endpoints.pns.is_empty() || client.endpoints.pns_client_id.is_empty() { + return Err(PushError::new( + "push_config_incomplete", + "The notification service is not configured", + )); + } + let body = serde_json::json!({ "previousRegistrationTokens": previous_tokens }).to_string(); + let response = client.http.execute(&HttpRequest { + url: client.endpoints.pns_unregister_url(), + method: HttpMethod::Post, + headers: vec![ + ("Content-Type".into(), "application/json".into()), + ("Accept".into(), "application/json".into()), + ( + "NV-Client-ID".into(), + client.endpoints.pns_client_id.clone(), + ), + ("Authorization".into(), format!("GFNJWT {gfnjwt}")), + ], + body: body.into_bytes(), + timeout: client.timeout, + })?; + match response.status { + 200 | 202 | 204 => Ok(()), + _ => Err(PushError::new( + "push_pns_refused", + "The notification service refused the unregistration", + )), + } +} + +impl PushEndpoints { + pub fn pns_registrations_url(&self) -> String { + format!("{}/registrations", self.pns.trim_end_matches('/')) + } + + pub fn pns_unregister_url(&self) -> String { + format!("{}/unregister", self.pns.trim_end_matches('/')) + } +} + +pub fn register( + client: &RegistrationClient<'_>, + identity: &PushIdentity, + existing: Option<&Registration>, + gfnjwt: &str, + device_id: &str, + guard: &mut dyn FnMut() -> bool, +) -> Result { + if identity.project_id.is_empty() || identity.api_key.is_empty() || identity.app_id.is_empty() { + return Err(PushError::new( + "push_config_incomplete", + "The push identity is incomplete", + )); + } + let (android_id, security_token, received) = match existing { + Some(registration) => ( + Some(registration.android_id), + Some(registration.security_token), + registration.received_persistent_ids.clone(), + ), + None => (None, None, Vec::new()), + }; + stage(guard)?; + let checked = checkin(client, android_id, security_token)?; + stage(guard)?; + let gcm_token = gcm_register(client, checked.android_id, checked.security_token, identity)?; + let keys = EceKeyPair::generate()?; + let fid = random_fid(); + stage(guard)?; + let installation_token = firebase_installation_token(client, identity, &fid)?; + stage(guard)?; + let (fcm_token, endpoint) = fcm_register( + client, + identity, + &installation_token, + &gcm_token, + keys.public_key(), + keys.auth_secret(), + )?; + let previous: Vec = existing + .map(|registration| registration.fcm_token.as_str()) + .filter(|token| !token.is_empty() && *token != fcm_token) + .map(str::to_owned) + .into_iter() + .collect(); + stage(guard)?; + pns_register(client, &fcm_token, device_id, &previous, gfnjwt)?; + Ok(Registration { + android_id: checked.android_id, + security_token: checked.security_token, + gcm_token, + fcm_token, + endpoint, + private_key: keys.private_key(), + public_key: keys.public_key().to_vec(), + auth_secret: keys.auth_secret().to_vec(), + received_persistent_ids: received, + created_at_seconds: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default(), + fingerprint: String::new(), + }) +} + +pub fn checkin_and_register( + client: &RegistrationClient<'_>, + identity: &PushIdentity, + existing: Option<&Registration>, + gfnjwt: &str, + device_id: &str, + guard: &mut dyn FnMut() -> bool, +) -> Result { + register(client, identity, existing, gfnjwt, device_id, guard) +} + +fn stage(guard: &mut dyn FnMut() -> bool) -> Result<(), PushError> { + if guard() { + Ok(()) + } else { + Err(PushError::new( + "push_scope_stale", + "The account changed while the push registration was running", + )) + } +} + +fn random_fid() -> String { + use rand::RngCore; + let mut bytes = [0_u8; 17]; + rand::rng().fill_bytes(&mut bytes); + bytes[0] = 0b0111_0000 + (bytes[0] % 0b0001_0000); + let encoded = URL_SAFE_NO_PAD.encode(bytes); + encoded[..22].to_owned() +} + +fn urlencode(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + encoded.push(byte as char); + } + other => encoded.push_str(&format!("%{other:02X}")), + } + } + encoded +} diff --git a/external/opennow/native/opennow-core/src/push/store.rs b/external/opennow/native/opennow-core/src/push/store.rs new file mode 100644 index 000000000..feab28f9f --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/store.rs @@ -0,0 +1,78 @@ +use crate::push::PushError; +use crate::push::registration::Registration; + +const PUSH_SERVICE_NAME: &str = "app.opennow.push"; + +pub trait PushStateStore: Send + Sync { + fn load(&self, account: &str) -> Result, PushError>; + fn save(&self, account: &str, registration: &Registration) -> Result<(), PushError>; + fn clear(&self, account: &str) -> Result<(), PushError>; +} + +pub struct RegistrationStore { + service: String, +} + +impl RegistrationStore { + pub fn new(service: &str) -> Self { + Self { + service: service.to_owned(), + } + } + + pub fn default_service() -> Self { + Self::new(PUSH_SERVICE_NAME) + } + + fn entry(&self, account: &str) -> Result { + keyring::Entry::new(&self.service, &format!("registration:{account}")).map_err(|_| { + PushError::new( + "push_store_unavailable", + "The OS credential store is unavailable for the push registration", + ) + }) + } +} + +impl PushStateStore for RegistrationStore { + fn load(&self, account: &str) -> Result, PushError> { + match self.entry(account)?.get_password() { + Ok(encoded) => serde_json::from_str(&encoded).map(Some).map_err(|_| { + PushError::new( + "push_store_corrupt", + "The stored push registration could not be read", + ) + }), + Err(keyring::Error::NoEntry) => Ok(None), + Err(_) => Err(PushError::new( + "push_store_unavailable", + "The OS credential store is unavailable for the push registration", + )), + } + } + + fn save(&self, account: &str, registration: &Registration) -> Result<(), PushError> { + let encoded = serde_json::to_string(registration).map_err(|_| { + PushError::new( + "push_store_failed", + "The push registration could not be encoded", + ) + })?; + self.entry(account)?.set_password(&encoded).map_err(|_| { + PushError::new( + "push_store_failed", + "The push registration could not be stored", + ) + }) + } + + fn clear(&self, account: &str) -> Result<(), PushError> { + match self.entry(account)?.delete_credential() { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(_) => Err(PushError::new( + "push_store_failed", + "The push registration could not be removed", + )), + } + } +} diff --git a/external/opennow/native/opennow-core/src/push/tests.rs b/external/opennow/native/opennow-core/src/push/tests.rs new file mode 100644 index 000000000..146925825 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/tests.rs @@ -0,0 +1,2881 @@ +use crate::push::decrypt::{EceKeyPair, decrypt_web_push, derive_content_keys}; +use crate::push::protocol::{ + CheckinRequest, FrameReader, MCS_VERSION, STREAM_ACK_EXTENSION_ID, TAG_DATA_MESSAGE_STANZA, + TAG_HEARTBEAT_ACK, TAG_HEARTBEAT_PING, TAG_IQ_STANZA, TAG_LOGIN_RESPONSE, + decode_checkin_response, decode_frame, encode_checkin_request, encode_heartbeat_ack, + encode_heartbeat_ping, encode_login_request, encode_stream_ack, +}; +use crate::push::registration::{ + HttpMethod, HttpRequest, HttpResponse, PushEndpoints, PushHttp, PushIdentity, Registration, + RegistrationClient, checkin_and_register, pns_register, +}; +use crate::push::store::PushStateStore; +use crate::push::transport::{PushTransport, PushTransportFactory}; +use crate::push::{MAXIMUM_CHANGED_IDS, MAXIMUM_FRAME_BYTES, PushError}; +use crate::push::{PushEvent, PushOwner, PushOwnerConfig, PushOwnerDeps, PushScope}; +use aes_gcm::aead::{Aead, KeyInit}; +use aes_gcm::{Aes128Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; +use std::time::{Duration, Instant}; + +fn scope() -> PushScope { + PushScope { + user_id: "user-1".into(), + provider_id: "nvidia".into(), + generation: 7, + } +} + +fn identity() -> PushIdentity { + PushIdentity { + project_id: "project".into(), + api_key: "api-key".into(), + sender_id: "123456789012".into(), + app_id: "app".into(), + firebase_app_id: "1:1:web:abc".into(), + vapid_key: Some("vapid".into()), + } +} + +fn endpoints() -> PushEndpoints { + PushEndpoints { + checkin: "https://push.test/checkin".into(), + c2dm: "https://push.test/c2dm".into(), + fis: "https://push.test/fis/v1".into(), + fcm_registration: "https://push.test/fcm/v1".into(), + fcm_send: "https://push.test/fcm/send".into(), + mcs_host: "push.test".into(), + mcs_port: 5228, + pns: "https://push.test/pns/v1".into(), + pns_client_id: "client-id".into(), + } +} + +struct FakeHttp { + responses: Mutex>, + requests: Mutex)>>, +} + +impl FakeHttp { + fn new(responses: Vec) -> Arc { + Arc::new(Self { + responses: Mutex::new(responses), + requests: Mutex::new(Vec::new()), + }) + } +} + +impl PushHttp for FakeHttp { + fn execute(&self, request: &HttpRequest) -> Result { + self.requests.lock().unwrap().push(( + request.url.clone(), + match request.method { + HttpMethod::Get => "GET".into(), + HttpMethod::Post => "POST".into(), + }, + request.body.clone(), + )); + let mut responses = self.responses.lock().unwrap(); + if responses.is_empty() { + return Err(PushError::new("push_http_failed", "No scripted response")); + } + Ok(responses.remove(0)) + } +} + +struct MemoryStore(Mutex>); + +impl MemoryStore { + fn new(existing: Option) -> Arc { + let mut map = BTreeMap::new(); + if let Some(registration) = existing { + map.insert(scope().account_key(), registration); + } + Arc::new(Self(Mutex::new(map))) + } +} + +impl PushStateStore for MemoryStore { + fn load(&self, account: &str) -> Result, PushError> { + Ok(self.0.lock().unwrap().get(account).cloned()) + } + + fn save(&self, account: &str, registration: &Registration) -> Result<(), PushError> { + self.0 + .lock() + .unwrap() + .insert(account.to_owned(), registration.clone()); + Ok(()) + } + + fn clear(&self, account: &str) -> Result<(), PushError> { + self.0.lock().unwrap().remove(account); + Ok(()) + } +} + +#[derive(Clone, Copy)] +struct HeartbeatPing { + last_received: Instant, + sent: Instant, +} + +#[derive(Clone, Copy, Default)] +struct HeartbeatTiming { + last_received: Option, + ping: Option, +} + +struct ScriptedFactory { + connections: Mutex>>>, + sent: Arc>>>, + connects: Arc, + fail_first: AtomicUsize, + active: Arc, + maximum_active: Arc, + spacing_millis: Arc, + heartbeat: Arc>, +} + +struct TrackedTransport { + chunks: Vec>, + sent: Arc>>>, + active: Arc, + spacing_millis: Arc, + heartbeat: Arc>, +} + +impl PushTransport for TrackedTransport { + fn send(&mut self, frame: &[u8], _timeout: Duration) -> Result<(), PushError> { + if frame.first() == Some(&TAG_HEARTBEAT_PING) { + let mut heartbeat = self.heartbeat.lock().unwrap(); + if heartbeat.ping.is_none() + && let Some(last_received) = heartbeat.last_received + { + heartbeat.ping = Some(HeartbeatPing { + last_received, + sent: Instant::now(), + }); + } + } + self.sent.lock().unwrap().push(frame.to_vec()); + Ok(()) + } + + fn recv(&mut self, _timeout: Duration) -> Result, PushError> { + if self.chunks.is_empty() { + std::thread::sleep(Duration::from_millis(5)); + return Ok(Vec::new()); + } + let spacing = self.spacing_millis.load(Ordering::SeqCst); + if spacing > 0 { + std::thread::sleep(Duration::from_millis(spacing as u64)); + } + let chunk = self.chunks.remove(0); + if !chunk.is_empty() { + self.heartbeat.lock().unwrap().last_received = Some(Instant::now()); + } + Ok(chunk) + } + + fn close(&mut self) {} +} + +impl Drop for TrackedTransport { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::SeqCst); + } +} + +impl PushTransportFactory for ScriptedFactory { + fn connect( + &self, + _host: &str, + _port: u16, + _timeout: Duration, + ) -> Result, PushError> { + self.connects.fetch_add(1, Ordering::SeqCst); + if self.fail_first.load(Ordering::SeqCst) > 0 { + self.fail_first.fetch_sub(1, Ordering::SeqCst); + return Err(PushError::new( + "push_transport_failed", + "The push connection failed", + )); + } + let mut connections = self.connections.lock().unwrap(); + if connections.is_empty() { + return Err(PushError::new( + "push_transport_failed", + "No scripted connection", + )); + } + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.maximum_active.fetch_max(active, Ordering::SeqCst); + Ok(Box::new(TrackedTransport { + chunks: connections.remove(0), + sent: Arc::clone(&self.sent), + active: Arc::clone(&self.active), + spacing_millis: Arc::clone(&self.spacing_millis), + heartbeat: Arc::clone(&self.heartbeat), + })) + } +} + +struct BlockingFactory { + released: Mutex, + wake: Condvar, + connects: Arc, + connections: Mutex>>>, + sent: Arc>>>, + active: Arc, + maximum_active: Arc, +} + +impl Drop for BlockingFactory { + fn drop(&mut self) { + *self.released.lock().unwrap() = true; + self.wake.notify_all(); + } +} + +impl PushTransportFactory for BlockingFactory { + fn connect( + &self, + _host: &str, + _port: u16, + _timeout: Duration, + ) -> Result, PushError> { + self.connects.fetch_add(1, Ordering::SeqCst); + let mut released = self.released.lock().unwrap(); + while !*released { + let (state, timeout) = self + .wake + .wait_timeout(released, Duration::from_millis(20)) + .unwrap(); + released = state; + if timeout.timed_out() { + return Err(PushError::new( + "push_transport_failed", + "The push connection timed out", + )); + } + } + drop(released); + let mut connections = self.connections.lock().unwrap(); + if connections.is_empty() { + return Err(PushError::new( + "push_transport_failed", + "No scripted connection", + )); + } + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.maximum_active.fetch_max(active, Ordering::SeqCst); + Ok(Box::new(TrackedTransport { + chunks: connections.remove(0), + sent: Arc::clone(&self.sent), + active: Arc::clone(&self.active), + spacing_millis: Arc::new(AtomicUsize::new(0)), + heartbeat: Arc::new(Mutex::new(HeartbeatTiming::default())), + })) + } +} + +fn registration() -> Registration { + let keys = EceKeyPair::generate().unwrap(); + Registration { + android_id: 42, + security_token: 7, + gcm_token: "gcm".into(), + fcm_token: "fcm".into(), + endpoint: "https://push.test/send/1".into(), + private_key: keys.private_key(), + public_key: keys.public_key().to_vec(), + auth_secret: keys.auth_secret().to_vec(), + received_persistent_ids: Vec::new(), + created_at_seconds: 1, + fingerprint: String::new(), + } +} + +fn fingerprint_for(scope: &PushScope) -> String { + PushOwnerConfig::bounded(endpoints(), identity(), "nvidia".into(), "device".into()) + .registration_fingerprint(scope) +} + +fn rfc_input_key(keys: &EceKeyPair, server_public: &[u8]) -> [u8; 32] { + let server_secret = + p256::SecretKey::from_slice(&decode("yfWPiYE-n46HLnH0KqZOF1fJJU3MYrct3AELtAQ-oRw")) + .unwrap(); + let user_public = p256::PublicKey::from_sec1_bytes(keys.public_key()).unwrap(); + let shared = + p256::ecdh::diffie_hellman(server_secret.to_nonzero_scalar(), user_public.as_affine()); + let mut key_info = Vec::new(); + key_info.extend_from_slice(b"WebPush: info\0"); + key_info.extend_from_slice(keys.public_key()); + key_info.extend_from_slice(server_public); + let mut input_key = [0_u8; 32]; + hkdf::Hkdf::::new(Some(keys.auth_secret()), shared.raw_secret_bytes()) + .expand(&key_info, &mut input_key) + .unwrap(); + input_key +} + +fn derive_test_input_key(keys: &EceKeyPair, server_public: &[u8]) -> [u8; 32] { + let mut server_scalar = [0_u8; 32]; + server_scalar[31] = 9; + let server_secret = p256::SecretKey::from_slice(&server_scalar).unwrap(); + let user_public = p256::PublicKey::from_sec1_bytes(keys.public_key()).unwrap(); + let shared = + p256::ecdh::diffie_hellman(server_secret.to_nonzero_scalar(), user_public.as_affine()); + let mut key_info = Vec::new(); + key_info.extend_from_slice(b"WebPush: info\0"); + key_info.extend_from_slice(keys.public_key()); + key_info.extend_from_slice(server_public); + let mut input_key = [0_u8; 32]; + hkdf::Hkdf::::new(Some(keys.auth_secret()), shared.raw_secret_bytes()) + .expand(&key_info, &mut input_key) + .unwrap(); + input_key +} + +fn encrypt_record( + content_key: &[u8; 16], + base_nonce: &[u8; 12], + sequence: u64, + record: &[u8], +) -> Vec { + let cipher = Aes128Gcm::new_from_slice(content_key).unwrap(); + let mut nonce = *base_nonce; + for (target, byte) in nonce[4..].iter_mut().zip(sequence.to_be_bytes().iter()) { + *target ^= *byte; + } + cipher.encrypt(Nonce::from_slice(&nonce), record).unwrap() +} + +fn aes128gcm_body(salt: &[u8; 16], server_public: &[u8], records: &[u8]) -> Vec { + aes128gcm_body_with_record_size(salt, server_public, 4096, records) +} + +fn aes128gcm_body_with_record_size( + salt: &[u8; 16], + server_public: &[u8], + record_size: u32, + records: &[u8], +) -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(salt); + body.extend_from_slice(&record_size.to_be_bytes()); + body.push(server_public.len() as u8); + body.extend_from_slice(server_public); + body.extend_from_slice(records); + body +} + +fn data_stanza( + keys: &EceKeyPair, + fields: &[(&str, String)], + persistent_id: &str, + immediate_ack: bool, +) -> Vec { + let mut data = serde_json::Map::new(); + for (key, value) in fields { + data.insert((*key).to_owned(), serde_json::Value::String(value.clone())); + } + let payload = serde_json::json!({ "data": data }).to_string(); + let (ciphertext, server_public, salt) = legacy_encryption(keys, payload.as_bytes()); + let mut stanza = Vec::new(); + write_string(3, "from", &mut stanza); + write_string(5, "category", &mut stanza); + write_app_data( + "crypto-key", + &format!("dh={}", URL_SAFE_NO_PAD.encode(&server_public)), + &mut stanza, + ); + write_app_data( + "encryption", + &format!("salt={}", URL_SAFE_NO_PAD.encode(salt)), + &mut stanza, + ); + write_string(9, persistent_id, &mut stanza); + write_bytes(21, &ciphertext, &mut stanza); + if immediate_ack { + write_tag(24, 0, &mut stanza); + write_varint(1, &mut stanza); + } + stanza +} + +fn data_frame(keys: &EceKeyPair, fields: &[(&str, String)], persistent_id: &str) -> Vec { + server_frame( + TAG_DATA_MESSAGE_STANZA, + &data_stanza(keys, fields, persistent_id, false), + ) +} + +fn legacy_encryption(keys: &EceKeyPair, plaintext: &[u8]) -> (Vec, Vec, [u8; 16]) { + let mut server_scalar = [0_u8; 32]; + server_scalar[31] = 9; + let server_secret = p256::SecretKey::from_slice(&server_scalar).unwrap(); + let server_public = p256::elliptic_curve::sec1::ToEncodedPoint::to_encoded_point( + &server_secret.public_key(), + false, + ) + .as_bytes() + .to_vec(); + let salt = [3_u8; 16]; + let user_public = p256::PublicKey::from_sec1_bytes(keys.public_key()).unwrap(); + let shared = + p256::ecdh::diffie_hellman(server_secret.to_nonzero_scalar(), user_public.as_affine()); + let mut input_key = [0_u8; 32]; + hkdf::Hkdf::::new(Some(keys.auth_secret()), shared.raw_secret_bytes()) + .expand(b"Content-Encoding: auth\0", &mut input_key) + .unwrap(); + let mut context = Vec::new(); + context.extend_from_slice(&[0, 65]); + context.extend_from_slice(keys.public_key()); + context.extend_from_slice(&[0, 65]); + context.extend_from_slice(&server_public); + let mut key_info = b"Content-Encoding: aesgcm\0P-256\0".to_vec(); + key_info.extend_from_slice(&context); + let mut nonce_info = b"Content-Encoding: nonce\0P-256\0".to_vec(); + nonce_info.extend_from_slice(&context); + let prk = hkdf::Hkdf::::new(Some(&salt), &input_key); + let mut content_key = [0_u8; 16]; + let mut nonce = [0_u8; 12]; + prk.expand(&key_info, &mut content_key).unwrap(); + prk.expand(&nonce_info, &mut nonce).unwrap(); + let mut record = vec![0, 0]; + record.extend_from_slice(plaintext); + let cipher = Aes128Gcm::new_from_slice(&content_key).unwrap(); + let ciphertext = cipher + .encrypt(Nonce::from_slice(&nonce), record.as_slice()) + .unwrap(); + (ciphertext, server_public, salt) +} + +fn aes128gcm_encryption(keys: &EceKeyPair, plaintext: &[u8]) -> (Vec, Vec, [u8; 16]) { + let mut server_scalar = [0_u8; 32]; + server_scalar[31] = 9; + let server_secret = p256::SecretKey::from_slice(&server_scalar).unwrap(); + let server_public = p256::elliptic_curve::sec1::ToEncodedPoint::to_encoded_point( + &server_secret.public_key(), + false, + ) + .as_bytes() + .to_vec(); + let salt = [3_u8; 16]; + let input_key = derive_test_input_key(keys, &server_public); + let (content_key, base_nonce) = derive_content_keys(&input_key, &salt).unwrap(); + let mut record = plaintext.to_vec(); + record.push(2); + let ciphertext = encrypt_record(&content_key, &base_nonce, 0, &record); + (ciphertext, server_public, salt) +} + +fn aes128gcm_data_frame( + keys: &EceKeyPair, + fields: &[(&str, String)], + persistent_id: &str, +) -> Vec { + let mut data = serde_json::Map::new(); + for (key, value) in fields { + data.insert((*key).to_owned(), serde_json::Value::String(value.clone())); + } + let payload = serde_json::json!({ "data": data }).to_string(); + let mut server_scalar = [0_u8; 32]; + server_scalar[31] = 9; + let server_secret = p256::SecretKey::from_slice(&server_scalar).unwrap(); + let server_public = p256::elliptic_curve::sec1::ToEncodedPoint::to_encoded_point( + &server_secret.public_key(), + false, + ) + .as_bytes() + .to_vec(); + let salt = [5_u8; 16]; + let input_key = derive_test_input_key(keys, &server_public); + let (content_key, base_nonce) = derive_content_keys(&input_key, &salt).unwrap(); + let mut record = payload.into_bytes(); + record.push(2); + let ciphertext = encrypt_record(&content_key, &base_nonce, 0, &record); + let mut stanza = Vec::new(); + write_string(3, "from", &mut stanza); + write_string(5, "category", &mut stanza); + write_string(9, persistent_id, &mut stanza); + write_bytes( + 21, + &aes128gcm_body(&salt, &server_public, &ciphertext), + &mut stanza, + ); + server_frame(TAG_DATA_MESSAGE_STANZA, &stanza) +} + +fn server_frame(tag: u8, payload: &[u8]) -> Vec { + let mut frame = Vec::with_capacity(payload.len() + 6); + frame.push(tag); + write_varint(payload.len() as u64, &mut frame); + frame.extend_from_slice(payload); + frame +} + +fn write_varint(mut value: u64, out: &mut Vec) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn write_tag(field: u64, wire: u64, out: &mut Vec) { + write_varint((field << 3) | wire, out); +} + +fn write_bytes(field: u64, value: &[u8], out: &mut Vec) { + write_tag(field, 2, out); + write_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +fn write_string(field: u64, value: &str, out: &mut Vec) { + write_bytes(field, value.as_bytes(), out); +} + +fn write_app_data(key: &str, value: &str, out: &mut Vec) { + let mut nested = Vec::new(); + write_string(1, key, &mut nested); + write_string(2, value, &mut nested); + write_bytes(7, &nested, out); +} + +fn checkin_body(android_id: u64, security_token: u64) -> Vec { + let mut body = Vec::new(); + write_tag(7, 1, &mut body); + body.extend_from_slice(&android_id.to_le_bytes()); + write_tag(8, 1, &mut body); + body.extend_from_slice(&security_token.to_le_bytes()); + body +} + +fn push_http(responses: Vec) -> Arc { + FakeHttp::new(responses) +} + +fn responses_for_registration() -> Vec { + vec![ + HttpResponse { + status: 200, + body: checkin_body(42, 7), + }, + HttpResponse { + status: 200, + body: b"token=gcm-token".to_vec(), + }, + HttpResponse { + status: 200, + body: br#"{"authToken":{"token":"fis-token"}}"#.to_vec(), + }, + HttpResponse { + status: 200, + body: br#"{"token":"fcm-token"}"#.to_vec(), + }, + HttpResponse { + status: 204, + body: Vec::new(), + }, + ] +} + +struct OwnerHarness { + owner: PushOwner, + factory: Arc, + store: Arc, + desired: Arc>>, + config: PushOwnerConfig, + http: Arc, +} + +fn owner_harness( + http: Arc, + connections: Vec>>, + sink: Arc, + initial: Option, + existing: Option, +) -> OwnerHarness { + tuned_owner_harness(http, connections, sink, initial, existing, |_| {}) +} + +fn tuned_owner_harness( + http: Arc, + connections: Vec>>, + sink: Arc, + initial: Option, + existing: Option, + tune: impl FnOnce(&mut PushOwnerConfig), +) -> OwnerHarness { + let store = MemoryStore::new(existing); + let sent = Arc::new(Mutex::new(Vec::new())); + let active = Arc::new(AtomicUsize::new(0)); + let maximum_active = Arc::new(AtomicUsize::new(0)); + let spacing_millis = Arc::new(AtomicUsize::new(0)); + let factory = Arc::new(ScriptedFactory { + connections: Mutex::new(connections), + sent, + connects: Arc::new(AtomicUsize::new(0)), + fail_first: AtomicUsize::new(0), + active: Arc::clone(&active), + maximum_active: Arc::clone(&maximum_active), + spacing_millis: Arc::clone(&spacing_millis), + heartbeat: Arc::new(Mutex::new(HeartbeatTiming::default())), + }); + let desired = Arc::new(Mutex::new(initial)); + let scope_source = Arc::clone(&desired); + let worker_scope = Arc::clone(&scope_source); + let token_source = Arc::new(move |scope: &PushScope| { + let current = worker_scope.lock().unwrap().clone()?; + (current.account_key() == scope.account_key()).then(|| "gfnjwt".to_owned()) + }); + let mut config = PushOwnerConfig { + connect_timeout: Duration::from_secs(1), + read_timeout: Duration::from_millis(20), + login_timeout: Duration::from_secs(2), + heartbeat_interval: Duration::from_secs(30), + heartbeat_ack_timeout: Duration::from_millis(500), + minimum_heartbeat_interval: Duration::from_millis(100), + maximum_heartbeat_interval: Duration::from_secs(2), + coalescing_window: Duration::from_millis(30), + backoff_initial: Duration::from_millis(10), + backoff_maximum: Duration::from_millis(40), + stop_deadline: Duration::from_secs(2), + ..PushOwnerConfig::bounded(endpoints(), identity(), "nvidia".into(), "device".into()) + }; + tune(&mut config); + let http_dep: Arc = http.clone(); + let owner = PushOwner::new( + config.clone(), + PushOwnerDeps { + http: http_dep, + transport: factory.clone(), + store: store.clone(), + sink, + scope_source: Arc::new(move || scope_source.lock().unwrap().clone()), + token_source, + }, + ); + OwnerHarness { + owner, + factory, + store, + desired, + config, + http, + } +} + +impl OwnerHarness { + fn config(&self) -> PushOwnerConfig { + self.config.clone() + } +} + +fn shutdown(harness: &mut OwnerHarness) { + *harness.desired.lock().unwrap() = None; + let deadline = Instant::now() + Duration::from_secs(2); + while harness.owner.is_running() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!(!harness.owner.is_running()); + assert!(harness.owner.shutdown(Duration::from_secs(2))); + assert_eq!(harness.owner.worker_count(), 0); +} + +#[test] +fn encryption_round_trip_decrypts_the_payload() { + let keys = EceKeyPair::generate().unwrap(); + let (ciphertext, server_public, salt) = legacy_encryption(&keys, b"{\"data\":{}}"); + let crypto_key = format!("dh={}", URL_SAFE_NO_PAD.encode(&server_public)); + let encryption = format!("salt={}", URL_SAFE_NO_PAD.encode(salt)); + let plaintext = decrypt_web_push( + &keys, + None, + Some(&crypto_key), + Some(&encryption), + &ciphertext, + ) + .unwrap(); + assert_eq!(plaintext, b"{\"data\":{}}"); + let (aes128_body, aes128_public, aes128_salt) = aes128gcm_encryption(&keys, b"{\"data\":{}}"); + let body = aes128gcm_body(&aes128_salt, &aes128_public, &aes128_body); + let plaintext = decrypt_web_push(&keys, None, None, None, &body).unwrap(); + assert_eq!(plaintext, b"{\"data\":{}}"); +} + +#[test] +fn decryption_matches_the_rfc8291_fixed_vector() { + let keys = EceKeyPair::from_parts( + &decode("q1dXpw3UpT5VOmu_cf_v6ih07Aems3njxI-JWgLcM94"), + &decode("BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4"), + &decode("BTBZMqHH6r4Tts7J_aSIgg"), + ) + .unwrap(); + let body = decode( + "DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPTpK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN", + ); + let plaintext = decrypt_web_push(&keys, None, None, None, &body).unwrap(); + assert_eq!(plaintext, b"When I grow up, I want to be a watermelon"); +} + +#[test] +fn decryption_rejects_a_tampered_rfc8291_body() { + let keys = EceKeyPair::from_parts( + &decode("q1dXpw3UpT5VOmu_cf_v6ih07Aems3njxI-JWgLcM94"), + &decode("BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4"), + &decode("BTBZMqHH6r4Tts7J_aSIgg"), + ) + .unwrap(); + let mut body = decode( + "DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPTpK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN", + ); + let last = body.len() - 1; + body[last] ^= 0x40; + assert_eq!( + decrypt_web_push(&keys, None, None, None, &body) + .unwrap_err() + .code, + "push_decrypt_failed" + ); +} + +#[test] +fn multi_record_payloads_use_the_documented_sequence_nonce() { + let keys = EceKeyPair::from_parts( + &decode("q1dXpw3UpT5VOmu_cf_v6ih07Aems3njxI-JWgLcM94"), + &decode("BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4"), + &decode("BTBZMqHH6r4Tts7J_aSIgg"), + ) + .unwrap(); + let salt = decode("DGv6ra1nlYgDCS1FRnbzlw"); + let salt: [u8; 16] = salt.try_into().unwrap(); + let server_public = decode( + "BP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A8", + ); + let input_key = rfc_input_key(&keys, &server_public); + assert_eq!( + input_key.to_vec(), + decode("S4lYMb_L0FxCeq0WhDx813KgSYqU26kOyzWUdsXYyrg"), + "the RFC key derivation must reproduce the published IKM" + ); + let (content_key, base_nonce) = derive_content_keys(&input_key, &salt).unwrap(); + assert_eq!( + content_key.to_vec(), + decode("oIhVW04MRdy2XN9CiKLxTg"), + "the RFC key derivation must reproduce the published CEK" + ); + assert_eq!( + base_nonce.to_vec(), + decode("4h_95klXJ5E_qnoN"), + "the RFC key derivation must reproduce the published nonce" + ); + let first = encrypt_record( + &content_key, + &base_nonce, + 0, + b"When I grow up, I want to be a watermelon\x02", + ); + assert_eq!( + first, + decode("8pfeW0KbunFT06SuDKoJH9Ql87S1QUrdirN6GcG7sFz1y1sqLgVi1VhjVkHsUoEsbI_0LpXMuGvnzQ") + ); + let mut records = Vec::new(); + records.extend_from_slice(&encrypt_record(&content_key, &base_nonce, 0, b"a\x01")); + records.extend_from_slice(&encrypt_record(&content_key, &base_nonce, 1, b"b\x02")); + let body = aes128gcm_body_with_record_size(&salt, &server_public, 18, &records); + let plaintext = decrypt_web_push(&keys, None, None, None, &body).unwrap(); + assert_eq!(plaintext, b"ab"); +} + +#[test] +fn decryption_honors_a_declared_aes128gcm_content_encoding() { + let keys = EceKeyPair::generate().unwrap(); + let (ciphertext, server_public, salt) = aes128gcm_encryption(&keys, b"{\"data\":{}}"); + let body = aes128gcm_body(&salt, &server_public, &ciphertext); + let misleading = format!("dh={}", URL_SAFE_NO_PAD.encode([1_u8; 65])); + let misleading_salt = format!("salt={}", URL_SAFE_NO_PAD.encode([2_u8; 16])); + let plaintext = decrypt_web_push( + &keys, + Some("aes128gcm"), + Some(&misleading), + Some(&misleading_salt), + &body, + ) + .unwrap(); + assert_eq!(plaintext, b"{\"data\":{}}"); +} + +#[test] +fn decryption_honors_a_declared_aesgcm_content_encoding() { + let keys = EceKeyPair::generate().unwrap(); + let (ciphertext, server_public, salt) = legacy_encryption(&keys, b"{\"data\":{}}"); + let crypto_key = format!("dh={}", URL_SAFE_NO_PAD.encode(&server_public)); + let encryption = format!("salt={}", URL_SAFE_NO_PAD.encode(salt)); + let plaintext = decrypt_web_push( + &keys, + Some("aesgcm"), + Some(&crypto_key), + Some(&encryption), + &ciphertext, + ) + .unwrap(); + assert_eq!(plaintext, b"{\"data\":{}}"); + assert_eq!( + decrypt_web_push(&keys, Some("aesgcm"), None, Some(&encryption), &ciphertext) + .err() + .map(|error| error.code), + Some("push_decrypt_invalid") + ); +} + +#[test] +fn decryption_rejects_a_declared_unsupported_content_encoding() { + let keys = EceKeyPair::generate().unwrap(); + let (ciphertext, server_public, salt) = aes128gcm_encryption(&keys, b"{\"data\":{}}"); + let body = aes128gcm_body(&salt, &server_public, &ciphertext); + assert_eq!( + decrypt_web_push(&keys, Some("br"), None, None, &body) + .err() + .map(|error| error.code), + Some("push_decrypt_unsupported") + ); +} + +#[test] +fn decryption_rejects_an_unsupported_record_size() { + let keys = EceKeyPair::generate().unwrap(); + let (ciphertext, server_public, salt) = aes128gcm_encryption(&keys, b"{\"data\":{}}"); + let mut body = aes128gcm_body(&salt, &server_public, &ciphertext); + body[16..20].copy_from_slice(&0_u32.to_be_bytes()); + assert_eq!( + decrypt_web_push(&keys, None, None, None, &body) + .unwrap_err() + .code, + "push_decrypt_invalid" + ); +} + +#[test] +fn stored_key_material_must_match_its_public_key() { + let first = EceKeyPair::generate().unwrap(); + let second = EceKeyPair::generate().unwrap(); + let error = match EceKeyPair::from_parts( + &first.private_key(), + second.public_key(), + first.auth_secret(), + ) { + Ok(_) => panic!("an unrelated public key was accepted"), + Err(error) => error, + }; + assert_eq!(error.code, "push_key_mismatch"); + EceKeyPair::from_parts( + &first.private_key(), + first.public_key(), + first.auth_secret(), + ) + .unwrap(); + let malformed = EceKeyPair::from_parts(&first.private_key(), &[0_u8; 5], first.auth_secret()); + assert_eq!( + malformed.err().map(|error| error.code), + Some("push_key_invalid") + ); + let unrelated = EceKeyPair::from_parts(&first.private_key(), &[0_u8; 65], first.auth_secret()); + assert_eq!( + unrelated.err().map(|error| error.code), + Some("push_key_mismatch") + ); +} + +#[test] +fn config_updates_restart_the_session_without_a_second_worker() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new()), session_chunks(Vec::new())], + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(harness.owner.worker_count(), 1); + let mut updated = harness.config(); + updated.coalescing_window = Duration::from_millis(10); + harness.owner.update_config(updated); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + harness.factory.connects.load(Ordering::SeqCst) >= 2, + "config update did not restart the session" + ); + assert_eq!( + harness.owner.worker_count(), + 1, + "config update spawned a second worker" + ); + assert!(harness.factory.maximum_active.load(Ordering::SeqCst) <= 1); + shutdown(&mut harness); +} + +#[test] +fn a_continuous_stream_never_postpones_a_coalesced_event() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut frames: Vec> = session_chunks(Vec::new()); + for index in 0..60 { + frames.push(data_frame( + &keys, + &[ + ("messageType", "\"LIBRARY_CHANGE\"".into()), + ("changedIds", format!("[\"app-{index}\"]")), + ], + &format!("p{index}"), + )); + } + let connections = vec![frames]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.factory.spacing_millis.store(5, Ordering::SeqCst); + harness.owner.update_config(PushOwnerConfig { + coalescing_window: Duration::from_millis(40), + ..harness.config() + }); + harness.owner.start().unwrap(); + let event = events_rx + .recv_timeout(Duration::from_millis(200)) + .expect("a coalescing window must close while messages keep arriving"); + match event { + PushEvent::LibraryChanged { changed_ids } => assert!(!changed_ids.is_empty()), + other => panic!("unexpected event {other:?}"), + } + let mut events = 1; + while events_rx.recv_timeout(Duration::from_millis(80)).is_ok() { + events += 1; + } + assert!(events >= 2, "later windows must keep closing, saw {events}"); + shutdown(&mut harness); +} + +#[test] +fn a_provider_mismatch_never_reaches_the_token_source() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let mut other_provider = scope(); + other_provider.provider_id = "alliance".into(); + *harness.desired.lock().unwrap() = Some(other_provider); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.owner.is_running() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + !harness.owner.is_running(), + "a foreign provider must stop the session" + ); + let connects = harness.factory.connects.load(Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(150)); + assert_eq!( + harness.factory.connects.load(Ordering::SeqCst), + connects, + "a foreign provider must not open a new push session" + ); + shutdown(&mut harness); +} + +#[test] +fn configuration_snapshots_are_atomic_with_their_generation() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let owner = Arc::new(harness.owner); + let reader = { + let owner = Arc::clone(&owner); + std::thread::spawn(move || { + let mut highest = 0_u64; + for _ in 0..200_000 { + let (config, generation) = owner.config(); + assert!( + generation >= highest, + "config generations must never move backwards" + ); + highest = generation; + assert_eq!( + config.coalescing_window, + expected_window(generation), + "torn configuration snapshot at generation {generation}" + ); + } + }) + }; + let writer = { + let owner = Arc::clone(&owner); + std::thread::spawn(move || { + for index in 1..=2000_u64 { + let (mut config, generation) = owner.config(); + assert_eq!(config.coalescing_window, expected_window(generation)); + config.coalescing_window = Duration::from_millis(index % 97 + 1); + owner.update_config(config); + } + }) + }; + let _ = writer.join(); + let _ = reader.join(); + let mut harness = OwnerHarness { + owner: Arc::try_unwrap(owner).ok().expect("owner"), + factory: harness.factory, + store: harness.store, + desired: harness.desired, + config: harness.config, + http: harness.http, + }; + shutdown(&mut harness); +} + +fn expected_window(generation: u64) -> Duration { + if generation == 1 { + Duration::from_millis(30) + } else { + Duration::from_millis(((generation - 1) % 97) + 1) + } +} + +fn login_refusal_frame(code: u64) -> Vec { + let mut refusal = Vec::new(); + write_tag(1, 0, &mut refusal); + write_varint(code, &mut refusal); + let mut response = Vec::new(); + write_bytes(3, &refusal, &mut response); + server_frame(TAG_LOGIN_RESPONSE, &response) +} + +fn login_response_frame(heartbeat_interval_ms: Option) -> Vec { + let mut payload = Vec::new(); + write_string(1, "login-id", &mut payload); + if let Some(interval) = heartbeat_interval_ms { + let mut config = Vec::new(); + write_tag(3, 0, &mut config); + write_varint(interval as u64, &mut config); + write_bytes(7, &config, &mut payload); + } + server_frame(TAG_LOGIN_RESPONSE, &payload) +} + +fn session_chunks(extra: Vec>) -> Vec> { + let mut chunks = vec![vec![MCS_VERSION], login_response_frame(None)]; + chunks.extend(extra); + chunks +} + +fn checkin_requests(harness: &OwnerHarness) -> usize { + harness + .http + .requests + .lock() + .unwrap() + .iter() + .filter(|(url, _, _)| url == &endpoints().checkin) + .count() +} + +#[test] +fn a_login_refusal_refreshes_the_registration_without_minting_a_new_identity() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "stale-fcm-token".into(); + let refused = login_refusal_frame(2); + let connections = vec![vec![vec![MCS_VERSION], refused], vec![vec![MCS_VERSION]]]; + let mut responses = responses_for_registration(); + responses[3] = HttpResponse { + status: 200, + body: br#"{"token":"refreshed-fcm-token"}"#.to_vec(), + }; + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(responses), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while harness.factory.connects.load(Ordering::SeqCst) < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + harness.factory.connects.load(Ordering::SeqCst) >= 2, + "the refused registration must reconnect" + ); + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("a refreshed registration must be stored"); + assert_eq!( + stored.fcm_token, "refreshed-fcm-token", + "the cached credentials must be replaced" + ); + assert_eq!( + (stored.android_id, stored.security_token), + (42, 7), + "a login refusal must retain the stored device identity" + ); + let requests = harness.http.requests.lock().unwrap().clone(); + let checkin = requests + .iter() + .find(|(url, _, _)| url == &endpoints().checkin) + .expect("the refresh must present the stored identity at check-in"); + assert_eq!( + checkin.2, + encode_checkin_request(&CheckinRequest { + android_id: Some(42), + security_token: Some(7), + }), + "the refresh must re-use the stored device identity" + ); + shutdown(&mut harness); +} + +#[test] +fn a_zero_login_error_code_is_not_a_rejection() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "kept-fcm-token".into(); + let zero_error = login_refusal_frame(0); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![vec![vec![MCS_VERSION], zero_error]], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + std::thread::sleep(Duration::from_millis(250)); + assert_eq!( + checkin_requests(&harness), + 0, + "a zero login error code must not trigger a registration refresh" + ); + assert_eq!( + harness.factory.connects.load(Ordering::SeqCst), + 1, + "a zero login error code must not restart the session" + ); + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("a zero login error code must keep the stored registration"); + assert_eq!(stored.fcm_token, "kept-fcm-token"); + assert!(harness.owner.is_running(), "the session must stay active"); + shutdown(&mut harness); +} + +#[test] +fn repeated_login_refusals_keep_refreshing_without_minting_a_new_identity() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "stale-fcm-token".into(); + let mut responses = Vec::new(); + for index in 0..3 { + let mut round = responses_for_registration(); + round[3] = HttpResponse { + status: 200, + body: format!("{{\"token\":\"refreshed-fcm-token-{index}\"}}").into_bytes(), + }; + responses.extend(round); + } + let mut harness = owner_harness( + push_http(responses), + vec![ + vec![vec![MCS_VERSION], login_refusal_frame(2)], + vec![vec![MCS_VERSION], login_refusal_frame(7)], + vec![vec![MCS_VERSION], login_refusal_frame(13)], + ], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(4); + while checkin_requests(&harness) < 3 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + checkin_requests(&harness) >= 3, + "a repeatedly refused registration must keep being refreshed" + ); + let requests = harness.http.requests.lock().unwrap().clone(); + let checkins: Vec<&(String, String, Vec)> = requests + .iter() + .filter(|(url, _, _)| url == &endpoints().checkin) + .collect(); + for (_, _, body) in &checkins { + assert_eq!( + body, + &encode_checkin_request(&CheckinRequest { + android_id: Some(42), + security_token: Some(7), + }), + "a refresh must retain the stored device identity" + ); + } + if let Some(stored) = harness.store.load(&scope().account_key()).unwrap() { + assert_ne!( + stored.fcm_token, "stale-fcm-token", + "a refused registration must never return to the cache" + ); + assert_eq!( + stored.android_id, 42, + "the device identity must be retained" + ); + } + shutdown(&mut harness); +} + +#[test] +fn corrupt_key_material_is_refreshed_without_a_session_attempt() { + let mismatched = EceKeyPair::generate().unwrap(); + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = mismatched.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "corrupt-fcm-token".into(); + let mut responses = responses_for_registration(); + responses[3] = HttpResponse { + status: 200, + body: br#"{"token":"fresh-fcm-token"}"#.to_vec(), + }; + let mut harness = owner_harness( + push_http(responses), + vec![session_chunks(Vec::new()), session_chunks(Vec::new())], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("unusable key material must be replaced"); + assert_eq!(stored.fcm_token, "fresh-fcm-token"); + assert!( + EceKeyPair::from_parts(&stored.private_key, &stored.public_key, &stored.auth_secret) + .is_ok(), + "the refreshed registration must carry loadable key material" + ); + assert_eq!( + harness.factory.connects.load(Ordering::SeqCst), + 1, + "unusable key material must never open a push session" + ); + shutdown(&mut harness); +} + +#[test] +fn a_rejected_device_identity_is_cleared_before_the_next_check_in() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.created_at_seconds = existing.created_at_seconds.saturating_sub(8 * 24 * 60 * 60); + let mut responses = vec![HttpResponse { + status: 401, + body: Vec::new(), + }]; + let mut round = responses_for_registration(); + round[0] = HttpResponse { + status: 200, + body: checkin_body(84, 9), + }; + responses.extend(round); + let mut harness = owner_harness( + push_http(responses), + vec![session_chunks(Vec::new())], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while checkin_requests(&harness) < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + while harness + .store + .load(&scope().account_key()) + .unwrap() + .is_none_or(|stored| stored.fcm_token != "fcm-token") + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let requests = harness.http.requests.lock().unwrap().clone(); + let checkins: Vec<&(String, String, Vec)> = requests + .iter() + .filter(|(url, _, _)| url == &endpoints().checkin) + .collect(); + assert!( + checkins.len() >= 2, + "a rejected device identity must be re-checked-in" + ); + assert_eq!( + checkins[0].2, + encode_checkin_request(&CheckinRequest { + android_id: Some(42), + security_token: Some(7), + }), + "the first check-in must present the stored device identity" + ); + assert_eq!( + checkins[1].2, + encode_checkin_request(&CheckinRequest { + android_id: None, + security_token: None, + }), + "a rejected device identity must not be presented again" + ); + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("a fresh registration must be stored"); + assert_eq!(stored.fcm_token, "fcm-token"); + assert_eq!( + (stored.android_id, stored.security_token), + (84, 9), + "a rejected device identity must be replaced by the fresh identity" + ); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + harness.owner.is_running(), + "the fresh registration must open a push session" + ); + assert_eq!(harness.factory.connects.load(Ordering::SeqCst), 1); + shutdown(&mut harness); +} + +#[test] +fn transient_failures_never_discard_a_stored_registration() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "kept-fcm-token".into(); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + Some(existing), + ); + harness.factory.fail_first.store(3, Ordering::SeqCst); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + while harness.factory.connects.load(Ordering::SeqCst) < 3 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("a transient failure must keep the stored registration"); + assert_eq!(stored.fcm_token, "kept-fcm-token"); + shutdown(&mut harness); +} + +#[test] +fn registration_reads_enforce_a_real_byte_limit() { + use crate::push::registration::{MAXIMUM_RESPONSE_BYTES, read_bounded}; + let exact = vec![b'a'; MAXIMUM_RESPONSE_BYTES as usize]; + let read = read_bounded(std::io::Cursor::new(exact.clone()), MAXIMUM_RESPONSE_BYTES).unwrap(); + assert_eq!(read.len(), exact.len()); + let oversized = vec![b'a'; MAXIMUM_RESPONSE_BYTES as usize + 1]; + assert_eq!( + read_bounded(std::io::Cursor::new(oversized), MAXIMUM_RESPONSE_BYTES) + .err() + .map(|error| error.code), + Some("push_http_failed") + ); + let unbounded = vec![b'a'; 4 * 1024 * 1024]; + assert!(read_bounded(std::io::Cursor::new(unbounded), MAXIMUM_RESPONSE_BYTES).is_err()); +} + +struct LocalHttpServer { + url: String, + _handle: std::thread::JoinHandle<()>, +} + +fn serve_local_http(responses: Vec>) -> LocalHttpServer { + use std::io::{Read as _, Write as _}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let handle = std::thread::spawn(move || { + for response in responses { + let Ok((mut socket, _)) = listener.accept() else { + return; + }; + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + loop { + match socket.read(&mut chunk) { + Ok(0) => break, + Ok(count) => { + request.extend_from_slice(&chunk[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + Err(_) => break, + } + } + let _ = socket.write_all(&response); + let _ = socket.shutdown(std::net::Shutdown::Write); + } + }); + LocalHttpServer { + url: format!("http://{address}"), + _handle: handle, + } +} + +fn execute_local( + url: &str, + timeout: Duration, +) -> Result { + use crate::push::registration::{HttpMethod, HttpRequest, ReqwestPushHttp}; + ReqwestPushHttp::new()? + .execute(&HttpRequest { + url: url.to_owned(), + method: HttpMethod::Get, + headers: Vec::new(), + body: Vec::new(), + timeout, + }) + .map_err(|error| PushError::new(error.code, error.message)) +} + +fn expect_request_error(result: Result) -> PushError { + match result { + Ok(_) => panic!("the push request must fail"), + Err(error) => error, + } +} + +#[test] +fn an_oversized_close_delimited_response_is_rejected_locally() { + use crate::push::registration::MAXIMUM_RESPONSE_BYTES; + let mut response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_vec(); + response.extend(std::iter::repeat_n( + b'a', + MAXIMUM_RESPONSE_BYTES as usize + 1, + )); + let server = serve_local_http(vec![response]); + let error = expect_request_error(execute_local(&server.url, Duration::from_secs(5))); + assert_eq!(error.code, "push_http_failed"); +} + +#[test] +fn an_oversized_chunked_response_is_rejected_locally() { + use crate::push::registration::MAXIMUM_RESPONSE_BYTES; + let chunk = vec![b'a'; 40 * 1024]; + let header = format!("{:x}\r\n", chunk.len()); + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + response.extend_from_slice(header.as_bytes()); + response.extend_from_slice(&chunk); + response.extend_from_slice(b"\r\n"); + response.extend_from_slice(header.as_bytes()); + response.extend_from_slice(&chunk); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + assert!( + (2 * chunk.len()) as u64 > MAXIMUM_RESPONSE_BYTES, + "the scripted body must exceed the accepted size" + ); + let server = serve_local_http(vec![response]); + let error = expect_request_error(execute_local(&server.url, Duration::from_secs(5))); + assert_eq!(error.code, "push_http_failed"); +} + +#[test] +fn a_response_at_the_byte_limit_is_accepted_locally() { + use crate::push::registration::MAXIMUM_RESPONSE_BYTES; + let body = vec![b'a'; MAXIMUM_RESPONSE_BYTES as usize]; + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(&body); + let server = serve_local_http(vec![response]); + let response = execute_local(&server.url, Duration::from_secs(5)) + .expect("a response at the limit must be accepted"); + assert_eq!(response.status, 200); + assert_eq!(response.body.len(), body.len()); +} + +#[test] +fn a_malformed_installation_response_is_rejected_locally() { + use crate::push::registration::firebase_installation_token; + let server = serve_local_http(vec![ + b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\nConnection: close\r\n\r\n{".to_vec(), + ]); + let mut endpoints = endpoints(); + endpoints.fis = server.url.clone(); + let error = expect_request_error(firebase_installation_token( + &RegistrationClient { + http: &crate::push::registration::ReqwestPushHttp::new().unwrap(), + endpoints: &endpoints, + timeout: Duration::from_secs(5), + }, + &identity(), + "fid", + )); + assert_eq!(error.code, "push_fis_incomplete"); +} + +#[test] +fn a_stalled_push_response_honors_the_request_deadline() { + use std::io::Read as _; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let handle = std::thread::spawn(move || { + let Ok((mut socket, _)) = listener.accept() else { + return; + }; + let mut chunk = [0_u8; 1024]; + let _ = socket.read(&mut chunk); + std::thread::sleep(Duration::from_millis(900)); + }); + let started = Instant::now(); + let error = expect_request_error(execute_local( + &format!("http://{address}"), + Duration::from_millis(300), + )); + assert_eq!(error.code, "push_http_failed"); + assert!( + started.elapsed() < Duration::from_secs(5), + "the request deadline was ignored: {:?}", + started.elapsed() + ); + let _ = handle.join(); +} + +#[test] +fn tls_transport_honors_a_short_connect_deadline() { + let started = Instant::now(); + let error = match crate::push::transport::TlsPushTransportFactory::new().connect( + "10.255.255.1", + 5228, + Duration::from_millis(300), + ) { + Ok(_) => panic!("an unroutable push address was reported as connected"), + Err(error) => error, + }; + assert_eq!(error.code, "push_transport_failed"); + assert!( + started.elapsed() < Duration::from_secs(10), + "the connect deadline was ignored: {:?}", + started.elapsed() + ); +} + +#[test] +fn frame_reader_accepts_the_versioned_first_frame_and_split_input() { + let mut reader = FrameReader::new(); + reader.extend(&[MCS_VERSION, TAG_LOGIN_RESPONSE, 3, 1, 2, 3]); + let frame = reader.next_frame(MAXIMUM_FRAME_BYTES).unwrap().unwrap(); + assert_eq!(frame.0, TAG_LOGIN_RESPONSE); + assert_eq!(frame.1, vec![1, 2, 3]); + let mut split = FrameReader::new(); + split.extend(&[MCS_VERSION, TAG_HEARTBEAT_PING]); + assert!(split.next_frame(MAXIMUM_FRAME_BYTES).unwrap().is_none()); + split.extend(&[0]); + assert_eq!( + split.next_frame(MAXIMUM_FRAME_BYTES).unwrap().unwrap().0, + TAG_HEARTBEAT_PING + ); +} + +#[test] +fn frame_reader_rejects_unsupported_versions_and_oversized_frames() { + let mut reader = FrameReader::new(); + reader.extend(&[7, TAG_LOGIN_RESPONSE, 0]); + assert_eq!( + reader.next_frame(MAXIMUM_FRAME_BYTES).unwrap_err().code, + "push_protocol_version" + ); + let mut oversize = FrameReader::new(); + let mut frame = vec![MCS_VERSION, TAG_DATA_MESSAGE_STANZA]; + write_varint((MAXIMUM_FRAME_BYTES + 1) as u64, &mut frame); + oversize.extend(&frame); + assert_eq!( + oversize.next_frame(MAXIMUM_FRAME_BYTES).unwrap_err().code, + "push_protocol_frame" + ); +} + +#[test] +fn login_request_uses_the_documented_wire_prefix() { + let login = encode_login_request(42, 7, &["persistent".into()]); + assert_eq!(login[0], MCS_VERSION); + assert_eq!(login[1], 2); + assert_eq!(login[3], 0x0a); + let login_text = String::from_utf8_lossy(&login); + assert!(login_text.contains("mcs.android.com")); + assert!(login_text.contains("android-2a")); + assert!(login_text.contains("persistent")); + assert!(login_text.contains("new_vc")); + assert!( + login.windows(2).any(|window| window == [0x80, 0x01]), + "auth_service field is missing" + ); +} + +#[test] +fn heartbeat_frames_carry_the_last_stream_id_received() { + let ack = encode_heartbeat_ack(Some(9)); + assert_eq!(ack, vec![1, 2, 0x10, 9]); + let ping = encode_heartbeat_ping(Some(3)); + assert_eq!(ping, vec![0, 2, 0x10, 3]); + assert_eq!(encode_heartbeat_ack(None), vec![1, 0]); + assert_eq!(encode_heartbeat_ping(None), vec![0, 0]); +} + +#[test] +fn checkin_request_and_response_follow_the_public_contract() { + let request = encode_checkin_request(&crate::push::protocol::CheckinRequest { + android_id: None, + security_token: None, + }); + assert_eq!(request[0], 0x22); + assert!(String::from_utf8_lossy(&request).contains("63.0.3234.0")); + let decoded = decode_checkin_response(&checkin_body(42, 7)).unwrap(); + assert_eq!(decoded.android_id, Some(42)); + assert_eq!(decoded.security_token, Some(7)); +} + +#[test] +fn data_message_decodes_app_data_and_payload() { + let mut stanza = Vec::new(); + write_string(3, "from", &mut stanza); + write_string(5, "category", &mut stanza); + write_app_data("crypto-key", "dh=abc", &mut stanza); + write_string(9, "persistent", &mut stanza); + write_bytes(21, &[1, 2, 3], &mut stanza); + let frame = server_frame(TAG_DATA_MESSAGE_STANZA, &stanza); + let mut reader = FrameReader::new(); + reader.extend(&[MCS_VERSION]); + reader.extend(&frame); + let (tag, payload) = reader.next_frame(MAXIMUM_FRAME_BYTES).unwrap().unwrap(); + match decode_frame(tag, &payload).unwrap() { + crate::push::protocol::Frame::DataMessage(message) => { + assert_eq!(message.from, "from"); + assert_eq!(message.app_data[0].0, "crypto-key"); + assert_eq!(message.persistent_id.as_deref(), Some("persistent")); + assert_eq!(message.raw_data.as_deref(), Some(&[1_u8, 2, 3][..])); + } + _ => panic!("expected a data message"), + } +} + +#[test] +fn registration_flow_uses_the_documented_steps_and_never_logs_tokens() { + let http = push_http(responses_for_registration()); + let registration = checkin_and_register( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + &identity(), + None, + "gfnjwt", + "device", + &mut || true, + ) + .unwrap(); + assert_eq!(registration.android_id, 42); + assert_eq!(registration.gcm_token, "gcm-token"); + assert_eq!(registration.fcm_token, "fcm-token"); + assert_eq!( + registration.endpoint, + "https://push.test/fcm/send/gcm-token" + ); + let requests = http.requests.lock().unwrap(); + assert_eq!(requests[0].0, "https://push.test/checkin"); + assert_eq!(requests[1].0, "https://push.test/c2dm"); + let c2dm = String::from_utf8_lossy(&requests[1].2).to_string(); + assert!(c2dm.contains("app=org.chromium.linux")); + assert!(c2dm.contains("X-subtype=app")); + assert!(c2dm.contains("device=42")); + assert!(c2dm.contains("sender=123456789012")); + assert_eq!( + requests[2].0, + "https://push.test/fis/v1/projects/project/installations" + ); + assert_eq!( + requests[3].0, + "https://push.test/fcm/v1/projects/project/registrations" + ); + assert_eq!(requests[4].0, "https://push.test/pns/v1/registrations"); + assert!(String::from_utf8_lossy(&requests[3].2).contains("fcm/send/gcm-token")); +} + +#[test] +fn registration_omits_the_sender_parameter_when_unconfigured() { + let http = push_http(responses_for_registration()); + let mut identity = identity(); + identity.sender_id = String::new(); + checkin_and_register( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + &identity, + None, + "gfnjwt", + "device", + &mut || true, + ) + .unwrap(); + let requests = http.requests.lock().unwrap(); + let c2dm = String::from_utf8_lossy(&requests[1].2); + assert!(!c2dm.contains("sender=")); + assert!(c2dm.contains("X-subtype=app")); +} + +#[test] +fn generated_installation_ids_follow_the_public_format() { + for _ in 0..32 { + let http = push_http(responses_for_registration()); + checkin_and_register( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + &identity(), + None, + "gfnjwt", + "device", + &mut || true, + ) + .unwrap(); + let requests = http.requests.lock().unwrap(); + let body: serde_json::Value = serde_json::from_slice(&requests[2].2).unwrap(); + let fid = body["fid"].as_str().unwrap(); + assert_eq!(fid.len(), 22, "fid {fid}"); + assert!(fid.starts_with(['c', 'd', 'e', 'f']), "fid {fid}"); + assert!( + fid.chars() + .all(|character| character.is_ascii_alphanumeric() + || character == '-' + || character == '_') + ); + } +} + +#[test] +fn pns_registration_validates_only_documented_statuses() { + let http = push_http(vec![HttpResponse { + status: 204, + body: Vec::new(), + }]); + pns_register( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + "token", + "device", + &[], + "jwt", + ) + .unwrap(); + let http = push_http(vec![HttpResponse { + status: 500, + body: Vec::new(), + }]); + assert_eq!( + pns_register( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + "token", + "device", + &[], + "jwt", + ) + .unwrap_err() + .code, + "push_pns_refused" + ); +} + +#[test] +fn pns_unregistration_validates_only_documented_statuses() { + let http = push_http(vec![HttpResponse { + status: 204, + body: Vec::new(), + }]); + crate::push::registration::pns_unregister( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + &["previous".to_owned()], + "jwt", + ) + .unwrap(); + let requests = http.requests.lock().unwrap(); + assert_eq!(requests[0].0, "https://push.test/pns/v1/unregister"); + let http = push_http(vec![HttpResponse { + status: 500, + body: Vec::new(), + }]); + assert_eq!( + crate::push::registration::pns_unregister( + &RegistrationClient { + http: http.as_ref(), + endpoints: &endpoints(), + timeout: Duration::from_secs(5), + }, + &["previous".to_owned()], + "jwt", + ) + .unwrap_err() + .code, + "push_pns_refused" + ); +} + +#[test] +fn production_edges_construct_without_reaching_the_network() { + crate::push::registration::ReqwestPushHttp::new().unwrap(); + let _factory = crate::push::transport::TlsPushTransportFactory::new(); + let _store = crate::push::store::RegistrationStore::default_service(); +} + +#[test] +fn owner_emits_coalesced_events_for_one_connection() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let frame_one = data_frame( + &keys, + &[ + ("messageType", "\"LIBRARY_CHANGE\"".into()), + ("changedIds", "[\"app-1\"]".into()), + ], + "p1", + ); + let frame_two = data_frame( + &keys, + &[ + ("messageType", "\"LIBRARY_CHANGE\"".into()), + ("changedIds", "[\"app-2\"]".into()), + ], + "p2", + ); + let connections = vec![vec![ + vec![MCS_VERSION], + login_response_frame(None), + frame_one, + frame_two, + ]]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let event = events_rx + .recv_timeout(Duration::from_secs(2)) + .expect("an event should arrive"); + match event { + PushEvent::LibraryChanged { changed_ids } => { + assert_eq!(changed_ids, vec!["app-1".to_owned(), "app-2".to_owned()]); + } + other => panic!("unexpected event {other:?}"), + } + assert!(events_rx.recv_timeout(Duration::from_millis(80)).is_err()); + shutdown(&mut harness); + assert_eq!(harness.factory.connects.load(Ordering::SeqCst), 1); + assert!( + harness + .store + .load(&scope().account_key()) + .unwrap() + .is_some() + ); +} + +#[test] +fn owner_accepts_aes128gcm_payloads_without_legacy_headers() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let frame = aes128gcm_data_frame( + &keys, + &[("messageType", "\"FAVORITES_CHANGE\"".into())], + "p1", + ); + let connections = vec![vec![vec![MCS_VERSION], login_response_frame(None), frame]]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let event = events_rx + .recv_timeout(Duration::from_secs(2)) + .expect("an aes128gcm event should arrive"); + assert!(matches!(event, PushEvent::FavoritesChanged { .. })); + shutdown(&mut harness); +} + +#[test] +fn owner_rejects_out_of_scope_events_and_stops() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let frame = data_frame( + &keys, + &[("messageType", "\"FAVORITES_CHANGE\"".into())], + "p1", + ); + let connections = vec![vec![vec![MCS_VERSION], login_response_frame(None), frame]]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + None, + Some(existing), + ); + harness.owner.start().unwrap(); + assert!(events_rx.recv_timeout(Duration::from_millis(120)).is_err()); + assert_eq!(harness.factory.connects.load(Ordering::SeqCst), 0); + shutdown(&mut harness); +} + +#[test] +fn owner_reconnects_after_a_failed_connection() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let frame = data_frame( + &keys, + &[("messageType", "\"SUBSCRIPTION_CHANGE\"".into())], + "p1", + ); + let connections = vec![vec![vec![MCS_VERSION], login_response_frame(None), frame]]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.factory.fail_first.store(1, Ordering::SeqCst); + harness.owner.start().unwrap(); + let event = events_rx + .recv_timeout(Duration::from_secs(3)) + .expect("an event should arrive after reconnecting"); + assert!(matches!(event, PushEvent::SubscriptionChanged { .. })); + shutdown(&mut harness); + assert!(harness.factory.connects.load(Ordering::SeqCst) >= 2); +} + +#[test] +fn owner_registers_when_no_registration_is_stored() { + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let http = push_http(responses_for_registration()); + let mut harness = owner_harness( + http.clone(), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + None, + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness + .store + .load(&scope().account_key()) + .unwrap() + .is_none() + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("registration should persist"); + assert_eq!(stored.fcm_token, "fcm-token"); + assert!(!stored.fingerprint.is_empty()); + assert_eq!(http.requests.lock().unwrap().len(), 5); + shutdown(&mut harness); +} + +#[test] +fn owner_replaces_an_expired_registration() { + let http = push_http(responses_for_registration()); + let mut expired = registration(); + expired.created_at_seconds = 1; + expired.fingerprint = fingerprint_for(&scope()); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + http.clone(), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + Some(expired), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(http.requests.lock().unwrap().len(), 5); + shutdown(&mut harness); +} + +#[test] +fn owner_never_reuses_another_account_registration() { + let http = push_http(responses_for_registration()); + let mut first = registration(); + first.created_at_seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + first.fingerprint = fingerprint_for(&scope()); + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + http.clone(), + vec![session_chunks(Vec::new()), session_chunks(Vec::new())], + sink, + Some(scope()), + Some(first), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(http.requests.lock().unwrap().len(), 0); + let second = PushScope { + user_id: "user-2".into(), + provider_id: "nvidia".into(), + generation: 8, + }; + *harness.desired.lock().unwrap() = Some(second.clone()); + let deadline = Instant::now() + Duration::from_secs(3); + while http.requests.lock().unwrap().len() < 5 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let requests = http.requests.lock().unwrap().len(); + assert_eq!( + requests, 5, + "the second account must register its own token" + ); + assert!(harness.store.load(&second.account_key()).unwrap().is_some()); + assert!(events_rx.recv_timeout(Duration::from_millis(50)).is_err()); + shutdown(&mut harness); +} + +#[test] +fn the_registration_fingerprint_covers_the_registration_identity() { + let base = PushOwnerConfig::bounded(endpoints(), identity(), "nvidia".into(), "device".into()); + let baseline = base.registration_fingerprint(&scope()); + let mut changes: Vec<(&str, PushOwnerConfig)> = Vec::new(); + let mut rotated = base.clone(); + rotated.identity.project_id = "other-project".into(); + changes.push(("project", rotated)); + let mut rotated = base.clone(); + rotated.identity.api_key = "other-key".into(); + changes.push(("api key", rotated)); + let mut rotated = base.clone(); + rotated.identity.sender_id = "999999999999".into(); + changes.push(("sender", rotated)); + let mut rotated = base.clone(); + rotated.identity.app_id = "other-app".into(); + changes.push(("application", rotated)); + let mut rotated = base.clone(); + rotated.identity.firebase_app_id = "1:2:web:other".into(); + changes.push(("firebase application", rotated)); + let mut rotated = base.clone(); + rotated.identity.vapid_key = Some("rotated-vapid".into()); + changes.push(("vapid key", rotated)); + let mut rotated = base.clone(); + rotated.endpoints.pns_client_id = "other-client".into(); + changes.push(("client id", rotated)); + let mut rotated = base.clone(); + rotated.device_id = "other-device".into(); + changes.push(("device", rotated)); + for (field, rotated) in changes { + assert_ne!( + baseline, + rotated.registration_fingerprint(&scope()), + "a changed {field} must invalidate the stored registration" + ); + } + assert_eq!( + baseline, + base.registration_fingerprint(&scope()), + "the fingerprint must stay stable for one configuration" + ); +} + +#[test] +fn owner_rapid_scope_changes_keep_one_worker_and_one_connection() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration(); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fingerprint = fingerprint_for(&scope()); + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let connections: Vec>> = (0..40).map(|_| session_chunks(Vec::new())).collect(); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + for _ in 0..40 { + *harness.desired.lock().unwrap() = None; + *harness.desired.lock().unwrap() = Some(scope()); + assert_eq!(harness.owner.worker_count(), 1); + } + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + shutdown(&mut harness); + assert_eq!(harness.factory.active.load(Ordering::SeqCst), 0); + assert!(harness.factory.maximum_active.load(Ordering::SeqCst) <= 1); +} + +#[test] +fn owner_cancels_a_blocked_connect_within_its_slice() { + let released = Arc::new(Mutex::new(false)); + let wake = Arc::new(Condvar::new()); + let connects = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let maximum_active = Arc::new(AtomicUsize::new(0)); + let desired = Arc::new(Mutex::new(Some(scope()))); + let source_scope = Arc::clone(&desired); + let mut owner = PushOwner::new( + PushOwnerConfig { + connect_timeout: Duration::from_millis(150), + read_timeout: Duration::from_millis(20), + login_timeout: Duration::from_secs(2), + heartbeat_interval: Duration::from_secs(30), + heartbeat_ack_timeout: Duration::from_millis(500), + coalescing_window: Duration::from_millis(30), + backoff_initial: Duration::from_millis(10), + backoff_maximum: Duration::from_millis(40), + stop_deadline: Duration::from_millis(300), + ..PushOwnerConfig::bounded(endpoints(), identity(), "nvidia".into(), "device".into()) + }, + PushOwnerDeps { + http: push_http(Vec::new()), + transport: Arc::new(BlockingFactory { + released: Mutex::new(false), + wake: Condvar::new(), + connects: Arc::clone(&connects), + connections: Mutex::new(vec![session_chunks(Vec::new())]), + sent: Arc::new(Mutex::new(Vec::new())), + active: Arc::clone(&active), + maximum_active: Arc::clone(&maximum_active), + }), + store: MemoryStore::new(Some(registration_with_current_age(&scope()))), + sink: Arc::new(|_: PushEvent, _: u64| {}), + scope_source: Arc::new(move || source_scope.lock().unwrap().clone()), + token_source: Arc::new(|_: &PushScope| Some("gfnjwt".to_owned())), + }, + ); + owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + connects.load(Ordering::SeqCst) >= 1, + "the connect must be attempted" + ); + assert!(owner.worker_count() <= 1); + *desired.lock().unwrap() = None; + *released.lock().unwrap() = true; + wake.notify_all(); + assert!(owner.shutdown(Duration::from_secs(3))); + assert_eq!(owner.worker_count(), 0); + assert_eq!(active.load(Ordering::SeqCst), 0); + assert!(maximum_active.load(Ordering::SeqCst) <= 1); +} +#[test] +fn changed_ids_stay_bounded() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let ids: Vec = (0..120).map(|index| format!("\"app-{index}\"")).collect(); + let frame = data_frame( + &keys, + &[ + ("messageType", "\"LIBRARY_CHANGE\"".into()), + ("changedIds", format!("[{}]", ids.join(","))), + ], + "p1", + ); + let connections = vec![vec![vec![MCS_VERSION], login_response_frame(None), frame]]; + let (events_tx, events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let event = events_rx + .recv_timeout(Duration::from_secs(2)) + .expect("an event should arrive"); + match event { + PushEvent::LibraryChanged { changed_ids } => { + assert_eq!(changed_ids.len(), MAXIMUM_CHANGED_IDS); + } + other => panic!("unexpected event {other:?}"), + } + shutdown(&mut harness); +} + +#[test] +fn encode_stream_ack_matches_the_documented_mcs_wire_shape() { + let mut extension = Vec::new(); + write_tag(1, 0, &mut extension); + write_varint(STREAM_ACK_EXTENSION_ID, &mut extension); + write_tag(2, 2, &mut extension); + write_varint(0, &mut extension); + let mut payload = Vec::new(); + write_tag(2, 0, &mut payload); + write_varint(1, &mut payload); + write_tag(3, 2, &mut payload); + write_varint(0, &mut payload); + write_tag(7, 2, &mut payload); + write_varint(extension.len() as u64, &mut payload); + payload.extend_from_slice(&extension); + write_tag(10, 0, &mut payload); + write_varint(6, &mut payload); + let mut expected = vec![TAG_IQ_STANZA]; + write_varint(payload.len() as u64, &mut expected); + expected.extend_from_slice(&payload); + assert_eq!(encode_stream_ack(Some(6)), expected); + let mut payload_without_stream_id = payload.clone(); + payload_without_stream_id.truncate(payload_without_stream_id.len() - 2); + let mut without_stream_id = vec![TAG_IQ_STANZA]; + write_varint( + payload_without_stream_id.len() as u64, + &mut without_stream_id, + ); + without_stream_id.extend_from_slice(&payload_without_stream_id); + assert_eq!(encode_stream_ack(None), without_stream_id); +} + +#[test] +fn the_session_advertises_every_received_stream_id_and_acknowledges_pings() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut chunks = session_chunks(Vec::new()); + chunks.push(server_frame(TAG_HEARTBEAT_ACK, &[])); + chunks.push(server_frame(5, &[])); + chunks.push(server_frame(TAG_IQ_STANZA, &[])); + chunks.push(data_frame( + &keys, + &[("messageType", "\"LIBRARY_CHANGE\"".into())], + "", + )); + chunks.push(server_frame(TAG_HEARTBEAT_PING, &[])); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![chunks], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.sent.lock().unwrap().len() < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let sent = harness.factory.sent.lock().unwrap().clone(); + let ack = sent + .iter() + .find(|frame| frame.first() == Some(&TAG_HEARTBEAT_ACK)) + .expect("the inbound ping must be acknowledged"); + assert_eq!( + ack, + &vec![TAG_HEARTBEAT_ACK, 2, 0x10, 6], + "the acknowledgement must advertise every received stream id" + ); + shutdown(&mut harness); +} + +#[test] +fn a_burst_of_persistent_messages_triggers_one_stream_ack() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut chunks = session_chunks(Vec::new()); + for index in 0..10 { + chunks.push(data_frame( + &keys, + &[("messageType", "\"LIBRARY_CHANGE\"".into())], + &format!("p{index}"), + )); + } + let mut harness = owner_harness( + push_http(Vec::new()), + vec![chunks], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.sent.lock().unwrap().len() < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let sent = harness.factory.sent.lock().unwrap().clone(); + let acks: Vec<&Vec> = sent + .iter() + .filter(|frame| frame.first() == Some(&TAG_IQ_STANZA)) + .collect(); + assert_eq!( + acks.len(), + 1, + "a burst of ten persistent messages must produce exactly one stream acknowledgement" + ); + shutdown(&mut harness); +} + +#[test] +fn an_immediate_ack_message_is_acknowledged_immediately() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let immediate = server_frame( + TAG_DATA_MESSAGE_STANZA, + &data_stanza( + &keys, + &[("messageType", "\"LIBRARY_CHANGE\"".into())], + "p-immediate", + true, + ), + ); + let mut chunks = session_chunks(Vec::new()); + chunks.push(immediate); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![chunks], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.sent.lock().unwrap().len() < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let sent = harness.factory.sent.lock().unwrap().clone(); + assert!( + sent.iter() + .any(|frame| frame.first() == Some(&TAG_IQ_STANZA)), + "an immediate-ack message must be acknowledged without waiting for the burst threshold" + ); + shutdown(&mut harness); +} + +#[test] +fn a_silent_connection_reconnects_without_replacing_the_registration() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + existing.fcm_token = "kept-fcm-token".into(); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new()), session_chunks(Vec::new())], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(4); + while harness.factory.connects.load(Ordering::SeqCst) < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + harness.factory.connects.load(Ordering::SeqCst) >= 2, + "a silent connection must be re-established after the heartbeat deadline" + ); + assert_eq!( + checkin_requests(&harness), + 0, + "a heartbeat timeout must not re-register the device identity" + ); + let stored = harness + .store + .load(&scope().account_key()) + .unwrap() + .expect("the registration must survive a heartbeat timeout"); + assert_eq!(stored.fcm_token, "kept-fcm-token"); + shutdown(&mut harness); +} + +#[test] +fn resumed_traffic_clears_the_heartbeat_deadline() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut chunks = session_chunks(Vec::new()); + chunks.push(server_frame(TAG_HEARTBEAT_PING, &[])); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![chunks], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.sent.lock().unwrap().len() < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(harness.factory.connects.load(Ordering::SeqCst), 1); + assert_eq!(checkin_requests(&harness), 0); + shutdown(&mut harness); +} + +#[test] +fn the_negotiated_heartbeat_interval_is_clamped_and_honored() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut chunks = vec![vec![MCS_VERSION], login_response_frame(Some(50))]; + chunks.push(server_frame(TAG_HEARTBEAT_PING, &[])); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![chunks], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while !harness + .factory + .sent + .lock() + .unwrap() + .iter() + .any(|frame| frame.first() == Some(&TAG_HEARTBEAT_PING)) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(5)); + } + let sent = harness.factory.sent.lock().unwrap().clone(); + assert!( + sent.iter() + .any(|frame| frame.first() == Some(&TAG_HEARTBEAT_PING)), + "the negotiated interval must still produce a keep-alive" + ); + let heartbeat = *harness.factory.heartbeat.lock().unwrap(); + let ping = heartbeat + .ping + .expect("the keep-alive must be observed with its preceding receive"); + let elapsed = ping.sent.saturating_duration_since(ping.last_received); + assert!( + elapsed >= Duration::from_millis(100), + "an out-of-range negotiated interval must be clamped to the configured bound: {elapsed:?}" + ); + shutdown(&mut harness); +} + +#[test] +fn a_missing_login_response_bounds_the_session() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![vec![vec![MCS_VERSION]], session_chunks(Vec::new())], + Arc::new(|_: PushEvent, _: u64| {}), + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(4); + while harness.factory.connects.load(Ordering::SeqCst) < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!( + harness.factory.connects.load(Ordering::SeqCst), + 2, + "a session without a login response must be bounded and retried" + ); + assert_eq!(checkin_requests(&harness), 0); + shutdown(&mut harness); +} + +#[test] +fn heartbeat_ping_triggers_an_acknowledgement() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let connections = vec![vec![ + vec![MCS_VERSION], + server_frame(TAG_HEARTBEAT_PING, &[]), + ]]; + let sent = Arc::new(Mutex::new(Vec::new())); + let active = Arc::new(AtomicUsize::new(0)); + let maximum_active = Arc::new(AtomicUsize::new(0)); + let factory = Arc::new(ScriptedFactory { + connections: Mutex::new(connections), + sent: Arc::clone(&sent), + connects: Arc::new(AtomicUsize::new(0)), + fail_first: AtomicUsize::new(0), + active, + maximum_active, + spacing_millis: Arc::new(AtomicUsize::new(0)), + heartbeat: Arc::new(Mutex::new(HeartbeatTiming::default())), + }); + let store = MemoryStore::new(Some(existing)); + let desired = Arc::new(Mutex::new(Some(scope()))); + let token_scope = Arc::clone(&desired); + let token_source = Arc::new(move |scope: &PushScope| { + let current = token_scope.lock().unwrap().clone()?; + (current.account_key() == scope.account_key()).then(|| "gfnjwt".to_owned()) + }); + let source_scope = Arc::clone(&desired); + let sink = Arc::new(|_: PushEvent, _: u64| {}); + let config = PushOwnerConfig { + connect_timeout: Duration::from_secs(1), + read_timeout: Duration::from_millis(20), + login_timeout: Duration::from_secs(2), + heartbeat_interval: Duration::from_secs(30), + heartbeat_ack_timeout: Duration::from_millis(500), + minimum_heartbeat_interval: Duration::from_millis(100), + maximum_heartbeat_interval: Duration::from_secs(2), + coalescing_window: Duration::from_millis(30), + backoff_initial: Duration::from_millis(10), + backoff_maximum: Duration::from_millis(40), + stop_deadline: Duration::from_secs(2), + ..PushOwnerConfig::bounded(endpoints(), identity(), "nvidia".into(), "device".into()) + }; + let mut owner = PushOwner::new( + config, + PushOwnerDeps { + http: push_http(Vec::new()), + transport: factory, + store, + sink, + scope_source: Arc::new(move || source_scope.lock().unwrap().clone()), + token_source, + }, + ); + owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while sent.lock().unwrap().len() < 2 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let frames = sent.lock().unwrap().clone(); + let ack = frames.last().expect("an acknowledgement should be sent"); + assert_eq!(ack, &vec![1, 2, 0x10, 1]); + *desired.lock().unwrap() = None; + let deadline = Instant::now() + Duration::from_secs(2); + while owner.is_running() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!(!owner.is_running()); + assert!(owner.shutdown(Duration::from_secs(2))); + assert_eq!(owner.worker_count(), 0); +} + +#[test] +fn message_types_map_only_to_the_supported_invalidation_events() { + let expected: Vec<(&str, bool)> = vec![ + ("FAVORITES_CHANGE", true), + ("LIBRARY_CHANGE", true), + ("LINKEDACCOUNT_CHANGE", true), + ("SUBSCRIPTION_CHANGE", true), + ("PLATFORM_SYNC_CHANGE", true), + ("SESSION_CHANGE", false), + ("CAMPAIGN_CHANGE", false), + ("KV_STORE_CHANGE", false), + ]; + let mut seen = 0; + for (name, supported) in expected { + let mapped = matches!( + crate::push::MessageType::parse(name), + Some( + crate::push::MessageType::LibraryChange + | crate::push::MessageType::FavoritesChange + | crate::push::MessageType::SubscriptionChange + | crate::push::MessageType::LinkedAccountChange + | crate::push::MessageType::PlatformSyncChange + ) + ); + assert_eq!(mapped, supported, "{name}"); + seen += 1; + } + assert_eq!(seen, 8); +} + +#[test] +fn stale_generation_cancels_the_worker_between_reconnects() { + let keys = EceKeyPair::generate().unwrap(); + let mut existing = registration_with_current_age(&scope()); + existing.private_key = keys.private_key(); + existing.public_key = keys.public_key().to_vec(); + existing.auth_secret = keys.auth_secret().to_vec(); + let connections = vec![session_chunks(Vec::new())]; + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + connections, + sink, + Some(scope()), + Some(existing), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + shutdown(&mut harness); +} + +#[test] +fn pausing_an_owner_keeps_one_worker_and_resumes_cleanly() { + let (events_tx, _events_rx) = mpsc::channel(); + let sink = Arc::new(move |event: PushEvent, _generation: u64| { + let _ = events_tx.send(event); + }); + let mut harness = owner_harness( + push_http(Vec::new()), + vec![session_chunks(Vec::new())], + sink, + Some(scope()), + Some(registration_with_current_age(&scope())), + ); + harness.owner.start().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.factory.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + harness.owner.stop(); + assert!(harness.owner.wait_idle(Duration::from_secs(2))); + assert_eq!( + harness.owner.worker_count(), + 1, + "pausing must keep the single worker" + ); + harness.owner.start().unwrap(); + assert_eq!( + harness.owner.worker_count(), + 1, + "resuming must never add a worker" + ); + assert!(harness.factory.maximum_active.load(Ordering::SeqCst) <= 1); + shutdown(&mut harness); +} + +fn registration_with_current_age(scope: &PushScope) -> Registration { + let mut registration = registration(); + registration.created_at_seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + registration.fingerprint = fingerprint_for(scope); + registration +} + +fn decode(value: &str) -> Vec { + URL_SAFE_NO_PAD + .decode(value.trim().replace(char::is_whitespace, "")) + .unwrap() +} diff --git a/external/opennow/native/opennow-core/src/push/transport.rs b/external/opennow/native/opennow-core/src/push/transport.rs new file mode 100644 index 000000000..fd1712e1b --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/transport.rs @@ -0,0 +1,320 @@ +use crate::push::PushError; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::sync::mpsc::{Receiver, TryRecvError}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +pub trait PushTransport: Send { + fn send(&mut self, frame: &[u8], timeout: Duration) -> Result<(), PushError>; + fn recv(&mut self, timeout: Duration) -> Result, PushError>; + fn close(&mut self); +} + +pub trait PushTransportFactory: Send + Sync { + fn connect( + &self, + host: &str, + port: u16, + timeout: Duration, + ) -> Result, PushError>; +} + +pub struct TlsPushTransport { + connection: rustls::ClientConnection, + socket: TcpStream, +} + +enum Resolution { + Addrs(Vec), + Failed, +} + +pub struct TlsPushTransportFactory { + pending: Mutex)>>, +} + +impl Default for TlsPushTransportFactory { + fn default() -> Self { + Self::new() + } +} + +impl TlsPushTransportFactory { + pub fn new() -> Self { + Self { + pending: Mutex::new(None), + } + } + + fn resolve( + &self, + host: &str, + port: u16, + timeout: Duration, + ) -> Result, PushError> { + let mut pending = self.pending.lock().expect("push resolver poisoned"); + if let Some((pending_host, pending_port, receiver)) = pending.take() { + match receiver.try_recv() { + Ok(Resolution::Addrs(addrs)) => { + if pending_host == host && pending_port == port { + return Ok(addrs); + } + } + Ok(Resolution::Failed) | Err(TryRecvError::Disconnected) => { + if pending_host == host && pending_port == port { + return Err(PushError::new( + "push_transport_failed", + "The push service could not be resolved", + )); + } + } + Err(TryRecvError::Empty) => { + *pending = Some((pending_host, pending_port, receiver)); + return Err(PushError::new( + "push_transport_resolving", + "A previous push service resolution is still in progress", + )); + } + } + } + let (sender, receiver) = std::sync::mpsc::channel(); + let owned_host = host.to_owned(); + std::thread::Builder::new() + .name("opennow-push-resolver".to_owned()) + .spawn(move || { + let resolved = (owned_host.as_str(), port) + .to_socket_addrs() + .map(|addrs| Resolution::Addrs(addrs.collect())) + .unwrap_or(Resolution::Failed); + let _ = sender.send(resolved); + }) + .map_err(|_| { + PushError::new( + "push_transport_failed", + "The push service resolver could not start", + ) + })?; + match receiver.recv_timeout(timeout) { + Ok(Resolution::Addrs(addrs)) => { + if addrs.is_empty() { + return Err(PushError::new( + "push_transport_failed", + "The push service could not be resolved", + )); + } + Ok(addrs) + } + Ok(Resolution::Failed) => Err(PushError::new( + "push_transport_failed", + "The push service could not be resolved", + )), + Err(_) => { + *pending = Some((host.to_owned(), port, receiver)); + Err(PushError::new( + "push_transport_failed", + "The push service resolution timed out", + )) + } + } + } +} + +fn tls_config() -> Arc { + static CONFIG: OnceLock> = OnceLock::new(); + Arc::clone(CONFIG.get_or_init(|| { + let mut roots = rustls::RootCertStore::empty(); + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + Arc::new( + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ) + })) +} + +fn is_timeout(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) +} + +fn apply_timeout(deadline: Instant) -> Option { + let remaining = deadline.saturating_duration_since(Instant::now()); + (!remaining.is_zero()).then_some(remaining) +} + +impl TlsPushTransport { + fn open(host: &str, address: SocketAddr, deadline: Instant) -> Result { + let server_name = + rustls::pki_types::ServerName::try_from(host.to_owned()).map_err(|_| { + PushError::new("push_transport_invalid", "The push service host is invalid") + })?; + let connection = + rustls::ClientConnection::new(tls_config(), server_name).map_err(|_| { + PushError::new( + "push_transport_failed", + "The push connection could not start", + ) + })?; + let Some(remaining) = apply_timeout(deadline) else { + return Err(PushError::new( + "push_transport_failed", + "The push service connection timed out", + )); + }; + let socket = TcpStream::connect_timeout(&address, remaining).map_err(|_| { + PushError::new("push_transport_failed", "The push service is unreachable") + })?; + let _ = socket.set_nodelay(true); + Ok(Self { connection, socket }) + } + + fn finish_handshake(&mut self, deadline: Instant) -> Result<(), PushError> { + while self.connection.is_handshaking() { + let Some(remaining) = apply_timeout(deadline) else { + return Err(PushError::new( + "push_transport_failed", + "The push service handshake timed out", + )); + }; + let _ = self.socket.set_read_timeout(Some(remaining)); + let _ = self.socket.set_write_timeout(Some(remaining)); + match self.connection.complete_io(&mut self.socket) { + Ok(_) => {} + Err(error) if is_timeout(&error) => continue, + Err(_) => { + return Err(PushError::new( + "push_transport_failed", + "The push service handshake failed", + )); + } + } + } + Ok(()) + } + + fn drain(&mut self, buffer: &mut [u8]) -> Result { + self.connection + .reader() + .read(buffer) + .map_err(|_| PushError::new("push_transport_failed", "The push connection failed")) + } +} + +impl PushTransport for TlsPushTransport { + fn send(&mut self, frame: &[u8], timeout: Duration) -> Result<(), PushError> { + let deadline = Instant::now() + timeout; + self.finish_handshake(deadline)?; + self.connection + .writer() + .write_all(frame) + .map_err(|_| PushError::new("push_transport_failed", "The push connection failed"))?; + while self.connection.wants_write() { + let Some(remaining) = apply_timeout(deadline) else { + return Err(PushError::new( + "push_transport_failed", + "The push message could not be sent in time", + )); + }; + let _ = self.socket.set_write_timeout(Some(remaining)); + match self.connection.write_tls(&mut self.socket) { + Ok(0) => { + return Err(PushError::new( + "push_transport_closed", + "The push connection ended", + )); + } + Ok(_) => {} + Err(error) if is_timeout(&error) => continue, + Err(_) => { + return Err(PushError::new( + "push_transport_failed", + "The push connection failed", + )); + } + } + } + Ok(()) + } + + fn recv(&mut self, timeout: Duration) -> Result, PushError> { + let deadline = Instant::now() + timeout; + let mut buffer = [0_u8; 16 * 1024]; + loop { + match self.drain(&mut buffer) { + Ok(0) => {} + Ok(count) => return Ok(buffer[..count].to_vec()), + Err(error) => return Err(error), + } + let Some(remaining) = apply_timeout(deadline) else { + return Ok(Vec::new()); + }; + let _ = self.socket.set_read_timeout(Some(remaining)); + match self.connection.read_tls(&mut self.socket) { + Ok(0) => { + return Err(PushError::new( + "push_transport_closed", + "The push connection ended", + )); + } + Ok(_) => { + self.connection.process_new_packets().map_err(|_| { + PushError::new("push_transport_failed", "The push connection failed") + })?; + } + Err(error) if is_timeout(&error) => {} + Err(_) => { + return Err(PushError::new( + "push_transport_failed", + "The push connection failed", + )); + } + } + } + } + + fn close(&mut self) { + self.connection.send_close_notify(); + let deadline = Instant::now() + Duration::from_millis(500); + while self.connection.wants_write() { + let Some(remaining) = apply_timeout(deadline) else { + break; + }; + let _ = self.socket.set_write_timeout(Some(remaining)); + if self.connection.write_tls(&mut self.socket).is_err() { + break; + } + } + let _ = self.socket.shutdown(std::net::Shutdown::Both); + } +} + +impl PushTransportFactory for TlsPushTransportFactory { + fn connect( + &self, + host: &str, + port: u16, + timeout: Duration, + ) -> Result, PushError> { + let deadline = Instant::now() + timeout; + let addresses = self.resolve(host, port, timeout)?; + let mut last_error = None; + for address in addresses.into_iter().take(8) { + match TlsPushTransport::open(host, address, deadline) { + Ok(mut transport) => { + transport.finish_handshake(deadline)?; + return Ok(Box::new(transport)); + } + Err(error) => last_error = Some(error), + } + if apply_timeout(deadline).is_none() { + break; + } + } + Err(last_error.unwrap_or_else(|| { + PushError::new("push_transport_failed", "The push service is unreachable") + })) + } +} diff --git a/external/opennow/native/opennow-core/src/push/vendor_default.rs b/external/opennow/native/opennow-core/src/push/vendor_default.rs new file mode 100644 index 000000000..c8a1f1d57 --- /dev/null +++ b/external/opennow/native/opennow-core/src/push/vendor_default.rs @@ -0,0 +1,103 @@ +use crate::push::owner::PushOwnerConfig; +use crate::push::registration::{PushEndpoints, PushIdentity}; + +const PROJECT_ID: &str = "nv-pushnotificationss-20220823"; +const API_KEY: &str = "AIzaSyCyGIQv16BXoVfMOSXesGclbZnC6krMBmU"; +const SENDER_ID: &str = "950535643107"; +const APP_ID: &str = "nv-pushnotificationss-20220823"; +const FIREBASE_APP_ID: &str = "1:950535643107:web:a2e08529da36c9456e0ad6"; +const VAPID_KEY: &str = + "BKXL_5beeXnhyvgZV-1BzEB6DH-bYL9gsql510ilbB8gYBxJavlpdHPF6VtjyrmowsskhD3qn72uvIMhiffB6UY"; +const PNS_SERVER: &str = "https://pns.geforcenow.com"; +const PNS_VERSION: &str = "v1"; +const PNS_CLIENT_ID: &str = "ec7e38d4-03af-4b58-b131-cfb0495903ab"; + +pub fn config_for_provider(provider_idp_id: &str, device_id: String) -> PushOwnerConfig { + PushOwnerConfig::bounded( + PushEndpoints { + pns: format!( + "{}/{}", + PNS_SERVER.trim_end_matches('/'), + PNS_VERSION.trim_matches('/') + ), + pns_client_id: PNS_CLIENT_ID.to_owned(), + ..PushEndpoints::default() + }, + PushIdentity { + project_id: PROJECT_ID.to_owned(), + api_key: API_KEY.to_owned(), + sender_id: SENDER_ID.to_owned(), + app_id: APP_ID.to_owned(), + firebase_app_id: FIREBASE_APP_ID.to_owned(), + vapid_key: Some(VAPID_KEY.to_owned()), + }, + provider_idp_id.to_owned(), + device_id, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_bundled_default_is_complete_and_public_only() { + let config = config_for_provider("nvidia", "device".into()); + assert_eq!(config.provider_id, "nvidia"); + assert_eq!(config.device_id, "device"); + assert_eq!(config.identity.project_id, PROJECT_ID); + assert!(!config.identity.api_key.is_empty()); + assert!(!config.identity.sender_id.is_empty()); + assert!(!config.identity.app_id.is_empty()); + assert!(!config.identity.firebase_app_id.is_empty()); + assert!( + config + .identity + .vapid_key + .as_deref() + .is_some_and(|key| !key.is_empty()) + ); + assert!(!config.endpoints.pns.is_empty()); + assert!(!config.endpoints.pns.ends_with('/')); + assert!(!config.endpoints.pns_client_id.is_empty()); + assert_eq!( + config.endpoints.pns, + format!( + "{}/{}", + PNS_SERVER.trim_end_matches('/'), + PNS_VERSION.trim_matches('/') + ) + ); + assert_eq!(config.endpoints.mcs_host, PushEndpoints::default().mcs_host); + } + + #[test] + fn the_bundled_default_is_bound_to_the_authenticated_provider() { + assert_eq!( + config_for_provider("nvidia", "device".into()).provider_id, + "nvidia" + ); + assert_eq!( + config_for_provider("alliance", "device".into()).provider_id, + "alliance" + ); + } + + #[test] + fn the_bundled_default_uses_a_usable_https_pns_base() { + let config = config_for_provider("nvidia", "device".into()); + assert!( + crate::push::config::pns_base_is_usable(&config.endpoints.pns), + "the bundled default must keep the evidenced https pns base" + ); + } + + #[test] + fn the_bundled_default_carries_no_device_or_user_secret() { + let first = config_for_provider("nvidia", "device-a".into()); + let second = config_for_provider("nvidia", "device-b".into()); + assert_eq!(first.identity, second.identity); + assert_eq!(first.endpoints, second.endpoints); + assert_ne!(first, second); + } +} diff --git a/external/opennow/native/opennow-core/src/push_registry.rs b/external/opennow/native/opennow-core/src/push_registry.rs new file mode 100644 index 000000000..c3bcbf2ac --- /dev/null +++ b/external/opennow/native/opennow-core/src/push_registry.rs @@ -0,0 +1,943 @@ +use crate::gfn::GfnService; +use opennow_core::push::{ + GfnTokenSource, PUSH_EVENT_NAME, PushConfig, PushError, PushOwner, PushOwnerConfig, + PushOwnerDeps, PushScope, PushSink, RegistrationStore, ReqwestPushHttp, ScopeSource, + TlsPushTransportFactory, bundled_push_config, push_config_for_provider, push_event_payload, +}; +use serde_json::Value; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::mpsc::Sender; +use std::time::Duration; + +pub struct DesiredPush { + pub config: PushOwnerConfig, +} + +pub fn desired_push(data_dir: &Path, scope: &PushScope, device_id: &str) -> Option { + let config = match push_config_for_provider(data_dir, &scope.provider_id, device_id.to_owned()) + { + PushConfig::Absent => bundled_push_config(&scope.provider_id, device_id.to_owned()), + PushConfig::Configured(config) => *config, + PushConfig::Suppressed => return None, + }; + if config.endpoints.pns.is_empty() || config.endpoints.pns_client_id.is_empty() { + return None; + } + Some(DesiredPush { config }) +} + +pub type DepsFactory = dyn Fn(Arc, ScopeSource, Sender) -> Result + + Send + + Sync; + +pub struct PushRegistry { + gfn: Arc, + output: Sender, + data_dir: PathBuf, + scope_source: ScopeSource, + owner: Option, + deps_factory: Arc, +} + +impl PushRegistry { + pub fn new(gfn: Arc, output: Sender, data_dir: PathBuf) -> Self { + let scope_source: ScopeSource = { + let gfn = Arc::clone(&gfn); + Arc::new(move || gfn.push_scope()) + }; + Self::with_deps_factory( + gfn, + output, + data_dir, + scope_source, + Arc::new(production_deps), + ) + } + + pub fn with_deps_factory( + gfn: Arc, + output: Sender, + data_dir: PathBuf, + scope_source: ScopeSource, + deps_factory: Arc, + ) -> Self { + Self { + gfn, + output, + data_dir, + scope_source, + owner: None, + deps_factory, + } + } + + pub fn reconcile(&mut self) -> Result<(), PushError> { + let scope = (self.scope_source)(); + let desired = scope + .as_ref() + .and_then(|scope| desired_push(&self.data_dir, scope, self.gfn.device_id())); + if let Some(owner) = self.owner.as_mut() { + let Some(desired) = desired else { + owner.stop(); + return Ok(()); + }; + if owner.config().0 != desired.config { + owner.update_config(desired.config); + } + return owner.start(); + } + let Some(desired) = desired else { + return Ok(()); + }; + let deps = (self.deps_factory)( + Arc::clone(&self.gfn), + Arc::clone(&self.scope_source), + self.output.clone(), + )?; + let mut owner = PushOwner::new(desired.config, deps); + owner.start()?; + self.owner = Some(owner); + Ok(()) + } + + pub fn stop(&mut self) { + if let Some(mut owner) = self.owner.take() { + owner.shutdown(Duration::from_secs(5)); + } + } +} + +pub fn production_deps( + gfn: Arc, + scope_source: ScopeSource, + output: Sender, +) -> Result { + let token_source: GfnTokenSource = { + let gfn = Arc::clone(&gfn); + Arc::new(move |scope: &PushScope| gfn.push_token_for_scope(scope)) + }; + let sink: PushSink = Arc::new(move |event, generation| { + let _ = output.send(serde_json::json!({ + "type": "event", + "name": PUSH_EVENT_NAME, + "payload": push_event_payload(&event, generation), + })); + }); + Ok(PushOwnerDeps { + http: Arc::new(ReqwestPushHttp::new()?), + transport: Arc::new(TlsPushTransportFactory::new()), + store: Arc::new(RegistrationStore::default_service()), + sink, + scope_source, + token_source, + }) +} + +impl Drop for PushRegistry { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opennow_core::push::{ + HttpRequest, HttpResponse, PushEvent, PushHttp, PushScope, PushStateStore, PushTransport, + PushTransportFactory, Registration, + }; + use serde_json::json; + use std::collections::BTreeMap; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Condvar, Mutex}; + use std::time::{Duration, Instant}; + + const HOLD_DEADLINE: Duration = Duration::from_secs(30); + + fn scope() -> PushScope { + PushScope { + user_id: "user-1".into(), + provider_id: "nvidia".into(), + generation: 3, + } + } + + fn config_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("opennow-push-registry-{name}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + raw_config().to_string(), + ) + .unwrap(); + dir + } + + fn empty_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("opennow-push-registry-{name}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn raw_config() -> Value { + json!({ + "providerIdpId": "nvidia", + "projectId": "project", + "apiKey": "api-key", + "senderId": "123456789012", + "appId": "app", + "firebaseAppId": "1:1:web:abc", + "pnsServer": "https://pns.example", + "pnsClientId": "client", + }) + } + + fn registration_body() -> Vec { + let mut body = vec![0x39]; + body.extend_from_slice(&42_u64.to_le_bytes()); + body.push(0x41); + body.extend_from_slice(&7_u64.to_le_bytes()); + body + } + + fn registration_responses() -> Vec { + vec![ + HttpResponse { + status: 200, + body: registration_body(), + }, + HttpResponse { + status: 200, + body: b"token=gcm-token".to_vec(), + }, + HttpResponse { + status: 200, + body: br#"{"authToken":{"token":"fis-token"}}"#.to_vec(), + }, + HttpResponse { + status: 200, + body: br#"{"token":"fcm-token"}"#.to_vec(), + }, + HttpResponse { + status: 204, + body: Vec::new(), + }, + ] + } + + struct FakeHttp { + responses: Mutex>, + calls: Arc, + release: Option>, + } + + struct Blocking { + released: Mutex, + wake: Condvar, + abandoned: AtomicBool, + } + + impl Blocking { + fn new() -> Arc { + Arc::new(Self { + released: Mutex::new(false), + wake: Condvar::new(), + abandoned: AtomicBool::new(false), + }) + } + + fn release(&self) { + *self.released.lock().unwrap() = true; + self.wake.notify_all(); + } + + fn abandoned(&self) -> bool { + self.abandoned.load(Ordering::SeqCst) + } + + fn wait(&self) { + let mut released = self.released.lock().unwrap(); + let deadline = Instant::now() + HOLD_DEADLINE; + loop { + if *released { + return; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + drop(released); + self.abandoned.store(true, Ordering::SeqCst); + panic!("the blocked fixture hold was not released within {HOLD_DEADLINE:?}"); + } + let (state, _) = self.wake.wait_timeout(released, remaining).unwrap(); + released = state; + } + } + + fn wait_bounded(&self, timeout: Duration) { + let mut released = self.released.lock().unwrap(); + let deadline = Instant::now() + timeout; + while !*released { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return; + } + let (state, _) = self.wake.wait_timeout(released, remaining).unwrap(); + released = state; + } + } + } + + impl PushHttp for FakeHttp { + fn execute(&self, _request: &HttpRequest) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + if let Some(release) = &self.release { + release.wait(); + } + let mut responses = self.responses.lock().unwrap(); + if responses.is_empty() { + return Err(PushError::new("push_http_failed", "No scripted response")); + } + Ok(responses.remove(0)) + } + } + + struct MemoryStore(Mutex>); + + impl MemoryStore { + fn empty() -> Arc { + Arc::new(Self(Mutex::new(BTreeMap::new()))) + } + } + + impl PushStateStore for MemoryStore { + fn load(&self, account: &str) -> Result, PushError> { + Ok(self.0.lock().unwrap().get(account).cloned()) + } + + fn save(&self, account: &str, registration: &Registration) -> Result<(), PushError> { + self.0 + .lock() + .unwrap() + .insert(account.to_owned(), registration.clone()); + Ok(()) + } + + fn clear(&self, account: &str) -> Result<(), PushError> { + self.0.lock().unwrap().remove(account); + Ok(()) + } + } + + struct BlockingTransport { + active: Arc, + } + + impl PushTransport for BlockingTransport { + fn send(&mut self, _frame: &[u8], _timeout: Duration) -> Result<(), PushError> { + Ok(()) + } + + fn recv(&mut self, _timeout: Duration) -> Result, PushError> { + std::thread::sleep(Duration::from_millis(5)); + Ok(Vec::new()) + } + + fn close(&mut self) {} + } + + impl Drop for BlockingTransport { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::SeqCst); + } + } + + struct BlockingFactory { + connection: Arc, + connects: Arc, + active: Arc, + maximum: Arc, + } + + impl PushTransportFactory for BlockingFactory { + fn connect( + &self, + _host: &str, + _port: u16, + _timeout: Duration, + ) -> Result, PushError> { + self.connects.fetch_add(1, Ordering::SeqCst); + self.connection.wait_bounded(Duration::from_millis(20)); + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.maximum.fetch_max(active, Ordering::SeqCst); + Ok(Box::new(BlockingTransport { + active: Arc::clone(&self.active), + })) + } + } + + struct Harness { + registry: PushRegistry, + scope: Arc>>, + tokens: Arc>>, + connect: Arc, + connects: Arc, + active: Arc, + maximum: Arc, + factories: Arc, + http_calls: Arc, + } + + impl Drop for Harness { + fn drop(&mut self) { + self.connect.release(); + if !std::thread::panicking() { + assert!( + !self.connect.abandoned(), + "the blocked fixture hold must be released before its safety deadline" + ); + } + } + } + + fn workers(harness: &Harness) -> usize { + harness + .registry + .owner + .as_ref() + .map_or(0, PushOwner::worker_count) + } + + fn harness(data_dir: PathBuf, script_registration: bool) -> Harness { + harness_with(data_dir, script_registration, false) + } + + fn harness_with( + data_dir: PathBuf, + script_registration: bool, + hold_first_stage: bool, + ) -> Harness { + let (output, _rx) = std::sync::mpsc::channel(); + let gfn = Arc::new(GfnService::new(std::env::temp_dir()).unwrap()); + let scope_state = Arc::new(Mutex::new(Some(scope()))); + let registry_scope: ScopeSource = { + let scope_state = Arc::clone(&scope_state); + Arc::new(move || scope_state.lock().unwrap().clone()) + }; + let tokens = Arc::new(Mutex::new(Vec::new())); + let connect = Blocking::new(); + let connects = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let maximum = Arc::new(AtomicUsize::new(0)); + let factories = Arc::new(AtomicUsize::new(0)); + let http_calls = Arc::new(AtomicUsize::new(0)); + let holds = Arc::new(AtomicBool::new(hold_first_stage)); + let scripted = Arc::new(AtomicBool::new(script_registration)); + let deps_scope = Arc::clone(&scope_state); + let deps_tokens = Arc::clone(&tokens); + let deps_connect = Arc::clone(&connect); + let deps_connects = Arc::clone(&connects); + let deps_active = Arc::clone(&active); + let deps_maximum = Arc::clone(&maximum); + let deps_factories = Arc::clone(&factories); + let deps_http_calls = Arc::clone(&http_calls); + let deps_holds = Arc::clone(&holds); + let deps_scripted = Arc::clone(&scripted); + let deps_factory: Arc = Arc::new(move |_gfn, _scope_source, _output| { + deps_factories.fetch_add(1, Ordering::SeqCst); + let scope_source: ScopeSource = { + let scope_state = Arc::clone(&deps_scope); + Arc::new(move || scope_state.lock().unwrap().clone()) + }; + let token_source: GfnTokenSource = { + let scope_state = Arc::clone(&deps_scope); + let tokens = Arc::clone(&deps_tokens); + Arc::new(move |scope: &PushScope| { + let current = scope_state.lock().unwrap().clone()?; + if current.user_id != scope.user_id + || current.provider_id != scope.provider_id + || current.generation != scope.generation + { + return None; + } + tokens.lock().unwrap().push(scope.clone()); + Some("gfnjwt".to_owned()) + }) + }; + let release = deps_holds + .load(Ordering::SeqCst) + .then(|| Arc::clone(&deps_connect)); + let http: Arc = Arc::new(FakeHttp { + responses: Mutex::new(if deps_scripted.load(Ordering::SeqCst) { + registration_responses() + } else { + Vec::new() + }), + calls: Arc::clone(&deps_http_calls), + release, + }); + Ok(PushOwnerDeps { + http, + transport: Arc::new(BlockingFactory { + connection: Arc::clone(&deps_connect), + connects: Arc::clone(&deps_connects), + active: Arc::clone(&deps_active), + maximum: Arc::clone(&deps_maximum), + }), + store: MemoryStore::empty(), + sink: Arc::new(|_: PushEvent, _: u64| {}), + scope_source, + token_source, + }) + }); + let registry = + PushRegistry::with_deps_factory(gfn, output, data_dir, registry_scope, deps_factory); + Harness { + registry, + scope: scope_state, + tokens, + connect, + connects, + active, + maximum, + factories, + http_calls, + } + } + + #[test] + fn desired_push_requires_a_matching_provider_and_service_configuration() { + let dir = config_dir("provider"); + assert!(desired_push(&dir, &scope(), "device").is_some()); + let mut other_provider = scope(); + other_provider.provider_id = "alliance".into(); + assert!(desired_push(&dir, &other_provider, "device").is_none()); + let missing_dir = config_dir("missing-client"); + let mut missing_client = raw_config(); + missing_client["pnsClientId"] = json!(""); + std::fs::write( + missing_dir.join(opennow_core::push::PUSH_CONFIG_FILE), + missing_client.to_string(), + ) + .unwrap(); + assert!(desired_push(&missing_dir, &scope(), "device").is_none()); + } + + #[test] + fn desired_push_uses_the_bundled_default_without_an_override() { + let dir = empty_dir("bundled-default"); + let desired = desired_push(&dir, &scope(), "device").expect("the bundled default applies"); + assert_eq!(desired.config.provider_id, "nvidia"); + assert_eq!(desired.config.device_id, "device"); + assert!(!desired.config.endpoints.pns.is_empty()); + assert!(!desired.config.endpoints.pns_client_id.is_empty()); + assert!(!desired.config.identity.api_key.is_empty()); + assert!( + dir.read_dir().unwrap().next().is_none(), + "the bundled default must never write to the data directory" + ); + let mut other_provider = scope(); + other_provider.provider_id = "alliance".into(); + let switched = desired_push(&dir, &other_provider, "device") + .expect("the bundled default follows the authenticated provider"); + assert_eq!(switched.config.provider_id, "alliance"); + assert_eq!( + desired.config.identity, switched.config.identity, + "the bundled public identity is provider independent" + ); + } + + #[test] + fn desired_push_config_changes_with_device_and_configuration() { + let dir = config_dir("key"); + let first = desired_push(&dir, &scope(), "device").unwrap(); + assert_eq!( + first.config, + desired_push(&dir, &scope(), "device").unwrap().config + ); + let other_device = desired_push(&dir, &scope(), "other").unwrap(); + assert_ne!(first.config, other_device.config); + let mut changed = raw_config(); + changed["pnsServer"] = json!("https://other.example"); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + changed.to_string(), + ) + .unwrap(); + assert_ne!( + first.config, + desired_push(&dir, &scope(), "device").unwrap().config + ); + let mut rotated = raw_config(); + rotated["apiKey"] = json!("rotated-api-key"); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + rotated.to_string(), + ) + .unwrap(); + assert_ne!( + first.config, + desired_push(&dir, &scope(), "device").unwrap().config, + "a rotated identity key must change the desired configuration" + ); + } + + #[test] + fn registry_propagates_identity_changes_across_the_same_owner() { + let dir = config_dir("identity-churn"); + let mut harness = harness(dir.clone(), true); + harness.registry.reconcile().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.tokens.lock().unwrap().is_empty() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let mut rotated = raw_config(); + rotated["apiKey"] = json!("rotated-api-key"); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + rotated.to_string(), + ) + .unwrap(); + harness.registry.reconcile().unwrap(); + let owner_config = harness.registry.owner.as_ref().unwrap().config().0; + assert_eq!( + owner_config.identity.api_key, "rotated-api-key", + "a changed identity must reach the live owner" + ); + assert_eq!( + harness.factories.load(Ordering::SeqCst), + 1, + "the configuration change must reuse the single owner" + ); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn registry_creates_no_owner_without_a_signed_in_scope() { + let mut harness = harness(config_dir("no-scope"), true); + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + harness.registry.reconcile().unwrap(); + assert_eq!(harness.factories.load(Ordering::SeqCst), 0); + assert!(harness.registry.owner.is_none()); + harness.registry.stop(); + } + + #[test] + fn registry_stays_inactive_without_an_authenticated_scope() { + let mut harness = harness(empty_dir("no-auth"), true); + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + assert!( + harness.registry.owner.is_none(), + "the bundled default must not activate while signed out" + ); + assert_eq!(harness.factories.load(Ordering::SeqCst), 0); + *harness.scope.lock().unwrap() = Some(scope()); + harness.registry.reconcile().unwrap(); + assert!( + harness.registry.owner.is_some(), + "signing in must activate the bundled default" + ); + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + assert!(!harness.registry.owner.as_ref().unwrap().is_running()); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn registry_activates_the_bundled_default_without_an_override_file() { + let mut harness = harness(empty_dir("bundled-activation"), true); + harness.registry.reconcile().unwrap(); + let config = harness + .registry + .owner + .as_ref() + .expect("the bundled default must activate the owner") + .config() + .0; + assert_eq!(config.provider_id, "nvidia"); + assert!(!config.identity.api_key.is_empty()); + assert!(!config.endpoints.pns.is_empty()); + assert_eq!(harness.factories.load(Ordering::SeqCst), 1); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn registry_prefers_an_explicit_override_over_the_bundled_default() { + let mut harness = harness(config_dir("override-wins"), true); + harness.registry.reconcile().unwrap(); + let config = harness + .registry + .owner + .as_ref() + .expect("the override must activate the owner") + .config() + .0; + assert_eq!(config.identity.api_key, "api-key"); + assert_eq!(config.endpoints.pns, "https://pns.example/v1"); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn desired_push_rejects_an_insecure_override_base() { + let dir = empty_dir("insecure-base"); + let mut config = raw_config(); + config["pnsServer"] = json!("http://pns.example"); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + config.to_string(), + ) + .unwrap(); + assert!( + desired_push(&dir, &scope(), "device").is_none(), + "an insecure PNS override must not activate a subscription" + ); + } + + #[cfg(unix)] + #[test] + fn desired_push_never_activates_a_dangling_override_link() { + let dir = empty_dir("dangling-link"); + std::os::unix::fs::symlink( + dir.join("missing.json"), + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + ) + .unwrap(); + assert!( + desired_push(&dir, &scope(), "device").is_none(), + "a dangling override link must suppress the bundled default" + ); + } + + #[cfg(unix)] + #[test] + fn desired_push_returns_without_a_writer_on_a_non_regular_override() { + use std::os::unix::ffi::OsStrExt as _; + let dir = empty_dir("fifo"); + let path = dir.join(opennow_core::push::PUSH_CONFIG_FILE); + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(desired_push(&dir, &scope(), "device").is_none()); + }); + assert!( + receiver + .recv_timeout(Duration::from_secs(2)) + .expect("a non-regular override entry must not block reconciliation"), + "a non-regular override must suppress the bundled default without a writer" + ); + } + + #[test] + fn registry_respects_an_explicitly_disabled_override() { + let dir = empty_dir("disabled"); + std::fs::write( + dir.join(opennow_core::push::PUSH_CONFIG_FILE), + json!({"enabled": false}).to_string(), + ) + .unwrap(); + let mut harness = harness(dir, true); + harness.registry.reconcile().unwrap(); + assert!( + harness.registry.owner.is_none(), + "an explicit disable must not fall back to the bundled default" + ); + assert_eq!(harness.factories.load(Ordering::SeqCst), 0); + harness.registry.stop(); + } + + #[test] + fn registry_binds_the_bundled_default_to_the_authenticated_provider() { + let mut harness = harness(empty_dir("provider-change"), true); + harness.registry.reconcile().unwrap(); + assert_eq!( + harness + .registry + .owner + .as_ref() + .unwrap() + .config() + .0 + .provider_id, + "nvidia" + ); + let mut switched = scope(); + switched.provider_id = "alliance".into(); + switched.generation = 4; + *harness.scope.lock().unwrap() = Some(switched); + harness.registry.reconcile().unwrap(); + let config = harness.registry.owner.as_ref().unwrap().config().0; + assert_eq!(config.provider_id, "alliance"); + assert_eq!( + harness.factories.load(Ordering::SeqCst), + 1, + "a provider change must reuse the single owner" + ); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn registry_reuses_one_owner_across_logout_and_login_cycles() { + let mut harness = harness(config_dir("cycles"), true); + harness.registry.reconcile().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + harness.connects.load(Ordering::SeqCst) >= 1, + "the worker must connect" + ); + assert_eq!(harness.factories.load(Ordering::SeqCst), 1); + for _ in 0..12 { + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + *harness.scope.lock().unwrap() = Some(scope()); + harness.registry.reconcile().unwrap(); + } + assert_eq!( + harness.factories.load(Ordering::SeqCst), + 1, + "a logout/login cycle must not create a second owner" + ); + assert!(workers(&harness) <= 1); + assert!(harness.maximum.load(Ordering::SeqCst) <= 1); + harness.connect.release(); + let deadline = Instant::now() + Duration::from_secs(3); + while workers(&harness) != 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + harness.registry.stop(); + assert_eq!(harness.active.load(Ordering::SeqCst), 0); + assert!(harness.maximum.load(Ordering::SeqCst) <= 1); + } + + #[test] + fn registry_never_registers_a_new_provider_with_the_old_config() { + let dir = config_dir("switch"); + let mut harness = harness(dir.clone(), true); + harness.registry.reconcile().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while harness.connects.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + harness.connects.load(Ordering::SeqCst) >= 1, + "the nvidia session connects only after its registration finishes" + ); + assert_eq!( + harness.tokens.lock().unwrap().as_slice(), + &[scope()], + "the nvidia account registers" + ); + let http_before = harness.http_calls.load(Ordering::SeqCst); + assert_eq!( + http_before, 5, + "a completed nvidia registration runs every registration request" + ); + let deadline = Instant::now() + Duration::from_secs(5); + while !harness.registry.owner.as_ref().unwrap().is_running() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + harness.registry.owner.as_ref().unwrap().is_running(), + "the nvidia session is live before the provider switches" + ); + let mut switched = scope(); + switched.provider_id = "alliance".into(); + switched.generation = 4; + *harness.scope.lock().unwrap() = Some(switched.clone()); + let deadline = Instant::now() + Duration::from_secs(5); + while harness.registry.owner.as_ref().unwrap().is_running() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + !harness.registry.owner.as_ref().unwrap().is_running(), + "the retired scope must end the nvidia session" + ); + assert!( + !harness + .tokens + .lock() + .unwrap() + .iter() + .any(|seen| seen.provider_id == "alliance"), + "the old provider config must never request the new account's token" + ); + assert_eq!( + harness.http_calls.load(Ordering::SeqCst), + http_before, + "the old provider config must not run registration HTTP for the new provider" + ); + assert_eq!(harness.factories.load(Ordering::SeqCst), 1); + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + harness.connect.release(); + harness.registry.stop(); + } + + #[test] + fn registry_stops_a_blocked_registration_before_later_stages() { + let mut harness = harness_with(config_dir("blocked-registration"), true, true); + harness.registry.reconcile().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while harness.http_calls.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!( + harness.http_calls.load(Ordering::SeqCst), + 1, + "the first stage must start" + ); + *harness.scope.lock().unwrap() = None; + harness.registry.reconcile().unwrap(); + harness.connect.release(); + let deadline = Instant::now() + Duration::from_secs(3); + while workers(&harness) != 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + harness.registry.stop(); + assert_eq!( + harness.http_calls.load(Ordering::SeqCst), + 1, + "no later registration stage may run after the scope retired" + ); + } + + #[test] + fn registry_sink_emits_the_documented_event_envelope() { + let (output, rx) = std::sync::mpsc::channel(); + let gfn = Arc::new(GfnService::new(std::env::temp_dir()).unwrap()); + let registry = PushRegistry::new(gfn, output, std::env::temp_dir()); + let sink: PushSink = { + let output = registry.output.clone(); + Arc::new(move |event, generation| { + let _ = output.send(json!({ + "type": "event", + "name": PUSH_EVENT_NAME, + "payload": push_event_payload(&event, generation), + })); + }) + }; + sink( + PushEvent::FavoritesChanged { + changed_ids: vec!["app-1".into()], + }, + 11, + ); + let message = rx.recv().unwrap(); + assert_eq!(message["type"], "event"); + assert_eq!(message["name"], "account.push.changed"); + assert_eq!(message["payload"]["generation"], 11); + assert_eq!(message["payload"]["kind"], "favorites"); + } +} diff --git a/external/opennow/native/opennow-core/src/queue_servers.rs b/external/opennow/native/opennow-core/src/queue_servers.rs new file mode 100644 index 000000000..c97c43de5 --- /dev/null +++ b/external/opennow/native/opennow-core/src/queue_servers.rs @@ -0,0 +1,523 @@ +use crate::gfn::ServiceError; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Map, Value, json}; +use std::collections::BTreeMap; +use std::io::Read; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const QUEUE_URL: &str = "https://api.printedwaste.com/gfn/queue/"; +const MAPPING_URL: &str = "https://remote.printedwaste.com/config/GFN_SERVERID_TO_REGION_MAPPING"; +const MAX_BODY: usize = 256 * 1024; +const MAX_ZONES: usize = 128; +const PROBE_WORKERS: usize = 32; +const MAX_AGE_SECONDS: u64 = 15 * 60; + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Location { + zone_id: String, + title: String, + region: String, + queue_position: u64, + eta_ms: Option, + last_updated: u64, + ping_ms: Option, + streaming_base_url: String, + alternate_count: usize, +} + +fn error(message: impl Into) -> ServiceError { + ServiceError { + code: "queue_servers_failed", + message: message.into(), + } +} + +pub(super) fn zone_url(zone: &str) -> Option { + if !(9..=14).contains(&zone.len()) { + return None; + } + let parts = zone.split('-').collect::>(); + if parts.len() != 3 + || parts[0] != "NP" + || !(3..=8).contains(&parts[1].len()) + || !parts[1] + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit()) + || !parts[1].bytes().next()?.is_ascii_uppercase() + || parts[2].len() != 2 + || !parts[2].bytes().all(|byte| byte.is_ascii_digit()) + { + return None; + } + Some(format!( + "https://{}.cloudmatchbeta.nvidiagrid.net/", + zone.to_ascii_lowercase() + )) +} + +fn client(url: &str) -> Result { + let parsed = url::Url::parse(url).map_err(|_| error("Queue endpoint is invalid"))?; + let host = parsed + .host_str() + .ok_or_else(|| error("Queue endpoint has no host"))?; + let port = parsed + .port_or_known_default() + .ok_or_else(|| error("Queue endpoint has no port"))?; + let addresses = crate::network::resolve_bounded(host.to_owned(), port) + .filter(|addresses| !addresses.is_empty()) + .ok_or_else(|| error("Queue endpoint could not be resolved"))?; + crate::requests::check()?; + Client::builder() + .timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(5)) + .resolve_to_addrs(host, &addresses) + .redirect(reqwest::redirect::Policy::none()) + .pool_max_idle_per_host(0) + .no_proxy() + .user_agent("opennow-core") + .build() + .map_err(|_| error("Could not create the queue HTTP client")) +} + +fn fetch(url: &str) -> Result { + crate::requests::check()?; + let client = client(url)?; + let mut response = client + .get(url) + .header("Accept", "application/json") + .send() + .map_err(|_| error("Queue data request failed"))?; + crate::requests::check()?; + if !response.status().is_success() { + return Err(error(format!( + "Queue data returned HTTP {}", + response.status().as_u16() + ))); + } + if response + .content_length() + .is_some_and(|size| size > MAX_BODY as u64) + { + return Err(error("Queue data response is too large")); + } + let mut bytes = Vec::new(); + let mut chunk = [0_u8; 8192]; + loop { + crate::requests::check()?; + let count = response + .read(&mut chunk) + .map_err(|_| error("Could not read queue data"))?; + if count == 0 { + break; + } + if bytes.len() + count > MAX_BODY { + return Err(error("Queue data response is too large")); + } + bytes.extend_from_slice(&chunk[..count]); + } + crate::requests::check()?; + serde_json::from_slice(&bytes).map_err(|_| error("Queue data is not valid JSON")) +} + +fn payload_data(payload: &Value) -> Result<&Map, ServiceError> { + if payload["status"].as_bool() != Some(true) { + return Err(error("Queue data did not report success")); + } + let data = payload["data"] + .as_object() + .ok_or_else(|| error("Queue data is missing its data object"))?; + if data.len() > 256 { + return Err(error("Queue data contains too many entries")); + } + Ok(data) +} + +fn text(value: &Value) -> Option<&str> { + value + .as_str() + .map(str::trim) + .filter(|text| !text.is_empty() && text.len() <= 128 && !text.chars().any(char::is_control)) +} + +fn continent(region: &str) -> &str { + match region { + "US" => "North America", + "CA" => "Canada", + "EU" => "Europe", + "JP" => "Japan", + "KR" => "South Korea", + "THAI" => "Southeast Asia", + "MY" => "Malaysia", + _ => region, + } +} + +fn parse_zones(queue: &Value, mapping: &Value, now: u64) -> Result, ServiceError> { + let data = payload_data(queue)?; + let mapping = payload_data(mapping).ok(); + let mut zones = Vec::new(); + for (zone_id, raw) in data { + let Some(streaming_base_url) = zone_url(zone_id) else { + continue; + }; + let entry = mapping + .and_then(|mapping| mapping.get(zone_id)) + .unwrap_or(&Value::Null); + if entry["nuked"].as_bool() == Some(true) { + continue; + } + let (Some(queue_position), Some(last_updated), Some(region)) = ( + raw["QueuePosition"].as_u64(), + raw["Last Updated"].as_u64(), + text(&raw["Region"]), + ) else { + continue; + }; + if last_updated == 0 + || last_updated > now.saturating_add(60) + || now.saturating_sub(last_updated) > MAX_AGE_SECONDS + { + continue; + } + zones.push(Location { + zone_id: zone_id.clone(), + title: text(&entry["title"]).unwrap_or(zone_id).to_owned(), + region: text(&entry["region"]) + .unwrap_or_else(|| continent(region)) + .to_owned(), + queue_position, + eta_ms: raw["eta"].as_u64(), + last_updated, + ping_ms: None, + streaming_base_url, + alternate_count: 0, + }); + } + if zones.len() > MAX_ZONES { + return Err(error("Queue data contains too many usable zones")); + } + if zones.is_empty() { + return Err(error("Queue data has no fresh usable NVIDIA zones")); + } + Ok(zones) +} + +fn score(zone: &Location, max_ping: u64, max_queue: u64) -> f64 { + 0.75 * zone.ping_ms.unwrap_or(max_ping) as f64 / max_ping as f64 + + 0.25 * zone.queue_position as f64 / max_queue as f64 +} + +fn best(zones: &[Location]) -> usize { + let measured = zones.iter().any(|zone| zone.ping_ms.is_some()); + let pool = zones + .iter() + .enumerate() + .filter(|(_, zone)| !measured || zone.ping_ms.is_some()) + .collect::>(); + let max_ping = pool + .iter() + .filter_map(|(_, zone)| zone.ping_ms) + .max() + .unwrap_or(1) + .max(1); + let max_queue = pool + .iter() + .map(|(_, zone)| zone.queue_position) + .max() + .unwrap_or(1) + .max(1); + let selected = pool + .iter() + .min_by(|(_, a), (_, b)| { + score(a, max_ping, max_queue) + .total_cmp(&score(b, max_ping, max_queue)) + .then_with(|| a.ping_ms.cmp(&b.ping_ms)) + .then_with(|| a.queue_position.cmp(&b.queue_position)) + .then_with(|| a.zone_id.cmp(&b.zone_id)) + }) + .map(|(index, _)| *index) + .unwrap_or(0); + if zones[selected].ping_ms.is_some_and(|ping| ping > 100) { + return pool + .iter() + .min_by_key(|(_, zone)| (zone.ping_ms, zone.queue_position, &zone.zone_id)) + .map(|(index, _)| *index) + .unwrap_or(selected); + } + selected +} + +fn locations(zones: Vec) -> Value { + let mut groups = BTreeMap::<(String, String), Vec>::new(); + for zone in zones { + groups + .entry((zone.region.clone(), zone.title.clone())) + .or_default() + .push(zone); + } + let mut locations = groups + .into_values() + .map(|group| { + let mut primary = group[best(&group)].clone(); + primary.alternate_count = group.len() - 1; + primary + }) + .collect::>(); + let recommended = (!locations.is_empty()).then(|| locations[best(&locations)].zone_id.clone()); + locations.sort_by(|a, b| { + (Some(&b.zone_id) == recommended.as_ref()) + .cmp(&(Some(&a.zone_id) == recommended.as_ref())) + .then_with(|| a.region.cmp(&b.region)) + .then_with(|| a.title.cmp(&b.title)) + }); + json!({"locations":locations,"recommendedZoneId":recommended}) +} + +pub fn list() -> Result { + crate::requests::check()?; + let cancellation = crate::requests::current(); + let (queue, mapping) = std::thread::scope(|scope| { + let mapping = + scope.spawn(|| crate::requests::scope(cancellation.clone(), || fetch(MAPPING_URL))); + let queue = fetch(QUEUE_URL); + ( + queue, + mapping + .join() + .unwrap_or_else(|_| Err(error("Queue mapping worker failed"))), + ) + }); + crate::requests::check()?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| error("System clock is invalid"))? + .as_secs(); + let mut zones = parse_zones(&queue?, &mapping.unwrap_or(Value::Null), now)?; + for batch in zones.chunks_mut(PROBE_WORKERS) { + crate::requests::check()?; + std::thread::scope(|scope| { + for zone in batch { + let cancellation = cancellation.clone(); + scope.spawn(move || { + zone.ping_ms = crate::requests::scope(cancellation, || { + crate::network::measure_queue_region(&zone.streaming_base_url) + }); + }); + } + }); + } + crate::requests::check()?; + Ok(locations(zones)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use std::net::TcpListener; + + fn queue() -> Value { + json!({"status":true,"data":{ + "NP-NEW9-01":{"QueuePosition":3,"Last Updated":10000,"Region":"US","eta":60000}, + "NP-NEW9-02":{"QueuePosition":10,"Last Updated":10000,"Region":"US"} + }}) + } + + fn options() -> Vec { + parse_zones(&queue(), &Value::Null, 10000).unwrap() + } + + #[test] + fn strict_zone_identity_allows_unknown_locations_not_host_injection_or_alliance() { + for zone in ["NP-NEW9-01", "NP-LAX-03", "NP-SJC6-04"] { + assert!(zone_url(zone).is_some()); + } + for zone in [ + "NPA-GKR-SEL-01", + "np-lax-03", + "NP-LAX-3", + "NP-LAX-003", + "NP-LAX-03.evil", + "NP-LAX-03/path", + "NP-LAX-03@evil", + "NP--03", + "NP-123-01", + "NP-LAX-03\n", + "NP-LAX-03-EXTRA", + ] { + assert!(zone_url(zone).is_none(), "{zone}"); + } + assert_eq!( + zone_url("NP-LAX-03").unwrap(), + "https://np-lax-03.cloudmatchbeta.nvidiagrid.net/" + ); + } + + #[test] + fn queue_errors_are_explicit_and_mapping_failure_preserves_raw_ids() { + for payload in [ + Value::Null, + json!({"status":false,"data":{}}), + json!({"status":true}), + json!({"status":true,"data":[]}), + json!({"status":true,"data":{}}), + ] { + assert!(parse_zones(&payload, &Value::Null, 10000).is_err()); + } + for mapping in [ + Value::Null, + json!({"status":false}), + json!({"status":true,"data":{}}), + ] { + let zones = parse_zones(&queue(), &mapping, 10000).unwrap(); + assert_eq!(zones[0].title, "NP-NEW9-01"); + assert_eq!(zones[0].region, "North America"); + assert_eq!(zones[0].eta_ms, Some(60000)); + assert_eq!(zones[1].eta_ms, None); + } + } + + #[test] + fn oversized_zone_sets_fail_before_any_probe_work() { + let mut data = Map::new(); + for index in 0..=MAX_ZONES { + data.insert( + format!("NP-LOC{index}-01"), + json!({"QueuePosition":1,"Last Updated":10000,"Region":"US"}), + ); + } + assert_eq!( + parse_zones(&json!({"status":true,"data":data}), &Value::Null, 10000) + .unwrap_err() + .message, + "Queue data contains too many usable zones" + ); + for index in 0..257 { + data.insert(index.to_string(), Value::Null); + } + assert_eq!( + payload_data(&json!({"status":true,"data":data})) + .unwrap_err() + .message, + "Queue data contains too many entries" + ); + } + + #[test] + fn nuked_stale_future_and_malformed_rows_are_not_selectable() { + let mut payload = queue(); + for (id, updated, position) in [ + ("NP-OLD-01", 9099, json!(0)), + ("NP-FUT-01", 10061, json!(0)), + ("NP-BAD-01", 10000, json!(-1)), + ("NP-BAD-02", 10000, json!(1.5)), + ("NP-BAD-03", 10000, json!("1")), + ("NPA-BAD-01", 10000, json!(1)), + ] { + payload["data"][id] = + json!({"QueuePosition":position,"Last Updated":updated,"Region":"US"}); + } + let mapping = json!({"status":true,"data":{"NP-NEW9-02":{"nuked":true}}}); + let zones = parse_zones(&payload, &mapping, 10000).unwrap(); + assert_eq!(zones.len(), 1); + assert_eq!(zones[0].zone_id, "NP-NEW9-01"); + assert!(parse_zones(&payload, &mapping, 20000).is_err()); + } + + #[test] + fn recommendation_weights_latency_and_queue_and_guards_high_latency() { + let mut zones = options(); + zones[0].ping_ms = Some(20); + zones[0].queue_position = 100; + zones[1].ping_ms = Some(25); + zones[1].queue_position = 0; + assert_eq!(best(&zones), 1); + zones[1].ping_ms = Some(80); + assert_eq!(best(&zones), 0); + zones[0].ping_ms = Some(110); + zones[1].ping_ms = Some(120); + assert_eq!(best(&zones), 0); + zones[0].ping_ms = None; + assert_eq!(best(&zones), 1); + zones[1].ping_ms = None; + assert_eq!(best(&zones), 1); + } + + #[test] + fn grouped_locations_use_measured_primary_and_recommend_returned_id() { + let mapping = json!({"status":true,"data":{ + "NP-NEW9-01":{"title":" New location ","region":" US West "}, + "NP-NEW9-02":{"title":"New location","region":"US West"} + }}); + let mut zones = parse_zones(&queue(), &mapping, 10000).unwrap(); + zones[1].ping_ms = Some(20); + let result = locations(zones); + assert_eq!(result["locations"].as_array().unwrap().len(), 1); + assert_eq!(result["locations"][0]["title"], "New location"); + assert_eq!(result["locations"][0]["alternateCount"], 1); + assert_eq!(result["locations"][0]["zoneId"], "NP-NEW9-02"); + assert_eq!( + result["recommendedZoneId"], + result["locations"][0]["zoneId"] + ); + } + + fn fetch_response(response: Vec) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let worker = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = [0; 8192]; + let count = stream.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..count]).to_ascii_lowercase(); + assert!(!request.contains("authorization:")); + assert!(!request.contains("cookie:")); + let _ = stream.write_all(&response); + }); + let result = fetch(&url); + worker.join().unwrap(); + result + } + + #[test] + fn http_rejects_errors_redirects_invalid_json_and_oversized_streamed_bodies() { + for response in [ + "HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\n\r\n", + "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:1/\r\nContent-Length: 0\r\n\r\n", + "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\n!", + "HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n", + ] { + assert!(fetch_response(response.as_bytes().to_vec()).is_err()); + } + let mut response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_vec(); + response.resize(response.len() + MAX_BODY + 1, b' '); + assert_eq!( + fetch_response(response).unwrap_err().message, + "Queue data response is too large" + ); + assert_eq!( + fetch_response(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}".to_vec()).unwrap(), + json!({}) + ); + } + + #[test] + fn cancelled_queue_request_never_starts_fetches_or_probes() { + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("queue", "queue.servers.list").unwrap(); + requests.cancel("queue"); + crate::requests::scope(permit.token.clone(), || { + assert_eq!(list().unwrap_err().code, "cancelled"); + assert_eq!(fetch("http://127.0.0.1:1").unwrap_err().code, "cancelled"); + assert_eq!( + crate::network::measure_queue_region("http://127.0.0.1:1"), + None + ); + }); + } +} diff --git a/external/opennow/native/opennow-core/src/reports.rs b/external/opennow/native/opennow-core/src/reports.rs new file mode 100644 index 000000000..981126077 --- /dev/null +++ b/external/opennow/native/opennow-core/src/reports.rs @@ -0,0 +1,529 @@ +use flate2::{Compression, write::GzEncoder}; +use reqwest::blocking::{Client, multipart}; +use serde_json::{Value, json}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub const DEFAULT_REPORTS_API: &str = "https://opennow-reports-production.up.railway.app"; +const MAX_REPORT_JSON_BYTES: usize = 64 * 1024; +const MAX_LOG_TEXT_BYTES: u64 = 32 * 1024 * 1024; +const MAX_LOG_GZIP_BYTES: usize = 5 * 1024 * 1024; +const MAX_PENDING_REPORTS: usize = 5; +const MAX_PENDING_AGE_MS: u128 = 7 * 24 * 60 * 60 * 1000; + +pub fn api_base_from_env() -> String { + std::env::var("OPENNOW_REPORTS_API") + .ok() + .map(|value| value.trim().trim_end_matches('/').to_owned()) + .filter(|value| value.starts_with("https://") || value.starts_with("http://")) + .unwrap_or_else(|| DEFAULT_REPORTS_API.to_owned()) +} + +pub fn valid_install_id(value: &str) -> bool { + value.len() == 32 && value.bytes().all(|value| value.is_ascii_hexdigit()) +} + +pub fn required_text( + value: &Value, + minimum: usize, + maximum: usize, + label: &str, +) -> Result { + let value = value.as_str().map(str::trim).unwrap_or_default(); + if !(minimum..=maximum).contains(&value.chars().count()) { + return Err(format!( + "{label} must be between {minimum} and {maximum} characters" + )); + } + Ok(value.to_owned()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Accepted { + pub report_id: String, + pub issue_id: Option, + pub issue_status: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Failure { + pub message: String, + pub retryable: bool, +} + +pub struct Document<'a> { + pub install_id: &'a str, + pub account: &'a Value, + pub activity: Value, + pub app: Value, +} + +impl Document<'_> { + pub fn automatic(self, trigger: Value) -> Value { + self.compose(true, trigger) + } + + pub fn manual(self, title: &str, description: &str) -> Value { + self.compose(false, json!({"title":title, "description":description})) + } + + fn compose(self, automatic: bool, detail: Value) -> Value { + let mut report = json!({ + "schemaVersion": 1, + "automatic": automatic, + "installId": self.install_id, + "account": if self.account.is_object() { self.account.clone() } else { Value::Null }, + "activity": self.activity, + "app": self.app + }); + report[if automatic { "trigger" } else { "manual" }] = detail; + report + } +} + +pub fn gzip_log(path: &Path) -> Option> { + if path + .metadata() + .map_or(true, |metadata| metadata.len() > MAX_LOG_TEXT_BYTES) + { + return None; + } + let text = fs::read(path).ok()?; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&text).ok()?; + encoder + .finish() + .ok() + .filter(|bytes| bytes.len() <= MAX_LOG_GZIP_BYTES) +} + +pub struct ReportsClient { + client: Client, + endpoint: String, + pending: PathBuf, +} + +impl ReportsClient { + pub fn new(base: &str, data_dir: &Path) -> Result { + Ok(Self { + client: Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) + .build() + .map_err(|error| error.to_string())?, + endpoint: format!("{base}/v1/reports"), + pending: data_dir.join("reports").join("pending"), + }) + } + + pub fn submit(&self, report: &Value, log: Option<&[u8]>) -> Result { + let body = report.to_string(); + if body.len() > MAX_REPORT_JSON_BYTES { + return Err(Failure { + message: "The bug report is too large to send.".to_owned(), + retryable: false, + }); + } + let mut form = multipart::Form::new().part( + "report", + multipart::Part::text(body) + .mime_str("application/json") + .map_err(|error| Failure { + message: error.to_string(), + retryable: false, + })?, + ); + if let Some(log) = log { + form = form.part( + "log", + multipart::Part::bytes(log.to_vec()) + .file_name("opennow-diagnostics.log.gz") + .mime_str("application/gzip") + .map_err(|error| Failure { + message: error.to_string(), + retryable: false, + })?, + ); + } + let response = self + .client + .post(&self.endpoint) + .multipart(form) + .send() + .map_err(|_| Failure { + message: "The reporting service is unavailable right now.".to_owned(), + retryable: true, + })?; + let status = response.status().as_u16(); + let body = response + .text() + .ok() + .and_then(|body| serde_json::from_str::(&body).ok()) + .unwrap_or(Value::Null); + if (200..300).contains(&status) { + let text = |key: &str| body[key].as_str().map(|value| bounded(value, 64)); + return Ok(Accepted { + report_id: text("reportId").unwrap_or_default(), + issue_id: text("issueId"), + issue_status: text("issueStatus"), + }); + } + Err(Failure { + message: body["error"]["message"] + .as_str() + .map(|message| bounded(message, 320)) + .filter(|message| !message.is_empty()) + .unwrap_or_else(|| match status { + 429 => "Too many bug reports were sent. Try again later.".to_owned(), + _ => format!("Bug report upload failed (HTTP {status})."), + }), + retryable: body["error"]["retryable"] + .as_bool() + .unwrap_or(status == 408 || status == 429 || status >= 500), + }) + } + + pub fn spool(&self, report: &Value, log: Option<&[u8]>, now_ms: u128) -> std::io::Result<()> { + fs::create_dir_all(&self.pending)?; + let name = format!("{now_ms:013}-{:08x}", rand::random::()); + if let Some(log) = log { + fs::write(self.pending.join(format!("{name}.log.gz")), log)?; + } + let path = self.pending.join(format!("{name}.json")); + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, report.to_string())?; + fs::rename(&temporary, &path)?; + self.prune(now_ms); + Ok(()) + } + + pub fn clear_pending(&self) { + let _ = fs::remove_dir_all(&self.pending); + } + + pub fn retry_pending(&self, now_ms: u128) -> Vec<(Value, Accepted)> { + let mut sent = Vec::new(); + for name in self.prune(now_ms) { + let path = self.pending.join(format!("{name}.json")); + let Some(report) = fs::read_to_string(&path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + else { + self.remove(&name); + continue; + }; + let log = fs::read(self.pending.join(format!("{name}.log.gz"))).ok(); + match self.submit(&report, log.as_deref()) { + Ok(accepted) => { + self.remove(&name); + sent.push((report, accepted)); + } + Err(failure) if failure.retryable => break, + Err(_) => self.remove(&name), + } + } + sent + } + + fn prune(&self, now_ms: u128) -> Vec { + let Ok(entries) = fs::read_dir(&self.pending) else { + return Vec::new(); + }; + let files = entries + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().into_string().ok()) + .collect::>(); + let mut reports = files + .iter() + .filter_map(|file| file.strip_suffix(".json")) + .map(ToOwned::to_owned) + .collect::>(); + reports.sort(); + let fresh = |name: &String| { + name.split('-') + .next() + .and_then(|stamp| stamp.parse::().ok()) + .is_some_and(|stamp| now_ms.saturating_sub(stamp) <= MAX_PENDING_AGE_MS) + }; + let excess = reports.len().saturating_sub(MAX_PENDING_REPORTS); + let (kept, dropped): (Vec<_>, Vec<_>) = reports + .into_iter() + .enumerate() + .partition(|(index, name)| *index >= excess && fresh(name)); + for (_, name) in dropped { + self.remove(&name); + } + let kept = kept.into_iter().map(|(_, name)| name).collect::>(); + for file in &files { + let orphan = file + .strip_suffix(".log.gz") + .or_else(|| file.strip_suffix(".json.tmp")) + .is_some_and(|name| !kept.iter().any(|kept| kept == name)); + if orphan { + let _ = fs::remove_file(self.pending.join(file)); + } + } + kept + } + + fn remove(&self, name: &str) { + let _ = fs::remove_file(self.pending.join(format!("{name}.json"))); + let _ = fs::remove_file(self.pending.join(format!("{name}.log.gz"))); + } +} + +fn bounded(value: &str, limit: usize) -> String { + value + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .take(limit) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::read::GzDecoder; + use std::io::{BufRead, BufReader, Read}; + use std::net::TcpListener; + use std::thread; + + const INSTALL: &str = "0123456789abcdef0123456789abcdef"; + + fn server(replies: Vec<(u16, Value)>) -> (String, thread::JoinHandle>>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let worker = thread::spawn(move || { + let mut bodies = Vec::new(); + for (status, reply) in replies { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&stream); + let mut head = String::new(); + let mut length = 0; + loop { + let mut line = String::new(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + head.push_str(&line); + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + let mut request = head.into_bytes(); + request.extend_from_slice(b"\r\n"); + request.extend_from_slice(&body); + bodies.push(request); + let reply = reply.to_string(); + write!(stream, "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).unwrap(); + } + bodies + }); + (base, worker) + } + + fn report() -> Value { + Document { + install_id: INSTALL, + account: &json!({"userId":"sub-1", "providerIdpId":"idp", "providerCode":"NVIDIA", + "providerName":"NVIDIA", "alliancePartner":false, "reporter":"Zortos", + "reporterBasis":"username", "membershipTier":"ULTIMATE"}), + activity: json!({"currentGame":{"title":"Portal 2","appId":"100"}, "recentGames":[]}), + app: json!({"version":"1.0.3", "os":"linux", "arch":"x86_64", "gpu":null, + "decoderBackend":"vaapi", "channel":"stable"}), + } + .automatic( + json!({"kind":"frame_drops", "code":"sustained_frame_drops", "message":"", + "metrics":{"droppedFrames":412.0}}), + ) + } + + fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) + } + + #[test] + fn manual_text_is_trimmed_and_bounded() { + assert!(required_text(&json!("short"), 8, 120, "Bug report title").is_err()); + assert_eq!( + required_text(&json!(" A valid title "), 8, 120, "Bug report title").unwrap(), + "A valid title" + ); + } + + #[test] + fn documents_follow_schema_v1_and_carry_only_one_detail_section() { + let automatic = report(); + assert_eq!(automatic["schemaVersion"], 1); + assert_eq!(automatic["automatic"], true); + assert!(automatic.get("manual").is_none()); + assert_eq!(automatic["trigger"]["code"], "sustained_frame_drops"); + let manual = Document { + install_id: INSTALL, + account: &Value::Null, + activity: json!({"currentGame":null, "recentGames":[]}), + app: json!({}), + } + .manual("Feedback: idea", "Add a dark mode please"); + assert_eq!(manual["automatic"], false); + assert_eq!(manual["account"], Value::Null); + assert!(manual.get("trigger").is_none()); + assert_eq!(manual["manual"]["title"], "Feedback: idea"); + } + + #[test] + fn reports_upload_as_multipart_with_json_and_a_gzip_log() { + let (base, worker) = server(vec![( + 201, + json!({"reportId":"br-7F3A21", "issueId":"iss_1", "issueStatus":"investigating"}), + )]); + let directory = tempfile::tempdir().unwrap(); + let client = ReportsClient::new(&base, directory.path()).unwrap(); + let log_path = directory.path().join("log.txt"); + fs::write(&log_path, "diagnostics line\n".repeat(64)).unwrap(); + let log = gzip_log(&log_path).unwrap(); + let accepted = client.submit(&report(), Some(&log)).unwrap(); + assert_eq!( + accepted, + Accepted { + report_id: "br-7F3A21".to_owned(), + issue_id: Some("iss_1".to_owned()), + issue_status: Some("investigating".to_owned()), + } + ); + let request = worker.join().unwrap().pop().unwrap(); + let text = String::from_utf8_lossy(&request); + assert!(text.starts_with("POST /v1/reports HTTP/1.1")); + assert!( + text.to_ascii_lowercase() + .contains("content-type: multipart/form-data; boundary=") + ); + assert!(text.contains("Content-Disposition: form-data; name=\"report\"")); + assert!(text.contains("Content-Type: application/json")); + assert!(text.contains(&report().to_string())); + assert!(text.contains( + "Content-Disposition: form-data; name=\"log\"; filename=\"opennow-diagnostics.log.gz\"" + )); + let start = find(&request, &[0x1f, 0x8b]).expect("gzip payload"); + let mut decoded = String::new(); + GzDecoder::new(&request[start..]) + .read_to_string(&mut decoded) + .unwrap(); + assert!(decoded.starts_with("diagnostics line\n")); + } + + #[test] + fn server_errors_report_their_retryability() { + let (base, worker) = server(vec![ + ( + 400, + json!({"error":{"code":"schema","message":" bad schema ","retryable":false}}), + ), + (503, Value::Null), + ( + 429, + json!({"error":{"code":"rate_limited","message":"Slow down","retryable":true}}), + ), + ]); + let directory = tempfile::tempdir().unwrap(); + let client = ReportsClient::new(&base, directory.path()).unwrap(); + assert_eq!( + client.submit(&report(), None).unwrap_err(), + Failure { + message: "bad schema".to_owned(), + retryable: false + } + ); + assert!(client.submit(&report(), None).unwrap_err().retryable); + assert!(client.submit(&report(), None).unwrap_err().retryable); + worker.join().unwrap(); + let unreachable = ReportsClient::new("http://127.0.0.1:9", directory.path()).unwrap(); + assert!(unreachable.submit(&report(), None).unwrap_err().retryable); + } + + #[test] + fn pending_reports_keep_the_newest_five_within_seven_days() { + let directory = tempfile::tempdir().unwrap(); + let client = ReportsClient::new("http://127.0.0.1:9", directory.path()).unwrap(); + let day = 24 * 60 * 60 * 1000; + let now = 100 * day; + client + .spool(&json!({"n":0}), Some(b"old"), now - 8 * day) + .unwrap(); + for index in 1..=6 { + client + .spool(&json!({"n":index}), None, now - 6 * day + index) + .unwrap(); + } + let kept = client.prune(now); + assert_eq!(kept.len(), 5); + let numbers = kept + .iter() + .map(|name| { + let text = fs::read_to_string(client.pending.join(format!("{name}.json"))).unwrap(); + serde_json::from_str::(&text).unwrap()["n"] + .as_i64() + .unwrap() + }) + .collect::>(); + assert_eq!(numbers, vec![2, 3, 4, 5, 6]); + let gz = fs::read_dir(&client.pending) + .unwrap() + .filter(|entry| { + entry + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".log.gz") + }) + .count(); + assert_eq!(gz, 0); + assert!(client.prune(now + 8 * day).is_empty()); + } + + #[test] + fn pending_reports_retry_until_the_service_is_unavailable() { + let directory = tempfile::tempdir().unwrap(); + let (base, worker) = server(vec![ + (201, json!({"reportId":"br-000001"})), + ( + 400, + json!({"error":{"code":"schema","message":"bad","retryable":false}}), + ), + (503, Value::Null), + ]); + let client = ReportsClient::new(&base, directory.path()).unwrap(); + let now = 10_000_000; + for index in 0..4 { + client + .spool(&json!({"n":index}), Some(b"log"), now + index) + .unwrap(); + } + let sent = client.retry_pending(now + 10); + worker.join().unwrap(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].0["n"], 0); + assert_eq!(sent[0].1.report_id, "br-000001"); + let remaining = client.prune(now + 10); + assert_eq!(remaining.len(), 2); + assert!( + client + .pending + .join(format!("{}.log.gz", remaining[0])) + .is_file() + ); + } +} diff --git a/external/opennow/native/opennow-core/src/requests.rs b/external/opennow/native/opennow-core/src/requests.rs new file mode 100644 index 000000000..5d2179600 --- /dev/null +++ b/external/opennow/native/opennow-core/src/requests.rs @@ -0,0 +1,226 @@ +//! Bounded RPC ownership, cooperative cancellation, and allocation receipts. +use crate::gfn::ServiceError; +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +const MAX_ACTIVE: usize = 8; +const MAX_BACKGROUND: usize = 4; + +#[derive(Default)] +struct RequestState { + cancelled: AtomicBool, + accepted: Mutex, + wake: Condvar, +} + +#[derive(Clone, Default)] +pub struct Cancellation(Arc); +impl Cancellation { + pub fn commit( + &self, + work: impl FnOnce() -> Result, + ) -> Result { + let _commit = self.0.accepted.lock().expect("request commit poisoned"); + self.check()?; + work() + } + pub fn cancelled(&self) -> bool { + self.0.cancelled.load(Ordering::Acquire) + } + + pub fn await_acceptance(&self, timeout: Duration) -> bool { + let accepted = self.0.accepted.lock().expect("request receipt poisoned"); + let (accepted, _) = self + .0 + .wake + .wait_timeout_while(accepted, timeout, |accepted| { + !*accepted && !self.cancelled() + }) + .expect("request receipt poisoned"); + *accepted && !self.cancelled() + } + pub fn check(&self) -> Result<(), ServiceError> { + if self.cancelled() { + Err(ServiceError { + code: "cancelled", + message: "Request cancelled".into(), + }) + } else { + Ok(()) + } + } +} + +#[derive(Default)] +pub struct Requests(Mutex>); +impl Requests { + pub fn admit(self: &Arc, id: &str, method: &str) -> Option { + let background = method.starts_with("catalog.") + || method.starts_with("artwork.") + || method == "network.regions.ping" + || method == "queue.servers.list"; + let mut active = self.0.lock().expect("request state poisoned"); + if active.contains_key(id) + || active.len() >= MAX_ACTIVE + || (background && active.values().filter(|(bg, _)| *bg).count() >= MAX_BACKGROUND) + { + return None; + } + let token = Cancellation::default(); + active.insert(id.into(), (background, token.clone())); + Some(Permit { + owner: self.clone(), + id: id.into(), + token, + }) + } + pub fn cancel(&self, id: &str) { + if let Some((_, token)) = self.0.lock().expect("request state poisoned").get(id) { + let _accepted = token.0.accepted.lock().expect("request receipt poisoned"); + token.0.cancelled.store(true, Ordering::Release); + token.0.wake.notify_all(); + } + } + + pub fn acknowledge(&self, id: &str) { + if let Some((_, token)) = self.0.lock().expect("request state poisoned").get(id) { + *token.0.accepted.lock().expect("request receipt poisoned") = true; + token.0.wake.notify_all(); + } + } +} + +pub struct Permit { + owner: Arc, + id: String, + pub token: Cancellation, +} +impl Drop for Permit { + fn drop(&mut self) { + self.owner + .0 + .lock() + .expect("request state poisoned") + .remove(&self.id); + } +} + +thread_local! { static CURRENT: RefCell = RefCell::default(); } +pub fn current() -> Cancellation { + CURRENT.with(|token| token.borrow().clone()) +} +pub fn check() -> Result<(), ServiceError> { + current().check() +} + +// A request worker is synchronous. Scoped child threads explicitly clone the +// token; cache/download workers intentionally have independent lifetimes. +pub fn scope(token: Cancellation, work: impl FnOnce() -> T) -> T { + struct Restore(Cancellation); + impl Drop for Restore { + fn drop(&mut self) { + CURRENT.with(|token| *token.borrow_mut() = self.0.clone()); + } + } + let _restore = Restore(CURRENT.with(|current| current.replace(token))); + work() +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn auth_commit_fence_rejects_prior_cancellation_and_finishes_entered_commit() { + let requests = Arc::new(Requests::default()); + let permit = requests.admit("login", "auth.device.complete").unwrap(); + requests.cancel("login"); + assert!( + permit + .token + .commit(|| -> Result<(), ServiceError> { panic!("cancelled commit ran") }) + .is_err() + ); + let permit = requests.admit("second", "auth.device.complete").unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let token = permit.token.clone(); + std::thread::scope(|scope| { + let worker = scope.spawn(move || { + token.commit(|| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok("committed") + }) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let cancellation = scope.spawn(|| requests.cancel("second")); + release_tx.send(()).unwrap(); + assert_eq!(worker.join().unwrap().unwrap(), "committed"); + cancellation.join().unwrap(); + }); + assert!(permit.token.cancelled()); + } + #[test] + fn allocation_receipt_requires_acceptance_and_cancellation_wins() { + let requests = Arc::new(Requests::default()); + let permit = requests.admit("create", "session.create").unwrap(); + assert!(!permit.token.await_acceptance(Duration::ZERO)); + requests.acknowledge("unknown"); + assert!(!permit.token.await_acceptance(Duration::ZERO)); + requests.acknowledge("create"); + assert!(permit.token.await_acceptance(Duration::ZERO)); + requests.cancel("create"); + assert!(!permit.token.await_acceptance(Duration::ZERO)); + } + + #[test] + fn cancellation_wakes_pending_allocation_receipt() { + let requests = Arc::new(Requests::default()); + let permit = requests.admit("create", "session.create").unwrap(); + std::thread::scope(|scope| { + let token = permit.token.clone(); + let worker = scope.spawn(move || token.await_acceptance(Duration::from_secs(10))); + requests.cancel("create"); + assert!(!worker.join().unwrap()); + }); + } + #[test] + fn background_load_reserves_control_capacity_until_workers_really_exit() { + let requests = Arc::new(Requests::default()); + let mut permits = Vec::new(); + for id in 0..4 { + permits.push( + requests + .admit(&id.to_string(), "catalog.store.local") + .unwrap(), + ); + } + requests.cancel("0"); + assert!(permits[0].token.cancelled()); + assert!(requests.admit("more", "catalog.store.local").is_none()); + for id in 4..8 { + permits.push(requests.admit(&id.to_string(), "session.poll").unwrap()); + } + assert!(requests.admit("overflow", "session.stop").is_none()); + permits.clear(); + assert!(requests.admit("new", "catalog.store.local").is_some()); + } + #[test] + fn unknown_cancels_do_not_accumulate_and_scopes_restore() { + let requests = Arc::new(Requests::default()); + for id in 0..10000 { + requests.cancel(&id.to_string()); + } + assert!(requests.0.lock().unwrap().is_empty()); + let permit = requests.admit("work", "catalog.store.local").unwrap(); + assert!(requests.admit("work", "session.poll").is_none()); + requests.cancel("work"); + scope(permit.token.clone(), || { + assert_eq!(check().unwrap_err().code, "cancelled") + }); + assert!(check().is_ok()); + } +} diff --git a/external/opennow/native/opennow-core/src/server_vpc_cache.rs b/external/opennow/native/opennow-core/src/server_vpc_cache.rs new file mode 100644 index 000000000..eefbb3d99 --- /dev/null +++ b/external/opennow/native/opennow-core/src/server_vpc_cache.rs @@ -0,0 +1,284 @@ +use crate::gfn::ServiceError; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const FRESHNESS: Duration = Duration::from_secs(5 * 60); +const FAILURE_BACKOFF: Duration = Duration::from_secs(30); + +struct Entry { + result: Result, + expires: Instant, +} + +type ScopeEntry = Arc>>; + +#[derive(Default)] +pub struct ServerVpcCache(Mutex>); + +impl ServerVpcCache { + pub fn resolve( + &self, + provider: &str, + account: &str, + token: &str, + fetch: impl FnOnce() -> Result, ServiceError>, + ) -> Result { + let scope = Sha256::digest(json!([provider, account, token]).to_string().as_bytes()).into(); + let slot = { + let mut entries = crate::store_requests::lock(&self.0)?; + if !entries.contains_key(&scope) && entries.len() >= 16 { + entries.retain(|_, entry| Arc::strong_count(entry) > 1); + if entries.len() >= 16 { + return Err(ServiceError { + code: "routing_busy", + message: "Too many provider lookups are in progress".into(), + }); + } + } + entries.entry(scope).or_default().clone() + }; + let mut cached = crate::store_requests::lock(&slot)?; + if let Some(entry) = cached + .as_ref() + .filter(|entry| entry.expires > Instant::now()) + { + return entry.result.clone(); + } + let result = fetch().and_then(|value| { + value + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| ServiceError { + code: "invalid_upstream_response", + message: "Server info did not include a verified VPC".into(), + }) + }); + crate::requests::check()?; + if result.as_ref().is_err_and(|error| { + matches!( + error.code, + "http_unauthorized" | "rate_limited" | "cancelled" | "stale_account" + ) + }) { + return result; + } + let freshness = if result.is_ok() { + FRESHNESS + } else { + FAILURE_BACKOFF + }; + *cached = Some(Entry { + result: result.clone(), + expires: Instant::now() + freshness, + }); + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + Arc, Barrier, + atomic::{AtomicUsize, Ordering}, + }; + + fn only_entry(cache: &ServerVpcCache) -> Arc>> { + cache.0.lock().unwrap().values().next().unwrap().clone() + } + + #[test] + fn unrelated_provider_lookup_does_not_wait_for_an_inflight_scope() { + let cache = ServerVpcCache::default(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|threads| { + let first = threads.spawn(|| { + cache.resolve("provider-a", "account-a", "token-a", move || { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(Some("vpc-a".into())) + }) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + cache + .resolve("provider-b", "account-b", "token-b", || Ok(Some( + "vpc-b".into() + ))) + .unwrap(), + "vpc-b" + ); + release_tx.send(()).unwrap(); + assert_eq!(first.join().unwrap().unwrap(), "vpc-a"); + }); + } + + #[test] + fn concurrent_lookups_share_one_fetch_and_reuse_it_for_five_minutes() { + let cache = ServerVpcCache::default(); + let ready = Barrier::new(4); + let calls = AtomicUsize::new(0); + let started = Instant::now(); + std::thread::scope(|threads| { + for _ in 0..4 { + let cache = &cache; + let ready = &ready; + let calls = &calls; + threads.spawn(move || { + ready.wait(); + let result = cache + .resolve("provider", "account", "token", || { + calls.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(100)); + Ok(Some("region-a".into())) + }) + .unwrap(); + assert_eq!(result, "region-a"); + }); + } + }); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!( + cache + .resolve("provider", "account", "token", || panic!( + "fresh metadata refetched" + )) + .unwrap(), + "region-a" + ); + let slot = only_entry(&cache); + let mut cached = slot.lock().unwrap(); + let entry = cached.as_mut().unwrap(); + assert!(entry.expires >= started + FRESHNESS); + assert!(entry.expires <= Instant::now() + FRESHNESS); + entry.expires = Instant::now() - Duration::from_secs(1); + drop(cached); + assert_eq!( + cache + .resolve("provider", "account", "token", || Ok(Some( + "region-b".into() + ))) + .unwrap(), + "region-b" + ); + } + + #[test] + fn providers_accounts_and_refreshed_tokens_do_not_share_metadata() { + let cache = ServerVpcCache::default(); + for (index, (provider, account, token)) in [ + ("provider-a", "account-a", "token-a"), + ("provider-b", "account-a", "token-a"), + ("provider-b", "account-b", "token-a"), + ("provider-b", "account-b", "token-b"), + ] + .into_iter() + .enumerate() + { + let expected = format!("region-{index}"); + assert_eq!( + cache + .resolve(provider, account, token, || Ok(Some(expected.clone()))) + .unwrap(), + expected + ); + } + } + + #[test] + fn failures_preserve_known_metadata_with_a_short_retry_backoff() { + let cache = ServerVpcCache::default(); + assert_eq!( + cache.resolve("p", "a", "t", || Ok(None)).unwrap_err().code, + "invalid_upstream_response" + ); + assert!( + only_entry(&cache) + .lock() + .unwrap() + .as_ref() + .unwrap() + .expires + .duration_since(Instant::now()) + <= FAILURE_BACKOFF + ); + cache + .resolve("p", "a", "t", || { + panic!("failed lookup retried immediately") + }) + .unwrap_err(); + only_entry(&cache).lock().unwrap().as_mut().unwrap().expires = + Instant::now() - Duration::from_secs(1); + assert_eq!( + cache + .resolve("p", "a", "t", || Ok(Some("known".into()))) + .unwrap(), + "known" + ); + only_entry(&cache).lock().unwrap().as_mut().unwrap().expires = + Instant::now() - Duration::from_secs(1); + assert_eq!( + cache.resolve("p", "a", "t", || Ok(None)).unwrap_err().code, + "invalid_upstream_response" + ); + assert_eq!( + cache + .resolve("other", "a", "t", || Ok(None)) + .unwrap_err() + .code, + "invalid_upstream_response" + ); + } + + #[test] + fn rate_limit_errors_are_not_cached_as_successful_metadata() { + let cache = ServerVpcCache::default(); + let error = cache + .resolve("p", "a", "t", || { + Err(ServiceError { + code: "rate_limited", + message: "Try later".into(), + }) + }) + .unwrap_err(); + assert_eq!(error.code, "rate_limited"); + assert!(only_entry(&cache).lock().unwrap().is_none()); + assert_eq!( + cache + .resolve("p", "a", "t", || Ok(Some("recovered".into()))) + .unwrap(), + "recovered" + ); + } + + #[test] + fn cancelled_lookups_neither_wait_for_the_cache_nor_publish_results() { + let cache = ServerVpcCache::default(); + let requests = Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("lookup", "catalog.store.local").unwrap(); + crate::requests::scope(permit.token.clone(), || { + assert_eq!( + cache + .resolve("p", "a", "t", || { + requests.cancel("lookup"); + Ok(Some("cancelled".into())) + }) + .unwrap_err() + .code, + "cancelled" + ); + let held = cache.0.lock().unwrap(); + assert!(held.values().all(|entry| entry.lock().unwrap().is_none())); + assert_eq!( + cache + .resolve("p", "a", "t", || panic!("cancelled fetch started")) + .unwrap_err() + .code, + "cancelled" + ); + }); + } +} diff --git a/external/opennow/native/opennow-core/src/settings.rs b/external/opennow/native/opennow-core/src/settings.rs new file mode 100644 index 000000000..a8eea09b7 --- /dev/null +++ b/external/opennow/native/opennow-core/src/settings.rs @@ -0,0 +1,3467 @@ +use crate::proxy::normalize_proxy_url; +use serde_json::{Map, Value, json}; +use std::collections::HashSet; +use std::env; +use std::fs; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +const NATIVE_TRANSPORT: &str = "nvst"; +const CONSOLE_POLICY_VERSION: &str = "qtConsoleModePolicyVersion"; +const WINDOWS_GPU_DEVICE_ID: &str = "windowsGpuDeviceId"; +const MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES: usize = 1024; +const MAXIMUM_BOOTSTRAP_SETTINGS_BYTES: u64 = 1024 * 1024; +const MAXIMUM_SHORTCUT_BYTES: usize = 80; +const SHORTCUT_KEYS: [&str; 9] = [ + "shortcutToggleStats", + "shortcutTogglePointerLock", + "shortcutToggleFullscreen", + "shortcutStopStream", + "shortcutToggleAntiAfk", + "shortcutToggleMicrophone", + "shortcutScreenshot", + "shortcutToggleRecording", + "shortcutSaveClip", +]; +const RESERVED_SHORTCUTS: [&str; 2] = ["Ctrl+G", "Shift+F3"]; +const HDR_NOTICE_SHOWN: &str = "hdrNoticeShownGameIds"; +const MAXIMUM_HDR_NOTICE_SHOWN: usize = 200; +const MAXIMUM_CONTROLLER_PROFILES: usize = 32; +const MAXIMUM_CONTROLLER_PROFILE_KEY_CHARS: usize = 160; +const MAXIMUM_CONTROLLER_PROFILE_NAME_CHARS: usize = 80; +const CONTROLLER_PROFILE_TUNING: [(&str, i64); 3] = [ + ("leftStickDeadzone", 50), + ("rightStickDeadzone", 50), + ("vibrationIntensity", 100), +]; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LoadPolicy { + ReadWrite, + ReadOnly, +} + +pub struct SettingsStore { + path: PathBuf, + values: Map, + // Preserve fields owned by an older/newer Electron build so trying the Qt + // client and then rolling back never destroys settings it does not know. + // Unknown values are not exposed over the shell/core contract. + passthrough: Map, +} + +impl SettingsStore { + pub fn load(data_dir: Option) -> io::Result { + Self::load_with_policy(data_dir, LoadPolicy::ReadWrite) + } + + pub(super) fn windows_gpu_device_id_read_only(data_dir: Option) -> io::Result { + let store = Self::load_with_policy(data_dir, LoadPolicy::ReadOnly)?; + Ok(store.values[WINDOWS_GPU_DEVICE_ID] + .as_str() + .unwrap_or_default() + .to_owned()) + } + + fn load_with_policy(data_dir: Option, policy: LoadPolicy) -> io::Result { + let path = data_dir + .unwrap_or_else(default_data_dir) + .join("settings.json"); + let defaults = defaults(); + let mut values = defaults.clone(); + let mut passthrough = Map::new(); + let mut migrate_onboarding = false; + let mut recovered_backup = false; + let backup = path.with_extension("json.bak"); + if path.exists() || backup.exists() { + let persisted = read_persisted_settings(&path, policy).or_else(|| { + let persisted = read_persisted_settings(&backup, policy); + recovered_backup = persisted.is_some(); + persisted + }); + match persisted { + Some(persisted) => { + if !persisted.contains_key("webrtcCompatibilityMode") + && persisted.get("allianceWebrtcCompatibility") == Some(&json!(true)) + { + values.insert("webrtcCompatibilityMode".to_owned(), json!("on")); + } + if !persisted.contains_key("startAsLastUsedAccount") + && persisted.get("consoleProfilePickerOnLaunch") == Some(&json!(false)) + { + values.insert("startAsLastUsedAccount".to_owned(), json!(true)); + } + migrate_onboarding = policy == LoadPolicy::ReadWrite + && !persisted.contains_key("onboardingCompleted"); + if migrate_onboarding { + values.insert("onboardingCompleted".to_owned(), json!(true)); + } + for (key, value) in persisted { + if defaults.contains_key(&key) { + let value = if key == "gameCollections" { + match normalize_game_collections(value) { + Ok(value) => value, + Err(_) if policy == LoadPolicy::ReadOnly => { + defaults["gameCollections"].clone() + } + Err(error) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + error, + )); + } + } + } else if policy == LoadPolicy::ReadWrite && key == "mouseAcceleration" + { + value.as_bool().map_or(value.clone(), |enabled| { + Value::Number((if enabled { 100 } else { 1 }).into()) + }) + } else { + value + }; + values.insert(key, value); + } else if !matches!(key.as_str(), "nativeHdrSupported" | "nativeHdrDisplay") + { + if policy == LoadPolicy::ReadWrite + && key == "sessionTimeRemainingDisplay" + && matches!(value.as_str(), Some("stats" | "both")) + { + values.insert( + "showSessionTimeRemainingInStatsOverlay".to_owned(), + Value::Bool(true), + ); + } + passthrough.insert(key, value); + } + } + } + None => { + if policy == LoadPolicy::ReadWrite { + let corrupt_path = path.with_extension("json.corrupt"); + let _ = fs::rename(&path, corrupt_path); + } + } + } + } + let mut store = Self { + path, + values, + passthrough, + }; + if policy == LoadPolicy::ReadWrite { + store.migrate_native_fullscreen_shortcut(); + } + // Old builds enabled automatic switching by default, so an existing + // true value is not reliable evidence of opt-in. Reset that policy once; + // subsequent explicit opt-ins survive every restart. + let migrate_console_policy = policy == LoadPolicy::ReadWrite + && store.passthrough.get(CONSOLE_POLICY_VERSION) != Some(&json!(1)); + if migrate_console_policy { + store + .values + .insert("switchToConsoleOnPad".to_owned(), json!(false)); + store + .passthrough + .insert(CONSOLE_POLICY_VERSION.to_owned(), json!(1)); + } + let codec_before_normalize = store.values["codec"].clone(); + let fallback_before_normalize = store.values["fallbackCodec"].clone(); + store.normalize(); + // Persist a first-launch codec/color heal: profiles saved before the + // settings page greyed out unsupported combinations are repaired toward + // Auto above; write the repaired values back so the fix sticks. + let codec_color_healed = policy == LoadPolicy::ReadWrite + && (store.values["codec"] != codec_before_normalize + || store.values["fallbackCodec"] != fallback_before_normalize); + if policy == LoadPolicy::ReadWrite + && (recovered_backup + || ((migrate_console_policy || migrate_onboarding || codec_color_healed) + && store.path.exists())) + { + store.save()?; + } + Ok(store) + } + + pub fn all(&self) -> Value { + Value::Object(self.values.clone()) + } + + pub fn set(&mut self, key: &str, mut value: Value) -> Result { + if matches!(key, "providerRegions" | "regionProviderIdpId") { + return Err("Provider region metadata is managed with the selected region".into()); + } + if !defaults().contains_key(key) { + return Err(format!("Unknown setting: {key}")); + } + if matches!(key, "codec" | "fallbackCodec") { + // Reject only recognized explicit codecs that the saved color mode + // cannot use. Unknown spellings fall through to normalize_choice, + // which clamps them to Auto. + if let Some(codec) = value.as_str() { + let name = codec.trim().to_ascii_lowercase(); + let known_explicit = + matches!(name.as_str(), "h264" | "avc" | "h265" | "hevc" | "av1"); + if known_explicit { + let color = self.values["colorQuality"].as_str().unwrap_or("8bit_420"); + if !crate::streamer::codec_supports_color_quality(&name, color) { + return Err(format!( + "{codec} cannot request {color}. Select Auto or H.265 for advanced color." + )); + } + } + } + } + if matches!(key, "gameLanguage" | "keyboardLayout") { + crate::language::validate_setting(key, &value)?; + } + if key == "audioOutputDevice" { + validate_bounded_string(&value, key, 1024)?; + } + if key == WINDOWS_GPU_DEVICE_ID { + validate_bounded_string(&value, key, MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES)?; + } + if key == "gameCollections" { + value = normalize_game_collections(value)?; + } + if key == "sessionProxyUrl" { + let raw = value + .as_str() + .ok_or_else(|| "sessionProxyUrl must be a string".to_owned())? + .trim(); + value = if raw.is_empty() { + Value::String(String::new()) + } else { + Value::String(normalize_proxy_url(raw)?.normalized_url) + }; + } + if key == "sessionProxyEnabled" && value.as_bool() == Some(true) { + let raw = self.values["sessionProxyUrl"].as_str().unwrap_or(""); + normalize_proxy_url(raw)?; + } + let previous_values = self.values.clone(); + if key == "allianceWebrtcCompatibility" { + self.values.insert( + "webrtcCompatibilityMode".to_owned(), + json!(if value == true { "on" } else { "off" }), + ); + } + self.values.insert(key.to_owned(), value); + self.normalize(); + if key == "microphoneMode" && self.values.get(key) == Some(&json!("voice-activity")) { + self.values + .insert("microphoneDeviceId".to_owned(), json!("")); + } + if key == "themePack" { + let light = matches!(self.values[key].as_str(), Some("bone" | "cobalt")); + self.values.insert( + "appTheme".to_owned(), + json!(if light { "light" } else { "dark" }), + ); + self.values + .insert("themeAccentOverride".to_owned(), json!(false)); + } else if key == "appAccentColor" { + self.values + .insert("themeAccentOverride".to_owned(), json!(true)); + } + if key == "launchInConsoleMode" && self.values.get(key) == Some(&json!(false)) { + self.values + .insert("switchToConsoleOnPad".to_owned(), json!(false)); + } + if let Err(error) = self.save() { + self.values = previous_values; + return Err(format!("Could not save settings: {error}")); + } + Ok(self.values.get(key).cloned().unwrap_or(Value::Null)) + } + + pub fn reset(&mut self) -> Result { + let previous_values = self.values.clone(); + self.values = defaults(); + self.values.insert( + "onboardingCompleted".to_owned(), + previous_values["onboardingCompleted"].clone(), + ); + self.normalize(); + if let Err(error) = self.save() { + self.values = previous_values; + return Err(format!("Could not reset settings: {error}")); + } + Ok(self.all()) + } + + pub fn set_shortcuts(&mut self, bindings: &Value) -> Result, String> { + let bindings = bindings + .as_object() + .filter(|bindings| !bindings.is_empty()) + .ok_or_else(|| "Shortcut bindings must be a non-empty object".to_owned())?; + let mut applied = Map::new(); + for (key, value) in bindings { + if !SHORTCUT_KEYS.contains(&key.as_str()) { + return Err(format!("Unknown shortcut setting: {key}")); + } + let chord = value + .as_str() + .ok_or_else(|| format!("{key} must be a string"))? + .trim(); + if chord.len() > MAXIMUM_SHORTCUT_BYTES { + return Err(format!("{key} is too long")); + } + if RESERVED_SHORTCUTS + .iter() + .any(|reserved| canonical_shortcut(reserved) == canonical_shortcut(chord)) + { + return Err(format!("{chord} is reserved")); + } + applied.insert(key.clone(), Value::String(chord.to_owned())); + } + let chord_of = |key: &str| { + applied + .get(key) + .or_else(|| self.values.get(key)) + .and_then(Value::as_str) + .map(canonical_shortcut) + .unwrap_or_default() + }; + for changed in applied.keys() { + let chord = chord_of(changed); + if chord.is_empty() { + continue; + } + if let Some(owner) = SHORTCUT_KEYS + .iter() + .find(|key| **key != changed.as_str() && chord_of(key) == chord) + { + return Err(format!( + "{} is assigned to both {changed} and {owner}", + applied[changed].as_str().unwrap_or_default() + )); + } + } + let previous_values = self.values.clone(); + for (key, value) in &applied { + self.values.insert(key.clone(), value.clone()); + } + self.normalize(); + if let Err(error) = self.save() { + self.values = previous_values; + return Err(format!("Could not save settings: {error}")); + } + Ok(applied + .keys() + .map(|key| (key.clone(), self.values[key].clone())) + .collect()) + } + + pub fn set_provider_region(&mut self, provider: &str, value: Value) -> Result { + validate_bounded_string(&value, "region", 256)?; + if provider.is_empty() || provider.len() > 256 { + return Err("Invalid region provider".into()); + } + let previous_values = self.values.clone(); + let mut regions = self.values["providerRegions"] + .as_object() + .cloned() + .unwrap_or_default(); + if !regions.contains_key(provider) && regions.len() >= 32 { + return Err("Too many saved region providers".into()); + } + regions.insert(provider.to_owned(), value.clone()); + self.values + .insert("providerRegions".into(), Value::Object(regions)); + self.values + .insert("regionProviderIdpId".into(), json!(provider)); + self.values.insert("region".into(), value.clone()); + if let Err(error) = self.save() { + self.values = previous_values; + return Err(format!("Could not save region: {error}")); + } + Ok(value) + } + + fn normalize(&mut self) { + normalize_types(&mut self.values); + normalize_recent_ids(&mut self.values, HDR_NOTICE_SHOWN, MAXIMUM_HDR_NOTICE_SHOWN); + if self.values["audioOutputDevice"] + .as_str() + .is_some_and(|device| device.len() > 1024 || device.contains('\0')) + { + self.values + .insert("audioOutputDevice".to_owned(), json!("")); + } + if self.values[WINDOWS_GPU_DEVICE_ID] + .as_str() + .is_some_and(|device| { + device.len() > MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES || device.contains('\0') + }) + { + self.values + .insert(WINDOWS_GPU_DEVICE_ID.to_owned(), json!("")); + } + normalize_resolution(&mut self.values); + normalize_choice( + &mut self.values, + "desktopBackground", + &["art", "gradient", "solid", "custom"], + "art", + ); + clamp_integer(&mut self.values, "desktopBackgroundOpacity", 0, 100, 30); + let background_image = self.values["desktopBackgroundImage"].as_str().unwrap_or(""); + if !background_image.is_empty() + && (background_image.len() > 8192 + || !url::Url::parse(background_image).is_ok_and(|url| { + url.scheme() == "file" + && url.to_file_path().is_ok() + && url.query().is_none() + && url.fragment().is_none() + })) + { + self.values + .insert("desktopBackgroundImage".to_owned(), json!("")); + } + normalize_choice( + &mut self.values, + "aspectRatio", + &["16:9", "16:10", "21:9", "32:9", "4:3"], + "16:9", + ); + normalize_choice( + &mut self.values, + "recordingResolution", + &["720p", "1080p", "1440p"], + "720p", + ); + normalize_choice(&mut self.values, "streamClientMode", &["native"], "native"); + normalize_choice( + &mut self.values, + "nativeVideoBackend", + &[ + "auto", + "d3d11", + "d3d12", + "nvdec", + "cuda", + "vaapi", + "v4l2", + "vulkan", + "videotoolbox", + "software", + ], + "auto", + ); + for key in ["nativeCloudGsyncMode", "nativeD3dFullscreenMode"] { + normalize_choice( + &mut self.values, + key, + &["auto", "disabled", "forced"], + "auto", + ); + } + self.values.insert( + "transportMode".to_owned(), + Value::String(NATIVE_TRANSPORT.to_owned()), + ); + normalize_choice( + &mut self.values, + "webrtcCompatibilityMode", + &["auto", "on", "off"], + "auto", + ); + self.values.insert( + "allianceWebrtcCompatibility".to_owned(), + json!(self.values["webrtcCompatibilityMode"] == "on"), + ); + normalize_choice( + &mut self.values, + "codec", + &["auto", "av1", "h264", "h265"], + "auto", + ); + normalize_choice( + &mut self.values, + "fallbackCodec", + &["auto", "h264", "h265"], + "auto", + ); + normalize_choice( + &mut self.values, + "colorQuality", + &["8bit_420", "10bit_420", "8bit_444", "10bit_444"], + "8bit_420", + ); + // Keep an explicitly saved codec compatible with the saved color mode, + // mirroring the official settings gating (H.264 is 8-bit 4:2:0 only, AV1 + // is 4:2:0 only). Repairing toward Auto preserves the saved quality + // choice. This heals older profiles on load and re-heals after an + // explicit color change; settings.set rejects newly incompatible + // explicit codec selections outright. + let color = self.values["colorQuality"] + .as_str() + .unwrap_or("8bit_420") + .to_owned(); + for key in ["codec", "fallbackCodec"] { + let codec = self.values[key].as_str().unwrap_or("auto"); + if !crate::streamer::codec_supports_color_quality(codec, &color) { + self.values.insert(key.to_owned(), json!("auto")); + } + } + normalize_choice(&mut self.values, "frameGeneration", &["off", "2x"], "off"); + let dlss_overrides = crate::dlss::normalize(&self.values[crate::dlss::SETTING]); + self.values + .insert(crate::dlss::SETTING.to_owned(), dlss_overrides); + normalize_choice( + &mut self.values, + "upscaling", + &["off", "metalfx", "fsr1"], + "off", + ); + clamp_integer(&mut self.values, "upscalingSharpness", 0, 15, 10); + clamp_integer(&mut self.values, "upscalingDenoise", 0, 20, 0); + for key in ["decoderPreference", "encoderPreference"] { + normalize_choice( + &mut self.values, + key, + &["auto", "hardware", "software"], + "auto", + ); + } + normalize_choice( + &mut self.values, + "automaticBugReports", + &["unset", "enabled", "disabled"], + "unset", + ); + if self.values["errorReportingConsent"] == "denied" + && self.values["automaticBugReports"] == "unset" + { + self.values + .insert("automaticBugReports".to_owned(), json!("disabled")); + } + normalize_choice( + &mut self.values, + "microphoneMode", + &["disabled", "voice-activity"], + "disabled", + ); + normalize_choice( + &mut self.values, + "statsOverlayPosition", + &["bottom-left", "bottom-right", "top-left", "top-right"], + "bottom-left", + ); + normalize_choice( + &mut self.values, + "appTheme", + &["light", "dark", "auto"], + "auto", + ); + normalize_choice( + &mut self.values, + "appLanguage", + &[ + "system", "de", "en", "es", "fr", "ja", "ko", "nl", "pl", "ro", "ru", "tr", "zh", + ], + "system", + ); + normalize_choice( + &mut self.values, + "themePack", + &[ + "default", "nocturne", "aurora", "kraft", "phosphor", "bone", "cobalt", "hibiscus", + "chapel", + ], + "nocturne", + ); + normalize_choice( + &mut self.values, + "appAccentColor", + &["green", "blue", "violet", "amber", "rose", "coral", "white"], + "green", + ); + normalize_choice( + &mut self.values, + "updateChannel", + &["stable", "nightly"], + crate::version::update_channel(crate::version::APPLICATION_VERSION), + ); + clamp_integer(&mut self.values, "mouseAcceleration", 1, 150, 1); + clamp_integer(&mut self.values, "controllerLeftStickDeadzone", 0, 50, 5); + clamp_integer(&mut self.values, "controllerRightStickDeadzone", 0, 50, 5); + clamp_integer( + &mut self.values, + "controllerVibrationIntensity", + 0, + 100, + 100, + ); + clamp_integer(&mut self.values, "fps", 30, 360, 60); + clamp_bitrate_mbps(&mut self.values); + clamp_integer(&mut self.values, "windowWidth", 960, 7680, 1400); + clamp_integer(&mut self.values, "windowHeight", 540, 4320, 900); + clamp_integer(&mut self.values, "recordingFps", 30, 60, 30); + clamp_integer(&mut self.values, "replayBufferSeconds", 15, 120, 30); + clamp_integer(&mut self.values, "replayBufferMemoryMiB", 64, 512, 256); + clamp_integer(&mut self.values, "antiAfkReminderEveryMinutes", 1, 120, 15); + clamp_integer(&mut self.values, "antiAfkReminderDurationSeconds", 1, 60, 5); + clamp_integer(&mut self.values, "sessionClockShowEveryMinutes", 1, 240, 60); + clamp_integer( + &mut self.values, + "sessionClockShowDurationSeconds", + 1, + 120, + 30, + ); + clamp_number(&mut self.values, "posterSizeScale", 0.75, 1.5, 1.05); + clamp_number(&mut self.values, "mouseSensitivity", 0.1, 3.0, 1.0); + clamp_number(&mut self.values, "desktopUiScale", 0.85, 1.25, 1.0); + clamp_number(&mut self.values, "statsOverlayScale", 0.85, 1.5, 1.0); + clamp_number(&mut self.values, "statsOverlayOpacity", 40.0, 100.0, 85.0); + normalize_optional_integer(&mut self.values, "recordingBitrateMbps", 1, 12); + normalize_optional_integer(&mut self.values, "windowX", -32768, 32767); + normalize_optional_integer(&mut self.values, "windowY", -32768, 32767); + normalize_bounded_strings(&mut self.values); + normalize_nested_settings(&mut self.values); + } + + fn migrate_native_fullscreen_shortcut(&mut self) { + let fullscreen = self + .values + .get("shortcutToggleFullscreen") + .and_then(Value::as_str); + let screenshot = self + .values + .get("shortcutScreenshot") + .and_then(Value::as_str); + if fullscreen == Some("F10") && screenshot == Some("F11") { + self.values.insert( + "shortcutToggleFullscreen".to_owned(), + Value::String("F11".to_owned()), + ); + self.values.insert( + "shortcutScreenshot".to_owned(), + Value::String("Ctrl+F11".to_owned()), + ); + if self + .values + .get("statsOverlayPosition") + .and_then(Value::as_str) + == Some("bottom-left") + { + self.values.insert( + "statsOverlayPosition".to_owned(), + Value::String("top-right".to_owned()), + ); + } + } + } + + fn save(&self) -> io::Result<()> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let temporary = self.path.with_extension("json.tmp"); + let backup = self.path.with_extension("json.bak"); + let mut persisted = self.passthrough.clone(); + persisted.extend(self.values.clone()); + let data = serde_json::to_vec_pretty(&persisted).map_err(io::Error::other)?; + let mut file = fs::File::create(&temporary)?; + file.write_all(&data)?; + file.sync_all()?; + drop(file); + if read_persisted_settings(&self.path, LoadPolicy::ReadWrite).is_some() { + fs::copy(&self.path, &backup)?; + fs::OpenOptions::new() + .write(true) + .open(&backup)? + .sync_all()?; + } + fs::rename(&temporary, &self.path)?; + Ok(()) + } +} + +fn read_persisted_settings(path: &Path, policy: LoadPolicy) -> Option> { + match policy { + LoadPolicy::ReadWrite => fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()), + LoadPolicy::ReadOnly => { + let mut data = Vec::new(); + fs::File::open(path) + .ok()? + .take(MAXIMUM_BOOTSTRAP_SETTINGS_BYTES + 1) + .read_to_end(&mut data) + .ok()?; + if data.len() as u64 > MAXIMUM_BOOTSTRAP_SETTINGS_BYTES { + return None; + } + serde_json::from_slice(&data).ok() + } + } +} + +fn validate_bounded_string(value: &Value, key: &str, maximum_bytes: usize) -> Result<(), String> { + let value = value + .as_str() + .ok_or_else(|| format!("{key} must be a string"))?; + if value.len() > maximum_bytes || value.contains('\0') { + return Err(format!( + "{key} must be at most {maximum_bytes} bytes without NUL characters" + )); + } + Ok(()) +} + +fn normalize_game_collections(mut value: Value) -> Result { + let collections = value + .as_array_mut() + .ok_or_else(|| "gameCollections must be an array".to_owned())?; + if collections.len() > 100 { + return Err("gameCollections must contain at most 100 collections".to_owned()); + } + let mut collection_ids = HashSet::new(); + for (index, collection) in collections.iter_mut().enumerate() { + let invalid = || format!("gameCollections[{index}] must contain id, name and gameIds"); + let collection = collection.as_object_mut().ok_or_else(invalid)?; + if collection.len() != 3 { + return Err(format!( + "gameCollections[{index}] must contain only id, name and gameIds" + )); + } + let id = collection + .get("id") + .and_then(Value::as_str) + .ok_or_else(invalid)?; + if id.trim().is_empty() || id.chars().count() > 128 { + return Err(format!( + "gameCollections[{index}].id must be nonempty and at most 128 characters" + )); + } + if !collection_ids.insert(id.to_owned()) { + return Err(format!("gameCollections[{index}].id must be unique")); + } + let name = collection + .get("name") + .and_then(Value::as_str) + .ok_or_else(invalid)? + .trim(); + if name.is_empty() || name.chars().count() > 80 { + return Err(format!( + "gameCollections[{index}].name must be nonempty and at most 80 characters after trimming" + )); + } + let name = name.to_owned(); + let game_ids = collection + .get("gameIds") + .and_then(Value::as_array) + .ok_or_else(invalid)?; + if game_ids.len() > 10_000 { + return Err(format!( + "gameCollections[{index}].gameIds must contain at most 10000 IDs" + )); + } + let mut seen = HashSet::new(); + for game_id in game_ids { + let game_id = game_id + .as_str() + .ok_or_else(|| format!("gameCollections[{index}].gameIds must contain strings"))?; + if game_id.trim().is_empty() || game_id.chars().count() > 128 { + return Err(format!( + "gameCollections[{index}].gameIds must contain nonempty IDs of at most 128 characters" + )); + } + if !seen.insert(game_id) { + return Err(format!( + "gameCollections[{index}].gameIds must contain unique IDs" + )); + } + } + collection.insert("name".to_owned(), Value::String(name)); + } + Ok(value) +} + +fn normalize_resolution(values: &mut Map) { + let valid = values["resolution"] + .as_str() + .and_then(|value| value.split_once('x')) + .and_then(|(width, height)| Some((width.parse::().ok()?, height.parse::().ok()?))) + .is_some_and(|(width, height)| { + (640..=7680).contains(&width) + && (480..=4320).contains(&height) + && width % 2 == 0 + && height % 2 == 0 + }); + if !valid { + values.insert("resolution".to_owned(), json!("1920x1080")); + } +} + +pub fn resolve_data_dir(data_dir: Option) -> PathBuf { + data_dir.unwrap_or_else(|| { + let primary = default_data_dir(); + select_existing_data_dir(primary.clone(), legacy_data_dirs(&primary)) + }) +} + +fn select_existing_data_dir( + primary: PathBuf, + legacy_candidates: impl IntoIterator, +) -> PathBuf { + if primary.exists() { + return primary; + } + legacy_candidates + .into_iter() + .find(|candidate| candidate.exists()) + .unwrap_or(primary) +} + +fn normalize_choice(values: &mut Map, key: &str, choices: &[&str], fallback: &str) { + let valid = values + .get(key) + .and_then(Value::as_str) + .is_some_and(|value| choices.contains(&value)); + if !valid { + values.insert(key.to_owned(), Value::String(fallback.to_owned())); + } +} + +fn clamp_integer( + values: &mut Map, + key: &str, + minimum: i64, + maximum: i64, + fallback: i64, +) { + let value = values + .get(key) + .and_then(Value::as_i64) + .unwrap_or(fallback) + .clamp(minimum, maximum); + values.insert(key.to_owned(), Value::Number(value.into())); +} + +fn clamp_bitrate_mbps(values: &mut Map) { + // 0.22 Mbps is 220 kbps. Whole numbers stay integers so existing settings + // and the 10–200 Mbps slider keep their previous JSON shape. + let raw = values + .get("maxBitrateMbps") + .and_then(Value::as_f64) + .filter(|value| value.is_finite()) + .unwrap_or(75.0); + let value = (raw.clamp(0.22, 200.0) * 100.0).round() / 100.0; + let stored = if (value - value.round()).abs() < 1e-9 { + Value::from(value.round() as i64) + } else { + serde_json::Number::from_f64(value) + .map(Value::Number) + .unwrap_or_else(|| Value::from(75)) + }; + values.insert("maxBitrateMbps".to_owned(), stored); +} + +fn clamp_number( + values: &mut Map, + key: &str, + minimum: f64, + maximum: f64, + fallback: f64, +) { + let value = values + .get(key) + .and_then(Value::as_f64) + .filter(|value| value.is_finite()) + .unwrap_or(fallback) + .clamp(minimum, maximum); + values.insert( + key.to_owned(), + serde_json::Number::from_f64(value).map_or(Value::from(fallback), Value::Number), + ); +} + +fn normalize_types(values: &mut Map) { + let expected = defaults(); + for (key, default) in expected { + let valid = values.get(&key).is_some_and(|value| match &default { + Value::Bool(_) => value.is_boolean(), + Value::String(_) => value.is_string(), + Value::Number(_) => value.is_number(), + Value::Array(_) => value.is_array(), + Value::Object(_) => value.is_object(), + Value::Null => value.is_null() || value.is_number(), + }); + if !valid { + values.insert(key, default); + } + } +} + +fn normalize_recent_ids(values: &mut Map, key: &str, limit: usize) { + let mut ids = Vec::new(); + for id in values[key].as_array().into_iter().flatten().rev() { + if ids.len() == limit { + break; + } + if id + .as_str() + .is_some_and(|id| !id.is_empty() && id.len() <= 256) + && !ids.contains(id) + { + ids.push(id.clone()); + } + } + ids.reverse(); + values.insert(key.to_owned(), Value::Array(ids)); +} + +fn normalize_optional_integer( + values: &mut Map, + key: &str, + minimum: i64, + maximum: i64, +) { + let Some(value) = values.get(key) else { + return; + }; + if value.is_null() { + return; + } + let normalized = value.as_i64().map(|value| value.clamp(minimum, maximum)); + values.insert( + key.to_owned(), + normalized.map_or(Value::Null, |value| Value::Number(value.into())), + ); +} + +fn canonical_shortcut(chord: &str) -> String { + let mut parts = chord + .split('+') + .map(|part| part.trim().to_ascii_lowercase()) + .map(|part| { + if part == "control" { + "ctrl".to_owned() + } else { + part + } + }) + .collect::>(); + let key = parts.pop().unwrap_or_default(); + parts.sort(); + parts.dedup(); + parts.push(key); + parts.join("+") +} + +fn normalize_bounded_strings(values: &mut Map) { + for (key, maximum) in [ + ("region", 256_usize), + ("regionProviderIdpId", 256_usize), + ("sessionProxyUrl", 2_048), + ("nativeStreamerExecutablePath", 2_048), + ("microphoneDeviceId", 512), + ("telemetryInstallId", 128), + ("lastSeenReleaseHighlightsVersion", 128), + ] { + let value = values + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .chars() + .take(maximum) + .collect::(); + values.insert(key.to_owned(), Value::String(value)); + } + for key in SHORTCUT_KEYS { + let value = values + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .chars() + .take(80) + .collect::(); + values.insert(key.to_owned(), Value::String(value)); + } + let favorites = values["favoriteGameIds"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(|value| value.chars().take(128).collect::()) + .filter(|value| !value.is_empty()) + .take(500) + .map(Value::String) + .collect(); + values.insert("favoriteGameIds".to_owned(), Value::Array(favorites)); + + let hidden_games = values["hiddenGameIds"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(|value| value.chars().take(128).collect::()) + .filter(|value| !value.is_empty()) + .take(500) + .map(Value::String) + .collect(); + values.insert("hiddenGameIds".to_owned(), Value::Array(hidden_games)); + + let tile_sizes = values["homeTileSizes"] + .as_object() + .into_iter() + .flat_map(|entries| entries.iter()) + .filter_map(|(id, value)| { + let id = id.chars().take(128).collect::(); + let size = value.as_str()?; + (!id.is_empty() && matches!(size, "square" | "wide")) + .then(|| (id, Value::String(size.to_owned()))) + }) + .take(500) + .collect(); + values.insert("homeTileSizes".to_owned(), Value::Object(tile_sizes)); + + values.insert( + "controllerProfiles".to_owned(), + Value::Object(normalize_controller_profiles(&values["controllerProfiles"])), + ); +} + +fn normalize_controller_profiles(value: &Value) -> Map { + let mut profiles: Vec<(String, Map, i64)> = value + .as_object() + .into_iter() + .flat_map(|entries| entries.iter()) + .filter_map(|(key, profile)| { + let key = key.trim(); + if key.is_empty() || key.chars().count() > MAXIMUM_CONTROLLER_PROFILE_KEY_CHARS { + return None; + } + let profile = profile.as_object()?; + let mut normalized = Map::new(); + for (field, maximum) in CONTROLLER_PROFILE_TUNING { + if let Some(value) = profile.get(field).and_then(Value::as_i64) { + normalized.insert(field.to_owned(), json!(value.clamp(0, maximum))); + } + } + if normalized.is_empty() { + return None; + } + if let Some(name) = profile.get("name").and_then(Value::as_str) { + let name: String = name + .trim() + .chars() + .take(MAXIMUM_CONTROLLER_PROFILE_NAME_CHARS) + .collect(); + if !name.is_empty() { + normalized.insert("name".to_owned(), Value::String(name)); + } + } + let last_seen = profile + .get("lastSeen") + .and_then(Value::as_i64) + .unwrap_or(0) + .max(0); + normalized.insert("lastSeen".to_owned(), json!(last_seen)); + Some((key.to_owned(), normalized, last_seen)) + }) + .collect(); + profiles.sort_by(|left, right| right.2.cmp(&left.2).then_with(|| left.0.cmp(&right.0))); + profiles + .into_iter() + .take(MAXIMUM_CONTROLLER_PROFILES) + .map(|(key, profile, _)| (key, Value::Object(profile))) + .collect() +} + +fn normalize_nested_settings(values: &mut Map) { + let mut interpolation = values["frameInterpolation"] + .as_object() + .cloned() + .unwrap_or_default(); + let enabled = interpolation + .get("enabled") + .and_then(Value::as_bool) + .unwrap_or(false); + let factor = interpolation + .get("factor") + .and_then(Value::as_i64) + .unwrap_or(2) + .clamp(2, 4); + let quality = interpolation + .get("quality") + .and_then(Value::as_i64) + .unwrap_or(480) + .clamp(120, 1_080); + interpolation.clear(); + interpolation.insert("enabled".to_owned(), Value::Bool(enabled)); + interpolation.insert("factor".to_owned(), Value::Number(factor.into())); + interpolation.insert("quality".to_owned(), Value::Number(quality.into())); + values.insert( + "frameInterpolation".to_owned(), + Value::Object(interpolation), + ); + + let mut shader = values["videoShader"] + .as_object() + .cloned() + .unwrap_or_default(); + let mut normalized = Map::new(); + normalized.insert( + "enabled".to_owned(), + Value::Bool( + shader + .remove("enabled") + .and_then(|value| value.as_bool()) + .unwrap_or(false), + ), + ); + for (key, fallback, minimum, maximum) in [ + ("sharpen", 40, 0, 100), + ("saturation", 100, 0, 200), + ("contrast", 100, 0, 200), + ("brightness", 100, 0, 200), + ("vibrance", 0, -100, 100), + ("filmGrain", 0, 0, 100), + ] { + let value = shader + .remove(key) + .and_then(|value| value.as_i64()) + .unwrap_or(fallback) + .clamp(minimum, maximum); + normalized.insert(key.to_owned(), Value::Number(value.into())); + } + values.insert("videoShader".to_owned(), Value::Object(normalized)); +} + +fn default_data_dir() -> PathBuf { + if let Some(path) = env::var_os("OPENNOW_DATA_DIR") { + return PathBuf::from(path); + } + #[cfg(target_os = "windows")] + if let Some(path) = env::var_os("APPDATA") { + return PathBuf::from(path).join("OpenNOW"); + } + #[cfg(target_os = "macos")] + if let Some(path) = env::var_os("HOME") { + return PathBuf::from(path).join("Library/Application Support/OpenNOW"); + } + if let Some(path) = env::var_os("XDG_CONFIG_HOME") { + return PathBuf::from(path).join("OpenNOW"); + } + env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")) + .join(".config/OpenNOW") +} + +fn legacy_data_dirs(primary: &Path) -> Vec { + #[cfg(target_os = "linux")] + { + let mut candidates = Vec::new(); + if let Some(parent) = primary.parent() { + candidates.push(parent.join("opennow")); + } + candidates + } + #[cfg(not(target_os = "linux"))] + { + let _ = primary; + Vec::new() + } +} + +fn defaults() -> Map { + json!({ + "onboardingCompleted":false, + "resolution":"1920x1080", "aspectRatio":"16:9", "posterSizeScale":1.05, + "fps":60, "frameGeneration":"off", "upscaling":"off", "upscalingSharpness":10, "upscalingDenoise":0, + "maxBitrateMbps":75, "saveBandwidth":false, "recordingBitrateMbps":null, + "recordingResolution":"720p", "recordingFps":30, "streamClientMode":"native", + "replayBufferEnabled":false, "replayBufferSeconds":30, "replayBufferMemoryMiB":256, + "nativeVideoBackend":"auto", "nativeStreamerExecutablePath":"", "audioOutputDevice":"", + "windowsGpuDeviceId":"", + "nativeCloudGsyncMode":"auto", "nativeD3dFullscreenMode":"auto", + "nativeExternalRenderer":false, "transportMode":"nvst", "allianceWebrtcCompatibility":false, "webrtcCompatibilityMode":"auto", "showNativeStreamerStats":false, + "codec":"auto", "fallbackCodec":"auto", "decoderPreference":"auto", + "encoderPreference":"auto", "colorQuality":"8bit_420", "enableHdr":false, "region":"", "regionProviderIdpId":"", "providerRegions":{}, + "suppressTenBitWarning":false, + "sessionProxyEnabled":false, "sessionProxyUrl":"", "clipboardPaste":false, "networkTest":false, + "suppressAwdlWhileStreaming":false, + "enableGyroscopeControls":false, "steamControllerCompatibilityMode":false, + "nativeCursorOverlay":true, "mouseSensitivity":1, "mouseAcceleration":1, + "shortcutToggleStats":"Ctrl+N", "shortcutTogglePointerLock":"F8", + "shortcutToggleFullscreen":"F11", "shortcutStopStream":"Ctrl+Shift+Q", + "shortcutToggleAntiAfk":"Ctrl+Shift+K", "shortcutToggleMicrophone":"Ctrl+Shift+M", + "shortcutScreenshot":"Ctrl+F11", "shortcutToggleRecording":"F12", + "shortcutSaveClip":"Ctrl+F12", + "microphoneMode":"disabled", "microphoneDeviceId":"", "hideStreamButtons":false, + "muteWhenOutOfFocus":false, "backgroundStreamReminder":false, + "showAntiAfkIndicator":true, "antiAfkReminderEveryMinutes":15, + "antiAfkReminderDurationSeconds":5, "showStatsOnLaunch":false, + "statsOverlayPosition":"top-right", "hideServerSelector":false, "hideQueueSelector":false, + "desktopUiScale":1.0, "statsOverlayScale":1.0, "statsOverlayOpacity":85, + "themeAccentOverride":false, + "statsShowFps":true, "statsShowRegion":true, "statsShowPing":true, + "statsShowBitrate":true, "statsShowJitter":true, "statsShowDrops":true, + "statsShowPacketLoss":true, "statsShowDecode":true, "statsShowLatency":true, + "statsShowVideo":true, "statsShowClock":true, "statsShowGraphs":true, + "appAccentColor":"green", "appTheme":"auto", "appLanguage":"system", "themePack":"nocturne", "translucentUI":false, + "showTileLabels":true, + "controllerMode":true, "controllerModePromptDismissed":false, "forwardFlightControls":true, + "controllerLeftStickDeadzone":5, "controllerRightStickDeadzone":5, + "controllerVibrationIntensity":100, "controllerProfiles":{}, + "reducedMotion":false, "uiSoundsEnabled":true, "confirmQuitWhenIdle":true, + "launchInConsoleMode":false, "startAsLastUsedAccount":false, + "desktopRailCollapsed":true, "desktopSidebarHover":true, "desktopBackground":"art", + "desktopBackgroundImage":"", "desktopBackgroundOpacity":30, + "switchToConsoleOnPad":false, "leaveConsoleOnPointer":true, + "autoFullScreen":true, "favoriteGameIds":[], "hiddenGameIds":[], "hdrNoticeShownGameIds":[], "gameCollections":[], "homeTileSizes":{}, "sessionCounterEnabled":false, + "showSessionReport":true, "showSessionTimeRemainingInStatsOverlay":false, + "sessionClockShowEveryMinutes":60, "sessionClockShowDurationSeconds":30, + "windowX":null, "windowY":null, "windowWidth":1400, "windowHeight":900, "keyboardLayout":"en-US", + "gameLanguage":"en_US", "enablePersistingInGameSettings":true, "enableL4S":false, + "identifyAsSteamDeck":false, "steamBigPictureMode":false, + "enableCloudGsync":false, "discordRichPresence":false, + "autoCheckForUpdates":true, "autoDownloadUpdates":false, + "updateChannel":crate::version::update_channel(crate::version::APPLICATION_VERSION), + "allowEscapeToExitFullscreen":false, "lastSeenReleaseHighlightsVersion":"", + "videoShader":{"enabled":false,"sharpen":40,"saturation":100,"contrast":100,"brightness":100,"vibrance":0,"filmGrain":0}, + "frameInterpolation":{"enabled":false,"factor":2,"quality":480}, + "errorReportingConsent":"unset", "automaticBugReports":"unset", "telemetryInstallId":"" + }) + .as_object() + .cloned() + .map(|mut values| { + values.insert( + crate::dlss::SETTING.to_owned(), + crate::dlss::default_overrides(), + ); + values + }) + .expect("settings defaults are an object") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn hdr_notice_history_keeps_the_newest_two_hundred_distinct_ids() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(store.all()[HDR_NOTICE_SHOWN], json!([])); + let mut ids: Vec = (0..205).map(|id| json!(id.to_string())).collect(); + ids.extend([json!("7"), json!(""), json!(42), json!("x".repeat(257))]); + store.set(HDR_NOTICE_SHOWN, Value::Array(ids)).unwrap(); + let kept = store.all()[HDR_NOTICE_SHOWN].as_array().unwrap().clone(); + assert_eq!(kept.len(), MAXIMUM_HDR_NOTICE_SHOWN); + assert_eq!(kept.first(), Some(&json!("5"))); + assert_eq!(kept.last(), Some(&json!("7"))); + assert!(!kept.contains(&json!("4"))); + assert_eq!(kept.iter().filter(|id| **id == "7").count(), 1); + let reloaded = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(reloaded.all()[HDR_NOTICE_SHOWN], Value::Array(kept)); + } + + #[test] + fn settings_backup_recovers_missing_and_corrupt_primary_without_bootstrap_writes() { + for corrupt in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let backup = path.with_extension("json.bak"); + let temporary = path.with_extension("json.tmp"); + let mut original = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + original + .set("windowsGpuDeviceId", json!("fixture-gpu")) + .unwrap(); + original.set("windowWidth", json!(1600)).unwrap(); + let expected = original.all(); + let bytes = fs::read(&path).unwrap(); + fs::rename(&path, &backup).unwrap(); + fs::write(&temporary, b"interrupted write").unwrap(); + if corrupt { + fs::write(&path, b"{").unwrap(); + } + assert_eq!( + SettingsStore::windows_gpu_device_id_read_only(Some(directory.path().to_owned())) + .unwrap(), + "fixture-gpu" + ); + assert_eq!(fs::read(&backup).unwrap(), bytes); + assert_eq!(fs::read(&temporary).unwrap(), b"interrupted write"); + assert!(!path.with_extension("json.corrupt").exists()); + if corrupt { + assert_eq!(fs::read(&path).unwrap(), b"{"); + } else { + assert!(!path.exists()); + } + let restored = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(restored.all(), expected); + assert_eq!(fs::read(&backup).unwrap(), bytes); + assert_eq!( + SettingsStore::load(Some(directory.path().to_owned())) + .unwrap() + .all(), + expected + ); + } + } + + #[test] + fn window_position_persists_across_monitors_and_rejects_non_integers() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(store.all()["windowX"], Value::Null); + assert_eq!(store.all()["windowY"], Value::Null); + assert_eq!(store.set("windowX", json!(-1920)).unwrap(), json!(-1920)); + assert_eq!(store.set("windowY", json!(99999)).unwrap(), json!(32767)); + let reloaded = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(reloaded.all()["windowX"], json!(-1920)); + assert_eq!(reloaded.all()["windowY"], json!(32767)); + assert_eq!(store.set("windowX", json!("left")).unwrap(), Value::Null); + } + + #[test] + fn settings_recovery_failure_keeps_the_valid_backup() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let backup = path.with_extension("json.bak"); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + store.set("windowWidth", json!(1600)).unwrap(); + let bytes = fs::read(&path).unwrap(); + fs::rename(&path, &backup).unwrap(); + fs::create_dir(path.with_extension("json.tmp")).unwrap(); + assert!(SettingsStore::load(Some(directory.path().to_owned())).is_err()); + assert!(!path.exists()); + assert_eq!(fs::read(&backup).unwrap(), bytes); + } + + #[test] + fn settings_backup_failure_keeps_the_primary_and_memory_unchanged() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + store.set("windowWidth", json!(1600)).unwrap(); + let expected = store.all(); + let bytes = fs::read(&path).unwrap(); + fs::create_dir(path.with_extension("json.bak")).unwrap(); + assert!(store.set("windowWidth", json!(1800)).is_err()); + assert_eq!(store.all(), expected); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[test] + fn settings_bootstrap_does_not_recover_invalid_or_oversized_backups() { + let directory = tempfile::tempdir().unwrap(); + let backup = directory.path().join("settings.json.bak"); + for bytes in [ + b"{".to_vec(), + vec![b' '; MAXIMUM_BOOTSTRAP_SETTINGS_BYTES as usize + 1], + ] { + fs::write(&backup, &bytes).unwrap(); + assert_eq!( + SettingsStore::windows_gpu_device_id_read_only(Some(directory.path().to_owned())) + .unwrap(), + "" + ); + assert_eq!(fs::read(&backup).unwrap(), bytes); + assert!(!directory.path().join("settings.json").exists()); + } + } + + #[test] + fn language_preferences_are_independent_and_rejected_writes_are_atomic() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + for (key, value) in [ + ("appLanguage", "ja"), + ("gameLanguage", "es_419"), + ("keyboardLayout", "ja-JP"), + ] { + store.set(key, json!(value)).unwrap(); + } + let saved = store.all(); + for (key, value) in [ + ("gameLanguage", "auto"), + ("gameLanguage", "system"), + ("gameLanguage", "en\nUS"), + ("keyboardLayout", "en_US"), + ("keyboardLayout", "m-us"), + ] { + assert!(store.set(key, json!(value)).is_err()); + assert_eq!(store.all(), saved); + } + assert_eq!( + SettingsStore::load(Some(directory.path().to_path_buf())) + .unwrap() + .all(), + saved + ); + store.set("appLanguage", json!("de")).unwrap(); + assert_eq!(store.all()["gameLanguage"], "es_419"); + assert_eq!(store.all()["keyboardLayout"], "ja-JP"); + store.set("gameLanguage", json!("future_001")).unwrap(); + assert_eq!(store.all()["appLanguage"], "de"); + assert_eq!(store.all()["keyboardLayout"], "ja-JP"); + std::fs::create_dir(store.path.with_extension("json.tmp")).unwrap(); + let saved = store.all(); + assert!(store.set("gameLanguage", json!("pt_BR")).is_err()); + assert_eq!(store.all(), saved); + } + + #[test] + fn restored_language_ids_are_not_rewritten_but_corrupt_values_never_reach_requests() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("settings.json"), + json!({ + "gameLanguage":"auto", "keyboardLayout":"unknown", "appLanguage":"fr", + "onboardingCompleted":true, "qtConsoleModePolicyVersion":1 + }) + .to_string(), + ) + .unwrap(); + let settings = SettingsStore::load(Some(directory.path().to_path_buf())) + .unwrap() + .all(); + assert_eq!(settings["gameLanguage"], "auto"); + assert_eq!(settings["keyboardLayout"], "unknown"); + let mut url = url::Url::parse("https://fixture.invalid/").unwrap(); + crate::language::append_session_preferences(&mut url, &settings); + assert_eq!(url.query(), Some("keyboardLayout=en-US&languageCode=en_US")); + } + + #[test] + fn alliance_webrtc_setting_is_default_off_boolean_and_independent_of_legacy_mode() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(store.all()["allianceWebrtcCompatibility"], false); + store.set("transportMode", json!("webrtc")).unwrap(); + assert_eq!(store.all()["transportMode"], "nvst"); + assert_eq!(store.all()["allianceWebrtcCompatibility"], false); + assert_eq!( + store + .set("allianceWebrtcCompatibility", json!("true")) + .unwrap(), + false + ); + store + .set("allianceWebrtcCompatibility", json!(true)) + .unwrap(); + let restored = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(restored.all()["allianceWebrtcCompatibility"], true); + assert_eq!(restored.all()["transportMode"], "nvst"); + } + + #[test] + fn automatic_webrtc_mode_preserves_manual_overrides_and_migrates_old_opt_in() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(store.all()["webrtcCompatibilityMode"], "auto"); + for mode in ["off", "on", "auto"] { + assert_eq!( + store.set("webrtcCompatibilityMode", json!(mode)).unwrap(), + mode + ); + let restored = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(restored.all()["webrtcCompatibilityMode"], mode); + assert_eq!(restored.all()["allianceWebrtcCompatibility"], mode == "on"); + } + for invalid in [json!(true), json!("unexpected"), Value::Null] { + assert_eq!( + store.set("webrtcCompatibilityMode", invalid).unwrap(), + "auto" + ); + } + for (legacy, expected) in [(true, "on"), (false, "auto")] { + std::fs::write( + directory.path().join("settings.json"), + json!({ + "allianceWebrtcCompatibility":legacy,"onboardingCompleted":true, + "qtConsoleModePolicyVersion":1 + }) + .to_string(), + ) + .unwrap(); + let mut restored = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(restored.all()["webrtcCompatibilityMode"], expected); + restored + .set("webrtcCompatibilityMode", json!("off")) + .unwrap(); + let again = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(again.all()["webrtcCompatibilityMode"], "off"); + } + } + + #[test] + fn provider_region_preferences_are_atomic_isolated_and_persisted() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + store + .set_provider_region("nvidia", json!("https://nvidia-region.nvidiagrid.net/")) + .unwrap(); + store + .set_provider_region("alliance", json!("https://alliance-region.nvidiagrid.net/")) + .unwrap(); + let restored = SettingsStore::load(Some(directory.path().to_path_buf())) + .unwrap() + .all(); + assert_eq!( + restored["providerRegions"]["nvidia"], + "https://nvidia-region.nvidiagrid.net/" + ); + assert_eq!(restored["providerRegions"]["alliance"], restored["region"]); + assert_eq!(restored["regionProviderIdpId"], "alliance"); + assert!(store.set("providerRegions", json!({})).is_err()); + assert!( + store + .set_provider_region("alliance", json!("x".repeat(257))) + .is_err() + ); + assert_eq!(store.all(), restored); + std::fs::create_dir(store.path.with_extension("json.tmp")).unwrap(); + assert!( + store + .set_provider_region("nvidia", json!("changed")) + .is_err() + ); + assert_eq!(store.all(), restored); + } + + #[test] + fn updater_preferences_are_independent_and_preserved() { + let directory = + env::temp_dir().join(format!("opennow-update-settings-{}", rand::random::())); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["autoDownloadUpdates"], json!(false)); + assert_eq!( + store.all()["updateChannel"], + json!(crate::version::update_channel( + crate::version::APPLICATION_VERSION + )) + ); + store.set("autoCheckForUpdates", json!(false)).unwrap(); + store.set("autoDownloadUpdates", json!(true)).unwrap(); + store.set("updateChannel", json!("stable")).unwrap(); + let settings = SettingsStore::load(Some(directory.clone())).unwrap().all(); + assert_eq!(settings["autoCheckForUpdates"], json!(false)); + assert_eq!(settings["autoDownloadUpdates"], json!(true)); + assert_eq!(settings["updateChannel"], json!("stable")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_new_profile_stays_incomplete_across_unrelated_writes() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-new-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(false)); + assert!(!directory.join("settings.json").exists()); + for (key, value) in [ + ("launchInConsoleMode", json!(true)), + ("windowWidth", json!(1600)), + ("windowHeight", json!(1000)), + ] { + store.set(key, value).unwrap(); + store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(false)); + } + store.set("onboardingCompleted", json!(true)).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["onboardingCompleted"], + json!(true) + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_existing_profiles_migrate_and_persist_completion() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-existing-{unique}")); + fs::create_dir_all(&directory).unwrap(); + for persisted in [ + json!({}), + json!({"qtConsoleModePolicyVersion": 1, "switchToConsoleOnPad": true}), + ] { + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&persisted).unwrap(), + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(true)); + if persisted.get("qtConsoleModePolicyVersion").is_some() { + assert_eq!(store.all()["switchToConsoleOnPad"], json!(true)); + } + let saved: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()) + .unwrap(); + assert_eq!(saved["onboardingCompleted"], json!(true)); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["onboardingCompleted"], + json!(true) + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_explicit_values_survive_migration_and_reset() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-reset-{unique}")); + fs::create_dir_all(&directory).unwrap(); + for completed in [false, true] { + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"onboardingCompleted": completed})).unwrap(), + ) + .unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(completed)); + store.set("windowWidth", json!(1600)).unwrap(); + store.set("launchInConsoleMode", json!(true)).unwrap(); + store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(completed)); + let reset = store.reset().unwrap(); + assert_eq!(reset["onboardingCompleted"], json!(completed)); + assert_eq!(reset["windowWidth"], json!(1400)); + assert_eq!(reset["launchInConsoleMode"], json!(false)); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all(), + reset + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_malformed_files_are_new_profiles() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-corrupt-{unique}")); + fs::create_dir_all(&directory).unwrap(); + for contents in ["{", "null", "[]", "true"] { + fs::write(directory.join("settings.json"), contents).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(false)); + assert_eq!( + fs::read_to_string(directory.join("settings.json.corrupt")).unwrap(), + contents + ); + fs::remove_file(directory.join("settings.json.corrupt")).unwrap(); + store.set("windowWidth", json!(1600)).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["onboardingCompleted"], + json!(false) + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn awdl_suppression_is_opt_in_boolean_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-awdl-setting-{unique}")); + fs::create_dir_all(&directory).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["suppressAwdlWhileStreaming"], json!(false)); + assert_eq!( + store + .set("suppressAwdlWhileStreaming", json!(true)) + .unwrap(), + json!(true) + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["suppressAwdlWhileStreaming"], + json!(true) + ); + for invalid in [json!("true"), json!(1), json!(null), json!([]), json!({})] { + assert_eq!( + store.set("suppressAwdlWhileStreaming", invalid).unwrap(), + json!(false) + ); + } + store + .set("suppressAwdlWhileStreaming", json!(true)) + .unwrap(); + store.reset().unwrap(); + assert_eq!(store.all()["suppressAwdlWhileStreaming"], json!(false)); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn flight_control_forwarding_defaults_on_and_persists_boolean_choices() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_path_buf())).unwrap(); + assert_eq!(store.all()["forwardFlightControls"], json!(true)); + assert_eq!( + store.set("forwardFlightControls", json!(false)).unwrap(), + json!(false) + ); + assert_eq!( + SettingsStore::load(Some(directory.path().to_path_buf())) + .unwrap() + .all()["forwardFlightControls"], + json!(false) + ); + for invalid in [json!("false"), json!(0), json!(null), json!([]), json!({})] { + assert_eq!( + store.set("forwardFlightControls", invalid).unwrap(), + json!(true) + ); + } + store.set("forwardFlightControls", json!(false)).unwrap(); + store.reset().unwrap(); + assert_eq!(store.all()["forwardFlightControls"], json!(true)); + } + + #[test] + fn onboarding_requires_boolean_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-types-{unique}")); + fs::create_dir_all(&directory).unwrap(); + for invalid in [json!("true"), json!(1), json!(null), json!([]), json!({})] { + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"onboardingCompleted": invalid})).unwrap(), + ) + .unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["onboardingCompleted"], json!(false)); + store.set("onboardingCompleted", json!(true)).unwrap(); + assert_eq!( + store.set("onboardingCompleted", invalid).unwrap(), + json!(false) + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["onboardingCompleted"], + json!(false) + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn queue_selector_preference_is_typed_persisted_and_resettable() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(store.all()["hideQueueSelector"], false); + store.set("hideQueueSelector", json!(true)).unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(store.all()["hideQueueSelector"], true); + for invalid in [json!("true"), json!(1), json!(null), json!([])] { + assert_eq!(store.set("hideQueueSelector", invalid).unwrap(), false); + } + store.set("hideQueueSelector", json!(true)).unwrap(); + assert_eq!(store.reset().unwrap()["hideQueueSelector"], false); + } + + #[test] + fn background_stream_preferences_are_opt_in_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-background-stream-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for key in ["muteWhenOutOfFocus", "backgroundStreamReminder"] { + assert_eq!(store.all()[key], json!(false)); + store.set(key, json!(true)).unwrap(); + } + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for key in ["muteWhenOutOfFocus", "backgroundStreamReminder"] { + assert_eq!(store.all()[key], json!(true)); + for invalid in [json!("true"), json!(1), json!(null), json!([])] { + assert_eq!(store.set(key, invalid).unwrap(), json!(false)); + } + } + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["muteWhenOutOfFocus"], json!(false)); + assert_eq!(store.all()["backgroundStreamReminder"], json!(false)); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn ten_bit_warning_opt_out_is_typed_persisted_and_resettable() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-ten-bit-warning-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["suppressTenBitWarning"], json!(false)); + store.set("suppressTenBitWarning", json!(true)).unwrap(); + store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["suppressTenBitWarning"], json!(true)); + for invalid in [json!("true"), json!(1), json!(null), json!([])] { + assert_eq!( + store.set("suppressTenBitWarning", invalid).unwrap(), + json!(false) + ); + } + store.set("suppressTenBitWarning", json!(true)).unwrap(); + assert_eq!( + store.reset().unwrap()["suppressTenBitWarning"], + json!(false) + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["suppressTenBitWarning"], + json!(false) + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_save_and_reset_failures_preserve_previous_state() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-onboarding-save-failure-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for completed in [false, true] { + store.set("onboardingCompleted", json!(completed)).unwrap(); + store.set("windowWidth", json!(1600)).unwrap(); + let original = store.all(); + let persisted = fs::read(directory.join("settings.json")).unwrap(); + fs::create_dir(directory.join("settings.json.tmp")).unwrap(); + assert!(store.set("onboardingCompleted", json!(!completed)).is_err()); + assert_eq!(store.all(), original); + assert!(store.reset().is_err()); + assert_eq!(store.all(), original); + assert_eq!( + fs::read(directory.join("settings.json")).unwrap(), + persisted + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all(), + original + ); + fs::remove_dir(directory.join("settings.json.tmp")).unwrap(); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn onboarding_migration_save_failure_leaves_existing_file_intact() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = + env::temp_dir().join(format!("opennow-onboarding-migration-failure-{unique}")); + fs::create_dir_all(directory.join("settings.json.tmp")).unwrap(); + let persisted = br#"{"qtConsoleModePolicyVersion":1}"#; + fs::write(directory.join("settings.json"), persisted).unwrap(); + assert!(SettingsStore::load(Some(directory.clone())).is_err()); + assert_eq!( + fs::read(directory.join("settings.json")).unwrap(), + persisted + ); + fs::remove_dir(directory.join("settings.json.tmp")).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["onboardingCompleted"], + json!(true) + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn shortcut_transaction_moves_chords_atomically() { + let directory = tempfile::tempdir().unwrap(); + let mut store = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + let applied = store + .set_shortcuts(&json!({"shortcutToggleStats":"Ctrl+F11","shortcutScreenshot":""})) + .unwrap(); + assert_eq!(applied["shortcutToggleStats"], json!("Ctrl+F11")); + assert_eq!(applied["shortcutScreenshot"], json!("")); + let reloaded = SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + assert_eq!(reloaded.all()["shortcutToggleStats"], json!("Ctrl+F11")); + assert_eq!(reloaded.all()["shortcutScreenshot"], json!("")); + + let before = store.all(); + for rejected in [ + json!({"shortcutToggleFullscreen":"ctrl + f11"}), + json!({"shortcutToggleFullscreen":"Ctrl+G"}), + json!({"shortcutSaveClip":"Shift+F3"}), + json!({"shortcutToggleStats":"F1","appTheme":"light"}), + json!({"shortcutToggleStats":7}), + json!({"shortcutToggleStats":"x".repeat(81)}), + json!({}), + json!([]), + ] { + assert!(store.set_shortcuts(&rejected).is_err(), "{rejected}"); + assert_eq!(store.all(), before, "{rejected} must not partially apply"); + } + + store.set("shortcutToggleRecording", json!("F8")).unwrap(); + store + .set_shortcuts(&json!({"shortcutSaveClip":"Alt+F12"})) + .expect("an existing duplicate must not block unrelated shortcut edits"); + assert!( + store + .set_shortcuts(&json!({"shortcutToggleRecording":"F8"})) + .is_err() + ); + + let defaults = defaults(); + let reset = SHORTCUT_KEYS + .iter() + .map(|key| (key.to_string(), defaults[*key].clone())) + .collect::>(); + store.set_shortcuts(&Value::Object(reset)).unwrap(); + for key in SHORTCUT_KEYS { + assert_eq!(store.all()[key], defaults[key]); + } + } + + #[test] + fn replay_is_opt_in_bounded_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-replay-settings-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["replayBufferEnabled"], json!(false)); + assert_eq!(store.all()["replayBufferSeconds"], json!(30)); + assert_eq!(store.all()["replayBufferMemoryMiB"], json!(256)); + assert_eq!(store.all()["shortcutToggleRecording"], json!("F12")); + assert_eq!(store.all()["shortcutSaveClip"], json!("Ctrl+F12")); + store.set("replayBufferEnabled", json!(true)).unwrap(); + store.set("replayBufferSeconds", json!(999)).unwrap(); + store.set("replayBufferMemoryMiB", json!(1)).unwrap(); + store.set("shortcutSaveClip", json!("Alt+F12")).unwrap(); + store.set("shortcutToggleRecording", json!("")).unwrap(); + let mut reloaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(reloaded.all()["replayBufferEnabled"], json!(true)); + assert_eq!(reloaded.all()["replayBufferSeconds"], json!(120)); + assert_eq!(reloaded.all()["replayBufferMemoryMiB"], json!(64)); + assert_eq!(reloaded.all()["shortcutSaveClip"], json!("Alt+F12")); + assert_eq!(reloaded.all()["shortcutToggleRecording"], json!("")); + reloaded.set("replayBufferSeconds", json!(-1)).unwrap(); + reloaded.set("replayBufferMemoryMiB", json!(9999)).unwrap(); + assert_eq!(reloaded.all()["replayBufferSeconds"], json!(15)); + assert_eq!(reloaded.all()["replayBufferMemoryMiB"], json!(512)); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn microphone_is_opt_in_and_open_mode_survives_reload() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-microphone-policy-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + store + .set("audioOutputDevice", json!("Selected headphones")) + .unwrap(); + assert_eq!(store.all()["microphoneMode"], json!("disabled")); + store + .set("microphoneDeviceId", json!("old-device-id")) + .unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["microphoneDeviceId"], + json!("old-device-id") + ); + assert_eq!( + store + .set("microphoneMode", json!("voice-activity")) + .unwrap(), + json!("voice-activity") + ); + let mut reloaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(reloaded.all()["microphoneMode"], json!("voice-activity")); + assert_eq!(reloaded.all()["microphoneDeviceId"], json!("")); + assert_eq!( + reloaded.all()["audioOutputDevice"], + json!("Selected headphones") + ); + assert_eq!( + reloaded + .set("microphoneMode", json!("push-to-talk")) + .unwrap(), + json!("disabled") + ); + assert_eq!( + reloaded.set("microphoneMode", json!("invalid")).unwrap(), + json!("disabled") + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn automatic_bug_report_choice_starts_unset_and_rejects_unknown_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-bug-report-choice-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["automaticBugReports"], json!("unset")); + assert_eq!( + store.set("automaticBugReports", json!("disabled")).unwrap(), + json!("disabled") + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["automaticBugReports"], + json!("disabled") + ); + assert_eq!( + store.set("automaticBugReports", json!("always")).unwrap(), + json!("unset") + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn denied_crash_reports_migrate_to_disabled_usage_and_bug_reports() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + for (legacy, choice, expected) in [ + ("denied", "unset", "disabled"), + ("denied", "enabled", "enabled"), + ("granted", "unset", "unset"), + ] { + let directory = env::temp_dir().join(format!( + "opennow-consent-migration-{unique}-{legacy}-{choice}" + )); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + json!({"errorReportingConsent":legacy, "automaticBugReports":choice}).to_string(), + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["automaticBugReports"], json!(expected)); + fs::remove_dir_all(directory).unwrap(); + } + } + + #[test] + fn audio_output_device_is_persisted_and_validated() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-audio-settings-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["audioOutputDevice"], json!("")); + let name = "USB Headphones – Audio"; + assert_eq!( + store.set("audioOutputDevice", json!(name)).unwrap(), + json!(name) + ); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["audioOutputDevice"], json!(name)); + for invalid in [ + json!(42), + json!(null), + json!("speaker\0other"), + json!("é".repeat(513)), + ] { + assert!(store.set("audioOutputDevice", invalid).is_err()); + assert_eq!(store.all()["audioOutputDevice"], json!(name)); + } + store.set("audioOutputDevice", json!("")).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["audioOutputDevice"], + json!("") + ); + let mut persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + persisted["audioOutputDevice"] = json!("bad\0device"); + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&persisted).unwrap(), + ) + .unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["audioOutputDevice"], + json!("") + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn theme_packs_and_accent_overrides_are_atomic_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-theme-policy-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for pack in [ + "nocturne", "aurora", "kraft", "phosphor", "bone", "cobalt", "hibiscus", "chapel", + ] { + store.set("appAccentColor", json!("rose")).unwrap(); + assert_eq!(store.all()["themeAccentOverride"], json!(true)); + store.set("themePack", json!(pack)).unwrap(); + let mut loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["themePack"], json!(pack)); + assert_eq!(loaded.all()["themeAccentOverride"], json!(false)); + assert_eq!( + loaded.all()["appTheme"], + json!(if matches!(pack, "bone" | "cobalt") { + "light" + } else { + "dark" + }) + ); + loaded.set("appTheme", json!("auto")).unwrap(); + loaded.set("appAccentColor", json!("violet")).unwrap(); + store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["appTheme"], json!("auto")); + assert_eq!(store.all()["themePack"], json!(pack)); + assert_eq!(store.all()["themeAccentOverride"], json!(true)); + assert_eq!(store.all()["appAccentColor"], json!("violet")); + } + let before = store.all(); + fs::create_dir(directory.join("settings.json.tmp")).unwrap(); + assert!(store.set("themePack", json!("bone")).is_err()); + assert_eq!(store.all(), before); + fs::remove_dir(directory.join("settings.json.tmp")).unwrap(); + store.set("themeAccentOverride", json!(false)).unwrap(); + let before = store.all(); + fs::create_dir(directory.join("settings.json.tmp")).unwrap(); + assert!(store.set("appAccentColor", json!("green")).is_err()); + assert_eq!(store.all(), before); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn in_game_settings_persistence_survives_restart_and_resets() { + let directory = tempfile::tempdir().unwrap(); + let load = || SettingsStore::load(Some(directory.path().to_owned())).unwrap(); + let mut store = load(); + assert_eq!(store.all()["enablePersistingInGameSettings"], true); + fs::write(directory.path().join("settings.json"), br#"{"fps":120}"#).unwrap(); + store = load(); + assert_eq!(store.all()["enablePersistingInGameSettings"], true); + for enabled in [true, false, true] { + store + .set("enablePersistingInGameSettings", json!(enabled)) + .unwrap(); + store = load(); + assert_eq!(store.all()["enablePersistingInGameSettings"], enabled); + store.set("fps", json!(120)).unwrap(); + store = load(); + assert_eq!(store.all()["enablePersistingInGameSettings"], enabled); + } + fs::create_dir(directory.path().join("settings.json.tmp")).unwrap(); + assert!( + store + .set("enablePersistingInGameSettings", json!(false)) + .is_err() + ); + assert_eq!(store.all()["enablePersistingInGameSettings"], true); + assert_eq!(load().all()["enablePersistingInGameSettings"], true); + fs::remove_dir(directory.path().join("settings.json.tmp")).unwrap(); + store + .set("enablePersistingInGameSettings", json!(false)) + .unwrap(); + store.reset().unwrap(); + assert_eq!(load().all()["enablePersistingInGameSettings"], true); + } + + #[test] + fn steam_big_picture_is_opt_in_and_persists_independently() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-big-picture-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], false); + store.set("launchInConsoleMode", json!(true)).unwrap(); + store.set("controllerMode", json!(true)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], false); + store.set("steamBigPictureMode", json!(true)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], true); + store.set("launchInConsoleMode", json!(false)).unwrap(); + store.set("controllerMode", json!(false)).unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], true); + store.set("steamBigPictureMode", json!(false)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], false); + store.set("steamBigPictureMode", json!(true)).unwrap(); + store.reset().unwrap(); + assert_eq!(store.all()["steamBigPictureMode"], false); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn start_as_last_used_account_persists_and_migrates_the_console_picker_toggle() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-account-picker-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["startAsLastUsedAccount"], false); + assert!(store.all().get("consoleProfilePickerOnLaunch").is_none()); + store.set("startAsLastUsedAccount", json!(true)).unwrap(); + let loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["startAsLastUsedAccount"], true); + fs::remove_dir_all(&directory).unwrap(); + + for (legacy, expected) in [(false, true), (true, false)] { + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + json!({"onboardingCompleted":true,"consoleProfilePickerOnLaunch":legacy}) + .to_string(), + ) + .unwrap(); + let migrated = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(migrated.all()["startAsLastUsedAccount"], expected); + fs::remove_dir_all(&directory).unwrap(); + } + } + + #[test] + fn ui_sounds_default_enabled_and_persist_independently() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-ui-sounds-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["uiSoundsEnabled"], true); + store.set("uiSoundsEnabled", json!(false)).unwrap(); + store.set("reducedMotion", json!(true)).unwrap(); + store.set("launchInConsoleMode", json!(true)).unwrap(); + let mut loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["uiSoundsEnabled"], false); + assert_eq!(loaded.all()["reducedMotion"], true); + loaded.reset().unwrap(); + assert_eq!(loaded.all()["uiSoundsEnabled"], true); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn game_collections_persist_and_survive_unrelated_changes() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-collections-persistence-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["gameCollections"], json!([])); + let collections = json!([ + {"id":"stable-b", "name":" Favorites ", "gameIds":["game-b", "game-a"]}, + {"id":"stable-a", "name":"Favorites", "gameIds":["game-a"]}, + {"id":"empty", "name":"Empty", "gameIds":[]} + ]); + let mut expected = collections.clone(); + expected[0]["name"] = json!("Favorites"); + assert_eq!( + store.set("gameCollections", collections.clone()).unwrap(), + expected + ); + store.set("fps", json!(120)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["gameCollections"], expected); + let mut persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + persisted["gameCollections"] = collections; + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&persisted).unwrap(), + ) + .unwrap(); + let loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["gameCollections"], expected); + expected[0]["name"] = json!("Renamed"); + store.set("gameCollections", expected.clone()).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["gameCollections"], + expected + ); + assert_eq!(store.reset().unwrap()["gameCollections"], json!([])); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["gameCollections"], + json!([]) + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn game_collections_reject_invalid_updates_without_data_loss() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-collections-invalid-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + let collection = json!({"id":"stable", "name":"Keep", "gameIds":["game-a"]}); + store + .set("gameCollections", json!([collection.clone()])) + .unwrap(); + let original = store.all(); + let persisted = fs::read(directory.join("settings.json")).unwrap(); + let mut invalid = vec![ + json!(null), + json!({}), + json!("wrong"), + json!(true), + json!([null]), + json!([[]]), + json!([collection.clone(), collection.clone()]), + json!([{"id":"stable", "name":"Keep", "gameIds":[], "unexpected":true}]), + ]; + for (key, values) in [ + ( + "id", + vec![ + json!(null), + json!(42), + json!(""), + json!(" \t"), + json!("x".repeat(129)), + ], + ), + ( + "name", + vec![ + json!(null), + json!(false), + json!(""), + json!(" \n"), + json!("界".repeat(81)), + ], + ), + ( + "gameIds", + vec![ + json!(null), + json!({}), + json!([null]), + json!([1]), + json!([""]), + json!([" \t"]), + json!(["x".repeat(129)]), + json!(["game-a", "game-a"]), + json!( + (0..10_001) + .map(|id| format!("game-{id}")) + .collect::>() + ), + ], + ), + ] { + let mut missing = collection.clone(); + missing.as_object_mut().unwrap().remove(key); + invalid.push(json!([missing])); + for value in values { + let mut malformed = collection.clone(); + malformed[key] = value; + invalid.push(json!([malformed])); + } + } + invalid.push(json!( + (0..101) + .map(|id| json!({"id":format!("collection-{id}"), "name":"Name", "gameIds":[]})) + .collect::>() + )); + for value in invalid { + let error = store.set("gameCollections", value).unwrap_err(); + assert!(error.contains("gameCollections"), "{error}"); + assert_eq!(store.all(), original); + assert_eq!( + fs::read(directory.join("settings.json")).unwrap(), + persisted + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn game_collections_accept_inclusive_unicode_and_count_bounds() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-collections-bounds-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + let mut collections = (0..100) + .map(|id| json!({"id":format!("collection-{id}"), "name":"Same name", "gameIds":[]})) + .collect::>(); + let mut game_ids = (0..9_999) + .map(|id| format!("game-{id}")) + .collect::>(); + game_ids.push("界".repeat(128)); + collections[0] = json!({"id":"界".repeat(128), "name":"界".repeat(80), "gameIds":game_ids}); + let expected = json!(collections); + assert_eq!( + store.set("gameCollections", expected.clone()).unwrap(), + expected + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["gameCollections"], + expected + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn game_collections_save_and_reset_failures_restore_previous_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-collections-save-failure-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + store + .set( + "gameCollections", + json!([{"id":"stable", "name":"Keep", "gameIds":["game-a"]}]), + ) + .unwrap(); + let original = store.all(); + let persisted = fs::read(directory.join("settings.json")).unwrap(); + fs::create_dir(directory.join("settings.json.tmp")).unwrap(); + assert!( + store + .set("gameCollections", json!([])) + .unwrap_err() + .contains("Could not save settings") + ); + assert_eq!(store.all(), original); + assert!( + store + .reset() + .unwrap_err() + .contains("Could not reset settings") + ); + assert_eq!(store.all(), original); + assert_eq!( + fs::read(directory.join("settings.json")).unwrap(), + persisted + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all(), + original + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn game_collections_invalid_persisted_data_is_not_rewritten() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-collections-load-failure-{unique}")); + fs::create_dir_all(&directory).unwrap(); + for collections in [ + json!(null), + json!([{"id":"stable", "name":"Keep", "gameIds":["game-a", "game-a"]}]), + ] { + let persisted = serde_json::to_vec(&json!({"gameCollections":collections})).unwrap(); + fs::write(directory.join("settings.json"), &persisted).unwrap(); + let error = SettingsStore::load(Some(directory.clone())).err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("gameCollections")); + assert_eq!( + fs::read(directory.join("settings.json")).unwrap(), + persisted + ); + assert!(!directory.join("settings.json.bak").exists()); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn console_switching_is_opt_in_and_manual_desktop_survives_reload() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-console-policy-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({ + "launchInConsoleMode": true, + "switchToConsoleOnPad": true, + "unrelatedLegacySetting": "preserve" + })) + .unwrap(), + ) + .unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["switchToConsoleOnPad"], json!(false)); + assert_eq!(store.all()["launchInConsoleMode"], json!(true)); + store.set("switchToConsoleOnPad", json!(true)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!( + store.all()["switchToConsoleOnPad"], + json!(true), + "do not migrate a new opt-in twice" + ); + // A failed atomic save must not change the in-memory policy either. + let blocked_temporary = directory.join("settings.json.tmp"); + fs::create_dir(&blocked_temporary).unwrap(); + assert!(store.set("launchInConsoleMode", json!(false)).is_err()); + assert_eq!(store.all()["launchInConsoleMode"], json!(true)); + assert_eq!(store.all()["switchToConsoleOnPad"], json!(true)); + fs::remove_dir(blocked_temporary).unwrap(); + store.set("launchInConsoleMode", json!(false)).unwrap(); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["launchInConsoleMode"], json!(false)); + assert_eq!(store.all()["switchToConsoleOnPad"], json!(false)); + store.set("launchInConsoleMode", json!(true)).unwrap(); + assert_eq!( + store.all()["switchToConsoleOnPad"], + json!(false), + "manual console is not automatic opt-in" + ); + let persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + assert_eq!(persisted["unrelatedLegacySetting"], json!("preserve")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn hdr_opt_in_persists_but_runtime_output_capability_does_not() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-hdr-settings-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["enableHdr"], false); + assert!(store.set("nativeHdrSupported", json!(true)).is_err()); + assert!( + store + .set( + "nativeHdrDisplay", + json!({"minimumNits":0.005,"maximumNits":620, + "maximumFullFrameNits":400,"redX":0.64,"redY":0.33, + "greenX":0.30,"greenY":0.60,"blueX":0.15,"blueY":0.06, + "whiteX":0.3127,"whiteY":0.329}) + ) + .is_err() + ); + assert_eq!(store.set("enableHdr", json!(true)).unwrap(), true); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["enableHdr"], + true + ); + let path = directory.join("settings.json"); + let mut persisted: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + persisted["nativeHdrSupported"] = json!(true); + persisted["nativeHdrDisplay"] = json!({"minimumNits":0.005,"maximumNits":620, + "maximumFullFrameNits":400,"redX":0.64,"redY":0.33, + "greenX":0.30,"greenY":0.60,"blueX":0.15,"blueY":0.06, + "whiteX":0.3127,"whiteY":0.329}); + fs::write(&path, serde_json::to_vec(&persisted).unwrap()).unwrap(); + let mut loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert!(loaded.all().get("nativeHdrSupported").is_none()); + assert!(loaded.all().get("nativeHdrDisplay").is_none()); + loaded.set("enableHdr", json!(true)).unwrap(); + let persisted: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert!(persisted.get("nativeHdrSupported").is_none()); + assert!(persisted.get("nativeHdrDisplay").is_none()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn custom_background_preferences_are_bounded_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-background-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["desktopBackgroundImage"], json!("")); + assert_eq!(store.all()["desktopBackgroundOpacity"], json!(30)); + assert_eq!( + store.set("desktopBackground", json!("custom")).unwrap(), + json!("custom") + ); + + let image = url::Url::from_file_path(directory.join("写真 #1.png")) + .unwrap() + .to_string(); + assert_eq!( + store.set("desktopBackgroundImage", json!(image)).unwrap(), + json!(image) + ); + for (input, expected) in [ + (json!(-10), 0), + (json!(110), 100), + (json!(0), 0), + (json!(65), 65), + (json!(null), 30), + (json!("50"), 30), + ] { + assert_eq!( + store.set("desktopBackgroundOpacity", input).unwrap(), + json!(expected) + ); + } + store.set("desktopBackgroundOpacity", json!(65)).unwrap(); + let reloaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(reloaded.all()["desktopBackground"], json!("custom")); + assert_eq!(reloaded.all()["desktopBackgroundImage"], json!(image)); + assert_eq!(reloaded.all()["desktopBackgroundOpacity"], json!(65)); + + for invalid in [ + "https://example.com/image.png", + "qrc:/image.png", + "relative.png", + "file:///image.png?download=1", + "file:///image.png#fragment", + ] { + assert_eq!( + store.set("desktopBackgroundImage", json!(invalid)).unwrap(), + json!("") + ); + } + assert_eq!( + store + .set("desktopBackgroundImage", json!("x".repeat(8193))) + .unwrap(), + json!("") + ); + store.reset().unwrap(); + assert_eq!(store.all()["desktopBackground"], json!("art")); + assert_eq!(store.all()["desktopBackgroundImage"], json!("")); + assert_eq!(store.all()["desktopBackgroundOpacity"], json!(30)); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn automatic_fullscreen_defaults_on_and_preserves_opt_out() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-fullscreen-settings-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["autoFullScreen"], json!(true)); + store.set("autoFullScreen", json!(false)).unwrap(); + let mut loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["autoFullScreen"], json!(false)); + loaded.reset().unwrap(); + assert_eq!(loaded.all()["autoFullScreen"], json!(true)); + fs::write(directory.join("settings.json"), r#"{"fps":60}"#).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()["autoFullScreen"], + json!(true) + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn windows_gpu_device_preference_roundtrips_and_resets() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-windows-gpu-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()[WINDOWS_GPU_DEVICE_ID], json!("")); + + let device_id = "é".repeat(MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES / 2); + assert_eq!(device_id.len(), MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES); + assert_eq!( + store + .set(WINDOWS_GPU_DEVICE_ID, json!(device_id.clone())) + .unwrap(), + json!(device_id) + ); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()[WINDOWS_GPU_DEVICE_ID], + json!(device_id) + ); + + let reset = store.reset().unwrap(); + assert_eq!(reset[WINDOWS_GPU_DEVICE_ID], json!("")); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()[WINDOWS_GPU_DEVICE_ID], + json!("") + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn windows_gpu_device_preference_rejects_invalid_set_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-windows-gpu-invalid-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + store + .set(WINDOWS_GPU_DEVICE_ID, json!("valid-device")) + .unwrap(); + + for invalid in [ + json!(null), + json!(false), + json!(42), + json!(["device"]), + json!({"device":"id"}), + json!("x".repeat(MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES + 1)), + json!("device\0id"), + ] { + assert!(store.set(WINDOWS_GPU_DEVICE_ID, invalid).is_err()); + assert_eq!(store.all()[WINDOWS_GPU_DEVICE_ID], json!("valid-device")); + } + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()[WINDOWS_GPU_DEVICE_ID], + json!("valid-device") + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn windows_gpu_device_preference_normalizes_invalid_saved_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-windows-gpu-saved-{unique}")); + fs::create_dir_all(&directory).unwrap(); + + for invalid in [ + json!(null), + json!(123), + json!("x".repeat(MAXIMUM_WINDOWS_GPU_DEVICE_ID_BYTES + 1)), + json!("device\0id"), + ] { + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"windowsGpuDeviceId":invalid})).unwrap(), + ) + .unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()[WINDOWS_GPU_DEVICE_ID], + json!("") + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn read_only_gpu_preference_load_never_migrates_or_renames_settings() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-windows-gpu-read-only-{unique}")); + let path = directory.join("settings.json"); + fs::create_dir_all(&directory).unwrap(); + + let legacy = br#"{"windowsGpuDeviceId":"legacy-device","mouseAcceleration":true,"gameCollections":[{"invalid":true}]}"#; + fs::write(&path, legacy).unwrap(); + assert_eq!( + SettingsStore::windows_gpu_device_id_read_only(Some(directory.clone())).unwrap(), + "legacy-device" + ); + assert_eq!(fs::read(&path).unwrap(), legacy); + assert!(!directory.join("settings.json.bak").exists()); + assert!(!directory.join("settings.json.tmp").exists()); + + let corrupt = b"{"; + fs::write(&path, corrupt).unwrap(); + assert_eq!( + SettingsStore::windows_gpu_device_id_read_only(Some(directory.clone())).unwrap(), + "" + ); + assert_eq!(fs::read(&path).unwrap(), corrupt); + assert!(!directory.join("settings.json.corrupt").exists()); + + let oversized = vec![b' '; MAXIMUM_BOOTSTRAP_SETTINGS_BYTES as usize + 1]; + fs::write(&path, &oversized).unwrap(); + assert_eq!( + SettingsStore::windows_gpu_device_id_read_only(Some(directory.clone())).unwrap(), + "" + ); + assert_eq!(fs::read(&path).unwrap(), oversized); + assert!(!directory.join("settings.json.corrupt").exists()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn persists_and_normalizes_settings() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-core-settings-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["launchInConsoleMode"], json!(false)); + assert_eq!(store.all()["transportMode"], json!("nvst")); + assert_eq!(store.all()["desktopRailCollapsed"], json!(true)); + assert_eq!(store.all()["desktopSidebarHover"], json!(true)); + assert_eq!(store.all()["confirmQuitWhenIdle"], json!(true)); + assert_eq!( + store.set("confirmQuitWhenIdle", json!(false)).unwrap(), + json!(false) + ); + assert_eq!(store.all()["desktopBackground"], json!("art")); + assert_eq!( + store.set("desktopBackground", json!("gradient")).unwrap(), + json!("gradient") + ); + assert_eq!( + store.set("desktopBackground", json!("invalid")).unwrap(), + json!("art") + ); + assert_eq!( + store.set("desktopSidebarHover", json!(false)).unwrap(), + json!(false) + ); + assert_eq!(store.all()["switchToConsoleOnPad"], json!(false)); + assert_eq!(store.all()["leaveConsoleOnPointer"], json!(true)); + assert_eq!(store.all()["shortcutToggleFullscreen"], json!("F11")); + assert_eq!(store.all()["shortcutScreenshot"], json!("Ctrl+F11")); + assert_eq!(store.all()["statsOverlayPosition"], json!("top-right")); + for key in [ + "statsShowFps", + "statsShowRegion", + "statsShowPing", + "statsShowBitrate", + "statsShowJitter", + "statsShowDrops", + "statsShowPacketLoss", + "statsShowDecode", + "statsShowLatency", + "statsShowVideo", + "statsShowClock", + "statsShowGraphs", + ] { + assert_eq!(store.all()[key], json!(true)); + assert_eq!(store.set(key, json!(false)).unwrap(), json!(false)); + } + assert_eq!( + store.set("statsOverlayScale", json!(99)).unwrap(), + json!(1.5) + ); + assert_eq!( + store.set("statsOverlayOpacity", json!(0)).unwrap(), + json!(40.0) + ); + assert_eq!(store.set("desktopUiScale", json!(0)).unwrap(), json!(0.85)); + let preferences = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(preferences.all()["desktopSidebarHover"], json!(false)); + assert_eq!(preferences.all()["confirmQuitWhenIdle"], json!(false)); + assert_eq!(preferences.all()["statsShowFps"], json!(false)); + assert_eq!(preferences.all()["statsShowRegion"], json!(false)); + assert_eq!(preferences.all()["statsOverlayScale"], json!(1.5)); + assert_eq!(store.set("fps", json!(999)).unwrap(), json!(360)); + assert_eq!(store.set("fps", json!(360)).unwrap(), json!(360)); + assert_eq!(store.set("fps", json!(240)).unwrap(), json!(240)); + assert_eq!(store.set("maxBitrateMbps", json!(200)).unwrap(), json!(200)); + let low_bitrate = store.set("maxBitrateMbps", json!(0.22)).unwrap(); + assert!((low_bitrate.as_f64().unwrap() - 0.22).abs() < 0.001); + let clamped_bitrate = store.set("maxBitrateMbps", json!(0.1)).unwrap(); + assert!((clamped_bitrate.as_f64().unwrap() - 0.22).abs() < 0.001); + assert_eq!(store.set("maxBitrateMbps", json!(27)).unwrap(), json!(27)); + assert_eq!(store.set("maxBitrateMbps", json!(200)).unwrap(), json!(200)); + assert_eq!( + store.set("launchInConsoleMode", json!(false)).unwrap(), + json!(false) + ); + assert_eq!( + store.set("reducedMotion", json!(true)).unwrap(), + json!(true) + ); + assert_eq!( + store.set("saveBandwidth", json!(true)).unwrap(), + json!(true) + ); + assert_eq!( + store.set("saveBandwidth", json!("yes")).unwrap(), + json!(false) + ); + assert_eq!( + store.set("saveBandwidth", json!(true)).unwrap(), + json!(true) + ); + let loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(loaded.all()["fps"], json!(240)); + assert_eq!(loaded.all()["maxBitrateMbps"], json!(200)); + assert_eq!(loaded.all()["saveBandwidth"], json!(true)); + assert_eq!(loaded.all()["launchInConsoleMode"], json!(false)); + assert_eq!(loaded.all()["reducedMotion"], json!(true)); + assert!(store.set("notASetting", json!(true)).is_err()); + assert_eq!(store.set("codec", json!("invalid")).unwrap(), json!("auto")); + assert_eq!( + store.set("resolution", json!("3440x1440")).unwrap(), + json!("3440x1440") + ); + assert_eq!( + store.set("resolution", json!("99999x1")).unwrap(), + json!("1920x1080") + ); + assert_eq!( + store.set("controllerMode", json!("yes")).unwrap(), + json!(true) + ); + assert_eq!( + store.set("mouseSensitivity", json!(99)).unwrap(), + json!(3.0) + ); + assert_eq!( + store + .set("frameInterpolation", json!({"enabled":true,"factor":99})) + .unwrap(), + json!({"enabled":true,"factor":4,"quality":480}) + ); + assert_eq!( + store.set("themePack", json!("chapel")).unwrap(), + json!("chapel") + ); + assert_eq!( + store.set("themePack", json!("unknown")).unwrap(), + json!("nocturne") + ); + assert_eq!(store.set("appLanguage", json!("de")).unwrap(), json!("de")); + assert_eq!( + store.set("appLanguage", json!("unsupported")).unwrap(), + json!("system") + ); + assert!(store.set("sessionProxyEnabled", json!(true)).is_err()); + assert_eq!( + store + .set("sessionProxyUrl", json!("proxy.example:8080")) + .unwrap(), + json!("http://proxy.example:8080/") + ); + assert_eq!( + store.set("sessionProxyEnabled", json!(true)).unwrap(), + json!(true) + ); + assert_eq!( + store + .set( + "homeTileSizes", + json!({"game-a":"wide","game-b":"giant","":"square"}), + ) + .unwrap(), + json!({"game-a":"wide"}) + ); + assert_eq!( + store + .set("hiddenGameIds", json!(["game-a", "", "game-b"])) + .unwrap(), + json!(["game-a", "game-b"]) + ); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn frame_generation_is_limited_to_off_or_2x() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-frame-generation-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + + assert_eq!(store.all()["frameGeneration"], json!("off")); + assert_eq!( + store.set("frameGeneration", json!("2x")).unwrap(), + json!("2x") + ); + assert_eq!( + store.set("frameGeneration", json!("3x")).unwrap(), + json!("off") + ); + assert_eq!( + store.set("frameGeneration", json!(true)).unwrap(), + json!("off") + ); + + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"frameGeneration": "invalid"})).unwrap(), + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["frameGeneration"], json!("off")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn upscaling_defaults_off_and_persists_only_supported_choices() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-upscaling-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscaling"], json!("off")); + assert_eq!( + store.set("upscaling", json!("metalfx")).unwrap(), + json!("metalfx") + ); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscaling"], json!("metalfx")); + assert_eq!(store.set("upscaling", json!("off")).unwrap(), json!("off")); + for invalid in [json!("unknown"), json!(true), json!(null)] { + assert_eq!(store.set("upscaling", invalid).unwrap(), json!("off")); + } + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"upscaling": "invalid"})).unwrap(), + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscaling"], json!("off")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn fsr_upscaling_persists_without_changing_stream_or_metalfx_preferences() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-fsr-upscaling-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + let fps = store.all()["fps"].clone(); + let resolution = store.all()["resolution"].clone(); + store.set("upscalingDenoise", json!(7)).unwrap(); + assert_eq!( + store.set("upscaling", json!("fsr1")).unwrap(), + json!("fsr1") + ); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscaling"], json!("fsr1")); + assert_eq!(store.all()["fps"], fps); + assert_eq!(store.all()["resolution"], resolution); + assert_eq!(store.all()["upscalingDenoise"], json!(7)); + store.set("upscaling", json!("off")).unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscaling"], json!("off")); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn upscaling_enhancement_defaults_bounds_and_persistence() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-upscaling-enhancement-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for (key, maximum, fallback) in + [("upscalingSharpness", 15, 10), ("upscalingDenoise", 20, 0)] + { + assert_eq!(store.all()[key], json!(fallback)); + assert_eq!(store.set(key, json!(-1)).unwrap(), json!(0)); + assert_eq!(store.set(key, json!(maximum + 1)).unwrap(), json!(maximum)); + for invalid in [json!("7"), json!(true), json!(null), json!(1.5)] { + assert_eq!(store.set(key, invalid).unwrap(), json!(fallback)); + } + assert_eq!(store.set(key, json!(7)).unwrap(), json!(7)); + } + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscalingSharpness"], json!(7)); + assert_eq!(store.all()["upscalingDenoise"], json!(7)); + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&json!({"upscalingSharpness": 100, "upscalingDenoise": -2})) + .unwrap(), + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["upscalingSharpness"], json!(15)); + assert_eq!(store.all()["upscalingDenoise"], json!(0)); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn legacy_f10_f11_pair_migrates_to_native_f11_fullscreen() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-core-f11-migration-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + r#"{"shortcutToggleFullscreen":"F10","shortcutScreenshot":"F11"}"#, + ) + .unwrap(); + + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["shortcutToggleFullscreen"], json!("F11")); + assert_eq!(store.all()["shortcutScreenshot"], json!("Ctrl+F11")); + assert_eq!(store.all()["statsOverlayPosition"], json!("top-right")); + + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn existing_legacy_profile_wins_only_when_primary_is_absent() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = env::temp_dir().join(format!("opennow-core-profile-{unique}")); + let primary = root.join("OpenNOW"); + let legacy = root.join("legacy-opennow"); + + fs::create_dir_all(&legacy).unwrap(); + assert_eq!( + select_existing_data_dir(primary.clone(), [legacy.clone()]), + legacy + ); + + fs::create_dir_all(&primary).unwrap(); + assert_eq!(select_existing_data_dir(primary.clone(), [legacy]), primary); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn historical_profile_spelling_respects_filesystem_case_sensitivity() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = env::temp_dir().join(format!("opennow-core-profile-case-{unique}")); + let primary = root.join("OpenNOW"); + let legacy = root.join("opennow"); + fs::create_dir_all(&legacy).unwrap(); + fs::write(legacy.join("settings.json"), b"{}").unwrap(); + + let expected = if primary.canonicalize().is_ok() { + primary.clone() + } else { + legacy.clone() + }; + let selected = select_existing_data_dir(primary, [legacy.clone()]); + assert_eq!(selected, expected); + assert_eq!(fs::read(selected.join("settings.json")).unwrap(), b"{}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn electron_settings_migrate_without_destroying_rollback_fields() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-core-legacy-settings-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + serde_json::to_vec_pretty(&json!({ + "fps": 120, + "mouseAcceleration": true, + "transportMode": "webrtc", + "sessionTimeRemainingDisplay": "both", + "futureElectronSetting": {"enabled": true} + })) + .unwrap(), + ) + .unwrap(); + + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["fps"], json!(120)); + assert_eq!(store.all()["mouseAcceleration"], json!(100)); + assert_eq!(store.all()["transportMode"], json!("nvst")); + assert_eq!( + store.all()["showSessionTimeRemainingInStatsOverlay"], + json!(true) + ); + assert!(store.all().get("futureElectronSetting").is_none()); + store.set("codec", json!("h264")).unwrap(); + + let persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + assert_eq!(persisted["futureElectronSetting"], json!({"enabled": true})); + assert_eq!(persisted["transportMode"], json!("nvst")); + assert_eq!(persisted["sessionTimeRemainingDisplay"], json!("both")); + assert_eq!(persisted["mouseAcceleration"], json!(100)); + assert_eq!(persisted["codec"], json!("h264")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn every_legacy_transport_selector_normalizes_and_persists_as_nvst() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-core-transport-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + + for legacy in [ + json!("webrtc"), + json!("browser"), + json!("auto"), + json!(null), + ] { + assert_eq!(store.set("transportMode", legacy).unwrap(), json!("nvst")); + } + + let persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + assert_eq!(persisted["transportMode"], json!("nvst")); + fs::remove_dir_all(directory).unwrap(); + } + #[test] + fn controller_tuning_is_bounded_and_persisted() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-controller-tuning-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + for (key, default, maximum) in [ + ("controllerLeftStickDeadzone", 5, 50), + ("controllerRightStickDeadzone", 5, 50), + ("controllerVibrationIntensity", 100, 100), + ] { + assert_eq!(store.all()[key], json!(default)); + assert_eq!(store.set(key, json!(-1)).unwrap(), json!(0)); + assert_eq!(store.set(key, json!(101)).unwrap(), json!(maximum)); + for invalid in [json!("bad"), json!(null), json!(false), json!(0.5)] { + assert_eq!(store.set(key, invalid).unwrap(), json!(default)); + } + store.set(key, json!(12)).unwrap(); + assert_eq!( + SettingsStore::load(Some(directory.clone())).unwrap().all()[key], + json!(12) + ); + } + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn controller_profiles_keep_optional_fields_and_fall_back_to_globals() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-controller-profiles-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["controllerProfiles"], json!({})); + store.set("controllerLeftStickDeadzone", json!(9)).unwrap(); + let saved = store + .set( + "controllerProfiles", + json!({ + "serial:054c:0ce6:abc": {"name": " DualSense ", "leftStickDeadzone": 12, "lastSeen": 20}, + "guid:030000005e040000e002000000006800": {"vibrationIntensity": 140, "rightStickDeadzone": -3}, + "guid:only-a-name": {"name": "No tuning"}, + "": {"leftStickDeadzone": 1}, + "guid:bad": "not an object", + "guid:wrong-type": {"leftStickDeadzone": "high", "vibrationIntensity": 0.5} + }), + ) + .unwrap(); + assert_eq!( + saved, + json!({ + "serial:054c:0ce6:abc": {"name": "DualSense", "leftStickDeadzone": 12, "lastSeen": 20}, + "guid:030000005e040000e002000000006800": {"vibrationIntensity": 100, "rightStickDeadzone": 0, "lastSeen": 0} + }) + ); + let profile = &saved["serial:054c:0ce6:abc"]; + assert!(profile.get("rightStickDeadzone").is_none()); + assert!(profile.get("vibrationIntensity").is_none()); + assert_eq!(store.all()["controllerLeftStickDeadzone"], json!(9)); + assert_eq!(store.all()["controllerRightStickDeadzone"], json!(5)); + assert_eq!(store.all()["controllerVibrationIntensity"], json!(100)); + let reloaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(reloaded.all()["controllerProfiles"], saved); + for invalid in [json!(null), json!([]), json!("profiles"), json!(7)] { + assert_eq!(store.set("controllerProfiles", invalid).unwrap(), json!({})); + } + assert_eq!(store.all()["controllerLeftStickDeadzone"], json!(9)); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn controller_profiles_are_bounded_by_most_recent_use() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-controller-profile-cap-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + let profiles: Map = (0..40) + .map(|index| { + ( + format!("guid:{index:02}"), + json!({"leftStickDeadzone": 10, "lastSeen": index}), + ) + }) + .collect(); + let long_key = "k".repeat(MAXIMUM_CONTROLLER_PROFILE_KEY_CHARS + 1); + let mut with_long_key = profiles.clone(); + with_long_key.insert( + long_key.clone(), + json!({"leftStickDeadzone": 1, "lastSeen": 99}), + ); + let saved = store + .set("controllerProfiles", Value::Object(with_long_key)) + .unwrap(); + let saved = saved.as_object().unwrap(); + assert_eq!(saved.len(), MAXIMUM_CONTROLLER_PROFILES); + assert!(!saved.contains_key(&long_key)); + assert!(saved.contains_key("guid:39")); + assert!(saved.contains_key("guid:08")); + assert!(!saved.contains_key("guid:07")); + let mut persisted: Value = + serde_json::from_slice(&fs::read(directory.join("settings.json")).unwrap()).unwrap(); + persisted["controllerProfiles"] = Value::Object(profiles); + fs::write( + directory.join("settings.json"), + serde_json::to_vec(&persisted).unwrap(), + ) + .unwrap(); + let loaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!( + loaded.all()["controllerProfiles"] + .as_object() + .unwrap() + .len(), + MAXIMUM_CONTROLLER_PROFILES + ); + assert_eq!(store.reset().unwrap()["controllerProfiles"], json!({})); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn settings_saved_before_controller_profiles_load_with_empty_profiles() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-controller-profile-legacy-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + br#"{"controllerLeftStickDeadzone":17,"controllerVibrationIntensity":40}"#, + ) + .unwrap(); + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["controllerLeftStickDeadzone"], json!(17)); + assert_eq!(store.all()["controllerVibrationIntensity"], json!(40)); + assert_eq!(store.all()["controllerProfiles"], json!({})); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn incompatible_saved_codec_color_combo_heals_to_auto_on_first_launch() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-codec-color-heal-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + r#"{"codec":"av1","fallbackCodec":"h264","colorQuality":"10bit_444"}"#, + ) + .unwrap(); + + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["colorQuality"], json!("10bit_444")); + assert_eq!(store.all()["codec"], json!("auto")); + assert_eq!(store.all()["fallbackCodec"], json!("auto")); + + // The repair persists so the next launch starts from a valid profile. + let reloaded = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(reloaded.all()["codec"], json!("auto")); + assert_eq!(reloaded.all()["fallbackCodec"], json!("auto")); + assert_eq!(reloaded.all()["colorQuality"], json!("10bit_444")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn compatible_saved_codec_color_combo_survives_reload_unchanged() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-codec-color-keep-{unique}")); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("settings.json"), + r#"{"codec":"h265","fallbackCodec":"auto","colorQuality":"10bit_444"}"#, + ) + .unwrap(); + + let store = SettingsStore::load(Some(directory.clone())).unwrap(); + assert_eq!(store.all()["codec"], json!("h265")); + assert_eq!(store.all()["fallbackCodec"], json!("auto")); + assert_eq!(store.all()["colorQuality"], json!("10bit_444")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn color_change_heals_an_incompatible_explicit_codec() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-color-change-heal-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + store.set("codec", json!("av1")).unwrap(); + store.set("colorQuality", json!("10bit_444")).unwrap(); + assert_eq!(store.all()["colorQuality"], json!("10bit_444")); + assert_eq!(store.all()["codec"], json!("auto")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn explicit_codec_selection_rejects_color_incompatible_values() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = env::temp_dir().join(format!("opennow-codec-reject-{unique}")); + let mut store = SettingsStore::load(Some(directory.clone())).unwrap(); + store.set("colorQuality", json!("10bit_444")).unwrap(); + for codec in ["av1", "h264"] { + let error = store.set("codec", json!(codec)).unwrap_err(); + assert!(error.contains(codec), "{error}"); + assert_eq!(store.all()["codec"], json!("auto")); + } + for codec in ["auto", "h265"] { + store.set("codec", json!(codec)).unwrap(); + assert_eq!(store.all()["codec"], json!(codec)); + } + let error = store.set("fallbackCodec", json!("h264")).unwrap_err(); + assert!(error.contains("h264"), "{error}"); + store.set("colorQuality", json!("8bit_420")).unwrap(); + store.set("codec", json!("h264")).unwrap(); + store.set("fallbackCodec", json!("h264")).unwrap(); + assert_eq!(store.all()["codec"], json!("h264")); + assert_eq!(store.all()["fallbackCodec"], json!("h264")); + fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/external/opennow/native/opennow-core/src/store_cache.rs b/external/opennow/native/opennow-core/src/store_cache.rs new file mode 100644 index 000000000..e252ded12 --- /dev/null +++ b/external/opennow/native/opennow-core/src/store_cache.rs @@ -0,0 +1,676 @@ +//! Bounded, account/provider-scoped Store responses persisted across core restarts. +//! Only successful protocol-sized results are cached; credentials never enter files. +use crate::{gfn::ServiceError, store_catalog_page::RESULT_BUDGET}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, Weak}; + +const MAX_BYTES: u64 = 64 * 1024 * 1024; +const MAX_ENTRIES: usize = 512; +const PAGE_TTL_MS: u64 = 15 * 60 * 1000; + +type FetchLocks = HashMap<(PathBuf, u64), Weak>>; + +pub struct CachePolicy { + pub ttl_ms: u64, + pub allow_stale: bool, +} + +pub struct StoreCache { + root: PathBuf, + index: Mutex>, + // IO is serialized, but network work never holds the lock. Invalidation + // advances the epoch so an older in-flight fetch cannot refill cleared data. + epoch: Mutex, + fetches: Mutex, + pub requests: crate::store_requests::StoreRequests, +} + +fn digest(value: &Value) -> String { + format!("{:x}", Sha256::digest(value.to_string().as_bytes())) +} + +impl StoreCache { + pub fn new(data_dir: PathBuf) -> Self { + Self { + root: data_dir.join("store-cache-v2"), + index: Mutex::new(None), + epoch: Mutex::new(0), + fetches: Mutex::new(HashMap::new()), + requests: crate::store_requests::StoreRequests::default(), + } + } + + pub fn load_or_fetch( + &self, + scope: &Value, + key: &Value, + refresh: bool, + fetch: impl FnOnce() -> Result, + ) -> Result { + self.load_with_policy( + scope, + key, + refresh, + CachePolicy { + ttl_ms: PAGE_TTL_MS, + allow_stale: false, + }, + fetch, + ) + } + + pub fn catalog_revision(&self) -> u64 { + let path = self.root.join("revision"); + let file = match File::open(&path) { + Ok(file) => file, + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + && fs::symlink_metadata(&path) + .is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound) => + { + return 0; + } + Err(_) => return now_ms(), + }; + let mut bytes = Vec::new(); + if file.take(32).read_to_end(&mut bytes).is_err() { + return now_ms(); + } + serde_json::from_slice::(&bytes) + .ok() + .filter(|revision| *revision < (1_u64 << 52)) + .unwrap_or_else(now_ms) + } + + pub fn invalidate(&self, revision: u64) -> Result<(), ServiceError> { + let mut epoch = self.epoch.lock().expect("Store cache poisoned"); + *epoch = epoch.wrapping_add(1); + *self.index.lock().expect("Store index poisoned") = None; + self.write( + &self.root.join("revision"), + &serde_json::json!(revision.max(self.catalog_revision())), + ) + .map_err(|_| ServiceError { + code: "catalog_cache_invalidation_failed", + message: "The catalog cache revision could not be saved. Retry the refresh.".into(), + }) + } + + pub fn invalidate_key(&self, scope: &Value, key: &Value) { + let mut epoch = self.epoch.lock().expect("Store cache poisoned"); + *epoch = epoch.wrapping_add(1); + let _ = fs::remove_file( + self.root + .join(format!("{}-{}.json", digest(scope), digest(key))), + ); + *self.index.lock().expect("Store index poisoned") = None; + } + + pub fn load_with_policy( + &self, + scope: &Value, + key: &Value, + refresh: bool, + policy: CachePolicy, + fetch: impl FnOnce() -> Result, + ) -> Result { + crate::requests::check()?; + let CachePolicy { + ttl_ms, + allow_stale, + } = policy; + let prefix = format!("{}-", digest(scope)); + let path = self.root.join(format!("{prefix}{}.json", digest(key))); + let prior_fetch = if refresh && allow_stale { + self.read(&path).map(|value| value["fetchedAt"].clone()) + } else { + None + }; + let epoch = { + let mut epoch = self.epoch.lock().expect("Store cache poisoned"); + if refresh && !allow_stale { + *self.index.lock().expect("Store index poisoned") = None; + *epoch = epoch.wrapping_add(1); + // Only this cache's generated, flat response files are targets. + if let Ok(entries) = fs::read_dir(&self.root) { + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with(&prefix) && name.ends_with(".json") { + let _ = fs::remove_file(entry.path()); + } + } + } + } else if !refresh && let Some(mut value) = self.read_fresh(&path) { + value["cacheHit"] = Value::Bool(true); + return Ok(value); + } + *epoch + }; + let fetch_lock = { + let mut fetches = self.fetches.lock().expect("Store fetches poisoned"); + fetches.retain(|_, lock| lock.strong_count() > 0); + let entry = fetches.entry((path.clone(), epoch)).or_default(); + match entry.upgrade() { + Some(lock) => lock, + None => { + let lock = Arc::new(Mutex::new(())); + *entry = Arc::downgrade(&lock); + lock + } + } + }; + let _fetch = crate::store_requests::lock(&fetch_lock)?; + if let Some(mut value) = self.read_fresh(&path) + && (!refresh || (allow_stale && prior_fetch.as_ref() != Some(&value["fetchedAt"]))) + { + value["cacheHit"] = Value::Bool(true); + return Ok(value); + } + let mut value = match fetch() { + Ok(value) => value, + Err(error) + if allow_stale + && !matches!( + error.code, + "cancelled" | "stale_account" | "http_unauthorized" + ) => + { + let mut value = self.read(&path).unwrap_or_else(|| serde_json::json!({})); + value["status"] = serde_json::json!(if value["fetchedAt"].is_number() { + "stale" + } else { + "error" + }); + value["freshness"] = value["status"].clone(); + value["error"] = serde_json::json!({"code":error.code,"message":error.message}); + return Ok(value); + } + Err(error) => return Err(error), + }; + crate::requests::check()?; + value["cacheHit"] = Value::Bool(false); + let fetched_at = now_ms(); + value["fetchedAt"] = serde_json::json!(fetched_at); + value["expiresAt"] = serde_json::json!(fetched_at.saturating_add(ttl_ms)); + value["freshness"] = serde_json::json!("fresh"); + value["status"] = serde_json::json!("success"); + value["error"] = Value::Null; + let value = crate::store_catalog_page::bounded_result(value)?; + let current = self.epoch.lock().expect("Store cache poisoned"); + if epoch == *current { + // Cache failures must not turn a successful catalog fetch into an + // unavailable Store (read-only disk, low space, interrupted writes). + if self.write(&path, &value).is_err() { + eprintln!("store cache: response could not be persisted"); + } + *self.index.lock().expect("Store index poisoned") = None; + } + Ok(value) + } + + pub fn local_query(&self, scope: &Value, params: &Value) -> Result { + crate::requests::check()?; + let scope_key = digest(scope); + let read_key = |key: &Value| { + self.read_fresh(&self.root.join(format!("{scope_key}-{}.json", digest(key)))) + }; + let mut cached = self.index.lock().expect("Store index poisoned"); + if params["refresh"] == true + || cached + .as_ref() + .is_none_or(|(key, _, expires)| key != &scope_key || *expires <= now_ms()) + { + let mut index = crate::store_index::StoreIndex::default(); + let mut expires = u64::MAX; + let mut cursor = String::new(); + let mut seen = std::collections::HashSet::new(); + for _ in 0..100 { + crate::requests::check()?; + let key = serde_json::json!(["page", 100, cursor, ""]); + let Some(page) = read_key(&key) else { + break; + }; + expires = expires.min(page["expiresAt"].as_u64().unwrap_or(0)); + for (at, game) in page["games"].as_array().into_iter().flatten().enumerate() { + index.add(game, key.clone(), format!("/games/{at}"), None); + } + index.pages += 1; + if page["hasNextPage"] == false { + index.complete = true; + break; + } + cursor = page["nextCursor"].as_str().unwrap_or("").to_owned(); + index.next_upstream = cursor.clone(); + if cursor.is_empty() || !seen.insert(cursor.clone()) { + break; + } + } + let key = serde_json::json!(["presentation", "panels"]); + if let Some(panels) = read_key(&key) { + expires = expires.min(panels["expiresAt"].as_u64().unwrap_or(0)); + index.add_panels(&panels, &key); + } + *cached = Some((scope_key.clone(), index, expires)); + } + let (_, index, _) = cached.as_ref().expect("Store index initialized"); + if index.pages == 0 { + return Err(ServiceError { + code: "store_cache_missing", + message: "No saved Store catalog yet. Load a catalog page first.".into(), + }); + } + index.query(params, read_key) + } + + fn read(&self, path: &PathBuf) -> Option { + let file = File::open(path).ok()?; + if !file.metadata().ok()?.is_file() { + return None; + } + let mut bytes = Vec::new(); + file.take(RESULT_BUDGET as u64 + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() > RESULT_BUDGET { + return None; + } + let value: Value = serde_json::from_slice(&bytes).ok()?; + // Versioned directory + bounded structural validation before crossing IPC. + let page = value["games"].is_array() + && value["hasNextPage"].is_boolean() + && value["nextCursor"].is_string(); + let section = matches!( + value["section"].as_str(), + Some("panels" | "marquee" | "filters") + ) && value["items"].is_array(); + let metadata = value["metadataKind"].is_string(); + (page || section || metadata).then_some(value) + } + + fn read_fresh(&self, path: &PathBuf) -> Option { + let value = self.read(path)?; + (value["expiresAt"].as_u64()? > now_ms()).then_some(value) + } + + fn write(&self, path: &PathBuf, value: &Value) -> std::io::Result<()> { + fs::create_dir_all(&self.root)?; + let bytes = serde_json::to_vec(value)?; + let temp = path.with_extension(format!( + "{}-{}.tmp", + std::process::id(), + rand::random::() + )); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp)?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + fs::rename(&temp, path)?; + self.prune(); + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp); + } + result + } + + fn prune(&self) { + let Ok(entries) = fs::read_dir(&self.root) else { + return; + }; + let mut files: Vec<_> = entries + .flatten() + .filter_map(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + if !(name.ends_with(".json") || name.ends_with(".tmp")) { + return None; + } + let meta = entry.metadata().ok()?; + meta.is_file() + .then_some((entry.path(), meta.len(), meta.modified().ok())) + }) + .collect(); + files.sort_by_key(|entry| entry.2); + let mut bytes: u64 = files.iter().map(|entry| entry.1).sum(); + let mut count = files.len(); + for (path, size, _) in files { + if count <= MAX_ENTRIES && bytes <= MAX_BYTES { + break; + } + if fs::remove_file(path).is_ok() { + count -= 1; + bytes = bytes.saturating_sub(size); + } + } + } +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + fn cache() -> StoreCache { + StoreCache::new( + std::env::temp_dir().join(format!("opennow-store-test-{}", rand::random::())), + ) + } + fn page(id: &str) -> Value { + json!({"games":[{"id":id}],"hasNextPage":false,"nextCursor":""}) + } + #[test] + fn restart_reads_disk_without_calling_the_fetcher() { + let cache = cache(); + let scope = json!(["account-a", "provider-a", "direct", "en_US"]); + let key = json!(["page", 100, "", ""]); + assert_eq!( + cache + .load_or_fetch(&scope, &key, false, || Ok(page("saved"))) + .unwrap()["cacheHit"], + false + ); + let restarted = StoreCache::new(cache.root.parent().unwrap().to_path_buf()); + let hit = restarted + .load_or_fetch(&scope, &key, false, || panic!("network called on restart")) + .unwrap(); + assert_eq!(hit["games"][0]["id"], "saved"); + assert_eq!(hit["cacheHit"], true); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + + #[test] + fn expired_disk_pages_require_network_and_metadata_refreshes_coalesce() { + let cache = cache(); + let scope = json!("scope"); + let key = json!("page"); + cache + .load_or_fetch(&scope, &key, false, || Ok(page("old"))) + .unwrap(); + let path = cache + .root + .join(format!("{}-{}.json", digest(&scope), digest(&key))); + let mut expired = cache.read(&path).unwrap(); + expired["expiresAt"] = json!(0); + cache.write(&path, &expired).unwrap(); + let refreshed = cache + .load_or_fetch(&scope, &key, false, || Ok(page("new"))) + .unwrap(); + assert_eq!(refreshed["cacheHit"], false); + assert_eq!(refreshed["games"][0]["id"], "new"); + let calls = std::sync::atomic::AtomicUsize::new(0); + let ready = std::sync::Barrier::new(4); + std::thread::scope(|threads| { + let mut workers = Vec::new(); + for _ in 0..4 { + workers.push(threads.spawn(|| { + ready.wait(); + cache + .load_with_policy( + &scope, + &key, + true, + CachePolicy { + ttl_ms: 1000, + allow_stale: true, + }, + || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(100)); + Ok(page("coalesced")) + }, + ) + .unwrap() + })); + } + for worker in workers { + assert_eq!(worker.join().unwrap()["games"][0]["id"], "coalesced"); + } + }); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + + #[test] + fn catalog_invalidation_survives_restart_and_out_of_order_writers() { + let cache = cache(); + cache.invalidate(2).unwrap(); + cache.invalidate(1).unwrap(); + let restarted = StoreCache::new(cache.root.parent().unwrap().to_path_buf()); + assert_eq!(restarted.catalog_revision(), 2); + fs::write(cache.root.join("revision"), b"corrupt").unwrap(); + assert!(restarted.catalog_revision() > 2); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + + #[test] + fn absent_revision_starts_in_the_initial_namespace() { + let cache = cache(); + assert_eq!(cache.catalog_revision(), 0); + } + + #[cfg(unix)] + #[test] + fn unreadable_revision_cannot_reuse_pre_sync_pages_or_erase_reference_metadata() { + let cache = cache(); + let page_key = json!(["page", 100, "", ""]); + cache + .load_or_fetch(&json!(["account", 0]), &page_key, false, || { + Ok(page("pre-sync")) + }) + .unwrap(); + let reference_scope = json!(["definitions", "account"]); + let reference_key = json!(["genres"]); + cache + .load_or_fetch(&reference_scope, &reference_key, false, || { + Ok(json!({"metadataKind":"genres","items":[{"genre":"ACTION","label":"Action"}]})) + }) + .unwrap(); + cache.invalidate(1).unwrap(); + let revision_path = cache.root.join("revision"); + fs::remove_file(&revision_path).unwrap(); + std::os::unix::fs::symlink("revision", &revision_path).unwrap(); + assert!(File::open(&revision_path).is_err()); + + let restarted = StoreCache::new(cache.root.parent().unwrap().to_path_buf()); + let revision = restarted.catalog_revision(); + assert_ne!(revision, 0); + let result = restarted + .load_or_fetch(&json!(["account", revision]), &page_key, false, || { + Ok(page("fresh-network")) + }) + .unwrap(); + assert_eq!(result["games"][0]["id"], "fresh-network"); + assert_eq!(result["cacheHit"], false); + let reference = restarted + .load_or_fetch(&reference_scope, &reference_key, false, || { + panic!("reference metadata was invalidated") + }) + .unwrap(); + assert_eq!(reference["items"][0]["genre"], "ACTION"); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + + #[test] + fn simultaneous_cache_misses_fetch_each_page_only_once() { + let cache = cache(); + let ready = std::sync::Barrier::new(4); + let calls = std::sync::atomic::AtomicUsize::new(0); + std::thread::scope(|threads| { + let mut workers = Vec::new(); + for _ in 0..4 { + workers.push(threads.spawn(|| { + ready.wait(); + cache + .load_or_fetch(&json!("a"), &json!("first"), false, || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(100)); + Ok(page("shared")) + }) + .unwrap() + })); + } + let results: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect(); + assert_eq!( + results + .iter() + .filter(|value| value["cacheHit"] == false) + .count(), + 1 + ); + assert!( + results + .iter() + .all(|value| value["games"][0]["id"] == "shared") + ); + }); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + + #[test] + fn failed_fetches_leave_saved_pages_available() { + let cache = cache(); + cache + .load_or_fetch(&json!("a"), &json!("saved"), false, || Ok(page("saved"))) + .unwrap(); + assert!( + cache + .load_or_fetch(&json!("a"), &json!("missing"), false, || { + Err(ServiceError { + code: "rate_limited", + message: "Try later".into(), + }) + }) + .is_err() + ); + let result = cache + .load_or_fetch(&json!("a"), &json!("saved"), false, || { + panic!("cached page fetched again") + }) + .unwrap(); + assert_eq!(result["cacheHit"], true); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + #[test] + fn accounts_providers_queries_and_cursors_do_not_share_results() { + let cache = cache(); + for scope in [ + json!(["a", "nvidia"]), + json!(["b", "nvidia"]), + json!(["a", "alliance"]), + ] { + for key in [json!(["", ""]), json!(["next", ""]), json!(["", "search"])] { + let result = cache + .load_or_fetch(&scope, &key, false, || Ok(page("new"))) + .unwrap(); + assert_eq!(result["cacheHit"], false); + } + } + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + #[test] + fn refresh_invalidates_all_pages_and_chrome_only_for_this_account() { + let cache = cache(); + for scope in [json!("a"), json!("b")] { + for key in [json!("first"), json!("next"), json!("chrome")] { + cache + .load_or_fetch(&scope, &key, false, || Ok(page("old"))) + .unwrap(); + } + } + cache + .load_or_fetch(&json!("a"), &json!("first"), true, || Ok(page("fresh"))) + .unwrap(); + for key in [json!("next"), json!("chrome")] { + assert_eq!( + cache + .load_or_fetch(&json!("a"), &key, false, || Ok(page("new"))) + .unwrap()["cacheHit"], + false + ); + assert_eq!( + cache + .load_or_fetch(&json!("b"), &key, false, || panic!()) + .unwrap()["cacheHit"], + true + ); + } + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + #[test] + fn failures_corruption_and_oversized_files_are_cache_misses() { + let cache = cache(); + let scope = json!("a"); + let key = json!("first"); + assert!( + cache + .load_or_fetch(&scope, &key, false, || Err(ServiceError { + code: "network_error", + message: "offline".into() + })) + .is_err() + ); + cache + .load_or_fetch(&scope, &key, false, || Ok(page("new"))) + .unwrap(); + let path = cache + .root + .join(format!("{}-{}.json", digest(&scope), digest(&key))); + for bytes in [b"broken json".to_vec(), vec![b' '; RESULT_BUDGET + 1]] { + fs::write(&path, bytes).unwrap(); + assert_eq!( + cache + .load_or_fetch(&scope, &key, false, || Ok(page("recovered"))) + .unwrap()["cacheHit"], + false + ); + } + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + #[test] + fn obsolete_fetch_cannot_repopulate_after_refresh() { + let cache = cache(); + cache + .load_or_fetch(&json!("a"), &json!("old"), false, || { + cache.load_or_fetch(&json!("a"), &json!("first"), true, || Ok(page("fresh")))?; + Ok(page("obsolete")) + }) + .unwrap(); + assert_eq!( + cache + .load_or_fetch(&json!("a"), &json!("old"), false, || Ok(page("new"))) + .unwrap()["cacheHit"], + false + ); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } + #[test] + fn disk_entry_count_is_bounded() { + let cache = cache(); + fs::create_dir_all(&cache.root).unwrap(); + for index in 0..MAX_ENTRIES + 3 { + fs::write(cache.root.join(format!("{index}.json")), b"{}").unwrap(); + } + cache.prune(); + assert_eq!(fs::read_dir(&cache.root).unwrap().count(), MAX_ENTRIES); + fs::remove_dir_all(cache.root.parent().unwrap()).unwrap(); + } +} diff --git a/external/opennow/native/opennow-core/src/store_catalog_page.rs b/external/opennow/native/opennow-core/src/store_catalog_page.rs new file mode 100644 index 000000000..857bb72b3 --- /dev/null +++ b/external/opennow/native/opennow-core/src/store_catalog_page.rs @@ -0,0 +1,246 @@ +//! One Store RPC contains one complete upstream page, never an entire catalog. +use crate::gfn::ServiceError; +use serde_json::{Value, json}; + +// Reserve ample room for the protocol envelope within the unchanged 1 MiB limit. +pub const RESULT_BUDGET: usize = 768 * 1024; +const MAX_CURSOR_BYTES: usize = 4096; + +pub struct PageRequest { + pub limit: usize, + pub cursor: String, + pub search: String, +} + +impl PageRequest { + pub fn parse(params: &Value) -> Result { + let text = |key: &str, max: usize| -> Result { + let value = match params.get(key) { + None => "", + Some(Value::String(value)) => value, + _ => { + return Err(error( + "invalid_params", + "Store cursor and search must be strings", + )); + } + }; + if value.len() > max { + return Err(error( + "invalid_params", + "Store cursor or search exceeds its size limit", + )); + } + Ok(value.to_owned()) + }; + Ok(Self { + limit: params["limit"].as_u64().unwrap_or(100).clamp(1, 100) as usize, + cursor: text("cursor", MAX_CURSOR_BYTES)?, + search: text("searchQuery", 512)?.trim().to_owned(), + }) + } +} + +fn error(code: &'static str, message: &str) -> ServiceError { + ServiceError { + code, + message: message.to_owned(), + } +} + +fn encoded_size(value: &Value) -> Result { + serde_json::to_vec(value) + .map(|bytes| bytes.len()) + .map_err(|_| { + error( + "invalid_upstream_response", + "Store result could not be encoded", + ) + }) +} + +pub fn bounded_result(value: Value) -> Result { + if encoded_size(&value)? > RESULT_BUDGET { + return Err(error( + "catalog_response_too_large", + "Store response exceeds the safe page size", + )); + } + Ok(value) +} + +pub fn fetch_bounded_page( + limit: usize, + mut fetch: impl FnMut(usize) -> Result, +) -> Result { + let mut count = limit.clamp(1, 100); + loop { + crate::requests::check()?; + let page = match fetch(count) { + Err(error) if error.code == "catalog_response_too_large" && count > 1 => { + count = (count / 2).max(1); + continue; + } + result => result?, + }; + crate::requests::check()?; + if encoded_size(&page)? <= RESULT_BUDGET { + return Ok(page); + } + if count == 1 { + return Err(error( + "catalog_response_too_large", + "A Store game exceeds the safe page size", + )); + } + count = (count / 2).max(1); + } +} + +pub fn page_result( + cursor: &str, + games: Vec, + info: &Value, + fetched_at: u64, +) -> Result { + let has_more = info["hasNextPage"].as_bool().ok_or_else(|| { + error( + "invalid_upstream_response", + "Store response has no pagination state", + ) + })?; + let next = info["endCursor"].as_str().unwrap_or(""); + if has_more && (next.is_empty() || next == cursor || next.len() > MAX_CURSOR_BYTES) { + return Err(error( + "invalid_upstream_response", + "Store pagination did not advance", + )); + } + Ok(json!({ + "count":games.len(), "totalCount":info["totalCount"].as_u64(), + "games":games, "hasNextPage":has_more, "nextCursor":if has_more {next} else {""}, + "source":"store-browse", "fetchedAt":fetched_at + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normal_sized_pages_keep_the_full_hundred_game_batch() { + let request = PageRequest::parse(&json!({"limit": 100})).unwrap(); + let mut calls = 0; + let page = fetch_bounded_page(request.limit, |count| { + calls += 1; + assert_eq!(count, 100); + page_result( + "", + (0..count).map(|id| json!({"id": id})).collect(), + &json!({"hasNextPage": true, "endCursor": "after-100", "totalCount": 2500}), + 0, + ) + }) + .unwrap(); + assert_eq!(calls, 1); + assert_eq!(page["count"], 100); + assert_eq!(page["nextCursor"], "after-100"); + } + + #[test] + fn cancelled_fetch_does_not_retry_an_oversized_page() { + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("page", "catalog.store.browse").unwrap(); + let mut calls = 0; + let result = crate::requests::scope(permit.token.clone(), || { + fetch_bounded_page(100, |_| { + calls += 1; + requests.cancel("page"); + Ok(json!({"items":["x".repeat(RESULT_BUDGET)]})) + }) + }); + assert_eq!(result.unwrap_err().code, "cancelled"); + assert_eq!(calls, 1); + } + + #[test] + fn page_requests_are_bounded_and_preserve_opaque_cursors() { + let request = PageRequest::parse( + &json!({"limit":2500,"cursor":"opaque+/=cursor","searchQuery":" game "}), + ) + .unwrap(); + assert_eq!(request.limit, 100); + assert_eq!(request.cursor, "opaque+/=cursor"); + assert_eq!(request.search, "game"); + assert!(PageRequest::parse(&json!({"cursor":"x".repeat(4097)})).is_err()); + assert!(PageRequest::parse(&json!({"searchQuery":5})).is_err()); + } + + #[test] + fn oversized_pages_refetch_smaller_without_skipping_items() { + let mut requested = Vec::new(); + let page = fetch_bounded_page(100, |count| { + requested.push(count); + let games = (0..count).map(|i| json!({"id":i,"title":"界\"".repeat(4000)})).collect(); + page_result("start", games, &json!({"hasNextPage":true,"endCursor":format!("offset-{count}"),"totalCount":2500}), 0) + }).unwrap(); + assert!(requested.len() > 1); + let count = *requested.last().unwrap(); + assert_eq!(page["count"], count); + assert_eq!(page["nextCursor"], format!("offset-{count}")); + let framed = json!({"type":"response","id":"999999","ok":true,"result":page}); + assert!(serde_json::to_vec(&framed).unwrap().len() < 1024 * 1024); + } + + #[test] + fn unpageable_records_and_chrome_return_scoped_errors() { + let huge = json!({"items":["x".repeat(RESULT_BUDGET)]}); + assert_eq!( + bounded_result(huge.clone()).unwrap_err().code, + "catalog_response_too_large" + ); + let mut attempts = 0; + let result = fetch_bounded_page(100, |_| { + attempts += 1; + Ok(huge.clone()) + }); + assert_eq!(result.unwrap_err().code, "catalog_response_too_large"); + assert_eq!(attempts, 7); + } + + #[test] + fn empty_final_pages_and_invalid_continuations_are_distinct() { + let end = page_result( + "last", + vec![], + &json!({"hasNextPage":false,"totalCount":12}), + 0, + ) + .unwrap(); + assert_eq!(end["hasNextPage"], false); + assert_eq!(end["nextCursor"], ""); + for next in ["", "last"] { + assert!( + page_result( + "last", + vec![], + &json!({"hasNextPage":true,"endCursor":next}), + 0 + ) + .is_err() + ); + } + assert!(page_result("last", vec![], &json!({}), 0).is_err()); + } + + #[test] + fn network_errors_do_not_trigger_size_retries() { + let mut attempts = 0; + let result = fetch_bounded_page(100, |_| { + attempts += 1; + Err(error("upstream_error", "HTTP 503")) + }); + assert_eq!(result.unwrap_err().code, "upstream_error"); + assert_eq!(attempts, 1); + } +} diff --git a/external/opennow/native/opennow-core/src/store_index.rs b/external/opennow/native/opennow-core/src/store_index.rs new file mode 100644 index 000000000..20af50888 --- /dev/null +++ b/external/opennow/native/opennow-core/src/store_index.rs @@ -0,0 +1,469 @@ +//! Compact local title/facet index. Full game records stay in the page cache; +//! only the selected page is materialized for Qt or the command palette. +use crate::{gfn::ServiceError, store_catalog_page}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeSet, HashMap, HashSet}; + +pub struct Entry { + id: String, + title: String, + words: Vec, + acronym: String, + owned: bool, + genres: Vec, + stores: Vec, + categories: HashSet, + key: Value, + pointer: String, +} + +#[derive(Default)] +pub struct StoreIndex { + entries: Vec, + identities: HashMap, + pub categories: Vec, + pub complete: bool, + pub next_upstream: String, + pub pages: usize, +} + +fn normalized(text: &str) -> String { + text.to_lowercase() + .chars() + .map(|c| if c.is_alphanumeric() { c } else { ' ' }) + .collect::() + .split_whitespace() + .collect::>() + .join(" ") +} +fn strings(value: &Value) -> Vec { + value + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} +fn error(message: &str) -> ServiceError { + ServiceError { + code: "invalid_params", + message: message.into(), + } +} + +impl StoreIndex { + pub fn add(&mut self, game: &Value, key: Value, pointer: String, category: Option<&str>) { + let id = game["id"].as_str().unwrap_or(""); + if id.is_empty() { + return; + } + if let Some(&at) = self.identities.get(id) { + if let Some(category) = category { + self.entries[at].categories.insert(category.into()); + } + return; + } + if self.entries.len() >= 12000 { + return; + } + let title = normalized(game["title"].as_str().unwrap_or("")); + let words: Vec = title.split_whitespace().map(str::to_owned).collect(); + let acronym = words + .iter() + .filter_map(|word| word.chars().next()) + .collect(); + let mut categories: HashSet = + strings(&game["storeCategories"]).into_iter().collect(); + if let Some(category) = category { + categories.insert(category.into()); + } + if game["freeToPlay"] == true + || game["storeFree"] == true + || game["playType"] + .as_str() + .unwrap_or("") + .to_uppercase() + .contains("FREE") + { + categories.insert("free".into()); + } + let state = normalized(game["playabilityState"].as_str().unwrap_or("")); + if state.contains("coming") || state.contains("unreleased") { + categories.insert("coming".into()); + } + if strings(&game["nvidiaTech"]) + .iter() + .any(|tech| tech.to_lowercase().contains("rtx")) + { + categories.insert("rtx".into()); + } + let mut stores = strings(&game["availableStores"]); + for variant in game["variants"].as_array().into_iter().flatten() { + if let Some(store) = variant["store"].as_str() { + if !stores.iter().any(|s| s == store) { + stores.push(store.into()); + } + } + } + self.identities.insert(id.into(), self.entries.len()); + self.entries.push(Entry { + id: id.into(), + title, + words, + acronym, + owned: game["isInLibrary"] == true || game["storeOwned"] == true, + genres: strings(&game["genres"]), + stores, + categories, + key, + pointer, + }); + } + + pub fn add_panels(&mut self, panels: &Value, key: &Value) { + for (p, panel) in panels["items"].as_array().into_iter().flatten().enumerate() { + for (s, section) in panel["sections"] + .as_array() + .into_iter() + .flatten() + .enumerate() + { + let category = format!("shelf:{p}:{s}"); + let title = section["title"].as_str().unwrap_or("Games"); + let lower = title.to_lowercase(); + let alias = if lower.contains("thursday") + || lower.contains("new") + || lower.contains("recent") + { + Some("new") + } else if lower.contains("rtx") { + Some("rtx") + } else if lower.contains("free") { + Some("free") + } else if lower.contains("coming") { + Some("coming") + } else { + None + }; + let games = section["games"] + .as_array() + .map(Vec::as_slice) + .unwrap_or(&[]); + for (g, game) in games.iter().enumerate() { + let pointer = format!("/items/{p}/sections/{s}/games/{g}"); + self.add(game, key.clone(), pointer.clone(), Some(&category)); + if let Some(alias) = alias { + self.add(game, key.clone(), pointer, Some(alias)); + } + } + if !games.is_empty() { + self.categories + .push(json!({"id":category,"label":title,"count":games.len()})); + } + } + } + } + + pub fn query( + &self, + params: &Value, + mut read: impl FnMut(&Value) -> Option, + ) -> Result { + let page = store_catalog_page::PageRequest::parse(params)?; + let query = normalized(&page.search); + let tokens: Vec<_> = query.split_whitespace().take(8).collect(); + let filter = |name: &str| -> Result { + match params.get(name) { + None => Ok(String::new()), + Some(Value::String(s)) if s.len() <= 256 => Ok(s.to_owned()), + _ => Err(error("Invalid local Store filter")), + } + }; + let genre = filter("genre")?; + let store = filter("store")?; + let category = filter("categoryId")?; + let fingerprint = format!( + "{:x}", + Sha256::digest( + json!([query, genre, store, category]) + .to_string() + .as_bytes() + ) + ); + let prefix = format!("local:{}:", &fingerprint[..16]); + let offset = if page.cursor.is_empty() { + 0 + } else { + page.cursor + .strip_prefix(&prefix) + .and_then(|s| s.split(':').next()) + .and_then(|s| s.parse::().ok()) + .filter(|n| *n <= 12000) + .ok_or_else(|| error("Local Store cursor does not match its search and filters"))? + }; + let mut matches: Vec<_> = self + .entries + .iter() + .enumerate() + .filter_map(|(at, entry)| { + if !genre.is_empty() && !entry.genres.iter().any(|g| g.eq_ignore_ascii_case(&genre)) + { + return None; + } + if !store.is_empty() && !entry.stores.iter().any(|s| s.eq_ignore_ascii_case(&store)) + { + return None; + } + if !category.is_empty() + && category != "featured" + && category != "all" + && !entry.categories.contains(&category) + { + return None; + } + score(entry, &query, &tokens).map(|score| (at, score)) + }) + .collect(); + if !query.is_empty() { + matches.sort_by_key(|&(at, score)| (std::cmp::Reverse(score), at)); + } + let genres: BTreeSet<_> = self + .entries + .iter() + .flat_map(|e| e.genres.iter().cloned()) + .collect(); + let stores: BTreeSet<_> = self + .entries + .iter() + .flat_map(|e| e.stores.iter().cloned()) + .collect(); + let facets = json!({"genres":genres,"stores":stores,"categories":self.categories}); + let mut loaded = HashMap::new(); + let selected: Vec = matches + .iter() + .skip(offset) + .take(page.limit.min(60)) + .map(|&(at, _)| { + let entry = &self.entries[at]; + let key = entry.key.to_string(); + let document = loaded.entry(key).or_insert_with(|| read(&entry.key)); + document + .as_ref() + .and_then(|doc| doc.pointer(&entry.pointer)) + .filter(|game| game["id"] == entry.id) + .cloned() + .ok_or_else(|| ServiceError { + code: "store_cache_changed", + message: "Saved Store data changed. Refresh the results.".into(), + }) + }) + .collect::>()?; + store_catalog_page::fetch_bounded_page(page.limit.min(60), |limit| { + let count = selected.len().min(limit); + let next = offset + count; + let more = next < matches.len() || !self.complete; + Ok( + json!({"games":&selected[..count],"count":count,"totalCount":matches.len(),"catalogTotalCount":self.entries.len(), + "hasNextPage":more,"nextCursor":if more {format!("{prefix}{next}:{}",self.pages)} else {String::new()}, + "source":"store-local","cacheHit":true,"cacheComplete":self.complete, + "localHasNextPage":next < matches.len(), + "upstreamCoverage":if self.complete {"complete-traversal"} else {"partial"}, + "facetsSource":"local-cache", + "upstreamCursor":self.next_upstream, + "facets":if offset == 0 {facets.clone()} else {Value::Null}}), + ) + }) + } +} + +fn score(entry: &Entry, query: &str, tokens: &[&str]) -> Option { + if query.is_empty() { + return Some(0); + } + if entry.title == query { + return Some(12000 + if entry.owned { 300 } else { 0 }); + } + if entry.acronym == query.replace(' ', "") { + return Some(11000 + if entry.owned { 300 } else { 0 }); + } + let mut score = if entry.title.starts_with(query) { + 9000 + } else if entry.title.contains(query) { + 7000 + } else { + 4000 + }; + for token in tokens { + let best = entry + .words + .iter() + .filter_map(|word| { + if word == token { + Some(300) + } else if word.starts_with(token) { + Some(200) + } else if token.len() >= 3 && word.contains(token) { + Some(100) + } else if token.len() >= 4 && one_typo(word, token) { + Some(30) + } else { + None + } + }) + .max()?; + score += best; + } + Some(score + if entry.owned { 300 } else { 0 }) +} + +// Bounded edit-distance-one matcher, including adjacent transpositions. It +// tolerates common typing errors without broad substring/subsequence noise. +fn one_typo(a: &str, b: &str) -> bool { + let a: Vec<_> = a.chars().take(64).collect(); + let b: Vec<_> = b.chars().take(64).collect(); + if a.len().abs_diff(b.len()) > 1 { + return false; + } + let first = a + .iter() + .zip(&b) + .position(|(a, b)| a != b) + .unwrap_or(a.len().min(b.len())); + if first == a.len().min(b.len()) { + return true; + } + if a.len() == b.len() { + a[first + 1..] == b[first + 1..] + || (first + 1 < a.len() + && a[first] == b[first + 1] + && a[first + 1] == b[first] + && a[first + 2..] == b[first + 2..]) + } else if a.len() > b.len() { + a[first + 1..] == b[first..] + } else { + a[first..] == b[first + 1..] + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> (StoreIndex, Value) { + let games = json!({"games":[{"id":"1","title":"Counter-Strike 2","genres":["ACTION"],"availableStores":["STEAM"]}, + {"id":"2","title":"Fortnite","genres":["ACTION"],"availableStores":["EPIC"]}, + {"id":"3","title":"Fortnite Festival","genres":["MUSIC"],"availableStores":["EPIC"]}]}); + let mut index = StoreIndex { + complete: true, + ..StoreIndex::default() + }; + for (i, g) in games["games"].as_array().unwrap().iter().enumerate() { + index.add(g, json!("page"), format!("/games/{i}"), None); + } + (index, games) + } + #[test] + fn ranked_search_handles_titles_acronyms_word_order_and_typos() { + let (index, games) = fixture(); + for (query, id) in [ + ("fortnite", "2"), + ("fortn", "2"), + ("fortntie", "2"), + ("cs2", "1"), + ("2 counter", "1"), + ] { + let result = index + .query(&json!({"searchQuery":query}), |_| Some(games.clone())) + .unwrap(); + assert_eq!(result["games"][0]["id"], id, "{query}"); + } + assert_eq!( + index + .query(&json!({"searchQuery":"zzzz"}), |_| panic!()) + .unwrap()["count"], + 0 + ); + } + #[test] + fn paging_is_bounded_and_facets_cover_unloaded_games() { + let (index, games) = fixture(); + let first = index + .query(&json!({"limit":1}), |_| Some(games.clone())) + .unwrap(); + assert_eq!(first["count"], 1); + assert_eq!(first["facets"]["genres"].as_array().unwrap().len(), 2); + let next = index + .query(&json!({"limit":1,"cursor":first["nextCursor"]}), |_| { + Some(games.clone()) + }) + .unwrap(); + assert_eq!(next["games"][0]["id"], "2"); + assert!( + index + .query( + &json!({"searchQuery":"other","cursor":first["nextCursor"]}), + |_| Some(games.clone()) + ) + .is_err() + ); + assert_eq!( + index + .query(&json!({"store":"EPIC","genre":"MUSIC"}), |_| Some( + games.clone() + )) + .unwrap()["games"][0]["id"], + "3" + ); + } + #[test] + fn only_selected_pages_are_read() { + let (index, games) = fixture(); + let mut reads = 0; + index + .query(&json!({"limit":2}), |_| { + reads += 1; + Some(games.clone()) + }) + .unwrap(); + assert_eq!(reads, 1); + assert!(!one_typo("fortnite", "fzzzzzzz")); + } + + #[test] + fn panel_categories_include_games_outside_the_first_page() { + let (mut index, games) = fixture(); + let panels = json!({"items":[{"sections":[{"title":"GFN Thursday", + "games":[games["games"][2].clone(), {"id":"4","title":"Coming Game"}]}]}]}); + index.add_panels(&panels, &json!("panels")); + let result = index + .query(&json!({"categoryId":"shelf:0:0"}), |key| { + Some(if key == "panels" { + panels.clone() + } else { + games.clone() + }) + }) + .unwrap(); + assert_eq!(result["count"], 2); + assert_eq!(result["games"][1]["id"], "4"); + assert_eq!(result["facets"]["categories"][0]["label"], "GFN Thursday"); + } + + #[test] + fn owned_game_breaks_ambiguous_acronym_ties() { + let games = json!({"games":[{"id":"1","title":"Clouds & Sheep 2"}, + {"id":"2","title":"Counter-Strike 2","isInLibrary":true}]}); + let mut index = StoreIndex { + complete: true, + ..StoreIndex::default() + }; + for (i, game) in games["games"].as_array().unwrap().iter().enumerate() { + index.add(game, json!("page"), format!("/games/{i}"), None); + } + let result = index + .query(&json!({"searchQuery":"cs2"}), |_| Some(games.clone())) + .unwrap(); + assert_eq!(result["games"][0]["id"], "2"); + } +} diff --git a/external/opennow/native/opennow-core/src/store_requests.rs b/external/opennow/native/opennow-core/src/store_requests.rs new file mode 100644 index 000000000..a7c9012c3 --- /dev/null +++ b/external/opennow/native/opennow-core/src/store_requests.rs @@ -0,0 +1,252 @@ +use crate::gfn::ServiceError; +use reqwest::blocking::{RequestBuilder, Response}; +use std::sync::{Mutex, MutexGuard, TryLockError}; +use std::time::{Duration, Instant, SystemTime}; + +const REQUEST_INTERVAL: Duration = Duration::from_millis(50); +const DEFAULT_COOLDOWN: Duration = Duration::from_secs(60); + +pub fn lock(mutex: &Mutex) -> Result, ServiceError> { + loop { + crate::requests::check()?; + match mutex.try_lock() { + Ok(guard) => return Ok(guard), + Err(TryLockError::WouldBlock) => std::thread::sleep(REQUEST_INTERVAL), + Err(TryLockError::Poisoned(_)) => panic!("Store request state poisoned"), + } + } +} + +#[derive(Default)] +struct Schedule { + next_request: Option, + cooldown: Option, +} + +#[derive(Default)] +pub struct StoreRequests(Mutex); + +fn rate_limited(remaining: Duration) -> ServiceError { + ServiceError { + code: "rate_limited", + message: format!( + "The Store is rate limited. Try again in {} seconds.", + remaining.as_secs().saturating_add(1) + ), + } +} + +impl StoreRequests { + pub fn send(&self, request: RequestBuilder, context: &str) -> Result { + self.send_mapped(request, |error| ServiceError::network(context, error)) + } + + pub fn send_mapped( + &self, + request: RequestBuilder, + network: impl FnOnce(reqwest::Error) -> ServiceError, + ) -> Result { + let mut schedule = lock(&self.0)?; + if let Some(remaining) = schedule + .cooldown + .and_then(|until| until.checked_duration_since(Instant::now())) + { + return Err(rate_limited(remaining)); + } + while let Some(remaining) = schedule + .next_request + .and_then(|until| until.checked_duration_since(Instant::now())) + { + crate::requests::check()?; + drop(schedule); + std::thread::sleep(remaining.min(REQUEST_INTERVAL)); + schedule = lock(&self.0)?; + } + if let Some(remaining) = schedule + .cooldown + .and_then(|until| until.checked_duration_since(Instant::now())) + { + return Err(rate_limited(remaining)); + } + crate::requests::check()?; + schedule.next_request = Some(Instant::now() + REQUEST_INTERVAL); + drop(schedule); + let response = request.send(); + let mut schedule = lock(&self.0)?; + schedule.next_request = Some(Instant::now() + REQUEST_INTERVAL); + let response = response.map_err(network)?; + if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { + let delay = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + httpdate::parse_http_date(value) + .ok() + .and_then(|date| date.duration_since(SystemTime::now()).ok()) + }) + }) + .unwrap_or(DEFAULT_COOLDOWN) + .max(REQUEST_INTERVAL); + let now = Instant::now(); + schedule.cooldown = Some(now.checked_add(delay).unwrap_or(now + DEFAULT_COOLDOWN)); + return Err(rate_limited(delay)); + } + Ok(response) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + use std::sync::Arc; + + #[test] + fn in_flight_network_does_not_hold_the_scheduler_lock() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (url, worker) = + crate::gfn::tests::mock_requests(vec![(200, serde_json::json!({}))], move |_, _| { + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + let requests = StoreRequests::default(); + std::thread::scope(|threads| { + let pending = threads + .spawn(|| requests.send(reqwest::blocking::Client::new().get(url), "fixture")); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let unlocked = requests.0.try_lock().is_ok(); + release_tx.send(()).unwrap(); + assert!(unlocked); + assert!(pending.join().unwrap().is_ok()); + }); + worker.join().unwrap(); + } + + fn server(responses: Vec) -> (String, std::thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let worker = std::thread::spawn(move || { + let mut arrivals = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + assert!(reader.read_until(b'\n', &mut request).unwrap() > 0); + assert!(request.len() <= 4096); + } + arrivals.push(Instant::now()); + stream.write_all(response.as_bytes()).unwrap(); + } + arrivals + }); + (url, worker) + } + + #[test] + fn concurrent_store_requests_are_paced() { + let (url, server) = server( + vec!["HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(); 4], + ); + let requests = Arc::new(StoreRequests::default()); + let client = reqwest::blocking::Client::builder() + .no_proxy() + .build() + .unwrap(); + std::thread::scope(|scope| { + for _ in 0..4 { + scope.spawn(|| requests.send(client.get(&url), "test").unwrap()); + } + }); + let arrivals = server.join().unwrap(); + assert!( + arrivals + .windows(2) + .all(|pair| pair[1].duration_since(pair[0]) >= REQUEST_INTERVAL) + ); + } + + #[test] + fn rate_limits_stop_subsequent_requests_including_retries() { + for header in [ + "Retry-After: 120\r\n".to_owned(), + format!( + "Retry-After: {}\r\n", + httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(120)) + ), + "Retry-After: invalid\r\n".to_owned(), + String::new(), + ] { + let (url, server) = server(vec![format!( + "HTTP/1.1 429 Too Many Requests\r\n{header}Content-Length: 0\r\nConnection: close\r\n\r\n" + )]); + let requests = StoreRequests::default(); + let client = reqwest::blocking::Client::builder() + .no_proxy() + .build() + .unwrap(); + assert_eq!( + requests.send(client.get(&url), "test").unwrap_err().code, + "rate_limited" + ); + server.join().unwrap(); + for _ in 0..100 { + assert_eq!( + requests.send(client.get(&url), "test").unwrap_err().code, + "rate_limited" + ); + } + let remaining = requests + .0 + .lock() + .unwrap() + .cooldown + .unwrap() + .duration_since(Instant::now()); + assert!( + remaining + >= Duration::from_secs(if header.contains("120") || header.contains("GMT") { + 118 + } else { + 58 + }) + ); + } + } + + #[test] + fn cooldown_expires_and_cancelled_waiters_never_send() { + let (url, server) = server(vec![ + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(), + ]); + let requests = StoreRequests::default(); + requests.0.lock().unwrap().cooldown = Some(Instant::now() - Duration::from_secs(1)); + let client = reqwest::blocking::Client::builder() + .no_proxy() + .build() + .unwrap(); + requests.send(client.get(&url), "test").unwrap(); + server.join().unwrap(); + let active = Arc::new(crate::requests::Requests::default()); + let permit = active.admit("cancelled", "catalog.store.local").unwrap(); + active.cancel("cancelled"); + let _held = requests.0.lock().unwrap(); + crate::requests::scope(permit.token.clone(), || { + assert_eq!( + requests.send(client.get(&url), "test").unwrap_err().code, + "cancelled" + ); + }); + } +} diff --git a/external/opennow/native/opennow-core/src/streamer.rs b/external/opennow/native/opennow-core/src/streamer.rs new file mode 100644 index 000000000..ae84bd82b --- /dev/null +++ b/external/opennow/native/opennow-core/src/streamer.rs @@ -0,0 +1,4313 @@ +#[cfg(all(test, unix))] +use rand::RngCore as _; +use serde_json::{Value, json}; +use std::fs; +use std::io::{BufRead, BufReader, ErrorKind, Write}; +use std::ops::{Deref, DerefMut}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const STREAMER_PROTOCOL_VERSION: u64 = 7; +const CHILD_MESSAGE_LIMIT: usize = 1024 * 1024; +const CHILD_START_TIMEOUT: Duration = Duration::from_secs(90); +const CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Debug)] +pub struct StreamerError { + pub code: &'static str, + pub message: String, +} + +#[derive(Clone)] +struct Snapshot { + status: String, + message: String, + session_id: Option, + process_id: Option, + transport: Option, + capabilities: Value, + executable: Option, + error_code: Option, + overlay_request_generation: u64, + screenshot_request_generation: u64, + recording_toggle_request_generation: u64, + shortcut_action_generation: u64, + shortcut_action: Option, + microphone_state: String, + microphone_enabled: bool, + microphone_message: Option, + started_at: Option, + session_started_at_ms: Option, + first_frame_latency_ms: Option, + media_backend: Option, + frames_per_second: Option, + bitrate_mbps: Option, + peak_bitrate_mbps: Option, + backend_fallback_count: u64, + decoder_error_count: u64, + output_error_count: u64, + device_loss_count: u64, + device_recovery_count: u64, + queue_drop_count: u64, + input_ready: bool, + input_unavailable_reason: Option, + input_pause_count: u64, + input_resume_count: u64, + surface_update_count: u64, + fullscreen_toggle_count: u64, + stats_toggle_count: u64, + recording_start_count: u64, + recording_stop_count: u64, +} + +impl Default for Snapshot { + fn default() -> Self { + Self { + status: "stopped".to_owned(), + message: "Native NVST streamer is not running".to_owned(), + session_id: None, + process_id: None, + transport: None, + capabilities: Value::Null, + executable: None, + error_code: None, + overlay_request_generation: 0, + screenshot_request_generation: 0, + recording_toggle_request_generation: 0, + shortcut_action_generation: 0, + shortcut_action: None, + microphone_state: "disabled".to_owned(), + microphone_enabled: false, + microphone_message: None, + started_at: None, + session_started_at_ms: None, + first_frame_latency_ms: None, + media_backend: None, + frames_per_second: None, + bitrate_mbps: None, + peak_bitrate_mbps: None, + backend_fallback_count: 0, + decoder_error_count: 0, + output_error_count: 0, + device_loss_count: 0, + device_recovery_count: 0, + queue_drop_count: 0, + input_ready: false, + input_unavailable_reason: None, + input_pause_count: 0, + input_resume_count: 0, + surface_update_count: 0, + fullscreen_toggle_count: 0, + stats_toggle_count: 0, + recording_start_count: 0, + recording_stop_count: 0, + } + } +} + +impl Snapshot { + fn value(&self) -> Value { + json!({"streamer":{ + "status":self.status, + "message":self.message, + "sessionId":self.session_id, + "processId":self.process_id, + "transport":self.transport, + "capabilities":self.capabilities, + "executable":self.executable.as_ref().map(|path| path.to_string_lossy().into_owned()), + "errorCode":self.error_code, + "overlayRequestGeneration":self.overlay_request_generation, + "screenshotRequestGeneration":self.screenshot_request_generation, + "recordingToggleRequestGeneration":self.recording_toggle_request_generation, + "shortcutActionGeneration":self.shortcut_action_generation, + "shortcutAction":self.shortcut_action, + "microphoneState":self.microphone_state, + "microphoneEnabled":self.microphone_enabled, + "microphoneMessage":self.microphone_message, + "sessionUptimeMs":self.started_at.map(|started| elapsed_millis(started.elapsed())), + "sessionStartedAtMs":self.session_started_at_ms.map(|value| value.to_string()), + "firstFrameLatencyMs":self.first_frame_latency_ms, + "mediaBackend":self.media_backend, + "framesPerSecond":self.frames_per_second, + "bitrateMbps":self.bitrate_mbps, + "peakBitrateMbps":self.peak_bitrate_mbps, + "backendFallbackCount":self.backend_fallback_count, + "decoderErrorCount":self.decoder_error_count, + "outputErrorCount":self.output_error_count, + "deviceLossCount":self.device_loss_count, + "deviceRecoveryCount":self.device_recovery_count, + "queueDropCount":self.queue_drop_count, + "inputReady":self.input_ready, + "inputUnavailableReason":self.input_unavailable_reason, + "inputPauseCount":self.input_pause_count, + "inputResumeCount":self.input_resume_count, + "surfaceUpdateCount":self.surface_update_count, + "fullscreenToggleCount":self.fullscreen_toggle_count, + "statsToggleCount":self.stats_toggle_count, + "recordingStartCount":self.recording_start_count, + "recordingStopCount":self.recording_stop_count + }}) + } + + fn acceptance_value(&self) -> Value { + let available_video_backends = self.capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool().unwrap_or(false)) + .filter_map(|backend| backend["backend"].as_str()) + .collect::>(); + json!({ + "schemaVersion": 1, + "kind": "opennow.stream.acceptance", + "status": self.status, + "transport": self.transport, + "errorCode": self.error_code, + "sessionUptimeMs": self.started_at.map(|started| elapsed_millis(started.elapsed())), + "sessionStartedAtMs": self.session_started_at_ms.map(|value| value.to_string()), + "firstFrameLatencyMs": self.first_frame_latency_ms, + "mediaBackend": self.media_backend, + "framesPerSecond": self.frames_per_second, + "bitrateMbps": self.bitrate_mbps, + "peakBitrateMbps": self.peak_bitrate_mbps, + "backendFallbackCount": self.backend_fallback_count, + "decoderErrorCount": self.decoder_error_count, + "outputErrorCount": self.output_error_count, + "deviceLossCount": self.device_loss_count, + "deviceRecoveryCount": self.device_recovery_count, + "queueDropCount": self.queue_drop_count, + "inputReady": self.input_ready, + "inputUnavailableReason": self.input_unavailable_reason, + "microphoneState": self.microphone_state, + "microphoneEnabled": self.microphone_enabled, + "inputPauseCount": self.input_pause_count, + "inputResumeCount": self.input_resume_count, + "surfaceUpdateCount": self.surface_update_count, + "fullscreenToggleCount": self.fullscreen_toggle_count, + "statsToggleCount": self.stats_toggle_count, + "recordingStartCount": self.recording_start_count, + "recordingStopCount": self.recording_stop_count, + "availableVideoBackends": available_video_backends, + "availableCodecs": available_codecs(&self.capabilities) + }) + } +} + +struct Worker { + control: Sender, + join: JoinHandle<()>, +} + +struct KillOnDrop(Child); + +impl Deref for KillOnDrop { + type Target = Child; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for KillOnDrop { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Drop for KillOnDrop { + fn drop(&mut self) { + if self.0.try_wait().ok().flatten().is_none() { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } +} + +enum WorkerCommand { + Stop(String), + InputPaused(bool), + Control(String), + Surface(Value), + Recording { + enabled: bool, + output_path: Option, + reply: mpsc::SyncSender>, + }, +} + +pub struct StreamerService { + state: Arc>, + worker: Mutex>, +} + +impl StreamerService { + pub fn new() -> Self { + Self { + state: Arc::new(Mutex::new(Snapshot::default())), + worker: Mutex::new(None), + } + } + + pub fn detect(&self, settings: &Value) -> Result { + let executable = resolve_executable(settings)?; + let metadata = fs::metadata(&executable).map_err(|error| StreamerError { + code: "streamer_not_found", + message: format!( + "Native streamer was not found at {}: {error}", + executable.display() + ), + })?; + if !metadata.is_file() { + return Err(StreamerError { + code: "streamer_not_found", + message: format!( + "Native streamer path is not a file: {}", + executable.display() + ), + }); + } + let capabilities = probe_capabilities(&executable, settings)?; + let available_codecs = available_codecs(&capabilities); + Ok(json!({ + "available":true, + "protocolVersion":STREAMER_PROTOCOL_VERSION, + "path":executable.to_string_lossy(), + "sizeBytes":metadata.len(), + "capabilities":capabilities, + "availableCodecs":available_codecs + })) + } + + pub fn validate_codec(&self, settings: &Value) -> Result<(), StreamerError> { + if settings["enableHdr"].as_bool() == Some(true) { + return Err(invalid( + "HDR requires current embedded window output capabilities", + )); + } + let requested = settings["codec"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase(); + // Auto resolves to local decode candidates at session time. Probe the + // portable H.264 baseline too: an explicit hardware-only decoder policy + // can make even that baseline unavailable. + let codec = match requested.as_str() { + "" | "auto" => "h264", + _ => normalize_codec_name(&requested).ok_or_else(|| invalid("Unknown video codec"))?, + }; + let detection = self.detect(settings)?; + ensure_codec_available(&detection["capabilities"], codec) + } + + pub fn color_quality_choices(settings: &Value, capabilities: &Value) -> Value { + json!( + ["8bit_420", "8bit_444", "10bit_420", "10bit_444"] + .into_iter() + .map(|quality| { + let mut candidate = settings.clone(); + candidate["colorQuality"] = json!(quality); + let reason = match Self::embedded_session_settings(&candidate, capabilities) { + Err(error) => Some(error.message), + Ok(resolved) => { + let delivered = + resolved["colorQuality"].as_str().unwrap_or("8bit_420"); + (!color_satisfies(delivered, quality)).then(|| { + format!( + "This device cannot decode {quality} with the selected codec and backend; the stream falls back to {delivered}." + ) + }) + } + }; + json!({"value":quality, "disabled":reason.is_some(), "reason":reason}) + }) + .collect::>() + ) + } + + pub fn codec_choices(settings: &Value, capabilities: &Value) -> Value { + json!( + ["auto", "av1", "h264", "h265"] + .into_iter() + .map(|codec| { + let mut candidate = settings.clone(); + candidate["codec"] = json!(codec); + candidate["colorQuality"] = json!("8bit_420"); + candidate["enableHdr"] = json!(false); + let result = Self::embedded_session_settings(&candidate, capabilities); + json!({"value":codec, "disabled":result.is_err(), + "reason":result.err().map(|error| error.message)}) + }) + .collect::>() + ) + } + + pub fn embedded_session_settings( + settings: &Value, + capabilities: &Value, + ) -> Result { + Self::resolve_embedded_session(settings, capabilities).map(|(resolved, _)| resolved) + } + + /// The in-process Qt streamer, not a separately installed executable, owns the usable + /// decode/presentation capabilities. Resolve Auto before asking CloudMatch for a seat. + /// Returns the effective session settings and every capability fallback applied to reach + /// them; unsupported 4:4:4, HDR, and 10-bit requests degrade instead of failing, while an + /// explicitly selected codec that cannot be decoded at all remains an error. + pub fn resolve_embedded_session( + settings: &Value, + capabilities: &Value, + ) -> Result<(Value, Vec), StreamerError> { + if capabilities["protocolVersion"].as_u64() != Some(STREAMER_PROTOCOL_VERSION) { + return Err(invalid( + "Embedded streamer capabilities are missing or incompatible", + )); + } + let mut fallbacks = Vec::new(); + let mut hdr = settings["enableHdr"].as_bool() == Some(true); + if hdr && capabilities["nativeHdrSupported"].as_bool() != Some(true) { + hdr = false; + fallbacks.push( + "HDR was requested, but the stream window's display has no active HDR output. Streaming SDR." + .to_owned(), + ); + } + let requested_backend = requested_embedded_backend(settings); + let software_requested = matches!(requested_backend.as_str(), "software" | "ffmpeg"); + let mut color = settings["colorQuality"] + .as_str() + .filter(|color| matches!(*color, "8bit_420" | "8bit_444" | "10bit_420" | "10bit_444")) + .unwrap_or("8bit_420"); + if software_requested && (hdr || color != "8bit_420") { + fallbacks.push(format!( + "{}{color} was requested, but software decoding presents 8-bit 4:2:0 SDR only. Falling back to 8-bit 4:2:0 SDR.", + if hdr { "HDR " } else { "" } + )); + hdr = false; + color = "8bit_420"; + } + let software_available = capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .any(|backend| { + matches!(backend["backend"].as_str(), Some("software" | "ffmpeg")) + && backend["available"].as_bool() == Some(true) + }); + let mut selected = capabilities.clone(); + let backends = selected["videoBackends"] + .as_array_mut() + .ok_or_else(|| invalid("Embedded streamer did not report video backends"))?; + for backend in backends.iter_mut() { + let name = backend["backend"].as_str().unwrap_or(""); + let matches = if software_requested { + matches!(name, "software" | "ffmpeg") + } else { + !matches!(name, "software" | "ffmpeg") + && (requested_backend == "auto" + || requested_backend == name + || (requested_backend == "nvdec" && name == "cuda")) + }; + if !matches { + backend["available"] = json!(false); + } + } + if !backends + .iter() + .any(|backend| backend["available"].as_bool() == Some(true)) + { + let message = if software_requested { + format!( + "The FFmpeg software decoder is unavailable for the requested {color} profile. Select Auto in Stream settings, or export diagnostics for backend probe failures." + ) + } else if requested_backend != "auto" { + let mut message = format!( + "The {} backend is unavailable for embedded streaming on this device. Select Auto in Stream settings.", + crate::diagnostics::runtime_failure_reason(&requested_backend) + ); + let evidence = crate::diagnostics::native_runtime_evidence(capabilities); + if let Some(reason) = evidence["videoBackends"] + .as_array() + .into_iter() + .flatten() + .find(|backend| { + backend["backend"] == requested_backend + || (requested_backend == "nvdec" && backend["backend"] == "cuda") + }) + .and_then(|backend| backend["reason"].as_str()) + .filter(|reason| !reason.is_empty()) + { + message.push_str(&format!(" {reason}")); + } + if let Some(hint) = hybrid_gpu_hint(capabilities) { + message.push(' '); + message.push_str(&hint); + } + message.chars().take(480).collect() + } else if let Some(message) = hybrid_gpu_message(capabilities) { + message + } else if software_available { + "No hardware video backend is available for embedded streaming on this device. Select Software (CPU) to decode on the CPU, or export diagnostics for probe failures.".to_owned() + } else { + "No hardware video backend is available for embedded streaming on this device. Check the hardware drivers and export diagnostics for backend probe failures.".to_owned() + }; + return Err(StreamerError { + code: "streamer_backend_unavailable", + message, + }); + } + let requested = settings["codec"] + .as_str() + .unwrap_or("auto") + .to_ascii_lowercase(); + let mut fixed_codec = if requested == "auto" { + None + } else { + Some(normalize_codec_name(&requested).ok_or_else(|| invalid("Unknown video codec"))?) + }; + if color.ends_with("_444") { + if (hdr && codec_decodes(&selected, "h265", "10bit_444", true)) + || codec_decodes(&selected, "h265", color, false) + { + if let Some(codec) = fixed_codec.filter(|codec| *codec != "h265") { + fallbacks.push(format!( + "{color} was requested with {}; 4:4:4 is carried only by H.265. Using H.265.", + codec_label(codec).unwrap_or(codec) + )); + } + fixed_codec = Some("h265"); + } else { + let reduced = if color == "10bit_444" { + "10bit_420" + } else { + "8bit_420" + }; + fallbacks.push(format!( + "{color} was requested, but this system cannot decode H.265 {color}. Falling back to {reduced}." + )); + color = reduced; + } + } + let hdr_color = if color.ends_with("_444") { + "10bit_444" + } else { + "10bit_420" + }; + let ten_bit_fallback = |color: &str, codec: &str| { + format!( + "{color} SDR video was requested, but this system cannot decode it with {codec}. Falling back to 8-bit 4:2:0 BT.709." + ) + }; + let codec = if let Some(codec) = fixed_codec { + ensure_codec_available(&selected, codec)?; + let label = codec_label(codec).unwrap_or(codec); + if hdr && !codec_decodes(&selected, codec, hdr_color, true) { + hdr = false; + fallbacks.push(if codec == "h264" { + "HDR was requested with H.264, which is SDR-only. Ignoring the HDR request." + .to_owned() + } else { + format!( + "HDR was requested, but this system cannot decode {label} HDR10 {hdr_color}. Ignoring the HDR request." + ) + }); + } + if !hdr && !codec_decodes(&selected, codec, color, false) { + if color != "8bit_420" { + fallbacks.push(ten_bit_fallback(color, label)); + color = "8bit_420"; + } + if !codec_decodes(&selected, codec, color, false) { + return Err(StreamerError { + code: "streamer_codec_unavailable", + message: format!( + "{label} cannot decode 8-bit 4:2:0 on the selected video backend. Select Auto in Stream settings." + ), + }); + } + } + codec + } else { + let mut chosen = None; + if hdr { + let av1_eligible = auto_av1_eligible(settings, &selected, true); + chosen = ["av1", "h265"].into_iter().find(|codec| { + (*codec != "av1" || av1_eligible) + && codec_decodes(&selected, codec, hdr_color, true) + }); + if chosen.is_none() { + hdr = false; + fallbacks.push(format!( + "HDR was requested, but no available codec can decode HDR10 {hdr_color}. Ignoring the HDR request." + )); + } + } + if chosen.is_none() { + let av1_eligible = auto_av1_eligible(settings, &selected, false); + let pick = |color: &str| { + ["av1", "h265", "h264"].into_iter().find(|codec| { + (*codec != "av1" || av1_eligible) + && codec_decodes(&selected, codec, color, false) + }) + }; + chosen = pick(color); + if chosen.is_none() && color != "8bit_420" { + fallbacks.push(ten_bit_fallback(color, "any available codec")); + color = "8bit_420"; + chosen = pick(color); + } + } + chosen.ok_or_else(|| StreamerError { + code: "streamer_codec_unavailable", + message: "No available codec can decode the stream on the selected video backend. Export diagnostics for decoder probe failures.".to_owned(), + })? + }; + let mut resolved = settings.clone(); + resolved["codec"] = json!(codec); + resolved["enableHdr"] = json!(hdr); + resolved["colorQuality"] = json!(if hdr { hdr_color } else { color }); + normalize_embedded_mode(&mut resolved, &selected, codec, &mut fallbacks)?; + resolved["nativeHdrSupported"] = json!( + capabilities["nativeHdrSupported"] + .as_bool() + .unwrap_or(false) + ); + if let Some(object) = resolved.as_object_mut() { + object.remove("nativeHdrDisplay"); + } + if let Some(display) = validated_native_hdr_display(&capabilities["nativeHdrDisplay"]) { + let mut snapshot = json!({"minimumNits":display.minimum_nits, + "maximumNits":display.maximum_nits}); + if let Some(metadata) = display.metadata { + snapshot["maximumFullFrameNits"] = json!(metadata.maximum_full_frame_nits); + for (key, coordinate) in HDR_CHROMATICITY_KEYS.iter().zip(metadata.coordinates) { + snapshot[*key] = json!(coordinate); + } + } + resolved["nativeHdrDisplay"] = snapshot; + } + Ok((resolved, fallbacks)) + } + + pub fn prepare_embedded( + &self, + params: &Value, + settings: &Value, + ) -> Result { + let session = params["session"] + .as_object() + .map(|_| params["session"].clone()) + .ok_or_else(|| invalid("streamer.prepare requires a ready session"))?; + let status = session["status"].as_i64().unwrap_or_default(); + if !matches!(status, 2 | 3) { + return Err(invalid( + "CloudMatch session is not ready for NVST media attachment", + )); + } + let profile = &session["negotiatedStreamProfile"]; + if session["transportMode"] == "webrtc" { + if session["reportedNativeTransport"] == true { + return Err(invalid( + "CloudMatch returned a native allocation for this WebRTC session", + )); + } + if !crate::cloudmatch::has_webrtc_endpoint(&session) { + return Err(invalid( + "The WebRTC session has no valid signaling endpoint", + )); + } + if profile["codec"] + .as_str() + .is_some_and(|codec| !matches!(codec, "H264" | "H265")) + || (profile["codecSource"] == "server" && profile["codec"].as_str().is_none()) + || profile["enableHdr"] == true + || profile["colorQuality"].as_str().is_some_and(|color| { + color != "8bit_420" && !(profile["codec"] == "H265" && color == "10bit_420") + }) + { + return Err(invalid( + "WebRTC compatibility requires H.264 8-bit or H.265 8/10-bit SDR 4:2:0", + )); + } + } + if profile["enableHdrSource"] == "server" && profile["enableHdr"].as_bool().is_none() { + return Err(invalid("CloudMatch returned an unsupported HDR mode")); + } + if ["bitDepthSource", "chromaFormatSource"].iter().any(|key| { + matches!( + profile[*key].as_str(), + Some("request" | "finalized" | "server") + ) + }) && profile["colorQuality"].as_str().is_none() + { + return Err(invalid( + "CloudMatch returned an incomplete or unsupported color profile", + )); + } + let mut context = streamer_context(session, settings); + if !params["runtimeCapabilities"].is_null() { + let mut negotiated_settings = context["settings"].clone(); + if context["settings"]["transportMode"] != "webrtc" + && context["session"]["negotiatedStreamProfile"]["codec"] + .as_str() + .is_none() + { + negotiated_settings["codec"] = json!(settings["codec"].as_str().unwrap_or("auto")); + } + let resolved = Self::embedded_session_settings( + &negotiated_settings, + ¶ms["runtimeCapabilities"], + )?; + let codec = resolved["codec"].as_str().unwrap_or("h264"); + let profile = &context["session"]["negotiatedStreamProfile"]; + if resolved["resolution"] != context["settings"]["resolution"] + || resolved["fps"] != context["settings"]["fps"] + { + return Err(StreamerError { + code: "streamer_mode_unavailable", + message: "This device cannot decode the accepted resolution and frame rate. Start a new session.".to_owned(), + }); + } + let accepted_color = profile["colorQuality"].as_str(); + if accepted_color.is_some_and(|color| resolved["colorQuality"] != color) + || resolved["enableHdr"] != context["settings"]["enableHdr"] + || profile["codec"] + .as_str() + .is_some_and(|accepted| normalize_codec_name(accepted) != Some(codec)) + { + let accepted = [ + profile["codec"].as_str(), + accepted_color, + (context["settings"]["enableHdr"] == true).then_some("HDR"), + ] + .into_iter() + .flatten() + .collect::>() + .join(" "); + return Err(StreamerError { + code: "streamer_color_unavailable", + message: format!( + "This device cannot decode the accepted {accepted} stream profile. Start a new session." + ), + }); + } + context["settings"]["codec"] = json!(codec.to_ascii_uppercase()); + context["settings"]["colorQuality"] = resolved["colorQuality"].clone(); + context["settings"]["nativeHdrSupported"] = resolved["nativeHdrSupported"].clone(); + } + // The codec is client-selected (CloudMatch no longer carries it, matching + // the official client), so validate HDR against the normalized settings + // codec (negotiated-or-settings) rather than the raw profile, which is + // only populated when the server explicitly reports one. + if context["settings"]["enableHdr"].as_bool() == Some(true) + && !matches!( + ( + context["settings"]["codec"].as_str(), + context["session"]["negotiatedStreamProfile"]["colorQuality"].as_str(), + ), + (Some("H265" | "HEVC"), Some("10bit_420" | "10bit_444")) + | (Some("AV1"), Some("10bit_420")) + ) + { + return Err(invalid( + "The accepted HDR session requires 10-bit HEVC 4:2:0/4:4:4 or AV1 4:2:0", + )); + } + if context["settings"]["enableHdr"].as_bool() == Some(true) + && params["runtimeCapabilities"].is_null() + { + return Err(invalid( + "Resuming HDR requires current embedded window output capabilities", + )); + } + context["surface"] = Value::Null; + Ok(json!({ + "protocolVersion": STREAMER_PROTOCOL_VERSION, + "context": context + })) + } + + pub fn start(&self, params: &Value, settings: &Value) -> Result { + if params["session"]["transportMode"] == "webrtc" { + return Err(invalid( + "WebRTC compatibility requires owned embedded stream preparation", + )); + } + self.reap_finished(); + let mut worker = self.worker.lock().expect("streamer worker poisoned"); + if worker.is_some() { + return Err(StreamerError { + code: "streamer_busy", + message: "A native streamer session is already running".to_owned(), + }); + } + let executable = resolve_executable(settings)?; + if !executable.is_file() { + return Err(StreamerError { + code: "streamer_not_found", + message: format!( + "Native streamer is not installed at {}", + executable.display() + ), + }); + } + let session = params["session"] + .as_object() + .map(|_| params["session"].clone()) + .ok_or_else(|| invalid("streamer.start requires a ready session"))?; + let status = session["status"].as_i64().unwrap_or_default(); + if !matches!(status, 2 | 3) { + return Err(invalid( + "CloudMatch session is not ready for NVST media attachment", + )); + } + let session_id = required_string(&session, "sessionId")?.to_owned(); + let microphone_mode = settings["microphoneMode"].as_str().unwrap_or("disabled"); + let mut context = streamer_context(session, settings); + let surface = normalize_surface(params.get("surface"), context_resolution(&context))?; + context["surface"] = surface; + let (control_tx, control_rx) = mpsc::channel(); + let state = Arc::clone(&self.state); + { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.status = "starting".to_owned(); + snapshot.message = "Launching the native NVST media runtime…".to_owned(); + snapshot.session_id = Some(session_id.clone()); + snapshot.process_id = None; + snapshot.transport = Some("nvst".to_owned()); + snapshot.executable = Some(executable.clone()); + snapshot.error_code = None; + snapshot.started_at = Some(Instant::now()); + snapshot.session_started_at_ms = Some(unix_time_millis()); + snapshot.first_frame_latency_ms = None; + snapshot.media_backend = None; + snapshot.frames_per_second = None; + snapshot.bitrate_mbps = None; + snapshot.peak_bitrate_mbps = None; + snapshot.backend_fallback_count = 0; + snapshot.decoder_error_count = 0; + snapshot.output_error_count = 0; + snapshot.device_loss_count = 0; + snapshot.device_recovery_count = 0; + snapshot.queue_drop_count = 0; + snapshot.input_ready = false; + snapshot.input_unavailable_reason = None; + snapshot.input_pause_count = 0; + snapshot.input_resume_count = 0; + snapshot.surface_update_count = 0; + snapshot.fullscreen_toggle_count = 0; + snapshot.stats_toggle_count = 0; + snapshot.recording_start_count = 0; + snapshot.recording_stop_count = 0; + match microphone_mode { + "voice-activity" => { + snapshot.microphone_state = "muted".to_owned(); + snapshot.microphone_enabled = false; + snapshot.microphone_message = None; + } + "push-to-talk" => { + snapshot.microphone_state = "unavailable".to_owned(); + snapshot.microphone_enabled = false; + snapshot.microphone_message = + Some("Push-to-talk is not supported by the native NVST runtime".to_owned()); + } + _ => { + snapshot.microphone_state = "disabled".to_owned(); + snapshot.microphone_enabled = false; + snapshot.microphone_message = None; + } + } + } + let join = thread::Builder::new() + .name("opennow-streamer-coordinator".to_owned()) + .spawn(move || { + if let Err(error) = run_worker(&executable, context, control_rx, Arc::clone(&state)) + { + eprintln!( + "native-streamer worker failed [{}]: {}", + error.code, error.message + ); + set_error(&state, error.code, error.message); + } + }) + .map_err(|error| StreamerError { + code: "streamer_spawn_failed", + message: error.to_string(), + })?; + *worker = Some(Worker { + control: control_tx, + join, + }); + drop(worker); + Ok(self.status()) + } + + pub fn status(&self) -> Value { + self.reap_finished(); + self.state.lock().expect("streamer state poisoned").value() + } + + pub fn acceptance_snapshot(&self) -> Value { + self.reap_finished(); + self.state + .lock() + .expect("streamer state poisoned") + .acceptance_value() + } + + pub fn stop(&self, reason: &str) -> Result { + let item = self.worker.lock().expect("streamer worker poisoned").take(); + if let Some(worker) = item { + let _ = worker.control.send(WorkerCommand::Stop(reason.to_owned())); + worker.join.join().map_err(|_| StreamerError { + code: "streamer_stop_failed", + message: "Native streamer coordinator panicked during shutdown".to_owned(), + })?; + } + let mut snapshot = self.state.lock().expect("streamer state poisoned"); + snapshot.status = "stopped".to_owned(); + snapshot.message = reason.to_owned(); + snapshot.session_id = None; + snapshot.process_id = None; + snapshot.transport = None; + snapshot.error_code = None; + snapshot.microphone_state = "disabled".to_owned(); + snapshot.microphone_enabled = false; + snapshot.microphone_message = None; + snapshot.started_at = None; + Ok(snapshot.value()) + } + + pub fn set_input_paused(&self, paused: bool) -> Result { + self.reap_finished(); + let worker = self.worker.lock().expect("streamer worker poisoned"); + let Some(worker) = worker.as_ref() else { + return Ok(json!({"paused":paused,"streamerRunning":false})); + }; + worker + .control + .send(WorkerCommand::InputPaused(paused)) + .map_err(|_| StreamerError { + code: "streamer_control_failed", + message: "Native streamer control channel is closed".to_owned(), + })?; + let mut snapshot = self.state.lock().expect("streamer state poisoned"); + if paused { + snapshot.input_pause_count = snapshot.input_pause_count.saturating_add(1); + } else { + snapshot.input_resume_count = snapshot.input_resume_count.saturating_add(1); + } + Ok(json!({"paused":paused,"streamerRunning":true})) + } + + pub fn control(&self, action: &str) -> Result { + let child_command = match action { + "toggle-fullscreen" => "fullscreen-toggle", + "toggle-microphone" => "microphone-toggle", + "anti-afk-pulse" => "anti-afk-pulse", + _ => return Err(invalid("Unknown native streamer control action")), + }; + self.reap_finished(); + let worker = self.worker.lock().expect("streamer worker poisoned"); + let Some(worker) = worker.as_ref() else { + return Err(StreamerError { + code: "streamer_not_running", + message: "Native streamer is not running".to_owned(), + }); + }; + worker + .control + .send(WorkerCommand::Control(child_command.to_owned())) + .map_err(|_| StreamerError { + code: "streamer_control_failed", + message: "Native streamer control channel is closed".to_owned(), + })?; + Ok(json!({"action":action,"streamerRunning":true})) + } + + pub fn update_surface(&self, params: &Value) -> Result { + let surface = normalize_surface(params.get("surface"), (1280, 720))?; + self.reap_finished(); + let worker = self.worker.lock().expect("streamer worker poisoned"); + let Some(worker) = worker.as_ref() else { + return Ok(json!({"applied":false,"streamerRunning":false})); + }; + worker + .control + .send(WorkerCommand::Surface(surface)) + .map_err(|_| StreamerError { + code: "streamer_control_failed", + message: "Native streamer control channel is closed".to_owned(), + })?; + let mut snapshot = self.state.lock().expect("streamer state poisoned"); + snapshot.surface_update_count = snapshot.surface_update_count.saturating_add(1); + Ok(json!({"applied":true,"streamerRunning":true})) + } + + pub fn recording(&self, params: &Value, enabled: bool) -> Result { + self.reap_finished(); + let output_path = if enabled { + let path = PathBuf::from( + params["outputPath"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid("Native recording requires an outputPath"))?, + ); + if !path.is_absolute() + || path.extension().and_then(|value| value.to_str()) != Some("mkv") + { + return Err(invalid( + "Native recording requires an absolute .mkv outputPath", + )); + } + Some(path) + } else { + None + }; + let (reply, response) = mpsc::sync_channel(1); + { + let worker = self.worker.lock().expect("streamer worker poisoned"); + let Some(worker) = worker.as_ref() else { + return Err(StreamerError { + code: "streamer_not_running", + message: "Native streamer is not running".to_owned(), + }); + }; + worker + .control + .send(WorkerCommand::Recording { + enabled, + output_path, + reply, + }) + .map_err(|_| StreamerError { + code: "streamer_control_failed", + message: "Native streamer control channel is closed".to_owned(), + })?; + } + let result = response + .recv_timeout(Duration::from_secs(15)) + .map_err(|_| StreamerError { + code: "streamer_timeout", + message: "Native streamer timed out finalizing the recording".to_owned(), + })??; + let mut snapshot = self.state.lock().expect("streamer state poisoned"); + if enabled { + snapshot.recording_start_count = snapshot.recording_start_count.saturating_add(1); + } else { + snapshot.recording_stop_count = snapshot.recording_stop_count.saturating_add(1); + } + Ok(result) + } + + fn reap_finished(&self) { + let finished = self + .worker + .lock() + .expect("streamer worker poisoned") + .as_ref() + .is_some_and(|worker| worker.join.is_finished()); + if finished + && let Some(worker) = self.worker.lock().expect("streamer worker poisoned").take() + { + let _ = worker.join.join(); + } + } +} + +impl Drop for StreamerService { + fn drop(&mut self) { + if let Some(worker) = self + .worker + .get_mut() + .expect("streamer worker poisoned") + .take() + { + let _ = worker + .control + .send(WorkerCommand::Stop("OpenNOW core shutdown".to_owned())); + let _ = worker.join.join(); + } + } +} + +fn streamer_command(executable: &Path, settings: &Value) -> Command { + let requested_backend = settings["nativeVideoBackend"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase(); + let decoder_preference = settings["decoderPreference"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase(); + let decoder_backend = match requested_backend.as_str() { + "auto" | "" => match decoder_preference.as_str() { + "hardware" => "hardware", + "software" => "software", + _ => "auto", + }, + explicit => explicit, + }; + let cursor_overlay = if settings["nativeCursorOverlay"].as_bool().unwrap_or(true) { + "1" + } else { + "0" + }; + let mouse_sensitivity = settings["mouseSensitivity"] + .as_f64() + .unwrap_or(1.0) + .clamp(0.1, 3.0) + .to_string(); + let mouse_acceleration = settings["mouseAcceleration"] + .as_f64() + .unwrap_or(1.0) + .clamp(1.0, 150.0) + .to_string(); + let mut command = Command::new(executable); + command + .env("OPENNOW_NATIVE_EXTERNAL_RENDERER", "1") + // The separate SDL stream window owns foreground keyboard/mouse events. + // Keep this paired with EXTERNAL_RENDERER, matching native-streamer-v2; + // otherwise the streamer deliberately disables capture to avoid duplicate + // events from an embedded UI owner. + .env("OPENNOW_NATIVE_INPUT_OWNER", "native") + .env("OPENNOW_NATIVE_VIDEO_BACKEND", decoder_backend) + .env("OPENNOW_NATIVE_CURSOR_OVERLAY", cursor_overlay) + .env("OPENNOW_MOUSE_SENSITIVITY", mouse_sensitivity) + .env("OPENNOW_MOUSE_ACCELERATION", mouse_acceleration) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command +} + +fn probe_capabilities(executable: &Path, settings: &Value) -> Result { + let mut child = KillOnDrop(streamer_command(executable, settings).spawn().map_err( + |error| StreamerError { + code: "streamer_spawn_failed", + message: format!("Could not probe {}: {error}", executable.display()), + }, + )?); + let mut stdin = child + .stdin + .take() + .ok_or_else(|| internal("Streamer probe stdin is unavailable"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| internal("Streamer probe stdout is unavailable"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| internal("Streamer probe stderr is unavailable"))?; + let (child_tx, child_rx) = mpsc::channel::(); + let reader = spawn_stdout_reader(stdout, child_tx)?; + let stderr_reader = thread::Builder::new() + .name("opennow-streamer-probe-stderr".to_owned()) + .spawn(move || { + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + eprintln!("native-streamer probe: {line}"); + } + }) + .map_err(|error| StreamerError { + code: "streamer_spawn_failed", + message: error.to_string(), + })?; + let state = Arc::new(Mutex::new(Snapshot::default())); + write_child( + &mut stdin, + &json!({"id":"hello","type":"hello","protocolVersion":STREAMER_PROTOCOL_VERSION}), + )?; + let hello = wait_for_child(&child_rx, "hello", CAPABILITY_PROBE_TIMEOUT, &state)?; + if hello["type"] != "ready" { + return Err(child_error( + &hello, + "Native streamer capability probe failed", + )); + } + let capabilities = hello["capabilities"].clone(); + if !capabilities.is_object() + || capabilities["protocolVersion"].as_u64() != Some(STREAMER_PROTOCOL_VERSION) + { + return Err(StreamerError { + code: "streamer_protocol_mismatch", + message: format!( + "Native streamer did not report protocol {STREAMER_PROTOCOL_VERSION} capabilities" + ), + }); + } + let _ = write_child( + &mut stdin, + &json!({"id":"shutdown","type":"shutdown","reason":"capability probe complete"}), + ); + wait_or_kill(&mut child); + drop(stdin); + let _ = reader.join(); + let _ = stderr_reader.join(); + Ok(capabilities) +} + +pub(crate) fn requested_embedded_backend(settings: &Value) -> String { + let requested = settings["nativeVideoBackend"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase(); + if !requested.is_empty() && requested != "auto" { + return requested; + } + match settings["decoderPreference"] + .as_str() + .unwrap_or("auto") + .trim() + .to_ascii_lowercase() + .as_str() + { + "software" => "software".to_owned(), + _ => "auto".to_owned(), + } +} + +fn gpu_label(value: &str) -> String { + let mut cleaned = String::new(); + for character in value.chars().take(80) { + if character.is_alphanumeric() + || matches!( + character, + ' ' | '(' | ')' | '[' | ']' | '+' | '-' | '.' | '/' | ',' | '&' + ) + { + cleaned.push(character); + } else if character.is_whitespace() { + cleaned.push(' '); + } + } + let cleaned = cleaned.split_whitespace().collect::>().join(" "); + if cleaned.is_empty() { + return String::new(); + } + let redacted = crate::diagnostics::runtime_failure_reason(&cleaned); + if redacted.is_empty() || redacted == "[redacted]" { + String::new() + } else { + redacted + } +} + +fn codec_label(codec: &str) -> Option<&'static str> { + match codec { + "h264" => Some("H.264"), + "h265" => Some("H.265"), + "av1" => Some("AV1"), + _ => None, + } +} + +fn alternate_decoding_gpus(capabilities: &Value) -> Vec { + let Some(adapters) = capabilities + .get("graphicsAdapters") + .and_then(Value::as_array) + else { + return Vec::new(); + }; + if adapters.len() < 2 { + return Vec::new(); + } + let mut capable = Vec::new(); + for adapter in adapters.iter().take(8) { + if adapter["active"].as_bool() == Some(true) { + continue; + } + let labels = adapter["codecs"] + .as_array() + .into_iter() + .flatten() + .filter_map(|codec| codec.as_str().and_then(codec_label)) + .collect::>(); + if labels.is_empty() { + continue; + } + let name = gpu_label(adapter["name"].as_str().unwrap_or("")); + let name = if name.is_empty() { + "another GPU".to_owned() + } else { + name + }; + capable.push(format!("{name} ({})", labels.join(", "))); + if capable.len() == 3 { + break; + } + } + capable +} + +fn hybrid_gpu_hint(capabilities: &Value) -> Option { + let capable = alternate_decoding_gpus(capabilities); + if capable.is_empty() { + return None; + } + let target = if capable.len() == 1 { + "that GPU" + } else { + "one of those GPUs" + }; + Some(format!( + "{} can decode. Select {target} in Settings → Stream → Graphics processor, then restart OpenNOW.", + capable.join("; ") + )) +} + +pub(crate) fn active_gpu_label(capabilities: &Value) -> Option { + let adapters = capabilities.get("graphicsAdapters")?.as_array()?; + let adapter = adapters + .iter() + .find(|adapter| adapter["active"].as_bool() == Some(true))?; + let name = gpu_label(adapter["name"].as_str().unwrap_or("")); + if name.is_empty() { None } else { Some(name) } +} + +fn hybrid_gpu_message(capabilities: &Value) -> Option { + let hint = hybrid_gpu_hint(capabilities)?; + let selected = active_gpu_label(capabilities).unwrap_or_else(|| "the selected GPU".to_owned()); + let message = format!("No hardware video backend is available on {selected}. {hint}"); + Some(message.chars().take(480).collect()) +} + +fn normalize_codec_name(value: &str) -> Option<&'static str> { + match value.trim().to_ascii_lowercase().as_str() { + "h264" | "avc" | "auto" | "" => Some("h264"), + "h265" | "hevc" => Some("h265"), + "av1" => Some("av1"), + _ => None, + } +} + +/// GFN codec/color support matrix, matching the official client's settings gating: +/// H.264 is 8-bit 4:2:0 only, AV1 is 4:2:0 only (8- or 10-bit), HEVC supports all +/// four color modes, and Auto always resolves to a supported codec. Unknown names +/// fail closed; settings normalization clamps stored values before consulting this. +pub(crate) fn codec_supports_color_quality(codec: &str, color: &str) -> bool { + match codec.trim().to_ascii_lowercase().as_str() { + "" | "auto" => true, + "h264" | "avc" => color.trim().eq_ignore_ascii_case("8bit_420"), + "av1" => matches!( + color.trim().to_ascii_lowercase().as_str(), + "8bit_420" | "10bit_420" + ), + "h265" | "hevc" => true, + _ => false, + } +} + +fn available_codecs(capabilities: &Value) -> Vec<&'static str> { + ["h264", "h265", "av1"] + .into_iter() + .filter(|codec| codec_available(capabilities, codec)) + .collect() +} + +fn requested_resolution_pixels(settings: &Value) -> Option { + let (width, height) = settings["resolution"].as_str()?.split_once('x')?; + let width = width.parse::().ok().filter(|width| *width > 0)?; + let height = height.parse::().ok().filter(|height| *height > 0)?; + width.checked_mul(height) +} + +fn auto_av1_eligible(settings: &Value, capabilities: &Value, hdr: bool) -> bool { + if crate::frame_rate::snap_frame_rate(settings["fps"].as_i64().unwrap_or(60)) > 120 { + return false; + }; + if settings.get("resolution").is_some() + && requested_resolution_pixels(settings).is_none_or(|pixels| pixels > 5120 * 2880) + { + return false; + } + capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool() == Some(true)) + .flat_map(|backend| backend["codecs"].as_array().into_iter().flatten()) + .any(|codec| { + codec["codec"] + .as_str() + .is_some_and(|name| name.eq_ignore_ascii_case("av1")) + && codec["available"].as_bool() == Some(true) + && (!hdr || codec["hdrSupported"].as_bool() == Some(true)) + }) +} + +const EMBEDDED_RESOLUTIONS: [(u64, u64); 21] = [ + (1280, 720), + (1600, 900), + (1920, 1080), + (2560, 1440), + (3200, 1800), + (3840, 2160), + (5120, 2880), + (7680, 4320), + (1280, 800), + (1440, 900), + (1680, 1050), + (1920, 1200), + (2560, 1600), + (3840, 2400), + (2560, 1080), + (3440, 1440), + (3840, 1600), + (3840, 1800), + (5120, 2160), + (3840, 1080), + (5120, 1440), +]; + +fn normalize_embedded_mode( + settings: &mut Value, + capabilities: &Value, + codec: &str, + fallbacks: &mut Vec, +) -> Result<(), StreamerError> { + if let Some(requested) = settings["fps"].as_i64() { + let capped = if matches!(codec, "h264" | "av1") { + requested.min(120) + } else { + requested + }; + let fps = crate::frame_rate::snap_frame_rate(capped); + if fps != requested { + let action = if fps < requested { + "Capping" + } else { + "Normalizing" + }; + fallbacks.push(format!( + "{} requested at {requested} FPS. {action} the frame rate at {fps} FPS to a supported codec mode.", + codec_label(codec).unwrap_or(codec) + )); + settings["fps"] = json!(fps); + } + } + let mut max_width = u64::MAX; + let mut max_height = u64::MAX; + for backend in capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"] == true) + { + for entry in backend["codecs"] + .as_array() + .into_iter() + .flatten() + .filter(|entry| { + codec_entry_decodes( + backend, + entry, + codec, + settings["colorQuality"].as_str().unwrap_or("8bit_420"), + settings["enableHdr"] == true, + ) + }) + { + let Some(limit) = entry.get("maxDecodeDimensions") else { + continue; + }; + let width = limit["width"] + .as_u64() + .filter(|value| (1..=u32::MAX as u64).contains(value)); + let height = limit["height"] + .as_u64() + .filter(|value| (1..=u32::MAX as u64).contains(value)); + let (Some(width), Some(height)) = (width, height) else { + return Err(invalid( + "Embedded streamer reported invalid maximum decode dimensions", + )); + }; + max_width = max_width.min(width); + max_height = max_height.min(height); + } + } + let Some(resolution) = settings["resolution"].as_str() else { + return Ok(()); + }; + let Some((width, height)) = resolution.split_once('x').and_then(|(width, height)| { + Some((width.parse::().ok()?, height.parse::().ok()?)) + }) else { + return Ok(()); + }; + if width.checked_mul(height).is_none() { + return Ok(()); + } + let max_pixels = if codec == "h264" { 4_953_599 } else { u64::MAX }; + let fits = |width: u64, height: u64| { + width <= max_width + && height <= max_height + && width + .checked_mul(height) + .is_some_and(|pixels| pixels <= max_pixels) + }; + if fits(width, height) { + return Ok(()); + } + let aspect = width as f64 / height as f64; + let (selected_width, selected_height) = EMBEDDED_RESOLUTIONS.into_iter() + .filter(|(w, h)| *w <= width && *h <= height && fits(*w, *h)) + .min_by(|(aw, ah), (bw, bh)| { + (aspect - *aw as f64 / *ah as f64).abs() + .total_cmp(&(aspect - *bw as f64 / *bh as f64).abs()) + .then_with(|| (bw * bh).cmp(&(aw * ah))) + }) + .ok_or_else(|| StreamerError { + code: "streamer_mode_unavailable", + message: "No supported resolution fits the selected decoder's limits. Select another video backend.".to_owned(), + })?; + fallbacks.push(format!( + "{} requested at {resolution}. Using the closest supported aspect mode {selected_width}x{selected_height} within the decoder's limits.", + codec_label(codec).unwrap_or(codec) + )); + settings["resolution"] = json!(format!("{selected_width}x{selected_height}")); + Ok(()) +} + +fn codec_available(capabilities: &Value, codec: &str) -> bool { + capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool().unwrap_or(false)) + .flat_map(|backend| backend["codecs"].as_array().into_iter().flatten()) + .any(|entry| { + entry["codec"] + .as_str() + .is_some_and(|value| value.eq_ignore_ascii_case(codec)) + && entry["available"].as_bool().unwrap_or(false) + }) +} + +fn codec_decodes(capabilities: &Value, codec: &str, color: &str, hdr: bool) -> bool { + codec_supports_color_quality(codec, color) + && capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .filter(|backend| backend["available"].as_bool() == Some(true)) + .any(|backend| { + backend["codecs"] + .as_array() + .into_iter() + .flatten() + .any(|entry| codec_entry_decodes(backend, entry, codec, color, hdr)) + }) +} + +fn codec_entry_decodes( + backend: &Value, + entry: &Value, + codec: &str, + color: &str, + hdr: bool, +) -> bool { + let listed = |qualities: &Value| { + qualities.as_array().is_some_and(|qualities| { + qualities + .iter() + .any(|quality| quality.as_str() == Some(color)) + }) + }; + let requires_profiles = hdr + || backend["backend"] == "vulkan" + || backend["platform"] == "linux" + || backend["backend"] == "videotoolbox"; + let hdr_supported = entry + .get("hdrSupported") + .map(|value| value.as_bool().unwrap_or(false)); + entry["available"].as_bool() == Some(true) + && entry["codec"] + .as_str() + .is_some_and(|name| name.eq_ignore_ascii_case(codec)) + && (!hdr + || (hdr_supported != Some(false) + && entry + .get("hdrColorQualities") + .map_or(color == "10bit_420", listed))) + && entry.get("colorQualities").map_or( + (hdr && color == "10bit_420" && hdr_supported == Some(true)) + || !requires_profiles + || color == "8bit_420", + listed, + ) +} + +fn color_satisfies(delivered: &str, requested: &str) -> bool { + delivered == requested + || requested + .strip_prefix("8bit") + .is_some_and(|chroma| delivered.strip_prefix("10bit") == Some(chroma)) +} + +fn ensure_codec_available(capabilities: &Value, codec: &str) -> Result<(), StreamerError> { + if codec_available(capabilities, codec) { + return Ok(()); + } + let reasons = capabilities["videoBackends"] + .as_array() + .into_iter() + .flatten() + .flat_map(|backend| backend["codecs"].as_array().into_iter().flatten()) + .filter(|entry| { + entry["codec"] + .as_str() + .is_some_and(|value| value.eq_ignore_ascii_case(codec)) + }) + .filter_map(|entry| entry["reason"].as_str()) + .collect::>(); + let detail = reasons + .first() + .copied() + .unwrap_or("no selected native backend advertises this codec"); + Err(StreamerError { + code: "streamer_codec_unavailable", + message: format!("{} is unavailable: {detail}", codec.to_ascii_uppercase()), + }) +} + +fn run_worker( + executable: &Path, + context: Value, + control: Receiver, + state: Arc>, +) -> Result<(), StreamerError> { + let mut child = KillOnDrop( + streamer_command(executable, &context["settings"]) + .spawn() + .map_err(|error| StreamerError { + code: "streamer_spawn_failed", + message: format!("Could not launch {}: {error}", executable.display()), + })?, + ); + let process_id = u64::from(child.id()); + let mut stdin = child + .stdin + .take() + .ok_or_else(|| internal("Streamer stdin is unavailable"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| internal("Streamer stdout is unavailable"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| internal("Streamer stderr is unavailable"))?; + let (child_tx, child_rx) = mpsc::channel::(); + let reader = spawn_stdout_reader(stdout, child_tx)?; + let stderr_reader = thread::Builder::new() + .name("opennow-streamer-stderr".to_owned()) + .spawn(move || { + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + eprintln!("native-streamer: {line}"); + } + }) + .map_err(|error| StreamerError { + code: "streamer_spawn_failed", + message: error.to_string(), + })?; + + write_child( + &mut stdin, + &json!({"id":"hello","type":"hello","protocolVersion":STREAMER_PROTOCOL_VERSION}), + )?; + let hello = wait_for_child(&child_rx, "hello", CHILD_START_TIMEOUT, &state)?; + if hello["type"] != "ready" { + return Err(child_error(&hello, "Native streamer handshake failed")); + } + let negotiated_codec = context["settings"]["codec"] + .as_str() + .and_then(normalize_codec_name) + .unwrap_or("h264"); + ensure_codec_available(&hello["capabilities"], negotiated_codec)?; + { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.process_id = hello["processId"].as_u64().or(Some(process_id)); + snapshot.capabilities = hello["capabilities"].clone(); + snapshot.status = "starting".to_owned(); + snapshot.message = "Preparing native video and audio output…".to_owned(); + } + if hello["capabilities"]["supportsOwnedNvstNegotiation"].as_bool() != Some(true) { + return Err(StreamerError { + code: "streamer_protocol_mismatch", + message: "Native streamer cannot start NVST: owned NVST negotiation and protocol 7 are required" + .to_owned(), + }); + } + let result = run_owned_nvst_child( + &context, &mut stdin, &mut child, &child_rx, &control, &state, + ); + let _ = write_child( + &mut stdin, + &json!({"id":"shutdown","type":"shutdown","reason":"OpenNOW session ended"}), + ); + wait_or_kill(&mut child); + drop(stdin); + let _ = reader.join(); + let _ = stderr_reader.join(); + result +} + +fn run_owned_nvst_child( + context: &Value, + stdin: &mut ChildStdin, + child: &mut Child, + child_rx: &Receiver, + control: &Receiver, + state: &Arc>, +) -> Result<(), StreamerError> { + { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.status = "negotiating".to_owned(); + snapshot.message = + "Native streamer is negotiating the GeForce NOW NVST session…".to_owned(); + } + start_child(stdin, child_rx, context, state)?; + ensure_child_running(child)?; + write_surface(stdin, &context["surface"])?; + { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.transport = Some("nvst".to_owned()); + snapshot.status = "streaming".to_owned(); + snapshot.message = "Native-owned NVST media transport is active".to_owned(); + } + + loop { + if let Ok(command) = control.try_recv() { + match command { + WorkerCommand::Stop(reason) => { + let _ = write_child(stdin, &json!({"id":"stop","type":"stop","reason":reason})); + return Ok(()); + } + WorkerCommand::InputPaused(paused) => { + write_child( + stdin, + &json!({"id":"input-paused","type":"input-paused","paused":paused}), + )?; + } + WorkerCommand::Control(command_type) => { + write_child(stdin, &json!({"id":"runtime-control","type":command_type}))?; + } + WorkerCommand::Surface(surface) => write_surface(stdin, &surface)?, + WorkerCommand::Recording { + enabled, + output_path, + reply, + } => { + let result = + execute_recording(stdin, child_rx, state, enabled, output_path.as_deref()); + let _ = reply.send(result); + } + } + } + if let Some(status) = child + .try_wait() + .map_err(|error| internal(error.to_string()))? + { + return Err(StreamerError { + code: "streamer_exited", + message: format!("Native streamer exited unexpectedly ({status})"), + }); + } + while let Ok(message) = child_rx.try_recv() { + handle_nvst_child_event(&message, state)?; + } + thread::sleep(Duration::from_millis(20)); + } +} + +fn start_child( + stdin: &mut ChildStdin, + child_rx: &Receiver, + context: &Value, + state: &Arc>, +) -> Result<(), StreamerError> { + write_child( + stdin, + &json!({"id":"start","type":"start","context":context}), + )?; + let started = wait_for_child(child_rx, "start", CHILD_START_TIMEOUT, state)?; + if started["type"] != "ok" { + return Err(child_error(&started, "Native NVST media startup failed")); + } + state.lock().expect("streamer state poisoned").transport = + started["transport"].as_str().map(ToOwned::to_owned); + Ok(()) +} + +fn handle_nvst_child_event( + message: &Value, + state: &Arc>, +) -> Result<(), StreamerError> { + apply_child_telemetry(message, state); + match message["type"].as_str().unwrap_or_default() { + "status" => { + let mut snapshot = state.lock().expect("streamer state poisoned"); + if let Some(value) = message["status"].as_str() { + snapshot.status = value.to_owned() + } + if let Some(value) = message["message"].as_str() { + snapshot.message = value.to_owned() + } + Ok(()) + } + "error" => Err(StreamerError { + code: "native_stream_error", + message: message["message"] + .as_str() + .unwrap_or("Native NVST media error") + .to_owned(), + }), + "overlay-request" => { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.overlay_request_generation = + snapshot.overlay_request_generation.wrapping_add(1); + Ok(()) + } + "screenshot-request" => { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.screenshot_request_generation = + snapshot.screenshot_request_generation.wrapping_add(1); + Ok(()) + } + "recording-toggle-request" => { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.recording_toggle_request_generation = + snapshot.recording_toggle_request_generation.wrapping_add(1); + Ok(()) + } + "shortcut-action" => { + let action = message["action"] + .as_str() + .filter(|action| { + matches!( + *action, + "toggle-stats" + | "toggle-fullscreen" + | "stop-stream" + | "toggle-anti-afk" + | "toggle-microphone" + | "screenshot" + | "toggle-recording" + ) + }) + .ok_or_else(|| StreamerError { + code: "streamer_protocol_error", + message: "Native streamer emitted an invalid shortcut action".to_owned(), + })?; + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.shortcut_action_generation = + snapshot.shortcut_action_generation.wrapping_add(1); + if action == "toggle-stats" { + snapshot.stats_toggle_count = snapshot.stats_toggle_count.saturating_add(1); + } else if action == "toggle-fullscreen" { + snapshot.fullscreen_toggle_count = + snapshot.fullscreen_toggle_count.saturating_add(1); + } + snapshot.shortcut_action = Some(action.to_owned()); + Ok(()) + } + "microphone-state" => { + apply_microphone_state(message, state); + Ok(()) + } + _ => Ok(()), + } +} + +fn apply_microphone_state(message: &Value, state: &Arc>) { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.microphone_state = message["state"] + .as_str() + .unwrap_or("unavailable") + .to_owned(); + snapshot.microphone_enabled = message["enabled"].as_bool().unwrap_or(false); + snapshot.microphone_message = message["message"].as_str().map(ToOwned::to_owned); +} + +fn apply_child_telemetry(message: &Value, state: &Arc>) { + let mut snapshot = state.lock().expect("streamer state poisoned"); + match message["type"].as_str().unwrap_or_default() { + "input-ready" => { + snapshot.input_ready = true; + snapshot.input_unavailable_reason = None; + } + "input-unavailable" => { + snapshot.input_ready = false; + snapshot.input_unavailable_reason = message["reason"].as_str().map(ToOwned::to_owned); + } + "telemetry" => { + let metric = |key: &str, maximum: f64| { + message[key] + .as_f64() + .filter(|value| value.is_finite() && (0.0..=maximum).contains(value)) + }; + if let Some(value) = metric("framesPerSecond", 1_000.0) { + snapshot.frames_per_second = Some(value); + } + if let Some(value) = metric("bitrateMbps", 10_000.0) { + snapshot.bitrate_mbps = Some(value); + } + if let Some(value) = metric("peakBitrateMbps", 10_000.0) { + snapshot.peak_bitrate_mbps = Some(value); + } + } + _ => {} + } + + match message["event"].as_str().unwrap_or_default() { + "first-frame" => { + if snapshot.first_frame_latency_ms.is_none() { + snapshot.first_frame_latency_ms = snapshot + .started_at + .map(|started| elapsed_millis(started.elapsed())); + } + snapshot.media_backend = message["backend"].as_str().map(ToOwned::to_owned); + } + "backend-fallback" => { + snapshot.backend_fallback_count = snapshot.backend_fallback_count.saturating_add(1); + } + "decoder-error" => { + snapshot.decoder_error_count = snapshot.decoder_error_count.saturating_add(1); + } + "output-error" => { + snapshot.output_error_count = snapshot.output_error_count.saturating_add(1); + } + "device-state" => { + if message["recovered"].as_bool().unwrap_or(false) { + snapshot.device_recovery_count = snapshot.device_recovery_count.saturating_add(1); + } else { + snapshot.device_loss_count = snapshot.device_loss_count.saturating_add(1); + } + } + "queue-dropped" => { + snapshot.queue_drop_count = snapshot + .queue_drop_count + .saturating_add(message["count"].as_u64().unwrap_or_default()); + } + _ => {} + } +} + +const HDR_CHROMATICITY_KEYS: [&str; 8] = [ + "redX", "redY", "greenX", "greenY", "blueX", "blueY", "whiteX", "whiteY", +]; + +pub(crate) struct NativeHdrDisplay { + pub minimum_nits: f64, + pub maximum_nits: f64, + pub metadata: Option, +} + +pub(crate) struct NativeHdrMetadata { + pub maximum_full_frame_nits: f64, + pub coordinates: [f64; 8], +} + +pub(crate) fn validated_native_hdr_display(display: &Value) -> Option { + let display = display.as_object()?; + let minimum = display.get("minimumNits")?.as_f64()?; + let maximum = display.get("maximumNits")?.as_f64()?; + if !minimum.is_finite() + || !maximum.is_finite() + || minimum < 0.0 + || maximum > 10_000.0 + || maximum <= minimum + { + return None; + } + let metadata = (|| { + let full_frame = display.get("maximumFullFrameNits")?.as_f64()?; + if !full_frame.is_finite() || full_frame <= minimum || full_frame > maximum { + return None; + } + let mut coordinates = [0.0; 8]; + for (coordinate, key) in coordinates.iter_mut().zip(HDR_CHROMATICITY_KEYS) { + *coordinate = display.get(key)?.as_f64()?; + } + if coordinates.chunks_exact(2).any(|xy| { + !xy[0].is_finite() + || !xy[1].is_finite() + || !(0.0..=1.0).contains(&xy[0]) + || !(0.0..=1.0).contains(&xy[1]) + || xy[0] + xy[1] > 1.0 + || xy[1] == 0.0 + }) { + return None; + } + let [rx, ry, gx, gy, bx, by, ..] = coordinates; + if ((gx - rx) * (by - ry) - (gy - ry) * (bx - rx)).abs() <= 1e-6 { + return None; + } + Some(NativeHdrMetadata { + maximum_full_frame_nits: full_frame, + coordinates, + }) + })(); + Some(NativeHdrDisplay { + minimum_nits: minimum, + maximum_nits: maximum, + metadata, + }) +} + +fn streamer_context(mut session: Value, settings: &Value) -> Value { + let mut normalized = settings.clone(); + let negotiated_codec = session["negotiatedStreamProfile"]["codec"].as_str(); + let requested_codec = if session["transportMode"] == "webrtc" { + "H264".to_owned() + } else { + normalized["codec"].as_str().unwrap_or("auto").to_owned() + }; + let codec = negotiated_codec.map(ToOwned::to_owned).unwrap_or_else(|| { + if requested_codec.eq_ignore_ascii_case("auto") { + "H264".to_owned() + } else { + requested_codec + } + }); + normalized["codec"] = Value::String(codec.to_ascii_uppercase()); + normalized["enableHdr"] = json!( + session["negotiatedStreamProfile"]["enableHdr"] + .as_bool() + .unwrap_or(false) + ); + normalized["nativeHdrSupported"] = json!(false); + if let Some(color) = session["negotiatedStreamProfile"]["colorQuality"].as_str() { + normalized["colorQuality"] = json!(color); + } + normalized["transportMode"] = json!(if session["transportMode"] == "webrtc" { + "webrtc" + } else { + "nvst" + }); + if session["transportMode"] == "webrtc" { + normalized["colorQuality"] = json!( + session["negotiatedStreamProfile"]["colorQuality"] + .as_str() + .unwrap_or("8bit_420") + ); + normalized["enableHdr"] = json!(false); + normalized["microphoneMode"] = json!("disabled"); + normalized["enableCloudGsync"] = json!(false); + normalized["nativeCloudGsyncMode"] = json!("disabled"); + normalized["resolution"] = json!("1920x1080"); + normalized["fps"] = json!(60); + } + for key in ["resolution", "fps"] { + if !session["negotiatedStreamProfile"][key].is_null() { + normalized[key] = session["negotiatedStreamProfile"][key].clone(); + } + } + if session["negotiatedStreamProfile"].is_null() { + session["negotiatedStreamProfile"] = json!({"codec":codec.to_ascii_uppercase()}); + } + json!({ + "session":session, + "settings":normalized, + "shortcuts":{ + "toggleStats":settings["shortcutToggleStats"], + "togglePointerLock":settings["shortcutTogglePointerLock"], + "toggleFullscreen":settings["shortcutToggleFullscreen"], + "stopStream":settings["shortcutStopStream"], + "toggleAntiAfk":settings["shortcutToggleAntiAfk"], + "toggleMicrophone":settings["shortcutToggleMicrophone"], + "screenshot":settings["shortcutScreenshot"], + "toggleRecording":settings["shortcutToggleRecording"], + "saveClip":settings["shortcutSaveClip"] + } + }) +} + +fn context_resolution(context: &Value) -> (u64, u64) { + context["settings"]["resolution"] + .as_str() + .and_then(|value| value.split_once('x')) + .and_then(|(width, height)| Some((width.parse().ok()?, height.parse().ok()?))) + .filter(|(width, height)| *width > 0 && *height > 0) + .unwrap_or((1280, 720)) +} + +fn normalize_surface( + raw: Option<&Value>, + fallback_size: (u64, u64), +) -> Result { + let fallback = json!({ + "rect":{"x":0,"y":0,"width":fallback_size.0,"height":fallback_size.1}, + "screenRect":{"x":0,"y":0,"width":fallback_size.0,"height":fallback_size.1}, + "visible":true, + "deviceScaleFactor":1.0 + }); + let value = raw.unwrap_or(&fallback); + let object = value + .as_object() + .ok_or_else(|| invalid("streamer surface must be an object"))?; + let visible = object + .get("visible") + .and_then(Value::as_bool) + .unwrap_or(true); + let screen = object + .get("screenRect") + .and_then(Value::as_object) + .or_else(|| object.get("rect").and_then(Value::as_object)) + .ok_or_else(|| invalid("streamer surface requires screenRect"))?; + let parse_rect = |value: &serde_json::Map, label: &str| { + let width = value + .get("width") + .and_then(Value::as_u64) + .filter(|value| (64..=16_384).contains(value)) + .ok_or_else(|| invalid(format!("streamer {label} width is out of range")))?; + let height = value + .get("height") + .and_then(Value::as_u64) + .filter(|value| (64..=16_384).contains(value)) + .ok_or_else(|| invalid(format!("streamer {label} height is out of range")))?; + let x = value.get("x").and_then(Value::as_i64).unwrap_or(0); + let y = value.get("y").and_then(Value::as_i64).unwrap_or(0); + if !(-100_000..=100_000).contains(&x) || !(-100_000..=100_000).contains(&y) { + return Err(invalid(format!( + "streamer {label} position is out of range" + ))); + } + Ok((x, y, width, height)) + }; + let (screen_x, screen_y, screen_width, screen_height) = parse_rect(screen, "screenRect")?; + let (local_x, local_y, local_width, local_height) = object + .get("rect") + .and_then(Value::as_object) + .map(|rect| parse_rect(rect, "rect")) + .transpose()? + .unwrap_or((0, 0, screen_width, screen_height)); + let scale = object + .get("deviceScaleFactor") + .and_then(Value::as_f64) + .unwrap_or(1.0); + if !scale.is_finite() || !(0.5..=8.0).contains(&scale) { + return Err(invalid("streamer surface scale is out of range")); + } + let window_handle = object + .get("windowHandle") + .and_then(Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= 512); + let mut normalized = json!({ + "rect":{"x":local_x,"y":local_y,"width":local_width,"height":local_height}, + "screenRect":{"x":screen_x,"y":screen_y,"width":screen_width,"height":screen_height}, + "visible":visible, + "deviceScaleFactor":scale + }); + if let Some(window_handle) = window_handle { + normalized["windowHandle"] = Value::String(window_handle.to_owned()); + } + Ok(normalized) +} + +fn write_surface(stdin: &mut ChildStdin, surface: &Value) -> Result<(), StreamerError> { + write_child( + stdin, + &json!({"id":"surface","type":"surface","surface":surface}), + ) +} + +fn execute_recording( + stdin: &mut ChildStdin, + child_rx: &Receiver, + state: &Arc>, + enabled: bool, + output_path: Option<&Path>, +) -> Result { + static RECORDING_COMMAND_SEQUENCE: AtomicU64 = AtomicU64::new(0); + let id = format!( + "recording-control-{}", + RECORDING_COMMAND_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ); + write_child( + stdin, + &json!({ + "id":id, + "type":if enabled { "recording-start" } else { "recording-stop" }, + "outputPath":output_path.map(|path| path.to_string_lossy().into_owned()), + }), + )?; + let response = wait_for_child(child_rx, &id, Duration::from_secs(12), state)?; + if response["type"] == "error" { + return Err(StreamerError { + code: "recording_failed", + message: response["message"] + .as_str() + .unwrap_or("Native recording failed") + .to_owned(), + }); + } + Ok(json!({ + "enabled":enabled, + "path":response["path"], + "videoPackets":response["videoPackets"], + "audioPackets":response["audioPackets"], + "requestId":if enabled { Value::String(id) } else { response["requestId"].clone() }, + "completion":response["completion"], + "streamerRunning":true, + })) +} + +fn spawn_stdout_reader( + stdout: impl std::io::Read + Send + 'static, + messages: Sender, +) -> Result, StreamerError> { + thread::Builder::new() + .name("opennow-streamer-stdout".to_owned()) + .spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if line.len() > CHILD_MESSAGE_LIMIT { + break; + } + if let Ok(message) = serde_json::from_str(&line) { + let _ = messages.send(message); + } + } + }) + .map_err(|error| StreamerError { + code: "streamer_spawn_failed", + message: error.to_string(), + }) +} + +fn wait_for_child( + receiver: &Receiver, + id: &str, + timeout: Duration, + state: &Arc>, +) -> Result { + let deadline = Instant::now() + timeout; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(StreamerError { + code: "streamer_timeout", + message: format!("Native streamer timed out waiting for {id}"), + }); + } + let message = match receiver.recv_timeout(remaining) { + Ok(message) => message, + Err(mpsc::RecvTimeoutError::Timeout) => { + return Err(StreamerError { + code: "streamer_timeout", + message: format!("Native streamer timed out waiting for {id}"), + }); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(StreamerError { + code: "streamer_exited", + message: format!( + "Native streamer closed its protocol output while waiting for {id}" + ), + }); + } + }; + if message["id"].as_str() == Some(id) { + return Ok(message); + } + if message["type"].as_str() == Some("status") { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.status = message["status"].as_str().unwrap_or("starting").to_owned(); + snapshot.message = message["message"].as_str().unwrap_or("").to_owned(); + } + } +} + +fn write_child(stdin: &mut ChildStdin, message: &Value) -> Result<(), StreamerError> { + serde_json::to_writer(&mut *stdin, message).map_err(|error| { + error + .io_error_kind() + .map(|kind| child_io_error(kind, error.to_string())) + .unwrap_or_else(|| internal(error.to_string())) + })?; + stdin + .write_all(b"\n") + .map_err(|error| child_io_error(error.kind(), error.to_string()))?; + stdin + .flush() + .map_err(|error| child_io_error(error.kind(), error.to_string())) +} + +fn child_io_error(kind: ErrorKind, message: String) -> StreamerError { + let code = if matches!( + kind, + ErrorKind::BrokenPipe + | ErrorKind::ConnectionAborted + | ErrorKind::ConnectionReset + | ErrorKind::UnexpectedEof + ) { + "streamer_exited" + } else { + "streamer_io_failed" + }; + StreamerError { code, message } +} + +fn wait_or_kill(child: &mut Child) { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + if child.try_wait().ok().flatten().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = child.kill(); + let _ = child.wait(); +} + +fn ensure_child_running(child: &mut Child) -> Result<(), StreamerError> { + match child.try_wait() { + Ok(None) => Ok(()), + Ok(Some(status)) => Err(StreamerError { + code: "streamer_exited", + message: format!("Native streamer exited unexpectedly ({status})"), + }), + Err(error) => Err(internal(format!( + "Could not inspect native streamer state: {error}" + ))), + } +} + +fn resolve_executable(settings: &Value) -> Result { + if let Some(configured) = settings["nativeStreamerExecutablePath"] + .as_str() + .filter(|value| !value.trim().is_empty()) + { + return Ok(PathBuf::from(configured)); + } + let current = std::env::current_exe().map_err(|error| StreamerError { + code: "streamer_not_found", + message: error.to_string(), + })?; + Ok(current + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(if cfg!(target_os = "windows") { + "opennow-streamer.exe" + } else { + "opennow-streamer" + })) +} + +fn required_string<'a>(value: &'a Value, key: &str) -> Result<&'a str, StreamerError> { + value[key] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid(format!("Missing session {key}"))) +} + +fn child_error(value: &Value, fallback: &str) -> StreamerError { + StreamerError { + code: "native_stream_error", + message: value["message"].as_str().unwrap_or(fallback).to_owned(), + } +} + +fn set_error(state: &Arc>, code: &str, message: String) { + let mut snapshot = state.lock().expect("streamer state poisoned"); + snapshot.status = "error".to_owned(); + snapshot.message = message; + snapshot.error_code = Some(code.to_owned()); + snapshot.process_id = None; + snapshot.microphone_enabled = false; + if snapshot.microphone_state != "disabled" { + snapshot.microphone_state = "unavailable".to_owned(); + } +} + +fn elapsed_millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +fn unix_time_millis() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +#[cfg(all(test, unix))] +fn random_u64() -> u64 { + let mut bytes = [0_u8; 8]; + rand::rng().fill_bytes(&mut bytes); + u64::from_le_bytes(bytes) +} + +fn invalid(message: impl Into) -> StreamerError { + StreamerError { + code: "invalid_params", + message: message.into(), + } +} + +fn internal(message: impl Into) -> StreamerError { + StreamerError { + code: "streamer_internal_error", + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn recording_bridge_preserves_completion_and_unique_start_identity() { + let mut child = Command::new("cat") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let responder = thread::spawn(move || { + let mut lines = BufReader::new(stdout).lines(); + let mut starts = Vec::new(); + for index in 0..4 { + let request: Value = serde_json::from_str(&lines.next().unwrap().unwrap()).unwrap(); + let response = if index < 2 { + starts.push(request["id"].clone()); + json!({"id":request["id"],"type":"recording-started","path":"/capture.mkv"}) + } else if index == 2 { + json!({"id":request["id"],"type":"recording-stopped","requestId":starts[1], + "path":"/capture.mkv","videoPackets":5,"audioPackets":3, + "completion":{"kind":"cut","reason":"queue-overflow"}}) + } else { + json!({"id":request["id"],"type":"error","message":"trailer failed"}) + }; + sender.send(response).unwrap(); + } + starts + }); + let state = Arc::new(Mutex::new(Snapshot::default())); + let first = execute_recording( + &mut stdin, + &receiver, + &state, + true, + Some(Path::new("/first.mkv")), + ) + .unwrap(); + let second = execute_recording( + &mut stdin, + &receiver, + &state, + true, + Some(Path::new("/second.mkv")), + ) + .unwrap(); + let stopped = execute_recording(&mut stdin, &receiver, &state, false, None).unwrap(); + let failed = execute_recording(&mut stdin, &receiver, &state, false, None).unwrap_err(); + drop(stdin); + assert!(child.wait().unwrap().success()); + let starts = responder.join().unwrap(); + assert_ne!(starts[0], starts[1]); + assert_eq!(first["requestId"], starts[0]); + assert_eq!(second["requestId"], starts[1]); + assert_eq!(stopped["requestId"], starts[1]); + assert_eq!( + stopped["completion"], + json!({"kind":"cut","reason":"queue-overflow"}) + ); + assert_eq!(stopped["videoPackets"], 5); + assert_eq!(stopped["audioPackets"], 3); + assert_eq!(failed.code, "recording_failed"); + assert_eq!(failed.message, "trailer failed"); + } + + #[test] + fn external_stream_window_is_the_native_input_owner() { + let command = streamer_command( + Path::new("opennow-streamer"), + &json!({ + "nativeVideoBackend":"auto", + "decoderPreference":"auto", + "nativeCursorOverlay":true, + "mouseSensitivity":1.0, + "mouseAcceleration":1.0 + }), + ); + let environment = command + .get_envs() + .filter_map(|(name, value)| Some((name.to_str()?, value?.to_str()?))) + .collect::>(); + assert_eq!( + environment.get("OPENNOW_NATIVE_EXTERNAL_RENDERER"), + Some(&"1") + ); + assert_eq!( + environment.get("OPENNOW_NATIVE_INPUT_OWNER"), + Some(&"native") + ); + } + + #[test] + fn crashed_child_is_reported_as_a_typed_streamer_failure() { + #[cfg(target_os = "windows")] + let mut child = Command::new("cmd") + .args(["/C", "exit", "23"]) + .spawn() + .expect("crash fixture"); + #[cfg(not(target_os = "windows"))] + let mut child = Command::new("sh") + .args(["-c", "exit 23"]) + .spawn() + .expect("crash fixture"); + child.wait().expect("crash fixture exit"); + + let failure = ensure_child_running(&mut child).expect_err("exited child must fail"); + assert_eq!(failure.code, "streamer_exited"); + assert!(failure.message.contains("23")); + } + + #[test] + fn webrtc_hevc_preparation_preserves_owned_output() { + for color in ["8bit_420", "10bit_420"] { + let session = json!({"sessionId":"hevc-seat","status":3,"transportMode":"webrtc", + "signalingUrl":"wss://seat.partner.example/nvst/", + "connectionInfo":[{"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","port":443,"resourcePath":"/nvst/"}], + "negotiatedStreamProfile":{"codec":"H265","codecSource":"request", + "colorQuality":color,"enableHdr":false,"resolution":"1920x1200","fps":60}}); + let settings = json!({"codec":"h264","colorQuality":"8bit_420", + "resolution":"1280x720","fps":30,"enableHdr":true}); + let prepared = StreamerService::new() + .prepare_embedded(&json!({"session":session}), &settings) + .unwrap(); + let output = &prepared["context"]["settings"]; + assert_eq!(output["codec"], "H265"); + assert_eq!(output["colorQuality"], color); + assert_eq!(output["resolution"], "1920x1200"); + assert_eq!(output["fps"], 60); + assert_eq!(output["enableHdr"], false); + assert_eq!(prepared["context"]["session"], session); + for available in [false, true] { + let capabilities = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true, + "codecs":[{"codec":"h265","available":available, + "colorQualities":["8bit_420","10bit_420"]}]}]}); + assert_eq!( + StreamerService::new() + .prepare_embedded( + &json!({"session":session,"runtimeCapabilities":capabilities}), + &settings, + ) + .is_ok(), + available + ); + } + } + } + + #[test] + fn alliance_webrtc_preparation_uses_owned_profile_not_current_preferences() { + let service = StreamerService::new(); + let session = json!({"sessionId":"seat","status":3,"transportMode":"webrtc", + "signalingUrl":"wss://seat.partner.example:8443/nvst/", + "connectionInfo":[{"usage":14,"appLevelProtocol":4,"ip":"seat.partner.example","port":8443,"resourcePath":"/nvst/"}], + "negotiatedStreamProfile":{"codec":"H264","codecSource":"request","colorQuality":"8bit_420", + "enableHdr":false,"resolution":"1280x720","fps":30}}); + let settings = json!({"transportMode":"nvst","allianceWebrtcCompatibility":false, + "codec":"av1","colorQuality":"10bit_444","enableHdr":true, + "microphoneMode":"open","resolution":"3840x2160","fps":240}); + assert!( + service + .start(&json!({"session":session}), &settings) + .is_err() + ); + let prepared = service + .prepare_embedded(&json!({"session":session}), &settings) + .unwrap(); + let output = &prepared["context"]["settings"]; + assert_eq!(output["transportMode"], "webrtc"); + assert_eq!(output["codec"], "H264"); + assert_eq!(output["colorQuality"], "8bit_420"); + assert_eq!(output["enableHdr"], false); + assert_eq!(output["microphoneMode"], "disabled"); + assert_eq!(output["resolution"], "1280x720"); + assert_eq!(output["fps"], 30); + assert_eq!(prepared["context"]["session"], session); + for (key, value) in [ + ("codec", json!("AV1")), + ("enableHdr", json!(true)), + ("colorQuality", json!("10bit_420")), + ] { + let mut incompatible = session.clone(); + incompatible["negotiatedStreamProfile"][key] = value; + assert!( + service + .prepare_embedded(&json!({"session":incompatible}), &settings) + .is_err() + ); + } + let mut native_endpoint = session.clone(); + native_endpoint["reportedNativeTransport"] = json!(true); + assert!( + service + .prepare_embedded(&json!({"session":native_endpoint}), &settings) + .is_err() + ); + native_endpoint["reportedNativeTransport"] = json!(false); + native_endpoint["connectionInfo"][0]["appLevelProtocol"] = json!(6); + assert!( + service + .prepare_embedded(&json!({"session":native_endpoint}), &settings) + .is_err() + ); + native_endpoint["signalingUrl"] = json!("wss://seat.partner.example/nvst/"); + assert!( + service + .prepare_embedded(&json!({"session":native_endpoint}), &settings) + .is_ok() + ); + } + + #[test] + fn closed_child_protocol_is_not_misreported_as_a_timeout() { + let (sender, receiver) = mpsc::channel(); + drop(sender); + let state = Arc::new(Mutex::new(Snapshot::default())); + let failure = wait_for_child(&receiver, "hello", Duration::from_secs(1), &state) + .expect_err("closed protocol must fail"); + assert_eq!(failure.code, "streamer_exited"); + } + + #[cfg(unix)] + #[test] + fn streamer_service_contains_a_crashing_child_and_publishes_error_state() { + use std::os::unix::fs::PermissionsExt as _; + + let fixture = std::env::temp_dir().join(format!( + "opennow-crashing-streamer-{}-{}", + std::process::id(), + random_u64() + )); + fs::write( + &fixture, + "#!/bin/sh\nread -r _line\nprintf '%s\\n' '{\"id\":\"hello\",\"type\":\"ready\",\"processId\":1,\"capabilities\":{\"protocolVersion\":7,\"supportsOwnedNvstNegotiation\":true,\"videoBackends\":[{\"available\":true,\"codecs\":[{\"codec\":\"h264\",\"available\":true}]}]}}'\nexit 23\n", + ) + .expect("write crash fixture"); + fs::set_permissions(&fixture, fs::Permissions::from_mode(0o700)) + .expect("make crash fixture executable"); + + let service = StreamerService::new(); + service + .start( + &json!({ + "session":{ + "status":2, + "sessionId":"crash-fixture", + "rtspsEndpoints":["rtsps://seat.nvidiagrid.net:322/session"] + } + }), + &json!({ + "nativeStreamerExecutablePath":fixture, + "transportMode":"nvst", + "codec":"H264", + "resolution":"1920x1080" + }), + ) + .expect("coordinator starts asynchronously"); + + let deadline = Instant::now() + Duration::from_secs(2); + let state = loop { + let state = service.status(); + if state["streamer"]["status"] == "error" || Instant::now() >= deadline { + break state; + } + thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(state["streamer"]["status"], "error"); + assert_eq!(state["streamer"]["errorCode"], "streamer_exited"); + assert_eq!(state["streamer"]["processId"], Value::Null); + + fs::remove_file(fixture).expect("remove crash fixture"); + } + + #[test] + fn color_descriptors_match_final_profile_validation_without_mutation() { + for backend in ["d3d11", "vulkan", "videotoolbox", "missing"] { + for codec in ["auto", "h264", "h265", "av1"] { + for hdr in [false, true] { + for known in [false, true] { + let caps = if known { + json!({"protocolVersion":7,"nativeHdrSupported":true, + "videoBackends":[{"backend":backend,"platform":"macos","available":true, + "codecs":[{"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","8bit_444","10bit_420","10bit_444"], + "hdrColorQualities":["10bit_420","10bit_444"]}, + {"codec":"av1","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420"]}]}]}) + } else { + json!({}) + }; + let settings = json!({"codec":codec,"enableHdr":hdr,"nativeVideoBackend":backend, + "colorQuality":"8bit_444"}); + let original = settings.clone(); + let choices = StreamerService::color_quality_choices(&settings, &caps); + assert_eq!(choices.as_array().unwrap().len(), 4); + for choice in choices.as_array().unwrap() { + let mut candidate = settings.clone(); + candidate["colorQuality"] = choice["value"].clone(); + match StreamerService::embedded_session_settings(&candidate, &caps) { + Err(error) => { + assert_eq!(choice["disabled"], true); + assert_eq!(choice["reason"], error.message); + } + Ok(resolved) => { + let delivered = resolved["colorQuality"].as_str().unwrap(); + let satisfied = color_satisfies( + delivered, + choice["value"].as_str().unwrap(), + ); + assert_eq!(choice["disabled"], !satisfied, "{candidate}"); + assert_eq!(choice["reason"].is_null(), satisfied); + } + } + if !known { + assert_eq!(choice["disabled"], true); + } + } + assert_eq!(settings, original); + } + } + } + } + } + + #[test] + fn codec_descriptors_disable_only_codecs_that_cannot_decode() { + let caps = json!({"protocolVersion":7,"nativeHdrSupported":true, + "videoBackends":[{"backend":"d3d11","available":true, + "codecs":[{"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","8bit_444","10bit_420","10bit_444"], + "hdrColorQualities":["10bit_420","10bit_444"]}, + {"codec":"av1","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420"]}]}]}); + let mut without_av1 = caps.clone(); + without_av1["videoBackends"][0]["codecs"][2]["available"] = json!(false); + for (caps, color, disabled) in [ + (&caps, "8bit_420", vec![]), + (&caps, "8bit_444", vec![]), + (&caps, "10bit_420", vec![]), + (&caps, "10bit_444", vec![]), + (&without_av1, "8bit_420", vec!["av1"]), + (&without_av1, "10bit_444", vec!["av1"]), + ] { + let settings = json!({"codec":"auto","colorQuality":color,"enableHdr":true}); + let choices = StreamerService::codec_choices(&settings, caps); + let choices = choices.as_array().unwrap(); + assert_eq!(choices.len(), 4); + for choice in choices { + let mut candidate = settings.clone(); + candidate["codec"] = choice["value"].clone(); + candidate["colorQuality"] = json!("8bit_420"); + candidate["enableHdr"] = json!(false); + let result = StreamerService::embedded_session_settings(&candidate, caps); + assert_eq!(choice["disabled"], result.is_err(), "{color}"); + let value = choice["value"].as_str().unwrap(); + assert_eq!( + choice["disabled"], + json!(disabled.contains(&value)), + "{color}/{value}" + ); + } + assert_eq!(settings["codec"], "auto"); + } + } + + #[test] + fn embedded_auto_selects_only_supported_codecs_and_preserves_manual_choices() { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true}, {"codec":"h265","available":true}, + {"codec":"av1","available":false}]}]}); + let settings = json!({"codec":"auto","nativeVideoBackend":"auto","colorQuality":"8bit_420", + "resolution":"1920x1080","fps":60}); + let resolve = |settings: &Value, caps: &Value| { + StreamerService::embedded_session_settings(settings, caps) + }; + assert_eq!(resolve(&settings, &caps).unwrap()["codec"], "h265"); + assert_eq!(settings["codec"], "auto"); // never rewrite the user's persisted Auto preference + caps["videoBackends"][0]["codecs"][1]["available"] = json!(false); + assert_eq!(resolve(&settings, &caps).unwrap()["codec"], "h264"); + assert!(resolve(&json!({"codec":"av1"}), &caps).is_err()); + let ten_bit = resolve(&json!({"codec":"auto","colorQuality":"10bit_420"}), &caps).unwrap(); + assert_eq!(ten_bit["codec"], "h264"); + assert_eq!(ten_bit["colorQuality"], "8bit_420"); + for backend in ["d3d12", "vulkan", "software"] { + let error = resolve(&json!({"nativeVideoBackend":backend}), &caps).unwrap_err(); + assert_eq!(error.code, "streamer_backend_unavailable"); + } + assert_eq!( + resolve(&json!({"codec":"h264","nativeVideoBackend":"d3d11"}), &caps).unwrap()["codec"], + "h264" + ); + caps["videoBackends"][0]["codecs"][2]["available"] = json!(true); + assert_eq!(resolve(&settings, &caps).unwrap()["codec"], "av1"); + caps["videoBackends"][0]["available"] = json!(false); + assert!(resolve(&settings, &caps).is_err()); + caps["videoBackends"][0]["available"] = json!(true); + caps["videoBackends"][0]["backend"] = json!("ffmpeg"); + assert!(resolve(&settings, &caps).is_err()); + assert!(resolve(&settings, &json!({})).is_err()); + } + + #[test] + fn embedded_mediacodec_preflight_preserves_the_hardware_only_profile() { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"mediacodec","platform":"linux","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"],"hdrSupported":false}, + {"codec":"h265","available":false,"colorQualities":[],"hdrSupported":false}, + {"codec":"av1","available":false,"colorQualities":[],"hdrSupported":false} + ] + }]}); + let settings = + json!({"codec":"auto","colorQuality":"8bit_420","fps":60,"resolution":"1280x720"}); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + "h264" + ); + for unsupported in [ + json!({"codec":"h265"}), + json!({"codec":"av1"}), + json!({"codec":"h264","nativeVideoBackend":"software"}), + ] { + assert!(StreamerService::embedded_session_settings(&unsupported, &caps).is_err()); + } + for degraded in [ + json!({"codec":"h264","colorQuality":"10bit_420"}), + json!({"codec":"h264","colorQuality":"8bit_444"}), + json!({"codec":"h264","enableHdr":true}), + ] { + let resolved = StreamerService::embedded_session_settings(°raded, &caps).unwrap(); + assert_eq!(resolved["codec"], "h264", "{degraded}"); + assert_eq!(resolved["colorQuality"], "8bit_420", "{degraded}"); + assert_eq!(resolved["enableHdr"], false, "{degraded}"); + } + caps["videoBackends"][0]["available"] = json!(false); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps) + .unwrap_err() + .code, + "streamer_backend_unavailable" + ); + } + + #[test] + fn explicit_444_request_forces_hevc_when_hevc_444_is_decodable() { + for profiles in [ + None, + Some(json!(["8bit_420", "10bit_420", "8bit_444", "10bit_444"])), + ] { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true}, + {"codec":"h265","available":true}, + {"codec":"av1","available":true} + ] + }]}); + if let Some(profiles) = profiles { + for codec in caps["videoBackends"][0]["codecs"].as_array_mut().unwrap() { + codec["colorQualities"] = profiles.clone(); + } + } + for codec in ["auto", "h264", "h265", "av1"] { + for color in ["8bit_444", "10bit_444"] { + let settings = json!({"codec":codec,"colorQuality":color}); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &caps).unwrap(); + assert_eq!(resolved["codec"], "h265", "{settings}"); + assert_eq!(resolved["colorQuality"], color, "{settings}"); + assert_eq!(settings["codec"], codec); + assert_eq!( + fallbacks.len(), + usize::from(matches!(codec, "h264" | "av1")) + ); + } + } + let resolved = StreamerService::embedded_session_settings( + &json!({"codec":"h264","colorQuality":"10bit_420"}), + &caps, + ) + .unwrap(); + assert_eq!(resolved["codec"], "h264"); + assert_eq!(resolved["colorQuality"], "8bit_420"); + } + } + + #[test] + fn hdr_preserves_requested_chroma_only_when_hevc_hdr_444_is_decodable() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420","8bit_444","10bit_444"], + "hdrColorQualities":["10bit_420","10bit_444"]}, + {"codec":"av1","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + let assert_profile = |settings: &Value, caps: &Value, color: &str, hdr: bool| { + let resolved = StreamerService::embedded_session_settings(settings, caps).unwrap(); + assert_eq!( + resolved["codec"], + if settings["codec"] == "auto" && color == "10bit_420" && hdr { + "av1" + } else { + "h265" + }, + "{settings} {caps}" + ); + assert_eq!(resolved["colorQuality"], color, "{settings} {caps}"); + assert_eq!(resolved["enableHdr"], hdr, "{settings} {caps}"); + }; + for codec in ["auto", "h265", "hevc", "av1"] { + for color in ["8bit_444", "10bit_444"] { + let settings = json!({"codec":codec,"colorQuality":color,"enableHdr":true}); + assert_profile(&settings, &capabilities, "10bit_444", true); + assert_eq!(settings["colorQuality"], color); + for qualities in [ + Value::Null, + json!([]), + json!(["10bit_420"]), + json!("10bit_444"), + ] { + let mut sdr_only = capabilities.clone(); + sdr_only["videoBackends"][0]["codecs"][0]["hdrColorQualities"] = qualities; + assert_profile(&settings, &sdr_only, color, false); + } + let mut sdr_only = capabilities.clone(); + sdr_only["videoBackends"][0]["codecs"][0] + .as_object_mut() + .unwrap() + .remove("hdrColorQualities"); + assert_profile(&settings, &sdr_only, color, false); + sdr_only = capabilities.clone(); + sdr_only["videoBackends"][0]["codecs"][0]["hdrSupported"] = json!(false); + assert_profile(&settings, &sdr_only, color, false); + let mut no_444 = capabilities.clone(); + no_444["videoBackends"][0]["codecs"][0]["colorQualities"] = + json!(["8bit_420", "10bit_420"]); + if codec != "av1" { + assert_profile(&settings, &no_444, "10bit_420", true); + } + } + } + } + + #[test] + fn invalid_accepted_color_never_falls_back_to_local_preferences() { + let service = StreamerService::new(); + for source in ["request", "finalized", "server"] { + let params = json!({"session":{"sessionId":"seat","status":2, + "negotiatedStreamProfile":{"codec":"H265", "colorQuality":null, + "bitDepthSource":source, "chromaFormat":1}}}); + let error = service + .prepare_embedded( + ¶ms, + &json!({"codec":"h265", "colorQuality":"10bit_444"}), + ) + .unwrap_err(); + assert!(error.message.contains("color profile")); + } + } + + #[test] + fn hdr_444_attachment_preserves_accepted_profile_and_rechecks_capabilities() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["10bit_420","10bit_444"],"hdrColorQualities":["10bit_420","10bit_444"]} + ] + }]}); + let settings = json!({"codec":"h264","colorQuality":"8bit_420","enableHdr":false}); + let service = StreamerService::new(); + for status in [2, 3] { + let params = json!({"session":{"sessionId":"hdr-444","status":status, + "negotiatedStreamProfile":{"codec":"H265","colorQuality":"10bit_444","enableHdr":true}}, + "runtimeCapabilities":capabilities}); + let prepared = service.prepare_embedded(¶ms, &settings).unwrap(); + assert_eq!(prepared["context"]["settings"]["codec"], "H265"); + assert_eq!(prepared["context"]["settings"]["colorQuality"], "10bit_444"); + assert_eq!(prepared["context"]["settings"]["enableHdr"], true); + assert_eq!(prepared["context"]["settings"]["nativeHdrSupported"], true); + for color in ["8bit_420", "8bit_444", "10bit_422"] { + let mut unsupported = params.clone(); + unsupported["session"]["negotiatedStreamProfile"]["colorQuality"] = json!(color); + assert!(service.prepare_embedded(&unsupported, &settings).is_err()); + } + let mut unavailable = params.clone(); + unavailable["runtimeCapabilities"]["videoBackends"][0]["codecs"][0]["colorQualities"] = + json!(["10bit_420"]); + assert!(service.prepare_embedded(&unavailable, &settings).is_err()); + unavailable = params.clone(); + unavailable["runtimeCapabilities"]["videoBackends"][0]["codecs"][0]["hdrColorQualities"] = + json!(["10bit_420"]); + assert!(service.prepare_embedded(&unavailable, &settings).is_err()); + unavailable = params.clone(); + unavailable["runtimeCapabilities"]["nativeHdrSupported"] = json!(false); + assert!(service.prepare_embedded(&unavailable, &settings).is_err()); + unavailable["runtimeCapabilities"] = Value::Null; + assert!(service.prepare_embedded(&unavailable, &settings).is_err()); + } + } + + #[test] + fn explicit_hdr_profiles_gate_hdr_without_affecting_sdr() { + let mut capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420"],"hdrColorQualities":["10bit_420"]} + ] + }]}); + let hdr = json!({"codec":"h265","enableHdr":true}); + assert!(StreamerService::embedded_session_settings(&hdr, &capabilities).is_ok()); + for profiles in [ + Value::Null, + json!([]), + json!(["10bit_444"]), + json!("10bit_420"), + ] { + capabilities["videoBackends"][0]["codecs"][0]["hdrColorQualities"] = profiles; + let resolved = StreamerService::embedded_session_settings(&hdr, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "8bit_420"); + let sdr = json!({"codec":"h265","enableHdr":false,"colorQuality":"10bit_420"}); + assert!(StreamerService::embedded_session_settings(&sdr, &capabilities).is_ok()); + } + } + + #[test] + fn hdr_requires_explicit_output_and_ten_bit_hardware_support() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]}, + {"codec":"av1","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + let settings = json!({"codec":"auto","colorQuality":"8bit_420","enableHdr":true, + "resolution":"1920x1080","fps":60}); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "10bit_420"); + assert_eq!(resolved["nativeHdrSupported"], true); + assert_eq!(settings["colorQuality"], "8bit_420"); + let assert_sdr = |settings: &Value, caps: &Value, codec: &str| { + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(settings, caps).unwrap(); + assert_eq!(resolved["codec"], codec, "{settings}"); + assert_eq!(resolved["colorQuality"], "8bit_420", "{settings}"); + assert_eq!(resolved["enableHdr"], false, "{settings}"); + assert_eq!(resolved["fps"], 60, "{settings}"); + assert_eq!(fallbacks.len(), 1, "{fallbacks:?}"); + }; + for capability in [Value::Null, json!(false), json!("true")] { + let mut unavailable = capabilities.clone(); + unavailable["nativeHdrSupported"] = capability; + assert_sdr(&settings, &unavailable, "av1"); + } + let mut eight_bit = capabilities.clone(); + for codec in eight_bit["videoBackends"][0]["codecs"] + .as_array_mut() + .unwrap() + { + codec["colorQualities"] = json!(["8bit_420"]); + } + assert_sdr(&settings, &eight_bit, "av1"); + for profile in [Value::Null, json!([])] { + let mut unavailable = capabilities.clone(); + for codec in unavailable["videoBackends"][0]["codecs"] + .as_array_mut() + .unwrap() + { + codec["colorQualities"] = profile.clone(); + } + assert_eq!( + StreamerService::embedded_session_settings(&settings, &unavailable) + .unwrap_err() + .code, + "streamer_codec_unavailable" + ); + } + let mut h264 = settings.clone(); + h264["codec"] = json!("h264"); + assert_sdr(&h264, &capabilities, "h264"); + for (key, value) in [ + ("nativeVideoBackend", "software"), + ("decoderPreference", "software"), + ] { + let mut unsupported = settings.clone(); + unsupported[key] = json!(value); + assert_eq!( + StreamerService::embedded_session_settings(&unsupported, &capabilities) + .unwrap_err() + .code, + "streamer_backend_unavailable" + ); + } + let mut sdr = settings.clone(); + sdr["enableHdr"] = json!(false); + let resolved = StreamerService::embedded_session_settings(&sdr, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "av1"); + assert_eq!(resolved["colorQuality"], "8bit_420"); + } + + #[test] + fn windows_hdr_capability_gates_hdr_without_changing_unknown_sdr_profiles() { + let mut capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h265","available":true,"hdrSupported":true} + ] + }]}); + let hdr = json!({"codec":"auto","enableHdr":true}); + let resolved = StreamerService::embedded_session_settings(&hdr, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "10bit_420"); + for supported in [json!(false), Value::Null, json!("true")] { + capabilities["videoBackends"][0]["codecs"][0]["hdrSupported"] = supported; + let resolved = StreamerService::embedded_session_settings(&hdr, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "8bit_420"); + let sdr = json!({"codec":"h265","colorQuality":"10bit_444","enableHdr":false}); + let resolved = StreamerService::embedded_session_settings(&sdr, &capabilities).unwrap(); + assert_eq!(resolved["colorQuality"], "10bit_444"); + } + capabilities["videoBackends"][0]["codecs"][0]["hdrSupported"] = json!(false); + capabilities["videoBackends"][0]["codecs"][0]["colorQualities"] = json!(["10bit_420"]); + assert_eq!( + StreamerService::embedded_session_settings(&hdr, &capabilities) + .unwrap_err() + .code, + "streamer_codec_unavailable" + ); + capabilities["videoBackends"][0]["codecs"][0]["hdrSupported"] = json!(true); + capabilities["videoBackends"][0]["codecs"][0]["colorQualities"] = json!([]); + assert_eq!( + StreamerService::embedded_session_settings(&hdr, &capabilities) + .unwrap_err() + .code, + "streamer_codec_unavailable" + ); + } + + #[test] + fn hdr_resume_uses_accepted_profile_and_rechecks_current_output() { + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + let mut params = json!({"session":{"sessionId":"hdr-resume","status":2, + "negotiatedStreamProfile":{"codec":"H265","colorQuality":"10bit_420","enableHdr":true}}, + "runtimeCapabilities":capabilities}); + let service = StreamerService::new(); + let settings = json!({"codec":"h264","colorQuality":"8bit_420","enableHdr":false}); + let prepared = service.prepare_embedded(¶ms, &settings).unwrap(); + assert_eq!(prepared["context"]["settings"]["enableHdr"], true); + assert_eq!(prepared["context"]["settings"]["colorQuality"], "10bit_420"); + assert_eq!(prepared["context"]["settings"]["nativeHdrSupported"], true); + params["runtimeCapabilities"]["nativeHdrSupported"] = json!(false); + assert!(service.prepare_embedded(¶ms, &settings).is_err()); + params["runtimeCapabilities"] = Value::Null; + assert!(service.prepare_embedded(¶ms, &settings).is_err()); + params["session"]["negotiatedStreamProfile"]["enableHdr"] = json!(false); + let stale_hdr = json!({"enableHdr":true,"nativeHdrSupported":true}); + let prepared = service.prepare_embedded(¶ms, &stale_hdr).unwrap(); + assert_eq!(prepared["context"]["settings"]["enableHdr"], false); + assert_eq!(prepared["context"]["settings"]["nativeHdrSupported"], false); + } + + #[test] + fn embedded_macos_auto_prefers_av1_then_hevc_independently_of_bitrate() { + let caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]}, + {"codec":"av1","available":true,"colorQualities":["8bit_420","10bit_420"]}]}]}); + for (resolution, fps, bitrate, color, expected) in [ + ("1920x1080", 60, 75, "8bit_420", "av1"), + ("1920x1080", 60, 50, "8bit_420", "av1"), + ("2560x1440", 60, 80, "8bit_420", "av1"), + ("1920x1080", 60, 30, "8bit_420", "av1"), + ("1920x1080", 60, 31, "8bit_420", "av1"), + ("3840x2160", 120, 80, "8bit_420", "av1"), + ("3840x2160", 144, 20, "8bit_420", "h265"), + ("1920x1080", 240, 75, "8bit_420", "h265"), + ("1920x1080", 60, 20, "10bit_420", "av1"), + ("1920x1080", 144, 75, "10bit_420", "h265"), + ] { + let settings = json!({"codec":"auto", "resolution":resolution, + "fps":fps, "maxBitrateMbps":bitrate, "colorQuality":color, + "transportMode":"nvst"}); + let resolved = StreamerService::embedded_session_settings(&settings, &caps).unwrap(); + assert_eq!(resolved["codec"], expected, "{settings}"); + assert_eq!(resolved["colorQuality"], color); + assert_eq!(resolved["transportMode"], "nvst"); + assert_eq!(settings["codec"], "auto"); + } + assert_eq!( + StreamerService::embedded_session_settings(&json!({"codec":"auto"}), &caps).unwrap()["codec"], + "av1" + ); + } + + #[test] + fn embedded_auto_av1_requires_supported_fps_and_resolution() { + for (platform, resolution, fps, expected) in [ + ("windows", "1920x1080", Some(60), "av1"), + ("windows", "5120x2880", Some(120), "av1"), + ("windows", "5120x2880", Some(121), "av1"), + ("windows", "5120x2881", Some(120), "h265"), + ("windows", "7680x4320", Some(60), "h265"), + ("windows", "1920x1080", None, "av1"), + ("windows", "1920x1080", Some(0), "av1"), + ("windows", "invalid", Some(60), "h265"), + ("windows", "0x1080", Some(60), "h265"), + ("windows", "18446744073709551615x2", Some(60), "h265"), + ("macos", "1920x1080", Some(60), "av1"), + ("linux", "1920x1080", Some(60), "av1"), + ("steamos", "1920x1080", Some(60), "av1"), + ("", "1920x1080", Some(60), "av1"), + ("unknown", "1920x1080", Some(60), "av1"), + ] { + let backend = match platform { + "macos" => "videotoolbox", + "linux" | "steamos" => "vulkan", + _ => "d3d11", + }; + let caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION, + "videoBackends":[{"backend":backend,"platform":platform,"available":true, + "codecs":[{"codec":"h264","available":true}, + {"codec":"h265","available":true}, + {"codec":"av1","available":true}]}]}); + let settings = json!({"codec":"auto","colorQuality":"8bit_420", + "resolution":resolution,"fps":fps,"maxBitrateMbps":20}); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + expected, + "{platform} {settings}" + ); + let mut without_resolution = settings.clone(); + without_resolution + .as_object_mut() + .unwrap() + .remove("resolution"); + assert_eq!( + StreamerService::embedded_session_settings(&without_resolution, &caps).unwrap()["codec"], + "av1", + "{platform} missing resolution" + ); + } + } + + #[test] + fn embedded_auto_av1_hdr_needs_probed_support_without_restricting_explicit_choice() { + let mut caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION, + "nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h265","available":false,"hdrSupported":true, + "colorQualities":["10bit_420"],"hdrColorQualities":["10bit_420"]}, + {"codec":"av1","available":true,"hdrSupported":true, + "colorQualities":["10bit_420"],"hdrColorQualities":["10bit_420"]}]}]}); + let settings = json!({"codec":"auto","enableHdr":true,"resolution":"3840x2160","fps":120}); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + "av1" + ); + for (platform, fps, hdr_supported) in [("windows", 144, Some(true)), ("windows", 120, None)] + { + caps["videoBackends"][0]["platform"] = json!(platform); + caps["videoBackends"][0]["codecs"][1]["hdrSupported"] = json!(hdr_supported); + let mut candidate = settings.clone(); + candidate["fps"] = json!(fps); + assert_eq!( + StreamerService::embedded_session_settings(&candidate, &caps) + .unwrap_err() + .code, + "streamer_codec_unavailable", + "{platform} {candidate}" + ); + } + caps["videoBackends"][0]["platform"] = json!("linux"); + caps["videoBackends"][0]["codecs"][1]["hdrSupported"] = json!(true); + let explicit = json!({"codec":"av1","enableHdr":true,"resolution":"3840x2160","fps":144}); + assert_eq!( + StreamerService::embedded_session_settings(&explicit, &caps).unwrap()["codec"], + "av1" + ); + } + + #[test] + fn unavailable_embedded_backends_distinguish_auto_from_explicit_selection() { + let capabilities = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"v4l2", "available":false, "reason":"HEVC topology probe failed: errno 13 (Permission denied) token=private-value"}, + {"backend":"cuda", "available":false, "reason":"CUDA unavailable"}, + {"backend":"software", "available":true, "reason":"unused software reason"} + ]}); + for selection in ["auto", "v4l2", "nvdec"] { + let error = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":selection}), + &capabilities, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_backend_unavailable"); + assert_eq!(error.message.contains("Select Auto"), selection != "auto"); + assert_eq!( + error + .message + .contains("HEVC topology probe failed: errno 13 (Permission denied)"), + selection == "v4l2" + ); + assert_eq!( + error.message.contains("CUDA unavailable"), + selection == "nvdec" + ); + assert!(!error.message.contains("private-value")); + assert!(!error.message.contains("unused software reason")); + } + let error = + StreamerService::embedded_session_settings(&json!({}), &capabilities).unwrap_err(); + assert!(error.message.starts_with("No hardware video backend")); + let mut oversized = capabilities.clone(); + oversized["videoBackends"][0]["reason"] = json!("failure ".repeat(10000)); + let concise = + StreamerService::embedded_session_settings(&json!({}), &oversized).unwrap_err(); + assert_eq!(concise.message, error.message); + assert!(concise.message.len() < 200); + let error = StreamerService::embedded_session_settings( + &json!({}), + &json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[]}), + ) + .unwrap_err(); + assert!(!error.message.contains("Select Auto")); + } + + #[test] + fn hybrid_gpu_failure_names_the_adapter_that_can_decode() { + let capabilities = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"d3d11","available":false,"reason":"Direct3D hardware decode or presentation is unavailable"} + ], "graphicsAdapters":[ + {"name":"NVIDIA GeForce MX110","active":true,"codecs":[],"h265Main10":false, + "reason":"no supported hardware decoder profile"}, + {"name":"Intel(R) HD Graphics 620","active":false,"codecs":["h264","h265","private-codec"], + "h265Main10":true}, + {"name":"Bearer secret-token","active":false,"codecs":["h264"]} + ]}); + let error = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"auto"}), + &capabilities, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_backend_unavailable"); + assert!( + error + .message + .starts_with("No hardware video backend is available on NVIDIA GeForce MX110.") + ); + assert!( + error + .message + .contains("Intel(R) HD Graphics 620 (H.264, H.265)") + ); + assert!(error.message.contains("Graphics processor")); + assert!(!error.message.contains("private-codec")); + assert!(!error.message.contains("secret-token")); + assert!(!error.message.contains("Select Software")); + assert!(error.message.len() <= 480); + + let explicit = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"d3d11"}), + &capabilities, + ) + .unwrap_err(); + assert!(explicit.message.contains("Select Auto")); + assert!( + explicit + .message + .contains("Intel(R) HD Graphics 620 (H.264, H.265)") + ); + assert!(!explicit.message.contains("secret-token")); + + let only_failed = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"d3d11","available":false} + ], "graphicsAdapters":[ + {"name":"NVIDIA GeForce MX110","active":true,"codecs":[]} + ]}); + let unchanged = + StreamerService::embedded_session_settings(&json!({}), &only_failed).unwrap_err(); + assert!(unchanged.message.contains("Check the hardware drivers")); + assert!(!unchanged.message.contains("Graphics processor")); + } + + #[test] + fn embedded_software_resolution_honors_the_explicit_cpu_policy() { + let software_caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"vulkan", "platform":"linux", "available":true, "codecs":[ + {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}, + {"codec":"h265", "available":true, "colorQualities":["8bit_420"]}]}, + {"backend":"ffmpeg", "platform":"linux", "available":true, "codecs":[ + {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}]} + ]}); + for settings in [ + json!({"nativeVideoBackend":"software", "codec":"h264"}), + json!({"nativeVideoBackend":"ffmpeg", "codec":"h264"}), + json!({"decoderPreference":"software", "codec":"h264"}), + ] { + let resolved = + StreamerService::embedded_session_settings(&settings, &software_caps).unwrap(); + assert_eq!(resolved["codec"], "h264", "{settings}"); + } + let error = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"software", "codec":"h265"}), + &software_caps, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_codec_unavailable"); + assert!( + !error.message.contains("hardware"), + "software requests must not borrow hardware codecs: {}", + error.message + ); + let mut hdr_capable = software_caps.clone(); + hdr_capable["nativeHdrSupported"] = json!(true); + for color in ["10bit_420", "8bit_444", "10bit_444"] { + for hdr in [false, true] { + let settings = json!({"nativeVideoBackend":"software", "colorQuality":color, + "enableHdr":hdr}); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &hdr_capable).unwrap(); + assert_eq!(resolved["codec"], "h264", "{settings}"); + assert_eq!(resolved["colorQuality"], "8bit_420", "{settings}"); + assert_eq!(resolved["enableHdr"], false, "{settings}"); + assert_eq!(fallbacks.len(), 1, "{fallbacks:?}"); + assert!(fallbacks[0].contains("software decoding presents 8-bit 4:2:0 SDR only")); + } + } + let hardware_only = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"vulkan", "platform":"linux", "available":true, "codecs":[ + {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}]} + ]}); + let error = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"software"}), + &hardware_only, + ) + .unwrap_err(); + assert!( + error + .message + .contains("FFmpeg software decoder is unavailable"), + "{}", + error.message + ); + let resolved = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"auto", "codec":"auto"}), + &software_caps, + ) + .unwrap(); + assert_eq!(resolved["codec"], "h265"); + } + + #[test] + fn embedded_software_av1_unavailable_falls_back_only_for_auto() { + let mut capabilities = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[ + {"backend":"ffmpeg", "platform":"linux", "available":true, "codecs":[ + {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}, + {"codec":"h265", "available":true, "colorQualities":["8bit_420"]}, + {"codec":"av1", "available":false, "colorQualities":["8bit_420"]}]} + ]}); + let settings = json!({"nativeVideoBackend":"software", "codec":"auto"}); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(settings["codec"], "auto"); + capabilities["videoBackends"][0]["codecs"][1]["available"] = json!(false); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h264"); + let error = StreamerService::embedded_session_settings( + &json!({"nativeVideoBackend":"software", "codec":"av1"}), + &capabilities, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_codec_unavailable"); + } + + #[test] + fn embedded_macos_codec_policy_never_falls_back_to_unsupported_hardware() { + let mut caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]}, + {"codec":"av1","available":false}]}, + {"backend":"software","available":true,"codecs":[ + {"codec":"av1","available":true,"colorQualities":["8bit_420","10bit_420"]}]}]}); + let settings = json!({"codec":"auto", "maxBitrateMbps":20}); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + "h265" + ); + assert_eq!( + StreamerService::embedded_session_settings( + &json!({"codec":"auto", "maxBitrateMbps":20, "resolution":"3840x2160", "fps":60}), + &caps + ) + .unwrap()["codec"], + "h265" + ); + assert_eq!( + StreamerService::embedded_session_settings(&json!({"codec":"av1"}), &caps) + .unwrap_err() + .code, + "streamer_codec_unavailable" + ); + assert_eq!( + StreamerService::embedded_session_settings(&json!({"codec":"h264"}), &caps).unwrap()["codec"], + "h264" + ); + caps["videoBackends"][0]["codecs"][1]["available"] = json!(false); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + "h264" + ); + for color in ["10bit_420", "8bit_444", "10bit_444"] { + for codec in ["auto", "h264"] { + let resolved = StreamerService::embedded_session_settings( + &json!({"codec":codec,"colorQuality":color}), + &caps, + ) + .unwrap(); + assert_eq!(resolved["codec"], "h264"); + assert_eq!(resolved["colorQuality"], "8bit_420"); + } + let error = StreamerService::embedded_session_settings( + &json!({"codec":"h265","colorQuality":color}), + &caps, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_codec_unavailable"); + assert!(error.message.contains("H265 is unavailable")); + } + caps["videoBackends"][0]["available"] = json!(false); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps) + .unwrap_err() + .code, + "streamer_backend_unavailable" + ); + } + + #[test] + fn embedded_macos_444_without_probed_hevc_444_falls_back_to_420_keeping_depth() { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true, + "codecs":[{"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420","10bit_444"]}] + }]}); + for codec in ["auto", "h265"] { + let settings = json!({"codec":codec,"colorQuality":"10bit_444"}); + let resolved = StreamerService::embedded_session_settings(&settings, &caps).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "10bit_444"); + } + let assert_fallback = |settings: Value, caps: &Value, color: &str| { + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, caps).unwrap(); + assert_eq!(resolved["codec"], "h265", "{settings}"); + assert_eq!(resolved["colorQuality"], color, "{settings}"); + assert_eq!(fallbacks.len(), 1, "{fallbacks:?}"); + assert!( + fallbacks[0].contains("cannot decode H.265"), + "{fallbacks:?}" + ); + }; + assert_fallback( + json!({"codec":"auto","colorQuality":"8bit_444"}), + &caps, + "8bit_420", + ); + caps["videoBackends"][0]["codecs"][0]["colorQualities"] = json!(["8bit_420", "10bit_420"]); + for codec in ["auto", "h265"] { + assert_fallback( + json!({"codec":codec,"colorQuality":"10bit_444"}), + &caps, + "10bit_420", + ); + } + } + + #[test] + fn auto_hdr_without_hdr_decode_streams_sdr_and_caps_h264_at_120_fps() { + let mut caps = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":false, + "colorQualities":["8bit_420","10bit_420"]}, + {"codec":"av1","available":false} + ] + }]}); + let settings = json!({"codec":"auto","enableHdr":true,"colorQuality":"10bit_420", + "resolution":"1920x1080","fps":240}); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &caps).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "10bit_420"); + assert_eq!(resolved["fps"], 240); + assert_eq!(fallbacks.len(), 1, "{fallbacks:?}"); + assert!(fallbacks[0].contains("Ignoring the HDR request")); + + caps["videoBackends"][0]["codecs"][1]["available"] = json!(false); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &caps).unwrap(); + assert_eq!(resolved["codec"], "h264"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "8bit_420"); + assert_eq!(resolved["fps"], 120); + assert_eq!(settings["fps"], 240); + assert_eq!(fallbacks.len(), 3, "{fallbacks:?}"); + assert!(fallbacks[2].contains("Capping the frame rate at 120 FPS")); + + let explicit = json!({"codec":"h264","enableHdr":true,"fps":240}); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&explicit, &caps).unwrap(); + assert_eq!(resolved["codec"], "h264"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(resolved["colorQuality"], "8bit_420"); + assert_eq!(resolved["fps"], 120); + assert_eq!(fallbacks.len(), 2); + assert!(fallbacks[0].contains("Ignoring the HDR request")); + assert!(fallbacks[1].contains("Capping the frame rate at 120 FPS")); + + let sdr = json!({"codec":"auto","enableHdr":false,"fps":240}); + assert_eq!( + StreamerService::embedded_session_settings(&sdr, &caps).unwrap()["fps"], + 120 + ); + } + + #[test] + fn embedded_resolution_fallbacks_match_the_qt_supported_mode_catalog() { + let qml = include_str!("../../../opennow-qt/qml/state/settings/SettingsState.qml"); + let modes = qml + .split_once("function resolutionItems()") + .unwrap() + .1 + .split_once("const items = []") + .unwrap() + .0; + let expression = regex::Regex::new(r#""([0-9]+)x([0-9]+)""#).unwrap(); + let choices = expression + .captures_iter(modes) + .map(|capture| { + ( + capture[1].parse::().unwrap(), + capture[2].parse::().unwrap(), + ) + }) + .collect::>(); + assert_eq!( + EMBEDDED_RESOLUTIONS + .into_iter() + .collect::>(), + choices + ); + } + + #[test] + fn embedded_codec_caps_and_decode_dimensions_are_session_local() { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"av1","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420"], + "maxDecodeDimensions":{"width":4096,"height":2304}} + ] + }]}); + for (codec, resolution, fps, expected_resolution, expected_fps) in [ + ("h264", "3440x1440", 120, "2560x1080", 120), + ("h264", "1920x1080", 240, "1920x1080", 120), + ("h264", "3840x2160", 360, "2560x1440", 120), + ("h264", "5120x1440", 165, "3840x1080", 120), + ("h264", "2560x1600", 90, "2560x1600", 90), + ("av1", "1920x1080", 240, "1920x1080", 120), + ("av1", "1920x1080", 360, "1920x1080", 120), + ("h265", "5120x2880", 165, "3840x2160", 165), + ("h265", "5120x2160", 165, "2560x1080", 165), + ("h265", "5120x1440", 165, "3840x1080", 165), + ("h265", "3840x2400", 165, "2560x1600", 165), + ("h265", "4096x2304", 165, "4096x2304", 165), + ] { + let settings = json!({"codec":codec,"resolution":resolution,"fps":fps}); + let saved = settings.clone(); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &caps).unwrap(); + assert_eq!(resolved["resolution"], expected_resolution, "{settings}"); + assert_eq!(resolved["fps"], expected_fps, "{settings}"); + assert_eq!(settings, saved); + assert_eq!( + fallbacks.is_empty(), + resolution == expected_resolution && fps == expected_fps + ); + } + caps["videoBackends"][0]["codecs"][1]["available"] = json!(false); + caps["videoBackends"][0]["codecs"][2]["available"] = json!(false); + let (resolved, fallbacks) = StreamerService::resolve_embedded_session( + &json!({"codec":"auto","resolution":"3440x1440","fps":240}), + &caps, + ) + .unwrap(); + assert_eq!(resolved["codec"], "h264"); + assert_eq!(resolved["resolution"], "2560x1080"); + assert_eq!(resolved["fps"], 120); + assert_eq!(fallbacks.len(), 2); + } + + #[test] + fn embedded_dimension_preflight_rejects_invalid_limits_and_oversized_resumes() { + let mut caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[{"codec":"h265","available":true, + "maxDecodeDimensions":{"width":4096,"height":2304}}] + }]}); + let error = StreamerService::new().prepare_embedded( + &json!({"runtimeCapabilities":caps,"session":{"status":2,"sessionId":"resize", + "negotiatedStreamProfile":{"codec":"H265","resolution":"5120x2880","fps":60,"colorQuality":"8bit_420","enableHdr":false}}}), + &json!({"codec":"h265","resolution":"1920x1080","fps":60}) + ).unwrap_err(); + assert_eq!(error.code, "streamer_mode_unavailable"); + for limit in [ + json!({"width":0,"height":2304}), + json!({"width":4096}), + json!(null), + ] { + caps["videoBackends"][0]["codecs"][0]["maxDecodeDimensions"] = limit; + assert!( + StreamerService::embedded_session_settings(&json!({"codec":"h265"}), &caps) + .is_err() + ); + } + caps["videoBackends"][0]["codecs"][0]["maxDecodeDimensions"] = + json!({"width":640,"height":480}); + assert_eq!( + StreamerService::embedded_session_settings( + &json!({"codec":"h265","resolution":"5120x2880"}), + &caps + ) + .unwrap_err() + .code, + "streamer_mode_unavailable" + ); + } + + #[test] + fn embedded_auto_uses_the_same_priority_for_sdr_and_hdr() { + for platform in ["windows", "macos", "linux"] { + let mut caps = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"hardware","platform":platform,"available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":true,"colorQualities":["8bit_420","10bit_420"],"hdrColorQualities":["10bit_420"]}, + {"codec":"av1","available":true,"hdrSupported":true,"colorQualities":["8bit_420","10bit_420"],"hdrColorQualities":["10bit_420"]} + ] + }]}); + for hdr in [false, true] { + let settings = + json!({"codec":"auto","enableHdr":hdr,"fps":120,"resolution":"1920x1080"}); + assert_eq!( + StreamerService::embedded_session_settings(&settings, &caps).unwrap()["codec"], + "av1" + ); + } + caps["videoBackends"][0]["codecs"][2]["available"] = json!(false); + assert_eq!( + StreamerService::embedded_session_settings( + &json!({"codec":"auto","enableHdr":true,"fps":120}), + &caps + ) + .unwrap()["codec"], + "h265" + ); + } + } + + #[test] + fn explicit_undecodable_codec_errors_unless_444_forces_hevc() { + let caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","platform":"windows","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420","10bit_444"]}, + {"codec":"av1","available":false,"reason":"AV1 decode is unavailable"} + ] + }]}); + for (color, hdr) in [ + ("8bit_420", false), + ("10bit_420", false), + ("8bit_444", false), + ("8bit_420", true), + ] { + let error = StreamerService::embedded_session_settings( + &json!({"codec":"av1","colorQuality":color,"enableHdr":hdr}), + &caps, + ) + .unwrap_err(); + assert_eq!(error.code, "streamer_codec_unavailable", "{color}"); + assert!( + error + .message + .contains("AV1 is unavailable: AV1 decode is unavailable") + ); + } + let resolved = StreamerService::embedded_session_settings( + &json!({"codec":"av1","colorQuality":"10bit_444"}), + &caps, + ) + .unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "10bit_444"); + let choices = StreamerService::codec_choices(&json!({"colorQuality":"10bit_444"}), &caps); + let disabled = choices + .as_array() + .unwrap() + .iter() + .filter(|choice| choice["disabled"] == true) + .map(|choice| choice["value"].as_str().unwrap()) + .collect::>(); + assert_eq!(disabled, ["av1"]); + } + + #[test] + fn color_choices_disable_profiles_that_fall_back_and_name_the_delivered_profile() { + let caps = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ] + }]}); + for (codec, enabled) in [ + ("auto", vec!["8bit_420", "10bit_420"]), + ("h264", vec!["8bit_420"]), + ] { + let choices = StreamerService::color_quality_choices(&json!({"codec":codec}), &caps); + for choice in choices.as_array().unwrap() { + let value = choice["value"].as_str().unwrap(); + assert_eq!( + choice["disabled"], + !enabled.contains(&value), + "{codec}/{value}" + ); + if choice["disabled"] == true { + assert!(choice["reason"].as_str().unwrap().contains("falls back to")); + } + } + } + } + + #[test] + fn prepare_adopts_the_resolved_color_when_the_session_omits_one() { + let service = StreamerService::new(); + let params = json!({"session":{"sessionId":"seat","status":2, + "negotiatedStreamProfile":{"codec":null,"codecSource":"unreported"}}, + "runtimeCapabilities":{"protocolVersion":7,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]} + ]}]}}); + let prepared = service + .prepare_embedded(¶ms, &json!({"codec":"auto","colorQuality":"10bit_444"})) + .unwrap(); + assert_eq!(prepared["context"]["settings"]["codec"], "H265"); + assert_eq!(prepared["context"]["settings"]["colorQuality"], "10bit_420"); + } + + #[test] + fn embedded_macos_prepare_checks_negotiated_color_before_attachment() { + let service = StreamerService::new(); + for color in ["8bit_444", "10bit_444"] { + let result = service.prepare_embedded( + &json!({"session": { + "sessionId":"resume", "status":2, + "negotiatedStreamProfile":{"codec":"H265", "colorQuality":color} + }, "runtimeCapabilities":{"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true, + "codecs":[{"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]}]}]}}), + &json!({"codec":"auto", "colorQuality":"8bit_420"}), + ); + assert_eq!(result.unwrap_err().code, "streamer_color_unavailable"); + } + let prepared = service.prepare_embedded( + &json!({"session": { + "sessionId":"resume-ten-bit", "status":2, + "negotiatedStreamProfile":{"codec":"H265", "colorQuality":"10bit_420"} + }, "runtimeCapabilities":{"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[{ + "backend":"videotoolbox","platform":"macos","available":true, + "codecs":[{"codec":"h265","available":true,"colorQualities":["8bit_420","10bit_420"]}]}]}}), + &json!({"codec":"auto", "colorQuality":"8bit_420"}), + ).unwrap(); + assert_eq!(prepared["context"]["settings"]["colorQuality"], "10bit_420"); + } + + #[test] + fn embedded_prepare_resolves_auto_for_an_unreported_codec() { + let service = StreamerService::new(); + let capabilities = json!({"protocolVersion":7,"nativeHdrSupported":true,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"hdrSupported":true, + "colorQualities":["8bit_420","10bit_420","10bit_444"], + "hdrColorQualities":["10bit_420","10bit_444"]}, + {"codec":"av1","available":false} + ] + }]}); + for color in ["10bit_420", "10bit_444", "8bit_420"] { + for hdr in [false, true] { + if hdr && color == "8bit_420" { + continue; + } + for codec in [json!("auto"), Value::Null] { + let params = json!({"session":{"sessionId":"auto-attach","status":2, + "negotiatedStreamProfile":{"codec":null,"codecSource":"unreported", + "colorQuality":color,"enableHdr":hdr}}, + "runtimeCapabilities":capabilities}); + let settings = + json!({"codec":codec,"colorQuality":"8bit_420","enableHdr":false}); + let prepared = service.prepare_embedded(¶ms, &settings).unwrap(); + assert_eq!(prepared["context"]["settings"]["codec"], "H265"); + assert_eq!(prepared["context"]["settings"]["colorQuality"], color); + assert_eq!(prepared["context"]["settings"]["enableHdr"], hdr); + assert_eq!(prepared["context"]["settings"]["nativeHdrSupported"], true); + assert_eq!(prepared["context"]["session"], params["session"]); + assert_eq!(settings["codec"], codec); + } + } + } + } + + #[test] + fn embedded_prepare_does_not_replace_an_explicit_codec() { + let service = StreamerService::new(); + let params = json!({"session":{"sessionId":"explicit-attach","status":2, + "negotiatedStreamProfile":{"codec":null,"codecSource":"unreported", + "colorQuality":"10bit_420","enableHdr":false}}, + "runtimeCapabilities":{"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["10bit_420"]} + ] + }]}}); + let error = service + .prepare_embedded(¶ms, &json!({"codec":"h264"})) + .unwrap_err(); + assert_eq!(error.code, "streamer_color_unavailable"); + assert!(error.message.contains("accepted 10bit_420 stream profile")); + } + + #[test] + fn embedded_prepare_rejects_an_incompatible_resumed_session() { + let service = StreamerService::new(); + let result = service.prepare_embedded( + &json!({"session": { + "sessionId":"resume", "status":2, "negotiatedStreamProfile":{"codec":"AV1"} + }, "runtimeCapabilities":{"protocolVersion":7,"videoBackends":[{ + "backend":"d3d11","available":true,"codecs":[{"codec":"h264","available":true}]}]}}), + &json!({"codec":"auto"}), + ); + assert_eq!(result.unwrap_err().code, "streamer_codec_unavailable"); + } + + #[test] + fn embedded_color_profiles_gate_auto_and_manual_before_allocation() { + let mut capabilities = json!({"protocolVersion":7,"videoBackends":[{ + "backend":"vulkan", "platform":"linux", "available":true, "codecs":[ + {"codec":"h264","available":true,"colorQualities":["8bit_420"]}, + {"codec":"h265","available":true,"colorQualities":["8bit_420"]}, + {"codec":"av1","available":true,"colorQualities":["8bit_420"]} + ] + }]}); + for (codec, expected) in [("auto", "av1"), ("h265", "h265"), ("av1", "av1")] { + let settings = + json!({"codec":codec,"nativeVideoBackend":"auto","colorQuality":"10bit_420"}); + let (resolved, fallbacks) = + StreamerService::resolve_embedded_session(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], expected); + assert_eq!(resolved["colorQuality"], "8bit_420"); + assert_eq!(resolved["enableHdr"], false); + assert_eq!(fallbacks.len(), 1); + assert!(fallbacks[0].contains("Falling back to 8-bit 4:2:0 BT.709")); + } + capabilities["videoBackends"][0]["codecs"][1]["colorQualities"] = + json!(["8bit_420", "10bit_420"]); + let settings = + json!({"codec":"auto","nativeVideoBackend":"auto","colorQuality":"10bit_420"}); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "10bit_420"); + assert_eq!(settings["codec"], "auto"); + for (color, expected) in [("8bit_444", "8bit_420"), ("10bit_444", "10bit_420")] { + for codec in ["auto", "h265"] { + let settings = json!({"codec":codec,"colorQuality":color}); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!( + resolved["codec"], + if codec == "auto" && color == "8bit_444" { + "av1" + } else { + "h265" + } + ); + assert_eq!(resolved["colorQuality"], expected); + } + } + let settings = + json!({"codec":"h265","nativeVideoBackend":"vulkan","colorQuality":"10bit_420"}); + assert!(StreamerService::embedded_session_settings(&settings, &capabilities).is_ok()); + capabilities["videoBackends"][0]["codecs"][1]["available"] = json!(false); + assert!(StreamerService::embedded_session_settings(&settings, &capabilities).is_err()); + } + + #[test] + fn embedded_linux_advanced_color_requires_explicit_well_formed_profiles() { + for backend in ["vulkan", "cuda"] { + for profiles in [ + None, + Some(Value::Null), + Some(json!([])), + Some(json!("10bit_420")), + ] { + let mut capabilities = json!({"protocolVersion":7,"videoBackends":[{ + "backend":backend,"platform":"linux","available":true, + "codecs":[{"codec":"h265","available":true}] + }]}); + let reported = profiles.is_some(); + if let Some(profiles) = profiles { + capabilities["videoBackends"][0]["codecs"][0]["colorQualities"] = profiles; + } + for color in ["10bit_420", "8bit_444", "10bit_444"] { + let settings = json!({"codec":"h265","colorQuality":color}); + let result = + StreamerService::embedded_session_settings(&settings, &capabilities); + if !reported { + assert_eq!(result.unwrap()["colorQuality"], "8bit_420"); + } else { + assert_eq!(result.unwrap_err().code, "streamer_codec_unavailable"); + } + } + } + } + } + + #[test] + fn embedded_color_profiles_cannot_be_borrowed_from_another_backend() { + let capabilities = json!({"protocolVersion":7,"videoBackends":[ + {"backend":"vulkan","platform":"linux","available":true, + "codecs":[{"codec":"h265","available":true,"colorQualities":["8bit_420"]}]}, + {"backend":"other","available":true, + "codecs":[{"codec":"h265","available":true,"colorQualities":["10bit_420"]}]} + ]}); + let settings = + json!({"codec":"h265","nativeVideoBackend":"vulkan","colorQuality":"10bit_420"}); + let resolved = + StreamerService::embedded_session_settings(&settings, &capabilities).unwrap(); + assert_eq!(resolved["codec"], "h265"); + assert_eq!(resolved["colorQuality"], "8bit_420"); + } + + #[test] + fn codec_capabilities_require_an_available_backend_and_codec() { + let capabilities = json!({ + "protocolVersion":7, + "videoBackends":[ + {"backend":"hardware","available":true,"codecs":[ + {"codec":"h264","available":true}, + {"codec":"h265","available":false,"reason":"not installed"} + ]}, + {"backend":"software","available":false,"codecs":[ + {"codec":"av1","available":true} + ]} + ] + }); + assert_eq!(available_codecs(&capabilities), vec!["h264"]); + assert!(ensure_codec_available(&capabilities, "h264").is_ok()); + let hevc = ensure_codec_available(&capabilities, "h265").expect_err("HEVC must fail"); + assert_eq!(hevc.code, "streamer_codec_unavailable"); + assert!(hevc.message.contains("not installed")); + assert!(ensure_codec_available(&capabilities, "av1").is_err()); + } + + #[cfg(unix)] + #[test] + fn detect_performs_a_protocol_handshake_and_reports_real_codecs() { + use std::os::unix::fs::PermissionsExt as _; + + let fixture = std::env::temp_dir().join(format!( + "opennow-capability-streamer-{}-{}", + std::process::id(), + random_u64() + )); + fs::write( + &fixture, + "#!/bin/sh\nread -r _hello\nprintf '%s\\n' '{\"id\":\"hello\",\"type\":\"ready\",\"processId\":1,\"capabilities\":{\"protocolVersion\":7,\"supportsOwnedNvstNegotiation\":true,\"videoBackends\":[{\"backend\":\"software\",\"available\":true,\"codecs\":[{\"codec\":\"h264\",\"available\":true},{\"codec\":\"av1\",\"available\":false,\"reason\":\"not built\"}]}]}}'\nread -r _shutdown\nprintf '%s\\n' '{\"id\":\"shutdown\",\"type\":\"ok\"}'\n", + ) + .expect("write capability fixture"); + fs::set_permissions(&fixture, fs::Permissions::from_mode(0o700)) + .expect("make capability fixture executable"); + + let service = StreamerService::new(); + let detected = service + .detect(&json!({"nativeStreamerExecutablePath":fixture})) + .expect("capability probe"); + assert_eq!(detected["protocolVersion"], 7); + assert_eq!(detected["availableCodecs"], json!(["h264"])); + assert_eq!( + detected["capabilities"]["videoBackends"][0]["backend"], + "software" + ); + + fs::remove_file(fixture).expect("remove capability fixture"); + } + + #[test] + fn context_resolves_auto_to_concrete_h264() { + let context = streamer_context( + json!({"sessionId":"one","status":2,"signalingUrl":"wss://server.nvidiagrid.net/nvst/"}), + &json!({"codec":"auto","transportMode":"webrtc","resolution":"1920x1080"}), + ); + assert_eq!(context["settings"]["codec"], "H264"); + assert_eq!(context["settings"]["transportMode"], "nvst"); + assert_eq!(context_resolution(&context), (1920, 1080)); + } + + #[test] + fn embedded_prepare_returns_nvst_context_without_a_native_surface() { + let service = StreamerService::new(); + let prepared = service + .prepare_embedded( + &json!({ + "session": { + "sessionId": "session-one", + "status": 2, + "signalingUrl": "wss://server.nvidiagrid.net/nvst/", + "negotiatedStreamProfile": { + "codec": "H264", + "dynamicStreamingMode": 1 + } + } + }), + &json!({ + "codec":"auto", + "transportMode":"webrtc", + "resolution":"1920x1080", + "maxBitrateMbps":200 + }), + ) + .expect("embedded context"); + + assert_eq!(prepared["protocolVersion"], STREAMER_PROTOCOL_VERSION); + assert_eq!(prepared["context"]["session"]["sessionId"], "session-one"); + assert_eq!(prepared["context"]["settings"]["codec"], "H264"); + assert_eq!(prepared["context"]["settings"]["transportMode"], "nvst"); + assert_eq!(prepared["context"]["settings"]["maxBitrateMbps"], 200); + assert_eq!( + prepared["context"]["session"]["negotiatedStreamProfile"]["dynamicStreamingMode"], + 1 + ); + assert_eq!(prepared["context"]["surface"], Value::Null); + assert!(service.worker.lock().expect("streamer worker").is_none()); + } + + #[test] + fn embedded_prepare_preserves_replay_opt_in_and_capture_bindings() { + for enabled in [false, true] { + for (recording, clip) in [("F12", "Alt+F9"), ("", "")] { + let service = StreamerService::new(); + let prepared = service + .prepare_embedded( + &json!({"session": {"sessionId": "replay-test", "status": 2}}), + &json!({"codec": "h264", "replayBufferEnabled": enabled, + "replayBufferSeconds": 60, "replayBufferMemoryMiB": 128, + "shortcutToggleRecording": recording, "shortcutSaveClip": clip}), + ) + .unwrap(); + assert_eq!( + prepared["context"]["settings"]["replayBufferEnabled"], + enabled + ); + assert_eq!(prepared["context"]["settings"]["replayBufferSeconds"], 60); + assert_eq!( + prepared["context"]["settings"]["replayBufferMemoryMiB"], + 128 + ); + assert_eq!( + prepared["context"]["shortcuts"]["toggleRecording"], + recording + ); + assert_eq!(prepared["context"]["shortcuts"]["saveClip"], clip); + } + } + } + + #[test] + fn embedded_prepare_preserves_selected_audio_output() { + for device in [ + "", + "pipewire:alsa_output.usb-headphones", + "coreaudio:BuiltInSpeakerDevice", + "USB Headphones", + ] { + let service = StreamerService::new(); + let prepared = service + .prepare_embedded( + &json!({"session": {"sessionId": "audio-test", "status": 2}}), + &json!({"codec": "h264", "audioOutputDevice": device}), + ) + .unwrap(); + assert_eq!(prepared["context"]["settings"]["audioOutputDevice"], device); + } + } + + #[test] + fn microphone_child_state_is_exposed_without_losing_the_message() { + let state = Arc::new(Mutex::new(Snapshot::default())); + apply_microphone_state( + &json!({ + "type":"microphone-state", + "state":"ready", + "enabled":true, + "message":"Microphone is streaming" + }), + &state, + ); + + let value = state.lock().expect("snapshot").value(); + assert_eq!(value["streamer"]["microphoneState"], "ready"); + assert_eq!(value["streamer"]["microphoneEnabled"], true); + assert_eq!( + value["streamer"]["microphoneMessage"], + "Microphone is streaming" + ); + } + + #[test] + fn native_shell_shortcuts_are_forwarded_to_qt_and_counted_once() { + let state = Arc::new(Mutex::new(Snapshot::default())); + handle_nvst_child_event( + &json!({"type":"shortcut-action","action":"toggle-stats"}), + &state, + ) + .expect("supported shortcut action"); + let value = state.lock().expect("snapshot").value(); + assert_eq!(value["streamer"]["shortcutAction"], "toggle-stats"); + assert_eq!(value["streamer"]["shortcutActionGeneration"], 1); + assert_eq!(value["streamer"]["statsToggleCount"], 1); + + handle_nvst_child_event( + &json!({"type":"shortcut-action","action":"toggle-fullscreen"}), + &state, + ) + .expect("supported fullscreen shortcut action"); + let value = state.lock().expect("snapshot").value(); + assert_eq!(value["streamer"]["shortcutAction"], "toggle-fullscreen"); + assert_eq!(value["streamer"]["shortcutActionGeneration"], 2); + assert_eq!(value["streamer"]["fullscreenToggleCount"], 1); + + let failure = handle_nvst_child_event( + &json!({"type":"shortcut-action","action":"launch-command"}), + &state, + ) + .expect_err("unsupported shortcut action must fail closed"); + assert_eq!(failure.code, "streamer_protocol_error"); + assert_eq!( + state.lock().expect("snapshot").shortcut_action_generation, + 2 + ); + } + + #[test] + fn structured_child_events_accumulate_redacted_acceptance_evidence() { + let state = Arc::new(Mutex::new(Snapshot { + started_at: Instant::now().checked_sub(Duration::from_millis(25)), + ..Snapshot::default() + })); + for event in [ + json!({"type":"input-ready","protocolVersion":1}), + json!({"type":"log","event":"backend-fallback","fromBackend":"hardware","toBackend":"software"}), + json!({"type":"log","event":"queue-dropped","media":"video","count":3}), + json!({"type":"log","event":"device-state","subsystem":"video","recovered":false}), + json!({"type":"log","event":"device-state","subsystem":"video","recovered":true}), + json!({"type":"status","event":"first-frame","backend":"ffmpeg","status":"streaming"}), + json!({"type":"telemetry","framesPerSecond":59.8,"bitrateMbps":42.5,"peakBitrateMbps":51.0}), + ] { + apply_child_telemetry(&event, &state); + } + + let value = state.lock().expect("snapshot").acceptance_value(); + assert_eq!(value["kind"], "opennow.stream.acceptance"); + assert_eq!(value["mediaBackend"], "ffmpeg"); + assert!(value["firstFrameLatencyMs"].as_u64().unwrap_or_default() >= 25); + assert_eq!(value["backendFallbackCount"], 1); + assert_eq!(value["queueDropCount"], 3); + assert_eq!(value["deviceLossCount"], 1); + assert_eq!(value["deviceRecoveryCount"], 1); + assert_eq!(value["inputReady"], true); + assert_eq!(value["framesPerSecond"], 59.8); + assert_eq!(value["bitrateMbps"], 42.5); + assert_eq!(value["peakBitrateMbps"], 51.0); + assert!(value.get("sessionId").is_none()); + assert!(value.get("executable").is_none()); + assert!(value.get("processId").is_none()); + } + + #[test] + fn dynamic_surface_preserves_embedding_handle_and_normalizes_geometry() { + let surface = normalize_surface( + Some(&json!({ + "rect":{"x":32,"y":64,"width":1600,"height":900}, + "screenRect":{"x":-120,"y":48,"width":1600,"height":900}, + "visible":true, + "deviceScaleFactor":1.5, + "windowHandle":"0x1234" + })), + (1280, 720), + ) + .unwrap(); + assert_eq!( + surface["rect"], + json!({"x":32,"y":64,"width":1600,"height":900}) + ); + assert_eq!(surface["screenRect"]["x"], -120); + assert_eq!(surface["deviceScaleFactor"], 1.5); + assert_eq!(surface["windowHandle"], "0x1234"); + assert!( + normalize_surface( + Some(&json!({"screenRect":{"width":32,"height":900}})), + (1280, 720) + ) + .is_err() + ); + } + + #[test] + fn nvst_context_preserves_explicit_transport() { + let context = streamer_context( + json!({"sessionId":"one","status":2,"rtspsEndpoints":["rtsps://203.0.113.20:322/session"]}), + &json!({"codec":"H264","transportMode":"nvst","resolution":"1920x1080"}), + ); + assert_eq!(context["settings"]["transportMode"], "nvst"); + } +} diff --git a/external/opennow/native/opennow-core/src/thanks.rs b/external/opennow/native/opennow-core/src/thanks.rs new file mode 100644 index 000000000..0f627acfa --- /dev/null +++ b/external/opennow/native/opennow-core/src/thanks.rs @@ -0,0 +1,278 @@ +use regex::Regex; +use reqwest::blocking::Client; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::collections::HashSet; +use std::thread; +use std::time::Duration; +use url::Url; + +const CONTRIBUTORS_URL: &str = + "https://api.github.com/repos/OpenCloudGaming/OpenNOW/contributors?per_page=100"; +const SUPPORTERS_URL: &str = "https://github.com/sponsors/zortos293"; +const MAXIMUM_RESPONSE_BYTES: u64 = 4 * 1024 * 1024; + +pub struct ThanksService { + client: Client, +} + +impl ThanksService { + pub fn new() -> Result { + Ok(Self { + client: Client::builder() + .connect_timeout(Duration::from_secs(8)) + .timeout(Duration::from_secs(8)) + .pool_idle_timeout(Duration::from_secs(30)) + .build() + .map_err(|error| error.to_string())?, + }) + } + + pub fn data(&self) -> Value { + let (contributors, supporters) = thread::scope(|scope| { + let contributors = scope.spawn(|| self.contributors()); + let supporters = scope.spawn(|| self.supporters()); + ( + contributors + .join() + .unwrap_or_else(|_| Err("Contributor request failed".to_owned())), + supporters + .join() + .unwrap_or_else(|_| Err("Supporter request failed".to_owned())), + ) + }); + let mut result = json!({ + "contributors":contributors.as_ref().cloned().unwrap_or_default(), + "supporters":supporters.as_ref().cloned().unwrap_or_else(|_| custom_supporters()) + }); + if let Err(error) = contributors { + result["contributorsError"] = json!(error); + } + if let Err(error) = supporters { + result["supportersError"] = json!(error); + } + result + } + + fn contributors(&self) -> Result, String> { + let response = self + .client + .get(CONTRIBUTORS_URL) + .header("Accept", "application/vnd.github+json") + .header("User-Agent", "OpenNOW-DesktopClient") + .send() + .map_err(|_| "Unable to load contributors right now".to_owned())?; + if !response.status().is_success() + || response.content_length().unwrap_or(0) > MAXIMUM_RESPONSE_BYTES + { + return Err(format!( + "GitHub contributors request failed ({})", + response.status().as_u16() + )); + } + let mut values = response + .json::>() + .map_err(|_| "GitHub contributors response was invalid".to_owned())? + .into_iter() + .filter_map(normalize_contributor) + .collect::>(); + values.sort_by(|left, right| { + right["contributions"] + .as_u64() + .cmp(&left["contributions"].as_u64()) + .then_with(|| left["login"].as_str().cmp(&right["login"].as_str())) + }); + Ok(values) + } + + fn supporters(&self) -> Result, String> { + let response = self + .client + .get(SUPPORTERS_URL) + .header("Accept", "text/html,application/xhtml+xml") + .header("User-Agent", "OpenNOW-DesktopClient") + .send() + .map_err(|_| "Unable to load supporters right now".to_owned())?; + if !response.status().is_success() + || response.content_length().unwrap_or(0) > MAXIMUM_RESPONSE_BYTES + { + return Err(format!( + "GitHub sponsors request failed ({})", + response.status().as_u16() + )); + } + let html = response + .text() + .map_err(|_| "GitHub sponsors response was invalid".to_owned())?; + let mut output = custom_supporters(); + output.extend(parse_supporters(&html)); + let mut seen = HashSet::new(); + output.retain(|value| { + let key = value["profileUrl"] + .as_str() + .map(|value| format!("profile:{}", value.to_ascii_lowercase())) + .unwrap_or_else(|| { + format!( + "name:{}|private:{}", + value["name"] + .as_str() + .unwrap_or_default() + .to_ascii_lowercase(), + value["isPrivate"].as_bool().unwrap_or(false) + ) + }); + seen.insert(key) + }); + if output.is_empty() { + return Err("No public supporters were found on GitHub Sponsors".to_owned()); + } + Ok(output) + } +} + +#[derive(Deserialize)] +struct Contributor { + login: Option, + avatar_url: Option, + html_url: Option, + contributions: Option, + #[serde(rename = "type")] + kind: Option, +} + +fn normalize_contributor(value: Contributor) -> Option { + let login = value.login?.trim().to_owned(); + let avatar = trusted_github_url(&value.avatar_url?)?; + let profile = trusted_github_url(&value.html_url?)?; + let excluded = login.ends_with("[bot]") + || value.kind.as_deref() == Some("Bot") + || ["copilot", "claude", "cappy"] + .iter() + .any(|needle| login.to_ascii_lowercase().contains(needle)); + if login.is_empty() || excluded { + return None; + } + Some(json!({ + "login":login, + "avatarUrl":avatar, + "profileUrl":profile, + "contributions":value.contributions.unwrap_or(0) + })) +} + +fn parse_supporters(html: &str) -> Vec { + let entry = Regex::new(r#"(?is)
]*>(.*?)
"#) + .expect("supporter entry regex"); + let image = Regex::new(r#"(?is)]+(?:alt|aria-label)="([^"]*)"[^>]*src="([^"]+)""#) + .expect("supporter image regex"); + let href = Regex::new(r#"(?is)]+href="([^"]+)""#).expect("supporter href regex"); + let mut values = Vec::new(); + for capture in entry.captures_iter(html).take(100) { + let body = capture + .get(1) + .map(|value| value.as_str()) + .unwrap_or_default(); + let private = body.to_ascii_lowercase().contains("private sponsor"); + let image_capture = image.captures(body); + let name = if private { + "Private".to_owned() + } else { + image_capture + .as_ref() + .and_then(|capture| capture.get(1)) + .map(|value| { + decode_html(value.as_str()) + .trim_start_matches('@') + .trim() + .to_owned() + }) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "Private".to_owned()) + }; + let avatar = image_capture + .as_ref() + .and_then(|capture| capture.get(2)) + .and_then(|value| normalize_github_url(value.as_str())); + let profile = (!private) + .then(|| href.captures(body)) + .flatten() + .and_then(|capture| capture.get(1)) + .and_then(|value| normalize_github_url(value.as_str())); + values.push(json!({ + "name":name, + "avatarUrl":avatar, + "profileUrl":profile, + "isPrivate":private || profile.is_none(), + "source":if private { "private" } else { "github" } + })); + } + values +} + +fn custom_supporters() -> Vec { + vec![json!({ + "name":"DarkevilPT", + "avatarUrl":"https://github.com/DarkevilPT.png?size=96", + "profileUrl":"https://github.com/DarkevilPT", + "isPrivate":false, + "source":"custom" + })] +} + +fn normalize_github_url(value: &str) -> Option { + let value = decode_html(value.trim()); + let absolute = if value.starts_with("//") { + format!("https:{value}") + } else if value.starts_with('/') { + format!("https://github.com{value}") + } else { + value + }; + trusted_github_url(&absolute) +} + +fn trusted_github_url(value: &str) -> Option { + let parsed = Url::parse(value).ok()?; + let host = parsed.host_str()?.to_ascii_lowercase(); + (parsed.scheme() == "https" + && matches!( + host.as_str(), + "github.com" | "avatars.githubusercontent.com" | "user-images.githubusercontent.com" + )) + .then(|| parsed.to_string()) +} + +fn decode_html(value: &str) -> String { + value + .replace("&", "&") + .replace(""", "\"") + .replace("'", "'") + .replace("<", "<") + .replace(">", ">") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn contributor_and_supporter_inputs_are_bounded_and_trusted() { + assert!( + normalize_contributor(Contributor { + login: Some("dependabot[bot]".to_owned()), + avatar_url: Some("https://avatars.githubusercontent.com/u/1".to_owned()), + html_url: Some("https://github.com/apps/dependabot".to_owned()), + contributions: Some(3), + kind: Some("Bot".to_owned()), + }) + .is_none() + ); + assert!(trusted_github_url("https://evil.example/avatar.png").is_none()); + let supporters = parse_supporters( + r#"
@player
"#, + ); + assert_eq!(supporters.len(), 1); + assert_eq!(supporters[0]["name"], "player"); + assert_eq!(supporters[0]["profileUrl"], "https://github.com/player"); + } +} diff --git a/external/opennow/native/opennow-core/src/update_apply/archive.rs b/external/opennow/native/opennow-core/src/update_apply/archive.rs new file mode 100644 index 000000000..7e7292126 --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/archive.rs @@ -0,0 +1,170 @@ +use std::collections::HashSet; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; + +const MAX_ENTRIES: usize = 100_000; +const MAX_EXPANDED_BYTES: u64 = 12 * 1024 * 1024 * 1024; + +pub(super) fn extract_zip( + package: &Path, + destination: &Path, + allow_links: bool, +) -> Result<(), String> { + fs::create_dir(destination).map_err(|error| error.to_string())?; + let result = extract(package, destination, allow_links); + if result.is_err() { + let _ = fs::remove_dir_all(destination); + } + result +} + +fn extract(package: &Path, destination: &Path, allow_links: bool) -> Result<(), String> { + let mut archive = zip::ZipArchive::new(File::open(package).map_err(|error| error.to_string())?) + .map_err(|error| format!("Invalid update ZIP: {error}"))?; + if archive.len() > MAX_ENTRIES { + return Err("Update archive has too many entries".to_owned()); + } + let mut paths = HashSet::new(); + let mut expanded = 0u64; + let mut links = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).map_err(|error| error.to_string())?; + let relative = archive_path(entry.name())?; + if !paths.insert(relative.to_string_lossy().to_lowercase()) { + return Err("Update archive contains duplicate paths".to_owned()); + } + expanded = expanded + .checked_add(entry.size()) + .ok_or("Update archive size overflow")?; + if expanded > MAX_EXPANDED_BYTES || entry.size() > super::verification::MAXIMUM_UPDATE_BYTES + { + return Err("Expanded update exceeds its size limit".to_owned()); + } + let mode = entry.unix_mode().unwrap_or(0o100644); + let file_type = mode & 0o170000; + let output = destination.join(&relative); + if file_type == 0o120000 { + if !allow_links || entry.size() > 4096 { + return Err("Unsupported update archive symlink".to_owned()); + } + let mut target = String::new(); + entry + .by_ref() + .take(4097) + .read_to_string(&mut target) + .map_err(|error| error.to_string())?; + validate_link(&relative, &target)?; + links.push((output, PathBuf::from(target))); + continue; + } + if !matches!(file_type, 0 | 0o040000 | 0o100000) { + return Err("Update archive contains a special file".to_owned()); + } + if entry.is_dir() { + fs::create_dir_all(&output).map_err(|error| error.to_string())?; + continue; + } + fs::create_dir_all(output.parent().ok_or("Invalid archive path")?) + .map_err(|error| error.to_string())?; + let expected = entry.size(); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .map_err(|error| error.to_string())?; + let copied = std::io::copy(&mut entry.by_ref().take(expected + 1), &mut file) + .map_err(|error| error.to_string())?; + if copied != expected { + return Err("Update ZIP entry has an invalid expanded size".to_owned()); + } + file.flush() + .and_then(|_| file.sync_all()) + .map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + &output, + fs::Permissions::from_mode(if mode & 0o111 != 0 { 0o755 } else { 0o644 }), + ) + .map_err(|error| error.to_string())?; + } + } + #[cfg(not(unix))] + if !links.is_empty() { + return Err("Update symlinks are unsupported on this platform".to_owned()); + } + #[cfg(unix)] + for (path, target) in &links { + fs::create_dir_all(path.parent().ok_or("Invalid symlink path")?) + .map_err(|error| error.to_string())?; + std::os::unix::fs::symlink(target, path).map_err(|error| error.to_string())?; + } + let root = fs::canonicalize(destination).map_err(|error| error.to_string())?; + for (path, _) in links { + if !fs::canonicalize(path) + .map_err(|error| format!("Invalid update symlink: {error}"))? + .starts_with(&root) + { + return Err("Update symlink escapes the package".to_owned()); + } + } + Ok(()) +} + +pub(super) fn archive_path(value: &str) -> Result { + if value.is_empty() || value.contains(['\\', ':', '\0']) || value.starts_with('/') { + return Err("Unsafe update archive path".to_owned()); + } + let value = value.trim_end_matches('/'); + for part in value.split('/') { + let stem = part + .split('.') + .next() + .unwrap_or_default() + .to_ascii_uppercase(); + if part.is_empty() + || part == "." + || part == ".." + || part.ends_with(['.', ' ']) + || matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && stem.as_bytes()[3].is_ascii_digit()) + { + return Err("Unsafe update archive component".to_owned()); + } + } + let path = PathBuf::from(value); + if !path + .components() + .all(|part| matches!(part, Component::Normal(_))) + { + return Err("Unsafe update archive path".to_owned()); + } + Ok(path) +} + +fn validate_link(path: &Path, target: &str) -> Result<(), String> { + if target.is_empty() || target.contains(['\\', ':', '\0']) || Path::new(target).is_absolute() { + return Err("Unsafe update symlink target".to_owned()); + } + let mut depth = path + .parent() + .map(|parent| parent.components().count()) + .unwrap_or(0); + for part in Path::new(target).components() { + match part { + Component::ParentDir => { + depth = depth + .checked_sub(1) + .ok_or("Update symlink escapes the package")?; + } + Component::Normal(_) => depth += 1, + Component::CurDir => (), + _ => return Err("Unsafe update symlink target".to_owned()), + } + } + Ok(()) +} diff --git a/external/opennow/native/opennow-core/src/update_apply/bundle.rs b/external/opennow/native/opennow-core/src/update_apply/bundle.rs new file mode 100644 index 000000000..5c65d237b --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/bundle.rs @@ -0,0 +1,267 @@ +use std::path::Path; + +pub(super) const MAX_COPY_ENTRIES: u64 = 100_000; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum CopyPolicy { + Bundle, + UserData, +} + +pub(super) struct CopyBudget { + remaining_entries: u64, + remaining_bytes: u64, +} + +impl CopyBudget { + pub fn new() -> Self { + Self { + remaining_entries: MAX_COPY_ENTRIES, + remaining_bytes: 12 * 1024 * 1024 * 1024, + } + } + + #[cfg(test)] + pub fn with_limits(entries: u64, bytes: u64) -> Self { + Self { + remaining_entries: entries, + remaining_bytes: bytes, + } + } + + pub fn consume_entry(&mut self) -> Result<(), String> { + self.remaining_entries = self + .remaining_entries + .checked_sub(1) + .ok_or("Preserved data exceeds its aggregate entry limit")?; + Ok(()) + } + + pub fn copy_file( + &mut self, + source: &Path, + destination: &Path, + policy: CopyPolicy, + ) -> Result<(), String> { + let copied = super::copy_synced_bounded( + source, + destination, + self.remaining_bytes + .min(super::verification::MAXIMUM_UPDATE_BYTES), + policy, + )?; + self.remaining_bytes -= copied; + Ok(()) + } +} + +#[cfg(target_os = "macos")] +fn run(command: &mut std::process::Command) -> Result<(), String> { + use std::process::Stdio; + use std::time::{Duration, Instant}; + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| error.to_string())?; + let start = Instant::now(); + loop { + if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { + return if status.success() { + Ok(()) + } else { + Err("macOS package operation failed".to_owned()) + }; + } + if start.elapsed() >= Duration::from_secs(120) { + let _ = child.kill(); + let _ = child.wait(); + return Err("macOS package operation exceeded its deadline".to_owned()); + } + std::thread::sleep(Duration::from_millis(100)); + } +} + +pub(super) fn extract_dmg(package: &Path, destination: &Path) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + use std::process::Command; + let mount = destination.with_extension("mount"); + std::fs::create_dir(&mount).map_err(|error| error.to_string())?; + let attached = run(Command::new("/usr/bin/hdiutil") + .args([ + "attach", + "-readonly", + "-nobrowse", + "-noautoopen", + "-mountpoint", + ]) + .arg(&mount) + .arg(package)); + let result = attached.and_then(|_| { + let apps: Vec<_> = std::fs::read_dir(&mount) + .map_err(|error| error.to_string())? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.extension().is_some_and(|value| value == "app") && path.is_dir() + }) + .collect(); + if apps.len() != 1 || apps[0].file_name().is_none_or(|name| name != "OpenNOW.app") { + return Err("DMG must contain exactly one OpenNOW.app bundle".to_owned()); + } + std::fs::create_dir(destination).map_err(|error| error.to_string())?; + copy_tree( + &apps[0], + &destination.join("OpenNOW.app"), + &mut CopyBudget::new(), + CopyPolicy::Bundle, + ) + }); + let detached = + run(Command::new("/usr/bin/hdiutil").arg("detach").arg(&mount)).or_else(|_| { + run(Command::new("/usr/bin/hdiutil") + .args(["detach", "-force"]) + .arg(&mount)) + }); + if detached.is_ok() { + let _ = std::fs::remove_dir(&mount); + } + result.and(detached) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (package, destination); + Err("DMG updates require macOS".to_owned()) + } +} + +pub(super) fn copy_tree( + source: &Path, + destination: &Path, + budget: &mut CopyBudget, + policy: CopyPolicy, +) -> Result<(), String> { + use std::fs; + budget.consume_entry()?; + let source = fs::canonicalize(source).map_err(|error| error.to_string())?; + if policy == CopyPolicy::UserData { + super::security::create_private_directory(destination)?; + } else { + fs::create_dir(destination).map_err(|error| error.to_string())?; + } + let result = (|| { + let mut pending = vec![(source.clone(), destination.to_path_buf())]; + let mut links = Vec::new(); + let mut directory_permissions = Vec::new(); + while let Some((from, to)) = pending.pop() { + if policy == CopyPolicy::UserData { + directory_permissions.push((from.clone(), to.clone())); + } + for entry in fs::read_dir(&from).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + budget.consume_entry()?; + let metadata = + fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?; + let path = to.join(entry.file_name()); + if metadata.is_symlink() { + let resolved = + fs::canonicalize(entry.path()).map_err(|error| error.to_string())?; + if !resolved.starts_with(&source) { + return Err("Application bundle symlink escapes its root".to_owned()); + } + let target = fs::read_link(entry.path()).map_err(|error| error.to_string())?; + if target.is_absolute() { + return Err("Application bundle contains an absolute symlink".to_owned()); + } + links.push((path, target)); + } else if metadata.is_dir() { + if policy == CopyPolicy::UserData { + super::security::create_private_directory(&path)?; + } else { + fs::create_dir(&path).map_err(|error| error.to_string())?; + } + pending.push((entry.path(), path)); + } else if metadata.is_file() { + budget.copy_file(&entry.path(), &path, policy)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if policy == CopyPolicy::Bundle { + fs::set_permissions( + &path, + fs::Permissions::from_mode( + if metadata.permissions().mode() & 0o111 != 0 { + 0o755 + } else { + 0o644 + }, + ), + ) + .map_err(|error| error.to_string())?; + } + } + } else { + return Err("Application bundle contains a special file".to_owned()); + } + } + } + #[cfg(not(unix))] + if !links.is_empty() { + return Err("Bundle symlinks require macOS or Unix".to_owned()); + } + #[cfg(unix)] + for (path, target) in links { + std::os::unix::fs::symlink(&target, &path).map_err(|error| error.to_string())?; + } + for (source, destination) in directory_permissions.into_iter().rev() { + super::security::preserve_permissions(&source, &destination)?; + } + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(destination); + } + result +} + +pub(super) fn verify_bundle(bundle: &Path, installed_bundle: &Path) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + use std::process::Command; + run(Command::new("/usr/bin/codesign") + .args(["--verify", "--deep", "--strict"]) + .arg(bundle))?; + let identity = |path: &Path| -> Result { + let output = super::command_output( + Command::new("/usr/bin/codesign") + .args(["-dv", "--verbose=4"]) + .arg(path), + std::time::Duration::from_secs(120), + )?; + if !output.status.success() || output.stderr.len() > 64 * 1024 { + return Err("Cannot read application signing identity".to_owned()); + } + let value = String::from_utf8_lossy(&output.stderr); + Ok(value + .lines() + .filter(|line| { + line.starts_with("TeamIdentifier=") || line.starts_with("Identifier=") + }) + .collect::>() + .join("\n")) + }; + let old = identity(installed_bundle)?; + if old.is_empty() || old != identity(bundle)? { + return Err( + "Updated app bundle signing identity does not match the installation".to_owned(), + ); + } + Ok(()) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (bundle, installed_bundle); + Err("macOS bundle verification requires macOS".to_owned()) + } +} diff --git a/external/opennow/native/opennow-core/src/update_apply/integration_tests.rs b/external/opennow/native/opennow-core/src/update_apply/integration_tests.rs new file mode 100644 index 000000000..77de2d51b --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/integration_tests.rs @@ -0,0 +1,490 @@ +use super::*; +use base64::Engine as _; +use ed25519_dalek::{Signer, SigningKey}; +use sha2::{Digest, Sha256}; +use tempfile::TempDir; + +struct OwnedChild(Child); + +impl Drop for OwnedChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +struct Fixture { + _temporary: TempDir, + plan: Plan, + directory: PathBuf, + original: Vec, + candidate: Vec, + application: OwnedChild, + core: OwnedChild, +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::write(self.directory.join("exit-fixture"), b"exit"); + if let Ok(ack) = read_json::(&self.directory.join("ack.json"), 64 * 1024) { + let start = Instant::now(); + while ack.application.is_running().unwrap_or(false) + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(50)); + } + } + let _ = self.application.0.kill(); + let _ = self.application.0.wait(); + let _ = self.core.0.kill(); + let _ = self.core.0.wait(); + } +} + +fn fixture_path(name: &str) -> PathBuf { + fs::canonicalize(std::env::var_os(name).unwrap_or_else(|| { + panic!("Run opennow-qt/tests/run_update_helper_integration.py; missing {name}") + })) + .unwrap() +} + +#[cfg(target_os = "macos")] +fn command_ok(command: &mut Command) { + let output = command.output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn sign(package: &Path) { + let seed: [u8; 32] = fs::read(fixture_path("OPENNOW_TEST_UPDATE_SEED_FILE")) + .unwrap() + .try_into() + .unwrap(); + let key = SigningKey::from_bytes(&seed); + assert_eq!( + verification::embedded_update_key().unwrap(), + key.verifying_key(), + "Test key must be compiled into the helper; runtime key overrides are forbidden" + ); + let bytes = fs::read(package).unwrap(); + let mut manifest = verification::UpdateManifest { + schema_version: 1, + version: "1.2.3".to_owned(), + asset: package.file_name().unwrap().to_str().unwrap().to_owned(), + size: bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(&bytes)), + signature: String::new(), + }; + manifest.signature = base64::engine::general_purpose::STANDARD.encode( + key.sign(verification::signature_payload(&manifest).as_bytes()) + .to_bytes(), + ); + atomic_json(&manifest_path(package).unwrap(), &manifest).unwrap(); +} + +fn stage_executable(source: &Path, target: &Path) { + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::copy(source, target).unwrap(); + make_executable(target).unwrap(); +} + +#[cfg(target_os = "macos")] +fn sign_bundle(root: &Path, identifier: &str, version: &str) { + fs::write(root.join("Contents/Info.plist"), format!(r#"CFBundleIdentifier{identifier}CFBundleExecutableOpenNOWCFBundlePackageTypeAPPLCFBundleShortVersionString{version}CFBundleVersion{version}"#)).unwrap(); + command_ok( + Command::new("/usr/bin/codesign") + .args([ + "--force", + "--deep", + "--sign", + "-", + "--identifier", + identifier, + ]) + .arg(root), + ); + command_ok( + Command::new("/usr/bin/codesign") + .args(["--verify", "--deep", "--strict"]) + .arg(root), + ); +} + +#[cfg(windows)] +fn zip_tree(root: &Path, package: &Path) { + let mut zip = zip::ZipWriter::new(File::create(package).unwrap()); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + continue; + } + let name = path + .strip_prefix(root) + .unwrap() + .to_str() + .unwrap() + .replace('\\', "/"); + zip.start_file( + name, + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + zip.write_all(&fs::read(path).unwrap()).unwrap(); + } + } + zip.finish().unwrap(); +} + +fn fixture(wrong_identity: bool) -> Fixture { + let temporary = TempDir::new().unwrap(); + let root = fs::canonicalize(temporary.path()).unwrap(); + let directory = root.join(".opennow-update-integration"); + fs::create_dir(&directory).unwrap(); + let previous = fixture_path("OPENNOW_TEST_UPDATE_PREVIOUS"); + let candidate = fixture_path("OPENNOW_TEST_UPDATE_CANDIDATE"); + #[cfg(any(windows, target_os = "macos"))] + let helper = fixture_path("OPENNOW_TEST_UPDATE_HELPER"); + #[cfg(target_os = "linux")] + let (kind, target, application, package, candidate_bytes) = { + let target = root.join("OpenNOW.AppImage"); + stage_executable(&previous, &target); + let package = directory.join("OpenNOW-1.2.3.AppImage"); + let mut bytes = fs::read(&candidate).unwrap(); + assert_eq!(&bytes[..4], b"\x7fELF"); + bytes[8..11].copy_from_slice(b"AI\x02"); + fs::write(&package, &bytes).unwrap(); + ( + InstallKind::AppImage, + target.clone(), + target, + package, + bytes, + ) + }; + #[cfg(any(windows, target_os = "macos"))] + let (kind, target, application, package, candidate_bytes) = { + #[cfg(windows)] + let (kind, target, relative, package) = ( + InstallKind::WindowsPortable, + root.join("installed"), + PathBuf::from("bin/OpenNOW.exe"), + directory.join("OpenNOW-1.2.3.zip"), + ); + #[cfg(target_os = "macos")] + let (kind, target, relative, package) = ( + InstallKind::MacBundle, + root.join("OpenNOW.app"), + PathBuf::from("Contents/MacOS/OpenNOW"), + directory.join("OpenNOW-1.2.3.dmg"), + ); + let application = target.join(&relative); + stage_executable(&previous, &application); + let image_root = root.join("image-root"); + #[cfg(target_os = "macos")] + let payload = image_root.join("OpenNOW.app"); + #[cfg(windows)] + let payload = image_root.clone(); + let payload_application = payload.join(&relative); + stage_executable(&candidate, &payload_application); + let suffix = if cfg!(windows) { ".exe" } else { "" }; + for (name, source) in [ + (format!("opennow-core{suffix}"), &previous), + (format!("opennow-update-helper{suffix}"), &helper), + ] { + stage_executable(source, &application.parent().unwrap().join(&name)); + stage_executable(source, &payload_application.parent().unwrap().join(&name)); + } + #[cfg(windows)] + { + if wrong_identity { + fs::rename( + &payload_application, + payload_application.with_file_name("OtherNOW.exe"), + ) + .unwrap(); + } + zip_tree(&payload, &package); + } + #[cfg(target_os = "macos")] + { + sign_bundle(&target, "org.opennow.integration", "1.0.0"); + sign_bundle( + &payload, + if wrong_identity { + "org.opennow.unrelated" + } else { + "org.opennow.integration" + }, + "1.2.3", + ); + command_ok( + Command::new("/usr/bin/hdiutil") + .args([ + "create", + "-quiet", + "-format", + "UDZO", + "-volname", + "OpenNOW integration", + "-srcfolder", + ]) + .arg(&image_root) + .arg(&package), + ); + } + let bytes = if payload_application.exists() { + fs::read(&payload_application).unwrap() + } else { + Vec::new() + }; + (kind, target, application, package, bytes) + }; + sign(&package); + let target = fs::canonicalize(target).unwrap(); + let application_path = fs::canonicalize(application).unwrap(); + let original = fs::read(&application_path).unwrap(); + let data_dir = if cfg!(windows) { + target.join("user-data") + } else { + root.join("settings") + }; + fs::create_dir(&data_dir).unwrap(); + fs::write(data_dir.join("preferences.json"), b"{\"preserved\":true}").unwrap(); + let application = OwnedChild( + Command::new(&application_path) + .arg("--hold") + .spawn() + .unwrap(), + ); + let core = OwnedChild(Command::new(&previous).arg("--hold").spawn().unwrap()); + let plan = Plan { + schema_version: 1, + version: if cfg!(target_os = "linux") && wrong_identity { + "1.2.4" + } else { + "1.2.3" + } + .to_owned(), + kind, + package, + target, + application_executable: application_path, + data_dir: fs::canonicalize(data_dir).unwrap(), + processes: vec![ + ProcessIdentity::capture(application.0.id()).unwrap(), + ProcessIdentity::capture(core.0.id()).unwrap(), + ], + nonce: "a".repeat(64), + managed_identity: None, + }; + atomic_json(&directory.join("plan.json"), &plan).unwrap(); + write_outcome( + &directory, + &plan, + OutcomeStatus::Prepared, + "Integration transaction prepared", + None, + None, + ) + .unwrap(); + Fixture { + _temporary: temporary, + plan, + directory, + original, + candidate: candidate_bytes, + application, + core, + } +} + +fn wait_for_status(fixture: &Fixture, helper: &mut OwnedChild, status: OutcomeStatus) { + let start = Instant::now(); + loop { + if read_outcome(&fixture.directory.join("outcome.json")) + .unwrap() + .is_some_and(|outcome| outcome.status == status) + { + return; + } + assert!( + helper.0.try_wait().unwrap().is_none(), + "Helper exited before {status:?}" + ); + assert!( + start.elapsed() < Duration::from_secs(150), + "Helper never reached {status:?}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn apply_after_exit(fixture: &mut Fixture) -> std::process::ExitStatus { + let mut command = Command::new(fixture_path("OPENNOW_TEST_UPDATE_HELPER")); + #[cfg(target_os = "linux")] + command + .env("APPDIR", "/tmp/opennow-old-mount") + .env("APPIMAGE", &fixture.plan.target) + .env("ARGV0", "/tmp/opennow-old-mount/AppRun"); + let mut helper = OwnedChild( + command + .arg("--apply") + .arg(fixture.directory.join("plan.json")) + .spawn() + .unwrap(), + ); + wait_for_status(fixture, &mut helper, OutcomeStatus::WaitingForExit); + assert_eq!( + fs::read(&fixture.plan.application_executable).unwrap(), + fixture.original + ); + assert!(!fixture.directory.join("ack.json").exists()); + fixture.application.0.kill().unwrap(); + fixture.application.0.wait().unwrap(); + fixture.core.0.kill().unwrap(); + fixture.core.0.wait().unwrap(); + let start = Instant::now(); + loop { + if let Some(status) = helper.0.try_wait().unwrap() { + return status; + } + assert!( + start.elapsed() < Duration::from_secs(120), + "Helper did not finish replacement" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +#[ignore = "Run opennow-qt/tests/run_update_helper_integration.py with an ephemeral pinned test key"] +fn signed_native_package_replaces_and_candidate_acknowledges_its_own_restart() { + let mut fixture = fixture(false); + assert!(apply_after_exit(&mut fixture).success()); + let outcome = read_outcome(&fixture.directory.join("outcome.json")) + .unwrap() + .unwrap(); + assert_eq!(outcome.status, OutcomeStatus::Completed); + assert_eq!(outcome.installed_version.as_deref(), Some("1.2.3")); + assert_eq!( + fs::read(&fixture.plan.application_executable).unwrap(), + fixture.candidate + ); + assert_ne!(fixture.candidate, fixture.original); + let ack: Acknowledgement = read_json(&fixture.directory.join("ack.json"), 64 * 1024).unwrap(); + assert_eq!(ack.version, "1.2.3"); + assert_eq!(ack.nonce, fixture.plan.nonce); + assert_ne!(ack.application.pid, std::process::id()); + assert_ne!(ack.application.pid, fixture.plan.processes[0].pid); + assert_eq!( + ack.application.executable, + fixture.plan.application_executable + ); + assert!(ack.application.is_running().unwrap()); + #[cfg(target_os = "linux")] + { + let environment = fs::read(format!("/proc/{}/environ", ack.application.pid)).unwrap(); + let environment = String::from_utf8(environment).unwrap(); + assert!(!environment.contains("/tmp/opennow-old-mount")); + } + assert_eq!( + fs::read_to_string(fixture.directory.join("candidate-started")).unwrap(), + ack.application.pid.to_string() + ); + assert_eq!( + fs::read(fixture.plan.data_dir.join("preferences.json")).unwrap(), + b"{\"preserved\":true}" + ); + assert!(!fixture.directory.join("previous").exists()); + assert!(!fixture.plan.package.exists()); + #[cfg(target_os = "macos")] + assert!(!fixture.directory.join("payload.mount").exists()); +} + +#[test] +#[ignore = "Run opennow-qt/tests/run_update_helper_integration.py with an ephemeral pinned test key"] +fn signed_native_package_with_failed_startup_rolls_back_and_restarts_previous() { + let mut fixture = fixture(false); + fs::write(fixture.directory.join("reject-startup"), b"reject").unwrap(); + assert!(!apply_after_exit(&mut fixture).success()); + assert_eq!( + read_outcome(&fixture.directory.join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::RolledBack + ); + assert_eq!( + fs::read(&fixture.plan.application_executable).unwrap(), + fixture.original + ); + assert!(!fixture.directory.join("ack.json").exists()); + assert!(fixture.directory.join("candidate-started").exists()); + let start = Instant::now(); + while !fixture.plan.data_dir.join("previous-restarted").exists() { + assert!( + start.elapsed() < Duration::from_secs(10), + "Previous application did not restart after rollback" + ); + std::thread::sleep(Duration::from_millis(50)); + } + assert_eq!( + fs::read(fixture.plan.data_dir.join("previous-restarted")).unwrap(), + b"1.0.0" + ); +} + +#[test] +#[ignore = "Run opennow-qt/tests/run_update_helper_integration.py with an ephemeral pinned test key"] +fn tampered_native_package_never_replaces_or_restarts() { + let fixture = fixture(false); + let mut bytes = fs::read(&fixture.plan.package).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 1; + fs::write(&fixture.plan.package, bytes).unwrap(); + assert!( + run_helper(&fixture.directory.join("plan.json")) + .unwrap_err() + .contains("SHA256") + ); + assert_eq!( + fs::read(&fixture.plan.application_executable).unwrap(), + fixture.original + ); + assert!(fixture.plan.processes[0].is_running().unwrap()); + assert!(!fixture.directory.join("candidate-started").exists()); + assert_eq!( + read_outcome(&fixture.directory.join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::Failed + ); +} + +#[test] +#[ignore = "Run opennow-qt/tests/run_update_helper_integration.py with an ephemeral pinned test key"] +fn signed_wrong_package_or_bundle_identity_never_replaces() { + let fixture = fixture(true); + assert!(run_helper(&fixture.directory.join("plan.json")).is_err()); + assert_eq!( + fs::read(&fixture.plan.application_executable).unwrap(), + fixture.original + ); + assert!(fixture.plan.processes[0].is_running().unwrap()); + assert!(!fixture.directory.join("candidate-started").exists()); + assert_eq!( + read_outcome(&fixture.directory.join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::Failed + ); +} diff --git a/external/opennow/native/opennow-core/src/update_apply/managed.rs b/external/opennow/native/opennow-core/src/update_apply/managed.rs new file mode 100644 index 000000000..de327457a --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/managed.rs @@ -0,0 +1,756 @@ +use super::{InstallKind, OutcomeStatus, Plan, write_outcome}; +use serde::{Deserialize, Serialize}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct Identity { + package: String, + version: String, + architecture: String, + installed_product: String, +} + +pub(super) fn prepare( + kind: InstallKind, + package: &Path, + target: &Path, + version: &str, +) -> Result, String> { + match kind { + InstallKind::DebianPackage => deb_identity(package, target, version).map(Some), + InstallKind::WindowsMsi => msi_identity(package, target, version).map(Some), + _ => Ok(None), + } +} + +pub(super) fn verify_identity( + kind: InstallKind, + package: &Path, + target: &Path, + version: &str, + expected: Option<&Identity>, +) -> Result<(), String> { + if prepare(kind, package, target, version)?.as_ref() != expected { + return Err("Native package identity changed after preparation".to_owned()); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn output(command: &mut Command) -> Result { + let result = super::command_output(command, Duration::from_secs(120))?; + if !result.status.success() { + return Err("Native package identity query failed".to_owned()); + } + if result.stdout.len() > 64 * 1024 { + return Err("Native package query exceeded its limit".to_owned()); + } + String::from_utf8(result.stdout) + .map(|value| value.trim().to_owned()) + .map_err(|error| error.to_string()) +} + +#[cfg(target_os = "linux")] +fn deb_identity(package: &Path, target: &Path, version: &str) -> Result { + if !Path::new("/usr/bin/pkexec").is_file() { + return Err( + "DEB updates require PolicyKit's pkexec and an interactive authorization agent" + .to_owned(), + ); + } + let field = |name| { + output( + Command::new("/usr/bin/dpkg-deb") + .arg("--field") + .arg(package) + .arg(name), + ) + }; + let name = field("Package")?; + let architecture = field("Architecture")?; + let expected_arch = if cfg!(target_arch = "x86_64") { + "amd64" + } else if cfg!(target_arch = "aarch64") { + "arm64" + } else { + return Err("Unsupported DEB architecture".to_owned()); + }; + let version = version + .trim_start_matches('v') + .replace("-nightly.", "~nightly.") + .replace("-supporter.", "~supporter."); + if name != "opennow" || architecture != expected_arch || field("Version")? != version { + return Err( + "DEB package identity does not match OpenNOW, architecture, or signed version" + .to_owned(), + ); + } + let files = deb_file_owners(target)?; + if !deb_owns_file(&files, target) { + return Err( + "Running application is not owned by the installed OpenNOW DEB package".to_owned(), + ); + } + let installed = output(Command::new("/usr/bin/dpkg-query").args([ + "--show", + "--showformat=${db:Status-Status}\t${Architecture}", + "opennow", + ]))?; + if installed != format!("installed\t{expected_arch}") { + return Err("Installed OpenNOW DEB is not configured for this architecture".to_owned()); + } + Ok(Identity { + package: name, + version, + architecture, + installed_product: "opennow".to_owned(), + }) +} + +#[cfg(target_os = "linux")] +fn deb_file_owners(target: &Path) -> Result { + let name = target.file_name().ok_or("Application has no file name")?; + let mut pattern = std::ffi::OsString::from("*"); + pattern.push(name); + output( + Command::new("/usr/bin/dpkg-query") + .arg("--search") + .arg(pattern), + ) +} + +#[cfg(target_os = "linux")] +fn deb_owns_file(files: &str, target: &Path) -> bool { + let Ok(target) = super::canonical_file(target) else { + return false; + }; + files.lines().any(|line| { + line.strip_prefix("opennow: ").is_some_and(|file| { + Path::new(file).is_absolute() + && super::canonical_file(Path::new(file)).is_ok_and(|path| path == target) + }) + }) +} + +#[cfg(not(target_os = "linux"))] +fn deb_identity(_: &Path, _: &Path, _: &str) -> Result { + Err("DEB updates require Linux".to_owned()) +} + +#[cfg(windows)] +mod windows { + use super::*; + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::System::ApplicationInstallationAndServicing::*; + pub(super) fn wide(value: impl AsRef) -> Vec { + value.as_ref().encode_wide().chain(Some(0)).collect() + } + + pub(super) fn argument_path(path: &Path) -> Result { + let value = path + .to_str() + .ok_or("Windows Installer requires a Unicode installation path")?; + if let Some(unc) = value.strip_prefix("\\\\?\\UNC\\") { + return Ok(format!("\\\\{unc}")); + } + Ok(value.strip_prefix("\\\\?\\").unwrap_or(value).to_owned()) + } + + pub(super) fn install_root_argument(target: &Path) -> Result { + let path = argument_path(target)?; + if path.contains('"') || path.contains('\n') || path.contains('\r') { + return Err( + "Windows Installer requires an installation path without quotes".to_owned(), + ); + } + if path.is_empty() || path.len() > 32767 { + return Err("Windows Installer requires a bounded installation path".to_owned()); + } + Ok(format!("INSTALL_ROOT=\"{path}\"")) + } + + struct Handle(u32); + impl Drop for Handle { + fn drop(&mut self) { + unsafe { + MsiCloseHandle(self.0); + } + } + } + + pub(super) fn property(package: &Path, name: &str) -> Result { + let mut database = 0; + if unsafe { MsiOpenDatabaseW(wide(package).as_ptr(), std::ptr::null(), &mut database) } != 0 + { + return Err("Cannot read signed MSI database".to_owned()); + } + let database = Handle(database); + let mut view = 0; + let query = format!("SELECT `Value` FROM `Property` WHERE `Property`='{name}'"); + if unsafe { MsiDatabaseOpenViewW(database.0, wide(query).as_ptr(), &mut view) } != 0 { + return Err("Cannot query MSI identity".to_owned()); + } + let view = Handle(view); + if unsafe { MsiViewExecute(view.0, 0) } != 0 { + return Err("Cannot execute MSI identity query".to_owned()); + } + let mut record = 0; + if unsafe { MsiViewFetch(view.0, &mut record) } != 0 { + return Err(format!("MSI identity property {name} is missing")); + } + let record = Handle(record); + let mut buffer = vec![0u16; 32768]; + let mut size = buffer.len() as u32; + if unsafe { MsiRecordGetStringW(record.0, 1, buffer.as_mut_ptr(), &mut size) } != 0 { + return Err("Cannot read MSI identity property".to_owned()); + } + String::from_utf16(&buffer[..size as usize]).map_err(|error| error.to_string()) + } + + pub(super) fn registered(product: &str, property: &str) -> Result { + let mut buffer = vec![0u16; 32768]; + let mut size = buffer.len() as u32; + if unsafe { + MsiGetProductInfoW( + wide(product).as_ptr(), + wide(property).as_ptr(), + buffer.as_mut_ptr(), + &mut size, + ) + } != 0 + { + return Err("Cannot read registered MSI installation identity".to_owned()); + } + String::from_utf16(&buffer[..size as usize]).map_err(|error| error.to_string()) + } + + fn architecture(package: &Path) -> Result { + let mut summary = 0; + if unsafe { MsiGetSummaryInformationW(0, wide(package).as_ptr(), 0, &mut summary) } != 0 { + return Err("Cannot read MSI architecture".to_owned()); + } + let summary = Handle(summary); + let mut buffer = [0u16; 1024]; + let mut size = buffer.len() as u32; + let mut kind = 0; + let mut integer = 0; + let mut time = windows_sys::Win32::Foundation::FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + if unsafe { + MsiSummaryInfoGetPropertyW( + summary.0, + 7, + &mut kind, + &mut integer, + &mut time, + buffer.as_mut_ptr(), + &mut size, + ) + } != 0 + { + return Err("Cannot read MSI architecture template".to_owned()); + } + Ok(String::from_utf16(&buffer[..size as usize]) + .map_err(|error| error.to_string())? + .split(';') + .next() + .unwrap_or_default() + .to_owned()) + } + + pub(super) fn family(version: &str) -> Result<&'static str, String> { + let version = semver::Version::parse(version.trim_start_matches('v')) + .map_err(|error| error.to_string())?; + if version.pre.is_empty() { + Ok("{6E81F7AE-B19D-4E87-A94A-2B2F01EBF762}") + } else if version.pre.as_str().starts_with("nightly.") { + Ok("{9661F4F8-656C-4B64-9035-01B04F4822B1}") + } else if version.pre.as_str().starts_with("supporter.") { + Ok("{B3AF8A40-5F44-445A-AD99-CECE00593601}") + } else { + Err("Unsupported MSI release channel".to_owned()) + } + } + + pub(super) fn installed_at(target: &Path, upgrade: &str) -> Result, String> { + for index in 0..256 { + let mut product = [0u16; 39]; + let status = unsafe { + MsiEnumRelatedProductsW(wide(upgrade).as_ptr(), 0, index, product.as_mut_ptr()) + }; + if status == 259 { + return Ok(None); + } + if status != 0 { + return Err("Cannot enumerate registered MSI products".to_owned()); + } + let product = String::from_utf16(&product[..38]).map_err(|error| error.to_string())?; + let location = registered(&product, "InstallLocation")?; + if location.is_empty() { + return Err("An OpenNOW MSI registration has no installation location; repair it before updating".to_owned()); + } + let location = std::fs::canonicalize(location).map_err(|_| "An OpenNOW MSI registration points to an unavailable installation; repair it before updating")?; + if location == target { + return Ok(Some(product)); + } + } + Err("Too many registered MSI products".to_owned()) + } + + pub(super) fn identity( + package: &Path, + target: &Path, + version: &str, + ) -> Result { + let upgrade = family(version)?; + let parsed = semver::Version::parse(version.trim_start_matches('v')) + .map_err(|error| error.to_string())?; + let (name, expected_version) = if parsed.pre.is_empty() { + ( + "OpenNOW", + format!("{}.{}.{}", parsed.major, parsed.minor, parsed.patch), + ) + } else { + let parts: Vec<_> = parsed.pre.as_str().split('.').collect(); + if parts.len() != 3 { + return Err("Invalid MSI channel version".to_owned()); + } + let run: u32 = parts[1].parse().map_err(|_| "Invalid MSI release run")?; + let attempt: u32 = parts[2] + .parse() + .map_err(|_| "Invalid MSI release attempt")?; + if !(1..=65535).contains(&run) || !(1..=65535).contains(&attempt) { + return Err("MSI release sequence is outside its supported range".to_owned()); + } + ( + if parts[0] == "nightly" { + "OpenNOW Nightly" + } else { + "OpenNOW Supporter" + }, + format!("{}.{}.{}", run / 256, run % 256, attempt), + ) + }; + if !property(package, "UpgradeCode")?.eq_ignore_ascii_case(upgrade) + || property(package, "ProductName")? != name + || property(package, "Manufacturer")? != "OpenCloudGaming" + { + return Err("MSI product family or channel does not match OpenNOW".to_owned()); + } + let arch = architecture(package)?; + let expected_arch = if cfg!(target_arch = "aarch64") { + "Arm64" + } else { + "x64" + }; + if !arch.eq_ignore_ascii_case(expected_arch) { + return Err("MSI architecture does not match this installation".to_owned()); + } + let installed_product = installed_at(target, upgrade)? + .ok_or("No matching Windows Installer registration owns this application")?; + let cached = registered(&installed_product, "LocalPackage")?; + if architecture(Path::new(&cached))? != arch { + return Err("MSI update architecture differs from the installed product".to_owned()); + } + let numeric = property(package, "ProductVersion")?; + if numeric != expected_version { + return Err("MSI product version does not match the signed release version".to_owned()); + } + Ok(Identity { + package: property(package, "ProductCode")?, + version: numeric, + architecture: arch, + installed_product, + }) + } +} + +#[cfg(windows)] +fn msi_identity(package: &Path, target: &Path, version: &str) -> Result { + windows::identity(package, target, version) +} +#[cfg(not(windows))] +fn msi_identity(_: &Path, _: &Path, _: &str) -> Result { + Err("MSI updates require Windows".to_owned()) +} + +pub(super) fn windows_managed(target: &Path) -> Result { + #[cfg(windows)] + { + for version in ["1.0.0", "1.0.0-nightly.1.1", "1.0.0-supporter.1.1"] { + if windows::installed_at(target, windows::family(version)?)?.is_some() { + return Ok(true); + } + } + } + let _ = target; + Ok(false) +} + +#[cfg(not(windows))] +fn msi_install_root_argument(target: &Path) -> Result { + let value = target + .to_str() + .ok_or("Windows Installer requires a Unicode installation path")?; + let path = if let Some(unc) = value.strip_prefix("\\\\?\\UNC\\") { + format!("\\\\{unc}") + } else { + value.strip_prefix("\\\\?\\").unwrap_or(value).to_owned() + }; + if path.contains('"') || path.contains('\n') || path.contains('\r') { + return Err("Windows Installer requires an installation path without quotes".to_owned()); + } + if path.is_empty() || path.len() > 32767 { + return Err("Windows Installer requires a bounded installation path".to_owned()); + } + Ok(format!("INSTALL_ROOT=\"{path}\"")) +} + +pub(super) fn install(plan: &Plan, directory: &Path) -> Result<(), String> { + let identity = plan + .managed_identity + .as_ref() + .ok_or("Missing prepared managed-package identity")?; + if matches!( + plan.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) { + super::recovery::record_install_boot(directory)?; + } + let mut command = match plan.kind { + InstallKind::DebianPackage => { + let mut command = Command::new("/usr/bin/pkexec"); + command + .args(["/usr/bin/dpkg", "--install"]) + .arg(&plan.package); + command + } + InstallKind::WindowsMsi => { + // apply() refuses this kind before install(), so in-app updates do not + // reach msiexec. The command stays while the WiX publish job still ships + // an MSI for manual install. + let system = + std::env::var_os("SystemRoot").ok_or("Windows system directory is unavailable")?; + let mut command = Command::new(Path::new(&system).join("System32/msiexec.exe")); + command.arg("/i"); + #[cfg(windows)] + { + let package = windows::argument_path(&plan.package)?; + if package.contains('"') { + return Err( + "Windows Installer requires a package path without quotes".to_owned() + ); + } + command.arg(package); + } + #[cfg(not(windows))] + command.arg(&plan.package); + command.args(["/passive", "/norestart", "REBOOT=ReallySuppress"]); + // msiexec parses the raw command line for PROPERTY=value tokens. A Rust + // Command::arg containing spaces would be quoted as a whole + // ("INSTALL_ROOT=C:\Program Files\..."), which msiexec rejects with 1639 + // and a help dialog. Emit INSTALL_ROOT="..." with quotes only around the + // value, matching the installer contract test. + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.raw_arg(windows::install_root_argument(&plan.target)?); + let log = windows::argument_path(&directory.join("msiexec.log"))?; + if log.contains('"') { + return Err("Windows Installer requires a log path without quotes".to_owned()); + } + command.arg("/l*v"); + command.arg(log); + } + #[cfg(not(windows))] + { + command.arg(msi_install_root_argument(&plan.target)?); + command.arg("/l*v"); + command.arg(directory.join("msiexec.log")); + } + command + } + _ => return Err("Not a managed package update".to_owned()), + }; + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| format!("Cannot start native package manager: {error}"))?; + let started = Instant::now(); + let code = loop { + if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { + break status.code().unwrap_or(-1); + } + if started.elapsed() > Duration::from_secs(15 * 60) { + write_outcome( + directory, + plan, + OutcomeStatus::ManagedPending, + "Native package manager is still running; installation completion is not confirmed. Do not start another update.", + None, + super::ProcessIdentity::capture(child.id()).ok(), + )?; + return Err("Native installer exceeded the monitoring deadline; it was not killed or reported as successful".to_owned()); + } + std::thread::sleep(Duration::from_millis(250)); + }; + if plan.kind == InstallKind::WindowsMsi && matches!(code, 3010 | 1641) { + write_outcome( + directory, + plan, + OutcomeStatus::RebootRequired, + "Windows Installer requires a system restart; application startup is not yet confirmed", + Some(identity.version.clone()), + None, + )?; + return Ok(()); + } + if code != 0 { + let mut message = installer_failure_message(plan.kind, code); + if plan.kind == InstallKind::WindowsMsi { + let log = directory.join("msiexec.log"); + if log.is_file() { + message.push_str(&format!("; installer log: {}", log.display())); + } + } + if prepare(plan.kind, &plan.package, &plan.target, &plan.version).is_ok() + && super::canonical_file(&plan.application_executable).is_ok() + { + match super::restart_previous(plan) { + Ok(_) => message.push_str("; the surviving registered application was restarted"), + Err(error) => message.push_str(&format!( + "; the surviving application could not restart: {error}" + )), + } + } else { + message + .push_str("; a usable registered installation could not be confirmed for restart"); + } + write_outcome(directory, plan, OutcomeStatus::Failed, &message, None, None)?; + return Err(message); + } + verify_installed(plan.kind, identity, &plan.target)?; + if let Err(error) = super::restart_and_acknowledge(plan, directory, Duration::from_secs(90)) { + let message = format!( + "Native package manager installed {}; application restart failed: {error}. No filesystem rollback was attempted for a managed installation.", + identity.version + ); + write_outcome( + directory, + plan, + OutcomeStatus::Failed, + &message, + Some(identity.version.clone()), + None, + )?; + return Err(message); + } + write_outcome( + directory, + plan, + OutcomeStatus::Completed, + "Native package manager confirmed installation and the updated application acknowledged healthy startup", + Some(plan.version.clone()), + None, + )?; + super::cleanup_completed(plan, directory) +} + +fn installer_failure_message(kind: InstallKind, code: i32) -> String { + let reason = match (kind, code) { + (InstallKind::DebianPackage, 126) => { + "Update authorization was cancelled. Try installing the update again and approve the authorization prompt" + } + (InstallKind::DebianPackage, 127) => { + "Update authorization failed. Ensure a PolicyKit authentication agent is running, then try again and approve the authorization prompt" + } + (InstallKind::WindowsMsi, 1602) => { + "Windows Installer was cancelled. Try installing the update again and complete the installer prompts" + } + (InstallKind::WindowsMsi, 1639) => { + "Windows Installer rejected the update command line. Try installing the update again; the installer log beside the update outcome has details" + } + _ => "Native package manager failed", + }; + format!("{reason} (exit code {code}); update completion is not confirmed") +} + +pub(super) fn verify_installed( + kind: InstallKind, + identity: &Identity, + target: &Path, +) -> Result<(), String> { + let _ = target; + match kind { + InstallKind::DebianPackage => { + let value = super::command_output( + Command::new("/usr/bin/dpkg-query").args([ + "--show", + "--showformat=${db:Status-Status}\t${Version}\t${Architecture}", + &identity.package, + ]), + Duration::from_secs(5), + )?; + if !value.status.success() + || value.stdout + != format!("installed\t{}\t{}", identity.version, identity.architecture) + .as_bytes() + { + return Err( + "The OpenNOW DEB is not configured at the expected version and architecture." + .to_owned(), + ); + } + #[cfg(target_os = "linux")] + { + let files = deb_file_owners(target)?; + if !deb_owns_file(&files, target) { + return Err("Installed OpenNOW DEB does not own the application".to_owned()); + } + } + Ok(()) + } + InstallKind::WindowsMsi => { + #[cfg(windows)] + { + if windows::registered(&identity.package, "VersionString")? != identity.version { + return Err( + "Windows Installer registration does not confirm the installed version" + .to_owned(), + ); + } + if std::fs::canonicalize(windows::registered(&identity.package, "InstallLocation")?) + .map_err(|error| error.to_string())? + != target + { + return Err( + "Windows Installer registered an unexpected installation location" + .to_owned(), + ); + } + Ok(()) + } + #[cfg(not(windows))] + Err("MSI updates require Windows".to_owned()) + } + _ => Err("Not a managed package".to_owned()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installer_failures_explain_authorization_and_cancellation_without_claiming_success() { + for (kind, code, explanation) in [ + ( + InstallKind::DebianPackage, + 126, + "authorization was cancelled", + ), + ( + InstallKind::DebianPackage, + 127, + "PolicyKit authentication agent", + ), + ( + InstallKind::WindowsMsi, + 1602, + "Windows Installer was cancelled", + ), + ( + InstallKind::WindowsMsi, + 1639, + "rejected the update command line", + ), + ] { + let message = installer_failure_message(kind, code); + assert!(message.contains(explanation), "{message}"); + assert!(message.contains(&format!("exit code {code}")), "{message}"); + assert!(message.ends_with("update completion is not confirmed")); + } + } + + #[test] + fn installer_failure_codes_are_interpreted_only_for_their_package_manager() { + for (kind, code) in [ + (InstallKind::DebianPackage, 1602), + (InstallKind::DebianPackage, 1639), + (InstallKind::WindowsMsi, 126), + (InstallKind::WindowsMsi, 127), + (InstallKind::DebianPackage, 1), + (InstallKind::WindowsMsi, 1603), + ] { + assert_eq!( + installer_failure_message(kind, code), + format!( + "Native package manager failed (exit code {code}); update completion is not confirmed" + ) + ); + } + } + + #[test] + fn msi_install_root_quotes_only_the_value_for_msiexec() { + #[cfg(windows)] + let argument = + windows::install_root_argument(Path::new(r"C:\Program Files\OpenNOW Nightly")).unwrap(); + #[cfg(not(windows))] + let argument = + msi_install_root_argument(Path::new(r"C:\Program Files\OpenNOW Nightly")).unwrap(); + assert_eq!( + argument, + r#"INSTALL_ROOT="C:\Program Files\OpenNOW Nightly""# + ); + #[cfg(windows)] + let stripped = windows::install_root_argument(Path::new(r"\\?\C:\OpenNOW")).unwrap(); + #[cfg(not(windows))] + let stripped = msi_install_root_argument(Path::new(r"\\?\C:\OpenNOW")).unwrap(); + assert_eq!(stripped, r#"INSTALL_ROOT="C:\OpenNOW""#); + #[cfg(windows)] + assert!(windows::install_root_argument(Path::new("C:\\evil\"quote")).is_err()); + #[cfg(not(windows))] + assert!(msi_install_root_argument(Path::new("C:\\evil\"quote")).is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + fn deb_ownership_matches_canonical_directories_but_rejects_file_symlinks() { + let directory = tempfile::tempdir().unwrap(); + let actual = directory.path().join("usr/bin"); + std::fs::create_dir_all(&actual).unwrap(); + let target = actual.join("opennow-qt"); + std::fs::write(&target, b"application").unwrap(); + let alias = directory.path().join("bin"); + std::os::unix::fs::symlink(&actual, &alias).unwrap(); + let registered = alias.join("opennow-qt"); + let owners = format!("opennow: {}", registered.display()); + assert!(deb_owns_file(&owners, &target)); + let unrelated = directory.path().join("unrelated"); + std::fs::write(&unrelated, b"other application").unwrap(); + assert!(!deb_owns_file(&owners, &unrelated)); + let link = actual.join("opennow-link"); + std::os::unix::fs::symlink(&unrelated, &link).unwrap(); + assert!(!deb_owns_file( + &format!("opennow: {}", link.display()), + &unrelated + )); + assert!(!deb_owns_file( + &format!("opennow: {}", unrelated.display()), + &link + )); + assert!(!deb_owns_file( + &format!("other-package: {}", target.display()), + &target + )); + assert!(!deb_owns_file("opennow: relative/path", &target)); + } +} diff --git a/external/opennow/native/opennow-core/src/update_apply/mod.rs b/external/opennow/native/opennow-core/src/update_apply/mod.rs new file mode 100644 index 000000000..0cc7af837 --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/mod.rs @@ -0,0 +1,1569 @@ +mod archive; +mod bundle; +mod managed; +mod process; +mod recovery; +mod security; +pub mod verification; + +use fs2::FileExt; +pub use process::ProcessIdentity; +pub use recovery::{ManagedRecovery, recover_managed_update}; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +struct TransactionLock(File); + +impl TransactionLock { + fn acquire(file: File) -> std::io::Result { + file.try_lock_exclusive()?; + Ok(Self(file)) + } +} + +impl Drop for TransactionLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum InstallKind { + AppImage, + WindowsPortable, + MacBundle, + WindowsMsi, + DebianPackage, +} + +#[derive(Debug)] +pub(super) enum RestartFailure { + RollbackSafe(String), + ProcessesMayBeRunning(String), +} + +impl std::fmt::Display for RestartFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RollbackSafe(message) | Self::ProcessesMayBeRunning(message) => { + formatter.write_str(message) + } + } + } +} + +#[derive(Clone, Debug)] +pub struct PrepareRequest { + pub package: PathBuf, + pub expected_version: String, + pub application_executable: PathBuf, + pub application_pid: u32, + pub core_pid: u32, + pub kind: InstallKind, + pub data_dir: PathBuf, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PreparedUpdate { + pub plan_path: PathBuf, + pub outcome_path: PathBuf, + pub version: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Plan { + schema_version: u32, + version: String, + kind: InstallKind, + package: PathBuf, + target: PathBuf, + application_executable: PathBuf, + data_dir: PathBuf, + processes: Vec, + nonce: String, + managed_identity: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum OutcomeStatus { + Prepared, + WaitingForExit, + BackingUp, + Installing, + AwaitingStartup, + Completed, + RolledBack, + Failed, + ManagedPending, + RebootRequired, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateOutcome { + pub schema_version: u32, + pub version: String, + pub status: OutcomeStatus, + pub message: String, + pub installed_version: Option, + pub restarted_process: Option, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Acknowledgement { + nonce: String, + version: String, + application: ProcessIdentity, +} + +pub const WINDOWS_INSTALLER_REPLACEMENT_MESSAGE: &str = "This OpenNOW installation is registered with Windows Installer. Replace it once with setup.exe. In-app updates do not run Windows Installer."; + +pub fn windows_installer_replacement_message() -> Result, String> { + #[cfg(windows)] + { + let Some(executable) = std::env::var_os("OPENNOW_APP_EXECUTABLE") else { + return Ok(None); + }; + let Ok(executable) = canonical_file(Path::new(&executable)) else { + return Ok(None); + }; + let Ok(root) = installation_target(InstallKind::WindowsMsi, &executable) else { + return Ok(None); + }; + if managed::windows_managed(&root)? { + return Ok(Some(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE)); + } + } + Ok(None) +} + +#[cfg_attr(not(any(windows, test)), allow(dead_code))] +pub(super) fn windows_update_package_extension( + msi_registered: bool, +) -> Result<&'static str, String> { + if msi_registered { + Err(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE.to_owned()) + } else { + Ok("zip") + } +} + +pub fn external_update_message() -> Option<&'static str> { + if cfg!(target_os = "linux") + && is_flatpak_installation( + std::env::var_os("FLATPAK_ID").as_deref(), + Path::new("/.flatpak-info"), + ) + { + Some( + "OpenNOW is managed by Flatpak. Update it through your software manager or run `flatpak update io.github.opencloudgaming.OpenNOW` on the host, then restart OpenNOW.", + ) + } else { + None + } +} + +fn is_flatpak_installation(flatpak_id: Option<&std::ffi::OsStr>, info_path: &Path) -> bool { + flatpak_id.is_some_and(|id| !id.is_empty()) || info_path.exists() +} + +fn require_native_updates() -> Result<(), String> { + match external_update_message() { + Some(message) => Err(message.to_owned()), + None => Ok(()), + } +} + +pub fn detect_install_kind(application: &Path, package: &Path) -> Result { + require_native_updates()?; + let extension = package + .extension() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + match extension.as_str() { + "appimage" if cfg!(target_os = "linux") => Ok(InstallKind::AppImage), + "deb" if cfg!(target_os = "linux") => Ok(InstallKind::DebianPackage), + "msi" if cfg!(windows) => Ok(InstallKind::WindowsMsi), + "zip" if cfg!(windows) => Ok(InstallKind::WindowsPortable), + "zip" | "dmg" + if cfg!(target_os = "macos") + && application + .ancestors() + .any(|path| path.extension().is_some_and(|extension| extension == "app")) => + { + Ok(InstallKind::MacBundle) + } + _ => Err( + "No safe native update mechanism exists for this package and installation".to_owned(), + ), + } +} + +pub fn compatible_package_extension() -> Result<&'static str, String> { + require_native_updates()?; + #[cfg(target_os = "linux")] + { + Ok( + if std::env::var_os("APPIMAGE").is_some_and(|path| !path.is_empty()) { + "appimage" + } else { + "deb" + }, + ) + } + #[cfg(target_os = "macos")] + { + Ok("dmg") + } + #[cfg(windows)] + { + let executable = std::env::var_os("OPENNOW_APP_EXECUTABLE") + .ok_or("Missing trusted application executable")?; + let executable = canonical_file(Path::new(&executable))?; + let root = installation_target(InstallKind::WindowsMsi, &executable)?; + windows_update_package_extension(managed::windows_managed(&root)?) + } +} + +pub fn prepare_update(request: PrepareRequest) -> Result { + require_native_updates()?; + if request.kind == InstallKind::WindowsMsi { + return Err(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE.to_owned()); + } + #[cfg(target_os = "linux")] + validate_linux_install_kind(request.kind, std::env::var_os("APPIMAGE").as_deref())?; + let package = canonical_file(&request.package)?; + let manifest = read_manifest(&package)?; + if manifest.version.trim_start_matches('v') != request.expected_version.trim_start_matches('v') + { + return Err("Signed update version does not match the selected release".to_owned()); + } + verification::verify_package(&package, &manifest)?; + let application = canonical_file(&request.application_executable)?; + if detect_install_kind(&application, &package)? != request.kind { + return Err("Update package does not match the installation type".to_owned()); + } + let app_process = ProcessIdentity::capture(request.application_pid)?; + app_process.matches_executable(&application)?; + let core_process = ProcessIdentity::capture(request.core_pid)?; + core_process + .matches_executable(&std::env::current_exe().map_err(|error| error.to_string())?)?; + let target = installation_target(request.kind, &application)?; + if request.kind == InstallKind::WindowsPortable && managed::windows_managed(&target)? { + return Err(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE.to_owned()); + } + let data_dir = fs::canonicalize(&request.data_dir).map_err(|error| error.to_string())?; + if (data_dir.starts_with(&target) && request.kind != InstallKind::WindowsPortable) + || target.starts_with(&data_dir) + { + return Err( + "Application settings must be outside the installation being replaced".to_owned(), + ); + } + let parent = if matches!( + request.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) { + data_dir.join("updates") + } else { + target + .parent() + .ok_or("Installation has no parent")? + .to_path_buf() + }; + fs::create_dir_all(&parent).map_err(|error| error.to_string())?; + let directory = parent.join(format!(".opennow-update-{:032x}", rand::random::())); + security::create_private_directory(&directory) + .map_err(|error| format!("Cannot write beside the installed application: {error}"))?; + let result = (|| { + let private_package = directory.join(&manifest.asset); + copy_synced(&package, &private_package)?; + copy_synced(&manifest_path(&package)?, &manifest_path(&private_package)?)?; + verification::verify_package(&private_package, &manifest)?; + let managed_identity = + managed::prepare(request.kind, &private_package, &target, &manifest.version)?; + let plan = Plan { + schema_version: 1, + version: manifest.version.trim_start_matches('v').to_owned(), + kind: request.kind, + package: private_package, + target, + application_executable: application, + data_dir, + processes: vec![app_process, core_process], + nonce: format!("{:064x}", rand::random::()), + managed_identity, + }; + let preflight = prepare_payload(&plan, &directory.join("preflight"))?; + preserve_portable_data(&plan, &preflight)?; + remove_path(&directory.join("preflight"))?; + let helper = std::env::current_exe() + .map_err(|error| error.to_string())? + .parent() + .ok_or("Core has no executable directory")? + .join(if cfg!(windows) { + "opennow-update-helper.exe" + } else { + "opennow-update-helper" + }); + copy_synced( + &canonical_file(&helper)?, + &directory.join(helper.file_name().ok_or("Invalid helper name")?), + )?; + make_executable(&directory.join(helper.file_name().ok_or("Invalid helper name")?))?; + #[cfg(windows)] + for name in ["vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"] { + let runtime = helper + .parent() + .ok_or("Missing helper directory")? + .join(name); + if runtime.is_file() { + copy_synced(&canonical_file(&runtime)?, &directory.join(name))?; + } + } + let plan_path = directory.join("plan.json"); + atomic_json(&plan_path, &plan)?; + write_outcome( + &directory, + &plan, + OutcomeStatus::Prepared, + "Update verified and prepared; application is still running", + None, + None, + )?; + Ok(PreparedUpdate { + plan_path, + outcome_path: directory.join("outcome.json"), + version: plan.version, + }) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&directory); + } + result +} + +#[cfg(target_os = "linux")] +fn validate_linux_install_kind( + kind: InstallKind, + appimage: Option<&std::ffi::OsStr>, +) -> Result<(), String> { + let is_appimage = appimage.is_some_and(|path| !path.is_empty()); + if (kind == InstallKind::AppImage && is_appimage) + || (kind == InstallKind::DebianPackage && !is_appimage) + { + Ok(()) + } else { + Err("Update package does not match the running Linux installation format".to_owned()) + } +} + +pub fn launch_prepared_update(prepared: &PreparedUpdate) -> Result { + require_native_updates()?; + let directory = prepared + .plan_path + .parent() + .ok_or("Update plan has no directory")?; + let helper = directory.join(if cfg!(windows) { + "opennow-update-helper.exe" + } else { + "opennow-update-helper" + }); + let mut command = Command::new(helper); + command + .current_dir(directory) + .arg("--apply") + .arg(&prepared.plan_path) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x00000008 | 0x00000200); + } + let plan: Plan = read_json(&prepared.plan_path, 64 * 1024)?; + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + let message = format!("Cannot start update helper: {error}"); + write_outcome( + directory, + &plan, + OutcomeStatus::Failed, + &message, + None, + None, + )?; + return Err(message); + } + }; + let result = (|| { + let start = Instant::now(); + loop { + if child + .try_wait() + .map_err(|error| error.to_string())? + .is_some() + { + return Err(read_outcome(&prepared.outcome_path)? + .map(|outcome| outcome.message) + .unwrap_or_else(|| "Update helper stopped before it was ready".to_owned())); + } + if read_outcome(&prepared.outcome_path)? + .is_some_and(|outcome| outcome.status == OutcomeStatus::WaitingForExit) + && helper_is_running(prepared)? + { + return Ok(child.id()); + } + if start.elapsed() > Duration::from_secs(300) { + return Err( + "Update helper preparation timed out; the application remains running" + .to_owned(), + ); + } + std::thread::sleep(Duration::from_millis(100)); + } + })(); + if let Err(error) = &result { + let _ = child.kill(); + let _ = child.wait(); + write_outcome(directory, &plan, OutcomeStatus::Failed, error, None, None)?; + } + result +} + +pub fn helper_is_running(prepared: &PreparedUpdate) -> Result { + let path = prepared + .plan_path + .parent() + .ok_or("Invalid update plan directory")? + .join("apply.lock"); + if !path.exists() { + return Ok(false); + } + let lock = OpenOptions::new() + .write(true) + .open(path) + .map_err(|error| error.to_string())?; + match TransactionLock::acquire(lock) { + Ok(_lock) => Ok(false), + Err(error) if error.raw_os_error() == fs2::lock_contended_error().raw_os_error() => { + Ok(true) + } + Err(error) => Err(error.to_string()), + } +} + +pub fn read_outcome(path: &Path) -> Result, String> { + if !path.exists() { + return Ok(None); + } + let outcome: UpdateOutcome = read_json(path, 64 * 1024)?; + if outcome.schema_version != 1 { + return Err("Unsupported update outcome schema".to_owned()); + } + Ok(Some(outcome)) +} + +pub fn acknowledge_startup_from_env(application_version: &str) -> Result { + let Some(plan_path) = std::env::var_os("OPENNOW_UPDATE_PLAN") else { + return Ok(false); + }; + let plan_path = PathBuf::from(plan_path); + let plan: Plan = read_json(&plan_path, 64 * 1024)?; + let nonce = + std::env::var("OPENNOW_UPDATE_NONCE").map_err(|_| "Missing update startup nonce")?; + if plan.schema_version != 1 + || plan.version != application_version.trim_start_matches('v') + || nonce != plan.nonce + { + return Err( + "Updated application startup identity does not match the update plan".to_owned(), + ); + } + let pid = std::env::var("OPENNOW_APP_PID") + .map_err(|_| "Missing trusted application PID")? + .parse() + .map_err(|_| "Invalid trusted application PID")?; + let application = ProcessIdentity::capture(pid)?; + if plan.kind == InstallKind::AppImage { + let image = std::env::var_os("APPIMAGE") + .ok_or("Updated process is not running from an AppImage")?; + if canonical_file(Path::new(&image))? != plan.target { + return Err("Updated AppImage does not match the installation target".to_owned()); + } + } else { + application.matches_executable(&plan.application_executable)?; + } + atomic_json( + &plan_path + .parent() + .ok_or("Invalid update plan path")? + .join("ack.json"), + &Acknowledgement { + nonce, + version: plan.version, + application, + }, + )?; + Ok(true) +} + +pub fn run_helper(plan_path: &Path) -> Result<(), String> { + require_native_updates()?; + let plan_path = canonical_file(plan_path)?; + let directory = plan_path.parent().ok_or("Invalid update plan path")?; + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(directory.join("apply.lock")) + .map_err(|error| error.to_string())?; + let _lock = TransactionLock::acquire(lock) + .map_err(|_| "Another helper owns this update transaction")?; + let plan: Plan = read_json(&plan_path, 64 * 1024)?; + if read_outcome(&directory.join("outcome.json"))?.is_some_and(|outcome| { + matches!( + outcome.status, + OutcomeStatus::Completed | OutcomeStatus::RolledBack | OutcomeStatus::RebootRequired + ) + }) { + return Ok(()); + } + let result = validate_plan(&plan, directory).and_then(|_| apply(&plan, directory)); + if let Err(error) = &result { + let outcome = read_outcome(&directory.join("outcome.json"))?; + if outcome.as_ref().is_none_or(|outcome| { + !matches!( + outcome.status, + OutcomeStatus::RolledBack + | OutcomeStatus::Failed + | OutcomeStatus::ManagedPending + | OutcomeStatus::RebootRequired + ) + }) { + write_outcome( + directory, + &plan, + OutcomeStatus::Failed, + error, + outcome + .as_ref() + .and_then(|outcome| outcome.installed_version.clone()), + outcome.and_then(|outcome| outcome.restarted_process), + )?; + } + } + result +} + +fn validate_plan(plan: &Plan, directory: &Path) -> Result<(), String> { + if plan.schema_version != 1 + || plan.nonce.len() != 64 + || !plan.nonce.bytes().all(|value| value.is_ascii_hexdigit()) + || plan.processes.len() != 2 + { + return Err("Invalid update plan schema or process identities".to_owned()); + } + if !plan.target.is_absolute() + || !plan.application_executable.is_absolute() + || !plan.data_dir.is_absolute() + || plan.package.parent() != Some(directory) + || (plan.kind != InstallKind::AppImage + && installation_target(plan.kind, &plan.application_executable)? != plan.target) + || (plan.data_dir.starts_with(&plan.target) && plan.kind != InstallKind::WindowsPortable) + || plan.target.starts_with(&plan.data_dir) + { + return Err("Invalid update installation paths".to_owned()); + } + if !matches!( + plan.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) && plan.target.parent() != directory.parent() + { + return Err("Update payload is not beside its installation".to_owned()); + } + if plan.processes[0].executable != plan.application_executable { + return Err("Update process identity does not match the application".to_owned()); + } + let manifest = read_manifest(&plan.package)?; + if manifest.version.trim_start_matches('v') != plan.version + || detect_install_kind(&plan.application_executable, &plan.package)? != plan.kind + { + return Err("Update plan does not match the signed manifest".to_owned()); + } + verification::verify_package(&plan.package, &manifest) +} + +fn apply(plan: &Plan, directory: &Path) -> Result<(), String> { + if plan.kind == InstallKind::WindowsMsi { + return Err(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE.to_owned()); + } + let previous = + read_outcome(&directory.join("outcome.json"))?.ok_or("Missing prepared update outcome")?; + if previous.status != OutcomeStatus::Prepared { + if matches!( + previous.status, + OutcomeStatus::BackingUp + | OutcomeStatus::Installing + | OutcomeStatus::AwaitingStartup + | OutcomeStatus::Failed + ) && !matches!( + plan.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) { + if previous.restarted_process.is_none() + && (previous.status == OutcomeStatus::AwaitingStartup + || (previous.status == OutcomeStatus::Failed + && directory.join("previous").exists())) + { + return Err("Interrupted startup has no confirmed process identity; automatic rollback is unsafe".to_owned()); + } + if previous + .restarted_process + .as_ref() + .map(process::owned_tree_is_running) + .transpose()? + .unwrap_or(false) + { + return Err( + "Interrupted update still has a running application; close it before recovery" + .to_owned(), + ); + } + rollback(plan, directory)?; + let message = match restart_previous(plan) { + Ok(_) => "Recovered an interrupted replacement; previous installation restored and restart requested".to_owned(), + Err(error) => format!("Recovered an interrupted replacement; previous installation restored but restart failed: {error}"), + }; + write_outcome( + directory, + plan, + OutcomeStatus::RolledBack, + &message, + None, + None, + )?; + return Ok(()); + } + return Err("Update transaction has already started or completed".to_owned()); + } + let payload = prepare_payload(plan, &directory.join("payload"))?; + let preserved = preserve_portable_data(plan, &payload)?; + managed::verify_identity( + plan.kind, + &plan.package, + &plan.target, + &plan.version, + plan.managed_identity.as_ref(), + )?; + write_outcome( + directory, + plan, + OutcomeStatus::WaitingForExit, + "Helper independently verified the signed update and is waiting for application shutdown", + None, + None, + )?; + process::wait_for_exit(&plan.processes, Duration::from_secs(90))?; + let refresh = (|| { + for (_, destination) in preserved { + remove_path(&destination)?; + } + preserve_portable_data(plan, &payload).map(|_| ()) + })(); + if let Err(error) = refresh { + let _ = restart_previous(plan); + return Err(error); + } + if matches!( + plan.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) { + write_outcome( + directory, + plan, + OutcomeStatus::Installing, + "Native package manager is installing the verified update", + None, + None, + )?; + return managed::install(plan, directory); + } + replace_and_restart(plan, directory, &payload, Duration::from_secs(90)) +} + +fn replace_and_restart( + plan: &Plan, + directory: &Path, + payload: &Path, + startup_timeout: Duration, +) -> Result<(), String> { + write_outcome( + directory, + plan, + OutcomeStatus::BackingUp, + "Preserving the previous installation", + None, + None, + )?; + let installed = rename_synced(&plan.target, &directory.join("previous")).and_then(|_| { + write_outcome( + directory, + plan, + OutcomeStatus::Installing, + "Replacing the complete installation", + None, + None, + )?; + rename_synced(payload, &plan.target)?; + if plan.kind == InstallKind::MacBundle { + bundle::verify_bundle(&plan.target, &directory.join("previous"))?; + } + Ok(()) + }); + let result = installed + .map_err(RestartFailure::RollbackSafe) + .and_then(|_| restart_and_acknowledge(plan, directory, startup_timeout)); + if let Err(error) = result { + return recover_replacement(plan, directory, error); + } + write_outcome( + directory, + plan, + OutcomeStatus::Completed, + "Updated application acknowledged healthy startup", + Some(plan.version.clone()), + None, + )?; + cleanup_completed(plan, directory)?; + Ok(()) +} + +fn recover_replacement( + plan: &Plan, + directory: &Path, + failure: RestartFailure, +) -> Result<(), String> { + let error = match failure { + RestartFailure::ProcessesMayBeRunning(error) => return Err(error), + RestartFailure::RollbackSafe(error) => error, + }; + if let Err(rollback_error) = rollback(plan, directory) { + let message = format!( + "{error}; rollback failed: {rollback_error}. Previous installation remains in {}", + directory.join("previous").display() + ); + write_outcome(directory, plan, OutcomeStatus::Failed, &message, None, None)?; + return Err(message); + } + let restart = restart_previous(plan); + let message = format!( + "{error}; previous installation restored{}", + restart + .err() + .map(|error| format!(" but restart failed: {error}")) + .unwrap_or_default() + ); + write_outcome( + directory, + plan, + OutcomeStatus::RolledBack, + &message, + None, + None, + )?; + Err(message) +} + +fn cleanup_completed(plan: &Plan, directory: &Path) -> Result<(), String> { + for path in [ + directory.join("previous"), + directory.join("payload"), + plan.package.clone(), + manifest_path(&plan.package)?, + ] { + if let Err(error) = remove_path(&path) { + write_outcome( + directory, + plan, + OutcomeStatus::Completed, + &format!( + "Updated application acknowledged healthy startup; cleanup of {} was deferred: {error}", + path.display() + ), + Some(plan.version.clone()), + None, + )?; + return Ok(()); + } + } + Ok(()) +} + +fn rollback(plan: &Plan, directory: &Path) -> Result<(), String> { + let backup = directory.join("previous"); + if !backup.exists() { + if plan.target.exists() { + return Ok(()); + } + return Err("Previous installation is missing".to_owned()); + } + if plan.target.exists() { + rename_synced(&plan.target, &directory.join("failed"))?; + } + rename_synced(&backup, &plan.target) +} + +fn restart_previous(plan: &Plan) -> Result { + Command::new(restart_executable(plan)) + .current_dir( + restart_executable(plan) + .parent() + .ok_or("Application executable has no directory")?, + ) + .env("OPENNOW_DATA_DIR", &plan.data_dir) + .env_remove("APPIMAGE") + .env_remove("APPDIR") + .env_remove("ARGV0") + .env_remove("OPENNOW_UPDATE_PLAN") + .env_remove("OPENNOW_UPDATE_NONCE") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| error.to_string()) +} + +fn restart_executable(plan: &Plan) -> &Path { + if plan.kind == InstallKind::AppImage { + &plan.target + } else { + &plan.application_executable + } +} + +fn restart_and_acknowledge( + plan: &Plan, + directory: &Path, + timeout: Duration, +) -> Result<(), RestartFailure> { + let mut command = Command::new(restart_executable(plan)); + command + .env("OPENNOW_DATA_DIR", &plan.data_dir) + .current_dir(restart_executable(plan).parent().ok_or_else(|| { + RestartFailure::RollbackSafe("Application executable has no directory".to_owned()) + })?) + .env_remove("APPIMAGE") + .env_remove("APPDIR") + .env_remove("ARGV0") + .env("OPENNOW_UPDATE_PLAN", directory.join("plan.json")) + .env("OPENNOW_UPDATE_NONCE", &plan.nonce) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + write_outcome( + directory, + plan, + OutcomeStatus::AwaitingStartup, + "Preparing to start the updated application", + Some(plan.version.clone()), + None, + ) + .map_err(RestartFailure::RollbackSafe)?; + let mut application = process::OwnedApplication::start(&mut command)?; + let result = (|| { + write_outcome( + directory, + plan, + OutcomeStatus::AwaitingStartup, + "Updated application started; waiting for healthy UI and core acknowledgement", + Some(plan.version.clone()), + Some(application.identity.clone()), + )?; + let started = Instant::now(); + loop { + if application + .child + .try_wait() + .map_err(|error| error.to_string())? + .is_some() + { + break Err("Updated application exited before healthy startup".to_owned()); + } + if directory.join("ack.json").exists() { + match read_json::(&directory.join("ack.json"), 64 * 1024) { + Ok(ack) + if ack.nonce == plan.nonce + && ack.version == plan.version + && application.owns(&ack.application)? + && (plan.kind == InstallKind::AppImage + || ack.application.executable == plan.application_executable) + && ack.application.is_running()? => + { + break Ok(()); + } + _ => { + break Err( + "Updated application returned an invalid startup acknowledgement" + .to_owned(), + ); + } + } + } + if started.elapsed() >= timeout { + break Err( + "Updated application did not acknowledge healthy startup before the deadline" + .to_owned(), + ); + } + std::thread::sleep(Duration::from_millis(100)); + } + })(); + let Err(error) = result else { + return Ok(()); + }; + let stopped = application.stop(); + Err(failed_restart(error, stopped, |message| { + write_outcome( + directory, + plan, + OutcomeStatus::Failed, + message, + Some(plan.version.clone()), + Some(application.identity.clone()), + ) + })) +} + +fn failed_restart( + error: String, + stopped: Result<(), String>, + record: impl FnOnce(&str) -> Result<(), String>, +) -> RestartFailure { + match stopped { + Ok(()) => RestartFailure::RollbackSafe(error), + Err(shutdown_error) => { + let message = format!( + "{error}; {shutdown_error}; rollback was not attempted while updated processes may still be running" + ); + let _ = record(&message); + RestartFailure::ProcessesMayBeRunning(message) + } + } +} + +fn prepare_payload(plan: &Plan, destination: &Path) -> Result { + match plan.kind { + InstallKind::AppImage => { + let mut file = File::open(&plan.package).map_err(|error| error.to_string())?; + let mut magic = [0u8; 11]; + file.read_exact(&mut magic) + .map_err(|error| error.to_string())?; + if &magic[..4] != b"\x7fELF" || &magic[8..11] != b"AI\x02" { + return Err("Signed package is not a type-2 AppImage".to_owned()); + } + validate_executable_architecture(&plan.package, plan.kind)?; + copy_synced(&plan.package, destination)?; + make_executable(destination)?; + Ok(destination.to_path_buf()) + } + InstallKind::WindowsPortable | InstallKind::MacBundle => { + if plan.kind == InstallKind::MacBundle + && plan + .package + .extension() + .is_some_and(|value| value.eq_ignore_ascii_case("dmg")) + { + bundle::extract_dmg(&plan.package, destination)?; + } else { + archive::extract_zip( + &plan.package, + destination, + plan.kind == InstallKind::MacBundle, + )?; + } + let relative = plan + .application_executable + .strip_prefix(&plan.target) + .map_err(|error| error.to_string())?; + let candidates = std::iter::once(destination.to_path_buf()).chain( + fs::read_dir(destination) + .map_err(|error| error.to_string())? + .filter_map(|entry| entry.ok().map(|entry| entry.path())), + ); + let roots: Vec<_> = candidates + .filter(|root| root.join(relative).is_file()) + .collect(); + if roots.len() != 1 { + return Err( + "Update archive does not contain exactly one expected application root" + .to_owned(), + ); + } + let root = roots.into_iter().next().ok_or("Missing update root")?; + let executable = root.join(relative); + canonical_file(&executable)?; + let core = executable + .parent() + .ok_or("Missing update executable directory")? + .join(if cfg!(windows) { + "opennow-core.exe" + } else { + "opennow-core" + }); + canonical_file(&core)?; + let helper = executable + .parent() + .ok_or("Missing update executable directory")? + .join(if cfg!(windows) { + "opennow-update-helper.exe" + } else { + "opennow-update-helper" + }); + canonical_file(&helper)?; + for path in [&executable, &core, &helper] { + validate_executable_architecture(path, plan.kind)?; + } + if plan.kind == InstallKind::MacBundle { + bundle::verify_bundle(&root, &plan.target)?; + } + Ok(root) + } + InstallKind::WindowsMsi | InstallKind::DebianPackage => Ok(plan.package.clone()), + } +} + +fn validate_executable_architecture(path: &Path, kind: InstallKind) -> Result<(), String> { + use std::io::{Seek, SeekFrom}; + if kind == InstallKind::MacBundle { + #[cfg(target_os = "macos")] + { + let architecture = if cfg!(target_arch = "aarch64") { + "arm64" + } else { + "x86_64" + }; + if !command_output( + Command::new("/usr/bin/lipo") + .arg(path) + .args(["-verify_arch", architecture]), + Duration::from_secs(120), + )? + .status + .success() + { + return Err( + "Application bundle does not contain this processor architecture".to_owned(), + ); + } + } + return Ok(()); + } + let mut file = File::open(path).map_err(|error| error.to_string())?; + let mut header = [0u8; 64]; + file.read_exact(&mut header) + .map_err(|error| error.to_string())?; + if kind == InstallKind::AppImage { + let expected: u16 = if cfg!(target_arch = "aarch64") { + 183 + } else { + 62 + }; + if header[4] != 2 + || header[5] != 1 + || u16::from_le_bytes([header[18], header[19]]) != expected + { + return Err("AppImage architecture does not match this installation".to_owned()); + } + return Ok(()); + } + if &header[..2] != b"MZ" { + return Err("Windows package contains a non-PE executable".to_owned()); + } + let offset = u32::from_le_bytes(header[60..64].try_into().map_err(|_| "Invalid PE header")?); + if offset > 1024 * 1024 { + return Err("Windows PE header exceeds its bound".to_owned()); + } + file.seek(SeekFrom::Start(offset as u64)) + .map_err(|error| error.to_string())?; + let mut pe = [0u8; 6]; + file.read_exact(&mut pe) + .map_err(|error| error.to_string())?; + let expected: u16 = if cfg!(target_arch = "aarch64") { + 0xaa64 + } else { + 0x8664 + }; + if &pe[..4] != b"PE\0\0" || u16::from_le_bytes([pe[4], pe[5]]) != expected { + return Err("Windows executable architecture does not match this installation".to_owned()); + } + Ok(()) +} + +fn preserve_portable_data(plan: &Plan, payload: &Path) -> Result, String> { + if plan.kind != InstallKind::WindowsPortable { + return Ok(Vec::new()); + } + let mut sources = Vec::new(); + for (index, entry) in fs::read_dir(&plan.target) + .map_err(|error| error.to_string())? + .enumerate() + { + if index as u64 >= bundle::MAX_COPY_ENTRIES { + return Err( + "Portable installation has too many root entries to preserve safely".to_owned(), + ); + } + let entry = entry.map_err(|error| error.to_string())?; + if !matches!( + entry.file_name().to_str(), + Some( + "bin" + | "plugins" + | "qml" + | "share" + | "translations" + | "LICENSE" + | "THIRD_PARTY_NOTICES.json" + ) + ) { + sources.push(entry.path()); + } + } + if plan.data_dir.starts_with(&plan.target) + && !sources.iter().any(|path| plan.data_dir.starts_with(path)) + { + sources.push(plan.data_dir.clone()); + } + let mut copied = Vec::new(); + let mut budget = bundle::CopyBudget::new(); + for source in sources { + let destination = payload.join( + source + .strip_prefix(&plan.target) + .map_err(|error| error.to_string())?, + ); + if destination.exists() { + return Err(format!( + "Update package overlaps preserved user data at {}; move that data outside the installation before updating", + source.display() + )); + } + copy_user_data(&source, &destination, &mut budget)?; + copied.push((source, destination)); + } + Ok(copied) +} + +fn copy_user_data( + source: &Path, + destination: &Path, + budget: &mut bundle::CopyBudget, +) -> Result<(), String> { + let metadata = fs::symlink_metadata(source).map_err(|error| error.to_string())?; + fs::create_dir_all(destination.parent().ok_or("User data has no parent")?) + .map_err(|error| error.to_string())?; + if metadata.is_dir() { + return bundle::copy_tree(source, destination, budget, bundle::CopyPolicy::UserData); + } + if !metadata.is_file() || metadata.len() > verification::MAXIMUM_UPDATE_BYTES { + return Err( + "Portable user data must contain bounded regular files or internal relative symlinks" + .to_owned(), + ); + } + budget.consume_entry()?; + budget.copy_file(source, destination, bundle::CopyPolicy::UserData) +} + +fn installation_target(kind: InstallKind, executable: &Path) -> Result { + let parent = executable.parent().ok_or("Application has no directory")?; + match kind { + InstallKind::AppImage => { + let image = std::env::var_os("APPIMAGE") + .map(PathBuf::from) + .unwrap_or_else(|| executable.to_path_buf()); + canonical_file(&image) + } + InstallKind::MacBundle => { + if parent.file_name().is_none_or(|name| name != "MacOS") + || parent + .parent() + .and_then(Path::file_name) + .is_none_or(|name| name != "Contents") + { + return Err("Application is not inside a standard macOS bundle".to_owned()); + } + let root = parent + .parent() + .and_then(Path::parent) + .ok_or("Invalid app bundle")?; + if root.extension().is_none_or(|extension| extension != "app") { + return Err("Application bundle has no .app suffix".to_owned()); + } + Ok(root.to_path_buf()) + } + InstallKind::WindowsPortable | InstallKind::WindowsMsi => { + if parent.file_name().is_none_or(|name| name != "bin") + || executable + .file_name() + .is_none_or(|name| name != "OpenNOW.exe") + { + return Err( + "Application does not use the supported Windows bin/OpenNOW.exe layout" + .to_owned(), + ); + } + Ok(parent + .parent() + .ok_or("Invalid Windows installation")? + .to_path_buf()) + } + InstallKind::DebianPackage => Ok(executable.to_path_buf()), + } +} + +fn read_manifest(package: &Path) -> Result { + let mut bytes = Vec::new(); + File::open(canonical_file(&manifest_path(package)?)?) + .map_err(|error| error.to_string())? + .take(verification::MAXIMUM_MANIFEST_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + verification::verify_signed_manifest(&bytes) +} + +fn manifest_path(package: &Path) -> Result { + Ok(package.with_file_name(format!( + "{}.manifest.json", + package + .file_name() + .and_then(|value| value.to_str()) + .ok_or("Invalid update package name")? + ))) +} + +fn canonical_file(path: &Path) -> Result { + if !fs::symlink_metadata(path) + .map_err(|error| error.to_string())? + .is_file() + { + return Err("Expected a regular file, not a link or directory".to_owned()); + } + fs::canonicalize(path).map_err(|error| error.to_string()) +} + +fn copy_synced(source: &Path, destination: &Path) -> Result<(), String> { + copy_synced_bounded( + source, + destination, + verification::MAXIMUM_UPDATE_BYTES, + bundle::CopyPolicy::Bundle, + ) + .map(|_| ()) +} + +fn copy_synced_bounded( + source: &Path, + destination: &Path, + maximum: u64, + policy: bundle::CopyPolicy, +) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + options.custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT); + } + let mut input = options.open(source).map_err(|error| error.to_string())?; + let metadata = input.metadata().map_err(|error| error.to_string())?; + if !metadata.is_file() || metadata.len() > maximum { + return Err( + "Update copy exceeds its remaining byte budget or is not a regular file".to_owned(), + ); + } + let mut output_options = OpenOptions::new(); + output_options.write(true).create_new(true); + #[cfg(unix)] + if policy == bundle::CopyPolicy::UserData { + use std::os::unix::fs::OpenOptionsExt; + output_options.mode(0o600); + } + let mut output = output_options + .open(destination) + .map_err(|error| error.to_string())?; + let result = (|| { + #[cfg(windows)] + if policy == bundle::CopyPolicy::UserData { + security::preserve_permissions(source, destination)?; + } + let copied = copy_bounded(&mut input, &mut output, maximum)?; + #[cfg(unix)] + if policy == bundle::CopyPolicy::UserData { + security::preserve_permissions(source, destination)?; + } + output.sync_all().map_err(|error| error.to_string())?; + Ok(copied) + })(); + drop(output); + if result.is_err() { + let _ = fs::remove_file(destination); + } + result +} + +fn copy_bounded( + input: &mut impl Read, + output: &mut impl Write, + maximum: u64, +) -> Result { + let copied = std::io::copy(&mut (&mut *input).take(maximum), output) + .map_err(|error| error.to_string())?; + let mut overflow = [0u8; 1]; + if input + .read(&mut overflow) + .map_err(|error| error.to_string())? + != 0 + { + return Err("Update copy exceeds its remaining streaming byte budget".to_owned()); + } + Ok(copied) +} + +fn make_executable(path: &Path) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o755)) + .map_err(|error| error.to_string())?; + } + let _ = path; + Ok(()) +} + +fn command_output( + command: &mut Command, + timeout: Duration, +) -> Result { + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let stdout = child.stdout.take().ok_or("Missing command stdout")?; + let stderr = child.stderr.take().ok_or("Missing command stderr")?; + let (sender, receiver) = std::sync::mpsc::sync_channel(2); + let read = |index: usize, stream: Box| { + let sender = sender.clone(); + std::thread::spawn(move || { + let result = (|| { + let mut bytes = Vec::new(); + stream + .take(64 * 1024 + 1) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 64 * 1024 { + return Err("Native package command output exceeds its limit".to_owned()); + } + Ok(bytes) + })(); + let _ = sender.send((index, result)); + }); + }; + read(0, Box::new(stdout)); + read(1, Box::new(stderr)); + drop(sender); + let start = Instant::now(); + let result = (|| { + let mut streams = [None, None]; + let mut status = None; + loop { + if status.is_none() { + status = child.try_wait().map_err(|error| error.to_string())?; + } + while let Ok((index, result)) = receiver.try_recv() { + streams[index] = Some(result?); + } + if let (Some(status), true) = (status, streams.iter().all(Option::is_some)) { + return Ok(std::process::Output { + status, + stdout: streams[0].take().ok_or("Missing command stdout")?, + stderr: streams[1].take().ok_or("Missing command stderr")?, + }); + } + if start.elapsed() > timeout { + return Err("Native package command exceeded its deadline".to_owned()); + } + std::thread::sleep(Duration::from_millis(50)); + } + })(); + if result.is_err() { + #[cfg(unix)] + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + let _ = child.kill(); + let _ = child.wait(); + } + result +} + +fn read_json(path: &Path, maximum: u64) -> Result { + let mut bytes = Vec::new(); + File::open(path) + .map_err(|error| error.to_string())? + .take(maximum + 1) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() as u64 > maximum { + return Err("Update transaction metadata exceeds its size limit".to_owned()); + } + serde_json::from_slice(&bytes) + .map_err(|error| format!("Invalid update transaction metadata: {error}")) +} + +fn atomic_json(path: &Path, value: &impl Serialize) -> Result<(), String> { + let temporary = path.with_extension(format!("{:016x}.tmp", rand::random::())); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| error.to_string())?; + let result = (|| { + file.write_all(&serde_json::to_vec(value).map_err(|error| error.to_string())?) + .and_then(|_| file.sync_all()) + .map_err(|error| error.to_string())?; + drop(file); + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::{ERROR_ACCESS_DENIED, ERROR_SHARING_VIOLATION}; + use windows_sys::Win32::Storage::FileSystem::{ + MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW, + }; + let old: Vec<_> = temporary.as_os_str().encode_wide().chain(Some(0)).collect(); + let new: Vec<_> = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let start = Instant::now(); + while unsafe { + MoveFileExW( + old.as_ptr(), + new.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + } == 0 + { + let error = std::io::Error::last_os_error(); + if !matches!( + error.raw_os_error().map(|code| code as u32), + Some(ERROR_ACCESS_DENIED | ERROR_SHARING_VIOLATION) + ) || start.elapsed() >= Duration::from_secs(1) + { + return Err(format!( + "Cannot publish update metadata at {}: {error}", + path.display() + )); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + #[cfg(not(windows))] + fs::rename(&temporary, path).map_err(|error| error.to_string())?; + sync_directory(path.parent().ok_or("Metadata has no parent")?) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn rename_synced(source: &Path, destination: &Path) -> Result<(), String> { + if destination.exists() { + return Err("Update rename destination already exists".to_owned()); + } + fs::rename(source, destination) + .map_err(|error| format!("Could not replace {}: {error}", source.display()))?; + sync_directory(source.parent().ok_or("Missing source parent")?)?; + sync_directory(destination.parent().ok_or("Missing destination parent")?) +} + +fn sync_directory(path: &Path) -> Result<(), String> { + #[cfg(unix)] + File::open(path) + .and_then(|file| file.sync_all()) + .map_err(|error| error.to_string())?; + let _ = path; + Ok(()) +} + +fn remove_path(path: &Path) -> Result<(), String> { + if path.is_dir() { + fs::remove_dir_all(path) + } else if path.exists() { + fs::remove_file(path) + } else { + Ok(()) + } + .map_err(|error| error.to_string()) +} + +fn write_outcome( + directory: &Path, + plan: &Plan, + status: OutcomeStatus, + message: &str, + installed_version: Option, + restarted_process: Option, +) -> Result<(), String> { + atomic_json( + &directory.join("outcome.json"), + &UpdateOutcome { + schema_version: 1, + version: plan.version.clone(), + status, + message: message.to_owned(), + installed_version, + restarted_process, + }, + ) +} + +#[cfg(test)] +mod integration_tests; +#[cfg(test)] +mod tests; diff --git a/external/opennow/native/opennow-core/src/update_apply/process.rs b/external/opennow/native/opennow-core/src/update_apply/process.rs new file mode 100644 index 000000000..76fb00a5a --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/process.rs @@ -0,0 +1,544 @@ +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +pub(super) struct OwnedApplication { + pub child: std::process::Child, + pub identity: ProcessIdentity, + #[cfg(windows)] + job: Job, +} + +impl OwnedApplication { + pub fn start(command: &mut std::process::Command) -> Result { + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x00000004); + } + let mut child = command.spawn().map_err(|error| { + super::RestartFailure::RollbackSafe(format!( + "Updated application could not start: {error}" + )) + })?; + #[cfg(windows)] + let job = match Job::assign(&child) { + Ok(job) => job, + Err(error) => { + return Err(failed_start(&mut child, error)); + } + }; + let identity = match ProcessIdentity::capture(child.id()) { + Ok(identity) => identity, + Err(error) => { + return Err(failed_start(&mut child, error)); + } + }; + #[cfg(windows)] + { + use std::os::windows::io::AsRawHandle; + #[link(name = "ntdll")] + unsafe extern "system" { + fn NtResumeProcess(process: windows_sys::Win32::Foundation::HANDLE) -> i32; + } + if unsafe { NtResumeProcess(child.as_raw_handle()) } != 0 { + return Err(failed_start( + &mut child, + "Cannot resume the owned updated application".to_owned(), + )); + } + } + Ok(Self { + child, + identity, + #[cfg(windows)] + job, + }) + } + + pub fn owns(&self, process: &ProcessIdentity) -> Result { + if !process.is_running()? || process.started < self.identity.started { + return Ok(false); + } + #[cfg(unix)] + { + Ok(process.group == self.identity.group) + } + #[cfg(windows)] + { + self.job.contains(process.pid) + } + } + + pub fn stop(&mut self) -> Result<(), String> { + #[cfg(unix)] + { + if capture(self.identity.pid)?.is_some_and(|current| { + current.started != self.identity.started || current.boot_id != self.identity.boot_id + }) { + return Err( + "Updated application process identity was reused; refusing group shutdown" + .to_owned(), + ); + } + if unsafe { libc::kill(-(self.identity.group as i32), libc::SIGKILL) } < 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error.to_string()); + } + } + } + #[cfg(windows)] + self.job.stop()?; + let start = Instant::now(); + loop { + let exited = self + .child + .try_wait() + .map_err(|error| error.to_string())? + .is_some(); + #[cfg(unix)] + let descendants = group_is_running(self.identity.group)?; + #[cfg(windows)] + let descendants = self.job.active()?; + if exited && !descendants { + return Ok(()); + } + if start.elapsed() > Duration::from_secs(10) { + return Err( + "Could not stop all updated application processes before rollback".to_owned(), + ); + } + std::thread::sleep(Duration::from_millis(50)); + } + } +} + +pub(super) fn failed_start( + child: &mut std::process::Child, + error: String, +) -> super::RestartFailure { + let cleanup = (|| { + #[cfg(unix)] + if unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) } < 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error.to_string()); + } + } + if let (Err(error), None) = ( + child.kill(), + child.try_wait().map_err(|error| error.to_string())?, + ) { + return Err(error.to_string()); + } + let start = Instant::now(); + loop { + let exited = child + .try_wait() + .map_err(|error| error.to_string())? + .is_some(); + #[cfg(unix)] + let descendants = group_is_running(child.id())?; + #[cfg(windows)] + let descendants = false; + if exited && !descendants { + return Ok(()); + } + if start.elapsed() >= Duration::from_secs(10) { + return Err( + "Failed startup process tree did not stop before the deadline".to_owned(), + ); + } + std::thread::sleep(Duration::from_millis(50)); + } + })(); + match cleanup { + Ok(()) => super::RestartFailure::RollbackSafe(error), + Err(cleanup_error) => super::RestartFailure::ProcessesMayBeRunning(format!( + "{error}; startup cleanup failed: {cleanup_error}" + )), + } +} + +#[cfg(target_os = "linux")] +fn group_is_running(group: u32) -> Result { + for entry in std::fs::read_dir("/proc").map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + if !entry + .file_name() + .to_string_lossy() + .bytes() + .all(|byte| byte.is_ascii_digit()) + { + continue; + } + let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { + continue; + }; + let Some((_, fields)) = stat.rsplit_once(')') else { + continue; + }; + let fields: Vec<_> = fields.split_whitespace().collect(); + if fields.first() != Some(&"Z") + && fields.get(2).and_then(|value| value.parse::().ok()) == Some(group) + { + return Ok(true); + } + } + Ok(false) +} + +#[cfg(target_os = "macos")] +fn group_is_running(group: u32) -> Result { + unsafe extern "C" { + fn proc_listpids(kind: u32, info: u32, buffer: *mut std::ffi::c_void, size: i32) -> i32; + } + let mut pids = vec![0u32; 100_000]; + let length = + unsafe { proc_listpids(2, group, pids.as_mut_ptr().cast(), (pids.len() * 4) as i32) }; + if length < 0 || length as usize >= pids.len() * 4 { + return Err("Cannot bound the updated application's process group".to_owned()); + } + for pid in pids + .into_iter() + .take(length as usize / 4) + .filter(|pid| *pid != 0) + { + if capture(pid)?.is_some() { + return Ok(true); + } + } + Ok(false) +} + +#[cfg(windows)] +struct Job(windows_sys::Win32::Foundation::HANDLE); + +#[cfg(windows)] +impl Drop for Job { + fn drop(&mut self) { + unsafe { + windows_sys::Win32::Foundation::CloseHandle(self.0); + } + } +} + +#[cfg(windows)] +impl Job { + fn assign(child: &std::process::Child) -> Result { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::JobObjects::*; + let handle = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; + if handle.is_null() { + return Err(std::io::Error::last_os_error().to_string()); + } + let job = Self(handle); + if unsafe { AssignProcessToJobObject(handle, child.as_raw_handle()) } == 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + Ok(job) + } + fn contains(&self, pid: u32) -> Result { + use windows_sys::Win32::System::{JobObjects::*, Threading::*}; + let process = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if process.is_null() { + return Err(std::io::Error::last_os_error().to_string()); + } + let mut member = 0; + let result = unsafe { IsProcessInJob(process, self.0, &mut member) }; + unsafe { + windows_sys::Win32::Foundation::CloseHandle(process); + } + if result == 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + Ok(member != 0) + } + fn stop(&self) -> Result<(), String> { + if unsafe { windows_sys::Win32::System::JobObjects::TerminateJobObject(self.0, 1) } == 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + Ok(()) + } + fn active(&self) -> Result { + use windows_sys::Win32::System::JobObjects::*; + let mut info: JOBOBJECT_BASIC_ACCOUNTING_INFORMATION = unsafe { std::mem::zeroed() }; + if unsafe { + QueryInformationJobObject( + self.0, + JobObjectBasicAccountingInformation, + (&mut info as *mut JOBOBJECT_BASIC_ACCOUNTING_INFORMATION).cast(), + std::mem::size_of_val(&info) as u32, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(std::io::Error::last_os_error().to_string()); + } + Ok(info.ActiveProcesses != 0) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ProcessIdentity { + pub pid: u32, + pub started: u64, + pub executable: PathBuf, + pub group: u32, + pub boot_id: String, +} + +impl ProcessIdentity { + pub fn capture(pid: u32) -> Result { + capture(pid)?.ok_or_else(|| format!("Process {pid} is not running")) + } + + pub fn matches_executable(&self, executable: &Path) -> Result<(), String> { + if std::fs::canonicalize(executable).map_err(|error| error.to_string())? != self.executable + { + return Err("Application process does not match its trusted executable".to_owned()); + } + Ok(()) + } + + pub fn is_running(&self) -> Result { + Ok(capture(self.pid)?.is_some_and(|current| { + current.started == self.started && current.boot_id == self.boot_id + })) + } +} + +pub(super) fn wait_for_exit( + processes: &[ProcessIdentity], + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + loop { + let mut running = false; + for process in processes { + running |= process.is_running()?; + } + if !running { + return Ok(()); + } + if start.elapsed() >= timeout { + return Err( + "Timed out waiting for the original application and core to exit".to_owned(), + ); + } + std::thread::sleep(Duration::from_millis(100)); + } +} + +pub(super) fn owned_tree_is_running(identity: &ProcessIdentity) -> Result { + #[cfg(target_os = "linux")] + if std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .map_err(|error| error.to_string())? + .trim() + != identity.boot_id + { + return Ok(false); + } + #[cfg(unix)] + { + group_is_running(identity.group) + } + #[cfg(windows)] + { + identity.is_running() + } +} + +#[cfg(target_os = "linux")] +fn capture(pid: u32) -> Result, String> { + let path = PathBuf::from(format!("/proc/{pid}")); + let stat = match std::fs::read_to_string(path.join("stat")) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + let fields: Vec<_> = stat + .rsplit_once(')') + .ok_or("Invalid process identity")? + .1 + .split_whitespace() + .collect(); + if fields.first() == Some(&"Z") { + return Ok(None); + } + let started = fields + .get(19) + .ok_or("Missing process start identity")? + .parse() + .map_err(|_| "Invalid process start identity")?; + let executable = match std::fs::read_link(path.join("exe")) { + Ok(path) => path, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + Ok(Some(ProcessIdentity { + pid, + started, + executable, + group: fields + .get(2) + .ok_or("Missing process group")? + .parse() + .map_err(|_| "Invalid process group")?, + boot_id: std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .map_err(|error| error.to_string())? + .trim() + .to_owned(), + })) +} + +#[cfg(target_os = "macos")] +fn capture(pid: u32) -> Result, String> { + #[repr(C)] + #[derive(Default)] + struct BsdInfo { + flags: u32, + status: u32, + xstatus: u32, + pid: u32, + ppid: u32, + uid: u32, + gid: u32, + ruid: u32, + rgid: u32, + svuid: u32, + svgid: u32, + rfu_1: u32, + comm: [u8; 16], + name: [u8; 32], + nfiles: u32, + pgid: u32, + pjobc: u32, + e_tdev: u32, + e_tpgid: u32, + nice: i32, + start_tvsec: u64, + start_tvusec: u64, + } + unsafe extern "C" { + fn proc_pidinfo( + pid: i32, + flavor: i32, + arg: u64, + buffer: *mut std::ffi::c_void, + size: i32, + ) -> i32; + fn proc_pidpath(pid: i32, buffer: *mut std::ffi::c_void, size: u32) -> i32; + } + let mut info = BsdInfo::default(); + let length = unsafe { + proc_pidinfo( + pid as i32, + 3, + 0, + (&mut info as *mut BsdInfo).cast(), + std::mem::size_of::() as i32, + ) + }; + if length == 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(None); + } + return Err(error.to_string()); + } + if length as usize != std::mem::size_of::() { + return Err("Incomplete macOS process identity".to_owned()); + } + if info.status == 5 { + return Ok(None); + } + let mut path = [0u8; 4096]; + let length = unsafe { proc_pidpath(pid as i32, path.as_mut_ptr().cast(), path.len() as u32) }; + if length <= 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(None); + } + return Err(error.to_string()); + } + use std::os::unix::ffi::OsStrExt; + let end = path + .iter() + .position(|byte| *byte == 0) + .unwrap_or(length as usize); + let executable = std::fs::canonicalize(Path::new(std::ffi::OsStr::from_bytes(&path[..end]))) + .map_err(|error| error.to_string())?; + Ok(Some(ProcessIdentity { + pid, + started: info.start_tvsec * 1_000_000 + info.start_tvusec, + executable, + group: info.pgid, + boot_id: String::new(), + })) +} + +#[cfg(windows)] +fn capture(pid: u32) -> Result, String> { + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::Foundation::{CloseHandle, ERROR_INVALID_PARAMETER, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetExitCodeProcess, GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + QueryFullProcessImageNameW, + }; + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if handle.is_null() { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(ERROR_INVALID_PARAMETER as i32) { + return Ok(None); + } + return Err(error.to_string()); + } + let result = (|| { + let mut exit = 0; + if unsafe { GetExitCodeProcess(handle, &mut exit) } == 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + if exit != 259 { + return Ok(None); + } + let mut creation = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let mut end = creation; + let mut kernel = creation; + let mut user = creation; + if unsafe { GetProcessTimes(handle, &mut creation, &mut end, &mut kernel, &mut user) } == 0 + { + return Err(std::io::Error::last_os_error().to_string()); + } + let mut buffer = vec![0u16; 32768]; + let mut length = buffer.len() as u32; + if unsafe { QueryFullProcessImageNameW(handle, 0, buffer.as_mut_ptr(), &mut length) } == 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + let executable = std::fs::canonicalize(PathBuf::from(std::ffi::OsString::from_wide( + &buffer[..length as usize], + ))) + .map_err(|error| error.to_string())?; + Ok(Some(ProcessIdentity { + pid, + started: (creation.dwHighDateTime as u64) << 32 | creation.dwLowDateTime as u64, + executable, + group: 0, + boot_id: String::new(), + })) + })(); + unsafe { + CloseHandle(handle); + } + result +} diff --git a/external/opennow/native/opennow-core/src/update_apply/recovery.rs b/external/opennow/native/opennow-core/src/update_apply/recovery.rs new file mode 100644 index 000000000..57559d02e --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/recovery.rs @@ -0,0 +1,313 @@ +use super::{InstallKind, OutcomeStatus, Plan, PreparedUpdate, UpdateOutcome, managed}; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +#[derive(Debug)] +pub enum ManagedRecovery { + Pending(String), + RebootRequired(String), + Finished(UpdateOutcome), +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct InstallBoot { + schema_version: u32, + boot_id: String, +} + +pub(super) fn record_install_boot(directory: &Path) -> Result<(), String> { + super::atomic_json( + &directory.join("install-boot.json"), + &InstallBoot { + schema_version: 1, + boot_id: current_boot_id()?, + }, + ) +} + +pub fn recover_managed_update( + prepared: &PreparedUpdate, + outcome: &UpdateOutcome, + running_version: &str, +) -> Result { + match super::helper_is_running(prepared) { + Ok(true) => { + return Ok(ManagedRecovery::Pending( + "The update helper is still finishing the native installation.".to_owned(), + )); + } + Err(error) => { + return Ok(ManagedRecovery::Pending(format!( + "Could not confirm that the update helper has finished: {error}" + ))); + } + Ok(false) => (), + } + let directory = prepared + .plan_path + .parent() + .ok_or("Missing update transaction directory.")?; + let unknown_installer = match &outcome.restarted_process { + Some(installer) => match installer.is_running() { + Ok(true) => return Ok(ManagedRecovery::Pending( + "The native package manager is still running; update completion is not confirmed." + .to_owned(), + )), + Err(error) => Some(error), + Ok(false) => None, + }, + None if outcome.status == OutcomeStatus::ManagedPending => { + Some("The native installer process identity was not recorded.".to_owned()) + } + None => None, + }; + if let Some(reason) = unknown_installer { + match reboot_completed(directory) { + Ok(true) => (), + Ok(false) => { + return Ok(ManagedRecovery::Pending(format!( + "Native installer completion cannot be confirmed: {reason} Restart your system before starting another update or session." + ))); + } + Err(error) => { + return Ok(ManagedRecovery::Pending(format!( + "Native installer completion cannot be confirmed: {reason} The restart boundary also could not be verified: {error}" + ))); + } + } + } + let plan: Plan = super::read_json(&prepared.plan_path, 64 * 1024)?; + if plan.schema_version != 1 + || plan.version != prepared.version + || outcome.version != prepared.version + || !matches!( + plan.kind, + InstallKind::WindowsMsi | InstallKind::DebianPackage + ) + || !matches!( + outcome.status, + OutcomeStatus::ManagedPending | OutcomeStatus::RebootRequired + ) + || (outcome.status == OutcomeStatus::RebootRequired && plan.kind != InstallKind::WindowsMsi) + { + return Err("Persisted managed update identity does not match its outcome.".to_owned()); + } + let identity = plan + .managed_identity + .as_ref() + .ok_or("Missing persisted native package identity.")?; + let reboot_completed = if outcome.status == OutcomeStatus::RebootRequired { + match reboot_completed(directory) { + Ok(value) => value, + Err(error) => { + return Ok(ManagedRecovery::RebootRequired(format!( + "Windows Installer requires a restart; reboot completion could not be verified: {error}" + ))); + } + } + } else { + true + }; + let recovery = classify_recovery(outcome, running_version, reboot_completed, || { + managed::verify_installed(plan.kind, identity, &plan.target) + }); + if let ManagedRecovery::Finished(result) = &recovery { + super::atomic_json(&prepared.outcome_path, result)?; + } + Ok(recovery) +} + +fn classify_recovery( + outcome: &UpdateOutcome, + running_version: &str, + reboot_completed: bool, + verify_installed: impl FnOnce() -> Result<(), String>, +) -> ManagedRecovery { + if outcome.status == OutcomeStatus::RebootRequired && !reboot_completed { + return ManagedRecovery::RebootRequired("Windows Installer requires a system restart before all updated files can be confirmed. Sessions remain available, but restart before checking for another update.".to_owned()); + } + let mut result = outcome.clone(); + result.restarted_process = None; + result.installed_version = None; + if let Err(error) = verify_installed() { + result.status = OutcomeStatus::Failed; + result.message = format!( + "The native installer has finished, but the expected installed package could not be verified: {error}" + ); + } else if running_version.trim_start_matches('v') != outcome.version.trim_start_matches('v') { + result.status = OutcomeStatus::Failed; + result.message = format!( + "The native package is installed, but OpenNOW is running {running_version} instead of {}. Restart OpenNOW before retrying the update.", + outcome.version + ); + } else { + result.status = OutcomeStatus::Completed; + result.installed_version = Some(outcome.version.clone()); + result.message = format!( + "Native package identity and running OpenNOW {} were verified after installation.", + outcome.version + ); + } + ManagedRecovery::Finished(result) +} + +fn reboot_completed(directory: &Path) -> Result { + let path = directory.join("install-boot.json"); + if !path.try_exists().map_err(|error| error.to_string())? { + record_install_boot(directory)?; + return Ok(false); + } + recorded_boot_changed(directory) +} + +fn recorded_boot_changed(directory: &Path) -> Result { + let previous: InstallBoot = super::read_json(&directory.join("install-boot.json"), 1024)?; + if previous.schema_version != 1 || previous.boot_id.is_empty() { + return Err("The recorded installation boot identity is invalid.".to_owned()); + } + Ok(previous.boot_id != current_boot_id()?) +} + +#[cfg(windows)] +fn current_boot_id() -> Result { + use windows_sys::Wdk::System::Threading::{ + NtQueryInformationProcess, ProcessTelemetryIdInformation, + }; + use windows_sys::Win32::System::Threading::GetCurrentProcess; + let mut returned = 0u32; + unsafe { + NtQueryInformationProcess( + GetCurrentProcess(), + ProcessTelemetryIdInformation, + std::ptr::null_mut(), + 0, + &mut returned, + ); + } + if !(64..=1024 * 1024).contains(&returned) { + return Err("Windows did not provide bounded process telemetry.".to_owned()); + } + let mut information = vec![0u64; (returned as usize).div_ceil(8)]; + let status = unsafe { + NtQueryInformationProcess( + GetCurrentProcess(), + ProcessTelemetryIdInformation, + information.as_mut_ptr().cast(), + (information.len() * 8) as u32, + &mut returned, + ) + }; + let header_size = information[0] as u32; + let process_id = (information[0] >> 32) as u32; + let boot_id = (information[7] >> 32) as u32; + if status < 0 + || returned < 64 + || header_size < 64 + || header_size > returned + || process_id != std::process::id() + { + return Err("Windows did not provide a valid boot sequence number.".to_owned()); + } + Ok(format!("windows-boot-sequence:{boot_id}")) +} + +#[cfg(target_os = "linux")] +fn current_boot_id() -> Result { + let value = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .map_err(|error| error.to_string())?; + let value = value.trim(); + if value.len() != 36 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() || byte == b'-') + { + return Err("The operating system boot identifier is invalid.".to_owned()); + } + Ok(value.to_owned()) +} + +#[cfg(not(any(windows, target_os = "linux")))] +fn current_boot_id() -> Result { + Err("Managed reboot recovery is not supported on this platform.".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn outcome(status: OutcomeStatus) -> UpdateOutcome { + UpdateOutcome { + schema_version: 1, + version: "1.2.3".to_owned(), + status, + message: "Native installation is pending".to_owned(), + installed_version: None, + restarted_process: None, + } + } + + #[test] + fn recovery_requires_native_identity_and_the_running_version() { + for status in [OutcomeStatus::ManagedPending, OutcomeStatus::RebootRequired] { + for (version, installed, expected) in [ + ("1.2.3", Ok(()), OutcomeStatus::Completed), + ("1.2.2", Ok(()), OutcomeStatus::Failed), + ( + "1.2.3", + Err("Native registration does not match".to_owned()), + OutcomeStatus::Failed, + ), + ] { + let ManagedRecovery::Finished(result) = + classify_recovery(&outcome(status), version, true, || installed) + else { + panic!("Finished installer must resolve to an outcome") + }; + assert_eq!(result.status, expected); + assert_eq!( + result.installed_version.is_some(), + expected == OutcomeStatus::Completed + ); + } + } + } + + #[test] + fn reboot_required_never_claims_success_before_a_proven_reboot() { + assert!(matches!( + classify_recovery( + &outcome(OutcomeStatus::RebootRequired), + "1.2.3", + false, + || panic!("Native package verification must not run before reboot") + ), + ManagedRecovery::RebootRequired(_) + )); + } + + #[cfg(any(windows, target_os = "linux"))] + #[test] + fn legacy_reboot_marker_requires_a_subsequent_boot() { + let directory = tempfile::tempdir().unwrap(); + assert!(!reboot_completed(directory.path()).unwrap()); + assert!(!reboot_completed(directory.path()).unwrap()); + super::super::atomic_json( + &directory.path().join("install-boot.json"), + &InstallBoot { + schema_version: 1, + boot_id: "a-previous-boot".to_owned(), + }, + ) + .unwrap(); + assert!(reboot_completed(directory.path()).unwrap()); + } + + #[cfg(windows)] + #[test] + fn windows_boot_sequence_is_available_and_stable() { + let first = current_boot_id().unwrap(); + assert!(first.starts_with("windows-boot-sequence:")); + assert_eq!(first, current_boot_id().unwrap()); + } +} diff --git a/external/opennow/native/opennow-core/src/update_apply/security.rs b/external/opennow/native/opennow-core/src/update_apply/security.rs new file mode 100644 index 000000000..ef974d0e9 --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/security.rs @@ -0,0 +1,355 @@ +use std::path::Path; + +pub(super) fn create_private_directory(path: &Path) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + let mut builder = std::fs::DirBuilder::new(); + builder.mode(0o700); + builder.create(path).map_err(|error| error.to_string()) + } + #[cfg(windows)] + { + windows::create_private(path) + } +} + +pub(super) fn preserve_permissions(source: &Path, destination: &Path) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::symlink_metadata(source).map_err(|error| error.to_string())?; + if metadata.is_symlink() { + return Err("Cannot preserve permissions through a user-data symlink".to_owned()); + } + std::os::unix::fs::chown(destination, Some(metadata.uid()), Some(metadata.gid())) + .map_err(|error| format!("Cannot preserve user-data ownership: {error}"))?; + std::fs::set_permissions(destination, metadata.permissions()) + .map_err(|error| format!("Cannot preserve user-data permissions: {error}"))?; + preserve_unix_acl(source, destination) + } + #[cfg(windows)] + { + windows::preserve(source, destination) + } +} + +#[cfg(target_os = "linux")] +fn preserve_unix_acl(source: &Path, destination: &Path) -> Result<(), String> { + use std::os::unix::ffi::OsStrExt; + let source = + std::ffi::CString::new(source.as_os_str().as_bytes()).map_err(|error| error.to_string())?; + let destination = std::ffi::CString::new(destination.as_os_str().as_bytes()) + .map_err(|error| error.to_string())?; + for name in [c"system.posix_acl_access", c"system.posix_acl_default"] { + let size = + unsafe { libc::getxattr(source.as_ptr(), name.as_ptr(), std::ptr::null_mut(), 0) }; + if size < 0 { + let error = std::io::Error::last_os_error(); + if !matches!(error.raw_os_error(), Some(libc::ENODATA | libc::ENOTSUP)) { + return Err(format!("Cannot inspect user-data ACL: {error}")); + } + if unsafe { libc::removexattr(destination.as_ptr(), name.as_ptr()) } < 0 { + let error = std::io::Error::last_os_error(); + if !matches!(error.raw_os_error(), Some(libc::ENODATA | libc::ENOTSUP)) { + return Err(format!("Cannot remove inherited user-data ACL: {error}")); + } + } + continue; + } + if size > 64 * 1024 { + return Err("User-data ACL exceeds its size limit".to_owned()); + } + let mut acl = vec![0u8; size as usize]; + let read = unsafe { + libc::getxattr( + source.as_ptr(), + name.as_ptr(), + acl.as_mut_ptr().cast(), + acl.len(), + ) + }; + if read != size { + return Err("User-data ACL changed while it was being copied".to_owned()); + } + if unsafe { + libc::setxattr( + destination.as_ptr(), + name.as_ptr(), + acl.as_ptr().cast(), + acl.len(), + 0, + ) + } < 0 + { + return Err(format!( + "Cannot preserve user-data ACL: {}", + std::io::Error::last_os_error() + )); + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn preserve_unix_acl(source: &Path, destination: &Path) -> Result<(), String> { + use std::os::unix::ffi::OsStrExt; + unsafe extern "C" { + fn copyfile( + source: *const libc::c_char, + destination: *const libc::c_char, + state: *mut libc::c_void, + flags: u32, + ) -> i32; + } + let source = + std::ffi::CString::new(source.as_os_str().as_bytes()).map_err(|error| error.to_string())?; + let destination = std::ffi::CString::new(destination.as_os_str().as_bytes()) + .map_err(|error| error.to_string())?; + if unsafe { + copyfile( + source.as_ptr(), + destination.as_ptr(), + std::ptr::null_mut(), + 1, + ) + } != 0 + { + return Err(format!( + "Cannot preserve user-data ACL: {}", + std::io::Error::last_os_error() + )); + } + Ok(()) +} + +#[cfg(windows)] +mod windows { + use super::*; + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::*; + use windows_sys::Win32::Security::*; + + struct Descriptor(PSECURITY_DESCRIPTOR); + impl Drop for Descriptor { + fn drop(&mut self) { + unsafe { + LocalFree(self.0); + } + } + } + + fn wide(path: &Path) -> Vec { + path.as_os_str().encode_wide().chain(Some(0)).collect() + } + + pub(super) fn create_private(path: &Path) -> Result<(), String> { + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, GetVolumeInformationByHandleW, + }; + let parent = std::fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open( + path.parent() + .ok_or("Private update directory has no parent")?, + ) + .map_err(|error| error.to_string())?; + let mut flags = 0; + if unsafe { + GetVolumeInformationByHandleW( + parent.as_raw_handle(), + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + &mut flags, + std::ptr::null_mut(), + 0, + ) + } == 0 + { + return Err(format!( + "Cannot verify update volume ACL support: {}", + std::io::Error::last_os_error() + )); + } + require_persistent_acls(flags)?; + let text: Vec<_> = "D:P(A;OICI;FA;;;OW)" + .encode_utf16() + .chain(Some(0)) + .collect(); + let mut descriptor = std::ptr::null_mut(); + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + text.as_ptr(), + 1, + &mut descriptor, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(std::io::Error::last_os_error().to_string()); + } + let descriptor = Descriptor(descriptor); + let attributes = SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + if unsafe { + windows_sys::Win32::Storage::FileSystem::CreateDirectoryW( + wide(path).as_ptr(), + &attributes, + ) + } == 0 + { + return Err(format!( + "Cannot create private update directory: {}", + std::io::Error::last_os_error() + )); + } + Ok(()) + } + + fn require_persistent_acls(flags: u32) -> Result<(), String> { + if flags & windows_sys::Win32::System::SystemServices::FILE_PERSISTENT_ACLS == 0 { + return Err("Update staging requires a filesystem with persistent ACLs; FAT and exFAT cannot protect the transaction".to_owned()); + } + Ok(()) + } + + pub(super) fn preserve(source: &Path, destination: &Path) -> Result<(), String> { + let mut owner = std::ptr::null_mut(); + let mut group = std::ptr::null_mut(); + let mut acl = std::ptr::null_mut(); + let mut descriptor = std::ptr::null_mut(); + let info = + OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION; + let code = unsafe { + GetNamedSecurityInfoW( + wide(source).as_ptr(), + SE_FILE_OBJECT, + info, + &mut owner, + &mut group, + &mut acl, + std::ptr::null_mut(), + &mut descriptor, + ) + }; + if code != 0 { + return Err(format!( + "Cannot inspect user-data security descriptor: Windows error {code}" + )); + } + let _descriptor = Descriptor(descriptor); + let code = unsafe { + SetNamedSecurityInfoW( + wide(destination).as_ptr(), + SE_FILE_OBJECT, + info | PROTECTED_DACL_SECURITY_INFORMATION, + owner, + group, + acl, + std::ptr::null_mut(), + ) + }; + if code != 0 { + return Err(format!( + "Cannot preserve the original effective user-data DACL and ownership: Windows error {code}" + )); + } + Ok(()) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn private_staging_requires_persistent_acl_support() { + assert!(require_persistent_acls(0).is_err()); + assert!( + require_persistent_acls( + !windows_sys::Win32::System::SystemServices::FILE_PERSISTENT_ACLS + ) + .is_err() + ); + assert!( + require_persistent_acls( + windows_sys::Win32::System::SystemServices::FILE_PERSISTENT_ACLS + ) + .is_ok() + ); + } + + fn dacl(path: &Path) -> (Vec>, bool) { + let mut acl = std::ptr::null_mut(); + let mut descriptor = std::ptr::null_mut(); + assert_eq!( + unsafe { + GetNamedSecurityInfoW( + wide(path).as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + std::ptr::null_mut(), + std::ptr::null_mut(), + &mut acl, + std::ptr::null_mut(), + &mut descriptor, + ) + }, + 0 + ); + let descriptor = Descriptor(descriptor); + let mut control = 0; + let mut revision = 0; + assert_ne!( + unsafe { GetSecurityDescriptorControl(descriptor.0, &mut control, &mut revision) }, + 0 + ); + assert!(!acl.is_null()); + let mut entries = Vec::new(); + for index in 0..unsafe { (*acl).AceCount } { + let mut ace = std::ptr::null_mut(); + assert_ne!(unsafe { GetAce(acl, index as u32, &mut ace) }, 0); + let header = ace.cast::(); + let mut bytes = unsafe { + std::slice::from_raw_parts(ace.cast::(), (*header).AceSize as usize) + } + .to_vec(); + bytes[1] &= !(INHERITED_ACE as u8); + entries.push(bytes); + } + (entries, control & SE_DACL_PROTECTED != 0) + } + + #[test] + fn private_transactions_and_preserved_profiles_keep_effective_dacls() { + let directory = tempfile::TempDir::new().unwrap(); + let profile = directory.path().join("profile"); + create_private(&profile).unwrap(); + std::fs::write(profile.join("account.json"), b"private fixture data").unwrap(); + let root_before = dacl(&profile); + let file_before = dacl(&profile.join("account.json")); + assert!(root_before.1); + let destination = directory.path().join("preserved"); + crate::update_apply::copy_user_data( + &profile, + &destination, + &mut crate::update_apply::bundle::CopyBudget::new(), + ) + .unwrap(); + let root_after = dacl(&destination); + let file_after = dacl(&destination.join("account.json")); + assert_eq!(root_before.0, root_after.0); + assert_eq!(file_before.0, file_after.0); + assert!(root_after.1); + assert!(file_after.1); + } + } +} diff --git a/external/opennow/native/opennow-core/src/update_apply/tests.rs b/external/opennow/native/opennow-core/src/update_apply/tests.rs new file mode 100644 index 000000000..7de8fd9b7 --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/tests.rs @@ -0,0 +1,1178 @@ +use super::*; +use base64::Engine as _; +use ed25519_dalek::{Signer, SigningKey}; +use sha2::{Digest, Sha256}; +use std::sync::OnceLock; +use tempfile::TempDir; + +#[cfg(target_os = "linux")] +#[test] +fn linux_update_preparation_rejects_cross_format_replacement() { + use std::ffi::OsStr; + let image = Some(OsStr::new("/home/user/OpenNOW.AppImage")); + assert!(validate_linux_install_kind(InstallKind::AppImage, image).is_ok()); + assert!(validate_linux_install_kind(InstallKind::DebianPackage, None).is_ok()); + assert!(validate_linux_install_kind(InstallKind::AppImage, None).is_err()); + assert!(validate_linux_install_kind(InstallKind::AppImage, Some(OsStr::new(""))).is_err()); + assert!(validate_linux_install_kind(InstallKind::DebianPackage, image).is_err()); + assert!(validate_linux_install_kind(InstallKind::WindowsMsi, None).is_err()); +} + +#[test] +fn windows_zip_is_selected_until_the_install_is_msi_registered() { + assert_eq!(windows_update_package_extension(false).unwrap(), "zip"); + assert_eq!( + windows_update_package_extension(true).unwrap_err(), + WINDOWS_INSTALLER_REPLACEMENT_MESSAGE + ); + assert!(WINDOWS_INSTALLER_REPLACEMENT_MESSAGE.contains("setup.exe")); +} + +#[test] +fn msi_registered_install_never_reaches_msiexec() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsMsi); + assert_eq!( + apply(&plan, directory.path()).unwrap_err(), + WINDOWS_INSTALLER_REPLACEMENT_MESSAGE + ); + assert!(!directory.path().join("install-boot.json").exists()); + assert_eq!( + prepare_update(PrepareRequest { + package: directory.path().join("package.msi"), + expected_version: "1.2.3".to_owned(), + application_executable: plan.application_executable, + application_pid: 1, + core_pid: 1, + kind: InstallKind::WindowsMsi, + data_dir: plan.data_dir, + }) + .unwrap_err(), + WINDOWS_INSTALLER_REPLACEMENT_MESSAGE + ); +} + +#[cfg(unix)] +#[test] +fn prepared_windows_swap_replaces_runtime_binaries_and_missing_ack_restores_them() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + let names = [ + "OpenNOW.exe", + "opennow-core.exe", + "opennow-update-helper.exe", + ]; + let old_core = b"old-opennow-core"; + let old_helper = b"old-opennow-update-helper"; + let new_core = b"new-opennow-core"; + let new_helper = b"new-opennow-update-helper"; + let old_app = b"#!/bin/sh\nexit 0\n"; + let new_app = b"#!/bin/sh\nbin=$(CDPATH= cd -- \"$(dirname \"$0\")\" && pwd)\nmkdir -p \"$OPENNOW_DATA_DIR\"\ncp \"$bin/OpenNOW.exe\" \"$OPENNOW_DATA_DIR/seen-OpenNOW.exe\"\ncp \"$bin/opennow-core.exe\" \"$OPENNOW_DATA_DIR/seen-opennow-core.exe\"\ncp \"$bin/opennow-update-helper.exe\" \"$OPENNOW_DATA_DIR/seen-opennow-update-helper.exe\"\nsleep 30\n"; + let old = [ + old_app.as_slice(), + old_core.as_slice(), + old_helper.as_slice(), + ]; + let new = [ + new_app.as_slice(), + new_core.as_slice(), + new_helper.as_slice(), + ]; + for (name, bytes) in names.iter().zip(old) { + let path = plan.target.join("bin").join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, bytes).unwrap(); + make_executable(&path).unwrap(); + } + let payload = directory.path().join("payload"); + for (name, bytes) in names.iter().zip(new) { + let path = payload.join("bin").join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, bytes).unwrap(); + make_executable(&path).unwrap(); + } + let error = replace_and_restart( + &plan, + directory.path(), + &payload, + Duration::from_millis(1500), + ) + .unwrap_err(); + assert!( + error.contains("did not acknowledge healthy startup"), + "{error}" + ); + for (name, bytes) in names.iter().zip(old) { + assert_eq!( + fs::read(plan.target.join("bin").join(name)).unwrap(), + bytes, + "{name} was not restored" + ); + } + for (name, bytes) in names.iter().zip(new) { + assert_eq!( + fs::read(directory.path().join("failed").join("bin").join(name)).unwrap(), + bytes, + "{name} was not swapped into place before rollback" + ); + let seen = match *name { + "OpenNOW.exe" => "seen-OpenNOW.exe", + "opennow-core.exe" => "seen-opennow-core.exe", + "opennow-update-helper.exe" => "seen-opennow-update-helper.exe", + _ => unreachable!(), + }; + assert_eq!( + fs::read(plan.data_dir.join(seen)).unwrap(), + bytes, + "the replacement process did not observe {name}" + ); + } + assert!(!directory.path().join("previous").exists()); + assert_eq!( + read_outcome(&directory.path().join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::RolledBack + ); +} + +#[test] +fn flatpak_detection_accepts_environment_or_sandbox_marker() { + use std::ffi::OsStr; + + let directory = tempfile::tempdir().unwrap(); + let info = directory.path().join(".flatpak-info"); + assert!(!is_flatpak_installation(None, &info)); + assert!(!is_flatpak_installation(Some(OsStr::new("")), &info)); + assert!(is_flatpak_installation( + Some(OsStr::new("io.github.opencloudgaming.OpenNOW")), + &info + )); + fs::write( + &info, + b"[Application]\nname=io.github.opencloudgaming.OpenNOW\n", + ) + .unwrap(); + assert!(is_flatpak_installation(None, &info)); + assert!(is_flatpak_installation(Some(OsStr::new("")), &info)); +} + +fn signed_manifest(asset: &str, bytes: &[u8]) -> (verification::UpdateManifest, SigningKey) { + let key = SigningKey::from_bytes(&[91; 32]); + let mut manifest = verification::UpdateManifest { + schema_version: 1, + version: "1.2.3".to_owned(), + asset: asset.to_owned(), + size: bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(bytes)), + signature: String::new(), + }; + manifest.signature = base64::engine::general_purpose::STANDARD.encode( + key.sign(verification::signature_payload(&manifest).as_bytes()) + .to_bytes(), + ); + (manifest, key) +} + +#[test] +fn signature_rejects_tampering_wrong_key_and_invalid_schema() { + let (manifest, key) = signed_manifest("OpenNOW.zip", b"valid package"); + verification::verify_manifest(&manifest, &key.verifying_key()).unwrap(); + assert!( + verification::verify_manifest( + &manifest, + &SigningKey::from_bytes(&[90; 32]).verifying_key() + ) + .is_err() + ); + let mut tampered = manifest.clone(); + tampered.version = "1.2.4".to_owned(); + assert!(verification::verify_manifest(&tampered, &key.verifying_key()).is_err()); + let mut invalid = manifest; + invalid.schema_version = 2; + assert!(verification::verify_manifest(&invalid, &key.verifying_key()).is_err()); +} + +#[test] +fn package_verification_rejects_tampering_and_wrong_filename() { + let dir = TempDir::new().unwrap(); + let bytes = b"valid package"; + let (manifest, _) = signed_manifest("OpenNOW.zip", bytes); + let package = dir.path().join("OpenNOW.zip"); + fs::write(&package, bytes).unwrap(); + verification::verify_package(&package, &manifest).unwrap(); + fs::write(&package, b"evil! package").unwrap(); + assert!( + verification::verify_package(&package, &manifest) + .unwrap_err() + .contains("SHA256") + ); + let other = dir.path().join("Other.zip"); + fs::write(&other, bytes).unwrap(); + assert!(verification::verify_package(&other, &manifest).is_err()); +} + +fn zip_file(path: &Path, entries: &[(&str, &[u8])]) { + let mut zip = zip::ZipWriter::new(File::create(path).unwrap()); + for (name, bytes) in entries { + zip.start_file( + *name, + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + zip.write_all(bytes).unwrap(); + } + zip.finish().unwrap(); +} + +#[test] +fn archive_rejects_traversal_case_aliases_and_partial_extraction() { + for entries in [ + vec![ + ("safe.txt", b"first".as_slice()), + ("../escaped", b"evil".as_slice()), + ], + vec![ + ("Bin/app", b"first".as_slice()), + ("bin/App", b"evil".as_slice()), + ], + vec![ + ("safe.txt", b"first".as_slice()), + ("C:/evil", b"evil".as_slice()), + ], + vec![ + ("safe.txt", b"first".as_slice()), + ("dir/CON.txt", b"evil".as_slice()), + ], + ] { + let dir = TempDir::new().unwrap(); + let package = dir.path().join("update.zip"); + zip_file(&package, &entries); + let destination = dir.path().join("extracted"); + assert!(archive::extract_zip(&package, &destination, false).is_err()); + assert!(!destination.exists()); + assert!(!dir.path().join("escaped").exists()); + } +} + +#[test] +fn archive_rejects_corruption_without_leaving_a_partial_tree() { + let dir = TempDir::new().unwrap(); + let package = dir.path().join("update.zip"); + zip_file(&package, &[("bin/app", b"candidate executable")]); + let mut bytes = fs::read(&package).unwrap(); + let offset = bytes + .windows(20) + .position(|window| window == b"candidate executable") + .unwrap(); + bytes[offset] ^= 1; + fs::write(&package, bytes).unwrap(); + let destination = dir.path().join("extracted"); + assert!(archive::extract_zip(&package, &destination, false).is_err()); + assert!(!destination.exists()); +} + +#[cfg(unix)] +#[test] +fn framework_symlinks_are_preserved_and_escaping_links_are_rejected() { + let dir = TempDir::new().unwrap(); + let source = dir.path().join("OpenNOW.app"); + let framework = source.join("Contents/Frameworks/Qt.framework"); + fs::create_dir_all(framework.join("Versions/A")).unwrap(); + fs::write(framework.join("Versions/A/Qt"), b"framework").unwrap(); + std::os::unix::fs::symlink("A", framework.join("Versions/Current")).unwrap(); + std::os::unix::fs::symlink("Versions/Current/Qt", framework.join("Qt")).unwrap(); + let copy = dir.path().join("copied.app"); + bundle::copy_tree( + &source, + ©, + &mut bundle::CopyBudget::new(), + bundle::CopyPolicy::Bundle, + ) + .unwrap(); + assert!( + fs::symlink_metadata(copy.join("Contents/Frameworks/Qt.framework/Qt")) + .unwrap() + .is_symlink() + ); + assert_eq!( + fs::read(copy.join("Contents/Frameworks/Qt.framework/Qt")).unwrap(), + b"framework" + ); + fs::write(dir.path().join("secret"), b"outside").unwrap(); + std::os::unix::fs::symlink("../secret", source.join("escape")).unwrap(); + let bad = dir.path().join("bad.app"); + assert!( + bundle::copy_tree( + &source, + &bad, + &mut bundle::CopyBudget::new(), + bundle::CopyPolicy::Bundle + ) + .is_err() + ); + assert!(!bad.exists()); +} + +fn fixture_executable() -> &'static Path { + static FIXTURE: OnceLock = OnceLock::new(); + FIXTURE + .get_or_init(|| { + let directory = TempDir::new().unwrap(); + let source = directory.path().join("fixture.rs"); + fs::write( + &source, + r#"fn main() { + if let Ok(path) = std::env::var("OPENNOW_FIXTURE_CHILD_PID") { + let child = std::process::Command::new(std::env::current_exe().unwrap()).env_remove("OPENNOW_FIXTURE_CHILD_PID").spawn().unwrap(); + let path = std::path::PathBuf::from(path); + let temporary = path.with_extension("tmp"); + std::fs::write(&temporary, child.id().to_string()).unwrap(); + std::fs::rename(temporary, path).unwrap(); + } + let lifetime = std::env::var("OPENNOW_FIXTURE_LIFETIME_MS").ok().and_then(|value| value.parse().ok()).unwrap_or(900); + std::thread::sleep(std::time::Duration::from_millis(lifetime)); + }"#, + ) + .unwrap(); + let mut compiler = Command::new("rustc"); + #[cfg(all(windows, target_env = "msvc"))] + { + let version = Command::new("rustc").arg("-vV").output().unwrap(); + assert!(version.status.success()); + let version = String::from_utf8(version.stdout).unwrap(); + let host = version.lines().find_map(|line| line.strip_prefix("host: ")).expect("rustc host triple"); + if host.ends_with("-windows-msvc") { + let variable = format!("CARGO_TARGET_{}_LINKER", host.replace('-', "_").to_ascii_uppercase()); + if let Some(linker) = std::env::var_os(variable) { + let mut option = std::ffi::OsString::from("linker="); + option.push(linker); + compiler.arg("-C").arg(option); + } + } + } + let output = compiler.arg(&source) + .arg("-o") + .arg(directory.path().join("fixture.exe")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + directory + }) + .path() +} + +#[test] +fn owned_application_accepts_descendants_and_stops_the_entire_tree() { + let directory = TempDir::new().unwrap(); + let pid_file = directory.path().join("child.pid"); + let mut command = Command::new(fixture_executable().join("fixture.exe")); + command + .env("OPENNOW_FIXTURE_CHILD_PID", &pid_file) + .env("OPENNOW_FIXTURE_LIFETIME_MS", "60000"); + let mut application = process::OwnedApplication::start(&mut command).unwrap(); + let start = Instant::now(); + while !pid_file.exists() { + assert!(start.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(10)); + } + let pid = fs::read_to_string(&pid_file).unwrap().parse().unwrap(); + let descendant = ProcessIdentity::capture(pid).unwrap(); + assert!(application.owns(&descendant).unwrap()); + assert!( + !application + .owns(&ProcessIdentity::capture(std::process::id()).unwrap()) + .unwrap() + ); + application.stop().unwrap(); + assert!(!descendant.is_running().unwrap()); + assert!(!application.identity.is_running().unwrap()); +} + +#[cfg(unix)] +#[test] +fn zip_framework_links_are_preserved_but_external_links_are_rejected() { + let directory = TempDir::new().unwrap(); + let package = directory.path().join("app.zip"); + let mut zip = zip::ZipWriter::new(File::create(&package).unwrap()); + zip.start_file( + "Contents/Frameworks/Qt/Versions/A/Qt", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(b"framework").unwrap(); + zip.add_symlink( + "Contents/Frameworks/Qt/Versions/Current", + "A", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.add_symlink( + "Contents/Frameworks/Qt/Qt", + "Versions/Current/Qt", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.finish().unwrap(); + let payload = directory.path().join("payload"); + archive::extract_zip(&package, &payload, true).unwrap(); + assert_eq!( + fs::read(payload.join("Contents/Frameworks/Qt/Qt")).unwrap(), + b"framework" + ); + let mut zip = zip::ZipWriter::new(File::create(&package).unwrap()); + zip.add_symlink( + "escape", + "../outside", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.finish().unwrap(); + assert!(archive::extract_zip(&package, &directory.path().join("bad"), true).is_err()); + assert!(!directory.path().join("bad").exists()); +} + +#[cfg(unix)] +#[test] +fn fixture_plan_uses_canonical_installation_paths() { + let directory = TempDir::new().unwrap(); + let target = directory.path().join("actual"); + fs::create_dir(&target).unwrap(); + let alias = directory.path().join("alias"); + std::os::unix::fs::symlink(&target, &alias).unwrap(); + let plan = plan(&alias, InstallKind::WindowsPortable); + assert_eq!( + plan.target, + fs::canonicalize(&target).unwrap().join("installed") + ); + assert_eq!( + plan.application_executable, + plan.target.join("bin/OpenNOW.exe") + ); +} + +fn plan(directory: &Path, kind: InstallKind) -> Plan { + let directory = fs::canonicalize(directory).unwrap(); + let target = directory.join(if kind == InstallKind::AppImage { + "OpenNOW.AppImage" + } else { + "installed" + }); + let application = if kind == InstallKind::AppImage { + target.clone() + } else { + target.join("bin/OpenNOW.exe") + }; + Plan { + schema_version: 1, + version: "1.2.3".to_owned(), + kind, + package: directory.join("package.zip"), + target, + application_executable: application, + data_dir: directory.join("settings"), + processes: vec![], + nonce: "a".repeat(64), + managed_identity: None, + } +} + +fn stage_native_executable(path: &Path) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::copy(fixture_executable().join("fixture.exe"), path).unwrap(); + make_executable(path).unwrap(); +} + +#[test] +fn failed_appimage_restart_restores_previous_binary() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::AppImage); + fs::write(&plan.target, b"original appimage").unwrap(); + let candidate = directory.path().join("candidate"); + fs::write(&candidate, b"not an executable").unwrap(); + make_executable(&candidate).unwrap(); + assert!( + replace_and_restart( + &plan, + directory.path(), + &candidate, + Duration::from_millis(100) + ) + .is_err() + ); + assert_eq!(fs::read(&plan.target).unwrap(), b"original appimage"); + assert_eq!( + fs::read(directory.path().join("failed")).unwrap(), + b"not an executable" + ); + assert_eq!( + read_outcome(&directory.path().join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::RolledBack + ); +} + +#[test] +fn failed_directory_replacement_restores_all_original_files_and_settings() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + fs::create_dir_all(plan.application_executable.parent().unwrap()).unwrap(); + fs::write(&plan.application_executable, b"original executable").unwrap(); + fs::write(plan.target.join("original.dll"), b"original dependency").unwrap(); + fs::create_dir(&plan.data_dir).unwrap(); + fs::write(plan.data_dir.join("settings.json"), b"keep settings").unwrap(); + let missing = directory.path().join("missing"); + assert!( + replace_and_restart( + &plan, + directory.path(), + &missing, + Duration::from_millis(100) + ) + .is_err() + ); + assert_eq!( + fs::read(&plan.application_executable).unwrap(), + b"original executable" + ); + assert_eq!( + fs::read(plan.target.join("original.dll")).unwrap(), + b"original dependency" + ); + assert_eq!( + fs::read(plan.data_dir.join("settings.json")).unwrap(), + b"keep settings" + ); +} + +#[test] +fn failed_shutdown_and_failed_outcome_write_never_authorize_rollback() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::AppImage); + fs::write(&plan.target, b"running candidate").unwrap(); + fs::write(directory.path().join("previous"), b"previous application").unwrap(); + write_outcome( + directory.path(), + &plan, + OutcomeStatus::AwaitingStartup, + "candidate running", + None, + None, + ) + .unwrap(); + let before = fs::read(directory.path().join("outcome.json")).unwrap(); + let mut attempted_record = false; + let failure = failed_restart( + "startup failed".to_owned(), + Err("process shutdown failed".to_owned()), + |_| { + attempted_record = true; + Err("disk full".to_owned()) + }, + ); + assert!(attempted_record); + assert!(matches!(failure, RestartFailure::ProcessesMayBeRunning(_))); + assert!(recover_replacement(&plan, directory.path(), failure).is_err()); + assert_eq!(fs::read(&plan.target).unwrap(), b"running candidate"); + assert_eq!( + fs::read(directory.path().join("previous")).unwrap(), + b"previous application" + ); + assert_eq!( + fs::read(directory.path().join("outcome.json")).unwrap(), + before + ); + assert!(!directory.path().join("failed").exists()); +} + +#[test] +fn interrupted_startup_without_identity_does_not_roll_back_under_unknown_processes() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::AppImage); + fs::write(&plan.target, b"candidate").unwrap(); + fs::write(directory.path().join("previous"), b"previous").unwrap(); + write_outcome( + directory.path(), + &plan, + OutcomeStatus::AwaitingStartup, + "startup interrupted", + None, + None, + ) + .unwrap(); + assert!( + apply(&plan, directory.path()) + .unwrap_err() + .contains("unsafe") + ); + assert_eq!(fs::read(&plan.target).unwrap(), b"candidate"); + assert_eq!( + fs::read(directory.path().join("previous")).unwrap(), + b"previous" + ); +} + +#[cfg(unix)] +#[test] +fn failed_start_cleans_the_forked_group_before_reaping_the_parent() { + use std::os::unix::process::CommandExt; + let directory = TempDir::new().unwrap(); + let pid_file = directory.path().join("child.pid"); + let mut child = Command::new(fixture_executable().join("fixture.exe")) + .process_group(0) + .env("OPENNOW_FIXTURE_CHILD_PID", &pid_file) + .env("OPENNOW_FIXTURE_LIFETIME_MS", "60000") + .spawn() + .unwrap(); + let start = Instant::now(); + while !pid_file.exists() { + assert!(start.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(10)); + } + let descendant = + ProcessIdentity::capture(fs::read_to_string(pid_file).unwrap().parse().unwrap()).unwrap(); + let failure = + process::failed_start(&mut child, "identity capture failed after fork".to_owned()); + assert!( + matches!(failure, RestartFailure::RollbackSafe(_)), + "{failure}" + ); + assert!(!descendant.is_running().unwrap()); + assert!(child.try_wait().unwrap().is_some()); +} + +#[test] +fn spawn_without_startup_acknowledgement_rolls_back_the_complete_directory() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + fs::create_dir_all(plan.application_executable.parent().unwrap()).unwrap(); + fs::write(&plan.application_executable, b"old executable").unwrap(); + let payload = directory.path().join("payload"); + stage_native_executable(&payload.join("bin/OpenNOW.exe")); + assert!( + replace_and_restart(&plan, directory.path(), &payload, Duration::from_millis(30)) + .unwrap_err() + .contains("acknowledge") + ); + assert_eq!( + fs::read(&plan.application_executable).unwrap(), + b"old executable" + ); + assert!(directory.path().join("failed/bin/OpenNOW.exe").is_file()); +} + +#[test] +fn portable_directory_replacement_requires_ack_and_preserves_settings() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + fs::create_dir_all(plan.application_executable.parent().unwrap()).unwrap(); + fs::write(&plan.application_executable, b"old executable").unwrap(); + fs::write(plan.target.join("old-only.dll"), b"old dependency").unwrap(); + fs::create_dir(&plan.data_dir).unwrap(); + fs::write(plan.data_dir.join("settings.json"), b"unchanged").unwrap(); + let payload = directory.path().join("payload"); + stage_native_executable(&payload.join("bin/OpenNOW.exe")); + fs::write(payload.join("new-only.dll"), b"new dependency").unwrap(); + let acknowledgement_directory = directory.path().to_path_buf(); + let nonce = plan.nonce.clone(); + let version = plan.version.clone(); + let worker = std::thread::spawn(move || { + let start = Instant::now(); + loop { + if let Some(outcome) = + read_outcome(&acknowledgement_directory.join("outcome.json")).unwrap() + { + if let (OutcomeStatus::AwaitingStartup, Some(application)) = + (outcome.status, outcome.restarted_process) + { + atomic_json( + &acknowledgement_directory.join("ack.json"), + &Acknowledgement { + nonce, + version, + application, + }, + ) + .unwrap(); + break; + } + } + assert!(start.elapsed() < Duration::from_secs(10)); + std::thread::sleep(Duration::from_millis(10)); + } + }); + replace_and_restart(&plan, directory.path(), &payload, Duration::from_secs(10)).unwrap(); + worker.join().unwrap(); + assert!(!plan.target.join("old-only.dll").exists()); + assert_eq!( + fs::read(plan.target.join("new-only.dll")).unwrap(), + b"new dependency" + ); + assert_eq!( + fs::read(plan.data_dir.join("settings.json")).unwrap(), + b"unchanged" + ); + assert!(!directory.path().join("previous").exists()); + assert_eq!( + read_outcome(&directory.path().join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::Completed + ); + std::thread::sleep(Duration::from_millis(950)); +} + +#[test] +fn acknowledged_application_remains_running_after_owner_drop() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + stage_native_executable(&plan.application_executable); + let acknowledgement_directory = directory.path().to_path_buf(); + let nonce = plan.nonce.clone(); + let version = plan.version.clone(); + let worker = std::thread::spawn(move || { + let start = Instant::now(); + loop { + if let Some(outcome) = + read_outcome(&acknowledgement_directory.join("outcome.json")).unwrap() + { + if let (OutcomeStatus::AwaitingStartup, Some(application)) = + (outcome.status, outcome.restarted_process) + { + atomic_json( + &acknowledgement_directory.join("ack.json"), + &Acknowledgement { + nonce, + version, + application: application.clone(), + }, + ) + .unwrap(); + return application; + } + } + assert!(start.elapsed() < Duration::from_secs(10)); + std::thread::sleep(Duration::from_millis(10)); + } + }); + restart_and_acknowledge(&plan, directory.path(), Duration::from_secs(10)).unwrap(); + let application = worker.join().unwrap(); + assert!(application.is_running().unwrap()); + std::thread::sleep(Duration::from_millis(100)); + assert!(application.is_running().unwrap()); + let deadline = Instant::now(); + while application.is_running().unwrap() { + assert!(deadline.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(20)); + } +} + +#[test] +fn interrupted_replacement_recovers_without_the_target_existing() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::AppImage); + fs::write(directory.path().join("previous"), b"original bytes").unwrap(); + write_outcome( + directory.path(), + &plan, + OutcomeStatus::Installing, + "interrupted", + None, + None, + ) + .unwrap(); + apply(&plan, directory.path()).unwrap(); + assert_eq!(fs::read(&plan.target).unwrap(), b"original bytes"); + assert_eq!( + read_outcome(&directory.path().join("outcome.json")) + .unwrap() + .unwrap() + .status, + OutcomeStatus::RolledBack + ); +} + +#[test] +fn repeated_helper_invocation_preserves_cleaned_up_terminal_outcomes() { + for status in [OutcomeStatus::Completed, OutcomeStatus::RolledBack] { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::AppImage); + atomic_json(&directory.path().join("plan.json"), &plan).unwrap(); + write_outcome( + directory.path(), + &plan, + status, + "terminal result", + Some(plan.version.clone()), + None, + ) + .unwrap(); + assert!(!plan.package.exists()); + let outcome = fs::read(directory.path().join("outcome.json")).unwrap(); + run_helper(&directory.path().join("plan.json")).unwrap(); + run_helper(&directory.path().join("plan.json")).unwrap(); + assert_eq!( + fs::read(directory.path().join("outcome.json")).unwrap(), + outcome + ); + } +} + +#[test] +fn advisory_lock_distinguishes_live_helper_from_stale_outcomes() { + let directory = TempDir::new().unwrap(); + let prepared = PreparedUpdate { + plan_path: directory.path().join("plan.json"), + outcome_path: directory.path().join("outcome.json"), + version: "1.2.3".to_owned(), + }; + assert!(!helper_is_running(&prepared).unwrap()); + let lock = OpenOptions::new() + .create_new(true) + .write(true) + .open(directory.path().join("apply.lock")) + .unwrap(); + lock.lock_exclusive().unwrap(); + assert!(helper_is_running(&prepared).unwrap()); + FileExt::unlock(&lock).unwrap(); + assert!(!helper_is_running(&prepared).unwrap()); +} + +#[cfg(windows)] +#[test] +fn atomic_metadata_publication_waits_for_a_windows_reader() { + use std::os::windows::fs::OpenOptionsExt; + use std::sync::mpsc::{RecvTimeoutError, channel}; + use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ; + + for share_mode in [None, Some(FILE_SHARE_READ)] { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("outcome.json"); + atomic_json(&path, &"prepared").unwrap(); + let mut options = OpenOptions::new(); + options.read(true); + if let Some(mode) = share_mode { + options.share_mode(mode); + } + let reader = options.open(&path).unwrap(); + std::thread::scope(|scope| { + let (sender, receiver) = channel(); + let path = &path; + scope.spawn(move || { + sender.send(atomic_json(path, &"waitingForExit")).unwrap(); + }); + let pending = receiver.recv_timeout(Duration::from_millis(100)); + assert_eq!(fs::read(path).unwrap(), b"\"prepared\""); + drop(reader); + assert!( + matches!(pending, Err(RecvTimeoutError::Timeout)), + "{pending:?}" + ); + receiver + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap(); + }); + assert_eq!(read_json::(&path, 1024).unwrap(), "waitingForExit"); + assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1); + } +} + +#[cfg(windows)] +#[test] +fn atomic_metadata_publication_fails_safely_for_a_persistent_windows_reader() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("outcome.json"); + atomic_json(&path, &"prepared").unwrap(); + let reader = File::open(&path).unwrap(); + let start = Instant::now(); + assert!(atomic_json(&path, &"waitingForExit").is_err()); + assert!(start.elapsed() < Duration::from_secs(5)); + assert_eq!(read_json::(&path, 1024).unwrap(), "prepared"); + assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1); + drop(reader); + atomic_json(&path, &"waitingForExit").unwrap(); + assert_eq!(read_json::(&path, 1024).unwrap(), "waitingForExit"); +} + +#[cfg(unix)] +#[test] +fn transaction_lock_releases_ownership_with_an_inherited_descriptor_open() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("apply.lock"); + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&path) + .unwrap(); + let owner = TransactionLock::acquire(file).unwrap(); + let inherited = owner.0.try_clone().unwrap(); + let next = OpenOptions::new().write(true).open(&path).unwrap(); + assert_eq!( + next.try_lock_exclusive().unwrap_err().raw_os_error(), + fs2::lock_contended_error().raw_os_error() + ); + drop(owner); + next.try_lock_exclusive().unwrap(); + FileExt::unlock(&next).unwrap(); + drop(inherited); +} + +#[test] +fn portable_qt_runtime_roots_are_replaced_without_preserving_stale_libraries() { + let directory = TempDir::new().unwrap(); + let mut plan = plan(directory.path(), InstallKind::WindowsPortable); + plan.data_dir = plan.target.join("profile"); + fs::create_dir_all(&plan.data_dir).unwrap(); + fs::write(plan.data_dir.join("settings.json"), b"user settings").unwrap(); + let payload = directory.path().join("payload"); + for root in ["bin", "plugins", "qml", "share", "translations"] { + fs::create_dir_all(plan.target.join(root)).unwrap(); + fs::write(plan.target.join(root).join("old-runtime"), b"old").unwrap(); + fs::create_dir_all(payload.join(root)).unwrap(); + fs::write(payload.join(root).join("new-runtime"), b"new").unwrap(); + } + let paths = preserve_portable_data(&plan, &payload).unwrap(); + assert_eq!(paths.len(), 1); + assert_eq!( + fs::read(payload.join("profile/settings.json")).unwrap(), + b"user settings" + ); + for root in ["bin", "plugins", "qml", "share", "translations"] { + assert!(!payload.join(root).join("old-runtime").exists()); + assert_eq!( + fs::read(payload.join(root).join("new-runtime")).unwrap(), + b"new" + ); + } +} + +#[test] +fn portable_explicit_profile_inside_qt_runtime_is_preserved_and_overlap_rejected() { + let directory = TempDir::new().unwrap(); + let mut plan = plan(directory.path(), InstallKind::WindowsPortable); + plan.data_dir = plan.target.join("plugins/profile"); + fs::create_dir_all(&plan.data_dir).unwrap(); + fs::write(plan.data_dir.join("settings.json"), b"user settings").unwrap(); + let payload = directory.path().join("payload"); + fs::create_dir_all(payload.join("plugins")).unwrap(); + let paths = preserve_portable_data(&plan, &payload).unwrap(); + assert_eq!(paths.len(), 1); + assert_eq!( + fs::read(payload.join("plugins/profile/settings.json")).unwrap(), + b"user settings" + ); + assert!( + preserve_portable_data(&plan, &payload) + .unwrap_err() + .contains("overlaps preserved user data") + ); + assert_eq!( + fs::read(plan.data_dir.join("settings.json")).unwrap(), + b"user settings" + ); +} + +#[test] +fn portable_nested_profile_and_extra_root_files_are_preserved() { + let directory = TempDir::new().unwrap(); + let mut plan = plan(directory.path(), InstallKind::WindowsPortable); + plan.data_dir = plan.target.join("bin/profile"); + fs::create_dir_all(&plan.data_dir).unwrap(); + fs::write(plan.data_dir.join("settings.json"), b"custom profile").unwrap(); + fs::write(plan.target.join("personal-notes.txt"), b"user-owned notes").unwrap(); + let payload = directory.path().join("payload"); + fs::create_dir_all(payload.join("bin")).unwrap(); + let paths = preserve_portable_data(&plan, &payload).unwrap(); + assert_eq!(paths.len(), 2); + assert_eq!( + fs::read(payload.join("bin/profile/settings.json")).unwrap(), + b"custom profile" + ); + assert_eq!( + fs::read(payload.join("personal-notes.txt")).unwrap(), + b"user-owned notes" + ); +} + +#[cfg(unix)] +#[test] +fn portable_profile_preserves_restrictive_unix_permissions_and_ownership() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let directory = TempDir::new().unwrap(); + let source = directory.path().join("profile"); + fs::create_dir_all(source.join("credentials")).unwrap(); + fs::write( + source.join("credentials/account.json"), + b"private fixture data", + ) + .unwrap(); + for path in [&source, &source.join("credentials")] { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); + } + fs::set_permissions( + source.join("credentials/account.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let destination = directory.path().join("preserved-profile"); + copy_user_data(&source, &destination, &mut bundle::CopyBudget::new()).unwrap(); + for relative in ["", "credentials", "credentials/account.json"] { + let before = fs::metadata(source.join(relative)).unwrap(); + let after = fs::metadata(destination.join(relative)).unwrap(); + assert_eq!(before.mode() & 0o7777, after.mode() & 0o7777); + assert_eq!(before.uid(), after.uid()); + assert_eq!(before.gid(), after.gid()); + } + assert_eq!( + fs::metadata(destination.join("credentials/account.json")) + .unwrap() + .mode() + & 0o777, + 0o600 + ); +} + +#[cfg(unix)] +#[test] +fn transaction_directory_is_private_at_creation() { + use std::os::unix::fs::PermissionsExt; + let directory = TempDir::new().unwrap(); + let transaction = directory.path().join("transaction"); + security::create_private_directory(&transaction).unwrap(); + assert_eq!( + fs::metadata(transaction).unwrap().permissions().mode() & 0o777, + 0o700 + ); +} + +#[test] +fn preserved_roots_share_one_entry_and_byte_budget() { + let directory = TempDir::new().unwrap(); + for name in ["first", "second"] { + fs::create_dir(directory.path().join(name)).unwrap(); + fs::write(directory.path().join(name).join("data"), b"123456").unwrap(); + } + let mut bytes = bundle::CopyBudget::with_limits(10, 10); + copy_user_data( + &directory.path().join("first"), + &directory.path().join("copy-first"), + &mut bytes, + ) + .unwrap(); + assert!( + copy_user_data( + &directory.path().join("second"), + &directory.path().join("copy-second"), + &mut bytes + ) + .unwrap_err() + .contains("byte budget") + ); + assert!(!directory.path().join("copy-second").exists()); + let mut entries = bundle::CopyBudget::with_limits(3, 100); + copy_user_data( + &directory.path().join("first"), + &directory.path().join("entries-first"), + &mut entries, + ) + .unwrap(); + assert!( + copy_user_data( + &directory.path().join("second"), + &directory.path().join("entries-second"), + &mut entries + ) + .unwrap_err() + .contains("entry limit") + ); + assert!(!directory.path().join("entries-second").exists()); +} + +#[test] +fn streaming_copy_rejects_file_growth_without_writing_past_the_budget() { + let directory = TempDir::new().unwrap(); + let source = directory.path().join("live-data"); + fs::write(&source, b"before").unwrap(); + let mut input = File::open(&source).unwrap(); + let observed_size = input.metadata().unwrap().len(); + OpenOptions::new() + .append(true) + .open(&source) + .unwrap() + .write_all(b"appended after metadata") + .unwrap(); + let mut output = Vec::new(); + assert!( + copy_bounded(&mut input, &mut output, observed_size) + .unwrap_err() + .contains("streaming byte budget") + ); + assert_eq!(output.len() as u64, observed_size); + assert_eq!(output, b"before"); +} + +#[test] +fn preserved_roots_are_rediscovered_and_rebudgeted_after_shutdown() { + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + fs::create_dir_all(plan.target.join("bin")).unwrap(); + fs::write(plan.target.join("first-profile"), b"old").unwrap(); + let payload = directory.path().join("payload"); + fs::create_dir(&payload).unwrap(); + let prepared = preserve_portable_data(&plan, &payload).unwrap(); + fs::write(plan.target.join("first-profile"), b"latest").unwrap(); + fs::write(plan.target.join("created-during-shutdown"), b"new").unwrap(); + for (_, destination) in prepared { + remove_path(&destination).unwrap(); + } + let refreshed = preserve_portable_data(&plan, &payload).unwrap(); + assert_eq!(refreshed.len(), 2); + assert_eq!(fs::read(payload.join("first-profile")).unwrap(), b"latest"); + assert_eq!( + fs::read(payload.join("created-during-shutdown")).unwrap(), + b"new" + ); +} + +#[cfg(windows)] +#[test] +fn windows_busy_file_rejects_replacement_without_losing_original() { + use std::os::windows::fs::OpenOptionsExt; + let directory = TempDir::new().unwrap(); + let plan = plan(directory.path(), InstallKind::WindowsPortable); + fs::create_dir_all(plan.application_executable.parent().unwrap()).unwrap(); + fs::write(&plan.application_executable, b"original executable").unwrap(); + let _busy = OpenOptions::new() + .read(true) + .share_mode(0) + .open(&plan.application_executable) + .unwrap(); + let payload = directory.path().join("payload"); + stage_native_executable(&payload.join("bin/OpenNOW.exe")); + assert!( + replace_and_restart(&plan, directory.path(), &payload, Duration::from_millis(30)).is_err() + ); + assert!(plan.application_executable.exists()); + assert!(!directory.path().join("previous").exists()); +} + +#[test] +fn running_process_identity_has_a_bounded_wait() { + let identity = ProcessIdentity::capture(std::process::id()).unwrap(); + assert!(identity.is_running().unwrap()); + assert!( + process::wait_for_exit(&[identity], Duration::from_millis(20)) + .unwrap_err() + .contains("Timed out") + ); +} diff --git a/external/opennow/native/opennow-core/src/update_apply/verification.rs b/external/opennow/native/opennow-core/src/update_apply/verification.rs new file mode 100644 index 000000000..88f9e3760 --- /dev/null +++ b/external/opennow/native/opennow-core/src/update_apply/verification.rs @@ -0,0 +1,134 @@ +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use ed25519_dalek::{Signature, VerifyingKey}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::io::Read; +use std::path::Path; + +pub const MAXIMUM_MANIFEST_BYTES: u64 = 64 * 1024; +pub const MAXIMUM_UPDATE_BYTES: u64 = 4 * 1024 * 1024 * 1024; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateManifest { + pub schema_version: u32, + pub version: String, + pub asset: String, + pub size: u64, + pub sha256: String, + pub signature: String, +} + +pub fn embedded_update_key() -> Result { + let encoded = option_env!("OPENNOW_UPDATE_ED25519_PUBLIC_KEY") + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or("This build has no pinned update signing key")?; + decode_verifying_key(encoded) +} + +pub fn decode_verifying_key(encoded: &str) -> Result { + let bytes: [u8; 32] = BASE64 + .decode(encoded) + .map_err(|_| "Pinned update signing key is not valid base64")? + .try_into() + .map_err(|_| "Pinned update signing key has the wrong length")?; + VerifyingKey::from_bytes(&bytes).map_err(|_| "Pinned update signing key is invalid".to_owned()) +} + +pub fn signature_payload(manifest: &UpdateManifest) -> String { + format!( + "OpenNOW update manifest v1\nversion={}\nasset={}\nsize={}\nsha256={}\n", + manifest.version.trim_start_matches('v'), + manifest.asset, + manifest.size, + manifest.sha256.to_ascii_lowercase() + ) +} + +pub fn verify_manifest(manifest: &UpdateManifest, key: &VerifyingKey) -> Result<(), String> { + if manifest.schema_version != 1 + || semver::Version::parse( + manifest + .version + .strip_prefix('v') + .unwrap_or(&manifest.version), + ) + .is_err() + || !safe_asset_name(&manifest.asset) + || manifest.size == 0 + || manifest.size > MAXIMUM_UPDATE_BYTES + || manifest.sha256.len() != 64 + || !manifest.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err("Signed update metadata fields are invalid".to_owned()); + } + let bytes = BASE64 + .decode(&manifest.signature) + .map_err(|_| "Update signature is not valid base64")?; + let signature = + Signature::from_slice(&bytes).map_err(|_| "Update signature has the wrong length")?; + key.verify_strict(signature_payload(manifest).as_bytes(), &signature) + .map_err(|_| "Update manifest signature is invalid".to_owned()) +} + +pub fn safe_asset_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 240 + && !value.contains("..") + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_')) +} + +pub fn verify_signed_manifest(bytes: &[u8]) -> Result { + if bytes.len() as u64 > MAXIMUM_MANIFEST_BYTES { + return Err("Update manifest exceeds its size limit".to_owned()); + } + let manifest = serde_json::from_slice(bytes) + .map_err(|error| format!("Invalid update manifest: {error}"))?; + verify_manifest(&manifest, &embedded_update_key()?)?; + Ok(manifest) +} + +pub fn verify_package(path: &Path, manifest: &UpdateManifest) -> Result<(), String> { + if path.file_name().and_then(|name| name.to_str()) != Some(manifest.asset.as_str()) { + return Err("Update package name does not match its signed manifest".to_owned()); + } + let metadata = std::fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if !metadata.is_file() || metadata.len() != manifest.size { + return Err( + "Update package size or file type does not match its signed manifest".to_owned(), + ); + } + let mut file = File::open(path).map_err(|error| error.to_string())?; + let (size, hash) = hash_reader(&mut file, MAXIMUM_UPDATE_BYTES)?; + if size != manifest.size || !hash.eq_ignore_ascii_case(&manifest.sha256) { + return Err("Update package SHA256 does not match its signed manifest".to_owned()); + } + Ok(()) +} + +pub(super) fn hash_reader(reader: &mut impl Read, maximum: u64) -> Result<(u64, String), String> { + let mut hasher = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + let length = reader + .read(&mut buffer) + .map_err(|error| error.to_string())?; + if length == 0 { + break; + } + total = total + .checked_add(length as u64) + .ok_or("Update size overflow")?; + if total > maximum { + return Err("Update data exceeds its size limit".to_owned()); + } + hasher.update(&buffer[..length]); + } + Ok((total, format!("{:x}", hasher.finalize()))) +} diff --git a/external/opennow/native/opennow-core/src/updater.rs b/external/opennow/native/opennow-core/src/updater.rs new file mode 100644 index 000000000..31408f456 --- /dev/null +++ b/external/opennow/native/opennow-core/src/updater.rs @@ -0,0 +1,2009 @@ +use crate::update_apply; +use crate::update_apply::verification::{ + MAXIMUM_MANIFEST_BYTES, MAXIMUM_UPDATE_BYTES, UpdateManifest, embedded_update_key, + safe_asset_name, verify_manifest, +}; +use reqwest::blocking::{Client, Response}; +use reqwest::header::{ACCEPT, USER_AGENT}; +use semver::Version; +use serde::Deserialize; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::Duration; + +const RELEASES_URL: &str = "https://api.github.com/repos/OpenCloudGaming/OpenNOW/releases"; +const RELEASES_PAGE: &str = "https://github.com/OpenCloudGaming/OpenNOW/releases"; +const RELEASE_ASSET_PREFIX: &str = "https://github.com/OpenCloudGaming/OpenNOW/releases/download/"; +const MAXIMUM_RELEASE_METADATA_BYTES: u64 = 8 * 1024 * 1024; + +#[derive(Clone)] +struct AvailableUpdate { + version: String, + asset: Asset, + manifest_url: String, +} + +#[derive(Clone)] +struct DownloadedUpdate { + version: String, + asset_name: String, + path: PathBuf, + size: u64, + sha256: String, +} + +#[derive(Clone, Copy)] +struct DownloadProgress { + received_bytes: u64, + total_bytes: u64, +} + +#[derive(Clone)] +struct State { + status: &'static str, + available_version: Option, + release_url: Option, + notes_version: Option, + notes: Option, + message: String, + last_checked_at: Option, + available: Option, + downloaded: Option, + download_progress: Option, + transaction: Option, +} + +#[derive(Deserialize)] +struct Release { + tag_name: String, + html_url: String, + body: Option, + draft: bool, + prerelease: bool, + assets: Vec, +} + +#[derive(Clone, Deserialize)] +struct Asset { + name: String, + browser_download_url: String, + size: u64, +} + +pub struct UpdaterService { + client: Client, + staging_dir: PathBuf, + data_dir: PathBuf, + operation: Mutex<()>, + state: Mutex, +} + +impl UpdaterService { + pub fn new(data_dir: &Path) -> Result { + let client = Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(300)) + .build() + .map_err(|error| error.to_string())?; + let staging_dir = data_dir.join("updates"); + fs::create_dir_all(&staging_dir) + .map_err(|error| format!("Could not create the update staging directory: {error}"))?; + let (transaction, status, message) = if let Some(message) = + update_apply::external_update_message() + { + (None, "unsupported", message.to_owned()) + } else { + match update_apply::windows_installer_replacement_message() { + Ok(Some(message)) => (None, "not-available", message.to_owned()), + Err(error) => ( + None, + "failed", + format!( + "Could not determine whether Windows Installer owns this installation: {error}" + ), + ), + Ok(None) => match read_prepared_update(&staging_dir) { + Ok(transaction) => ( + transaction, + "idle", + "Ready to check GitHub Releases".to_owned(), + ), + Err(error) => { + // A corrupt transaction must not block every future launch. Report it + // once for this session, then drop the unreadable persistence so the + // next startup begins clean. Diagnostics remain in the message. + let _ = fs::remove_file(staging_dir.join("active-apply.json")); + ( + None, + "failed", + format!("Could not recover update status: {error}"), + ) + } + }, + } + }; + Ok(Self { + client, + staging_dir, + data_dir: data_dir.to_path_buf(), + operation: Mutex::new(()), + state: Mutex::new(State { + status, + available_version: None, + release_url: None, + notes_version: None, + notes: None, + message, + last_checked_at: None, + available: None, + downloaded: None, + download_progress: None, + transaction, + }), + }) + } + + pub fn state(&self) -> Value { + let mut state = self.state.lock().expect("updater state poisoned"); + reconcile_transaction(&mut state, &self.staging_dir); + state_json(&state) + } + + pub fn installation_pending(&self) -> bool { + self.state(); + matches!( + self.state.lock().expect("updater state poisoned").status, + "preparing" + | "awaiting-exit" + | "applying" + | "restarting" + | "installing" + | "managed-pending" + ) + } + + pub fn request_failed(&self, message: &str) { + if update_apply::windows_installer_replacement_message() + .ok() + .flatten() + .is_some() + { + return; + } + let mut state = self.state.lock().expect("updater state poisoned"); + if !matches!( + state.status, + "unsupported" + | "checking" + | "downloading" + | "preparing" + | "awaiting-exit" + | "applying" + | "restarting" + | "installing" + ) { + state.message = message.to_owned(); + } + } + + pub fn check(&self, params: &Value) -> Result { + if update_apply::external_update_message().is_some() { + return Ok(self.state()); + } + if let Some(message) = update_apply::windows_installer_replacement_message()? { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "not-available"; + state.available = None; + state.available_version = None; + state.downloaded = None; + state.transaction = None; + state.message = message.to_owned(); + return Ok(state_json(&state)); + } + let _operation = self.begin_operation()?; + let channel = params["channel"] + .as_str() + .unwrap_or_else(|| crate::version::update_channel(crate::version::APPLICATION_VERSION)); + if !matches!(channel, "stable" | "nightly") { + return Err("Unsupported update channel".to_owned()); + } + { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "checking"; + state.transaction = None; + state.message = "Checking GitHub Releases…".to_owned(); + } + let _ = fs::remove_file(self.staging_dir.join("active-apply.json")); + let releases = self.fetch_releases(RELEASES_URL); + let releases = match releases { + Ok(releases) => releases, + Err(error) => { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "error"; + state.message = error.clone(); + return Err(error); + } + }; + let current = parse_version(crate::version::APPLICATION_VERSION) + .ok_or_else(|| "Current application version is invalid".to_owned())?; + let release = select_installable_release(&releases, channel, ¤t) + .or_else(|| select_release(&releases, channel, current)); + let mut state = self.state.lock().expect("updater state poisoned"); + state.last_checked_at = Some(now_ms()); + // Reading release notes is independent of installing a newer version. + // A development build can be ahead of every published release. + update_highlights(&mut state, select_latest_release(&releases, channel)); + if let Some(release) = release { + let version = release.tag_name.trim_start_matches('v').to_owned(); + let compatible = compatible_asset(&release.assets).cloned(); + let manifest_url = compatible + .as_ref() + .and_then(|asset| manifest_asset(release, asset)) + .map(|manifest| manifest.browser_download_url.clone()); + state.status = "available"; + state.available_version = Some(version.clone()); + state.release_url = trusted_release_url(&release.html_url) + .then(|| release.html_url.clone()) + .or_else(|| Some(RELEASES_PAGE.to_owned())); + state.available = + compatible + .zip(manifest_url) + .map(|(asset, manifest_url)| AvailableUpdate { + version: version.clone(), + asset, + manifest_url, + }); + state.message = if state.available.is_none() { + format!( + "OpenNOW {version} is available; this release has no signed Qt update package for this platform." + ) + } else if embedded_update_key().is_err() { + format!( + "OpenNOW {version} is available; this build has no pinned update signing key. Install a signed-update build manually once to enable future automatic updates." + ) + } else { + format!("OpenNOW {version} is available with signed update metadata.") + }; + } else { + let has_release = select_latest_release(&releases, channel).is_some(); + state.status = if has_release { + "not-available" + } else { + "error" + }; + state.available_version = None; + state.available = None; + state.message = if has_release { + "OpenNOW is up to date.".to_owned() + } else { + format!( + "No published releases were found for the {channel} update channel. Try again later." + ) + }; + } + restore_downloaded_status(&mut state); + Ok(state_json(&state)) + } + + fn fetch_releases(&self, releases_url: &str) -> Result, String> { + let mut releases = Vec::new(); + for (url, latest) in [ + (format!("{releases_url}?per_page=100"), false), + (format!("{releases_url}/latest"), true), + ] { + let response = self + .client + .get(url) + .timeout(Duration::from_secs(10)) + .header( + USER_AGENT, + concat!("OpenNOW-Qt/", env!("CARGO_PKG_VERSION")), + ) + .header(ACCEPT, "application/vnd.github+json") + .send() + .map_err(|error| friendly_network_error(&error.to_string()))?; + if latest && response.status() == reqwest::StatusCode::NOT_FOUND { + continue; + } + if !response.status().is_success() { + return Err(format!( + "GitHub Releases returned HTTP {}", + response.status().as_u16() + )); + } + let bytes = read_bounded(response, MAXIMUM_RELEASE_METADATA_BYTES)?; + if latest { + let release: Release = serde_json::from_slice(&bytes) + .map_err(|_| "GitHub Releases returned invalid metadata".to_owned())?; + releases.retain(|existing: &Release| existing.tag_name != release.tag_name); + releases.push(release); + } else { + releases = serde_json::from_slice(&bytes) + .map_err(|_| "GitHub Releases returned invalid metadata".to_owned())?; + } + } + Ok(releases) + } + + /// Downloads and verifies the available package. on_progress receives the + /// updater state each time the received byte count crosses another whole + /// percent of the signed size, so the shell can show determinate progress. + pub fn download(&self, on_progress: &dyn Fn(Value)) -> Result { + let _operation = self.begin_operation()?; + let (available, started) = { + let mut state = self.state.lock().expect("updater state poisoned"); + let available = state + .available + .clone() + .ok_or_else(|| "No signed update package is available".to_owned())?; + embedded_update_key()?; + state.status = "downloading"; + state.transaction = None; + state.message = format!("Downloading OpenNOW {}…", available.version); + state.download_progress = Some(DownloadProgress { + received_bytes: 0, + total_bytes: available.asset.size, + }); + (available, state_json(&state)) + }; + on_progress(started); + let _ = fs::remove_file(self.staging_dir.join("active-apply.json")); + let mut reported_percent = 0; + let result = self.download_verified(&available, &mut |received_bytes| { + let percent = download_percent(received_bytes, available.asset.size); + let snapshot = { + let mut state = self.state.lock().expect("updater state poisoned"); + state.download_progress = Some(DownloadProgress { + received_bytes, + total_bytes: available.asset.size, + }); + (percent > reported_percent).then(|| state_json(&state)) + }; + if let Some(snapshot) = snapshot { + reported_percent = percent; + on_progress(snapshot); + } + }); + self.state + .lock() + .expect("updater state poisoned") + .download_progress = None; + match result { + Ok(downloaded) => { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "downloaded"; + state.message = format!( + "OpenNOW {} downloaded and verified. Ready to install.", + downloaded.version + ); + state.downloaded = Some(downloaded); + Ok(state_json(&state)) + } + Err(error) => { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "available"; + state.message = format!("Update download failed verification: {error}"); + Err(error) + } + } + } + + pub fn install(&self, params: &Value) -> Result { + let _operation = self.begin_operation()?; + if params["confirmed"].as_bool() != Some(true) { + return Err("Update installation requires explicit confirmation".to_owned()); + } + let downloaded = self + .state + .lock() + .expect("updater state poisoned") + .downloaded + .clone() + .ok_or_else(|| "No verified update has been downloaded".to_owned())?; + { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "preparing"; + state.message = + "Verifying and preparing a complete replacement before shutdown.".to_owned(); + state.transaction = None; + } + let _ = fs::remove_file(self.staging_dir.join("active-apply.json")); + let result = (|| { + verify_downloaded_file(&downloaded)?; + let application = std::env::var_os("OPENNOW_APP_EXECUTABLE") + .map(PathBuf::from) + .ok_or("The Qt application executable identity is unavailable")?; + let application_pid = std::env::var("OPENNOW_APP_PID") + .map_err(|_| "The Qt application process identity is unavailable")? + .parse::() + .map_err(|_| "The Qt application process identity is invalid")?; + let kind = update_apply::detect_install_kind(&application, &downloaded.path)?; + let prepared = update_apply::prepare_update(update_apply::PrepareRequest { + package: downloaded.path.clone(), + expected_version: downloaded.version.clone(), + application_executable: application, + application_pid, + core_pid: std::process::id(), + kind, + data_dir: self.data_dir.clone(), + })?; + save_prepared_update(&self.staging_dir, &prepared)?; + self.state + .lock() + .expect("updater state poisoned") + .transaction = Some(prepared.clone()); + update_apply::launch_prepared_update(&prepared)?; + Ok::<(), String>(()) + })(); + if let Err(error) = result { + let mut state = self.state.lock().expect("updater state poisoned"); + state.status = "failed"; + state.message = format!("Update preparation failed; OpenNOW remains open: {error}"); + state.transaction = None; + let _ = fs::remove_file(self.staging_dir.join("active-apply.json")); + return Err(error); + } + Ok(self.state()) + } + + pub fn highlights(&self) -> Value { + let state = self.state.lock().expect("updater state poisoned"); + json!({ + "version": state.notes_version, + "title": state.notes_version.as_ref().map(|version| format!("OpenNOW {version}")), + "bodyMarkdown": state.notes, + "releaseUrl": state.release_url + }) + } + + fn begin_operation(&self) -> Result, String> { + if let Some(message) = update_apply::external_update_message() { + return Err(message.to_owned()); + } + if let Some(message) = update_apply::windows_installer_replacement_message()? { + return Err(message.to_owned()); + } + let operation = self + .operation + .try_lock() + .map_err(|_| "An update operation is already in progress".to_owned())?; + if self.installation_pending() { + return Err("Update installation is already in progress".to_owned()); + } + if self.state.lock().expect("updater state poisoned").status == "reboot-required" { + return Err("Restart your system before starting another update operation.".to_owned()); + } + Ok(operation) + } + + fn download_verified( + &self, + available: &AvailableUpdate, + progress: &mut dyn FnMut(u64), + ) -> Result { + let key = embedded_update_key()?; + let manifest_response = self + .client + .get(&available.manifest_url) + .header(USER_AGENT, "OpenNOW-Qt/0.5") + .send() + .map_err(|_| "Could not download signed update metadata".to_owned())?; + ensure_asset_response(&manifest_response, MAXIMUM_MANIFEST_BYTES)?; + let manifest_bytes = read_bounded(manifest_response, MAXIMUM_MANIFEST_BYTES)?; + let manifest = serde_json::from_slice::(&manifest_bytes) + .map_err(|_| "Signed update metadata is invalid JSON".to_owned())?; + verify_manifest(&manifest, &key)?; + if !manifest_matches_release(&manifest, available) { + return Err( + "Signed update metadata does not match the selected release asset".to_owned(), + ); + } + if !safe_asset_name(&manifest.asset) || !trusted_asset_url(&available.asset) { + return Err("Update asset name or URL is not trusted".to_owned()); + } + + let response = self + .client + .get(&available.asset.browser_download_url) + .header(USER_AGENT, "OpenNOW-Qt/0.5") + .send() + .map_err(|_| "Could not download the update package".to_owned())?; + ensure_asset_response(&response, manifest.size)?; + let final_path = self.staging_dir.join(&manifest.asset); + let partial_path = self.staging_dir.join(format!(".{}.part", manifest.asset)); + let _ = fs::remove_file(&partial_path); + let result = write_verified_asset(response, &partial_path, &manifest, progress); + if let Err(error) = result { + let _ = fs::remove_file(&partial_path); + return Err(error); + } + if final_path.exists() { + fs::remove_file(&final_path) + .map_err(|error| format!("Could not replace the staged update: {error}"))?; + } + fs::rename(&partial_path, &final_path) + .map_err(|error| format!("Could not finalize the staged update: {error}"))?; + fs::write( + self.staging_dir + .join(format!("{}.manifest.json", manifest.asset)), + manifest_bytes, + ) + .map_err(|error| format!("Could not preserve signed update metadata: {error}"))?; + Ok(DownloadedUpdate { + version: available.version.clone(), + asset_name: manifest.asset, + path: final_path, + size: manifest.size, + sha256: manifest.sha256.to_ascii_lowercase(), + }) + } +} + +fn state_json(state: &State) -> Value { + let can_download = matches!(state.status, "available" | "error" | "downloaded") + && state.available.is_some() + && embedded_update_key().is_ok() + && state.available.as_ref().is_some_and(|available| { + state + .downloaded + .as_ref() + .is_none_or(|downloaded| downloaded.asset_name != available.asset.name) + }); + json!({ + "status": state.status, + "currentVersion": crate::version::APPLICATION_VERSION, + "availableVersion": state.available_version, + "downloadedVersion": state.downloaded.as_ref().map(|value| value.version.clone()), + "releaseUrl": state.release_url, + "message": state.message, + "lastCheckedAt": state.last_checked_at.map(|value| value.to_string()), + "downloadProgress": state.download_progress.map(|progress| json!({ + "receivedBytes": progress.received_bytes, + "totalBytes": progress.total_bytes, + })), + "canCheck": !matches!(state.status, "unsupported" | "checking" | "downloading" | "preparing" | "awaiting-exit" | "applying" | "restarting" | "installing" | "managed-pending" | "reboot-required"), + "canDownload": can_download, + "canInstall": matches!(state.status, "downloaded" | "error" | "available" | "not-available" | "failed" | "rolled-back") && state.downloaded.is_some(), + "exitRequired": state.status == "awaiting-exit", + "installVersion": state.transaction.as_ref().map(|transaction| &transaction.version), + "updateSource": if state.status == "unsupported" { "flatpak" } else { "github-releases" }, + "signaturePolicy": if embedded_update_key().is_ok() { "ed25519-pinned" } else { "unconfigured-fail-closed" } + }) +} + +fn read_prepared_update(directory: &Path) -> Result, String> { + let path = directory.join("active-apply.json"); + let file = match fs::File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("Could not read update transaction: {error}")), + }; + let bytes = read_bounded(file, MAXIMUM_MANIFEST_BYTES)?; + let prepared: update_apply::PreparedUpdate = + serde_json::from_slice(&bytes).map_err(|_| "Persisted update transaction is malformed")?; + if !prepared.plan_path.is_absolute() + || !prepared.outcome_path.is_absolute() + || prepared + .plan_path + .file_name() + .is_none_or(|name| name != "plan.json") + || prepared + .outcome_path + .file_name() + .is_none_or(|name| name != "outcome.json") + || prepared.plan_path.parent() != prepared.outcome_path.parent() + || parse_version(&prepared.version).is_none() + { + return Err("Persisted update transaction paths or version are invalid".to_owned()); + } + Ok(Some(prepared)) +} + +fn save_prepared_update( + directory: &Path, + prepared: &update_apply::PreparedUpdate, +) -> Result<(), String> { + let temporary = directory.join(format!(".active-apply-{:016x}.part", rand::random::())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| error.to_string())?; + let result = (|| { + file.write_all(&serde_json::to_vec(prepared).map_err(|error| error.to_string())?) + .and_then(|_| file.sync_all()) + .map_err(|error| error.to_string())?; + drop(file); + let destination = directory.join("active-apply.json"); + match fs::remove_file(&destination) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return Err(error.to_string()), + } + fs::rename(&temporary, destination).map_err(|error| error.to_string())?; + #[cfg(unix)] + fs::File::open(directory) + .and_then(|file| file.sync_all()) + .map_err(|error| error.to_string())?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn reconcile_transaction(state: &mut State, staging_dir: &Path) { + let Some(transaction) = &state.transaction else { + return; + }; + let outcome = update_apply::read_outcome(&transaction.outcome_path); + let outcome = match outcome { + Ok(Some(outcome)) if outcome.version == transaction.version => outcome, + Ok(None) | Ok(Some(_)) => { + state.status = "failed"; + state.message = + "The update helper outcome is missing or does not match the prepared version." + .to_owned(); + // Terminal: report once for this launch, then drop persistence so the next + // startup does not re-show the same failure dialog. + clear_finished_transaction(state, staging_dir); + return; + } + Err(error) => { + state.status = "failed"; + state.message = error; + clear_finished_transaction(state, staging_dir); + return; + } + }; + use update_apply::OutcomeStatus; + if matches!( + outcome.status, + OutcomeStatus::ManagedPending | OutcomeStatus::RebootRequired + ) { + match update_apply::recover_managed_update( + transaction, + &outcome, + crate::version::APPLICATION_VERSION, + ) { + Ok(update_apply::ManagedRecovery::Pending(message)) => { + state.status = "managed-pending"; + state.message = message; + return; + } + Ok(update_apply::ManagedRecovery::RebootRequired(message)) => { + state.status = "reboot-required"; + state.message = message; + return; + } + Ok(update_apply::ManagedRecovery::Finished(result)) => { + state.status = if result.status == OutcomeStatus::Completed { + "succeeded" + } else { + "failed" + }; + state.message = result.message; + } + Err(error) => { + state.status = "failed"; + state.message = format!("Could not recover native update completion: {error}"); + } + } + clear_finished_transaction(state, staging_dir); + return; + } + if matches!( + outcome.status, + OutcomeStatus::WaitingForExit + | OutcomeStatus::BackingUp + | OutcomeStatus::Installing + | OutcomeStatus::AwaitingStartup + ) || (outcome.status == OutcomeStatus::Prepared && state.status != "preparing") + { + match update_apply::helper_is_running(transaction) { + Ok(true) => (), + Ok(false) => { + state.status = "failed"; + state.message = "The update helper stopped before completing the transaction. OpenNOW will not close automatically.".to_owned(); + clear_finished_transaction(state, staging_dir); + return; + } + Err(error) => { + state.status = "failed"; + state.message = format!("Could not confirm update helper ownership: {error}"); + clear_finished_transaction(state, staging_dir); + return; + } + } + } + state.status = match outcome.status { + OutcomeStatus::Prepared => "preparing", + OutcomeStatus::WaitingForExit => "awaiting-exit", + OutcomeStatus::BackingUp | OutcomeStatus::Installing => "applying", + OutcomeStatus::AwaitingStartup => "restarting", + OutcomeStatus::Completed => "succeeded", + OutcomeStatus::RolledBack => "rolled-back", + OutcomeStatus::Failed => "failed", + OutcomeStatus::ManagedPending => "managed-pending", + OutcomeStatus::RebootRequired => "reboot-required", + }; + state.message = outcome.message; + if matches!( + outcome.status, + OutcomeStatus::Completed | OutcomeStatus::RolledBack | OutcomeStatus::Failed + ) { + // Terminal helper outcomes are reported for this launch only. Clearing the + // persisted transaction prevents the same failure dialog on every restart, + // matching the managed-recovery behavior documented in core-protocol.md. + clear_finished_transaction(state, staging_dir); + } +} + +fn clear_finished_transaction(state: &mut State, staging_dir: &Path) { + state.transaction = None; + match fs::remove_file(staging_dir.join("active-apply.json")) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => state.message.push_str(&format!( + "; could not remove the completed update transaction: {error}" + )), + } +} + +fn restore_downloaded_status(state: &mut State) { + if let Some(downloaded) = &state.downloaded { + state.status = "downloaded"; + state.message = format!( + "OpenNOW {} remains downloaded and verified. Ready to install.", + downloaded.version + ); + } +} + +fn select_release<'a>( + releases: &'a [Release], + channel: &str, + current: Version, +) -> Option<&'a Release> { + select_latest_release(releases, channel).filter(|release| { + parse_version(&release.tag_name) + .is_some_and(|version| version.cmp_precedence(¤t).is_gt()) + }) +} + +fn select_latest_release<'a>(releases: &'a [Release], channel: &str) -> Option<&'a Release> { + releases + .iter() + .filter(|release| !release.draft && (channel == "nightly" || !release.prerelease)) + .filter_map(|release| parse_version(&release.tag_name).map(|version| (release, version))) + .filter(|(_, version)| channel == "nightly" || version.pre.is_empty()) + .max_by(|(_, left), (_, right)| left.cmp_precedence(right)) + .map(|(release, _)| release) +} + +fn select_installable_release<'a>( + releases: &'a [Release], + channel: &str, + current: &Version, +) -> Option<&'a Release> { + releases + .iter() + .filter(|release| !release.draft && (channel == "nightly" || !release.prerelease)) + .filter_map(|release| parse_version(&release.tag_name).map(|version| (release, version))) + .filter(|(_, version)| { + (channel == "nightly" || version.pre.is_empty()) + && version.cmp_precedence(current).is_gt() + }) + .filter(|(release, _)| { + compatible_asset(&release.assets) + .and_then(|asset| manifest_asset(release, asset)) + .is_some() + }) + .max_by(|(_, left), (_, right)| left.cmp_precedence(right)) + .map(|(release, _)| release) +} + +fn manifest_asset<'a>(release: &'a Release, asset: &Asset) -> Option<&'a Asset> { + let expected_name = format!("{}.manifest.json", asset.name); + let expected_url = format!("{}.manifest.json", asset.browser_download_url); + release.assets.iter().find(|manifest| { + manifest.name == expected_name + && manifest.browser_download_url == expected_url + && manifest.size > 0 + && manifest.size <= MAXIMUM_MANIFEST_BYTES + }) +} + +fn update_highlights(state: &mut State, release: Option<&Release>) { + state.notes_version = + release.map(|release| release.tag_name.trim_start_matches('v').to_owned()); + state.release_url = Some( + release + .filter(|release| trusted_release_url(&release.html_url)) + .map_or_else( + || RELEASES_PAGE.to_owned(), + |release| release.html_url.clone(), + ), + ); + state.notes = Some(match release { + Some(release) => release + .body + .as_deref() + .map(str::trim) + .filter(|body| !body.is_empty()) + .map(|body| bounded(body, 20_000)) + .unwrap_or_else(|| "No release notes were published for this release.".to_owned()), + None => "No published releases were found for this update channel.".to_owned(), + }); +} + +fn compatible_asset(assets: &[Asset]) -> Option<&Asset> { + let extension = update_apply::compatible_package_extension().ok()?; + let os_aliases: &[&str] = if cfg!(target_os = "windows") { + &["win", "windows"] + } else if cfg!(target_os = "macos") { + &["mac", "macos", "darwin"] + } else { + &["linux"] + }; + let arch_aliases: &[&str] = if cfg!(target_arch = "aarch64") { + &["arm64", "aarch64"] + } else { + &["x64", "x86_64", "amd64"] + }; + assets.iter().find(|asset| { + let name = asset.name.to_ascii_lowercase(); + let package = name + .rsplit_once('.') + .is_some_and(|(_, suffix)| suffix == extension); + trusted_asset_url(asset) + && safe_asset_name(&asset.name) + && name.contains("qt") + && os_aliases.iter().any(|alias| name.contains(alias)) + && arch_aliases.iter().any(|alias| name.contains(alias)) + && package + && asset.size > 0 + && asset.size <= MAXIMUM_UPDATE_BYTES + }) +} + +fn manifest_matches_release(manifest: &UpdateManifest, available: &AvailableUpdate) -> bool { + parse_version(&manifest.version) + .is_some_and(|version| Some(version) == parse_version(&available.version)) + && manifest.asset == available.asset.name + && manifest.size == available.asset.size + && manifest.size > 0 + && manifest.size <= MAXIMUM_UPDATE_BYTES +} + +fn download_percent(received_bytes: u64, total_bytes: u64) -> u64 { + if total_bytes == 0 { + return 0; + } + (u128::from(received_bytes.min(total_bytes)) * 100 / u128::from(total_bytes)) as u64 +} + +fn write_verified_asset( + mut response: impl Read, + path: &Path, + manifest: &UpdateManifest, + progress: &mut dyn FnMut(u64), +) -> Result<(), String> { + let mut output = fs::File::create(path) + .map_err(|error| format!("Could not create the staged update: {error}"))?; + let mut hasher = Sha256::new(); + let mut total = 0_u64; + let mut buffer = [0_u8; 128 * 1024]; + loop { + let count = response + .read(&mut buffer) + .map_err(|_| "Update download ended unexpectedly".to_owned())?; + if count == 0 { + break; + } + total = total + .checked_add(count as u64) + .ok_or_else(|| "Update size overflowed".to_owned())?; + if total > manifest.size || total > MAXIMUM_UPDATE_BYTES { + return Err("Update download exceeded its signed size".to_owned()); + } + hasher.update(&buffer[..count]); + output + .write_all(&buffer[..count]) + .map_err(|error| format!("Could not write the staged update: {error}"))?; + progress(total); + } + output + .sync_all() + .map_err(|error| format!("Could not flush the staged update: {error}"))?; + let digest = format!("{:x}", hasher.finalize()); + if total != manifest.size || !digest.eq_ignore_ascii_case(&manifest.sha256) { + return Err("Update package hash or size did not match signed metadata".to_owned()); + } + Ok(()) +} + +fn verify_downloaded_file(downloaded: &DownloadedUpdate) -> Result<(), String> { + let manifest = fs::File::open( + downloaded + .path + .with_file_name(format!("{}.manifest.json", downloaded.asset_name)), + ) + .map_err(|_| "The staged update manifest could not be read".to_owned())?; + let manifest = update_apply::verification::verify_signed_manifest(&read_bounded( + manifest, + MAXIMUM_MANIFEST_BYTES, + )?)?; + if manifest.version.trim_start_matches('v') != downloaded.version + || manifest.asset != downloaded.asset_name + || manifest.size != downloaded.size + || !manifest.sha256.eq_ignore_ascii_case(&downloaded.sha256) + { + return Err("The staged update package changed after verification".to_owned()); + } + update_apply::verification::verify_package(&downloaded.path, &manifest) +} + +fn ensure_asset_response(response: &Response, maximum: u64) -> Result<(), String> { + if !response.status().is_success() { + return Err(format!( + "Update asset returned HTTP {}", + response.status().as_u16() + )); + } + if response + .content_length() + .is_some_and(|length| length > maximum) + { + return Err("Update asset exceeded its allowed size".to_owned()); + } + Ok(()) +} + +fn read_bounded(mut response: impl Read, maximum: u64) -> Result, String> { + let mut bytes = Vec::new(); + response + .by_ref() + .take(maximum + 1) + .read_to_end(&mut bytes) + .map_err(|_| "Could not read update metadata".to_owned())?; + if bytes.len() as u64 > maximum { + return Err("Update metadata exceeded the size limit".to_owned()); + } + Ok(bytes) +} + +fn trusted_asset_url(asset: &Asset) -> bool { + asset.browser_download_url.starts_with(RELEASE_ASSET_PREFIX) + && !asset.browser_download_url[RELEASE_ASSET_PREFIX.len()..].is_empty() +} + +fn parse_version(value: &str) -> Option { + Version::parse(value.strip_prefix('v').unwrap_or(value)).ok() +} + +fn trusted_release_url(value: &str) -> bool { + value + .strip_prefix("https://github.com/OpenCloudGaming/OpenNOW/releases/") + .is_some_and(|suffix| !suffix.is_empty()) +} + +fn bounded(value: &str, limit: usize) -> String { + value.chars().take(limit).collect() +} + +fn friendly_network_error(value: &str) -> String { + let lower = value.to_ascii_lowercase(); + if lower.contains("timeout") || lower.contains("connect") || lower.contains("dns") { + "Unable to reach GitHub Releases right now.".to_owned() + } else { + "Update check failed.".to_owned() + } +} + +fn now_ms() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::update_apply::verification::{decode_verifying_key, signature_payload}; + use base64::Engine as _; + use base64::engine::general_purpose::STANDARD as BASE64; + use ed25519_dalek::{Signer as _, SigningKey}; + + #[cfg(target_os = "linux")] + #[test] + fn flatpak_updates_are_external_even_with_cached_native_updates() { + if std::env::var_os("OPENNOW_FLATPAK_UPDATER_TEST").is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "updater::tests::flatpak_updates_are_external_even_with_cached_native_updates", + "--nocapture", + ]) + .env("OPENNOW_FLATPAK_UPDATER_TEST", "1") + .env("FLATPAK_ID", "io.github.opencloudgaming.OpenNOW") + .env("APPIMAGE", "/host/OpenNOW.AppImage") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + let directory = tempfile::tempdir().unwrap(); + let updates = directory.path().join("updates"); + fs::create_dir(&updates).unwrap(); + let persisted = updates.join("active-apply.json"); + fs::write(&persisted, b"partial host update transaction").unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + let message = update_apply::external_update_message().unwrap(); + assert!(message.contains("flatpak update io.github.opencloudgaming.OpenNOW")); + assert!(!updater.installation_pending()); + for state in [updater.state(), updater.check(&json!({})).unwrap()] { + assert_eq!(state["status"], "unsupported"); + assert_eq!(state["updateSource"], "flatpak"); + assert_eq!(state["message"], message); + for capability in ["canCheck", "canDownload", "canInstall", "exitRequired"] { + assert_eq!(state[capability], false); + } + assert!(state["installVersion"].is_null()); + } + updater.request_failed("A generic failure must not replace the update instruction"); + assert_eq!(updater.state()["message"], message); + assert_eq!(updater.download(&|_| {}).unwrap_err(), message); + assert_eq!( + updater.install(&json!({"confirmed": true})).unwrap_err(), + message + ); + assert_eq!( + fs::read(&persisted).unwrap(), + b"partial host update transaction" + ); + assert_eq!(fs::read_dir(&updates).unwrap().count(), 1); + assert_eq!( + update_apply::compatible_package_extension().unwrap_err(), + message + ); + for (name, kind) in [ + ("OpenNOW.AppImage", update_apply::InstallKind::AppImage), + ("OpenNOW.deb", update_apply::InstallKind::DebianPackage), + ] { + let package = directory.path().join(name); + let application = directory.path().join("OpenNOW"); + assert_eq!( + update_apply::detect_install_kind(&application, &package).unwrap_err(), + message + ); + assert_eq!( + update_apply::prepare_update(update_apply::PrepareRequest { + package, + expected_version: "1.2.3".to_owned(), + application_executable: application, + application_pid: std::process::id(), + core_pid: std::process::id(), + kind, + data_dir: directory.path().to_path_buf(), + }) + .unwrap_err(), + message + ); + } + let prepared = update_apply::PreparedUpdate { + plan_path: directory.path().join("plan.json"), + outcome_path: directory.path().join("outcome.json"), + version: "1.2.3".to_owned(), + }; + assert_eq!( + update_apply::launch_prepared_update(&prepared).unwrap_err(), + message + ); + assert_eq!( + update_apply::run_helper(&prepared.plan_path).unwrap_err(), + message + ); + assert!(!directory.path().join("apply.lock").exists()); + } + + fn persisted_managed_fixture( + directory: &Path, + status: update_apply::OutcomeStatus, + installer: Option, + ) -> update_apply::PreparedUpdate { + let updates = directory.join("updates"); + fs::create_dir_all(&updates).unwrap(); + let prepared = update_apply::PreparedUpdate { + plan_path: directory.join("plan.json"), + outcome_path: directory.join("outcome.json"), + version: "1.1.0".to_owned(), + }; + save_prepared_update(&updates, &prepared).unwrap(); + fs::write(&prepared.plan_path, serde_json::to_vec(&json!({ + "schemaVersion": 1, + "version": prepared.version, + "kind": if status == update_apply::OutcomeStatus::RebootRequired || cfg!(windows) { "windowsMsi" } else { "debianPackage" }, + "package": directory.join("update-package"), + "target": directory.join("installed"), + "applicationExecutable": directory.join("installed/OpenNOW"), + "dataDir": directory.join("data"), + "processes": [], + "nonce": "a".repeat(64), + "managedIdentity": { + "package": if cfg!(windows) { "{00000000-0000-0000-0000-000000000000}" } else { "opennow-recovery-test-missing" }, + "version": "1.1.0", "architecture": "amd64", "installed_product": "opennow-recovery-test-missing" + } + })).unwrap()).unwrap(); + fs::write( + &prepared.outcome_path, + serde_json::to_vec(&update_apply::UpdateOutcome { + schema_version: 1, + version: prepared.version.clone(), + status, + message: "Native installation completion is pending".to_owned(), + installed_version: None, + restarted_process: installer, + }) + .unwrap(), + ) + .unwrap(); + prepared + } + + #[test] + fn a_live_native_installer_keeps_persisted_updates_blocking() { + for status in [ + update_apply::OutcomeStatus::ManagedPending, + update_apply::OutcomeStatus::RebootRequired, + ] { + let directory = tempfile::tempdir().unwrap(); + persisted_managed_fixture( + directory.path(), + status, + Some(update_apply::ProcessIdentity::capture(std::process::id()).unwrap()), + ); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], "managed-pending"); + assert_eq!(updater.state()["canCheck"], false); + assert!(updater.installation_pending()); + assert!(directory.path().join("updates/active-apply.json").exists()); + } + } + + #[test] + fn a_live_helper_keeps_native_recovery_blocking() { + use fs2::FileExt; + let directory = tempfile::tempdir().unwrap(); + let mut installer = update_apply::ProcessIdentity::capture(std::process::id()).unwrap(); + installer.started = installer.started.wrapping_add(1); + persisted_managed_fixture( + directory.path(), + update_apply::OutcomeStatus::ManagedPending, + Some(installer), + ); + let lock = fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .read(true) + .open(directory.path().join("apply.lock")) + .unwrap(); + lock.try_lock_exclusive().unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert!(updater.installation_pending()); + assert_eq!(updater.state()["canCheck"], false); + drop(lock); + assert!(!updater.installation_pending()); + assert_eq!(updater.state()["canCheck"], true); + } + + #[test] + fn finished_native_installer_verification_failure_unblocks_and_clears_persistence() { + let directory = tempfile::tempdir().unwrap(); + let mut installer = update_apply::ProcessIdentity::capture(std::process::id()).unwrap(); + installer.started = installer.started.wrapping_add(1); + let prepared = persisted_managed_fixture( + directory.path(), + update_apply::OutcomeStatus::ManagedPending, + Some(installer), + ); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], "failed"); + assert_eq!(updater.state()["canCheck"], true); + assert!(!updater.installation_pending()); + assert!(updater.begin_operation().is_ok()); + assert!(!directory.path().join("updates/active-apply.json").exists()); + assert_eq!( + update_apply::read_outcome(&prepared.outcome_path) + .unwrap() + .unwrap() + .status, + update_apply::OutcomeStatus::Failed + ); + let restarted = UpdaterService::new(directory.path()).unwrap(); + assert!(!restarted.installation_pending()); + assert_eq!(restarted.state()["canCheck"], true); + } + + #[cfg(any(windows, target_os = "linux"))] + #[test] + fn missing_native_installer_identity_requires_a_proven_reboot_before_recovery() { + let directory = tempfile::tempdir().unwrap(); + persisted_managed_fixture( + directory.path(), + update_apply::OutcomeStatus::ManagedPending, + None, + ); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert!(updater.installation_pending()); + assert_eq!(updater.state()["canCheck"], false); + assert!( + updater.state()["message"] + .as_str() + .unwrap() + .contains("Restart your system") + ); + assert!(directory.path().join("updates/active-apply.json").exists()); + assert!(directory.path().join("install-boot.json").exists()); + fs::write( + directory.path().join("install-boot.json"), + br#"{"schemaVersion":1,"bootId":"previous-boot"}"#, + ) + .unwrap(); + let restarted = UpdaterService::new(directory.path()).unwrap(); + assert!(!restarted.installation_pending()); + assert_eq!(restarted.state()["status"], "failed"); + assert_eq!(restarted.state()["canCheck"], true); + assert!(!directory.path().join("updates/active-apply.json").exists()); + } + + #[cfg(any(windows, target_os = "linux"))] + #[test] + fn reboot_warning_allows_sessions_but_blocks_update_operations_until_boot_change() { + let directory = tempfile::tempdir().unwrap(); + persisted_managed_fixture( + directory.path(), + update_apply::OutcomeStatus::RebootRequired, + None, + ); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], "reboot-required"); + assert_eq!(updater.state()["canCheck"], false); + assert!(!updater.installation_pending()); + assert!( + updater + .begin_operation() + .unwrap_err() + .contains("Restart your system") + ); + assert!(directory.path().join("updates/active-apply.json").exists()); + fs::write( + directory.path().join("install-boot.json"), + br#"{"schemaVersion":1,"bootId":"previous-boot"}"#, + ) + .unwrap(); + let restarted = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(restarted.state()["status"], "failed"); + assert!(!restarted.installation_pending()); + assert_eq!(restarted.state()["canCheck"], true); + assert!(!directory.path().join("updates/active-apply.json").exists()); + } + + #[test] + fn persisted_apply_outcomes_require_a_live_helper_before_quit() { + let directory = tempfile::tempdir().unwrap(); + let updates = directory.path().join("updates"); + fs::create_dir(&updates).unwrap(); + let prepared = update_apply::PreparedUpdate { + plan_path: directory.path().join("plan.json"), + outcome_path: directory.path().join("outcome.json"), + version: "1.1.0".to_owned(), + }; + save_prepared_update(&updates, &prepared).unwrap(); + assert_eq!( + read_prepared_update(&updates).unwrap().unwrap().version, + "1.1.0" + ); + for status in [ + update_apply::OutcomeStatus::Prepared, + update_apply::OutcomeStatus::WaitingForExit, + update_apply::OutcomeStatus::BackingUp, + update_apply::OutcomeStatus::Installing, + update_apply::OutcomeStatus::AwaitingStartup, + ] { + // Terminal failures clear persistence after the first report, so each + // stale-helper case needs fresh persistence to exercise the same path. + save_prepared_update(&updates, &prepared).unwrap(); + fs::write( + &prepared.outcome_path, + serde_json::to_vec(&update_apply::UpdateOutcome { + schema_version: 1, + version: prepared.version.clone(), + status, + message: "A stale helper snapshot".to_owned(), + installed_version: None, + restarted_process: None, + }) + .unwrap(), + ) + .unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], "failed"); + assert_eq!(updater.state()["exitRequired"], false); + assert!(!updater.installation_pending()); + // The failure is reported for this launch only; the next startup begins + // clean instead of re-showing the same dialog. + assert!(!updates.join("active-apply.json").exists()); + } + } + + #[test] + fn terminal_helper_outcomes_are_reported_once_then_cleared() { + for (outcome, expected) in [ + (update_apply::OutcomeStatus::Failed, "failed"), + (update_apply::OutcomeStatus::RolledBack, "rolled-back"), + (update_apply::OutcomeStatus::Completed, "succeeded"), + ] { + let directory = tempfile::tempdir().unwrap(); + let updates = directory.path().join("updates"); + fs::create_dir(&updates).unwrap(); + let prepared = update_apply::PreparedUpdate { + plan_path: directory.path().join("plan.json"), + outcome_path: directory.path().join("outcome.json"), + version: "1.1.0".to_owned(), + }; + save_prepared_update(&updates, &prepared).unwrap(); + fs::write( + &prepared.outcome_path, + serde_json::to_vec(&update_apply::UpdateOutcome { + schema_version: 1, + version: prepared.version.clone(), + status: outcome, + message: "Terminal helper result".to_owned(), + installed_version: None, + restarted_process: None, + }) + .unwrap(), + ) + .unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], expected); + assert!(!updater.installation_pending()); + assert!(!updates.join("active-apply.json").exists()); + let restarted = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(restarted.state()["status"], "idle"); + assert_eq!(restarted.state()["canCheck"], true); + assert!(!restarted.installation_pending()); + } + } + + #[test] + fn corrupt_update_status_does_not_prevent_core_startup() { + let directory = tempfile::tempdir().unwrap(); + fs::create_dir(directory.path().join("updates")).unwrap(); + fs::write( + directory.path().join("updates/active-apply.json"), + b"partial JSON", + ) + .unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(updater.state()["status"], "failed"); + assert_eq!(updater.state()["exitRequired"], false); + assert_eq!(updater.state()["canCheck"], true); + // Corrupt persistence is dropped after the first report so the next launch + // starts clean instead of failing forever. + assert!(!directory.path().join("updates/active-apply.json").exists()); + let restarted = UpdaterService::new(directory.path()).unwrap(); + assert_eq!(restarted.state()["status"], "idle"); + } + + #[test] + fn checks_and_errors_preserve_verified_install_capability() { + let mut state = notes_state(); + state.downloaded = Some(DownloadedUpdate { + version: "1.1.0".to_owned(), + asset_name: "OpenNOW-Qt-linux-x64.AppImage".to_owned(), + path: PathBuf::from("/not-read-by-state"), + size: 123, + sha256: "ab".repeat(32), + }); + for status in ["available", "not-available", "error"] { + state.status = status; + assert_eq!(state_json(&state)["canInstall"], true); + restore_downloaded_status(&mut state); + assert_eq!(state.status, "downloaded"); + assert_eq!(state_json(&state)["downloadedVersion"], "1.1.0"); + } + for status in ["checking", "downloading", "installing"] { + state.status = status; + assert_eq!(state_json(&state)["canInstall"], false); + } + } + + #[test] + fn install_requires_consent_before_accessing_packages() { + let path = + std::env::temp_dir().join(format!("opennow-update-consent-{}", rand::random::())); + let updater = UpdaterService::new(&path).unwrap(); + for params in [ + json!({}), + json!({"confirmed":false}), + json!({"confirmed":"true"}), + ] { + assert_eq!( + updater.install(¶ms).unwrap_err(), + "Update installation requires explicit confirmation" + ); + } + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn overlapping_update_operations_are_rejected_without_mutating_state() { + let path = + std::env::temp_dir().join(format!("opennow-update-busy-{}", rand::random::())); + let updater = UpdaterService::new(&path).unwrap(); + let operation = updater.begin_operation().unwrap(); + let before = updater.state(); + for result in [ + updater.check(&json!({"channel":"invalid"})), + updater.download(&|_| {}), + updater.install(&json!({"confirmed":true})), + ] { + assert_eq!( + result.unwrap_err(), + "An update operation is already in progress" + ); + assert_eq!(updater.state(), before); + } + drop(operation); + assert!(updater.begin_operation().is_ok()); + updater.state.lock().unwrap().status = "installing"; + for result in [ + updater.check(&json!({"channel":"invalid"})), + updater.download(&|_| {}), + updater.install(&json!({"confirmed":true})), + ] { + assert_eq!( + result.unwrap_err(), + "Update installation is already in progress" + ); + } + std::fs::remove_dir_all(path).unwrap(); + } + + fn release(version: &str, prerelease: bool) -> Release { + Release { + tag_name: version.to_owned(), + html_url: format!("https://github.com/OpenCloudGaming/OpenNOW/releases/tag/{version}"), + body: None, + draft: false, + prerelease, + assets: vec![], + } + } + + #[cfg(target_os = "linux")] + fn linux_update(version: &str, prerelease: bool) -> Release { + let mut release = release(version, prerelease); + let architecture = if cfg!(target_arch = "aarch64") { + "arm64" + } else { + "x64" + }; + let extension = update_apply::compatible_package_extension().unwrap(); + let name = format!( + "OpenNOW-Qt-{}-Linux-{architecture}.{extension}", + version.trim_start_matches('v') + ); + release.assets = [name.clone(), format!("{name}.manifest.json")] + .into_iter() + .map(|name| Asset { + browser_download_url: format!("{RELEASE_ASSET_PREFIX}{version}/{name}"), + name, + size: 128, + }) + .collect(); + release + } + + #[cfg(target_os = "linux")] + #[test] + fn incomplete_releases_do_not_hide_a_compatible_signed_update() { + let mut releases = vec![linux_update("v1.1.0", false), linux_update("v1.2.0", false)]; + releases[1].assets.pop(); + assert_eq!( + select_installable_release(&releases, "stable", &Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.1.0" + ); + assert!(select_installable_release(&releases, "stable", &Version::new(1, 1, 0)).is_none()); + releases.push(linux_update("v1.3.0-nightly.1.1", true)); + assert_eq!( + select_installable_release(&releases, "nightly", &Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.3.0-nightly.1.1" + ); + assert_eq!( + select_installable_release(&releases, "stable", &Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.1.0" + ); + releases[2].draft = true; + assert_eq!( + select_installable_release(&releases, "nightly", &Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.1.0" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn manifests_must_belong_to_the_selected_release_and_be_bounded() { + let mut release = linux_update("v1.1.0", false); + assert!(manifest_asset(&release, &release.assets[0]).is_some()); + for size in [0, MAXIMUM_MANIFEST_BYTES + 1] { + release.assets[1].size = size; + assert!(manifest_asset(&release, &release.assets[0]).is_none()); + } + release.assets[1].size = 128; + release.assets[1].browser_download_url = release.assets[1] + .browser_download_url + .replace("/v1.1.0/", "/v1.0.0/"); + assert!(manifest_asset(&release, &release.assets[0]).is_none()); + } + + #[test] + fn release_discovery_reaches_stable_beyond_the_nightly_window() { + use std::io::{BufRead, BufReader}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/releases", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + for path in ["/releases?per_page=100", "/releases/latest"] { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(socket.try_clone().unwrap()); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert!(line.starts_with(&format!("GET {path} ")), "{line}"); + loop { + line.clear(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + } + let metadata = |version: &str, prerelease| { + json!({ + "tag_name":version,"html_url":format!("{RELEASES_PAGE}/tag/{version}"), + "body":null,"draft":false,"prerelease":prerelease,"assets":[] + }) + }; + let body = if path.ends_with("/latest") { + metadata("v1.1.0", false).to_string() + } else { + serde_json::to_string( + &(1..=100) + .map(|run| metadata(&format!("v1.2.0-nightly.{run}.1"), true)) + .collect::>(), + ) + .unwrap() + }; + write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap(); + } + }); + let directory = tempfile::tempdir().unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + let releases = updater.fetch_releases(&url).unwrap(); + server.join().unwrap(); + assert_eq!(releases.len(), 101); + assert_eq!( + select_release(&releases, "stable", Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.1.0" + ); + assert_eq!( + select_release(&releases, "nightly", Version::new(1, 0, 0)) + .unwrap() + .tag_name, + "v1.2.0-nightly.100.1" + ); + } + + #[test] + fn versions_and_channels_are_selected_without_lexical_ordering() { + let releases = vec![release("v0.9.0", false), release("v0.10.0-beta", true)]; + assert_eq!( + select_release(&releases, "stable", Version::new(0, 8, 0)) + .unwrap() + .tag_name, + "v0.9.0" + ); + assert_eq!( + select_release(&releases, "nightly", Version::new(0, 8, 0)) + .unwrap() + .tag_name, + "v0.10.0-beta" + ); + assert_eq!( + parse_version("v10.2.3-beta.1"), + Some(Version::parse("10.2.3-beta.1").unwrap()) + ); + assert!(select_release(&releases, "stable", Version::new(0, 9, 0)).is_none()); + assert!(select_release(&releases, "nightly", Version::new(0, 10, 0)).is_none()); + } + + fn notes_state() -> State { + State { + status: "not-available", + available_version: None, + release_url: None, + notes_version: None, + notes: None, + message: String::new(), + last_checked_at: None, + available: None, + downloaded: None, + download_progress: None, + transaction: None, + } + } + + #[test] + fn nightly_runs_and_attempts_are_ordered_numerically() { + let releases = vec![ + release("v1.0.0-nightly.10.2", true), + release("v1.0.0-nightly.9.10", true), + release("v1.0.0-nightly.10.10", true), + ]; + for current in ["1.0.0-nightly.9.10", "1.0.0-nightly.10.2"] { + assert_eq!( + select_release(&releases, "nightly", parse_version(current).unwrap()) + .unwrap() + .tag_name, + "v1.0.0-nightly.10.10" + ); + } + for current in ["1.0.0-nightly.10.10", "1.0.0-nightly.11.1", "1.0.0"] { + assert!( + select_release(&releases, "nightly", parse_version(current).unwrap()).is_none() + ); + } + } + + #[test] + fn stable_promotes_the_same_base_nightly_on_both_channels() { + let releases = vec![ + release("v1.0.0", false), + release("v1.0.0-nightly.999.1", true), + ]; + for channel in ["stable", "nightly"] { + assert_eq!( + select_release( + &releases, + channel, + parse_version("1.0.0-nightly.999.1").unwrap() + ) + .unwrap() + .tag_name, + "v1.0.0" + ); + assert!(select_release(&releases, channel, Version::new(1, 0, 0)).is_none()); + } + let mislabeled = [release("v1.1.0-nightly.1.1", false)]; + assert!(select_latest_release(&mislabeled, "stable").is_none()); + } + + #[test] + fn build_metadata_does_not_offer_an_update() { + let releases = [release("v1.0.0-nightly.10.1+z", true)]; + assert!( + select_release( + &releases, + "nightly", + parse_version("1.0.0-nightly.10.1+a").unwrap() + ) + .is_none() + ); + } + + #[test] + fn invalid_semver_suffixes_and_prefixes_are_rejected() { + for version in [ + "1.0.0-nightly.01.1", + "1.0.0-nightly..1", + "1.0.0-", + "1.0.0+", + "01.0.0", + "vv1.0.0", + "1.0.0\n", + ] { + assert!(parse_version(version).is_none(), "{version:?}"); + } + } + + #[test] + fn manifest_identity_preserves_prerelease_and_build_metadata() { + let available = AvailableUpdate { + version: "1.0.0-nightly.10.2+build".to_owned(), + asset: Asset { + name: "OpenNOW-Qt-1.0.0-nightly.10.2-Linux-x64.deb".to_owned(), + browser_download_url: String::new(), + size: 123, + }, + manifest_url: String::new(), + }; + let mut manifest = UpdateManifest { + schema_version: 1, + version: format!("v{}", available.version), + asset: available.asset.name.clone(), + size: available.asset.size, + sha256: "ab".repeat(32), + signature: String::new(), + }; + assert!(manifest_matches_release(&manifest, &available)); + for version in [ + "1.0.0-nightly.10.1+build", + "1.0.0-nightly.10.2+other", + "1.0.0-nightly.10.2", + "1.0.0", + "invalid", + ] { + manifest.version = version.to_owned(); + assert!( + !manifest_matches_release(&manifest, &available), + "{version}" + ); + } + } + + #[test] + fn nightly_manifests_require_valid_signatures() { + let signing = SigningKey::from_bytes(&[7_u8; 32]); + let mut manifest = UpdateManifest { + schema_version: 1, + version: "1.0.0-nightly.10.2".to_owned(), + asset: "OpenNOW-Qt-1.0.0-nightly.10.2-Linux-x64.deb".to_owned(), + size: 123, + sha256: "ab".repeat(32), + signature: String::new(), + }; + let key = signing.verifying_key(); + assert!(verify_manifest(&manifest, &key).is_err()); + manifest.signature = BASE64.encode( + signing + .sign(signature_payload(&manifest).as_bytes()) + .to_bytes(), + ); + verify_manifest(&manifest, &key).unwrap(); + let original_version = manifest.version.clone(); + for version in ["1.0.0-nightly.10.3", "1.0.0"] { + manifest.version = version.to_owned(); + assert!(verify_manifest(&manifest, &key).is_err()); + } + manifest.version = original_version; + assert!( + verify_manifest( + &manifest, + &SigningKey::from_bytes(&[8_u8; 32]).verifying_key() + ) + .is_err() + ); + } + + #[test] + fn builds_without_a_pinned_key_cannot_download_updates() { + if option_env!("OPENNOW_UPDATE_ED25519_PUBLIC_KEY") + .is_some_and(|value| !value.trim().is_empty()) + { + return; + } + let path = + std::env::temp_dir().join(format!("opennow-update-no-key-{}", rand::random::())); + let updater = UpdaterService::new(&path).unwrap(); + updater.state.lock().unwrap().available = Some(AvailableUpdate { + version: "1.0.0-nightly.10.2".to_owned(), + asset: Asset { + name: "OpenNOW-Qt-1.0.0-nightly.10.2-Linux-x64.deb".to_owned(), + browser_download_url: String::new(), + size: 123, + }, + manifest_url: String::new(), + }); + updater.state.lock().unwrap().status = "available"; + assert_eq!(updater.state()["canDownload"], false); + assert_eq!( + updater.state()["signaturePolicy"], + "unconfigured-fail-closed" + ); + assert_eq!( + updater.download(&|_| {}).unwrap_err(), + "This build has no pinned update signing key" + ); + assert!(updater.state()["downloadedVersion"].is_null()); + fs::remove_dir_all(path).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn nightly_debian_package_matches_the_platform() { + let arch = if cfg!(target_arch = "aarch64") { + "arm64" + } else { + "x64" + }; + let name = format!("OpenNOW-Qt-1.0.0-nightly.10.2-Linux-{arch}.deb"); + let assets = [Asset { + browser_download_url: format!("{RELEASE_ASSET_PREFIX}v1.0.0-nightly.10.2/{name}"), + name, + size: 123, + }]; + assert!(compatible_asset(&assets).is_some()); + } + + #[test] + fn current_and_ahead_builds_keep_published_notes_without_offering_a_downgrade() { + let mut published = release("v0.5.4", false); + published.body = Some("# Changes\n\n- **Fixed** release notes".to_owned()); + let releases = vec![published]; + for current in [Version::new(0, 5, 4), Version::new(1, 0, 0)] { + assert!(select_release(&releases, "stable", current).is_none()); + let mut state = notes_state(); + update_highlights(&mut state, select_latest_release(&releases, "stable")); + assert_eq!(state.notes_version.as_deref(), Some("0.5.4")); + assert_eq!(state.notes, releases[0].body); + assert_eq!( + state.release_url.as_deref(), + Some(releases[0].html_url.as_str()) + ); + assert!(state_json(&state)["availableVersion"].is_null()); + assert_eq!(state_json(&state)["canDownload"], false); + } + } + + #[test] + fn notes_follow_channel_and_ignore_drafts_and_invalid_versions() { + let mut draft = release("v99.0.0", false); + draft.draft = true; + let releases = vec![ + release("v1.0.0", false), + release("v1.1.0-beta", true), + draft, + release("not-a-version", false), + ]; + let mut state = notes_state(); + update_highlights(&mut state, select_latest_release(&releases, "nightly")); + assert_eq!(state.notes_version.as_deref(), Some("1.1.0-beta")); + update_highlights(&mut state, select_latest_release(&releases, "stable")); + assert_eq!(state.notes_version.as_deref(), Some("1.0.0")); + assert_eq!( + select_release(&releases, "stable", Version::new(0, 9, 0)) + .unwrap() + .tag_name, + "v1.0.0" + ); + } + + #[test] + fn missing_bodies_and_empty_channels_replace_stale_notes_with_feedback() { + let mut state = notes_state(); + for body in [None, Some(" \n\t".to_owned())] { + let mut published = release("v1.0.0", false); + published.body = body; + update_highlights(&mut state, Some(&published)); + assert_eq!( + state.notes.as_deref(), + Some("No release notes were published for this release.") + ); + } + update_highlights(&mut state, None); + assert!(state.notes_version.is_none()); + assert_eq!( + state.notes.as_deref(), + Some("No published releases were found for this update channel.") + ); + assert_eq!(state.release_url.as_deref(), Some(RELEASES_PAGE)); + } + + #[test] + fn update_urls_assets_and_names_are_strictly_scoped() { + assert!(trusted_release_url( + "https://github.com/OpenCloudGaming/OpenNOW/releases/tag/v1.0.0" + )); + assert!(!trusted_release_url("https://example.com/releases/tag/v1")); + assert!(safe_asset_name("OpenNOW-Qt-linux-x64.AppImage")); + assert!(!safe_asset_name("../OpenNOW.AppImage")); + let malicious = Asset { + name: "OpenNOW-Qt-linux-x64.AppImage".to_owned(), + browser_download_url: "https://example.com/update".to_owned(), + size: 1, + }; + assert!(compatible_asset(&[malicious]).is_none()); + } + + #[test] + fn signed_manifests_verify_canonical_fields_and_reject_tampering() { + let signing = SigningKey::from_bytes(&[7_u8; 32]); + let mut manifest = UpdateManifest { + schema_version: 1, + version: "0.6.0".to_owned(), + asset: "OpenNOW-Qt-linux-x64.AppImage".to_owned(), + size: 123, + sha256: "ab".repeat(32), + signature: String::new(), + }; + manifest.signature = BASE64.encode( + signing + .sign(signature_payload(&manifest).as_bytes()) + .to_bytes(), + ); + let encoded_key = BASE64.encode(signing.verifying_key().as_bytes()); + let key = decode_verifying_key(&encoded_key).unwrap(); + verify_manifest(&manifest, &key).unwrap(); + manifest.size += 1; + assert!(verify_manifest(&manifest, &key).is_err()); + } + + #[test] + fn verified_downloads_report_monotonic_byte_progress() { + let directory = tempfile::tempdir().unwrap(); + let payload = vec![7_u8; 300 * 1024]; + let manifest = UpdateManifest { + schema_version: 1, + version: "1.0.0".to_owned(), + asset: "OpenNOW-Qt-1.0.0-Linux-x64.AppImage".to_owned(), + size: payload.len() as u64, + sha256: format!("{:x}", Sha256::digest(&payload)), + signature: String::new(), + }; + let mut reported = Vec::new(); + write_verified_asset( + std::io::Cursor::new(payload.clone()), + &directory.path().join("asset.part"), + &manifest, + &mut |received| reported.push(received), + ) + .unwrap(); + assert!(reported.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(reported.last().copied(), Some(manifest.size)); + assert_eq!(download_percent(0, manifest.size), 0); + assert_eq!(download_percent(manifest.size / 2, manifest.size), 50); + assert_eq!(download_percent(manifest.size * 2, manifest.size), 100); + assert_eq!(download_percent(10, 0), 0); + } + + #[test] + fn download_progress_is_reported_only_while_downloading() { + let directory = tempfile::tempdir().unwrap(); + let updater = UpdaterService::new(directory.path()).unwrap(); + assert!(updater.state()["downloadProgress"].is_null()); + updater.state.lock().unwrap().download_progress = Some(DownloadProgress { + received_bytes: 40_500_000, + total_bytes: 96_487_928, + }); + assert_eq!( + updater.state()["downloadProgress"], + json!({"receivedBytes":40_500_000_u64, "totalBytes":96_487_928_u64}) + ); + } +} diff --git a/external/opennow/native/opennow-core/src/version.rs b/external/opennow/native/opennow-core/src/version.rs new file mode 100644 index 000000000..a986fdcfb --- /dev/null +++ b/external/opennow/native/opennow-core/src/version.rs @@ -0,0 +1,40 @@ +pub const APPLICATION_VERSION: &str = match option_env!("OPENNOW_BUILD_VERSION") { + Some(version) => version, + None => env!("CARGO_PKG_VERSION"), +}; + +pub fn update_channel(version: &str) -> &'static str { + if semver::Version::parse(version) + .is_ok_and(|version| version.pre.as_str().split('.').next() == Some("nightly")) + { + "nightly" + } else { + "stable" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn application_version_is_semver() { + semver::Version::parse(APPLICATION_VERSION).expect("application version must be semver"); + } + + #[test] + fn update_channel_follows_build_identity() { + assert_eq!(update_channel("1.0.0-nightly.123.2"), "nightly"); + for version in ["1.0.0", "1.0.0-beta.1", "1.0.0+nightly", "invalid"] { + assert_eq!(update_channel(version), "stable"); + } + } + + #[test] + fn nightly_version_preserves_run_and_attempt() { + let version = semver::Version::parse("1.0.0-nightly.123456.2").unwrap(); + assert_eq!((version.major, version.minor, version.patch), (1, 0, 0)); + assert_eq!(version.pre.as_str(), "nightly.123456.2"); + assert_eq!(version.to_string(), "1.0.0-nightly.123456.2"); + } +} diff --git a/external/opennow/native/opennow-core/tests/core_protocol_cli.rs b/external/opennow/native/opennow-core/tests/core_protocol_cli.rs new file mode 100644 index 000000000..abe58c002 --- /dev/null +++ b/external/opennow/native/opennow-core/tests/core_protocol_cli.rs @@ -0,0 +1,390 @@ +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn hello(version: u32) -> Value { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!("opennow-protocol-{version}-{unique}")); + let mut child = Command::new(env!("CARGO_BIN_EXE_opennow-core")) + .arg("--data-dir") + .arg(&directory) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "{}", + json!({ + "type":"request","id":"hello","method":"core.hello", + "params":{"protocolVersion":version,"shell":"qt"} + }) + ) + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let reader = std::thread::spawn(move || { + let mut line = String::new(); + let result = BufReader::new(stdout).read_line(&mut line).map(|_| line); + sender.send(result).unwrap(); + }); + let response = receiver.recv_timeout(Duration::from_secs(10)); + let _ = child.kill(); + child.wait().unwrap(); + reader.join().unwrap(); + if directory.exists() { + std::fs::remove_dir_all(directory).unwrap(); + } + serde_json::from_str(&response.expect("core handshake timed out").unwrap()).unwrap() +} + +#[test] +fn protocol_three_shells_are_rejected_before_the_paged_library_contract() { + let response = hello(3); + assert_eq!(response["ok"], false); + assert_eq!(response["error"]["code"], "incompatible_protocol"); + assert!(response.get("result").is_none()); +} + +#[test] +fn protocol_four_shells_receive_the_paged_library_capabilities() { + let response = hello(4); + assert_eq!(response["ok"], false); + assert_eq!(response["error"]["code"], "incompatible_protocol"); +} + +#[test] +fn protocol_five_shells_receive_the_paged_library_capabilities() { + let response = hello(5); + assert_eq!(response["ok"], true); + assert_eq!(response["result"]["protocolVersion"], 5); + let capabilities = response["result"]["capabilities"].as_array().unwrap(); + for capability in [ + "catalog.libraryPages.v1", + "catalog.metadata.v1", + "account.syncObservation.v1", + "account.pushInvalidation.v1", + "catalog.languages.v1", + "queue.servers.v1", + "auth.accountPicker.v1", + "session.terminalCleanup.v1", + ] { + assert!(capabilities.contains(&json!(capability))); + } +} + +#[test] +fn writable_cores_exclusively_own_the_resolved_profile_until_exit() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!("opennow-profile-lock-{unique}")); + let other_directory = directory.join("other-profile"); + let mut first = Command::new(env!("CARGO_BIN_EXE_opennow-core")) + .args(["--data-dir", directory.to_str().unwrap()]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + writeln!(first.stdin.as_mut().unwrap(), "{}", json!({ + "type":"request","id":"owner","method":"core.hello","params":{"protocolVersion":5,"shell":"qt"} + })).unwrap(); + let stdout = first.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let reader = std::thread::spawn(move || { + let mut line = String::new(); + let _ = sender.send(BufReader::new(stdout).read_line(&mut line).map(|_| line)); + }); + let ready = receiver.recv_timeout(Duration::from_secs(10)); + let launch = |path: &std::path::Path, graphics: bool| { + let mut command = Command::new(env!("CARGO_BIN_EXE_opennow-core")); + command.arg("--data-dir").arg(path).stdin(Stdio::null()); + if graphics { + command.arg("--graphics-preferences"); + } + command.output().unwrap() + }; + let second = launch(&directory, false); + let independent = launch(&other_directory, false); + let read_only = launch(&directory, true); + first.kill().unwrap(); + first.wait().unwrap(); + reader.join().unwrap(); + let recovered = launch(&directory, false); + std::fs::remove_dir_all(&directory).unwrap(); + let hello: Value = + serde_json::from_str(&ready.expect("owner handshake timed out").unwrap()).unwrap(); + assert_eq!(hello["ok"], true); + assert!( + !second.status.success(), + "a second writable core admitted the same profile" + ); + assert!(String::from_utf8_lossy(&second.stderr).contains("data directory is already in use")); + assert!( + independent.status.success(), + "a distinct profile was blocked" + ); + assert!( + read_only.status.success(), + "read-only graphics preferences were blocked" + ); + assert!( + recovered.status.success(), + "process exit left a stale profile lock" + ); +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +mod pending_profile_owner { + use super::*; + use fs2::FileExt; + use std::fs::{File, OpenOptions}; + use std::io::{self, Read}; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::process::CommandExt; + use std::process::Child; + use std::thread::{self, JoinHandle}; + use std::time::Instant; + + struct ReapedChild { + child: Child, + reader: Option>, + } + + impl Drop for ReapedChild { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + if let Some(reader) = self.reader.take() { + let _ = reader.join(); + } + } + } + + fn pipe() -> (File, File) { + let mut descriptors = [-1; 2]; + assert_eq!( + unsafe { libc::pipe2(descriptors.as_mut_ptr(), libc::O_CLOEXEC) }, + 0 + ); + unsafe { + ( + File::from_raw_fd(descriptors[0]), + File::from_raw_fd(descriptors[1]), + ) + } + } + + fn signal(reader: &mut File, expected: u8) { + let mut descriptor = libc::pollfd { + fd: reader.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + assert_eq!( + unsafe { libc::poll(&mut descriptor, 1, 10_000) }, + 1, + "fixture signal timed out" + ); + let mut byte = [0]; + reader.read_exact(&mut byte).unwrap(); + assert_eq!(byte[0], expected); + } + + fn response(receiver: &mpsc::Receiver>, id: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let line = receiver + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("core response timed out") + .unwrap(); + let value: Value = serde_json::from_str(&line).unwrap(); + if value["id"] == id { + return value; + } + } + } + + #[test] + fn pending_mutable_worker_keeps_profile_locked_after_main_loop_error() { + let directory = tempfile::tempdir().unwrap(); + let fixture = directory.path().join("profile-fsync-gate.so"); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/profile_fsync_gate.c"); + let compiled = Command::new("cc") + .args(["-shared", "-fPIC", "-Wall", "-Wextra", "-Werror"]) + .arg(source) + .args(["-o"]) + .arg(&fixture) + .arg("-ldl") + .output() + .unwrap(); + assert!( + compiled.status.success(), + "fsync fixture compilation failed" + ); + let armed = directory.path().join("armed"); + let (mut ready_read, ready_write) = pipe(); + let (mut error_read, error_write) = pipe(); + let (release_read, mut release_write) = pipe(); + let (_stderr_read, mut stderr_write) = pipe(); + let inherited = [ + ready_write.as_raw_fd(), + error_write.as_raw_fd(), + release_read.as_raw_fd(), + ]; + let executable = std::env::var_os("OPENNOW_PROFILE_TEST_CORE") + .unwrap_or_else(|| env!("CARGO_BIN_EXE_opennow-core").into()); + let mut command = Command::new(&executable); + command + .arg("--data-dir") + .arg(directory.path()) + .env("LD_PRELOAD", fixture) + .env("PROFILE_FSYNC_ARMED", &armed) + .env("PROFILE_FSYNC_READY_FD", inherited[0].to_string()) + .env("PROFILE_ERROR_FD", inherited[1].to_string()) + .env("PROFILE_FSYNC_RELEASE_FD", inherited[2].to_string()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(stderr_write.try_clone().unwrap())); + unsafe { + command.pre_exec(move || { + for fd in inherited { + if libc::fcntl(fd, libc::F_SETFD, 0) < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + }); + } + let mut first = ReapedChild { + child: command.spawn().unwrap(), + reader: None, + }; + drop((ready_write, error_write, release_read)); + let stdout = first.child.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + first.reader = Some(thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if sender.send(line).is_err() { + break; + } + } + })); + writeln!( + first.child.stdin.as_mut().unwrap(), + "{}", + json!({ + "type":"request", "id":"owner", "method":"core.hello", + "params":{"protocolVersion":5,"shell":"qt"} + }) + ) + .unwrap(); + assert_eq!(response(&receiver, "owner")["ok"], true); + + let fd = stderr_write.as_raw_fd(); + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + assert!(flags >= 0); + assert_eq!( + unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) }, + 0 + ); + loop { + match stderr_write.write(&[b'x'; 4096]) { + Ok(count) => assert!(count > 0), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => break, + Err(error) => panic!("could not fill fixture stderr pipe: {error}"), + } + } + assert_eq!(unsafe { libc::fcntl(fd, libc::F_SETFL, flags) }, 0); + File::create(&armed).unwrap(); + writeln!( + first.child.stdin.as_mut().unwrap(), + "{}", + json!({ + "type":"request", "id":"width", "method":"settings.set", + "params":{"key":"windowWidth","value":1234} + }) + ) + .unwrap(); + signal(&mut ready_read, b's'); + writeln!(first.child.stdin.as_mut().unwrap(), "invalid-json").unwrap(); + signal(&mut error_read, b'e'); + assert!(first.child.try_wait().unwrap().is_none()); + + let mut second = ReapedChild { + child: Command::new(&executable) + .arg("--data-dir") + .arg(directory.path()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + reader: None, + }; + let deadline = Instant::now() + Duration::from_secs(10); + let status = loop { + if let Some(status) = second.child.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "second core did not exit"); + thread::sleep(Duration::from_millis(5)); + }; + let mut error = String::new(); + second + .child + .stderr + .take() + .unwrap() + .read_to_string(&mut error) + .unwrap(); + assert!( + !status.success(), + "a second writable core admitted a profile while a pending worker survived main-loop failure" + ); + assert!(error.contains("data directory is already in use")); + + release_write.write_all(b"r").unwrap(); + let saved = response(&receiver, "width"); + assert_eq!(saved["ok"], true); + assert_eq!(saved["result"]["value"], 1234); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let tasks = std::fs::read_dir(format!("/proc/{}/task", first.child.id())).unwrap(); + let rpc_running = tasks.filter_map(Result::ok).any(|entry| { + std::fs::read_to_string(entry.path().join("comm")) + .is_ok_and(|name| name.starts_with("opennow-rpc-")) + }); + if !rpc_running { + break; + } + assert!(Instant::now() < deadline, "RPC worker did not finish"); + thread::sleep(Duration::from_millis(5)); + } + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(directory.path().join("core.lock")) + .unwrap(); + assert!(first.child.try_wait().unwrap().is_none()); + let error = lock + .try_lock_exclusive() + .expect_err("live core process released profile ownership after RPC teardown"); + assert_eq!( + error.raw_os_error(), + fs2::lock_contended_error().raw_os_error() + ); + drop(first); + lock.try_lock_exclusive() + .expect("terminated core retained profile ownership"); + } +} diff --git a/external/opennow/native/opennow-core/tests/fixtures/profile_fsync_gate.c b/external/opennow/native/opennow-core/tests/fixtures/profile_fsync_gate.c new file mode 100644 index 000000000..0576bf393 --- /dev/null +++ b/external/opennow/native/opennow-core/tests/fixtures/profile_fsync_gate.c @@ -0,0 +1,61 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +static void report_main_error(int fd, const void *data, size_t length) +{ + static int reported; + if (fd != STDERR_FILENO || !memmem(data, length, "opennow-core:", 13)) return; + const char *signal_fd = getenv("PROFILE_ERROR_FD"); + if (!signal_fd || __atomic_exchange_n(&reported, 1, __ATOMIC_SEQ_CST)) return; + ssize_t (*real_write)(int, const void *, size_t) = dlsym(RTLD_NEXT, "write"); + const char signal = 'e'; + real_write(atoi(signal_fd), &signal, 1); +} + +ssize_t write(int fd, const void *data, size_t length) +{ + ssize_t (*real_write)(int, const void *, size_t) = dlsym(RTLD_NEXT, "write"); + report_main_error(fd, data, length); + return real_write(fd, data, length); +} + +ssize_t writev(int fd, const struct iovec *vectors, int count) +{ + ssize_t (*real_writev)(int, const struct iovec *, int) = dlsym(RTLD_NEXT, "writev"); + for (int i = 0; i < count; ++i) + report_main_error(fd, vectors[i].iov_base, vectors[i].iov_len); + return real_writev(fd, vectors, count); +} + +int fsync(int fd) +{ + static int entered; + int (*real_fsync)(int) = dlsym(RTLD_NEXT, "fsync"); + int result = real_fsync(fd); + int saved_errno = errno; + const char *armed = getenv("PROFILE_FSYNC_ARMED"); + char link[64], path[4096]; + snprintf(link, sizeof(link), "/proc/self/fd/%d", fd); + ssize_t length = readlink(link, path, sizeof(path) - 1); + if (length > 0) path[length] = 0; + if (armed && access(armed, F_OK) == 0 && length > 0 + && strstr(path, "/settings.json.tmp") + && !__atomic_exchange_n(&entered, 1, __ATOMIC_SEQ_CST)) { + const char *ready_fd = getenv("PROFILE_FSYNC_READY_FD"); + const char *release_fd = getenv("PROFILE_FSYNC_RELEASE_FD"); + if (ready_fd && release_fd) { + const char signal = 's'; + write(atoi(ready_fd), &signal, 1); + char release; + while (read(atoi(release_fd), &release, 1) < 0 && errno == EINTR) {} + } + } + errno = saved_errno; + return result; +} diff --git a/external/opennow/native/opennow-core/tests/graphics_preferences_cli.rs b/external/opennow/native/opennow-core/tests/graphics_preferences_cli.rs new file mode 100644 index 000000000..0c57237a4 --- /dev/null +++ b/external/opennow/native/opennow-core/tests/graphics_preferences_cli.rs @@ -0,0 +1,48 @@ +use serde_json::json; +use std::fs; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn graphics_preferences_cli_prints_one_exact_json_line_without_mutating_settings() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!("opennow-graphics-cli-{unique}")); + let path = directory.join("settings.json"); + fs::create_dir_all(&directory).unwrap(); + let persisted = serde_json::to_vec(&json!({ + "windowsGpuDeviceId":"gpu-ü-1", + "mouseAcceleration":true, + "futureSecret":"do-not-print" + })) + .unwrap(); + fs::write(&path, &persisted).unwrap(); + + let mut explicit = Command::new(env!("CARGO_BIN_EXE_opennow-core")); + explicit + .arg("--graphics-preferences") + .arg("--data-dir") + .arg(&directory); + let mut environment = Command::new(env!("CARGO_BIN_EXE_opennow-core")); + environment + .arg("--graphics-preferences") + .env("OPENNOW_DATA_DIR", &directory); + + for mut command in [explicit, environment] { + let output = command.output().unwrap(); + assert!(output.status.success()); + assert_eq!( + output.stdout, + "{\"version\":1,\"windowsGpuDeviceId\":\"gpu-ü-1\"}\n".as_bytes() + ); + assert!(output.stderr.is_empty()); + } + assert_eq!(fs::read(&path).unwrap(), persisted); + assert!(!directory.join("settings.json.bak").exists()); + assert!(!directory.join("settings.json.tmp").exists()); + assert!(!directory.join("settings.json.corrupt").exists()); + assert_eq!(fs::read_dir(&directory).unwrap().count(), 1); + fs::remove_dir_all(directory).unwrap(); +} diff --git a/external/opennow/native/opennow-core/tests/legacy_contract.rs b/external/opennow/native/opennow-core/tests/legacy_contract.rs new file mode 100644 index 000000000..9d0b74e71 --- /dev/null +++ b/external/opennow/native/opennow-core/tests/legacy_contract.rs @@ -0,0 +1,112 @@ +use serde_json::Value; +use std::collections::HashSet; + +const MANIFEST: &str = include_str!("../contracts/legacy-open-now-api.json"); +const SCHEMA: &str = include_str!("../contracts/legacy-open-now-api.schema.json"); +const CORE_DISPATCH: &str = include_str!("../src/main.rs"); + +#[test] +fn legacy_api_manifest_is_complete_unique_and_fixture_backed() { + let manifest: Value = serde_json::from_str(MANIFEST).expect("contract manifest must be JSON"); + let schema: Value = serde_json::from_str(SCHEMA).expect("contract schema must be JSON"); + assert_eq!(manifest["version"], 1); + assert_eq!(schema["properties"]["operations"]["minItems"], 109); + let expected_columns = [ + "legacy", + "kind", + "requestSchema", + "resultSchema", + "owner", + "replacement", + "status", + "requestFixture", + "resultFixture", + ]; + assert_eq!( + manifest["columns"].as_array().expect("columns"), + &expected_columns.map(Value::from) + ); + + let operations = manifest["operations"].as_array().expect("operations array"); + assert_eq!( + operations.len(), + 109, + "OpenNowApi surface changed without a contract update" + ); + let mut names = HashSet::new(); + for operation in operations { + let fields = operation.as_array().expect("operation tuple"); + assert_eq!(fields.len(), 9); + let name = fields[0].as_str().expect("legacy name"); + assert!(names.insert(name), "duplicate legacy operation {name}"); + assert!(matches!( + fields[1].as_str(), + Some("request" | "send" | "event") + )); + assert!( + !fields[2].as_str().unwrap_or_default().is_empty(), + "{name} request schema" + ); + assert!( + !fields[3].as_str().unwrap_or_default().is_empty(), + "{name} result schema" + ); + assert!(matches!( + fields[4].as_str(), + Some("core" | "qt" | "streamer") + )); + assert!( + !fields[5].as_str().unwrap_or_default().is_empty(), + "{name} replacement" + ); + assert!(matches!( + fields[6].as_str(), + Some("ported" | "superseded" | "pending") + )); + assert!(fields.get(7).is_some(), "{name} request fixture"); + assert!(fields.get(8).is_some(), "{name} result fixture"); + let serialized = serde_json::to_string(operation).expect("fixture must serialize"); + for secret in ["access_token", "refresh_token", "client_token", "password"] { + assert!( + !serialized.to_ascii_lowercase().contains(secret), + "{name} fixture leaks {secret}" + ); + } + } +} + +#[test] +fn legacy_api_manifest_has_no_pending_implementation() { + let manifest: Value = serde_json::from_str(MANIFEST).expect("contract manifest must be JSON"); + for operation in manifest["operations"].as_array().expect("operations array") { + let fields = operation.as_array().expect("operation tuple"); + assert_ne!( + fields[6].as_str(), + Some("pending"), + "{} still has no Qt/Rust parity disposition", + fields[0].as_str().unwrap_or("unnamed operation") + ); + } +} + +#[test] +fn every_ported_core_rpc_is_present_in_the_dispatcher() { + let manifest: Value = serde_json::from_str(MANIFEST).unwrap(); + for operation in manifest["operations"].as_array().unwrap() { + let fields = operation.as_array().unwrap(); + if fields[4] != "core" || fields[6] != "ported" { + continue; + } + let replacement = fields[5].as_str().unwrap(); + for method in replacement.split(" + ") { + if !method.contains('.') || method.contains(' ') { + continue; + } + assert!( + CORE_DISPATCH.contains(&format!("\"{method}\"")), + "{} claims missing core method {method}", + fields[0] + ); + } + } +} diff --git a/external/opennow/native/opennow-core/tests/media_env_cli.rs b/external/opennow/native/opennow-core/tests/media_env_cli.rs new file mode 100644 index 000000000..33b58bb1e --- /dev/null +++ b/external/opennow/native/opennow-core/tests/media_env_cli.rs @@ -0,0 +1,132 @@ +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc; +use std::time::Duration; +use tempfile::TempDir; + +struct Session { + child: Child, + stdin: ChildStdin, + lines: mpsc::Receiver, + _directory: TempDir, +} + +impl Session { + fn start(environment: &[(&str, &str)]) -> Self { + let directory = TempDir::new().unwrap(); + let mut command = Command::new(env!("CARGO_BIN_EXE_opennow-core")); + command + .arg("--data-dir") + .arg(directory.path()) + .env_remove("OPENNOW_PICTURES_DIR") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + for (name, value) in environment { + command.env(name, value); + } + let mut child = command.spawn().unwrap(); + let stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + if sender.send(line).is_err() { + break; + } + } + }); + Self { + child, + stdin, + lines, + _directory: directory, + } + } + + fn request(&mut self, id: &str, method: &str, params: Value) -> Value { + writeln!( + self.stdin, + "{}", + json!({"type":"request","id":id,"method":method,"params":params}) + ) + .unwrap(); + loop { + let line = self + .lines + .recv_timeout(Duration::from_secs(10)) + .expect("core response timed out"); + let value: Value = serde_json::from_str(&line).unwrap(); + if value["type"] == "response" && value["id"] == id { + return value; + } + } + } + + fn handshake(&mut self) { + let response = self.request( + "hello", + "core.hello", + json!({"protocolVersion":5,"shell":"qt"}), + ); + assert_eq!(response["ok"], true); + } +} + +impl Drop for Session { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +#[test] +fn empty_pictures_override_keeps_unrelated_startup_working_while_media_is_unavailable() { + let mut session = Session::start(&[("OPENNOW_PICTURES_DIR", "")]); + session.handshake(); + let media = session.request("media", "media.root.get", json!({})); + assert_eq!(media["ok"], false); + assert_eq!(media["error"]["code"], "media_unavailable"); + let unrelated = session.request("thanks", "thanks.data.get", json!({})); + assert_eq!(unrelated["ok"], true); +} + +#[test] +fn explicit_pictures_override_is_published_by_the_running_core() { + let directory = TempDir::new().unwrap(); + { + let mut session = + Session::start(&[("OPENNOW_PICTURES_DIR", directory.path().to_str().unwrap())]); + session.handshake(); + let media = session.request("media", "media.root.get", json!({})); + assert_eq!(media["ok"], true); + assert_eq!( + media["result"]["path"], + directory.path().join("OpenNOW").to_string_lossy().as_ref() + ); + } +} + +#[test] +fn absent_pictures_override_keeps_the_platform_fallback() { + let home = TempDir::new().unwrap(); + { + let mut session = Session::start(&[ + ("HOME", home.path().to_str().unwrap()), + ("USERPROFILE", home.path().to_str().unwrap()), + ]); + session.handshake(); + let media = session.request("media", "media.root.get", json!({})); + assert_eq!(media["ok"], true); + assert_eq!( + media["result"]["path"], + home.path() + .join("Pictures") + .join("OpenNOW") + .to_string_lossy() + .as_ref() + ); + } +} diff --git a/external/opennow/native/opennow-core/tests/settings_event_order_cli.rs b/external/opennow/native/opennow-core/tests/settings_event_order_cli.rs new file mode 100644 index 000000000..b4c41e496 --- /dev/null +++ b/external/opennow/native/opennow-core/tests/settings_event_order_cli.rs @@ -0,0 +1,126 @@ +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::time::Duration; + +#[test] +fn settings_events_precede_acknowledgements_and_other_events_keep_their_order() { + let directory = tempfile::tempdir().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_opennow-core")) + .args(["--data-dir", directory.path().to_str().unwrap()]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + if sender + .send(serde_json::from_str::(&line).unwrap()) + .is_err() + { + break; + } + } + }); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut serial = 0; + let mut request = |method: &str, params: Value| { + serial += 1; + let id = serial.to_string(); + writeln!( + child.stdin.as_mut().unwrap(), + "{}", + json!({"type":"request","id":id,"method":method,"params":params}) + ) + .unwrap(); + id + }; + let next = || { + receiver + .recv_timeout(Duration::from_secs(5)) + .expect("core response timed out") + }; + for (key, value, changes) in [ + ( + "webrtcCompatibilityMode", + json!("on"), + Some(json!({"allianceWebrtcCompatibility":true})), + ), + ( + "webrtcCompatibilityMode", + json!("off"), + Some(json!({"allianceWebrtcCompatibility":false})), + ), + ( + "webrtcCompatibilityMode", + json!("auto"), + Some(json!({"allianceWebrtcCompatibility":false})), + ), + ( + "allianceWebrtcCompatibility", + json!(true), + Some(json!({"webrtcCompatibilityMode":"on"})), + ), + ( + "allianceWebrtcCompatibility", + json!(false), + Some(json!({"webrtcCompatibilityMode":"off"})), + ), + ("gameLanguage", json!("es_419"), None), + ("gameLanguage", json!("zh_Hant_TW"), None), + ("launchInConsoleMode", json!(true), None), + ( + "launchInConsoleMode", + json!(false), + Some(json!({"switchToConsoleOnPad":false})), + ), + ( + "themePack", + json!("bone"), + Some(json!({"appTheme":"light","themeAccentOverride":false})), + ), + ( + "microphoneMode", + json!("voice-activity"), + Some(json!({"microphoneDeviceId":""})), + ), + ] { + let id = request("settings.set", json!({"key":key,"value":value})); + let event = next(); + assert_eq!( + event["type"], "event", + "a queued next write must not start before this event" + ); + assert_eq!(event["name"], "settings.changed"); + assert_eq!(event["payload"]["key"], key); + assert_eq!(event["payload"]["value"], value); + if let Some(changes) = changes { + assert_eq!(event["payload"]["changes"], changes); + } + let response = next(); + assert_eq!(response["id"], id); + assert_eq!(response["ok"], true); + assert_eq!(response["result"], event["payload"]); + } + let id = request("settings.set", json!({"key":"gameLanguage","value":"auto"})); + let rejected = next(); + assert_eq!(rejected["id"], id); + assert_eq!(rejected["ok"], false); + let id = request("settings.reset", json!({})); + let response = next(); + assert_eq!(response["id"], id); + assert_eq!(response["ok"], true); + assert_eq!(next()["name"], "settings.reset"); + })); + let _ = child.kill(); + child.wait().unwrap(); + reader.join().unwrap(); + if let Err(error) = result { + std::panic::resume_unwind(error); + } +} diff --git a/external/opennow/native/opennow-streamer/Cargo.lock b/external/opennow/native/opennow-streamer/Cargo.lock new file mode 100644 index 000000000..fac63a008 --- /dev/null +++ b/external/opennow/native/opennow-streamer/Cargo.lock @@ -0,0 +1,3176 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "ash-window" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52bca67b61cb81e5553babde81b8211f713cb6db79766f80168f3e5f40ea6c82" +dependencies = [ + "ash", + "raw-window-handle", + "raw-window-metal", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "audiopus_sys" +version = "0.2.2" +dependencies = [ + "cmake", + "log", + "pkg-config", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.117", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.3", + "shlex 1.3.0", + "syn 2.0.117", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytestring" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "ccm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847" +dependencies = [ + "aead", + "cipher", + "ctr", + "subtle", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cocoa" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6140449f97a6e97f9511815c5632d84c8aacf8ac271ad77c559218161a1373c" +dependencies = [ + "bitflags 1.3.2", + "block", + "cocoa-foundation", + "core-foundation 0.9.4", + "core-graphics", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6234cbb2e4c785b456c0644748b1ac416dd045799740356f8363dfe00c93f7" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation 0.9.4", + "core-graphics-types", + "libc", + "objc", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cros-codecs-extended" +version = "0.0.5-extended.2" +dependencies = [ + "anyhow", + "byteorder", + "crc32fast", + "cros-libva-extended", + "drm", + "drm-fourcc", + "log", + "nix", + "thiserror 1.0.69", + "zerocopy", +] + +[[package]] +name = "cros-libva-extended" +version = "0.0.13-extended.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a734fee2508191b121a756bd5eb35caf0aa096e7f20b7b6020e689a75edd92e5" +dependencies = [ + "bindgen 0.70.1", + "bitflags 2.13.1", + "log", + "pkg-config", + "regex", + "thiserror 1.0.69", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dimpl" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6aa42b0c64c3e5311a2afad224b32db1ee129d21c63daaaf8ea747b846cdbc" +dependencies = [ + "aes", + "aes-gcm", + "arrayvec", + "aws-lc-rs", + "ccm", + "chacha20", + "chacha20poly1305", + "der", + "ecdsa", + "generic-array", + "hkdf", + "hmac", + "log", + "nom 8.0.0", + "once_cell", + "p256", + "p384", + "pkcs8", + "rand", + "rand_core 0.6.4", + "rcgen", + "sec1", + "sha2", + "signature", + "spki", + "subtle", + "time", + "x25519-dalek", + "x509-cert", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "drm" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98888c4bbd601524c11a7ed63f814b8825f420514f78e96f752c437ae9cbb5d1" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "rustix", +] + +[[package]] +name = "drm-ffi" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97c98727e48b7ccb4f4aea8cfe881e5b07f702d17b7875991881b41af7278d53" +dependencies = [ + "drm-sys", + "rustix", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd39dde40b6e196c2e8763f23d119ddb1a8714534bf7d77fa97a65b0feda3986" +dependencies = [ + "libc", + "linux-raw-sys 0.6.5", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "ffmpeg-next" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6380599799e175191eb7ffe82c97f36a2a90a36cbc54c738a903e5287d7f516a" +dependencies = [ + "bitflags 2.13.1", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "9.0.0" +dependencies = [ + "bindgen 0.72.1", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08f9118d003d441f79c1070e84d0a2a89f35721ca8ae0cd5e1f9026dc4a2517" +dependencies = [ + "crc", + "serde", + "str0m-proto", + "subtle", + "tracing", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a385b1be4e5c3e362ad2ffa73c392e53f031eaa5b7d648e64cd87f27f6063d7" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "mtu" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6915088ba81d679b0a07ecc674f31d66af5e7edaeb0c34782669be38cdf67b5" +dependencies = [ + "bindgen 0.72.1", + "cfg_aliases 0.2.2", + "libc", + "static_assertions", + "windows", +] + +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "log", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", + "objc2-exception-helper", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image", + "objc2-core-text", + "objc2-core-video", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.1", + "block2", + "dispatch2", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "block2", + "dispatch2", + "libc", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.1", + "block2", + "dispatch2", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", + "objc2-metal", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-exception-helper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a1c5fbb72d7735b076bb47b578523aedc40f3c439bea6dfd595c089d79d98a" +dependencies = [ + "cc", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.13.1", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.13.1", + "block2", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-video", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-video-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openh264" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc9071a0b5f9c501ddd01066c2600d922cc799dc450a52975222bcda7755a08" +dependencies = [ + "openh264-sys2", + "wide", +] + +[[package]] +name = "openh264-sys2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ada29a4cadc13d4d326737af87c13e224210d7d99fe47594e9f3f9b0102fd70" +dependencies = [ + "cc", + "nasm-rs", + "walkdir", +] + +[[package]] +name = "opennow-awdl-helper" +version = "0.2.0" +dependencies = [ + "libc", +] + +[[package]] +name = "opennow-streamer" +version = "0.2.0" +dependencies = [ + "opennow-streamer-core", + "opennow-streamer-platform", + "opennow-streamer-protocol", + "serde_json", + "tracing-subscriber", +] + +[[package]] +name = "opennow-streamer-core" +version = "0.2.0" +dependencies = [ + "base64", + "getrandom 0.3.4", + "openh264", + "opennow-streamer-hid", + "opennow-streamer-platform", + "opennow-streamer-protocol", + "opennow-streamer-transport", + "rcgen", + "rustls", + "serde_json", + "str0m", + "tungstenite", +] + +[[package]] +name = "opennow-streamer-ffi" +version = "0.2.0" +dependencies = [ + "opennow-streamer-core", + "opennow-streamer-hid", + "opennow-streamer-platform", + "opennow-streamer-protocol", + "serde_json", +] + +[[package]] +name = "opennow-streamer-hid" +version = "0.2.0" + +[[package]] +name = "opennow-streamer-platform" +version = "0.2.0" +dependencies = [ + "base64", + "cc", + "image", + "libc", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "openh264", + "opennow-streamer-platform-linux", + "opennow-streamer-platform-macos", + "opennow-streamer-platform-windows", + "opennow-streamer-protocol", + "opus", + "oxideav-core", + "oxideav-mkv", + "raw-window-handle", + "scuffle-av1", + "sdl2", + "serde_json", + "windows-sys 0.61.2", + "x11-dl", +] + +[[package]] +name = "opennow-streamer-platform-linux" +version = "0.2.0" +dependencies = [ + "anyhow", + "ash", + "ash-window", + "cros-codecs-extended", + "ffmpeg-next", + "ffmpeg-sys-next", + "libc", + "libloading", + "opus", + "raw-window-handle", + "serde_json", + "static_assertions", + "thiserror 2.0.18", +] + +[[package]] +name = "opennow-streamer-platform-macos" +version = "0.2.0" +dependencies = [ + "block2", + "libc", + "objc2", + "objc2-app-kit", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation", + "objc2-io-surface", + "objc2-metal", + "objc2-quartz-core", + "objc2-video-toolbox", + "opus", + "thiserror 2.0.18", +] + +[[package]] +name = "opennow-streamer-platform-windows" +version = "0.2.0" +dependencies = [ + "opennow-streamer-protocol", + "windows", +] + +[[package]] +name = "opennow-streamer-protocol" +version = "0.2.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "opennow-streamer-transport" +version = "0.2.0" +dependencies = [ + "aes", + "crc32fast", + "ctr", + "getrandom 0.3.4", + "ghash", + "hmac", + "mtu", + "opennow-streamer-hid", + "opennow-streamer-protocol", + "serde", + "serde_json", + "sha1", + "socket2", + "str0m", + "subtle", + "thiserror 2.0.18", + "windows-sys 0.61.2", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3809943dff6fbad5f0484449ea26bdb9cb7d8efdf26ed50d3c7f227f69eb5c" +dependencies = [ + "audiopus_sys", +] + +[[package]] +name = "oxideav-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a68d9a74d6d0a7373589eca0c0bb8ab380c46fd1d7fc807e696d716cdad078c" +dependencies = [ + "bytemuck", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "oxideav-mkv" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8841f155fb729296f6162a143b8509074ff6a2a1b4c09a2b6576c204a50c482" +dependencies = [ + "oxideav-core", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e8caa82e31bb98fee12fa8f051c94a6aa36b07cddb03f0d4fc558988360ff1" +dependencies = [ + "cocoa", + "core-graphics", + "objc", + "raw-window-handle", +] + +[[package]] +name = "rcgen" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted 0.9.0", +] + +[[package]] +name = "safe_arch" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "sctp-proto" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f895c3c33ae20283f9129bd09db2ca69138798b372ef2b98bd2946d23ea4819b" +dependencies = [ + "bytes", + "crc", + "log", + "rand", + "rustc-hash 2.1.3", + "slab", + "thiserror 2.0.18", +] + +[[package]] +name = "scuffle-av1" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028eddc8b17fe9dba817b238c56d3acf03748bdbed4c35783cfb93857ef15955" +dependencies = [ + "byteorder", + "bytes", + "scuffle-bytes-util", + "scuffle-workspace-hack", +] + +[[package]] +name = "scuffle-bytes-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0417748c2a42f4a08d4e634b68b1d64f22a8c24bef2e7ac93df33aa61202a45b" +dependencies = [ + "byteorder", + "bytes", + "bytestring", + "scuffle-workspace-hack", +] + +[[package]] +name = "scuffle-workspace-hack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8028ded836a0d9fabdfa4d713389b76a2098b5153f50a135c8faed7e3a3d5ae2" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "git+https://github.com/zortos293/rust-sdl2.git?rev=8c812c4a0087eaf664a11a844634200cebe9afb8#8c812c4a0087eaf664a11a844634200cebe9afb8" +dependencies = [ + "bitflags 2.13.1", + "lazy_static", + "libc", + "raw-window-handle", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "git+https://github.com/zortos293/rust-sdl2.git?rev=8c812c4a0087eaf664a11a844634200cebe9afb8#8c812c4a0087eaf664a11a844634200cebe9afb8" +dependencies = [ + "bindgen 0.72.1", + "cfg-if", + "cmake", + "libc", + "version-compare", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "str0m" +version = "0.23.0" +dependencies = [ + "arrayvec", + "base64ct", + "combine", + "dimpl", + "fastrand", + "is", + "sctp-proto", + "serde", + "str0m-proto", + "str0m-rust-crypto", + "str0m-wincrypto", + "subtle", + "time", + "tracing", +] + +[[package]] +name = "str0m-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f3d99f2cf6c76a502e45feb896ea71e54787ede8744bcdb215acfbfcb905dd" +dependencies = [ + "base64ct", + "dimpl", + "fastrand", + "serde", + "subtle", + "time", +] + +[[package]] +name = "str0m-rust-crypto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947c2417bf43e47504911f92f47c22b6125a57d162e1661f1c77b78ebb3aa9d3" +dependencies = [ + "aes", + "aes-gcm", + "ctr", + "dimpl", + "hmac", + "p256", + "sha1", + "sha2", + "str0m-proto", + "time", +] + +[[package]] +name = "str0m-wincrypto" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2816a81106af3dfdc321b37da5148238759c900fda9e7f8c11ebe77f1b7a2b73" +dependencies = [ + "dimpl", + "str0m-proto", + "tracing", + "windows", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wide" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de2aaf408e58689c2096682331b1f42bb2d9f2ed6b11560407d023cd0a6c634e" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core 0.6.4", + "serde", + "zeroize", +] + +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/external/opennow/native/opennow-streamer/Cargo.toml b/external/opennow/native/opennow-streamer/Cargo.toml new file mode 100644 index 000000000..96b743b03 --- /dev/null +++ b/external/opennow/native/opennow-streamer/Cargo.toml @@ -0,0 +1,75 @@ +[workspace] +members = [ + "crates/opennow-awdl-helper", + "crates/opennow-streamer", + "crates/opennow-streamer-core", + "crates/opennow-streamer-ffi", + "crates/opennow-streamer-hid", + "crates/opennow-streamer-platform", + "crates/opennow-streamer-platform-linux", + "crates/opennow-streamer-platform-macos", + "crates/opennow-streamer-platform-windows", + "crates/opennow-streamer-protocol", + "crates/opennow-streamer-transport", +] +default-members = ["crates/opennow-streamer"] +exclude = ["vendor/cros-codecs-extended"] +resolver = "2" + +[workspace.package] +version = "0.2.0" +edition = "2024" +license = "MIT" +rust-version = "1.85" + +[workspace.dependencies] +aes = "0.8" +aes-gcm = "0.10" +audiopus_sys = { version = "0.2.2", features = ["static"] } +base64 = "0.22" +ctr = "0.9" +crc32fast = "1" +getrandom = "0.3" +ghash = "0.5" +hmac = "0.12" +image = { version = "0.25", default-features = false, features = ["bmp", "ico", "png"] } +mtu = "0.3" +openh264 = { version = "0.9.8", default-features = false, features = ["source"] } +oxideav-core = "0.1.34" +oxideav-mkv = "0.0.9" +opus = "0.3.1" +# Pin rust-sdl2 after its bundled SDL2 update. SDL 2.26.4 from the crates.io +# release does not compile against current PipeWire headers with modern GCC. +# +# This is upstream adeb978d plus a cherry-pick of upstream #1512, which Windows +# needs because clang types C enums as `int` for MSVC and `unsigned` for Linux +# and `set_mod_state` only accepted the Linux form. The fix cannot be taken by +# moving the pin forward: the commit before it rewrote `Texture<'a>` to +# `Texture<'_>` inside the `unsafe_textures` block, where `Texture` has no +# lifetime, so every revision from there through master fails to build with the +# feature we rely on. Drop this fork once upstream repairs `unsafe_textures`. +sdl2 = { git = "https://github.com/zortos293/rust-sdl2.git", rev = "8c812c4a0087eaf664a11a844634200cebe9afb8", default-features = false, features = ["bundled", "raw-window-handle", "static-link", "unsafe_textures"] } +scuffle-av1 = "0.1.4" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha1 = "0.10" +socket2 = { version = "0.5", features = ["all"] } +str0m = { version = "0.23", default-features = false } +subtle = "2" +thiserror = "2" +rustls = { version = "0.23", default-features = false, features = ["std", "ring", "tls12"] } +tungstenite = { version = "0.28", default-features = false, features = ["handshake", "rustls-tls-native-roots"] } + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = true + +[patch.crates-io] +str0m = { path = "vendor/str0m" } +# Always compile the vendored Opus sources for the distributable streamer. +audiopus_sys = { path = "vendor/audiopus-sys" } +# Upstream does not expose an NVDEC-only bundled build. This patch keeps CUDA +# and nvcuvid as driver-loaded interfaces without requiring the CUDA toolkit. +ffmpeg-sys-next = { path = "vendor/ffmpeg-sys-next" } +cros-codecs-extended = { path = "vendor/cros-codecs-extended" } diff --git a/external/opennow/native/opennow-streamer/README.md b/external/opennow/native/opennow-streamer/README.md new file mode 100644 index 000000000..e9adfc372 --- /dev/null +++ b/external/opennow/native/opennow-streamer/README.md @@ -0,0 +1,315 @@ +# OpenNOW native streamer + +This workspace implements the native GeForce NOW NVST runtime. It owns RTSPS negotiation, the dedicated Mjolnir SRTP video socket, the NVST ICE/DTLS/SCTP bundle used for audio, microphone, RTCP and input, bounded media queues, platform decode/audio output, source-stream Matroska recording and GPU frame publication. It does not implement the browser WebRTC offer/answer or trickle-ICE protocol. + +The Qt application loads `opennow-streamer-ffi` as an in-process shared library. Qt supplies one complete CloudMatch session context; the embedded engine reserves its bundle and Mjolnir sockets and performs OPTIONS, DESCRIBE, SETUP, ANNOUNCE, PLAY, keepalive and TEARDOWN. The runtime does not load or redistribute NVIDIA client libraries. + +## Crates + +### Hardware selection in the Qt app + +Stream settings default to `nativeVideoBackend: auto`. On Windows the embedded presenter +currently uses D3D11; its picker offers an explicit DX11 override and shows DX12/Vulkan as +unavailable. Standalone D3D12 support does **not** imply embedded D3D12 texture interop. +Unsupported forced backends fail explicitly; they never silently select a different API. +On Linux supported decoder overrides are passed per session, without changing process-global +environment variables. Qt's compositor still uses the platform-native presentation API. + +Windows codec detection requires both a usable Media Foundation transform and a matching +DXVA decoder profile/configuration on the probed GPU. Decoder startup repeats the check on +Qt's adopted device with the actual codec, dimensions, bit depth and chroma. Installing an +AV1 codec package on a GPU without AV1 hardware support no longer makes it hardware-capable. +The core resolves Auto against the embedded capability report before CloudMatch allocation. +This is capability selection, not an automatic reconnect or codec change inside a live stream. + +GPU import/shader failures are reported to the shell and Qt log. To diagnose an individual +black-screen report, collect `diagnostics/native-streamer.log` plus the Qt log and distinguish +an entirely black application window from a stream-only black video surface. + +### Workspace components + +- `opennow-streamer-protocol`: versioned local command and session DTOs. +- `opennow-streamer-core`: NVST lifecycle, command routing, media feedback and recording. +- `opennow-streamer-transport`: Mjolnir SRTP plus the NVST-required ICE/DTLS/SCTP, RTCP and input implementation. +- `opennow-streamer-platform`: bounded media queues, decode/audio output, recording and GPU-frame publication. +- `opennow-streamer-platform-{windows,macos,linux}`: platform decoders and native GPU texture producers. +- `opennow-streamer-ffi`: bounded C ABI used in process by Qt. +- `opennow-streamer`: a development JSON-lines host for the same engine; Qt packages do not include it. + +## Qt GPU integration + +The Qt path does not create an SDL video window or a child streamer process. `NativeStreamRuntime` owns the embedded Rust handle, and `StreamVideoItem` drives the GPU-only FFI from Qt's render thread: + +1. Qt lends the current QRhi native graphics objects to the runtime. +2. Platform decode publishes a native GPU frame into a one-frame, drop-stale mailbox. +3. Qt acquires the latest opaque frame token and asks Rust to record conversion and synchronization into the same QRhi command buffer that will render it. +4. Qt imports the returned RGBA8 native texture and samples it in the scene graph, so QML overlays compose above video normally. +5. Qt releases the token after GPU use and shuts down the scene-graph binding on the render thread before QRhi teardown. + +The FFI exposes no CPU image, encoded-frame callback, swap chain, window or Qt object. See `crates/opennow-streamer-ffi/README.md` for ownership and threading details. + +### Local playback mute + +Protocol 7 accepts `{"type":"setAudioMuted","id":"audio-mute-1","muted":true}` +and returns the correlated `ok` response. `muted` must be a boolean; omitting it +returns `missing-muted`. Set it to `false` to restore speaker playback. + +The mute state belongs to the media runtime, defaults to false, and can be set +before starting a session. It survives stops, reconnects, backend fallback, and +audio device recovery within that runtime. Creating a new runtime resets it, so +the shell must resend its desired state when its native runtime becomes ready. +The FFI ABI is unchanged. + +Mute replaces samples with silence only at SDL, ALSA/PipeWire, CoreAudio, and +WASAPI playback boundaries. Playback queues keep draining; transport, Opus decode, +recording, replay capture, microphone capture, and video remain active. Samples +already handed to the operating system or device may finish playing during its +existing bounded output latency; mute does not stop or flush those devices. + +### Audio output selection + +Protocol 5 accepts the additive request `{"type":"audioDevices","id":"audio-1"}` +and returns `{"type":"audioDevices","id":"audio-1","devices":[{"id":"...","name":"..."}]}`. +Failures return a correlated `error` with code `audio-devices-unavailable`; an empty +successful list is not an enumeration failure. Queries run on the existing native +host worker with a two-second response timeout and at most one outstanding query. +Lists exceeding 512 KiB return an error rather than exceeding Qt's callback bound. +Queries neither open a playback device +nor pause, recreate, or stop the active session. + +The optional session-context `settings.audioOutputDevice` string is an opaque ID +from this response. An absent or empty string uses the system default and preserves +existing backend fallback behavior. Nonempty IDs are used only when the next session +starts, must contain no NUL and be at most 1024 UTF-8 bytes, and must identify exactly +one available output. Missing or ambiguous fixed outputs fail explicitly rather than +opening the default. Clients persist IDs, never list positions, device indices, or +display names. Duplicate identities are excluded from enumeration. + +The Qt embedded backends use their actual playback identities: + +- Windows uses exact SDL2 playback device names. SDL playback and enumeration share + the existing embedded host worker; Qt does not create a second audio owner. +- Linux uses `pipewire:` or `alsa:`. PipeWire enumeration + requires `pw-dump` and is bounded to 1.5 seconds and 4 MiB; playback uses `pw-cat`. + ALSA enumeration uses the playback PCM hints from the same dynamically loaded + libasound backend. Default-routing aliases are excluded from fixed ALSA choices. + Fixed outputs disable cross-backend fallback; PipeWire streams also set + `node.dont-fallback` and `node.dont-reconnect`. +- macOS uses `coreaudio:` and passes that UID through native backend + startup and audio recovery rather than persisting an AudioDeviceID. + +The development standalone host uses SDL identities for SDL playback and CoreAudio +identities for native macOS playback. Standalone Windows native WASAPI output does +not support fixed SDL device selection; use the Qt embedded path instead. +The startup selection guarantee does not promise that the operating system will +never reroute an already-playing stream after device loss. + +This extension does not change protocol or FFI ABI versions: old clients omit the +setting and keep default behavior, and new clients must tolerate `unknown-command` +from older runtimes that do not implement enumeration. + +## Packaging + +The Qt CMake build always compiles `opennow-streamer-ffi` with Cargo's release profile and links the resulting shared library to `opennow-qt`. CPack installs that library beside the Qt executable and `opennow-core`; there is no separate streamer executable, helper application or native video window in the Qt package. Linux packages enable the bundled FFmpeg fallback while optional GPU driver interfaces remain dynamically discovered. + +The standalone `opennow-streamer` binary remains a development host and is not evidence for the packaged Qt presentation path. + +### Embedded latency and resource bounds + +- Windows decoder workers wait for actionable input or a bounded 1 ms output + poll; queued frames do not wake a worker that has no decoder input credits. + Closing the queue interrupts either wait immediately. +- Windows conversion targets are allocated lazily for the QRhi slots actually + used, with an eight-slot upper bound. Format changes discard the old slot set. +- Stereo SDL playout retains the 120 ms overflow bound. A backlog above 60 ms + lasting 200 ms enables allocation-free catch-up resampling (at most 2% faster) + until the queued tail reaches 40 ms. Normal playback is bit-exact; clearing + the buffer resets recovery. This controls the application queue, not latency + inside the OS audio device, and the temporary correction can slightly alter pitch. +- Qt samples the converted surface directly in its scene pass; it does not + allocate a second video-sized intermediate render target. Native conversion + and Qt drawing remain on the same graphics command stream. + +## Checks + +### Microphone upstream + +Microphone capture defaults off. The `voice-activity` settings value is exposed as +**Open microphone**: it sends continuous audio, not a voice-activity detector or +push-to-talk implementation. Select it before starting a session; NVST does not +renegotiate microphone support during a stream. Capture uses the system default +input device. A nonempty `microphoneDeviceId` is rejected rather than silently +capturing a different device. + +The native RTSP path requires the server's DESCRIBE to advertise +`x-nv-general.rtcMicOnNativeBundle:1`. Only then, and with microphone opt-in, +ANNOUNCE includes that flag and `x-nv-mic.micSsrcConfig.senderSsrc:1`. The existing +str0m bundle sends the negotiated microphone Opus payload type on SSRC 1, +independently of downlink audio. Handoffs without a microphone payload type retain +PT 111. When the handoff supplies a RED payload type and a nonzero redundancy +level, the sender wraps Opus in RFC 2198 RED with up to three previous frames. +History is cleared on mute/unmute, and unrepresentable offsets, block lengths, +or packets above the payload budget reduce redundancy rather than dropping the +primary frame. This follows the native bundle findings documented in OpenNOW-Mac's +`docs/StreamTransportArchitecture.md` (reference revision `88a09bd68598651b367aa4744e7528da9c074d28`). +Legacy RTSP/UDP microphone carriage is not implemented. + +Capture produces mono 48 kHz PCM and an off-callback encoder produces 10 ms Opus +frames at 16 kbps. The frame size must match the server's `mic.frameSize:10`: the +cloud PC's microphone stays silent when it receives 20 ms frames. PCM, encoded audio, and transport queues each hold at most five +frames and drop old data under load. Mute closes capture and clears pending audio; +unmute preserves the RTP clock and transport sequence lifetime. Capture or uplink +failure disables only the microphone, not game audio/video. Session termination +also closes capture. Queue drops are reported in microphone diagnostics without +logging audio contents. + +Embedded capture devices are owned by the existing embedded runtime host, sharing +SDL ownership with Windows playback and output-device enumeration. Engine keeps +only the microphone session's shared control; the uplink worker receives encoded +packets without owning SDL. Mute pauses hardware synchronously, and the host +reclaims stopped capture on its next bounded poll. The host sleeps without a +polling timer when no capture is active. Output enumeration and input-pause +commands never open, mute, or restart capture. The actual audio-device-aware +stream-start entry point resets the microphone clock; mute/unmute does not. +When SDL reports an active capture device as disconnected, the same host closes +it and tries the default endpoint again at 250 ms intervals for up to one second. +PCM, encoded, and in-flight encoder generations are flushed during reopening. +The original sample-clock origin is retained. Reopening failure disables only +the mic. Disabled capture never opens or retries a device. + +### Timestamp audio jitter buffer + +Authenticated NVST audio RTP is reordered by sender timestamp and sequence before +RED recovery and decoding. The network buffer starts at 45 ms and increases its +target by 20 ms for a new late-arrival burst after playout starts, capped at 200 ms. +It bounds timestamp span to 200 ms, packet count to 80, and individual payload +size to the existing eight-redundant-block RED receive limit. Duplicates and stale +originals are discarded; source changes reset the timeline. `audio-jitter` logs +report the target, queue depth, reordering, stale/overflow/invalid drops, source +resets, and underflow observations once a second without audio contents. + +The Windows GFN logs establish an 80 ms initial target, a dynamic 45 ms floor, +the ceiling, and the growth value, but not the complete target-reduction +predicate. OpenNOW deliberately starts at that 45 ms floor rather than imposing +80 ms permanently without a verified downward adaptation path. After N distinct +underflow bursts, the target is `min(45 + 20*N, 200)` ms until a source reset. +Downward adaptation remains deferred. This adds 45 ms before decode on a clean +network, rising only on late bursts; the existing PCM capacity and roughly 40 ms +catch-up target are unchanged. These are separate queues, so the addition is +45 ms, not merely 5 ms compared with the PCM catch-up target. This is not a total +end-to-end latency cap. OS endpoint buffering is outside the network buffer's +bound. + +The local JSON protocol is version 7; the C ABI is unchanged. `hello` exposes +`supportsMicrophone` for runtime capture support, while the `start` response +reports the negotiated session capability. `microphone-set` requires a boolean +`enabled`; `microphone-toggle` toggles the current capture state. Both require an +active, opted-in, negotiated session. `microphone-state` events contain `state` +(`disabled`, `muted`, `ready`, `unavailable`, or `error`), `enabled`, and an optional +`message`. `ready` confirms local capture/queueing, not that a remote game has +selected the microphone. + +The `start` command accepts an optional boolean `microphoneEnabled`. Setting it +to false reserves negotiated microphone support but does not open capture; Qt +uses this to retain mute during same-session recovery. It cannot override a +disabled microphone mode or a server that does not support bundle capture. + +Live release acceptance must confirm remote voice reception and uninterrupted +game audio, then mute/unmute, disconnect the input device, end/restart the session, +and deny OS permission. Use headphones: acoustic echo cancellation and noise +suppression are not provided by this capture path. + +### Embedded session diagnostics + +Protocol 5 telemetry includes optional `jitterMs` (RTP interarrival jitter on +the video 90 kHz clock) and `packetLossPercent` (authenticated RTP reception loss +over the latest sampled interval, normally one second). Loss sampling requires +new traffic and an interval between one and three seconds. The first sample +establishes a baseline and returns null. A longer gap, stream change, or counter +reset starts a new baseline. Late recovery cannot produce negative loss. Sampling +does not change cumulative RTCP loss or advance RTCP report intervals. Qt forwards +measured values without converting nulls into zeros. + +Qt's shared connection-health state requires at least 0.5% loss sustained for two +seconds before showing an unstable header or popup. A fresh sample below 0.1% +restores healthy status and dismisses the loss popup. Missing or invalid samples, +or five seconds without a fresh measurement, clear health to unknown. Historical +video frame and audio packet drop totals remain unchanged. Loss popups keep their +four-second lifetime and thirty-second cooldown across local overlay changes. + +The optional `pingMs` field measures network round-trip time on the active session. It prefers +the nominated ICE candidate pair's measured RTT, then authenticated STUN/NATT replies +on the dedicated video socket or control/audio bundle. RTSPS keepalive response times +are not used: they include application-level request handling and can overstate network latency. +STUN replies must match an outstanding transaction, the peer address, fingerprint, and +message integrity. Each receiver tracks at most 64 probes. ICE statistics only refresh +the sample when the pair's response count changes; rereading old statistics cannot keep +an old measurement alive. + +The bundle remembers up to 64 emitted ICE transactions and forwards each authenticated +success response to the ICE library only once. Delayed duplicate replies must not overwrite +the original completion time and turn the age of an old probe into the displayed network RTT. +This does not cap genuine network latency or filter first replies based on their timing. + +Session liveness follows [OpenNOW-Mac's live-tested keepalive method](https://github.com/OpenCloudGaming/OpenNOW-Mac/blob/90627114383501dd18ef165baa005d9ea603fdf3/GFN/NVST/Rtsp/NvstRtspConnection.swift#L161-L234): +send `GET_PARAMETER` with the RTSP Session header every two seconds over the existing +RTSPS WebSocket, and match the response. A `551 Option Not Supported` response +still completes the keepalive. Some seats do not answer STUN/ICE probes and close the +connection on client WebSocket pings, so those pings are replaced by session-scoped +RTSP keepalives. Server-initiated WebSocket pings are still answered with pongs. + +Measurements expire after five seconds, and new sessions start without a sample. +Missing or expired ping is emitted as null, including on seats that only answer RTSPS +keepalives. No discovery-server ping, remote-clock +assumptions, stale handshake timing, or synthesized ICE replies supply this metric. +Ping is network RTT, not control-request, decode, one-way video, or input-to-display latency. Decode duration +and end-to-end latency remain unavailable without a measurement source; Qt hides those +two metrics when unavailable in the panel, compact bar, and copied statistics. + +Queue discard events retain `type: "log"`, `event: "queue-dropped"`, `media`, and +the per-source delta `count`. The additive `unit` field is `frames` for video +decode/presentation sources, `samples` for `audio-output`, `packets` for compressed +audio or queued PCM blocks, and `items` for unknown sources. `audio-output` also +includes `sampleRate: 48000` and `channels: 2`; its interleaved sample count converts +to milliseconds as `count * 1000 / (sampleRate * channels)`. Audio packet/block +duration is unknown and must not be inferred from its count. Never sum unlike +units or treat these queue-stage counters as unique lost video frames or network +packet loss. + +Pending deltas flush every second, including when no further drops occur. Full +event queues retain pending counts for retry. Shutdown joins the media producers +and drains their remaining drop feedback before final best-effort delivery. +Final summaries never wait for event-queue capacity: undeliverable deltas are +written directly to the native log with `delivery=event-queue-unavailable`, then +removed. These deltas will be missing from callback/UI totals under saturation. +Each delivered or directly logged final delta is removed, so it is never counted twice. +Native-to-Qt file logs retain every delivered per-source delta and its unit, +independently of periodic telemetry logging. Routine progress/file telemetry and +repeated keyframe-attempt logs use ten-second intervals; errors, warnings and +recovery transitions remain visible. Callback telemetry and wire feedback/retry +cadences are unchanged. + +During Windows decoder recreation, the worker retains one pending recovery +keyframe outside the bounded input queue. Already-queued descendants remain in +FIFO order instead of being silently cleared. This adds at most one owned access +unit while the replacement MFT waits for input credits; it adds no presenter or +CPU pixel-copy path. Regression coverage verifies reference-chain ordering. + +```sh +cargo fmt --manifest-path native/opennow-streamer/Cargo.toml --all -- --check +cargo clippy --manifest-path native/opennow-streamer/Cargo.toml --workspace --all-targets -- -D warnings +cargo test --manifest-path native/opennow-streamer/Cargo.toml --workspace +``` + +Controlled output tests can also run without physical audio hardware: + +```sh +SDL_AUDIODRIVER=dummy cargo test --manifest-path native/opennow-streamer/Cargo.toml -p opennow-streamer-platform selected_sdl_output -- --ignored +ALSA_CONFIG_PATH="$PWD/native/opennow-streamer/crates/opennow-streamer-platform-linux/tests/fixtures/audio-null.conf" cargo test --manifest-path native/opennow-streamer/Cargo.toml -p opennow-streamer-platform-linux selected_alsa_output -- --ignored +``` + +The ignored `selected_pipewire_output` test requires an isolated PipeWire server +and session manager with an `Audio/Sink` node named `opennow_test_output`. Point +`XDG_RUNTIME_DIR` at that server's private runtime directory when running the test. + +Release validation must additionally run authorized live sessions on Windows, Linux/X11, Linux/Wayland, Intel macOS and Apple Silicon to validate NVST interoperability, native GPU import, audio, input, recovery and device-loss behavior. diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/Cargo.toml new file mode 100644 index 000000000..3ed536bf8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "opennow-streamer-core" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +base64.workspace = true +getrandom.workspace = true +opennow-streamer-platform = { path = "../opennow-streamer-platform" } +opennow-streamer-hid = { path = "../opennow-streamer-hid" } +opennow-streamer-protocol = { path = "../opennow-streamer-protocol" } +opennow-streamer-transport = { path = "../opennow-streamer-transport" } +rustls.workspace = true +serde_json.workspace = true +tungstenite.workspace = true + +[dev-dependencies] +openh264.workspace = true +str0m.workspace = true +opennow-streamer-platform = { path = "../opennow-streamer-platform", features = ["test-runtime"] } +rcgen = { version = "0.14", default-features = false, features = ["crypto", "ring"] } diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/decode_progress.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/decode_progress.rs new file mode 100644 index 000000000..69c4c8051 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/decode_progress.rs @@ -0,0 +1,707 @@ +use std::time::{Duration, Instant}; + +use opennow_streamer_platform::DecodeTimingsReport; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeProgressPolicy { + pub stall: Duration, + pub keyframe_grace: Duration, + pub recovery_grace: Duration, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum DecodeProgressStage { + #[default] + Tracking, + KeyframePending, + RecoveryRequired, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecodeProgressEvent { + KeyframeRequested { + idle_for: Duration, + in_flight: usize, + }, + RecoveryNeeded { + idle_for: Duration, + in_flight: usize, + }, +} + +#[derive(Debug, Default)] +pub(crate) struct DecodeProgressWatchdog { + stage: DecodeProgressStage, + stage_started_at: Option, + decoder_epoch: Option, + unsubmitted_since: Option, +} + +impl DecodeProgressWatchdog { + pub(crate) fn stage(&self) -> DecodeProgressStage { + self.stage + } + + pub(crate) fn poll( + &mut self, + timings: &DecodeTimingsReport, + upstream_stalled: bool, + last_assembled_at: Option, + now: Instant, + policy: DecodeProgressPolicy, + ) -> Option { + if self.decoder_epoch != Some(timings.epoch) { + self.decoder_epoch = Some(timings.epoch); + self.clear_episode(); + self.unsubmitted_since = None; + } + if self.stage == DecodeProgressStage::RecoveryRequired { + if !timings.last_output_at.is_some_and(|output| { + self.stage_started_at + .is_some_and(|started| output > started) + }) { + return None; + } + self.clear_episode(); + self.unsubmitted_since = None; + } + if upstream_stalled { + self.clear_episode(); + self.unsubmitted_since = None; + return None; + } + let reference = if timings.in_flight == 0 { + if !last_assembled_at.is_some_and(|assembled| { + now.saturating_duration_since(assembled) < policy.stall + && timings + .last_output_at + .is_none_or(|output| assembled > output) + }) { + self.clear_episode(); + self.unsubmitted_since = None; + return None; + } + let first = *self.unsubmitted_since.get_or_insert(now); + if timings.last_output_at.is_some_and(|output| output >= first) { + self.clear_episode(); + self.unsubmitted_since = None; + return None; + } + first + } else { + self.unsubmitted_since = None; + progress_reference(timings)? + }; + let idle_for = now.saturating_duration_since(reference); + if idle_for < policy.stall { + self.clear_episode(); + return None; + } + match self.stage { + DecodeProgressStage::Tracking => { + self.stage = DecodeProgressStage::KeyframePending; + self.stage_started_at = Some(now); + Some(DecodeProgressEvent::KeyframeRequested { + idle_for, + in_flight: timings.in_flight, + }) + } + DecodeProgressStage::KeyframePending => { + let grace_started = self.stage_started_at.unwrap_or(now); + if now.saturating_duration_since(grace_started) < policy.keyframe_grace { + return None; + } + self.stage = DecodeProgressStage::RecoveryRequired; + self.stage_started_at = Some(now); + Some(DecodeProgressEvent::RecoveryNeeded { + idle_for, + in_flight: timings.in_flight, + }) + } + DecodeProgressStage::RecoveryRequired => None, + } + } + + fn clear_episode(&mut self) { + self.stage = DecodeProgressStage::Tracking; + self.stage_started_at = None; + } +} + +pub(crate) fn progress_reference(timings: &DecodeTimingsReport) -> Option { + let this_epoch_output = timings + .last_output_at + .filter(|at| timings.epoch_started_at.is_none_or(|floor| *at >= floor)); + match (this_epoch_output, timings.oldest_in_flight_at) { + (Some(last), Some(oldest)) => Some(last.max(oldest)), + (Some(last), None) => Some(last), + (None, Some(oldest)) => Some( + timings + .epoch_started_at + .map_or(oldest, |floor| oldest.max(floor)), + ), + (None, None) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn timings(epoch: u64, epoch_started_at: Option) -> DecodeTimingsReport { + DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 0, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: None, + last_output_at: None, + in_flight: 0, + oldest_in_flight_at: None, + epoch, + epoch_started_at, + unmatched_outputs: 0, + unmatched_submissions: 0, + } + } + + fn policy() -> DecodeProgressPolicy { + DecodeProgressPolicy { + stall: Duration::from_secs(8), + keyframe_grace: Duration::from_secs(4), + recovery_grace: Duration::from_secs(8), + } + } + + fn single_pending_submission(base: Instant) -> DecodeTimingsReport { + let mut report = timings(1, Some(base)); + report.submissions_total = 1; + report.in_flight = 1; + report.oldest_in_flight_at = Some(base); + report.last_submission_at = Some(base); + report + } + + #[test] + fn no_outstanding_work_never_reports_a_decode_stall() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let report = timings(1, Some(base)); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(60), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + } + + #[test] + fn arriving_video_without_decoder_submissions_requests_recovery() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = timings(1, Some(base)); + report.last_output_at = Some(base); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(1)), + base + Duration::from_secs(1), + policy() + ), + None + ); + for second in 2..9 { + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(second)), + base + Duration::from_secs(second), + policy() + ), + None + ); + } + assert!(matches!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(9)), + base + Duration::from_secs(9), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { in_flight: 0, .. }) + )); + assert!(matches!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(13)), + base + Duration::from_secs(13), + policy() + ), + Some(DecodeProgressEvent::RecoveryNeeded { in_flight: 0, .. }) + )); + } + + #[test] + fn stale_assembly_and_resumed_output_do_not_trigger_recovery() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = timings(1, Some(base)); + report.last_output_at = Some(base); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(1)), + base + Duration::from_secs(1), + policy() + ), + None + ); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(1)), + base + Duration::from_secs(10), + policy() + ), + None + ); + report.last_output_at = Some(base + Duration::from_secs(11)); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(12)), + base + Duration::from_secs(12), + policy() + ), + None + ); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(19)), + base + Duration::from_secs(19), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + } + + #[test] + fn upstream_stall_restarts_the_unsubmitted_frame_grace_period() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let report = timings(1, Some(base)); + assert_eq!( + watchdog.poll(&report, false, Some(base), base, policy()), + None + ); + assert_eq!( + watchdog.poll( + &report, + true, + Some(base + Duration::from_secs(7)), + base + Duration::from_secs(7), + policy() + ), + None + ); + assert_eq!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(10)), + base + Duration::from_secs(10), + policy() + ), + None + ); + assert!(matches!( + watchdog.poll( + &report, + false, + Some(base + Duration::from_secs(18)), + base + Duration::from_secs(18), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { in_flight: 0, .. }) + )); + } + + #[test] + fn one_pending_submission_with_a_stale_assembly_timestamp_still_escalates() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let report = single_pending_submission(base); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(7), + policy() + ), + None + ); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(8), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { in_flight: 1, .. }) + )); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(12), + policy() + ), + Some(DecodeProgressEvent::RecoveryNeeded { in_flight: 1, .. }) + )); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(60), + policy() + ), + None + ); + } + + #[test] + fn fresh_output_clears_a_pending_keyframe_episode() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = single_pending_submission(base); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(8), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + report.last_output_at = Some(base + Duration::from_secs(9)); + report.oldest_in_flight_at = Some(base + Duration::from_secs(9)); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(9), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + for second in 10..40 { + report.last_output_at = Some(base + Duration::from_secs(second)); + report.oldest_in_flight_at = Some(base + Duration::from_secs(second)); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(second), + policy() + ), + None + ); + } + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + } + + #[test] + fn zero_work_clears_a_pending_episode_and_later_work_starts_fresh() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = single_pending_submission(base); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(8), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + report.in_flight = 0; + report.oldest_in_flight_at = None; + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(9), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + + let later = base + Duration::from_secs(100); + report.in_flight = 1; + report.oldest_in_flight_at = Some(later); + assert_eq!( + watchdog.poll( + &report, + false, + None, + later + Duration::from_secs(1), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + later + Duration::from_secs(8), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + } + + #[test] + fn upstream_ownership_suppresses_until_the_transport_stage_resolves() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let report = single_pending_submission(base); + assert_eq!( + watchdog.poll( + &report, + true, + None, + base + Duration::from_secs(30), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(31), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + } + + #[test] + fn idle_outputs_before_the_epoch_do_not_mask_a_stall() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = timings(2, Some(base + Duration::from_secs(10))); + report.last_output_at = Some(base); + report.in_flight = 1; + report.oldest_in_flight_at = Some(base + Duration::from_secs(10)); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(19), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + } + + #[test] + fn decoder_epoch_change_resets_a_terminal_episode() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let report = single_pending_submission(base); + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(8), + policy(), + ); + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(12), + policy(), + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::RecoveryRequired); + + let mut restarted = timings(2, Some(base + Duration::from_secs(20))); + restarted.in_flight = 1; + restarted.oldest_in_flight_at = Some(base + Duration::from_secs(20)); + assert_eq!( + watchdog.poll( + &restarted, + false, + None, + base + Duration::from_secs(27), + policy() + ), + None + ); + assert!(matches!( + watchdog.poll( + &restarted, + false, + None, + base + Duration::from_secs(28), + policy() + ), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + } + + #[test] + fn same_epoch_output_rearms_recovery_for_a_later_stall() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = single_pending_submission(base); + assert!(matches!( + watchdog.poll(&report, false, None, base + policy().stall, policy()), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + let recovery_at = base + policy().stall + policy().keyframe_grace; + assert!(matches!( + watchdog.poll(&report, false, None, recovery_at, policy()), + Some(DecodeProgressEvent::RecoveryNeeded { .. }) + )); + report.last_output_at = Some(recovery_at - Duration::from_secs(1)); + assert_eq!( + watchdog.poll(&report, false, None, recovery_at, policy()), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::RecoveryRequired); + + let resumed_at = recovery_at + Duration::from_secs(1); + report.last_output_at = Some(resumed_at); + report.outputs_total = 1; + report.in_flight = 0; + report.oldest_in_flight_at = None; + assert_eq!( + watchdog.poll(&report, false, None, resumed_at, policy()), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + + report.in_flight = 1; + report.oldest_in_flight_at = Some(resumed_at); + let stalled_at = resumed_at + policy().stall; + assert!(matches!( + watchdog.poll(&report, false, None, stalled_at, policy()), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + assert!(matches!( + watchdog.poll( + &report, + false, + None, + stalled_at + policy().keyframe_grace, + policy() + ), + Some(DecodeProgressEvent::RecoveryNeeded { .. }) + )); + } + + #[test] + fn outstanding_work_changes_do_not_rearm_without_recovered_output() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = single_pending_submission(base); + watchdog.poll(&report, false, None, base + policy().stall, policy()); + let recovery_at = base + policy().stall + policy().keyframe_grace; + assert!(matches!( + watchdog.poll(&report, false, None, recovery_at, policy()), + Some(DecodeProgressEvent::RecoveryNeeded { .. }) + )); + for second in 1..30 { + let now = recovery_at + Duration::from_secs(second); + report.in_flight = usize::from(second % 2 != 0); + report.oldest_in_flight_at = (report.in_flight != 0).then_some(now); + assert_eq!( + watchdog.poll(&report, false, Some(now), now, policy()), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::RecoveryRequired); + } + } + + #[test] + fn recovering_epoch_does_not_escalate_from_a_prior_epoch_output() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = single_pending_submission(base); + report.last_output_at = Some(base - Duration::from_secs(30)); + report.epoch = 4; + report.epoch_started_at = Some(base); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(1), + policy() + ), + None + ); + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + } + + #[test] + fn advancing_decode_output_keeps_the_watchdog_tracking() { + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let mut report = timings(1, Some(base)); + report.in_flight = 2; + report.oldest_in_flight_at = Some(base); + report.last_output_at = Some(base); + for second in 1..20 { + report.last_output_at = Some(base + Duration::from_secs(second)); + report.oldest_in_flight_at = Some(base + Duration::from_secs(second)); + assert_eq!( + watchdog.poll( + &report, + false, + None, + base + Duration::from_secs(second + 1), + policy() + ), + None + ); + } + assert_eq!(watchdog.stage(), DecodeProgressStage::Tracking); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/lib.rs new file mode 100644 index 000000000..28a15dec5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/lib.rs @@ -0,0 +1,6858 @@ +use std::net::UdpSocket; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::{Receiver, Sender, SyncSender, TrySendError}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use decode_progress::{ + DecodeProgressEvent, DecodeProgressPolicy, DecodeProgressStage, DecodeProgressWatchdog, +}; +use opennow_streamer_hid::HidRuntime; +use opennow_streamer_platform::{ + CapturedInput, CapturedInputQueue, CapturedInputSample, DecodeStageTimings, + DecodeTimingsReport, EncodedFrame, ManualRecordingSummary, MediaCodec, MediaColorQuality, + MediaControl, MediaFeedback, MediaRuntime, MediaRuntimeControl, MediaSession, MediaSink, + MediaStreamConfig, MediaVideoCodec, PushOutcome, StreamShortcutAction, StreamShortcutBindings, + record_matroska, record_replay_matroska, supports_audio_decode, supports_audio_output, + video_backends, +}; +use opennow_streamer_protocol::{ + Capabilities, Command, PROTOCOL_VERSION, RecordingCutReason, ReplayBufferConfig, + SessionContext, error, event, response, +}; +use opennow_streamer_transport::{ + FrameStageTimings, NvstControllerRumble, NvstDropReason, NvstReceiveEvent, NvstReceiverState, + NvstRecovery, NvstUdpReceiverControl, NvstUdpReceiverSession, ReservedNvstBundle, + SharedNvstFeedback, TransportError, parse_nvst_video_handoff, reserve_nvst_mjolnir_udp_socket, + spawn_nvst_mjolnir_receiver, spawn_nvst_udp_receiver_with_socket, +}; +use serde_json::{Value, json}; + +#[cfg(test)] +static LOG_SINK_TEST_LOCK: Mutex<()> = Mutex::new(()); + +mod decode_progress; +mod microphone; +mod nvst_rtsp; +mod queue_drops; +#[cfg(test)] +mod recording_tests; +mod telemetry; +mod webrtc; + +use microphone::MicrophoneController; + +use nvst_rtsp::{ActiveNvstRtspSession, prepare_owned_nvst}; +use queue_drops::QueueDropReports; +use telemetry::{TELEMETRY_INTERVAL, media_telemetry}; + +pub use opennow_streamer_transport::{EncodedMediaFrame, MediaConsumer}; + +#[derive(Clone)] +pub struct EventSender { + inner: EventSenderInner, + start_id: Option>, +} + +#[derive(Clone)] +enum EventSenderInner { + Unbounded(Sender), + Bounded(SyncSender), +} + +impl EventSender { + fn unbounded(sender: Sender) -> Self { + Self { + inner: EventSenderInner::Unbounded(sender), + start_id: None, + } + } + + pub fn bounded(sender: SyncSender) -> Self { + Self { + inner: EventSenderInner::Bounded(sender), + start_id: None, + } + } + + fn for_session(&self, start_id: &str) -> Self { + Self { + inner: self.inner.clone(), + start_id: Some(Arc::from(start_id)), + } + } + + fn scope_lifecycle(&self, mut value: Value) -> Value { + if !matches!(value["type"].as_str(), Some("status" | "error")) + || value.get("startId").is_some() + { + return value; + } + if let Some(start_id) = &self.start_id { + value["startId"] = Value::String(start_id.to_string()); + } + value + } + + fn send(&self, value: Value) -> Result<(), ()> { + let value = self.scope_lifecycle(value); + match &self.inner { + EventSenderInner::Unbounded(sender) => sender.send(value).map_err(|_| ()), + EventSenderInner::Bounded(sender) => match sender.try_send(value) { + Ok(()) => Ok(()), + Err(TrySendError::Full(_) | TrySendError::Disconnected(_)) => Err(()), + }, + } + } + + fn send_terminal(&self, value: Value) -> Result<(), ()> { + let value = self.scope_lifecycle(value); + match &self.inner { + EventSenderInner::Unbounded(sender) => sender.send(value).map_err(|_| ()), + EventSenderInner::Bounded(sender) => sender.send(value).map_err(|_| ()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + Idle, + Connected, +} + +const ENCODED_MEDIA_QUEUE_CAPACITY: usize = 8; +const NVST_RECOVERY_ATTEMPT_LIMIT: usize = 1; +const NATIVE_INPUT_POLL_INTERVAL: Duration = Duration::from_micros(250); +const MAX_STREAM_FPS: u32 = 360; + +trait NvstSessionResources { + fn take_rumble(&self) -> ([Option; 4], usize) { + ([None; 4], 0) + } + fn ping_ms(&self) -> Option { + None + } + fn network_metrics(&self) -> Option<(f64, f64)> { + None + } + fn socket_receive_bytes(&self) -> Option { + None + } + fn frame_stage_timings(&self) -> Option { + None + } + fn decode_progress_policy(&self) -> DecodeProgressPolicy { + DecodeProgressPolicy { + stall: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + keyframe_grace: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + recovery_grace: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + } + } + fn request_keyframe(&self); + fn cut_recording(&self, reason: RecordingCutReason); + fn acknowledge_video_frame(&self, frame_index: u32, bytes: u32); + fn send_captured_input(&self, bytes: Vec) -> Result<(), TransportError>; + fn send_captured_text( + &self, + text: opennow_streamer_protocol::text_input::UnicodeText, + timestamp_us: u64, + ) -> Result<(), TransportError>; + fn apply_cursor(&self, bytes: Vec); + fn recover(&self) -> Result<(), String>; + fn stop(&self); +} + +struct ActiveNvstResources { + bundle: NvstUdpReceiverControl, + mjolnir: Option, + feedback: SharedNvstFeedback, + media: Option, +} + +impl NvstSessionResources for ActiveNvstResources { + fn take_rumble(&self) -> ([Option; 4], usize) { + self.feedback.haptics.take() + } + fn ping_ms(&self) -> Option { + self.feedback.ping_ms(Instant::now()) + } + fn network_metrics(&self) -> Option<(f64, f64)> { + self.feedback.recent_network_metrics(Instant::now()) + } + fn socket_receive_bytes(&self) -> Option { + Some(self.feedback.socket_receive_bytes()) + } + fn frame_stage_timings(&self) -> Option { + let timings = self.feedback.frame_stage_timings(); + (!timings.is_empty()).then_some(timings) + } + fn request_keyframe(&self) { + self.feedback.request_keyframe(); + } + + fn cut_recording(&self, reason: RecordingCutReason) { + if let Some(media) = self.media.as_ref() { + media.cut_recording(reason); + } + } + + fn acknowledge_video_frame(&self, frame_index: u32, bytes: u32) { + self.feedback + .publish_accepted_frame(frame_index, bytes, Instant::now()); + } + + fn send_captured_input(&self, bytes: Vec) -> Result<(), TransportError> { + self.bundle.queue_input(bytes, false) + } + + fn send_captured_text( + &self, + text: opennow_streamer_protocol::text_input::UnicodeText, + timestamp_us: u64, + ) -> Result<(), TransportError> { + self.bundle.queue_text(text, timestamp_us) + } + + fn apply_cursor(&self, bytes: Vec) { + if let Some(media) = self.media.as_ref() { + media.update_cursor(bytes); + } + } + + fn recover(&self) -> Result<(), String> { + self.bundle + .recover() + .map_err(|error| format!("bundle recovery failed: {error}"))?; + if let Some(mjolnir) = self.mjolnir.as_ref() { + mjolnir + .recover() + .map_err(|error| format!("Mjolnir recovery failed: {error}"))?; + } + Ok(()) + } + + fn stop(&self) { + let _ = self.bundle.stop(); + if let Some(mjolnir) = self.mjolnir.as_ref() { + let _ = mjolnir.stop(); + } + if let Some(media) = self.media.as_ref() { + media.stop(); + } + } +} + +pub struct Engine { + lifecycle: Arc>, + webrtc_session: Option, + nvst_transport: Option, + nvst_mjolnir_transport: Option, + reserved_nvst_bundle: Option, + nvst_hole_punch_socket: Option, + nvst_rtsp: Option, + events: EventSender, + media_consumer: Option, + media_runtime: Option, + media_session: Option, + media_worker: Option>, + media_feedback: Option>, + feedback_worker: Option>, + recording_worker: Option, + clip_worker: Option>, + clip_cancelled: Arc, + replay_budget: Arc, + microphone: Option, + hid_runtime: Arc, +} + +struct RecordingWorker { + request_id: String, + thread: JoinHandle>, + completed: Arc, +} + +#[derive(Debug)] +struct Lifecycle { + state: State, + context: Option, + generation: u64, +} + +impl Engine { + pub fn new(events: Sender) -> Self { + Self::with_event_sender(EventSender::unbounded(events)) + } + + pub fn with_event_sender(events: EventSender) -> Self { + Self { + lifecycle: Arc::new(Mutex::new(Lifecycle { + state: State::Idle, + context: None, + generation: 0, + })), + nvst_transport: None, + webrtc_session: None, + nvst_mjolnir_transport: None, + reserved_nvst_bundle: None, + nvst_hole_punch_socket: None, + nvst_rtsp: None, + events, + media_consumer: None, + media_runtime: None, + media_session: None, + media_worker: None, + media_feedback: None, + feedback_worker: None, + recording_worker: None, + clip_worker: None, + clip_cancelled: Arc::new(AtomicBool::new(false)), + replay_budget: Arc::new(AtomicUsize::new(0)), + microphone: None, + hid_runtime: Arc::new(HidRuntime::new()), + } + } + + pub fn embedded(events: EventSender) -> Self { + Self::with_event_sender(events) + } + + pub fn with_media_consumer(events: Sender, media_consumer: MediaConsumer) -> Self { + Self::with_media_consumer_and_event_sender(EventSender::unbounded(events), media_consumer) + } + + pub fn with_media_consumer_and_event_sender( + events: EventSender, + media_consumer: MediaConsumer, + ) -> Self { + Self { + lifecycle: Arc::new(Mutex::new(Lifecycle { + state: State::Idle, + context: None, + generation: 0, + })), + nvst_transport: None, + webrtc_session: None, + nvst_mjolnir_transport: None, + reserved_nvst_bundle: None, + nvst_hole_punch_socket: None, + nvst_rtsp: None, + events, + media_consumer: Some(media_consumer), + media_runtime: None, + media_session: None, + media_worker: None, + media_feedback: None, + feedback_worker: None, + recording_worker: None, + clip_worker: None, + clip_cancelled: Arc::new(AtomicBool::new(false)), + replay_budget: Arc::new(AtomicUsize::new(0)), + microphone: None, + hid_runtime: Arc::new(HidRuntime::new()), + } + } + + pub fn with_media_runtime(events: Sender, media_runtime: MediaRuntime) -> Self { + Self::with_media_runtime_and_event_sender(EventSender::unbounded(events), media_runtime) + } + + pub fn with_media_runtime_and_event_sender( + events: EventSender, + media_runtime: MediaRuntime, + ) -> Self { + Self { + lifecycle: Arc::new(Mutex::new(Lifecycle { + state: State::Idle, + context: None, + generation: 0, + })), + nvst_transport: None, + webrtc_session: None, + nvst_mjolnir_transport: None, + reserved_nvst_bundle: None, + nvst_hole_punch_socket: None, + nvst_rtsp: None, + events, + media_consumer: None, + media_runtime: Some(media_runtime), + media_session: None, + media_worker: None, + media_feedback: None, + feedback_worker: None, + recording_worker: None, + clip_worker: None, + clip_cancelled: Arc::new(AtomicBool::new(false)), + replay_budget: Arc::new(AtomicUsize::new(0)), + microphone: None, + hid_runtime: Arc::new(HidRuntime::new()), + } + } + + pub fn with_embedded_media_runtime(events: EventSender, media_runtime: MediaRuntime) -> Self { + Self::with_media_runtime_and_event_sender(events, media_runtime) + } + + pub fn with_embedded_media_runtime_and_hid( + events: EventSender, + media_runtime: MediaRuntime, + hid_runtime: Arc, + ) -> Self { + let mut engine = Self::with_media_runtime_and_event_sender(events, media_runtime); + engine.hid_runtime = hid_runtime; + engine + } + + pub fn handle(&mut self, command: Command) -> (Vec, bool) { + let summary = opennow_streamer_protocol::log::message_summary(&json!({ + "id": &command.id, "type": &command.kind + })); + opennow_streamer_protocol::log::log_line( + "INFO", + "engine-command", + &format!("begin {summary}"), + ); + let mut stage = opennow_streamer_protocol::log::Stage::begin("engine.command"); + let id = command.id.clone(); + let result = match command.kind.as_str() { + "hello" => self.hello(&command), + "audioDevices" => self.audio_devices(&command), + "setAudioMuted" => self.set_audio_muted(command), + "nvst-bind" => self.nvst_bind(command), + "nvst-unbind" => self.nvst_unbind(command), + "nvst-send" => self.nvst_send(command), + "start" => self.start(command), + "input-paused" => self.set_paused(command), + "surface" => self.update_surface(command), + "stats-toggle" => Ok(vec![ + response(id, "ok"), + event( + "shortcut-action", + json!({"action":"toggle-stats", "source":"command"}), + ), + ]), + "fullscreen-toggle" => Ok(vec![ + response(id, "ok"), + event( + "shortcut-action", + json!({"action":"toggle-fullscreen", "source":"command"}), + ), + ]), + "anti-afk-pulse" => self.anti_afk_pulse(command), + "recording-start" => self.start_recording(command), + "recording-stop" => self.stop_recording(command), + "clip-save" => self.save_clip(command), + "replay-stop" => { + self.stop_replay(); + Ok(vec![response(id, "replay-stopped")]) + } + "microphone-set" | "microphone-toggle" => self.set_microphone(command), + "bitrate" | "update-shortcuts" => Err(error( + Some(&id), + "unsupported-command", + format!("Native streamer cannot apply the {} command", command.kind), + )), + "stop" => { + self.stop(command.reason.as_deref().unwrap_or("stopped")); + Ok(vec![response(id, "ok")]) + } + "shutdown" => { + self.stop(command.reason.as_deref().unwrap_or("shutdown")); + stage.complete(); + return (vec![response(id, "ok")], false); + } + other => Err(error( + Some(&id), + "unknown-command", + format!("Unknown command: {other}"), + )), + }; + + if result.is_ok() { + stage.complete(); + } + opennow_streamer_protocol::log::log_line( + "INFO", + "engine-command", + &format!("end {summary} success={}", result.is_ok()), + ); + match result { + Ok(values) => (values, true), + Err(value) => (vec![value], true), + } + } + + fn hello(&self, command: &Command) -> Result, Value> { + if command.protocol_version != Some(PROTOCOL_VERSION) { + return Err(error( + Some(&command.id), + "protocol-version-mismatch", + format!("Native streamer requires protocol {PROTOCOL_VERSION}"), + )); + } + let backends = self + .media_runtime + .as_ref() + .map(MediaRuntime::video_backends) + .unwrap_or_else(video_backends); + let graphics_adapters = self + .media_runtime + .as_ref() + .map(MediaRuntime::graphics_adapters) + .unwrap_or_default(); + let media_ready = self.media_runtime.is_some(); + let video_ready = media_ready && backends.iter().any(|backend| backend.available); + opennow_streamer_protocol::log::log_line( + "INFO", + "handshake", + &format!( + "protocol={PROTOCOL_VERSION} media_runtime={media_ready} video_available={video_ready} backend_count={} graphics_adapters={}", + backends.len(), + graphics_adapters.len() + ), + ); + let capabilities = Capabilities { + protocol_version: PROTOCOL_VERSION, + backend: "native", + supports_input: media_ready, + supports_video_decode: video_ready, + supports_video_present: video_ready, + supports_audio_decode: media_ready && supports_audio_decode(), + supports_audio_output: media_ready && supports_audio_output(), + supports_microphone: media_ready, + supports_owned_nvst_negotiation: media_ready, + video_backends: backends, + graphics_adapters, + }; + let ready = json!({ + "id": command.id, + "type": "ready", + "processId": std::process::id(), + "capabilities": capabilities, + }); + Ok(vec![ready]) + } + + fn nvst_bind(&mut self, command: Command) -> Result, Value> { + if self.reserved_nvst_bundle.is_none() { + let bundle = ReservedNvstBundle::reserve().map_err(|bind_error| { + error( + Some(&command.id), + "nvst-bind-failed", + format!("failed to reserve NVST UDP socket: {bind_error}"), + ) + })?; + eprintln!( + "NVST reserved video UDP socket on {} (Mjolnir on {})", + bundle + .local_addr() + .map(|addr| addr.to_string()) + .unwrap_or_else(|_| "unknown".to_owned()), + bundle + .mjolnir_local_addr() + .map(|addr| addr.to_string()) + .unwrap_or_else(|_| "unknown".to_owned()), + ); + self.reserved_nvst_bundle = Some(bundle); + } + let bundle = self.reserved_nvst_bundle.as_mut().ok_or_else(|| { + error( + Some(&command.id), + "nvst-bind-failed", + "reserved NVST UDP socket has no local port", + ) + })?; + let local_addr = bundle.local_addr().map_err(|_| { + error( + Some(&command.id), + "nvst-bind-failed", + "reserved NVST UDP socket has no local port", + ) + })?; + let mjolnir_addr = bundle.mjolnir_local_addr().map_err(|_| { + error( + Some(&command.id), + "nvst-bind-failed", + "reserved NVST Mjolnir UDP socket has no local port", + ) + })?; + let port = local_addr.port(); + let local_address = bundle.advertised_local_address(); + let identity = bundle.identity(); + Ok(vec![json!({ + "id": command.id, + "type": "nvst-bound", + "port": port, + "mjolnirPort": mjolnir_addr.port(), + "localAddress": local_address, + "iceUsernameFragment": identity.ice_username_fragment, + "icePassword": identity.ice_password, + "dtlsFingerprint": identity.dtls_fingerprint, + })]) + } + + fn nvst_send(&mut self, command: Command) -> Result, Value> { + let host = command.host.ok_or_else(|| { + error( + Some(&command.id), + "nvst-send-failed", + "nvst-send requires host", + ) + })?; + let port = command.port.ok_or_else(|| { + error( + Some(&command.id), + "nvst-send-failed", + "nvst-send requires port", + ) + })?; + let payload = BASE64 + .decode(command.payload_base64.unwrap_or_default()) + .map_err(|decode_error| { + error( + Some(&command.id), + "nvst-send-failed", + format!("nvst-send payload is not valid base64: {decode_error}"), + ) + })?; + let send_result = if let Some(bundle) = self.reserved_nvst_bundle.as_ref() { + bundle.send_to(&payload, host.as_str(), port) + } else if let Some(socket) = self.nvst_hole_punch_socket.as_ref() { + socket.send_to(&payload, (host.as_str(), port)) + } else { + return Err(error( + Some(&command.id), + "nvst-send-failed", + "NVST UDP socket has not been reserved", + )); + }; + send_result.map_err(|send_error| { + error( + Some(&command.id), + "nvst-send-failed", + format!("failed to send NVST UDP datagram: {send_error}"), + ) + })?; + Ok(vec![response(command.id, "ok")]) + } + + fn nvst_unbind(&mut self, command: Command) -> Result, Value> { + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.state != State::Idle + || self.nvst_transport.is_some() + || self.nvst_mjolnir_transport.is_some() + { + return Err(error( + Some(&command.id), + "nvst-unbind-in-use", + "Cannot release an NVST UDP reservation after session start", + )); + } + drop(lifecycle); + self.reserved_nvst_bundle = None; + self.nvst_hole_punch_socket = None; + Ok(vec![response(command.id, "ok")]) + } + + fn audio_devices(&self, command: &Command) -> Result, Value> { + let runtime = self.media_runtime.as_ref().ok_or_else(|| { + error( + Some(&command.id), + "audio-devices-unavailable", + "Native media runtime is unavailable", + ) + })?; + let devices = runtime + .audio_devices() + .map_err(|message| error(Some(&command.id), "audio-devices-unavailable", message))?; + let response = json!({"type": "audioDevices", "id": command.id, "devices": devices}); + if serde_json::to_vec(&response) + .map_err(|error_value| { + error( + Some(&command.id), + "audio-devices-unavailable", + error_value.to_string(), + ) + })? + .len() + > 512 * 1024 + { + return Err(error( + Some(&command.id), + "audio-devices-unavailable", + "Audio device list exceeds the response size limit", + )); + } + Ok(vec![response]) + } + + fn start(&mut self, command: Command) -> Result, Value> { + let mut context = parse_context(command.context, &command.id)?; + validate_context(&context, &command.id)?; + let audio_device = + opennow_streamer_protocol::AudioOutputDevice::from_settings(&context.settings) + .map_err(|message| error(Some(&command.id), "invalid-context", message))?; + opennow_streamer_protocol::log::log_line( + "INFO", + "session", + "context validated; checking media backend", + ); + if let Some(runtime) = &self.media_runtime { + runtime + .validate_backend( + context + .settings + .get("nativeVideoBackend") + .and_then(Value::as_str) + .unwrap_or("auto"), + ) + .map_err(|message| { + error(Some(&command.id), "media-backend-unavailable", message) + })?; + } + { + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.state != State::Idle { + return Err(invalid_state(&command.id, "start", lifecycle.state, "Idle")); + } + } + if let Some(session) = self.webrtc_session.take() { + session.stop(); + } + self.events = self.events.for_session(&command.id); + if context + .settings + .get("transportMode") + .and_then(Value::as_str) + == Some("webrtc") + { + return self.start_webrtc(command.id, context, audio_device); + } + let wants_owned_nvst = context + .settings + .get("transportMode") + .and_then(Value::as_str) + .is_some_and(|mode| mode.eq_ignore_ascii_case("nvst")) + && context.nvst_video.is_none(); + let mut prepared_nvst = if wants_owned_nvst { + if self.reserved_nvst_bundle.is_none() { + self.reserved_nvst_bundle = + Some(ReservedNvstBundle::reserve().map_err(|error_value| { + error( + Some(&command.id), + "nvst-bind-failed", + format!( + "Native streamer could not reserve its NVST sockets: {error_value}" + ), + ) + })?); + } + let prepared_result = { + let mut stage = opennow_streamer_protocol::log::Stage::begin("nvst.negotiate"); + let bundle = self + .reserved_nvst_bundle + .as_mut() + .expect("NVST reservation created above"); + let result = prepare_owned_nvst(&context, bundle); + if result.is_ok() { + stage.complete(); + } + result + }; + let prepared = match prepared_result { + Ok(prepared) => prepared, + Err(negotiation_error) => { + self.reserved_nvst_bundle = None; + return Err(error( + Some(&command.id), + negotiation_error.code, + negotiation_error.message, + )); + } + }; + context.nvst_video = Some(prepared.handoff.clone()); + Some(prepared) + } else { + None + }; + let transport_context = serde_json::to_value(&context).map_err(|context_error| { + error( + Some(&command.id), + "invalid-context", + format!("Session context is not serializable: {context_error}"), + ) + })?; + let nvst_config = match parse_nvst_video_handoff(&transport_context) { + Ok(Some(config)) => Some(config), + Ok(None) => { + return Err(error( + Some(&command.id), + "nvst-handoff-required", + "Native streaming requires an NVST handoff", + )); + } + Err(reason) => { + return Err(error( + Some(&command.id), + "invalid-nvst-handoff", + format!("NVST transport is invalid: {reason}"), + )); + } + }; + let nvst_bundle_available = nvst_config + .as_ref() + .is_some_and(|config| config.remote_dtls_fingerprint().is_some()); + let nvst_audio_negotiated = nvst_config + .as_ref() + .is_some_and(|config| config.audio_track().is_some()); + let microphone_requested = + context.settings["microphoneMode"].as_str() == Some("voice-activity"); + let microphone_available = self.media_runtime.is_some() + && nvst_config + .as_ref() + .is_some_and(|config| config.microphone_available()); + let microphone_device = context.settings["microphoneDeviceId"] + .as_str() + .unwrap_or("") + .to_owned(); + + self.microphone = None; + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + if let Some(transport) = self.nvst_mjolnir_transport.take() { + transport.stop(); + } + if let Some(mut rtsp) = self.nvst_rtsp.take() { + rtsp.shutdown(); + } + self.stop_media_resources(); + if let Some(runtime) = self.media_runtime.clone() { + let (feedback_sender, feedback_receiver) = std::sync::mpsc::channel(); + let stream_config = prepared_nvst + .as_ref() + .map(|prepared| prepared.media_config) + .unwrap_or_else(|| media_stream_config(&context)); + opennow_streamer_protocol::log::log_line( + "INFO", + "media-config", + &format!( + "codec={:?} color={:?} width={} height={} fps={} bitrate_bps={} audio_negotiated={} dtls_bundle={}", + stream_config.codec, + stream_config.color_quality, + stream_config.width, + stream_config.height, + stream_config.fps, + stream_config.bitrate_bps, + nvst_audio_negotiated, + nvst_bundle_available + ), + ); + let session = runtime + .start_with_audio_device( + feedback_sender, + stream_config, + context + .settings + .get("nativeVideoBackend") + .and_then(Value::as_str) + .unwrap_or("auto"), + audio_device, + ) + .map_err(|message| error(Some(&command.id), "media-output-unavailable", message))?; + session.control().start_replay( + ReplayBufferConfig::from_settings(&context.settings), + Arc::clone(&self.replay_budget), + ); + let sink = session.sink(); + let (media_consumer, media_receiver) = + std::sync::mpsc::sync_channel(ENCODED_MEDIA_QUEUE_CAPACITY); + let output = self.events.clone(); + let media_worker = match thread::Builder::new() + .name("opennow-media-consumer".to_owned()) + .spawn(move || consume_encoded_media(&output, media_receiver, sink)) + { + Ok(worker) => worker, + Err(spawn_error) => { + session.stop(); + return Err(error( + Some(&command.id), + "media-worker-failed", + spawn_error.to_string(), + )); + } + }; + self.media_consumer = Some(media_consumer); + self.media_session = Some(session); + self.media_worker = Some(media_worker); + self.media_feedback = Some(feedback_receiver); + } + + if let Some(prepared) = prepared_nvst.as_mut() + && let Err(negotiation_error) = prepared.announce() + { + self.stop_media_resources(); + self.reserved_nvst_bundle = None; + return Err(error( + Some(&command.id), + negotiation_error.code, + negotiation_error.message, + )); + } + + let mut nvst_events = None; + let mut nvst_resources = None; + let mut nvst_upstream_ready = None; + let mut nvst_authenticated_ready = None; + if let Some(config) = nvst_config { + let Some(media_consumer) = self.media_consumer.clone() else { + self.stop_media_resources(); + return Err(error( + Some(&command.id), + "media-consumer-unavailable", + "NVST video requires an in-process encoded media consumer", + )); + }; + let (event_sender, event_receiver) = std::sync::mpsc::channel(); + let (reserved_socket, reserved_rtc, reserved_mjolnir) = + match self.reserved_nvst_bundle.take() { + Some(bundle) => { + self.nvst_hole_punch_socket = bundle.try_clone_socket().ok(); + let (socket, rtc, mjolnir_socket) = bundle.into_parts(); + (Some(socket), Some(rtc), Some(mjolnir_socket)) + } + None => (None, None, None), + }; + let mjolnir_udp_port = config.mjolnir_udp_port(); + let feedback = config.feedback(); + nvst_authenticated_ready = Some(feedback.clone()); + let (upstream_ready, upstream_waiter) = if prepared_nvst.is_some() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + (Some(sender), Some(receiver)) + } else { + (None, None) + }; + let transport = match spawn_nvst_udp_receiver_with_socket( + config.clone(), + media_consumer.clone(), + event_sender.clone(), + reserved_socket, + reserved_rtc, + Arc::clone(&self.hid_runtime), + upstream_waiter, + ) { + Ok(transport) => transport, + Err(transport_error) => { + drop(media_consumer); + self.stop_media_resources(); + return Err(error( + Some(&command.id), + "nvst-start-failed", + transport_error.to_string(), + )); + } + }; + let bundle_control = transport.control(); + self.nvst_transport = Some(transport); + let mut mjolnir_control = None; + if let Some(expected_port) = mjolnir_udp_port { + // Official two-socket model: video RTP/SRTP arrives on the + // dedicated NATT-only Mjolnir socket, not on the ICE/DTLS bundle. + let mjolnir_socket = match reserved_mjolnir { + Some(socket) => { + let actual_port = socket.local_addr().map(|addr| addr.port()).unwrap_or(0); + if actual_port != expected_port { + eprintln!( + "NVST Mjolnir socket port mismatch: reserved {actual_port}, handoff expects {expected_port}; NATT keepalive determines routing" + ); + } + socket + } + None => { + eprintln!( + "NVST Mjolnir reservation missing at start; binding a fresh video UDP socket" + ); + reserve_nvst_mjolnir_udp_socket().map_err(|bind_error| { + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + self.stop_media_resources(); + error( + Some(&command.id), + "nvst-start-failed", + format!("failed to reserve NVST Mjolnir UDP socket: {bind_error}"), + ) + })? + } + }; + let mjolnir = spawn_nvst_mjolnir_receiver( + mjolnir_socket, + config, + media_consumer, + event_sender, + ) + .map_err(|mjolnir_error| { + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + self.stop_media_resources(); + error( + Some(&command.id), + "nvst-start-failed", + mjolnir_error.to_string(), + ) + })?; + mjolnir_control = Some(mjolnir.control()); + self.nvst_mjolnir_transport = Some(mjolnir); + } + nvst_resources = Some(ActiveNvstResources { + bundle: bundle_control, + mjolnir: mjolnir_control, + feedback, + media: self.media_session.as_ref().map(MediaSession::control), + }); + nvst_events = Some(event_receiver); + nvst_upstream_ready = upstream_ready; + } else { + self.reserved_nvst_bundle = None; + self.nvst_hole_punch_socket = None; + } + + let replay_enabled = self.media_session.is_some() + && ReplayBufferConfig::from_settings(&context.settings).enabled; + let generation = { + let mut lifecycle = lock_lifecycle(&self.lifecycle); + lifecycle.generation = lifecycle.generation.wrapping_add(1); + lifecycle.context = Some(context); + lifecycle.state = State::Connected; + lifecycle.generation + }; + if self.hid_runtime.bind_session(generation).is_none() { + self.stop("NVST association exited before the session start was accepted"); + return Err(error( + Some(&command.id), + "nvst-start-failed", + "NVST association exited before the session start was accepted", + )); + } + if let Some(nvst_events) = nvst_events { + let output = self.events.clone(); + let lifecycle = self.lifecycle.clone(); + let media_feedback = self.media_feedback.take(); + let captured_input = self + .media_session + .as_ref() + .map(MediaSession::captured_input); + let shortcut_runtime = self.media_runtime.clone(); + let start_id = command.id.clone(); + let nvst_resources = nvst_resources.expect("NVST events require active resources"); + self.feedback_worker = thread::Builder::new() + .name("opennow-nvst-events".to_owned()) + .spawn(move || { + forward_nvst_session_events( + &output, + &lifecycle, + generation, + NvstSessionEventResources { + start_id, + nvst_events, + media_feedback, + captured_input, + shortcut_runtime, + transport: nvst_resources, + }, + ) + }) + .ok(); + if self.feedback_worker.is_none() { + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + if let Some(transport) = self.nvst_mjolnir_transport.take() { + transport.stop(); + } + self.stop_media_resources(); + let mut lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation == generation { + lifecycle.context = None; + lifecycle.state = State::Idle; + } + drop(lifecycle); + self.hid_runtime.unbind_session(generation); + return Err(error( + Some(&command.id), + "media-worker-failed", + "Failed to start NVST lifecycle worker", + )); + } + } + if let Some(prepared) = prepared_nvst { + let readiness = nvst_rtsp::wait_authenticated_transport( + || { + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation != generation || lifecycle.state != State::Connected { + return Err(nvst_rtsp::NvstRtspError { + code: "nvst-start-aborted", + message: + "NVST session stopped while waiting for authenticated transport" + .to_owned(), + }); + } + Ok(nvst_authenticated_ready + .as_ref() + .is_some_and(|feedback| feedback.authenticated_transport_ready())) + }, + Duration::from_secs(20), + ); + if let Err(readiness_error) = readiness { + self.stop("NVST authenticated transport readiness failed"); + return Err(error( + Some(&command.id), + readiness_error.code, + readiness_error.message, + )); + } + match prepared.finish() { + Ok(active) => { + self.nvst_rtsp = Some(active); + if nvst_upstream_ready + .take() + .is_some_and(|ready| ready.try_send(()).is_err()) + { + self.stop("NVST bundle exited before PLAY completed"); + return Err(error( + Some(&command.id), + "nvst-start-failed", + "NVST bundle exited before PLAY completed", + )); + } + } + Err(negotiation_error) => { + self.stop("Native-owned NVST negotiation failed"); + return Err(error( + Some(&command.id), + negotiation_error.code, + negotiation_error.message, + )); + } + } + } + if microphone_available { + let mut microphone = MicrophoneController::new( + self.media_runtime + .as_ref() + .expect("microphone requires media runtime") + .clone(), + self.nvst_transport + .as_ref() + .expect("microphone requires bundle") + .control(), + microphone_device, + self.events.clone(), + self.lifecycle.clone(), + generation, + ); + let _ = + microphone.set_enabled(command.microphone_enabled.unwrap_or(microphone_requested)); + self.microphone = Some(microphone); + } else { + let _ = self.events.send(event("microphone-state", json!({ + "state":if microphone_requested { "unavailable" } else { "disabled" }, + "enabled":false, + "message":if microphone_requested { Some("The server did not offer microphone audio on the native bundle") } else { None } + }))); + } + let _ = self.events.send(event( + "status", + json!({ + "status": "ready", + "message": "NVST authenticated media path initialized" + }), + )); + let mut start_response = response(command.id, "ok"); + start_response["replayEnabled"] = json!(replay_enabled); + start_response["transport"] = Value::String("nvst".to_owned()); + start_response["capabilities"] = json!({ + "supportsInput": nvst_bundle_available, + "supportsAudioDecode": nvst_audio_negotiated && supports_audio_decode(), + "supportsAudioOutput": nvst_audio_negotiated && supports_audio_output(), + "supportsMicrophone": microphone_available, + }); + Ok(vec![start_response]) + } + + fn set_microphone(&mut self, command: Command) -> Result, Value> { + let state = lock_lifecycle(&self.lifecycle).state; + if state != State::Connected { + return Err(invalid_state( + &command.id, + &command.kind, + state, + "Connected", + )); + } + let microphone = self.microphone.as_mut().ok_or_else(|| { + error( + Some(&command.id), + "microphone-unavailable", + "Microphone audio was not enabled and negotiated for this session", + ) + })?; + let enabled = if command.kind == "microphone-toggle" { + !microphone.enabled() + } else { + command.enabled.ok_or_else(|| { + error( + Some(&command.id), + "invalid-command", + "microphone-set requires a boolean enabled value", + ) + })? + }; + microphone + .set_enabled(enabled) + .map_err(|message| error(Some(&command.id), "microphone-failed", message))?; + Ok(vec![response(command.id, "ok")]) + } + + fn set_audio_muted(&self, command: Command) -> Result, Value> { + let muted = command.muted.ok_or_else(|| { + error( + Some(&command.id), + "missing-muted", + "Audio mute command requires muted state", + ) + })?; + let runtime = self.media_runtime.as_ref().ok_or_else(|| { + error( + Some(&command.id), + "unsupported-command", + "Native streamer has no audio playback runtime", + ) + })?; + runtime.set_audio_muted(muted); + Ok(vec![response(command.id, "ok")]) + } + + fn set_paused(&self, command: Command) -> Result, Value> { + if let Some(session) = self.webrtc_session.as_ref() { + let paused = command.paused.ok_or_else(|| { + error( + Some(&command.id), + "missing-paused", + "Pause command does not include paused state", + ) + })?; + session + .set_paused(paused) + .map_err(|message| error(Some(&command.id), "webrtc-input-unavailable", message))?; + return Ok(vec![response(command.id, "ok")]); + } + let Some(runtime) = self.media_runtime.as_ref() else { + return Err(error( + Some(&command.id), + "unsupported-command", + "Native streamer has no media runtime for input-paused", + )); + }; + let paused = command.paused.ok_or_else(|| { + error( + Some(&command.id), + "missing-paused", + "Pause command does not include paused state", + ) + })?; + runtime + .set_paused(paused) + .map_err(|message| error(Some(&command.id), "media-host-unavailable", message))?; + if let Some(media) = self.media_session.as_ref() { + media.set_paused(paused); + } + if let Some(transport) = self.nvst_transport.as_ref() { + let result = if paused { + transport.pause() + } else { + transport.resume() + }; + result.map_err(|transport_error| { + error( + Some(&command.id), + "nvst-control-failed", + transport_error.to_string(), + ) + })?; + } + if let Some(transport) = self.nvst_mjolnir_transport.as_ref() { + let result = if paused { + transport.pause() + } else { + transport.resume() + }; + result.map_err(|transport_error| { + error( + Some(&command.id), + "nvst-control-failed", + transport_error.to_string(), + ) + })?; + } + Ok(vec![response(command.id, "ok")]) + } + + fn update_surface(&self, command: Command) -> Result, Value> { + let Some(runtime) = self.media_runtime.as_ref() else { + return Err(error( + Some(&command.id), + "unsupported-command", + "Native streamer has no media runtime for surface", + )); + }; + let surface = command.surface.ok_or_else(|| { + error( + Some(&command.id), + "missing-surface", + "Surface command does not include a render surface", + ) + })?; + runtime + .update_surface(surface) + .map_err(|message| error(Some(&command.id), "media-host-unavailable", message))?; + Ok(vec![response(command.id, "ok")]) + } + + fn stop(&mut self, reason: &str) { + self.clip_cancelled.store(true, Ordering::Release); + if let Some(session) = self.media_session.as_ref() { + session.control().unsubscribe_recording(); + } + if let Some(generation) = self.hid_runtime.session_generation() { + self.hid_runtime.unbind_session(generation); + } + let was_active = { + let mut lifecycle = lock_lifecycle(&self.lifecycle); + let was_active = lifecycle.state != State::Idle; + lifecycle.generation = lifecycle.generation.wrapping_add(1); + lifecycle.context = None; + lifecycle.state = State::Idle; + was_active + }; + if let Some(session) = self.webrtc_session.take() { + session.stop(); + } + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + if let Some(transport) = self.nvst_mjolnir_transport.take() { + transport.stop(); + } + if let Some(mut rtsp) = self.nvst_rtsp.take() { + rtsp.shutdown(); + } + self.reserved_nvst_bundle = None; + self.nvst_hole_punch_socket = None; + self.stop_media_resources(); + if was_active { + let _ = self.events.send(event( + "microphone-state", + json!({"state":"disabled", "enabled":false}), + )); + let _ = self.events.send(event( + "status", + json!({ "status": "stopped", "message": reason }), + )); + } + } + + fn stop_media_resources(&mut self) { + self.stop_replay(); + self.microphone = None; + let _ = self.stop_recording_inner(); + if self.media_runtime.is_some() { + self.media_consumer = None; + if let Some(session) = self.media_session.take() { + session.stop(); + } + if let Some(worker) = self.media_worker.take() { + let _ = worker.join(); + } + self.media_feedback = None; + } + if let Some(worker) = self.feedback_worker.take() + && let Ok(mut pending) = worker.join() + { + if let Some(feedback) = pending.receiver { + for feedback in feedback.try_iter() { + if let MediaFeedback::QueueDropped { media, count } = feedback { + pending.reports.record(media, count); + } + } + } + pending.reports.flush(&self.events, Instant::now(), true); + } + } + + fn stop_replay(&mut self) { + { + let _lifecycle = lock_lifecycle(&self.lifecycle); + self.clip_cancelled.store(true, Ordering::Release); + } + if let Some(session) = self.media_session.as_ref() { + session.control().stop_replay(); + } + } + + fn save_clip(&mut self, command: Command) -> Result, Value> { + if self + .clip_worker + .as_ref() + .is_some_and(|worker| !worker.is_finished()) + { + return Err(error( + Some(&command.id), + "clip-already-saving", + "A clip export is already in progress", + )); + } + if let Some(worker) = self.clip_worker.take() { + let _ = worker.join(); + } + let path = command + .output_path + .as_deref() + .map(std::path::PathBuf::from) + .filter(|path| { + path.is_absolute() + && path.extension().and_then(|value| value.to_str()) == Some("mkv") + }) + .ok_or_else(|| { + error( + Some(&command.id), + "invalid-clip-output", + "Clips require an absolute .mkv output path", + ) + })?; + let control = self + .media_session + .as_ref() + .map(MediaSession::control) + .ok_or_else(|| { + error( + Some(&command.id), + "replay-not-enabled", + "Replay requires an enabled active session", + ) + })?; + let (stream, snapshot) = control.replay_snapshot().map_err(|code| { + let message = match code { + "replay-not-enabled" => { + "Enable replay buffering before starting a session to save clips" + } + "replay-not-ready" => { + "Waiting for a source video keyframe; previous replay history may have expired" + } + "clip-already-saving" => "Another clip export is already in progress", + _ => "Replay capture is unavailable", + }; + error(Some(&command.id), code, message) + })?; + let cancelled = Arc::new(AtomicBool::new(false)); + self.clip_cancelled = Arc::clone(&cancelled); + let events = self.events.clone(); + let lifecycle = Arc::clone(&self.lifecycle); + let generation = lock_lifecycle(&lifecycle).generation; + let request_id = command.id.clone(); + let worker_path = path.clone(); + let worker = thread::Builder::new().name("opennow-replay-export".to_owned()).spawn(move || { + let result = record_replay_matroska(&worker_path, stream, snapshot, &cancelled); + let saved = result.is_ok(); + let payload = match result { + Ok(summary) => json!({"state":"saved", "path":summary.path, "message":"Clip saved", "requestId":request_id, "videoPackets":summary.video_packets, "audioPackets":summary.audio_packets}), + Err(message) => json!({"state":"failed", "path":worker_path, "message":message, "requestId":request_id}), + }; + let completion = event("clip-state", payload); + loop { + let current = lock_lifecycle(&lifecycle); + if current.generation != generation || current.state != State::Connected || cancelled.load(Ordering::Acquire) { + drop(current); + if saved { + let _ = std::fs::remove_file(&worker_path); + } + return; + } + if events.send(completion.clone()).is_ok() { + return; + } + drop(current); + thread::sleep(Duration::from_millis(10)); + } + }).map_err(|err| error(Some(&command.id), "clip-worker-failed", err.to_string()))?; + self.clip_worker = Some(worker); + Ok(vec![ + json!({"id":command.id, "type":"clip-saving", "path":path}), + ]) + } + + fn start_recording(&mut self, command: Command) -> Result, Value> { + if self.recording_worker.as_ref().is_some_and(|worker| { + worker.completed.load(Ordering::Acquire) || worker.thread.is_finished() + }) { + let _ = self.stop_recording_inner(); + } + if self.recording_worker.is_some() { + return Err(error( + Some(&command.id), + "recording-already-active", + "A native stream recording is already active", + )); + } + let output_path = command + .output_path + .filter(|path| !path.is_empty()) + .ok_or_else(|| { + error( + Some(&command.id), + "invalid-recording-output", + "Native recording requires an absolute .mkv output path", + ) + })?; + let path = std::path::PathBuf::from(&output_path); + if !path.is_absolute() || path.extension().and_then(|value| value.to_str()) != Some("mkv") { + return Err(error( + Some(&command.id), + "invalid-recording-output", + "Native recording requires an absolute .mkv output path", + )); + } + let control = self + .media_session + .as_ref() + .map(MediaSession::control) + .ok_or_else(|| { + error( + Some(&command.id), + "media-output-unavailable", + "Native recording requires an active media session", + ) + })?; + let (stream, receiver) = control + .subscribe_recording() + .map_err(|message| error(Some(&command.id), "recording-start-failed", message))?; + let events = self.events.clone(); + let worker_path = path.clone(); + let request_id = command.id.clone(); + let completed = Arc::new(AtomicBool::new(false)); + let worker_completed = Arc::clone(&completed); + let worker = thread::Builder::new() + .name("opennow-matroska-recording".to_owned()) + .spawn(move || { + let result = record_matroska(&worker_path, stream, receiver); + worker_completed.store(true, Ordering::Release); + let payload = match &result { + Ok(summary) => json!({ + "state":"saved", + "path":summary.media.path, + "videoPackets":summary.media.video_packets, + "audioPackets":summary.media.audio_packets, + "completion":summary.completion, + "requestId":request_id, + }), + Err(message) => { + json!({"state":"failed","message":message,"requestId":request_id}) + } + }; + let _ = events.send(event("recording-state", payload)); + result + }) + .map_err(|spawn_error| { + control.unsubscribe_recording(); + error( + Some(&command.id), + "recording-worker-failed", + spawn_error.to_string(), + ) + })?; + self.recording_worker = Some(RecordingWorker { + request_id: command.id.clone(), + thread: worker, + completed, + }); + Ok(vec![json!({ + "id":command.id, + "type":"recording-started", + "path":path, + })]) + } + + fn stop_recording(&mut self, command: Command) -> Result, Value> { + match self.stop_recording_inner() { + Ok(Some((request_id, summary))) => Ok(vec![json!({ + "id":command.id, + "type":"recording-stopped", + "path":summary.media.path, + "videoPackets":summary.media.video_packets, + "audioPackets":summary.media.audio_packets, + "completion":summary.completion, + "requestId":request_id, + })]), + Ok(None) => Ok(vec![response(command.id, "recording-not-active")]), + Err(message) => Err(error(Some(&command.id), "recording-failed", message)), + } + } + + fn anti_afk_pulse(&self, command: Command) -> Result, Value> { + let state = lock_lifecycle(&self.lifecycle).state; + if state != State::Connected { + return Err(invalid_state( + &command.id, + "anti-afk-pulse", + state, + "Connected with an initialized input channel", + )); + } + if let Some(session) = self.webrtc_session.as_ref() { + session + .anti_afk() + .map_err(|message| error(Some(&command.id), "webrtc-input-unavailable", message))?; + return Ok(vec![response(command.id, "ok")]); + } + let send = |input| { + let bytes = captured_input_packet(input, 0); + if let Some(transport) = self.nvst_transport.as_ref() { + transport.send_input(bytes, false) + } else { + Err(opennow_streamer_transport::TransportError::Closed) + } + }; + send(CapturedInput::Key { + virtual_key: 0x7c, + modifiers: 0, + pressed: true, + }) + .and_then(|_| { + send(CapturedInput::Key { + virtual_key: 0x7c, + modifiers: 0, + pressed: false, + }) + }) + .map_err(|transport_error| { + error( + Some(&command.id), + transport_error.code(), + transport_error.to_string(), + ) + })?; + Ok(vec![response(command.id, "ok")]) + } + + fn stop_recording_inner(&mut self) -> Result, String> { + let Some(worker) = self.recording_worker.take() else { + return Ok(None); + }; + if let Some(session) = self.media_session.as_ref() { + session.control().unsubscribe_recording(); + } + worker + .thread + .join() + .map_err(|_| "native recording worker panicked".to_owned())? + .map(|summary| Some((worker.request_id, summary))) + } +} + +impl Drop for Engine { + fn drop(&mut self) { + self.stop("process closed"); + if let Some(worker) = self.clip_worker.take() { + let _ = worker.join(); + } + } +} + +fn parse_context(context: Option, id: &str) -> Result { + let context = context.ok_or_else(|| { + error( + Some(id), + "missing-context", + "Command requires session context", + ) + })?; + serde_json::from_value(context).map_err(|context_error| { + error( + Some(id), + "invalid-context", + format!("Invalid session context: {context_error}"), + ) + }) +} + +fn validate_context(context: &SessionContext, id: &str) -> Result<(), Value> { + opennow_streamer_protocol::AudioOutputDevice::from_settings(&context.settings) + .map_err(|message| error(Some(id), "invalid-context", message))?; + if context.session.session_id.trim().is_empty() { + return Err(error( + Some(id), + "invalid-context", + "Session context requires a non-empty sessionId", + )); + } + if context.session.server_ip.trim().is_empty() { + return Err(error( + Some(id), + "invalid-context", + "Session context requires a non-empty serverIp endpoint", + )); + } + if !context.settings.is_object() || !context.shortcuts.is_object() { + return Err(error( + Some(id), + "invalid-context", + "Session context settings and shortcuts must be objects", + )); + } + if let Some(profile) = context.session.extra.get("negotiatedStreamProfile") { + let color_reported = ["bitDepthSource", "chromaFormatSource"].iter().any(|key| { + matches!( + profile[*key].as_str(), + Some("request" | "finalized" | "server") + ) + }); + if (color_reported && profile["colorQuality"].as_str().is_none()) + || (profile["enableHdrSource"] == "server" && profile["enableHdr"].as_bool().is_none()) + { + return Err(error( + Some(id), + "invalid-context", + "The accepted color or HDR profile is incomplete or unsupported", + )); + } + let codec = profile["codec"] + .as_str() + .unwrap_or_default() + .trim() + .to_ascii_uppercase(); + let color = profile["colorQuality"] + .as_str() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + if (codec == "H264" && matches!(color.as_str(), "8bit_444" | "10bit_420" | "10bit_444")) + || (codec == "AV1" && matches!(color.as_str(), "8bit_444" | "10bit_444")) + { + return Err(error( + Some(id), + "invalid-context", + "The accepted codec and color profile cannot be preserved by the streamer", + )); + } + } + // HDR acceptance comes from the negotiated profile, but the codec is + // client-selected (the official client never sends it to CloudMatch), so + // validate the effective stream config rather than the raw profile: a + // session without a server-reported codec must not fail when the client + // selected HEVC/AV1 with 10-bit color. + let stream = media_stream_config(context); + if stream.hdr + && !matches!( + (stream.codec, stream.color_quality), + ( + MediaVideoCodec::H265, + MediaColorQuality::TenBit420 | MediaColorQuality::TenBit444 + ) | (MediaVideoCodec::Av1, MediaColorQuality::TenBit420) + ) + { + return Err(error( + Some(id), + "invalid-context", + "HDR requires an accepted HEVC/AV1 10-bit profile with supported chroma", + )); + } + if let Some(endpoint) = &context.session.media_connection_info { + if endpoint.ip.trim().is_empty() || endpoint.port == 0 || endpoint.port > u16::MAX.into() { + return Err(error( + Some(id), + "invalid-context", + "mediaConnectionInfo requires a hostname and a port in 1..=65535", + )); + } + } + serde_json::to_value(context).map_err(|context_error| { + error( + Some(id), + "invalid-context", + format!("Session context is not serializable: {context_error}"), + ) + })?; + Ok(()) +} + +fn invalid_state(id: &str, command: &str, state: State, required: &str) -> Value { + error( + Some(id), + "invalid-state", + format!("Cannot apply {command} while lifecycle is {state:?}; required state: {required}"), + ) +} + +fn lock_lifecycle(lifecycle: &Mutex) -> MutexGuard<'_, Lifecycle> { + lifecycle + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn forward_shortcut_action( + output: &EventSender, + runtime: Option<&MediaRuntime>, + action: StreamShortcutAction, +) { + if action == StreamShortcutAction::TogglePointerLock + && runtime.is_some_and(MediaRuntime::is_embedded) + { + let _ = output.send(event( + "shortcut-action", + json!({"action":action.protocol_name(), "source":"keyboard"}), + )); + return; + } + let control = match action { + StreamShortcutAction::ToggleStats => None, + StreamShortcutAction::ToggleFullscreen => None, + StreamShortcutAction::TogglePointerLock => Some(MediaRuntimeControl::PointerLock), + _ => None, + }; + if let Some(control) = control { + let result = runtime + .ok_or_else(|| "native media runtime is unavailable".to_owned()) + .and_then(|runtime| runtime.control(control)); + if let Err(message) = result { + let _ = output.send(event( + "log", + json!({"level":"warn", "message":format!("Shortcut {} failed: {message}", action.protocol_name())}), + )); + } + return; + } + let _ = output.send(event( + "shortcut-action", + json!({"action":action.protocol_name(), "source":"keyboard"}), + )); +} + +struct NvstSessionEventResources { + start_id: String, + nvst_events: Receiver, + media_feedback: Option>, + captured_input: Option>, + shortcut_runtime: Option, + transport: R, +} + +struct PendingMediaFeedback { + receiver: Option>, + reports: QueueDropReports, +} + +fn forward_nvst_session_events( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + event_resources: NvstSessionEventResources, +) -> PendingMediaFeedback { + let NvstSessionEventResources { + start_id, + nvst_events, + media_feedback, + captured_input, + shortcut_runtime, + transport: resources, + } = event_resources; + let mut feedback_state = NvstMediaFeedbackState::new(false); + feedback_state.previous_socket_receive_bytes = resources.socket_receive_bytes().unwrap_or(0); + feedback_state.start_id = start_id.clone(); + if let Some(queue) = captured_input.as_ref() { + queue.set_text_ready(generation, false); + } + let mut pending_rumble = [None; 4]; + let mut pending_input = None; + let mut pending_control_output = NvstControlOutput { + start_id: start_id.clone(), + pending: None, + pending_timer: None, + pending_text_error: None, + }; + 'session: loop { + flush_control_output(output, lifecycle, generation, &mut pending_control_output); + if let Some(feedback) = media_feedback.as_ref() { + while let Ok(feedback) = feedback.try_recv() { + forward_nvst_media_feedback( + output, + lifecycle, + generation, + &resources, + feedback, + &mut feedback_state, + ); + } + } + feedback_state + .drop_reports + .flush(output, Instant::now(), false); + if lock_lifecycle(lifecycle).generation != generation { + break; + } + let (rumble, coalesced) = resources.take_rumble(); + feedback_state + .drop_reports + .record("controller-rumble-coalesced", coalesced); + for command in rumble.into_iter().flatten() { + if pending_rumble[usize::from(command.controller_id)] + .replace(command) + .is_some() + { + feedback_state + .drop_reports + .record("controller-rumble-coalesced", 1); + } + } + for pending in &mut pending_rumble { + if let Some(command) = *pending + && forward_controller_rumble(output, lifecycle, generation, &start_id, command) + { + *pending = None; + } + } + if let Some(captured_input) = captured_input.as_ref() { + if !feedback_state.input_available { + if pending_input.is_none() { + captured_input.clear(); + } + } else if captured_input.take_overflowed() { + let _ = emit_nvst_terminal( + output, + lifecycle, + generation, + &resources, + "native-input-capture-overflow", + "Native input capture queue overflowed; stopping to prevent stuck input" + .to_owned(), + ); + break; + } else { + // Preserve high-polling-rate RawInput/SDL samples rather than + // turning several reports into one uneven movement burst. + for _ in 0..32 { + let Some(input) = pending_input + .take() + .or_else(|| captured_input.take_sample()) + else { + break; + }; + if matches!(input.input, CapturedInput::Guide) { + let _ = output.send(event("overlay-request", json!({"source":"gamepad"}))); + continue; + } + if matches!(input.input, CapturedInput::Screenshot) { + let _ = + output.send(event("screenshot-request", json!({"source":"keyboard"}))); + continue; + } + if matches!(input.input, CapturedInput::RecordingToggle) { + let _ = output.send(event( + "recording-toggle-request", + json!({"source":"keyboard"}), + )); + continue; + } + if let CapturedInput::Shortcut(action) = &input.input { + forward_shortcut_action(output, shortcut_runtime.as_ref(), *action); + continue; + } + if let Err(error) = + forward_nvst_captured_sample(&resources, &input, &feedback_state) + { + if matches!(error, TransportError::InputNotReady) { + pending_input = Some(input); + break; + } + let _ = emit_nvst_terminal( + output, + lifecycle, + generation, + &resources, + "native-input-capture-failed", + format!("Native window input capture failed: {error}"), + ); + break 'session; + } + } + } + } + match nvst_events.recv_timeout(NATIVE_INPUT_POLL_INTERVAL) { + Ok(nvst_event) => { + match &nvst_event { + NvstReceiveEvent::InputReady(_) => { + feedback_state.input_available = true; + if let Some(queue) = captured_input.as_ref() { + queue.set_text_ready(generation, true); + } + } + NvstReceiveEvent::InputUnavailable(_) => { + feedback_state.input_available = false; + if let Some(queue) = captured_input.as_ref() { + queue.set_text_ready(generation, false); + } + } + _ => {} + } + match nvst_event { + NvstReceiveEvent::FrameProgressStall { .. } + | NvstReceiveEvent::RecoveryNeeded(NvstRecovery::FrameProgress { .. }) => { + feedback_state.transport_frame_progress_stalled = true; + } + NvstReceiveEvent::FrameProgressResumed => { + feedback_state.transport_frame_progress_stalled = false; + } + _ => {} + } + let terminal = forward_nvst_event( + output, + lifecycle, + generation, + &resources, + &mut feedback_state.recovery_attempts, + &mut pending_control_output, + feedback_state.last_video_decoder_error.as_deref(), + nvst_event, + ); + if terminal { + break; + } + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + let _ = emit_nvst_terminal( + output, + lifecycle, + generation, + &resources, + "nvst-event-channel-closed", + "NVST receiver event channel closed unexpectedly".to_owned(), + ); + break; + } + } + if let Some(timings) = feedback_state.decode_timings { + let last_assembled_at = if timings.in_flight == 0 { + resources + .frame_stage_timings() + .and_then(|stage| stage.last_assembled_at) + } else { + None + }; + let policy = resources.decode_progress_policy(); + let decode_event = feedback_state.decode_progress.poll( + &timings, + feedback_state.transport_frame_progress_stalled, + last_assembled_at, + Instant::now(), + policy, + ); + if let Some(decode_event) = decode_event { + match decode_event { + DecodeProgressEvent::KeyframeRequested { + idle_for, + in_flight, + } => { + resources.request_keyframe(); + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": format!( + "Decoder produced no frame for {idle_for:?} with {in_flight} frame(s) outstanding; requested a keyframe" + ) + }), + )); + } + DecodeProgressEvent::RecoveryNeeded { + idle_for, + in_flight, + } => { + let reason = format!( + "decoder stall: no output for {idle_for:?} with {in_flight} frame(s) outstanding" + ); + if attempt_nvst_recovery( + output, + lifecycle, + generation, + &resources, + &mut feedback_state.recovery_attempts, + reason, + ) { + break; + } + feedback_state.decode_recovery_deadline = + Some(Instant::now() + policy.recovery_grace); + } + } + } + if feedback_state.decode_progress.stage() != DecodeProgressStage::RecoveryRequired { + feedback_state.decode_recovery_deadline = None; + } + if feedback_state + .decode_recovery_deadline + .is_some_and(|deadline| Instant::now() >= deadline) + { + emit_nvst_terminal( + output, + lifecycle, + generation, + &resources, + "nvst-recovery-exhausted", + "Decoder did not resume output after NVST recovery".into(), + ); + break; + } + } + if feedback_state.telemetry_started + || resources + .socket_receive_bytes() + .is_some_and(|bytes| bytes > feedback_state.previous_socket_receive_bytes) + { + feedback_state.telemetry_started = true; + flush_nvst_telemetry(output, &resources, &mut feedback_state); + } + } + if let Some(queue) = captured_input.as_ref() { + queue.set_text_ready(generation, false); + } + feedback_state + .drop_reports + .flush(output, Instant::now(), true); + PendingMediaFeedback { + receiver: media_feedback, + reports: feedback_state.drop_reports, + } +} + +fn forward_controller_rumble( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + start_id: &str, + command: NvstControllerRumble, +) -> bool { + let current = lock_lifecycle(lifecycle); + if current.generation != generation || current.state != State::Connected { + return true; + } + let mut payload = json!({ + "startId": start_id, + "controllerId": command.controller_id, + "lowFrequency": command.low_frequency, + "highFrequency": command.high_frequency, + "durationMs": command.duration_ms, + }); + if let Some(incarnation) = command.source_incarnation + && let Some(object) = payload.as_object_mut() + { + object.insert("sourceIncarnation".to_owned(), json!(incarnation)); + } + output.send(event("controller-rumble", payload)).is_ok() +} + +#[derive(Default)] +struct NvstControlOutput { + start_id: String, + pending: Option, + pending_timer: Option, + pending_text_error: Option<&'static str>, +} + +fn flush_control_output( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + state: &mut NvstControlOutput, +) { + if let Some(reason) = state.pending_text_error { + let current = lock_lifecycle(lifecycle); + if current.generation != generation + || current.context.is_none() + || output + .send(event( + "text-input-error", + json!({"startId": state.start_id, "reason": reason}), + )) + .is_ok() + { + state.pending_text_error = None; + } + } + if let Some(timer) = state.pending_timer { + let current = lock_lifecycle(lifecycle); + if current.generation != generation + || current.context.is_none() + || output + .send(event( + "session-timer", + json!({ + "startId": state.start_id, + "timerType": timer.timer_type, + "secondsLeft": timer.seconds_left, + "active": timer.active, + }), + )) + .is_ok() + { + state.pending_timer = None; + } + } + let Some(composited) = state.pending else { + return; + }; + let current = lock_lifecycle(lifecycle); + if current.generation != generation + || current.context.is_none() + || output + .send(event( + "cursor-capture", + json!({ "startId": state.start_id, "composited": composited }), + )) + .is_ok() + { + state.pending = None; + } +} + +#[allow(clippy::too_many_arguments)] +fn forward_nvst_event( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + resources: &R, + recovery_attempts: &mut usize, + pending_control_output: &mut NvstControlOutput, + last_video_decoder_error: Option<&str>, + nvst_event: NvstReceiveEvent, +) -> bool { + if lock_lifecycle(lifecycle).generation != generation { + return true; + } + + match nvst_event { + NvstReceiveEvent::MicrophoneError(message) => { + opennow_streamer_protocol::log::log_line("WARN", "microphone", &message); + false + } + NvstReceiveEvent::FrameProgressStall { + idle_for, + recovery_required: false, + .. + } => { + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": format!( + "Produced-frame stall for {idle_for:?}; requested a fresh keyframe" + ) + }), + )); + false + } + NvstReceiveEvent::FrameProgressStall { .. } | NvstReceiveEvent::FrameProgressResumed => { + false + } + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { + first_missing_index, + last_missing_index, + }) => { + // The receiver owns gap recovery (reference invalidation or keyframe, + // plus discontinuity marking); a keyframe request here would turn every + // reference-invalidation recovery back into an IDR. + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": format!( + "Recovering NVST packet gap: {first_missing_index}..={last_missing_index}" + ) + }), + )); + false + } + NvstReceiveEvent::RecoveryNeeded(recovery) => attempt_nvst_recovery( + output, + lifecycle, + generation, + resources, + recovery_attempts, + format!("{recovery:?}"), + ), + NvstReceiveEvent::Lifecycle(NvstReceiverState::RecoveryRequired) => attempt_nvst_recovery( + output, + lifecycle, + generation, + resources, + recovery_attempts, + "authenticated media timeout".to_owned(), + ), + NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped(cause)) => emit_nvst_terminal( + output, + lifecycle, + generation, + resources, + "nvst-transport-stopped", + format!("NVST receiver stopped unexpectedly ({cause})"), + ), + NvstReceiveEvent::Dropped(NvstDropReason::MediaConsumerBackpressured) => { + // A bounded decode queue protects latency. A momentary full queue + // means this access unit is stale, not that the network session is + // dead. Keep receiving and ask for a clean decoder reference. + resources.request_keyframe(); + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": "Dropped a backpressured NVST video frame and requested a fresh keyframe" + }), + )); + false + } + NvstReceiveEvent::Dropped(NvstDropReason::MediaConsumerClosed) => emit_nvst_terminal( + output, + lifecycle, + generation, + resources, + "media-consumer-closed", + match last_video_decoder_error { + Some(error) => format!( + "NVST receiver stopped because the decoded media path closed after {error}" + ), + None => "NVST receiver stopped because the decoded media path closed".to_owned(), + }, + ), + NvstReceiveEvent::Lifecycle(NvstReceiverState::Running) => { + lock_lifecycle(lifecycle).state = State::Connected; + let _ = output.send(event( + "status", + json!({ "status": "streaming", "message": "NVST SRTP video receiver is running" }), + )); + false + } + NvstReceiveEvent::Lifecycle(NvstReceiverState::Paused) => { + let _ = output.send(event( + "status", + json!({ "status": "paused", "message": "NVST SRTP video receiver is paused" }), + )); + false + } + NvstReceiveEvent::TransportReady(phase) => { + let _ = output.send(event("nvst-transport-ready", json!({ "phase": phase }))); + false + } + NvstReceiveEvent::InputReady(protocol_version) => { + let _ = output.send(event( + "input-ready", + json!({ "protocolVersion": protocol_version }), + )); + false + } + NvstReceiveEvent::InputUnavailable(reason) => { + opennow_streamer_protocol::log::log_line( + "WARN", + "nvst-input", + &format!("input-unavailable reason={reason}"), + ); + let _ = output.send(event("input-unavailable", json!({ "reason": reason }))); + false + } + NvstReceiveEvent::UserInput => { + pending_control_output.pending_timer = + Some(opennow_streamer_transport::NvstSessionTimer { + timer_type: 4, + seconds_left: 0, + active: false, + }); + flush_control_output(output, lifecycle, generation, pending_control_output); + false + } + NvstReceiveEvent::TextRejected(reason) => { + pending_control_output.pending_text_error = Some(reason); + flush_control_output(output, lifecycle, generation, pending_control_output); + false + } + NvstReceiveEvent::SessionTimer(timer) => { + pending_control_output.pending_timer = Some(timer); + flush_control_output(output, lifecycle, generation, pending_control_output); + false + } + NvstReceiveEvent::ServerTermination(termination) => { + let resumable = termination.resumable(); + let message = format!( + "The server ended the streaming session (0x{:08x})", + termination.wire_code, + ); + let termination = json!({ + "source": "nvst-server", + "startId": pending_control_output.start_id, + "code": "server-terminated", + "wireCode": termination.wire_code, + "reason": termination.reason, + "extendedCode": termination.extended_code, + "resumable": resumable, + }); + emit_nvst_terminal_with_termination( + output, + lifecycle, + generation, + resources, + "server-terminated", + message, + termination, + ) + } + NvstReceiveEvent::Cursor(bytes) => { + resources.apply_cursor(bytes); + false + } + NvstReceiveEvent::CursorCapture(composited) => { + pending_control_output.pending = Some(composited); + flush_control_output(output, lifecycle, generation, pending_control_output); + false + } + NvstReceiveEvent::Dropped( + NvstDropReason::AwaitingStartOfFrame + | NvstDropReason::StaleRtpPacket { .. } + | NvstDropReason::DuplicateRtpPacket { .. }, + ) => { + // These are expected while a packet-gap recovery waits for the + // requested keyframe. Logging every following datagram can flood + // stdout and steal time from the receive/decode threads. + false + } + NvstReceiveEvent::Dropped(reason) => { + let _ = output.send(event( + "log", + json!({ "level": "debug", "message": format!("Dropped NVST datagram: {reason:?}") }), + )); + false + } + NvstReceiveEvent::Frame(_) => false, + } +} + +fn attempt_nvst_recovery( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + resources: &R, + recovery_attempts: &mut usize, + reason: String, +) -> bool { + if *recovery_attempts >= NVST_RECOVERY_ATTEMPT_LIMIT { + return emit_nvst_terminal( + output, + lifecycle, + generation, + resources, + "nvst-recovery-exhausted", + format!("NVST recovery failed after one attempt: {reason}"), + ); + } + + *recovery_attempts += 1; + resources.cut_recording(RecordingCutReason::Interrupted); + resources.request_keyframe(); + if let Err(recovery_error) = resources.recover() { + return emit_nvst_terminal( + output, + lifecycle, + generation, + resources, + "nvst-recovery-failed", + format!("NVST recovery could not be started: {recovery_error}"), + ); + } + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": format!("Attempting bounded NVST recovery with a fresh keyframe: {reason}") + }), + )); + false +} + +fn emit_nvst_terminal( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + resources: &R, + code: &str, + message: String, +) -> bool { + emit_nvst_terminal_with_termination( + output, + lifecycle, + generation, + resources, + code, + message, + json!({"source":"nvst-transport","code":code,"resumable":null}), + ) +} + +fn emit_nvst_terminal_with_termination( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + resources: &R, + code: &str, + message: String, + termination: Value, +) -> bool { + { + let mut lifecycle = lock_lifecycle(lifecycle); + if lifecycle.generation != generation || lifecycle.context.is_none() { + return true; + } + lifecycle.context = None; + lifecycle.state = State::Idle; + } + resources.stop(); + opennow_streamer_protocol::log::log_line("WARN", "transport", &format!("{code}: {message}")); + let _ = output.send_terminal(event( + "error", + json!({ "code": code, "message": &message, "termination": &termination }), + )); + let _ = output.send_terminal(event( + "status", + json!({ "status": "stopped", "message": message, "termination": termination }), + )); + true +} + +fn frame_stage_timings_event(timings: Option) -> Value { + let Some(timings) = timings else { + return Value::Null; + }; + let stage = |summary: Option| match summary { + Some(summary) => json!({ + "p50": summary.p50_ms, + "p95": summary.p95_ms, + "max": summary.max_ms, + }), + None => Value::Null, + }; + json!({ + "deliveryToAdmissionMs": stage(timings.delivery_to_admission), + "admissionToControlQueueMs": stage(timings.admission_to_control_queue), + "assembledToControlQueueMs": stage(timings.assembled_to_control_queue), + "deliveryWindowSamples": timings.delivery_window_samples, + "ackWindowSamples": timings.ack_window_samples, + "assembledFramesTotal": timings.assembled_frames_total, + "admittedFramesTotal": timings.admitted_frames_total, + "queuedAckFramesTotal": timings.queued_ack_frames_total, + "undeliveredFramesTotal": timings.undelivered_frames_total, + "pendingDeliveries": timings.pending_deliveries, + "unmatchedDeliveries": timings.unmatched_deliveries, + "unmatchedAdmissions": timings.unmatched_admissions, + }) +} + +fn decode_progress_stage_name(stage: DecodeProgressStage) -> &'static str { + match stage { + DecodeProgressStage::Tracking => "tracking", + DecodeProgressStage::KeyframePending => "keyframe-pending", + DecodeProgressStage::RecoveryRequired => "recovery-required", + } +} + +fn decode_timings_event(timings: Option) -> Value { + let Some(timings) = timings else { + return Value::Null; + }; + let stage = |stage: Option| match stage { + Some(stage) => json!({ + "p50": stage.p50_us as f64 / 1_000.0, + "p95": stage.p95_us as f64 / 1_000.0, + "max": stage.max_us as f64 / 1_000.0, + }), + None => Value::Null, + }; + json!({ + "call": stage(timings.call), + "residence": stage(timings.residence), + "callWindowSamples": timings.call_window_samples, + "residenceWindowSamples": timings.residence_window_samples, + "submissionsTotal": timings.submissions_total, + "outputsTotal": timings.outputs_total, + "outputCallsTotal": timings.output_calls_total, + "inFlight": timings.in_flight, + "epoch": timings.epoch, + "unmatchedOutputs": timings.unmatched_outputs, + "unmatchedSubmissions": timings.unmatched_submissions, + }) +} + +struct NvstMediaFeedbackState { + drop_reports: QueueDropReports, + recovery_attempts: usize, + input_origin: Instant, + input_available: bool, + telemetry_window_started: Instant, + telemetry_frames: u64, + telemetry_bytes: u64, + telemetry_started: bool, + previous_socket_receive_bytes: u64, + peak_bitrate_mbps: f64, + decode_timings: Option, + decode_progress: DecodeProgressWatchdog, + decode_recovery_deadline: Option, + transport_frame_progress_stalled: bool, + last_video_decoder_error: Option, + start_id: String, +} + +impl NvstMediaFeedbackState { + fn new(input_available: bool) -> Self { + Self { + drop_reports: QueueDropReports::new(), + recovery_attempts: 0, + input_origin: Instant::now(), + input_available, + telemetry_window_started: Instant::now(), + telemetry_frames: 0, + telemetry_bytes: 0, + telemetry_started: false, + previous_socket_receive_bytes: 0, + peak_bitrate_mbps: 0.0, + decode_timings: None, + decode_progress: DecodeProgressWatchdog::default(), + decode_recovery_deadline: None, + transport_frame_progress_stalled: false, + last_video_decoder_error: None, + start_id: String::new(), + } + } +} + +fn flush_nvst_telemetry( + output: &EventSender, + resources: &R, + state: &mut NvstMediaFeedbackState, +) { + let elapsed = state.telemetry_window_started.elapsed(); + if elapsed < TELEMETRY_INTERVAL { + return; + } + let mut payload = media_telemetry( + elapsed, + state.telemetry_frames, + state.telemetry_bytes, + resources.socket_receive_bytes(), + &mut state.previous_socket_receive_bytes, + &mut state.peak_bitrate_mbps, + ); + let network = resources.network_metrics(); + payload.as_object_mut().unwrap().extend( + json!({ + "pingMs": resources.ping_ms(), + "jitterMs": network.map(|metrics| metrics.0), + "packetLossPercent": network.map(|metrics| metrics.1), + "frameStageTimings": frame_stage_timings_event(resources.frame_stage_timings()), + "decodeTimeMs": state.decode_timings.and_then(|timings| timings.call) + .map(|stage| stage.p50_us as f64 / 1_000.0), + "decoderResidenceMs": state.decode_timings.and_then(|timings| timings.residence) + .map(|stage| stage.p50_us as f64 / 1_000.0), + "decodeTimings": decode_timings_event(state.decode_timings), + "decodeProgressStage": decode_progress_stage_name(state.decode_progress.stage()), + "transportFrameProgressStalled": state.transport_frame_progress_stalled, + "startId": state.start_id, + }) + .as_object() + .unwrap() + .clone(), + ); + let _ = output.send(event("telemetry", payload)); + state.telemetry_window_started = Instant::now(); + state.telemetry_frames = 0; + state.telemetry_bytes = 0; +} + +fn forward_nvst_media_feedback( + output: &EventSender, + lifecycle: &Mutex, + generation: u64, + resources: &R, + feedback: MediaFeedback, + state: &mut NvstMediaFeedbackState, +) { + if let MediaFeedback::QueueDropped { media, count } = feedback { + state.drop_reports.record(media, count); + return; + } + if lock_lifecycle(lifecycle).generation != generation { + return; + } + match feedback { + MediaFeedback::VideoFrameAccepted { + frame_index, + bytes, + keyframe, + .. + } => { + state.transport_frame_progress_stalled = false; + if let Some(frame_index) = frame_index { + resources.acknowledge_video_frame(frame_index, bytes); + } + if keyframe { + state.recovery_attempts = 0; + } + state.telemetry_frames = state.telemetry_frames.saturating_add(1); + state.telemetry_bytes = state.telemetry_bytes.saturating_add(u64::from(bytes)); + state.telemetry_started = true; + flush_nvst_telemetry(output, resources, state); + } + MediaFeedback::PlaybackStarted { backend } => { + let _ = output.send(event( + "status", + json!({ + "event": "first-frame", + "backend": backend, + "status": "streaming", + "message": format!("{backend} presented the first NVST video frame") + }), + )); + } + MediaFeedback::BackendFallback { from, to, reason } => { + let _ = output.send(event( + "log", + json!({ + "event": "backend-fallback", + "fromBackend": from, + "toBackend": to, + "reason": reason, + "level": "warn", + "message": format!("{from} startup failed; using {to}: {reason}") + }), + )); + } + MediaFeedback::ColorFormatChanged { requested, actual } => { + let lifecycle = lock_lifecycle(lifecycle); + if lifecycle.generation != generation { + return; + } + let Some(context) = &lifecycle.context else { + return; + }; + let session_id = context.session.session_id.clone(); + drop(lifecycle); + let _ = output.send(event( + "log", + json!({ + "event": "color-format-changed", + "requestedColorQuality": requested.protocol_name(), + "actualColorQuality": actual.protocol_name(), + "sessionId": session_id, + "source": "decoder", + "level": if requested == actual { "info" } else { "warn" }, + "message": if requested == actual { + "The decoded video color format matches the requested format" + } else { + "The decoded video color format differs from the requested format" + } + }), + )); + } + MediaFeedback::RequestKeyframe { reason, .. } => { + resources.request_keyframe(); + let _ = output.send(event( + "log", + json!({ + "event": "keyframe-request", + "reason": reason, + "level": "info", + "message": format!("Requested an NVST video keyframe: {reason}") + }), + )); + } + MediaFeedback::DecoderError { codec, message } => { + let message = format!("{codec} decoder error: {message}"); + state.last_video_decoder_error = Some(message.chars().take(240).collect()); + let _ = output.send(event( + "error", + json!({ + "event": "decoder-error", + "codec": codec, + "code": "media-decode-error", + "message": message + }), + )); + } + MediaFeedback::AudioDecoderError { + message, + consecutive, + } => { + let _ = output.send(event( + "log", + json!({ + "event": "decoder-error", + "codec": "opus", + "consecutive": consecutive, + "level": "warn", + "message": format!("Opus decoder error: {message}") + }), + )); + } + MediaFeedback::AudioUnavailable { + backend, + reason, + rejected, + } => { + let message = format!("Audio is unavailable for the rest of this session: {reason}"); + opennow_streamer_protocol::log::log_line("WARN", "media-audio", &message); + let _ = output.send(event( + "log", + json!({ + "event": "audio-unavailable", + "backend": backend, + "rejectedPackets": rejected, + "level": "warn", + "message": message + }), + )); + } + MediaFeedback::OutputError { message } => { + let _ = output.send(event( + "error", + json!({ "event": "output-error", "code": "media-output-error", "message": message }), + )); + } + MediaFeedback::DeviceLost { + subsystem, + recovered, + message, + } => { + let _ = output.send(event( + "log", + json!({ + "event": "device-state", + "subsystem": subsystem, + "recovered": recovered, + "level": if recovered { "info" } else { "warn" }, + "message": message.unwrap_or_else(|| format!( + "{subsystem} device {}", + if recovered { "recovered" } else { "was lost" } + )) + }), + )); + } + MediaFeedback::DecodeTimings(timings) => { + state.decode_timings = Some(timings); + flush_nvst_telemetry(output, resources, state); + } + MediaFeedback::QueueDropped { .. } => unreachable!(), + } +} + +#[cfg(test)] +fn forward_nvst_captured_input( + resources: &R, + input: CapturedInput, + state: &NvstMediaFeedbackState, +) -> Result<(), TransportError> { + if matches!( + input, + CapturedInput::Guide + | CapturedInput::Screenshot + | CapturedInput::RecordingToggle + | CapturedInput::Shortcut(_) + ) { + return Ok(()); + } + let timestamp_us = u64::try_from(state.input_origin.elapsed().as_micros()).unwrap_or(u64::MAX); + if let CapturedInput::Text(text) = input { + return resources.send_captured_text(text, timestamp_us); + } + resources.send_captured_input(captured_input_packet(input, timestamp_us)) +} + +fn forward_nvst_captured_sample( + resources: &R, + sample: &CapturedInputSample, + state: &NvstMediaFeedbackState, +) -> Result<(), TransportError> { + if matches!( + sample.input, + CapturedInput::Guide + | CapturedInput::Screenshot + | CapturedInput::RecordingToggle + | CapturedInput::Shortcut(_) + ) { + return Ok(()); + } + // Bifrost timestamps native input at OS capture, before aggregation and + // SCTP sending. Keeping that time prevents a delayed queue drain from + // making a group of older reports look newly generated. + let captured = sample + .captured_at + .checked_duration_since(state.input_origin) + .unwrap_or_default(); + let timestamp_us = u64::try_from(captured.as_micros()).unwrap_or(u64::MAX); + if let CapturedInput::Text(text) = &sample.input { + return resources.send_captured_text(text.clone(), timestamp_us); + } + resources.send_captured_input(captured_input_packet(sample.input.clone(), timestamp_us)) +} + +fn captured_input_packet(input: CapturedInput, timestamp_us: u64) -> Vec { + match input { + CapturedInput::Text(_) => unreachable!("text uses typed transport submission"), + CapturedInput::Key { + virtual_key, + modifiers, + pressed, + } => { + let mut packet = Vec::with_capacity(18); + packet.extend_from_slice(&(if pressed { 3_u32 } else { 4_u32 }).to_le_bytes()); + packet.extend_from_slice(&virtual_key.to_be_bytes()); + packet.extend_from_slice(&modifiers.to_be_bytes()); + packet.extend_from_slice(&0_u16.to_be_bytes()); + packet.extend_from_slice(×tamp_us.to_be_bytes()); + packet + } + CapturedInput::MouseMove { delta_x, delta_y } => { + let (delta_x, delta_y) = tune_relative_mouse(delta_x, delta_y, input_tuning()); + let mut packet = Vec::with_capacity(22); + packet.extend_from_slice(&7_u32.to_le_bytes()); + packet.extend_from_slice(&delta_x.to_be_bytes()); + packet.extend_from_slice(&delta_y.to_be_bytes()); + packet.extend_from_slice(&[0; 6]); + packet.extend_from_slice(×tamp_us.to_be_bytes()); + packet + } + CapturedInput::MouseAbsolute { + x, + y, + width, + height, + } => { + let mut packet = Vec::with_capacity(26); + packet.extend_from_slice(&5_u32.to_le_bytes()); + packet.extend_from_slice(&x.to_be_bytes()); + packet.extend_from_slice(&y.to_be_bytes()); + packet.extend_from_slice(&0_u16.to_be_bytes()); + packet.extend_from_slice(&width.to_be_bytes()); + packet.extend_from_slice(&height.to_be_bytes()); + packet.extend_from_slice(&0_u32.to_be_bytes()); + packet.extend_from_slice(×tamp_us.to_be_bytes()); + packet + } + CapturedInput::MouseButton { button, pressed } => { + let mut packet = Vec::with_capacity(18); + packet.extend_from_slice(&(if pressed { 8_u32 } else { 9_u32 }).to_le_bytes()); + packet.extend_from_slice(&[button, 0]); + packet.extend_from_slice(&[0; 4]); + packet.extend_from_slice(×tamp_us.to_be_bytes()); + packet + } + CapturedInput::MouseWheel { delta_x, delta_y } => { + let mut packet = Vec::with_capacity(22); + packet.extend_from_slice(&10_u32.to_le_bytes()); + packet.extend_from_slice(&delta_x.to_be_bytes()); + packet.extend_from_slice(&delta_y.to_be_bytes()); + packet.extend_from_slice(&[0; 6]); + packet.extend_from_slice(×tamp_us.to_be_bytes()); + packet + } + CapturedInput::Gamepad { + controller_id, + bitmap, + buttons, + left_trigger, + right_trigger, + left_stick_x, + left_stick_y, + right_stick_x, + right_stick_y, + } => { + let mut packet = Vec::with_capacity(38); + packet.extend_from_slice(&12_u32.to_le_bytes()); + packet.extend_from_slice(&26_u16.to_le_bytes()); + packet.extend_from_slice(&u16::from(controller_id & 0x03).to_le_bytes()); + packet.extend_from_slice(&bitmap.to_le_bytes()); + packet.extend_from_slice(&20_u16.to_le_bytes()); + packet.extend_from_slice(&buttons.to_le_bytes()); + packet.extend_from_slice( + &(u16::from(left_trigger) | (u16::from(right_trigger) << 8)).to_le_bytes(), + ); + packet.extend_from_slice(&left_stick_x.to_le_bytes()); + packet.extend_from_slice(&left_stick_y.to_le_bytes()); + packet.extend_from_slice(&right_stick_x.to_le_bytes()); + packet.extend_from_slice(&right_stick_y.to_le_bytes()); + packet.extend_from_slice(&0_u16.to_le_bytes()); + packet.extend_from_slice(&85_u16.to_le_bytes()); + packet.extend_from_slice(&0_u16.to_le_bytes()); + packet.extend_from_slice(×tamp_us.to_le_bytes()); + packet + } + CapturedInput::Guide + | CapturedInput::Screenshot + | CapturedInput::RecordingToggle + | CapturedInput::Shortcut(_) => Vec::new(), + } +} + +#[derive(Debug, Clone, Copy)] +struct InputTuning { + sensitivity: f64, + acceleration_percent: f64, +} + +fn input_tuning() -> InputTuning { + static TUNING: OnceLock = OnceLock::new(); + *TUNING.get_or_init(|| InputTuning { + sensitivity: std::env::var("OPENNOW_MOUSE_SENSITIVITY") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(1.0) + .clamp(0.1, 3.0), + acceleration_percent: std::env::var("OPENNOW_MOUSE_ACCELERATION") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(1.0) + .clamp(1.0, 150.0), + }) +} + +fn tune_relative_mouse(delta_x: i16, delta_y: i16, tuning: InputTuning) -> (i16, i16) { + let mut x = f64::from(delta_x) * tuning.sensitivity; + let mut y = f64::from(delta_y) * tuning.sensitivity; + if tuning.acceleration_percent > 1.0 { + let speed = x.hypot(y); + let strength = (tuning.acceleration_percent - 1.0) / 149.0; + // Match the legacy client curve: preserve low-speed precision and cap + // the maximum turn boost at 60% for the 150% setting. + let factor = 1.0 + (0.6 * strength).min(speed / 50.0 * strength); + x *= factor; + y *= factor; + } + let clamp = |value: f64| { + value + .round() + .clamp(f64::from(i16::MIN), f64::from(i16::MAX)) as i16 + }; + (clamp(x), clamp(y)) +} + +fn media_stream_config(context: &SessionContext) -> MediaStreamConfig { + let codec_name = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("codec")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("codec").and_then(Value::as_str)) + .unwrap_or("H264"); + let codec = match codec_name.trim().to_ascii_uppercase().as_str() { + "H265" | "HEVC" => MediaVideoCodec::H265, + "AV1" => MediaVideoCodec::Av1, + _ => MediaVideoCodec::H264, + }; + let color_quality_name = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("colorQuality")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("colorQuality").and_then(Value::as_str)) + .unwrap_or("8bit_420"); + let color_quality = match color_quality_name.trim().to_ascii_lowercase().as_str() { + "8bit_444" if codec == MediaVideoCodec::H265 => MediaColorQuality::EightBit444, + "10bit_420" if codec != MediaVideoCodec::H264 => MediaColorQuality::TenBit420, + "10bit_444" if codec == MediaVideoCodec::H265 => MediaColorQuality::TenBit444, + "10bit_444" if codec == MediaVideoCodec::Av1 => MediaColorQuality::TenBit420, + _ => MediaColorQuality::EightBit420, + }; + let resolution = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("resolution")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("resolution").and_then(Value::as_str)) + .and_then(|value| { + let lowercase = value.to_ascii_lowercase(); + let (width, height) = lowercase.split_once('x')?; + Some((width.parse::().ok()?, height.parse::().ok()?)) + }) + .filter(|(width, height)| (48..=4096).contains(width) && (48..=2304).contains(height)) + .unwrap_or((1920, 1080)); + let fps = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("fps")) + .or_else(|| context.settings.get("fps")) + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .unwrap_or(60) + .clamp(1, MAX_STREAM_FPS); + let bitrate_mbps = context + .settings + .get("maxBitrateMbps") + .and_then(Value::as_f64) + .filter(|value| value.is_finite()) + .unwrap_or(75.0) + .clamp(0.22, 200.0); + let bitrate_bps = (bitrate_mbps * 1_000_000.0) + .round() + .clamp(1.0, f64::from(u32::MAX)) as u32; + let requested_cloud_gsync = match context + .settings + .get("nativeCloudGsyncMode") + .and_then(Value::as_str) + .unwrap_or("auto") + { + "disabled" => false, + "forced" => true, + _ => context + .settings + .get("enableCloudGsync") + .and_then(Value::as_bool) + .unwrap_or(false), + }; + // `enableCloudGsync` is the resolved client request, but CloudMatch may + // explicitly reject it in the finalized profile. Never switch Linux into + // unthrottled VRR pacing when the server negotiated the feature off. + let negotiated_cloud_gsync = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("enableCloudGsync")) + .and_then(Value::as_bool); + let cloud_gsync = requested_cloud_gsync && negotiated_cloud_gsync.unwrap_or(true); + let hdr = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("enableHdr")) + .and_then(Value::as_bool) + .unwrap_or(false); + MediaStreamConfig { + codec, + color_quality, + hdr, + width: resolution.0, + height: resolution.1, + fps, + bitrate_bps, + cloud_gsync, + shortcuts: StreamShortcutBindings::from_json(&context.shortcuts), + } +} + +fn consume_encoded_media( + output: &EventSender, + receiver: Receiver, + sink: MediaSink, +) { + let mut video = 0_u64; + let mut audio = 0_u64; + let mut keyframes = 0_u64; + let mut dropped = 0_u64; + let mut paused = 0_u64; + let origin = Instant::now(); + let mut last_report = origin; + opennow_streamer_protocol::log::log_line( + "INFO", + "media-ingress", + "consumer started; awaiting assembled media", + ); + loop { + if last_report.elapsed() >= Duration::from_secs(10) { + opennow_streamer_protocol::log::log_async( + "INFO", + "media-ingress", + &format!( + "elapsed_ms={} video={video} audio={audio} keyframes={keyframes} dropped={dropped} paused={paused}", + origin.elapsed().as_millis() + ), + ); + last_report = Instant::now(); + } + let frame = match receiver.recv_timeout(Duration::from_millis(250)) { + Ok(frame) => frame, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, + }; + if frame.codec.eq_ignore_ascii_case("opus") { + audio += 1; + } else { + video += 1; + } + if frame.keyframe { + keyframes += 1; + } + if video == 1 && !frame.codec.eq_ignore_ascii_case("opus") { + opennow_streamer_protocol::log::log_line( + "INFO", + "media-ingress", + &format!( + "first assembled video bytes={} keyframe={} contiguous={} elapsed_ms={}", + frame.payload.len(), + frame.keyframe, + frame.contiguous, + origin.elapsed().as_millis() + ), + ); + } + let codec = if frame.codec.eq_ignore_ascii_case("h264") { + MediaCodec::H264 + } else if frame.codec.eq_ignore_ascii_case("h265") + || frame.codec.eq_ignore_ascii_case("hevc") + { + MediaCodec::H265 + } else if frame.codec.eq_ignore_ascii_case("av1") { + MediaCodec::Av1 + } else if frame.codec.eq_ignore_ascii_case("opus") { + MediaCodec::Opus { + channels: frame.channels.unwrap_or(2).clamp(1, 2), + } + } else { + MediaCodec::Unsupported(frame.codec) + }; + match sink.push(EncodedFrame { + mid: frame.mid, + codec, + data: frame.payload, + frame_index: frame.frame_index, + timestamp: frame.rtp_timestamp, + clock_rate_hz: frame.clock_rate_hz, + keyframe: frame.keyframe, + contiguous: frame.contiguous, + ssrc: frame.ssrc, + }) { + PushOutcome::Unsupported => { + dropped += 1; + let _ = output.send(event( + "log", + json!({ + "level": "warn", + "message": "Dropping a frame for a codec not built into native streamer v2" + }), + )); + } + PushOutcome::Closed => break, + PushOutcome::DroppedOldest => dropped += 1, + PushOutcome::Paused => paused += 1, + PushOutcome::Queued => {} + } + } + opennow_streamer_protocol::log::log_line( + "INFO", + "media-ingress", + &format!( + "consumer stopped elapsed_ms={} video={video} audio={audio} keyframes={keyframes} dropped={dropped} paused={paused}", + origin.elapsed().as_millis() + ), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::UdpSocket; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use std::sync::mpsc; + use std::time::Instant; + + fn command(value: Value) -> Command { + serde_json::from_value(value).expect("command") + } + + #[test] + fn lifecycle_events_keep_the_session_that_created_the_sender() { + let (sender, receiver) = mpsc::channel(); + let base = EventSender::unbounded(sender); + let old = base.for_session("old-start"); + let current = old.for_session("new-start"); + old.send(event("status", json!({"status":"streaming"}))) + .unwrap(); + current + .send(event("status", json!({"status":"streaming"}))) + .unwrap(); + old.send_terminal(event("error", json!({"message":"old session ended"}))) + .unwrap(); + current + .send(event( + "status", + json!({"status":"paused","startId":"sender-authored"}), + )) + .unwrap(); + current + .send(event("telemetry", json!({"videoFramesReceived":1}))) + .unwrap(); + assert_eq!(receiver.recv().unwrap()["startId"], "old-start"); + assert_eq!(receiver.recv().unwrap()["startId"], "new-start"); + assert_eq!(receiver.recv().unwrap()["startId"], "old-start"); + assert_eq!(receiver.recv().unwrap()["startId"], "sender-authored"); + assert!(receiver.recv().unwrap().get("startId").is_none()); + } + + #[test] + fn bounded_lifecycle_sender_scopes_normal_and_terminal_delivery() { + let (sender, receiver) = mpsc::sync_channel(2); + let sender = EventSender::bounded(sender).for_session("bounded-start"); + sender + .send(event("status", json!({"status":"streaming"}))) + .unwrap(); + sender + .send_terminal(event("status", json!({"status":"stopped"}))) + .unwrap(); + assert_eq!(receiver.recv().unwrap()["startId"], "bounded-start"); + assert_eq!(receiver.recv().unwrap()["startId"], "bounded-start"); + } + + #[test] + fn replay_clip_commands_complete_asynchronously_with_correlated_success_and_failure() { + let (host, runtime) = opennow_streamer_platform::create_test_runtime(); + let (events, received) = std::sync::mpsc::channel(); + let mut engine = Engine::with_media_runtime(events, runtime.clone()); + let (feedback, _feedback_receiver) = std::sync::mpsc::channel(); + let session = runtime + .start(feedback, MediaStreamConfig::default()) + .unwrap(); + session.control().start_replay( + ReplayBufferConfig::from_settings(&json!({"replayBufferEnabled":true})), + Arc::clone(&engine.replay_budget), + ); + let sink = session.sink(); + engine.media_session = Some(session); + lock_lifecycle(&engine.lifecycle).state = State::Connected; + let directory = std::env::temp_dir().join(format!( + "opennow-engine-clip-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + for (id, timestamp, data, expected_state) in [ + ( + "saved", + 90_000, + vec![ + 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x28, 0xde, 0xad, 0, 0, 0, 1, 0x68, 0xee, 0x3c, + 0x80, 0, 0, 0, 1, 0x65, 0x88, 0x84, 0x00, 0x10, + ], + "saved", + ), + ("failed", 180_000, vec![0, 0, 0, 1, 0x65], "failed"), + ] { + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from(data), + frame_index: Some(1), + timestamp, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }); + let path = directory.join(format!("{id}.mkv")); + let (responses, _) = engine.handle(command( + json!({"id":id,"type":"clip-save","outputPath":path}), + )); + assert_eq!(responses[0]["type"], "clip-saving"); + assert_eq!(responses[0]["path"], json!(path)); + let completion = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(completion["type"], "clip-state"); + assert_eq!(completion["requestId"], id); + assert_eq!(completion["state"], expected_state); + assert_eq!(completion["path"], json!(path)); + assert!(completion["message"].is_string()); + assert_eq!(path.exists(), expected_state == "saved"); + engine.clip_worker.take().unwrap().join().unwrap(); + } + engine.stop("test complete"); + runtime.shutdown(); + host.join().unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn replay_commands_fail_closed_and_stop_cancels_without_waiting_for_export() { + let (sender, receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let path = std::env::temp_dir().join("opennow-command-test.mkv"); + let (responses, _) = engine.handle(command( + json!({"id":"disabled","type":"clip-save","outputPath":path}), + )); + assert_eq!(responses[0]["code"], "replay-not-enabled"); + let (responses, _) = engine.handle(command( + json!({"id":"invalid","type":"clip-save","outputPath":"relative.mkv"}), + )); + assert_eq!(responses[0]["code"], "invalid-clip-output"); + let (release, blocked) = std::sync::mpsc::channel(); + engine.clip_worker = Some(thread::spawn(move || { + let _ = blocked.recv(); + })); + engine.clip_cancelled.store(false, Ordering::Release); + let (responses, _) = engine.handle(command( + json!({"id":"busy","type":"clip-save","outputPath":path}), + )); + assert_eq!(responses[0]["code"], "clip-already-saving"); + let (responses, _) = engine.handle(command(json!({"id":"stop","type":"replay-stop"}))); + assert_eq!(responses[0]["type"], "replay-stopped"); + assert!(engine.clip_cancelled.load(Ordering::Acquire)); + assert!(!engine.clip_worker.as_ref().unwrap().is_finished()); + assert!(receiver.try_recv().is_err()); + release.send(()).unwrap(); + engine.clip_worker.take().unwrap().join().unwrap(); + let (sender, receiver) = std::sync::mpsc::channel(); + forward_shortcut_action( + &EventSender::unbounded(sender), + None, + StreamShortcutAction::SaveClip, + ); + let action = receiver.recv().unwrap(); + assert_eq!(action["type"], "shortcut-action"); + assert_eq!(action["action"], "save-clip"); + } + + fn synthetic_context(session_id: &str, ice_servers: Value) -> Value { + json!({ + "session": { + "sessionId": session_id, + "serverIp": "127-0-0-1.synthetic.invalid", + "iceServers": ice_servers, + "mediaConnectionInfo": { + "ip": "127-0-0-1.media.synthetic.invalid", + "port": 18_784, + "usage": 17 + }, + "syntheticExtension": "preserved" + }, + "settings": { "codec": "H264", "fps": 60 }, + "shortcuts": { "stopStream": "Ctrl+Shift+Q" }, + "syntheticContextExtension": true + }) + } + + fn lifecycle_state(engine: &Engine) -> State { + lock_lifecycle(&engine.lifecycle).state + } + + #[test] + fn audio_mute_without_playback_runtime_fails_without_changing_lifecycle() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let (responses, keep_running) = engine.handle(command(json!({ + "type": "setAudioMuted", "id": "mute", "muted": true + }))); + assert!(keep_running); + assert_eq!(responses[0]["type"], "error"); + assert_eq!(responses[0]["id"], "mute"); + assert_eq!(responses[0]["code"], "unsupported-command"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + + #[test] + fn audio_devices_without_runtime_returns_correlated_error() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let (responses, keep_running) = + engine.handle(command(json!({"type": "audioDevices", "id": "audio"}))); + assert!(keep_running); + assert_eq!(responses[0]["type"], "error"); + assert_eq!(responses[0]["id"], "audio"); + assert_eq!(responses[0]["code"], "audio-devices-unavailable"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + + #[test] + fn start_rejects_invalid_audio_output_before_changing_state() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + for value in [json!(1), json!(null), json!("a\0b"), json!("é".repeat(513))] { + let mut context = synthetic_context("audio-test", json!([])); + context["settings"]["audioOutputDevice"] = value; + let (responses, _) = engine.handle(command( + json!({"type": "start", "id": "audio", "context": context}), + )); + assert_eq!(responses[0]["code"], "invalid-context"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + } + + #[test] + fn irrelevant_cloudmatch_connections_do_not_reject_a_valid_media_endpoint() { + let mut context = synthetic_context("seat", json!([])); + context["session"]["connectionInfo"] = json!([ + {"usage":15,"ip":"unused.example","port":0}, + {"usage":14,"ip":"signaling.example","port":322} + ]); + context["session"]["mediaConnectionInfo"] = + json!({"ip":"203.0.113.20","port":5004,"usage":17}); + let context: SessionContext = serde_json::from_value(context).unwrap(); + assert!(validate_context(&context, "start").is_ok()); + + let mut invalid_media = context; + invalid_media + .session + .media_connection_info + .as_mut() + .unwrap() + .port = 0; + assert_eq!( + validate_context(&invalid_media, "start").unwrap_err()["code"], + "invalid-context" + ); + } + + #[test] + fn shell_shortcuts_emit_typed_actions() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + forward_shortcut_action(&sender, None, StreamShortcutAction::ToggleStats); + let action = receiver.recv().expect("stats shortcut action"); + assert_eq!(action["type"], "shortcut-action"); + assert_eq!(action["action"], "toggle-stats"); + + forward_shortcut_action(&sender, None, StreamShortcutAction::ToggleFullscreen); + let action = receiver.recv().expect("fullscreen shortcut action"); + assert_eq!(action["type"], "shortcut-action"); + assert_eq!(action["action"], "toggle-fullscreen"); + } + + #[test] + fn legacy_stats_toggle_routes_to_the_shell_without_native_rendering() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let (responses, keep_running) = engine.handle(command(json!({ + "id": "stats", + "type": "stats-toggle" + }))); + + assert!(keep_running); + assert_eq!(responses[0]["type"], "ok"); + assert_eq!(responses[1]["type"], "shortcut-action"); + assert_eq!(responses[1]["action"], "toggle-stats"); + assert_eq!(responses[1]["source"], "command"); + } + + #[test] + fn legacy_fullscreen_toggle_routes_to_the_shell_without_native_mutation() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let (responses, keep_running) = engine.handle(command(json!({ + "id": "fullscreen", + "type": "fullscreen-toggle" + }))); + + assert!(keep_running); + assert_eq!(responses[0]["type"], "ok"); + assert_eq!(responses[1]["type"], "shortcut-action"); + assert_eq!(responses[1]["action"], "toggle-fullscreen"); + assert_eq!(responses[1]["source"], "command"); + } + + fn unused_udp_port() -> u16 { + let socket = UdpSocket::bind("127.0.0.1:0").expect("ephemeral UDP port"); + let port = socket.local_addr().expect("socket address").port(); + drop(socket); + port + } + + #[derive(Default)] + struct TestNvstResources { + rumble: Arc; 4]>>, + ping_ms: Option, + socket_bytes: Option>, + frame_stage_timings: Option, + decode_progress_policy: Option, + keyframe_requests: Arc, + recording_cuts: Mutex>, + acknowledged_frames: AtomicUsize, + acknowledged_frame_data: Mutex>, + recoveries: Arc, + stops: AtomicUsize, + captured_inputs: Mutex>>, + input_not_ready: Arc, + input_closed: bool, + input_sent: Option>>, + } + + impl NvstSessionResources for TestNvstResources { + fn take_rumble(&self) -> ([Option; 4], usize) { + (std::mem::take(&mut *self.rumble.lock().unwrap()), 0) + } + fn ping_ms(&self) -> Option { + self.ping_ms + } + + fn socket_receive_bytes(&self) -> Option { + self.socket_bytes + .as_ref() + .map(|bytes| bytes.load(Ordering::Relaxed)) + } + + fn frame_stage_timings(&self) -> Option { + self.frame_stage_timings + } + + fn decode_progress_policy(&self) -> DecodeProgressPolicy { + self.decode_progress_policy + .unwrap_or_else(|| DecodeProgressPolicy { + stall: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + keyframe_grace: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + recovery_grace: Duration::from_millis(nvst_rtsp::VIDEO_TIMEOUT_MS), + }) + } + + fn request_keyframe(&self) { + self.keyframe_requests.fetch_add(1, Ordering::Relaxed); + } + + fn cut_recording(&self, reason: RecordingCutReason) { + self.recording_cuts.lock().unwrap().push(reason); + } + + fn acknowledge_video_frame(&self, frame_index: u32, bytes: u32) { + self.acknowledged_frames.fetch_add(1, Ordering::Relaxed); + self.acknowledged_frame_data + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push((frame_index, bytes)); + } + + fn send_captured_input(&self, bytes: Vec) -> Result<(), TransportError> { + if self.input_closed { + return Err(TransportError::Closed); + } + let mut remaining = self.input_not_ready.load(Ordering::Acquire); + while remaining != 0 { + match self.input_not_ready.compare_exchange_weak( + remaining, + remaining - 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Err(TransportError::InputNotReady), + Err(current) => remaining = current, + } + } + if let Some(sent) = &self.input_sent { + let _ = sent.send(bytes.clone()); + } + self.captured_inputs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(bytes); + Ok(()) + } + + fn apply_cursor(&self, _bytes: Vec) {} + + fn send_captured_text( + &self, + text: opennow_streamer_protocol::text_input::UnicodeText, + _timestamp_us: u64, + ) -> Result<(), TransportError> { + self.captured_inputs + .lock() + .unwrap() + .push(text.as_str().as_bytes().to_vec()); + Ok(()) + } + + fn recover(&self) -> Result<(), String> { + self.recoveries.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + fn stop(&self) { + self.stops.fetch_add(1, Ordering::Relaxed); + } + } + + fn connected_lifecycle() -> Mutex { + Mutex::new(Lifecycle { + state: State::Connected, + context: Some( + serde_json::from_value(synthetic_context("nvst-recovery", json!([]))) + .expect("session context"), + ), + generation: 7, + }) + } + + mod server_control_tests { + include!("nvst_server_control_tests.rs"); + } + + #[test] + fn decoder_keyframe_feedback_routes_to_nvst_pli_handle() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::RequestKeyframe { + mid: "nvst-video-0".to_owned(), + reason: "decoder reference loss".to_owned(), + }, + &mut state, + ); + + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 1); + let message = receiver.recv().expect("keyframe log"); + assert_eq!(message["type"], "log"); + assert!( + message["message"] + .as_str() + .is_some_and(|message| message.contains("decoder reference loss")) + ); + } + + #[test] + fn audio_decode_feedback_stays_non_fatal_and_audio_scoped() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::AudioDecoderError { + message: "corrupted stream".to_owned(), + consecutive: 3, + }, + &mut state, + ); + + let message = receiver.recv().expect("audio decode log"); + assert_eq!(message["type"], "log"); + assert_eq!(message["event"], "decoder-error"); + assert_eq!(message["codec"], "opus"); + assert_eq!(message["level"], "warn"); + assert_eq!(message["consecutive"], 3); + assert!( + message["message"] + .as_str() + .is_some_and(|message| message.contains("corrupted stream")) + ); + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::AudioUnavailable { + backend: "ALSA", + reason: "audio output was lost".to_owned(), + rejected: 0, + }, + &mut state, + ); + + let message = receiver.recv().expect("audio unavailable log"); + assert_eq!(message["type"], "log"); + assert_eq!(message["event"], "audio-unavailable"); + assert_eq!(message["backend"], "ALSA"); + assert_eq!(message["rejectedPackets"], 0); + assert_eq!(message["level"], "warn"); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); + } + + #[test] + fn audio_output_device_loss_feedback_stays_non_fatal() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::DeviceLost { + subsystem: "ALSA", + recovered: false, + message: Some("Alsa device was lost: Broken pipe (os error 32)".to_owned()), + }, + &mut state, + ); + + let message = receiver.recv().expect("device state log"); + assert_eq!(message["type"], "log"); + assert_eq!(message["event"], "device-state"); + assert_eq!(message["subsystem"], "ALSA"); + assert_eq!(message["recovered"], false); + assert_eq!(message["level"], "warn"); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); + } + + #[test] + fn color_format_feedback_reports_actual_output_without_restarting_media() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + for generation in [6, 7] { + forward_nvst_media_feedback( + &sender, + &lifecycle, + generation, + &resources, + MediaFeedback::ColorFormatChanged { + requested: MediaColorQuality::TenBit444, + actual: MediaColorQuality::TenBit420, + }, + &mut state, + ); + if generation == 6 { + assert!(receiver.try_recv().is_err()); + } + } + let message = receiver.try_recv().expect("color format notification"); + assert_eq!(message["type"], "log"); + assert_eq!(message["event"], "color-format-changed"); + assert_eq!(message["source"], "decoder"); + assert_eq!( + message["sessionId"], + lock_lifecycle(&lifecycle) + .context + .as_ref() + .unwrap() + .session + .session_id + ); + assert_eq!(message["requestedColorQuality"], "10bit_444"); + assert_eq!(message["actualColorQuality"], "10bit_420"); + assert_eq!(message["level"], "warn"); + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::ColorFormatChanged { + requested: MediaColorQuality::TenBit444, + actual: MediaColorQuality::TenBit444, + }, + &mut state, + ); + let restored = receiver.try_recv().expect("restored color format"); + assert_eq!(restored["actualColorQuality"], "10bit_444"); + assert_eq!(restored["level"], "info"); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 0); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn accepted_video_keyframe_routes_pacing_feedback_and_resets_recovery_budget() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + state.recovery_attempts = 1; + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(71), + timestamp: 90_000, + bytes: 1_024, + keyframe: true, + }, + &mut state, + ); + + assert_eq!(resources.acknowledged_frames.load(Ordering::Relaxed), 1); + assert_eq!( + *resources + .acknowledged_frame_data + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + [(71, 1_024)] + ); + assert_eq!(state.recovery_attempts, 0); + } + + #[test] + fn accepted_video_frames_emit_bounded_shell_telemetry() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(72), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + + let telemetry = receiver.recv().expect("stream telemetry"); + assert_eq!(telemetry["type"], "telemetry"); + assert!( + telemetry["framesPerSecond"] + .as_f64() + .is_some_and(|value| value > 0.0) + ); + assert!( + telemetry["bitrateMbps"] + .as_f64() + .is_some_and(|value| (0.9..=1.0).contains(&value)) + ); + assert_eq!(telemetry["peakBitrateMbps"], telemetry["bitrateMbps"]); + } + + #[test] + fn socket_receive_rate_tracks_cumulative_bytes_through_idle_and_reset() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let bytes = Arc::new(AtomicU64::new(50_000)); + let resources = TestNvstResources { + socket_bytes: Some(Arc::clone(&bytes)), + ..Default::default() + }; + let mut state = NvstMediaFeedbackState::new(true); + state.previous_socket_receive_bytes = resources.socket_receive_bytes().unwrap(); + bytes.store(300_000, Ordering::Relaxed); + state.telemetry_window_started = Instant::now() - Duration::from_secs(2); + flush_nvst_telemetry(&sender, &resources, &mut state); + let first = receiver.recv().unwrap(); + assert!((first["receiveBitrateMbps"].as_f64().unwrap() - 1.0).abs() < 0.01); + assert_eq!(first["bitrateMbps"], json!(0.0)); + + bytes.store(550_000, Ordering::Relaxed); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &resources, &mut state); + let second = receiver.recv().unwrap(); + assert!((second["receiveBitrateMbps"].as_f64().unwrap() - 2.0).abs() < 0.02); + + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &resources, &mut state); + assert_eq!(receiver.recv().unwrap()["receiveBitrateMbps"], json!(0.0)); + + bytes.store(100, Ordering::Relaxed); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &resources, &mut state); + assert_eq!(receiver.recv().unwrap()["receiveBitrateMbps"], Value::Null); + + bytes.store(125_100, Ordering::Relaxed); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &resources, &mut state); + assert!( + (receiver.recv().unwrap()["receiveBitrateMbps"] + .as_f64() + .unwrap() + - 1.0) + .abs() + < 0.01 + ); + + let mut next_session = NvstMediaFeedbackState::new(true); + let new_resources = TestNvstResources { + socket_bytes: Some(Arc::new(AtomicU64::new(0))), + ..Default::default() + }; + next_session.telemetry_window_started = Instant::now() - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &new_resources, &mut next_session); + assert_eq!(receiver.recv().unwrap()["receiveBitrateMbps"], json!(0.0)); + } + + #[test] + fn accepted_video_telemetry_preserves_measured_and_unavailable_ping() { + for ping_ms in [None, Some(0.0), Some(25.5)] { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let resources = TestNvstResources { + ping_ms, + ..Default::default() + }; + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(72), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver.recv().unwrap(); + assert_eq!(telemetry["type"], "telemetry"); + assert!(telemetry.get("pingMs").is_some()); + assert_eq!(telemetry["pingMs"], json!(ping_ms)); + assert_eq!(telemetry["frameStageTimings"], json!(null)); + } + } + + #[test] + fn sanitized_progress_distinguishes_assembly_submission_output_and_restart() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let mut state = NvstMediaFeedbackState::new(true); + state.start_id = "private-start-id".to_owned(); + for (assembled, submissions, outputs, in_flight, epoch) in [ + (100, 98, 97, 1, 2), + (200, 98, 97, 1, 2), + (300, 200, 97, 103, 2), + (1, 1, 0, 1, 3), + ] { + let resources = TestNvstResources { + frame_stage_timings: Some(FrameStageTimings { + assembled_frames_total: assembled, + ..Default::default() + }), + ..Default::default() + }; + let now = Instant::now(); + state.telemetry_window_started = now; + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: submissions, + outputs_total: outputs, + output_calls_total: outputs, + last_submission_at: Some(now), + last_output_at: (outputs > 0).then_some(now), + in_flight, + oldest_in_flight_at: Some(now), + epoch, + epoch_started_at: Some(now), + unmatched_outputs: 0, + unmatched_submissions: 0, + }), + &mut state, + ); + assert!(receiver.try_recv().is_err()); + state.telemetry_window_started = now - Duration::from_secs(1); + flush_nvst_telemetry(&sender, &resources, &mut state); + let summary = + opennow_streamer_protocol::log::message_summary(&receiver.try_recv().unwrap()); + for field in [ + format!("frameStageTimings.assembledFramesTotal={assembled}"), + format!("decodeTimings.submissionsTotal={submissions}"), + format!("decodeTimings.outputsTotal={outputs}"), + format!("decodeTimings.inFlight={in_flight}"), + format!("decodeTimings.epoch={epoch}"), + ] { + assert!(summary.split(' ').any(|entry| entry == field), "{summary}"); + } + assert!(!summary.contains("private-start-id")); + } + } + + #[test] + fn sanitized_keyframe_feedback_preserves_only_known_recovery_codes() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + for (reason, expected) in [ + ( + "embedded Linux decoder queue overflow", + "type=log event=keyframe-request reasonCode=decoder-queue-overflow", + ), + ( + "Linux decoder requires a fresh keyframe", + "type=log event=keyframe-request reasonCode=decoder-reference-required", + ), + ( + "private session token https://private.example", + "type=log event=keyframe-request", + ), + ] { + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::RequestKeyframe { + mid: "private-media-id".to_owned(), + reason: reason.to_owned(), + }, + &mut state, + ); + assert_eq!( + opennow_streamer_protocol::log::message_summary(&receiver.try_recv().unwrap()), + expected + ); + } + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 3); + } + + #[test] + fn telemetry_reports_only_measured_frame_stage_timings() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let resources = TestNvstResources { + frame_stage_timings: Some(FrameStageTimings { + delivery_to_admission: Some(opennow_streamer_transport::StageSummary { + p50_ms: 3.5, + p95_ms: 8.25, + max_ms: 11.0, + }), + admission_to_control_queue: Some(opennow_streamer_transport::StageSummary { + p50_ms: 1.5, + p95_ms: 4.0, + max_ms: 6.0, + }), + assembled_to_control_queue: Some(opennow_streamer_transport::StageSummary { + p50_ms: 5.0, + p95_ms: 12.0, + max_ms: 17.0, + }), + delivery_window_samples: 42, + ack_window_samples: 40, + assembled_frames_total: 900, + admitted_frames_total: 897, + queued_ack_frames_total: 896, + pending_deliveries: 2, + unmatched_deliveries: 1, + unmatched_admissions: 3, + ..Default::default() + }), + ..Default::default() + }; + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(73), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver + .try_recv() + .expect("frame stage telemetry for the completed window"); + let timings = &telemetry["frameStageTimings"]; + assert_eq!(timings["deliveryToAdmissionMs"]["p50"], json!(3.5)); + assert_eq!(timings["deliveryToAdmissionMs"]["p95"], json!(8.25)); + assert_eq!(timings["admissionToControlQueueMs"]["max"], json!(6.0)); + assert_eq!(timings["assembledToControlQueueMs"]["p95"], json!(12.0)); + assert_eq!(timings["deliveryWindowSamples"], json!(42)); + assert_eq!(timings["ackWindowSamples"], json!(40)); + assert_eq!(timings["assembledFramesTotal"], json!(900)); + assert_eq!(timings["admittedFramesTotal"], json!(897)); + assert_eq!(timings["queuedAckFramesTotal"], json!(896)); + assert_eq!(timings["pendingDeliveries"], json!(2)); + assert_eq!(timings["unmatchedDeliveries"], json!(1)); + assert_eq!(timings["unmatchedAdmissions"], json!(3)); + } + + #[test] + fn decode_timings_feedback_reaches_telemetry_with_totals_and_measured_stages() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + let submitted_at = Instant::now(); + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: Some(DecodeStageTimings { + p50_us: 2_400, + p95_us: 5_000, + max_us: 7_500, + }), + residence: Some(DecodeStageTimings { + p50_us: 9_000, + p95_us: 14_000, + max_us: 21_000, + }), + call_window_samples: 256, + residence_window_samples: 256, + submissions_total: 4_096, + outputs_total: 4_090, + output_calls_total: 4_090, + last_submission_at: Some(submitted_at), + last_output_at: Some(submitted_at), + in_flight: 0, + oldest_in_flight_at: None, + epoch: 0, + epoch_started_at: None, + unmatched_outputs: 1, + unmatched_submissions: 2, + }), + &mut state, + ); + assert!( + receiver.try_recv().is_err(), + "timings stay for the next telemetry window" + ); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(74), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver + .try_recv() + .expect("decode timings telemetry for the completed window"); + assert_eq!(telemetry["decodeTimeMs"], json!(2.4)); + assert_eq!(telemetry["decoderResidenceMs"], json!(9.0)); + let timings = &telemetry["decodeTimings"]; + assert_eq!(timings["call"]["p50"], json!(2.4)); + assert_eq!(timings["call"]["p95"], json!(5.0)); + assert_eq!(timings["residence"]["max"], json!(21.0)); + assert_eq!(timings["callWindowSamples"], json!(256)); + assert_eq!(timings["residenceWindowSamples"], json!(256)); + assert_eq!(timings["submissionsTotal"], json!(4_096)); + assert_eq!(timings["outputsTotal"], json!(4_090)); + assert_eq!(timings["outputCallsTotal"], json!(4_090)); + assert_eq!(timings["unmatchedOutputs"], json!(1)); + assert_eq!(timings["unmatchedSubmissions"], json!(2)); + assert_eq!(timings["inFlight"], json!(0)); + assert_eq!(timings["epoch"], json!(0)); + } + + #[test] + fn decode_timings_report_input_work_before_any_output() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + let submitted_at = Instant::now(); + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(submitted_at), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(submitted_at), + epoch: 2, + epoch_started_at: Some(submitted_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }), + &mut state, + ); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &lifecycle, + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(76), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver + .try_recv() + .expect("decode telemetry for outstanding input"); + assert_eq!(telemetry["decodeTimeMs"], json!(null)); + assert_eq!(telemetry["decoderResidenceMs"], json!(null)); + let timings = &telemetry["decodeTimings"]; + assert_eq!(timings["call"], json!(null)); + assert_eq!(timings["residence"], json!(null)); + assert_eq!(timings["submissionsTotal"], json!(1)); + assert_eq!(timings["outputsTotal"], json!(0)); + assert_eq!(timings["inFlight"], json!(1)); + assert_eq!(timings["epoch"], json!(2)); + } + + #[test] + fn telemetry_publishes_outstanding_decoder_input_without_a_new_accepted_frame() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + let submitted_at = Instant::now(); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(submitted_at), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(submitted_at), + epoch: 3, + epoch_started_at: Some(submitted_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }), + &mut state, + ); + let telemetry = receiver + .try_recv() + .expect("a decoder hung on its only submission still publishes telemetry"); + assert_eq!(telemetry["type"], json!("telemetry")); + let timings = &telemetry["decodeTimings"]; + assert_eq!(timings["submissionsTotal"], json!(1)); + assert_eq!(timings["outputsTotal"], json!(0)); + assert_eq!(timings["inFlight"], json!(1)); + assert_eq!(timings["epoch"], json!(3)); + assert_eq!( + telemetry["decodeTimeMs"], + json!(null), + "an unmeasured decode duration stays null instead of zero" + ); + assert_eq!(telemetry["decoderResidenceMs"], json!(null)); + assert_eq!( + telemetry["framesPerSecond"], + json!(0.0), + "the window truly contained no accepted frames" + ); + assert_eq!(telemetry["bitrateMbps"], json!(0.0)); + assert_eq!(telemetry["peakBitrateMbps"], json!(0.0)); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(submitted_at), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(submitted_at), + epoch: 3, + epoch_started_at: Some(submitted_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }), + &mut state, + ); + assert!( + receiver.try_recv().is_err(), + "a report inside the open window cannot repeat the previous rates" + ); + } + + #[test] + fn telemetry_rates_are_per_window_and_not_repeated_from_a_stale_window() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(11), + timestamp: 90_000, + bytes: 1_000_000, + keyframe: false, + }, + &mut state, + ); + let measured = receiver.try_recv().expect("frame window telemetry"); + assert!( + measured["framesPerSecond"].as_f64().expect("rate") > 0.0, + "a window with an accepted frame reports a measured rate" + ); + assert!(measured["bitrateMbps"].as_f64().expect("bitrate") > 0.0); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: Some(DecodeStageTimings { + p50_us: 2_400, + p95_us: 5_000, + max_us: 7_500, + }), + residence: None, + call_window_samples: 1, + residence_window_samples: 0, + submissions_total: 2, + outputs_total: 1, + output_calls_total: 1, + last_submission_at: Some(Instant::now()), + last_output_at: Some(Instant::now()), + in_flight: 0, + oldest_in_flight_at: None, + epoch: 0, + epoch_started_at: None, + unmatched_outputs: 0, + unmatched_submissions: 0, + }), + &mut state, + ); + let stalled = receiver.try_recv().expect("decode window telemetry"); + assert_eq!( + stalled["framesPerSecond"], + json!(0.0), + "the refreshed window reports its own frames, not the previous window's rate" + ); + assert_eq!(stalled["bitrateMbps"], json!(0.0)); + assert_eq!(stalled["decodeTimeMs"], json!(2.4)); + assert_eq!(stalled["decoderResidenceMs"], json!(null)); + } + + #[test] + fn decode_timings_stay_unavailable_until_the_decoder_reports() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &sender, + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(75), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver + .try_recv() + .expect("decode telemetry with unavailable stages"); + assert_eq!(telemetry["decodeTimeMs"], json!(null)); + assert_eq!(telemetry["decoderResidenceMs"], json!(null)); + assert_eq!(telemetry["decodeTimings"], json!(null)); + assert_eq!(decode_timings_event(None), json!(null)); + } + + #[test] + fn frame_stage_timings_event_keeps_unmeasured_stages_null() { + assert_eq!(frame_stage_timings_event(None), json!(null)); + let timings = frame_stage_timings_event(Some(FrameStageTimings { + admission_to_control_queue: Some(opennow_streamer_transport::StageSummary { + p50_ms: 2.0, + p95_ms: 2.5, + max_ms: 3.0, + }), + ack_window_samples: 1, + ..Default::default() + })); + assert_eq!(timings["deliveryToAdmissionMs"], json!(null)); + assert_eq!(timings["assembledToControlQueueMs"], json!(null)); + assert_eq!(timings["admissionToControlQueueMs"]["p50"], json!(2.0)); + assert_eq!(timings["deliveryWindowSamples"], json!(0)); + assert_eq!(timings["assembledFramesTotal"], json!(0)); + assert_eq!(timings["pendingDeliveries"], json!(0)); + } + + #[test] + fn active_video_telemetry_requires_a_network_ping_sample() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); + let config = parse_nvst_video_handoff(&json!({ + "nvstVideo": { + "clientUdpPort": socket.local_addr().unwrap().port(), + "videoPeerIp": "127.0.0.1", + "videoPeerPort": peer.local_addr().unwrap().port(), + "srtpAesKeyHex": "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + "srtpSaltHex": "00000000000000009ECA935E", + "codec": "H264" + } + })) + .unwrap() + .unwrap(); + let feedback = config.feedback(); + let (media_sender, _media_receiver) = std::sync::mpsc::sync_channel(1); + let (event_sender, _event_receiver) = std::sync::mpsc::channel(); + let transport = spawn_nvst_udp_receiver_with_socket( + config, + media_sender, + event_sender, + Some(socket), + None, + Arc::new(HidRuntime::new()), + None, + ) + .unwrap(); + let resources = ActiveNvstResources { + bundle: transport.control(), + mjolnir: None, + feedback, + media: None, + }; + let (sender, receiver) = std::sync::mpsc::channel(); + let mut state = NvstMediaFeedbackState::new(true); + state.telemetry_window_started = Instant::now() - Duration::from_secs(1); + forward_nvst_media_feedback( + &EventSender::unbounded(sender), + &connected_lifecycle(), + 7, + &resources, + MediaFeedback::VideoFrameAccepted { + frame_index: Some(72), + timestamp: 90_000, + bytes: 125_000, + keyframe: false, + }, + &mut state, + ); + let telemetry = receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + transport.stop(); + assert_eq!(telemetry["type"], "telemetry"); + assert!(telemetry["framesPerSecond"].as_f64().unwrap() > 0.0); + assert_eq!(telemetry.get("pingMs"), Some(&Value::Null)); + } + + #[test] + fn captured_sdl_input_routes_through_the_nvst_input_codec_packet_shape() { + let resources = TestNvstResources::default(); + let state = NvstMediaFeedbackState::new(true); + + assert!( + forward_nvst_captured_input( + &resources, + CapturedInput::Key { + virtual_key: 0x57, + modifiers: 0x01, + pressed: true, + }, + &state, + ) + .is_ok() + ); + assert!( + forward_nvst_captured_input( + &resources, + CapturedInput::MouseMove { + delta_x: -12, + delta_y: 34, + }, + &state, + ) + .is_ok() + ); + assert!( + forward_nvst_captured_input( + &resources, + CapturedInput::MouseAbsolute { + x: 321, + y: 180, + width: 1280, + height: 720, + }, + &state, + ) + .is_ok() + ); + + let inputs = resources + .captured_inputs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inputs.len(), 3); + assert_eq!(u32::from_le_bytes(inputs[0][0..4].try_into().unwrap()), 3); + assert_eq!( + u16::from_be_bytes(inputs[0][4..6].try_into().unwrap()), + 0x57 + ); + assert_eq!( + u16::from_be_bytes(inputs[0][6..8].try_into().unwrap()), + 0x01 + ); + assert_eq!(inputs[0].len(), 18); + assert_eq!(u32::from_le_bytes(inputs[1][0..4].try_into().unwrap()), 7); + assert_eq!(i16::from_be_bytes(inputs[1][4..6].try_into().unwrap()), -12); + assert_eq!(i16::from_be_bytes(inputs[1][6..8].try_into().unwrap()), 34); + assert_eq!(inputs[1].len(), 22); + assert_eq!(u32::from_le_bytes(inputs[2][0..4].try_into().unwrap()), 5); + assert_eq!(u16::from_be_bytes(inputs[2][4..6].try_into().unwrap()), 321); + assert_eq!(u16::from_be_bytes(inputs[2][6..8].try_into().unwrap()), 180); + assert_eq!( + u16::from_be_bytes(inputs[2][10..12].try_into().unwrap()), + 1280 + ); + assert_eq!( + u16::from_be_bytes(inputs[2][12..14].try_into().unwrap()), + 720 + ); + assert_eq!(inputs[2].len(), 26); + } + + #[test] + fn captured_input_preserves_the_os_capture_timestamp() { + let resources = TestNvstResources::default(); + let input_origin = Instant::now(); + let mut state = NvstMediaFeedbackState::new(true); + state.input_origin = input_origin; + forward_nvst_captured_sample( + &resources, + &CapturedInputSample { + input: CapturedInput::MouseMove { + delta_x: 1, + delta_y: -1, + }, + captured_at: input_origin + Duration::from_micros(4_242), + }, + &state, + ) + .expect("captured input"); + + let inputs = resources + .captured_inputs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!( + u64::from_be_bytes(inputs[0][14..22].try_into().unwrap()), + 4_242 + ); + } + + #[test] + fn captured_input_readiness_races_preserve_order_and_release_without_stopping_media() { + let (output, received) = mpsc::channel(); + let output = EventSender::unbounded(output); + let lifecycle = Arc::new(connected_lifecycle()); + let worker_lifecycle = lifecycle.clone(); + let queue = Arc::new(CapturedInputQueue::default()); + let worker_queue = queue.clone(); + let (transport, nvst_events) = mpsc::channel(); + let (input_sent, inputs) = mpsc::channel(); + let input_not_ready = Arc::new(AtomicUsize::new(42)); + let remaining = input_not_ready.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &output, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "input-readiness".to_owned(), + nvst_events, + media_feedback: None, + captured_input: Some(worker_queue), + shortcut_runtime: None, + transport: TestNvstResources { + input_not_ready, + input_sent: Some(input_sent), + ..TestNvstResources::default() + }, + }, + ) + }); + transport.send(NvstReceiveEvent::InputReady(2)).unwrap(); + let event = received.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(event["type"], "input-ready"); + let captured_at = Instant::now(); + for pressed in [true, false] { + queue.push_sample(CapturedInputSample { + input: CapturedInput::Key { + virtual_key: 0x41, + modifiers: 0, + pressed, + }, + captured_at, + }); + } + let press = inputs.recv_timeout(Duration::from_secs(3)).unwrap(); + let release = inputs.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(u32::from_le_bytes(press[..4].try_into().unwrap()), 3); + assert_eq!(u32::from_le_bytes(release[..4].try_into().unwrap()), 4); + assert_eq!(press[10..18], release[10..18]); + assert_eq!(remaining.load(Ordering::Acquire), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + assert!(inputs.try_recv().is_err()); + assert!( + received + .try_iter() + .all(|event| event["type"] != "error" && event["type"] != "status") + ); + } + + #[test] + fn captured_input_closed_transport_still_stops_the_session() { + let (output, received) = mpsc::channel(); + let output = EventSender::unbounded(output); + let lifecycle = Arc::new(connected_lifecycle()); + let worker_lifecycle = lifecycle.clone(); + let queue = Arc::new(CapturedInputQueue::default()); + let worker_queue = queue.clone(); + let (transport, nvst_events) = mpsc::channel(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &output, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "input-closed".to_owned(), + nvst_events, + media_feedback: None, + captured_input: Some(worker_queue), + shortcut_runtime: None, + transport: TestNvstResources { + input_closed: true, + ..TestNvstResources::default() + }, + }, + ) + }); + transport.send(NvstReceiveEvent::InputReady(2)).unwrap(); + assert_eq!( + received.recv_timeout(Duration::from_secs(3)).unwrap()["type"], + "input-ready" + ); + queue.push(CapturedInput::Key { + virtual_key: 0x41, + modifiers: 0, + pressed: false, + }); + let error = received.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(error["type"], "error"); + assert_eq!(error["code"], "native-input-capture-failed"); + worker.join().unwrap(); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Idle); + } + + #[test] + fn captured_unicode_text_uses_typed_transport_without_key_expansion() { + use opennow_streamer_protocol::text_input::TextInputSlot; + let resources = TestNvstResources::default(); + let state = NvstMediaFeedbackState::new(true); + let text = TextInputSlot::default() + .submit("é世界🦫".as_bytes()) + .unwrap(); + forward_nvst_captured_sample( + &resources, + &CapturedInputSample { + input: CapturedInput::Text(text), + captured_at: state.input_origin, + }, + &state, + ) + .unwrap(); + assert_eq!( + *resources.captured_inputs.lock().unwrap(), + vec!["é世界🦫".as_bytes().to_vec()] + ); + } + + #[test] + fn captured_gamepad_matches_the_official_38_byte_packet() { + let packet = captured_input_packet( + CapturedInput::Gamepad { + controller_id: 2, + bitmap: 0x0404, + buttons: 0x5101, + left_trigger: 17, + right_trigger: 231, + left_stick_x: -12_345, + left_stick_y: 23_456, + right_stick_x: -30_000, + right_stick_y: 30_001, + }, + 0x0102_0304_0506_0708, + ); + + assert_eq!(packet.len(), 38); + assert_eq!(u32::from_le_bytes(packet[0..4].try_into().unwrap()), 12); + assert_eq!(u16::from_le_bytes(packet[4..6].try_into().unwrap()), 26); + assert_eq!(u16::from_le_bytes(packet[6..8].try_into().unwrap()), 2); + assert_eq!( + u16::from_le_bytes(packet[8..10].try_into().unwrap()), + 0x0404 + ); + assert_eq!(u16::from_le_bytes(packet[10..12].try_into().unwrap()), 20); + assert_eq!( + u16::from_le_bytes(packet[12..14].try_into().unwrap()), + 0x5101 + ); + assert_eq!( + u16::from_le_bytes(packet[14..16].try_into().unwrap()), + 0xe711 + ); + assert_eq!( + i16::from_le_bytes(packet[16..18].try_into().unwrap()), + -12_345 + ); + assert_eq!( + i16::from_le_bytes(packet[18..20].try_into().unwrap()), + 23_456 + ); + assert_eq!( + i16::from_le_bytes(packet[20..22].try_into().unwrap()), + -30_000 + ); + assert_eq!( + i16::from_le_bytes(packet[22..24].try_into().unwrap()), + 30_001 + ); + assert_eq!(u16::from_le_bytes(packet[26..28].try_into().unwrap()), 85); + assert_eq!( + u64::from_le_bytes(packet[30..38].try_into().unwrap()), + 0x0102_0304_0506_0708 + ); + assert!(captured_input_packet(CapturedInput::Guide, 1).is_empty()); + assert!(captured_input_packet(CapturedInput::Screenshot, 1).is_empty()); + assert!(captured_input_packet(CapturedInput::RecordingToggle, 1).is_empty()); + } + + #[test] + fn relative_mouse_tuning_matches_sensitivity_and_bounded_acceleration() { + assert_eq!( + tune_relative_mouse( + 20, + -10, + InputTuning { + sensitivity: 0.5, + acceleration_percent: 1.0, + }, + ), + (10, -5) + ); + let accelerated = tune_relative_mouse( + 100, + 0, + InputTuning { + sensitivity: 1.0, + acceleration_percent: 150.0, + }, + ); + assert_eq!(accelerated, (160, 0)); + assert_eq!( + tune_relative_mouse( + i16::MAX, + i16::MIN, + InputTuning { + sensitivity: 3.0, + acceleration_percent: 150.0, + }, + ), + (i16::MAX, i16::MIN) + ); + } + + #[test] + fn queue_drop_reports_do_not_mix_audio_samples_with_video_frames() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let mut reports = QueueDropReports::new(); + reports.record("audio-output", 96_000); + reports.record("linux-present", 47); + reports.flush(&sender, Instant::now(), true); + let reports: std::collections::HashMap<_, _> = receiver + .try_iter() + .map(|value| (value["media"].as_str().unwrap().to_owned(), value)) + .collect(); + assert_eq!(reports["audio-output"]["count"], 96_000); + assert_eq!(reports["audio-output"]["unit"], "samples"); + assert_eq!(reports["linux-present"]["count"], 47); + assert_eq!(reports["linux-present"]["unit"], "frames"); + } + + #[test] + fn queue_drop_feedback_flushes_periodically_without_another_drop() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + feedback_sender + .send(MediaFeedback::QueueDropped { + media: "video", + count: 5, + }) + .unwrap(); + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "test-session".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: TestNvstResources::default(), + }, + ) + }); + let report = receiver + .recv_timeout(Duration::from_secs(3)) + .expect("idle periodic flush"); + assert_eq!(report["event"], "queue-dropped"); + assert_eq!(report["count"], 5); + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + assert!(receiver.try_recv().is_err()); + drop(transport_sender); + } + + #[test] + fn decode_recovery_has_a_terminal_deadline_unless_output_resumes() { + for resumes in [false, true] { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(200), + keyframe_grace: Duration::from_millis(50), + recovery_grace: Duration::from_millis(300), + }), + ..Default::default() + }; + let recoveries = Arc::clone(&resources.recoveries); + let keyframes = Arc::clone(&resources.keyframe_requests); + let stalled_at = Instant::now() - Duration::from_secs(5); + let mut report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(stalled_at), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(stalled_at), + epoch: 1, + epoch_started_at: Some(stalled_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "decode-deadline".into(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + let deadline = Instant::now() + Duration::from_secs(2); + let mut messages = Vec::new(); + let mut resumed = false; + while Instant::now() < deadline { + if let Ok(message) = receiver.recv_timeout(Duration::from_millis(20)) { + let stopped = message["status"] == "stopped"; + messages.push(message); + if stopped { + break; + } + } + if resumes && !resumed && recoveries.load(Ordering::Relaxed) == 1 { + report.last_output_at = Some(Instant::now()); + report.outputs_total = 1; + report.in_flight = 0; + report.oldest_in_flight_at = None; + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + resumed = true; + } + } + let final_state = lock_lifecycle(&lifecycle).state; + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + drop(transport_sender); + assert_eq!(recoveries.load(Ordering::Relaxed), 1); + assert_eq!(keyframes.load(Ordering::Relaxed), 2); + if resumes { + assert!(resumed); + assert_eq!(final_state, State::Connected); + assert!( + !messages + .iter() + .any(|message| message["status"] == "stopped") + ); + } else { + assert_eq!(final_state, State::Idle); + assert_eq!( + messages + .iter() + .filter(|message| message["type"] == "error" + && message["code"] == "nvst-recovery-exhausted") + .count(), + 1 + ); + assert_eq!( + messages + .iter() + .filter(|message| message["status"] == "stopped") + .count(), + 1 + ); + } + } + } + + #[test] + fn decode_stall_escalates_from_keyframe_to_bounded_recovery() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let stalled_at = Instant::now() - Duration::from_secs(5); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(200), + keyframe_grace: Duration::from_millis(200), + recovery_grace: Duration::from_secs(8), + }), + frame_stage_timings: Some(FrameStageTimings { + last_assembled_at: Some(stalled_at), + assembled_frames_total: 12, + ..Default::default() + }), + ..Default::default() + }; + let keyframe_requests = Arc::clone(&resources.keyframe_requests); + let recoveries = Arc::clone(&resources.recoveries); + let report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 12, + outputs_total: 11, + output_calls_total: 11, + last_submission_at: Some(stalled_at), + last_output_at: Some(stalled_at), + in_flight: 1, + oldest_in_flight_at: Some(stalled_at), + epoch: 3, + epoch_started_at: Some(stalled_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + let feeder = thread::spawn(move || { + for _ in 0..20 { + thread::sleep(Duration::from_millis(200)); + if feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .is_err() + { + return; + } + } + }); + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "decode-stall".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + + let deadline = Instant::now() + Duration::from_secs(6); + let mut messages = Vec::new(); + let mut saw_stage = false; + while Instant::now() < deadline { + if let Ok(message) = receiver.recv_timeout(Duration::from_millis(200)) { + if message["type"] == "telemetry" + && message["decodeProgressStage"] == "recovery-required" + { + saw_stage = true; + } + messages.push(message); + if recoveries.load(Ordering::Relaxed) > 0 && saw_stage { + break; + } + } + } + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + drop(transport_sender); + let _ = feeder.join(); + + assert_eq!( + recoveries.load(Ordering::Relaxed), + 1, + "the decode stall must reach the bounded same-session recovery" + ); + assert_eq!( + keyframe_requests.load(Ordering::Relaxed), + 2, + "one keyframe from the stall stage and one from the recovery attempt" + ); + assert!( + messages.iter().any(|message| message["message"] + .as_str() + .is_some_and(|text| text.contains("Decoder produced no frame"))), + "the stall stage must be observable: {messages:?}" + ); + assert!( + messages.iter().any(|message| message["message"] + .as_str() + .is_some_and(|text| text.contains("Attempting bounded NVST recovery"))), + "the recovery stage must be observable: {messages:?}" + ); + assert!( + saw_stage, + "telemetry must publish the recovery-required decode stage: {messages:?}" + ); + } + + #[test] + fn transport_frame_progress_stall_owns_escalation_over_the_decode_stage() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(200), + keyframe_grace: Duration::from_millis(200), + recovery_grace: Duration::from_secs(8), + }), + ..Default::default() + }; + let keyframe_requests = Arc::clone(&resources.keyframe_requests); + let recoveries = Arc::clone(&resources.recoveries); + let stalled_at = Instant::now() - Duration::from_secs(5); + let report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(stalled_at), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(stalled_at), + epoch: 1, + epoch_started_at: Some(stalled_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + let feeder = thread::spawn(move || { + for _ in 0..20 { + thread::sleep(Duration::from_millis(200)); + if feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .is_err() + { + return; + } + } + }); + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "decode-owned".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + + transport_sender + .send(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::FrameProgress { + idle_for: Duration::from_secs(8), + last_assembled_frame_index: None, + }, + )) + .unwrap(); + thread::sleep(Duration::from_millis(1200)); + assert_eq!( + recoveries.load(Ordering::Relaxed), + 1, + "the transport stall owns the single recovery attempt" + ); + assert_eq!( + keyframe_requests.load(Ordering::Relaxed), + 1, + "only the transport recovery may request a keyframe, not the decode stage" + ); + let mut messages = Vec::new(); + while let Ok(message) = receiver.try_recv() { + messages.push(message); + } + assert!( + !messages.iter().any(|message| message["message"] + .as_str() + .is_some_and(|text| text.contains("Decoder produced no frame"))), + "downstream stall must stay silent while the transport owns it: {messages:?}" + ); + lock_lifecycle(&lifecycle).generation += 1; + let _ = transport_sender.send(NvstReceiveEvent::InputUnavailable("")); + worker.join().unwrap(); + let _ = feeder.join(); + } + + #[test] + fn transport_stall_ownership_reaches_telemetry_in_both_phases() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(200), + keyframe_grace: Duration::from_millis(200), + recovery_grace: Duration::from_secs(8), + }), + ..Default::default() + }; + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "stall-ownership".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + + transport_sender + .send(NvstReceiveEvent::FrameProgressStall { + idle_for: Duration::from_millis(800), + last_assembled_frame_index: Some(9), + recovery_required: false, + }) + .unwrap(); + let idle_at = Instant::now(); + let decode_report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 9, + outputs_total: 9, + output_calls_total: 9, + last_submission_at: Some(idle_at), + last_output_at: Some(idle_at), + in_flight: 0, + oldest_in_flight_at: None, + epoch: 1, + epoch_started_at: Some(idle_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + feedback_sender + .send(MediaFeedback::DecodeTimings(decode_report)) + .unwrap(); + let feeder = thread::spawn(move || { + for _ in 0..20 { + thread::sleep(Duration::from_millis(200)); + if feedback_sender + .send(MediaFeedback::DecodeTimings(decode_report)) + .is_err() + { + return; + } + } + }); + + let deadline = Instant::now() + Duration::from_secs(6); + let mut stalled = None; + let mut logged_stall = false; + while Instant::now() < deadline && stalled.is_none() { + if let Ok(message) = receiver.recv_timeout(Duration::from_millis(200)) { + if message["message"] + .as_str() + .is_some_and(|text| text.contains("Produced-frame stall")) + { + logged_stall = true; + } + if message["type"] == "telemetry" + && message["transportFrameProgressStalled"] + .as_bool() + .unwrap_or(false) + { + stalled = Some(true); + } + } + } + assert!( + logged_stall, + "the keyframe-pending phase must be observable" + ); + assert_eq!( + stalled, + Some(true), + "the keyframe-pending phase must reach the consumer boundary" + ); + + transport_sender + .send(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::FrameProgress { + idle_for: Duration::from_secs(2), + last_assembled_frame_index: Some(9), + }, + )) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(4); + let mut recovery_owned = false; + while Instant::now() < deadline && !recovery_owned { + if let Ok(message) = receiver.recv_timeout(Duration::from_millis(200)) + && message["type"] == "telemetry" + && message["transportFrameProgressStalled"] + .as_bool() + .unwrap_or(false) + { + recovery_owned = true; + } + } + assert!( + recovery_owned, + "the recovery phase must keep ownership visible to the consumer" + ); + + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + let _ = feeder.join(); + drop(transport_sender); + } + + #[test] + fn assembly_resumption_returns_ownership_to_the_decode_stage() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(200), + keyframe_grace: Duration::from_millis(200), + recovery_grace: Duration::from_secs(8), + }), + ..Default::default() + }; + let keyframe_requests = Arc::clone(&resources.keyframe_requests); + let recoveries = Arc::clone(&resources.recoveries); + let stalled_at = Instant::now() - Duration::from_secs(5); + let report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 7, + outputs_total: 6, + output_calls_total: 6, + last_submission_at: Some(stalled_at), + last_output_at: Some(stalled_at), + in_flight: 1, + oldest_in_flight_at: Some(stalled_at), + epoch: 2, + epoch_started_at: Some(stalled_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "assembly-resumed".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + + transport_sender + .send(NvstReceiveEvent::FrameProgressStall { + idle_for: Duration::from_secs(8), + last_assembled_frame_index: None, + recovery_required: false, + }) + .unwrap(); + thread::sleep(Duration::from_millis(300)); + assert_eq!( + recoveries.load(Ordering::Relaxed), + 0, + "the transport owns the stall while it is unresolved" + ); + + transport_sender + .send(NvstReceiveEvent::FrameProgressResumed) + .unwrap(); + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + let feeder = thread::spawn(move || { + for _ in 0..20 { + thread::sleep(Duration::from_millis(200)); + if feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .is_err() + { + return; + } + } + }); + + let deadline = Instant::now() + Duration::from_secs(5); + while Instant::now() < deadline && recoveries.load(Ordering::Relaxed) == 0 { + let _ = receiver.recv_timeout(Duration::from_millis(100)); + } + assert_eq!( + recoveries.load(Ordering::Relaxed), + 1, + "with assembly resumed and the decoder still outstanding, the decode stage \ + must regain ownership and recover" + ); + assert!( + keyframe_requests.load(Ordering::Relaxed) >= 1, + "the decode stall must request a fresh keyframe" + ); + + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + let _ = feeder.join(); + drop(transport_sender); + } + + #[test] + fn idle_decoder_without_outstanding_work_never_escalates() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (transport_sender, nvst_events) = std::sync::mpsc::channel(); + let resources = TestNvstResources { + decode_progress_policy: Some(DecodeProgressPolicy { + stall: Duration::from_millis(100), + keyframe_grace: Duration::from_millis(100), + recovery_grace: Duration::from_secs(8), + }), + ..Default::default() + }; + let keyframe_requests = Arc::clone(&resources.keyframe_requests); + let recoveries = Arc::clone(&resources.recoveries); + let idle_at = Instant::now() - Duration::from_secs(30); + let report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 4, + outputs_total: 4, + output_calls_total: 4, + last_submission_at: Some(idle_at), + last_output_at: Some(idle_at), + in_flight: 0, + oldest_in_flight_at: None, + epoch: 1, + epoch_started_at: Some(idle_at), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .unwrap(); + let feeder = thread::spawn(move || { + for _ in 0..20 { + thread::sleep(Duration::from_millis(200)); + if feedback_sender + .send(MediaFeedback::DecodeTimings(report)) + .is_err() + { + return; + } + } + }); + let worker_lifecycle = lifecycle.clone(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &sender, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "idle-decoder".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + let mut saw_tracking = false; + let deadline = Instant::now() + Duration::from_secs(4); + while Instant::now() < deadline && !saw_tracking { + if let Ok(message) = receiver.recv_timeout(Duration::from_millis(200)) + && message["type"] == "telemetry" + { + assert_eq!(message["decodeProgressStage"], "tracking"); + saw_tracking = true; + } + } + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + drop(transport_sender); + let _ = feeder.join(); + assert_eq!(keyframe_requests.load(Ordering::Relaxed), 0); + assert_eq!(recoveries.load(Ordering::Relaxed), 0); + assert!(saw_tracking, "idle decoder must still publish its stage"); + } + + #[test] + fn rumble_stop_is_retried_after_event_queue_backpressure() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + sender.send(json!({"type":"busy"})).unwrap(); + let output = EventSender::bounded(sender); + let lifecycle = Arc::new(connected_lifecycle()); + let resources = TestNvstResources::default(); + resources.rumble.lock().unwrap()[2] = Some(NvstControllerRumble { + controller_id: 2, + low_frequency: 0, + high_frequency: 0, + duration_ms: 1000, + source_incarnation: None, + }); + let pending = resources.rumble.clone(); + let worker_lifecycle = lifecycle.clone(); + let (_sender, nvst_events) = std::sync::mpsc::channel(); + let worker = thread::spawn(move || { + forward_nvst_session_events( + &output, + &worker_lifecycle, + 7, + NvstSessionEventResources { + start_id: "start-7".to_owned(), + nvst_events, + media_feedback: None, + captured_input: None, + shortcut_runtime: None, + transport: resources, + }, + ) + }); + let deadline = Instant::now() + Duration::from_secs(3); + while pending.lock().unwrap()[2].is_some() { + assert!(Instant::now() < deadline); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!( + receiver.recv_timeout(Duration::from_secs(3)).unwrap()["type"], + "busy" + ); + let stop = receiver.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(stop["type"], "controller-rumble"); + assert_eq!(stop["controllerId"], 2); + assert_eq!(stop["lowFrequency"], 0); + assert_eq!(stop["highFrequency"], 0); + lock_lifecycle(&lifecycle).generation += 1; + worker.join().unwrap(); + } + + #[test] + fn rumble_events_are_typed_session_scoped_and_nonblocking() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let output = EventSender::bounded(sender); + let lifecycle = connected_lifecycle(); + let command = opennow_streamer_transport::NvstControllerRumble { + controller_id: 3, + low_frequency: 65535, + high_frequency: 12345, + duration_ms: 65535, + source_incarnation: None, + }; + assert!(forward_controller_rumble( + &output, &lifecycle, 7, "start-7", command + )); + assert!(!forward_controller_rumble( + &output, &lifecycle, 7, "start-7", command + )); + assert_eq!( + receiver.try_recv().unwrap(), + json!({"type":"controller-rumble", + "startId":"start-7", "controllerId":3, "lowFrequency":65535, + "highFrequency":12345, "durationMs":65535}) + ); + assert!(forward_controller_rumble( + &output, + &lifecycle, + 6, + "old-start", + command + )); + lock_lifecycle(&lifecycle).state = State::Idle; + assert!(forward_controller_rumble( + &output, &lifecycle, 7, "start-7", command + )); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn sony_rumble_carries_the_source_incarnation_and_stays_session_scoped() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let output = EventSender::bounded(sender); + let lifecycle = connected_lifecycle(); + assert!(forward_controller_rumble( + &output, + &lifecycle, + 7, + "start-7", + opennow_streamer_transport::NvstControllerRumble { + controller_id: 1, + low_frequency: 0x4000, + high_frequency: 0x8000, + duration_ms: 0, + source_incarnation: Some(91), + } + )); + assert_eq!( + receiver.try_recv().expect("sony rumble event"), + json!({"type":"controller-rumble", + "startId":"start-7", "controllerId":1, "lowFrequency":0x4000, + "highFrequency":0x8000, "durationMs":0, "sourceIncarnation":91}) + ); + } + + #[test] + fn queue_drop_shutdown_preserves_invalidated_and_late_producer_feedback() { + let (sender, receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let output = engine.events.clone(); + let lifecycle = engine.lifecycle.clone(); + let (feedback_sender, feedback) = std::sync::mpsc::channel(); + let (_transport_sender, nvst_events) = std::sync::mpsc::channel(); + let (finished, completion) = std::sync::mpsc::channel(); + feedback_sender + .send(MediaFeedback::QueueDropped { + media: "video", + count: 3, + }) + .unwrap(); + engine.feedback_worker = Some(thread::spawn(move || { + let pending = forward_nvst_session_events( + &output, + &lifecycle, + u64::MAX, + NvstSessionEventResources { + start_id: "test-session".to_owned(), + nvst_events, + media_feedback: Some(feedback), + captured_input: None, + shortcut_runtime: None, + transport: TestNvstResources::default(), + }, + ); + finished.send(()).unwrap(); + pending + })); + completion.recv_timeout(Duration::from_secs(3)).unwrap(); + feedback_sender + .send(MediaFeedback::QueueDropped { + media: "video", + count: 9, + }) + .unwrap(); + engine.stop("test shutdown"); + let reports: Vec<_> = receiver + .try_iter() + .filter(|value| value["event"] == "queue-dropped") + .collect(); + assert_eq!(reports.len(), 2); + assert_eq!(reports[0]["count"], 3); + assert_eq!(reports[1]["count"], 9); + engine.stop("repeated shutdown"); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn nvst_recovery_is_attempted_once_with_pli() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + + let terminal = forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::RecoveryNeeded(opennow_streamer_transport::NvstRecovery::Timeout { + idle_for: Duration::from_secs(2), + }), + ); + + assert!(!terminal); + assert_eq!(recovery_attempts, 1); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 1); + assert_eq!( + *resources.recording_cuts.lock().unwrap(), + vec![RecordingCutReason::Interrupted] + ); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 1); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert!( + receiver + .try_iter() + .all(|message| message["type"] != "error") + ); + } + + #[test] + fn cursor_capture_output_survives_setup_before_running() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let sender = EventSender::bounded(sender); + let lifecycle = connected_lifecycle(); + lock_lifecycle(&lifecycle).state = State::Idle; + let mut state = NvstControlOutput { + start_id: "cursor-setup".to_owned(), + pending: Some(false), + pending_timer: None, + pending_text_error: None, + }; + sender.send(json!({ "type": "log" })).unwrap(); + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert_eq!(state.pending, Some(false)); + receiver.try_recv().unwrap(); + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert_eq!(state.pending, None); + assert_eq!(receiver.try_recv().unwrap()["composited"], false); + lock_lifecycle(&lifecycle).state = State::Connected; + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert!(receiver.try_recv().is_err()); + state.pending = Some(true); + lock_lifecycle(&lifecycle).context = None; + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert_eq!(state.pending, None); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn cursor_capture_output_retries_latest_state_after_backpressure() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let sender = EventSender::bounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + let mut state = NvstControlOutput { + start_id: "cursor-session".to_owned(), + pending: None, + pending_timer: None, + pending_text_error: None, + }; + sender.send(json!({ "type": "log" })).unwrap(); + for composited in [true, false] { + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut state, + None, + NvstReceiveEvent::CursorCapture(composited), + )); + assert_eq!(state.pending, Some(composited)); + } + assert_eq!(receiver.try_recv().unwrap()["type"], "log"); + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert_eq!(state.pending, None); + let message = receiver.try_recv().unwrap(); + assert_eq!(message["type"], "cursor-capture"); + assert_eq!(message["startId"], "cursor-session"); + assert_eq!(message["composited"], false); + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert!(receiver.try_recv().is_err()); + state.pending = Some(true); + lock_lifecycle(&lifecycle).generation += 1; + flush_control_output(&sender, &lifecycle, 7, &mut state); + assert_eq!(state.pending, None); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn cursor_capture_events_preserve_composition_across_reactivation() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + + for composited in [true, false, true, false] { + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::CursorCapture(composited), + )); + let message = receiver.try_recv().expect("cursor composition event"); + assert_eq!(message["type"], "cursor-capture"); + assert_eq!(message["composited"], composited); + } + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn repeated_packet_gaps_leave_recovery_to_the_receiver_without_stopping_the_session() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + + for first_missing_index in [100, 200] { + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { + first_missing_index, + last_missing_index: first_missing_index + 31, + }), + )); + } + + assert_eq!(recovery_attempts, 0); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); + assert!(resources.recording_cuts.lock().unwrap().is_empty()); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert!( + receiver + .try_iter() + .all(|message| message["type"] != "error") + ); + } + + #[test] + fn transient_media_backpressure_requests_keyframe_without_stopping_session() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::Dropped(NvstDropReason::MediaConsumerBackpressured), + )); + + assert_eq!(recovery_attempts, 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 1); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert!( + receiver + .try_iter() + .all(|message| message["type"] != "error" && message["type"] != "status") + ); + } + + #[test] + fn exhausted_nvst_recovery_stops_every_leg_and_emits_terminal_status() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + let recovery = || { + NvstReceiveEvent::RecoveryNeeded(opennow_streamer_transport::NvstRecovery::Timeout { + idle_for: Duration::from_secs(2), + }) + }; + + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + recovery(), + )); + assert!(forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + recovery(), + )); + + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 1); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 1); + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + let lifecycle = lock_lifecycle(&lifecycle); + assert_eq!(lifecycle.state, State::Idle); + assert!(lifecycle.context.is_none()); + drop(lifecycle); + let events = receiver.try_iter().collect::>(); + assert!(events.iter().any(|message| { + message["type"] == "error" && message["code"] == "nvst-recovery-exhausted" + })); + for message in &events { + if matches!(message["type"].as_str(), Some("error" | "status")) { + assert_eq!(message["termination"]["source"], "nvst-transport"); + assert_eq!(message["termination"]["code"], "nvst-recovery-exhausted"); + assert!(message["termination"]["resumable"].is_null()); + } + } + assert!( + events + .iter() + .any(|message| { message["type"] == "status" && message["status"] == "stopped" }) + ); + } + + #[test] + fn unexpected_receiver_stop_reports_its_cause_under_the_stable_code() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + + assert!(forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut 0, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped( + opennow_streamer_transport::NvstStopCause::InputDeliveryFailed + )), + )); + + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + let error = receiver + .try_iter() + .find(|message| message["type"] == "error") + .expect("terminal error"); + assert_eq!(error["code"], "nvst-transport-stopped"); + assert_eq!(error["termination"]["code"], "nvst-transport-stopped"); + assert_eq!( + error["message"], + "NVST receiver stopped unexpectedly (cause=input-delivery-failed)" + ); + } + + #[test] + fn input_unavailable_reason_reaches_the_native_log() { + let _sink = LOG_SINK_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let path = std::env::temp_dir().join(format!( + "opennow-input-unavailable-{}.log", + std::process::id() + )); + let _ = std::fs::remove_file(&path); + opennow_streamer_protocol::log::set_log_file(path.to_str().unwrap()).unwrap(); + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &TestNvstResources::default(), + &mut 0, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::InputUnavailable("reliable input data channel closed"), + )); + + assert_eq!(receiver.recv().unwrap()["type"], "input-unavailable"); + let log = std::fs::read_to_string(&path).unwrap(); + assert!( + log.lines().any(|line| line.ends_with( + "WARN nvst-input input-unavailable reason=reliable input data channel closed" + )), + "{log}" + ); + let _ = std::fs::remove_file(path); + } + + #[test] + fn media_consumer_closure_reports_the_preceding_decoder_error() { + let terminal_message = |decoder_error: Option| { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut state = NvstMediaFeedbackState::new(true); + if let Some(feedback) = decoder_error { + forward_nvst_media_feedback( + &sender, &lifecycle, 7, &resources, feedback, &mut state, + ); + } + assert!(forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut state.recovery_attempts, + &mut NvstControlOutput::default(), + state.last_video_decoder_error.as_deref(), + NvstReceiveEvent::Dropped(NvstDropReason::MediaConsumerClosed), + )); + receiver + .try_iter() + .find(|message| message["code"] == "media-consumer-closed") + .expect("terminal error")["message"] + .clone() + }; + + assert_eq!( + terminal_message(None), + "NVST receiver stopped because the decoded media path closed" + ); + assert_eq!( + terminal_message(Some(MediaFeedback::DecoderError { + codec: "H264", + message: format!("V4L2 dequeue failed (os error 32){}", "x".repeat(400)), + })), + format!( + "NVST receiver stopped because the decoded media path closed after H264 decoder error: V4L2 dequeue failed (os error 32){}", + "x".repeat(240 - "H264 decoder error: V4L2 dequeue failed (os error 32)".len()) + ) + ); + } + + #[test] + fn repeated_decode_stall_after_resumed_output_exhausts_session_recovery() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 0; + let mut watchdog = DecodeProgressWatchdog::default(); + let base = Instant::now(); + let policy = resources.decode_progress_policy(); + let mut report = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: 1, + outputs_total: 0, + output_calls_total: 0, + last_submission_at: Some(base), + last_output_at: None, + in_flight: 1, + oldest_in_flight_at: Some(base), + epoch: 1, + epoch_started_at: Some(base), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + + let mut episode_started_at = base; + for episode in 0..2 { + let stalled_at = episode_started_at + policy.stall; + assert!(matches!( + watchdog.poll(&report, false, None, stalled_at, policy), + Some(DecodeProgressEvent::KeyframeRequested { .. }) + )); + resources.request_keyframe(); + let recovery_at = stalled_at + policy.keyframe_grace; + assert!(matches!( + watchdog.poll(&report, false, None, recovery_at, policy), + Some(DecodeProgressEvent::RecoveryNeeded { .. }) + )); + assert_eq!( + attempt_nvst_recovery( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + "scripted decoder stall".to_owned(), + ), + episode == 1 + ); + if episode == 0 { + episode_started_at = recovery_at + Duration::from_secs(1); + report.last_output_at = Some(episode_started_at); + report.outputs_total = 1; + report.oldest_in_flight_at = Some(episode_started_at); + assert_eq!( + watchdog.poll(&report, false, None, episode_started_at, policy), + None + ); + } + } + + assert_eq!(recovery_attempts, 1); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 1); + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Idle); + assert!(lock_lifecycle(&lifecycle).context.is_none()); + assert!(receiver.try_iter().any(|message| { + message["type"] == "status" + && message["status"] == "stopped" + && message["termination"]["code"] == "nvst-recovery-exhausted" + })); + } + + #[test] + fn assembled_keyframe_does_not_reset_recovery_episode_budget() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut recovery_attempts = 1; + + assert!(!forward_nvst_event( + &sender, + &lifecycle, + 7, + &resources, + &mut recovery_attempts, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::Frame(opennow_streamer_transport::EncodedVideoAccessUnit { + codec: opennow_streamer_transport::NvstVideoCodec::H264, + timestamp: 1, + frame_index: 1, + first_stream_packet_index: 1, + frame_type: None, + keyframe: true, + contiguous: true, + bytes: vec![0, 0, 0, 1, 0x65], + }), + )); + assert_eq!(recovery_attempts, 1); + } + + #[test] + fn hello_reports_honest_transport_only_capabilities() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let command = command(json!({ + "id": "hello", + "type": "hello", + "protocolVersion": PROTOCOL_VERSION, + })); + let (responses, _) = engine.handle(command); + assert_eq!(responses[0]["type"], "ready"); + assert!( + responses[0]["capabilities"] + .get("supportsOfferAnswer") + .is_none() + ); + assert!( + responses[0]["capabilities"] + .get("supportsRemoteIce") + .is_none() + ); + assert_eq!(responses[0]["capabilities"]["supportsVideoPresent"], false); + } + + #[test] + fn accepted_color_profiles_are_not_silently_downgraded() { + let mut value = synthetic_context("accepted-color-profile", json!([])); + for (codec, color) in [ + ("H264", "8bit_444"), + ("H264", "10bit_420"), + ("H264", "10bit_444"), + ("AV1", "8bit_444"), + ("AV1", "10bit_444"), + (" av1 ", " 10bit_444 "), + ] { + value["session"]["negotiatedStreamProfile"] = json!({ + "codec": codec, "colorQuality": color, "enableHdr": false + }); + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!( + validate_context(&context, "invalid-color").is_err(), + "{codec} {color}" + ); + } + for (codec, color) in [ + ("H264", "8bit_420"), + ("H265", "8bit_444"), + ("H265", "10bit_444"), + ("AV1", "10bit_420"), + ] { + value["session"]["negotiatedStreamProfile"] = json!({ + "codec": codec, "colorQuality": color, "enableHdr": false + }); + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!( + validate_context(&context, "valid-color").is_ok(), + "{codec} {color}" + ); + } + } + + #[test] + fn unknown_accepted_color_is_not_replaced_by_local_settings() { + let mut value = synthetic_context("invalid-accepted-color", json!([])); + value["settings"]["colorQuality"] = json!("10bit_444"); + value["session"]["negotiatedStreamProfile"] = json!({ + "codec":"H265", "colorQuality":null, "bitDepthSource":"finalized" + }); + let context: SessionContext = serde_json::from_value(value).unwrap(); + assert!(validate_context(&context, "color").is_err()); + } + + #[test] + fn accepted_color_maps_to_native_decode_depth_and_chroma() { + for (color, expected) in [ + ("8bit_420", MediaColorQuality::EightBit420), + ("10bit_420", MediaColorQuality::TenBit420), + ("10bit_444", MediaColorQuality::TenBit444), + ] { + let mut value = synthetic_context("accepted-color", json!([])); + value["settings"]["colorQuality"] = json!("8bit_420"); + value["session"]["negotiatedStreamProfile"] = + json!({"codec":"H265","colorQuality":color}); + let context: SessionContext = serde_json::from_value(value).unwrap(); + assert!(validate_context(&context, "color").is_ok()); + assert_eq!(media_stream_config(&context).color_quality, expected); + } + } + + #[test] + fn media_hdr_uses_only_accepted_profile_including_sdr_fallback() { + let mut value = synthetic_context("hdr-media-config", json!([])); + value["settings"] = json!({"enableHdr":true,"codec":"H264","colorQuality":"8bit_420"}); + value["session"]["negotiatedStreamProfile"] = json!({ + "codec":"H265","colorQuality":"10bit_420","enableHdr":true + }); + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!(media_stream_config(&context).hdr); + assert_eq!( + media_stream_config(&context).color_quality, + MediaColorQuality::TenBit420 + ); + assert!(validate_context(&context, "hdr").is_ok()); + value["settings"]["enableHdr"] = json!(false); + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!(media_stream_config(&context).hdr); + value["settings"]["enableHdr"] = json!(true); + for accepted in [json!(false), Value::Null] { + value["session"]["negotiatedStreamProfile"]["enableHdr"] = accepted; + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!(!media_stream_config(&context).hdr); + } + value["session"]["negotiatedStreamProfile"]["enableHdr"] = json!(true); + for (codec, color) in [ + ("H264", "10bit_420"), + ("H265", "8bit_420"), + ("AV1", "10bit_444"), + ] { + value["session"]["negotiatedStreamProfile"]["codec"] = json!(codec); + value["session"]["negotiatedStreamProfile"]["colorQuality"] = json!(color); + let context: SessionContext = serde_json::from_value(value.clone()).unwrap(); + assert!(validate_context(&context, "invalid-hdr").is_err()); + } + } + + #[test] + fn accepted_hevc_hdr_444_preserves_bit_depth_chroma_and_hdr() { + for codec in ["H265", "HEVC"] { + let mut value = synthetic_context("hdr-444-media-config", json!([])); + value["settings"] = json!({"enableHdr": false, "colorQuality": "8bit_420"}); + value["session"]["negotiatedStreamProfile"] = json!({ + "codec": codec, "colorQuality": "10bit_444", "enableHdr": true + }); + let context: SessionContext = serde_json::from_value(value).unwrap(); + assert!(validate_context(&context, "hdr-444").is_ok()); + let stream = media_stream_config(&context); + assert_eq!(stream.codec, MediaVideoCodec::H265); + assert_eq!(stream.color_quality, MediaColorQuality::TenBit444); + assert!(stream.hdr); + } + } + + #[test] + fn microphone_commands_require_an_active_negotiated_session() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + for kind in ["microphone-set", "microphone-toggle"] { + let (responses, keep_running) = engine.handle(command(json!({ + "id":"mic", "type":kind, "enabled":true + }))); + assert!(keep_running); + assert_eq!(responses[0]["type"], "error"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + lock_lifecycle(&engine.lifecycle).state = State::Connected; + let (responses, _) = engine.handle(command(json!({ + "id":"mic", "type":"microphone-set", "enabled":true + }))); + assert_eq!(responses[0]["code"], "microphone-unavailable"); + assert_eq!(lifecycle_state(&engine), State::Connected); + } + + #[test] + fn microphone_capability_is_not_advertised_without_capture_runtime() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let (responses, _) = engine.handle(command(json!({ + "id":"hello", "type":"hello", "protocolVersion":PROTOCOL_VERSION + }))); + assert_eq!(responses[0]["capabilities"]["supportsMicrophone"], false); + assert!( + serde_json::from_value::(json!({ + "id":"mic", "type":"microphone-set", "enabled":"true" + })) + .is_err() + ); + } + + #[test] + fn muted_start_retains_microphone_capability_without_starting_capture() { + let (_host, runtime) = opennow_streamer_platform::create_test_runtime(); + let local_ip = + opennow_streamer_transport::advertised_nvst_ipv4().expect("local IPv4 route"); + let peer = UdpSocket::bind((local_ip, 0)).unwrap(); + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::with_media_runtime(sender, runtime); + let mut context = synthetic_context("muted-mic", json!([])); + context["settings"]["microphoneMode"] = json!("disabled"); + context["nvstVideo"] = json!({ + "clientUdpPort":unused_udp_port(), "videoPeerIp":local_ip.to_string(), + "videoPeerPort":peer.local_addr().unwrap().port(), + "srtpAesKeyHex":"000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + "srtpSaltHex":"00000000000000009ECA935E", "codec":"H264", + "microphoneOnBundle":true, "pingVersion":6, + "localIceUsernameFragment":"loc1", "localIcePassword":"local-password-value-01", + "remoteIceUsernameFragment":"remote01", "remoteIcePassword":"remote-password-value-01", + "remoteDtlsFingerprint":format!("sha-256 {}AA", "AA:".repeat(31)) + }); + let (responses, _) = engine.handle(command( + json!({"id":"start", "type":"start", "context":context}), + )); + assert_eq!(responses[0]["type"], "ok", "{responses:?}"); + assert_eq!(responses[0]["capabilities"]["supportsMicrophone"], true); + assert!(!engine.microphone.as_ref().unwrap().enabled()); + let (responses, _) = engine.handle(command( + json!({"id":"mic", "type":"microphone-set", "enabled":false}), + )); + assert_eq!(responses[0]["type"], "ok", "{responses:?}"); + engine.stop("test complete"); + } + + #[test] + fn derives_bounded_windows_media_configuration_from_stream_settings() { + let mut value = synthetic_context("media-config", json!([])); + value["settings"] = json!({ + "codec": "H264", + "resolution": "2560x1440", + "fps": 120, + "maxBitrateMbps": 75, + "enableCloudGsync": true, + "autoFullScreen": true + }); + value["session"]["negotiatedStreamProfile"] = json!({ + "enableCloudGsync": true + }); + let context: SessionContext = serde_json::from_value(value).expect("context"); + + assert_eq!( + media_stream_config(&context), + MediaStreamConfig { + codec: MediaVideoCodec::H264, + color_quality: MediaColorQuality::EightBit420, + hdr: false, + width: 2560, + height: 1440, + fps: 120, + bitrate_bps: 75_000_000, + cloud_gsync: true, + shortcuts: StreamShortcutBindings::default(), + } + ); + let mut low_rate = context.clone(); + low_rate.settings["maxBitrateMbps"] = json!(0.22); + assert_eq!(media_stream_config(&low_rate).bitrate_bps, 220_000); + + let fallback: SessionContext = + serde_json::from_value(synthetic_context("fallback-config", json!([]))) + .expect("context"); + assert_eq!(media_stream_config(&fallback), MediaStreamConfig::default()); + + let mut high_fps = synthetic_context("high-fps-config", json!([])); + high_fps["settings"] = json!({ + "codec": "H264", + "resolution": "1920x1080", + "fps": 360, + "maxBitrateMbps": 100 + }); + high_fps["session"]["negotiatedStreamProfile"] = json!({ + "codec": "AV1", + "fps": 400, + "colorQuality": "10bit_444" + }); + let high_fps: SessionContext = serde_json::from_value(high_fps).expect("context"); + assert_eq!(media_stream_config(&high_fps).codec, MediaVideoCodec::Av1); + assert_eq!(media_stream_config(&high_fps).fps, 360); + assert_eq!( + media_stream_config(&high_fps).color_quality, + MediaColorQuality::TenBit420 + ); + + let mut top_tier = synthetic_context("top-tier-config", json!([])); + top_tier["settings"] = json!({ + "codec": "H265", + "resolution": "1920x1080", + "fps": 360, + "maxBitrateMbps": 100 + }); + top_tier["session"]["negotiatedStreamProfile"] = json!({"fps": 360}); + let top_tier: SessionContext = serde_json::from_value(top_tier).expect("context"); + assert_eq!(media_stream_config(&top_tier).fps, 360); + + let mut rejected_vrr = synthetic_context("rejected-vrr-config", json!([])); + rejected_vrr["settings"] = json!({ "enableCloudGsync": true }); + rejected_vrr["session"]["negotiatedStreamProfile"] = json!({ + "enableCloudGsync": false + }); + let rejected_vrr: SessionContext = serde_json::from_value(rejected_vrr).expect("context"); + assert!(!media_stream_config(&rejected_vrr).cloud_gsync); + + let mut forced_vrr = synthetic_context("forced-vrr-config", json!([])); + forced_vrr["settings"] = json!({ + "enableCloudGsync": false, + "nativeCloudGsyncMode": "forced", + "showNativeStreamerStats": true, + "statsOverlayPosition": "top-right" + }); + forced_vrr["session"]["negotiatedStreamProfile"] = json!({ + "enableCloudGsync": true + }); + let forced_vrr: SessionContext = serde_json::from_value(forced_vrr).expect("context"); + let forced_config = media_stream_config(&forced_vrr); + assert!(forced_config.cloud_gsync); + + let mut legacy_overlay = synthetic_context("legacy-overlay-config", json!([])); + legacy_overlay["settings"] = json!({ + "showNativeStreamerStats": true, + "showStatsOnLaunch": true, + "statsOverlayPosition": "bottom-left", + "autoFullScreen": true + }); + let legacy_overlay: SessionContext = + serde_json::from_value(legacy_overlay).expect("context"); + assert_eq!( + media_stream_config(&legacy_overlay), + MediaStreamConfig::default() + ); + } + + #[test] + fn valid_nvst_handoff_starts_udp_video_and_rejects_removed_offer_command() { + let (sender, receiver) = std::sync::mpsc::channel(); + let (media_sender, _media_receiver) = std::sync::mpsc::sync_channel(4); + let mut engine = Engine::with_media_consumer(sender, media_sender); + let mut context = synthetic_context("nvst-session", json!([])); + context["settings"]["codec"] = json!("AV1"); + context["nvstVideo"] = json!({ + "clientUdpPort": unused_udp_port(), + "videoPeerIp": "127.0.0.1", + "videoPeerPort": 5004, + "srtpAesKeyHex": "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + "srtpSaltHex": "00000000000000009ECA935E", + "codec": "H264" + }); + let (responses, _) = engine.handle(command(json!({ + "id": "start", + "type": "start", + "context": context.clone(), + }))); + + assert_eq!(responses[0]["type"], "ok"); + assert_eq!(responses[0]["transport"], "nvst"); + assert!( + responses[0]["capabilities"] + .get("supportsOfferAnswer") + .is_none() + ); + assert!( + responses[0]["capabilities"] + .get("supportsRemoteIce") + .is_none() + ); + assert_eq!(responses[0]["capabilities"]["supportsInput"], false); + assert_eq!(responses[0]["capabilities"]["supportsAudioDecode"], false); + assert_eq!(lifecycle_state(&engine), State::Connected); + assert!(engine.nvst_transport.is_some()); + assert!(receiver.try_iter().any(|message| { + message["type"] == "status" + && message["message"] + .as_str() + .is_some_and(|text| text.contains("NVST")) + })); + + let (responses, _) = engine.handle(command(json!({ + "id": "offer", + "type": "offer", + "context": context, + }))); + assert_eq!(responses[0]["code"], "unknown-command"); + + let (responses, _) = engine.handle(command(json!({ + "id": "stop", + "type": "stop", + "reason": "test complete", + }))); + assert_eq!(responses[0]["type"], "ok"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + + #[test] + fn accepted_start_binds_the_hid_endpoint_and_termination_closes_it() { + // Hold a live peer for the whole test. On Windows, sending to a closed + // UDP port makes the next receive fail with WSAECONNRESET. That exits + // the bundle thread, which unbinds HID before the assertion below can + // observe the binding start() just installed. + let peer = UdpSocket::bind("127.0.0.1:0").expect("HID test peer socket"); + let peer_port = peer.local_addr().expect("HID test peer address").port(); + let (sender, _receiver) = std::sync::mpsc::channel(); + let (media_sender, _media_receiver) = std::sync::mpsc::sync_channel(4); + let mut engine = Engine::with_media_consumer(sender, media_sender); + let mut context = synthetic_context("hid-endpoint-lifecycle", json!([])); + context["settings"]["codec"] = json!("AV1"); + context["nvstVideo"] = json!({ + "clientUdpPort": unused_udp_port(), + "videoPeerIp": "127.0.0.1", + "videoPeerPort": peer_port, + "srtpAesKeyHex": "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + "srtpSaltHex": "00000000000000009ECA935E", + "codec": "H264" + }); + let (responses, _) = engine.handle(command(json!({ + "id": "start", + "type": "start", + "context": context, + }))); + assert_eq!(responses[0]["type"], "ok"); + let generation = lock_lifecycle(&engine.lifecycle).generation; + assert_eq!(engine.hid_runtime.session_generation(), Some(generation)); + assert!( + engine + .hid_runtime + .bind_session(generation.wrapping_add(1)) + .is_some() + ); + engine + .hid_runtime + .unbind_session(generation.wrapping_add(1)); + assert_eq!(engine.hid_runtime.session_generation(), None); + + if let Some(transport) = engine.nvst_transport.take() { + transport.stop(); + } + assert_eq!( + engine.hid_runtime.session_generation(), + None, + "terminating the owned transport must close the HID endpoint" + ); + assert!( + engine.hid_runtime.bind_session(9_999).is_none(), + "a closed endpoint must refuse late binding" + ); + + let (responses, _) = engine.handle(command(json!({ + "id": "stop", + "type": "stop", + "reason": "test complete", + }))); + assert_eq!(responses[0]["type"], "ok"); + assert_eq!(lifecycle_state(&engine), State::Idle); + assert!(engine.hid_runtime.bind_session(10_000).is_none()); + } + + #[test] + fn explicit_invalid_nvst_handoff_fails_closed() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let mut context = synthetic_context("invalid-nvst-session", json!([])); + context["nvstVideo"] = json!({ + "clientUdpPort": 0, + "codec": "H264" + }); + + let (responses, _) = engine.handle(command(json!({ + "id": "start-invalid-nvst", + "type": "start", + "context": context, + }))); + + assert_eq!(responses[0]["code"], "invalid-nvst-handoff"); + assert_eq!(lifecycle_state(&engine), State::Idle); + assert!(engine.nvst_transport.is_none()); + assert!(engine.hid_runtime.bind_session(1).is_none()); + assert_eq!(engine.hid_runtime.session_generation(), None); + } + + #[test] + fn explicit_nvst_mode_without_endpoint_fails_closed() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let mut context = synthetic_context("missing-nvst-session", json!([])); + context["settings"]["transportMode"] = json!("nvst"); + + let (responses, _) = engine.handle(command(json!({ + "id": "start-missing-nvst", + "type": "start", + "context": context, + }))); + + assert_eq!(responses[0]["code"], "missing-rtsps-endpoint"); + assert_eq!(lifecycle_state(&engine), State::Idle); + assert!(engine.nvst_transport.is_none()); + } + + #[test] + fn unused_nvst_reservation_can_be_released_idempotently() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + + let (responses, _) = engine.handle(command(json!({ + "id": "bind", + "type": "nvst-bind", + }))); + assert_eq!(responses[0]["type"], "nvst-bound"); + assert!(engine.reserved_nvst_bundle.is_some()); + + for id in ["unbind", "unbind-again"] { + let (responses, _) = engine.handle(command(json!({ + "id": id, + "type": "nvst-unbind", + }))); + assert_eq!(responses[0]["type"], "ok"); + assert!(engine.reserved_nvst_bundle.is_none()); + } + } + + #[test] + fn start_rejects_invalid_contexts_and_missing_nvst_handoffs() { + let (sender, _receiver) = std::sync::mpsc::channel(); + let mut engine = Engine::new(sender); + let invalid = command(json!({ + "id": "invalid", + "type": "start", + "context": { + "session": { "sessionId": "", "serverIp": "host", "iceServers": [] }, + "settings": {}, + "shortcuts": {} + } + })); + let (responses, _) = engine.handle(invalid); + assert_eq!(responses[0]["code"], "invalid-context"); + assert_eq!(lifecycle_state(&engine), State::Idle); + + let (responses, _) = engine.handle(command(json!({ + "id": "missing-nvst", + "type": "start", + "context": synthetic_context("synthetic-session", json!([])), + }))); + assert_eq!(responses[0]["code"], "nvst-handoff-required"); + assert_eq!(lifecycle_state(&engine), State::Idle); + } + + #[test] + fn derives_initial_media_dimensions_from_session_settings() { + assert_eq!( + media_stream_config( + &serde_json::from_value(json!({ + "session": { + "sessionId": "test", + "serverIp": "127.0.0.1", + "negotiatedStreamProfile": { "resolution": "3840x2160" } + }, + "settings": { "resolution": "1920x1080" }, + "shortcuts": {} + })) + .expect("context") + ), + MediaStreamConfig { + width: 3840, + height: 2160, + ..MediaStreamConfig::default() + } + ); + assert_eq!( + media_stream_config( + &serde_json::from_value(json!({ + "session": { "sessionId": "test", "serverIp": "127.0.0.1" }, + "settings": { "resolution": "invalid" }, + "shortcuts": {} + })) + .expect("context") + ), + MediaStreamConfig::default() + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/microphone.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/microphone.rs new file mode 100644 index 000000000..80c1de5f3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/microphone.rs @@ -0,0 +1,262 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use opennow_streamer_platform::{MediaRuntime, MicrophoneReceiver, MicrophoneSession}; +use opennow_streamer_protocol::event; +use opennow_streamer_transport::NvstUdpReceiverControl; +use serde_json::json; + +use crate::{EventSender, Lifecycle, State, lock_lifecycle}; + +pub(super) struct MicrophoneController { + runtime: MediaRuntime, + device_id: String, + events: EventSender, + lifecycle: Arc>, + generation: u64, + enabled: Arc, + transport: NvstUdpReceiverControl, + capture: Option, + worker: Option>, +} + +impl MicrophoneController { + pub fn new( + runtime: MediaRuntime, + transport: NvstUdpReceiverControl, + device_id: String, + events: EventSender, + lifecycle: Arc>, + generation: u64, + ) -> Self { + Self { + runtime, + device_id, + events, + lifecycle, + generation, + enabled: Arc::new(AtomicBool::new(false)), + transport, + capture: None, + worker: None, + } + } + + pub fn enabled(&self) -> bool { + self.enabled.load(Ordering::Acquire) + } + + pub fn set_enabled(&mut self, enabled: bool) -> Result<(), String> { + if enabled && self.enabled() { + return Ok(()); + } + self.stop_capture(); + if !enabled { + publish( + &self.events, + &self.lifecycle, + self.generation, + "muted", + false, + None, + ); + return Ok(()); + } + let result = self.start_capture(); + if let Err(message) = &result { + self.stop_capture(); + publish( + &self.events, + &self.lifecycle, + self.generation, + "error", + false, + Some(message), + ); + } + result + } + + fn start_capture(&mut self) -> Result<(), String> { + let capture = self.runtime.start_microphone(&self.device_id)?; + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation != self.generation || lifecycle.state != State::Connected { + return Err("Microphone session ended during capture setup".to_owned()); + } + self.transport + .set_microphone_enabled(true) + .map_err(|error| error.to_string())?; + self.enabled.store(true, Ordering::Release); + let receiver = capture.receiver(); + self.capture = Some(capture); + let pump = MicrophonePump { + transport: self.transport.clone(), + events: self.events.clone(), + lifecycle: self.lifecycle.clone(), + generation: self.generation, + enabled: self.enabled.clone(), + receiver, + }; + self.worker = Some( + thread::Builder::new() + .name("opennow-microphone-uplink".to_owned()) + .spawn(move || pump.run()) + .map_err(|error| format!("Microphone worker could not start: {error}"))?, + ); + let _ = self.events.send(event( + "microphone-state", + json!({"state":"ready", "enabled":true}), + )); + drop(lifecycle); + Ok(()) + } + + fn stop_capture(&mut self) { + self.enabled.store(false, Ordering::Release); + let _ = self.transport.set_microphone_enabled(false); + if let Some(mut capture) = self.capture.take() { + capture.stop(); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl Drop for MicrophoneController { + fn drop(&mut self) { + self.stop_capture(); + } +} + +fn publish( + events: &EventSender, + lifecycle: &Mutex, + generation: u64, + state: &str, + enabled: bool, + message: Option<&str>, +) { + let lifecycle = lock_lifecycle(lifecycle); + if lifecycle.generation == generation && lifecycle.state == State::Connected { + let _ = events.send(event( + "microphone-state", + json!({"state":state, "enabled":enabled, "message":message}), + )); + } +} + +struct MicrophonePump { + transport: NvstUdpReceiverControl, + events: EventSender, + lifecycle: Arc>, + generation: u64, + enabled: Arc, + receiver: MicrophoneReceiver, +} + +impl MicrophonePump { + fn active(&self) -> bool { + let lifecycle = lock_lifecycle(&self.lifecycle); + self.enabled.load(Ordering::Acquire) + && lifecycle.generation == self.generation + && lifecycle.state == State::Connected + } + + fn run(self) { + let mut last_drops = (0, 0, 0); + let mut last_report = Instant::now(); + while self.active() { + let mut failure = None; + for _ in 0..5 { + match self.receiver.try_recv() { + Ok(Some(frame)) => { + if let Err(error) = self + .transport + .send_microphone_opus(frame.opus, frame.timestamp) + { + failure = Some(error.to_string()); + break; + } + } + Ok(None) => break, + Err(error) => { + failure = Some(error); + break; + } + } + } + let status = self.receiver.status(); + if !status.enabled && failure.is_none() { + failure = Some( + status + .error + .unwrap_or_else(|| "Microphone capture stopped".to_owned()), + ); + } + if let Some(message) = failure { + if self.enabled.swap(false, Ordering::AcqRel) { + publish( + &self.events, + &self.lifecycle, + self.generation, + "error", + false, + Some(&message), + ); + } + break; + } + if last_report.elapsed() >= Duration::from_secs(1) { + let drops = ( + status.dropped_pcm_frames, + status.dropped_encoded_frames, + self.transport.microphone_dropped_frames(), + ); + if drops != last_drops { + opennow_streamer_protocol::log::log_line( + "WARN", + "microphone", + &format!( + "dropped_pcm={} dropped_encoded={} dropped_transport={}", + drops.0, drops.1, drops.2 + ), + ); + last_drops = drops; + } + last_report = Instant::now(); + } + thread::sleep(Duration::from_millis(5)); + } + self.enabled.store(false, Ordering::Release); + self.receiver.stop(); + let _ = self.transport.set_microphone_enabled(false); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn microphone_status_cannot_escape_its_session_generation() { + let (sender, receiver) = std::sync::mpsc::channel(); + let events = EventSender::unbounded(sender); + let lifecycle = Mutex::new(Lifecycle { + state: State::Connected, + context: None, + generation: 2, + }); + publish(&events, &lifecycle, 1, "ready", true, None); + assert!(receiver.try_recv().is_err()); + publish(&events, &lifecycle, 2, "ready", true, None); + let value = receiver.try_recv().unwrap(); + assert_eq!(value["type"], "microphone-state"); + assert_eq!(value["enabled"], true); + lock_lifecycle(&lifecycle).state = State::Idle; + publish(&events, &lifecycle, 2, "error", false, Some("device lost")); + assert!(receiver.try_recv().is_err()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp.rs new file mode 100644 index 000000000..963d10a47 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp.rs @@ -0,0 +1,3921 @@ +use std::collections::HashMap; +use std::io::ErrorKind; +use std::net::{IpAddr, TcpStream}; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use opennow_streamer_platform::MediaStreamConfig; +use opennow_streamer_protocol::SessionContext; +use opennow_streamer_transport::nvst::{ + MAX_CONTROL_REPORT_BYTES, MAX_NVST_VIDEO_PEER_PORTS, MIN_CONTROL_REPORT_BYTES, +}; +use opennow_streamer_transport::{ReservedNvstBundle, nvst_video_packet_size}; +use serde_json::{Value, json}; +use tungstenite::client::IntoClientRequest; +use tungstenite::http::{HeaderValue, Uri}; +use tungstenite::stream::MaybeTlsStream; +use tungstenite::{Message, WebSocket, connect}; + +#[path = "nvst_rtsp_color.rs"] +mod color; +#[path = "nvst_rtsp_negotiation.rs"] +mod negotiation; +#[path = "nvst_rtsp_transport_diagnostics.rs"] +mod transport_diagnostics; +use color::announce_color_lines; +use negotiation::{microphone_handoff, negotiated_announce, offered_bool, recovery_policy}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); +// A rig whose video streamer is still starting answers SETUP with 200 but no +// Transport peer yet. Re-sweep on a bounded pace then instead of failing in +// under a second; pure rejections still fail immediately. Worst case adds 9s +// inside the shared 20s budget above. +const SETUP_PEER_RETRY_ROUNDS: u32 = 3; +const SETUP_PEER_RETRY_DELAY: Duration = Duration::from_secs(3); +const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(2); +#[cfg(test)] +const CONTROL_PING_EXPIRY: Duration = Duration::from_secs(5); +const CONTROL_IO_TIMEOUT: Duration = Duration::from_millis(100); +const MAX_REQUEST_RESPONSE_BYTES: usize = 4 * 1024 * 1024; +const MAX_CONTROL_RESPONSE_BYTES: usize = 64 * 1024; +const MAX_STREAM_BITRATE_MBPS: u64 = 200; +// GeForce NOW 2.0.87.131 reports video[0].timeoutLengthMs=8000 and +// video[0].sendFrameTimeoutMs=7000. Waiting sixty seconds left a dead Mjolnir media leg on screen +// while audio/control remained alive; use the official receiver timeout so the existing bounded +// transport recovery runs promptly. +pub(crate) const VIDEO_TIMEOUT_MS: u64 = 8_000; +const VIDEO_STARTUP_TIMEOUT_MS: u64 = if cfg!(windows) { + 60_000 +} else { + VIDEO_TIMEOUT_MS +}; + +#[derive(Debug)] +pub struct NvstRtspError { + pub code: &'static str, + pub message: String, +} + +impl NvstRtspError { + fn new(code: &'static str, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } +} + +pub(crate) fn wait_authenticated_transport( + mut ready: impl FnMut() -> Result, + timeout: Duration, +) -> Result<(), NvstRtspError> { + let deadline = Instant::now() + timeout; + loop { + if ready()? { + return Ok(()); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(NvstRtspError::new( + "nvst-transport-ready-timeout", + "NVST transport did not authenticate before the startup deadline", + )); + } + thread::sleep(remaining.min(Duration::from_millis(10))); + } +} + +struct RtspResponse { + status: u16, + status_text: String, + headers: HashMap, + body: String, +} + +struct SweepFailure { + error: NvstRtspError, + peerless_200: Option>, +} + +struct RtspClient { + socket: WebSocket>, + cseq: u64, + buffer: String, +} + +struct VideoSetup { + response: RtspResponse, + peer: (String, u16, u16), + bundle_fallback: bool, +} + +#[derive(Clone, Copy)] +struct BundleEndpoint<'a> { + ip: &'a str, + port: u32, + source: &'static str, +} + +/// Resolves the ICE/DTLS bundle peer the way GFN's `getBundlePortNumber` does: the +/// CloudMatch BUNDLE connection (usage 17) wins over the VIDEO one (usage 2), which +/// alliance seats publish as a separate NAT mapping. +fn bundle_endpoint(context: &SessionContext) -> Option> { + context + .session + .connection_info + .iter() + .flatten() + .find_map(|connection| { + let ip = connection.ip.as_deref()?; + (connection.usage == 17 + && (1..=65535).contains(&connection.port) + && ip.parse::().is_ok()) + .then_some(BundleEndpoint { + ip, + port: connection.port, + source: "cloudmatch-bundle", + }) + }) + .or_else(|| { + context + .session + .media_connection_info + .as_ref() + .map(|media| BundleEndpoint { + ip: &media.ip, + port: media.port, + source: "cloudmatch-media", + }) + }) +} + +fn connection_usage_summary(context: &SessionContext) -> String { + context + .session + .connection_info + .iter() + .flatten() + .map(|connection| format!("{}:{}", connection.usage, connection.port)) + .collect::>() + .join(",") +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum QosVersionOffer { + #[default] + Missing, + Malformed, + Version(u8), +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct VideoQosOffers { + feedback: QosVersionOffer, + timings: QosVersionOffer, + blob_stats: QosVersionOffer, +} + +impl VideoQosOffers { + fn uses_v5_timings(&self) -> bool { + matches!(self.timings, QosVersionOffer::Version(5..)) + && matches!(self.blob_stats, QosVersionOffer::Version(9..)) + } + + fn validate(&self) -> Result<(), NvstRtspError> { + for (offer, minimum) in [(self.feedback, 7), (self.timings, 5), (self.blob_stats, 9)] { + match offer { + QosVersionOffer::Malformed => { + return Err(NvstRtspError::new( + "nvst-qos-version-invalid", + "Server advertised a malformed QoS version", + )); + } + QosVersionOffer::Version(version) if version < minimum => { + return Err(NvstRtspError::new( + "nvst-qos-version-unsupported", + "Server requires an older QoS wire format that is not implemented", + )); + } + _ => {} + } + } + Ok(()) + } + + fn pacing_feedback_mode(&self, sdp: &str) -> Result { + match sdp_attribute(sdp, "video[0].framePacing.feedbackMode").as_deref() { + Some("0") => Ok(0), + Some("1") => Ok(1), + None => Ok(u8::from(!self.uses_v5_timings())), + _ => Err(NvstRtspError::new( + "nvst-qos-pacing-unsupported", + "Server advertised an unsupported frame pacing feedback mode", + )), + } + } + + fn record(&mut self, parameter: &str) { + let parameter = parameter.trim(); + let parameter = parameter.strip_prefix("a=").unwrap_or(parameter); + let (name, raw_version) = parameter.split_once(['=', ':']).unwrap_or((parameter, "")); + let offer = match name.trim() { + name if name.eq_ignore_ascii_case("nv-video-qos-feedback-version") => { + &mut self.feedback + } + name if name.eq_ignore_ascii_case("nv-video-qos-timings-version") => &mut self.timings, + name if name.eq_ignore_ascii_case("nv-video-qos-blob-stats-version") => { + &mut self.blob_stats + } + _ => return, + }; + let raw_version = raw_version.trim(); + *offer = if matches!(offer, QosVersionOffer::Missing) + && !raw_version.is_empty() + && raw_version.bytes().all(|byte| byte.is_ascii_digit()) + { + raw_version + .parse::() + .map_or(QosVersionOffer::Malformed, QosVersionOffer::Version) + } else { + QosVersionOffer::Malformed + }; + } + + fn add_to_handoff(&self, handoff: &mut Value) { + if let QosVersionOffer::Version(version) = self.feedback { + handoff["qosFeedbackVersion"] = json!(version); + } + if let QosVersionOffer::Version(version) = self.timings { + handoff["qosTimingsVersion"] = json!(version); + } + if let QosVersionOffer::Version(version) = self.blob_stats { + handoff["qosBlobStatsVersion"] = json!(version); + } + } +} + +#[derive(Clone, Default)] +struct NvstControlPing { + sample: Arc>>, +} + +impl NvstControlPing { + #[cfg(test)] + fn ping_ms(&self, now: Instant) -> Option { + let mut sample = self.sample.lock().ok()?; + let (received_at, elapsed) = (*sample)?; + if now.checked_duration_since(received_at)? >= CONTROL_PING_EXPIRY { + *sample = None; + return None; + } + Some(elapsed.as_secs_f64() * 1000.0) + } + + fn record(&self, sent_at: Instant, received_at: Instant) { + if let Ok(mut sample) = self.sample.lock() { + *sample = received_at + .checked_duration_since(sent_at) + .map(|elapsed| (received_at, elapsed)); + } + } + + fn clear(&self) { + if let Ok(mut sample) = self.sample.lock() { + *sample = None; + } + } +} + +impl RtspClient { + #[allow(clippy::too_many_arguments)] + fn setup_video( + &mut self, + control: &str, + target: &str, + headers: &[(&str, String)], + client_port: u16, + bundle_video_peer: Option<(String, u16, u16)>, + ) -> Result { + self.setup_video_with_retry( + control, + target, + headers, + client_port, + SETUP_PEER_RETRY_ROUNDS, + SETUP_PEER_RETRY_DELAY, + bundle_video_peer, + ) + } + + #[allow(clippy::too_many_arguments)] + fn setup_video_with_retry( + &mut self, + control: &str, + target: &str, + headers: &[(&str, String)], + client_port: u16, + max_peer_retries: u32, + peer_retry_delay: Duration, + bundle_video_peer: Option<(String, u16, u16)>, + ) -> Result { + // A rig whose video streamer is still starting answers SETUP with 200 + // but no Transport peer yet. Re-sweep on a bounded pace then: the + // official client negotiates through progress callbacks instead of + // one fast burst. Pure rejections (400/404/459+) mean the forms are + // wrong for this server, so those still fail immediately. + let candidates = video_setup_candidates(control, target); + let deadline = Instant::now() + REQUEST_TIMEOUT; + let mut headers = headers.to_vec(); + headers.push(("Transport", String::new())); + let transport_index = headers.len() - 1; + let mut round = 0u32; + let mut peerless_200_seen = false; + loop { + match self.setup_video_sweep( + &candidates, + &mut headers, + transport_index, + client_port, + &deadline, + ) { + Ok(setup) => return Ok(setup), + Err(failure) => { + if let Some(response) = failure.peerless_200 { + peerless_200_seen = true; + if let Some(peer) = bundle_video_peer.clone() { + opennow_streamer_protocol::log::log_line( + "WARN", + "rtsps", + "video-peer-fallback source=cloudmatch-bundle", + ); + return Ok(VideoSetup { + response: *response, + peer, + bundle_fallback: true, + }); + } + } + let mut error = failure.error; + if error.code != "missing-video-peer" { + if error.code == "nvst-rtsp-failed" && peerless_200_seen { + error.code = "missing-video-peer"; + } else { + return Err(error); + } + } + if round >= max_peer_retries { + return Err(error); + } + round += 1; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(error); + } + let sleep_for = peer_retry_delay.min(remaining); + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "video-setup-retry round={round}/{max_peer_retries} sleep_ms={}", + sleep_for.as_millis(), + ), + ); + std::thread::sleep(sleep_for); + } + } + } + } + + #[allow(clippy::too_many_arguments)] + fn setup_video_sweep( + &mut self, + candidates: &[String], + headers: &mut [(&str, String)], + transport_index: usize, + client_port: u16, + deadline: &Instant, + ) -> Result { + let mut last_status = 0; + for transport in [ + String::new(), + format!( + "unicast;X-GS-ClientPort={client_port}-{}", + client_port.saturating_add(1) + ), + ] { + let transport_form = if transport.is_empty() { + "empty" + } else { + "client-udp" + }; + headers[transport_index].1 = transport; + for (index, candidate) in candidates.iter().enumerate() { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(SweepFailure { + error: NvstRtspError::new( + "nvst-rtsp-timeout", + "RTSPS video SETUP timed out", + ), + peerless_200: None, + }); + } + let response = self + .request_with_timeout("SETUP", candidate, headers, "", remaining) + .map_err(|error| SweepFailure { + error, + peerless_200: None, + })?; + let transport = header_value(&response, "transport"); + let peer = transport + .and_then(parse_video_peer) + .filter(|(ip, _, _)| ip.parse::().is_ok()); + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "video-setup candidate={}/{} transport_form={transport_form} status={} transport_present={} video_peer_valid={} ping_version_present={} ping_payload_present={}", + index + 1, + candidates.len(), + response.status, + transport.is_some(), + peer.is_some(), + header_value(&response, "x-nv-ping").is_some(), + header_value(&response, "x-nv-ping-payload").is_some(), + ), + ); + last_status = response.status; + match response.status { + 200 => { + if let Some(peer) = peer { + return Ok(VideoSetup { + response, + peer, + bundle_fallback: false, + }); + } + // 200 without a peer: stop sweeping immediately. Live + // alliance rigs accept the first SETUP but poison the + // session once further forms are tried (later rounds + // degrade to pure 400s and ANNOUNCE is then rejected), + // while a single SETUP followed by ANNOUNCE succeeds. + // Every observed working session peers on its first + // 200, so stopping here changes nothing for them. + if let Some(transport) = transport.filter(|t| !t.trim().is_empty()) { + opennow_streamer_protocol::log::log_line( + "WARN", + "rtsps", + &format!( + "video-setup-transport candidate={} {}", + index + 1, + transport_diagnostics::summarize(transport), + ), + ); + } + return Err(SweepFailure { + error: NvstRtspError::new( + "missing-video-peer", + format!( + "SETUP stopped after a peerless 200 without a usable NVST video peer (last status {last_status})", + ), + ), + peerless_200: Some(Box::new(response)), + }); + } + 400 | 404 | 459 | 460 | 461 => {} + _ => { + return Err(SweepFailure { + error: NvstRtspError::new( + "nvst-rtsp-failed", + format!("SETUP failed with status {}", response.status), + ), + peerless_200: None, + }); + } + } + } + } + Err(SweepFailure { + error: NvstRtspError::new( + "nvst-rtsp-failed", + format!( + "SETUP did not return a usable NVST video peer after {} URI forms and 2 Transport forms (last status {last_status})", + candidates.len(), + ), + ), + peerless_200: None, + }) + } + + fn connect(endpoint: &str, session_id: &str) -> Result<(Self, String), NvstRtspError> { + let (wss, target) = rtsp_endpoint_urls(endpoint)?; + let mut request = wss + .into_client_request() + .map_err(|error| NvstRtspError::new("nvst-connect-failed", error.to_string()))?; + request.headers_mut().insert( + "x-nv-sessionid", + HeaderValue::from_str(session_id).map_err(|_| { + NvstRtspError::new("invalid-session", "Invalid NVST session identity") + })?, + ); + request + .headers_mut() + .insert("content-length", HeaderValue::from_static("0")); + let (mut socket, _) = connect_with_retry(|| connect(request.clone()))?; + set_io_timeout(&mut socket, REQUEST_TIMEOUT); + Ok(( + Self { + socket, + cseq: 0, + buffer: String::new(), + }, + target, + )) + } + + fn request( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, String)], + body: &str, + ) -> Result { + self.request_with_timeout(method, uri, headers, body, REQUEST_TIMEOUT) + } + + fn request_with_timeout( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, String)], + body: &str, + timeout: Duration, + ) -> Result { + let mut stage = opennow_streamer_protocol::log::Stage::begin("rtsps.request"); + let deadline = Instant::now() + timeout; + set_io_timeout(&mut self.socket, timeout); + self.socket.set_config(|config| { + config.max_message_size = Some(MAX_REQUEST_RESPONSE_BYTES); + config.max_frame_size = Some(MAX_REQUEST_RESPONSE_BYTES); + }); + self.send_request(method, uri, headers, body)?; + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "request method={method} cseq={} timeout_ms={} body_bytes={}", + self.cseq, + timeout.as_millis(), + body.len() + ), + ); + loop { + if self.buffer.len() > MAX_REQUEST_RESPONSE_BYTES { + return Err(NvstRtspError::new( + "nvst-rtsp-failed", + format!( + "RTSPS {method} response exceeds request buffer limit: {} bytes (limit {MAX_REQUEST_RESPONSE_BYTES})", + self.buffer.len() + ), + )); + } + if Instant::now() >= deadline { + return Err(NvstRtspError::new( + "nvst-rtsp-timeout", + format!("RTSPS {method} timed out"), + )); + } + let response = match take_rtsp_response(&mut self.buffer, self.cseq) { + Ok(response) => response, + Err(error) + if method == "TEARDOWN" && error.code == "nvst-rtsp-sequence-mismatch" => + { + continue; + } + Err(error) => return Err(error), + }; + if let Some(response) = response { + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "response method={method} cseq={} status={} body_bytes={}", + self.cseq, + response.status, + response.body.len() + ), + ); + stage.complete(); + return Ok(response); + } + set_io_timeout( + &mut self.socket, + deadline + .saturating_duration_since(Instant::now()) + .max(Duration::from_millis(1)), + ); + match self.socket.read() { + Ok(Message::Text(text)) => self.buffer.push_str(text.as_str()), + Ok(Message::Binary(bytes)) => { + self.buffer.push_str(&String::from_utf8_lossy(&bytes)) + } + Ok(Message::Ping(bytes)) => { + let _ = self.socket.send(Message::Pong(bytes)); + } + Ok(Message::Close(_)) => { + return Err(NvstRtspError::new( + "nvst-network-failed", + "RTSPS control channel closed", + )); + } + Ok(_) => {} + Err(tungstenite::Error::Io(error)) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {} + Err(error) => return Err(rtsp_io_error(&error)), + } + } + } + + fn send_request( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, String)], + body: &str, + ) -> Result { + self.cseq += 1; + let mut request = format!( + "{method} {uri} RTSP/1.0\r\nCSeq: {}\r\nRequest-Id: {}\r\n", + self.cseq, self.cseq + ); + for (name, value) in headers { + request.push_str(&format!("{name}: {value}\r\n")); + } + if !body.is_empty() { + request.push_str(&format!("Content-Length: {}\r\n", body.len())); + } + request.push_str("\r\n"); + request.push_str(body); + self.socket + .send(Message::Text(request.into())) + .map_err(|error| rtsp_io_error(&error))?; + Ok(self.cseq) + } +} + +fn connect_with_retry( + mut attempt: impl FnMut() -> Result, +) -> Result { + for retry in 0..3 { + match attempt() { + Ok(value) => return Ok(value), + Err(error) => { + let failure = rtsp_connect_error(&error); + let transient = matches!( + failure.code, + "nvst-connect-failed" | "nvst-service-unavailable" + ); + opennow_streamer_protocol::log::log_line( + "WARN", + "rtsp", + &format!( + "signaling-connect-failed attempt={}/3 code={}", + retry + 1, + failure.code + ), + ); + if !transient || retry == 2 { + return Err(failure); + } + } + } + } + Err(NvstRtspError::new( + "nvst-connect-failed", + "RTSPS connection attempts exhausted", + )) +} + +fn rtsp_connect_error(error: &tungstenite::Error) -> NvstRtspError { + if let tungstenite::Error::Http(response) = error { + let status = response.status().as_u16(); + return NvstRtspError::new( + if status == 401 || status == 403 { + "nvst-signaling-forbidden" + } else if status == 503 { + "nvst-service-unavailable" + } else { + "nvst-signaling-rejected" + }, + if status == 503 { + "The GeForce NOW RTSPS service is temporarily unavailable (HTTP 503). The media connection was not established; this is not a video decoder error.".to_owned() + } else { + format!("Could not open RTSPS control channel: HTTP {status}") + }, + ); + } + NvstRtspError::new( + if matches!( + error, + tungstenite::Error::Io(_) + | tungstenite::Error::ConnectionClosed + | tungstenite::Error::AlreadyClosed + ) { + "nvst-connect-failed" + } else { + "nvst-signaling-rejected" + }, + format!("Could not open RTSPS control channel: {error}"), + ) +} + +fn rtsp_io_error(error: &tungstenite::Error) -> NvstRtspError { + NvstRtspError::new( + if matches!( + error, + tungstenite::Error::Io(_) + | tungstenite::Error::ConnectionClosed + | tungstenite::Error::AlreadyClosed + ) { + "nvst-network-failed" + } else { + "nvst-rtsp-failed" + }, + error.to_string(), + ) +} + +pub struct PreparedNvstRtspSession { + control_ping: NvstControlPing, + client: Option, + target: String, + common_headers: Vec<(&'static str, String)>, + rtsp_session: String, + disable_play: bool, + announce_body: String, + announced: bool, + owns_session: bool, + pub handoff: Value, + pub media_config: MediaStreamConfig, +} + +impl PreparedNvstRtspSession { + pub fn announce(&mut self) -> Result<(), NvstRtspError> { + if self.announced { + return Ok(()); + } + let client = self + .client + .as_mut() + .ok_or_else(|| NvstRtspError::new("nvst-rtsp-failed", "RTSPS client is unavailable"))?; + let mut announce_headers = self.common_headers.clone(); + announce_headers.push(("Session", self.rtsp_session.clone())); + announce_headers.push(("Content-Type", "application/sdp".to_owned())); + let announce = client.request( + "ANNOUNCE", + &self.target, + &announce_headers, + &self.announce_body, + )?; + ensure_rtsp_ok("ANNOUNCE", &announce)?; + self.announced = true; + Ok(()) + } + + pub fn finish(mut self) -> Result { + self.announce()?; + + if !self.disable_play { + let mut play_headers = self.common_headers.clone(); + play_headers.push(("Session", self.rtsp_session.clone())); + let play = self + .client + .as_mut() + .ok_or_else(|| { + NvstRtspError::new("nvst-rtsp-failed", "RTSPS client is unavailable") + })? + .request("PLAY", &self.target, &play_headers, "")?; + ensure_rtsp_ok("PLAY", &play)?; + } + + let client = self + .client + .take() + .ok_or_else(|| NvstRtspError::new("nvst-rtsp-failed", "RTSPS client is unavailable"))?; + self.owns_session = false; + ActiveNvstRtspSession::spawn(client, self.control_ping.clone()) + } +} + +impl Drop for PreparedNvstRtspSession { + fn drop(&mut self) { + if !self.owns_session { + return; + } + let Some(client) = self.client.as_mut() else { + return; + }; + set_io_timeout(&mut client.socket, CONTROL_IO_TIMEOUT); + let mut headers = self.common_headers.clone(); + headers.push(("Session", self.rtsp_session.clone())); + let _ = client.request_with_timeout( + "TEARDOWN", + &self.target, + &headers, + "", + Duration::from_secs(1), + ); + let _ = client.socket.close(None); + } +} + +enum Control { + Shutdown, +} + +pub struct ActiveNvstRtspSession { + control: Sender, + worker: Option>, +} + +impl ActiveNvstRtspSession { + fn spawn(mut client: RtspClient, control_ping: NvstControlPing) -> Result { + set_io_timeout(&mut client.socket, CONTROL_IO_TIMEOUT); + client.socket.set_config(|config| { + config.max_message_size = Some(MAX_CONTROL_RESPONSE_BYTES); + config.max_frame_size = Some(MAX_CONTROL_RESPONSE_BYTES); + }); + let (control, receiver) = mpsc::channel(); + let worker = thread::Builder::new() + .name("opennow-nvst-rtsps".to_owned()) + .spawn(move || { + let mut last_ping = Instant::now(); + let mut outstanding: Option<(u64, Instant)> = None; + let mut ping_sequence = 0u64; + loop { + if receiver.try_recv().is_ok() { + control_ping.clear(); + let _ = client.socket.close(None); + break; + } + let now = Instant::now(); + if outstanding.is_some_and(|(_, sent_at)| { + now.duration_since(sent_at) >= KEEPALIVE_INTERVAL + }) { + outstanding = None; + control_ping.clear(); + } + if outstanding.is_none() && now.duration_since(last_ping) >= KEEPALIVE_INTERVAL + { + ping_sequence = ping_sequence.wrapping_add(1); + if client + .socket + .send(Message::Ping(ping_sequence.to_be_bytes().to_vec().into())) + .is_err() + { + break; + } + outstanding = Some((ping_sequence, now)); + last_ping = now; + } + match client.socket.read() { + Ok(Message::Text(text)) => client.buffer.push_str(text.as_str()), + Ok(Message::Binary(bytes)) => { + client.buffer.push_str(&String::from_utf8_lossy(&bytes)); + } + Ok(Message::Ping(bytes)) => { + if client.socket.send(Message::Pong(bytes)).is_err() { + break; + } + } + Ok(Message::Pong(bytes)) => { + if let Some((sequence, sent_at)) = outstanding { + if bytes.as_ref() == sequence.to_be_bytes() { + outstanding = None; + control_ping.record(sent_at, Instant::now()); + } + } + } + Ok(Message::Close(_)) => break, + Ok(_) => {} + Err(tungstenite::Error::Io(error)) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::TimedOut + ) => {} + Err(_) => break, + } + if client.buffer.len() > MAX_CONTROL_RESPONSE_BYTES { + break; + } + while !client.buffer.is_empty() { + match take_rtsp_response(&mut client.buffer, client.cseq) { + Ok(Some(_)) => {} + Ok(None) => break, + Err(error) if error.code == "nvst-rtsp-sequence-mismatch" => {} + Err(_) => { + control_ping.clear(); + return; + } + } + } + } + control_ping.clear(); + }) + .map_err(|error| NvstRtspError::new("nvst-control-failed", error.to_string()))?; + Ok(Self { + control, + worker: Some(worker), + }) + } + + pub fn shutdown(&mut self) { + let _ = self.control.send(Control::Shutdown); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl Drop for ActiveNvstRtspSession { + fn drop(&mut self) { + self.shutdown(); + } +} + +pub fn prepare_owned_nvst( + context: &SessionContext, + bundle: &mut ReservedNvstBundle, +) -> Result { + ensure_tls_crypto_provider()?; + let endpoints = context + .session + .extra + .get("rtspsEndpoints") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(Value::as_str) + .collect::>(); + let session_id = context.session.session_id.trim(); + if session_id.is_empty() { + return Err(NvstRtspError::new( + "invalid-session", + "NVST negotiation requires a session ID", + )); + } + + try_signaling_endpoints(&endpoints, |endpoint| { + prepare_on_endpoint(context, bundle, session_id, endpoint) + }) +} + +fn try_signaling_endpoints( + endpoints: &[Option<&str>], + mut handshake: impl FnMut(&str) -> Result, +) -> Result { + if endpoints.is_empty() { + return Err(NvstRtspError::new( + "missing-rtsps-endpoint", + "CloudMatch did not provide an RTSPS endpoint for NVST", + )); + } + let mut failures = Vec::new(); + for (index, endpoint) in endpoints.iter().enumerate() { + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "signaling-attempt endpoint={}/{}", + index + 1, + endpoints.len() + ), + ); + let attempt = endpoint + .ok_or_else(|| { + NvstRtspError::new( + "invalid-rtsps-endpoint", + "RTSPS endpoint is not a URL string", + ) + }) + .and_then(&mut handshake); + match attempt { + Ok(session) => return Ok(session), + Err(error) => { + opennow_streamer_protocol::log::log_line( + "WARN", + "rtsps", + &format!( + "signaling-failed endpoint={}/{} code={}", + index + 1, + endpoints.len(), + error.code + ), + ); + if !matches!( + error.code, + "nvst-connect-failed" + | "nvst-network-failed" + | "nvst-rtsp-timeout" + | "nvst-service-unavailable" + ) { + return Err(error); + } + failures.push((index + 1, error)); + } + } + } + let summary = failures + .iter() + .map(|(index, error)| format!("{index}:{}", error.code)) + .collect::>() + .join(", "); + let (_, last) = failures.pop().ok_or_else(|| { + NvstRtspError::new( + "missing-rtsps-endpoint", + "CloudMatch did not provide an RTSPS endpoint for NVST", + ) + })?; + Err(NvstRtspError::new( + last.code, + format!( + "All {} NVST signaling endpoints failed ({}); last: {}", + endpoints.len(), + summary, + last.message, + ), + )) +} + +fn prepare_on_endpoint( + context: &SessionContext, + bundle: &mut ReservedNvstBundle, + session_id: &str, + endpoint: &str, +) -> Result { + let client_port = bundle + .local_addr() + .map_err(|error| NvstRtspError::new("nvst-bind-failed", error.to_string()))? + .port(); + let mjolnir_port = bundle + .mjolnir_local_addr() + .map_err(|error| NvstRtspError::new("nvst-bind-failed", error.to_string()))? + .port(); + let identity = bundle.identity(); + + let mut connect_stage = opennow_streamer_protocol::log::Stage::begin("rtsps.connect"); + let (mut client, target) = RtspClient::connect(endpoint, session_id)?; + connect_stage.complete(); + drop(connect_stage); + let host = target + .strip_prefix("rtsps://") + .or_else(|| target.strip_prefix("rtsp://")) + .unwrap_or(&target) + .to_owned(); + let common_headers = vec![ + ("X-GS-Version", "14.2".to_owned()), + ("Host", host), + ("x-nv-sessionid", session_id.to_owned()), + ]; + + let options = client.request("OPTIONS", &target, &common_headers, "")?; + ensure_rtsp_ok("OPTIONS", &options)?; + let mut describe_headers = common_headers.clone(); + describe_headers.push(("Accept", "application/sdp".to_owned())); + describe_headers.push(("x-nv-abtesting", "2".to_owned())); + let describe = client.request("DESCRIBE", &target, &describe_headers, "")?; + ensure_rtsp_ok("DESCRIBE", &describe)?; + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!("describe-shape {}", describe_media_shape(&describe.body),), + ); + let stream = super::media_stream_config(context); + opennow_streamer_protocol::log::log_line( + "INFO", + "nvst-color", + &format!( + "color={} hdr={} announce={:?}", + stream.color_quality.protocol_name(), + stream.hdr, + announce_color_lines(stream), + ), + ); + + let rtsp_session = header_value(&describe, "session") + .and_then(|value| value.split(';').next()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + NvstRtspError::new("nvst-rtsp-failed", "DESCRIBE did not include a session") + })? + .to_owned(); + let video_control = media_control(&describe.body, "video").ok_or_else(|| { + NvstRtspError::new( + "missing-video-control", + "DESCRIBE did not include a video control stream", + ) + })?; + let video_qos_offers = video_qos_offers(&describe.body); + video_qos_offers.validate()?; + let max_control_report_bytes = qos_messages_size(&describe.body)?; + let described_ping_version = sdp_attribute(&describe.body, "general.pingVersion") + .and_then(|value| value.parse::().ok()) + .unwrap_or(6); + let remote_ufrag = sdp_attribute(&describe.body, "general.iceUserNameFragmentV2") + .or_else(|| sdp_attribute(&describe.body, "general.iceUsernameFragment")); + let remote_password = sdp_attribute(&describe.body, "general.icePasswordV2") + .or_else(|| sdp_attribute(&describe.body, "general.iceUsernamePwd")); + let remote_fingerprint = sdp_attribute(&describe.body, "general.dtlsFingerprintV2") + .or_else(|| sdp_attribute(&describe.body, "general.dtlsFingerprint")); + let disable_play = sdp_attribute(&describe.body, "general.disablePlay").as_deref() == Some("1"); + let native_bundle = sdp_attribute(&describe.body, "general.nativeRtcOnBundlePort"); + if native_bundle.as_deref() != Some("1") { + return Err(NvstRtspError::new( + "nvst-legacy-transport-unsupported", + "This seat requires the retired multi-socket NVST transport", + )); + } + let rtcp_on_sctp = sdp_attribute(&describe.body, "general.rtcpOnSctp").as_deref() == Some("1"); + let hid_device_mask = sdp_attribute(&describe.body, "ri.hidDeviceMask") + .as_deref() + .map(parse_hid_device_mask) + .unwrap_or(0); + let microphone_available = negotiate_microphone(context, &describe.body); + + let mut setup_headers = common_headers.clone(); + setup_headers.push(("Session", rtsp_session.clone())); + setup_headers.push(("x-nv-ping", described_ping_version.to_string())); + let bundle_endpoint = bundle_endpoint(context); + let described_unified_socket = + offered_bool(&describe.body, "general.enableUnifiedSocket").unwrap_or(false); + opennow_streamer_protocol::log::log_line( + "INFO", + "rtsps", + &format!( + "bundle-route source={} unified_socket={described_unified_socket} server_bundle_port={} rtc_video_on_native_bundle={} connection_usages=[{}]", + bundle_endpoint.map_or("setup", |endpoint| endpoint.source), + sdp_attribute(&describe.body, "general.serverBundlePort").unwrap_or_default(), + sdp_attribute(&describe.body, "general.rtcVideoOnNativeBundle").unwrap_or_default(), + connection_usage_summary(context), + ), + ); + // CloudMatch bundle peer doubles as the video peer when rigs omit + // Transport (validated here; the substitution itself is logged where it + // happens inside setup_video). + let bundle_video_peer = bundle_endpoint.and_then(|endpoint| { + let port = u16::try_from(endpoint.port) + .ok() + .filter(|port| *port != 0)?; + endpoint.ip.parse::().ok()?; + Some((endpoint.ip.to_owned(), port, port)) + }); + let VideoSetup { + response: setup, + peer: (video_peer_ip, video_peer_port, video_peer_port_end), + bundle_fallback, + } = client.setup_video( + &video_control, + &target, + &setup_headers, + if described_unified_socket { + client_port + } else { + mjolnir_port + }, + bundle_video_peer, + )?; + let (bundle_peer_ip, bundle_peer_port) = bundle_endpoint + .map(|endpoint| (endpoint.ip, endpoint.port)) + .unwrap_or((&video_peer_ip, u32::from(video_peer_port))); + let bundle_peer_port = u16::try_from(bundle_peer_port) + .ok() + .filter(|port| *port != 0) + .ok_or_else(|| { + NvstRtspError::new("invalid-media-peer", "NVST media peer port is invalid") + })?; + let bundle_peer = bundle_peer_ip + .parse::() + .map(|ip| std::net::SocketAddr::new(ip, bundle_peer_port)) + .map_err(|_| { + NvstRtspError::new("invalid-media-peer", "NVST media peer is not an IP address") + })?; + let unified_socket = described_unified_socket || bundle_fallback; + let (video_peer_ip, video_peer_port, video_peer_port_end) = if unified_socket { + ( + bundle_peer.ip().to_string(), + bundle_peer.port(), + bundle_peer.port(), + ) + } else { + (video_peer_ip, video_peer_port, video_peer_port_end) + }; + let video_local_port = if unified_socket { + client_port + } else { + mjolnir_port + }; + let local_address = bundle + .advertised_local_address_for(bundle_peer) + .map_err(|error| { + NvstRtspError::new( + "nvst-bind-failed", + format!("Could not select the local route to the NVST media peer: {error}"), + ) + })?; + let video_peer_ip_parsed = video_peer_ip.parse().map_err(|_| { + NvstRtspError::new("invalid-media-peer", "NVST video peer is not an IP address") + })?; + let video_packet_size = nvst_video_packet_size(video_peer_ip_parsed) + .map_err(|error| NvstRtspError::new("nvst-video-mtu-invalid", error.to_string()))?; + let video_packet_size = measured_path_packet_size(context, video_peer_ip_parsed) + .map_or(video_packet_size, |measured| { + video_packet_size.min(measured) + }); + opennow_streamer_protocol::log::log_line( + "INFO", + "transport", + &format!( + "NVST media route local={local_address} bundlePort={client_port} videoPort={video_local_port} bundlePeer={bundle_peer} videoPeer={video_peer_ip}:{video_peer_port} unifiedSocket={unified_socket}" + ), + ); + let setup_ping_payload = header_value(&setup, "x-nv-ping-payload").map(ToOwned::to_owned); + let ping_version = header_value(&setup, "x-nv-ping") + .and_then(|value| value.parse::().ok()) + .unwrap_or(described_ping_version); + if ping_version == 6 && setup_ping_payload.is_none() { + return Err(NvstRtspError::new( + "missing-ice-credentials", + "SETUP selected ping version 6 without an X-Nv-Ping-Payload", + )); + } + let bundle_username = (ping_version == 6) + .then(|| bundle_natt_username(&describe.body)) + .flatten(); + let remote_ufrag = resolve_remote_ufrag( + setup_ping_payload.as_deref(), + remote_ufrag.as_deref(), + bundle_username.as_deref(), + ping_version, + ) + .ok_or_else(|| { + NvstRtspError::new( + "missing-ice-credentials", + "NVST negotiation did not provide a remote ICE username fragment", + ) + })?; + let ping_payload = setup_ping_payload.unwrap_or_else(|| "PING".to_owned()); + let remote_password = remote_password.ok_or_else(|| { + NvstRtspError::new( + "missing-ice-credentials", + "DESCRIBE did not return NVST ICE credentials", + ) + })?; + let (key, key_id) = match runtime_key(&describe.body) { + Some(value) => value, + None => random_runtime_key()?, + }; + let salt = format!("{key_id:024X}"); + let codec = negotiated_codec(context); + let srtp_profile = + advertised_srtp_profile(&setup, &describe.body).unwrap_or("AEAD_AES_256_GCM_8"); + + let mut handoff = json!({ + "clientUdpPort":client_port, + "packetSize":video_packet_size, + "mjolnirUdpPort":(!unified_socket).then_some(mjolnir_port), + "unifiedSocket":unified_socket, + "videoPeerIp":video_peer_ip, + "videoPeerPort":video_peer_port, + "videoPeerPortEnd":video_peer_port_end, + "srtpAesKeyHex":key, + "srtpKeyId":key_id, + "srtpSaltHex":salt, + "srtpProfile":srtp_profile, + "pingPayload":ping_payload, + "pingVersion":ping_version, + "localIceUsernameFragment":identity.ice_username_fragment, + "localIcePassword":identity.ice_password, + "remoteIceUsernameFragment":remote_ufrag, + "remoteIcePassword":remote_password, + "localDtlsFingerprint":identity.dtls_fingerprint, + "remoteDtlsFingerprint":remote_fingerprint, + "rtcpOnSctp":rtcp_on_sctp, + "hidDeviceMask":hid_device_mask, + "microphoneOnBundle":microphone_available, + "codec":codec, + "audioTrack":{"payloadType":111,"codec":"opus","clockRateHz":48000,"channels":2,"mid":"0"}, + "timeoutMs":VIDEO_TIMEOUT_MS, + "startupTimeoutMs":VIDEO_STARTUP_TIMEOUT_MS + }); + video_qos_offers.add_to_handoff(&mut handoff); + if let Some(max_control_report_bytes) = max_control_report_bytes { + handoff["maxQosMessagesSize"] = json!(max_control_report_bytes); + } + if let Some(username) = bundle_username { + handoff["bundleNattRemoteUsername"] = json!(username); + } + handoff["bundlePeerIp"] = json!(bundle_peer.ip().to_string()); + handoff["bundlePeerPort"] = json!(bundle_peer.port()); + handoff["recoveryPolicy"] = json!(recovery_policy(&describe.body)); + microphone_handoff(&describe.body, &mut handoff); + + // Only log transport shape, never SDP, runtime keys, or ICE credentials. + opennow_streamer_protocol::log::log_line( + "INFO", + "nvst-handoff", + &format!( + "video_local_port={video_local_port} bundle_local_port={client_port} video_peer_port={video_peer_port} video_peer_port_end={video_peer_port_end} bundle_peer_port={} same_peer_host={} unified_socket={unified_socket} ping_version={ping_version} ping_bytes={} legacy_ping_payload={} srtp_profile={srtp_profile} rtcp_on_sctp={rtcp_on_sctp} sockets_retained=true reachability=unverified video_startup_timeout_ms={VIDEO_STARTUP_TIMEOUT_MS} video_idle_timeout_ms={VIDEO_TIMEOUT_MS}", + bundle_peer.port(), + bundle_peer.ip().to_string() == video_peer_ip, + ping_payload.len(), + ping_payload == "PING" + ), + ); + + let effective_announce = negotiated_announce( + &build_announce( + context, + AnnounceParams { + stream, + key: handoff["srtpAesKeyHex"].as_str().unwrap_or_default(), + key_id, + port: client_port, + address: &local_address, + ufrag: handoff["localIceUsernameFragment"] + .as_str() + .unwrap_or_default(), + password: handoff["localIcePassword"].as_str().unwrap_or_default(), + fingerprint: handoff["localDtlsFingerprint"].as_str().unwrap_or_default(), + video_port: video_peer_port, + video_packet_size, + rtcp_on_sctp, + microphone_available, + qos_timings_v5: video_qos_offers.uses_v5_timings(), + unified_socket, + }, + ), + &describe.body, + &mut handoff, + )?; + let announce_body = omit_server_announce_attributes(&effective_announce, &describe.body); + Ok(PreparedNvstRtspSession { + control_ping: NvstControlPing::default(), + client: Some(client), + target, + common_headers, + rtsp_session, + disable_play, + announce_body, + announced: false, + owns_session: true, + handoff, + media_config: stream, + }) +} + +fn ensure_tls_crypto_provider() -> Result<(), NvstRtspError> { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + if rustls::crypto::CryptoProvider::get_default().is_none() { + return Err(NvstRtspError::new( + "tls-provider-unavailable", + "Could not initialize the TLS crypto provider", + )); + } + Ok(()) +} + +struct AnnounceParams<'a> { + stream: MediaStreamConfig, + key: &'a str, + key_id: u32, + port: u16, + address: &'a str, + ufrag: &'a str, + password: &'a str, + fingerprint: &'a str, + video_port: u16, + video_packet_size: usize, + rtcp_on_sctp: bool, + microphone_available: bool, + qos_timings_v5: bool, + unified_socket: bool, +} + +fn negotiate_microphone(_context: &SessionContext, describe: &str) -> bool { + sdp_attribute(describe, "general.rtcMicOnNativeBundle").as_deref() == Some("1") +} + +fn advertised_bitrate_kbps(settings: &Value) -> u64 { + let mbps = settings + .get("maxBitrateMbps") + .and_then(Value::as_f64) + .filter(|value| value.is_finite()) + .unwrap_or(75.0) + .clamp(0.22, MAX_STREAM_BITRATE_MBPS as f64); + (mbps * 1000.0).round() as u64 +} + +fn omit_server_announce_attributes(announce: &str, describe: &str) -> String { + const SERVER_NEGOTIATED: &[&str] = &[ + "video[0].updateSplitEncodeStateDynamically", + "video[0].enableRtpNack", + "video[0].rtpNackQueueLength", + "video[0].rtpNackQueueMaxPackets", + "video[0].rtpNackMaxPacketCount", + "video[0].framePacing.mode", + "video[0].framePacing.feedbackMode", + "video[0].initialBitrateKbps", + "video[0].initialPeakBitrateKbps", + "video[0].mapRtpTimestampsToFrames", + "vqos[0].fec.enable", + "vqos[0].fec.rateDropWindow", + "vqos[0].fec.minRequiredFecPackets", + "vqos[0].fec.repairPercent", + "vqos[0].fec.repairMinPercent", + "vqos[0].fec.repairMaxPercent", + "vqos[0].bllFec.enable", + "vqos[0].drc.enable", + "vqos[0].dfc.adjustResAndFps", + "vqos[0].calculateAvgVideoStreamingBitrate", + "vqos[0].bw.minimumBitrateKbps", + "vqos[0].drc.bitrateIirFilterFactor", + "vqos[0].resControl.bitrateIirFilterFactor", + "packetPacing.version", + "packetPacing.mode", + "packetPacing.numGroups", + "packetPacing.maxDelayUs", + "packetPacing.minNumPacketsFrame", + "packetPacing.minNumPacketsPerGroup", + "packetPacing.enableAccurateSleep", + "packetPacing.enableSmoothTransition", + "packetPacing.allowFpsBasedToggle", + "ri.partialReliableThresholdMs", + "ri.timestampsEnabled", + "ri.useMultipleGamepads", + "ri.usePartiallyReliableUdpChannel", + "ri.enablePartiallyReliableTransferGamepad", + "ri.enablePartiallyReliableTransferHid", + "bwe.useOwdCongestionControl", + "general.rtspWebSocketPerConnection", + "general.pingIntervalBeforeConnectionMs", + "general.pingIntervalAfterConnectionMs", + "runtime.audioSrtp", + "runtime.micSrtp", + "runtime.videoSrtp", + "general.nativeRtcOnBundlePort", + "general.rtcVideoOnNativeBundle", + "general.rtcAudioOnNativeBundle", + "general.rtcDataChannelOnNativeBundle", + "general.enableUnifiedSocket", + "general.rtcpOnSctp", + "video[0].refPicInvalidation", + "general.clientSupportsIntraRefresh", + ]; + let server_values: HashMap = describe + .split("||") + .next() + .unwrap_or_default() + .split(";;") + .flat_map(str::lines) + .filter_map(|line| { + let (name, value) = line.trim().strip_prefix("a=")?.split_once(':')?; + let name = name.strip_prefix("x-nv-").unwrap_or(name); + let value = value.trim(); + (!value.is_empty()).then(|| (name.to_ascii_lowercase(), value)) + }) + .collect(); + let mut filtered = String::with_capacity(announce.len()); + for line in announce.split_inclusive("\r\n") { + let Some((name, value)) = line + .strip_prefix("a=x-nv-") + .and_then(|attribute| attribute.split_once(':')) + else { + filtered.push_str(line); + continue; + }; + if SERVER_NEGOTIATED.contains(&name) + && server_values + .get(&name.to_ascii_lowercase()) + .is_some_and(|server| { + *server == value.trim() + || server + .parse::() + .ok() + .zip(value.trim().parse::().ok()) + .is_some_and(|(server, client)| server == client) + }) + { + continue; + } + filtered.push_str(line); + } + filtered +} + +fn build_announce(context: &SessionContext, params: AnnounceParams<'_>) -> String { + let (width, height) = resolution(context); + let fps = negotiated_fps(context); + let bitrate = advertised_bitrate_kbps(&context.settings); + let minimum_bitrate = bitrate.min(1_000); + let codec = negotiated_codec(context); + let format = if codec.eq_ignore_ascii_case("AV1") { + 2 + } else if codec.eq_ignore_ascii_case("H265") || codec.eq_ignore_ascii_case("HEVC") { + 1 + } else { + 0 + }; + let dynamic_streaming_mode = negotiated_dynamic_streaming_mode(context); + let adjust_res_and_fps = negotiated_adjustment_enabled(dynamic_streaming_mode); + let mut lines = vec![ + "v=0".to_owned(), + "o=unknown 0 14 IN IPv4 127.0.0.1".to_owned(), + "s=NVIDIA Streaming Client".to_owned(), + format!("a=x-nv-video[0].clientViewportWd:{width}"), + format!("a=x-nv-video[0].clientViewportHt:{height}"), + "a=x-nv-video[0].maxCodecProfile:3".to_owned(), + "a=x-nv-video[0].maxCodecLevel:51".to_owned(), + "a=x-nv-video[0].maxH264Profile:3".to_owned(), + "a=x-nv-video[0].maxH264Level:51".to_owned(), + "a=x-nv-video[0].updateSplitEncodeStateDynamically:1".to_owned(), + format!("a=x-nv-video[0].packetSize:{}", params.video_packet_size), + "a=x-nv-video[0].enableRtpNack:1".to_owned(), + "a=x-nv-video[0].rtpNackQueueLength:2048".to_owned(), + "a=x-nv-video[0].rtpNackQueueMaxPackets:1024".to_owned(), + "a=x-nv-video[0].rtpNackMaxPacketCount:64".to_owned(), + format!( + "a=x-nv-video[0].framePacing.mode:{}", + if params.qos_timings_v5 { 2 } else { 1 } + ), + format!( + "a=x-nv-video[0].framePacing.feedbackMode:{}", + if params.qos_timings_v5 { 0 } else { 1 } + ), + "a=x-nv-video[0].maxNumReferenceFrames:0".to_owned(), + "a=x-nv-video[0].refPicInvalidation:1".to_owned(), + "a=x-nv-general.clientSupportsIntraRefresh:1".to_owned(), + "a=x-nv-video[0].prefilterParams.prefilterMode:0".to_owned(), + "a=x-nv-video[0].prefilterParams.prefilterModel:4".to_owned(), + "a=x-nv-video[0].prefilterParams.denoiseLevel:0".to_owned(), + "a=x-nv-video[0].prefilterParams.sharpnessLevel:0".to_owned(), + "a=x-nv-video[0].encoderCscMode:2".to_owned(), + "a=x-nv-video[0].encoderHdrCscMode:4".to_owned(), + "a=x-nv-video[0].mapRtpTimestampsToFrames:0".to_owned(), + format!("a=x-nv-video[0].maxFPS:{fps}"), + format!("a=x-nv-video[0].initialBitrateKbps:{bitrate}"), + format!("a=x-nv-video[0].initialPeakBitrateKbps:{bitrate}"), + format!("a=x-nv-vqos[0].bitStreamFormat:{format}"), + "a=x-nv-vqos[0].fec.enable:1".to_owned(), + "a=x-nv-vqos[0].fec.rateDropWindow:10".to_owned(), + "a=x-nv-vqos[0].fec.minRequiredFecPackets:2".to_owned(), + "a=x-nv-vqos[0].fec.repairPercent:20".to_owned(), + "a=x-nv-vqos[0].fec.repairMinPercent:20".to_owned(), + "a=x-nv-vqos[0].fec.repairMaxPercent:35".to_owned(), + "a=x-nv-vqos[0].bllFec.enable:0".to_owned(), + "a=x-nv-vqos[0].drc.enable:0".to_owned(), + format!("a=x-nv-vqos[0].dfc.adjustResAndFps:{adjust_res_and_fps}"), + "a=x-nv-vqos[0].calculateAvgVideoStreamingBitrate:1".to_owned(), + format!("a=x-nv-vqos[0].bw.maximumBitrateKbps:{bitrate}"), + format!("a=x-nv-vqos[0].bw.minimumBitrateKbps:{minimum_bitrate}"), + "a=x-nv-vqos[0].drc.bitrateIirFilterFactor:128".to_owned(), + "a=x-nv-vqos[0].resControl.bitrateIirFilterFactor:128".to_owned(), + format!("a=x-nv-vqos[0].dynamicStreamingMode:{dynamic_streaming_mode}"), + "a=x-nv-packetPacing.version:3".to_owned(), + "a=x-nv-packetPacing.mode:1".to_owned(), + "a=x-nv-packetPacing.numGroups:5".to_owned(), + format!( + "a=x-nv-packetPacing.maxDelayUs:{}", + if fps >= 100 { 4000 } else { 2000 } + ), + "a=x-nv-packetPacing.minNumPacketsFrame:10".to_owned(), + "a=x-nv-packetPacing.minNumPacketsPerGroup:15".to_owned(), + "a=x-nv-packetPacing.enableAccurateSleep:1".to_owned(), + "a=x-nv-packetPacing.enableSmoothTransition:1".to_owned(), + "a=x-nv-packetPacing.allowFpsBasedToggle:1".to_owned(), + "a=x-nv-ri.partialReliableThresholdMs:300".to_owned(), + "a=x-nv-ri.timestampsEnabled:1".to_owned(), + "a=x-nv-ri.useMultipleGamepads:1".to_owned(), + "a=x-nv-ri.usePartiallyReliableUdpChannel:0".to_owned(), + "a=x-nv-ri.enablePartiallyReliableTransferGamepad:255".to_owned(), + "a=x-nv-ri.enablePartiallyReliableTransferHid:-1".to_owned(), + "a=x-nv-bwe.useOwdCongestionControl:1".to_owned(), + "a=x-nv-general.rtspWebSocketPerConnection:1".to_owned(), + "a=x-nv-general.pingIntervalBeforeConnectionMs:20".to_owned(), + "a=x-nv-general.pingIntervalAfterConnectionMs:100".to_owned(), + "a=x-nv-runtime.audioSrtp:0".to_owned(), + "a=x-nv-runtime.micSrtp:0".to_owned(), + "a=x-nv-runtime.mouseCursorCapture:3".to_owned(), + "a=x-nv-runtime.mimicRemoteCursor:0".to_owned(), + "a=x-nv-runtime.videoSrtp:1".to_owned(), + format!("a=x-nv-runtime.encryptionKey:{}", params.key), + format!("a=x-nv-runtime.encryptionKeyId:{}", params.key_id), + "a=x-nv-general.clientPorts.video:0".to_owned(), + "a=x-nv-general.clientPorts.audio:0".to_owned(), + "a=x-nv-general.clientPorts.mic:0".to_owned(), + "a=x-nv-general.clientPorts.control:0".to_owned(), + "a=x-nv-general.clientPorts.bundle:0".to_owned(), + "a=x-nv-general.clientPorts.session:0".to_owned(), + format!("a=x-nv-general.clientPorts.localAddress:{}", params.address), + "a=x-nv-general.clientPorts.useReserved:1".to_owned(), + "a=x-nv-general.clientPorts.fallbackDynamic:1".to_owned(), + format!("a=x-nv-general.clientBundlePort:{}", params.port), + "a=x-nv-general.nativeRtcOnBundlePort:1".to_owned(), + "a=x-nv-general.rtcVideoOnNativeBundle:0".to_owned(), + "a=x-nv-general.rtcAudioOnNativeBundle:1".to_owned(), + "a=x-nv-general.rtcDataChannelOnNativeBundle:1".to_owned(), + format!( + "a=x-nv-general.enableUnifiedSocket:{}", + u8::from(params.unified_socket) + ), + format!( + "a=x-nv-general.rtcpOnSctp:{}", + u8::from(params.rtcp_on_sctp) + ), + format!("a=x-nv-general.iceUserNameFragmentV2:{}", params.ufrag), + format!("a=x-nv-general.icePasswordV2:{}", params.password), + format!("a=x-nv-general.dtlsFingerprintV2:{}", params.fingerprint), + "a=ice-options:trickle".to_owned(), + format!("a=ice-ufrag:{}", params.ufrag), + format!("a=ice-pwd:{}", params.password), + format!("a=fingerprint:sha-256 {}", params.fingerprint), + "a=setup:actpass".to_owned(), + format!( + "a=candidate:1 1 udp 2122260223 {} {} typ host", + params.address, params.port + ), + ]; + if params.microphone_available { + lines.push("a=x-nv-general.rtcMicOnNativeBundle:1".to_owned()); + lines.push("a=x-nv-mic.micSsrcConfig.senderSsrc:1".to_owned()); + } + lines.extend(announce_color_lines(params.stream)); + lines.extend([ + "t=0 0".to_owned(), + format!("m=video {}", params.video_port), + "c=IN IP4 0.0.0.0".to_owned(), + "i=DeviceString, DeviceName".to_owned(), + String::new(), + ]); + lines.join("\r\n") +} + +fn resolution(context: &SessionContext) -> (u64, u64) { + let value = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("resolution")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("resolution").and_then(Value::as_str)) + .unwrap_or("1920x1080"); + value + .split_once(['x', 'X']) + .and_then(|(width, height)| Some((width.parse().ok()?, height.parse().ok()?))) + .unwrap_or((1920, 1080)) +} + +fn negotiated_fps(context: &SessionContext) -> u64 { + context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("fps")) + .and_then(Value::as_u64) + .or_else(|| context.settings.get("fps").and_then(Value::as_u64)) + .unwrap_or(60) + .clamp(30, u64::from(super::MAX_STREAM_FPS)) +} + +fn measured_path_packet_size(context: &SessionContext, peer: IpAddr) -> Option { + let datagram = context + .session + .extra + .get("networkTest")? + .get("measuredDatagramBytes")? + .as_u64()?; + opennow_streamer_transport::measured_video_packet_size(usize::try_from(datagram).ok()?, peer) +} + +fn negotiated_codec(context: &SessionContext) -> String { + context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("codec")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("codec").and_then(Value::as_str)) + .unwrap_or("H264") + .to_ascii_uppercase() +} + +fn negotiated_dynamic_streaming_mode(context: &SessionContext) -> u8 { + // The dynamic quality policy is RTSP-only in the official client: it never + // travels in CloudMatch requestedStreamingFeatures, so a negotiated value can + // only come from a server-finalized override. Otherwise the live client + // setting decides, matching the official Data Saver behavior at ANNOUNCE time. + if let Some(policy) = context + .session + .extra + .get("negotiatedStreamProfile") + .and_then(|profile| profile.get("dynamicStreamingMode")) + .and_then(Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .filter(|value| *value <= 3) + { + return policy; + } + u8::from( + context + .settings + .get("saveBandwidth") + .and_then(Value::as_bool) + .unwrap_or(false), + ) +} + +fn negotiated_adjustment_enabled(policy: u8) -> u8 { + u8::from(policy != 0) +} + +fn advertised_srtp_profile<'a>(response: &'a RtspResponse, sdp: &'a str) -> Option<&'a str> { + const PROFILES: [&str; 8] = [ + "AEAD_AES_128_GCM_8", + "AEAD_AES_256_GCM_8", + "AEAD_AES_128_GCM", + "AEAD_AES_256_GCM", + "AES_CM_128_HMAC_SHA1_32", + "AES_CM_128_HMAC_SHA1_80", + "AES_CM_256_HMAC_SHA1_32", + "AES_CM_256_HMAC_SHA1_80", + ]; + response + .headers + .iter() + .filter(|(name, _)| { + name.eq_ignore_ascii_case("transport") + || name.to_ascii_lowercase().contains("srtp") + || name.to_ascii_lowercase().contains("crypto") + }) + .map(|(_, value)| value.as_str()) + .chain(sdp.lines()) + .find_map(|value| { + let upper = value.to_ascii_uppercase(); + PROFILES.into_iter().find(|profile| { + upper + .split(|character: char| !character.is_ascii_alphanumeric() && character != '_') + .any(|token| token == *profile) + }) + }) +} + +fn ensure_rtsp_ok(step: &str, response: &RtspResponse) -> Result<(), NvstRtspError> { + if response.status == 200 { + Ok(()) + } else { + let failure = NvstRtspError::new( + if response.status == 503 { + "nvst-service-unavailable" + } else if matches!(response.status, 401 | 403) { + "nvst-signaling-forbidden" + } else { + "nvst-rtsp-failed" + }, + format!( + "{step} failed: {} {}", + response.status, response.status_text + ), + ); + opennow_streamer_protocol::log::log_line("WARN", "rtsp", &failure.message); + Err(failure) + } +} + +fn header_value<'a>(response: &'a RtspResponse, name: &str) -> Option<&'a str> { + response + .headers + .get(&name.to_ascii_lowercase()) + .map(String::as_str) +} + +fn take_rtsp_response( + buffer: &mut String, + expected_cseq: u64, +) -> Result, NvstRtspError> { + // Some Bifrost seats put an extra blank CRLF block between WebSocket-carried + // RTSP responses. It is transport padding, not an empty RTSP response. Drop + // only leading line separators while waiting for the next status line. + let status_start = buffer + .find(|character: char| character != '\r' && character != '\n') + .unwrap_or(buffer.len()); + if status_start > 0 { + buffer.drain(..status_start); + } + let Some(header_end) = buffer.find("\r\n\r\n").or_else(|| buffer.find("\n\n")) else { + return Ok(None); + }; + let separator = if buffer[header_end..].starts_with("\r\n\r\n") { + 4 + } else { + 2 + }; + let header_text = &buffer[..header_end]; + let content_length = header_text + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.trim().parse::().ok()) + }) + .unwrap_or(0); + let total = (header_end + separator) + .checked_add(content_length) + .ok_or_else(|| NvstRtspError::new("nvst-rtsp-failed", "Invalid RTSPS content length"))?; + if total > MAX_REQUEST_RESPONSE_BYTES { + return Err(NvstRtspError::new( + "nvst-rtsp-failed", + format!( + "RTSPS response exceeds request buffer limit: {total} bytes (limit {MAX_REQUEST_RESPONSE_BYTES})" + ), + )); + } + if buffer.len() < total { + return Ok(None); + } + if !buffer.is_char_boundary(total) { + return Err(NvstRtspError::new( + "nvst-rtsp-failed", + "Invalid RTSPS body encoding", + )); + } + let raw = buffer[..total].to_owned(); + buffer.drain(..total); + let (head, body) = raw.split_at(header_end + separator); + let mut lines = head.lines(); + let status_line = lines.next().unwrap_or_default(); + let mut parts = status_line.splitn(3, ' '); + let _ = parts.next(); + let status = parts + .next() + .and_then(|value| value.parse().ok()) + .ok_or_else(|| { + let printable = status_line + .chars() + .take(120) + .map(|character| { + if character.is_ascii_graphic() || character == ' ' { + character + } else { + '�' + } + }) + .collect::(); + NvstRtspError::new( + "nvst-rtsp-failed", + format!("Invalid RTSPS status line: {printable:?}"), + ) + })?; + let status_text = parts.next().unwrap_or_default().trim().to_owned(); + let mut headers = HashMap::new(); + for line in lines { + if let Some((name, value)) = line.split_once(':') { + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_owned()); + } + } + let response_cseq = headers + .get("cseq") + .and_then(|value| value.parse::().ok()); + let response_request_id = headers + .get("request-id") + .and_then(|value| value.parse::().ok()); + let sequence_matches = if headers.contains_key("cseq") { + response_cseq == Some(expected_cseq) + } else { + response_request_id == Some(expected_cseq) + }; + if !sequence_matches { + return Err(NvstRtspError::new( + "nvst-rtsp-sequence-mismatch", + format!( + "RTSPS response sequence mismatch: expected {expected_cseq}, CSeq={}, Request-Id={}", + response_cseq + .map(|value| value.to_string()) + .unwrap_or_else(|| "missing".to_owned()), + response_request_id + .map(|value| value.to_string()) + .unwrap_or_else(|| "missing".to_owned()), + ), + )); + } + Ok(Some(RtspResponse { + status, + status_text, + headers, + body: body.to_owned(), + })) +} + +fn rtsp_endpoint_urls(endpoint: &str) -> Result<(String, String), NvstRtspError> { + if endpoint.starts_with("rtsp://") { + return Err(NvstRtspError::new( + "nvst-raw-rtsp-unsupported", + "Raw RTSP signaling is not supported by the NVST WebSocket client", + )); + } + let translated = endpoint + .strip_prefix("rtsps://") + .map(|rest| format!("https://{rest}")) + .ok_or_else(|| { + NvstRtspError::new( + "invalid-rtsps-endpoint", + "Unsupported RTSPS endpoint scheme", + ) + })?; + let parsed = translated + .parse::() + .map_err(|_| NvstRtspError::new("invalid-rtsps-endpoint", "Invalid RTSPS endpoint"))?; + let host = parsed.host().ok_or_else(|| { + NvstRtspError::new("invalid-rtsps-endpoint", "RTSPS endpoint has no host") + })?; + let authority = parsed.authority().ok_or_else(|| { + NvstRtspError::new("invalid-rtsps-endpoint", "RTSPS endpoint has no authority") + })?; + if authority.as_str().contains('@') + || endpoint.contains('#') + || endpoint.chars().any(char::is_control) + { + return Err(NvstRtspError::new( + "invalid-rtsps-endpoint", + "RTSPS endpoint contains invalid authority or characters", + )); + } + if host.starts_with('[') + && host + .strip_prefix('[') + .and_then(|host| host.strip_suffix(']')) + .is_none_or(|host| host.parse::().is_err()) + { + return Err(NvstRtspError::new( + "invalid-rtsps-endpoint", + "RTSPS endpoint IPv6 host is invalid", + )); + } + let explicit_port = authority.as_str().strip_prefix(host).unwrap_or_default(); + let port = if explicit_port.is_empty() { + 322 + } else { + explicit_port + .strip_prefix(':') + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| { + NvstRtspError::new("invalid-rtsps-endpoint", "RTSPS endpoint port is invalid") + })? + }; + if port == 0 { + return Err(NvstRtspError::new( + "invalid-rtsps-endpoint", + "RTSPS endpoint port is zero", + )); + } + Ok(( + format!("wss://{host}:{port}/rtsp"), + format!("rtsps://{host}:{port}"), + )) +} + +/// Shape-only DESCRIBE summary for diagnosing peerless SETUP responses. +/// Logs media sections with ports, control shape classes, and connection +/// presence. Never logs addresses, ports of the connection line, control +/// values, or any SDP attribute values (ICE credentials, fingerprints). +fn describe_media_shape(sdp: &str) -> String { + let mut sections = Vec::new(); + let mut current: Option<(String, String)> = None; + let mut connection_present = false; + let mut control_count = 0u32; + let mut video_control_shape = "absent"; + let mut in_video = false; + for line in sdp + .split("||") + .next() + .unwrap_or_default() + .split(";;") + .next() + .unwrap_or_default() + .lines() + .map(str::trim) + { + if let Some(media) = line.strip_prefix("m=") { + if let Some((media, port)) = current.take() { + sections.push(format!("{media}/{port}")); + } + let mut parts = media.split_whitespace(); + let kind = parts.next().unwrap_or(""); + let media = if kind.eq_ignore_ascii_case("video") { + "video" + } else if kind.eq_ignore_ascii_case("audio") { + "audio" + } else if kind.eq_ignore_ascii_case("application") { + "application" + } else { + "other" + }; + current = Some(( + media.to_owned(), + parts + .next() + .and_then(|port| port.parse::().ok()) + .map_or_else(|| "?".to_owned(), |port| port.to_string()), + )); + in_video = media == "video"; + } else if line.starts_with("c=") { + connection_present = true; + } else if let Some(value) = line.strip_prefix("a=control:") { + control_count += 1; + if in_video && video_control_shape == "absent" { + let lower = value.to_ascii_lowercase(); + video_control_shape = if value == "*" || value.is_empty() { + "empty" + } else if lower.starts_with("rtsps://") || lower.starts_with("rtsp://") { + "absolute-url" + } else if lower.starts_with("streamid=") { + "relative-streamid" + } else { + "other" + }; + } + } + } + if let Some((media, port)) = current.take() { + sections.push(format!("{media}/{port}")); + } + format!( + "sections=[{}] connection_present={connection_present} control_count={control_count} video_control={video_control_shape}", + sections.join(","), + ) +} + +fn media_control(sdp: &str, kind: &str) -> Option { + let mut current = ""; + for line in sdp + .split("||") + .next()? + .split(";;") + .next()? + .lines() + .map(str::trim) + { + if let Some(media) = line.strip_prefix("m=") { + current = media.split_whitespace().next().unwrap_or(""); + } else if current.eq_ignore_ascii_case(kind) + && let Some(value) = line.strip_prefix("a=control:") + && value != "*" + && !value.is_empty() + { + return Some(value.to_owned()); + } + } + None +} + +fn video_qos_offers(sdp: &str) -> VideoQosOffers { + let mut offers = VideoQosOffers::default(); + let mut video_formats = Vec::new(); + let mut in_video = false; + for line in sdp + .split("||") + .next() + .unwrap_or_default() + .split(";;") + .next() + .unwrap_or_default() + .lines() + .map(str::trim) + { + if let Some(media) = line.strip_prefix("m=") { + let mut parts = media.split_whitespace(); + if in_video { + break; + } + in_video = parts + .next() + .is_some_and(|kind| kind.eq_ignore_ascii_case("video")); + if in_video { + video_formats = parts.skip(2).collect(); + } + } else if in_video { + if let Some(fmtp) = line.strip_prefix("a=fmtp:") { + let mut parts = fmtp.splitn(2, char::is_whitespace); + if parts + .next() + .is_some_and(|format| video_formats.contains(&format)) + { + for parameter in parts.next().unwrap_or_default().split(';') { + offers.record(parameter); + } + } + } else if line.starts_with("a=") { + offers.record(line); + } + } + } + offers +} + +fn parse_hid_device_mask(value: &str) -> u32 { + let trimmed = value.trim(); + let (radix, digits) = if let Some(hex) = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + { + (16, hex) + } else if trimmed + .chars() + .any(|character| character.is_ascii_hexdigit()) + && trimmed + .chars() + .any(|character| character.is_ascii_alphabetic()) + { + (16, trimmed) + } else { + (10, trimmed) + }; + u32::from_str_radix(digits, radix).unwrap_or(0) +} + +fn qos_messages_size(sdp: &str) -> Result, NvstRtspError> { + let mut selected = None; + for line in sdp + .split("||") + .next() + .unwrap_or_default() + .split(";;") + .flat_map(str::lines) + .map(str::trim) + { + let Some(attribute) = line.strip_prefix("a=") else { + continue; + }; + let (name, value) = attribute.split_once([':', '=']).unwrap_or((attribute, "")); + if !name.eq_ignore_ascii_case("x-nv-general.maxQosMessagesSize") + && !name.eq_ignore_ascii_case("general.maxQosMessagesSize") + { + continue; + } + selected = Some(( + attribute.as_bytes().get(name.len()) == Some(&b':'), + value.trim(), + )); + } + let Some((valid_separator, value)) = selected else { + return Ok(None); + }; + if !valid_separator || value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(NvstRtspError::new( + "nvst-qos-message-size-invalid", + "Server advertised a malformed QoS message size", + )); + } + let offered = value.parse::().map_err(|_| { + NvstRtspError::new( + "nvst-qos-message-size-invalid", + "Server advertised a malformed QoS message size", + ) + })?; + if offered < MIN_CONTROL_REPORT_BYTES { + return Err(NvstRtspError::new( + "nvst-qos-message-size-unsupported", + "Server QoS message size cannot hold a complete frame report", + )); + } + Ok(Some(offered.min(MAX_CONTROL_REPORT_BYTES))) +} + +fn sdp_attribute(sdp: &str, name: &str) -> Option { + let candidates = [ + format!("a=x-nv-{name}:").to_ascii_lowercase(), + format!("a={name}:").to_ascii_lowercase(), + ]; + sdp.split("||") + .next()? + .split(";;") + .flat_map(str::lines) + .map(str::trim) + .filter_map(|line| { + let lower = line.to_ascii_lowercase(); + candidates.iter().find_map(|prefix| { + lower.strip_prefix(prefix).and_then(|_| { + line.get(prefix.len()..) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + }) + }) + }) + .last() +} + +fn official_video_setup_control(control: &str) -> String { + let lower = control.to_ascii_lowercase(); + if lower.starts_with("streamid=video/") && control.matches('/').count() == 1 { + format!("{control}/0") + } else { + control.to_owned() + } +} + +fn video_setup_candidates(control: &str, target: &str) -> Vec { + let mut candidates = vec![official_video_setup_control(control)]; + if candidates[0] != control { + candidates.push(control.to_owned()); + } + for index in 0..candidates.len() { + let control = &candidates[index]; + let lower = control.to_ascii_lowercase(); + if lower.starts_with("rtsps://") || lower.starts_with("rtsp://") { + continue; + } + let absolute = format!( + "{}/{}", + target.trim_end_matches('/'), + control.trim_start_matches('/') + ); + if !candidates.contains(&absolute) { + candidates.push(absolute); + } + } + candidates +} + +fn parse_video_peer(transport: &str) -> Option<(String, u16, u16)> { + let mut ip = None; + let mut port = None; + let mut port_end = None; + for part in transport.split([';', ',']) { + let Some((name, value)) = part.trim().split_once('=') else { + continue; + }; + if name.eq_ignore_ascii_case("source") { + ip = Some(value.trim().to_owned()); + } else if name.eq_ignore_ascii_case("X-GS-ServerPort") { + let (first, last) = value + .trim() + .split_once('-') + .unwrap_or((value.trim(), value.trim())); + let first = first.parse::().ok().filter(|port| *port != 0)?; + port = Some(first); + port_end = Some( + last.parse::() + .ok() + .filter(|last| *last >= first && *last - first < MAX_NVST_VIDEO_PEER_PORTS) + .unwrap_or(first), + ); + } + } + Some((ip?, port?, port_end?)) +} + +fn increment_hex(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let mut bytes = value.as_bytes().to_vec(); + let mut carry = true; + for byte in bytes.iter_mut().rev() { + if !carry { + break; + } + let digit = (*byte as char).to_digit(16)?; + if digit == 15 { + *byte = b'0'; + } else { + *byte = char::from_digit(digit + 1, 16)?.to_ascii_lowercase() as u8; + carry = false; + } + } + if carry { + bytes.insert(0, b'1'); + } + String::from_utf8(bytes).ok() +} + +fn resolve_remote_ufrag( + ping_payload: Option<&str>, + described_ufrag: Option<&str>, + bundle_username: Option<&str>, + ping_version: u8, +) -> Option { + if ping_version == 6 + && let Some(username) = bundle_username + { + return Some(username.to_owned()); + } + if let Some(payload) = ping_payload { + if let Some(incremented) = increment_hex(payload) { + return Some(incremented); + } + if payload.eq_ignore_ascii_case("PING") || ping_version == 6 { + return Some(payload.to_owned()); + } + } + described_ufrag.map(ToOwned::to_owned) +} + +fn bundle_natt_username(sdp: &str) -> Option { + let ufrag = sdp_attribute(sdp, "general.iceUsernameFragment") + .or_else(|| sdp_attribute(sdp, "general.iceUserNameFragmentV2"))?; + let port = sdp_attribute(sdp, "general.serverBundlePort")? + .parse::() + .ok() + .filter(|port| *port != 0)?; + let port_text = port.to_string(); + if ufrag.is_empty() + || ufrag.len() + port_text.len() > 256 + || !ufrag + .bytes() + .all(|byte| byte.is_ascii_graphic() && byte != b':') + { + return None; + } + Some(format!("{ufrag}{port_text}")) +} + +fn runtime_key(sdp: &str) -> Option<(String, u32)> { + let key = sdp_attribute(sdp, "runtime.encryptionKey")?; + if key.len() != 64 || !key.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let raw = sdp_attribute(sdp, "runtime.encryptionKeyId")? + .parse::() + .ok()?; + Some((key.to_ascii_uppercase(), raw as u32)) +} + +fn random_runtime_key() -> Result<(String, u32), NvstRtspError> { + let mut key = [0_u8; 32]; + let mut id = [0_u8; 4]; + getrandom::fill(&mut key).map_err(|error| { + NvstRtspError::new( + "randomness-unavailable", + format!("Could not generate the NVST runtime key: {error}"), + ) + })?; + getrandom::fill(&mut id).map_err(|error| { + NvstRtspError::new( + "randomness-unavailable", + format!("Could not generate the NVST runtime key ID: {error}"), + ) + })?; + Ok(( + key.iter().map(|byte| format!("{byte:02X}")).collect(), + u32::from_be_bytes(id), + )) +} + +fn set_io_timeout(socket: &mut WebSocket>, timeout: Duration) { + match socket.get_mut() { + MaybeTlsStream::Plain(stream) => { + let _ = stream.set_read_timeout(Some(timeout)); + let _ = stream.set_write_timeout(Some(timeout)); + } + MaybeTlsStream::Rustls(stream) => { + let _ = stream.get_mut().set_read_timeout(Some(timeout)); + let _ = stream.get_mut().set_write_timeout(Some(timeout)); + } + _ => {} + } +} + +#[cfg(test)] +#[path = "nvst_rtsp_control_ping_tests.rs"] +mod control_ping_tests; + +#[cfg(test)] +#[path = "nvst_rtsp_tls_tests.rs"] +mod tls_tests; + +#[cfg(test)] +#[path = "nvst_rtsp_setup_tests.rs"] +mod setup_tests; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn stream_config(context: &SessionContext) -> MediaStreamConfig { + super::super::media_stream_config(context) + } + + fn announce_color_format(context: &SessionContext) -> (u8, u8) { + let stream = stream_config(context); + ( + stream.color_quality.bit_depth(), + if stream.color_quality.is_444() { 3 } else { 1 }, + ) + } + + #[test] + fn video_setup_retains_only_bounded_advertised_port_ranges() { + for (ports, expected) in [ + ("5004", (5004, 5004)), + ("5004-5005", (5004, 5005)), + ("5004-5019", (5004, 5019)), + ("65534-65535", (65534, 65535)), + ("65535", (65535, 65535)), + ("5005-5004", (5005, 5005)), + ("5004-5020", (5004, 5004)), + ("5004-65536", (5004, 5004)), + ("5004-invalid", (5004, 5004)), + ("5004-5005-5006", (5004, 5004)), + ] { + assert_eq!( + parse_video_peer(&format!( + "unicast;X-GS-ServerPort={ports};source=192.0.2.10" + )), + Some(("192.0.2.10".to_owned(), expected.0, expected.1)), + "{ports}" + ); + } + for transport in [ + "source=192.0.2.10", + "X-GS-ServerPort=5004-5005", + "source=192.0.2.10;X-GS-ServerPort=0-1", + "source=192.0.2.10;X-GS-ServerPort=65536", + "source=192.0.2.10;X-GS-ServerPort=invalid", + ] { + assert_eq!(parse_video_peer(transport), None, "{transport}"); + } + } + + #[test] + fn http_503_is_a_control_service_failure_not_a_decoder_failure() { + let response = tungstenite::http::Response::builder() + .status(503) + .body(Some(b"private response body".to_vec())) + .unwrap(); + let error = rtsp_connect_error(&tungstenite::Error::Http(Box::new(response))); + assert_eq!(error.code, "nvst-service-unavailable"); + assert!(error.message.contains("HTTP 503")); + assert!(!error.message.contains("private response body")); + } + + #[test] + fn forbidden_upgrade_stops_signaling_fallback() { + let response = tungstenite::http::Response::builder() + .status(403) + .body(Some(b"private response body".to_vec())) + .unwrap(); + let failure = rtsp_connect_error(&tungstenite::Error::Http(Box::new(response))); + assert_eq!(failure.code, "nvst-signaling-forbidden"); + assert!(!failure.message.contains("private response body")); + let mut attempted = Vec::new(); + let error = try_signaling_endpoints(&[Some("first"), Some("second")], |endpoint| { + attempted.push(endpoint.to_owned()); + Err::<(), _>(NvstRtspError::new(failure.code, failure.message.clone())) + }) + .unwrap_err(); + assert_eq!(attempted, ["first"]); + assert_eq!(error.code, "nvst-signaling-forbidden"); + let mut attempts = 0; + let error = connect_with_retry::<()>(|| { + attempts += 1; + let response = tungstenite::http::Response::builder() + .status(403) + .body(None) + .unwrap(); + Err(tungstenite::Error::Http(Box::new(response))) + }) + .unwrap_err(); + assert_eq!(attempts, 1); + assert_eq!(error.code, "nvst-signaling-forbidden"); + } + + #[test] + fn semantic_and_auth_failures_do_not_try_other_endpoints() { + for code in [ + "nvst-offer-unsupported", + "invalid-rtsps-endpoint", + "missing-ice-credentials", + "nvst-signaling-forbidden", + "nvst-signaling-rejected", + "nvst-rtsp-failed", + ] { + let mut attempts = 0; + let failure = try_signaling_endpoints::<()>(&[Some("first"), Some("second")], |_| { + attempts += 1; + Err(NvstRtspError::new(code, "semantic failure")) + }) + .unwrap_err(); + assert_eq!(attempts, 1, "{code}"); + assert_eq!(failure.code, code); + } + for status in [401, 400, 404] { + let failure = rtsp_connect_error(&tungstenite::Error::Http(Box::new( + tungstenite::http::Response::builder() + .status(status) + .body(None) + .unwrap(), + ))); + assert!(!matches!( + failure.code, + "nvst-connect-failed" | "nvst-service-unavailable" + )); + } + } + + #[test] + fn play_rejects_invalid_state_and_readiness_has_a_bounded_deadline() { + let response = RtspResponse { + status: 455, + status_text: "Invalid State".to_owned(), + headers: HashMap::new(), + body: String::new(), + }; + assert!(ensure_rtsp_ok("PLAY", &response).is_err()); + assert!(wait_authenticated_transport(|| Ok(true), Duration::ZERO).is_ok()); + assert_eq!( + wait_authenticated_transport(|| Ok(false), Duration::from_millis(1)) + .unwrap_err() + .code, + "nvst-transport-ready-timeout" + ); + let mut calls = 0; + assert!( + wait_authenticated_transport( + || { + calls += 1; + Ok(calls > 1) + }, + Duration::from_secs(1) + ) + .is_ok() + ); + assert_eq!( + wait_authenticated_transport( + || Err(NvstRtspError::new( + "nvst-start-aborted", + "transport stopped" + )), + Duration::from_secs(20) + ) + .unwrap_err() + .code, + "nvst-start-aborted" + ); + } + + #[test] + fn announce_suppresses_only_equal_values_and_retains_unified_fallback() { + let announce = "a=x-nv-general.enableUnifiedSocket:1\r\na=x-nv-ri.timestampsEnabled:1\r\na=x-nv-runtime.videoSrtp:1\r\n"; + let describe = "a=x-nv-general.enableUnifiedSocket:0\r\na=x-nv-ri.timestampsEnabled:01\r\n;;a=x-nv-runtime.videoSrtp:1\r\n||a=x-nv-general.enableUnifiedSocket:1\r\n"; + let filtered = omit_server_announce_attributes(announce, describe); + assert_eq!( + sdp_attribute(&filtered, "general.enableUnifiedSocket").as_deref(), + Some("1") + ); + assert_eq!(sdp_attribute(&filtered, "ri.timestampsEnabled"), None); + assert_eq!(sdp_attribute(&filtered, "runtime.videoSrtp"), None); + assert_eq!( + offered_bool( + "a=x-nv-general.enableUnifiedSocket:2\r\n", + "general.enableUnifiedSocket" + ), + None + ); + } + + #[test] + fn negotiation_preserves_server_limits_and_initial_rate_without_deriving_a_ratio() { + let value = context(); + let announce = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: true, + qos_timings_v5: false, + unified_socket: false, + }, + ); + let offer = "a=x-nv-video[0].initialBitrateKbps:12135\r\na=x-nv-video[0].initialPeakBitrateKbps:12135\r\na=x-nv-vqos[0].fec.repairMinPercent:5\r\na=x-nv-vqos[0].fec.repairMaxPercent:40\r\na=x-nv-vqos[0].bllFec.enable:1\r\na=x-nv-packetPacing.minNumPacketsPerGroup:0\r\na=x-nv-packetPacing.maxDelayUs:1000\r\n"; + let effective = negotiated_announce(&announce, offer, &mut json!({})).unwrap(); + for (name, expected) in [ + ("video[0].initialBitrateKbps", "12135"), + ("video[0].initialPeakBitrateKbps", "12135"), + ("vqos[0].fec.repairMinPercent", "5"), + ("vqos[0].fec.repairMaxPercent", "40"), + ("vqos[0].bllFec.enable", "1"), + ("packetPacing.minNumPacketsPerGroup", "0"), + ("packetPacing.maxDelayUs", "1000"), + ] { + assert_eq!(sdp_attribute(&effective, name).as_deref(), Some(expected)); + assert_eq!( + sdp_attribute(&omit_server_announce_attributes(&effective, offer), name), + None + ); + } + assert_eq!( + sdp_attribute(&effective, "vqos[0].bw.maximumBitrateKbps"), + sdp_attribute(&announce, "vqos[0].bw.maximumBitrateKbps") + ); + assert_eq!( + sdp_attribute( + &negotiated_announce(&announce, "", &mut json!({})).unwrap(), + "video[0].initialBitrateKbps" + ), + sdp_attribute(&announce, "video[0].initialBitrateKbps") + ); + let defaults = negotiated_announce(&announce, "", &mut json!({})).unwrap(); + for unsupported in [ + "a=x-nv-vqos[0].fec.repairMinPercent:101", + "a=x-nv-vqos[0].fec.repairMinPercent:40\r\na=x-nv-vqos[0].fec.repairMaxPercent:5", + "a=x-nv-packetPacing.version:4", + "a=x-nv-video[0].framePacing.mode:0", + "a=x-nv-video[0].framePacing.mode:3", + "a=x-nv-video[0].framePacing.feedbackMode:2", + "a=x-nv-general.disablePlay:2", + "a=x-nv-video[0].initialBitrateKbps:0", + "a=x-nv-vqos[0].fec.enable:yes", + ] { + assert_eq!( + negotiated_announce(&announce, unsupported, &mut json!({})).unwrap(), + defaults, + "{unsupported}" + ); + } + } + + #[test] + fn transient_connection_retries_are_bounded_per_endpoint() { + let mut attempts = 0; + let result = connect_with_retry(|| { + attempts += 1; + if attempts < 3 { + Err(tungstenite::Error::Io(std::io::Error::from( + ErrorKind::ConnectionRefused, + ))) + } else { + Ok(42) + } + }) + .unwrap(); + assert_eq!(result, 42); + assert_eq!(attempts, 3); + let mut attempts = 0; + let error = connect_with_retry::<()>(|| { + attempts += 1; + Err(tungstenite::Error::Io(std::io::Error::from( + ErrorKind::ConnectionRefused, + ))) + }) + .unwrap_err(); + assert_eq!(attempts, 3); + assert_eq!(error.code, "nvst-connect-failed"); + } + + #[test] + fn signaling_attempts_every_endpoint_in_order_with_a_fresh_handshake() { + let mut attempted = Vec::new(); + let session = try_signaling_endpoints( + &[Some("unavailable"), Some("working"), Some("unused")], + |endpoint| { + attempted.push(endpoint.to_owned()); + match endpoint { + "unavailable" => { + Err(NvstRtspError::new("nvst-service-unavailable", "HTTP 503")) + } + _ => Ok(endpoint.to_owned()), + } + }, + ) + .unwrap(); + assert_eq!(session, "working"); + assert_eq!(attempted, ["unavailable", "working"]); + + let error = try_signaling_endpoints::<()>( + &[Some("unavailable"), Some("unavailable")], + |endpoint| { + Err(NvstRtspError::new( + if endpoint == "invalid" { + "invalid-rtsps-endpoint" + } else { + "nvst-service-unavailable" + }, + "failed", + )) + }, + ) + .unwrap_err(); + assert_eq!(error.code, "nvst-service-unavailable"); + assert!( + error + .message + .contains("1:nvst-service-unavailable, 2:nvst-service-unavailable") + ); + assert_eq!( + try_signaling_endpoints::<()>(&[], |_| Ok(())) + .unwrap_err() + .code, + "missing-rtsps-endpoint" + ); + } + + #[test] + fn bundle_peer_prefers_the_cloudmatch_bundle_mapping_over_video() { + let mut context = context(); + context.session.media_connection_info = Some( + serde_json::from_value(json!({"ip":"203.0.113.2","port":17181,"usage":2})).unwrap(), + ); + context.session.connection_info = Some( + serde_json::from_value(json!([ + {"ip":"203.0.113.2","port":17181,"usage":2}, + {"ip":"seat.partner.example","port":17999,"usage":17}, + {"ip":"203.0.113.2","port":0,"usage":17}, + {"ip":"203.0.113.2","port":17182,"usage":17}, + {"ip":"203.0.113.2","port":322,"usage":16} + ])) + .unwrap(), + ); + let endpoint = bundle_endpoint(&context).expect("bundle endpoint"); + assert_eq!((endpoint.ip, endpoint.port), ("203.0.113.2", 17182)); + assert_eq!(endpoint.source, "cloudmatch-bundle"); + assert_eq!( + connection_usage_summary(&context), + "2:17181,17:17999,17:0,17:17182,16:322" + ); + + context.session.connection_info = None; + let endpoint = bundle_endpoint(&context).expect("media endpoint"); + assert_eq!((endpoint.ip, endpoint.port), ("203.0.113.2", 17181)); + assert_eq!(endpoint.source, "cloudmatch-media"); + + context.session.media_connection_info = None; + assert!(bundle_endpoint(&context).is_none()); + } + + fn context() -> SessionContext { + serde_json::from_value(json!({ + "session": { + "sessionId": "session", + "serverIp": "seat.nvidiagrid.net", + "rtspsEndpoints": ["rtsps://seat.nvidiagrid.net:322/session"], + "iceServers": [], + "negotiatedStreamProfile": { + "codec": "AV1", + "fps": 120, + "colorQuality": "10bit_444" + } + }, + "settings": { + "transportMode": "nvst", + "codec": "H264", + "resolution": "2560x1440", + "fps": 60, + "maxBitrateMbps": 75 + }, + "shortcuts": {} + })) + .expect("context") + } + + #[test] + fn rtsps_supports_tls12_and_tls13_without_legacy_versions() { + let versions: Vec<_> = rustls::DEFAULT_VERSIONS + .iter() + .map(|version| version.version) + .collect(); + assert_eq!( + versions, + vec![ + rustls::ProtocolVersion::TLSv1_3, + rustls::ProtocolVersion::TLSv1_2 + ] + ); + } + + #[test] + fn installs_a_process_level_tls_crypto_provider() { + ensure_tls_crypto_provider().expect("TLS provider"); + assert!(rustls::crypto::CryptoProvider::get_default().is_some()); + } + + #[test] + fn announce_reports_the_selected_socket_layout() { + let value = context(); + for (unified_socket, expected) in [ + (false, "a=x-nv-general.enableUnifiedSocket:0\r\n"), + (true, "a=x-nv-general.enableUnifiedSocket:1\r\n"), + ] { + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket, + }, + ); + assert!(sdp.contains(expected)); + } + } + + #[test] + fn announce_carries_the_documented_top_tier_frame_rate() { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"] = json!({"codec":"AV1", "fps":360}); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].maxFPS:360")); + assert!(sdp.contains("a=x-nv-packetPacing.maxDelayUs:4000")); + + let mut runaway = context(); + runaway.session.extra["negotiatedStreamProfile"] = json!({"codec":"AV1", "fps":600}); + let sdp = build_announce( + &runaway, + AnnounceParams { + stream: stream_config(&runaway), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].maxFPS:360")); + assert!(!sdp.contains("a=x-nv-video[0].maxFPS:600")); + } + + #[test] + fn announce_with_v5_timings_does_not_request_legacy_pacing_feedback() { + let value = context(); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: true, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].framePacing.mode:2\r\n")); + assert!(sdp.contains("a=x-nv-video[0].framePacing.feedbackMode:0\r\n")); + } + + #[test] + fn owned_announce_matches_current_official_bundle_baseline() { + let value = context(); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].maxFPS:120")); + assert!(sdp.contains("a=x-nv-video[0].bitDepth:10")); + assert!(sdp.contains("a=x-nv-video[0].chromaFormat:1")); + assert!(sdp.contains("a=x-nv-video[0].maxCodecProfile:3")); + assert!(sdp.contains("a=x-nv-video[0].maxCodecLevel:51")); + assert!(sdp.contains("a=x-nv-video[0].maxH264Level:51")); + assert!(!sdp.contains("a=x-nv-video[0].videoSplitEncodeStripsPerFrame:")); + assert!(!sdp.contains("a=x-nv-vqos[0].grc.enable:")); + assert!(!sdp.contains("a=x-nv-clientSupportHevc:")); + assert!(sdp.contains("a=x-nv-video[0].encoderCscMode:2")); + assert!(sdp.contains("a=x-nv-vqos[0].bitStreamFormat:2")); + assert!(sdp.contains("a=x-nv-general.clientBundlePort:49006")); + assert!(sdp.contains("a=x-nv-general.rtcDataChannelOnNativeBundle:1")); + assert!(sdp.contains("a=x-nv-runtime.encryptionKey:")); + assert!(sdp.contains("m=video 5004")); + } + + #[test] + fn announce_preserves_the_route_selected_video_packet_size() { + for video_packet_size in [1280, 1216, 1200, 1136] { + let sdp = build_announce( + &context(), + AnnounceParams { + stream: stream_config(&context()), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert_eq!( + sdp_attribute(&sdp, "video[0].packetSize"), + Some(video_packet_size.to_string()) + ); + } + } + + #[test] + fn announce_uses_the_measured_authenticated_path_when_it_is_tighter() { + let mut context = context(); + context.session.extra.insert( + "networkTest".to_owned(), + json!({"sessionId":"nt-1", "measuredDatagramBytes":1_200}), + ); + let peer: IpAddr = "192.0.2.1".parse().unwrap(); + let packet_size = measured_path_packet_size(&context, peer).expect("measured packet size"); + assert_eq!(packet_size, 1_168); + assert!(packet_size < nvst_video_packet_size(peer).unwrap()); + + let sdp = build_announce( + &context, + AnnounceParams { + stream: stream_config(&context), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: packet_size, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert_eq!( + sdp_attribute(&sdp, "video[0].packetSize"), + Some(packet_size.to_string()) + ); + } + + #[test] + fn announce_ignores_an_absent_or_unusable_measurement() { + let base = context(); + let peer: IpAddr = "192.0.2.1".parse().unwrap(); + assert_eq!(measured_path_packet_size(&base, peer), None); + + for measured in [0_u64, 1, 12] { + let mut context = context(); + context.session.extra.insert( + "networkTest".to_owned(), + json!({"measuredDatagramBytes":measured}), + ); + assert_eq!( + measured_path_packet_size(&context, peer), + None, + "{measured}" + ); + } + } + + #[test] + fn announce_prefers_a_finalized_policy_but_falls_back_to_live_settings() { + // The official client never sends dynamicStreamingMode to CloudMatch, so a + // negotiated value can only be a server-finalized override. Otherwise the + // live saveBandwidth setting decides, matching the official RTSP-only policy. + for (profile, save, policy, adjust) in [ + (None, false, 0, 0), + (None, true, 1, 1), + (Some(1), false, 1, 1), + (Some(2), false, 2, 1), + (Some(3), false, 3, 1), + (Some(7), true, 1, 1), + (Some(7), false, 0, 0), + ] { + let mut value = context(); + value.settings["saveBandwidth"] = json!(save); + if let Some(profile) = profile { + value.session.extra["negotiatedStreamProfile"]["dynamicStreamingMode"] = + json!(profile); + } + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains(&format!("a=x-nv-vqos[0].dynamicStreamingMode:{policy}\r\n"))); + assert!(sdp.contains(&format!("a=x-nv-vqos[0].dfc.adjustResAndFps:{adjust}\r\n"))); + assert!(sdp.contains("a=x-nv-vqos[0].drc.enable:0\r\n")); + } + } + + #[test] + fn announce_dynamic_range_follows_accepted_hdr_not_saved_intent() { + for (accepted, requested, hdr) in [ + (json!(true), false, true), + (json!(false), true, false), + (Value::Null, true, false), + ] { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"]["codec"] = json!("H265"); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!("10bit_420"); + value.session.extra["negotiatedStreamProfile"]["enableHdr"] = accepted; + value.settings["enableHdr"] = json!(requested); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + // HDR carries an explicit :1; SDR omits the line, like the official client. + assert_eq!(sdp.contains("a=x-nv-video[0].dynamicRangeMode:1\r\n"), hdr); + assert!(!sdp.contains("a=x-nv-video[0].dynamicRangeMode:0")); + assert!(sdp.contains("a=x-nv-video[0].bitDepth:10\r\n")); + assert!(sdp.contains("a=x-nv-video[0].chromaFormat:1\r\n")); + } + } + + #[test] + fn owned_announce_preserves_the_configured_200_mbps_ceiling() { + let mut value = context(); + value.settings["maxBitrateMbps"] = json!(200); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].initialBitrateKbps:200000")); + assert!(sdp.contains("a=x-nv-video[0].initialPeakBitrateKbps:200000")); + assert!(sdp.contains("a=x-nv-vqos[0].bw.maximumBitrateKbps:200000")); + assert!(sdp.contains("a=x-nv-vqos[0].bw.minimumBitrateKbps:1000")); + } + + #[test] + fn owned_announce_can_request_220_kbps_without_a_1_mbps_floor() { + let mut value = context(); + value.settings["maxBitrateMbps"] = json!(0.22); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert!(sdp.contains("a=x-nv-video[0].initialBitrateKbps:220")); + assert!(sdp.contains("a=x-nv-vqos[0].bw.maximumBitrateKbps:220")); + assert!(sdp.contains("a=x-nv-vqos[0].bw.minimumBitrateKbps:220")); + } + + #[test] + fn microphone_announce_requires_request_and_server_offer() { + let mut value = context(); + assert!(negotiate_microphone( + &value, + "a=x-nv-general.rtcMicOnNativeBundle:1\r\n" + )); + for mode in ["disabled", "voice-activity", "push-to-talk"] { + value.settings["microphoneMode"] = json!(mode); + for offer in [ + "", + "a=x-nv-general.rtcMicOnNativeBundle:0\r\n", + "a=x-nv-general.rtcMicOnNativeBundle:1\r\n", + ] { + let available = negotiate_microphone(&value, offer); + assert_eq!(available, offer.contains(":1")); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: available, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert_eq!( + sdp.contains("a=x-nv-general.rtcMicOnNativeBundle:1\r\n"), + available + ); + assert_eq!( + sdp.contains("a=x-nv-mic.micSsrcConfig.senderSsrc:1\r\n"), + available + ); + assert_eq!(sdp.contains("rtcMicOnNativeBundle"), available); + assert!(sdp.contains("a=x-nv-general.rtcAudioOnNativeBundle:1\r\n")); + } + } + } + + #[test] + fn h264_announce_stays_eight_bit_420() { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"]["codec"] = json!("H264"); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!("10bit_444"); + assert_eq!(announce_color_format(&value), (8, 1)); + } + + #[test] + fn av1_announce_stays_420_but_preserves_ten_bit_depth() { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!("10bit_444"); + assert_eq!(announce_color_format(&value), (10, 1)); + } + + #[test] + fn accepted_hevc_color_preserves_nvst_depth_and_chroma_enum_space() { + for (color, format) in [ + ("8bit_420", (8, 1)), + ("8bit_444", (8, 3)), + ("10bit_420", (10, 1)), + ("10bit_444", (10, 3)), + ] { + let mut value = context(); + value.settings["colorQuality"] = json!("8bit_420"); + value.session.extra["negotiatedStreamProfile"]["codec"] = json!("H265"); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!(color); + assert_eq!(announce_color_format(&value), format); + } + } + + #[test] + fn h265_announce_supports_ten_bit_444() { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"]["codec"] = json!("H265"); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!("10bit_444"); + assert_eq!(announce_color_format(&value), (10, 3)); + } + + #[test] + fn full_announce_always_states_depth_chroma_and_hdr_explicitly() { + for (codec, color, hdr, depth, chroma) in [ + ("H265", "10bit_444", false, "10", "3"), + ("H265", "10bit_420", false, "10", "1"), + ("H264", "8bit_420", false, "8", "1"), + ] { + let mut value = context(); + value.session.extra["negotiatedStreamProfile"]["codec"] = json!(codec); + value.session.extra["negotiatedStreamProfile"]["colorQuality"] = json!(color); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: false, + qos_timings_v5: false, + unified_socket: false, + }, + ); + assert_eq!( + sdp_attribute(&sdp, "video[0].bitDepth"), + Some(depth.to_owned()) + ); + assert_eq!( + sdp_attribute(&sdp, "video[0].chromaFormat"), + Some(chroma.to_owned()) + ); + // SDR never carries a dynamic-range line; HDR always carries :1. + assert_eq!( + sdp_attribute(&sdp, "video[0].dynamicRangeMode"), + hdr.then(|| "1".to_owned()) + ); + for field in ["bitDepth", "chromaFormat"] { + assert_eq!( + sdp.matches(&format!("a=x-nv-video[0].{field}:")).count(), + 1, + "{codec}/{color}" + ); + } + // Encoder identity the seat reads before initializing. + for line in [ + "a=x-nv-video[0].maxCodecProfile:3", + "a=x-nv-video[0].maxCodecLevel:51", + "a=x-nv-video[0].maxH264Profile:3", + "a=x-nv-video[0].maxH264Level:51", + ] { + assert!(sdp.contains(line), "{line}"); + } + } + } + + #[test] + fn announce_does_not_echo_server_config_or_invent_server_owned_values() { + let mut value = context(); + value.settings["maxBitrateMbps"] = json!(150); + let sdp = build_announce( + &value, + AnnounceParams { + stream: stream_config(&value), + key: &"01".repeat(32), + key_id: 7, + port: 49006, + address: "192.0.2.10", + ufrag: "abcd", + password: "abcdefghijklmnopqrstuv", + fingerprint: "AA:BB", + video_port: 5004, + video_packet_size: 1280, + rtcp_on_sctp: true, + microphone_available: true, + qos_timings_v5: false, + unified_socket: false, + }, + ); + for server_value in ["0", "23"] { + let server_bitrate = if server_value == "0" { 100_000 } else { 50_000 }; + let describe = format!( + "v=0\r\na=x-nv-video[0].framePacing.mode:2\r\na=x-nv-vqos[0].fec.repairMinPercent:5\r\na=x-nv-vqos[0].grc.enable:{server_value}\r\na=x-nv-vqos[0].bw.maximumBitrateKbps:{server_bitrate}\r\na=x-nv-video[0].maxCodecLevel:61\r\na=x-nv-video[0].chromaFormat:0\r\na=x-nv-video[0].packetSize:1408\r\n;;v=0\r\na=x-nv-video[0].framePacing.feedbackMode:0\r\na=x-nv-packetPacing.maxDelayUs:1000\r\na=x-nv-packetPacing.minNumPacketsPerGroup:0\r\n||v=0\r\na=x-nv-vqos[0].fec.enable:0\r\n" + ); + assert_eq!( + sdp_attribute(&describe, "video[0].framePacing.feedbackMode"), + Some("0".to_owned()) + ); + let mut handoff = json!({"framePacingFeedbackMode":0}); + let effective = negotiated_announce(&sdp, &describe, &mut handoff).unwrap(); + let announce = omit_server_announce_attributes(&effective, &describe); + for field in [ + "vqos[0].fec.repairMinPercent", + "vqos[0].grc.enable", + "packetPacing.maxDelayUs", + "packetPacing.minNumPacketsPerGroup", + ] { + assert_eq!(sdp_attribute(&announce, field), None, "{field}"); + } + assert_eq!( + sdp_attribute(&announce, "video[0].framePacing.mode").as_deref(), + Some("1") + ); + assert_eq!( + sdp_attribute(&announce, "video[0].framePacing.feedbackMode").as_deref(), + Some("1") + ); + assert_eq!( + handoff["framePacingFeedbackMode"], 1, + "transport must use the announced client intent, not the differing server baseline" + ); + assert_eq!( + sdp_attribute(&announce, "vqos[0].fec.enable"), + Some("1".to_owned()) + ); + assert_eq!( + sdp_attribute(&announce, "video[0].maxCodecLevel"), + Some("51".to_owned()) + ); + assert_eq!( + sdp_attribute(&announce, "video[0].chromaFormat"), + Some("1".to_owned()) + ); + assert_eq!( + sdp_attribute(&announce, "vqos[0].bw.maximumBitrateKbps"), + Some("150000".to_owned()) + ); + for (field, value) in [ + ("video[0].packetSize", "1280"), + ("video[0].maxFPS", "120"), + ("general.iceUserNameFragmentV2", "abcd"), + ("general.rtcMicOnNativeBundle", "1"), + ("mic.micSsrcConfig.senderSsrc", "1"), + ] { + assert_eq!(sdp_attribute(&announce, field).as_deref(), Some(value)); + } + for server_owned in [ + "video[0].videoSplitEncodeStripsPerFrame", + "video[0].adaptiveQuantization.spatialAQStrength", + "vqos[0].grc.enable", + "aqos.redundancyLevel", + "general.enetControlChannel.mtuSize", + ] { + assert_eq!( + sdp_attribute(&announce, server_owned), + None, + "{server_owned}" + ); + } + } + for invalid_cap in ["0", "not-a-number"] { + let describe = format!("a=x-nv-vqos[0].bw.maximumBitrateKbps:{invalid_cap}\r\n"); + let announce = omit_server_announce_attributes(&sdp, &describe); + assert_eq!( + sdp_attribute(&announce, "vqos[0].bw.maximumBitrateKbps"), + Some("150000".to_owned()), + "{invalid_cap}" + ); + } + } + + #[test] + fn rtsp_parser_waits_for_body_and_checks_cseq() { + let mut buffer = "RTSP/1.0 200 OK\r\nCSeq: 3\r\nContent-Length: 4\r\n\r\ntest".to_owned(); + let response = take_rtsp_response(&mut buffer, 3).unwrap().unwrap(); + assert_eq!(response.status, 200); + assert_eq!(response.body, "test"); + assert!(buffer.is_empty()); + } + + #[test] + fn rtsp_parser_accepts_matching_request_id_when_setup_omits_cseq() { + let mut buffer = "RTSP/1.0 200 OK\r\nRequest-Id: 3\r\nContent-Length: 0\r\n\r\n".to_owned(); + assert!(take_rtsp_response(&mut buffer, 3).unwrap().is_some()); + + let mut uncorrelated = "RTSP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n".to_owned(); + assert!(take_rtsp_response(&mut uncorrelated, 3).is_err()); + } + + #[test] + fn rtsp_parser_ignores_blank_transport_padding_before_next_response() { + let mut buffer = + "\r\n\r\nRTSP/1.0 200 OK\r\nCSeq: 4\r\nRequest-Id: 4\r\nContent-Length: 0\r\n\r\n" + .to_owned(); + let response = take_rtsp_response(&mut buffer, 4).unwrap().unwrap(); + assert_eq!(response.status, 200); + assert!(buffer.is_empty()); + } + + #[test] + fn official_setup_preserves_the_relative_video_control_target() { + assert_eq!( + official_video_setup_control("streamid=video/0"), + "streamid=video/0/0" + ); + assert_eq!( + official_video_setup_control("streamid=video/0/0"), + "streamid=video/0/0" + ); + } + + #[test] + fn describe_features_override_main_without_reading_upstream_offer() { + let describe = "v=0\r\na=x-nv-general.disablePlay:0\r\na=x-nv-general.nativeRtcOnBundlePort:0\r\nm=video 5004\r\na=control:streamid=video/0\r\n;;v=0\r\na=x-nv-general.nativeRtcOnBundlePort:1\r\na=x-nv-general.disablePlay:1\r\nm=video 6000\r\na=control:wrong-control\r\n||v=0\r\na=x-nv-general.disablePlay:0\r\n"; + assert_eq!( + sdp_attribute(describe, "general.nativeRtcOnBundlePort").as_deref(), + Some("1") + ); + assert_eq!( + sdp_attribute(describe, "general.disablePlay").as_deref(), + Some("1") + ); + assert_eq!( + media_control(describe, "video").as_deref(), + Some("streamid=video/0") + ); + assert_eq!( + media_control("v=0\r\n||m=video 5004\r\na=control:upstream\r\n", "video"), + None + ); + assert_eq!(sdp_attribute("v=0\r\n", "general.disablePlay"), None); + } + + #[test] + fn describe_video_qos_offers_parse_official_fmtp_and_attributes() { + let describe = "v=0\r\na=nv-video-qos-feedback-version:99\r\n\ + m=audio 5006 RTP/SAVPF 111\r\na=fmtp:111 nv-video-qos-feedback-version=99\r\n\ + m=video 5004 RTP/SAVPF 96 97\r\n\ + a=fmtp:111 nv-video-qos-feedback-version=99\r\n\ + a=fmtp:96 packetization-mode=1; nv-video-qos-feedback-version=7; nv-video-qos-timings-version=5\r\n\ + a=nv-video-qos-blob-stats-version:9\r\n\ + m=application 6000 UDP/DTLS/SCTP webrtc-datachannel\r\n\ + a=nv-video-qos-blob-stats-version:99\r\n"; + let offers = video_qos_offers(describe); + assert_eq!( + offers, + VideoQosOffers { + feedback: QosVersionOffer::Version(7), + timings: QosVersionOffer::Version(5), + blob_stats: QosVersionOffer::Version(9), + } + ); + let mut handoff = json!({}); + offers.add_to_handoff(&mut handoff); + assert_eq!( + handoff, + json!({"qosFeedbackVersion":7,"qosTimingsVersion":5,"qosBlobStatsVersion":9}) + ); + assert!(offers.uses_v5_timings()); + assert_eq!( + offers + .pacing_feedback_mode( + "m=video 5004 RTP/SAVPF 96\r\na=x-nv-video[0].framePacing.mode:2\r\na=x-nv-video[0].framePacing.feedbackMode:0\r\n", + ) + .unwrap(), + 0 + ); + assert_eq!( + offers + .pacing_feedback_mode( + "m=video 5004 RTP/SAVPF 96\r\na=x-nv-video[0].framePacing.mode:1\r\na=x-nv-video[0].framePacing.feedbackMode:1\r\n", + ) + .unwrap(), + 1 + ); + assert_eq!(offers.pacing_feedback_mode("v=0\r\n").unwrap(), 0); + assert_eq!( + VideoQosOffers::default() + .pacing_feedback_mode("v=0\r\n") + .unwrap(), + 1 + ); + assert_eq!( + offers + .pacing_feedback_mode("a=x-nv-video[0].framePacing.feedbackMode:2\r\n") + .unwrap_err() + .code, + "nvst-qos-pacing-unsupported" + ); + } + + #[test] + fn describe_qos_message_size_defaults_and_clamps_without_exceeding_safe_limit() { + assert_eq!(qos_messages_size("v=0\r\n").unwrap(), None); + for (offer, expected) in [(106, 106), (1071, 1071), (4000, 1071)] { + let describe = format!( + "v=0\r\na=x-nv-general.maxQosMessagesSize:{offer}\r\n;;a=general.disablePlay:1\r\n" + ); + assert_eq!(qos_messages_size(&describe).unwrap(), Some(expected)); + } + assert_eq!( + qos_messages_size("a=general.maxQosMessagesSize:212\r\n").unwrap(), + Some(212) + ); + assert_eq!( + qos_messages_size( + "a=x-nv-general.maxQosMessagesSize:106\r\n;;a=general.maxQosMessagesSize:212\r\n" + ) + .unwrap(), + Some(212) + ); + assert_eq!( + qos_messages_size( + "a=x-nv-general.maxQosMessagesSize:bad\r\n;;a=general.maxQosMessagesSize:212\r\n" + ) + .unwrap(), + Some(212) + ); + assert_eq!( + qos_messages_size("v=0\r\n||a=x-nv-general.maxQosMessagesSize:106\r\n").unwrap(), + None + ); + } + + #[test] + fn describe_qos_message_size_rejects_malformed_and_too_small_effective_offers() { + for value in ["", "-1", "+106", "106x", "999999999999999999999999999999"] { + let describe = format!("a=x-nv-general.maxQosMessagesSize:{value}\r\n"); + assert_eq!( + qos_messages_size(&describe).unwrap_err().code, + "nvst-qos-message-size-invalid", + "{value}" + ); + } + assert_eq!( + qos_messages_size("a=x-nv-general.maxQosMessagesSize\r\n") + .unwrap_err() + .code, + "nvst-qos-message-size-invalid" + ); + assert_eq!( + qos_messages_size("a=x-nv-general.maxQosMessagesSize=106\r\n") + .unwrap_err() + .code, + "nvst-qos-message-size-invalid" + ); + assert_eq!( + qos_messages_size( + "a=x-nv-general.maxQosMessagesSize:212\r\n;;a=general.maxQosMessagesSize:bad\r\n" + ) + .unwrap_err() + .code, + "nvst-qos-message-size-invalid" + ); + for value in [0, 56, 105] { + let describe = format!("a=x-nv-general.maxQosMessagesSize:{value}\r\n"); + assert_eq!( + qos_messages_size(&describe).unwrap_err().code, + "nvst-qos-message-size-unsupported" + ); + } + } + + #[test] + fn describe_video_qos_offers_retain_lower_and_higher_versions() { + for (feedback, timings, blob_stats) in [(5, 3, 8), (8, 6, 10), (0, 255, 9)] { + let describe = format!( + "v=0\r\nm=video 5004 RTP/SAVPF 96\r\n\ + a=fmtp:96 nv-video-qos-feedback-version={feedback};nv-video-qos-timings-version={timings};nv-video-qos-blob-stats-version={blob_stats}\r\n" + ); + let offers = video_qos_offers(&describe); + assert_eq!(offers.feedback, QosVersionOffer::Version(feedback)); + assert_eq!(offers.timings, QosVersionOffer::Version(timings)); + assert_eq!(offers.blob_stats, QosVersionOffer::Version(blob_stats)); + if feedback < 7 || timings < 5 || blob_stats < 9 { + assert_eq!( + offers.validate().unwrap_err().code, + "nvst-qos-version-unsupported" + ); + } else { + offers.validate().unwrap(); + } + let mut handoff = json!({}); + offers.add_to_handoff(&mut handoff); + assert_eq!(handoff["qosFeedbackVersion"], feedback); + assert_eq!(handoff["qosTimingsVersion"], timings); + assert_eq!(handoff["qosBlobStatsVersion"], blob_stats); + } + } + + #[test] + fn describe_video_qos_offers_distinguish_malformed_from_missing() { + for malformed in [ + "", + "-1", + "+7", + "7x", + "256", + "999999999999999999999999999999", + ] { + let describe = format!( + "m=video 5004 RTP/SAVPF 96\r\n\ + a=fmtp:96 nv-video-qos-feedback-version={malformed};nv-video-qos-timings-version=5\r\n" + ); + let offers = video_qos_offers(&describe); + assert_eq!(offers.feedback, QosVersionOffer::Malformed, "{malformed}"); + assert_eq!( + offers.validate().unwrap_err().code, + "nvst-qos-version-invalid" + ); + assert_eq!(offers.timings, QosVersionOffer::Version(5)); + assert_eq!(offers.blob_stats, QosVersionOffer::Missing); + let mut handoff = json!({}); + offers.add_to_handoff(&mut handoff); + assert_eq!(handoff, json!({"qosTimingsVersion":5})); + } + let offers = video_qos_offers( + "m=video 5004 RTP/SAVPF 96\r\na=nv-video-qos-feedback-version\r\n\ + a=nv-video-qos-timings-version:5\r\na=nv-video-qos-timings-version:6\r\n", + ); + assert_eq!(offers.feedback, QosVersionOffer::Malformed); + assert_eq!(offers.timings, QosVersionOffer::Malformed); + assert_eq!(offers.blob_stats, QosVersionOffer::Missing); + } + + #[test] + fn describe_video_qos_offers_read_only_first_describe_video_section() { + let describe = "v=0\r\nm=video 5004 RTP/SAVPF 96\r\n\ + a=fmtp:96 nv-video-qos-feedback-version=7\r\n\ + m=video 5006 RTP/SAVPF 97\r\na=nv-video-qos-timings-version:5\r\n\ + ;;v=0\r\nm=video 6000 RTP/SAVPF 96\r\na=nv-video-qos-blob-stats-version:9\r\n\ + ||v=0\r\nm=video 7000 RTP/SAVPF 96\r\na=nv-video-qos-timings-version:8\r\n"; + assert_eq!( + video_qos_offers(describe), + VideoQosOffers { + feedback: QosVersionOffer::Version(7), + timings: QosVersionOffer::Missing, + blob_stats: QosVersionOffer::Missing, + } + ); + assert_eq!( + video_qos_offers( + "v=0\r\n||m=video 5004 RTP/SAVPF 96\r\na=nv-video-qos-feedback-version:7\r\n" + ), + VideoQosOffers::default() + ); + } + + #[test] + fn describe_qos_log_shape_excludes_offers_and_credentials() { + let describe = "v=0\r\nm=video 5004 RTP/SAVPF 96\r\n\ + a=fmtp:96 nv-video-qos-feedback-version=7;password=secret-credential\r\n\ + a=nv-video-qos-timings-version:5\r\n\ + a=x-nv-general.icePasswordV2:secret-credential\r\n"; + let shape = describe_media_shape(describe); + assert_eq!( + shape, + "sections=[video/5004] connection_present=false control_count=0 video_control=absent" + ); + assert!(!shape.contains("nv-video-qos")); + assert!(!shape.contains("secret-credential")); + let malformed = describe_media_shape( + "m=secret-credential 5004 RTP/SAVPF 96\r\nm=video secret-credential RTP/SAVPF 96\r\n", + ); + assert_eq!( + malformed, + "sections=[other/5004,video/?] connection_present=false control_count=0 video_control=absent" + ); + assert!(!malformed.contains("secret-credential")); + } + + #[test] + fn describe_shape_reports_sections_and_controls_without_values() { + let shape = describe_media_shape( + "v=0\r\no=- 0 0 IN IP4 203.0.113.9\r\nc=IN IP4 203.0.113.9\r\n\ + m=video 5004 RTP/SAVPF 96\r\na=control:streamid=video/0\r\n\ + a=x-nv-general.iceUserNameFragment:secret\r\n\ + m=audio 5006 RTP/SAVPF 111\r\na=control:*\r\n", + ); + assert_eq!( + shape, + "sections=[video/5004,audio/5006] connection_present=true \ + control_count=2 video_control=relative-streamid" + ); + assert!(!shape.contains("203.0.113.9")); + assert!(!shape.contains("secret")); + assert!(!shape.contains("streamid=video/0")); + let disabled = describe_media_shape("v=0\r\nm=video 0 RTP/SAVPF 96\r\n"); + assert!(disabled.contains("video/0")); + assert!(disabled.contains("video_control=absent")); + } + + #[test] + fn endpoint_urls_preserve_one_ipv6_bracket_pair_and_the_selected_port() { + for (endpoint, port) in [ + ("rtsps://[2001:4860:4860::8888]:48322/session", 48322), + ("rtsps://[2001:4860:4860::8888]/session", 322), + ] { + let (wss, target) = rtsp_endpoint_urls(endpoint).unwrap(); + let authority = format!("[2001:4860:4860::8888]:{port}"); + assert_eq!(wss, format!("wss://{authority}/rtsp")); + assert_eq!(target, format!("rtsps://{authority}")); + let request = wss.into_client_request().unwrap(); + assert_eq!(request.headers()["host"], authority); + assert_eq!(request.uri().host(), Some("[2001:4860:4860::8888]")); + assert_eq!(request.uri().port_u16(), Some(port)); + let target = target.parse::().unwrap(); + assert_eq!(target.authority().unwrap().as_str(), authority); + } + } + + #[test] + fn endpoint_urls_preserve_dns_and_ipv4_behavior() { + for (endpoint, authority) in [ + ( + "rtsps://seat.nvidiagrid.net/session", + "seat.nvidiagrid.net:322", + ), + ("rtsps://8.8.8.8:48322/session", "8.8.8.8:48322"), + ] { + assert_eq!( + rtsp_endpoint_urls(endpoint).unwrap(), + ( + format!("wss://{authority}/rtsp"), + format!("rtsps://{authority}"), + ) + ); + } + } + + #[test] + fn endpoint_urls_reject_malformed_authorities_not_partner_or_private_hosts() { + for endpoint in [ + "rtsps://[seat.nvidiagrid.net]:322", + "rtsps://[8.8.8.8]:322", + "rtsps://[[2001:4860:4860::8888]]:322", + "https://partner.example:322", + "rtsp://partner.example:322", + "rtsps://user@partner.example:322", + "rtsps://partner.example:65536", + "rtsps://partner.example:0", + "rtsps://partner.example:wrong", + "rtsps://partner.example:322/#fragment", + ] { + assert!(rtsp_endpoint_urls(endpoint).is_err(), "{endpoint}"); + } + for endpoint in [ + "rtsps://partner.example:322", + "rtsps://127.0.0.1:322", + "rtsps://10.0.0.8:322", + "rtsps://[::1]:322", + ] { + assert!(rtsp_endpoint_urls(endpoint).is_ok(), "{endpoint}"); + } + assert_eq!( + rtsp_endpoint_urls("rtsp://partner.example:322") + .unwrap_err() + .code, + "nvst-raw-rtsp-unsupported" + ); + } + + #[test] + fn endpoint_urls_accept_ipv4_mapped_public_addresses() { + assert_eq!( + rtsp_endpoint_urls("rtsps://[::ffff:8.8.8.8]:48322/session").unwrap(), + ( + "wss://[::ffff:8.8.8.8]:48322/rtsp".to_owned(), + "rtsps://[::ffff:8.8.8.8]:48322".to_owned(), + ) + ); + } + + #[test] + fn ping_identity_increment_preserves_width_and_carry() { + assert_eq!(increment_hex("00ff").as_deref(), Some("0100")); + assert_eq!(increment_hex("ffff").as_deref(), Some("10000")); + assert_eq!(increment_hex("PING"), None); + assert_eq!( + resolve_remote_ufrag(Some("00ff"), Some("described"), None, 6).as_deref(), + Some("0100") + ); + assert_eq!( + resolve_remote_ufrag(None, Some("described"), None, 5).as_deref(), + Some("described") + ); + } + + #[test] + fn bundle_natt_username_uses_server_ufrag_and_decimal_bundle_port() { + let sdp = "a=x-nv-general.iceUsernameFragment:server\r\na=x-nv-general.serverBundlePort:47998\r\n;;a=x-nv-general.serverBundlePort:47999\r\n||a=x-nv-general.serverBundlePort:1234\r\n"; + assert_eq!(bundle_natt_username(sdp).as_deref(), Some("server47999")); + assert_ne!( + bundle_natt_username(sdp).as_deref(), + increment_hex("server47998").as_deref() + ); + assert_eq!(bundle_natt_username("a=x-nv-general.iceUserNameFragmentV2:next\r\na=x-nv-general.serverBundlePort:5004\r\n").as_deref(), Some("next5004")); + for invalid in [ + "a=x-nv-general.iceUsernameFragment:server\r\n", + "a=x-nv-general.serverBundlePort:47999\r\n", + "a=x-nv-general.iceUsernameFragment:server\r\na=x-nv-general.serverBundlePort:0\r\n", + "a=x-nv-general.iceUsernameFragment:server\r\na=x-nv-general.serverBundlePort:65536\r\n", + "a=x-nv-general.iceUsernameFragment:bad:value\r\na=x-nv-general.serverBundlePort:47999\r\n", + ] { + assert_eq!(bundle_natt_username(invalid), None); + } + let oversized = format!( + "a=x-nv-general.iceUsernameFragment:{}\r\na=x-nv-general.serverBundlePort:47999\r\n", + "a".repeat(252) + ); + assert_eq!(bundle_natt_username(&oversized), None); + } + + #[test] + fn bundle_natt_handoff_keeps_the_ice_fragment_distinct() { + let describe = "v=0\r\na=x-nv-general.iceUsernameFragment:a1b2\r\na=x-nv-general.serverBundlePort:48000\r\n;;a=x-nv-general.serverBundlePort:48001\r\n||a=x-nv-general.serverBundlePort:47000\r\n"; + let bundle_username = bundle_natt_username(describe); + let ice_fragment = resolve_remote_ufrag( + Some("a1b247998"), + sdp_attribute(describe, "general.iceUsernameFragment").as_deref(), + bundle_username.as_deref(), + 6, + ) + .unwrap(); + let handoff = json!({"remoteIceUsernameFragment": ice_fragment, + "bundleNattRemoteUsername": bundle_username}); + assert_eq!(handoff["remoteIceUsernameFragment"], "a1b248001"); + assert_eq!(handoff["bundleNattRemoteUsername"], "a1b248001"); + + assert_eq!( + resolve_remote_ufrag( + Some("a1b247998"), + Some("a1b2"), + bundle_natt_username("a=x-nv-general.iceUsernameFragment:a1b2\r\n").as_deref(), + 6, + ) + .as_deref(), + Some("a1b247999") + ); + assert_eq!( + resolve_remote_ufrag( + None, + Some("a1b2"), + handoff["bundleNattRemoteUsername"].as_str(), + 5 + ) + .as_deref(), + Some("a1b2") + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color.rs new file mode 100644 index 000000000..b4c1190ef --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color.rs @@ -0,0 +1,34 @@ +use opennow_streamer_platform::MediaStreamConfig; + +/// NVST ANNOUNCE color encoding, matching the official wire format. +/// +/// A vendor capture of a `10bit_420` session carries `bitDepth:10 +/// chromaFormat:1`: depth is literal (8 or 10) and chroma follows +/// `chroma_format_idc` (1 = 4:2:0, 3 = 4:4:4). Both lines are always sent +/// explicitly; the seat never sees a lone `bitDepth` line. `dynamicRangeMode` +/// is sent only for HDR (`1`); SDR omits it, matching the captured baseline. +/// +/// The internal 0/1 chroma enum from the client's app-to-NVST conversion must +/// not reach the wire: emitting it (or suppressing the chroma line for 4:2:0) +/// leaves seats unable to initialize the encoder, which surfaces as an +/// accepted session that never delivers video. +pub(super) fn announce_color_lines(stream: MediaStreamConfig) -> Vec { + let mut lines = vec![ + format!( + "a=x-nv-video[0].bitDepth:{}", + stream.color_quality.bit_depth() + ), + format!( + "a=x-nv-video[0].chromaFormat:{}", + if stream.color_quality.is_444() { 3 } else { 1 } + ), + ]; + if stream.hdr { + lines.push("a=x-nv-video[0].dynamicRangeMode:1".to_owned()); + } + lines +} + +#[cfg(test)] +#[path = "nvst_rtsp_color_tests.rs"] +mod tests; diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color_tests.rs new file mode 100644 index 000000000..7cad931dc --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_color_tests.rs @@ -0,0 +1,77 @@ +use super::*; +use opennow_streamer_platform::{MediaColorQuality, MediaVideoCodec}; + +fn stream(color_quality: MediaColorQuality, hdr: bool) -> MediaStreamConfig { + MediaStreamConfig { + codec: MediaVideoCodec::H265, + color_quality, + hdr, + width: 2560, + height: 1440, + fps: 120, + ..MediaStreamConfig::default() + } +} + +#[test] +fn wire_values_use_literal_depth_and_chroma_format_idc() { + for (quality, depth, chroma) in [ + (MediaColorQuality::EightBit420, 8, 1), + (MediaColorQuality::EightBit444, 8, 3), + (MediaColorQuality::TenBit420, 10, 1), + (MediaColorQuality::TenBit444, 10, 3), + ] { + assert_eq!( + announce_color_lines(stream(quality, false)), + vec![ + format!("a=x-nv-video[0].bitDepth:{depth}"), + format!("a=x-nv-video[0].chromaFormat:{chroma}"), + ] + ); + } +} + +#[test] +fn dynamic_range_mode_is_hdr_only() { + assert_eq!( + announce_color_lines(stream(MediaColorQuality::TenBit420, true)), + vec![ + "a=x-nv-video[0].bitDepth:10", + "a=x-nv-video[0].chromaFormat:1", + "a=x-nv-video[0].dynamicRangeMode:1", + ] + ); + assert_eq!( + announce_color_lines(stream(MediaColorQuality::TenBit444, true)), + vec![ + "a=x-nv-video[0].bitDepth:10", + "a=x-nv-video[0].chromaFormat:3", + "a=x-nv-video[0].dynamicRangeMode:1", + ] + ); + for quality in [ + MediaColorQuality::EightBit420, + MediaColorQuality::EightBit444, + MediaColorQuality::TenBit420, + MediaColorQuality::TenBit444, + ] { + assert!( + announce_color_lines(stream(quality, false)) + .iter() + .all(|line| !line.contains("dynamicRangeMode")), + "{quality:?}" + ); + } +} + +#[test] +fn ten_bit_420_matches_the_vendor_capture() { + // The only vendor-observed 10-bit 4:2:0 value on the wire. + assert_eq!( + announce_color_lines(stream(MediaColorQuality::TenBit420, false)), + vec![ + "a=x-nv-video[0].bitDepth:10", + "a=x-nv-video[0].chromaFormat:1", + ] + ); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_control_ping_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_control_ping_tests.rs new file mode 100644 index 000000000..8173b3ecd --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_control_ping_tests.rs @@ -0,0 +1,330 @@ +use super::*; +use std::net::TcpListener; + +fn socket_pair() -> (RtspClient, WebSocket) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(6))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(1))) + .unwrap(); + tungstenite::accept(stream).unwrap() + }); + let stream = TcpStream::connect(address).unwrap(); + stream.set_nodelay(true).unwrap(); + let (socket, _) = tungstenite::client( + format!("ws://{address}/rtsp"), + MaybeTlsStream::Plain(stream), + ) + .unwrap(); + ( + RtspClient { + socket, + cseq: 7, + buffer: String::new(), + }, + server.join().unwrap(), + ) +} + +fn active_session(client: RtspClient, ping: &NvstControlPing) -> ActiveNvstRtspSession { + ActiveNvstRtspSession::spawn(client, ping.clone()).unwrap() +} + +fn server_ping(server: &mut WebSocket) { + server.send(Message::Ping(vec![1, 2, 3].into())).unwrap(); + assert_eq!(server.read().unwrap(), Message::Pong(vec![1, 2, 3].into())); +} + +fn client_ping(server: &mut WebSocket) -> Vec { + let message = server.read().unwrap(); + let Message::Ping(bytes) = message else { + panic!("expected WebSocket ping, received {message:?}"); + }; + assert_eq!(bytes.len(), 8); + bytes.to_vec() +} + +fn wait_until(mut condition: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(1); + while !condition() { + assert!(Instant::now() < deadline, "worker did not respond promptly"); + thread::sleep(Duration::from_millis(2)); + } +} + +#[test] +fn describe_accepts_large_text_binary_and_split_responses() { + for (binary, chunk_size) in [ + (false, usize::MAX), + (true, usize::MAX), + (false, 16 * 1024), + (true, 16 * 1024), + ] { + let (mut client, mut server) = socket_pair(); + let body = "a=x-nv-test:description\r\n".repeat(12_000); + assert!(body.len() > MAX_CONTROL_RESPONSE_BYTES); + let expected_body = body.clone(); + let sender = thread::spawn(move || { + let Message::Text(request) = server.read().unwrap() else { + panic!("expected DESCRIBE request"); + }; + assert!(request.starts_with("DESCRIBE ")); + assert!(request.contains("\r\nCSeq: 8\r\n")); + let response = format!( + "RTSP/1.0 200 OK\r\nCSeq: 8\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + for chunk in response.as_bytes().chunks(chunk_size) { + let message = if binary { + Message::Binary(chunk.to_vec().into()) + } else { + Message::Text(String::from_utf8(chunk.to_vec()).unwrap().into()) + }; + server.send(message).unwrap(); + } + }); + let response = client + .request("DESCRIBE", "rtsps://seat.nvidiagrid.net:322", &[], "") + .unwrap(); + assert_eq!(response.status, 200); + assert_eq!(response.body, expected_body); + assert!(client.buffer.is_empty()); + sender.join().unwrap(); + } +} + +#[test] +fn request_rejects_unbounded_partial_response() { + let (mut client, mut server) = socket_pair(); + let sender = thread::spawn(move || { + server.read().unwrap(); + server + .send(Message::Text("x".repeat(MAX_REQUEST_RESPONSE_BYTES).into())) + .unwrap(); + server.send(Message::Text("x".into())).unwrap(); + }); + let error = client + .request("DESCRIBE", "rtsps://seat.nvidiagrid.net:322", &[], "") + .err() + .unwrap(); + assert_eq!(error.code, "nvst-rtsp-failed"); + assert!( + error + .message + .contains("DESCRIBE response exceeds request buffer limit") + ); + sender.join().unwrap(); +} + +#[test] +fn request_bounds_websocket_message_size() { + let (mut client, mut server) = socket_pair(); + let sender = thread::spawn(move || { + server.read().unwrap(); + let _ = server.send(Message::Text( + "x".repeat(MAX_REQUEST_RESPONSE_BYTES + 1).into(), + )); + }); + let error = client + .request("DESCRIBE", "rtsps://seat.nvidiagrid.net:322", &[], "") + .err() + .unwrap(); + assert_eq!(error.code, "nvst-rtsp-failed"); + assert!(error.message.contains("Space limit exceeded")); + assert!(client.buffer.is_empty()); + sender.join().unwrap(); +} + +#[test] +fn rtsp_parser_accepts_response_at_request_size_limit() { + let prefix = "RTSP/1.0 200 OK\r\nCSeq: 8\r\nContent-Length: "; + let body_length = MAX_REQUEST_RESPONSE_BYTES + - prefix.len() + - MAX_REQUEST_RESPONSE_BYTES.to_string().len() + - 4; + let body = "x".repeat(body_length); + let mut buffer = format!("{prefix}{body_length}\r\n\r\n{body}"); + assert_eq!(buffer.len(), MAX_REQUEST_RESPONSE_BYTES); + let response = take_rtsp_response(&mut buffer, 8).unwrap().unwrap(); + assert_eq!(response.body, body); + assert!(buffer.is_empty()); +} + +#[test] +fn rtsp_parser_bounds_declared_response_size_before_receiving_body() { + let mut oversized = format!( + "RTSP/1.0 200 OK\r\nCSeq: 8\r\nContent-Length: {MAX_REQUEST_RESPONSE_BYTES}\r\n\r\n" + ); + let error = take_rtsp_response(&mut oversized, 8).err().unwrap(); + assert!( + error + .message + .contains("response exceeds request buffer limit") + ); + + let mut bounded = "RTSP/1.0 200 OK\r\nCSeq: 8\r\nContent-Length: 262144\r\n\r\n".to_owned(); + assert!(take_rtsp_response(&mut bounded, 8).unwrap().is_none()); +} + +#[test] +fn control_ping_parser_rejects_overflowing_and_invalid_body_boundaries() { + let mut overflowing = format!( + "RTSP/1.0 551 Response\r\nCSeq: 8\r\nContent-Length: {}\r\n\r\n", + usize::MAX + ); + assert!(take_rtsp_response(&mut overflowing, 8).is_err()); + let mut invalid = "RTSP/1.0 551 Response\r\nCSeq: 8\r\nContent-Length: 1\r\n\r\né".to_owned(); + assert!(take_rtsp_response(&mut invalid, 8).is_err()); +} + +#[test] +fn control_ping_samples_are_shared_expiring_and_clearable() { + let ping = NvstControlPing::default(); + let clone = ping.clone(); + let sent = Instant::now(); + let received = sent + Duration::from_micros(4_750); + assert_eq!(ping.ping_ms(sent), None); + ping.record(sent, received); + assert_eq!(clone.ping_ms(received), Some(4.75)); + assert_eq!(clone.ping_ms(sent), None); + assert_eq!(ping.ping_ms(received + Duration::from_secs(4)), Some(4.75)); + assert_eq!(ping.ping_ms(received + CONTROL_PING_EXPIRY), None); + assert_eq!(clone.ping_ms(received), None); + ping.record(sent, received); + clone.clear(); + assert_eq!(ping.ping_ms(received), None); + ping.record(sent, sent); + assert_eq!(ping.ping_ms(sent), Some(0.0)); +} + +#[test] +fn live_control_ping_uses_websocket_without_rtsp_requests() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + server_ping(&mut server); + let payload = client_ping(&mut server); + server.send(Message::Pong(vec![0; 8].into())).unwrap(); + assert_eq!(ping.ping_ms(Instant::now()), None); + server.send(Message::Pong(payload.into())).unwrap(); + wait_until(|| ping.ping_ms(Instant::now()).is_some()); + assert!(ping.ping_ms(Instant::now()).unwrap() < 1000.0); + server_ping(&mut server); + server.close(None).unwrap(); + wait_until(|| active.worker.as_ref().unwrap().is_finished()); + assert_eq!(ping.ping_ms(Instant::now()), None); + let shutdown_started = Instant::now(); + active.shutdown(); + assert!(shutdown_started.elapsed() < Duration::from_secs(1)); +} + +#[test] +fn live_control_ping_times_out_without_accepting_late_pongs() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + let first = client_ping(&mut server); + server.send(Message::Pong(first.into())).unwrap(); + wait_until(|| ping.ping_ms(Instant::now()).is_some()); + let missing = client_ping(&mut server); + server.send(Message::Pong(vec![0; 8].into())).unwrap(); + let next = client_ping(&mut server); + server.send(Message::Pong(vec![0; 8].into())).unwrap(); + assert_eq!( + u64::from_be_bytes(next.clone().try_into().unwrap()), + u64::from_be_bytes(missing.clone().try_into().unwrap()) + 1 + ); + assert_eq!(ping.ping_ms(Instant::now()), None); + server.send(Message::Pong(missing.into())).unwrap(); + server_ping(&mut server); + assert_eq!(ping.ping_ms(Instant::now()), None); + server.send(Message::Pong(next.into())).unwrap(); + wait_until(|| ping.ping_ms(Instant::now()).is_some()); + drop(server); + wait_until(|| active.worker.as_ref().unwrap().is_finished()); + assert_eq!(ping.ping_ms(Instant::now()), None); + active.shutdown(); +} + +#[test] +fn unanswered_websocket_ping_does_not_end_the_session() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + let first = client_ping(&mut server); + server.send(Message::Pong(vec![0; 8].into())).unwrap(); + let second = client_ping(&mut server); + server.send(Message::Pong(vec![0; 8].into())).unwrap(); + assert_ne!(first, second); + assert_eq!(ping.ping_ms(Instant::now()), None); + assert!(!active.worker.as_ref().unwrap().is_finished()); + let shutdown = thread::spawn(move || active.shutdown()); + assert!(matches!(server.read().unwrap(), Message::Close(_))); + shutdown.join().unwrap(); +} + +#[test] +fn live_control_ping_shutdown_closes_without_rtsp_teardown() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + client_ping(&mut server); + let shutdown_started = Instant::now(); + let shutdown = thread::spawn(move || active.shutdown()); + assert!(matches!(server.read().unwrap(), Message::Close(_))); + shutdown.join().unwrap(); + assert!(shutdown_started.elapsed() < Duration::from_secs(1)); + assert_eq!(ping.ping_ms(Instant::now()), None); +} + +#[test] +fn live_control_ping_shutdown_does_not_wait_for_pending_pong() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + client_ping(&mut server); + let shutdown_started = Instant::now(); + let shutdown = thread::spawn(move || active.shutdown()); + assert!(matches!(server.read().unwrap(), Message::Close(_))); + assert_eq!(ping.ping_ms(Instant::now()), None); + shutdown.join().unwrap(); + assert!(shutdown_started.elapsed() < Duration::from_secs(1)); +} + +#[test] +fn live_control_ping_shutdown_clears_sample() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + let payload = client_ping(&mut server); + server.send(Message::Pong(payload.into())).unwrap(); + wait_until(|| ping.ping_ms(Instant::now()).is_some()); + let shutdown_started = Instant::now(); + let shutdown = thread::spawn(move || active.shutdown()); + assert!(matches!(server.read().unwrap(), Message::Close(_))); + assert_eq!(ping.ping_ms(Instant::now()), None); + shutdown.join().unwrap(); + assert!(shutdown_started.elapsed() < Duration::from_secs(1)); +} + +#[test] +fn live_control_ping_rejects_unbounded_partial_response() { + let (client, mut server) = socket_pair(); + let ping = NvstControlPing::default(); + let mut active = active_session(client, &ping); + let payload = client_ping(&mut server); + server.send(Message::Pong(payload.into())).unwrap(); + wait_until(|| ping.ping_ms(Instant::now()).is_some()); + server + .send(Message::Text("x".repeat(MAX_CONTROL_RESPONSE_BYTES).into())) + .unwrap(); + server.send(Message::Text("x".into())).unwrap(); + wait_until(|| active.worker.as_ref().unwrap().is_finished()); + assert_eq!(ping.ping_ms(Instant::now()), None); + active.shutdown(); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_negotiation.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_negotiation.rs new file mode 100644 index 000000000..492b60bee --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_negotiation.rs @@ -0,0 +1,295 @@ +use super::{NvstRtspError, sdp_attribute}; +use opennow_streamer_transport::NvstRecoveryPolicy; +use serde_json::{Value, json}; +use std::ops::RangeInclusive; + +fn ignore_offer(name: &str, value: &str) { + opennow_streamer_protocol::log::log_line( + "WARN", + "transport", + &format!("ignored-server-offer name={name} value={value}"), + ); +} + +pub(super) fn offered_bool(sdp: &str, name: &str) -> Option { + let value = sdp_attribute(sdp, name)?; + match value.as_str() { + "0" => Some(false), + "1" => Some(true), + _ => { + ignore_offer(name, &value); + None + } + } +} + +fn offered_u32(sdp: &str, name: &str) -> Option { + let value = sdp_attribute(sdp, name)?; + value + .parse() + .inspect_err(|_| ignore_offer(name, &value)) + .ok() +} + +fn offered_in(sdp: &str, name: &str, range: RangeInclusive) -> Option { + let value = offered_u32(sdp, name)?; + if range.contains(&value) { + return Some(value); + } + ignore_offer(name, &value.to_string()); + None +} + +pub(super) fn recovery_policy(sdp: &str) -> NvstRecoveryPolicy { + NvstRecoveryPolicy { + ref_pic_invalidation: offered_bool(sdp, "video[0].refPicInvalidation"), + stream_recovery_type: offered_u32(sdp, "video[0].streamRecoveryType"), + frames_with_invalid_ref_threshold: offered_u32( + sdp, + "video[0].framesWithInvalidRefThreshold", + ), + enable_recovery_mode: offered_bool(sdp, "general.enableRecoveryMode"), + recovery_mode_trigger_threshold_ms: offered_u32( + sdp, + "general.recoveryModeTriggerThresholdMs", + ), + enable_intra_refresh_in_recovery_mode: offered_bool( + sdp, + "general.enableIntraRefreshInRecoveryMode", + ), + encoder_ltr_feature_setting: offered_u32(sdp, "video[0].encoderLtrFeatureSetting"), + } +} + +pub(super) fn negotiated_announce( + announce: &str, + describe: &str, + handoff: &mut Value, +) -> Result { + let mut values = serde_json::Map::new(); + for (name, minimum, maximum) in [ + ("vqos[0].fec.enable", 0, 1), + ("vqos[0].fec.rateDropWindow", 0, u32::MAX), + ("vqos[0].fec.minRequiredFecPackets", 0, u32::MAX), + ("vqos[0].fec.repairPercent", 0, 100), + ("vqos[0].fec.repairMinPercent", 0, 100), + ("vqos[0].fec.repairMaxPercent", 0, 100), + ("vqos[0].bllFec.enable", 0, 1), + ("packetPacing.version", 3, 3), + ("packetPacing.mode", 1, 1), + ("packetPacing.numGroups", 1, u32::MAX), + ("packetPacing.maxDelayUs", 0, u32::MAX), + ("packetPacing.minNumPacketsFrame", 0, u32::MAX), + ("packetPacing.minNumPacketsPerGroup", 0, u32::MAX), + ("packetPacing.enableAccurateSleep", 0, 1), + ("packetPacing.enableSmoothTransition", 0, 1), + ("packetPacing.allowFpsBasedToggle", 0, 1), + ] { + let value = offered_in(describe, name, minimum..=maximum) + .or(offered_u32(announce, name)) + .unwrap(); + values.insert(name.to_owned(), json!(value)); + } + let mut min = values["vqos[0].fec.repairMinPercent"].as_u64().unwrap(); + let mut max = values["vqos[0].fec.repairMaxPercent"].as_u64().unwrap(); + if min > max { + ignore_offer("vqos[0].fec.repairMinPercent", &format!("{min}>{max}")); + min = u64::from(offered_u32(announce, "vqos[0].fec.repairMinPercent").unwrap()); + max = u64::from(offered_u32(announce, "vqos[0].fec.repairMaxPercent").unwrap()); + values["vqos[0].fec.repairMinPercent"] = json!(min); + values["vqos[0].fec.repairMaxPercent"] = json!(max); + } + let repair = values["vqos[0].fec.repairPercent"].as_u64().unwrap(); + values["vqos[0].fec.repairPercent"] = json!(repair.clamp(min, max)); + let ceiling = offered_u32(announce, "vqos[0].bw.maximumBitrateKbps").unwrap(); + let initial = offered_in(describe, "video[0].initialBitrateKbps", 1..=u32::MAX) + .unwrap_or(ceiling) + .min(ceiling); + let peak = offered_u32(describe, "video[0].initialPeakBitrateKbps") + .unwrap_or(ceiling) + .clamp(initial, ceiling); + values.insert("video[0].initialBitrateKbps".to_owned(), json!(initial)); + values.insert("video[0].initialPeakBitrateKbps".to_owned(), json!(peak)); + values.insert("vqos[0].bw.maximumBitrateKbps".to_owned(), json!(ceiling)); + opennow_streamer_protocol::log::log_line( + "INFO", + "transport", + &format!("negotiated-encoding={}", Value::Object(values.clone())), + ); + let mut result = String::with_capacity(announce.len()); + for line in announce.split_inclusive("\r\n") { + let attribute = line + .strip_prefix("a=x-nv-") + .and_then(|line| line.split_once(':')); + if let Some((name, _)) = attribute + && let Some(value) = values.get(name) + { + result.push_str(&format!("a=x-nv-{name}:{value}\r\n")); + } else { + result.push_str(line); + } + } + handoff["framePacingFeedbackMode"] = + json!(super::video_qos_offers(describe).pacing_feedback_mode(&result)?); + Ok(result) +} + +pub(super) fn microphone_handoff(sdp: &str, handoff: &mut Value) { + let enabled = offered_bool(sdp, "aqos.enableRedundancyForMic").unwrap_or(false); + let level = if enabled { + offered_in(sdp, "aqos.redundancyLevelForMic", 0..=3).unwrap_or(0) + } else { + 0 + }; + handoff["microphoneRedundancyLevel"] = json!(level); + let mut in_mic = false; + let mut payloads = Vec::new(); + for line in sdp + .split("||") + .next() + .unwrap_or_default() + .split(";;") + .next() + .unwrap_or_default() + .lines() + .map(str::trim) + { + if let Some(media) = line.strip_prefix("m=") { + let mut parts = media.split_whitespace(); + in_mic = parts + .next() + .is_some_and(|kind| kind.eq_ignore_ascii_case("mic")); + payloads = if in_mic { + parts.skip(2).collect() + } else { + Vec::new() + }; + } else if in_mic + && let Some(mapping) = line.strip_prefix("a=rtpmap:") + && let Some((payload, encoding)) = mapping.split_once(char::is_whitespace) + { + if !payloads.contains(&payload) { + continue; + } + let Some(codec) = encoding.split('/').next() else { + continue; + }; + let field = if codec.eq_ignore_ascii_case("opus") { + "microphoneOpusPayloadType" + } else if codec.eq_ignore_ascii_case("red") { + "microphoneRedPayloadType" + } else { + continue; + }; + let Some(payload) = payload.parse::().ok().filter(|value| *value <= 127) else { + ignore_offer(field, payload); + continue; + }; + handoff[field] = json!(payload); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recovery_describe_is_typed_and_does_not_guess_defaults() { + let policy = recovery_policy( + "a=x-nv-video[0].refPicInvalidation:1\r\na=x-nv-video[0].streamRecoveryType:2\r\na=x-nv-video[0].framesWithInvalidRefThreshold:4\r\na=x-nv-general.enableRecoveryMode:1\r\na=x-nv-general.recoveryModeTriggerThresholdMs:6000\r\na=x-nv-general.enableIntraRefreshInRecoveryMode:1\r\na=x-nv-video[0].encoderLtrFeatureSetting:2\r\n", + ); + assert_eq!(policy.ref_pic_invalidation, Some(true)); + assert_eq!(policy.stream_recovery_type, Some(2)); + assert_eq!(policy.frames_with_invalid_ref_threshold, Some(4)); + assert_eq!(policy.recovery_mode_trigger_threshold_ms, Some(6000)); + assert_eq!(policy.encoder_ltr_feature_setting, Some(2)); + assert_eq!(recovery_policy(""), NvstRecoveryPolicy::default()); + assert_eq!( + recovery_policy( + "a=x-nv-video[0].refPicInvalidation:2\r\na=x-nv-video[0].framesWithInvalidRefThreshold:-1\r\n" + ), + NvstRecoveryPolicy::default() + ); + } + + #[test] + fn mic_handoff_uses_only_its_offered_payloads_and_red_depth() { + let mut handoff = json!({}); + microphone_handoff( + "a=x-nv-aqos.enableRedundancyForMic:1\r\na=x-nv-aqos.redundancyLevelForMic:3\r\nm=audio 5000 RTP/AVP 111 112\r\na=rtpmap:111 opus/48000/2\r\na=rtpmap:112 red/48000/2\r\nm=mic 5002 RTP/AVP 113 114\r\na=rtpmap:113 opus/48000/1\r\na=rtpmap:114 red/48000/1\r\na=rtpmap:115 red/48000/1\r\n", + &mut handoff, + ); + assert_eq!( + handoff, + json!({"microphoneOpusPayloadType":113,"microphoneRedPayloadType":114,"microphoneRedundancyLevel":3}) + ); + for sdp in [ + "", + "a=x-nv-aqos.enableRedundancyForMic:0\r\na=x-nv-aqos.redundancyLevelForMic:3\r\nm=audio 5000 RTP/AVP 111\r\na=rtpmap:111 opus/48000/2\r\n", + ] { + let mut handoff = json!({}); + microphone_handoff(sdp, &mut handoff); + assert_eq!(handoff, json!({"microphoneRedundancyLevel":0})); + } + for sdp in [ + "m=mic 5002 RTP/AVP 128\r\na=rtpmap:128 opus/48000/1\r\n", + "a=x-nv-aqos.enableRedundancyForMic:1\r\na=x-nv-aqos.redundancyLevelForMic:4", + ] { + let mut handoff = json!({}); + microphone_handoff(sdp, &mut handoff); + assert_eq!(handoff, json!({"microphoneRedundancyLevel":0})); + } + } + + #[test] + fn mic_describe_handoff_reaches_the_live_transport_parser() { + use opennow_streamer_transport::{NvstConfigError, NvstVideoConfig}; + + let mut handoff = json!({ + "clientUdpPort":49005,"videoPeerIp":"192.0.2.1","videoPeerPort":5004, + "srtpAesKeyHex":"01".repeat(32),"srtpSaltHex":"02".repeat(12),"codec":"H264", + "microphoneOnBundle":true,"pingVersion":6, + "localIceUsernameFragment":"loc1","localIcePassword":"local-password-value-01", + "remoteIceUsernameFragment":"remote01","remoteIcePassword":"remote-password-value-01", + "remoteDtlsFingerprint":"AA:BB" + }); + microphone_handoff( + "a=x-nv-aqos.enableRedundancyForMic:1\r\na=x-nv-aqos.redundancyLevelForMic:3\r\nm=mic 5002 RTP/AVP 113 114\r\na=rtpmap:113 opus/48000/1\r\na=rtpmap:114 red/48000/1\r\n", + &mut handoff, + ); + assert_eq!(handoff["microphoneOpusPayloadType"], 113); + assert_eq!(handoff["microphoneRedPayloadType"], 114); + assert_eq!(handoff["microphoneRedundancyLevel"], 3); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + + for (field, invalid) in [ + ("microphoneOpusPayloadType", 78), + ("microphoneRedPayloadType", 128), + ("microphoneRedPayloadType", 113), + ("microphoneRedundancyLevel", 4), + ] { + let mut invalid_handoff = handoff.clone(); + invalid_handoff[field] = json!(invalid); + assert!( + matches!(NvstVideoConfig::from_legacy_handoff(&invalid_handoff, None), + Err(NvstConfigError::OutOfRange {field: rejected}) if rejected == field), + "the live transport parser must consume and validate {field}" + ); + } + microphone_handoff( + "a=x-nv-aqos.enableRedundancyForMic:0\r\na=x-nv-aqos.redundancyLevelForMic:3\r\n", + &mut handoff, + ); + assert_eq!(handoff["microphoneRedundancyLevel"], 0); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_setup_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_setup_tests.rs new file mode 100644 index 000000000..f62aa1778 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_setup_tests.rs @@ -0,0 +1,441 @@ +use super::*; +use std::net::TcpListener; + +const TARGET: &str = "rtsps://seat.nvidiagrid.net:322"; +const VALID_PEER: &str = "Transport: unicast;source=192.0.2.10;X-GS-ServerPort=5004-5005\r\nX-Nv-Ping: 6\r\nX-Nv-Ping-Payload: 00ff\r\n"; +const EMPTY_TRANSPORT_URIS: [&str; 4] = [ + "streamid=video/0/0", + "streamid=video/0", + "rtsps://seat.nvidiagrid.net:322/streamid=video/0/0", + "rtsps://seat.nvidiagrid.net:322/streamid=video/0", +]; + +#[derive(Clone, Copy)] +struct Reply { + uri: &'static str, + transport: &'static str, + status: u16, + headers: &'static str, +} + +fn scripted_setup(replies: Vec) -> Result { + scripted_setup_with_retry(replies, 0, Duration::ZERO, None) +} + +fn scripted_setup_with_retry( + replies: Vec, + max_peer_retries: u32, + peer_retry_delay: Duration, + bundle_video_peer: Option<(String, u16, u16)>, +) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let stream = TcpStream::connect(address).unwrap(); + let (server_stream, _) = listener.accept().unwrap(); + for socket in [&stream, &server_stream] { + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + } + let server = thread::spawn(move || { + let mut socket = tungstenite::accept(server_stream).unwrap(); + for (index, reply) in replies.iter().enumerate() { + let Message::Text(request) = socket.read().unwrap() else { + panic!("expected a SETUP request"); + }; + assert!( + request.starts_with(&format!("SETUP {} RTSP/1.0\r\n", reply.uri)), + "{request}" + ); + assert!(request.contains(&format!("\r\nTransport: {}\r\n", reply.transport))); + assert!(request.contains("\r\nSession: rtsp-session\r\n")); + assert!(request.contains("\r\nx-nv-sessionid: nv-session\r\n")); + assert!(request.contains("\r\nx-nv-ping: 6\r\n")); + let cseq = index + 3; + assert!(request.contains(&format!("\r\nCSeq: {cseq}\r\n"))); + socket + .send(Message::Text( + format!( + "RTSP/1.0 {} Response\r\nCSeq: {cseq}\r\n{}Content-Length: 0\r\n\r\n", + reply.status, reply.headers + ) + .into(), + )) + .unwrap(); + } + }); + let (socket, _) = tungstenite::client( + format!("ws://{address}/rtsp"), + MaybeTlsStream::Plain(stream), + ) + .unwrap(); + let mut client = RtspClient { + socket, + cseq: 2, + buffer: String::new(), + }; + let result = client.setup_video_with_retry( + "streamid=video/0", + TARGET, + &[ + ("Session", "rtsp-session".to_owned()), + ("x-nv-sessionid", "nv-session".to_owned()), + ("x-nv-ping", "6".to_owned()), + ], + 49005, + max_peer_retries, + peer_retry_delay, + bundle_video_peer, + ); + drop(client); + server.join().unwrap(); + result +} + +#[test] +fn video_setup_keeps_the_official_first_attempt_and_its_metadata() { + let setup = scripted_setup(vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: VALID_PEER, + }]) + .unwrap(); + assert_eq!(setup.peer, ("192.0.2.10".to_owned(), 5004, 5005)); + assert!(!setup.bundle_fallback); + assert_eq!( + header_value(&setup.response, "x-nv-ping-payload"), + Some("00ff") + ); +} + +#[test] +fn video_setup_stops_at_first_peerless_success_to_protect_announce() { + // Live alliance rigs accept the first SETUP but poison the session once + // further forms are tried (later rounds degrade to pure 400s and ANNOUNCE + // is then rejected), while a single SETUP followed by ANNOUNCE succeeds. + // The scripted server holds exactly one reply: any extra request fails it. + let error = match scripted_setup(vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "", + }]) { + Ok(_) => panic!("a peerless 200 must not yield a peer"), + Err(error) => error, + }; + assert_eq!(error.code, "missing-video-peer"); + assert!(error.message.contains("peerless 200")); +} + +#[test] +fn video_setup_tries_absolute_forms_after_uri_rejections() { + let replies = EMPTY_TRANSPORT_URIS + .iter() + .enumerate() + .map(|(index, uri)| Reply { + uri, + transport: "", + status: if index == 3 { 200 } else { 404 }, + headers: if index == 3 { VALID_PEER } else { "" }, + }) + .collect(); + let setup = scripted_setup(replies).unwrap(); + assert_eq!(setup.peer.1, 5004); +} + +#[test] +fn video_setup_tries_client_udp_transport_after_all_empty_transport_forms() { + let mut replies: Vec<_> = EMPTY_TRANSPORT_URIS + .iter() + .map(|uri| Reply { + uri, + transport: "", + status: 461, + headers: "", + }) + .collect(); + replies.extend( + EMPTY_TRANSPORT_URIS + .iter() + .enumerate() + .map(|(index, uri)| Reply { + uri, + transport: "unicast;X-GS-ClientPort=49005-49006", + status: if index == 3 { 200 } else { 400 }, + headers: if index == 3 { VALID_PEER } else { "" }, + }), + ); + let setup = scripted_setup(replies).unwrap(); + assert_eq!(setup.peer.1, 5004); + assert_eq!( + header_value(&setup.response, "x-nv-ping-payload"), + Some("00ff") + ); +} + +#[test] +fn video_setup_never_invents_a_peer_when_success_omits_transport() { + // Single peerless 200: the sweep stops immediately (one scripted reply) + // instead of trying further forms that would poison strict sessions. + let error = match scripted_setup(vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "", + }]) { + Ok(_) => panic!("SETUP without a peer must not succeed"), + Err(error) => error, + }; + assert_eq!(error.code, "missing-video-peer"); + assert!(error.message.contains("peerless 200")); +} + +#[test] +fn video_setup_rejects_partial_and_invalid_transport_metadata() { + // Present-but-unusable Transport (missing/invalid source or ports) also + // stops the sweep: no live rig has ever yielded a peer on a later form + // after a peerless 200, while extra SETUPs poison strict sessions. + for headers in [ + "Transport: unicast;X-GS-ServerPort=5004\r\n", + "Transport: unicast;source=192.0.2.10\r\n", + "Transport: unicast;source=not-an-ip;X-GS-ServerPort=5004\r\n", + ] { + let error = match scripted_setup(vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers, + }]) { + Ok(_) => panic!("unusable Transport must not yield a peer"), + Err(error) => error, + }; + assert_eq!(error.code, "missing-video-peer"); + } +} + +#[test] +fn video_setup_stops_on_auth_session_and_server_errors() { + for status in [401, 403, 454, 455, 500, 503] { + let result = scripted_setup(vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status, + headers: "", + }]); + let error = match result { + Ok(_) => panic!("a fatal SETUP response must not succeed"), + Err(error) => error, + }; + assert_eq!(error.code, "nvst-rtsp-failed"); + assert!(error.message.contains(&status.to_string())); + } +} + +#[test] +fn video_setup_candidates_match_mac_order_without_duplicates() { + assert_eq!( + video_setup_candidates("streamid=video/0", TARGET), + EMPTY_TRANSPORT_URIS + ); + assert_eq!( + video_setup_candidates("streamid=video/0/0", TARGET), + vec![ + "streamid=video/0/0", + "rtsps://seat.nvidiagrid.net:322/streamid=video/0/0", + ] + ); + assert_eq!( + video_setup_candidates("/streamid=video/0", TARGET), + vec![ + "/streamid=video/0", + "rtsps://seat.nvidiagrid.net:322/streamid=video/0", + ] + ); + for absolute in [ + "rtsp://seat.nvidiagrid.net:322/video", + "rtsps://seat.nvidiagrid.net:322/video", + ] { + assert_eq!(video_setup_candidates(absolute, TARGET), vec![absolute]); + } +} + +#[test] +fn rtsp_request_deadline_bounds_a_partial_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let stream = TcpStream::connect(address).unwrap(); + let (server_stream, _) = listener.accept().unwrap(); + for socket in [&stream, &server_stream] { + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + } + let (release, released) = mpsc::channel(); + let server = thread::spawn(move || { + let mut socket = tungstenite::accept(server_stream).unwrap(); + assert!(matches!(socket.read().unwrap(), Message::Text(_))); + socket + .send(Message::Text("RTSP/1.0 200 OK\r\nCSeq: 1\r\n".into())) + .unwrap(); + let _ = released.recv_timeout(Duration::from_secs(2)); + }); + let (socket, _) = tungstenite::client( + format!("ws://{address}/rtsp"), + MaybeTlsStream::Plain(stream), + ) + .unwrap(); + let mut client = RtspClient { + socket, + cseq: 0, + buffer: String::new(), + }; + let start = Instant::now(); + let result = client.request_with_timeout( + "SETUP", + "streamid=video/0/0", + &[], + "", + Duration::from_millis(80), + ); + let elapsed = start.elapsed(); + let _ = release.send(()); + server.join().unwrap(); + let error = match result { + Ok(_) => panic!("an incomplete RTSP response must time out"), + Err(error) => error, + }; + assert_eq!(error.code, "nvst-rtsp-timeout"); + assert!( + elapsed < Duration::from_secs(1), + "deadline exceeded: {elapsed:?}" + ); +} + +#[test] +fn video_setup_resweeps_when_successes_omit_a_peer_until_the_rig_is_ready() { + // Round 1 draws one peerless 200 and stops at it; round 2 re-sends the + // same form and the rig has a peer by then. Mirrors a late-starting + // encoder without spraying further forms that poison strict sessions. + let replies = vec![ + Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "", + }, + Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: VALID_PEER, + }, + ]; + let setup = scripted_setup_with_retry(replies, 3, Duration::ZERO, None).unwrap(); + assert_eq!(setup.peer, ("192.0.2.10".to_owned(), 5004, 5005)); +} + +#[test] +fn video_setup_peer_retry_stays_bounded_and_keeps_the_terminal_code() { + // Every round draws one peerless 200 and stops there: retries exhaust, + // then the original missing-video-peer error (not a timeout, not a new + // code) is returned. + let replies = vec![ + Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "", + }; + 3 + ]; + let error = match scripted_setup_with_retry(replies, 2, Duration::ZERO, None) { + Ok(_) => panic!("SETUP without a peer must not succeed"), + Err(error) => error, + }; + assert_eq!(error.code, "missing-video-peer"); + assert!(error.message.contains("peerless 200")); +} + +#[test] +fn video_setup_never_retries_pure_rejections() { + // No 200 seen at all: the forms are wrong for this server, so fail fast + // without burning retry rounds. + let replies: Vec<_> = ["", "unicast;X-GS-ClientPort=49005-49006"] + .iter() + .flat_map(|transport| { + EMPTY_TRANSPORT_URIS.iter().map(move |uri| Reply { + uri, + transport, + status: 400, + headers: "", + }) + }) + .collect(); + let error = match scripted_setup_with_retry(replies, 3, Duration::ZERO, None) { + Ok(_) => panic!("rejected SETUP forms must not succeed"), + Err(error) => error, + }; + assert_eq!(error.code, "nvst-rtsp-failed"); +} + +#[test] +fn video_setup_degraded_retry_round_keeps_missing_peer_code() { + // Round 1 draws a peerless 200 and stops, so a retry is scheduled; round + // 2 then degrades to pure 400s (repeat SETUPs rejected). The terminal code + // must stay missing-video-peer so callers can try the bundle peer instead + // of treating it as wrong forms. Round 2 sweeps all forms because it drew + // no 200 at all. + let mut replies = vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "", + }]; + replies.extend( + ["", "unicast;X-GS-ClientPort=49005-49006"] + .iter() + .flat_map(|transport| { + EMPTY_TRANSPORT_URIS.iter().map(move |uri| Reply { + uri, + transport, + status: 400, + headers: "", + }) + }), + ); + let error = match scripted_setup_with_retry(replies, 1, Duration::ZERO, None) { + Ok(_) => panic!("peerless SETUP must not succeed"), + Err(error) => error, + }; + assert_eq!(error.code, "missing-video-peer"); +} + +#[test] +fn video_setup_uses_bundle_peer_on_first_peerless_success_without_retry() { + let peerless = vec![Reply { + uri: EMPTY_TRANSPORT_URIS[0], + transport: "", + status: 200, + headers: "Transport: \r\nX-Nv-Ping: 6\r\nX-Nv-Ping-Payload: 00ff\r\n", + }]; + let setup = scripted_setup_with_retry( + peerless, + 3, + Duration::ZERO, + Some(("192.0.2.99".to_owned(), 13749, 13749)), + ) + .unwrap(); + assert_eq!(setup.peer, ("192.0.2.99".to_owned(), 13749, 13749)); + assert!(setup.bundle_fallback); + assert_eq!( + header_value(&setup.response, "x-nv-ping-payload"), + Some("00ff") + ); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_tls_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_tls_tests.rs new file mode 100644 index 000000000..2292ff3f8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_tls_tests.rs @@ -0,0 +1,136 @@ +use super::*; +use std::net::TcpListener; + +fn exchange_over_tls( + server_versions: &[&'static rustls::SupportedProtocolVersion], + expected_version: rustls::ProtocolVersion, +) { + ensure_tls_crypto_provider().unwrap(); + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).unwrap(); + let server_config = rustls::ServerConfig::builder_with_protocol_versions(server_versions) + .with_no_client_auth() + .with_single_cert( + vec![cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(signing_key.serialize_der()).into(), + ) + .unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let (server_stream, _) = listener.accept().unwrap(); + server_stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + server_stream + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let server = thread::spawn(move || { + let connection = rustls::ServerConnection::new(Arc::new(server_config)).unwrap(); + let mut socket = + tungstenite::accept(rustls::StreamOwned::new(connection, server_stream)).unwrap(); + assert_eq!( + socket.get_ref().conn.protocol_version(), + Some(expected_version) + ); + let Message::Text(request) = socket.read().unwrap() else { + panic!("expected an RTSP request"); + }; + assert!(request.starts_with("OPTIONS rtsps://localhost:322 RTSP/1.0\r\n")); + assert!(request.contains("\r\nCSeq: 1\r\n")); + socket + .send(Message::Text( + "RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 0\r\n\r\n".into(), + )) + .unwrap(); + }); + let (socket, _) = tungstenite::client_tls_with_config( + format!("wss://localhost:{}/rtsp", address.port()), + stream, + None, + Some(tungstenite::Connector::Rustls(Arc::new(client_config))), + ) + .unwrap(); + let MaybeTlsStream::Rustls(tls) = socket.get_ref() else { + panic!("RTSPS must remain encrypted"); + }; + assert_eq!(tls.conn.protocol_version(), Some(expected_version)); + let mut client = RtspClient { + socket, + cseq: 0, + buffer: String::new(), + }; + let response = client + .request("OPTIONS", "rtsps://localhost:322", &[], "") + .unwrap(); + assert_eq!(response.status, 200); + server.join().unwrap(); +} + +#[test] +fn rtsps_exchanges_requests_with_tls12_only_servers() { + exchange_over_tls(&[&rustls::version::TLS12], rustls::ProtocolVersion::TLSv1_2); +} + +#[test] +fn rtsps_exchanges_requests_with_tls13_only_servers() { + exchange_over_tls(&[&rustls::version::TLS13], rustls::ProtocolVersion::TLSv1_3); +} + +#[test] +fn rtsps_prefers_tls13_when_both_versions_are_available() { + exchange_over_tls(rustls::DEFAULT_VERSIONS, rustls::ProtocolVersion::TLSv1_3); +} + +#[test] +fn partner_endpoint_does_not_bypass_tls_hostname_verification() { + ensure_tls_crypto_provider().unwrap(); + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).unwrap(); + let server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(signing_key.serialize_der()).into(), + ) + .unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let connection = rustls::ServerConnection::new(Arc::new(server_config)).unwrap(); + let _ = tungstenite::accept(rustls::StreamOwned::new(connection, stream)); + }); + let stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let failure = tungstenite::client_tls_with_config( + format!("wss://partner.example:{}/rtsp", address.port()), + stream, + None, + Some(tungstenite::Connector::Rustls(Arc::new(client_config))), + ); + assert!(failure.is_err()); + server.join().unwrap(); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_transport_diagnostics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_transport_diagnostics.rs new file mode 100644 index 000000000..54c66e5dc --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_rtsp_transport_diagnostics.rs @@ -0,0 +1,176 @@ +use std::fmt::Write; +use std::net::{IpAddr, SocketAddr}; + +use opennow_streamer_transport::nvst::MAX_NVST_VIDEO_PEER_PORTS; + +pub(super) fn summarize(transport: &str) -> String { + let bytes = transport.as_bytes(); + let prefix = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]); + let lower = prefix.to_ascii_lowercase(); + let mut summary = format!( + "bytes={} input_truncated={} source_marker={} xgs_port_marker={} standard_port_marker={}", + bytes.len(), + bytes.len() > 2048, + lower.contains("source="), + lower.contains("x-gs-serverport="), + lower.contains("server_port="), + ); + let mut fields = prefix.split([';', ',']); + for (index, field) in fields.by_ref().take(16).enumerate() { + let Some((name, value)) = field.trim().split_once('=') else { + let kind = match field.trim().to_ascii_lowercase().as_str() { + "unicast" => "unicast", + "multicast" => "multicast", + "rtp/avp" | "rtp/avp/udp" => "rtp-avp", + "rtp/savp" | "rtp/savp/udp" => "rtp-savp", + "" => "empty", + _ => "other-redacted", + }; + let _ = write!(summary, " field{index}={kind}"); + continue; + }; + let (name_kind, value_kind) = match name.trim().to_ascii_lowercase().as_str() { + "source" => ("source", source_kind(value.trim())), + "x-gs-serverport" => ("xgs-port", port_kind(value.trim())), + "server_port" => ("standard-port", port_kind(value.trim())), + _ => ("other", "redacted"), + }; + let _ = write!( + summary, + " field{index}={name_kind}:{value_kind}:key-spacing-{}", + name != name.trim(), + ); + } + let _ = write!(summary, " fields_truncated={}", fields.next().is_some()); + summary +} + +fn source_kind(value: &str) -> &'static str { + if value.is_empty() { + return "empty"; + } + if let Ok(ip) = value.parse::() { + return if ip.is_ipv4() { "ipv4" } else { "ipv6" }; + } + if value.starts_with('"') && value.ends_with('"') { + return "quoted"; + } + if value + .strip_prefix('[') + .and_then(|v| v.strip_suffix(']')) + .is_some_and(|v| v.parse::().is_ok()) + { + return "bracketed-ip"; + } + if value.parse::().is_ok() { + return "ip-with-port"; + } + if value + .split_whitespace() + .next() + .is_some_and(|v| v.parse::().is_ok()) + { + return "ip-with-trailing-data"; + } + "non-ip" +} + +fn port_kind(value: &str) -> &'static str { + if value.is_empty() { + return "empty"; + } + if value.starts_with('"') && value.ends_with('"') { + return "quoted"; + } + if value.chars().any(char::is_whitespace) { + return "internal-whitespace"; + } + let (first, last) = value.split_once('-').unwrap_or((value, value)); + let Ok(first) = first.parse::() else { + return "invalid-start"; + }; + if first == 0 || first > u64::from(u16::MAX) { + return "start-out-of-range"; + } + if !value.contains('-') { + return "single-port"; + } + if last.parse::().ok().is_some_and(|last| { + last <= u64::from(u16::MAX) + && last >= first + && last - first < u64::from(MAX_NVST_VIDEO_PEER_PORTS) + }) { + "valid-range" + } else { + "range-falls-back-to-start" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transport_diagnostics_identify_expected_fields_without_their_values() { + let summary = summarize("unicast;source=192.0.2.10;X-GS-ServerPort=5004-5005"); + assert!( + summary.contains("source_marker=true xgs_port_marker=true standard_port_marker=false") + ); + assert!(summary.contains("field0=unicast field1=source:ipv4:key-spacing-false field2=xgs-port:valid-range:key-spacing-false")); + assert!(!summary.contains("192.0.2.10")); + assert!(!summary.contains("5004")); + } + + #[test] + fn transport_diagnostics_separate_source_and_port_incompatibilities() { + for (value, expected) in [ + ("source=", "source:empty"), + ("source=seat.example.test", "source:non-ip"), + ("source=\"192.0.2.10\"", "source:quoted"), + ("source=[2001:db8::1]", "source:bracketed-ip"), + ("source=192.0.2.10:5004", "source:ip-with-port"), + ("source=192.0.2.10 extra", "source:ip-with-trailing-data"), + ("source =192.0.2.10", "source:ipv4:key-spacing-true"), + ("server_port=5004", "standard-port:single-port"), + ("X-GS-ServerPort=0", "xgs-port:start-out-of-range"), + ("X-GS-ServerPort=65536", "xgs-port:start-out-of-range"), + ("X-GS-ServerPort=not-a-port", "xgs-port:invalid-start"), + ( + "X-GS-ServerPort=5004 - 5005", + "xgs-port:internal-whitespace", + ), + ("X-GS-ServerPort=\"5004\"", "xgs-port:quoted"), + ( + "X-GS-ServerPort=5004-65535", + "xgs-port:range-falls-back-to-start", + ), + ] { + assert!(summarize(value).contains(expected), "{expected}"); + } + } + + #[test] + fn transport_diagnostics_never_echo_unknown_fields_or_sensitive_values() { + let summary = summarize( + "secret-flag;secret-key=secret-value;source=secret-host;X-GS-ServerPort=secret-port;token=Bearer-secret\r\nAuthorization: secret-auth", + ); + assert!(!summary.contains("secret")); + assert!(!summary.contains("Bearer")); + assert!(!summary.contains("Authorization")); + assert!(!summary.contains('\r')); + assert!(!summary.contains('\n')); + } + + #[test] + fn transport_diagnostics_bound_input_fields_and_unicode_output() { + let summary = summarize(&"source=secret;".repeat(1000)); + assert!(summary.contains("input_truncated=true")); + assert!(summary.contains("fields_truncated=true")); + assert!(!summary.contains("field16=")); + assert!(summary.len() < 2048); + let unicode = summarize(&format!("{}🦀", "a".repeat(2047))); + assert!(unicode.contains("input_truncated=true")); + assert!(!unicode.contains('🦀')); + assert!(unicode.len() < 2048); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_server_control_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_server_control_tests.rs new file mode 100644 index 000000000..bca9dfa38 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/nvst_server_control_tests.rs @@ -0,0 +1,239 @@ +use super::*; +use opennow_streamer_transport::{NvstServerTermination, NvstSessionTimer}; + +#[test] +fn text_rejection_is_nonterminal_and_retries_after_callback_backpressure() { + let (events, received) = mpsc::sync_channel(1); + let output = EventSender::bounded(events); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut control = NvstControlOutput { start_id: "paste-session".to_owned(), ..Default::default() }; + output.send(json!({"type":"log"})).unwrap(); + assert!(!forward_nvst_event(&output, &lifecycle, 7, &resources, &mut 0, &mut control, None, NvstReceiveEvent::TextRejected("paste-queue-full"))); + assert_eq!(control.pending_text_error, Some("paste-queue-full")); + received.try_recv().unwrap(); + flush_control_output(&output, &lifecycle, 7, &mut control); + assert_eq!(received.try_recv().unwrap(), json!({"type":"text-input-error", "startId":"paste-session", "reason":"paste-queue-full"})); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert!(control.pending_text_error.is_none()); +} + +#[test] +fn idle_timer_and_input_clear_events_do_not_disturb_media() { + let (events, received) = mpsc::channel(); + let output = EventSender::unbounded(events); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut control = NvstControlOutput { + start_id: "timer-session".to_owned(), + ..Default::default() + }; + let mut attempts = 0; + for event in [ + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 4, + seconds_left: 60, + active: true, + }), + NvstReceiveEvent::UserInput, + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 5, + seconds_left: 0, + active: false, + }), + ] { + assert!(!forward_nvst_event( + &output, + &lifecycle, + 7, + &resources, + &mut attempts, + &mut control, + None, + event + )); + } + let events = received.try_iter().collect::>(); + assert_eq!(events.len(), 3); + assert_eq!( + events[0], + json!({"type":"session-timer", "startId":"timer-session", "timerType":4,"secondsLeft":60,"active":true}) + ); + assert_eq!(events[1]["active"], false); + assert_eq!(events[2]["timerType"], 5); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 0); + assert_eq!(resources.keyframe_requests.load(Ordering::Relaxed), 0); +} + +#[test] +fn idle_timer_retries_only_the_latest_state_after_event_backpressure() { + let (events, received) = mpsc::sync_channel(1); + let output = EventSender::bounded(events); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let mut control = NvstControlOutput::default(); + output.send(json!({"type":"log"})).unwrap(); + for event in [ + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 4, + seconds_left: 60, + active: true, + }), + NvstReceiveEvent::UserInput, + ] { + assert!(!forward_nvst_event( + &output, + &lifecycle, + 7, + &resources, + &mut 0, + &mut control, + None, + event + )); + } + assert_eq!( + control.pending_timer, + Some(NvstSessionTimer { + timer_type: 4, + seconds_left: 0, + active: false, + }) + ); + received.try_recv().unwrap(); + flush_control_output(&output, &lifecycle, 7, &mut control); + assert_eq!(received.try_recv().unwrap()["active"], false); + assert!(control.pending_timer.is_none()); +} + +#[test] +fn server_termination_preserves_classification_and_stops_every_leg_once() { + for (reason, extended_code, resumable) in [ + (Some(0x38), Some(0x8003_0007), json!(false)), + (Some(0x37), Some(0x8004_0010), json!(true)), + (None, None, Value::Null), + ] { + let (events, received) = mpsc::channel(); + let output = EventSender::unbounded(events); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + let termination = NvstServerTermination { + wire_code: 0x8003_0023, + reason, + extended_code, + }; + for _ in 0..2 { + assert!(forward_nvst_event( + &output, + &lifecycle, + 7, + &resources, + &mut 0, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::ServerTermination(termination) + )); + } + let events = received.try_iter().collect::>(); + assert_eq!(events.len(), 2); + assert_eq!(events[0]["type"], "error"); + assert_eq!(events[1]["status"], "stopped"); + for event in events { + assert_eq!(event["termination"]["source"], "nvst-server"); + assert_eq!(event["termination"]["wireCode"], 0x8003_0023_u32); + assert_eq!(event["termination"]["reason"], json!(reason)); + assert_eq!(event["termination"]["extendedCode"], json!(extended_code)); + assert_eq!(event["termination"]["resumable"], resumable); + } + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + assert_eq!(resources.recoveries.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Idle); + } +} + +#[test] +fn stale_server_control_events_cannot_change_a_replacement_session() { + let (events, received) = mpsc::channel(); + let output = EventSender::unbounded(events); + let lifecycle = connected_lifecycle(); + let resources = TestNvstResources::default(); + for event in [ + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 4, + seconds_left: 60, + active: true, + }), + NvstReceiveEvent::ServerTermination(NvstServerTermination { + wire_code: 0x8003_0023, + reason: Some(0x38), + extended_code: Some(0x8003_0007), + }), + ] { + assert!(forward_nvst_event( + &output, + &lifecycle, + 6, + &resources, + &mut 0, + &mut NvstControlOutput::default(), + None, + event + )); + } + assert!(received.try_recv().is_err()); + assert_eq!(resources.stops.load(Ordering::Relaxed), 0); + assert_eq!(lock_lifecycle(&lifecycle).state, State::Connected); +} + +#[test] +fn server_termination_during_setup_is_not_mistaken_for_a_duplicate() { + let (events, received) = mpsc::channel(); + let output = EventSender::unbounded(events); + let lifecycle = connected_lifecycle(); + lock_lifecycle(&lifecycle).state = State::Idle; + let resources = TestNvstResources::default(); + assert!(forward_nvst_event(&output, &lifecycle, 7, &resources, &mut 0, &mut NvstControlOutput::default(), None, NvstReceiveEvent::ServerTermination(NvstServerTermination {wire_code: 0x8003_0023, reason: Some(0x38), extended_code: Some(0x8003_0007)}))); + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + assert_eq!(received.try_iter().count(), 2); +} + +#[test] +fn terminal_events_survive_a_full_bounded_callback_queue() { + let (events, received) = mpsc::sync_channel(1); + let output = EventSender::bounded(events); + output.send(json!({"type":"log"})).unwrap(); + let worker = thread::spawn(move || { + let resources = TestNvstResources::default(); + assert!(forward_nvst_event( + &output, + &connected_lifecycle(), + 7, + &resources, + &mut 0, + &mut NvstControlOutput::default(), + None, + NvstReceiveEvent::ServerTermination(NvstServerTermination { + wire_code: 0x8004_0010, + reason: Some(0x37), + extended_code: Some(0x8004_0010) + }) + )); + assert_eq!(resources.stops.load(Ordering::Relaxed), 1); + }); + assert_eq!( + received.recv_timeout(Duration::from_secs(2)).unwrap()["type"], + "log" + ); + assert_eq!( + received.recv_timeout(Duration::from_secs(2)).unwrap()["termination"]["resumable"], + true + ); + assert_eq!( + received.recv_timeout(Duration::from_secs(2)).unwrap()["status"], + "stopped" + ); + worker.join().unwrap(); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/queue_drops.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/queue_drops.rs new file mode 100644 index 000000000..c2924238e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/queue_drops.rs @@ -0,0 +1,228 @@ +use std::collections::HashMap; +use std::time::{Duration, Instant}; + +use opennow_streamer_protocol::event; +use serde_json::{Value, json}; + +use crate::EventSender; + +pub(crate) struct QueueDropReports { + pending: HashMap<&'static str, usize>, + last_flush: Instant, +} + +impl QueueDropReports { + pub(crate) fn new() -> Self { + Self { + pending: HashMap::new(), + last_flush: Instant::now(), + } + } + + pub(crate) fn record(&mut self, media: &'static str, count: usize) { + if count != 0 { + let pending = self.pending.entry(media).or_default(); + *pending = pending.saturating_add(count); + } + } + + pub(crate) fn flush(&mut self, output: &EventSender, now: Instant, final_flush: bool) { + if !final_flush && now.duration_since(self.last_flush) < Duration::from_secs(1) { + return; + } + self.last_flush = now; + self.pending.retain(|media, count| { + let report = queue_drop_event(media, *count); + let summary = + final_flush.then(|| opennow_streamer_protocol::log::message_summary(&report)); + if output.send(report).is_ok() { + return false; + } + if let Some(summary) = summary { + opennow_streamer_protocol::log::log_line( + "WARN", + "native-queue-drop", + &format!("{summary} delivery=event-queue-unavailable"), + ); + return false; + } + true + }); + } +} + +fn queue_drop_event(media: &str, count: usize) -> Value { + let unit = match media { + "video" | "d3d11-video" | "d3d11-decode" | "present" | "linux-present" | "videotoolbox" + | "videotoolbox-hevc" | "videotoolbox-av1" | "video-decode" | "video-presentation" => { + "frames" + } + "audio-output" => "samples", + "audio" | "wasapi" | "linux-audio" | "coreaudio" => "packets", + _ => "items", + }; + let mut report = event( + "log", + json!({ + "event": "queue-dropped", + "media": media, + "count": count, + "unit": unit, + "level": "debug", + "message": format!("Low-latency {media} queues dropped {count} stale {unit}") + }), + ); + if unit == "samples" { + report["sampleRate"] = json!(48_000); + report["channels"] = json!(2); + } + report +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_units_preserve_count_and_only_describe_known_sample_duration() { + for (sources, unit) in [ + ( + &[ + "video", + "d3d11-video", + "d3d11-decode", + "present", + "linux-present", + "videotoolbox", + "videotoolbox-hevc", + "videotoolbox-av1", + "video-decode", + "video-presentation", + ][..], + "frames", + ), + ( + &["audio", "wasapi", "linux-audio", "coreaudio"][..], + "packets", + ), + (&["audio-output"][..], "samples"), + (&["unknown", "future-audio", "future-video"][..], "items"), + ] { + for source in sources { + let report = queue_drop_event(source, 37); + assert_eq!(report["event"], "queue-dropped"); + assert_eq!(report["media"], *source); + assert_eq!(report["count"], 37); + assert_eq!(report["unit"], unit); + if unit == "samples" { + assert_eq!(report["sampleRate"], 48_000); + assert_eq!(report["channels"], 2); + } else { + assert!(report.get("sampleRate").is_none()); + assert!(report.get("channels").is_none()); + } + } + } + } + + #[test] + fn periodic_flush_delivers_idle_sources_and_final_flush_is_exact() { + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = EventSender::unbounded(sender); + let mut reports = QueueDropReports::new(); + let now = reports.last_flush; + reports.record("video", 3); + reports.record("video", 4); + reports.record("audio-output", 96_000); + reports.record("audio", 2); + reports.record("unknown", 0); + reports.flush(&sender, now, false); + assert!(receiver.try_recv().is_err()); + reports.flush(&sender, now + Duration::from_secs(1), false); + let delivered: HashMap<_, _> = receiver + .try_iter() + .map(|value| { + ( + value["media"].as_str().unwrap().to_owned(), + value["count"].as_u64().unwrap(), + ) + }) + .collect(); + assert_eq!( + delivered, + HashMap::from([ + ("video".to_owned(), 7), + ("audio-output".to_owned(), 96_000), + ("audio".to_owned(), 2), + ]) + ); + reports.record("video", 5); + reports.flush(&sender, now + Duration::from_secs(1), true); + assert_eq!(receiver.try_recv().unwrap()["count"], 5); + reports.flush(&sender, now + Duration::from_secs(2), true); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn full_event_queue_retains_pending_counts_for_retry() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let sender = EventSender::bounded(sender); + sender.send(json!({"type": "telemetry"})).unwrap(); + let mut reports = QueueDropReports::new(); + let now = reports.last_flush; + reports.record("video", 3); + reports.flush(&sender, now + Duration::from_secs(1), false); + receiver.recv().unwrap(); + reports.record("video", 4); + reports.flush(&sender, now + Duration::from_secs(2), false); + assert_eq!(receiver.recv().unwrap()["count"], 7); + reports.flush(&sender, now + Duration::from_secs(3), false); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn final_flush_does_not_wait_for_dispatcher_capacity() { + let _sink = crate::LOG_SINK_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let path = std::env::temp_dir().join(format!( + "opennow-final-queue-drops-{}.log", + std::process::id() + )); + let _ = std::fs::remove_file(&path); + opennow_streamer_protocol::log::set_log_file(path.to_str().unwrap()).unwrap(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let sender = EventSender::bounded(sender); + sender.send(json!({"type": "telemetry"})).unwrap(); + let (finished, completion) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let mut reports = QueueDropReports::new(); + reports.record("audio-output", 96_000); + reports.flush(&sender, Instant::now(), true); + assert!(reports.pending.is_empty()); + reports.flush(&sender, Instant::now(), true); + finished.send(()).unwrap(); + }); + completion.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(receiver.recv().unwrap()["type"], "telemetry"); + worker.join().unwrap(); + assert!(receiver.try_recv().is_err()); + let log = std::fs::read_to_string(&path).unwrap(); + let reports: Vec<_> = log + .lines() + .filter(|line| line.contains("delivery=event-queue-unavailable")) + .collect(); + assert_eq!(reports.len(), 1); + for field in [ + "event=queue-dropped", + "media=audio-output", + "unit=samples", + "count=96000", + "sampleRate=48000", + "channels=2", + ] { + assert!(reports[0].contains(field), "{}", reports[0]); + } + let _ = std::fs::remove_file(path); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/recording_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/recording_tests.rs new file mode 100644 index 000000000..7a6fed968 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/recording_tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn command(value: Value) -> Command { + serde_json::from_value(value).expect("command") +} + +#[test] +fn recording_cut_preserves_a_prefix_and_emits_one_completion() { + let (host, runtime) = opennow_streamer_platform::create_test_runtime(); + let (events, received) = std::sync::mpsc::channel(); + let mut engine = Engine::with_media_runtime(events, runtime.clone()); + let (feedback, _feedback_receiver) = std::sync::mpsc::channel(); + let session = runtime + .start(feedback, MediaStreamConfig::default()) + .unwrap(); + let sink = session.sink(); + engine.media_session = Some(session); + lock_lifecycle(&engine.lifecycle).state = State::Connected; + let directory = std::env::temp_dir().join(format!( + "opennow-cut-completion-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + let output = directory.join("cut.mkv"); + let (started, _) = engine.handle(command(json!({ + "id":"cut-start", "type":"recording-start", "outputPath":output + }))); + assert_eq!(started[0]["type"], "recording-started"); + let first_frame = EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([ + 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x28, 0xde, 0xad, 0, 0, 0, 1, 0x68, 0xee, 0x3c, 0x80, 0, + 0, 0, 1, 0x65, 0x88, 0x84, 0x00, 0x10, + ]), + frame_index: Some(10), + timestamp: 90_000, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: Some(7), + }; + sink.push(first_frame.clone()); + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0, 0, 0, 1, 0x41]), + frame_index: Some(11), + timestamp: 93_000, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: false, + ssrc: Some(7), + }); + let completion = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(completion["type"], "recording-state"); + assert_eq!(completion["state"], "saved"); + assert_eq!(completion["completion"]["kind"], "cut"); + assert_eq!(completion["completion"]["reason"], "discontinuity"); + assert_eq!(completion["requestId"], "cut-start"); + assert_eq!(completion["videoPackets"], 1); + assert!(output.is_file()); + let original = std::fs::read(&output).unwrap(); + let (stopped, _) = engine.handle(command(json!({"id":"first-stop", "type":"recording-stop"}))); + assert_eq!(stopped[0]["type"], "recording-stopped"); + assert_eq!(stopped[0]["requestId"], "cut-start"); + assert_eq!(stopped[0]["completion"], completion["completion"]); + let (repeated, _) = engine.handle(command( + json!({"id":"second-stop", "type":"recording-stop"}), + )); + assert_eq!(repeated[0]["type"], "recording-not-active"); + assert!(received.try_recv().is_err()); + assert_eq!(std::fs::read(&output).unwrap(), original); + let next_output = directory.join("next.mkv"); + let (restarted, _) = engine.handle(command(json!({ + "id":"next-start", "type":"recording-start", "outputPath":next_output + }))); + assert_eq!(restarted[0]["type"], "recording-started"); + sink.push(first_frame); + engine.stop("test complete"); + let saved = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(saved["state"], "saved"); + assert_eq!(saved["completion"]["kind"], "complete"); + assert_eq!(saved["requestId"], "next-start"); + assert!(next_output.is_file()); + runtime.shutdown(); + host.join().unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn recording_can_restart_after_finished_worker_failure() { + let (host, runtime) = opennow_streamer_platform::create_test_runtime(); + let (events, received) = std::sync::mpsc::channel(); + let mut engine = Engine::with_media_runtime(events, runtime.clone()); + let (feedback, _feedback_receiver) = std::sync::mpsc::channel(); + let session = runtime + .start(feedback, MediaStreamConfig::default()) + .unwrap(); + let sink = session.sink(); + engine.media_session = Some(session); + lock_lifecycle(&engine.lifecycle).state = State::Connected; + let directory = std::env::temp_dir().join(format!( + "opennow-recording-retry-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + let (started, _) = engine.handle(command(json!({ + "id":"first", "type":"recording-start", "outputPath":directory.join("first.mkv") + }))); + assert_eq!(started[0]["type"], "recording-started"); + let (busy, _) = engine.handle(command(json!({ + "id":"busy", "type":"recording-start", "outputPath":directory.join("busy.mkv") + }))); + assert_eq!(busy[0]["code"], "recording-already-active"); + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0, 0, 0, 1, 0x41]), + frame_index: Some(2), + timestamp: 90_000, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: false, + ssrc: None, + }); + let completion = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(completion["type"], "recording-state"); + assert_eq!(completion["state"], "failed"); + assert_eq!( + completion["message"], + "recording stopped because the video stream was discontinuous" + ); + assert!( + engine + .recording_worker + .as_ref() + .unwrap() + .completed + .load(Ordering::Acquire) + ); + let deadline = Instant::now() + Duration::from_secs(5); + while !engine + .recording_worker + .as_ref() + .unwrap() + .thread + .is_finished() + { + assert!(Instant::now() < deadline, "recording worker did not finish"); + thread::sleep(Duration::from_millis(1)); + } + let output = directory.join("retry.mkv"); + let (retried, _) = engine.handle(command(json!({ + "id":"retry", "type":"recording-start", "outputPath":output + }))); + assert_eq!(retried[0]["type"], "recording-started"); + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([ + 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x28, 0xde, 0xad, 0, 0, 0, 1, 0x68, 0xee, 0x3c, 0x80, 0, + 0, 0, 1, 0x65, 0x88, 0x84, 0x00, 0x10, + ]), + frame_index: Some(3), + timestamp: 91_500, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }); + let (stopped, _) = engine.handle(command(json!({"id":"stop", "type":"recording-stop"}))); + assert_eq!(stopped[0]["type"], "recording-stopped"); + assert_eq!(stopped[0]["videoPackets"], 1); + let completion = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(completion["state"], "saved"); + assert!(output.is_file()); + assert!(engine.recording_worker.is_none()); + engine.stop("test complete"); + runtime.shutdown(); + host.join().unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn recording_retry_reaps_terminal_worker_before_thread_exit() { + let (host, runtime) = opennow_streamer_platform::create_test_runtime(); + let (events, received) = std::sync::mpsc::channel(); + let mut engine = Engine::with_media_runtime(events, runtime.clone()); + let (feedback, _feedback_receiver) = std::sync::mpsc::channel(); + let session = runtime + .start(feedback, MediaStreamConfig::default()) + .unwrap(); + let (_, receiver) = session.control().subscribe_recording().unwrap(); + engine.media_session = Some(session); + lock_lifecycle(&engine.lifecycle).state = State::Connected; + let completed = Arc::new(AtomicBool::new(false)); + let worker_completed = Arc::clone(&completed); + let events = engine.events.clone(); + engine.recording_worker = Some(RecordingWorker { + request_id: "terminal-fixture".to_owned(), + completed, + thread: thread::spawn(move || { + worker_completed.store(true, Ordering::Release); + events + .send(event("recording-state", json!({"state":"failed"}))) + .unwrap(); + assert!(receiver.recv().is_err()); + Err("terminal recording failure".to_owned()) + }), + }); + let completion = received.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(completion["state"], "failed"); + assert!( + !engine + .recording_worker + .as_ref() + .unwrap() + .thread + .is_finished() + ); + let path = std::env::temp_dir().join(format!( + "opennow-recording-terminal-retry-{}.mkv", + std::process::id() + )); + let (retried, _) = engine.handle(command(json!({ + "id":"retry", "type":"recording-start", "outputPath":path + }))); + let stopped = engine.stop_recording_inner(); + engine.stop("test complete"); + runtime.shutdown(); + host.join().unwrap(); + assert_eq!(retried[0]["type"], "recording-started"); + assert_eq!( + stopped.unwrap_err(), + "recording ended before a decodable video keyframe arrived" + ); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/telemetry.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/telemetry.rs new file mode 100644 index 000000000..2a5e264eb --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/telemetry.rs @@ -0,0 +1,30 @@ +use super::*; + +pub(super) const TELEMETRY_INTERVAL: Duration = Duration::from_secs(1); + +pub(super) fn media_telemetry( + elapsed: Duration, + frames: u64, + bytes: u64, + socket_bytes: Option, + previous_socket_bytes: &mut u64, + peak_bitrate_mbps: &mut f64, +) -> Value { + let seconds = elapsed.as_secs_f64(); + let bitrate_mbps = bytes as f64 * 8.0 / seconds / 1_000_000.0; + let receive_bitrate_mbps = socket_bytes.and_then(|bytes| { + bytes + .checked_sub(*previous_socket_bytes) + .map(|delta| delta as f64 * 8.0 / seconds / 1_000_000.0) + }); + if let Some(bytes) = socket_bytes { + *previous_socket_bytes = bytes; + } + *peak_bitrate_mbps = peak_bitrate_mbps.max(bitrate_mbps); + json!({ + "framesPerSecond": frames as f64 / seconds, + "bitrateMbps": bitrate_mbps, + "receiveBitrateMbps": receive_bitrate_mbps, + "peakBitrateMbps": peak_bitrate_mbps, + }) +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/input.rs new file mode 100644 index 000000000..9c4827f7d --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/input.rs @@ -0,0 +1,185 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use super::*; + +#[derive(Default)] +pub(super) struct InputState { + pub(super) paused: bool, + keys: BTreeSet, + buttons: BTreeSet, + gamepads: BTreeMap, +} + +impl InputState { + pub(super) fn sent(&mut self, input: &CapturedInput) -> Result<(), Failure> { + match input { + CapturedInput::Key { + virtual_key, + pressed, + .. + } => { + if *pressed { + if self.keys.len() >= 256 && !self.keys.contains(virtual_key) { + return Err(Failure { + code: "webrtc-input-overflow", + message: "Too many concurrently held keys".to_owned(), + }); + } + self.keys.insert(*virtual_key); + } else { + self.keys.remove(virtual_key); + } + } + CapturedInput::MouseButton { button, pressed } => { + if *pressed { + self.buttons.insert(*button); + } else { + self.buttons.remove(button); + } + } + CapturedInput::Gamepad { + controller_id, + bitmap, + .. + } => { + self.gamepads.insert(*controller_id & 3, *bitmap); + } + _ => {} + } + Ok(()) + } + + pub(super) fn set_paused(&mut self, paused: bool) -> Vec { + self.paused = paused; + if !paused { + return Vec::new(); + } + let mut neutral = Vec::new(); + neutral.extend( + std::mem::take(&mut self.keys) + .into_iter() + .map(|virtual_key| CapturedInput::Key { + virtual_key, + modifiers: 0, + pressed: false, + }), + ); + neutral.extend(std::mem::take(&mut self.buttons).into_iter().map(|button| { + CapturedInput::MouseButton { + button, + pressed: false, + } + })); + neutral.extend(std::mem::take(&mut self.gamepads).into_iter().map( + |(controller_id, bitmap)| CapturedInput::Gamepad { + controller_id, + bitmap, + buttons: 0, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + }, + )); + neutral + } +} + +pub(super) fn send_and_flush( + control: &TransportControl, + inputs: Vec, + timestamp: u64, +) -> Result<(), transport::TransportError> { + if inputs.is_empty() { + return Ok(()); + } + let deadline = Instant::now() + Duration::from_millis(150); + for input in inputs { + let packet = captured_input_packet(input, timestamp); + loop { + if Instant::now() >= deadline { + return Err(transport::TransportError::Backpressured); + } + match control.send_input(packet.clone(), false) { + Ok(()) => break, + Err(transport::TransportError::Backpressured) => { + control.flush_input(deadline.saturating_duration_since(Instant::now()))? + } + Err(error) => return Err(error), + } + } + } + control.flush_input(deadline.saturating_duration_since(Instant::now())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn input_pause_releases_keys_buttons_and_controller_without_repressing_on_resume() { + let mut state = InputState::default(); + state + .sent(&CapturedInput::Key { + virtual_key: 65, + modifiers: 1, + pressed: true, + }) + .unwrap(); + state + .sent(&CapturedInput::MouseButton { + button: 1, + pressed: true, + }) + .unwrap(); + state + .sent(&CapturedInput::Gamepad { + controller_id: 2, + bitmap: 0x0404, + buttons: 0xffff, + left_trigger: 255, + right_trigger: 255, + left_stick_x: 32767, + left_stick_y: 32767, + right_stick_x: 32767, + right_stick_y: 32767, + }) + .unwrap(); + let release = state.set_paused(true); + assert_eq!(release.len(), 3); + assert!(matches!( + release[0], + CapturedInput::Key { + virtual_key: 65, + pressed: false, + modifiers: 0 + } + )); + assert!(matches!( + release[1], + CapturedInput::MouseButton { + button: 1, + pressed: false + } + )); + assert!(matches!( + release[2], + CapturedInput::Gamepad { + controller_id: 2, + bitmap: 0x0404, + buttons: 0, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0 + } + )); + assert!(state.set_paused(true).is_empty()); + assert!(state.set_paused(false).is_empty()); + assert!(!state.paused); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/mod.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/mod.rs new file mode 100644 index 000000000..3743ef0d1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/mod.rs @@ -0,0 +1,818 @@ +use std::sync::mpsc::{self, Receiver, SyncSender}; + +use opennow_streamer_protocol::AudioOutputDevice; +use opennow_streamer_transport::webrtc::{self as transport, TransportControl, TransportEvent}; + +use super::*; + +mod input; +mod sdp; +mod signaling; + +const MAX_SDP_BYTES: usize = 256 * 1024; +const MAX_PENDING_CANDIDATES: usize = 64; +const STARTUP_TIMEOUT: Duration = Duration::from_secs(20); +const VIDEO_TIMEOUT: Duration = Duration::from_secs(8); +const RECOVERY_GRACE: Duration = Duration::from_secs(4); +const SIGNALING_POLL_INTERVAL: Duration = Duration::from_millis(1); + +#[derive(Debug)] +struct Failure { + code: &'static str, + message: String, +} + +impl Failure { + fn signaling(message: impl Into) -> Self { + Self { + code: "webrtc-signaling-failed", + message: message.into(), + } + } + + fn transport(error: transport::TransportError) -> Self { + Self { + code: error.code(), + message: error.to_string(), + } + } +} + +pub(super) struct OwnedSession { + cancelled: Arc, + input_ready: Arc, + commands: SyncSender, + worker: Option>, +} + +enum SessionCommand { + AntiAfk, + Pause(bool), +} + +impl OwnedSession { + pub(super) fn stop(mut self) { + self.cancelled.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } + + pub(super) fn anti_afk(&self) -> Result<(), String> { + if !self.input_ready.load(Ordering::Acquire) || self.cancelled.load(Ordering::Acquire) { + return Err("WebRTC input handshake is not ready".to_owned()); + } + self.commands + .try_send(SessionCommand::AntiAfk) + .map_err(|_| "WebRTC input command queue unavailable".to_owned()) + } + + pub(super) fn set_paused(&self, paused: bool) -> Result<(), String> { + self.commands + .try_send(SessionCommand::Pause(paused)) + .map_err(|_| "WebRTC input pause queue unavailable".to_owned()) + } +} + +impl Drop for OwnedSession { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl Engine { + pub(super) fn start_webrtc( + &mut self, + id: String, + mut context: SessionContext, + audio_device: AudioOutputDevice, + ) -> Result, Value> { + if let Some(profile) = context + .session + .extra + .get_mut("negotiatedStreamProfile") + .and_then(Value::as_object_mut) + { + if profile.get("fps").is_some_and(Value::is_null) { + profile.remove("fps"); + } + } + let stream = media_stream_config(&context); + let accepted = context.session.extra.get("negotiatedStreamProfile"); + let codec = accepted + .and_then(|profile| profile.get("codec")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("codec").and_then(Value::as_str)) + .unwrap_or("H264"); + let color = accepted + .and_then(|profile| profile.get("colorQuality")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("colorQuality").and_then(Value::as_str)) + .unwrap_or("8bit_420"); + let video_codec = match ( + codec.to_ascii_uppercase().as_str(), + color, + stream.codec, + stream.color_quality, + ) { + ("H264", "8bit_420", MediaVideoCodec::H264, MediaColorQuality::EightBit420) => { + Some(transport::NegotiatedVideoCodec::H264) + } + ( + "H265" | "HEVC", + "8bit_420", + MediaVideoCodec::H265, + MediaColorQuality::EightBit420, + ) => Some(transport::NegotiatedVideoCodec::H265Main), + ("H265" | "HEVC", "10bit_420", MediaVideoCodec::H265, MediaColorQuality::TenBit420) => { + Some(transport::NegotiatedVideoCodec::H265Main10) + } + _ => None, + }; + let Some(video_codec) = video_codec.filter(|_| !stream.hdr) else { + return Err(error( + Some(&id), + "webrtc-profile-unsupported", + "WebRTC requires H264 8-bit or H265 8/10-bit 4:2:0 SDR", + )); + }; + let resolution_matches = accepted + .and_then(|profile| profile.get("resolution")) + .and_then(Value::as_str) + .or_else(|| context.settings.get("resolution").and_then(Value::as_str)) + .is_none_or(|resolution| { + resolution + .to_ascii_lowercase() + .split_once('x') + .and_then(|(width, height)| { + Some((width.parse::().ok()?, height.parse::().ok()?)) + }) + == Some((stream.width, stream.height)) + }); + let fps_matches = accepted + .and_then(|profile| profile.get("fps")) + .or_else(|| context.settings.get("fps")) + .is_none_or(|fps| fps.as_u64() == Some(u64::from(stream.fps))); + if !resolution_matches || !fps_matches { + return Err(error( + Some(&id), + "webrtc-profile-unsupported", + "WebRTC resolution and frame rate must match the supported native stream configuration", + )); + } + signaling::sign_in_url(&context.session, "validation") + .map_err(|failure| error(Some(&id), failure.code, failure.message))?; + if context.nvst_video.is_some() { + return Err(error( + Some(&id), + "webrtc-context-invalid", + "A WebRTC context cannot contain an NVST handoff", + )); + } + if let Some(session) = self.webrtc_session.take() { + session.stop(); + } + if let Some(transport) = self.nvst_transport.take() { + transport.stop(); + } + if let Some(transport) = self.nvst_mjolnir_transport.take() { + transport.stop(); + } + if let Some(mut rtsp) = self.nvst_rtsp.take() { + rtsp.shutdown(); + } + self.reserved_nvst_bundle = None; + self.nvst_hole_punch_socket = None; + self.stop_media_resources(); + if let Some(runtime) = self.media_runtime.clone() { + let (feedback_sender, feedback_receiver) = mpsc::channel(); + let session = runtime + .start_with_audio_device( + feedback_sender, + stream, + context + .settings + .get("nativeVideoBackend") + .and_then(Value::as_str) + .unwrap_or("auto"), + audio_device, + ) + .map_err(|message| error(Some(&id), "media-output-unavailable", message))?; + session.control().start_replay( + ReplayBufferConfig::from_settings(&context.settings), + Arc::clone(&self.replay_budget), + ); + let sink = session.sink(); + let (consumer, receiver) = mpsc::sync_channel(ENCODED_MEDIA_QUEUE_CAPACITY); + let output = self.events.clone(); + let worker = match thread::Builder::new() + .name("opennow-media-consumer".to_owned()) + .spawn(move || consume_encoded_media(&output, receiver, sink)) + { + Ok(worker) => worker, + Err(spawn_error) => { + session.stop(); + return Err(error( + Some(&id), + "media-worker-failed", + spawn_error.to_string(), + )); + } + }; + self.media_consumer = Some(consumer); + self.media_session = Some(session); + self.media_worker = Some(worker); + self.media_feedback = Some(feedback_receiver); + } + let Some(consumer) = self.media_consumer.clone() else { + self.stop_media_resources(); + return Err(error( + Some(&id), + "media-consumer-unavailable", + "WebRTC requires an in-process encoded media consumer", + )); + }; + let generation = { + let mut lifecycle = lock_lifecycle(&self.lifecycle); + lifecycle.generation = lifecycle.generation.wrapping_add(1); + lifecycle.context = Some(context.clone()); + lifecycle.state = State::Connected; + lifecycle.generation + }; + let replay_enabled = self.media_session.is_some() + && ReplayBufferConfig::from_settings(&context.settings).enabled; + let cancelled = Arc::new(AtomicBool::new(false)); + let input_ready = Arc::new(AtomicBool::new(false)); + let (command_sender, commands) = mpsc::sync_channel(4); + let resources = Worker { + context, + stream, + video_codec, + start_id: id.clone(), + lifecycle: self.lifecycle.clone(), + generation, + cancelled: cancelled.clone(), + input_ready: input_ready.clone(), + output: self.events.clone(), + feedback: self.media_feedback.take(), + captured_input: self + .media_session + .as_ref() + .map(MediaSession::captured_input), + media: self.media_session.as_ref().map(MediaSession::control), + runtime: self.media_runtime.clone(), + commands, + }; + let worker = match thread::Builder::new() + .name("opennow-webrtc-session".to_owned()) + .spawn(move || { + let result = resources.run(consumer); + if let Err(failure) = result { + resources.terminal(failure); + } + resources.input_ready.store(false, Ordering::Release); + if let Some(queue) = &resources.captured_input { + queue.set_text_ready(generation, false); + queue.clear(); + } + if let Some(media) = &resources.media { + media.stop(); + } + }) { + Ok(worker) => worker, + Err(spawn_error) => { + self.stop("WebRTC worker startup failed"); + return Err(error( + Some(&id), + "media-worker-failed", + spawn_error.to_string(), + )); + } + }; + self.webrtc_session = Some(OwnedSession { + cancelled, + input_ready, + commands: command_sender, + worker: Some(worker), + }); + let mut result = response(id, "ok"); + result["transport"] = json!("webrtc"); + result["replayEnabled"] = json!(replay_enabled); + result["capabilities"] = json!({ + "supportsInput":true,"supportsAudioDecode":supports_audio_decode(), + "supportsAudioOutput":supports_audio_output(),"supportsMicrophone":false + }); + Ok(vec![result]) + } +} + +struct Worker { + context: SessionContext, + stream: MediaStreamConfig, + video_codec: transport::NegotiatedVideoCodec, + start_id: String, + lifecycle: Arc>, + generation: u64, + cancelled: Arc, + input_ready: Arc, + output: EventSender, + feedback: Option>, + captured_input: Option>, + media: Option, + runtime: Option, + commands: Receiver, +} + +impl Worker { + fn active(&self) -> bool { + !self.cancelled.load(Ordering::Acquire) + && lock_lifecycle(&self.lifecycle).generation == self.generation + } + + fn emit(&self, name: &str, mut payload: Value) { + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation != self.generation + || lifecycle.state == State::Idle + || self.cancelled.load(Ordering::Acquire) + { + return; + } + payload["startId"] = json!(self.start_id); + payload["transport"] = json!("webrtc"); + let _ = self.output.send(event(name, payload)); + } + + fn terminal(&self, failure: Failure) { + let mut lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation != self.generation + || lifecycle.state == State::Idle + || self.cancelled.load(Ordering::Acquire) + { + return; + } + lifecycle.context = None; + lifecycle.state = State::Idle; + let termination = json!({"source":"webrtc-transport","code":failure.code,"resumable":null}); + let _ = self.output.send(event("error", json!({"code":failure.code,"message":failure.message,"termination":termination,"startId":self.start_id,"transport":"webrtc"}))); + let _ = self.output.send(event("status", json!({"status":"stopped","message":failure.message,"termination":termination,"startId":self.start_id,"transport":"webrtc"}))); + } + + fn run(&self, consumer: MediaConsumer) -> Result<(), Failure> { + let published_baseline = self + .runtime + .as_ref() + .and_then(MediaRuntime::published_video_frames); + if let Some(queue) = &self.captured_input { + queue.set_text_ready(self.generation, false); + queue.clear(); + } + self.emit( + "status", + json!({"status":"connecting","message":"Connecting WebRTC compatibility transport"}), + ); + self.emit("microphone-state", json!({"state":"unavailable","enabled":false,"message":"WebRTC compatibility does not support microphone audio"})); + let mut signaling = signaling::Signaling::connect(&self.context.session, &self.cancelled)?; + if !self.active() { + return Ok(()); + } + let peer_info = signaling + .protocol + .peer_info(self.stream.width, self.stream.height); + signaling.send(peer_info)?; + let deadline = Instant::now() + STARTUP_TIMEOUT; + let mut candidates = Vec::new(); + let offer = loop { + if !self.active() { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(Failure::signaling("Timed out waiting for the WebRTC offer")); + } + match signaling.poll()? { + Some(signaling::Incoming::Offer(offer)) => break offer, + Some(signaling::Incoming::Candidate(candidate)) => { + if candidates.len() >= MAX_PENDING_CANDIDATES { + return Err(Failure::signaling("Too many queued ICE candidates")); + } + candidates.push(candidate); + } + Some(signaling::Incoming::Closed(detail)) => { + return Err(Failure::signaling(format!( + "Signaling peer closed before the offer ({detail})" + ))); + } + None => thread::sleep(SIGNALING_POLL_INTERVAL), + } + }; + let threshold = sdp::partial_reliability(&offer)?; + let mut transport_session = self.context.session.clone(); + let host = transport_session + .media_connection_info + .as_mut() + .map(|endpoint| &mut endpoint.ip) + .unwrap_or(&mut transport_session.server_ip); + if host.parse::().is_err() { + let addresses = signaling::resolve(host.clone(), 443, &self.cancelled)?; + let address = addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| addresses.first()) + .ok_or_else(|| Failure::signaling("Media endpoint DNS returned no address"))?; + *host = address.ip().to_string(); + } + if !self.active() { + return Ok(()); + } + let (sender, receiver) = mpsc::sync_channel(64); + let negotiated = transport::negotiate( + &offer, + &transport_session, + self.video_codec, + threshold, + sender, + consumer, + ) + .map_err(Failure::transport)?; + if !self.active() { + return Ok(()); + } + let nvst = sdp::nvst_answer(&offer, &negotiated.answer_sdp, self.stream)?; + let video_mid = negotiated.video_mid.to_string(); + signaling.send_peer(json!({"type":"answer","sdp":negotiated.answer_sdp,"nvstSdp":nvst}))?; + signaling.send_peer( + serde_json::to_value(&negotiated.local_candidate) + .map_err(|_| Failure::signaling("Could not serialize local ICE"))?, + )?; + let session = negotiated.session; + for candidate in candidates { + session + .add_remote_candidate(&candidate) + .map_err(Failure::transport)?; + } + let control = session.control(); + let mut progress = Progress::new(Instant::now(), published_baseline); + let mut input_origin = Instant::now(); + let mut input_state = input::InputState::default(); + let mut drops = QueueDropReports::new(); + let mut telemetry = Telemetry::new(Instant::now()); + let result = (|| -> Result<(), Failure> { + while self.active() { + let iteration_started = Instant::now(); + for incoming in receiver.try_iter().take(64) { + match incoming { + TransportEvent::Connected => self.emit("log", json!({"level":"info","message":"WebRTC transport connected; waiting for decoded media"})), + TransportEvent::Disconnected(message) => return Err(Failure { code:"webrtc-disconnected", message }), + TransportEvent::InputReady(version) => { + if !self.input_ready.swap(true, Ordering::AcqRel) { + input_origin = Instant::now(); + } + self.emit("input-ready", json!({"protocolVersion":version})); + } + TransportEvent::InputUnavailable(message) => return Err(Failure { code:"webrtc-input-unavailable", message }), + TransportEvent::Log(message) => self.emit("log", json!({"level":"info","message":message})), + TransportEvent::Stats(stats) => telemetry.transport = stats, + } + } + if let Some(feedback) = &self.feedback { + for feedback in feedback.try_iter().take(128) { + self.feedback( + feedback, + &control, + &video_mid, + &mut progress, + &mut drops, + &mut telemetry, + )?; + } + } + self.flush_telemetry(&mut telemetry, Instant::now()); + progress.observe_published( + self.runtime + .as_ref() + .and_then(MediaRuntime::published_video_frames), + Instant::now(), + ); + if progress.poll(Instant::now())? { + control + .request_keyframe(video_mid.clone()) + .map_err(Failure::transport)?; + } + { + let lifecycle = lock_lifecycle(&self.lifecycle); + if lifecycle.generation != self.generation { + break; + } + drops.flush(&self.output, Instant::now(), false); + } + if let Some(queue) = &self.captured_input { + if !self.input_ready.load(Ordering::Acquire) { + queue.clear(); + } else if queue.take_overflowed() { + return Err(Failure { + code: "webrtc-input-overflow", + message: + "Captured input queue overflowed; stopping to prevent stuck controls" + .to_owned(), + }); + } + for _ in 0..128 { + let Some(sample) = queue.take_sample() else { + break; + }; + if matches!(sample.input, CapturedInput::Guide) { + self.emit("overlay-request", json!({"source":"gamepad"})); + continue; + } + if matches!(sample.input, CapturedInput::Screenshot) { + self.emit("screenshot-request", json!({"source":"keyboard"})); + continue; + } + if matches!(sample.input, CapturedInput::RecordingToggle) { + self.emit("recording-toggle-request", json!({"source":"keyboard"})); + continue; + } + let action = match &sample.input { + CapturedInput::Shortcut(action) => Some(*action), + _ => None, + }; + if let Some(action) = action { + if self.active() { + forward_shortcut_action( + &self.output, + self.runtime.as_ref(), + action, + ); + } + continue; + } + if matches!(sample.input, CapturedInput::Text(_)) { + return Err(Failure { + code: "webrtc-text-unavailable", + message: "WebRTC text submission is unavailable".to_owned(), + }); + } + if self.input_ready.load(Ordering::Acquire) && !input_state.paused { + let timestamp = sample + .captured_at + .saturating_duration_since(input_origin) + .as_micros() + .min(u128::from(u64::MAX)) + as u64; + input_state.sent(&sample.input)?; + control + .send_input(captured_input_packet(sample.input, timestamp), false) + .map_err(Failure::transport)?; + } + } + } + for command in self.commands.try_iter().take(4) { + let inputs = match command { + SessionCommand::Pause(paused) => { + if let Some(queue) = &self.captured_input { + queue.clear(); + } + input_state.set_paused(paused) + } + SessionCommand::AntiAfk if !input_state.paused => [true, false] + .map(|pressed| CapturedInput::Key { + virtual_key: 0x7c, + modifiers: 0, + pressed, + }) + .to_vec(), + SessionCommand::AntiAfk => Vec::new(), + }; + if !self.input_ready.load(Ordering::Acquire) { + continue; + } + let timestamp = + input_origin.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + input::send_and_flush(&control, inputs, timestamp) + .map_err(Failure::transport)?; + } + match signaling.poll()? { + Some(signaling::Incoming::Candidate(candidate)) => session + .add_remote_candidate(&candidate) + .map_err(Failure::transport)?, + Some(signaling::Incoming::Offer(_)) => { + return Err(Failure::signaling( + "WebRTC renegotiation requires a fresh session start", + )); + } + Some(signaling::Incoming::Closed(detail)) => { + return Err(Failure::signaling(format!( + "Signaling peer closed ({detail})" + ))); + } + None => {} + } + thread::sleep(SIGNALING_POLL_INTERVAL.saturating_sub(iteration_started.elapsed())); + } + Ok(()) + })(); + let neutral = input_state.set_paused(true); + if !neutral.is_empty() && self.input_ready.load(Ordering::Acquire) { + let timestamp = input_origin.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let _ = input::send_and_flush(&control, neutral, timestamp); + } + session.stop(); + result + } + + fn feedback( + &self, + feedback: MediaFeedback, + control: &TransportControl, + video_mid: &str, + progress: &mut Progress, + drops: &mut QueueDropReports, + telemetry: &mut Telemetry, + ) -> Result<(), Failure> { + progress.observe_feedback(&feedback, Instant::now()); + telemetry.observe(&feedback); + match feedback { + MediaFeedback::VideoFrameAccepted { .. } | MediaFeedback::DecodeTimings(_) => {} + MediaFeedback::PlaybackStarted { backend } => { + self.emit("status", json!({"event":"first-frame","backend":backend,"status":"streaming","message":"WebRTC presented the first video frame"})); + } + MediaFeedback::RequestKeyframe { mid, .. } => control.request_keyframe(mid).map_err(Failure::transport)?, + MediaFeedback::DecoderError { message, .. } => { + self.emit("log", json!({"level":"warn","message":message})); + control.request_keyframe(video_mid).map_err(Failure::transport)?; + } + MediaFeedback::OutputError { message } => return Err(Failure { code:"media-output-error", message }), + MediaFeedback::AudioDecoderError { message, consecutive } => self.emit("log", json!({"level":"warn","message":message,"consecutive":consecutive})), + MediaFeedback::AudioUnavailable { backend, reason, rejected } => self.emit("log", json!({"level":"warn","backend":backend,"message":reason,"rejected":rejected})), + MediaFeedback::BackendFallback { from, to, reason } => self.emit("log", json!({"event":"backend-fallback","fromBackend":from,"toBackend":to,"reason":reason})), + MediaFeedback::ColorFormatChanged { requested, actual } => self.emit("log", json!({"level":"warn","message":format!("Color format changed from {requested:?} to {actual:?}")})), + MediaFeedback::DeviceLost { subsystem, recovered, message } => { + self.emit("log", json!({"event":"device-state","subsystem":subsystem,"recovered":recovered,"message":message})); + if recovered { control.request_keyframe(video_mid).map_err(Failure::transport)?; } + } + MediaFeedback::QueueDropped { media, count } => drops.record(media, count), + } + Ok(()) + } + + fn flush_telemetry(&self, telemetry: &mut Telemetry, now: Instant) { + if let Some(payload) = telemetry.sample(now) { + self.emit("telemetry", payload); + } + } +} + +struct Telemetry { + window_started: Instant, + frames: u64, + bytes: u64, + previous_socket_bytes: u64, + peak_bitrate_mbps: f64, + transport: transport::TransportStats, + decode_timings: Option, +} + +impl Telemetry { + fn new(now: Instant) -> Self { + Self { + window_started: now, + frames: 0, + bytes: 0, + previous_socket_bytes: 0, + peak_bitrate_mbps: 0.0, + transport: transport::TransportStats::default(), + decode_timings: None, + } + } + + fn observe(&mut self, feedback: &MediaFeedback) { + match feedback { + MediaFeedback::VideoFrameAccepted { bytes, .. } => { + self.frames = self.frames.saturating_add(1); + self.bytes = self.bytes.saturating_add(u64::from(*bytes)); + } + MediaFeedback::DecodeTimings(timings) => self.decode_timings = Some(*timings), + _ => {} + } + } + + fn sample(&mut self, now: Instant) -> Option { + let elapsed = now.saturating_duration_since(self.window_started); + if elapsed < TELEMETRY_INTERVAL { + return None; + } + let mut payload = media_telemetry( + elapsed, + self.frames, + self.bytes, + self.transport.socket_receive_bytes, + &mut self.previous_socket_bytes, + &mut self.peak_bitrate_mbps, + ); + payload["pingMs"] = json!(self.transport.ping_ms); + payload["jitterMs"] = json!(self.transport.jitter_ms); + payload["packetLossPercent"] = json!(self.transport.packet_loss_percent); + payload["decodeTimings"] = decode_timings_event(self.decode_timings); + payload["decodeTimeMs"] = json!( + self.decode_timings + .and_then(|timings| timings.call) + .map(|stage| stage.p50_us as f64 / 1_000.0) + ); + payload["decoderResidenceMs"] = json!( + self.decode_timings + .and_then(|timings| timings.residence) + .map(|stage| stage.p50_us as f64 / 1_000.0) + ); + self.window_started = now; + self.frames = 0; + self.bytes = 0; + Some(payload) + } +} + +struct Progress { + started: Instant, + presented: bool, + last_decoded: Option, + requested_keyframe: Option, + published: Option, + timings_observed: bool, +} + +impl Progress { + fn new(started: Instant, published: Option) -> Self { + Self { + started, + presented: false, + last_decoded: None, + requested_keyframe: None, + published, + timings_observed: false, + } + } + fn observe_published(&mut self, published: Option, now: Instant) { + if self + .published + .zip(published) + .is_some_and(|(previous, current)| previous != current) + { + self.decoded(now); + } + self.published = published; + } + fn observe_feedback(&mut self, feedback: &MediaFeedback, now: Instant) { + match feedback { + MediaFeedback::PlaybackStarted { .. } => self.presented(now), + MediaFeedback::DecodeTimings(timings) => { + self.timings_observed = true; + if let Some(at) = timings.last_output_at.filter(|at| *at >= self.started) { + self.decoded(at); + } + } + _ => {} + } + } + fn presented(&mut self, at: Instant) { + self.presented = true; + self.decoded(at); + } + fn decoded(&mut self, at: Instant) { + if self.last_decoded.is_none_or(|previous| at > previous) { + self.last_decoded = Some(at); + self.requested_keyframe = None; + } + } + fn poll(&mut self, now: Instant) -> Result { + if !self.presented && now.saturating_duration_since(self.started) >= STARTUP_TIMEOUT { + return Err(Failure { + code: "webrtc-video-startup-timeout", + message: "WebRTC connected but no video frame was presented".to_owned(), + }); + } + if (self.published.is_some() || self.timings_observed) + && let Some(last) = self.last_decoded + && now.saturating_duration_since(last) >= VIDEO_TIMEOUT + { + match self.requested_keyframe { + None => { + self.requested_keyframe = Some(now); + return Ok(true); + } + Some(requested) if now.saturating_duration_since(requested) >= RECOVERY_GRACE => { + return Err(Failure { + code: "webrtc-video-stalled", + message: "WebRTC video stopped decoding after a keyframe request" + .to_owned(), + }); + } + Some(_) => {} + } + } + Ok(false) + } +} + +#[cfg(test)] +mod tests; diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/sdp.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/sdp.rs new file mode 100644 index 000000000..233ff514e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/sdp.rs @@ -0,0 +1,210 @@ +use opennow_streamer_platform::MediaStreamConfig; + +use super::Failure; + +pub(super) fn partial_reliability(offer: &str) -> Result { + let value = attribute(offer, "ri.partialReliableThresholdMs").unwrap_or("30"); + value + .parse::() + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| Failure::signaling("Invalid input partial reliability threshold")) +} + +fn attribute<'a>(sdp: &'a str, name: &str) -> Option<&'a str> { + let prefix = format!("a={name}:"); + sdp.lines() + .find_map(|line| line.strip_prefix(&prefix).map(str::trim)) +} + +fn input_mask(offer: &str, name: &str, fallback: u32) -> Result { + match attribute(offer, name) { + None => Ok(fallback), + Some(value) => { + let parsed = if let Some(hex) = value.strip_prefix("0x") { + u32::from_str_radix(hex, 16).ok() + } else if value == "-1" { + Some(u32::MAX) + } else { + value.parse::().ok() + }; + parsed.ok_or_else(|| Failure::signaling("Invalid input capability mask")) + } + } +} + +pub(super) fn nvst_answer( + offer: &str, + answer: &str, + stream: MediaStreamConfig, +) -> Result { + let ufrag = attribute(answer, "ice-ufrag").filter(|value| !value.is_empty()); + let password = attribute(answer, "ice-pwd").filter(|value| !value.is_empty()); + let fingerprint = answer + .lines() + .find_map(|line| line.strip_prefix("a=fingerprint:sha-256 ")) + .map(str::trim) + .filter(|value| !value.is_empty()); + let (Some(ufrag), Some(password), Some(fingerprint)) = (ufrag, password, fingerprint) else { + return Err(Failure::signaling( + "Local WebRTC answer lacks ICE credentials or SHA-256 fingerprint", + )); + }; + let threshold = partial_reliability(offer)?; + let hid = input_mask(offer, "ri.hidDeviceMask", u32::MAX)?; + let partial_hid = input_mask(offer, "ri.enablePartiallyReliableTransferHid", hid)?; + let partial_gamepad = input_mask(offer, "ri.enablePartiallyReliableTransferGamepad", 15)?; + let maximum = (stream.bitrate_bps / 1000).clamp(1000, 200_000); + let minimum = maximum.min(5000); + let initial = minimum.max(maximum / 4); + let frame_time = (950_000 / stream.fps.max(1)).max(1000); + let mut lines = vec![ + "v=0".to_owned(), + "o=SdpTest test_id_13 14 IN IPv4 127.0.0.1".to_owned(), + "s=-".to_owned(), + "t=0 0".to_owned(), + format!("a=general.icePassword:{password}"), + format!("a=general.iceUserNameFragment:{ufrag}"), + format!("a=general.dtlsFingerprint:{fingerprint}"), + ]; + lines.extend( + include_str!("video_attributes.sdp") + .lines() + .map(str::to_owned), + ); + lines.push(format!( + "a=video.framePacing.pid.minTargetFrameTimeUs:{frame_time}" + )); + if stream.fps > 60 { + lines.extend([ + "a=vqos.resControl.dfc.useClientFpsPerf:0".to_owned(), + "a=bwe.iirFilterFactor:8".to_owned(), + "a=video.encoderFeatureSetting:47".to_owned(), + "a=video.encoderPreset:6".to_owned(), + format!("a=vqos.maxStreamFpsEstimate:{}", stream.fps), + ]); + if let Some((grab, decode)) = match stream.fps { + 90 => Some((9, 11)), + 120 => Some((6, 9)), + 240.. => Some((18, 9)), + _ => None, + } { + lines.push(format!("a=video.fbcDynamicFpsGrabTimeoutMs:{grab}")); + lines.push(format!( + "a=vqos.resControl.cpmRtc.decodeTimeThresholdMs:{decode}" + )); + } + } + if stream.fps >= 240 { + lines.extend( + [ + "a=video.enableNextCaptureMode:1", + "a=video.videoSplitEncodeStripsPerFrame:3", + "a=video.updateSplitEncodeStateDynamically:1", + "a=vqos.rtcPreemptiveIdrSettings.minBurstNackSize:65535", + "a=vqos.rtcPreemptiveIdrSettings.minNackPacketCaptureAgeMs:65535", + ] + .map(str::to_owned), + ); + } + lines.extend([ + format!( + "a=packetPacing.numGroups:{}", + if stream.fps == 120 { 3 } else { 5 } + ), + format!("a=video.clientViewportWd:{}", stream.width), + format!("a=video.clientViewportHt:{}", stream.height), + format!("a=video.maxFPS:{}", stream.fps), + format!("a=video.bitDepth:{}", stream.color_quality.bit_depth()), + format!("a=video.initialBitrateKbps:{initial}"), + format!("a=video.initialPeakBitrateKbps:{initial}"), + format!("a=vqos.bw.maximumBitrateKbps:{maximum}"), + format!("a=vqos.bw.minimumBitrateKbps:{minimum}"), + format!("a=vqos.bw.peakBitrateKbps:{maximum}"), + format!("a=vqos.bw.serverPeakBitrateKbps:{maximum}"), + format!("a=vqos.grc.maximumBitrateKbps:{maximum}"), + "m=audio 0 RTP/AVP".to_owned(), + "a=msid:audio".to_owned(), + "m=application 0 RTP/AVP".to_owned(), + "a=msid:input_1".to_owned(), + format!("a=ri.partialReliableThresholdMs:{threshold}"), + format!("a=ri.hidDeviceMask:{hid}"), + format!("a=ri.enablePartiallyReliableTransferGamepad:{partial_gamepad}"), + format!("a=ri.enablePartiallyReliableTransferHid:{partial_hid}"), + String::new(), + ]); + Ok(lines.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn answer_uses_local_credentials_and_echoes_input_masks() { + let answer = "v=0\r\na=ice-ufrag:local\r\na=ice-pwd:local-password\r\na=fingerprint:sha-256 AA:BB\r\n"; + let offer = "a=ice-ufrag:remote\na=ri.partialReliableThresholdMs:44\na=ri.hidDeviceMask:0xffffffff\na=ri.enablePartiallyReliableTransferHid:128\na=ri.enablePartiallyReliableTransferGamepad:3\n"; + let sdp = nvst_answer(offer, answer, MediaStreamConfig::default()).unwrap(); + for expected in [ + "a=general.iceUserNameFragment:local", + "a=general.icePassword:local-password", + "a=general.dtlsFingerprint:AA:BB", + "a=ri.partialReliableThresholdMs:44", + "a=ri.hidDeviceMask:4294967295", + "a=ri.enablePartiallyReliableTransferHid:128", + "a=ri.enablePartiallyReliableTransferGamepad:3", + "a=video.bitDepth:8", + "a=video.dynamicRangeMode:0", + ] { + assert!( + sdp.lines().any(|line| line == expected), + "missing {expected}" + ); + } + assert!(!sdp.contains("remote")); + assert!(!sdp.contains("m=mic")); + } + + #[test] + fn hevc_main10_sdr_answer_preserves_1920x1200_at_60_fps() { + use opennow_streamer_platform::{MediaColorQuality, MediaVideoCodec}; + let answer = "v=0\r\na=ice-ufrag:local\r\na=ice-pwd:local-password\r\na=fingerprint:sha-256 AA:BB\r\n"; + let stream = MediaStreamConfig { + codec: MediaVideoCodec::H265, + color_quality: MediaColorQuality::TenBit420, + width: 1920, + height: 1200, + fps: 60, + hdr: false, + ..MediaStreamConfig::default() + }; + let sdp = nvst_answer("", answer, stream).unwrap(); + for expected in [ + "a=video.clientViewportWd:1920", + "a=video.clientViewportHt:1200", + "a=video.maxFPS:60", + "a=video.bitDepth:10", + "a=video.dynamicRangeMode:0", + "a=video.framePacing.pid.minTargetFrameTimeUs:15833", + ] { + assert!( + sdp.lines().any(|line| line == expected), + "missing {expected}" + ); + } + assert_eq!( + sdp.lines() + .filter(|line| line.starts_with("a=video.bitDepth:")) + .count(), + 1 + ); + } + + #[test] + fn missing_credentials_and_invalid_input_limits_are_rejected() { + assert!(nvst_answer("", "v=0", MediaStreamConfig::default()).is_err()); + assert!(partial_reliability("a=ri.partialReliableThresholdMs:65536").is_err()); + assert!(partial_reliability("a=ri.partialReliableThresholdMs:0").is_err()); + assert_eq!(partial_reliability("").unwrap(), 30); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling.rs new file mode 100644 index 000000000..6deda56cb --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling.rs @@ -0,0 +1,470 @@ +use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use opennow_streamer_protocol::Session; +use opennow_streamer_transport::webrtc::IceCandidate; +use serde_json::{Value, json}; +use tungstenite::client::IntoClientRequest; +use tungstenite::http::{HeaderValue, Uri}; +use tungstenite::protocol::WebSocketConfig; +use tungstenite::stream::MaybeTlsStream; +use tungstenite::{Message, WebSocket}; + +use super::{Failure, MAX_SDP_BYTES}; + +pub(super) const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const WRITE_TIMEOUT: Duration = Duration::from_millis(250); +const MAX_SIGNALING_BYTES: usize = 512 * 1024; + +struct ResolveRequest { + host: String, + port: u16, + result: mpsc::SyncSender>>, +} + +pub(super) fn resolve( + host: String, + port: u16, + cancelled: &AtomicBool, +) -> Result, Failure> { + if let Ok(ip) = host.parse::() { + return Ok(vec![SocketAddr::new(ip, port)]); + } + static RESOLVER: OnceLock>> = OnceLock::new(); + let resolver = RESOLVER + .get_or_init(|| { + let (sender, receiver) = mpsc::sync_channel::(1); + thread::Builder::new() + .name("opennow-webrtc-dns".to_owned()) + .spawn(move || { + while let Ok(request) = receiver.recv() { + let result = (request.host.as_str(), request.port) + .to_socket_addrs() + .map(|addresses| addresses.take(8).collect::>()); + let _ = request.result.try_send(result); + } + }) + .ok() + .map(|_| sender) + }) + .as_ref() + .ok_or_else(|| Failure::signaling("Could not start bounded DNS resolver"))?; + let (sender, receiver) = mpsc::sync_channel(1); + resolver + .try_send(ResolveRequest { + host, + port, + result: sender, + }) + .map_err(|_| Failure::signaling("WebRTC DNS resolver is busy"))?; + let deadline = Instant::now() + CONNECT_TIMEOUT; + loop { + if cancelled.load(Ordering::Acquire) || Instant::now() >= deadline { + return Err(Failure::signaling("WebRTC DNS cancelled or timed out")); + } + match receiver.recv_timeout(Duration::from_millis(25)) { + Ok(Ok(addresses)) if !addresses.is_empty() => return Ok(addresses), + Ok(_) | Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(Failure::signaling("WebRTC DNS failed")); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +pub(super) enum Incoming { + Offer(String), + Candidate(IceCandidate), + Closed(String), +} + +/// Names the transport-level reason without addresses, URLs or server text. +fn error_detail(error: &tungstenite::Error) -> String { + match error { + tungstenite::Error::ConnectionClosed => "connection-closed".to_owned(), + tungstenite::Error::AlreadyClosed => "already-closed".to_owned(), + tungstenite::Error::Io(error) => match error.raw_os_error() { + Some(code) => format!("io kind={:?} os_code={code}", error.kind()), + None => format!("io kind={:?} os_code=none", error.kind()), + }, + tungstenite::Error::Tls(_) => "tls".to_owned(), + tungstenite::Error::Capacity(_) => "capacity".to_owned(), + tungstenite::Error::Protocol(error) => { + let variant = format!("{error:?}"); + format!( + "protocol={}", + variant.split(['(', ' ', '{']).next().unwrap_or_default() + ) + } + tungstenite::Error::WriteBufferFull(_) => "write-buffer-full".to_owned(), + tungstenite::Error::Utf8(_) => "utf8".to_owned(), + tungstenite::Error::AttackAttempt => "attack-attempt".to_owned(), + tungstenite::Error::Url(_) => "url".to_owned(), + tungstenite::Error::Http(response) => format!("http status={}", response.status().as_u16()), + tungstenite::Error::HttpFormat(_) => "http-format".to_owned(), + } +} + +fn failure(context: &str, error: &tungstenite::Error) -> Failure { + let detail = error_detail(error); + opennow_streamer_protocol::log::log_line( + "WARN", + "webrtc-signaling", + &format!("{context} {detail}"), + ); + Failure::signaling(format!("{context} ({detail})")) +} + +pub(super) struct Protocol { + peer_name: String, + peer_id: u64, + remote_id: u64, + ack: u64, +} + +impl Protocol { + pub(super) fn new(peer_name: String) -> Self { + Self { + peer_name, + peer_id: 0, + remote_id: 1, + ack: 0, + } + } + + pub(super) fn peer_info(&mut self, width: u32, height: u32) -> Value { + self.ack += 1; + json!({"ackid":self.ack,"peer_info":{ + "browser":"Chrome","browserVersion":"131","connected":true, + "id":self.peer_id,"name":self.peer_name,"peerRole":0, + "resolution":format!("{width}x{height}"),"version":2 + }}) + } + + pub(super) fn peer_message(&mut self, payload: Value) -> Value { + self.ack += 1; + json!({"ackid":self.ack,"peer_msg":{ + "from":self.peer_id,"to":self.remote_id,"msg":payload.to_string() + }}) + } + + pub(super) fn receive( + &mut self, + text: &str, + ) -> Result<(Vec, Option), Failure> { + if text.len() > MAX_SIGNALING_BYTES { + return Err(Failure::signaling("Signaling message exceeds size limit")); + } + let packet: Value = + serde_json::from_str(text).map_err(|_| Failure::signaling("Invalid signaling JSON"))?; + if packet["peer_info"]["name"].as_str() == Some(&self.peer_name) + && let Some(id) = packet["peer_info"]["id"].as_u64() + { + self.peer_id = id; + } + let mut replies = Vec::new(); + if let Some(ack) = packet["ackid"].as_u64() + && packet["peer_info"]["id"].as_u64() != Some(self.peer_id) + { + replies.push(json!({"ack":ack})); + } + if packet["hb"].as_u64().is_some_and(|hb| hb != 0) { + replies.push(json!({"hb":1})); + return Ok((replies, None)); + } + if packet["error"].as_str() == Some("peerRemoved") { + return Ok((replies, Some(Incoming::Closed("peer-removed".to_owned())))); + } + if packet.get("error").is_some_and(|value| !value.is_null()) { + return Err(Failure::signaling("Server rejected signaling request")); + } + let Some(message) = packet["peer_msg"]["msg"].as_str() else { + return Ok((replies, None)); + }; + if let Some(id) = packet["peer_msg"]["from"].as_u64() { + self.remote_id = id; + } + if message.trim() == "BYE" { + return Ok((replies, Some(Incoming::Closed("bye".to_owned())))); + } + let payload: Value = serde_json::from_str(message) + .map_err(|_| Failure::signaling("Invalid nested peer message"))?; + if payload["type"].as_str() == Some("offer") { + let sdp = payload["sdp"] + .as_str() + .filter(|sdp| !sdp.is_empty() && sdp.len() <= MAX_SDP_BYTES) + .ok_or_else(|| Failure::signaling("Offer SDP is missing or exceeds size limit"))?; + return Ok((replies, Some(Incoming::Offer(sdp.to_owned())))); + } + if payload["candidate"].is_string() { + let mut candidate: IceCandidate = serde_json::from_value(payload) + .map_err(|_| Failure::signaling("Invalid remote ICE candidate"))?; + if candidate.candidate.len() > 4096 { + return Err(Failure::signaling( + "Remote ICE candidate exceeds size limit", + )); + } + if candidate.sdp_mid.is_none() && candidate.sdp_m_line_index.is_none() { + candidate.sdp_m_line_index = Some(0); + } + return Ok((replies, Some(Incoming::Candidate(candidate)))); + } + Ok((replies, None)) + } +} + +fn query_component(value: &str) -> String { + let mut encoded = String::new(); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + use std::fmt::Write; + let _ = write!(encoded, "%{byte:02X}"); + } + } + encoded +} + +pub(super) fn sign_in_url(session: &Session, peer: &str) -> Result { + let base = session + .extra + .get("signalingUrl") + .and_then(Value::as_str) + .filter(|url| !url.trim().is_empty()) + .ok_or_else(|| Failure::signaling("WebRTC requires an explicit signalingUrl"))?; + let uri: Uri = base + .parse() + .map_err(|_| Failure::signaling("Invalid signalingUrl"))?; + let authority = uri + .authority() + .ok_or_else(|| Failure::signaling("Signaling URL has no host"))?; + let test_plaintext = + cfg!(test) && uri.scheme_str() == Some("ws") && uri.host() == Some("127.0.0.1"); + if (uri.scheme_str() != Some("wss") && !test_plaintext) + || authority.as_str().contains('@') + || base.contains('#') + { + return Err(Failure::signaling( + "Signaling requires a credential-free WSS endpoint", + )); + } + let path = uri.path().trim_end_matches('/'); + let path = if path.ends_with("/sign_in") { + path.to_owned() + } else { + format!("{path}/sign_in") + }; + let reconnect = session + .extra + .get("signalingReconnect") + .and_then(Value::as_bool) + .unwrap_or(false); + let scheme = if test_plaintext { "ws" } else { "wss" }; + Ok(format!( + "{scheme}://{authority}{path}?peer_id={}&version={}&peer_role=1&pairing_id={}{}", + query_component(peer), + if reconnect { "3.0" } else { "2" }, + query_component(&session.session_id), + if reconnect { "&reconnect=1" } else { "" } + )) +} + +pub(super) struct Signaling> { + socket: WebSocket, + pub(super) protocol: Protocol, + heartbeat_at: Instant, + write_pending_since: Option, +} + +impl Signaling { + pub(super) fn connect(session: &Session, cancelled: &AtomicBool) -> Result { + let mut random = [0_u8; 12]; + getrandom::fill(&mut random) + .map_err(|_| Failure::signaling("Could not generate signaling identity"))?; + let peer = format!( + "peer-{}", + random + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + let url = sign_in_url(session, &peer)?; + let mut request = url + .into_client_request() + .map_err(|_| Failure::signaling("Invalid WSS request"))?; + request.headers_mut().insert( + "Origin", + HeaderValue::from_static("https://play.geforcenow.com"), + ); + request.headers_mut().insert( + "User-Agent", + HeaderValue::from_static( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/131.0.0.0 Safari/537.36", + ), + ); + request.headers_mut().insert( + "Sec-WebSocket-Protocol", + HeaderValue::from_str(&format!("x-nv-sessionid.{}", session.session_id)) + .map_err(|_| Failure::signaling("Invalid signaling session ID"))?, + ); + let host = request + .uri() + .host() + .ok_or_else(|| Failure::signaling("Missing signaling host"))? + .to_owned(); + let port = request.uri().port_u16().unwrap_or(443); + let addresses = resolve(host, port, cancelled)?; + let deadline = Instant::now() + CONNECT_TIMEOUT; + let mut connected = None; + for address in addresses { + if cancelled.load(Ordering::Acquire) || Instant::now() >= deadline { + break; + } + let budget = deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(500)); + if let Ok(stream) = TcpStream::connect_timeout(&address, budget) { + connected = Some(stream); + break; + } + } + let stream = connected + .ok_or_else(|| Failure::signaling("Signaling TCP connection failed or timed out"))?; + stream + .set_nonblocking(true) + .map_err(|_| Failure::signaling("Could not bound signaling handshake"))?; + let config = WebSocketConfig::default() + .max_write_buffer_size(MAX_SIGNALING_BYTES * 2) + .max_message_size(Some(MAX_SIGNALING_BYTES)) + .max_frame_size(Some(MAX_SIGNALING_BYTES)); + let mut handshake = + tungstenite::client_tls_with_config(request, stream, Some(config), None); + let mut socket = loop { + if cancelled.load(Ordering::Acquire) || Instant::now() >= deadline { + return Err(Failure::signaling( + "Signaling handshake cancelled or timed out", + )); + } + match handshake { + Ok((socket, _)) => break socket, + Err(tungstenite::HandshakeError::Interrupted(pending)) => { + thread::sleep(Duration::from_millis(5)); + handshake = pending.handshake(); + } + Err(tungstenite::HandshakeError::Failure(error)) => { + return Err(failure("Signaling TLS/WebSocket handshake failed", &error)); + } + } + }; + let stream = match socket.get_mut() { + MaybeTlsStream::Plain(stream) => stream, + MaybeTlsStream::Rustls(stream) => stream.get_mut(), + _ => return Err(Failure::signaling("Unsupported signaling TLS backend")), + }; + stream + .set_nonblocking(true) + .map_err(|_| Failure::signaling("Could not configure signaling socket"))?; + Ok(Self { + socket, + protocol: Protocol::new(peer), + heartbeat_at: Instant::now(), + write_pending_since: None, + }) + } +} + +impl Signaling { + fn write_result(&mut self, result: Result<(), tungstenite::Error>) -> Result<(), Failure> { + match result { + Ok(()) => { + self.write_pending_since = None; + Ok(()) + } + Err(tungstenite::Error::Io(error)) if error.kind() == ErrorKind::WouldBlock => { + let since = self.write_pending_since.get_or_insert_with(Instant::now); + if since.elapsed() >= WRITE_TIMEOUT { + return Err(Failure::signaling("Signaling pending write timed out")); + } + Ok(()) + } + Err(tungstenite::Error::WriteBufferFull(_)) => Err(Failure::signaling( + "Signaling write buffer exceeds size limit", + )), + Err(error) => Err(failure("Signaling write failed", &error)), + } + } + + fn flush(&mut self) -> Result<(), Failure> { + if self + .write_pending_since + .is_some_and(|since| since.elapsed() >= WRITE_TIMEOUT) + { + return Err(Failure::signaling("Signaling pending write timed out")); + } + let result = self.socket.flush(); + self.write_result(result) + } + + pub(super) fn send(&mut self, payload: Value) -> Result<(), Failure> { + let text = payload.to_string(); + if text.len() > MAX_SIGNALING_BYTES { + return Err(Failure::signaling( + "Outgoing signaling message exceeds size limit", + )); + } + let result = self.socket.send(Message::Text(text.into())); + self.write_result(result) + } + + pub(super) fn send_peer(&mut self, payload: Value) -> Result<(), Failure> { + let packet = self.protocol.peer_message(payload); + self.send(packet) + } + + pub(super) fn poll(&mut self) -> Result, Failure> { + self.flush()?; + if self.heartbeat_at.elapsed() >= HEARTBEAT_INTERVAL { + self.send(json!({"hb":1}))?; + self.heartbeat_at = Instant::now(); + } + let packet = match self.socket.read() { + Ok(Message::Text(text)) => text.to_string(), + Ok(Message::Binary(bytes)) => String::from_utf8(bytes.to_vec()) + .map_err(|_| Failure::signaling("Invalid signaling UTF-8"))?, + Ok(Message::Close(frame)) => { + let detail = frame.map_or_else( + || "close_code=none".to_owned(), + |frame| format!("close_code={}", u16::from(frame.code)), + ); + opennow_streamer_protocol::log::log_line( + "WARN", + "webrtc-signaling", + &format!("peer-close {detail}"), + ); + return Ok(Some(Incoming::Closed(detail))); + } + Ok(_) => return Ok(None), + Err(tungstenite::Error::Io(error)) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => + { + return Ok(None); + } + Err(error) => return Err(failure("Signaling connection closed", &error)), + }; + let (replies, incoming) = self.protocol.receive(&packet)?; + for reply in replies { + self.send(reply)?; + } + Ok(incoming) + } +} + +#[cfg(test)] +mod tests; diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling/tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling/tests.rs new file mode 100644 index 000000000..7fe6b5dac --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/signaling/tests.rs @@ -0,0 +1,284 @@ +use super::*; + +fn session(url: &str) -> Session { + serde_json::from_value( + json!({"sessionId":"seat/one","serverIp":"192.0.2.1","signalingUrl":url}), + ) + .unwrap() +} + +#[test] +fn sign_in_normalizes_path_query_and_reconnect() { + let mut session = session("wss://example.test/nvst/?old=ignored"); + assert_eq!( + sign_in_url(&session, "peer one").unwrap(), + "wss://example.test/nvst/sign_in?peer_id=peer%20one&version=2&peer_role=1&pairing_id=seat%2Fone" + ); + session.extra.insert( + "signalingUrl".to_owned(), + json!("wss://example.test/nvst/sign_in"), + ); + session + .extra + .insert("signalingReconnect".to_owned(), json!(true)); + let url = sign_in_url(&session, "peer").unwrap(); + assert!(url.contains("version=3.0")); + assert!(url.ends_with("&reconnect=1")); + assert!(!url.contains("sign_in/sign_in")); +} + +#[test] +fn signaling_rejects_insecure_and_credential_urls() { + for url in [ + "http://example.test/nvst/", + "ws://example.test/nvst/", + "wss://user:password@example.test/nvst/", + "wss://example.test/nvst/#fragment", + ] { + assert!(sign_in_url(&session(url), "peer").is_err()); + } +} + +#[test] +fn peer_info_ack_heartbeat_and_nested_answer_match_wire_contract() { + let mut protocol = Protocol::new("peer-local".to_owned()); + let info = protocol.peer_info(1920, 1080); + assert_eq!(info["peer_info"]["resolution"], "1920x1080"); + assert_eq!(info["peer_info"]["peerRole"], 0); + let (replies, _) = protocol + .receive(r#"{"ackid":1,"peer_info":{"id":7,"name":"peer-local"}}"#) + .unwrap(); + assert!(replies.is_empty()); + let (replies, _) = protocol.receive(r#"{"ackid":2,"hb":1}"#).unwrap(); + assert_eq!(replies, vec![json!({"ack":2}), json!({"hb":1})]); + let offer = json!({"peer_msg":{"from":9,"to":7,"msg":json!({"type":"offer","sdp":"v=0\r\n"}).to_string()},"ackid":3}); + let (replies, incoming) = protocol.receive(&offer.to_string()).unwrap(); + assert_eq!(replies, vec![json!({"ack":3})]); + assert!(matches!(incoming, Some(Incoming::Offer(sdp)) if sdp == "v=0\r\n")); + let answer = + protocol.peer_message(json!({"type":"answer","sdp":"local","nvstSdp":"attributes"})); + assert_eq!(answer["peer_msg"]["from"], 7); + assert_eq!(answer["peer_msg"]["to"], 9); + let nested: Value = serde_json::from_str(answer["peer_msg"]["msg"].as_str().unwrap()).unwrap(); + assert_eq!(nested["nvstSdp"], "attributes"); +} + +#[test] +fn candidate_defaults_and_terminal_messages_are_parsed() { + let mut protocol = Protocol::new("peer".to_owned()); + let candidate = json!({"peer_msg":{"from":1,"msg":json!({"candidate":"candidate:1 1 udp 1 127.0.0.1 9999 typ host"}).to_string()}}); + let (_, incoming) = protocol.receive(&candidate.to_string()).unwrap(); + assert!( + matches!(incoming, Some(Incoming::Candidate(candidate)) if candidate.sdp_m_line_index == Some(0)) + ); + for (packet, detail) in [ + (json!({"error":"peerRemoved"}), "peer-removed"), + (json!({"peer_msg":{"msg":"BYE"}}), "bye"), + ] { + assert!(matches!( + protocol.receive(&packet.to_string()).unwrap().1, + Some(Incoming::Closed(closed)) if closed == detail + )); + } + assert!(protocol.receive("not JSON").is_err()); + assert!( + protocol + .receive(&"x".repeat(MAX_SIGNALING_BYTES + 1)) + .is_err() + ); +} + +#[derive(Default)] +struct NonblockingStream { + incoming: std::io::Cursor>, + written: Vec, + write_budget: usize, +} + +impl Read for NonblockingStream { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + let count = self.incoming.read(output)?; + if count == 0 { + return Err(std::io::Error::from(ErrorKind::WouldBlock)); + } + Ok(count) + } +} + +impl Write for NonblockingStream { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.write_budget == 0 { + return Err(std::io::Error::from(ErrorKind::WouldBlock)); + } + let count = bytes.len().min(self.write_budget); + self.written.extend_from_slice(&bytes[..count]); + self.write_budget -= count; + Ok(count) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn nonblocking_signaling( + write_buffer_size: usize, + max_write_buffer_size: usize, +) -> Signaling { + Signaling { + socket: WebSocket::from_raw_socket( + NonblockingStream::default(), + tungstenite::protocol::Role::Client, + Some( + WebSocketConfig::default() + .write_buffer_size(write_buffer_size) + .max_write_buffer_size(max_write_buffer_size), + ), + ), + protocol: Protocol::new("fixture".to_owned()), + heartbeat_at: Instant::now(), + write_pending_since: None, + } +} + +#[test] +fn partial_writes_are_flushed_once_in_order_without_resending_messages() { + for write_buffer_size in [0, 128] { + let mut signaling = nonblocking_signaling(write_buffer_size, 1024); + signaling.socket.get_mut().write_budget = 3; + signaling.send(json!({"order":1})).unwrap(); + let pending_since = signaling.write_pending_since.unwrap(); + signaling.send(json!({"order":2})).unwrap(); + assert_eq!(signaling.write_pending_since, Some(pending_since)); + for _ in 0..8 { + assert!(signaling.poll().unwrap().is_none()); + } + assert_eq!(signaling.socket.get_ref().written.len(), 3); + signaling.socket.get_mut().write_budget = usize::MAX; + assert!(signaling.poll().unwrap().is_none()); + assert!(signaling.write_pending_since.is_none()); + let bytes = signaling.socket.get_ref().written.clone(); + let mut server = WebSocket::from_raw_socket( + std::io::Cursor::new(bytes), + tungstenite::protocol::Role::Server, + None, + ); + for order in [1, 2] { + let message: Value = + serde_json::from_str(&server.read().unwrap().into_text().unwrap()).unwrap(); + assert_eq!(message, json!({"order":order})); + } + assert!(server.read().is_err()); + } +} + +#[test] +fn incomplete_incoming_message_does_not_prevent_pending_write_flush() { + let mut signaling = nonblocking_signaling(0, 1024); + signaling.socket.get_mut().incoming = std::io::Cursor::new(vec![0x01, 0x01, b'{']); + assert!(signaling.poll().unwrap().is_none()); + signaling.send(json!({"hb":1})).unwrap(); + assert!(signaling.write_pending_since.is_some()); + signaling.socket.get_mut().write_budget = usize::MAX; + assert!(signaling.poll().unwrap().is_none()); + assert!(signaling.write_pending_since.is_none()); + assert!(!signaling.socket.get_ref().written.is_empty()); +} + +#[test] +fn pending_writes_have_a_fixed_deadline_and_a_bounded_buffer() { + let mut signaling = nonblocking_signaling(0, 128); + signaling.send(json!("a".repeat(80))).unwrap(); + assert!( + signaling + .send(json!("b".repeat(80))) + .unwrap_err() + .message + .contains("buffer exceeds") + ); + signaling.write_pending_since = Some(Instant::now() - WRITE_TIMEOUT); + assert!( + signaling + .poll() + .err() + .unwrap() + .message + .contains("timed out") + ); + let mut signaling = nonblocking_signaling(0, 128); + assert!( + signaling + .send(json!("a".repeat(MAX_SIGNALING_BYTES))) + .unwrap_err() + .message + .contains("message exceeds") + ); + assert!(signaling.socket.get_ref().written.is_empty()); +} + +struct FailingStream(Option); + +impl Read for FailingStream { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + self.0.take().map_or(Ok(0), Err) + } +} + +impl Write for FailingStream { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::new( + ErrorKind::ConnectionAborted, + "203.0.113.9:443 private", + )) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn failing_signaling(error: Option) -> Signaling { + Signaling { + socket: WebSocket::from_raw_socket( + FailingStream(error), + tungstenite::protocol::Role::Client, + None, + ), + protocol: Protocol::new("fixture".to_owned()), + heartbeat_at: Instant::now(), + write_pending_since: None, + } +} + +#[test] +fn signaling_failures_name_the_transport_cause_without_endpoints() { + let error = std::io::Error::from_raw_os_error(10054); + let expected = format!( + "Signaling connection closed (io kind={:?} os_code=10054)", + error.kind() + ); + let failure = failing_signaling(Some(error)).poll().err().unwrap(); + assert_eq!(failure.code, "webrtc-signaling-failed"); + assert_eq!(failure.message, expected); + + assert_eq!( + failing_signaling(None).poll().err().unwrap().message, + "Signaling connection closed (protocol=ResetWithoutClosingHandshake)" + ); + + let failure = failing_signaling(None).send(json!({"hb":1})).unwrap_err(); + assert_eq!( + failure.message, + "Signaling write failed (io kind=ConnectionAborted os_code=none)" + ); + assert!(!failure.message.contains("203.0.113.9") && !failure.message.contains("private")); + + let mut signaling = nonblocking_signaling(0, 1024); + let mut close = vec![0x88, 9, 0x03, 0xf3]; + close.extend_from_slice(b"private"); + signaling.socket.get_mut().incoming = std::io::Cursor::new(close); + assert!(matches!( + signaling.poll().unwrap(), + Some(Incoming::Closed(detail)) if detail == "close_code=1011" + )); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/tests.rs new file mode 100644 index 000000000..799145126 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/tests.rs @@ -0,0 +1,1288 @@ +use super::*; + +#[test] +fn accepted_frames_emit_webrtc_telemetry_with_optional_transport_stats() { + let now = Instant::now(); + let (events, receiver) = mpsc::channel(); + let worker = worker(EventSender::unbounded(events)); + let mut telemetry = Telemetry::new(now); + for index in 0..60 { + telemetry.observe(&MediaFeedback::VideoFrameAccepted { + frame_index: Some(index), + timestamp: u64::from(index) * 1_500, + bytes: 100_000, + keyframe: index == 0, + }); + } + worker.flush_telemetry(&mut telemetry, now + Duration::from_millis(999)); + assert!(receiver.try_recv().is_err()); + worker.flush_telemetry(&mut telemetry, now + Duration::from_secs(1)); + let first = receiver.try_recv().unwrap(); + assert_eq!(first["type"], "telemetry"); + assert_eq!(first["transport"], "webrtc"); + assert_eq!(first["startId"], "fixture"); + assert_eq!(first["framesPerSecond"], 60.0); + assert_eq!(first["bitrateMbps"], 48.0); + assert_eq!(first["peakBitrateMbps"], 48.0); + for key in [ + "receiveBitrateMbps", + "pingMs", + "jitterMs", + "packetLossPercent", + "decodeTimings", + ] { + assert!(first[key].is_null(), "{key} must remain unavailable"); + } + telemetry.transport = transport::TransportStats { + socket_receive_bytes: Some(1_000_000), + ping_ms: Some(12.0), + jitter_ms: Some(2.5), + packet_loss_percent: Some(0.5), + }; + telemetry.observe(&MediaFeedback::DecodeTimings(DecodeTimingsReport { + epoch: 7, + outputs_total: 60, + call: Some(DecodeStageTimings { + p50_us: 1_500, + p95_us: 1_500, + max_us: 1_500, + }), + residence: Some(DecodeStageTimings { + p50_us: 2_000, + p95_us: 2_000, + max_us: 2_000, + }), + call_window_samples: 60, + residence_window_samples: 60, + submissions_total: 60, + output_calls_total: 60, + last_submission_at: Some(now), + last_output_at: Some(now), + in_flight: 0, + oldest_in_flight_at: None, + epoch_started_at: Some(now), + unmatched_outputs: 0, + unmatched_submissions: 0, + })); + assert!(receiver.try_recv().is_err()); + worker.flush_telemetry(&mut telemetry, now + Duration::from_secs(2)); + let second = receiver.try_recv().unwrap(); + assert_eq!(second["framesPerSecond"], 0.0); + assert_eq!(second["bitrateMbps"], 0.0); + assert_eq!(second["peakBitrateMbps"], 48.0); + assert_eq!(second["receiveBitrateMbps"], 8.0); + assert_eq!(second["pingMs"], 12.0); + assert_eq!(second["jitterMs"], 2.5); + assert_eq!(second["packetLossPercent"], 0.5); + assert_eq!(second["decodeTimings"]["epoch"], 7); + assert_eq!(second["decodeTimings"]["outputsTotal"], 60); + assert_eq!(second["decodeTimeMs"], 1.5); + assert_eq!(second["decoderResidenceMs"], 2.0); + telemetry.transport.socket_receive_bytes = Some(1_500_000); + worker.flush_telemetry(&mut telemetry, now + Duration::from_secs(3)); + assert_eq!(receiver.try_recv().unwrap()["receiveBitrateMbps"], 4.0); + telemetry.transport = transport::TransportStats::default(); + worker.flush_telemetry(&mut telemetry, now + Duration::from_secs(4)); + let missing = receiver.try_recv().unwrap(); + for key in [ + "receiveBitrateMbps", + "pingMs", + "jitterMs", + "packetLossPercent", + ] { + assert!(missing[key].is_null()); + } + assert!(receiver.try_recv().is_err()); +} + +#[test] +fn video_progress_requires_presentation_and_does_not_accept_audio_as_progress() { + let now = Instant::now(); + let mut progress = Progress::new(now, None); + assert!( + !progress + .poll(now + STARTUP_TIMEOUT - Duration::from_millis(1)) + .unwrap() + ); + assert_eq!( + progress.poll(now + STARTUP_TIMEOUT).unwrap_err().code, + "webrtc-video-startup-timeout" + ); +} + +#[test] +fn stalled_decoder_gets_one_keyframe_then_terminal_error() { + let now = Instant::now(); + let mut progress = Progress::new(now, Some(0)); + progress.presented(now); + assert!(progress.poll(now + VIDEO_TIMEOUT).unwrap()); + assert!( + !progress + .poll(now + VIDEO_TIMEOUT + Duration::from_millis(1)) + .unwrap() + ); + assert_eq!( + progress + .poll(now + VIDEO_TIMEOUT + RECOVERY_GRACE) + .unwrap_err() + .code, + "webrtc-video-stalled" + ); + progress.decoded(now + VIDEO_TIMEOUT + RECOVERY_GRACE); + assert!(!progress.poll(now + VIDEO_TIMEOUT + RECOVERY_GRACE).unwrap()); +} + +#[test] +fn published_decoded_output_remains_healthy_for_ninety_seconds() { + let now = Instant::now(); + let mut progress = Progress::new(now, Some(1000)); + progress.observe_feedback(&MediaFeedback::PlaybackStarted { backend: "fixture" }, now); + for second in 1..=90 { + let at = now + Duration::from_secs(second); + progress.observe_published(Some(1000 + second * 60), at); + assert!(!progress.poll(at).unwrap()); + } + assert!(progress.poll(now + Duration::from_secs(98)).unwrap()); + assert_eq!( + progress + .poll(now + Duration::from_secs(102)) + .unwrap_err() + .code, + "webrtc-video-stalled" + ); +} + +#[test] +fn published_output_does_not_replace_first_presentation_or_count_prior_sessions() { + let now = Instant::now(); + let mut progress = Progress::new(now, Some(1000)); + progress.observe_published(Some(1000), now + Duration::from_secs(1)); + assert!(progress.last_decoded.is_none()); + for second in 2..=19 { + progress.observe_published(Some(1000 + second), now + Duration::from_secs(second)); + } + assert!(!progress.presented); + assert_eq!( + progress.poll(now + STARTUP_TIMEOUT).unwrap_err().code, + "webrtc-video-startup-timeout" + ); +} + +#[test] +fn accepted_compressed_frames_do_not_refresh_observable_decoded_health() { + let now = Instant::now(); + let mut progress = Progress::new(now, Some(100)); + progress.observe_feedback(&MediaFeedback::PlaybackStarted { backend: "fixture" }, now); + let mut requests = 0; + for second in 1..=12 { + let at = now + Duration::from_secs(second); + progress.observe_feedback( + &MediaFeedback::VideoFrameAccepted { + frame_index: None, + timestamp: second * 90_000, + bytes: 1000, + keyframe: second == 1, + }, + at, + ); + progress.observe_published(Some(100), at); + let result = progress.poll(at); + if second == 12 { + assert_eq!(result.unwrap_err().code, "webrtc-video-stalled"); + } else if result.unwrap() { + requests += 1; + assert_eq!(second, 8); + } + } + assert_eq!(requests, 1); +} + +#[test] +fn missing_output_observer_does_not_invent_a_decode_stall_after_presentation() { + let now = Instant::now(); + let mut progress = Progress::new(now, None); + progress.observe_feedback(&MediaFeedback::PlaybackStarted { backend: "fixture" }, now); + for second in 1..=90 { + let at = now + Duration::from_secs(second); + progress.observe_published(None, at); + assert!(!progress.poll(at).unwrap()); + } +} + +#[test] +fn explicit_decode_timings_drive_health_without_a_graphics_publisher() { + let now = Instant::now(); + let mut progress = Progress::new(now, None); + progress.observe_feedback(&MediaFeedback::PlaybackStarted { backend: "fixture" }, now); + for second in 1..=90 { + let at = now + Duration::from_secs(second); + let timings = DecodeTimingsReport { + call: None, + residence: None, + call_window_samples: 0, + residence_window_samples: 0, + submissions_total: second * 60, + outputs_total: second * 60, + output_calls_total: second * 60, + last_submission_at: Some(at), + last_output_at: Some(at), + in_flight: 0, + oldest_in_flight_at: None, + epoch: 1, + epoch_started_at: Some(now), + unmatched_outputs: 0, + unmatched_submissions: 0, + }; + progress.observe_feedback(&MediaFeedback::DecodeTimings(timings), at); + progress.observe_published(None, at); + assert!(!progress.poll(at).unwrap()); + } + assert!(progress.poll(now + Duration::from_secs(98)).unwrap()); + assert_eq!( + progress + .poll(now + Duration::from_secs(102)) + .unwrap_err() + .code, + "webrtc-video-stalled" + ); +} + +fn worker(output: EventSender) -> Worker { + let context: SessionContext = serde_json::from_value(json!({ + "session":{"sessionId":"fixture","serverIp":"127.0.0.1"},"settings":{},"shortcuts":{} + })) + .unwrap(); + let (_, commands) = mpsc::sync_channel(4); + Worker { + lifecycle: Arc::new(Mutex::new(Lifecycle { + state: State::Connected, + context: Some(context.clone()), + generation: 1, + })), + context, + stream: MediaStreamConfig::default(), + video_codec: transport::NegotiatedVideoCodec::H264, + start_id: "fixture".to_owned(), + generation: 1, + cancelled: Arc::new(AtomicBool::new(false)), + input_ready: Arc::new(AtomicBool::new(false)), + output, + feedback: None, + captured_input: None, + media: None, + runtime: None, + commands, + } +} + +#[test] +fn stopped_generation_rejects_late_status_and_terminal_events() { + let (events, receiver) = mpsc::channel(); + let worker = worker(EventSender::unbounded(events)); + lock_lifecycle(&worker.lifecycle).generation = 2; + worker.emit("status", json!({"status":"streaming"})); + worker.terminal(Failure::signaling("late failure")); + assert!(receiver.try_recv().is_err()); + assert_eq!(lock_lifecycle(&worker.lifecycle).state, State::Connected); +} + +#[test] +fn explicit_cancellation_rejects_late_status_and_terminal_events() { + let (events, receiver) = mpsc::channel(); + let worker = worker(EventSender::unbounded(events)); + worker.cancelled.store(true, Ordering::Release); + worker.emit("status", json!({"status":"streaming"})); + worker.terminal(Failure::signaling("late failure")); + assert!(receiver.try_recv().is_err()); +} + +#[test] +fn terminal_failure_is_emitted_once() { + let (events, receiver) = mpsc::channel(); + let worker = worker(EventSender::unbounded(events)); + worker.terminal(Failure::signaling("failed")); + worker.terminal(Failure::signaling("duplicate")); + assert_eq!(receiver.try_iter().count(), 2); + assert_eq!(lock_lifecycle(&worker.lifecycle).state, State::Idle); +} + +#[test] +fn incompatible_profile_is_rejected_before_opening_signaling() { + let (output, _) = mpsc::channel(); + let (consumer, _receiver) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + for (codec, color) in [ + ("H265", "8bit_444"), + ("H265", "10bit_444"), + ("H264", "10bit_420"), + ("AV1", "8bit_420"), + ] { + let command = serde_json::from_value(json!({"id":"invalid-profile","type":"start","context":{ + "session":{"sessionId":"fixture","serverIp":"127.0.0.1","signalingUrl":"wss://127.0.0.1/nvst/"}, + "settings":{"transportMode":"webrtc","codec":codec,"colorQuality":color},"shortcuts":{} + }})).unwrap(); + let (response, _) = engine.handle(command); + assert_eq!( + response[0]["code"], "webrtc-profile-unsupported", + "{response:?}" + ); + assert!(engine.webrtc_session.is_none()); + } +} + +struct PeerFixture { + signaling_url: String, + media_port: u16, + stream: MediaStreamConfig, + stopped: Arc, + worker: Option>, + input: Receiver>, + partial_signaling_sent: Arc, +} + +#[test] +fn unsupported_webrtc_dimensions_fps_and_hdr_are_rejected_before_signaling() { + let (output, _) = mpsc::channel(); + let (consumer, _receiver) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + for profile in [ + json!({"resolution":"5120x2880","fps":60}), + json!({"resolution":"7680x4320","fps":60}), + json!({"resolution":"1920x1200","fps":1000}), + json!({"resolution":"1920x1200","fps":0}), + json!({"resolution":"1920x1200","fps":"60"}), + json!({"resolution":"invalid","fps":60}), + json!({"resolution":"1920x1200","fps":60,"enableHdr":true}), + ] { + for finalized in [false, true] { + let mut context = json!({ + "session":{"sessionId":"fixture","serverIp":"127.0.0.1","signalingUrl":"wss://127.0.0.1/nvst/"}, + "settings":{"transportMode":"webrtc","codec":"H265","colorQuality":"10bit_420"}, + "shortcuts":{} + }); + if finalized || profile.get("enableHdr").is_some() { + context["session"]["negotiatedStreamProfile"] = profile.clone(); + context["session"]["negotiatedStreamProfile"]["codec"] = json!("H265"); + context["session"]["negotiatedStreamProfile"]["colorQuality"] = json!("10bit_420"); + } else { + context["settings"] + .as_object_mut() + .unwrap() + .extend(profile.as_object().unwrap().clone()); + } + let command = serde_json::from_value( + json!({"id":"invalid-profile","type":"start","context":context}), + ) + .unwrap(); + let (response, _) = engine.handle(command); + assert_eq!( + response[0]["code"], "webrtc-profile-unsupported", + "{profile:?}: {response:?}" + ); + assert!(engine.webrtc_session.is_none()); + } + } +} + +impl PeerFixture { + fn start(video: Vec) -> Self { + Self::start_with_partial_signaling(video, false) + } + + fn start_with_partial_signaling(video: Vec, partial_signaling: bool) -> Self { + Self::start_with_stream( + video, + partial_signaling, + MediaStreamConfig { + width: 64, + height: 64, + ..MediaStreamConfig::default() + }, + ) + } + + fn start_with_stream( + video: Vec, + partial_signaling: bool, + profile: MediaStreamConfig, + ) -> Self { + use std::io::ErrorKind; + use str0m::change::SdpAnswer; + use str0m::format::Codec; + use str0m::media::{Direction, Frequency, MediaKind, MediaTime}; + use str0m::net::{Protocol, Receive}; + use str0m::{Candidate, Event, Input, Output, RtcConfig}; + use tungstenite::Message; + + opennow_streamer_transport::install_crypto(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let signaling_url = format!("ws://{}/nvst/", listener.local_addr().unwrap()); + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let address = socket.local_addr().unwrap(); + let media_port = address.port(); + let stopped = Arc::new(AtomicBool::new(false)); + let cancellation = stopped.clone(); + let partial_signaling_sent = Arc::new(AtomicBool::new(false)); + let partial_sent = partial_signaling_sent.clone(); + let (input_sender, input) = mpsc::channel(); + let worker = thread::spawn(move || { + let mut config = RtcConfig::new().clear_codecs().enable_opus(true); + let video_codec = if profile.codec == MediaVideoCodec::H265 { + config.codec_config().add_h265( + 96.into(), + Some(97.into()), + if profile.color_quality == MediaColorQuality::TenBit420 { + 2 + } else { + 1 + }, + 0, + 153, + ); + Codec::H265 + } else { + config = config.enable_h264(true); + Codec::H264 + }; + let mut rtc = config.build(Instant::now()); + rtc.add_local_candidate(Candidate::host(address, "udp").unwrap()); + let mut changes = rtc.sdp_api(); + let microphone_mid = + changes.add_media(MediaKind::Audio, Direction::RecvOnly, None, None, None); + let video_mid = + changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + let audio_mid = + changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + changes.add_channel("server-bootstrap".to_owned()); + let (offer, pending) = changes.apply().unwrap(); + let mut pending = Some(pending); + let deadline = Instant::now() + Duration::from_secs(10); + let stream = loop { + if cancellation.load(Ordering::Acquire) { + return; + } + assert!( + Instant::now() < deadline, + "mock signaling was not contacted" + ); + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)) + } + Err(error) => panic!("mock accept failed: {error}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + #[allow(clippy::result_large_err)] + let mut websocket = tungstenite::accept_hdr( + stream, + |request: &tungstenite::handshake::server::Request, + mut response: tungstenite::handshake::server::Response| { + assert!(request.uri().path().ends_with("/nvst/sign_in")); + assert!( + request + .uri() + .query() + .unwrap() + .contains("version=2&peer_role=1&pairing_id=fixture-seat") + ); + assert_eq!(request.headers()["Origin"], "https://play.geforcenow.com"); + assert_eq!( + request.headers()["Sec-WebSocket-Protocol"], + "x-nv-sessionid.fixture-seat" + ); + response.headers_mut().insert( + "Sec-WebSocket-Protocol", + request.headers()["Sec-WebSocket-Protocol"].clone(), + ); + Ok(response) + }, + ) + .unwrap(); + let peer_info = websocket.read().unwrap().into_text().unwrap(); + let peer_info: Value = serde_json::from_str(&peer_info).unwrap(); + assert_eq!( + peer_info["peer_info"]["resolution"], + format!("{}x{}", profile.width, profile.height) + ); + websocket.get_mut().set_nonblocking(true).unwrap(); + websocket.set_config(|config| config.max_write_buffer_size = 1024 * 1024); + fixture_write( + &mut websocket, + Message::Text( + json!({"peer_info":{"name":peer_info["peer_info"]["name"],"id":7},"ackid":1}) + .to_string() + .into(), + ), + ); + fixture_write(&mut websocket, Message::Text(json!({"peer_msg":{"from":9,"to":7,"msg":json!({"type":"offer","sdp":offer.to_sdp_string()}).to_string()},"ackid":2}).to_string().into())); + let mut connected = false; + let mut signaling_open = true; + let mut partial_queued = false; + let mut next_media = Instant::now(); + let mut timestamp = 900_123_u64; + while !cancellation.load(Ordering::Acquire) { + if signaling_open { + match websocket.flush() { + Ok(()) => { + if partial_queued { + partial_sent.store(true, Ordering::Release); + } + } + Err(tungstenite::Error::Io(error)) + if error.kind() == ErrorKind::WouldBlock => {} + Err(error) => panic!("fixture signaling flush: {error}"), + } + match websocket.read() { + Ok(Message::Text(packet)) => { + let packet: Value = serde_json::from_str(&packet).unwrap(); + if let Some(nested) = packet["peer_msg"]["msg"].as_str() { + let message: Value = serde_json::from_str(nested).unwrap(); + if message["type"] == "answer" { + assert_eq!(packet["peer_msg"]["from"], 7); + assert_eq!(packet["peer_msg"]["to"], 9); + let answer = message["sdp"].as_str().unwrap(); + let nvst = message["nvstSdp"].as_str().unwrap(); + let local_ufrag = answer + .lines() + .find_map(|line| line.strip_prefix("a=ice-ufrag:")) + .unwrap(); + assert!(nvst.contains(&format!( + "a=general.iceUserNameFragment:{local_ufrag}" + ))); + for expected in [ + format!("a=video.clientViewportWd:{}", profile.width), + format!("a=video.clientViewportHt:{}", profile.height), + format!("a=video.maxFPS:{}", profile.fps), + format!( + "a=video.bitDepth:{}", + profile.color_quality.bit_depth() + ), + "a=video.dynamicRangeMode:0".to_owned(), + ] { + assert!( + nvst.lines().any(|line| line == expected), + "missing {expected}" + ); + } + rtc.sdp_api() + .accept_answer( + pending.take().unwrap(), + SdpAnswer::from_sdp_string(answer).unwrap(), + ) + .unwrap(); + assert_eq!( + rtc.media(microphone_mid).unwrap().direction(), + Direction::Inactive + ); + assert_eq!( + answer + .lines() + .filter(|line| line.starts_with("m=")) + .count(), + 4 + ); + if partial_signaling { + fixture_write(&mut websocket, Message::Frame(tungstenite::protocol::frame::Frame::message( + vec![b'{'], + tungstenite::protocol::frame::coding::OpCode::Data(tungstenite::protocol::frame::coding::Data::Text), + false, + ))); + partial_queued = true; + } + } + } + } + Err(tungstenite::Error::Io(error)) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::TimedOut + ) => {} + Err(_) | Ok(Message::Close(_)) => signaling_open = false, + Ok(_) => {} + } + } + rtc.handle_input(Input::Timeout(Instant::now())).unwrap(); + loop { + match rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) => { + match socket.send_to(&packet.contents, packet.destination) { + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + ErrorKind::ConnectionReset + | ErrorKind::ConnectionRefused + | ErrorKind::Interrupted + ) => {} + Err(error) => panic!("fixture RTP send: {error}"), + } + } + Output::Event(Event::Connected) => connected = true, + Output::Event(Event::ChannelOpen(id, label)) + if label == "input_channel_v1" => + { + assert!( + rtc.channel(id) + .unwrap() + .write(true, &[0x0e, 0x02, 3, 0]) + .unwrap() + ); + } + Output::Event(Event::ChannelData(data)) => { + let _ = input_sender.send(data.data); + } + _ => {} + } + } + let mut buffer = [0; 65_536]; + loop { + match socket.recv_from(&mut buffer) { + Ok((length, source)) => rtc + .handle_input(Input::Receive( + Instant::now(), + Receive { + proto: Protocol::Udp, + source, + destination: address, + contents: (&buffer[..length]).try_into().unwrap(), + }, + )) + .unwrap(), + Err(error) if error.kind() == ErrorKind::WouldBlock => break, + Err(error) + if matches!( + error.kind(), + ErrorKind::ConnectionReset + | ErrorKind::ConnectionRefused + | ErrorKind::Interrupted + ) => + { + continue; + } + Err(error) => panic!("fixture RTP receive: {error}"), + } + } + if connected && Instant::now() >= next_media { + for (mid, codec, time, bytes) in [ + ( + video_mid, + video_codec, + MediaTime::from_90khz(timestamp), + video.clone(), + ), + ( + audio_mid, + Codec::Opus, + MediaTime::new(48_321, Frequency::FORTY_EIGHT_KHZ), + vec![0xf8, 0xff, 0xfe], + ), + ] { + let pt = rtc + .writer(mid) + .unwrap() + .payload_params() + .find(|params| params.spec().codec == codec) + .unwrap() + .pt(); + rtc.writer(mid) + .unwrap() + .write(pt, Instant::now(), time, bytes) + .unwrap(); + } + timestamp += 9_000; + next_media = Instant::now() + Duration::from_millis(100); + } + thread::sleep(Duration::from_millis(1)); + } + }); + Self { + signaling_url, + media_port, + stream: profile, + stopped, + worker: Some(worker), + input, + partial_signaling_sent, + } + } +} + +fn fixture_write( + socket: &mut tungstenite::WebSocket, + message: tungstenite::Message, +) { + match socket.send(message) { + Ok(()) => {} + Err(tungstenite::Error::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => panic!("fixture signaling write: {error}"), + } +} + +impl Drop for PeerFixture { + fn drop(&mut self) { + self.stopped.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let result = worker.join(); + if !thread::panicking() { + result.unwrap(); + } + } + } +} + +fn start_command(peer: &PeerFixture, id: &str) -> Command { + serde_json::from_value(json!({"id":id,"type":"start","context":{ + "session":{"sessionId":"fixture-seat","serverIp":"127.0.0.1","signalingUrl":peer.signaling_url, + "mediaConnectionInfo":{"ip":"127.0.0.1","port":peer.media_port,"usage":15}}, + "settings":{"transportMode":"webrtc", + "codec":if peer.stream.codec == MediaVideoCodec::H265 { "H265" } else { "H264" }, + "colorQuality":if peer.stream.color_quality == MediaColorQuality::TenBit420 { "10bit_420" } else { "8bit_420" }, + "resolution":format!("{}x{}", peer.stream.width, peer.stream.height), + "fps":peer.stream.fps,"maxBitrateMbps":10},"shortcuts":{} + }})).unwrap() +} + +#[test] +fn nullable_finalized_fps_uses_settings_without_changing_the_webrtc_stream() { + for fps in [30, 60, 120] { + for finalized_fps in [None, Some(Value::Null)] { + let video = vec![0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + let peer = PeerFixture::start_with_stream( + video.clone(), + false, + MediaStreamConfig { + width: 64, + height: 64, + fps, + ..MediaStreamConfig::default() + }, + ); + let (output, events) = mpsc::channel(); + let (consumer, received) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + let mut command = start_command(&peer, "nullable-fps"); + let context = command.context.as_mut().unwrap(); + context["session"]["negotiatedStreamProfile"] = json!({ + "codec":"H264", "colorQuality":"8bit_420", "resolution":null, "enableHdr":false + }); + if let Some(finalized_fps) = finalized_fps { + context["session"]["negotiatedStreamProfile"]["fps"] = finalized_fps; + } + let (responses, _) = engine.handle(command); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let deadline = Instant::now() + Duration::from_secs(8); + loop { + assert!( + Instant::now() < deadline, + "media did not arrive: {:?}", + events.try_iter().collect::>() + ); + if let Ok(frame) = received.recv_timeout(Duration::from_millis(50)) { + if frame.codec == "H264" { + assert_eq!(frame.payload.as_ref(), video); + break; + } + } + } + engine.stop("fixture stop"); + } + } +} + +#[test] +fn engine_webrtc_session_emits_frame_and_network_telemetry() { + let video = include_bytes!( + "../../../opennow-streamer-platform-windows/fixtures/probe/hevc-p010-sdr.hevc" + ) + .to_vec(); + let peer = PeerFixture::start_with_stream( + video, + false, + MediaStreamConfig { + codec: MediaVideoCodec::H265, + color_quality: MediaColorQuality::TenBit420, + width: 2560, + height: 1600, + fps: 60, + hdr: false, + ..MediaStreamConfig::default() + }, + ); + let (output, events) = mpsc::channel(); + let (consumer, received) = mpsc::sync_channel(8); + let (feedback, feedback_receiver) = mpsc::channel(); + let mut engine = Engine::with_media_consumer(output, consumer); + engine.media_feedback = Some(feedback_receiver); + let (responses, _) = engine.handle(start_command(&peer, "telemetry-fixture")); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + assert!( + Instant::now() < deadline, + "session telemetry did not arrive" + ); + if let Ok(frame) = received.recv_timeout(Duration::from_millis(20)) { + if frame.codec == "H265" { + feedback + .send(MediaFeedback::VideoFrameAccepted { + frame_index: frame.frame_index, + timestamp: frame.rtp_timestamp, + bytes: frame.payload.len().try_into().unwrap(), + keyframe: frame.keyframe, + }) + .unwrap(); + } + } + if let Some(event) = events.try_iter().find(|event| { + event["type"] == "telemetry" + && event["framesPerSecond"] + .as_f64() + .is_some_and(|fps| fps > 0.0) + && event["receiveBitrateMbps"] + .as_f64() + .is_some_and(|rate| rate > 0.0) + && event["pingMs"].is_number() + && event["jitterMs"].is_number() + && event["packetLossPercent"].is_number() + }) { + assert_eq!(event["transport"], "webrtc"); + assert_eq!(event["startId"], "telemetry-fixture"); + assert!(event["bitrateMbps"].as_f64().unwrap() > 0.0); + assert!( + event["peakBitrateMbps"].as_f64().unwrap() + >= event["bitrateMbps"].as_f64().unwrap() + ); + assert_eq!(event["packetLossPercent"], 0.0); + break; + } + } + engine.stop("fixture stop"); +} + +#[test] +fn engine_signaling_delivers_hevc_main10_sdr_to_native_media_consumer() { + let video = include_bytes!( + "../../../opennow-streamer-platform-windows/fixtures/probe/hevc-p010-sdr.hevc" + ) + .to_vec(); + let peer = PeerFixture::start_with_stream( + video, + false, + MediaStreamConfig { + codec: MediaVideoCodec::H265, + color_quality: MediaColorQuality::TenBit420, + width: 1920, + height: 1080, + fps: 60, + hdr: false, + ..MediaStreamConfig::default() + }, + ); + let (output, events) = mpsc::channel(); + let (consumer, received) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + let (responses, _) = engine.handle(start_command(&peer, "hevc-main10")); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let deadline = Instant::now() + Duration::from_secs(8); + loop { + assert!( + Instant::now() < deadline, + "HEVC did not arrive: {:?}", + events.try_iter().collect::>() + ); + if let Ok(frame) = received.recv_timeout(Duration::from_millis(50)) { + if frame.codec == "H265" { + assert!(frame.keyframe); + assert_eq!(frame.clock_rate_hz, 90_000); + assert!(frame.ssrc.is_some()); + assert_eq!(frame.frame_index, None); + for nal_type in [32, 33, 34, 20] { + assert!( + frame.payload.windows(6).any(|nal| { + nal.starts_with(&[0, 0, 0, 1]) && (nal[4] >> 1) & 0x3f == nal_type + }), + "missing HEVC NAL type {nal_type}" + ); + } + break; + } + } + } + assert!( + !events + .try_iter() + .any(|event| event["status"] == "streaming") + ); + engine.stop("fixture stop"); + assert_eq!(lock_lifecycle(&engine.lifecycle).state, State::Idle); +} + +#[test] +fn engine_signaling_to_real_h264_opus_decode_input_stop_and_restart() { + use openh264::formats::{RgbSliceU8, YUVBuffer, YUVSource}; + let rgb = vec![96_u8; 64 * 64 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (64, 64))); + let video = openh264::encoder::Encoder::new() + .unwrap() + .encode(&yuv) + .unwrap() + .to_vec(); + let (output, events) = mpsc::channel(); + let (consumer, received) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + for id in ["first", "restart"] { + let peer = PeerFixture::start(video.clone()); + let (feedback, feedback_receiver) = mpsc::channel(); + engine.media_feedback = Some(feedback_receiver); + let (responses, _) = engine.handle(start_command(&peer, id)); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let mut decoder = openh264::decoder::Decoder::new().unwrap(); + let mut decoded = false; + let mut audio = false; + let deadline = Instant::now() + Duration::from_secs(8); + while !(decoded && audio) { + assert!( + Instant::now() < deadline, + "media did not arrive: {:?}", + events.try_iter().collect::>() + ); + let Ok(frame) = received.recv_timeout(Duration::from_millis(50)) else { + continue; + }; + assert_eq!(frame.frame_index, None); + assert!(frame.ssrc.is_some()); + if frame.codec == "H264" { + assert_eq!(frame.clock_rate_hz, 90_000); + let picture = decoder.decode(&frame.payload).unwrap().unwrap(); + assert_eq!(picture.dimensions(), (64, 64)); + decoded = true; + } else if frame.codec == "Opus" { + assert_eq!(frame.payload.as_ref(), &[0xf8, 0xff, 0xfe]); + assert_eq!(frame.clock_rate_hz, 48_000); + audio = true; + } + } + assert!( + !events + .try_iter() + .any(|event| event["status"] == "streaming") + ); + feedback + .send(MediaFeedback::PlaybackStarted { + backend: "fixture-decoder", + }) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + loop { + assert!( + Instant::now() < deadline, + "first decoded frame event was not forwarded" + ); + if let Ok(event) = events.recv_timeout(Duration::from_millis(20)) { + if event["status"] == "streaming" { + assert_eq!(event["startId"], id); + break; + } + } + } + let deadline = Instant::now() + Duration::from_secs(2); + while !engine + .webrtc_session + .as_ref() + .unwrap() + .input_ready + .load(Ordering::Acquire) + { + assert!( + Instant::now() < deadline, + "input handshake timed out for {id}; lifecycle={:?}, peer_finished={}, events={:?}", + lock_lifecycle(&engine.lifecycle).state, + peer.worker.as_ref().is_some_and(JoinHandle::is_finished), + events.try_iter().collect::>() + ); + thread::sleep(Duration::from_millis(5)); + } + engine.webrtc_session.as_ref().unwrap().anti_afk().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + loop { + assert!(Instant::now() < deadline, "input did not arrive"); + if let Ok(packet) = peer.input.recv_timeout(Duration::from_millis(20)) { + if packet.first() == Some(&0x23) { + assert_eq!(packet[9], 0x22); + assert_eq!(&packet[10..14], &3_u32.to_le_bytes()); + break; + } + } + } + let before = Instant::now(); + engine.stop("fixture stop"); + assert!(before.elapsed() < Duration::from_secs(1)); + assert_eq!(lock_lifecycle(&engine.lifecycle).state, State::Idle); + drop(peer); + while received.try_recv().is_ok() {} + while events.try_recv().is_ok() {} + } +} + +#[test] +fn incomplete_signaling_message_does_not_stall_input_media_or_stop() { + use openh264::formats::{RgbSliceU8, YUVBuffer}; + let rgb = vec![96_u8; 64 * 64 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (64, 64))); + let video = openh264::encoder::Encoder::new() + .unwrap() + .encode(&yuv) + .unwrap() + .to_vec(); + let peer = PeerFixture::start_with_partial_signaling(video, true); + let (output, events) = mpsc::channel(); + let (consumer, received) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + let (responses, _) = engine.handle(start_command(&peer, "partial-signaling")); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let deadline = Instant::now() + Duration::from_secs(2); + while !peer.partial_signaling_sent.load(Ordering::Acquire) + || !engine + .webrtc_session + .as_ref() + .unwrap() + .input_ready + .load(Ordering::Acquire) + { + assert!( + Instant::now() < deadline, + "input stalled with partial signaling: {:?}", + events.try_iter().collect::>() + ); + thread::sleep(Duration::from_millis(1)); + } + engine.webrtc_session.as_ref().unwrap().anti_afk().unwrap(); + let packets = input_bodies(&peer, &[3, 4]); + assert_eq!(&packets[0][4..6], &0x7c_u16.to_be_bytes()); + let deadline = Instant::now() + Duration::from_secs(2); + let mut video_timestamps = std::collections::BTreeSet::new(); + let mut audio_received = false; + while video_timestamps.len() < 3 || !audio_received { + assert!( + Instant::now() < deadline, + "media stalled with partial signaling: {:?}", + events.try_iter().collect::>() + ); + if let Ok(frame) = received.recv_timeout(Duration::from_millis(10)) { + match frame.codec.as_str() { + "H264" => { + video_timestamps.insert(frame.rtp_timestamp); + } + "Opus" => audio_received = true, + other => panic!("unexpected codec {other}"), + } + } + } + let before = Instant::now(); + engine.stop("partial signaling stop"); + assert!(before.elapsed() < Duration::from_secs(1)); + assert!(!events.try_iter().any(|event| event["type"] == "error")); +} + +#[test] +fn engine_stop_cancels_an_incomplete_signaling_handshake() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("ws://{}/nvst/", listener.local_addr().unwrap()); + let (accepted, accepted_receiver) = mpsc::channel(); + let (finish, finished) = mpsc::channel(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + accepted.send(()).unwrap(); + let _ = finished.recv_timeout(Duration::from_secs(3)); + drop(stream); + }); + let (output, events) = mpsc::channel(); + let (consumer, _received) = mpsc::sync_channel(8); + let mut engine = Engine::with_media_consumer(output, consumer); + let command: Command = serde_json::from_value(json!({"id":"cancelled","type":"start","context":{ + "session":{"sessionId":"fixture-seat","serverIp":"127.0.0.1","signalingUrl":url}, + "settings":{"transportMode":"webrtc","codec":"H264","colorQuality":"8bit_420"},"shortcuts":{} + }})).unwrap(); + let (responses, _) = engine.handle(command); + assert_eq!(responses[0]["transport"], "webrtc"); + accepted_receiver + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + let before = Instant::now(); + engine.stop("cancel handshake"); + assert!(before.elapsed() < Duration::from_millis(500)); + assert_eq!(lock_lifecycle(&engine.lifecycle).state, State::Idle); + finish.send(()).unwrap(); + server.join().unwrap(); + let events = events.try_iter().collect::>(); + assert!( + !events + .iter() + .any(|event| event["type"] == "error" || event["status"] == "streaming") + ); +} + +fn input_bodies(peer: &PeerFixture, expected: &[u32]) -> Vec> { + let deadline = Instant::now() + Duration::from_secs(2); + let mut bodies = Vec::new(); + while bodies.len() < expected.len() { + assert!( + Instant::now() < deadline, + "missing input bodies, received {bodies:?}" + ); + if let Ok(packet) = peer.input.recv_timeout(Duration::from_millis(10)) { + if packet.first() != Some(&0x23) { + continue; + } + let offset = match packet[9] { + 0x22 => 10, + 0x21 => 12, + other => panic!("unexpected wrapper {other}"), + }; + let body = packet[offset..].to_vec(); + assert_eq!( + u32::from_le_bytes(body[..4].try_into().unwrap()), + expected[bodies.len()] + ); + bodies.push(body); + } + } + bodies +} + +fn held_controller() -> CapturedInput { + CapturedInput::Gamepad { + controller_id: 0, + bitmap: 0x0101, + buttons: 0xffff, + left_trigger: 255, + right_trigger: 255, + left_stick_x: 32767, + left_stick_y: 32767, + right_stick_x: 32767, + right_stick_y: 32767, + } +} + +#[test] +fn engine_input_pause_preserves_media_releases_controls_and_stop_flushes_neutral_state() { + use openh264::formats::{RgbSliceU8, YUVBuffer}; + let rgb = vec![96_u8; 64 * 64 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (64, 64))); + let video = openh264::encoder::Encoder::new() + .unwrap() + .encode(&yuv) + .unwrap() + .to_vec(); + let peer = PeerFixture::start(video); + let (host, runtime) = opennow_streamer_platform::create_test_runtime(); + let (output, events) = mpsc::channel(); + let mut engine = Engine::with_media_runtime(output, runtime.clone()); + let (responses, _) = engine.handle(start_command(&peer, "pause")); + assert_eq!(responses[0]["transport"], "webrtc", "{responses:?}"); + let deadline = Instant::now() + Duration::from_secs(5); + while !engine + .webrtc_session + .as_ref() + .unwrap() + .input_ready + .load(Ordering::Acquire) + { + assert!( + Instant::now() < deadline, + "input not ready: {:?}", + events.try_iter().collect::>() + ); + thread::sleep(Duration::from_millis(5)); + } + let input = runtime.captured_input(); + input.push(CapturedInput::Key { + virtual_key: 65, + modifiers: 0, + pressed: true, + }); + input.push(CapturedInput::MouseButton { + button: 1, + pressed: true, + }); + input.push(held_controller()); + input_bodies(&peer, &[3, 8, 12]); + let command = + serde_json::from_value(json!({"id":"pause-input","type":"input-paused","paused":true})) + .unwrap(); + let (response, _) = engine.handle(command); + assert_eq!(response[0]["type"], "ok"); + let releases = input_bodies(&peer, &[4, 9, 12]); + assert_eq!(&releases[2][12..24], &[0; 12]); + let (_, recording) = engine + .media_session + .as_ref() + .unwrap() + .control() + .subscribe_recording() + .unwrap(); + let (frame_sender, frame_receiver) = mpsc::channel(); + let frame_worker = thread::spawn(move || { + let _ = frame_sender.send(recording.recv()); + }); + let frame = frame_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("media stopped during input pause") + .unwrap(); + assert!(matches!( + frame.codec, + MediaCodec::H264 | MediaCodec::Opus { .. } + )); + frame_worker.join().unwrap(); + engine + .media_session + .as_ref() + .unwrap() + .control() + .unsubscribe_recording(); + input.push(CapturedInput::Key { + virtual_key: 66, + modifiers: 0, + pressed: true, + }); + let command = + serde_json::from_value(json!({"id":"resume-input","type":"input-paused","paused":false})) + .unwrap(); + engine.handle(command); + engine.webrtc_session.as_ref().unwrap().anti_afk().unwrap(); + let pulse = input_bodies(&peer, &[3, 4]); + assert_eq!(&pulse[0][4..6], &0x7c_u16.to_be_bytes()); + assert_eq!(&pulse[1][4..6], &0x7c_u16.to_be_bytes()); + input.push(CapturedInput::Key { + virtual_key: 67, + modifiers: 0, + pressed: true, + }); + input.push(CapturedInput::MouseButton { + button: 2, + pressed: true, + }); + input.push(held_controller()); + input_bodies(&peer, &[3, 8, 12]); + let before = Instant::now(); + engine.stop("held input stop"); + assert!(before.elapsed() < Duration::from_secs(1)); + let releases = input_bodies(&peer, &[4, 9, 12]); + assert_eq!(&releases[0][4..6], &67_u16.to_be_bytes()); + assert_eq!(&releases[2][12..24], &[0; 12]); + assert!(!events.try_iter().any(|event| event["type"] == "error")); + drop(engine); + runtime.shutdown(); + host.join().unwrap(); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/video_attributes.sdp b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/video_attributes.sdp new file mode 100644 index 000000000..1f5142525 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-core/src/webrtc/video_attributes.sdp @@ -0,0 +1,74 @@ +m=video 0 RTP/AVP +a=msid:fbc-video-0 +a=vqos.fec.rateDropWindow:10 +a=vqos.fec.minRequiredFecPackets:2 +a=vqos.fec.repairMinPercent:5 +a=vqos.fec.repairPercent:5 +a=vqos.fec.repairMaxPercent:35 +a=vqos.bllFec.enable:0 +a=vqos.dynamicStreamingMode:0 +a=vqos.drc.enable:0 +a=vqos.dfc.enable:0 +a=vqos.dfc.adjustResAndFps:0 +a=vqos.calculateAvgVideoStreamingBitrate:1 +a=video.dx9EnableNv12:1 +a=video.dx9EnableHdr:0 +a=vqos.qpg.enable:1 +a=vqos.resControl.qp.qpg.featureSetting:7 +a=video.adaptiveQuantization.spatialAQSetting:7 +a=video.adaptiveQuantization.temporalAQSetting:0 +a=video.adaptiveQuantization.spatialAQStrength:12 +a=video.adaptiveQuantization.qpThresholdAdjPercent:2 +a=video.adaptiveQuantization.saqAdaptMinQpThresholdPercent:40 +a=video.adaptiveQuantization.saqAdaptMaxQpThresholdPercent:100 +a=video.adaptiveQuantization.saqAdaptDecayStrengthX100:250 +a=video.adaptiveQuantization.perfAdjEnablement:1 +a=video.framePacing.mode:2 +a=bwe.useOwdCongestionControl:1 +a=video.enableRtpNack:1 +a=vqos.bw.txRxLag.minFeedbackTxDeltaMs:200 +a=vqos.drc.bitrateIirFilterFactor:18 +a=video.packetSize:1140 +a=packetPacing.version:3 +a=packetPacing.mode:1 +a=packetPacing.minNumPacketsPerGroup:15 +a=packetPacing.enableAccurateSleep:1 +a=packetPacing.enableSmoothTransition:1 +a=packetPacing.allowFpsBasedToggle:1 +a=vqos.relaxMaxBitrate.overrideAvgBitrateThresholdPercent:4 +a=vqos.relaxMaxBitrate.customAvgBitrateThresholdPercent:65 +a=vqos.relaxMaxBitrate.overrideAvgQpThresholdPercent:7 +a=vqos.relaxMaxBitrate.customAvgQpThresholdPercent:51 +a=vqos.relaxMaxBitrate.iirFilterFactor:120 +a=vqos.qpDelta.qpDeltaMaxPercent:10 +a=vqos.qpDelta.qpDeltaSurfaceAdjustmentStrengthPercent:70 +a=vqos.qpDelta.qpDeltaVbvUsageFactorPercentH264:100 +a=vqos.qpDelta.qpDeltaVbvUsageFactorPercentH265:100 +a=vqos.qpDelta.qpDeltaVbvUsageFactorPercentAv1:100 +a=vqos.qpDelta.qpDeltaMinPercent:60 +a=vqos.qpDelta.qpDeltaIirFactor:60 +a=vqos.qpDelta.qpDeltaThrottlePercent:100 +a=vqos.adjustStreamingFpsDuringOutOfFocus:0 +a=vqos.resControl.cpmRtc.ignoreOutOfFocusWindowState:1 +a=vqos.resControl.perfHistory.rtcIgnoreOutOfFocusWindowState:1 +a=vqos.resControl.cpmRtc.featureMask:0 +a=vqos.resControl.cpmRtc.enable:0 +a=vqos.resControl.cpmRtc.minResolutionPercent:100 +a=vqos.resControl.cpmRtc.resolutionChangeHoldonMs:999999 +a=packetPacing.maxDelayUs:1000 +a=packetPacing.minNumPacketsFrame:10 +a=video.rtpNackQueueLength:1024 +a=video.rtpNackQueueMaxPackets:512 +a=video.rtpNackMaxPacketCount:25 +a=vqos.drc.qpMaxResThresholdAdj:4 +a=vqos.grc.qpMaxResThresholdAdj:4 +a=vqos.drc.iirFilterFactor:100 +a=vqos.bw.enableBandwidthEstimation:1 +a=vqos.bw.disableBitrateLimit:0 +a=vqos.grc.enable:0 +a=video.maxNumReferenceFrames:4 +a=video.mapRtpTimestampsToFrames:1 +a=video.encoderCscMode:3 +a=video.dynamicRangeMode:0 +a=video.scalingFeature1:0 +a=video.prefilterParams.prefilterModel:0 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/Cargo.toml new file mode 100644 index 000000000..4ed6fd866 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "opennow-streamer-ffi" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = [] +linux-vaapi = ["opennow-streamer-platform/linux-vaapi"] +linux-ffmpeg = ["opennow-streamer-platform/linux-ffmpeg"] +linux-ffmpeg-bundled = ["opennow-streamer-platform/linux-ffmpeg-bundled"] + +[dependencies] +opennow-streamer-core = { path = "../opennow-streamer-core" } +opennow-streamer-hid = { path = "../opennow-streamer-hid" } +opennow-streamer-platform = { path = "../opennow-streamer-platform" } +opennow-streamer-protocol = { path = "../opennow-streamer-protocol" } +serde_json.workspace = true + +[dev-dependencies] +opennow-streamer-platform = { path = "../opennow-streamer-platform", features = ["test-runtime"] } diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/README.md b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/README.md new file mode 100644 index 000000000..af9d2885e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/README.md @@ -0,0 +1,301 @@ +# OpenNOW streamer FFI + +This crate exposes `opennow-streamer-core::Engine` as a C-compatible in-process library. It is an integration boundary only: it does not start a child process, read or write standard streams, create an SDL/native window, or depend on Qt. + +## Contract + +- Initialize every `OpenNowStreamerConfig` field, set `abi_version` to `OPENNOW_STREAMER_FFI_ABI_VERSION`, and set `struct_size` to `sizeof(OpenNowStreamerConfig)`. +- `opennow_streamer_create` writes one owned opaque handle to `output`. On failure it writes `NULL` when the supplied config header is readable. +- `opennow_streamer_send` copies one serialized JSON protocol command before returning. `OPENNOW_STREAMER_OK` means the bounded command queue accepted it, not that the protocol command succeeded. Parse errors and protocol results arrive at `response_callback`. +- Values returned directly by `Engine::handle` go to `response_callback`. Unsolicited engine events go to `event_callback`. Each byte slice is UTF-8 JSON and is valid only for the duration of that callback. +- `frame_available_callback` is separate from the protocol callbacks. It only asks the shell to schedule a render; it carries no frame or protocol payload and may run on a decoder thread. +- Response and event callbacks each run serially on their own dispatcher thread, so the two callbacks may overlap. They must return promptly and must not re-enter this handle. `user_data` must remain valid until `opennow_streamer_destroy` returns. +- `opennow_streamer_destroy` consumes the handle exactly once, waits for the engine and both callback queues to drain, and then returns. No call may race with destroy. A null handle is rejected; reusing a destroyed pointer is caller-side undefined behavior. +- Every exported function catches Rust panics before they can unwind through the C ABI. A worker-thread panic closes the command queue. + +### Generic flight controls and wheels (ABI 15) + +Rebuild Qt and the native library together with ABI 15. The generic-HID exports are +`opennow_streamer_replace_generic_hid_inventory`, +`opennow_streamer_submit_generic_hid_report`, +`opennow_streamer_take_generic_hid_output`, and +`opennow_streamer_get_generic_hid_admissions`. + +The admission query takes a versioned `OpenNowGenericHidAdmissions` snapshot initialized +with `OPENNOW_STREAMER_GENERIC_HID_VERSION` and its C size. It returns at most eight +rows keyed by slot and incarnation: Pending until a sent attach receives a valid +control-1 response, Admitted after acceptance, or Declined after refusal. Detach +fallback and session replacement reset admission to Pending without inventing acceptance. + +Inventory rows use versioned `OpenNowGenericHidInterface` structures. Each row identifies +one interface of a source slot and carries the source incarnation, VID, PID, revision, +and a descriptor byte slice. Rows with the same slot must agree on device identity. +Replacement is atomic. There are at most eight sources and 16 interfaces per source, +with descriptors of at most 4,096 bytes. Empty descriptors retain explicit detach-on-pause +fallback behavior. The caller's descriptor memory is copied before the call returns. + +`OpenNowGenericHidReport` submits an owned copy of at most 256 bytes. Normal input uses +operation 0. Output and feature responses use their corresponding operation and an error +byte. Source incarnations and interfaces are validated before queue admission. Each source +has a 64-report, 16-KiB bound; overflow faults the source and preserves its release path. + +Before calling `opennow_streamer_take_generic_hid_output`, initialize the output's +`version` to `OPENNOW_STREAMER_GENERIC_HID_VERSION` and `struct_size` to its C size. +It returns `OK` with one copied command, or `CLOSED` when no command is pending. +The output queue holds at most 32 commands and 8 KiB. The Qt host executes physical +HID operations on its input worker, never inside an FFI callback or graphics callback. + +Capture closure generates descriptor-derived releases before forwarding stops. Missing +or unsafe descriptors use detach and reattach instead. Session and device identity checks +prevent old reports and outputs from reaching a replacement device. See +[`docs/flight-controls.md`](../../../../docs/flight-controls.md) for wire semantics and +the physical checks still required. + +### Server timers and termination (ABI 13) + +Rebuild Qt and the native library together with the current ABI 15. Structure layouts and callback +ownership are unchanged. `event_callback` now delivers generation-scoped +`session-timer` events with `startId`, `timerType`, `secondsLeft` and `active`. +Timer type 4 warns of idle expiry; type 5 clears the warning. Successful gameplay +input also clears an active warning with a type-4 event whose `active` is false. +The shell displays this state above its existing video item without changing capture +or media ownership. + +Server termination uses the existing error and stopped-status events. Their +`termination` object has `source: "nvst-server"`, `startId`, `code: "server-terminated"`, +the original `wireCode`, optional `reason` and `extendedCode`, and `resumable`. +The wire result and the application's termination reason are different namespaces. +Unknown classification fields are null; the shell retains CloudMatch reconciliation +for these rather than guessing that the session has ended. A false `resumable` is +authoritative and must prevent automatic native restarts and seat-resume attempts. +Lifecycle timers coalesce to their latest state when the callback queue is full; +terminal delivery waits on that bounded queue from the stopped session's event worker, +never from the network or render thread. + +### Deterministic presentation incompatibility (ABI 12) + +`opennow_streamer_record_frame` returns `OPENNOW_STREAMER_GRAPHICS_UNSUPPORTED` when the decoder's +output kind cannot be imported with the graphics context's enabled capabilities, for example a +DMA-BUF frame on a Vulkan context without `OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT`. +The same decoder and device fail identically on every attempt, so the host must not restart the +media transport for it. `OPENNOW_STREAMER_RENDER_FAILED` remains the status for failures that a +fresh device, surface, or session may clear. Rebuild Qt and the native library together. + +### Windows adapter selection (ABI 10) + +ABI 10 appends `uint64_t windows_adapter_luid` to `OpenNowStreamerConfig`; rebuild the Qt shell +and native library together. Initialize it to zero on non-Windows platforms and when Windows +should use the system default adapter. A nonzero value contains the packed bits of the Windows +adapter `LUID` selected by the shell. + +Engine creation copies the selection into its embedded media runtime. Windows hardware capability +and HDR/profile probes resolve that exact `IDXGIAdapter`, create their D3D device with +`D3D_DRIVER_TYPE_UNKNOWN`, and report the backend unavailable when the LUID cannot be resolved. +They never retry against the default adapter. The selection is immutable for the engine lifetime; +changing it requires destroying and recreating the engine. Standalone runtime paths retain their +default-adapter behavior. + +The live embedded decoder still adopts the D3D11 device and immediate context supplied by Qt in +`OpenNowStreamerGraphicsContext`. Before installing a D3D11 context, the FFI reads that device's +DXGI adapter LUID and rejects a mismatch with `OPENNOW_STREAMER_GRAPHICS_UNAVAILABLE`, with both +LUIDs in the native graphics diagnostic. The configured LUID does not replace, wrap, or switch +the borrowed Qt graphics device. + +### Clipboard text (ABI 9) + +Rebuild Qt and the native library together with ABI version 9. Existing structure +layouts and the graphics render-command version are unchanged. The new export is +`opennow_streamer_submit_text(const OpenNowStreamer *, const uint8_t *, size_t)`; +`OPENNOW_STREAMER_MAX_TEXT_BYTES` is 65,536. + +The caller supplies nonempty, strictly valid UTF-8 without embedded NUL or a trailing +terminator. The library copies the complete text before returning; it never truncates, +normalizes newlines, interprets text as key combinations, or logs its contents. +Null pointers return `NULL_POINTER`, empty/invalid/NUL-containing text returns +`INVALID_CONFIG`, and lengths over the bound return `MESSAGE_TOO_LARGE` before the +buffer is read. An oversized length takes precedence over a null text pointer. + +Admission requires active embedded capture and negotiated session input. Otherwise +it returns `CLOSED`. One paste occupies a capture-queue entry and retains a single-flight +reservation through the ordered transport command queue. A second paste or a full +256-entry capture queue returns `QUEUE_FULL` without admitting any text, evicting +gameplay input, or triggering the control-overflow shutdown. `OK` acknowledges whole +local admission, not delivery to or insertion by the remote application. + +Deferred discrete input and paste share one ordered transport FIFO. One accepted +paste keeps its single-flight reservation until all encoded chunks are admitted, +cancelled, or the session stops. A paste retains at most 65,536 UTF-8 bytes plus +bounded wire framing; the separate encoded-text bound is 128 KiB. Later gameplay +input cannot overtake it, and successful prefixes are never resent. Cancellation +discards only the remaining text and preserves queued key/button releases. +Temporary SCTP pressure retries without ending media. If a second paste or an +oversized encoded batch is rejected locally, ABI 13 emits a generation-scoped +`text-input-error` event with `startId` and `reason`; Qt displays nonterminal retry +feedback without including text contents in diagnostics. + +Capture loss, input unavailability, and session teardown cancel text still pending +in capture or transport. Readiness changes are generation-scoped, so stale session +events cannot enable or cancel text belonging to a replacement session. A paste +already submitted to SCTP cannot be recalled. +Transport checks capacity for the complete encoded batch across all eight negotiated +SCTP channels before writing any chunk. This matches str0m 0.23's 128-KiB aggregate +send-buffer limit; the receive worker owns all writes and does not poll the network +between chunks. Insufficient capacity rejects the whole batch and emits a payload-free +diagnostic rather than retrying a paste that might duplicate text. Connection failure +can interrupt remote delivery; the protocol provides no paste-level acknowledgement +or rollback. + +Each packet uses ordered reliable control command `0x0206` and remote-input type 23, +with raw UTF-8 bodies of at most `0x3f8` (1,016) bytes. Overflow cuts back to the nearest +code-point boundary (at most three bytes). The RI header is a big-endian 32-bit length +of `4 + body bytes`, then little-endian type 23. Following the verified +`NvstRemoteInput.utf8TextPackets` / `sendFramedRemoteInput` reference contract, inner +packets up to 1,007 bytes receive a type-14 envelope with zero padding to +`((inner_length + 8) / 8) * 8 + 8` bytes and one little-endian 64-bit capture timestamp. +Larger inner packets (text bodies of 1,000–1,016 bytes) are sent bare. The older +serialized ASCII text command retains its key-stroke conversion for compatibility; +the typed clipboard path never enters that codec or its raw-input diagnostics. + +### Controller rumble events + +The additive `controller-rumble` JSON event uses the existing `event_callback` (no C ABI +layout change): `startId` is the originating start command's ID, `controllerId` is a wire +slot from 0 through 3, `lowFrequency` and `highFrequency` are unsigned 16-bit motor +amplitudes, and `durationMs` is an integer from 1 through 65535. Hosts must discard +events whose `startId` does not match the currently authorized start, and stop motors +on session replacement, stop, failure, capture loss, and device removal. + +NVST control command `0x010b` uses a little-endian `u16 code, u16 payloadLength` +envelope. Its payload begins with `u16 kind, u16 blobLength`. Kind 1 contains 6-byte +records (`u16 slot, u16 left, u16 right`); kind 2 contains 8-byte records with an +additional `u16 durationMs`. Absent or zero durations become 1000 ms, matching the +reference client. Zero motor amplitudes are stop commands. Unknown kinds and slots +above 3 are ignored. Complete records within the smaller of the declared blob length +and available payload are accepted; truncated outer envelopes are not dispatched. + +Transport reception coalesces into four session-owned latest-value slots, including +zero-amplitude stops. The engine polls these alongside input and forwards events +nonblockingly through the bounded FFI queue. If that queue is full, four latest-value +slots retain commands for retry, so stops are not permanently lost. Coalesced records +are included in the periodic queue-drop diagnostics. Qt also bounds callback storage, +validates every numeric field and start ID, and emits `controllerRumbleRequested` on +the GUI thread. Its `controllerRumbleStopped` signal accompanies presentation +invalidation. Callback overflow stops motors and discards rumble queued before the +overflow, without discarding status or response callbacks. The shell owns +physical-device routing and shutdown. Individual rumble events are +not logged on the high-frequency callback path. + +### Optional MetalFX spatial upscaling (ABI 8, render command 3) + +ABI 8 requires rebuilding the Qt shell and native runtime together. Version 3 of +`OpenNowStreamerRecordCommand` appends `upscale_sharpness` and `upscale_denoise` +after the version-2 `upscale_width` and `upscale_height` fields. Earlier command +versions and truncated layouts are rejected. Initialize sharpness to 10 and denoise +to 0; valid inclusive ranges are 0–15 and 0–20 respectively. Invalid controls are +rejected even when scaling is disabled. Other graphics backends ignore valid controls. +Set both target dimensions to zero for normal scaling. On Metal, nonzero dimensions request spatial +MetalFX at the video viewport's physical pixel size, excluding letterboxing. +Dimensions must both be zero or each be in `1..=16384`. Other graphics backends +ignore the target. Unsupported MetalFX configurations fall back to the original +converted texture without stopping the session. + +The Mac producer encodes scaling after YUV-to-RGB conversion in the same borrowed +command buffer. Clarity and denoise filter source-resolution RGB in that conversion +pass, before MetalFX; they are not MetalFX descriptor settings. The four cardinal +neighbors form `blur`, with `sharpness = value / 10` and +`denoise = min(value / 10 * 0.65, 1)`. The shader computes +`denoised = mix(center, blur, denoise)`, then clamps +`denoised + (denoised - blur) * sharpness` to 0–1, matching OpenNOW-Mac's native +spatial shader. Zero controls skip the neighborhood samples. Changes update +uniforms without rebuilding the scaler or restarting the session. +New control values apply when the next decoded frame is recorded; a retained output +keeps its existing processing until that frame arrives. Token ownership and the +single-record contract remain unchanged. +The returned texture dimensions describe the actual output; +`OpenNowStreamerFrameInfo` still describes the decoded source. Source sequence, +timestamps, color precision, frame-slot ownership, and scene-graph retirement +remain unchanged. Qt must refresh the target after resize, fullscreen, or display +scale changes, and must not infer source video geometry from upscaled textures. + +Qt publishes the decoded display size after a successful frame recording, using +the actual recorded size for the Windows pending-frame path and retaining the +decoded source size when MetalFX changes the texture size. Updates are coalesced +and tagged with Qt's presentation generation, so a queued resize from a stopped +or replaced session cannot change the next session's geometry. The existing video +item uses the requested profile only until decoded dimensions arrive, then uses +the decoded size for aspect fitting, pointer mapping and confinement. A decoded +resize does not recreate the item, transport or session. This does not change the +FFI layout or equate texture-slot generations with presentation generations. + +### HDR texture metadata (introduced in ABI 6) + +ABI 6 adds `color_space` to `OpenNowStreamerRecordedFrame` and rejects clients compiled for earlier ABI versions. Rebuild the native runtime and Qt shell together. `texture_format` now accepts `RGBA16F` in addition to `RGBA8` and `RGB10A2`. + +The color-space constants identify the encoded RGB signal: `SDR709` is the existing SDR output, `PQ2020` is ST 2084 with BT.2020 primaries, and `HLG2020` is HLG with BT.2020 primaries. A 10-bit texture does not imply HDR. Windows converts HDR decoder surfaces to PQ BT.2020 RGB10A2; Linux preserves PQ/HLG BT.2020 in RGBA16F. HDR in RGBA8 is rejected at the FFI boundary. The shell must transform the signal to its actual swapchain color space, or explicitly tone-map HDR for SDR output. + +Color metadata belongs to each retained frame, so HDR/SDR transitions cannot reuse a previous frame's color interpretation. The texture ownership, bounded slots, generation, and sender presentation timestamps keep the same lifecycle. + +### Shared Vulkan device (introduced in ABI 5) + +ABI 5 appends `const OpenNowStreamerVulkanDevice *vulkan_device` to the engine config and rejects older config versions. Initialize this field to `NULL` on platforms that do not adopt a shared device. + +On Linux, call `opennow_streamer_vulkan_device_create` before creating the Qt graphics device. Success returns one opaque owner; unsupported builds, unavailable Vulkan Video devices, and device-creation failures return `OPENNOW_STREAMER_GRAPHICS_UNAVAILABLE` with a null output. `opennow_streamer_vulkan_device_info` writes the complete version-1 `OpenNowStreamerVulkanDeviceInfo`, including its size, instance, physical device, logical device, graphics queue, queue family and index, and API version. The output needs writable storage for the complete structure; it does not need preinitialization. These native objects are borrowed, not transferred to the caller. + +Linux bootstrap failures retain the device-creation reason in a single line capped at 512 characters, sent to the native file log when configured and to stderr even before runtime logging is initialized. This diagnostic contains the typed device error, not native object addresses or session credentials. Stderr write failures do not change the FFI result. + +Qt adopts these exact graphics objects and passes the owner in `OpenNowStreamerConfig`. Engine creation clones the native reference before starting its worker, and the runtime passes that same owner to every Linux decoder session. Keep the shell's owner alive until all adopted Qt graphics resources, windows, and Vulkan instance wrappers have been released. `opennow_streamer_vulkan_device_destroy` consumes only the shell's reference; it does not invalidate a reference retained by an engine or decoder. No call may race with owner destruction. Null device handles are rejected by info and destroy. + +Embedded Vulkan capabilities come from the attached logical device's codec profiles rather than the standalone device probe. Session startup also validates the negotiated codec, dimensions, and color depth: 4:2:0 8-bit uses NV12, supported 4:2:0 10-bit uses P010, and 4:4:4 is rejected instead of silently downgraded. A null owner keeps embedded Vulkan decode unavailable and preserves the existing non-Vulkan fallback. Standalone backend selection and other operating systems are unchanged. + +The protocol-6 hello report includes `videoBackends[].codecs[].colorQualities` for embedded Linux codecs. Vulkan lists only the attached device's supported `8bit_420`/`10bit_420` profiles. VAAPI lists `10bit_420` for HEVC/AV1 only when FFmpeg supports that decoder and a render node reports the corresponding decode profile and 10-bit render-target format. Other Linux paths remain 8-bit only. Windows reports additive `hdrSupported` flags from actual P010 decoder and PQ video-processor conversion probes without changing its existing SDR color-profile contract. Qt adds the current window's `nativeHdrSupported` to the transient runtime capabilities sent to core. Both Auto and manual codec choices are checked before CloudMatch allocation; the media-start check remains a second guard. + +The shared decoder copies decoded images into bounded GPU snapshots isolated from FFmpeg's reference-picture pool. Completed snapshots remain immutable while Qt samples them. This path is GPU-only, with no CPU readback, but it is not zero-copy and does not advertise a zero-copy mode. + +All three queues are bounded. Command submission returns `OPENNOW_STREAMER_QUEUE_FULL` rather than blocking. Responses backpressure the engine worker so an accepted command's response is retained. Unsolicited events use a drop-newest policy when their queue is full because the engine's event path cannot block latency-sensitive transport workers. + +Queue-drop diagnostic deltas are retained per source when periodic event delivery +finds a full queue, then retried at the next one-second flush. The engine joins +media producers and drains remaining drop feedback before completing stop. +Final event delivery is also nonblocking; undeliverable final deltas are written +directly to the native log and omitted from callback/UI totals rather than waiting +for a stalled dispatcher. The dispatcher remains alive through engine shutdown. Callback owners must +keep accepting events through the stop response to receive these final deltas; +native file logging still records them if a host has already disabled callbacks. +This does not change media queues or normal event-path backpressure. + +## GPU frame lifecycle + +The graphics API is GPU-only. It exposes no window, swap chain, `QWindow`, CPU image, pixel buffer, or encoded-video callback. + +1. On the QQuick render thread, call `opennow_streamer_set_graphics_context` with the versioned native objects borrowed from the current QRhi. +2. When `frame_available_callback` schedules a frame, call `opennow_streamer_acquire_latest_frame`. The bounded mailbox contains one pending frame: publishing a newer frame releases the stale pending frame. A successful acquisition transfers one retained reference into the opaque token. +3. After QRhi has opened the frame and provided its native command buffer, but **before** the Qt scene pass begins, call `opennow_streamer_record_frame`. Conversion and synchronization are encoded into that exact command stream. The function never creates, submits, commits, or waits for another command buffer. It returns one producer-owned RGBA8, RGB10A2, or RGBA16F GPU texture with explicit color-space metadata for the selected in-flight slot. +4. Import and sample that texture inside the item's render pass. Keep the token until QRhi has finished every GPU use of the slot, then call `opennow_streamer_release_frame`. A token records at most once and must be released exactly once. +5. At session-generation changes and scene-graph invalidation, call `QRhi::finish()` outside a render pass to submit and drain commands, release tokens and imported Qt texture wrappers, and drain Qt's deferred releases. Then call `opennow_streamer_scene_graph_shutdown` on the bound render thread before QRhi destroys its native objects. This explicitly retires the Linux producer's in-flight slots and imported resources even when the decoder/publisher still holds that producer. A session worker may clear the mailbox and stop decoding but never destroys the render-owned resource lease. Destroy rejects a still-active scene graph instead of dropping GPU state from the wrong thread. + +Replacing the graphics device or session renderer requires explicit shutdown first. Shutdown clears the one-frame mailbox and advances its epoch. Previously acquired tokens stay releasable but become stale and cannot record against the replacement context. Graphics calls are bound to the thread that installed the active context; a new scene graph can bind a different thread after shutdown. Only one session renderer is retained, bounding ownership across repeated session restarts. + +### Vulkan enabled capabilities (graphics context 3) + +Shutdown invalidates the context even if resource retirement returns `OPENNOW_STREAMER_RENDER_FAILED`. A lost Vulkan device can retire normally. If an idle wait fails for another reason, resources whose completion cannot be proven are deliberately abandoned instead of being destroyed by a later worker drop against a dead device; the error is returned to the host. + +`enabled_capabilities` is an explicit logical-device contract, not physical-device discovery. Set `OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT` only when the host created the device with external-memory, external-memory-fd, DMA-BUF, and DRM-modifier support enabled, including their prerequisites. Unknown bits and capabilities on non-Vulkan contexts are rejected; a zero mask disables DMA-BUF import while leaving explicitly CPU-backed NV12 presentation available. + +Graphics context version 3 adds the independent +`OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT` capability. Set it only +for Vulkan 1.1+ instances/devices with external-memory-fd, DMA-BUF, and +`VK_EXT_queue_family_foreign` enabled on the logical device. The Pi SAND path +requires this capability, queries external TRANSFER_SRC buffer import support, +and transfers ownership from/to the foreign decoder around its GPU copy. It does +not require SAND image-modifier sampling support. Setting the image-import bit +alone does not enable this path; zero remains a safe default for both bits. + +Qt requests the extensions through `QT_VULKAN_DEVICE_EXTENSIONS` before any window/device creation. Qt 6.8's Vulkan backend enables each requested extension that the selected physical device advertises. The host contract checks both that this startup request was installed and that every non-core extension is requested and supported on the selected device, with Vulkan 1.1 or newer on the instance and physical device. Vulkan 1.1 provides the external-memory, bind-memory, memory-requirements, sampler-YCbCr, and maintenance prerequisites; the request also includes their extension names. `VK_KHR_image_format_list` must be enabled unless both instance and physical device provide Vulkan 1.2. Support is not inferred from advertisement alone. This contract applies to Qt-created devices, not arbitrary adopted devices. + +Embedded Linux sessions reject FFmpeg's independent-device Vulkan decoder before opening it. Vulkan decode is available only through the ABI 5 shared owner described above. CUDA/NVDEC remains an explicit CPU-transfer backend, and downloaded/software frames carry CPU planes without foreign Vulkan metadata. No failed GPU import triggers readback in the embedded presenter. + +## Integration boundary + +The public C constructor creates the embedded engine used by Qt. GPU producers publish through the platform runtime's `GraphicsFramePublisher`; the FFI owns the mailbox, token epochs, thread checks, and C ownership boundary while platform decoders own native texture creation and same-command-stream conversion. + +There is no callback cancellation or timeout. A callback that never returns will eventually backpressure responses and will make destroy wait indefinitely. Dropped unsolicited events are not yet summarized with an overflow event. The header is handwritten and must be validated by each C/C++ consumer's compile-time layout assertions. diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/build.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/build.rs new file mode 100644 index 000000000..65313b1f0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/build.rs @@ -0,0 +1,15 @@ +fn main() { + match std::env::var("CARGO_CFG_TARGET_OS").as_deref() { + Ok("linux") => { + println!("cargo:rustc-link-arg=-Wl,-soname,libopennow_streamer_ffi.so"); + } + Ok("macos") => { + println!("cargo:rustc-link-arg=-Wl,-install_name,@rpath/libopennow_streamer_ffi.dylib"); + } + Ok("windows") if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("msvc") => { + println!("cargo:rustc-link-arg=/DELAYLOAD:mfplat.dll"); + println!("cargo:rustc-link-lib=delayimp"); + } + _ => {} + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/include/opennow_streamer_ffi.h b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/include/opennow_streamer_ffi.h new file mode 100644 index 000000000..48342773e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/include/opennow_streamer_ffi.h @@ -0,0 +1,415 @@ +#ifndef OPENNOW_STREAMER_FFI_H +#define OPENNOW_STREAMER_FFI_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define OPENNOW_STREAMER_FFI_ABI_VERSION 15u +#define OPENNOW_STREAMER_GENERIC_HID_VERSION 1u +#define OPENNOW_STREAMER_EVENT_SESSION_TIMER "session-timer" +#define OPENNOW_STREAMER_EVENT_TEXT_INPUT_ERROR "text-input-error" +#define OPENNOW_STREAMER_TERMINATION_SOURCE_SERVER "nvst-server" +#define OPENNOW_STREAMER_TIMER_IDLE_WARNING 4u +#define OPENNOW_STREAMER_TIMER_IDLE_EXPIRY 5u +#define OPENNOW_STREAMER_MAX_TEXT_BYTES 65536u +#define OPENNOW_STREAMER_VULKAN_DEVICE_INFO_VERSION 1u +#define OPENNOW_STREAMER_GRAPHICS_CONTEXT_VERSION 3u +#define OPENNOW_STREAMER_RENDER_COMMAND_VERSION 3u + +#define OPENNOW_STREAMER_GRAPHICS_API_D3D11 1u +#define OPENNOW_STREAMER_GRAPHICS_API_VULKAN 2u +#define OPENNOW_STREAMER_GRAPHICS_API_METAL 3u +#define OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT 1u +#define OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT 2u + +#define OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA8 1u +#define OPENNOW_STREAMER_TEXTURE_FORMAT_RGB10A2 2u +#define OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA16F 3u + +#define OPENNOW_STREAMER_COLOR_SPACE_SDR709 0u +#define OPENNOW_STREAMER_COLOR_SPACE_PQ2020 1u +#define OPENNOW_STREAMER_COLOR_SPACE_HLG2020 2u + +#define OPENNOW_STREAMER_LOCAL_ACTION_GUIDE 1u +#define OPENNOW_STREAMER_LOCAL_ACTION_SCREENSHOT 2u +#define OPENNOW_STREAMER_LOCAL_ACTION_RECORDING_TOGGLE 3u + +typedef struct OpenNowStreamer OpenNowStreamer; +typedef struct OpenNowStreamerFrame OpenNowStreamerFrame; +typedef struct OpenNowStreamerVulkanDevice OpenNowStreamerVulkanDevice; + +typedef struct OpenNowGenericHidInterface { + uint32_t version; + size_t struct_size; + uint8_t slot; + uint8_t interface_number; + uint16_t reserved; + uint64_t incarnation; + uint16_t vendor; + uint16_t product; + uint16_t revision; + uint16_t reserved2; + const uint8_t *descriptor; + size_t descriptor_len; +} OpenNowGenericHidInterface; + +typedef struct OpenNowGenericHidReport { + uint32_t version; + size_t struct_size; + uint8_t slot; + uint8_t interface_number; + uint8_t operation; + uint8_t error; + uint64_t incarnation; + const uint8_t *bytes; + size_t byte_count; +} OpenNowGenericHidReport; + +typedef struct OpenNowGenericHidOutput { + uint32_t version; + size_t struct_size; + uint8_t slot; + uint8_t interface_number; + uint8_t operation; + uint8_t reserved; + uint64_t incarnation; + size_t byte_count; + uint8_t bytes[256]; +} OpenNowGenericHidOutput; + +typedef enum OpenNowGenericHidAdmissionState { + OPENNOW_GENERIC_HID_PENDING = 0, + OPENNOW_GENERIC_HID_ADMITTED = 1, + OPENNOW_GENERIC_HID_DECLINED = 2 +} OpenNowGenericHidAdmissionState; + +typedef struct OpenNowGenericHidAdmission { + uint8_t slot; + uint8_t admission; + uint8_t reserved[6]; + uint64_t incarnation; +} OpenNowGenericHidAdmission; + +typedef struct OpenNowGenericHidAdmissions { + uint32_t version; + size_t struct_size; + size_t count; + OpenNowGenericHidAdmission devices[8]; +} OpenNowGenericHidAdmissions; + +typedef struct OpenNowStreamerVulkanDeviceInfo { + uint32_t version; + size_t struct_size; + void *instance; + void *physical_device; + void *device; + void *graphics_queue; + uint32_t graphics_queue_family_index; + uint32_t graphics_queue_index; + uint32_t api_version; +} OpenNowStreamerVulkanDeviceInfo; + +typedef void (*OpenNowStreamerCallback)( + const uint8_t *bytes, + size_t length, + void *user_data); + +typedef void (*OpenNowStreamerFrameAvailableCallback)(void *user_data); + +typedef struct OpenNowStreamerConfig { + uint32_t abi_version; + size_t struct_size; + size_t command_queue_capacity; + size_t response_queue_capacity; + size_t event_queue_capacity; + size_t max_command_bytes; + OpenNowStreamerCallback response_callback; + OpenNowStreamerCallback event_callback; + OpenNowStreamerFrameAvailableCallback frame_available_callback; + OpenNowStreamerCallback cursor_callback; + void *user_data; + const OpenNowStreamerVulkanDevice *vulkan_device; + /* Packed Windows LUID bits. Zero selects the system default adapter. */ + uint64_t windows_adapter_luid; +} OpenNowStreamerConfig; + +/* + * Borrowed native objects from one live QRhi. The shell retains ownership. + * + * D3D11: device is ID3D11Device and queue is its immediate ID3D11DeviceContext. + * Vulkan: instance, physical_device, device, queue, and queue_family_index are required. + * enabled_capabilities describes support enabled on this logical device, not advertised by its + * physical device. DMA-BUF import requires Vulkan 1.1 and enabled external-memory/fd/dma_buf/ + * drm_format_modifier extensions with their prerequisites. DMA-BUF buffer import instead requires + * Vulkan 1.1 and enabled external-memory-fd, dma_buf, and queue_family_foreign extensions. + * The image and buffer capability bits are independent. Unknown bits are rejected. + * Metal: device is id and queue is id. + */ +typedef struct OpenNowStreamerGraphicsContext { + uint32_t version; + size_t struct_size; + uint32_t graphics_api; + void *instance; + void *physical_device; + void *device; + void *queue; + uint32_t queue_family_index; + uint32_t enabled_capabilities; +} OpenNowStreamerGraphicsContext; + +/* + * One shell-owned in-flight slot and the current QRhi native command buffer. + * record_frame writes conversion/synchronization commands into this command stream; it never + * creates, commits, submits, or waits for another command buffer. + * Sharpness is 0..15 (default 10), denoise is 0..20 (default 0); both apply before MetalFX. + */ +typedef struct OpenNowStreamerRecordCommand { + uint32_t version; + size_t struct_size; + void *command_buffer; + uint32_t frame_slot; + uint32_t upscale_width; + uint32_t upscale_height; + uint32_t upscale_sharpness; + uint32_t upscale_denoise; +} OpenNowStreamerRecordCommand; + +typedef struct OpenNowStreamerFrameInfo { + uint32_t width; + uint32_t height; + uint64_t sequence; + uint64_t presentation_time_ns; +} OpenNowStreamerFrameInfo; + +/* + * One GPU texture populated by record_frame. texture_format identifies RGBA8, RGB10A2 or RGBA16F. + * color_space identifies encoded SDR Rec.709, PQ Rec.2020 or HLG Rec.2020 RGB; it is independent + * of texture precision. HDR is never published in RGBA8. On D3D + * and Metal, resource is the native texture pointer encoded as uint64_t and resource_view is zero. + * On Vulkan, resource is VkImage and resource_view is VkImageView. The producer owns both handles; + * the frame token retains their backing slot until release. + */ +typedef struct OpenNowStreamerRecordedFrame { + uint64_t resource; + uint64_t resource_view; + uint32_t graphics_api; + uint32_t texture_format; + uint32_t color_space; + uint32_t width; + uint32_t height; + uint32_t frame_slot; + uint64_t generation; + uint64_t presentation_time_ns; +} OpenNowStreamerRecordedFrame; + +#define OPENNOW_STREAMER_SDL_MAX_SOURCES 4u +#define OPENNOW_STREAMER_SONY_CONTACTS 2u +#define OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION 1u + +typedef struct OpenNowSdlDeviceClaim { + uint8_t slot; + uint8_t reserved[7]; + uint64_t incarnation; + uint16_t vendor; + uint16_t product; +} OpenNowSdlDeviceClaim; + +typedef struct OpenNowSonySnapshot { + uint32_t version; + size_t struct_size; + uint8_t slot; + uint8_t touchpad_click; + uint8_t reserved[2]; + uint64_t incarnation; + uint16_t buttons; + uint8_t left_trigger; + uint8_t right_trigger; + int16_t left_stick_x; + int16_t left_stick_y; + int16_t right_stick_x; + int16_t right_stick_y; + uint8_t contact_active[OPENNOW_STREAMER_SONY_CONTACTS]; + uint8_t reserved2[2]; + float contact_x[OPENNOW_STREAMER_SONY_CONTACTS]; + float contact_y[OPENNOW_STREAMER_SONY_CONTACTS]; + uint32_t reserved3; + uint64_t observed_at_us; +} OpenNowSonySnapshot; + +typedef enum OpenNowStreamerStatus { + OPENNOW_STREAMER_OK = 0, + OPENNOW_STREAMER_NULL_POINTER = 1, + OPENNOW_STREAMER_INVALID_CONFIG = 2, + OPENNOW_STREAMER_MESSAGE_TOO_LARGE = 3, + OPENNOW_STREAMER_QUEUE_FULL = 4, + OPENNOW_STREAMER_CLOSED = 5, + OPENNOW_STREAMER_NO_FRAME = 6, + OPENNOW_STREAMER_GRAPHICS_UNAVAILABLE = 7, + OPENNOW_STREAMER_WRONG_THREAD = 8, + OPENNOW_STREAMER_STALE_FRAME = 9, + OPENNOW_STREAMER_RENDER_FAILED = 10, + OPENNOW_STREAMER_SCENE_GRAPH_ACTIVE = 11, + OPENNOW_STREAMER_FRAME_ALREADY_RECORDED = 12, + OPENNOW_STREAMER_SONY_INACTIVE = 13, + OPENNOW_STREAMER_SDL_CLAIM_REJECTED = 14, + OPENNOW_STREAMER_GRAPHICS_UNSUPPORTED = 15, + OPENNOW_STREAMER_PANIC = 255 +} OpenNowStreamerStatus; + +OpenNowStreamerStatus opennow_streamer_create( + const OpenNowStreamerConfig *config, + OpenNowStreamer **output); + +OpenNowStreamerStatus opennow_streamer_vulkan_device_create( + OpenNowStreamerVulkanDevice **output); + +OpenNowStreamerStatus opennow_streamer_vulkan_device_info( + const OpenNowStreamerVulkanDevice *handle, + OpenNowStreamerVulkanDeviceInfo *output); + +OpenNowStreamerStatus opennow_streamer_vulkan_device_destroy( + OpenNowStreamerVulkanDevice *handle); + +/* Points the embedded file log at a UTF-8 path (rotating past 2 MiB). + * The Qt shell passes its diagnostics native-streamer.log here so packaged + * builds log the video pipeline without spawning the legacy child streamer. + * Additive since ABI v3; safe to call more than once. Never fails streaming. */ +OpenNowStreamerStatus opennow_streamer_set_log_file(const char *path); + +OpenNowStreamerStatus opennow_streamer_send( + const OpenNowStreamer *handle, + const uint8_t *bytes, + size_t length); + +OpenNowStreamerStatus opennow_streamer_submit_key( + const OpenNowStreamer *handle, + uint16_t virtual_key, + uint16_t modifiers, + bool pressed); + +/* Copies nonempty strict UTF-8 without NUL or a terminator (at most MAX_TEXT_BYTES). + * OK admits the entire paste locally, not a remote delivery acknowledgement. + * Requires active capture and negotiated session input. One paste may be in flight. + * Invalid/empty: INVALID_CONFIG; oversize: MESSAGE_TOO_LARGE; null: NULL_POINTER; + * inactive/unavailable: CLOSED; occupied/full capture queue: QUEUE_FULL. + * Pending text is discarded when capture or session input becomes unavailable. */ +OpenNowStreamerStatus opennow_streamer_submit_text( + const OpenNowStreamer *handle, + const uint8_t *text, + size_t text_len); + +OpenNowStreamerStatus opennow_streamer_submit_mouse_relative( + const OpenNowStreamer *handle, + int16_t delta_x, + int16_t delta_y); + +OpenNowStreamerStatus opennow_streamer_submit_mouse_absolute( + const OpenNowStreamer *handle, + uint16_t x, + uint16_t y, + uint16_t width, + uint16_t height); + +OpenNowStreamerStatus opennow_streamer_submit_mouse_button( + const OpenNowStreamer *handle, + uint8_t button, + bool pressed); + +OpenNowStreamerStatus opennow_streamer_submit_mouse_wheel( + const OpenNowStreamer *handle, + int16_t delta_x, + int16_t delta_y); + +OpenNowStreamerStatus opennow_streamer_submit_gamepad( + const OpenNowStreamer *handle, + uint8_t controller_id, + uint16_t bitmap, + uint16_t buttons, + uint8_t left_trigger, + uint8_t right_trigger, + int16_t left_stick_x, + int16_t left_stick_y, + int16_t right_stick_x, + int16_t right_stick_y); + +OpenNowStreamerStatus opennow_streamer_replace_sdl_device_claims( + const OpenNowStreamer *handle, + const OpenNowSdlDeviceClaim *claims, + size_t claim_count); + +OpenNowStreamerStatus opennow_streamer_submit_sony_snapshot( + const OpenNowStreamer *handle, + const OpenNowSonySnapshot *snapshot); + +OpenNowStreamerStatus opennow_streamer_replace_generic_hid_inventory( + const OpenNowStreamer *handle, + const OpenNowGenericHidInterface *interfaces, + size_t count); + +OpenNowStreamerStatus opennow_streamer_submit_generic_hid_report( + const OpenNowStreamer *handle, + const OpenNowGenericHidReport *report); + +OpenNowStreamerStatus opennow_streamer_take_generic_hid_output( + const OpenNowStreamer *handle, + OpenNowGenericHidOutput *output); + +OpenNowStreamerStatus opennow_streamer_get_generic_hid_admissions( + const OpenNowStreamer *handle, + OpenNowGenericHidAdmissions *admissions); + +OpenNowStreamerStatus opennow_streamer_submit_local_action( + const OpenNowStreamer *handle, + uint32_t action); + +OpenNowStreamerStatus opennow_streamer_set_capture_active( + const OpenNowStreamer *handle, + bool active, + bool relative_mouse, + uintptr_t window_handle, + bool *raw_input_active); + +OpenNowStreamerStatus opennow_streamer_set_graphics_context( + const OpenNowStreamer *handle, + const OpenNowStreamerGraphicsContext *context); + +OpenNowStreamerStatus opennow_streamer_acquire_latest_frame( + const OpenNowStreamer *handle, + OpenNowStreamerFrame **output, + OpenNowStreamerFrameInfo *info); + +/* + * Call after QRhi has opened its frame and supplied the native command buffer, but BEFORE + * QQuickRhiItem begins its render pass. The conversion may open its own offscreen pass or encode + * barriers, which is invalid inside the item's pass. After this returns, import/sample output's + * GPU texture in the QQuickRhiItem pass on the same command stream. + */ +OpenNowStreamerStatus opennow_streamer_record_frame( + const OpenNowStreamer *handle, + const OpenNowStreamerFrame *frame, + const OpenNowStreamerRecordCommand *command, + OpenNowStreamerRecordedFrame *output); + +OpenNowStreamerStatus opennow_streamer_release_frame(OpenNowStreamerFrame *frame); + +/* + * Call from the render thread on session-generation changes and sceneGraphInvalidated/ + * releaseResources, after submitting/draining native commands and releasing frame tokens and Qt + * imported wrappers. The borrowed device must still be live. RENDER_FAILED still invalidates the + * context; WRONG_THREAD does not. Retained decoder/publisher references may outlive this call. + */ +OpenNowStreamerStatus opennow_streamer_scene_graph_shutdown( + const OpenNowStreamer *handle); + +OpenNowStreamerStatus opennow_streamer_destroy(OpenNowStreamer *handle); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/src/lib.rs new file mode 100644 index 000000000..5eff44145 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-ffi/src/lib.rs @@ -0,0 +1,4230 @@ +use std::ffi::{CStr, c_char, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, SyncSender, TrySendError, sync_channel}; +use std::thread::{self, JoinHandle}; + +use opennow_streamer_core::{Engine, EventSender}; +use opennow_streamer_hid::generic::{ + GenericAdmission, GenericDevice, GenericError, GenericInterface, GenericReport, +}; +use opennow_streamer_hid::{ + HidRuntime, InventoryOutcome, SdlDeviceClaim, SnapshotAdmission, SonySnapshot, +}; +use opennow_streamer_platform::{ + CapturedInput, CapturedInputQueue, EmbeddedInputCapture, EmbeddedLocalAction, + EmbeddedRuntimeConfig, GraphicsApi, GraphicsContext, GraphicsFramePublisher, + GraphicsFrameToken, GraphicsRecordCommand, GraphicsRecordedFrame, GraphicsRuntimeError, + GraphicsTextureFormat, RenderThreadGraphics, SharedVulkanDevice, WindowsAdapterLuid, + create_embedded_runtime_with_config, +}; +use opennow_streamer_protocol::log; +use opennow_streamer_protocol::{Command, error}; +use serde_json::Value; + +static FIRST_FRAME_LOGGED: AtomicBool = AtomicBool::new(false); + +pub const OPENNOW_STREAMER_FFI_ABI_VERSION: u32 = 15; +pub const OPENNOW_STREAMER_MAX_TEXT_BYTES: usize = + opennow_streamer_protocol::text_input::MAX_TEXT_BYTES; +pub const OPENNOW_STREAMER_VULKAN_DEVICE_INFO_VERSION: u32 = 1; +const DEFAULT_MAX_COMMAND_BYTES: usize = 1024 * 1024; +const MAX_QUEUE_CAPACITY: usize = 4096; +const MAX_COMMAND_BYTES: usize = 16 * 1024 * 1024; + +pub type OpenNowStreamerCallback = + Option; +pub type OpenNowStreamerFrameAvailableCallback = + Option; +pub type OpenNowStreamerCursorCallback = + Option; + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowSdlDeviceClaim { + pub slot: u8, + pub reserved: [u8; 7], + pub incarnation: u64, + pub vendor: u16, + pub product: u16, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowSonySnapshot { + pub version: u32, + pub struct_size: usize, + pub slot: u8, + pub touchpad_click: u8, + pub reserved: [u8; 2], + pub incarnation: u64, + pub buttons: u16, + pub left_trigger: u8, + pub right_trigger: u8, + pub left_stick_x: i16, + pub left_stick_y: i16, + pub right_stick_x: i16, + pub right_stick_y: i16, + pub contact_active: [u8; 2], + pub reserved2: [u8; 2], + pub contact_x: [f32; 2], + pub contact_y: [f32; 2], + pub reserved3: u32, + pub observed_at_us: u64, +} + +pub const OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION: u32 = 1; + +pub const OPENNOW_STREAMER_GENERIC_HID_VERSION: u32 = 1; + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct OpenNowGenericHidAdmission { + pub slot: u8, + pub admission: u8, + pub reserved: [u8; 6], + pub incarnation: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowGenericHidAdmissions { + pub version: u32, + pub struct_size: usize, + pub count: usize, + pub devices: [OpenNowGenericHidAdmission; 8], +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowGenericHidInterface { + pub version: u32, + pub struct_size: usize, + pub slot: u8, + pub interface_number: u8, + pub reserved: u16, + pub incarnation: u64, + pub vendor: u16, + pub product: u16, + pub revision: u16, + pub reserved2: u16, + pub descriptor: *const u8, + pub descriptor_len: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowGenericHidReport { + pub version: u32, + pub struct_size: usize, + pub slot: u8, + pub interface_number: u8, + pub operation: u8, + pub error: u8, + pub incarnation: u64, + pub bytes: *const u8, + pub byte_count: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowGenericHidOutput { + pub version: u32, + pub struct_size: usize, + pub slot: u8, + pub interface_number: u8, + pub operation: u8, + pub reserved: u8, + pub incarnation: u64, + pub byte_count: usize, + pub bytes: [u8; 256], +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct OpenNowStreamerConfig { + pub abi_version: u32, + pub struct_size: usize, + pub command_queue_capacity: usize, + pub response_queue_capacity: usize, + pub event_queue_capacity: usize, + pub max_command_bytes: usize, + pub response_callback: OpenNowStreamerCallback, + pub event_callback: OpenNowStreamerCallback, + pub frame_available_callback: OpenNowStreamerFrameAvailableCallback, + pub cursor_callback: OpenNowStreamerCursorCallback, + pub user_data: *mut c_void, + pub vulkan_device: *const OpenNowStreamerVulkanDevice, + pub windows_adapter_luid: u64, +} + +pub struct OpenNowStreamerVulkanDevice { + device: Arc, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct OpenNowStreamerVulkanDeviceInfo { + pub version: u32, + pub struct_size: usize, + pub instance: *mut c_void, + pub physical_device: *mut c_void, + pub device: *mut c_void, + pub graphics_queue: *mut c_void, + pub graphics_queue_family_index: u32, + pub graphics_queue_index: u32, + pub api_version: u32, +} + +#[unsafe(no_mangle)] +/// Creates an owned device for adoption by the shell and embedded decoder. +/// +/// # Safety +/// `output` must point to writable storage for one handle pointer. +pub unsafe extern "C" fn opennow_streamer_vulkan_device_create( + output: *mut *mut OpenNowStreamerVulkanDevice, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if output.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + unsafe { output.write(ptr::null_mut()) }; + #[cfg(target_os = "linux")] + { + let device = match SharedVulkanDevice::create() { + Ok(device) => device, + Err(error) => { + use std::io::Write as _; + let reason: String = error + .to_string() + .chars() + .take(1536) + .map(|character| { + if character.is_control() { + ' ' + } else { + character + } + }) + .collect(); + log::log_line("WARN", "vulkan-device", &reason); + let _ = writeln!( + std::io::stderr().lock(), + "embedded Vulkan unavailable: {reason}" + ); + return OpenNowStreamerStatus::GraphicsUnavailable; + } + }; + unsafe { + output.write(Box::into_raw(Box::new(OpenNowStreamerVulkanDevice { + device, + }))) + }; + OpenNowStreamerStatus::Ok + } + #[cfg(not(target_os = "linux"))] + OpenNowStreamerStatus::GraphicsUnavailable + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Returns borrowed native objects retained by the device owner. +/// +/// # Safety +/// `handle` must be live and must not race with destroy. `output` must point to writable +/// storage for one complete info structure. Native objects must not outlive their owner. +pub unsafe extern "C" fn opennow_streamer_vulkan_device_info( + handle: *const OpenNowStreamerVulkanDevice, + output: *mut OpenNowStreamerVulkanDeviceInfo, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || output.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + #[cfg(target_os = "linux")] + { + let info = unsafe { &*handle }.device.info(); + unsafe { + output.write(OpenNowStreamerVulkanDeviceInfo { + version: OPENNOW_STREAMER_VULKAN_DEVICE_INFO_VERSION, + struct_size: size_of::(), + instance: info.instance as *mut c_void, + physical_device: info.physical_device as *mut c_void, + device: info.device as *mut c_void, + graphics_queue: info.graphics_queue as *mut c_void, + graphics_queue_family_index: info.graphics_queue_family_index, + graphics_queue_index: info.graphics_queue_index, + api_version: info.api_version, + }); + } + OpenNowStreamerStatus::Ok + } + #[cfg(not(target_os = "linux"))] + OpenNowStreamerStatus::GraphicsUnavailable + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Releases the device owner after the shell has stopped using its native objects. +/// +/// # Safety +/// `handle` must be a live handle returned by device creation and consumed exactly once, +/// without concurrent access. The shell must release all adopted graphics resources first. +pub unsafe extern "C" fn opennow_streamer_vulkan_device_destroy( + handle: *mut OpenNowStreamerVulkanDevice, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + unsafe { drop(Box::from_raw(handle)) }; + OpenNowStreamerStatus::Ok + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[repr(i32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpenNowStreamerStatus { + Ok = 0, + NullPointer = 1, + InvalidConfig = 2, + MessageTooLarge = 3, + QueueFull = 4, + Closed = 5, + NoFrame = 6, + GraphicsUnavailable = 7, + WrongThread = 8, + StaleFrame = 9, + RenderFailed = 10, + SceneGraphActive = 11, + FrameAlreadyRecorded = 12, + SonyInactive = 13, + SdlClaimRejected = 14, + GraphicsUnsupported = 15, + Panic = 255, +} + +pub const OPENNOW_STREAMER_GRAPHICS_CONTEXT_VERSION: u32 = 3; +pub const OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT: u32 = 1; +pub const OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT: u32 = 2; +pub const OPENNOW_STREAMER_RENDER_COMMAND_VERSION: u32 = 3; +pub const OPENNOW_STREAMER_GRAPHICS_API_D3D11: u32 = 1; +pub const OPENNOW_STREAMER_GRAPHICS_API_VULKAN: u32 = 2; +pub const OPENNOW_STREAMER_GRAPHICS_API_METAL: u32 = 3; +pub const OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA8: u32 = 1; +pub const OPENNOW_STREAMER_TEXTURE_FORMAT_RGB10A2: u32 = 2; +pub const OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA16F: u32 = 3; +pub const OPENNOW_STREAMER_COLOR_SPACE_SDR709: u32 = 0; +pub const OPENNOW_STREAMER_COLOR_SPACE_PQ2020: u32 = 1; +pub const OPENNOW_STREAMER_COLOR_SPACE_HLG2020: u32 = 2; +pub const OPENNOW_STREAMER_LOCAL_ACTION_GUIDE: u32 = 1; +pub const OPENNOW_STREAMER_LOCAL_ACTION_SCREENSHOT: u32 = 2; +pub const OPENNOW_STREAMER_LOCAL_ACTION_RECORDING_TOGGLE: u32 = 3; + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct OpenNowStreamerGraphicsContext { + pub version: u32, + pub struct_size: usize, + pub graphics_api: u32, + pub instance: *mut c_void, + pub physical_device: *mut c_void, + pub device: *mut c_void, + pub queue: *mut c_void, + pub queue_family_index: u32, + pub enabled_capabilities: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct OpenNowStreamerRecordCommand { + pub version: u32, + pub struct_size: usize, + pub command_buffer: *mut c_void, + pub frame_slot: u32, + pub upscale_width: u32, + pub upscale_height: u32, + pub upscale_sharpness: u32, + pub upscale_denoise: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OpenNowStreamerFrameInfo { + pub width: u32, + pub height: u32, + pub sequence: u64, + pub presentation_time_ns: u64, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OpenNowStreamerRecordedFrame { + pub resource: u64, + pub resource_view: u64, + pub graphics_api: u32, + pub texture_format: u32, + pub color_space: u32, + pub width: u32, + pub height: u32, + pub frame_slot: u32, + pub generation: u64, + pub presentation_time_ns: u64, +} + +pub struct OpenNowStreamerFrame { + token: GraphicsFrameToken, +} + +#[derive(Clone, Copy)] +struct Callback { + function: OpenNowStreamerCallback, + user_data: usize, +} + +#[derive(Clone, Copy)] +struct FrameAvailableCallback { + function: OpenNowStreamerFrameAvailableCallback, + user_data: usize, +} + +impl FrameAvailableCallback { + fn invoke(self) { + let Some(function) = self.function else { + return; + }; + unsafe { + function(self.user_data as *mut c_void); + } + } +} + +impl Callback { + fn invoke(self, value: &Value) { + let Some(function) = self.function else { + return; + }; + let Ok(bytes) = serde_json::to_vec(value) else { + return; + }; + unsafe { + function(bytes.as_ptr(), bytes.len(), self.user_data as *mut c_void); + } + } + + fn invoke_bytes(self, bytes: &[u8]) { + let Some(function) = self.function else { + return; + }; + unsafe { + function(bytes.as_ptr(), bytes.len(), self.user_data as *mut c_void); + } + } +} + +enum WorkerCommand { + Send(Vec), + Destroy, +} + +pub struct OpenNowStreamer { + commands: Option>, + worker: Option>, + response_dispatcher: Option>, + event_dispatcher: Option>, + cursor_dispatcher: Option>, + max_command_bytes: usize, + graphics: RenderThreadGraphics, + frame_publisher: GraphicsFramePublisher, + input: EmbeddedInputCapture, + hid: Arc, + #[cfg(target_os = "windows")] + windows_adapter_luid: Option, +} + +impl OpenNowStreamer { + fn create( + config: OpenNowStreamerConfig, + captured_input: Arc, + hid: Arc, + engine_factory: impl FnOnce(EventSender, GraphicsFramePublisher, SyncSender>) -> Engine + + Send + + 'static, + exit_hook: impl FnOnce() + Send + 'static, + ) -> Result { + validate_config(&config)?; + let callback = |function| Callback { + function, + user_data: config.user_data as usize, + }; + let frame_available = FrameAvailableCallback { + function: config.frame_available_callback, + user_data: config.user_data as usize, + }; + let (graphics, frame_publisher) = + RenderThreadGraphics::new(move || frame_available.invoke()); + let engine_frame_publisher = frame_publisher.clone(); + let (commands, command_receiver) = sync_channel(config.command_queue_capacity); + let (responses, response_receiver) = sync_channel(config.response_queue_capacity); + let (events, event_receiver) = sync_channel(config.event_queue_capacity); + let (cursor_updates, cursor_receiver) = sync_channel(config.event_queue_capacity); + let response_dispatcher = spawn_dispatcher( + "opennow-ffi-responses", + response_receiver, + callback(config.response_callback), + )?; + let event_dispatcher = match spawn_dispatcher( + "opennow-ffi-events", + event_receiver, + callback(config.event_callback), + ) { + Ok(dispatcher) => dispatcher, + Err(status) => { + drop(responses); + let _ = response_dispatcher.join(); + return Err(status); + } + }; + let cursor_dispatcher = + match spawn_cursor_dispatcher(cursor_receiver, callback(config.cursor_callback)) { + Ok(dispatcher) => dispatcher, + Err(status) => { + drop(responses); + drop(events); + let _ = response_dispatcher.join(); + let _ = event_dispatcher.join(); + return Err(status); + } + }; + let worker = match thread::Builder::new() + .name("opennow-ffi-engine".to_owned()) + .spawn(move || { + let _ = catch_unwind(AssertUnwindSafe(|| { + run_engine( + command_receiver, + responses, + EventSender::bounded(events), + move |events| { + engine_factory(events, engine_frame_publisher, cursor_updates) + }, + ); + })); + exit_hook(); + }) { + Ok(worker) => worker, + Err(_) => { + drop(commands); + let _ = response_dispatcher.join(); + let _ = event_dispatcher.join(); + let _ = cursor_dispatcher.join(); + return Err(OpenNowStreamerStatus::Closed); + } + }; + Ok(Self { + commands: Some(commands), + worker: Some(worker), + response_dispatcher: Some(response_dispatcher), + event_dispatcher: Some(event_dispatcher), + cursor_dispatcher: Some(cursor_dispatcher), + max_command_bytes: if config.max_command_bytes == 0 { + DEFAULT_MAX_COMMAND_BYTES + } else { + config.max_command_bytes + }, + graphics, + frame_publisher, + input: EmbeddedInputCapture::new(captured_input), + hid, + #[cfg(target_os = "windows")] + windows_adapter_luid: WindowsAdapterLuid::new(config.windows_adapter_luid), + }) + } + + fn send(&self, bytes: &[u8]) -> OpenNowStreamerStatus { + if bytes.len() > self.max_command_bytes { + return OpenNowStreamerStatus::MessageTooLarge; + } + let Some(commands) = self.commands.as_ref() else { + return OpenNowStreamerStatus::Closed; + }; + match commands.try_send(WorkerCommand::Send(bytes.to_vec())) { + Ok(()) => OpenNowStreamerStatus::Ok, + Err(TrySendError::Full(_)) => OpenNowStreamerStatus::QueueFull, + Err(TrySendError::Disconnected(_)) => OpenNowStreamerStatus::Closed, + } + } + + fn shutdown(&mut self) { + if let Some(commands) = self.commands.take() { + let _ = commands.send(WorkerCommand::Destroy); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + if let Some(dispatcher) = self.response_dispatcher.take() { + let _ = dispatcher.join(); + } + if let Some(dispatcher) = self.event_dispatcher.take() { + let _ = dispatcher.join(); + } + if let Some(dispatcher) = self.cursor_dispatcher.take() { + let _ = dispatcher.join(); + } + } + + #[cfg(test)] + fn frame_publisher(&self) -> GraphicsFramePublisher { + self.frame_publisher.clone() + } +} + +impl Drop for OpenNowStreamer { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn validate_config(config: &OpenNowStreamerConfig) -> Result<(), OpenNowStreamerStatus> { + let valid_capacity = |capacity| (1..=MAX_QUEUE_CAPACITY).contains(&capacity); + if config.abi_version != OPENNOW_STREAMER_FFI_ABI_VERSION + || config.struct_size < size_of::() + || !valid_capacity(config.command_queue_capacity) + || !valid_capacity(config.response_queue_capacity) + || !valid_capacity(config.event_queue_capacity) + || config.max_command_bytes > MAX_COMMAND_BYTES + || config.response_callback.is_none() + { + return Err(OpenNowStreamerStatus::InvalidConfig); + } + Ok(()) +} + +#[cfg(any(target_os = "windows", test))] +fn validate_windows_adapter_luid( + configured: WindowsAdapterLuid, + actual: WindowsAdapterLuid, +) -> Result<(), String> { + if configured == actual { + return Ok(()); + } + Err(format!( + "configured Windows adapter LUID {:#018x} does not match Qt D3D11 device LUID {:#018x}", + configured.get(), + actual.get(), + )) +} + +fn spawn_dispatcher( + name: &str, + receiver: Receiver, + callback: Callback, +) -> Result, OpenNowStreamerStatus> { + thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let mut last_telemetry = std::time::Instant::now(); + while let Ok(value) = receiver.recv() { + let kind = value["type"].as_str().unwrap_or(""); + if should_log_event(kind, &mut last_telemetry, std::time::Instant::now()) { + log::log_line( + match (kind, value["level"].as_str()) { + ("error", _) | (_, Some("error")) => "ERROR", + (_, Some("warn")) => "WARN", + (_, Some("debug")) => "DEBUG", + _ => "INFO", + }, + "native-to-qt", + &log::message_summary(&value), + ); + } + callback.invoke(&value); + } + }) + .map_err(|_| OpenNowStreamerStatus::Closed) +} + +fn should_log_event( + kind: &str, + last_telemetry: &mut std::time::Instant, + now: std::time::Instant, +) -> bool { + if kind == "controller-rumble" { + return false; + } + if !matches!(kind, "telemetry" | "stats") { + return true; + } + if now.duration_since(*last_telemetry) < std::time::Duration::from_secs(10) { + return false; + } + *last_telemetry = now; + true +} + +fn spawn_cursor_dispatcher( + receiver: Receiver>, + callback: Callback, +) -> Result, OpenNowStreamerStatus> { + thread::Builder::new() + .name("opennow-ffi-cursor".to_owned()) + .spawn(move || { + while let Ok(bytes) = receiver.recv() { + callback.invoke_bytes(&bytes); + } + }) + .map_err(|_| OpenNowStreamerStatus::Closed) +} + +fn run_engine( + commands: Receiver, + responses: SyncSender, + events: EventSender, + engine_factory: impl FnOnce(EventSender) -> Engine, +) { + let mut engine = engine_factory(events); + while let Ok(command) = commands.recv() { + let bytes = match command { + WorkerCommand::Send(bytes) => bytes, + WorkerCommand::Destroy => break, + }; + let command: Command = match serde_json::from_slice(&bytes) { + Ok(command) => command, + Err(parse_error) => { + if responses + .send(error(None, "invalid-command", parse_error.to_string())) + .is_err() + { + break; + } + continue; + } + }; + let (messages, keep_running) = engine.handle(command); + for message in messages { + if responses.send(message).is_err() { + return; + } + } + if !keep_running { + break; + } + } +} + +fn graphics_context( + context: OpenNowStreamerGraphicsContext, +) -> Result { + if context.version != OPENNOW_STREAMER_GRAPHICS_CONTEXT_VERSION + || context.struct_size < size_of::() + { + return Err(OpenNowStreamerStatus::InvalidConfig); + } + let api = match context.graphics_api { + OPENNOW_STREAMER_GRAPHICS_API_D3D11 => GraphicsApi::D3d11, + OPENNOW_STREAMER_GRAPHICS_API_VULKAN => GraphicsApi::Vulkan, + OPENNOW_STREAMER_GRAPHICS_API_METAL => GraphicsApi::Metal, + _ => return Err(OpenNowStreamerStatus::InvalidConfig), + }; + let supported_capabilities = OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT + | OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT; + if context.enabled_capabilities & !supported_capabilities != 0 + || (api != GraphicsApi::Vulkan && context.enabled_capabilities != 0) + { + return Err(OpenNowStreamerStatus::InvalidConfig); + } + Ok(GraphicsContext { + api, + instance: context.instance as usize, + physical_device: context.physical_device as usize, + device: context.device as usize, + queue: context.queue as usize, + queue_family_index: context.queue_family_index, + vulkan_dmabuf_import_enabled: context.enabled_capabilities + & OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT + != 0, + vulkan_dmabuf_buffer_import_enabled: context.enabled_capabilities + & OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT + != 0, + }) +} + +fn render_command( + command: OpenNowStreamerRecordCommand, +) -> Result { + if command.version != OPENNOW_STREAMER_RENDER_COMMAND_VERSION + || command.struct_size < size_of::() + || command.upscale_sharpness > 15 + || command.upscale_denoise > 20 + { + return Err(OpenNowStreamerStatus::InvalidConfig); + } + Ok(GraphicsRecordCommand { + command_buffer: command.command_buffer as usize, + frame_slot: command.frame_slot, + upscale_width: command.upscale_width, + upscale_height: command.upscale_height, + upscale_sharpness: command.upscale_sharpness, + upscale_denoise: command.upscale_denoise, + }) +} + +fn recorded_frame( + api: GraphicsApi, + frame: GraphicsRecordedFrame, +) -> Result { + if frame.resource == 0 || frame.width == 0 || frame.height == 0 { + return Err(OpenNowStreamerStatus::RenderFailed); + } + if frame.color_space != opennow_streamer_platform::GraphicsColorSpace::Sdr709 + && frame.texture_format == GraphicsTextureFormat::Rgba8 + { + return Err(OpenNowStreamerStatus::RenderFailed); + } + Ok(OpenNowStreamerRecordedFrame { + resource: frame.resource, + resource_view: frame.resource_view, + graphics_api: match api { + GraphicsApi::D3d11 => OPENNOW_STREAMER_GRAPHICS_API_D3D11, + GraphicsApi::Vulkan => OPENNOW_STREAMER_GRAPHICS_API_VULKAN, + GraphicsApi::Metal => OPENNOW_STREAMER_GRAPHICS_API_METAL, + }, + texture_format: match frame.texture_format { + GraphicsTextureFormat::Rgba8 => OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA8, + GraphicsTextureFormat::Rgb10A2 => OPENNOW_STREAMER_TEXTURE_FORMAT_RGB10A2, + GraphicsTextureFormat::Rgba16Float => OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA16F, + }, + color_space: match frame.color_space { + opennow_streamer_platform::GraphicsColorSpace::Sdr709 => { + OPENNOW_STREAMER_COLOR_SPACE_SDR709 + } + opennow_streamer_platform::GraphicsColorSpace::Pq2020 => { + OPENNOW_STREAMER_COLOR_SPACE_PQ2020 + } + opennow_streamer_platform::GraphicsColorSpace::Hlg2020 => { + OPENNOW_STREAMER_COLOR_SPACE_HLG2020 + } + }, + width: frame.width, + height: frame.height, + frame_slot: frame.frame_slot, + generation: frame.generation, + presentation_time_ns: frame.presentation_time_ns, + }) +} + +fn graphics_status(error: GraphicsRuntimeError) -> OpenNowStreamerStatus { + match error { + GraphicsRuntimeError::InvalidContext(_) | GraphicsRuntimeError::InvalidRenderCommand(_) => { + OpenNowStreamerStatus::InvalidConfig + } + GraphicsRuntimeError::SceneGraphInactive => OpenNowStreamerStatus::GraphicsUnavailable, + GraphicsRuntimeError::WrongThread => OpenNowStreamerStatus::WrongThread, + GraphicsRuntimeError::NoFrame => OpenNowStreamerStatus::NoFrame, + GraphicsRuntimeError::StaleFrame => OpenNowStreamerStatus::StaleFrame, + GraphicsRuntimeError::FrameAlreadyRecorded => OpenNowStreamerStatus::FrameAlreadyRecorded, + GraphicsRuntimeError::RecordUnsupported(_) => OpenNowStreamerStatus::GraphicsUnsupported, + GraphicsRuntimeError::RecordFailed(_) => OpenNowStreamerStatus::RenderFailed, + } +} + +fn graphics_api_name(api: u32) -> &'static str { + match api { + OPENNOW_STREAMER_GRAPHICS_API_D3D11 => "d3d11", + OPENNOW_STREAMER_GRAPHICS_API_VULKAN => "vulkan", + OPENNOW_STREAMER_GRAPHICS_API_METAL => "metal", + _ => "unknown", + } +} + +/// Best-effort command-type label for the log. Payloads may carry session +/// secrets, so only the `type` discriminator is ever extracted. +fn command_kind_label(bytes: &[u8]) -> String { + serde_json::from_slice::(bytes) + .ok() + .and_then(|value| value.get("type")?.as_str().map(str::to_owned)) + .map(|kind| kind.chars().take(64).collect()) + .unwrap_or_else(|| "".to_owned()) +} + +fn ffi_status(body: impl FnOnce() -> OpenNowStreamerStatus) -> OpenNowStreamerStatus { + catch_unwind(AssertUnwindSafe(body)).unwrap_or(OpenNowStreamerStatus::Panic) +} + +#[unsafe(no_mangle)] +/// Points the embedded file log at `path` (UTF-8, NUL-terminated), creating +/// parent directories and rotating a previous log over 2 MiB to +/// `.previous`. The Qt shell passes its diagnostics +/// `native-streamer.log` here at startup so packaged builds — which never +/// spawn the legacy child streamer — still produce video-pipeline logs. +/// Safe to call more than once; logging never affects streaming results. +/// +/// # Safety +/// +/// `path` must point to readable NUL-terminated bytes for this call. +pub unsafe extern "C" fn opennow_streamer_set_log_file( + path: *const c_char, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if path.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let path = unsafe { CStr::from_ptr(path) }; + let path = match path.to_str() { + Ok(path) => path, + Err(_) => return OpenNowStreamerStatus::InvalidConfig, + }; + match log::set_log_file(path) { + Ok(()) => { + log::log_line("INFO", "engine", "file log configured"); + OpenNowStreamerStatus::Ok + } + Err(reason) => { + log::log_line("WARN", "engine", &format!("file log unavailable: {reason}")); + OpenNowStreamerStatus::InvalidConfig + } + } + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Creates one engine handle owned by the caller. +/// +/// # Safety +/// +/// `config` must point to a readable configuration whose first `struct_size` bytes remain valid +/// for this call. `output` must point to writable storage for one handle pointer. Callback pointers +/// and `user_data` must remain valid until destroy returns. A non-null `vulkan_device` must be +/// live throughout this call; the engine retains its own reference before the worker starts. +pub unsafe extern "C" fn opennow_streamer_create( + config: *const OpenNowStreamerConfig, + output: *mut *mut OpenNowStreamer, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if config.is_null() || output.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + unsafe { + output.write(ptr::null_mut()); + } + let abi_version = unsafe { ptr::addr_of!((*config).abi_version).read() }; + let struct_size = unsafe { ptr::addr_of!((*config).struct_size).read() }; + if abi_version != OPENNOW_STREAMER_FFI_ABI_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let config = unsafe { config.read() }; + let vulkan_device = + unsafe { config.vulkan_device.as_ref() }.map(|owner| Arc::clone(&owner.device)); + let windows_adapter_luid = WindowsAdapterLuid::new(config.windows_adapter_luid); + let captured_input = Arc::new(CapturedInputQueue::default()); + let runtime_input = Arc::clone(&captured_input); + let hid = Arc::new(HidRuntime::new()); + let engine_hid = Arc::clone(&hid); + match OpenNowStreamer::create( + config, + captured_input, + Arc::clone(&hid), + move |events, frames, cursor_updates| { + let cursor_update = Arc::new(move |bytes: Vec| { + let _ = cursor_updates.try_send(bytes); + }); + Engine::with_embedded_media_runtime_and_hid( + events, + create_embedded_runtime_with_config( + frames, + runtime_input, + Some(cursor_update), + EmbeddedRuntimeConfig { + vulkan_device, + windows_adapter_luid, + }, + ), + engine_hid, + ) + }, + || {}, + ) { + Ok(handle) => { + unsafe { + output.write(Box::into_raw(Box::new(handle))); + } + log::log_line( + "INFO", + "engine", + &format!( + "engine created (abi={abi_version} cmdq={} rspq={} evtq={} max_bytes={})", + config.command_queue_capacity, + config.response_queue_capacity, + config.event_queue_capacity, + config.max_command_bytes, + ), + ); + OpenNowStreamerStatus::Ok + } + Err(status) => { + log::log_line( + "WARN", + "engine", + &format!("engine create failed: {status:?}"), + ); + status + } + } + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Atomically admits one nonempty UTF-8 paste to the bounded captured input queue. +/// Success acknowledges local admission, not remote delivery. +/// +/// # Safety +/// +/// `handle` must be null or point to a live handle not being destroyed. +/// `text` must be null or readable for `text_len` bytes for the duration of the call. +pub unsafe extern "C" fn opennow_streamer_submit_text( + handle: *const OpenNowStreamer, + text: *const u8, + text_len: usize, +) -> OpenNowStreamerStatus { + ffi_status(|| { + use opennow_streamer_protocol::text_input::TextInputError; + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if text_len > OPENNOW_STREAMER_MAX_TEXT_BYTES { + return OpenNowStreamerStatus::MessageTooLarge; + } + if text.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + if text_len == 0 { + return OpenNowStreamerStatus::InvalidConfig; + } + let bytes = unsafe { std::slice::from_raw_parts(text, text_len) }; + if bytes.contains(&0) || std::str::from_utf8(bytes).is_err() { + return OpenNowStreamerStatus::InvalidConfig; + } + match handle.input.submit_text(bytes) { + Ok(()) => OpenNowStreamerStatus::Ok, + Err(TextInputError::Invalid) => OpenNowStreamerStatus::InvalidConfig, + Err(TextInputError::TooLarge) => OpenNowStreamerStatus::MessageTooLarge, + Err(TextInputError::Busy) => OpenNowStreamerStatus::QueueFull, + Err(TextInputError::Unavailable) => OpenNowStreamerStatus::Closed, + } + }) +} + +#[unsafe(no_mangle)] +/// Submits one keyboard transition to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_key( + handle: *const OpenNowStreamer, + virtual_key: u16, + modifiers: u16, + pressed: bool, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + handle.input.submit(CapturedInput::Key { + virtual_key, + modifiers, + pressed, + }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Submits one relative mouse movement to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_mouse_relative( + handle: *const OpenNowStreamer, + delta_x: i16, + delta_y: i16, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + handle + .input + .submit(CapturedInput::MouseMove { delta_x, delta_y }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Submits one absolute mouse position to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_mouse_absolute( + handle: *const OpenNowStreamer, + x: u16, + y: u16, + width: u16, + height: u16, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if width == 0 || height == 0 { + return OpenNowStreamerStatus::InvalidConfig; + } + handle.input.submit(CapturedInput::MouseAbsolute { + x, + y, + width, + height, + }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Submits one mouse-button transition to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_mouse_button( + handle: *const OpenNowStreamer, + button: u8, + pressed: bool, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if !(1..=5).contains(&button) { + return OpenNowStreamerStatus::InvalidConfig; + } + handle + .input + .submit(CapturedInput::MouseButton { button, pressed }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Submits one mouse-wheel movement to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_mouse_wheel( + handle: *const OpenNowStreamer, + delta_x: i16, + delta_y: i16, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + handle + .input + .submit(CapturedInput::MouseWheel { delta_x, delta_y }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Submits one gamepad state snapshot to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_gamepad( + handle: *const OpenNowStreamer, + controller_id: u8, + bitmap: u16, + buttons: u16, + left_trigger: u8, + right_trigger: u8, + left_stick_x: i16, + left_stick_y: i16, + right_stick_x: i16, + right_stick_y: i16, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if controller_id >= 4 { + return OpenNowStreamerStatus::InvalidConfig; + } + handle.input.submit(CapturedInput::Gamepad { + controller_id, + bitmap, + buttons, + left_trigger, + right_trigger, + left_stick_x, + left_stick_y, + right_stick_x, + right_stick_y, + }); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn opennow_streamer_replace_sdl_device_claims( + handle: *const OpenNowStreamer, + claims: *const OpenNowSdlDeviceClaim, + claim_count: usize, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if claim_count > 4 { + return OpenNowStreamerStatus::InvalidConfig; + } + if claim_count > 0 && claims.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let mut entries = [None; 4]; + for (index, entry) in entries.iter_mut().enumerate().take(claim_count) { + let claim = unsafe { claims.add(index).read() }; + let Some(claim) = + SdlDeviceClaim::new(claim.slot, claim.incarnation, claim.vendor, claim.product) + else { + return OpenNowStreamerStatus::SdlClaimRejected; + }; + *entry = Some(claim); + } + match handle.hid.replace_inventory(&entries).outcome { + InventoryOutcome::Replaced => OpenNowStreamerStatus::Ok, + InventoryOutcome::DuplicateSlot + | InventoryOutcome::DuplicateIncarnation + | InventoryOutcome::MalformedEntry => OpenNowStreamerStatus::SdlClaimRejected, + } + }) +} + +fn generic_hid_status(result: Result<(), GenericError>) -> OpenNowStreamerStatus { + match result { + Ok(()) => OpenNowStreamerStatus::Ok, + Err(GenericError::StaleSource | GenericError::Unbound | GenericError::Inactive) => { + OpenNowStreamerStatus::Closed + } + Err(GenericError::Overflow | GenericError::Faulted) => OpenNowStreamerStatus::QueueFull, + Err(GenericError::DescriptorTooLarge) => OpenNowStreamerStatus::MessageTooLarge, + Err(_) => OpenNowStreamerStatus::InvalidConfig, + } +} + +#[unsafe(no_mangle)] +/// Atomically replaces generic HID inventory, copying all descriptors before returning. +/// +/// # Safety +/// +/// `handle` must be null or a live handle not being destroyed. `interfaces` must be null +/// or point to `count` readable entries, each with a readable version and size header. +/// Entries with a valid header must be fully readable and their nonempty descriptors +/// must point to `descriptor_len` readable bytes for the duration of this call. +pub unsafe extern "C" fn opennow_streamer_replace_generic_hid_inventory( + handle: *const OpenNowStreamer, + interfaces: *const OpenNowGenericHidInterface, + count: usize, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if count > 128 { + return OpenNowStreamerStatus::InvalidConfig; + } + if count > 0 && interfaces.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let mut devices: Vec = Vec::with_capacity(8); + for index in 0..count { + let entry = unsafe { interfaces.add(index) }; + let version = unsafe { ptr::addr_of!((*entry).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*entry).struct_size).read() }; + if version != OPENNOW_STREAMER_GENERIC_HID_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let entry = unsafe { entry.read() }; + if entry.slot >= 8 + || entry.incarnation == 0 + || entry.reserved != 0 + || entry.reserved2 != 0 + { + return OpenNowStreamerStatus::InvalidConfig; + } + if entry.descriptor_len > 4096 { + return OpenNowStreamerStatus::MessageTooLarge; + } + if entry.descriptor_len > 0 && entry.descriptor.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let device = + if let Some(device) = devices.iter_mut().find(|device| device.slot == entry.slot) { + if device.incarnation != entry.incarnation + || device.vendor != entry.vendor + || device.product != entry.product + || device.revision != entry.revision + { + return OpenNowStreamerStatus::InvalidConfig; + } + device + } else { + devices.push(GenericDevice { + slot: entry.slot, + incarnation: entry.incarnation, + vendor: entry.vendor, + product: entry.product, + revision: entry.revision, + interfaces: Vec::new(), + }); + devices.last_mut().expect("inserted device") + }; + if device.interfaces.len() >= 16 { + return OpenNowStreamerStatus::InvalidConfig; + } + let descriptor = if entry.descriptor_len == 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(entry.descriptor, entry.descriptor_len) } + .to_vec() + }; + device.interfaces.push(GenericInterface { + number: entry.interface_number, + descriptor, + }); + } + generic_hid_status(handle.hid.generic.replace_inventory(devices)) + }) +} + +#[unsafe(no_mangle)] +/// Copies one bounded generic HID report into the runtime queue. +/// +/// # Safety +/// +/// `handle` must be null or a live handle not being destroyed. `report` must be null +/// or have a readable version and size header. A valid header requires a fully +/// readable report and `byte_count` readable bytes when the payload is nonempty. +pub unsafe extern "C" fn opennow_streamer_submit_generic_hid_report( + handle: *const OpenNowStreamer, + report: *const OpenNowGenericHidReport, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if report.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let version = unsafe { ptr::addr_of!((*report).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*report).struct_size).read() }; + if version != OPENNOW_STREAMER_GENERIC_HID_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let report = unsafe { report.read() }; + if report.slot >= 8 || report.incarnation == 0 || report.error > 1 || report.operation > 4 { + return OpenNowStreamerStatus::InvalidConfig; + } + if report.byte_count > 256 { + return OpenNowStreamerStatus::MessageTooLarge; + } + if report.byte_count > 0 && report.bytes.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let bytes = if report.byte_count == 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(report.bytes, report.byte_count) }.to_vec() + }; + generic_hid_status(handle.hid.generic.submit_report(GenericReport { + slot: report.slot, + incarnation: report.incarnation, + interface: report.interface_number, + operation: report.operation, + error: report.error == 1, + bytes, + })) + }) +} + +#[unsafe(no_mangle)] +/// Takes one pending generic HID output command, or returns Closed if none is pending. +/// +/// # Safety +/// +/// `handle` must be null or a live handle not being destroyed. `output` must be null +/// or have a readable version and size header. A valid header requires writable +/// storage for one complete `OpenNowGenericHidOutput` for the duration of this call. +pub unsafe extern "C" fn opennow_streamer_take_generic_hid_output( + handle: *const OpenNowStreamer, + output: *mut OpenNowGenericHidOutput, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if output.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let version = unsafe { ptr::addr_of!((*output).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*output).struct_size).read() }; + if version != OPENNOW_STREAMER_GENERIC_HID_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let Some(command) = handle.hid.generic.take_output() else { + return OpenNowStreamerStatus::Closed; + }; + let mut result = OpenNowGenericHidOutput { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + slot: command.slot, + interface_number: command.interface, + operation: command.operation, + reserved: 0, + incarnation: command.incarnation, + byte_count: command.bytes.len(), + bytes: [0; 256], + }; + result.bytes[..command.bytes.len()].copy_from_slice(&command.bytes); + unsafe { output.write(result) }; + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// # Safety +/// `handle` must be null or live and not being destroyed. `admissions` must be null +/// or have a readable version/size header and writable storage for the complete snapshot. +pub unsafe extern "C" fn opennow_streamer_get_generic_hid_admissions( + handle: *const OpenNowStreamer, + admissions: *mut OpenNowGenericHidAdmissions, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if admissions.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let version = unsafe { ptr::addr_of!((*admissions).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*admissions).struct_size).read() }; + if version != OPENNOW_STREAMER_GENERIC_HID_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let snapshot = handle.hid.generic.admissions_snapshot(); + let mut result = OpenNowGenericHidAdmissions { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + count: snapshot.len(), + devices: [OpenNowGenericHidAdmission::default(); 8], + }; + for (target, source) in result.devices.iter_mut().zip(snapshot) { + *target = OpenNowGenericHidAdmission { + slot: source.slot, + incarnation: source.incarnation, + admission: match source.admission { + GenericAdmission::Pending => 0, + GenericAdmission::Admitted => 1, + GenericAdmission::Declined => 2, + }, + reserved: [0; 6], + }; + } + unsafe { admissions.write(result) }; + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn opennow_streamer_submit_sony_snapshot( + handle: *const OpenNowStreamer, + snapshot: *const OpenNowSonySnapshot, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + let Some(raw) = (unsafe { snapshot.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + if raw.version != OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION + || raw.struct_size < size_of::() + || raw.reserved != [0; 2] + || raw.reserved2 != [0; 2] + || raw.reserved3 != 0 + { + return OpenNowStreamerStatus::InvalidConfig; + } + if raw.touchpad_click > 1 || raw.contact_active.iter().any(|flag| *flag > 1) { + return OpenNowStreamerStatus::InvalidConfig; + } + if raw + .contact_x + .iter() + .chain(raw.contact_y.iter()) + .any(|coordinate| !coordinate.is_finite()) + { + return OpenNowStreamerStatus::InvalidConfig; + } + let mut snapshot = SonySnapshot::neutral(raw.slot, raw.incarnation, raw.observed_at_us); + snapshot.buttons = raw.buttons; + snapshot.left_trigger = raw.left_trigger; + snapshot.right_trigger = raw.right_trigger; + snapshot.left_stick_x = raw.left_stick_x; + snapshot.left_stick_y = raw.left_stick_y; + snapshot.right_stick_x = raw.right_stick_x; + snapshot.right_stick_y = raw.right_stick_y; + snapshot.touchpad_click = raw.touchpad_click == 1; + for index in 0..2 { + snapshot.contacts[index] = opennow_streamer_hid::SonyContact { + active: raw.contact_active[index] == 1, + x: opennow_streamer_hid::ds4::normalize_touch_axis(raw.contact_x[index]), + y: opennow_streamer_hid::ds4::normalize_touch_axis(raw.contact_y[index]), + }; + } + match handle + .hid + .submit_snapshot_with(snapshot, |snapshot, bitmap| { + handle.input.submit(CapturedInput::Gamepad { + controller_id: snapshot.slot, + bitmap, + buttons: snapshot.buttons, + left_trigger: snapshot.left_trigger, + right_trigger: snapshot.right_trigger, + left_stick_x: snapshot.left_stick_x, + left_stick_y: snapshot.left_stick_y.saturating_neg(), + right_stick_x: snapshot.right_stick_x, + right_stick_y: snapshot.right_stick_y.saturating_neg(), + }); + }) { + SnapshotAdmission::Admitted => OpenNowStreamerStatus::Ok, + SnapshotAdmission::AdmittedInactive => OpenNowStreamerStatus::SonyInactive, + SnapshotAdmission::Unbound | SnapshotAdmission::StaleSource => { + OpenNowStreamerStatus::Closed + } + SnapshotAdmission::Faulted | SnapshotAdmission::Overflow => { + OpenNowStreamerStatus::QueueFull + } + } + }) +} + +#[unsafe(no_mangle)] +/// Submits one shell-local action to the embedded input queue. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +pub unsafe extern "C" fn opennow_streamer_submit_local_action( + handle: *const OpenNowStreamer, + action: u32, +) -> OpenNowStreamerStatus { + ffi_status(|| { + let Some(handle) = (unsafe { handle.as_ref() }) else { + return OpenNowStreamerStatus::NullPointer; + }; + let action = match action { + OPENNOW_STREAMER_LOCAL_ACTION_GUIDE => EmbeddedLocalAction::Guide, + OPENNOW_STREAMER_LOCAL_ACTION_SCREENSHOT => EmbeddedLocalAction::Screenshot, + OPENNOW_STREAMER_LOCAL_ACTION_RECORDING_TOGGLE => EmbeddedLocalAction::RecordingToggle, + _ => return OpenNowStreamerStatus::InvalidConfig, + }; + handle.input.submit_local_action(action); + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Changes native input capture state for the Qt window. +/// +/// # Safety +/// +/// `handle` must be null or point to a live engine handle that is not being destroyed. +/// `raw_input_active` must be null or point to writable storage for one `bool`. +pub unsafe extern "C" fn opennow_streamer_set_capture_active( + handle: *const OpenNowStreamer, + active: bool, + relative_mouse: bool, + window_handle: usize, + raw_input_active: *mut bool, +) -> OpenNowStreamerStatus { + ffi_status(|| { + if handle.is_null() || raw_input_active.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let handle = unsafe { &*handle }; + handle.hid.set_active(active); + let raw = handle + .input + .set_active(active, relative_mouse, window_handle); + unsafe { + raw_input_active.write(raw); + } + OpenNowStreamerStatus::Ok + }) +} + +#[unsafe(no_mangle)] +/// Binds the caller's live graphics objects to the current scene-graph render thread. +/// +/// Repeating the call with an identical context is idempotent. A changed context invalidates any +/// queued or acquired frame from the previous context. +/// +/// # Safety +/// +/// `handle` must be live and must not race with destroy. `context` must point to a readable +/// versioned context. Its native graphics objects must remain live until scene-graph shutdown, +/// and this call must run on the thread that will acquire and record frames. +pub unsafe extern "C" fn opennow_streamer_set_graphics_context( + handle: *const OpenNowStreamer, + context: *const OpenNowStreamerGraphicsContext, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || context.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let version = unsafe { ptr::addr_of!((*context).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*context).struct_size).read() }; + if version != OPENNOW_STREAMER_GRAPHICS_CONTEXT_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let api_name = graphics_api_name(unsafe { ptr::addr_of!((*context).graphics_api).read() }); + let context = match graphics_context(unsafe { context.read() }) { + Ok(context) => context, + Err(status) => { + log::log_line( + "WARN", + "graphics", + &format!("graphics context rejected (api={api_name}): {status:?}"), + ); + return status; + } + }; + let handle = unsafe { &*handle }; + #[cfg(target_os = "windows")] + if context.api == GraphicsApi::D3d11 + && let Some(configured) = handle.windows_adapter_luid + { + let validation = unsafe { + opennow_streamer_platform::d3d11_adapter_luid(context.device as *mut c_void) + } + .and_then(|actual| validate_windows_adapter_luid(configured, actual)); + if let Err(error) = validation { + log::log_line( + "WARN", + "graphics", + &format!("graphics context rejected: {error}"), + ); + return OpenNowStreamerStatus::GraphicsUnavailable; + } + } + let status = handle + .graphics + .initialize(context) + .map_or_else(graphics_status, |()| OpenNowStreamerStatus::Ok); + log::log_line( + if status == OpenNowStreamerStatus::Ok { + "INFO" + } else { + "WARN" + }, + "graphics", + &format!( + "graphics context initialized (api={api_name} dmabuf_image_import={} dmabuf_buffer_import={}): {status:?}", + context.vulkan_dmabuf_import_enabled, context.vulkan_dmabuf_buffer_import_enabled + ), + ); + status + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Acquires the newest pending GPU frame and transfers one retained reference to the caller. +/// +/// # Safety +/// +/// `handle` must be live. `output` and `info` must point to writable storage. The call must run on +/// the bound render thread. A successful token must be released exactly once. +pub unsafe extern "C" fn opennow_streamer_acquire_latest_frame( + handle: *const OpenNowStreamer, + output: *mut *mut OpenNowStreamerFrame, + info: *mut OpenNowStreamerFrameInfo, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || output.is_null() || info.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + unsafe { + output.write(ptr::null_mut()); + info.write(OpenNowStreamerFrameInfo::default()); + } + let token = match unsafe { &*handle }.graphics.acquire_latest() { + Ok(token) => token, + Err(error) => { + let status = graphics_status(error); + if status != OpenNowStreamerStatus::NoFrame { + log::log_throttled( + "acquire-latest-frame", + "WARN", + "present", + &format!("acquire latest frame failed: {status:?}"), + ); + } + return status; + } + }; + let frame_info = token.info(); + if !FIRST_FRAME_LOGGED.swap(true, Ordering::Relaxed) { + log::log_line( + "INFO", + "present", + &format!( + "first frame acquired ({}x{} seq={})", + frame_info.width, frame_info.height, frame_info.sequence, + ), + ); + } + unsafe { + info.write(OpenNowStreamerFrameInfo { + width: frame_info.width, + height: frame_info.height, + sequence: frame_info.sequence, + presentation_time_ns: frame_info.presentation_time_ns, + }); + output.write(Box::into_raw(Box::new(OpenNowStreamerFrame { token }))); + } + OpenNowStreamerStatus::Ok + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Records the acquired frame into the caller's current GPU command stream. +/// +/// # Safety +/// +/// `handle` and `frame` must be live and belong to the same streamer. `command` must describe a +/// writable command buffer owned by the bound graphics context. This call must happen before the +/// QQuickRhiItem render pass begins. `output` receives textures that can then be sampled inside the +/// item pass. All native objects must remain valid for the call, which must run on the bound render +/// thread. +pub unsafe extern "C" fn opennow_streamer_record_frame( + handle: *const OpenNowStreamer, + frame: *const OpenNowStreamerFrame, + command: *const OpenNowStreamerRecordCommand, + output: *mut OpenNowStreamerRecordedFrame, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || frame.is_null() || command.is_null() || output.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + unsafe { + output.write(OpenNowStreamerRecordedFrame::default()); + } + let version = unsafe { ptr::addr_of!((*command).version).read() }; + let struct_size = unsafe { ptr::addr_of!((*command).struct_size).read() }; + if version != OPENNOW_STREAMER_RENDER_COMMAND_VERSION + || struct_size < size_of::() + { + return OpenNowStreamerStatus::InvalidConfig; + } + let command = match render_command(unsafe { command.read() }) { + Ok(command) => command, + Err(status) => return status, + }; + let handle = unsafe { &*handle }; + let recorded = match handle.graphics.record(&unsafe { &*frame }.token, command) { + Ok(recorded) => recorded, + Err(error) => { + // Expected during asynchronous decoder bootstrap. Qt keeps the + // current surface and waits for the decoder's frame-ready event. + if error == GraphicsRuntimeError::NoFrame { + log::log_throttled( + "record-not-ready", + "INFO", + "present", + "decoder output pending; waiting for frame-ready", + ); + return OpenNowStreamerStatus::NoFrame; + } + // Preserve the platform failure before collapsing it to an ABI + // status. Graphics errors contain local stage/device details, + // not session context; bound the diagnostic on the render path. + let detail: String = error.to_string().chars().take(512).collect(); + let status = graphics_status(error); + log::log_throttled( + "record-frame", + "WARN", + "present", + &format!("record frame failed: {status:?} detail={detail}"), + ); + return status; + } + }; + let api = match handle.frame_publisher.context() { + Some(lease) => lease.context().api, + None => return OpenNowStreamerStatus::GraphicsUnavailable, + }; + let recorded = match recorded_frame(api, recorded) { + Ok(recorded) => recorded, + Err(status) => return status, + }; + unsafe { + output.write(recorded); + } + OpenNowStreamerStatus::Ok + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Releases one acquired frame token. +/// +/// # Safety +/// +/// `frame` must be a live token returned by acquire and must be released exactly once. Release may +/// run on any thread and remains valid after scene-graph shutdown. +pub unsafe extern "C" fn opennow_streamer_release_frame( + frame: *mut OpenNowStreamerFrame, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if frame.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + drop(unsafe { Box::from_raw(frame) }); + OpenNowStreamerStatus::Ok + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Invalidates queued frames and releases all scene-graph-owned graphics state. +/// +/// Acquired tokens remain releasable but cannot be recorded. The handle can later bind a new +/// context, including from a replacement render thread. +/// +/// # Safety +/// +/// `handle` must be live and the call must run on the currently bound render thread. It must not +/// race with another graphics operation or destroy. Submit and drain commands and release host +/// texture wrappers first, while the borrowed device is still live. RenderFailed still invalidates +/// the graphics context; WrongThread does not. +pub unsafe extern "C" fn opennow_streamer_scene_graph_shutdown( + handle: *const OpenNowStreamer, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + let status = unsafe { &*handle } + .graphics + .shutdown() + .map_or_else(graphics_status, |()| OpenNowStreamerStatus::Ok); + FIRST_FRAME_LOGGED.store(false, Ordering::Relaxed); + log::log_line( + "INFO", + "graphics", + &format!("scene graph shutdown: {status:?}"), + ); + status + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Copies and queues one serialized protocol command. +/// +/// # Safety +/// +/// `handle` must be a live handle returned by create and must not race with destroy. When `length` +/// is nonzero, `bytes` must identify a readable allocation of at least `length` bytes. +pub unsafe extern "C" fn opennow_streamer_send( + handle: *const OpenNowStreamer, + bytes: *const u8, + length: usize, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || (bytes.is_null() && length != 0) { + return OpenNowStreamerStatus::NullPointer; + } + let handle = unsafe { &*handle }; + let bytes = if length == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(bytes, length) } + }; + let summary = serde_json::from_slice::(bytes) + .map(|value| log::message_summary(&value)) + .unwrap_or_else(|_| command_kind_label(bytes)); + let status = handle.send(bytes); + log::log_line( + "INFO", + "qt-to-native", + &format!("{summary} bytes={length} queue_result={status:?}"), + ); + status + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[unsafe(no_mangle)] +/// Stops the engine, drains callbacks, and consumes the handle. +/// +/// # Safety +/// +/// `handle` must be a live handle returned by create. It must be passed to destroy exactly once, +/// with no concurrent or subsequent access through the pointer. +pub unsafe extern "C" fn opennow_streamer_destroy( + handle: *mut OpenNowStreamer, +) -> OpenNowStreamerStatus { + match catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + return OpenNowStreamerStatus::NullPointer; + } + if unsafe { &*handle }.graphics.is_active() { + return OpenNowStreamerStatus::SceneGraphActive; + } + let mut handle = unsafe { Box::from_raw(handle) }; + handle.shutdown(); + FIRST_FRAME_LOGGED.store(false, Ordering::Relaxed); + log::log_line("INFO", "engine", "engine destroyed"); + OpenNowStreamerStatus::Ok + })) { + Ok(status) => status, + Err(_) => OpenNowStreamerStatus::Panic, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::Barrier; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Condvar, Mutex}; + use std::time::{Duration, Instant}; + + use opennow_streamer_platform::{GraphicsFrame, GraphicsFrameInfo, create_test_runtime}; + use serde_json::json; + + use super::*; + use opennow_streamer_hid::SourceMode; + + #[test] + fn logging_throttles_only_periodic_snapshots_not_drop_deltas_or_warnings() { + let now = Instant::now(); + let mut last_telemetry = now; + for seconds in 0..10 { + let current = now + Duration::from_secs(seconds); + assert!(!should_log_event("telemetry", &mut last_telemetry, current)); + assert!(!should_log_event("stats", &mut last_telemetry, current)); + for kind in ["log", "error", "status"] { + assert!(should_log_event(kind, &mut last_telemetry, current)); + } + assert_eq!(last_telemetry, now); + } + assert!(should_log_event( + "telemetry", + &mut last_telemetry, + now + Duration::from_secs(10) + )); + assert_eq!(last_telemetry, now + Duration::from_secs(10)); + } + + #[test] + fn hdr_recorded_frames_preserve_color_space_and_precision() { + use opennow_streamer_platform::GraphicsColorSpace; + for (color_space, wire_color) in [ + ( + GraphicsColorSpace::Sdr709, + OPENNOW_STREAMER_COLOR_SPACE_SDR709, + ), + ( + GraphicsColorSpace::Pq2020, + OPENNOW_STREAMER_COLOR_SPACE_PQ2020, + ), + ( + GraphicsColorSpace::Hlg2020, + OPENNOW_STREAMER_COLOR_SPACE_HLG2020, + ), + ] { + for (texture_format, wire_format) in [ + ( + GraphicsTextureFormat::Rgba8, + OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA8, + ), + ( + GraphicsTextureFormat::Rgb10A2, + OPENNOW_STREAMER_TEXTURE_FORMAT_RGB10A2, + ), + ( + GraphicsTextureFormat::Rgba16Float, + OPENNOW_STREAMER_TEXTURE_FORMAT_RGBA16F, + ), + ] { + let result = recorded_frame( + GraphicsApi::Vulkan, + GraphicsRecordedFrame { + resource: 1, + resource_view: 2, + texture_format, + color_space, + width: 1920, + height: 1080, + frame_slot: 0, + generation: 7, + presentation_time_ns: 123456789, + }, + ); + if color_space != GraphicsColorSpace::Sdr709 + && texture_format == GraphicsTextureFormat::Rgba8 + { + assert_eq!(result, Err(OpenNowStreamerStatus::RenderFailed)); + } else { + let frame = result.expect("supported color texture"); + assert_eq!(frame.color_space, wire_color); + assert_eq!(frame.texture_format, wire_format); + assert_eq!(frame.generation, 7); + assert_eq!(frame.presentation_time_ns, 123456789); + } + } + } + } + + #[test] + fn hdr_abi_rejects_pre_color_metadata_clients() { + let messages = CallbackMessages::default(); + let mut config = test_config(&messages); + config.abi_version = 5; + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + assert_eq!( + std::mem::offset_of!(OpenNowStreamerRecordedFrame, color_space), + std::mem::offset_of!(OpenNowStreamerRecordedFrame, texture_format) + size_of::() + ); + } + + #[test] + fn abi_five_appends_the_shared_vulkan_owner() { + assert_eq!(OPENNOW_STREAMER_VULKAN_DEVICE_INFO_VERSION, 1); + assert_eq!( + std::mem::offset_of!(OpenNowStreamerConfig, vulkan_device), + std::mem::offset_of!(OpenNowStreamerConfig, user_data) + size_of::<*mut c_void>(), + ); + assert_eq!( + std::mem::offset_of!(OpenNowStreamerVulkanDeviceInfo, physical_device), + std::mem::offset_of!(OpenNowStreamerVulkanDeviceInfo, instance) + + size_of::<*mut c_void>(), + ); + assert_eq!( + std::mem::offset_of!(OpenNowStreamerVulkanDeviceInfo, graphics_queue_index), + std::mem::offset_of!(OpenNowStreamerVulkanDeviceInfo, graphics_queue_family_index) + + size_of::(), + ); + let messages = CallbackMessages::default(); + let mut config = test_config(&messages); + config.abi_version = 4; + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + config.abi_version = OPENNOW_STREAMER_FFI_ABI_VERSION; + config.struct_size = std::mem::offset_of!(OpenNowStreamerConfig, vulkan_device); + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + + #[test] + fn abi_twelve_adds_the_deterministic_graphics_unsupported_status() { + assert_eq!(OPENNOW_STREAMER_FFI_ABI_VERSION, 15); + assert_eq!(OpenNowStreamerStatus::GraphicsUnsupported as i32, 15); + let header = include_str!("../include/opennow_streamer_ffi.h"); + assert!(header.contains("OPENNOW_STREAMER_GRAPHICS_UNSUPPORTED = 15,")); + } + + #[test] + fn abi_eleven_appends_the_sony_snapshot_contract() { + const { assert!(OPENNOW_STREAMER_FFI_ABI_VERSION >= 11) }; + assert_eq!(std::mem::offset_of!(OpenNowSdlDeviceClaim, incarnation), 8); + assert_eq!(std::mem::offset_of!(OpenNowSdlDeviceClaim, vendor), 16); + assert_eq!(std::mem::offset_of!(OpenNowSdlDeviceClaim, product), 18); + assert_eq!(size_of::(), 24); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, struct_size), 8); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, slot), 16); + assert_eq!( + std::mem::offset_of!(OpenNowSonySnapshot, touchpad_click), + 17 + ); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, incarnation), 24); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, buttons), 32); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, left_trigger), 34); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, left_stick_x), 36); + assert_eq!( + std::mem::offset_of!(OpenNowSonySnapshot, contact_active), + 44 + ); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, contact_x), 48); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, contact_y), 56); + assert_eq!(std::mem::offset_of!(OpenNowSonySnapshot, reserved3), 64); + assert_eq!( + std::mem::offset_of!(OpenNowSonySnapshot, observed_at_us), + 72 + ); + assert_eq!(size_of::(), 80); + } + + #[test] + fn abi_ten_appends_the_windows_adapter_luid() { + const { assert!(OPENNOW_STREAMER_FFI_ABI_VERSION >= 10) }; + assert_eq!( + std::mem::offset_of!(OpenNowStreamerConfig, windows_adapter_luid), + std::mem::offset_of!(OpenNowStreamerConfig, vulkan_device) + + size_of::<*const OpenNowStreamerVulkanDevice>(), + ); + let messages = CallbackMessages::default(); + let mut config = test_config(&messages); + config.struct_size = std::mem::offset_of!(OpenNowStreamerConfig, windows_adapter_luid); + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + config.struct_size = size_of::(); + config.windows_adapter_luid = 0; + assert_eq!(WindowsAdapterLuid::new(config.windows_adapter_luid), None); + config.windows_adapter_luid = 0x1122_3344_ffff_fffe; + assert_eq!( + WindowsAdapterLuid::new(config.windows_adapter_luid).map(WindowsAdapterLuid::get), + Some(0x1122_3344_ffff_fffe) + ); + } + + #[test] + fn configured_windows_adapter_must_match_the_adopted_device() { + let configured = WindowsAdapterLuid::new(0x0000_0001_1122_3344).expect("configured LUID"); + assert_eq!( + validate_windows_adapter_luid(configured, configured), + Ok(()) + ); + let actual = WindowsAdapterLuid::new(0x0000_0002_5566_7788).expect("actual LUID"); + assert_eq!( + validate_windows_adapter_luid(configured, actual), + Err("configured Windows adapter LUID 0x0000000111223344 does not match Qt D3D11 device LUID 0x0000000255667788".to_owned()) + ); + } + + #[test] + fn vulkan_owner_exports_reject_null_pointers() { + assert_eq!( + unsafe { opennow_streamer_vulkan_device_create(ptr::null_mut()) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_vulkan_device_info(ptr::null(), ptr::null_mut()) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_vulkan_device_destroy(ptr::null_mut()) }, + OpenNowStreamerStatus::NullPointer + ); + } + + #[cfg(not(all( + target_os = "linux", + any(feature = "linux-ffmpeg", feature = "linux-ffmpeg-bundled") + )))] + #[test] + fn unsupported_vulkan_creation_clears_the_output() { + let mut output = ptr::dangling_mut::(); + assert_eq!( + unsafe { opennow_streamer_vulkan_device_create(&mut output) }, + OpenNowStreamerStatus::GraphicsUnavailable + ); + assert!(output.is_null()); + } + + #[derive(Default)] + struct CallbackMessages { + values: Mutex>, + changed: Condvar, + frames_available: AtomicUsize, + } + + impl CallbackMessages { + fn wait_for_id(&self, id: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(5); + let mut values = self.values.lock().expect("callback values"); + loop { + if let Some(value) = values.iter().find(|value| value["id"] == id) { + return value.clone(); + } + let timeout = deadline.saturating_duration_since(Instant::now()); + assert!(!timeout.is_zero(), "timed out waiting for response {id}"); + let (next, result) = self + .changed + .wait_timeout(values, timeout) + .expect("callback wait"); + values = next; + assert!(!result.timed_out(), "timed out waiting for response {id}"); + } + } + } + + unsafe extern "C" fn collect_response(bytes: *const u8, length: usize, user_data: *mut c_void) { + let bytes = unsafe { std::slice::from_raw_parts(bytes, length) }; + let Ok(value) = serde_json::from_slice(bytes) else { + return; + }; + let messages = unsafe { &*(user_data as *const CallbackMessages) }; + messages.values.lock().expect("callback values").push(value); + messages.changed.notify_all(); + } + + unsafe extern "C" fn collect_frame_available(user_data: *mut c_void) { + let messages = unsafe { &*(user_data as *const CallbackMessages) }; + messages.frames_available.fetch_add(1, Ordering::Relaxed); + } + + fn test_config(messages: &CallbackMessages) -> OpenNowStreamerConfig { + OpenNowStreamerConfig { + abi_version: OPENNOW_STREAMER_FFI_ABI_VERSION, + struct_size: size_of::(), + command_queue_capacity: 4, + response_queue_capacity: 4, + event_queue_capacity: 4, + max_command_bytes: 4096, + response_callback: Some(collect_response), + event_callback: None, + frame_available_callback: Some(collect_frame_available), + cursor_callback: None, + user_data: ptr::from_ref(messages).cast_mut().cast(), + vulkan_device: ptr::null(), + windows_adapter_luid: 0, + } + } + + #[test] + fn controller_rumble_crosses_the_event_callback_without_per_packet_logging() { + let messages = CallbackMessages::default(); + let (sender, receiver) = sync_channel(1); + let dispatcher = spawn_dispatcher( + "test-rumble-events", + receiver, + Callback { + function: Some(collect_response), + user_data: ptr::from_ref(&messages) as usize, + }, + ) + .unwrap(); + let event = json!({"type":"controller-rumble", "startId":"start-7", + "controllerId":3, "lowFrequency":65535, "highFrequency":12345, "durationMs":65535}); + sender.send(event.clone()).unwrap(); + drop(sender); + dispatcher.join().unwrap(); + assert_eq!(*messages.values.lock().unwrap(), vec![event]); + let now = Instant::now(); + let mut last = now; + assert!(!should_log_event("controller-rumble", &mut last, now)); + } + + #[test] + fn server_timer_and_termination_cross_the_bounded_json_callback() { + let messages = CallbackMessages::default(); + let (sender, receiver) = sync_channel(3); + let dispatcher = spawn_dispatcher( + "test-server-control-events", + receiver, + Callback { + function: Some(collect_response), + user_data: ptr::from_ref(&messages) as usize, + }, + ) + .unwrap(); + let events = vec![ + json!({"type":"session-timer", "startId":"start-7", "timerType":4,"secondsLeft":60,"active":true}), + json!({"type":"session-timer", "startId":"start-7", "timerType":4,"secondsLeft":0,"active":false}), + json!({"type":"status", "status":"stopped", "termination":{"source":"nvst-server","wireCode":0x8003_0023_u32,"reason":56,"extendedCode":0x8003_0007_u32,"resumable":false}}), + ]; + for event in &events { + sender.send(event.clone()).unwrap(); + } + drop(sender); + dispatcher.join().unwrap(); + assert_eq!(*messages.values.lock().unwrap(), events); + let header = include_str!("../include/opennow_streamer_ffi.h"); + assert!(header.contains("#define OPENNOW_STREAMER_EVENT_SESSION_TIMER \"session-timer\"")); + assert!( + header.contains("#define OPENNOW_STREAMER_TERMINATION_SOURCE_SERVER \"nvst-server\"") + ); + assert!(header.contains("#define OPENNOW_STREAMER_TIMER_IDLE_WARNING 4u")); + assert!(header.contains("#define OPENNOW_STREAMER_TIMER_IDLE_EXPIRY 5u")); + } + + #[test] + fn text_input_rejection_crosses_the_bounded_json_callback() { + let messages = CallbackMessages::default(); + let (sender, receiver) = sync_channel(1); + let dispatcher = spawn_dispatcher( + "test-text-input-events", + receiver, + Callback { + function: Some(collect_response), + user_data: ptr::from_ref(&messages) as usize, + }, + ) + .unwrap(); + let event = json!({"type":"text-input-error", "startId":"paste-session", "reason":"paste-queue-full"}); + sender.send(event.clone()).unwrap(); + drop(sender); + dispatcher.join().unwrap(); + assert_eq!(*messages.values.lock().unwrap(), vec![event]); + assert!( + include_str!("../include/opennow_streamer_ffi.h") + .contains("#define OPENNOW_STREAMER_EVENT_TEXT_INPUT_ERROR \"text-input-error\"") + ); + } + + #[test] + fn recording_completion_metadata_crosses_the_unchanged_json_callback() { + use opennow_streamer_protocol::{ + PROTOCOL_VERSION, RecordingCompletion, RecordingCutReason, + }; + + let messages = CallbackMessages::default(); + let (sender, receiver) = sync_channel(4); + let dispatcher = spawn_dispatcher( + "test-recording-events", + receiver, + Callback { + function: Some(collect_response), + user_data: ptr::from_ref(&messages) as usize, + }, + ) + .unwrap(); + let expected: Vec<_> = [ + RecordingCompletion::Complete, + RecordingCompletion::Cut { + reason: RecordingCutReason::Discontinuity, + }, + RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + }, + RecordingCompletion::Cut { + reason: RecordingCutReason::Interrupted, + }, + ] + .into_iter() + .map(|completion| { + json!({"type":"recording-state", "state":"saved", + "requestId":"recording-start-9", "path":"fixture.mkv", "videoPackets":3, + "audioPackets":3, "completion":completion}) + }) + .collect(); + for event in &expected { + sender.send(event.clone()).unwrap(); + } + drop(sender); + dispatcher.join().unwrap(); + assert_eq!(*messages.values.lock().unwrap(), expected); + assert_eq!(PROTOCOL_VERSION, 7); + } + + fn create_with_test_runtime( + messages: &CallbackMessages, + ) -> ( + OpenNowStreamer, + opennow_streamer_platform::TestMediaRuntimeHost, + ) { + let (host, runtime) = create_test_runtime(); + let shutdown_runtime = runtime.clone(); + let handle = OpenNowStreamer::create( + test_config(messages), + runtime.captured_input(), + Arc::new(HidRuntime::new()), + move |events, _frames, _cursor| Engine::with_embedded_media_runtime(events, runtime), + move || shutdown_runtime.shutdown(), + ) + .expect("FFI handle"); + (handle, host) + } + + #[derive(Default)] + struct RecordedCommands { + values: Mutex>, + drops: AtomicUsize, + } + + struct TestGraphicsFrame { + sequence: u64, + recorded: Arc, + panic_on_record: bool, + error: Option, + } + + impl GraphicsFrame for TestGraphicsFrame { + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: 2560, + height: 1440, + sequence: self.sequence, + presentation_time_ns: 8_333_333 * self.sequence, + } + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + assert!(!self.panic_on_record, "injected render panic"); + if let Some(error) = &self.error { + return Err(error.clone()); + } + self.recorded + .values + .lock() + .expect("recorded commands") + .push((context, command)); + Ok(GraphicsRecordedFrame { + resource: 0xfeed, + resource_view: 0xbeef, + texture_format: GraphicsTextureFormat::Rgb10A2, + color_space: opennow_streamer_platform::GraphicsColorSpace::Sdr709, + width: 2560, + height: 1440, + frame_slot: command.frame_slot, + generation: self.sequence, + presentation_time_ns: 8_333_333 * self.sequence, + }) + } + } + + impl Drop for TestGraphicsFrame { + fn drop(&mut self) { + self.recorded.drops.fetch_add(1, Ordering::Relaxed); + } + } + + fn ffi_graphics_context() -> OpenNowStreamerGraphicsContext { + OpenNowStreamerGraphicsContext { + version: OPENNOW_STREAMER_GRAPHICS_CONTEXT_VERSION, + struct_size: size_of::(), + graphics_api: OPENNOW_STREAMER_GRAPHICS_API_VULKAN, + instance: ptr::dangling_mut(), + physical_device: ptr::dangling_mut(), + device: ptr::dangling_mut(), + queue: ptr::dangling_mut(), + queue_family_index: 5, + enabled_capabilities: 0, + } + } + + #[test] + fn graphics_capabilities_are_explicit_versioned_and_vulkan_only() { + let mut context = ffi_graphics_context(); + assert!( + !graphics_context(context) + .unwrap() + .vulkan_dmabuf_import_enabled + ); + context.enabled_capabilities = OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT; + assert!( + graphics_context(context) + .unwrap() + .vulkan_dmabuf_import_enabled + ); + assert!( + !graphics_context(context) + .unwrap() + .vulkan_dmabuf_buffer_import_enabled + ); + context.enabled_capabilities = OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT; + let buffer_only = graphics_context(context).unwrap(); + assert!(buffer_only.vulkan_dmabuf_buffer_import_enabled); + assert!(!buffer_only.vulkan_dmabuf_import_enabled); + context.enabled_capabilities |= OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT; + let both = graphics_context(context).unwrap(); + assert!(both.vulkan_dmabuf_import_enabled); + assert!(both.vulkan_dmabuf_buffer_import_enabled); + context.enabled_capabilities = 4; + assert_eq!( + graphics_context(context), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + context.enabled_capabilities = OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_IMPORT; + context.graphics_api = OPENNOW_STREAMER_GRAPHICS_API_D3D11; + assert_eq!( + graphics_context(context), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + context.enabled_capabilities = OPENNOW_STREAMER_GRAPHICS_CAP_VULKAN_DMABUF_BUFFER_IMPORT; + assert_eq!( + graphics_context(context), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + context = ffi_graphics_context(); + for version in [1, 2] { + context.version = version; + assert_eq!( + graphics_context(context), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + context = ffi_graphics_context(); + context.struct_size -= 1; + assert_eq!( + graphics_context(context), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + + fn ffi_render_command() -> OpenNowStreamerRecordCommand { + OpenNowStreamerRecordCommand { + version: OPENNOW_STREAMER_RENDER_COMMAND_VERSION, + struct_size: size_of::(), + command_buffer: ptr::dangling_mut(), + frame_slot: 2, + upscale_width: 0, + upscale_height: 0, + upscale_sharpness: 10, + upscale_denoise: 0, + } + } + + #[test] + fn render_command_three_preserves_controls_and_rejects_previous_layout_and_ranges() { + let mut command = ffi_render_command(); + let native = render_command(command).unwrap(); + assert_eq!(native.upscale_sharpness, 10); + assert_eq!(native.upscale_denoise, 0); + command.upscale_sharpness = 15; + command.upscale_denoise = 20; + let native = render_command(command).unwrap(); + assert_eq!(native.upscale_sharpness, 15); + assert_eq!(native.upscale_denoise, 20); + command.version = 2; + assert_eq!( + render_command(command), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + command.version = OPENNOW_STREAMER_RENDER_COMMAND_VERSION; + command.struct_size = std::mem::offset_of!(OpenNowStreamerRecordCommand, upscale_sharpness); + assert_eq!( + render_command(command), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + for (sharpness, denoise) in [(16, 0), (0, 21), (u32::MAX, u32::MAX)] { + command = ffi_render_command(); + command.upscale_sharpness = sharpness; + command.upscale_denoise = denoise; + assert_eq!( + render_command(command), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + let messages = CallbackMessages::default(); + let mut config = test_config(&messages); + config.abi_version = 7; + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + + #[test] + fn render_command_two_preserves_upscaling_and_rejects_old_layouts() { + let mut command = ffi_render_command(); + assert_eq!(OPENNOW_STREAMER_RENDER_COMMAND_VERSION, 3); + assert_eq!(render_command(command).unwrap().upscale_width, 0); + assert_eq!(render_command(command).unwrap().upscale_height, 0); + command.upscale_width = 2560; + command.upscale_height = 1440; + let native = render_command(command).unwrap(); + assert_eq!(native.upscale_width, 2560); + assert_eq!(native.upscale_height, 1440); + assert_eq!(native.frame_slot, command.frame_slot); + command.version = 1; + assert_eq!( + render_command(command), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + command.version = OPENNOW_STREAMER_RENDER_COMMAND_VERSION; + command.struct_size = std::mem::offset_of!(OpenNowStreamerRecordCommand, upscale_width); + assert_eq!( + render_command(command), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + let messages = CallbackMessages::default(); + let mut config = test_config(&messages); + config.abi_version = 6; + assert_eq!( + validate_config(&config), + Err(OpenNowStreamerStatus::InvalidConfig) + ); + } + + fn graphics_test_handle(messages: &CallbackMessages) -> OpenNowStreamer { + OpenNowStreamer::create( + test_config(messages), + Arc::new(CapturedInputQueue::default()), + Arc::new(HidRuntime::new()), + |events, _frames, _cursor| Engine::embedded(events), + || {}, + ) + .expect("FFI handle") + } + + #[test] + fn typed_submit_api_routes_only_captured_events_to_the_rust_queue() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + handle.input.set_active(true, false, 0); + + assert_eq!( + unsafe { opennow_streamer_submit_key(&handle, 0x57, 0x02, true) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_relative(&handle, -12, 34) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_absolute(&handle, 20, 30, 640, 360) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_button(&handle, 5, true) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_wheel(&handle, 0, -120) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { + opennow_streamer_submit_gamepad(&handle, 2, 0x0404, 0x1001, 4, 5, -6, 7, -8, 9) + }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { + opennow_streamer_submit_local_action(&handle, OPENNOW_STREAMER_LOCAL_ACTION_GUIDE) + }, + OpenNowStreamerStatus::Ok + ); + + let queue = handle.input.queue(); + assert_eq!( + queue.take(), + Some(CapturedInput::Key { + virtual_key: 0x57, + modifiers: 0x02, + pressed: true, + }) + ); + assert_eq!( + queue.take(), + Some(CapturedInput::MouseMove { + delta_x: -12, + delta_y: 34, + }) + ); + assert_eq!( + queue.take(), + Some(CapturedInput::MouseAbsolute { + x: 20, + y: 30, + width: 640, + height: 360, + }) + ); + assert_eq!( + queue.take(), + Some(CapturedInput::MouseButton { + button: 5, + pressed: true, + }) + ); + assert_eq!( + queue.take(), + Some(CapturedInput::MouseWheel { + delta_x: 0, + delta_y: -120, + }) + ); + assert_eq!( + queue.take(), + Some(CapturedInput::Gamepad { + controller_id: 2, + bitmap: 0x0404, + buttons: 0x1001, + left_trigger: 4, + right_trigger: 5, + left_stick_x: -6, + left_stick_y: 7, + right_stick_x: -8, + right_stick_y: 9, + }) + ); + assert_eq!(queue.take(), Some(CapturedInput::Guide)); + assert_eq!(queue.take(), None); + + handle.shutdown(); + } + + #[test] + fn typed_text_submission_validates_copies_and_bounds_the_whole_paste() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + let submit = |bytes: &[u8]| unsafe { + opennow_streamer_submit_text(&handle, bytes.as_ptr(), bytes.len()) + }; + assert_eq!( + unsafe { opennow_streamer_submit_text(ptr::null(), b"a".as_ptr(), 1) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_submit_text(&handle, ptr::null(), 1) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_submit_text(&handle, ptr::null(), usize::MAX) }, + OpenNowStreamerStatus::MessageTooLarge + ); + for invalid in [ + &b""[..], + &b"a\0b"[..], + &[0xff], + &[0xc0, 0x80], + &[0xed, 0xa0, 0x80], + ] { + assert_eq!(submit(invalid), OpenNowStreamerStatus::InvalidConfig); + } + assert_eq!( + submit(&vec![b'a'; OPENNOW_STREAMER_MAX_TEXT_BYTES + 1]), + OpenNowStreamerStatus::MessageTooLarge + ); + assert_eq!(submit(b"paste"), OpenNowStreamerStatus::Closed); + handle.input.set_active(true, false, 0); + assert_eq!(submit(b"paste"), OpenNowStreamerStatus::Closed); + let queue = handle.input.queue(); + queue.set_text_ready(1, true); + let mut bytes = "café 世界 🦫\r\n".as_bytes().to_vec(); + assert_eq!(submit(&bytes), OpenNowStreamerStatus::Ok); + bytes.fill(b'x'); + assert_eq!(submit(b"again"), OpenNowStreamerStatus::QueueFull); + let Some(CapturedInput::Text(text)) = queue.take() else { + panic!("missing paste") + }; + assert_eq!(text.as_str(), "café 世界 🦫\r\n"); + assert_eq!(submit(b"again"), OpenNowStreamerStatus::QueueFull); + drop(text); + assert_eq!( + submit(&vec![b'a'; OPENNOW_STREAMER_MAX_TEXT_BYTES]), + OpenNowStreamerStatus::Ok + ); + handle.input.set_active(false, false, 0); + assert!(queue.take().is_none()); + handle.shutdown(); + } + + #[test] + fn abi_10_header_and_text_bound_match_rust() { + let header = include_str!("../include/opennow_streamer_ffi.h"); + assert!(header.contains(&format!( + "#define OPENNOW_STREAMER_FFI_ABI_VERSION {}u", + OPENNOW_STREAMER_FFI_ABI_VERSION + ))); + assert!(header.contains("uint64_t windows_adapter_luid;")); + assert!(header.contains(&format!( + "#define OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION {}u", + OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION + ))); + assert!(header.contains("OpenNowSonySnapshot")); + assert!(header.contains("opennow_streamer_submit_sony_snapshot(")); + assert!(header.contains("opennow_streamer_replace_sdl_device_claims(")); + assert!(header.contains("#define OPENNOW_STREAMER_MAX_TEXT_BYTES 65536u")); + assert_eq!(OPENNOW_STREAMER_MAX_TEXT_BYTES, 65_536); + assert!(header.contains("opennow_streamer_submit_text(")); + } + + fn generic_interface(slot: u8, number: u8) -> OpenNowGenericHidInterface { + OpenNowGenericHidInterface { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + slot, + interface_number: number, + reserved: 0, + incarnation: u64::from(slot) + 41, + vendor: 0x044f, + product: 0x0402, + revision: 1, + reserved2: 0, + descriptor: ptr::null(), + descriptor_len: 0, + } + } + + #[test] + fn generic_inventory_boundary_validates_before_atomic_replacement() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + let replace = |entries: &[OpenNowGenericHidInterface]| unsafe { + opennow_streamer_replace_generic_hid_inventory(&handle, entries.as_ptr(), entries.len()) + }; + assert_eq!( + unsafe { opennow_streamer_replace_generic_hid_inventory(ptr::null(), ptr::null(), 0) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_replace_generic_hid_inventory(&handle, ptr::null(), 129) }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_replace_generic_hid_inventory(&handle, ptr::null(), 1) }, + OpenNowStreamerStatus::NullPointer + ); + let descriptor = [0x05, 0x01]; + let mut entry = generic_interface(0, 3); + entry.descriptor = descriptor.as_ptr(); + entry.descriptor_len = descriptor.len(); + assert_eq!( + replace(&[entry, generic_interface(0, 4)]), + OpenNowStreamerStatus::Ok + ); + let inventory = handle.hid.generic.inventory_snapshot().0; + assert_eq!(inventory.len(), 1); + assert_eq!(inventory[0].interfaces.len(), 2); + assert_eq!(inventory[0].interfaces[0].descriptor, descriptor); + for malformed in [ + OpenNowGenericHidInterface { + version: 2, + ..entry + }, + OpenNowGenericHidInterface { + struct_size: size_of::() - 1, + ..entry + }, + OpenNowGenericHidInterface { slot: 8, ..entry }, + OpenNowGenericHidInterface { + incarnation: 0, + ..entry + }, + OpenNowGenericHidInterface { + reserved: 1, + ..entry + }, + OpenNowGenericHidInterface { + reserved2: 1, + ..entry + }, + OpenNowGenericHidInterface { + product: 0xffff, + ..entry + }, + ] { + assert_eq!(replace(&[malformed]), OpenNowStreamerStatus::InvalidConfig); + assert_eq!(handle.hid.generic.inventory_snapshot().0, inventory); + } + for malformed in [ + OpenNowGenericHidInterface { + incarnation: 99, + ..entry + }, + OpenNowGenericHidInterface { + vendor: 0x046d, + ..entry + }, + OpenNowGenericHidInterface { + product: 0xb67c, + ..entry + }, + OpenNowGenericHidInterface { + revision: 2, + ..entry + }, + entry, + ] { + assert_eq!( + replace(&[entry, malformed]), + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!(handle.hid.generic.inventory_snapshot().0, inventory); + } + assert_eq!( + replace(&[OpenNowGenericHidInterface { + descriptor: ptr::null(), + ..entry + }]), + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + replace(&[OpenNowGenericHidInterface { + descriptor_len: 4097, + ..entry + }]), + OpenNowStreamerStatus::MessageTooLarge + ); + let entries: Vec<_> = (0..17).map(|number| generic_interface(0, number)).collect(); + assert_eq!(replace(&entries), OpenNowStreamerStatus::InvalidConfig); + let entries: Vec<_> = (0..8) + .flat_map(|slot| (0..16).map(move |number| generic_interface(slot, number))) + .collect(); + assert_eq!(replace(&entries), OpenNowStreamerStatus::Ok); + assert_eq!(handle.hid.generic.inventory_snapshot().0.len(), 8); + assert_eq!( + unsafe { opennow_streamer_replace_generic_hid_inventory(&handle, ptr::null(), 0) }, + OpenNowStreamerStatus::Ok + ); + assert!(handle.hid.generic.inventory_snapshot().0.is_empty()); + } + + #[test] + fn generic_reports_reject_malformed_and_stale_sources_and_copy_payloads() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + assert_eq!( + unsafe { + opennow_streamer_replace_generic_hid_inventory(&handle, &generic_interface(0, 3), 1) + }, + OpenNowStreamerStatus::Ok + ); + handle.hid.generic.bind_session(1); + handle.hid.generic.set_capture(true); + let mut bytes = [7u8; 256]; + let report = OpenNowGenericHidReport { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + slot: 0, + interface_number: 3, + operation: 0, + error: 0, + incarnation: 41, + bytes: bytes.as_ptr(), + byte_count: bytes.len(), + }; + let submit = |report: &OpenNowGenericHidReport| unsafe { + opennow_streamer_submit_generic_hid_report(&handle, report) + }; + assert_eq!( + unsafe { opennow_streamer_submit_generic_hid_report(&handle, ptr::null()) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_submit_generic_hid_report(ptr::null(), &report) }, + OpenNowStreamerStatus::NullPointer + ); + for malformed in [ + OpenNowGenericHidReport { + version: 2, + ..report + }, + OpenNowGenericHidReport { + struct_size: size_of::() - 1, + ..report + }, + OpenNowGenericHidReport { slot: 8, ..report }, + OpenNowGenericHidReport { + incarnation: 0, + ..report + }, + OpenNowGenericHidReport { + operation: 5, + ..report + }, + OpenNowGenericHidReport { error: 2, ..report }, + OpenNowGenericHidReport { + interface_number: 4, + ..report + }, + ] { + assert_eq!(submit(&malformed), OpenNowStreamerStatus::InvalidConfig); + } + assert_eq!( + submit(&OpenNowGenericHidReport { + incarnation: 40, + ..report + }), + OpenNowStreamerStatus::Closed + ); + assert_eq!( + submit(&OpenNowGenericHidReport { + byte_count: 257, + ..report + }), + OpenNowStreamerStatus::MessageTooLarge + ); + assert_eq!( + submit(&OpenNowGenericHidReport { + bytes: ptr::null(), + ..report + }), + OpenNowStreamerStatus::NullPointer + ); + assert_eq!(submit(&report), OpenNowStreamerStatus::Ok); + bytes.fill(9); + let queued = handle.hid.generic.drain(1); + let opennow_streamer_hid::generic_runtime::GenericIngress::Report(queued) = &queued[0] + else { + panic!("report expected") + }; + assert_eq!(queued.bytes, vec![7; 256]); + for operation in 0..=4 { + assert_eq!( + submit(&OpenNowGenericHidReport { + operation, + error: 1, + ..report + }), + OpenNowStreamerStatus::Ok + ); + } + } + + #[test] + fn generic_output_contract_is_versioned_bounded_and_matches_header() { + let header = include_str!("../include/opennow_streamer_ffi.h"); + assert!(header.contains("#define OPENNOW_STREAMER_FFI_ABI_VERSION 15u")); + assert!(header.contains("#define OPENNOW_STREAMER_GENERIC_HID_VERSION 1u")); + for name in [ + "OpenNowGenericHidInterface", + "OpenNowGenericHidReport", + "OpenNowGenericHidOutput", + "opennow_streamer_replace_generic_hid_inventory(", + "opennow_streamer_submit_generic_hid_report(", + "opennow_streamer_take_generic_hid_output(", + ] { + assert!(header.contains(name)); + } + assert_eq!(size_of::(), 56); + assert_eq!(size_of::(), 48); + assert_eq!(size_of::(), 296); + assert_eq!( + std::mem::offset_of!(OpenNowGenericHidInterface, incarnation), + 24 + ); + assert_eq!(std::mem::offset_of!(OpenNowGenericHidReport, bytes), 32); + assert_eq!(std::mem::offset_of!(OpenNowGenericHidOutput, bytes), 40); + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + let mut output = OpenNowGenericHidOutput { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + slot: 0, + interface_number: 0, + operation: 0, + reserved: 0, + incarnation: 0, + byte_count: 0, + bytes: [0; 256], + }; + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(&handle, ptr::null_mut()) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(ptr::null(), &mut output) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { + opennow_streamer_replace_generic_hid_inventory(&handle, &generic_interface(0, 3), 1) + }, + OpenNowStreamerStatus::Ok + ); + handle.hid.generic.bind_session(1); + handle.hid.generic.set_capture(true); + handle + .hid + .generic + .push_output(opennow_streamer_hid::generic::GenericOutput { + slot: 0, + incarnation: 41, + interface: 3, + operation: 4, + bytes: vec![8; 256], + }) + .expect("queued output"); + output.version = 2; + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(&handle, &mut output) }, + OpenNowStreamerStatus::InvalidConfig + ); + output.version = OPENNOW_STREAMER_GENERIC_HID_VERSION; + output.struct_size -= 1; + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(&handle, &mut output) }, + OpenNowStreamerStatus::InvalidConfig + ); + output.struct_size += 1; + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(&handle, &mut output) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + ( + output.slot, + output.interface_number, + output.operation, + output.incarnation + ), + (0, 3, 4, 41) + ); + assert_eq!(output.reserved, 0); + assert_eq!(output.byte_count, 256); + assert_eq!(output.bytes, [8; 256]); + assert_eq!( + unsafe { opennow_streamer_take_generic_hid_output(&handle, &mut output) }, + OpenNowStreamerStatus::Closed + ); + } + + #[test] + fn generic_admission_query_is_versioned_bounded_and_keyed_by_identity() { + use opennow_streamer_hid::generic::GenericSession; + let header = include_str!("../include/opennow_streamer_ffi.h"); + assert!(header.contains("opennow_streamer_get_generic_hid_admissions(")); + assert_eq!(size_of::(), 16); + assert_eq!(size_of::(), 152); + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + let mut output = OpenNowGenericHidAdmissions { + version: OPENNOW_STREAMER_GENERIC_HID_VERSION, + struct_size: size_of::(), + count: 0, + devices: [OpenNowGenericHidAdmission::default(); 8], + }; + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(ptr::null(), &mut output) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, ptr::null_mut()) }, + OpenNowStreamerStatus::NullPointer + ); + output.version = 2; + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, &mut output) }, + OpenNowStreamerStatus::InvalidConfig + ); + output.version = OPENNOW_STREAMER_GENERIC_HID_VERSION; + output.struct_size -= 1; + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, &mut output) }, + OpenNowStreamerStatus::InvalidConfig + ); + output.struct_size += 1; + let interfaces = (0..8) + .map(|slot| generic_interface(slot, 3)) + .collect::>(); + assert_eq!( + unsafe { + opennow_streamer_replace_generic_hid_inventory( + &handle, + interfaces.as_ptr(), + interfaces.len(), + ) + }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, &mut output) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(output.count, 8); + for (slot, row) in output.devices.iter().enumerate() { + assert_eq!(row.slot, slot as u8); + assert_eq!(row.incarnation, slot as u64 + 41); + assert_eq!(row.admission, 0); + assert_eq!(row.reserved, [0; 6]); + } + handle.hid.generic.bind_session(1); + let mut session = GenericSession::new(1); + let (inventory, _) = handle.hid.generic.inventory_snapshot(); + for action in session.reconcile(&inventory, true) { + session.sent(&action); + } + session.handle_response(10, 1, 0, 22); + session.handle_response(11, 0, 0, 23); + handle.hid.generic.publish_admissions(&session); + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, &mut output) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(output.devices[0].admission, 1); + assert_eq!(output.devices[1].admission, 2); + handle.hid.generic.unbind_session(1); + assert_eq!( + unsafe { opennow_streamer_get_generic_hid_admissions(&handle, &mut output) }, + OpenNowStreamerStatus::Ok + ); + assert!(output.devices.iter().all(|row| row.admission == 0)); + } + + #[test] + fn sdl_claims_replace_atomically_and_reject_malformed_entries() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + let claim = |slot: u8, incarnation: u64| OpenNowSdlDeviceClaim { + slot, + reserved: [0; 7], + incarnation, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims( + &handle, + [claim(0, 11), claim(1, 12)].as_ptr(), + 2, + ) + }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(handle.hid.inventory().incarnation(0), Some(11)); + assert_eq!(handle.hid.inventory().incarnation(1), Some(12)); + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims( + &handle, + [claim(0, 21), claim(0, 22)].as_ptr(), + 2, + ) + }, + OpenNowStreamerStatus::SdlClaimRejected + ); + assert_eq!(handle.hid.inventory().incarnation(0), Some(11)); + assert_eq!(handle.hid.inventory().incarnation(1), Some(12)); + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims(&handle, [claim(4, 31)].as_ptr(), 1) + }, + OpenNowStreamerStatus::SdlClaimRejected + ); + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims(&handle, [claim(0, 0)].as_ptr(), 1) + }, + OpenNowStreamerStatus::SdlClaimRejected + ); + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, std::ptr::null(), 1) }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims(std::ptr::null(), std::ptr::null(), 0) + }, + OpenNowStreamerStatus::NullPointer + ); + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims(&handle, [claim(0, 11)].as_ptr(), 5) + }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, std::ptr::null(), 0) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(handle.hid.inventory().incarnation(0), None); + assert_eq!(handle.hid.inventory().incarnation(1), None); + } + + #[test] + fn sony_snapshots_validate_version_size_and_source_incarnation() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, std::ptr::null()) }, + OpenNowStreamerStatus::NullPointer + ); + let mut snapshot = OpenNowSonySnapshot { + version: OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION, + struct_size: size_of::(), + slot: 0, + touchpad_click: 0, + reserved: [0; 2], + incarnation: 41, + buttons: 0, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + contact_active: [0; 2], + reserved2: [0; 2], + contact_x: [0.0; 2], + contact_y: [0.0; 2], + reserved3: 0, + observed_at_us: 1000, + }; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + let claim = OpenNowSdlDeviceClaim { + slot: 0, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + handle.hid.set_active(true); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(1).is_some()); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + handle + .hid + .set_session_admission(0b0001, handle.hid.inventory_snapshot().1); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Ok + ); + snapshot.incarnation = 42; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + snapshot.incarnation = 41; + snapshot.version = OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION + 1; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::InvalidConfig + ); + snapshot.version = OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION; + snapshot.reserved = [1; 2]; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::InvalidConfig + ); + snapshot.reserved = [0; 2]; + snapshot.struct_size = 8; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::InvalidConfig + ); + snapshot.struct_size = size_of::(); + snapshot.buttons = 0x1000; + let mut accepted = 0_usize; + while unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) } + == OpenNowStreamerStatus::Ok + { + accepted += 1; + assert!(accepted <= 64); + } + assert!(accepted > 0); + assert_eq!(handle.hid.pending_states(), 0); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::QueueFull + ); + handle.hid.set_active(false); + snapshot.buttons = 0x1000; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::SonyInactive + ); + assert_eq!(handle.hid.pending_states(), 0); + snapshot.buttons = 0; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::SonyInactive + ); + assert_eq!(handle.hid.pending_states(), 0); + } + + #[test] + fn fallback_admission_is_linearized_against_session_unbinding() { + let messages = Box::new(CallbackMessages::default()); + let queue = Arc::new(CapturedInputQueue::default()); + let hid = Arc::new(HidRuntime::new()); + let handle = OpenNowStreamer::create( + test_config(&messages), + Arc::clone(&queue), + Arc::clone(&hid), + |events, _frames, _cursor| Engine::embedded(events), + || {}, + ) + .expect("FFI handle"); + handle.input.set_active(true, false, 0); + let claim = OpenNowSdlDeviceClaim { + slot: 1, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + let endpoint = hid.open_endpoint(); + assert!(hid.bind_session(7).is_some()); + hid.set_active(true); + hid.set_session_admission(0, hid.inventory_snapshot().1); + + let entered = Arc::new(Barrier::new(2)); + let unbind_requested = Arc::new(Barrier::new(2)); + let unbind_completed = Arc::new(AtomicBool::new(false)); + let unbinder = { + let hid = Arc::clone(&hid); + let entered = Arc::clone(&entered); + let unbind_requested = Arc::clone(&unbind_requested); + let unbind_completed = Arc::clone(&unbind_completed); + std::thread::spawn(move || { + entered.wait(); + unbind_requested.wait(); + hid.unbind_session(7); + unbind_completed.store(true, Ordering::Release); + }) + }; + + let mut snapshot = opennow_streamer_hid::SonySnapshot::neutral(1, 41, 5_000); + snapshot.buttons = 0x1000; + let admission = hid.submit_snapshot_with(snapshot, |snapshot, bitmap| { + entered.wait(); + unbind_requested.wait(); + assert_eq!(bitmap, 0x0202); + handle.input.submit(CapturedInput::Gamepad { + controller_id: snapshot.slot, + bitmap, + buttons: snapshot.buttons, + left_trigger: snapshot.left_trigger, + right_trigger: snapshot.right_trigger, + left_stick_x: snapshot.left_stick_x, + left_stick_y: snapshot.left_stick_y, + right_stick_x: snapshot.right_stick_x, + right_stick_y: snapshot.right_stick_y, + }); + }); + assert_eq!(admission, SnapshotAdmission::Admitted); + unbinder.join().unwrap(); + assert!(unbind_completed.load(Ordering::Acquire)); + assert!( + queue.take().is_some(), + "the linearized admission must be enqueued before the unbind wins" + ); + assert_eq!(hid.session_generation(), None); + let mut rejected = opennow_streamer_hid::SonySnapshot::neutral(1, 41, 6_000); + rejected.buttons = 0x1000; + assert_eq!( + hid.submit_snapshot_with(rejected, |_, _| { + panic!("unbound sessions must not admit fallback input") + }), + SnapshotAdmission::Unbound + ); + hid.close_endpoint(endpoint); + } + + #[test] + fn parent_review_fallback_rejects_stale_incarnation() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + handle.input.set_active(true, false, 0); + let claim = OpenNowSdlDeviceClaim { + slot: 1, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(1).is_some()); + handle.hid.set_active(true); + handle + .hid + .set_session_admission(0, handle.hid.inventory_snapshot().1); + let mut snapshot = OpenNowSonySnapshot { + version: OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION, + struct_size: size_of::(), + slot: 1, + touchpad_click: 0, + reserved: [0; 2], + incarnation: 41, + buttons: 0x1000, + left_trigger: 7, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 12_000, + right_stick_x: 0, + right_stick_y: -12_000, + contact_active: [0; 2], + reserved2: [0; 2], + contact_x: [0.0; 2], + contact_y: [0.0; 2], + reserved3: 0, + observed_at_us: 1000, + }; + snapshot.incarnation = 99; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + assert!(handle.input.queue().take().is_none()); + } + + #[test] + fn parent_review_changed_inventory_requires_reconciliation() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + handle.input.set_active(true, false, 0); + let claim = OpenNowSdlDeviceClaim { + slot: 1, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(1).is_some()); + handle.hid.set_active(true); + handle + .hid + .set_session_admission(0, handle.hid.inventory_snapshot().1); + assert_eq!( + handle.hid.source_mode(1), + Some(SourceMode::OrdinaryFallback) + ); + let snapshot = OpenNowSonySnapshot { + version: OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION, + struct_size: size_of::(), + slot: 1, + touchpad_click: 0, + reserved: [0; 2], + incarnation: 41, + buttons: 0x1000, + left_trigger: 7, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 12_000, + right_stick_x: 0, + right_stick_y: -12_000, + contact_active: [0; 2], + reserved2: [0; 2], + contact_x: [0.0; 2], + contact_y: [0.0; 2], + reserved3: 0, + observed_at_us: 1000, + }; + let mut replacement = claim; + replacement.incarnation = 99; + assert_eq!( + unsafe { + opennow_streamer_replace_sdl_device_claims(&handle, [replacement].as_ptr(), 1) + }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(handle.hid.source_mode(snapshot.slot), None); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + assert!(handle.input.queue().take().is_none()); + let stale_epoch = handle.hid.inventory_snapshot().1 - 1; + assert!(!handle.hid.set_session_admission(0, stale_epoch)); + assert_eq!(handle.hid.source_mode(snapshot.slot), None); + } + + #[test] + fn session_unbinding_closes_snapshot_admission_for_the_next_association() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + let claim = OpenNowSdlDeviceClaim { + slot: 0, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + handle.input.set_active(true, false, 0); + assert_eq!(handle.hid.session_generation(), None); + let endpoint = handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(7).is_some()); + handle.hid.set_active(true); + handle + .hid + .set_session_admission(0b0001, handle.hid.inventory_snapshot().1); + let snapshot = OpenNowSonySnapshot { + version: OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION, + struct_size: size_of::(), + slot: 0, + touchpad_click: 0, + reserved: [0; 2], + incarnation: 41, + buttons: 0x1000, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + contact_active: [0; 2], + reserved2: [0; 2], + contact_x: [0.0; 2], + contact_y: [0.0; 2], + reserved3: 0, + observed_at_us: 1000, + }; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(handle.hid.pending_states(), 1); + handle.hid.unbind_session(7); + assert_eq!(handle.hid.session_generation(), None); + assert_eq!(handle.hid.pending_states(), 0); + assert!(handle.hid.drain(8).is_empty()); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + assert_eq!(handle.hid.source_mode(0), None); + handle.hid.close_endpoint(endpoint); + assert!(handle.hid.bind_session(8).is_none()); + assert_eq!(handle.hid.session_generation(), None); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Closed + ); + handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(8).is_some()); + assert_eq!(handle.hid.session_generation(), Some(8)); + assert_eq!(handle.hid.source_mode(0), None); + assert_eq!(handle.hid.pending_states(), 0); + } + + #[test] + fn sony_fallback_routes_snapshots_through_the_ordinary_formatter() { + let messages = Box::new(CallbackMessages::default()); + let handle = graphics_test_handle(&messages); + handle.input.set_active(true, false, 0); + let claim = OpenNowSdlDeviceClaim { + slot: 1, + reserved: [0; 7], + incarnation: 41, + vendor: 0x054c, + product: 0x05c4, + }; + assert_eq!( + unsafe { opennow_streamer_replace_sdl_device_claims(&handle, [claim].as_ptr(), 1) }, + OpenNowStreamerStatus::Ok + ); + handle.hid.open_endpoint(); + assert!(handle.hid.bind_session(1).is_some()); + handle.hid.set_active(true); + handle + .hid + .set_session_admission(0, handle.hid.inventory_snapshot().1); + assert_eq!( + handle.hid.source_mode(1), + Some(SourceMode::OrdinaryFallback) + ); + let snapshot = OpenNowSonySnapshot { + version: OPENNOW_STREAMER_SONY_SNAPSHOT_VERSION, + struct_size: size_of::(), + slot: 1, + touchpad_click: 0, + reserved: [0; 2], + incarnation: 41, + buttons: 0x1000, + left_trigger: 7, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 12_000, + right_stick_x: 0, + right_stick_y: -12_000, + contact_active: [0; 2], + reserved2: [0; 2], + contact_x: [0.0; 2], + contact_y: [0.0; 2], + reserved3: 0, + observed_at_us: 1000, + }; + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Ok + ); + let queue = handle.input.queue(); + let Some(CapturedInput::Gamepad { + controller_id, + bitmap, + buttons, + left_trigger, + left_stick_y, + right_stick_y, + .. + }) = queue.take() + else { + panic!("missing fallback gamepad report") + }; + assert_eq!(controller_id, 1); + assert_eq!(bitmap, 0x0202); + assert_eq!(buttons, 0x1000); + assert_eq!(left_trigger, 7); + assert_eq!(left_stick_y, -12_000); + assert_eq!(right_stick_y, 12_000); + assert_eq!(handle.hid.pending_states(), 0); + handle + .hid + .set_session_admission(0b0010, handle.hid.inventory_snapshot().1); + assert_eq!(handle.hid.source_mode(1), Some(SourceMode::Rich)); + assert_eq!( + unsafe { opennow_streamer_submit_sony_snapshot(&handle, &snapshot) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(handle.hid.pending_states(), 1); + assert!(handle.input.queue().take().is_none()); + } + + #[test] + fn typed_submit_api_validates_boundaries_without_crossing_the_ffi() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_absolute(&handle, 0, 0, 0, 1) }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_submit_mouse_button(&handle, 6, true) }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_submit_gamepad(&handle, 4, 0, 0, 0, 0, 0, 0, 0, 0) }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_submit_local_action(&handle, u32::MAX) }, + OpenNowStreamerStatus::InvalidConfig + ); + assert_eq!( + unsafe { opennow_streamer_submit_key(ptr::null(), 0, 0, false) }, + OpenNowStreamerStatus::NullPointer + ); + handle.shutdown(); + } + + #[test] + fn graphics_ffi_waits_for_decoder_without_losing_context_or_masking_errors() { + use opennow_streamer_platform::GraphicsFrameError; + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + let context = ffi_graphics_context(); + assert_eq!( + unsafe { opennow_streamer_set_graphics_context(&handle, &context) }, + OpenNowStreamerStatus::Ok + ); + let publisher = handle.frame_publisher(); + let lease = publisher.context().unwrap(); + let recorded = Arc::new(RecordedCommands::default()); + let cases = [ + ( + Some(GraphicsFrameError::NotReady), + OpenNowStreamerStatus::NoFrame, + ), + ( + Some(GraphicsFrameError::NotReady), + OpenNowStreamerStatus::NoFrame, + ), + (None, OpenNowStreamerStatus::Ok), + ( + Some(GraphicsFrameError::Unsupported("no DMA-BUF import".into())), + OpenNowStreamerStatus::GraphicsUnsupported, + ), + ( + Some(GraphicsFrameError::Failed("device removed".into())), + OpenNowStreamerStatus::RenderFailed, + ), + ]; + for (index, (error, expected)) in cases.into_iter().enumerate() { + publisher + .publish( + lease, + Arc::new(TestGraphicsFrame { + sequence: index as u64, + recorded: Arc::clone(&recorded), + panic_on_record: false, + error, + }), + ) + .unwrap(); + let mut token = ptr::null_mut(); + let mut info = OpenNowStreamerFrameInfo::default(); + assert_eq!( + unsafe { opennow_streamer_acquire_latest_frame(&handle, &mut token, &mut info) }, + OpenNowStreamerStatus::Ok + ); + let mut output = OpenNowStreamerRecordedFrame::default(); + assert_eq!( + unsafe { + opennow_streamer_record_frame( + &handle, + token, + &ffi_render_command(), + &mut output, + ) + }, + expected + ); + assert_eq!(output.resource != 0, expected == OpenNowStreamerStatus::Ok); + assert_eq!( + unsafe { opennow_streamer_release_frame(token) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(recorded.drops.load(Ordering::Relaxed), index + 1); + } + assert_eq!(messages.frames_available.load(Ordering::Relaxed), 5); + assert_eq!(recorded.values.lock().unwrap().len(), 1); + assert_eq!( + unsafe { opennow_streamer_scene_graph_shutdown(&handle) }, + OpenNowStreamerStatus::Ok + ); + handle.shutdown(); + } + + #[test] + fn graphics_ffi_acquires_records_and_releases_the_latest_frame() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + let context = ffi_graphics_context(); + assert_eq!( + unsafe { opennow_streamer_set_graphics_context(&handle, &context) }, + OpenNowStreamerStatus::Ok + ); + let publisher = handle.frame_publisher(); + let lease = publisher.context().expect("graphics context lease"); + let recorded = Arc::new(RecordedCommands::default()); + publisher + .publish( + lease, + Arc::new(TestGraphicsFrame { + sequence: 42, + recorded: Arc::clone(&recorded), + panic_on_record: false, + error: None, + }), + ) + .expect("publish frame"); + assert_eq!(messages.frames_available.load(Ordering::Relaxed), 1); + + let mut token = ptr::null_mut(); + let mut info = OpenNowStreamerFrameInfo::default(); + assert_eq!( + unsafe { opennow_streamer_acquire_latest_frame(&handle, &mut token, &mut info) }, + OpenNowStreamerStatus::Ok + ); + assert!(!token.is_null()); + assert_eq!( + info, + OpenNowStreamerFrameInfo { + width: 2560, + height: 1440, + sequence: 42, + presentation_time_ns: 349_999_986, + } + ); + let command = ffi_render_command(); + let mut output = OpenNowStreamerRecordedFrame::default(); + assert_eq!( + unsafe { opennow_streamer_record_frame(&handle, token, &command, &mut output) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(output.graphics_api, OPENNOW_STREAMER_GRAPHICS_API_VULKAN); + assert_eq!( + output.texture_format, + OPENNOW_STREAMER_TEXTURE_FORMAT_RGB10A2 + ); + assert_eq!(output.width, 2560); + assert_eq!(output.height, 1440); + assert_eq!(output.frame_slot, 2); + assert_eq!(output.generation, 42); + assert_eq!(output.resource, 0xfeed); + assert_eq!(output.resource_view, 0xbeef); + assert_eq!(output.presentation_time_ns, 349_999_986); + assert_eq!( + unsafe { opennow_streamer_record_frame(&handle, token, &command, &mut output) }, + OpenNowStreamerStatus::FrameAlreadyRecorded + ); + assert_eq!(recorded.drops.load(Ordering::Relaxed), 0); + assert_eq!( + unsafe { opennow_streamer_release_frame(token) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(recorded.drops.load(Ordering::Relaxed), 1); + + let values = recorded.values.lock().expect("recorded commands"); + assert_eq!(values.len(), 1); + assert_eq!(values[0].0.api, GraphicsApi::Vulkan); + assert_eq!(values[0].0.device, context.device as usize); + assert_eq!(values[0].0.queue, context.queue as usize); + assert_eq!(values[0].1.command_buffer, command.command_buffer as usize); + assert_eq!(values[0].1.frame_slot, 2); + drop(values); + + assert_eq!( + unsafe { opennow_streamer_scene_graph_shutdown(&handle) }, + OpenNowStreamerStatus::Ok + ); + handle.shutdown(); + } + + #[test] + fn scene_graph_shutdown_invalidates_tokens_without_leaking_them() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + let context = ffi_graphics_context(); + assert_eq!( + unsafe { opennow_streamer_set_graphics_context(&handle, &context) }, + OpenNowStreamerStatus::Ok + ); + let publisher = handle.frame_publisher(); + let recorded = Arc::new(RecordedCommands::default()); + publisher + .publish( + publisher.context().expect("graphics context lease"), + Arc::new(TestGraphicsFrame { + sequence: 1, + recorded: Arc::clone(&recorded), + panic_on_record: false, + error: None, + }), + ) + .expect("publish frame"); + let mut token = ptr::null_mut(); + let mut info = OpenNowStreamerFrameInfo::default(); + assert_eq!( + unsafe { opennow_streamer_acquire_latest_frame(&handle, &mut token, &mut info) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_scene_graph_shutdown(&handle) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { + opennow_streamer_record_frame( + &handle, + token, + &ffi_render_command(), + &mut OpenNowStreamerRecordedFrame::default(), + ) + }, + OpenNowStreamerStatus::GraphicsUnavailable + ); + assert_eq!(recorded.drops.load(Ordering::Relaxed), 0); + assert_eq!( + unsafe { opennow_streamer_release_frame(token) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!(recorded.drops.load(Ordering::Relaxed), 1); + handle.shutdown(); + } + + #[test] + fn graphics_ffi_contains_panics_from_frame_recording() { + let messages = Box::new(CallbackMessages::default()); + let mut handle = graphics_test_handle(&messages); + assert_eq!( + unsafe { opennow_streamer_set_graphics_context(&handle, &ffi_graphics_context()) }, + OpenNowStreamerStatus::Ok + ); + let publisher = handle.frame_publisher(); + let recorded = Arc::new(RecordedCommands::default()); + publisher + .publish( + publisher.context().expect("graphics context lease"), + Arc::new(TestGraphicsFrame { + sequence: 1, + recorded, + panic_on_record: true, + error: None, + }), + ) + .expect("publish frame"); + let mut token = ptr::null_mut(); + let mut info = OpenNowStreamerFrameInfo::default(); + assert_eq!( + unsafe { opennow_streamer_acquire_latest_frame(&handle, &mut token, &mut info) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { + opennow_streamer_record_frame( + &handle, + token, + &ffi_render_command(), + &mut OpenNowStreamerRecordedFrame::default(), + ) + }, + OpenNowStreamerStatus::Panic + ); + assert_eq!( + unsafe { opennow_streamer_release_frame(token) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_scene_graph_shutdown(&handle) }, + OpenNowStreamerStatus::Ok + ); + handle.shutdown(); + } + + #[test] + fn destroy_rejects_an_active_scene_graph_without_consuming_the_handle() { + let messages = Box::new(CallbackMessages::default()); + let handle = Box::into_raw(Box::new(graphics_test_handle(&messages))); + assert_eq!( + unsafe { opennow_streamer_set_graphics_context(handle, &ffi_graphics_context()) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_destroy(handle) }, + OpenNowStreamerStatus::SceneGraphActive + ); + assert_eq!( + unsafe { opennow_streamer_scene_graph_shutdown(handle) }, + OpenNowStreamerStatus::Ok + ); + assert_eq!( + unsafe { opennow_streamer_destroy(handle) }, + OpenNowStreamerStatus::Ok + ); + } + + #[test] + fn hello_round_trips_through_the_response_callback_with_test_media_runtime() { + let messages = Box::new(CallbackMessages::default()); + let (mut handle, host) = create_with_test_runtime(&messages); + let command = serde_json::to_vec(&json!({ + "id": "hello-1", + "type": "hello", + "protocolVersion": 7 + })) + .expect("hello command"); + + assert_eq!(handle.send(&command), OpenNowStreamerStatus::Ok); + let response = messages.wait_for_id("hello-1"); + assert_eq!(response["type"], "ready"); + assert!( + response["capabilities"] + .get("supportsOfferAnswer") + .is_none() + ); + assert!(response["capabilities"].get("microphoneDevices").is_none()); + + handle.shutdown(); + host.join().expect("test media runtime"); + } + + #[test] + fn audio_mute_round_trips_before_start_and_after_stop() { + let messages = Box::new(CallbackMessages::default()); + let (mut handle, host) = create_with_test_runtime(&messages); + for (id, command) in [ + ( + "mute-before-start", + json!({"type": "setAudioMuted", "muted": true}), + ), + ("stop-muted", json!({"type": "stop"})), + ( + "unmute-after-stop", + json!({"type": "setAudioMuted", "muted": false}), + ), + ] { + let mut command = command; + command["id"] = json!(id); + assert_eq!( + handle.send(&serde_json::to_vec(&command).unwrap()), + OpenNowStreamerStatus::Ok + ); + assert_eq!(messages.wait_for_id(id)["type"], "ok"); + } + assert_eq!( + handle.send(br#"{"id":"missing-muted","type":"setAudioMuted"}"#), + OpenNowStreamerStatus::Ok + ); + assert_eq!( + messages.wait_for_id("missing-muted")["code"], + "missing-muted" + ); + handle.shutdown(); + host.join().expect("test media runtime"); + } + + #[test] + fn audio_devices_round_trips_without_changing_the_media_runtime() { + let messages = Box::new(CallbackMessages::default()); + let (mut handle, host) = create_with_test_runtime(&messages); + assert_eq!( + handle.send(br#"{"id":"audio-1","type":"audioDevices"}"#), + OpenNowStreamerStatus::Ok + ); + let response = messages.wait_for_id("audio-1"); + assert_eq!(response["type"], "audioDevices"); + assert_eq!(response["devices"], json!([])); + assert_eq!( + handle.send(br#"{"id":"hello-after-audio","type":"hello","protocolVersion":7}"#), + OpenNowStreamerStatus::Ok + ); + assert_eq!(messages.wait_for_id("hello-after-audio")["type"], "ready"); + handle.shutdown(); + host.join().expect("test media runtime"); + } + + #[test] + fn public_constructor_installs_the_production_in_process_media_runtime() { + let messages = Box::new(CallbackMessages::default()); + let config = test_config(&messages); + let mut handle = ptr::null_mut(); + assert_eq!( + unsafe { opennow_streamer_create(&config, &mut handle) }, + OpenNowStreamerStatus::Ok + ); + assert!(!handle.is_null()); + let command = serde_json::to_vec(&json!({ + "id": "hello-production", + "type": "hello", + "protocolVersion": 7 + })) + .expect("hello command"); + assert_eq!( + unsafe { &*handle }.send(&command), + OpenNowStreamerStatus::Ok + ); + let response = messages.wait_for_id("hello-production"); + assert!( + response["capabilities"] + .get("supportsOfferAnswer") + .is_none() + ); + assert_eq!(response["capabilities"]["supportsAudioDecode"], true); + assert_eq!(response["capabilities"]["supportsAudioOutput"], true); + assert_eq!( + unsafe { opennow_streamer_destroy(handle) }, + OpenNowStreamerStatus::Ok + ); + } + + #[test] + fn replay_commands_cross_ffi_without_activating_disabled_capture() { + let messages = Box::new(CallbackMessages::default()); + let (mut handle, host) = create_with_test_runtime(&messages); + let path = std::env::temp_dir().join("opennow-ffi-clip.mkv"); + let command = + serde_json::to_vec(&json!({"id":"clip-disabled","type":"clip-save","outputPath":path})) + .unwrap(); + assert_eq!(handle.send(&command), OpenNowStreamerStatus::Ok); + let response = messages.wait_for_id("clip-disabled"); + assert_eq!(response["code"], "replay-not-enabled"); + assert_eq!( + handle.send(br#"{"id":"replay-stop","type":"replay-stop"}"#), + OpenNowStreamerStatus::Ok + ); + assert_eq!( + messages.wait_for_id("replay-stop")["type"], + "replay-stopped" + ); + handle.shutdown(); + host.join().unwrap(); + } + + #[test] + fn shutdown_responds_and_closes_the_command_queue() { + let messages = Box::new(CallbackMessages::default()); + let (mut handle, host) = create_with_test_runtime(&messages); + let command = br#"{"id":"shutdown-1","type":"shutdown","reason":"ffi test"}"#; + + assert_eq!(handle.send(command), OpenNowStreamerStatus::Ok); + let response = messages.wait_for_id("shutdown-1"); + assert_eq!(response["type"], "ok"); + let deadline = Instant::now() + Duration::from_secs(5); + while handle.send(br#"{"id":"late","type":"hello"}"#) != OpenNowStreamerStatus::Closed { + assert!(Instant::now() < deadline, "command queue did not close"); + thread::yield_now(); + } + + handle.shutdown(); + host.join().expect("test media runtime"); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/Cargo.toml new file mode 100644 index 000000000..d05560111 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "opennow-streamer-hid" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[features] +default = [] +test-runtime = [] + +[dependencies] diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/allowlist.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/allowlist.rs new file mode 100644 index 000000000..0389eebc8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/allowlist.rs @@ -0,0 +1,62 @@ +pub const GENERIC_HID_ALLOWLIST: [(u16, u16); 18] = [ + (0x044f, 0x0402), + (0x044f, 0x0404), + (0x044f, 0xb67c), + (0x044f, 0xb687), + (0x044f, 0xb68d), + (0x044f, 0xb68f), + (0x044f, 0xb697), + (0x046d, 0xc24f), + (0x046d, 0xc262), + (0x046d, 0xc266), + (0x046d, 0xc268), + (0x046d, 0xc26e), + (0x046d, 0xc272), + (0x046d, 0xc276), + (0x06a3, 0x075c), + (0x06a3, 0x0762), + (0x0738, 0x2221), + (0x0738, 0xa221), +]; + +pub fn generic_hid_allowed(vendor: u16, product: u16) -> bool { + GENERIC_HID_ALLOWLIST.contains(&(vendor, product)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn allowlist_is_exact_and_does_not_admit_entire_vendors() { + let expected = [ + (0x044f, 0x0402), + (0x044f, 0x0404), + (0x044f, 0xb67c), + (0x044f, 0xb687), + (0x044f, 0xb68d), + (0x044f, 0xb68f), + (0x044f, 0xb697), + (0x046d, 0xc24f), + (0x046d, 0xc262), + (0x046d, 0xc266), + (0x046d, 0xc268), + (0x046d, 0xc26e), + (0x046d, 0xc272), + (0x046d, 0xc276), + (0x06a3, 0x075c), + (0x06a3, 0x0762), + (0x0738, 0x2221), + (0x0738, 0xa221), + ]; + assert_eq!(GENERIC_HID_ALLOWLIST, expected); + for (vendor, product) in expected { + assert!(generic_hid_allowed(vendor, product)); + assert!(!generic_hid_allowed(vendor, product ^ 1)); + assert!(!generic_hid_allowed(vendor ^ 0x8000, product)); + } + assert!(!generic_hid_allowed(0x054c, 0x05c4)); + assert!(!generic_hid_allowed(0x044f, 0xb10a)); + assert!(!generic_hid_allowed(0x046d, 0xffff)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/descriptor.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/descriptor.rs new file mode 100644 index 000000000..2c049cea7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/descriptor.rs @@ -0,0 +1,772 @@ +use std::collections::BTreeMap; +use std::fmt; + +const MAX_DESCRIPTOR_BYTES: usize = 4096; +const MAX_REPORT_BYTES: usize = 256; +const MAX_REPORT_IDS: usize = 64; +const MAX_FIELDS: usize = 1024; +const MAX_USAGES: usize = 1024; +const MAX_DEPTH: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DescriptorError { + Malformed, + Unsupported, + Ambiguous, + LimitExceeded, + UnknownReportId(u8), + ShortReport, + ReportTooLong, +} + +impl fmt::Display for DescriptorError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Malformed => formatter.write_str("malformed HID descriptor"), + Self::Unsupported => formatter.write_str("unsupported HID descriptor item"), + Self::Ambiguous => formatter.write_str("ambiguous HID report layout"), + Self::LimitExceeded => formatter.write_str("HID descriptor exceeds bounded limits"), + Self::UnknownReportId(id) => write!(formatter, "unknown HID input report ID {id}"), + Self::ShortReport => formatter.write_str("HID input report is too short"), + Self::ReportTooLong => formatter.write_str("HID input report exceeds 256 bytes"), + } + } +} + +impl std::error::Error for DescriptorError {} + +#[derive(Debug, Clone)] +pub struct ReportDescriptor { + reports: BTreeMap, + numbered: bool, +} + +#[derive(Debug, Clone, Default)] +struct ReportLayout { + bits: [usize; 3], + release_fields: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct ReleaseField { + offset: usize, + width: usize, + value: u32, +} + +#[derive(Clone, Copy, Default)] +struct Globals { + usage_page: u16, + logical_min: i64, + logical_max: i64, + size: usize, + count: usize, + report_id: u8, +} + +#[derive(Default)] +struct Locals { + usages: Vec, + minimum: Option, +} + +impl Locals { + fn usage(&mut self, usage: u32) -> Result<(), DescriptorError> { + if self.minimum.is_some() { + return Err(DescriptorError::Ambiguous); + } + if self.usages.len() == MAX_USAGES { + return Err(DescriptorError::LimitExceeded); + } + self.usages.push(usage); + Ok(()) + } + + fn maximum(&mut self, maximum: u32) -> Result<(), DescriptorError> { + let minimum = self.minimum.take().ok_or(DescriptorError::Malformed)?; + if minimum > maximum || minimum >> 16 != maximum >> 16 { + return Err(DescriptorError::Malformed); + } + let count = (maximum - minimum) as usize + 1; + if count > MAX_USAGES - self.usages.len() { + return Err(DescriptorError::LimitExceeded); + } + self.usages.extend(minimum..=maximum); + Ok(()) + } +} + +impl ReportDescriptor { + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DESCRIPTOR_BYTES { + return Err(DescriptorError::LimitExceeded); + } + let mut descriptor = Self { + reports: BTreeMap::new(), + numbered: false, + }; + let mut globals = Globals::default(); + let mut stack = Vec::new(); + let mut locals = Locals::default(); + let mut depth = 0_usize; + let mut field_count = 0_usize; + let mut cursor = 0; + while cursor < bytes.len() { + let prefix = bytes[cursor]; + cursor += 1; + if prefix == 0xfe { + return Err(DescriptorError::Unsupported); + } + let length = match prefix & 3 { + 3 => 4, + length => usize::from(length), + }; + let data = bytes + .get(cursor..cursor + length) + .ok_or(DescriptorError::Malformed)?; + cursor += length; + let value = unsigned(data); + let tag = prefix >> 4; + match (prefix >> 2) & 3 { + 0 => { + if locals.minimum.is_some() { + return Err(DescriptorError::Malformed); + } + match tag { + 8 | 9 | 11 => { + if length == 0 || depth == 0 { + return Err(DescriptorError::Malformed); + } + if value & !0x1ff != 0 || value & 0x100 != 0 { + return Err(DescriptorError::Unsupported); + } + let kind = match tag { + 8 => 0, + 9 => 1, + _ => 2, + }; + descriptor.add_fields( + globals, + &locals, + kind, + value, + &mut field_count, + )?; + } + 10 => { + if length != 1 || value > 6 { + return Err(DescriptorError::Unsupported); + } + if depth == MAX_DEPTH { + return Err(DescriptorError::LimitExceeded); + } + depth += 1; + } + 12 => { + if length != 0 || depth == 0 { + return Err(DescriptorError::Malformed); + } + depth -= 1; + } + _ => return Err(DescriptorError::Unsupported), + } + locals = Locals::default(); + } + 1 => match tag { + 10 => { + if length != 0 { + return Err(DescriptorError::Malformed); + } + if stack.len() == MAX_DEPTH { + return Err(DescriptorError::LimitExceeded); + } + stack.push(globals); + } + 11 => { + if length != 0 { + return Err(DescriptorError::Malformed); + } + globals = stack.pop().ok_or(DescriptorError::Malformed)?; + } + 0..=9 => { + if length == 0 { + return Err(DescriptorError::Malformed); + } + match tag { + 0 => { + globals.usage_page = u16::try_from(value) + .map_err(|_| DescriptorError::Unsupported)?; + } + 1 => globals.logical_min = signed(data), + 2 => { + globals.logical_max = if globals.logical_min < 0 { + signed(data) + } else { + i64::from(value) + }; + } + 3..=6 => {} + 7 => globals.size = value as usize, + 8 => { + if length != 1 || value == 0 { + return Err(DescriptorError::Malformed); + } + globals.report_id = value as u8; + descriptor.numbered = true; + descriptor.layout(globals.report_id)?; + } + 9 => globals.count = value as usize, + _ => unreachable!(), + } + } + _ => return Err(DescriptorError::Unsupported), + }, + 2 => { + if length == 0 { + return Err(DescriptorError::Malformed); + } + let usage = if length == 4 { + value + } else { + (u32::from(globals.usage_page) << 16) | value + }; + match tag { + 0 => locals.usage(usage)?, + 1 => { + if locals.minimum.replace(usage).is_some() { + return Err(DescriptorError::Malformed); + } + } + 2 => locals.maximum(usage)?, + 3..=5 | 7..=9 => {} + _ => return Err(DescriptorError::Unsupported), + } + } + _ => return Err(DescriptorError::Unsupported), + } + } + if depth != 0 || !stack.is_empty() || locals.minimum.is_some() || !locals.usages.is_empty() + { + return Err(DescriptorError::Malformed); + } + if descriptor.numbered && descriptor.reports.contains_key(&0) { + return Err(DescriptorError::Ambiguous); + } + if !descriptor + .reports + .values() + .any(|report| report.bits[0] != 0) + { + return Err(DescriptorError::Malformed); + } + Ok(descriptor) + } + + pub fn report_id(&self, report: &[u8]) -> Result { + if report.len() > MAX_REPORT_BYTES { + return Err(DescriptorError::ReportTooLong); + } + let id = if self.numbered { + *report.first().ok_or(DescriptorError::ShortReport)? + } else { + 0 + }; + let layout = self + .reports + .get(&id) + .filter(|layout| layout.bits[0] != 0) + .ok_or(DescriptorError::UnknownReportId(id))?; + let required = layout.bits[0].div_ceil(8) + usize::from(self.numbered); + if report.len() < required { + return Err(DescriptorError::ShortReport); + } + Ok(id) + } + + pub fn release(&self, report: &[u8]) -> Result, DescriptorError> { + let id = self.report_id(report)?; + let layout = &self.reports[&id]; + let mut released = report.to_vec(); + let payload_offset = usize::from(self.numbered) * 8; + for field in &layout.release_fields { + for bit in 0..field.width { + let offset = payload_offset + field.offset + bit; + let mask = 1 << (offset % 8); + let byte = &mut released[offset / 8]; + if field.value & (1 << bit) != 0 { + *byte |= mask; + } else { + *byte &= !mask; + } + } + } + Ok(released) + } + + fn layout(&mut self, id: u8) -> Result<&mut ReportLayout, DescriptorError> { + if !self.reports.contains_key(&id) && self.reports.len() == MAX_REPORT_IDS { + return Err(DescriptorError::LimitExceeded); + } + Ok(self.reports.entry(id).or_default()) + } + + fn add_fields( + &mut self, + globals: Globals, + locals: &Locals, + kind: usize, + flags: u32, + field_count: &mut usize, + ) -> Result<(), DescriptorError> { + if globals.size == 0 || globals.count == 0 { + return Err(DescriptorError::Malformed); + } + if globals.size > 32 || globals.count > MAX_FIELDS - *field_count { + return Err(DescriptorError::LimitExceeded); + } + *field_count += globals.count; + let numbered = self.numbered || globals.report_id != 0; + let report = self.layout(globals.report_id)?; + let offset = report.bits[kind]; + let bits = globals.size * globals.count; + let next = offset + bits; + if next.div_ceil(8) + usize::from(numbered) > MAX_REPORT_BYTES { + return Err(DescriptorError::LimitExceeded); + } + report.bits[kind] = next; + if flags & 1 != 0 { + return Ok(()); + } + if locals.usages.is_empty() || globals.logical_min > globals.logical_max { + return Err(DescriptorError::Malformed); + } + let (minimum, maximum) = field_range(globals.size, globals.logical_min < 0); + if globals.logical_min < minimum || globals.logical_max > maximum { + return Err(DescriptorError::Malformed); + } + if kind != 0 { + return Ok(()); + } + let variable = flags & 2 != 0; + if variable && locals.usages.len() > globals.count { + return Err(DescriptorError::Ambiguous); + } + if !variable { + let all_buttons = locals.usages.iter().all(|usage| usage >> 16 == 9); + let any_buttons = locals.usages.iter().any(|usage| usage >> 16 == 9); + let any_hats = locals.usages.contains(&0x0001_0039); + if any_hats || any_buttons != all_buttons { + return Err(DescriptorError::Ambiguous); + } + } + for index in 0..globals.count { + let usage = locals.usages[index.min(locals.usages.len() - 1)]; + let value = if usage >> 16 == 9 { + Some(0) + } else if variable && usage == 0x0001_0039 && flags & 0x40 != 0 { + let null = if globals.logical_max < maximum { + globals.logical_max + 1 + } else if globals.logical_min > minimum { + globals.logical_min - 1 + } else { + return Err(DescriptorError::Unsupported); + }; + Some(null as u32) + } else { + None + }; + if let Some(value) = value { + report.release_fields.push(ReleaseField { + offset: offset + index * globals.size, + width: globals.size, + value, + }); + } + } + Ok(()) + } +} + +fn unsigned(data: &[u8]) -> u32 { + data.iter().enumerate().fold(0, |value, (index, byte)| { + value | (u32::from(*byte) << (index * 8)) + }) +} + +fn signed(data: &[u8]) -> i64 { + let value = i64::from(unsigned(data)); + let bits = data.len() * 8; + if value & (1 << (bits - 1)) != 0 { + value - (1 << bits) + } else { + value + } +} + +fn field_range(width: usize, signed: bool) -> (i64, i64) { + if signed { + (-(1_i64 << (width - 1)), (1_i64 << (width - 1)) - 1) + } else { + (0, (1_i64 << width) - 1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Synthetic fixtures are constructed here; report examples are synthetic even for real descriptors. + const SYNTHETIC_PACKED: &[u8] = &[ + 0x05, 0x01, 0x09, 0x04, 0xa1, 0x01, 0x85, 0x07, 0x05, 0x09, 0x19, 0x01, 0x29, 0x03, 0x15, + 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x03, 0x81, 0x02, 0x75, 0x02, 0x95, 0x01, 0x81, 0x01, + 0x05, 0x01, 0x09, 0x39, 0x25, 0x07, 0x75, 0x04, 0x81, 0x42, 0x09, 0x30, 0x26, 0xff, 0x00, + 0x75, 0x08, 0x81, 0x02, 0x09, 0x39, 0x25, 0x07, 0x75, 0x04, 0x81, 0x02, 0x75, 0x03, 0x81, + 0x01, 0xc0, + ]; + + // Real 044f:b351 Cougar MFD descriptor published by Luiz Renault, March 28, 2022: + // https://forum.dcs.world/topic/297091-in-game-thrustmaster-warthog-and-cougar-mfd-brightness-control-mod/ + const REAL_COUGAR_MFD: &[u8] = &[ + 0x05, 0x01, 0x09, 0x05, 0xa1, 0x01, 0x05, 0x09, 0x19, 0x01, 0x29, 0x1c, 0x15, 0x00, 0x25, + 0x01, 0x35, 0x00, 0x45, 0x01, 0x75, 0x01, 0x95, 0x1c, 0x81, 0x02, 0x75, 0x04, 0x95, 0x01, + 0x81, 0x01, 0x06, 0x00, 0xff, 0x09, 0x01, 0x75, 0x10, 0x95, 0x01, 0xb1, 0x02, 0xc0, + ]; + + // Real 044f:b315 Dual Analog 4, USB Device Tree Viewer capture published January 4, 2014: + // http://www.classiccomp.de/Reste/GamePads/GP3.html + const REAL_DUAL_ANALOG_4: &[u8] = &[ + 0x05, 0x01, 0x09, 0x05, 0xa1, 0x01, 0x05, 0x01, 0x09, 0x01, 0xa1, 0x00, 0x05, 0x09, 0x19, + 0x01, 0x29, 0x0c, 0x15, 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x0c, 0x81, 0x02, 0x75, 0x08, + 0x95, 0x01, 0x81, 0x01, 0x05, 0x01, 0x09, 0x39, 0x25, 0x07, 0x35, 0x00, 0x46, 0x0e, 0x01, + 0x66, 0x40, 0x00, 0x75, 0x04, 0x81, 0x42, 0x09, 0x30, 0x09, 0x31, 0x15, 0x80, 0x25, 0x7f, + 0x46, 0xff, 0x00, 0x66, 0x00, 0x00, 0x75, 0x08, 0x95, 0x02, 0x81, 0x02, 0x09, 0x35, 0x95, + 0x01, 0x81, 0x02, 0x09, 0x36, 0x16, 0x00, 0x00, 0x26, 0xff, 0x00, 0x81, 0x02, 0x09, 0xbb, + 0x15, 0x00, 0x26, 0xff, 0x00, 0x35, 0x00, 0x46, 0xff, 0x00, 0x75, 0x08, 0x95, 0x04, 0x91, + 0x02, 0xc0, 0xc0, + ]; + + // Real allowlisted 044f:0402 Warthog stick, sysfs capture published by ho0ver, January 2024: + // https://forums.x-plane.org/forums/topic/299033-xp12-linux-winwing-orion-2-throttle-has-too-many-buttons/ + // The posted lsusb line is a different device; the capture path and later correction identify the stick. + const REAL_WARTHOG_STICK: &[u8] = &[ + 0x05, 0x01, 0x09, 0x04, 0xa1, 0x01, 0x85, 0x01, 0x05, 0x09, 0x19, 0x01, 0x29, 0x13, 0x15, + 0x00, 0x25, 0x01, 0x35, 0x00, 0x45, 0x01, 0x75, 0x01, 0x95, 0x13, 0x81, 0x02, 0x75, 0x01, + 0x95, 0x01, 0x81, 0x01, 0x05, 0x01, 0x09, 0x39, 0x25, 0x07, 0x46, 0x3b, 0x01, 0x65, 0x14, + 0x75, 0x04, 0x81, 0x42, 0x09, 0x30, 0x09, 0x31, 0x65, 0x00, 0x27, 0xff, 0xff, 0x00, 0x00, + 0x47, 0xff, 0xff, 0x00, 0x00, 0x75, 0x10, 0x95, 0x02, 0x81, 0x02, 0x06, 0xff, 0x00, 0x09, + 0x01, 0x09, 0x02, 0x81, 0x02, 0x09, 0xbb, 0x75, 0x08, 0x95, 0x0b, 0x26, 0xff, 0x00, 0x46, + 0xff, 0x00, 0x91, 0x02, 0x85, 0x02, 0x06, 0xff, 0x00, 0x09, 0x03, 0x81, 0x02, 0xc0, + ]; + + fn synthetic(fields: &[u8]) -> Vec { + let mut bytes = vec![0x05, 0x01, 0x09, 0x04, 0xa1, 0x01]; + bytes.extend_from_slice(fields); + bytes.push(0xc0); + bytes + } + + fn button_fields() -> Vec { + vec![ + 0x05, 0x09, 0x19, 0x01, 0x29, 0x08, 0x15, 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x08, + 0x81, 0x02, + ] + } + + #[test] + fn packed_release_preserves_axes_constants_non_nullable_hat_and_id() { + let descriptor = ReportDescriptor::parse(SYNTHETIC_PACKED).unwrap(); + let original = [0x07, 0x7f, 0xaa, 0xaa]; + assert_eq!(descriptor.report_id(&original), Ok(7)); + assert_eq!( + descriptor.release(&original).unwrap(), + [0x07, 0x18, 0xab, 0xaa] + ); + assert_eq!(original, [0x07, 0x7f, 0xaa, 0xaa]); + let released = descriptor.release(&original).unwrap(); + assert_eq!(descriptor.release(&released).unwrap(), released); + } + + #[test] + fn published_real_descriptors_have_exact_release_bytes() { + assert_eq!(REAL_COUGAR_MFD.len(), 44); + let mfd = ReportDescriptor::parse(REAL_COUGAR_MFD).unwrap(); + assert_eq!(mfd.report_id(&[0xff; 4]), Ok(0)); + assert_eq!(mfd.release(&[0xff; 4]).unwrap(), [0, 0, 0, 0xf0]); + assert_eq!(REAL_DUAL_ANALOG_4.len(), 108); + let pad = ReportDescriptor::parse(REAL_DUAL_ANALOG_4).unwrap(); + assert_eq!( + pad.release(&[0xff, 0xdf, 0x35, 0x80, 0x7f, 0xab, 0xcd, 0xee]) + .unwrap(), + [0x00, 0xd0, 0x85, 0x80, 0x7f, 0xab, 0xcd, 0xee] + ); + assert_eq!(REAL_WARTHOG_STICK.len(), 104); + let stick = ReportDescriptor::parse(REAL_WARTHOG_STICK).unwrap(); + assert_eq!( + stick + .release(&[ + 1, 0xff, 0xff, 0x3f, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef + ]) + .unwrap(), + [ + 1, 0, 0, 0x88, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef + ] + ); + assert_eq!(stick.release(&[2; 12]).unwrap(), [2; 12]); + } + + #[test] + fn push_pop_restores_report_id_usage_page_size_and_logical_bounds() { + let mut fields = vec![0x85, 0x01]; + fields.extend(button_fields()); + fields.extend([ + 0xa4, 0x85, 0x02, 0x05, 0x01, 0x09, 0x30, 0x15, 0x80, 0x25, 0x7f, 0x75, 0x08, 0x95, + 0x01, 0x81, 0x02, 0xb4, 0x09, 0x01, 0x81, 0x02, + ]); + let descriptor = ReportDescriptor::parse(&synthetic(&fields)).unwrap(); + assert_eq!(descriptor.release(&[1, 0xff, 0xff]).unwrap(), [1, 0, 0]); + assert_eq!(descriptor.release(&[2, 0x81]).unwrap(), [2, 0x81]); + assert_eq!(descriptor.report_id(&[2, 0x81]), Ok(2)); + } + + #[test] + fn usage_lists_ranges_and_extended_usages_clear_only_button_fields() { + let descriptor = ReportDescriptor::parse(&synthetic(&[ + 0x15, 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x06, 0x0b, 0x01, 0x00, 0x09, 0x00, 0x1b, + 0x02, 0x00, 0x09, 0x00, 0x2b, 0x04, 0x00, 0x09, 0x00, 0x0b, 0x30, 0x00, 0x01, 0x00, + 0x0b, 0x05, 0x00, 0x09, 0x00, 0x81, 0x02, 0x75, 0x02, 0x95, 0x01, 0x81, 0x01, + ])) + .unwrap(); + assert_eq!(descriptor.release(&[0xff]).unwrap(), [0xd0]); + } + + #[test] + fn button_arrays_clear_each_slot_and_do_not_change_vendor_arrays() { + let descriptor = ReportDescriptor::parse(&synthetic(&[ + 0x05, 0x09, 0x19, 0x00, 0x29, 0x0f, 0x15, 0x00, 0x25, 0x0f, 0x75, 0x04, 0x95, 0x03, + 0x81, 0x00, 0x06, 0x00, 0xff, 0x19, 0x00, 0x29, 0x0f, 0x95, 0x01, 0x81, 0x00, + ])) + .unwrap(); + assert_eq!(descriptor.release(&[0x21, 0xb3]).unwrap(), [0, 0xb0]); + } + + #[test] + fn unsigned_and_signed_hat_nulls_choose_a_representable_outside_value() { + let descriptor = ReportDescriptor::parse(&synthetic(&[ + 0x09, 0x39, 0x15, 0x01, 0x25, 0x0f, 0x75, 0x04, 0x95, 0x01, 0x81, 0x42, 0x09, 0x39, + 0x15, 0xfe, 0x25, 0x07, 0x81, 0x42, 0x09, 0x39, 0x15, 0xfc, 0x25, 0x03, 0x81, 0x42, + 0x75, 0x04, 0x81, 0x01, + ])) + .unwrap(); + assert_eq!(descriptor.release(&[0x51, 0xa2]).unwrap(), [0xd0, 0xa4]); + let full_range = synthetic(&[ + 0x09, 0x39, 0x15, 0x00, 0x25, 0x0f, 0x75, 0x04, 0x95, 0x01, 0x81, 0x42, + ]); + assert!(matches!( + ReportDescriptor::parse(&full_range), + Err(DescriptorError::Unsupported) + )); + } + + #[test] + fn signed_16_and_32_bit_axes_and_32_bit_hat_are_not_truncated() { + let descriptor = ReportDescriptor::parse(&synthetic(&[ + 0x09, 0x30, 0x16, 0x00, 0x80, 0x26, 0xff, 0x7f, 0x75, 0x10, 0x95, 0x01, 0x81, 0x02, + 0x09, 0x31, 0x17, 0x00, 0x00, 0x00, 0x80, 0x27, 0xff, 0xff, 0xff, 0x7f, 0x75, 0x20, + 0x81, 0x02, 0x09, 0x39, 0x17, 0x00, 0x00, 0x00, 0x00, 0x27, 0xfe, 0xff, 0xff, 0xff, + 0x81, 0x42, + ])) + .unwrap(); + assert_eq!( + descriptor + .release(&[0x00, 0x80, 0xff, 0xff, 0xff, 0x7f, 0x42, 0, 0, 0]) + .unwrap(), + [0x00, 0x80, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff] + ); + } + + #[test] + fn collection_boundaries_reset_locals_but_preserve_report_offsets() { + let mut descriptor = synthetic(&button_fields()); + descriptor.extend(synthetic(&[ + 0x09, 0x30, 0x15, 0x00, 0x26, 0xff, 0x00, 0x75, 0x08, 0x95, 0x01, 0x81, 0x02, + ])); + let parsed = ReportDescriptor::parse(&descriptor).unwrap(); + assert_eq!(parsed.release(&[0xff, 0x67]).unwrap(), [0, 0x67]); + let missing_usage = synthetic(&[0x75, 0x01, 0x95, 0x01, 0x15, 0, 0x25, 1, 0x81, 2]); + assert!(matches!( + ReportDescriptor::parse(&missing_usage), + Err(DescriptorError::Malformed) + )); + let mut missing_next_usage = button_fields(); + missing_next_usage.extend([0x81, 2]); + assert!(ReportDescriptor::parse(&synthetic(&missing_next_usage)).is_err()); + } + + #[test] + fn output_and_feature_layouts_do_not_shift_input_offsets_or_admit_output_only_ids() { + let mut fields = vec![0x85, 0x01]; + fields.extend(button_fields()); + fields.extend([0x09, 0x01, 0x91, 0x02, 0x09, 0x01, 0xb1, 0x02]); + fields.extend([0x85, 0x02, 0x09, 0x01, 0x91, 0x02]); + let descriptor = ReportDescriptor::parse(&synthetic(&fields)).unwrap(); + assert_eq!(descriptor.release(&[1, 0xff]).unwrap(), [1, 0]); + assert_eq!( + descriptor.release(&[2, 0xff]), + Err(DescriptorError::UnknownReportId(2)) + ); + } + + #[test] + fn report_errors_fail_closed_and_padding_is_preserved() { + let numbered = ReportDescriptor::parse(SYNTHETIC_PACKED).unwrap(); + for report in [&[][..], &[7][..], &[7, 0xff, 0xff][..]] { + assert_eq!(numbered.release(report), Err(DescriptorError::ShortReport)); + assert_eq!( + numbered.report_id(report), + Err(DescriptorError::ShortReport) + ); + } + assert_eq!( + numbered.release(&[8; 4]), + Err(DescriptorError::UnknownReportId(8)) + ); + assert_eq!( + numbered.release(&[0; 4]), + Err(DescriptorError::UnknownReportId(0)) + ); + assert_eq!( + numbered.release(&[7; 257]), + Err(DescriptorError::ReportTooLong) + ); + let implicit = ReportDescriptor::parse(&synthetic(&button_fields())).unwrap(); + assert_eq!(implicit.release(&[]), Err(DescriptorError::ShortReport)); + assert_eq!(implicit.release(&[0xff, 0xab]).unwrap(), [0, 0xab]); + assert_eq!(implicit.report_id(&[0xff]), Ok(0)); + } + + #[test] + fn malformed_unsupported_and_ambiguous_items_are_rejected() { + let mut mixed_ids = button_fields(); + mixed_ids.extend([0x85, 0x01, 0x09, 0x01, 0x81, 0x02]); + assert!(matches!( + ReportDescriptor::parse(&synthetic(&mixed_ids)), + Err(DescriptorError::Ambiguous) + )); + let cases: &[&[u8]] = &[ + &[], + &[0x05], + &[0x05, 1], + &[0xfe, 0, 0], + &[0x0c], + &[0xc0], + &[0xa1, 1], + &[0xa4], + &[0xb4], + &[0x85, 0], + &[0x86, 1, 0], + &[0x75, 1, 0x95, 1, 0x81, 1], + ]; + for bytes in cases { + assert!(ReportDescriptor::parse(bytes).is_err(), "{bytes:02x?}"); + } + for bad_fields in [ + vec![0xa4], + vec![0xa1, 0], + vec![0xc0], + vec![0xa9, 1], + vec![0x19, 2, 0x29, 1], + vec![0x19, 1], + vec![0x29, 1], + vec![0x19, 1, 0x19, 2, 0x29, 3], + vec![0x19, 1, 0x09, 2], + vec![0x09, 0x30, 0x15, 2, 0x25, 1, 0x75, 1, 0x95, 1, 0x81, 2], + vec![0x09, 0x30, 0x15, 0, 0x25, 4, 0x75, 1, 0x95, 1, 0x81, 2], + vec![0x09, 0x39, 0x15, 0, 0x25, 7, 0x75, 4, 0x95, 1, 0x81, 0x40], + vec![0x0b, 1, 0, 9, 0, 0x09, 0x30, 0x75, 1, 0x95, 2, 0x81, 0], + vec![0x09, 0x30, 0x09, 0x31, 0x75, 1, 0x95, 1, 0x81, 2], + vec![0x75, 1, 0x95, 1, 0x82, 1, 1], + ] { + assert!( + ReportDescriptor::parse(&synthetic(&bad_fields)).is_err(), + "{bad_fields:02x?}" + ); + } + } + + #[test] + fn descriptor_report_field_usage_and_nesting_limits_are_bounded() { + assert!(matches!( + ReportDescriptor::parse(&[0; 4097]), + Err(DescriptorError::LimitExceeded) + )); + let mut ids = Vec::new(); + for id in 1..=65 { + ids.extend([0x85, id]); + } + assert!(matches!( + ReportDescriptor::parse(&synthetic(&ids)), + Err(DescriptorError::LimitExceeded) + )); + ids.truncate(64 * 2); + ids.extend(button_fields()); + assert!(ReportDescriptor::parse(&synthetic(&ids)).is_ok()); + let payload = synthetic(&[0x75, 32, 0x95, 64, 0x81, 1]); + let descriptor = ReportDescriptor::parse(&payload).unwrap(); + assert_eq!(descriptor.release(&[0xab; 256]).unwrap(), [0xab; 256]); + let too_long = synthetic(&[0x85, 1, 0x75, 32, 0x95, 64, 0x81, 1]); + assert!(matches!( + ReportDescriptor::parse(&too_long), + Err(DescriptorError::LimitExceeded) + )); + for fields in [ + vec![0x75, 33, 0x95, 1, 0x81, 1], + vec![0x75, 1, 0x96, 1, 4, 0x81, 1], + vec![0x19, 0, 0x2a, 0, 4], + vec![ + 0x77, 0xff, 0xff, 0xff, 0xff, 0x97, 0xff, 0xff, 0xff, 0xff, 0x81, 1, + ], + ] { + assert!(matches!( + ReportDescriptor::parse(&synthetic(&fields)), + Err(DescriptorError::LimitExceeded) + )); + } + let pushes = vec![0xa4; 33]; + assert!(matches!( + ReportDescriptor::parse(&synthetic(&pushes)), + Err(DescriptorError::LimitExceeded) + )); + let nested: Vec = [0xa1, 0].repeat(32); + assert!(matches!( + ReportDescriptor::parse(&synthetic(&nested)), + Err(DescriptorError::LimitExceeded) + )); + } + + #[test] + fn every_truncation_and_single_byte_input_is_safe() { + for bytes in [ + SYNTHETIC_PACKED, + REAL_COUGAR_MFD, + REAL_DUAL_ANALOG_4, + REAL_WARTHOG_STICK, + ] { + for length in 0..bytes.len() { + assert!(ReportDescriptor::parse(&bytes[..length]).is_err()); + } + for index in 0..bytes.len() { + for value in [0, 0xff] { + let mut mutated = bytes.to_vec(); + mutated[index] = value; + if let Ok(descriptor) = ReportDescriptor::parse(&mutated) { + let _ = descriptor.release(&[0xff; 256]); + let _ = descriptor.release(&[]); + } + } + } + } + for byte in 0..=255 { + assert!(ReportDescriptor::parse(&[byte]).is_err()); + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/ds4.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/ds4.rs new file mode 100644 index 000000000..c2c48bbf2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/ds4.rs @@ -0,0 +1,440 @@ +use crate::{SonyContact, SonySnapshot}; + +pub const DS4_REPORT_ID: u8 = 0x01; +pub const DS4_REPORT_BYTES: usize = 64; +pub const DS4_INTERFACE: u8 = 4; +pub const DS4_LOW_ID_BASE: u8 = 6; +pub const DS4_TOUCH_SAMPLE_COUNT: u8 = 1; +pub const DS4_STATUS_BYTE: u8 = 0x1b; + +pub const DS4_BUTTON_SQUARE: u16 = 0x4000; +pub const DS4_BUTTON_CROSS: u16 = 0x1000; +pub const DS4_BUTTON_CIRCLE: u16 = 0x2000; +pub const DS4_BUTTON_TRIANGLE: u16 = 0x8000; +pub const DS4_BUTTON_L1: u16 = 0x0100; +pub const DS4_BUTTON_R1: u16 = 0x0200; +pub const DS4_BUTTON_SHARE: u16 = 0x0020; +pub const DS4_BUTTON_OPTIONS: u16 = 0x0010; +pub const DS4_BUTTON_L3: u16 = 0x0040; +pub const DS4_BUTTON_R3: u16 = 0x0080; +pub const DS4_BUTTON_DPAD_UP: u16 = 0x0001; +pub const DS4_BUTTON_DPAD_DOWN: u16 = 0x0002; +pub const DS4_BUTTON_DPAD_LEFT: u16 = 0x0004; +pub const DS4_BUTTON_DPAD_RIGHT: u16 = 0x0008; + +const HAT_TABLE: [[u8; 3]; 3] = [[0x07, 0x06, 0x05], [0x00, 0x08, 0x04], [0x01, 0x02, 0x03]]; + +pub fn low_id_for_slot(slot: u8) -> Option { + (usize::from(slot) < crate::MAX_SOURCES).then_some(slot + DS4_LOW_ID_BASE) +} + +pub fn slot_for_low_id(low_id: u8) -> Option { + low_id + .checked_sub(DS4_LOW_ID_BASE) + .filter(|slot| usize::from(*slot) < crate::MAX_SOURCES) +} + +pub fn normalize_touch_axis(value: f32) -> i16 { + if value.is_nan() { + return 0; + } + if value <= 0.0 { + return i16::MIN; + } + if value >= 1.0 { + return i16::MAX; + } + let scaled = ((2.0_f64 * f64::from(value)) - 1.0) as f32; + if scaled < 0.0 { + (scaled * 32768.0).round().clamp(-32768.0, 32767.0) as i16 + } else if scaled > 0.0 { + (scaled * 32767.0).round().clamp(-32768.0, 32767.0) as i16 + } else { + 0 + } +} + +pub fn encode_contact(tracking_id: u8, contact: SonyContact) -> [u8; 4] { + let x = scale_axis(contact.x, 1919.0); + let y = scale_axis(contact.y, 942.0); + let mut bytes = [0_u8; 4]; + bytes[0] = (tracking_id & 0x7f) | if contact.active { 0 } else { 0x80 }; + bytes[1] = (x & 0xff) as u8; + bytes[2] = (((x >> 8) & 0x0f) as u8) | (((y & 0x0f) as u8) << 4); + bytes[3] = ((y >> 4) & 0xff) as u8; + bytes +} + +fn scale_axis(raw: i16, extent: f32) -> u16 { + let value = ((f32::from(raw) + 32768.0) / 65535.0) * extent; + value.clamp(0.0, extent) as u16 +} + +fn centered_axis(raw: i16) -> u8 { + ((i32::from(raw) + 32768) >> 8) as u8 +} + +fn hat_byte(buttons: u16) -> u8 { + let x = i32::from(buttons & DS4_BUTTON_DPAD_RIGHT != 0) + - i32::from(buttons & DS4_BUTTON_DPAD_LEFT != 0); + let y = i32::from(buttons & DS4_BUTTON_DPAD_DOWN != 0) + - i32::from(buttons & DS4_BUTTON_DPAD_UP != 0); + HAT_TABLE[(x + 1) as usize][(y + 1) as usize] +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Ds4ReportState { + contacts: [SonyContact; crate::CONTACTS_PER_SOURCE], + tracking_ids: [u8; crate::CONTACTS_PER_SOURCE], + touch_timestamp_byte: u8, + last_observed_at_us: u64, + seeded: bool, +} + +impl Default for Ds4ReportState { + fn default() -> Self { + Self { + contacts: [SonyContact::default(); crate::CONTACTS_PER_SOURCE], + tracking_ids: [0; crate::CONTACTS_PER_SOURCE], + touch_timestamp_byte: 0, + last_observed_at_us: 0, + seeded: false, + } + } +} + +impl Ds4ReportState { + pub fn tracking_id(&self, contact: usize) -> u8 { + self.tracking_ids[contact] + } + + pub fn touch_timestamp_byte(&self) -> u8 { + self.touch_timestamp_byte + } + + pub fn contacts(&self) -> &[SonyContact; crate::CONTACTS_PER_SOURCE] { + &self.contacts + } + + pub fn observe(&mut self, observed_at_us: u64) { + self.last_observed_at_us = self.last_observed_at_us.max(observed_at_us); + } + + pub fn last_observed_at_us(&self) -> u64 { + self.last_observed_at_us + } + + pub fn build(&mut self, snapshot: &SonySnapshot) -> [u8; DS4_REPORT_BYTES] { + self.observe(snapshot.observed_at_us); + let next = snapshot.contacts; + for (index, contact) in next.iter().enumerate() { + if contact.active && !self.contacts[index].active { + self.tracking_ids[index] = self.tracking_ids[index].wrapping_add(1) & 0x7f; + } + } + if !self.seeded || next != self.contacts { + self.touch_timestamp_byte = (snapshot.observed_at_us / 1000) as u8; + self.seeded = true; + } + self.contacts = next; + + let mut report = [0_u8; DS4_REPORT_BYTES]; + report[0] = DS4_REPORT_ID; + report[1] = centered_axis(snapshot.left_stick_x); + report[2] = centered_axis(snapshot.left_stick_y); + report[3] = centered_axis(snapshot.right_stick_x); + report[4] = centered_axis(snapshot.right_stick_y); + report[5] = hat_byte(snapshot.buttons) + | if snapshot.buttons & DS4_BUTTON_SQUARE != 0 { + 0x10 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_CROSS != 0 { + 0x20 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_CIRCLE != 0 { + 0x40 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_TRIANGLE != 0 { + 0x80 + } else { + 0 + }; + report[6] = if snapshot.buttons & DS4_BUTTON_L1 != 0 { + 0x01 + } else { + 0 + } | if snapshot.buttons & DS4_BUTTON_R1 != 0 { + 0x02 + } else { + 0 + } | if snapshot.left_trigger != 0 { 0x04 } else { 0 } + | if snapshot.right_trigger != 0 { 0x08 } else { 0 } + | if snapshot.buttons & DS4_BUTTON_SHARE != 0 { + 0x10 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_OPTIONS != 0 { + 0x20 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_L3 != 0 { + 0x40 + } else { + 0 + } + | if snapshot.buttons & DS4_BUTTON_R3 != 0 { + 0x80 + } else { + 0 + }; + report[7] = if snapshot.touchpad_click { 0x02 } else { 0 }; + report[8] = snapshot.left_trigger; + report[9] = snapshot.right_trigger; + let counter = snapshot + .observed_at_us + .wrapping_mul(3) + .wrapping_shr(4) + .to_le_bytes(); + report[10] = counter[0]; + report[11] = counter[1]; + report[30] = DS4_STATUS_BYTE; + report[33] = DS4_TOUCH_SAMPLE_COUNT; + report[34] = self.touch_timestamp_byte; + report[35..39].copy_from_slice(&encode_contact(self.tracking_ids[0], next[0])); + report[39..43].copy_from_slice(&encode_contact(self.tracking_ids[1], next[1])); + report + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MAX_SOURCES; + + fn contact(active: bool, x: i16, y: i16) -> SonyContact { + SonyContact { active, x, y } + } + + fn snapshot_with(contacts: [SonyContact; 2]) -> SonySnapshot { + let mut snapshot = SonySnapshot::neutral(0, 1, 0); + snapshot.contacts = contacts; + snapshot + } + + #[test] + fn normalization_matches_the_proved_vectors() { + assert_eq!(normalize_touch_axis(0.0), -32768); + assert_eq!(normalize_touch_axis(0.25), -16384); + assert_eq!(normalize_touch_axis(0.5), 0); + assert_eq!(normalize_touch_axis(0.75), 16384); + assert_eq!(normalize_touch_axis(1.0), 32767); + assert_eq!(normalize_touch_axis(-0.25), -32768); + assert_eq!(normalize_touch_axis(1.5), 32767); + assert_eq!(normalize_touch_axis(f32::NAN), 0); + assert_eq!(normalize_touch_axis(f32::INFINITY), 32767); + assert_eq!(normalize_touch_axis(f32::NEG_INFINITY), -32768); + } + + #[test] + fn normalization_keeps_the_float32_intermediate() { + assert_eq!(normalize_touch_axis(f32::from_bits(0x3e7f_fe01)), -16385); + } + + #[test] + fn contact_encoding_matches_the_proved_vectors_byte_for_byte() { + let cases: [(i16, i16, bool, [u8; 4]); 6] = [ + (-32768, -32768, true, [0x01, 0x00, 0x00, 0x00]), + (0, 0, false, [0x81, 0xbf, 0x73, 0x1d]), + (0, 1, true, [0x01, 0xbf, 0x73, 0x1d]), + (1, 0, true, [0x01, 0xbf, 0x73, 0x1d]), + (16384, -16384, true, [0x01, 0x9f, 0xb5, 0x0e]), + (32767, 32767, true, [0x01, 0x7f, 0xe7, 0x3a]), + ]; + for (x, y, active, expected) in cases { + assert_eq!( + encode_contact(1, contact(active, x, y)), + expected, + "contact vector ({x},{y}) active={active}" + ); + } + } + + #[test] + fn tracking_ids_increment_only_on_activation_and_start_at_one() { + let mut state = Ds4ReportState::default(); + let _ = state.build(&snapshot_with([SonyContact::default(); 2])); + assert_eq!(state.tracking_id(0), 0); + let center = contact(true, 0, 0); + let _ = state.build(&snapshot_with([center, SonyContact::default()])); + assert_eq!(state.tracking_id(0), 1); + let _ = state.build(&snapshot_with([ + contact(true, 100, 100), + SonyContact::default(), + ])); + assert_eq!(state.tracking_id(0), 1); + let _ = state.build(&snapshot_with([ + contact(true, 100, 100), + contact(true, -100, -100), + ])); + assert_eq!(state.tracking_id(0), 1); + assert_eq!(state.tracking_id(1), 1); + let _ = state.build(&snapshot_with([center, contact(true, -100, -100)])); + assert_eq!(state.tracking_id(0), 1); + let _ = state.build(&snapshot_with([ + SonyContact::default(), + contact(true, -100, -100), + ])); + let _ = state.build(&snapshot_with([center, contact(true, -100, -100)])); + assert_eq!(state.tracking_id(0), 2); + } + + #[test] + fn tracking_ids_wrap_modulo_one_twenty_eight() { + let mut state = Ds4ReportState::default(); + let up = SonyContact::default(); + let down = contact(true, 0, 0); + let _ = state.build(&snapshot_with([up, up])); + for _ in 0..127 { + let _ = state.build(&snapshot_with([down, up])); + let _ = state.build(&snapshot_with([up, up])); + } + assert_eq!(state.tracking_id(0), 127); + let _ = state.build(&snapshot_with([down, up])); + assert_eq!(state.tracking_id(0), 0); + } + + #[test] + fn first_build_always_latches_a_deterministic_touch_timestamp() { + let mut state = Ds4ReportState::default(); + let mut snapshot = SonySnapshot::neutral(0, 1, 1_500_000); + let report = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), (1_500_000_u64 / 1000) as u8); + assert_eq!(report[34], state.touch_timestamp_byte()); + let latched = state.touch_timestamp_byte(); + snapshot.observed_at_us = 9_999_000; + let _ = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), latched); + } + + #[test] + fn touch_timestamp_refreshes_on_every_contact_tuple_change() { + let mut state = Ds4ReportState::default(); + let mut snapshot = SonySnapshot::neutral(0, 1, 1_000_000); + let _ = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), (1_000_000_u64 / 1000) as u8); + snapshot.observed_at_us = 2_000_000; + snapshot.contacts[0] = contact(true, 0, 0); + let _ = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), (2_000_000_u64 / 1000) as u8); + snapshot.observed_at_us = 3_000_000; + snapshot.contacts[0] = contact(false, 0, 0); + let _ = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), (3_000_000_u64 / 1000) as u8); + snapshot.observed_at_us = 4_000_000; + snapshot.contacts[1] = contact(true, 1, 2); + let _ = state.build(&snapshot); + assert_eq!(state.touch_timestamp_byte(), (4_000_000_u64 / 1000) as u8); + } + + #[test] + fn every_report_owns_all_sixty_four_bytes_with_fixed_values() { + let mut state = Ds4ReportState::default(); + let mut snapshot = SonySnapshot::neutral(0, 1, 2_500); + snapshot.buttons = DS4_BUTTON_SQUARE | DS4_BUTTON_DPAD_UP; + snapshot.left_trigger = 0x40; + snapshot.right_trigger = 0x80; + snapshot.touchpad_click = true; + snapshot.contacts[0] = contact(true, 16384, -16384); + let report = state.build(&snapshot); + assert_eq!(report.len(), DS4_REPORT_BYTES); + assert_eq!( + &report[0..10], + &[0x01, 128, 128, 128, 128, 0x10, 0x0c, 0x02, 0x40, 0x80] + ); + assert_eq!(report[10], ((2_500_u64 * 3) >> 4) as u8); + assert_eq!(report[11], ((2_500_u64 * 3) >> 12) as u8); + assert_eq!(report[12], 0); + assert!(report[13..30].iter().all(|byte| *byte == 0)); + assert_eq!(report[30], DS4_STATUS_BYTE); + assert_eq!(report[31], 0); + assert_eq!(report[32], 0); + assert_eq!(report[33], DS4_TOUCH_SAMPLE_COUNT); + assert_eq!(&report[35..39], &[0x01, 0x9f, 0xb5, 0x0e]); + assert_eq!(&report[39..43], &[0x80, 0xbf, 0x73, 0x1d]); + assert!(report[43..].iter().all(|byte| *byte == 0)); + } + + #[test] + fn ps_bit_stays_clear_even_when_every_button_bit_is_set() { + let mut state = Ds4ReportState::default(); + let mut snapshot = SonySnapshot::neutral(0, 1, 0); + snapshot.buttons = 0xffff; + let report = state.build(&snapshot); + assert_eq!(report[7] & 0x01, 0); + assert_eq!(report[7] & 0x02, 0); + assert_eq!(report[7], 0); + } + + #[test] + fn capture_loss_neutral_releases_contacts_at_their_last_positions() { + let mut state = Ds4ReportState::default(); + let mut snapshot = SonySnapshot::neutral(0, 1, 0); + snapshot.contacts[0] = contact(true, 32767, 32767); + snapshot.contacts[1] = contact(true, -32768, -32768); + let _ = state.build(&snapshot); + let mut release = SonySnapshot::neutral(0, 1, 1000); + release.contacts[0] = contact(false, 32767, 32767); + release.contacts[1] = contact(false, -32768, -32768); + let report = state.build(&release); + assert_eq!(&report[1..5], &[128, 128, 128, 128]); + assert_eq!(report[5], 0x08); + assert_eq!(report[6], 0); + assert_eq!(report[7], 0); + assert_eq!(&report[35..39], &[0x81, 0x7f, 0xe7, 0x3a]); + assert_eq!(&report[39..43], &[0x81, 0x00, 0x00, 0x00]); + } + + #[test] + fn two_active_center_contacts_remain_independently_active() { + let mut state = Ds4ReportState::default(); + let center = contact(true, 0, 0); + let report = state.build(&snapshot_with([center, center])); + assert_eq!(report[35], 0x01); + assert_eq!(report[39], 0x01); + assert_eq!(&report[35..39], &report[39..43]); + assert_eq!(state.tracking_id(0), 1); + assert_eq!(state.tracking_id(1), 1); + } + + #[test] + fn hat_table_covers_the_full_grid() { + assert_eq!(hat_byte(0), 0x08); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_UP), 0x00); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_RIGHT), 0x02); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_DOWN), 0x04); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_LEFT), 0x06); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_UP | DS4_BUTTON_DPAD_RIGHT), 0x01); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_DOWN | DS4_BUTTON_DPAD_LEFT), 0x05); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_UP | DS4_BUTTON_DPAD_LEFT), 0x07); + assert_eq!(hat_byte(DS4_BUTTON_DPAD_DOWN | DS4_BUTTON_DPAD_RIGHT), 0x03); + } + + #[test] + fn low_id_mapping_round_trips_within_the_reserved_domain() { + for slot in 0..MAX_SOURCES as u8 { + let low_id = low_id_for_slot(slot).unwrap(); + assert_eq!(slot_for_low_id(low_id), Some(slot)); + } + assert_eq!(low_id_for_slot(MAX_SOURCES as u8), None); + assert_eq!(slot_for_low_id(5), None); + assert_eq!(slot_for_low_id(10), None); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic.rs new file mode 100644 index 000000000..e1bb6e7fd --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic.rs @@ -0,0 +1,1136 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +use crate::descriptor::ReportDescriptor; + +pub const MAX_GENERIC_SOURCES: usize = 8; +pub const MAX_GENERIC_INTERFACES: usize = 16; +pub const MAX_REPORT_BYTES: usize = 256; +pub const MAX_DESCRIPTOR_BYTES: usize = 4096; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GenericDevice { + pub slot: u8, + pub incarnation: u64, + pub vendor: u16, + pub product: u16, + pub revision: u16, + pub interfaces: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GenericInterface { + pub number: u8, + pub descriptor: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GenericReport { + pub slot: u8, + pub incarnation: u64, + pub interface: u8, + pub operation: u8, + pub error: bool, + pub bytes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GenericOutput { + pub slot: u8, + pub incarnation: u64, + pub interface: u8, + pub operation: u8, + pub bytes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GenericError { + InvalidSlot, + InvalidIncarnation, + DuplicateSlot, + DuplicateInterface, + TooManyInterfaces, + DescriptorTooLarge, + NotAllowlisted, + StaleSource, + UnknownInterface, + InvalidOperation, + InvalidPayload, + Unbound, + Inactive, + Overflow, + Faulted, +} + +impl fmt::Display for GenericError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "generic HID: {self:?}") + } +} + +impl std::error::Error for GenericError {} + +pub(crate) fn validate_inventory(devices: &[GenericDevice]) -> Result<(), GenericError> { + let mut slots = BTreeSet::new(); + for device in devices { + if usize::from(device.slot) >= MAX_GENERIC_SOURCES { + return Err(GenericError::InvalidSlot); + } + if device.incarnation == 0 { + return Err(GenericError::InvalidIncarnation); + } + if !slots.insert(device.slot) { + return Err(GenericError::DuplicateSlot); + } + if !crate::allowlist::generic_hid_allowed(device.vendor, device.product) { + return Err(GenericError::NotAllowlisted); + } + if device.interfaces.is_empty() || device.interfaces.len() > MAX_GENERIC_INTERFACES { + return Err(GenericError::TooManyInterfaces); + } + let mut interfaces = BTreeSet::new(); + for interface in &device.interfaces { + if interface.number == u8::MAX { + return Err(GenericError::UnknownInterface); + } + if !interfaces.insert(interface.number) { + return Err(GenericError::DuplicateInterface); + } + if interface.descriptor.len() > MAX_DESCRIPTOR_BYTES { + return Err(GenericError::DescriptorTooLarge); + } + } + } + Ok(()) +} + +pub(crate) fn validate_operation( + operation: u8, + bytes: &[u8], + output: bool, +) -> Result<(), GenericError> { + if operation > 4 { + return Err(GenericError::InvalidOperation); + } + if bytes.is_empty() || bytes.len() > MAX_REPORT_BYTES { + return Err(GenericError::InvalidPayload); + } + if output && matches!(operation, 2 | 3) { + if bytes.len() != 3 { + return Err(GenericError::InvalidPayload); + } + let requested = usize::from(u16::from_le_bytes([bytes[1], bytes[2]])); + if requested == 0 || requested > MAX_REPORT_BYTES { + return Err(GenericError::InvalidPayload); + } + } + Ok(()) +} + +pub(crate) fn validate_report(report: &GenericReport) -> Result<(), GenericError> { + if report.error && matches!(report.operation, 1..=4) && report.bytes.is_empty() { + return Ok(()); + } + validate_operation(report.operation, &report.bytes, false) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GenericOutbound { + Attach { + request_id: u8, + device: GenericDevice, + }, + Detach { + device_id: u8, + }, + Report { + device_id: u8, + interface: u8, + operation: u8, + error: bool, + bytes: Vec, + slot: u8, + incarnation: u64, + release: bool, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Admission { + Dormant, + AwaitingDetach(u8), + Pending(u8), + Admitted(u8), + Refused, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GenericAdmission { + Pending, + Admitted, + Declined, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GenericAdmissionStatus { + pub slot: u8, + pub incarnation: u64, + pub admission: GenericAdmission, +} + +#[derive(Debug)] +struct Source { + device: GenericDevice, + admission: Admission, + descriptors: BTreeMap, + forwarded: BTreeMap<(u8, u8), Vec>, + detach_fallback: bool, +} + +impl Source { + fn new(device: GenericDevice) -> Self { + let descriptors: BTreeMap<_, _> = device + .interfaces + .iter() + .filter_map(|interface| { + ReportDescriptor::parse(&interface.descriptor) + .ok() + .map(|parsed| (interface.number, parsed)) + }) + .collect(); + let detach_fallback = descriptors.len() != device.interfaces.len(); + if detach_fallback { + eprintln!( + "generic HID {:04x}:{:04x} slot {}: descriptor unavailable or unsafe; capture release will detach", + device.vendor, device.product, device.slot + ); + } + Self { + device, + admission: Admission::Dormant, + descriptors, + forwarded: BTreeMap::new(), + detach_fallback, + } + } + + fn release(&mut self) -> Vec { + let Admission::Admitted(device_id) = self.admission else { + return Vec::new(); + }; + let mut releases = Vec::with_capacity(self.forwarded.len()); + if !self.detach_fallback { + for ((interface, _), bytes) in &self.forwarded { + let released = self + .descriptors + .get(interface) + .and_then(|descriptor| descriptor.release(bytes).ok()); + let Some(bytes) = released else { + self.detach_fallback = true; + eprintln!( + "generic HID slot {}: report cannot be safely released; detaching", + self.device.slot + ); + break; + }; + releases.push(GenericOutbound::Report { + device_id, + interface: *interface, + operation: 0, + error: false, + bytes, + slot: self.device.slot, + incarnation: self.device.incarnation, + release: true, + }); + } + } + if self.detach_fallback { + releases.clear(); + releases.push(GenericOutbound::Detach { device_id }); + self.admission = Admission::AwaitingDetach(device_id); + self.forwarded.clear(); + } + releases + } +} + +#[derive(Debug, Clone, Copy)] +enum Request { + Pending { + slot: u8, + incarnation: u64, + sent: Option, + }, + RetiredPending, + Tombstone, +} + +#[derive(Debug)] +pub struct GenericSession { + generation: u64, + sources: BTreeMap, + requests: BTreeMap, + used_device_ids: BTreeSet, + retired_server_ids_eligible: BTreeMap, + send_order: u64, + next_request: u16, + capture: bool, + cleanup: BTreeSet, +} + +impl Default for GenericSession { + fn default() -> Self { + Self { + generation: 0, + sources: BTreeMap::new(), + requests: BTreeMap::new(), + used_device_ids: BTreeSet::new(), + retired_server_ids_eligible: BTreeMap::new(), + send_order: 0, + next_request: 10, + capture: false, + cleanup: BTreeSet::new(), + } + } +} + +impl GenericSession { + pub fn new(generation: u64) -> Self { + Self { + generation, + ..Self::default() + } + } + + pub fn generation(&self) -> u64 { + self.generation + } + + pub fn admission(&self, device: &GenericDevice) -> GenericAdmission { + match self.sources.get(&device.slot) { + Some(source) if source.device == *device => match source.admission { + Admission::Admitted(_) => GenericAdmission::Admitted, + Admission::Refused => GenericAdmission::Declined, + _ => GenericAdmission::Pending, + }, + _ => GenericAdmission::Pending, + } + } + + fn attach(&mut self, slot: u8) -> Option { + let source = self.sources.get_mut(&slot)?; + if source.admission != Admission::Dormant || !self.capture { + return None; + } + if self.next_request > 254 { + source.admission = Admission::Refused; + eprintln!( + "generic HID slot {slot}: registration IDs exhausted; a new session is required" + ); + return None; + } + let request_id = self.next_request as u8; + self.next_request += 1; + source.admission = Admission::Pending(request_id); + self.requests.insert( + request_id, + Request::Pending { + slot, + incarnation: source.device.incarnation, + sent: None, + }, + ); + Some(GenericOutbound::Attach { + request_id, + device: source.device.clone(), + }) + } + + fn retire(&mut self, slot: u8) -> Vec { + let Some(mut source) = self.sources.remove(&slot) else { + return Vec::new(); + }; + if let Admission::Pending(request_id) = source.admission { + let sent = matches!( + self.requests.get(&request_id), + Some(Request::Pending { sent: Some(_), .. }) + ); + self.requests.insert( + request_id, + if sent { + Request::RetiredPending + } else { + Request::Tombstone + }, + ); + } + let mut result = source.release(); + if let Admission::Admitted(device_id) = source.admission { + result.push(GenericOutbound::Detach { device_id }); + } + result + } + + pub fn reconcile( + &mut self, + inventory: &[GenericDevice], + capture_active: bool, + ) -> Vec { + let mut result = self + .cleanup + .iter() + .map(|device_id| GenericOutbound::Detach { + device_id: *device_id, + }) + .collect::>(); + self.cleanup.clear(); + if validate_inventory(inventory).is_err() { + result.extend(self.retire_all()); + return result; + } + let removed = self + .sources + .iter() + .filter_map(|(slot, source)| (!inventory.contains(&source.device)).then_some(*slot)) + .collect::>(); + for slot in removed { + result.extend(self.retire(slot)); + } + result.extend(self.set_capture(capture_active)); + for device in inventory { + self.sources + .entry(device.slot) + .or_insert_with(|| Source::new(device.clone())); + if let Some(attach) = self.attach(device.slot) { + result.push(attach); + } + } + result + } + + pub fn release_all(&mut self) -> Vec { + self.sources + .values_mut() + .flat_map(Source::release) + .collect() + } + + pub fn set_capture(&mut self, active: bool) -> Vec { + if self.capture == active { + return Vec::new(); + } + self.capture = active; + if !active { + return self.release_all(); + } + let slots = self.sources.keys().copied().collect::>(); + slots + .into_iter() + .filter_map(|slot| self.attach(slot)) + .collect() + } + + pub fn handle_response(&mut self, request_id: u8, control: u8, _opaque: u32, device_id: u8) { + let Some(request) = self.requests.get(&request_id).copied() else { + return; + }; + if matches!( + request, + Request::Tombstone | Request::Pending { sent: None, .. } + ) { + return; + } + self.requests.insert(request_id, Request::Tombstone); + let valid = control == 1 && (10..=254).contains(&device_id); + if let Request::RetiredPending = request { + if valid + && !self + .sources + .values() + .any(|source| source.admission == Admission::Admitted(device_id)) + { + self.used_device_ids.insert(device_id); + self.cleanup.insert(device_id); + } + return; + } + let Request::Pending { + slot, + incarnation, + sent: Some(sent_order), + } = request + else { + return; + }; + let active_id = self + .sources + .values() + .any(|source| source.admission == Admission::Admitted(device_id)); + let reused_id = (self.used_device_ids.contains(&device_id) + && self + .retired_server_ids_eligible + .get(&device_id) + .is_none_or(|detach_order| *detach_order > sent_order)) + || self.cleanup.contains(&device_id); + let Some(source) = self.sources.get_mut(&slot) else { + return; + }; + if source.device.incarnation != incarnation + || source.admission != Admission::Pending(request_id) + { + return; + } + source.admission = if valid && !active_id && !reused_id { + self.used_device_ids.insert(device_id); + self.retired_server_ids_eligible.remove(&device_id); + Admission::Admitted(device_id) + } else { + if valid && !active_id { + if self.retired_server_ids_eligible.contains_key(&device_id) + && !self.cleanup.contains(&device_id) + { + Admission::Dormant + } else { + self.cleanup.insert(device_id); + Admission::AwaitingDetach(device_id) + } + } else { + Admission::Refused + } + }; + eprintln!( + "generic HID {:04x}:{:04x} slot {slot}: registration {}", + source.device.vendor, + source.device.product, + if source.admission == Admission::Admitted(device_id) { + "accepted" + } else { + "refused" + } + ); + if !self.capture + && source.detach_fallback + && source.admission == Admission::Admitted(device_id) + { + self.cleanup.insert(device_id); + source.admission = Admission::AwaitingDetach(device_id); + } + } + + pub fn report(&self, report: GenericReport) -> Option { + validate_report(&report).ok()?; + if !self.capture { + return None; + } + let source = self.sources.get(&report.slot)?; + let Admission::Admitted(device_id) = source.admission else { + return None; + }; + if source.device.incarnation != report.incarnation + || !source + .device + .interfaces + .iter() + .any(|interface| interface.number == report.interface) + { + return None; + } + Some(GenericOutbound::Report { + device_id, + interface: report.interface, + operation: report.operation, + error: report.error, + bytes: report.bytes, + slot: report.slot, + incarnation: report.incarnation, + release: false, + }) + } + + pub fn sent(&mut self, outbound: &GenericOutbound) { + match outbound { + GenericOutbound::Detach { device_id } => { + if self + .sources + .values() + .any(|source| source.admission == Admission::Admitted(*device_id)) + { + return; + } + if self.used_device_ids.contains(device_id) { + self.send_order += 1; + self.retired_server_ids_eligible + .insert(*device_id, self.send_order); + } + self.cleanup.remove(device_id); + for source in self.sources.values_mut() { + if source.admission == Admission::AwaitingDetach(*device_id) { + source.admission = Admission::Dormant; + } + } + } + GenericOutbound::Attach { request_id, device } => { + if let Some(Request::Pending { + slot, + incarnation, + sent, + }) = self.requests.get_mut(request_id) + && *slot == device.slot + && *incarnation == device.incarnation + && sent.is_none() + { + self.send_order += 1; + *sent = Some(self.send_order); + } + } + GenericOutbound::Report { + device_id, + interface, + operation: 0, + error: false, + bytes, + slot, + incarnation, + .. + } => { + let Some(source) = self.sources.get_mut(slot) else { + return; + }; + if source.device.incarnation != *incarnation + || source.admission != Admission::Admitted(*device_id) + { + return; + } + let id = source + .descriptors + .get(interface) + .and_then(|descriptor| descriptor.report_id(bytes).ok()); + if let Some(id) = id { + source.forwarded.insert((*interface, id), bytes.clone()); + } else if !source.detach_fallback { + source.detach_fallback = true; + eprintln!( + "generic HID slot {}: report does not match its descriptor; capture release will detach", + source.device.slot + ); + } + } + _ => {} + } + } + + pub fn fault(&mut self, slot: u8, incarnation: u64) -> Vec { + let Some(source) = self.sources.get(&slot) else { + return Vec::new(); + }; + if source.device.incarnation != incarnation { + return Vec::new(); + } + eprintln!("generic HID slot {slot}: retiring source after a bounded input/output failure"); + let device = source.device.clone(); + let result = self.retire(slot); + let mut refused = Source::new(device); + refused.admission = Admission::Refused; + self.sources.insert(slot, refused); + result + } + + pub fn output( + &self, + device_id: u8, + interface: u8, + operation: u8, + bytes: &[u8], + ) -> Option { + validate_operation(operation, bytes, true).ok()?; + if !self.capture { + return None; + } + let source = self + .sources + .values() + .find(|source| source.admission == Admission::Admitted(device_id))?; + if !source + .device + .interfaces + .iter() + .any(|entry| entry.number == interface) + { + return None; + } + Some(GenericOutput { + slot: source.device.slot, + incarnation: source.device.incarnation, + interface, + operation, + bytes: bytes.to_vec(), + }) + } + + pub fn retire_all(&mut self) -> Vec { + let slots = self.sources.keys().copied().collect::>(); + let mut result = self + .cleanup + .iter() + .map(|device_id| GenericOutbound::Detach { + device_id: *device_id, + }) + .collect::>(); + self.cleanup.clear(); + for slot in slots { + result.extend(self.retire(slot)); + } + result + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + pub(crate) fn device(slot: u8, incarnation: u64) -> GenericDevice { + GenericDevice { + slot, + incarnation, + vendor: 0x044f, + product: 0x0402, + revision: 0x0100, + interfaces: vec![GenericInterface { + number: 2, + descriptor: vec![ + 0x05, 0x01, 0x09, 0x04, 0xa1, 0x01, 0x05, 0x09, 0x19, 0x01, 0x29, 0x08, 0x15, + 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, 0x05, 0x01, 0x09, 0x30, + 0x15, 0x00, 0x26, 0xff, 0x00, 0x75, 0x08, 0x95, 0x01, 0x81, 0x02, 0x09, 0x39, + 0x15, 0x00, 0x25, 0x07, 0x75, 0x04, 0x95, 0x01, 0x81, 0x42, 0x75, 0x04, 0x95, + 0x01, 0x81, 0x03, 0xc0, + ], + }], + } + } + + pub(crate) fn report(slot: u8, incarnation: u64) -> GenericReport { + GenericReport { + slot, + incarnation, + interface: 2, + operation: 0, + error: false, + bytes: vec![0xff, 0xa5, 0xb3], + } + } + + fn attach(session: &mut GenericSession, devices: &[GenericDevice], server_id: u8) -> u8 { + let outgoing = session.reconcile(devices, true); + let GenericOutbound::Attach { request_id, .. } = &outgoing[0] else { + panic!("expected attach"); + }; + let request_id = *request_id; + session.sent(&outgoing[0]); + session.handle_response(request_id, 1, 0, server_id); + request_id + } + + #[test] + fn admission_requires_a_sent_attach_and_verified_response() { + let mut session = GenericSession::new(1); + let outgoing = session.reconcile(&[device(0, 1)], true); + assert!(session.report(report(0, 1)).is_none()); + session.handle_response(10, 1, 0, 20); + assert!(session.report(report(0, 1)).is_none()); + session.sent(&outgoing[0]); + session.handle_response(10, 1, 0, 20); + assert!(matches!( + session.report(report(0, 1)), + Some(GenericOutbound::Report { device_id: 20, .. }) + )); + session.handle_response(10, 1, 1, 255); + assert!(session.report(report(0, 1)).is_some()); + } + + #[test] + fn control_one_admits_regardless_of_the_opaque_response_dword() { + for opaque in [0, 0xf, u32::MAX] { + let mut session = GenericSession::new(1); + let outgoing = session.reconcile(&[device(0, 1)], true); + session.sent(&outgoing[0]); + session.handle_response(10, 1, opaque, 20); + assert!(matches!( + session.report(report(0, 1)), + Some(GenericOutbound::Report { device_id: 20, .. }) + )); + } + } + + #[test] + fn pause_releases_only_forwarded_reports_and_resume_uses_next_poll() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 21); + let outgoing = session.report(report(0, 1)).unwrap(); + assert!(session.set_capture(false).is_empty()); + assert!(session.set_capture(true).is_empty()); + session.sent(&outgoing); + let release = session.set_capture(false); + assert!( + matches!(&release[0], GenericOutbound::Report { bytes, release: true, .. } if bytes == &[0, 0xa5, 0xb8]) + ); + assert!(session.report(report(0, 1)).is_none()); + session.sent(&release[0]); + assert!(session.set_capture(true).is_empty()); + assert!( + matches!(session.report(report(0, 1)), Some(GenericOutbound::Report { bytes, release: false, .. }) if bytes == vec![0xff, 0xa5, 0xb3]) + ); + } + + #[test] + fn release_tracks_each_forwarded_report_id_and_interface_separately() { + let mut source = device(0, 1); + let descriptor = source.interfaces[0].descriptor.clone(); + let fields = &descriptor[6..descriptor.len() - 1]; + let mut numbered = descriptor[..6].to_vec(); + numbered.extend_from_slice(&[0x85, 1]); + numbered.extend_from_slice(fields); + numbered.extend_from_slice(&[0x85, 2]); + numbered.extend_from_slice(fields); + numbered.push(0xc0); + source.interfaces[0].descriptor = numbered.clone(); + source.interfaces.push(GenericInterface { + number: 3, + descriptor: numbered, + }); + let mut session = GenericSession::new(1); + attach(&mut session, &[source], 22); + for (interface, bytes) in [ + (2, vec![1, 0xff, 0x11, 0x33]), + (2, vec![2, 0xaa, 0x22, 0x43]), + (3, vec![1, 0xbb, 0x33, 0x53]), + ] { + let mut input = report(0, 1); + input.interface = interface; + input.bytes = bytes; + let outgoing = session.report(input).unwrap(); + session.sent(&outgoing); + } + let released = session.set_capture(false); + let states = released + .iter() + .map(|record| { + let GenericOutbound::Report { + interface, + bytes, + release: true, + .. + } = record + else { + panic!("expected descriptor release"); + }; + (*interface, bytes.clone()) + }) + .collect::>(); + assert_eq!( + states, + vec![ + (2, vec![1, 0, 0x11, 0x38]), + (2, vec![2, 0, 0x22, 0x48]), + (3, vec![1, 0, 0x33, 0x58]) + ] + ); + } + + #[test] + fn descriptor_mismatched_report_is_forwarded_but_release_detaches() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 22); + let mut input = report(0, 1); + input.bytes = vec![1]; + let outgoing = session.report(input).unwrap(); + session.sent(&outgoing); + assert_eq!( + session.set_capture(false), + vec![GenericOutbound::Detach { device_id: 22 }] + ); + } + + #[test] + fn invalid_descriptor_passes_through_then_detaches_and_reattaches() { + let mut source = device(0, 1); + source.interfaces[0].descriptor.clear(); + let mut session = GenericSession::new(1); + attach(&mut session, &[source], 21); + let outgoing = session.report(report(0, 1)).unwrap(); + session.sent(&outgoing); + let paused = session.set_capture(false); + assert_eq!(paused, vec![GenericOutbound::Detach { device_id: 21 }]); + session.sent(&paused[0]); + let resumed = session.set_capture(true); + assert!(matches!( + &resumed[0], + GenericOutbound::Attach { request_id: 11, .. } + )); + assert!(session.report(report(0, 1)).is_none()); + session.sent(&resumed[0]); + session.handle_response(10, 1, 0, 22); + assert!(session.report(report(0, 1)).is_none()); + session.handle_response(11, 1, 0, 22); + assert!(session.report(report(0, 1)).is_some()); + } + + #[test] + fn fallback_resume_may_reuse_server_id_for_the_same_source_incarnation() { + let mut source = device(0, 1); + source.interfaces[0].descriptor.clear(); + let mut session = GenericSession::new(1); + attach(&mut session, &[source.clone()], 22); + let paused = session.set_capture(false); + assert_eq!(paused, vec![GenericOutbound::Detach { device_id: 22 }]); + assert!(session.set_capture(true).is_empty()); + session.sent(&paused[0]); + let resumed = session.reconcile(&[source], true); + session.sent(&resumed[0]); + session.handle_response(11, 1, 0xf, 22); + assert!(session.report(report(0, 1)).is_some()); + assert_eq!(session.output(22, 2, 0, &[1]).unwrap().incarnation, 1); + } + + #[test] + fn removal_releases_before_detach_and_old_incarnations_do_not_resurrect() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 21); + let outgoing = session.report(report(0, 1)).unwrap(); + session.sent(&outgoing); + let retired = session.reconcile(&[device(0, 2)], true); + assert!(matches!( + retired[0], + GenericOutbound::Report { release: true, .. } + )); + assert_eq!(retired[1], GenericOutbound::Detach { device_id: 21 }); + assert!(matches!( + retired[2], + GenericOutbound::Attach { request_id: 11, .. } + )); + session.sent(&retired[2]); + session.handle_response(10, 1, 0, 30); + assert!(session.report(report(0, 1)).is_none()); + assert!(session.report(report(0, 2)).is_none()); + session.handle_response(11, 1, 0, 22); + assert!(session.report(report(0, 2)).is_some()); + let ended = session.retire_all(); + assert_eq!(ended, vec![GenericOutbound::Detach { device_id: 22 }]); + assert!(session.report(report(0, 2)).is_none()); + let mut restarted = GenericSession::new(1); + assert_eq!(attach(&mut restarted, &[device(0, 2)], 22), 10); + } + + #[test] + fn late_response_to_retired_pending_attach_only_schedules_cleanup() { + let mut session = GenericSession::new(1); + let outgoing = session.reconcile(&[device(0, 1)], true); + session.sent(&outgoing[0]); + assert!(session.reconcile(&[], false).is_empty()); + session.handle_response(10, 1, 0, 22); + assert_eq!( + session.reconcile(&[], false), + vec![GenericOutbound::Detach { device_id: 22 }] + ); + assert!(session.report(report(0, 1)).is_none()); + } + + #[test] + fn retired_server_ids_cannot_route_stale_output_to_a_replacement() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 22); + let outgoing = session.reconcile(&[device(0, 2)], true); + session.sent(&outgoing[1]); + session.handle_response(11, 1, 0, 22); + assert!(session.output(22, 2, 0, &[1]).is_none()); + assert!(session.report(report(0, 2)).is_none()); + assert_eq!( + session.reconcile(&[device(0, 2)], true), + vec![GenericOutbound::Detach { device_id: 22 }] + ); + } + + #[test] + fn queued_detach_then_fresh_attach_allows_assigned_id_reuse_on_reconnect() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 10); + let stale_output = session.output(10, 2, 0, &[1]).unwrap(); + let outgoing = session.reconcile(&[device(0, 2)], true); + assert_eq!(outgoing[0], GenericOutbound::Detach { device_id: 10 }); + assert!(matches!( + outgoing[1], + GenericOutbound::Attach { request_id: 11, .. } + )); + assert!(session.output(10, 2, 0, &[1]).is_none()); + session.sent(&outgoing[0]); + assert!(session.output(10, 2, 0, &[1]).is_none()); + session.sent(&outgoing[1]); + assert!(session.output(10, 2, 0, &[1]).is_none()); + session.handle_response(11, 1, 0xf, 10); + assert_eq!(session.output(10, 2, 0, &[1]).unwrap().incarnation, 2); + assert!(session.report(report(0, 1)).is_none()); + assert!(session.report(report(0, 2)).is_some()); + let runtime = crate::generic_runtime::GenericRuntime::new(); + runtime.replace_inventory(vec![device(0, 2)]).unwrap(); + runtime.bind_session(1); + runtime.set_capture(true); + assert_eq!( + runtime.push_output(stale_output), + Err(GenericError::StaleSource) + ); + } + + #[test] + fn deferred_pending_cleanup_cannot_detach_a_newly_admitted_replacement() { + let mut session = GenericSession::new(1); + let first = session.reconcile(&[device(0, 1)], true); + session.sent(&first[0]); + let second = session.reconcile(&[device(0, 2)], true); + session.sent(&second[0]); + session.handle_response(10, 1, 0, 22); + session.handle_response(11, 1, 0, 22); + assert!(session.report(report(0, 2)).is_none()); + let cleanup = session.reconcile(&[device(0, 2)], true); + assert_eq!(cleanup, vec![GenericOutbound::Detach { device_id: 22 }]); + assert!(session.reconcile(&[device(0, 2)], true).is_empty()); + session.sent(&cleanup[0]); + let retry = session.reconcile(&[device(0, 2)], true); + assert!(matches!( + retry[0], + GenericOutbound::Attach { request_id: 12, .. } + )); + session.sent(&retry[0]); + assert!(session.output(22, 2, 0, &[1]).is_none()); + session.handle_response(11, 1, 0, 22); + assert!(session.output(22, 2, 0, &[1]).is_none()); + session.handle_response(12, 1, 0xf, 22); + assert_eq!(session.output(22, 2, 0, &[1]).unwrap().incarnation, 2); + assert!(session.reconcile(&[device(0, 2)], true).is_empty()); + } + + #[test] + fn acknowledgement_for_attach_queued_before_cleanup_requires_a_fresh_retry() { + let mut session = GenericSession::new(1); + let first = session.reconcile(&[device(0, 1)], true); + session.sent(&first[0]); + let second = session.reconcile(&[device(0, 2)], true); + session.sent(&second[0]); + session.handle_response(10, 1, 0, 22); + let cleanup = session.reconcile(&[device(0, 2)], true); + assert_eq!(cleanup, vec![GenericOutbound::Detach { device_id: 22 }]); + session.sent(&cleanup[0]); + session.handle_response(11, 1, 0, 22); + assert!(session.output(22, 2, 0, &[1]).is_none()); + let retry = session.reconcile(&[device(0, 2)], true); + assert_eq!(retry.len(), 1); + assert!(matches!( + retry[0], + GenericOutbound::Attach { request_id: 12, .. } + )); + session.sent(&retry[0]); + session.handle_response(12, 1, 0, 22); + assert_eq!(session.output(22, 2, 0, &[1]).unwrap().incarnation, 2); + assert!(session.reconcile(&[device(0, 2)], true).is_empty()); + } + + #[test] + fn refusal_invalid_ids_and_duplicate_server_ids_are_fail_closed() { + for (control, id) in [(0, 22), (1, 9), (1, 255)] { + let mut session = GenericSession::new(1); + let outgoing = session.reconcile(&[device(0, 1)], true); + session.sent(&outgoing[0]); + session.handle_response(10, control, 0, id); + assert!(session.report(report(0, 1)).is_none()); + session.handle_response(10, 1, 0, 22); + assert!(session.reconcile(&[device(0, 1)], true).is_empty()); + assert!(session.report(report(0, 1)).is_none()); + } + let mut session = GenericSession::new(1); + let outgoing = session.reconcile(&[device(0, 1), device(1, 2)], true); + for entry in &outgoing { + session.sent(entry); + } + session.handle_response(10, 1, 0, 22); + session.handle_response(11, 1, 0, 22); + assert!(session.report(report(0, 1)).is_some()); + assert!(session.report(report(1, 2)).is_none()); + } + + #[test] + fn request_ids_never_reuse_and_exhaust_at_254() { + let mut session = GenericSession::default(); + for incarnation in 1..=245 { + let outgoing = session.reconcile(&[device(0, incarnation)], true); + assert!( + matches!(&outgoing[0], GenericOutbound::Attach { request_id, .. } if u64::from(*request_id) == incarnation + 9) + ); + session.sent(&outgoing[0]); + session.handle_response((incarnation + 9) as u8, 0, 0, 255); + } + assert!(session.reconcile(&[device(0, 246)], true).is_empty()); + assert!(session.report(report(0, 246)).is_none()); + } + + #[test] + fn fault_is_sticky_until_inventory_incarnation_changes() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 22); + let outgoing = session.report(report(0, 1)).unwrap(); + session.sent(&outgoing); + assert_eq!(session.fault(0, 99), Vec::new()); + let fault = session.fault(0, 1); + assert!(matches!( + fault[0], + GenericOutbound::Report { release: true, .. } + )); + assert_eq!(fault[1], GenericOutbound::Detach { device_id: 22 }); + assert!(session.reconcile(&[device(0, 1)], true).is_empty()); + assert!(session.report(report(0, 1)).is_none()); + assert!(matches!( + &session.reconcile(&[device(0, 2)], true)[0], + GenericOutbound::Attach { request_id: 11, .. } + )); + } + + #[test] + fn outputs_require_admission_capture_interface_and_valid_get_length() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 7)], 22); + for operation in 0..=4 { + let bytes = if matches!(operation, 2 | 3) { + vec![0, 0, 1] + } else { + vec![0, 1] + }; + let output = session.output(22, 2, operation, &bytes).unwrap(); + assert_eq!(output.incarnation, 7); + } + for bytes in [&[][..], &[0, 0][..], &[0, 0, 0][..], &[0, 1, 1][..]] { + assert!(session.output(22, 2, 2, bytes).is_none()); + } + assert!(session.output(22, 2, 5, &[1]).is_none()); + assert!(session.output(22, 3, 0, &[1]).is_none()); + assert!(session.output(23, 2, 0, &[1]).is_none()); + assert!(session.output(22, 2, 0, &[0; 257]).is_none()); + session.set_capture(false); + assert!(session.output(22, 2, 0, &[1]).is_none()); + } + + #[test] + fn empty_error_control_replies_pass_through_without_becoming_release_state() { + let mut session = GenericSession::new(1); + attach(&mut session, &[device(0, 1)], 22); + for operation in 0..=5 { + let mut input = report(0, 1); + input.operation = operation; + input.error = true; + input.bytes.clear(); + let outgoing = session.report(input); + assert_eq!(outgoing.is_some(), (1..=4).contains(&operation)); + if let Some(outgoing) = outgoing { + session.sent(&outgoing); + } + } + assert!(session.set_capture(false).is_empty()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic_runtime.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic_runtime.rs new file mode 100644 index 000000000..b66a4f788 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/generic_runtime.rs @@ -0,0 +1,761 @@ +use std::collections::VecDeque; +use std::sync::Mutex; + +use crate::generic::{ + GenericAdmission, GenericAdmissionStatus, GenericDevice, GenericError, GenericOutput, + GenericReport, GenericSession, MAX_GENERIC_SOURCES, validate_inventory, validate_operation, + validate_report, +}; + +const MAX_REPORTS_PER_SOURCE: usize = 64; +const MAX_REPORT_BYTES_PER_SOURCE: usize = 16 * 1024; +const MAX_OUTPUTS: usize = 32; +const MAX_OUTPUT_BYTES: usize = 8 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GenericIngress { + Report(GenericReport), + Release { slot: u8, incarnation: u64 }, + Fault { slot: u8, incarnation: u64 }, +} + +#[derive(Debug, Default)] +struct Lane { + reports: VecDeque, + bytes: usize, + release: Option, + fault: Option, + faulted: Option, +} + +impl Lane { + fn clear_reports(&mut self) { + self.reports.clear(); + self.bytes = 0; + } +} + +#[derive(Debug, Default)] +struct State { + inventory: Vec, + inventory_epoch: u64, + binding: Option, + capture: bool, + release_epoch: u64, + lanes: [Lane; MAX_GENERIC_SOURCES], + next_source: usize, + outputs: VecDeque, + output_bytes: usize, + admissions: [Option; MAX_GENERIC_SOURCES], +} + +impl State { + fn clear_queues(&mut self) { + self.lanes = Default::default(); + self.next_source = 0; + self.outputs.clear(); + self.output_bytes = 0; + self.admissions = Default::default(); + } + + fn source(&self, slot: u8, incarnation: u64, interface: u8) -> Result<(), GenericError> { + let device = self + .inventory + .iter() + .find(|device| device.slot == slot && device.incarnation == incarnation) + .ok_or(GenericError::StaleSource)?; + if !device + .interfaces + .iter() + .any(|entry| entry.number == interface) + { + return Err(GenericError::UnknownInterface); + } + if self.lanes[usize::from(slot)].faulted == Some(incarnation) { + return Err(GenericError::Faulted); + } + Ok(()) + } +} + +#[derive(Debug, Default)] +pub struct GenericRuntime { + state: Mutex, +} + +impl GenericRuntime { + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub fn replace_inventory(&self, inventory: Vec) -> Result<(), GenericError> { + validate_inventory(&inventory)?; + let mut state = self.lock(); + if state.inventory == inventory { + return Ok(()); + } + let previous = std::mem::replace(&mut state.inventory, inventory); + for device in previous { + if state.inventory.contains(&device) { + continue; + } + state.admissions[usize::from(device.slot)] = None; + let lane = &mut state.lanes[usize::from(device.slot)]; + lane.clear_reports(); + lane.release.get_or_insert(device.incarnation); + if !state + .inventory + .iter() + .any(|entry| entry.slot == device.slot && entry.incarnation == device.incarnation) + { + state.lanes[usize::from(device.slot)].faulted = None; + } + } + state.outputs.clear(); + state.output_bytes = 0; + state.inventory_epoch = state.inventory_epoch.wrapping_add(1); + Ok(()) + } + + pub fn inventory_snapshot(&self) -> (Vec, u64) { + let state = self.lock(); + (state.inventory.clone(), state.inventory_epoch) + } + + pub fn publish_admissions(&self, session: &GenericSession) { + let mut state = self.lock(); + if state.binding != Some(session.generation()) { + return; + } + for index in 0..state.inventory.len() { + let device = &state.inventory[index]; + let status = GenericAdmissionStatus { + slot: device.slot, + incarnation: device.incarnation, + admission: session.admission(device), + }; + state.admissions[usize::from(status.slot)] = Some(status); + } + } + + pub fn admissions_snapshot(&self) -> Vec { + let state = self.lock(); + state + .inventory + .iter() + .map(|device| { + state.admissions[usize::from(device.slot)].unwrap_or(GenericAdmissionStatus { + slot: device.slot, + incarnation: device.incarnation, + admission: GenericAdmission::Pending, + }) + }) + .collect() + } + + pub fn submit_report(&self, report: GenericReport) -> Result<(), GenericError> { + validate_report(&report)?; + let mut state = self.lock(); + state.source(report.slot, report.incarnation, report.interface)?; + if state.binding.is_none() { + return Err(GenericError::Unbound); + } + if !state.capture { + return Err(GenericError::Inactive); + } + let lane = &mut state.lanes[usize::from(report.slot)]; + if lane.reports.len() >= MAX_REPORTS_PER_SOURCE + || lane.bytes + report.bytes.len() > MAX_REPORT_BYTES_PER_SOURCE + { + lane.clear_reports(); + lane.fault = Some(report.incarnation); + lane.faulted = Some(report.incarnation); + return Err(GenericError::Overflow); + } + lane.bytes += report.bytes.len(); + lane.reports.push_back(report); + Ok(()) + } + + pub fn drain(&self, limit: usize) -> Vec { + let mut state = self.lock(); + let mut result = Vec::with_capacity(limit.min(MAX_GENERIC_SOURCES * 66)); + for (slot, lane) in state.lanes.iter_mut().enumerate() { + if result.len() == limit { + break; + } + if let Some(incarnation) = lane.fault.take() { + result.push(GenericIngress::Fault { + slot: slot as u8, + incarnation, + }); + } + if result.len() == limit { + break; + } + if let Some(incarnation) = lane.release.take() { + result.push(GenericIngress::Release { + slot: slot as u8, + incarnation, + }); + } + } + while result.len() < limit { + let mut progressed = false; + for _ in 0..MAX_GENERIC_SOURCES { + if result.len() == limit { + break; + } + let slot = state.next_source; + state.next_source = (slot + 1) % MAX_GENERIC_SOURCES; + let lane = &mut state.lanes[slot]; + if let Some(report) = lane.reports.pop_front() { + lane.bytes -= report.bytes.len(); + result.push(GenericIngress::Report(report)); + progressed = true; + } + } + if !progressed { + break; + } + } + result + } + + pub fn set_capture(&self, active: bool) { + let mut state = self.lock(); + if state.capture == active { + return; + } + state.capture = active; + if active { + return; + } + state.release_epoch = state.release_epoch.wrapping_add(1); + for lane in &mut state.lanes { + lane.clear_reports(); + } + state.outputs.clear(); + state.output_bytes = 0; + } + + pub fn capture_state(&self) -> (bool, u64) { + let state = self.lock(); + (state.capture, state.release_epoch) + } + + pub fn bind_session(&self, generation: u64) { + let mut state = self.lock(); + state.clear_queues(); + state.binding = Some(generation); + } + + pub fn unbind_session(&self, generation: u64) { + let mut state = self.lock(); + if state.binding != Some(generation) { + return; + } + state.binding = None; + state.clear_queues(); + } + + pub fn push_output(&self, output: GenericOutput) -> Result<(), GenericError> { + validate_operation(output.operation, &output.bytes, true)?; + let mut state = self.lock(); + state.source(output.slot, output.incarnation, output.interface)?; + if state.binding.is_none() { + return Err(GenericError::Unbound); + } + if !state.capture { + return Err(GenericError::Inactive); + } + if state.outputs.len() >= MAX_OUTPUTS + || state.output_bytes + output.bytes.len() > MAX_OUTPUT_BYTES + { + return Err(GenericError::Overflow); + } + state.output_bytes += output.bytes.len(); + state.outputs.push_back(output); + Ok(()) + } + + pub fn take_output(&self) -> Option { + let mut state = self.lock(); + let output = state.outputs.pop_front()?; + state.output_bytes -= output.bytes.len(); + Some(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::generic::tests::{device, report}; + use crate::generic::{GenericInterface, MAX_DESCRIPTOR_BYTES, MAX_GENERIC_INTERFACES}; + + fn runtime() -> GenericRuntime { + let runtime = GenericRuntime::new(); + runtime.replace_inventory(vec![device(0, 1)]).unwrap(); + runtime.bind_session(1); + runtime.set_capture(true); + runtime + } + + #[test] + fn admission_tracks_verified_responses_and_resets_on_detach_or_replacement() { + let runtime = runtime(); + let mut source = device(0, 1); + source.interfaces[0].descriptor.clear(); + runtime.replace_inventory(vec![source.clone()]).unwrap(); + let mut session = GenericSession::new(1); + let actions = session.reconcile(&[source.clone()], true); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Pending + ); + session.handle_response(10, 1, 0, 22); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Pending + ); + session.sent(&actions[0]); + session.handle_response(10, 1, 0, 22); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Admitted + ); + let detach = session.set_capture(false); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Pending + ); + session.sent(&detach[0]); + let actions = session.set_capture(true); + session.sent(&actions[0]); + session.handle_response(11, 2, 0, 22); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Declined + ); + source.incarnation = 2; + runtime.replace_inventory(vec![source]).unwrap(); + runtime.publish_admissions(&session); + assert_eq!( + runtime.admissions_snapshot(), + vec![GenericAdmissionStatus { + slot: 0, + incarnation: 2, + admission: GenericAdmission::Pending, + }] + ); + runtime.replace_inventory(vec![]).unwrap(); + assert!(runtime.admissions_snapshot().is_empty()); + } + + #[test] + fn admission_reset_is_session_scoped_and_bounded() { + let runtime = runtime(); + let devices = (0..8) + .map(|slot| device(slot, u64::from(slot) + 1)) + .collect::>(); + runtime.replace_inventory(devices.clone()).unwrap(); + let mut session = GenericSession::new(1); + for action in session.reconcile(&devices, true) { + session.sent(&action); + } + for slot in 0..8 { + session.handle_response(10 + slot, 1, 0, 30 + slot); + } + runtime.publish_admissions(&session); + let statuses = runtime.admissions_snapshot(); + assert_eq!(statuses.len(), 8); + assert!( + statuses + .iter() + .all(|row| row.admission == GenericAdmission::Admitted) + ); + runtime.unbind_session(9); + assert_eq!(runtime.admissions_snapshot(), statuses); + runtime.unbind_session(1); + runtime.publish_admissions(&session); + assert!( + runtime + .admissions_snapshot() + .iter() + .all(|row| row.admission == GenericAdmission::Pending) + ); + runtime.bind_session(2); + runtime.publish_admissions(&session); + assert!( + runtime + .admissions_snapshot() + .iter() + .all(|row| row.admission == GenericAdmission::Pending) + ); + } + + fn output() -> GenericOutput { + GenericOutput { + slot: 0, + incarnation: 1, + interface: 2, + operation: 0, + bytes: vec![1], + } + } + + #[test] + fn invalid_inventory_does_not_mutate_existing_inventory() { + let runtime = runtime(); + let original = runtime.inventory_snapshot(); + let mut invalid = device(0, 1); + invalid.slot = 8; + assert_eq!( + runtime.replace_inventory(vec![invalid]), + Err(GenericError::InvalidSlot) + ); + let mut invalid = device(0, 0); + assert_eq!( + runtime.replace_inventory(vec![invalid.clone()]), + Err(GenericError::InvalidIncarnation) + ); + invalid.incarnation = 1; + invalid.product = 1; + assert_eq!( + runtime.replace_inventory(vec![invalid]), + Err(GenericError::NotAllowlisted) + ); + assert_eq!( + runtime.replace_inventory(vec![device(0, 1), device(0, 2)]), + Err(GenericError::DuplicateSlot) + ); + let mut invalid = device(0, 1); + invalid.interfaces.push(invalid.interfaces[0].clone()); + assert_eq!( + runtime.replace_inventory(vec![invalid]), + Err(GenericError::DuplicateInterface) + ); + let mut invalid = device(0, 1); + invalid.interfaces[0].descriptor = vec![0; MAX_DESCRIPTOR_BYTES + 1]; + assert_eq!( + runtime.replace_inventory(vec![invalid]), + Err(GenericError::DescriptorTooLarge) + ); + let mut invalid = device(0, 1); + invalid.interfaces = (0..=MAX_GENERIC_INTERFACES) + .map(|number| GenericInterface { + number: number as u8, + descriptor: vec![], + }) + .collect(); + assert_eq!( + runtime.replace_inventory(vec![invalid]), + Err(GenericError::TooManyInterfaces) + ); + assert_eq!(runtime.inventory_snapshot(), original); + } + + #[test] + fn unknown_interface_255_is_rejected_at_inventory_boundary() { + let runtime = runtime(); + let mut source = device(0, 1); + source.interfaces[0].number = 255; + assert_eq!( + runtime.replace_inventory(vec![source]), + Err(GenericError::UnknownInterface) + ); + assert_eq!(runtime.inventory_snapshot().0, vec![device(0, 1)]); + } + + #[test] + fn only_error_control_replies_can_have_empty_payloads() { + let runtime = runtime(); + for operation in 0..=5 { + for error in [false, true] { + let mut input = report(0, 1); + input.operation = operation; + input.error = error; + input.bytes.clear(); + let admitted = error && (1..=4).contains(&operation); + assert_eq!(runtime.submit_report(input).is_ok(), admitted); + } + } + assert_eq!(runtime.drain(32).len(), 4); + } + + #[test] + fn inventory_accepts_all_128_interfaces_and_unreadable_descriptors() { + let runtime = GenericRuntime::new(); + let devices = (0..8) + .map(|slot| { + let mut entry = device(slot, u64::from(slot) + 1); + entry.interfaces = (0..16) + .map(|number| GenericInterface { + number, + descriptor: vec![], + }) + .collect(); + entry + }) + .collect::>(); + runtime.replace_inventory(devices.clone()).unwrap(); + assert_eq!(runtime.inventory_snapshot().0, devices); + } + + #[test] + fn capture_off_on_discards_backlog_but_preserves_release_epoch() { + let runtime = runtime(); + runtime.submit_report(report(0, 1)).unwrap(); + runtime.push_output(output()).unwrap(); + runtime.set_capture(false); + assert_eq!( + runtime.submit_report(report(0, 1)), + Err(GenericError::Inactive) + ); + assert_eq!(runtime.push_output(output()), Err(GenericError::Inactive)); + runtime.set_capture(true); + assert_eq!(runtime.capture_state(), (true, 1)); + assert!(runtime.take_output().is_none()); + assert!(runtime.drain(0).is_empty()); + runtime.submit_report(report(0, 1)).unwrap(); + assert_eq!(runtime.drain(1), vec![GenericIngress::Report(report(0, 1))]); + runtime.set_capture(false); + runtime.set_capture(false); + assert_eq!(runtime.capture_state(), (false, 2)); + } + + #[test] + fn collapsed_capture_transition_generates_one_descriptor_release() { + use crate::generic::{GenericOutbound, GenericSession}; + + let runtime = runtime(); + let mut session = GenericSession::new(1); + let attach = session.reconcile(&runtime.inventory_snapshot().0, true); + session.sent(&attach[0]); + session.handle_response(10, 1, 0, 22); + runtime.submit_report(report(0, 1)).unwrap(); + let GenericIngress::Report(input) = runtime.drain(1).pop().unwrap() else { + panic!("expected input"); + }; + let outgoing = session.report(input).unwrap(); + session.sent(&outgoing); + runtime.set_capture(false); + runtime.set_capture(true); + let (active, release_epoch) = runtime.capture_state(); + assert_eq!((active, release_epoch), (true, 1)); + let mut outbound = session.set_capture(false); + outbound.extend(session.reconcile(&runtime.inventory_snapshot().0, active)); + assert_eq!(outbound.len(), 1); + assert!(matches!( + outbound[0], + GenericOutbound::Report { release: true, .. } + )); + session.sent(&outbound[0]); + assert!(runtime.drain(8).is_empty()); + assert!( + session + .reconcile(&runtime.inventory_snapshot().0, active) + .is_empty() + ); + runtime.submit_report(report(0, 1)).unwrap(); + let GenericIngress::Report(input) = runtime.drain(1).pop().unwrap() else { + panic!("expected input"); + }; + assert!(matches!( + session.report(input), + Some(GenericOutbound::Report { release: false, .. }) + )); + } + + #[test] + fn overflow_drops_backlog_and_fault_is_sticky_across_drain_and_capture() { + let runtime = runtime(); + for _ in 0..64 { + runtime.submit_report(report(0, 1)).unwrap(); + } + assert_eq!( + runtime.submit_report(report(0, 1)), + Err(GenericError::Overflow) + ); + assert_eq!( + runtime.submit_report(report(0, 1)), + Err(GenericError::Faulted) + ); + runtime.set_capture(false); + runtime.set_capture(true); + assert!(runtime.drain(0).is_empty()); + assert_eq!( + runtime.drain(1), + vec![GenericIngress::Fault { + slot: 0, + incarnation: 1 + }] + ); + assert!(runtime.drain(64).is_empty()); + assert_eq!(runtime.push_output(output()), Err(GenericError::Faulted)); + assert_eq!( + runtime.submit_report(report(0, 1)), + Err(GenericError::Faulted) + ); + runtime.replace_inventory(vec![device(0, 2)]).unwrap(); + runtime.submit_report(report(0, 2)).unwrap(); + } + + #[test] + fn sources_have_independent_64_report_and_16kib_budgets() { + let runtime = runtime(); + runtime + .replace_inventory(vec![device(0, 1), device(1, 2)]) + .unwrap(); + let mut first = report(0, 1); + first.bytes = vec![1; 256]; + for _ in 0..64 { + runtime.submit_report(first.clone()).unwrap(); + } + runtime.submit_report(report(1, 2)).unwrap(); + assert_eq!(runtime.submit_report(first), Err(GenericError::Overflow)); + assert_eq!( + runtime.drain(2), + vec![ + GenericIngress::Fault { + slot: 0, + incarnation: 1 + }, + GenericIngress::Report(report(1, 2)) + ] + ); + } + + #[test] + fn small_drain_budgets_do_not_starve_later_sources() { + let runtime = runtime(); + runtime + .replace_inventory(vec![device(0, 1), device(1, 2)]) + .unwrap(); + for _ in 0..4 { + runtime.submit_report(report(0, 1)).unwrap(); + } + runtime.submit_report(report(1, 2)).unwrap(); + assert_eq!(runtime.drain(1), vec![GenericIngress::Report(report(0, 1))]); + assert_eq!(runtime.drain(1), vec![GenericIngress::Report(report(1, 2))]); + } + + #[test] + fn removal_and_restart_reject_stale_reports_and_clear_session_queues() { + let runtime = runtime(); + runtime.submit_report(report(0, 1)).unwrap(); + runtime.push_output(output()).unwrap(); + runtime.replace_inventory(vec![device(0, 2)]).unwrap(); + assert_eq!( + runtime.submit_report(report(0, 1)), + Err(GenericError::StaleSource) + ); + assert_eq!( + runtime.drain(1), + vec![GenericIngress::Release { + slot: 0, + incarnation: 1 + }] + ); + assert!(runtime.take_output().is_none()); + runtime.submit_report(report(0, 2)).unwrap(); + runtime.bind_session(2); + assert!(runtime.drain(8).is_empty()); + assert_eq!(runtime.inventory_snapshot().0, vec![device(0, 2)]); + runtime.unbind_session(1); + runtime.submit_report(report(0, 2)).unwrap(); + runtime.unbind_session(2); + assert!(runtime.drain(8).is_empty()); + assert_eq!( + runtime.submit_report(report(0, 2)), + Err(GenericError::Unbound) + ); + } + + #[test] + fn output_queue_is_bounded_and_capture_or_session_closure_clears_it() { + let runtime = runtime(); + let mut outgoing = output(); + outgoing.bytes = vec![1; 256]; + for _ in 0..32 { + runtime.push_output(outgoing.clone()).unwrap(); + } + assert_eq!( + runtime.push_output(outgoing.clone()), + Err(GenericError::Overflow) + ); + assert_eq!(runtime.take_output(), Some(outgoing.clone())); + runtime.push_output(outgoing).unwrap(); + runtime.set_capture(false); + assert!(runtime.take_output().is_none()); + runtime.set_capture(true); + runtime.push_output(output()).unwrap(); + runtime.unbind_session(1); + assert!(runtime.take_output().is_none()); + assert_eq!(runtime.push_output(output()), Err(GenericError::Unbound)); + } + + #[test] + fn hid_runtime_propagates_session_capture_and_endpoint_closure() { + let runtime = crate::HidRuntime::new(); + runtime + .generic + .replace_inventory(vec![device(0, 1)]) + .unwrap(); + let endpoint = runtime.open_endpoint(); + runtime.bind_session(1).unwrap(); + runtime.set_active(true); + runtime.generic.submit_report(report(0, 1)).unwrap(); + runtime.set_active(false); + runtime.set_active(true); + assert_eq!(runtime.generic.capture_state(), (true, 1)); + assert!(runtime.generic.drain(1).is_empty()); + runtime.generic.submit_report(report(0, 1)).unwrap(); + runtime.close_endpoint(endpoint); + assert!(runtime.generic.drain(1).is_empty()); + assert_eq!( + runtime.generic.submit_report(report(0, 1)), + Err(GenericError::Unbound) + ); + assert_eq!(runtime.generic.inventory_snapshot().0, vec![device(0, 1)]); + } + + #[test] + fn replaced_endpoint_clears_generic_queues_and_stale_endpoint_cannot_close_new_session() { + let runtime = crate::HidRuntime::new(); + runtime + .generic + .replace_inventory(vec![device(0, 1)]) + .unwrap(); + let first = runtime.open_endpoint(); + runtime.bind_session(1).unwrap(); + runtime.set_active(true); + runtime.generic.submit_report(report(0, 1)).unwrap(); + let second = runtime.open_endpoint(); + assert!(runtime.generic.drain(8).is_empty()); + assert_eq!( + runtime.generic.submit_report(report(0, 1)), + Err(GenericError::Unbound) + ); + runtime.bind_session(2).unwrap(); + runtime.close_endpoint(first); + runtime.generic.submit_report(report(0, 1)).unwrap(); + runtime.close_endpoint(second); + assert!(runtime.generic.drain(8).is_empty()); + assert_eq!( + runtime.generic.submit_report(report(0, 1)), + Err(GenericError::Unbound) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/lib.rs new file mode 100644 index 000000000..2b121930e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/lib.rs @@ -0,0 +1,443 @@ +use std::fmt; + +pub mod allowlist; +pub mod descriptor; +pub mod ds4; +pub mod generic; +pub mod generic_runtime; +pub mod runtime; +pub mod session; + +pub use ds4::{DS4_INTERFACE, DS4_LOW_ID_BASE, DS4_REPORT_BYTES, Ds4ReportState}; +pub use generic_runtime::{GenericIngress, GenericRuntime}; +pub use runtime::{ + EndpointToken, HidRuntime, IngressItem, InventoryUpdate, LaneFault, ReleaseRequest, + SessionBinding, SourceMode, +}; +pub use session::{HidAttachment, HidOutbound, HidSession, SonyRumble}; +pub const SONY_VENDOR: u16 = 0x054c; +pub const SONY_DS4_PRODUCTS: [u16; 3] = [0x05c4, 0x09cc, 0x0ba0]; +pub const SONY_DS5_PRODUCTS: [u16; 2] = [0x0ce6, 0x0df2]; + +pub const MAX_SOURCES: usize = 4; +pub const CONTACTS_PER_SOURCE: usize = 2; +pub const MAX_QUEUED_STATES: usize = 64; +pub const MAX_QUEUED_STATE_BYTES: usize = 8 * 1024; +pub const DRAIN_PER_ITERATION: usize = 32; +pub const MAX_EXACT_INCARNATION: u64 = 9_007_199_254_740_991; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SonyTarget { + Ds4, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SonyCapability { + pub synthesized_ds4_support: bool, + pub server_mask: u32, + pub ds4_source_allowed: bool, + pub dualsense_source_allowed: bool, + pub cross_synthesize_ds5_to_ds4: bool, +} + +impl Default for SonyCapability { + fn default() -> Self { + Self { + synthesized_ds4_support: true, + server_mask: 0, + ds4_source_allowed: true, + dualsense_source_allowed: true, + cross_synthesize_ds5_to_ds4: true, + } + } +} + +impl SonyCapability { + pub const SERVER_DS4_TARGET_BIT: u32 = 0x4; + pub const SERVER_GAME_PERMISSION_BIT: u32 = 0x1; + + pub fn server_supports_ds4(self) -> bool { + self.server_mask & Self::SERVER_DS4_TARGET_BIT != 0 + } + + pub fn game_allows_ds4(self) -> bool { + self.server_mask & Self::SERVER_GAME_PERMISSION_BIT != 0 + } + + pub fn admits(self, target: SonyTarget, cross_synthesis: bool) -> bool { + self.synthesized_ds4_support + && self.server_supports_ds4() + && self.game_allows_ds4() + && match (target, cross_synthesis) { + (SonyTarget::Ds4, false) => self.ds4_source_allowed, + (SonyTarget::Ds4, true) => { + self.dualsense_source_allowed && self.cross_synthesize_ds5_to_ds4 + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SdlDeviceClaim { + pub slot: u8, + pub incarnation: u64, + pub vendor: u16, + pub product: u16, +} + +impl SdlDeviceClaim { + pub fn new(slot: u8, incarnation: u64, vendor: u16, product: u16) -> Option { + if usize::from(slot) >= MAX_SOURCES + || incarnation == 0 + || incarnation > MAX_EXACT_INCARNATION + { + return None; + } + Some(Self { + slot, + incarnation, + vendor, + product, + }) + } + + pub fn sony_product(self) -> Option<(SonyTarget, bool)> { + if self.vendor != SONY_VENDOR { + return None; + } + if SONY_DS4_PRODUCTS.contains(&self.product) { + return Some((SonyTarget::Ds4, false)); + } + if SONY_DS5_PRODUCTS.contains(&self.product) { + return Some((SonyTarget::Ds4, true)); + } + None + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SonyContact { + pub active: bool, + pub x: i16, + pub y: i16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SonySnapshot { + pub slot: u8, + pub incarnation: u64, + pub buttons: u16, + pub left_trigger: u8, + pub right_trigger: u8, + pub left_stick_x: i16, + pub left_stick_y: i16, + pub right_stick_x: i16, + pub right_stick_y: i16, + pub touchpad_click: bool, + pub contacts: [SonyContact; CONTACTS_PER_SOURCE], + pub observed_at_us: u64, +} + +impl SonySnapshot { + pub fn neutral(slot: u8, incarnation: u64, observed_at_us: u64) -> Self { + Self { + slot, + incarnation, + buttons: 0, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + touchpad_click: false, + contacts: [SonyContact::default(); CONTACTS_PER_SOURCE], + observed_at_us, + } + } + + pub fn validate(self) -> Result { + if usize::from(self.slot) >= MAX_SOURCES { + return Err(SonySnapshotError::SlotOutOfRange(self.slot)); + } + if self.incarnation == 0 { + return Err(SonySnapshotError::MissingIncarnation); + } + if self.incarnation > MAX_EXACT_INCARNATION { + return Err(SonySnapshotError::IncarnationOutOfRange(self.incarnation)); + } + Ok(self) + } + + pub fn is_neutral(self) -> bool { + let centered = |axis: i16| axis == 0; + self.buttons == 0 + && self.left_trigger == 0 + && self.right_trigger == 0 + && centered(self.left_stick_x) + && centered(self.left_stick_y) + && centered(self.right_stick_x) + && centered(self.right_stick_y) + && !self.touchpad_click + && self.contacts.iter().all(|contact| !contact.active) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SonySnapshotError { + SlotOutOfRange(u8), + MissingIncarnation, + IncarnationOutOfRange(u64), +} + +impl fmt::Display for SonySnapshotError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SlotOutOfRange(slot) => { + write!(formatter, "sony snapshot slot {slot} is out of range") + } + Self::MissingIncarnation => { + formatter.write_str("sony snapshot has no source incarnation") + } + Self::IncarnationOutOfRange(incarnation) => { + write!( + formatter, + "sony snapshot incarnation {incarnation} exceeds the exactly representable range" + ) + } + } + } +} + +impl std::error::Error for SonySnapshotError {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SnapshotAdmission { + Admitted, + AdmittedInactive, + Unbound, + Faulted, + StaleSource, + Overflow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InventoryOutcome { + Replaced, + DuplicateSlot, + DuplicateIncarnation, + MalformedEntry, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct InventoryView { + claims: [Option; MAX_SOURCES], +} + +impl InventoryView { + pub fn claims(&self) -> &[Option; MAX_SOURCES] { + &self.claims + } + + pub fn claim(&self, slot: u8) -> Option { + self.claims.get(usize::from(slot)).copied().flatten() + } + + pub fn incarnation(&self, slot: u8) -> Option { + self.claim(slot).map(|claim| claim.incarnation) + } + + pub fn replace(&mut self, claims: &[Option]) -> InventoryOutcome { + if claims.len() > MAX_SOURCES { + return InventoryOutcome::MalformedEntry; + } + let mut next = [None; MAX_SOURCES]; + let mut seen_incarnations: [u64; MAX_SOURCES] = [0; MAX_SOURCES]; + for (seen, claim) in claims.iter().flatten().enumerate().take(MAX_SOURCES) { + if usize::from(claim.slot) >= MAX_SOURCES || claim.incarnation == 0 { + return InventoryOutcome::MalformedEntry; + } + if next[usize::from(claim.slot)].is_some() { + return InventoryOutcome::DuplicateSlot; + } + if seen_incarnations[..seen].contains(&claim.incarnation) { + return InventoryOutcome::DuplicateIncarnation; + } + seen_incarnations[seen] = claim.incarnation; + next[usize::from(claim.slot)] = Some(*claim); + } + self.claims = next; + InventoryOutcome::Replaced + } + + pub fn rich_slot_mask(&self) -> u16 { + let mut mask = 0_u16; + for (slot, claim) in self.claims.iter().enumerate() { + if claim.is_some_and(|claim| claim.sony_product().is_some()) { + mask |= 1 << slot; + } + } + mask + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ds4(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, SONY_VENDOR, 0x05c4) + } + + fn xbox(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, 0x045e, 0x02ea) + } + + #[test] + fn claim_rejects_out_of_range_slot_and_missing_incarnation() { + assert!(SdlDeviceClaim::new(4, 1, SONY_VENDOR, 0x05c4).is_none()); + assert!(SdlDeviceClaim::new(0, 0, SONY_VENDOR, 0x05c4).is_none()); + assert!(SdlDeviceClaim::new(3, 7, SONY_VENDOR, 0x05c4).is_some()); + assert!(SdlDeviceClaim::new(3, MAX_EXACT_INCARNATION, SONY_VENDOR, 0x05c4).is_some()); + assert!(SdlDeviceClaim::new(3, MAX_EXACT_INCARNATION + 1, SONY_VENDOR, 0x05c4).is_none()); + assert!(SdlDeviceClaim::new(3, u64::MAX, SONY_VENDOR, 0x05c4).is_none()); + } + + #[test] + fn sony_product_classifies_ds4_and_ds5_only_for_the_sony_vendor() { + let ds4 = SdlDeviceClaim::new(0, 1, SONY_VENDOR, 0x09cc).unwrap(); + assert_eq!(ds4.sony_product(), Some((SonyTarget::Ds4, false))); + let ds5 = SdlDeviceClaim::new(0, 1, SONY_VENDOR, 0x0ce6).unwrap(); + assert_eq!(ds5.sony_product(), Some((SonyTarget::Ds4, true))); + let edge = SdlDeviceClaim::new(0, 1, SONY_VENDOR, 0x0df2).unwrap(); + assert_eq!(edge.sony_product(), Some((SonyTarget::Ds4, true))); + let other = SdlDeviceClaim::new(0, 1, SONY_VENDOR, 0x0eee).unwrap(); + assert_eq!(other.sony_product(), None); + let clone = SdlDeviceClaim::new(0, 1, 0x045e, 0x0ce6).unwrap(); + assert_eq!(clone.sony_product(), None); + } + + #[test] + fn inventory_replacement_is_atomic_and_rejects_malformed_entries() { + let mut inventory = InventoryView::default(); + let good = [ds4(0, 11), xbox(1, 12), None, ds4(3, 14)]; + assert_eq!(inventory.replace(&good), InventoryOutcome::Replaced); + assert_eq!(inventory.incarnation(0), Some(11)); + assert_eq!(inventory.rich_slot_mask(), 0b1001); + + let mut out_of_range = [None; MAX_SOURCES]; + out_of_range[0] = Some(SdlDeviceClaim { + slot: 255, + incarnation: 21, + vendor: SONY_VENDOR, + product: 0x05c4, + }); + assert_eq!( + inventory.replace(&out_of_range), + InventoryOutcome::MalformedEntry + ); + assert_eq!(inventory.incarnation(0), Some(11)); + + let mut duplicate_port = [None; MAX_SOURCES]; + duplicate_port[0] = ds4(0, 31); + duplicate_port[1] = Some(SdlDeviceClaim { + slot: 0, + incarnation: 32, + vendor: SONY_VENDOR, + product: 0x05c4, + }); + assert_eq!( + inventory.replace(&duplicate_port), + InventoryOutcome::DuplicateSlot + ); + assert_eq!(inventory.incarnation(0), Some(11)); + + let mut duplicate_incarnation = [None; MAX_SOURCES]; + duplicate_incarnation[0] = ds4(0, 41); + duplicate_incarnation[1] = xbox(1, 41); + assert_eq!( + inventory.replace(&duplicate_incarnation), + InventoryOutcome::DuplicateIncarnation + ); + assert_eq!(inventory.incarnation(0), Some(11)); + + let mut zero_incarnation = [None; MAX_SOURCES]; + zero_incarnation[0] = Some(SdlDeviceClaim { + slot: 0, + incarnation: 0, + vendor: SONY_VENDOR, + product: 0x05c4, + }); + assert_eq!( + inventory.replace(&zero_incarnation), + InventoryOutcome::MalformedEntry + ); + + let oversized = [None; MAX_SOURCES + 1]; + assert_eq!( + inventory.replace(&oversized), + InventoryOutcome::MalformedEntry + ); + assert_eq!(inventory.incarnation(0), Some(11)); + assert_eq!(inventory.incarnation(3), Some(14)); + } + + #[test] + fn inventory_entries_may_arrive_in_any_order() { + let mut inventory = InventoryView::default(); + let reordered = [None, ds4(1, 5), None, None]; + assert_eq!(inventory.replace(&reordered), InventoryOutcome::Replaced); + assert_eq!(inventory.incarnation(1), Some(5)); + let shifted = [ + Some(SdlDeviceClaim { + slot: 3, + ..ds4(3, 6).unwrap() + }), + None, + None, + None, + ]; + assert_eq!(inventory.replace(&shifted), InventoryOutcome::Replaced); + assert_eq!(inventory.incarnation(3), Some(6)); + assert_eq!(inventory.incarnation(1), None); + } + + #[test] + fn snapshot_validation_rejects_bad_slot_and_missing_incarnation() { + let snapshot = SonySnapshot::neutral(4, 1, 0); + assert_eq!( + snapshot.validate(), + Err(SonySnapshotError::SlotOutOfRange(4)) + ); + let snapshot = SonySnapshot::neutral(2, 0, 0); + assert_eq!( + snapshot.validate(), + Err(SonySnapshotError::MissingIncarnation) + ); + assert!(SonySnapshot::neutral(2, 9, 10).validate().is_ok()); + assert!( + SonySnapshot::neutral(2, MAX_EXACT_INCARNATION, 10) + .validate() + .is_ok() + ); + assert_eq!( + SonySnapshot::neutral(2, MAX_EXACT_INCARNATION + 1, 10).validate(), + Err(SonySnapshotError::IncarnationOutOfRange( + MAX_EXACT_INCARNATION + 1 + )) + ); + assert_eq!( + SonySnapshot::neutral(2, u64::MAX, 10).validate(), + Err(SonySnapshotError::IncarnationOutOfRange(u64::MAX)) + ); + } + + #[test] + fn neutral_snapshot_centers_axes_and_releases_contacts() { + let snapshot = SonySnapshot::neutral(1, 3, 42); + assert_eq!(snapshot.buttons, 0); + assert_eq!(snapshot.left_stick_x, 0); + assert_eq!(snapshot.observed_at_us, 42); + assert!(snapshot.contacts.iter().all(|contact| !contact.active)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/runtime.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/runtime.rs new file mode 100644 index 000000000..840b3b383 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/runtime.rs @@ -0,0 +1,738 @@ +use std::collections::VecDeque; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::{ + InventoryOutcome, InventoryView, MAX_QUEUED_STATE_BYTES, MAX_QUEUED_STATES, MAX_SOURCES, + SdlDeviceClaim, SnapshotAdmission, SonySnapshot, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReleaseRequest { + pub slot: u8, + pub incarnation: u64, + pub observed_at_us: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LaneFault { + Overflow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceMode { + Rich, + OrdinaryFallback, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EndpointToken(u64); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IngressItem { + Snapshot(SonySnapshot), + Release { + slot: u8, + incarnation: u64, + observed_at_us: u64, + }, + Fault { + slot: u8, + incarnation: u64, + observed_at_us: u64, + }, +} + +#[derive(Debug, Default)] +struct SourceLane { + states: VecDeque, + bytes: usize, + release: Option, + fault: Option, + last_observed_at_us: u64, + dropped: u64, +} + +impl SourceLane { + fn reset(&mut self) { + self.states.clear(); + self.bytes = 0; + self.release = None; + self.fault = None; + self.last_observed_at_us = 0; + } + + fn drop_buffered(&mut self) { + self.states.clear(); + self.bytes = 0; + } + + fn arm_release(&mut self, slot: u8, incarnation: u64) { + if self.release.is_none() { + self.release = Some(ReleaseRequest { + slot, + incarnation, + observed_at_us: self.last_observed_at_us, + }); + } + } + + fn push(&mut self, snapshot: SonySnapshot) -> SnapshotAdmission { + if self.fault.is_some() { + self.dropped = self.dropped.saturating_add(1); + return SnapshotAdmission::Faulted; + } + self.last_observed_at_us = self.last_observed_at_us.max(snapshot.observed_at_us); + let bytes = core::mem::size_of::(); + if self.states.len() >= MAX_QUEUED_STATES + || self.bytes.saturating_add(bytes) > MAX_QUEUED_STATE_BYTES + { + self.drop_buffered(); + self.fault = Some(LaneFault::Overflow); + self.dropped = self.dropped.saturating_add(1); + self.arm_release(snapshot.slot, snapshot.incarnation); + return SnapshotAdmission::Overflow; + } + self.bytes += bytes; + self.states.push_back(snapshot); + SnapshotAdmission::Admitted + } +} + +#[derive(Debug, Default)] +struct RuntimeState { + binding: Option, + capture_active: bool, + inventory: InventoryView, + inventory_epoch: u64, + endpoint: Option, + lanes: [SourceLane; MAX_SOURCES], + session_evaluated_epoch: Option, + session_admitted: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SessionBinding { + pub generation: u64, +} + +#[derive(Debug, Default)] +pub struct HidRuntime { + pub generic: crate::generic_runtime::GenericRuntime, + state: Mutex, + incarnation_source: AtomicU64, + inventory_epoch: AtomicU64, + endpoint_source: AtomicU64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InventoryUpdate { + pub outcome: InventoryOutcome, + pub epoch: u64, +} + +impl HidRuntime { + pub fn new() -> Self { + Self { + generic: crate::generic_runtime::GenericRuntime::new(), + state: Mutex::new(RuntimeState::default()), + incarnation_source: AtomicU64::new(1), + inventory_epoch: AtomicU64::new(0), + endpoint_source: AtomicU64::new(1), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, RuntimeState> { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub fn next_incarnation(&self) -> u64 { + self.incarnation_source.fetch_add(1, Ordering::AcqRel) + } + + pub fn inventory_epoch(&self) -> u64 { + self.inventory_epoch.load(Ordering::Acquire) + } + + pub fn inventory(&self) -> InventoryView { + self.lock().inventory + } + + pub fn replace_inventory(&self, claims: &[Option]) -> InventoryUpdate { + let mut state = self.lock(); + let previous = state.inventory; + let outcome = state.inventory.replace(claims); + if outcome != InventoryOutcome::Replaced { + return InventoryUpdate { + outcome, + epoch: state.inventory_epoch, + }; + } + for slot in 0..MAX_SOURCES as u8 { + if previous.incarnation(slot) == state.inventory.incarnation(slot) { + continue; + } + state.lanes[usize::from(slot)].reset(); + } + state.session_evaluated_epoch = None; + state.session_admitted = 0; + state.inventory_epoch = state.inventory_epoch.wrapping_add(1); + let epoch = state.inventory_epoch; + self.inventory_epoch.store(epoch, Ordering::Release); + InventoryUpdate { outcome, epoch } + } + + pub fn inventory_snapshot(&self) -> (InventoryView, u64) { + let state = self.lock(); + (state.inventory, state.inventory_epoch) + } + + pub fn open_endpoint(&self) -> EndpointToken { + let mut state = self.lock(); + if let Some(generation) = state.binding { + self.generic.unbind_session(generation); + } + let token = self.endpoint_source.fetch_add(1, Ordering::AcqRel); + for lane in state.lanes.iter_mut() { + lane.reset(); + } + state.session_evaluated_epoch = None; + state.session_admitted = 0; + state.endpoint = Some(token); + EndpointToken(token) + } + + pub fn close_endpoint(&self, token: EndpointToken) { + let mut state = self.lock(); + if state.endpoint != Some(token.0) { + return; + } + if let Some(generation) = state.binding { + self.generic.unbind_session(generation); + } + state.endpoint = None; + for lane in state.lanes.iter_mut() { + lane.reset(); + } + state.session_evaluated_epoch = None; + state.session_admitted = 0; + state.binding = None; + } + + pub fn bind_session(&self, generation: u64) -> Option { + let mut state = self.lock(); + state.endpoint?; + self.generic.bind_session(generation); + for lane in state.lanes.iter_mut() { + lane.reset(); + } + state.session_evaluated_epoch = None; + state.session_admitted = 0; + state.binding = Some(generation); + Some(SessionBinding { generation }) + } + + pub fn unbind_session(&self, generation: u64) { + let mut state = self.lock(); + if state.binding.is_some_and(|bound| bound != generation) { + return; + } + self.generic.unbind_session(generation); + for lane in state.lanes.iter_mut() { + lane.reset(); + } + state.session_evaluated_epoch = None; + state.session_admitted = 0; + state.binding = None; + } + + pub fn session_generation(&self) -> Option { + self.lock().binding + } + + pub fn set_session_admission(&self, admitted: u16, epoch: u64) -> bool { + let mut state = self.lock(); + if epoch != state.inventory_epoch { + return false; + } + state.session_evaluated_epoch = Some(epoch); + state.session_admitted = admitted; + true + } + + fn mode_for_epoch(state: &RuntimeState, slot: u8) -> Option { + if state.session_evaluated_epoch != Some(state.inventory_epoch) { + return None; + } + state.inventory.incarnation(slot)?; + Some(if state.session_admitted & (1 << slot) != 0 { + SourceMode::Rich + } else { + SourceMode::OrdinaryFallback + }) + } + + pub fn source_mode(&self, slot: u8) -> Option { + Self::mode_for_epoch(&self.lock(), slot) + } + + pub fn set_active(&self, active: bool) { + let mut state = self.lock(); + self.generic.set_capture(active); + if state.capture_active == active { + return; + } + state.capture_active = active; + if active { + return; + } + for slot in 0..MAX_SOURCES as u8 { + let Some(incarnation) = state.inventory.incarnation(slot) else { + continue; + }; + let lane = &mut state.lanes[usize::from(slot)]; + lane.drop_buffered(); + lane.arm_release(slot, incarnation); + } + } + + pub fn is_active(&self) -> bool { + self.lock().capture_active + } + + pub fn output_admitted(&self, slot: u8) -> bool { + let state = self.lock(); + if !state.capture_active || state.binding.is_none() { + return false; + } + if state.inventory.incarnation(slot).is_none() { + return false; + } + state + .lanes + .get(usize::from(slot)) + .is_some_and(|lane| lane.fault.is_none()) + } + + pub fn submit_snapshot_with( + &self, + snapshot: SonySnapshot, + admit_ordinary: F, + ) -> SnapshotAdmission + where + F: FnOnce(&SonySnapshot, u16), + { + let snapshot = match snapshot.validate() { + Ok(snapshot) => snapshot, + Err(_) => return SnapshotAdmission::StaleSource, + }; + let mut state = self.lock(); + if state.binding.is_none() { + return SnapshotAdmission::Unbound; + } + let Some(incarnation) = state.inventory.incarnation(snapshot.slot) else { + return SnapshotAdmission::StaleSource; + }; + if incarnation != snapshot.incarnation { + return SnapshotAdmission::StaleSource; + } + let Some(mode) = Self::mode_for_epoch(&state, snapshot.slot) else { + return SnapshotAdmission::StaleSource; + }; + if mode == SourceMode::OrdinaryFallback { + let mut bitmap = 0_u16; + for slot in 0..MAX_SOURCES as u8 { + if state.inventory.incarnation(slot).is_some() { + bitmap |= (1_u16 << slot) | (1_u16 << (slot + 8)); + } + } + admit_ordinary(&snapshot, bitmap); + return SnapshotAdmission::Admitted; + } + let active = state.capture_active; + let lane = &mut state.lanes[usize::from(snapshot.slot)]; + lane.last_observed_at_us = lane.last_observed_at_us.max(snapshot.observed_at_us); + if !active { + if snapshot.is_neutral() { + lane.arm_release(snapshot.slot, snapshot.incarnation); + } + return SnapshotAdmission::AdmittedInactive; + } + lane.push(snapshot) + } + + pub fn submit_snapshot(&self, snapshot: SonySnapshot) -> SnapshotAdmission { + self.submit_snapshot_with(snapshot, |_, _| {}) + } + + pub fn drain(&self, budget: usize) -> Vec { + let mut drained = Vec::with_capacity(budget.min(crate::DRAIN_PER_ITERATION)); + let mut state = self.lock(); + for lane in state.lanes.iter_mut() { + if drained.len() >= budget { + break; + } + if let Some(release) = lane.release.take() { + let item = match lane.fault.take() { + Some(_) => IngressItem::Fault { + slot: release.slot, + incarnation: release.incarnation, + observed_at_us: release.observed_at_us, + }, + None => IngressItem::Release { + slot: release.slot, + incarnation: release.incarnation, + observed_at_us: release.observed_at_us, + }, + }; + drained.push(item); + if drained.len() >= budget { + break; + } + } + while drained.len() < budget { + let Some(snapshot) = lane.states.pop_front() else { + break; + }; + lane.bytes = lane + .bytes + .saturating_sub(core::mem::size_of::()); + drained.push(IngressItem::Snapshot(snapshot)); + } + } + drained + } + + pub fn pending_states(&self) -> usize { + self.lock().lanes.iter().map(|lane| lane.states.len()).sum() + } + + pub fn dropped_states(&self) -> u64 { + self.lock().lanes.iter().map(|lane| lane.dropped).sum() + } + + pub fn faulted_sources(&self) -> u16 { + let mut mask = 0_u16; + for (slot, lane) in self.lock().lanes.iter().enumerate() { + if lane.fault.is_some() { + mask |= 1 << slot; + } + } + mask + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{SONY_VENDOR, SonyContact}; + + fn ds4(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, SONY_VENDOR, 0x05c4) + } + + fn runtime_with(claims: [Option; 4]) -> HidRuntime { + let runtime = HidRuntime::new(); + assert_eq!( + runtime.replace_inventory(&claims).outcome, + InventoryOutcome::Replaced + ); + runtime.open_endpoint(); + assert!(runtime.bind_session(1).is_some()); + publish_rich_admission(&runtime, &claims); + runtime + } + + fn publish_rich_admission(runtime: &HidRuntime, claims: &[Option; 4]) { + let rich = claims + .iter() + .flatten() + .filter(|claim| claim.sony_product().is_some()) + .fold(0_u16, |mask, claim| mask | (1 << claim.slot)); + assert!(runtime.set_session_admission(rich, runtime.inventory_snapshot().1)); + } + + fn snapshot(slot: u8, incarnation: u64) -> SonySnapshot { + SonySnapshot::neutral(slot, incarnation, 0) + } + + #[test] + fn incarnation_source_never_repeats() { + let runtime = HidRuntime::new(); + let first = runtime.next_incarnation(); + let second = runtime.next_incarnation(); + assert_ne!(first, second); + assert!(second > first); + } + + #[test] + fn unsupported_inventory_replacement_leaves_the_previous_view_intact() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + let before = runtime.inventory_epoch(); + let mut duplicate = [None; 4]; + duplicate[0] = ds4(2, 9); + duplicate[1] = Some(SdlDeviceClaim { + slot: 2, + incarnation: 10, + vendor: SONY_VENDOR, + product: 0x05c4, + }); + assert_eq!( + runtime.replace_inventory(&duplicate).outcome, + InventoryOutcome::DuplicateSlot + ); + assert_eq!(runtime.inventory_epoch(), before); + assert_eq!(runtime.inventory().incarnation(0), Some(1)); + assert_eq!(runtime.inventory().incarnation(2), None); + } + + #[test] + fn snapshots_for_unknown_or_replaced_sources_are_rejected() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 2)), + SnapshotAdmission::StaleSource + ); + assert_eq!( + runtime.submit_snapshot(snapshot(1, 1)), + SnapshotAdmission::StaleSource + ); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Admitted + ); + } + + #[test] + fn inactive_capture_drops_non_neutral_snapshots_but_keeps_the_release() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + let mut active_snapshot = snapshot(0, 1); + active_snapshot.left_stick_x = 4096; + assert_eq!( + runtime.submit_snapshot(active_snapshot), + SnapshotAdmission::AdmittedInactive + ); + assert!(runtime.drain(8).is_empty()); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::AdmittedInactive + ); + assert_eq!( + runtime.drain(8), + vec![IngressItem::Release { + slot: 0, + incarnation: 1, + observed_at_us: 0, + }] + ); + } + + #[test] + fn capture_loss_arms_one_release_per_source_that_resume_preserves_until_drained() { + let runtime = runtime_with([ + Some(ds4(0, 1).unwrap()), + None, + Some(ds4(2, 3).unwrap()), + None, + ]); + runtime.set_active(true); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Admitted + ); + runtime.set_active(false); + runtime.set_active(false); + let items = runtime.drain(8); + assert_eq!(items.len(), 2); + assert!( + items + .iter() + .all(|item| matches!(item, IngressItem::Release { .. })) + ); + } + + #[test] + fn repeated_capture_toggles_do_not_accumulate_duplicate_releases() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + for _ in 0..64 { + runtime.set_active(false); + runtime.set_active(true); + let items = runtime.drain(8); + assert_eq!(items.len(), 1); + assert_eq!( + items[0], + IngressItem::Release { + slot: 0, + incarnation: 1, + observed_at_us: 0, + } + ); + } + } + + #[test] + fn overflow_faults_the_source_and_keeps_the_release_until_consumed() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + for _ in 0..MAX_QUEUED_STATES { + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Admitted + ); + } + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Overflow + ); + assert!(runtime.dropped_states() >= 1); + assert_eq!(runtime.faulted_sources(), 0b0001); + assert!(!runtime.output_admitted(0)); + assert_eq!( + runtime.submit_snapshot({ + let mut held = snapshot(0, 1); + held.buttons = 0x1000; + held + }), + SnapshotAdmission::Faulted + ); + let items = runtime.drain(crate::MAX_QUEUED_STATES + 8); + assert_eq!(items.len(), 1); + assert!(matches!(items[0], IngressItem::Fault { slot: 0, .. })); + assert_eq!(runtime.pending_states(), 0); + assert_eq!(runtime.faulted_sources(), 0); + assert!(runtime.output_admitted(0)); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Admitted + ); + } + + #[test] + fn unbound_sessions_reject_admission_and_drop_queued_input() { + let runtime = HidRuntime::new(); + assert_eq!( + runtime + .replace_inventory(&[Some(ds4(0, 1).unwrap()), None, None, None]) + .outcome, + InventoryOutcome::Replaced + ); + runtime.set_active(true); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Unbound + ); + assert_eq!(runtime.session_generation(), None); + assert!(!runtime.output_admitted(0)); + runtime.open_endpoint(); + assert_eq!(runtime.bind_session(7).unwrap().generation, 7); + assert_eq!(runtime.session_generation(), Some(7)); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::StaleSource + ); + assert!(runtime.set_session_admission(0b0001, runtime.inventory_snapshot().1)); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Admitted + ); + assert!( + runtime + .drain(8) + .iter() + .any(|item| matches!(item, IngressItem::Snapshot(_))) + ); + assert!(runtime.output_admitted(0)); + runtime.unbind_session(8); + assert_eq!(runtime.session_generation(), Some(7)); + runtime.unbind_session(7); + assert_eq!(runtime.session_generation(), None); + assert!(runtime.drain(8).is_empty()); + assert_eq!(runtime.pending_states(), 0); + assert!(!runtime.output_admitted(0)); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::Unbound + ); + } + + #[test] + fn output_admission_requires_capture_a_binding_and_a_healthy_source() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + assert!(!runtime.output_admitted(0)); + runtime.set_active(true); + assert!(runtime.output_admitted(0)); + assert!(!runtime.output_admitted(1)); + runtime.set_active(false); + assert!(!runtime.output_admitted(0)); + } + + #[test] + fn drain_budget_bounds_work_per_iteration() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + for _ in 0..MAX_QUEUED_STATES { + let _ = runtime.submit_snapshot(snapshot(0, 1)); + } + let first = runtime.drain(32); + assert_eq!(first.len(), 32); + assert_eq!(runtime.pending_states(), MAX_QUEUED_STATES - 32); + let second = runtime.drain(32); + assert_eq!(second.len(), 32); + assert_eq!(runtime.pending_states(), 0); + } + + #[test] + fn inventory_replacement_resets_the_lane_and_drops_previous_states() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + for _ in 0..8 { + let _ = runtime.submit_snapshot(snapshot(0, 1)); + } + runtime.set_active(false); + assert!( + runtime + .replace_inventory(&[None, Some(ds4(1, 2).unwrap()), None, None]) + .outcome + == InventoryOutcome::Replaced + ); + assert_eq!(runtime.pending_states(), 0); + assert!(runtime.drain(8).is_empty()); + } + + #[test] + fn repeated_neutral_reports_arm_one_release_until_drained() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + runtime.set_active(true); + runtime.set_active(false); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::AdmittedInactive + ); + assert_eq!( + runtime.submit_snapshot(snapshot(0, 1)), + SnapshotAdmission::AdmittedInactive + ); + assert_eq!(runtime.drain(8).len(), 1); + assert!(runtime.drain(8).is_empty()); + } + + #[test] + fn neutral_snapshot_keeps_contacts_but_is_recognized_as_a_release() { + let runtime = runtime_with([Some(ds4(0, 1).unwrap()), None, None, None]); + let mut neutral = snapshot(0, 1); + neutral.contacts[0] = SonyContact { + active: false, + x: 100, + y: -100, + }; + assert_eq!( + runtime.submit_snapshot(neutral), + SnapshotAdmission::AdmittedInactive + ); + assert_eq!(runtime.drain(8).len(), 1); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/session.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/session.rs new file mode 100644 index 000000000..4c6af1baf --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-hid/src/session.rs @@ -0,0 +1,673 @@ +use crate::ds4::{Ds4ReportState, low_id_for_slot, slot_for_low_id}; +use crate::{InventoryView, SdlDeviceClaim, SonyCapability, SonySnapshot}; + +pub const DS4_OUTPUT_REPORT_ID: u8 = 5; +pub const DS4_OUTPUT_ENABLE_BIT: u8 = 0x01; +pub const DS4_OUTPUT_MIN_BYTES: usize = 6; +pub const DS4_OUTPUT_LOW_BYTE: usize = 4; +pub const DS4_OUTPUT_HIGH_BYTE: usize = 5; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SonyRumble { + pub slot: u8, + pub incarnation: u64, + pub low_frequency: u16, + pub high_frequency: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HidAttachment { + Attached, + Removed, + Refused, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HidOutbound { + Attach { + low_id: u8, + }, + Report { + low_id: u8, + bytes: Box<[u8; crate::ds4::DS4_REPORT_BYTES]>, + }, + Release { + low_id: u8, + bytes: Box<[u8; crate::ds4::DS4_REPORT_BYTES]>, + }, + Removal { + low_id: u8, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AdmittedSource { + slot: u8, + incarnation: u64, + vendor: u16, + product: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HidSession { + capability: SonyCapability, + sources: [Option; crate::MAX_SOURCES], + tombstones: [Option; crate::MAX_SOURCES], + report_states: [Ds4ReportState; crate::MAX_SOURCES], +} + +impl Default for HidSession { + fn default() -> Self { + Self::new(SonyCapability::default()) + } +} + +impl HidSession { + pub fn new(capability: SonyCapability) -> Self { + Self { + capability, + sources: [None; crate::MAX_SOURCES], + tombstones: [None; crate::MAX_SOURCES], + report_states: Default::default(), + } + } + + pub fn capability(&self) -> SonyCapability { + self.capability + } + + pub fn set_server_mask(&mut self, server_mask: u32) { + self.capability.server_mask = server_mask; + } + pub fn attached(&self, slot: u8) -> Option { + self.sources + .get(usize::from(slot)) + .copied() + .flatten() + .map(|source| source.incarnation) + } + + pub fn is_tombstoned(&self, slot: u8) -> bool { + self.tombstones + .get(usize::from(slot)) + .is_some_and(|entry| entry.is_some()) + } + + pub fn rich_slot_mask(&self) -> u16 { + let mut mask = 0_u16; + for (slot, source) in self.sources.iter().enumerate() { + if source.is_some() { + mask |= 1 << slot; + } + } + mask + } + + pub fn reconcile(&mut self, inventory: &InventoryView) -> Vec { + let mut outbound = Vec::with_capacity(crate::MAX_SOURCES * 2); + for slot in 0..crate::MAX_SOURCES as u8 { + let wanted = inventory.claim(slot); + match (self.sources[usize::from(slot)], wanted) { + (Some(current), Some(claim)) if current.incarnation == claim.incarnation => {} + (Some(_), _) => { + self.retire_slot(slot, &mut outbound); + if let Some(claim) = wanted { + self.admit(claim, &mut outbound); + } + } + (None, Some(claim)) => { + self.admit(claim, &mut outbound); + } + (None, None) => {} + } + } + outbound + } + + fn admit(&mut self, claim: SdlDeviceClaim, outbound: &mut Vec) -> HidAttachment { + if usize::from(claim.slot) >= crate::MAX_SOURCES || claim.incarnation == 0 { + return HidAttachment::Refused; + } + let index = usize::from(claim.slot); + let Some((target, cross_synthesis)) = claim.sony_product() else { + return HidAttachment::Refused; + }; + if !self.capability.admits(target, cross_synthesis) { + return HidAttachment::Refused; + } + let Some(low_id) = low_id_for_slot(claim.slot) else { + return HidAttachment::Refused; + }; + if self.tombstones[index].is_some() { + return HidAttachment::Refused; + } + self.sources[index] = Some(AdmittedSource { + slot: claim.slot, + incarnation: claim.incarnation, + vendor: claim.vendor, + product: claim.product, + }); + self.report_states[index] = Ds4ReportState::default(); + outbound.push(HidOutbound::Attach { low_id }); + HidAttachment::Attached + } + + pub fn retire(&mut self, slot: u8) -> Vec { + let mut outbound = Vec::with_capacity(2); + self.retire_slot(slot, &mut outbound); + outbound + } + + fn retire_slot(&mut self, slot: u8, outbound: &mut Vec) { + let index = usize::from(slot); + let Some(source) = self.sources.get(index).copied().flatten() else { + return; + }; + self.sources[index] = None; + self.tombstones[index] = Some(source.incarnation); + if let Some(low_id) = low_id_for_slot(slot) { + let bytes = self.release_report(slot); + self.report_states[index] = Ds4ReportState::default(); + outbound.push(HidOutbound::Release { + low_id, + bytes: Box::new(bytes), + }); + outbound.push(HidOutbound::Removal { low_id }); + } else { + self.report_states[index] = Ds4ReportState::default(); + } + } + + fn release_report(&mut self, slot: u8) -> [u8; crate::ds4::DS4_REPORT_BYTES] { + let state = &mut self.report_states[usize::from(slot)]; + let mut release = SonySnapshot::neutral(slot, 1, state.last_observed_at_us()); + let previous = *state.contacts(); + release.contacts = previous.map(|contact| crate::SonyContact { + active: false, + x: contact.x, + y: contact.y, + }); + state.build(&release) + } + + pub fn observe_slot(&mut self, slot: u8, observed_at_us: u64) { + if let Some(state) = self.report_states.get_mut(usize::from(slot)) { + state.observe(observed_at_us); + } + } + + pub fn retire_all(&mut self) -> Vec { + let mut outbound = Vec::with_capacity(crate::MAX_SOURCES * 2); + for slot in 0..crate::MAX_SOURCES as u8 { + self.retire_slot(slot, &mut outbound); + } + outbound + } + + pub fn build_report( + &mut self, + snapshot: &SonySnapshot, + ) -> Option<(u8, [u8; crate::ds4::DS4_REPORT_BYTES])> { + let index = usize::from(snapshot.slot); + let source = self.sources.get(index).copied().flatten()?; + if source.incarnation != snapshot.incarnation { + return None; + } + let low_id = low_id_for_slot(snapshot.slot)?; + Some((low_id, self.report_states[index].build(snapshot))) + } + + pub fn release_snapshot( + &mut self, + slot: u8, + ) -> Option<(u8, [u8; crate::ds4::DS4_REPORT_BYTES])> { + let index = usize::from(slot); + self.sources.get(index).copied().flatten()?; + let low_id = low_id_for_slot(slot)?; + Some((low_id, self.release_report(slot))) + } + + pub fn take_output(&self, payload: &[u8], low_id: u8) -> Option { + let slot = slot_for_low_id(low_id)?; + let source = self.sources.get(usize::from(slot)).copied().flatten()?; + if payload.len() < DS4_OUTPUT_MIN_BYTES { + return None; + } + if payload[0] != DS4_OUTPUT_REPORT_ID { + return None; + } + if payload[1] & DS4_OUTPUT_ENABLE_BIT == 0 { + return None; + } + Some(SonyRumble { + slot, + incarnation: source.incarnation, + low_frequency: u16::from(payload[DS4_OUTPUT_LOW_BYTE]) << 8, + high_frequency: u16::from(payload[DS4_OUTPUT_HIGH_BYTE]) << 8, + }) + } + + pub fn adopt(&mut self, claim: SdlDeviceClaim) -> Result { + if claim.sony_product().is_none() { + return Err(HidSessionError::UnsupportedIdentity); + } + let mut outbound = Vec::new(); + Ok(self.admit(claim, &mut outbound)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HidSessionError { + UnsupportedIdentity, +} + +impl std::fmt::Display for HidSessionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnsupportedIdentity => { + formatter.write_str("identity is not an admitted Sony source") + } + } + } +} + +impl std::error::Error for HidSessionError {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{InventoryOutcome, SONY_VENDOR, SonyContact, SonyTarget}; + + fn ds4(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, SONY_VENDOR, 0x05c4) + } + + fn ds5(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, SONY_VENDOR, 0x0ce6) + } + + fn xbox(slot: u8, incarnation: u64) -> Option { + SdlDeviceClaim::new(slot, incarnation, 0x045e, 0x02ea) + } + + fn inventory(claims: [Option; 4]) -> InventoryView { + let mut view = InventoryView::default(); + assert_eq!(view.replace(&claims), InventoryOutcome::Replaced); + view + } + + fn snapshot(slot: u8, incarnation: u64) -> SonySnapshot { + SonySnapshot::neutral(slot, incarnation, 0) + } + + #[test] + fn admission_attaches_only_supported_sony_sources() { + let mut session = session_with_server_mask(0x5); + let outbound = session.reconcile(&inventory([ + Some(ds4(0, 1).unwrap()), + xbox(1, 2), + None, + None, + ])); + assert_eq!(outbound, vec![HidOutbound::Attach { low_id: 6 }]); + assert_eq!(session.attached(0), Some(1)); + assert_eq!(session.attached(1), None); + assert_eq!(session.rich_slot_mask(), 0b0001); + } + + #[test] + fn dualsense_cross_synthesis_is_locally_enabled_and_needs_no_ds5_server_bit() { + let mut session = session_with_server_mask(0x5); + assert_eq!( + session.adopt(ds5(0, 1).unwrap()).unwrap(), + HidAttachment::Attached + ); + assert_eq!(session.attached(0), Some(1)); + assert!(!session.is_tombstoned(0)); + } + + #[test] + fn ds4_support_without_game_permission_falls_back_to_ordinary_behavior() { + let mut session = session_with_server_mask(0x4); + assert_eq!( + session.adopt(ds4(0, 1).unwrap()).unwrap(), + HidAttachment::Refused + ); + assert_eq!(session.attached(0), None); + let mut session = session_with_server_mask(0x1); + assert_eq!( + session.adopt(ds4(0, 1).unwrap()).unwrap(), + HidAttachment::Refused + ); + let mut session = session_with_server_mask(0x0); + assert_eq!( + session.adopt(ds5(0, 1).unwrap()).unwrap(), + HidAttachment::Refused + ); + } + + #[test] + fn a_disabled_local_cross_synthesis_policy_refuses_dualsense_but_keeps_ds4() { + let mut capability = SonyCapability { + server_mask: 0x5, + ..SonyCapability::default() + }; + capability.cross_synthesize_ds5_to_ds4 = false; + let mut session = HidSession::new(capability); + assert_eq!( + session.adopt(ds5(0, 1).unwrap()).unwrap(), + HidAttachment::Refused + ); + assert_eq!( + session.adopt(ds4(1, 2).unwrap()).unwrap(), + HidAttachment::Attached + ); + } + + #[test] + fn a_disabled_local_ds4_support_byte_refuses_every_sony_source() { + let capability = SonyCapability { + server_mask: 0x5, + synthesized_ds4_support: false, + ..SonyCapability::default() + }; + let mut session = HidSession::new(capability); + assert_eq!( + session.adopt(ds4(0, 1).unwrap()).unwrap(), + HidAttachment::Refused + ); + assert_eq!( + session.adopt(ds5(1, 2).unwrap()).unwrap(), + HidAttachment::Refused + ); + assert_eq!(session.rich_slot_mask(), 0); + } + + fn session_with_server_mask(server_mask: u32) -> HidSession { + let mut session = HidSession::new(SonyCapability { + server_mask, + ..SonyCapability::default() + }); + session.set_server_mask(server_mask); + session + } + + #[test] + fn unsupported_identity_is_rejected_before_any_state_change() { + let mut session = session_with_server_mask(0x5); + assert_eq!( + session.adopt(xbox(0, 1).unwrap()), + Err(HidSessionError::UnsupportedIdentity) + ); + assert!(session.retire_all().is_empty()); + } + + #[test] + fn publication_requires_the_admitted_incarnation() { + let mut session = session_with_server_mask(0x5); + let adopted = session.adopt(ds4(2, 77).unwrap()).unwrap(); + assert_eq!(adopted, HidAttachment::Attached); + assert_eq!(session.attached(2), Some(77)); + assert!(session.build_report(&snapshot(2, 78)).is_none()); + assert_eq!( + session.build_report(&snapshot(2, 77)).map(|(id, _)| id), + Some(8) + ); + assert!(session.build_report(&snapshot(1, 77)).is_none()); + } + + #[test] + fn retire_sends_release_before_removal_and_keeps_a_tombstone() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(1, 40).unwrap()); + let outbound = session.retire(1); + assert_eq!(outbound.len(), 2); + assert!(matches!( + outbound[0], + HidOutbound::Release { low_id: 7, .. } + )); + assert_eq!(outbound[1], HidOutbound::Removal { low_id: 7 }); + assert!(session.is_tombstoned(1)); + assert_eq!(session.attached(1), None); + } + + #[test] + fn retired_low_ids_are_not_rebound_to_another_incarnation_on_the_same_association() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(1, 40).unwrap()); + let _ = session.retire(1); + assert_eq!( + session.adopt(ds4(1, 41).unwrap()).unwrap(), + HidAttachment::Refused + ); + assert_eq!(session.attached(1), None); + let outbound = session.reconcile(&inventory([None, ds4(1, 41), None, None])); + assert!(outbound.is_empty()); + assert_eq!(session.attached(1), None); + } + + #[test] + fn a_new_association_clears_tombstones_and_reopens_the_slot() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(1, 40).unwrap()); + let _ = session.retire(1); + let mut fresh = session_with_server_mask(0x5); + assert_eq!( + fresh.adopt(ds4(1, 41).unwrap()).unwrap(), + HidAttachment::Attached + ); + assert_eq!(fresh.attached(1), Some(41)); + assert!(!session.is_tombstoned(0)); + } + + #[test] + fn reconcile_retires_the_old_source_before_attaching_the_replacement_elsewhere() { + let mut session = session_with_server_mask(0x5); + let first = inventory([Some(ds4(0, 1).unwrap()), None, None, None]); + assert_eq!( + session.reconcile(&first), + vec![HidOutbound::Attach { low_id: 6 }] + ); + let second = inventory([None, Some(ds4(1, 2).unwrap()), None, None]); + let outbound = session.reconcile(&second); + assert!(matches!( + outbound[0], + HidOutbound::Release { low_id: 6, .. } + )); + assert_eq!(outbound[1], HidOutbound::Removal { low_id: 6 }); + assert_eq!(outbound[2], HidOutbound::Attach { low_id: 7 }); + assert_eq!(session.attached(0), None); + assert_eq!(session.attached(1), Some(2)); + } + + #[test] + fn repeated_reconcile_with_the_same_inventory_is_idempotent() { + let mut session = session_with_server_mask(0x5); + let view = inventory([Some(ds4(0, 1).unwrap()), None, None, None]); + let _ = session.reconcile(&view); + assert!(session.reconcile(&view).is_empty()); + assert!(session.reconcile(&view).is_empty()); + } + + #[test] + fn report_building_publishes_the_full_64_byte_report_for_the_admitted_slot() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(3, 9).unwrap()); + let mut state = snapshot(3, 9); + state.contacts[0] = SonyContact { + active: true, + x: 0, + y: 0, + }; + let (low_id, report) = session.build_report(&state).unwrap(); + assert_eq!(low_id, 9); + assert_eq!(report.len(), 64); + assert_eq!(report[35], 0x01); + } + + #[test] + fn low_id_output_only_accepts_the_proved_report_id_and_enable_bit() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 5).unwrap()); + assert_eq!( + session.take_output(&[5, 0x01, 0, 0, 0x40, 0x80], 6), + Some(SonyRumble { + slot: 0, + incarnation: 5, + low_frequency: 0x4000, + high_frequency: 0x8000, + }) + ); + assert_eq!(session.take_output(&[5, 0x00, 0, 0, 0x40, 0x80], 6), None); + assert_eq!(session.take_output(&[4, 0x01, 0, 0, 0x40, 0x80], 6), None); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x40], 6), None); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x40, 0x80], 5), None); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x40, 0x80], 10), None); + } + + #[test] + fn low_id_output_for_a_retired_slot_is_dropped() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 5).unwrap()); + let _ = session.retire(0); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x40, 0x80], 6), None); + } + + #[test] + fn low_id_output_cannot_drive_a_replacement_controller_in_the_same_slot() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 5).unwrap()); + let stale = session + .take_output(&[5, 0x01, 0, 0, 0x11, 0x22], 6) + .unwrap(); + assert_eq!(stale.incarnation, 5); + let _ = session.retire(0); + let _ = session.adopt(ds4(0, 6).unwrap()); + assert_ne!(session.attached(0), Some(stale.incarnation)); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x11, 0x22], 6), None); + } + + #[test] + fn retire_all_releases_every_admitted_source_in_slot_order() { + let mut session = session_with_server_mask(0x5); + let _ = session.reconcile(&inventory([ + Some(ds4(0, 1).unwrap()), + Some(ds4(1, 2).unwrap()), + None, + Some(ds4(3, 4).unwrap()), + ])); + let outbound = session.retire_all(); + assert_eq!(outbound.len(), 6); + assert!(matches!( + outbound[0], + HidOutbound::Release { low_id: 6, .. } + )); + assert_eq!(outbound[1], HidOutbound::Removal { low_id: 6 }); + assert!(matches!( + outbound[2], + HidOutbound::Release { low_id: 7, .. } + )); + assert_eq!(outbound[3], HidOutbound::Removal { low_id: 7 }); + assert!(matches!( + outbound[4], + HidOutbound::Release { low_id: 9, .. } + )); + assert_eq!(outbound[5], HidOutbound::Removal { low_id: 9 }); + assert!(session.retire_all().is_empty()); + } + + #[test] + fn release_report_retains_last_contact_positions_but_clears_activity() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 1).unwrap()); + let mut state = snapshot(0, 1); + state.contacts[0] = SonyContact { + active: true, + x: 32767, + y: 32767, + }; + let _ = session.build_report(&state); + let (_, release) = session.release_snapshot(0).unwrap(); + assert_eq!(release[35], 0x81); + assert_eq!(&release[35..39], &[0x81, 0x7f, 0xe7, 0x3a]); + assert_eq!(&release[1..5], &[128, 128, 128, 128]); + } + + #[test] + fn parent_review_release_retains_last_observation_counter() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 1).unwrap()); + let mut state = snapshot(0, 1); + state.observed_at_us = 1_000_000; + state.buttons = 0x1000; + state.contacts[0].active = true; + let (_, prior) = session.build_report(&state).unwrap(); + let (_, release) = session.release_snapshot(0).unwrap(); + assert_ne!(&prior[10..12], &[0, 0]); + assert_eq!( + &release[10..12], + &prior[10..12], + "capture release must carry the chosen last-observation timestamp, not zero" + ); + assert_eq!(release[34], prior[34]); + } + + #[test] + fn release_snapshot_is_unavailable_for_an_unpublished_slot() { + let mut session = session_with_server_mask(0x5); + assert!(session.release_snapshot(2).is_none()); + } + + #[test] + fn retirement_release_keeps_the_last_contact_positions() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(1, 3).unwrap()); + let mut state = snapshot(1, 3); + state.contacts[0] = SonyContact { + active: true, + x: 16384, + y: -16384, + }; + let _ = session.build_report(&state); + let outbound = session.retire(1); + let HidOutbound::Release { low_id, bytes } = &outbound[0] else { + panic!("retirement must release before removal"); + }; + assert_eq!(*low_id, 7); + assert_eq!(&bytes[35..39], &[0x81, 0x9f, 0xb5, 0x0e]); + assert_eq!(&bytes[1..5], &[128, 128, 128, 128]); + assert_eq!(outbound[1], HidOutbound::Removal { low_id: 7 }); + } + + #[test] + fn report_building_rejects_a_slot_the_source_map_cannot_index() { + let mut session = session_with_server_mask(0x5); + let _ = session.adopt(ds4(0, 1).unwrap()); + let mut out_of_range = snapshot(0, 1); + out_of_range.slot = 9; + assert!(session.build_report(&out_of_range).is_none()); + assert!(session.release_snapshot(9).is_none()); + assert_eq!(session.take_output(&[5, 0x01, 0, 0, 0x40, 0x80], 15), None); + } + + #[test] + fn adoption_refuses_a_claim_the_slot_bounds_cannot_hold() { + let mut session = session_with_server_mask(0x5); + let claim = SdlDeviceClaim { + slot: 9, + incarnation: 1, + vendor: SONY_VENDOR, + product: 0x05c4, + }; + assert_eq!(session.adopt(claim).unwrap(), HidAttachment::Refused); + assert_eq!(session.rich_slot_mask(), 0); + } + + #[test] + fn target_selection_stays_ds4_for_every_admitted_sony_source() { + let claim = ds5(0, 1).unwrap(); + assert_eq!(claim.sony_product(), Some((SonyTarget::Ds4, true))); + let ds4_claim = ds4(0, 1).unwrap(); + assert_eq!(ds4_claim.sony_product(), Some((SonyTarget::Ds4, false))); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml new file mode 100644 index 000000000..842c7a988 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "opennow-streamer-platform-windows" +version.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +rust-version.workspace = true + +[dependencies] +opennow-streamer-protocol = { path = "../opennow-streamer-protocol" } + + +[target.'cfg(windows)'.dependencies] +windows = { version = "0.62", features = [ + "Win32_Foundation", + "Win32_Graphics_Direct3D10", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D_Fxc", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Direct3D11on12", + "Win32_Graphics_Direct3D12", + "Win32_Graphics_Dxgi", + "Win32_Graphics_Dxgi_Common", + "Win32_Graphics_Gdi", + "Win32_Media_Audio", + "Win32_Media_MediaFoundation", + "Win32_Media_Multimedia", + "Win32_System_Com", + "Win32_System_Com_StructuredStorage", + "Win32_System_LibraryLoader", + "Win32_System_Threading", + "Win32_System_Variant", + "Win32_UI_WindowsAndMessaging", +] } diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/README.md b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/README.md new file mode 100644 index 000000000..493bd8c2f --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/README.md @@ -0,0 +1,226 @@ +# OpenNOW Windows media backend + +This isolated crate provides the Windows media end of the native streamer. It accepts complete +H.264, HEVC/H.265, or AV1 access units and interleaved `f32` PCM that has already been decoded from +Opus. + +The backend uses only Windows system APIs: + +- Media Foundation hardware MFTs for H.264/HEVC/AV1 decode into NV12/P010, or AYUV/Y410 when the + installed decoder exposes those exact formats. Hardware selection checks the selected adapter's + DXVA profile, output format, and decoder configuration before trying adapter-matched transforms + and D3D11-aware registered MFTs. A separate software mode accepts only + synchronous/asynchronous software MFTs that are D3D11-aware and publish D3D11-backed output; + the two modes are never silently mixed. +- A D3D11 video processor and a two-buffer flip-model DXGI swap chain for NV12 conversion, aspect-correct scaling, and presentation. The swap chain can target a caller-owned HWND, or the backend can create and own a child or top-level HWND. +- WASAPI shared-mode rendering for interleaved IEEE-float PCM. Windows performs endpoint format conversion when the active mix format differs. + +`WindowsBackend::probe_for` accepts an explicit `Option` and creates a hidden +presentation path using either D3D11 or a D3D12-backed D3D11-on-12 device. `None` preserves the +system default; `Some` resolves that exact DXGI adapter and fails without a default-adapter retry +when it is unavailable. The probe independently checks adapter-matched hardware transforms and registered +software transforms for H.264, HEVC and AV1, and starts a 48 kHz stereo WASAPI client. A codec +capability is reported only when its selected decoder class and the complete presentation/audio +operation succeed. `WindowsBackend::probe` remains the D3D11 compatibility entry point. +Non-Windows builds expose the same typed API but report the backend as unavailable. + +`CapabilityProbe.h265_hdr`, `av1_hdr`, and `supports_hdr(codec)` report HDR separately from +SDR codec support. `h265_10bit`, `av1_10bit`, `h265_444`, `h265_10bit_444`, and +`supports_format(codec, pixel_format, hdr)` distinguish each supported depth/chroma combination. +Advanced-format probes decode a synthetic 1920x1080 access unit through the selected MFT, +require a D3D11-backed output frame preserving the requested depth, chroma, and transfer, +then execute the embedded converter: `VideoProcessorBlt` for P010/HDR and a UINT shader for +SDR Y410. Both paths produce the actual RGB10A2 texture used by Qt. +An exposed media subtype or provisional startup format alone is not a successful probe. +The fixtures and their generation/verification commands are in `fixtures/probe/`. +These checks use the exact optional adapter passed to `probe_for`; +the Qt-adopted device is checked independently when its decoder and converter are created. +The capability probe does not establish that a monitor or the Qt swapchain supports HDR. +Decoded-output polling is bounded to 500 ms per advanced profile, at most three seconds per +graphics API, excluding Windows API call latency. Results are reused for the same adapter LUID +and D3D API; device removal or an adapter change invalidates them. Timeouts are not cached. +Presentation and audio are always checked again. Live playback independently validates the adopted +Qt device and actual output. It permits AYUV-to-NV12 and Y410-to-P010 chroma-only fallback at unchanged +bit depth; capability probes still require the exact requested output format. The parent shell +notifies the user using the validated actual format exposed by the acquired `D3d11Frame`. +Installing a different decoder requires restarting the process to refresh cached MFT failures. + +## Embedded HDR color contract + +`VideoFormat` carries explicit transfer function, primaries, matrix, range, and chroma siting. +Callers supply the negotiated defaults, including `VideoChromaSiting::Left` for GFN. Actual +Media Foundation output metadata overrides these defaults; unsupported metadata and HDR-to-SDR +decoder downgrades fail rather than being mislabeled. HDR requires ten-bit BT.2020 output; +ten-bit SDR remains SDR. Color-only changes invalidate converter resources and texture slots. + +The embedded renderer publishes `D3d11RecordedFrame` with `texture_format: Rgb10A2` and +`color_space: Pq2020` for HDR. PQ uses the matching LEFT or TOPLEFT input colorspace. HLG can +convert to PQ only for TOPLEFT siting and explicit driver support; HLG with LEFT siting and +full-range PQ fail because no exact DXGI input colorspace is available. SDR uses BT.709 RGB, +with RGBA8 for eight-bit input and RGB10A2 for ten-bit input. +For packed 4:4:4, chroma exists at every luma pixel, so LEFT and TOPLEFT do not describe different +sample positions. The converter selects an available DXGI siting label without resampling chroma; +4:2:0 still requires its exact negotiated siting. +The standalone HWND presenter explicitly rejects HDR; HDR presentation belongs to the embedded +Qt path, which must preserve the recorded texture's color interpretation through scan-out. +When the host configured an explicit adapter LUID, the FFI verifies that the borrowed Qt D3D11 +device reports the same DXGI adapter before installing the graphics context. A mismatch fails +explicitly; it cannot authorize decode using capabilities from a different default adapter. +## Embedded Qt SDR conversion + +The embedded frame producer converts NV12/AYUV to RGBA8 and P010/Y410 to RGB10A2, +preserving ten-bit precision until Qt's final SDR composition pass. RGB10A2 is an SDR +intermediate, not an HDR or ten-bit scan-out guarantee. Both full- and limited-range +BT.709 decoder output are converted to full-range BT.709 RGB. + +Embedded conversion requires `ID3D11VideoContext1` for explicit color-space selection and +`ID3D11VideoProcessorEnumerator1` to validate the input/output format and color-space pair. +Unsupported conversions fail explicitly; the producer does not silently replace RGB10A2 +with RGBA8 or rely on legacy driver color defaults. Format, range, and extent changes +invalidate the processor, cached input views, and frame slots before conversion resumes. + +Media Foundation may temporarily negotiate a lower-precision output type before reading +the first sequence header. That startup compatibility does not permit lower-bit-depth frames: +each decoded surface must match its output media type and preserve at least the negotiated +bit depth before entering the decoded queue. Chroma loss is limited to the two same-depth +fallbacks above, and conversion uses the validated actual format and color metadata. +Provisional startup metadata is not validated as an actual HDR frame before the sequence header +arrives. The first actual sample and every output-type change undergo strict validation, including +HDR color metadata; lower-precision frames never reach conversion. + +## Exact SDR Y410 conversion + +On the validated RTX3080 driver, the video processor preserved all decoded Y410 values but +applied eight-bit-normalized nominal-range/chroma offsets during RGB conversion. A neutral +10-bit white became R1017/G1021/B1016 instead of 1023, and disabling automatic processing +made no difference. The decoded surface itself was sample-exact. + +SDR BT.709 Y410 therefore uses a deterministic GPU shader: one GPU copy crops the decoded +array slice into a sampleable Y410 texture, a UINT view exposes the exact ten-bit U/Y/V codes, +and explicit full/limited-range matrix math writes RGB10A2. The input texture and visited Qt +output slots are bounded and reused. A deferred command list restores Qt's immediate-context +state; format/range/extent changes recreate converter resources. Unsupported views or formats +fail explicitly, with no CPU download or lower-precision fallback. + +P010/PQ HDR retains the video processor. A valid Main10 PQ ramp was checked against a software +decoder reference and preserved 875 RGB10A2/PQ gray levels within two code values across three +decoder recreations on the same RTX3080. This verifies the encoded color pipeline, not monitor +scan-out or optical HDR accuracy. Shader compilation uses Windows' system D3DCompiler 47. + +## HEVC availability in Qt + +The embedded Qt stream view uses the same Media Foundation decoder configuration as the +Windows capability probe. HEVC input types explicitly carry the profile matching the negotiated +bit depth and chroma before the decoder selects its output type. Hardware profile checks and +D3D11 surface requirements still apply; setting a profile does not enable an unsupported GPU. + +Windows must have a registered, activatable HEVC Media Foundation decoder, such as Microsoft's +HEVC Video Extensions, in addition to a compatible GPU and driver. The app does not bundle a +separate Windows HEVC decoder. An installed extension alone does not prove that decoder +activation or D3D11 configuration succeeds. + +Microsoft's older [HEVC decoder documentation](https://learn.microsoft.com/en-us/windows/win32/medfound/h-265---hevc-video-decoder) +lists Main/Main10 4:2:0 and NV12/P010, but that is not an exhaustive description of current +HEVCVideoExtension MFTs. On an RTX3080 with driver 32.0.16.1047, the D3D11 path decoded actual +AYUV and Main44410/Y410 access units and completed embedded conversion; the same device's +D3D12-backed path rejected 4:4:4. Main44410 initially negotiated provisional NV12 before +producing Y410 after the sequence header. Runtime probing of the selected API, GPU, installed +MFT, actual decoded texture, and converter is authoritative; neither old documentation nor +a GPU model name alone establishes support. Unsupported combinations report their failure stage. +AV1 4:4:4 and HDR 4:4:4 are not advertised. No alternative FFmpeg or NVDEC decoder is bundled. + +When Qt disables H.265, `%APPDATA%\OpenNOW\diagnostics\native-streamer.log` records +`Windows hardware decoder probe failed` with `codec=H.265` and the decoder error, even if H.264 +remains available. `OPENNOW_DATA_DIR` overrides the data directory. Configuration failures include +the MFT name and activation, D3D manager, or input-type stage where applicable. + +## Integration API + +```rust,no_run +use std::num::NonZeroU32; +use opennow_streamer_platform_windows::{ + AudioFormat, BackendConfig, Bounds, OwnedWindow, SurfaceTarget, VideoCodec, VideoFormat, + VideoChromaFormat, VideoChromaSiting, VideoPixelFormat, VideoTransferFunction, + VideoColorPrimaries, VideoColorMatrix, WindowsBackend, +}; + +let backend = WindowsBackend::start(BackendConfig { + video: VideoFormat { + codec: VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: NonZeroU32::new(120).unwrap(), + frame_rate_denominator: NonZeroU32::new(1).unwrap(), + average_bitrate: 75_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + chroma_siting: VideoChromaSiting::Left, + full_range: false, + transfer_function: VideoTransferFunction::Sdr, + color_primaries: VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + }, + audio: AudioFormat { + sample_rate: 48_000, + channels: 2, + }, + surface: SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 1920, + height: 1080, + }, + visible: true, + }), + video_queue_capacity: 3, + audio_queue_capacity: 8, +})?; +# Ok::<(), Box>(()) +``` + +`submit_video` and `submit_audio` never wait for the media thread. When a configured queue is full, the oldest packet is discarded and the method returns `PushOutcome::DroppedOldest`. The WASAPI staging buffer is independently capped at two endpoint buffers. Events use a bounded drop-oldest queue as well, so a stalled event consumer cannot create unbounded memory growth. + +### Existing and parented surfaces + +`SurfaceTarget::Existing` is only for a dedicated HWND that the caller reserves for D3D presentation. Do not pass the shell's interactive top-level HWND as an existing surface because the swap chain and window input policy would then share that interactive window. + +An embedded shell integration must use `SurfaceTarget::Owned` with the shell HWND in `OwnedWindow::parent`. The backend creates a child with `WS_EX_NOACTIVATE | WS_EX_TRANSPARENT`, returns `HTTRANSPARENT` from `WM_NCHITTEST`, and rejects mouse activation. The child therefore cannot take focus or consume pointer input from the shell. `OwnedWindow::bounds` are parent-client coordinates relative to the shell. Calling `set_surface` with the same parent updates those bounds and `visible` state in place with `SWP_NOACTIVATE`; changing the parent recreates the owned child and its swap chain. + +Video and audio reconfiguration flush their respective queues. D3D11 or WASAPI failures move the backend to `Recovering`, clear stale packets, and retry for five seconds. Successful D3D recovery emits `KeyFrameRequired`; exhausted recovery emits a fatal device-loss event. `stop` closes both queues and joins the media thread, including when recovery is in progress. + +## Checks + +The crate is a member of the native streamer workspace and can also be checked directly: + +```sh +cargo test --manifest-path native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml +cargo check --manifest-path native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml --all-targets --target x86_64-pc-windows-msvc +cargo check --manifest-path native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml --all-targets --target aarch64-pc-windows-msvc +cargo run --manifest-path native/opennow-streamer/crates/opennow-streamer-platform-windows/Cargo.toml --example probe +``` + +On a Windows GPU machine, the example prints D3D11 and D3D12-backed capability results and the +specific failures for unsupported profiles. The Windows-only unit tests exercise Media Foundation +metadata and D3D11 processing; cross-target `cargo check` compiles but does not execute them. + +The explicit hardware precision regression decodes a lossless Main44410/RExt fixture, checks +the actual Y410 decoder surface, converts it through the embedded GPU converter, and reads +back the test-only RGB10A2 target. It verifies at least 800 distinct gray levels, every sample +of an 877-level gray ramp, and 1920 alternating one-pixel chroma samples across three decoder +restarts. It has no unsupported-hardware skip once explicitly selected: + +```sh +cargo test --manifest-path native/opennow-streamer/Cargo.toml -p opennow-streamer-platform-windows hevc_444_hardware_decode_and_conversion_preserve_precision_and_chroma -- --ignored --nocapture --test-threads=1 +``` + +The cross-target checks validate the Win32 bindings for x64 and ARM64. Hardware decode, presentation, audio output, device removal, and endpoint switching still require tests on real Windows hardware. + +The corresponding HDR precision regression uses a valid Main10/PQ fixture and a committed +software-decoded luma reference. It keeps the same two-code conversion tolerance rather than +folding codec errors into a wider tolerance: + +```sh +cargo test --manifest-path native/opennow-streamer/Cargo.toml -p opennow-streamer-platform-windows hevc_hdr_hardware_decode_and_conversion_preserve_pq_precision -- --ignored --nocapture --test-threads=1 +``` diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/examples/probe.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/examples/probe.rs new file mode 100644 index 000000000..ebefec750 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/examples/probe.rs @@ -0,0 +1,7 @@ +use opennow_streamer_platform_windows::{WindowsBackend, WindowsGraphicsApi}; + +fn main() { + for api in [WindowsGraphicsApi::D3d11, WindowsGraphicsApi::D3d12] { + println!("{api:?}: {:#?}", WindowsBackend::probe_for(api, None)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/README.md b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/README.md new file mode 100644 index 000000000..09b03fb26 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/README.md @@ -0,0 +1,111 @@ +# Windows advanced-format decode probes + +These synthetic black access units exercise the actual Media Foundation decoder before +advertising 10-bit, 4:4:4, or HDR support. Each contains one independent 1920x1080 frame +at 60 Hz with no reordering. HEVC uses Annex B; AV1 uses low-overhead OBU framing. +All samples use limited range. SDR is BT.709; HDR10 is BT.2020 nonconstant-luminance/PQ. +They contain no captured user or game content. +HEVC uses `keyint=60` even though only one IDR is emitted: `keyint=1` would select +the intra-only RExt profile instead of Main10 for the P010 samples. + +Generated with FFmpeg 6.1.1, libx265 and libaom-av1. FFmpeg is a development-only +fixture generator, not a Windows runtime dependency. From this directory: + +```sh +for spec in 'p010 yuv420p10le' 'ayuv yuv444p' 'y410 yuv444p10le'; do + set -- $spec + ffmpeg -hide_banner -loglevel error \ + -f lavfi -i 'color=black:size=1920x1080:rate=60' -frames:v 1 \ + -pix_fmt "$2" -c:v libx265 -preset ultrafast \ + -x265-params 'log-level=error:pools=1:frame-threads=1:info=0:keyint=60:bframes=0:repeat-headers=1:colorprim=bt709:transfer=bt709:colormatrix=bt709:chromaloc=0' \ + -f hevc -y "hevc-$1-sdr.hevc" +done +ffmpeg -hide_banner -loglevel error \ + -f lavfi -i 'color=black:size=1920x1080:rate=60' -frames:v 1 \ + -pix_fmt yuv420p10le -c:v libx265 -preset ultrafast \ + -x265-params 'log-level=error:pools=1:frame-threads=1:info=0:keyint=60:bframes=0:repeat-headers=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:chromaloc=0' \ + -f hevc -y hevc-p010-pq.hevc +ffmpeg -hide_banner -loglevel error \ + -f lavfi -i 'color=black:size=1920x1080:rate=60' -frames:v 1 \ + -pix_fmt yuv444p10le -c:v libx265 -preset ultrafast \ + -x265-params 'log-level=error:pools=1:frame-threads=1:info=0:keyint=60:bframes=0:repeat-headers=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:chromaloc=0' \ + -f hevc -y hevc-y410-pq.hevc +for transfer in sdr pq; do + if [ "$transfer" = pq ]; then + colors='-color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc' + else + colors='-color_primaries bt709 -color_trc bt709 -colorspace bt709' + fi + ffmpeg -hide_banner -loglevel error \ + -f lavfi -i 'color=black:size=1920x1080:rate=60' -frames:v 1 \ + -pix_fmt yuv420p10le -c:v libaom-av1 -cpu-used 8 -threads 2 \ + -lag-in-frames 0 -g 1 $colors -color_range tv \ + -f obu -y "av1-p010-$transfer.obu" +done +``` + +Verify the encoded dimensions, pixel format, and color metadata with: + +```sh +for file in *.hevc *.obu; do + ffprobe -v error -show_entries \ + stream=codec_name,profile,width,height,pix_fmt,color_space,color_transfer,color_primaries \ + -of compact "$file" + ffmpeg -v error -i "$file" -frames:v 1 -f null - +done +``` + +## Non-black hardware precision regression + +`hevc-y410-precision.hevc` is a separate lossless Main44410/RExt access unit for the explicit +hardware regression, not a startup capability probe. The first 32 rows contain gray code values +64 through 940, repeating every 877 pixels, with neutral chroma. Remaining rows have Y=512, +alternating U=384/640 and V=640/384 at every pixel. All 6,220,800 decoded Y/U/V samples were +verified against this source using FFmpeg software decoding. The hardware test allows two RGB +code values of error for the ramp and four for the chroma matrix conversion; it does not allow +an 8-bit or chroma-subsampled result. + +```sh +ffmpeg -hide_banner -loglevel error -f lavfi \ + -i "nullsrc=size=1920x1080:rate=60,format=yuv444p10le,geq=lum='if(lt(Y,32),64+mod(X,877),512)':cb='if(lt(Y,32),512,if(mod(X,2),640,384))':cr='if(lt(Y,32),512,if(mod(X,2),384,640))'" \ + -frames:v 1 -c:v libx265 -preset ultrafast \ + -x265-params 'log-level=error:pools=1:frame-threads=1:info=0:keyint=60:bframes=0:repeat-headers=1:lossless=1:colorprim=bt709:transfer=bt709:colormatrix=bt709:chromaloc=0' \ + -f hevc -y hevc-y410-precision.hevc +``` + +## Main44410 PQ probes and hardware precision regression + +`hevc-y410-pq-precision.hevc` uses the same lossless 4:4:4 precision source and encoder +options as `hevc-y410-precision.hevc`, replacing `colorprim=bt709:transfer=bt709:colormatrix=bt709` +with `colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc`. The HDR test keeps the same +decoder precision, chroma, restart, and output precision assertions, uses BT.2020 conversion +coefficients, and requires PQ/BT.2020 output metadata. Neither this test nor the startup probe +permits software decoding. GPU readback is used only by the opt-in precision test. + +The macOS `profile_probe.rs` HDR444 parameter-set constants and length-prefixed IDR use the +HDR444 black probe command above with `size=64x64:rate=1`. Each Annex-B NAL is extracted without +its start code; the IDR is prefixed by its four-byte big-endian length. The decoded frame must +have two full-sized IOSurface planes and pass the existing PQ/BT.2020 Metal import checks. + +## Main10 PQ hardware precision regression + +`hevc-p010-pq-precision.hevc` was generated with FFmpeg 9.0.1 and libx265. It is Main10 +4:2:0, not lossless RExt: QP0 is used with psychovisual processing, SAO and deblocking disabled. +The source repeats Y codes64–940 every877 pixels, with neutral Cb/Cr=512. Quantization produces +875 distinct decoded gray levels and at most two source-code values of codec error. The binary +`hevc-p010-pq-precision-luma.bin` contains the first877 software-decoded luma values as little-endian +u16, so the hardware test compares against the actual coded signal and keeps the independent +RGB conversion tolerance at two codes. + +```sh +ffmpeg -hide_banner -loglevel error -f lavfi \ + -i "nullsrc=size=1920x1080:rate=60,format=yuv420p10le,geq=lum='64+mod(X,877)':cb=512:cr=512" \ + -frames:v 1 -c:v libx265 -preset ultrafast \ + -x265-params 'log-level=error:pools=1:frame-threads=1:info=0:keyint=60:bframes=0:repeat-headers=1:qp=0:psy-rd=0:psy-rdoq=0:sao=0:deblock=0:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:chromaloc=0' \ + -f hevc -y hevc-p010-pq-precision.hevc +ffprobe -v error -show_entries stream=profile,pix_fmt,color_space,color_transfer,color_primaries \ + -of compact hevc-p010-pq-precision.hevc +ffmpeg -v error -i hevc-p010-pq-precision.hevc -frames:v 1 -pix_fmt yuv420p10le \ + -f rawvideo -y pq-decoded.yuv +python -c "from pathlib import Path; Path('hevc-p010-pq-precision-luma.bin').write_bytes(Path('pq-decoded.yuv').read_bytes()[:877*2])" +``` diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-pq.obu b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-pq.obu new file mode 100644 index 0000000000000000000000000000000000000000..1e0bc68b4f768b31cfd154647ec11ca06cb23bd7 GIT binary patch literal 77 zcmV-T0J8rQ016HO002s>zr%AYxY2ATu20V3@<2=wm5uL7;o!C1-(#{?_ literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-sdr.obu b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/av1-p010-sdr.obu new file mode 100644 index 0000000000000000000000000000000000000000..1a21c61ea182e207b4bd7f85aae871164658a15c GIT binary patch literal 77 zcmV-T0J8rQ016HO002s>zr%AY6{K{7cI0FVF(AOH#g0s+N%P-96!xY2ATu20V3@<2=wm5uL7;o!C1-;2s`w literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-ayuv-sdr.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-ayuv-sdr.hevc new file mode 100644 index 0000000000000000000000000000000000000000..ecb300a99566e26e8f46a1d7c96832947b787ceb GIT binary patch literal 666 zcmZQzU|@7$e|+|IfDLw z?`F1bb#V1t0MZRZOi5gSmKXO-`~VjLF=`!TzUWS@KoR56ko&EKs*G7qF$qN>GW(G!ObP=8#=ptcYKf+uIlw}e^Rl@g@@ zJ_!`ABrs?ZA;zGA5`YZ03MjHMsPnOaJD7@duJSHvx$)7`s literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision-luma.bin b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision-luma.bin new file mode 100644 index 0000000000000000000000000000000000000000..7108490f17deea1b17448e236b2830d1dde33cae GIT binary patch literal 1754 zcmWN<1C$U500qF0Y}>Z&&9-gZuB|rPwr$(CZQJhd%@jfa3Zn>$q8N&!1WKY5N}~+Q zq8!Sj0xF^sDx(Ujq8h5B25O=fYNHP7q8{p_0UDwa8lwrCq8XZ_1zMsNTB8lxq8-|! z13ID;I-?7^q8qxS2YR9xdZQ2eq96KW00trugD@CF5QL!^hT#~2kr;*17=y7Ghw+$z ziI{}Rn1ZR8hUu7rnV5yyn1i{Phxu55g;<2eSc0WkhUHj+l~{$;hy6H!gE)l4ID(@%hT}MalQ@ObID@k|hx53Ai@1c#xPq&= zhU>V2o4AGBxP!a6hx>Sdhj@g?c!H;RhUa*Jmw1KOc!Rfihxhn^kNAYo_=2zahVS@+ zpZJB}_=CUr2apuaU<}R>49QRo%`gnha174~jL1lg%qWb?XpGJnjLBGx%{Yw9c#O{k zOvpq`%p^?8WK7N!OvzMC%`{BQbWG0-%*ag4%q+~xY|PFa%*kBL%{R?oIFqwDn{zmq^EjUixR8sum`k{n%eb5? zxRR^5nrpb0>$sj9xRINH=XjnMc#)TQnOAs~*La;bc$2qyn|FAZ_jsQV_>hnIm{0hW&-k1#_>!;qns4}) z@A#e{_>rIZnP2#o-}s$B_>;f*n}7J1{|Hf%6->buLLn7Op%q496;9z5K@k;6krhQz z6;06sg*`)l}_oEK^c`vnUzIZl}*`| zLpha8xs^wGl~4IqKm}Du0V=E_Dym{Ct`aJ#QYx)7DywoTuL`QDN~)|Xs;X+Lt{SST zTB@x&s;hdcuLf$UMry1kYN}>xt`=&kR%)#_YO8i?uMX;{PU@^K>Z)$)t{&>CUh1tr z>Z^Y0uK^mUKn>Dh4N;JWYM6#=ghpzVMr(}5YMjPvf+lK`CTohOYMQ2NhGuG(W^0b- zYM$n6ffj0!7Hf%?YMGX6g;r{nR%?ycYMs_=gEnfDHfxKvYMZuehjwb0c59FJYM=J& zfDY=A4(o`H>X?q}gih*|PV0=$>YUE&f-dTkF6)Y}>YA?WhHmPXZtITj>YncFfgb9S z9_xvo>Y1MFgYd)}gFfn$KI@CV>YKjnhkojpe(R6^>Yrd*wu3piLpY>E zIkdw#tiw6HBRHZXIkKZTs-ro&V>qT`Ikw|CuH!kr6F8w0IkA&CsgpUmQ#hqlIknR` zt(;{XM_~9q2(G>>&>FP!IEPkMKy3@@S9oSda5~Pw+%f q@?=l(R8RAC&+tso@@&uXT+j0YFY?0wE%p*G^)fI2-%79XD*p%2Vzt)* literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq-precision.hevc new file mode 100644 index 0000000000000000000000000000000000000000..09cf8dc116413fc14a3012b8f7ac76755a82a5bd GIT binary patch literal 2173 zcmZQzU|@7$3mGKvYWy z%ZCP#1|T8F0@C5ac%ZOZ2*}W2TqAW*V0DTcx6^?Q+FXYwEYRRQlrTe;(G#ebbEE3={}dL>sE8a* z+LIWRp7(F--4E@O4Gj%kS?}df*8I7+@agU7r%M%*)Hvjslid%ez|J` zvN!7$pF}o))OvUSP4a>GKlS%+y$#uV_R^b`KFOC=3%o)VD=%)@^P@}Pxo*bxg0<|n z$64PBCGjsU?yvi|W@*FZQ$1@ZKA$%6W99W!DW^rp_iFR(mz6o>ej@nfZPgX?+)d1K z-d{;`dTpg4$-%n%@bSMJHXJp0#gcvCjfI+*r`Cp;_3Nz@Bj?|B^5MPS>rqm(?1b6} zll8&-x|Du={@Rv0Z^5>iYnt}|sQ)|t@8+L#dB0`eT<_k%uKE1H{@BnPTX(Rx)ii(M zT)sc)zr^>lDPpY~k1*`+S;V`<=cM?dp2m=y1#`Uii5V|a-@T9Lar=?#N9p-Md_kL_QMWUp0$sSwjj;Xy?-(4*pRXpHKbG>pPIq&?94GTl^ zR_^>0sy|PZ;ag;cRBM6VcB{#)*2e#~&G~ahj6v+mjP;*Gz6O2Y^04CA^OeibJpCE_ zD>1QBmEV9hX2XM zw0HHYnwrj|97>{!XZj6Z-#RGhu`NFC-28qK<%!F+ZvoRpu5OpA&f)a5pRZ%SIcB=Z z+47oq-^pRsV41U^ z_U8so_kPS5xgKZU`+eJzufJCx_U&c3zWn9y->08{JAL|B_hLh~O^YM!#IE(PlXdr%TDOm^KrdF!pf?=M_s(*?dyDLC4DDH`N~ zzLd!w8*_e32B+Jw{mD%+)T=k$+VI(N!@`{({-_&jLo*yW+c6zN=7HGWw|54g2PMwI&kJhlSk*8KiIv1NTtjS%Nnd* z-a8;HLpKaud_V<%HF(@0P9-x#!hs-S+zK>a{z2FRqD*BvmvDe6g%%646o^sC>{mpL zfzkq{2}BsTrJ?2lVP(u+%?^ZB@mq8fp-S&TUjt#Y8XG1j5LCtJJ(aM9Ovh#tG>LV| zTm?e5ajsaDFUxzgh2&qbOSVz#ZDu)dY ngv^=Ju#uoS8yYqfH0MaemIOj}-E7#}KuFE&hHV7x`qcmc)C>YP literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-p010-pq.hevc new file mode 100644 index 0000000000000000000000000000000000000000..002ce36caec5ec48ac76f46082fb2556e0290e08 GIT binary patch literal 665 zcmZQzU|@7$3mGKvYWy z%ZCP#1|T8F0@C5ac%b-TCy=4RxJHx3^{A$$mtph0?>o;G&v|ZnE<5Vl*0ni;{(tYz zQB5%DfZ7JJlcAwO^fhCP1FGPm1}P3mGKvY9R z!-oct1|T8F0@C5ac%b-TCy=4RxJHx3^{A$$mtph0?>o;G&v|ZnE<5Vl*0ni;{(tYz zQB5%DfZ7JJlcAwO^fhCP1FGPm1}P)oF zV?+We;(W%emw%aa0a$<uXzETeD|suL!wL6pt1jFU%>F-{!F*zU`x5yd6c<{j}O!TZ1!ecAH(J z?prRtE~C!dOgu=ecNzV~e}p+serxN?!;O-y6OT$pNvWhZk_##IG1X&jQrj>7BdA5| zx3&%`edbs_De!$sS40*WFT%-|$KH_{D$3*|4px%)BMK^>wX)%_D+G zwOqH^-Pz;9QLSJ6M^tHVZD=~=9h=Uxx>QQDgC!ZZnHMJ#$ddeve+B=yt*vSLS|Jw$ z+04{*pY*bH@oiqevv2$87jMTMioUI_V(sts8(y1*o9UZ(Hg9ip#^&AaqhI_h?mBIs z?v!^)$38k7YJ03CteGgW%|?iYzh>eu{uL%GTU!GvYa*LXU!Cg@Ked%Vti8>7c0#uD zfAFuP+O4fkEti*f7hWtsUUmsSx@@=2$Af*Bk1qe>Uvc8u*4D&odQF?G`Sq@A{7fgW zzS-tlW`=7gul?d*@m6SSvq-ha{W~ExWKbi5P_WltBSNp!Xq8a#i+_c_-PYFD>y5Ma z7c8npFY&bU2JndUwDR%r)$<1Y;y=QD)rM@2i!i#B(|@k=QZ+5P9i*#%#<$iuylHys z)I`7t93n|arpBoHNzbC#(#j!av3v{rsK-)dFuH-^4p?5%z8ZAX9ImB*f7~MchI$sI zF#JkTD0>iOf8)`E3OM;J${%6nXTh^=jv)$JhUy&e*e;m>wI3-{m;eWK+ZzQc4!nyj znjC{s_pi&s+(Jk802QC^EwOyiAGO+5T~IB;PQZ9DyJd4#!4=M1jZ*)r2WP@>;a;I2< z7&i?8v$^0Ux&%9pht*Gx_Ij`PDUtvDwKv8)4X6I}cC5U8tF?GCGXKt6=)=#?&B&7; z`P9kOcCYgtN@=k7`uM)7=_IY2{6BXRs17YQ6!*D!2MzpS2O(lSPG}lsOxelid35^yGkRoQp%>bEO>zIY5*W5=7ZxS>GW?FuHsl0%nWH zj#g)ZW!Gn`Hv{>En4>7AtV*NJwD-u3VR1tOytjyb(!6l}8jpa%65vA~7#jzs=j}H( zCqt?-#f5V4##7e&-T{tD?z!FU{;me~#UwZ~D6MNcXv&2pP&uWxFw*vmJ`A@!)QGSp zXG=64YO&_N$O^cnE}E8{mUKQmXzpzDAOhC+Lv=Az`uxf>R1RgDh|VcGQ)HXs2QT=T zKI8!%muu!b;dk_s%Wzr$Rbp>D+-H?!0gCc=YMU(iAgiiMlY*<63W$d05{v8j3qcum zMgqu%a~v>ya_gMLys94zxh9QA9;ZH?dD6Zb@VCS_ejff2bYb#oKcuyMG32jT*5qPp>(c2}UHwY94OqKVYn2QLAirpTx%d$W^i#QE#(*sYYdQ$v#v-#T!{1DigP zHUD`>ktVkX3ySGKw7N-io}5Z9ybMnSNDX&U>WnMZ#nfG7kDmEr`xrQN48O=*B&-H@#{Z*cpZfBdlH8pkzcPR zj#og^)>*3?Rw*jnK(~|QQlvpfme*9XtS6GK`!IH;{P1n(t$YI2Rk;<8yBrB-?cg#Z zyW;{%5$-+}T)X&Yl#AjN@PxNMIw)$kaFU2<+|a2)fECL%bG+|(sX`{a595Co0hk+S zU-Ym9=0^g<_n6Vdy%BLS;z(5{hNU#g>$5|DPVN3f!C)5pym{ni+w|?`FYU^WKDrY+=tyg}swG?^hM(k>_&eqfB`1es;sRJ)gUz8)w>4zuH;3-)+1O zQ;o-_$@?mBd#1k-an7X;sa&kP{Y8e$(jli$iq~n!`%aUqgG4Oymecxk&&MT4P-Ajw zQ&`S&i52X-L#9s85jK#z>XzTAkN^@oR&EwP7DB0+$WJxes?)$9+u2=A<2II3>rIB} z1?5xAB0mK%C}m=yVVF5M8>jC`T1wK8L~%DV+TI2N^SPkO)X8y_zuC{+wN12pQT2s; zCeC1edi;j4vKwoC@easxHwy?fWYY5qxK74m`Y@lJ(;BAZ+7@fT-^Y@_pu_JWhl!~l zhUKTzfUBL8=|_;z4>fGxH`c91ZGk0Id`nb%bDI0GtM{5D@S)zZ1w#e7@BI)v!l{ML z-=EWt4nY?@uO1tCiQ>XYy41`-cyqNb-Z^fx3$QY8qewMCPx0d=e#X3z8}nn0GeHoq zGma&H1@J90SijuJ?x?eup$C}HThNCg$UM{Vj|4Dl*Yp;C6D3ojD!7wOmoEmIVxg>u z?@Ne)%4h4t!=uBePmpn3SN5Jw)>hcJ7+)@bEpiB+vxi#N|2U?H4zK)8`a>?SV4GmT;5fCUu@IScE=$!Gj`h z(*6pwPsv#K0vGvK`OMChtYZogwpzHH<<;~e-So*%l#QkQ)x$iO!JsE6V!(dJ%QcXpBOx(!6t8oH5DqbQf@^?9nG5 z#J4Qcy!Iqw???{+xZy+N(P`xh%(F?diyMnW11PeebEG+xX+2@P54Z;kX<_OOZxUrH zELo{x%qh}jw~2DHC(uzzdHgW{?LxyWTD{+qnw|LG6n2#UxDc?NpZ?GF#q)g!O&}S9 zR2ZW6?yEr#Prpz-kb7s)tK_69>}rY|KZ>35)Itr{$Eds#U-^ZPHw!ZD^8nG3ql1oQ zo<<~fcq)^^*hr36f#W`+e~22mW7Qm0=T*t-L}b9_Ywcd4fM-9Et!glE;W=%xo?rj! zv|T1*)K*|C-Y;g&4$O3R#~l@Zzlja%16*t0mqitUN4kw+Etd(8E9`< zuOCt#|6sP$?i`sW=H1cWu18hBHmLNGY4AqWZ$HptgP5%CNmc%@M3JJbBIQX%3#J-5%_iJtDZ8MVvRac{0A zysv@LsU}yv#c2xB*||&{w`EsrJb>?U{&6{8HFILeI2igC7o3&_h;^P#gw8815r_vZD+dB(ugKp!PM&_;D&H0Lfj zf2#3`0)_7pW-MyVgXWnSm&Xrj>_Nn8j>0d+&%y*n9*uK9rvzHNPkdHp=adQjmlZ0d zg@1$y9mTH`aoX%Z*2w2BQ&rHxI%Li`Q`uz<+?)}t%fzg2>mXW=5$-RW{YJod$&HjC zsg6nfamE}&FF8FqzH>yZ4#X$#eB31Pw;(SECdMd6$FX6ausdv4tqS-okQvSooDcH} zh_w&JS{kt3^;_JN8?frJs2&Io_g(^bKG)JX8RzG5RW29^EMo+XVD;5G0#;12;Q2hT zDMWJMca8kRBTS5J$l$HE+4b+M5F67gE49mS z%TY=OnTx_G*ml{}-c&NYuEhI{grqN4Cp#+J(N_c8r=O9Q1Vog3@1rkFpL_2BBca@8 zhzK^{SiZF%q&8YAVyAd#3izmDK6`$9QpA|`{V_oy}(P@ z5#di{T|}fO=i0Ho+L3n`(KhjoHVu?jgx0z>wAJ2*jJR}E;zdpOD>r)b1cK5Vr*ntK zNrF-;vBB`wey($Ke1mZmrE6~cS%t#G@;-``HEWkA`I$uQ&MQlW%qM7dvOs+M5Mb@!%TBv7UtVy!Sz<=hh7b=hQAmim>bUMesG+t?wDmR6K52c)8_nw{#|+T{j<>x~Y!)_2 zdF9nfKYBOIbIJMPo-rI)^?al=a7{^5F0}L0{i(g6e?+wZ#ICc) zj@>xWIRgrP=kN0;>Aez-8`}%xrVkc~kDY(K1{Ue(S8TEE5}M%2I)=Ft&AiY1(&>J0 zhh0E7cG=3&!BJ1R8Lpq66&XdTLw9=LG*nIw5Pua6;<;&N?s7-R1_1e*qR?6Ze03Hs zev-c5WJ!qq#Z)~!(Co5Nm$Ho|aq zM3XU33Y4-wxRb(elALiu<#je}e2KX?jblvE*iuAsd4XHK6g}(J?Ogz-0TV3^ZELP-03HPsnSFby=G~w*#*K|(+6jW z$cMN({bNl>*-C-yXWV|FOnB6mY+yyIpa?j=ywkMe{+r8*ZtU=uK`Ik3`)qs&H#Uw6 z?$B8ZN?x=xD}JyHi3UuEQW|B_b)SBz2aYqtXKk0CGR5)h5B*td3YSjgZ_&wZeLQ3|!dvr_3&iStvSBLbgj3D_s?E1{>VUrIlGr-QP1I`oC6h6bGi z7#P$YDhGN^;`J!AsNKmDkUD!p*9OJe1%oHw33HUyR`K@js$;3SXH;LbV`?7+T}+f0 zPg)eE%7A0sWbyeuE7;JL-~?nseTXYKb?H?Y3--)!f+=EoYV$J-E#oP9;7L-+x{dAK zt4Sm>5V7PmPaaMsmLnoYj=+*jDAN6?`>c)t!!&ma=zu<(_f9s!Bno$tI1XxhClA!> z*K{=#Fk{Y$Un5AS)Gb|0zP3*tFcg}%bY_9c(@@7m`^Nv0jO$hV;vVIr*?DATy|T>f zc<+se(#8NL1G+Ye508BARdL!wrc<4cBhoaGZ*SF`fr{X<%^80#Ll*CuqhzXPhL2~Y z=Xce83@(Np^&{3|DPji<0P`x^b*v*V|5nH@I(zV9E_AE&PMPeh|IGZCD-j{ug$T-@ zfAX{GnIlR`3^>u%+*RYi4F|;%d{6=oZCV{H8+=GF0ef9u?joYdL+f_I=dA=L*WJ-{ z>4w&g0l|Ik7_(KG$#J3Pd+R#aeBiL1$Ds>xEK`q~W)qMM_CqV*8|Tu^%D*syh9@RH zjX?|TT&bo;1dJPNy@yXcIs+WV^z0i$Chr@oPj!*7a&ulhAtg@>c6^y;v+6h!j;Q!-nFmh%Y9UTOQyK9WaDCKISZ*#hi zej^(?Y>rSh;a|=(3&F;y7vMTE76Ju#?kQ-1)BPR8i*914MNGy!$P7;(>9kVZF}KT@ zT>9&?UryxaE*cI$i6%QS(~`s^p3CPJ!Rf^kq)U{WATb*x0dJrh)a z2HvFG+l$QMTOA}8t*?pk8|Z*Q!Zg0iqz7ITg3 zyBA#6^<=8-yY^N^(ozk{Q6;d0QogsRxX06#fT?~I-14rerp6&?$m;i-bg(bME`KG* zb`7{fH47}p5WAp;cBf|yLv*oi1WKK7#9zdpQ+XKe@QXjh)ml@Nqb>VY<}FS(yJdy zJ)<07Yxk}vHRuIOb(t*+d+(ElV4%wO)g*LO{PL~qXCElZR0D;U#f#JPKPJnHz&(zu zv6-s9Jru63kQuu~lip*S5;O!aw)|;^gFP9ZUQGnW#rY+L1aBHjI+PhcWmu_M8(T&n zM(Nx>W-yIG_OTurwprGKSfHjTDwE>|opiJ4K0hxUaNez_a0)1u8HVBGS#>h|~j^_GF#6@Zf~E+?fZMx0dq0lm1QFqo7(g z{n2YgHXg*2n)eX!$bnZfjIBhJ{_;CD&hFZ0CWEne{QiY5V2*TI1j8cAxLLmg6q5r7 z{;u*h!^fw$bT}TfEi|GmAzieNjz$BGai@FzFsY9DUV+yy?Uz&cLg}gY+8r)De7(aD zQXUsRxefnD+@m9TdA#ySx&jDs!gS;{&}GtjpFWh$_`l>**w@F%k@VK+A5wZ4%idjs zxuFI2lUK-0#Np}_m;sWkvrWanNEb@j+s+vVcgFk4*j~%*%ZmBOZ(UTREr6t)r8}5g z5|Wt&Dl(?1#Q()TgLJwm5O(a_Ik;W95ZFJGy_P6bTl)#rpKZiud}f@^Re-9d-7OsQ?5r0Y!F{kJuxKN?gTb#`Q_8T0+@x?wiE4`A)StZ zoKLIHfsS*)pg{`XCB2Kqj0D+hF7Zn6o4_J-KKM(U?AzYAfRU1adhNxZbQR>1z|{*c zfI(;a$K#P}XsiS8%UfN$A1c}PY5;qp_w1$ityfH|v?0t`noHkmWGY$TL@VOa-zQhX zIv4NZF%2ME(T38j2pPT3cpbOj{2#O`hvb)1rH<127G3-h720>&*5*$2ci~ ziz=JOQ`hLzoTE&)KSd24=Pmq%snLS$;_*!?{;1Fl9)W0G!KVp&ek!E^E(Sy z&kX|IkN3~KorY6}3nWKi&-v`SF^kbj2O>^MmG&Nrr3U%64FcT#>glx@MJJ)99zBMc zdLT6MJg4@%8|>I=a4U+8VGH4=I3IVW`H@K((3FG@oXkI&5)cDt?XcH&qLeMq$7Gpg zk+B%Cs_?b$0nF!XJCNS7F+PS8*$=Ffc5-fns4?tTk4qDqnVj;tduk1&T`52C_a>Nt z>2ok%dU-iHIBiai2q&m-+=;)x>$T!VhXmy7&Ri)EDL1$OFwN;6nvlK)|D?OaX>Ukw zoCFR3GtEWIMG>dbzmKonxh93sgS<5h4?4ZaFrI?9CWYcVmY-)fp>$fzIlcb94g;D0(25Mp zzx<4Sm#L|7zz9+%l@9cL>259rDogYJeds^?!rcD-pJs^&af2a9wP%mWFt7;5Fy)W^ zQI{WdvK4&gT19TiDIriOqvDFLY*$%aBQb&s&q_#+OV1?z!RB*ZLBh5qX;xs^LF zU4WAF+at-)2ma)EFtHR3<|W|lMcA>QV@C4N*DV0M^EHaO$T>%DX4tr>-tPm9%KN$v zoeg!vNbtmA=yMuSsw^nH@jT}PFO$Cd%PlIO&!iaVB{Z!yV`{=q4eG)0snNq3 z*Zilxm5z~deBp|-TdWidLc`yiAw4wyIZf`~TdsI*0%kz|bE#k8@o@{-z1;YPB67x0 zi?nYQPc2wrk#t8~veP120mQ!BgaZ54>*oYy=q%Gl5%A&fHF$Ln3drtmk%7>Pc7^Bj z4XfjS`?tto7g!xF@m<0|`EP9w``@&Nla0R{7yhTpxJXH9bfwR143Sn?e$TU-eQ}L@ zLbv@5s81xfRcaAUNaB%7*c&Xfb1e&3C3Lc%8liuyn@@xWpxfLs2l2H*-t+e`ubHmC z4~8l!!xg`2-%wkWk;1mg`9tft#?T4&`G0fU&)it8Svq&k@J=2j;Kj#d(Q)+t9RiXf zEJ&#VoUg*H??>K0%$k7kolbPS0yLX4AJwW{z}=)@HqrimP=VCv1eEdf-Tl#ANhW7S z#-9XZ_Ksa~dwr2V-;PokrM+A1Au@s($h%65l||P#e1Cv7U@zo zaK(aSL~6n2D~D66G_ZdbHGjL3_4VdZ&^);Msx77trt(vhab>&qglB_(mM}pp|H|h} zZ@w}k`cw1HQ(6-Mkk!qgH%hC>R~Ie&?Z_yLQn&6`!*idn&Y9E-eC@nF zYISD%z}I#Xfl=S<_+i{*KjV?7V$$WB!lu(m-PQAlluGFl90wt!T>;0+ON}p}VZgEC z&i~(sI?>kz1=6!l4v(}%@i6u>Jc_a)-m~3Z$J%gSXmSz5z^VFI;6Uf%7P+rc-M!DDo3N)@GOdU|*= zW->HA7!22N_;A2n|8ha$Z7tz{NBZU~OIbZaqE3Na%Gs|MQyIPdI+cxpJBLd8%IZnc z+^Ho$6UTrd?i)6AO4eu<&U;P z;h#-)|2B0)PVXzfNv}%d#0IPaR=!<+e;XT=+fwl!RX?)W_f7VKW#T--@J5G1t?bN+ z!`8{)K|-xBy!TkSN65=Z+vX;8gd#tEkqG3Phuk9&d`)rzj6d1?^##GbwI zkRQv{`Lx4*A5eRQg(fIKu%r$fEgcP>nm{pkQ_#m8_FSJ2qYE>_L^$<5H|x&w-ru*h z2rhzIUTXd-scZ+H?wTnjn_>L!cz%Yjk3NnL1|?CzHJ`L z_`Udx{G5FImu0Al-#V8MwC}wA>|DyM*Q8=8VVCEq@ncF-HZLCIrxUHh4x+IhvL+nH zZOvq)=5FM&{U2sI%O|Q(anLNsDp#t&Eg4se@(Dz;k?Is5ec*a9s(@X~S>7O=3#7`s z`~cYg`IJK}Vv~}yUEnn?1Y2c4F*d$-uNg=eE>@*~zMbfU zQeV@f>h^>4=!oH|daU%EVeA;jWMjg@&cdpj2>f-|Lxu2v8E!y5kCD(9DT5dahY}(G zH!ttioDHx~hc}$^mlK%oWv(JZ-%x>EL$L;`IDD`sLmFIdz3vqN$CxQy&Saq+lK2V< zvQ0b%$F>yiCNb^ziTnw^pj7T*bxUKRZB#>4o3jYddG`VN#jGu{0S}ZyROnswa<8q3Uha~i)LsmKm}1;p);yY0r4gYSecy#>ky8hpAft_C$;sV;@@*Tz1r zXDYqg3TVUQVj@xkTmo!TNN{+`$B4R%^?)KTr4!IE_BnS47g5DtamxVp-s;l&iu!#| z$1saZ8*`Z;?OpcF{X);-*#Ze}cy&Kis*$%L!4subenII&Ag2@% zIFAb3Vwk$#lPQ}e{B8x7UotsRLI*-V%Jd5B3Uw7I#i20V_J;5dQa{%%WEG$ z#C|WOe2#C{>x;n4T-rhHtnlRnUBT!Ww)5aaDI1OyS7Ti}^e^@?`VZW=P0B62Z@oM# zsM{~nsR$_-3K211^U91JLCVKvSXWr*t^OE>$)~1s9oXVEO;tNK`@mQBpKJ ze)m1@PGE5#Lo&@_CE{$g(>aIhZa}|p+eAiypnjo;`%kCg2DE=BQr_b9%6zLB^AE1c zKNsNWS!r>mM*Ml5pBt}%O;IJ(KDIKy(5y4_PnGz8G)q$RZ`jjE##&j8oNg|5jP1`^ zL7D7zU(CnOpPU|phM6)>3!q*0-5q=`*aO=SFre`T2LtU1#yg1kP$`*|r5171 zx-$txT#w17!0nfWnt*`8C3wml=s10LGxZLaI*N5Icdv1Cd?VIk#GDRcf%QW9Og`w+ zf(jmUIIVTh;ir}&MgNhLIQGNoh{rb%2{x31#xS|q6ZVJI z6qraE---upYNzl0k027Vd#++1Epp;vPRNnQ*BYj01Qh<~%778Ljok#I9X0jGg?%J@ zVpA@iZYv;%*4pW1k2~4q6n^#r#jaC?$cJod?Y`X0F!N+3{FwtTh$MAa`D?s5o$0 za6#)4+wfBzU>Yqe3I35qcU^F<%Uk2rNZ#(N zN8~9R=G`Aypj0_Wn@4Yt#?N@djXhMt?@XSSR(lRC1E+G&BM;M#k1a1qC*hbz@dOtG z=RP?yZe?SvyB&=G?`eg(WpfQrgn5gIo$Q>f1ZI?Oc$dadCY)M5uoP;_sf$t7FrVd2 zQ}l{xFCl={g(WPVeX8|?udXU8gOLCAlmC zZ&BKryfNfr^#u3Fq@DZuT@W%AtNXz?irAm!=3pB;WU3AnI8&{S<&Cd$3ZRuzq5Fb> zv9tK;^W-+lyYaz@1>Y{6Uih>yn-8$M7K?Yu4d}2 zc*!h`4ihIFkG=hNfa6V~1ADy!GGk;r!s5(uM+3xK5toN3pAqut-6sM?3I}&%3+*g9 zt?51R%n6F-d*$z5Xh(GD$OFN^dcAsem1p_hT|%1#9Hx1r67W61L=A9iAJa-df`&!3 zv$4hgA#);T-5tLzQ~Y77j{p7{#@|wFvA}&2t&sQ$RQP7IH9c?>kg(CufZIB^?_e_C zPG@0A9=KlS-!e--;NIBf*J+LoEtJ|{!mh5U@?}3TAL`|$sH4c>9YZwL7JW2Q&s{!~ zcz*)qMiV{F9P=h#HThJ)K-zhVdlQ)&EI^ye@VP4UpGh2|KAkd$GLg-@?;I)}%7U2x zzPtd7PpAYBY% zsD2~RG#u>yS`ij`=_f@k5%Fh#l|-%Q8Aa$9CQ$Y%6jToOH9S0L*~yO}!!BIWzD73l z@$4wBswKkmVORF`Y;c%5R{eJocmWuK9}fm-O_$7ha5sWhM&!Qq!gR$QWGu(%kI%Ls zYan8Dv4h*2xqBB1oreoKBPPR5wv`1%kwj*IICb;gvQ8-T0k+Mx!Zo$}0EbyHW4(|U`h1;VyFzvb z=wfnB6LdxOL@1xpg36T(+-0+6kt;+T-D6WB%NLI{jtAZjhFes*m6tv!Ur?A*Bx2g4 zB!kj}Zv5~Bkueb?H!`x(xqcy2vkKeQ*p1RSA5k859yX07{mr%slu5O=rCiFicJy%Q zGJ6aPSb?baK1t>hU>SJgEk~puhc|Oe=9s*hn{iEf3HHHHr2XK?#QP~%{2(hm7vTTJ zNap{wYI1)xXlTs4dEspGfb{&osbMwidY(8P%jq>{>4v=>Oos%{3eaqlF|T|B3W$HT zS+jOGk5Fc>{(~&~Kh>czg?_xEb%I2o60}wBb1G;aEPOZ>g)la2yKx+s>CGyVx&;z^ zwr?e)9MklrjS~L9Zqk44S^V5Wt+Y!|bU*Z3?M<=XMW8<=>gs0MQm^s)WXdwq5!2R% zUh<>%?HVLg*Lp?E$?Z5fu3vp(FJe+`rChcDd3LHmDVY+RE>qZ3_-d*G0tM?cTo?4} z*)<=jqb#b%y7AWhO^0G!@n9a2-sDKfksyIZu%71$<76trexJ&7|3Zq!Y*T$ug$G~1ha&V zo*6-#j)^0|wJ2Shd#*=GyR=64(vAN}?8SW8v*#c+h6BZKXz|0zh)XniCZ2(%?-WSL zj}L_BOwZ|NVt~7^Q709C_buHz`^}WxQZk+l7BV{@C&HiXN`EUcF^b)Z=aUWbT=~>* zOZ3Yt96U1wNZ3%fJs-{RS>rThpZwpQ!l%P$ycivY-Jdlt>;@+t31nJ&Dyf!7hxt#I zVSJ6(^Ok!n5kkrQE19?<`bt@LheP`Jrj&uRAby50Od!qrA-oFS%qM_Zpl2c2W+~d2 z`k4`!6g7*f4Xf?nUehTVxca0WlouOt%|*@nYUDo@yB6S)JYXaAK))QWnX;yi**|M4 zqHPHJS9Z+7wj)nZ8Z0$>pS6|1?4HUJUpEhH+|niit@V#fi;0V&)g)*ZQa-h{5bnaY z;eta24#yr%Y?De~+)XzGmn|ze8myn~mjlZBm?QMu;8rmAAyeBH`)zea0ck#SPo7hr z^@%NT1`@;3w3SGSlOvbMg>-W_2 z^tQvZoP624e1!IOj zZO=z3iYdBl6{EYj?oI^5_)xB@b=G@(sT*lyIMKQ4Ck;=kZZ?B??CQ15OV9GpGiOtw zbe}|y-I!GHBN~+t&HjaH!O-JXX(g%@gO#uDAkZc!II5wnw}rS9%bKqK+R!a zPVPW7%Jh_%lKTR;iz5ME)QQkn0G+Gh%(B=wj0=_^1Ik@(w>&7_@3LoN5?xi|V?azj zZ<8eVXkr9Kx~{U+)d9=Z?6{9vs!&TX2)ZR(*6>bm{tL=C!%{EePr=1(h7BU%gxDj6 z$b|FQpP|{nR?6gHbm~2w9HZf7(m!#NBNQLZ1l^fhnAbBuwZhIZ_g9`-gR!zqI+;*r zb(S|J_3Vwd9ZxOs1J{`F&`144VU%GuqQ?<*_ zz6ehDo)f5c+mkDL;`K8E%Ts+8?eWM>@s$k}EFP6=Ctth%Wq5m0f8~DaRy^%;{paN! z@VUlv$JMh|pAt#VV>shGp!Hk44YA&}lt-LGF z!QIol=MiP50|87gJl9>@-N+Pm7tcLaa}LY-q~FF2eQ^SLN^`fhCSBtR*y$-{)z_8J zSxvz-72#JYomn$0KM;V@9g0a3E;ZoOQ=!iU<&@7&nU$Fq&BQ-!`~Oaa{ENd{?bgKx ze#euh$eUQo2fCVpUp5(#FVs_#o(#a@ z(-QhpUhv|=MC)-stc-2lPJ-6O$SGXdTR44_Kb2ZgCVXsZHxZ-0`#??$l1|_W9f<|2 z?1-`Zh`j>4H8YCD+|PM_7n_bP2Z5B$wXX1=8bW?TYb2ceLh8}_*KzBq?zezFel;f| zG{X820fDj`InRMH-rzzyP95EMA)^=cPW;@>H^OIYOvc(5W@>Jm|M}m$J3n+js87b& zyeNKs5;&V)X$VcmJVGP=c52=oC)A0q)om|MW@`t9yi6%QYda!O#7y#d-7H$7ZBx+{ zLmBE6)pdxgLrg`?n=u%wg5>PH{jUR)bLfF4)cP)gLVJ1Q;+IassO^;%witq11OidF@zYe?s#Nit_LT&3m#S1SIzdJ)bhN3-y|*&FHKNs zc_MqTawhtm-G@9w7=WVSzFPYpa~XAhFo@^V<0a#$g3dd0)CxN^C|CgN%q|+Ur)EiJ z2{8NPNKxruy)u->me!X(AM!>;yKp(noFxAf7}6cW?<=FP^h-5{rl;TDj_4s`%S-yd zykn_Vp3a137Oq${vi*5kL9cG$|b}-*O4|M%@;a%FI@scHT$#g8pprA$lL!E{cTBBc1dZtvh&-9yqa7v zN?Bw3MfGmrz)vN5w?A#mVvbf&zyJFtVLI5y>m@Q|DGUndy91K_$VQf;6?0=Bu)FDt z;NkZPSLBjP_b02NxIoYg!E4<||1{WA@H(nU4@h_SZsze1ZbU{$a}{HjS#7}vx#;}= zq08>he{)O|JfZOPkvY}ir9Ic=PFSKmZcPijkR@dyg6O6nwzT`(NFTfo@7m zW5PoizNH@|jHpW`X=Hw|!3KDHoRC>Of_|ct`hI*dL=ITXRZd$H|5*CiQr18JyHDhf z@LFN7#>4xsUo4yhE`>Lyt;aHL&OVry0q=#&gLG8AW1+-LO9W<3s~j;kGt2B82_l$$ zE?Nc2t(1!>#S0&w3}&1(=&YXK3j-94Y-Tswr@hKuiP$eM!Y>M4H8uPoA6cFS;krHC<$by(1UU)=F+$8{^uT_uG0dc zba$1ROhmrFH>%i@$WZKR9Z>ct+Wtpg_FfD{e=gI54~H{U)DG0Z0c@A!^Mza5+=1LA z#P?ptm$#{Rr_Xi-G=j?bORtu-+_uWJBh0WG!oYZ{*st;We>SxIHgFykP*69oTrxi(IFEf+JSuM2f>ujwVyMBX3?V!@;wa9{Qyj4n{>g6A*vzo|P=syT=59W%C&|o`7KG)K(Qk zWjOTsyX$ARn{&wU~gLW*T;WJmD>9 zvnQ?}Dlgi5oY2ISv>x0~#QMgYwcBbv#m?ZgvmpF)j-+JgsXJnl`vBu&eXJchxL&0E z9}O*$E9doi{D8CJTHn0MB(9DUwFR%dyUH!}Z?q%{6e0~Lc$mTX%)t$!lgpc0z~=Fb zhMosef&E;s^iClmX#9+_!-?qf%kSU8o?g)ldgHWm_;kw8D7BU(r8bH#Q_Zazfw-q& zdBQ{krz8J0HX=ngD=L^&cUih4nFV;iOBK$2ovT(E`xhqL zKa*9}HPdEf0eeseZyMVYWmyE0T74hr$w*RikLzl(1Ce^^ndAkEB7S>vk+?Zx{q)Qg)c=v)K541ni}+_M4+NRu_9qjRO<)7tpVh?c!o&| ziNC7``tWat@>|>-3t>I;Mo3R^?tcEo2W_InS~7Ktdbp#QBYV;qoLIi7@C>DtDxOor zen+*nA6*CjOChWaZVS?4C~`~>b2bpzwbO6M4ab!&%)|Nz$wy8=bKym)SO87^)|waQ zotoxjIu!|-@jYrrt)(R~L-S0Ub#IfD$QnP^G_Y;HCGC&4wk-5R5`^;amVOnG@iuYn zEDwBKCU@3kw6jmSazZ}LB*Wt8QjHns<4Kh~Mdxo+x(>&Z13;qfwQ3QX-IZ{ky!``N zVwUd(H<_M2+xc}dZj*EKP5o>6O&QqSimqQKVb`|eK$}b{Yx&6ce)YLyPmkKP*5+(n zH@VQc)o@^U=iE~wvb4m@KjHZPvT6O+=F#zQBHD`T?(M1vtE?_3cTZ?dTP0c7r--;) z?6=vp-jU)varoOt{d4 z!xNcL;?x)0k4ni}d}m<$ui)`t!Q;Pz$A1Nnf8)CPui)`t!Q=0)tN%X;9tGr}$6kx@ z?en0fgQ3-Q{;TYj?3J9|+X&bO=?{F#`I0jSqK-emXKdyLf4C5_fLXw-46eMxlweBe z;(mes$Ts{a_^wSq-cSx=S4Scz`>D_< z$cw?^RF+#(LsG-_n(H;m<;mqKq+hsEu#F^=akZp?8Od_#OEj zCx<47A3JTs`P|64kx&RK7p-bMqp)j_@%L?O#rH(_MEBzDh0_-OUzjf2#u$t9wc(Dn zQb#GK2d^bxOTG<%8{Rq4Inby03-2y(BLS26D)y$Lg`t8FH+wsKJ4ay3|I^-?MpM~^ zeOxKC%(IYrCPO7;o`sM(L?UI5P&$&aGK36emYGB{BoQfOj))_Xc}fZy@AsTu&-<=( z-`0B0YPFs(?scv&EvL0=+k5}7ef@ve-dE!oXc=g0;!vM>8#)_0TO8}vU+n-x?E%aF zbB3tTmE zTDV$~SdpC1MSm_RqdwZY05kXH(Hs3fiJVhpo`BUdV6`GVy08-?N8eEu|8P zXag#o$_IC%Ow1?pt8U{v!uYEeDsND6Zh82h_~1&@p%W25N<7O&toIxhG_NgoqHZES zbBk)7Vq)@$i2gD9)cc z3o5kUy!U# zpM6LC-_JzcYMI|W2}OoaZ?+2RH=;twv^RMtnvB{aSzaCMcaxb|cgbFN$-#emeJ6_B zUd~JE%$(-r?=0OALB+EqsfL{pBe;qrCPF~0PNNSCWuTM_yL{Y|iUtwYYyMY6z&X+Tx!se4u^iHs~x$M6z*y89od1);* zaV_;Bg&W;YxDr0mw(_i5o(j;(*1UjlXg@IBirBVpDPZ@Mkjfv7-74BBX#TqNZMVd)&n>%gLfS zF4@fUQg05!+i}Evy;nzYX!VNPKIQ|i6xF%Pz#9K%1oEoM&6IrKyqTPQl6qX=1IP45 zhf1AFF1FYJeZo`bD#IJrhYRi}rp&szG}h8?nw?h=0L~iZAK-C%bE{T$i|MCJW^5s1 zy5VXor)OWZ$E|k!6U+mp4ePpIz17zLmzC=NYjC*^XVa~oJ zRMQ0UGu~G|pW60Xu2H&KkR#53G`lg5lQ@#3ek`1|gA5sT9@-m}z2dHJ?{ zoxwisgQ8@V{tQZv9x{0mP(;2O^Wzu;$|YqjKj@ zFSulZzJ0T%{4Xe1E4GI>5c3=7*mbw%U08zXUQ!!)pB_0LIMn(CAE?D2R{pGBnbv?y zLLw?!(^mup->?6Gv+?4gNa!o}Uw7MGwd9tYUJSUGw!Yow`IS0~2Hx2b4)^3KMQc{&1J0>T%`E049<&J?laG#@zTh$xy*jJ`| z&POZDG_!#j?`59Y9s<*ktL8&4o{1q-u@;yM$eGE6I^5JbpEGrkHj+X{`8!p8_Lv*8 zK49cv9UZer%(m{D+}Ov^dY9tJKA(>YIT_c&KJ$1de|o|92UHhYu)SXr zQCsP`l9B!_I?oxA{YYY0p}ITL0V-u{0}>46^A9L~s&%m{yC!b)`o8nCEYfsTdK<#X z?BJG`C#X`Pq?R_9LN5BmUMgwABk^LT-eq^pbx$Zy>Z}~emdNmheh~d%dpr7?cVAJ{ z{V^&I8C)jx){>oRTy9d(nD%jPcwC^pFIHA8{>t-E`^Kndl ziST~PzHszc?l}ul`t$n+`N=x*@@=T7&`=wmQL9vA!y((gsViG@yx&ASTP&f)mPd(M zZM(4IET!)JUA52@cJHt+3d74>XJ^$9MDuc#?f$^L#|Y~shWK* z&|Af76JwYX0y{|o%wdwJ2?B2WyjB9ND9MQGH^HmrW$c}I6$K}1 z`rSr7PWwE(bcvmmog`F@t;q!$r7vmsA~zaw#SA@_-stWN5PIi9L>x$mW8v10o=QM; zAT9AnA0BCXNdf(ga3Trb&P;lqU+J@lmwlh}<&mN1MEYAtR_dt3v!8#=%zmnH8mw?0 zs`hA2qCL#@If_4serSf~_a2H9kXZBf)>lGyew;pgzvPu`pL!ruAb{HGLb080DMfLW z5e0gTdX$8@Ld&Ndejbjg$D=`yQM-qPm_H*DGyr|{T112-K|}& z$*!6Td@2Xww;g>dL}t{T zuPKK;DLB)TCFzF2O>SYj*_C#x6WSx<2tAu4gBgcgyQ+u9mr7dR2{32PINJ=SXLfq& z63`o?e{)i;XE_4}ybNAEJ`p}uZ>@0)HBdNL4USiK%ADs^k!M7W(5v>1qt}BD0#ZPd zT%sH|YJ?^mQb`TsLT^qY+>4!^-=jt-88uM3_TWhv@VJC_h?M1askeN66?$W8w`nRb zH7?EhM9_yUAPCN2L=99t626C;M{tSIU$K{Z$ndu|D~H&97rNG`I-6L8+=j3D z4Xs{_bWWzK0bU**>SOL$zAY)(^Gv^WBy_LOC8NkKI_J;Uk~@IcPbG$ta+MG;&J!A`n@)nE_YE)S=eW5Uu7b{qELe8 zQq;|SJRI20OEtKyqaQNj6Es1WU}0gox;+)*A>q8*x{2PG-tqY+=>VAfC5J15Uc4mtM76j@AP0h)J>u-R9r>(qx0-WGzrNj4T*@ zRJwxRmy+`x8MzVbpmYGd=_Hg2D*$tOD~gh<{PpO4i80jf&AZ_(jD>kI7$R7^Y&wLe zUA9441YZ$=nMIg!nFBvL*71D$-G9C1pucBx^oE==@5{-IA5|aBRb(`da0W(GavfxQ z?D*dE4Ci)#nRnPIPiPA60rQ9nYB~a8oXW3BsY{4vfQvwou5d}mOVrUQONEiz;Y_-! zsP1>AkJ2;Cjt2THXi--GO}ejUvbXP<^KY5+noK+)cv7o|?JUvJ>K*~TBE{!K z`wog<$K3UEAzE~Rcc9s_CG`S_m3tvOdSA*7w$P$*!rbE>?EWZ25~PI4V}{oR~VWgII*5TR!W@?#E;0dH? zw7(_Q?8(hkMCSuI=LCTVz$K5fSr+HhsCxl*Lq;ZY<>WUShR6qqhDd;NotiY*& zLkKt(K-^&gr2>#?B6#;Q*L6@T07=qO7&a)8-esDgu$O<&qQd0qZv49GqvLe1km7{$ z2vwkZB|#Kxjsnnfcx>P?VPLxt!(E_24kn;83v(hFKWJ)`}^ExED%b~v6PUD~UFb>hirj+<+&8=FJxVKb}okFVS>Rc)q`xIMf9N-W)^ z0Z?MeTL6F)%R9BdK#AoY0GwE+0NbF%(jgJRJlzC52J4qiQx!e{N-U{Z5kEnRB^LT+ z!X;U|208^p&VUXN5Fl;#&?*l#n#=K2cVp@btc%&NO(=YmKGX}82|P*wot$Viju3$} z0qV8|P$n>@^FlY5W6aZ#m0jgly5)vrB2nSy|}s)<6bX@bT(<^25xHn%R9 zhkhPpz2f1p|CQop!b0w+i8MxgelneZc2O#O#B-fjKS+~?*D(Bt6WI^o!uygAk*mG_ zM0ElR> z|B^-)ckD$f5AZ=fkMoE*#xkwL%>Y-6?jm_b);cyvHk;SPM}7~JV^W#Ys`SQ#>mFW~ zz0&AmI;|wvNM`b_pf~Tdx;H9gM-3AVq<%!J@;tic!Ko(ISK}tXOH5}-D&yEeg^JD= zEb2#CRW+5NW?Z&%KP0NczguX0{~q&J zmqEs~d_Ln@_WKWHXP&d{({2=}Zq+Y>vv2yj=0oQl3>U8b)az1GAtm!9H#Y`eOb&@6 zIm_suj8@-S5YQ#q>$$fHfN8@W>aTVvotpb;O^p|v-EmpAaOGJgw`=k=AA)QYP!wI zZP)D!n~CUR<)IsrI%Y>UlDArFs;WhP7b^#{(^E%73a|Cf^khGlL08N}7>T06mB_p7 z|8Y?OPffHUK04N!S(7AJEUo2MdyDFbGyZn(>ZJj@vy};FM%02>DBth-3eyE(X?@N0 z)2TD1D9>{WKaGlVK4TfP2%;1laKAvPS2w3>Gg^%-B!6I?xkY^Z93OeNkL21|EKEqj?`5wN5#u zl`b5}_Gl6S)lPW!mW^WSvWel2Z;4eqG+pFzU(bj$c!YM@Sxofp_I$At&s>-zPL+x- z(G!On1reOeL&~OP8gHBM@OyI-3!k6Ef67wl9c=-2ctusSK@P88;7Oo^bJ7+s%h3>3 z&<$dwwmk^WqT7_lVt^VHUU%;Lv3c}@t8n&-Qvx1WcOTTtWU4_GbWwHPG;(k);bb8A znC_|=@!wXyFUl=-RasE*P6vrReO13f{3UKaQzd4tITnAh+0gZS!#gsj#2$EGDtxMA zxTsER(a=w3E$jc?#L4^vkU!hYVepAP%zFHG+#%yix|%(1@0u2GVvN!cCe&SOwkvL5 z_IVr?r<1Vg@=(eAKyhe@eBomF=e1NcqGU2IVzAfFaT7*#e9r7^f`>;|Lq4;5zO(v1 zrXIQJ?kxF+XPv$!pFcN0Bg>inUp{^mWPkBTE7DZ+4e2wh{ydMao9x%dISHni z5^CxLfFz)uwKX;7cip5TIPbl(oJVbyam8`F-*ppd!yfTXJuseQ0?za=n41cK@f;hF zuhT|?1)zkLuTrLo(_ts?9<0dXo}7HsbFFXkWFECTuxYz3e3R`4!Gdj7#DsKA{hpsn zDL{+=I7y()xb(@**&oI2QCj5@r}%2A=!lm$>&T@!0_;i=RyF0$meL^FEVPv~bw z_b_W-nDKWxAk?a%`t|IM2bD%6=0Q&9JNJ66@)@wEwV8S&$U9?L3q4S+YhGP_4Ll@voHZ z7Y}DH=dSPhsZ=Jnru6cW;Vsd`4i1@nOXBt8UR~p_O~1IZ-q;Y*IV)v2Zg5N$o=P&w zue9P$T~)X8^IKl+`!wk-cV8G-is67{)yv4cr?Gm8&H3MRu8ff6T?o( z*C4=jgi#N_XS=vzAT~CuM8w-3zzKxdO>LCPgfdYWbA-sZ+PB9okwft)lL>92=yiaw zr|^Rs;7q9T{(sEr;;O||G`qiR44kNFFZ`K>c z42be_wBH6%UP>S;Lsj&QH~^-+G~_lwl-C8hbr9u60A_o0A=vBz-v4+i#RJ^-(bkdF zTZ&$oNK$j*O8MfpgD0Yfur$&tk(ndZ1S8fj}n@0U-~lh?m1sq zj%qTan`P0SKgD&AwJvE>j0Uh&YocedX)qsYvq}SiH*Z-0ym@zlb~DuG4el~&yzfp1fSN40Pl2Fe z2{A0XOt6A>y9BfV%oR7==fpPHaCpeA6)+LkZ_|<#Y^R5N2eL5No6hHW;&-fDgO;-o zziMQizI#lt|CyD)!nZ9@-Q+|vs5OK?_ACIdn?Tx{Y8nyXx=9BD0M$(-uxM+#VDaA~ zy4?WgihHpq|I-Phu z=v}UFHKbU*ny}Q9;fi@bu2bRS&c7*;r_!iiLL z45kl+JuOXbLGW$ngq+RT^Y#g*!u2ekc6Yo53}($27qKUhw`ZQbHzV#L@EHJ;6k(ta zS(`i+P~%D+?&Grn=KTqGk;VX+h+hT3M7(kb03zZchkaU=Cb=yT5fAAt=xl*p3jpcZ z9DR*VZ-H`=$Tr4`n=L+97Vd;`2T+!$Lx?lT7}#rgZaii+7g6kk50t+fWFEjNj(S*+ z?$l9%6>b-WV;)qT_iO!j{4q(LI8N*Sbw^m?Knmwm@={U}F`w@BYU0|XZaixS|BL(l zB){!-D{dG6M6J9|+L%LJ_T%XA-J3$c@_k|d1mvI85xoon28QRg=BoRSYYquqE$Y5i z04p3+;Uec>a4F$UhG$qeKr0R=1fYaKG-<)Yhq9IeH_=S=Y~qT8 z2>}ox@N|DMyW9$B#lZ>(B?MrF!@LL-b-N%);dE6O`FSI^Kr5~WCIldafCe6@ry0K9 z1Z3-)>@&U(D;#LWJ=leTgB1?*o?TQY6G-85GOyE6tHHMjfNnS0&DOo`xKTDQmXFlt zT}0(egm$p43$k?!aSEDXg#)j+UBD$+;lTI8;(!#+psSYX4hH~Qaj>ns3%I24+;6}u z@vJj9x@e*nSlSEQx|p`^N1U@`u)={?9ISBIyj}rPxV)(^Dk>BJXvOux;1U#EVjFlh zFRh(lL4__;X4ad+wk`-WU36p2URJfj>t^paL0BFTs zhHYJ_tsCV8GMQjo7i8=D)A+y&2YMPWY&SF)RyfS7(3R6b3OD0>!tD%f>w=#D$}rjm zT!L*~%(r8@cyU3tE(%eN=t^4oKr$TLA7+ivOJMDV439NAcrf|Ay zY%sV4ef}G^b^rCzaMUATAX^s(mkxk0qIZK84)Ovj^l3||!sTSfz~B<}fnC_v{R3MU z2A2XLPdt}t!V32X3TM#u9tM{_0cR8{hfV_MO zy^jJ0mtb%S*BrB1`41H?CE^AQF2Uf^zaH73RdA@S3xi8AxP--#4I72iRa1t+B^X?C z$84U29@$<)9=d?RCFqe22A8lnvSFieIhh$SxCDbsRhZ3_P(ibf1y493N`KIO6fn31 zgG*TSnqZ@F23_5E;2W1h}?I={xcBz8H;1Z!f>yy{a0y(L#7g0056i>g5)3Z=Yp)5k3J!xy zvEXh>L1;<^-?)TD6&#A@rbMK{;1Ucj!Qhg^&%-hGcr>k(;z{iu5@HX*88lW$HW*y` zYv7U|3@*Xo5)3X$_#SE=!6lmTNNl&4ddP4ZoIyifHWTQP4F;F6II=-~UMQNIlbH*H zOE9?fS8QDvT!N-l@Qq85Zc1ns9E#=|boIbDE`e(V@O>12bRPu_EUumPlJc6#X&&>N%6lv1soEs*^MT7!fN z+AcK$*w)3OMgZTq1i2&^RyZtw3NU&(0+~vpy+qzhgxX>y%)YgTcih~vIFDV=hE{#H~;IOUxN472uF2M@- zmlO^Lm)u}m_b=JHFt`LO92P}M7+k{Y$OhZGf63N`pNfMO?k_1EeB%=8r4RTaoqv7% z6s!9v;HTnXG#9>c$q(r4nv&)TBoRROS0+t3*g^IeXblo7XuG6T@O>0m)Cgd3i4WXO z2`d~HMMgogiDSyc1Mu4W!Z0sT#Cjf@<*>-f~K*$4qrC;Kfihp(bP!@dtsJ|WxZ zMA3$vCzOks4)XOT_I{%%PeSlO^{@I=$q?j^e}1cIvcvDaUweyVA(%l)l;D zA(ymDNzgKT8NZwTTE-~ zJG~v!yid#Y83P~pmn_gfAUs_reTAEmuu1DEaXRIfT;ZSx7pm0}S%vmQ4!$qOZ!p$P zUhmlVquunIUIM@*ViJGF4sqlj#q`A_rZ`P)(nC#_t8odp?btptRhy&*o^Mg&HB@Il zi_9l<}`= zEP>gcvJH9)kFJeO2gnL+Pf?Wd#GVQ{+$hho^Q*p9v5}_QnWDGO?8VC|-L~9W$Eqve zq>TuVFVrd*tBra~@U!^$H_m9+v8I2HZojCzpH&!lrja(-+`h_*?-9dv-BEegr4Qs+ z9y2yvyG%`fPfFd6kaD5(=WCX@OLc7Gg{HPfw}xCtImYY!Ofov(Us)B(61Oe!XLGZ= zO!fPJtyQh`;efC4RwsvpF0WshKU)HpV9jN IzqjE3KOPK`f&c&j literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-pq.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-pq.hevc new file mode 100644 index 0000000000000000000000000000000000000000..eb6aadde5a6d35a986337d8e31b96851ce4a12eb GIT binary patch literal 667 zcmZQzU|@7$H_!n^ yHtzrmcS-{a_W&yQH42we7FjK$Hwq6W>=|dG$n8eqT}R=)MB)8L<}tCM@WcR?eLf!m literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-precision.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-precision.hevc new file mode 100644 index 0000000000000000000000000000000000000000..fd7e0db1e3c6505fb72d6c21abf223e0ba4c42b8 GIT binary patch literal 94032 zcmeFZc{EiI`#yZ{bB=kQXAY8i3YGbgDI}RPI}}lwLP^Osl~5s~k|a||rwo~$M5)L; zW=au}kjm`U=Xuue{q%f(&+y;-{l4p2*SfE@*Ky8T&R%=p`@XOHx(@cV;h+QLx- zU?KevhOH_L0k;37iy*(ho&dqj82s{$H-X7%&X{f-jw=-fzQ{V@Cfe#o^N-~y!WI{W5!Y<$edX$oY#UlyO3`hU^ zf|7}UY1r1*`r6jk*6f+uD?+Xl#iNDC3v&wPw|T6HZ~N#MZ$}YzKdtuG*5J&V-DcOQ z`<9EZ%c%1<6Au#WT}FTLA7PG@-`e`}aHC}F#G{f?QYxv9FZ!?oDizb7>Fa9I!w%uOm|3Q`mpIbDQ7AjhySh z{fqyIJD;|;3YWg04n2G^E91IbvPY8hb+?q`H++&ke(@i1Hf(DvGjEA*eO)VH^N8S4 zE!S;!clNk&RO=W25mnk-8=4Mz$ENeFE|t>kU`d8;=EaEwvLyfFU%~%vYipXmR>;Lb zHZwKdC%r6Pe4E$r?At#2#oKX*qHk-fSo?eZhSz4{X8Pux&D-0Yv3Ym<=okNryH4Au zJLO%{v5yXi+8!$jYbHu;vk_w9ubKFZe}&1)*4BW^n#g9;SLgb}Pi^H7Yj1O&osg~k zAN=d6c57=>%jM>WYrptcycOEoEK==p|4xVv8Ptd%6zsLvh|udaS|t?x;$NX}x3#tPdgHA9 z1&eCYOFXT-0X*V7t$aLu^}GSU_>XX3wIQ41B8)EO^q;G|R83262kEMx@vSuuZai3VjBa4K1D02`uLj*ThimEIAGZj zCdXjZ{p+$Yx6qM2K*gtfODrGsN3C{M7gUR|6EGgkZrNN_aK)=Wy=gdUu|GH_2Fv*5 ztKMCfeG6K>L9V>6&YAv3O%)hf1`tb8}GxDTI zK6Ns+-RpdZQX1^NKE7{iI!Ws$|IeKSszZwn#eFW`K?6V7L5LWS6PiXDQ#Y7)lX1F{ z47vo0C@}QD3#_b(_;jgss(mCxPi%5|PmF)?#jn+#AS3ng=c&$lSvjkue5gOSsOuha zlx`BJpUgwM1e{4>RrcMx`t3@eFCN3y*zsm8ZYYp;JHy1hc}GAT#btWhdRk?sfZ5`) zqt#hp+4b4#%|JdO<|s-jtI}vQ?LBg1Slo~R?=51VG%sAg#v@>`1o)5##>Ro^dHapc z$&ji{aiJW%@s#zxcYtG(du}(ozpFugF$s4)~C7llsnmgM(h=BF|P+iQFKELt|l|z{(qH~JQ6xpWu!3#d7 z4|zbx<(m0U_#M6EGF;YwmDt-3_gN)bfTFye+9pdr$f~N+q~L0%0-~Y0#Ns;sLQqDX zkpObx90v@a+&U*Quj&Uwu1VvO$EiHErh*DHUW_>bgTdJ9= z90K06u9%}XM@8-sk^MjFxjlfj=7m>PZPiTh4bo&mR}UNg`vImFAlh)%4P&@{MGafxNrP+Q-_` zuL6oPRfD&H=Ib7r=JM-ryf~sdK~A=+UfED-_5&HC-7mcTbi=#9bTN?E|2mB#H;(Dr z`T+L1sBXNc-4&>4ZS87+Xd<=t!AroWDKcux-t1%=asGNcb}Oar)X?O^w+@`~z@|@R z&41oeq{;2Uf@1m)t!~nsC#RCz>3~7`+vLsgo5z|YU~}0n`M6-O@E6^7V-Wl3CxgV$ zbE7iUWAOMiFT)c7Qo~)8I^#-pF*Tgu9N^q(di-c(r8og|aA;RXvEYN7JAnU_E!7bs zlJS0A?c%LSUjbleu3<>i1|>t4>HZ|2g@dR<;a4ZNeUl%xJoj%BsN^JgbEtm^U32Bq z?M9e0pPMHjuD~*Mj7-5DcJ*b+OMi{i0k@i=ewK6M@(IF?$LwkZ?794#YIduZQX#PK z2~&TxeVj^uA#E;7OWCUw>&ms0#kg|^$Tgqt622rXokhg#E4Z@i1Hy;#%D5sHV{)R1 zC#1{!O^;TN#CV}pxAI@B&4afQRgpuyefd40u&&^mOCm6GEwrSGq~fVQEl&326x!RD zK9m4N-krwYpzKxpFRK8#%{KYZJHE1y7hRc?jjE=Ph{JGhL< z?zn(bgu71#*Dk&p<)SzRJmIa64vLyBoFpO|H*~5HV8wFH9Pc|`s*nlq!}woC0OrQo z7dL%t_ed?3GG~yE$GO|>2pERL!r0fDN??n+`(V^>i;8KEE`*F3m%P4%Ii`a; zoTQ5~9C5iQdrp?PX$!d@Z%}yvO=if6vy140LN z?!C2s%bi3-V!y5h$pG%%@w3AB+=vv1>%}&SOD#;MJlog!YaT_JGW%JT;(}N+RN%&? zH!~?Cs!l$G^lhpEAJ2A3`pC|r8)A5s}tR7i$v zEhS$I>vy0KB091gCljv4gZ8_jXEw7+mDSD#+#R80hv2Uf9|qdgbql7+VO)RXpW5beMBi$SxAp1CtWh!cN?$6 zRO7K}^1ce(p6M?{oO5YIDi`Z+f05y`bjazG;&s~bzSHFDAQ6ka<+T3X^Kr=$)Ru|kg3yigbk#wy5%=2B!GmDm79f+g-~iH@>7kr>NN1jc6JxjxQ(UMdXph~ zLHX3O$WH+bN|{(_7-kO6#_2nfmXb6iQQVD;wzq-6d@g7*b#ff#Z}u~HZ4>QYRDI!| zi8ENA9={>1?8aJOyaTe_%>n`qne==Du9LBtKFnw5w1(-pw#6Fo_p#(J=b)ije5iM9!B9c&dq2dEaB5-m z_vf^uL(m1!tH%ajqPQ@UE;Ta{-dwGVca9tF0<6s2C{hj3Q~Y>|pD{1w#{5|0Oc2ED zjAO}P0ep)L)-N}*JL>FZ=mF;Q7W82VGS4*pBLU3XHNA!3M9EaB3hpG+<%@x)SSah^ z`w}9c^4a?E@aXXA6J#9MmAz+^wH5X)M&4D1+u^BVS#ifDN;N4_OwXY^o_zV70#`xQ zfFX;4h{0CX$|d!Wp)CEVn=Nu6g47NL(`@SupB zw7`Qz(u}QKC^Qr>zD$Btrjk4c{ROAH+}LGWn(FS^)Sz6FzCsN7_gu5a!y;= zb&XBnr71ADvUWXA_OLThcykIgKXx~nHkq0d09TtV=Ed8q_3(&w*Lf((NGpeQ8JhaF zr7^7blNr7}w-pKLAYrVL6F|V&9JB5oEjX}xJGU8Vr+y%Y5{9hv&2VWeNBbfAmpp=G zrX8L3ePu2MGs+>BR`7sK8z$rdBHwe_D=HF8E^~e<8l#Y_H1C`iXH0TD-38n!d-TZ% z@hyuquRV#_JCegcZuro6bXvIr^K6pr;>P080E+DA9BB?^T2I*S1MY!BT9|snn?#uk zOIB(abBZ+CZK9m)33OCa9zV=~yU;L;R_}MDW+%Qkg&n0oE(C1nr~h+(@qFJw6G(<2 z6^5w2`)ZKG(=SvHfL*9?`23?7utwqY}Zit^9v9sF%2*MNG97{!c-$nk_D%hXKhg|DS3#tV^T z>@jOk69hro^LEihxY!bKbY;bJ4dF8d3Ut8>(Ljr0hLCO4Jv(P8hq36^3@NyKuNmn)&E*a=03N3B?vAa zcguCl-H8W`^J~`9D<_0(9v4j4fnA%a1DJ8*MmJXdkh=pxS8lhaWtsuIn4m>vCL*m= z@Y%O#=f;7qqd*aO4(=z5UJgDA`?fmo_L8wLxz5M7#Wst3g8Wy`By9qGfA2bTUyhlC z_}{*r7lqK0nJ?UBgxw+fb!n0bRX1@f;+a%i(isz#tq$qsvfQvUmMnJY(tXBF*xJf%JvP##2aPnVISe~Q2;M#~7dMxY7 z+ADI&2>d4^4+ISsT`8yxyWU?ofNcV$ThsQVV=h~7yD8))APssU7+?#6& z?`vRms>u~^ahgJOb}kdgZQ0e@c%a7QJSh3D2J(|JUl)-2QsC46A>afI58!*8f1FJ> z9txgfHd=rt8<(wat*%Q9hTyUV4-OPl#vm7?Ydppz{NqdaSDw{{J6%2r7=3O{gTu^E z{Aowa*TH%;XyVBDkd}-;mmiY0)1jp&rkFeS(FsLRe|?#EPkQLt7c);25bnT_g4hwH zG6Fc9*Hc#_yA(7Pjxyu1Vw~;Y0`l_IeCVuE#>>vZy?MQHo-yz=&_@Xmv{4-x&AAKC zpK5%fK;e6Y8H*b8pm`?7Wra#< z;U6JFNAc@KoHqN9HS)R3R26iv4w*B~RCXByH)jOvGBNAhI*68Ig!{{8zY*|Vaw8>3 zs$&v=oH57HOHNOY?;H`U1M$f_A2$j7Ey&A(i7|@Nacr0;><*h%s{%d?WQOwt=fivg zV(mk*mIiEh{TBD+2CRB4st1C@y_dk9&$Tp8#`$?%l?w&}%NRi;SbcSlfEAN0cs>to z3XvT6T_gYS2oob4GI*O6~I7 za+H!m=AtkPwp}*0H1U)R0TJci`{)bP=iWQONGP`% zB7)5~xrszdwoF)|J#uXScuIz(y4gt(GmB6ph>?>Vaf$3?ip%;bM}EB0K6ycCFYpp} zMEDa~7ZK^nxpr)?cI4efv`u`YO#@{Wp|x%eZMC-{BQ714cu~{+%8i~pfuQup>D-}l zlAx4IY%qMapX=Nl-(Va?>6+VqR-y2)ypJMf&D!NjekKvS^U6{o^9fp=ED+zm#19cL zdvo2oA6pfz#{=K1W$oxpD3{wmwpq0N`vd-;pPf)FX`Iz0C#uS5ks4(!Q7g{fu<-33 z-c5kRh9p{5-My(736!Y|Y>~1B#|CqXJ%`{2!@F{l>}^cX#0eA^PaLl=)h@IT1Xav{ zgFlMZdf4!FmYC7BA;ber6cQq?I<7kqYN+iJZG8>2Pg-B{M)Uf~F++5K<83b#n}tnM zUU_xWkKWDlTylQ6XAB2cJs;@|TvO7N3+?=Le`+u29}(^U_b?T&VWMS`TP7ydap#|#`eOv>4OF0W9J{QfkpcH6=>d$;e5GjWApt z(PYe%0;Q}E?xe7rBxjsZd7TX#Ut;b};}{b(wiHoZUf@DUe_H+zEie7=4mclrrnsx;95w8Mj|36CSlC8(5JlC<2Zz?=-Er|K@U{8#}yZkjli%J{upxjg6y% zJ9L(Uk{9jFiXSXPq5;#Plt!6!-KSsbf#b~ZS=;5OOmY1BLx0wq!le`WTXb?8c~Hk? zMBt=`S&P$9(}2f*ltL}=tW>&G;`|ioh`=XW0`^J!O6aNTm(ma4>7Xp04t=4ip+Tnr z1_pJ9%7I>!cs>R4*-8Pyl4IZex4* zY7&VIL@YVYlZR7@<%o!pBe3KWigZ8fKC2_ZFwLC;I-t+yy^~EaiNakZj)R)s$pf|e zHC@dF%$PIc*9ej+bxYThukBL@429+`ompV=G}JNCzVW{#<9gM;xJUVDb{<(-uPpOA z-h1Prv@w9mfUZsA!y})2Rh;&a=~SoVh%^o4+gtT!pdxr|bH<;`ki~oED4D96;o}+U z`CWA%gNtED{fM<#ir4`Iz`Tlf9qY);zZJ5J&K|s&3*G9xQzrZBKQsU3N<@fuA%gPf zpZsik=7>@f15R``chxv>!$GkGAC!PYn^p(Q1|QN(z+RV^yND?A(7Ijlc`Jd*b$2vf zx}kMrKyY6>#%xt)a$KnS-nz~;A2@91ap*!E%haQ$*#snm{m{zy#<_H}@-IxF;fYC4 zW6(l7SE{KI0prG6@8J`V&HzU-J^RLx$@|9YQ(Yvi+?*FrNXgSextS}#SbwnuYl4-N zCO{RG_ZsBxm=uUo9V-!V&ji(< zfj8;)_9AomRtJei>uX~C26|xZ)c9t9hUKdXrSoJ09p+8@!nujmIyu^ppe*c_#a!e1 z?gf{1J(()|uDw-}v{XZKR0-^$l<(~+?(uXbV5%Plx4dhrsc{Gzvikid9qdc6%U{W{ zT?4LA%>s)t#4f0z-RT*_5M68=fl?~-DAfi z$bX#TCrjrD)H#of&OTH+-lcNq2>D<1t`i1PJCz!}Gz`w%i2)II?&GW=gU6`HFT>r$ zrT7Q4`K2j9Pv=PYAmj*LuDtf^d8Db*^i-jKL#ef_BAME7_>6nBdyrBB9P3UH>w);xyJK=s_)Dr~PYYBX+0K(6h}Us6mrBGdL!(q5Pon;&Taehf5!JCGX4rPW<8CGi6#+K2C zQ98Ge8BAl4eXK`@ZI-nl7N{wT%H+5~C*3T%&(8~ooGiHCr*$?gFSYea52(~!TwW5P zF2F}o&x}!x7R~JK4Y+cQ=LLY+>^*6t-pAih@D|@5qex$BBH`W>4{VL*0nbLN?Lf^y zEN%>0VtgL6U+xC$v}m&k7`J?`y8V5By=CBb1>ht|mYAR(cjf`+t);y0q<>TPD5#cA zfAkuWjR*0h<~;;Fa^RH=V=ED*zx+;(v%B`0$zbdqzki_%m?K>l!LW!jZr1Mr#pJ+& zzpH%B@bT#_9gfFr3ymmCNEfZ6qtQTP-05CFOsZqPSK#$a`{mTVPgdO{K4sKU21on?)uO-UV)_wx@XB)8@pBblf72x~&o47do zmn?zF5X^iwX0+wU@D>qfqBq#p=o(_%kD+^!uJKwZO?XA8?G_v$ze(!QFJreCj3gkZ z(r)3L%gE9WJ<27p8zs!jo;=pulOS}^NCa}ny5B}07`?mKjV5H=qUVHH;T?M%$aP`6q zV9=TV@p$AK8tcIO@>bXGhe~$68o-|DJ$tEr>lM=~Z3r`#=F+zsnM&3-(TaHV_sNy8 z&c%CpOaq8kw4pRBLPoDMUdQb>{|BwgA^D|LsiU;MMHfFrh4!7cwK;653Y4MyF-{8L zqROW6)HV7v=jf9Jq=K@);Nj~6A1>fXrji#LL-snq@2U@|W$pqdl|#uvV-Vdq@o4c= zmmik8afRzb57zr&U*Ih`brf^%g9_C?a}P;-GV`kGyIohkBqk3g&BGy@vvmyP{LaGF zbAv$lSxG$o+}C-YCH1jN8uJM6WcC}qp@F~KGSIQ6ky$L2@ z`W%dxUS3WPPMcFB!U^gdcjE8wdaZcTAp!ZiGgrz(%FXRROmn)2CZun{Kk4pp+8dG^ zCqcviOmoq4QN(HV@8fOW$gVF(TB(g8{r>q-&vA=6F}~b#4AD4iB_POpEj!$9Ugv?9au zFF#}7Wol|1FoKjxr2{=*x|<7u%F?`lANtR}Ft>mIr&%IG++YY&?b#zT3@m~%O!;Gf z)a8dINtBK`LG?2S$4fVk*eJkPiGUY%U18pP27*>W^eT0)Oup;u=ma-zc(6L@w&3-V zGeahJ2)I{*L%5B0M@3SQsUe_ENDOOe{r%c?mds5q9k7n34SRbqm1me2ro*a?X*P88$Ae_xk{&^1g0E zXG7gE5f% zHUFt^rDJ3qU%2Az7AwVq(D3(WNDqyFPLsR$mMdPHfEkegTF{x_}RWaIC~h5xBCE>cn&UFkC$L!=d!-}9_yUtHs! z&~1MM>J!Orm0Cm-l6a&N_6E!BT+6~$37zbxM(E$_<`ba-=r*^^L40kH_xwG~Yo@F3 zgP}^waK&%hH`Eqoq_Ayr{?IzEF?51`{@>j8GdEUimd;%>ypu->c=7RAbR4~Zhk&FA z3sPzT=c_R5`;qq#vnC*XrxV?-0L`Y%N3|*!a5w3fO|-usR3P;^0cHGrcYicjlF3<- z@h8EUy<=D0USH(Tx1$tBY3~+$$a7}sc#StdBqP^SFWrlZ^fXd%1e8i+@m^cC zT(RI7ky^0%%HfnM4eXyq&EKwMeZ4spG!L%6YKy6Zsr=MrT-mNY;n|>{B}~xDzw-Ig zo3G4>{?xqll-9(vaA4{`QhRX>KIm!c1$_(@UjsU^+P4o-aG$X&$jsmIMdSTP4{?#< zML}K7FIVWW2PEEdD$6kF=kt0QWOXy>jnXRe)kVvGJ2J|m)UEr~@Z9ICb0)O{UpsG) zTAi6b@U@*pVAS_Iei--I&v@jim~^?Ou<0~XclG=srBZqX$3X~bSHQ9IQsWC~7;vn( z^Z)muPV_ZFf%L4C!y_$GJdC{zkD@Gy_iT69u{N9+xlZG%#Y(yg7km9RWNkxwqHj8r zF{k79+}z|@F=_$9_Nx&$h)8`$Yy%;Cmez1!m_V(Nmv=kncJPg5@EBd1Qbj46o*tfz znG8)22E#QRJ{&ODzg$pwTTA%gk-quLQdWq@qm2a2d-^K>zwp6@F)sHOpeUrUlnK+LyywRahD?4-I zuyyixkWlLj?>$!T5%TiUwz)}dpZvj8{7wHw{yiw)26xd8S~GHX=$&qeqzyS+arx5HI{Tpb3^DStAh6vf6~62i}OKQ059 zalyM=)#OJ5l(&2&APL^_xCL= zf{S35mzuvyD%*kQ`w0v`iXQB2F`sShE879OWi00ka(Z4nSp^{|&FGoVMZL+7W`>6h zK#sB5sw1<*pf(eG)cqp+w04 z&C7c=X9Mih;SHz!6a~zyqZa)p|g@OH8pc z2ZF5o13xnMKFi%agmH{{G_MS0;oWGx`14{fD+ziFrkGseFI_A}>d)XiW}*Zw-L!$u z;GAQzW?i>|#xgVBoW^fQDzZag0dYIzZoBd1;5(s9Z-Fv_2A^(=t3i!ds!QSfwXsj@ znM$v=0^0Dnn23}BmjIg-5*%LgF{18bJ)p=-=>+tPea_v%MO3j@+%iDDx4N{xqJH1g zG0dXU##|;ydzU?PztD4dwm^a#UfmCsYUFK5@I+~zQL=ZN!&S+&G8t=WZBd|HyukF{ z6Lgx`nE6OETFBEE;+`@>4&mP;ggQV+_4>Tqjiv`DEs|q_Cf}Yr@+nQVUr_oG$SDN` z&ZEM%7^bfGWXfg5`9`bVlapZ32JNq-mMm=>e1-jn;x^4iA_ zvENH6pW~bL`XcZ$mv&G)D}4DtS1>w;?L7EU%7!Dw)mYaK{fm8!{sT8|lX461TQAQF z>h_CtDniPILPX5hyfR})kn(XE))m%yt3QTe@~P=u2ex=kQ`L^mKJb-&`Ig_SGWT}; za}3JtgL_%BZp76XcBT)H@&BPjzcX#zD+=8Jy6oh#`OFntrexNwsehJ$Zi{a&;=6QP zm3dw$0vB>c+nDv{JxOh|7&v}+*QNK`W4`tT3Yl+L3h-ds*z^6k)MVB}bz>B9@0?@5 z-+hm}6Ik5GkW6z}i8x#Bbk5S6OufUUj$&QQ-D}(&--xvsF{eXVV7*X2lMlMI zpn}I7PHWwB_^G8x(SPLR_`Bl38PLT^W{VErvvnaIj{R^t;_=Nxf(@mhF-$J@g#BSP z1twC)x8ebt+Ua}$BZ!3To~zhLi=23v6LO^SwT9^#0fqm$GGGL5V>f|lM@{{4VIRq! z*py4B+X~2`wRU>h<4!g?g`a&uvFj8e@*$gAyD#@L%sg2Mf98M-B1s+Ji;B!&vuxRu z`TT5zb0*xMrHi<15yP^8{yr;hk00F4bMoGehWRX@j&3lO>XdpG)(*#?xQ^UKDS5vp zA4xL0xx@cF74Yv*(URem9vL9~%~1yIEDCEsKm4qzAj7K7dpaJVaW@&RdhG)zDh^y0 zT+n*NHvCivm`2M=f`4Su9T$9FV2*J-$Q5mDvLicxl^>(z^UBx$iuYbW6KNDNjRoaJi*1l zxlfLaTiF=vZU^K4ds<;`*<8aDVcsHQCp#xAff=P6-lZ{=38z*MEQOkK>S9zi%x5{% z6ulzaO9)_fVF^oTpKAT!`08zxK5!@jKcn}yl<$QWsLj?9{cN5IoTR;V6;E9w+Wn1T zN@0YUnhz3dn6WIYEZlh_mOFW5LJDs$LP2=@83ILjU%BWY;ZCZNKrzC8*J(>*NiIvk zTaV7beBKBvwIoQSynW_T?&Qxn-dE={`0%)aF=)NFe z>@0rzJh_eXZhSD}ICdLzwx~}k0lW6z<52!N+e>A1I@_bgL^6pIuiyqG(QVV9tC>10 zUNQ@#!^8>4V{gA5;CPehz+SI_%oy2@usAc^(Ezbl#N{E%XM{X@_lZD}!ol6xLOV-N zYkChnbAqDzUirHh+7TT(@<1@KUawwVL@aJ#}G}mMIVjSbC=H~ z-k$)u(L_%($GnMGO+FPckanKp-bAJb3(%%Ae6Gs;XA*~~Pp8bGOl0%!JBLb#vLNQa zFE7C26Dq+2o6NMHHqS=2ITII-^)2Khmn{Pwaf|eA`1IV1pxSw7)XQV+hPRjjGs0f3 za!n=I+uoz3C>SJuwmtn&-JxD{igz#6gGAck)Ih+2|4i9Dzq`lq8&GjhD87EkMcZLj z`VxeNf`)H!q;$U}Aa^YYsiwenRNX7#c|uBu0wUaFu(9WFK1H`EQ&cCo>(b4A-*DOz zs^17S4F|iwR)j@f`bkksMEu!bB~j~nMiKgj36y;b1(kz+4G+&*cJd?0unSkTuaV7s zJUfc3YKgFX*p+=f8yu#NRsUTCUI2#R$AbY{(M}>5xFnDFkNv68Ot&Hvb0C$E+@#SHVZ&mA_6IPThRBtP{$7fNgWFa80c~z+o25STE#-K40h8u8^Gp zx|m$k1YJ=*5z1$@pmOB`ciC)N;TBbH<)shG7Zj!xiI}!1 z$)NP08$UcjWK6`!jf`w`u3yO1tipCRcB3@TN0f)1hfQNif3s}@Wm2teDVH*>9X%Yn z%pQXRRv@aqPm;L=SO%VW%Mt0v;mzEVIVNxBW?WNVf_?B4X+JnJ@qWq`KgdeY1^9n4 zlKFqFn%o}^8XEI%UO1aPAU*$YYFN#>o+nPna(azfx?yhz(;z{~(M0PjzTap&zekogfjY1Z|c3oC=x;3m;BJA&kx1ZXCyDdb5h8Zh-`! z?OVwx$25IuqlEvjoAh6M7C*O8EA7$~-4DH1dsD1;5$I2ey1H4m)N8yxnX-&@#I$vx zm;9)Gy9UYBwO-M3ayw3r>sO!Hi6vuMI~1WC_nes5 zpHrg%22q=p6gN4F0IkMbmKn~dodsO>^Vq{;Xv^lTKsS_;u1}siDzKxI|UN* z;{)M2({s9+7~t+})JcWkeM`5_elsPvl#C~Xh0M;!iSQ@8(%(uC;xY+@agawFGfdU_h-!uyTM6E0-2VcN~-12Vg8e4 z7+>S{yyf0Xgitd7N+xcIzEYOm;gJ5lDP`a+h@T-06G*du2(N-S^9f)U=vfH1S&H_h zer5zFMa^Pr!)p7t*K|q-u0Cl8<;4bEb5XOt8u<^!t_64`57-Dj&@YE;rmU%B_RpG% zXd8n5l^t`i?a0%U21||JXKf`gyQi|m*UiHkx3oz>YyIQWV&YucKYvYXXC)TaHMoHwCmVV%XY^Rs!4fC_3&P|{U}Y+nCQjwJ7HeEuZsPU zlkuy>`aShL zz3uQUC!tZ{Z^N3nE0_brokO^_7*bCK+ANMzJtQBSi0Y`sCDA!I+^> z+w)P1Vv6os#po`syA#1MK9s9!o%P;c>PFfaPIRvNNyC$>o6TSzyLv72(zE>Y%-K{Z z-6xS_H)a)z8{Yk}fIaYz!@AP?K*Jd32|Dp%{omNt!&3!9xof%Vx83A+cW7LLLGzvf zg)EYZc{p&bV&X&lh(_f@vwvY)F!Xp;T8S#fVCAbj2(-xwj_gEf159idGbpkLRs@U% zpr^^$C-tKtep;U*=dfF+A--3S#uiRk2U3yHKjj(Y{+@vY+dFfW;*ske9P__7x`CJG9@#FIUz6r*ngz$_vPvo$ z;dYaG*`(i<2f}xYWvn=)v z+a$?7nixTmuB$9{b-;2pJMLqaD%27Tf^Nx{HN4ZC|AO+(u+)qAQ*bexVS@-bA@)ci zGT}V-XJ|ICl`=UPoqA6v$7pz&^iSO62*n38L3gGW=Jm`^t*~><{gr3dV5}^YP9~ID z9kZwWOmXQ5O5G9Z8EFgX+@rXLr`giM2DAi>lZTFE?{w$>P7ugw3A%wIc{iAM>U-!) z_5%Ce6AKrKe<-;d#0Pe*PG{yY-pmu}zDRPFM! zFM`v(=LD+V_T-A5c>RpP@>HKidpvSed}RX#i$|r}$=9xb8QxyhU%8*U6;Hcd|9N=_ ze6F$FarLa#r$mzT7|!?(xtO(>cE2B}tncF{Zr5Um&j=u^WTCr#uTaK)wKTzxgHw{o zP?CjcQ~|Bf+XGE=O@qtJaKS&huNcMEAF*exJb>=5jJAH;TgP!?Fe71%n#(OsEANVP zaQF1?c|@7%KmgMV&vn;!H!?-t#dA;9oWpWH>9;XMUz|Xm(%fyWN!NG+c6v%#^>yWQ zR#PxdMfg=pXV%Qh4+Nlehhmb1OAYw+ROmB7IpuRxW@V;DGw~1G{=X9;|KhM#yLGXF z-|?g=@+Ow@fv)DI5B>V>WuWK#Hbc^l%|KXsb(}!E)9TY6J-;P`PpTTAAnPZWGaWCN zRIPU(JaK zjj%pMK%neK&U0XlH@J|FQ%Cn*$mj*V6F+zJjquqTld<-NnVQ??fByIG&JUdr>XR`x zFN$BE1kR>c8bXsXkI+cJotk&Y33Z}tb=!-R+1f!NFH=g-+K$K*F_S!AH;a~N+f+2g zP=-21bsggB5K|HJW($~Eka5TYkv<9 z3X@v`#&o|c-{&DdXorg!OEbl1j8ajDcQNOn_@$FDYI|jcEk>du!jG>2@XOiUn;7hv zQ1ur)+{p|X+?Psi=XXlu0Lo=<3?as=J02OB>%occf=AW$RkM94wLGuiH;D_xOA}OD zp2!}ooQXbX_aV;^2B0XouhzcDTt=NA4C49pc*!`bp!3cgwZaY!3KqaRvx~;;sacX) z0?ht6QdIg^uMDNJrS+xHhrChIE?mwsC&~W=hIEJU`^xAm{Zfsg>FKw(BYKF~@{;~9 z?^tRT`J8QN;YyXnw8JYev8ko z4Fs_@-u9H&Cy}xisp8XR^M%gcOP7F9&Hn7W#_?}2^7cPPe_N83T~ZpZ?EJPNuO=6a zQr6ghQN0^D@KcH2?N8gXn4=Zc@Bh9@m=5;wdWj5K3WLJ=?to-JvXP}|#oX8j>~8uZ zc=&z76}hC+{mE)5E)euW@LKoLKMl4NypAf;1Jd2Sn|b_$8-yG8fPbfTnWKK1BY0ove6P75CThqcWWJy_wAiC*?Ep7f-L0CA-M0M{U+s&LE zTol~29cWg56wvDn$nh`pj|q@q_R-Igbf(I&HFRqj6_t;#7TdIP`B*K}{#UnSpqtXt znD7vWZ|MgKBkEE~8krw#umRp4CuA0npr7cZz8_x#E%&p6tVMJgr{K7OL`f=?H5lPVUoY?w+sgqx=qXD%??5{*KtW`gr`o2iSCVbU(mT?w4N&=ZF^q?Gyx%BP2|GCGf>$E^9 z-CboS6Or%ljViVzG8DU72b4XEw*Qfry%$5#pUd>%!{H1SwF5P90NdsGeBqWhcOW+j z@x7Pv9ZXHji56A(yL`Hx2-bm2s5mPFfg7f_G`TUpA9Yl`0)*(@z-$Xkd#SE zx<7RaWQ6V&=c(m>bjZYrb|ByRK``aI=mp;kN)I!UNE6{0A~5FOBVFR+V}n!U;RWW` zT$g{a1j-Uv_`c=E+D3K67;jc2U@oVqws!DqvQ)l3_@9}f?{Rz=(qW1;UlulEg5Opk zeUNgg!czseqsbKR$eWkTaPVxUhkj{>gHh4W1jOIFXC+JZ?lD1N*?dNgCm>ikwN=Fs z`7l$UFHvJ{n1^$^YP9xP!vB)#>ElxesaCS2$M8cm*N!bG$j5AcEhgZmnTDMoPk0O3 z?1}4#%8T|MCp0l7tq1oLvA*$U?Y3G^u`@XBEC~OcBPrQ=>W-M?KESwGA8SVrt`{l) zM?*{G%6UB=Kj3V*);Di5iL0YTZNV$=u5t_g8!br!g-F8*9%e8;b8v&`BxVLjY!eWiV7yxU6$@hW&z&sQiXG0=c-l4{)Nf*c>dc`hHg5>m%wtLvV)TJ z&tz3~&9oU=z#f#ro5r?8Sr&n$R^JDDGLn?svSS;Y-mE_0x^MrUrc}5vZt7tVmcB)%wA2YXCVeo?((g z;_s?~KKz@Z{1!LILRind5z-T!yPtpYL7OPCmQ0QV8pU+k&(hiX79!oDBqa?eyDm!*OK`^RWIw@{tqJTzF9`7C=+KwdRF+ zr>6OsPDMgye2Vty+X;cO~2>Z~s7+ znB{xHO{Qnhc79!q+vMDQQ~z3iQwBD-qU)DQ*tM-V&?ZyLT0Zia5IQ7N$qf)XK-x=8cD|q}@@c6Ib@n6B?-?*;+D|q}@@c6sy>i-XdM*%tLvDYGe z`#h-WU}!a+|0;VWdnIT0HUhRm`U78bzU0h-sN>J?8Jl^*A1*{JU=}bdgDdYaC72Sr zxL;sDvJF29zH8HuHE3r(BcYdg1L z!)L=6tsbpj>hKF6&TWItCs0yZen$4##V7-=7hEp{uL@ojvk|kA5&VT+liQfbekwEy z@?x+!mF1SykkoL!=6X$Xd2)FQ=@)JkY@-OSf4i*}aP0QumY&zq+bGyZ+cuVd;SJ9= zc!8r%wz}!5zpm{$?j75ZjFF6yTb5f^_WT9a{%wq5C2OQpqy67eeOh^U=-r_qen)=C z$)U;N$4=XDJ~wi1Bou%l#XPq3?W0AWhRjfNkj^nBjQM8o{~bw`#q=E^SG`RF2%3oqsPep7+6+p(rSracy} z7Oqw#RwUTA_PI_X0Xpe#W&1;LDsGEq- z+@e~in3y~wqQ8ueo2%b;V#Po?e&+ax_sY|^ai2<~;=nK6z@1RKsF-TPQlM!`x~(wA zf(orSFaL>iH6uQQ9-((-1!BMJQh(Q_Io&g~6J5HzbhQc^CZo1UmRHC6-DD=#U9#6*a`2yC--+V3 zm-CW3Gp9NEJ4-i2Q1L8Ds$nO@2(BW|ZfCD+wOL3uphK=<^Plj)PWrrTmYFB+8;yb? zI*{iYcf#ma;<652TsywmW~h1~DxzNtQvPq$J=uS^jCt}4!-VHO-gywi^QIxUXS6JBpZs5SF^W{vau=ykuy%TJ0F8ePFwm3RYURp~{ zTuXgO;YPO;u7ppttvqX%rvh}cH7}qdSK_w%PB_0aesGZCN4dR4-8R{mZL$&mrQ)4f zd{6(XTKuY23DvC%b5B%M+m~P8iF*nw8pfr5++Bf2`>bhEVV7;T6X`R3eAm2YCNHZQ zUVW#HiW?oRn>(SmUln0drBMvLT>X4<=vcW$<Dd?UajPBw1oJ@YgUGED&WV8$FP3y%rRpYeADQXb09m)XFHsO8d`+4qPFx#K9+~^SINGf5t!bzKu#4EhouWN7s^TCyM+% z=?PGOG`Vl1kd32Mp)8%4dF_?`xb1@oBPru|D&iao=I@&|k00W~Gi1=SQ*f=`ue#E| zVf#p)W{Jw=^m)m^55)eml|<58wx1CSm-rrMM?cgM2Hrff|N6}GC|UmmL%;p`F4&Wy$No$*juwSs57G2Mv#7}P>5y!e z1Ye^p95IG4{=WNO#NxD!_bhc$UcN0~ zXRuHEpePxoKf{&E2vd4(OS>K%y?ycrfSY;;zFIzdQX+V&{n=b(>8Yl5NyYE@S0qh_ zx1R&1wT7=A_&lHJdt~Y4@D>5^A!5Eo6;qo}yJWqJTR&EK05K`zfyiVvthqS$l4z)hP2Ws($l|QRjrZwP_ zkcf)b^c4ZY_v=64Y`l0V68eh$*WGqkExF~U7X$94t#7w^ex;5gLb`2hLBHV{76kCA zZ|?1!*hd`b#K+vx;XCnoL@>Jm;P`N#nzSZ@t*zlW2~u1s zwig(?w8VI{_(;i`{dj8JO+)>MwVUCmlvE|hj>!pbK3nQ&xnp1s+~;WGR`td^_Lb?L z^U=yO&1_)CdzmM;hrsmXs`-$MXJW`ytOe!*a%M824mWkq=S&@>jiiuK{!Ue&J?4h2 z4;VREN5||Dv#q-(H})~K-laIQ&*!5;PR6yc&ph7ApI-1iLOQUBMzg1pdR_Z8Z10yu z)K+?~WTZcf&T~d&Ka$v0sP2w*fJ)igfCK~i`~!-gYF(_#u8G^czVG}ji!>dT-i9zT zJGiCg393{msin=Okc&RCmr9!ONW567ciA0t-4n`_Ix9!AB{F=WA4LDx-j2TJ-B;9f ze~gMl2A2uFHP9nXxNsz^C}jdJ*!$a}zw(vo?twJC3U;+9?R$$PPUx@Lx>?8Kd>m6> zBD|lnFC6`qd(J|X{`|f{ezH!yd>bk%G}MM?)GF23aLBfA>dKZJ?>EuT7E5TcRO#S!_>8_#ufConIC7 zZNF^?M=E@hVLNZp?jhAWdGw;%ZQzr^T;Sz}vw=!yNR=hY--kukt~1KwUs${qg#L@O`U1BF?CkYi}YjQzG=}X$Z$c;u^F+)$KH@f=*gx+}&5eL%YSh%&LrxFky zNK5?Dhew)TQb0c=oJfMVGn1a@SNiPXW#8v~d1UB0k^a_^l{)J1?B^ddv!5!Q1}mJ0 zsy$khXb*FJj^YoZADW^0y@%oiB-XsW^_7sFAE(dWFL~wKryj@@2%vVlP;94LN>N;8 zM1dZo9wlL}(DEsVpNC`W@o3Ov)b1f6=Ff-(4S*i5Ip%75s)_UOv$BhKe?_)hcWakx zva6;7-+{*zb0Kro0%x;;7OSz1F@Z+f_1{~OT9YjB;s~;hspf~A0CS;xNsDC;73g=@ zsDGz+)lNEJDfII_?gI&w)*Al!>IFbJ?psP_!+lTMH>BsjJ)^5E27aWjHLS%f**Byf zx*@gLEA_ZAYI9kr{R8 zYsz6y3eL1-NxEThlUtZ>cBP%_g!YIyLeJ*NV8-FruIgd&rIMC+0?b)6&NhSTnVnv` z1oXz}-<(wISMdVih2EIjZJNqU zjZ1Sr5%eJo2!b;hQ3DlE`NsUJgzur|5nLklSL~%8GW@O0${}{&g|79f&L$Qix8ZAk zL#x*!os+3*fR{&y`j|VGZ%YdHJkxI-3Ek^+$tZG*&N+0$8j0t4_E7DD$eU# ze(6FHGd<13oPtrM((_5|*Tenlok|1_Yo6wlc-(!jey`1?%Ux7c7WUcNSDDDKD3sv2 z6m>Hn4+pmMQVnkF=!cB>1WnK-SXfxDZcl}HNI0*yZld?4cYMA{IsoQ=Nu$XR48<verB|(%qqRUgVv=k^xB2(JG}&M{SqoGRBMSx} zm9C)orR01^MsCD9C>;QAItiu13cy_6ilXEye?59%Vhpu=^KN(xV_{wlh6vU!n-1Y= zmu*lM!B+%eW)Wsw=D<&mbv&Pb_g`;0=Jp+E;35#DD_qj?5_L4nQemWaIFqg_ zs{38(qx8(Oqk%pPTGW;9^_=2klkTgT?CpEz{9ER{CKFExp46&gJ4}Nxi^fZEmJqw{dy;G*R2`XzW{^reW9e+u5KAODZ3Hg1031a?lbEaybYeYEWgn7zkg}xbad(X7Y2H@u385DlX^uRD}tgn66AIs?5A_0N0*?|vCnp+>BShd# zfVyn~lnIO|AV8VGcm^vd6JYrF!azc)P~ih`CIC*+F9u_?=U`J$Fyq3p_K5VU&bw=K zHd813q{LLs0@cFJ4XcgPhrXZlLYNC*9nxOGcIe&kF&8^Gj zp`Qm?uXs4@f2DYtu#o#{B8}0WpG@bUU6jfm@m%NC57MOJH4OjZMD_!?@V=x&%di|s)xs~Puw(;G8wmxtW0g+rruTwolg$=MA3g^h%Oa z$!*!bDmKcJJn~iPcbUmBVFR=an$ocWD~20K7&jiAN7YVHWhSvva3xAy36e!)wF_B8 zstH1rC9G@t$PV@L=fn(*7cSk;R>1Y@lI~NQ^>suV$VA7Qm{g{;D!uXGx`&r# zuQYm?PAkbZl9_xf=*>H=?v2XWQNu(7sUOj*Jddti5)ws#;64M!y$~bmVp`x<| zi~12(RZZnI5+jEku_nPs@&PiTOB_x2E+t4#qz%I-IgEO+8JDfx4~eSq?-m;0zsJ1Q zWsor~pU-%f{r&^lnddC~v>U~#TlI_J?3;eB`OtX>!-Z=<^}3W)NXb0O&5eN?HaZqxk_06rqB~z zV2aTcC#<^bj9Iidj3zJoq%#vWdOVX5U96-;^;c?t{wn`H@IDRBWpuGpdaiMSnr?G) z+jaZGW+J*+dFY0uj@gloVh zbm~kg%JZDUPotuo&sfGRf+)oX+%FL7RgT%P=|b0+*u6<8oTU@=EZ8dde~oZQ)t7iS zW7(csK5Lfhf1PLc;CFo~i2gNLt;F?yJ@)@*xteNh*;b9RQBc*BlnY;;P7D9MWyt&X z+wb~P5AOkR&fcC5vDe5m|Kq9vgT;#!GxpQ24)jFUeEwNmUlbRMfrsDPXx@iMty4~E zr3(kLJ(>hSwG*DbWuut7Y+|_MTVmA?O&59G*E8Y_9-&=!788BDJzuQEGZ*HFQ>CIy z^u(b?K?JArkg_S6#@i-5{N9|z!sqAkpR&|>M_Yg$UQyL-ki)ANcoOK~oV3Nuax_F0 zbb}bFZ4ZL8=r*OX7@!7)*PXk5Y#zPfDx7`dlz_+8-3RqDnQBl4T~u8+jT~G{I2i~& zrn@Re{I`|wi*id{RTdPy(?KFnU)66Ae~FvVREb$@j>TVWHgx^o@Q#cru?OCl3ZLp2 zE~?X7H1v~M%ldyeaWek^* zgqr#QAPHz^ZB32&T{r0n&U>#c=TTc_Tyfm)cilwVut$7T4~*xSfHVCI=B5H*JjVv) z>$H(z0VrYRtCVTtblAze2P?9;Cnw+ZT|SN0w=cNv+}KcCK*PtcycpQ7r+Fg$Q7%p3@8cs(Oe{3|8< z#lxA)x$Ap=DwWBtDZPATcuO>~gG1)tl6d{NSJ(J!(=V>9H#UTH&Po}M8yr)Gr;<$a zE3LRwSJkcj{FYbyK23Vd-4_NHIe+2OHAciPWwu1}%iI?uLo9?|6IcCqqXUf$7P`mh zy#b@Mh8e|MpekH9KLD=62>{?K+;sq4h0Cx9KvlRHux)#neT;P*gyul-O9wkFY+I;d zvJ1MKVH64QxI~XfCR5T4{~jnt;ek`f*Q%GPdcTRNf8w|h9?kFY2Gk#*)q`xr#IO_c zH3%>rVbsI#*)DDvh>guE5%IPMZ~`HAQyXP6p-dFU93k?p_U&;?^v<#j=B9YlE%fZ5($2sV3w_dlLW@c_4dv~?sk z`SO7ArE^)HeNP7DDw?E`94}A(nw(5cx#=_CUAE_^l2(j(v#8n~0^fOovM1dyl1%hg z!&x^5ho+}L^?BH$x|?Jw=i09Cb>t)qXigkU;AlJ`>-7LVn>=UBqlBj0m%hxrd(PLD zqngaY9l~R^{EXr`J$50&l;53jEqtjOH$_ zp3!TVq<%vF8RKmXoJSOA8q7!9tkMAB&07`#Z{A&?-3+yPgS$)`@4J%$pe76MQy^$q zLJW&86Re=!E&(k7bH&Z}Ik62k93FCO1x&>C+q5JF+v(xnfh^4Rrt>+T_#Nxkpylkt zuNqmW?;aEEe`e*c@NElJH#w0EY7OC!Jqv*ACXlwKnnnb;Zqk7OKy?!dEZUkbSp2t$ zZa09r;$AEds$0&cUTtuV?zBOh-<3r&FbQ|b{h|*y!FL6+b6M!!+qe{OEOIl(S@nVx zqb#wOkYHJ9Tlt=+y?2{R&eM?|RWAPREG52;vvm2D@Kgd#C4+t^_wU1?oq~~kuNRlf zYS+YNVh(txg*@p-rvguEyt7WOh2t_WBprf3An=qdG~PA6Uu zdY5aeq=6$1QA8vKcaaUx4z)^)WTfr&hNer1C5+AcobUW&UAN-O)8$;Yd1!v};hE?H zWoZz2db$n(4^wbpwBbRPq+@m}|J&8#*tR$QND^(9&)!Td=0*T>lOLE~ynt&#BJa$;ECPtmz&qh7}H~a3U2Q zgXsfdPfJr<5PX|CA!jr8ynTYHa6OBs-5qZMgIV*%MeGUW?U^U<&4_yldCgW{c02g*#!~0hHzG5aJ9n2KE}B8;@DdMHKts1Lf}qnFnx+qaN0y zJ9SiGh1*5pmvyZZzAG|P{mhXiX4y16hOy0Taz)Ho2{)X^!TXzL|Hl;bP!u|=w zKjCCD9?H4&z-aV5sz-;~TT94n&)>ZS@=vS}S}G&vt-O-6&zgs!$LP$6;B#2vpoD<> zvEpi-5Bx=M>LZC{co*Z$V1)xITuOM8;ThHq(29cz0Vp95Op2>}Qppn(VKX@;*i z0ol4H`;70y3I|$o4|ZYTV1>iHXBXAU1X8%1%P@&6|nf0cytqTH|W^wqJV1>hcEk%fOH#8SkILrro!sHn-6|U!}Yd}UQ09tXE zVOtk!>qa?&OeWaY1=+g(G(NDxfu6<-+YQZy6%O+%bmcUV!p-=ea61Frx}fL3GK_Ws zmtb2L^X-@}UR;o^ivpKmg#$f4uq&DiD;(x)a~XPgAcd1<*2~R+ZC%ic+pX~%wsk=! z6U5eq!6oPem$1TN@frbC;Zh;C)msZd)N2A81SPP<-h0xR6VDV(kv z8w@T%pZ|t!-G6;F9QDW-$kv6yr32uL=-pt2gS>zWecBSLa5CM^|G?IT z!KDDm6VGLuu)_U;!WneEhry+f0Bq~BD2R#I9j%~8+PZ}|LIv$9ZCCH5^u&ukV2$Kuk<%%zetqX%o(7WPca0#22!66FQ^V5SJ2A9BZ z%EPuU&PU7ya+hbCptddyELydAlis~I|>!FU8>+PxCH4nfx#s#dQGrVxSpS$`(bbi2A43KC!t3+7+iw7Y|tay zQy5$Vb0mMJa5KIqZ^7Ua3@-f@TNehGpjB`fTmly*u~InM!}2h=1cOWe+G_%>g2Uia zEV!Fe5SmiKH!fjO1&5-!DG_NfxCDbsFu3IK^KeW(9!=|{cv8EEgxEuH294E`4F;F~ z8n~nfgG(^D1cOTwzK5DeaET^765H*i9x|K;XV6fW%>;U6gTW;%j%-k$7mDWQWah%) z5)3Z=6_Rh-A4g~OVE@GzHtfCO$qgR zp=d4)E}G^v38irBrKY3uJ$R)*zvR zwo8owwso(NC%7NCKz0T!KHtHxoekR6KoU?2AA%_wk{TehK;QY-?#)T91Jc+)~++k;$K+2 z6@)4*p)j~M5-gf0cR8}bwk{SVF0={`wRPdA;$VgQO9}^rOQX;#IBe_wk*y1ZOR&QI zC53~*B{$gC{Y$nk3@*V6hec5m2A8lpvcb0QU$S-Kr{Z9R`%4N3-?)T&=>vX9=U?AG z#p*r^_^CJ;&4q7V@&h`%rlffSNd(aSl}Qs0c98uAT7!fN+Ab*-d>;iCH3Ar1;sbY6 z!U~5)Q4%YKgTW;jT!Or7vI~g|-?;Q=J}<0rf243QxCDbskataX5i}TF!lu^*2ABR8 zxMW8D`6mO=w+&Dn-nv{T@d|f9W_)3OV8Ui&wKl@k)gq0~O*ORSq|xOc&jYwh60Gx# z8y^R2e2maINnsZMCF3-W)JDJ2mn_M(f!CD%t)hknquolJvud2l1VvhlXSnYqZOt*? zOp6!uP%alIq2d2k7S(*RtC>epK!1~3BO^!dIzIJlHbQ{Z$$m@B;j8G+u`8IM2x!)zwcFC44H?sN^$^ws87cS09|R zTwFJ}HB3&in3-!jF5d3@Ws$avd8oZJX6;mhv4l`bxJDgSC_WXZVfkkdu8SIO6(4Yk z7_Wb<#dTSgk1VIo!Ce#=AHGIu=jYq}WXb)JPA@sJqxkT$ojNUE$}wKrt~7EwrEm6k z$R(|k(odD#8@@`gVNKg~P;-UQtUdDfg1r0x{~Zego;rHm$*QC~fAS2*awg=%#~R-rwSgYS#+8;o_6 z*E{z8XgB?)mjEz{n8aVPLmatBF?}(KDNa+H^iY%KYFxr?JGPHZ)h20y=UbF`4b_>? zBJ&G=zBX3m`4y$UK{`~ZrzkppB$o4>C^M2_O39}qooZ8^(=T}8BkvJ<_ox<1uGOf+ zSL%mD)d!O4`BR0rmFPLTYZq=YDf%drupM5GIX5z$d z$BYfvE>n}=lTx=Mq+ICy`I;r}QXQLkp{cFWts&P@j`2D_lZ?*yS5}3x#BEFb+1%_d zQ~my5YgH?KIN)o%)yd(Y%j*~B&sO9aPA~n0S^38Q{a4Deuh{4K`?K}2mwa~lXSDwR I?=ATM4@cgS1^@s6 literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-sdr.hevc b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/fixtures/probe/hevc-y410-sdr.hevc new file mode 100644 index 0000000000000000000000000000000000000000..9b4a43e2f9409b75a503e0b9e331f87f14a69064 GIT binary patch literal 667 zcmZQzU|@7$H_!n^ yHtzrmcS-{a_W&yQH42we7FjK$Hwq6W>=|dG$n8eqT}R=)MB)8L<}tCM@WcReT0Nrx literal 0 HcmV?d00001 diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/adapter_decode.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/adapter_decode.rs new file mode 100644 index 000000000..dd9cf2251 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/adapter_decode.rs @@ -0,0 +1,203 @@ +use std::sync::OnceLock; + +use opennow_streamer_protocol::GraphicsAdapterCapability; + +/// Hardware decode profiles observed on one physical adapter. +/// +/// High-performance enumeration order is preserved. Automatic selection uses +/// the first adapter that exposes any profile, so a hybrid laptop whose +/// discrete GPU cannot decode (GeForce MX110 on an Intel HD Graphics 620 +/// machine, for example) falls through to the integrated GPU that can. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdapterDecodeIndex { + pub luid: u64, + pub name: String, + pub h264: bool, + pub h265: bool, + pub h265_main10: bool, + pub av1: bool, + pub reason: Option, +} + +#[cfg(any(windows, test))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DecodeProfile { + H264, + H265Main, + H265Main10, + Av1, + Other, +} + +impl AdapterDecodeIndex { + #[cfg(any(windows, test))] + pub(crate) fn named(luid: u64, name: impl Into) -> Self { + Self { + luid, + name: sanitize_adapter_name(&name.into()), + h264: false, + h265: false, + h265_main10: false, + av1: false, + reason: None, + } + } + + pub fn can_decode(&self) -> bool { + self.h264 || self.h265 || self.av1 + } + + pub fn codec_names(&self) -> Vec<&'static str> { + let mut names = Vec::new(); + if self.h264 { + names.push("h264"); + } + if self.h265 { + names.push("h265"); + } + if self.av1 { + names.push("av1"); + } + names + } + + #[cfg(any(windows, test))] + pub(crate) fn apply_profile(&mut self, profile: DecodeProfile, supported: bool) { + if !supported { + return; + } + match profile { + DecodeProfile::H264 => self.h264 = true, + DecodeProfile::H265Main => self.h265 = true, + DecodeProfile::H265Main10 => { + self.h265 = true; + self.h265_main10 = true; + } + DecodeProfile::Av1 => self.av1 = true, + DecodeProfile::Other => {} + } + } + + #[cfg(any(windows, test))] + pub(crate) fn finish(&mut self) { + if self.reason.is_none() && !self.can_decode() { + self.reason = Some("no supported hardware decoder profile".to_owned()); + } + } +} + +#[cfg(any(windows, test))] +pub(crate) fn sanitize_adapter_name(value: &str) -> String { + let mut cleaned = String::new(); + for character in value.chars().take(128) { + if character.is_control() { + cleaned.push(' '); + } else { + cleaned.push(character); + } + } + let cleaned = cleaned.split_whitespace().collect::>().join(" "); + if cleaned.is_empty() { + "GPU".to_owned() + } else { + cleaned + } +} + +pub(crate) fn report_adapter_decode( + adapters: Vec, + active_luid: Option, +) -> Vec { + adapters + .into_iter() + .take(8) + .map(|adapter| { + let codecs = adapter + .codec_names() + .into_iter() + .map(str::to_owned) + .collect(); + GraphicsAdapterCapability { + name: adapter.name, + active: active_luid == Some(adapter.luid), + codecs, + h265_main10: adapter.h265_main10, + reason: adapter.reason, + } + }) + .collect() +} + +/// Indexes every physical adapter's hardware decode profiles once per process. +pub fn graphics_adapter_capabilities(active_luid: Option) -> Vec { + report_adapter_decode(index_adapter_decode(), active_luid) +} + +fn index_adapter_decode() -> Vec { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(probe_adapter_decode).clone() +} + +fn probe_adapter_decode() -> Vec { + #[cfg(windows)] + { + crate::windows::probe_adapter_decode() + } + #[cfg(not(windows))] + { + Vec::new() + } +} + +#[cfg(test)] +mod tests { + use super::{AdapterDecodeIndex, DecodeProfile, report_adapter_decode, sanitize_adapter_name}; + + #[test] + fn adapter_names_keep_vendor_text_and_drop_controls() { + assert_eq!( + sanitize_adapter_name("Intel(R) HD Graphics 620"), + "Intel(R) HD Graphics 620" + ); + assert_eq!( + sanitize_adapter_name("NVIDIA GeForce MX110\nextra"), + "NVIDIA GeForce MX110 extra" + ); + assert_eq!(sanitize_adapter_name(" \n\t "), "GPU"); + } + + #[test] + fn mx110_without_profiles_is_empty_and_hd620_indexes_h264_and_h265() { + let mut mx110 = AdapterDecodeIndex::named(2, "NVIDIA GeForce MX110"); + mx110.apply_profile(DecodeProfile::H264, false); + mx110.apply_profile(DecodeProfile::Av1, false); + mx110.finish(); + assert!(mx110.codec_names().is_empty()); + assert!(!mx110.can_decode()); + assert_eq!( + mx110.reason.as_deref(), + Some("no supported hardware decoder profile") + ); + + let mut hd620 = AdapterDecodeIndex::named(1, "Intel(R) HD Graphics 620"); + hd620.apply_profile(DecodeProfile::H264, true); + hd620.apply_profile(DecodeProfile::H265Main, true); + hd620.apply_profile(DecodeProfile::H265Main10, true); + hd620.apply_profile(DecodeProfile::Av1, false); + hd620.apply_profile(DecodeProfile::Other, true); + hd620.finish(); + assert_eq!(hd620.codec_names(), vec!["h264", "h265"]); + assert!(hd620.h265_main10); + assert!(hd620.reason.is_none()); + + let reported = report_adapter_decode(vec![mx110, hd620], Some(1)); + assert!(!reported[0].active); + assert!(reported[0].codecs.is_empty()); + assert!(reported[1].active); + assert_eq!( + reported[1].codecs, + vec!["h264".to_owned(), "h265".to_owned()] + ); + assert!(reported[1].h265_main10); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/aperture.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/aperture.rs new file mode 100644 index 000000000..510e98c35 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/aperture.rs @@ -0,0 +1,110 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct VideoAperture { + pub(crate) x: u32, + pub(crate) y: u32, + pub(crate) width: u32, + pub(crate) height: u32, +} + +impl VideoAperture { + pub(crate) fn new( + media_width: u32, + media_height: u32, + area: Option<(i32, i32, i32, i32)>, + ) -> Result { + let aperture = match area { + None | Some((0, 0, 0, 0)) => Self { + x: 0, + y: 0, + width: media_width, + height: media_height, + }, + Some((x, y, width, height)) if x >= 0 && y >= 0 && width > 0 && height > 0 => Self { + x: x as u32, + y: y as u32, + width: width as u32, + height: height as u32, + }, + Some(_) => return Err("invalid decoder display aperture".to_owned()), + }; + aperture.validate_extent(media_width, media_height)?; + Ok(aperture) + } + + pub(crate) fn validate_extent(self, width: u32, height: u32) -> Result<(), String> { + if width == 0 + || height == 0 + || width > i32::MAX as u32 + || height > i32::MAX as u32 + || self.width == 0 + || self.height == 0 + || self + .x + .checked_add(self.width) + .is_none_or(|right| right > width) + || self + .y + .checked_add(self.height) + .is_none_or(|bottom| bottom > height) + { + return Err(format!( + "decoder display aperture {self:?} exceeds frame extent {width}x{height}" + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn padded_allocation_does_not_change_visible_extent() { + let aperture = VideoAperture::new(1920, 1080, None).unwrap(); + for _ in 0..120 { + aperture.validate_extent(1920, 1088).unwrap(); + assert_eq!((aperture.width, aperture.height), (1920, 1080)); + } + } + + #[test] + fn explicit_aperture_preserves_offsets_and_excludes_padding() { + let aperture = VideoAperture::new(1920, 1088, Some((8, 4, 1904, 1080))).unwrap(); + assert_eq!((aperture.x, aperture.y), (8, 4)); + assert_eq!( + (aperture.x + aperture.width, aperture.y + aperture.height), + (1912, 1084) + ); + aperture.validate_extent(1920, 1088).unwrap(); + assert!(aperture.validate_extent(1920, 1080).is_err()); + } + + #[test] + fn absent_and_default_apertures_use_media_not_allocation_extent() { + assert_eq!( + VideoAperture::new(1920, 1080, None), + VideoAperture::new(1920, 1080, Some((0, 0, 0, 0))) + ); + } + + #[test] + fn malformed_apertures_are_rejected_without_clamping() { + for area in [ + (-1, 0, 1920, 1080), + (0, -1, 1920, 1080), + (0, 0, -1, 1080), + (0, 0, 1920, 0), + (1, 0, 1920, 1080), + (0, 9, 1920, 1080), + (1, 0, 0, 0), + ] { + assert!( + VideoAperture::new(1920, 1088, Some(area)).is_err(), + "{area:?}" + ); + } + assert!(VideoAperture::new(0, 1080, None).is_err()); + assert!(VideoAperture::new(u32::MAX, 1080, None).is_err()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_color.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_color.rs new file mode 100644 index 000000000..42b56fe6a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_color.rs @@ -0,0 +1,226 @@ +use crate::{ + VideoChromaSiting, VideoColorMatrix, VideoColorPrimaries, VideoFormat, VideoTransferFunction, +}; + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) enum Descriptor { + #[default] + Unspecified, + Recognized(T), + Unsupported(u32), +} + +impl Descriptor { + fn resolve(self, field_bit: u32, field: &str, fallback: T) -> T { + match self { + Self::Unspecified => fallback, + Self::Recognized(value) => value, + Self::Unsupported(value) => { + static REPORTED: std::sync::atomic::AtomicU32 = + std::sync::atomic::AtomicU32::new(0); + let bit = 1 << field_bit; + if REPORTED.fetch_or(bit, std::sync::atomic::Ordering::Relaxed) & bit == 0 { + eprintln!( + "warning: decoded Media Foundation {field} {value} is not presentable; using negotiated {fallback:?}" + ); + } + fallback + } + } + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct ColorDescriptors { + pub transfer: Descriptor, + pub primaries: Descriptor, + pub matrix: Descriptor, + pub range: Descriptor, + pub chroma_siting: Descriptor, +} + +impl ColorDescriptors { + pub fn resolve(self, fallback: VideoFormat) -> VideoFormat { + let mut format = VideoFormat { + transfer_function: self.transfer.resolve( + 0, + "transfer function", + fallback.transfer_function, + ), + color_primaries: self + .primaries + .resolve(1, "color primaries", fallback.color_primaries), + color_matrix: self + .matrix + .resolve(2, "YCbCr matrix", fallback.color_matrix), + full_range: self.range.resolve(3, "nominal range", fallback.full_range), + chroma_siting: self + .chroma_siting + .resolve(4, "chroma siting", fallback.chroma_siting), + ..fallback + }; + if matches!(self.matrix, Descriptor::Unspecified) + && format.transfer_function == VideoTransferFunction::Sdr + && format.color_primaries == VideoColorPrimaries::Bt709 + { + format.color_matrix = VideoColorMatrix::Bt709; + } + format + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{VideoChromaFormat, VideoCodec, VideoPixelFormat}; + use std::num::NonZeroU32; + + fn hdr_format() -> VideoFormat { + VideoFormat { + codec: VideoCodec::H265, + width: 1920, + height: 1080, + frame_rate_numerator: NonZeroU32::new(60).unwrap(), + frame_rate_denominator: NonZeroU32::new(1).unwrap(), + average_bitrate: 20_000_000, + pixel_format: VideoPixelFormat::P010, + chroma_format: VideoChromaFormat::Cs420, + chroma_siting: VideoChromaSiting::TopLeft, + full_range: true, + transfer_function: VideoTransferFunction::Pq, + color_primaries: VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + } + } + + #[test] + fn unsupported_descriptors_fall_back_independently() { + let fallback = hdr_format(); + for descriptors in [ + ColorDescriptors { + transfer: Descriptor::Unsupported(999), + ..Default::default() + }, + ColorDescriptors { + primaries: Descriptor::Unsupported(999), + ..Default::default() + }, + ColorDescriptors { + matrix: Descriptor::Unsupported(999), + ..Default::default() + }, + ColorDescriptors { + range: Descriptor::Unsupported(3), + ..Default::default() + }, + ColorDescriptors { + chroma_siting: Descriptor::Unsupported(999), + ..Default::default() + }, + ColorDescriptors { + transfer: Descriptor::Unsupported(u32::MAX), + primaries: Descriptor::Unsupported(u32::MAX), + matrix: Descriptor::Unsupported(u32::MAX), + range: Descriptor::Unsupported(u32::MAX), + chroma_siting: Descriptor::Unsupported(u32::MAX), + }, + ] { + assert_eq!(descriptors.resolve(fallback), fallback); + } + } + + #[test] + fn unsupported_fields_do_not_discard_recognized_fields() { + let fallback = hdr_format(); + let descriptors = ColorDescriptors { + transfer: Descriptor::Recognized(VideoTransferFunction::Hlg), + primaries: Descriptor::Unsupported(999), + matrix: Descriptor::Unsupported(999), + range: Descriptor::Recognized(false), + chroma_siting: Descriptor::Recognized(VideoChromaSiting::Left), + }; + assert_eq!( + descriptors.resolve(fallback), + VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + full_range: false, + chroma_siting: VideoChromaSiting::Left, + ..fallback + } + ); + } + + #[test] + fn unsupported_matrix_does_not_use_the_unspecified_sdr_rule() { + let fallback = hdr_format(); + let descriptors = ColorDescriptors { + transfer: Descriptor::Recognized(VideoTransferFunction::Sdr), + primaries: Descriptor::Recognized(VideoColorPrimaries::Bt709), + matrix: Descriptor::Unsupported(999), + ..Default::default() + }; + let output = descriptors.resolve(fallback); + assert_eq!(output.color_matrix, fallback.color_matrix); + assert!(output.validate_color().is_err()); + let output = ColorDescriptors { + matrix: Descriptor::Unspecified, + ..descriptors + } + .resolve(fallback); + assert_eq!(output.color_matrix, VideoColorMatrix::Bt709); + assert!(output.validate_color().is_ok()); + } + + #[test] + fn recognized_incompatible_fields_are_not_repaired() { + let fallback = hdr_format(); + for descriptors in [ + ColorDescriptors { + primaries: Descriptor::Recognized(VideoColorPrimaries::Bt709), + ..Default::default() + }, + ColorDescriptors { + matrix: Descriptor::Recognized(VideoColorMatrix::Bt709), + transfer: Descriptor::Unsupported(999), + ..Default::default() + }, + ] { + assert!(descriptors.resolve(fallback).validate_color().is_err()); + } + let output = ColorDescriptors::default().resolve(VideoFormat { + pixel_format: VideoPixelFormat::Nv12, + ..fallback + }); + assert!(output.validate_color().is_err()); + } + + #[test] + fn unspecified_fields_and_recognized_transitions_are_preserved() { + let hdr = hdr_format(); + assert_eq!(ColorDescriptors::default().resolve(hdr), hdr); + let sdr = VideoFormat { + transfer_function: VideoTransferFunction::Sdr, + color_primaries: VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + ..hdr + }; + for format in [ + sdr, + hdr, + VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + ..hdr + }, + ] { + let descriptors = ColorDescriptors { + transfer: Descriptor::Recognized(format.transfer_function), + primaries: Descriptor::Recognized(format.color_primaries), + matrix: Descriptor::Recognized(format.color_matrix), + range: Descriptor::Recognized(format.full_range), + chroma_siting: Descriptor::Recognized(format.chroma_siting), + }; + assert_eq!(descriptors.resolve(sdr), format); + assert_eq!(descriptors.resolve(hdr), format); + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_order.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_order.rs new file mode 100644 index 000000000..2a9ae8c0d --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/decoder_order.rs @@ -0,0 +1,62 @@ +/// Where a hardware-mode decoder search looks for Media Foundation activations. +#[cfg_attr(not(windows), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DecoderCandidateSource { + /// Hardware MFTs registered against the Qt adapter's LUID. + AdapterHardware, + /// Hardware MFTs that are not tagged with an adapter LUID. Intel iGPU + /// decoders are commonly visible only in this set; skipping it selects the + /// Microsoft software H.264 MFT instead. + UnscopedHardware, + /// Every registered decoder. NVIDIA and AMD expose DXVA through the + /// Microsoft MFT, which is why this remains the last hardware-mode step. + Registered, +} + +/// Adapter-scoped hardware MFTs win when any exist. Otherwise try unscoped +/// hardware MFTs before the registered software/DXVA fallback. Trying the +/// registered list first on an Intel iGPU sticks playback on the software +/// decoder, which cannot sustain the stream and then stalls. +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) fn decoder_candidate_sources( + adapter_hardware_count: usize, +) -> &'static [DecoderCandidateSource] { + if adapter_hardware_count == 0 { + &[ + DecoderCandidateSource::UnscopedHardware, + DecoderCandidateSource::Registered, + ] + } else { + &[ + DecoderCandidateSource::AdapterHardware, + DecoderCandidateSource::Registered, + ] + } +} + +#[cfg(test)] +mod tests { + use super::{DecoderCandidateSource, decoder_candidate_sources}; + + #[test] + fn intel_style_adapters_search_unscoped_hardware_before_software() { + assert_eq!( + decoder_candidate_sources(0), + &[ + DecoderCandidateSource::UnscopedHardware, + DecoderCandidateSource::Registered, + ] + ); + } + + #[test] + fn adapter_matched_hardware_is_not_replaced_by_another_gpu() { + assert_eq!( + decoder_candidate_sources(1), + &[ + DecoderCandidateSource::AdapterHardware, + DecoderCandidateSource::Registered, + ] + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/format.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/format.rs new file mode 100644 index 000000000..50b00b07b --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/format.rs @@ -0,0 +1,658 @@ +use std::num::{NonZeroIsize, NonZeroU32}; + +use crate::BackendError; + +pub const MAX_DECODE_WIDTH: u32 = 4096; +pub const MAX_DECODE_HEIGHT: u32 = 2304; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoCodec { + H264, + H265, + Av1, +} + +impl VideoCodec { + pub const fn label(self) -> &'static str { + match self { + Self::H264 => "H.264", + Self::H265 => "H.265", + Self::Av1 => "AV1", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoPixelFormat { + Nv12, + P010, + Ayuv, + Y410, +} + +impl VideoPixelFormat { + pub const fn bit_depth(self) -> u8 { + match self { + Self::Nv12 | Self::Ayuv => 8, + Self::P010 | Self::Y410 => 10, + } + } + + #[cfg(any(windows, test))] + pub(crate) fn supports_decoded_output(self, output: Self) -> bool { + match self { + Self::Nv12 => true, + Self::P010 => matches!(output, Self::P010 | Self::Y410), + Self::Ayuv => matches!(output, Self::Ayuv | Self::Y410 | Self::Nv12), + Self::Y410 => matches!(output, Self::Y410 | Self::P010), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoChromaFormat { + Cs420, + Cs444, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoChromaSiting { + Left, + TopLeft, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoTransferFunction { + Sdr, + Pq, + Hlg, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoColorPrimaries { + Bt709, + Bt2020, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoColorMatrix { + Bt601, + Bt709, + Bt2020, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VideoFormat { + pub codec: VideoCodec, + pub width: u32, + pub height: u32, + pub frame_rate_numerator: NonZeroU32, + pub frame_rate_denominator: NonZeroU32, + pub average_bitrate: u32, + /// Preferred decoder output before the MFT reports its actual subtype. + pub pixel_format: VideoPixelFormat, + pub chroma_format: VideoChromaFormat, + pub chroma_siting: VideoChromaSiting, + /// Whether the decoded YCbCr samples use full (0-255) rather than studio range. + pub full_range: bool, + pub transfer_function: VideoTransferFunction, + pub color_primaries: VideoColorPrimaries, + pub color_matrix: VideoColorMatrix, +} + +impl VideoFormat { + pub fn validate(self) -> Result<(), BackendError> { + self.validate_color()?; + if !(48..=MAX_DECODE_WIDTH).contains(&self.width) + || !(48..=MAX_DECODE_HEIGHT).contains(&self.height) + { + return Err(BackendError::InvalidConfig(format!( + "{} dimensions must be at least 48x48 and no larger than 4096x2304", + self.codec.label() + ))); + } + let fps = self.frame_rate_numerator.get() as f64 / self.frame_rate_denominator.get() as f64; + if !(1.0..=360.0).contains(&fps) { + return Err(BackendError::InvalidConfig( + "video frame rate must be between 1 and 360 fps".to_owned(), + )); + } + if self.average_bitrate == 0 { + return Err(BackendError::InvalidConfig( + "average video bitrate must be non-zero".to_owned(), + )); + } + Ok(()) + } + + pub fn validate_color(self) -> Result<(), BackendError> { + if self.transfer_function != VideoTransferFunction::Sdr + && (self.pixel_format.bit_depth() != 10 + || self.color_primaries != VideoColorPrimaries::Bt2020 + || self.color_matrix != VideoColorMatrix::Bt2020) + { + return Err(BackendError::InvalidConfig( + "HDR requires 10-bit decoder output with BT.2020 primaries and matrix".to_owned(), + )); + } + if (self.color_primaries == VideoColorPrimaries::Bt2020) + != (self.color_matrix == VideoColorMatrix::Bt2020) + { + return Err(BackendError::InvalidConfig( + "decoder color primaries and YCbCr matrix are incompatible".to_owned(), + )); + } + Ok(()) + } + + pub fn frame_duration_100ns(self) -> i64 { + (10_000_000_u64 * self.frame_rate_denominator.get() as u64 + / self.frame_rate_numerator.get() as u64) as i64 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AudioFormat { + pub sample_rate: u32, + pub channels: u16, +} + +impl AudioFormat { + pub fn validate(self) -> Result<(), BackendError> { + if !(8_000..=192_000).contains(&self.sample_rate) { + return Err(BackendError::InvalidConfig( + "PCM sample rate must be between 8 kHz and 192 kHz".to_owned(), + )); + } + if !(1..=8).contains(&self.channels) { + return Err(BackendError::InvalidConfig( + "PCM channel count must be between 1 and 8".to_owned(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct WindowHandle(NonZeroIsize); + +impl WindowHandle { + pub fn new(raw: isize) -> Option { + NonZeroIsize::new(raw).map(Self) + } + + pub fn get(self) -> isize { + self.0.get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Bounds { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +impl Bounds { + pub fn validate(self) -> Result<(), BackendError> { + if self.width == 0 || self.height == 0 { + return Err(BackendError::InvalidConfig( + "surface bounds must be non-empty".to_owned(), + )); + } + if self.width > i32::MAX as u32 || self.height > i32::MAX as u32 { + return Err(BackendError::InvalidConfig( + "surface bounds exceed Win32 coordinate limits".to_owned(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExistingWindow { + pub hwnd: WindowHandle, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OwnedWindow { + pub parent: Option, + pub bounds: Bounds, + pub visible: bool, +} + +impl OwnedWindow { + #[cfg_attr(not(windows), allow(dead_code))] + pub(crate) fn has_same_parent(self, other: Self) -> bool { + self.parent == other.parent + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SurfaceTarget { + /// A dedicated renderer HWND whose ownership, activation, and input policy belongs to the caller. + Existing(ExistingWindow), + /// An input-transparent, non-activating HWND owned by this backend. + Owned(OwnedWindow), +} + +impl SurfaceTarget { + pub fn validate(self) -> Result<(), BackendError> { + if let Self::Owned(window) = self { + window.bounds.validate()?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedVideoFrame { + pub codec: VideoCodec, + pub data: Vec, + pub timestamp_100ns: i64, + pub duration_100ns: i64, + pub key_frame: bool, + /// Discard every decoder reference and decoded surface before accepting + /// this recovery keyframe. A clean-point flag alone does not make an MFT + /// forget damaged H.264 references after packet loss. + pub reset_decoder: bool, +} + +impl EncodedVideoFrame { + pub fn validate(&self) -> Result<(), BackendError> { + if self.data.is_empty() { + return Err(BackendError::InvalidFrame(format!( + "{} access unit is empty", + self.codec.label() + ))); + } + if matches!(self.codec, VideoCodec::H264 | VideoCodec::H265) + && !self.data.starts_with(&[0, 0, 1]) + && !self.data.starts_with(&[0, 0, 0, 1]) + { + return Err(BackendError::InvalidFrame(format!( + "{} input must use Annex B start codes", + self.codec.label() + ))); + } + if self.timestamp_100ns < 0 || self.duration_100ns <= 0 { + return Err(BackendError::InvalidFrame( + "video timestamps must be non-negative with positive duration".to_owned(), + )); + } + if self.reset_decoder && !self.key_frame { + return Err(BackendError::InvalidFrame( + "a decoder reset must be attached to a recovery keyframe".to_owned(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PcmFrame { + pub samples: Vec, + pub format: AudioFormat, +} + +impl PcmFrame { + pub fn validate(&self) -> Result<(), BackendError> { + self.format.validate().map_err(|error| { + BackendError::InvalidFrame(format!("PCM packet format is invalid: {error}")) + })?; + if self.samples.is_empty() { + return Err(BackendError::InvalidFrame( + "PCM frame must contain samples and at least one channel".to_owned(), + )); + } + if self.samples.len() % self.format.channels as usize != 0 { + return Err(BackendError::InvalidFrame( + "interleaved PCM sample count is not divisible by channel count".to_owned(), + )); + } + if self.samples.iter().any(|sample| !sample.is_finite()) { + return Err(BackendError::InvalidFrame( + "PCM samples must be finite f32 values".to_owned(), + )); + } + let maximum_frames = self.format.sample_rate as usize * 120 / 1_000; + if self.frame_count() > maximum_frames { + return Err(BackendError::InvalidFrame( + "PCM packet exceeds the 120 ms Opus frame limit".to_owned(), + )); + } + Ok(()) + } + + pub fn frame_count(&self) -> usize { + self.samples.len() / self.format.channels as usize + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BackendConfig { + pub video: VideoFormat, + pub audio: AudioFormat, + pub surface: SurfaceTarget, + pub video_queue_capacity: usize, + pub audio_queue_capacity: usize, +} + +impl BackendConfig { + pub fn validate(self) -> Result<(), BackendError> { + self.video.validate()?; + self.audio.validate()?; + self.surface.validate()?; + if !(1..=32).contains(&self.video_queue_capacity) { + return Err(BackendError::InvalidConfig( + "video queue capacity must be between 1 and 32 frames".to_owned(), + )); + } + if !(1..=256).contains(&self.audio_queue_capacity) { + return Err(BackendError::InvalidConfig( + "audio queue capacity must be between 1 and 256 packets".to_owned(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn video_format() -> VideoFormat { + VideoFormat { + codec: VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: NonZeroU32::new(120).unwrap(), + frame_rate_denominator: NonZeroU32::new(1).unwrap(), + average_bitrate: 75_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + } + } + + #[test] + fn decoded_chroma_fallback_preserves_negotiated_bit_depth() { + for (requested, output) in [ + (VideoPixelFormat::Ayuv, VideoPixelFormat::Nv12), + (VideoPixelFormat::Y410, VideoPixelFormat::P010), + ] { + assert!(requested.supports_decoded_output(output)); + assert_eq!(requested.bit_depth(), output.bit_depth()); + } + assert!(!VideoPixelFormat::Ayuv.supports_decoded_output(VideoPixelFormat::P010)); + } + + #[test] + fn decoded_output_never_reduces_negotiated_bit_depth() { + for requested in [VideoPixelFormat::P010, VideoPixelFormat::Y410] { + for output in [VideoPixelFormat::Nv12, VideoPixelFormat::Ayuv] { + assert!(!requested.supports_decoded_output(output)); + } + } + for pixel_format in [ + VideoPixelFormat::Nv12, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] { + assert!(pixel_format.supports_decoded_output(pixel_format)); + } + } + + #[test] + fn ten_bit_sdr_is_not_hdr() { + for pixel_format in [VideoPixelFormat::P010, VideoPixelFormat::Y410] { + let format = VideoFormat { + pixel_format, + ..video_format() + }; + assert!(format.validate_color().is_ok()); + assert_eq!(format.transfer_function, VideoTransferFunction::Sdr); + } + } + + #[test] + fn hdr_requires_ten_bit_bt2020() { + for transfer_function in [VideoTransferFunction::Pq, VideoTransferFunction::Hlg] { + let format = VideoFormat { + pixel_format: VideoPixelFormat::P010, + transfer_function, + color_primaries: VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + ..video_format() + }; + assert!(format.validate().is_ok()); + assert!( + VideoFormat { + pixel_format: VideoPixelFormat::Nv12, + ..format + } + .validate_color() + .is_err() + ); + assert!( + VideoFormat { + color_primaries: VideoColorPrimaries::Bt709, + ..format + } + .validate_color() + .is_err() + ); + assert!( + VideoFormat { + color_matrix: VideoColorMatrix::Bt709, + ..format + } + .validate_color() + .is_err() + ); + } + } + + #[test] + fn color_changes_are_part_of_format_identity() { + let format = video_format(); + assert_ne!( + format, + VideoFormat { + chroma_siting: VideoChromaSiting::TopLeft, + ..format + } + ); + assert_ne!( + format, + VideoFormat { + full_range: true, + ..format + } + ); + assert_ne!( + format, + VideoFormat { + transfer_function: VideoTransferFunction::Pq, + ..format + } + ); + assert_ne!( + format, + VideoFormat { + color_primaries: VideoColorPrimaries::Bt2020, + ..format + } + ); + assert_ne!( + format, + VideoFormat { + color_matrix: VideoColorMatrix::Bt2020, + ..format + } + ); + } + + #[test] + fn rejects_sdr_primaries_and_matrix_mismatches() { + let format = video_format(); + assert!( + VideoFormat { + color_primaries: VideoColorPrimaries::Bt2020, + ..format + } + .validate_color() + .is_err() + ); + assert!( + VideoFormat { + color_matrix: VideoColorMatrix::Bt2020, + ..format + } + .validate_color() + .is_err() + ); + assert!( + VideoFormat { + color_primaries: VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + ..format + } + .validate_color() + .is_ok() + ); + } + + #[test] + fn validates_supported_h264_format() { + let format = video_format(); + assert!(format.validate().is_ok()); + assert_eq!(format.frame_duration_100ns(), 83_333); + } + + #[test] + fn rejects_out_of_range_h264_format() { + assert!( + VideoFormat { + width: 16, + ..video_format() + } + .validate() + .is_err() + ); + assert!( + VideoFormat { + average_bitrate: 0, + ..video_format() + } + .validate() + .is_err() + ); + assert!( + VideoFormat { + frame_rate_numerator: NonZeroU32::new(360).unwrap(), + ..video_format() + } + .validate() + .is_ok() + ); + assert!( + VideoFormat { + frame_rate_numerator: NonZeroU32::new(480).unwrap(), + ..video_format() + } + .validate() + .is_err() + ); + } + + #[test] + fn requires_annex_b_access_units() { + let valid = EncodedVideoFrame { + codec: VideoCodec::H264, + data: vec![0, 0, 0, 1, 0x67], + timestamp_100ns: 0, + duration_100ns: 166_667, + key_frame: true, + reset_decoder: false, + }; + assert!(valid.validate().is_ok()); + assert!( + EncodedVideoFrame { + data: vec![1, 2, 3], + ..valid + } + .validate() + .is_err() + ); + assert!( + EncodedVideoFrame { + key_frame: false, + reset_decoder: true, + ..valid + } + .validate() + .is_err() + ); + } + + #[test] + fn validates_interleaved_pcm_shape() { + let frame = PcmFrame { + samples: vec![0.0, 0.25, -0.25, 0.0], + format: AudioFormat { + sample_rate: 48_000, + channels: 2, + }, + }; + assert_eq!(frame.frame_count(), 2); + assert!(frame.validate().is_ok()); + assert!( + PcmFrame { + samples: vec![0.0], + format: frame.format, + } + .validate() + .is_err() + ); + } + + #[test] + fn owned_surface_updates_preserve_parent_ownership() { + let parent = WindowHandle::new(1).unwrap(); + let original = OwnedWindow { + parent: Some(parent), + bounds: Bounds { + x: 0, + y: 0, + width: 1280, + height: 720, + }, + visible: true, + }; + let moved_and_hidden = OwnedWindow { + bounds: Bounds { + x: 40, + y: 20, + width: 960, + height: 540, + }, + visible: false, + ..original + }; + let reparented = OwnedWindow { + parent: WindowHandle::new(2), + ..moved_and_hidden + }; + + assert!(original.has_same_parent(moved_and_hidden)); + assert!(!original.has_same_parent(reparented)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/lib.rs new file mode 100644 index 000000000..5eadc3aa1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/lib.rs @@ -0,0 +1,916 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(any(windows, test))] +mod aperture; +#[cfg(any(windows, test))] +mod decoder_color; +mod decoder_order; +mod format; +mod queue; +#[cfg(any(windows, test))] +mod y410_color; + +mod adapter_decode; + +#[cfg(windows)] +mod windows; + +pub use adapter_decode::{AdapterDecodeIndex, graphics_adapter_capabilities}; + +#[cfg(windows)] +pub use windows::{ + AdoptedD3d11Context, D3d11ColorSpace, D3d11Frame, D3d11FrameProducer, D3d11FrameSubmitter, + D3d11RecordedFrame, D3d11TextureFormat, +}; + +use std::num::NonZeroU64; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::JoinHandle; + +use opennow_streamer_protocol::log; + +use crate::queue::BoundedQueue; + +pub use format::{ + AudioFormat, BackendConfig, Bounds, EncodedVideoFrame, ExistingWindow, MAX_DECODE_HEIGHT, + MAX_DECODE_WIDTH, OwnedWindow, PcmFrame, SurfaceTarget, VideoChromaFormat, VideoChromaSiting, + VideoCodec, VideoColorMatrix, VideoColorPrimaries, VideoFormat, VideoPixelFormat, + VideoTransferFunction, WindowHandle, +}; +pub use queue::{CompressedAdmit, GapAdmit, PushOutcome, ReferenceGap, admit_compressed_frame}; + +/// Burst allowance for adaptive video delivery. The decoded presenter uses +/// the same bound and trims stale frames before presentation, keeping latency +/// low without treating ordinary game-FPS fluctuations as decoder loss. +pub const ADAPTIVE_VIDEO_QUEUE_CAPACITY: usize = 7; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum LifecycleState { + Starting = 0, + Running = 1, + Reconfiguring = 2, + Recovering = 3, + Stopping = 4, + Stopped = 5, + Failed = 6, +} + +impl LifecycleState { + fn from_raw(value: u8) -> Self { + match value { + 0 => Self::Starting, + 1 => Self::Running, + 2 => Self::Reconfiguring, + 3 => Self::Recovering, + 4 => Self::Stopping, + 5 => Self::Stopped, + _ => Self::Failed, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Subsystem { + VideoDecode, + VideoPresentation, + Audio, +} + +/// Graphics API that owns the Windows decode/presentation device. D3D12 uses +/// Microsoft's D3D11-on-12 layer so Media Foundation can keep its required +/// D3D11 device-manager contract while work is submitted to a D3D12 queue. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WindowsGraphicsApi { + D3d11, + D3d12, +} + +/// Restricts Media Foundation decoder discovery to the requested execution +/// class. Keeping this explicit prevents a synchronous software MFT from being +/// reported as hardware merely because it can publish D3D11 surfaces. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WindowsDecoderMode { + Hardware, + Software, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct WindowsAdapterLuid(NonZeroU64); + +impl WindowsAdapterLuid { + pub const fn new(raw: u64) -> Option { + match NonZeroU64::new(raw) { + Some(raw) => Some(Self(raw)), + None => None, + } + } + + pub const fn get(self) -> u64 { + self.0.get() + } +} + +#[cfg(windows)] +/// Reads the DXGI adapter LUID from a borrowed D3D11 device. +/// +/// # Safety +/// +/// `device` must point to a live `ID3D11Device` for the duration of the call. +pub unsafe fn d3d11_adapter_luid( + device: *mut std::ffi::c_void, +) -> Result { + unsafe { windows::d3d11_adapter_luid(device) } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BackendEvent { + StateChanged(LifecycleState), + FirstFramePresented, + QueueOverflow(Subsystem), + VideoFormatChanged(VideoFormat), + DeviceLost { + subsystem: Subsystem, + message: String, + }, + DeviceRecovered(Subsystem), + KeyFrameRequired, + Fatal(BackendError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CapabilityProbe { + pub available: bool, + pub h264_hardware_decode: bool, + pub h265_hardware_decode: bool, + pub av1_hardware_decode: bool, + pub h265_hdr: bool, + pub av1_hdr: bool, + pub h265_10bit: bool, + pub av1_10bit: bool, + pub h265_444: bool, + pub h265_10bit_444: bool, + pub h265_hdr_444: bool, + pub h264_software_decode: bool, + pub h265_software_decode: bool, + pub av1_software_decode: bool, + pub d3d11_presentation: bool, + pub wasapi_render: bool, + pub reason: Option, +} + +impl CapabilityProbe { + pub const fn supports_format( + &self, + codec: VideoCodec, + pixel_format: VideoPixelFormat, + hdr: bool, + ) -> bool { + if !self.d3d11_presentation { + return false; + } + match (codec, pixel_format, hdr) { + (VideoCodec::H264, VideoPixelFormat::Nv12, false) => self.h264_hardware_decode, + (VideoCodec::H265, VideoPixelFormat::Nv12, false) => self.h265_hardware_decode, + (VideoCodec::Av1, VideoPixelFormat::Nv12, false) => self.av1_hardware_decode, + (VideoCodec::H265, VideoPixelFormat::P010, false) => self.h265_10bit, + (VideoCodec::Av1, VideoPixelFormat::P010, false) => self.av1_10bit, + (VideoCodec::H265, VideoPixelFormat::Ayuv, false) => self.h265_444, + (VideoCodec::H265, VideoPixelFormat::Y410, false) => self.h265_10bit_444, + (VideoCodec::H265, VideoPixelFormat::Y410, true) => self.h265_hdr_444, + (VideoCodec::H265, VideoPixelFormat::P010, true) => self.h265_hdr, + (VideoCodec::Av1, VideoPixelFormat::P010, true) => self.av1_hdr, + _ => false, + } + } + + pub const fn supports_hdr(&self, codec: VideoCodec) -> bool { + self.d3d11_presentation + && match codec { + VideoCodec::H264 => false, + VideoCodec::H265 => self.h265_hardware_decode && self.h265_hdr, + VideoCodec::Av1 => self.av1_hardware_decode && self.av1_hdr, + } + } + + pub const fn bundled_backend_available(&self) -> bool { + (self.h264_hardware_decode || self.h265_hardware_decode || self.av1_hardware_decode) + && self.d3d11_presentation + && self.wasapi_render + } + + pub const fn software_backend_available(&self) -> bool { + (self.h264_software_decode || self.h265_software_decode || self.av1_software_decode) + && self.d3d11_presentation + && self.wasapi_render + } +} + +#[cfg(any(windows, test))] +struct DefaultEndpointTracker { + active_id: String, + reported_id: Option, +} + +#[cfg(any(windows, test))] +impl DefaultEndpointTracker { + fn new(active_id: String) -> Self { + Self { + active_id, + reported_id: None, + } + } + + fn observe(&mut self, current_id: String) -> bool { + if current_id == self.active_id { + self.reported_id = None; + return false; + } + if self.reported_id.as_ref() == Some(¤t_id) { + return false; + } + self.reported_id = Some(current_id); + true + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BackendError { + UnsupportedPlatform, + InvalidConfig(String), + InvalidFrame(String), + NotRunning(LifecycleState), + Startup(String), + Reconfigure(String), + DeviceLost { + subsystem: Subsystem, + message: String, + }, + WorkerDisconnected, +} + +impl std::fmt::Display for BackendError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnsupportedPlatform => { + formatter.write_str("the Windows media backend is unavailable on this platform") + } + Self::InvalidConfig(message) => { + write!(formatter, "invalid backend configuration: {message}") + } + Self::InvalidFrame(message) => write!(formatter, "invalid media frame: {message}"), + Self::NotRunning(state) => write!(formatter, "backend is not running ({state:?})"), + Self::Startup(message) => write!(formatter, "backend startup failed: {message}"), + Self::Reconfigure(message) => { + write!(formatter, "backend reconfiguration failed: {message}") + } + Self::DeviceLost { subsystem, message } => { + write!(formatter, "{subsystem:?} device was lost: {message}") + } + Self::WorkerDisconnected => formatter.write_str("backend worker disconnected"), + } + } +} + +impl std::error::Error for BackendError {} + +#[derive(Debug)] +#[cfg_attr(not(windows), allow(dead_code))] +enum Control { + ReconfigureVideo(VideoFormat), + ReconfigureAudio(AudioFormat), + SetSurface(SurfaceTarget), + SetPaused(bool), + Stop, +} + +#[derive(Debug)] +struct Shared { + #[cfg(windows)] + audio_muted: Arc, + video: BoundedQueue, + audio: BoundedQueue, + video_format: Mutex, + audio_format: Mutex, + surface: Mutex, + state: AtomicU8, + paused: std::sync::atomic::AtomicBool, + events: BoundedQueue, + presented_frames: AtomicU64, +} + +impl Shared { + fn state(&self) -> LifecycleState { + LifecycleState::from_raw(self.state.load(Ordering::Acquire)) + } + + fn set_state(&self, state: LifecycleState) { + self.state.store(state as u8, Ordering::Release); + let _ = self.events.push(BackendEvent::StateChanged(state)); + } +} + +pub struct WindowsBackend { + shared: Arc, + control: mpsc::Sender, + worker: Mutex>>, +} + +impl WindowsBackend { + pub fn probe() -> CapabilityProbe { + Self::probe_for(WindowsGraphicsApi::D3d11, None) + } + + pub fn probe_for( + api: WindowsGraphicsApi, + adapter_luid: Option, + ) -> CapabilityProbe { + #[cfg(windows)] + { + windows::probe(api, adapter_luid) + } + #[cfg(not(windows))] + { + let _ = (api, adapter_luid); + CapabilityProbe { + available: false, + h264_hardware_decode: false, + h265_hardware_decode: false, + av1_hardware_decode: false, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: false, + h265_software_decode: false, + av1_software_decode: false, + d3d11_presentation: false, + wasapi_render: false, + reason: Some("the current target is not Windows".to_owned()), + } + } + } + + pub fn start(config: BackendConfig) -> Result { + Self::start_for(WindowsGraphicsApi::D3d11, config) + } + + pub fn start_for(api: WindowsGraphicsApi, config: BackendConfig) -> Result { + Self::start_for_mode( + api, + WindowsDecoderMode::Hardware, + config, + Arc::new(std::sync::atomic::AtomicBool::new(false)), + ) + } + + pub fn start_for_mode( + api: WindowsGraphicsApi, + decoder_mode: WindowsDecoderMode, + config: BackendConfig, + audio_muted: Arc, + ) -> Result { + config.validate()?; + #[cfg(windows)] + let video = config.video; + #[cfg(windows)] + let describe = || { + let fps = + video.frame_rate_numerator.get() as f64 / video.frame_rate_denominator.get() as f64; + format!( + "windows backend starting (api={api:?} decoder={decoder_mode:?} codec={} {}x{} {:.0}fps {}kbps pixel={:?} chroma={:?} full_range={})", + video.codec.label(), + video.width, + video.height, + fps, + video.average_bitrate, + video.pixel_format, + video.chroma_format, + video.full_range, + ) + }; + + #[cfg(not(windows))] + { + let _ = api; + let _ = (decoder_mode, config, audio_muted); + log::log_line( + "WARN", + "decode", + "windows backend requested on non-windows build", + ); + Err(BackendError::UnsupportedPlatform) + } + + #[cfg(windows)] + { + log::log_line("INFO", "decode", &describe()); + let shared = Arc::new(Shared { + audio_muted, + video: BoundedQueue::new(config.video_queue_capacity), + audio: BoundedQueue::new(config.audio_queue_capacity), + video_format: Mutex::new(config.video), + audio_format: Mutex::new(config.audio), + surface: Mutex::new(config.surface), + state: AtomicU8::new(LifecycleState::Starting as u8), + paused: std::sync::atomic::AtomicBool::new(false), + events: BoundedQueue::new(64), + presented_frames: AtomicU64::new(0), + }); + let (control_sender, control_receiver) = mpsc::channel(); + let worker = match windows::spawn( + api, + decoder_mode, + config, + Arc::clone(&shared), + control_receiver, + ) { + Ok(worker) => worker, + Err(error) => { + log::log_line( + "WARN", + "decode", + &format!("windows backend spawn failed: {error:?}"), + ); + return Err(error); + } + }; + log::log_line("INFO", "decode", "windows backend worker spawned"); + Ok(Self { + shared, + control: control_sender, + worker: Mutex::new(Some(worker)), + }) + } + } + + pub fn state(&self) -> LifecycleState { + self.shared.state() + } + + pub fn presented_frames(&self) -> u64 { + self.shared.presented_frames.load(Ordering::Relaxed) + } + + pub fn submit_video(&self, frame: EncodedVideoFrame) -> Result { + if let Err(error) = frame.validate().and(self.ensure_media_accepting()) { + log::log_throttled( + "submit-video-reject", + "WARN", + "decode", + &format!("video frame rejected: {error:?}"), + ); + return Err(error); + } + if self.shared.paused.load(Ordering::Acquire) { + return Ok(PushOutcome::Paused); + } + let key_frame = frame.key_frame; + let outcome = self + .shared + .video + .push_or_clear_on_overflow(frame, key_frame, |queued| queued.key_frame) + .map_err(|_| BackendError::NotRunning(self.state()))?; + if outcome == PushOutcome::DroppedOldest { + let _ = self + .shared + .events + .push(BackendEvent::QueueOverflow(Subsystem::VideoDecode)); + log::log_throttled( + "submit-video-overflow", + "WARN", + "decode", + "video queue overflow, oldest frame dropped", + ); + } + Ok(outcome) + } + + pub fn submit_audio(&self, frame: PcmFrame) -> Result { + frame.validate()?; + self.ensure_media_accepting()?; + if self.shared.paused.load(Ordering::Acquire) { + return Ok(PushOutcome::Paused); + } + let expected_format = *self + .shared + .audio_format + .lock() + .unwrap_or_else(|error| error.into_inner()); + if frame.format != expected_format { + return Err(BackendError::InvalidFrame(format!( + "PCM packet format {:?} does not match active format {:?}", + frame.format, expected_format + ))); + } + let outcome = self + .shared + .audio + .push(frame) + .map_err(|_| BackendError::NotRunning(self.state()))?; + if outcome == PushOutcome::DroppedOldest { + let _ = self + .shared + .events + .push(BackendEvent::QueueOverflow(Subsystem::Audio)); + } + Ok(outcome) + } + + pub fn reconfigure_video(&self, format: VideoFormat) -> Result<(), BackendError> { + format.validate()?; + self.ensure_controllable()?; + self.shared.video.clear(); + *self + .shared + .video_format + .lock() + .unwrap_or_else(|error| error.into_inner()) = format; + self.shared.set_state(LifecycleState::Reconfiguring); + self.control + .send(Control::ReconfigureVideo(format)) + .map_err(|_| BackendError::WorkerDisconnected) + } + + pub fn reconfigure_audio(&self, format: AudioFormat) -> Result<(), BackendError> { + format.validate()?; + self.ensure_controllable()?; + self.shared.audio.clear(); + *self + .shared + .audio_format + .lock() + .unwrap_or_else(|error| error.into_inner()) = format; + self.shared.set_state(LifecycleState::Reconfiguring); + self.control + .send(Control::ReconfigureAudio(format)) + .map_err(|_| BackendError::WorkerDisconnected) + } + + pub fn set_surface(&self, surface: SurfaceTarget) -> Result<(), BackendError> { + surface.validate()?; + self.ensure_controllable()?; + *self + .shared + .surface + .lock() + .unwrap_or_else(|error| error.into_inner()) = surface; + self.shared.set_state(LifecycleState::Reconfiguring); + self.control + .send(Control::SetSurface(surface)) + .map_err(|_| BackendError::WorkerDisconnected) + } + + pub fn set_paused(&self, paused: bool) -> Result<(), BackendError> { + self.ensure_controllable()?; + if self.shared.paused.swap(paused, Ordering::AcqRel) == paused { + return Ok(()); + } + self.shared.video.clear(); + self.shared.audio.clear(); + self.control + .send(Control::SetPaused(paused)) + .map_err(|_| BackendError::WorkerDisconnected) + } + + pub fn try_event(&self) -> Option { + self.shared.events.try_pop() + } + + pub fn stop(&self) { + let state = self.state(); + if matches!(state, LifecycleState::Stopped | LifecycleState::Stopping) { + return; + } + let was_failed = state == LifecycleState::Failed; + if !was_failed { + self.shared.set_state(LifecycleState::Stopping); + } + self.shared.video.close(); + self.shared.audio.close(); + let _ = self.control.send(Control::Stop); + if let Some(worker) = self + .worker + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + let _ = worker.join(); + } + if !was_failed && self.state() != LifecycleState::Failed { + self.shared.set_state(LifecycleState::Stopped); + } + } + + fn ensure_media_accepting(&self) -> Result<(), BackendError> { + let state = self.state(); + if matches!( + state, + LifecycleState::Running | LifecycleState::Reconfiguring + ) { + Ok(()) + } else { + Err(BackendError::NotRunning(state)) + } + } + + fn ensure_controllable(&self) -> Result<(), BackendError> { + let state = self.state(); + if matches!( + state, + LifecycleState::Running | LifecycleState::Reconfiguring | LifecycleState::Recovering + ) { + Ok(()) + } else { + Err(BackendError::NotRunning(state)) + } + } +} + +impl Drop for WindowsBackend { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn adapter_luid_models_zero_as_no_selection() { + assert_eq!(WindowsAdapterLuid::new(0), None); + let luid = WindowsAdapterLuid::new(0xffff_fffe_1122_3344).expect("non-zero LUID"); + assert_eq!(luid.get(), 0xffff_fffe_1122_3344); + } + + fn test_backend(state: LifecycleState) -> (WindowsBackend, Arc) { + let (control_sender, _control_receiver) = mpsc::channel(); + let shared = Arc::new(Shared { + video: BoundedQueue::new(2), + audio: BoundedQueue::new(2), + video_format: Mutex::new(VideoFormat { + codec: VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }), + audio_format: Mutex::new(AudioFormat { + sample_rate: 48_000, + channels: 2, + }), + surface: Mutex::new(SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 1280, + height: 720, + }, + visible: false, + })), + state: AtomicU8::new(state as u8), + #[cfg(windows)] + audio_muted: Arc::new(std::sync::atomic::AtomicBool::new(false)), + paused: std::sync::atomic::AtomicBool::new(false), + events: BoundedQueue::new(8), + presented_frames: AtomicU64::new(0), + }); + let backend = WindowsBackend { + shared: Arc::clone(&shared), + control: control_sender, + worker: Mutex::new(None), + }; + (backend, shared) + } + + #[test] + fn non_windows_probe_never_advertises_capabilities() { + if !cfg!(windows) { + let probe = WindowsBackend::probe(); + assert!(!probe.available); + assert!(!probe.h264_hardware_decode); + assert!(!probe.h265_hardware_decode); + assert!(!probe.av1_hardware_decode); + assert!(!probe.h264_software_decode); + assert!(!probe.h265_software_decode); + assert!(!probe.av1_software_decode); + assert!(!probe.h265_hdr); + assert!(!probe.av1_hdr); + assert!(!probe.supports_hdr(VideoCodec::H265)); + assert!(!probe.supports_hdr(VideoCodec::Av1)); + assert!(!probe.d3d11_presentation); + assert!(!probe.wasapi_render); + } + } + + #[test] + fn bundled_backend_requires_wasapi_start() { + let mut probe = CapabilityProbe { + available: false, + h264_hardware_decode: true, + h265_hardware_decode: true, + av1_hardware_decode: true, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: true, + h265_software_decode: true, + av1_software_decode: true, + d3d11_presentation: true, + wasapi_render: false, + reason: Some("WASAPI failed to start".to_owned()), + }; + + assert!(!probe.bundled_backend_available()); + probe.wasapi_render = true; + assert!(probe.bundled_backend_available()); + assert!(probe.software_backend_available()); + } + + #[test] + fn hdr_capability_requires_explicit_codec_and_presentation_support() { + let mut probe = CapabilityProbe { + available: true, + h264_hardware_decode: true, + h265_hardware_decode: true, + av1_hardware_decode: true, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: true, + h265_software_decode: true, + av1_software_decode: true, + d3d11_presentation: true, + wasapi_render: true, + reason: None, + }; + assert!(!probe.supports_hdr(VideoCodec::H265)); + assert!(!probe.supports_hdr(VideoCodec::Av1)); + probe.h265_hdr = true; + assert!(probe.supports_hdr(VideoCodec::H265)); + assert!(!probe.supports_hdr(VideoCodec::Av1)); + probe.av1_hdr = true; + assert!(probe.supports_hdr(VideoCodec::Av1)); + assert!(!probe.supports_hdr(VideoCodec::H264)); + probe.h265_hardware_decode = false; + assert!(!probe.supports_hdr(VideoCodec::H265)); + probe.d3d11_presentation = false; + assert!(!probe.supports_hdr(VideoCodec::Av1)); + } + + #[test] + fn format_capabilities_never_infer_depth_or_chroma_from_codec_support() { + let mut probe = CapabilityProbe { + available: true, + h264_hardware_decode: true, + h265_hardware_decode: true, + av1_hardware_decode: true, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: false, + h265_software_decode: false, + av1_software_decode: false, + d3d11_presentation: true, + wasapi_render: true, + reason: None, + }; + for codec in [VideoCodec::H264, VideoCodec::H265, VideoCodec::Av1] { + assert!(probe.supports_format(codec, VideoPixelFormat::Nv12, false)); + for pixel_format in [ + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] { + assert!(!probe.supports_format(codec, pixel_format, false)); + assert!(!probe.supports_format(codec, pixel_format, true)); + } + } + probe.h265_10bit = true; + assert!(probe.supports_format(VideoCodec::H265, VideoPixelFormat::P010, false)); + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::P010, true)); + assert!(!probe.supports_format(VideoCodec::Av1, VideoPixelFormat::P010, false)); + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, false)); + probe.h265_444 = true; + assert!(probe.supports_format(VideoCodec::H265, VideoPixelFormat::Ayuv, false)); + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, false)); + probe.h265_10bit_444 = true; + assert!(probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, false)); + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, true)); + assert!(!probe.supports_format(VideoCodec::Av1, VideoPixelFormat::Y410, false)); + probe.d3d11_presentation = false; + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, false)); + } + + #[test] + fn hdr_444_requires_its_own_probe_and_live_gpu_presentation() { + let mut probe = WindowsBackend::probe(); + probe.d3d11_presentation = true; + probe.h265_hardware_decode = true; + probe.h265_hdr = true; + probe.h265_10bit_444 = true; + probe.h265_hdr_444 = false; + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, true)); + probe.h265_hdr_444 = true; + assert!(probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, true)); + assert!(!probe.supports_format(VideoCodec::Av1, VideoPixelFormat::Y410, true)); + probe.d3d11_presentation = false; + assert!(!probe.supports_format(VideoCodec::H265, VideoPixelFormat::Y410, true)); + } + + #[test] + fn endpoint_change_is_reported_once_per_transition() { + let mut tracker = DefaultEndpointTracker::new("speakers".to_owned()); + + assert!(!tracker.observe("speakers".to_owned())); + assert!(tracker.observe("headset".to_owned())); + assert!(!tracker.observe("headset".to_owned())); + assert!(!tracker.observe("speakers".to_owned())); + assert!(tracker.observe("headset".to_owned())); + } + + #[test] + fn lifecycle_state_round_trips_atomic_values() { + for state in [ + LifecycleState::Starting, + LifecycleState::Running, + LifecycleState::Reconfiguring, + LifecycleState::Recovering, + LifecycleState::Stopping, + LifecycleState::Stopped, + LifecycleState::Failed, + ] { + assert_eq!(LifecycleState::from_raw(state as u8), state); + } + } + + #[test] + fn stop_closes_media_queues_and_is_idempotent() { + let (backend, shared) = test_backend(LifecycleState::Running); + + backend.stop(); + backend.stop(); + + assert_eq!(backend.state(), LifecycleState::Stopped); + assert!(shared.video.is_closed()); + assert!(shared.audio.is_closed()); + + shared + .state + .store(LifecycleState::Failed as u8, Ordering::Release); + backend.stop(); + assert_eq!(backend.state(), LifecycleState::Failed); + } + + #[test] + fn surface_reconfiguration_keeps_accepting_media_frames() { + let (backend, shared) = test_backend(LifecycleState::Reconfiguring); + let video_outcome = backend.submit_video(EncodedVideoFrame { + codec: VideoCodec::H264, + data: vec![0, 0, 0, 1, 0x67], + timestamp_100ns: 0, + duration_100ns: 166_667, + key_frame: true, + reset_decoder: false, + }); + let audio_outcome = backend.submit_audio(PcmFrame { + samples: vec![0.0, 0.0], + format: AudioFormat { + sample_rate: 48_000, + channels: 2, + }, + }); + + assert_eq!(video_outcome, Ok(PushOutcome::Queued)); + assert_eq!(audio_outcome, Ok(PushOutcome::Queued)); + assert!(shared.video.try_pop().is_some()); + assert!(shared.audio.try_pop().is_some()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/queue.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/queue.rs new file mode 100644 index 000000000..a46a62ce8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/queue.rs @@ -0,0 +1,493 @@ +use std::collections::VecDeque; +use std::sync::{Condvar, Mutex}; +use std::time::Duration; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PushOutcome { + Queued, + DroppedOldest, + /// The incoming delta was dropped and the queue was left unchanged because + /// it already holds a keyframe. Clearing that keyframe to admit one more + /// P-frame makes the decoder wait forever for a reference it just lost. + Backpressured, + Paused, +} + +/// How one compressed access unit is admitted into a bounded decode queue. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompressedAdmit { + Append, + ReplaceWithKeyframe { dropped: usize }, + PreserveQueuedKeyframe, + DiscardChain { dropped: usize }, +} + +/// A full queue that already holds a keyframe keeps that keyframe. An incoming +/// keyframe still replaces the stale chain. A full queue with no keyframe is +/// discarded, because every remaining delta depends on a frame that will not +/// be decoded. +pub fn admit_compressed_frame( + len: usize, + capacity: usize, + incoming_is_keyframe: bool, + queue_has_keyframe: bool, +) -> CompressedAdmit { + if len < capacity { + return CompressedAdmit::Append; + } + if incoming_is_keyframe { + return CompressedAdmit::ReplaceWithKeyframe { dropped: len }; + } + if queue_has_keyframe { + return CompressedAdmit::PreserveQueuedKeyframe; + } + CompressedAdmit::DiscardChain { + dropped: len.saturating_add(1), + } +} + +/// Reference state of a compressed queue after it dropped a delta. Every later +/// delta depends on the dropped frame, so none may be queued until a keyframe +/// arrives. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum ReferenceGap { + #[default] + Intact, + /// The queue still holds the keyframe in front of the gap. A replacement + /// requested now arrives before the decoder dequeues that keyframe and + /// evicts it, so slow decoders would request one keyframe per arrival. + BehindQueuedKeyframe, + KeyframeRequested, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GapAdmit { + Admit, + Discard, + DiscardAndRequestKeyframe, +} + +impl ReferenceGap { + pub fn admit( + &mut self, + incoming_is_keyframe: bool, + queue_has_keyframe: impl FnOnce() -> bool, + ) -> GapAdmit { + if incoming_is_keyframe { + *self = Self::Intact; + return GapAdmit::Admit; + } + match self { + Self::Intact => GapAdmit::Admit, + Self::BehindQueuedKeyframe if queue_has_keyframe() => GapAdmit::Discard, + Self::BehindQueuedKeyframe => { + *self = Self::KeyframeRequested; + GapAdmit::DiscardAndRequestKeyframe + } + Self::KeyframeRequested => GapAdmit::Discard, + } + } + + pub fn record(&mut self, admitted: CompressedAdmit) { + match admitted { + CompressedAdmit::PreserveQueuedKeyframe => *self = Self::BehindQueuedKeyframe, + CompressedAdmit::DiscardChain { .. } => *self = Self::KeyframeRequested, + CompressedAdmit::Append | CompressedAdmit::ReplaceWithKeyframe { .. } => {} + } + } +} + +#[derive(Debug)] +struct Inner { + values: VecDeque, + closed: bool, +} + +#[derive(Debug)] +pub(crate) struct BoundedQueue { + capacity: usize, + inner: Mutex>, + ready: Condvar, +} + +#[cfg_attr(not(windows), allow(dead_code))] +impl BoundedQueue { + pub(crate) fn new(capacity: usize) -> Self { + assert!(capacity > 0, "queue capacity must be non-zero"); + Self { + capacity, + inner: Mutex::new(Inner { + values: VecDeque::with_capacity(capacity), + closed: false, + }), + ready: Condvar::new(), + } + } + + pub(crate) fn push(&self, value: T) -> Result { + let mut inner = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + if inner.closed { + return Err(value); + } + let outcome = if inner.values.len() == self.capacity { + inner.values.pop_front(); + PushOutcome::DroppedOldest + } else { + PushOutcome::Queued + }; + inner.values.push_back(value); + self.ready.notify_one(); + Ok(outcome) + } + + /// Queues compressed inter-frame video without creating a broken reference + /// chain. A full queue with no keyframe is discarded: retaining a delta + /// after dropping its reference makes the decoder reject the stream. An + /// incoming keyframe replaces that stale chain. A full queue that already + /// holds a keyframe stays intact (`Backpressured`); wiping it to admit one + /// more delta drops the only frame the decoder can restart from. + pub(crate) fn push_or_clear_on_overflow( + &self, + value: T, + incoming_is_keyframe: bool, + queued_is_keyframe: impl Fn(&T) -> bool, + ) -> Result { + let mut inner = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + if inner.closed { + return Err(value); + } + let queue_has_keyframe = inner.values.iter().any(queued_is_keyframe); + match admit_compressed_frame( + inner.values.len(), + self.capacity, + incoming_is_keyframe, + queue_has_keyframe, + ) { + CompressedAdmit::Append => { + inner.values.push_back(value); + self.ready.notify_one(); + Ok(PushOutcome::Queued) + } + CompressedAdmit::ReplaceWithKeyframe { .. } => { + inner.values.clear(); + inner.values.push_back(value); + self.ready.notify_one(); + Ok(PushOutcome::DroppedOldest) + } + CompressedAdmit::PreserveQueuedKeyframe => Ok(PushOutcome::Backpressured), + CompressedAdmit::DiscardChain { .. } => { + inner.values.clear(); + Ok(PushOutcome::DroppedOldest) + } + } + } + + pub(crate) fn try_pop(&self) -> Option { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .values + .pop_front() + } + + pub(crate) fn pop_timeout(&self, timeout: Duration) -> Option { + let inner = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + let mut inner = self + .ready + .wait_timeout_while(inner, timeout, |inner| { + inner.values.is_empty() && !inner.closed + }) + .unwrap_or_else(|error| error.into_inner()) + .0; + inner.values.pop_front() + } + + /// Waits for actionable input, shutdown, or the next decoder-output poll. + /// Queued input cannot wake a decoder that has no input credits. In that + /// case even producer notifications must leave it asleep until the bounded + /// poll deadline (or shutdown), instead of spinning on a nonempty queue. + pub(crate) fn wait_for_decoder(&self, timeout: Duration, accepts_input: bool) -> bool { + let inner = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + let inner = self + .ready + .wait_timeout_while(inner, timeout, |inner| { + (!accepts_input || inner.values.is_empty()) && !inner.closed + }) + .unwrap_or_else(|error| error.into_inner()) + .0; + !inner.values.is_empty() + } + + pub(crate) fn clear(&self) { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .values + .clear(); + } + + pub(crate) fn any(&self, predicate: impl Fn(&T) -> bool) -> bool { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .values + .iter() + .any(predicate) + } + + pub(crate) fn len(&self) -> usize { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .values + .len() + } + + pub(crate) fn close(&self) { + let mut inner = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + inner.closed = true; + inner.values.clear(); + self.ready.notify_all(); + } + + #[cfg(test)] + pub(crate) fn is_closed(&self) -> bool { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .closed + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn drops_oldest_value_when_full() { + let queue = BoundedQueue::new(2); + assert_eq!(queue.push(1), Ok(PushOutcome::Queued)); + assert_eq!(queue.push(2), Ok(PushOutcome::Queued)); + assert_eq!(queue.push(3), Ok(PushOutcome::DroppedOldest)); + assert_eq!(queue.try_pop(), Some(2)); + assert_eq!(queue.try_pop(), Some(3)); + } + + #[test] + fn video_overflow_discards_the_pending_reference_chain() { + let queue = BoundedQueue::new(2); + assert_eq!( + queue.push_or_clear_on_overflow(1, false, |_| false), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(2, false, |_| false), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(3, false, |_| false), + Ok(PushOutcome::DroppedOldest) + ); + assert_eq!(queue.try_pop(), None); + } + + #[test] + fn video_overflow_retains_an_incoming_recovery_keyframe() { + let queue = BoundedQueue::new(2); + assert_eq!( + queue.push_or_clear_on_overflow(1, false, |_| false), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(2, false, |_| false), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(3, true, |_| false), + Ok(PushOutcome::DroppedOldest) + ); + assert_eq!(queue.try_pop(), Some(3)); + assert_eq!(queue.try_pop(), None); + } + + #[test] + fn video_overflow_keeps_a_queued_keyframe_when_a_delta_does_not_fit() { + let queue = BoundedQueue::new(2); + assert_eq!( + queue.push_or_clear_on_overflow(-1, true, is_keyframe), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(2, false, is_keyframe), + Ok(PushOutcome::Queued) + ); + assert_eq!( + queue.push_or_clear_on_overflow(3, false, is_keyframe), + Ok(PushOutcome::Backpressured) + ); + assert_eq!(queue.try_pop(), Some(-1)); + assert_eq!(queue.try_pop(), Some(2)); + assert_eq!(queue.try_pop(), None); + } + + #[test] + fn admit_compressed_frame_preserves_a_queued_keyframe() { + assert_eq!( + admit_compressed_frame(7, 7, false, true), + CompressedAdmit::PreserveQueuedKeyframe + ); + assert_eq!( + admit_compressed_frame(7, 7, true, true), + CompressedAdmit::ReplaceWithKeyframe { dropped: 7 } + ); + assert_eq!( + admit_compressed_frame(7, 7, false, false), + CompressedAdmit::DiscardChain { dropped: 8 } + ); + assert_eq!( + admit_compressed_frame(3, 7, false, false), + CompressedAdmit::Append + ); + } + + fn is_keyframe(value: &i32) -> bool { + *value < 0 + } + + fn submit(queue: &BoundedQueue, gap: &mut ReferenceGap, value: i32) -> GapAdmit { + let keyframe = is_keyframe(&value); + let admitted = gap.admit(keyframe, || queue.any(is_keyframe)); + if admitted == GapAdmit::Admit { + match queue.push_or_clear_on_overflow(value, keyframe, is_keyframe) { + Ok(PushOutcome::Backpressured) => { + gap.record(CompressedAdmit::PreserveQueuedKeyframe) + } + Ok(PushOutcome::DroppedOldest) if !keyframe => { + gap.record(CompressedAdmit::DiscardChain { dropped: 0 }) + } + _ => {} + } + } + admitted + } + + #[test] + fn queue_reports_whether_any_value_matches() { + let queue = BoundedQueue::new(3); + queue.push(1).unwrap(); + assert!(!queue.any(is_keyframe)); + queue.push(-2).unwrap(); + assert!(queue.any(is_keyframe)); + assert_eq!(queue.try_pop(), Some(1)); + assert_eq!(queue.try_pop(), Some(-2)); + assert!(!queue.any(is_keyframe)); + } + + #[test] + fn stalled_decoder_does_not_request_a_keyframe_per_arrival() { + let queue = BoundedQueue::new(3); + let mut gap = ReferenceGap::default(); + for value in [-1, 2, 3] { + assert_eq!(submit(&queue, &mut gap, value), GapAdmit::Admit); + } + assert_eq!(submit(&queue, &mut gap, 4), GapAdmit::Admit); + assert_eq!(gap, ReferenceGap::BehindQueuedKeyframe); + for value in 5..300 { + assert_eq!(submit(&queue, &mut gap, value), GapAdmit::Discard); + } + assert_eq!(queue.try_pop(), Some(-1)); + assert_eq!( + submit(&queue, &mut gap, 300), + GapAdmit::DiscardAndRequestKeyframe + ); + assert_eq!(submit(&queue, &mut gap, 301), GapAdmit::Discard); + assert_eq!(queue.try_pop(), Some(2)); + assert_eq!(queue.try_pop(), Some(3)); + assert_eq!(queue.try_pop(), None); + assert_eq!(submit(&queue, &mut gap, 302), GapAdmit::Discard); + assert_eq!(submit(&queue, &mut gap, -303), GapAdmit::Admit); + assert_eq!(gap, ReferenceGap::Intact); + assert_eq!(submit(&queue, &mut gap, 304), GapAdmit::Admit); + assert_eq!(queue.try_pop(), Some(-303)); + assert_eq!(queue.try_pop(), Some(304)); + } + + #[test] + fn keyframe_arriving_behind_a_queued_keyframe_closes_the_gap() { + let queue = BoundedQueue::new(2); + let mut gap = ReferenceGap::default(); + submit(&queue, &mut gap, -1); + submit(&queue, &mut gap, 2); + submit(&queue, &mut gap, 3); + assert_eq!(gap, ReferenceGap::BehindQueuedKeyframe); + assert_eq!(submit(&queue, &mut gap, -4), GapAdmit::Admit); + assert_eq!(gap, ReferenceGap::Intact); + assert_eq!(queue.try_pop(), Some(-4)); + assert_eq!(queue.try_pop(), None); + } + + #[test] + fn discarded_chain_drops_deltas_until_a_keyframe_without_new_requests() { + let queue = BoundedQueue::new(2); + let mut gap = ReferenceGap::default(); + submit(&queue, &mut gap, 1); + submit(&queue, &mut gap, 2); + assert_eq!(submit(&queue, &mut gap, 3), GapAdmit::Admit); + assert_eq!(gap, ReferenceGap::KeyframeRequested); + assert_eq!(queue.try_pop(), None); + assert_eq!(submit(&queue, &mut gap, 4), GapAdmit::Discard); + assert_eq!(queue.try_pop(), None); + assert_eq!(submit(&queue, &mut gap, -5), GapAdmit::Admit); + assert_eq!(queue.try_pop(), Some(-5)); + } + + #[test] + fn close_discards_values_and_rejects_writes() { + let queue = BoundedQueue::new(2); + queue.push(1).unwrap(); + queue.close(); + assert!(queue.is_closed()); + assert_eq!(queue.try_pop(), None); + assert_eq!(queue.push(2), Err(2)); + } + + #[test] + fn timeout_returns_without_a_value() { + let queue = BoundedQueue::::new(1); + assert_eq!(queue.pop_timeout(Duration::from_millis(1)), None); + } + + #[test] + fn decoder_wait_does_not_consume_actionable_input() { + let queue = BoundedQueue::new(1); + queue.push(7).unwrap(); + assert!(queue.wait_for_decoder(Duration::from_secs(1), true)); + assert_eq!(queue.try_pop(), Some(7)); + } + + #[test] + fn decoder_backpressure_waits_even_with_queued_input() { + let queue = BoundedQueue::new(1); + queue.push(7).unwrap(); + let started = std::time::Instant::now(); + let interval = Duration::from_millis(20); + assert!(queue.wait_for_decoder(interval, false)); + assert!(started.elapsed() >= interval); + assert_eq!(queue.try_pop(), Some(7)); + } + + #[test] + fn closing_queue_interrupts_decoder_backpressure_wait() { + let queue = std::sync::Arc::new(BoundedQueue::new(1)); + queue.push(7).unwrap(); + let worker_queue = std::sync::Arc::clone(&queue); + let (sent, received) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + worker_queue.wait_for_decoder(Duration::from_secs(30), false); + sent.send(()).unwrap(); + }); + queue.close(); + received.recv_timeout(Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/adapter_index.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/adapter_index.rs new file mode 100644 index 000000000..205b9bd9f --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/adapter_index.rs @@ -0,0 +1,233 @@ +use std::collections::HashSet; + +use ::windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE_UNKNOWN, D3D_FEATURE_LEVEL_10_0, D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1, +}; +use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0, D3D11_DECODER_PROFILE_H264_VLD_FGT, + D3D11_DECODER_PROFILE_H264_VLD_NOFGT, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN, + D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10, D3D11_SDK_VERSION, D3D11_VIDEO_DECODER_DESC, + D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11VideoDevice, +}; +use ::windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_P010}; +use ::windows::Win32::Graphics::Dxgi::{ + CreateDXGIFactory1, DXGI_ADAPTER_FLAG_REMOTE, DXGI_ADAPTER_FLAG_SOFTWARE, DXGI_ERROR_NOT_FOUND, + DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, IDXGIAdapter, IDXGIAdapter1, IDXGIFactory1, + IDXGIFactory6, +}; +use ::windows::core::{BOOL, GUID, Interface}; + +use crate::adapter_decode::{AdapterDecodeIndex, DecodeProfile}; + +const PROFILE_WIDTHS: [(u32, u32); 2] = [(1920, 1080), (1280, 720)]; + +pub(crate) fn probe_adapter_decode() -> Vec { + let adapters = match enumerate_adapters() { + Ok(adapters) => adapters, + Err(error) => { + video_log!("Windows graphics adapter decode index failed: {error}"); + return Vec::new(); + } + }; + let mut indexed = Vec::with_capacity(adapters.len()); + for (adapter, name, luid) in adapters { + let mut index = AdapterDecodeIndex::named(luid, name); + match query_decode_profiles(&adapter) { + Ok(profiles) => { + for (profile, supported) in profiles { + index.apply_profile(profile, supported); + } + index.finish(); + } + Err(error) => { + index.reason = Some(bounded_reason(error)); + } + } + video_log!( + "Windows adapter decode index name={} h264={} h265={} h265_main10={} av1={} reason={}", + index.name, + index.h264, + index.h265, + index.h265_main10, + index.av1, + index.reason.as_deref().unwrap_or("") + ); + indexed.push(index); + } + indexed +} + +fn enumerate_adapters() -> Result, String> { + unsafe { + let factory: IDXGIFactory1 = + CreateDXGIFactory1().map_err(|error| format!("CreateDXGIFactory1: {error}"))?; + let preferred: Option = factory.cast().ok(); + let mut adapters = Vec::new(); + let mut seen = HashSet::new(); + for index in 0..64u32 { + let adapter = if let Some(preferred) = &preferred { + match preferred.EnumAdapterByGpuPreference::( + index, + DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, + ) { + Ok(adapter) => adapter, + Err(error) if error.code() == DXGI_ERROR_NOT_FOUND => break, + Err(error) => { + return Err(format!("EnumAdapterByGpuPreference: {error}")); + } + } + } else { + match factory.EnumAdapters1(index) { + Ok(adapter) => adapter, + Err(error) if error.code() == DXGI_ERROR_NOT_FOUND => break, + Err(error) => return Err(format!("EnumAdapters1: {error}")), + } + }; + let description = match adapter.GetDesc1() { + Ok(description) => description, + Err(error) => { + video_log!("Windows adapter {index} description failed: {error}"); + continue; + } + }; + let excluded = (DXGI_ADAPTER_FLAG_SOFTWARE.0 | DXGI_ADAPTER_FLAG_REMOTE.0) as u32; + if description.Flags & excluded != 0 { + continue; + } + let luid = ((description.AdapterLuid.HighPart as u32 as u64) << 32) + | u64::from(description.AdapterLuid.LowPart); + if luid == 0 || !seen.insert(luid) { + continue; + } + let name = adapter_name(&description.Description); + adapters.push((adapter, name, luid)); + } + Ok(adapters) + } +} + +fn query_decode_profiles(adapter: &IDXGIAdapter1) -> Result, String> { + let dxgi_adapter: IDXGIAdapter = adapter + .cast() + .map_err(|error| format!("DXGI adapter interface: {error}"))?; + let (device, _context) = create_video_device(&dxgi_adapter)?; + let video: ID3D11VideoDevice = device + .cast() + .map_err(|error| format!("D3D11 video device: {error}"))?; + let count = unsafe { video.GetVideoDecoderProfileCount() }.min(64); + let mut profiles = Vec::new(); + for index in 0..count { + let profile = match unsafe { video.GetVideoDecoderProfile(index) } { + Ok(profile) => profile, + Err(error) => { + video_log!("Windows decoder profile {index} failed: {error}"); + continue; + } + }; + let kind = classify_profile(profile); + if kind == DecodeProfile::Other { + continue; + } + let format = if kind == DecodeProfile::H265Main10 { + DXGI_FORMAT_P010 + } else { + DXGI_FORMAT_NV12 + }; + let supported = profile_supported(&video, profile, format) + || (kind == DecodeProfile::Av1 && profile_supported(&video, profile, DXGI_FORMAT_P010)); + profiles.push((kind, supported)); + } + Ok(profiles) +} + +fn create_video_device( + adapter: &IDXGIAdapter, +) -> Result<(ID3D11Device, ID3D11DeviceContext), String> { + let levels = [ + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + ]; + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + Some(adapter), + D3D_DRIVER_TYPE_UNKNOWN, + ::windows::Win32::Foundation::HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .map_err(|error| format!("D3D11CreateDevice: {error}"))?; + } + Ok(( + device.ok_or("D3D11CreateDevice returned no device")?, + context.ok_or("D3D11CreateDevice returned no immediate context")?, + )) +} + +fn classify_profile(profile: GUID) -> DecodeProfile { + if profile == D3D11_DECODER_PROFILE_H264_VLD_NOFGT + || profile == D3D11_DECODER_PROFILE_H264_VLD_FGT + { + DecodeProfile::H264 + } else if profile == D3D11_DECODER_PROFILE_HEVC_VLD_MAIN { + DecodeProfile::H265Main + } else if profile == D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10 { + DecodeProfile::H265Main10 + } else if profile == D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0 { + DecodeProfile::Av1 + } else { + DecodeProfile::Other + } +} + +fn profile_supported(video: &ID3D11VideoDevice, profile: GUID, format: DXGI_FORMAT) -> bool { + unsafe { + let supported = video + .CheckVideoDecoderFormat(&profile, format) + .unwrap_or(BOOL(0)); + if !supported.as_bool() { + return false; + } + PROFILE_WIDTHS.iter().any(|(width, height)| { + let description = D3D11_VIDEO_DECODER_DESC { + Guid: profile, + SampleWidth: *width, + SampleHeight: *height, + OutputFormat: format, + }; + video.GetVideoDecoderConfigCount(&description).unwrap_or(0) > 0 + }) + } +} + +fn adapter_name(description: &[u16]) -> String { + let end = description + .iter() + .position(|unit| *unit == 0) + .unwrap_or(description.len()); + crate::adapter_decode::sanitize_adapter_name(&String::from_utf16_lossy(&description[..end])) +} + +fn bounded_reason(value: impl AsRef) -> String { + value + .as_ref() + .chars() + .map(|character| { + if character.is_control() { + ' ' + } else { + character + } + }) + .take(180) + .collect() +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/audio.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/audio.rs new file mode 100644 index 000000000..a6c0bb676 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/audio.rs @@ -0,0 +1,371 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use ::windows::Win32::Media::Audio::{ + AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM, + AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, IAudioClient, IAudioRenderClient, IMMDeviceEnumerator, + MMDeviceEnumerator, WAVEFORMATEX, eConsole, eRender, +}; +use ::windows::Win32::System::Com::{CLSCTX_ALL, CoCreateInstance, CoTaskMemFree}; + +use crate::queue::BoundedQueue; +use crate::{AudioFormat, DefaultEndpointTracker, PcmFrame}; + +const WAVE_FORMAT_IEEE_FLOAT: u16 = 3; +const BUFFER_DURATION_100NS: i64 = 200_000; +const ENDPOINT_CHECK_INTERVAL: Duration = Duration::from_millis(250); +const INITIAL_PREROLL_MS: u32 = 80; +const MIN_PREROLL_MS: u32 = 60; +const MAX_PREROLL_MS: u32 = 200; +const PREROLL_ADJUSTMENT_MS: u32 = 5; +const PREROLL_STABLE_INTERVAL: Duration = Duration::from_secs(20); +const MAX_PENDING_AUDIO_MS: usize = 200; + +struct AdaptivePreroll { + target_ms: u32, + stable_since: Instant, + underrun_active: bool, +} + +impl AdaptivePreroll { + fn new(now: Instant) -> Self { + Self { + target_ms: INITIAL_PREROLL_MS, + stable_since: now, + underrun_active: false, + } + } + + fn target_frames(&self, sample_rate: u32) -> usize { + sample_rate as usize * self.target_ms as usize / 1_000 + } + + fn on_started(&mut self, now: Instant) { + self.stable_since = now; + self.underrun_active = false; + } + + fn on_underrun(&mut self, now: Instant) -> bool { + if self.underrun_active { + return false; + } + self.underrun_active = true; + let previous = self.target_ms; + self.target_ms = self + .target_ms + .saturating_add(PREROLL_ADJUSTMENT_MS) + .min(MAX_PREROLL_MS); + self.stable_since = now; + self.target_ms != previous + } + + fn on_buffered(&mut self) { + self.underrun_active = false; + } + + fn observe_stable(&mut self, now: Instant) -> bool { + if self.target_ms == MIN_PREROLL_MS + || now.saturating_duration_since(self.stable_since) < PREROLL_STABLE_INTERVAL + { + return false; + } + self.target_ms = self + .target_ms + .saturating_sub(PREROLL_ADJUSTMENT_MS) + .max(MIN_PREROLL_MS); + self.stable_since = now; + true + } +} + +pub(super) struct AudioRenderer { + enumerator: IMMDeviceEnumerator, + render: IAudioRenderClient, + client: IAudioClient, + buffer_frames: u32, + format: AudioFormat, + pending: VecDeque, + pending_capacity: usize, + paused: bool, + started: bool, + preroll: AdaptivePreroll, + endpoint_tracker: DefaultEndpointTracker, + next_endpoint_check: Instant, +} + +impl AudioRenderer { + pub(super) fn probe() -> Result<(), String> { + let mut renderer = Self::new(AudioFormat { + sample_rate: 48_000, + channels: 2, + })?; + unsafe { + renderer.client.Start().map_err(|error| error.to_string())?; + } + renderer.started = true; + renderer.stop(); + Ok(()) + } + + pub(super) fn new(format: AudioFormat) -> Result { + let block_align = format + .channels + .checked_mul(4) + .ok_or("PCM block alignment overflow")?; + let wave_format = WAVEFORMATEX { + wFormatTag: WAVE_FORMAT_IEEE_FLOAT, + nChannels: format.channels, + nSamplesPerSec: format.sample_rate, + nAvgBytesPerSec: format + .sample_rate + .checked_mul(block_align as u32) + .ok_or("PCM byte rate overflow")?, + nBlockAlign: block_align, + wBitsPerSample: 32, + cbSize: 0, + }; + + unsafe { + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) + .map_err(|error| error.to_string())?; + let device = enumerator + .GetDefaultAudioEndpoint(eRender, eConsole) + .map_err(|error| error.to_string())?; + let endpoint_id = device_id(&device)?; + let client: IAudioClient = device + .Activate(CLSCTX_ALL, None) + .map_err(|error| error.to_string())?; + client + .Initialize( + AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, + BUFFER_DURATION_100NS, + 0, + &wave_format, + None, + ) + .map_err(|error| error.to_string())?; + let buffer_frames = client.GetBufferSize().map_err(|error| error.to_string())?; + let render: IAudioRenderClient = + client.GetService().map_err(|error| error.to_string())?; + let now = Instant::now(); + let pending_frames = (format.sample_rate as usize * MAX_PENDING_AUDIO_MS / 1_000) + .max(buffer_frames as usize * 4); + Ok(Self { + enumerator, + render, + client, + buffer_frames, + format, + pending: VecDeque::new(), + pending_capacity: pending_frames * format.channels as usize, + paused: false, + started: false, + preroll: AdaptivePreroll::new(now), + endpoint_tracker: DefaultEndpointTracker::new(endpoint_id), + next_endpoint_check: now + ENDPOINT_CHECK_INTERVAL, + }) + } + } + + pub(super) fn default_endpoint_changed(&mut self) -> Result { + let now = Instant::now(); + if now < self.next_endpoint_check { + return Ok(false); + } + self.next_endpoint_check = now + ENDPOINT_CHECK_INTERVAL; + let device = unsafe { + self.enumerator + .GetDefaultAudioEndpoint(eRender, eConsole) + .map_err(|error| error.to_string())? + }; + let endpoint_id = device_id(&device)?; + Ok(self.endpoint_tracker.observe(endpoint_id)) + } + + pub(super) fn render( + &mut self, + queue: &BoundedQueue, + muted: &std::sync::atomic::AtomicBool, + ) -> Result { + let mut dropped = false; + while let Some(frame) = queue.try_pop() { + if frame.format != self.format { + return Err(format!( + "PCM packet format {:?} does not match active format {:?}", + frame.format, self.format + )); + } + self.pending.extend(frame.samples); + while self.pending.len() > self.pending_capacity { + self.pending.pop_front(); + dropped = true; + } + } + if self.paused { + return Ok(dropped); + } + + unsafe { + let padding = self + .client + .GetCurrentPadding() + .map_err(|error| error.to_string())?; + if self.started && padding == 0 { + if self.preroll.on_underrun(Instant::now()) { + eprintln!( + "WASAPI underrun: increasing audio pre-roll to {}ms", + self.preroll.target_ms + ); + } + // Shared-mode WASAPI already renders silence while starved. + // Keep its clock running and refill it below instead of + // Stop+Reset, which turns a short network gap into a click and + // another full pre-roll delay. + } else if padding != 0 { + self.preroll.on_buffered(); + } + + let pending_frames = self.pending.len() / self.format.channels as usize; + if !self.started && pending_frames < self.preroll.target_frames(self.format.sample_rate) + { + return Ok(dropped); + } + + let available_frames = self.buffer_frames.saturating_sub(padding); + let write_frames = available_frames.min(pending_frames as u32); + if write_frames == 0 { + return Ok(dropped); + } + let sample_count = write_frames as usize * self.format.channels as usize; + let destination = self + .render + .GetBuffer(write_frames) + .map_err(|error| error.to_string())? as *mut f32; + let muted = muted.load(std::sync::atomic::Ordering::Acquire); + for index in 0..sample_count { + let sample = self.pending.pop_front().unwrap_or(0.0); + destination + .add(index) + .write(if muted { 0.0 } else { sample }); + } + self.render + .ReleaseBuffer(write_frames, 0) + .map_err(|error| error.to_string())?; + if !self.started { + self.client.Start().map_err(|error| error.to_string())?; + self.started = true; + self.preroll.on_started(Instant::now()); + } else if self.preroll.observe_stable(Instant::now()) { + eprintln!( + "WASAPI playback stable: reducing audio pre-roll to {}ms", + self.preroll.target_ms + ); + } + } + Ok(dropped) + } + + pub(super) fn stop(&mut self) { + unsafe { + if self.started { + let _ = self.client.Stop(); + } + } + self.pending.clear(); + self.started = false; + self.paused = true; + } + + pub(super) fn set_paused(&mut self, paused: bool) -> Result<(), String> { + if self.paused == paused { + return Ok(()); + } + self.pending.clear(); + unsafe { + if paused { + if self.started { + self.client.Stop().map_err(|error| error.to_string())?; + } + self.client.Reset().map_err(|error| error.to_string())?; + } + } + self.started = false; + if !paused { + // Playback restarts only after a short pre-roll. Starting an empty + // shared-mode endpoint is the source of the repeated startup and + // post-focus audio glitches this path used to produce. + self.preroll.on_started(Instant::now()); + } + self.paused = paused; + Ok(()) + } +} + +impl Drop for AudioRenderer { + fn drop(&mut self) { + self.stop(); + } +} + +fn device_id(device: &::windows::Win32::Media::Audio::IMMDevice) -> Result { + unsafe { + let id = device.GetId().map_err(|error| error.to_string())?; + let result = id.to_string().map_err(|error| error.to_string()); + CoTaskMemFree(Some(id.0.cast())); + result + } +} + +#[cfg(test)] +mod tests { + use super::{ + AdaptivePreroll, INITIAL_PREROLL_MS, MAX_PREROLL_MS, MIN_PREROLL_MS, PREROLL_ADJUSTMENT_MS, + PREROLL_STABLE_INTERVAL, + }; + use std::time::{Duration, Instant}; + + #[test] + fn preroll_increases_on_underrun_and_is_bounded() { + let now = Instant::now(); + let mut preroll = AdaptivePreroll::new(now); + assert_eq!(preroll.target_frames(48_000), 3_840); + assert_eq!(preroll.target_ms, INITIAL_PREROLL_MS); + + for step in 1..=32 { + preroll.on_underrun(now + Duration::from_millis(step)); + preroll.on_buffered(); + } + assert_eq!(preroll.target_ms, MAX_PREROLL_MS); + } + + #[test] + fn one_underrun_episode_only_adjusts_the_target_once() { + let now = Instant::now(); + let mut preroll = AdaptivePreroll::new(now); + + assert!(preroll.on_underrun(now)); + assert!(!preroll.on_underrun(now + Duration::from_millis(1))); + assert_eq!( + preroll.target_ms, + INITIAL_PREROLL_MS + PREROLL_ADJUSTMENT_MS + ); + + preroll.on_buffered(); + assert!(preroll.on_underrun(now + Duration::from_millis(2))); + } + + #[test] + fn stable_playback_returns_to_the_low_latency_target_gradually() { + let now = Instant::now(); + let mut preroll = AdaptivePreroll::new(now); + preroll.target_ms = MIN_PREROLL_MS + PREROLL_ADJUSTMENT_MS * 2; + + assert!(!preroll.observe_stable(now + PREROLL_STABLE_INTERVAL / 2)); + assert!(preroll.observe_stable(now + PREROLL_STABLE_INTERVAL)); + assert_eq!(preroll.target_ms, MIN_PREROLL_MS + PREROLL_ADJUSTMENT_MS); + assert!(preroll.observe_stable(now + PREROLL_STABLE_INTERVAL * 2)); + assert_eq!(preroll.target_ms, MIN_PREROLL_MS); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/color.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/color.rs new file mode 100644 index 000000000..7c04b6e4c --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/color.rs @@ -0,0 +1,73 @@ +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_COLOR_SPACE_TYPE, DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601, + DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709, DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020, DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709, DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020, +}; + +use crate::{ + VideoChromaFormat, VideoChromaSiting, VideoColorMatrix, VideoFormat, VideoTransferFunction, +}; + +pub(super) fn input_color_space(format: VideoFormat) -> Result { + format.validate_color().map_err(|error| error.to_string())?; + let chroma_siting = if format.chroma_format == VideoChromaFormat::Cs444 { + match format.transfer_function { + VideoTransferFunction::Hlg => VideoChromaSiting::TopLeft, + _ => VideoChromaSiting::Left, + } + } else { + format.chroma_siting + }; + match ( + format.transfer_function, + chroma_siting, + format.color_matrix, + format.full_range, + ) { + (VideoTransferFunction::Pq, VideoChromaSiting::Left, _, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020) + } + (VideoTransferFunction::Pq, VideoChromaSiting::TopLeft, _, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020) + } + (VideoTransferFunction::Hlg, VideoChromaSiting::TopLeft, _, true) => { + Ok(DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020) + } + (VideoTransferFunction::Hlg, VideoChromaSiting::TopLeft, _, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt601, true) => { + Ok(DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt601, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt709, true) => { + Ok(DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt709, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt2020, true) => { + Ok(DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020) + } + (VideoTransferFunction::Sdr, VideoChromaSiting::Left, VideoColorMatrix::Bt2020, false) => { + Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020) + } + ( + VideoTransferFunction::Sdr, + VideoChromaSiting::TopLeft, + VideoColorMatrix::Bt2020, + false, + ) => Ok(DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020), + _ => Err(format!( + "no DXGI input color space for {:?} {:?} {:?} fullRange={}", + format.transfer_function, format.chroma_siting, format.color_matrix, format.full_range + )), + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/decoder.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/decoder.rs new file mode 100644 index 000000000..3fb45153c --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/decoder.rs @@ -0,0 +1,2085 @@ +use std::collections::VecDeque; +use std::ffi::c_void; +use std::mem::ManuallyDrop; +use std::mem::size_of; +use std::ptr; + +use ::windows::Win32::Foundation::{E_NOTIMPL, LUID}; +use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0, D3D11_DECODER_PROFILE_H264_VLD_FGT, + D3D11_DECODER_PROFILE_H264_VLD_NOFGT, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN, + D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10, D3D11_TEXTURE2D_DESC, D3D11_VIDEO_DECODER_DESC, + ID3D11Texture2D, ID3D11VideoDevice, +}; +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_FORMAT, DXGI_FORMAT_AYUV, DXGI_FORMAT_NV12, DXGI_FORMAT_P010, DXGI_FORMAT_Y410, +}; +use ::windows::Win32::Media::MediaFoundation::{ + D3D12_VIDEO_DECODE_PROFILE_HEVC_MAIN_444, D3D12_VIDEO_DECODE_PROFILE_HEVC_MAIN10_444, + MF_MT_TRANSFER_FUNCTION, MF_MT_VIDEO_CHROMA_SITING, MF_MT_VIDEO_PRIMARIES, MF_MT_YUV_MATRIX, + MFNominalRange_16_235, MFVideoChromaSubsampling_Cosited, MFVideoChromaSubsampling_MPEG2, + MFVideoChromaSubsampling_ProgressiveChroma, MFVideoPrimaries_BT709, MFVideoPrimaries_BT2020, + MFVideoTransFunc_22, MFVideoTransFunc_709, MFVideoTransFunc_2020, MFVideoTransFunc_2084, + MFVideoTransFunc_HLG, MFVideoTransFunc_sRGB, MFVideoTransferMatrix_BT601, + MFVideoTransferMatrix_BT709, MFVideoTransferMatrix_BT2020_10, +}; +use ::windows::Win32::Media::MediaFoundation::{ + IMFActivate, IMFAttributes, IMFDXGIBuffer, IMFMediaEventGenerator, IMFMediaType, IMFSample, + IMFTransform, METransformHaveOutput, METransformNeedInput, MF_E_ATTRIBUTENOTFOUND, + MF_E_NO_EVENTS_AVAILABLE, MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, + MF_EVENT_FLAG_NO_WAIT, MF_LOW_LATENCY, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, + MF_MT_GEOMETRIC_APERTURE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, + MF_MT_MINIMUM_DISPLAY_APERTURE, MF_MT_MPEG2_PROFILE, MF_MT_PAN_SCAN_APERTURE, + MF_MT_PAN_SCAN_ENABLED, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, MF_MT_VIDEO_NOMINAL_RANGE, + MF_SA_D3D11_AWARE, MF_TRANSFORM_ASYNC, MF_TRANSFORM_ASYNC_UNLOCK, MFCreateAttributes, + MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample, MFMediaType_Video, + MFNominalRange_0_255, MFSampleExtension_CleanPoint, MFSampleExtension_FrameCorruption, + MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_ADAPTER_LUID, MFT_ENUM_FLAG, MFT_ENUM_FLAG_ASYNCMFT, + MFT_ENUM_FLAG_HARDWARE, MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_SYNCMFT, + MFT_ENUM_HARDWARE_URL_Attribute, MFT_FRIENDLY_NAME_Attribute, MFT_MESSAGE_COMMAND_DRAIN, + MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, + MFT_MESSAGE_NOTIFY_END_OF_STREAM, MFT_MESSAGE_NOTIFY_END_STREAMING, + MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, + MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, + MFT_REGISTER_TYPE_INFO, MFTEnum2, MFVideoArea, MFVideoFormat_AV1, MFVideoFormat_AYUV, + MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoFormat_P010, + MFVideoFormat_Y410, MFVideoInterlace_MixedInterlaceOrProgressive, + eAVEncH265VProfile_Main_420_8, eAVEncH265VProfile_Main_420_10, eAVEncH265VProfile_Main_444_8, + eAVEncH265VProfile_Main_444_10, +}; +use ::windows::Win32::System::Com::CoTaskMemFree; +use ::windows::core::Interface; + +use crate::aperture::VideoAperture; +use crate::decoder_color::{ColorDescriptors, Descriptor}; +use crate::queue::BoundedQueue; +use crate::{ + ADAPTIVE_VIDEO_QUEUE_CAPACITY, BackendEvent, EncodedVideoFrame, Subsystem, VideoChromaFormat, + VideoChromaSiting, VideoCodec, VideoColorMatrix, VideoColorPrimaries, VideoFormat, + VideoPixelFormat, VideoTransferFunction, WindowsDecoderMode, +}; + +pub(super) trait DecoderDevice { + fn device_manager(&self) -> &::windows::Win32::Media::MediaFoundation::IMFDXGIDeviceManager; + fn adapter_luid(&self) -> Result; + fn video_format(&self) -> VideoFormat; +} + +pub(super) struct DecodedVideoFrame { + pub(super) format: VideoFormat, + pub(super) aperture: VideoAperture, + pub(super) texture: ID3D11Texture2D, + pub(super) subresource: u32, + pub(super) timestamp_100ns: i64, + pub(super) duration_100ns: i64, + // The sample, not the texture's COM reference, leases the decoder's array + // slice. Releasing it early returns that slice to the MFT's surface pool. + // Keep it alive through the decoded queue and conversion to Qt's RGB target. + // See Microsoft's "Supporting Direct3D 11 Video Decoding in Media Foundation", + // Decoding: the tracked-sample release callback makes the surface reusable. + _sample: IMFSample, +} + +impl DecodedVideoFrame { + pub(super) fn from_sample( + sample: IMFSample, + format: VideoFormat, + aperture: VideoAperture, + preferred_pixel_format: VideoPixelFormat, + ) -> Result { + match unsafe { sample.GetUINT32(&MFSampleExtension_FrameCorruption) } { + Ok(corrupted) if corrupted != 0 => { + return Err("Media Foundation decoder reported a corrupted output frame".to_owned()); + } + Ok(_) => {} + Err(error) if error.code() == MF_E_ATTRIBUTENOTFOUND => {} + Err(error) => return Err(format!("read decoder output corruption flag: {error}")), + } + let (texture, subresource) = dxgi_surface(&sample)?; + let mut description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + texture.GetDesc(&mut description); + } + validate_decoded_output_format( + description.Format, + format.pixel_format, + preferred_pixel_format, + )?; + let timestamp_100ns = unsafe { sample.GetSampleTime().unwrap_or(0) }; + let duration_100ns = unsafe { + sample + .GetSampleDuration() + .ok() + .filter(|duration| *duration > 0) + .unwrap_or(format.frame_duration_100ns()) + }; + Ok(Self { + format, + aperture, + texture, + subresource, + timestamp_100ns, + duration_100ns, + _sample: sample, + }) + } +} + +pub(super) struct Decoder { + events: Option, + transform: IMFTransform, + activation: IMFActivate, + input_stream: u32, + output_stream: u32, + input_credits: u32, + format: VideoFormat, + aperture: VideoAperture, + negotiated_format: VideoFormat, + output_validated: bool, + output_provides_samples: bool, + stopped: bool, +} + +impl Decoder { + pub(super) fn probe( + graphics: &G, + codec: VideoCodec, + mode: WindowsDecoderMode, + ) -> Result<(), String> { + let mut decoder = Self::new( + graphics, + VideoFormat { + codec, + ..graphics.video_format() + }, + mode, + )?; + decoder.validate_output()?; + decoder.stop(); + Ok(()) + } + + pub(super) fn new( + graphics: &G, + format: VideoFormat, + mode: WindowsDecoderMode, + ) -> Result { + if mode == WindowsDecoderMode::Hardware { + ensure_hardware_format(graphics, format)?; + } + let activations = enumerate_decoders(graphics.adapter_luid()?, format.codec, mode)?; + let mut failures = Vec::new(); + for activation in activations { + let friendly_name = activation_string(&activation, &MFT_FRIENDLY_NAME_Attribute) + .unwrap_or_else(|| "unknown".to_owned()); + match configure_transform(&activation, graphics, format) { + Ok(( + transform, + events, + input_stream, + output_stream, + output_provides_samples, + output_format, + aperture, + )) => { + let pixel_format = output_format.pixel_format; + let full_range = output_format.full_range; + let input_credits = u32::from(events.is_none()); + let hardware_registered = + activation_string(&activation, &MFT_ENUM_HARDWARE_URL_Attribute).is_some(); + video_log!( + "Windows decoder configured codec={} mft={friendly_name:?} hardwareRegistered={hardware_registered} async={} output={pixel_format:?} bitDepth={} chroma={:?} range={} sampleLease=retained-through-video-blit", + format.codec.label(), + events.is_some(), + pixel_format.bit_depth(), + chroma_format(pixel_format), + if full_range { "full" } else { "limited" }, + ); + return Ok(Self { + events, + transform, + activation, + input_stream, + output_stream, + input_credits, + format: output_format, + aperture, + negotiated_format: format, + output_validated: false, + output_provides_samples, + stopped: false, + }); + } + Err(error) => { + failures.push(format!("{friendly_name}: {error}")); + unsafe { + let _ = activation.ShutdownObject(); + } + } + } + } + Err(if failures.is_empty() { + format!( + "no D3D11-aware {} decoder MFT is registered", + format.codec.label() + ) + } else { + format!( + "registered {} decoders rejected the D3D11 configuration: {}", + format.codec.label(), + failures.join("; ") + ) + }) + } + + pub(super) fn wants_input(&self) -> bool { + self.input_credits > 0 && !self.stopped + } + + pub(super) fn format(&self) -> VideoFormat { + self.format + } + + pub(super) fn validate_output(&mut self) -> Result { + let (aperture, format) = output_format( + &self.transform, + self.output_stream, + self.negotiated_format, + false, + )?; + validate_decoded_output_format( + decoder_surface_format(format.pixel_format), + format.pixel_format, + self.negotiated_format.pixel_format, + )?; + self.aperture = aperture; + self.format = format; + self.output_validated = true; + Ok(format) + } + + pub(super) fn probe_frame(&mut self, data: &[u8]) -> Result { + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(500); + let mut frames = VecDeque::new(); + let events = BoundedQueue::new(16); + let mut submitted = false; + while std::time::Instant::now() < deadline { + self.poll_output(&mut frames, &events)?; + if let Some(frame) = frames.pop_front() { + return Ok(frame); + } + if !submitted && self.wants_input() { + self.submit(EncodedVideoFrame { + codec: self.negotiated_format.codec, + data: data.to_vec(), + timestamp_100ns: 0, + duration_100ns: self.negotiated_format.frame_duration_100ns(), + key_frame: true, + reset_decoder: false, + })?; + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) + .map_err(|error| format!("drain decoder probe: {error}"))?; + } + submitted = true; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + Err(format!( + "decoder probe timed out waiting for {:?} output (submitted={submitted})", + self.negotiated_format.pixel_format + )) + } + + pub(super) fn submit(&mut self, frame: EncodedVideoFrame) -> Result<(), String> { + if self.input_credits == 0 { + return Err("decoder input was submitted without a NeedInput credit".to_owned()); + } + unsafe { + if frame.codec != self.format.codec { + return Err(format!( + "{} decoder received a {} access unit", + self.format.codec.label(), + frame.codec.label() + )); + } + let buffer = + MFCreateMemoryBuffer(frame.data.len() as u32).map_err(|error| error.to_string())?; + let mut destination = ptr::null_mut(); + buffer + .Lock(&mut destination, None, None) + .map_err(|error| error.to_string())?; + ptr::copy_nonoverlapping(frame.data.as_ptr(), destination, frame.data.len()); + let unlock_result = buffer.Unlock(); + unlock_result.map_err(|error| error.to_string())?; + buffer + .SetCurrentLength(frame.data.len() as u32) + .map_err(|error| error.to_string())?; + let sample = MFCreateSample().map_err(|error| error.to_string())?; + sample + .AddBuffer(&buffer) + .map_err(|error| error.to_string())?; + sample + .SetSampleTime(frame.timestamp_100ns) + .map_err(|error| error.to_string())?; + sample + .SetSampleDuration(frame.duration_100ns) + .map_err(|error| error.to_string())?; + if frame.key_frame { + sample + .SetUINT32(&MFSampleExtension_CleanPoint, 1) + .map_err(|error| error.to_string())?; + } + self.transform + .ProcessInput(self.input_stream, &sample, 0) + .map_err(|error| error.to_string())?; + } + self.input_credits -= 1; + Ok(()) + } + + pub(super) fn poll_output( + &mut self, + decoded_frames: &mut VecDeque, + event_queue: &BoundedQueue, + ) -> Result { + let mut produced = 0; + if let Some(events) = self.events.clone() { + loop { + let event = match unsafe { events.GetEvent(MF_EVENT_FLAG_NO_WAIT) } { + Ok(event) => event, + Err(error) if error.code() == MF_E_NO_EVENTS_AVAILABLE => break, + Err(error) => return Err(error.to_string()), + }; + let status = unsafe { event.GetStatus().map_err(|error| error.to_string())? }; + status.ok().map_err(|error| error.to_string())?; + let event_type = unsafe { event.GetType().map_err(|error| error.to_string())? }; + if event_type == METransformNeedInput.0 as u32 { + self.input_credits = self.input_credits.saturating_add(1); + } else if event_type == METransformHaveOutput.0 as u32 { + if self.process_output(decoded_frames, event_queue)? == OutputPoll::Produced { + produced += 1; + } + // Keep decode and presentation interleaved on the shared + // media worker. Draining every pending MFT output here can + // create an artificial burst that looks stale even when + // the source cadence and display cadence are healthy. + break; + } + } + } else { + if self.process_output(decoded_frames, event_queue)? == OutputPoll::Produced { + produced += 1; + } + self.input_credits = 1; + } + Ok(produced) + } + + fn process_output( + &mut self, + decoded_frames: &mut VecDeque, + event_queue: &BoundedQueue, + ) -> Result { + let mut output = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream, + ..Default::default() + }; + let mut status = 0; + let result = unsafe { + self.transform + .ProcessOutput(0, std::slice::from_mut(&mut output), &mut status) + }; + let sample = unsafe { ManuallyDrop::take(&mut output.pSample) }; + let output_events = unsafe { ManuallyDrop::take(&mut output.pEvents) }; + drop(output_events); + + if let Err(error) = result { + drop(sample); + if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT { + return Ok(OutputPoll::NeedsInput); + } + if error.code() == MF_E_TRANSFORM_STREAM_CHANGE { + self.select_video_output_type()?; + let updated = self.validate_output()?; + let _ = event_queue.push(BackendEvent::VideoFormatChanged(updated)); + return Ok(OutputPoll::Produced); + } + return Err(error.to_string()); + } + + let sample = sample.ok_or_else(|| { + if self.output_provides_samples { + "hardware decoder reported output without a sample".to_owned() + } else { + "hardware decoder requires caller-allocated output samples".to_owned() + } + })?; + if !self.output_validated { + self.validate_output()?; + } + let frame = DecodedVideoFrame::from_sample( + sample, + self.format, + self.aperture, + self.negotiated_format.pixel_format, + )?; + if decoded_frames.len() == ADAPTIVE_VIDEO_QUEUE_CAPACITY { + decoded_frames.pop_front(); + let _ = event_queue.push(BackendEvent::QueueOverflow(Subsystem::VideoPresentation)); + } + decoded_frames.push_back(frame); + Ok(OutputPoll::Produced) + } + + pub(super) fn stop(&mut self) { + if self.stopped { + return; + } + unsafe { + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0); + let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + let _ = self.activation.ShutdownObject(); + } + self.input_credits = 0; + self.stopped = true; + } + + fn select_video_output_type(&self) -> Result { + select_video_output_type( + &self.transform, + self.output_stream, + self.negotiated_format.pixel_format, + ) + } +} + +fn activation_string(activation: &IMFActivate, key: &::windows::core::GUID) -> Option { + unsafe { + let length = activation.GetStringLength(key).ok()? as usize; + let mut value = vec![0_u16; length.saturating_add(1)]; + activation.GetString(key, &mut value, None).ok()?; + String::from_utf16(&value[..length]).ok() + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum OutputPoll { + Produced, + NeedsInput, +} + +impl Drop for Decoder { + fn drop(&mut self) { + self.stop(); + } +} + +type ConfiguredTransform = ( + IMFTransform, + Option, + u32, + u32, + bool, + VideoFormat, + VideoAperture, +); + +fn configure_transform( + activation: &IMFActivate, + graphics: &G, + format: VideoFormat, +) -> Result { + unsafe { + let transform: IMFTransform = activation + .ActivateObject() + .map_err(|error| format!("ActivateObject: {error}"))?; + let attributes = transform + .GetAttributes() + .map_err(|error| error.to_string())?; + let asynchronous = attributes.GetUINT32(&MF_TRANSFORM_ASYNC).unwrap_or(0) != 0; + let d3d11_aware = attributes.GetUINT32(&MF_SA_D3D11_AWARE).unwrap_or(0) != 0; + if !d3d11_aware { + return Err("MFT is not D3D11-aware".to_owned()); + } + attributes + .SetUINT32(&MF_LOW_LATENCY, 1) + .map_err(|error| format!("decoder low-latency mode: {error}"))?; + if asynchronous { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .map_err(|error| error.to_string())?; + } + transform + .ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + graphics.device_manager().as_raw() as usize, + ) + .map_err(|error| format!("MFT_MESSAGE_SET_D3D_MANAGER: {error}"))?; + + let (input_stream, output_stream) = stream_ids(&transform)?; + let input_type = video_input_type(format)?; + transform + .SetInputType(input_stream, &input_type, 0) + .map_err(|error| format!("SetInputType: {error}"))?; + select_video_output_type(&transform, output_stream, format.pixel_format)?; + let (aperture, decoded_format) = output_format(&transform, output_stream, format, true)?; + + let stream_info = transform + .GetOutputStreamInfo(output_stream) + .map_err(|error| error.to_string())?; + let provider_flags = MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 as u32 + | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES.0 as u32; + if stream_info.dwFlags & provider_flags == 0 { + return Err("MFT cannot allocate D3D11 output samples".to_owned()); + } + let events = asynchronous + .then(|| transform.cast::()) + .transpose() + .map_err(|error| error.to_string())?; + transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) + .map_err(|error| error.to_string())?; + transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) + .map_err(|error| error.to_string())?; + Ok(( + transform, + events, + input_stream, + output_stream, + true, + decoded_format, + aperture, + )) + } +} + +fn video_input_type(format: VideoFormat) -> Result { + format.validate_color().map_err(|error| error.to_string())?; + unsafe { + let input_type = MFCreateMediaType().map_err(|error| error.to_string())?; + input_type + .SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) + .map_err(|error| error.to_string())?; + input_type + .SetGUID(&MF_MT_SUBTYPE, video_subtype(format.codec)) + .map_err(|error| error.to_string())?; + if format.codec == VideoCodec::H265 { + let profile = match format.pixel_format { + VideoPixelFormat::Nv12 => eAVEncH265VProfile_Main_420_8, + VideoPixelFormat::P010 => eAVEncH265VProfile_Main_420_10, + VideoPixelFormat::Ayuv => eAVEncH265VProfile_Main_444_8, + VideoPixelFormat::Y410 => eAVEncH265VProfile_Main_444_10, + }; + input_type + .SetUINT32(&MF_MT_MPEG2_PROFILE, profile.0 as u32) + .map_err(|error| format!("HEVC input profile: {error}"))?; + } + input_type + .SetUINT64(&MF_MT_FRAME_SIZE, pack_pair(format.width, format.height)) + .map_err(|error| error.to_string())?; + input_type + .SetUINT64( + &MF_MT_FRAME_RATE, + pack_pair( + format.frame_rate_numerator.get(), + format.frame_rate_denominator.get(), + ), + ) + .map_err(|error| error.to_string())?; + input_type + .SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, pack_pair(1, 1)) + .map_err(|error| error.to_string())?; + input_type + .SetUINT32( + &MF_MT_INTERLACE_MODE, + MFVideoInterlace_MixedInterlaceOrProgressive.0 as u32, + ) + .map_err(|error| error.to_string())?; + input_type + .SetUINT32(&MF_MT_AVG_BITRATE, format.average_bitrate) + .map_err(|error| error.to_string())?; + for (key, value) in [ + ( + MF_MT_VIDEO_CHROMA_SITING, + match format.chroma_siting { + VideoChromaSiting::Left => MFVideoChromaSubsampling_MPEG2.0, + VideoChromaSiting::TopLeft => MFVideoChromaSubsampling_Cosited.0, + }, + ), + ( + MF_MT_TRANSFER_FUNCTION, + match format.transfer_function { + VideoTransferFunction::Sdr => MFVideoTransFunc_709.0, + VideoTransferFunction::Pq => MFVideoTransFunc_2084.0, + VideoTransferFunction::Hlg => MFVideoTransFunc_HLG.0, + }, + ), + ( + MF_MT_VIDEO_PRIMARIES, + match format.color_primaries { + VideoColorPrimaries::Bt709 => MFVideoPrimaries_BT709.0, + VideoColorPrimaries::Bt2020 => MFVideoPrimaries_BT2020.0, + }, + ), + ( + MF_MT_YUV_MATRIX, + match format.color_matrix { + VideoColorMatrix::Bt601 => MFVideoTransferMatrix_BT601.0, + VideoColorMatrix::Bt709 => MFVideoTransferMatrix_BT709.0, + VideoColorMatrix::Bt2020 => MFVideoTransferMatrix_BT2020_10.0, + }, + ), + ( + MF_MT_VIDEO_NOMINAL_RANGE, + if format.full_range { + MFNominalRange_0_255.0 + } else { + MFNominalRange_16_235.0 + }, + ), + ] { + input_type + .SetUINT32(&key, value as u32) + .map_err(|error| format!("decoder input color metadata: {error}"))?; + } + Ok(input_type) + } +} + +/// Applies the shared candidate-source policy to already enumerated lists. +/// Adapter-tagged hardware wins when any exist. Otherwise untagged hardware +/// (Intel Quick Sync, which often omits `MFT_ENUM_ADAPTER_LUID`) comes before +/// the registered Microsoft MFT that NVIDIA and AMD use for DXVA. +#[cfg(test)] +fn hardware_decoder_order( + adapter_tagged: Vec, + untagged_hardware: Vec, + registered: Vec, +) -> Vec { + let mut adapter_tagged = adapter_tagged; + let mut untagged_hardware = untagged_hardware; + let mut registered = registered; + let mut activations = Vec::new(); + for source in crate::decoder_order::decoder_candidate_sources(adapter_tagged.len()) { + match source { + crate::decoder_order::DecoderCandidateSource::AdapterHardware => { + activations.append(&mut adapter_tagged); + } + crate::decoder_order::DecoderCandidateSource::UnscopedHardware => { + activations.append(&mut untagged_hardware); + } + crate::decoder_order::DecoderCandidateSource::Registered => { + activations.append(&mut registered); + } + } + } + activations +} + +fn enumerate_decoders( + adapter_luid: LUID, + codec: VideoCodec, + mode: WindowsDecoderMode, +) -> Result, String> { + let registered = MFT_ENUM_FLAG( + MFT_ENUM_FLAG_SYNCMFT.0 | MFT_ENUM_FLAG_ASYNCMFT.0 | MFT_ENUM_FLAG_SORTANDFILTER.0, + ); + unsafe { + match mode { + WindowsDecoderMode::Hardware => { + let mut attributes = None; + MFCreateAttributes(&mut attributes, 1).map_err(|error| error.to_string())?; + let attributes = + attributes.ok_or("MFCreateAttributes returned no attribute store")?; + let luid_bytes = std::slice::from_raw_parts( + (&adapter_luid as *const LUID).cast::(), + size_of::(), + ); + attributes + .SetBlob(&MFT_ENUM_ADAPTER_LUID, luid_bytes) + .map_err(|error| error.to_string())?; + let hardware_flags = + MFT_ENUM_FLAG(MFT_ENUM_FLAG_HARDWARE.0 | MFT_ENUM_FLAG_SORTANDFILTER.0); + let mut adapter_hardware = + enumerate_matching_decoders(hardware_flags, Some(&attributes), codec)?; + let mut activations = Vec::new(); + // `decoder_candidate_sources` is the shared order: adapter-tagged + // hardware when any exist, otherwise untagged hardware, and the + // registered Microsoft MFT last. configure_transform still rejects + // any MFT that cannot accept this device's D3D manager. + for source in + crate::decoder_order::decoder_candidate_sources(adapter_hardware.len()) + { + match source { + crate::decoder_order::DecoderCandidateSource::AdapterHardware => { + activations.append(&mut adapter_hardware); + } + crate::decoder_order::DecoderCandidateSource::UnscopedHardware => { + // Intel Quick Sync / Iris / Arc hardware MFTs usually omit + // MFT_ENUM_ADAPTER_LUID, so the adapter filter above is empty + // even when the D3D11 device is already that GPU. Without + // this step the registered fallback's first success is + // "Microsoft H264 Video Decoder MFT" (software). + video_log!( + "Windows hardware decoder enumeration matched no MFT for this adapter; trying hardware transforms that do not publish an adapter LUID" + ); + activations.extend(enumerate_matching_decoders( + hardware_flags, + None, + codec, + )?); + } + crate::decoder_order::DecoderCandidateSource::Registered => { + // NVIDIA and AMD drive decode through DXVA2 instead of + // registering a hardware-flagged MFT. The D3D11-aware + // Microsoft MFT is their GPU path, so it stays last. + activations + .extend(enumerate_matching_decoders(registered, None, codec)?); + } + } + } + Ok(activations) + } + WindowsDecoderMode::Software => enumerate_matching_decoders(registered, None, codec), + } + } +} + +unsafe fn enumerate_matching_decoders( + flags: MFT_ENUM_FLAG, + attributes: Option<&IMFAttributes>, + codec: VideoCodec, +) -> Result, String> { + let input = MFT_REGISTER_TYPE_INFO { + guidMajorType: MFMediaType_Video, + guidSubtype: *video_subtype(codec), + }; + let mut entries: *mut Option = ptr::null_mut(); + let mut count = 0; + unsafe { + MFTEnum2( + MFT_CATEGORY_VIDEO_DECODER, + flags, + Some(&input), + None, + attributes, + &mut entries, + &mut count, + ) + .map_err(|error| error.to_string())?; + } + let mut activations = Vec::with_capacity(count as usize); + if !entries.is_null() { + unsafe { + for index in 0..count as usize { + if let Some(activation) = (*entries.add(index)).take() { + activations.push(activation); + } + } + CoTaskMemFree(Some(entries as *const c_void)); + } + } + Ok(activations) +} + +fn stream_ids(transform: &IMFTransform) -> Result<(u32, u32), String> { + unsafe { + let mut input_count = 0; + let mut output_count = 0; + transform + .GetStreamCount(&mut input_count, &mut output_count) + .map_err(|error| error.to_string())?; + if input_count != 1 || output_count != 1 { + return Err(format!( + "expected one input and output stream, got {input_count} and {output_count}" + )); + } + let mut input = [0]; + let mut output = [0]; + if let Err(error) = transform.GetStreamIDs(&mut input, &mut output) { + if error.code() == E_NOTIMPL { + return Ok((0, 0)); + } + return Err(error.to_string()); + } + Ok((input[0], output[0])) + } +} + +// MFT registration and D3D11 awareness do not prove hardware codec support. In particular, +// an installed AV1 MFT can exist on a pre-AV1 GPU. Check the actual decoder device, including +// the requested depth, chroma and dimensions, before configuring either probe or live decode. +fn hardware_profiles( + codec: VideoCodec, + pixel: VideoPixelFormat, +) -> &'static [::windows::core::GUID] { + match (codec, pixel) { + (VideoCodec::H264, VideoPixelFormat::Nv12) => &[ + D3D11_DECODER_PROFILE_H264_VLD_NOFGT, + D3D11_DECODER_PROFILE_H264_VLD_FGT, + ], + (VideoCodec::H265, VideoPixelFormat::Nv12) => &[D3D11_DECODER_PROFILE_HEVC_VLD_MAIN], + (VideoCodec::H265, VideoPixelFormat::P010) => &[D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10], + // DXVA decode profile GUIDs are shared between the D3D11 and D3D12 APIs. + (VideoCodec::H265, VideoPixelFormat::Ayuv) => &[D3D12_VIDEO_DECODE_PROFILE_HEVC_MAIN_444], + (VideoCodec::H265, VideoPixelFormat::Y410) => &[D3D12_VIDEO_DECODE_PROFILE_HEVC_MAIN10_444], + (VideoCodec::Av1, VideoPixelFormat::Nv12 | VideoPixelFormat::P010) => { + &[D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0] + } + _ => &[], + } +} + +fn ensure_hardware_format( + graphics: &G, + format: VideoFormat, +) -> Result<(), String> { + let manager = graphics.device_manager(); + let device = unsafe { + let handle = manager + .OpenDeviceHandle() + .map_err(|error| error.to_string())?; + let mut service = ptr::null_mut(); + let result = manager.GetVideoService(handle, &ID3D11VideoDevice::IID, &mut service); + let _ = manager.CloseDeviceHandle(handle); + result.map_err(|error| format!("D3D11 decoder device: {error}"))?; + if service.is_null() { + return Err("D3D11 decoder device is null".to_owned()); + } + ID3D11VideoDevice::from_raw(service) + }; + let output = decoder_surface_format(format.pixel_format); + let candidates = hardware_profiles(format.codec, format.pixel_format); + unsafe { + for index in 0..device.GetVideoDecoderProfileCount() { + let profile = device + .GetVideoDecoderProfile(index) + .map_err(|error| error.to_string())?; + if !candidates.contains(&profile) + || !device + .CheckVideoDecoderFormat(&profile, output) + .is_ok_and(|supported| supported.as_bool()) + { + continue; + } + let description = D3D11_VIDEO_DECODER_DESC { + Guid: profile, + SampleWidth: format.width, + SampleHeight: format.height, + OutputFormat: output, + }; + if device + .GetVideoDecoderConfigCount(&description) + .is_ok_and(|count| count > 0) + { + return Ok(()); + } + } + } + Err(format!( + "The selected GPU does not support hardware {} {:?} decoding at {}x{}. Use Auto codec or a supported color depth/resolution.", + format.codec.label(), + format.pixel_format, + format.width, + format.height + )) +} + +fn select_video_output_type( + transform: &IMFTransform, + output_stream: u32, + preferred: VideoPixelFormat, +) -> Result { + let mut supported = Vec::new(); + for index in 0..64 { + let media_type = match unsafe { transform.GetOutputAvailableType(output_stream, index) } { + Ok(media_type) => media_type, + Err(_) if !supported.is_empty() => break, + Err(error) => return Err(format!("video output type is unavailable: {error}")), + }; + let subtype = unsafe { + media_type + .GetGUID(&MF_MT_SUBTYPE) + .map_err(|error| error.to_string())? + }; + if let Some(pixel_format) = pixel_format_from_subtype(subtype) { + supported.push((pixel_format, media_type)); + } + } + let exposed = supported + .iter() + .map(|(format, _)| format!("{format:?}")) + .collect::>() + .join(", "); + let mut rejected = Vec::new(); + for candidate in output_format_preferences(preferred) { + for (pixel_format, media_type) in &supported { + if pixel_format != candidate { + continue; + } + match unsafe { transform.SetOutputType(output_stream, media_type, 0) } { + Ok(()) => { + if *pixel_format != preferred { + video_log!( + "Windows decoder requested {preferred:?}, temporarily using compatible {pixel_format:?} output" + ); + } + return Ok(*pixel_format); + } + Err(error) => rejected.push(format!("{pixel_format:?}: {error}")), + } + } + } + let suffix = if rejected.is_empty() { + String::new() + } else { + format!("; rejected types: {}", rejected.join("; ")) + }; + Err(format!( + "hardware decoder could not configure requested {preferred:?} output (exposed: {exposed}){suffix}" + )) +} + +fn output_format_preferences(preferred: VideoPixelFormat) -> &'static [VideoPixelFormat] { + match preferred { + VideoPixelFormat::Nv12 => &[VideoPixelFormat::Nv12], + // Some HEVC/AV1 MFTs advertise NV12 until the first sequence header is + // parsed, then raise MF_E_TRANSFORM_STREAM_CHANGE and expose P010. Let + // startup proceed in NV12 and keep preferring P010 on every reselection. + VideoPixelFormat::P010 => &[VideoPixelFormat::P010, VideoPixelFormat::Nv12], + VideoPixelFormat::Ayuv => &[VideoPixelFormat::Ayuv, VideoPixelFormat::Nv12], + VideoPixelFormat::Y410 => &[ + VideoPixelFormat::Y410, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Nv12, + ], + } +} + +fn decoder_surface_format(pixel_format: VideoPixelFormat) -> DXGI_FORMAT { + match pixel_format { + VideoPixelFormat::Nv12 => DXGI_FORMAT_NV12, + VideoPixelFormat::P010 => DXGI_FORMAT_P010, + VideoPixelFormat::Ayuv => DXGI_FORMAT_AYUV, + VideoPixelFormat::Y410 => DXGI_FORMAT_Y410, + } +} + +fn validate_decoded_output_format( + surface_format: DXGI_FORMAT, + output: VideoPixelFormat, + preferred: VideoPixelFormat, +) -> Result<(), String> { + if surface_format != decoder_surface_format(output) { + return Err(format!( + "Media Foundation decoder surface format {} does not match output media format {output:?}", + surface_format.0 + )); + } + if !preferred.supports_decoded_output(output) { + return Err(format!( + "Media Foundation decoder produced {output:?} output below negotiated {preferred:?} precision or chroma" + )); + } + Ok(()) +} + +fn pixel_format_from_subtype(subtype: ::windows::core::GUID) -> Option { + if subtype == MFVideoFormat_NV12 { + Some(VideoPixelFormat::Nv12) + } else if subtype == MFVideoFormat_P010 { + Some(VideoPixelFormat::P010) + } else if subtype == MFVideoFormat_AYUV { + Some(VideoPixelFormat::Ayuv) + } else if subtype == MFVideoFormat_Y410 { + Some(VideoPixelFormat::Y410) + } else { + None + } +} + +fn chroma_format(pixel_format: VideoPixelFormat) -> VideoChromaFormat { + match pixel_format { + VideoPixelFormat::Nv12 | VideoPixelFormat::P010 => VideoChromaFormat::Cs420, + VideoPixelFormat::Ayuv | VideoPixelFormat::Y410 => VideoChromaFormat::Cs444, + } +} + +fn output_format( + transform: &IMFTransform, + output_stream: u32, + fallback: VideoFormat, + allow_provisional: bool, +) -> Result<(VideoAperture, VideoFormat), String> { + unsafe { + let media_type = transform + .GetOutputCurrentType(output_stream) + .map_err(|error| error.to_string())?; + let aperture = output_aperture(&media_type, fallback)?; + let subtype = media_type + .GetGUID(&MF_MT_SUBTYPE) + .map_err(|error| error.to_string())?; + let pixel_format = pixel_format_from_subtype(subtype) + .ok_or("decoder output has an unsupported pixel format")?; + let format = output_media_format( + &media_type, + fallback, + aperture, + pixel_format, + allow_provisional, + )?; + Ok((aperture, format)) + } +} + +fn output_media_format( + media_type: &IMFMediaType, + negotiated: VideoFormat, + aperture: VideoAperture, + pixel_format: VideoPixelFormat, + allow_provisional: bool, +) -> Result { + let format = VideoFormat { + width: aperture.width, + height: aperture.height, + pixel_format, + chroma_format: chroma_format(pixel_format), + ..negotiated + }; + if allow_provisional && pixel_format != negotiated.pixel_format { + return Ok(format); + } + output_color_format(media_type, format) +} + +fn output_color_format( + media_type: &IMFMediaType, + fallback: VideoFormat, +) -> Result { + let mut descriptors = ColorDescriptors::default(); + for key in [ + MF_MT_TRANSFER_FUNCTION, + MF_MT_VIDEO_PRIMARIES, + MF_MT_YUV_MATRIX, + MF_MT_VIDEO_NOMINAL_RANGE, + MF_MT_VIDEO_CHROMA_SITING, + ] { + let value = match unsafe { media_type.GetUINT32(&key) } { + Ok(0) => continue, + Ok(value) => value, + Err(error) if error.code() == MF_E_ATTRIBUTENOTFOUND => continue, + Err(error) => return Err(format!("decoder output color metadata: {error}")), + }; + if key == MF_MT_TRANSFER_FUNCTION { + descriptors.transfer = match value as i32 { + value if value == MFVideoTransFunc_2084.0 => { + Descriptor::Recognized(VideoTransferFunction::Pq) + } + value if value == MFVideoTransFunc_HLG.0 => { + Descriptor::Recognized(VideoTransferFunction::Hlg) + } + value + if [ + MFVideoTransFunc_22.0, + MFVideoTransFunc_709.0, + MFVideoTransFunc_sRGB.0, + MFVideoTransFunc_2020.0, + ] + .contains(&value) => + { + Descriptor::Recognized(VideoTransferFunction::Sdr) + } + _ => Descriptor::Unsupported(value), + }; + } else if key == MF_MT_VIDEO_PRIMARIES { + descriptors.primaries = match value as i32 { + value if value == MFVideoPrimaries_BT709.0 => { + Descriptor::Recognized(VideoColorPrimaries::Bt709) + } + value if value == MFVideoPrimaries_BT2020.0 => { + Descriptor::Recognized(VideoColorPrimaries::Bt2020) + } + _ => Descriptor::Unsupported(value), + }; + } else if key == MF_MT_YUV_MATRIX { + descriptors.matrix = match value as i32 { + value if value == MFVideoTransferMatrix_BT601.0 => { + Descriptor::Recognized(VideoColorMatrix::Bt601) + } + value if value == MFVideoTransferMatrix_BT709.0 => { + Descriptor::Recognized(VideoColorMatrix::Bt709) + } + value if value == MFVideoTransferMatrix_BT2020_10.0 => { + Descriptor::Recognized(VideoColorMatrix::Bt2020) + } + _ => Descriptor::Unsupported(value), + }; + } else if key == MF_MT_VIDEO_CHROMA_SITING { + descriptors.chroma_siting = + match value as i32 & !MFVideoChromaSubsampling_ProgressiveChroma.0 { + value if value == MFVideoChromaSubsampling_MPEG2.0 => { + Descriptor::Recognized(VideoChromaSiting::Left) + } + value if value == MFVideoChromaSubsampling_Cosited.0 => { + Descriptor::Recognized(VideoChromaSiting::TopLeft) + } + _ => Descriptor::Unsupported(value), + }; + } else { + descriptors.range = match value as i32 { + value if value == MFNominalRange_0_255.0 => Descriptor::Recognized(true), + value if value == MFNominalRange_16_235.0 => Descriptor::Recognized(false), + _ => Descriptor::Unsupported(value), + }; + } + } + let format = descriptors.resolve(fallback); + format.validate_color().map_err(|error| { + let present = |key| { + unsafe { media_type.GetUINT32(key) } + .ok() + .is_some_and(|value| value != 0) + }; + format!( + "{error}: negotiated={:?}/{:?}/{:?} decoded={:?}/{:?}/{:?} explicit={}/{}/{}", + fallback.transfer_function, + fallback.color_primaries, + fallback.color_matrix, + format.transfer_function, + format.color_primaries, + format.color_matrix, + present(&MF_MT_TRANSFER_FUNCTION), + present(&MF_MT_VIDEO_PRIMARIES), + present(&MF_MT_YUV_MATRIX), + ) + })?; + Ok(format) +} + +fn output_aperture( + media_type: &IMFMediaType, + fallback: VideoFormat, +) -> Result { + let packed = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } { + Ok(packed) => packed, + Err(error) if error.code() == MF_E_ATTRIBUTENOTFOUND => { + pack_pair(fallback.width, fallback.height) + } + Err(error) => return Err(error.to_string()), + }; + let pan_scan = unsafe { media_type.GetUINT32(&MF_MT_PAN_SCAN_ENABLED).unwrap_or(0) != 0 }; + for key in [ + pan_scan.then_some(&MF_MT_PAN_SCAN_APERTURE), + Some(&MF_MT_MINIMUM_DISPLAY_APERTURE), + Some(&MF_MT_GEOMETRIC_APERTURE), + ] + .into_iter() + .flatten() + { + let size = match unsafe { media_type.GetBlobSize(key) } { + Ok(size) => size as usize, + Err(error) if error.code() == MF_E_ATTRIBUTENOTFOUND => continue, + Err(error) => return Err(error.to_string()), + }; + if size != size_of::() { + return Err(format!("invalid decoder aperture blob size {size}")); + } + let mut bytes = [0_u8; size_of::()]; + unsafe { media_type.GetBlob(key, &mut bytes, None) }.map_err(|error| error.to_string())?; + let area = unsafe { ptr::read_unaligned(bytes.as_ptr().cast::()) }; + if area.OffsetX.fract != 0 || area.OffsetY.fract != 0 { + return Err("fractional decoder aperture offsets cannot be represented by a D3D11 source rectangle".to_owned()); + } + let area = ( + i32::from(area.OffsetX.value), + i32::from(area.OffsetY.value), + area.Area.cx, + area.Area.cy, + ); + if area == (0, 0, 0, 0) { + continue; + } + return VideoAperture::new((packed >> 32) as u32, packed as u32, Some(area)); + } + VideoAperture::new((packed >> 32) as u32, packed as u32, None) +} + +fn video_subtype(codec: VideoCodec) -> &'static ::windows::core::GUID { + match codec { + VideoCodec::H264 => &MFVideoFormat_H264, + VideoCodec::H265 => &MFVideoFormat_HEVC, + VideoCodec::Av1 => &MFVideoFormat_AV1, + } +} + +fn dxgi_surface(sample: &IMFSample) -> Result<(ID3D11Texture2D, u32), String> { + unsafe { + let buffer = sample + .GetBufferByIndex(0) + .map_err(|error| error.to_string())?; + let dxgi_buffer: IMFDXGIBuffer = buffer.cast().map_err(|_| { + "hardware decoder returned a system-memory buffer instead of a D3D11 surface".to_owned() + })?; + let mut resource = ptr::null_mut(); + dxgi_buffer + .GetResource(&ID3D11Texture2D::IID, &mut resource) + .map_err(|error| error.to_string())?; + if resource.is_null() { + return Err("decoder returned a null D3D11 texture".to_owned()); + } + let texture = ID3D11Texture2D::from_raw(resource); + let subresource = dxgi_buffer + .GetSubresourceIndex() + .map_err(|error| error.to_string())?; + Ok((texture, subresource)) + } +} + +fn pack_pair(high: u32, low: u32) -> u64 { + ((high as u64) << 32) | low as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hdr_test_format() -> VideoFormat { + VideoFormat { + codec: VideoCodec::H265, + width: 1920, + height: 1080, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 20_000_000, + pixel_format: VideoPixelFormat::P010, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: VideoTransferFunction::Pq, + color_primaries: VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + } + } + + #[test] + fn empty_adapter_luid_filter_tries_untagged_hardware_before_the_microsoft_decoder() { + let order = hardware_decoder_order( + Vec::<&str>::new(), + vec!["Intel Quick Sync Video H.264 Decoder"], + vec!["Microsoft H264 Video Decoder MFT"], + ); + assert_eq!( + order, + [ + "Intel Quick Sync Video H.264 Decoder", + "Microsoft H264 Video Decoder MFT", + ] + ); + } + + #[test] + fn adapter_tagged_hardware_mft_stays_ahead_of_the_microsoft_decoder() { + let order = hardware_decoder_order( + vec!["Adapter H.264 hardware decoder"], + vec!["Intel Quick Sync Video H.264 Decoder"], + vec!["Microsoft H264 Video Decoder MFT"], + ); + assert_eq!( + order, + [ + "Adapter H.264 hardware decoder", + "Microsoft H264 Video Decoder MFT", + ] + ); + } + + #[test] + fn vendor_dxva_path_keeps_the_microsoft_decoder_when_no_hardware_mft_is_registered() { + let order = hardware_decoder_order( + Vec::<&str>::new(), + Vec::<&str>::new(), + vec!["Microsoft H264 Video Decoder MFT"], + ); + assert_eq!(order, ["Microsoft H264 Video Decoder MFT"]); + } + + #[test] + fn provisional_nv12_allows_hdr_startup_but_never_validated_output() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let negotiated = hdr_test_format(); + let aperture = VideoAperture::new(negotiated.width, negotiated.height, None).unwrap(); + let media_type = video_input_type(negotiated).unwrap(); + let provisional = output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::Nv12, + true, + ) + .unwrap(); + assert_eq!(provisional.pixel_format, VideoPixelFormat::Nv12); + assert!( + output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::Nv12, + false, + ) + .is_err() + ); + assert!( + validate_decoded_output_format( + DXGI_FORMAT_NV12, + provisional.pixel_format, + negotiated.pixel_format, + ) + .is_err() + ); + assert_eq!( + output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::P010, + false, + ) + .unwrap(), + negotiated + ); + + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32) + .unwrap(); + } + assert!( + output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::Nv12, + true, + ) + .is_ok() + ); + let sdr_output = output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::P010, + false, + ) + .unwrap(); + assert_eq!(sdr_output.transfer_function, VideoTransferFunction::Sdr); + assert_eq!(sdr_output.pixel_format, VideoPixelFormat::P010); + } + + #[test] + fn chroma_fallback_validates_actual_output_color_metadata() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let negotiated = VideoFormat { + pixel_format: VideoPixelFormat::Y410, + chroma_format: VideoChromaFormat::Cs444, + ..hdr_test_format() + }; + let aperture = VideoAperture::new(negotiated.width, negotiated.height, None).unwrap(); + let media_type = video_input_type(negotiated).unwrap(); + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_HLG.0 as u32) + .unwrap(); + media_type + .SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_0_255.0 as u32) + .unwrap(); + } + let output = output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::P010, + false, + ) + .unwrap(); + assert_eq!(output.pixel_format, VideoPixelFormat::P010); + assert_eq!(output.chroma_format, VideoChromaFormat::Cs420); + assert_eq!(output.transfer_function, VideoTransferFunction::Hlg); + assert!(output.full_range); + assert!( + validate_decoded_output_format( + DXGI_FORMAT_P010, + output.pixel_format, + negotiated.pixel_format, + ) + .is_ok() + ); + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32) + .unwrap(); + } + let sdr_output = output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::P010, + false, + ) + .unwrap(); + assert_eq!(sdr_output.transfer_function, VideoTransferFunction::Sdr); + assert_eq!(sdr_output.pixel_format, VideoPixelFormat::P010); + unsafe { + media_type.SetUINT32(&MF_MT_TRANSFER_FUNCTION, 999).unwrap(); + } + let output = output_media_format( + &media_type, + negotiated, + aperture, + VideoPixelFormat::P010, + false, + ) + .unwrap(); + assert_eq!(output.transfer_function, negotiated.transfer_function); + assert_eq!(output.pixel_format, VideoPixelFormat::P010); + assert!(output.full_range); + } + + #[test] + fn decoder_color_defaults_and_actual_metadata_are_explicit() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = hdr_test_format(); + let media_type = unsafe { MFCreateMediaType().unwrap() }; + assert_eq!(output_color_format(&media_type, format).unwrap(), format); + unsafe { + media_type.SetUINT32(&MF_MT_TRANSFER_FUNCTION, 0).unwrap(); + media_type.SetUINT32(&MF_MT_VIDEO_PRIMARIES, 0).unwrap(); + media_type.SetUINT32(&MF_MT_YUV_MATRIX, 0).unwrap(); + media_type.SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, 0).unwrap(); + } + assert_eq!(output_color_format(&media_type, format).unwrap(), format); + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_HLG.0 as u32) + .unwrap(); + media_type + .SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_0_255.0 as u32) + .unwrap(); + } + let updated = output_color_format(&media_type, format).unwrap(); + assert_eq!(updated.transfer_function, VideoTransferFunction::Hlg); + assert!(updated.full_range); + let sdr_defaults = VideoFormat { + transfer_function: VideoTransferFunction::Sdr, + color_primaries: VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + ..format + }; + let actual_hdr_type = video_input_type(format).unwrap(); + assert_eq!( + output_color_format(&actual_hdr_type, sdr_defaults).unwrap(), + format + ); + for transfer_function in [VideoTransferFunction::Pq, VideoTransferFunction::Hlg] { + let negotiated = VideoFormat { + transfer_function, + ..format + }; + assert_eq!( + output_color_format(&video_input_type(negotiated).unwrap(), negotiated).unwrap(), + negotiated + ); + } + } + + #[test] + fn decoder_preserves_actual_chroma_siting_or_rejects_it() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = hdr_test_format(); + let media_type = unsafe { MFCreateMediaType().unwrap() }; + for (value, expected) in [ + (0, VideoChromaSiting::Left), + (MFVideoChromaSubsampling_MPEG2.0, VideoChromaSiting::Left), + ( + MFVideoChromaSubsampling_Cosited.0, + VideoChromaSiting::TopLeft, + ), + ( + MFVideoChromaSubsampling_MPEG2.0 | MFVideoChromaSubsampling_ProgressiveChroma.0, + VideoChromaSiting::Left, + ), + ( + MFVideoChromaSubsampling_Cosited.0 | MFVideoChromaSubsampling_ProgressiveChroma.0, + VideoChromaSiting::TopLeft, + ), + ] { + unsafe { + media_type + .SetUINT32(&MF_MT_VIDEO_CHROMA_SITING, value as u32) + .unwrap(); + } + assert_eq!( + output_color_format(&media_type, format) + .unwrap() + .chroma_siting, + expected + ); + let negotiated = VideoFormat { + chroma_siting: expected, + ..format + }; + assert_eq!( + output_color_format(&video_input_type(negotiated).unwrap(), format) + .unwrap() + .chroma_siting, + expected + ); + } + for value in [1, 2, 4, 6, 8, 999] { + unsafe { + media_type + .SetUINT32(&MF_MT_VIDEO_CHROMA_SITING, value) + .unwrap(); + } + assert_eq!(output_color_format(&media_type, format).unwrap(), format); + } + } + + #[test] + fn decoder_follows_explicit_sdr_hdr_output_transitions() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let hdr = hdr_test_format(); + let sdr = VideoFormat { + transfer_function: VideoTransferFunction::Sdr, + color_primaries: VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + ..hdr + }; + let sdr_type = video_input_type(sdr).unwrap(); + let hdr_type = video_input_type(hdr).unwrap(); + assert_eq!(output_color_format(&sdr_type, hdr).unwrap(), sdr); + assert_eq!(output_color_format(&hdr_type, sdr).unwrap(), hdr); + let unspecified_type = unsafe { MFCreateMediaType().unwrap() }; + assert_eq!(output_color_format(&unspecified_type, hdr).unwrap(), hdr); + } + + #[test] + fn decoder_uses_sdr_matrix_when_hdr_output_switches_to_sdr_without_one() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let hdr = hdr_test_format(); + let media_type = unsafe { MFCreateMediaType().unwrap() }; + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32) + .unwrap(); + media_type + .SetUINT32(&MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0 as u32) + .unwrap(); + } + let sdr = VideoFormat { + transfer_function: VideoTransferFunction::Sdr, + color_primaries: VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + ..hdr + }; + assert_eq!(output_color_format(&media_type, hdr).unwrap(), sdr); + unsafe { + media_type.SetUINT32(&MF_MT_YUV_MATRIX, 0).unwrap(); + } + assert_eq!(output_color_format(&media_type, hdr).unwrap(), sdr); + unsafe { + media_type + .SetUINT32(&MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT2020_10.0 as u32) + .unwrap(); + } + assert!(output_color_format(&media_type, hdr).is_err()); + } + + #[test] + fn decoder_rejects_hdr_downgrades_and_unsupported_metadata() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = hdr_test_format(); + for (key, value) in [ + (MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0), + (MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709.0), + ] { + let media_type = unsafe { MFCreateMediaType().unwrap() }; + unsafe { + media_type.SetUINT32(&key, value as u32).unwrap(); + } + assert!( + output_color_format(&media_type, format).is_err(), + "metadata {key:?}={value}" + ); + } + let media_type = unsafe { MFCreateMediaType().unwrap() }; + assert!( + output_color_format( + &media_type, + VideoFormat { + pixel_format: VideoPixelFormat::Nv12, + ..format + } + ) + .is_err() + ); + } + + #[test] + fn decoder_falls_back_field_by_field_for_unsupported_mf_descriptors() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = hdr_test_format(); + let unsupported = [ + (MF_MT_TRANSFER_FUNCTION, 999), + (MF_MT_VIDEO_PRIMARIES, 999), + (MF_MT_YUV_MATRIX, 999), + (MF_MT_VIDEO_NOMINAL_RANGE, 3), + (MF_MT_VIDEO_CHROMA_SITING, 999), + ]; + for (key, value) in unsupported { + let media_type = unsafe { MFCreateMediaType().unwrap() }; + unsafe { + media_type.SetUINT32(&key, value).unwrap(); + } + assert_eq!(output_color_format(&media_type, format).unwrap(), format); + } + let media_type = unsafe { MFCreateMediaType().unwrap() }; + for (key, _) in unsupported { + unsafe { + media_type.SetUINT32(&key, u32::MAX).unwrap(); + } + } + assert_eq!(output_color_format(&media_type, format).unwrap(), format); + unsafe { + media_type + .SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_HLG.0 as u32) + .unwrap(); + media_type + .SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_0_255.0 as u32) + .unwrap(); + media_type + .SetUINT32( + &MF_MT_VIDEO_CHROMA_SITING, + MFVideoChromaSubsampling_Cosited.0 as u32, + ) + .unwrap(); + } + assert_eq!( + output_color_format(&media_type, format).unwrap(), + VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + full_range: true, + chroma_siting: VideoChromaSiting::TopLeft, + ..format + } + ); + } + + #[test] + fn decoder_does_not_ignore_malformed_mf_color_attributes() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let media_type = unsafe { MFCreateMediaType().unwrap() }; + unsafe { + media_type + .SetGUID(&MF_MT_TRANSFER_FUNCTION, &MFMediaType_Video) + .unwrap(); + } + assert!(output_color_format(&media_type, hdr_test_format()).is_err()); + } + + #[test] + fn hevc_input_types_set_the_negotiated_profile_before_decoder_configuration() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + for (pixel_format, profile) in [ + (VideoPixelFormat::Nv12, eAVEncH265VProfile_Main_420_8), + (VideoPixelFormat::P010, eAVEncH265VProfile_Main_420_10), + (VideoPixelFormat::Ayuv, eAVEncH265VProfile_Main_444_8), + (VideoPixelFormat::Y410, eAVEncH265VProfile_Main_444_10), + ] { + let format = VideoFormat { + codec: VideoCodec::H265, + width: 2560, + height: 1440, + frame_rate_numerator: std::num::NonZeroU32::new(120).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 75_000_000, + pixel_format, + chroma_format: chroma_format(pixel_format), + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + let input_type = video_input_type(format).unwrap(); + unsafe { + assert_eq!( + input_type.GetGUID(&MF_MT_MAJOR_TYPE).unwrap(), + MFMediaType_Video + ); + assert_eq!( + input_type.GetGUID(&MF_MT_SUBTYPE).unwrap(), + MFVideoFormat_HEVC + ); + assert_eq!( + input_type.GetUINT32(&MF_MT_MPEG2_PROFILE).unwrap(), + profile.0 as u32 + ); + assert_eq!( + input_type.GetUINT64(&MF_MT_FRAME_SIZE).unwrap(), + pack_pair(2560, 1440) + ); + assert_eq!( + input_type.GetUINT64(&MF_MT_FRAME_RATE).unwrap(), + pack_pair(120, 1) + ); + assert_eq!( + input_type.GetUINT32(&MF_MT_AVG_BITRATE).unwrap(), + 75_000_000 + ); + assert_eq!( + input_type.GetUINT64(&MF_MT_PIXEL_ASPECT_RATIO).unwrap(), + pack_pair(1, 1) + ); + assert_eq!( + input_type.GetUINT32(&MF_MT_INTERLACE_MODE).unwrap(), + MFVideoInterlace_MixedInterlaceOrProgressive.0 as u32 + ); + } + for codec in [VideoCodec::H264, VideoCodec::Av1] { + let input_type = video_input_type(VideoFormat { codec, ..format }).unwrap(); + unsafe { + assert_eq!( + input_type.GetGUID(&MF_MT_SUBTYPE).unwrap(), + *video_subtype(codec) + ); + assert_eq!( + input_type + .GetUINT32(&MF_MT_MPEG2_PROFILE) + .unwrap_err() + .code(), + MF_E_ATTRIBUTENOTFOUND + ); + } + } + } + } + + #[test] + fn output_samples_reject_reported_corruption_before_surface_extraction() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = VideoFormat { + codec: VideoCodec::H264, + width: 2560, + height: 1440, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 78_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + let aperture = VideoAperture::new(format.width, format.height, None).unwrap(); + for (corruption, discontinuity) in [ + (None, false), + (Some(0), false), + (None, true), + (Some(0), true), + (Some(1), false), + (Some(1), true), + ] { + let sample = unsafe { + let sample = MFCreateSample().unwrap(); + sample.AddBuffer(&MFCreateMemoryBuffer(1).unwrap()).unwrap(); + if let Some(corruption) = corruption { + sample + .SetUINT32(&MFSampleExtension_FrameCorruption, corruption) + .unwrap(); + } + if discontinuity { + sample + .SetUINT32( + &::windows::Win32::Media::MediaFoundation::MFSampleExtension_Discontinuity, + 1, + ) + .unwrap(); + } + sample + }; + let error = + DecodedVideoFrame::from_sample(sample, format, aperture, format.pixel_format) + .err() + .expect("system-memory output cannot be published as a decoded GPU frame"); + assert_eq!( + error, + if corruption == Some(1) { + "Media Foundation decoder reported a corrupted output frame" + } else { + "hardware decoder returned a system-memory buffer instead of a D3D11 surface" + }, + "corruption={corruption:?} discontinuity={discontinuity}" + ); + } + } + + #[test] + fn media_type_apertures_handle_padding_defaults_precedence_and_invalid_offsets() { + let _runtime = super::super::MediaRuntime::initialize().unwrap(); + let format = VideoFormat { + codec: VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: true, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + let media_type = unsafe { MFCreateMediaType().unwrap() }; + assert_eq!( + output_aperture(&media_type, format).unwrap(), + VideoAperture::new(1920, 1080, None).unwrap() + ); + unsafe { + media_type + .SetUINT64(&MF_MT_FRAME_SIZE, pack_pair(1920, 1088)) + .unwrap(); + } + assert_eq!(output_aperture(&media_type, format).unwrap().height, 1088); + let set_area = |key, area: MFVideoArea| { + let bytes = unsafe { + std::slice::from_raw_parts( + (&area as *const MFVideoArea).cast::(), + size_of::(), + ) + }; + unsafe { + media_type.SetBlob(key, bytes).unwrap(); + } + }; + let mut area = MFVideoArea::default(); + set_area(&MF_MT_MINIMUM_DISPLAY_APERTURE, area); + assert_eq!(output_aperture(&media_type, format).unwrap().height, 1088); + area.Area.cx = 1904; + area.Area.cy = 1080; + area.OffsetX.value = 8; + area.OffsetY.value = 4; + set_area(&MF_MT_GEOMETRIC_APERTURE, area); + let visible = output_aperture(&media_type, format).unwrap(); + assert_eq!( + visible, + VideoAperture::new(1920, 1088, Some((8, 4, 1904, 1080))).unwrap() + ); + area.Area.cy = 1072; + set_area(&MF_MT_MINIMUM_DISPLAY_APERTURE, area); + assert_eq!(output_aperture(&media_type, format).unwrap().height, 1072); + area.Area.cy = 1064; + set_area(&MF_MT_PAN_SCAN_APERTURE, area); + assert_eq!(output_aperture(&media_type, format).unwrap().height, 1072); + unsafe { + media_type.SetUINT32(&MF_MT_PAN_SCAN_ENABLED, 1).unwrap(); + } + assert_eq!(output_aperture(&media_type, format).unwrap().height, 1064); + area.OffsetX.fract = 1; + set_area(&MF_MT_PAN_SCAN_APERTURE, area); + assert!( + output_aperture(&media_type, format) + .unwrap_err() + .contains("fractional") + ); + area.OffsetX.fract = 0; + area.OffsetX.value = -1; + set_area(&MF_MT_PAN_SCAN_APERTURE, area); + assert!(output_aperture(&media_type, format).is_err()); + unsafe { + media_type + .SetBlob(&MF_MT_PAN_SCAN_APERTURE, &[0_u8; 1]) + .unwrap(); + } + assert!( + output_aperture(&media_type, format) + .unwrap_err() + .contains("blob size") + ); + } + + #[test] + fn hardware_profiles_do_not_confuse_codec_depth_or_chroma() { + let legacy_gpu = [ + D3D11_DECODER_PROFILE_H264_VLD_NOFGT, + D3D11_DECODER_PROFILE_HEVC_VLD_MAIN, + ]; + assert!( + hardware_profiles(VideoCodec::H264, VideoPixelFormat::Nv12) + .iter() + .any(|p| legacy_gpu.contains(p)) + ); + assert!( + !hardware_profiles(VideoCodec::Av1, VideoPixelFormat::Nv12) + .iter() + .any(|p| legacy_gpu.contains(p)) + ); + assert!( + !hardware_profiles(VideoCodec::H265, VideoPixelFormat::P010) + .iter() + .any(|p| legacy_gpu.contains(p)) + ); + assert!(hardware_profiles(VideoCodec::H264, VideoPixelFormat::P010).is_empty()); + assert!(hardware_profiles(VideoCodec::Av1, VideoPixelFormat::Y410).is_empty()); + assert_ne!( + hardware_profiles(VideoCodec::H265, VideoPixelFormat::Y410), + hardware_profiles(VideoCodec::H265, VideoPixelFormat::P010) + ); + } + + #[test] + fn media_foundation_subtypes_preserve_depth_and_chroma() { + let cases = [ + ( + MFVideoFormat_NV12, + VideoPixelFormat::Nv12, + VideoChromaFormat::Cs420, + ), + ( + MFVideoFormat_P010, + VideoPixelFormat::P010, + VideoChromaFormat::Cs420, + ), + ( + MFVideoFormat_AYUV, + VideoPixelFormat::Ayuv, + VideoChromaFormat::Cs444, + ), + ( + MFVideoFormat_Y410, + VideoPixelFormat::Y410, + VideoChromaFormat::Cs444, + ), + ]; + for (subtype, expected_pixel, expected_chroma) in cases { + let pixel = pixel_format_from_subtype(subtype).expect("supported subtype"); + assert_eq!(pixel, expected_pixel); + assert_eq!(chroma_format(pixel), expected_chroma); + } + assert_eq!(VideoPixelFormat::Y410.bit_depth(), 10); + } + + #[test] + fn ten_bit_output_preferences_allow_safe_decoder_startup_fallbacks() { + assert_eq!( + output_format_preferences(VideoPixelFormat::P010), + &[VideoPixelFormat::P010, VideoPixelFormat::Nv12] + ); + assert_eq!( + output_format_preferences(VideoPixelFormat::Y410), + &[ + VideoPixelFormat::Y410, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Nv12, + ] + ); + assert_eq!( + output_format_preferences(VideoPixelFormat::Nv12), + &[VideoPixelFormat::Nv12] + ); + } + + #[test] + fn decoded_output_rejects_depth_or_chroma_loss_after_startup() { + for (preferred, accepted) in [ + (VideoPixelFormat::Nv12, [true, true, true, true]), + (VideoPixelFormat::P010, [false, true, false, true]), + (VideoPixelFormat::Ayuv, [true, false, true, true]), + (VideoPixelFormat::Y410, [false, true, false, true]), + ] { + for (output, accepted) in [ + VideoPixelFormat::Nv12, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] + .into_iter() + .zip(accepted) + { + let result = validate_decoded_output_format( + decoder_surface_format(output), + output, + preferred, + ); + assert_eq!( + result.is_ok(), + accepted, + "requested={preferred:?} output={output:?}: {result:?}" + ); + if !accepted { + let error = result.unwrap_err(); + assert!(error.contains(&format!("produced {output:?}"))); + assert!(error.contains(&format!("negotiated {preferred:?}"))); + } + } + } + } + + #[test] + fn chroma_fallback_rejects_mismatched_decoder_surfaces() { + for (preferred, output) in [ + (VideoPixelFormat::Ayuv, VideoPixelFormat::Nv12), + (VideoPixelFormat::Y410, VideoPixelFormat::P010), + ] { + for surface in [ + DXGI_FORMAT_NV12, + DXGI_FORMAT_P010, + DXGI_FORMAT_AYUV, + DXGI_FORMAT_Y410, + ] { + let result = validate_decoded_output_format(surface, output, preferred); + assert_eq!(result.is_ok(), surface == decoder_surface_format(output)); + if let Err(error) = result { + assert!(error.contains("does not match output media format")); + } + } + } + } + + #[test] + fn decoded_surface_must_match_media_type_before_precision_validation() { + for surface in [ + DXGI_FORMAT_NV12, + DXGI_FORMAT_P010, + DXGI_FORMAT_AYUV, + DXGI_FORMAT_Y410, + ::windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_R8G8B8A8_UNORM, + ] { + for output in [ + VideoPixelFormat::Nv12, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] { + let result = validate_decoded_output_format(surface, output, output); + assert_eq!(result.is_ok(), surface == decoder_surface_format(output)); + if let Err(error) = result { + assert!(error.contains("does not match output media format")); + } + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/embedded.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/embedded.rs new file mode 100644 index 000000000..e6db5da6b --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/embedded.rs @@ -0,0 +1,2945 @@ +use std::collections::{HashMap, VecDeque}; +use std::ffi::c_void; +use std::mem::ManuallyDrop; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{Receiver, TryRecvError, sync_channel}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle, ThreadId}; +use std::time::{Duration, Instant}; + +use ::windows::Win32::Foundation::{LUID, RECT}; +use ::windows::Win32::Graphics::Direct3D10::ID3D10Multithread; +use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, + D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_STREAM, + D3D11_VIDEO_USAGE_OPTIMAL_SPEED, D3D11_VPIV_DIMENSION_TEXTURE2D, + D3D11_VPOV_DIMENSION_TEXTURE2D, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, + ID3D11VideoContext, ID3D11VideoContext1, ID3D11VideoDevice, ID3D11VideoProcessor, + ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorEnumerator1, ID3D11VideoProcessorInputView, + ID3D11VideoProcessorOutputView, +}; +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, DXGI_FORMAT, DXGI_FORMAT_AYUV, DXGI_FORMAT_NV12, + DXGI_FORMAT_P010, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_R10G10B10A2_UNORM, DXGI_FORMAT_Y410, + DXGI_RATIONAL, DXGI_SAMPLE_DESC, +}; +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020, DXGI_COLOR_SPACE_TYPE, +}; +use ::windows::Win32::Graphics::Dxgi::IDXGIDevice; +use ::windows::Win32::Media::MediaFoundation::{ + IMFDXGIDeviceManager, MF_VERSION, MFCreateDXGIDeviceManager, MFSTARTUP_LITE, MFShutdown, + MFStartup, +}; +use ::windows::Win32::System::Com::{ + CO_MTA_USAGE_COOKIE, COINIT_MULTITHREADED, CoDecrementMTAUsage, CoIncrementMTAUsage, + CoInitializeEx, CoUninitialize, +}; +use ::windows::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_ABOVE_NORMAL, +}; +use ::windows::core::{IUnknown, Interface}; + +use crate::queue::BoundedQueue; +use crate::{ + ADAPTIVE_VIDEO_QUEUE_CAPACITY, BackendError, BackendEvent, EncodedVideoFrame, PushOutcome, + Subsystem, VideoFormat, VideoPixelFormat, VideoTransferFunction, WindowsDecoderMode, +}; + +use super::color::input_color_space; +use super::decoder::{DecodedVideoFrame, Decoder, DecoderDevice}; + +// Async Media Foundation transforms do not provide a waitable output handle. Polling at one +// millisecond keeps decoder progress independent from Qt's render cadence without spinning an +// entire core. In particular, the final HaveOutput event must be drained even when transport has +// stopped delivering compressed access units temporarily. +const DECODER_POLL_INTERVAL: Duration = Duration::from_millis(1); +pub(super) const MAX_FRAME_SLOTS: usize = 8; +// A broken driver may never return from codec work. Never join it on Qt's render +// thread, but also never accumulate unbounded abandoned workers across retries. +static LIVE_DECODER_WORKERS: AtomicUsize = AtomicUsize::new(0); +struct DecoderWorkerLease; +impl DecoderWorkerLease { + fn acquire() -> Result { + let mut count = LIVE_DECODER_WORKERS.load(Ordering::Acquire); + while let Some(next) = (count < 4).then_some(count + 1) { + match LIVE_DECODER_WORKERS.compare_exchange_weak( + count, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(Self), + Err(current) => count = current, + } + } + Err(BackendError::Startup( + "Previous decoder workers are still stopping; restart OpenNOW before retrying".into(), + )) + } +} +impl Drop for DecoderWorkerLease { + fn drop(&mut self) { + LIVE_DECODER_WORKERS.fetch_sub(1, Ordering::AcqRel); + } +} + +fn poll_decoder_startup( + startup: &mut Option>>, +) -> Result { + let Some(receiver) = startup.as_ref() else { + return Ok(true); + }; + match receiver.try_recv() { + Err(TryRecvError::Empty) => Ok(false), + result => { + *startup = None; + match result { + Ok(Ok(())) => Ok(true), + Ok(Err(error)) => Err(BackendError::Startup(format!( + "Media Foundation decoder: {error}" + ))), + _ => Err(BackendError::Startup( + "Decoder worker exited during startup".into(), + )), + } + } + } +} + +/// Borrowed Qt D3D11 objects used by the embedded frame producer. +/// +/// Both pointers must identify the D3D11 device and its immediate context from +/// Qt's adopted QRhi. The producer takes its own COM references and never +/// assumes ownership of either incoming reference. +#[derive(Debug, Clone, Copy)] +pub struct AdoptedD3d11Context { + pub device: *mut c_void, + pub immediate_context: *mut c_void, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum D3d11TextureFormat { + Rgba8, + Rgb10A2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum D3d11ColorSpace { + Sdr709, + Pq2020, + Hlg2020, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct D3d11RecordedFrame { + pub texture: *mut c_void, + pub texture_format: D3d11TextureFormat, + pub color_space: D3d11ColorSpace, + pub width: u32, + pub height: u32, + pub frame_slot: u32, + pub generation: u64, + pub presentation_time_ns: u64, +} + +#[derive(Clone)] +pub struct D3d11FrameSubmitter { + encoded: Arc>, + events: Arc>, +} + +impl D3d11FrameSubmitter { + pub fn submit_video(&self, frame: EncodedVideoFrame) -> Result { + frame.validate()?; + let key_frame = frame.key_frame; + let outcome = self + .encoded + .push_or_clear_on_overflow(frame, key_frame, |queued| queued.key_frame) + .map_err(|_| BackendError::WorkerDisconnected)?; + if outcome == PushOutcome::DroppedOldest { + let _ = self + .events + .push(BackendEvent::QueueOverflow(Subsystem::VideoDecode)); + // The submitting owner handles DroppedOldest synchronously and + // invalidates its compressed chain before submitting again. A second + // asynchronous KeyFrameRequired can arrive after the recovery IDR + // and incorrectly discard that new, healthy chain. + } + Ok(outcome) + } + + pub fn holds_keyframe(&self) -> bool { + self.encoded.any(|queued| queued.key_frame) + } +} + +struct EmbeddedMediaRuntime { + mta_cookie: usize, + media_foundation_started: bool, +} + +impl EmbeddedMediaRuntime { + fn initialize() -> Result { + super::ensure_media_foundation_available()?; + unsafe { + let mta_cookie = + CoIncrementMTAUsage().map_err(|error| format!("CoIncrementMTAUsage: {error}"))?; + if let Err(error) = MFStartup(MF_VERSION, MFSTARTUP_LITE) { + let _ = CoDecrementMTAUsage(mta_cookie); + return Err(format!("MFStartup: {error}")); + } + Ok(Self { + mta_cookie: mta_cookie.0 as usize, + media_foundation_started: true, + }) + } + } +} + +impl Drop for EmbeddedMediaRuntime { + fn drop(&mut self) { + unsafe { + if self.media_foundation_started { + let _ = MFShutdown(); + } + let _ = CoDecrementMTAUsage(CO_MTA_USAGE_COOKIE(self.mta_cookie as *mut c_void)); + } + } +} + +struct FrameSlot { + texture: ID3D11Texture2D, + output_view: ID3D11VideoProcessorOutputView, +} + +impl FrameSlot { + fn new( + device: &ID3D11Device, + video_device: &ID3D11VideoDevice, + enumerator: &ID3D11VideoProcessorEnumerator, + width: u32, + height: u32, + format: DXGI_FORMAT, + ) -> Result { + let description = frame_slot_description(width, height, format); + let mut texture = None; + unsafe { + device + .CreateTexture2D(&description, None, Some(&mut texture)) + .map_err(|error| format!("CreateTexture2D frame slot: {error}"))?; + } + let texture = texture.ok_or("D3D11 returned no frame-slot texture")?; + let description = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { + ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 }, + }, + }; + let mut output_view = None; + unsafe { + video_device + .CreateVideoProcessorOutputView( + &texture, + enumerator, + &description, + Some(&mut output_view), + ) + .map_err(|error| format!("CreateVideoProcessorOutputView: {error}"))?; + } + Ok(Self { + texture, + output_view: output_view.ok_or("D3D11 returned no frame-slot output view")?, + }) + } +} + +struct ProcessorResources { + input_width: u32, + input_height: u32, + input_format: DXGI_FORMAT, + output_width: u32, + output_height: u32, + output_format: DXGI_FORMAT, + enumerator: ID3D11VideoProcessorEnumerator, + processor: ID3D11VideoProcessor, + input_views: HashMap<(usize, u32), ID3D11VideoProcessorInputView>, + slots: [Option; MAX_FRAME_SLOTS], +} + +struct AdoptedResources { + device: ID3D11Device, + context: ID3D11DeviceContext, + video_device: ID3D11VideoDevice, + video_context: ID3D11VideoContext, + video_context_1: ID3D11VideoContext1, + manager: IMFDXGIDeviceManager, + format: VideoFormat, + generation: u64, + processor: Option, + y410: Option, +} + +impl AdoptedResources { + unsafe fn new(adopted: AdoptedD3d11Context, format: VideoFormat) -> Result { + if adopted.device.is_null() || adopted.immediate_context.is_null() { + return Err("Qt supplied a null D3D11 device or immediate context".to_owned()); + } + let device = unsafe { clone_interface::(adopted.device)? }; + let context = unsafe { clone_interface::(adopted.immediate_context)? }; + let context_device = unsafe { + context + .GetDevice() + .map_err(|error| format!("ID3D11DeviceContext::GetDevice: {error}"))? + }; + if com_identity(&device)? != com_identity(&context_device)? { + return Err( + "Qt D3D11 immediate context does not belong to the adopted device".to_owned(), + ); + } + enable_multithread_protection(&context)?; + let video_device = device + .cast() + .map_err(|error| format!("Qt D3D11 device has no video interface: {error}"))?; + let video_context: ID3D11VideoContext = context + .cast() + .map_err(|error| format!("Qt immediate context has no video interface: {error}"))?; + let video_context_1 = video_context.cast().map_err(|error| { + format!("Qt D3D11 context cannot configure explicit color spaces: {error}") + })?; + let mut reset_token = 0; + let mut manager = None; + unsafe { + MFCreateDXGIDeviceManager(&mut reset_token, &mut manager) + .map_err(|error| format!("MFCreateDXGIDeviceManager: {error}"))?; + } + let manager = manager.ok_or("MFCreateDXGIDeviceManager returned no manager")?; + unsafe { + manager + .ResetDevice(&device, reset_token) + .map_err(|error| format!("IMFDXGIDeviceManager::ResetDevice: {error}"))?; + } + Ok(Self { + device, + context, + video_device, + video_context, + video_context_1, + manager, + format, + generation: 0, + processor: None, + y410: None, + }) + } + + fn reconfigure(&mut self, format: VideoFormat) { + if self.format != format { + self.format = format; + self.processor = None; + self.y410 = None; + } + } + + fn reset_decoder_views(&mut self) { + if let Some(processor) = self.processor.as_mut() { + processor.input_views.clear(); + } + } + + unsafe fn validate_adopted_context(&self, adopted: AdoptedD3d11Context) -> Result<(), String> { + let device = unsafe { clone_interface::(adopted.device)? }; + let context = unsafe { clone_interface::(adopted.immediate_context)? }; + if com_identity(&device)? != com_identity(&self.device)? + || com_identity(&context)? != com_identity(&self.video_context)? + { + return Err("D3D11 frame belongs to an older Qt graphics context".to_owned()); + } + Ok(()) + } + + fn record( + &mut self, + frame_slot: u32, + frame: &DecodedVideoFrame, + ) -> Result { + let slot = usize::try_from(frame_slot) + .ok() + .filter(|slot| *slot < MAX_FRAME_SLOTS) + .ok_or_else(|| format!("invalid D3D11 frame slot {frame_slot}"))?; + let mut input_description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + frame.texture.GetDesc(&mut input_description); + } + let pixel_format = pixel_format_from_dxgi(input_description.Format).ok_or_else(|| { + format!( + "Media Foundation returned unsupported D3D11 texture format {}", + input_description.Format.0 + ) + })?; + if pixel_format != frame.format.pixel_format { + return Err(format!( + "decoder texture format {pixel_format:?} does not match media format {:?}", + frame.format.pixel_format + )); + } + frame + .aperture + .validate_extent(input_description.Width, input_description.Height)?; + let array_slice = decoder_array_slice( + frame.subresource, + input_description.MipLevels, + input_description.ArraySize, + )?; + self.reconfigure(frame.format); + if frame.format.pixel_format == VideoPixelFormat::Y410 { + if self.y410.is_none() { + self.y410 = Some(super::y410::Y410Converter::new( + &self.device, + &self.context, + frame.format, + )?); + } + let texture = self + .y410 + .as_mut() + .ok_or("no Y410 converter")? + .record(slot, frame)?; + self.generation = self.generation.wrapping_add(1).max(1); + return Ok(D3d11RecordedFrame { + texture: texture.as_raw(), + texture_format: D3d11TextureFormat::Rgb10A2, + color_space: if frame.format.transfer_function == VideoTransferFunction::Pq { + D3d11ColorSpace::Pq2020 + } else { + D3d11ColorSpace::Sdr709 + }, + width: frame.format.width, + height: frame.format.height, + frame_slot, + generation: self.generation, + presentation_time_ns: u64::try_from(frame.timestamp_100ns.max(0)) + .unwrap_or(0) + .saturating_mul(100), + }); + } + self.ensure_processor( + input_description.Width, + input_description.Height, + input_description.Format, + frame.aperture.width, + frame.aperture.height, + )?; + let processor = self + .processor + .as_mut() + .ok_or("embedded video processor is unavailable")?; + let input_key = (frame.texture.as_raw() as usize, array_slice); + let input_view = if let Some(view) = processor.input_views.get(&input_key) { + view.clone() + } else { + let description = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { + FourCC: 0, + ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPIV { + MipSlice: 0, + ArraySlice: array_slice, + }, + }, + }; + let mut view = None; + unsafe { + self.video_device + .CreateVideoProcessorInputView( + &frame.texture, + &processor.enumerator, + &description, + Some(&mut view), + ) + .map_err(|error| format!("CreateVideoProcessorInputView: {error}"))?; + } + let view = view.ok_or("D3D11 returned no video processor input view")?; + processor.input_views.insert(input_key, view.clone()); + view + }; + let source = RECT { + left: frame.aperture.x as i32, + top: frame.aperture.y as i32, + right: (frame.aperture.x + frame.aperture.width) as i32, + bottom: (frame.aperture.y + frame.aperture.height) as i32, + }; + let destination = RECT { + left: 0, + top: 0, + right: processor.output_width as i32, + bottom: processor.output_height as i32, + }; + if processor.slots[slot].is_none() { + processor.slots[slot] = Some(FrameSlot::new( + &self.device, + &self.video_device, + &processor.enumerator, + processor.output_width, + processor.output_height, + processor.output_format, + )?); + } + let active_slot = processor.slots[slot] + .as_ref() + .expect("initialized frame slot"); + let output_view = active_slot.output_view.clone(); + unsafe { + self.video_context.VideoProcessorSetStreamFrameFormat( + &processor.processor, + 0, + D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + ); + self.video_context.VideoProcessorSetStreamSourceRect( + &processor.processor, + 0, + true, + Some(&source), + ); + self.video_context.VideoProcessorSetStreamDestRect( + &processor.processor, + 0, + true, + Some(&destination), + ); + self.video_context.VideoProcessorSetOutputTargetRect( + &processor.processor, + true, + Some(&RECT { + left: 0, + top: 0, + right: processor.output_width as i32, + bottom: processor.output_height as i32, + }), + ); + let mut stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + pInputSurface: ManuallyDrop::new(Some(input_view)), + ..Default::default() + }; + let result = self.video_context.VideoProcessorBlt( + &processor.processor, + &output_view, + 0, + std::slice::from_ref(&stream), + ); + ManuallyDrop::drop(&mut stream.pInputSurface); + result.map_err(|error| format!("VideoProcessorBlt: {error}"))?; + } + self.generation = self.generation.wrapping_add(1).max(1); + Ok(D3d11RecordedFrame { + texture: active_slot.texture.as_raw(), + texture_format: d3d11_texture_format(processor.output_format), + color_space: recorded_color_space(self.format.transfer_function), + width: processor.output_width, + height: processor.output_height, + frame_slot, + generation: self.generation, + presentation_time_ns: u64::try_from(frame.timestamp_100ns.max(0)) + .unwrap_or(0) + .saturating_mul(100), + }) + } + + fn ensure_processor( + &mut self, + input_width: u32, + input_height: u32, + input_format: DXGI_FORMAT, + output_width: u32, + output_height: u32, + ) -> Result<(), String> { + self.format + .validate_color() + .map_err(|error| error.to_string())?; + let input_color_space = input_color_space(self.format)?; + let output_format = output_dxgi_format(self.format); + if self.processor.as_ref().is_some_and(|processor| { + processor.input_width == input_width + && processor.input_height == input_height + && processor.input_format == input_format + && processor.output_width == output_width + && processor.output_height == output_height + && processor.output_format == output_format + }) { + return Ok(()); + } + let description = D3D11_VIDEO_PROCESSOR_CONTENT_DESC { + InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + InputFrameRate: DXGI_RATIONAL { + Numerator: self.format.frame_rate_numerator.get(), + Denominator: self.format.frame_rate_denominator.get(), + }, + InputWidth: input_width, + InputHeight: input_height, + OutputFrameRate: DXGI_RATIONAL { + Numerator: self.format.frame_rate_numerator.get(), + Denominator: self.format.frame_rate_denominator.get(), + }, + OutputWidth: output_width, + OutputHeight: output_height, + Usage: D3D11_VIDEO_USAGE_OPTIMAL_SPEED, + }; + let enumerator = unsafe { + self.video_device + .CreateVideoProcessorEnumerator(&description) + .map_err(|error| format!("CreateVideoProcessorEnumerator: {error}"))? + }; + validate_conversion(&enumerator, input_format, output_format, self.format)?; + let processor = unsafe { + self.video_device + .CreateVideoProcessor(&enumerator, 0) + .map_err(|error| format!("CreateVideoProcessor: {error}"))? + }; + unsafe { + self.video_context_1.VideoProcessorSetStreamColorSpace1( + &processor, + 0, + input_color_space, + ); + self.video_context_1.VideoProcessorSetOutputColorSpace1( + &processor, + output_color_space(self.format.transfer_function), + ); + // Driver auto-processing modes apply vendor-specific picture + // enhancements (e.g. dynamic contrast and brightness boosts on Intel/AMD) + // that distort SDR gamma and blow out highlights. Disable them unconditionally. + self.video_context_1 + .VideoProcessorSetStreamAutoProcessingMode(&processor, 0, false); + } + // QRhi owns slot reuse. Allocate only the slots it actually visits; + // reserving the ABI maximum would waste seven 4K textures on D3D11. + let slots = std::array::from_fn(|_| None); + self.processor = Some(ProcessorResources { + input_width, + input_height, + input_format, + output_width, + output_height, + output_format, + enumerator, + processor, + input_views: HashMap::new(), + slots, + }); + Ok(()) + } +} + +pub(super) unsafe fn d3d11_adapter_luid( + device: *mut c_void, +) -> Result { + if device.is_null() { + return Err("Qt supplied a null D3D11 device".to_owned()); + } + let device = unsafe { clone_interface::(device)? }; + let dxgi_device: IDXGIDevice = device + .cast() + .map_err(|error| format!("Qt D3D11 DXGI device: {error}"))?; + let adapter = unsafe { + dxgi_device + .GetAdapter() + .map_err(|error| format!("Qt D3D11 adapter: {error}"))? + }; + let description = unsafe { + adapter + .GetDesc() + .map_err(|error| format!("Qt D3D11 adapter description: {error}"))? + }; + let raw = u64::from(description.AdapterLuid.LowPart) + | ((description.AdapterLuid.HighPart as u32 as u64) << 32); + crate::WindowsAdapterLuid::new(raw) + .ok_or_else(|| "Qt D3D11 device reported a zero adapter LUID".to_owned()) +} + +#[cfg(test)] +pub(super) unsafe fn probe_hdr_conversion( + adopted: AdoptedD3d11Context, + format: VideoFormat, +) -> Result<(), String> { + if format.transfer_function != VideoTransferFunction::Pq + || format.pixel_format != VideoPixelFormat::P010 + { + return Err("HDR10 capability requires actual P010/PQ decoder output".to_owned()); + } + unsafe { probe_conversion(adopted, format) } +} + +#[cfg(test)] +unsafe fn probe_conversion( + adopted: AdoptedD3d11Context, + format: VideoFormat, +) -> Result<(), String> { + let input_format = match format.pixel_format { + VideoPixelFormat::Nv12 => DXGI_FORMAT_NV12, + VideoPixelFormat::P010 => DXGI_FORMAT_P010, + VideoPixelFormat::Ayuv => DXGI_FORMAT_AYUV, + VideoPixelFormat::Y410 => DXGI_FORMAT_Y410, + }; + let mut resources = unsafe { AdoptedResources::new(adopted, format)? }; + resources.ensure_processor( + format.width, + format.height, + input_format, + format.width, + format.height, + ) +} + +pub(super) unsafe fn probe_decoded_conversion( + adopted: AdoptedD3d11Context, + frame: &DecodedVideoFrame, +) -> Result<(), String> { + let mut resources = unsafe { AdoptedResources::new(adopted, frame.format)? }; + resources.record(0, frame)?; + Ok(()) +} + +fn enable_multithread_protection(context: &ID3D11DeviceContext) -> Result<(), String> { + // Media Foundation may use the adopted device and immediate context from an asynchronous + // decoder work-queue thread while Qt records and presents on QSGRenderThread. D3D11 immediate + // contexts are not thread-safe unless ID3D10Multithread protection is enabled. Without it, + // NVIDIA's user-mode driver can deadlock one thread in decoder-buffer acquisition and the + // other in DXGI Present. + let multithread: ID3D10Multithread = context + .cast() + .map_err(|error| format!("Qt D3D11 context has no multithread interface: {error}"))?; + unsafe { + let _ = multithread.SetMultithreadProtected(true); + if !multithread.GetMultithreadProtected().as_bool() { + return Err("Qt D3D11 context rejected multithread protection".to_owned()); + } + } + Ok(()) +} + +impl DecoderDevice for AdoptedResources { + fn device_manager(&self) -> &IMFDXGIDeviceManager { + &self.manager + } + + fn adapter_luid(&self) -> Result { + unsafe { + let dxgi_device: IDXGIDevice = self + .device + .cast() + .map_err(|error| format!("Qt D3D11 DXGI device: {error}"))?; + let adapter = dxgi_device + .GetAdapter() + .map_err(|error| format!("Qt D3D11 adapter: {error}"))?; + let description = adapter + .GetDesc() + .map_err(|error| format!("Qt D3D11 adapter description: {error}"))?; + let name_end = description + .Description + .iter() + .position(|c| *c == 0) + .unwrap_or(description.Description.len()); + video_log!( + "Qt decoder adapter name={} vendor={:04x} device={:04x} dedicated_video_mb={}", + String::from_utf16_lossy(&description.Description[..name_end]), + description.VendorId, + description.DeviceId, + description.DedicatedVideoMemory / (1024 * 1024) + ); + Ok(description.AdapterLuid) + } + } + + fn video_format(&self) -> VideoFormat { + self.format + } +} + +#[derive(Clone)] +struct DecoderDeviceSnapshot { + manager: IMFDXGIDeviceManager, + adapter_luid: LUID, + format: VideoFormat, +} + +// IMFDXGIDeviceManager is the documented synchronization boundary for sharing one D3D11 device +// with Media Foundation. The adopted immediate context has ID3D10Multithread protection enabled +// before this snapshot is created, and the snapshot remains alive until the decoder thread joins. +unsafe impl Send for DecoderDeviceSnapshot {} + +impl DecoderDeviceSnapshot { + fn new(resources: &AdoptedResources) -> Result { + Ok(Self { + manager: resources.manager.clone(), + adapter_luid: resources.adapter_luid()?, + format: resources.format, + }) + } +} + +impl DecoderDevice for DecoderDeviceSnapshot { + fn device_manager(&self) -> &IMFDXGIDeviceManager { + &self.manager + } + + fn adapter_luid(&self) -> Result { + Ok(self.adapter_luid) + } + + fn video_format(&self) -> VideoFormat { + self.format + } +} + +struct ReadyDecodedFrame { + frame: DecodedVideoFrame, + decoder_generation: u64, +} + +// The decoder surface AND its Media Foundation sample lease move together. +// The sample prevents decoder-pool reuse while Qt records the video blit on +// the same protected immediate context. Neither is mutated through this queue. +unsafe impl Send for ReadyDecodedFrame {} + +struct D3d11Pipeline { + owner_thread: ThreadId, + resources: AdoptedResources, + decoded: Arc>>, + encoded: Arc>, + events: Arc>, + presented_decoder_generation: u64, + stopping: Arc, + decoder_worker: Option>, + startup: Option>>, + _runtime: Arc, +} + +unsafe impl Send for D3d11Pipeline {} + +pub struct D3d11Frame { + frame: DecodedVideoFrame, + state: Arc>, + sequence: u64, +} + +unsafe impl Send for D3d11Frame {} +unsafe impl Sync for D3d11Frame {} + +impl D3d11Frame { + pub fn format(&self) -> VideoFormat { + self.frame.format + } + + pub fn width(&self) -> u32 { + self.frame.aperture.width + } + + pub fn height(&self) -> u32 { + self.frame.aperture.height + } + + pub const fn sequence(&self) -> u64 { + self.sequence + } + + pub fn presentation_time_ns(&self) -> u64 { + u64::try_from(self.frame.timestamp_100ns.max(0)) + .unwrap_or(0) + .saturating_mul(100) + } + + /// Converts this decoded surface into Qt's matching 8-bit or 10-bit RGB frame-slot target. + /// + /// # Safety + /// + /// `target.texture` must identify a live `ID3D11Texture2D` from the adopted + /// Qt device. Recording must run on the render thread that created the + /// producer. + pub unsafe fn record( + &self, + adopted: AdoptedD3d11Context, + frame_slot: u32, + ) -> Result { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.ensure_render_thread()?; + unsafe { state.resources.validate_adopted_context(adopted) }.map_err(|message| { + BackendError::DeviceLost { + subsystem: Subsystem::VideoPresentation, + message, + } + })?; + state + .resources + .record(frame_slot, &self.frame) + .map_err(|message| BackendError::DeviceLost { + subsystem: Subsystem::VideoPresentation, + message, + }) + } +} + +#[derive(Clone)] +pub struct D3d11FrameProducer { + state: Arc>, + sequence: Arc, +} + +impl D3d11FrameProducer { + /// Adopts Qt's D3D11 device and immediate context without creating a + /// window, swap chain, SDL video subsystem, or presentation path. + /// + /// # Safety + /// + /// Both COM pointers must be valid for this call and identify live + /// `ID3D11Device` and `ID3D11DeviceContext` interfaces. The context must be + /// Qt's immediate context, and all producer methods except submission and + /// event polling must remain on the creating render thread. + pub unsafe fn new( + adopted: AdoptedD3d11Context, + format: VideoFormat, + decoder_mode: WindowsDecoderMode, + frame_ready: Arc, + ) -> Result<(Self, D3d11FrameSubmitter), BackendError> { + format.validate()?; + let worker_lease = DecoderWorkerLease::acquire()?; + let runtime = Arc::new(EmbeddedMediaRuntime::initialize().map_err(BackendError::Startup)?); + let resources = + unsafe { AdoptedResources::new(adopted, format) }.map_err(BackendError::Startup)?; + let decoder_device = + DecoderDeviceSnapshot::new(&resources).map_err(BackendError::Startup)?; + let encoded = Arc::new(BoundedQueue::new(ADAPTIVE_VIDEO_QUEUE_CAPACITY)); + let events = Arc::new(BoundedQueue::new(64)); + let decoded = Arc::new(Mutex::new(VecDeque::with_capacity( + ADAPTIVE_VIDEO_QUEUE_CAPACITY, + ))); + let decoder_generation = Arc::new(AtomicU64::new(1)); + let stopping = Arc::new(AtomicBool::new(false)); + let submitter = D3d11FrameSubmitter { + encoded: Arc::clone(&encoded), + events: Arc::clone(&events), + }; + let (startup_sender, startup_receiver) = sync_channel(1); + let worker_encoded = Arc::clone(&encoded); + let worker_decoded = Arc::clone(&decoded); + let worker_events = Arc::clone(&events); + let worker_generation = Arc::clone(&decoder_generation); + let worker_stopping = Arc::clone(&stopping); + let worker_runtime = Arc::clone(&runtime); + let decoder_worker = thread::Builder::new() + .name("opennow-mf-video-decode".to_owned()) + .spawn(move || { + let _worker_lease = worker_lease; + let _runtime = worker_runtime; + let notify = Arc::clone(&frame_ready); + let stopping = Arc::clone(&worker_stopping); + run_decoder_worker( + decoder_device, + format, + decoder_mode, + worker_encoded, + worker_decoded, + worker_events, + worker_generation, + worker_stopping, + frame_ready, + startup_sender, + ); + // Startup failures must wake the presenter to consume the error. + if !stopping.load(Ordering::Acquire) { + notify(); + } + }) + .map_err(|error| { + BackendError::Startup(format!("start embedded decoder worker: {error}")) + })?; + let state = Arc::new(Mutex::new(D3d11Pipeline { + owner_thread: thread::current().id(), + resources, + decoded, + encoded, + events, + presented_decoder_generation: 0, + stopping, + decoder_worker: Some(decoder_worker), + startup: Some(startup_receiver), + _runtime: runtime, + })); + Ok(( + Self { + state, + sequence: Arc::new(AtomicU64::new(0)), + }, + submitter, + )) + } + + pub fn acquire_latest(&self) -> Result, BackendError> { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let frame = state.acquire_latest()?; + Ok(frame.map(|frame| D3d11Frame { + frame, + state: Arc::clone(&self.state), + sequence: self.sequence.fetch_add(1, Ordering::AcqRel) + 1, + })) + } + + pub fn try_event(&self) -> Option { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .events + .try_pop() + } +} + +impl D3d11Pipeline { + fn acquire_latest(&mut self) -> Result, BackendError> { + self.ensure_render_thread()?; + if !poll_decoder_startup(&mut self.startup)? { + return Ok(None); + } + let mut decoded = self + .decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let Some(ready) = decoded.pop_back() else { + return Ok(None); + }; + if !decoded.is_empty() { + decoded.clear(); + let _ = self + .events + .push(BackendEvent::QueueOverflow(Subsystem::VideoPresentation)); + } + drop(decoded); + if ready.decoder_generation != self.presented_decoder_generation { + self.resources.reset_decoder_views(); + self.presented_decoder_generation = ready.decoder_generation; + } + Ok(Some(ready.frame)) + } + + fn ensure_render_thread(&self) -> Result<(), BackendError> { + if thread::current().id() == self.owner_thread { + Ok(()) + } else { + Err(BackendError::InvalidConfig( + "embedded D3D11 frame production must stay on Qt's render thread".to_owned(), + )) + } + } +} + +impl Drop for D3d11Pipeline { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Release); + self.encoded.close(); + // The worker retains its device, queues, and MF runtime until it exits. + // Dropping a JoinHandle detaches, without waiting for driver shutdown. + drop(self.decoder_worker.take()); + self.decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clear(); + } +} + +struct DecoderThreadApartment; + +impl DecoderThreadApartment { + fn initialize() -> Result { + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED) + .ok() + .map_err(|error| format!("CoInitializeEx: {error}"))?; + let _ = SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL); + } + Ok(Self) + } +} + +impl Drop for DecoderThreadApartment { + fn drop(&mut self) { + unsafe { + CoUninitialize(); + } + } +} + +#[allow(clippy::too_many_arguments)] +fn run_decoder_worker( + device: DecoderDeviceSnapshot, + format: VideoFormat, + mode: WindowsDecoderMode, + encoded: Arc>, + decoded: Arc>>, + events: Arc>, + decoder_generation: Arc, + stopping: Arc, + frame_ready: Arc, + startup_sender: std::sync::mpsc::SyncSender>, +) { + let _apartment = match DecoderThreadApartment::initialize() { + Ok(apartment) => apartment, + Err(error) => { + video_log!("Embedded D3D11 COM initialization failed: {error}"); + let _ = startup_sender.send(Err(error)); + return; + } + }; + let mut decoder = match Decoder::new(&device, format, mode) { + Ok(decoder) => decoder, + Err(error) => { + video_log!("Embedded D3D11 decoder initialization failed: {error}"); + let _ = startup_sender.send(Err(error)); + return; + } + }; + if startup_sender.send(Ok(())).is_err() { + decoder.stop(); + return; + } + + opennow_streamer_protocol::log::diagnostic( + "INFO", + "decode", + &format!( + "Embedded D3D11 decoder worker started codec={} pollMs={}", + format.codec.label(), + DECODER_POLL_INTERVAL.as_millis() + ), + ); + let mut submitted_any = false; + // One worker-owned access unit may be waiting for a replacement MFT's + // NeedInput event. Keep its already-queued descendants in order. + let mut pending_frame = None; + let mut submitted_frames = 0_u64; + let mut produced_frames = 0_u64; + let mut last_progress_log = Instant::now(); + let mut output = VecDeque::with_capacity(2); + while !stopping.load(Ordering::Acquire) { + let mut made_progress = false; + output.clear(); + match decoder.poll_output(&mut output, &events) { + Ok(produced) => { + made_progress |= produced > 0; + produced_frames = produced_frames.saturating_add(produced as u64); + if produced > 0 { + let generation = decoder_generation.load(Ordering::Acquire); + let mut ready = decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + while let Some(frame) = output.pop_front() { + if ready.len() == ADAPTIVE_VIDEO_QUEUE_CAPACITY { + ready.pop_front(); + let _ = events + .push(BackendEvent::QueueOverflow(Subsystem::VideoPresentation)); + } + ready.push_back(ReadyDecodedFrame { + frame, + decoder_generation: generation, + }); + } + drop(ready); + frame_ready(); + } + } + Err(message) => { + decoder.stop(); + decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clear(); + encoded.clear(); + decoder_generation.fetch_add(1, Ordering::AcqRel); + opennow_streamer_protocol::log::diagnostic( + "WARN", + "decode", + &format!("Embedded D3D11 decoder failed: {message}"), + ); + let _ = events.push(BackendEvent::DeviceLost { + subsystem: Subsystem::VideoDecode, + message, + }); + let _ = events.push(BackendEvent::KeyFrameRequired); + frame_ready(); + pending_frame = wait_for_recovery_keyframe( + &device, + format, + mode, + &encoded, + &decoded, + &events, + &decoder_generation, + &stopping, + frame_ready.as_ref(), + &mut decoder, + &mut submitted_any, + ); + continue; + } + } + + while decoder.wants_input() { + let Some(frame) = next_encoded_frame(&mut pending_frame, &encoded) else { + break; + }; + if frame.reset_decoder && submitted_any { + decoder.stop(); + decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clear(); + decoder_generation.fetch_add(1, Ordering::AcqRel); + match Decoder::new(&device, format, mode) { + Ok(replacement) => { + decoder = replacement; + submitted_any = false; + // The queued frames follow this keyframe. Clearing them + // silently breaks the next P-frame's reference chain. + pending_frame = Some(frame); + made_progress = true; + break; + } + Err(message) => { + opennow_streamer_protocol::log::diagnostic( + "WARN", + "decode", + &format!("Embedded D3D11 decoder reset failed: {message}"), + ); + let _ = events.push(BackendEvent::DeviceLost { + subsystem: Subsystem::VideoDecode, + message, + }); + let _ = events.push(BackendEvent::KeyFrameRequired); + frame_ready(); + pending_frame = wait_for_recovery_keyframe( + &device, + format, + mode, + &encoded, + &decoded, + &events, + &decoder_generation, + &stopping, + frame_ready.as_ref(), + &mut decoder, + &mut submitted_any, + ); + made_progress = true; + break; + } + } + } + if let Err(message) = decoder.submit(frame) { + decoder.stop(); + decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clear(); + encoded.clear(); + decoder_generation.fetch_add(1, Ordering::AcqRel); + opennow_streamer_protocol::log::diagnostic( + "WARN", + "decode", + &format!("Embedded D3D11 decoder input failed: {message}"), + ); + let _ = events.push(BackendEvent::DeviceLost { + subsystem: Subsystem::VideoDecode, + message, + }); + let _ = events.push(BackendEvent::KeyFrameRequired); + frame_ready(); + pending_frame = wait_for_recovery_keyframe( + &device, + format, + mode, + &encoded, + &decoded, + &events, + &decoder_generation, + &stopping, + frame_ready.as_ref(), + &mut decoder, + &mut submitted_any, + ); + made_progress = true; + break; + } + submitted_any = true; + submitted_frames = submitted_frames.saturating_add(1); + made_progress = true; + } + + if last_progress_log.elapsed() >= Duration::from_secs(10) { + let decoded_ready = decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .len(); + opennow_streamer_protocol::log::diagnostic( + "INFO", + "decode", + &format!( + "Embedded D3D11 decoder progress codec={} submitted={submitted_frames} produced={produced_frames} encodedQueued={} decodedReady={decoded_ready} generation={}", + format.codec.label(), + encoded.len(), + decoder_generation.load(Ordering::Acquire), + ), + ); + last_progress_log = Instant::now(); + } + + if !made_progress { + let _ = encoded.wait_for_decoder(DECODER_POLL_INTERVAL, decoder.wants_input()); + } else { + thread::yield_now(); + } + } + decoder.stop(); + video_log!("Embedded D3D11 decoder worker stopped"); +} + +fn next_encoded_frame( + pending: &mut Option, + encoded: &BoundedQueue, +) -> Option { + pending.take().or_else(|| encoded.try_pop()) +} + +#[allow(clippy::too_many_arguments)] +fn wait_for_recovery_keyframe( + device: &DecoderDeviceSnapshot, + format: VideoFormat, + mode: WindowsDecoderMode, + encoded: &BoundedQueue, + decoded: &Mutex>, + events: &BoundedQueue, + decoder_generation: &AtomicU64, + stopping: &AtomicBool, + frame_ready: &dyn Fn(), + decoder: &mut Decoder, + submitted_any: &mut bool, +) -> Option { + while !stopping.load(Ordering::Acquire) { + let Some(frame) = encoded.pop_timeout(DECODER_POLL_INTERVAL) else { + continue; + }; + if !frame.key_frame { + continue; + } + match Decoder::new(device, format, mode) { + Ok(replacement) => { + *decoder = replacement; + *submitted_any = false; + decoded + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clear(); + decoder_generation.fetch_add(1, Ordering::AcqRel); + let _ = events.push(BackendEvent::DeviceRecovered(Subsystem::VideoDecode)); + frame_ready(); + return Some(frame); + } + Err(message) => { + opennow_streamer_protocol::log::diagnostic( + "WARN", + "decode", + &format!("Embedded D3D11 decoder recovery failed: {message}"), + ); + let _ = events.push(BackendEvent::DeviceLost { + subsystem: Subsystem::VideoDecode, + message, + }); + let _ = events.push(BackendEvent::KeyFrameRequired); + frame_ready(); + } + } + } + None +} + +unsafe fn clone_interface(pointer: *mut c_void) -> Result { + if pointer.is_null() { + return Err("cannot adopt a null COM interface".to_owned()); + } + let borrowed = ManuallyDrop::new(unsafe { T::from_raw(pointer) }); + Ok((*borrowed).clone()) +} + +fn com_identity(interface: &T) -> Result { + interface + .cast::() + .map(|unknown| unknown.as_raw() as usize) + .map_err(|error| error.to_string()) +} + +fn decoder_array_slice(subresource: u32, mip_levels: u32, array_size: u32) -> Result { + let mip_levels = mip_levels.max(1); + let array_size = array_size.max(1); + let total = mip_levels + .checked_mul(array_size) + .ok_or("decoder texture subresource count overflowed")?; + if subresource >= total { + return Err(format!( + "decoder subresource {subresource} exceeds texture layout ({mip_levels} mips, {array_size} slices)" + )); + } + Ok(subresource / mip_levels) +} + +fn frame_slot_description(width: u32, height: u32, format: DXGI_FORMAT) -> D3D11_TEXTURE2D_DESC { + D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET).0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + } +} + +fn validate_conversion( + enumerator: &ID3D11VideoProcessorEnumerator, + input_format: DXGI_FORMAT, + output_format: DXGI_FORMAT, + format: VideoFormat, +) -> Result<(), String> { + let input_color_space = input_color_space(format)?; + unsafe { + let support = enumerator + .CheckVideoProcessorFormat(input_format) + .map_err(|error| format!("query decoder video-processor input: {error}"))?; + if support & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT.0 as u32 == 0 { + return Err(format!( + "D3D11 video processor does not support input format {}", + input_format.0 + )); + } + let support = enumerator + .CheckVideoProcessorFormat(output_format) + .map_err(|error| format!("query RGB video-processor output: {error}"))?; + if support & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT.0 as u32 == 0 { + return Err(format!( + "D3D11 video processor does not support output format {}", + output_format.0 + )); + } + let enumerator_1 = enumerator + .cast::() + .map_err(|error| format!("D3D11 cannot validate explicit color conversion: {error}"))?; + let supported = enumerator_1 + .CheckVideoProcessorFormatConversion( + input_format, + input_color_space, + output_format, + output_color_space(format.transfer_function), + ) + .map_err(|error| format!("query D3D11 embedded conversion: {error}"))?; + if !supported.as_bool() { + return Err(format!( + "D3D11 driver rejects embedded video conversion {} (color space {}) -> {} (color space {})", + input_format.0, + input_color_space.0, + output_format.0, + output_color_space(format.transfer_function).0, + )); + } + } + Ok(()) +} + +fn output_dxgi_format(format: VideoFormat) -> DXGI_FORMAT { + if format.transfer_function != VideoTransferFunction::Sdr || format.pixel_format.bit_depth() > 8 + { + DXGI_FORMAT_R10G10B10A2_UNORM + } else { + DXGI_FORMAT_R8G8B8A8_UNORM + } +} + +fn output_color_space(transfer: VideoTransferFunction) -> DXGI_COLOR_SPACE_TYPE { + match transfer { + VideoTransferFunction::Sdr => DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, + VideoTransferFunction::Pq | VideoTransferFunction::Hlg => { + DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 + } + } +} + +fn recorded_color_space(transfer: VideoTransferFunction) -> D3d11ColorSpace { + match transfer { + VideoTransferFunction::Sdr => D3d11ColorSpace::Sdr709, + VideoTransferFunction::Pq | VideoTransferFunction::Hlg => D3d11ColorSpace::Pq2020, + } +} + +fn d3d11_texture_format(format: DXGI_FORMAT) -> D3d11TextureFormat { + if format == DXGI_FORMAT_R10G10B10A2_UNORM { + D3d11TextureFormat::Rgb10A2 + } else { + D3d11TextureFormat::Rgba8 + } +} + +fn pixel_format_from_dxgi(format: DXGI_FORMAT) -> Option { + match format { + DXGI_FORMAT_NV12 => Some(VideoPixelFormat::Nv12), + DXGI_FORMAT_P010 => Some(VideoPixelFormat::P010), + DXGI_FORMAT_AYUV => Some(VideoPixelFormat::Ayuv), + DXGI_FORMAT_Y410 => Some(VideoPixelFormat::Y410), + _ => None, + } +} + +#[cfg(test)] +fn chroma_format(format: VideoPixelFormat) -> crate::VideoChromaFormat { + use crate::VideoChromaFormat; + match format { + VideoPixelFormat::Nv12 | VideoPixelFormat::P010 => VideoChromaFormat::Cs420, + VideoPixelFormat::Ayuv | VideoPixelFormat::Y410 => VideoChromaFormat::Cs444, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{VideoChromaFormat, VideoChromaSiting, VideoColorMatrix}; + use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_CPU_ACCESS_READ, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_USAGE_STAGING, + }; + use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709, DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020, + DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020, + }; + + #[test] + fn padded_decoder_frames_crop_and_reuse_processor_views_and_slots() { + let _runtime = EmbeddedMediaRuntime::initialize().expect("Media Foundation"); + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 test device"); + } + let device = device.unwrap(); + let context = context.unwrap(); + if let Err(error) = device.cast::() { + assert_eq!( + error.code().0 as u32, + 0x80004002, + "unexpected video probe error: {error}" + ); + eprintln!("SKIP GPU conversion: D3D11 video interface unavailable ({error})"); + return; + } + let format = VideoFormat { + codec: crate::VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: true, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + for (allocation_width, aperture) in [ + ( + 1920, + crate::aperture::VideoAperture::new(1920, 1080, None).unwrap(), + ), + ( + 1936, + crate::aperture::VideoAperture::new(1936, 1088, Some((8, 4, 1920, 1080))).unwrap(), + ), + ] { + let mut resources = unsafe { + AdoptedResources::new( + AdoptedD3d11Context { + device: device.as_raw(), + immediate_context: context.as_raw(), + }, + format, + ) + } + .unwrap(); + let mut description = frame_slot_description(allocation_width, 1088, DXGI_FORMAT_NV12); + description.BindFlags = + ::windows::Win32::Graphics::Direct3D11::D3D11_BIND_DECODER.0 as u32; + description.ArraySize = 2; + let mut texture = None; + let sample = unsafe { + device + .CreateTexture2D(&description, None, Some(&mut texture)) + .unwrap(); + let buffer = + MFCreateDXGISurfaceBuffer(&ID3D11Texture2D::IID, &texture.unwrap(), 1, false) + .unwrap(); + let sample = MFCreateSample().unwrap(); + sample.AddBuffer(&buffer).unwrap(); + sample.SetSampleTime(1_234_567).unwrap(); + sample.SetSampleDuration(166_667).unwrap(); + sample + }; + let frame = + DecodedVideoFrame::from_sample(sample, format, aperture, format.pixel_format) + .unwrap(); + let first = resources.record(0, &frame).unwrap(); + let processor = resources.processor.as_ref().unwrap().processor.clone(); + for _ in 0..3 { + resources.reconfigure(format); + let recorded = resources.record(0, &frame).unwrap(); + assert_eq!(resources.format, format); + assert_eq!((recorded.width, recorded.height), (1920, 1080)); + assert_eq!(recorded.texture, first.texture); + assert_eq!(recorded.presentation_time_ns, 123_456_700); + assert_eq!(frame.subresource, 1); + assert_eq!(frame.duration_100ns, 166_667); + let active = resources.processor.as_ref().unwrap(); + assert_eq!(active.processor.as_raw(), processor.as_raw()); + assert_eq!( + (active.input_width, active.input_height), + (allocation_width, 1088) + ); + assert_eq!(active.input_views.len(), 1); + let mut enabled = Default::default(); + let mut source = RECT::default(); + unsafe { + resources.video_context.VideoProcessorGetStreamSourceRect( + &processor, + 0, + &mut enabled, + &mut source, + ); + } + assert!(enabled.as_bool()); + assert_eq!( + (source.left, source.top, source.right, source.bottom), + ( + aperture.x as i32, + aperture.y as i32, + (aperture.x + 1920) as i32, + (aperture.y + 1080) as i32 + ) + ); + } + resources.reset_decoder_views(); + assert!(resources.processor.as_ref().unwrap().input_views.is_empty()); + assert_eq!(resources.record(0, &frame).unwrap().texture, first.texture); + } + } + #[test] + fn decoder_worker_retries_are_bounded_until_old_workers_exit() { + let mut leases: Vec<_> = (0..4) + .map(|_| DecoderWorkerLease::acquire().unwrap()) + .collect(); + assert!(DecoderWorkerLease::acquire().is_err()); + leases.pop(); + let replacement = DecoderWorkerLease::acquire().unwrap(); + assert!(DecoderWorkerLease::acquire().is_err()); + drop(replacement); + drop(leases); + assert!(DecoderWorkerLease::acquire().is_ok()); + } + #[test] + fn decoder_startup_is_nonblocking_and_preserves_failures() { + let (sender, receiver) = sync_channel(1); + let mut startup = Some(receiver); + assert!(!poll_decoder_startup(&mut startup).unwrap()); + sender.send(Ok(())).unwrap(); + assert!(poll_decoder_startup(&mut startup).unwrap()); + assert!(poll_decoder_startup(&mut startup).unwrap()); + + let (sender, receiver) = sync_channel(1); + let mut startup = Some(receiver); + sender.send(Err("driver refused codec".into())).unwrap(); + assert!( + poll_decoder_startup(&mut startup) + .unwrap_err() + .to_string() + .contains("driver refused codec") + ); + let (sender, receiver) = sync_channel(1); + drop(sender); + assert!(poll_decoder_startup(&mut Some(receiver)).is_err()); + } + use ::windows::Win32::Foundation::HMODULE; + use ::windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1, + }; + use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_SDK_VERSION, D3D11CreateDevice, + }; + use ::windows::Win32::Graphics::Dxgi::IDXGIAdapter; + use ::windows::Win32::Media::MediaFoundation::{ + IMFSample, MFCreateDXGISurfaceBuffer, MFCreateSample, + }; + + // Test-only observation of the concrete MF sample's COM ownership. Keeping + // a texture alive alone must not satisfy the sample-lease regression check. + fn sample_ref_count(sample: &IMFSample) -> u32 { + let identity = sample.cast::<::windows::core::IUnknown>().unwrap(); + unsafe { + (identity.vtable().AddRef)(identity.as_raw()); + (identity.vtable().Release)(identity.as_raw()) - 1 + } + } + + fn encoded_frame(sequence: i64, key_frame: bool) -> EncodedVideoFrame { + EncodedVideoFrame { + codec: crate::VideoCodec::H264, + data: vec![0, 0, 0, 1, if key_frame { 0x65 } else { 0x41 }], + timestamp_100ns: sequence * 166_667, + duration_100ns: 166_667, + key_frame, + reset_decoder: key_frame, + } + } + + #[test] + fn decoder_reset_keeps_keyframe_and_queued_descendants_in_order() { + let queue = BoundedQueue::new(2); + let mut pending = Some(encoded_frame(10, true)); + queue.push(encoded_frame(11, false)).unwrap(); + queue.push(encoded_frame(12, false)).unwrap(); + for sequence in 10..=12 { + let frame = next_encoded_frame(&mut pending, &queue).unwrap(); + assert_eq!(frame.timestamp_100ns, sequence * 166_667); + assert_eq!(frame.key_frame, sequence == 10); + } + assert!(next_encoded_frame(&mut pending, &queue).is_none()); + } + + #[test] + fn adopted_context_enables_d3d11_multithread_protection() { + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 test device"); + } + let context = context.expect("D3D11 immediate context"); + let multithread: ID3D10Multithread = context.cast().expect("multithread interface"); + unsafe { + let _ = multithread.SetMultithreadProtected(false); + } + assert!(!unsafe { multithread.GetMultithreadProtected().as_bool() }); + + enable_multithread_protection(&context).expect("enable multithread protection"); + + assert!(unsafe { multithread.GetMultithreadProtected().as_bool() }); + } + + #[test] + fn full_decode_queue_retains_an_incoming_recovery_keyframe() { + let encoded = Arc::new(BoundedQueue::new(2)); + let events = Arc::new(BoundedQueue::new(8)); + let submitter = D3d11FrameSubmitter { + encoded: Arc::clone(&encoded), + events: Arc::clone(&events), + }; + assert_eq!( + submitter.submit_video(encoded_frame(0, false)), + Ok(PushOutcome::Queued) + ); + assert_eq!( + submitter.submit_video(encoded_frame(1, false)), + Ok(PushOutcome::Queued) + ); + + assert_eq!( + submitter.submit_video(encoded_frame(2, true)), + Ok(PushOutcome::DroppedOldest) + ); + assert!(encoded.try_pop().is_some_and(|frame| frame.key_frame)); + assert!(encoded.try_pop().is_none()); + assert_eq!( + events.try_pop(), + Some(BackendEvent::QueueOverflow(Subsystem::VideoDecode)) + ); + assert!(events.try_pop().is_none()); + } + + #[test] + #[ignore = "requires a hardware HEVC Main10 MFT and P010/PQ-to-RGB10A2/PQ conversion"] + fn hevc_hdr_hardware_decode_and_conversion_preserve_pq_precision() { + let _runtime = EmbeddedMediaRuntime::initialize().expect("Media Foundation"); + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 hardware device"); + } + let device = device.unwrap(); + let context = context.unwrap(); + let format = VideoFormat { + transfer_function: VideoTransferFunction::Pq, + color_primaries: crate::VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + ..color_test_format() + }; + let reference = include_bytes!("../../fixtures/probe/hevc-p010-pq-precision-luma.bin") + .chunks_exact(2) + .map(|bytes| u16::from_le_bytes([bytes[0], bytes[1]])) + .collect::>(); + assert_eq!(reference.len(), 877); + let mut resources = unsafe { + AdoptedResources::new( + AdoptedD3d11Context { + device: device.as_raw(), + immediate_context: context.as_raw(), + }, + format, + ) + } + .unwrap(); + for restart in 0..3 { + resources.reset_decoder_views(); + let mut decoder = + Decoder::new(&resources, format, WindowsDecoderMode::Hardware).unwrap(); + let frame = decoder + .probe_frame(include_bytes!( + "../../fixtures/probe/hevc-p010-pq-precision.hevc" + )) + .expect("decode valid Main10 PQ precision access unit"); + assert_eq!(frame.format.pixel_format, VideoPixelFormat::P010); + assert_eq!(frame.format.transfer_function, VideoTransferFunction::Pq); + let read_row = |texture: &ID3D11Texture2D, subresource: u32| { + let mut description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + texture.GetDesc(&mut description); + } + description.Usage = D3D11_USAGE_STAGING; + description.BindFlags = 0; + description.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; + description.ArraySize = 1; + description.MipLevels = 1; + description.MiscFlags = 0; + let mut staging = None; + unsafe { + device + .CreateTexture2D(&description, None, Some(&mut staging)) + .unwrap(); + } + let staging = staging.unwrap(); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + context.CopySubresourceRegion(&staging, 0, 0, 0, 0, texture, subresource, None); + context + .Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) + .unwrap(); + let bytes = std::slice::from_raw_parts( + mapped.pData.cast::(), + 877 * if description.Format == DXGI_FORMAT_P010 { + 2 + } else { + 4 + }, + ) + .to_vec(); + context.Unmap(&staging, 0); + (description.Format, bytes) + } + }; + let (input_format, input) = read_row(&frame.texture, frame.subresource); + assert_eq!(input_format, DXGI_FORMAT_P010); + for (index, bytes) in input.chunks_exact(2).enumerate() { + let actual = u16::from_le_bytes([bytes[0], bytes[1]]); + assert_eq!(actual & 63, 0, "P010 sample alignment at {index}"); + assert_eq!( + actual >> 6, + reference[index], + "Main10 decoded luma at {index}" + ); + } + let recorded = resources + .record(0, &frame) + .expect("convert actual P010/PQ decoder surface"); + assert_eq!(recorded.texture_format, D3d11TextureFormat::Rgb10A2); + assert_eq!(recorded.color_space, D3d11ColorSpace::Pq2020); + let output = unsafe { clone_interface::(recorded.texture) }.unwrap(); + let (output_format, bytes) = read_row(&output, 0); + assert_eq!(output_format, DXGI_FORMAT_R10G10B10A2_UNORM); + let mut levels = std::collections::HashSet::new(); + for (index, bytes) in bytes.chunks_exact(4).enumerate() { + let pixel = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let expected = ((f64::from(reference[index]) - 64.0) * 1023.0 / 876.0) + .round() + .clamp(0.0, 1023.0) as i32; + levels.insert(pixel & 1023); + for shift in [0, 10, 20] { + let actual = ((pixel >> shift) & 1023) as i32; + assert!( + (actual - expected).abs() <= 2, + "restart={restart} PQ sample={index} channel={shift} actual={actual} expected={expected}" + ); + } + } + assert!( + levels.len() >= 800, + "retained only {} PQ gray levels", + levels.len() + ); + eprintln!( + "HEVC Main10 PQ restart={restart}: P010 source matches software reference; RGB10A2/PQ retains {} gray levels", + levels.len() + ); + drop(frame); + decoder.stop(); + } + } + + #[test] + #[ignore = "requires a hardware HEVC Main44410 MFT accepting P010 output and RGB10A2 conversion"] + fn hevc_444_request_accepts_p010_output_without_decoder_restart() { + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 hardware device"); + } + let device = device.unwrap(); + let context = context.unwrap(); + let adopted = AdoptedD3d11Context { + device: device.as_raw(), + immediate_context: context.as_raw(), + }; + let requested = VideoFormat { + pixel_format: VideoPixelFormat::Y410, + chroma_format: VideoChromaFormat::Cs444, + ..color_test_format() + }; + let (ready_sender, ready_receiver) = sync_channel(1); + let (producer, submitter) = unsafe { + D3d11FrameProducer::new( + adopted, + requested, + WindowsDecoderMode::Hardware, + Arc::new(move || { + let _ = ready_sender.try_send(()); + }), + ) + } + .expect("adopt D3D11 device with a requested Y410 decoder"); + submitter + .submit_video(EncodedVideoFrame { + codec: requested.codec, + data: include_bytes!("../../fixtures/probe/hevc-p010-sdr.hevc").to_vec(), + timestamp_100ns: 0, + duration_100ns: requested.frame_duration_100ns(), + key_frame: true, + reset_decoder: false, + }) + .expect("submit HEVC P010 SDR access unit"); + let deadline = Instant::now() + Duration::from_secs(5); + let frame = loop { + let frame = producer + .acquire_latest() + .expect("acquire validated decoder output"); + while let Some(event) = producer.try_event() { + assert!( + matches!(event, BackendEvent::VideoFormatChanged(_)), + "unexpected decoder recovery event: {event:?}" + ); + } + if let Some(frame) = frame { + break frame; + } + ready_receiver + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("P010 output must arrive without a replacement keyframe"); + }; + let actual = frame.format(); + assert_eq!(actual.pixel_format, VideoPixelFormat::P010); + assert_eq!(actual.chroma_format, VideoChromaFormat::Cs420); + assert_eq!(actual.pixel_format.bit_depth(), 10); + assert_eq!(actual.transfer_function, VideoTransferFunction::Sdr); + assert_eq!(actual.color_primaries, requested.color_primaries); + assert_eq!(actual.color_matrix, requested.color_matrix); + assert_eq!( + producer.state.lock().unwrap().presented_decoder_generation, + 1, + "chroma fallback must not restart the decoder" + ); + let recorded = unsafe { frame.record(adopted, 0) } + .expect("convert validated P010 output into the adopted RGB10A2 frame slot"); + assert_eq!(recorded.texture_format, D3d11TextureFormat::Rgb10A2); + assert_eq!(recorded.color_space, D3d11ColorSpace::Sdr709); + let output = unsafe { clone_interface::(recorded.texture) }.unwrap(); + let mut description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + output.GetDesc(&mut description); + } + assert_eq!(description.Format, DXGI_FORMAT_R10G10B10A2_UNORM); + } + + #[test] + #[ignore = "requires a hardware HEVC Main44410 MFT and Y410 UINT shader conversion"] + fn hevc_444_hardware_decode_and_conversion_preserve_precision_and_chroma() { + verify_hevc_444_hardware_precision(false); + } + + #[test] + #[ignore = "requires a hardware HEVC Main44410 MFT and Y410 PQ shader conversion"] + fn hevc_hdr_444_hardware_decode_and_conversion_preserve_precision_and_chroma() { + verify_hevc_444_hardware_precision(true); + } + + fn verify_hevc_444_hardware_precision(hdr: bool) { + let _runtime = EmbeddedMediaRuntime::initialize().expect("Media Foundation"); + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 hardware device"); + } + let device = device.unwrap(); + let context = context.unwrap(); + let format = VideoFormat { + codec: crate::VideoCodec::H265, + width: 1920, + height: 1080, + pixel_format: VideoPixelFormat::Y410, + chroma_format: crate::VideoChromaFormat::Cs444, + transfer_function: if hdr { + VideoTransferFunction::Pq + } else { + VideoTransferFunction::Sdr + }, + color_primaries: if hdr { + crate::VideoColorPrimaries::Bt2020 + } else { + crate::VideoColorPrimaries::Bt709 + }, + color_matrix: if hdr { + VideoColorMatrix::Bt2020 + } else { + VideoColorMatrix::Bt709 + }, + ..color_test_format() + }; + let mut resources = unsafe { + AdoptedResources::new( + AdoptedD3d11Context { + device: device.as_raw(), + immediate_context: context.as_raw(), + }, + format, + ) + } + .expect("adopt D3D11 hardware device"); + for restart in 0..3 { + resources.reset_decoder_views(); + let mut decoder = Decoder::new(&resources, format, WindowsDecoderMode::Hardware) + .expect("configure hardware HEVC Main44410 decoder"); + let frame = decoder + .probe_frame(if hdr { + include_bytes!("../../fixtures/probe/hevc-y410-pq-precision.hevc") + } else { + include_bytes!("../../fixtures/probe/hevc-y410-precision.hevc") + }) + .expect("decode lossless Main44410 precision access unit"); + assert_eq!(frame.format.pixel_format, VideoPixelFormat::Y410); + assert_eq!(frame.format.chroma_format, crate::VideoChromaFormat::Cs444); + assert_eq!(frame.format.pixel_format.bit_depth(), 10); + assert_eq!(frame.format.transfer_function, format.transfer_function); + assert_eq!(frame.format.color_primaries, format.color_primaries); + assert_eq!(frame.format.color_matrix, format.color_matrix); + let mut input_description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + frame.texture.GetDesc(&mut input_description); + } + assert_eq!(input_description.Format, DXGI_FORMAT_Y410); + let read_rows = |texture: &ID3D11Texture2D, subresource: u32| { + let mut description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + texture.GetDesc(&mut description); + } + description.Usage = D3D11_USAGE_STAGING; + description.BindFlags = 0; + description.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; + description.ArraySize = 1; + description.MipLevels = 1; + description.MiscFlags = 0; + let mut staging = None; + unsafe { + device + .CreateTexture2D(&description, None, Some(&mut staging)) + .unwrap(); + } + let staging = staging.unwrap(); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + context.CopySubresourceRegion(&staging, 0, 0, 0, 0, texture, subresource, None); + context + .Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) + .unwrap(); + let gray = std::slice::from_raw_parts(mapped.pData.cast::(), 877).to_vec(); + let chroma = std::slice::from_raw_parts( + mapped + .pData + .cast::() + .add(mapped.RowPitch as usize * 64) + .cast::(), + 1920, + ) + .to_vec(); + context.Unmap(&staging, 0); + (gray, chroma) + } + }; + let decoded_pixels = read_rows(&frame.texture, frame.subresource); + let recorded = resources + .record(0, &frame) + .expect("convert actual Y410 decoder surface"); + assert_eq!(recorded.texture_format, D3d11TextureFormat::Rgb10A2); + assert_eq!( + recorded.color_space, + if hdr { + D3d11ColorSpace::Pq2020 + } else { + D3d11ColorSpace::Sdr709 + } + ); + let output = unsafe { clone_interface::(recorded.texture) }.unwrap(); + let mut output_description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + output.GetDesc(&mut output_description); + } + assert_eq!(output_description.Format, DXGI_FORMAT_R10G10B10A2_UNORM); + let pixels = read_rows(&output, 0); + for (index, pixel) in decoded_pixels.0.iter().copied().enumerate() { + assert_eq!( + (pixel >> 10) & 1023, + index as u32 + 64, + "decoder luma precision at {index}" + ); + assert_eq!(pixel & 1023, 512, "decoder neutral U at {index}"); + assert_eq!((pixel >> 20) & 1023, 512, "decoder neutral V at {index}"); + } + for (index, pixel) in decoded_pixels.1.iter().copied().enumerate() { + assert_eq!( + (pixel >> 10) & 1023, + 512, + "decoder chroma-row luma at {index}" + ); + assert_eq!( + pixel & 1023, + if index % 2 == 0 { 384 } else { 640 }, + "decoder U at {index}" + ); + assert_eq!( + (pixel >> 20) & 1023, + if index % 2 == 0 { 640 } else { 384 }, + "decoder V at {index}" + ); + } + let mut levels = std::collections::HashSet::new(); + for (index, pixel) in pixels.0.iter().copied().enumerate() { + let expected = (index as f64 * 1023.0 / 876.0).round() as i32; + levels.insert(pixel & 1023); + for shift in [0, 10, 20] { + let actual = ((pixel >> shift) & 1023) as i32; + assert!( + (actual - expected).abs() <= 2, + "restart={restart} grayscale sample={index} channel={shift} actual={actual} expected={expected}" + ); + } + } + assert!( + levels.len() >= 800, + "restart={restart} retained only {} grayscale levels", + levels.len() + ); + for (index, pixel) in pixels.1.iter().copied().enumerate() { + let y = (512.0 - 64.0) / 876.0; + let u = if index % 2 == 0 { + -128.0 / 896.0 + } else { + 128.0 / 896.0 + }; + let v = -u; + let expected = if hdr { + [ + y + 1.4746 * v, + y - 0.1645531268 * u - 0.5713531268 * v, + y + 1.8814 * u, + ] + } else { + [ + y + 1.5748 * v, + y - 0.1873242729 * u - 0.4681242729 * v, + y + 1.8556 * u, + ] + }; + for (channel, expected) in expected.into_iter().enumerate() { + let expected = (expected * 1023.0_f64).round() as i32; + let actual = ((pixel >> (channel * 10)) & 1023) as i32; + assert!( + (actual - expected).abs() <= 4, + "restart={restart} chroma sample={index} channel={channel} actual={actual} expected={expected}" + ); + } + } + eprintln!( + "HEVC Main44410 hardware restart={restart}: actual Y410 -> RGB10A2, {} distinct gray levels, 1920 alternating chroma pixels verified", + levels.len() + ); + drop(frame); + decoder.stop(); + } + } + + #[test] + fn conversion_allocates_only_visited_qrhi_slots_and_reuses_them() { + let _runtime = EmbeddedMediaRuntime::initialize().expect("Media Foundation"); + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .expect("D3D11 test device"); + } + let device = device.unwrap(); + let context = context.unwrap(); + // Headless Windows CI may expose D3D11 without its video interfaces. + // Only that capability absence is a skip; conversion errors on capable + // hardware must continue to fail this test. + if let Err(error) = device.cast::() { + assert_eq!( + error.code().0 as u32, + 0x80004002, + "unexpected video probe error: {error}" + ); + eprintln!("SKIP GPU conversion: D3D11 video interface unavailable ({error})"); + return; + } + let format = VideoFormat { + codec: crate::VideoCodec::H264, + width: 64, + height: 64, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format: VideoPixelFormat::Nv12, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + let mut resources = unsafe { + AdoptedResources::new( + AdoptedD3d11Context { + device: device.as_raw(), + immediate_context: context.as_raw(), + }, + format, + ) + } + .expect("adopt device"); + resources + .ensure_processor(64, 64, DXGI_FORMAT_NV12, 64, 64) + .unwrap(); + assert!( + resources + .processor + .as_ref() + .unwrap() + .slots + .iter() + .all(Option::is_none) + ); + let mut description = frame_slot_description(64, 64, DXGI_FORMAT_NV12); + description.BindFlags = ::windows::Win32::Graphics::Direct3D11::D3D11_BIND_DECODER.0 as u32; + let mut texture = None; + unsafe { + device + .CreateTexture2D(&description, None, Some(&mut texture)) + .unwrap(); + } + let texture = texture.unwrap(); + let sample = unsafe { + let buffer = + MFCreateDXGISurfaceBuffer(&ID3D11Texture2D::IID, &texture, 0, false).unwrap(); + let sample = MFCreateSample().unwrap(); + sample.AddBuffer(&buffer).unwrap(); + sample + }; + let baseline_refs = sample_ref_count(&sample); + for preferred in [ + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] { + let decoded = DecodedVideoFrame::from_sample( + sample.clone(), + format, + crate::aperture::VideoAperture::new(format.width, format.height, None).unwrap(), + preferred, + ); + if preferred == VideoPixelFormat::Ayuv { + let frame = decoded.expect("eight-bit chroma fallback retains its actual format"); + assert_eq!(frame.format.pixel_format, VideoPixelFormat::Nv12); + assert_eq!(frame.format.chroma_format, VideoChromaFormat::Cs420); + assert_eq!(sample_ref_count(&sample), baseline_refs + 1); + drop(frame); + } else { + let error = decoded + .err() + .expect("startup fallback must not lose bit depth"); + assert!(error.contains("produced Nv12 output below negotiated")); + } + assert_eq!(sample_ref_count(&sample), baseline_refs); + } + let error = DecodedVideoFrame::from_sample( + sample.clone(), + VideoFormat { + pixel_format: VideoPixelFormat::P010, + ..format + }, + crate::aperture::VideoAperture::new(format.width, format.height, None).unwrap(), + VideoPixelFormat::P010, + ) + .err() + .expect("an eight-bit surface cannot claim ten-bit output"); + assert!(error.contains("does not match output media format P010")); + assert_eq!(sample_ref_count(&sample), baseline_refs); + let frame = DecodedVideoFrame::from_sample( + sample.clone(), + format, + crate::aperture::VideoAperture::new(format.width, format.height, None).unwrap(), + format.pixel_format, + ) + .unwrap(); + assert_eq!( + sample_ref_count(&sample), + baseline_refs + 1, + "decoded frame must lease the sample, not only its texture" + ); + let mut ready = VecDeque::from([frame]); + assert_eq!(sample_ref_count(&sample), baseline_refs + 1); + let frame = ready.pop_back().unwrap(); + let first = resources.record(0, &frame).unwrap(); + assert_eq!(first.texture, resources.record(0, &frame).unwrap().texture); + assert_ne!(first.texture, resources.record(3, &frame).unwrap().texture); + assert_eq!( + sample_ref_count(&sample), + baseline_refs + 1, + "recording must not release the sample before the frame token" + ); + assert_eq!( + resources + .processor + .as_ref() + .unwrap() + .slots + .iter() + .filter(|slot| slot.is_some()) + .count(), + 2 + ); + assert!(resources.record(8, &frame).is_err()); + drop(frame); + assert_eq!( + sample_ref_count(&sample), + baseline_refs, + "frame release must return its lease without leaking samples" + ); + for changed in [ + VideoFormat { + chroma_siting: VideoChromaSiting::TopLeft, + ..format + }, + VideoFormat { + full_range: true, + ..format + }, + VideoFormat { + transfer_function: VideoTransferFunction::Pq, + ..format + }, + VideoFormat { + color_primaries: crate::VideoColorPrimaries::Bt2020, + ..format + }, + VideoFormat { + color_matrix: VideoColorMatrix::Bt601, + ..format + }, + ] { + resources.reconfigure(changed); + assert!( + resources.processor.is_none(), + "color changes must discard converters and texture slots" + ); + resources.reconfigure(format); + resources + .ensure_processor(64, 64, DXGI_FORMAT_NV12, 64, 64) + .unwrap(); + } + resources.reconfigure(VideoFormat { + width: 128, + ..format + }); + assert!(resources.processor.is_none()); + resources + .ensure_processor(128, 64, DXGI_FORMAT_NV12, 128, 64) + .unwrap(); + assert!( + resources + .processor + .as_ref() + .unwrap() + .slots + .iter() + .all(Option::is_none) + ); + let enumerator = resources.processor.as_ref().unwrap().enumerator.clone(); + let enumerator_1: ID3D11VideoProcessorEnumerator1 = enumerator.cast().unwrap(); + let mut previous_texture = None; + for (input_format, pixel_format, chroma_format) in [ + ( + DXGI_FORMAT_NV12, + VideoPixelFormat::Nv12, + VideoChromaFormat::Cs420, + ), + ( + DXGI_FORMAT_P010, + VideoPixelFormat::P010, + VideoChromaFormat::Cs420, + ), + ( + DXGI_FORMAT_Y410, + VideoPixelFormat::Y410, + VideoChromaFormat::Cs444, + ), + ( + DXGI_FORMAT_AYUV, + VideoPixelFormat::Ayuv, + VideoChromaFormat::Cs444, + ), + ( + DXGI_FORMAT_NV12, + VideoPixelFormat::Nv12, + VideoChromaFormat::Cs420, + ), + ] { + for full_range in [true, false] { + let format = VideoFormat { + width: 128, + pixel_format, + chroma_format, + full_range, + ..format + }; + resources.reconfigure(format); + assert!(resources.processor.is_none()); + let output_format = output_dxgi_format(format); + let supported = unsafe { + enumerator.CheckVideoProcessorFormat(input_format).unwrap() + & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT.0 as u32 + != 0 + && enumerator.CheckVideoProcessorFormat(output_format).unwrap() + & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT.0 as u32 + != 0 + && enumerator_1 + .CheckVideoProcessorFormatConversion( + input_format, + input_color_space(format).unwrap(), + output_format, + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, + ) + .unwrap() + .as_bool() + }; + let result = resources.ensure_processor(128, 64, input_format, 128, 64); + if !supported { + assert!( + result.is_err(), + "unsupported conversion must not fall back to RGBA8" + ); + assert!(resources.processor.is_none()); + continue; + } + result.unwrap(); + let processor = resources.processor.as_mut().unwrap(); + assert_eq!(processor.output_format, output_format); + assert!(processor.input_views.is_empty()); + assert!(processor.slots.iter().all(Option::is_none)); + unsafe { + assert_eq!( + resources + .video_context_1 + .VideoProcessorGetStreamColorSpace1(&processor.processor, 0,), + input_color_space(format).unwrap() + ); + assert_eq!( + resources + .video_context_1 + .VideoProcessorGetOutputColorSpace1(&processor.processor,), + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709 + ); + } + let slot = FrameSlot::new( + &device, + &resources.video_device, + &processor.enumerator, + 128, + 64, + processor.output_format, + ) + .unwrap(); + let mut description = D3D11_TEXTURE2D_DESC::default(); + unsafe { + slot.texture.GetDesc(&mut description); + } + assert_eq!(description.Format, output_format); + if let Some(previous) = previous_texture.as_ref() { + assert_ne!(&slot.texture, previous); + } + previous_texture = Some(slot.texture.clone()); + processor.slots[0] = Some(slot); + let identity = processor.processor.clone(); + resources + .ensure_processor(128, 64, input_format, 128, 64) + .unwrap(); + assert_eq!(resources.processor.as_ref().unwrap().processor, identity); + assert!(resources.processor.as_ref().unwrap().slots[0].is_some()); + } + } + } + + #[test] + fn full_decode_queue_reports_loss_synchronously_without_a_stale_recovery_event() { + let encoded = Arc::new(BoundedQueue::new(2)); + let events = Arc::new(BoundedQueue::new(8)); + let submitter = D3d11FrameSubmitter { + encoded: Arc::clone(&encoded), + events: Arc::clone(&events), + }; + assert_eq!( + submitter.submit_video(encoded_frame(0, false)), + Ok(PushOutcome::Queued) + ); + assert_eq!( + submitter.submit_video(encoded_frame(1, false)), + Ok(PushOutcome::Queued) + ); + + assert_eq!( + submitter.submit_video(encoded_frame(2, false)), + Ok(PushOutcome::DroppedOldest) + ); + assert!(encoded.try_pop().is_none()); + assert_eq!( + events.try_pop(), + Some(BackendEvent::QueueOverflow(Subsystem::VideoDecode)) + ); + assert_eq!(events.try_pop(), None); + } + + #[test] + fn decoder_subresource_preserves_array_slice() { + assert_eq!(decoder_array_slice(0, 1, 8), Ok(0)); + assert_eq!(decoder_array_slice(5, 1, 8), Ok(5)); + assert_eq!(decoder_array_slice(7, 1, 8), Ok(7)); + assert_eq!(decoder_array_slice(6, 3, 4), Ok(2)); + assert!(decoder_array_slice(12, 3, 4).is_err()); + } + + #[test] + fn all_decoder_formats_map_to_video_processor_inputs() { + let cases = [ + ( + DXGI_FORMAT_NV12, + VideoPixelFormat::Nv12, + VideoChromaFormat::Cs420, + ), + ( + DXGI_FORMAT_P010, + VideoPixelFormat::P010, + VideoChromaFormat::Cs420, + ), + ( + DXGI_FORMAT_AYUV, + VideoPixelFormat::Ayuv, + VideoChromaFormat::Cs444, + ), + ( + DXGI_FORMAT_Y410, + VideoPixelFormat::Y410, + VideoChromaFormat::Cs444, + ), + ]; + for (dxgi, pixel, chroma) in cases { + let mapped = pixel_format_from_dxgi(dxgi).expect("supported decoder format"); + assert_eq!(mapped, pixel); + assert_eq!(chroma_format(mapped), chroma); + } + assert_eq!(DXGI_FORMAT_R8G8B8A8_UNORM.0, 28); + } + + #[test] + fn frame_slots_convert_decoder_surfaces_to_qt_sdr_composition_targets() { + let description = frame_slot_description(1920, 1080, DXGI_FORMAT_R8G8B8A8_UNORM); + assert_eq!(description.Width, 1920); + assert_eq!(description.Height, 1080); + assert_eq!(description.Format, DXGI_FORMAT_R8G8B8A8_UNORM); + assert_ne!( + description.BindFlags & D3D11_BIND_SHADER_RESOURCE.0 as u32, + 0 + ); + assert_ne!(description.BindFlags & D3D11_BIND_RENDER_TARGET.0 as u32, 0); + + let p010 = VideoFormat { + pixel_format: VideoPixelFormat::P010, + ..color_test_format() + }; + let y410 = VideoFormat { + pixel_format: VideoPixelFormat::Y410, + ..color_test_format() + }; + let ten_bit = frame_slot_description(2560, 1440, output_dxgi_format(p010)); + assert_eq!(ten_bit.Format, DXGI_FORMAT_R10G10B10A2_UNORM); + assert_eq!(output_dxgi_format(y410), ten_bit.Format); + assert_eq!( + d3d11_texture_format(ten_bit.Format), + D3d11TextureFormat::Rgb10A2 + ); + } + + fn color_test_format() -> VideoFormat { + VideoFormat { + codec: crate::VideoCodec::H265, + width: 1920, + height: 1080, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 20_000_000, + pixel_format: VideoPixelFormat::P010, + chroma_format: VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + } + } + + #[test] + fn packed_444_color_conversion_does_not_require_subsampled_chroma_siting() { + for pixel_format in [VideoPixelFormat::Ayuv, VideoPixelFormat::Y410] { + for full_range in [false, true] { + let format = VideoFormat { + pixel_format, + chroma_format: crate::VideoChromaFormat::Cs444, + full_range, + ..color_test_format() + }; + assert_eq!( + input_color_space(format).unwrap(), + input_color_space(VideoFormat { + chroma_siting: VideoChromaSiting::TopLeft, + ..format + }) + .unwrap(), + ); + } + } + assert!( + input_color_space(VideoFormat { + chroma_siting: VideoChromaSiting::TopLeft, + ..color_test_format() + }) + .is_err() + ); + } + + #[test] + fn hdr_capability_probe_requires_actual_p010_pq_before_gpu_checks() { + for format in [ + color_test_format(), + VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + ..color_test_format() + }, + VideoFormat { + transfer_function: VideoTransferFunction::Pq, + pixel_format: VideoPixelFormat::Nv12, + ..color_test_format() + }, + ] { + let result = unsafe { + probe_hdr_conversion( + AdoptedD3d11Context { + device: std::ptr::null_mut(), + immediate_context: std::ptr::null_mut(), + }, + format, + ) + }; + assert_eq!( + result.unwrap_err(), + "HDR10 capability requires actual P010/PQ decoder output" + ); + } + } + + #[test] + fn hdr_targets_preserve_pq_bt2020_in_rgb10a2() { + for transfer in [VideoTransferFunction::Pq, VideoTransferFunction::Hlg] { + let format = VideoFormat { + transfer_function: transfer, + color_primaries: crate::VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + ..color_test_format() + }; + assert_eq!(output_dxgi_format(format), DXGI_FORMAT_R10G10B10A2_UNORM); + assert_eq!( + output_color_space(transfer), + DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 + ); + assert_eq!(recorded_color_space(transfer), D3d11ColorSpace::Pq2020); + assert_eq!( + d3d11_texture_format(output_dxgi_format(format)), + D3d11TextureFormat::Rgb10A2 + ); + } + } + + #[test] + fn hdr_input_spaces_preserve_transfer_and_range() { + let pq = VideoFormat { + transfer_function: VideoTransferFunction::Pq, + color_primaries: crate::VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + ..color_test_format() + }; + assert_eq!( + input_color_space(pq).unwrap(), + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020 + ); + assert_eq!( + input_color_space(VideoFormat { + chroma_siting: VideoChromaSiting::TopLeft, + ..pq + }) + .unwrap(), + DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020 + ); + assert!( + input_color_space(VideoFormat { + full_range: true, + ..pq + }) + .is_err() + ); + let hlg = VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + chroma_siting: VideoChromaSiting::TopLeft, + ..pq + }; + assert!( + input_color_space(VideoFormat { + chroma_siting: VideoChromaSiting::Left, + ..hlg + }) + .is_err() + ); + assert_eq!( + input_color_space(hlg).unwrap(), + DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020 + ); + assert_eq!( + input_color_space(VideoFormat { + full_range: true, + ..hlg + }) + .unwrap(), + DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020 + ); + assert!( + input_color_space(VideoFormat { + pixel_format: VideoPixelFormat::Nv12, + ..pq + }) + .is_err() + ); + } + + #[test] + fn sdr_conversion_preserves_decoder_precision_and_expands_only_limited_range() { + for (pixel_format, chroma_format, output_format, texture_format) in [ + ( + VideoPixelFormat::Nv12, + VideoChromaFormat::Cs420, + DXGI_FORMAT_R8G8B8A8_UNORM, + D3d11TextureFormat::Rgba8, + ), + ( + VideoPixelFormat::Ayuv, + VideoChromaFormat::Cs444, + DXGI_FORMAT_R8G8B8A8_UNORM, + D3d11TextureFormat::Rgba8, + ), + ( + VideoPixelFormat::P010, + VideoChromaFormat::Cs420, + DXGI_FORMAT_R10G10B10A2_UNORM, + D3d11TextureFormat::Rgb10A2, + ), + ( + VideoPixelFormat::Y410, + VideoChromaFormat::Cs444, + DXGI_FORMAT_R10G10B10A2_UNORM, + D3d11TextureFormat::Rgb10A2, + ), + ] { + for full_range in [false, true] { + let format = VideoFormat { + codec: crate::VideoCodec::H265, + width: 64, + height: 64, + frame_rate_numerator: std::num::NonZeroU32::new(60).unwrap(), + frame_rate_denominator: std::num::NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format, + chroma_format, + full_range, + chroma_siting: VideoChromaSiting::Left, + transfer_function: VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: VideoColorMatrix::Bt709, + }; + assert_eq!(output_dxgi_format(format), output_format); + assert_eq!(d3d11_texture_format(output_format), texture_format); + assert_eq!( + recorded_color_space(format.transfer_function), + D3d11ColorSpace::Sdr709 + ); + assert_eq!( + output_color_space(format.transfer_function), + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709 + ); + assert_eq!( + input_color_space(format).unwrap(), + if full_range { + DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709 + } else { + DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709 + } + ); + } + } + } + + #[test] + fn embedded_module_has_no_window_swapchain_sdl_or_present_path() { + let source = include_str!("embedded.rs"); + for forbidden in [ + concat!("Create", "Window"), + concat!("Create", "SwapChain"), + concat!(".Pre", "sent("), + concat!("sdl", "2::"), + ] { + assert!( + !source.contains(forbidden), + "embedded producer contains forbidden presentation token {forbidden}" + ); + } + } + + #[test] + fn encoded_submitter_is_safe_to_move_to_the_transport_thread() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::(); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/graphics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/graphics.rs new file mode 100644 index 000000000..6f106f62a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/graphics.rs @@ -0,0 +1,1565 @@ +use super::color::input_color_space; +use std::collections::HashMap; +use std::ffi::c_void; +use std::mem::ManuallyDrop; +use std::mem::size_of; +use std::num::NonZeroU32; + +use ::windows::Win32::Foundation::{ + ERROR_CLASS_ALREADY_EXISTS, GetLastError, HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, LRESULT, + LUID, RECT, WAIT_FAILED, WAIT_OBJECT_0, WAIT_TIMEOUT, WPARAM, +}; +use ::windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_UNKNOWN, D3D_FEATURE_LEVEL_10_0, + D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1, +}; +use ::windows::Win32::Graphics::Direct3D10::ID3D10Multithread; +#[cfg(test)] +use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_BIND_DECODER, D3D11_BIND_SHADER_RESOURCE, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, +}; +use ::windows::Win32::Graphics::Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, + D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, + D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VIDEO_USAGE_OPTIMAL_SPEED, D3D11_VPIV_DIMENSION_TEXTURE2D, + D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, + ID3D11Texture2D, ID3D11VideoContext, ID3D11VideoContext1, ID3D11VideoDevice, + ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorEnumerator1, + ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, +}; +use ::windows::Win32::Graphics::Direct3D11on12::{D3D11On12CreateDevice, ID3D11On12Device}; +use ::windows::Win32::Graphics::Direct3D12::{ + D3D12_COMMAND_LIST_TYPE_DIRECT, D3D12_COMMAND_QUEUE_DESC, D3D12_COMMAND_QUEUE_FLAG_NONE, + D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, D3D12CreateDevice, ID3D12CommandQueue, ID3D12Device, +}; +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_ALPHA_MODE_IGNORE, DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, DXGI_FORMAT, DXGI_FORMAT_AYUV, + DXGI_FORMAT_NV12, DXGI_FORMAT_P010, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_R10G10B10A2_UNORM, + DXGI_FORMAT_Y410, DXGI_RATIONAL, DXGI_SAMPLE_DESC, +}; +use ::windows::Win32::Graphics::Dxgi::{ + CreateDXGIFactory1, DXGI_FEATURE_PRESENT_ALLOW_TEARING, DXGI_MWA_NO_ALT_ENTER, DXGI_PRESENT, + DXGI_PRESENT_ALLOW_TEARING, DXGI_SCALING_STRETCH, DXGI_SWAP_CHAIN_DESC1, DXGI_SWAP_CHAIN_FLAG, + DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING, DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT, + DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL, DXGI_USAGE_RENDER_TARGET_OUTPUT, IDXGIAdapter, IDXGIDevice, + IDXGIFactory2, IDXGIFactory4, IDXGIFactory5, IDXGISwapChain1, IDXGISwapChain2, IDXGISwapChain3, +}; +use ::windows::Win32::Media::MediaFoundation::{IMFDXGIDeviceManager, MFCreateDXGIDeviceManager}; +use ::windows::Win32::System::LibraryLoader::GetModuleHandleW; +use ::windows::Win32::System::Threading::WaitForSingleObject; +#[cfg(test)] +use ::windows::Win32::UI::WindowsAndMessaging::WS_POPUP; +use ::windows::Win32::UI::WindowsAndMessaging::{ + CS_HREDRAW, CS_VREDRAW, CreateWindowExW, DefWindowProcW, DestroyWindow, DispatchMessageW, + GetClientRect, HTTRANSPARENT, IsWindow, MA_NOACTIVATE, MSG, PM_REMOVE, PeekMessageW, + RegisterClassW, SWP_HIDEWINDOW, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW, SetWindowPos, + TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_MOUSEACTIVATE, WM_NCHITTEST, WNDCLASSW, + WS_CHILD, WS_CLIPCHILDREN, WS_CLIPSIBLINGS, WS_EX_APPWINDOW, WS_EX_NOACTIVATE, + WS_EX_TRANSPARENT, WS_OVERLAPPEDWINDOW, +}; +use ::windows::core::{BOOL, IUnknown, Interface, w}; + +use crate::{ + Bounds, ExistingWindow, OwnedWindow, SurfaceTarget, VideoChromaFormat, VideoCodec, VideoFormat, + VideoPixelFormat, WindowHandle, WindowsAdapterLuid, WindowsGraphicsApi, +}; + +use super::decoder::DecoderDevice; + +const WINDOW_CLASS: ::windows::core::PCWSTR = w!("OpenNOWStreamerD3D11Surface"); + +struct DeviceResources { + device: ID3D11Device, + context: ID3D11DeviceContext, + video_device: ID3D11VideoDevice, + video_context: ID3D11VideoContext, + video_context_1: Option, + manager: IMFDXGIDeviceManager, + _d3d12: Option, +} + +struct D3d12Owners { + _device: ID3D12Device, + _queue: ID3D12CommandQueue, + _on12: ID3D11On12Device, +} + +impl DeviceResources { + fn new( + api: WindowsGraphicsApi, + adapter_luid: Option, + ) -> Result { + let adapter = resolve_adapter(adapter_luid)?; + match api { + WindowsGraphicsApi::D3d11 => Self::new_d3d11(adapter.as_ref()), + WindowsGraphicsApi::D3d12 => Self::new_d3d12(adapter.as_ref()), + } + } + + fn finish_d3d11_device( + device: ID3D11Device, + context: ID3D11DeviceContext, + d3d12: Option, + ) -> Result { + unsafe { + let multithread: ID3D10Multithread = + context.cast().map_err(|error| error.to_string())?; + let _ = multithread.SetMultithreadProtected(true); + let video_device: ID3D11VideoDevice = + device.cast().map_err(|error| error.to_string())?; + let video_context: ID3D11VideoContext = + context.cast().map_err(|error| error.to_string())?; + let video_context_1 = video_context.cast().ok(); + let mut reset_token = 0; + let mut manager = None; + MFCreateDXGIDeviceManager(&mut reset_token, &mut manager) + .map_err(|error| error.to_string())?; + let manager = manager.ok_or("MFCreateDXGIDeviceManager returned no manager")?; + manager + .ResetDevice(&device, reset_token) + .map_err(|error| error.to_string())?; + Ok(Self { + device, + context, + video_device, + video_context, + video_context_1, + manager, + _d3d12: d3d12, + }) + } + } + + fn new_d3d11(adapter: Option<&IDXGIAdapter>) -> Result { + unsafe { + let mut device = None; + let mut context = None; + let levels = [ + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + ]; + D3D11CreateDevice( + adapter, + if adapter.is_some() { + D3D_DRIVER_TYPE_UNKNOWN + } else { + D3D_DRIVER_TYPE_HARDWARE + }, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + .map_err(|error| error.to_string())?; + Self::finish_d3d11_device( + device.ok_or("D3D11CreateDevice returned no device")?, + context.ok_or("D3D11CreateDevice returned no immediate context")?, + None, + ) + } + } + + fn new_d3d12(adapter: Option<&IDXGIAdapter>) -> Result { + unsafe { + let adapter = adapter + .map(|adapter| adapter.cast::()) + .transpose() + .map_err(|error| format!("selected D3D12 adapter interface: {error}"))?; + let mut d3d12_device = None; + D3D12CreateDevice(adapter.as_ref(), D3D_FEATURE_LEVEL_11_0, &mut d3d12_device) + .map_err(|error| format!("D3D12CreateDevice: {error}"))?; + let d3d12_device: ID3D12Device = + d3d12_device.ok_or("D3D12CreateDevice returned no device")?; + let queue: ID3D12CommandQueue = d3d12_device + .CreateCommandQueue(&D3D12_COMMAND_QUEUE_DESC { + Type: D3D12_COMMAND_LIST_TYPE_DIRECT, + Priority: D3D12_COMMAND_QUEUE_PRIORITY_NORMAL.0, + Flags: D3D12_COMMAND_QUEUE_FLAG_NONE, + NodeMask: 0, + }) + .map_err(|error| format!("ID3D12Device::CreateCommandQueue: {error}"))?; + let queue_unknown = queue.cast().map_err(|error| error.to_string())?; + let mut device = None; + let mut context = None; + D3D11On12CreateDevice( + &d3d12_device, + (D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT).0, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + Some(&[Some(queue_unknown)]), + 0, + Some(&mut device), + Some(&mut context), + None, + ) + .map_err(|error| format!("D3D11On12CreateDevice: {error}"))?; + let device = device.ok_or("D3D11On12CreateDevice returned no D3D11 device")?; + let context = context.ok_or("D3D11On12CreateDevice returned no immediate context")?; + let on12 = device + .cast() + .map_err(|error| format!("D3D11-on-12 device interface: {error}"))?; + Self::finish_d3d11_device( + device, + context, + Some(D3d12Owners { + _device: d3d12_device, + _queue: queue, + _on12: on12, + }), + ) + } + } +} + +fn resolve_adapter( + adapter_luid: Option, +) -> Result, String> { + resolve_adapter_with(adapter_luid, |luid, raw| unsafe { + let factory: IDXGIFactory4 = CreateDXGIFactory1().map_err(|error| { + format!("create DXGI factory for adapter LUID {raw:#018x}: {error}") + })?; + factory + .EnumAdapterByLuid(luid) + .map_err(|error| format!("selected adapter LUID {raw:#018x} is unavailable: {error}")) + }) +} + +fn resolve_adapter_with( + adapter_luid: Option, + resolve: impl FnOnce(LUID, u64) -> Result, +) -> Result, String> { + let Some(adapter_luid) = adapter_luid else { + return Ok(None); + }; + let raw = adapter_luid.get(); + let luid = LUID { + LowPart: raw as u32, + HighPart: (raw >> 32) as u32 as i32, + }; + resolve(luid, raw).map(Some) +} + +enum RenderWindow { + Existing(HWND), + Owned { + hwnd: HWND, + parent: Option, + visible: bool, + }, +} + +impl RenderWindow { + fn new(target: SurfaceTarget) -> Result { + match target { + SurfaceTarget::Existing(ExistingWindow { hwnd }) => { + let hwnd = hwnd_from_handle(hwnd); + if unsafe { !IsWindow(Some(hwnd)).as_bool() } { + return Err("supplied HWND is not a live Win32 window".to_owned()); + } + Ok(Self::Existing(hwnd)) + } + SurfaceTarget::Owned(window) => create_owned_window(window).map(|hwnd| Self::Owned { + hwnd, + parent: window.parent, + visible: window.visible, + }), + } + } + + fn hwnd(&self) -> HWND { + match self { + Self::Existing(hwnd) | Self::Owned { hwnd, .. } => *hwnd, + } + } + + fn can_update(&self, target: SurfaceTarget) -> bool { + match (self, target) { + (Self::Existing(current), SurfaceTarget::Existing(target)) => { + current.0 == hwnd_from_handle(target.hwnd).0 + } + ( + Self::Owned { + parent: current, .. + }, + SurfaceTarget::Owned(target), + ) => OwnedWindow { + parent: *current, + bounds: target.bounds, + visible: target.visible, + } + .has_same_parent(target), + _ => false, + } + } + + fn update(&mut self, target: SurfaceTarget) -> Result<(), String> { + match (self, target) { + (Self::Existing(hwnd), SurfaceTarget::Existing(_)) => { + if unsafe { IsWindow(Some(*hwnd)).as_bool() } { + Ok(()) + } else { + Err("supplied HWND is no longer a live Win32 window".to_owned()) + } + } + (Self::Owned { hwnd, visible, .. }, SurfaceTarget::Owned(window)) => { + position_owned_window(*hwnd, window)?; + *visible = window.visible; + Ok(()) + } + _ => Err("surface ownership changed during in-place update".to_owned()), + } + } + + fn client_size(&self) -> Result<(u32, u32), String> { + let mut rect = RECT::default(); + unsafe { + GetClientRect(self.hwnd(), &mut rect).map_err(|error| error.to_string())?; + } + Ok(( + (rect.right - rect.left).max(1) as u32, + (rect.bottom - rect.top).max(1) as u32, + )) + } + + fn is_visible(&self) -> bool { + match self { + Self::Existing(_) => true, + Self::Owned { visible, .. } => *visible, + } + } +} + +impl Drop for RenderWindow { + fn drop(&mut self) { + if let Self::Owned { hwnd, .. } = self { + unsafe { + let _ = DestroyWindow(*hwnd); + } + } + } +} + +struct ProcessorResources { + input_width: u32, + input_height: u32, + output_width: u32, + output_height: u32, + enumerator: ID3D11VideoProcessorEnumerator, + processor: ID3D11VideoProcessor, + input_views: HashMap<(usize, u32), ID3D11VideoProcessorInputView>, + output_view: ID3D11VideoProcessorOutputView, +} + +struct SwapChainResources { + swap_chain: IDXGISwapChain1, + frame_latency_waitable: HANDLE, + flags: DXGI_SWAP_CHAIN_FLAG, + allow_tearing: bool, +} + +struct FormatProbeCache { + adapter_luid: (u32, i32), + device: ID3D11Device, + results: Vec<(VideoCodec, VideoPixelFormat, bool, Result<(), String>)>, +} + +static FORMAT_PROBE_CACHE: std::sync::Mutex<[Option; 2]> = + std::sync::Mutex::new([None, None]); + +pub(super) struct Graphics { + processor: Option, + swap_chain: IDXGISwapChain1, + frame_latency_waitable: HANDLE, + swap_chain_flags: DXGI_SWAP_CHAIN_FLAG, + allow_tearing: bool, + has_presented: bool, + window: RenderWindow, + resources: DeviceResources, + video_format: VideoFormat, + swap_size: (u32, u32), + swap_format: DXGI_FORMAT, + ten_bit_output_supported: bool, +} + +impl Graphics { + pub(super) fn probe_hdr(&self, codec: VideoCodec) -> Result<(), String> { + self.probe_format(codec, VideoPixelFormat::P010, true) + } + + pub(super) fn probe_format( + &self, + codec: VideoCodec, + pixel_format: VideoPixelFormat, + hdr: bool, + ) -> Result<(), String> { + let luid = self.adapter_luid()?; + let adapter_luid = (luid.LowPart, luid.HighPart); + let mut cache = FORMAT_PROBE_CACHE + .lock() + .unwrap_or_else(|error| error.into_inner()); + let entry = &mut cache[usize::from(self.resources._d3d12.is_some())]; + if entry.as_ref().is_some_and(|entry| { + entry.adapter_luid != adapter_luid + || unsafe { entry.device.GetDeviceRemovedReason() }.is_err() + }) { + *entry = None; + } + let entry = entry.get_or_insert_with(|| FormatProbeCache { + adapter_luid, + device: self.resources.device.clone(), + results: Vec::with_capacity(6), + }); + if let Some((_, _, _, result)) = + entry + .results + .iter() + .find(|(cached_codec, cached_pixel, cached_hdr, _)| { + (*cached_codec, *cached_pixel, *cached_hdr) == (codec, pixel_format, hdr) + }) + { + return result.clone(); + } + let result = self.probe_format_uncached(codec, pixel_format, hdr); + if entry.results.len() < 6 + && !result + .as_ref() + .is_err_and(|error| error.contains("timed out")) + { + entry + .results + .push((codec, pixel_format, hdr, result.clone())); + } + result + } + + fn probe_format_uncached( + &self, + codec: VideoCodec, + pixel_format: VideoPixelFormat, + hdr: bool, + ) -> Result<(), String> { + if hdr && codec == VideoCodec::H264 { + return Err("H.264 HDR is unsupported".to_owned()); + } + let format = VideoFormat { + codec, + pixel_format, + chroma_format: chroma_format(pixel_format), + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: if hdr { + crate::VideoTransferFunction::Pq + } else { + crate::VideoTransferFunction::Sdr + }, + color_primaries: if hdr { + crate::VideoColorPrimaries::Bt2020 + } else { + crate::VideoColorPrimaries::Bt709 + }, + color_matrix: if hdr { + crate::VideoColorMatrix::Bt2020 + } else { + crate::VideoColorMatrix::Bt709 + }, + full_range: false, + ..self.video_format + }; + let mut decoder = + super::decoder::Decoder::new(self, format, crate::WindowsDecoderMode::Hardware)?; + let data: &[u8] = match (codec, pixel_format, hdr) { + (VideoCodec::H265, VideoPixelFormat::P010, false) => { + include_bytes!("../../fixtures/probe/hevc-p010-sdr.hevc") + } + (VideoCodec::H265, VideoPixelFormat::P010, true) => { + include_bytes!("../../fixtures/probe/hevc-p010-pq.hevc") + } + (VideoCodec::Av1, VideoPixelFormat::P010, false) => { + include_bytes!("../../fixtures/probe/av1-p010-sdr.obu") + } + (VideoCodec::Av1, VideoPixelFormat::P010, true) => { + include_bytes!("../../fixtures/probe/av1-p010-pq.obu") + } + (VideoCodec::H265, VideoPixelFormat::Ayuv, false) => { + include_bytes!("../../fixtures/probe/hevc-ayuv-sdr.hevc") + } + (VideoCodec::H265, VideoPixelFormat::Y410, false) => { + include_bytes!("../../fixtures/probe/hevc-y410-sdr.hevc") + } + (VideoCodec::H265, VideoPixelFormat::Y410, true) => { + include_bytes!("../../fixtures/probe/hevc-y410-pq.hevc") + } + _ => return Err("unsupported Windows decoder probe format".to_owned()), + }; + let frame = decoder.probe_frame(data)?; + if frame.format.pixel_format != pixel_format + || frame.format.transfer_function != format.transfer_function + { + return Err("decoder did not configure the exact requested probe format".to_owned()); + } + let result = unsafe { + super::embedded::probe_decoded_conversion( + super::embedded::AdoptedD3d11Context { + device: self.resources.device.as_raw(), + immediate_context: self.resources.context.as_raw(), + }, + &frame, + ) + }; + drop(frame); + decoder.stop(); + result + } + + pub(super) fn probe( + api: WindowsGraphicsApi, + adapter_luid: Option, + ) -> Result { + let format = VideoFormat { + codec: VideoCodec::H264, + width: 1920, + height: 1080, + frame_rate_numerator: NonZeroU32::new(60).expect("non-zero"), + frame_rate_denominator: NonZeroU32::new(1).expect("non-zero"), + average_bitrate: 10_000_000, + pixel_format: crate::VideoPixelFormat::Nv12, + chroma_format: crate::VideoChromaFormat::Cs420, + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + }; + Self::new( + api, + adapter_luid, + SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 64, + height: 64, + }, + visible: false, + }), + format, + ) + } + + pub(super) fn new( + api: WindowsGraphicsApi, + adapter_luid: Option, + target: SurfaceTarget, + video_format: VideoFormat, + ) -> Result { + validate_standalone_color(video_format)?; + let resources = DeviceResources::new(api, adapter_luid)?; + let window = RenderWindow::new(target)?; + let swap_size = window.client_size()?; + let swap_format = swap_chain_format(video_format, true); + let swap_chain = + create_swap_chain(&resources.device, window.hwnd(), swap_size, swap_format)?; + eprintln!( + "Windows presenter configured swapchain={}x{} format={} bitDepth={}", + swap_size.0, + swap_size.1, + swap_format.0, + video_format.pixel_format.bit_depth(), + ); + let mut graphics = Self { + processor: None, + swap_chain: swap_chain.swap_chain, + frame_latency_waitable: swap_chain.frame_latency_waitable, + swap_chain_flags: swap_chain.flags, + allow_tearing: swap_chain.allow_tearing, + has_presented: false, + window, + resources, + video_format, + swap_size, + swap_format, + ten_bit_output_supported: true, + }; + graphics.ensure_processor(video_format.width, video_format.height)?; + Ok(graphics) + } + + pub(super) fn device_manager(&self) -> &IMFDXGIDeviceManager { + &self.resources.manager + } + + pub(super) fn adapter_luid(&self) -> Result { + unsafe { + let dxgi_device: IDXGIDevice = self + .resources + .device + .cast() + .map_err(|error| error.to_string())?; + let adapter = dxgi_device + .GetAdapter() + .map_err(|error| error.to_string())?; + Ok(adapter + .GetDesc() + .map_err(|error| error.to_string())? + .AdapterLuid) + } + } + + pub(super) fn video_format(&self) -> VideoFormat { + self.video_format + } + + pub(super) fn is_visible(&self) -> bool { + self.window.is_visible() + } + + pub(super) fn set_surface( + &mut self, + target: SurfaceTarget, + video_format: VideoFormat, + ) -> Result<(), String> { + validate_standalone_color(video_format)?; + if self.window.can_update(target) { + self.window.update(target)?; + if self.video_format != video_format { + self.processor = None; + } + self.video_format = video_format; + self.resize_if_needed()?; + return self.ensure_processor(video_format.width, video_format.height); + } + let window = RenderWindow::new(target)?; + let swap_size = window.client_size()?; + let swap_format = swap_chain_format(video_format, self.ten_bit_output_supported); + let swap_chain = create_swap_chain( + &self.resources.device, + window.hwnd(), + swap_size, + swap_format, + )?; + self.processor = None; + let old_swap_chain = std::mem::replace(&mut self.swap_chain, swap_chain.swap_chain); + drop(old_swap_chain); + self.frame_latency_waitable = swap_chain.frame_latency_waitable; + self.swap_chain_flags = swap_chain.flags; + self.allow_tearing = swap_chain.allow_tearing; + self.has_presented = false; + self.window = window; + self.swap_size = swap_size; + self.swap_format = swap_format; + self.video_format = video_format; + self.ensure_processor(video_format.width, video_format.height) + } + + pub(super) fn reconfigure_video(&mut self, format: VideoFormat) -> Result<(), String> { + validate_standalone_color(format)?; + self.video_format = format; + self.processor = None; + self.resize_if_needed()?; + self.ensure_processor(format.width, format.height) + } + + pub(super) fn is_present_ready(&self) -> Result { + if !self.has_presented { + return Ok(true); + } + let result = unsafe { WaitForSingleObject(self.frame_latency_waitable, 0) }; + if result == WAIT_OBJECT_0 { + Ok(true) + } else if result == WAIT_TIMEOUT { + Ok(false) + } else if result == WAIT_FAILED { + Err(format!( + "DXGI frame-latency wait failed: {}", + unsafe { GetLastError() }.0 + )) + } else { + Err(format!( + "unexpected DXGI frame-latency wait result: {}", + result.0 + )) + } + } + + pub(super) fn present( + &mut self, + texture: &ID3D11Texture2D, + subresource: u32, + _timestamp_100ns: i64, + _duration_100ns: i64, + ) -> Result<(), String> { + let mut texture_description = Default::default(); + unsafe { + texture.GetDesc(&mut texture_description); + } + self.synchronize_decoder_surface_format(texture_description.Format)?; + self.resize_if_needed()?; + self.ensure_processor(texture_description.Width, texture_description.Height)?; + let processor = self + .processor + .as_mut() + .ok_or("video processor is unavailable")?; + + unsafe { + let input_description = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { + FourCC: 0, + ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPIV { + MipSlice: 0, + ArraySlice: subresource, + }, + }, + }; + let input_key = (texture.as_raw() as usize, subresource); + let input_view = if let Some(view) = processor.input_views.get(&input_key) { + view.clone() + } else { + let mut input_view = None; + self.resources + .video_device + .CreateVideoProcessorInputView( + texture, + &processor.enumerator, + &input_description, + Some(&mut input_view), + ) + .map_err(|error| error.to_string())?; + let input_view = + input_view.ok_or("D3D11 returned no video processor input view")?; + processor.input_views.insert(input_key, input_view.clone()); + input_view + }; + + let source = RECT { + left: 0, + top: 0, + right: processor.input_width as i32, + bottom: processor.input_height as i32, + }; + let destination = fit_rect( + processor.input_width, + processor.input_height, + processor.output_width, + processor.output_height, + ); + self.resources + .video_context + .VideoProcessorSetStreamFrameFormat( + &processor.processor, + 0, + D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + ); + self.resources + .video_context + .VideoProcessorSetStreamSourceRect(&processor.processor, 0, true, Some(&source)); + self.resources + .video_context + .VideoProcessorSetStreamDestRect(&processor.processor, 0, true, Some(&destination)); + self.resources + .video_context + .VideoProcessorSetOutputTargetRect( + &processor.processor, + true, + Some(&RECT { + left: 0, + top: 0, + right: processor.output_width as i32, + bottom: processor.output_height as i32, + }), + ); + let mut stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + pInputSurface: ManuallyDrop::new(Some(input_view)), + ..Default::default() + }; + let blit = self.resources.video_context.VideoProcessorBlt( + &processor.processor, + &processor.output_view, + 0, + std::slice::from_ref(&stream), + ); + ManuallyDrop::drop(&mut stream.pInputSurface); + blit.map_err(|error| error.to_string())?; + if self.resources._d3d12.is_some() { + self.resources.context.Flush(); + } + let present_flags = if self.allow_tearing { + DXGI_PRESENT_ALLOW_TEARING + } else { + DXGI_PRESENT(0) + }; + self.swap_chain + .Present(0, present_flags) + .ok() + .map_err(|error| error.to_string())?; + self.has_presented = true; + } + Ok(()) + } + + pub(super) fn pump_window_messages(&self) { + let mut message = MSG::default(); + unsafe { + while PeekMessageW(&mut message, None, 0, 0, PM_REMOVE).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + } + + fn resize_if_needed(&mut self) -> Result<(), String> { + let size = self.window.client_size()?; + let format = swap_chain_format(self.video_format, self.ten_bit_output_supported); + if size == self.swap_size && format == self.swap_format { + return Ok(()); + } + self.resize_swap_chain(size, format) + } + + fn resize_swap_chain(&mut self, size: (u32, u32), format: DXGI_FORMAT) -> Result<(), String> { + self.processor = None; + unsafe { + self.swap_chain + .ResizeBuffers(2, size.0, size.1, format, self.swap_chain_flags) + .map_err(|error| error.to_string())?; + } + self.swap_size = size; + self.swap_format = format; + set_swap_chain_color_space(&self.swap_chain)?; + eprintln!( + "Windows presenter resized swapchain={}x{} format={} bitDepth={}", + size.0, + size.1, + format.0, + self.video_format.pixel_format.bit_depth(), + ); + Ok(()) + } + + fn synchronize_decoder_surface_format(&mut self, format: DXGI_FORMAT) -> Result<(), String> { + let Some(pixel_format) = pixel_format_from_dxgi(format) else { + return Err(format!( + "Media Foundation returned unsupported D3D11 texture format {}", + format.0 + )); + }; + if pixel_format == self.video_format.pixel_format { + return Ok(()); + } + eprintln!( + "Windows presenter corrected decoder metadata {:?} to live texture {:?} (DXGI format={})", + self.video_format.pixel_format, pixel_format, format.0, + ); + self.video_format.pixel_format = pixel_format; + self.video_format.chroma_format = chroma_format(pixel_format); + self.processor = None; + Ok(()) + } + + fn ensure_processor(&mut self, input_width: u32, input_height: u32) -> Result<(), String> { + if self.processor.as_ref().is_some_and(|processor| { + processor.input_width == input_width + && processor.input_height == input_height + && processor.output_width == self.swap_size.0 + && processor.output_height == self.swap_size.1 + }) { + return Ok(()); + } + let description = D3D11_VIDEO_PROCESSOR_CONTENT_DESC { + InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + InputFrameRate: DXGI_RATIONAL { + Numerator: self.video_format.frame_rate_numerator.get(), + Denominator: self.video_format.frame_rate_denominator.get(), + }, + InputWidth: input_width, + InputHeight: input_height, + OutputFrameRate: DXGI_RATIONAL { + Numerator: self.video_format.frame_rate_numerator.get(), + Denominator: self.video_format.frame_rate_denominator.get(), + }, + OutputWidth: self.swap_size.0, + OutputHeight: self.swap_size.1, + Usage: D3D11_VIDEO_USAGE_OPTIMAL_SPEED, + }; + unsafe { + let enumerator = self + .resources + .video_device + .CreateVideoProcessorEnumerator(&description) + .map_err(|error| error.to_string())?; + if let Err(reason) = validate_video_processor_conversion( + &enumerator, + self.video_format, + self.swap_format, + ) { + if self.swap_format == DXGI_FORMAT_R10G10B10A2_UNORM { + // A number of Windows drivers expose P010 decode and an R10 + // swap-chain independently, but cannot connect those formats + // through the D3D11 video processor. VideoProcessorBlt may still + // return S_OK and silently paint black. Keep the negotiated + // 10-bit decoder surface and let the video processor dither it + // into the universally-supported SDR scan-out format instead. + self.ten_bit_output_supported = false; + eprintln!( + "Windows presenter falling back from 10-bit scan-out to 8-bit SDR: {reason}" + ); + self.resize_swap_chain(self.swap_size, DXGI_FORMAT_R8G8B8A8_UNORM)?; + return self.ensure_processor(input_width, input_height); + } + return Err(reason); + } + let processor = self + .resources + .video_device + .CreateVideoProcessor(&enumerator, 0) + .map_err(|error| error.to_string())?; + if let Some(video_context) = self.resources.video_context_1.as_ref() { + let input_color_space = input_color_space(self.video_format)?; + video_context.VideoProcessorSetStreamColorSpace1(&processor, 0, input_color_space); + video_context.VideoProcessorSetOutputColorSpace1( + &processor, + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, + ); + video_context.VideoProcessorSetStreamAutoProcessingMode(&processor, 0, false); + } + let back_buffer: ID3D11Texture2D = self + .swap_chain + .GetBuffer(0) + .map_err(|error| error.to_string())?; + let output_description = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { + ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 }, + }, + }; + let mut output_view = None; + self.resources + .video_device + .CreateVideoProcessorOutputView( + &back_buffer, + &enumerator, + &output_description, + Some(&mut output_view), + ) + .map_err(|error| error.to_string())?; + self.processor = Some(ProcessorResources { + input_width, + input_height, + output_width: self.swap_size.0, + output_height: self.swap_size.1, + enumerator, + processor, + input_views: HashMap::new(), + output_view: output_view.ok_or("D3D11 returned no video processor output view")?, + }); + eprintln!( + "Windows video processor conversion input={} output={} range={}", + dxgi_format(self.video_format.pixel_format).0, + self.swap_format.0, + if self.video_format.full_range { + "full" + } else { + "limited" + }, + ); + } + Ok(()) + } +} + +impl DecoderDevice for Graphics { + fn device_manager(&self) -> &IMFDXGIDeviceManager { + self.device_manager() + } + + fn adapter_luid(&self) -> Result { + self.adapter_luid() + } + + fn video_format(&self) -> VideoFormat { + self.video_format() + } +} + +fn create_swap_chain( + device: &ID3D11Device, + hwnd: HWND, + size: (u32, u32), + format: DXGI_FORMAT, +) -> Result { + unsafe { + let dxgi_device: IDXGIDevice = device.cast().map_err(|error| error.to_string())?; + let adapter = dxgi_device + .GetAdapter() + .map_err(|error| error.to_string())?; + let factory: IDXGIFactory2 = adapter.GetParent().map_err(|error| error.to_string())?; + let allow_tearing = supports_tearing(&factory); + let flags = if allow_tearing { + DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT | DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING + } else { + DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT + }; + let description = DXGI_SWAP_CHAIN_DESC1 { + Width: size.0, + Height: size.1, + Format: format, + Stereo: false.into(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + BufferUsage: DXGI_USAGE_RENDER_TARGET_OUTPUT, + BufferCount: 2, + Scaling: DXGI_SCALING_STRETCH, + SwapEffect: DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL, + AlphaMode: DXGI_ALPHA_MODE_IGNORE, + // IDXGISwapChain2::SetMaximumFrameLatency is only valid for a + // waitable swap chain. Preserve the flag in ResizeBuffers too. + Flags: flags.0 as u32, + }; + let swap_chain = factory + .CreateSwapChainForHwnd(device, hwnd, &description, None, None) + .map_err(|error| error.to_string())?; + let _ = factory.MakeWindowAssociation(hwnd, DXGI_MWA_NO_ALT_ENTER); + let swap_chain_2: IDXGISwapChain2 = swap_chain.cast().map_err(|error| error.to_string())?; + swap_chain_2 + .SetMaximumFrameLatency(1) + .map_err(|error| error.to_string())?; + let frame_latency_waitable = swap_chain_2.GetFrameLatencyWaitableObject(); + if frame_latency_waitable.0.is_null() { + return Err("DXGI returned no frame-latency waitable object".to_owned()); + } + set_swap_chain_color_space(&swap_chain)?; + Ok(SwapChainResources { + swap_chain, + frame_latency_waitable, + flags, + allow_tearing, + }) + } +} + +fn validate_standalone_color(format: VideoFormat) -> Result<(), String> { + if format.transfer_function != crate::VideoTransferFunction::Sdr { + return Err("HDR requires the embedded Qt HDR presenter; the standalone Windows swapchain is SDR-only".to_owned()); + } + input_color_space(format)?; + Ok(()) +} + +fn swap_chain_format(format: VideoFormat, ten_bit_output_supported: bool) -> DXGI_FORMAT { + if format.pixel_format.bit_depth() > 8 && ten_bit_output_supported { + DXGI_FORMAT_R10G10B10A2_UNORM + } else { + // Match the official Windows client. Its streaming swap chain starts at + // DXGI format 0x1c (R8G8B8A8) and changes to 0x18 for 10-bit scan-out. + DXGI_FORMAT_R8G8B8A8_UNORM + } +} + +fn dxgi_format(format: VideoPixelFormat) -> DXGI_FORMAT { + match format { + VideoPixelFormat::Nv12 => DXGI_FORMAT_NV12, + VideoPixelFormat::P010 => DXGI_FORMAT_P010, + VideoPixelFormat::Ayuv => DXGI_FORMAT_AYUV, + VideoPixelFormat::Y410 => DXGI_FORMAT_Y410, + } +} + +fn pixel_format_from_dxgi(format: DXGI_FORMAT) -> Option { + match format { + DXGI_FORMAT_NV12 => Some(VideoPixelFormat::Nv12), + DXGI_FORMAT_P010 => Some(VideoPixelFormat::P010), + DXGI_FORMAT_AYUV => Some(VideoPixelFormat::Ayuv), + DXGI_FORMAT_Y410 => Some(VideoPixelFormat::Y410), + _ => None, + } +} + +fn chroma_format(format: VideoPixelFormat) -> VideoChromaFormat { + match format { + VideoPixelFormat::Nv12 | VideoPixelFormat::P010 => VideoChromaFormat::Cs420, + VideoPixelFormat::Ayuv | VideoPixelFormat::Y410 => VideoChromaFormat::Cs444, + } +} + +fn validate_video_processor_conversion( + enumerator: &ID3D11VideoProcessorEnumerator, + input: VideoFormat, + output_format: DXGI_FORMAT, +) -> Result<(), String> { + unsafe { + let output_support = enumerator + .CheckVideoProcessorFormat(output_format) + .map_err(|error| format!("query output format {}: {error}", output_format.0))?; + if output_support & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT.0 as u32 == 0 { + return Err(format!( + "D3D11 video processor does not support output format {}", + output_format.0 + )); + } + + if let Ok(enumerator_1) = enumerator.cast::() { + let supported = enumerator_1 + .CheckVideoProcessorFormatConversion( + dxgi_format(input.pixel_format), + input_color_space(input)?, + output_format, + DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, + ) + .map_err(|error| { + format!( + "query D3D11 conversion {} -> {}: {error}", + dxgi_format(input.pixel_format).0, + output_format.0, + ) + })?; + if !supported.as_bool() { + return Err(format!( + "D3D11 driver rejects video conversion {} -> {}", + dxgi_format(input.pixel_format).0, + output_format.0, + )); + } + } + Ok(()) + } +} + +fn set_swap_chain_color_space(swap_chain: &IDXGISwapChain1) -> Result<(), String> { + let swap_chain_3: IDXGISwapChain3 = swap_chain.cast().map_err(|error| error.to_string())?; + unsafe { + swap_chain_3 + .SetColorSpace1(DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709) + .map_err(|error| error.to_string()) + } +} + +fn supports_tearing(factory: &IDXGIFactory2) -> bool { + let Ok(factory): Result = factory.cast() else { + return false; + }; + let mut supported = BOOL::default(); + unsafe { + factory + .CheckFeatureSupport( + DXGI_FEATURE_PRESENT_ALLOW_TEARING, + (&mut supported as *mut BOOL).cast(), + size_of::() as u32, + ) + .is_ok() + && supported.as_bool() + } +} + +fn create_owned_window(window: OwnedWindow) -> Result { + register_window_class()?; + let module = unsafe { GetModuleHandleW(None).map_err(|error| error.to_string())? }; + let instance = HINSTANCE(module.0); + let parent = window.parent.map(hwnd_from_handle); + let (style, extended_style) = owned_window_styles(parent.is_some()); + let hwnd = unsafe { + CreateWindowExW( + extended_style, + WINDOW_CLASS, + w!("OpenNOW Stream"), + style, + window.bounds.x, + window.bounds.y, + window.bounds.width as i32, + window.bounds.height as i32, + parent, + None, + Some(instance), + None, + ) + .map_err(|error| error.to_string())? + }; + if let Err(error) = position_owned_window(hwnd, window) { + unsafe { + let _ = DestroyWindow(hwnd); + } + return Err(error); + } + Ok(hwnd) +} + +fn owned_window_styles(has_parent: bool) -> (WINDOW_STYLE, WINDOW_EX_STYLE) { + let style = WS_CLIPCHILDREN + | WS_CLIPSIBLINGS + | if has_parent { + WS_CHILD + } else { + WS_OVERLAPPEDWINDOW + }; + let extended_style = WS_EX_NOACTIVATE + | WS_EX_TRANSPARENT + | if has_parent { + WINDOW_EX_STYLE(0) + } else { + WS_EX_APPWINDOW + }; + (style, extended_style) +} + +fn position_owned_window(hwnd: HWND, window: OwnedWindow) -> Result<(), String> { + let visibility = if window.visible { + SWP_SHOWWINDOW + } else { + SWP_HIDEWINDOW + }; + unsafe { + SetWindowPos( + hwnd, + None, + window.bounds.x, + window.bounds.y, + window.bounds.width as i32, + window.bounds.height as i32, + SWP_NOACTIVATE | SWP_NOZORDER | visibility, + ) + .map_err(|error| error.to_string()) + } +} + +fn register_window_class() -> Result<(), String> { + let module = unsafe { GetModuleHandleW(None).map_err(|error| error.to_string())? }; + let class = WNDCLASSW { + style: CS_HREDRAW | CS_VREDRAW, + lpfnWndProc: Some(window_proc), + hInstance: HINSTANCE(module.0), + lpszClassName: WINDOW_CLASS, + ..Default::default() + }; + if unsafe { RegisterClassW(&class) } != 0 { + return Ok(()); + } + let error = unsafe { GetLastError() }; + if error == ERROR_CLASS_ALREADY_EXISTS { + Ok(()) + } else { + Err(format!("RegisterClassW failed: {error:?}")) + } +} + +unsafe extern "system" fn window_proc( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if let Some(result) = owned_window_message_result(message) { + result + } else { + unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } + } +} + +fn owned_window_message_result(message: u32) -> Option { + match message { + WM_NCHITTEST => Some(LRESULT(HTTRANSPARENT as isize)), + WM_MOUSEACTIVATE => Some(LRESULT(MA_NOACTIVATE as isize)), + _ => None, + } +} + +fn hwnd_from_handle(handle: WindowHandle) -> HWND { + HWND(handle.get() as *mut c_void) +} + +fn fit_rect(input_width: u32, input_height: u32, output_width: u32, output_height: u32) -> RECT { + let input_aspect = input_width as f64 / input_height as f64; + let output_aspect = output_width as f64 / output_height as f64; + if output_aspect > input_aspect { + let width = (output_height as f64 * input_aspect).round() as i32; + let left = (output_width as i32 - width) / 2; + RECT { + left, + top: 0, + right: left + width, + bottom: output_height as i32, + } + } else { + let height = (output_width as f64 / input_aspect).round() as i32; + let top = (output_height as i32 - height) / 2; + RECT { + left: 0, + top, + right: output_width as i32, + bottom: top + height, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ::windows::Win32::UI::WindowsAndMessaging::WS_VISIBLE; + + fn video_format(pixel_format: VideoPixelFormat) -> VideoFormat { + VideoFormat { + codec: VideoCodec::H265, + width: 2560, + height: 1440, + frame_rate_numerator: NonZeroU32::new(120).expect("non-zero"), + frame_rate_denominator: NonZeroU32::new(1).expect("non-zero"), + average_bitrate: 100_000_000, + pixel_format, + chroma_format: chroma_format(pixel_format), + full_range: false, + chroma_siting: crate::VideoChromaSiting::Left, + transfer_function: crate::VideoTransferFunction::Sdr, + color_primaries: crate::VideoColorPrimaries::Bt709, + color_matrix: crate::VideoColorMatrix::Bt709, + } + } + + #[test] + fn standalone_presenter_rejects_hdr_instead_of_labeling_it_sdr() { + let format = video_format(VideoPixelFormat::P010); + assert!(validate_standalone_color(format).is_ok()); + for transfer_function in [ + crate::VideoTransferFunction::Pq, + crate::VideoTransferFunction::Hlg, + ] { + assert!( + validate_standalone_color(VideoFormat { + transfer_function, + color_primaries: crate::VideoColorPrimaries::Bt2020, + color_matrix: crate::VideoColorMatrix::Bt2020, + ..format + }) + .is_err() + ); + } + } + + fn is_missing_hardware_video_support(error: &str) -> bool { + error.contains("0x80004002") || error.contains("0x887A0004") + } + + #[test] + fn swap_chain_formats_match_official_windows_client() { + assert_eq!( + swap_chain_format(video_format(VideoPixelFormat::Nv12), true), + DXGI_FORMAT_R8G8B8A8_UNORM + ); + assert_eq!( + swap_chain_format(video_format(VideoPixelFormat::P010), true), + DXGI_FORMAT_R10G10B10A2_UNORM + ); + assert_eq!( + swap_chain_format(video_format(VideoPixelFormat::P010), false), + DXGI_FORMAT_R8G8B8A8_UNORM + ); + } + + #[test] + fn decoder_surface_formats_round_trip_to_dxgi() { + for format in [ + VideoPixelFormat::Nv12, + VideoPixelFormat::P010, + VideoPixelFormat::Ayuv, + VideoPixelFormat::Y410, + ] { + assert_eq!(pixel_format_from_dxgi(dxgi_format(format)), Some(format)); + } + } + + #[test] + fn letterboxes_wide_target() { + assert_eq!( + fit_rect(1920, 1080, 1024, 1024), + RECT { + left: 0, + top: 224, + right: 1024, + bottom: 800, + } + ); + } + + #[test] + fn owned_child_style_is_non_activating_and_input_transparent() { + let (style, extended_style) = owned_window_styles(true); + + assert_ne!(style.0 & WS_CHILD.0, 0); + assert_eq!(style.0 & WS_POPUP.0, 0); + assert_eq!(style.0 & WS_VISIBLE.0, 0); + assert_ne!(extended_style.0 & WS_EX_NOACTIVATE.0, 0); + assert_ne!(extended_style.0 & WS_EX_TRANSPARENT.0, 0); + } + + #[test] + fn owned_top_level_style_is_a_distinct_non_activating_app_window() { + let (style, extended_style) = owned_window_styles(false); + + assert_ne!(style.0 & WS_OVERLAPPEDWINDOW.0, 0); + assert_eq!(style.0 & WS_POPUP.0, 0); + assert_eq!(style.0 & WS_CHILD.0, 0); + assert_eq!(style.0 & WS_VISIBLE.0, 0); + assert_ne!(extended_style.0 & WS_EX_APPWINDOW.0, 0); + assert_ne!(extended_style.0 & WS_EX_NOACTIVATE.0, 0); + assert_ne!(extended_style.0 & WS_EX_TRANSPARENT.0, 0); + } + + #[test] + fn owned_window_rejects_hit_testing_and_mouse_activation() { + assert_eq!( + owned_window_message_result(WM_NCHITTEST), + Some(LRESULT(HTTRANSPARENT as isize)) + ); + assert_eq!( + owned_window_message_result(WM_MOUSEACTIVATE), + Some(LRESULT(MA_NOACTIVATE as isize)) + ); + assert_eq!(owned_window_message_result(0), None); + } + + #[test] + fn d3d11_on_12_exposes_video_processing_or_reports_unsupported_hardware() { + if let Err(error) = Graphics::probe(WindowsGraphicsApi::D3d12, None) { + assert!( + is_missing_hardware_video_support(&error), + "D3D11-on-12 probe failed unexpectedly: {error}" + ); + } + } + + #[test] + fn selected_adapter_luid_is_routed_without_default_fallback() { + let luid = WindowsAdapterLuid::new(0xffff_fffe_1122_3344).expect("non-zero LUID"); + let result = resolve_adapter_with(Some(luid), |native, raw| { + assert_eq!(native.LowPart, 0x1122_3344); + assert_eq!(native.HighPart, -2); + assert_eq!(raw, luid.get()); + Err("selected adapter unavailable".to_owned()) + }); + let error = match result { + Ok(_) => panic!("selected adapter resolution must preserve failure"), + Err(error) => error, + }; + assert_eq!(error, "selected adapter unavailable"); + + let default = resolve_adapter_with(None, |_, _| -> Result { + panic!("default selection must not resolve an explicit adapter") + }) + .expect("default adapter selection"); + assert!(default.is_none()); + } + + #[test] + fn p010_presentation_selects_a_scanout_or_reports_unsupported_hardware() { + let result = Graphics::new( + WindowsGraphicsApi::D3d11, + None, + SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 64, + height: 64, + }, + visible: false, + }), + video_format(VideoPixelFormat::P010), + ); + match result { + Ok(graphics) => assert!( + graphics.swap_format == DXGI_FORMAT_R10G10B10A2_UNORM + || graphics.swap_format == DXGI_FORMAT_R8G8B8A8_UNORM + ), + Err(error) => assert!( + is_missing_hardware_video_support(&error), + "P010 presenter failed unexpectedly: {error}" + ), + } + } + + #[test] + fn p010_presentation_executes_a_video_processor_blit() { + // Probe only the device interface. Do not turn later swap-chain, + // processor, texture, or presentation failures into capability skips. + let mut device = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + .expect("D3D11 capability probe device"); + } + let device = device.expect("D3D11 returned a device"); + if let Err(error) = device.cast::() { + assert_eq!( + error.code().0 as u32, + 0x80004002, + "unexpected video probe error: {error}" + ); + eprintln!("SKIP P010 GPU blit: D3D11 video interface unavailable ({error})"); + return; + } + let mut graphics = Graphics::new( + WindowsGraphicsApi::D3d11, + None, + SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 64, + height: 64, + }, + visible: false, + }), + video_format(VideoPixelFormat::P010), + ) + .expect("P010 presentation must initialize"); + let description = D3D11_TEXTURE2D_DESC { + Width: 64, + Height: 64, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_P010, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_DECODER.0 as u32 | D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut texture = None; + unsafe { + graphics + .resources + .device + .CreateTexture2D(&description, None, Some(&mut texture)) + .expect("create synthetic P010 decoder surface"); + } + graphics + .present( + &texture.expect("D3D11 returned a P010 surface"), + 0, + 0, + 166_667, + ) + .expect("P010 video-processor blit must complete"); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/mod.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/mod.rs new file mode 100644 index 000000000..9b0045f55 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/mod.rs @@ -0,0 +1,1104 @@ +// Packaged GUI applications have no stderr console. Keep these low-frequency +// decoder diagnostics in the FFI file sink as well as stderr for CLI tooling. +macro_rules! video_log { + ($($args:tt)*) => {{ + let message = format!($($args)*); + opennow_streamer_protocol::log::log_line("INFO", "windows-video", &message); + eprintln!("{message}"); + }}; +} + +mod adapter_index; +mod audio; +mod color; +mod decoder; +mod embedded; +mod graphics; +mod y410; + +pub(crate) use adapter_index::probe_adapter_decode; + +pub(crate) unsafe fn d3d11_adapter_luid( + device: *mut std::ffi::c_void, +) -> Result { + unsafe { embedded::d3d11_adapter_luid(device) } +} +pub use embedded::{ + AdoptedD3d11Context, D3d11ColorSpace, D3d11Frame, D3d11FrameProducer, D3d11FrameSubmitter, + D3d11RecordedFrame, D3d11TextureFormat, +}; + +use std::collections::VecDeque; +use std::sync::{Arc, mpsc}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use ::windows::Win32::Foundation::FreeLibrary; +use ::windows::Win32::Media::MediaFoundation::{MF_VERSION, MFSTARTUP_LITE, MFShutdown, MFStartup}; +use ::windows::Win32::Media::{TIMERR_NOERROR, timeBeginPeriod, timeEndPeriod}; +use ::windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoUninitialize}; +use ::windows::Win32::System::LibraryLoader::{LOAD_LIBRARY_SEARCH_SYSTEM32, LoadLibraryExW}; +use ::windows::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_ABOVE_NORMAL, +}; +use ::windows::core::w; + +use crate::{ + ADAPTIVE_VIDEO_QUEUE_CAPACITY, BackendConfig, BackendError, BackendEvent, CapabilityProbe, + Control, LifecycleState, Shared, Subsystem, VideoCodec, WindowsDecoderMode, WindowsGraphicsApi, +}; + +use self::audio::AudioRenderer; +use self::decoder::{DecodedVideoFrame, Decoder}; +use self::graphics::Graphics; + +const RECOVERY_TIMEOUT: Duration = Duration::from_secs(5); +const AUDIO_RECOVERY_RETRY_INTERVAL: Duration = Duration::from_millis(250); +const AUDIO_RECOVERY_LOG_INTERVAL: Duration = Duration::from_secs(5); +const PRESENTATION_SPIN_THRESHOLD: Duration = Duration::from_micros(300); +const ARRIVAL_HISTORY_LENGTH: usize = 120; +const QUEUE_HISTORY_LENGTH: usize = 3; +const RENDER_HISTORY_LENGTH: usize = 60; +const PACING_OUTLIER: Duration = Duration::from_millis(75); +const PACING_DIAGNOSTIC_INTERVAL: Duration = Duration::from_secs(5); +const MAX_CATCH_UP_RATE: f64 = 0.10; + +/// Spaces decoded frames at the negotiated stream cadence without depending on +/// GFN's RTP timestamps. H.265/AV1 can assign one timestamp to a small group of +/// frames, so presenting immediately on decode makes a 120 fps stream arrive at +/// the display as visible bursts. Slower, game-driven streams remain dynamic: +/// when a frame arrives after its deadline it is presented immediately and the +/// clock is rebased instead of trying to catch up. +struct PresentationClock { + nominal_interval: Duration, + arrival_history: VecDeque, + queue_history: VecDeque, + render_history: VecDeque, + last_arrival: Option, + last_presented: Option, + next_deadline: Option, + queue_integral: f64, + last_diagnostics: Instant, +} + +impl PresentationClock { + fn new(frame_duration_100ns: i64) -> Self { + let nominal_interval = frame_interval(frame_duration_100ns); + Self { + nominal_interval, + arrival_history: VecDeque::with_capacity(ARRIVAL_HISTORY_LENGTH), + queue_history: VecDeque::with_capacity(QUEUE_HISTORY_LENGTH), + render_history: VecDeque::with_capacity(RENDER_HISTORY_LENGTH), + last_arrival: None, + last_presented: None, + next_deadline: None, + queue_integral: 0.0, + last_diagnostics: Instant::now(), + } + } + + fn reset(&mut self, frame_duration_100ns: i64) { + self.nominal_interval = frame_interval(frame_duration_100ns); + self.arrival_history.clear(); + self.queue_history.clear(); + self.render_history.clear(); + self.last_arrival = None; + self.last_presented = None; + self.next_deadline = None; + self.queue_integral = 0.0; + self.last_diagnostics = Instant::now(); + } + + fn observe_decoded_frame(&mut self, now: Instant) { + if let Some(previous) = self.last_arrival.replace(now) { + let interval = now.saturating_duration_since(previous); + if interval <= PACING_OUTLIER { + push_bounded(&mut self.arrival_history, interval, ARRIVAL_HISTORY_LENGTH); + } else { + // A loading screen or recovery gap must not pull the active + // stream cadence down for the next 120 frames. + self.arrival_history.clear(); + self.next_deadline = None; + } + } + } + + fn is_due(&self, now: Instant) -> bool { + self.next_deadline.is_none_or(|deadline| now >= deadline) + } + + fn mark_presented(&mut self, now: Instant, queued_frames: usize) { + if let Some(previous) = self.last_presented.replace(now) { + push_bounded( + &mut self.render_history, + now.saturating_duration_since(previous), + RENDER_HISTORY_LENGTH, + ); + } + push_bounded(&mut self.queue_history, queued_frames, QUEUE_HISTORY_LENGTH); + let queue_average = average_usize(&self.queue_history); + self.queue_integral = (self.queue_integral * 0.9 + queue_average).clamp(0.0, 12.0); + let interval = self.controlled_interval(queue_average); + let next = self + .next_deadline + .map_or(now + interval, |deadline| deadline + interval); + // Preserve the presentation phase while decoded frames are queued. A + // D3D11 hardware decoder commonly releases AV1/H.265 frames in small + // bursts; rebasing every slightly-late deadline to `now + interval` + // makes that harmless lateness permanent and eventually forces the + // low-latency queue to discard frames. Keeping the old phase lets the + // worker use the next DXGI-ready slot to catch up. When the queue is + // empty, rebase so a genuinely dynamic/slow game stream is never + // chased with synthetic catch-up frames. + self.next_deadline = Some(if queued_frames == 0 && next <= now { + now + interval + } else { + next + }); + + if now.duration_since(self.last_diagnostics) >= PACING_DIAGNOSTIC_INTERVAL { + let observed = average_duration(&self.render_history).unwrap_or(Duration::ZERO); + let jitter = percentile_absolute_deviation(&self.render_history, observed, 0.99); + eprintln!( + "Adaptive presentation pacing: target={:.3}ms observed={:.3}ms p99Jitter={:.3}ms queue={queue_average:.2} arrivalSamples={}", + interval.as_secs_f64() * 1_000.0, + observed.as_secs_f64() * 1_000.0, + jitter.as_secs_f64() * 1_000.0, + self.arrival_history.len(), + ); + self.last_diagnostics = now; + } + } + + fn time_until_due(&self, now: Instant) -> Duration { + self.next_deadline.map_or(Duration::ZERO, |deadline| { + deadline.saturating_duration_since(now) + }) + } + + fn controlled_interval(&self, queue_average: f64) -> Duration { + let estimated = average_duration(&self.arrival_history) + .unwrap_or(self.nominal_interval) + .clamp( + self.nominal_interval, + self.nominal_interval.saturating_mul(2), + ); + // Queue feedback is intentionally conservative. P drains a burst now; + // I prevents a small persistent backlog, while the negotiated cadence + // remains the hard upper-FPS limit. + let correction = (queue_average * 0.10 + self.queue_integral * 0.004).clamp(0.0, 0.30); + let minimum = if queue_average > 0.0 { + self.nominal_interval.as_secs_f64() * (1.0 - MAX_CATCH_UP_RATE) + } else { + self.nominal_interval.as_secs_f64() + }; + Duration::from_secs_f64((estimated.as_secs_f64() / (1.0 + correction)).max(minimum)) + } +} + +fn frame_interval(frame_duration_100ns: i64) -> Duration { + Duration::from_nanos(u64::try_from(frame_duration_100ns.max(1)).unwrap_or(1) * 100) +} + +fn push_bounded(values: &mut VecDeque, value: T, capacity: usize) { + if values.len() == capacity { + values.pop_front(); + } + values.push_back(value); +} + +fn average_duration(values: &VecDeque) -> Option { + if values.is_empty() { + return None; + } + let total_ns = values.iter().map(Duration::as_nanos).sum::(); + Some(Duration::from_nanos( + u64::try_from(total_ns / values.len() as u128).unwrap_or(u64::MAX), + )) +} + +fn average_usize(values: &VecDeque) -> f64 { + if values.is_empty() { + 0.0 + } else { + values.iter().sum::() as f64 / values.len() as f64 + } +} + +fn percentile_absolute_deviation( + values: &VecDeque, + center: Duration, + percentile: f64, +) -> Duration { + if values.is_empty() { + return Duration::ZERO; + } + let center_ns = center.as_nanos(); + let mut deviations = values + .iter() + .map(|value| value.as_nanos().abs_diff(center_ns)) + .collect::>(); + deviations.sort_unstable(); + let index = ((deviations.len() - 1) as f64 * percentile).round() as usize; + Duration::from_nanos(u64::try_from(deviations[index]).unwrap_or(u64::MAX)) +} + +/// Keeps the native media loop out of Windows' coarse default timer cadence. +/// The official GFN renderer uses an accurate-sleep pacing path and elevated +/// RTP/media threads; without this, a nominal 1 ms idle sleep can occasionally +/// become a visible multi-frame stall. +struct LowLatencyThreadGuard { + timer_resolution_active: bool, +} + +impl LowLatencyThreadGuard { + fn enter() -> Self { + let timer_resolution_active = unsafe { timeBeginPeriod(1) == TIMERR_NOERROR }; + unsafe { + let _ = SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL); + } + Self { + timer_resolution_active, + } + } +} + +impl Drop for LowLatencyThreadGuard { + fn drop(&mut self) { + if self.timer_resolution_active { + unsafe { + let _ = timeEndPeriod(1); + } + } + } +} + +struct MediaRuntime; + +fn ensure_media_foundation_available() -> Result<(), String> { + unsafe { + let module = LoadLibraryExW(w!("mfplat.dll"), None, LOAD_LIBRARY_SEARCH_SYSTEM32) + .map_err(|error| format!("Media Foundation is unavailable: {error}"))?; + FreeLibrary(module).map_err(|error| format!("FreeLibrary(mfplat.dll): {error}")) + } +} + +impl MediaRuntime { + fn initialize() -> Result { + ensure_media_foundation_available().map_err(BackendError::Startup)?; + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED) + .ok() + .map_err(|error| BackendError::Startup(format!("CoInitializeEx: {error}")))?; + if let Err(error) = MFStartup(MF_VERSION, MFSTARTUP_LITE) { + CoUninitialize(); + return Err(BackendError::Startup(format!("MFStartup: {error}"))); + } + } + Ok(Self) + } +} + +impl Drop for MediaRuntime { + fn drop(&mut self) { + unsafe { + let _ = MFShutdown(); + CoUninitialize(); + } + } +} + +pub(super) fn probe( + api: WindowsGraphicsApi, + adapter_luid: Option, +) -> CapabilityProbe { + let _runtime = match MediaRuntime::initialize() { + Ok(runtime) => runtime, + Err(error) => { + return CapabilityProbe { + available: false, + h264_hardware_decode: false, + h265_hardware_decode: false, + av1_hardware_decode: false, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: false, + h265_software_decode: false, + av1_software_decode: false, + d3d11_presentation: false, + wasapi_render: false, + reason: Some(error.to_string()), + }; + } + }; + + let graphics = Graphics::probe(api, adapter_luid); + let h264_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::H264, WindowsDecoderMode::Hardware) + .map_err(|error| error.to_string()) + }); + let h265_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::H265, WindowsDecoderMode::Hardware) + .map_err(|error| error.to_string()) + }); + let av1_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::Av1, WindowsDecoderMode::Hardware) + .map_err(|error| error.to_string()) + }); + for (codec, result) in [ + (VideoCodec::H264, &h264_decoder), + (VideoCodec::H265, &h265_decoder), + (VideoCodec::Av1, &av1_decoder), + ] { + if let Err(error) = result { + video_log!( + "Windows hardware decoder probe failed api={api:?} codec={}: {error}", + codec.label() + ); + } + } + let h264_software_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::H264, WindowsDecoderMode::Software) + .map_err(|error| error.to_string()) + }); + let h265_software_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::H265, WindowsDecoderMode::Software) + .map_err(|error| error.to_string()) + }); + let av1_software_decoder = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| { + Decoder::probe(graphics, VideoCodec::Av1, WindowsDecoderMode::Software) + .map_err(|error| error.to_string()) + }); + let hdr_support = [VideoCodec::H265, VideoCodec::Av1].map(|codec| { + let result = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| graphics.probe_hdr(codec)); + if let Err(error) = &result { + video_log!( + "Windows HDR10 probe failed api={api:?} codec={}: {error}", + codec.label() + ); + } + result.is_ok() + }); + let format_support = [ + (VideoCodec::H265, crate::VideoPixelFormat::P010, false), + (VideoCodec::Av1, crate::VideoPixelFormat::P010, false), + (VideoCodec::H265, crate::VideoPixelFormat::Ayuv, false), + (VideoCodec::H265, crate::VideoPixelFormat::Y410, false), + (VideoCodec::H265, crate::VideoPixelFormat::Y410, true), + ].map(|(codec, pixel_format, hdr)| { + let result = graphics + .as_ref() + .map_err(Clone::clone) + .and_then(|graphics| graphics.probe_format(codec, pixel_format, hdr)); + if let Err(error) = &result { + video_log!( + "Windows format probe failed api={api:?} codec={} pixelFormat={pixel_format:?} hdr={hdr}: {error}", + codec.label() + ); + } + result.is_ok() + }); + let audio = AudioRenderer::probe().map_err(|error| error.to_string()); + let h264_hardware_decode = h264_decoder.is_ok(); + let h265_hardware_decode = h265_decoder.is_ok(); + let av1_hardware_decode = av1_decoder.is_ok(); + let h264_software_decode = h264_software_decoder.is_ok(); + let h265_software_decode = h265_software_decoder.is_ok(); + let av1_software_decode = av1_software_decoder.is_ok(); + let d3d11_presentation = graphics.is_ok(); + let wasapi_render = audio.is_ok(); + if let Err(error) = &audio { + video_log!("Windows standalone WASAPI output probe failed: {error}"); + } + let mut probe = CapabilityProbe { + available: false, + h264_hardware_decode, + h265_hardware_decode, + av1_hardware_decode, + h265_hdr: hdr_support[0], + av1_hdr: hdr_support[1], + h265_10bit: format_support[0], + av1_10bit: format_support[1], + h265_444: format_support[2], + h265_10bit_444: format_support[3], + h265_hdr_444: format_support[4], + h264_software_decode, + h265_software_decode, + av1_software_decode, + d3d11_presentation, + wasapi_render, + reason: None, + }; + probe.available = probe.bundled_backend_available(); + if !probe.available { + probe.reason = Some( + [ + graphics.err().map(|error| format!("{api:?}: {error}")), + h264_decoder.err().map(|error| format!("H.264: {error}")), + audio.err().map(|error| format!("WASAPI: {error}")), + ] + .into_iter() + .flatten() + .collect::>() + .join("; "), + ); + } + probe +} + +pub(super) fn spawn( + api: WindowsGraphicsApi, + decoder_mode: WindowsDecoderMode, + config: BackendConfig, + shared: Arc, + controls: mpsc::Receiver, +) -> Result, BackendError> { + let (ready_sender, ready_receiver) = mpsc::sync_channel(1); + let worker_shared = Arc::clone(&shared); + let worker = thread::Builder::new() + .name("opennow-windows-media".to_owned()) + .spawn(move || { + let runtime = match Worker::new(api, decoder_mode, config, worker_shared, controls) { + Ok(worker) => { + let _ = ready_sender.send(Ok(())); + worker + } + Err(error) => { + let _ = ready_sender.send(Err(error)); + return; + } + }; + runtime.run(); + }) + .map_err(|error| BackendError::Startup(format!("failed to spawn media thread: {error}")))?; + + match ready_receiver.recv() { + Ok(Ok(())) => Ok(worker), + Ok(Err(error)) => { + let _ = worker.join(); + Err(error) + } + Err(_) => { + let _ = worker.join(); + Err(BackendError::Startup( + "media thread exited during initialization".to_owned(), + )) + } + } +} + +struct Worker { + api: WindowsGraphicsApi, + decoder_mode: WindowsDecoderMode, + config: BackendConfig, + shared: Arc, + controls: mpsc::Receiver, + graphics: Graphics, + decoder: Decoder, + decoded_video: VecDeque, + presentation_clock: PresentationClock, + first_frame_presented: bool, + audio: Option, + audio_recovery: Option, + _runtime: MediaRuntime, +} + +struct AudioRecovery { + reason: String, + next_attempt: Instant, + next_log: Instant, +} + +impl Worker { + fn new( + api: WindowsGraphicsApi, + decoder_mode: WindowsDecoderMode, + config: BackendConfig, + shared: Arc, + controls: mpsc::Receiver, + ) -> Result { + let runtime = MediaRuntime::initialize()?; + let graphics = Graphics::new(api, None, config.surface, config.video) + .map_err(|error| BackendError::Startup(format!("{api:?} presentation: {error}")))?; + let decoder = Decoder::new(&graphics, config.video, decoder_mode).map_err(|error| { + BackendError::Startup(format!( + "Media Foundation {} {} decoder: {error}", + match decoder_mode { + WindowsDecoderMode::Hardware => "hardware", + WindowsDecoderMode::Software => "software", + }, + config.video.codec.label() + )) + })?; + let audio = AudioRenderer::new(config.audio) + .map_err(|error| BackendError::Startup(format!("WASAPI: {error}")))?; + let presentation_clock = PresentationClock::new(config.video.frame_duration_100ns()); + shared.set_state(LifecycleState::Running); + Ok(Self { + api, + decoder_mode, + config, + shared, + controls, + graphics, + decoder, + decoded_video: VecDeque::with_capacity(ADAPTIVE_VIDEO_QUEUE_CAPACITY), + presentation_clock, + first_frame_presented: false, + audio: Some(audio), + audio_recovery: None, + _runtime: runtime, + }) + } + + fn run(mut self) { + let _low_latency_thread = LowLatencyThreadGuard::enter(); + let mut stopping = false; + while !stopping { + let mut did_work = false; + self.graphics.pump_window_messages(); + while let Ok(control) = self.controls.try_recv() { + did_work = true; + match control { + Control::Stop => { + stopping = true; + break; + } + Control::ReconfigureVideo(format) => { + self.config.video = format; + self.shared.video.clear(); + if let Err(error) = self.rebuild_video( + Subsystem::VideoDecode, + error_label("video reconfiguration"), + ) { + self.fail(error); + return; + } + } + Control::ReconfigureAudio(format) => { + self.config.audio = format; + self.begin_audio_recovery(error_label("audio reconfiguration")); + } + Control::SetSurface(surface) => { + self.config.surface = surface; + self.decoded_video.clear(); + self.presentation_clock + .reset(self.config.video.frame_duration_100ns()); + self.first_frame_presented = false; + if let Err(error) = self.graphics.set_surface(surface, self.config.video) { + if let Err(error) = self.recover_video( + Subsystem::VideoPresentation, + format!("surface reconfiguration failed: {error}"), + ) { + self.fail(error); + return; + } + } else { + self.shared.set_state(LifecycleState::Running); + } + } + Control::SetPaused(paused) => { + self.decoded_video.clear(); + self.presentation_clock + .reset(self.config.video.frame_duration_100ns()); + self.first_frame_presented = false; + let audio_error = self + .audio + .as_mut() + .and_then(|audio| audio.set_paused(paused).err()); + if let Some(error) = audio_error { + self.begin_audio_recovery(format!( + "audio {} failed: {error}", + if paused { "pause" } else { "resume" } + )); + } + if !paused { + if let Err(error) = self + .rebuild_video(Subsystem::VideoDecode, "video resume".to_owned()) + { + self.fail(error); + return; + } + } + } + } + } + if stopping { + break; + } + let endpoint_result = self + .audio + .as_mut() + .map(AudioRenderer::default_endpoint_changed); + match endpoint_result { + Some(Ok(true)) => { + self.begin_audio_recovery("default audio endpoint changed".to_owned()); + } + Some(Ok(false)) | None => {} + Some(Err(error)) => { + self.begin_audio_recovery(format!( + "default audio endpoint check failed: {error}" + )); + } + } + self.poll_audio_recovery(); + if self + .shared + .paused + .load(std::sync::atomic::Ordering::Acquire) + { + thread::sleep(Duration::from_millis(1)); + continue; + } + + let produced = match self + .decoder + .poll_output(&mut self.decoded_video, &self.shared.events) + { + Ok(produced) => produced, + Err(error) => { + if let Err(error) = self.rebuild_video(Subsystem::VideoDecode, error) { + self.fail(error); + return; + } + continue; + } + }; + did_work |= produced > 0; + if produced > 0 { + self.presentation_clock + .observe_decoded_frame(Instant::now()); + } + let decoded_format = self.decoder.format(); + if decoded_format != self.config.video { + if let Err(error) = self.graphics.reconfigure_video(decoded_format) { + if let Err(error) = self.recover_video(Subsystem::VideoPresentation, error) { + self.fail(error); + return; + } + continue; + } + self.config.video = decoded_format; + self.presentation_clock + .reset(decoded_format.frame_duration_100ns()); + *self + .shared + .video_format + .lock() + .unwrap_or_else(|error| error.into_inner()) = decoded_format; + } + while self.decoder.wants_input() { + if let Some(frame) = self.shared.video.try_pop() { + did_work = true; + if frame.reset_decoder + && let Err(error) = self.reset_decoder_for_keyframe() + { + if let Err(error) = self.rebuild_video(Subsystem::VideoDecode, error) { + self.fail(error); + return; + } + // rebuild_video deliberately requests another clean + // keyframe; do not feed this one to a decoder whose + // recovery path failed midway. + continue; + } + if let Err(error) = self.decoder.submit(frame) { + if let Err(error) = self.rebuild_video(Subsystem::VideoDecode, error) { + self.fail(error); + return; + } + continue; + } + } else { + break; + } + } + // DXGI's frame-latency handle behaves like an auto-reset event. + // Checking it without a frame ready would consume the signal and + // leave the swap chain waiting forever for a Present that cannot + // happen. Only consume readiness when presentation can follow. + let presentation_now = Instant::now(); + if !self.decoded_video.is_empty() && self.presentation_clock.is_due(presentation_now) { + match self.graphics.is_present_ready() { + Ok(true) => { + // Keep every decoded frame while the bounded queue has + // capacity. GFN's dynamic stream durations describe + // game output cadence, not whether an older decoded + // surface is stale; comparing them discarded normal + // AV1/H.265 decoder bursts. The phase-preserving clock + // drains bursts, while process_output's seven-surface + // cap still bounds latency during a real overload. + let frame = self.decoded_video.pop_front(); + if let Some(frame) = frame { + did_work = true; + if let Err(error) = self.graphics.present( + &frame.texture, + frame.subresource, + frame.timestamp_100ns, + frame.duration_100ns, + ) { + if let Err(error) = + self.recover_video(Subsystem::VideoPresentation, error) + { + self.fail(error); + return; + } + continue; + } + self.shared + .presented_frames + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.presentation_clock + .mark_presented(Instant::now(), self.decoded_video.len()); + if !self.first_frame_presented && self.graphics.is_visible() { + self.first_frame_presented = true; + let _ = self.shared.events.push(BackendEvent::FirstFramePresented); + } + } + } + Ok(false) => {} + Err(error) => { + if let Err(error) = self.recover_video(Subsystem::VideoPresentation, error) + { + self.fail(error); + return; + } + continue; + } + } + } + let audio_result = self + .audio + .as_mut() + .map(|audio| audio.render(&self.shared.audio, &self.shared.audio_muted)); + match audio_result { + Some(Ok(true)) => { + did_work = true; + let _ = self + .shared + .events + .push(BackendEvent::QueueOverflow(Subsystem::Audio)); + } + Some(Ok(false)) => {} + Some(Err(error)) => { + self.begin_audio_recovery(error); + } + None => { + // Audio remains real-time while an endpoint is unavailable. + // Discard stale PCM instead of allowing it to backpressure video. + self.shared.audio.clear(); + } + } + let presentation_wait = if self.decoded_video.is_empty() { + Duration::ZERO + } else { + self.presentation_clock.time_until_due(Instant::now()) + }; + if !presentation_wait.is_zero() && presentation_wait <= PRESENTATION_SPIN_THRESHOLD { + // Avoid handing the final fraction of an 8.33 ms interval + // back to the Windows scheduler. The bounded 300 us precision + // phase costs less than four percent of one core at 120 Hz. + while !self + .presentation_clock + .time_until_due(Instant::now()) + .is_zero() + { + std::hint::spin_loop(); + } + } else if presentation_wait > PRESENTATION_SPIN_THRESHOLD { + thread::sleep( + presentation_wait + .saturating_sub(PRESENTATION_SPIN_THRESHOLD) + .min(Duration::from_millis(1)), + ); + } else if !did_work { + thread::sleep(Duration::from_millis(1)); + } else { + thread::yield_now(); + } + } + + self.decoder.stop(); + if let Some(audio) = self.audio.as_mut() { + audio.stop(); + } + } + + fn recover_video(&mut self, subsystem: Subsystem, message: String) -> Result<(), BackendError> { + let _ = self.shared.events.push(BackendEvent::DeviceLost { + subsystem, + message: message.clone(), + }); + self.shared.set_state(LifecycleState::Recovering); + self.shared.video.clear(); + self.decoded_video.clear(); + self.presentation_clock + .reset(self.config.video.frame_duration_100ns()); + self.first_frame_presented = false; + let start = Instant::now(); + loop { + if self.shared.state() == LifecycleState::Stopping { + return Ok(()); + } + self.config.video = *self + .shared + .video_format + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.config.surface = *self + .shared + .surface + .lock() + .unwrap_or_else(|error| error.into_inner()); + match Graphics::new(self.api, None, self.config.surface, self.config.video).and_then( + |graphics| { + Decoder::new(&graphics, self.config.video, self.decoder_mode) + .map(|decoder| (graphics, decoder)) + }, + ) { + Ok((graphics, decoder)) => { + self.graphics = graphics; + self.decoder = decoder; + let _ = self + .shared + .events + .push(BackendEvent::DeviceRecovered(subsystem)); + let _ = self.shared.events.push(BackendEvent::KeyFrameRequired); + self.shared.set_state(LifecycleState::Running); + return Ok(()); + } + Err(error) if start.elapsed() < RECOVERY_TIMEOUT => { + let _ = error; + thread::sleep(Duration::from_millis(100)); + } + Err(error) => { + return Err(BackendError::DeviceLost { + subsystem, + message: format!("{message}; recovery timed out: {error}"), + }); + } + } + } + } + + fn rebuild_video(&mut self, subsystem: Subsystem, message: String) -> Result<(), BackendError> { + self.decoder.stop(); + self.shared.video.clear(); + self.decoded_video.clear(); + self.presentation_clock + .reset(self.config.video.frame_duration_100ns()); + self.first_frame_presented = false; + match Decoder::new(&self.graphics, self.config.video, self.decoder_mode) { + Ok(decoder) => { + self.decoder = decoder; + self.graphics + .reconfigure_video(self.config.video) + .map_err(|error| { + BackendError::Reconfigure(format!("D3D11 presenter: {error}")) + })?; + let _ = self.shared.events.push(BackendEvent::KeyFrameRequired); + self.shared.set_state(LifecycleState::Running); + Ok(()) + } + Err(error) => self.recover_video(subsystem, format!("{message}: {error}")), + } + } + + fn reset_decoder_for_keyframe(&mut self) -> Result<(), String> { + self.decoder.stop(); + self.decoded_video.clear(); + self.presentation_clock + .reset(self.config.video.frame_duration_100ns()); + self.first_frame_presented = false; + self.decoder = Decoder::new(&self.graphics, self.config.video, self.decoder_mode) + .map_err(|error| format!("recovery-keyframe decoder reset failed: {error}"))?; + self.graphics + .reconfigure_video(self.config.video) + .map_err(|error| format!("recovery-keyframe presenter reset failed: {error}"))?; + Ok(()) + } + + fn begin_audio_recovery(&mut self, message: String) { + if self.audio_recovery.is_none() { + let _ = self.shared.events.push(BackendEvent::DeviceLost { + subsystem: Subsystem::Audio, + message: message.clone(), + }); + } + self.shared.audio.clear(); + if let Some(mut audio) = self.audio.take() { + audio.stop(); + } + let now = Instant::now(); + self.audio_recovery = Some(AudioRecovery { + reason: message, + next_attempt: now, + next_log: now + AUDIO_RECOVERY_LOG_INTERVAL, + }); + } + + fn poll_audio_recovery(&mut self) { + let Some(recovery) = self.audio_recovery.as_mut() else { + return; + }; + let now = Instant::now(); + if now < recovery.next_attempt { + return; + } + self.config.audio = *self + .shared + .audio_format + .lock() + .unwrap_or_else(|error| error.into_inner()); + let paused = self + .shared + .paused + .load(std::sync::atomic::Ordering::Acquire); + match AudioRenderer::new(self.config.audio).and_then(|mut audio| { + if paused { + audio.set_paused(true)?; + } + Ok(audio) + }) { + Ok(audio) => { + self.audio = Some(audio); + self.audio_recovery = None; + let _ = self + .shared + .events + .push(BackendEvent::DeviceRecovered(Subsystem::Audio)); + } + Err(error) => { + recovery.next_attempt = now + AUDIO_RECOVERY_RETRY_INTERVAL; + if now >= recovery.next_log { + eprintln!( + "WASAPI endpoint still unavailable after {}: {error}; video remains active", + recovery.reason + ); + recovery.next_log = now + AUDIO_RECOVERY_LOG_INTERVAL; + } + } + } + } + + fn fail(&self, error: BackendError) { + self.shared.set_state(LifecycleState::Failed); + let _ = self.shared.events.push(BackendEvent::Fatal(error)); + self.shared.video.close(); + self.shared.audio.close(); + } +} + +fn error_label(value: &str) -> String { + value.to_owned() +} + +#[cfg(test)] +mod tests { + use super::PresentationClock; + use std::time::{Duration, Instant}; + + #[test] + fn presentation_clock_spaces_a_decoded_burst() { + let start = Instant::now(); + let mut clock = PresentationClock::new(83_333); + + assert!(clock.is_due(start)); + clock.mark_presented(start, 3); + assert!(!clock.is_due(start + Duration::from_millis(4))); + assert!(clock.is_due(start + Duration::from_micros(8_334))); + } + + #[test] + fn presentation_clock_rebases_after_a_slow_frame() { + let start = Instant::now(); + let mut clock = PresentationClock::new(83_333); + clock.mark_presented(start, 0); + + let slow_arrival = start + Duration::from_millis(17); + assert!(clock.is_due(slow_arrival)); + clock.mark_presented(slow_arrival, 0); + assert!(!clock.is_due(slow_arrival + Duration::from_millis(4))); + assert!(clock.is_due(slow_arrival + Duration::from_micros(8_334))); + } + + #[test] + fn presentation_clock_preserves_phase_to_drain_a_decoder_burst() { + let start = Instant::now(); + let mut clock = PresentationClock::new(83_333); + clock.mark_presented(start, 3); + + let late = start + Duration::from_millis(18); + assert!(clock.is_due(late)); + clock.mark_presented(late, 2); + + // The next buffered frame is already due because the clock retained + // the original cadence instead of adding another full frame period to + // the late presentation time. + assert!(clock.is_due(late)); + } + + #[test] + fn presentation_clock_learns_a_dynamic_sixty_fps_source() { + let start = Instant::now(); + let mut clock = PresentationClock::new(83_333); + for frame in 0..120 { + clock.observe_decoded_frame(start + Duration::from_micros(frame * 16_667)); + } + + let interval = clock.controlled_interval(0.0); + assert!(interval >= Duration::from_micros(16_660)); + assert!(interval <= Duration::from_micros(16_670)); + } + + #[test] + fn presentation_clock_averages_grouped_frame_arrivals() { + let start = Instant::now(); + let mut clock = PresentationClock::new(83_333); + for group in 0..30 { + let group_start = start + Duration::from_millis(group * 33); + for frame in 0..4 { + clock.observe_decoded_frame(group_start + Duration::from_micros(frame * 20)); + } + } + + let interval = clock.controlled_interval(0.0); + assert!(interval >= Duration::from_micros(8_200)); + assert!(interval <= Duration::from_micros(8_500)); + } + + #[test] + fn presentation_clock_uses_a_bounded_catch_up_rate_for_backlog() { + let clock = PresentationClock::new(83_333); + let interval = clock.controlled_interval(3.0); + + assert!(interval < Duration::from_nanos(8_333_300)); + assert!(interval >= Duration::from_nanos(7_499_970)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.hlsl b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.hlsl new file mode 100644 index 000000000..b97f11a92 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.hlsl @@ -0,0 +1,14 @@ +Texture2D source_y410 : register(t0); +cbuffer Quantization : register(b0) { float4 scale_bias; float4 yuv_coefficients; }; +float4 vertex_main(uint id : SV_VertexID) : SV_Position { + float2 corner = float2((id << 1) & 2, id & 2); + return float4(corner * float2(2, -2) + float2(-1, 1), 0, 1); +} +float4 pixel_main(float4 position : SV_Position) : SV_Target { + float3 encoded = float3(source_y410.Load(int3(position.xy, 0)).xyz); + float y = encoded.y * scale_bias.x + scale_bias.y; + float u = encoded.x * scale_bias.z + scale_bias.w; + float v = encoded.z * scale_bias.z + scale_bias.w; + return float4(saturate(float3(y + yuv_coefficients.x * v, + y + yuv_coefficients.y * u + yuv_coefficients.z * v, y + yuv_coefficients.w * u)), 1); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.rs new file mode 100644 index 000000000..13288fad6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/windows/y410.rs @@ -0,0 +1,283 @@ +use super::decoder::DecodedVideoFrame; +use super::embedded::MAX_FRAME_SLOTS; +use crate::VideoFormat; +use crate::y410_color::Y410Constants; +use ::windows::Win32::Graphics::Direct3D::Fxc::D3DCompile; +use ::windows::Win32::Graphics::Direct3D::{ + D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D_SRV_DIMENSION_TEXTURE2D, ID3DBlob, +}; +use ::windows::Win32::Graphics::Direct3D11::*; +use ::windows::Win32::Graphics::Dxgi::Common::{ + DXGI_FORMAT_R10G10B10A2_UINT, DXGI_FORMAT_R10G10B10A2_UNORM, DXGI_FORMAT_Y410, DXGI_SAMPLE_DESC, +}; +use ::windows::core::PCSTR; +struct OutputSlot { + texture: ID3D11Texture2D, + view: ID3D11RenderTargetView, +} +pub(super) struct Y410Converter { + device: ID3D11Device, + immediate: ID3D11DeviceContext, + deferred: ID3D11DeviceContext, + input: ID3D11Texture2D, + input_view: ID3D11ShaderResourceView, + vertex: ID3D11VertexShader, + pixel: ID3D11PixelShader, + quantization: ID3D11Buffer, + rasterizer: ID3D11RasterizerState, + slots: [Option; MAX_FRAME_SLOTS], + format: VideoFormat, +} +impl Y410Converter { + pub(super) fn new( + device: &ID3D11Device, + immediate: &ID3D11DeviceContext, + format: VideoFormat, + ) -> Result { + let constants = Y410Constants::new(format)?; + let mut input = None; + let mut input_view = None; + let mut deferred = None; + unsafe { + device + .CreateTexture2D( + &D3D11_TEXTURE2D_DESC { + Width: format.width, + Height: format.height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_Y410, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + ..Default::default() + }, + None, + Some(&mut input), + ) + .map_err(|error| format!("create Y410 shader input: {error}"))?; + } + let input = input.ok_or("no Y410 shader input")?; + unsafe { + device + .CreateShaderResourceView( + &input, + Some(&D3D11_SHADER_RESOURCE_VIEW_DESC { + Format: DXGI_FORMAT_R10G10B10A2_UINT, + ViewDimension: D3D_SRV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_SHADER_RESOURCE_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_SRV { + MostDetailedMip: 0, + MipLevels: 1, + }, + }, + }), + Some(&mut input_view), + ) + .map_err(|error| format!("create exact Y410 UINT view: {error}"))?; + device + .CreateDeferredContext(0, Some(&mut deferred)) + .map_err(|error| format!("create Y410 deferred context: {error}"))?; + } + let vertex_code = compile(c"vertex_main", c"vs_5_0")?; + let pixel_code = compile(c"pixel_main", c"ps_5_0")?; + let mut vertex = None; + let mut pixel = None; + unsafe { + device + .CreateVertexShader( + std::slice::from_raw_parts( + vertex_code.GetBufferPointer().cast(), + vertex_code.GetBufferSize(), + ), + None, + Some(&mut vertex), + ) + .map_err(|error| format!("create Y410 vertex shader: {error}"))?; + device + .CreatePixelShader( + std::slice::from_raw_parts( + pixel_code.GetBufferPointer().cast(), + pixel_code.GetBufferSize(), + ), + None, + Some(&mut pixel), + ) + .map_err(|error| format!("create Y410 pixel shader: {error}"))?; + } + let mut quantization = None; + let mut rasterizer = None; + unsafe { + device + .CreateBuffer( + &D3D11_BUFFER_DESC { + ByteWidth: std::mem::size_of::() as u32, + Usage: D3D11_USAGE_IMMUTABLE, + BindFlags: D3D11_BIND_CONSTANT_BUFFER.0 as u32, + ..Default::default() + }, + Some(&D3D11_SUBRESOURCE_DATA { + pSysMem: std::ptr::from_ref(&constants).cast(), + ..Default::default() + }), + Some(&mut quantization), + ) + .map_err(|error| format!("create Y410 quantization constants: {error}"))?; + device + .CreateRasterizerState( + &D3D11_RASTERIZER_DESC { + FillMode: D3D11_FILL_SOLID, + CullMode: D3D11_CULL_NONE, + DepthClipEnable: true.into(), + ..Default::default() + }, + Some(&mut rasterizer), + ) + .map_err(|error| format!("create Y410 rasterizer: {error}"))?; + } + Ok(Self { + device: device.clone(), + immediate: immediate.clone(), + deferred: deferred.ok_or("no Y410 deferred context")?, + input, + input_view: input_view.ok_or("no Y410 shader view")?, + vertex: vertex.ok_or("no Y410 vertex shader")?, + pixel: pixel.ok_or("no Y410 pixel shader")?, + quantization: quantization.ok_or("no Y410 quantization buffer")?, + rasterizer: rasterizer.ok_or("no Y410 rasterizer")?, + slots: std::array::from_fn(|_| None), + format, + }) + } + pub(super) fn record( + &mut self, + slot: usize, + frame: &DecodedVideoFrame, + ) -> Result<&ID3D11Texture2D, String> { + if slot >= MAX_FRAME_SLOTS || frame.format != self.format { + return Err("Y410 converter frame or slot mismatch".to_owned()); + } + if self.slots[slot].is_none() { + let mut texture = None; + let mut view = None; + unsafe { + self.device + .CreateTexture2D( + &D3D11_TEXTURE2D_DESC { + Width: self.format.width, + Height: self.format.height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R10G10B10A2_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET).0 + as u32, + ..Default::default() + }, + None, + Some(&mut texture), + ) + .map_err(|error| format!("create Y410 RGB10A2 slot: {error}"))?; + } + let texture = texture.ok_or("no Y410 RGB10A2 slot")?; + unsafe { + self.device + .CreateRenderTargetView(&texture, None, Some(&mut view)) + } + .map_err(|error| format!("create Y410 RGB10A2 view: {error}"))?; + self.slots[slot] = Some(OutputSlot { + texture, + view: view.ok_or("no Y410 RGB10A2 view")?, + }); + } + let output = self.slots[slot].as_ref().ok_or("no Y410 output slot")?; + let region = D3D11_BOX { + left: frame.aperture.x, + top: frame.aperture.y, + front: 0, + right: frame.aperture.x + frame.aperture.width, + bottom: frame.aperture.y + frame.aperture.height, + back: 1, + }; + unsafe { + self.deferred.CopySubresourceRegion( + &self.input, + 0, + 0, + 0, + 0, + &frame.texture, + frame.subresource, + Some(®ion), + ); + self.deferred + .IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + self.deferred.VSSetShader(&self.vertex, None); + self.deferred.PSSetShader(&self.pixel, None); + self.deferred + .PSSetShaderResources(0, Some(&[Some(self.input_view.clone())])); + self.deferred + .PSSetConstantBuffers(0, Some(&[Some(self.quantization.clone())])); + self.deferred.RSSetState(&self.rasterizer); + self.deferred.RSSetViewports(Some(&[D3D11_VIEWPORT { + Width: self.format.width as f32, + Height: self.format.height as f32, + MaxDepth: 1.0, + ..Default::default() + }])); + self.deferred + .OMSetRenderTargets(Some(&[Some(output.view.clone())]), None); + self.deferred.Draw(3, 0); + let mut commands = None; + self.deferred + .FinishCommandList(false, Some(&mut commands)) + .map_err(|error| format!("finish Y410 conversion commands: {error}"))?; + self.immediate + .ExecuteCommandList(&commands.ok_or("no Y410 conversion commands")?, true); + self.device + .GetDeviceRemovedReason() + .map_err(|error| format!("Y410 conversion device lost: {error}"))?; + } + Ok(&output.texture) + } +} +fn compile(entry: &std::ffi::CStr, target: &std::ffi::CStr) -> Result { + let source = include_bytes!("y410.hlsl"); + let mut code = None; + let mut errors = None; + let result = unsafe { + D3DCompile( + source.as_ptr().cast(), + source.len(), + PCSTR::null(), + None, + None, + PCSTR(entry.as_ptr().cast()), + PCSTR(target.as_ptr().cast()), + 0, + 0, + &mut code, + Some(&mut errors), + ) + }; + if let Err(error) = result { + let detail = errors + .map(|errors| unsafe { + String::from_utf8_lossy(std::slice::from_raw_parts( + errors.GetBufferPointer().cast(), + errors.GetBufferSize(), + )) + .into_owned() + }) + .unwrap_or_default(); + return Err(format!("compile Y410 shader: {error}: {detail}")); + } + code.ok_or_else(|| "no Y410 shader bytecode".to_owned()) +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/y410_color.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/y410_color.rs new file mode 100644 index 000000000..524daa2ce --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform-windows/src/y410_color.rs @@ -0,0 +1,118 @@ +use crate::{ + VideoColorMatrix, VideoColorPrimaries, VideoFormat, VideoPixelFormat, VideoTransferFunction, +}; + +#[repr(C)] +pub(crate) struct Y410Constants { + pub(crate) scale_bias: [f32; 4], + pub(crate) matrix: [f32; 4], +} + +impl Y410Constants { + pub(crate) fn new(format: VideoFormat) -> Result { + format.validate_color().map_err(|error| error.to_string())?; + if format.pixel_format != VideoPixelFormat::Y410 { + return Err("Y410 shader conversion requires packed 10-bit 4:4:4 output".to_owned()); + } + let matrix = match ( + format.transfer_function, + format.color_primaries, + format.color_matrix, + ) { + (VideoTransferFunction::Sdr, VideoColorPrimaries::Bt709, VideoColorMatrix::Bt709) => { + [1.5748, -0.187_324_27, -0.468_124_27, 1.8556] + } + (VideoTransferFunction::Pq, VideoColorPrimaries::Bt2020, VideoColorMatrix::Bt2020) => { + [1.4746, -0.164_553_12, -0.571_353_14, 1.8814] + } + _ => return Err("Y410 shader conversion requires SDR BT.709 or PQ BT.2020".to_owned()), + }; + Ok(Self { + scale_bias: if format.full_range { + [1.0 / 1023.0, 0.0, 1.0 / 1023.0, -512.0 / 1023.0] + } else { + [1.0 / 876.0, -64.0 / 876.0, 1.0 / 896.0, -512.0 / 896.0] + }, + matrix, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{VideoChromaFormat, VideoChromaSiting, VideoCodec}; + use std::num::NonZeroU32; + + fn hdr_format(full_range: bool) -> VideoFormat { + VideoFormat { + codec: VideoCodec::H265, + width: 1920, + height: 1080, + frame_rate_numerator: NonZeroU32::new(60).unwrap(), + frame_rate_denominator: NonZeroU32::new(1).unwrap(), + average_bitrate: 10_000_000, + pixel_format: VideoPixelFormat::Y410, + chroma_format: VideoChromaFormat::Cs444, + chroma_siting: VideoChromaSiting::Left, + full_range, + transfer_function: VideoTransferFunction::Pq, + color_primaries: VideoColorPrimaries::Bt2020, + color_matrix: VideoColorMatrix::Bt2020, + } + } + + #[test] + fn hdr_y410_preserves_pq_codes_and_uses_bt2020_matrix_in_both_ranges() { + assert_eq!(std::mem::size_of::(), 32); + for full_range in [false, true] { + let constants = Y410Constants::new(hdr_format(full_range)).unwrap(); + let [ys, yb, cs, cb] = constants.scale_bias; + let [rv, gu, gv, bu] = constants.matrix; + let (black, white) = if full_range { (0, 1023) } else { (64, 940) }; + for code in black..=white { + let y = code as f32 * ys + yb; + let chroma = 512.0 * cs + cb; + assert!(chroma.abs() < 0.000001); + let expected = (code - black) as f32 / (white - black) as f32; + assert!((y - expected).abs() < 0.000001); + } + let y = 512.0 * ys + yb; + let u = 384.0 * cs + cb; + let v = 640.0 * cs + cb; + let rgb = [y + rv * v, y + gu * u + gv * v, y + bu * u]; + let kr = 0.2627_f32; + let kb = 0.0593_f32; + let expected = [ + y + 2.0 * (1.0 - kr) * v, + y - 2.0 * kb * (1.0 - kb) / (1.0 - kr - kb) * u + - 2.0 * kr * (1.0 - kr) / (1.0 - kr - kb) * v, + y + 2.0 * (1.0 - kb) * u, + ]; + for (actual, expected) in rgb.into_iter().zip(expected) { + assert!((actual - expected).abs() < 0.000001); + } + assert!((rgb[0] - (y + 1.5748 * v)).abs() > 0.01); + } + } + + #[test] + fn hdr_y410_rejects_mismatched_color_metadata_and_hlg() { + for format in [ + VideoFormat { + color_matrix: VideoColorMatrix::Bt709, + ..hdr_format(false) + }, + VideoFormat { + transfer_function: VideoTransferFunction::Hlg, + ..hdr_format(false) + }, + VideoFormat { + pixel_format: VideoPixelFormat::Ayuv, + ..hdr_format(false) + }, + ] { + assert!(Y410Constants::new(format).is_err()); + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/Cargo.toml new file mode 100644 index 000000000..5af50eaec --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "opennow-streamer-platform" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[build-dependencies] +cc = "1" + +[features] +default = [] +test-runtime = [] +linux-vaapi = ["opennow-streamer-platform-linux/vaapi"] +linux-ffmpeg = ["opennow-streamer-platform-linux/ffmpeg"] +linux-ffmpeg-bundled = ["opennow-streamer-platform-linux/ffmpeg-bundled"] + +[dependencies] +base64.workspace = true +image.workspace = true +openh264.workspace = true +opennow-streamer-protocol = { path = "../opennow-streamer-protocol" } +opennow-streamer-platform-windows = { path = "../opennow-streamer-platform-windows" } +oxideav-core.workspace = true +oxideav-mkv.workspace = true +opus.workspace = true +sdl2.workspace = true +scuffle-av1.workspace = true +serde_json.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +raw-window-handle = "0.6" +# sdl2-sys ships a single checked-in sdl_bindings.rs generated on Linux and +# copies it for every target unless use-bindgen is on. Its X11 layout assertions +# underflow against Windows LLP64, so Windows must generate bindings from the +# bundled SDL2 headers instead. Requires libclang on Windows build machines. +sdl2 = { workspace = true, features = ["use-bindgen"] } +windows-sys = { version = "0.61", features = [ + "Win32_Foundation", + "Win32_Graphics_Gdi", + "Win32_Storage_FileSystem", + "Win32_System_LibraryLoader", + "Win32_System_Threading", + "Win32_UI_Input", + "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging", +] } + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" +opennow-streamer-platform-linux = { path = "../opennow-streamer-platform-linux" } +raw-window-handle = "0.6" +x11-dl = "2.21" + +[target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" +objc2 = "0.6" +objc2-app-kit = { version = "0.3", default-features = false, features = ["NSGraphics", "NSResponder", "NSRunningApplication", "NSView", "NSWindow", "NSWorkspace"] } +objc2-foundation = { version = "0.3", default-features = false, features = ["NSGeometry"] } +opennow-streamer-platform-macos = { path = "../opennow-streamer-platform-macos" } +raw-window-handle = "0.6" diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/build.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/build.rs new file mode 100644 index 000000000..07956a74a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/build.rs @@ -0,0 +1,29 @@ +use std::path::PathBuf; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + + let output = cc::Build::new() + .get_compiler() + .to_command() + .arg("--print-resource-dir") + .output() + .expect("query the macOS C compiler's runtime directory"); + assert!( + output.status.success(), + "could not locate the macOS Clang runtime: {}", + String::from_utf8_lossy(&output.stderr) + ); + let resource_dir = String::from_utf8(output.stdout).expect("UTF-8 Clang resource directory"); + let runtime_dir = PathBuf::from(resource_dir.trim()).join("lib/darwin"); + assert!( + runtime_dir.join("libclang_rt.osx.a").is_file(), + "macOS Clang runtime is missing from {}; install the Xcode command-line tools", + runtime_dir.display() + ); + println!("cargo:rustc-link-search=native={}", runtime_dir.display()); + println!("cargo:rustc-link-lib=static=clang_rt.osx"); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/audio_playout.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/audio_playout.rs new file mode 100644 index 000000000..accda113e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/audio_playout.rs @@ -0,0 +1,160 @@ +//! Bounded stereo playout with gentle recovery from a persistent burst backlog. +//! +//! Ordinary packets are copied unchanged. Only a backlog above 60 ms sustained +//! for 200 ms enables a temporary, at-most-2% catch-up resample. Hysteresis stops +//! recovery at 40 ms. The callback neither allocates nor drops whole chunks of +//! audible PCM; both channels always use the same fractional source position. + +use std::collections::VecDeque; + +const CHANNELS: usize = 2; +const SAMPLE_RATE: usize = 48_000; +const TARGET_FRAMES: usize = SAMPLE_RATE * 40 / 1_000; +const HIGH_FRAMES: usize = SAMPLE_RATE * 60 / 1_000; +const PERSISTENT_FRAMES: usize = SAMPLE_RATE * 200 / 1_000; + +#[derive(Debug)] +pub(crate) struct AudioPlayoutBuffer { + samples: VecDeque, + capacity: usize, + high_frames: usize, + recovering: bool, +} + +impl AudioPlayoutBuffer { + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.samples.is_empty() + } + + pub(crate) fn new(capacity: usize) -> Self { + assert!(capacity >= CHANNELS && capacity % CHANNELS == 0); + Self { + samples: VecDeque::with_capacity(capacity), + capacity, + high_frames: 0, + recovering: false, + } + } + + pub(crate) fn push(&mut self, samples: &[f32]) -> usize { + // Never allow a partial interleaved frame to shift channel alignment. + let samples = &samples[..samples.len() / CHANNELS * CHANNELS]; + let overflow = self + .samples + .len() + .saturating_add(samples.len()) + .saturating_sub(self.capacity); + self.samples.drain(..overflow.min(self.samples.len())); + self.samples.extend( + samples[samples.len().saturating_sub(self.capacity)..] + .iter() + .copied(), + ); + overflow + } + + pub(crate) fn clear(&mut self) { + self.samples.clear(); + self.high_frames = 0; + self.recovering = false; + } + + pub(crate) fn fill(&mut self, destination: &mut [f32]) { + let output_frames = destination.len() / CHANNELS; + let available = self.samples.len() / CHANNELS; + let backlog = available.saturating_sub(output_frames); + if backlog > HIGH_FRAMES { + self.high_frames = self.high_frames.saturating_add(output_frames); + self.recovering |= self.high_frames >= PERSISTENT_FRAMES; + } else { + self.high_frames = 0; + } + if backlog <= TARGET_FRAMES { + self.recovering = false; + } + let extra = if self.recovering { + (output_frames / 50).min(backlog.saturating_sub(TARGET_FRAMES)) + } else { + 0 + }; + if extra == 0 { + for sample in destination.iter_mut().take(output_frames * CHANNELS) { + *sample = self.samples.pop_front().unwrap_or(0.0); + } + } else { + let consumed = output_frames + extra; + for frame in 0..output_frames { + let position = frame as f64 * consumed as f64 / output_frames as f64; + let first = position as usize; + let fraction = (position - first as f64) as f32; + for channel in 0..CHANNELS { + let a = self.samples[first * CHANNELS + channel]; + let b = self.samples[(first + 1) * CHANNELS + channel]; + destination[frame * CHANNELS + channel] = a + (b - a) * fraction; + } + } + self.samples.drain(..consumed * CHANNELS); + } + if destination.len() % CHANNELS != 0 { + *destination.last_mut().unwrap() = 0.0; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ordinary_audio_is_bit_exact_and_underruns_are_silent() { + let mut audio = AudioPlayoutBuffer::new(11_520); + assert_eq!(audio.push(&[0.25, -0.25, 0.5, -0.5]), 0); + let mut output = [1.0; 6]; + audio.fill(&mut output); + assert_eq!(output, [0.25, -0.25, 0.5, -0.5, 0.0, 0.0]); + } + + #[test] + fn overflow_preserves_newest_complete_stereo_frames() { + let mut audio = AudioPlayoutBuffer::new(4); + audio.push(&[1.0, 2.0]); + assert_eq!(audio.push(&[3.0, 4.0, 5.0, 6.0]), 2); + let mut output = [0.0; 4]; + audio.fill(&mut output); + assert_eq!(output, [3.0, 4.0, 5.0, 6.0]); + } + + #[test] + fn sustained_backlog_recovers_without_underruns_or_channel_skew() { + let mut audio = AudioPlayoutBuffer::new(11_520); + let packet: Vec<_> = (0..480).flat_map(|_| [0.25, -0.5]).collect(); + for _ in 0..11 { + audio.push(&packet); + } + let mut output = [0.0; 960]; + for _ in 0..600 { + audio.push(&packet); + audio.fill(&mut output); + assert!(output.chunks_exact(2).all(|v| v == [0.25, -0.5])); + } + assert_eq!(audio.samples.len() / CHANNELS, TARGET_FRAMES); + assert!(!audio.recovering); + } + + #[test] + fn short_burst_does_not_enable_resampling_and_clear_resets_recovery() { + let mut audio = AudioPlayoutBuffer::new(11_520); + audio.push(&vec![0.5; 11_520]); + let mut output = [0.0; 960]; + for _ in 0..10 { + audio.fill(&mut output); + } + assert!(!audio.recovering); + audio.high_frames = PERSISTENT_FRAMES; + audio.recovering = true; + audio.clear(); + assert_eq!(audio.high_frames, 0); + assert!(!audio.recovering); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/embedded_input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/embedded_input.rs new file mode 100644 index 000000000..5b54e20a0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/embedded_input.rs @@ -0,0 +1,442 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::{CapturedInput, CapturedInputQueue}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EmbeddedLocalAction { + Guide, + Screenshot, + RecordingToggle, +} + +pub struct EmbeddedInputCapture { + queue: Arc, + active: AtomicBool, + gamepads: Mutex<[Option; 4]>, + #[cfg(target_os = "linux")] + raw: Mutex>, + #[cfg(target_os = "windows")] + raw: Mutex>, +} + +impl EmbeddedInputCapture { + pub fn new(queue: Arc) -> Self { + Self { + queue, + active: AtomicBool::new(false), + gamepads: Mutex::new([None; 4]), + #[cfg(any(target_os = "linux", target_os = "windows"))] + raw: Mutex::new(None), + } + } + + pub fn submit(&self, input: CapturedInput) { + let mut gamepads = self + .gamepads + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let CapturedInput::Gamepad { + controller_id, + bitmap, + buttons, + left_trigger, + right_trigger, + left_stick_x, + left_stick_y, + right_stick_x, + right_stick_y, + } = &input + { + if usize::from(*controller_id) >= gamepads.len() { + return; + } + let neutral = ( + *buttons, + *left_trigger, + *right_trigger, + *left_stick_x, + *left_stick_y, + *right_stick_x, + *right_stick_y, + ) == (0, 0, 0, 0, 0, 0, 0); + if !self.active.load(Ordering::Acquire) && !neutral { + return; + } + for value in gamepads.iter_mut().flatten() { + *value = *bitmap; + } + gamepads[usize::from(*controller_id)] = Some(*bitmap); + if neutral && !self.active.load(Ordering::Acquire) { + self.queue.release_gamepad(*controller_id, *bitmap); + return; + } + } + if self.active.load(Ordering::Acquire) { + self.queue.push(input); + } + } + + pub fn submit_local_action(&self, action: EmbeddedLocalAction) { + self.submit(match action { + EmbeddedLocalAction::Guide => CapturedInput::Guide, + EmbeddedLocalAction::Screenshot => CapturedInput::Screenshot, + EmbeddedLocalAction::RecordingToggle => CapturedInput::RecordingToggle, + }); + } + + pub fn submit_text( + &self, + bytes: &[u8], + ) -> Result<(), opennow_streamer_protocol::text_input::TextInputError> { + let _guard = self + .gamepads + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !self.active.load(Ordering::Acquire) { + return Err(opennow_streamer_protocol::text_input::TextInputError::Unavailable); + } + self.queue.submit_text(bytes) + } + + pub fn set_active(&self, active: bool, relative_mouse: bool, window_handle: usize) -> bool { + let mut gamepads = self + .gamepads + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !active { + self.queue.discard_text(); + if self.active.load(Ordering::Acquire) { + for (controller_id, bitmap) in gamepads.iter_mut().enumerate() { + if let Some(bitmap) = bitmap.take() { + self.queue.release_gamepad(controller_id as u8, bitmap); + } + } + } + self.active.store(false, Ordering::Release); + } else { + self.active.store(true, Ordering::Release); + } + + #[cfg(target_os = "linux")] + { + let raw_enabled = x11_raw_capture_enabled(active, relative_mouse, window_handle); + let mut raw = self + .raw + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if raw_enabled && raw.is_none() { + match crate::linux_xinput::LinuxXInputController::start(Arc::clone(&self.queue)) { + Ok(controller) => *raw = Some(controller), + Err(error) => eprintln!("Embedded XInput2 capture unavailable: {error}"), + } + } + if let Some(raw) = raw.as_ref() { + raw.set_enabled(raw_enabled); + raw_enabled + } else { + false + } + } + + #[cfg(target_os = "windows")] + { + // Qt owns absolute position, buttons, and wheel as one ordered event + // stream. Raw Input is required only for relative motion. Letting the + // raw thread own buttons in absolute mode races the Qt position event, + // so the host can apply a click at its previous cursor coordinate. + let raw_enabled = raw_capture_enabled(active, relative_mouse); + let mut raw = self + .raw + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if active && raw.is_none() && window_handle != 0 { + match crate::windows_raw_input::WindowsRawInputController::start( + window_handle as isize, + Arc::clone(&self.queue), + ) { + Ok(controller) => *raw = Some(controller), + Err(error) => eprintln!("Embedded Raw Input capture unavailable: {error}"), + } + } + if let Some(raw) = raw.as_ref() { + if window_handle != 0 { + raw.set_foreground_owner(window_handle as isize); + } + raw.set_capture(raw_enabled, relative_mouse); + raw_enabled + } else { + false + } + } + + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + { + let _ = (relative_mouse, window_handle); + false + } + } + + pub fn queue(&self) -> Arc { + Arc::clone(&self.queue) + } +} + +#[cfg(any(target_os = "linux", test))] +const fn x11_raw_capture_enabled(active: bool, relative_mouse: bool, x11_window: usize) -> bool { + raw_capture_enabled(active, relative_mouse) && x11_window != 0 +} + +#[cfg(any(target_os = "linux", target_os = "windows", test))] +const fn raw_capture_enabled(active: bool, relative_mouse: bool) -> bool { + active && relative_mouse +} + +#[cfg(test)] +mod tests { + #[test] + fn old_session_teardown_cannot_cancel_or_enable_new_session_text() { + use opennow_streamer_protocol::text_input::TextInputError; + let queue = std::sync::Arc::new(crate::CapturedInputQueue::default()); + let capture = super::EmbeddedInputCapture::new(queue.clone()); + capture.set_active(true, false, 0); + queue.set_text_ready(1, true); + capture.submit_text(b"old").unwrap(); + let Some(crate::CapturedInput::Text(old)) = queue.take() else { + panic!("missing old paste") + }; + queue.set_text_ready(2, false); + assert!(old.is_cancelled()); + drop(old); + queue.set_text_ready(1, true); + assert_eq!( + capture.submit_text(b"new"), + Err(TextInputError::Unavailable) + ); + queue.set_text_ready(2, true); + capture.submit_text(b"new").unwrap(); + queue.set_text_ready(1, false); + let Some(crate::CapturedInput::Text(new)) = queue.take() else { + panic!("missing new paste") + }; + assert!(!new.is_cancelled()); + } + + #[test] + fn text_requires_capture_and_session_and_cancels_in_flight_on_focus_loss() { + use opennow_streamer_protocol::text_input::TextInputError; + let queue = std::sync::Arc::new(crate::CapturedInputQueue::default()); + let capture = super::EmbeddedInputCapture::new(queue.clone()); + assert_eq!( + capture.submit_text(b"paste"), + Err(TextInputError::Unavailable) + ); + capture.set_active(true, false, 0); + assert_eq!( + capture.submit_text(b"paste"), + Err(TextInputError::Unavailable) + ); + queue.set_text_ready(1, true); + assert_eq!(capture.submit_text(b"paste"), Ok(())); + assert_eq!(capture.submit_text(b"again"), Err(TextInputError::Busy)); + let Some(crate::CapturedInput::Text(text)) = queue.take() else { + panic!("missing text") + }; + assert_eq!(capture.submit_text(b"again"), Err(TextInputError::Busy)); + capture.set_active(false, false, 0); + assert!(text.is_cancelled()); + drop(text); + capture.set_active(true, false, 0); + assert_eq!(capture.submit_text(b"again"), Ok(())); + queue.set_text_ready(1, false); + assert!(queue.take().is_none()); + assert_eq!( + capture.submit_text(b"again"), + Err(TextInputError::Unavailable) + ); + } + + #[test] + fn full_capture_queue_rejects_paste_atomically_without_control_overflow() { + use opennow_streamer_protocol::text_input::TextInputError; + let queue = std::sync::Arc::new(crate::CapturedInputQueue::default()); + let capture = super::EmbeddedInputCapture::new(queue.clone()); + capture.set_active(true, false, 0); + queue.set_text_ready(1, true); + for _ in 0..256 { + capture.submit(crate::CapturedInput::Guide); + } + assert_eq!(capture.submit_text(b"paste"), Err(TextInputError::Busy)); + assert!(!queue.take_overflowed()); + for _ in 0..256 { + assert_eq!(queue.take(), Some(crate::CapturedInput::Guide)); + } + assert!(queue.take().is_none()); + assert_eq!(capture.submit_text(b"paste"), Ok(())); + } + + use super::*; + + fn gamepad(controller_id: u8, bitmap: u16, buttons: u16) -> CapturedInput { + CapturedInput::Gamepad { + controller_id, + bitmap, + buttons, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + } + } + + #[test] + fn closing_capture_neutralizes_all_controllers_before_rejecting_held_input() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + capture.set_active(true, false, 0); + for id in 0..4 { + capture.submit(gamepad(id, 0x0f0f, 0x1000)); + assert_eq!(queue.take(), Some(gamepad(id, 0x0f0f, 0x1000))); + } + capture.set_active(false, false, 0); + capture.submit(gamepad(0, 0x0f0f, 0x1000)); + for id in 0..4 { + assert_eq!(queue.take(), Some(gamepad(id, 0x0f0f, 0))); + } + assert_eq!(queue.take(), None); + capture.set_active(false, false, 0); + assert_eq!(queue.take(), None); + } + + #[test] + fn controller_disconnect_is_preserved_after_capture_closes() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + capture.set_active(true, false, 0); + capture.submit(gamepad(0, 0x0101, 0x1000)); + capture.set_active(false, false, 0); + capture.submit(gamepad(0, 0, 0)); + assert_eq!(queue.take(), Some(gamepad(0, 0, 0))); + assert_eq!(queue.take(), None); + } + + #[test] + fn concurrent_controller_submission_cannot_follow_a_capture_closure_neutral() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = Arc::new(EmbeddedInputCapture::new(Arc::clone(&queue))); + capture.set_active(true, false, 0); + capture.submit(gamepad(0, 0x0101, 0x1000)); + queue.take(); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let producer = { + let capture = Arc::clone(&capture); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + for _ in 0..64 { + capture.submit(gamepad(0, 0x0101, 0x1000)); + } + }) + }; + barrier.wait(); + capture.set_active(false, false, 0); + producer.join().unwrap(); + assert_eq!(queue.take(), Some(gamepad(0, 0x0101, 0))); + assert_eq!(queue.take(), None); + assert!(!queue.take_overflowed()); + } + + #[test] + fn capture_closure_resets_triggers_and_both_sticks_without_a_button_press() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + capture.set_active(true, false, 0); + capture.submit(CapturedInput::Gamepad { + controller_id: 0, + bitmap: 0x0101, + buttons: 0, + left_trigger: 255, + right_trigger: 128, + left_stick_x: 32767, + left_stick_y: -32768, + right_stick_x: 24000, + right_stick_y: -24000, + }); + queue.take(); + capture.set_active(false, false, 0); + assert_eq!(queue.take(), Some(gamepad(0, 0x0101, 0))); + assert_eq!(queue.take(), None); + } + + #[test] + fn capture_closure_reserves_neutral_slots_even_when_normal_queue_is_full() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + capture.set_active(true, false, 0); + for id in 0..4 { + capture.submit(gamepad(id, 0x0f0f, 0x1000)); + queue.take(); + } + for _ in 0..256 { + capture.submit_local_action(EmbeddedLocalAction::Guide); + } + capture.set_active(false, false, 0); + assert!(!queue.take_overflowed()); + for _ in 0..256 { + assert_eq!(queue.take(), Some(CapturedInput::Guide)); + } + for id in 0..4 { + assert_eq!(queue.take(), Some(gamepad(id, 0x0f0f, 0))); + } + assert_eq!(queue.take(), None); + } + + #[test] + fn capture_reacquisition_does_not_replay_stale_controller_state() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + for _ in 0..3 { + capture.set_active(true, false, 0); + assert_eq!(queue.take(), None); + capture.submit(gamepad(0, 0x0101, 0x1000)); + capture.set_active(false, false, 0); + assert_eq!(queue.take(), Some(gamepad(0, 0x0101, 0))); + assert_eq!(queue.take(), None); + } + } + + #[test] + fn xinput_requires_an_explicit_x11_window_even_with_relative_mode_enabled() { + assert!(!x11_raw_capture_enabled(true, true, 0)); + assert!(x11_raw_capture_enabled(true, true, 42)); + assert!(!x11_raw_capture_enabled(true, false, 42)); + assert!(!x11_raw_capture_enabled(false, true, 42)); + } + + #[test] + fn inactive_capture_drops_input_and_active_capture_preserves_typed_events() { + let queue = Arc::new(CapturedInputQueue::default()); + let capture = EmbeddedInputCapture::new(Arc::clone(&queue)); + capture.submit(CapturedInput::Key { + virtual_key: 0x57, + modifiers: 0, + pressed: true, + }); + assert_eq!(queue.take(), None); + + capture.active.store(true, Ordering::Release); + capture.submit_local_action(EmbeddedLocalAction::Guide); + assert_eq!(queue.take(), Some(CapturedInput::Guide)); + } + + #[test] + fn raw_capture_is_reserved_for_relative_mouse_mode() { + assert!(!raw_capture_enabled(false, false)); + assert!(!raw_capture_enabled(true, false)); + assert!(!raw_capture_enabled(false, true)); + assert!(raw_capture_enabled(true, true)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/graphics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/graphics.rs new file mode 100644 index 000000000..1704458e0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/graphics.rs @@ -0,0 +1,1149 @@ +use std::fmt; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, ThreadId}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GraphicsApi { + D3d11, + Vulkan, + Metal, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GraphicsTextureFormat { + Rgba8, + Rgb10A2, + Rgba16Float, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum GraphicsColorSpace { + #[default] + Sdr709, + Pq2020, + Hlg2020, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsContext { + pub api: GraphicsApi, + pub instance: usize, + pub physical_device: usize, + pub device: usize, + pub queue: usize, + pub queue_family_index: u32, + pub vulkan_dmabuf_import_enabled: bool, + pub vulkan_dmabuf_buffer_import_enabled: bool, +} + +impl GraphicsContext { + fn validate(self) -> Result { + if self.device == 0 || self.queue == 0 { + return Err(GraphicsRuntimeError::InvalidContext( + "the graphics device and queue must be non-null", + )); + } + if self.api == GraphicsApi::Vulkan && (self.instance == 0 || self.physical_device == 0) { + return Err(GraphicsRuntimeError::InvalidContext( + "Vulkan requires non-null instance and physical-device handles", + )); + } + Ok(self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsRecordCommand { + pub command_buffer: usize, + pub frame_slot: u32, + pub upscale_width: u32, + pub upscale_height: u32, + pub upscale_sharpness: u32, + pub upscale_denoise: u32, +} + +impl GraphicsRecordCommand { + fn validate(self) -> Result { + if self.command_buffer == 0 { + return Err(GraphicsRuntimeError::InvalidRenderCommand( + "the command buffer must be non-null", + )); + } + if (self.upscale_width == 0) != (self.upscale_height == 0) + || self.upscale_width > 16384 + || self.upscale_height > 16384 + { + return Err(GraphicsRuntimeError::InvalidRenderCommand( + "upscale dimensions must both be zero or between 1 and 16384", + )); + } + if self.upscale_sharpness > 15 || self.upscale_denoise > 20 { + return Err(GraphicsRuntimeError::InvalidRenderCommand( + "upscale sharpness must be between 0 and 15 and denoise between 0 and 20", + )); + } + Ok(self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsRecordedFrame { + pub resource: u64, + pub resource_view: u64, + pub texture_format: GraphicsTextureFormat, + pub color_space: GraphicsColorSpace, + pub width: u32, + pub height: u32, + pub frame_slot: u32, + pub generation: u64, + pub presentation_time_ns: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsFrameInfo { + pub width: u32, + pub height: u32, + pub sequence: u64, + pub presentation_time_ns: u64, +} + +/// A bootstrap frame can initialize an asynchronous decoder on the render +/// thread without having decoded output yet. This is not a render failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GraphicsFrameError { + NotReady, + Unsupported(String), + Failed(String), +} + +impl From for GraphicsFrameError { + fn from(message: String) -> Self { + Self::Failed(message) + } +} + +pub trait GraphicsRenderResources: Send + Sync + 'static { + /// Releases borrowed-device state on the render thread after host submission + /// has drained. Even on error, no subsequent drop may use the retired device. + fn retire(&self) -> Result<(), String>; +} + +pub trait GraphicsFrame: Send + Sync + 'static { + fn info(&self) -> GraphicsFrameInfo; + + fn render_resources(&self) -> Option> { + None + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result; +} + +#[cfg(target_os = "linux")] +impl GraphicsRenderResources for opennow_streamer_platform_linux::LinuxGpuRenderResources { + fn retire(&self) -> Result<(), String> { + self.retire().map_err(|error| error.to_string()) + } +} + +#[cfg(target_os = "linux")] +impl GraphicsFrame for opennow_streamer_platform_linux::LinuxGpuFrame { + fn render_resources(&self) -> Option> { + Some(self.render_resources()) + } + + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: self.width(), + height: self.height(), + sequence: self.sequence(), + presentation_time_ns: self.presentation_time_ns(), + } + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + if context.api != GraphicsApi::Vulkan { + return Err("a Linux decoded frame requires a Vulkan graphics context" + .to_owned() + .into()); + } + let frame = unsafe { + self.record( + opennow_streamer_platform_linux::VulkanRenderDevice { + instance: context.instance, + physical_device: context.physical_device, + device: context.device, + queue: context.queue, + queue_family: context.queue_family_index, + dmabuf_import_enabled: context.vulkan_dmabuf_import_enabled, + dmabuf_buffer_import_enabled: context.vulkan_dmabuf_buffer_import_enabled, + }, + command.command_buffer, + command.frame_slot, + ) + } + .map_err(|error| match error { + opennow_streamer_platform_linux::Error::FrameNotReady => GraphicsFrameError::NotReady, + error @ opennow_streamer_platform_linux::Error::ImportUnsupported(_) => { + GraphicsFrameError::Unsupported(error.to_string()) + } + error => GraphicsFrameError::from(error.to_string()), + })?; + Ok(GraphicsRecordedFrame { + resource: frame.image, + resource_view: frame.image_view, + texture_format: match frame.texture_format { + opennow_streamer_platform_linux::GpuTextureFormat::Rgba8 => { + GraphicsTextureFormat::Rgba8 + } + opennow_streamer_platform_linux::GpuTextureFormat::Rgb10A2 => { + GraphicsTextureFormat::Rgb10A2 + } + opennow_streamer_platform_linux::GpuTextureFormat::Rgba16Float => { + GraphicsTextureFormat::Rgba16Float + } + }, + color_space: match frame.color_space { + opennow_streamer_platform_linux::LinuxTextureColorSpace::Sdr709 => { + GraphicsColorSpace::Sdr709 + } + opennow_streamer_platform_linux::LinuxTextureColorSpace::Pq2020 => { + GraphicsColorSpace::Pq2020 + } + opennow_streamer_platform_linux::LinuxTextureColorSpace::Hlg2020 => { + GraphicsColorSpace::Hlg2020 + } + }, + width: frame.width, + height: frame.height, + frame_slot: frame.slot, + generation: frame.generation, + presentation_time_ns: frame.timestamp_us.saturating_mul(1_000), + }) + } +} + +#[cfg(target_os = "macos")] +impl GraphicsFrame for opennow_streamer_platform_macos::MetalFrame { + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: self.width(), + height: self.height(), + sequence: self.sequence(), + presentation_time_ns: self.presentation_time_ns(), + } + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + if context.api != GraphicsApi::Metal { + return Err( + "a macOS VideoToolbox frame requires a Metal graphics context" + .to_owned() + .into(), + ); + } + let frame = unsafe { + self.record( + opennow_streamer_platform_macos::AdoptedMetalContext { + device: context.device as *mut std::ffi::c_void, + command_buffer: command.command_buffer as *mut std::ffi::c_void, + upscale_width: command.upscale_width, + upscale_height: command.upscale_height, + upscale_sharpness: command.upscale_sharpness, + upscale_denoise: command.upscale_denoise, + }, + command.frame_slot, + ) + } + .map_err(|error| error.to_string())?; + let color_space = macos_frame_color_space(frame.color_space, frame.transfer, frame.format)?; + Ok(GraphicsRecordedFrame { + resource: frame.texture as usize as u64, + resource_view: 0, + color_space, + texture_format: match frame.format { + opennow_streamer_platform_macos::MetalFrameFormat::Rgba8Unorm => { + GraphicsTextureFormat::Rgba8 + } + opennow_streamer_platform_macos::MetalFrameFormat::Rgb10a2Unorm => { + GraphicsTextureFormat::Rgb10A2 + } + opennow_streamer_platform_macos::MetalFrameFormat::Rgba16Float => { + GraphicsTextureFormat::Rgba16Float + } + }, + width: frame.width, + height: frame.height, + frame_slot: frame.frame_slot, + generation: frame.generation, + presentation_time_ns: frame.presentation_time_ns, + }) + } +} + +#[cfg(target_os = "macos")] +fn macos_frame_color_space( + color_space: opennow_streamer_platform_macos::VideoColorSpace, + transfer: opennow_streamer_platform_macos::VideoTransfer, + format: opennow_streamer_platform_macos::MetalFrameFormat, +) -> Result { + use opennow_streamer_platform_macos::{MetalFrameFormat, VideoColorSpace, VideoTransfer}; + match (color_space, transfer, format) { + (VideoColorSpace::Bt2020, VideoTransfer::Pq, MetalFrameFormat::Rgba16Float) => { + Ok(GraphicsColorSpace::Pq2020) + } + ( + VideoColorSpace::Bt601 | VideoColorSpace::Bt709, + VideoTransfer::Sdr, + MetalFrameFormat::Rgba8Unorm | MetalFrameFormat::Rgb10a2Unorm, + ) => Ok(GraphicsColorSpace::Sdr709), + _ => Err( + "Metal frame color metadata does not match its texture format" + .to_owned() + .into(), + ), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GraphicsRuntimeError { + InvalidContext(&'static str), + InvalidRenderCommand(&'static str), + SceneGraphInactive, + WrongThread, + NoFrame, + StaleFrame, + FrameAlreadyRecorded, + RecordUnsupported(String), + RecordFailed(String), +} + +impl fmt::Display for GraphicsRuntimeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidContext(message) | Self::InvalidRenderCommand(message) => { + formatter.write_str(message) + } + Self::SceneGraphInactive => formatter.write_str("the scene graph is not initialized"), + Self::WrongThread => { + formatter.write_str("graphics access must stay on the render thread") + } + Self::NoFrame => formatter.write_str("no decoded GPU frame is pending"), + Self::StaleFrame => { + formatter.write_str("the GPU frame belongs to an older scene graph") + } + Self::FrameAlreadyRecorded => { + formatter.write_str("the GPU frame token was already recorded") + } + Self::RecordUnsupported(message) => { + write!(formatter, "GPU frame recording is unsupported: {message}") + } + Self::RecordFailed(message) => { + write!(formatter, "GPU frame recording failed: {message}") + } + } + } +} + +impl std::error::Error for GraphicsRuntimeError {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GraphicsPublishOutcome { + Published, + Replaced, +} + +pub struct GraphicsFrameToken { + epoch: u64, + frame: Arc, + owner: Arc, + recorded: AtomicBool, +} + +impl GraphicsFrameToken { + pub fn info(&self) -> GraphicsFrameInfo { + self.frame.info() + } +} + +#[derive(Clone)] +pub struct GraphicsFramePublisher { + inner: Arc, +} + +impl GraphicsFramePublisher { + pub fn published_frames(&self) -> u64 { + self.inner.published_frames.load(Ordering::Relaxed) + } + + pub fn clear(&self) { + lock_scene(&self.inner.scene).latest = None; + } + + pub fn context(&self) -> Option { + let scene = lock_scene(&self.inner.scene); + scene.context.map(|context| GraphicsContextLease { + context, + epoch: scene.epoch, + }) + } + + pub fn publish( + &self, + lease: GraphicsContextLease, + frame: Arc, + ) -> Result { + self.publish_frame(lease, frame, true) + } + + #[cfg(any(windows, test))] + pub(crate) fn publish_pending( + &self, + lease: GraphicsContextLease, + frame: Arc, + ) -> Result { + self.publish_frame(lease, frame, false) + } + + fn publish_frame( + &self, + lease: GraphicsContextLease, + frame: Arc, + decoded: bool, + ) -> Result { + let outcome = { + let mut scene = lock_scene(&self.inner.scene); + if scene.context.is_none() { + return Err(GraphicsRuntimeError::SceneGraphInactive); + } + if scene.epoch != lease.epoch || scene.context != Some(lease.context) { + return Err(GraphicsRuntimeError::StaleFrame); + } + let replaced = scene.latest.replace(FrameSlot { + epoch: lease.epoch, + frame, + }); + if decoded { + self.inner.published_frames.fetch_add(1, Ordering::Relaxed); + } + if replaced.is_some() { + GraphicsPublishOutcome::Replaced + } else { + GraphicsPublishOutcome::Published + } + }; + (self.inner.frame_available)(); + Ok(outcome) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsContextLease { + context: GraphicsContext, + epoch: u64, +} + +impl GraphicsContextLease { + pub fn context(self) -> GraphicsContext { + self.context + } +} + +#[derive(Clone)] +pub struct RenderThreadGraphics { + inner: Arc, +} + +impl RenderThreadGraphics { + pub fn new( + frame_available: impl Fn() + Send + Sync + 'static, + ) -> (Self, GraphicsFramePublisher) { + let inner = Arc::new(GraphicsRuntimeInner { + scene: Mutex::new(SceneGraphState::default()), + frame_available: Box::new(frame_available), + published_frames: AtomicU64::new(0), + }); + ( + Self { + inner: Arc::clone(&inner), + }, + GraphicsFramePublisher { inner }, + ) + } + + pub fn initialize(&self, context: GraphicsContext) -> Result<(), GraphicsRuntimeError> { + let context = context.validate()?; + let current_thread = thread::current().id(); + let mut scene = lock_scene(&self.inner.scene); + if let Some(render_thread) = scene.render_thread { + if render_thread != current_thread { + return Err(GraphicsRuntimeError::WrongThread); + } + if scene.context == Some(context) { + return Ok(()); + } + return Err(GraphicsRuntimeError::InvalidContext( + "shut down the previous scene graph before replacing its device", + )); + } + scene.latest = None; + scene.epoch = scene.epoch.wrapping_add(1); + scene.context = Some(context); + scene.render_thread = Some(current_thread); + Ok(()) + } + + pub fn acquire_latest(&self) -> Result { + let mut scene = lock_scene(&self.inner.scene); + require_render_thread(&scene)?; + let slot = scene.latest.take().ok_or(GraphicsRuntimeError::NoFrame)?; + if slot.epoch != scene.epoch { + return Err(GraphicsRuntimeError::StaleFrame); + } + Ok(GraphicsFrameToken { + epoch: slot.epoch, + frame: slot.frame, + owner: Arc::clone(&self.inner), + recorded: AtomicBool::new(false), + }) + } + + pub fn record( + &self, + frame: &GraphicsFrameToken, + command: GraphicsRecordCommand, + ) -> Result { + let command = command.validate()?; + if !Arc::ptr_eq(&self.inner, &frame.owner) { + return Err(GraphicsRuntimeError::StaleFrame); + } + let context = { + let mut scene = lock_scene(&self.inner.scene); + require_render_thread(&scene)?; + if frame.epoch != scene.epoch { + return Err(GraphicsRuntimeError::StaleFrame); + } + if frame.recorded.swap(true, Ordering::AcqRel) { + return Err(GraphicsRuntimeError::FrameAlreadyRecorded); + } + if let Some(resources) = frame.frame.render_resources() { + if scene + .resources + .as_ref() + .is_some_and(|active| !Arc::ptr_eq(active, &resources)) + { + return Err(GraphicsRuntimeError::RecordFailed( + "retire the previous session renderer before recording a new session" + .to_owned(), + )); + } + scene.resources = Some(resources); + } + scene + .context + .ok_or(GraphicsRuntimeError::SceneGraphInactive)? + }; + frame + .frame + .record(context, command) + .map_err(|error| match error { + GraphicsFrameError::NotReady => GraphicsRuntimeError::NoFrame, + GraphicsFrameError::Unsupported(message) => { + GraphicsRuntimeError::RecordUnsupported(message) + } + GraphicsFrameError::Failed(message) => GraphicsRuntimeError::RecordFailed(message), + }) + } + + /// The host must submit and drain all commands and release its imported + /// texture wrappers before shutdown, while the borrowed device is still live. + pub fn shutdown(&self) -> Result<(), GraphicsRuntimeError> { + let mut scene = lock_scene(&self.inner.scene); + if scene.context.is_none() { + return Ok(()); + } + require_render_thread(&scene)?; + let retirement = scene.retire_resources(); + scene.latest = None; + scene.context = None; + scene.render_thread = None; + scene.epoch = scene.epoch.wrapping_add(1); + retirement + } + + pub fn is_active(&self) -> bool { + lock_scene(&self.inner.scene).context.is_some() + } +} + +struct GraphicsRuntimeInner { + scene: Mutex, + frame_available: Box, + published_frames: AtomicU64, +} + +#[derive(Default)] +struct SceneGraphState { + context: Option, + render_thread: Option, + epoch: u64, + latest: Option, + resources: Option>, +} + +impl SceneGraphState { + fn retire_resources(&mut self) -> Result<(), GraphicsRuntimeError> { + if let Some(resources) = self.resources.take() { + resources + .retire() + .map_err(GraphicsRuntimeError::RecordFailed)?; + } + Ok(()) + } +} + +struct FrameSlot { + epoch: u64, + frame: Arc, +} + +fn require_render_thread(scene: &SceneGraphState) -> Result<(), GraphicsRuntimeError> { + let render_thread = scene + .render_thread + .ok_or(GraphicsRuntimeError::SceneGraphInactive)?; + if render_thread != thread::current().id() { + return Err(GraphicsRuntimeError::WrongThread); + } + Ok(()) +} + +fn lock_scene(scene: &Mutex) -> std::sync::MutexGuard<'_, SceneGraphState> { + scene + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[cfg(target_os = "macos")] + #[test] + fn metal_hdr_metadata_requires_bt2020_pq_float_output() { + use opennow_streamer_platform_macos::{MetalFrameFormat, VideoColorSpace, VideoTransfer}; + assert!(matches!( + macos_frame_color_space( + VideoColorSpace::Bt2020, + VideoTransfer::Pq, + MetalFrameFormat::Rgba16Float, + ), + Ok(GraphicsColorSpace::Pq2020) + )); + assert!(matches!( + macos_frame_color_space( + VideoColorSpace::Bt709, + VideoTransfer::Sdr, + MetalFrameFormat::Rgb10a2Unorm, + ), + Ok(GraphicsColorSpace::Sdr709) + )); + for (color, transfer, format) in [ + ( + VideoColorSpace::Bt709, + VideoTransfer::Pq, + MetalFrameFormat::Rgba16Float, + ), + ( + VideoColorSpace::Bt2020, + VideoTransfer::Sdr, + MetalFrameFormat::Rgb10a2Unorm, + ), + ( + VideoColorSpace::Bt2020, + VideoTransfer::Pq, + MetalFrameFormat::Rgb10a2Unorm, + ), + ( + VideoColorSpace::Bt2020, + VideoTransfer::Pq, + MetalFrameFormat::Rgba8Unorm, + ), + ] { + assert!(macos_frame_color_space(color, transfer, format).is_err()); + } + } + + struct RetainedResources(Arc>>); + + impl GraphicsRenderResources for RetainedResources { + fn retire(&self) -> Result<(), String> { + self.0.lock().unwrap().push(thread::current().id()); + Ok(()) + } + } + + struct ResourceFrame(Arc); + + struct FailedRetirement; + + impl GraphicsRenderResources for FailedRetirement { + fn retire(&self) -> Result<(), String> { + Err("device retirement failed".to_owned()) + } + } + + impl GraphicsFrame for ResourceFrame { + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: 2, + height: 2, + sequence: 1, + presentation_time_ns: 1, + } + } + + fn render_resources(&self) -> Option> { + Some(self.0.clone()) + } + + fn record( + &self, + _: GraphicsContext, + _: GraphicsRecordCommand, + ) -> Result { + Err(GraphicsFrameError::NotReady) + } + } + + #[test] + fn session_stop_keeps_render_resources_until_render_thread_retirement() { + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + let retired = Arc::new(Mutex::new(Vec::new())); + for _ in 0..3 { + graphics.initialize(context()).unwrap(); + let resources = Arc::new(RetainedResources(retired.clone())); + let weak = Arc::downgrade(&resources); + publisher + .publish( + publisher.context().unwrap(), + Arc::new(ResourceFrame(resources.clone())), + ) + .unwrap(); + let token = graphics.acquire_latest().unwrap(); + assert_eq!( + graphics.record(&token, command()), + Err(GraphicsRuntimeError::NoFrame) + ); + drop(token); + let worker_publisher = publisher.clone(); + thread::spawn(move || { + worker_publisher.clear(); + drop(resources); + }) + .join() + .unwrap(); + assert!(weak.upgrade().is_some()); + graphics.shutdown().unwrap(); + assert!(weak.upgrade().is_none()); + } + assert_eq!(*retired.lock().unwrap(), vec![thread::current().id(); 3]); + } + + #[test] + fn replacing_a_live_device_requires_explicit_shutdown() { + let (graphics, _) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).unwrap(); + let replacement = GraphicsContext { + device: 9, + ..context() + }; + assert!(matches!( + graphics.initialize(replacement), + Err(GraphicsRuntimeError::InvalidContext(_)) + )); + graphics.shutdown().unwrap(); + graphics.initialize(replacement).unwrap(); + graphics.shutdown().unwrap(); + } + + #[test] + fn failed_retirement_still_invalidates_the_borrowed_context() { + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).unwrap(); + publisher + .publish( + publisher.context().unwrap(), + Arc::new(ResourceFrame(Arc::new(FailedRetirement))), + ) + .unwrap(); + let token = graphics.acquire_latest().unwrap(); + assert_eq!( + graphics.record(&token, command()), + Err(GraphicsRuntimeError::NoFrame) + ); + assert!(matches!( + graphics.shutdown(), + Err(GraphicsRuntimeError::RecordFailed(_)) + )); + assert!(!graphics.is_active()); + assert!(publisher.context().is_none()); + assert_eq!( + graphics.record(&token, command()), + Err(GraphicsRuntimeError::SceneGraphInactive) + ); + graphics.shutdown().unwrap(); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_frame_implements_shared_graphics_frame_contract() { + fn assert_graphics_frame() {} + assert_graphics_frame::(); + } + + struct TestFrame { + sequence: u64, + drops: Arc, + records: Arc>>, + } + + impl GraphicsFrame for TestFrame { + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: 1920, + height: 1080, + sequence: self.sequence, + presentation_time_ns: self.sequence * 1_000, + } + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + self.records + .lock() + .expect("record calls") + .push((context, command)); + Ok(GraphicsRecordedFrame { + resource: 10, + resource_view: 11, + texture_format: GraphicsTextureFormat::Rgba8, + color_space: GraphicsColorSpace::Sdr709, + width: 1920, + height: 1080, + frame_slot: command.frame_slot, + generation: self.sequence, + presentation_time_ns: self.sequence * 1_000, + }) + } + } + + impl Drop for TestFrame { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::Relaxed); + } + } + + fn context() -> GraphicsContext { + GraphicsContext { + api: GraphicsApi::Vulkan, + instance: 1, + physical_device: 2, + device: 3, + queue: 4, + queue_family_index: 5, + vulkan_dmabuf_import_enabled: false, + vulkan_dmabuf_buffer_import_enabled: false, + } + } + + fn command() -> GraphicsRecordCommand { + GraphicsRecordCommand { + command_buffer: 6, + frame_slot: 7, + upscale_width: 0, + upscale_height: 0, + upscale_sharpness: 10, + upscale_denoise: 0, + } + } + + #[test] + fn upscale_target_is_optional_and_bounded() { + for (width, height, valid) in [ + (0, 0, true), + (2560, 1440, true), + (16384, 16384, true), + (0, 1440, false), + (2560, 0, false), + (16385, 1440, false), + (2560, u32::MAX, false), + ] { + let request = GraphicsRecordCommand { + upscale_width: width, + upscale_height: height, + ..command() + }; + assert_eq!(request.validate().is_ok(), valid, "{width}x{height}"); + } + } + + #[test] + fn upscale_controls_are_bounded_even_when_scaling_is_disabled() { + for (sharpness, denoise, valid) in [ + (0, 0, true), + (10, 0, true), + (15, 20, true), + (16, 0, false), + (0, 21, false), + (u32::MAX, u32::MAX, false), + ] { + for (width, height) in [(0, 0), (2560, 1440)] { + let request = GraphicsRecordCommand { + upscale_width: width, + upscale_height: height, + upscale_sharpness: sharpness, + upscale_denoise: denoise, + ..command() + }; + assert_eq!(request.validate().is_ok(), valid); + } + } + } + + fn frame( + sequence: u64, + drops: &Arc, + records: &Arc>>, + ) -> Arc { + Arc::new(TestFrame { + sequence, + drops: Arc::clone(drops), + records: Arc::clone(records), + }) + } + + #[test] + fn decoded_resize_keeps_source_dimensions_on_retained_tokens() { + struct DecodedFrame(GraphicsFrameInfo); + + impl GraphicsFrame for DecodedFrame { + fn info(&self) -> GraphicsFrameInfo { + self.0 + } + + fn record( + &self, + _: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + Ok(GraphicsRecordedFrame { + resource: 10, + resource_view: 11, + texture_format: GraphicsTextureFormat::Rgba8, + color_space: GraphicsColorSpace::Sdr709, + width: self.0.width, + height: self.0.height, + frame_slot: command.frame_slot, + generation: 1, + presentation_time_ns: self.0.presentation_time_ns, + }) + } + } + + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).unwrap(); + let lease = publisher.context().unwrap(); + let mut retained: Vec<(GraphicsFrameToken, GraphicsFrameInfo)> = Vec::new(); + for (width, height, sequence) in [(1920, 1080, 42), (1600, 1200, 47), (3440, 1440, 53)] { + let info = GraphicsFrameInfo { + width, + height, + sequence, + presentation_time_ns: sequence * 1_000, + }; + publisher + .publish(lease, Arc::new(DecodedFrame(info))) + .unwrap(); + let token = graphics.acquire_latest().unwrap(); + assert_eq!(token.info(), info); + let recorded = graphics.record(&token, command()).unwrap(); + assert_eq!((recorded.width, recorded.height), (width, height)); + assert_eq!(recorded.presentation_time_ns, info.presentation_time_ns); + for (old, old_info) in &retained { + assert_eq!(old.info(), *old_info); + } + retained.push((token, info)); + } + drop(retained); + graphics.shutdown().unwrap(); + } + + #[test] + fn publication_progress_survives_consumption_and_rejects_stale_frames() { + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).unwrap(); + let lease = publisher.context().unwrap(); + let drops = Arc::new(AtomicUsize::new(0)); + let records = Arc::new(Mutex::new(Vec::new())); + assert_eq!(publisher.published_frames(), 0); + publisher + .publish_pending(lease, frame(0, &drops, &records)) + .unwrap(); + assert_eq!(publisher.published_frames(), 0); + publisher + .publish(lease, frame(1, &drops, &records)) + .unwrap(); + publisher + .publish(lease, frame(2, &drops, &records)) + .unwrap(); + assert_eq!(publisher.published_frames(), 2); + drop(graphics.acquire_latest().unwrap()); + publisher.clear(); + assert_eq!(publisher.published_frames(), 2); + graphics.shutdown().unwrap(); + assert!( + publisher + .publish(lease, frame(3, &drops, &records)) + .is_err() + ); + graphics.initialize(context()).unwrap(); + assert!( + publisher + .publish(lease, frame(4, &drops, &records)) + .is_err() + ); + assert_eq!(publisher.published_frames(), 2); + publisher + .publish(publisher.context().unwrap(), frame(5, &drops, &records)) + .unwrap(); + assert_eq!(publisher.published_frames(), 3); + } + + #[test] + fn mailbox_keeps_only_the_latest_frame_and_transfers_ownership_to_the_token() { + let notifications = Arc::new(AtomicUsize::new(0)); + let callback_notifications = Arc::clone(¬ifications); + let (graphics, publisher) = RenderThreadGraphics::new(move || { + callback_notifications.fetch_add(1, Ordering::Relaxed); + }); + graphics.initialize(context()).expect("graphics context"); + let lease = publisher.context().expect("context lease"); + let drops = Arc::new(AtomicUsize::new(0)); + let records = Arc::new(Mutex::new(Vec::new())); + + assert_eq!( + publisher + .publish(lease, frame(1, &drops, &records)) + .expect("first frame"), + GraphicsPublishOutcome::Published + ); + assert_eq!( + publisher + .publish(lease, frame(2, &drops, &records)) + .expect("replacement frame"), + GraphicsPublishOutcome::Replaced + ); + assert_eq!(drops.load(Ordering::Relaxed), 1); + assert_eq!(notifications.load(Ordering::Relaxed), 2); + + let token = graphics.acquire_latest().expect("latest frame"); + assert_eq!(token.info().sequence, 2); + assert_eq!( + graphics.acquire_latest().err(), + Some(GraphicsRuntimeError::NoFrame) + ); + assert_eq!(drops.load(Ordering::Relaxed), 1); + drop(token); + assert_eq!(drops.load(Ordering::Relaxed), 2); + } + + #[test] + fn record_uses_the_active_context_and_callers_command_stream() { + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).expect("graphics context"); + let lease = publisher.context().expect("context lease"); + let drops = Arc::new(AtomicUsize::new(0)); + let records = Arc::new(Mutex::new(Vec::new())); + publisher + .publish(lease, frame(1, &drops, &records)) + .expect("frame"); + let token = graphics.acquire_latest().expect("latest frame"); + + let output = graphics.record(&token, command()).expect("record frame"); + + assert_eq!( + &*records.lock().expect("record calls"), + &[(context(), command())] + ); + assert_eq!(output.resource, 10); + assert_eq!(output.resource_view, 11); + } + + #[test] + fn shutdown_drops_pending_frames_and_invalidates_tokens_and_leases() { + let (graphics, publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).expect("graphics context"); + let old_lease = publisher.context().expect("context lease"); + let drops = Arc::new(AtomicUsize::new(0)); + let records = Arc::new(Mutex::new(Vec::new())); + publisher + .publish(old_lease, frame(1, &drops, &records)) + .expect("first frame"); + let token = graphics.acquire_latest().expect("latest frame"); + publisher + .publish(old_lease, frame(2, &drops, &records)) + .expect("pending frame"); + + graphics.shutdown().expect("scene graph shutdown"); + assert_eq!(drops.load(Ordering::Relaxed), 1); + assert_eq!( + graphics.record(&token, command()).err(), + Some(GraphicsRuntimeError::SceneGraphInactive) + ); + assert_eq!( + publisher + .publish(old_lease, frame(3, &drops, &records)) + .err(), + Some(GraphicsRuntimeError::SceneGraphInactive) + ); + + graphics.initialize(context()).expect("new scene graph"); + assert_eq!( + graphics.record(&token, command()).err(), + Some(GraphicsRuntimeError::StaleFrame) + ); + assert_eq!( + publisher + .publish(old_lease, frame(4, &drops, &records)) + .err(), + Some(GraphicsRuntimeError::StaleFrame) + ); + } + + #[test] + fn graphics_calls_are_bound_to_the_initializing_render_thread() { + let (graphics, _publisher) = RenderThreadGraphics::new(|| {}); + graphics.initialize(context()).expect("graphics context"); + let other_thread = std::thread::spawn(move || graphics.acquire_latest().err()) + .join() + .expect("render-thread check"); + assert_eq!(other_thread, Some(GraphicsRuntimeError::WrongThread)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/h264.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/h264.rs new file mode 100644 index 000000000..8a967119e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/h264.rs @@ -0,0 +1,138 @@ +use std::collections::BTreeMap; + +#[derive(Default)] +pub(crate) struct H264ParameterSets { + units: BTreeMap<(u8, u8), Vec>, +} + +impl H264ParameterSets { + pub fn retain(&mut self, access_unit: &[u8]) -> Result<(), String> { + let mut updates = BTreeMap::new(); + for unit in openh264::nal_units(access_unit) { + let Some(&header) = unit.get(3) else { + continue; + }; + let kind = header & 0x1f; + if !matches!(kind, 7 | 8) { + continue; + } + if unit.len() > 64 * 1024 { + return Err("H.264 parameter set exceeds 64 KiB".to_owned()); + } + let id = parameter_set_id(kind, &unit[4..]) + .ok_or_else(|| "invalid H.264 parameter-set identifier".to_owned())?; + updates.insert((kind, id), unit); + } + for (key, unit) in updates { + if self.units.get(&key).is_none_or(|cached| cached != unit) { + self.units.insert(key, unit.to_vec()); + } + } + Ok(()) + } + + pub fn replay(&self) -> Vec { + self.units.values().flatten().copied().collect() + } +} + +fn parameter_set_id(kind: u8, bytes: &[u8]) -> Option { + let mut zeroes = 0; + let rbsp = bytes.iter().copied().filter(|&byte| { + if zeroes == 2 && byte == 3 { + zeroes = 0; + return false; + } + zeroes = if byte == 0 { (zeroes + 1).min(2) } else { 0 }; + true + }); + let mut bits = rbsp + .flat_map(|byte| (0..8).rev().map(move |shift| (byte >> shift) & 1)) + .skip(if kind == 7 { 24 } else { 0 }); + let mut leading_zeroes = 0; + while bits.next()? == 0 { + leading_zeroes += 1; + if leading_zeroes > 8 { + return None; + } + } + let mut value = 1_u16; + for _ in 0..leading_zeroes { + value = (value << 1) | u16::from(bits.next()?); + } + let id = u8::try_from(value - 1).ok()?; + (kind != 7 || id < 32).then_some(id) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parameter_sets_replace_only_matching_ids_and_replay_sequence_sets_first() { + let mut sets = H264ParameterSets::default(); + sets.retain(&[0, 0, 1, 0x68, 0x80]).unwrap(); + sets.retain(&[0, 0, 1, 0x67, 66, 0, 30, 0x80]).unwrap(); + sets.retain(&[0, 0, 1, 0x68, 0x40]).unwrap(); + sets.retain(&[0, 0, 1, 0x68, 0xa0]).unwrap(); + assert_eq!(sets.units.len(), 3); + assert_eq!( + sets.replay(), + [ + 0, 0, 1, 0x67, 66, 0, 30, 0x80, 0, 0, 1, 0x68, 0xa0, 0, 0, 1, 0x68, 0x40 + ] + ); + } + + #[test] + fn invalid_parameter_set_does_not_replace_cached_configuration() { + let mut sets = H264ParameterSets::default(); + sets.retain(&[0, 0, 1, 0x68, 0x80]).unwrap(); + let before = sets.replay(); + assert!(sets.retain(&[0, 0, 1, 0x68, 0xa0, 0, 0, 1, 0x67]).is_err()); + assert_eq!(sets.replay(), before); + assert!(sets.retain(&[0, 0, 1, 0x68, 0, 0, 0]).is_err()); + assert_eq!(sets.replay(), before); + } + + #[test] + fn parameter_set_ids_are_bounded_and_skip_emulation_prevention() { + assert_eq!(parameter_set_id(7, &[66, 0, 30, 0x80]), Some(0)); + assert_eq!(parameter_set_id(7, &[0, 0, 3, 0, 0x80]), Some(0)); + assert_eq!(parameter_set_id(7, &[66, 0, 30, 0x04, 0x00]), Some(31)); + assert_eq!(parameter_set_id(7, &[66, 0, 30, 0x04, 0x20]), None); + assert_eq!(parameter_set_id(8, &[0, 0x80, 0]), Some(255)); + assert_eq!(parameter_set_id(8, &[0, 0x80, 0x80]), None); + } + + #[test] + fn oversized_parameter_set_does_not_change_the_cache() { + let mut sets = H264ParameterSets::default(); + sets.retain(&[0, 0, 1, 0x68, 0x80]).unwrap(); + let before = sets.replay(); + let mut oversized = vec![0xff; 64 * 1024 + 1]; + oversized[..5].copy_from_slice(&[0, 0, 1, 0x68, 0x80]); + assert!(sets.retain(&oversized).is_err()); + assert_eq!(sets.replay(), before); + } + + #[test] + fn mixed_annex_b_prefixes_retain_the_same_parameter_set_ids() { + let mut sets = H264ParameterSets::default(); + sets.retain(&[0, 0, 0, 1, 0x68, 0x80, 0, 0, 1, 0x67, 66, 0, 30, 0x80]) + .unwrap(); + let expected = [0, 0, 1, 0x67, 66, 0, 30, 0x80, 0, 0, 1, 0x68, 0x80]; + assert_eq!(sets.replay(), expected); + sets.retain(&[0, 0, 0, 1, 0x67, 66, 0, 30, 0x80]).unwrap(); + sets.retain(&[0, 0, 1, 0x68, 0x80]).unwrap(); + assert_eq!(sets.replay(), expected); + sets.retain(&[0, 0, 1, 0x67, 66, 0, 30, 0x80, 0, 0, 0, 1, 0x68, 0x80]) + .unwrap(); + assert_eq!(sets.units.len(), 2); + let replay = sets.replay(); + let kinds: Vec = openh264::nal_units(&replay) + .map(|unit| unit[3] & 0x1f) + .collect(); + assert_eq!(kinds, [7, 8]); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/lib.rs new file mode 100644 index 000000000..9ad532507 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/lib.rs @@ -0,0 +1,1108 @@ +mod audio_playout; +mod embedded_input; +mod graphics; +mod h264; +#[cfg(target_os = "linux")] +mod linux_backend; +#[cfg(target_os = "linux")] +mod linux_frame_pacing; +#[cfg(target_os = "linux")] +mod linux_xinput; +#[cfg(target_os = "macos")] +mod macos_backend; +mod media; +mod microphone; +mod native_surface; +mod output; +mod queue; +mod recording; +mod replay; +mod runtime; +mod video_queue; +#[cfg(target_os = "windows")] +mod windows_graphics; +#[cfg(target_os = "windows")] +mod windows_raw_input; + +pub use embedded_input::{EmbeddedInputCapture, EmbeddedLocalAction}; +pub use graphics::{ + GraphicsApi, GraphicsColorSpace, GraphicsContext, GraphicsContextLease, GraphicsFrame, + GraphicsFrameError, GraphicsFrameInfo, GraphicsFramePublisher, GraphicsFrameToken, + GraphicsPublishOutcome, GraphicsRecordCommand, GraphicsRecordedFrame, GraphicsRenderResources, + GraphicsRuntimeError, GraphicsTextureFormat, RenderThreadGraphics, +}; +pub use media::{ + CapturedInput, CapturedInputQueue, CapturedInputSample, DecodeStageTimings, + DecodeTimingsReport, EncodedFrame, EncodedRecordingReceiver, MediaCodec, MediaColorQuality, + MediaControl, MediaFeedback, MediaSession, MediaSink, MediaStreamConfig, MediaVideoCodec, + PushOutcome, ShortcutChord, StreamShortcutAction, StreamShortcutBindings, +}; +pub use microphone::{ + EncodedMicrophoneFrame, MICROPHONE_FRAME_SAMPLES, MICROPHONE_SAMPLE_RATE, MicrophoneReceiver, + MicrophoneSession, MicrophoneStatus, +}; +#[cfg(target_os = "linux")] +pub use opennow_streamer_platform_linux::{ + LinuxGpuFrame, LinuxGpuFrameProducer, SharedVulkanDevice, +}; +#[cfg(not(target_os = "linux"))] +pub enum SharedVulkanDevice {} +#[cfg(target_os = "macos")] +pub use opennow_streamer_platform_macos::{ + AdoptedMetalContext, EmbeddedFrameProducer, MetalFrame, MetalRecordedFrame, +}; +pub use opennow_streamer_platform_windows::WindowsAdapterLuid; +#[cfg(target_os = "windows")] +pub use opennow_streamer_platform_windows::{ + AdoptedD3d11Context, D3d11Frame, D3d11FrameProducer, D3d11FrameSubmitter, D3d11RecordedFrame, + D3d11TextureFormat, d3d11_adapter_luid, +}; +pub use recording::{ + ManualRecordingSummary, RecordingSummary, record_matroska, record_replay_matroska, +}; +pub use replay::ReplaySnapshot; +pub use runtime::{ + EmbeddedRuntimeConfig, MainThreadHost, MediaRuntime, MediaRuntimeControl, + create_embedded_runtime, create_embedded_runtime_with_config, + create_embedded_runtime_with_input, create_runtime, +}; +#[cfg(feature = "test-runtime")] +pub use runtime::{TestMediaRuntimeHost, create_test_runtime}; + +use opennow_streamer_protocol::{ + CodecCapability, GraphicsAdapterCapability, VideoBackendCapability, +}; + +#[cfg(any(target_os = "windows", test))] +fn windows_decode_dimensions() -> Option { + Some(opennow_streamer_protocol::DecodeDimensions { + width: opennow_streamer_platform_windows::MAX_DECODE_WIDTH, + height: opennow_streamer_platform_windows::MAX_DECODE_HEIGHT, + }) +} + +pub(crate) fn graphics_adapter_capabilities( + active_luid: Option, +) -> Vec { + opennow_streamer_platform_windows::graphics_adapter_capabilities(active_luid) +} + +pub fn video_backends() -> Vec { + #[cfg(target_os = "linux")] + let mut backends = { + let mut backends = linux_backend::video_backends(); + backends.push(software_backend()); + backends + }; + #[cfg(target_os = "windows")] + let mut backends = { + use opennow_streamer_platform_windows::WindowsGraphicsApi; + vec![ + windows_hardware_backend( + WindowsGraphicsApi::D3d12, + "d3d12", + "d3d11on12-nv12", + None, + WindowsOutputMode::Standalone, + ), + windows_hardware_backend( + WindowsGraphicsApi::D3d11, + "d3d11", + "d3d11-nv12", + None, + WindowsOutputMode::Standalone, + ), + software_backend(), + ] + }; + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + let mut backends = vec![hardware_backend(), software_backend()]; + apply_backend_policy( + &mut backends, + std::env::var("OPENNOW_NATIVE_VIDEO_BACKEND") + .ok() + .as_deref(), + ); + backends +} + +pub fn embedded_video_backends() -> Vec { + embedded_video_backends_with_config(None, None) +} + +pub(crate) fn embedded_video_backends_with_config( + _device: Option<&SharedVulkanDevice>, + _windows_adapter_luid: Option, +) -> Vec { + #[cfg(target_os = "linux")] + let mut backends = linux_backend::mediacodec_capability() + .map(|backend| vec![backend]) + .unwrap_or_else(linux_backend::video_backends); + #[cfg(target_os = "linux")] + for backend in &mut backends { + if backend.backend == "vulkan" { + use opennow_streamer_platform_linux::{PixelFormat, VideoCodec}; + for codec in &mut backend.codecs { + let profile = match codec.codec { + "h264" => VideoCodec::H264, + "h265" => VideoCodec::H265, + _ => VideoCodec::Av1, + }; + let colors = [ + ("8bit_420", PixelFormat::Nv12), + ("10bit_420", PixelFormat::P010), + ("8bit_444", PixelFormat::Nv24), + ("10bit_444", PixelFormat::P410), + ] + .into_iter() + .filter(|(color, _)| match codec.codec { + "h264" => *color == "8bit_420", + "av1" => !color.ends_with("444"), + _ => true, + }) + .filter_map(|(color, format)| { + _device + .is_some_and(|device| device.codec_format_support(profile, format)) + .then_some(color) + }) + .collect::>(); + codec.available = !colors.is_empty(); + codec.max_decode_dimensions = _device + .and_then(|device| device.max_decode_dimensions(profile)) + .map( + |[width, height]| opennow_streamer_protocol::DecodeDimensions { + width, + height, + }, + ); + codec.color_qualities = Some(colors); + codec.reason = (!codec.available) + .then_some("the attached Vulkan device does not support this decode profile"); + } + backend.available = backend.codecs.iter().any(|codec| codec.available); + backend.reason = (!backend.available) + .then_some("no compatible shared Vulkan decode device is attached"); + backend.zero_copy_modes.clear(); + } else { + for codec in &mut backend.codecs { + let mut colors = if codec.available { + vec!["8bit_420"] + } else { + Vec::new() + }; + if backend.backend == "vaapi" { + let profile = match codec.codec { + "h265" => opennow_streamer_platform_linux::VideoCodec::H265, + "av1" => opennow_streamer_platform_linux::VideoCodec::Av1, + _ => opennow_streamer_platform_linux::VideoCodec::H264, + }; + let support = opennow_streamer_platform_linux::vaapi_color_support(profile); + if apply_vaapi_color_support(&mut colors, support) { + codec.available = true; + codec.reason = None; + } + } + codec.color_qualities = Some(colors); + } + if backend.backend == "vaapi" { + backend.available = backend.codecs.iter().any(|codec| codec.available); + if backend.available { + backend.reason = None; + } + } + } + } + #[cfg(target_os = "linux")] + for backend in &mut backends { + for codec in &mut backend.codecs { + codec.hdr_color_qualities = Some( + codec + .color_qualities + .as_ref() + .into_iter() + .flatten() + .copied() + .filter(|color| { + backend.available + && codec.available + && matches!(backend.backend, "vulkan" | "vaapi") + && matches!(codec.codec, "h265" | "av1") + && color.starts_with("10bit_") + }) + .collect(), + ); + codec.hdr_supported = Some( + backend.available + && codec.available + && matches!(backend.backend, "vulkan" | "vaapi") + && matches!(codec.codec, "h265" | "av1") + && codec + .color_qualities + .as_ref() + .is_some_and(|colors| colors.contains(&"10bit_420")), + ); + } + } + #[cfg(target_os = "windows")] + let mut backends = { + use opennow_streamer_platform_windows::WindowsGraphicsApi; + vec![windows_hardware_backend( + WindowsGraphicsApi::D3d11, + "d3d11", + "d3d11-nv12", + _windows_adapter_luid, + WindowsOutputMode::Embedded, + )] + }; + #[cfg(target_os = "macos")] + let mut backends = vec![hardware_backend()]; + #[cfg(target_os = "macos")] + for backend in &mut backends { + for codec in &mut backend.codecs { + let mut colors = if !codec.available { + Vec::new() + } else if codec.codec == "h264" { + vec!["8bit_420"] + } else { + vec!["8bit_420", "10bit_420"] + }; + if codec.available + && codec.codec == "h265" + && opennow_streamer_platform_macos::probe_h265_444_ten_bit_hardware() + { + colors.push("10bit_444"); + } + codec.hdr_supported = Some( + codec.available + && codec.codec == "h265" + && opennow_streamer_platform_macos::probe_h265_hdr_hardware(), + ); + let mut hdr_colors = Vec::new(); + if codec.hdr_supported == Some(true) { + hdr_colors.push("10bit_420"); + } + if codec.available + && codec.codec == "h265" + && opennow_streamer_platform_macos::probe_h265_hdr_444_hardware() + { + hdr_colors.push("10bit_444"); + } + codec.hdr_color_qualities = Some(hdr_colors); + codec.color_qualities = Some(colors); + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + let mut backends = Vec::new(); + apply_backend_policy( + &mut backends, + std::env::var("OPENNOW_NATIVE_VIDEO_BACKEND") + .ok() + .as_deref(), + ); + backends +} + +#[cfg(target_os = "linux")] +fn apply_vaapi_color_support( + colors: &mut Vec<&'static str>, + support: opennow_streamer_platform_linux::VaapiColorSupport, +) -> bool { + if support.eight_bit_420 { + colors.push("8bit_420"); + } + if support.ten_bit_420 { + colors.push("10bit_420"); + } + support.eight_bit_420 || support.ten_bit_420 +} + +fn apply_backend_policy(backends: &mut [VideoBackendCapability], requested: Option<&str>) { + let requested = requested + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("auto") + .to_ascii_lowercase(); + if requested == "auto" { + return; + } + for backend in backends { + if backend_allowed_by_policy(&requested, backend.backend) { + continue; + } + backend.available = false; + backend.reason = Some("video backend was disabled by decoder policy"); + backend.zero_copy_modes.clear(); + for codec in &mut backend.codecs { + codec.available = false; + codec.hdr_supported = Some(false); + codec.hdr_color_qualities = Some(Vec::new()); + codec.color_qualities = Some(Vec::new()); + codec.reason = Some("video backend was disabled by decoder policy"); + } + } +} + +fn backend_allowed_by_policy(requested: &str, backend: &str) -> bool { + match requested { + "auto" | "" => true, + "software" => matches!(backend, "software" | "ffmpeg"), + "hardware" => !matches!(backend, "software" | "ffmpeg"), + "nvdec" => backend == "cuda", + explicit => backend == explicit, + } +} + +pub const fn supports_audio_decode() -> bool { + true +} + +pub const fn supports_audio_output() -> bool { + true +} + +#[cfg(any(target_os = "windows", test))] +#[derive(Clone, Copy)] +enum WindowsOutputMode { + Embedded, + Standalone, +} + +#[cfg(target_os = "windows")] +fn windows_hardware_backend( + api: opennow_streamer_platform_windows::WindowsGraphicsApi, + backend: &'static str, + zero_copy_mode: &'static str, + adapter_luid: Option, + output_mode: WindowsOutputMode, +) -> VideoBackendCapability { + if !runtime::backend_preference_allows(backend) { + return unavailable_backend( + backend, + "windows", + "Direct3D hardware decode was disabled by configuration", + ); + } + let probe = opennow_streamer_platform_windows::WindowsBackend::probe_for(api, adapter_luid); + windows_hardware_capability(&probe, backend, zero_copy_mode, output_mode) +} + +#[cfg(any(target_os = "windows", test))] +fn windows_hardware_capability( + probe: &opennow_streamer_platform_windows::CapabilityProbe, + backend: &'static str, + zero_copy_mode: &'static str, + output_mode: WindowsOutputMode, +) -> VideoBackendCapability { + use opennow_streamer_platform_windows::{VideoCodec, VideoPixelFormat}; + + let media_output_available = probe.d3d11_presentation + && (matches!(output_mode, WindowsOutputMode::Embedded) || probe.wasapi_render); + let available = media_output_available + && (probe.h264_hardware_decode || probe.h265_hardware_decode || probe.av1_hardware_decode); + let color_qualities = |codec| { + [ + ("8bit_420", VideoPixelFormat::Nv12), + ("10bit_420", VideoPixelFormat::P010), + ("8bit_444", VideoPixelFormat::Ayuv), + ("10bit_444", VideoPixelFormat::Y410), + ] + .into_iter() + .filter_map(|(color, format)| { + (media_output_available && probe.supports_format(codec, format, false)).then_some(color) + }) + .collect::>() + }; + let hdr_color_qualities = |codec| { + [ + ("10bit_420", VideoPixelFormat::P010), + ("10bit_444", VideoPixelFormat::Y410), + ] + .into_iter() + .filter_map(|(color, format)| { + (media_output_available && probe.supports_format(codec, format, true)).then_some(color) + }) + .collect::>() + }; + let reason = if available { + None + } else { + Some(match output_mode { + WindowsOutputMode::Embedded => { + "Direct3D hardware decode or presentation is unavailable" + } + WindowsOutputMode::Standalone => { + "Direct3D hardware decode, presentation, or WASAPI output is unavailable" + } + }) + }; + VideoBackendCapability { + backend, + platform: "windows", + codecs: vec![ + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: Some(false), + hdr_color_qualities: Some(hdr_color_qualities(VideoCodec::H264)), + color_qualities: Some(color_qualities(VideoCodec::H264)), + codec: "h264", + available: media_output_available && probe.h264_hardware_decode, + reason: (!(media_output_available && probe.h264_hardware_decode)).then_some( + "H.264 Media Foundation hardware decode or media output is unavailable", + ), + }, + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: Some(media_output_available && probe.h265_hdr), + hdr_color_qualities: Some(hdr_color_qualities(VideoCodec::H265)), + color_qualities: Some(color_qualities(VideoCodec::H265)), + codec: "h265", + available: media_output_available && probe.h265_hardware_decode, + reason: (!(media_output_available && probe.h265_hardware_decode)).then_some( + "H.265 Media Foundation hardware decode or media output is unavailable", + ), + }, + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: Some(media_output_available && probe.av1_hdr), + hdr_color_qualities: Some(hdr_color_qualities(VideoCodec::Av1)), + color_qualities: Some(color_qualities(VideoCodec::Av1)), + codec: "av1", + available: media_output_available && probe.av1_hardware_decode, + reason: (!(media_output_available && probe.av1_hardware_decode)).then_some( + "AV1 Media Foundation hardware decode or media output is unavailable", + ), + }, + ], + zero_copy_modes: available + .then_some(vec![zero_copy_mode]) + .unwrap_or_default(), + available, + reason, + } +} + +#[cfg(target_os = "macos")] +fn hardware_backend() -> VideoBackendCapability { + if !runtime::backend_preference_allows("videotoolbox") { + return unavailable_backend( + "videotoolbox", + "macos", + "VideoToolbox hardware decode was disabled by configuration", + ); + } + const UNAVAILABLE: &str = "VideoToolbox hardware decode or Metal is unavailable"; + let available = macos_backend::available(); + let h264_available = macos_backend::h264_available(); + let h265_available = macos_backend::h265_available(); + let av1_available = macos_backend::av1_available(); + VideoBackendCapability { + backend: "videotoolbox", + platform: "macos", + codecs: vec![ + CodecCapability { + max_decode_dimensions: None, + hdr_supported: Some(false), + hdr_color_qualities: None, + color_qualities: Some(vec!["8bit_420"]), + codec: "h264", + available: h264_available, + reason: (!h264_available) + .then_some("H.264 VideoToolbox hardware decode or Metal is unavailable"), + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: Some(false), + hdr_color_qualities: None, + color_qualities: Some(vec!["8bit_420", "10bit_420"]), + codec: "h265", + available: h265_available, + reason: (!h265_available) + .then_some("H.265 VideoToolbox hardware decode or Metal is unavailable"), + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: Some(false), + hdr_color_qualities: None, + color_qualities: Some(vec!["8bit_420", "10bit_420"]), + codec: "av1", + available: av1_available, + reason: (!av1_available) + .then_some("AV1 VideoToolbox hardware decode or Metal is unavailable"), + }, + ], + zero_copy_modes: available + .then_some(vec!["cvpixelbuffer-iosurface-metal"]) + .unwrap_or_default(), + available, + reason: (!available).then_some(UNAVAILABLE), + } +} + +#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] +fn hardware_backend() -> VideoBackendCapability { + unavailable_backend("unsupported", "other", "Unsupported operating system") +} + +fn software_backend() -> VideoBackendCapability { + #[cfg(target_os = "windows")] + { + let probe = opennow_streamer_platform_windows::WindowsBackend::probe_for( + opennow_streamer_platform_windows::WindowsGraphicsApi::D3d11, + None, + ); + let media_output_available = probe.d3d11_presentation && probe.wasapi_render; + // OpenH264 is bundled for the guaranteed H.264 path. HEVC and AV1 can + // additionally use a D3D11-aware synchronous Media Foundation decoder. + let available = true; + VideoBackendCapability { + backend: "software", + platform: "windows", + codecs: vec![ + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h264", + available: true, + reason: None, + }, + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h265", + available: media_output_available && probe.h265_software_decode, + reason: (!(media_output_available && probe.h265_software_decode)).then_some( + "H.265 Media Foundation software decode or media output is unavailable", + ), + }, + CodecCapability { + max_decode_dimensions: windows_decode_dimensions(), + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "av1", + available: media_output_available && probe.av1_software_decode, + reason: (!(media_output_available && probe.av1_software_decode)).then_some( + "AV1 Media Foundation software decode or media output is unavailable", + ), + }, + ], + zero_copy_modes: Vec::new(), + available, + reason: None, + } + } + #[cfg(not(target_os = "windows"))] + VideoBackendCapability { + backend: "software", + platform: "cross-platform", + codecs: vec![ + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h264", + available: true, + reason: None, + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h265", + available: false, + reason: Some("H.265 decoder is not built into this binary"), + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "av1", + available: false, + reason: Some("AV1 decoder is not built into this binary"), + }, + ], + zero_copy_modes: Vec::new(), + available: true, + reason: None, + } +} + +#[cfg(not(target_os = "linux"))] +fn unavailable_backend( + backend: &'static str, + platform: &'static str, + reason: &'static str, +) -> VideoBackendCapability { + VideoBackendCapability { + backend, + platform, + codecs: ["h264", "h265", "av1"] + .into_iter() + .map(|codec| CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec, + available: false, + reason: Some(reason), + }) + .collect(), + zero_copy_modes: Vec::new(), + available: false, + reason: Some(reason), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decoder_policy_separates_hardware_software_and_explicit_backends() { + assert!(backend_allowed_by_policy("auto", "software")); + assert!(backend_allowed_by_policy("software", "ffmpeg")); + assert!(backend_allowed_by_policy("software", "software")); + assert!(!backend_allowed_by_policy("software", "vulkan")); + assert!(backend_allowed_by_policy("hardware", "d3d11")); + assert!(!backend_allowed_by_policy("hardware", "software")); + assert!(!backend_allowed_by_policy("hardware", "ffmpeg")); + assert!(backend_allowed_by_policy("nvdec", "cuda")); + assert!(!backend_allowed_by_policy("nvdec", "vulkan")); + assert!(backend_allowed_by_policy("videotoolbox", "videotoolbox")); + } + + #[test] + fn disabled_backends_do_not_retain_hdr_or_color_profiles() { + let mut backends = vec![VideoBackendCapability { + backend: "vulkan", + platform: "linux", + codecs: vec![CodecCapability { + max_decode_dimensions: None, + codec: "h265", + available: true, + hdr_supported: Some(true), + hdr_color_qualities: None, + color_qualities: Some(vec!["8bit_420", "10bit_420"]), + reason: None, + }], + zero_copy_modes: vec!["vulkan"], + available: true, + reason: None, + }]; + apply_backend_policy(&mut backends, Some("software")); + assert!(!backends[0].available); + assert!(backends[0].zero_copy_modes.is_empty()); + assert!(!backends[0].codecs[0].available); + assert_eq!(backends[0].codecs[0].hdr_supported, Some(false)); + assert_eq!(backends[0].codecs[0].color_qualities, Some(Vec::new())); + } + + #[test] + fn windows_embedded_video_capabilities_do_not_require_standalone_audio() { + use opennow_streamer_platform_windows::CapabilityProbe; + + let mut probe = CapabilityProbe { + available: false, + h264_hardware_decode: true, + h265_hardware_decode: false, + av1_hardware_decode: false, + h265_hdr: false, + av1_hdr: false, + h265_10bit: false, + av1_10bit: false, + h265_444: false, + h265_10bit_444: false, + h265_hdr_444: false, + h264_software_decode: true, + h265_software_decode: false, + av1_software_decode: false, + d3d11_presentation: true, + wasapi_render: false, + reason: Some("WASAPI failed to start".to_owned()), + }; + let capability = |probe: &CapabilityProbe, mode| { + windows_hardware_capability(probe, "d3d11", "d3d11-nv12", mode) + }; + + let embedded = capability(&probe, WindowsOutputMode::Embedded); + assert!(embedded.available); + assert_eq!(embedded.reason, None); + assert_eq!(embedded.zero_copy_modes, vec!["d3d11-nv12"]); + assert!(embedded.codecs[0].available); + assert_eq!(embedded.codecs[0].reason, None); + assert_eq!(embedded.codecs[0].color_qualities, Some(vec!["8bit_420"])); + for codec in &embedded.codecs { + let limit = codec.max_decode_dimensions.as_ref().unwrap(); + assert_eq!((limit.width, limit.height), (4096, 2304)); + } + assert!(embedded.codecs[1..].iter().all(|codec| !codec.available)); + + let standalone = capability(&probe, WindowsOutputMode::Standalone); + assert!(!standalone.available); + assert!(standalone.zero_copy_modes.is_empty()); + assert!(standalone.codecs.iter().all(|codec| !codec.available)); + + probe.wasapi_render = true; + assert_eq!( + serde_json::to_value(capability(&probe, WindowsOutputMode::Embedded)).unwrap(), + serde_json::to_value(embedded).unwrap(), + ); + assert!(capability(&probe, WindowsOutputMode::Standalone).available); + + probe.wasapi_render = false; + probe.h265_hardware_decode = true; + probe.h265_10bit = true; + probe.h265_hdr = true; + let hdr = capability(&probe, WindowsOutputMode::Embedded); + assert!(hdr.codecs[1].available); + assert_eq!(hdr.codecs[1].hdr_supported, Some(true)); + assert_eq!( + hdr.codecs[1].color_qualities, + Some(vec!["8bit_420", "10bit_420"]) + ); + assert_eq!(hdr.codecs[1].hdr_color_qualities, Some(vec!["10bit_420"])); + assert!(!hdr.codecs[2].available); + + probe.d3d11_presentation = false; + for mode in [WindowsOutputMode::Embedded, WindowsOutputMode::Standalone] { + let unavailable = capability(&probe, mode); + assert!(!unavailable.available); + assert!(unavailable.reason.is_some()); + assert!(unavailable.zero_copy_modes.is_empty()); + for codec in unavailable.codecs { + assert!(!codec.available); + assert_eq!(codec.hdr_supported, Some(false)); + assert_eq!(codec.color_qualities, Some(vec![])); + assert_eq!(codec.hdr_color_qualities, Some(vec![])); + } + } + + probe.d3d11_presentation = true; + probe.h264_hardware_decode = false; + probe.h265_hardware_decode = false; + probe.h265_10bit = false; + probe.h265_hdr = false; + assert!(!capability(&probe, WindowsOutputMode::Embedded).available); + probe.av1_hardware_decode = true; + let av1 = capability(&probe, WindowsOutputMode::Embedded); + assert!(av1.available); + assert!(!av1.codecs[0].available); + assert!(av1.codecs[2].available); + } + + #[test] + fn advertises_only_the_linked_software_codec() { + let backends = video_backends(); + let software = backends + .iter() + .find(|backend| backend.backend == "software") + .expect("software backend"); + assert!(software.available); + assert!( + software + .codecs + .iter() + .find(|codec| codec.codec == "h264") + .expect("h264") + .available + ); + #[cfg(not(target_os = "windows"))] + assert!( + software + .codecs + .iter() + .filter(|codec| codec.codec != "h264") + .all(|codec| !codec.available) + ); + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + assert!( + backends + .iter() + .filter(|backend| backend.backend != "software") + .all(|backend| !backend.available) + ); + #[cfg(target_os = "windows")] + { + use opennow_streamer_platform_windows::WindowsGraphicsApi; + let software_probe = opennow_streamer_platform_windows::WindowsBackend::probe_for( + WindowsGraphicsApi::D3d11, + None, + ); + assert_eq!( + software + .codecs + .iter() + .find(|codec| codec.codec == "h265") + .expect("h265") + .available, + software_probe.h265_software_decode + && software_probe.d3d11_presentation + && software_probe.wasapi_render, + ); + assert_eq!( + software + .codecs + .iter() + .find(|codec| codec.codec == "av1") + .expect("av1") + .available, + software_probe.av1_software_decode + && software_probe.d3d11_presentation + && software_probe.wasapi_render, + ); + for (backend_name, api) in [ + ("d3d12", WindowsGraphicsApi::D3d12), + ("d3d11", WindowsGraphicsApi::D3d11), + ] { + let hardware = backends + .iter() + .find(|backend| backend.backend == backend_name) + .expect("Direct3D backend"); + let probe = opennow_streamer_platform_windows::WindowsBackend::probe_for(api, None); + assert_eq!(hardware.available, probe.bundled_backend_available()); + assert_eq!( + hardware + .codecs + .iter() + .find(|codec| codec.codec == "h265") + .expect("h265") + .available, + probe.h265_hardware_decode && probe.d3d11_presentation && probe.wasapi_render, + ); + assert_eq!( + hardware + .codecs + .iter() + .find(|codec| codec.codec == "av1") + .expect("av1") + .available, + probe.av1_hardware_decode && probe.d3d11_presentation && probe.wasapi_render, + ); + } + } + #[cfg(target_os = "macos")] + { + let hardware = backends + .iter() + .find(|backend| backend.backend == "videotoolbox") + .expect("VideoToolbox backend"); + for (codec, expected) in [ + ("h264", macos_backend::h264_available()), + ("h265", macos_backend::h265_available()), + ("av1", macos_backend::av1_available()), + ] { + assert_eq!( + hardware + .codecs + .iter() + .find(|capability| capability.codec == codec) + .expect(codec) + .available, + expected, + ); + } + assert_eq!(hardware.available, macos_backend::available()); + assert_eq!( + hardware.available, + hardware + .codecs + .iter() + .any(|capability| capability.available) + ); + } + } + + #[test] + fn embedded_capabilities_exclude_standalone_only_backends() { + let backends = embedded_video_backends(); + assert!(backends.iter().all(|backend| backend.backend != "software")); + #[cfg(target_os = "linux")] + { + let vulkan = backends + .iter() + .find(|backend| backend.backend == "vulkan") + .unwrap(); + assert!(!vulkan.available); + assert!(vulkan.codecs.iter().all(|codec| !codec.available)); + assert!(vulkan.zero_copy_modes.is_empty()); + } + #[cfg(target_os = "windows")] + assert_eq!( + backends + .iter() + .map(|backend| backend.backend) + .collect::>(), + vec!["d3d11"] + ); + #[cfg(target_os = "macos")] + assert_eq!( + backends + .iter() + .map(|backend| backend.backend) + .collect::>(), + vec!["videotoolbox"] + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn embedded_linux_hdr_requires_an_available_ten_bit_output_path() { + for backend in embedded_video_backends_with_config(None, None) { + for codec in backend.codecs { + assert_eq!( + codec.hdr_supported, + Some( + backend.available + && codec.available + && matches!(backend.backend, "vulkan" | "vaapi") + && matches!(codec.codec, "h265" | "av1") + && codec + .color_qualities + .as_ref() + .is_some_and(|colors| colors.contains(&"10bit_420")) + ) + ); + } + } + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_windows_advertises_only_probed_color_profiles() { + use opennow_streamer_platform_windows::{ + VideoCodec, VideoPixelFormat, WindowsBackend, WindowsGraphicsApi, + }; + let probe = WindowsBackend::probe_for(WindowsGraphicsApi::D3d11, None); + for backend in embedded_video_backends() { + for codec in backend.codecs { + let profile = match codec.codec { + "h264" => VideoCodec::H264, + "h265" => VideoCodec::H265, + _ => VideoCodec::Av1, + }; + let colors = codec.color_qualities.expect("explicit Windows profiles"); + for (color, format) in [ + ("8bit_420", VideoPixelFormat::Nv12), + ("10bit_420", VideoPixelFormat::P010), + ("8bit_444", VideoPixelFormat::Ayuv), + ("10bit_444", VideoPixelFormat::Y410), + ] { + assert_eq!( + colors.contains(&color), + codec.available && probe.supports_format(profile, format, false), + "{} {color}", + codec.codec + ); + } + } + } + } + + #[cfg(target_os = "linux")] + #[test] + fn embedded_linux_reports_explicit_color_profiles_without_claiming_zero_copy() { + for backend in embedded_video_backends_with_config(None, None) { + for codec in backend.codecs { + let colors = codec + .color_qualities + .expect("embedded Linux color profiles"); + match backend.backend { + "vulkan" => { + assert!(colors.is_empty()); + assert!(backend.zero_copy_modes.is_empty()); + } + "vaapi" => { + for color in &colors { + assert!( + matches!(*color, "8bit_420" | "10bit_420"), + "{} {color}", + codec.codec + ); + if *color == "10bit_420" { + assert!( + matches!(codec.codec, "h265" | "av1"), + "{} {color}", + codec.codec + ); + } + } + } + _ => assert!( + colors.iter().all(|color| *color == "8bit_420"), + "{} {colors:?}", + backend.backend + ), + } + } + } + } + + #[cfg(target_os = "linux")] + #[test] + fn vaapi_color_support_extends_the_advertised_profiles_in_wire_order() { + use opennow_streamer_platform_linux::VaapiColorSupport; + let support = |eight_bit_420, ten_bit_420| VaapiColorSupport { + eight_bit_420, + ten_bit_420, + }; + + let mut colors = Vec::new(); + assert!(!apply_vaapi_color_support( + &mut colors, + support(false, false) + )); + assert!(colors.is_empty()); + + assert!(apply_vaapi_color_support(&mut colors, support(true, false))); + assert_eq!(colors, ["8bit_420"]); + + let mut colors = Vec::new(); + assert!(apply_vaapi_color_support(&mut colors, support(false, true))); + assert_eq!(colors, ["10bit_420"]); + + let mut colors = Vec::new(); + assert!(apply_vaapi_color_support(&mut colors, support(true, true))); + assert_eq!(colors, ["8bit_420", "10bit_420"]); + } + + #[cfg(target_os = "macos")] + #[test] + fn embedded_macos_reports_only_biplanar_color_profiles() { + for backend in embedded_video_backends() { + for codec in backend.codecs { + let colors = codec + .color_qualities + .expect("embedded macOS color profiles"); + assert_eq!( + codec.hdr_supported, + Some( + codec.available + && codec.codec == "h265" + && opennow_streamer_platform_macos::probe_h265_hdr_hardware() + ) + ); + if !codec.available { + assert!(colors.is_empty()); + } else if codec.codec == "h264" { + assert_eq!(colors, ["8bit_420"]); + } else { + let mut expected = vec!["8bit_420", "10bit_420"]; + if codec.codec == "h265" + && opennow_streamer_platform_macos::probe_h265_444_ten_bit_hardware() + { + expected.push("10bit_444"); + } + assert_eq!(colors, expected); + } + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_backend.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_backend.rs new file mode 100644 index 000000000..746d890d4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_backend.rs @@ -0,0 +1,1136 @@ +use std::sync::OnceLock; + +use opennow_streamer_platform_linux::{ + BackendCapability, DecoderPreference, PresentationCapability, probe_video_capabilities, +}; +use opennow_streamer_protocol::{CodecCapability, VideoBackendCapability}; + +const VIDEO_BACKEND_ENV: &str = "OPENNOW_NATIVE_VIDEO_BACKEND"; +const WINDOW_SYSTEM_ENV: &str = "OPENNOW_NATIVE_WINDOW_SYSTEM"; +const CODECS: [&str; 3] = ["h264", "h265", "av1"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LinuxVideoPath { + Hardware(DecoderPreference), + Software, +} + +#[derive(Debug, Clone)] +pub(crate) struct LinuxVideoSelection { + pub(crate) path: LinuxVideoPath, + pub(crate) use_vulkan_output: bool, + pub(crate) fallback_reason: Option, +} + +#[derive(Debug, Clone)] +struct LinuxCapabilitySnapshot { + decoders: Vec, + presentation: PresentationCapability, +} + +impl LinuxCapabilitySnapshot { + fn probe() -> Self { + let (decoders, presentation) = probe_video_capabilities(); + Self { + decoders, + presentation, + } + } + + fn decoder(&self, name: &str) -> BackendCapability { + self.decoders + .iter() + .find(|capability| capability.name == name) + .cloned() + .unwrap_or(BackendCapability { + name: "missing-decoder-probe", + available: false, + detail: format!("capability probe did not return {name}"), + }) + } + + fn codec_decoder(&self, prefix: &str, codec: &str) -> BackendCapability { + self.decoder(&format!("{prefix}-{codec}")) + } + + fn backend_supports_all_codecs(&self, prefix: &str) -> bool { + CODECS + .iter() + .all(|codec| self.codec_decoder(prefix, codec).available) + } + + fn presentation_available(&self, window_system: &str) -> bool { + presentation_available(&self.presentation, window_system) + } +} + +fn presentation_available(presentation: &PresentationCapability, window_system: &str) -> bool { + presentation.available && presentation.window_systems.contains(&window_system) +} + +fn presentation_unavailable_reason( + presentation: &PresentationCapability, + window_system: &str, +) -> String { + if !presentation.available { + return presentation.detail.clone(); + } + format!( + "Vulkan presentation lacks {window_system} WSI required by the active Linux window system" + ) +} + +fn selected_window_system() -> &'static str { + selected_window_system_for( + std::env::var(WINDOW_SYSTEM_ENV).ok().as_deref(), + std::env::var("SDL_VIDEODRIVER").ok().as_deref(), + std::env::var_os("WAYLAND_DISPLAY").is_some(), + std::env::var_os("DISPLAY").is_some(), + ) +} + +fn selected_window_system_for( + requested: Option<&str>, + sdl_driver: Option<&str>, + has_wayland_display: bool, + has_x11_display: bool, +) -> &'static str { + for value in [requested, sdl_driver].into_iter().flatten() { + match value.trim().to_ascii_lowercase().as_str() { + "wayland" => return "wayland", + "x11" => return "x11", + _ => {} + } + } + if has_wayland_display && !has_x11_display { + "wayland" + } else { + "x11" + } +} + +fn capabilities() -> &'static LinuxCapabilitySnapshot { + static CAPABILITIES: OnceLock = OnceLock::new(); + CAPABILITIES.get_or_init(LinuxCapabilitySnapshot::probe) +} + +pub(crate) fn select_video_path() -> LinuxVideoSelection { + select_video_path_for( + std::env::var(VIDEO_BACKEND_ENV).ok().as_deref(), + capabilities(), + selected_window_system(), + ) +} + +pub(crate) fn select_embedded_video_path( + requested: &str, + device: Option<&crate::SharedVulkanDevice>, + stream: crate::MediaStreamConfig, +) -> LinuxVideoSelection { + let policy = std::env::var(VIDEO_BACKEND_ENV).unwrap_or_else(|_| "auto".to_owned()); + let requested = if requested == "auto" { + policy.as_str() + } else { + requested + }; + let requested = requested.trim().to_ascii_lowercase(); + let requested = if requested.is_empty() { + "auto" + } else { + &requested + }; + if requested == "mediacodec" { + return select_mediacodec(stream); + } + let codec = match stream.codec { + crate::MediaVideoCodec::H264 => opennow_streamer_platform_linux::VideoCodec::H264, + crate::MediaVideoCodec::H265 => opennow_streamer_platform_linux::VideoCodec::H265, + crate::MediaVideoCodec::Av1 => opennow_streamer_platform_linux::VideoCodec::Av1, + }; + if requested == "vulkan" + || (matches!(requested, "auto" | "hardware") + && device.is_some_and(|device| { + device.supports_format( + codec, + stream.color_quality.linux_pixel_format(), + stream.width, + stream.height, + ) + })) + { + return LinuxVideoSelection { + path: LinuxVideoPath::Hardware(DecoderPreference::VulkanOnly), + use_vulkan_output: true, + fallback_reason: None, + }; + } + select_embedded_fallback( + requested, + stream, + &crate::embedded_video_backends_with_config(device, None), + ) +} + +fn select_mediacodec(stream: crate::MediaStreamConfig) -> LinuxVideoSelection { + if stream.codec == crate::MediaVideoCodec::H264 + && stream.color_quality == crate::MediaColorQuality::EightBit420 + && !stream.hdr + && stream.width <= 2560 + && stream.height <= 1440 + { + return LinuxVideoSelection { + path: LinuxVideoPath::Hardware(DecoderPreference::MediaCodecOnly), + use_vulkan_output: true, + fallback_reason: None, + }; + } + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: Some( + "The opt-in Android MediaCodec bridge supports H.264 8-bit 4:2:0 SDR up to 2560x1440 only; it does not fall back to software.".to_owned(), + ), + } +} + +pub(crate) fn mediacodec_capability() -> Option { + let requested = std::env::var(VIDEO_BACKEND_ENV).unwrap_or_default(); + if requested.trim().eq_ignore_ascii_case("mediacodec") { + Some(mediacodec_capability_for( + opennow_streamer_platform_linux::probe_mediacodec().is_ok(), + )) + } else { + None + } +} + +fn mediacodec_capability_for(available: bool) -> VideoBackendCapability { + let reason = (!available).then_some( + "The opt-in Android hardware MediaCodec bridge did not answer a valid hardware probe. Check the DroidDeck session and OPENNOW_MEDIACODEC_SOCKET.", + ); + VideoBackendCapability { + backend: "mediacodec", + platform: "linux", + available, + reason, + zero_copy_modes: Vec::new(), + codecs: CODECS + .into_iter() + .map(|codec| { + let supported = codec == "h264"; + CodecCapability { + max_decode_dimensions: supported.then_some( + opennow_streamer_protocol::DecodeDimensions { + width: opennow_streamer_platform_linux::MEDIACODEC_MAX_WIDTH, + height: opennow_streamer_platform_linux::MEDIACODEC_MAX_HEIGHT, + }, + ), + codec, + available: available && supported, + hdr_supported: Some(false), + hdr_color_qualities: Some(Vec::new()), + color_qualities: Some(if available && supported { + vec!["8bit_420"] + } else { + Vec::new() + }), + reason: if supported { + reason + } else { + Some("The Android compatibility bridge currently supports H.264 only") + }, + } + }) + .collect(), + } +} + +fn select_embedded_fallback( + requested: &str, + stream: crate::MediaStreamConfig, + backends: &[VideoBackendCapability], +) -> LinuxVideoSelection { + if matches!(requested, "software" | "ffmpeg") { + return select_embedded_software(stream, backends); + } + if !stream.color_quality.is_444() { + let color = if stream.color_quality.bit_depth() == 10 { + "10bit_420" + } else { + "8bit_420" + }; + for (name, preference) in [ + ("cuda", DecoderPreference::CudaOnly), + ("vaapi", DecoderPreference::VaApiOnly), + ("v4l2", DecoderPreference::V4l2Only), + ] { + if name == "v4l2" && stream.hdr { + continue; + } + if stream.color_quality.bit_depth() == 10 && name != "vaapi" { + continue; + } + if !matches!(requested, "auto" | "hardware") + && requested != name + && !(requested == "nvdec" && name == "cuda") + { + continue; + } + if backends.iter().any(|backend| { + backend.backend == name + && backend.available + && backend.codecs.iter().any(|codec| { + codec.codec == stream.codec.label() + && codec.available + && codec + .color_qualities + .as_ref() + .map_or(color == "8bit_420", |qualities| qualities.contains(&color)) + }) + }) { + return LinuxVideoSelection { + path: LinuxVideoPath::Hardware( + if stream.color_quality.bit_depth() == 8 + && matches!(requested, "auto" | "hardware") + { + DecoderPreference::HardwareOnly + } else { + preference + }, + ), + use_vulkan_output: true, + fallback_reason: None, + }; + } + } + } + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: Some(format!( + "No compatible embedded {requested} decoder supports the negotiated {} color profile", + stream.codec.label(), + )), + } +} + +fn select_embedded_software( + stream: crate::MediaStreamConfig, + backends: &[VideoBackendCapability], +) -> LinuxVideoSelection { + let software = backends.iter().find(|backend| { + backend.backend == "ffmpeg" + && backend.available + && backend.codecs.iter().any(|codec| { + codec.codec == stream.codec.label() + && codec.available + && codec + .color_qualities + .as_ref() + .is_none_or(|qualities| qualities.contains(&"8bit_420")) + }) + }); + let reason = if stream.hdr || stream.color_quality != crate::MediaColorQuality::EightBit420 { + format!( + "Software decoding presents 8-bit 4:2:0 SDR only; the negotiated {} {} profile cannot be preserved", + stream.codec.label(), + stream.color_quality.protocol_name(), + ) + } else if software.is_none() { + format!( + "The FFmpeg software decoder is unavailable for the negotiated {} stream. Select Auto in Stream settings or export diagnostics for backend probe failures.", + stream.codec.label(), + ) + } else { + return LinuxVideoSelection { + path: LinuxVideoPath::Hardware(DecoderPreference::SoftwareOnly), + use_vulkan_output: true, + fallback_reason: None, + }; + }; + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: Some(reason), + } +} + +fn select_video_path_for( + requested: Option<&str>, + capabilities: &LinuxCapabilitySnapshot, + window_system: &str, +) -> LinuxVideoSelection { + let requested = requested + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("auto") + .to_ascii_lowercase(); + let presentation = capabilities.presentation_available(window_system); + let vulkan = capabilities.backend_supports_all_codecs("vulkan-video"); + let cuda = capabilities.backend_supports_all_codecs("cuda"); + let ffmpeg = capabilities.backend_supports_all_codecs("ffmpeg-software"); + let vaapi = capabilities.decoder("vaapi-h264").available; + let v4l2 = capabilities.decoder("v4l2-h264").available; + let hardware = vulkan || cuda || vaapi || v4l2; + + let preference = match requested.as_str() { + "vulkan" if vulkan => Some(DecoderPreference::VulkanOnly), + "cuda" | "nvdec" if cuda => Some(DecoderPreference::CudaOnly), + "vaapi" if vaapi => Some(DecoderPreference::VaApiOnly), + "v4l2" if v4l2 => Some(DecoderPreference::V4l2Only), + "software" | "ffmpeg" if ffmpeg => Some(DecoderPreference::SoftwareOnly), + "hardware" if hardware => Some(DecoderPreference::HardwareOnly), + "auto" if vulkan || cuda || vaapi || v4l2 || ffmpeg => Some(DecoderPreference::Automatic), + _ => None, + }; + if let Some(preference) = preference { + return LinuxVideoSelection { + path: LinuxVideoPath::Hardware(preference), + use_vulkan_output: presentation, + fallback_reason: (!presentation).then(|| { + format!( + "{}; using SDL NV12 presentation", + presentation_unavailable_reason(&capabilities.presentation, window_system) + ) + }), + }; + } + + let reason = match requested.as_str() { + "vulkan" => backend_unavailable_reason(capabilities, "vulkan-video"), + "cuda" | "nvdec" => backend_unavailable_reason(capabilities, "cuda"), + "vaapi" => capabilities.decoder("vaapi-h264").detail, + "v4l2" => capabilities.decoder("v4l2-h264").detail, + "software" | "ffmpeg" => backend_unavailable_reason(capabilities, "ffmpeg-software"), + "hardware" => format!( + "no Linux hardware decoder is available (Vulkan Video: {}; CUDA/NVDEC: {}; VA-API: {}; V4L2: {})", + backend_unavailable_reason(capabilities, "vulkan-video"), + backend_unavailable_reason(capabilities, "cuda"), + capabilities.decoder("vaapi-h264").detail, + capabilities.decoder("v4l2-h264").detail, + ), + "auto" => format!( + "no Linux decoder is available (Vulkan Video: {}; CUDA/NVDEC: {}; FFmpeg software: {})", + backend_unavailable_reason(capabilities, "vulkan-video"), + backend_unavailable_reason(capabilities, "cuda"), + backend_unavailable_reason(capabilities, "ffmpeg-software"), + ), + other => format!("video backend {other:?} is not supported on Linux"), + }; + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: Some(format!("{reason}; using OpenH264/SDL fallback")), + } +} + +fn backend_unavailable_reason(capabilities: &LinuxCapabilitySnapshot, prefix: &str) -> String { + CODECS + .iter() + .filter_map(|codec| { + let capability = capabilities.codec_decoder(prefix, codec); + (!capability.available).then(|| format!("{codec}: {}", capability.detail)) + }) + .collect::>() + .join(", ") +} + +pub(crate) fn video_backends() -> Vec { + let capabilities = capabilities(); + let window_system = selected_window_system(); + vec![ + multi_codec_capability("vulkan", "vulkan-video", capabilities, window_system, false), + multi_codec_capability("cuda", "cuda", capabilities, window_system, false), + h264_capability("vaapi", "vaapi-h264", capabilities, window_system), + v4l2_capability(capabilities, window_system), + multi_codec_capability( + "ffmpeg", + "ffmpeg-software", + capabilities, + window_system, + false, + ), + ] +} + +fn v4l2_capability( + capabilities: &LinuxCapabilitySnapshot, + window_system: &str, +) -> VideoBackendCapability { + let mut backend = h264_capability("v4l2", "v4l2-h264", capabilities, window_system); + let decoder = capabilities.decoder("v4l2-h265"); + let presentation = capabilities.presentation_available(window_system); + let codec = backend + .codecs + .iter_mut() + .find(|codec| codec.codec == "h265") + .unwrap(); + codec.available = decoder.available && presentation; + codec.color_qualities = Some(if codec.available { + vec!["8bit_420"] + } else { + Vec::new() + }); + codec.hdr_supported = Some(false); + codec.reason = if !decoder.available { + Some(static_reason(decoder.detail)) + } else if !presentation { + Some(static_reason(presentation_unavailable_reason( + &capabilities.presentation, + window_system, + ))) + } else { + None + }; + backend.available = backend.codecs.iter().any(|codec| codec.available); + if backend.available { + backend.reason = None; + } else { + backend.reason = Some(static_reason( + backend + .codecs + .iter() + .filter(|codec| matches!(codec.codec, "h264" | "h265")) + .filter_map(|codec| { + codec + .reason + .map(|reason| format!("{}: {reason}", codec.codec)) + }) + .collect::>() + .join("; "), + )); + } + backend +} + +fn multi_codec_capability( + backend: &'static str, + prefix: &str, + capabilities: &LinuxCapabilitySnapshot, + window_system: &str, + requires_presentation: bool, +) -> VideoBackendCapability { + let presentation = !requires_presentation || capabilities.presentation_available(window_system); + let codecs = CODECS + .into_iter() + .map(|codec| { + let decoder = capabilities.codec_decoder(prefix, codec); + let available = decoder.available && presentation; + let reason = if !decoder.available { + Some(static_reason(decoder.detail)) + } else if !presentation { + Some(static_reason(presentation_unavailable_reason( + &capabilities.presentation, + window_system, + ))) + } else { + None + }; + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec, + available, + reason, + } + }) + .collect::>(); + let available = codecs.iter().any(|codec| codec.available); + let reason = (!available).then(|| { + codecs + .iter() + .find_map(|codec| codec.reason) + .unwrap_or("no supported codecs") + }); + VideoBackendCapability { + backend, + platform: "linux", + codecs, + zero_copy_modes: (backend == "vulkan" + && available + && capabilities.presentation_available(window_system)) + .then_some(vec!["vulkan-video-same-device-nv12"]) + .unwrap_or_default(), + available, + reason, + } +} + +fn h264_capability( + backend: &'static str, + decoder_name: &str, + capabilities: &LinuxCapabilitySnapshot, + window_system: &str, +) -> VideoBackendCapability { + let decoder = capabilities.decoder(decoder_name); + let h264_available = decoder.available; + let h264_reason = if !decoder.available { + Some(static_reason(decoder.detail)) + } else { + None + }; + VideoBackendCapability { + backend, + platform: "linux", + codecs: vec![ + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h264", + available: h264_available, + reason: h264_reason, + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "h265", + available: false, + reason: Some("native backend currently supports H.264 only"), + }, + CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + hdr_color_qualities: None, + color_qualities: None, + codec: "av1", + available: false, + reason: Some("native backend currently supports H.264 only"), + }, + ], + zero_copy_modes: (backend == "vaapi" + && h264_available + && capabilities.presentation_available(window_system)) + .then_some(vec!["vaapi-drm-prime-dmabuf-vulkan"]) + .unwrap_or_default(), + available: h264_available, + reason: h264_reason, + } +} + +fn static_reason(reason: String) -> &'static str { + Box::leak(reason.into_boxed_str()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mediacodec_selection_is_exact_and_rejects_unsupported_profiles() { + let stream = crate::MediaStreamConfig::default(); + let selected = select_mediacodec(stream); + assert_eq!( + selected.path, + LinuxVideoPath::Hardware(DecoderPreference::MediaCodecOnly) + ); + assert!(selected.use_vulkan_output); + assert!(selected.fallback_reason.is_none()); + for unsupported in [ + crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + ..stream + }, + crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::Av1, + ..stream + }, + crate::MediaStreamConfig { + color_quality: crate::MediaColorQuality::TenBit420, + ..stream + }, + crate::MediaStreamConfig { + color_quality: crate::MediaColorQuality::EightBit444, + ..stream + }, + crate::MediaStreamConfig { + hdr: true, + ..stream + }, + crate::MediaStreamConfig { + width: 3840, + height: 2160, + ..stream + }, + ] { + let rejected = select_mediacodec(unsupported); + assert_eq!(rejected.path, LinuxVideoPath::Software); + assert!(!rejected.use_vulkan_output); + assert!( + rejected + .fallback_reason + .unwrap() + .contains("does not fall back to software") + ); + } + } + + #[test] + fn mediacodec_capabilities_require_a_successful_hardware_probe() { + let missing = mediacodec_capability_for(false); + assert!(!missing.available); + assert!(missing.codecs.iter().all(|codec| !codec.available)); + let available = mediacodec_capability_for(true); + assert!(available.available); + assert!(available.zero_copy_modes.is_empty()); + for codec in available.codecs { + assert_eq!(codec.available, codec.codec == "h264"); + assert_eq!( + codec + .max_decode_dimensions + .as_ref() + .map(|limit| (limit.width, limit.height)), + (codec.codec == "h264").then_some((2560, 1440)) + ); + assert_eq!(codec.hdr_supported, Some(false)); + assert_eq!( + codec.color_qualities, + Some(if codec.codec == "h264" { + vec!["8bit_420"] + } else { + Vec::new() + }) + ); + } + } + + #[test] + fn unavailable_v4l2_summary_preserves_the_hevc_failure() { + let mut capabilities = snapshot(false, false, false, true, vec!["wayland"]); + capabilities.decoders.push(BackendCapability { + name: "v4l2-h265", + available: false, + detail: "MEDIA_IOC_REQUEST_ALLOC failed: Permission denied (os error 13)".to_owned(), + }); + let backend = v4l2_capability(&capabilities, "wayland"); + assert!(!backend.available); + let reason = backend.reason.unwrap(); + assert!(reason.contains("h264:")); + assert!(reason.contains("h265: MEDIA_IOC_REQUEST_ALLOC failed")); + assert!(reason.contains("os error 13")); + } + + #[test] + fn hevc_request_is_available_without_stateful_h264_and_never_ten_bit() { + let mut capabilities = snapshot(false, false, false, true, vec!["wayland"]); + capabilities.decoders.push(capability("v4l2-h265", true)); + let backend = v4l2_capability(&capabilities, "wayland"); + assert!(backend.available); + assert!( + !backend + .codecs + .iter() + .find(|codec| codec.codec == "h264") + .unwrap() + .available + ); + let codec = backend + .codecs + .iter() + .find(|codec| codec.codec == "h265") + .unwrap(); + assert!(codec.available); + assert_eq!(codec.color_qualities, Some(vec!["8bit_420"])); + assert_eq!(codec.hdr_supported, Some(false)); + let mut stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + color_quality: crate::MediaColorQuality::EightBit420, + ..Default::default() + }; + assert_eq!( + select_embedded_fallback("v4l2", stream, std::slice::from_ref(&backend)).path, + LinuxVideoPath::Hardware(DecoderPreference::V4l2Only) + ); + stream.color_quality = crate::MediaColorQuality::TenBit420; + assert_eq!( + select_embedded_fallback("v4l2", stream, std::slice::from_ref(&backend)).path, + LinuxVideoPath::Software + ); + stream.color_quality = crate::MediaColorQuality::EightBit420; + stream.hdr = true; + assert_eq!( + select_embedded_fallback("v4l2", stream, &[backend]).path, + LinuxVideoPath::Software + ); + assert!(!v4l2_capability(&capabilities, "x11").available); + } + + #[test] + fn ten_bit_fallback_uses_only_explicit_vaapi_profiles() { + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + color_quality: crate::MediaColorQuality::TenBit420, + hdr: true, + ..Default::default() + }; + let mut backend = VideoBackendCapability { + backend: "vaapi", + platform: "linux", + codecs: vec![CodecCapability { + max_decode_dimensions: None, + codec: "h265", + available: true, + hdr_color_qualities: None, + color_qualities: Some(vec!["8bit_420", "10bit_420"]), + hdr_supported: None, + reason: None, + }], + zero_copy_modes: vec![], + available: true, + reason: None, + }; + for requested in ["auto", "hardware", "vaapi"] { + assert_eq!( + select_embedded_fallback(requested, stream, &[backend.clone()]).path, + LinuxVideoPath::Hardware(DecoderPreference::VaApiOnly) + ); + } + for name in ["cuda", "v4l2", "ffmpeg"] { + backend.backend = name; + assert_eq!( + select_embedded_fallback("auto", stream, &[backend.clone()]).path, + LinuxVideoPath::Software + ); + } + backend.backend = "vaapi"; + for qualities in [None, Some(vec![]), Some(vec!["8bit_420"])] { + backend.codecs[0].color_qualities = qualities; + assert_eq!( + select_embedded_fallback("auto", stream, &[backend.clone()]).path, + LinuxVideoPath::Software + ); + } + } + + #[test] + fn embedded_fallback_uses_the_selected_codec_not_all_codec_support() { + let capabilities = snapshot(true, false, true, true, vec!["x11"]); + let mut cuda = multi_codec_capability("cuda", "cuda", &capabilities, "x11", false); + cuda.available = true; + cuda.codecs + .iter_mut() + .find(|codec| codec.codec == "h265") + .unwrap() + .available = true; + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + ..Default::default() + }; + for (requested, preference) in [ + ("auto", DecoderPreference::HardwareOnly), + ("hardware", DecoderPreference::HardwareOnly), + ("cuda", DecoderPreference::CudaOnly), + ("nvdec", DecoderPreference::CudaOnly), + ] { + assert_eq!( + select_embedded_fallback(requested, stream, &[cuda.clone()]).path, + LinuxVideoPath::Hardware(preference), + ); + } + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::Av1, + ..stream + }; + assert_eq!( + select_embedded_fallback("auto", stream, &[cuda]).path, + LinuxVideoPath::Software + ); + } + + #[test] + fn embedded_fallback_never_reselects_standalone_vulkan_or_software() { + let capabilities = snapshot(true, false, true, true, vec!["x11"]); + let mut backends = vec![ + multi_codec_capability("vulkan", "vulkan-video", &capabilities, "x11", false), + multi_codec_capability("ffmpeg", "ffmpeg-software", &capabilities, "x11", false), + h264_capability("vaapi", "vaapi-h264", &capabilities, "x11"), + ]; + let stream = crate::MediaStreamConfig::default(); + let unavailable = select_embedded_fallback("auto", stream, &backends); + assert_eq!(unavailable.path, LinuxVideoPath::Software); + assert!( + unavailable + .fallback_reason + .unwrap() + .contains("No compatible embedded") + ); + backends[2].available = true; + backends[2].codecs[0].available = true; + assert_eq!( + select_embedded_fallback("auto", stream, &backends).path, + LinuxVideoPath::Hardware(DecoderPreference::HardwareOnly), + ); + for color_quality in [ + crate::MediaColorQuality::TenBit420, + crate::MediaColorQuality::EightBit444, + ] { + assert_eq!( + select_embedded_fallback( + "auto", + crate::MediaStreamConfig { + color_quality, + ..stream + }, + &backends + ) + .path, + LinuxVideoPath::Software, + ); + } + } + + fn software_backend( + codec: &'static str, + color_qualities: Vec<&'static str>, + ) -> VideoBackendCapability { + VideoBackendCapability { + backend: "ffmpeg", + platform: "linux", + codecs: vec![CodecCapability { + max_decode_dimensions: None, + codec, + available: true, + hdr_color_qualities: None, + color_qualities: Some(color_qualities), + hdr_supported: Some(false), + reason: None, + }], + zero_copy_modes: vec![], + available: true, + reason: None, + } + } + + #[test] + fn explicit_software_request_selects_the_bounded_cpu_decoder() { + for requested in ["software", "ffmpeg"] { + let selected = select_embedded_fallback( + requested, + crate::MediaStreamConfig::default(), + &[software_backend("h264", vec!["8bit_420"])], + ); + assert_eq!( + selected.path, + LinuxVideoPath::Hardware(DecoderPreference::SoftwareOnly) + ); + assert!(selected.use_vulkan_output); + assert!(selected.fallback_reason.is_none()); + } + for codec in ["h264", "h265", "av1"] { + let stream = crate::MediaStreamConfig { + codec: match codec { + "h265" => crate::MediaVideoCodec::H265, + "av1" => crate::MediaVideoCodec::Av1, + _ => crate::MediaVideoCodec::H264, + }, + ..crate::MediaStreamConfig::default() + }; + assert_eq!( + select_embedded_fallback( + "software", + stream, + &[software_backend(codec, vec!["8bit_420"])] + ) + .path, + LinuxVideoPath::Hardware(DecoderPreference::SoftwareOnly), + "{codec}" + ); + } + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::Av1, + ..crate::MediaStreamConfig::default() + }; + for qualities in [vec![], vec!["10bit_420"]] { + let selected = + select_embedded_fallback("software", stream, &[software_backend("av1", qualities)]); + assert_eq!(selected.path, LinuxVideoPath::Software); + assert!( + selected + .fallback_reason + .unwrap() + .contains("FFmpeg software decoder is unavailable") + ); + } + } + + #[test] + fn explicit_software_request_refuses_profiles_the_cpu_path_cannot_preserve() { + let backends = [software_backend("h265", vec!["8bit_420"])]; + for (color_quality, hdr) in [ + (crate::MediaColorQuality::EightBit420, true), + (crate::MediaColorQuality::TenBit420, false), + (crate::MediaColorQuality::EightBit444, false), + (crate::MediaColorQuality::TenBit444, false), + ] { + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + color_quality, + hdr, + ..crate::MediaStreamConfig::default() + }; + let selected = select_embedded_fallback("software", stream, &backends); + assert_eq!(selected.path, LinuxVideoPath::Software); + assert!(!selected.use_vulkan_output); + let reason = selected.fallback_reason.unwrap(); + assert!( + reason.contains("Software decoding presents 8-bit 4:2:0 SDR only"), + "{color_quality:?} hdr={hdr}: {reason}" + ); + assert!(reason.contains(color_quality.protocol_name())); + } + } + + #[test] + fn automatic_selection_never_substitutes_software_for_hardware() { + let backends = [software_backend("h264", vec!["8bit_420"])]; + for requested in ["auto", "hardware"] { + let selected = + select_embedded_fallback(requested, crate::MediaStreamConfig::default(), &backends); + assert_eq!(selected.path, LinuxVideoPath::Software, "{requested}"); + assert!( + selected + .fallback_reason + .unwrap() + .contains("No compatible embedded") + ); + } + } + + fn capability(name: &'static str, available: bool) -> BackendCapability { + BackendCapability { + name, + available, + detail: format!("{name} probe"), + } + } + + fn snapshot( + vulkan_video: bool, + cuda: bool, + software: bool, + presentation: bool, + window_systems: Vec<&'static str>, + ) -> LinuxCapabilitySnapshot { + let mut decoders = Vec::new(); + for codec in CODECS { + decoders.push(capability( + match codec { + "h264" => "vulkan-video-h264", + "h265" => "vulkan-video-h265", + _ => "vulkan-video-av1", + }, + vulkan_video, + )); + decoders.push(capability( + match codec { + "h264" => "cuda-h264", + "h265" => "cuda-h265", + _ => "cuda-av1", + }, + cuda, + )); + decoders.push(capability( + match codec { + "h264" => "ffmpeg-software-h264", + "h265" => "ffmpeg-software-h265", + _ => "ffmpeg-software-av1", + }, + software, + )); + } + decoders.push(capability("vaapi-h264", false)); + decoders.push(capability("v4l2-h264", false)); + LinuxCapabilitySnapshot { + decoders, + presentation: PresentationCapability { + available: presentation, + api: "vulkan", + window_systems, + detail: "vulkan presentation probe".to_owned(), + }, + } + } + + #[test] + fn explicit_backends_select_distinct_decoders() { + let available = snapshot(true, true, true, true, vec!["x11"]); + assert_eq!( + select_video_path_for(Some("vulkan"), &available, "x11").path, + LinuxVideoPath::Hardware(DecoderPreference::VulkanOnly) + ); + assert_eq!( + select_video_path_for(Some("nvdec"), &available, "x11").path, + LinuxVideoPath::Hardware(DecoderPreference::CudaOnly) + ); + assert_eq!( + select_video_path_for(Some("software"), &available, "x11").path, + LinuxVideoPath::Hardware(DecoderPreference::SoftwareOnly) + ); + assert_eq!( + select_video_path_for(Some("hardware"), &available, "x11").path, + LinuxVideoPath::Hardware(DecoderPreference::HardwareOnly) + ); + } + + #[test] + fn automatic_selection_uses_the_complete_codec_pipeline() { + let available = snapshot(true, true, true, true, vec!["wayland"]); + assert_eq!( + select_video_path_for(None, &available, "wayland").path, + LinuxVideoPath::Hardware(DecoderPreference::Automatic) + ); + } + + #[test] + fn presentation_must_match_the_active_window_system() { + let available = snapshot(true, true, true, true, vec!["wayland"]); + let selected = select_video_path_for(None, &available, "x11"); + assert_eq!( + selected.path, + LinuxVideoPath::Hardware(DecoderPreference::Automatic) + ); + assert!(!selected.use_vulkan_output); + assert!( + selected + .fallback_reason + .as_deref() + .is_some_and(|reason| reason.contains("lacks x11 WSI")) + ); + } + + #[test] + fn partial_codec_support_is_not_treated_as_a_complete_explicit_backend() { + let mut available = snapshot(true, true, true, true, vec!["x11"]); + available + .decoders + .iter_mut() + .find(|capability| capability.name == "vulkan-video-av1") + .unwrap() + .available = false; + let selected = select_video_path_for(Some("vulkan"), &available, "x11"); + assert_eq!(selected.path, LinuxVideoPath::Software); + assert!(selected.fallback_reason.unwrap().contains("av1")); + } + + #[test] + fn window_system_selection_prefers_explicit_runtime_contract() { + assert_eq!( + selected_window_system_for(Some("wayland"), Some("x11"), false, true), + "wayland" + ); + assert_eq!( + selected_window_system_for(None, Some("wayland"), true, true), + "wayland" + ); + assert_eq!( + selected_window_system_for(None, None, true, false), + "wayland" + ); + assert_eq!(selected_window_system_for(None, None, true, true), "x11"); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_frame_pacing.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_frame_pacing.rs new file mode 100644 index 000000000..26e12a726 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_frame_pacing.rs @@ -0,0 +1,333 @@ +use std::time::{Duration, Instant}; + +const PACING_OUTLIER: Duration = Duration::from_millis(75); +const MAX_ADAPTIVE_QUEUE_DEPTH: usize = 2; +const JITTER_QUEUE_THRESHOLD: f64 = 0.25; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FrameSelectionPolicy { + OldestReady, + LatestReady, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct PacingDecision { + pub(crate) selection: FrameSelectionPolicy, + pub(crate) target_queue_depth: usize, +} + +/// Display-clocked frame scheduler modelled after the native GFN client's +/// timestamp/adaptive queue. Decode is allowed to run independently, but only +/// one selected frame is submitted to the FIFO swapchain per presentation +/// interval. Fast streams deliberately sample the newest decoded frame rather +/// than building a backlog that WSI must discard later. +#[derive(Debug)] +pub(crate) struct LinuxFramePacer { + stream_interval: Duration, + display_interval: Option, + presentation_interval: Duration, + fast_stream: bool, + framerate_mismatch: bool, + vrr_enabled: bool, + last_arrival: Option<(Instant, u64)>, + jitter_ema_ns: f64, + jitter_bound_ns: f64, + next_deadline: Option, +} + +impl LinuxFramePacer { + pub(crate) fn new(stream_fps: u32, display_refresh_hz: Option, vrr_enabled: bool) -> Self { + let stream_interval = frame_interval(stream_fps); + let display_interval = display_refresh_hz.map(frame_interval); + let (presentation_interval, fast_stream, framerate_mismatch) = + pacing_mode(stream_interval, display_interval); + Self { + stream_interval, + display_interval, + presentation_interval, + fast_stream, + framerate_mismatch, + vrr_enabled, + last_arrival: None, + jitter_ema_ns: 0.0, + jitter_bound_ns: 0.0, + next_deadline: None, + } + } + + pub(crate) fn reconfigure(&mut self, stream_fps: u32, display_refresh_hz: Option) -> bool { + let stream_interval = frame_interval(stream_fps); + let display_interval = display_refresh_hz.map(frame_interval); + if self.stream_interval == stream_interval && self.display_interval == display_interval { + return false; + } + self.stream_interval = stream_interval; + self.display_interval = display_interval; + ( + self.presentation_interval, + self.fast_stream, + self.framerate_mismatch, + ) = pacing_mode(stream_interval, display_interval); + self.reset(); + true + } + + pub(crate) fn reset(&mut self) { + self.last_arrival = None; + self.jitter_ema_ns = 0.0; + self.jitter_bound_ns = 0.0; + self.next_deadline = None; + } + + pub(crate) fn is_due(&self, now: Instant) -> bool { + if self.vrr_enabled { + return true; + } + self.next_deadline.is_none_or(|deadline| now >= deadline) + } + + pub(crate) fn decision(&self, now: Instant) -> PacingDecision { + // With compositor VRR enabled, submit the freshest complete server + // frame immediately. FIFO provides tear-free scanout while the output + // refresh follows server arrival timing; replaying a burst after a + // hitch would defeat Cloud G-SYNC and add input latency. + if self.vrr_enabled { + return PacingDecision { + selection: FrameSelectionPolicy::LatestReady, + target_queue_depth: 0, + }; + } + let stalled = self + .next_deadline + .is_some_and(|deadline| now.saturating_duration_since(deadline) > PACING_OUTLIER); + if self.fast_stream || self.framerate_mismatch || stalled { + return PacingDecision { + selection: FrameSelectionPolicy::LatestReady, + target_queue_depth: 0, + }; + } + PacingDecision { + selection: FrameSelectionPolicy::OldestReady, + target_queue_depth: self.adaptive_queue_depth(), + } + } + + pub(crate) fn observe_frame(&mut self, arrived_at: Instant, timestamp_us: u64) { + let Some((previous_arrival, previous_timestamp_us)) = + self.last_arrival.replace((arrived_at, timestamp_us)) + else { + return; + }; + let arrival_interval = arrived_at.saturating_duration_since(previous_arrival); + let Some(timestamp_delta_us) = timestamp_us.checked_sub(previous_timestamp_us) else { + self.clear_timing_history(); + return; + }; + let media_interval = Duration::from_micros(timestamp_delta_us); + if arrival_interval.is_zero() + || media_interval.is_zero() + || arrival_interval > PACING_OUTLIER + || media_interval > PACING_OUTLIER + { + self.clear_timing_history(); + return; + } + + let jitter_ns = arrival_interval.abs_diff(media_interval).as_nanos() as f64; + self.jitter_ema_ns = if self.jitter_ema_ns == 0.0 { + jitter_ns + } else { + self.jitter_ema_ns * 0.90 + jitter_ns * 0.10 + }; + // Keep a decaying bound so a transient burst is absorbed, but does not + // leave the stream permanently buffered after conditions recover. + self.jitter_bound_ns = (self.jitter_bound_ns * 0.98).max(jitter_ns); + } + + pub(crate) fn mark_presented(&mut self, now: Instant) { + if self.vrr_enabled { + self.next_deadline = None; + return; + } + let next = self + .next_deadline + .map_or(now + self.presentation_interval, |deadline| { + deadline + self.presentation_interval + }); + // Never replay missed deadlines. A hitch rebases the presentation + // clock and stale frames are handled by `LatestReady` on the next tick. + self.next_deadline = Some(if next <= now { + now + self.presentation_interval + } else { + next + }); + } + + pub(crate) const fn fast_stream(&self) -> bool { + self.fast_stream + } + + pub(crate) const fn vrr_enabled(&self) -> bool { + self.vrr_enabled + } + + pub(crate) fn presentation_hz(&self) -> f64 { + 1.0 / self.presentation_interval.as_secs_f64() + } + + fn adaptive_queue_depth(&self) -> usize { + if self.jitter_bound_ns == 0.0 { + return 0; + } + let interval_ns = self.presentation_interval.as_nanos() as f64; + let target_ns = (self.jitter_ema_ns + self.jitter_bound_ns * 0.5) * 1.25; + if target_ns < interval_ns * JITTER_QUEUE_THRESHOLD { + return 0; + } + ((target_ns / interval_ns).ceil() as usize).min(MAX_ADAPTIVE_QUEUE_DEPTH) + } + + fn clear_timing_history(&mut self) { + self.last_arrival = None; + self.jitter_ema_ns = 0.0; + self.jitter_bound_ns = 0.0; + self.next_deadline = None; + } +} + +fn frame_interval(fps: u32) -> Duration { + Duration::from_secs_f64(1.0 / f64::from(fps.max(1))) +} + +fn pacing_mode( + stream_interval: Duration, + display_interval: Option, +) -> (Duration, bool, bool) { + let Some(display_interval) = display_interval else { + return (stream_interval, false, false); + }; + let stream_ns = stream_interval.as_nanos(); + let display_ns = display_interval.as_nanos(); + let fast_stream = 3 * stream_ns < 2 * display_ns; + let framerate_mismatch = 4 * stream_ns < 3 * display_ns || 2 * stream_ns > 3 * display_ns; + ( + stream_interval.max(display_interval), + fast_stream, + framerate_mismatch, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fast_stream_samples_latest_at_display_rate() { + let pacer = LinuxFramePacer::new(360, Some(165), false); + assert!(pacer.fast_stream()); + assert_eq!( + pacer.decision(Instant::now()).selection, + FrameSelectionPolicy::LatestReady + ); + assert!((pacer.presentation_hz() - 165.0).abs() < 0.01); + } + + #[test] + fn matched_stream_uses_ordered_adaptive_queue() { + let pacer = LinuxFramePacer::new(120, Some(165), false); + assert!(!pacer.fast_stream()); + assert_eq!( + pacer.decision(Instant::now()), + PacingDecision { + selection: FrameSelectionPolicy::OldestReady, + target_queue_depth: 0, + } + ); + assert!((pacer.presentation_hz() - 120.0).abs() < 0.01); + } + + #[test] + fn fast_stream_and_mismatch_boundaries_are_strict() { + let display = Duration::from_millis(120); + let ns = Duration::from_nanos(1); + let cases = [ + (Duration::from_millis(80) - ns, true, true), + (Duration::from_millis(80), false, true), + (Duration::from_millis(80) + ns, false, true), + (Duration::from_millis(90) - ns, false, true), + (Duration::from_millis(90), false, false), + (Duration::from_millis(90) + ns, false, false), + (Duration::from_millis(180) - ns, false, false), + (Duration::from_millis(180), false, false), + (Duration::from_millis(180) + ns, false, true), + ]; + for (stream, fast, mismatch) in cases { + assert_eq!( + pacing_mode(stream, Some(display)), + (stream.max(display), fast, mismatch) + ); + } + assert_eq!(pacing_mode(display, None), (display, false, false)); + } + + #[test] + fn framerate_mismatch_samples_latest_without_changing_clock() { + for (stream_fps, display_hz) in [(240, 165), (170, 120), (60, 120)] { + let pacer = LinuxFramePacer::new(stream_fps, Some(display_hz), false); + assert!(!pacer.fast_stream()); + assert_eq!( + pacer.decision(Instant::now()), + PacingDecision { + selection: FrameSelectionPolicy::LatestReady, + target_queue_depth: 0, + } + ); + assert!((pacer.presentation_hz() - f64::from(stream_fps.min(display_hz))).abs() < 0.01); + } + } + + #[test] + fn missed_deadline_is_rebased_without_catch_up_burst() { + let mut pacer = LinuxFramePacer::new(120, Some(120), false); + let origin = Instant::now(); + pacer.mark_presented(origin); + let late = origin + Duration::from_millis(20); + pacer.mark_presented(late); + assert!(!pacer.is_due(late)); + assert!(pacer.is_due(late + Duration::from_millis(9))); + } + + #[test] + fn sustained_arrival_jitter_adds_bounded_queue_depth() { + let mut pacer = LinuxFramePacer::new(120, Some(165), false); + let origin = Instant::now(); + pacer.observe_frame(origin, 0); + for index in 1..20_u64 { + pacer.observe_frame( + origin + Duration::from_micros(index * 13_000), + index * 8_333, + ); + } + let decision = pacer.decision(origin + Duration::from_millis(300)); + assert_eq!(decision.selection, FrameSelectionPolicy::OldestReady); + assert!(decision.target_queue_depth > 0); + assert!(decision.target_queue_depth <= MAX_ADAPTIVE_QUEUE_DEPTH); + } + + #[test] + fn vrr_submits_latest_frame_without_a_fixed_deadline() { + let mut pacer = LinuxFramePacer::new(120, Some(165), true); + let now = Instant::now(); + assert!(pacer.is_due(now)); + assert!(pacer.vrr_enabled()); + assert_eq!( + pacer.decision(now), + PacingDecision { + selection: FrameSelectionPolicy::LatestReady, + target_queue_depth: 0, + } + ); + pacer.mark_presented(now); + assert!(pacer.is_due(now)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_xinput.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_xinput.rs new file mode 100644 index 000000000..962f6bff7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/linux_xinput.rs @@ -0,0 +1,380 @@ +use std::collections::HashSet; +use std::ffi::c_void; +use std::ptr::null; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; + +use x11_dl::{xinput2, xlib}; + +use crate::media::{CapturedInput, CapturedInputQueue}; + +struct XInputState { + enabled: AtomicBool, + pressed_buttons: Mutex>, + captured_input: Arc, +} + +/// Dedicated XInput2 raw-mouse controller matching the official Linux GFN +/// client. It owns a separate X connection and blocks in poll independently +/// of Vulkan presentation, so FIFO WSI back-pressure can never delay input. +pub(crate) struct LinuxXInputController { + state: Arc, + stop_fd: libc::c_int, + worker: Option>, +} + +impl LinuxXInputController { + pub(crate) fn start(captured_input: Arc) -> Result { + let stop_fd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC | libc::EFD_NONBLOCK) }; + if stop_fd < 0 { + return Err(format!( + "failed to create XInput2 wake event: {}", + std::io::Error::last_os_error() + )); + } + let state = Arc::new(XInputState { + enabled: AtomicBool::new(false), + pressed_buttons: Mutex::new(HashSet::new()), + captured_input, + }); + let thread_state = Arc::clone(&state); + let (ready_sender, ready_receiver) = mpsc::sync_channel(1); + let worker = match thread::Builder::new() + .name("opennow-xinput2".to_owned()) + .spawn(move || run_xinput_thread(thread_state, stop_fd, ready_sender)) + { + Ok(worker) => worker, + Err(error) => { + unsafe { libc::close(stop_fd) }; + return Err(format!("failed to spawn XInput2 thread: {error}")); + } + }; + match ready_receiver.recv() { + Ok(Ok(())) => Ok(Self { + state, + stop_fd, + worker: Some(worker), + }), + Ok(Err(error)) => { + let _ = worker.join(); + unsafe { libc::close(stop_fd) }; + Err(error) + } + Err(_) => { + let _ = worker.join(); + unsafe { libc::close(stop_fd) }; + Err("XInput2 thread exited during initialization".to_owned()) + } + } + } + + pub(crate) fn set_enabled(&self, enabled: bool) { + if self.state.enabled.swap(enabled, Ordering::AcqRel) != enabled && !enabled { + release_pressed_buttons(&self.state); + } + } +} + +impl Drop for LinuxXInputController { + fn drop(&mut self) { + self.state.enabled.store(false, Ordering::Release); + release_pressed_buttons(&self.state); + let wake = 1_u64; + unsafe { + let _ = libc::write( + self.stop_fd, + (&wake as *const u64).cast::(), + std::mem::size_of::(), + ); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + unsafe { libc::close(self.stop_fd) }; + } +} + +fn run_xinput_thread( + state: Arc, + stop_fd: libc::c_int, + ready: mpsc::SyncSender>, +) { + let xlib = match xlib::Xlib::open() { + Ok(xlib) => xlib, + Err(error) => { + let _ = ready.send(Err(format!( + "could not load bundled X11 interface: {error}" + ))); + return; + } + }; + let xi = match xinput2::XInput2::open() { + Ok(xi) => xi, + Err(error) => { + let _ = ready.send(Err(format!("could not load XInput2: {error}"))); + return; + } + }; + let display = unsafe { (xlib.XOpenDisplay)(null()) }; + if display.is_null() { + let _ = ready.send(Err("XInput2 could not open DISPLAY".to_owned())); + return; + } + + let result = unsafe { initialize_xinput(&xlib, &xi, display) }; + let opcode = match result { + Ok(opcode) => opcode, + Err(error) => { + unsafe { (xlib.XCloseDisplay)(display) }; + let _ = ready.send(Err(error)); + return; + } + }; + let connection_fd = unsafe { (xlib.XConnectionNumber)(display) }; + if connection_fd < 0 { + unsafe { (xlib.XCloseDisplay)(display) }; + let _ = ready.send(Err("XInput2 returned an invalid X11 connection".to_owned())); + return; + } + let _ = ready.send(Ok(())); + + let mut residual_x = 0.0_f64; + let mut residual_y = 0.0_f64; + let mut poll_fds = [ + libc::pollfd { + fd: connection_fd, + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: stop_fd, + events: libc::POLLIN, + revents: 0, + }, + ]; + loop { + let poll_result = unsafe { libc::poll(poll_fds.as_mut_ptr(), poll_fds.len() as _, -1) }; + if poll_result < 0 { + if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted { + continue; + } + break; + } + if poll_fds[1].revents & libc::POLLIN != 0 { + break; + } + let x_revents = poll_fds[0].revents; + if x_revents & (libc::POLLIN | libc::POLLERR | libc::POLLHUP | libc::POLLNVAL) == 0 { + continue; + } + while unsafe { (xlib.XPending)(display) } > 0 { + let mut event = xlib::XEvent { pad: [0; 24] }; + unsafe { (xlib.XNextEvent)(display, &mut event) }; + if event.get_type() != xlib::GenericEvent { + continue; + } + let cookie = unsafe { &mut event.generic_event_cookie }; + if cookie.extension != opcode || unsafe { (xlib.XGetEventData)(display, cookie) } == 0 { + continue; + } + if !cookie.data.is_null() { + let raw = unsafe { &*(cookie.data.cast::()) }; + process_raw_event(&state, raw, &mut residual_x, &mut residual_y); + } + unsafe { (xlib.XFreeEventData)(display, cookie) }; + } + if x_revents & (libc::POLLERR | libc::POLLHUP | libc::POLLNVAL) != 0 { + break; + } + } + unsafe { (xlib.XCloseDisplay)(display) }; +} + +unsafe fn initialize_xinput( + xlib: &xlib::Xlib, + xi: &xinput2::XInput2, + display: *mut xlib::Display, +) -> Result { + let mut opcode = 0; + let mut first_event = 0; + let mut first_error = 0; + if unsafe { + (xlib.XQueryExtension)( + display, + c"XInputExtension".as_ptr(), + &mut opcode, + &mut first_event, + &mut first_error, + ) + } == 0 + { + return Err("XInput2 extension is unavailable".to_owned()); + } + let mut major = 2; + let mut minor = 0; + if unsafe { (xi.XIQueryVersion)(display, &mut major, &mut minor) } != 0 || major < 2 { + return Err(format!( + "XInput2 2.0 is required (server reported {major}.{minor})" + )); + } + let mut mask = [0_u8; 4]; + xinput2::XISetMask(&mut mask, xinput2::XI_RawButtonPress); + xinput2::XISetMask(&mut mask, xinput2::XI_RawButtonRelease); + xinput2::XISetMask(&mut mask, xinput2::XI_RawMotion); + let mut event_mask = xinput2::XIEventMask { + deviceid: xinput2::XIAllMasterDevices, + mask_len: mask.len() as libc::c_int, + mask: mask.as_mut_ptr(), + }; + let root = unsafe { (xlib.XDefaultRootWindow)(display) }; + if unsafe { (xi.XISelectEvents)(display, root, &mut event_mask, 1) } != 0 { + return Err("XInput2 could not select raw mouse events".to_owned()); + } + unsafe { (xlib.XFlush)(display) }; + eprintln!("Dedicated Linux XInput2 raw mouse thread ready (XI {major}.{minor})"); + Ok(opcode) +} + +fn process_raw_event( + state: &XInputState, + raw: &xinput2::XIRawEvent, + residual_x: &mut f64, + residual_y: &mut f64, +) { + if !state.enabled.load(Ordering::Acquire) { + return; + } + match raw.evtype { + xinput2::XI_RawMotion => { + let (x, y) = raw_motion(raw); + *residual_x += x; + *residual_y += y; + let delta_x = residual_x.trunc() as i32; + let delta_y = residual_y.trunc() as i32; + *residual_x -= f64::from(delta_x); + *residual_y -= f64::from(delta_y); + push_mouse_delta(&state.captured_input, delta_x, delta_y); + } + xinput2::XI_RawButtonPress => process_button(state, raw.detail, true), + xinput2::XI_RawButtonRelease => process_button(state, raw.detail, false), + _ => {} + } +} + +fn raw_motion(raw: &xinput2::XIRawEvent) -> (f64, f64) { + let valuators = raw.valuators; + if valuators.mask_len <= 0 || valuators.mask.is_null() || raw.raw_values.is_null() { + return (0.0, 0.0); + } + let mask = + unsafe { std::slice::from_raw_parts(valuators.mask, valuators.mask_len.max(0) as usize) }; + let mut value_index = 0_usize; + let mut result = (0.0, 0.0); + for axis in 0..valuators.mask_len.saturating_mul(8) { + if !xinput2::XIMaskIsSet(mask, axis) { + continue; + } + let value = unsafe { *raw.raw_values.add(value_index) }; + value_index += 1; + match axis { + 0 => result.0 = value, + 1 => result.1 = value, + _ => {} + } + } + result +} + +fn process_button(state: &XInputState, detail: libc::c_int, pressed: bool) { + if pressed { + match detail { + 4 => { + state.captured_input.push(CapturedInput::MouseWheel { + delta_x: 0, + delta_y: 120, + }); + return; + } + 5 => { + state.captured_input.push(CapturedInput::MouseWheel { + delta_x: 0, + delta_y: -120, + }); + return; + } + _ => {} + } + } else if matches!(detail, 4..=7) { + return; + } + let button = match detail { + 1 => 1, + 2 => 2, + 3 => 3, + 8 => 4, + 9 => 5, + _ => return, + }; + let mut pressed_buttons = state + .pressed_buttons + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.enabled.load(Ordering::Acquire) { + return; + } + let changed = if pressed { + pressed_buttons.insert(button) + } else { + pressed_buttons.remove(&button) + }; + if changed { + state + .captured_input + .push(CapturedInput::MouseButton { button, pressed }); + } +} + +fn release_pressed_buttons(state: &XInputState) { + let mut pressed_buttons = state + .pressed_buttons + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for button in pressed_buttons.drain() { + state.captured_input.push(CapturedInput::MouseButton { + button, + pressed: false, + }); + } +} + +fn push_mouse_delta(queue: &CapturedInputQueue, mut delta_x: i32, mut delta_y: i32) { + while delta_x != 0 || delta_y != 0 { + let x = delta_x.clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + let y = delta_y.clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + queue.push(CapturedInput::MouseMove { + delta_x: x, + delta_y: y, + }); + delta_x -= i32::from(x); + delta_y -= i32::from(y); + } +} + +#[cfg(test)] +mod tests { + use super::push_mouse_delta; + use crate::media::{CapturedInput, CapturedInputQueue}; + + #[test] + fn large_xinput_deltas_are_split_without_loss() { + let queue = CapturedInputQueue::default(); + push_mouse_delta(&queue, 40_000, -40_000); + let mut total = (0_i32, 0_i32); + while let Some(CapturedInput::MouseMove { delta_x, delta_y }) = queue.take() { + total.0 += i32::from(delta_x); + total.1 += i32::from(delta_y); + } + assert_eq!(total, (40_000, -40_000)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/macos_backend.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/macos_backend.rs new file mode 100644 index 000000000..774f0d461 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/macos_backend.rs @@ -0,0 +1,881 @@ +use opennow_streamer_platform_macos::{ + AudioFormat, Av1Format, BackendConfig, BorrowedNsView, H264Format, H264Framing, + H264ParameterSets, H265Format, H265ParameterSets, MacOsBackend, OwnedOverlayConfig, + QueueLimits, ScreenRect, StreamSink, SurfaceTarget, VideoColorSpace, probe_av1_hardware, + probe_h264_hardware, probe_h265_hardware, +}; +use opennow_streamer_protocol::{RenderSurface, RenderSurfaceRect}; +use raw_window_handle::{HasWindowHandle, RawWindowHandle}; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use crate::media::{CapturedInput, MediaStreamConfig}; +use crate::output::{ + OutputControl, SdlInputCapture, external_renderer_enabled, native_input_capture_enabled, +}; + +const HIDDEN_SURFACE: ScreenRect = ScreenRect::new(0.0, 0.0, 2.0, 2.0); +const ORDERING_POLL_INTERVAL: Duration = Duration::from_millis(100); +type H264ParameterSetBytes = (Option>, Option>); +type H265ParameterSetBytes = (Option>, Option>, Option>); + +pub(crate) fn available() -> bool { + h264_available() || h265_available() || av1_available() +} + +pub(crate) fn h264_available() -> bool { + h264_availability().load(Ordering::Acquire) +} + +pub(crate) fn h265_available() -> bool { + h265_availability().load(Ordering::Acquire) +} + +pub(crate) fn av1_available() -> bool { + av1_availability().load(Ordering::Acquire) +} + +pub(crate) fn disable() { + h264_availability().store(false, Ordering::Release); + h265_availability().store(false, Ordering::Release); + av1_availability().store(false, Ordering::Release); +} + +pub(crate) fn disable_h265() { + h265_availability().store(false, Ordering::Release); +} + +pub(crate) fn disable_av1() { + av1_availability().store(false, Ordering::Release); +} + +fn h264_availability() -> &'static AtomicBool { + static AVAILABLE: OnceLock = OnceLock::new(); + AVAILABLE.get_or_init(|| AtomicBool::new(probe_h264_hardware())) +} + +fn h265_availability() -> &'static AtomicBool { + static AVAILABLE: OnceLock = OnceLock::new(); + AVAILABLE.get_or_init(|| AtomicBool::new(probe_h265_hardware())) +} + +fn av1_availability() -> &'static AtomicBool { + static AVAILABLE: OnceLock = OnceLock::new(); + AVAILABLE.get_or_init(|| AtomicBool::new(probe_av1_hardware())) +} + +pub(crate) struct MacOutput { + audio_muted: std::sync::Arc, + backend: Option, + audio_output_device: Option, + // The backend must detach its CAMetalLayer before SDL destroys the NSView. + external_surface: Option, + screen_rect: ScreenRect, + visible: bool, + paused: bool, + last_ordering_check: Instant, + failure_reported: bool, +} + +impl MacOutput { + pub(crate) fn initialize( + stream: MediaStreamConfig, + audio_device: &opennow_streamer_protocol::AudioOutputDevice, + audio_muted: std::sync::Arc, + ) -> Result { + if let Some(id) = audio_device.device_name() { + let devices = opennow_streamer_platform_macos::audio_output_devices() + .map_err(|error| error.to_string())?; + if devices.iter().filter(|device| device.id == id).count() != 1 { + return Err("The selected audio output device is unavailable or ambiguous. Select another device or System default.".to_owned()); + } + } + let external_surface = external_renderer_enabled() + .then(|| MacExternalSurface::initialize(stream)) + .transpose()?; + Ok(Self { + backend: None, + audio_muted, + audio_output_device: audio_device.device_name().map(str::to_owned), + external_surface, + screen_rect: HIDDEN_SURFACE, + visible: false, + paused: false, + last_ordering_check: Instant::now(), + failure_reported: false, + }) + } + + pub(crate) fn start(&mut self, surface: Option<&RenderSurface>) -> Result<(), String> { + self.paused = false; + self.failure_reported = false; + if let Some(surface) = surface { + self.update_surface(surface)?; + } + Ok(()) + } + + pub(crate) fn configure_h264( + &mut self, + parameter_sets: H264ParameterSets, + ) -> Result { + if self.backend.is_some() { + return Err("macOS VideoToolbox backend is already configured".to_owned()); + } + let surface = match self.external_surface.as_ref() { + Some(surface) => surface.target()?, + None => SurfaceTarget::OwnedOverlay(OwnedOverlayConfig::new( + self.screen_rect, + self.visible && !self.paused, + )), + }; + let mut backend = MacOsBackend::start(BackendConfig { + audio_muted: self.audio_muted.clone(), + surface, + video: H264Format::new(parameter_sets, VideoColorSpace::Bt709).into(), + audio: AudioFormat::OPUS_STEREO_48KHZ, + audio_output_device: self.audio_output_device.clone(), + queues: QueueLimits::default(), + }) + .map_err(|error| format!("VideoToolbox backend initialization failed: {error}"))?; + backend + .set_paused(self.paused) + .map_err(|error| format!("VideoToolbox pause state failed: {error}"))?; + let sink = backend.sink(); + self.backend = Some(backend); + Ok(sink) + } + + pub(crate) fn configure_h265( + &mut self, + parameter_sets: H265ParameterSets, + ) -> Result { + if self.backend.is_some() { + return Err("macOS VideoToolbox backend is already configured".to_owned()); + } + let surface = match self.external_surface.as_ref() { + Some(surface) => surface.target()?, + None => SurfaceTarget::OwnedOverlay(OwnedOverlayConfig::new( + self.screen_rect, + self.visible && !self.paused, + )), + }; + let mut backend = MacOsBackend::start(BackendConfig { + surface, + video: H265Format::new(parameter_sets, VideoColorSpace::Bt709).into(), + audio_muted: self.audio_muted.clone(), + audio: AudioFormat::OPUS_STEREO_48KHZ, + audio_output_device: self.audio_output_device.clone(), + queues: QueueLimits::default(), + }) + .map_err(|error| format!("VideoToolbox HEVC backend initialization failed: {error}"))?; + backend + .set_paused(self.paused) + .map_err(|error| format!("VideoToolbox pause state failed: {error}"))?; + let sink = backend.sink(); + self.backend = Some(backend); + Ok(sink) + } + + pub(crate) fn configure_av1(&mut self, format: Av1Format) -> Result { + if self.backend.is_some() { + return Err("macOS VideoToolbox backend is already configured".to_owned()); + } + let surface = match self.external_surface.as_ref() { + Some(surface) => surface.target()?, + None => SurfaceTarget::OwnedOverlay(OwnedOverlayConfig::new( + self.screen_rect, + self.visible && !self.paused, + )), + }; + let mut backend = MacOsBackend::start(BackendConfig { + surface, + video: format.into(), + audio_muted: self.audio_muted.clone(), + audio: AudioFormat::OPUS_STEREO_48KHZ, + audio_output_device: self.audio_output_device.clone(), + queues: QueueLimits::default(), + }) + .map_err(|error| format!("VideoToolbox AV1 backend initialization failed: {error}"))?; + backend + .set_paused(self.paused) + .map_err(|error| format!("VideoToolbox pause state failed: {error}"))?; + let sink = backend.sink(); + self.backend = Some(backend); + Ok(sink) + } + + pub(crate) fn set_paused(&mut self, paused: bool) -> Result<(), String> { + self.paused = paused; + if let Some(surface) = self.external_surface.as_mut() { + surface + .input_capture + .set_input_paused(paused, &surface.sdl, &mut surface.window); + surface.set_visible(self.visible && !paused); + } + if let Some(backend) = self.backend.as_mut() { + backend + .set_paused(paused) + .map_err(|error| format!("macOS media pause failed: {error}"))?; + if self.external_surface.is_none() { + backend + .update_owned_overlay(self.screen_rect, self.visible && !paused) + .map_err(|error| format!("macOS overlay pause failed: {error}"))?; + } + } + Ok(()) + } + + pub(crate) fn stop(&mut self) { + if let Some(mut backend) = self.backend.take() { + backend.stop(); + } + if let Some(surface) = self.external_surface.as_mut() { + surface.hide(); + } + self.paused = false; + } + + pub(crate) fn update_surface(&mut self, surface: &RenderSurface) -> Result<(), String> { + if let Some(external) = self.external_surface.as_mut() { + self.visible = surface.visible; + if surface.visible { + external.update_geometry(surface)?; + } + external.set_visible(surface.visible && !self.paused); + return Ok(()); + } + if surface.visible { + self.screen_rect = surface.screen_rect.map(screen_rect).ok_or_else(|| { + "visible macOS overlay is missing absolute screen bounds".to_owned() + })?; + } + self.visible = surface.visible && surface.screen_rect.is_some(); + if let Some(backend) = self.backend.as_mut() { + backend + .update_owned_overlay(self.screen_rect, self.visible && !self.paused) + .map_err(|error| format!("macOS overlay update failed: {error}"))?; + } + Ok(()) + } + + pub(crate) fn pump(&mut self) -> Result<(), String> { + if let Some(surface) = self.external_surface.as_mut() { + surface.pump(); + } + if !self.failure_reported + && let Some(failure) = self.backend.as_ref().and_then(MacOsBackend::fatal_failure) + { + self.failure_reported = true; + return Err(format!( + "{} backend failed: {}", + failure.subsystem.name(), + failure.message + )); + } + if self.external_surface.is_some() + || self.last_ordering_check.elapsed() < ORDERING_POLL_INTERVAL + { + return Ok(()); + } + self.last_ordering_check = Instant::now(); + if let Some(backend) = self.backend.as_mut() { + backend + .refresh_overlay_ordering() + .map_err(|error| format!("macOS overlay ordering refresh failed: {error}"))?; + } + Ok(()) + } + + pub(crate) fn take_captured_input(&mut self) -> Vec { + self.external_surface + .as_mut() + .map(MacExternalSurface::take_captured_input) + .unwrap_or_default() + } + + pub(crate) fn update_cursor(&mut self, bytes: &[u8]) { + if let Some(surface) = self.external_surface.as_mut() { + surface.update_cursor(bytes); + } + } + + pub(crate) fn control(&mut self, control: OutputControl) -> Result<(), String> { + let Some(surface) = self.external_surface.as_mut() else { + return Err("Runtime controls require the external stream window".to_owned()); + }; + match control { + OutputControl::PointerLock => { + surface + .input_capture + .toggle_pointer_lock(&surface.sdl, &mut surface.window); + Ok(()) + } + } + } +} + +struct MacExternalSurface { + // Input-owned SDL cursors must drop before the window and SDL root. + input_capture: SdlInputCapture, + event_pump: sdl2::EventPump, + window: sdl2::video::Window, + sdl: sdl2::Sdl, + stream_size: (u32, u32), + visible: bool, +} + +impl MacExternalSurface { + fn initialize(stream: MediaStreamConfig) -> Result { + sdl2::hint::set("SDL_MOUSE_RELATIVE_MODE_WARP", "0"); + sdl2::hint::set("SDL_MOUSE_RELATIVE_SCALING", "0"); + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let video = sdl + .video() + .map_err(|error| format!("SDL video initialization failed: {error}"))?; + let mut window_builder = video.window("OpenNOW Stream", 1280, 800); + window_builder + .position_centered() + .resizable() + .allow_highdpi() + .hidden() + .metal_view(); + let window = window_builder + .build() + .map_err(|error| format!("external macOS stream window creation failed: {error}"))?; + // Resolve the AppKit handle now so startup fails clearly instead of waiting for the first + // keyframe to discover that VideoToolbox has nowhere to present. + Self::target_for_window(&window)?; + let event_pump = sdl + .event_pump() + .map_err(|error| format!("external macOS event pump creation failed: {error}"))?; + let capture_input = native_input_capture_enabled(); + eprintln!( + "External macOS stream window ready (native keyboard/mouse capture: {capture_input})" + ); + let mut input_capture = SdlInputCapture::new(capture_input, false, false, stream.shortcuts); + input_capture.enable_gamepads(&sdl); + Ok(Self { + input_capture, + event_pump, + window, + sdl, + stream_size: (stream.width.max(1), stream.height.max(1)), + visible: false, + }) + } + + fn target(&self) -> Result { + Self::target_for_window(&self.window) + } + + fn target_for_window(window: &sdl2::video::Window) -> Result { + let raw = window + .window_handle() + .map_err(|error| format!("SDL AppKit handle unavailable: {error}"))? + .as_raw(); + let RawWindowHandle::AppKit(handle) = raw else { + return Err("SDL did not create an AppKit stream window".to_owned()); + }; + // SAFETY: the SDL window owns this dedicated NSView, remains alive after the backend is + // constructed, and all calls occur on the streamer's AppKit main thread. + Ok(SurfaceTarget::NsView(unsafe { + BorrowedNsView::from_raw(handle.ns_view) + })) + } + + fn set_visible(&mut self, visible: bool) { + if visible { + opennow_streamer_platform_macos::activate_stream_application(); + self.window.show(); + if !self.visible { + self.window.raise(); + } + } else { + self.release_input(); + self.window.hide(); + } + self.visible = visible; + } + + fn update_geometry(&mut self, surface: &RenderSurface) -> Result<(), String> { + let bounds = surface + .screen_rect + .ok_or_else(|| "visible external macOS surface is missing screen bounds".to_owned())?; + let scale = surface.device_scale_factor.clamp(0.5, 8.0); + self.window.set_position( + sdl2::video::WindowPos::Positioned((bounds.x as f32 / scale).round() as i32), + sdl2::video::WindowPos::Positioned((bounds.y as f32 / scale).round() as i32), + ); + self.window + .set_size( + ((bounds.width as f32 / scale).round() as u32).max(2), + ((bounds.height as f32 / scale).round() as u32).max(2), + ) + .map_err(|error| format!("failed to resize external macOS surface: {error}")) + } + + fn pump(&mut self) { + let window_id = self.window.id(); + for event in self.event_pump.poll_iter().collect::>() { + if matches!(event, sdl2::event::Event::Quit { .. }) + || matches!( + event, + sdl2::event::Event::Window { + window_id: event_window_id, + win_event: sdl2::event::WindowEvent::Close, + .. + } if event_window_id == window_id + ) + { + self.release_input(); + self.window.hide(); + self.visible = false; + } else { + self.input_capture.handle_event( + &self.sdl, + &mut self.window, + self.stream_size, + event, + ); + } + } + } + + fn release_input(&mut self) { + self.input_capture.release(&self.sdl, &mut self.window); + } + + fn hide(&mut self) { + self.release_input(); + self.window.hide(); + self.visible = false; + } + + fn take_captured_input(&mut self) -> Vec { + self.input_capture.take() + } + + fn update_cursor(&mut self, bytes: &[u8]) { + self.input_capture + .apply_cursor(&self.sdl, &mut self.window, self.stream_size, bytes); + } +} + +fn screen_rect(rect: RenderSurfaceRect) -> ScreenRect { + ScreenRect::new( + f64::from(rect.x), + f64::from(rect.y), + f64::from(rect.width), + f64::from(rect.height), + ) +} + +#[derive(Default)] +pub(crate) struct H264ParameterSetTracker { + committed: Option, + bootstrap_sequence: Option>, + bootstrap_picture: Option>, + candidate: Option, +} + +impl H264ParameterSetTracker { + pub(crate) fn observe(&mut self, access_unit: &[u8]) -> Result { + // Parameter-set replacement is transactional. A damaged access unit must never combine a + // new SPS with the previously committed PPS (or vice versa), because that creates a pair + // which VideoToolbox quite correctly rejects. During initial bootstrap we may accumulate + // SPS/PPS across access units; after a decoder is committed, only a complete pair from the + // same access unit is eligible to replace it. + self.candidate = None; + let (framing, sequence, picture) = if find_start_code(access_unit, 0).is_some() { + let (sequence, picture) = Self::parameter_sets_from_annex_b(access_unit)?; + (H264Framing::AnnexB, sequence, picture) + } else { + let (sequence, picture) = Self::parameter_sets_from_avcc(access_unit)?; + (H264Framing::Avcc, sequence, picture) + }; + + if let (Some(sequence), Some(picture)) = (sequence.as_ref(), picture.as_ref()) { + self.candidate = Some( + H264ParameterSets::new(sequence, picture) + .map_err(|error| format!("invalid H.264 parameter sets: {error}"))?, + ); + } else if self.committed.is_none() { + if sequence.is_some() { + self.bootstrap_sequence = sequence; + } + if picture.is_some() { + self.bootstrap_picture = picture; + } + if let (Some(sequence), Some(picture)) = + (&self.bootstrap_sequence, &self.bootstrap_picture) + { + self.candidate = Some( + H264ParameterSets::new(sequence, picture) + .map_err(|error| format!("invalid H.264 parameter sets: {error}"))?, + ); + } + } + Ok(framing) + } + + pub(crate) fn parameter_sets(&self) -> Option { + self.candidate.clone().or_else(|| self.committed.clone()) + } + + pub(crate) fn commit_parameter_sets(&mut self, parameter_sets: H264ParameterSets) { + self.committed = Some(parameter_sets); + self.bootstrap_sequence = None; + self.bootstrap_picture = None; + self.candidate = None; + } + + fn parameter_sets_from_annex_b(access_unit: &[u8]) -> Result { + let Some((mut start, prefix)) = find_start_code(access_unit, 0) else { + return Err("H.264 access unit has no Annex B start code".to_owned()); + }; + if access_unit[..start].iter().any(|byte| *byte != 0) { + return Err("H.264 access unit has invalid Annex B framing".to_owned()); + } + start += prefix; + let mut sequence = None; + let mut picture = None; + loop { + let next = find_start_code(access_unit, start); + let end = next.map_or(access_unit.len(), |(offset, _)| offset); + Self::observe_nal(&access_unit[start..end], &mut sequence, &mut picture)?; + let Some((next_start, next_prefix)) = next else { + return Ok((sequence, picture)); + }; + start = next_start + next_prefix; + } + } + + fn parameter_sets_from_avcc(access_unit: &[u8]) -> Result { + let mut offset = 0usize; + let mut sequence = None; + let mut picture = None; + while offset < access_unit.len() { + let length_bytes = access_unit + .get(offset..offset + 4) + .ok_or_else(|| "H.264 access unit has truncated AVCC framing".to_owned())?; + let length = u32::from_be_bytes( + length_bytes + .try_into() + .expect("AVCC length was checked as four bytes"), + ) as usize; + offset += 4; + let end = offset + .checked_add(length) + .filter(|end| *end <= access_unit.len()) + .ok_or_else(|| "H.264 access unit has invalid AVCC framing".to_owned())?; + Self::observe_nal(&access_unit[offset..end], &mut sequence, &mut picture)?; + offset = end; + } + if offset == 0 { + return Err("H.264 access unit is empty".to_owned()); + } + Ok((sequence, picture)) + } + + fn observe_nal( + nal: &[u8], + sequence: &mut Option>, + picture: &mut Option>, + ) -> Result<(), String> { + let header = *nal + .first() + .ok_or_else(|| "H.264 access unit contains an empty NAL unit".to_owned())?; + match header & 0x1f { + 7 => *sequence = Some(nal.to_vec()), + 8 => *picture = Some(nal.to_vec()), + _ => {} + } + Ok(()) + } +} + +#[derive(Default)] +pub(crate) struct H265ParameterSetTracker { + committed: Option, + bootstrap_video: Option>, + bootstrap_sequence: Option>, + bootstrap_picture: Option>, + candidate: Option, +} + +impl H265ParameterSetTracker { + pub(crate) fn observe(&mut self, access_unit: &[u8]) -> Result { + self.candidate = None; + let (framing, video, sequence, picture) = if find_start_code(access_unit, 0).is_some() { + let (video, sequence, picture) = Self::parameter_sets_from_annex_b(access_unit)?; + (H264Framing::AnnexB, video, sequence, picture) + } else { + let (video, sequence, picture) = Self::parameter_sets_from_avcc(access_unit)?; + (H264Framing::Avcc, video, sequence, picture) + }; + + if let (Some(video), Some(sequence), Some(picture)) = + (video.as_ref(), sequence.as_ref(), picture.as_ref()) + { + self.candidate = Some( + H265ParameterSets::new(video, sequence, picture) + .map_err(|error| format!("invalid H.265 parameter sets: {error}"))?, + ); + } else if self.committed.is_none() { + if video.is_some() { + self.bootstrap_video = video; + } + if sequence.is_some() { + self.bootstrap_sequence = sequence; + } + if picture.is_some() { + self.bootstrap_picture = picture; + } + if let (Some(video), Some(sequence), Some(picture)) = ( + &self.bootstrap_video, + &self.bootstrap_sequence, + &self.bootstrap_picture, + ) { + self.candidate = Some( + H265ParameterSets::new(video, sequence, picture) + .map_err(|error| format!("invalid H.265 parameter sets: {error}"))?, + ); + } + } + Ok(framing) + } + + pub(crate) fn parameter_sets(&self) -> Option { + self.candidate.clone().or_else(|| self.committed.clone()) + } + + pub(crate) fn commit_parameter_sets(&mut self, parameter_sets: H265ParameterSets) { + self.committed = Some(parameter_sets); + self.bootstrap_video = None; + self.bootstrap_sequence = None; + self.bootstrap_picture = None; + self.candidate = None; + } + + fn parameter_sets_from_annex_b(access_unit: &[u8]) -> Result { + let Some((mut start, prefix)) = find_start_code(access_unit, 0) else { + return Err("H.265 access unit has no Annex B start code".to_owned()); + }; + if access_unit[..start].iter().any(|byte| *byte != 0) { + return Err("H.265 access unit has invalid Annex B framing".to_owned()); + } + start += prefix; + let mut video = None; + let mut sequence = None; + let mut picture = None; + loop { + let next = find_start_code(access_unit, start); + let end = next.map_or(access_unit.len(), |(offset, _)| offset); + Self::observe_nal( + &access_unit[start..end], + &mut video, + &mut sequence, + &mut picture, + )?; + let Some((next_start, next_prefix)) = next else { + return Ok((video, sequence, picture)); + }; + start = next_start + next_prefix; + } + } + + fn parameter_sets_from_avcc(access_unit: &[u8]) -> Result { + let mut offset = 0usize; + let mut video = None; + let mut sequence = None; + let mut picture = None; + while offset < access_unit.len() { + let length_bytes = access_unit + .get(offset..offset + 4) + .ok_or_else(|| "H.265 access unit has truncated HVCC framing".to_owned())?; + let length = u32::from_be_bytes( + length_bytes + .try_into() + .expect("HVCC length was checked as four bytes"), + ) as usize; + offset += 4; + let end = offset + .checked_add(length) + .filter(|end| *end <= access_unit.len()) + .ok_or_else(|| "H.265 access unit has invalid HVCC framing".to_owned())?; + Self::observe_nal( + &access_unit[offset..end], + &mut video, + &mut sequence, + &mut picture, + )?; + offset = end; + } + if offset == 0 { + return Err("H.265 access unit is empty".to_owned()); + } + Ok((video, sequence, picture)) + } + + fn observe_nal( + nal: &[u8], + video: &mut Option>, + sequence: &mut Option>, + picture: &mut Option>, + ) -> Result<(), String> { + let header = *nal + .first() + .ok_or_else(|| "H.265 access unit contains an empty NAL unit".to_owned())?; + match (header >> 1) & 0x3f { + 32 => *video = Some(nal.to_vec()), + 33 => *sequence = Some(nal.to_vec()), + 34 => *picture = Some(nal.to_vec()), + _ => {} + } + Ok(()) + } +} + +fn find_start_code(bytes: &[u8], from: usize) -> Option<(usize, usize)> { + let mut index = from; + while index + 3 <= bytes.len() { + if bytes[index..].starts_with(&[0, 0, 0, 1]) { + return Some((index, 4)); + } + if bytes[index..].starts_with(&[0, 0, 1]) { + return Some((index, 3)); + } + index += 1; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_parameter_sets_from_annex_b_access_unit() { + let mut tracker = H264ParameterSetTracker::default(); + let framing = tracker + .observe(&[ + 0, 0, 0, 1, 0x67, 0x64, 0, 0x29, 0, 0, 1, 0x68, 0xee, 0x3c, 0x80, 0, 0, 0, 1, 0x65, + 1, + ]) + .unwrap(); + assert_eq!(framing, H264Framing::AnnexB); + let parameters = tracker.parameter_sets().unwrap(); + assert_eq!(parameters.sequence(), &[0x67, 0x64, 0, 0x29]); + assert_eq!(parameters.picture(), &[0x68, 0xee, 0x3c, 0x80]); + } + + #[test] + fn extracts_parameter_sets_across_avcc_access_units() { + let mut tracker = H264ParameterSetTracker::default(); + assert_eq!( + tracker.observe(&[0, 0, 0, 2, 0x67, 1]).unwrap(), + H264Framing::Avcc + ); + assert!(tracker.parameter_sets().is_none()); + tracker.observe(&[0, 0, 0, 2, 0x68, 2]).unwrap(); + assert!(tracker.parameter_sets().is_some()); + } + + #[test] + fn committed_parameter_sets_are_not_mixed_with_a_single_set_update() { + let mut tracker = H264ParameterSetTracker::default(); + tracker + .observe(&[ + 0, 0, 0, 1, 0x67, 0x64, 0, 0x29, 0, 0, 0, 1, 0x68, 0xee, 0x3c, 0x80, + ]) + .unwrap(); + let committed = tracker.parameter_sets().unwrap(); + tracker.commit_parameter_sets(committed.clone()); + + // A lone SPS may be damaged or belong to an incomplete stream transition. It must not be + // paired with the old PPS and offered to VideoToolbox as a reconfiguration. + tracker + .observe(&[0, 0, 0, 1, 0x67, 0x42, 0, 0x1f, 0, 0, 0, 1, 0x65, 1]) + .unwrap(); + assert_eq!(tracker.parameter_sets(), Some(committed)); + } + + #[test] + fn candidate_parameter_sets_do_not_replace_committed_sets_until_commit() { + let mut tracker = H264ParameterSetTracker::default(); + tracker + .observe(&[0, 0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x68, 2]) + .unwrap(); + let committed = tracker.parameter_sets().unwrap(); + tracker.commit_parameter_sets(committed.clone()); + + tracker + .observe(&[0, 0, 0, 1, 0x67, 3, 0, 0, 0, 1, 0x68, 4]) + .unwrap(); + assert_ne!(tracker.parameter_sets(), Some(committed.clone())); + + // Observing the next regular frame discards the uncommitted candidate and restores the + // last known-good pair, which models a failed decoder reconfiguration. + tracker.observe(&[0, 0, 0, 1, 0x65, 5]).unwrap(); + assert_eq!(tracker.parameter_sets(), Some(committed)); + } + + #[test] + fn extracts_hevc_parameter_sets_from_annex_b_access_unit() { + let mut tracker = H265ParameterSetTracker::default(); + let framing = tracker + .observe(&[ + 0, 0, 0, 1, 0x40, 0x01, 0x0c, 0, 0, 1, 0x42, 0x01, 0x01, 0, 0, 0, 1, 0x44, 0x01, + 0xc0, 0, 0, 1, 0x26, 0x01, + ]) + .unwrap(); + assert_eq!(framing, H264Framing::AnnexB); + let parameters = tracker.parameter_sets().unwrap(); + assert_eq!(parameters.video(), &[0x40, 0x01, 0x0c]); + assert_eq!(parameters.sequence(), &[0x42, 0x01, 0x01]); + assert_eq!(parameters.picture(), &[0x44, 0x01, 0xc0]); + } + + #[test] + fn extracts_hevc_parameter_sets_across_hvcc_access_units() { + let mut tracker = H265ParameterSetTracker::default(); + assert_eq!( + tracker.observe(&[0, 0, 0, 3, 0x40, 1, 1]).unwrap(), + H264Framing::Avcc + ); + assert!(tracker.parameter_sets().is_none()); + tracker.observe(&[0, 0, 0, 3, 0x42, 1, 2]).unwrap(); + assert!(tracker.parameter_sets().is_none()); + tracker.observe(&[0, 0, 0, 3, 0x44, 1, 3]).unwrap(); + assert!(tracker.parameter_sets().is_some()); + } + + #[test] + fn committed_hevc_sets_are_replaced_only_as_a_complete_triplet() { + let mut tracker = H265ParameterSetTracker::default(); + tracker + .observe(&[ + 0, 0, 0, 1, 0x40, 1, 1, 0, 0, 1, 0x42, 1, 2, 0, 0, 1, 0x44, 1, 3, + ]) + .unwrap(); + let committed = tracker.parameter_sets().unwrap(); + tracker.commit_parameter_sets(committed.clone()); + + tracker + .observe(&[0, 0, 0, 1, 0x42, 1, 9, 0, 0, 0, 1, 0x26, 1]) + .unwrap(); + assert_eq!(tracker.parameter_sets(), Some(committed.clone())); + + tracker + .observe(&[ + 0, 0, 0, 1, 0x40, 1, 7, 0, 0, 1, 0x42, 1, 8, 0, 0, 1, 0x44, 1, 9, + ]) + .unwrap(); + assert_ne!(tracker.parameter_sets(), Some(committed.clone())); + + tracker.observe(&[0, 0, 0, 1, 0x26, 1]).unwrap(); + assert_eq!(tracker.parameter_sets(), Some(committed)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media.rs new file mode 100644 index 000000000..9273f87f9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media.rs @@ -0,0 +1,5784 @@ +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, TrySendError, sync_channel}; +use std::sync::{Arc, Mutex, Weak}; +use std::thread::{self, JoinHandle}; +#[cfg(target_os = "windows")] +use std::time::Duration; +use std::time::Instant; + +use opennow_streamer_protocol::{RecordingCompletion, RecordingCutReason}; + +use openh264::OpenH264API; +use openh264::decoder::{Decoder as OpenH264Decoder, DecoderConfig}; +use openh264::formats::YUVSource; +use opus::{Channels, Decoder as OpusNativeDecoder}; + +#[cfg(target_os = "windows")] +use crate::output::WindowsBridge; +use crate::output::{DecodedVideoFrame, OutputBuffers}; +use crate::queue::{BoundedQueue, PushResult}; +use crate::runtime::HostCommand; +use crate::video_queue::VideoQueue; + +#[cfg(target_os = "linux")] +use crate::linux_backend::{LinuxVideoPath, LinuxVideoSelection}; + +const VIDEO_QUEUE_CAPACITY: usize = 2; +const VIDEO_QUEUE_MAX_CAPACITY: usize = 60; +// Ten 20 ms Opus packets cover the official client's 200 ms adaptive ceiling. +// The queue remains bounded and drop-oldest, so recovery cannot grow latency +// without limit under a stalled decoder. +const AUDIO_QUEUE_CAPACITY: usize = 10; +const OPUS_SAMPLE_RATE: u32 = 48_000; +const MAX_OPUS_FRAME_SAMPLES_PER_CHANNEL: usize = 5_760; +const RECORDING_TAP_QUEUE_CAPACITY: usize = 256; +const RECORDING_RUNNING: u64 = 0; +const RECORDING_GRACEFUL: u64 = 1; +const RECORDING_DISCONTINUITY: u64 = 2; +const RECORDING_OVERFLOW: u64 = 3; +const RECORDING_INTERRUPTED: u64 = 4; +const RECORDING_STATE_MASK: u64 = 7; + +#[cfg(test)] +#[path = "media_recording_tests.rs"] +mod recording_tests; + +pub struct EncodedRecordingReceiver { + receiver: Receiver, + terminal: Arc, + wake: Arc>>, + _lifetime: Arc<()>, + #[cfg(test)] + before_wait: Option>, + #[cfg(test)] + wait_release: Option>, +} + +impl EncodedRecordingReceiver { + pub fn recv(&self) -> Result { + loop { + if let Some(completion) = self.cut() { + return Err(completion); + } + match self.receiver.try_recv() { + Ok(frame) => return self.cut().map_or(Ok(frame), Err), + Err(TryRecvError::Empty) => { + let wake = self + .wake + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + #[cfg(test)] + if let Some(before_wait) = &self.before_wait { + let _ = before_wait.send(()); + } + #[cfg(test)] + if let Some(wait_release) = &self.wait_release { + let _ = wait_release.recv(); + } + if wake.recv().is_err() { + return self + .receiver + .recv() + .map_err(|_| self.cut().unwrap_or(RecordingCompletion::Complete)); + } + } + Err(TryRecvError::Disconnected) => { + return Err(self.cut().unwrap_or(RecordingCompletion::Complete)); + } + } + } + } + + fn cut(&self) -> Option { + let reason = match self.terminal.load(Ordering::Acquire) & RECORDING_STATE_MASK { + RECORDING_RUNNING | RECORDING_GRACEFUL => return None, + RECORDING_DISCONTINUITY => RecordingCutReason::Discontinuity, + RECORDING_OVERFLOW => RecordingCutReason::QueueOverflow, + RECORDING_INTERRUPTED => RecordingCutReason::Interrupted, + _ => unreachable!(), + }; + Some(RecordingCompletion::Cut { reason }) + } + + pub fn overflowed(&self) -> bool { + self.cut() + == Some(RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + }) + } + + #[cfg(test)] + pub(crate) fn from_receiver(receiver: Receiver) -> Self { + let (_, wake) = sync_channel(1); + Self { + receiver, + terminal: Arc::new(AtomicU64::new(RECORDING_RUNNING)), + _lifetime: Arc::new(()), + wake: Arc::new(Mutex::new(wake)), + #[cfg(test)] + before_wait: None, + #[cfg(test)] + wait_release: None, + } + } +} + +#[derive(Default)] +struct RecordingSubscription { + sender: Option>, + receiver: Weak<()>, +} + +struct RecordingTap { + subscription: Mutex, + terminal: Arc, + wake_sender: SyncSender<()>, + wake_receiver: Arc>>, +} + +impl Default for RecordingTap { + fn default() -> Self { + let (wake_sender, wake_receiver) = sync_channel(1); + Self { + subscription: Mutex::default(), + terminal: Arc::new(AtomicU64::new(RECORDING_RUNNING)), + wake_sender, + wake_receiver: Arc::new(Mutex::new(wake_receiver)), + } + } +} + +impl RecordingTap { + fn subscribe(&self) -> Result { + let mut active = self + .subscription + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if active.sender.is_some() || active.receiver.upgrade().is_some() { + return Err("a native stream recording is already active".to_owned()); + } + let generation = (self.terminal.load(Ordering::Acquire) & !RECORDING_STATE_MASK) + .wrapping_add(RECORDING_STATE_MASK + 1); + self.terminal.store(generation, Ordering::Release); + let (sender, receiver) = sync_channel(RECORDING_TAP_QUEUE_CAPACITY); + let lifetime = Arc::new(()); + active.sender = Some(sender); + active.receiver = Arc::downgrade(&lifetime); + Ok(EncodedRecordingReceiver { + receiver, + terminal: Arc::clone(&self.terminal), + _lifetime: lifetime, + wake: Arc::clone(&self.wake_receiver), + #[cfg(test)] + before_wait: None, + #[cfg(test)] + wait_release: None, + }) + } + + fn unsubscribe(&self) { + self.close(RecordingCompletion::Complete); + } + + fn close(&self, completion: RecordingCompletion) { + let mut active = self + .subscription + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.signal(self.terminal.load(Ordering::Acquire), completion); + active.sender.take(); + let _ = self.wake_sender.try_send(()); + } + + fn signal(&self, status: u64, completion: RecordingCompletion) { + if status & RECORDING_STATE_MASK != RECORDING_RUNNING { + return; + } + let code = match completion { + RecordingCompletion::Complete => RECORDING_GRACEFUL, + RecordingCompletion::Cut { + reason: RecordingCutReason::Discontinuity, + } => RECORDING_DISCONTINUITY, + RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + } => RECORDING_OVERFLOW, + RecordingCompletion::Cut { + reason: RecordingCutReason::Interrupted, + } => RECORDING_INTERRUPTED, + }; + let _ = self.terminal.compare_exchange( + status, + status | code, + Ordering::AcqRel, + Ordering::Acquire, + ); + let _ = self.wake_sender.try_send(()); + } + + fn publish(&self, frame: &EncodedFrame) { + let status = self.terminal.load(Ordering::Acquire); + let Ok(mut active) = self.subscription.try_lock() else { + self.signal( + status, + RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + }, + ); + return; + }; + if self.terminal.load(Ordering::Acquire) != status { + return; + } + if self.terminal.load(Ordering::Acquire) & RECORDING_STATE_MASK != RECORDING_RUNNING { + active.sender.take(); + return; + } + let Some(sender) = active.sender.as_ref() else { + return; + }; + match sender.try_send(frame.clone()) { + Ok(()) => { + let _ = self.wake_sender.try_send(()); + } + Err(TrySendError::Full(_)) => { + self.signal( + self.terminal.load(Ordering::Acquire), + RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + }, + ); + active.sender.take(); + } + Err(TrySendError::Disconnected(_)) => { + active.sender.take(); + } + } + } +} + +fn video_queue_capacity(fps: u32) -> usize { + let frames_for_recovery_burst = fps.max(1).div_ceil(4); + usize::try_from(frames_for_recovery_burst) + .unwrap_or(VIDEO_QUEUE_MAX_CAPACITY) + .clamp(VIDEO_QUEUE_CAPACITY, VIDEO_QUEUE_MAX_CAPACITY) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaVideoCodec { + H264, + H265, + Av1, +} + +impl MediaVideoCodec { + pub const fn label(self) -> &'static str { + match self { + Self::H264 => "h264", + Self::H265 => "h265", + Self::Av1 => "av1", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ShortcutChord { + pub virtual_key: u16, + pub modifiers: u16, +} + +impl ShortcutChord { + pub fn parse(value: &str) -> Option { + let mut modifiers = 0_u16; + let mut virtual_key = None; + for part in value.split('+').map(str::trim) { + if part.is_empty() { + return None; + } + let normalized = part.to_ascii_lowercase(); + let modifier = match normalized.as_str() { + "shift" => Some(0x01), + "ctrl" | "control" => Some(0x02), + "alt" | "option" => Some(0x04), + "meta" | "command" | "cmd" | "super" | "win" => Some(0x08), + key if virtual_key.is_none() => { + virtual_key = shortcut_virtual_key(key); + None + } + _ => return None, + }; + if let Some(modifier) = modifier { + if modifiers & modifier != 0 { + return None; + } + modifiers |= modifier; + } + } + virtual_key.map(|virtual_key| Self { + virtual_key, + modifiers, + }) + } +} + +fn shortcut_virtual_key(value: &str) -> Option { + if value.len() == 1 { + let byte = value.as_bytes()[0].to_ascii_uppercase(); + if byte.is_ascii_alphanumeric() { + return Some(u16::from(byte)); + } + } + if let Some(number) = value + .strip_prefix('f') + .and_then(|number| number.parse::().ok()) + && (1..=24).contains(&number) + { + return Some(0x6f + number); + } + Some(match value { + "enter" | "return" => 0x0d, + "escape" | "esc" => 0x1b, + "backspace" => 0x08, + "tab" => 0x09, + "space" => 0x20, + "left" => 0x25, + "up" => 0x26, + "right" => 0x27, + "down" => 0x28, + "insert" => 0x2d, + "delete" | "del" => 0x2e, + "home" => 0x24, + "end" => 0x23, + "pageup" | "pgup" => 0x21, + "pagedown" | "pgdown" => 0x22, + "printscreen" | "print" => 0x2a, + "pause" => 0x13, + _ => return None, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamShortcutAction { + ToggleStats, + TogglePointerLock, + ToggleFullscreen, + StopStream, + ToggleAntiAfk, + Screenshot, + ToggleRecording, + SaveClip, +} + +impl StreamShortcutAction { + pub const fn protocol_name(self) -> &'static str { + match self { + Self::ToggleStats => "toggle-stats", + Self::TogglePointerLock => "toggle-pointer-lock", + Self::ToggleFullscreen => "toggle-fullscreen", + Self::StopStream => "stop-stream", + Self::ToggleAntiAfk => "toggle-anti-afk", + Self::Screenshot => "screenshot", + Self::ToggleRecording => "toggle-recording", + Self::SaveClip => "save-clip", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StreamShortcutBindings { + bindings: [(StreamShortcutAction, Option); 8], +} + +impl StreamShortcutBindings { + pub fn from_json(value: &serde_json::Value) -> Self { + let read = |key: &str, fallback: &str| match value + .get(key) + .and_then(serde_json::Value::as_str) + { + Some("") => None, + Some(chord) => ShortcutChord::parse(chord).or_else(|| ShortcutChord::parse(fallback)), + None => ShortcutChord::parse(fallback), + }; + Self { + bindings: [ + ( + StreamShortcutAction::ToggleStats, + read("toggleStats", "Ctrl+N"), + ), + ( + StreamShortcutAction::TogglePointerLock, + read("togglePointerLock", "F8"), + ), + ( + StreamShortcutAction::ToggleFullscreen, + read("toggleFullscreen", "F11"), + ), + ( + StreamShortcutAction::StopStream, + read("stopStream", "Ctrl+Shift+Q"), + ), + ( + StreamShortcutAction::ToggleAntiAfk, + read("toggleAntiAfk", "Ctrl+Shift+K"), + ), + ( + StreamShortcutAction::Screenshot, + read("screenshot", "Ctrl+F11"), + ), + ( + StreamShortcutAction::ToggleRecording, + read("toggleRecording", "F12"), + ), + (StreamShortcutAction::SaveClip, read("saveClip", "Ctrl+F12")), + ], + } + } + + pub const fn action(self, virtual_key: u16, modifiers: u16) -> Option { + let mut index = 0; + while index < self.bindings.len() { + let (action, chord) = self.bindings[index]; + if let Some(chord) = chord + && chord.virtual_key == virtual_key + && chord.modifiers == modifiers + { + return Some(action); + } + index += 1; + } + None + } +} + +impl Default for StreamShortcutBindings { + fn default() -> Self { + Self::from_json(&serde_json::Value::Null) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum MediaColorQuality { + #[default] + EightBit420, + EightBit444, + TenBit420, + TenBit444, +} + +impl MediaColorQuality { + pub const fn protocol_name(self) -> &'static str { + match self { + Self::EightBit420 => "8bit_420", + Self::EightBit444 => "8bit_444", + Self::TenBit420 => "10bit_420", + Self::TenBit444 => "10bit_444", + } + } + + pub const fn bit_depth(self) -> u8 { + match self { + Self::EightBit420 | Self::EightBit444 => 8, + Self::TenBit420 | Self::TenBit444 => 10, + } + } + + pub const fn is_444(self) -> bool { + matches!(self, Self::EightBit444 | Self::TenBit444) + } + + #[cfg(target_os = "linux")] + pub(crate) const fn linux_pixel_format(self) -> opennow_streamer_platform_linux::PixelFormat { + use opennow_streamer_platform_linux::PixelFormat; + match self { + Self::EightBit420 => PixelFormat::Nv12, + Self::EightBit444 => PixelFormat::Nv24, + Self::TenBit420 => PixelFormat::P010, + Self::TenBit444 => PixelFormat::P410, + } + } + + #[cfg(target_os = "linux")] + pub(crate) const fn from_linux_pixel_format( + format: opennow_streamer_platform_linux::PixelFormat, + ) -> Option { + use opennow_streamer_platform_linux::PixelFormat; + match format { + PixelFormat::Nv12 | PixelFormat::I420 => Some(Self::EightBit420), + PixelFormat::Nv24 => Some(Self::EightBit444), + PixelFormat::P010 => Some(Self::TenBit420), + PixelFormat::P410 => Some(Self::TenBit444), + PixelFormat::Bgra8 | PixelFormat::Rgba8 => None, + } + } + + #[cfg(target_os = "macos")] + const fn macos_bit_depth(self) -> opennow_streamer_platform_macos::VideoBitDepth { + use opennow_streamer_platform_macos::VideoBitDepth; + match self { + Self::EightBit420 | Self::EightBit444 => VideoBitDepth::Eight, + Self::TenBit420 | Self::TenBit444 => VideoBitDepth::Ten, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MediaStreamConfig { + pub codec: MediaVideoCodec, + /// Color class accepted by CloudMatch and requested again during NVST setup. + pub color_quality: MediaColorQuality, + pub hdr: bool, + pub width: u32, + pub height: u32, + pub fps: u32, + pub bitrate_bps: u32, + /// CloudMatch accepted Cloud G-SYNC and the host presentation path is VRR-capable. + pub cloud_gsync: bool, + pub shortcuts: StreamShortcutBindings, +} + +impl Default for MediaStreamConfig { + fn default() -> Self { + Self { + codec: MediaVideoCodec::H264, + color_quality: MediaColorQuality::default(), + hdr: false, + width: 1920, + height: 1080, + fps: 60, + bitrate_bps: 75_000_000, + cloud_gsync: false, + shortcuts: StreamShortcutBindings::default(), + } + } +} + +#[cfg(target_os = "macos")] +impl MediaStreamConfig { + fn macos_color_space(self) -> opennow_streamer_platform_macos::VideoColorSpace { + use opennow_streamer_platform_macos::VideoColorSpace; + if self.hdr { + VideoColorSpace::Bt2020 + } else { + VideoColorSpace::Bt709 + } + } + + fn macos_transfer(self) -> opennow_streamer_platform_macos::VideoTransfer { + use opennow_streamer_platform_macos::VideoTransfer; + if self.hdr { + VideoTransfer::Pq + } else { + VideoTransfer::Sdr + } + } + + fn macos_h265_format( + self, + parameter_sets: opennow_streamer_platform_macos::H265ParameterSets, + ) -> opennow_streamer_platform_macos::H265Format { + use opennow_streamer_platform_macos::{H265Format, VideoChroma}; + H265Format::new(parameter_sets, self.macos_color_space()) + .with_bit_depth(self.color_quality.macos_bit_depth()) + .with_chroma(if self.color_quality.is_444() { + VideoChroma::Yuv444 + } else { + VideoChroma::Yuv420 + }) + .with_transfer(self.macos_transfer()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MediaCodec { + H264, + H265, + Av1, + Opus { channels: u8 }, + Unsupported(String), +} + +#[derive(Debug, Clone)] +pub struct EncodedFrame { + pub mid: String, + pub codec: MediaCodec, + pub data: Arc<[u8]>, + /// Sender-authored NVST video frame index used by the feedback protocol. + pub frame_index: Option, + pub timestamp: u64, + pub clock_rate_hz: u32, + pub keyframe: bool, + pub contiguous: bool, + pub ssrc: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CapturedInput { + Text(opennow_streamer_protocol::text_input::UnicodeText), + Key { + virtual_key: u16, + modifiers: u16, + pressed: bool, + }, + MouseMove { + delta_x: i16, + delta_y: i16, + }, + MouseAbsolute { + x: u16, + y: u16, + width: u16, + height: u16, + }, + MouseButton { + button: u8, + pressed: bool, + }, + MouseWheel { + delta_x: i16, + delta_y: i16, + }, + Gamepad { + controller_id: u8, + bitmap: u16, + buttons: u16, + left_trigger: u8, + right_trigger: u8, + left_stick_x: i16, + left_stick_y: i16, + right_stick_x: i16, + right_stick_y: i16, + }, + Guide, + Screenshot, + RecordingToggle, + Shortcut(StreamShortcutAction), +} + +const CAPTURED_INPUT_CAPACITY: usize = 256; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CapturedInputSample { + pub input: CapturedInput, + pub captured_at: Instant, +} + +#[derive(Debug, Default)] +pub struct CapturedInputQueue { + pending: Mutex>, + overflowed: AtomicBool, + text_ready: AtomicBool, + text_generation: AtomicU64, + text_slot: opennow_streamer_protocol::text_input::TextInputSlot, +} + +impl CapturedInputQueue { + pub fn set_text_ready(&self, generation: u64, ready: bool) { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = self.text_generation.load(Ordering::Relaxed); + if generation.wrapping_sub(previous) > u64::MAX / 2 { + return; + } + self.text_generation.store(generation, Ordering::Relaxed); + self.text_ready.store(ready, Ordering::Release); + if !ready || generation != previous { + self.text_slot.cancel(); + pending.retain(|sample| !matches!(sample.input, CapturedInput::Text(_))); + } + } + + pub(crate) fn submit_text( + &self, + bytes: &[u8], + ) -> Result<(), opennow_streamer_protocol::text_input::TextInputError> { + use opennow_streamer_protocol::text_input::TextInputError; + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !self.text_ready.load(Ordering::Acquire) { + return Err(TextInputError::Unavailable); + } + if pending.len() >= CAPTURED_INPUT_CAPACITY { + return Err(TextInputError::Busy); + } + let text = self.text_slot.submit(bytes)?; + pending.push_back(CapturedInputSample { + input: CapturedInput::Text(text), + captured_at: Instant::now(), + }); + Ok(()) + } + + pub(crate) fn discard_text(&self) { + self.text_slot.cancel(); + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|sample| !matches!(sample.input, CapturedInput::Text(_))); + } + + pub fn push(&self, input: CapturedInput) { + self.push_sample(CapturedInputSample { + input, + captured_at: Instant::now(), + }); + } + + pub fn push_sample(&self, sample: CapturedInputSample) { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if matches!(&sample.input, CapturedInput::MouseAbsolute { .. }) + && matches!( + pending.back(), + Some(CapturedInputSample { + input: CapturedInput::MouseAbsolute { .. }, + .. + }) + ) + { + pending.pop_back(); + } + if pending.len() >= CAPTURED_INPUT_CAPACITY { + if let Some(index) = pending.iter().position(|event| { + matches!( + &event.input, + CapturedInput::MouseMove { .. } | CapturedInput::MouseAbsolute { .. } + ) + }) { + pending.remove(index); + } else { + self.overflowed.store(true, Ordering::Release); + return; + } + } + pending.push_back(sample); + } + + pub(crate) fn release_gamepad(&self, controller_id: u8, bitmap: u16) { + assert!(controller_id < 4); + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending.retain(|sample| { + !matches!(sample.input, + CapturedInput::Gamepad { controller_id: id, .. } if id == controller_id) + }); + pending.push_back(CapturedInputSample { + input: CapturedInput::Gamepad { + controller_id, + bitmap, + buttons: 0, + left_trigger: 0, + right_trigger: 0, + left_stick_x: 0, + left_stick_y: 0, + right_stick_x: 0, + right_stick_y: 0, + }, + captured_at: Instant::now(), + }); + debug_assert!(pending.len() <= CAPTURED_INPUT_CAPACITY + 4); + } + + pub fn take(&self) -> Option { + self.take_sample().map(|sample| sample.input) + } + + pub fn take_sample(&self) -> Option { + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .pop_front() + } + + pub fn take_overflowed(&self) -> bool { + self.overflowed.swap(false, Ordering::AcqRel) + } + + pub fn clear(&self) { + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.overflowed.store(false, Ordering::Release); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeStageTimings { + pub p50_us: u64, + pub p95_us: u64, + pub max_us: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeTimingsReport { + pub call: Option, + pub residence: Option, + pub call_window_samples: usize, + pub residence_window_samples: usize, + pub submissions_total: u64, + pub outputs_total: u64, + pub output_calls_total: u64, + pub last_submission_at: Option, + pub last_output_at: Option, + pub in_flight: usize, + pub oldest_in_flight_at: Option, + pub epoch: u64, + pub epoch_started_at: Option, + pub unmatched_outputs: u64, + pub unmatched_submissions: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MediaFeedback { + VideoFrameAccepted { + frame_index: Option, + timestamp: u64, + bytes: u32, + keyframe: bool, + }, + PlaybackStarted { + backend: &'static str, + }, + BackendFallback { + from: &'static str, + to: &'static str, + reason: String, + }, + ColorFormatChanged { + requested: MediaColorQuality, + actual: MediaColorQuality, + }, + RequestKeyframe { + mid: String, + reason: String, + }, + DecoderError { + codec: &'static str, + message: String, + }, + AudioDecoderError { + message: String, + consecutive: u32, + }, + AudioUnavailable { + backend: &'static str, + reason: String, + rejected: u64, + }, + QueueDropped { + media: &'static str, + count: usize, + }, + OutputError { + message: String, + }, + DeviceLost { + subsystem: &'static str, + recovered: bool, + message: Option, + }, + DecodeTimings(DecodeTimingsReport), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PushOutcome { + Queued, + DroppedOldest, + Paused, + Unsupported, + Closed, +} + +struct SharedPipeline { + video: Arc, + audio: Arc>, + output: Arc, + feedback: Sender, + paused: AtomicBool, + video_desynced: AtomicBool, + keyframe_requested: AtomicBool, + stopped: AtomicBool, + recording_tap: RecordingTap, + replay_tap: crate::replay::ReplayTap, + stream: MediaStreamConfig, + #[cfg(target_os = "macos")] + mac_sink: Mutex>, + #[cfg(target_os = "macos")] + mac_software_fallback: AtomicBool, + #[cfg(target_os = "windows")] + windows_bridge: Arc, + #[cfg(target_os = "linux")] + linux_session: Mutex>, + #[cfg(target_os = "linux")] + linux_software_fallback: Arc, + #[cfg(target_os = "linux")] + linux_video_mid: Mutex, + #[cfg(target_os = "linux")] + linux_codec: MediaVideoCodec, +} + +#[derive(Clone)] +pub struct MediaSink { + shared: Arc, +} + +impl MediaSink { + pub fn push(&self, frame: EncodedFrame) -> PushOutcome { + if self.shared.stopped.load(Ordering::Acquire) { + return PushOutcome::Closed; + } + if self.shared.paused.load(Ordering::Acquire) { + return PushOutcome::Paused; + } + match frame.codec { + MediaCodec::H264 | MediaCodec::H265 | MediaCodec::Av1 | MediaCodec::Opus { .. } => { + self.shared.recording_tap.publish(&frame); + self.shared.replay_tap.publish(&frame); + } + MediaCodec::Unsupported(_) => {} + } + match frame.codec { + MediaCodec::H264 | MediaCodec::H265 | MediaCodec::Av1 => self.push_video(frame), + MediaCodec::Opus { .. } => self.push_audio(frame), + MediaCodec::Unsupported(_) => PushOutcome::Unsupported, + } + } + + fn push_video(&self, frame: EncodedFrame) -> PushOutcome { + let mid = frame.mid.clone(); + let Ok(result) = self.shared.video.push(frame) else { + return PushOutcome::Closed; + }; + if result.request_keyframe { + opennow_streamer_protocol::log::log_async( + "WARN", + "video-reference", + "compressed chain invalidated; waiting for a fresh keyframe", + ); + let _ = self.shared.feedback.send(MediaFeedback::RequestKeyframe { + mid, + reason: "compressed video discontinuity or queue overflow".to_owned(), + }); + } + if result.dropped > 0 { + let _ = self.shared.feedback.send(MediaFeedback::QueueDropped { + media: "video", + count: result.dropped, + }); + PushOutcome::DroppedOldest + } else { + PushOutcome::Queued + } + } + + fn push_audio(&self, frame: EncodedFrame) -> PushOutcome { + match self.shared.audio.push(frame) { + PushResult::Queued => PushOutcome::Queued, + PushResult::DroppedOldest => { + let _ = self.shared.feedback.send(MediaFeedback::QueueDropped { + media: "audio", + count: 1, + }); + PushOutcome::DroppedOldest + } + PushResult::Closed => PushOutcome::Closed, + } + } +} + +pub(crate) struct MediaSessionConfig<'a> { + pub(crate) stream: MediaStreamConfig, + pub(crate) audio_device: &'a opennow_streamer_protocol::AudioOutputDevice, +} + +pub struct MediaSession { + sink: MediaSink, + video_worker: Option>, + audio_worker: Option>, + #[cfg(target_os = "linux")] + linux_monitor: Option>, + embedded_host_worker: Option>, + #[cfg(target_os = "windows")] + embedded_d3d11: Option>>, + embedded_frames: Option, + host_commands: Sender, +} + +#[derive(Clone)] +pub struct MediaControl { + shared: Arc, + host_commands: Sender, +} + +impl MediaSession { + pub fn captured_input(&self) -> Arc { + self.sink.shared.output.captured_input() + } + + pub(crate) fn spawn( + output: Arc, + feedback: Sender, + host_commands: Sender, + use_hardware: bool, + stream: MediaStreamConfig, + #[cfg(target_os = "windows")] windows_bridge: Arc, + #[cfg(target_os = "linux")] linux_selection: LinuxVideoSelection, + #[cfg(target_os = "linux")] linux_software_fallback: Arc, + ) -> Result { + #[cfg(not(target_os = "linux"))] + let _ = &stream; + #[cfg(target_os = "macos")] + if use_hardware { + return Self::spawn_macos(output, feedback, host_commands, stream); + } + #[cfg(target_os = "windows")] + let use_windows_backend = windows_bridge.backend().is_some(); + #[cfg(target_os = "windows")] + let _ = use_hardware; + #[cfg(not(target_os = "windows"))] + let use_windows_backend = false; + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + let _ = use_hardware; + #[cfg(target_os = "linux")] + if !linux_software_fallback.load(Ordering::Acquire) + && let LinuxVideoPath::Hardware(decoder_preference) = linux_selection.path + { + match Self::spawn_linux( + Arc::clone(&output), + feedback.clone(), + host_commands.clone(), + stream, + decoder_preference, + Arc::clone(&linux_software_fallback), + ) { + Ok(session) => return Ok(session), + Err(reason) => { + if stream.codec != MediaVideoCodec::H264 { + return Err(format!( + "Linux {} decoder startup failed: {reason}", + stream.codec.label().to_ascii_uppercase() + )); + } + linux_software_fallback.store(true, Ordering::Release); + let _ = host_commands.send(HostCommand::FallbackLinux { + reason: format!("Linux hardware media startup failed: {reason}"), + }); + } + } + } + if !use_windows_backend && stream.codec != MediaVideoCodec::H264 { + return Err(format!( + "{} requires the Windows hardware decoder", + stream.codec.label().to_ascii_uppercase() + )); + } + let video_decoder = (!use_windows_backend).then(H264Decoder::new).transpose()?; + let audio_decoder = OpusDecoder::new(2)?; + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + #[cfg(target_os = "macos")] + mac_sink: Mutex::new(None), + #[cfg(target_os = "macos")] + mac_software_fallback: AtomicBool::new(false), + #[cfg(target_os = "windows")] + windows_bridge, + #[cfg(target_os = "linux")] + linux_session: Mutex::new(None), + #[cfg(target_os = "linux")] + linux_software_fallback, + #[cfg(target_os = "linux")] + linux_video_mid: Mutex::new(String::new()), + #[cfg(target_os = "linux")] + linux_codec: stream.codec, + }); + let video_shared = Arc::clone(&shared); + let video_worker = thread::Builder::new() + .name(format!("opennow-{}-decode", stream.codec.label())) + .spawn(move || { + #[cfg(target_os = "windows")] + if use_windows_backend { + run_windows_video(video_shared, stream.fps); + return; + } + run_video_decoder( + video_shared, + video_decoder.expect("software decoder was initialized"), + ); + }) + .map_err(|error| format!("failed to start video decoder worker: {error}"))?; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-opus-decode".to_owned()) + .spawn(move || run_audio_decoder(audio_shared, audio_decoder)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + return Err(format!("failed to start Opus decoder worker: {error}")); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + #[cfg(target_os = "linux")] + linux_monitor: None, + embedded_host_worker: None, + #[cfg(target_os = "windows")] + embedded_d3d11: None, + embedded_frames: None, + host_commands, + }) + } + + #[cfg(target_os = "macos")] + fn spawn_macos( + output: Arc, + feedback: Sender, + host_commands: Sender, + stream: MediaStreamConfig, + ) -> Result { + let shared = Arc::new(SharedPipeline { + // Keep a bounded scheduler-burst reserve. The VideoToolbox worker drains this queue + // asynchronously; decoded frames remain latest-first at the Metal presentation edge. + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + mac_sink: Mutex::new(None), + mac_software_fallback: AtomicBool::new(false), + }); + let video_shared = Arc::clone(&shared); + let video_commands = host_commands.clone(); + let video_worker = thread::Builder::new() + .name(format!( + "opennow-videotoolbox-{}-submit", + stream.codec.label() + )) + .spawn(move || match stream.codec { + MediaVideoCodec::H264 => { + run_macos_h264_video(video_shared, video_commands, stream.fps, true); + } + MediaVideoCodec::H265 => { + run_macos_h265_video(video_shared, video_commands, stream.fps); + } + MediaVideoCodec::Av1 => { + run_macos_av1_video(video_shared, video_commands, stream); + } + }) + .map_err(|error| format!("failed to start VideoToolbox submit worker: {error}"))?; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-coreaudio-submit".to_owned()) + .spawn(move || run_macos_audio(audio_shared)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + return Err(format!("failed to start CoreAudio submit worker: {error}")); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + #[cfg(target_os = "linux")] + linux_monitor: None, + embedded_host_worker: None, + embedded_frames: None, + host_commands, + }) + } + + #[cfg(target_os = "linux")] + fn spawn_linux( + output: Arc, + feedback: Sender, + host_commands: Sender, + stream: MediaStreamConfig, + decoder_preference: opennow_streamer_platform_linux::DecoderPreference, + software_fallback: Arc, + ) -> Result { + let format = opennow_streamer_platform_linux::StreamFormat::video_default( + stream.width, + stream.height, + ) + .map_err(|error| error.to_string())?; + let mut config = opennow_streamer_platform_linux::SessionConfig::new(format); + config.codec = match stream.codec { + MediaVideoCodec::H264 => opennow_streamer_platform_linux::VideoCodec::H264, + MediaVideoCodec::H265 => opennow_streamer_platform_linux::VideoCodec::H265, + MediaVideoCodec::Av1 => opennow_streamer_platform_linux::VideoCodec::Av1, + }; + config.decoder_preference = decoder_preference; + config.audio = None; + let session = opennow_streamer_platform_linux::LinuxSession::start(config) + .map_err(|error| error.to_string())?; + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + linux_session: Mutex::new(Some(session)), + linux_software_fallback: software_fallback, + linux_video_mid: Mutex::new(String::new()), + linux_codec: stream.codec, + }); + let video_shared = Arc::clone(&shared); + let video_commands = host_commands.clone(); + let video_worker = thread::Builder::new() + .name("opennow-linux-video-submit".to_owned()) + .spawn(move || run_linux_video(video_shared, video_commands)) + .map_err(|error| format!("failed to start Linux video submit worker: {error}"))?; + let audio_decoder = OpusDecoder::new(2)?; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-opus-decode".to_owned()) + .spawn(move || run_audio_decoder(audio_shared, audio_decoder)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + return Err(format!( + "failed to start Linux audio submit worker: {error}" + )); + } + }; + let monitor_shared = Arc::clone(&shared); + let monitor_commands = host_commands.clone(); + let linux_monitor = match thread::Builder::new() + .name("opennow-linux-media-events".to_owned()) + .spawn(move || run_linux_monitor(monitor_shared, monitor_commands)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + shared.audio.close(); + let _ = video_worker.join(); + let _ = audio_worker.join(); + return Err(format!("failed to start Linux media monitor: {error}")); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + linux_monitor: Some(linux_monitor), + embedded_host_worker: None, + embedded_frames: None, + host_commands, + }) + } + + pub(crate) fn spawn_embedded( + output: Arc, + feedback: Sender, + host_commands: Sender, + config: MediaSessionConfig<'_>, + frames: crate::GraphicsFramePublisher, + #[cfg(target_os = "linux")] linux_selection: LinuxVideoSelection, + #[cfg(target_os = "linux")] vulkan_device: Option>, + ) -> Result { + #[cfg(target_os = "linux")] + { + return Self::spawn_embedded_linux( + output, + feedback, + host_commands, + config, + frames, + linux_selection, + vulkan_device, + ); + } + #[cfg(target_os = "macos")] + { + return Self::spawn_embedded_macos( + output, + feedback, + host_commands, + config.stream, + config.audio_device, + frames, + ); + } + #[cfg(target_os = "windows")] + { + return Self::spawn_embedded_windows( + output, + feedback, + host_commands, + config.stream, + frames, + ); + } + #[allow(unreachable_code)] + Err("embedded media is unsupported on this platform".to_owned()) + } + + #[cfg(target_os = "linux")] + fn spawn_embedded_linux( + output: Arc, + feedback: Sender, + host_commands: Sender, + config: MediaSessionConfig<'_>, + frames: crate::GraphicsFramePublisher, + linux_selection: LinuxVideoSelection, + vulkan_device: Option>, + ) -> Result { + let MediaSessionConfig { + stream, + audio_device, + } = config; + let LinuxVideoPath::Hardware(decoder_preference) = linux_selection.path else { + return Err(linux_selection.fallback_reason.unwrap_or_else(|| { + "embedded Linux output requires a native NV12 decoder".to_owned() + })); + }; + if decoder_preference == opennow_streamer_platform_linux::DecoderPreference::SoftwareOnly + && (stream.hdr || stream.color_quality != crate::MediaColorQuality::EightBit420) + { + return Err( + "Software decoding presents 8-bit 4:2:0 SDR only. Select 8-bit 4:2:0, or disable HDR, or select a hardware backend.".to_owned(), + ); + } + let format = opennow_streamer_platform_linux::StreamFormat::video_default( + stream.width, + stream.height, + ) + .map_err(|error| error.to_string())?; + let mut config = opennow_streamer_platform_linux::SessionConfig::new(format); + config.codec = match stream.codec { + MediaVideoCodec::H264 => opennow_streamer_platform_linux::VideoCodec::H264, + MediaVideoCodec::H265 => opennow_streamer_platform_linux::VideoCodec::H265, + MediaVideoCodec::Av1 => opennow_streamer_platform_linux::VideoCodec::Av1, + }; + config.decoder_preference = decoder_preference; + config.embedded_presentation = true; + config.stream_format.pixel_format = stream.color_quality.linux_pixel_format(); + if stream.hdr { + config.stream_format.color_transfer = + opennow_streamer_platform_linux::ColorTransfer::Pq; + config.stream_format.color_primaries = + opennow_streamer_platform_linux::ColorPrimaries::Bt2020; + config.stream_format.color_matrix = + opennow_streamer_platform_linux::ColorMatrix::Bt2020; + } + if let Some(audio) = config.audio.as_mut() { + audio.muted = Arc::clone(&output.audio_muted); + audio.output_device = audio_device.device_name().unwrap_or_default().to_owned(); + } + if stream.color_quality.is_444() + && (decoder_preference + != opennow_streamer_platform_linux::DecoderPreference::VulkanOnly + || !vulkan_device.as_ref().is_some_and(|device| { + device.supports_format( + config.codec, + config.stream_format.pixel_format, + stream.width, + stream.height, + ) + })) + { + return Err("embedded Linux decode does not support negotiated 4:4:4 color".to_owned()); + } + if decoder_preference == opennow_streamer_platform_linux::DecoderPreference::VulkanOnly { + let device = vulkan_device.as_ref().ok_or_else(|| { + "embedded Vulkan decode requires an attached shared device".to_owned() + })?; + if !device.supports_format( + config.codec, + config.stream_format.pixel_format, + stream.width, + stream.height, + ) { + return Err("the attached Vulkan device does not support the negotiated codec, color depth, or dimensions".to_owned()); + } + } else if stream.color_quality.bit_depth() == 10 + && decoder_preference != opennow_streamer_platform_linux::DecoderPreference::VaApiOnly + { + return Err("negotiated 10-bit embedded Linux decode requires Vulkan Video or VAAPI P010 import".to_owned()); + } + config.vulkan_device = vulkan_device; + let session = opennow_streamer_platform_linux::LinuxSession::start(config) + .map_err(|error| error.to_string())?; + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + linux_session: Mutex::new(Some(session)), + linux_software_fallback: Arc::new(AtomicBool::new(false)), + linux_video_mid: Mutex::new(String::new()), + linux_codec: stream.codec, + }); + let video_shared = Arc::clone(&shared); + let video_worker = thread::Builder::new() + .name("opennow-embedded-linux-video-submit".to_owned()) + .spawn(move || run_embedded_linux_video(video_shared)) + .map_err(|error| format!("failed to start embedded Linux video submitter: {error}"))?; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-embedded-linux-audio-submit".to_owned()) + .spawn(move || run_embedded_linux_audio(audio_shared)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + stop_linux_session(&shared); + return Err(format!( + "failed to start embedded Linux audio submitter: {error}" + )); + } + }; + let producer = crate::LinuxGpuFrameProducer::new(8).map_err(|error| error.to_string())?; + let embedded_frames = frames.clone(); + let monitor_shared = Arc::clone(&shared); + let monitor_label = linux_embedded_backend_label(decoder_preference); + let linux_monitor = match thread::Builder::new() + .name("opennow-embedded-linux-frame-publisher".to_owned()) + .spawn(move || { + run_embedded_linux_monitor(monitor_shared, frames, producer, monitor_label) + }) { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + shared.audio.close(); + let _ = video_worker.join(); + let _ = audio_worker.join(); + stop_linux_session(&shared); + return Err(format!( + "failed to start embedded Linux frame publisher: {error}" + )); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + linux_monitor: Some(linux_monitor), + embedded_host_worker: None, + embedded_frames: Some(embedded_frames), + host_commands, + }) + } + + #[cfg(target_os = "macos")] + fn spawn_embedded_macos( + output: Arc, + feedback: Sender, + _host_commands: Sender, + stream: MediaStreamConfig, + audio_device: &opennow_streamer_protocol::AudioOutputDevice, + frames: crate::GraphicsFramePublisher, + ) -> Result { + if stream.color_quality.is_444() + && (stream.codec != MediaVideoCodec::H265 + || stream.color_quality != MediaColorQuality::TenBit444 + || !opennow_streamer_platform_macos::probe_h265_444_ten_bit_hardware()) + { + return Err( + "embedded VideoToolbox decode does not support negotiated 4:4:4 color".to_owned(), + ); + } + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + mac_sink: Mutex::new(None), + mac_software_fallback: AtomicBool::new(false), + }); + let audio_output_device = audio_device.device_name().map(str::to_owned); + let audio_muted = Arc::clone(&shared.output.audio_muted); + let (host_commands, host_receiver) = std::sync::mpsc::channel(); + let embedded_frames = frames.clone(); + let embedded_host_worker = thread::Builder::new() + .name("opennow-embedded-videotoolbox-host".to_owned()) + .spawn(move || { + use opennow_streamer_platform_macos::{ + AudioFormat, EmbeddedBackendConfig, H264Format, MacOsBackend, QueueLimits, + VideoColorSpace, + }; + + let mut backend: Option = None; + let mut paused = false; + while let Ok(command) = host_receiver.recv() { + let start = |video| { + let publisher = frames.clone(); + MacOsBackend::start_embedded_with_publisher( + EmbeddedBackendConfig { + audio_muted: Arc::clone(&audio_muted), + video, + audio: AudioFormat::OPUS_STEREO_48KHZ, + audio_output_device: audio_output_device.clone(), + queues: QueueLimits::default(), + }, + move |frame| { + let Some(lease) = publisher.context() else { + return false; + }; + matches!( + publisher.publish(lease, Arc::new(frame)), + Ok(crate::GraphicsPublishOutcome::Replaced) + ) + }, + ) + }; + match command { + HostCommand::Start { reply, .. } => { + let _ = reply + .send(Err("embedded VideoToolbox host is already session-scoped" + .to_owned())); + } + HostCommand::ConfigureMacH264 { + parameter_sets, + reply, + } => { + let result = start( + H264Format::new(parameter_sets, VideoColorSpace::Bt709).into(), + ) + .and_then(|mut started| { + started.set_paused(paused)?; + let sink = started.sink(); + backend = Some(started); + Ok(crate::runtime::MacH264Configuration::Hardware(sink)) + }) + .map_err(|error| error.to_string()); + let _ = reply.send(result); + } + HostCommand::ConfigureMacH265 { + parameter_sets, + reply, + } => { + let result = start(stream.macos_h265_format(parameter_sets).into()) + .and_then(|mut started| { + started.set_paused(paused)?; + let sink = started.sink(); + backend = Some(started); + Ok(sink) + }) + .map_err(|error| error.to_string()); + let _ = reply.send(result); + } + HostCommand::ConfigureMacAv1 { format, reply } => { + let result = start(format.into()) + .and_then(|mut started| { + started.set_paused(paused)?; + let sink = started.sink(); + backend = Some(started); + Ok(sink) + }) + .map_err(|error| error.to_string()); + let _ = reply.send(result); + } + HostCommand::Pause { + paused: new_paused, + reply, + } => { + let result = backend.as_mut().map_or(Ok(()), |backend| { + backend + .set_paused(new_paused) + .map_err(|error| error.to_string()) + }); + if result.is_ok() { + paused = new_paused; + } + if let Some(reply) = reply { + let _ = reply.send(result); + } + } + HostCommand::Stop | HostCommand::Shutdown => break, + HostCommand::StartMicrophone { reply, .. } => { + let _ = reply.send(Err( + "microphone capture is owned by the embedded runtime host" + .to_owned(), + )); + } + HostCommand::Surface { reply, .. } | HostCommand::Control { reply, .. } => { + let _ = reply.send(Ok(())); + } + HostCommand::AudioDevices(request) => { + let _ = request.reply.send(Err( + "Audio enumeration is owned by the embedded runtime host" + .to_owned(), + )); + } + HostCommand::Cursor(_) => {} + } + } + }) + .map_err(|error| format!("failed to start embedded VideoToolbox host: {error}"))?; + let video_shared = Arc::clone(&shared); + let video_commands = host_commands.clone(); + let video_worker = match thread::Builder::new() + .name(format!( + "opennow-embedded-videotoolbox-{}-submit", + stream.codec.label() + )) + .spawn(move || match stream.codec { + MediaVideoCodec::H264 => { + run_macos_h264_video(video_shared, video_commands, stream.fps, false); + } + MediaVideoCodec::H265 => { + run_macos_h265_video(video_shared, video_commands, stream.fps); + } + MediaVideoCodec::Av1 => run_macos_av1_video(video_shared, video_commands, stream), + }) { + Ok(worker) => worker, + Err(error) => { + let _ = host_commands.send(HostCommand::Shutdown); + let _ = embedded_host_worker.join(); + return Err(format!( + "failed to start embedded VideoToolbox submitter: {error}" + )); + } + }; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-embedded-coreaudio-submit".to_owned()) + .spawn(move || run_macos_audio(audio_shared)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + let _ = host_commands.send(HostCommand::Shutdown); + let _ = embedded_host_worker.join(); + return Err(format!( + "failed to start embedded CoreAudio submitter: {error}" + )); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + embedded_host_worker: Some(embedded_host_worker), + embedded_frames: Some(embedded_frames), + host_commands, + }) + } + + #[cfg(target_os = "windows")] + fn spawn_embedded_windows( + output: Arc, + feedback: Sender, + host_commands: Sender, + stream: MediaStreamConfig, + frames: crate::GraphicsFramePublisher, + ) -> Result { + let bridge = Arc::new(WindowsBridge::new()); + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + windows_bridge: bridge, + }); + // Keep compressed-frame submission independent from the render-owned decoder state. + // The old path made the 120 Hz submit worker take the same mutex that Qt held while + // Media Foundation polled output and D3D11 converted a frame. A single slow AV1/P010 + // decode step could therefore fill the two-frame ingress queue, discard the prediction + // chain, and repeatedly reset an otherwise healthy decoder while audio continued. + let submission = Arc::new(Mutex::new(EmbeddedD3d11Submission::new())); + let producer = Arc::new(Mutex::new(EmbeddedD3d11State::new( + stream, + Arc::clone(&submission), + ))); + let notifier = Arc::new(EmbeddedD3d11FrameNotifier { + state: Arc::downgrade(&producer), + frames: frames.clone(), + shared: Arc::clone(&shared), + stream, + mid: Mutex::new(String::new()), + presentation_time_ns: AtomicU64::new(0), + sequence: AtomicU64::new(0), + }); + let weak_notifier = Arc::downgrade(¬ifier); + producer + .lock() + .unwrap_or_else(|error| error.into_inner()) + .set_frame_ready(Arc::new(move || { + if let Some(notifier) = weak_notifier.upgrade() { + notifier.publish_decoded(); + } + })); + let embedded_frames = frames.clone(); + let video_shared = Arc::clone(&shared); + let video_submission = Arc::clone(&submission); + let video_notifier = Arc::clone(¬ifier); + let video_worker = thread::Builder::new() + .name("opennow-embedded-d3d11-submit".to_owned()) + .spawn(move || { + opennow_streamer_protocol::log::log_line("INFO", "d3d11-submit", &format!( + "worker started width={} height={} fps={}; awaiting compressed video", stream.width, stream.height, stream.fps)); + let mut received = 0_u64; + let mut skipped_paused = 0_u64; + let mut skipped_reference = 0_u64; + let mut deferred_keyframe_request = false; + let mut last_report = Instant::now(); + let mut clock = AdaptiveSampleClock::new(stream.fps); + while let Some(packet) = video_shared.video.pop_packet() { + let frame = packet.frame; + received += 1; + if received == 1 || last_report.elapsed() >= Duration::from_secs(10) { + opennow_streamer_protocol::log::log_async("INFO", "d3d11-submit", &format!( + "received={received} skipped_paused={skipped_paused} skipped_reference={skipped_reference} keyframe={} bytes={}", frame.keyframe, frame.data.len())); + last_report = Instant::now(); + } + if video_shared.paused.load(Ordering::Acquire) { + skipped_paused += 1; + continue; + } + let timestamp_100ns = + media_timestamp_100ns(frame.timestamp, frame.clock_rate_hz); + let duration_100ns = clock.observe(timestamp_100ns); + let encoded = opennow_streamer_platform_windows::EncodedVideoFrame { + codec: match frame.codec { + MediaCodec::H264 => opennow_streamer_platform_windows::VideoCodec::H264, + MediaCodec::H265 => opennow_streamer_platform_windows::VideoCodec::H265, + MediaCodec::Av1 => opennow_streamer_platform_windows::VideoCodec::Av1, + _ => continue, + }, + data: frame.data.to_vec(), + timestamp_100ns, + duration_100ns, + key_frame: frame.keyframe, + reset_decoder: packet.reset_decoder, + }; + // Validate after allocation, under the same lock that publishes loss. + // Only compressed-queue admission runs here, never GPU/codec work. + let Some(submission_result) = video_shared.video.submit_if_current( + packet.generation, + || video_submission + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(encoded), + ) else { + skipped_reference += 1; + continue; + }; + let submission = match submission_result { + Ok(submission) => submission, + Err(message) => { + let _ = video_shared.feedback.send(MediaFeedback::DecoderError { + codec: match frame.codec { + MediaCodec::H264 => "h264", + MediaCodec::H265 => "h265", + MediaCodec::Av1 => "av1", + _ => "video", + }, + message, + }); + invalidate_embedded_video( + &video_shared, + &frame.mid, + "embedded D3D11 submission failed", + ); + continue; + } + }; + if submission.dropped > 0 { + if !(submission.preserved_keyframe && deferred_keyframe_request) { + eprintln!( + "Embedded D3D11 compressed queue overflow: dropped={} recoveryKeyframeRetained={}", + submission.dropped, + (submission.queued && frame.keyframe) || submission.preserved_keyframe, + ); + } + let _ = video_shared.feedback.send(MediaFeedback::QueueDropped { + media: "d3d11-video", + count: submission.dropped, + }); + } + if !submission.queued { + if submission.preserved_keyframe { + if !deferred_keyframe_request { + deferred_keyframe_request = true; + opennow_streamer_protocol::log::log_async( + "WARN", + "video-reference", + "embedded D3D11 compressed queue retained its keyframe under backpressure; keyframe request deferred until the decoder dequeues it", + ); + } + continue; + } + let reason = if std::mem::take(&mut deferred_keyframe_request) { + "embedded D3D11 decoder dequeued its retained keyframe after a dropped delta" + } else { + "embedded D3D11 compressed-video queue overflow" + }; + // One request per gap. Repeating this for every delta + // clears `request_pending` and asks the sender for a + // new IDR faster than the decoder can consume the last + // one, which is what freezes the picture. + if !video_shared + .keyframe_requested + .swap(true, Ordering::AcqRel) + { + invalidate_embedded_video(&video_shared, &frame.mid, reason); + } + continue; + } + if frame.keyframe { + deferred_keyframe_request = false; + video_shared + .keyframe_requested + .store(false, Ordering::Release); + } + report_video_frame_accepted(&video_shared, &frame); + if submission.needs_graphics { + video_notifier.publish_initial( + &frame.mid, + u64::try_from(timestamp_100ns.max(0)) + .unwrap_or(0) + .saturating_mul(100), + ); + } + } + }) + .map_err(|error| format!("failed to start embedded D3D11 submitter: {error}"))?; + let audio_shared = Arc::clone(&shared); + let audio_decoder = OpusDecoder::new(2)?; + let audio_worker = match thread::Builder::new() + .name("opennow-embedded-opus-decode".to_owned()) + .spawn(move || run_audio_decoder(audio_shared, audio_decoder)) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + return Err(format!("failed to start embedded Opus decoder: {error}")); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + embedded_host_worker: None, + embedded_d3d11: Some(producer), + embedded_frames: Some(embedded_frames), + host_commands, + }) + } + + #[cfg(feature = "test-runtime")] + pub(crate) fn spawn_test( + output: Arc, + feedback: Sender, + host_commands: Sender, + stream: MediaStreamConfig, + #[cfg(target_os = "windows")] windows_bridge: Arc, + #[cfg(target_os = "linux")] linux_software_fallback: Arc, + ) -> Result { + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(video_queue_capacity(stream.fps))), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output, + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(false), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + #[cfg(target_os = "macos")] + mac_sink: Mutex::new(None), + #[cfg(target_os = "macos")] + mac_software_fallback: AtomicBool::new(false), + #[cfg(target_os = "windows")] + windows_bridge, + #[cfg(target_os = "linux")] + linux_session: Mutex::new(None), + #[cfg(target_os = "linux")] + linux_software_fallback, + #[cfg(target_os = "linux")] + linux_video_mid: Mutex::new(String::new()), + #[cfg(target_os = "linux")] + linux_codec: stream.codec, + }); + let video_shared = Arc::clone(&shared); + let video_worker = thread::Builder::new() + .name("opennow-test-video-consumer".to_owned()) + .spawn(move || { + while let Some(frame) = video_shared.video.pop() { + report_video_frame_accepted(&video_shared, &frame); + } + }) + .map_err(|error| format!("failed to start test video consumer: {error}"))?; + let audio_shared = Arc::clone(&shared); + let audio_worker = match thread::Builder::new() + .name("opennow-test-audio-consumer".to_owned()) + .spawn(move || while audio_shared.audio.pop().is_some() {}) + { + Ok(worker) => worker, + Err(error) => { + shared.video.close(); + let _ = video_worker.join(); + return Err(format!("failed to start test audio consumer: {error}")); + } + }; + Ok(Self { + sink: MediaSink { shared }, + video_worker: Some(video_worker), + audio_worker: Some(audio_worker), + #[cfg(target_os = "linux")] + linux_monitor: None, + embedded_host_worker: None, + #[cfg(target_os = "windows")] + embedded_d3d11: None, + embedded_frames: None, + host_commands, + }) + } + + pub fn sink(&self) -> MediaSink { + self.sink.clone() + } + + pub fn control(&self) -> MediaControl { + MediaControl { + shared: Arc::clone(&self.sink.shared), + host_commands: self.host_commands.clone(), + } + } + + pub fn set_paused(&self, paused: bool) { + self.sink.shared.paused.store(paused, Ordering::Release); + self.sink.shared.video.clear(); + self.sink.shared.audio.clear(); + self.sink.shared.output.clear(); + if let Some(frames) = &self.embedded_frames { + frames.clear(); + } + #[cfg(target_os = "windows")] + if let Some(state) = &self.embedded_d3d11 { + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .reset(); + } + #[cfg(target_os = "linux")] + if let Some(session) = self + .sink + .shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + { + let _ = session.set_paused(paused); + } + if !paused { + self.sink + .shared + .video_desynced + .store(true, Ordering::Release); + self.sink + .shared + .keyframe_requested + .store(false, Ordering::Release); + } + let _ = self.host_commands.send(HostCommand::Pause { + paused, + reply: None, + }); + } + + pub fn stop(mut self) { + self.stop_inner(); + } + + fn stop_inner(&mut self) { + self.control().stop(); + if let Some(worker) = self.video_worker.take() { + let _ = worker.join(); + } + if let Some(worker) = self.audio_worker.take() { + let _ = worker.join(); + } + #[cfg(target_os = "linux")] + if let Some(worker) = self.linux_monitor.take() { + let _ = worker.join(); + } + #[cfg(target_os = "macos")] + { + self.sink + .shared + .mac_sink + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + } + if let Some(worker) = self.embedded_host_worker.take() { + let _ = worker.join(); + } + // Clear after all publishers stop, so a late decoder callback cannot + // republish the previous session's surface into the next session. + if let Some(frames) = &self.embedded_frames { + frames.clear(); + } + } +} + +#[cfg(target_os = "windows")] +struct EmbeddedD3d11State { + format: opennow_streamer_platform_windows::VideoFormat, + submission: Arc>, + producer: Option<(crate::GraphicsContext, crate::D3d11FrameProducer)>, + frame_ready: Arc, + keyframe_required: bool, + first_frame_recorded: bool, + reported_pixel_format: Option, +} + +#[cfg(target_os = "windows")] +struct EmbeddedD3d11Submission { + pending: VecDeque, + submitter: Option, + gap: opennow_streamer_platform_windows::ReferenceGap, +} + +#[cfg(target_os = "windows")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EmbeddedD3d11SubmissionOutcome { + queued: bool, + dropped: usize, + needs_graphics: bool, + /// The decoder queue still holds the keyframe in front of a dropped delta, + /// so this delta was dropped and its keyframe request deferred. + preserved_keyframe: bool, +} + +#[cfg(target_os = "windows")] +impl EmbeddedD3d11State { + fn new(stream: MediaStreamConfig, submission: Arc>) -> Self { + use opennow_streamer_platform_windows::{ + VideoChromaFormat, VideoCodec, VideoFormat, VideoPixelFormat, + }; + use std::num::NonZeroU32; + + Self { + format: VideoFormat { + codec: match stream.codec { + MediaVideoCodec::H264 => VideoCodec::H264, + MediaVideoCodec::H265 => VideoCodec::H265, + MediaVideoCodec::Av1 => VideoCodec::Av1, + }, + width: stream.width, + height: stream.height, + frame_rate_numerator: NonZeroU32::new(stream.fps.max(1)).expect("non-zero fps"), + frame_rate_denominator: NonZeroU32::new(1).expect("one is non-zero"), + average_bitrate: stream.bitrate_bps.max(1), + pixel_format: match ( + stream.color_quality.bit_depth(), + stream.color_quality.is_444(), + ) { + (10, true) => VideoPixelFormat::Y410, + (10, false) => VideoPixelFormat::P010, + (_, true) => VideoPixelFormat::Ayuv, + _ => VideoPixelFormat::Nv12, + }, + chroma_format: if stream.color_quality.is_444() { + VideoChromaFormat::Cs444 + } else { + VideoChromaFormat::Cs420 + }, + full_range: false, + chroma_siting: opennow_streamer_platform_windows::VideoChromaSiting::Left, + transfer_function: if stream.hdr { + opennow_streamer_platform_windows::VideoTransferFunction::Pq + } else { + opennow_streamer_platform_windows::VideoTransferFunction::Sdr + }, + color_primaries: if stream.hdr { + opennow_streamer_platform_windows::VideoColorPrimaries::Bt2020 + } else { + opennow_streamer_platform_windows::VideoColorPrimaries::Bt709 + }, + color_matrix: if stream.hdr { + opennow_streamer_platform_windows::VideoColorMatrix::Bt2020 + } else { + opennow_streamer_platform_windows::VideoColorMatrix::Bt709 + }, + }, + submission, + producer: None, + frame_ready: Arc::new(|| {}), + keyframe_required: false, + first_frame_recorded: false, + reported_pixel_format: None, + } + } + + fn set_frame_ready(&mut self, frame_ready: Arc) { + self.frame_ready = frame_ready; + } + + fn reset(&mut self) { + detach_d3d11_producer(&self.submission, &mut self.producer); + self.keyframe_required = false; + } + + fn take_keyframe_required(&mut self) -> bool { + std::mem::take(&mut self.keyframe_required) + } + + fn take_first_recorded_frame(&mut self, succeeded: bool) -> bool { + succeeded && !std::mem::replace(&mut self.first_frame_recorded, true) + } + + fn record( + &mut self, + context: crate::GraphicsContext, + command: crate::GraphicsRecordCommand, + ) -> Result<(crate::GraphicsRecordedFrame, Option), crate::GraphicsFrameError> + { + use opennow_streamer_platform_windows::{ + AdoptedD3d11Context, D3d11FrameProducer, WindowsDecoderMode, + }; + + if context.api != crate::GraphicsApi::D3d11 { + return Err( + "an embedded Windows frame requires a D3D11 graphics context" + .to_owned() + .into(), + ); + } + if self + .producer + .as_ref() + .is_none_or(|(adopted, _)| *adopted != context) + { + let replacing_context = self.producer.is_some(); + if replacing_context { + // Frames submitted through the old Qt device cannot be replayed on the new one. + // Switch the submit path back to its bounded pre-context queue before adopting + // the replacement device, then request one clean prediction chain. + detach_d3d11_producer(&self.submission, &mut self.producer); + self.keyframe_required = true; + } + let (producer, submitter) = unsafe { + D3d11FrameProducer::new( + AdoptedD3d11Context { + device: context.device as *mut std::ffi::c_void, + immediate_context: context.queue as *mut std::ffi::c_void, + }, + self.format, + WindowsDecoderMode::Hardware, + Arc::clone(&self.frame_ready), + ) + } + .map_err(|error| error.to_string())?; + let dropped = self + .submission + .lock() + .unwrap_or_else(|error| error.into_inner()) + .install(submitter)?; + if dropped > 0 { + self.keyframe_required = true; + } + self.producer = Some((context, producer)); + } + let (_, producer) = self.producer.as_ref().expect("producer initialized"); + while let Some(event) = producer.try_event() { + match event { + opennow_streamer_platform_windows::BackendEvent::KeyFrameRequired + | opennow_streamer_platform_windows::BackendEvent::DeviceLost { + subsystem: opennow_streamer_platform_windows::Subsystem::VideoDecode, + .. + } => self.keyframe_required = true, + _ => {} + } + } + let frame = producer + .acquire_latest() + .map_err(|error| error.to_string())? + .ok_or(crate::GraphicsFrameError::NotReady)?; + let actual_format = frame.format(); + let recorded = unsafe { + frame.record( + AdoptedD3d11Context { + device: context.device as *mut std::ffi::c_void, + immediate_context: context.queue as *mut std::ffi::c_void, + }, + command.frame_slot, + ) + } + .map_err(|error| error.to_string())?; + let color_change = if self.reported_pixel_format != Some(actual_format.pixel_format) { + use opennow_streamer_platform_windows::VideoPixelFormat; + let quality = |format| match format { + VideoPixelFormat::Nv12 => MediaColorQuality::EightBit420, + VideoPixelFormat::P010 => MediaColorQuality::TenBit420, + VideoPixelFormat::Ayuv => MediaColorQuality::EightBit444, + VideoPixelFormat::Y410 => MediaColorQuality::TenBit444, + }; + self.reported_pixel_format = Some(actual_format.pixel_format); + Some(MediaFeedback::ColorFormatChanged { + requested: quality(self.format.pixel_format), + actual: quality(actual_format.pixel_format), + }) + } else { + None + }; + Ok(( + crate::GraphicsRecordedFrame { + resource: recorded.texture as usize as u64, + resource_view: 0, + color_space: recorded.color_space.into(), + texture_format: match recorded.texture_format { + opennow_streamer_platform_windows::D3d11TextureFormat::Rgba8 => { + crate::GraphicsTextureFormat::Rgba8 + } + opennow_streamer_platform_windows::D3d11TextureFormat::Rgb10A2 => { + crate::GraphicsTextureFormat::Rgb10A2 + } + }, + width: recorded.width, + height: recorded.height, + frame_slot: recorded.frame_slot, + generation: recorded.generation, + presentation_time_ns: recorded.presentation_time_ns, + }, + color_change, + )) + } +} + +#[cfg(target_os = "windows")] +fn detach_d3d11_producer(submission: &Mutex, producer: &mut Option) { + // Drop closes the decoder queue; its worker retains its resources until exit. + // First remove the submitter under the submission lock. Concurrent frames use the staging queue, + // never a closed decoder queue (reported as "backend worker disconnected"). + submission + .lock() + .unwrap_or_else(|error| error.into_inner()) + .reset(); + drop(producer.take()); +} + +#[cfg(target_os = "windows")] +impl EmbeddedD3d11Submission { + fn new() -> Self { + Self { + pending: VecDeque::with_capacity( + opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY, + ), + submitter: None, + gap: opennow_streamer_platform_windows::ReferenceGap::default(), + } + } + + /// Queues one compressed access unit without retaining a broken inter-frame chain. + /// Once graphics is installed this writes directly to the decoder's thread-safe queue; it + /// never waits for the render-owned decoder/processor mutex. + fn push( + &mut self, + frame: opennow_streamer_platform_windows::EncodedVideoFrame, + ) -> Result { + use opennow_streamer_platform_windows::{ + CompressedAdmit, GapAdmit, PushOutcome, ReferenceGap, + }; + + let key_frame = frame.key_frame; + let needs_graphics = self.submitter.is_none(); + let (pending, submitter) = (&self.pending, self.submitter.as_ref()); + let gap_admit = self.gap.admit(key_frame, || { + submitter.map_or_else( + || pending.iter().any(|queued| queued.key_frame), + crate::D3d11FrameSubmitter::holds_keyframe, + ) + }); + if gap_admit != GapAdmit::Admit { + return Ok(EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: 1, + needs_graphics, + preserved_keyframe: self.gap == ReferenceGap::BehindQueuedKeyframe, + }); + } + if let Some(submitter) = self.submitter.as_ref() { + let outcome = submitter + .submit_video(frame) + .map_err(|error| error.to_string())?; + return Ok(match outcome { + PushOutcome::Queued | PushOutcome::Paused => EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped: 0, + needs_graphics, + preserved_keyframe: false, + }, + PushOutcome::Backpressured => { + self.gap.record(CompressedAdmit::PreserveQueuedKeyframe); + EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: 1, + needs_graphics, + preserved_keyframe: true, + } + } + PushOutcome::DroppedOldest => { + let dropped = opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY + + usize::from(!key_frame); + if !key_frame { + self.gap.record(CompressedAdmit::DiscardChain { dropped }); + } + EmbeddedD3d11SubmissionOutcome { + queued: key_frame, + dropped, + needs_graphics, + preserved_keyframe: false, + } + } + }); + } + let admitted = opennow_streamer_platform_windows::admit_compressed_frame( + self.pending.len(), + opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY, + key_frame, + self.pending.iter().any(|queued| queued.key_frame), + ); + self.gap.record(admitted); + match admitted { + CompressedAdmit::Append => { + self.pending.push_back(frame); + Ok(EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped: 0, + needs_graphics: true, + preserved_keyframe: false, + }) + } + CompressedAdmit::ReplaceWithKeyframe { dropped } => { + self.pending.clear(); + self.pending.push_back(frame); + Ok(EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped, + needs_graphics: true, + preserved_keyframe: false, + }) + } + CompressedAdmit::PreserveQueuedKeyframe => Ok(EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: 1, + needs_graphics: true, + preserved_keyframe: true, + }), + CompressedAdmit::DiscardChain { dropped } => { + self.pending.clear(); + Ok(EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped, + needs_graphics: true, + preserved_keyframe: false, + }) + } + } + } + + fn reset(&mut self) { + self.pending.clear(); + self.submitter = None; + } + + fn install(&mut self, submitter: crate::D3d11FrameSubmitter) -> Result { + let mut dropped = 0; + while let Some(frame) = self.pending.pop_front() { + let outcome = submitter + .submit_video(frame) + .map_err(|error| error.to_string())?; + if matches!( + outcome, + opennow_streamer_platform_windows::PushOutcome::DroppedOldest + | opennow_streamer_platform_windows::PushOutcome::Backpressured + ) { + // The decoder queue rejected this access unit. Later pending + // deltas reference it, so they must not follow a preserved or + // discarded chain. + dropped = dropped + .max(opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY + 1); + self.pending.clear(); + break; + } + } + self.submitter = Some(submitter); + Ok(dropped) + } +} + +#[cfg(target_os = "windows")] +struct EmbeddedD3d11FrameNotifier { + state: std::sync::Weak>, + frames: crate::GraphicsFramePublisher, + shared: Arc, + stream: MediaStreamConfig, + mid: Mutex, + presentation_time_ns: AtomicU64, + sequence: AtomicU64, +} + +#[cfg(target_os = "windows")] +impl EmbeddedD3d11FrameNotifier { + fn publish_initial(&self, mid: &str, presentation_time_ns: u64) { + *self + .mid + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = mid.to_owned(); + self.presentation_time_ns + .store(presentation_time_ns, Ordering::Release); + self.publish(false); + } + + fn publish_decoded(&self) { + self.publish(true); + } + + fn publish(&self, decoded: bool) { + let Some(state) = self.state.upgrade() else { + return; + }; + let Some(lease) = self.frames.context() else { + opennow_streamer_protocol::log::log_throttled( + "d3d11-publish-no-context", + "WARN", + "d3d11-publish", + "frame ready but Qt graphics context is absent", + ); + return; + }; + let sequence = self.sequence.fetch_add(1, Ordering::AcqRel) + 1; + let pending = PendingD3d11Frame { + state, + shared: Arc::clone(&self.shared), + mid: self + .mid + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(), + info: crate::GraphicsFrameInfo { + width: self.stream.width, + height: self.stream.height, + sequence, + presentation_time_ns: self.presentation_time_ns.load(Ordering::Acquire), + }, + }; + let result = if decoded { + self.frames.publish(lease, Arc::new(pending)) + } else { + self.frames.publish_pending(lease, Arc::new(pending)) + }; + if sequence == 1 || result.is_err() { + opennow_streamer_protocol::log::log_throttled( + "d3d11-publish-result", + "INFO", + "d3d11-publish", + &format!("sequence={sequence} result={result:?}"), + ); + } + } +} + +#[cfg(target_os = "windows")] +struct PendingD3d11Frame { + state: Arc>, + shared: Arc, + mid: String, + info: crate::GraphicsFrameInfo, +} + +#[cfg(target_os = "windows")] +impl crate::GraphicsRenderResources for Mutex { + fn retire(&self) -> Result<(), String> { + let mut state = self.lock().unwrap_or_else(|error| error.into_inner()); + state.reset(); + state.keyframe_required = true; + Ok(()) + } +} + +#[cfg(target_os = "windows")] +impl crate::GraphicsFrame for PendingD3d11Frame { + fn info(&self) -> crate::GraphicsFrameInfo { + self.info + } + + fn render_resources(&self) -> Option> { + Some(self.state.clone()) + } + + fn record( + &self, + context: crate::GraphicsContext, + command: crate::GraphicsRecordCommand, + ) -> Result { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + let mut color_change = None; + let result = state.record(context, command).map(|(frame, change)| { + color_change = change; + frame + }); + let first_frame = state.take_first_recorded_frame(result.is_ok()); + let keyframe_required = state.take_keyframe_required(); + drop(state); + if first_frame { + let _ = self.shared.feedback.send(MediaFeedback::PlaybackStarted { + backend: "D3D11/Qt", + }); + } + if let Some(change) = color_change { + let _ = self.shared.feedback.send(change); + } + if keyframe_required { + invalidate_embedded_video( + &self.shared, + &self.mid, + "embedded D3D11 decoder lost references or graphics device", + ); + let _ = self.shared.feedback.send(MediaFeedback::QueueDropped { + media: "d3d11-decode", + count: 1, + }); + } + result + } +} + +impl MediaControl { + pub fn start_replay( + &self, + config: opennow_streamer_protocol::ReplayBufferConfig, + reserved: Arc, + ) { + self.shared.replay_tap.start_with_budget(config, reserved); + } + + pub fn stop_replay(&self) { + self.shared.replay_tap.stop(); + } + + pub fn replay_snapshot( + &self, + ) -> Result<(MediaStreamConfig, crate::replay::ReplaySnapshot), &'static str> { + self.shared + .replay_tap + .snapshot() + .map(|snapshot| (self.shared.stream, snapshot)) + } + + pub fn subscribe_recording( + &self, + ) -> Result<(MediaStreamConfig, EncodedRecordingReceiver), String> { + self.shared + .recording_tap + .subscribe() + .map(|receiver| (self.shared.stream, receiver)) + } + + pub fn unsubscribe_recording(&self) { + self.shared.recording_tap.unsubscribe(); + } + + pub fn cut_recording(&self, reason: RecordingCutReason) { + self.shared + .recording_tap + .close(RecordingCompletion::Cut { reason }); + } + + pub fn update_cursor(&self, bytes: Vec) { + let _ = self.host_commands.send(HostCommand::Cursor(bytes)); + } + + pub fn stop(&self) { + if self.shared.stopped.swap(true, Ordering::AcqRel) { + return; + } + self.shared.video.close(); + self.shared.audio.close(); + self.cut_recording(RecordingCutReason::Interrupted); + self.shared.replay_tap.stop(); + self.shared.output.stop_microphone(); + self.shared.output.clear(); + let _ = self.host_commands.send(HostCommand::Stop); + } +} + +#[cfg(target_os = "windows")] +fn run_windows_video(shared: Arc, maximum_fps: u32) { + use opennow_streamer_platform_windows::{ + EncodedVideoFrame, PushOutcome as WindowsPushOutcome, VideoCodec, + }; + + let mut sample_clock = AdaptiveSampleClock::new(maximum_fps); + while let Some(frame) = shared.video.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + if shared.windows_bridge.use_software() { + shared.video_desynced.store(true, Ordering::Release); + match H264Decoder::new() { + Ok(decoder) => run_video_decoder_from(shared, decoder, Some(frame)), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + } + } + return; + } + if (shared.video_desynced.load(Ordering::Acquire) + || shared.windows_bridge.keyframe_required()) + && !frame.keyframe + { + continue; + } + let Some(backend) = shared.windows_bridge.backend() else { + shared.video_desynced.store(true, Ordering::Release); + request_keyframe(&shared, &frame.mid, "D3D11 backend is unavailable"); + continue; + }; + shared.windows_bridge.set_last_video_mid(&frame.mid); + let timestamp_100ns = media_timestamp_100ns(frame.timestamp, frame.clock_rate_hz); + let duration_100ns = sample_clock.observe(timestamp_100ns); + let reset_decoder = frame.keyframe + && (shared.video_desynced.load(Ordering::Acquire) + || shared.windows_bridge.keyframe_required()); + match backend.submit_video(EncodedVideoFrame { + codec: match frame.codec { + MediaCodec::H264 => VideoCodec::H264, + MediaCodec::H265 => VideoCodec::H265, + MediaCodec::Av1 => VideoCodec::Av1, + _ => continue, + }, + data: frame.data.to_vec(), + timestamp_100ns, + duration_100ns, + key_frame: frame.keyframe, + reset_decoder, + }) { + Ok(WindowsPushOutcome::Queued) => { + report_video_frame_accepted(&shared, &frame); + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + shared.windows_bridge.accept_keyframe(); + } + } + Ok(WindowsPushOutcome::Paused) => {} + Ok(WindowsPushOutcome::Backpressured) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "d3d11-video", + count: 1, + }); + shared.video_desynced.store(true, Ordering::Release); + shared.windows_bridge.require_keyframe(); + request_keyframe( + &shared, + &frame.mid, + "D3D11 input queue kept its keyframe under backpressure", + ); + } + Ok(WindowsPushOutcome::DroppedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "d3d11-video", + count: opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY + + usize::from(!frame.keyframe), + }); + if frame.keyframe { + // The Windows queue cleared the stale prediction chain but retained this + // recovery keyframe. Accept it immediately instead of requesting and then + // discarding another keyframe. + report_video_frame_accepted(&shared, &frame); + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + shared.windows_bridge.accept_keyframe(); + } else { + shared.video_desynced.store(true, Ordering::Release); + shared.windows_bridge.require_keyframe(); + request_keyframe(&shared, &frame.mid, "D3D11 input queue overflow"); + } + } + Err(error) => { + shared.video_desynced.store(true, Ordering::Release); + shared.windows_bridge.require_keyframe(); + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: match frame.codec { + MediaCodec::H264 => "h264", + MediaCodec::H265 => "h265", + MediaCodec::Av1 => "av1", + _ => "video", + }, + message: error.to_string(), + }); + request_keyframe(&shared, &frame.mid, "D3D11 decoder rejected an access unit"); + } + } + } +} + +#[cfg(any(target_os = "windows", test))] +struct AdaptiveSampleClock { + previous_timestamp_100ns: Option, + nominal_duration_100ns: i64, + recent_durations_100ns: VecDeque, +} + +#[cfg(any(target_os = "windows", test))] +impl AdaptiveSampleClock { + const HISTORY_LENGTH: usize = 120; + + fn new(maximum_fps: u32) -> Self { + Self { + previous_timestamp_100ns: None, + nominal_duration_100ns: 10_000_000_i64 / i64::from(maximum_fps.max(1)), + recent_durations_100ns: VecDeque::with_capacity(Self::HISTORY_LENGTH), + } + } + + fn observe(&mut self, timestamp_100ns: i64) -> i64 { + let observed = self + .previous_timestamp_100ns + .replace(timestamp_100ns) + .and_then(|previous| timestamp_100ns.checked_sub(previous)) + .filter(|duration| *duration > 0); + + if let Some(duration) = observed { + if self.recent_durations_100ns.len() == Self::HISTORY_LENGTH { + self.recent_durations_100ns.pop_front(); + } + self.recent_durations_100ns.push_back(duration); + return duration; + } + + if self.recent_durations_100ns.is_empty() { + self.nominal_duration_100ns + } else { + self.recent_durations_100ns.iter().copied().sum::() + / self.recent_durations_100ns.len() as i64 + } + } +} + +#[cfg(any(target_os = "windows", test))] +fn media_timestamp_100ns(timestamp: u64, clock_rate_hz: u32) -> i64 { + if clock_rate_hz == 0 { + return 0; + } + let value = u128::from(timestamp) + .saturating_mul(10_000_000) + .checked_div(u128::from(clock_rate_hz)) + .unwrap_or(0); + i64::try_from(value).unwrap_or(i64::MAX) +} + +#[cfg(target_os = "windows")] +fn invalidate_embedded_video(shared: &SharedPipeline, mid: &str, reason: &str) { + // Invalidate queued AND already-dequeued access units. An older keyframe + // submission must never clear a newer decoder/transport failure. The hold + // keeps the following deltas from each requesting another IDR; `clear` + // alone re-arms that request. + shared.video.clear(); + shared.video.hold_keyframe_request(); + opennow_streamer_protocol::log::log_async("WARN", "video-reference", reason); + let _ = shared.feedback.send(MediaFeedback::RequestKeyframe { + mid: mid.to_owned(), + reason: reason.to_owned(), + }); +} + +#[cfg(target_os = "windows")] +fn request_keyframe(shared: &SharedPipeline, mid: &str, reason: &str) { + if !shared.keyframe_requested.swap(true, Ordering::AcqRel) { + let _ = shared.feedback.send(MediaFeedback::RequestKeyframe { + mid: mid.to_owned(), + reason: reason.to_owned(), + }); + } +} + +impl Drop for MediaSession { + fn drop(&mut self) { + self.stop_inner(); + } +} + +#[cfg(test)] +type TestVideoDecode = Box Result, String> + Send>; + +struct H264Decoder { + decoder: OpenH264Decoder, + parameter_sets: crate::h264::H264ParameterSets, + #[cfg(test)] + decode_for_test: Option, +} + +impl H264Decoder { + fn new() -> Result { + OpenH264Decoder::with_api_config( + OpenH264API::from_source(), + DecoderConfig::new().debug(false), + ) + .map(|decoder| Self { + decoder, + parameter_sets: crate::h264::H264ParameterSets::default(), + #[cfg(test)] + decode_for_test: None, + }) + .map_err(|error| format!("OpenH264 decoder initialization failed: {error}")) + } + + fn reset(&mut self) -> Result<(), String> { + let mut replacement = Self::new()?; + let parameter_sets = self.parameter_sets.replay(); + if !parameter_sets.is_empty() { + replacement + .decoder + .decode(¶meter_sets) + .map_err(|error| format!("H.264 parameter-set replay failed: {error}"))?; + } + self.decoder = replacement.decoder; + Ok(()) + } + + fn decode(&mut self, encoded: &[u8]) -> Result, String> { + #[cfg(test)] + if let Some(decode) = self.decode_for_test.as_mut() { + return decode(encoded); + } + let yuv = self + .decoder + .decode(encoded) + .map_err(|error| error.to_string())?; + self.parameter_sets.retain(encoded)?; + let Some(yuv) = yuv else { + return Ok(None); + }; + let (width, height) = yuv.dimensions(); + let mut rgb = vec![0; yuv.rgb8_len()]; + yuv.write_rgb8(&mut rgb); + Ok(Some(DecodedVideoFrame { + width: width as u32, + height: height as u32, + rgb, + })) + } +} + +struct OpusDecoder { + decoder: OpusNativeDecoder, + channels: u8, + scratch: Vec, + last_frame_samples_per_channel: usize, +} + +impl OpusDecoder { + fn new(channels: u8) -> Result { + let opus_channels = match channels { + 1 => Channels::Mono, + 2 => Channels::Stereo, + other => return Err(format!("unsupported Opus channel count: {other}")), + }; + OpusNativeDecoder::new(OPUS_SAMPLE_RATE, opus_channels) + .map(|decoder| Self { + decoder, + channels, + scratch: vec![0.0; MAX_OPUS_FRAME_SAMPLES_PER_CHANNEL * channels as usize], + // GFN audio uses 20 ms Opus packets. A discontinuity can + // arrive before the first successful decode establishes the + // packet duration, so begin with that negotiated baseline. + last_frame_samples_per_channel: 960, + }) + .map_err(|error| format!("Opus decoder initialization failed: {error}")) + } + + fn decode(&mut self, encoded: &[u8]) -> Result<&[f32], String> { + let samples_per_channel = self + .decoder + .decode_float(encoded, &mut self.scratch, false) + .map_err(|error| error.to_string())?; + if !encoded.is_empty() { + self.last_frame_samples_per_channel = samples_per_channel; + } + Ok(&self.scratch[..samples_per_channel * self.channels as usize]) + } + + fn decode_packet_loss(&mut self) -> Result<&[f32], String> { + // An empty Opus packet invokes decoder packet-loss concealment. In-band + // FEC remains disabled; RFC 2198 recovery is handled by the transport. + // Bound the output to the preceding packet duration. Giving libopus + // the full 120 ms scratch buffer makes one loss synthesize 120 ms and + // creates an audible latency spike. + let output_len = self.last_frame_samples_per_channel * self.channels as usize; + let samples_per_channel = self + .decoder + .decode_float(&[], &mut self.scratch[..output_len], false) + .map_err(|error| error.to_string())?; + Ok(&self.scratch[..samples_per_channel * self.channels as usize]) + } +} + +fn run_video_decoder(shared: Arc, decoder: H264Decoder) { + run_video_decoder_from(shared, decoder, None); +} + +fn run_video_decoder_from( + shared: Arc, + mut decoder: H264Decoder, + mut pending: Option, +) { + loop { + let Some(frame) = pending.take().or_else(|| shared.video.pop()) else { + return; + }; + if shared.paused.load(Ordering::Acquire) { + continue; + } + if !frame.contiguous { + shared.video_desynced.store(true, Ordering::Release); + } + if shared.video_desynced.load(Ordering::Acquire) { + if !frame.keyframe { + continue; + } + match decoder.reset() { + Ok(()) => {} + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + request_software_keyframe(&shared, &frame.mid); + continue; + } + } + } + match decoder.decode(&frame.data) { + Ok(Some(decoded)) => { + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + } + shared.keyframe_requested.store(false, Ordering::Release); + report_video_frame_accepted(&shared, &frame); + if shared.output.replace_video(decoded) { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "present", + count: 1, + }); + } + } + Ok(None) => { + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + } + } + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + shared.video_desynced.store(true, Ordering::Release); + request_software_keyframe(&shared, &frame.mid); + } + } + } +} + +fn request_software_keyframe(shared: &SharedPipeline, mid: &str) { + if !shared.keyframe_requested.swap(true, Ordering::AcqRel) { + let _ = shared.feedback.send(MediaFeedback::RequestKeyframe { + mid: mid.to_owned(), + reason: "H.264 decoder rejected an access unit".to_owned(), + }); + } +} + +fn run_audio_decoder(shared: Arc, decoder: OpusDecoder) { + run_audio_decoder_from(shared, decoder, None); +} + +fn run_audio_decoder_from( + shared: Arc, + mut decoder: OpusDecoder, + mut pending: Option, +) { + let mut configured_channels = 2; + loop { + let Some(frame) = pending.take().or_else(|| shared.audio.pop()) else { + return; + }; + if shared.paused.load(Ordering::Acquire) { + continue; + } + let MediaCodec::Opus { channels } = frame.codec else { + continue; + }; + let channels = channels.clamp(1, 2); + if channels != configured_channels { + match OpusDecoder::new(channels) { + Ok(new_decoder) => { + decoder = new_decoder; + configured_channels = channels; + } + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message, + }); + continue; + } + } + } + if !frame.contiguous { + match decoder.decode_packet_loss() { + Ok(samples) => submit_decoded_audio(&shared, samples.to_vec(), configured_channels), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus-plc", + message, + }); + } + } + } + match decoder.decode(&frame.data) { + Ok(samples) => submit_decoded_audio(&shared, samples.to_vec(), configured_channels), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message, + }); + } + } + } +} + +fn submit_decoded_audio(shared: &SharedPipeline, samples: Vec, channels: u8) { + let samples = if channels == 1 { + samples + .into_iter() + .flat_map(|sample| [sample, sample]) + .collect::>() + } else { + samples + }; + #[cfg(target_os = "windows")] + if !shared.windows_bridge.use_software() + && let Some(backend) = shared.windows_bridge.backend() + { + use opennow_streamer_platform_windows::{ + AudioFormat, PcmFrame, PushOutcome as WindowsPushOutcome, + }; + match backend.submit_audio(PcmFrame { + samples, + format: AudioFormat { + sample_rate: OPUS_SAMPLE_RATE, + channels: 2, + }, + }) { + Ok(WindowsPushOutcome::DroppedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "wasapi", + count: 1, + }); + } + Ok( + WindowsPushOutcome::Queued + | WindowsPushOutcome::Paused + | WindowsPushOutcome::Backpressured, + ) => {} + Err(error) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: error.to_string(), + }); + } + } + return; + } + let dropped = shared.output.push_audio(&samples); + if dropped > 0 { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "audio-output", + count: dropped, + }); + } +} + +#[cfg(target_os = "linux")] +fn run_linux_video(shared: Arc, host_commands: Sender) { + while let Some(frame) = shared.video.pop() { + if shared.linux_software_fallback.load(Ordering::Acquire) { + match H264Decoder::new() { + Ok(decoder) => run_video_decoder_from(shared, decoder, Some(frame)), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + } + } + return; + } + if shared.paused.load(Ordering::Acquire) { + continue; + } + *shared + .linux_video_mid + .lock() + .unwrap_or_else(|error| error.into_inner()) = frame.mid.clone(); + let timestamp_us = media_timestamp_us(frame.timestamp, frame.clock_rate_hz); + let encoded = match opennow_streamer_platform_linux::EncodedVideoFrame::new( + Arc::clone(&frame.data), + timestamp_us, + frame.keyframe, + ) { + Ok(encoded) => encoded, + Err(error) => { + trigger_linux_fallback( + &shared, + &host_commands, + format!("Linux decoder rejected encoded video framing: {error}"), + ); + continue; + } + }; + let result = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .ok_or_else(|| "Linux hardware session is unavailable".to_owned()) + .and_then(|session| { + session + .submit_video(encoded) + .map_err(|error| error.to_string()) + }); + match result { + Ok(opennow_streamer_platform_linux::PushOutcome::Queued) => { + report_video_frame_accepted(&shared, &frame); + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + } + } + Ok(opennow_streamer_platform_linux::PushOutcome::DroppedOldest) => { + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + } else { + shared.video_desynced.store(true, Ordering::Release); + } + } + Ok(opennow_streamer_platform_linux::PushOutcome::Paused) => {} + Ok(opennow_streamer_platform_linux::PushOutcome::AudioDisabled) => { + unreachable!("audio-disabled outcomes are not produced by video submission") + } + Err(reason) => trigger_linux_fallback( + &shared, + &host_commands, + format!("Linux hardware video submission failed: {reason}"), + ), + } + } +} + +#[cfg(target_os = "linux")] +fn run_embedded_linux_video(shared: Arc) { + while let Some(frame) = shared.video.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + *shared + .linux_video_mid + .lock() + .unwrap_or_else(|error| error.into_inner()) = frame.mid.clone(); + let encoded = match opennow_streamer_platform_linux::EncodedVideoFrame::new( + Arc::clone(&frame.data), + media_timestamp_us(frame.timestamp, frame.clock_rate_hz), + frame.keyframe, + ) { + Ok(encoded) => encoded, + Err(error) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: shared.linux_codec.label(), + message: error.to_string(), + }); + request_linux_keyframe(&shared, "Linux decoder rejected encoded video framing"); + continue; + } + }; + let result = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .ok_or_else(|| "embedded Linux media session is unavailable".to_owned()) + .and_then(|session| { + session + .submit_video(encoded) + .map_err(|error| error.to_string()) + }); + match result { + Ok(opennow_streamer_platform_linux::PushOutcome::Queued) => { + report_video_frame_accepted(&shared, &frame); + if frame.keyframe { + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + } + } + Ok(opennow_streamer_platform_linux::PushOutcome::DroppedOldest) => { + shared.video_desynced.store(true, Ordering::Release); + request_linux_keyframe(&shared, "embedded Linux decoder queue overflow"); + } + Ok(opennow_streamer_platform_linux::PushOutcome::Paused) => {} + Ok(opennow_streamer_platform_linux::PushOutcome::AudioDisabled) => { + unreachable!("audio-disabled outcomes are not produced by video submission") + } + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: shared.linux_codec.label(), + message, + }); + request_linux_keyframe(&shared, "embedded Linux video submission failed"); + } + } + } +} + +#[cfg(target_os = "linux")] +fn run_embedded_linux_audio(shared: Arc) { + while let Some(frame) = shared.audio.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + let MediaCodec::Opus { .. } = frame.codec else { + continue; + }; + let Some(ssrc) = frame.ssrc else { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: "embedded Linux audio frame carries no sender source identifier" + .to_owned(), + }); + continue; + }; + let Ok(rtp_timestamp) = u32::try_from(frame.timestamp) else { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: "embedded Linux audio frame carries an out-of-range RTP timestamp" + .to_owned(), + }); + continue; + }; + let packet = match opennow_streamer_platform_linux::AudioPacket::new( + Arc::clone(&frame.data), + rtp_timestamp, + frame.clock_rate_hz, + ssrc, + ) { + Ok(packet) => packet, + Err(error) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: error.to_string(), + }); + continue; + } + }; + let result = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .ok_or_else(|| "embedded Linux media session is unavailable".to_owned()) + .and_then(|session| { + session + .submit_audio(packet) + .map_err(|error| error.to_string()) + }); + match result { + Ok(opennow_streamer_platform_linux::PushOutcome::DroppedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "linux-audio", + count: 1, + }); + } + Ok(opennow_streamer_platform_linux::PushOutcome::Queued) + | Ok(opennow_streamer_platform_linux::PushOutcome::Paused) + | Ok(opennow_streamer_platform_linux::PushOutcome::AudioDisabled) => {} + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message, + }); + } + } + } +} + +#[cfg(target_os = "linux")] +fn run_embedded_linux_monitor( + shared: Arc, + publisher: crate::GraphicsFramePublisher, + producer: crate::LinuxGpuFrameProducer, + backend_label: &'static str, +) { + use std::time::Duration; + + let mut playback_started = false; + let mut last_decode_timings_report = Instant::now(); + let mut reported_color = None; + while !shared.stopped.load(Ordering::Acquire) { + let report_decode_timings = last_decode_timings_report.elapsed() >= Duration::from_secs(1); + let (frame, events, decode_timings) = { + let session = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()); + let Some(session) = session.as_ref() else { + return; + }; + let frame = session.try_recv_latest_frame(); + let mut events = Vec::new(); + while let Some(event) = session.try_recv_event() { + events.push(event); + } + let decode_timings = report_decode_timings.then(|| session.decode_timings()); + (frame, events, decode_timings) + }; + if let Some(timings) = decode_timings { + last_decode_timings_report = Instant::now(); + if timings.has_observable_state() { + let stage = |stage: opennow_streamer_platform_linux::DecodeStagePercentiles| { + DecodeStageTimings { + p50_us: stage.p50_us, + p95_us: stage.p95_us, + max_us: stage.max_us, + } + }; + let _ = shared + .feedback + .send(MediaFeedback::DecodeTimings(DecodeTimingsReport { + call: timings.call.map(stage), + residence: timings.residence.map(stage), + call_window_samples: timings.call_window_samples, + residence_window_samples: timings.residence_window_samples, + submissions_total: timings.submissions_total, + outputs_total: timings.outputs_total, + output_calls_total: timings.output_calls_total, + last_submission_at: timings.last_submission_at, + last_output_at: timings.last_output_at, + in_flight: timings.in_flight, + oldest_in_flight_at: timings.oldest_in_flight_at, + epoch: timings.epoch, + epoch_started_at: timings.epoch_started_at, + unmatched_outputs: timings.unmatched_outputs, + unmatched_submissions: timings.unmatched_submissions, + })); + } + } + if !shared.paused.load(Ordering::Acquire) { + if let Some((decoded, skipped)) = frame { + if skipped > 0 { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "decoded-video", + count: skipped, + }); + } + if let Some(lease) = publisher.context() { + match producer + .frame(decoded) + .map_err(|error| error.to_string()) + .and_then(|frame| { + publisher + .publish(lease, Arc::new(frame)) + .map_err(|error| error.to_string()) + }) { + Ok(_) if !playback_started => { + playback_started = true; + let _ = shared.feedback.send(MediaFeedback::PlaybackStarted { + backend: backend_label, + }); + } + Ok(_) => {} + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::OutputError { message }); + } + } + } + } + } + for event in events { + match event { + opennow_streamer_platform_linux::BackendEvent::DecoderChanged { + from, + to, + reason, + } => { + let _ = shared.feedback.send(MediaFeedback::BackendFallback { + from: linux_decoder_name(from), + to: linux_decoder_name(to), + reason, + }); + } + opennow_streamer_platform_linux::BackendEvent::NeedKeyframe => { + request_linux_keyframe(&shared, "Linux decoder requires a fresh keyframe"); + } + opennow_streamer_platform_linux::BackendEvent::QueueOverflow { media } => { + let _ = shared + .feedback + .send(MediaFeedback::QueueDropped { media, count: 1 }); + } + opennow_streamer_platform_linux::BackendEvent::DeviceLost { subsystem, reason } => { + let _ = shared.feedback.send(MediaFeedback::DeviceLost { + subsystem: linux_subsystem_name(subsystem), + recovered: false, + message: Some(reason), + }); + request_linux_keyframe(&shared, "Linux decoder device was lost"); + } + opennow_streamer_platform_linux::BackendEvent::Error(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: shared.linux_codec.label(), + message, + }); + shared.stopped.store(true, Ordering::Release); + shared.video.close(); + shared.audio.close(); + stop_linux_session(&shared); + return; + } + opennow_streamer_platform_linux::BackendEvent::StateChanged( + opennow_streamer_platform_linux::LifecycleState::Failed, + ) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: shared.linux_codec.label(), + message: "embedded Linux media session failed".to_owned(), + }); + shared.stopped.store(true, Ordering::Release); + shared.video.close(); + shared.audio.close(); + stop_linux_session(&shared); + return; + } + opennow_streamer_platform_linux::BackendEvent::AudioDecodeError { + message, + consecutive, + } => { + let _ = shared.feedback.send(MediaFeedback::AudioDecoderError { + message, + consecutive, + }); + } + opennow_streamer_platform_linux::BackendEvent::AudioOutputError { + backend, + message, + } => forward_linux_audio_output_loss(&shared, backend, message), + opennow_streamer_platform_linux::BackendEvent::AudioOutputRecovered { + from, + to, + } => forward_linux_audio_output_recovery(&shared, from, to), + opennow_streamer_platform_linux::BackendEvent::AudioUnavailable { + backend, + reason, + rejected, + } => { + let _ = shared.feedback.send(MediaFeedback::AudioUnavailable { + backend: linux_audio_backend_name(backend), + reason, + rejected, + }); + } + opennow_streamer_platform_linux::BackendEvent::FormatChanged(format) => { + report_linux_color_format_change(&shared, &mut reported_color, format); + } + opennow_streamer_platform_linux::BackendEvent::StateChanged(_) + | opennow_streamer_platform_linux::BackendEvent::DecoderSelected(_) + | opennow_streamer_platform_linux::BackendEvent::AudioSelected(_) => {} + } + } + thread::sleep(Duration::from_millis(2)); + } + stop_linux_session(&shared); +} + +#[cfg(target_os = "linux")] +fn run_linux_monitor(shared: Arc, host_commands: Sender) { + use std::time::Duration; + + let mut reported_color = None; + while !shared.stopped.load(Ordering::Acquire) { + if shared.linux_software_fallback.load(Ordering::Acquire) { + request_linux_keyframe(&shared, "Linux decoder fallback requires a fresh keyframe"); + stop_linux_session(&shared); + return; + } + let (frames, events) = { + let session = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()); + let Some(session) = session.as_ref() else { + return; + }; + let mut frames = Vec::new(); + while let Some(frame) = session.try_recv_frame() { + frames.push(frame); + } + let mut events = Vec::new(); + while let Some(event) = session.try_recv_event() { + events.push(event); + } + (frames, events) + }; + if !shared.paused.load(Ordering::Acquire) { + for frame in frames { + if shared.output.queue_linux_video(frame) { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "linux-present", + count: 1, + }); + } + } + } + for event in events { + match event { + opennow_streamer_platform_linux::BackendEvent::DecoderChanged { + from, + to, + reason, + } => { + let _ = shared.feedback.send(MediaFeedback::BackendFallback { + from: linux_decoder_name(from), + to: linux_decoder_name(to), + reason, + }); + } + opennow_streamer_platform_linux::BackendEvent::NeedKeyframe => { + request_linux_keyframe(&shared, "Linux decoder requires a fresh keyframe"); + } + opennow_streamer_platform_linux::BackendEvent::QueueOverflow { media } => { + let _ = shared + .feedback + .send(MediaFeedback::QueueDropped { media, count: 1 }); + } + opennow_streamer_platform_linux::BackendEvent::DeviceLost { subsystem, reason } => { + let _ = shared.feedback.send(MediaFeedback::DeviceLost { + subsystem: linux_subsystem_name(subsystem), + recovered: false, + message: Some(reason.clone()), + }); + trigger_linux_fallback( + &shared, + &host_commands, + format!("{subsystem:?} device was lost: {reason}"), + ); + } + opennow_streamer_platform_linux::BackendEvent::Error(reason) => { + trigger_linux_fallback(&shared, &host_commands, reason) + } + opennow_streamer_platform_linux::BackendEvent::StateChanged( + opennow_streamer_platform_linux::LifecycleState::Failed, + ) => trigger_linux_fallback( + &shared, + &host_commands, + "Linux hardware media session failed".to_owned(), + ), + opennow_streamer_platform_linux::BackendEvent::AudioDecodeError { + message, + consecutive, + } => { + let _ = shared.feedback.send(MediaFeedback::AudioDecoderError { + message, + consecutive, + }); + } + opennow_streamer_platform_linux::BackendEvent::AudioOutputError { + backend, + message, + } => forward_linux_audio_output_loss(&shared, backend, message), + opennow_streamer_platform_linux::BackendEvent::AudioOutputRecovered { + from, + to, + } => forward_linux_audio_output_recovery(&shared, from, to), + opennow_streamer_platform_linux::BackendEvent::AudioUnavailable { + backend, + reason, + rejected, + } => { + let _ = shared.feedback.send(MediaFeedback::AudioUnavailable { + backend: linux_audio_backend_name(backend), + reason, + rejected, + }); + } + opennow_streamer_platform_linux::BackendEvent::FormatChanged(format) => { + report_linux_color_format_change(&shared, &mut reported_color, format); + } + opennow_streamer_platform_linux::BackendEvent::StateChanged(_) + | opennow_streamer_platform_linux::BackendEvent::DecoderSelected(_) + | opennow_streamer_platform_linux::BackendEvent::AudioSelected(_) => {} + } + } + thread::sleep(Duration::from_millis(2)); + } + stop_linux_session(&shared); +} + +#[cfg(target_os = "linux")] +fn trigger_linux_fallback( + shared: &SharedPipeline, + host_commands: &Sender, + reason: String, +) { + if shared.linux_codec != MediaVideoCodec::H264 { + shared.stopped.store(true, Ordering::Release); + shared.video.close(); + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: shared.linux_codec.label(), + message: reason, + }); + return; + } + if shared.linux_software_fallback.swap(true, Ordering::AcqRel) { + return; + } + request_linux_keyframe(shared, "Linux decoder fallback requires a fresh keyframe"); + let _ = host_commands.send(HostCommand::FallbackLinux { reason }); +} + +#[cfg(target_os = "linux")] +fn request_linux_keyframe(shared: &SharedPipeline, reason: &str) { + shared.video_desynced.store(true, Ordering::Release); + if shared.keyframe_requested.swap(true, Ordering::AcqRel) { + return; + } + let mid = shared + .linux_video_mid + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if !mid.is_empty() { + let _ = shared.feedback.send(MediaFeedback::RequestKeyframe { + mid, + reason: reason.to_owned(), + }); + } +} + +#[cfg(target_os = "linux")] +fn stop_linux_session(shared: &SharedPipeline) { + if let Some(mut session) = shared + .linux_session + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + let _ = session.stop(); + } +} + +#[cfg(target_os = "linux")] +fn report_linux_color_format_change( + shared: &SharedPipeline, + reported: &mut Option, + format: opennow_streamer_platform_linux::StreamFormat, +) { + let Some(actual) = MediaColorQuality::from_linux_pixel_format(format.pixel_format) else { + return; + }; + if *reported == Some(actual) { + return; + } + *reported = Some(actual); + let _ = shared.feedback.send(MediaFeedback::ColorFormatChanged { + requested: shared.stream.color_quality, + actual, + }); +} + +#[cfg(target_os = "linux")] +const fn linux_decoder_name( + backend: opennow_streamer_platform_linux::DecoderBackend, +) -> &'static str { + match backend { + opennow_streamer_platform_linux::DecoderBackend::Vulkan => "Vulkan Video/Vulkan", + opennow_streamer_platform_linux::DecoderBackend::Cuda => "CUDA/NVDEC (CPU transfer)/Vulkan", + opennow_streamer_platform_linux::DecoderBackend::VaApi => "VA-API/Vulkan", + opennow_streamer_platform_linux::DecoderBackend::V4l2 => "V4L2/Vulkan", + opennow_streamer_platform_linux::DecoderBackend::Ffmpeg => "FFmpeg software/Vulkan", + opennow_streamer_platform_linux::DecoderBackend::MediaCodec => { + "Android MediaCodec hardware (CPU NV12 transfer)/Vulkan" + } + } +} + +#[cfg(target_os = "linux")] +const fn linux_embedded_backend_label( + preference: opennow_streamer_platform_linux::DecoderPreference, +) -> &'static str { + match preference { + opennow_streamer_platform_linux::DecoderPreference::MediaCodecOnly => { + linux_decoder_name(opennow_streamer_platform_linux::DecoderBackend::MediaCodec) + } + opennow_streamer_platform_linux::DecoderPreference::SoftwareOnly => { + linux_decoder_name(opennow_streamer_platform_linux::DecoderBackend::Ffmpeg) + } + _ => "Linux decoder/embedded Vulkan", + } +} + +#[cfg(target_os = "linux")] +const fn linux_audio_backend_name( + backend: opennow_streamer_platform_linux::AudioBackend, +) -> &'static str { + match backend { + opennow_streamer_platform_linux::AudioBackend::PipeWire => "PipeWire", + opennow_streamer_platform_linux::AudioBackend::Alsa => "ALSA", + } +} + +#[cfg(target_os = "linux")] +fn forward_linux_audio_output_loss( + shared: &SharedPipeline, + backend: opennow_streamer_platform_linux::AudioBackend, + message: String, +) { + let _ = shared.feedback.send(MediaFeedback::DeviceLost { + subsystem: linux_audio_backend_name(backend), + recovered: false, + message: Some(message), + }); +} + +#[cfg(target_os = "linux")] +fn forward_linux_audio_output_recovery( + shared: &SharedPipeline, + from: opennow_streamer_platform_linux::AudioBackend, + to: opennow_streamer_platform_linux::AudioBackend, +) { + let _ = shared.feedback.send(MediaFeedback::DeviceLost { + subsystem: linux_audio_backend_name(from), + recovered: true, + message: Some(format!( + "{} accepted audio output after the {} sink failed", + linux_audio_backend_name(to), + linux_audio_backend_name(from) + )), + }); +} + +#[cfg(target_os = "linux")] +const fn linux_subsystem_name( + subsystem: opennow_streamer_platform_linux::Subsystem, +) -> &'static str { + match subsystem { + opennow_streamer_platform_linux::Subsystem::VaApi => "VA-API", + opennow_streamer_platform_linux::Subsystem::V4l2 => "V4L2", + opennow_streamer_platform_linux::Subsystem::Vulkan => "Vulkan", + opennow_streamer_platform_linux::Subsystem::Ffmpeg => "FFmpeg", + opennow_streamer_platform_linux::Subsystem::MediaCodec => "Android MediaCodec bridge", + opennow_streamer_platform_linux::Subsystem::Opus => "Opus", + opennow_streamer_platform_linux::Subsystem::PipeWire => "PipeWire", + opennow_streamer_platform_linux::Subsystem::Alsa => "ALSA", + opennow_streamer_platform_linux::Subsystem::Session => "Linux media session", + } +} + +#[cfg(target_os = "linux")] +fn media_timestamp_us(timestamp: u64, clock_rate_hz: u32) -> u64 { + if clock_rate_hz == 0 { + return 0; + } + timestamp.saturating_mul(1_000_000) / u64::from(clock_rate_hz) +} + +#[cfg(target_os = "macos")] +fn run_macos_h264_video( + shared: Arc, + host_commands: Sender, + stream_fps: u32, + allow_software_fallback: bool, +) { + use std::sync::mpsc; + use std::time::Duration; + + use opennow_streamer_platform_macos::{ + FrameTiming, H264Format, SubmitOutcome, VideoColorSpace, + }; + + use crate::runtime::MacH264Configuration; + + let mut tracker = crate::macos_backend::H264ParameterSetTracker::default(); + let mut configured_parameter_sets = None; + let mut backend_sink: Option = None; + let mut playback_started = false; + while let Some(frame) = shared.video.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + if let Some(sink) = backend_sink.as_ref() { + let mut decode_loss = None; + while let Some(loss) = sink.pop_video_decode_loss() { + decode_loss = Some(loss); + } + if let Some(loss) = decode_loss { + let reason = + loss.status + .map_or("VideoToolbox produced no decoded pixel buffer", |status| { + if status == -12_909 { + "VideoToolbox rejected damaged H.264 data" + } else { + "VideoToolbox lost decoder synchronization" + } + }); + mark_macos_video_desynced(&shared, &frame.mid, reason); + } + if let Some(failure) = sink.fatal_failure() { + if !allow_software_fallback { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message: format!( + "{}; the Qt Metal stream view requires hardware decoding", + failure.message + ), + }); + return; + } + shared.mac_software_fallback.store(true, Ordering::Release); + mark_macos_video_desynced( + &shared, + &frame.mid, + &format!( + "{} failure requires software decode", + failure.subsystem.name() + ), + ); + } + } + if shared.mac_software_fallback.load(Ordering::Acquire) { + match H264Decoder::new() { + Ok(decoder) => run_video_decoder_from(shared, decoder, Some(frame)), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + } + } + return; + } + let framing = match tracker.observe(&frame.data) { + Ok(framing) => framing, + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + mark_macos_video_desynced(&shared, &frame.mid, "invalid H.264 framing"); + continue; + } + }; + let parameter_sets = tracker.parameter_sets(); + if shared.video_desynced.load(Ordering::Acquire) && !frame.keyframe { + continue; + } + if backend_sink.is_none() { + let Some(parameter_sets) = parameter_sets.clone() else { + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox is waiting for H.264 SPS/PPS", + ); + continue; + }; + let (reply, response) = mpsc::channel(); + if host_commands + .send(HostCommand::ConfigureMacH264 { + parameter_sets: parameter_sets.clone(), + reply, + }) + .is_err() + { + return; + } + match response.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(MacH264Configuration::Hardware(sink))) => { + *shared + .mac_sink + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(sink.clone()); + tracker.commit_parameter_sets(parameter_sets.clone()); + configured_parameter_sets = Some(parameter_sets); + backend_sink = Some(sink); + } + Ok(Ok(MacH264Configuration::SoftwareFallback { reason })) => { + shared.mac_software_fallback.store(true, Ordering::Release); + let _ = shared.feedback.send(MediaFeedback::BackendFallback { + from: "VideoToolbox/Metal", + to: "OpenH264/SDL", + reason, + }); + match H264Decoder::new() { + Ok(decoder) => run_video_decoder_from(shared, decoder, Some(frame)), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + } + } + return; + } + Ok(Err(message)) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message, + }); + return; + } + Err(_) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h264", + message: "VideoToolbox initialization timed out on the main thread" + .to_owned(), + }); + return; + } + } + } else if let Some(ref parameter_sets) = parameter_sets + && (configured_parameter_sets.as_ref() != Some(parameter_sets) + || !frame.contiguous + || shared.video_desynced.load(Ordering::Acquire)) + && frame.keyframe + { + let format = H264Format::new(parameter_sets.clone(), VideoColorSpace::Bt709); + let Some(sink) = backend_sink.as_ref() else { + return; + }; + if let Err(error) = sink.reconfigure_h264(format) { + eprintln!( + "Rejected H.264 parameter-set update; retaining the last known-good VideoToolbox decoder: {error} (spsBytes={}, ppsBytes={})", + parameter_sets.sequence().len(), + parameter_sets.picture().len(), + ); + mark_macos_video_desynced( + &shared, + &frame.mid, + "rejected H.264 parameter-set update; waiting for a clean keyframe", + ); + continue; + } + tracker.commit_parameter_sets(parameter_sets.clone()); + configured_parameter_sets = Some(parameter_sets.clone()); + } else if let Some(parameter_sets) = parameter_sets + && configured_parameter_sets.as_ref() == Some(¶meter_sets) + { + // Clear an identical candidate pair without perturbing the active decoder. + tracker.commit_parameter_sets(parameter_sets); + } + + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + let timescale = i32::try_from(frame.clock_rate_hz) + .ok() + .filter(|timescale| *timescale > 0) + .unwrap_or(90_000); + let timing = FrameTiming::new( + i64::try_from(frame.timestamp).unwrap_or(i64::MAX), + i64::from(timescale) / i64::from(stream_fps.max(1)), + timescale, + ); + let Some(sink) = backend_sink.as_ref() else { + return; + }; + match sink.submit_h264(&frame.data, framing, timing) { + Ok(SubmitOutcome::Accepted) => report_video_frame_accepted(&shared, &frame), + Ok(SubmitOutcome::Paused) => {} + Ok(SubmitOutcome::Backpressured | SubmitOutcome::ReplacedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "videotoolbox", + count: 1, + }); + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox decode queue backpressure", + ); + } + Err(error) => { + mark_macos_video_desynced( + &shared, + &frame.mid, + &format!("VideoToolbox rejected an H.264 access unit: {error}"), + ); + } + } + let stats = sink.stats(); + if !playback_started && (stats.video_presented > 0 || stats.video_metal_completed > 0) { + playback_started = true; + let _ = shared.feedback.send(MediaFeedback::PlaybackStarted { + backend: "VideoToolbox/Metal", + }); + } + } +} + +#[cfg(target_os = "macos")] +fn run_macos_h265_video( + shared: Arc, + host_commands: Sender, + stream_fps: u32, +) { + use std::sync::mpsc; + use std::time::Duration; + + use opennow_streamer_platform_macos::{FrameTiming, SubmitOutcome}; + + let mut tracker = crate::macos_backend::H265ParameterSetTracker::default(); + let mut configured_parameter_sets = None; + let mut backend_sink: Option = None; + let mut playback_started = false; + while let Some(frame) = shared.video.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + if let Some(sink) = backend_sink.as_ref() { + let mut decode_loss = None; + while let Some(loss) = sink.pop_video_decode_loss() { + decode_loss = Some(loss); + } + if let Some(loss) = decode_loss { + let reason = loss.status.map_or( + "VideoToolbox produced no decoded HEVC pixel buffer", + |_| "VideoToolbox lost HEVC decoder synchronization", + ); + mark_macos_video_desynced(&shared, &frame.mid, reason); + } + if let Some(failure) = sink.fatal_failure() { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h265", + message: format!( + "{} failed while decoding HEVC: {}", + failure.subsystem.name(), + failure.message + ), + }); + mark_macos_video_desynced(&shared, &frame.mid, "VideoToolbox HEVC backend failed"); + return; + } + } + let framing = match tracker.observe(&frame.data) { + Ok(framing) => framing, + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h265", + message, + }); + mark_macos_video_desynced(&shared, &frame.mid, "invalid H.265 framing"); + continue; + } + }; + let parameter_sets = tracker.parameter_sets(); + if shared.video_desynced.load(Ordering::Acquire) && !frame.keyframe { + continue; + } + if backend_sink.is_none() { + let Some(parameter_sets) = parameter_sets.clone() else { + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox is waiting for H.265 VPS/SPS/PPS", + ); + continue; + }; + let (reply, response) = mpsc::channel(); + if host_commands + .send(HostCommand::ConfigureMacH265 { + parameter_sets: parameter_sets.clone(), + reply, + }) + .is_err() + { + return; + } + match response.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(sink)) => { + *shared + .mac_sink + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(sink.clone()); + tracker.commit_parameter_sets(parameter_sets.clone()); + configured_parameter_sets = Some(parameter_sets); + backend_sink = Some(sink); + } + Ok(Err(message)) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h265", + message, + }); + return; + } + Err(_) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "h265", + message: "VideoToolbox HEVC initialization timed out on the main thread" + .to_owned(), + }); + return; + } + } + } else if let Some(ref parameter_sets) = parameter_sets + && configured_parameter_sets.as_ref() != Some(parameter_sets) + && frame.keyframe + { + let format = shared.stream.macos_h265_format(parameter_sets.clone()); + let Some(sink) = backend_sink.as_ref() else { + return; + }; + if let Err(error) = sink.reconfigure_h265(format) { + eprintln!( + "Rejected H.265 parameter-set update; retaining the last known-good VideoToolbox decoder: {error} (vpsBytes={}, spsBytes={}, ppsBytes={})", + parameter_sets.video().len(), + parameter_sets.sequence().len(), + parameter_sets.picture().len(), + ); + mark_macos_video_desynced( + &shared, + &frame.mid, + "rejected H.265 parameter-set update; waiting for a clean keyframe", + ); + continue; + } + tracker.commit_parameter_sets(parameter_sets.clone()); + configured_parameter_sets = Some(parameter_sets.clone()); + } else if let Some(parameter_sets) = parameter_sets + && configured_parameter_sets.as_ref() == Some(¶meter_sets) + { + tracker.commit_parameter_sets(parameter_sets); + } + + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + let timescale = i32::try_from(frame.clock_rate_hz) + .ok() + .filter(|timescale| *timescale > 0) + .unwrap_or(90_000); + let timing = FrameTiming::new( + i64::try_from(frame.timestamp).unwrap_or(i64::MAX), + i64::from(timescale) / i64::from(stream_fps.max(1)), + timescale, + ); + let Some(sink) = backend_sink.as_ref() else { + return; + }; + match sink.submit_h265(&frame.data, framing, timing) { + Ok(SubmitOutcome::Accepted) => report_video_frame_accepted(&shared, &frame), + Ok(SubmitOutcome::Paused) => {} + Ok(SubmitOutcome::Backpressured | SubmitOutcome::ReplacedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "videotoolbox-hevc", + count: 1, + }); + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox HEVC decode queue backpressure", + ); + } + Err(error) => { + mark_macos_video_desynced( + &shared, + &frame.mid, + &format!("VideoToolbox rejected an H.265 access unit: {error}"), + ); + } + } + let stats = sink.stats(); + if !playback_started && (stats.video_presented > 0 || stats.video_metal_completed > 0) { + playback_started = true; + let _ = shared.feedback.send(MediaFeedback::PlaybackStarted { + backend: "VideoToolbox HEVC/Metal", + }); + } + } +} + +#[cfg(target_os = "macos")] +fn run_macos_av1_video( + shared: Arc, + host_commands: Sender, + stream: MediaStreamConfig, +) { + use std::sync::mpsc; + use std::time::Duration; + + use opennow_streamer_platform_macos::{Av1Format, FrameTiming, SubmitOutcome}; + + let mut configured_codec_configuration: Option> = None; + let mut backend_sink: Option = None; + let mut playback_started = false; + while let Some(frame) = shared.video.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + if let Some(sink) = backend_sink.as_ref() { + let mut decode_loss = None; + while let Some(loss) = sink.pop_video_decode_loss() { + decode_loss = Some(loss); + } + if let Some(loss) = decode_loss { + let reason = loss.status.map_or( + "VideoToolbox produced no decoded AV1 pixel buffer", + |_| "VideoToolbox lost AV1 decoder synchronization", + ); + mark_macos_video_desynced(&shared, &frame.mid, reason); + } + if let Some(failure) = sink.fatal_failure() { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "av1", + message: format!( + "{} failed while decoding AV1: {}", + failure.subsystem.name(), + failure.message + ), + }); + mark_macos_video_desynced(&shared, &frame.mid, "VideoToolbox AV1 backend failed"); + return; + } + } + if shared.video_desynced.load(Ordering::Acquire) && !frame.keyframe { + continue; + } + + let candidate_configuration = frame + .keyframe + .then(|| crate::recording::av1_codec_private(&frame.data)); + if backend_sink.is_none() { + let codec_configuration = match candidate_configuration { + Some(Ok(configuration)) => configuration, + Some(Err(_)) | None => { + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox is waiting for an AV1 sequence-header keyframe", + ); + continue; + } + }; + let format = match Av1Format::new( + &codec_configuration, + stream.width, + stream.height, + stream.macos_color_space(), + ) { + Ok(format) => format.with_transfer(stream.macos_transfer()), + Err(error) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "av1", + message: format!("invalid AV1 VideoToolbox format: {error}"), + }); + mark_macos_video_desynced( + &shared, + &frame.mid, + "invalid AV1 VideoToolbox configuration", + ); + continue; + } + }; + let (reply, response) = mpsc::channel(); + if host_commands + .send(HostCommand::ConfigureMacAv1 { format, reply }) + .is_err() + { + return; + } + match response.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(sink)) => { + *shared + .mac_sink + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(sink.clone()); + configured_codec_configuration = Some(codec_configuration); + backend_sink = Some(sink); + } + Ok(Err(message)) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "av1", + message, + }); + return; + } + Err(_) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "av1", + message: "VideoToolbox AV1 initialization timed out on the main thread" + .to_owned(), + }); + return; + } + } + } else if let Some(Ok(codec_configuration)) = candidate_configuration + && configured_codec_configuration.as_ref() != Some(&codec_configuration) + { + let format = match Av1Format::new( + &codec_configuration, + stream.width, + stream.height, + stream.macos_color_space(), + ) { + Ok(format) => format.with_transfer(stream.macos_transfer()), + Err(error) => { + eprintln!("Rejected AV1 configuration update: {error}"); + mark_macos_video_desynced( + &shared, + &frame.mid, + "rejected AV1 configuration update; waiting for a clean keyframe", + ); + continue; + } + }; + let Some(sink) = backend_sink.as_ref() else { + return; + }; + if let Err(error) = sink.reconfigure_av1(format) { + eprintln!( + "Rejected AV1 sequence-header update; retaining the last known-good VideoToolbox decoder: {error} (configurationBytes={})", + codec_configuration.len(), + ); + mark_macos_video_desynced( + &shared, + &frame.mid, + "rejected AV1 sequence-header update; waiting for a clean keyframe", + ); + continue; + } + configured_codec_configuration = Some(codec_configuration); + } + + shared.video_desynced.store(false, Ordering::Release); + shared.keyframe_requested.store(false, Ordering::Release); + let timescale = i32::try_from(frame.clock_rate_hz) + .ok() + .filter(|timescale| *timescale > 0) + .unwrap_or(90_000); + let timing = FrameTiming::new( + i64::try_from(frame.timestamp).unwrap_or(i64::MAX), + i64::from(timescale) / i64::from(stream.fps.max(1)), + timescale, + ); + let Some(sink) = backend_sink.as_ref() else { + return; + }; + match sink.submit_av1(&frame.data, timing) { + Ok(SubmitOutcome::Accepted) => report_video_frame_accepted(&shared, &frame), + Ok(SubmitOutcome::Paused) => {} + Ok(SubmitOutcome::Backpressured | SubmitOutcome::ReplacedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "videotoolbox-av1", + count: 1, + }); + mark_macos_video_desynced( + &shared, + &frame.mid, + "VideoToolbox AV1 decode queue backpressure", + ); + } + Err(error) => { + mark_macos_video_desynced( + &shared, + &frame.mid, + &format!("VideoToolbox rejected an AV1 temporal unit: {error}"), + ); + } + } + let stats = sink.stats(); + if !playback_started && (stats.video_presented > 0 || stats.video_metal_completed > 0) { + playback_started = true; + let _ = shared.feedback.send(MediaFeedback::PlaybackStarted { + backend: "VideoToolbox AV1/Metal", + }); + } + } +} + +fn report_video_frame_accepted(shared: &SharedPipeline, frame: &EncodedFrame) { + let _ = shared.feedback.send(MediaFeedback::VideoFrameAccepted { + frame_index: frame.frame_index, + timestamp: frame.timestamp, + bytes: u32::try_from(frame.data.len()).unwrap_or(u32::MAX), + keyframe: frame.keyframe, + }); +} + +#[cfg(target_os = "macos")] +fn run_macos_audio(shared: Arc) { + use opennow_streamer_platform_macos::{AudioFormat, SubmitOutcome}; + + let mut configured_channels = 2; + while let Some(frame) = shared.audio.pop() { + if shared.paused.load(Ordering::Acquire) { + continue; + } + let native_sink = shared + .mac_sink + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if native_sink + .as_ref() + .and_then(|sink| sink.fatal_failure()) + .is_some() + { + shared.mac_software_fallback.store(true, Ordering::Release); + } + if shared.mac_software_fallback.load(Ordering::Acquire) { + match OpusDecoder::new(2) { + Ok(decoder) => run_audio_decoder_from(shared, decoder, Some(frame)), + Err(message) => { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message, + }); + } + } + return; + } + let MediaCodec::Opus { channels } = frame.codec else { + continue; + }; + let Some(sink) = native_sink else { + continue; + }; + let channels = channels.clamp(1, 2); + if channels != configured_channels { + if let Err(error) = sink.reconfigure_audio(AudioFormat::new(48_000, channels)) { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: error.to_string(), + }); + continue; + } + configured_channels = channels; + } + match sink.submit_opus(&frame.data) { + Ok(SubmitOutcome::Accepted | SubmitOutcome::Backpressured | SubmitOutcome::Paused) => {} + Ok(SubmitOutcome::ReplacedOldest) => { + let _ = shared.feedback.send(MediaFeedback::QueueDropped { + media: "coreaudio", + count: 1, + }); + } + Err(error) => { + if !shared.stopped.load(Ordering::Acquire) { + let _ = shared.feedback.send(MediaFeedback::DecoderError { + codec: "opus", + message: error.to_string(), + }); + } + } + } + } +} + +#[cfg(target_os = "macos")] +fn mark_macos_video_desynced(shared: &SharedPipeline, mid: &str, reason: &str) { + shared.video_desynced.store(true, Ordering::Release); + if !shared.keyframe_requested.swap(true, Ordering::AcqRel) { + let _ = shared.feedback.send(MediaFeedback::RequestKeyframe { + mid: mid.to_owned(), + reason: reason.to_owned(), + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(target_os = "macos")] + #[test] + fn macos_color_metadata_tracks_accepted_hdr_without_inference_from_depth() { + use opennow_streamer_platform_macos::{VideoColorSpace, VideoTransfer}; + for color_quality in [MediaColorQuality::TenBit420, MediaColorQuality::TenBit444] { + let mut stream = MediaStreamConfig { + color_quality, + ..Default::default() + }; + assert_eq!(stream.macos_color_space(), VideoColorSpace::Bt709); + assert_eq!(stream.macos_transfer(), VideoTransfer::Sdr); + stream.hdr = true; + assert_eq!(stream.macos_color_space(), VideoColorSpace::Bt2020); + assert_eq!(stream.macos_transfer(), VideoTransfer::Pq); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_stream_color_profiles_preserve_chroma_and_bit_depth() { + use opennow_streamer_platform_linux::PixelFormat; + for (color, format) in [ + (MediaColorQuality::EightBit420, PixelFormat::Nv12), + (MediaColorQuality::EightBit444, PixelFormat::Nv24), + (MediaColorQuality::TenBit420, PixelFormat::P010), + (MediaColorQuality::TenBit444, PixelFormat::P410), + ] { + assert_eq!(color.linux_pixel_format(), format); + assert_eq!( + MediaColorQuality::from_linux_pixel_format(format), + Some(color) + ); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_decoded_formats_without_a_color_class_are_not_guessed() { + use opennow_streamer_platform_linux::PixelFormat; + assert_eq!( + MediaColorQuality::from_linux_pixel_format(PixelFormat::I420), + Some(MediaColorQuality::EightBit420) + ); + for format in [PixelFormat::Bgra8, PixelFormat::Rgba8] { + assert_eq!(MediaColorQuality::from_linux_pixel_format(format), None); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_format_change_reports_the_decoded_color_class_once_per_change() { + use opennow_streamer_platform_linux::{ColorTransfer, PixelFormat, StreamFormat}; + + let (feedback, receiver) = std::sync::mpsc::channel(); + let stream = MediaStreamConfig { + codec: MediaVideoCodec::H265, + color_quality: MediaColorQuality::TenBit420, + ..MediaStreamConfig::default() + }; + let shared = SharedPipeline { + video: Arc::new(VideoQueue::new(VIDEO_QUEUE_CAPACITY)), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output: Arc::new(OutputBuffers::new()), + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(false), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream, + linux_session: Mutex::new(None), + linux_software_fallback: Arc::new(AtomicBool::new(false)), + linux_video_mid: Mutex::new(String::new()), + linux_codec: stream.codec, + }; + let format = |pixel_format| StreamFormat { + pixel_format, + color_transfer: ColorTransfer::Sdr, + ..StreamFormat::video_default(1920, 1080).expect("valid default format") + }; + + let mut reported = None; + report_linux_color_format_change(&shared, &mut reported, format(PixelFormat::Nv12)); + assert_eq!( + receiver.try_recv().expect("downgrade report"), + MediaFeedback::ColorFormatChanged { + requested: MediaColorQuality::TenBit420, + actual: MediaColorQuality::EightBit420, + } + ); + report_linux_color_format_change(&shared, &mut reported, format(PixelFormat::Nv12)); + assert!(receiver.try_recv().is_err()); + + report_linux_color_format_change( + &shared, + &mut reported, + StreamFormat { + width: 1280, + height: 720, + ..format(PixelFormat::Nv12) + }, + ); + assert!(receiver.try_recv().is_err()); + + report_linux_color_format_change(&shared, &mut reported, format(PixelFormat::P010)); + assert_eq!( + receiver.try_recv().expect("restored format report"), + MediaFeedback::ColorFormatChanged { + requested: MediaColorQuality::TenBit420, + actual: MediaColorQuality::TenBit420, + } + ); + assert!(receiver.try_recv().is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + fn embedded_linux_rejects_color_downgrades_and_missing_vulkan_owner() { + use opennow_streamer_platform_linux::DecoderPreference; + for (color_quality, preference, expected) in [ + ( + MediaColorQuality::EightBit444, + DecoderPreference::VulkanOnly, + "4:4:4", + ), + ( + MediaColorQuality::TenBit444, + DecoderPreference::VulkanOnly, + "4:4:4", + ), + ( + MediaColorQuality::TenBit420, + DecoderPreference::CudaOnly, + "10-bit", + ), + ( + MediaColorQuality::EightBit420, + DecoderPreference::VulkanOnly, + "attached shared device", + ), + ( + MediaColorQuality::TenBit420, + DecoderPreference::VulkanOnly, + "attached shared device", + ), + ( + MediaColorQuality::TenBit420, + DecoderPreference::SoftwareOnly, + "8-bit 4:2:0 SDR only", + ), + ( + MediaColorQuality::EightBit444, + DecoderPreference::SoftwareOnly, + "8-bit 4:2:0 SDR only", + ), + ( + MediaColorQuality::TenBit444, + DecoderPreference::SoftwareOnly, + "8-bit 4:2:0 SDR only", + ), + ] { + let (feedback, _feedback_receiver) = std::sync::mpsc::channel(); + let (commands, _command_receiver) = std::sync::mpsc::channel(); + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let result = MediaSession::spawn_embedded_linux( + Arc::new(OutputBuffers::new()), + feedback, + commands, + MediaSessionConfig { + stream: MediaStreamConfig { + color_quality, + ..MediaStreamConfig::default() + }, + audio_device: &opennow_streamer_protocol::AudioOutputDevice::default(), + }, + frames, + LinuxVideoSelection { + path: LinuxVideoPath::Hardware(preference), + use_vulkan_output: true, + fallback_reason: None, + }, + None, + ); + assert!(result.err().is_some_and(|error| error.contains(expected))); + } + } + + #[cfg(target_os = "linux")] + fn device_lost_feedback( + receiver: &Receiver, + ) -> (&'static str, bool, Option) { + match receiver + .try_recv() + .expect("a device state feedback for every audio output event") + { + MediaFeedback::DeviceLost { + subsystem, + recovered, + message, + } => (subsystem, recovered, message), + other => panic!("expected a device state feedback, saw {other:?}"), + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_audio_output_feedback_pairs_loss_and_recovery_without_touching_video() { + use opennow_streamer_platform_linux::AudioBackend; + + let (shared, receiver) = software_test_pipeline(); + forward_linux_audio_output_loss(&shared, AudioBackend::PipeWire, "Broken pipe".to_owned()); + forward_linux_audio_output_recovery(&shared, AudioBackend::PipeWire, AudioBackend::Alsa); + + let loss = device_lost_feedback(&receiver); + assert_eq!(loss.0, "PipeWire"); + assert!(!loss.1); + assert_eq!(loss.2.as_deref(), Some("Broken pipe")); + + let recovery = device_lost_feedback(&receiver); + assert_eq!( + recovery.0, "PipeWire", + "recovery must clear the audio subsystem that lost output" + ); + assert!(recovery.1); + assert!( + recovery + .2 + .as_deref() + .is_some_and(|message| message.contains("ALSA") && message.contains("PipeWire")), + "recovery must name the sink that accepted output, saw {:?}", + recovery.2 + ); + + assert!( + receiver.try_recv().is_err(), + "one loss and one recovery only" + ); + assert!(!shared.stopped.load(Ordering::Acquire)); + assert!(!shared.keyframe_requested.load(Ordering::Acquire)); + assert!(shared.video_desynced.load(Ordering::Acquire)); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_audio_output_recovery_pairs_a_backend_that_recovered_itself() { + use opennow_streamer_platform_linux::AudioBackend; + + let (shared, receiver) = software_test_pipeline(); + forward_linux_audio_output_loss(&shared, AudioBackend::Alsa, "Device lost".to_owned()); + forward_linux_audio_output_recovery(&shared, AudioBackend::Alsa, AudioBackend::Alsa); + + let loss = device_lost_feedback(&receiver); + let recovery = device_lost_feedback(&receiver); + assert_eq!(loss, ("ALSA", false, Some("Device lost".to_owned()))); + assert_eq!( + recovery.0, loss.0, + "a backend that accepted output again clears its own loss" + ); + assert!(recovery.1); + assert!(receiver.try_recv().is_err()); + assert!(!shared.stopped.load(Ordering::Acquire)); + assert!(!shared.keyframe_requested.load(Ordering::Acquire)); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_playback_starts_only_once_after_a_successful_record() { + let mut state = EmbeddedD3d11State::new( + MediaStreamConfig::default(), + Arc::new(Mutex::new(EmbeddedD3d11Submission::new())), + ); + assert!(!state.take_first_recorded_frame(false)); + assert!(!state.take_first_recorded_frame(false)); + assert!(state.take_first_recorded_frame(true)); + assert!(!state.take_first_recorded_frame(true)); + state.reset(); + assert!( + !state.take_first_recorded_frame(true), + "surface recreation is not a new session" + ); + } + + #[cfg(target_os = "windows")] + fn embedded_h264_frame( + timestamp_100ns: i64, + key_frame: bool, + ) -> opennow_streamer_platform_windows::EncodedVideoFrame { + opennow_streamer_platform_windows::EncodedVideoFrame { + codec: opennow_streamer_platform_windows::VideoCodec::H264, + data: vec![0, 0, 0, 1, if key_frame { 0x65 } else { 0x41 }], + timestamp_100ns, + duration_100ns: 166_667, + key_frame, + reset_decoder: key_frame, + } + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_retirement_rearms_graphics_and_reference_recovery() { + use crate::GraphicsFrame; + + let submission = Arc::new(Mutex::new(EmbeddedD3d11Submission::new())); + let state = Arc::new(Mutex::new(EmbeddedD3d11State::new( + MediaStreamConfig::default(), + Arc::clone(&submission), + ))); + state.lock().unwrap().first_frame_recorded = true; + let (shared, _feedback) = software_test_pipeline(); + let frame = PendingD3d11Frame { + state: Arc::clone(&state), + shared, + mid: "video".to_owned(), + info: crate::GraphicsFrameInfo { + width: 1920, + height: 1080, + sequence: 1, + presentation_time_ns: 0, + }, + }; + let resources = frame.render_resources().expect("D3D11 render resources"); + assert!(Arc::ptr_eq(&resources, &frame.render_resources().unwrap())); + + for cycle in 0..3 { + submission + .lock() + .unwrap() + .push(embedded_h264_frame(cycle, true)) + .unwrap(); + resources.retire().unwrap(); + resources.retire().unwrap(); + + let mut state = state.lock().unwrap(); + assert!(state.producer.is_none()); + assert!(state.take_keyframe_required()); + assert!(!state.take_keyframe_required()); + assert!(!state.take_first_recorded_frame(true)); + drop(state); + + let mut submission = submission.lock().unwrap(); + assert!(submission.pending.is_empty()); + assert!(submission.submitter.is_none()); + let outcome = submission + .push(embedded_h264_frame(cycle + 1, false)) + .unwrap(); + assert!(outcome.queued && outcome.needs_graphics); + } + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_detaches_submission_before_retiring_decoder() { + struct RetiredDecoder<'a>(&'a Mutex); + impl Drop for RetiredDecoder<'_> { + fn drop(&mut self) { + // Simulate a concurrent submission while decoder teardown joins its + // worker. No old prediction chain or locked submission mutex may remain. + let mut submission = self.0.try_lock().expect("teardown held submission lock"); + assert!(submission.pending.is_empty()); + assert!(submission.submitter.is_none()); + let outcome = submission.push(embedded_h264_frame(100, true)).unwrap(); + assert!(outcome.queued && outcome.needs_graphics); + } + } + let submission = Mutex::new(EmbeddedD3d11Submission::new()); + submission + .lock() + .unwrap() + .push(embedded_h264_frame(1, true)) + .unwrap(); + let mut producer = Some(RetiredDecoder(&submission)); + detach_d3d11_producer(&submission, &mut producer); + assert!(producer.is_none()); + assert_eq!( + submission + .lock() + .unwrap() + .pending + .front() + .unwrap() + .timestamp_100ns, + 100 + ); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_overflow_discards_the_entire_reference_chain() { + let mut state = EmbeddedD3d11Submission::new(); + let capacity = opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY; + for index in 0..capacity { + assert_eq!( + state.push(embedded_h264_frame(index as i64, false)), + Ok(EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped: 0, + needs_graphics: true, + preserved_keyframe: false, + }) + ); + } + assert_eq!(state.pending.len(), capacity); + + assert_eq!( + state.push(embedded_h264_frame(capacity as i64, false)), + Ok(EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: capacity + 1, + needs_graphics: true, + preserved_keyframe: false, + }) + ); + assert!(state.pending.is_empty()); + + assert_eq!( + state.push(embedded_h264_frame(100, true)), + Ok(EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped: 0, + needs_graphics: true, + preserved_keyframe: false, + }) + ); + assert_eq!(state.pending.len(), 1); + assert!(state.pending.front().is_some_and(|frame| frame.key_frame)); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_overflow_keeps_a_queued_keyframe() { + let mut state = EmbeddedD3d11Submission::new(); + let capacity = opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY; + for index in 0..capacity { + assert!( + state + .push(embedded_h264_frame(index as i64, index == 0)) + .is_ok_and(|outcome| outcome.queued && !outcome.preserved_keyframe) + ); + } + assert_eq!( + state.push(embedded_h264_frame(capacity as i64, false)), + Ok(EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: 1, + needs_graphics: true, + preserved_keyframe: true, + }) + ); + assert_eq!(state.pending.len(), capacity); + assert!(state.pending.front().is_some_and(|frame| frame.key_frame)); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_defers_recovery_until_the_retained_keyframe_is_dequeued() { + let mut state = EmbeddedD3d11Submission::new(); + let capacity = opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY; + let deferred = EmbeddedD3d11SubmissionOutcome { + queued: false, + dropped: 1, + needs_graphics: true, + preserved_keyframe: true, + }; + let discarded = EmbeddedD3d11SubmissionOutcome { + preserved_keyframe: false, + ..deferred + }; + for index in 0..capacity { + assert!( + state + .push(embedded_h264_frame(index as i64, index == 0)) + .is_ok_and(|outcome| outcome.queued) + ); + } + for index in capacity..capacity + 300 { + assert_eq!( + state.push(embedded_h264_frame(index as i64, false)), + Ok(deferred) + ); + } + assert_eq!(state.pending.len(), capacity); + + assert!( + state + .pending + .pop_front() + .is_some_and(|frame| frame.key_frame) + ); + assert_eq!(state.push(embedded_h264_frame(1_000, false)), Ok(discarded)); + assert_eq!(state.push(embedded_h264_frame(1_001, false)), Ok(discarded)); + assert_eq!(state.pending.len(), capacity - 1); + + state.pending.clear(); + assert!( + state + .push(embedded_h264_frame(1_002, true)) + .is_ok_and(|outcome| outcome.queued) + ); + assert!( + state + .push(embedded_h264_frame(1_003, false)) + .is_ok_and(|outcome| outcome.queued) + ); + assert_eq!(state.pending.len(), 2); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_d3d11_overflow_retains_the_incoming_recovery_keyframe() { + let mut state = EmbeddedD3d11Submission::new(); + let capacity = opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY; + for index in 0..capacity { + assert!( + state + .push(embedded_h264_frame(index as i64, index == 0)) + .is_ok_and(|outcome| outcome.queued && outcome.dropped == 0) + ); + } + + assert_eq!( + state.push(embedded_h264_frame(capacity as i64, true)), + Ok(EmbeddedD3d11SubmissionOutcome { + queued: true, + dropped: capacity, + needs_graphics: true, + preserved_keyframe: false, + }) + ); + assert_eq!(state.pending.len(), 1); + assert!(state.pending.front().is_some_and(|frame| frame.key_frame)); + } + + #[test] + fn shortcut_chords_parse_supported_keys_and_reject_ambiguous_input() { + assert_eq!( + ShortcutChord::parse("Ctrl+Shift+Q"), + Some(ShortcutChord { + virtual_key: u16::from(b'Q'), + modifiers: 0x03, + }) + ); + assert_eq!( + ShortcutChord::parse("command+f24"), + Some(ShortcutChord { + virtual_key: 0x87, + modifiers: 0x08, + }) + ); + assert_eq!(ShortcutChord::parse("Ctrl"), None); + assert_eq!(ShortcutChord::parse("Ctrl+Control+Q"), None); + assert_eq!(ShortcutChord::parse("Ctrl+Q+W"), None); + assert_eq!(ShortcutChord::parse("Ctrl+"), None); + assert_eq!(ShortcutChord::parse("F25"), None); + } + + #[test] + fn shortcut_bindings_use_custom_values_defaults_and_stable_conflict_order() { + let bindings = StreamShortcutBindings::from_json(&serde_json::json!({ + "toggleStats":"Alt+S", + "togglePointerLock":"Alt+S", + "toggleFullscreen":"not-a-key" + })); + assert_eq!( + bindings.action(u16::from(b'S'), 0x04), + Some(StreamShortcutAction::ToggleStats) + ); + assert_eq!( + bindings.action(0x7a, 0), + Some(StreamShortcutAction::ToggleFullscreen) + ); + assert_eq!( + StreamShortcutBindings::default().action(0x7a, 0x02), + Some(StreamShortcutAction::Screenshot) + ); + } + + #[test] + fn explicitly_cleared_shortcuts_do_not_capture_gameplay_keys() { + let bindings = StreamShortcutBindings::from_json(&serde_json::json!({ + "toggleStats":"", "toggleRecording":"", "saveClip":"" + })); + assert_eq!(bindings.action(u16::from(b'N'), 0x02), None); + assert_eq!(bindings.action(0x7b, 0), None); + assert_eq!(bindings.action(0x7b, 0x02), None); + assert_eq!( + bindings.action(0x7a, 0), + Some(StreamShortcutAction::ToggleFullscreen) + ); + } + + #[test] + fn replay_and_manual_recording_share_payloads_and_stop_independently() { + let recording = RecordingTap::default(); + let receiver = recording.subscribe().unwrap(); + let replay = crate::replay::ReplayTap::default(); + replay.start( + opennow_streamer_protocol::ReplayBufferConfig::from_settings( + &serde_json::json!({"replayBufferEnabled":true}), + ), + ); + let frame = EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0_u8, 0, 0, 1, 0x65]), + frame_index: Some(7), + timestamp: 90_000, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }; + recording.publish(&frame); + replay.publish(&frame); + let recorded = receiver.recv().unwrap(); + let snapshot = replay.snapshot().unwrap(); + assert!(Arc::ptr_eq(&recorded.data, &snapshot.frames[0].frame.data)); + assert_eq!(recorded.timestamp, snapshot.frames[0].frame.timestamp); + replay.stop(); + recording.publish(&frame); + assert_eq!(receiver.recv().unwrap().frame_index, Some(7)); + assert!(recording.subscribe().is_err()); + recording.unsubscribe(); + assert!(receiver.recv().is_err()); + } + + #[test] + fn replay_shortcut_and_qt_portable_key_aliases_are_supported() { + assert_eq!( + StreamShortcutBindings::default().action(0x7b, 0x02), + Some(StreamShortcutAction::SaveClip) + ); + assert_eq!( + StreamShortcutBindings::default().action(0x7b, 0), + Some(StreamShortcutAction::ToggleRecording) + ); + assert_eq!( + StreamShortcutBindings::from_json(&serde_json::json!({"saveClip":"Alt+C"})) + .action(u16::from(b'C'), 0x04), + Some(StreamShortcutAction::SaveClip) + ); + assert_eq!(StreamShortcutAction::SaveClip.protocol_name(), "save-clip"); + for (portable, native) in [ + ("Ctrl+PgUp", "Ctrl+PageUp"), + ("Alt+PgDown", "Alt+PageDown"), + ("Print", "PrintScreen"), + ] { + assert_eq!(ShortcutChord::parse(portable), ShortcutChord::parse(native)); + assert!(ShortcutChord::parse(portable).is_some()); + } + } + + #[test] + fn encoded_recording_tap_fails_closed_on_overflow() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().expect("recording subscription"); + let frame = EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0_u8, 0, 0, 1, 0x65]), + frame_index: Some(1), + timestamp: 0, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }; + for _ in 0..=RECORDING_TAP_QUEUE_CAPACITY { + tap.publish(&frame); + } + assert!(receiver.overflowed()); + assert_eq!( + receiver.recv().unwrap_err(), + RecordingCompletion::Cut { + reason: RecordingCutReason::QueueOverflow, + } + ); + } + + use openh264::encoder::Encoder; + use openh264::formats::{RgbSliceU8, YUVBuffer}; + use opus::{Application, Encoder as OpusEncoder}; + + #[cfg(target_os = "macos")] + #[test] + fn macos_encoded_queue_keeps_bounded_scheduler_burst_tolerance() { + assert_eq!(video_queue_capacity(30), 8); + assert_eq!(video_queue_capacity(60), 15); + assert_eq!(video_queue_capacity(120), 30); + assert_eq!(video_queue_capacity(240), 60); + } + + #[test] + fn video_ingress_capacity_stays_bounded_for_invalid_and_extreme_fps() { + assert_eq!(video_queue_capacity(0), 2); + assert_eq!(video_queue_capacity(60), 15); + assert_eq!(video_queue_capacity(u32::MAX), 60); + } + + #[test] + fn decodes_a_synthetic_h264_keyframe() { + let width = 32; + let height = 32; + let mut rgb = vec![0_u8; width * height * 3]; + for (index, pixel) in rgb.chunks_exact_mut(3).enumerate() { + pixel.copy_from_slice(&[(index % 255) as u8, 64, 192]); + } + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (width, height))); + let mut encoder = Encoder::new().expect("encoder"); + let encoded = encoder.encode(&yuv).expect("encode").to_vec(); + let mut decoder = H264Decoder::new().expect("decoder"); + let decoded = decoder + .decode(&encoded) + .expect("decode") + .expect("decoded frame"); + assert_eq!( + (decoded.width, decoded.height), + (width as u32, height as u32) + ); + assert_eq!(decoded.rgb.len(), width * height * 3); + } + + #[test] + fn software_recovery_replays_parameter_sets_for_an_idr_only_access_unit() { + let rgb = vec![96_u8; 32 * 32 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (32, 32))); + let mut encoder = Encoder::new().unwrap(); + let initial = encoder.encode(&yuv).unwrap().to_vec(); + encoder.force_intra_frame(); + let recovery = encoder.encode(&yuv).unwrap().to_vec(); + let idr_only: Vec = openh264::nal_units(&recovery) + .filter(|nal| nal.get(3).is_some_and(|header| header & 0x1f == 5)) + .flatten() + .copied() + .collect(); + assert!(!idr_only.is_empty()); + assert!(H264Decoder::new().unwrap().decode(&idr_only).is_err()); + + let mut reference = H264Decoder::new().unwrap(); + reference.decode(&initial).unwrap(); + let expected = reference.decode(&idr_only).unwrap().unwrap(); + let mut decoder = H264Decoder::new().unwrap(); + assert!(decoder.decode(&initial).unwrap().is_some()); + let parameter_sets = decoder.parameter_sets.replay(); + assert!(!parameter_sets.is_empty()); + assert!(decoder.decode(&[0, 0, 1, 0x67, 0xff]).is_err()); + assert_eq!(decoder.parameter_sets.replay(), parameter_sets); + decoder.reset().unwrap(); + let recovered = decoder.decode(&idr_only).unwrap().unwrap(); + assert_eq!((recovered.width, recovered.height), (32, 32)); + assert_eq!(recovered.rgb.len(), 32 * 32 * 3); + assert_eq!(recovered.rgb, expected.rgb); + decoder.reset().unwrap(); + assert!(decoder.decode(&idr_only).unwrap().is_some()); + } + + #[test] + fn captured_input_queue_preserves_raw_motion_and_fails_closed_on_control_overflow() { + let queue = CapturedInputQueue::default(); + for _ in 0..3 { + queue.push(CapturedInput::MouseMove { + delta_x: 1, + delta_y: -1, + }); + } + for _ in 0..3 { + assert_eq!( + queue.take(), + Some(CapturedInput::MouseMove { + delta_x: 1, + delta_y: -1, + }) + ); + } + + for virtual_key in 0..=u16::try_from(CAPTURED_INPUT_CAPACITY).unwrap() { + queue.push(CapturedInput::Key { + virtual_key, + modifiers: 0, + pressed: true, + }); + } + assert!(queue.take_overflowed()); + assert_eq!( + queue.take(), + Some(CapturedInput::Key { + virtual_key: 0, + modifiers: 0, + pressed: true, + }) + ); + } + + fn software_test_pipeline() -> (Arc, Receiver) { + let (feedback, receiver) = std::sync::mpsc::channel(); + let shared = Arc::new(SharedPipeline { + video: Arc::new(VideoQueue::new(VIDEO_QUEUE_CAPACITY)), + audio: Arc::new(BoundedQueue::new(AUDIO_QUEUE_CAPACITY)), + output: Arc::new(OutputBuffers::new()), + feedback, + paused: AtomicBool::new(false), + video_desynced: AtomicBool::new(true), + keyframe_requested: AtomicBool::new(false), + stopped: AtomicBool::new(false), + recording_tap: RecordingTap::default(), + replay_tap: crate::replay::ReplayTap::default(), + stream: MediaStreamConfig::default(), + #[cfg(target_os = "macos")] + mac_sink: Mutex::new(None), + #[cfg(target_os = "macos")] + mac_software_fallback: AtomicBool::new(true), + #[cfg(target_os = "windows")] + windows_bridge: Arc::new(WindowsBridge::new()), + #[cfg(target_os = "linux")] + linux_session: Mutex::new(None), + #[cfg(target_os = "linux")] + linux_software_fallback: Arc::new(AtomicBool::new(true)), + #[cfg(target_os = "linux")] + linux_video_mid: Mutex::new(String::new()), + #[cfg(target_os = "linux")] + linux_codec: MediaVideoCodec::H264, + }); + (shared, receiver) + } + + #[test] + fn software_handoff_decodes_the_pending_h264_keyframe() { + let width = 32; + let height = 32; + let rgb = vec![96_u8; width * height * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (width, height))); + let mut encoder = Encoder::new().expect("encoder"); + let encoded: Arc<[u8]> = encoder.encode(&yuv).expect("encode").to_vec().into(); + let (shared, _receiver) = software_test_pipeline(); + let output = Arc::clone(&shared.output); + shared.video.close(); + run_video_decoder_from( + shared, + H264Decoder::new().expect("decoder"), + Some(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: encoded, + frame_index: Some(1), + timestamp: 0, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }), + ); + let decoded = output.take_video().expect("decoded pending frame"); + assert_eq!( + (decoded.width, decoded.height), + (width as u32, height as u32) + ); + } + + #[test] + fn software_worker_recovers_an_idr_without_repeated_parameter_sets() { + let rgb = vec![96_u8; 32 * 32 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (32, 32))); + let mut encoder = Encoder::new().unwrap(); + let initial = encoder.encode(&yuv).unwrap().to_vec(); + encoder.force_intra_frame(); + let recovery = encoder.encode(&yuv).unwrap().to_vec(); + let idr_only: Vec = openh264::nal_units(&recovery) + .filter(|nal| nal.get(3).is_some_and(|header| header & 0x1f == 5)) + .flatten() + .copied() + .collect(); + let mut decoder = H264Decoder::new().unwrap(); + assert!(decoder.decode(&initial).unwrap().is_some()); + assert!(decoder.decode(&[0, 0, 1, 0x67, 0xff]).is_err()); + let (shared, feedback) = software_test_pipeline(); + let output = Arc::clone(&shared.output); + shared.video.close(); + run_video_decoder_from( + shared, + decoder, + Some(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: idr_only.into(), + frame_index: Some(71), + timestamp: 4500, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: false, + ssrc: None, + }), + ); + let decoded = output.take_video().expect("recovered IDR"); + assert_eq!((decoded.width, decoded.height), (32, 32)); + assert!(matches!( + feedback.try_recv(), + Ok(MediaFeedback::VideoFrameAccepted { + frame_index: Some(71), + timestamp: 4500, + keyframe: true, + .. + }) + )); + assert!(feedback.try_recv().is_err()); + } + + #[test] + fn software_worker_does_not_decode_a_discontinuous_delta_frame() { + let rgb = vec![96_u8; 32 * 32 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (32, 32))); + let mut encoder = Encoder::new().unwrap(); + let initial = encoder.encode(&yuv).unwrap().to_vec(); + let delta = encoder.encode(&yuv).unwrap().to_vec(); + assert!(!delta.is_empty()); + assert!( + !openh264::nal_units(&delta) + .any(|nal| nal.get(3).is_some_and(|header| header & 0x1f == 5)) + ); + let mut decoder = H264Decoder::new().unwrap(); + assert!(decoder.decode(&initial).unwrap().is_some()); + let (shared, feedback) = software_test_pipeline(); + shared.video_desynced.store(false, Ordering::Release); + shared.video.close(); + run_video_decoder_from( + Arc::clone(&shared), + decoder, + Some(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: delta.into(), + frame_index: Some(72), + timestamp: 5250, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: false, + ssrc: None, + }), + ); + assert!(shared.video_desynced.load(Ordering::Acquire)); + assert!(shared.output.take_video().is_none()); + assert!(feedback.try_recv().is_err()); + } + + #[test] + fn software_worker_requests_once_until_a_keyframe_decodes() { + let rgb = vec![96_u8; 32 * 32 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (32, 32))); + let valid_keyframe: Arc<[u8]> = Encoder::new() + .unwrap() + .encode(&yuv) + .unwrap() + .to_vec() + .into(); + let (shared, feedback) = software_test_pipeline(); + let worker_shared = Arc::clone(&shared); + let worker = std::thread::spawn(move || { + run_video_decoder(worker_shared, H264Decoder::new().unwrap()); + }); + let make_frame = |id, data: Arc<[u8]>| EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data, + frame_index: Some(id), + timestamp: u64::from(id) * 750, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }; + let malformed: Arc<[u8]> = Arc::from([0, 0, 1, 0x67, 0xff]); + for id in 1..=3 { + shared + .video + .push(make_frame(id, Arc::clone(&malformed))) + .unwrap(); + assert!(matches!( + feedback.recv_timeout(std::time::Duration::from_secs(2)), + Ok(MediaFeedback::DecoderError { codec: "h264", .. }) + )); + if id == 1 { + assert!(matches!( + feedback.recv_timeout(std::time::Duration::from_secs(2)), + Ok(MediaFeedback::RequestKeyframe { .. }) + )); + } else { + assert!(feedback.try_recv().is_err()); + } + assert!(shared.video_desynced.load(Ordering::Acquire)); + assert!(shared.output.take_video().is_none()); + } + shared.video.push(make_frame(4, valid_keyframe)).unwrap(); + assert!(matches!( + feedback.recv_timeout(std::time::Duration::from_secs(2)), + Ok(MediaFeedback::VideoFrameAccepted { + frame_index: Some(4), + .. + }) + )); + shared.video.close(); + worker.join().unwrap(); + assert!(!shared.video_desynced.load(Ordering::Acquire)); + assert!(!shared.keyframe_requested.load(Ordering::Acquire)); + assert!(shared.output.take_video().is_some()); + assert!(feedback.try_recv().is_err()); + } + + #[test] + fn software_worker_accepts_a_buffered_keyframe_before_output() { + let (shared, feedback) = software_test_pipeline(); + shared.keyframe_requested.store(true, Ordering::Release); + let mut decoder = H264Decoder::new().unwrap(); + let decoder_shared = Arc::clone(&shared); + let mut has_reference = false; + decoder.decode_for_test = Some(Box::new(move |encoded| match encoded { + b"idr" => { + has_reference = true; + Ok(None) + } + b"delta" if has_reference => { + assert!(decoder_shared.keyframe_requested.load(Ordering::Acquire)); + Ok(Some(DecodedVideoFrame { + width: 1, + height: 1, + rgb: vec![96; 3], + })) + } + _ => Err("missing reference".to_owned()), + })); + let worker_shared = Arc::clone(&shared); + let worker = std::thread::spawn(move || run_video_decoder(worker_shared, decoder)); + shared + .video + .push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from(&b"idr"[..]), + frame_index: Some(1), + timestamp: 750, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }) + .unwrap(); + shared + .video + .push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from(&b"delta"[..]), + frame_index: Some(2), + timestamp: 1500, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: true, + ssrc: None, + }) + .unwrap(); + assert!(matches!( + feedback.recv_timeout(std::time::Duration::from_secs(2)), + Ok(MediaFeedback::VideoFrameAccepted { + frame_index: Some(2), + .. + }) + )); + shared.video.close(); + worker.join().unwrap(); + assert!(!shared.video_desynced.load(Ordering::Acquire)); + assert!(!shared.keyframe_requested.load(Ordering::Acquire)); + assert!(shared.output.take_video().is_some()); + assert!(feedback.try_recv().is_err()); + } + + #[test] + fn decodes_synthetic_stereo_opus() { + let mut encoder = OpusEncoder::new(OPUS_SAMPLE_RATE, Channels::Stereo, Application::Audio) + .expect("encoder"); + let input: Vec = (0..960 * 2) + .map(|sample| ((sample as f32 / 24.0).sin()) * 0.25) + .collect(); + let mut packet = vec![0_u8; 4_000]; + let encoded_len = encoder.encode_float(&input, &mut packet).expect("encode"); + let mut decoder = OpusDecoder::new(2).expect("decoder"); + let decoded = decoder.decode(&packet[..encoded_len]).expect("decode"); + assert_eq!(decoded.len(), input.len()); + assert!(decoded.iter().any(|sample| sample.abs() > 0.001)); + + let concealed = decoder + .decode_packet_loss() + .expect("packet-loss concealment"); + assert_eq!(concealed.len(), input.len()); + assert!(concealed.iter().all(|sample| sample.is_finite())); + } + + #[test] + fn converts_rtp_video_timestamps_to_media_foundation_time() { + assert_eq!(media_timestamp_100ns(0, 90_000), 0); + assert_eq!(media_timestamp_100ns(90_000, 90_000), 10_000_000); + assert_eq!(media_timestamp_100ns(45_000, 90_000), 5_000_000); + assert_eq!(media_timestamp_100ns(90_000, 0), 0); + } + + #[test] + fn adaptive_sample_clock_uses_the_negotiated_fps_only_as_a_ceiling() { + let mut clock = AdaptiveSampleClock::new(120); + assert_eq!(clock.observe(0), 83_333); + assert_eq!(clock.observe(83_333), 83_333); + assert_eq!(clock.observe(250_000), 166_667); + assert_eq!(clock.observe(583_334), 333_334); + } + + #[test] + fn adaptive_sample_clock_recovers_from_a_repeated_timestamp() { + let mut clock = AdaptiveSampleClock::new(120); + assert_eq!(clock.observe(0), 83_333); + assert_eq!(clock.observe(83_333), 83_333); + assert_eq!(clock.observe(83_333), 83_333); + } + + #[test] + fn paused_and_stopped_sessions_reject_frames() { + let (feedback, _receiver) = std::sync::mpsc::channel(); + let (commands, _host) = std::sync::mpsc::channel(); + let session = MediaSession::spawn( + Arc::new(OutputBuffers::new()), + feedback, + commands, + false, + MediaStreamConfig::default(), + #[cfg(target_os = "windows")] + Arc::new(WindowsBridge::new()), + #[cfg(target_os = "linux")] + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: None, + }, + #[cfg(target_os = "linux")] + Arc::new(AtomicBool::new(true)), + ) + .expect("session"); + let sink = session.sink(); + session.set_paused(true); + assert_eq!( + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([]), + frame_index: Some(1), + timestamp: 0, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: true, + ssrc: None, + }), + PushOutcome::Paused + ); + session.stop(); + assert_eq!( + sink.push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([]), + frame_index: Some(2), + timestamp: 0, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: true, + ssrc: None, + }), + PushOutcome::Closed + ); + } + + #[test] + fn requests_a_keyframe_when_video_starts_mid_gop() { + let (feedback, receiver) = std::sync::mpsc::channel(); + let (commands, _host) = std::sync::mpsc::channel(); + let session = MediaSession::spawn( + Arc::new(OutputBuffers::new()), + feedback, + commands, + false, + MediaStreamConfig::default(), + #[cfg(target_os = "windows")] + Arc::new(WindowsBridge::new()), + #[cfg(target_os = "linux")] + LinuxVideoSelection { + path: LinuxVideoPath::Software, + use_vulkan_output: false, + fallback_reason: None, + }, + #[cfg(target_os = "linux")] + Arc::new(AtomicBool::new(true)), + ) + .expect("session"); + assert_eq!( + session.sink().push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0_u8, 0, 0, 1, 1]), + frame_index: Some(1), + timestamp: 0, + clock_rate_hz: 90_000, + keyframe: false, + contiguous: true, + ssrc: None, + }), + PushOutcome::DroppedOldest + ); + assert!(matches!( + receiver.recv_timeout(std::time::Duration::from_secs(1)), + Ok(MediaFeedback::RequestKeyframe { mid, .. }) if mid == "video" + )); + session.stop(); + } + + #[cfg(target_os = "linux")] + #[test] + fn converts_rtp_timestamps_to_microseconds_without_overflow() { + assert_eq!(media_timestamp_us(180_000, 90_000), 2_000_000); + assert_eq!(media_timestamp_us(48_000, 48_000), 1_000_000); + assert_eq!(media_timestamp_us(u64::MAX, 90_000), u64::MAX / 90_000); + assert_eq!(media_timestamp_us(42, 0), 0); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media_recording_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media_recording_tests.rs new file mode 100644 index 000000000..85c715a27 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/media_recording_tests.rs @@ -0,0 +1,553 @@ +use super::*; +use std::fs::{File, OpenOptions}; +use std::io::{self, Seek, SeekFrom, Write}; +use std::sync::mpsc; +use std::time::Duration; + +const DEADLOCK_TIMEOUT: Duration = Duration::from_secs(5); + +fn frame(index: u32) -> EncodedFrame { + EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0, 0, 0, 1, 0x65, 0x88]), + frame_index: Some(index), + timestamp: 90_000 + u64::from(index) * 3_000, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: Some(9), + } +} + +fn cut(reason: RecordingCutReason) -> RecordingCompletion { + RecordingCompletion::Cut { reason } +} + +fn terminal(receiver: &EncodedRecordingReceiver) -> RecordingCompletion { + receiver + .recv() + .expect_err("recording must not admit another frame") +} + +fn waiting_receiver( + mut receiver: EncodedRecordingReceiver, +) -> ( + mpsc::Receiver<()>, + mpsc::Receiver>, + JoinHandle<()>, +) { + let (waiting, entered_wait) = mpsc::sync_channel(0); + receiver.before_wait = Some(waiting); + let (result, completed) = mpsc::sync_channel(1); + let worker = thread::spawn(move || { + let _ = result.send(receiver.recv()); + }); + (entered_wait, completed, worker) +} + +#[test] +fn close_before_receive_needs_no_later_frame() { + for completion in [ + RecordingCompletion::Complete, + cut(RecordingCutReason::Discontinuity), + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ] { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + tap.close(completion); + assert_eq!(terminal(&receiver), completion); + assert_eq!(terminal(&receiver), completion); + } +} + +#[test] +fn signal_between_empty_check_and_wait_is_not_lost() { + for completion in [ + RecordingCompletion::Complete, + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ] { + let tap = RecordingTap::default(); + let mut receiver = tap.subscribe().unwrap(); + let (release, wait_release) = mpsc::sync_channel(0); + receiver.wait_release = Some(wait_release); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + tap.close(completion); + release.send(()).unwrap(); + drop(release); + assert_eq!( + completed + .recv_timeout(DEADLOCK_TIMEOUT) + .unwrap() + .unwrap_err(), + completion + ); + worker.join().unwrap(); + } +} + +#[test] +fn close_wakes_receiver_after_it_enters_wait_without_more_frames() { + for completion in [ + RecordingCompletion::Complete, + cut(RecordingCutReason::Discontinuity), + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ] { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + assert!(matches!( + tap.wake_receiver.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + assert!(matches!( + completed.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + tap.close(completion); + assert_eq!( + completed + .recv_timeout(DEADLOCK_TIMEOUT) + .unwrap() + .unwrap_err(), + completion + ); + worker.join().unwrap(); + } +} + +#[test] +fn graceful_signal_before_sender_drop_can_wake_and_wait_again() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + tap.signal( + tap.terminal.load(Ordering::Acquire), + RecordingCompletion::Complete, + ); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + assert!(matches!( + completed.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + tap.unsubscribe(); + assert_eq!( + completed + .recv_timeout(DEADLOCK_TIMEOUT) + .unwrap() + .unwrap_err(), + RecordingCompletion::Complete + ); + worker.join().unwrap(); +} + +#[test] +fn publishing_wakes_a_waiting_receiver_with_the_original_frame() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + let expected = frame(73); + tap.publish(&expected); + let received = completed.recv_timeout(DEADLOCK_TIMEOUT).unwrap().unwrap(); + assert_eq!(received.frame_index, expected.frame_index); + assert_eq!(received.timestamp, expected.timestamp); + assert_eq!(received.clock_rate_hz, expected.clock_rate_hz); + assert_eq!(received.ssrc, expected.ssrc); + assert!(Arc::ptr_eq(&received.data, &expected.data)); + worker.join().unwrap(); +} + +#[test] +fn queued_data_and_close_between_empty_check_and_wait_obey_completion() { + for completion in [ + RecordingCompletion::Complete, + cut(RecordingCutReason::Interrupted), + ] { + let tap = RecordingTap::default(); + let mut receiver = tap.subscribe().unwrap(); + let (release, wait_release) = mpsc::sync_channel(0); + receiver.wait_release = Some(wait_release); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + tap.publish(&frame(81)); + tap.close(completion); + release.send(()).unwrap(); + drop(release); + let result = completed.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + match completion { + RecordingCompletion::Complete => assert_eq!(result.unwrap().frame_index, Some(81)), + RecordingCompletion::Cut { .. } => assert_eq!(result.unwrap_err(), completion), + } + worker.join().unwrap(); + } +} + +#[test] +fn subscription_contention_cuts_without_blocking_the_publisher_or_more_traffic() { + for hold_before_block in [true, false] { + let tap = Arc::new(RecordingTap::default()); + let mut receiver = tap.subscribe().unwrap(); + let (release, wait_release) = mpsc::sync_channel(0); + if hold_before_block { + receiver.wait_release = Some(wait_release); + } + let (entered_wait, completed, receiver_worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + let subscription = tap.subscription.lock().unwrap(); + let publisher_tap = Arc::clone(&tap); + let (published, publication_finished) = mpsc::sync_channel(1); + let publisher = thread::spawn(move || { + publisher_tap.publish(&frame(1)); + published.send(()).unwrap(); + }); + publication_finished.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + if hold_before_block { + release.send(()).unwrap(); + } + drop(release); + assert_eq!( + completed + .recv_timeout(DEADLOCK_TIMEOUT) + .unwrap() + .unwrap_err(), + cut(RecordingCutReason::QueueOverflow) + ); + assert!(subscription.sender.is_some()); + drop(subscription); + publisher.join().unwrap(); + receiver_worker.join().unwrap(); + } +} + +#[test] +fn graceful_close_drains_accepted_frames_but_every_cut_rejects_them() { + let completions = [ + RecordingCompletion::Complete, + cut(RecordingCutReason::Discontinuity), + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ]; + for completion in completions { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + tap.publish(&frame(1)); + tap.publish(&frame(2)); + tap.close(completion); + tap.publish(&frame(3)); + if completion == RecordingCompletion::Complete { + assert_eq!(receiver.recv().unwrap().frame_index, Some(1)); + assert_eq!(receiver.recv().unwrap().frame_index, Some(2)); + } + assert_eq!(terminal(&receiver), completion); + } +} + +#[test] +fn terminal_reason_is_first_wins_even_when_close_is_repeated() { + let completions = [ + RecordingCompletion::Complete, + cut(RecordingCutReason::Discontinuity), + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ]; + for first in completions { + for second in completions { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + tap.close(first); + tap.close(second); + tap.unsubscribe(); + assert_eq!(terminal(&receiver), first); + } + } +} + +#[test] +fn old_receiver_lifetime_prevents_terminal_state_reset_on_resubscribe() { + for completion in [ + RecordingCompletion::Complete, + cut(RecordingCutReason::QueueOverflow), + cut(RecordingCutReason::Interrupted), + ] { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + assert!(tap.subscribe().is_err()); + tap.close(completion); + let old_status = tap.terminal.load(Ordering::Acquire); + assert!(tap.subscribe().is_err()); + assert_eq!(tap.terminal.load(Ordering::Acquire), old_status); + assert_eq!(terminal(&receiver), completion); + assert!(tap.subscribe().is_err()); + drop(receiver); + let replacement = tap.subscribe().unwrap(); + assert_ne!(tap.terminal.load(Ordering::Acquire), old_status); + tap.publish(&frame(2)); + tap.unsubscribe(); + assert_eq!(replacement.recv().unwrap().frame_index, Some(2)); + assert_eq!(terminal(&replacement), RecordingCompletion::Complete); + } +} + +#[test] +fn stale_publisher_compare_exchange_cannot_cut_the_next_subscription() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + let stale_running_status = tap.terminal.load(Ordering::Acquire); + tap.unsubscribe(); + drop(receiver); + let receiver = tap.subscribe().unwrap(); + let current_status = tap.terminal.load(Ordering::Acquire); + assert_ne!(current_status, stale_running_status); + tap.signal(stale_running_status, cut(RecordingCutReason::QueueOverflow)); + assert_eq!(tap.terminal.load(Ordering::Acquire), current_status); + assert_eq!(receiver.cut(), None); + tap.publish(&frame(42)); + tap.unsubscribe(); + assert_eq!(receiver.recv().unwrap().frame_index, Some(42)); + assert_eq!(terminal(&receiver), RecordingCompletion::Complete); +} + +#[test] +fn stale_wake_is_consumed_once_then_the_new_receiver_waits_for_its_own_signal() { + let tap = RecordingTap::default(); + let previous = tap.subscribe().unwrap(); + tap.unsubscribe(); + assert_eq!(terminal(&previous), RecordingCompletion::Complete); + drop(previous); + let receiver = tap.subscribe().unwrap(); + let (entered_wait, completed, worker) = waiting_receiver(receiver); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + entered_wait.recv_timeout(DEADLOCK_TIMEOUT).unwrap(); + drop(entered_wait); + assert!(matches!( + tap.wake_receiver.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + assert!(matches!( + completed.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + tap.close(cut(RecordingCutReason::Interrupted)); + assert_eq!( + completed + .recv_timeout(DEADLOCK_TIMEOUT) + .unwrap() + .unwrap_err(), + cut(RecordingCutReason::Interrupted) + ); + worker.join().unwrap(); +} + +#[test] +fn wake_notifications_coalesce_and_overflow_does_not_drain_pending_frames() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + for index in 0..RECORDING_TAP_QUEUE_CAPACITY { + tap.publish(&frame(index as u32)); + } + { + let wake = tap.wake_receiver.lock().unwrap(); + assert_eq!(wake.try_recv(), Ok(())); + assert_eq!(wake.try_recv(), Err(mpsc::TryRecvError::Empty)); + } + tap.publish(&frame(RECORDING_TAP_QUEUE_CAPACITY as u32)); + assert!(receiver.overflowed()); + assert_eq!(terminal(&receiver), cut(RecordingCutReason::QueueOverflow)); + tap.unsubscribe(); + assert_eq!(terminal(&receiver), cut(RecordingCutReason::QueueOverflow)); +} + +fn directory(name: &str) -> std::path::PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + for _ in 0..128 { + let directory = std::env::temp_dir().join(format!( + "opennow-recording-{name}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed), + )); + match std::fs::create_dir(&directory) { + Ok(()) => return directory, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => panic!("failed to create recording fixture directory: {error}"), + } + } + panic!("exhausted recording fixture directory allocation attempts"); +} + +#[test] +fn recording_fixture_directories_are_exclusively_owned() { + let first = directory("same-clock-tick"); + let second = directory("same-clock-tick"); + assert_ne!(first, second); + std::fs::remove_dir_all(first).unwrap(); + std::fs::remove_dir_all(second).unwrap(); +} + +#[test] +fn overflow_before_any_mux_commit_does_not_create_a_saved_recording() { + let tap = RecordingTap::default(); + let receiver = tap.subscribe().unwrap(); + for index in 0..=RECORDING_TAP_QUEUE_CAPACITY { + tap.publish(&frame(index as u32)); + } + let directory = directory("no-prefix-overflow"); + let output = directory.join("overflow.mkv"); + let error = crate::recording::record_matroska(&output, MediaStreamConfig::default(), receiver) + .expect_err("queued frames are not a mux-committed prefix"); + assert_eq!( + error, + "recording ended before a decodable video keyframe arrived" + ); + assert!(!output.exists()); + assert!(!directory.join(".overflow.mkv.part").exists()); + std::fs::remove_dir_all(directory).unwrap(); +} + +struct CutAfterPayload { + file: File, + payload: Vec, + triggered: Arc, + tap: Arc, + reason: RecordingCutReason, +} + +impl Write for CutAfterPayload { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let written = self.file.write(bytes)?; + if bytes[..written].ends_with(&self.payload) && !self.triggered.swap(true, Ordering::AcqRel) + { + match self.reason { + RecordingCutReason::QueueOverflow => { + for index in 1..=RECORDING_TAP_QUEUE_CAPACITY + 1 { + self.tap.publish(&frame(index as u32)); + } + } + reason => { + self.tap.publish(&frame(1)); + self.tap.close(cut(reason)); + self.tap.publish(&frame(2)); + } + } + } + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } +} + +impl Seek for CutAfterPayload { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.file.seek(position) + } +} + +#[test] +fn recording_overflow_saves_only_the_mux_committed_video_prefix() { + assert_mux_committed_prefix(RecordingCutReason::QueueOverflow); +} + +#[test] +fn recording_interruption_saves_only_the_mux_committed_video_prefix() { + assert_mux_committed_prefix(RecordingCutReason::Interrupted); +} + +#[test] +fn recording_discontinuity_cut_saves_only_the_mux_committed_video_prefix() { + assert_mux_committed_prefix(RecordingCutReason::Discontinuity); +} + +fn assert_mux_committed_prefix(reason: RecordingCutReason) { + use openh264::encoder::Encoder; + use openh264::formats::{RgbSliceU8, YUVBuffer}; + use oxideav_core::{NullCodecResolver, ReadSeek, WriteSeek}; + use oxideav_mkv::avc::annexb_to_avcc; + + let tap = Arc::new(RecordingTap::default()); + let receiver = tap.subscribe().unwrap(); + let rgb = vec![64_u8; 32 * 32 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (32, 32))); + let encoded = Encoder::new().unwrap().encode(&yuv).unwrap().to_vec(); + let packetized = annexb_to_avcc(&encoded).packetized; + assert!(!packetized.is_empty()); + let first = EncodedFrame { + data: encoded.into(), + ..frame(0) + }; + tap.publish(&first); + let directory = directory("committed-prefix"); + let output = directory.join("overflow.mkv"); + let triggered = Arc::new(AtomicBool::new(false)); + let writer_triggered = Arc::clone(&triggered); + let writer_tap = Arc::clone(&tap); + let expected_payload = packetized.clone(); + let worker_output = output.clone(); + let (finished, result) = mpsc::sync_channel(1); + let worker = thread::spawn(move || { + let summary = crate::recording::record_matroska_with( + &worker_output, + MediaStreamConfig { + width: 32, + height: 32, + fps: 30, + ..MediaStreamConfig::default() + }, + receiver, + move |path| { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|error| error.to_string())?; + Ok(Box::new(CutAfterPayload { + file, + payload: expected_payload, + triggered: writer_triggered, + tap: writer_tap, + reason, + }) as Box) + }, + |part, output| { + crate::recording::publish_manual_recording(part, output) + .map_err(|error| error.to_string()) + }, + ); + let _ = finished.send(summary); + }); + let result = result.recv_timeout(DEADLOCK_TIMEOUT); + if result.is_err() { + tap.close(cut(RecordingCutReason::Interrupted)); + } + worker.join().unwrap(); + let summary = result + .expect("recorder must finish after the payload-triggered cut without more input") + .expect("a committed valid packet must survive a recording cut"); + assert!(triggered.load(Ordering::Acquire)); + assert_eq!(summary.completion, cut(reason)); + assert_eq!(summary.media.video_packets, 1); + assert_eq!(summary.media.audio_packets, 0); + assert!(!directory.join(".overflow.mkv.part").exists()); + let file: Box = Box::new(File::open(&output).unwrap()); + let mut demuxer = oxideav_mkv::demux::open(file, &NullCodecResolver).unwrap(); + assert_eq!(demuxer.streams()[0].params.codec_id.as_str(), "h264"); + let saved = demuxer.next_packet().unwrap(); + assert_eq!(saved.data.as_slice(), packetized); + assert_eq!(saved.pts, Some(0)); + assert!(demuxer.next_packet().is_err()); + drop(demuxer); + std::fs::remove_dir_all(directory).unwrap(); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/microphone.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/microphone.rs new file mode 100644 index 000000000..33d27fc6d --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/microphone.rs @@ -0,0 +1,1114 @@ +use std::cell::UnsafeCell; +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use sdl2::audio::AudioCallback; +use sdl2::sys::{self, SDL_AudioStatus}; + +pub const MICROPHONE_SAMPLE_RATE: u32 = 48_000; +// The server describes `mic.frameSize:10`: 10 ms at 48 kHz. +pub const MICROPHONE_FRAME_SAMPLES: usize = 480; +const QUEUE_CAPACITY: usize = 5; +const MAX_FRAME_AGE: Duration = Duration::from_millis(100); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedMicrophoneFrame { + pub opus: Vec, + pub timestamp: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MicrophoneStatus { + pub enabled: bool, + pub error: Option, + pub dropped_pcm_frames: u64, + pub dropped_encoded_frames: u64, +} + +#[derive(Debug)] +struct PcmFrame { + samples: [i16; MICROPHONE_FRAME_SAMPLES], + timestamp: u32, + captured_at: Instant, + generation: u64, +} + +#[derive(Debug)] +struct EncodedPacket { + frame: EncodedMicrophoneFrame, + captured_at: Instant, +} + +#[derive(Debug)] +struct QueueState { + pcm: VecDeque, + encoded: VecDeque, + stopped: bool, + error: Option, + last_pcm: Instant, + generation: u64, +} + +#[derive(Debug)] +pub(crate) struct MicrophoneShared { + state: Mutex, + ready: Condvar, + dropped_pcm: AtomicU64, + dropped_encoded: AtomicU64, + stopped: AtomicBool, + device: Mutex>, +} + +impl MicrophoneShared { + pub(crate) fn new() -> Arc { + Arc::new(Self { + state: Mutex::new(QueueState { + pcm: VecDeque::with_capacity(QUEUE_CAPACITY), + encoded: VecDeque::with_capacity(QUEUE_CAPACITY), + stopped: false, + error: None, + last_pcm: Instant::now(), + generation: 0, + }), + ready: Condvar::new(), + dropped_pcm: AtomicU64::new(0), + dropped_encoded: AtomicU64::new(0), + stopped: AtomicBool::new(false), + device: Mutex::new(None), + }) + } + + pub(crate) fn close(&self, error: Option) { + self.stopped.store(true, Ordering::Release); + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if !state.stopped { + state.stopped = true; + if error.is_some() { + state.error = error; + } + state.pcm.clear(); + state.encoded.clear(); + self.ready.notify_all(); + } + drop(state); + self.pause_device(); + } + + pub(crate) fn stopped(&self) -> bool { + self.stopped.load(Ordering::Acquire) + } + + fn pause_device(&self) { + let device = self + .device + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Some(id) = *device { + unsafe { sys::SDL_PauseAudioDevice(id, 1) }; + } + } + + fn device_status(&self) -> Option { + let device = self + .device + .lock() + .unwrap_or_else(|error| error.into_inner()); + device.map(|id| unsafe { sys::SDL_GetAudioDeviceStatus(id) }) + } + + fn poll_device(&self) { + if self.stopped() { + return; + } + let failed = self + .device_status() + .is_some_and(|status| status == SDL_AudioStatus::SDL_AUDIO_PAUSED); + if failed { + self.close(Some( + "microphone capture device stopped or disconnected".to_owned(), + )); + } + } + + fn push_pcm( + &self, + samples: &[i16; MICROPHONE_FRAME_SAMPLES], + timestamp: u32, + captured_at: Instant, + ) { + let Ok(mut state) = self.state.try_lock() else { + self.dropped_pcm.fetch_add(1, Ordering::Relaxed); + return; + }; + if state.stopped { + return; + } + state.last_pcm = Instant::now(); + if state.pcm.len() == QUEUE_CAPACITY { + state.pcm.pop_front(); + self.dropped_pcm.fetch_add(1, Ordering::Relaxed); + } + let generation = state.generation; + state.pcm.push_back(PcmFrame { + samples: *samples, + timestamp, + captured_at, + generation, + }); + self.ready.notify_one(); + } + + fn pop_pcm(&self) -> Option { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + loop { + if state.stopped { + return None; + } + if let Some(frame) = state.pcm.pop_front() { + if frame.captured_at.elapsed() > MAX_FRAME_AGE { + self.dropped_pcm.fetch_add(1, Ordering::Relaxed); + continue; + } + return Some(frame); + } + let (next, _) = self + .ready + .wait_timeout(state, Duration::from_secs(1)) + .unwrap_or_else(|error| error.into_inner()); + state = next; + if !state.stopped && state.last_pcm.elapsed() >= Duration::from_secs(2) { + drop(state); + self.close(Some("microphone capture stopped delivering PCM".to_owned())); + return None; + } + } + } + + fn push_encoded(&self, frame: EncodedMicrophoneFrame, captured_at: Instant, generation: u64) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.stopped { + return; + } + if state.generation != generation { + self.dropped_encoded.fetch_add(1, Ordering::Relaxed); + return; + } + if captured_at.elapsed() > MAX_FRAME_AGE { + self.dropped_encoded.fetch_add(1, Ordering::Relaxed); + return; + } + if state.encoded.len() == QUEUE_CAPACITY { + state.encoded.pop_front(); + self.dropped_encoded.fetch_add(1, Ordering::Relaxed); + } + state + .encoded + .push_back(EncodedPacket { frame, captured_at }); + } + + fn flush_for_reopen(&self) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.generation = state.generation.wrapping_add(1); + self.dropped_pcm + .fetch_add(state.pcm.len() as u64, Ordering::Relaxed); + self.dropped_encoded + .fetch_add(state.encoded.len() as u64, Ordering::Relaxed); + state.pcm.clear(); + state.encoded.clear(); + } +} + +struct CaptureCallback { + shared: Arc, + samples: [i16; MICROPHONE_FRAME_SAMPLES], + filled: usize, + timestamp: u32, + captured_at: Instant, +} + +impl CaptureCallback { + fn new(shared: Arc) -> Self { + Self { + shared, + samples: [0; MICROPHONE_FRAME_SAMPLES], + filled: 0, + timestamp: 0, + captured_at: Instant::now(), + } + } +} + +impl AudioCallback for CaptureCallback { + type Channel = i16; + + fn callback(&mut self, mut input: &mut [i16]) { + if self.shared.stopped() { + self.filled = 0; + self.samples.fill(0); + return; + } + while !input.is_empty() { + if self.filled == 0 { + self.captured_at = Instant::now(); + } + let count = input.len().min(MICROPHONE_FRAME_SAMPLES - self.filled); + self.samples[self.filled..self.filled + count].copy_from_slice(&input[..count]); + self.filled += count; + input = &mut input[count..]; + if self.filled == MICROPHONE_FRAME_SAMPLES { + self.shared + .push_pcm(&self.samples, self.timestamp, self.captured_at); + self.timestamp = self.timestamp.wrapping_add(MICROPHONE_FRAME_SAMPLES as u32); + self.filled = 0; + } + } + } +} + +fn encoder() -> Result { + let mut encoder = opus::Encoder::new( + MICROPHONE_SAMPLE_RATE, + opus::Channels::Mono, + opus::Application::Voip, + ) + .map_err(|error| format!("microphone Opus encoder initialization failed: {error}"))?; + encoder + .set_bitrate(opus::Bitrate::Bits(16_000)) + .map_err(|error| format!("microphone Opus bitrate configuration failed: {error}"))?; + Ok(encoder) +} + +fn run_encoder(shared: Arc, mut encoder: opus::Encoder) { + let mut packet = [0_u8; 1275]; + while let Some(frame) = shared.pop_pcm() { + match encoder.encode(&frame.samples, &mut packet) { + Ok(size) => shared.push_encoded( + EncodedMicrophoneFrame { + opus: packet[..size].to_vec(), + timestamp: frame.timestamp, + }, + frame.captured_at, + frame.generation, + ), + Err(error) => { + shared.close(Some(format!("microphone Opus encoding failed: {error}"))); + break; + } + } + } +} + +struct CaptureDevice { + id: u32, + shared: Arc, + _callback: Box>, + _audio: sdl2::AudioSubsystem, +} + +unsafe extern "C" fn capture_callback( + userdata: *mut std::ffi::c_void, + input: *mut u8, + length: i32, +) { + if userdata.is_null() || input.is_null() || length <= 0 || length % 2 != 0 { + return; + } + let callback = unsafe { &mut *userdata.cast::() }; + let samples = + unsafe { std::slice::from_raw_parts_mut(input.cast::(), length as usize / 2) }; + callback.callback(samples); +} + +impl CaptureDevice { + fn open( + audio: sdl2::AudioSubsystem, + shared: Arc, + origin: Instant, + resume: bool, + ) -> Result { + if shared.stopped() { + return Err("microphone capture request was cancelled".to_owned()); + } + let callback = Box::new(UnsafeCell::new(CaptureCallback::new(Arc::clone(&shared)))); + let desired = sys::SDL_AudioSpec { + freq: MICROPHONE_SAMPLE_RATE as i32, + format: sys::AUDIO_S16SYS as u16, + channels: 1, + silence: 0, + samples: 1024, + padding: 0, + size: 0, + callback: Some(capture_callback), + userdata: callback.get().cast(), + }; + let mut obtained = desired; + let id = + unsafe { sys::SDL_OpenAudioDevice(std::ptr::null(), 1, &desired, &mut obtained, 0) }; + if id == 0 { + return Err(format!( + "microphone capture could not open the default device: {}", + sdl2::get_error() + )); + } + unsafe { + (*callback.get()).timestamp = (origin.elapsed().as_micros() + * u128::from(MICROPHONE_SAMPLE_RATE) + / 1_000_000) as u32; + } + let device = Self { + id, + shared, + _callback: callback, + _audio: audio, + }; + if obtained.freq != MICROPHONE_SAMPLE_RATE as i32 + || obtained.channels != 1 + || obtained.format != sys::AUDIO_S16SYS as u16 + { + return Err( + "microphone device did not provide signed 16-bit mono 48 kHz PCM".to_owned(), + ); + } + { + let mut current = device + .shared + .device + .lock() + .unwrap_or_else(|error| error.into_inner()); + if device.shared.stopped() { + return Err("microphone capture request was cancelled".to_owned()); + } + if resume { + unsafe { sys::SDL_PauseAudioDevice(id, 0) }; + } + *current = Some(id); + } + Ok(device) + } + + fn resume(&self) -> Result<(), String> { + let device = self + .shared + .device + .lock() + .unwrap_or_else(|error| error.into_inner()); + if self.shared.stopped() { + return Err("microphone capture request was cancelled".to_owned()); + } + let id = device.ok_or_else(|| "microphone capture device was closed".to_owned())?; + unsafe { sys::SDL_PauseAudioDevice(id, 0) }; + Ok(()) + } + + fn pause(&self) { + self.shared.pause_device(); + } + + fn close(&mut self) { + if self.id == 0 { + return; + } + let id = std::mem::replace(&mut self.id, 0); + { + let mut current = self + .shared + .device + .lock() + .unwrap_or_else(|error| error.into_inner()); + if *current == Some(id) { + *current = None; + } + } + unsafe { sys::SDL_CloseAudioDevice(id) }; + } + + #[cfg(test)] + fn status(&self) -> SDL_AudioStatus { + self.shared + .device_status() + .unwrap_or(SDL_AudioStatus::SDL_AUDIO_STOPPED) + } +} + +impl Drop for CaptureDevice { + fn drop(&mut self) { + self.close(); + } +} + +pub(crate) struct MicrophoneCapture { + device: CaptureDevice, + worker: Option>, + shared: Arc, + _sdl: sdl2::Sdl, + origin: Instant, + reopen_started: Option, + next_reopen: Instant, +} + +impl MicrophoneCapture { + pub(crate) fn start( + device_id: &str, + shared: Arc, + origin: Instant, + ) -> Result { + if shared.stopped() { + return Err("microphone capture request was cancelled".to_owned()); + } + if !device_id.is_empty() { + return Err( + "microphone device selection is not supported; select the default input device" + .to_owned(), + ); + } + let encoder = encoder()?; + let sdl = sdl2::init() + .map_err(|error| format!("microphone SDL initialization failed: {error}"))?; + let audio = sdl + .audio() + .map_err(|error| format!("microphone audio initialization failed: {error}"))?; + let device = CaptureDevice::open(audio, Arc::clone(&shared), origin, false)?; + shared + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .last_pcm = Instant::now(); + let worker_shared = Arc::clone(&shared); + let worker = thread::Builder::new() + .name("opennow-microphone-opus".to_owned()) + .spawn(move || run_encoder(worker_shared, encoder)) + .map_err(|error| format!("microphone encoder worker could not start: {error}"))?; + let mut capture = Self { + device, + worker: Some(worker), + shared: Arc::clone(&shared), + _sdl: sdl, + origin, + reopen_started: None, + next_reopen: Instant::now(), + }; + if shared.stopped() { + return Err("microphone capture request was cancelled".to_owned()); + } + capture.device.resume()?; + capture.poll(); + if let Some(error) = shared + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .error + .clone() + { + return Err(error); + } + Ok(capture) + } + + pub(crate) fn poll(&mut self) { + self.shared.poll_device(); + if self.shared.stopped() { + return; + } + let now = Instant::now(); + if self.reopen_started.is_some() + || self.shared.device_status() == Some(SDL_AudioStatus::SDL_AUDIO_STOPPED) + { + self.device.close(); + let started = *self.reopen_started.get_or_insert_with(|| { + self.shared.flush_for_reopen(); + now + }); + if now >= self.next_reopen { + self.next_reopen = now + Duration::from_millis(250); + match CaptureDevice::open( + self.device._audio.clone(), + Arc::clone(&self.shared), + self.origin, + true, + ) { + Ok(device) => { + self.device = device; + self.shared.flush_for_reopen(); + self.shared + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .last_pcm = now; + self.reopen_started = None; + } + Err(error) if now.duration_since(started) >= Duration::from_secs(1) => { + self.shared + .close(Some(format!("microphone device recovery failed: {error}"))); + } + Err(_) => {} + } + } + } + if self.worker.as_ref().is_some_and(JoinHandle::is_finished) { + self.shared.close(Some( + "microphone encoder worker stopped unexpectedly".to_owned(), + )); + } + } + + pub(crate) fn stopped(&self) -> bool { + self.shared.stopped() + } +} + +impl Drop for MicrophoneCapture { + fn drop(&mut self) { + self.shared.close(None); + self.device.pause(); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +pub struct MicrophoneSession { + shared: Arc, +} + +pub struct MicrophoneReceiver { + shared: Arc, +} + +impl MicrophoneReceiver { + #[cfg(test)] + pub(crate) fn capture_device_open(&self) -> bool { + self.shared.device_status().is_some() + } + + #[cfg(test)] + pub(crate) fn pause_capture_device(&self) { + self.shared.pause_device(); + } + + pub fn try_recv(&self) -> Result, String> { + self.shared.poll_device(); + let mut state = self + .shared + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Some(error) = &state.error { + return Err(error.clone()); + } + while let Some(packet) = state.encoded.pop_front() { + if packet.captured_at.elapsed() > MAX_FRAME_AGE { + self.shared.dropped_encoded.fetch_add(1, Ordering::Relaxed); + continue; + } + return Ok(Some(packet.frame)); + } + Ok(None) + } + + pub fn status(&self) -> MicrophoneStatus { + self.shared.poll_device(); + let state = self + .shared + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + MicrophoneStatus { + enabled: !state.stopped, + error: state.error.clone(), + dropped_pcm_frames: self.shared.dropped_pcm.load(Ordering::Relaxed), + dropped_encoded_frames: self.shared.dropped_encoded.load(Ordering::Relaxed), + } + } + + pub fn stop(&self) { + self.shared.close(None); + } +} + +impl MicrophoneSession { + pub(crate) fn from_shared(shared: Arc) -> Self { + Self { shared } + } + + pub fn receiver(&self) -> MicrophoneReceiver { + MicrophoneReceiver { + shared: Arc::clone(&self.shared), + } + } + + pub fn try_recv(&mut self) -> Result, String> { + self.receiver().try_recv() + } + + pub fn status(&mut self) -> MicrophoneStatus { + self.receiver().status() + } + + pub fn stop(&mut self) { + self.shared.close(None); + } +} + +impl Drop for MicrophoneSession { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct LocalMicrophoneSession { + session: MicrophoneSession, + capture: Option, + } + + impl std::ops::Deref for LocalMicrophoneSession { + type Target = MicrophoneSession; + + fn deref(&self) -> &Self::Target { + &self.session + } + } + + impl std::ops::DerefMut for LocalMicrophoneSession { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.session + } + } + + impl LocalMicrophoneSession { + fn stop(&mut self) { + self.session.stop(); + self.capture = None; + } + } + + impl MicrophoneSession { + fn start_local(device_id: &str, origin: Instant) -> Result { + let shared = MicrophoneShared::new(); + let session = Self::from_shared(Arc::clone(&shared)); + let capture = MicrophoneCapture::start(device_id, shared, origin)?; + Ok(LocalMicrophoneSession { + session, + capture: Some(capture), + }) + } + } + + fn wait_for_packet(receiver: &MicrophoneReceiver) -> EncodedMicrophoneFrame { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if let Some(packet) = receiver.try_recv().expect("capture remains healthy") { + return packet; + } + assert!( + Instant::now() < deadline, + "encoder did not produce a packet" + ); + thread::sleep(Duration::from_millis(2)); + } + } + + #[test] + fn receiver_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } + + #[test] + fn stalled_encoder_discards_expired_pcm_without_rewriting_new_frame_clock() { + let shared = MicrophoneShared::new(); + shared.push_pcm(&[1; MICROPHONE_FRAME_SAMPLES], 960, Instant::now()); + shared + .state + .lock() + .unwrap() + .pcm + .front_mut() + .unwrap() + .captured_at = Instant::now() - Duration::from_secs(1); + shared.push_pcm(&[2; MICROPHONE_FRAME_SAMPLES], 48_000, Instant::now()); + let frame = shared.pop_pcm().unwrap(); + assert_eq!(frame.timestamp, 48_000); + assert_eq!(shared.dropped_pcm.load(Ordering::Relaxed), 1); + } + + #[test] + fn stalled_consumer_never_receives_expired_encoded_audio() { + let shared = MicrophoneShared::new(); + let session = MicrophoneSession::from_shared(Arc::clone(&shared)); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![1], + timestamp: 960, + }, + Instant::now(), + 0, + ); + shared + .state + .lock() + .unwrap() + .encoded + .front_mut() + .unwrap() + .captured_at = Instant::now() - Duration::from_secs(1); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![2], + timestamp: 48_000, + }, + Instant::now(), + 0, + ); + let receiver = session.receiver(); + assert_eq!(receiver.try_recv().unwrap().unwrap().timestamp, 48_000); + assert_eq!(receiver.status().dropped_encoded_frames, 1); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![3], + timestamp: 999, + }, + Instant::now() - Duration::from_secs(1), + 0, + ); + assert_eq!(receiver.try_recv(), Ok(None)); + assert_eq!(receiver.status().dropped_encoded_frames, 2); + } + + #[test] + fn callback_assembles_exact_frames_and_preserves_sample_clock_wrap() { + const HALF: usize = MICROPHONE_FRAME_SAMPLES / 2; + let shared = MicrophoneShared::new(); + let mut callback = CaptureCallback::new(Arc::clone(&shared)); + callback.timestamp = u32::MAX - (HALF as u32 - 1); + callback.callback(&mut [11; HALF]); + assert!(shared.state.lock().unwrap().pcm.is_empty()); + callback.callback(&mut [22; 3 * HALF]); + let first = shared.pop_pcm().unwrap(); + assert_eq!(first.timestamp, u32::MAX - (HALF as u32 - 1)); + assert_eq!(&first.samples[..HALF], &[11; HALF]); + assert_eq!(&first.samples[HALF..], &[22; HALF]); + let second = shared.pop_pcm().unwrap(); + assert_eq!(second.timestamp, HALF as u32); + assert_eq!(second.samples, [22; MICROPHONE_FRAME_SAMPLES]); + } + + #[test] + fn partial_frame_retains_capture_age_across_callback_stall() { + const HALF: usize = MICROPHONE_FRAME_SAMPLES / 2; + let shared = MicrophoneShared::new(); + let mut callback = CaptureCallback::new(Arc::clone(&shared)); + callback.callback(&mut [1; HALF]); + callback.captured_at = Instant::now() - Duration::from_secs(1); + callback.callback(&mut [2; 3 * HALF]); + let frame = shared.pop_pcm().unwrap(); + assert_eq!(frame.timestamp, MICROPHONE_FRAME_SAMPLES as u32); + assert_eq!(frame.samples, [2; MICROPHONE_FRAME_SAMPLES]); + assert_eq!(shared.dropped_pcm.load(Ordering::Relaxed), 1); + } + + #[test] + fn pcm_overflow_discards_oldest_and_preserves_timestamp_gaps() { + let shared = MicrophoneShared::new(); + let mut callback = CaptureCallback::new(Arc::clone(&shared)); + for value in 0..QUEUE_CAPACITY + 2 { + callback.callback(&mut [value as i16; MICROPHONE_FRAME_SAMPLES]); + } + assert_eq!(shared.state.lock().unwrap().pcm.len(), QUEUE_CAPACITY); + assert_eq!(shared.dropped_pcm.load(Ordering::Relaxed), 2); + let frame = shared.pop_pcm().unwrap(); + assert_eq!(frame.timestamp, 2 * MICROPHONE_FRAME_SAMPLES as u32); + assert_eq!(frame.samples, [2; MICROPHONE_FRAME_SAMPLES]); + } + + #[test] + fn callback_drops_instead_of_waiting_on_busy_queue() { + let shared = MicrophoneShared::new(); + let mut callback = CaptureCallback::new(Arc::clone(&shared)); + let guard = shared.state.lock().unwrap(); + callback.callback(&mut [1; MICROPHONE_FRAME_SAMPLES]); + drop(guard); + assert_eq!(shared.dropped_pcm.load(Ordering::Relaxed), 1); + callback.callback(&mut [2; MICROPHONE_FRAME_SAMPLES]); + assert_eq!( + shared.pop_pcm().unwrap().timestamp, + MICROPHONE_FRAME_SAMPLES as u32 + ); + } + + #[test] + fn encoded_queue_is_bounded_and_stop_discards_pending_and_in_flight_frames() { + let shared = MicrophoneShared::new(); + let mut session = MicrophoneSession::from_shared(Arc::clone(&shared)); + for timestamp in 0..QUEUE_CAPACITY as u32 + 2 { + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![1], + timestamp, + }, + Instant::now(), + 0, + ); + } + assert_eq!(session.status().dropped_encoded_frames, 2); + assert_eq!(session.try_recv().unwrap().unwrap().timestamp, 2); + shared.push_pcm(&[1; MICROPHONE_FRAME_SAMPLES], 0, Instant::now()); + session.stop(); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![2], + timestamp: 999, + }, + Instant::now(), + 0, + ); + shared.push_pcm(&[2; MICROPHONE_FRAME_SAMPLES], 999, Instant::now()); + assert!(!session.status().enabled); + assert_eq!(session.try_recv(), Ok(None)); + assert!(shared.pop_pcm().is_none()); + } + + #[test] + fn worker_encodes_decodable_mono_ten_millisecond_opus() { + let shared = MicrophoneShared::new(); + let mut session = MicrophoneSession::from_shared(Arc::clone(&shared)); + let receiver = session.receiver(); + let worker_shared = Arc::clone(&shared); + let worker = thread::spawn(move || run_encoder(worker_shared, encoder().unwrap())); + shared.push_pcm(&[100; MICROPHONE_FRAME_SAMPLES], 12_345, Instant::now()); + let packet = wait_for_packet(&receiver); + assert_eq!(packet.timestamp, 12_345); + let mut decoder = opus::Decoder::new(MICROPHONE_SAMPLE_RATE, opus::Channels::Mono).unwrap(); + let mut pcm = [0; MICROPHONE_FRAME_SAMPLES * 2]; + assert_eq!( + decoder.decode(&packet.opus, &mut pcm, false).unwrap(), + MICROPHONE_FRAME_SAMPLES + ); + session.stop(); + worker.join().unwrap(); + assert_eq!(receiver.try_recv(), Ok(None)); + } + + #[test] + fn microphone_encoder_uses_verified_voice_bitrate() { + assert_eq!( + encoder().unwrap().get_bitrate().unwrap(), + opus::Bitrate::Bits(16_000) + ); + } + + #[test] + fn reopening_flushes_pcm_encoded_and_in_flight_generations() { + let shared = MicrophoneShared::new(); + shared.push_pcm(&[1; MICROPHONE_FRAME_SAMPLES], 0, Instant::now()); + let in_flight = shared.pop_pcm().unwrap(); + shared.push_pcm(&[2; MICROPHONE_FRAME_SAMPLES], 480, Instant::now()); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![2], + timestamp: 480, + }, + Instant::now(), + 0, + ); + shared.flush_for_reopen(); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![1], + timestamp: in_flight.timestamp, + }, + in_flight.captured_at, + in_flight.generation, + ); + assert!(shared.state.lock().unwrap().pcm.is_empty()); + assert!(shared.state.lock().unwrap().encoded.is_empty()); + assert_eq!(shared.dropped_pcm.load(Ordering::Relaxed), 1); + assert_eq!(shared.dropped_encoded.load(Ordering::Relaxed), 2); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![3], + timestamp: 960, + }, + Instant::now(), + 1, + ); + let mut session = MicrophoneSession::from_shared(Arc::clone(&shared)); + assert_eq!(session.try_recv().unwrap().unwrap().timestamp, 960); + } + + #[test] + #[ignore = "requires SDL_AUDIODRIVER=dummy and a dedicated test process"] + fn dummy_hotplug_reopens_default_capture_without_closing_uplink() { + assert_eq!(std::env::var("SDL_AUDIODRIVER").as_deref(), Ok("dummy")); + let shared = MicrophoneShared::new(); + let mut capture = + MicrophoneCapture::start("", Arc::clone(&shared), Instant::now()).unwrap(); + let session = MicrophoneSession::from_shared(Arc::clone(&shared)); + let first = wait_for_packet(&session.receiver()); + capture.reopen_started = Some(Instant::now()); + capture.poll(); + assert_eq!(shared.state.lock().unwrap().generation, 1); + assert!(!shared.stopped()); + assert!(capture.device.status() == SDL_AudioStatus::SDL_AUDIO_PLAYING); + assert_eq!(*shared.device.lock().unwrap(), Some(capture.device.id)); + let next = wait_for_packet(&session.receiver()); + assert!(next.timestamp.wrapping_sub(first.timestamp) >= MICROPHONE_FRAME_SAMPLES as u32); + let id = capture.device.id; + session.receiver().stop(); + capture.reopen_started = Some(Instant::now()); + capture.poll(); + assert!(shared.stopped()); + assert_eq!(capture.device.id, id); + assert_eq!(shared.state.lock().unwrap().generation, 1); + } + + #[test] + #[ignore = "requires SDL_AUDIODRIVER=dummy and a dedicated test process"] + fn dummy_hotplug_retry_budget_fails_only_microphone_when_default_is_busy() { + assert_eq!(std::env::var("SDL_AUDIODRIVER").as_deref(), Ok("dummy")); + let shared = MicrophoneShared::new(); + let origin = Instant::now(); + let mut capture = MicrophoneCapture::start("", Arc::clone(&shared), origin).unwrap(); + capture.device.close(); + let blocker = CaptureDevice::open( + capture.device._audio.clone(), + MicrophoneShared::new(), + origin, + false, + ) + .unwrap(); + capture.reopen_started = Some(Instant::now() - Duration::from_secs(2)); + capture.poll(); + assert!(shared.stopped()); + assert!( + shared + .state + .lock() + .unwrap() + .error + .as_ref() + .unwrap() + .contains("microphone device recovery failed") + ); + assert_eq!( + blocker.status() as u32, + SDL_AudioStatus::SDL_AUDIO_PAUSED as u32 + ); + assert_eq!(*shared.device.lock().unwrap(), None); + } + + #[test] + fn dropping_session_stops_idle_worker_and_receiver() { + let shared = MicrophoneShared::new(); + let session = MicrophoneSession::from_shared(Arc::clone(&shared)); + let receiver = session.receiver(); + let worker = thread::spawn(move || run_encoder(shared, encoder().unwrap())); + drop(session); + worker.join().unwrap(); + assert!(!receiver.status().enabled); + } + + #[test] + fn device_failure_disables_and_flushes_without_leaking_into_next_session() { + let shared = MicrophoneShared::new(); + let mut session = MicrophoneSession::from_shared(Arc::clone(&shared)); + shared.push_encoded( + EncodedMicrophoneFrame { + opus: vec![1], + timestamp: 0, + }, + Instant::now(), + 0, + ); + shared.close(Some("device disconnected".to_owned())); + assert_eq!(session.try_recv(), Err("device disconnected".to_owned())); + assert!(!session.status().enabled); + session.stop(); + assert_eq!( + session.status().error.as_deref(), + Some("device disconnected") + ); + let mut replacement = MicrophoneSession::from_shared(MicrophoneShared::new()); + assert_eq!(replacement.status().error, None); + assert_eq!(replacement.try_recv(), Ok(None)); + } + + #[test] + fn missing_pcm_reports_failure_instead_of_claiming_enabled_forever() { + let shared = MicrophoneShared::new(); + let session = MicrophoneSession::from_shared(Arc::clone(&shared)); + let receiver = session.receiver(); + shared.state.lock().unwrap().last_pcm = Instant::now() - Duration::from_secs(3); + let worker = thread::spawn(move || run_encoder(shared, encoder().unwrap())); + worker.join().unwrap(); + assert!(!receiver.status().enabled); + assert_eq!( + receiver.status().error.as_deref(), + Some("microphone capture stopped delivering PCM") + ); + } + + #[test] + fn explicit_device_selection_fails_before_opening_sdl() { + let error = MicrophoneSession::start_local("unsupported-device-id", Instant::now()) + .err() + .expect("nondefault device rejected"); + assert!(error.contains("device selection is not supported")); + } + + #[test] + fn cancelled_host_request_never_opens_a_device() { + let shared = MicrophoneShared::new(); + shared.close(None); + let error = MicrophoneCapture::start("", shared, Instant::now()) + .err() + .expect("cancelled request rejected"); + assert_eq!(error, "microphone capture request was cancelled"); + } + + #[test] + #[ignore = "requires SDL_AUDIODRIVER=dummy and a dedicated test process"] + fn dummy_capture_restarts_with_continuous_clock_and_no_stale_audio() { + assert_eq!(std::env::var("SDL_AUDIODRIVER").as_deref(), Ok("dummy")); + let origin = Instant::now(); + let mut session = MicrophoneSession::start_local("", origin).unwrap(); + assert!(session.status().enabled); + let receiver = session.receiver(); + let first = wait_for_packet(&receiver); + let stop_receiver = session.receiver(); + thread::spawn(move || stop_receiver.stop()).join().unwrap(); + assert!( + session.capture.as_ref().unwrap().device.status() == SDL_AudioStatus::SDL_AUDIO_PAUSED + ); + session.stop(); + assert!(!receiver.status().enabled); + assert_eq!(receiver.try_recv(), Ok(None)); + thread::sleep(Duration::from_millis(25)); + let mut restarted = MicrophoneSession::start_local("", origin).unwrap(); + receiver.stop(); + assert!( + restarted.capture.as_ref().unwrap().device.status() + == SDL_AudioStatus::SDL_AUDIO_PLAYING + ); + let next = wait_for_packet(&restarted.receiver()); + assert!(next.timestamp.wrapping_sub(first.timestamp) >= MICROPHONE_FRAME_SAMPLES as u32); + restarted.capture.as_ref().unwrap().device.pause(); + let stopped_receiver = restarted.receiver(); + let status = stopped_receiver.status(); + assert!(!status.enabled); + assert_eq!( + status.error.as_deref(), + Some("microphone capture device stopped or disconnected") + ); + assert!(stopped_receiver.try_recv().is_err()); + restarted.stop(); + assert!(restarted.capture.is_none()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/native_surface.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/native_surface.rs new file mode 100644 index 000000000..03ff88e4e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/native_surface.rs @@ -0,0 +1,569 @@ +use opennow_streamer_protocol::RenderSurfaceRect; +use sdl2::video::Window; + +pub(crate) struct NativeSurface { + inner: platform::Surface, +} + +impl NativeSurface { + pub(crate) fn new(window: &Window) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + platform::Surface::new(window) + })) + .map_err(|_| "SDL could not expose a native presentation handle".to_owned())? + .map(|inner| Self { inner }) + } + + pub(crate) fn attach_and_show( + &mut self, + parent_handle: &str, + rect: RenderSurfaceRect, + screen_rect: Option, + scale: f32, + ) -> Result<(), String> { + self.inner + .attach_and_show(parent_handle, rect, screen_rect, scale) + } + + pub(crate) fn hide(&mut self) { + self.inner.hide(); + } + + #[cfg(target_os = "windows")] + pub(crate) fn hide_checked(&mut self) -> Result<(), String> { + self.inner.hide_checked() + } + + pub(crate) fn refresh_ordering(&mut self) -> Result<(), String> { + self.inner.refresh_ordering() + } + + #[cfg(target_os = "windows")] + pub(crate) fn window_handle(&self) -> isize { + self.inner.window_handle() + } +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn parse_handle(value: &str) -> Result { + let trimmed = value.trim(); + let parsed = if let Some(hex) = trimmed.strip_prefix("0x") { + usize::from_str_radix(hex, 16) + } else { + trimmed.parse() + }; + parsed + .ok() + .filter(|handle| *handle != 0) + .ok_or_else(|| format!("invalid shell native window handle: {value}")) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn physical_rect(rect: RenderSurfaceRect, _scale: f32) -> (i32, i32, u32, u32) { + (rect.x, rect.y, rect.width.max(2), rect.height.max(2)) +} + +#[cfg(target_os = "windows")] +mod platform { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + use windows_sys::Win32::Foundation::{GetLastError, SetLastError}; + use windows_sys::Win32::UI::Input::KeyboardAndMouse::SetFocus; + use windows_sys::Win32::UI::WindowsAndMessaging::{ + GWL_EXSTYLE, GWL_STYLE, GetWindowLongPtrW, HWND_TOP, SW_HIDE, SWP_FRAMECHANGED, + SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SWP_NOZORDER, SWP_SHOWWINDOW, SetForegroundWindow, + SetParent, SetWindowLongPtrW, SetWindowPos, ShowWindow, WS_CAPTION, WS_CHILD, + WS_CLIPSIBLINGS, WS_DISABLED, WS_EX_APPWINDOW, WS_EX_NOACTIVATE, WS_EX_TRANSPARENT, + WS_MAXIMIZEBOX, WS_MINIMIZEBOX, WS_POPUP, WS_SYSMENU, WS_THICKFRAME, WS_VISIBLE, + }; + + use super::*; + + fn child_style(style: u32) -> u32 { + (style + & !(WS_VISIBLE + | WS_POPUP + | WS_DISABLED + | WS_CAPTION + | WS_THICKFRAME + | WS_MINIMIZEBOX + | WS_MAXIMIZEBOX + | WS_SYSMENU)) + | WS_CHILD + | WS_CLIPSIBLINGS + } + + fn child_extended_style(style: u32) -> u32 { + style & !(WS_EX_APPWINDOW | WS_EX_NOACTIVATE | WS_EX_TRANSPARENT) + } + + pub(crate) struct Surface { + child: windows_sys::Win32::Foundation::HWND, + parent: windows_sys::Win32::Foundation::HWND, + standalone_style: u32, + standalone_extended_style: u32, + shown: bool, + } + + impl Surface { + pub(crate) fn new(window: &Window) -> Result { + let child = match window + .window_handle() + .map_err(|error| format!("SDL Win32 handle unavailable: {error}"))? + .as_raw() + { + RawWindowHandle::Win32(handle) => handle.hwnd.get() as _, + _ => return Err("SDL did not create a Win32 presentation window".to_owned()), + }; + Ok(Self { + child, + parent: std::ptr::null_mut(), + standalone_style: unsafe { GetWindowLongPtrW(child, GWL_STYLE) as u32 }, + standalone_extended_style: unsafe { GetWindowLongPtrW(child, GWL_EXSTYLE) as u32 }, + shown: false, + }) + } + + pub(crate) fn window_handle(&self) -> isize { + self.child as isize + } + + pub(crate) fn attach_and_show( + &mut self, + parent_handle: &str, + rect: RenderSurfaceRect, + _screen_rect: Option, + _scale: f32, + ) -> Result<(), String> { + let parent = parse_handle(parent_handle)? as _; + let (x, y, width, height) = physical_rect(rect, _scale); + unsafe { + SetWindowLongPtrW( + self.child, + GWL_STYLE, + child_style(self.standalone_style) as isize, + ); + SetWindowLongPtrW( + self.child, + GWL_EXSTYLE, + child_extended_style(self.standalone_extended_style) as isize, + ); + if self.parent != parent { + SetLastError(0); + if SetParent(self.child, parent).is_null() && GetLastError() != 0 { + SetWindowLongPtrW( + self.child, + GWL_STYLE, + (self.standalone_style & !WS_VISIBLE) as isize, + ); + SetWindowLongPtrW( + self.child, + GWL_EXSTYLE, + self.standalone_extended_style as isize, + ); + return Err( + "failed to attach SDL video surface to the Qt window".to_owned() + ); + } + self.parent = parent; + } + if SetWindowPos( + self.child, + HWND_TOP, + x, + y, + width as i32, + height as i32, + SWP_FRAMECHANGED | SWP_NOACTIVATE | SWP_SHOWWINDOW, + ) == 0 + { + return Err("failed to position the Qt child video surface".to_owned()); + } + if !self.shown { + let _ = SetForegroundWindow(parent); + let _ = SetFocus(self.child); + self.shown = true; + } + } + Ok(()) + } + + pub(crate) fn hide_checked(&mut self) -> Result<(), String> { + unsafe { + ShowWindow(self.child, SW_HIDE); + if !self.parent.is_null() { + SetLastError(0); + if SetParent(self.child, std::ptr::null_mut()).is_null() && GetLastError() != 0 + { + self.shown = false; + return Err( + "failed to detach SDL video surface from the Qt window".to_owned() + ); + } + self.parent = std::ptr::null_mut(); + SetWindowLongPtrW( + self.child, + GWL_STYLE, + (self.standalone_style & !WS_VISIBLE) as isize, + ); + SetWindowLongPtrW( + self.child, + GWL_EXSTYLE, + self.standalone_extended_style as isize, + ); + let _ = SetWindowPos( + self.child, + std::ptr::null_mut(), + 0, + 0, + 0, + 0, + SWP_FRAMECHANGED | SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER, + ); + } + } + self.shown = false; + Ok(()) + } + + pub(crate) fn hide(&mut self) { + let _ = self.hide_checked(); + } + + pub(crate) fn refresh_ordering(&mut self) -> Result<(), String> { + Ok(()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn embedded_presenter_is_an_enabled_input_capable_child() { + let style = child_style( + WS_POPUP + | WS_VISIBLE + | WS_DISABLED + | WS_CAPTION + | WS_THICKFRAME + | WS_MINIMIZEBOX + | WS_MAXIMIZEBOX + | WS_SYSMENU, + ); + assert_ne!(style & WS_CHILD, 0); + assert_ne!(style & WS_CLIPSIBLINGS, 0); + assert_eq!( + style + & (WS_POPUP + | WS_VISIBLE + | WS_DISABLED + | WS_CAPTION + | WS_THICKFRAME + | WS_MINIMIZEBOX + | WS_MAXIMIZEBOX + | WS_SYSMENU), + 0 + ); + + let extended = + child_extended_style(WS_EX_APPWINDOW | WS_EX_NOACTIVATE | WS_EX_TRANSPARENT); + assert_eq!( + extended & (WS_EX_APPWINDOW | WS_EX_NOACTIVATE | WS_EX_TRANSPARENT), + 0 + ); + } + } +} + +#[cfg(all(test, any(target_os = "windows", target_os = "linux")))] +mod tests { + use super::*; + + #[test] + fn renderer_rect_is_already_physical_at_common_dpi_scales() { + let rect = RenderSurfaceRect { + x: 120, + y: 80, + width: 1280, + height: 720, + }; + + for scale in [1.0, 1.5, 2.0] { + assert_eq!(physical_rect(rect, scale), (120, 80, 1280, 720)); + } + } + + #[test] + fn physical_rect_only_clamps_empty_dimensions() { + let rect = RenderSurfaceRect { + x: -10, + y: 20, + width: 0, + height: 1, + }; + + assert_eq!(physical_rect(rect, 2.0), (-10, 20, 2, 2)); + } +} + +#[cfg(target_os = "linux")] +mod platform { + use std::sync::atomic::{AtomicI32, Ordering}; + + use raw_window_handle::{HasDisplayHandle, HasWindowHandle, RawDisplayHandle, RawWindowHandle}; + use x11_dl::xlib; + + use super::*; + + static X11_ERROR_CODE: AtomicI32 = AtomicI32::new(0); + + unsafe extern "C" fn record_x11_error( + _display: *mut xlib::Display, + event: *mut xlib::XErrorEvent, + ) -> i32 { + if !event.is_null() { + X11_ERROR_CODE.store(unsafe { (*event).error_code.into() }, Ordering::Release); + } + 0 + } + + pub(crate) struct Surface { + xlib: xlib::Xlib, + display: *mut xlib::Display, + child: xlib::Window, + parent: xlib::Window, + } + + impl Surface { + pub(crate) fn new(window: &Window) -> Result { + let child = match window + .window_handle() + .map_err(|error| format!("SDL X11 window handle unavailable: {error}"))? + .as_raw() + { + RawWindowHandle::Xlib(handle) => handle.window, + RawWindowHandle::Wayland(_) => { + return Err( + "native shell surface embedding requires an X11/XWayland session" + .to_owned(), + ); + } + _ => return Err("SDL did not create an X11 presentation window".to_owned()), + }; + let display = match window + .display_handle() + .map_err(|error| format!("SDL X11 display handle unavailable: {error}"))? + .as_raw() + { + RawDisplayHandle::Xlib(handle) => handle + .display + .map(|display| display.as_ptr()) + .unwrap_or(std::ptr::null_mut()), + _ => std::ptr::null_mut(), + }; + if display.is_null() { + return Err("SDL X11 display pointer is null".to_owned()); + } + Ok(Self { + xlib: xlib::Xlib::open() + .map_err(|error| format!("failed to load X11 embedding API: {error}"))?, + display: display.cast(), + child, + parent: 0, + }) + } + + pub(crate) fn attach_and_show( + &mut self, + parent_handle: &str, + rect: RenderSurfaceRect, + _screen_rect: Option, + _scale: f32, + ) -> Result<(), String> { + let parent = parse_handle(parent_handle)? as xlib::Window; + let (x, y, width, height) = physical_rect(rect, _scale); + let error_code = unsafe { + (self.xlib.XSync)(self.display, xlib::False); + X11_ERROR_CODE.store(0, Ordering::Release); + let previous = (self.xlib.XSetErrorHandler)(Some(record_x11_error)); + if self.parent != parent { + (self.xlib.XUnmapWindow)(self.display, self.child); + (self.xlib.XReparentWindow)(self.display, self.child, parent, x, y); + (self.xlib.XSelectInput)(self.display, self.child, xlib::StructureNotifyMask); + self.parent = parent; + } + (self.xlib.XMoveResizeWindow)(self.display, self.child, x, y, width, height); + (self.xlib.XLowerWindow)(self.display, self.child); + (self.xlib.XMapWindow)(self.display, self.child); + (self.xlib.XSync)(self.display, xlib::False); + (self.xlib.XSetErrorHandler)(previous); + X11_ERROR_CODE.load(Ordering::Acquire) + }; + if error_code != 0 { + self.parent = 0; + return Err(format!( + "X11 could not attach the native child surface (error {error_code})" + )); + } + Ok(()) + } + + pub(crate) fn hide(&mut self) { + unsafe { + (self.xlib.XSync)(self.display, xlib::False); + X11_ERROR_CODE.store(0, Ordering::Release); + let previous = (self.xlib.XSetErrorHandler)(Some(record_x11_error)); + (self.xlib.XUnmapWindow)(self.display, self.child); + (self.xlib.XSync)(self.display, xlib::False); + (self.xlib.XSetErrorHandler)(previous); + } + } + + pub(crate) fn refresh_ordering(&mut self) -> Result<(), String> { + Ok(()) + } + } +} + +#[cfg(target_os = "macos")] +mod platform { + use std::time::{Duration, Instant}; + + use objc2::rc::Retained; + use objc2_app_kit::{NSView, NSWindow, NSWorkspace}; + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + + use super::*; + + const ORDERING_POLL_INTERVAL: Duration = Duration::from_millis(100); + + pub(crate) struct Surface { + child: Retained, + raw_window: *mut sdl2::sys::SDL_Window, + parent_pid: libc::pid_t, + requested_visible: bool, + ordered: bool, + last_ordering_check: Option, + } + + impl Surface { + pub(crate) fn new(window: &Window) -> Result { + let child_view = match window + .window_handle() + .map_err(|error| format!("SDL AppKit handle unavailable: {error}"))? + .as_raw() + { + RawWindowHandle::AppKit(handle) => unsafe { + &*handle.ns_view.as_ptr().cast::() + }, + _ => return Err("SDL did not create an AppKit presentation window".to_owned()), + }; + let child = child_view + .window() + .ok_or_else(|| "SDL AppKit view has no window".to_owned())?; + child.setIgnoresMouseEvents(true); + Ok(Self { + child, + raw_window: window.raw(), + parent_pid: unsafe { libc::getppid() }, + requested_visible: false, + ordered: false, + last_ordering_check: None, + }) + } + + pub(crate) fn attach_and_show( + &mut self, + _parent_handle: &str, + _rect: RenderSurfaceRect, + screen_rect: Option, + _scale: f32, + ) -> Result<(), String> { + let screen_rect = screen_rect.ok_or_else(|| { + "macOS native surface is missing absolute screen bounds".to_owned() + })?; + let width = i32::try_from(screen_rect.width) + .map_err(|_| "macOS native surface width is out of range".to_owned())?; + let height = i32::try_from(screen_rect.height) + .map_err(|_| "macOS native surface height is out of range".to_owned())?; + unsafe { + sdl2::sys::SDL_SetWindowPosition(self.raw_window, screen_rect.x, screen_rect.y); + sdl2::sys::SDL_SetWindowSize(self.raw_window, width, height); + } + self.requested_visible = true; + self.last_ordering_check = None; + self.refresh_ordering() + } + + pub(crate) fn hide(&mut self) { + self.requested_visible = false; + if self.ordered { + unsafe { + sdl2::sys::SDL_HideWindow(self.raw_window); + } + self.ordered = false; + } + } + + pub(crate) fn refresh_ordering(&mut self) -> Result<(), String> { + if self + .last_ordering_check + .is_some_and(|last| last.elapsed() < ORDERING_POLL_INTERVAL) + { + return Ok(()); + } + self.last_ordering_check = Some(Instant::now()); + let should_order = self.requested_visible && self.parent_is_frontmost(); + if should_order == self.ordered { + return Ok(()); + } + unsafe { + if should_order { + sdl2::sys::SDL_ShowWindow(self.raw_window); + } else { + sdl2::sys::SDL_HideWindow(self.raw_window); + } + } + if should_order { + self.child.orderFrontRegardless(); + } + self.ordered = should_order; + Ok(()) + } + + fn parent_is_frontmost(&self) -> bool { + NSWorkspace::sharedWorkspace() + .frontmostApplication() + .is_some_and(|application| application.processIdentifier() == self.parent_pid) + } + } +} + +#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] +mod platform { + use super::*; + + pub(crate) struct Surface; + + impl Surface { + pub(crate) fn new(_window: &Window) -> Result { + Err("native presentation is unsupported on this operating system".to_owned()) + } + + pub(crate) fn attach_and_show( + &mut self, + _parent_handle: &str, + _rect: RenderSurfaceRect, + _screen_rect: Option, + _scale: f32, + ) -> Result<(), String> { + Err("native presentation is unsupported on this operating system".to_owned()) + } + + pub(crate) fn hide(&mut self) {} + + pub(crate) fn refresh_ordering(&mut self) -> Result<(), String> { + Ok(()) + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/output.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/output.rs new file mode 100644 index 000000000..f1695dcc6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/output.rs @@ -0,0 +1,3455 @@ +#[cfg(target_os = "linux")] +use std::collections::VecDeque; +use std::collections::{HashMap, HashSet}; +use std::io::Cursor as IoCursor; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use crate::audio_playout::AudioPlayoutBuffer; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use image::ImageReader; +use opennow_streamer_protocol::{AudioDevice as PlaybackDevice, AudioOutputDevice, RenderSurface}; +use sdl2::audio::{AudioCallback, AudioDevice, AudioSpecDesired}; +use sdl2::pixels::{Color, PixelFormatEnum}; +use sdl2::rect::Rect; +use sdl2::render::{Texture, WindowCanvas}; + +#[cfg(target_os = "linux")] +use crate::linux_frame_pacing::{FrameSelectionPolicy, LinuxFramePacer}; +#[cfg(target_os = "linux")] +use crate::linux_xinput::LinuxXInputController; +use crate::media::{CapturedInput, CapturedInputQueue, MediaStreamConfig, StreamShortcutBindings}; +use crate::native_surface::NativeSurface; +#[cfg(target_os = "windows")] +use crate::windows_raw_input::WindowsRawInputController; + +#[cfg(target_os = "linux")] +use opennow_streamer_platform_linux::DecodedVideoFrame as LinuxDecodedVideoFrame; +#[cfg(target_os = "windows")] +use opennow_streamer_platform_windows::{ + AudioFormat, BackendConfig, BackendEvent, Bounds, ExistingWindow, OwnedWindow, Subsystem, + SurfaceTarget, VideoChromaFormat, VideoCodec, VideoFormat, VideoPixelFormat, WindowHandle, + WindowsBackend, WindowsDecoderMode, WindowsGraphicsApi, +}; + +const AUDIO_SAMPLE_RATE: i32 = 48_000; +const AUDIO_CHANNELS: u8 = 2; +const AUDIO_BUFFER_FRAMES: u16 = 480; +const MAX_AUDIO_LATENCY_MS: usize = 120; + +pub(crate) fn audio_devices() -> Result, String> { + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let audio = sdl + .audio() + .map_err(|error| format!("SDL audio initialization failed: {error}"))?; + let devices = playback_devices(&audio)?; + Ok(unique_playback_devices(devices)) +} + +fn unique_playback_devices(devices: Vec) -> Vec { + let mut counts = HashMap::new(); + for device in &devices { + *counts.entry(device.id.clone()).or_insert(0) += 1; + } + devices + .into_iter() + .filter(|device| counts[&device.id] == 1) + .collect() +} + +fn playback_devices(audio: &sdl2::AudioSubsystem) -> Result, String> { + let count = audio + .num_audio_playback_devices() + .ok_or_else(|| format!("SDL audio device enumeration failed: {}", sdl2::get_error()))?; + if count > 1024 { + return Err("SDL returned more than 1024 audio playback devices".to_owned()); + } + (0..count) + .map(|index| { + let name = audio + .audio_playback_device_name(index) + .map_err(|error| format!("SDL audio device enumeration failed: {error}"))?; + AudioOutputDevice::new(name.clone())?; + if name.is_empty() { + return Err("SDL returned an empty audio playback device name".to_owned()); + } + Ok(PlaybackDevice { + id: name.clone(), + name, + }) + }) + .collect() +} + +fn select_playback_device<'a>( + selected: &'a AudioOutputDevice, + devices: &[PlaybackDevice], +) -> Result, String> { + let Some(name) = selected.device_name() else { + return Ok(None); + }; + if devices.iter().filter(|device| device.id == name).count() != 1 { + return Err("The selected audio output device is unavailable or ambiguous. Select another device or System default.".to_owned()); + } + Ok(Some(name)) +} + +fn open_audio_playback( + audio: &sdl2::AudioSubsystem, + output: &Arc, + selected: &AudioOutputDevice, +) -> Result, String> { + let devices = if selected.device_name().is_some() { + playback_devices(audio)? + } else { + Vec::new() + }; + let name = select_playback_device(selected, &devices)?; + let desired = AudioSpecDesired { + freq: Some(AUDIO_SAMPLE_RATE), + channels: Some(AUDIO_CHANNELS), + samples: Some(AUDIO_BUFFER_FRAMES), + }; + let callback_output = Arc::clone(output); + audio + .open_playback(name, &desired, move |_| StreamAudioCallback { + output: callback_output, + }) + .map_err(|error| format!("native audio output creation failed: {error}")) +} +#[cfg(target_os = "linux")] +const LINUX_VIDEO_QUEUE_CAPACITY: usize = 6; + +#[derive(Debug)] +pub(crate) struct DecodedVideoFrame { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) rgb: Vec, +} + +#[cfg(target_os = "linux")] +#[derive(Debug)] +struct QueuedLinuxVideoFrame { + frame: LinuxDecodedVideoFrame, + arrived_at: Instant, +} + +#[derive(Debug)] +pub(crate) struct OutputBuffers { + video: Mutex>, + #[cfg(target_os = "linux")] + linux_video: Mutex>, + audio: Mutex, + pub(crate) audio_muted: Arc, + captured_input: Arc, + microphone: Mutex>>, + microphone_clock: Mutex, +} + +#[cfg(target_os = "windows")] +pub(crate) struct WindowsBridge { + backend: Mutex>>, + software_fallback: AtomicBool, + keyframe_required: AtomicBool, + last_video_mid: Mutex, +} + +#[cfg(target_os = "windows")] +impl WindowsBridge { + pub(crate) fn new() -> Self { + Self { + backend: Mutex::new(None), + software_fallback: AtomicBool::new(false), + keyframe_required: AtomicBool::new(true), + last_video_mid: Mutex::new("video".to_owned()), + } + } + + pub(crate) fn backend(&self) -> Option> { + self.backend + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } + + fn replace_backend(&self, backend: Option>) { + *self + .backend + .lock() + .unwrap_or_else(|error| error.into_inner()) = backend; + } + + pub(crate) fn use_software(&self) -> bool { + self.software_fallback.load(Ordering::Acquire) + } + + pub(crate) fn fall_back_to_software(&self) { + self.software_fallback.store(true, Ordering::Release); + self.keyframe_required.store(true, Ordering::Release); + self.replace_backend(None); + } + + pub(crate) fn reset(&self) { + self.software_fallback.store(false, Ordering::Release); + self.keyframe_required.store(true, Ordering::Release); + self.replace_backend(None); + } + + pub(crate) fn set_last_video_mid(&self, mid: &str) { + *self + .last_video_mid + .lock() + .unwrap_or_else(|error| error.into_inner()) = mid.to_owned(); + } + + pub(crate) fn last_video_mid(&self) -> String { + self.last_video_mid + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } + + pub(crate) fn keyframe_required(&self) -> bool { + self.keyframe_required.load(Ordering::Acquire) + } + + pub(crate) fn require_keyframe(&self) { + self.keyframe_required.store(true, Ordering::Release); + } + + pub(crate) fn accept_keyframe(&self) { + self.keyframe_required.store(false, Ordering::Release); + } +} + +impl OutputBuffers { + #[cfg(test)] + pub(crate) fn audio_samples_pending(&self) -> bool { + !self + .audio + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty() + } + + pub(crate) fn new() -> Self { + Self::with_captured_input(Arc::new(CapturedInputQueue::default())) + } + + pub(crate) fn with_captured_input(captured_input: Arc) -> Self { + Self { + video: Mutex::new(None), + #[cfg(target_os = "linux")] + linux_video: Mutex::new(VecDeque::with_capacity(LINUX_VIDEO_QUEUE_CAPACITY)), + audio: Mutex::new(AudioPlayoutBuffer::new( + AUDIO_SAMPLE_RATE as usize * AUDIO_CHANNELS as usize * MAX_AUDIO_LATENCY_MS / 1_000, + )), + audio_muted: Arc::new(AtomicBool::new(false)), + captured_input, + microphone: Mutex::new(None), + microphone_clock: Mutex::new(Instant::now()), + } + } + + pub(crate) fn captured_input(&self) -> Arc { + Arc::clone(&self.captured_input) + } + + pub(crate) fn set_microphone(&self, shared: &Arc) { + let mut microphone = self + .microphone + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Some(previous) = microphone.take().and_then(|previous| previous.upgrade()) { + previous.close(None); + } + *microphone = Some(Arc::downgrade(shared)); + } + + pub(crate) fn microphone_clock(&self) -> Instant { + *self + .microphone_clock + .lock() + .unwrap_or_else(|error| error.into_inner()) + } + + pub(crate) fn reset_microphone_clock(&self) { + *self + .microphone_clock + .lock() + .unwrap_or_else(|error| error.into_inner()) = Instant::now(); + } + + pub(crate) fn stop_microphone(&self) { + if let Some(microphone) = self + .microphone + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + .and_then(|microphone| microphone.upgrade()) + { + microphone.close(None); + } + } + + pub(crate) fn replace_video(&self, frame: DecodedVideoFrame) -> bool { + self.video + .lock() + .unwrap_or_else(|error| error.into_inner()) + .replace(frame) + .is_some() + } + + pub(crate) fn take_video(&self) -> Option { + self.video + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + } + + #[cfg(target_os = "linux")] + pub(crate) fn queue_linux_video(&self, frame: LinuxDecodedVideoFrame) -> bool { + let mut queue = self + .linux_video + .lock() + .unwrap_or_else(|error| error.into_inner()); + let dropped = if queue.len() == LINUX_VIDEO_QUEUE_CAPACITY { + queue.pop_front(); + true + } else { + false + }; + queue.push_back(QueuedLinuxVideoFrame { + frame, + arrived_at: Instant::now(), + }); + dropped + } + + #[cfg(target_os = "linux")] + fn take_linux_video(&self) -> Option { + self.linux_video + .lock() + .unwrap_or_else(|error| error.into_inner()) + .pop_front() + } + + #[cfg(target_os = "linux")] + fn take_linux_video_for_presentation( + &self, + selection: FrameSelectionPolicy, + target_queue_depth: usize, + ) -> Option { + let mut queue = self + .linux_video + .lock() + .unwrap_or_else(|error| error.into_inner()); + if queue.len() <= target_queue_depth { + return None; + } + match selection { + FrameSelectionPolicy::OldestReady => queue.pop_front(), + FrameSelectionPolicy::LatestReady => { + let frame = queue.pop_back(); + queue.clear(); + frame + } + } + } + + #[cfg(target_os = "linux")] + fn clear_linux_video(&self) { + self.linux_video + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clear(); + } + + pub(crate) fn push_audio(&self, samples: &[f32]) -> usize { + self.audio + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(samples) + } + + pub(crate) fn clear(&self) { + self.take_video(); + #[cfg(target_os = "linux")] + self.clear_linux_video(); + self.audio + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clear(); + } + + fn fill_audio(&self, destination: &mut [f32]) { + self.audio + .lock() + .unwrap_or_else(|error| error.into_inner()) + .fill(destination); + if self.audio_muted.load(Ordering::Acquire) { + destination.fill(0.0); + } + } +} + +#[cfg(any(target_os = "windows", test))] +pub(crate) struct HeadlessAudioOutput { + device: AudioDevice, + output: Arc, + _sdl: sdl2::Sdl, +} + +#[cfg(any(target_os = "windows", test))] +impl HeadlessAudioOutput { + pub(crate) fn start( + output: Arc, + audio_device: &AudioOutputDevice, + ) -> Result { + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let audio_subsystem = sdl + .audio() + .map_err(|error| format!("SDL audio initialization failed: {error}"))?; + let device = open_audio_playback(&audio_subsystem, &output, audio_device)?; + device.resume(); + Ok(Self { + device, + output, + _sdl: sdl, + }) + } + + pub(crate) fn set_paused(&self, paused: bool) { + if paused { + self.device.pause(); + self.output.clear(); + } else { + self.device.resume(); + } + } +} + +#[cfg(any(target_os = "windows", test))] +impl Drop for HeadlessAudioOutput { + fn drop(&mut self) { + self.device.pause(); + self.output.clear(); + } +} + +#[cfg(target_os = "linux")] +pub(crate) struct LinuxHardwareOutput { + // These fields own resources derived from the SDL Wayland/X11 display. + // Rust drops fields in declaration order, so keep the presenter and + // window ahead of the SDL root to preserve their native lifetimes even + // when normal stop handling is bypassed during unwinding. + presenter: Option, + native_surface: Result, + input_capture: SdlInputCapture, + raw_input: Option, + window: sdl2::video::Window, + video: sdl2::VideoSubsystem, + event_pump: sdl2::EventPump, + audio: AudioDevice, + output: Arc, + external_renderer: bool, + stream_size: (u32, u32), + surface_size: Option<(u32, u32)>, + stream_fps: u32, + frame_pacer: LinuxFramePacer, + visible: bool, + paused: bool, + _sdl: sdl2::Sdl, +} + +#[cfg(target_os = "linux")] +impl LinuxHardwareOutput { + fn initialize( + output: Arc, + stream: MediaStreamConfig, + audio_device: &AudioOutputDevice, + ) -> Result { + configure_linux_sdl_video_driver(); + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let video = sdl + .video() + .map_err(|error| format!("SDL video initialization failed: {error}"))?; + let audio_subsystem = sdl + .audio() + .map_err(|error| format!("SDL audio initialization failed: {error}"))?; + let video_driver = video.current_video_driver(); + if !matches!(video_driver, "x11" | "wayland") { + return Err(format!( + "Linux Vulkan presentation requires X11 or Wayland, but SDL selected {video_driver}", + )); + } + let external_renderer = external_renderer_enabled(); + if video_driver == "wayland" && !external_renderer { + return Err( + "Wayland presentation must use a compositor-managed top-level window".to_owned(), + ); + } + let mut window_builder = video.window("OpenNOW Stream", 1280, 720); + window_builder + .position_centered() + .resizable() + .allow_highdpi() + .hidden() + .vulkan(); + if !external_renderer { + window_builder.borderless(); + } + let window = window_builder + .build() + .map_err(|error| format!("native Vulkan window creation failed: {error}"))?; + let display_refresh_hz = linux_display_refresh_hz(&video, &window); + let frame_pacer = LinuxFramePacer::new(stream.fps, display_refresh_hz, stream.cloud_gsync); + eprintln!( + "Linux presentation pacing: stream={}fps display={} output={:.1}Hz mode={}", + stream.fps, + display_refresh_hz.map_or_else(|| "unknown".to_owned(), |value| format!("{value}Hz")), + frame_pacer.presentation_hz(), + if frame_pacer.vrr_enabled() { + "cloud-gsync-vrr" + } else if frame_pacer.fast_stream() { + "nonblocking-fast-stream" + } else { + "adaptive-timestamp" + }, + ); + let native_surface = if external_renderer { + Err("external Linux output does not use child-window embedding".to_owned()) + } else { + NativeSurface::new(&window) + }; + let audio = open_audio_playback(&audio_subsystem, &output, audio_device)?; + let event_pump = sdl + .event_pump() + .map_err(|error| format!("native window event pump creation failed: {error}"))?; + let capture_input = external_renderer && native_input_capture_enabled(); + let raw_input = if capture_input && video_driver == "x11" { + match LinuxXInputController::start(output.captured_input()) { + Ok(controller) => Some(controller), + Err(error) => { + eprintln!( + "Dedicated Linux XInput2 unavailable; retaining SDL raw motion: {error}" + ); + None + } + } + } else { + None + }; + let external_relative_motion = raw_input.is_some(); + let mut input_capture = SdlInputCapture::new( + capture_input, + external_relative_motion, + false, + stream.shortcuts, + ); + input_capture.enable_gamepads(&sdl); + Ok(Self { + presenter: None, + native_surface, + input_capture, + raw_input, + window, + video, + event_pump, + audio, + output, + external_renderer, + stream_size: (stream.width.max(1), stream.height.max(1)), + surface_size: None, + stream_fps: stream.fps.max(1), + frame_pacer, + visible: false, + paused: false, + _sdl: sdl, + }) + } + + fn start(&mut self, surface: Option<&RenderSurface>) -> Result<(), String> { + self.paused = false; + self.frame_pacer.reset(); + self.output.clear(); + self.audio.resume(); + if let Some(surface) = surface { + self.update_surface(surface)?; + } + Ok(()) + } + + fn set_paused(&mut self, paused: bool) { + self.paused = paused; + self.frame_pacer.reset(); + if paused { + self.input_capture + .set_input_paused(true, &self._sdl, &mut self.window); + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + self.audio.pause(); + self.output.clear(); + } else { + self.input_capture + .set_input_paused(false, &self._sdl, &mut self.window); + self.audio.resume(); + } + } + + fn stop(&mut self) { + self.input_capture.release(&self._sdl, &mut self.window); + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + self.output.clear(); + self.audio.pause(); + self.presenter = None; + self.surface_size = None; + if let Ok(surface) = self.native_surface.as_mut() { + surface.hide(); + } + if self.external_renderer { + self.window.hide(); + } + self.visible = false; + self.paused = false; + self.frame_pacer.reset(); + } + + fn update_surface(&mut self, surface: &RenderSurface) -> Result<(), String> { + let Some(rect) = surface.rect.filter(|_| surface.visible) else { + self.input_capture.release(&self._sdl, &mut self.window); + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + if let Ok(native_surface) = self.native_surface.as_mut() { + native_surface.hide(); + } + if self.external_renderer { + self.window.hide(); + } + self.presenter = None; + self.surface_size = None; + self.visible = false; + return Ok(()); + }; + if self.external_renderer { + let bounds = surface.screen_rect.unwrap_or(rect); + self.window.set_position( + sdl2::video::WindowPos::Positioned(bounds.x), + sdl2::video::WindowPos::Positioned(bounds.y), + ); + self.window + .set_size(bounds.width.max(2), bounds.height.max(2)) + .map_err(|error| format!("failed to resize external Vulkan surface: {error}"))?; + self.window.show(); + if !self.visible { + self.window.raise(); + } + } else { + let parent_handle = surface.window_handle.as_deref().ok_or_else(|| { + "visible native surface is missing the shell window handle".to_owned() + })?; + self.native_surface + .as_mut() + .map_err(|error| error.clone())? + .attach_and_show( + parent_handle, + rect, + surface.screen_rect, + surface.device_scale_factor, + )?; + } + let display_refresh_hz = linux_display_refresh_hz(&self.video, &self.window); + if self + .frame_pacer + .reconfigure(self.stream_fps, display_refresh_hz) + { + eprintln!( + "Linux presentation pacing changed: stream={}fps display={} output={:.1}Hz mode={}", + self.stream_fps, + display_refresh_hz + .map_or_else(|| "unknown".to_owned(), |value| format!("{value}Hz")), + self.frame_pacer.presentation_hz(), + if self.frame_pacer.vrr_enabled() { + "cloud-gsync-vrr" + } else if self.frame_pacer.fast_stream() { + "nonblocking-fast-stream" + } else { + "adaptive-timestamp" + }, + ); + self.output.clear_linux_video(); + } + let size = if self.external_renderer { + let (width, height) = self.window.vulkan_drawable_size(); + (width.max(2), height.max(2)) + } else { + (rect.width.max(2), rect.height.max(2)) + }; + if self.presenter.is_some() && self.surface_size != Some(size) { + if let Some(presenter) = self.presenter.as_mut() { + presenter + .reconfigure(size.0, size.1) + .map_err(|error| error.to_string())?; + } + } + self.surface_size = Some(size); + self.visible = true; + Ok(()) + } + + fn pump(&mut self) -> Result { + if !self.external_renderer + && let Ok(surface) = self.native_surface.as_mut() + { + surface.refresh_ordering()?; + } + for event in self.event_pump.poll_iter().collect::>() { + if matches!(event, sdl2::event::Event::Quit { .. }) { + self.input_capture.release(&self._sdl, &mut self.window); + if let Ok(surface) = self.native_surface.as_mut() { + surface.hide(); + } + if self.external_renderer { + self.window.hide(); + } + self.presenter = None; + self.visible = false; + } else { + self.input_capture.handle_event( + &self._sdl, + &mut self.window, + self.stream_size, + event, + ); + } + } + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled( + self.visible + && self.input_capture.focused + && self.input_capture.relative_mouse_enabled(), + ); + } + // NVST input must not sit behind Vulkan presentation. FIFO acquire can + // legitimately report back-pressure, and queue_present may enter the + // compositor, so publish every event while we are still ahead of WSI. + // The runtime's normal post-pump drain remains as a fallback for input + // generated by lifecycle operations outside this event batch. + let captured_input = self.output.captured_input(); + for input in self.input_capture.take() { + captured_input.push(input); + } + if self.paused || !self.visible { + self.output.clear_linux_video(); + return Ok(false); + } + if self.external_renderer { + let (width, height) = self.window.vulkan_drawable_size(); + let drawable_size = (width.max(2), height.max(2)); + if self.surface_size != Some(drawable_size) { + if let Some(presenter) = self.presenter.as_mut() { + presenter + .reconfigure(drawable_size.0, drawable_size.1) + .map_err(|error| error.to_string())?; + } + self.surface_size = Some(drawable_size); + } + } + let now = Instant::now(); + if !self.frame_pacer.is_due(now) { + return Ok(false); + } + let decision = self.frame_pacer.decision(now); + let frame = self + .output + .take_linux_video_for_presentation(decision.selection, decision.target_queue_depth); + let Some(queued_frame) = frame else { + return Ok(false); + }; + self.frame_pacer + .observe_frame(queued_frame.arrived_at, queued_frame.frame.timestamp_us); + let frame = queued_frame.frame; + self.stream_size = (frame.format.width.max(1), frame.format.height.max(1)); + if frame.vulkan.as_ref().is_some_and(|vulkan| { + self.presenter + .as_ref() + .is_some_and(|presenter| !presenter.matches_vulkan_video_device(vulkan)) + }) { + // Decoder recovery can replace the FFmpeg Vulkan device. Rebuild + // the surface resources on the new device before touching it. + self.presenter = None; + } + if self.presenter.is_none() { + let size = self + .surface_size + .ok_or_else(|| "Linux Vulkan presenter has no visible surface extent".to_owned())?; + self.presenter = Some(create_linux_presenter( + &self.window, + size.0, + size.1, + frame.vulkan.as_deref(), + )?); + } + let presented = self + .presenter + .as_mut() + .ok_or_else(|| { + "Linux Vulkan presenter is not attached to a visible surface".to_owned() + })? + .present(&frame) + .map_err(|error| error.to_string())?; + if presented { + self.frame_pacer.mark_presented(Instant::now()); + } + Ok(presented) + } + + fn take_captured_input(&mut self) -> Vec { + self.input_capture.take() + } + + fn update_cursor(&mut self, bytes: &[u8]) { + if self.external_renderer { + self.input_capture + .apply_cursor(&self._sdl, &mut self.window, self.stream_size, bytes); + } + } +} + +#[cfg(target_os = "linux")] +impl Drop for LinuxHardwareOutput { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(target_os = "linux")] +fn create_linux_presenter( + window: &sdl2::video::Window, + width: u32, + height: u32, + vulkan_video: Option<&opennow_streamer_platform_linux::VulkanVideoFrame>, +) -> Result { + use std::ffi::c_void; + use std::num::NonZeroU64; + use std::ptr::NonNull; + + use raw_window_handle::{HasDisplayHandle, HasWindowHandle, RawDisplayHandle, RawWindowHandle}; + + let display_handle = window + .display_handle() + .map_err(|error| format!("SDL display handle unavailable: {error}"))?; + let window_handle = window + .window_handle() + .map_err(|error| format!("SDL window handle unavailable: {error}"))?; + let surface = match (display_handle.as_raw(), window_handle.as_raw()) { + (RawDisplayHandle::Xlib(display), RawWindowHandle::Xlib(window)) => { + let screen = display.screen; + let display = display + .display + .ok_or_else(|| "SDL X11 display pointer is null".to_owned())?; + let window = NonZeroU64::new(window.window) + .ok_or_else(|| "SDL returned an empty X11 window handle".to_owned())?; + let display = NonNull::::new(display.as_ptr().cast()) + .ok_or_else(|| "SDL X11 display pointer is null".to_owned())?; + unsafe { + opennow_streamer_platform_linux::NativeSurface::borrow_x11(display, window, screen) + } + } + (RawDisplayHandle::Wayland(display), RawWindowHandle::Wayland(window)) => unsafe { + opennow_streamer_platform_linux::NativeSurface::borrow_wayland( + display.display, + window.surface, + ) + }, + _ => { + return Err( + "SDL returned mismatched or unsupported Linux handles for Vulkan presentation" + .to_owned(), + ); + } + }; + match vulkan_video { + Some(frame) => opennow_streamer_platform_linux::VulkanPresenter::new_for_vulkan_video( + &surface, width, height, frame, + ), + None => opennow_streamer_platform_linux::VulkanPresenter::new(&surface, width, height), + } + .map_err(|error| error.to_string()) +} + +#[cfg(target_os = "linux")] +fn configure_linux_sdl_video_driver() { + let native_wayland = std::env::var("OPENNOW_LINUX_NATIVE_WAYLAND") + .ok() + .is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }); + let explicit_driver = std::env::var("SDL_VIDEODRIVER") + .ok() + .is_some_and(|value| !value.trim().is_empty()); + let xwayland_available = std::env::var("DISPLAY") + .ok() + .is_some_and(|value| !value.trim().is_empty()); + if !native_wayland && !explicit_driver && xwayland_available { + // NVIDIA's native Linux client deliberately runs through X11/XWayland + // and couples its xlib Vulkan surface to a dedicated XInput2 raw-input + // worker. Match that path by default; native Wayland remains available + // on systems without XWayland or through the explicit opt-in above. + let _ = sdl2::hint::set("SDL_VIDEODRIVER", "x11"); + } +} + +#[cfg(target_os = "linux")] +fn linux_display_refresh_hz( + video: &sdl2::VideoSubsystem, + window: &sdl2::video::Window, +) -> Option { + let display_index = window.display_index().ok()?; + let refresh_rate = video.current_display_mode(display_index).ok()?.refresh_rate; + u32::try_from(refresh_rate).ok().filter(|value| *value > 0) +} + +struct StreamAudioCallback { + output: Arc, +} + +impl AudioCallback for StreamAudioCallback { + type Channel = f32; + + fn callback(&mut self, output: &mut [f32]) { + self.output.fill_audio(output); + } +} + +pub(crate) struct SdlInputCapture { + captured: Vec, + pressed_keys: HashMap, + pressed_shortcuts: HashSet, + pressed_buttons: HashSet, + configured: bool, + enabled: bool, + focused: bool, + relative_mouse: bool, + external_relative_motion: bool, + external_mouse_buttons: bool, + cursor_state: RemoteCursorState, + cursors: HashMap<(u8, u8), sdl2::mouse::Cursor>, + game_controller: Option, + gamepads: HashMap, + gamepad_states: [GamepadState; 4], + last_gamepad_keepalive: Instant, + shortcuts: StreamShortcutBindings, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct GamepadState { + connected: bool, + buttons: u16, + left_trigger: u8, + right_trigger: u8, + raw_left_stick_x: i16, + raw_left_stick_y: i16, + raw_right_stick_x: i16, + raw_right_stick_y: i16, + left_stick_x: i16, + left_stick_y: i16, + right_stick_x: i16, + right_stick_y: i16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RemoteCursorState { + Unknown, + Hidden, + Visible, +} + +impl SdlInputCapture { + pub(crate) fn new( + enabled: bool, + external_relative_motion: bool, + external_mouse_buttons: bool, + shortcuts: StreamShortcutBindings, + ) -> Self { + Self { + captured: Vec::new(), + pressed_keys: HashMap::new(), + pressed_shortcuts: HashSet::new(), + pressed_buttons: HashSet::new(), + configured: enabled, + enabled, + focused: false, + relative_mouse: false, + external_relative_motion, + external_mouse_buttons, + // Do not infer hidden-cursor gameplay before the first server + // update. GFN sends a distinct predefined cursor ID 0 when the + // game actually wants locked relative input. + cursor_state: RemoteCursorState::Unknown, + cursors: HashMap::new(), + game_controller: None, + gamepads: HashMap::new(), + gamepad_states: [GamepadState::default(); 4], + last_gamepad_keepalive: Instant::now(), + shortcuts, + } + } + + pub(crate) fn enable_gamepads(&mut self, sdl: &sdl2::Sdl) { + let Ok(subsystem) = sdl.game_controller() else { + return; + }; + let count = subsystem.num_joysticks().unwrap_or(0); + self.game_controller = Some(subsystem); + for index in 0..count { + self.open_gamepad(index); + } + } + + fn open_gamepad(&mut self, joystick_index: u32) { + let Some(subsystem) = self.game_controller.as_ref() else { + return; + }; + if !subsystem.is_game_controller(joystick_index) { + return; + } + let Ok(controller) = subsystem.open(joystick_index) else { + return; + }; + let instance_id = controller.instance_id(); + if self.gamepads.contains_key(&instance_id) { + return; + } + let Some(slot) = + (0_u8..4).find(|slot| !self.gamepads.values().any(|(current, _)| current == slot)) + else { + return; + }; + self.gamepads.insert(instance_id, (slot, controller)); + self.gamepad_states[usize::from(slot)] = GamepadState { + connected: true, + ..GamepadState::default() + }; + self.push_gamepad(slot); + } + + fn close_gamepad(&mut self, instance_id: u32) { + let Some((slot, _)) = self.gamepads.remove(&instance_id) else { + return; + }; + self.gamepad_states[usize::from(slot)] = GamepadState::default(); + self.push_gamepad(slot); + } + + fn gamepad_bitmap(&self) -> u16 { + self.gamepad_states + .iter() + .enumerate() + .filter(|(_, state)| state.connected) + .fold(0_u16, |bitmap, (slot, _)| { + bitmap | (1 << slot) | (1 << (slot + 8)) + }) + } + + fn push_gamepad(&mut self, slot: u8) { + let state = self.gamepad_states[usize::from(slot)]; + self.captured.push(CapturedInput::Gamepad { + controller_id: slot, + bitmap: self.gamepad_bitmap(), + buttons: state.buttons, + left_trigger: state.left_trigger, + right_trigger: state.right_trigger, + left_stick_x: state.left_stick_x, + left_stick_y: state.left_stick_y, + right_stick_x: state.right_stick_x, + right_stick_y: state.right_stick_y, + }); + } + + fn update_gamepad_button( + &mut self, + instance_id: u32, + button: sdl2::controller::Button, + pressed: bool, + ) { + let Some(slot) = self.gamepads.get(&instance_id).map(|(slot, _)| *slot) else { + return; + }; + if button == sdl2::controller::Button::Guide { + if pressed { + self.captured.push(CapturedInput::Guide); + } + return; + } + let Some(mask) = gamepad_button_mask(button) else { + return; + }; + let state = &mut self.gamepad_states[usize::from(slot)]; + if pressed { + state.buttons |= mask; + } else { + state.buttons &= !mask; + } + self.push_gamepad(slot); + } + + fn update_gamepad_axis(&mut self, instance_id: u32, axis: sdl2::controller::Axis, value: i16) { + let Some(slot) = self.gamepads.get(&instance_id).map(|(slot, _)| *slot) else { + return; + }; + let state = &mut self.gamepad_states[usize::from(slot)]; + match axis { + sdl2::controller::Axis::LeftX => state.raw_left_stick_x = value, + sdl2::controller::Axis::LeftY => state.raw_left_stick_y = value, + sdl2::controller::Axis::RightX => state.raw_right_stick_x = value, + sdl2::controller::Axis::RightY => state.raw_right_stick_y = value, + sdl2::controller::Axis::TriggerLeft => state.left_trigger = trigger_axis(value), + sdl2::controller::Axis::TriggerRight => state.right_trigger = trigger_axis(value), + } + (state.left_stick_x, state.left_stick_y) = + radial_deadzone(state.raw_left_stick_x, state.raw_left_stick_y); + state.left_stick_y = state.left_stick_y.saturating_neg(); + (state.right_stick_x, state.right_stick_y) = + radial_deadzone(state.raw_right_stick_x, state.raw_right_stick_y); + state.right_stick_y = state.right_stick_y.saturating_neg(); + self.push_gamepad(slot); + } + + pub(crate) fn set_input_paused( + &mut self, + paused: bool, + sdl: &sdl2::Sdl, + window: &mut sdl2::video::Window, + ) { + if paused { + self.release(sdl, window); + // Publish neutral states before capture is suspended so a remote + // game cannot retain a held trigger, stick, or button while the + // Qt shell owns input. + for slot in 0_u8..4 { + if self.gamepad_states[usize::from(slot)].connected { + self.gamepad_states[usize::from(slot)] = GamepadState { + connected: true, + ..GamepadState::default() + }; + self.push_gamepad(slot); + } + } + self.enabled = false; + } else { + self.enabled = self.configured; + self.last_gamepad_keepalive = Instant::now(); + if self.enabled { + let disconnected = self + .gamepads + .iter() + .filter_map(|(instance_id, (_, controller))| { + (!controller.attached()).then_some(*instance_id) + }) + .collect::>(); + for instance_id in disconnected { + self.close_gamepad(instance_id); + } + let count = self + .game_controller + .as_ref() + .and_then(|controller| controller.num_joysticks().ok()) + .unwrap_or(0); + for joystick_index in 0..count { + self.open_gamepad(joystick_index); + } + } + } + } + + pub(crate) fn handle_event( + &mut self, + sdl: &sdl2::Sdl, + window: &mut sdl2::video::Window, + stream_size: (u32, u32), + event: sdl2::event::Event, + ) { + use sdl2::event::{Event, WindowEvent}; + + if !self.enabled { + return; + } + let window_size = window.size(); + match event { + Event::ControllerDeviceAdded { which, .. } => self.open_gamepad(which), + Event::ControllerDeviceRemoved { which, .. } => self.close_gamepad(which), + Event::ControllerButtonDown { which, button, .. } => { + self.update_gamepad_button(which, button, true) + } + Event::ControllerButtonUp { which, button, .. } => { + self.update_gamepad_button(which, button, false) + } + Event::ControllerAxisMotion { + which, axis, value, .. + } => self.update_gamepad_axis(which, axis, value), + Event::KeyDown { + scancode: Some(scancode), + keymod, + repeat: false, + .. + } => { + if is_native_guide_shortcut(scancode, keymod) { + // Match the GeForce NOW desktop client: Ctrl+G belongs to + // the local stream menu and must never reach the game. + // Tracking it as a pressed shortcut also consumes key-up + // even when Ctrl is released before G. + self.pressed_shortcuts.insert(scancode); + self.captured.push(CapturedInput::Guide); + return; + } + if let Some(virtual_key) = sdl_virtual_key(scancode) + && let Some(action) = self + .shortcuts + .action(virtual_key, sdl_modifiers(scancode, keymod)) + { + self.pressed_shortcuts.insert(scancode); + self.captured.push(CapturedInput::Shortcut(action)); + return; + } + let Some(virtual_key) = sdl_virtual_key(scancode) else { + return; + }; + if self.pressed_keys.insert(scancode, virtual_key).is_none() { + self.captured.push(CapturedInput::Key { + virtual_key, + modifiers: sdl_modifiers(scancode, keymod), + pressed: true, + }); + } + } + Event::KeyUp { + scancode: Some(scancode), + keymod, + .. + } => { + if self.pressed_shortcuts.remove(&scancode) { + return; + } + if let Some(virtual_key) = self.pressed_keys.remove(&scancode) { + self.captured.push(CapturedInput::Key { + virtual_key, + modifiers: sdl_modifiers(scancode, keymod), + pressed: false, + }); + } + } + Event::MouseMotion { .. } if self.relative_mouse && self.external_relative_motion => {} + Event::MouseMotion { xrel, yrel, .. } if self.relative_mouse => { + push_mouse_motion(&mut self.captured, xrel, yrel); + } + Event::MouseMotion { x, y, .. } if self.cursor_state != RemoteCursorState::Hidden => { + let absolute = + map_window_point_to_stream_viewport((x, y), stream_size, window_size); + self.captured.push(CapturedInput::MouseAbsolute { + x: absolute.x, + y: absolute.y, + width: absolute.width, + height: absolute.height, + }); + } + Event::MouseButtonDown { mouse_btn, .. } => { + // A button event proves this window owns foreground input even + // if SDL delivered FocusGained later in the same pump batch. + self.focused = true; + if self.cursor_state == RemoteCursorState::Hidden { + self.enable_relative_mouse(sdl, window); + } + if !self.external_mouse_buttons + && let Some(button) = sdl_mouse_button(mouse_btn) + && self.pressed_buttons.insert(button) + { + self.captured.push(CapturedInput::MouseButton { + button, + pressed: true, + }); + } + } + Event::MouseButtonUp { mouse_btn, .. } => { + if !self.external_mouse_buttons + && let Some(button) = sdl_mouse_button(mouse_btn) + && self.pressed_buttons.remove(&button) + { + self.captured.push(CapturedInput::MouseButton { + button, + pressed: false, + }); + } + } + Event::MouseWheel { + x, y, direction, .. + } if !self.external_mouse_buttons && (x != 0 || y != 0) => { + let direction = if direction == sdl2::mouse::MouseWheelDirection::Flipped { + -1 + } else { + 1 + }; + self.captured.push(CapturedInput::MouseWheel { + delta_x: clamp_i16(x.saturating_mul(120).saturating_mul(direction)), + delta_y: clamp_i16(y.saturating_mul(120).saturating_mul(direction)), + }); + } + Event::Window { + win_event: WindowEvent::FocusGained, + .. + } => { + self.focused = true; + if self.cursor_state == RemoteCursorState::Hidden { + self.enable_relative_mouse(sdl, window); + } + } + Event::Window { + win_event: WindowEvent::FocusLost, + .. + } => { + self.focused = false; + self.release(sdl, window); + } + _ => {} + } + } + + fn enable_relative_mouse(&mut self, sdl: &sdl2::Sdl, window: &mut sdl2::video::Window) { + if self.focused && !self.relative_mouse { + window.set_mouse_grab(true); + sdl.mouse().set_relative_mouse_mode(true); + sdl.mouse().show_cursor(false); + self.relative_mouse = true; + eprintln!("External SDL mouse control mode: locked relative"); + } + } + + fn disable_relative_mouse(&mut self, sdl: &sdl2::Sdl, window: &mut sdl2::video::Window) { + if self.relative_mouse { + sdl.mouse().set_relative_mouse_mode(false); + window.set_mouse_grab(false); + self.relative_mouse = false; + eprintln!("External SDL mouse control mode: absolute cursor"); + } + } + + pub(crate) fn toggle_pointer_lock( + &mut self, + sdl: &sdl2::Sdl, + window: &mut sdl2::video::Window, + ) { + if self.relative_mouse { + self.disable_relative_mouse(sdl, window); + sdl.mouse().show_cursor(true); + } else { + self.enable_relative_mouse(sdl, window); + } + } + + pub(crate) fn apply_cursor( + &mut self, + sdl: &sdl2::Sdl, + window: &mut sdl2::video::Window, + stream_size: (u32, u32), + bytes: &[u8], + ) { + let Some((&message_type, remainder)) = bytes.split_first() else { + return; + }; + if !matches!(message_type, 0 | 1) { + return; + } + let Some(&cursor_id) = remainder.first() else { + return; + }; + let was_cursor_visible = self.cursor_state == RemoteCursorState::Visible; + self.cursor_state = cursor_state_for_message(message_type, cursor_id); + eprintln!( + "GFN cursor applied: sourceType={message_type} cursorId={cursor_id} state={:?} bytes={}", + self.cursor_state, + bytes.len(), + ); + if self.cursor_state == RemoteCursorState::Hidden { + self.enable_relative_mouse(sdl, window); + return; + } + self.disable_relative_mouse(sdl, window); + if !native_cursor_overlay_enabled() { + sdl.mouse().show_cursor(false); + return; + } + let cursor_key = (message_type, cursor_id); + if message_type == 1 { + match custom_sdl_cursor(bytes) { + Ok(cursor) => { + self.cursors.insert(cursor_key, cursor); + } + Err(error) => { + eprintln!("GFN custom cursor {cursor_id} could not be decoded: {error}"); + } + } + } else if let std::collections::hash_map::Entry::Vacant(entry) = + self.cursors.entry(cursor_key) + { + let system_cursor = match cursor_id { + 2 => sdl2::mouse::SystemCursor::IBeam, + 3 => sdl2::mouse::SystemCursor::Wait, + 4 => sdl2::mouse::SystemCursor::Crosshair, + 5 => sdl2::mouse::SystemCursor::WaitArrow, + 6 => sdl2::mouse::SystemCursor::SizeNWSE, + 7 => sdl2::mouse::SystemCursor::SizeNESW, + 8 => sdl2::mouse::SystemCursor::SizeWE, + 9 => sdl2::mouse::SystemCursor::SizeNS, + 10 => sdl2::mouse::SystemCursor::SizeAll, + 12 => sdl2::mouse::SystemCursor::Hand, + _ => sdl2::mouse::SystemCursor::Arrow, + }; + if let Ok(cursor) = sdl2::mouse::Cursor::from_system(system_cursor) { + entry.insert(cursor); + } + } + if let Some(cursor) = self.cursors.get(&cursor_key) { + cursor.set(); + } else if let Ok(cursor) = + sdl2::mouse::Cursor::from_system(sdl2::mouse::SystemCursor::Arrow) + { + cursor.set(); + self.cursors.insert(cursor_key, cursor); + } + sdl.mouse().show_cursor(true); + if !was_cursor_visible && let Some(position) = parse_cursor_position(bytes) { + let viewport = aspect_fit( + stream_size.0.max(1), + stream_size.1.max(1), + window.size().0.max(1), + window.size().1.max(1), + ); + let x = viewport.x() + normalized_cursor_coordinate(position.0, viewport.width()); + let y = viewport.y() + normalized_cursor_coordinate(position.1, viewport.height()); + sdl.mouse().warp_mouse_in_window(window, x, y); + } + } + + pub(crate) fn release(&mut self, sdl: &sdl2::Sdl, window: &mut sdl2::video::Window) { + for (_, virtual_key) in self.pressed_keys.drain() { + self.captured.push(CapturedInput::Key { + virtual_key, + modifiers: 0, + pressed: false, + }); + } + self.pressed_shortcuts.clear(); + for button in self.pressed_buttons.drain() { + self.captured.push(CapturedInput::MouseButton { + button, + pressed: false, + }); + } + self.disable_relative_mouse(sdl, window); + window.set_mouse_grab(false); + sdl.mouse().show_cursor(true); + } + + pub(crate) fn take(&mut self) -> Vec { + if self.enabled && self.last_gamepad_keepalive.elapsed().as_millis() >= 100 { + for slot in 0_u8..4 { + if self.gamepad_states[usize::from(slot)].connected { + self.push_gamepad(slot); + } + } + self.last_gamepad_keepalive = Instant::now(); + } + std::mem::take(&mut self.captured) + } + + #[cfg(any(target_os = "linux", target_os = "windows"))] + pub(crate) const fn relative_mouse_enabled(&self) -> bool { + self.relative_mouse + } + + #[cfg(target_os = "windows")] + pub(crate) const fn capture_enabled(&self) -> bool { + self.enabled + } +} + +fn gamepad_button_mask(button: sdl2::controller::Button) -> Option { + use sdl2::controller::Button; + Some(match button { + Button::A => 0x1000, + Button::B => 0x2000, + Button::X => 0x4000, + Button::Y => 0x8000, + Button::Back => 0x0020, + Button::Start => 0x0010, + Button::LeftStick => 0x0040, + Button::RightStick => 0x0080, + Button::LeftShoulder => 0x0100, + Button::RightShoulder => 0x0200, + Button::DPadUp => 0x0001, + Button::DPadDown => 0x0002, + Button::DPadLeft => 0x0004, + Button::DPadRight => 0x0008, + _ => return None, + }) +} + +fn radial_deadzone(x: i16, y: i16) -> (i16, i16) { + const DEADZONE: f32 = 0.15; + let normalized_x = f32::from(x) / 32767.0; + let normalized_y = f32::from(y) / 32767.0; + let magnitude = normalized_x.hypot(normalized_y); + if magnitude < DEADZONE { + return (0, 0); + } + let scaled = ((magnitude - DEADZONE) / (1.0 - DEADZONE)).clamp(0.0, 1.0); + let factor = scaled / magnitude; + ( + (normalized_x * factor * 32767.0).round() as i16, + (normalized_y * factor * 32767.0).round() as i16, + ) +} + +fn trigger_axis(value: i16) -> u8 { + let normalized = (f32::from(value) / 32767.0).clamp(0.0, 1.0); + (normalized * 255.0).round() as u8 +} + +pub(crate) struct SoftwareOutput { + // Texture and window-owned resources must be released before the SDL + // renderer/window, and SDL itself must outlive every derived resource. + texture: Option, + native_surface: Result, + input_capture: SdlInputCapture, + #[cfg(target_os = "linux")] + raw_input: Option, + canvas: WindowCanvas, + texture_size: Option<(u32, u32)>, + texture_format: Option, + event_pump: sdl2::EventPump, + audio: AudioDevice, + output: Arc, + external_renderer: bool, + #[cfg(target_os = "linux")] + presented_linux_frame: bool, + visible: bool, + paused: bool, + _sdl: sdl2::Sdl, +} + +impl SoftwareOutput { + fn initialize( + output: Arc, + stream: MediaStreamConfig, + audio_device: &AudioOutputDevice, + ) -> Result { + #[cfg(target_os = "linux")] + configure_linux_sdl_video_driver(); + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let video = sdl + .video() + .map_err(|error| format!("SDL video initialization failed: {error}"))?; + let audio_subsystem = sdl + .audio() + .map_err(|error| format!("SDL audio initialization failed: {error}"))?; + let external_renderer = external_renderer_enabled(); + let mut window_builder = video.window("OpenNOW Stream", 1280, 720); + window_builder + .position_centered() + .resizable() + .hidden() + .metal_view(); + if external_renderer && cfg!(target_os = "linux") { + window_builder.allow_highdpi(); + } + if !external_renderer { + window_builder.borderless(); + } + let window = window_builder + .build() + .map_err(|error| format!("native video window creation failed: {error}"))?; + let mut canvas = window + .into_canvas() + .build() + .map_err(|error| format!("native video renderer creation failed: {error}"))?; + canvas.set_draw_color(Color::BLACK); + canvas.clear(); + canvas.present(); + + let audio = open_audio_playback(&audio_subsystem, &output, audio_device)?; + if audio.spec().freq != AUDIO_SAMPLE_RATE || audio.spec().channels != AUDIO_CHANNELS { + return Err(format!( + "native audio output returned unsupported format: {} Hz, {} channels", + audio.spec().freq, + audio.spec().channels + )); + } + let event_pump = sdl + .event_pump() + .map_err(|error| format!("native window event pump creation failed: {error}"))?; + let native_surface = if video.current_video_driver() == "dummy" { + Err("dummy SDL video driver has no native presentation handle".to_owned()) + } else { + NativeSurface::new(canvas.window()) + }; + let capture_input = cfg!(any(target_os = "windows", target_os = "linux")) + && external_renderer + && native_input_capture_enabled(); + #[cfg(target_os = "linux")] + let raw_input = if capture_input && video.current_video_driver() == "x11" { + match LinuxXInputController::start(output.captured_input()) { + Ok(controller) => Some(controller), + Err(error) => { + eprintln!( + "Dedicated Linux XInput2 unavailable; retaining SDL raw motion: {error}" + ); + None + } + } + } else { + None + }; + #[cfg(target_os = "linux")] + let external_relative_motion = raw_input.is_some(); + #[cfg(not(target_os = "linux"))] + let external_relative_motion = false; + let mut input_capture = SdlInputCapture::new( + capture_input, + external_relative_motion, + false, + stream.shortcuts, + ); + input_capture.enable_gamepads(&sdl); + + Ok(Self { + texture: None, + native_surface, + input_capture, + #[cfg(target_os = "linux")] + raw_input, + canvas, + texture_size: None, + texture_format: None, + event_pump, + audio, + output, + external_renderer, + #[cfg(target_os = "linux")] + presented_linux_frame: false, + visible: false, + paused: false, + _sdl: sdl, + }) + } + + fn start(&mut self, surface: Option<&RenderSurface>) -> Result<(), String> { + self.paused = false; + #[cfg(target_os = "linux")] + { + self.presented_linux_frame = false; + } + self.output.clear(); + self.audio.resume(); + if let Some(surface) = surface { + self.update_surface(surface)?; + } + Ok(()) + } + + fn set_paused(&mut self, paused: bool) { + self.paused = paused; + if paused { + self.input_capture + .set_input_paused(true, &self._sdl, self.canvas.window_mut()); + #[cfg(target_os = "linux")] + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + self.audio.pause(); + self.output.clear(); + } else { + self.input_capture + .set_input_paused(false, &self._sdl, self.canvas.window_mut()); + self.audio.resume(); + } + } + + fn stop(&mut self) { + self.input_capture + .release(&self._sdl, self.canvas.window_mut()); + #[cfg(target_os = "linux")] + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + self.flush_captured_input(); + self.audio.pause(); + self.output.clear(); + if let Ok(surface) = self.native_surface.as_mut() { + surface.hide(); + } + if self.external_renderer { + self.canvas.window_mut().hide(); + } + self.visible = false; + self.paused = false; + self.texture = None; + self.texture_size = None; + self.texture_format = None; + #[cfg(target_os = "linux")] + { + self.presented_linux_frame = false; + } + } + + fn update_surface(&mut self, surface: &RenderSurface) -> Result<(), String> { + let Some(rect) = surface.rect.filter(|_| surface.visible) else { + #[cfg(target_os = "linux")] + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled(false); + } + if let Ok(native_surface) = self.native_surface.as_mut() { + native_surface.hide(); + } + if self.external_renderer { + self.canvas.window_mut().hide(); + } + self.visible = false; + return Ok(()); + }; + if self.external_renderer { + let bounds = surface.screen_rect.unwrap_or(rect); + let window = self.canvas.window_mut(); + window.set_position( + sdl2::video::WindowPos::Positioned(bounds.x), + sdl2::video::WindowPos::Positioned(bounds.y), + ); + window + .set_size(bounds.width.max(2), bounds.height.max(2)) + .map_err(|error| format!("failed to resize external SDL video surface: {error}"))?; + window.show(); + window.raise(); + self.visible = true; + return Ok(()); + } + let parent_handle = surface.window_handle.as_deref().ok_or_else(|| { + "visible native surface is missing the shell window handle".to_owned() + })?; + self.native_surface + .as_mut() + .map_err(|error| error.clone())? + .attach_and_show( + parent_handle, + rect, + surface.screen_rect, + surface.device_scale_factor, + )?; + self.visible = true; + Ok(()) + } + + fn pump(&mut self) -> Result { + if !self.external_renderer + && let Ok(surface) = self.native_surface.as_mut() + { + surface.refresh_ordering()?; + } + let events = self.event_pump.poll_iter().collect::>(); + for event in events { + if matches!(event, sdl2::event::Event::Quit { .. }) { + self.input_capture + .release(&self._sdl, self.canvas.window_mut()); + if let Ok(surface) = self.native_surface.as_mut() { + surface.hide(); + } + if self.external_renderer { + self.canvas.window_mut().hide(); + } + self.visible = false; + continue; + } + let window_size = self.canvas.window().size(); + let stream_size = self.texture_size.unwrap_or(window_size); + self.input_capture.handle_event( + &self._sdl, + self.canvas.window_mut(), + stream_size, + event, + ); + } + #[cfg(target_os = "linux")] + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_enabled( + self.visible + && self.input_capture.focused + && self.input_capture.relative_mouse_enabled(), + ); + } + if self.paused || !self.visible { + self.output.take_video(); + #[cfg(target_os = "linux")] + self.output.clear_linux_video(); + return Ok(false); + } + #[cfg(target_os = "linux")] + if let Some(frame) = self.output.take_linux_video() { + return self.present_linux_frame(frame.frame); + } + let Some(frame) = self.output.take_video() else { + return Ok(false); + }; + #[cfg(target_os = "linux")] + { + self.presented_linux_frame = false; + } + if self.texture_size != Some((frame.width, frame.height)) + || self.texture_format != Some(PixelFormatEnum::RGB24) + { + self.texture = Some( + self.canvas + .texture_creator() + .create_texture_streaming(PixelFormatEnum::RGB24, frame.width, frame.height) + .map_err(|error| format!("video texture creation failed: {error}"))?, + ); + self.texture_size = Some((frame.width, frame.height)); + self.texture_format = Some(PixelFormatEnum::RGB24); + } + let texture = self.texture.as_mut().expect("texture was just created"); + texture + .update(None, &frame.rgb, frame.width as usize * 3) + .map_err(|error| format!("video texture upload failed: {error}"))?; + let (output_width, output_height) = self.canvas.output_size()?; + let target = aspect_fit(frame.width, frame.height, output_width, output_height); + self.canvas.set_draw_color(Color::BLACK); + self.canvas.clear(); + self.canvas.copy(texture, None, target)?; + self.canvas.present(); + Ok(true) + } + + #[cfg(target_os = "linux")] + fn present_linux_frame(&mut self, frame: LinuxDecodedVideoFrame) -> Result { + use opennow_streamer_platform_linux::PixelFormat; + + if frame.format.pixel_format != PixelFormat::Nv12 || frame.planes.len() != 2 { + return Err(format!( + "SDL Linux presentation requires NV12 with two planes, received {:?}", + frame.format.pixel_format + )); + } + let width = frame.format.width; + let height = frame.format.height; + if self.texture_size != Some((width, height)) + || self.texture_format != Some(PixelFormatEnum::NV12) + { + self.texture = Some( + self.canvas + .texture_creator() + .create_texture_streaming(PixelFormatEnum::NV12, width, height) + .map_err(|error| format!("NV12 texture creation failed: {error}"))?, + ); + self.texture_size = Some((width, height)); + self.texture_format = Some(PixelFormatEnum::NV12); + } + let y_plane = &frame.planes[0]; + let uv_plane = &frame.planes[1]; + self.texture + .as_mut() + .expect("NV12 texture was just created") + .with_lock(None, |destination, pitch| { + for row in 0..height as usize { + let source_start = row * y_plane.stride; + let destination_start = row * pitch; + destination[destination_start..destination_start + width as usize] + .copy_from_slice( + &y_plane.data[source_start..source_start + width as usize], + ); + } + let uv_base = pitch * height as usize; + for row in 0..height as usize / 2 { + let source_start = row * uv_plane.stride; + let destination_start = uv_base + row * pitch; + destination[destination_start..destination_start + width as usize] + .copy_from_slice( + &uv_plane.data[source_start..source_start + width as usize], + ); + } + }) + .map_err(|error| format!("NV12 texture upload failed: {error}"))?; + let (output_width, output_height) = self.canvas.output_size()?; + let target = aspect_fit(width, height, output_width, output_height); + self.canvas.set_draw_color(Color::BLACK); + self.canvas.clear(); + self.canvas.copy( + self.texture.as_ref().expect("NV12 texture exists"), + None, + target, + )?; + self.canvas.present(); + self.presented_linux_frame = true; + Ok(true) + } + + fn backend_label(&self) -> &'static str { + #[cfg(target_os = "linux")] + if self.presented_linux_frame { + return "Linux decoder/SDL NV12"; + } + "OpenH264/SDL" + } + + fn take_captured_input(&mut self) -> Vec { + self.input_capture.take() + } + + fn update_cursor(&mut self, bytes: &[u8]) { + if self.external_renderer { + let stream_size = self + .texture_size + .unwrap_or_else(|| self.canvas.window().size()); + self.input_capture.apply_cursor( + &self._sdl, + self.canvas.window_mut(), + stream_size, + bytes, + ); + } + } + + fn flush_captured_input(&mut self) { + let captured_input = self.output.captured_input(); + for input in self.take_captured_input() { + captured_input.push(input); + } + } +} + +impl Drop for SoftwareOutput { + fn drop(&mut self) { + self.stop(); + } +} + +fn sdl_virtual_key(scancode: sdl2::keyboard::Scancode) -> Option { + use sdl2::keyboard::Scancode; + + Some(match scancode { + Scancode::A => 0x41, + Scancode::B => 0x42, + Scancode::C => 0x43, + Scancode::D => 0x44, + Scancode::E => 0x45, + Scancode::F => 0x46, + Scancode::G => 0x47, + Scancode::H => 0x48, + Scancode::I => 0x49, + Scancode::J => 0x4a, + Scancode::K => 0x4b, + Scancode::L => 0x4c, + Scancode::M => 0x4d, + Scancode::N => 0x4e, + Scancode::O => 0x4f, + Scancode::P => 0x50, + Scancode::Q => 0x51, + Scancode::R => 0x52, + Scancode::S => 0x53, + Scancode::T => 0x54, + Scancode::U => 0x55, + Scancode::V => 0x56, + Scancode::W => 0x57, + Scancode::X => 0x58, + Scancode::Y => 0x59, + Scancode::Z => 0x5a, + Scancode::Num0 => 0x30, + Scancode::Num1 => 0x31, + Scancode::Num2 => 0x32, + Scancode::Num3 => 0x33, + Scancode::Num4 => 0x34, + Scancode::Num5 => 0x35, + Scancode::Num6 => 0x36, + Scancode::Num7 => 0x37, + Scancode::Num8 => 0x38, + Scancode::Num9 => 0x39, + Scancode::Return | Scancode::KpEnter => 0x0d, + Scancode::Escape => 0x1b, + Scancode::Backspace => 0x08, + Scancode::Tab => 0x09, + Scancode::Space => 0x20, + Scancode::Minus => 0xbd, + Scancode::Equals | Scancode::KpEquals => 0xbb, + Scancode::LeftBracket => 0xdb, + Scancode::RightBracket => 0xdd, + Scancode::Backslash => 0xdc, + Scancode::NonUsBackslash => 0xe2, + Scancode::Semicolon => 0xba, + Scancode::Apostrophe => 0xde, + Scancode::Grave => 0xc0, + Scancode::Comma => 0xbc, + Scancode::Period => 0xbe, + Scancode::Slash => 0xbf, + Scancode::F1 => 0x70, + Scancode::F2 => 0x71, + Scancode::F3 => 0x72, + Scancode::F4 => 0x73, + Scancode::F5 => 0x74, + Scancode::F6 => 0x75, + Scancode::F7 => 0x76, + Scancode::F8 => 0x77, + Scancode::F9 => 0x78, + Scancode::F10 => 0x79, + Scancode::F11 => 0x7a, + Scancode::F12 => 0x7b, + Scancode::F13 => 0x7c, + Scancode::F14 => 0x7d, + Scancode::F15 => 0x7e, + Scancode::F16 => 0x7f, + Scancode::F17 => 0x80, + Scancode::F18 => 0x81, + Scancode::F19 => 0x82, + Scancode::F20 => 0x83, + Scancode::F21 => 0x84, + Scancode::F22 => 0x85, + Scancode::F23 => 0x86, + Scancode::F24 => 0x87, + Scancode::Right => 0x27, + Scancode::Left => 0x25, + Scancode::Down => 0x28, + Scancode::Up => 0x26, + Scancode::LCtrl => 0xa2, + Scancode::LShift => 0xa0, + Scancode::LAlt => 0xa4, + Scancode::LGui => 0x5b, + Scancode::RCtrl => 0xa3, + Scancode::RShift => 0xa1, + Scancode::RAlt => 0xa5, + Scancode::RGui => 0x5c, + Scancode::CapsLock => 0x14, + Scancode::NumLockClear => 0x90, + Scancode::Insert => 0x2d, + Scancode::Delete => 0x2e, + Scancode::Home => 0x24, + Scancode::End => 0x23, + Scancode::PageUp => 0x21, + Scancode::PageDown => 0x22, + Scancode::PrintScreen => 0x2a, + Scancode::ScrollLock => 0x91, + Scancode::Pause => 0x13, + Scancode::Application => 0x5d, + Scancode::Kp0 => 0x60, + Scancode::Kp1 => 0x61, + Scancode::Kp2 => 0x62, + Scancode::Kp3 => 0x63, + Scancode::Kp4 => 0x64, + Scancode::Kp5 => 0x65, + Scancode::Kp6 => 0x66, + Scancode::Kp7 => 0x67, + Scancode::Kp8 => 0x68, + Scancode::Kp9 => 0x69, + Scancode::KpPlus => 0x6b, + Scancode::KpMinus => 0x6d, + Scancode::KpMultiply => 0x6a, + Scancode::KpDivide => 0x6f, + Scancode::KpPeriod => 0x6e, + _ => return None, + }) +} + +fn sdl_modifiers(scancode: sdl2::keyboard::Scancode, modifiers: sdl2::keyboard::Mod) -> u16 { + use sdl2::keyboard::{Mod, Scancode}; + + let mut flags = 0; + if !matches!(scancode, Scancode::LShift | Scancode::RShift) + && modifiers.intersects(Mod::LSHIFTMOD | Mod::RSHIFTMOD) + { + flags |= 0x01; + } + if !matches!(scancode, Scancode::LCtrl | Scancode::RCtrl) + && modifiers.intersects(Mod::LCTRLMOD | Mod::RCTRLMOD) + { + flags |= 0x02; + } + if !matches!(scancode, Scancode::LAlt | Scancode::RAlt) + && modifiers.intersects(Mod::LALTMOD | Mod::RALTMOD) + { + flags |= 0x04; + } + if !matches!(scancode, Scancode::LGui | Scancode::RGui) + && modifiers.intersects(Mod::LGUIMOD | Mod::RGUIMOD) + { + flags |= 0x08; + } + flags +} + +fn sdl_mouse_button(button: sdl2::mouse::MouseButton) -> Option { + use sdl2::mouse::MouseButton; + + match button { + MouseButton::Left => Some(1), + MouseButton::Middle => Some(2), + MouseButton::Right => Some(3), + MouseButton::X1 => Some(4), + MouseButton::X2 => Some(5), + MouseButton::Unknown => None, + } +} + +fn push_mouse_motion(input: &mut Vec, delta_x: i32, delta_y: i32) { + if delta_x == 0 && delta_y == 0 { + return; + } + input.push(CapturedInput::MouseMove { + delta_x: clamp_i16(delta_x), + delta_y: clamp_i16(delta_y), + }); +} + +fn clamp_i16(value: i32) -> i16 { + value.clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16 +} + +fn clamp_i32_u16(value: i32) -> u16 { + value.clamp(0, i32::from(u16::MAX)) as u16 +} + +fn clamp_u32_u16(value: u32) -> u16 { + value.min(u32::from(u16::MAX)) as u16 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AbsoluteMouseViewport { + x: u16, + y: u16, + width: u16, + height: u16, +} + +struct CustomCursorPayload<'a> { + hotspot_x: u8, + hotspot_y: u8, + image_base64: &'a [u8], + scale: f32, +} + +fn cursor_state_for_message(message_type: u8, cursor_id: u8) -> RemoteCursorState { + if message_type == 0 && cursor_id == 0 { + RemoteCursorState::Hidden + } else { + // Custom cursor ID 0 is still a visible bitmap. Only predefined ID 0 + // is the protocol's hidden/raw-input cursor. + RemoteCursorState::Visible + } +} + +fn parse_custom_cursor_payload(bytes: &[u8]) -> Result, String> { + if bytes.first() != Some(&1) || bytes.len() < 7 { + return Err("not a complete custom cursor message".to_owned()); + } + let mime_length = usize::from(bytes[4]); + let mime_end = 5_usize + .checked_add(mime_length) + .ok_or_else(|| "custom cursor MIME length overflow".to_owned())?; + let image_length_bytes = bytes + .get(mime_end..mime_end + 2) + .ok_or_else(|| "custom cursor is missing its image length".to_owned())?; + let image_length = usize::from(u16::from_le_bytes([ + image_length_bytes[0], + image_length_bytes[1], + ])); + let image_start = mime_end + 2; + let image_end = image_start + .checked_add(image_length) + .ok_or_else(|| "custom cursor image length overflow".to_owned())?; + let image_base64 = bytes + .get(image_start..image_end) + .ok_or_else(|| "custom cursor image is truncated".to_owned())?; + let position_end = image_end.saturating_add(4); + let scale = bytes + .get(position_end..position_end + 2) + .map(|raw| f32::from(u16::from_le_bytes([raw[0], raw[1]])) / 100.0) + .filter(|scale| scale.is_finite() && *scale > 0.0) + .unwrap_or(1.0); + Ok(CustomCursorPayload { + hotspot_x: bytes[2], + hotspot_y: bytes[3], + image_base64, + scale, + }) +} + +fn custom_sdl_cursor(bytes: &[u8]) -> Result { + const MAX_CURSOR_EXTENT: u32 = 256; + let payload = parse_custom_cursor_payload(bytes)?; + let encoded = std::str::from_utf8(payload.image_base64) + .map_err(|error| format!("cursor image base64 is not UTF-8: {error}"))?; + let compressed = BASE64 + .decode(encoded) + .map_err(|error| format!("cursor image base64 is invalid: {error}"))?; + let image = ImageReader::new(IoCursor::new(compressed)) + .with_guessed_format() + .map_err(|error| format!("cursor image format is unknown: {error}"))? + .decode() + .map_err(|error| format!("cursor image decode failed: {error}"))? + .to_rgba8(); + let (source_width, source_height) = image.dimensions(); + if source_width == 0 + || source_height == 0 + || source_width > MAX_CURSOR_EXTENT + || source_height > MAX_CURSOR_EXTENT + { + return Err(format!( + "cursor image dimensions are outside 1..={MAX_CURSOR_EXTENT}: {source_width}x{source_height}" + )); + } + let width = ((source_width as f32 / payload.scale).round() as u32).clamp(1, MAX_CURSOR_EXTENT); + let height = + ((source_height as f32 / payload.scale).round() as u32).clamp(1, MAX_CURSOR_EXTENT); + let rgba = if (width, height) == (source_width, source_height) { + image + } else { + image::imageops::resize(&image, width, height, image::imageops::FilterType::Triangle) + }; + let row_bytes = width as usize * 4; + let mut surface = sdl2::surface::Surface::new(width, height, PixelFormatEnum::RGBA32)?; + let pitch = surface.pitch() as usize; + surface.with_lock_mut(|destination| { + for (source, destination) in rgba + .as_raw() + .chunks_exact(row_bytes) + .zip(destination.chunks_mut(pitch)) + { + destination[..row_bytes].copy_from_slice(source); + } + }); + let hotspot_x = ((f32::from(payload.hotspot_x) / payload.scale).round() as i32) + .clamp(0, width.saturating_sub(1) as i32); + let hotspot_y = ((f32::from(payload.hotspot_y) / payload.scale).round() as i32) + .clamp(0, height.saturating_sub(1) as i32); + sdl2::mouse::Cursor::from_surface(surface, hotspot_x, hotspot_y) +} + +fn map_window_point_to_stream_viewport( + point: (i32, i32), + stream_size: (u32, u32), + window_size: (u32, u32), +) -> AbsoluteMouseViewport { + let viewport = aspect_fit( + stream_size.0.max(1), + stream_size.1.max(1), + window_size.0.max(1), + window_size.1.max(1), + ); + let width = viewport.width().max(1); + let height = viewport.height().max(1); + AbsoluteMouseViewport { + x: clamp_i32_u16((point.0 - viewport.x()).clamp(0, width.saturating_sub(1) as i32)), + y: clamp_i32_u16((point.1 - viewport.y()).clamp(0, height.saturating_sub(1) as i32)), + width: clamp_u32_u16(width), + height: clamp_u32_u16(height), + } +} + +fn parse_cursor_position(bytes: &[u8]) -> Option<(u16, u16)> { + if bytes.len() < 7 || !matches!(bytes[0], 0 | 1) { + return None; + } + let mime_length = usize::from(bytes[4]); + let image_length_offset = 5_usize.checked_add(mime_length)?; + let image_length_bytes = bytes.get(image_length_offset..image_length_offset + 2)?; + let image_length = usize::from(u16::from_le_bytes([ + image_length_bytes[0], + image_length_bytes[1], + ])); + let position_offset = image_length_offset + .checked_add(2)? + .checked_add(image_length)?; + let position = bytes.get(position_offset..position_offset + 4)?; + Some(( + u16::from_le_bytes([position[0], position[1]]), + u16::from_le_bytes([position[2], position[3]]), + )) +} + +fn normalized_cursor_coordinate(value: u16, viewport_extent: u32) -> i32 { + let extent = viewport_extent.max(1); + let coordinate = (u64::from(value) * u64::from(extent) / u64::from(u16::MAX)) as u32; + coordinate.min(extent.saturating_sub(1)) as i32 +} + +fn is_native_guide_shortcut( + scancode: sdl2::keyboard::Scancode, + keymod: sdl2::keyboard::Mod, +) -> bool { + scancode == sdl2::keyboard::Scancode::G && sdl_modifiers(scancode, keymod) == 0x02 +} + +pub(crate) fn native_input_capture_enabled() -> bool { + native_input_capture_enabled_value(std::env::var("OPENNOW_NATIVE_INPUT_OWNER").ok().as_deref()) +} + +fn native_input_capture_enabled_value(owner: Option<&str>) -> bool { + owner.is_some_and(|owner| owner.trim().eq_ignore_ascii_case("native")) +} + +pub(crate) enum ActiveOutput { + Software(Box), + #[cfg(target_os = "windows")] + Windows(Box), + #[cfg(target_os = "linux")] + LinuxHardware(Box), + #[cfg(target_os = "macos")] + Mac(Box), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OutputControl { + PointerLock, +} + +impl ActiveOutput { + #[cfg(target_os = "macos")] + pub(crate) fn is_macos_hardware(&self) -> bool { + matches!(self, Self::Mac(_)) + } + + #[cfg(target_os = "linux")] + pub(crate) fn is_linux_hardware(&self) -> bool { + matches!(self, Self::LinuxHardware(_)) + } + + #[cfg(target_os = "windows")] + pub(crate) fn is_windows_hardware(&self) -> bool { + matches!(self, Self::Windows(output) if output.is_hardware()) + } + + pub(crate) fn initialize( + output: Arc, + use_hardware: bool, + config: crate::media::MediaSessionConfig<'_>, + #[cfg(target_os = "windows")] windows_bridge: Arc, + use_linux_hardware: bool, + ) -> Result { + let crate::media::MediaSessionConfig { + stream, + audio_device, + } = config; + #[cfg(target_os = "macos")] + if use_hardware { + return crate::macos_backend::MacOutput::initialize( + stream, + audio_device, + Arc::clone(&output.audio_muted), + ) + .map(Box::new) + .map(Self::Mac); + } + #[cfg(target_os = "windows")] + if use_hardware || stream.codec != crate::media::MediaVideoCodec::H264 { + if audio_device.device_name().is_some() { + return Err("Fixed audio output selection requires the embedded Windows stream view or SDL software output".to_owned()); + } + let decoder_mode = if use_hardware { + WindowsDecoderMode::Hardware + } else { + WindowsDecoderMode::Software + }; + return WindowsOutput::initialize(windows_bridge, output, stream, decoder_mode) + .map(Box::new) + .map(Self::Windows); + } + #[cfg(target_os = "linux")] + if use_linux_hardware { + return LinuxHardwareOutput::initialize(output, stream, audio_device) + .map(Box::new) + .map(Self::LinuxHardware); + } + let _ = use_hardware; + let _ = use_linux_hardware; + SoftwareOutput::initialize(output, stream, audio_device) + .map(Box::new) + .map(Self::Software) + } + + pub(crate) fn start(&mut self, surface: Option<&RenderSurface>) -> Result<(), String> { + match self { + Self::Software(output) => output.start(surface), + #[cfg(target_os = "windows")] + Self::Windows(output) => output.start(surface), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.start(surface), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.start(surface), + } + } + + pub(crate) fn set_paused(&mut self, paused: bool) -> Result<(), String> { + match self { + Self::Software(output) => { + output.set_paused(paused); + Ok(()) + } + #[cfg(target_os = "windows")] + Self::Windows(output) => output.set_paused(paused), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => { + output.set_paused(paused); + Ok(()) + } + #[cfg(target_os = "macos")] + Self::Mac(output) => output.set_paused(paused), + } + } + + pub(crate) fn stop(&mut self) { + match self { + Self::Software(output) => output.stop(), + #[cfg(target_os = "windows")] + Self::Windows(output) => output.stop(), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.stop(), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.stop(), + } + } + + pub(crate) fn update_surface(&mut self, surface: &RenderSurface) -> Result<(), String> { + match self { + Self::Software(output) => output.update_surface(surface), + #[cfg(target_os = "windows")] + Self::Windows(output) => output.update_surface(surface), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.update_surface(surface), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.update_surface(surface), + } + } + + pub(crate) fn pump(&mut self) -> Result { + match self { + Self::Software(output) => { + let presented = output.pump()?; + Ok(if presented { + OutputEvent::Presented(output.backend_label()) + } else { + OutputEvent::None + }) + } + #[cfg(target_os = "windows")] + Self::Windows(output) => output.pump(), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.pump().map(|presented| { + if presented { + OutputEvent::Presented("Linux decoder/Vulkan") + } else { + OutputEvent::None + } + }), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.pump().map(|_| OutputEvent::None), + } + } + + pub(crate) fn take_captured_input(&mut self) -> Vec { + match self { + Self::Software(output) => output.take_captured_input(), + #[cfg(target_os = "windows")] + Self::Windows(output) => output.take_captured_input(), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.take_captured_input(), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.take_captured_input(), + } + } + + pub(crate) fn update_cursor(&mut self, bytes: &[u8]) { + match self { + Self::Software(output) => output.update_cursor(bytes), + #[cfg(target_os = "windows")] + Self::Windows(output) => output.update_cursor(bytes), + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => output.update_cursor(bytes), + #[cfg(target_os = "macos")] + Self::Mac(output) => output.update_cursor(bytes), + } + } + + pub(crate) fn control(&mut self, control: OutputControl) -> Result<(), String> { + match self { + Self::Software(output) => match control { + OutputControl::PointerLock => { + output + .input_capture + .toggle_pointer_lock(&output._sdl, output.canvas.window_mut()); + Ok(()) + } + }, + #[cfg(target_os = "windows")] + Self::Windows(output) => { + let Some(surface) = output.external_surface.as_mut() else { + return Err("Runtime controls require the external stream window".to_owned()); + }; + match control { + OutputControl::PointerLock => { + surface + .input_capture + .toggle_pointer_lock(&surface.sdl, &mut surface.window); + surface.sync_raw_input(); + Ok(()) + } + } + } + #[cfg(target_os = "linux")] + Self::LinuxHardware(output) => match control { + OutputControl::PointerLock => { + output + .input_capture + .toggle_pointer_lock(&output._sdl, &mut output.window); + Ok(()) + } + }, + #[cfg(target_os = "macos")] + Self::Mac(output) => output.control(control), + } + } + + #[cfg(target_os = "macos")] + pub(crate) fn configure_macos_h264( + &mut self, + parameter_sets: opennow_streamer_platform_macos::H264ParameterSets, + ) -> Result { + match self { + Self::Mac(output) => output.configure_h264(parameter_sets), + Self::Software(_) => Err("VideoToolbox is not the selected media backend".to_owned()), + } + } + + #[cfg(target_os = "macos")] + pub(crate) fn configure_macos_h265( + &mut self, + parameter_sets: opennow_streamer_platform_macos::H265ParameterSets, + ) -> Result { + match self { + Self::Mac(output) => output.configure_h265(parameter_sets), + Self::Software(_) => Err("VideoToolbox is not the selected media backend".to_owned()), + } + } + + #[cfg(target_os = "macos")] + pub(crate) fn configure_macos_av1( + &mut self, + format: opennow_streamer_platform_macos::Av1Format, + ) -> Result { + match self { + Self::Mac(output) => output.configure_av1(format), + Self::Software(_) => Err("VideoToolbox is not the selected media backend".to_owned()), + } + } +} + +pub(crate) enum OutputEvent { + None, + Presented(&'static str), + #[cfg(target_os = "windows")] + RequestKeyframe, + #[cfg(target_os = "windows")] + DeviceLost { + subsystem: &'static str, + recovered: bool, + message: Option, + }, + #[cfg(target_os = "windows")] + QueueDropped(&'static str), + #[cfg(target_os = "windows")] + Fatal(String), +} + +#[cfg(target_os = "windows")] +struct WindowsExternalSdlSurface { + sdl: sdl2::Sdl, + window: sdl2::video::Window, + event_pump: sdl2::EventPump, + native_surface: NativeSurface, + input_capture: SdlInputCapture, + raw_input: Option, + stream_size: (u32, u32), + visible: bool, + session_paused: bool, + input_suspended: bool, +} + +#[cfg(target_os = "windows")] +impl WindowsExternalSdlSurface { + fn initialize( + stream: MediaStreamConfig, + captured_input: Arc, + ) -> Result { + // Force the Windows RawInput path. Warp-relative motion is quantized by + // cursor recentering and Windows pointer scaling, which is particularly + // noticeable in 120 FPS first-person games. + sdl2::hint::set("SDL_WINDOWS_DPI_AWARENESS", "permonitorv2"); + sdl2::hint::set("SDL_WINDOWS_DPI_SCALING", "1"); + sdl2::hint::set("SDL_MOUSE_RELATIVE_MODE_WARP", "0"); + sdl2::hint::set("SDL_MOUSE_RELATIVE_SCALING", "0"); + let sdl = sdl2::init().map_err(|error| format!("SDL initialization failed: {error}"))?; + let video = sdl + .video() + .map_err(|error| format!("SDL video initialization failed: {error}"))?; + let mut window_builder = video.window("OpenNOW Stream", 1280, 720); + window_builder + .position_centered() + .resizable() + .borderless() + .hidden(); + let mut window = window_builder + .build() + .map_err(|error| format!("external SDL video window creation failed: {error}"))?; + let native_surface = NativeSurface::new(&window)?; + let event_pump = sdl + .event_pump() + .map_err(|error| format!("external SDL event pump creation failed: {error}"))?; + let capture_input = native_input_capture_enabled(); + let raw_input = if capture_input { + match WindowsRawInputController::start(native_surface.window_handle(), captured_input) { + Ok(controller) => { + eprintln!("Dedicated Windows Raw Input mouse thread ready"); + Some(controller) + } + Err(error) => { + eprintln!( + "Dedicated Windows Raw Input unavailable; retaining SDL motion: {error}" + ); + None + } + } + } else { + None + }; + eprintln!( + "External SDL stream window ready (native keyboard/mouse capture: {capture_input})" + ); + let external_relative_motion = raw_input.is_some(); + let mut input_capture = SdlInputCapture::new( + capture_input, + external_relative_motion, + raw_input.is_some(), + stream.shortcuts, + ); + input_capture.enable_gamepads(&sdl); + input_capture.set_input_paused(true, &sdl, &mut window); + Ok(Self { + sdl, + window, + event_pump, + native_surface, + input_capture, + raw_input, + stream_size: (stream.width, stream.height), + visible: false, + session_paused: false, + input_suspended: true, + }) + } + + fn target(&self) -> Result { + let handle = WindowHandle::new(self.native_surface.window_handle()) + .ok_or_else(|| "external SDL window returned an empty HWND".to_owned())?; + Ok(SurfaceTarget::Existing(ExistingWindow { hwnd: handle })) + } + + fn update(&mut self, surface: &RenderSurface) -> Result<(), String> { + let Some(rect) = surface.rect.filter(|_| surface.visible) else { + self.visible = false; + self.sync_input_ownership(); + self.native_surface.hide_checked()?; + return Ok(()); + }; + let parent_handle = surface.window_handle.as_deref().ok_or_else(|| { + "visible Windows stream surface is missing the Qt window handle".to_owned() + })?; + let foreground_owner = parse_windows_handle(parent_handle)?.get(); + self.native_surface.attach_and_show( + parent_handle, + rect, + surface.screen_rect, + surface.device_scale_factor, + )?; + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.set_foreground_owner(foreground_owner); + } + self.visible = true; + self.sync_input_ownership(); + Ok(()) + } + + fn pump(&mut self) { + let stream_window_id = self.window.id(); + for event in self.event_pump.poll_iter().collect::>() { + if matches!(event, sdl2::event::Event::Quit { .. }) { + if let Err(error) = self.release() { + eprintln!("{error}"); + } + continue; + } + let focus_lost = matches!( + event, + sdl2::event::Event::Window { + window_id, + win_event: sdl2::event::WindowEvent::FocusLost, + .. + } if window_id == stream_window_id + ); + self.input_capture + .handle_event(&self.sdl, &mut self.window, self.stream_size, event); + if focus_lost { + if let Some(raw_input) = self.raw_input.as_ref() { + raw_input.release_buttons(); + } + } + } + self.sync_raw_input(); + } + + fn release(&mut self) -> Result<(), String> { + self.visible = false; + self.sync_input_ownership(); + self.native_surface.hide_checked() + } + + fn set_paused(&mut self, paused: bool) { + self.session_paused = paused; + self.sync_input_ownership(); + } + + fn sync_input_ownership(&mut self) { + let suspended = self.session_paused || !self.visible; + if suspended == self.input_suspended { + return; + } + self.input_capture + .set_input_paused(suspended, &self.sdl, &mut self.window); + self.input_suspended = suspended; + self.sync_raw_input(); + } + + fn take_captured_input(&mut self) -> Vec { + self.input_capture.take() + } + + fn update_cursor(&mut self, bytes: &[u8]) { + self.input_capture + .apply_cursor(&self.sdl, &mut self.window, self.stream_size, bytes); + // SDL changes its process-wide Raw Input registration when relative mode changes. + // Reclaim it immediately so the first button after a server cursor transition cannot + // fall into an ownership gap. + self.sync_raw_input(); + } + + fn sync_raw_input(&self) { + if let Some(raw_input) = self.raw_input.as_ref() { + // Keep Raw Input registered for the whole visible session. It is the sole button + // and wheel owner in both cursor modes; foreground validation still happens on the + // Raw Input thread. Only motion switches between SDL absolute and raw relative. + raw_input.set_capture( + self.visible && self.input_capture.capture_enabled(), + self.input_capture.relative_mouse_enabled(), + ); + } + } +} + +#[cfg(target_os = "windows")] +pub(crate) struct WindowsOutput { + backend: Arc, + graphics_api: WindowsGraphicsApi, + decoder_mode: WindowsDecoderMode, + bridge: Arc, + output: Arc, + external_surface: Option, + stopped: bool, +} + +#[cfg(target_os = "windows")] +impl WindowsOutput { + fn is_hardware(&self) -> bool { + self.decoder_mode == WindowsDecoderMode::Hardware + } + + fn initialize( + bridge: Arc, + output: Arc, + stream: MediaStreamConfig, + decoder_mode: WindowsDecoderMode, + ) -> Result { + bridge.reset(); + let external_renderer = external_renderer_enabled(); + let graphics_api = if decoder_mode == WindowsDecoderMode::Software { + WindowsGraphicsApi::D3d11 + } else { + selected_windows_graphics_api(stream.codec) + }; + let external_surface = if external_renderer { + Some(WindowsExternalSdlSurface::initialize( + stream, + output.captured_input(), + )?) + } else { + None + }; + let initial_surface = match external_surface.as_ref() { + Some(surface) => surface.target()?, + None => hidden_windows_surface(), + }; + let backend = Arc::new( + WindowsBackend::start_for_mode( + graphics_api, + decoder_mode, + BackendConfig { + video: VideoFormat { + codec: match stream.codec { + crate::media::MediaVideoCodec::H264 => VideoCodec::H264, + crate::media::MediaVideoCodec::H265 => VideoCodec::H265, + crate::media::MediaVideoCodec::Av1 => VideoCodec::Av1, + }, + width: stream.width, + height: stream.height, + frame_rate_numerator: std::num::NonZeroU32::new(stream.fps.max(1)) + .expect("fps is clamped non-zero"), + frame_rate_denominator: std::num::NonZeroU32::new(1) + .expect("one is non-zero"), + average_bitrate: stream.bitrate_bps.max(1), + pixel_format: match ( + stream.color_quality.bit_depth(), + stream.color_quality.is_444(), + ) { + (10, true) => VideoPixelFormat::Y410, + (10, false) => VideoPixelFormat::P010, + (_, true) => VideoPixelFormat::Ayuv, + _ => VideoPixelFormat::Nv12, + }, + chroma_format: if stream.color_quality.is_444() { + VideoChromaFormat::Cs444 + } else { + VideoChromaFormat::Cs420 + }, + // Official GFN SDR sessions report limited-range BT.709. + full_range: false, + chroma_siting: opennow_streamer_platform_windows::VideoChromaSiting::Left, + transfer_function: + opennow_streamer_platform_windows::VideoTransferFunction::Sdr, + color_primaries: + opennow_streamer_platform_windows::VideoColorPrimaries::Bt709, + color_matrix: opennow_streamer_platform_windows::VideoColorMatrix::Bt709, + }, + audio: AudioFormat { + sample_rate: AUDIO_SAMPLE_RATE as u32, + channels: AUDIO_CHANNELS as u16, + }, + surface: initial_surface, + video_queue_capacity: + opennow_streamer_platform_windows::ADAPTIVE_VIDEO_QUEUE_CAPACITY, + // 10 x 20 ms PCM packets matches the adaptive renderer's + // maximum jitter-buffer target without becoming unbounded. + audio_queue_capacity: 10, + }, + Arc::clone(&output.audio_muted), + ) + .map_err(|error| error.to_string())?, + ); + bridge.replace_backend(Some(Arc::clone(&backend))); + Ok(Self { + backend, + graphics_api, + decoder_mode, + bridge, + output, + external_surface, + stopped: false, + }) + } + + fn start(&mut self, surface: Option<&RenderSurface>) -> Result<(), String> { + if let Some(surface) = surface { + self.update_surface(surface)?; + } + Ok(()) + } + + fn set_paused(&mut self, paused: bool) -> Result<(), String> { + if let Some(surface) = self.external_surface.as_mut() { + surface.set_paused(paused); + } + self.backend + .set_paused(paused) + .map_err(|error| error.to_string()) + } + + fn update_surface(&mut self, surface: &RenderSurface) -> Result<(), String> { + if let Some(external_surface) = self.external_surface.as_mut() { + external_surface.update(surface)?; + return self + .backend + .set_surface(external_surface.target()?) + .map_err(|error| error.to_string()); + } + self.backend + .set_surface(windows_surface(surface)?) + .map_err(|error| error.to_string()) + } + + fn pump(&mut self) -> Result { + if let Some(surface) = self.external_surface.as_mut() { + surface.pump(); + } + let Some(event) = self.backend.try_event() else { + return Ok(OutputEvent::None); + }; + Ok(match event { + BackendEvent::FirstFramePresented => { + OutputEvent::Presented(match (self.graphics_api, self.decoder_mode) { + (WindowsGraphicsApi::D3d12, WindowsDecoderMode::Hardware) => { + "Media Foundation hardware/D3D11-on-12/D3D12/WASAPI" + } + (WindowsGraphicsApi::D3d11, WindowsDecoderMode::Hardware) => { + "Media Foundation hardware/D3D11/WASAPI" + } + (_, WindowsDecoderMode::Software) => "Media Foundation software/D3D11/WASAPI", + }) + } + BackendEvent::KeyFrameRequired => { + self.bridge.require_keyframe(); + OutputEvent::RequestKeyframe + } + BackendEvent::DeviceLost { subsystem, message } => OutputEvent::DeviceLost { + subsystem: subsystem_label(subsystem), + recovered: false, + message: Some(message), + }, + BackendEvent::DeviceRecovered(subsystem) => OutputEvent::DeviceLost { + subsystem: subsystem_label(subsystem), + recovered: true, + message: None, + }, + BackendEvent::Fatal(error) => OutputEvent::Fatal(error.to_string()), + BackendEvent::QueueOverflow(subsystem) => { + if subsystem == Subsystem::VideoDecode { + self.bridge.require_keyframe(); + } + OutputEvent::QueueDropped(subsystem_label(subsystem)) + } + BackendEvent::StateChanged(_) | BackendEvent::VideoFormatChanged(_) => { + OutputEvent::None + } + }) + } + + fn stop(&mut self) { + if self.stopped { + return; + } + self.bridge.replace_backend(None); + if let Some(surface) = self.external_surface.as_mut() { + if let Err(error) = surface.release() { + eprintln!("{error}"); + } + let captured_input = self.output.captured_input(); + for input in surface.take_captured_input() { + captured_input.push(input); + } + } + self.backend.stop(); + self.stopped = true; + } + + fn take_captured_input(&mut self) -> Vec { + self.external_surface + .as_mut() + .map(WindowsExternalSdlSurface::take_captured_input) + .unwrap_or_default() + } + + fn update_cursor(&mut self, bytes: &[u8]) { + if let Some(surface) = self.external_surface.as_mut() { + surface.update_cursor(bytes); + } + } +} + +#[cfg(target_os = "windows")] +fn selected_windows_graphics_api(codec: crate::media::MediaVideoCodec) -> WindowsGraphicsApi { + let d3d12_allowed = crate::runtime::backend_preference_allows("d3d12"); + let d3d11_allowed = crate::runtime::backend_preference_allows("d3d11"); + select_windows_graphics_api( + codec, + d3d12_allowed, + d3d11_allowed, + WindowsBackend::probe_for(WindowsGraphicsApi::D3d12, None).bundled_backend_available(), + ) +} + +#[cfg(target_os = "windows")] +fn select_windows_graphics_api( + codec: crate::media::MediaVideoCodec, + d3d12_allowed: bool, + d3d11_allowed: bool, + d3d12_available: bool, +) -> WindowsGraphicsApi { + // Media Foundation exposes AV1 decode surfaces through D3D11. Under Auto, + // keep those surfaces on D3D11 instead of synchronizing and flushing them + // through D3D11-on-12 for every 120 Hz frame. An explicit D3D12 selection + // remains authoritative for diagnostics and user preference. + if d3d12_allowed + && (!d3d11_allowed || (codec != crate::media::MediaVideoCodec::Av1 && d3d12_available)) + { + WindowsGraphicsApi::D3d12 + } else { + WindowsGraphicsApi::D3d11 + } +} + +#[cfg(target_os = "windows")] +impl Drop for WindowsOutput { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(target_os = "windows")] +fn hidden_windows_surface() -> SurfaceTarget { + SurfaceTarget::Owned(OwnedWindow { + parent: None, + bounds: Bounds { + x: 0, + y: 0, + width: 2, + height: 2, + }, + visible: false, + }) +} + +#[cfg(target_os = "windows")] +fn windows_surface(surface: &RenderSurface) -> Result { + let Some(rect) = surface.rect.filter(|_| surface.visible) else { + return Ok(hidden_windows_surface()); + }; + let parent = surface + .window_handle + .as_deref() + .ok_or_else(|| "visible native surface is missing the shell window handle".to_owned())?; + let parent = parse_windows_handle(parent)?; + Ok(SurfaceTarget::Owned(OwnedWindow { + parent: Some(parent), + bounds: Bounds { + x: rect.x, + y: rect.y, + width: rect.width.max(2), + height: rect.height.max(2), + }, + visible: true, + })) +} + +pub(crate) fn external_renderer_enabled() -> bool { + external_renderer_enabled_value( + std::env::var("OPENNOW_NATIVE_EXTERNAL_RENDERER") + .ok() + .as_deref(), + ) +} + +fn native_cursor_overlay_enabled() -> bool { + std::env::var("OPENNOW_NATIVE_CURSOR_OVERLAY") + .ok() + .as_deref() + .is_none_or(|value| { + !matches!( + value.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "no" | "off" + ) + }) +} + +fn external_renderer_enabled_value(value: Option<&str>) -> bool { + value.is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +#[cfg(target_os = "windows")] +fn parse_windows_handle(value: &str) -> Result { + let trimmed = value.trim(); + let raw = if let Some(hex) = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + { + isize::from_str_radix(hex, 16) + } else { + trimmed.parse::() + } + .map_err(|error| format!("invalid shell HWND {value:?}: {error}"))?; + WindowHandle::new(raw).ok_or_else(|| "shell HWND must be non-zero".to_owned()) +} + +#[cfg(target_os = "windows")] +fn subsystem_label(subsystem: Subsystem) -> &'static str { + match subsystem { + Subsystem::VideoDecode => "video-decode", + Subsystem::VideoPresentation => "video-presentation", + Subsystem::Audio => "audio", + } +} + +fn aspect_fit(source_width: u32, source_height: u32, width: u32, height: u32) -> Rect { + let scale = (width as f64 / source_width as f64).min(height as f64 / source_height as f64); + let target_width = (source_width as f64 * scale).round() as u32; + let target_height = (source_height as f64 * scale).round() as u32; + Rect::new( + ((width - target_width) / 2) as i32, + ((height - target_height) / 2) as i32, + target_width, + target_height, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audio_selection_requires_one_exact_device_match() { + let devices = vec![PlaybackDevice { + id: "USB Speakers".to_owned(), + name: "USB Speakers".to_owned(), + }]; + assert_eq!( + select_playback_device(&AudioOutputDevice::default(), &[]).unwrap(), + None + ); + let selected = AudioOutputDevice::new("USB Speakers".to_owned()).unwrap(); + assert_eq!( + select_playback_device(&selected, &devices).unwrap(), + Some("USB Speakers") + ); + assert!(select_playback_device(&selected, &[]).is_err()); + assert!( + select_playback_device(&selected, &[devices[0].clone(), devices[0].clone()]).is_err() + ); + let wrong_case = AudioOutputDevice::new("usb speakers".to_owned()).unwrap(); + assert!(select_playback_device(&wrong_case, &devices).is_err()); + } + + #[test] + fn audio_enumeration_excludes_all_ambiguous_names() { + let device = |name: &str| PlaybackDevice { + id: name.to_owned(), + name: name.to_owned(), + }; + assert_eq!( + unique_playback_devices(vec![device("A"), device("B"), device("A")]), + vec![device("B")], + ); + } + + #[test] + #[ignore = "requires SDL_AUDIODRIVER=dummy in an isolated test process"] + fn selected_sdl_output_survives_enumeration_and_missing_device_open() { + let sdl = sdl2::init().expect("SDL"); + let audio = sdl.audio().expect("SDL audio"); + let output = Arc::new(OutputBuffers::new()); + let default = open_audio_playback(&audio, &output, &AudioOutputDevice::default()) + .expect("default output"); + drop(default); + let devices = audio_devices().expect("devices"); + let selected = + AudioOutputDevice::new(devices.first().expect("dummy output").id.clone()).unwrap(); + let playback = open_audio_playback(&audio, &output, &selected).expect("fixed output"); + playback.resume(); + assert_eq!(audio_devices().expect("enumeration while playing"), devices); + let missing = AudioOutputDevice::new("opennow-test-missing-output".to_owned()).unwrap(); + assert!(open_audio_playback(&audio, &output, &missing).is_err()); + assert_eq!(playback.status(), sdl2::audio::AudioStatus::Playing); + } + + #[cfg(target_os = "windows")] + #[test] + fn automatic_av1_uses_direct_d3d11_but_explicit_d3d12_is_preserved() { + use crate::media::MediaVideoCodec; + + assert_eq!( + select_windows_graphics_api(MediaVideoCodec::Av1, true, true, true), + WindowsGraphicsApi::D3d11 + ); + assert_eq!( + select_windows_graphics_api(MediaVideoCodec::Av1, true, false, true), + WindowsGraphicsApi::D3d12 + ); + assert_eq!( + select_windows_graphics_api(MediaVideoCodec::H265, true, true, true), + WindowsGraphicsApi::D3d12 + ); + } + + #[test] + fn playback_mute_drains_queued_audio_and_resumes_without_replay() { + let output = Arc::new(OutputBuffers::new()); + let mut callback = StreamAudioCallback { + output: Arc::clone(&output), + }; + let mut samples = [1.0; 4]; + output.push_audio(&[0.25, -0.25, 0.5, -0.5]); + output.audio_muted.store(true, Ordering::Release); + callback.callback(&mut samples); + assert_eq!(samples, [0.0; 4]); + assert!(!output.audio_samples_pending()); + output.clear(); + assert!(output.audio_muted.load(Ordering::Acquire)); + output.push_audio(&[0.75, -0.75]); + callback.callback(&mut samples); + assert_eq!(samples, [0.0; 4]); + output.audio_muted.store(false, Ordering::Release); + output.push_audio(&[0.125, -0.125]); + callback.callback(&mut samples); + assert_eq!(samples, [0.125, -0.125, 0.0, 0.0]); + } + + #[test] + fn audio_buffer_drops_oldest_samples() { + let output = OutputBuffers { + video: Mutex::new(None), + #[cfg(target_os = "linux")] + linux_video: Mutex::new(VecDeque::with_capacity(LINUX_VIDEO_QUEUE_CAPACITY)), + audio: Mutex::new(AudioPlayoutBuffer::new(4)), + audio_muted: Arc::new(AtomicBool::new(false)), + captured_input: Arc::new(CapturedInputQueue::default()), + microphone: Mutex::new(None), + microphone_clock: Mutex::new(Instant::now()), + }; + assert_eq!(output.push_audio(&[1.0, 2.0]), 0); + assert_eq!(output.push_audio(&[3.0, 4.0, 5.0, 6.0]), 2); + let mut values = [0.0; 4]; + output.fill_audio(&mut values); + assert_eq!(values, [3.0, 4.0, 5.0, 6.0]); + } + + #[test] + fn video_slot_keeps_only_the_latest_frame() { + let output = OutputBuffers::new(); + assert!(!output.replace_video(DecodedVideoFrame { + width: 1, + height: 1, + rgb: vec![1, 2, 3], + })); + assert!(output.replace_video(DecodedVideoFrame { + width: 2, + height: 1, + rgb: vec![4; 6], + })); + assert_eq!(output.take_video().expect("frame").width, 2); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_video_queue_smooths_bursts_and_bounds_recovery_latency() { + use opennow_streamer_platform_linux::StreamFormat; + + let frame = |timestamp_us| LinuxDecodedVideoFrame { + format: StreamFormat::video_default(2, 2).expect("format"), + planes: Vec::new(), + dmabuf: None, + vulkan: None, + timestamp_us, + }; + let output = OutputBuffers::new(); + for timestamp_us in 0..LINUX_VIDEO_QUEUE_CAPACITY as u64 { + assert!(!output.queue_linux_video(frame(timestamp_us))); + } + assert!(output.queue_linux_video(frame(LINUX_VIDEO_QUEUE_CAPACITY as u64))); + assert_eq!( + output + .take_linux_video() + .expect("oldest frame") + .frame + .timestamp_us, + 1 + ); + assert_eq!( + output + .take_linux_video() + .expect("next frame") + .frame + .timestamp_us, + 2 + ); + + let recovery = OutputBuffers::new(); + for timestamp_us in 10..12 { + assert!(!recovery.queue_linux_video(frame(timestamp_us))); + } + assert_eq!( + recovery + .take_linux_video_for_presentation(FrameSelectionPolicy::LatestReady, 0) + .expect("latest recovery frame") + .frame + .timestamp_us, + 11, + ); + assert!(recovery.take_linux_video().is_none()); + } + + #[test] + fn aspect_fit_letterboxes_without_stretching() { + assert_eq!( + aspect_fit(1920, 1080, 1000, 1000), + Rect::new(0, 218, 1000, 563) + ); + } + + #[test] + fn absolute_mouse_coordinates_exclude_letterbox_bars() { + assert_eq!( + map_window_point_to_stream_viewport((500, 500), (1920, 1080), (1000, 1000)), + AbsoluteMouseViewport { + x: 500, + y: 282, + width: 1000, + height: 563, + } + ); + assert_eq!( + map_window_point_to_stream_viewport((500, 0), (1920, 1080), (1000, 1000)), + AbsoluteMouseViewport { + x: 500, + y: 0, + width: 1000, + height: 563, + } + ); + } + + #[test] + fn cursor_channel_position_uses_normalized_stream_coordinates() { + let message = [0, 12, 0, 0, 0, 0, 0, 0x00, 0x80, 0xff, 0xff]; + assert_eq!(parse_cursor_position(&message), Some((32768, 65535))); + assert_eq!(normalized_cursor_coordinate(32768, 2560), 1280); + assert_eq!(normalized_cursor_coordinate(65535, 1440), 1439); + } + + #[test] + fn sdl_keys_map_to_windows_virtual_keys_and_gfn_modifiers() { + use sdl2::keyboard::{Mod, Scancode}; + + assert_eq!(sdl_virtual_key(Scancode::W), Some(0x57)); + assert_eq!(sdl_virtual_key(Scancode::Escape), Some(0x1b)); + assert_eq!(sdl_virtual_key(Scancode::LCtrl), Some(0xa2)); + assert_eq!( + sdl_modifiers(Scancode::W, Mod::LSHIFTMOD | Mod::LCTRLMOD), + 0x03 + ); + assert_eq!(sdl_modifiers(Scancode::LShift, Mod::LSHIFTMOD), 0); + } + + #[test] + fn ctrl_g_is_reserved_for_the_shell_stream_menu() { + use sdl2::keyboard::{Mod, Scancode}; + + assert!(is_native_guide_shortcut(Scancode::G, Mod::LCTRLMOD)); + assert!(is_native_guide_shortcut(Scancode::G, Mod::RCTRLMOD)); + assert!(!is_native_guide_shortcut(Scancode::G, Mod::NOMOD)); + assert!(!is_native_guide_shortcut( + Scancode::G, + Mod::LCTRLMOD | Mod::LSHIFTMOD + )); + assert!(!is_native_guide_shortcut(Scancode::F1, Mod::LCTRLMOD)); + } + + #[test] + fn adjacent_sdl_mouse_motion_is_preserved_and_clamped() { + let mut input = Vec::new(); + push_mouse_motion(&mut input, 10, -20); + push_mouse_motion(&mut input, i32::MAX, -30); + + assert_eq!( + input, + vec![ + CapturedInput::MouseMove { + delta_x: 10, + delta_y: -20, + }, + CapturedInput::MouseMove { + delta_x: i16::MAX, + delta_y: -30, + }, + ] + ); + } + + #[test] + fn native_input_capture_requires_explicit_ownership() { + assert!(!native_input_capture_enabled_value(None)); + assert!(!native_input_capture_enabled_value(Some("electron"))); + assert!(native_input_capture_enabled_value(Some(" native "))); + } + + #[test] + fn gamepad_sticks_use_the_official_radial_deadzone() { + assert_eq!(radial_deadzone(2_000, 2_000), (0, 0)); + let diagonal = radial_deadzone(16_384, 16_384); + assert_eq!(diagonal.0, diagonal.1); + assert!(diagonal.0 > 10_000); + assert_eq!(radial_deadzone(i16::MAX, 0), (i16::MAX, 0)); + } + + #[test] + fn native_input_capture_waits_for_the_first_server_cursor_mode() { + let capture = SdlInputCapture::new(true, false, false, StreamShortcutBindings::default()); + assert!(!capture.focused); + assert!(!capture.relative_mouse); + assert_eq!(capture.cursor_state, RemoteCursorState::Unknown); + } + + #[test] + fn only_predefined_cursor_zero_enters_relative_mouse_mode() { + assert_eq!(cursor_state_for_message(0, 0), RemoteCursorState::Hidden); + assert_eq!(cursor_state_for_message(0, 1), RemoteCursorState::Visible); + assert_eq!(cursor_state_for_message(1, 0), RemoteCursorState::Visible); + } + + #[test] + fn custom_cursor_parser_preserves_bitmap_hotspot_and_scale() { + let mime = b"image/png"; + let image = b"AAAA"; + let mut bytes = vec![1, 0, 3, 4, mime.len() as u8]; + bytes.extend_from_slice(mime); + bytes.extend_from_slice(&(image.len() as u16).to_le_bytes()); + bytes.extend_from_slice(image); + bytes.extend_from_slice(&10_u16.to_le_bytes()); + bytes.extend_from_slice(&20_u16.to_le_bytes()); + bytes.extend_from_slice(&150_u16.to_le_bytes()); + + let cursor = parse_custom_cursor_payload(&bytes).expect("custom cursor"); + assert_eq!(cursor.hotspot_x, 3); + assert_eq!(cursor.hotspot_y, 4); + assert_eq!(cursor.image_base64, image); + assert_eq!(cursor.scale, 1.5); + } + + #[test] + fn external_renderer_environment_values_are_explicit() { + assert!(external_renderer_enabled_value(Some("1"))); + assert!(external_renderer_enabled_value(Some(" TRUE "))); + assert!(!external_renderer_enabled_value(Some("0"))); + assert!(!external_renderer_enabled_value(None)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/queue.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/queue.rs new file mode 100644 index 000000000..89cd6a4a0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/queue.rs @@ -0,0 +1,107 @@ +use std::collections::VecDeque; +use std::sync::{Condvar, Mutex}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PushResult { + Queued, + DroppedOldest, + Closed, +} + +#[derive(Debug)] +struct QueueState { + values: VecDeque, + closed: bool, +} + +#[derive(Debug)] +pub(crate) struct BoundedQueue { + capacity: usize, + state: Mutex>, + ready: Condvar, +} + +impl BoundedQueue { + pub(crate) fn new(capacity: usize) -> Self { + assert!(capacity > 0, "bounded queue capacity must be non-zero"); + Self { + capacity, + state: Mutex::new(QueueState { + values: VecDeque::with_capacity(capacity), + closed: false, + }), + ready: Condvar::new(), + } + } + + pub(crate) fn push(&self, value: T) -> PushResult { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.closed { + return PushResult::Closed; + } + let result = if state.values.len() == self.capacity { + state.values.pop_front(); + PushResult::DroppedOldest + } else { + PushResult::Queued + }; + state.values.push_back(value); + self.ready.notify_one(); + result + } + + pub(crate) fn pop(&self) -> Option { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + loop { + if let Some(value) = state.values.pop_front() { + return Some(value); + } + if state.closed { + return None; + } + state = self + .ready + .wait(state) + .unwrap_or_else(|error| error.into_inner()); + } + } + + pub(crate) fn clear(&self) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .values + .clear(); + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.closed = true; + state.values.clear(); + self.ready.notify_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn drops_the_oldest_value_at_capacity() { + let queue = BoundedQueue::new(2); + assert_eq!(queue.push(1), PushResult::Queued); + assert_eq!(queue.push(2), PushResult::Queued); + assert_eq!(queue.push(3), PushResult::DroppedOldest); + assert_eq!(queue.pop(), Some(2)); + assert_eq!(queue.pop(), Some(3)); + } + + #[test] + fn close_wakes_and_rejects_producers() { + let queue = BoundedQueue::new(1); + queue.push(1); + queue.close(); + assert_eq!(queue.pop(), None); + assert_eq!(queue.push(2), PushResult::Closed); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/recording.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/recording.rs new file mode 100644 index 000000000..7fdecea74 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/recording.rs @@ -0,0 +1,1578 @@ +use std::fs::OpenOptions; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use opennow_streamer_protocol::{RecordingCompletion, RecordingCutReason}; +use oxideav_core::{ + CodecId, CodecParameters, Muxer, Packet, Rational, StreamInfo, TimeBase, WriteSeek, +}; +use oxideav_mkv::avc::annexb_to_avcc; +use oxideav_mkv::mux::MkvMuxer; +use scuffle_av1::{ObuHeader, ObuType, seq::SequenceHeaderObu}; + +use crate::media::{ + EncodedFrame, EncodedRecordingReceiver, MediaCodec, MediaStreamConfig, MediaVideoCodec, +}; + +const VIDEO_STREAM_INDEX: u32 = 0; +const AUDIO_STREAM_INDEX: u32 = 1; +const OPUS_SAMPLE_RATE: u32 = 48_000; +const OPUS_PRE_SKIP: u16 = 312; + +pub fn record_replay_matroska( + output_path: impl AsRef, + stream: MediaStreamConfig, + mut snapshot: crate::replay::ReplaySnapshot, + cancelled: &AtomicBool, +) -> Result { + static EXPORT_ID: AtomicU64 = AtomicU64::new(0); + let output_path = validate_output_path(output_path.as_ref())?; + let part_path = output_path.with_extension(format!( + "mkv.{}.{}.part", + std::process::id(), + EXPORT_ID.fetch_add(1, Ordering::Relaxed) + )); + if output_path.exists() || part_path.exists() { + return Err("clip output already exists".to_owned()); + } + let result = (|| { + let first = snapshot.frames.front().ok_or("replay buffer is empty")?; + let origin = first.time; + let channels = snapshot + .frames + .iter() + .find_map(|entry| match entry.frame.codec { + MediaCodec::Opus { channels } => Some(channels.clamp(1, 2)), + _ => None, + }) + .unwrap_or(2); + if cancelled.load(Ordering::Acquire) || snapshot.cancelled.load(Ordering::Acquire) { + return Err("clip export cancelled".to_owned()); + } + let mut active = start_muxer(&part_path, stream, channels, &first.frame)?; + for entry in &snapshot.frames { + if cancelled.load(Ordering::Acquire) || snapshot.cancelled.load(Ordering::Acquire) { + return Err("clip export cancelled".to_owned()); + } + if let Some(time) = entry.time.checked_sub(origin) { + write_frame_at(&mut active, entry.frame.clone(), Some(time))?; + } + } + active + .muxer + .write_trailer() + .map_err(|error| format!("failed to finalize clip: {error}"))?; + drop(active.muxer); + if cancelled.load(Ordering::Acquire) || snapshot.cancelled.load(Ordering::Acquire) { + return Err("clip export cancelled".to_owned()); + } + std::fs::hard_link(&part_path, &output_path) + .map_err(|error| format!("failed to publish clip: {error}"))?; + if cancelled.load(Ordering::Acquire) || snapshot.cancelled.load(Ordering::Acquire) { + let _ = std::fs::remove_file(&output_path); + return Err("clip export cancelled".to_owned()); + } + Ok(RecordingSummary { + path: output_path, + video_packets: active.video_packets, + audio_packets: active.audio_packets, + }) + })(); + snapshot.frames.clear(); + let _ = std::fs::remove_file(&part_path); + result +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecordingSummary { + pub path: PathBuf, + pub video_packets: u64, + pub audio_packets: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManualRecordingSummary { + pub media: RecordingSummary, + pub completion: RecordingCompletion, +} + +struct ActiveMuxer { + muxer: MkvMuxer, + video_codec: MediaVideoCodec, + video_clock: TrackClock, + audio_clock: TrackClock, + video_packets: u64, + audio_packets: u64, +} + +#[derive(Default)] +struct TrackClock { + base: Option, + last_pts: i64, +} + +impl TrackClock { + fn pts(&mut self, timestamp: u64) -> i64 { + let base = *self.base.get_or_insert(timestamp); + let delta = rtp_timestamp_delta(base, timestamp); + let pts = i64::try_from(delta).unwrap_or(i64::MAX); + self.last_pts = self.last_pts.max(pts); + self.last_pts + } +} + +pub fn record_matroska( + output_path: impl AsRef, + stream: MediaStreamConfig, + receiver: EncodedRecordingReceiver, +) -> Result { + record_matroska_with( + output_path.as_ref(), + stream, + receiver, + |path| { + OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map(|file| Box::new(file) as Box) + .map_err(|error| format!("failed to create recording: {error}")) + }, + |part, output| { + publish_manual_recording(part, output) + .map_err(|error| format!("failed to publish completed recording: {error}")) + }, + ) +} + +pub(crate) fn publish_manual_recording(part: &Path, output: &Path) -> std::io::Result<()> { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let part = CString::new(part.as_os_str().as_bytes())?; + let output = CString::new(output.as_os_str().as_bytes())?; + #[cfg(target_os = "linux")] + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + part.as_ptr(), + libc::AT_FDCWD, + output.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + #[cfg(target_os = "macos")] + let result = unsafe { libc::renamex_np(part.as_ptr(), output.as_ptr(), libc::RENAME_EXCL) }; + if result == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + } + #[cfg(target_os = "windows")] + { + use std::os::windows::ffi::OsStrExt; + let part = std::fs::canonicalize(part)?; + let output = std::fs::canonicalize(output.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "recording output has no directory", + ) + })?)? + .join(output.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "recording output has no filename", + ) + })?); + let part: Vec = part.as_os_str().encode_wide().chain(Some(0)).collect(); + let output: Vec = output.as_os_str().encode_wide().chain(Some(0)).collect(); + let result = unsafe { + windows_sys::Win32::Storage::FileSystem::MoveFileExW(part.as_ptr(), output.as_ptr(), 0) + }; + if result != 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + { + let _ = (part, output); + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "atomic recording publication is unavailable on this platform", + )) + } +} + +pub(crate) fn record_matroska_with( + output_path: &Path, + stream: MediaStreamConfig, + receiver: EncodedRecordingReceiver, + open: impl FnOnce(&Path) -> Result, String>, + publish: impl FnOnce(&Path, &Path) -> Result<(), String>, +) -> Result { + let output_path = validate_output_path(output_path)?; + let part_path = part_path_for(&output_path)?; + if output_path.exists() { + return Err(format!( + "recording output already exists: {}", + output_path.display() + )); + } + + let mut owns_partial = false; + let result = record_matroska_inner( + &part_path, + &output_path, + stream, + &receiver, + |path| { + let output = open(path)?; + owns_partial = true; + Ok(output) + }, + publish, + ); + if owns_partial && result.is_err() { + let _ = std::fs::remove_file(&part_path); + } + result +} + +fn record_matroska_inner( + part_path: &Path, + output_path: &Path, + stream: MediaStreamConfig, + receiver: &EncodedRecordingReceiver, + open: impl FnOnce(&Path) -> Result, String>, + publish: impl FnOnce(&Path, &Path) -> Result<(), String>, +) -> Result { + let mut active: Option = None; + let mut audio_channels = 2_u8; + let mut open = Some(open); + + let completion = loop { + let frame = match receiver.recv() { + Ok(frame) => frame, + Err(completion) => break completion, + }; + if !frame.contiguous { + if active.is_none() { + return Err(format!( + "recording stopped because the {} stream was discontinuous", + frame.mid + )); + } + break RecordingCompletion::Cut { + reason: RecordingCutReason::Discontinuity, + }; + } + + if let MediaCodec::Opus { channels } = frame.codec { + audio_channels = channels.clamp(1, 2); + } + + if active.is_none() { + if !frame.keyframe || !is_video_codec(&frame.codec, stream.codec) { + continue; + } + if !has_video_picture(&frame, stream.codec) { + return Err("recording keyframe did not contain a video picture".to_owned()); + } + active = Some(start_muxer_with( + part_path, + stream, + audio_channels, + &frame, + open.take().expect("recording output opens once"), + )?); + } + + write_frame(active.as_mut().expect("muxer was initialized"), frame)?; + }; + + let Some(mut active) = active else { + return Err("recording ended before a decodable video keyframe arrived".to_owned()); + }; + if active.video_packets == 0 { + return Err("recording ended without a video picture".to_owned()); + } + active + .muxer + .write_trailer() + .map_err(|error| format!("failed to finalize Matroska recording: {error}"))?; + let media = RecordingSummary { + path: output_path.to_owned(), + video_packets: active.video_packets, + audio_packets: active.audio_packets, + }; + drop(active); + publish(part_path, output_path)?; + Ok(ManualRecordingSummary { media, completion }) +} + +fn has_video_picture(frame: &EncodedFrame, codec: MediaVideoCodec) -> bool { + match codec { + MediaVideoCodec::H264 => { + let nals = split_annex_b(&frame.data); + nals.iter().any(|nal| nal.len() > 1 && nal[0] & 0x1f == 5) + && nals.iter().any(|nal| nal.len() >= 4 && nal[0] & 0x1f == 7) + && nals.iter().any(|nal| nal.len() > 1 && nal[0] & 0x1f == 8) + } + MediaVideoCodec::H265 => { + let nals = split_annex_b(&frame.data); + nals.iter() + .any(|nal| nal.len() > 2 && matches!((nal[0] >> 1) & 0x3f, 16..=21)) + && [32, 33, 34].into_iter().all(|kind| { + nals.iter() + .any(|nal| nal.len() > 2 && (nal[0] >> 1) & 0x3f == kind) + }) + } + MediaVideoCodec::Av1 => { + let mut cursor = std::io::Cursor::new(frame.data.as_ref()); + let mut frame_header = false; + let mut tiles = false; + while (cursor.position() as usize) < frame.data.len() { + let Ok(header) = ObuHeader::parse(&mut cursor) else { + return false; + }; + let Some(size) = header.size else { + return false; + }; + let Some(end) = cursor.position().checked_add(size) else { + return false; + }; + if end > frame.data.len() as u64 { + return false; + } + if size > 0 { + match header.obu_type { + ObuType::Frame => { + frame_header = true; + tiles = true; + } + ObuType::FrameHeader => frame_header = true, + ObuType::TileGroup => tiles = true, + _ => {} + } + } + cursor.set_position(end); + } + frame_header && tiles + } + } +} + +fn start_muxer( + path: &Path, + stream: MediaStreamConfig, + audio_channels: u8, + first_video: &EncodedFrame, +) -> Result { + start_muxer_with(path, stream, audio_channels, first_video, |path| { + OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map(|file| Box::new(file) as Box) + .map_err(|error| format!("failed to create recording: {error}")) + }) +} + +fn start_muxer_with( + path: &Path, + stream: MediaStreamConfig, + audio_channels: u8, + first_video: &EncodedFrame, + open: impl FnOnce(&Path) -> Result, String>, +) -> Result { + let (codec_id, codec_private) = match stream.codec { + MediaVideoCodec::H264 => { + let repacked = annexb_to_avcc(&first_video.data); + if repacked.config_record.is_empty() { + return Err( + "H.264 recording keyframe did not include SPS/PPS codec configuration" + .to_owned(), + ); + } + ("h264", repacked.config_record) + } + MediaVideoCodec::H265 => { + let repacked = annexb_to_hvcc(&first_video.data)?; + if repacked.config_record.is_empty() { + return Err( + "H.265 recording keyframe did not include VPS/SPS/PPS codec configuration" + .to_owned(), + ); + } + ("h265", repacked.config_record) + } + MediaVideoCodec::Av1 => ("av1", av1_codec_private(&first_video.data)?), + }; + + let mut video_params = CodecParameters::video(CodecId::new(codec_id)); + video_params.width = Some(stream.width); + video_params.height = Some(stream.height); + video_params.frame_rate = Some(Rational::new(i64::from(stream.fps.max(1)), 1)); + video_params.bit_rate = Some(u64::from(stream.bitrate_bps)); + video_params.extradata = codec_private; + let video_time_base = TimeBase::from_rate(first_video.clock_rate_hz.max(1)); + let video_stream = StreamInfo { + index: VIDEO_STREAM_INDEX, + time_base: video_time_base, + duration: None, + start_time: Some(0), + params: video_params, + }; + + let mut audio_params = CodecParameters::audio(CodecId::new("opus")); + audio_params.sample_rate = Some(OPUS_SAMPLE_RATE); + audio_params.channels = Some(u16::from(audio_channels)); + audio_params.extradata = opus_head(audio_channels); + let audio_stream = StreamInfo { + index: AUDIO_STREAM_INDEX, + time_base: TimeBase::from_rate(OPUS_SAMPLE_RATE), + duration: None, + start_time: Some(0), + params: audio_params, + }; + + let output = open(path)?; + let mut muxer = MkvMuxer::new_matroska(output, &[video_stream, audio_stream]) + .map_err(|error| format!("failed to configure Matroska recording: {error}"))?; + muxer + .write_header() + .map_err(|error| format!("failed to write Matroska header: {error}"))?; + Ok(ActiveMuxer { + muxer, + video_codec: stream.codec, + video_clock: TrackClock::default(), + audio_clock: TrackClock::default(), + video_packets: 0, + audio_packets: 0, + }) +} + +fn write_frame(active: &mut ActiveMuxer, frame: EncodedFrame) -> Result<(), String> { + write_frame_at(active, frame, None) +} + +fn write_frame_at( + active: &mut ActiveMuxer, + frame: EncodedFrame, + time: Option, +) -> Result<(), String> { + let aligned_pts = time.map(|time| { + (time.as_nanos() * u128::from(frame.clock_rate_hz) / 1_000_000_000).min(i64::MAX as u128) + as i64 + }); + let (stream_index, pts, data, keyframe) = match frame.codec { + MediaCodec::H264 if active.video_codec == MediaVideoCodec::H264 => { + let repacked = annexb_to_avcc(&frame.data); + if repacked.packetized.is_empty() { + return Ok(()); + } + ( + VIDEO_STREAM_INDEX, + aligned_pts.unwrap_or_else(|| active.video_clock.pts(frame.timestamp)), + repacked.packetized, + frame.keyframe, + ) + } + MediaCodec::H265 if active.video_codec == MediaVideoCodec::H265 => { + let repacked = annexb_to_hvcc(&frame.data)?; + if repacked.packetized.is_empty() { + return Ok(()); + } + ( + VIDEO_STREAM_INDEX, + aligned_pts.unwrap_or_else(|| active.video_clock.pts(frame.timestamp)), + repacked.packetized, + frame.keyframe, + ) + } + MediaCodec::Av1 if active.video_codec == MediaVideoCodec::Av1 => ( + VIDEO_STREAM_INDEX, + aligned_pts.unwrap_or_else(|| active.video_clock.pts(frame.timestamp)), + frame.data.to_vec(), + frame.keyframe, + ), + MediaCodec::Opus { .. } => ( + AUDIO_STREAM_INDEX, + aligned_pts.unwrap_or_else(|| active.audio_clock.pts(frame.timestamp)), + frame.data.to_vec(), + true, + ), + MediaCodec::Unsupported(_) | MediaCodec::H264 | MediaCodec::H265 | MediaCodec::Av1 => { + return Err("recording stream codec changed during the session".to_owned()); + } + }; + + let time_base = if stream_index == VIDEO_STREAM_INDEX { + TimeBase::from_rate(frame.clock_rate_hz.max(1)) + } else { + TimeBase::from_rate(OPUS_SAMPLE_RATE) + }; + let packet = Packet::new(stream_index, time_base, data) + .with_pts(pts) + .with_dts(pts) + .with_keyframe(keyframe); + active + .muxer + .write_packet(&packet) + .map_err(|error| format!("failed to write Matroska packet: {error}"))?; + if stream_index == VIDEO_STREAM_INDEX { + active.video_packets = active.video_packets.saturating_add(1); + } else { + active.audio_packets = active.audio_packets.saturating_add(1); + } + Ok(()) +} + +fn validate_output_path(path: &Path) -> Result { + if !path.is_absolute() { + return Err("recording output path must be absolute".to_owned()); + } + if path.extension().and_then(|value| value.to_str()) != Some("mkv") { + return Err("native stream recordings must use the .mkv extension".to_owned()); + } + let parent = path + .parent() + .ok_or_else(|| "recording output must have a parent directory".to_owned())?; + if !parent.is_dir() { + return Err("recording output directory does not exist".to_owned()); + } + Ok(path.to_owned()) +} + +fn part_path_for(path: &Path) -> Result { + let file_name = path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| "recording output file name must be valid UTF-8".to_owned())?; + Ok(path.with_file_name(format!(".{file_name}.part"))) +} + +fn is_video_codec(codec: &MediaCodec, expected: MediaVideoCodec) -> bool { + matches!( + (codec, expected), + (MediaCodec::H264, MediaVideoCodec::H264) + | (MediaCodec::H265, MediaVideoCodec::H265) + | (MediaCodec::Av1, MediaVideoCodec::Av1) + ) +} + +fn opus_head(channels: u8) -> Vec { + let mut out = Vec::with_capacity(19); + out.extend_from_slice(b"OpusHead"); + out.push(1); + out.push(channels.clamp(1, 2)); + out.extend_from_slice(&OPUS_PRE_SKIP.to_le_bytes()); + out.extend_from_slice(&OPUS_SAMPLE_RATE.to_le_bytes()); + out.extend_from_slice(&0_i16.to_le_bytes()); + out.push(0); + out +} + +fn rtp_timestamp_delta(base: u64, timestamp: u64) -> u64 { + if timestamp >= base { + return timestamp - base; + } + let base_low = base as u32; + let timestamp_low = timestamp as u32; + if base_low.wrapping_sub(timestamp_low) > (u32::MAX / 2) { + u64::from(timestamp_low.wrapping_sub(base_low)) + } else { + 0 + } +} + +pub(crate) fn av1_codec_private(temporal_unit: &[u8]) -> Result, String> { + let mut cursor = std::io::Cursor::new(temporal_unit); + while usize::try_from(cursor.position()).unwrap_or(usize::MAX) < temporal_unit.len() { + let obu_start = usize::try_from(cursor.position()) + .map_err(|_| "AV1 OBU position is out of range".to_owned())?; + let header = ObuHeader::parse(&mut cursor) + .map_err(|error| format!("invalid AV1 OBU header: {error}"))?; + let payload_start = usize::try_from(cursor.position()) + .map_err(|_| "AV1 OBU position is out of range".to_owned())?; + let payload_length = header + .size + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(|| "AV1 sequence-header OBU must carry an explicit size".to_owned())?; + let payload_end = payload_start + .checked_add(payload_length) + .filter(|end| *end <= temporal_unit.len()) + .ok_or_else(|| "AV1 OBU payload is truncated".to_owned())?; + if header.obu_type == ObuType::SequenceHeader { + let sequence = SequenceHeaderObu::parse( + header, + &mut std::io::Cursor::new(&temporal_unit[payload_start..payload_end]), + ) + .map_err(|error| format!("invalid AV1 sequence header: {error}"))?; + let operating_point = sequence + .operating_points + .first() + .ok_or_else(|| "AV1 sequence header has no operating point".to_owned())?; + let mut config = Vec::with_capacity(4 + payload_end - obu_start); + config.push(0x81); + config.push((sequence.seq_profile << 5) | (operating_point.seq_level_idx & 0x1f)); + config.push( + (u8::from(operating_point.seq_tier) << 7) + | (u8::from(sequence.color_config.bit_depth > 8) << 6) + | (u8::from(sequence.color_config.bit_depth == 12) << 5) + | (u8::from(sequence.color_config.mono_chrome) << 4) + | (u8::from(sequence.color_config.subsampling_x) << 3) + | (u8::from(sequence.color_config.subsampling_y) << 2) + | (sequence.color_config.chroma_sample_position & 0x03), + ); + config.push(0); + config.extend_from_slice(&temporal_unit[obu_start..payload_end]); + return Ok(config); + } + cursor.set_position( + u64::try_from(payload_end) + .map_err(|_| "AV1 OBU position is out of range".to_owned())?, + ); + } + Err("AV1 recording keyframe did not include a sequence-header OBU".to_owned()) +} + +struct HvccRepack { + config_record: Vec, + packetized: Vec, +} + +#[derive(Clone, Copy)] +struct HevcProfile { + profile_space: u8, + tier_flag: bool, + profile_idc: u8, + compatibility_flags: u32, + constraint_flags: [u8; 6], + level_idc: u8, + max_sub_layers_minus_one: u8, + temporal_id_nested: bool, + chroma_format_idc: u8, + bit_depth_luma_minus_eight: u8, + bit_depth_chroma_minus_eight: u8, +} + +fn annexb_to_hvcc(stream: &[u8]) -> Result { + let mut vps = Vec::new(); + let mut sps = Vec::new(); + let mut pps = Vec::new(); + let mut packetized = Vec::with_capacity(stream.len()); + for nal in split_annex_b(stream) { + if nal.len() < 2 { + continue; + } + match (nal[0] >> 1) & 0x3f { + 32 => push_unique(&mut vps, nal), + 33 => push_unique(&mut sps, nal), + 34 => push_unique(&mut pps, nal), + _ => { + let length = u32::try_from(nal.len()) + .map_err(|_| "HEVC NAL unit exceeds the Matroska packet limit".to_owned())?; + packetized.extend_from_slice(&length.to_be_bytes()); + packetized.extend_from_slice(nal); + } + } + } + if sps.is_empty() || pps.is_empty() { + if packetized.is_empty() { + return Ok(HvccRepack { + config_record: Vec::new(), + packetized, + }); + } + return Ok(HvccRepack { + config_record: Vec::new(), + packetized, + }); + } + let profile = parse_hevc_sps(sps[0])?; + let config_record = build_hvcc(profile, &[(&vps, 32), (&sps, 33), (&pps, 34)])?; + Ok(HvccRepack { + config_record, + packetized, + }) +} + +fn push_unique<'a>(target: &mut Vec<&'a [u8]>, nal: &'a [u8]) { + if !target.contains(&nal) { + target.push(nal); + } +} + +fn build_hvcc(profile: HevcProfile, arrays: &[(&Vec<&[u8]>, u8)]) -> Result, String> { + let populated = arrays.iter().filter(|(nals, _)| !nals.is_empty()).count(); + let mut out = Vec::new(); + out.push(1); + out.push( + (profile.profile_space << 6) + | (u8::from(profile.tier_flag) << 5) + | (profile.profile_idc & 0x1f), + ); + out.extend_from_slice(&profile.compatibility_flags.to_be_bytes()); + out.extend_from_slice(&profile.constraint_flags); + out.push(profile.level_idc); + out.extend_from_slice(&0xf000_u16.to_be_bytes()); + out.push(0xfc); + out.push(0xfc | (profile.chroma_format_idc & 0x03)); + out.push(0xf8 | (profile.bit_depth_luma_minus_eight & 0x07)); + out.push(0xf8 | (profile.bit_depth_chroma_minus_eight & 0x07)); + out.extend_from_slice(&0_u16.to_be_bytes()); + out.push( + ((profile.max_sub_layers_minus_one.saturating_add(1) & 0x07) << 3) + | (u8::from(profile.temporal_id_nested) << 2) + | 0x03, + ); + out.push(u8::try_from(populated).map_err(|_| "too many HEVC parameter arrays".to_owned())?); + for (nals, nal_type) in arrays.iter().filter(|(nals, _)| !nals.is_empty()) { + out.push(0x80 | (*nal_type & 0x3f)); + out.extend_from_slice( + &u16::try_from(nals.len()) + .map_err(|_| "too many HEVC parameter sets".to_owned())? + .to_be_bytes(), + ); + for nal in *nals { + out.extend_from_slice( + &u16::try_from(nal.len()) + .map_err(|_| "HEVC parameter set is too large".to_owned())? + .to_be_bytes(), + ); + out.extend_from_slice(nal); + } + } + Ok(out) +} + +fn parse_hevc_sps(nal: &[u8]) -> Result { + if nal.len() < 4 { + return Err("HEVC SPS is truncated".to_owned()); + } + let rbsp = remove_emulation_prevention(&nal[2..]); + let mut bits = BitReader::new(&rbsp); + bits.skip(4)?; + let max_sub_layers_minus_one = bits.read(3)? as u8; + let temporal_id_nested = bits.read(1)? != 0; + let profile_space = bits.read(2)? as u8; + let tier_flag = bits.read(1)? != 0; + let profile_idc = bits.read(5)? as u8; + let compatibility_flags = bits.read(32)? as u32; + let mut constraint_flags = [0_u8; 6]; + for value in &mut constraint_flags { + *value = bits.read(8)? as u8; + } + let level_idc = bits.read(8)? as u8; + let mut sub_layer_profile_present = [false; 8]; + let mut sub_layer_level_present = [false; 8]; + for index in 0..usize::from(max_sub_layers_minus_one) { + sub_layer_profile_present[index] = bits.read(1)? != 0; + sub_layer_level_present[index] = bits.read(1)? != 0; + } + if max_sub_layers_minus_one > 0 { + for _ in max_sub_layers_minus_one..8 { + bits.skip(2)?; + } + } + for index in 0..usize::from(max_sub_layers_minus_one) { + if sub_layer_profile_present[index] { + bits.skip(88)?; + } + if sub_layer_level_present[index] { + bits.skip(8)?; + } + } + let _sps_id = bits.read_ue()?; + let chroma_format_idc = + u8::try_from(bits.read_ue()?).map_err(|_| "invalid HEVC chroma format".to_owned())?; + if chroma_format_idc == 3 { + bits.skip(1)?; + } + let _width = bits.read_ue()?; + let _height = bits.read_ue()?; + if bits.read(1)? != 0 { + for _ in 0..4 { + let _ = bits.read_ue()?; + } + } + let bit_depth_luma_minus_eight = + u8::try_from(bits.read_ue()?).map_err(|_| "invalid HEVC luma bit depth".to_owned())?; + let bit_depth_chroma_minus_eight = + u8::try_from(bits.read_ue()?).map_err(|_| "invalid HEVC chroma bit depth".to_owned())?; + Ok(HevcProfile { + profile_space, + tier_flag, + profile_idc, + compatibility_flags, + constraint_flags, + level_idc, + max_sub_layers_minus_one, + temporal_id_nested, + chroma_format_idc, + bit_depth_luma_minus_eight, + bit_depth_chroma_minus_eight, + }) +} + +fn remove_emulation_prevention(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(bytes.len()); + let mut zeroes = 0_u8; + for &value in bytes { + if zeroes >= 2 && value == 3 { + zeroes = 0; + continue; + } + out.push(value); + if value == 0 { + zeroes = zeroes.saturating_add(1); + } else { + zeroes = 0; + } + } + out +} + +struct BitReader<'a> { + bytes: &'a [u8], + bit: usize, +} + +impl<'a> BitReader<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, bit: 0 } + } + + fn read(&mut self, count: usize) -> Result { + if count > 64 || self.bit.saturating_add(count) > self.bytes.len().saturating_mul(8) { + return Err("HEVC SPS bitstream is truncated".to_owned()); + } + let mut value = 0_u64; + for _ in 0..count { + let byte = self.bytes[self.bit / 8]; + let shift = 7 - (self.bit % 8); + value = (value << 1) | u64::from((byte >> shift) & 1); + self.bit += 1; + } + Ok(value) + } + + fn skip(&mut self, count: usize) -> Result<(), String> { + self.read(count).map(|_| ()) + } + + fn read_ue(&mut self) -> Result { + let mut leading_zeroes = 0_usize; + while self.read(1)? == 0 { + leading_zeroes += 1; + if leading_zeroes > 31 { + return Err("HEVC SPS Exp-Golomb value is too large".to_owned()); + } + } + if leading_zeroes == 0 { + return Ok(0); + } + Ok(((1_u64 << leading_zeroes) - 1) + self.read(leading_zeroes)?) + } +} + +fn split_annex_b(data: &[u8]) -> Vec<&[u8]> { + let mut result = Vec::new(); + let mut cursor = 0_usize; + while let Some((start, prefix)) = find_start_code(&data[cursor..]) { + let nal_start = cursor + start + prefix; + let next = find_start_code(&data[nal_start..]) + .map(|(offset, _)| nal_start + offset) + .unwrap_or(data.len()); + let mut nal_end = next; + while nal_end > nal_start && data[nal_end - 1] == 0 { + nal_end -= 1; + } + if nal_end > nal_start { + result.push(&data[nal_start..nal_end]); + } + cursor = next; + if cursor >= data.len() { + break; + } + } + result +} + +fn find_start_code(data: &[u8]) -> Option<(usize, usize)> { + let mut index = 0_usize; + while index + 2 < data.len() { + if data[index] == 0 && data[index + 1] == 0 { + if data[index + 2] == 1 { + return Some((index, 3)); + } + if index + 3 < data.len() && data[index + 2] == 0 && data[index + 3] == 1 { + return Some((index, 4)); + } + } + index += 1; + } + None +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::mpsc::channel; + use std::time::{SystemTime, UNIX_EPOCH}; + + use oxideav_core::{NullCodecResolver, ReadSeek}; + + use super::*; + + #[test] + fn replay_exports_source_packets_with_shared_audio_offset_and_no_clobber() { + let directory = std::env::temp_dir().join(format!( + "opennow-replay-test-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + let output = directory.join("clip.mkv"); + let tap = crate::replay::ReplayTap::default(); + tap.start( + opennow_streamer_protocol::ReplayBufferConfig::from_settings( + &serde_json::json!({"replayBufferEnabled":true}), + ), + ); + let video = frame(MediaCodec::H264, h264_keyframe(), 9_000_000, true); + let audio = frame( + MediaCodec::Opus { channels: 2 }, + vec![0x80, 1, 2, 3], + 400_000, + false, + ); + tap.publish(&video); + tap.publish(&audio); + tap.publish(&frame( + MediaCodec::H264, + vec![0, 0, 0, 1, 0x41, 0x9a, 0x22], + 9_018_000, + false, + )); + let mut snapshot = tap.snapshot().unwrap(); + snapshot.frames[0].time = Duration::from_secs(4); + snapshot.frames[1].time = Duration::from_millis(4_100); + snapshot.frames[2].time = Duration::from_millis(4_200); + assert_eq!(snapshot.frames[1].frame.timestamp, 400_000); + assert!(Arc::ptr_eq(&snapshot.frames[0].frame.data, &video.data)); + let summary = record_replay_matroska( + &output, + MediaStreamConfig::default(), + snapshot, + &AtomicBool::new(false), + ) + .unwrap(); + assert_eq!((summary.video_packets, summary.audio_packets), (2, 1)); + let file: Box = Box::new(std::fs::File::open(&output).unwrap()); + let mut demuxer = oxideav_mkv::demux::open(file, &NullCodecResolver).unwrap(); + let mut packets = Vec::new(); + while let Ok(packet) = demuxer.next_packet() { + packets.push(packet); + } + assert_eq!(packets.len(), 3); + assert_eq!(packets[0].pts, Some(0)); + assert_eq!(packets[1].pts, Some(100)); + assert_eq!(packets[2].pts, Some(200)); + assert_eq!(packets[1].data.as_slice(), audio.data.as_ref()); + let original = std::fs::read(&output).unwrap(); + tap.publish(&frame(MediaCodec::H264, h264_keyframe(), 9_036_000, true)); + assert!( + record_replay_matroska( + &output, + MediaStreamConfig::default(), + tap.snapshot().unwrap(), + &AtomicBool::new(false) + ) + .is_err() + ); + assert_eq!(std::fs::read(&output).unwrap(), original); + tap.publish(&frame(MediaCodec::H264, h264_keyframe(), 9_054_000, true)); + let cancelled = tap.snapshot().unwrap(); + tap.stop(); + let cancelled_path = directory.join("cancelled.mkv"); + assert!( + record_replay_matroska( + &cancelled_path, + MediaStreamConfig::default(), + cancelled, + &AtomicBool::new(false) + ) + .is_err() + ); + assert!(!cancelled_path.exists()); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 1); + std::fs::remove_dir_all(directory).unwrap(); + } + + fn frame(codec: MediaCodec, data: Vec, timestamp: u64, keyframe: bool) -> EncodedFrame { + let is_audio = matches!(codec, MediaCodec::Opus { .. }); + EncodedFrame { + mid: if is_audio { "audio" } else { "video" }.to_owned(), + codec, + data: Arc::from(data), + frame_index: (!is_audio).then_some(1), + timestamp, + clock_rate_hz: if is_audio { 48_000 } else { 90_000 }, + keyframe, + contiguous: true, + ssrc: None, + } + } + + fn h264_keyframe() -> Vec { + let mut out = Vec::new(); + for nal in [ + &[0x67, 0x64, 0x00, 0x28, 0xde, 0xad][..], + &[0x68, 0xee, 0x3c, 0x80][..], + &[0x65, 0x88, 0x84, 0x00, 0x10][..], + ] { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(nal); + } + out + } + + fn real_h264_keyframe() -> Vec { + use openh264::encoder::Encoder; + use openh264::formats::{RgbSliceU8, YUVBuffer}; + + let rgb = vec![64_u8; 64 * 64 * 3]; + let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&rgb, (64, 64))); + Encoder::new().unwrap().encode(&yuv).unwrap().to_vec() + } + + fn recording_test_directory(name: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "opennow-{name}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + directory + } + + #[test] + fn discontinuity_saves_only_the_committed_video_and_audio_prefix() { + let directory = recording_test_directory("recording-prefix"); + let output = directory.join("cut.mkv"); + let (sender, receiver) = channel(); + let video = frame(MediaCodec::H264, real_h264_keyframe(), 90_000, true); + let mut encoder = opus::Encoder::new( + OPUS_SAMPLE_RATE, + opus::Channels::Stereo, + opus::Application::Audio, + ) + .unwrap(); + let mut packet = [0_u8; 1275]; + let samples: Vec = (0..960) + .flat_map(|index| { + let sample = (index as f32 * std::f32::consts::TAU * 440.0 / 48_000.0).sin() * 0.1; + [sample, sample] + }) + .collect(); + let length = encoder.encode_float(&samples, &mut packet).unwrap(); + let audio = frame( + MediaCodec::Opus { channels: 2 }, + packet[..length].to_vec(), + 48_000, + false, + ); + sender.send(video.clone()).unwrap(); + sender.send(audio.clone()).unwrap(); + let mut broken = frame( + MediaCodec::H264, + vec![0, 0, 0, 1, 0x41, 0xff], + 93_000, + false, + ); + broken.contiguous = false; + sender.send(broken).unwrap(); + sender + .send(frame(MediaCodec::H264, real_h264_keyframe(), 96_000, true)) + .unwrap(); + drop(sender); + let result = record_matroska( + &output, + MediaStreamConfig { + width: 64, + height: 64, + ..MediaStreamConfig::default() + }, + EncodedRecordingReceiver::from_receiver(receiver), + ); + let summary = result.expect("a valid prefix must survive the first discontinuity"); + assert_eq!( + (summary.media.video_packets, summary.media.audio_packets), + (1, 1) + ); + assert_eq!( + summary.completion, + RecordingCompletion::Cut { + reason: RecordingCutReason::Discontinuity + } + ); + assert!(!part_path_for(&output).unwrap().exists()); + let file: Box = Box::new(std::fs::File::open(&output).unwrap()); + let mut demuxer = oxideav_mkv::demux::open(file, &NullCodecResolver).unwrap(); + let saved_video = demuxer.next_packet().unwrap(); + let saved_audio = demuxer.next_packet().unwrap(); + assert_eq!( + saved_video.data.as_slice(), + annexb_to_avcc(&video.data).packetized + ); + assert_eq!(saved_audio.data.as_slice(), audio.data.as_ref()); + assert!(demuxer.next_packet().is_err()); + let mut decoder = opus::Decoder::new(OPUS_SAMPLE_RATE, opus::Channels::Stereo).unwrap(); + let mut decoded = [0_f32; 1920]; + assert_eq!( + decoder + .decode_float(&saved_audio.data, &mut decoded, false) + .unwrap(), + 960 + ); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn create_failure_does_not_remove_an_existing_partial_file() { + let directory = recording_test_directory("recording-owned-partial"); + let output = directory.join("existing.mkv"); + let partial = part_path_for(&output).unwrap(); + std::fs::write(&partial, b"another writer owns this file").unwrap(); + let (sender, receiver) = channel(); + sender + .send(frame(MediaCodec::H264, real_h264_keyframe(), 90_000, true)) + .unwrap(); + drop(sender); + assert!( + record_matroska( + &output, + MediaStreamConfig::default(), + EncodedRecordingReceiver::from_receiver(receiver), + ) + .is_err() + ); + assert_eq!( + std::fs::read(&partial).unwrap(), + b"another writer owns this file" + ); + assert!(!output.exists()); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn empty_missing_and_configuration_only_prefixes_are_not_saved() { + let directory = recording_test_directory("recording-empty-prefix"); + let mut broken = frame(MediaCodec::H264, real_h264_keyframe(), 90_000, true); + broken.contiguous = false; + let config_only = split_annex_b(&real_h264_keyframe()) + .into_iter() + .filter(|nal| matches!(nal[0] & 0x1f, 7 | 8)) + .flat_map(|nal| [0, 0, 0, 1].into_iter().chain(nal.iter().copied())) + .collect(); + for (index, frames) in [ + vec![], + vec![broken], + vec![frame( + MediaCodec::H264, + vec![0, 0, 1, 0x41, 1], + 90_000, + false, + )], + vec![frame(MediaCodec::H264, vec![], 90_000, true)], + vec![frame(MediaCodec::H264, config_only, 90_000, true)], + vec![frame( + MediaCodec::H264, + vec![0, 0, 1, 0x65, 1], + 90_000, + true, + )], + ] + .into_iter() + .enumerate() + { + let output = directory.join(format!("{index}.mkv")); + let (sender, receiver) = channel(); + for frame in frames { + sender.send(frame).unwrap(); + } + drop(sender); + assert!( + record_matroska( + &output, + MediaStreamConfig::default(), + EncodedRecordingReceiver::from_receiver(receiver) + ) + .is_err() + ); + assert!(!output.exists()); + assert!(!part_path_for(&output).unwrap().exists()); + } + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn hevc_and_av1_configuration_without_a_picture_cannot_be_saved() { + let directory = recording_test_directory("recording-codec-empty"); + let sps = synthetic_hevc_sps(); + let hevc: Vec = [&[0x40, 1, 0x0c][..], sps.as_slice(), &[0x44, 1, 0xc0][..]] + .into_iter() + .flat_map(|nal| [0, 0, 0, 1].into_iter().chain(nal.iter().copied())) + .collect(); + for (codec, media, bytes, name) in [ + (MediaVideoCodec::H265, MediaCodec::H265, hevc, "hevc"), + ( + MediaVideoCodec::Av1, + MediaCodec::Av1, + b"\x0a\x0f\0\0\0j\xef\xbf\xe1\xbc\x02\x19\x90\x10\x10\x10@".to_vec(), + "av1", + ), + ] { + let output = directory.join(format!("{name}.mkv")); + let (sender, receiver) = channel(); + sender.send(frame(media, bytes, 90_000, true)).unwrap(); + drop(sender); + assert!( + record_matroska( + &output, + MediaStreamConfig { + codec, + ..MediaStreamConfig::default() + }, + EncodedRecordingReceiver::from_receiver(receiver) + ) + .is_err() + ); + assert!(!output.exists()); + assert!(!part_path_for(&output).unwrap().exists()); + } + std::fs::remove_dir_all(directory).unwrap(); + } + + struct FailingFile { + file: std::fs::File, + payload: Vec, + packets: usize, + fail_packet: bool, + fail_flush: bool, + failed: Arc, + } + + impl std::io::Write for FailingFile { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.ends_with(&self.payload) { + self.packets += 1; + if self.fail_packet && self.packets == 2 { + self.failed.store(true, Ordering::Release); + return Err(std::io::Error::other("injected packet write failure")); + } + } + std::io::Write::write(&mut self.file, bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + if self.fail_flush { + self.failed.store(true, Ordering::Release); + return Err(std::io::Error::other("injected trailer flush failure")); + } + std::io::Write::flush(&mut self.file) + } + } + + impl std::io::Seek for FailingFile { + fn seek(&mut self, position: std::io::SeekFrom) -> std::io::Result { + std::io::Seek::seek(&mut self.file, position) + } + } + + #[test] + fn packet_trailer_and_publication_failures_never_publish_a_cut() { + let directory = recording_test_directory("recording-io-failures"); + let video = real_h264_keyframe(); + let payload = annexb_to_avcc(&video).packetized; + for failure in ["packet", "trailer", "publication"] { + let output = directory.join(format!("{failure}.mkv")); + let (sender, receiver) = channel(); + sender + .send(frame(MediaCodec::H264, video.clone(), 90_000, true)) + .unwrap(); + sender + .send(frame(MediaCodec::H264, video.clone(), 93_000, true)) + .unwrap(); + let mut broken = frame(MediaCodec::H264, vec![], 96_000, false); + broken.contiguous = false; + sender.send(broken).unwrap(); + drop(sender); + let failed = Arc::new(AtomicBool::new(false)); + let published = AtomicBool::new(false); + let result = record_matroska_with( + &output, + MediaStreamConfig::default(), + EncodedRecordingReceiver::from_receiver(receiver), + |path| { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .unwrap(); + Ok(Box::new(FailingFile { + file, + payload: payload.clone(), + packets: 0, + fail_packet: failure == "packet", + fail_flush: failure == "trailer", + failed: Arc::clone(&failed), + })) + }, + |_, _| { + published.store(true, Ordering::Release); + Err("injected publication failure".to_owned()) + }, + ); + let error = result.unwrap_err(); + assert!(error.contains(failure), "{failure}: {error}"); + assert_eq!(failed.load(Ordering::Acquire), failure != "publication"); + assert_eq!(published.load(Ordering::Acquire), failure == "publication"); + assert!(!output.exists()); + assert!(!part_path_for(&output).unwrap().exists()); + } + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn manual_publication_does_not_overwrite_a_destination_created_during_recording() { + let directory = recording_test_directory("recording-publish-race"); + let output = directory.join("existing.mkv"); + let (sender, receiver) = channel(); + sender + .send(frame(MediaCodec::H264, real_h264_keyframe(), 90_000, true)) + .unwrap(); + drop(sender); + let result = record_matroska_with( + &output, + MediaStreamConfig::default(), + EncodedRecordingReceiver::from_receiver(receiver), + |path| { + std::fs::write(&output, b"existing destination").unwrap(); + Ok(Box::new( + OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .unwrap(), + )) + }, + |part, output| { + publish_manual_recording(part, output).map_err(|error| error.to_string()) + }, + ); + assert!(result.is_err()); + assert_eq!(std::fs::read(&output).unwrap(), b"existing destination"); + assert!(!part_path_for(&output).unwrap().exists()); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[cfg(target_os = "windows")] + #[test] + fn windows_manual_publication_preserves_long_unicode_paths_and_existing_targets() { + let directory = recording_test_directory("recording-long-path"); + let mut nested = directory.clone(); + for _ in 0..8 { + nested.push("recording-長い名前-abcdefghijklmnopqrstuvwxyz"); + } + std::fs::create_dir_all(&nested).unwrap(); + let part = nested.join(".capture.mkv.part"); + let output = nested.join("capture.mkv"); + std::fs::write(&part, b"valid prefix").unwrap(); + publish_manual_recording(&part, &output).unwrap(); + assert!(!part.exists()); + assert_eq!(std::fs::read(&output).unwrap(), b"valid prefix"); + std::fs::write(&part, b"later prefix").unwrap(); + assert!(publish_manual_recording(&part, &output).is_err()); + assert_eq!(std::fs::read(&output).unwrap(), b"valid prefix"); + assert_eq!(std::fs::read(&part).unwrap(), b"later prefix"); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn writes_atomic_h264_and_opus_matroska() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "opennow-recording-test-{}-{}", + std::process::id(), + unique + )); + std::fs::create_dir_all(&directory).expect("temporary directory"); + let output = directory.join("capture.mkv"); + let (sender, raw_receiver) = channel(); + sender + .send(frame(MediaCodec::H264, h264_keyframe(), 90_000, true)) + .unwrap(); + sender + .send(frame( + MediaCodec::Opus { channels: 2 }, + vec![0x80, 1, 2, 3], + 48_000, + false, + )) + .unwrap(); + sender + .send(frame( + MediaCodec::H264, + vec![0, 0, 0, 1, 0x41, 0x9a, 0x22], + 91_500, + false, + )) + .unwrap(); + drop(sender); + + let summary = record_matroska( + &output, + MediaStreamConfig { + width: 1920, + height: 1080, + fps: 60, + ..MediaStreamConfig::default() + }, + EncodedRecordingReceiver::from_receiver(raw_receiver), + ) + .expect("recording"); + assert_eq!(summary.media.video_packets, 2); + assert_eq!(summary.media.audio_packets, 1); + assert_eq!(summary.completion, RecordingCompletion::Complete); + assert!(output.exists()); + assert!(!part_path_for(&output).unwrap().exists()); + + let file: Box = Box::new(std::fs::File::open(&output).unwrap()); + let mut demuxer = oxideav_mkv::demux::open(file, &NullCodecResolver).unwrap(); + assert_eq!(demuxer.streams().len(), 2); + assert_eq!(demuxer.streams()[0].params.codec_id.as_str(), "h264"); + assert_eq!(demuxer.streams()[1].params.codec_id.as_str(), "opus"); + let mut packets = 0; + while demuxer.next_packet().is_ok() { + packets += 1; + } + assert_eq!(packets, 3); + + std::fs::remove_file(output).unwrap(); + std::fs::remove_dir(directory).unwrap(); + } + + #[test] + fn timestamp_delta_handles_rtp_wrap() { + assert_eq!(rtp_timestamp_delta(u64::from(u32::MAX) - 10, 20), 31); + assert_eq!(rtp_timestamp_delta(100, 90), 0); + } + + #[test] + fn h265_repack_builds_configuration_and_length_prefixes() { + // Minimal synthetic SPS bitstream for parser coverage. The profile fields are Main, + // level 4.0, one temporal layer, 4:2:0, 8-bit, 1920x1080. + let sps = synthetic_hevc_sps(); + let mut input = Vec::new(); + for nal in [ + &[0x40, 0x01, 0x0c][..], + sps.as_slice(), + &[0x44, 0x01, 0xc0][..], + &[0x26, 0x01, 0xaa, 0xbb][..], + ] { + input.extend_from_slice(&[0, 0, 0, 1]); + input.extend_from_slice(nal); + } + let repacked = annexb_to_hvcc(&input).expect("HEVC repack"); + assert_eq!(repacked.config_record[0], 1); + assert_eq!(repacked.config_record[22], 3); + assert_eq!( + repacked.packetized, + vec![0, 0, 0, 4, 0x26, 0x01, 0xaa, 0xbb] + ); + } + + #[test] + fn av1_configuration_is_derived_from_the_sequence_header_obu() { + let sequence_header = b"\x0a\x0f\0\0\0j\xef\xbf\xe1\xbc\x02\x19\x90\x10\x10\x10@"; + let config = av1_codec_private(sequence_header).expect("AV1 configuration"); + assert_eq!(&config[..4], &[0x81, 0x0d, 0x0c, 0x00]); + assert_eq!(&config[4..], sequence_header); + } + + fn synthetic_hevc_sps() -> Vec { + let mut bits = BitWriter::default(); + bits.write(0, 4); // sps_video_parameter_set_id + bits.write(0, 3); // max_sub_layers_minus1 + bits.write(1, 1); // temporal nesting + bits.write(0, 2); // profile space + bits.write(0, 1); // tier + bits.write(1, 5); // Main profile + bits.write(0x6000_0000, 32); // compatibility + bits.write(0, 48); // constraints + bits.write(120, 8); // level 4.0 + bits.write_ue(0); // sps id + bits.write_ue(1); // 4:2:0 + bits.write_ue(1920); + bits.write_ue(1080); + bits.write(0, 1); // no conformance window + bits.write_ue(0); // 8-bit luma + bits.write_ue(0); // 8-bit chroma + let mut nal = vec![0x42, 0x01]; + nal.extend_from_slice(&bits.finish()); + nal + } + + #[derive(Default)] + struct BitWriter { + bits: Vec, + } + + impl BitWriter { + fn write(&mut self, value: u64, count: usize) { + for shift in (0..count).rev() { + self.bits.push(((value >> shift) & 1) != 0); + } + } + + fn write_ue(&mut self, value: u64) { + let code = value + 1; + let width = (64 - code.leading_zeros()) as usize; + self.write(0, width - 1); + self.write(code, width); + } + + fn finish(mut self) -> Vec { + self.bits.push(true); + while !self.bits.len().is_multiple_of(8) { + self.bits.push(false); + } + self.bits + .chunks(8) + .map(|chunk| { + chunk + .iter() + .fold(0_u8, |value, bit| (value << 1) | u8::from(*bit)) + }) + .collect() + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/replay.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/replay.rs new file mode 100644 index 000000000..cc7ed6ed4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/replay.rs @@ -0,0 +1,448 @@ +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use opennow_streamer_protocol::ReplayBufferConfig; + +use crate::media::{EncodedFrame, MediaCodec}; + +const MAX_FRAMES: usize = 32_768; +const FRAME_OVERHEAD: usize = 256; + +pub(crate) struct ReplayFrame { + pub frame: EncodedFrame, + pub time: Duration, +} + +pub struct ReplaySnapshot { + pub(crate) frames: VecDeque, + pub(crate) cancelled: Arc, + reserved: Arc, +} + +impl Drop for ReplaySnapshot { + fn drop(&mut self) { + self.frames.clear(); + self.reserved.store(0, Ordering::Release); + } +} + +struct Clock { + timestamp: u64, + rate: u32, + time: Duration, + codec: MediaCodec, +} + +impl Clock { + fn advance(&mut self, frame: &EncodedFrame) -> Option { + if frame.clock_rate_hz == 0 || self.rate != frame.clock_rate_hz || self.codec != frame.codec + { + return None; + } + let delta = + if self.timestamp <= u64::from(u32::MAX) && frame.timestamp <= u64::from(u32::MAX) { + u64::from((frame.timestamp as u32).wrapping_sub(self.timestamp as u32)) + } else { + frame.timestamp.checked_sub(self.timestamp)? + }; + if delta > u64::from(self.rate) * 10 { + return None; + } + self.time += Duration::from_secs_f64(delta as f64 / f64::from(self.rate)); + self.timestamp = frame.timestamp; + Some(self.time) + } +} + +struct Buffer { + config: ReplayBufferConfig, + frames: VecDeque, + bytes: usize, + origin: Instant, + video: Option, + audio: Option, + latest: Duration, + cancelled: Arc, + reserved: Arc, +} + +impl Buffer { + fn clear(&mut self) { + self.frames.clear(); + self.bytes = 0; + self.video = None; + self.audio = None; + self.origin = Instant::now(); + self.latest = Duration::ZERO; + } + + fn pop_front(&mut self) { + if let Some(frame) = self.frames.pop_front() { + self.bytes -= frame.frame.data.len() + frame.frame.mid.len() + FRAME_OVERHEAD; + } + } + + fn push(&mut self, frame: &EncodedFrame, reserved: usize, received_at: Instant) { + if !frame.contiguous { + self.clear(); + return; + } + let video = !matches!(frame.codec, MediaCodec::Opus { .. }); + let clock = if video { + &mut self.video + } else { + &mut self.audio + }; + let time = if let Some(clock) = clock { + let Some(time) = clock.advance(frame) else { + self.clear(); + return; + }; + time + } else { + if frame.clock_rate_hz == 0 { + return; + } + let time = received_at.saturating_duration_since(self.origin); + *clock = Some(Clock { + timestamp: frame.timestamp, + rate: frame.clock_rate_hz, + time, + codec: frame.codec.clone(), + }); + time + }; + self.latest = self.latest.max(time); + if self.frames.front().is_some_and(|first| time < first.time) { + return; + } + if self.frames.is_empty() && !(video && frame.keyframe) { + return; + } + let budget = self.config.memory_bytes.saturating_sub(reserved); + let bytes = frame + .data + .len() + .saturating_add(frame.mid.len()) + .saturating_add(FRAME_OVERHEAD); + if bytes > budget { + self.clear(); + return; + } + self.frames.push_back(ReplayFrame { + frame: frame.clone(), + time, + }); + self.bytes += bytes; + while self.bytes > budget + || self.frames.len() > MAX_FRAMES + || self + .frames + .front() + .is_some_and(|first| self.latest.saturating_sub(first.time) > self.config.duration) + { + self.pop_front(); + } + while self.frames.front().is_some_and(|first| { + !first.frame.keyframe || matches!(first.frame.codec, MediaCodec::Opus { .. }) + }) { + self.pop_front(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(bytes: usize, seconds: u64) -> ReplayBufferConfig { + ReplayBufferConfig { + enabled: true, + memory_bytes: bytes, + duration: Duration::from_secs(seconds), + } + } + + fn frame(timestamp: u64, keyframe: bool) -> EncodedFrame { + EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0_u8; 16]), + frame_index: Some(timestamp as u32), + timestamp, + clock_rate_hz: 1, + keyframe, + contiguous: true, + ssrc: None, + } + } + + #[test] + fn long_gop_is_discarded_until_a_new_source_keyframe() { + let tap = ReplayTap::default(); + tap.start(config(100_000, 2)); + for index in 0..10 { + tap.publish(&frame(index, index == 0)); + } + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + tap.publish(&frame(10, true)); + let snapshot = tap.snapshot().unwrap(); + assert_eq!(snapshot.frames.len(), 1); + assert_eq!(snapshot.frames[0].frame.timestamp, 10); + } + + #[test] + fn audio_offset_uses_receive_anchor_then_source_clock_not_arrival_jitter() { + let tap = ReplayTap::default(); + tap.start(config(100_000, 30)); + let mut guard = tap.buffer.lock().unwrap(); + let buffer = guard.as_mut().unwrap(); + let origin = buffer.origin; + let mut video = frame(9_000_000, true); + video.clock_rate_hz = 90_000; + let mut audio = frame(400_000, false); + audio.codec = MediaCodec::Opus { channels: 2 }; + audio.clock_rate_hz = 48_000; + buffer.push(&video, 0, origin + Duration::from_secs(1)); + buffer.push(&audio, 0, origin + Duration::from_millis(1_100)); + audio.timestamp += 960; + buffer.push(&audio, 0, origin + Duration::from_millis(1_500)); + video.timestamp += 18_000; + video.keyframe = false; + buffer.push(&video, 0, origin + Duration::from_millis(1_600)); + assert_eq!(buffer.frames[0].time, Duration::from_secs(1)); + assert_eq!(buffer.frames[1].time, Duration::from_millis(1_100)); + assert_eq!(buffer.frames[2].time, Duration::from_millis(1_120)); + assert_eq!(buffer.frames[3].time, Duration::from_millis(1_200)); + assert_eq!(buffer.frames[1].frame.timestamp, 400_000); + assert_eq!(buffer.frames[2].frame.timestamp, 400_960); + } + + #[test] + fn cancelled_previous_session_keeps_its_reservation_until_worker_releases_it() { + let budget = Arc::new(AtomicUsize::new(0)); + let old = ReplayTap::default(); + old.start_with_budget(config(16 + FRAME_OVERHEAD + 5, 30), Arc::clone(&budget)); + old.publish(&frame(0, true)); + let snapshot = old.snapshot().unwrap(); + old.stop(); + assert!(snapshot.cancelled.load(Ordering::Acquire)); + let new = ReplayTap::default(); + new.start_with_budget(config(16 + FRAME_OVERHEAD + 5, 30), Arc::clone(&budget)); + new.publish(&frame(0, true)); + assert!( + new.buffer + .lock() + .unwrap() + .as_ref() + .unwrap() + .frames + .is_empty() + ); + drop(snapshot); + new.publish(&frame(1, true)); + assert!(new.snapshot().is_ok()); + } + + #[test] + fn disabled_replay_allocates_no_buffer_and_retains_no_payload() { + let tap = ReplayTap::default(); + tap.start(ReplayBufferConfig::from_settings(&serde_json::json!({}))); + let frame = frame(0, true); + tap.publish(&frame); + assert!(tap.buffer.lock().unwrap().is_none()); + assert_eq!(Arc::strong_count(&frame.data), 1); + assert_eq!(tap.snapshot().err(), Some("replay-not-enabled")); + } + + #[test] + fn evicts_whole_gops_for_byte_and_source_time_bounds() { + for cfg in [ + config(3 * (16 + FRAME_OVERHEAD + 5), 100), + config(100_000, 2), + ] { + let tap = ReplayTap::default(); + tap.start(cfg); + for index in 0..6 { + tap.publish(&frame(index, index % 2 == 0)); + } + let snapshot = tap.snapshot().unwrap(); + assert_eq!(snapshot.frames.front().unwrap().frame.timestamp, 4); + assert_eq!(snapshot.frames.len(), 2); + assert!(snapshot.frames.front().unwrap().frame.keyframe); + assert!(snapshot.reserved.load(Ordering::Acquire) <= cfg.memory_bytes); + assert!( + snapshot.frames.back().unwrap().time - snapshot.frames.front().unwrap().time + <= cfg.duration + ); + } + } + + #[test] + fn discontinuity_and_contention_reset_until_keyframe() { + let tap = ReplayTap::default(); + tap.start(config(100_000, 30)); + tap.publish(&frame(0, true)); + let mut broken = frame(1, false); + broken.contiguous = false; + tap.publish(&broken); + tap.publish(&frame(2, false)); + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + tap.publish(&frame(3, true)); + let guard = tap.buffer.lock().unwrap(); + tap.publish(&frame(4, false)); + drop(guard); + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + tap.publish(&frame(5, true)); + assert_eq!(tap.snapshot().unwrap().frames.len(), 1); + } + + #[test] + fn export_reserves_shared_budget_and_only_one_snapshot() { + let tap = ReplayTap::default(); + tap.start(config(2 * (16 + FRAME_OVERHEAD + 5), 30)); + let source = frame(0, true); + tap.publish(&source); + let snapshot = tap.snapshot().unwrap(); + assert_eq!(Arc::strong_count(&source.data), 2); + for index in 1..5 { + tap.publish(&frame(index, true)); + } + assert_eq!(tap.snapshot().err(), Some("clip-already-saving")); + let guard = tap.buffer.lock().unwrap(); + assert!( + guard.as_ref().unwrap().bytes + snapshot.reserved.load(Ordering::Acquire) + <= 2 * (16 + FRAME_OVERHEAD + 5) + ); + drop(guard); + drop(snapshot); + assert_eq!(Arc::strong_count(&source.data), 1); + assert!(tap.snapshot().is_ok()); + tap.stop(); + assert!(tap.buffer.lock().unwrap().is_none()); + } + + #[test] + fn oversized_frames_clock_changes_and_backward_timestamps_reset() { + let tap = ReplayTap::default(); + tap.start(config(16 + FRAME_OVERHEAD + 5, 30)); + tap.publish(&frame(10, true)); + tap.publish(&frame(9, false)); + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + tap.publish(&frame(10, true)); + let mut changed = frame(11, false); + changed.clock_rate_hz = 2; + tap.publish(&changed); + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + let mut oversized = frame(0, true); + oversized.data = Arc::from([0_u8; 17]); + tap.publish(&oversized); + assert_eq!(tap.snapshot().err(), Some("replay-not-ready")); + } + + #[test] + fn source_clock_wrap_preserves_elapsed_time() { + let mut clock = Clock { + timestamp: u64::from(u32::MAX), + rate: 1, + time: Duration::from_secs(5), + codec: MediaCodec::H264, + }; + assert_eq!(clock.advance(&frame(1, true)), Some(Duration::from_secs(7))); + } +} + +#[derive(Default)] +pub(crate) struct ReplayTap { + enabled: AtomicBool, + reset: AtomicBool, + buffer: Mutex>, +} + +impl ReplayTap { + #[cfg(test)] + pub fn start(&self, config: ReplayBufferConfig) { + self.start_with_budget(config, Arc::new(AtomicUsize::new(0))); + } + + pub fn start_with_budget(&self, config: ReplayBufferConfig, reserved: Arc) { + self.stop(); + if !config.enabled { + return; + } + *self + .buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Buffer { + config, + frames: VecDeque::new(), + bytes: 0, + origin: Instant::now(), + video: None, + audio: None, + latest: Duration::ZERO, + cancelled: Arc::new(AtomicBool::new(false)), + reserved, + }); + self.enabled.store(true, Ordering::Release); + } + + pub fn stop(&self) { + self.enabled.store(false, Ordering::Release); + if let Some(buffer) = self + .buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + buffer.cancelled.store(true, Ordering::Release); + } + self.reset.store(false, Ordering::Release); + } + + pub fn publish(&self, frame: &EncodedFrame) { + if !self.enabled.load(Ordering::Acquire) { + return; + } + let Ok(mut guard) = self.buffer.try_lock() else { + self.reset.store(true, Ordering::Release); + return; + }; + if let Some(buffer) = guard.as_mut() { + if self.reset.swap(false, Ordering::AcqRel) { + buffer.clear(); + } + buffer.push( + frame, + buffer.reserved.load(Ordering::Acquire), + Instant::now(), + ); + } + } + + pub fn snapshot(&self) -> Result { + let mut guard = self + .buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let buffer = guard.as_mut().ok_or("replay-not-enabled")?; + if self.reset.swap(false, Ordering::AcqRel) { + buffer.clear(); + } + if buffer.reserved.load(Ordering::Acquire) != 0 { + return Err("clip-already-saving"); + } + if buffer.frames.is_empty() { + return Err("replay-not-ready"); + } + buffer.reserved.store(buffer.bytes, Ordering::Release); + buffer.bytes = 0; + Ok(ReplaySnapshot { + frames: std::mem::take(&mut buffer.frames), + reserved: Arc::clone(&buffer.reserved), + cancelled: Arc::clone(&buffer.cancelled), + }) + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/runtime.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/runtime.rs new file mode 100644 index 000000000..e167834ed --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/runtime.rs @@ -0,0 +1,2103 @@ +use std::marker::PhantomData; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::time::Duration; + +use opennow_streamer_protocol::{AudioDevice, AudioOutputDevice, RenderSurface}; + +use crate::GraphicsFramePublisher; +use crate::media::{MediaFeedback, MediaSession, MediaSessionConfig, MediaStreamConfig}; +#[cfg(target_os = "windows")] +use crate::output::WindowsBridge; +use crate::output::{ActiveOutput, OutputBuffers, OutputControl, OutputEvent}; + +#[cfg(target_os = "linux")] +use crate::linux_backend::{LinuxVideoPath, LinuxVideoSelection}; + +// Keep native input sampling below one rendered frame even at 120 Hz. SDL can +// coalesce the individual raw-input events, so this improves delivery cadence +// without allowing the queue to grow with redundant motion packets. +const HOST_POLL_INTERVAL: Duration = Duration::from_micros(250); +const HOST_START_TIMEOUT: Duration = Duration::from_secs(10); +const HOST_CONTROL_TIMEOUT: Duration = Duration::from_secs(2); +const MICROPHONE_HOST_POLL_INTERVAL: Duration = Duration::from_millis(20); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaRuntimeControl { + PointerLock, +} + +#[cfg(target_os = "macos")] +pub(crate) enum MacH264Configuration { + Hardware(opennow_streamer_platform_macos::StreamSink), + SoftwareFallback { reason: String }, +} + +pub(crate) struct AudioDeviceRequest { + pub(crate) reply: Sender, String>>, + pending: Arc, +} + +impl Drop for AudioDeviceRequest { + fn drop(&mut self) { + self.pending.store(false, Ordering::Release); + } +} + +pub(crate) enum HostCommand { + StartMicrophone { + device_id: String, + shared: Arc, + reply: Sender>, + }, + AudioDevices(AudioDeviceRequest), + Start { + reply: Sender>, + feedback: Sender, + stream: MediaStreamConfig, + audio_device: AudioOutputDevice, + }, + Pause { + paused: bool, + reply: Option>>, + }, + Surface { + surface: RenderSurface, + reply: Sender>, + }, + Control { + control: MediaRuntimeControl, + reply: Sender>, + }, + Cursor(Vec), + #[cfg(target_os = "macos")] + ConfigureMacH264 { + parameter_sets: opennow_streamer_platform_macos::H264ParameterSets, + reply: Sender>, + }, + #[cfg(target_os = "macos")] + ConfigureMacH265 { + parameter_sets: opennow_streamer_platform_macos::H265ParameterSets, + reply: Sender>, + }, + #[cfg(target_os = "macos")] + ConfigureMacAv1 { + format: opennow_streamer_platform_macos::Av1Format, + reply: Sender>, + }, + #[cfg(target_os = "linux")] + FallbackLinux { + reason: String, + }, + Stop, + Shutdown, +} + +#[derive(Clone)] +pub struct MediaRuntime { + commands: Sender, + audio_query_pending: Arc, + output: Arc, + paused: Arc, + use_hardware: Arc, + #[cfg(target_os = "windows")] + windows_bridge: Arc, + #[cfg(target_os = "linux")] + linux_selection: LinuxVideoSelection, + #[cfg(target_os = "linux")] + linux_software_fallback: Arc, + mode: MediaRuntimeMode, +} + +#[derive(Clone)] +enum MediaRuntimeMode { + Standalone, + Embedded { + frames: GraphicsFramePublisher, + config: EmbeddedRuntimeConfig, + }, + #[cfg(feature = "test-runtime")] + Test, +} + +#[derive(Clone, Default)] +pub struct EmbeddedRuntimeConfig { + pub vulkan_device: Option>, + pub windows_adapter_luid: Option, +} + +impl MediaRuntime { + pub fn published_video_frames(&self) -> Option { + match &self.mode { + MediaRuntimeMode::Embedded { frames, .. } => Some(frames.published_frames()), + _ => None, + } + } + + pub fn set_audio_muted(&self, muted: bool) { + self.output.audio_muted.store(muted, Ordering::Release); + } + + pub const fn is_embedded(&self) -> bool { + matches!(self.mode, MediaRuntimeMode::Embedded { .. }) + } + + pub fn video_backends(&self) -> Vec { + match &self.mode { + MediaRuntimeMode::Embedded { config, .. } => { + crate::embedded_video_backends_with_config( + config.vulkan_device.as_deref(), + self.windows_adapter_luid(), + ) + } + _ => crate::video_backends(), + } + } + + pub fn graphics_adapters(&self) -> Vec { + crate::graphics_adapter_capabilities(self.windows_adapter_luid().map(|luid| luid.get())) + } + + fn windows_adapter_luid(&self) -> Option { + match &self.mode { + MediaRuntimeMode::Embedded { config, .. } => config.windows_adapter_luid, + _ => None, + } + } + + pub fn captured_input(&self) -> Arc { + self.output.captured_input() + } + + pub fn start_microphone(&self, device_id: &str) -> Result { + self.output.stop_microphone(); + let shared = crate::microphone::MicrophoneShared::new(); + let session = crate::MicrophoneSession::from_shared(Arc::clone(&shared)); + self.output.set_microphone(&shared); + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::StartMicrophone { + device_id: device_id.to_owned(), + shared, + reply, + }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_START_TIMEOUT) + .map_err(|_| "native media host did not start microphone capture".to_owned())??; + Ok(session) + } + + pub fn audio_devices(&self) -> Result, String> { + if self.audio_query_pending.swap(true, Ordering::AcqRel) { + return Err("native audio device enumeration is already in progress".to_owned()); + } + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::AudioDevices(AudioDeviceRequest { + reply, + pending: Arc::clone(&self.audio_query_pending), + })) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_CONTROL_TIMEOUT) + .map_err(|_| "native audio device enumeration timed out".to_owned())? + } + + pub fn start( + &self, + feedback: Sender, + stream: MediaStreamConfig, + ) -> Result { + self.start_with_backend(feedback, stream, "auto") + } + + pub fn validate_backend(&self, requested: &str) -> Result<(), String> { + if !self.is_embedded() { + return Ok(()); + } + #[cfg(target_os = "windows")] + let supported = matches!(requested, "auto" | "d3d11"); + #[cfg(target_os = "macos")] + let supported = matches!(requested, "auto" | "videotoolbox"); + #[cfg(target_os = "linux")] + let supported = requested == "auto" + || self.video_backends().iter().any(|backend| { + backend.available + && (backend.backend == requested + || (requested == "nvdec" && backend.backend == "cuda") + || (matches!(requested, "software" | "ffmpeg") + && backend.backend == "ffmpeg")) + }); + #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] + let supported = requested == "auto"; + if supported { + Ok(()) + } else { + Err(format!( + "Backend {requested} is not supported by this embedded stream view. Select Auto in Stream settings." + )) + } + } + + pub fn start_with_backend( + &self, + feedback: Sender, + stream: MediaStreamConfig, + requested: &str, + ) -> Result { + self.start_with_audio_device(feedback, stream, requested, AudioOutputDevice::default()) + } + + pub fn start_with_audio_device( + &self, + feedback: Sender, + stream: MediaStreamConfig, + requested: &str, + audio_device: AudioOutputDevice, + ) -> Result { + self.validate_backend(requested)?; + if stream.hdr { + if stream.codec == crate::MediaVideoCodec::H264 + || stream.color_quality.bit_depth() != 10 + { + return Err("HDR requires a negotiated 10-bit HEVC or AV1 stream".to_owned()); + } + if stream.color_quality.is_444() && !self.supports_hdr_444(stream) { + return Err("HDR requires a negotiated 10-bit 4:2:0 stream unless HEVC 10-bit 4:4:4 hardware decode and GPU output are supported".to_owned()); + } + if !cfg!(any( + target_os = "windows", + target_os = "linux", + target_os = "macos" + )) { + return Err( + "HDR is supported only by the Windows, Linux, and macOS embedded stream view" + .to_owned(), + ); + } + if matches!(self.mode, MediaRuntimeMode::Standalone) { + return Err("HDR requires the color-managed embedded Qt stream view".to_owned()); + } + #[cfg(target_os = "macos")] + if stream.codec != crate::MediaVideoCodec::H265 + || !(if stream.color_quality.is_444() { + opennow_streamer_platform_macos::probe_h265_hdr_444_hardware() + } else { + opennow_streamer_platform_macos::probe_h265_hdr_hardware() + }) + { + return Err( + "HDR requires a verified HEVC VideoToolbox and Metal output path".to_owned(), + ); + } + } + if (stream.codec == crate::MediaVideoCodec::H264 + && stream.color_quality != crate::MediaColorQuality::EightBit420) + || (stream.codec == crate::MediaVideoCodec::Av1 && stream.color_quality.is_444()) + { + return Err("The negotiated codec and color profile cannot be preserved by the embedded decoder".to_owned()); + } + self.output.stop_microphone(); + self.output.reset_microphone_clock(); + if let MediaRuntimeMode::Embedded { + frames, + config: _config, + } = &self.mode + { + #[cfg(target_os = "macos")] + if let Some(id) = audio_device.device_name() { + if self + .audio_devices()? + .iter() + .filter(|device| device.id == id) + .count() + != 1 + { + return Err("The selected audio output device is unavailable or ambiguous. Select another device or System default.".to_owned()); + } + } + #[cfg(target_os = "windows")] + { + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::Start { + reply, + feedback: feedback.clone(), + stream, + audio_device: audio_device.clone(), + }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_START_TIMEOUT) + .map_err(|_| "native audio output startup timed out".to_owned())??; + } + let session = MediaSession::spawn_embedded( + Arc::clone(&self.output), + feedback, + self.commands.clone(), + crate::media::MediaSessionConfig { + stream, + audio_device: &audio_device, + }, + frames.clone(), + #[cfg(target_os = "linux")] + crate::linux_backend::select_embedded_video_path( + requested, + _config.vulkan_device.as_deref(), + stream, + ), + #[cfg(target_os = "linux")] + _config.vulkan_device.clone(), + ) + .inspect_err(|_| { + let _ = self.commands.send(HostCommand::Stop); + })?; + if self.paused.load(Ordering::Acquire) { + session.set_paused(true); + } + return Ok(session); + } + #[cfg(feature = "test-runtime")] + if matches!(self.mode, MediaRuntimeMode::Test) { + let session = MediaSession::spawn_test( + Arc::clone(&self.output), + feedback, + self.commands.clone(), + stream, + #[cfg(target_os = "windows")] + Arc::clone(&self.windows_bridge), + #[cfg(target_os = "linux")] + Arc::clone(&self.linux_software_fallback), + )?; + if self.paused.load(Ordering::Acquire) { + session.set_paused(true); + } + return Ok(session); + } + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::Start { + reply, + feedback: feedback.clone(), + stream, + audio_device, + }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_START_TIMEOUT) + .map_err(|_| "native media host did not start on the UI thread".to_owned())??; + #[cfg(target_os = "linux")] + if let Some(reason) = self.linux_selection.fallback_reason.clone() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "Linux Vulkan presentation", + to: if matches!(self.linux_selection.path, LinuxVideoPath::Hardware(_)) { + "Linux decoder/SDL NV12" + } else { + "OpenH264/SDL" + }, + reason, + }); + } + match MediaSession::spawn( + Arc::clone(&self.output), + feedback, + self.commands.clone(), + self.use_hardware.load(Ordering::Acquire), + stream, + #[cfg(target_os = "windows")] + Arc::clone(&self.windows_bridge), + #[cfg(target_os = "linux")] + self.linux_selection.clone(), + #[cfg(target_os = "linux")] + Arc::clone(&self.linux_software_fallback), + ) { + Ok(session) => { + if self.paused.load(Ordering::Acquire) { + session.set_paused(true); + } + Ok(session) + } + Err(error) => { + let _ = self.commands.send(HostCommand::Stop); + Err(error) + } + } + } + + fn supports_hdr_444(&self, stream: MediaStreamConfig) -> bool { + if !self.is_embedded() || stream.codec != crate::MediaVideoCodec::H265 { + return false; + } + #[cfg(target_os = "windows")] + { + use opennow_streamer_platform_windows::{VideoCodec, VideoPixelFormat, WindowsBackend}; + WindowsBackend::probe_for( + opennow_streamer_platform_windows::WindowsGraphicsApi::D3d11, + self.windows_adapter_luid(), + ) + .supports_format(VideoCodec::H265, VideoPixelFormat::Y410, true) + } + #[cfg(target_os = "linux")] + { + use opennow_streamer_platform_linux::{PixelFormat, VideoCodec}; + matches!(&self.mode, MediaRuntimeMode::Embedded { config, .. } + if config.vulkan_device.as_ref().is_some_and(|device| device.supports_format(VideoCodec::H265, PixelFormat::P410, stream.width, stream.height))) + } + #[cfg(target_os = "macos")] + { + opennow_streamer_platform_macos::probe_h265_hdr_444_hardware() + } + #[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] + false + } + + pub fn update_surface(&self, surface: RenderSurface) -> Result<(), String> { + if !matches!(self.mode, MediaRuntimeMode::Standalone) { + return Ok(()); + } + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::Surface { surface, reply }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_CONTROL_TIMEOUT) + .map_err(|_| "native media host did not apply the shell surface update".to_owned())? + } + + pub fn set_paused(&self, paused: bool) -> Result<(), String> { + self.paused.store(paused, Ordering::Release); + if !matches!(self.mode, MediaRuntimeMode::Standalone) { + return Ok(()); + } + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::Pause { + paused, + reply: Some(reply), + }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_CONTROL_TIMEOUT) + .map_err(|_| "native media host did not apply the pause update".to_owned())? + } + + pub fn control(&self, control: MediaRuntimeControl) -> Result<(), String> { + if !matches!(self.mode, MediaRuntimeMode::Standalone) { + return Ok(()); + } + let (reply, response) = mpsc::channel(); + self.commands + .send(HostCommand::Control { control, reply }) + .map_err(|_| "native media host is no longer running".to_owned())?; + response + .recv_timeout(HOST_CONTROL_TIMEOUT) + .map_err(|_| "native media host did not apply the runtime control".to_owned())? + } + + pub fn shutdown(&self) { + self.output.stop_microphone(); + let _ = self.commands.send(HostCommand::Shutdown); + } +} + +pub struct MainThreadHost { + commands: Receiver, + output: Arc, + _not_send: PhantomData>, + use_macos_hardware: Arc, + #[cfg(target_os = "windows")] + windows_bridge: Arc, + #[cfg(target_os = "linux")] + linux_selection: LinuxVideoSelection, + #[cfg(target_os = "linux")] + linux_software_fallback: Arc, +} + +impl MainThreadHost { + pub fn run(self) { + let mut active: Option = None; + let mut microphone: Option = None; + let mut surface: Option = None; + let mut paused = false; + let mut feedback: Option> = None; + let mut software_playback_started = false; + let mut active_stream = MediaStreamConfig::default(); + let mut active_audio_device = AudioOutputDevice::default(); + loop { + match self.commands.recv_timeout(HOST_POLL_INTERVAL) { + Ok(HostCommand::StartMicrophone { + device_id, + shared, + reply, + }) => { + microphone = None; + let result = crate::microphone::MicrophoneCapture::start( + &device_id, + shared, + self.output.microphone_clock(), + ) + .map(|capture| microphone = Some(capture)); + if reply.send(result).is_err() { + microphone = None; + } + } + Ok(HostCommand::Start { + reply, + feedback: session_feedback, + stream, + audio_device, + }) => { + active_stream = stream; + self.output.stop_microphone(); + microphone = None; + active_audio_device = audio_device; + if let Some(output) = active.as_mut() { + output.stop(); + } + let requested_hardware = self.use_macos_hardware.load(Ordering::Acquire); + #[cfg(target_os = "linux")] + let requested_linux_hardware = self.linux_selection.use_vulkan_output + && matches!(self.linux_selection.path, LinuxVideoPath::Hardware(_)) + && !self.linux_software_fallback.load(Ordering::Acquire); + #[cfg(not(target_os = "linux"))] + let requested_linux_hardware = false; + let mut startup_fallback = None; + let initialized = match initialize_output( + Arc::clone(&self.output), + surface.as_ref(), + paused, + requested_hardware, + MediaSessionConfig { + stream, + audio_device: &active_audio_device, + }, + requested_linux_hardware, + #[cfg(target_os = "windows")] + Arc::clone(&self.windows_bridge), + ) { + Ok(output) => Ok(output), + Err(hardware_error) if requested_hardware || requested_linux_hardware => { + self.use_macos_hardware.store(false, Ordering::Release); + #[cfg(target_os = "windows")] + self.windows_bridge.fall_back_to_software(); + startup_fallback = Some(hardware_error); + initialize_output( + Arc::clone(&self.output), + surface.as_ref(), + paused, + false, + MediaSessionConfig { + stream, + audio_device: &active_audio_device, + }, + false, + #[cfg(target_os = "windows")] + Arc::clone(&self.windows_bridge), + ) + } + Err(error) => Err(error), + }; + match initialized { + Ok(output) => { + active = Some(output); + feedback = Some(session_feedback); + software_playback_started = false; + if let (Some(reason), Some(feedback)) = + (startup_fallback, feedback.as_ref()) + { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: hardware_backend_label(), + to: if requested_linux_hardware { + "Linux decoder/SDL NV12" + } else if cfg!(target_os = "windows") + && stream.codec != crate::media::MediaVideoCodec::H264 + { + "Media Foundation software/D3D11/WASAPI" + } else { + "OpenH264/SDL" + }, + reason, + }); + } + let _ = reply.send(Ok(())); + } + Err(error) => { + active = None; + feedback = None; + let _ = reply.send(Err(error)); + } + } + } + Ok(HostCommand::AudioDevices(request)) => { + let reply = &request.reply; + #[cfg(target_os = "windows")] + if self.use_macos_hardware.load(Ordering::Acquire) + || active + .as_ref() + .is_some_and(|output| matches!(output, ActiveOutput::Windows(_))) + { + let _ = reply.send(Err("Fixed audio output selection is unavailable in the standalone Windows native output. Use the Qt embedded stream view.".to_owned())); + continue; + } + #[cfg(target_os = "macos")] + if self.use_macos_hardware.load(Ordering::Acquire) { + let devices = opennow_streamer_platform_macos::audio_output_devices() + .map(|devices| { + devices + .into_iter() + .map(|device| AudioDevice { + id: device.id, + name: device.name, + }) + .collect() + }) + .map_err(|error| error.to_string()); + let _ = reply.send(devices); + continue; + } + let _ = reply.send(crate::output::audio_devices()); + } + Ok(HostCommand::Pause { + paused: new_paused, + reply, + }) => { + let result = active + .as_mut() + .map_or(Ok(()), |output| output.set_paused(new_paused)); + if result.is_ok() { + paused = new_paused; + } + if let Some(reply) = reply { + let _ = reply.send(result); + } + } + Ok(HostCommand::Surface { + surface: new_surface, + reply, + }) => { + static SURFACE_LOG_REMAINING: AtomicU64 = AtomicU64::new(12); + let mut remaining = SURFACE_LOG_REMAINING.load(Ordering::Relaxed); + let log_surface = loop { + let Some(next) = remaining.checked_sub(1) else { + break false; + }; + match SURFACE_LOG_REMAINING.compare_exchange_weak( + remaining, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break true, + Err(current) => remaining = current, + } + }; + if log_surface { + eprintln!( + "NVST surface-update visible={} rect={:?}", + new_surface.visible, new_surface.screen_rect + ); + } + #[cfg(target_os = "linux")] + let linux_hardware = + active.as_ref().is_some_and(ActiveOutput::is_linux_hardware); + let result = if let Some(output) = active.as_mut() { + output.update_surface(&new_surface) + } else { + Ok(()) + }; + #[cfg(target_os = "linux")] + let mut result = result; + #[cfg(target_os = "linux")] + if linux_hardware && let Err(reason) = &result { + match initialize_output( + Arc::clone(&self.output), + Some(&new_surface), + paused, + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + false, + ) { + Ok(output) => { + if let Some(current) = active.as_mut() { + current.stop(); + } + active = Some(output); + software_playback_started = false; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "Linux decoder/Vulkan", + to: "Linux decoder/SDL NV12", + reason: format!( + "Linux surface reconfiguration failed: {reason}" + ), + }); + } + result = Ok(()); + } + Err(error) => result = Err(error), + } + } + if let Err(message) = &result { + if let Some(output) = active.as_mut() { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { + message: message.clone(), + }); + } + output.stop(); + active = None; + feedback = None; + } + } + surface = Some(new_surface); + let _ = reply.send(result); + } + Ok(HostCommand::Control { control, reply }) => { + let output_control = match control { + MediaRuntimeControl::PointerLock => OutputControl::PointerLock, + }; + let result = active + .as_mut() + .ok_or_else(|| "native media output is not active".to_owned()) + .and_then(|output| output.control(output_control)); + let _ = reply.send(result); + } + #[cfg(target_os = "macos")] + Ok(HostCommand::ConfigureMacH264 { + parameter_sets, + reply, + }) => { + let hardware_result = active + .as_mut() + .ok_or_else(|| "native media output is not active".to_owned()) + .and_then(|output| output.configure_macos_h264(parameter_sets)); + match hardware_result { + Ok(sink) => { + let _ = reply.send(Ok(MacH264Configuration::Hardware(sink))); + } + Err(hardware_error) => { + self.use_macos_hardware.store(false, Ordering::Release); + crate::macos_backend::disable(); + if let Some(output) = active.as_mut() { + output.stop(); + } + let fallback = ActiveOutput::initialize( + Arc::clone(&self.output), + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + false, + ) + .and_then(|mut output| { + output.start(surface.as_ref())?; + output.set_paused(paused)?; + Ok(output) + }); + match fallback { + Ok(output) => { + active = Some(output); + let _ = + reply.send(Ok(MacH264Configuration::SoftwareFallback { + reason: hardware_error, + })); + } + Err(fallback_error) => { + active = None; + let _ = reply.send(Err(format!( + "VideoToolbox startup failed ({hardware_error}); software output fallback also failed: {fallback_error}" + ))); + } + } + } + } + } + #[cfg(target_os = "macos")] + Ok(HostCommand::ConfigureMacH265 { + parameter_sets, + reply, + }) => { + let result = active + .as_mut() + .ok_or_else(|| "native media output is not active".to_owned()) + .and_then(|output| output.configure_macos_h265(parameter_sets)); + if result.is_err() { + crate::macos_backend::disable_h265(); + } + let _ = reply.send(result); + } + #[cfg(target_os = "macos")] + Ok(HostCommand::ConfigureMacAv1 { format, reply }) => { + let result = active + .as_mut() + .ok_or_else(|| "native media output is not active".to_owned()) + .and_then(|output| output.configure_macos_av1(format)); + if result.is_err() { + crate::macos_backend::disable_av1(); + } + let _ = reply.send(result); + } + Ok(HostCommand::Cursor(bytes)) => { + if let Some(output) = active.as_mut() { + output.update_cursor(&bytes); + } + } + #[cfg(target_os = "linux")] + Ok(HostCommand::FallbackLinux { reason }) => { + self.linux_software_fallback.store(true, Ordering::Release); + let replacement = initialize_output( + Arc::clone(&self.output), + surface.as_ref(), + paused, + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + false, + ); + match replacement { + Ok(output) => { + if let Some(current) = active.as_mut() { + current.stop(); + } + active = Some(output); + software_playback_started = false; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "Linux decoder pipeline", + to: "OpenH264/SDL", + reason, + }); + } + } + Err(fallback_error) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { + message: format!( + "Linux hardware fallback failed: {fallback_error}" + ), + }); + } + active = None; + feedback = None; + } + } + } + Ok(HostCommand::Stop) => { + self.output.stop_microphone(); + microphone = None; + if let Some(output) = active.as_mut() { + output.stop(); + } + active = None; + feedback = None; + software_playback_started = false; + } + Ok(HostCommand::Shutdown) | Err(mpsc::RecvTimeoutError::Disconnected) => { + self.output.stop_microphone(); + drop(microphone); + if let Some(output) = active.as_mut() { + output.stop(); + } + return; + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if let Some(capture) = microphone.as_mut() { + capture.poll(); + if capture.stopped() { + microphone = None; + } + } + if let Some(output) = active.as_mut() { + let output_event = output.pump(); + let captured_input = self.output.captured_input(); + for input in output.take_captured_input() { + captured_input.push(input); + } + match output_event { + Ok(OutputEvent::Presented(backend)) if !software_playback_started => { + software_playback_started = true; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::PlaybackStarted { backend }); + } + } + #[cfg(target_os = "windows")] + Ok(OutputEvent::RequestKeyframe) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::RequestKeyframe { + mid: current_video_mid( + #[cfg(target_os = "windows")] + &self.windows_bridge, + ), + reason: "D3D11 decoder recovery requires a fresh keyframe" + .to_owned(), + }); + } + } + #[cfg(target_os = "windows")] + Ok(OutputEvent::DeviceLost { + subsystem, + recovered, + message, + }) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::DeviceLost { + subsystem, + recovered, + message, + }); + } + } + #[cfg(target_os = "windows")] + Ok(OutputEvent::QueueDropped(media)) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::QueueDropped { media, count: 1 }); + } + } + #[cfg(target_os = "windows")] + Ok(OutputEvent::Fatal(message)) => { + if !output.is_windows_hardware() { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { message }); + } + output.stop(); + active = None; + feedback = None; + software_playback_started = false; + continue; + } + let fallback = (|| { + output.stop(); + self.use_macos_hardware.store(false, Ordering::Release); + #[cfg(target_os = "windows")] + self.windows_bridge.fall_back_to_software(); + let mut replacement = ActiveOutput::initialize( + Arc::clone(&self.output), + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + #[cfg(target_os = "windows")] + Arc::clone(&self.windows_bridge), + false, + )?; + replacement.start(surface.as_ref())?; + replacement.set_paused(paused)?; + Ok::<_, String>(replacement) + })(); + match fallback { + Ok(replacement) => { + *output = replacement; + software_playback_started = false; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "Media Foundation/D3D11/WASAPI", + to: if active_stream.codec + == crate::media::MediaVideoCodec::H264 + { + "OpenH264/SDL" + } else { + "Media Foundation software/D3D11/WASAPI" + }, + reason: message, + }); + let _ = feedback.send(MediaFeedback::RequestKeyframe { + mid: current_video_mid( + #[cfg(target_os = "windows")] + &self.windows_bridge, + ), + reason: "D3D11 device recovery failed".to_owned(), + }); + } + } + Err(fallback_error) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { + message: format!( + "D3D11 output failed ({message}); software fallback failed: {fallback_error}" + ), + }); + } + active = None; + feedback = None; + software_playback_started = false; + } + } + } + Ok(OutputEvent::None | OutputEvent::Presented(_)) => {} + Err(message) => { + #[cfg(target_os = "macos")] + if output.is_macos_hardware() { + self.use_macos_hardware.store(false, Ordering::Release); + crate::macos_backend::disable(); + output.stop(); + let fallback = initialize_output( + Arc::clone(&self.output), + surface.as_ref(), + paused, + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + false, + ); + match fallback { + Ok(replacement) => { + *output = replacement; + software_playback_started = false; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "VideoToolbox/Metal/CoreAudio", + to: "OpenH264/SDL", + reason: message, + }); + } + continue; + } + Err(fallback_error) => { + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { + message: format!( + "macOS hardware output failed ({message}); software fallback failed: {fallback_error}" + ), + }); + } + active = None; + feedback = None; + software_playback_started = false; + continue; + } + } + } + #[cfg(target_os = "linux")] + if output.is_linux_hardware() { + output.stop(); + let fallback = initialize_output( + Arc::clone(&self.output), + surface.as_ref(), + paused, + false, + MediaSessionConfig { + stream: active_stream, + audio_device: &active_audio_device, + }, + false, + ); + if let Ok(replacement) = fallback { + *output = replacement; + software_playback_started = false; + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::BackendFallback { + from: "Linux decoder/Vulkan", + to: "Linux decoder/SDL NV12", + reason: format!( + "Linux Vulkan presentation failed: {message}" + ), + }); + } + continue; + } + } + if let Some(feedback) = feedback.as_ref() { + let _ = feedback.send(MediaFeedback::OutputError { message }); + } + output.stop(); + active = None; + feedback = None; + software_playback_started = false; + } + } + } + // The host loop replaces `NSApplication.run()`; without draining AppKit's queue the + // stream window never finishes ordering in and its controls stay dead. + #[cfg(target_os = "macos")] + opennow_streamer_platform_macos::pump_app_events(); + } + } +} + +pub fn create_runtime() -> Result<(MainThreadHost, MediaRuntime), String> { + ensure_macos_main_thread()?; + let use_macos_hardware = Arc::new(AtomicBool::new(use_hardware_backend())); + #[cfg(target_os = "linux")] + let linux_selection = crate::linux_backend::select_video_path(); + #[cfg(target_os = "linux")] + let linux_software_fallback = Arc::new(AtomicBool::new(matches!( + linux_selection.path, + LinuxVideoPath::Software + ))); + let (commands, receiver) = mpsc::channel(); + let output = Arc::new(OutputBuffers::new()); + let paused = Arc::new(AtomicBool::new(false)); + #[cfg(target_os = "windows")] + let windows_bridge = Arc::new(WindowsBridge::new()); + Ok(( + MainThreadHost { + commands: receiver, + output: Arc::clone(&output), + _not_send: PhantomData, + use_macos_hardware: Arc::clone(&use_macos_hardware), + #[cfg(target_os = "windows")] + windows_bridge: Arc::clone(&windows_bridge), + #[cfg(target_os = "linux")] + linux_selection: linux_selection.clone(), + #[cfg(target_os = "linux")] + linux_software_fallback: Arc::clone(&linux_software_fallback), + }, + MediaRuntime { + commands, + audio_query_pending: Arc::new(AtomicBool::new(false)), + output, + paused, + use_hardware: use_macos_hardware, + #[cfg(target_os = "windows")] + windows_bridge, + #[cfg(target_os = "linux")] + linux_selection, + #[cfg(target_os = "linux")] + linux_software_fallback, + mode: MediaRuntimeMode::Standalone, + }, + )) +} + +pub fn create_embedded_runtime(frames: GraphicsFramePublisher) -> MediaRuntime { + create_embedded_runtime_with_input(frames, Arc::new(crate::CapturedInputQueue::default()), None) +} + +pub fn create_embedded_runtime_with_input( + frames: GraphicsFramePublisher, + captured_input: Arc, + cursor_update: Option) + Send + Sync>>, +) -> MediaRuntime { + create_embedded_runtime_with_config( + frames, + captured_input, + cursor_update, + EmbeddedRuntimeConfig::default(), + ) +} + +pub fn create_embedded_runtime_with_config( + frames: GraphicsFramePublisher, + captured_input: Arc, + cursor_update: Option) + Send + Sync>>, + config: EmbeddedRuntimeConfig, +) -> MediaRuntime { + let (commands, receiver) = mpsc::channel(); + let output = Arc::new(OutputBuffers::with_captured_input(captured_input)); + let host_output = Arc::clone(&output); + let _ = std::thread::Builder::new() + .name("opennow-embedded-host".to_owned()) + .spawn(move || { + #[cfg(any(target_os = "windows", test))] + let mut audio = None; + let mut microphone: Option = None; + loop { + let command = if microphone.is_some() { + receiver.recv_timeout(MICROPHONE_HOST_POLL_INTERVAL) + } else { + receiver + .recv() + .map_err(|_| mpsc::RecvTimeoutError::Disconnected) + }; + let command = match command { + Ok(command) => Some(command), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + if let Some(command) = command { + match command { + HostCommand::StartMicrophone { + device_id, + shared, + reply, + } => { + microphone = None; + let result = crate::microphone::MicrophoneCapture::start( + &device_id, + shared, + host_output.microphone_clock(), + ) + .map(|capture| microphone = Some(capture)); + if reply.send(result).is_err() { + microphone = None; + } + } + HostCommand::Cursor(bytes) => { + if let Some(update) = &cursor_update { + update(bytes); + } + } + HostCommand::AudioDevices(request) => { + let reply = &request.reply; + #[cfg(all(target_os = "linux", not(test)))] + let devices = opennow_streamer_platform_linux::audio_output_devices() + .map(|devices| { + devices + .into_iter() + .map(|device| AudioDevice { + id: device.id, + name: device.name, + }) + .collect() + }) + .map_err(|error| error.to_string()); + #[cfg(any(target_os = "windows", test))] + let devices = crate::output::audio_devices(); + #[cfg(all(target_os = "macos", not(test)))] + let devices = opennow_streamer_platform_macos::audio_output_devices() + .map(|devices| { + devices + .into_iter() + .map(|device| AudioDevice { + id: device.id, + name: device.name, + }) + .collect() + }) + .map_err(|error| error.to_string()); + #[cfg(not(any( + target_os = "linux", + target_os = "windows", + target_os = "macos", + test + )))] + let devices = + Err("Native audio device enumeration is unavailable".to_owned()); + let _ = reply.send(devices); + } + #[cfg(any(target_os = "windows", test))] + HostCommand::Start { + reply, + audio_device, + .. + } => { + host_output.stop_microphone(); + microphone = None; + audio = None; + let result = crate::output::HeadlessAudioOutput::start( + Arc::clone(&host_output), + &audio_device, + ) + .map(|started| { + audio = Some(started); + }); + if reply.send(result).is_err() { + audio = None; + } + } + #[cfg(any(target_os = "windows", test))] + HostCommand::Pause { paused, reply } => { + if let Some(audio) = &audio { + audio.set_paused(paused); + } + if let Some(reply) = reply { + let _ = reply.send(Ok(())); + } + } + HostCommand::Stop => { + host_output.stop_microphone(); + microphone = None; + #[cfg(any(target_os = "windows", test))] + { + audio = None; + } + } + HostCommand::Shutdown => break, + _ => {} + } + } + if let Some(capture) = microphone.as_mut() { + capture.poll(); + if capture.stopped() { + microphone = None; + } + } + } + host_output.stop_microphone(); + drop(microphone); + }); + #[cfg(target_os = "linux")] + let linux_selection = crate::linux_backend::select_video_path(); + MediaRuntime { + commands, + audio_query_pending: Arc::new(AtomicBool::new(false)), + output, + paused: Arc::new(AtomicBool::new(false)), + use_hardware: Arc::new(AtomicBool::new(true)), + #[cfg(target_os = "windows")] + windows_bridge: Arc::new(WindowsBridge::new()), + #[cfg(target_os = "linux")] + linux_selection, + #[cfg(target_os = "linux")] + linux_software_fallback: Arc::new(AtomicBool::new(false)), + mode: MediaRuntimeMode::Embedded { frames, config }, + } +} + +#[cfg(feature = "test-runtime")] +pub struct TestMediaRuntimeHost { + worker: Option>, +} + +#[cfg(feature = "test-runtime")] +impl TestMediaRuntimeHost { + pub fn join(mut self) -> std::thread::Result<()> { + self.worker.take().expect("test media worker").join() + } +} + +#[cfg(feature = "test-runtime")] +pub fn create_test_runtime() -> (TestMediaRuntimeHost, MediaRuntime) { + let use_macos_hardware = Arc::new(AtomicBool::new(false)); + #[cfg(target_os = "linux")] + let linux_selection = crate::linux_backend::select_video_path(); + #[cfg(target_os = "linux")] + let linux_software_fallback = Arc::new(AtomicBool::new(matches!( + linux_selection.path, + LinuxVideoPath::Software + ))); + let (commands, receiver) = mpsc::channel(); + let output = Arc::new(OutputBuffers::new()); + let paused = Arc::new(AtomicBool::new(false)); + #[cfg(target_os = "windows")] + let windows_bridge = Arc::new(WindowsBridge::new()); + let worker = std::thread::Builder::new() + .name("opennow-test-media-runtime".to_owned()) + .spawn(move || { + while let Ok(command) = receiver.recv() { + match command { + HostCommand::StartMicrophone { reply, .. } => { + let _ = + reply + .send(Err("test media runtime has no microphone capture device" + .to_owned())); + } + HostCommand::AudioDevices(request) => { + let reply = &request.reply; + let _ = reply.send(Ok(Vec::new())); + } + HostCommand::Start { reply, .. } => { + let _ = reply.send(Err( + "test media runtime does not start media sessions".to_owned() + )); + } + HostCommand::Pause { reply, .. } => { + if let Some(reply) = reply { + let _ = reply.send(Ok(())); + } + } + HostCommand::Surface { reply, .. } | HostCommand::Control { reply, .. } => { + let _ = reply.send(Ok(())); + } + HostCommand::Cursor(_) | HostCommand::Stop => {} + #[cfg(target_os = "macos")] + HostCommand::ConfigureMacH264 { reply, .. } => { + let _ = reply.send(Err("test media runtime has no decoder".to_owned())); + } + #[cfg(target_os = "macos")] + HostCommand::ConfigureMacH265 { reply, .. } => { + let _ = reply.send(Err("test media runtime has no decoder".to_owned())); + } + #[cfg(target_os = "macos")] + HostCommand::ConfigureMacAv1 { reply, .. } => { + let _ = reply.send(Err("test media runtime has no decoder".to_owned())); + } + #[cfg(target_os = "linux")] + HostCommand::FallbackLinux { .. } => {} + HostCommand::Shutdown => break, + } + } + }) + .expect("test media runtime worker"); + ( + TestMediaRuntimeHost { + worker: Some(worker), + }, + MediaRuntime { + commands, + audio_query_pending: Arc::new(AtomicBool::new(false)), + output, + paused, + use_hardware: use_macos_hardware, + #[cfg(target_os = "windows")] + windows_bridge, + #[cfg(target_os = "linux")] + linux_selection, + #[cfg(target_os = "linux")] + linux_software_fallback, + mode: MediaRuntimeMode::Test, + }, + ) +} + +fn initialize_output( + output: Arc, + surface: Option<&RenderSurface>, + paused: bool, + use_hardware: bool, + config: MediaSessionConfig<'_>, + use_linux_hardware: bool, + #[cfg(target_os = "windows")] windows_bridge: Arc, +) -> Result { + let mut output = ActiveOutput::initialize( + output, + use_hardware, + config, + #[cfg(target_os = "windows")] + windows_bridge, + use_linux_hardware, + )?; + if let Err(error) = output.start(surface) { + output.stop(); + return Err(error); + } + if let Err(error) = output.set_paused(paused) { + output.stop(); + return Err(error); + } + Ok(output) +} + +#[cfg(target_os = "macos")] +fn use_hardware_backend() -> bool { + backend_preference_allows("videotoolbox") && crate::macos_backend::available() +} + +#[cfg(target_os = "windows")] +fn use_hardware_backend() -> bool { + backend_preference_allows("d3d12") || backend_preference_allows("d3d11") +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +const fn use_hardware_backend() -> bool { + false +} + +#[cfg(any(target_os = "windows", target_os = "macos"))] +pub(crate) fn backend_preference_allows(hardware_backend: &str) -> bool { + backend_preference_allows_value( + std::env::var("OPENNOW_NATIVE_VIDEO_BACKEND") + .ok() + .as_deref(), + hardware_backend, + ) +} + +#[cfg(any(target_os = "windows", target_os = "macos", test))] +fn backend_preference_allows_value(value: Option<&str>, hardware_backend: &str) -> bool { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .is_none_or(|value| { + let value = value.to_ascii_lowercase(); + value == "auto" || value == "hardware" || value == hardware_backend + }) +} + +#[cfg(target_os = "windows")] +fn current_video_mid(bridge: &WindowsBridge) -> String { + bridge.last_video_mid() +} + +#[cfg(target_os = "windows")] +const fn hardware_backend_label() -> &'static str { + "Media Foundation/Direct3D/WASAPI" +} + +#[cfg(target_os = "macos")] +const fn hardware_backend_label() -> &'static str { + "VideoToolbox/Metal/CoreAudio" +} + +#[cfg(not(any(target_os = "windows", target_os = "macos")))] +const fn hardware_backend_label() -> &'static str { + "Linux decoder/Vulkan" +} + +#[cfg(target_os = "macos")] +fn ensure_macos_main_thread() -> Result<(), String> { + if unsafe { libc::pthread_main_np() } == 1 { + Ok(()) + } else { + Err( + "the macOS native media host must be created and run on the process main thread" + .to_owned(), + ) + } +} + +#[cfg(not(target_os = "macos"))] +fn ensure_macos_main_thread() -> Result<(), String> { + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn embedded_runtime_snapshots_windows_adapter_selection() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let luid = crate::WindowsAdapterLuid::new(0xffff_fffe_1122_3344).expect("non-zero LUID"); + let runtime = super::create_embedded_runtime_with_config( + frames, + std::sync::Arc::new(crate::CapturedInputQueue::default()), + None, + super::EmbeddedRuntimeConfig { + vulkan_device: None, + windows_adapter_luid: Some(luid), + }, + ); + let super::MediaRuntimeMode::Embedded { config, .. } = &runtime.mode else { + panic!("embedded runtime mode") + }; + assert_eq!(config.windows_adapter_luid, Some(luid)); + assert_eq!(runtime.windows_adapter_luid(), Some(luid)); + } + + #[test] + #[ignore = "requires SDL_AUDIODRIVER=dummy in an isolated test process"] + fn embedded_host_shares_sdl_playback_capture_and_enumeration_lifetimes() { + use super::{HostCommand, MediaRuntime}; + use std::sync::{Arc, mpsc}; + use std::time::{Duration, Instant}; + + fn wait_until(mut condition: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(2); + while !condition() { + assert!( + Instant::now() < deadline, + "embedded audio host did not settle" + ); + std::thread::sleep(Duration::from_millis(2)); + } + } + + fn start_playback( + runtime: &MediaRuntime, + device: &opennow_streamer_protocol::AudioOutputDevice, + ) { + let (reply, response) = mpsc::channel(); + let (feedback, _) = mpsc::channel(); + runtime + .commands + .send(HostCommand::Start { + reply, + feedback, + stream: crate::MediaStreamConfig::default(), + audio_device: device.clone(), + }) + .unwrap(); + response + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + } + + fn playback_consumes(runtime: &MediaRuntime) { + runtime.output.push_audio(&[0.25; 4096]); + wait_until(|| !runtime.output.audio_samples_pending()); + } + + fn next_microphone(receiver: &crate::MicrophoneReceiver) -> u32 { + let mut timestamp = None; + wait_until(|| { + timestamp = receiver.try_recv().unwrap().map(|frame| frame.timestamp); + timestamp.is_some() + }); + timestamp.unwrap() + } + + assert_eq!(std::env::var("SDL_AUDIODRIVER").as_deref(), Ok("dummy")); + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let (cursor_sent, cursor_received) = mpsc::channel(); + let runtime = super::create_embedded_runtime_with_input( + frames, + Arc::new(crate::CapturedInputQueue::default()), + Some(Arc::new(move |bytes| { + let _ = cursor_sent.send(bytes); + })), + ); + let devices = runtime.audio_devices().unwrap(); + let selected = opennow_streamer_protocol::AudioOutputDevice::new( + devices.first().expect("dummy playback device").id.clone(), + ) + .unwrap(); + start_playback(&runtime, &selected); + playback_consumes(&runtime); + assert!( + sdl2::init().is_err(), + "the embedded host must remain SDL's sole initialization thread" + ); + + let mut microphone = runtime.start_microphone("").unwrap(); + let receiver = microphone.receiver(); + let first = next_microphone(&receiver); + assert_eq!(runtime.audio_devices().unwrap(), devices); + playback_consumes(&runtime); + for paused in [true, false] { + let (reply, response) = mpsc::channel(); + runtime + .commands + .send(HostCommand::Pause { + paused, + reply: Some(reply), + }) + .unwrap(); + response + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + next_microphone(&receiver); + assert!(receiver.status().enabled); + } + playback_consumes(&runtime); + + let stop_receiver = microphone.receiver(); + std::thread::spawn(move || stop_receiver.stop()) + .join() + .unwrap(); + wait_until(|| !receiver.capture_device_open()); + assert!(!receiver.status().enabled); + assert_eq!(receiver.try_recv(), Ok(None)); + assert_eq!(runtime.audio_devices().unwrap(), devices); + assert!(!receiver.capture_device_open()); + playback_consumes(&runtime); + microphone.stop(); + assert!(runtime.start_microphone("unsupported-input").is_err()); + playback_consumes(&runtime); + + let restarted = runtime.start_microphone("").unwrap(); + let restarted_receiver = restarted.receiver(); + assert!( + next_microphone(&restarted_receiver).wrapping_sub(first) + >= crate::MICROPHONE_FRAME_SAMPLES as u32 + ); + receiver.stop(); + assert!(restarted_receiver.status().enabled); + playback_consumes(&runtime); + + restarted_receiver.pause_capture_device(); + wait_until(|| !restarted_receiver.capture_device_open()); + assert!(!restarted_receiver.status().enabled); + assert!(restarted_receiver.status().error.is_some()); + playback_consumes(&runtime); + let terminal_microphone = runtime.start_microphone("").unwrap(); + let terminal_receiver = terminal_microphone.receiver(); + next_microphone(&terminal_receiver); + + runtime.commands.send(HostCommand::Stop).unwrap(); + runtime.commands.send(HostCommand::Cursor(vec![1])).unwrap(); + assert_eq!( + cursor_received + .recv_timeout(Duration::from_secs(2)) + .unwrap(), + vec![1] + ); + assert!(!restarted_receiver.status().enabled); + assert!(!restarted_receiver.capture_device_open()); + assert!(!terminal_receiver.status().enabled); + assert!(!terminal_receiver.capture_device_open()); + runtime.output.push_audio(&[0.25; 4096]); + std::thread::sleep(Duration::from_millis(50)); + assert!(runtime.output.audio_samples_pending()); + + start_playback(&runtime, &selected); + let final_microphone = runtime.start_microphone("").unwrap(); + let final_receiver = final_microphone.receiver(); + next_microphone(&final_receiver); + runtime.shutdown(); + assert!(!final_receiver.status().enabled); + assert_eq!( + cursor_received.recv_timeout(Duration::from_secs(2)), + Err(mpsc::RecvTimeoutError::Disconnected) + ); + assert!(!final_receiver.capture_device_open()); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn timed_out_audio_query_does_not_queue_more_requests() { + let (host, mut runtime) = super::create_test_runtime(); + let original = runtime.clone(); + let (commands, receiver) = std::sync::mpsc::channel(); + runtime.commands = commands; + assert!(runtime.audio_devices().unwrap_err().contains("timed out")); + assert!( + runtime + .audio_devices() + .unwrap_err() + .contains("already in progress") + ); + drop(receiver.try_recv().expect("one queued query")); + assert!(receiver.try_recv().is_err()); + assert!( + !runtime + .audio_query_pending + .load(std::sync::atomic::Ordering::Acquire) + ); + drop(receiver); + assert!( + runtime + .audio_devices() + .unwrap_err() + .contains("no longer running") + ); + assert!( + !runtime + .audio_query_pending + .load(std::sync::atomic::Ordering::Acquire) + ); + original.shutdown(); + host.join().expect("test host"); + } + + use super::backend_preference_allows_value; + + #[test] + fn hdr_rejects_h264_and_eight_bit_before_decoder_start() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + for (codec, color_quality) in [ + ( + crate::MediaVideoCodec::H264, + crate::MediaColorQuality::TenBit420, + ), + ( + crate::MediaVideoCodec::H265, + crate::MediaColorQuality::EightBit420, + ), + ( + crate::MediaVideoCodec::Av1, + crate::MediaColorQuality::EightBit444, + ), + ] { + let (feedback, _) = std::sync::mpsc::channel(); + let result = runtime.start( + feedback, + crate::MediaStreamConfig { + codec, + color_quality, + hdr: true, + ..Default::default() + }, + ); + assert!(matches!(result, Err(message) if message.contains("10-bit HEVC or AV1"))); + } + runtime.shutdown(); + } + + #[test] + fn hdr_444_fails_before_decoder_start() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let mut runtime = super::create_embedded_runtime(frames); + runtime.mode = super::MediaRuntimeMode::Standalone; + for codec in [crate::MediaVideoCodec::H265, crate::MediaVideoCodec::Av1] { + let (feedback, _) = std::sync::mpsc::channel(); + let result = runtime.start( + feedback, + crate::MediaStreamConfig { + codec, + color_quality: crate::MediaColorQuality::TenBit444, + hdr: true, + ..Default::default() + }, + ); + assert!(matches!(result, Err(message) if message.contains("10-bit 4:2:0"))); + } + runtime.shutdown(); + } + + #[cfg(target_os = "linux")] + #[test] + fn hdr_444_without_shared_vulkan_device_never_falls_back_to_cpu() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + let stream = crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + color_quality: crate::MediaColorQuality::TenBit444, + hdr: true, + ..Default::default() + }; + assert!(!runtime.supports_hdr_444(stream)); + let (feedback, _) = std::sync::mpsc::channel(); + assert!(matches!(runtime.start(feedback, stream), Err(message) + if message.contains("4:4:4 hardware decode and GPU output"))); + runtime.shutdown(); + } + + #[test] + fn incompatible_color_profiles_fail_before_decoder_start() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + for (codec, color_quality) in [ + ( + crate::MediaVideoCodec::H264, + crate::MediaColorQuality::EightBit444, + ), + ( + crate::MediaVideoCodec::H264, + crate::MediaColorQuality::TenBit420, + ), + ( + crate::MediaVideoCodec::H264, + crate::MediaColorQuality::TenBit444, + ), + ( + crate::MediaVideoCodec::Av1, + crate::MediaColorQuality::EightBit444, + ), + ( + crate::MediaVideoCodec::Av1, + crate::MediaColorQuality::TenBit444, + ), + ] { + let (feedback, _) = std::sync::mpsc::channel(); + let result = runtime.start( + feedback, + crate::MediaStreamConfig { + codec, + color_quality, + ..Default::default() + }, + ); + assert!(matches!(result, Err(message) if message.contains("cannot be preserved"))); + } + runtime.shutdown(); + } + + #[test] + fn standalone_output_never_presents_hdr_as_sdr() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let mut runtime = super::create_embedded_runtime(frames); + runtime.mode = super::MediaRuntimeMode::Standalone; + let (feedback, _) = std::sync::mpsc::channel(); + let result = runtime.start( + feedback, + crate::MediaStreamConfig { + codec: crate::MediaVideoCodec::H265, + color_quality: crate::MediaColorQuality::TenBit420, + hdr: true, + ..Default::default() + }, + ); + assert!(matches!(result, Err(message) if message.contains("embedded"))); + runtime.shutdown(); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn audio_mute_survives_session_restart_without_pausing_media_or_microphone() { + use std::sync::atomic::Ordering; + + let (host, runtime) = super::create_test_runtime(); + runtime.set_audio_muted(true); + for _ in 0..2 { + let (feedback, _) = std::sync::mpsc::channel(); + let media = runtime + .start(feedback, crate::MediaStreamConfig::default()) + .unwrap(); + assert!(runtime.output.audio_muted.load(Ordering::Acquire)); + assert!(!runtime.paused.load(Ordering::Acquire)); + let shared = crate::microphone::MicrophoneShared::new(); + let microphone = crate::MicrophoneSession::from_shared(std::sync::Arc::clone(&shared)); + runtime.output.set_microphone(&shared); + let clock = runtime.output.microphone_clock(); + for muted in [false, true] { + runtime.set_audio_muted(muted); + assert_eq!(runtime.output.audio_muted.load(Ordering::Acquire), muted); + assert!(!runtime.paused.load(Ordering::Acquire)); + assert!(microphone.receiver().status().enabled); + assert_eq!(runtime.output.microphone_clock(), clock); + } + media.stop(); + assert!(runtime.output.audio_muted.load(Ordering::Acquire)); + } + runtime.shutdown(); + host.join().unwrap(); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn audio_device_start_resets_microphone_but_enumeration_does_not() { + let (host, runtime) = super::create_test_runtime(); + let shared = crate::microphone::MicrophoneShared::new(); + let microphone = crate::MicrophoneSession::from_shared(std::sync::Arc::clone(&shared)); + runtime.output.set_microphone(&shared); + let receiver = microphone.receiver(); + let previous_clock = runtime.output.microphone_clock(); + assert_eq!(runtime.audio_devices().unwrap(), Vec::new()); + assert!(receiver.status().enabled); + assert_eq!(runtime.output.microphone_clock(), previous_clock); + std::thread::sleep(std::time::Duration::from_millis(1)); + let (feedback, _) = std::sync::mpsc::channel(); + let media = runtime + .start_with_audio_device( + feedback, + crate::MediaStreamConfig::default(), + "auto", + opennow_streamer_protocol::AudioOutputDevice::default(), + ) + .unwrap(); + assert!(!receiver.status().enabled); + assert!(runtime.output.microphone_clock() > previous_clock); + drop(media); + runtime.shutdown(); + host.join().unwrap(); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn terminal_media_stop_synchronously_disables_microphone_receiver() { + let (host, runtime) = super::create_test_runtime(); + let (feedback, _) = std::sync::mpsc::channel(); + let media = runtime + .start(feedback, crate::MediaStreamConfig::default()) + .unwrap(); + let shared = crate::microphone::MicrophoneShared::new(); + let session = crate::MicrophoneSession::from_shared(std::sync::Arc::clone(&shared)); + runtime.output.set_microphone(&shared); + let receiver = session.receiver(); + assert!(receiver.status().enabled); + media.control().stop(); + assert!(!receiver.status().enabled); + assert_eq!(receiver.try_recv(), Ok(None)); + drop(media); + runtime.shutdown(); + host.join().unwrap(); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn test_runtime_reports_no_capture_device_instead_of_enabled() { + let (host, runtime) = super::create_test_runtime(); + let error = runtime + .start_microphone("") + .err() + .expect("test host has no device"); + assert!(error.contains("no microphone capture device")); + runtime.shutdown(); + host.join().unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn embedded_runtime_without_an_owner_disables_vulkan() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + let backends = runtime.video_backends(); + let vulkan = backends + .iter() + .find(|backend| backend.backend == "vulkan") + .unwrap(); + assert!(!vulkan.available); + assert!(vulkan.codecs.iter().all(|codec| !codec.available)); + assert!(runtime.validate_backend("vulkan").is_err()); + assert!(runtime.validate_backend("auto").is_ok()); + runtime.shutdown(); + } + + #[cfg(target_os = "linux")] + #[test] + fn embedded_runtime_accepts_software_only_when_the_cpu_decoder_is_available() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + let software_available = runtime + .video_backends() + .iter() + .any(|backend| backend.backend == "ffmpeg" && backend.available); + for requested in ["software", "ffmpeg"] { + assert_eq!( + runtime.validate_backend(requested).is_ok(), + software_available, + "{requested}" + ); + } + for unsupported in ["hardware", "d3d12", "videotoolbox", "invalid"] { + assert!( + runtime.validate_backend(unsupported).is_err(), + "{unsupported}" + ); + } + runtime.shutdown(); + } + + #[cfg(target_os = "windows")] + #[test] + fn embedded_windows_rejects_unimplemented_backend_overrides() { + let (_graphics, frames) = crate::RenderThreadGraphics::new(|| {}); + let runtime = super::create_embedded_runtime(frames); + assert!(runtime.validate_backend("auto").is_ok()); + assert!(runtime.validate_backend("d3d11").is_ok()); + for backend in ["d3d12", "vulkan", "software", "invalid"] { + assert!( + runtime + .validate_backend(backend) + .unwrap_err() + .contains("Select Auto") + ); + } + runtime.shutdown(); + } + + #[test] + fn video_backend_preference_selects_only_auto_or_the_requested_hardware() { + assert!(backend_preference_allows_value(None, "d3d11")); + assert!(backend_preference_allows_value(Some(""), "d3d11")); + assert!(backend_preference_allows_value(Some("AUTO"), "d3d11")); + assert!(backend_preference_allows_value(Some("d3d11"), "d3d11")); + assert!(!backend_preference_allows_value(Some("software"), "d3d11")); + assert!(!backend_preference_allows_value(Some("d3d12"), "d3d11")); + assert!(backend_preference_allows_value(Some("d3d12"), "d3d12")); + assert!(!backend_preference_allows_value(Some("d3d11"), "d3d12")); + } + + #[cfg(feature = "test-runtime")] + #[test] + fn embedded_style_start_creates_a_live_media_consumer_without_a_render_surface() { + use std::sync::Arc; + use std::sync::mpsc; + use std::time::Duration; + + use crate::{EncodedFrame, MediaCodec, MediaFeedback, MediaStreamConfig, PushOutcome}; + + let (host, runtime) = super::create_test_runtime(); + let (feedback, received) = mpsc::channel(); + let session = runtime + .start(feedback, MediaStreamConfig::default()) + .expect("headless media session starts without a RenderSurface"); + assert_eq!( + session.sink().push(EncodedFrame { + mid: "video".to_owned(), + codec: MediaCodec::H264, + data: Arc::from([0_u8, 0, 0, 1, 0x65]), + frame_index: Some(9), + timestamp: 90_000, + clock_rate_hz: 90_000, + keyframe: true, + contiguous: true, + ssrc: None, + }), + PushOutcome::Queued + ); + assert!(matches!( + received.recv_timeout(Duration::from_secs(2)), + Ok(MediaFeedback::VideoFrameAccepted { .. }) + )); + + session.stop(); + runtime.shutdown(); + host.join().expect("test media runtime"); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/video_queue.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/video_queue.rs new file mode 100644 index 000000000..8f0165435 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/video_queue.rs @@ -0,0 +1,360 @@ +//! Compressed video cannot use an audio-style drop-oldest queue: every retained +//! delta may depend on the removed frame. Admission and loss epochs share one lock. +use std::collections::VecDeque; +use std::sync::{Condvar, Mutex}; + +use crate::media::EncodedFrame; + +pub(crate) struct VideoPacket { + pub frame: EncodedFrame, + #[cfg(any(windows, test))] + pub generation: u64, + pub reset_decoder: bool, +} + +pub(crate) struct VideoPush { + pub dropped: usize, + pub request_keyframe: bool, +} + +struct State { + frames: VecDeque, + generation: u64, + waiting_for_keyframe: bool, + request_pending: bool, + closed: bool, +} + +pub(crate) struct VideoQueue { + capacity: usize, + state: Mutex, + ready: Condvar, +} + +impl VideoQueue { + pub fn new(capacity: usize) -> Self { + assert!(capacity > 0); + Self { + capacity, + state: Mutex::new(State { + frames: VecDeque::with_capacity(capacity), + generation: 1, + waiting_for_keyframe: true, + request_pending: false, + closed: false, + }), + ready: Condvar::new(), + } + } + + fn invalidate_locked(state: &mut State) -> usize { + let dropped = state.frames.len(); + state.frames.clear(); + state.generation = state.generation.wrapping_add(1); + state.waiting_for_keyframe = true; + state.request_pending = false; + dropped + } + + pub fn push(&self, frame: EncodedFrame) -> Result { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.closed { + return Err(()); + } + let mut dropped = 0; + if !state.waiting_for_keyframe && (!frame.contiguous || state.frames.len() == self.capacity) + { + if frame.contiguous + && !frame.keyframe + && state.frames.iter().any(|queued| queued.frame.keyframe) + { + state.waiting_for_keyframe = true; + state.request_pending = true; + return Ok(VideoPush { + dropped: 1, + request_keyframe: true, + }); + } + dropped += Self::invalidate_locked(&mut state); + } + if state.waiting_for_keyframe && !frame.keyframe { + let request_keyframe = !state.request_pending; + state.request_pending = true; + return Ok(VideoPush { + dropped: dropped + 1, + request_keyframe, + }); + } + if state.waiting_for_keyframe && !state.frames.is_empty() { + dropped += Self::invalidate_locked(&mut state); + } + let reset_decoder = state.waiting_for_keyframe; + state.waiting_for_keyframe = false; + state.request_pending = false; + #[cfg(any(windows, test))] + let generation = state.generation; + state.frames.push_back(VideoPacket { + frame, + #[cfg(any(windows, test))] + generation, + reset_decoder, + }); + self.ready.notify_one(); + Ok(VideoPush { + dropped, + request_keyframe: false, + }) + } + + pub fn pop_packet(&self) -> Option { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + loop { + if let Some(frame) = state.frames.pop_front() { + return Some(frame); + } + if state.closed { + return None; + } + state = self + .ready + .wait(state) + .unwrap_or_else(|error| error.into_inner()); + } + } + + // Compatibility for other platform consumers: local queue loss is a decode + // discontinuity too. Sender IDs/timestamps and recording input stay untouched. + pub fn pop(&self) -> Option { + self.pop_packet().map(|packet| { + let mut frame = packet.frame; + frame.contiguous &= !packet.reset_decoder; + frame + }) + } + + /// The closure may only enqueue compressed data, never run codec/GPU work. + /// A loss cannot interleave between epoch validation and downstream admission. + #[cfg(any(windows, test))] + pub fn submit_if_current(&self, generation: u64, enqueue: impl FnOnce() -> T) -> Option { + let state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.closed || state.generation != generation { + return None; + } + Some(enqueue()) + } + + pub fn clear(&self) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + Self::invalidate_locked(&mut state); + } + + /// The caller already asked for one IDR. Keep discarding deltas until that + /// IDR arrives, and do not ask again for each discarded delta. + /// + /// Embedded D3D11 is the production caller. The same gate is covered by unit tests. + #[cfg(any(windows, test))] + pub fn hold_keyframe_request(&self) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.waiting_for_keyframe = true; + state.request_pending = true; + } + + pub fn close(&self) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + Self::invalidate_locked(&mut state); + state.closed = true; + self.ready.notify_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::media::MediaCodec; + use std::sync::Arc; + + fn frame(id: u32, keyframe: bool) -> EncodedFrame { + EncodedFrame { + mid: "video".into(), + codec: MediaCodec::H264, + data: Arc::from([1]), + frame_index: Some(id), + timestamp: u64::from(id) * 750, + clock_rate_hz: 90_000, + keyframe, + contiguous: true, + ssrc: None, + } + } + + #[test] + fn old_keyframe_cannot_reopen_a_newer_missing_reference_chain() { + let queue = VideoQueue::new(2); + queue.push(frame(1, true)).unwrap(); + let old_keyframe = queue.pop_packet().unwrap(); + queue.push(frame(2, false)).unwrap(); + queue.push(frame(3, false)).unwrap(); + let loss = queue.push(frame(4, false)).unwrap(); + assert_eq!(loss.dropped, 3); + assert!(loss.request_keyframe); + assert!( + queue + .submit_if_current(old_keyframe.generation, || panic!("stale submission")) + .is_none() + ); + let dependent = queue.push(frame(5, false)).unwrap(); + assert_eq!(dependent.dropped, 1); + assert!(!dependent.request_keyframe); + queue.push(frame(6, true)).unwrap(); + queue.push(frame(7, false)).unwrap(); + let fresh = queue.pop_packet().unwrap(); + assert!(fresh.reset_decoder); + assert_eq!(fresh.frame.frame_index, Some(6)); + assert_eq!(fresh.frame.timestamp, 4500); + assert_eq!( + queue.submit_if_current(fresh.generation, || true), + Some(true) + ); + let delta = queue.pop_packet().unwrap(); + assert_eq!(delta.frame.frame_index, Some(7)); + assert!(!delta.reset_decoder); + } + + #[test] + fn packet_gap_discards_dependents_and_accepts_a_complete_keyframe() { + let queue = VideoQueue::new(3); + queue.push(frame(1, true)).unwrap(); + queue.push(frame(2, false)).unwrap(); + let mut recovery = frame(4, true); + recovery.contiguous = false; + let result = queue.push(recovery).unwrap(); + assert_eq!(result.dropped, 2); + assert!(!result.request_keyframe); + let packet = queue.pop_packet().unwrap(); + assert_eq!(packet.frame.frame_index, Some(4)); + assert!(packet.reset_decoder); + } + + #[test] + fn missing_reference_discards_the_chain_until_recovery() { + let queue = VideoQueue::new(3); + queue.push(frame(1, true)).unwrap(); + queue.push(frame(2, false)).unwrap(); + let mut missing_reference = frame(4, false); + missing_reference.contiguous = false; + let gap = queue.push(missing_reference).unwrap(); + assert_eq!(gap.dropped, 3); + assert!(gap.request_keyframe); + let dependent = queue.push(frame(5, false)).unwrap(); + assert_eq!(dependent.dropped, 1); + assert!(!dependent.request_keyframe); + assert_eq!(queue.push(frame(6, true)).unwrap().dropped, 0); + queue.push(frame(7, false)).unwrap(); + let recovery = queue.pop_packet().unwrap(); + assert_eq!(recovery.frame.frame_index, Some(6)); + assert!(recovery.reset_decoder); + let following = queue.pop_packet().unwrap(); + assert_eq!(following.frame.frame_index, Some(7)); + assert!(!following.reset_decoder); + } + + #[test] + fn held_keyframe_request_does_not_repeat_for_each_delta() { + let queue = VideoQueue::new(2); + queue.push(frame(1, true)).unwrap(); + queue.clear(); + queue.hold_keyframe_request(); + let delta = queue.push(frame(2, false)).unwrap(); + assert_eq!(delta.dropped, 1); + assert!(!delta.request_keyframe); + let idr = queue.push(frame(3, true)).unwrap(); + assert!(!idr.request_keyframe); + assert!(queue.pop_packet().unwrap().reset_decoder); + } + + #[test] + fn overflow_retains_incoming_keyframe_without_extra_round_trip() { + let queue = VideoQueue::new(2); + queue.push(frame(1, true)).unwrap(); + queue.push(frame(2, false)).unwrap(); + let result = queue.push(frame(3, true)).unwrap(); + assert_eq!(result.dropped, 2); + assert!(!result.request_keyframe); + assert_eq!(queue.pop_packet().unwrap().frame.frame_index, Some(3)); + } + + #[test] + fn overflow_preserves_queued_reference_and_gates_dependent_frames() { + let queue = VideoQueue::new(3); + queue.push(frame(1, true)).unwrap(); + queue.push(frame(2, false)).unwrap(); + queue.push(frame(3, false)).unwrap(); + let loss = queue.push(frame(4, false)).unwrap(); + assert_eq!(loss.dropped, 1); + assert!(loss.request_keyframe); + assert_eq!(queue.push(frame(5, false)).unwrap().dropped, 1); + assert!(!queue.push(frame(6, false)).unwrap().request_keyframe); + let first = queue.pop_packet().unwrap(); + assert_eq!(first.frame.frame_index, Some(1)); + assert!(first.reset_decoder); + assert_eq!(queue.pop_packet().unwrap().frame.frame_index, Some(2)); + assert_eq!(queue.pop_packet().unwrap().frame.frame_index, Some(3)); + queue.push(frame(7, true)).unwrap(); + assert!(queue.pop_packet().unwrap().reset_decoder); + } + + #[test] + fn recovery_keyframe_replaces_unconsumed_chain_after_overflow() { + let queue = VideoQueue::new(2); + queue.push(frame(1, true)).unwrap(); + queue.push(frame(2, false)).unwrap(); + assert!(queue.push(frame(3, false)).unwrap().request_keyframe); + let old = queue.pop_packet().unwrap(); + let recovery = queue.push(frame(4, true)).unwrap(); + assert_eq!(recovery.dropped, 1); + assert!(!recovery.request_keyframe); + assert!(queue.submit_if_current(old.generation, || ()).is_none()); + let packet = queue.pop_packet().unwrap(); + assert_eq!(packet.frame.frame_index, Some(4)); + assert!(packet.reset_decoder); + } + + #[test] + fn clear_and_close_invalidate_in_flight_frames() { + let queue = VideoQueue::new(2); + queue.push(frame(1, true)).unwrap(); + let old = queue.pop_packet().unwrap(); + queue.clear(); + assert!(queue.submit_if_current(old.generation, || ()).is_none()); + assert!(queue.push(frame(2, false)).unwrap().request_keyframe); + queue.push(frame(3, true)).unwrap(); + let new = queue.pop_packet().unwrap(); + queue.close(); + assert!(queue.submit_if_current(new.generation, || ()).is_none()); + assert!(queue.pop_packet().is_none()); + assert!(queue.push(frame(4, true)).is_err()); + } + + #[test] + fn loss_and_downstream_admission_share_the_same_lock() { + let queue = Arc::new(VideoQueue::new(2)); + queue.push(frame(1, true)).unwrap(); + let packet = queue.pop_packet().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let worker_queue = Arc::clone(&queue); + let submit = std::thread::spawn(move || { + worker_queue.submit_if_current(packet.generation, || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }) + }); + entered_rx.recv().unwrap(); + // Loss cannot become visible in the middle of downstream admission. + assert!(queue.state.try_lock().is_err()); + release_tx.send(()).unwrap(); + assert_eq!(submit.join().unwrap(), Some(())); + queue.clear(); + assert!(queue.submit_if_current(packet.generation, || ()).is_none()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_graphics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_graphics.rs new file mode 100644 index 000000000..fb0a27745 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_graphics.rs @@ -0,0 +1,74 @@ +use opennow_streamer_platform_windows::{AdoptedD3d11Context, D3d11Frame}; + +use crate::{ + GraphicsApi, GraphicsColorSpace, GraphicsContext, GraphicsFrame, GraphicsFrameInfo, + GraphicsRecordCommand, GraphicsRecordedFrame, GraphicsTextureFormat, +}; + +impl From for GraphicsColorSpace { + fn from(value: opennow_streamer_platform_windows::D3d11ColorSpace) -> Self { + match value { + opennow_streamer_platform_windows::D3d11ColorSpace::Sdr709 => Self::Sdr709, + opennow_streamer_platform_windows::D3d11ColorSpace::Pq2020 => Self::Pq2020, + opennow_streamer_platform_windows::D3d11ColorSpace::Hlg2020 => Self::Hlg2020, + } + } +} + +impl GraphicsFrame for D3d11Frame { + fn info(&self) -> GraphicsFrameInfo { + GraphicsFrameInfo { + width: self.width(), + height: self.height(), + sequence: self.sequence(), + presentation_time_ns: self.presentation_time_ns(), + } + } + + fn record( + &self, + context: GraphicsContext, + command: GraphicsRecordCommand, + ) -> Result { + if context.api != GraphicsApi::D3d11 { + return Err("a Windows decoded frame requires a D3D11 graphics context" + .to_owned() + .into()); + } + if command.command_buffer != context.queue { + return Err( + "D3D11 record command does not use Qt's adopted immediate context" + .to_owned() + .into(), + ); + } + let frame = unsafe { + self.record( + AdoptedD3d11Context { + device: context.device as *mut std::ffi::c_void, + immediate_context: context.queue as *mut std::ffi::c_void, + }, + command.frame_slot, + ) + } + .map_err(|error| error.to_string())?; + Ok(GraphicsRecordedFrame { + resource: frame.texture as usize as u64, + resource_view: 0, + color_space: frame.color_space.into(), + texture_format: match frame.texture_format { + opennow_streamer_platform_windows::D3d11TextureFormat::Rgba8 => { + GraphicsTextureFormat::Rgba8 + } + opennow_streamer_platform_windows::D3d11TextureFormat::Rgb10A2 => { + GraphicsTextureFormat::Rgb10A2 + } + }, + width: frame.width, + height: frame.height, + frame_slot: frame.frame_slot, + generation: frame.generation, + presentation_time_ns: frame.presentation_time_ns, + }) + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_raw_input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_raw_input.rs new file mode 100644 index 000000000..375ed3219 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-platform/src/windows_raw_input.rs @@ -0,0 +1,472 @@ +use std::collections::HashSet; +use std::ffi::c_void; +use std::mem::{MaybeUninit, size_of}; +use std::ptr::{null, null_mut}; +use std::sync::atomic::{AtomicBool, AtomicIsize, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; + +use windows_sys::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; +use windows_sys::Win32::System::LibraryLoader::GetModuleHandleW; +use windows_sys::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_ABOVE_NORMAL, +}; +use windows_sys::Win32::UI::Input::{ + GetRawInputData, HRAWINPUT, MOUSE_MOVE_ABSOLUTE, RAWINPUT, RAWINPUTDEVICE, RAWINPUTHEADER, + RID_INPUT, RIDEV_INPUTSINK, RIDEV_REMOVE, RIM_TYPEMOUSE, RegisterRawInputDevices, +}; +use windows_sys::Win32::UI::WindowsAndMessaging::{ + CREATESTRUCTW, CreateWindowExW, DefWindowProcW, DestroyWindow, DispatchMessageW, GWLP_USERDATA, + GetForegroundWindow, GetMessageW, GetWindowLongPtrW, HWND_MESSAGE, MSG, PostMessageW, + PostQuitMessage, RI_MOUSE_BUTTON_4_DOWN, RI_MOUSE_BUTTON_4_UP, RI_MOUSE_BUTTON_5_DOWN, + RI_MOUSE_BUTTON_5_UP, RI_MOUSE_HWHEEL, RI_MOUSE_LEFT_BUTTON_DOWN, RI_MOUSE_LEFT_BUTTON_UP, + RI_MOUSE_MIDDLE_BUTTON_DOWN, RI_MOUSE_MIDDLE_BUTTON_UP, RI_MOUSE_RIGHT_BUTTON_DOWN, + RI_MOUSE_RIGHT_BUTTON_UP, RI_MOUSE_WHEEL, RegisterClassW, SetWindowLongPtrW, TranslateMessage, + WM_APP, WM_CLOSE, WM_INPUT, WM_NCCREATE, WM_NCDESTROY, WNDCLASSW, +}; + +use crate::media::{CapturedInput, CapturedInputQueue}; + +const RAW_INPUT_CLASS: &[u16] = &[ + b'O' as u16, + b'p' as u16, + b'e' as u16, + b'n' as u16, + b'N' as u16, + b'O' as u16, + b'W' as u16, + b'R' as u16, + b'a' as u16, + b'w' as u16, + b'I' as u16, + b'n' as u16, + b'p' as u16, + b'u' as u16, + b't' as u16, + 0, +]; +const WM_RAW_INPUT_REREGISTER: u32 = WM_APP + 1; + +struct RawInputState { + foreground_owner: AtomicIsize, + enabled: AtomicBool, + relative_motion: AtomicBool, + pressed_buttons: Mutex>, + captured_input: Arc, +} + +pub(crate) struct WindowsRawInputController { + state: Arc, + message_window: isize, + worker: Option>, +} + +impl WindowsRawInputController { + pub(crate) fn start( + foreground_owner: isize, + captured_input: Arc, + ) -> Result { + let state = Arc::new(RawInputState { + foreground_owner: AtomicIsize::new(foreground_owner), + enabled: AtomicBool::new(false), + relative_motion: AtomicBool::new(false), + pressed_buttons: Mutex::new(HashSet::new()), + captured_input, + }); + let thread_state = Arc::clone(&state); + let (ready_sender, ready_receiver) = mpsc::sync_channel(1); + let worker = thread::Builder::new() + .name("opennow-raw-input".to_owned()) + .spawn(move || run_raw_input_thread(thread_state, ready_sender)) + .map_err(|error| format!("failed to spawn Raw Input thread: {error}"))?; + let message_window = match ready_receiver.recv() { + Ok(Ok(hwnd)) => hwnd, + Ok(Err(error)) => { + let _ = worker.join(); + return Err(error); + } + Err(_) => { + let _ = worker.join(); + return Err("Raw Input thread exited during initialization".to_owned()); + } + }; + Ok(Self { + state, + message_window, + worker: Some(worker), + }) + } + + pub(crate) fn set_foreground_owner(&self, foreground_owner: isize) { + self.state + .foreground_owner + .store(foreground_owner, Ordering::Release); + } + + pub(crate) fn set_capture(&self, enabled: bool, relative_motion: bool) { + let motion_changed = self + .state + .relative_motion + .swap(relative_motion, Ordering::AcqRel) + != relative_motion; + let enabled_changed = self.state.enabled.swap(enabled, Ordering::AcqRel) != enabled; + if enabled && (enabled_changed || motion_changed) { + // SDL also uses Raw Input for relative mode. Register our dedicated + // message window after SDL toggles the mode so motion is delivered + // directly to this thread rather than SDL's event pump. + unsafe { + let _ = PostMessageW(self.message_window as HWND, WM_RAW_INPUT_REREGISTER, 0, 0); + } + } else if enabled_changed { + release_pressed_buttons(&self.state); + } + } + + pub(crate) fn release_buttons(&self) { + release_pressed_buttons(&self.state); + } +} + +impl Drop for WindowsRawInputController { + fn drop(&mut self) { + self.state.enabled.store(false, Ordering::Release); + release_pressed_buttons(&self.state); + unsafe { + let _ = PostMessageW(self.message_window as HWND, WM_CLOSE, 0, 0); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn run_raw_input_thread(state: Arc, ready: mpsc::SyncSender>) { + unsafe { + let _ = SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL); + let instance = GetModuleHandleW(null()); + let window_class = WNDCLASSW { + lpfnWndProc: Some(raw_input_window_proc), + hInstance: instance, + lpszClassName: RAW_INPUT_CLASS.as_ptr(), + ..Default::default() + }; + let _ = RegisterClassW(&window_class); + let state_pointer = Arc::as_ptr(&state); + let hwnd = CreateWindowExW( + 0, + RAW_INPUT_CLASS.as_ptr(), + RAW_INPUT_CLASS.as_ptr(), + 0, + 0, + 0, + 0, + 0, + HWND_MESSAGE, + null_mut(), + instance, + state_pointer.cast::(), + ); + if hwnd.is_null() { + let _ = ready.send(Err("failed to create Raw Input message window".to_owned())); + return; + } + if !register_raw_mouse(hwnd) { + let _ = DestroyWindow(hwnd); + let _ = ready.send(Err("failed to register the Raw Input mouse".to_owned())); + return; + } + let _ = ready.send(Ok(hwnd as isize)); + let mut message = MSG::default(); + while GetMessageW(&mut message, null_mut(), 0, 0) > 0 { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } +} + +unsafe extern "system" fn raw_input_window_proc( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if message == WM_NCCREATE { + let create = unsafe { &*(lparam as *const CREATESTRUCTW) }; + unsafe { + SetWindowLongPtrW(hwnd, GWLP_USERDATA, create.lpCreateParams as isize); + } + return 1; + } + + let state_pointer = unsafe { GetWindowLongPtrW(hwnd, GWLP_USERDATA) } as *const RawInputState; + match message { + WM_RAW_INPUT_REREGISTER => { + // Internal wake after SDL changes relative mode: reclaim the raw + // mouse registration for this dedicated message thread. + unsafe { + let _ = register_raw_mouse(hwnd); + } + 0 + } + WM_INPUT => { + if !state_pointer.is_null() { + unsafe { + process_raw_input(&*state_pointer, lparam as HRAWINPUT); + } + } + 0 + } + WM_CLOSE => { + unsafe { + unregister_raw_mouse(); + let _ = DestroyWindow(hwnd); + } + 0 + } + WM_NCDESTROY => { + unsafe { + SetWindowLongPtrW(hwnd, GWLP_USERDATA, 0); + PostQuitMessage(0); + } + 0 + } + _ => unsafe { DefWindowProcW(hwnd, message, wparam, lparam) }, + } +} + +unsafe fn register_raw_mouse(hwnd: HWND) -> bool { + let device = RAWINPUTDEVICE { + usUsagePage: 0x01, + usUsage: 0x02, + dwFlags: RIDEV_INPUTSINK, + hwndTarget: hwnd, + }; + unsafe { RegisterRawInputDevices(&device, 1, size_of::() as u32) != 0 } +} + +unsafe fn unregister_raw_mouse() { + let device = RAWINPUTDEVICE { + usUsagePage: 0x01, + usUsage: 0x02, + dwFlags: RIDEV_REMOVE, + hwndTarget: null_mut(), + }; + unsafe { + let _ = RegisterRawInputDevices(&device, 1, size_of::() as u32); + } +} + +unsafe fn process_raw_input(state: &RawInputState, handle: HRAWINPUT) { + if !state.enabled.load(Ordering::Acquire) { + return; + } + let mut raw = MaybeUninit::::uninit(); + let mut size = size_of::() as u32; + let copied = unsafe { + GetRawInputData( + handle, + RID_INPUT, + raw.as_mut_ptr().cast::(), + &mut size, + size_of::() as u32, + ) + }; + if copied == u32::MAX || copied < size_of::() as u32 { + return; + } + let raw = unsafe { raw.assume_init() }; + if raw.header.dwType != RIM_TYPEMOUSE { + return; + } + let mouse = unsafe { raw.data.mouse }; + let owns_foreground = + unsafe { GetForegroundWindow() } as isize == state.foreground_owner.load(Ordering::Acquire); + if owns_foreground + && state.relative_motion.load(Ordering::Acquire) + && mouse.usFlags & MOUSE_MOVE_ABSOLUTE == 0 + { + // SDL continues to own absolute cursor coordinates. This thread owns raw relative + // deltas, plus buttons and wheel in both cursor modes. + push_mouse_delta(&state.captured_input, mouse.lLastX, mouse.lLastY); + } + + let buttons = unsafe { mouse.Anonymous.Anonymous }; + let button_flags = if owns_foreground { + buttons.usButtonFlags + } else { + // A click can transiently move foreground before WM_INPUT delivers the + // matching release. Never discard releases for buttons that we already + // sent down: doing so leaves both the local de-duplicator and host stuck. + buttons.usButtonFlags & raw_mouse_button_up_mask() + }; + push_raw_mouse_buttons(state, button_flags); + if owns_foreground && u32::from(buttons.usButtonFlags) & RI_MOUSE_WHEEL != 0 { + state.captured_input.push(CapturedInput::MouseWheel { + delta_x: 0, + delta_y: buttons.usButtonData as i16, + }); + } + if owns_foreground && u32::from(buttons.usButtonFlags) & RI_MOUSE_HWHEEL != 0 { + state.captured_input.push(CapturedInput::MouseWheel { + delta_x: buttons.usButtonData as i16, + delta_y: 0, + }); + } +} + +fn raw_mouse_button_up_mask() -> u16 { + (RI_MOUSE_LEFT_BUTTON_UP + | RI_MOUSE_MIDDLE_BUTTON_UP + | RI_MOUSE_RIGHT_BUTTON_UP + | RI_MOUSE_BUTTON_4_UP + | RI_MOUSE_BUTTON_5_UP) as u16 +} + +fn push_raw_mouse_buttons(state: &RawInputState, flags: u16) { + const BUTTON_FLAGS: &[(u32, u32, u8)] = &[ + (RI_MOUSE_LEFT_BUTTON_DOWN, RI_MOUSE_LEFT_BUTTON_UP, 1), + (RI_MOUSE_MIDDLE_BUTTON_DOWN, RI_MOUSE_MIDDLE_BUTTON_UP, 2), + (RI_MOUSE_RIGHT_BUTTON_DOWN, RI_MOUSE_RIGHT_BUTTON_UP, 3), + (RI_MOUSE_BUTTON_4_DOWN, RI_MOUSE_BUTTON_4_UP, 4), + (RI_MOUSE_BUTTON_5_DOWN, RI_MOUSE_BUTTON_5_UP, 5), + ]; + let flags = u32::from(flags); + for &(down, up, button) in BUTTON_FLAGS { + if flags & down != 0 { + push_mouse_button(state, button, true); + } + if flags & up != 0 { + push_mouse_button(state, button, false); + } + } +} + +fn push_mouse_button(state: &RawInputState, button: u8, pressed: bool) { + let mut pressed_buttons = state + .pressed_buttons + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.enabled.load(Ordering::Acquire) { + return; + } + let changed = if pressed { + pressed_buttons.insert(button) + } else { + pressed_buttons.remove(&button) + }; + if changed { + state + .captured_input + .push(CapturedInput::MouseButton { button, pressed }); + } +} + +fn release_pressed_buttons(state: &RawInputState) { + let mut pressed_buttons = state + .pressed_buttons + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for button in pressed_buttons.drain() { + state.captured_input.push(CapturedInput::MouseButton { + button, + pressed: false, + }); + } +} + +fn push_mouse_delta(queue: &CapturedInputQueue, mut delta_x: i32, mut delta_y: i32) { + while delta_x != 0 || delta_y != 0 { + let x = delta_x.clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + let y = delta_y.clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + queue.push(CapturedInput::MouseMove { + delta_x: x, + delta_y: y, + }); + delta_x -= i32::from(x); + delta_y -= i32::from(y); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::Ordering; + + use super::{RawInputState, push_mouse_delta, push_raw_mouse_buttons, release_pressed_buttons}; + use crate::media::{CapturedInput, CapturedInputQueue}; + use windows_sys::Win32::UI::WindowsAndMessaging::{ + RI_MOUSE_LEFT_BUTTON_DOWN, RI_MOUSE_LEFT_BUTTON_UP, RI_MOUSE_RIGHT_BUTTON_DOWN, + }; + + fn state() -> RawInputState { + RawInputState { + foreground_owner: Default::default(), + enabled: true.into(), + relative_motion: false.into(), + pressed_buttons: Default::default(), + captured_input: Arc::new(CapturedInputQueue::default()), + } + } + + #[test] + fn large_raw_mouse_deltas_are_split_without_loss() { + let queue = CapturedInputQueue::default(); + push_mouse_delta(&queue, 40_000, -40_000); + + let mut total_x = 0_i32; + let mut total_y = 0_i32; + while let Some(CapturedInput::MouseMove { delta_x, delta_y }) = queue.take() { + total_x += i32::from(delta_x); + total_y += i32::from(delta_y); + } + assert_eq!((total_x, total_y), (40_000, -40_000)); + } + + #[test] + fn raw_buttons_keep_one_owner_across_cursor_mode_changes() { + let state = state(); + push_raw_mouse_buttons( + &state, + (RI_MOUSE_LEFT_BUTTON_DOWN | RI_MOUSE_RIGHT_BUTTON_DOWN) as u16, + ); + // A server cursor update can switch motion between SDL absolute and Raw Input relative, + // but button ownership and pressed state remain on this controller. + state.relative_motion.store(true, Ordering::Release); + push_raw_mouse_buttons(&state, RI_MOUSE_LEFT_BUTTON_DOWN as u16); + state.relative_motion.store(false, Ordering::Release); + push_raw_mouse_buttons(&state, RI_MOUSE_LEFT_BUTTON_UP as u16); + + assert_eq!( + state.captured_input.take(), + Some(CapturedInput::MouseButton { + button: 1, + pressed: true, + }) + ); + assert_eq!( + state.captured_input.take(), + Some(CapturedInput::MouseButton { + button: 3, + pressed: true, + }) + ); + assert_eq!( + state.captured_input.take(), + Some(CapturedInput::MouseButton { + button: 1, + pressed: false, + }) + ); + + state.enabled.store(false, Ordering::Release); + release_pressed_buttons(&state); + assert_eq!( + state.captured_input.take(), + Some(CapturedInput::MouseButton { + button: 3, + pressed: false, + }) + ); + assert_eq!(state.captured_input.take(), None); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/Cargo.toml new file mode 100644 index 000000000..1a1670cbe --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "opennow-streamer-protocol" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/lib.rs new file mode 100644 index 000000000..c548941ab --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/lib.rs @@ -0,0 +1,606 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +pub mod log; +pub mod text_input; + +pub const PROTOCOL_VERSION: u64 = 7; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RecordingCutReason { + Discontinuity, + QueueOverflow, + Interrupted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum RecordingCompletion { + Complete, + Cut { reason: RecordingCutReason }, +} + +#[derive(Debug, Clone, Copy)] +pub struct ReplayBufferConfig { + pub enabled: bool, + pub duration: std::time::Duration, + pub memory_bytes: usize, +} + +impl ReplayBufferConfig { + pub fn from_settings(settings: &Value) -> Self { + let bounded = |key: &str, default: u64, minimum: u64, maximum: u64| { + settings[key] + .as_i64() + .map(|value| value.clamp(minimum as i64, maximum as i64) as u64) + .or_else(|| { + settings[key] + .as_u64() + .map(|value| value.clamp(minimum, maximum)) + }) + .unwrap_or(default) + }; + Self { + enabled: settings["replayBufferEnabled"].as_bool() == Some(true), + duration: std::time::Duration::from_secs(bounded("replayBufferSeconds", 30, 15, 120)), + memory_bytes: bounded("replayBufferMemoryMiB", 256, 64, 512) as usize * 1024 * 1024, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AudioOutputDevice(String); + +impl AudioOutputDevice { + pub fn from_settings(settings: &Value) -> Result { + match settings.get("audioOutputDevice") { + None => Ok(Self::default()), + Some(Value::String(name)) => Self::new(name.clone()), + Some(_) => Err("audioOutputDevice must be a string".to_owned()), + } + } + + pub fn new(name: String) -> Result { + if name.len() > 1024 || name.contains('\0') { + return Err( + "audioOutputDevice must be at most 1024 UTF-8 bytes and contain no NUL".to_owned(), + ); + } + Ok(Self(name)) + } + + pub fn device_name(&self) -> Option<&str> { + (!self.0.is_empty()).then_some(self.0.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct AudioDevice { + pub id: String, + pub name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Command { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(default)] + pub protocol_version: Option, + #[serde(default)] + pub context: Option, + #[serde(default)] + pub paused: Option, + #[serde(default)] + pub muted: Option, + #[serde(default)] + pub enabled: Option, + #[serde(default)] + pub microphone_enabled: Option, + #[serde(default)] + pub surface: Option, + #[serde(default)] + pub max_bitrate_kbps: Option, + #[serde(default)] + pub reason: Option, + #[serde(default)] + pub shortcuts: Option, + #[serde(default)] + pub host: Option, + #[serde(default)] + pub port: Option, + #[serde(default)] + pub payload_base64: Option, + #[serde(default)] + pub output_path: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RenderSurface { + #[serde(default)] + pub rect: Option, + #[serde(default)] + pub visible: bool, + #[serde(default = "default_device_scale_factor")] + pub device_scale_factor: f32, + #[serde(default)] + pub window_handle: Option, + #[serde(default)] + pub screen_rect: Option, +} + +impl Default for RenderSurface { + fn default() -> Self { + Self { + rect: None, + visible: false, + device_scale_factor: default_device_scale_factor(), + window_handle: None, + screen_rect: None, + } + } +} + +fn default_device_scale_factor() -> f32 { + 1.0 +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RenderSurfaceRect { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SessionContext { + pub session: Session, + pub settings: Value, + pub shortcuts: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nvst_video: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Session { + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sub_session_id: Option, + pub server_ip: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_connection_info: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub connection_info: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ConnectionInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ip: Option, + pub port: u32, + pub usage: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub protocol: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_level_protocol: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_path: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaConnectionInfo { + pub ip: String, + pub port: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Capabilities { + pub protocol_version: u64, + pub backend: &'static str, + pub supports_input: bool, + pub supports_video_decode: bool, + pub supports_video_present: bool, + pub supports_audio_decode: bool, + pub supports_audio_output: bool, + pub supports_microphone: bool, + pub supports_owned_nvst_negotiation: bool, + pub video_backends: Vec, + /// Per-adapter hardware decode profiles for the current boot. Empty on + /// platforms that do not enumerate adapters. LUIDs are not included. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub graphics_adapters: Vec, +} + +/// One physical GPU and the hardware decode profiles its video device exposed. +/// `active` is the adapter the embedded runtime is using for this process. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GraphicsAdapterCapability { + pub name: String, + pub active: bool, + pub codecs: Vec, + pub h265_main10: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct VideoBackendCapability { + pub backend: &'static str, + pub platform: &'static str, + pub codecs: Vec, + pub zero_copy_modes: Vec<&'static str>, + pub available: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option<&'static str>, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DecodeDimensions { + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CodecCapability { + pub codec: &'static str, + pub available: bool, + #[serde(rename = "colorQualities", skip_serializing_if = "Option::is_none")] + pub color_qualities: Option>, + #[serde(rename = "hdrColorQualities", skip_serializing_if = "Option::is_none")] + pub hdr_color_qualities: Option>, + #[serde(rename = "hdrSupported", skip_serializing_if = "Option::is_none")] + pub hdr_supported: Option, + #[serde( + rename = "maxDecodeDimensions", + skip_serializing_if = "Option::is_none" + )] + pub max_decode_dimensions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option<&'static str>, +} + +#[cfg(test)] +mod graphics_adapter_tests { + use super::GraphicsAdapterCapability; + + #[test] + fn graphics_adapter_index_serializes_codec_profiles_without_an_empty_reason() { + let decoding = GraphicsAdapterCapability { + name: "Intel(R) HD Graphics 620".to_owned(), + active: false, + codecs: vec!["h264".to_owned(), "h265".to_owned()], + h265_main10: true, + reason: None, + }; + let value = serde_json::to_value(&decoding).unwrap(); + assert_eq!(value["name"], "Intel(R) HD Graphics 620"); + assert_eq!(value["active"], false); + assert_eq!(value["codecs"], serde_json::json!(["h264", "h265"])); + assert_eq!(value["h265Main10"], true); + assert!(value.get("reason").is_none()); + + let discrete = GraphicsAdapterCapability { + name: "NVIDIA GeForce MX110".to_owned(), + active: true, + codecs: Vec::new(), + h265_main10: false, + reason: Some("no supported hardware decoder profile".to_owned()), + }; + let value = serde_json::to_value(&discrete).unwrap(); + assert_eq!(value["codecs"], serde_json::json!([])); + assert_eq!(value["h265Main10"], false); + assert_eq!(value["reason"], "no supported hardware decoder profile"); + } +} + +#[cfg(test)] +mod hdr_profile_tests { + use super::{CodecCapability, DecodeDimensions}; + + #[test] + fn exact_hdr_profiles_serialize_independently_of_sdr_formats() { + let mut codec = CodecCapability { + max_decode_dimensions: None, + codec: "h265", + available: true, + color_qualities: Some(vec!["10bit_420", "10bit_444"]), + hdr_color_qualities: Some(vec!["10bit_420"]), + hdr_supported: Some(true), + reason: None, + }; + let value = serde_json::to_value(&codec).unwrap(); + assert_eq!(value["hdrColorQualities"], serde_json::json!(["10bit_420"])); + assert_eq!( + value["colorQualities"], + serde_json::json!(["10bit_420", "10bit_444"]) + ); + codec.hdr_color_qualities = Some(Vec::new()); + assert_eq!( + serde_json::to_value(&codec).unwrap()["hdrColorQualities"], + serde_json::json!([]) + ); + codec.hdr_color_qualities = None; + assert!( + serde_json::to_value(&codec) + .unwrap() + .get("hdrColorQualities") + .is_none() + ); + assert!( + serde_json::to_value(&codec) + .unwrap() + .get("maxDecodeDimensions") + .is_none() + ); + codec.max_decode_dimensions = Some(DecodeDimensions { + width: 4096, + height: 2304, + }); + assert_eq!( + serde_json::to_value(&codec).unwrap()["maxDecodeDimensions"], + serde_json::json!({"width":4096,"height":2304}) + ); + } +} + +pub fn response(id: impl Into, kind: &str) -> Value { + serde_json::json!({ "id": id.into(), "type": kind }) +} + +pub fn error(id: Option<&str>, code: &str, message: impl Into) -> Value { + let mut value = serde_json::json!({ + "type": "error", + "code": code, + "message": message.into(), + }); + if let Some(id) = id { + value["id"] = Value::String(id.to_owned()); + } + value +} + +pub fn event(kind: &str, fields: Value) -> Value { + let mut object = fields.as_object().cloned().unwrap_or_default(); + object.insert("type".to_owned(), Value::String(kind.to_owned())); + Value::Object(object) +} + +#[cfg(test)] +mod tests { + #[test] + fn audio_mute_command_accepts_only_boolean_states() { + use super::Command; + use serde_json::json; + + for muted in [false, true] { + let command: Command = serde_json::from_value(json!({ + "id": "mute", "type": "setAudioMuted", "muted": muted + })) + .unwrap(); + assert_eq!(command.muted, Some(muted)); + } + for muted in [json!("true"), json!(1), json!({}), json!([])] { + assert!( + serde_json::from_value::(json!({ + "id": "mute", "type": "setAudioMuted", "muted": muted + })) + .is_err() + ); + } + } + + #[test] + fn replay_settings_are_opt_in_and_bounded_and_commands_are_additive() { + use super::{Command, ReplayBufferConfig}; + use serde_json::json; + let defaults = ReplayBufferConfig::from_settings(&json!({})); + assert!(!defaults.enabled); + assert_eq!(defaults.duration.as_secs(), 30); + assert_eq!(defaults.memory_bytes, 256 * 1024 * 1024); + for enabled in [json!(false), json!("true"), json!(1), json!(null)] { + assert!( + !ReplayBufferConfig::from_settings(&json!({"replayBufferEnabled":enabled})).enabled + ); + } + for (seconds, memory, expected_seconds, expected_memory) in [ + (-1, -1, 15, 64), + (5, 20, 15, 64), + (60, 128, 60, 128), + (999, 999, 120, 512), + ] { + let config = ReplayBufferConfig::from_settings( + &json!({"replayBufferEnabled":true,"replayBufferSeconds":seconds,"replayBufferMemoryMiB":memory}), + ); + assert!(config.enabled); + assert_eq!(config.duration.as_secs(), expected_seconds); + assert_eq!(config.memory_bytes, expected_memory * 1024 * 1024); + } + let save: Command = serde_json::from_value( + json!({"id":"clip-1","type":"clip-save","outputPath":"/clips/game.mkv"}), + ) + .unwrap(); + assert_eq!(save.output_path.as_deref(), Some("/clips/game.mkv")); + let stop: Command = + serde_json::from_value(json!({"id":"stop-1","type":"replay-stop"})).unwrap(); + assert_eq!(stop.kind, "replay-stop"); + } + + #[test] + fn audio_output_device_defaults_and_preserves_exact_names() { + use super::AudioOutputDevice; + use serde_json::json; + + for settings in [json!({}), json!({"audioOutputDevice": ""})] { + assert_eq!( + AudioOutputDevice::from_settings(&settings) + .unwrap() + .device_name(), + None + ); + } + let settings = json!({"audioOutputDevice": " USB Speakers — 音声 "}); + let selected = AudioOutputDevice::from_settings(&settings).unwrap(); + assert_eq!(selected.device_name(), Some(" USB Speakers — 音声 ")); + } + + #[test] + fn audio_output_device_rejects_invalid_names_and_types() { + use super::AudioOutputDevice; + use serde_json::json; + + for value in [ + json!(null), + json!(1), + json!(true), + json!([]), + json!({}), + json!("a\0b"), + json!("é".repeat(513)), + ] { + assert!( + AudioOutputDevice::from_settings(&json!({"audioOutputDevice": value})).is_err() + ); + } + assert!(AudioOutputDevice::new("é".repeat(512)).is_ok()); + } + + use super::*; + + #[test] + fn hdr_capabilities_distinguish_unknown_supported_and_unsupported() { + let mut codec = CodecCapability { + max_decode_dimensions: None, + codec: "h265", + available: true, + hdr_color_qualities: None, + color_qualities: None, + hdr_supported: None, + reason: None, + }; + assert!( + serde_json::to_value(&codec) + .unwrap() + .get("hdrSupported") + .is_none() + ); + for supported in [false, true] { + codec.hdr_supported = Some(supported); + let value = serde_json::to_value(&codec).unwrap(); + assert_eq!(value["hdrSupported"], supported); + assert!(value.get("colorQualities").is_none()); + } + } + + #[test] + fn codec_color_profiles_are_additive_and_preserve_explicit_empty_support() { + let mut codec = CodecCapability { + max_decode_dimensions: None, + hdr_supported: None, + codec: "h265", + available: true, + hdr_color_qualities: None, + color_qualities: None, + reason: None, + }; + let value = serde_json::to_value(&codec).unwrap(); + assert!(value.get("colorQualities").is_none()); + codec.color_qualities = Some(vec!["8bit_420", "10bit_420"]); + let value = serde_json::to_value(&codec).unwrap(); + assert_eq!( + value["colorQualities"], + serde_json::json!(["8bit_420", "10bit_420"]) + ); + codec.color_qualities = Some(Vec::new()); + let value = serde_json::to_value(&codec).unwrap(); + assert_eq!(value["colorQualities"], serde_json::json!([])); + } + + #[test] + fn parses_forward_compatible_commands() { + let command: Command = serde_json::from_value(serde_json::json!({ + "id": "1", + "type": "start", + "context": { "session": { "sessionId": "session" } }, + "futureField": true + })) + .expect("command"); + assert_eq!(command.kind, "start"); + assert!(command.context.is_some()); + } + + #[test] + fn unsolicited_errors_do_not_serialize_a_null_request_id() { + let value = error(None, "invalid-command", "bad JSON"); + assert!(value.get("id").is_none()); + assert_eq!(value["type"], "error"); + } + + #[test] + fn session_context_round_trips_required_and_forward_compatible_fields() { + let fixture = serde_json::json!({ + "session": { + "sessionId": "synthetic-session", + "subSessionId": "synthetic-subsession", + "serverIp": "127-0-0-1.synthetic.invalid", + "iceServers": [], + "mediaConnectionInfo": { + "ip": "198.51.100.20", + "port": 18_784, + "usage": 17, + "futureEndpointField": true + }, + "connectionInfo": [ + { + "ip": "198.51.100.10", + "port": 443, + "usage": 14, + "protocol": 1, + "resourcePath": "/nvst/" + }, + { + "ip": "198.51.100.20", + "port": 48322, + "usage": 16, + "protocol": 1, + "appLevelProtocol": 6, + "resourcePath": "rtsps://198.51.100.20:48322/session", + "futureConnectionField": true + } + ], + "futureSessionField": "preserved" + }, + "settings": { "codec": "H264", "fps": 60 }, + "shortcuts": { "stopStream": "Ctrl+Shift+Q" }, + "futureContextField": 42 + }); + + let context: SessionContext = serde_json::from_value(fixture.clone()).expect("context"); + assert_eq!(context.session.session_id, "synthetic-session"); + assert_eq!( + serde_json::to_value(context).expect("serializable context"), + fixture + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/log.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/log.rs new file mode 100644 index 000000000..301393deb --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/log.rs @@ -0,0 +1,729 @@ +//! Minimal file logger for the in-process streamer. +//! +//! The Qt shell writes the legacy child-process streamer's stdout to +//! `diagnostics/native-streamer.log`, but the embedded FFI path never spawns +//! that child, so video-pipeline failures were invisible. This module gives +//! the FFI runtime its own append-only sink into the same file. +//! +//! Deliberately dependency-free and redaction-by-construction: call sites +//! pass pre-formatted `area`/`message` strings, so secrets (tokens, URLs) +//! can never reach the log unless a call site formats them in. + +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::Path; +use std::sync::{Mutex, OnceLock}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +/// Matches the Qt shell's rotation policy for `native-streamer.log`. +const MAX_LOG_BYTES: u64 = 2 * 1024 * 1024; +/// Hot-path failures (per-frame acquire/record) log the first hit and then +/// every Nth repeat so a wedged pipeline cannot flood the disk. +const THROTTLE_EVERY: u64 = 600; + +struct LogState { + file: File, + path: String, + bytes: u64, +} + +fn state() -> &'static Mutex> { + static STATE: OnceLock>> = OnceLock::new(); + STATE.get_or_init(|| Mutex::new(None)) +} + +fn throttle_counts() -> &'static Mutex> { + static COUNTS: OnceLock>> = OnceLock::new(); + COUNTS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + +/// Points the file sink at `path`, creating parent directories and rotating +/// a previous log over 2 MiB to `.previous`. Replaces any earlier sink +/// so tests and restarts never append to a stale handle. Safe to call more +/// than once; logging must never break streaming, so failures are returned +/// for the caller to ignore or surface. +pub fn set_log_file(path: &str) -> Result<(), String> { + let trimmed = path.trim(); + if trimmed.is_empty() { + return Err("log file path is empty".to_owned()); + } + let file_path = Path::new(trimmed); + if let Some(parent) = file_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create log directory: {error}"))?; + } + if file_path.exists() { + let size = std::fs::metadata(file_path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + if size > MAX_LOG_BYTES { + let previous = format!("{trimmed}.previous"); + let _ = std::fs::remove_file(&previous); + let _ = std::fs::rename(file_path, &previous); + } + } + let file = OpenOptions::new() + .create(true) + .append(true) + .open(file_path) + .map_err(|error| format!("cannot open log file: {error}"))?; + let mut guard = state().lock().expect("log state poisoned"); + *guard = Some(LogState { + bytes: file.metadata().map(|metadata| metadata.len()).unwrap_or(0), + file, + path: trimmed.to_owned(), + }); + Ok(()) +} + +/// Returns the currently configured log path, if any. +pub fn log_file_path() -> Option { + state() + .lock() + .expect("log state poisoned") + .as_ref() + .map(|state| state.path.clone()) +} + +fn write_line(level: &str, area: &str, message: &str) { + let mut guard = match state().lock() { + Ok(guard) => guard, + Err(_) => return, + }; + let Some(state) = guard.as_mut() else { + return; + }; + // Single-line discipline: embedded newlines would break log parsers. + let flat: String = message + .chars() + .map(|c| if c == '\n' || c == '\r' { ' ' } else { c }) + .take(2048) + .collect(); + let line = format!("{} {level} {area} {flat}\n", now_ms()); + if state.bytes + line.len() as u64 > MAX_LOG_BYTES { + // Keep the append handle valid on Windows, including other existing writers. + if std::fs::copy(&state.path, format!("{}.previous", state.path)).is_ok() + && state.file.set_len(0).is_ok() + { + state.bytes = 0; + } else { + return; // Do not grow an unbounded log if rotation fails. + } + } + if state.file.write_all(line.as_bytes()).is_ok() { + state.bytes += line.len() as u64; + } +} + +/// Timed, payload-free connection stage. An early return is visible as incomplete. +pub struct Stage { + name: &'static str, + started: Instant, + complete: bool, +} + +impl Stage { + pub fn begin(name: &'static str) -> Self { + log_line("INFO", "stage", &format!("{name} begin")); + Self { + name, + started: Instant::now(), + complete: false, + } + } + + pub fn complete(&mut self) { + self.complete = true; + } +} + +impl Drop for Stage { + fn drop(&mut self) { + log_line( + if self.complete { "INFO" } else { "WARN" }, + "stage", + &format!( + "{} {} elapsed_ms={}", + self.name, + if self.complete { + "complete" + } else { + "incomplete" + }, + self.started.elapsed().as_millis() + ), + ); + } +} + +/// Only protocol discriminators and numeric counters, never payloads, free-form +/// server messages, session IDs, addresses, credentials, SDP or user paths. +pub fn message_summary(value: &serde_json::Value) -> String { + let mut fields = Vec::new(); + for key in [ + "id", + "type", + "event", + "status", + "code", + "backend", + "phase", + "protocolVersion", + "protocol", + "paused", + "fps", + "width", + "height", + "framesReceived", + "framesDecoded", + "framesDropped", + "media", + "unit", + ] { + let Some(value) = value.get(key) else { + continue; + }; + let text = match value { + serde_json::Value::String(text) + if text.len() <= 64 + && text + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_. /".contains(c)) + && !text.contains('/') => + { + text.clone() + } + serde_json::Value::Number(_) | serde_json::Value::Bool(_) => value.to_string(), + _ => continue, + }; + fields.push(format!("{key}={text}")); + } + for key in ["count", "sampleRate", "channels"] { + if let Some(value) = value.get(key).and_then(serde_json::Value::as_u64) { + fields.push(format!("{key}={value}")); + } + } + if value["type"] == "log" && value["event"] == "keyframe-request" { + let code = match value["reason"].as_str() { + Some("compressed video discontinuity or queue overflow") => { + Some("compressed-video-discontinuity-or-overflow") + } + Some("H.264 decoder rejected an access unit") => Some("decoder-access-unit-rejected"), + Some("Linux decoder rejected encoded video framing") => Some("invalid-video-framing"), + Some("embedded Linux decoder queue overflow") => Some("decoder-queue-overflow"), + Some("embedded Linux video submission failed") => Some("video-submission-failed"), + Some("Linux decoder requires a fresh keyframe") => Some("decoder-reference-required"), + Some("Linux decoder device was lost") => Some("decoder-device-lost"), + Some("Linux decoder fallback requires a fresh keyframe") => Some("decoder-fallback"), + _ => None, + }; + if let Some(code) = code { + fields.push(format!("reasonCode={code}")); + } + } + if value["type"] == "telemetry" { + if let Some(stage @ ("tracking" | "keyframe-pending" | "recovery-required")) = + value["decodeProgressStage"].as_str() + { + fields.push(format!("decodeProgressStage={stage}")); + } + if let Some(stalled) = value["transportFrameProgressStalled"].as_bool() { + fields.push(format!("transportFrameProgressStalled={stalled}")); + } + for (group, keys) in [ + ( + "decodeTimings", + [ + "epoch", + "submissionsTotal", + "outputsTotal", + "outputCallsTotal", + "inFlight", + "unmatchedOutputs", + "unmatchedSubmissions", + ], + ), + ( + "frameStageTimings", + [ + "assembledFramesTotal", + "admittedFramesTotal", + "queuedAckFramesTotal", + "undeliveredFramesTotal", + "pendingDeliveries", + "unmatchedDeliveries", + "unmatchedAdmissions", + ], + ), + ] { + for key in keys { + if let Some(counter) = value + .get(group) + .and_then(|timings| timings.get(key)) + .and_then(serde_json::Value::as_u64) + { + fields.push(format!("{group}.{key}={counter}")); + } + } + } + } + fields.join(" ") +} + +/// Appends one line. No-op until [`set_log_file`] succeeds. +pub fn log_line(level: &str, area: &str, message: &str) { + write_line(level, area, message); +} + +/// Periodic media/transport counters must not wait for disk I/O. The diagnostics +/// worker has a fixed queue; under slow storage it drops traces, never media. +pub fn log_async(level: &'static str, area: &'static str, message: &str) { + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::mpsc::{SyncSender, sync_channel}; + type Line = (&'static str, &'static str, String); + static DROPPED: AtomicU64 = AtomicU64::new(0); + static SENDER: OnceLock>> = OnceLock::new(); + let sender = SENDER.get_or_init(|| { + let (sender, receiver) = sync_channel::(64); + std::thread::Builder::new() + .name("opennow-diagnostics".to_owned()) + .spawn(move || { + while let Ok((level, area, message)) = receiver.recv() { + write_line(level, area, &message); + let dropped = DROPPED.swap(0, Ordering::Relaxed); + if dropped > 0 { + write_line( + "WARN", + "diagnostics", + &format!("trace_queue_dropped={dropped}"), + ); + } + } + }) + .ok() + .map(|_| sender) + }); + if let Some(sender) = sender + && sender + .try_send((level, area, message.chars().take(2048).collect())) + .is_err() + { + DROPPED.fetch_add(1, Ordering::Relaxed); + } +} + +/// Low-frequency pipeline diagnostics must be visible both in standalone stderr +/// captures and in the embedded Qt file sink. Never pass credentials or payloads. +pub fn diagnostic(level: &'static str, area: &'static str, message: &str) { + log_async(level, area, message); + eprintln!("{message}"); +} + +/// Appends the first hit for `key` and then every [`THROTTLE_EVERY`]th +/// repeat, tagging repeats with their running count. For hot paths where +/// the same failure recurs per frame. +pub fn log_throttled(key: &'static str, level: &str, area: &str, message: &str) { + let count = { + let mut counts = throttle_counts().lock().expect("log counts poisoned"); + let entry = counts.entry(key).or_insert(0); + *entry += 1; + *entry + }; + if count == 1 { + write_line(level, area, message); + } else if count % THROTTLE_EVERY == 0 { + write_line(level, area, &format!("{message} (repeat #{count})")); + } +} + +#[cfg(test)] +pub(crate) fn reset_for_tests() { + *state().lock().expect("log state poisoned") = None; + throttle_counts() + .lock() + .expect("log counts poisoned") + .clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn summaries_exclude_payloads_and_untrusted_messages() { + let summary = message_summary(&serde_json::json!({ + "id":"native-12", "type":"error", "code":"nvst-rtsp-timeout", + "message":"secret https://private.example", "context":{"token":"credential"}, + "phase":"https://private.example", "width":1920 + })); + assert!(summary.contains("id=native-12")); + assert!(summary.contains("code=nvst-rtsp-timeout")); + assert!(summary.contains("width=1920")); + for forbidden in ["secret", "private", "credential", "context", "message"] { + assert!(!summary.contains(forbidden)); + } + } + + #[test] + fn queue_drop_summaries_include_only_safe_units_and_numeric_counts() { + let summary = message_summary(&serde_json::json!({ + "type": "log", "event": "queue-dropped", "media": "audio-output", + "unit": "samples", "count": 96000, "sampleRate": 48000, "channels": 2, + "message": "private payload", "context": {"token": "credential"} + })); + for field in [ + "media=audio-output", + "unit=samples", + "count=96000", + "sampleRate=48000", + "channels=2", + ] { + assert!(summary.contains(field), "{summary}"); + } + for forbidden in ["private", "payload", "credential", "context", "message"] { + assert!(!summary.contains(forbidden)); + } + let summary = message_summary(&serde_json::json!({ + "media": "https://private.example", "unit": "invalid\nunit", + "count": "credential", "sampleRate": -1, "channels": {"token": "secret"} + })); + assert!(summary.is_empty(), "{summary}"); + } + + #[test] + fn telemetry_summaries_preserve_typed_pipeline_progress() { + let summary = message_summary(&serde_json::json!({ + "type": "telemetry", + "decodeProgressStage": "keyframe-pending", + "transportFrameProgressStalled": false, + "frameStageTimings": { + "assembledFramesTotal": 900, "admittedFramesTotal": 897, + "queuedAckFramesTotal": 896, "undeliveredFramesTotal": 1, + "pendingDeliveries": 2, "unmatchedDeliveries": 1, + "unmatchedAdmissions": 3 + }, + "decodeTimings": { + "epoch": 2, "submissionsTotal": 850, "outputsTotal": 820, + "outputCallsTotal": 800, "inFlight": 30, + "unmatchedOutputs": 1, "unmatchedSubmissions": 2 + } + })); + for field in [ + "decodeProgressStage=keyframe-pending", + "transportFrameProgressStalled=false", + "frameStageTimings.assembledFramesTotal=900", + "frameStageTimings.admittedFramesTotal=897", + "frameStageTimings.queuedAckFramesTotal=896", + "frameStageTimings.undeliveredFramesTotal=1", + "frameStageTimings.pendingDeliveries=2", + "frameStageTimings.unmatchedDeliveries=1", + "frameStageTimings.unmatchedAdmissions=3", + "decodeTimings.epoch=2", + "decodeTimings.submissionsTotal=850", + "decodeTimings.outputsTotal=820", + "decodeTimings.outputCallsTotal=800", + "decodeTimings.inFlight=30", + "decodeTimings.unmatchedOutputs=1", + "decodeTimings.unmatchedSubmissions=2", + ] { + assert!(summary.split(' ').any(|entry| entry == field), "{summary}"); + } + } + + #[test] + fn keyframe_summaries_map_only_exact_known_reasons_to_codes() { + for (reason, code) in [ + ( + "compressed video discontinuity or queue overflow", + "compressed-video-discontinuity-or-overflow", + ), + ( + "H.264 decoder rejected an access unit", + "decoder-access-unit-rejected", + ), + ( + "Linux decoder rejected encoded video framing", + "invalid-video-framing", + ), + ( + "embedded Linux decoder queue overflow", + "decoder-queue-overflow", + ), + ( + "embedded Linux video submission failed", + "video-submission-failed", + ), + ( + "Linux decoder requires a fresh keyframe", + "decoder-reference-required", + ), + ("Linux decoder device was lost", "decoder-device-lost"), + ( + "Linux decoder fallback requires a fresh keyframe", + "decoder-fallback", + ), + ] { + let summary = message_summary(&serde_json::json!({ + "type": "log", "event": "keyframe-request", "reason": reason, + "message": "private payload", "sessionId": "secret" + })); + assert_eq!( + summary, + format!("type=log event=keyframe-request reasonCode={code}") + ); + } + for reason in [ + "credential", + "https://private.example", + "/home/private/file", + "embedded Linux decoder queue overflow\nsecret", + "embedded Linux decoder queue overflow secret", + ] { + let summary = message_summary(&serde_json::json!({ + "type": "log", "event": "keyframe-request", "reason": reason, + "reasonCode": "credential" + })); + assert_eq!(summary, "type=log event=keyframe-request"); + } + for value in [ + serde_json::json!({"reason": "embedded Linux decoder queue overflow"}), + serde_json::json!({"type": "error", "event": "keyframe-request", "reason": "embedded Linux decoder queue overflow"}), + serde_json::json!({"type": "log", "event": "backend-fallback", "reason": "embedded Linux decoder queue overflow"}), + ] { + assert!(!message_summary(&value).contains("reasonCode=")); + } + } + + #[test] + fn pipeline_summaries_reject_untrusted_types_and_nested_payloads() { + for invalid in [ + serde_json::json!("credential"), + serde_json::json!("123"), + serde_json::json!(-1), + serde_json::json!(1.5), + serde_json::json!(true), + serde_json::json!(null), + serde_json::json!([1]), + serde_json::json!({"token": "secret"}), + ] { + let summary = message_summary(&serde_json::json!({ + "type": "telemetry", "decodeProgressStage": invalid, + "transportFrameProgressStalled": invalid, + "frameStageTimings": { + "assembledFramesTotal": invalid, "admittedFramesTotal": invalid, + "queuedAckFramesTotal": invalid, "undeliveredFramesTotal": invalid, + "pendingDeliveries": invalid, "unmatchedDeliveries": invalid, + "unmatchedAdmissions": invalid + }, + "decodeTimings": { + "epoch": invalid, "submissionsTotal": invalid, "outputsTotal": invalid, + "outputCallsTotal": invalid, "inFlight": invalid, + "unmatchedOutputs": invalid, "unmatchedSubmissions": invalid + } + })); + let expected = if invalid.is_boolean() { + "type=telemetry transportFrameProgressStalled=true" + } else { + "type=telemetry" + }; + assert_eq!(summary, expected); + } + let summary = message_summary(&serde_json::json!({ + "type": "telemetry", "decodeProgressStage": "credential", + "startId": "secret", "sessionId": "secret", "accountId": "secret", + "reason": "private", "context": {"token": "credential"}, + "frameStageTimings": {"assembledFramesTotal": 0, "url": "https://private.example"}, + "decodeTimings": {"outputsTotal": 0, "path": "/home/private/file", "token": "credential"} + })); + assert_eq!( + summary, + "type=telemetry decodeTimings.outputsTotal=0 frameStageTimings.assembledFramesTotal=0" + ); + for group in [ + serde_json::json!(null), + serde_json::json!("secret"), + serde_json::json!([1]), + ] { + assert_eq!( + message_summary(&serde_json::json!({ + "type": "telemetry", "decodeTimings": group, "frameStageTimings": group + })), + "type=telemetry" + ); + } + assert_eq!( + message_summary(&serde_json::json!({ + "type": "log", "decodeProgressStage": "tracking", + "transportFrameProgressStalled": true, + "decodeTimings": {"inFlight": 1}, "frameStageTimings": {"pendingDeliveries": 1} + })), + "type=log" + ); + } + + #[test] + fn pipeline_summary_size_is_bounded_and_missing_values_stay_unavailable() { + let mut telemetry = serde_json::json!({"type": "telemetry"}); + assert_eq!(message_summary(&telemetry), "type=telemetry"); + telemetry["decodeTimings"] = serde_json::json!({ + "epoch": u64::MAX, "submissionsTotal": u64::MAX, "outputsTotal": u64::MAX, + "outputCallsTotal": u64::MAX, "inFlight": u64::MAX, + "unmatchedOutputs": u64::MAX, "unmatchedSubmissions": u64::MAX + }); + telemetry["frameStageTimings"] = serde_json::json!({ + "assembledFramesTotal": u64::MAX, "admittedFramesTotal": u64::MAX, + "queuedAckFramesTotal": u64::MAX, "undeliveredFramesTotal": u64::MAX, + "pendingDeliveries": u64::MAX, "unmatchedDeliveries": u64::MAX, + "unmatchedAdmissions": u64::MAX + }); + telemetry["decodeProgressStage"] = serde_json::json!("recovery-required"); + telemetry["transportFrameProgressStalled"] = serde_json::json!(true); + let summary = message_summary(&telemetry); + assert!(summary.len() < 1024, "{}", summary.len()); + assert_eq!(summary.split(' ').count(), 17); + assert!(summary.contains(&format!("decodeTimings.outputsTotal={}", u64::MAX))); + telemetry["decodeTimings"]["private"] = serde_json::json!("secret".repeat(10_000)); + assert_eq!(message_summary(&telemetry), summary); + } + + #[test] + fn active_log_rotation_and_incomplete_stages_are_visible() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + let path = temp_path("active-rotation.log"); + let _ = std::fs::remove_file(&path); + set_log_file(&path).unwrap(); + for _ in 0..1100 { + log_line("INFO", "test", &"x".repeat(2048)); + } + { + let _stage = Stage::begin("test.incomplete"); + } + { + let mut stage = Stage::begin("test.complete"); + stage.complete(); + } + let body = std::fs::read_to_string(&path).unwrap(); + assert!(body.len() as u64 <= MAX_LOG_BYTES); + assert!(Path::new(&format!("{path}.previous")).exists()); + assert!(body.contains("test.incomplete incomplete elapsed_ms=")); + assert!(body.contains("test.complete complete elapsed_ms=")); + reset_for_tests(); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(format!("{path}.previous")); + } + + // These tests replace the process-global log sink and throttle counters. + // Hold one guard across setup, assertions and teardown, not just each write. + static TEST_LOG: Mutex<()> = Mutex::new(()); + + #[test] + fn asynchronous_counters_reach_the_sink() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + let path = temp_path("async.log"); + let _ = std::fs::remove_file(&path); + set_log_file(&path).unwrap(); + log_async("INFO", "transport", "inbound=0 frames=0"); + diagnostic("INFO", "transport", "NVST rx-stats inbound=0 pings=97"); + let deadline = Instant::now() + std::time::Duration::from_secs(2); + loop { + let body = std::fs::read_to_string(&path).unwrap(); + if body.contains("inbound=0 frames=0") + && body.contains("INFO transport NVST rx-stats inbound=0 pings=97") + { + break; + } + assert!( + Instant::now() < deadline, + "diagnostic worker did not write the trace" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + reset_for_tests(); + let _ = std::fs::remove_file(&path); + } + + fn temp_path(name: &str) -> String { + let mut path = std::env::temp_dir(); + path.push(format!( + "opennow-streamer-log-test-{}-{name}", + std::process::id() + )); + path.to_string_lossy().into_owned() + } + + #[test] + fn log_lines_reach_the_configured_file() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + let path = temp_path("lines.log"); + let _ = std::fs::remove_file(&path); + set_log_file(&path).unwrap(); + assert_eq!(log_file_path().as_deref(), Some(path.as_str())); + log_line("INFO", "engine", "hello"); + let body = std::fs::read_to_string(&path).unwrap(); + assert!(body.contains("INFO engine hello"), "body was: {body}"); + reset_for_tests(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn oversized_logs_rotate_to_previous() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + let path = temp_path("rotate.log"); + let previous = format!("{path}.previous"); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&previous); + std::fs::write(&path, vec![b'x'; (MAX_LOG_BYTES + 8) as usize]).unwrap(); + set_log_file(&path).unwrap(); + assert!( + std::fs::metadata(&previous).unwrap().len() > MAX_LOG_BYTES, + "previous log should hold the oversized body" + ); + assert!( + std::fs::metadata(&path).unwrap().len() < 1024, + "fresh log should start near-empty" + ); + reset_for_tests(); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&previous); + } + + #[test] + fn throttled_repeats_stay_bounded() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + let path = temp_path("throttle.log"); + let _ = std::fs::remove_file(&path); + set_log_file(&path).unwrap(); + for _ in 0..(THROTTLE_EVERY * 2 + 10) { + log_throttled("unit-test-key", "WARN", "decode", "boom"); + } + let body = std::fs::read_to_string(&path).unwrap(); + assert_eq!(body.lines().count(), 3, "body was: {body}"); + assert!(body.contains("(repeat #1200)")); + reset_for_tests(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn logging_without_a_sink_is_a_silent_noop() { + let _guard = TEST_LOG.lock().unwrap_or_else(|error| error.into_inner()); + reset_for_tests(); + log_line("INFO", "engine", "nowhere"); + log_throttled("noop-key", "WARN", "decode", "nowhere"); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/text_input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/text_input.rs new file mode 100644 index 000000000..36799ccb9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-protocol/src/text_input.rs @@ -0,0 +1,134 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, Weak}; + +pub const MAX_TEXT_BYTES: usize = 65_536; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextInputError { + Invalid, + TooLarge, + Busy, + Unavailable, +} + +#[derive(Debug, Default)] +pub struct TextInputSlot(Mutex>); + +impl TextInputSlot { + pub fn submit(&self, bytes: &[u8]) -> Result { + if bytes.len() > MAX_TEXT_BYTES { + return Err(TextInputError::TooLarge); + } + if bytes.is_empty() || bytes.contains(&0) { + return Err(TextInputError::Invalid); + } + let text = std::str::from_utf8(bytes).map_err(|_| TextInputError::Invalid)?; + let mut slot = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if slot.upgrade().is_some() { + return Err(TextInputError::Busy); + } + let payload = Arc::new(TextPayload { + text: text.to_owned(), + cancelled: AtomicBool::new(false), + }); + *slot = Arc::downgrade(&payload); + Ok(UnicodeText(payload)) + } + + pub fn cancel(&self) { + if let Some(payload) = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .upgrade() + { + payload.cancelled.store(true, Ordering::Release); + } + } +} + +struct TextPayload { + text: String, + cancelled: AtomicBool, +} + +#[derive(Clone)] +pub struct UnicodeText(Arc); + +impl UnicodeText { + pub fn as_str(&self) -> &str { + &self.0.text + } + + pub fn is_cancelled(&self) -> bool { + self.0.cancelled.load(Ordering::Acquire) + } +} + +impl std::fmt::Debug for UnicodeText { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UnicodeText") + .field("bytes", &self.0.text.len()) + .finish_non_exhaustive() + } +} + +impl PartialEq for UnicodeText { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for UnicodeText {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_text_without_truncation_or_payload_diagnostics() { + let slot = TextInputSlot::default(); + for invalid in [ + &b""[..], + &b"a\0b"[..], + &[0xff], + &[0xc0, 0x80], + &[0xed, 0xa0, 0x80], + ] { + assert_eq!(slot.submit(invalid), Err(TextInputError::Invalid)); + } + assert_eq!( + slot.submit(&vec![b'a'; MAX_TEXT_BYTES + 1]), + Err(TextInputError::TooLarge) + ); + let text = slot.submit("secret café 世界 🦫\r\n".as_bytes()).unwrap(); + assert_eq!(text.as_str(), "secret café 世界 🦫\r\n"); + assert!(!format!("{text:?}").contains("secret")); + drop(text); + assert_eq!( + slot.submit(&vec![b'a'; MAX_TEXT_BYTES]) + .unwrap() + .as_str() + .len(), + MAX_TEXT_BYTES + ); + } + + #[test] + fn one_paste_remains_reserved_through_clones_and_cancellation() { + let slot = TextInputSlot::default(); + let text = slot.submit(b"first").unwrap(); + let transport = text.clone(); + drop(text); + assert_eq!(slot.submit(b"second"), Err(TextInputError::Busy)); + slot.cancel(); + assert!(transport.is_cancelled()); + assert_eq!(slot.submit(b"second"), Err(TextInputError::Busy)); + drop(transport); + let next = slot.submit(b"second").unwrap(); + assert!(!next.is_cancelled()); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/Cargo.toml b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/Cargo.toml new file mode 100644 index 000000000..ad44f0b95 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "opennow-streamer-transport" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +aes.workspace = true +ctr.workspace = true +crc32fast.workspace = true +getrandom.workspace = true +ghash.workspace = true +hmac.workspace = true +mtu.workspace = true +opennow-streamer-hid = { path = "../opennow-streamer-hid" } +opennow-streamer-protocol = { path = "../opennow-streamer-protocol" } +serde.workspace = true +serde_json.workspace = true +sha1.workspace = true +socket2.workspace = true +subtle.workspace = true +thiserror.workspace = true + +[target.'cfg(windows)'.dependencies] +str0m = { workspace = true, features = ["wincrypto-dimpl"] } +windows-sys = { version = "0.61", features = ["Win32_Networking_WinSock", "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", "Win32_System_IO"] } + +[target.'cfg(target_os = "macos")'.dependencies] +# Keep transport crypto architecture-independent. The Apple/Swift provider's +# static archive discovery breaks x86_64 cross-builds on Apple Silicon, while +# WebRTC crypto is not on the VideoToolbox/Metal media hot path. +str0m = { workspace = true, features = ["rust-crypto"] } + +[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies] +str0m = { workspace = true, features = ["rust-crypto"] } + +[dev-dependencies] +serde_json.workspace = true + +[features] +sony-peer-probe = [] + +[[bin]] +name = "nvst-peer-probe" +path = "src/bin/nvst_peer_probe.rs" +required-features = ["sony-peer-probe"] diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/bin/nvst_peer_probe.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/bin/nvst_peer_probe.rs new file mode 100644 index 000000000..7d4547cd3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/bin/nvst_peer_probe.rs @@ -0,0 +1,360 @@ +use std::io::{BufRead, Read, Write}; +use std::net::{SocketAddr, UdpSocket}; +use std::process::ExitCode; +use std::time::{Duration, Instant}; + +use opennow_streamer_transport::nvst::logical_ice_addr; +use str0m::net::{Protocol, Receive}; +use str0m::{Candidate, Event, IceCreds, Input, Output, RtcConfig}; + +const CONTROL_RELIABLE_LABEL: &str = "control_channel_reliable"; +const MAX_STDIN_LINE_BYTES: u64 = 4 * 1024; +const MAX_PENDING_COMMANDS: usize = 32; +const MAX_COMMANDS_PER_ITERATION: usize = 8; +const INPUT_PROTOCOL_VERSION: u16 = 3; +const INPUT_VERSION_COMMAND_CODE: u16 = 0x020e; + +fn input_version_command() -> Vec { + let mut command = Vec::with_capacity(6); + command.extend_from_slice(&INPUT_VERSION_COMMAND_CODE.to_le_bytes()); + command.extend_from_slice(&2_u16.to_le_bytes()); + command.extend_from_slice(&INPUT_PROTOCOL_VERSION.to_le_bytes()); + command +} + +struct Options { + bind_ip: std::net::IpAddr, + remote_ip: std::net::IpAddr, + port: u16, + remote_port: u16, + local_ufrag: String, + local_password: String, + remote_ufrag: String, + remote_password: String, + timeout: Duration, +} + +fn parse_options() -> Result { + let mut bind_ip = std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + let mut remote_ip = std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + let mut port = None; + let mut remote_port = None; + let mut local_ufrag = None; + let mut local_password = None; + let mut remote_ufrag = None; + let mut remote_password = None; + let mut timeout_ms = 120_000_u64; + let mut arguments = std::env::args().skip(1); + while let Some(argument) = arguments.next() { + let mut value = || { + arguments + .next() + .ok_or_else(|| format!("missing {argument}")) + }; + match argument.as_str() { + "--bind-ip" => bind_ip = value()?.parse().map_err(|_| "bad bind ip")?, + "--remote-ip" => remote_ip = value()?.parse().map_err(|_| "bad remote ip")?, + "--port" => port = Some(value()?.parse::().map_err(|_| "bad port")?), + "--remote-port" => { + remote_port = Some(value()?.parse::().map_err(|_| "bad remote port")?) + } + "--local-ufrag" => local_ufrag = Some(value()?), + "--local-password" => local_password = Some(value()?), + "--remote-ufrag" => remote_ufrag = Some(value()?), + "--remote-password" => remote_password = Some(value()?), + "--timeout-ms" => timeout_ms = value()?.parse::().map_err(|_| "bad timeout")?, + other => return Err(format!("unknown argument {other}")), + } + } + Ok(Options { + bind_ip, + remote_ip, + port: port.ok_or("missing --port")?, + remote_port: remote_port.ok_or("missing --remote-port")?, + local_ufrag: local_ufrag.ok_or("missing --local-ufrag")?, + local_password: local_password.ok_or("missing --local-password")?, + remote_ufrag: remote_ufrag.ok_or("missing --remote-ufrag")?, + remote_password: remote_password.ok_or("missing --remote-password")?, + timeout: Duration::from_millis(timeout_ms), + }) +} + +fn fingerprint_hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|byte| format!("{byte:02X}")) + .collect::>() + .join(":") +} + +fn payload_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +enum PeerCommand { + Quit, + Cursor { id: u32 }, + Rumble { low_id: u8, low: u8, high: u8 }, +} + +fn parse_peer_command(line: &str) -> Option { + let mut parts = line.split_whitespace(); + match parts.next()? { + "QUIT" => Some(PeerCommand::Quit), + "CURSOR" => Some(PeerCommand::Cursor { + id: parts.next()?.parse().ok()?, + }), + "RUMBLE" => Some(PeerCommand::Rumble { + low_id: parts.next()?.parse().ok()?, + low: parts.next()?.parse().ok()?, + high: parts.next()?.parse().ok()?, + }), + _ => None, + } +} + +fn system_cursor_command(id: u32) -> Vec { + let mut payload = Vec::with_capacity(8); + payload.extend_from_slice(&id.to_le_bytes()); + payload.extend_from_slice(&0_u16.to_le_bytes()); + payload.extend_from_slice(&0_u16.to_le_bytes()); + let mut bytes = Vec::with_capacity(4 + payload.len()); + bytes.extend_from_slice(&0x010f_u16.to_le_bytes()); + bytes.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + bytes.extend_from_slice(&payload); + bytes +} + +fn sony_rumble_command(low_id: u8, low: u8, high: u8) -> Vec { + let mut payload = Vec::with_capacity(13); + payload.extend_from_slice(&0x11_u32.to_le_bytes()); + payload.push(low_id); + payload.push(4); + payload.push(0); + payload.extend_from_slice(&[5, 0x01, 0, 0, low, high]); + let mut bytes = Vec::with_capacity(4 + payload.len()); + bytes.extend_from_slice(&0x0206_u16.to_le_bytes()); + bytes.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + bytes.extend_from_slice(&payload); + bytes +} + +fn emit(line: &str) { + let mut stdout = std::io::stdout().lock(); + let _ = writeln!(stdout, "{line}"); + let _ = stdout.flush(); +} + +fn watch_stdin( + commands: std::sync::mpsc::SyncSender, + stop: std::sync::Arc, +) { + std::thread::spawn(move || { + let stdin = std::io::stdin(); + let mut buffer = String::new(); + let mut discarding_line = false; + loop { + buffer.clear(); + let read = { + let mut bounded = stdin.lock().take(MAX_STDIN_LINE_BYTES); + bounded.read_line(&mut buffer) + }; + match read { + Ok(0) | Err(_) => break, + Ok(_) => { + if !buffer.ends_with('\n') { + discarding_line = true; + continue; + } + if discarding_line { + discarding_line = false; + continue; + } + match parse_peer_command(buffer.trim()) { + Some(PeerCommand::Quit) => break, + Some(command) => { + if commands.try_send(command).is_err() { + continue; + } + } + None => {} + } + } + } + } + stop.store(true, std::sync::atomic::Ordering::Release); + }); +} + +fn run() -> Result<(), String> { + let options = parse_options()?; + let socket = UdpSocket::bind(SocketAddr::new(options.bind_ip, options.port)) + .map_err(|error| error.to_string())?; + socket + .set_read_timeout(Some(Duration::from_millis(1))) + .map_err(|error| error.to_string())?; + let physical_local = socket.local_addr().map_err(|error| error.to_string())?; + if physical_local.ip().is_unspecified() { + return Err("--bind-ip must name the physical local address".to_owned()); + } + let physical_remote = SocketAddr::new(options.remote_ip, options.remote_port); + let local_candidate = logical_ice_addr(physical_local, 2); + let remote_candidate = logical_ice_addr(physical_remote, 1); + + let mut rtc = RtcConfig::new() + .set_fingerprint_verification(false) + .build(Instant::now()); + rtc.direct_api().set_local_ice_credentials(IceCreds { + ufrag: options.local_ufrag, + pass: options.local_password, + }); + rtc.direct_api().set_remote_ice_credentials(IceCreds { + ufrag: options.remote_ufrag, + pass: options.remote_password, + }); + rtc.add_local_candidate( + Candidate::host(local_candidate, "udp").map_err(|error| error.to_string())?, + ); + rtc.add_remote_candidate( + Candidate::host(remote_candidate, "udp").map_err(|error| error.to_string())?, + ); + rtc.direct_api().set_ice_controlling(false); + let placeholder = rtc.direct_api().local_dtls_fingerprint().clone(); + rtc.direct_api().set_remote_fingerprint(placeholder); + rtc.direct_api() + .start_dtls(false) + .map_err(|error| error.to_string())?; + rtc.direct_api().start_sctp(false); + emit(&format!( + "READY {}", + fingerprint_hex(&rtc.direct_api().local_dtls_fingerprint().bytes) + )); + emit(&format!( + "ADDRESSES physicalLocal={physical_local} physicalRemote={physical_remote} logicalLocal={local_candidate} logicalRemote={remote_candidate}" + )); + + let deadline = Instant::now() + options.timeout; + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (command_sender, command_receiver) = std::sync::mpsc::sync_channel(MAX_PENDING_COMMANDS); + watch_stdin(command_sender, stop.clone()); + let mut control_channel: Option = None; + let mut buffer = [0_u8; 65_536]; + let mut payloads = 0_u64; + let mut connected = false; + let mut reported_error = false; + let mut dropped_datagrams = 0_u64; + while Instant::now() < deadline && !stop.load(std::sync::atomic::Ordering::Acquire) { + let now = Instant::now(); + for _ in 0..MAX_COMMANDS_PER_ITERATION { + let Ok(command) = command_receiver.try_recv() else { + break; + }; + match command { + PeerCommand::Quit => {} + PeerCommand::Cursor { id } => { + let sent = control_channel.and_then(|id| rtc.channel(id)).is_some_and( + |mut channel| { + channel + .write(true, &system_cursor_command(id)) + .unwrap_or(false) + }, + ); + emit(&format!("SENT CURSOR id={id} sent={sent}")); + } + PeerCommand::Rumble { low_id, low, high } => { + let sent = control_channel.and_then(|id| rtc.channel(id)).is_some_and( + |mut channel| { + channel + .write(true, &sony_rumble_command(low_id, low, high)) + .unwrap_or(false) + }, + ); + emit(&format!( + "SENT RUMBLE low_id={low_id} low={} high={} sent={sent}", + u16::from(low) << 8, + u16::from(high) << 8 + )); + } + } + } + let _ = rtc.handle_input(Input::Timeout(now)); + loop { + match rtc.poll_output() { + Ok(Output::Timeout(_)) => break, + Ok(Output::Transmit(transmit)) => { + let _ = socket.send_to(transmit.contents.as_ref(), physical_remote); + } + Ok(Output::Event(event)) => match event { + Event::Connected => { + if !connected { + connected = true; + emit("CONNECTED"); + } + } + Event::ChannelOpen(id, label) => { + if label == CONTROL_RELIABLE_LABEL { + control_channel = Some(id); + if let Some(mut channel) = rtc.channel(id) { + let _ = channel.write(true, &input_version_command()); + } + } + emit(&format!("OPEN {id:?} {label}")); + } + Event::ChannelData(data) => { + payloads += 1; + emit(&format!("DATA {payloads} {}", payload_hex(&data.data))); + } + _ => {} + }, + Err(error) => { + if !reported_error { + reported_error = true; + emit(&format!("ERROR {error}")); + } + break; + } + } + } + if let Ok((length, source)) = socket.recv_from(&mut buffer) { + if source != physical_remote { + continue; + } + let contents = match buffer[..length].try_into() { + Ok(contents) => contents, + Err(error) => { + if dropped_datagrams == 0 { + emit(&format!( + "DROPPED source={source} length={length} error={error}" + )); + } + dropped_datagrams += 1; + continue; + } + }; + let receive = Receive { + proto: Protocol::Udp, + source: remote_candidate, + destination: local_candidate, + contents, + }; + if let Err(error) = rtc.handle_input(Input::Receive(now, receive)) { + if !reported_error { + reported_error = true; + emit(&format!("ERROR {error}")); + } + } + } + } + emit(&format!("DONE {payloads}")); + Ok(()) +} + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + emit(&format!("ERROR {error}")); + ExitCode::FAILURE + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/frame_stage_timing.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/frame_stage_timing.rs new file mode 100644 index 000000000..71cc172cc --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/frame_stage_timing.rs @@ -0,0 +1,468 @@ +use std::collections::VecDeque; +use std::time::Instant; + +pub(crate) const STAGE_SAMPLE_CAPACITY: usize = 256; + +pub(crate) const DELIVERY_STAMP_CAPACITY: usize = 512; + +const NANOS_PER_MILLI: f64 = 1_000_000.0; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct StageSummary { + pub p50_ms: f64, + pub p95_ms: f64, + pub max_ms: f64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct FrameStageTimings { + pub delivery_to_admission: Option, + pub admission_to_control_queue: Option, + pub assembled_to_control_queue: Option, + pub delivery_window_samples: usize, + pub ack_window_samples: usize, + pub assembled_frames_total: u64, + pub admitted_frames_total: u64, + pub queued_ack_frames_total: u64, + pub undelivered_frames_total: u64, + pub last_assembled_at: Option, + pub last_admitted_at: Option, + pub last_ack_queued_at: Option, + pub pending_deliveries: usize, + pub unmatched_deliveries: u64, + pub unmatched_admissions: u64, +} + +impl FrameStageTimings { + pub fn is_empty(&self) -> bool { + self.delivery_to_admission.is_none() + && self.admission_to_control_queue.is_none() + && self.assembled_to_control_queue.is_none() + } + + pub fn has_progress(&self) -> bool { + self.assembled_frames_total > 0 + } + + pub fn log_line(&self) -> String { + let stage = |summary: Option| match summary { + Some(summary) => format!( + "p50={:.2} p95={:.2} max={:.2}", + summary.p50_ms, summary.p95_ms, summary.max_ms + ), + None => "unavailable".to_owned(), + }; + format!( + "deliveryToAdmission[{}]={} admissionToControlQueue[{}]={} assembledToControlQueue[{}]={} assembled={} admitted={} queuedAcks={} undelivered={} pendingDeliveries={} unmatchedDelivery={} unmatchedAdmission={}", + self.delivery_window_samples, + stage(self.delivery_to_admission), + self.ack_window_samples, + stage(self.admission_to_control_queue), + self.ack_window_samples, + stage(self.assembled_to_control_queue), + self.assembled_frames_total, + self.admitted_frames_total, + self.queued_ack_frames_total, + self.undelivered_frames_total, + self.pending_deliveries, + self.unmatched_deliveries, + self.unmatched_admissions, + ) + } +} + +#[derive(Debug, Default)] +pub(crate) struct FrameStageTimingsAccumulator { + delivered: VecDeque<(u32, Instant)>, + delivery_to_admission_us: VecDeque, + admission_to_control_queue_us: VecDeque, + assembled_to_control_queue_us: VecDeque, + assembled_frames_total: u64, + admitted_frames_total: u64, + queued_ack_frames_total: u64, + undelivered_frames_total: u64, + last_assembled_at: Option, + last_admitted_at: Option, + last_ack_queued_at: Option, + unmatched_deliveries: u64, + unmatched_admissions: u64, + epoch_started_at: Option, +} + +impl FrameStageTimingsAccumulator { + pub(crate) fn record_assembly(&mut self, frame_index: u32, at: Instant) { + if let Some(position) = self + .delivered + .iter() + .position(|(delivered, _)| *delivered == frame_index) + { + self.delivered.remove(position); + self.unmatched_deliveries = self.unmatched_deliveries.saturating_add(1); + } + while self.delivered.len() >= DELIVERY_STAMP_CAPACITY { + self.delivered.pop_front(); + self.unmatched_deliveries = self.unmatched_deliveries.saturating_add(1); + } + self.delivered.push_back((frame_index, at)); + self.assembled_frames_total = self.assembled_frames_total.saturating_add(1); + self.last_assembled_at = Some(at); + } + + pub(crate) fn retire_undelivered(&mut self, frame_index: u32) { + if let Some(position) = self + .delivered + .iter() + .position(|(delivered, _)| *delivered == frame_index) + { + self.delivered.remove(position); + } + self.undelivered_frames_total = self.undelivered_frames_total.saturating_add(1); + } + + pub(crate) fn record_admission(&mut self, frame_index: u32, at: Instant) -> Option { + self.admitted_frames_total = self.admitted_frames_total.saturating_add(1); + self.last_admitted_at = Some(at); + let Some(position) = self + .delivered + .iter() + .position(|(delivered, _)| *delivered == frame_index) + else { + self.unmatched_admissions = self.unmatched_admissions.saturating_add(1); + return None; + }; + let (_, delivered_at) = self.delivered.remove(position).expect("position found"); + push_sample( + &mut self.delivery_to_admission_us, + at.saturating_duration_since(delivered_at), + ); + Some(delivered_at) + } + + pub(crate) fn record_ack_queued( + &mut self, + assembled_at: Option, + admitted_at: Instant, + at: Instant, + ) { + self.queued_ack_frames_total = self.queued_ack_frames_total.saturating_add(1); + self.last_ack_queued_at = Some(at); + if self + .epoch_started_at + .is_some_and(|epoch_started_at| admitted_at < epoch_started_at) + { + return; + } + push_sample( + &mut self.admission_to_control_queue_us, + at.saturating_duration_since(admitted_at), + ); + if let Some(assembled_at) = assembled_at { + push_sample( + &mut self.assembled_to_control_queue_us, + at.saturating_duration_since(assembled_at), + ); + } + } + + pub(crate) fn reset_epoch(&mut self, at: Instant) { + let retired = self.delivered.len() as u64; + self.delivered.clear(); + self.unmatched_deliveries = self.unmatched_deliveries.saturating_add(retired); + self.epoch_started_at = Some(at); + } + + pub(crate) fn last_assembled_at(&self) -> Option { + self.last_assembled_at + } + + pub(crate) fn snapshot(&self) -> FrameStageTimings { + FrameStageTimings { + delivery_to_admission: summarize(&self.delivery_to_admission_us), + admission_to_control_queue: summarize(&self.admission_to_control_queue_us), + assembled_to_control_queue: summarize(&self.assembled_to_control_queue_us), + delivery_window_samples: self.delivery_to_admission_us.len(), + ack_window_samples: self.admission_to_control_queue_us.len(), + assembled_frames_total: self.assembled_frames_total, + admitted_frames_total: self.admitted_frames_total, + queued_ack_frames_total: self.queued_ack_frames_total, + undelivered_frames_total: self.undelivered_frames_total, + last_assembled_at: self.last_assembled_at, + last_admitted_at: self.last_admitted_at, + last_ack_queued_at: self.last_ack_queued_at, + pending_deliveries: self.delivered.len(), + unmatched_deliveries: self.unmatched_deliveries, + unmatched_admissions: self.unmatched_admissions, + } + } +} + +fn push_sample(samples: &mut VecDeque, value: std::time::Duration) { + while samples.len() >= STAGE_SAMPLE_CAPACITY { + samples.pop_front(); + } + samples.push_back(value.as_nanos().try_into().unwrap_or(u64::MAX)); +} + +fn summarize(samples: &VecDeque) -> Option { + if samples.is_empty() { + return None; + } + let mut sorted: Vec = samples.iter().copied().collect(); + sorted.sort_unstable(); + Some(StageSummary { + p50_ms: percentile_ms(&sorted, 50), + p95_ms: percentile_ms(&sorted, 95), + max_ms: *sorted.last().expect("non-empty") as f64 / NANOS_PER_MILLI, + }) +} + +fn percentile_ms(sorted: &[u64], percentile: u32) -> f64 { + let rank = (sorted.len() * percentile as usize).div_ceil(100).max(1); + sorted[rank.min(sorted.len()) - 1] as f64 / NANOS_PER_MILLI +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn at(base: Instant, millis: u64) -> Instant { + base + Duration::from_millis(millis) + } + + #[test] + fn empty_accumulator_reports_no_stage_instead_of_zero() { + let timings = FrameStageTimingsAccumulator::default().snapshot(); + assert!(timings.is_empty()); + assert_eq!(timings.delivery_window_samples, 0); + assert_eq!(timings.ack_window_samples, 0); + assert_eq!(timings.assembled_frames_total, 0); + assert_eq!(timings.admitted_frames_total, 0); + assert_eq!(timings.queued_ack_frames_total, 0); + assert!(timings.last_assembled_at.is_none()); + assert!(timings.last_admitted_at.is_none()); + assert!(timings.last_ack_queued_at.is_none()); + assert!(!timings.has_progress()); + assert_eq!(timings.pending_deliveries, 0); + assert_eq!(timings.unmatched_deliveries, 0); + assert_eq!(timings.unmatched_admissions, 0); + } + + #[test] + fn matched_samples_become_measured_stage_percentiles() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + for frame in 1..=20_u32 { + accumulator.record_assembly(frame, base); + let assembled = accumulator + .record_admission(frame, at(base, u64::from(frame))) + .expect("delivery stamp"); + assert_eq!(assembled, base); + accumulator.record_ack_queued( + Some(assembled), + at(base, u64::from(frame)), + at(base, 100), + ); + } + let timings = accumulator.snapshot(); + let delivery = timings.delivery_to_admission.expect("delivery samples"); + assert_eq!(delivery.max_ms, 20.0); + assert_eq!(delivery.p50_ms, 10.0); + assert_eq!(delivery.p95_ms, 19.0); + let ack = timings.admission_to_control_queue.expect("ack samples"); + assert_eq!(ack.max_ms, 99.0); + assert_eq!(ack.p50_ms, 89.0); + let age = timings.assembled_to_control_queue.expect("age samples"); + assert_eq!(age.max_ms, 100.0); + assert_eq!(age.p50_ms, 100.0); + assert_eq!(timings.delivery_window_samples, 20); + assert_eq!(timings.ack_window_samples, 20); + assert_eq!(timings.assembled_frames_total, 20); + assert_eq!(timings.admitted_frames_total, 20); + assert_eq!(timings.queued_ack_frames_total, 20); + assert_eq!(timings.pending_deliveries, 0); + assert_eq!(timings.last_assembled_at, Some(base)); + assert_eq!(timings.last_admitted_at, Some(at(base, 20))); + assert_eq!(timings.last_ack_queued_at, Some(at(base, 100))); + assert!(timings.has_progress()); + } + + #[test] + fn admission_without_a_delivery_stamp_is_counted_not_estimated() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + assert_eq!(accumulator.record_admission(7, base), None); + let timings = accumulator.snapshot(); + assert!(timings.delivery_to_admission.is_none()); + assert_eq!(timings.unmatched_admissions, 1); + assert_eq!(timings.pending_deliveries, 0); + } + + #[test] + fn evicted_delivery_stamps_are_counted_as_unmatched() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + for frame in 0..DELIVERY_STAMP_CAPACITY as u32 { + accumulator.record_assembly(frame, base); + } + accumulator.record_assembly(DELIVERY_STAMP_CAPACITY as u32, base); + let timings = accumulator.snapshot(); + assert_eq!(timings.unmatched_deliveries, 1); + assert_eq!(timings.pending_deliveries, DELIVERY_STAMP_CAPACITY); + assert!(timings.delivery_to_admission.is_none()); + } + + #[test] + fn repeated_frame_identifier_retires_the_older_stamp() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_assembly(5, base); + accumulator.record_assembly(5, at(base, 40)); + assert_eq!(accumulator.snapshot().unmatched_deliveries, 1); + let assembled = accumulator + .record_admission(5, at(base, 50)) + .expect("newest stamp"); + assert_eq!(assembled, at(base, 40)); + let timings = accumulator.snapshot(); + assert_eq!( + timings.delivery_to_admission.expect("measured").p50_ms, + 10.0 + ); + assert_eq!(timings.pending_deliveries, 0); + } + + #[test] + fn epoch_reset_retires_pending_stamps_without_fabricating_a_sample() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_assembly(7, base); + accumulator.reset_epoch(at(base, 5_000)); + assert_eq!(accumulator.record_admission(7, at(base, 5_500)), None); + let after_stale_admission = accumulator.snapshot(); + assert!( + after_stale_admission.delivery_to_admission.is_none(), + "a stale admission can never match a retired stamp" + ); + assert_eq!(after_stale_admission.unmatched_deliveries, 1); + assert_eq!(after_stale_admission.unmatched_admissions, 1); + assert_eq!(after_stale_admission.pending_deliveries, 0); + accumulator.record_assembly(7, at(base, 6_000)); + let assembled = accumulator + .record_admission(7, at(base, 6_100)) + .expect("epoch-local stamp"); + assert_eq!(assembled, at(base, 6_000)); + let timings = accumulator.snapshot(); + assert_eq!( + timings.delivery_to_admission.expect("measured").p50_ms, + 100.0 + ); + assert_eq!(timings.assembled_frames_total, 2); + assert_eq!(timings.admitted_frames_total, 2); + } + + #[test] + fn acks_for_pre_epoch_admissions_report_no_cross_epoch_sample() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_assembly(3, base); + let assembled = accumulator + .record_admission(3, at(base, 20)) + .expect("delivery stamp"); + accumulator.reset_epoch(at(base, 4_000)); + accumulator.record_ack_queued(Some(assembled), at(base, 20), at(base, 4_100)); + let timings = accumulator.snapshot(); + assert!( + timings.admission_to_control_queue.is_none(), + "an ACK that spans the epoch boundary reports no stage sample" + ); + assert!(timings.assembled_to_control_queue.is_none()); + assert_eq!(timings.queued_ack_frames_total, 1); + assert_eq!(timings.last_ack_queued_at, Some(at(base, 4_100))); + accumulator.record_assembly(4, at(base, 4_200)); + let assembled = accumulator + .record_admission(4, at(base, 4_250)) + .expect("delivery stamp"); + accumulator.record_ack_queued(Some(assembled), at(base, 4_250), at(base, 4_300)); + let timings = accumulator.snapshot(); + assert_eq!( + timings + .admission_to_control_queue + .expect("epoch-local ack") + .p50_ms, + 50.0 + ); + } + + #[test] + fn stage_rings_stay_bounded_under_continued_load() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + for frame in 0..(STAGE_SAMPLE_CAPACITY as u32 * 3) { + accumulator.record_assembly(frame, base); + let assembled = accumulator + .record_admission(frame, at(base, 1)) + .expect("delivery stamp"); + accumulator.record_ack_queued(Some(assembled), at(base, 1), at(base, 2)); + } + let timings = accumulator.snapshot(); + assert_eq!(timings.delivery_window_samples, STAGE_SAMPLE_CAPACITY); + assert_eq!(timings.ack_window_samples, STAGE_SAMPLE_CAPACITY); + assert_eq!(timings.pending_deliveries, 0); + assert_eq!(timings.unmatched_deliveries, 0); + assert_eq!( + timings.assembled_frames_total, + u64::from(STAGE_SAMPLE_CAPACITY as u32 * 3), + "cumulative totals keep counting after the window saturates" + ); + assert_eq!( + timings.queued_ack_frames_total, + u64::from(STAGE_SAMPLE_CAPACITY as u32 * 3) + ); + } + + #[test] + fn ack_samples_require_admission_and_never_use_sender_time() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_ack_queued(None, base, at(base, 3)); + let timings = accumulator.snapshot(); + assert_eq!( + timings + .admission_to_control_queue + .expect("admission sample") + .p50_ms, + 3.0 + ); + assert!(timings.assembled_to_control_queue.is_none()); + } + + #[test] + fn log_line_names_stages_and_reports_unavailable_honestly() { + let empty = FrameStageTimings::default().log_line(); + assert!(empty.contains("deliveryToAdmission[0]=unavailable")); + assert!(empty.contains("unmatchedDelivery=0")); + + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_assembly(1, base); + accumulator.record_admission(1, at(base, 4)); + accumulator.record_ack_queued(Some(base), at(base, 4), at(base, 9)); + let line = accumulator.snapshot().log_line(); + assert!(line.contains("deliveryToAdmission[1]=p50=4.00")); + assert!(line.contains("assembledToControlQueue[1]=p50=9.00")); + } + + #[test] + fn percentiles_use_nearest_rank_and_never_exceed_the_sample_count() { + let base = Instant::now(); + let mut accumulator = FrameStageTimingsAccumulator::default(); + accumulator.record_assembly(1, base); + let assembled = accumulator.record_admission(1, at(base, 2)).expect("stamp"); + accumulator.record_ack_queued(Some(assembled), at(base, 2), at(base, 3)); + let timings = accumulator.snapshot(); + let delivery = timings.delivery_to_admission.expect("single sample"); + assert_eq!(delivery.p50_ms, 2.0); + assert_eq!(delivery.p95_ms, 2.0); + assert_eq!(delivery.max_ms, 2.0); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/lib.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/lib.rs new file mode 100644 index 000000000..7f59f8d9b --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/lib.rs @@ -0,0 +1,138 @@ +use std::sync::Arc; +use std::sync::Once; +use std::sync::mpsc::{SyncSender, TrySendError}; + +use str0m::crypto::from_feature_flags; +use thiserror::Error; + +mod frame_stage_timing; +pub mod nvst; +mod nvst_audio_jitter; +mod nvst_bandwidth; +mod nvst_budget; +mod nvst_control; +mod nvst_cursor; +mod nvst_haptics; +mod nvst_input; +mod nvst_input_delivery; +mod nvst_invalidation; +mod nvst_microphone; +mod nvst_network; +mod nvst_recovery_policy; +mod nvst_server_control; +pub mod webrtc; + +pub use frame_stage_timing::{FrameStageTimings, StageSummary}; +pub use nvst_haptics::{NvstControllerRumble, NvstHaptics}; +pub use nvst_recovery_policy::NvstRecoveryPolicy; +pub use nvst_server_control::{NvstServerTermination, NvstSessionTimer}; + +pub use nvst::{ + BoundedFrameQueue, EncodedVideoAccessUnit, NvstBundleIdentity, NvstConfigError, NvstDropReason, + NvstFrameProgress, NvstFrameProgressEvent, NvstFrameProgressPolicy, NvstFrameProgressStage, + NvstFrameType, NvstReceiveEvent, NvstReceiverState, NvstRecovery, NvstSrtpProfile, + NvstStopCause, NvstUdpReceiverControl, NvstUdpReceiverError, NvstUdpReceiverSession, + NvstUnsupportedFeature, NvstVideoCodec, NvstVideoConfig, NvstVideoReceiver, ReservedNvstBundle, + SharedNvstFeedback, advertised_nvst_ipv4, measured_video_packet_size, nvst_video_packet_size, + parse_nvst_video_handoff, reserve_nvst_mjolnir_udp_socket, reserve_nvst_udp_socket, + spawn_nvst_mjolnir_receiver, spawn_nvst_udp_receiver, spawn_nvst_udp_receiver_with_socket, +}; + +static INSTALL_CRYPTO: Once = Once::new(); + +#[derive(Debug, Error)] +pub enum TransportError { + #[error("input channel is not ready")] + InputNotReady, + #[error("encoded media consumer is no longer running")] + MediaConsumerClosed, + #[error("encoded media consumer is backpressured")] + MediaConsumerBackpressured, + #[error("transport worker is no longer running")] + Closed, +} + +impl TransportError { + pub const fn code(&self) -> &'static str { + match self { + Self::InputNotReady => "input-not-ready", + Self::MediaConsumerClosed => "media-consumer-closed", + Self::MediaConsumerBackpressured => "media-consumer-backpressured", + Self::Closed => "transport-closed", + } + } +} + +#[derive(Debug, Clone)] +pub struct EncodedMediaFrame { + pub mid: String, + pub codec: String, + pub payload: Arc<[u8]>, + /// Sender-authored video frame index from the NVST GS packet header. + /// Audio and transports without an equivalent identifier leave this unset. + pub frame_index: Option, + pub rtp_timestamp: u64, + pub clock_rate_hz: u32, + pub channels: Option, + pub received_at_us: u64, + pub keyframe: bool, + pub contiguous: bool, + pub ssrc: Option, +} + +pub type MediaConsumer = SyncSender; + +pub fn install_crypto() { + INSTALL_CRYPTO.call_once(|| from_feature_flags().install_process_default()); +} + +fn deliver_media_frame( + consumer: &MediaConsumer, + frame: EncodedMediaFrame, +) -> Result<(), TransportError> { + consumer.try_send(frame).map_err(|error| match error { + TrySendError::Full(_) => TransportError::MediaConsumerBackpressured, + TrySendError::Disconnected(_) => TransportError::MediaConsumerClosed, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame() -> EncodedMediaFrame { + EncodedMediaFrame { + mid: "video-0".to_owned(), + codec: "H264".to_owned(), + payload: Arc::from([1_u8, 2, 3, 4]), + frame_index: Some(42), + rtp_timestamp: 180_000, + clock_rate_hz: 90_000, + channels: None, + received_at_us: 2_500, + keyframe: true, + contiguous: true, + ssrc: None, + } + } + + #[test] + fn media_delivery_preserves_shared_payload() { + let (consumer, receiver) = std::sync::mpsc::sync_channel(1); + let expected = frame(); + let payload = expected.payload.clone(); + deliver_media_frame(&consumer, expected).expect("frame delivery"); + let delivered = receiver.recv().expect("delivered frame"); + assert!(Arc::ptr_eq(&delivered.payload, &payload)); + } + + #[test] + fn media_delivery_reports_bounded_queue_backpressure() { + let (consumer, _receiver) = std::sync::mpsc::sync_channel(1); + deliver_media_frame(&consumer, frame()).expect("first frame delivery"); + assert!(matches!( + deliver_media_frame(&consumer, frame()), + Err(TransportError::MediaConsumerBackpressured) + )); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs new file mode 100644 index 000000000..2f5b1ee4f --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs @@ -0,0 +1,14695 @@ +//! Independently authored NVST video receive transport. +//! +//! This module only implements the receive side of the classic NVST video handoff: +//! authenticated SRTP video datagrams from the negotiated peer become bounded H.264 +//! Annex-B access units. Standard Opus RTP and the existing input data-channel contract +//! use the negotiated DTLS bundle. NVIDIA's systematic Reed-Solomon video FEC is repaired before +//! access-unit assembly so isolated UDP loss does not flush the hardware decoder reference chain. + +use std::collections::{BTreeMap, VecDeque}; +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender, TryRecvError}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use super::nvst_audio_jitter::{AudioJitterBuffer, AudioPacket}; +use aes::cipher::{BlockEncrypt, KeyIvInit, StreamCipher}; +use aes::{Aes128, Aes256}; +use crc32fast::hash as crc32; +use ctr::{Ctr32BE, Ctr128BE}; +use ghash::{GHash, universal_hash::UniversalHash}; +use hmac::{Hmac, Mac}; +use serde_json::Value; +use sha1::Sha1; +use socket2::{Domain, Protocol, Socket, Type}; +use subtle::ConstantTimeEq; +use thiserror::Error; + +use super::nvst_microphone::{ + MICROPHONE_QUEUE_CAPACITY, MicrophonePacketizer, MicrophoneQueue, MicrophoneRtpConfig, +}; +use super::nvst_network::is_vpn_interface; +use super::nvst_server_control::{ + ControlKeepaliveHealth, FramePacingFeedbackMode, NvstServerTermination, NvstSessionTimer, + ServerControlDiagnostics, ServerControlMessage, server_control_messages, +}; +use str0m::channel::ChannelId; +use str0m::config::Fingerprint; +use str0m::format::{Codec, FormatParams}; +use str0m::media::{Frequency, MediaKind, Mid}; +use str0m::net::{Protocol as RtcProtocol, Receive}; +use str0m::rtp::{RtpHeader as BundleRtpHeader, Ssrc}; +use str0m::stats::CandidatePairStats; +use str0m::{ + Candidate, Event, IceConnectionState, IceCreds, Input, Output, Rtc, RtcConfig, RtcError, +}; + +use super::frame_stage_timing::FrameStageTimingsAccumulator; +use super::nvst_bandwidth::{BandwidthEstimator, PacketBandwidthSample}; +use super::nvst_control::{ + DEFAULT_FRAME_TIME_US, FRAME_ACK_PAYLOAD_LEN, FRAME_PACING_INTERVAL, MAX_NACK_PACKET_COUNT, + QOS_REPORT_INTERVAL, QosPacketSnapshot, QosReport, frame_ack, frame_invalidation_range, + frame_pacing_report, idr_request, nack_v2, +}; +use super::nvst_cursor::{CursorCommand, NvstCursorCapture, valid_cursor_channel_message}; +use super::nvst_haptics::NvstHaptics; +use super::nvst_input::{ + GenericHidMessage, NvstEncodedInput, NvstInputChannelState, NvstInputChannels, NvstInputCodec, + NvstInputWrite, SonyDeviceControl, for_each_generic_hid_message, for_each_sony_output, + generic_hid_command, native_input_type_is_motion, native_input_type_name, native_input_types, + next_control_keepalive, server_cursor_messages, sony_device_change_command, + sony_report_command, +}; +use super::nvst_input_delivery::{NvstInputDelivery, NvstInputDeliveryError}; +pub use super::nvst_invalidation::NvstFrameType; +use super::nvst_invalidation::{FrameInvalidation, FrameInvalidationFeedback}; +use super::{ + EncodedMediaFrame, MediaConsumer, TransportError, deliver_media_frame, install_crypto, +}; +use opennow_streamer_hid::generic::{GenericError, GenericOutbound, GenericSession}; +use opennow_streamer_hid::generic_runtime::{GenericIngress, GenericRuntime}; +use opennow_streamer_hid::{ + DRAIN_PER_ITERATION, EndpointToken, HidOutbound, HidRuntime, HidSession, IngressItem, + SonyCapability, +}; + +const RTP_FIXED_HEADER_LEN: usize = 12; +pub const MAX_NVST_VIDEO_PEER_PORTS: u16 = 16; +const SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN: usize = 10; +/// RFC 7714 `AEAD_AES_*_GCM` profiles carry a 16-byte authentication tag. +const SRTP_AEAD_AES_GCM_TAG_LEN: usize = 16; +/// NVIDIA's `SecureRtp` (libBifrost2) maps `sec_serv_conf_and_auth` + 256-bit keys +/// to `srtp_crypto_policy_set_aes_gcm_256_8_auth` — an 8-byte tag, not RFC 7714's 16. +const SRTP_AEAD_AES_GCM_8_TAG_LEN: usize = 8; + +/// RFC 3711 / libsrtp SRTP labels. Hook captures of 0x03/0x05 were SRTCP (same +/// master key, separate session keys). Mjolnir video is RTP, so use 0x00/0x02. +const GFN_SRTP_KEY_LABEL: u8 = 0x00; +const GFN_SRTP_SALT_LABEL: u8 = 0x02; +/// RFC 3711 SRTCP KDF labels (same master key, separate session keys). +const GFN_SRTCP_KEY_LABEL: u8 = 0x03; +const GFN_SRTCP_SALT_LABEL: u8 = 0x05; +const SRTCP_ENCRYPTED_FLAG: u32 = 0x8000_0000; +const RTCP_SENDER_SSRC: u32 = 0x4f4e_4f57; // "ONOW" +const SRTCP_RR_INTERVAL: Duration = Duration::from_secs(1); +const RTCP_RECOVERY_INTERVAL: Duration = Duration::from_millis(4); +const NACK_RETRY_INTERVAL: Duration = Duration::from_millis(4); +const DEFAULT_NACK_RTT: Duration = Duration::from_millis(30); +const NACK_TRACKING_TIMEOUT: Duration = Duration::from_millis(52); +const MAX_NACK_ATTEMPTS: u8 = 3; +const RFI_BACKSTOP_TIMEOUT: Duration = Duration::from_millis(250); +const IDR_RETRY_INTERVAL: Duration = Duration::from_millis(100); +const MAX_PENDING_NACK_PACKETS: usize = 2048; +const MAX_PENDING_FRAME_ACKS: usize = 512; +pub const MAX_CONTROL_REPORT_BYTES: usize = 1_071; +pub const MIN_CONTROL_REPORT_BYTES: usize = 4 + FRAME_ACK_PAYLOAD_LEN; +const FRAME_ACK_RECORD_LEN: usize = 4 + FRAME_ACK_PAYLOAD_LEN; +const STREAM_PING_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_PENDING_STREAM_PINGS: usize = 64; + +#[derive(Default)] +struct StreamPingTracker { + pending: VecDeque<([u8; 12], Instant)>, +} + +#[derive(Default)] +struct IceResponseTracker { + requests: VecDeque<([u8; 12], bool)>, +} + +impl IceResponseTracker { + fn sent(&mut self, packet: &[u8]) { + if !looks_like_stun(packet) || packet[..2] != STUN_BINDING_REQUEST.to_be_bytes() { + return; + } + let transaction_id = packet[8..20].try_into().expect("STUN header checked"); + if self.requests.iter().any(|(id, _)| *id == transaction_id) { + return; + } + if self.requests.len() == MAX_PENDING_STREAM_PINGS { + self.requests.pop_front(); + } + self.requests.push_back((transaction_id, false)); + } + + fn accept(&mut self, packet: &[u8], remote_password: &str) -> bool { + if !looks_like_stun(packet) + || packet[..2] != STUN_BINDING_SUCCESS_RESPONSE.to_be_bytes() + || STUN_HEADER_LEN + usize::from(u16::from_be_bytes([packet[2], packet[3]])) + != packet.len() + { + return true; + } + let Some((_, completed)) = self + .requests + .iter_mut() + .find(|(id, _)| id.as_slice() == &packet[8..20]) + else { + return true; + }; + if !valid_stun_message_integrity(packet, remote_password.as_bytes()) + || (find_stun_attribute(packet, STUN_ATTR_FINGERPRINT).is_some() + && !valid_stun_fingerprint(packet)) + { + return true; + } + !std::mem::replace(completed, true) + } +} + +impl StreamPingTracker { + fn sent(&mut self, transaction_id: [u8; 12], now: Instant) { + self.pending + .retain(|(_, sent)| now.saturating_duration_since(*sent) < STREAM_PING_TIMEOUT); + if self.pending.len() == MAX_PENDING_STREAM_PINGS { + self.pending.pop_front(); + } + self.pending.push_back((transaction_id, now)); + } + + fn receive( + &mut self, + packet: &[u8], + source: SocketAddr, + credentials: &NvstStunCredentials, + now: Instant, + ) -> Option { + if packet.get(..2)? != STUN_BINDING_SUCCESS_RESPONSE.to_be_bytes() { + return None; + } + let transaction_id = packet.get(8..20)?; + let index = self + .pending + .iter() + .position(|(id, _)| id == transaction_id)?; + if !matches!( + handle_stun_datagram(packet, source, credentials), + StunDatagram::Handled(None) + ) { + return None; + } + let (_, sent) = self.pending.remove(index)?; + let elapsed = now.checked_duration_since(sent)?; + (elapsed < STREAM_PING_TIMEOUT).then_some(elapsed) + } +} + +fn verbose_diagnostics_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("OPENNOW_NVST_TRACE") + .ok() + .is_some_and(|value| matches!(value.trim(), "1" | "true" | "TRUE" | "yes" | "on")) + }) +} +const MAX_NACK_FCI_ENTRIES: usize = MAX_NACK_PACKET_COUNT.div_ceil(17); +// At 120 FPS several FEC blocks can arrive during a single retransmission round trip. Keep them +// ordered instead of throwing away the incomplete head block when its successor arrives. +const MAX_PENDING_FEC_BLOCKS: usize = 128; +const NV_VIDEO_PACKET_LEN: usize = 16; +// GameStream's Reed-Solomon shards cover packetSize plus room for the fixed RTP header and +// extension prefix. This is MAX_RTP_HEADER_SIZE in Moonlight's reference receive path. +const NVST_FEC_RTP_HEADER_ALLOWANCE: usize = 16; +const DEFAULT_NVST_VIDEO_PACKET_SIZE: usize = 1_280; +const MIN_NVST_VIDEO_PACKET_SIZE: usize = 256; +const MAX_NVST_VIDEO_PACKET_SIZE: usize = 65_519; +const UDP_HEADER_BYTES: usize = 8; +// Match the official client's bounded NACK/dejitter envelope: it keeps up to +// 1,024 RTP packets available for late or retransmitted packets and permits a +// 2,048-entry NACK queue. A 32-packet window is only a few milliseconds at +// 150 Mbps and turns recoverable reordering into visible keyframe hitches. +const DEFAULT_REORDER_WINDOW: usize = 1_024; +const MAX_REORDER_WINDOW: usize = 2_048; +// Cloud packet reordering plus RTCP-over-SCTP NACK round trips can exceed one or two frame times. +// Keep the wait exceptional and bounded, but long enough for a retransmission to beat an IDR +// reset. This adds latency only while a sequence gap is open; the normal path still drains at once. +const MJOLNIR_REORDER_DEQUEUE_TIMEOUT: Duration = Duration::from_millis(52); +// The replay filter must be at least as deep as the reorder/NACK window. The old 64-packet bitmap +// rejected valid retransmissions after roughly 10 ms on a high-bitrate stream, guaranteeing that +// every NACK eventually degraded into a keyframe reset. +const SRTP_REPLAY_WINDOW_PACKETS: usize = MAX_REORDER_WINDOW; +const SRTP_REPLAY_WINDOW_WORDS: usize = SRTP_REPLAY_WINDOW_PACKETS.div_ceil(u64::BITS as usize); +const NVST_UDP_RECEIVE_BUFFER_BYTES: usize = 8 * 1024 * 1024; +const DEFAULT_MAX_ACCESS_UNIT_BYTES: usize = 2 * 1024 * 1024; +const MAX_ACCESS_UNIT_BYTES: usize = 16 * 1024 * 1024; +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); +const MIN_TIMEOUT: Duration = Duration::from_millis(250); +const MAX_TIMEOUT: Duration = Duration::from_secs(90); +const MAX_PING_BYTES: usize = 512; +const PING_INTERVAL_BEFORE_CONNECTION: Duration = Duration::from_millis(20); +const PING_INTERVAL_AFTER_CONNECTION: Duration = Duration::from_millis(100); +const UDP_RECEIVE_POLL_INTERVAL: Duration = Duration::from_millis(10); +// The WebRTC bundle owns the SCTP input channels. A 10 ms socket wait batches +// raw mouse reports and makes high-refresh streams feel closer to 100 Hz. +const CONTROL_RECEIVE_POLL_INTERVAL: Duration = Duration::from_millis(1); +const STUN_HEADER_LEN: usize = 20; +const STUN_MAGIC_COOKIE: u32 = 0x2112_a442; +const STUN_BINDING_REQUEST: u16 = 0x0001; +const STUN_BINDING_SUCCESS_RESPONSE: u16 = 0x0101; +const STUN_ATTR_USERNAME: u16 = 0x0006; +#[cfg(test)] +const STUN_ATTR_PRIORITY: u16 = 0x0024; +const STUN_ATTR_MESSAGE_INTEGRITY: u16 = 0x0008; +const STUN_ATTR_XOR_MAPPED_ADDRESS: u16 = 0x0020; +const STUN_ATTR_FINGERPRINT: u16 = 0x8028; +const STUN_FINGERPRINT_XOR: u32 = 0x5354_554e; +const MAX_ICE_CREDENTIAL_BYTES: usize = 256; + +/// The independently documented `NV_VIDEO_PACKET` flag values used by an earlier OpenNOW +/// implementation. This module does not borrow code or binaries from NVIDIA. +const FLAG_CONTAINS_PIC_DATA: u8 = 0x01; +const FLAG_EOF: u8 = 0x02; +const FLAG_SOF: u8 = 0x04; +const STREAM_PACKET_INDEX_MASK: u32 = 0x00ff_ffff; +/// Mjolnir video RTP packets carry the per-packet video metadata in a fixed +/// 16-byte RTP extension block with profile `0x4753` ("GS"), not in the payload. +const GS_VIDEO_EXTENSION_PROFILE: u16 = 0x4753; +const MAX_GS_FRAME_HEADER_BYTES: usize = 64; +const GS_SHORT_FRAME_HEADER_BYTES: usize = 8; +// GFN's current cloud NVST protocol uses a compact extended frame header. It +// is not the 44-byte extended header used by current consumer GameStream. +const GFN_EXTENDED_FRAME_HEADER_BYTES: usize = 20; +const GFN_RED_PAYLOAD_TYPE: u8 = 63; +const GFN_OPUS_PAYLOAD_TYPE: u8 = 111; +const MAX_REDUNDANT_AUDIO_BLOCKS: usize = 8; +const MAX_OPUS_PACKET_BYTES: usize = 1_275; + +fn diagnostic_hex(bytes: &[u8], limit: usize) -> String { + let mut value = bytes + .iter() + .take(limit) + .map(|byte| format!("{byte:02x}")) + .collect::(); + if bytes.len() > limit { + value.push_str("..."); + } + value +} + +type Aes256Ctr = Ctr128BE; +type Aes128Ctr = Ctr128BE; +type HmacSha1 = Hmac; + +/// The SRTP profile must come from negotiated metadata. The legacy `nvstVideo` handoff has no +/// profile field; Bifrost's Mjolnir video path hardcodes `aes_gcm_256_8_auth`, so the legacy +/// default selects the 8-byte-tag GCM variant. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstSrtpProfile { + AeadAes128Gcm, + AeadAes256Gcm, + AeadAes128Gcm8, + AeadAes256Gcm8, + AesCm128HmacSha1_32, + AesCm128HmacSha1_80, + AesCm256HmacSha1_32, + AesCm256HmacSha1_80, +} + +impl NvstSrtpProfile { + fn parse(value: &str) -> Result { + match value.trim().to_ascii_uppercase().as_str() { + "AEAD_AES_128_GCM" | "SRTP_AEAD_AES_128_GCM" => Ok(Self::AeadAes128Gcm), + "AEAD_AES_256_GCM" | "SRTP_AEAD_AES_256_GCM" => Ok(Self::AeadAes256Gcm), + "AEAD_AES_128_GCM_8" | "SRTP_AEAD_AES_128_GCM_8" => Ok(Self::AeadAes128Gcm8), + "AEAD_AES_256_GCM_8" | "SRTP_AEAD_AES_256_GCM_8" => Ok(Self::AeadAes256Gcm8), + "AES_CM_128_HMAC_SHA1_32" | "SRTP_AES_CM_128_HMAC_SHA1_32" => { + Ok(Self::AesCm128HmacSha1_32) + } + "AES_CM_128_HMAC_SHA1_80" | "SRTP_AES_CM_128_HMAC_SHA1_80" => { + Ok(Self::AesCm128HmacSha1_80) + } + "AES_CM_256_HMAC_SHA1_32" | "SRTP_AES_CM_256_HMAC_SHA1_32" => { + Ok(Self::AesCm256HmacSha1_32) + } + "AES_CM_256_HMAC_SHA1_80" | "SRTP_AES_CM_256_HMAC_SHA1_80" => { + Ok(Self::AesCm256HmacSha1_80) + } + other => Err(NvstConfigError::UnsupportedSrtpProfile(other.to_owned())), + } + } +} + +/// Video codecs accepted by the native NVST receive path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstVideoCodec { + H264, + H265, + Av1, +} + +impl NvstVideoCodec { + fn parse(value: &str) -> Result { + match value.trim().to_ascii_uppercase().as_str() { + "H264" | "AVC" => Ok(Self::H264), + "H265" | "HEVC" => Ok(Self::H265), + "AV1" => Ok(Self::Av1), + other => Err(NvstConfigError::UnsupportedCodec(other.to_owned())), + } + } + + pub const fn label(self) -> &'static str { + match self { + Self::H264 => "H264", + Self::H265 => "H265", + Self::Av1 => "AV1", + } + } +} + +#[derive(Debug, Error)] +pub enum NvstConfigError { + #[error("nvstVideo must be an object")] + HandoffNotObject, + #[error("nvstVideo is missing {0}")] + MissingField(&'static str), + #[error("nvstVideo.{field} must be a {expected}")] + InvalidFieldType { + field: &'static str, + expected: &'static str, + }, + #[error("nvstVideo.{field} is out of range")] + OutOfRange { field: &'static str }, + #[error("nvstVideo.videoPeerIp is not a routable unicast IP address: {0}")] + InvalidPeerIp(String), + #[error("nvstVideo.srtpAesKeyHex is invalid for the selected SRTP profile")] + InvalidAesKey, + #[error("nvstVideo.srtpSaltHex is invalid for the selected SRTP profile")] + InvalidSrtpSalt, + #[error("nvstVideo.srtpProfile {0} is not implemented")] + UnsupportedSrtpProfile(String), + #[error("NVST video codec {0} is not implemented; supported codecs are H264, H265, and AV1")] + UnsupportedCodec(String), + #[error("nvstVideo.audioTrack must contain standard negotiated Opus RTP metadata")] + InvalidAudioTrack, + #[error("nvstTransport cannot be used without the required nvstVideo handoff")] + MissingNvstVideoHandoff, +} + +/// Explicitly records transport features that cannot be sent correctly from current wire data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstUnsupportedFeature { + FecRepair, +} + +impl fmt::Display for NvstUnsupportedFeature { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::FecRepair => "FEC repair", + }; + formatter.write_str(name) + } +} + +/// Feedback plane shared between the Mjolnir video receiver (which learns the +/// stream SSRC/sequence and detects unrecoverable loss) and the ICE/DTLS bundle +/// (which owns the `rtcp1` SCTP data channel used for RTCP feedback). +/// +/// The official client sends RTCP Receiver Reports / PLI over an SCTP data +/// channel on the bundle ("RTCP over SCTP is a must for One SDK video to +/// function"). Without it the server stops video after a short provisional +/// window. This state lets the bundle build accurate reports for the stream the +/// Mjolnir receiver is actually seeing. +#[derive(Debug, Default)] +struct ReceptionTiming { + first_arrival: Option, + first_rtp_timestamp: u32, + last_rtp_timestamp: u32, + latest_rtp_timestamp: Option, + qos_packet_snapshot: Option, + last_transit: i64, + jitter: f64, + bandwidth: BandwidthEstimator, +} + +#[derive(Debug, Clone, Copy)] +struct RtcpReportBlock { + media_ssrc: u32, + fraction_lost: u8, + cumulative_lost: i32, + highest_sequence: u32, + jitter: u32, +} + +#[derive(Debug, Clone, Copy)] +struct CompletedFrameFeedback { + frame_number: u32, + bytes: u32, + accepted_at: Instant, + assembled_at: Option, + packet_timing: Option, +} + +#[derive(Debug, Clone, Copy)] +struct FramePacketTiming { + frame_number: u32, + first_at: Instant, + assembled: bool, +} + +impl CompletedFrameFeedback { + fn command(&self, origin: Instant) -> super::nvst_control::NvstControlCommand { + let first_packet_time_ms = self.packet_timing.map(|timing| { + timing + .first_at + .saturating_duration_since(origin) + .as_secs_f64() + * 1_000.0 + }); + frame_ack(self.frame_number, first_packet_time_ms, self.bytes) + } +} + +struct ControlReportBatch { + max_bytes: usize, + bytes: Vec, + queued_at: Option, + frames: Vec, + qos: Option, + last_qos_report: QosReport, + last_qos_send: Instant, + frame_acks_sent: u64, + qos_reports_sent: u64, + last_ack_frame: Option, +} + +impl ControlReportBatch { + fn new(now: Instant, max_bytes: usize) -> Self { + Self { + max_bytes, + bytes: Vec::with_capacity(max_bytes), + queued_at: None, + frames: Vec::with_capacity(max_bytes / FRAME_ACK_RECORD_LEN), + qos: None, + last_qos_report: QosReport::default(), + last_qos_send: now - QOS_REPORT_INTERVAL, + frame_acks_sent: 0, + qos_reports_sent: 0, + last_ack_frame: None, + } + } + + fn fits(&self, record_len: usize) -> bool { + self.bytes.len().saturating_add(record_len) <= self.max_bytes + } + + fn append(&mut self, record: &[u8], now: Instant) { + assert!(self.fits(record.len())); + self.queued_at.get_or_insert(now); + self.bytes.extend_from_slice(record); + } + + fn append_ack(&mut self, frame: CompletedFrameFeedback, record: &[u8], now: Instant) { + self.append(record, now); + self.frames.push(frame); + } + + fn append_qos(&mut self, report: QosReport, record: &[u8], now: Instant) { + self.append(record, now); + self.qos = Some(report); + } + + fn flush_due(&self, now: Instant) -> bool { + self.qos.is_some() + || self.queued_at.is_some_and(|queued_at| { + now.saturating_duration_since(queued_at) >= QOS_REPORT_INTERVAL + }) + } + + fn flush( + &mut self, + now: Instant, + feedback: &NvstFeedbackState, + send: impl FnOnce(&[u8]) -> bool, + ) -> bool { + if self.bytes.is_empty() { + return true; + } + if !send(&self.bytes) { + return false; + } + for frame in self.frames.drain(..) { + feedback + .frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .record_ack_queued(frame.assembled_at, frame.accepted_at, now); + self.frame_acks_sent = self.frame_acks_sent.saturating_add(1); + self.last_ack_frame = Some(frame.frame_number); + } + if let Some(report) = self.qos.take() { + self.last_qos_report = report; + self.last_qos_send = now; + self.qos_reports_sent = self.qos_reports_sent.saturating_add(1); + } + self.bytes.clear(); + self.queued_at = None; + true + } +} + +#[derive(Debug, Clone, Copy)] +struct PendingNack { + requested_at: Instant, + last_sent_at: Option, + attempts: u8, +} + +#[derive(Debug, Default, Clone, Copy)] +enum VideoStartup { + #[default] + Immediate, + WaitingForPlay, + PlayComplete(Instant), +} + +const RECENT_NETWORK_MAX_SAMPLE_GAP: Duration = Duration::from_secs(3); +const RECENT_NETWORK_SAMPLE_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct NetworkCounters { + ssrc: u32, + base: u32, + highest: u32, + received: u32, +} + +#[derive(Debug, Default)] +struct RecentNetworkMetrics { + previous: Option<(Instant, NetworkCounters)>, +} + +#[derive(Debug, Clone, Copy)] +enum StreamSocket { + Bundle, + Video, +} + +impl RecentNetworkMetrics { + fn sample(&mut self, now: Instant, counters: NetworkCounters) -> Option { + if counters.ssrc == 0 || counters.base == u32::MAX || counters.highest < counters.base { + self.previous = None; + return None; + } + if let Some((sampled_at, previous)) = self.previous { + if now < sampled_at { + return None; + } + if counters.ssrc != previous.ssrc + || counters.base != previous.base + || counters.highest < previous.highest + || counters.received < previous.received + { + self.previous = Some((now, counters)); + return None; + } else if counters.received == previous.received { + return None; + } else if now.duration_since(sampled_at) > RECENT_NETWORK_MAX_SAMPLE_GAP { + self.previous = Some((now, counters)); + return None; + } else if now.duration_since(sampled_at) < RECENT_NETWORK_SAMPLE_INTERVAL { + return None; + } + let expected = counters.highest - previous.highest; + let received = counters.received - previous.received; + self.previous = Some((now, counters)); + return (expected > 0).then(|| { + f64::from(expected.saturating_sub(received)) * 100.0 / f64::from(expected) + }); + } + self.previous = Some((now, counters)); + None + } +} + +#[derive(Debug)] +pub struct NvstFeedbackState { + pub(crate) idle_warning_active: AtomicBool, + frame_pacing_enabled: AtomicBool, + pub haptics: crate::NvstHaptics, + /// Bound video stream SSRC (0 until the first packet is authenticated). + video_ssrc: AtomicU32, + /// Highest extended sequence number received on the video stream. + highest_sequence: AtomicU32, + base_sequence: AtomicU32, + received_packets: AtomicU32, + report_prior: Mutex<(u32, u32)>, + recent_network_metrics: Mutex, + bundle_receive_bytes: AtomicU64, + authenticated_transport_ready: AtomicBool, + video_receive_bytes: AtomicU64, + reception_timing: Mutex, + ice_ping: Mutex>, + video_ping: Mutex>, + bundle_ping: Mutex>, + /// Remains set through send attempts until assembly receives a fresh keyframe. + keyframe_needed: AtomicBool, + pub(crate) frame_invalidation: FrameInvalidationFeedback, + pending_nacks: Mutex>, + nack_overflows: AtomicU64, + last_sender_frame: AtomicU32, + completed_frame_bytes: AtomicU64, + pending_frame_acks: Mutex>, + frame_packet_timings: Mutex>, + frame_stage_timings: Mutex, + video_startup: Mutex, +} + +impl Default for NvstFeedbackState { + fn default() -> Self { + Self { + idle_warning_active: AtomicBool::new(false), + frame_pacing_enabled: AtomicBool::new(true), + haptics: crate::NvstHaptics::default(), + video_ssrc: AtomicU32::new(0), + highest_sequence: AtomicU32::new(0), + base_sequence: AtomicU32::new(u32::MAX), + received_packets: AtomicU32::new(0), + report_prior: Mutex::new((0, 0)), + recent_network_metrics: Mutex::new(RecentNetworkMetrics::default()), + bundle_receive_bytes: AtomicU64::new(0), + authenticated_transport_ready: AtomicBool::new(false), + video_receive_bytes: AtomicU64::new(0), + reception_timing: Mutex::new(ReceptionTiming::default()), + ice_ping: Mutex::new(None), + video_ping: Mutex::new(None), + bundle_ping: Mutex::new(None), + keyframe_needed: AtomicBool::new(false), + frame_invalidation: FrameInvalidationFeedback::default(), + pending_nacks: Mutex::new(BTreeMap::new()), + nack_overflows: AtomicU64::new(0), + last_sender_frame: AtomicU32::new(0), + completed_frame_bytes: AtomicU64::new(0), + pending_frame_acks: Mutex::new(VecDeque::new()), + frame_packet_timings: Mutex::new(VecDeque::new()), + frame_stage_timings: Mutex::new(FrameStageTimingsAccumulator::default()), + video_startup: Mutex::new(VideoStartup::Immediate), + } + } +} + +impl NvstFeedbackState { + fn receive_server_control( + &self, + bytes: &[u8], + diagnostics: &mut ServerControlDiagnostics, + events: &Sender, + ) -> bool { + for message in server_control_messages(bytes) { + match message { + Ok(ServerControlMessage::Timer(timer)) => { + self.idle_warning_active + .store(timer.active, Ordering::Release); + let _ = events.send(NvstReceiveEvent::SessionTimer(timer)); + } + Ok(ServerControlMessage::Termination { wire_code, extra }) => { + self.idle_warning_active.store(false, Ordering::Release); + let _ = events.send(NvstReceiveEvent::ServerTermination( + NvstServerTermination::from_wire(wire_code, extra), + )); + return true; + } + Ok(ServerControlMessage::FramePacingMode { stream_index, mode }) => { + if stream_index == 0 { + self.frame_pacing_enabled + .store(mode == FramePacingFeedbackMode::Interval, Ordering::Release); + } else { + diagnostics.invalid_stream = diagnostics.invalid_stream.saturating_add(1); + } + } + Ok(ServerControlMessage::Unhandled) => {} + Ok(ServerControlMessage::Unknown(code)) => { + diagnostics.unknown = diagnostics.unknown.saturating_add(1); + diagnostics.last_unknown = Some(code); + } + Err(_) => diagnostics.malformed = diagnostics.malformed.saturating_add(1), + } + } + false + } + + pub fn authenticated_transport_ready(&self) -> bool { + self.authenticated_transport_ready.load(Ordering::Acquire) + || self.socket_receive_bytes() > 0 + } + + fn wait_for_play(&self) { + *self + .video_startup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = VideoStartup::WaitingForPlay; + } + + fn complete_play(&self, now: Instant) { + let mut startup = self + .video_startup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if matches!(*startup, VideoStartup::WaitingForPlay) { + *startup = VideoStartup::PlayComplete(now); + } + } + + fn video_timeout_origin(&self, default: Instant) -> Option { + match *self + .video_startup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + { + VideoStartup::Immediate => Some(default), + VideoStartup::WaitingForPlay => None, + VideoStartup::PlayComplete(at) => Some(at.max(default)), + } + } + + fn record_socket_receive( + &self, + socket: StreamSocket, + peer_matches: bool, + authenticated: bool, + length: usize, + ) { + if !peer_matches || !authenticated { + return; + } + let counter = match socket { + StreamSocket::Bundle => &self.bundle_receive_bytes, + StreamSocket::Video => &self.video_receive_bytes, + }; + let mut bytes = counter.load(Ordering::Relaxed); + loop { + let next = bytes.saturating_add(length as u64); + match counter.compare_exchange_weak(bytes, next, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(current) => bytes = current, + } + } + } + + pub fn socket_receive_bytes(&self) -> u64 { + self.bundle_receive_bytes + .load(Ordering::Relaxed) + .saturating_add(self.video_receive_bytes.load(Ordering::Relaxed)) + } + + fn update_ice_ping( + &self, + pair: Option<&CandidatePairStats>, + previous_responses: &mut u64, + now: Instant, + ) { + let mut sample = self + .ice_ping + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(pair) = pair else { + *previous_responses = 0; + *sample = None; + return; + }; + if pair.responses_received != *previous_responses { + *previous_responses = pair.responses_received; + *sample = pair.current_round_trip_time.map(|elapsed| (now, elapsed)); + } + } + + fn publish_ping(&self, video: bool, now: Instant, elapsed: Duration) { + let sample = if video { + &self.video_ping + } else { + &self.bundle_ping + }; + *sample + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some((now, elapsed)); + } + + pub fn ping_ms(&self, now: Instant) -> Option { + self.ping(now).map(|elapsed| elapsed.as_secs_f64() * 1000.0) + } + + fn ping(&self, now: Instant) -> Option { + [&self.ice_ping, &self.video_ping, &self.bundle_ping] + .into_iter() + .find_map(|sample| { + sample + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .filter(|(received, _)| { + now.saturating_duration_since(*received) < STREAM_PING_TIMEOUT + }) + .map(|(_, elapsed)| elapsed) + }) + } + + #[cfg(test)] + fn publish_stream(&self, ssrc: u32, highest_sequence: u32, rtp_timestamp: u32, now: Instant) { + self.publish_stream_packet(ssrc, highest_sequence, rtp_timestamp, now, None); + } + + fn publish_stream_packet( + &self, + ssrc: u32, + highest_sequence: u32, + rtp_timestamp: u32, + now: Instant, + video: Option<(u32, u32, usize, bool, bool, bool)>, + ) { + self.video_ssrc.store(ssrc, Ordering::Release); + let _ = self.base_sequence.compare_exchange( + u32::MAX, + highest_sequence, + Ordering::AcqRel, + Ordering::Acquire, + ); + self.highest_sequence + .fetch_max(highest_sequence, Ordering::AcqRel); + self.received_packets.fetch_add(1, Ordering::AcqRel); + + let mut timing = self + .reception_timing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if timing + .qos_packet_snapshot + .is_some_and(|previous| previous.ssrc != ssrc) + { + timing.bandwidth = BandwidthEstimator::default(); + timing.first_arrival = None; + timing.latest_rtp_timestamp = None; + timing.jitter = 0.0; + timing.last_transit = 0; + } + if let Some((frame_number, sequence, bytes, started, ended, parity)) = video { + if parity { + timing.bandwidth.observe_parity(frame_number, sequence); + } else { + timing.bandwidth.observe(PacketBandwidthSample { + frame_number, + sequence, + timestamp: rtp_timestamp, + bytes, + started, + ended, + received_at: now, + }); + } + } + timing.qos_packet_snapshot = Some(match timing.qos_packet_snapshot { + Some(previous) if previous.ssrc == ssrc && previous.received < u32::MAX => { + QosPacketSnapshot { + highest: previous.highest.max(highest_sequence), + received: previous.received + 1, + ..previous + } + } + _ => QosPacketSnapshot { + ssrc, + base: highest_sequence, + highest: highest_sequence, + received: 1, + }, + }); + if timing + .latest_rtp_timestamp + .is_none_or(|latest| rtp_timestamp.wrapping_sub(latest) as i32 > 0) + { + timing.latest_rtp_timestamp = Some(rtp_timestamp); + } + let Some(first_arrival) = timing.first_arrival else { + timing.first_arrival = Some(now); + timing.first_rtp_timestamp = rtp_timestamp; + timing.last_rtp_timestamp = rtp_timestamp; + return; + }; + if timing.last_rtp_timestamp == rtp_timestamp { + return; + } + let arrival_ticks = now + .saturating_duration_since(first_arrival) + .as_secs_f64() + .mul_add(90_000.0, 0.0) as i64; + let rtp_ticks = i64::from(rtp_timestamp.wrapping_sub(timing.first_rtp_timestamp)); + let transit = arrival_ticks - rtp_ticks; + let delta = transit.abs_diff(timing.last_transit) as f64; + timing.last_rtp_timestamp = rtp_timestamp; + timing.last_transit = transit; + timing.jitter += (delta - timing.jitter) / 16.0; + } + + pub fn request_keyframe(&self) { + self.keyframe_needed.store(true, Ordering::Release); + } + + fn request_nack(&self, first_missing_index: u64, last_missing_index: u64, now: Instant) { + if first_missing_index > last_missing_index { + return; + } + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending.retain(|_, packet| { + now.saturating_duration_since(packet.requested_at) < NACK_TRACKING_TIMEOUT + }); + for index in first_missing_index..=last_missing_index { + if pending.contains_key(&index) { + continue; + } + if pending.len() == MAX_PENDING_NACK_PACKETS { + self.nack_overflows.fetch_add(1, Ordering::Relaxed); + self.request_keyframe(); + break; + } + pending.insert( + index, + PendingNack { + requested_at: now, + last_sent_at: None, + attempts: 0, + }, + ); + } + } + + fn resolve_nack(&self, received_index: u64) -> bool { + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending + .remove(&received_index) + .is_some_and(|packet| packet.last_sent_at.is_some()) + } + + fn publish_completed_frame(&self, frame: &EncodedVideoAccessUnit) { + self.last_sender_frame + .store(frame.frame_index, Ordering::Release); + self.completed_frame_bytes.fetch_add( + u64::try_from(frame.bytes.len()).unwrap_or(u64::MAX), + Ordering::AcqRel, + ); + if frame.keyframe && self.keyframe_needed.swap(false, Ordering::AcqRel) { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-keyframe", + &format!( + "assembled frame={} bytes={} recovery_pending=false", + frame.frame_index, + frame.bytes.len() + ), + ); + } + } + + pub fn publish_assembled_frame(&self, frame_number: u32, assembled_at: Instant) { + self.frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .record_assembly(frame_number, assembled_at); + } + + fn record_frame_packet(&self, frame_number: u32, received_at: Instant) { + let mut timings = self + .frame_packet_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(timing) = timings + .iter_mut() + .rev() + .find(|entry| entry.frame_number == frame_number) + { + if !timing.assembled { + timing.first_at = timing.first_at.min(received_at); + } + return; + } + if timings.len() == MAX_PENDING_FRAME_ACKS { + timings.pop_front(); + } + timings.push_back(FramePacketTiming { + frame_number, + first_at: received_at, + assembled: false, + }); + } + + fn complete_frame_packets(&self, frame_number: u32) { + let mut timings = self + .frame_packet_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(timing) = timings + .iter_mut() + .rev() + .find(|entry| entry.frame_number == frame_number) + { + timing.assembled = true; + } + } + + pub fn retire_undelivered_frame(&self, frame_number: u32) { + self.frame_packet_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|entry| entry.frame_number != frame_number); + self.frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retire_undelivered(frame_number); + } + + pub fn publish_accepted_frame(&self, frame_number: u32, bytes: u32, accepted_at: Instant) { + let packet_timing = { + let mut timings = self + .frame_packet_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + timings + .iter() + .position(|entry| entry.frame_number == frame_number && entry.assembled) + .and_then(|position| timings.remove(position)) + }; + let assembled_at = self + .frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .record_admission(frame_number, accepted_at); + let mut pending = self + .pending_frame_acks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if pending.len() == MAX_PENDING_FRAME_ACKS { + pending.pop_front(); + } + pending.push_back(CompletedFrameFeedback { + frame_number, + bytes, + accepted_at, + assembled_at, + packet_timing, + }); + } + + pub fn last_assembled_frame_at(&self) -> Option { + self.frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .last_assembled_at() + } + + pub fn frame_stage_timings(&self) -> crate::FrameStageTimings { + self.frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .snapshot() + } + + pub fn reset_frame_stage_epoch(&self) { + self.frame_stage_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reset_epoch(Instant::now()); + self.reception_timing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .bandwidth = BandwidthEstimator::default(); + } + + fn take_completed_frame(&self) -> Option { + self.pending_frame_acks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .pop_front() + } + + fn completed_frame_snapshot(&self) -> (u32, u32) { + ( + self.last_sender_frame.load(Ordering::Acquire), + self.completed_frame_bytes.load(Ordering::Acquire) as u32, + ) + } + + #[cfg(test)] + fn qos_report(&self, previous: &QosReport, elapsed: Duration) -> QosReport { + self.qos_report_at(previous, elapsed, Instant::now()) + } + + fn qos_report_at(&self, previous: &QosReport, elapsed: Duration, now: Instant) -> QosReport { + let (sender_frame_number, bytes_received) = self.completed_frame_snapshot(); + let timing = self + .reception_timing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let packet_snapshot = timing.qos_packet_snapshot; + let bandwidth = timing.bandwidth.report(now); + drop(timing); + let loss_per_ten_thousand = previous + .packet_snapshot + .zip(packet_snapshot) + .filter(|(prior, current)| { + prior.ssrc == current.ssrc + && prior.base == current.base + && current.highest >= prior.highest + && current.received >= prior.received + }) + .map_or(0, |(prior, current)| { + let expected = current.highest - prior.highest; + let received = current.received - prior.received; + if expected == 0 { + return 0; + } + ((u64::from(expected.saturating_sub(received)) * 10_000) / u64::from(expected)) + as u16 + }); + QosReport { + sequence: previous.sequence.wrapping_add(1), + sender_frame_number, + bytes_received, + loss_per_ten_thousand, + client_time_90khz: elapsed.as_millis().wrapping_mul(90) as u32, + packet_snapshot, + bandwidth, + } + } + + /// SSRC + highest sequence for the next Receiver Report, if a stream is bound. + fn stream_snapshot(&self) -> Option<(u32, u32)> { + let ssrc = self.video_ssrc.load(Ordering::Acquire); + (ssrc != 0).then(|| (ssrc, self.highest_sequence.load(Ordering::Acquire))) + } + + /// Measured RTP interarrival jitter (90 kHz video clock) and cumulative + /// packet loss percentage. Reading UI telemetry must not reset RTCP's + /// interval counters or substitute discovery ping for streaming RTT. + pub fn network_metrics(&self) -> Option<(f64, f64)> { + let report = self.report_snapshot(false)?; + let base = self.base_sequence.load(Ordering::Acquire); + let expected = report + .highest_sequence + .saturating_sub(base) + .saturating_add(1); + let loss = f64::from(report.cumulative_lost.max(0)) * 100.0 / f64::from(expected.max(1)); + Some((f64::from(report.jitter) / 90.0, loss.clamp(0.0, 100.0))) + } + + pub fn recent_network_metrics(&self, now: Instant) -> Option<(f64, f64)> { + let mut recent = self + .recent_network_metrics + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let read_counters = || NetworkCounters { + ssrc: self.video_ssrc.load(Ordering::Acquire), + base: self.base_sequence.load(Ordering::Acquire), + highest: self.highest_sequence.load(Ordering::Acquire), + received: self.received_packets.load(Ordering::Acquire), + }; + let counters = read_counters(); + if counters != read_counters() { + return None; + } + let loss = recent.sample(now, counters)?; + let jitter = self + .reception_timing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .jitter + .clamp(0.0, f64::from(u32::MAX)) as u32; + Some((f64::from(jitter) / 90.0, loss)) + } + + fn report_snapshot(&self, update_interval: bool) -> Option { + let media_ssrc = self.video_ssrc.load(Ordering::Acquire); + let base = self.base_sequence.load(Ordering::Acquire); + if media_ssrc == 0 || base == u32::MAX { + return None; + } + let highest_sequence = self.highest_sequence.load(Ordering::Acquire); + let received = self.received_packets.load(Ordering::Acquire); + let expected = highest_sequence.wrapping_sub(base).wrapping_add(1); + let cumulative_lost = i64::from(expected) - i64::from(received); + let cumulative_lost = cumulative_lost.clamp(-0x80_0000, 0x7f_ffff) as i32; + let fraction_lost = if update_interval { + let mut prior = self + .report_prior + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let expected_interval = expected.wrapping_sub(prior.0); + let received_interval = received.wrapping_sub(prior.1); + *prior = (expected, received); + let lost_interval = expected_interval.saturating_sub(received_interval); + if expected_interval == 0 { + 0 + } else { + ((u64::from(lost_interval) << 8) / u64::from(expected_interval)).min(255) as u8 + } + } else { + 0 + }; + let jitter = self + .reception_timing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .jitter + .clamp(0.0, f64::from(u32::MAX)) as u32; + Some(RtcpReportBlock { + media_ssrc, + fraction_lost, + cumulative_lost, + highest_sequence, + jitter, + }) + } + + fn keyframe_request_pending(&self) -> bool { + self.keyframe_needed.load(Ordering::Acquire) + } + + /// Control IDR does not need a video SSRC; RTCP PLI does. Sending either + /// request is not evidence that a keyframe was received, so keep retrying. + fn keyframe_request_routes( + &self, + now: Instant, + last_attempt: Instant, + rtcp_open: bool, + control_open: bool, + ) -> (bool, bool) { + if !self.keyframe_request_pending() + || now.saturating_duration_since(last_attempt) < IDR_RETRY_INTERVAL + { + return (false, false); + } + (rtcp_open && self.stream_snapshot().is_some(), control_open) + } + + fn take_nack(&self, now: Instant, retry_rtt: Option) -> Option<(u64, u64)> { + let (retry_interval, max_attempts) = + retry_rtt.map_or((NACK_RETRY_INTERVAL, MAX_NACK_ATTEMPTS), |rtt| { + let attempts = if rtt.is_zero() { + MAX_NACK_ATTEMPTS + } else { + (NACK_TRACKING_TIMEOUT.as_nanos() / rtt.as_nanos()) + .clamp(1, u128::from(MAX_NACK_ATTEMPTS)) as u8 + }; + (rtt.saturating_add(NACK_RETRY_INTERVAL), attempts) + }); + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending.retain(|_, packet| { + now.saturating_duration_since(packet.requested_at) < NACK_TRACKING_TIMEOUT + }); + let eligible = |packet: &PendingNack| { + packet.attempts < max_attempts + && packet.last_sent_at.is_none_or(|last_sent| { + now.saturating_duration_since(last_sent) >= retry_interval + }) + }; + let first = *pending.iter().find(|(_, packet)| eligible(packet))?.0; + let mut last = first; + for (&index, packet) in pending.range_mut(first..).take(MAX_NACK_PACKET_COUNT) { + if index > last.saturating_add(1) || !eligible(packet) { + break; + } + packet.last_sent_at = Some(now); + packet.attempts = packet.attempts.saturating_add(1); + last = index; + } + Some((first, last)) + } + + fn mark_nack_send_failed(&self, first: u64, last: u64) { + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for (_, packet) in pending.range_mut(first..=last) { + packet.last_sent_at = None; + packet.attempts = packet.attempts.saturating_sub(1); + } + } + + fn clear_nacks(&self) { + self.pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + } +} + +/// Shared handle to the NVST feedback plane (cheap to clone, shared across threads). +pub type SharedNvstFeedback = Arc; + +/// Legacy `nvstVideo` configuration normalized into the bounded receive transport. +/// +/// Secret material is never exposed through `Debug`. The legacy `nvstVideo` handoff defaults to +/// `AEAD_AES_256_GCM`; every SRTP profile requires an explicit `srtpSaltHex`. +#[derive(Clone)] +pub struct NvstVideoConfig { + pub recovery_policy: crate::NvstRecoveryPolicy, + client_udp_port: u16, + video_peer: SocketAddr, + video_peer_port_end: Option, + bundle_peer: Option, + srtp: NvstSrtpMaterial, + ping_payload: Vec, + ping_version: Option, + bundle_natt_remote_username: Option, + stun_credentials: Option, + remote_dtls_fingerprint: Option, + /// The peer assigned RTCP feedback to the `rtcp1` SCTP data channel. When true, the + /// dedicated Mjolnir socket must not send a second raw SRTCP Receiver Report. + rtcp_on_sctp: bool, + qos_timings_v5: bool, + frame_pacing_feedback: bool, + max_control_report_bytes: usize, + hid_device_mask: u32, + /// Dedicated NATT-only video (Mjolnir) socket port in the official two-socket + /// cloud model. When set, video RTP/SRTP arrives on this socket while the + /// ICE/DTLS bundle socket only carries control/audio keepalive traffic. + mjolnir_udp_port: Option, + /// GFN unified-socket mode: Mjolnir video SRTP shares the ICE/DTLS bundle socket + /// instead of a dedicated video socket, so the bundle demuxes raw video RTP. + unified_socket: bool, + codec: NvstVideoCodec, + audio_track: Option, + microphone_available: bool, + microphone_rtp: MicrophoneRtpConfig, + expected_payload_type: Option, + expected_ssrc: Option, + reorder_window_packets: usize, + max_access_unit_bytes: usize, + timeout: Duration, + startup_timeout: Duration, + frame_time_us: u32, + /// Negotiated `x-nv-video[0].packetSize`; FEC shards are this plus 16 RTP bytes. + video_packet_size: usize, + /// Feedback plane shared with the ICE/DTLS bundle (cloned configs share it). + feedback: SharedNvstFeedback, +} + +impl fmt::Debug for NvstVideoConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NvstVideoConfig") + .field("client_udp_port", &self.client_udp_port) + .field("video_peer", &self.video_peer) + .field("video_peer_port_end", &self.video_peer_port_end) + .field("bundle_peer", &self.bundle_peer) + .field("srtp", &self.srtp) + .field("ping_payload_len", &self.ping_payload.len()) + .field("ping_version", &self.ping_version) + .field( + "bundle_natt_remote_username", + &self + .bundle_natt_remote_username + .as_ref() + .map(|_| "[redacted]"), + ) + .field("stun_credentials", &self.stun_credentials) + .field( + "remote_dtls_fingerprint_bytes", + &self.remote_dtls_fingerprint.as_ref().map(String::len), + ) + .field("rtcp_on_sctp", &self.rtcp_on_sctp) + .field("qos_timings_v5", &self.qos_timings_v5) + .field("frame_pacing_feedback", &self.frame_pacing_feedback) + .field("max_control_report_bytes", &self.max_control_report_bytes) + .field("hid_device_mask", &self.hid_device_mask) + .field("mjolnir_udp_port", &self.mjolnir_udp_port) + .field("unified_socket", &self.unified_socket) + .field("codec", &self.codec) + .field("audio_track", &self.audio_track) + .field("expected_payload_type", &self.expected_payload_type) + .field("expected_ssrc", &self.expected_ssrc) + .field("reorder_window_packets", &self.reorder_window_packets) + .field("max_access_unit_bytes", &self.max_access_unit_bytes) + .field("timeout", &self.timeout) + .field("startup_timeout", &self.startup_timeout) + .field("frame_time_us", &self.frame_time_us) + .field("video_packet_size", &self.video_packet_size) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NvstAudioTrack { + pub payload_type: u8, + pub clock_rate_hz: u32, + pub channels: u8, + pub mid: String, + pub ssrc: Option, +} + +#[derive(Clone)] +struct NvstStunCredentials { + local_username_fragment: String, + local_password: String, + remote_username_fragment: String, + remote_password: String, +} + +impl fmt::Debug for NvstStunCredentials { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NvstStunCredentials") + .field("local_username_fragment", &"[redacted]") + .field("local_password", &"[redacted]") + .field("remote_username_fragment", &"[redacted]") + .field("remote_password", &"[redacted]") + .finish() + } +} + +#[derive(Clone)] +enum NvstSrtpMaterial { + AeadAes128Gcm { + master_key: [u8; 16], + master_salt: [u8; 12], + authentication_tag_len: usize, + }, + AeadAes256Gcm { + master_key: [u8; 32], + master_salt: [u8; 12], + authentication_tag_len: usize, + }, + AesCm128HmacSha1 { + master_key: [u8; 16], + master_salt: [u8; 14], + authentication_tag_len: usize, + }, + AesCm256HmacSha1 { + master_key: [u8; 32], + master_salt: [u8; 14], + authentication_tag_len: usize, + }, +} + +impl fmt::Debug for NvstSrtpMaterial { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NvstSrtpMaterial") + .field("profile", &self.profile()) + .field("master_key", &"[redacted]") + .field("master_salt", &"[redacted]") + .finish() + } +} + +impl NvstSrtpMaterial { + fn profile(&self) -> NvstSrtpProfile { + match self { + Self::AeadAes128Gcm { + authentication_tag_len: SRTP_AEAD_AES_GCM_8_TAG_LEN, + .. + } => NvstSrtpProfile::AeadAes128Gcm8, + Self::AeadAes128Gcm { .. } => NvstSrtpProfile::AeadAes128Gcm, + Self::AeadAes256Gcm { + authentication_tag_len: SRTP_AEAD_AES_GCM_8_TAG_LEN, + .. + } => NvstSrtpProfile::AeadAes256Gcm8, + Self::AeadAes256Gcm { .. } => NvstSrtpProfile::AeadAes256Gcm, + Self::AesCm128HmacSha1 { + authentication_tag_len: SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN, + .. + } => NvstSrtpProfile::AesCm128HmacSha1_80, + Self::AesCm128HmacSha1 { .. } => NvstSrtpProfile::AesCm128HmacSha1_32, + Self::AesCm256HmacSha1 { + authentication_tag_len: SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN, + .. + } => NvstSrtpProfile::AesCm256HmacSha1_80, + Self::AesCm256HmacSha1 { .. } => NvstSrtpProfile::AesCm256HmacSha1_32, + } + } +} + +impl NvstVideoConfig { + /// Parses the stable `nvstVideo` handoff supplied by the RTSP probe. + pub fn from_legacy_handoff( + handoff: &Value, + settings_codec: Option<&str>, + ) -> Result { + let object = handoff + .as_object() + .ok_or(NvstConfigError::HandoffNotObject)?; + let client_udp_port = required_u16(object, "clientUdpPort")?; + if client_udp_port == 0 { + return Err(NvstConfigError::OutOfRange { + field: "clientUdpPort", + }); + } + + let peer_ip_text = required_string(object, "videoPeerIp")?; + let peer_ip: IpAddr = peer_ip_text + .parse() + .map_err(|_| NvstConfigError::InvalidPeerIp(peer_ip_text.to_owned()))?; + if !is_unicast_peer(peer_ip) { + return Err(NvstConfigError::InvalidPeerIp(peer_ip_text.to_owned())); + } + let video_peer_port = required_u16(object, "videoPeerPort")?; + if video_peer_port == 0 { + return Err(NvstConfigError::OutOfRange { + field: "videoPeerPort", + }); + } + let video_peer_port_end = optional_u16(object, "videoPeerPortEnd")?; + if video_peer_port_end.is_some_and(|end| { + end < video_peer_port || end - video_peer_port >= MAX_NVST_VIDEO_PEER_PORTS + }) { + return Err(NvstConfigError::OutOfRange { + field: "videoPeerPortEnd", + }); + } + let bundle_peer = + if object.contains_key("bundlePeerIp") || object.contains_key("bundlePeerPort") { + let bundle_ip_text = required_string(object, "bundlePeerIp")?; + let bundle_ip: IpAddr = bundle_ip_text + .parse() + .map_err(|_| NvstConfigError::InvalidPeerIp(bundle_ip_text.to_owned()))?; + if !is_unicast_peer(bundle_ip) { + return Err(NvstConfigError::InvalidPeerIp(bundle_ip_text.to_owned())); + } + let bundle_port = required_u16(object, "bundlePeerPort")?; + if bundle_port == 0 { + return Err(NvstConfigError::OutOfRange { + field: "bundlePeerPort", + }); + } + Some(SocketAddr::new(bundle_ip, bundle_port)) + } else { + None + }; + + let master_key = required_string(object, "srtpAesKeyHex")?; + // Bifrost's SignalingHandler initializes Mjolnir video as + // sec_serv_conf_and_auth + 256-bit keys → AES-256-GCM with an 8-byte tag. + let srtp_profile = optional_string(object, "srtpProfile")? + .map(NvstSrtpProfile::parse) + .transpose()? + .unwrap_or(NvstSrtpProfile::AeadAes256Gcm8); + let srtp = match srtp_profile { + NvstSrtpProfile::AeadAes128Gcm | NvstSrtpProfile::AeadAes128Gcm8 => { + let master_salt = required_string(object, "srtpSaltHex").and_then(|salt| { + decode_fixed_hex::<12>(salt, NvstConfigError::InvalidSrtpSalt) + })?; + NvstSrtpMaterial::AeadAes128Gcm { + master_key: decode_fixed_hex::<16>(master_key, NvstConfigError::InvalidAesKey)?, + master_salt, + authentication_tag_len: match srtp_profile { + NvstSrtpProfile::AeadAes128Gcm8 => SRTP_AEAD_AES_GCM_8_TAG_LEN, + _ => SRTP_AEAD_AES_GCM_TAG_LEN, + }, + } + } + NvstSrtpProfile::AeadAes256Gcm | NvstSrtpProfile::AeadAes256Gcm8 => { + let master_salt = required_string(object, "srtpSaltHex").and_then(|salt| { + decode_fixed_hex::<12>(salt, NvstConfigError::InvalidSrtpSalt) + })?; + NvstSrtpMaterial::AeadAes256Gcm { + master_key: decode_fixed_hex::<32>(master_key, NvstConfigError::InvalidAesKey)?, + master_salt, + authentication_tag_len: match srtp_profile { + NvstSrtpProfile::AeadAes256Gcm8 => SRTP_AEAD_AES_GCM_8_TAG_LEN, + _ => SRTP_AEAD_AES_GCM_TAG_LEN, + }, + } + } + NvstSrtpProfile::AesCm128HmacSha1_32 | NvstSrtpProfile::AesCm128HmacSha1_80 => { + let master_salt = required_string(object, "srtpSaltHex").and_then(|salt| { + decode_salt_hex::<14>(salt, NvstConfigError::InvalidSrtpSalt) + })?; + NvstSrtpMaterial::AesCm128HmacSha1 { + master_key: decode_fixed_hex::<16>(master_key, NvstConfigError::InvalidAesKey)?, + master_salt, + authentication_tag_len: match srtp_profile { + NvstSrtpProfile::AesCm128HmacSha1_32 => 4, + NvstSrtpProfile::AesCm128HmacSha1_80 => SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN, + _ => unreachable!("AES-CM profile selected above"), + }, + } + } + NvstSrtpProfile::AesCm256HmacSha1_32 | NvstSrtpProfile::AesCm256HmacSha1_80 => { + let master_salt = required_string(object, "srtpSaltHex").and_then(|salt| { + decode_salt_hex::<14>(salt, NvstConfigError::InvalidSrtpSalt) + })?; + NvstSrtpMaterial::AesCm256HmacSha1 { + master_key: decode_fixed_hex::<32>(master_key, NvstConfigError::InvalidAesKey)?, + master_salt, + authentication_tag_len: match srtp_profile { + NvstSrtpProfile::AesCm256HmacSha1_32 => 4, + NvstSrtpProfile::AesCm256HmacSha1_80 => SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN, + _ => unreachable!("AES-CM profile selected above"), + }, + } + } + }; + + let codec_name = optional_string(object, "codec")? + .or(settings_codec) + .ok_or(NvstConfigError::MissingField("codec"))?; + let codec = NvstVideoCodec::parse(codec_name)?; + let audio_track = object + .get("audioTrack") + .map(parse_audio_track) + .transpose()?; + let ping_payload = optional_string(object, "pingPayload")? + .map_or_else(|| b"PING".to_vec(), |payload| payload.as_bytes().to_vec()); + if ping_payload.is_empty() || ping_payload.len() > MAX_PING_BYTES { + return Err(NvstConfigError::OutOfRange { + field: "pingPayload", + }); + } + let ping_version = optional_u8(object, "pingVersion")?; + let bundle_natt_remote_username = optional_string(object, "bundleNattRemoteUsername")? + .map(|value| { + if value.is_empty() + || value.len() > MAX_ICE_CREDENTIAL_BYTES + || !value + .bytes() + .all(|byte| byte.is_ascii_graphic() && byte != b':') + || ping_version != Some(6) + { + return Err(NvstConfigError::OutOfRange { + field: "bundleNattRemoteUsername", + }); + } + Ok(value.to_owned()) + }) + .transpose()?; + let remote_dtls_fingerprint = + optional_string(object, "remoteDtlsFingerprint")?.map(str::to_owned); + let rtcp_on_sctp = optional_bool(object, "rtcpOnSctp")?.unwrap_or(false); + let feedback_version = optional_u8(object, "qosFeedbackVersion")?; + let timings_version = optional_u8(object, "qosTimingsVersion")?; + let blob_version = optional_u8(object, "qosBlobStatsVersion")?; + for (field, version, minimum) in [ + ("qosFeedbackVersion", feedback_version, 7), + ("qosTimingsVersion", timings_version, 5), + ("qosBlobStatsVersion", blob_version, 9), + ] { + if version.is_some_and(|version| version < minimum) { + return Err(NvstConfigError::OutOfRange { field }); + } + } + let qos_timings_v5 = timings_version.is_some() && blob_version.is_some(); + let frame_pacing_feedback = match optional_u8(object, "framePacingFeedbackMode")? { + Some(0) => false, + Some(1) => true, + Some(_) => { + return Err(NvstConfigError::OutOfRange { + field: "framePacingFeedbackMode", + }); + } + None => !qos_timings_v5, + }; + let max_control_report_bytes = + optional_usize(object, "maxQosMessagesSize")?.unwrap_or(MAX_CONTROL_REPORT_BYTES); + if max_control_report_bytes < MIN_CONTROL_REPORT_BYTES { + return Err(NvstConfigError::OutOfRange { + field: "maxQosMessagesSize", + }); + } + let max_control_report_bytes = max_control_report_bytes.min(MAX_CONTROL_REPORT_BYTES); + let mjolnir_udp_port = optional_u16(object, "mjolnirUdpPort")?; + if mjolnir_udp_port == Some(0) { + return Err(NvstConfigError::OutOfRange { + field: "mjolnirUdpPort", + }); + } + let unified_socket = optional_bool(object, "unifiedSocket")?.unwrap_or(false); + if unified_socket && mjolnir_udp_port.is_some() { + return Err(NvstConfigError::OutOfRange { + field: "unifiedSocket", + }); + } + let hid_device_mask = optional_u32(object, "hidDeviceMask")?.unwrap_or(0); + let stun_credentials = if ping_version == Some(6) || remote_dtls_fingerprint.is_some() { + Some(NvstStunCredentials { + local_username_fragment: required_ice_credential( + object, + "localIceUsernameFragment", + )?, + local_password: required_ice_credential(object, "localIcePassword")?, + remote_username_fragment: required_ice_credential( + object, + "remoteIceUsernameFragment", + )?, + remote_password: required_ice_credential(object, "remoteIcePassword")?, + }) + } else { + None + }; + + let expected_payload_type = optional_u8(object, "rtpPayloadType")?; + let expected_ssrc = optional_u32(object, "rtpSsrc")?; + let reorder_window_packets = + optional_usize(object, "reorderWindowPackets")?.unwrap_or(DEFAULT_REORDER_WINDOW); + if !(1..=MAX_REORDER_WINDOW).contains(&reorder_window_packets) { + return Err(NvstConfigError::OutOfRange { + field: "reorderWindowPackets", + }); + } + let max_access_unit_bytes = + optional_usize(object, "maxAccessUnitBytes")?.unwrap_or(DEFAULT_MAX_ACCESS_UNIT_BYTES); + if !(1..=MAX_ACCESS_UNIT_BYTES).contains(&max_access_unit_bytes) { + return Err(NvstConfigError::OutOfRange { + field: "maxAccessUnitBytes", + }); + } + let timeout_ms = optional_u64(object, "timeoutMs")?; + let timeout = timeout_ms + .map(Duration::from_millis) + .unwrap_or(DEFAULT_TIMEOUT); + if !(MIN_TIMEOUT..=MAX_TIMEOUT).contains(&timeout) { + return Err(NvstConfigError::OutOfRange { field: "timeoutMs" }); + } + let startup_timeout = optional_u64(object, "startupTimeoutMs")? + .map(Duration::from_millis) + .unwrap_or(timeout); + if !(timeout..=MAX_TIMEOUT).contains(&startup_timeout) { + return Err(NvstConfigError::OutOfRange { + field: "startupTimeoutMs", + }); + } + let video_packet_size = + optional_usize(object, "packetSize")?.unwrap_or(DEFAULT_NVST_VIDEO_PACKET_SIZE); + if !(MIN_NVST_VIDEO_PACKET_SIZE..=MAX_NVST_VIDEO_PACKET_SIZE).contains(&video_packet_size) { + return Err(NvstConfigError::OutOfRange { + field: "packetSize", + }); + } + + let microphone_available = object.get("microphoneOnBundle").and_then(Value::as_bool) + == Some(true) + && remote_dtls_fingerprint.is_some() + && stun_credentials.is_some(); + let microphone_rtp = MicrophoneRtpConfig { + opus_payload_type: optional_u8(object, "microphoneOpusPayloadType")? + .unwrap_or(GFN_OPUS_PAYLOAD_TYPE), + red_payload_type: optional_u8(object, "microphoneRedPayloadType")?, + redundancy_level: optional_u8(object, "microphoneRedundancyLevel")?.unwrap_or(0), + }; + for (field, payload_type) in [ + ( + "microphoneOpusPayloadType", + Some(microphone_rtp.opus_payload_type), + ), + ("microphoneRedPayloadType", microphone_rtp.red_payload_type), + ] { + if payload_type.is_some_and(|pt| pt > 127 || (64..=95).contains(&pt)) { + return Err(NvstConfigError::OutOfRange { field }); + } + } + if microphone_rtp.redundancy_level > 3 { + return Err(NvstConfigError::OutOfRange { + field: "microphoneRedundancyLevel", + }); + } + if microphone_rtp.red_payload_type == Some(microphone_rtp.opus_payload_type) { + return Err(NvstConfigError::OutOfRange { + field: "microphoneRedPayloadType", + }); + } + Ok(Self { + recovery_policy: object + .get("recoveryPolicy") + .map(|value| serde_json::from_value(value.clone())) + .transpose() + .map_err(|_| NvstConfigError::InvalidFieldType { + field: "recoveryPolicy", + expected: "a recovery policy object", + })? + .unwrap_or_default(), + client_udp_port, + video_peer: SocketAddr::new(peer_ip, video_peer_port), + video_peer_port_end, + bundle_peer, + srtp, + ping_payload, + ping_version, + bundle_natt_remote_username, + stun_credentials, + remote_dtls_fingerprint, + rtcp_on_sctp, + qos_timings_v5, + frame_pacing_feedback, + max_control_report_bytes, + mjolnir_udp_port, + unified_socket, + hid_device_mask, + codec, + audio_track, + microphone_available, + microphone_rtp, + expected_payload_type, + expected_ssrc, + reorder_window_packets, + max_access_unit_bytes, + timeout, + startup_timeout, + frame_time_us: DEFAULT_FRAME_TIME_US, + video_packet_size, + feedback: Arc::new(NvstFeedbackState::default()), + }) + } + + pub fn client_udp_port(&self) -> u16 { + self.client_udp_port + } + + /// Shared feedback plane (RTCP-over-SCTP state) for this session. + pub fn feedback(&self) -> SharedNvstFeedback { + self.feedback.clone() + } + + pub fn video_peer(&self) -> SocketAddr { + self.video_peer + } + + fn video_peer_ports(&self) -> std::ops::RangeInclusive { + self.video_peer.port()..=self.video_peer_port_end.unwrap_or(self.video_peer.port()) + } + + fn accepts_video_source(&self, source: SocketAddr) -> bool { + source.ip() == self.video_peer.ip() && self.video_peer_ports().contains(&source.port()) + } + + pub fn bundle_peer(&self) -> SocketAddr { + self.bundle_peer.unwrap_or(self.video_peer) + } + + pub fn codec(&self) -> NvstVideoCodec { + self.codec + } + + pub fn audio_track(&self) -> Option<&NvstAudioTrack> { + self.audio_track.as_ref() + } + + pub fn microphone_available(&self) -> bool { + self.microphone_available + } + + pub fn srtp_profile(&self) -> NvstSrtpProfile { + self.srtp.profile() + } + + pub fn timeout(&self) -> Duration { + self.timeout + } + + pub fn remote_dtls_fingerprint(&self) -> Option<&str> { + self.remote_dtls_fingerprint.as_deref() + } + + pub fn rtcp_on_sctp(&self) -> bool { + self.rtcp_on_sctp + } + + pub fn hid_device_mask(&self) -> u32 { + self.hid_device_mask + } + + pub fn mjolnir_udp_port(&self) -> Option { + self.mjolnir_udp_port + } + + /// Unified-socket video is still Mjolnir video: it recovers through the private + /// NACK-v2 and control IDR dialect, not bundle RTCP Generic NACK/PLI. + fn carries_mjolnir_video(&self) -> bool { + self.mjolnir_udp_port.is_some() || self.unified_socket + } +} + +fn parse_audio_track(value: &Value) -> Result { + let object = value + .as_object() + .ok_or(NvstConfigError::InvalidAudioTrack)?; + let payload_type = object + .get("payloadType") + .and_then(Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .filter(|value| *value <= 127) + .ok_or(NvstConfigError::InvalidAudioTrack)?; + object + .get("codec") + .and_then(Value::as_str) + .filter(|value| value.eq_ignore_ascii_case("opus")) + .ok_or(NvstConfigError::InvalidAudioTrack)?; + let clock_rate_hz = object + .get("clockRateHz") + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| *value == 48_000) + .ok_or(NvstConfigError::InvalidAudioTrack)?; + let channels = object + .get("channels") + .and_then(Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .filter(|value| matches!(*value, 1 | 2)) + .ok_or(NvstConfigError::InvalidAudioTrack)?; + let mid = match object.get("mid") { + Some(Value::String(value)) if !value.is_empty() && value.len() <= 64 => value.clone(), + Some(_) => return Err(NvstConfigError::InvalidAudioTrack), + None => "audio".to_owned(), + }; + let ssrc = match object.get("ssrc") { + Some(value) => Some( + value + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| *value != 0) + .ok_or(NvstConfigError::InvalidAudioTrack)?, + ), + None => None, + }; + Ok(NvstAudioTrack { + payload_type, + clock_rate_hz, + channels, + mid, + ssrc, + }) +} + +fn required_ice_credential( + object: &serde_json::Map, + field: &'static str, +) -> Result { + let value = required_string(object, field)?; + if value.is_empty() || value.len() > MAX_ICE_CREDENTIAL_BYTES { + return Err(NvstConfigError::OutOfRange { field }); + } + Ok(value.to_owned()) +} + +/// Parses `context.nvstVideo` without tying the rest of the transport to JSON field names. +pub fn parse_nvst_video_handoff( + context: &Value, +) -> Result, NvstConfigError> { + let Some(handoff) = context.get("nvstVideo") else { + if context.get("nvstTransport").is_some() { + return Err(NvstConfigError::MissingNvstVideoHandoff); + } + return Ok(None); + }; + let settings_codec = context.pointer("/settings/codec").and_then(Value::as_str); + NvstVideoConfig::from_legacy_handoff(handoff, settings_codec).map(Some) +} + +fn required_string<'a>( + object: &'a serde_json::Map, + field: &'static str, +) -> Result<&'a str, NvstConfigError> { + let value = object + .get(field) + .ok_or(NvstConfigError::MissingField(field))?; + value.as_str().ok_or(NvstConfigError::InvalidFieldType { + field, + expected: "string", + }) +} + +fn optional_string<'a>( + object: &'a serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + match object.get(field) { + None | Some(Value::Null) => Ok(None), + Some(value) => value + .as_str() + .map(Some) + .ok_or(NvstConfigError::InvalidFieldType { + field, + expected: "string", + }), + } +} + +fn required_u16( + object: &serde_json::Map, + field: &'static str, +) -> Result { + let value = required_u64(object, field)?; + u16::try_from(value).map_err(|_| NvstConfigError::OutOfRange { field }) +} + +fn required_u64( + object: &serde_json::Map, + field: &'static str, +) -> Result { + let value = object + .get(field) + .ok_or(NvstConfigError::MissingField(field))?; + value.as_u64().ok_or(NvstConfigError::InvalidFieldType { + field, + expected: "unsigned integer", + }) +} + +fn optional_u64( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + match object.get(field) { + None | Some(Value::Null) => Ok(None), + Some(value) => value + .as_u64() + .map(Some) + .ok_or(NvstConfigError::InvalidFieldType { + field, + expected: "unsigned integer", + }), + } +} + +fn optional_bool( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + match object.get(field) { + None | Some(Value::Null) => Ok(None), + Some(Value::Bool(value)) => Ok(Some(*value)), + Some(_) => Err(NvstConfigError::InvalidFieldType { + field, + expected: "boolean", + }), + } +} + +fn optional_u16( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + optional_u64(object, field)?.map_or(Ok(None), |value| { + u16::try_from(value) + .map(Some) + .map_err(|_| NvstConfigError::OutOfRange { field }) + }) +} + +fn optional_u8( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + optional_u64(object, field)?.map_or(Ok(None), |value| { + u8::try_from(value) + .map(Some) + .map_err(|_| NvstConfigError::OutOfRange { field }) + }) +} + +fn optional_u32( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + optional_u64(object, field)?.map_or(Ok(None), |value| { + u32::try_from(value) + .map(Some) + .map_err(|_| NvstConfigError::OutOfRange { field }) + }) +} + +fn optional_usize( + object: &serde_json::Map, + field: &'static str, +) -> Result, NvstConfigError> { + optional_u64(object, field)?.map_or(Ok(None), |value| { + usize::try_from(value) + .map(Some) + .map_err(|_| NvstConfigError::OutOfRange { field }) + }) +} + +fn decode_fixed_hex( + value: &str, + error: NvstConfigError, +) -> Result<[u8; N], NvstConfigError> { + if value.len() != N * 2 { + return Err(error); + } + let mut decoded = [0_u8; N]; + for (index, output) in decoded.iter_mut().enumerate() { + let offset = index * 2; + let pair = match value.get(offset..offset + 2) { + Some(pair) => pair, + None => return Err(error), + }; + *output = match u8::from_str_radix(pair, 16) { + Ok(byte) => byte, + Err(_) => return Err(error), + }; + } + Ok(decoded) +} + +/// GFN packs the keyId into the low 4 bytes of a 12-byte salt; libsrtp right-pads the +/// AES-CM master salt to 14 bytes. Accept the 12-byte probe form and right-pad with zeros. +fn decode_salt_hex( + value: &str, + error: NvstConfigError, +) -> Result<[u8; N], NvstConfigError> { + if value.len() % 2 != 0 || value.len() > N * 2 { + return Err(error); + } + let mut decoded = [0_u8; N]; + for (index, output) in decoded.iter_mut().take(value.len() / 2).enumerate() { + let offset = index * 2; + let pair = match value.get(offset..offset + 2) { + Some(pair) => pair, + None => return Err(error), + }; + *output = match u8::from_str_radix(pair, 16) { + Ok(byte) => byte, + Err(_) => return Err(error), + }; + } + Ok(decoded) +} + +fn is_unicast_peer(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => !ip.is_unspecified() && !ip.is_multicast() && ip != Ipv4Addr::BROADCAST, + IpAddr::V6(ip) => !ip.is_unspecified() && !ip.is_multicast() && ip != Ipv6Addr::UNSPECIFIED, + } +} + +/// A received encoded video access unit ready for a decoder queue. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedVideoAccessUnit { + pub codec: NvstVideoCodec, + pub timestamp: u32, + pub frame_index: u32, + pub first_stream_packet_index: u32, + /// GameStream frame-header frame type; `None` when the stream carries no frame header. + pub frame_type: Option, + pub keyframe: bool, + pub contiguous: bool, + pub bytes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstReceiverState { + Running, + Paused, + RecoveryRequired, + Stopped(NvstStopCause), +} + +/// Why a receiver worker stopped. Carries only operation names and OS error codes, never +/// endpoints or payloads, so it can reach diagnostics and the reported error message. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstStopCause { + Requested, + ControllerClosed, + UpstreamGateClosed, + MediaConsumerClosed, + InputDeliveryFailed, + HidOutputBlocked, + RandomnessUnavailable, + Socket { + operation: &'static str, + kind: std::io::ErrorKind, + os_code: Option, + }, + WebRtc { + operation: &'static str, + error: &'static str, + }, +} + +impl NvstStopCause { + fn socket(operation: &'static str, error: &std::io::Error) -> Self { + Self::Socket { + operation, + kind: error.kind(), + os_code: error.raw_os_error(), + } + } + + fn webrtc(operation: &'static str, error: &RtcError) -> Self { + let error = match error { + RtcError::RemoteSdp(_) | RtcError::Sdp(_) => "sdp", + RtcError::Rtp(_) | RtcError::Packet(..) => "rtp", + RtcError::Io(_) => "io", + RtcError::Dtls(_) => "dtls", + RtcError::Net(_) => "net", + RtcError::Ice(_) => "ice", + RtcError::Sctp(_) => "sctp", + _ => "other", + }; + Self::WebRtc { operation, error } + } +} + +impl std::fmt::Display for NvstStopCause { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Requested => formatter.write_str("cause=requested"), + Self::ControllerClosed => formatter.write_str("cause=controller-closed"), + Self::UpstreamGateClosed => formatter.write_str("cause=upstream-gate-closed"), + Self::MediaConsumerClosed => formatter.write_str("cause=media-consumer-closed"), + Self::InputDeliveryFailed => formatter.write_str("cause=input-delivery-failed"), + Self::HidOutputBlocked => formatter.write_str("cause=hid-output-blocked"), + Self::RandomnessUnavailable => formatter.write_str("cause=randomness-unavailable"), + Self::Socket { + operation, + kind, + os_code, + } => { + write!( + formatter, + "cause=socket operation={operation} kind={kind:?}" + )?; + match os_code { + Some(code) => write!(formatter, " os_code={code}"), + None => formatter.write_str(" os_code=none"), + } + } + Self::WebRtc { operation, error } => { + write!( + formatter, + "cause=webrtc operation={operation} error={error}" + ) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NvstDropReason { + UnexpectedSource { + expected: SocketAddr, + actual: SocketAddr, + }, + Paused, + Stopped, + RecoveryRequired, + MalformedRtp(RtpParseError), + AuthenticationFailed, + ReplayRejected, + UnexpectedPayloadType { + expected: u8, + actual: u8, + }, + UnexpectedSsrc { + expected: u32, + actual: u32, + }, + StaleRtpPacket { + index: u64, + }, + DuplicateRtpPacket { + index: u64, + }, + Unsupported(NvstUnsupportedFeature), + AwaitingStartOfFrame, + FrameDiscontinuity, + MissingAnnexBStartCode, + InvalidLastPacketPayloadLength { + reported: usize, + frame_header_size: usize, + available: usize, + }, + AccessUnitTooLarge { + limit: usize, + }, + MalformedRedAudio, + MediaConsumerBackpressured, + MediaConsumerClosed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RtpParseError { + TooShort, + InvalidVersion, + InvalidCsrcLength, + InvalidExtensionLength, + MissingAuthenticationTag, + InvalidPadding, + MissingNvVideoHeader, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NvstRecovery { + PacketGap { + first_missing_index: u64, + last_missing_index: u64, + }, + Timeout { + idle_for: Duration, + }, + FrameProgress { + idle_for: Duration, + last_assembled_frame_index: Option, + }, +} + +/// All receive decisions are explicit so callers can collect operational metrics without +/// treating malformed network traffic as a fatal thread error. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NvstReceiveEvent { + Frame(EncodedVideoAccessUnit), + TransportReady(&'static str), + InputReady(u16), + InputUnavailable(&'static str), + UserInput, + TextRejected(&'static str), + SessionTimer(NvstSessionTimer), + ServerTermination(NvstServerTermination), + MicrophoneError(String), + Cursor(Vec), + CursorCapture(bool), + Dropped(NvstDropReason), + RecoveryNeeded(NvstRecovery), + FrameProgressStall { + idle_for: Duration, + last_assembled_frame_index: Option, + recovery_required: bool, + }, + FrameProgressResumed, + Lifecycle(NvstReceiverState), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NvstFrameProgressPolicy { + pub stall: Duration, + pub keyframe_grace: Duration, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum NvstFrameProgressStage { + #[default] + Tracking, + KeyframePending, + RecoveryRequired, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NvstFrameProgress { + pub authenticated_packets: u64, + pub last_authenticated_packet: Option, + pub assembled_frames_total: u64, + pub last_assembled_frame_index: Option, + pub last_assembled_at: Option, + pub stage: NvstFrameProgressStage, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstFrameProgressEvent { + KeyframeRequested { + idle_for: Duration, + last_assembled_frame_index: Option, + }, + RecoveryNeeded { + idle_for: Duration, + last_assembled_frame_index: Option, + }, +} + +#[derive(Debug, Clone, Copy, Default)] +struct FrameProgressWatchdog { + first_authenticated_at: Option, + first_keyframe_assembled: bool, + last_assembled_frame_index: Option, + stage: NvstFrameProgressStage, + keyframe_requested_at: Option, +} + +impl FrameProgressWatchdog { + fn authenticated(&mut self, now: Instant) { + self.first_authenticated_at.get_or_insert(now); + } + + fn assembled(&mut self, frame_index: u32, keyframe: bool) -> bool { + self.last_assembled_frame_index = Some(frame_index); + if !self.first_keyframe_assembled && !keyframe { + return false; + } + self.first_keyframe_assembled = true; + let closed_episode = self.stage != NvstFrameProgressStage::Tracking; + self.stage = NvstFrameProgressStage::Tracking; + self.keyframe_requested_at = None; + closed_episode + } + + fn reset(&mut self) { + *self = Self::default(); + } +} + +#[derive(Debug, Clone, Copy)] +struct RtpHeader { + payload_type: u8, + sequence_number: u16, + timestamp: u32, + ssrc: u32, + payload_offset: usize, + has_padding: bool, + gs_video_header: Option<[u8; NV_VIDEO_PACKET_LEN]>, +} + +impl RtpHeader { + fn parse(packet: &[u8]) -> Result { + if packet.len() < RTP_FIXED_HEADER_LEN { + return Err(RtpParseError::TooShort); + } + let first = packet[0]; + if first >> 6 != 2 { + return Err(RtpParseError::InvalidVersion); + } + let csrc_count = usize::from(first & 0x0f); + let mut payload_offset = RTP_FIXED_HEADER_LEN + .checked_add(csrc_count.saturating_mul(4)) + .ok_or(RtpParseError::InvalidCsrcLength)?; + if packet.len() < payload_offset { + return Err(RtpParseError::InvalidCsrcLength); + } + let mut gs_video_header = None; + if first & 0x10 != 0 { + if packet.len() < payload_offset + 4 { + return Err(RtpParseError::InvalidExtensionLength); + } + let profile = u16::from_be_bytes([packet[payload_offset], packet[payload_offset + 1]]); + let words = + u16::from_be_bytes([packet[payload_offset + 2], packet[payload_offset + 3]]); + let extension_len = usize::from(words) + .checked_mul(4) + .and_then(|length| length.checked_add(4)) + .ok_or(RtpParseError::InvalidExtensionLength)?; + let extension_end = payload_offset + .checked_add(extension_len) + .ok_or(RtpParseError::InvalidExtensionLength)?; + if packet.len() < extension_end { + return Err(RtpParseError::InvalidExtensionLength); + } + if profile == GS_VIDEO_EXTENSION_PROFILE { + if usize::from(words) * 4 != NV_VIDEO_PACKET_LEN { + return Err(RtpParseError::InvalidExtensionLength); + } + let body = &packet[payload_offset + 4..extension_end]; + gs_video_header = Some( + body[..NV_VIDEO_PACKET_LEN] + .try_into() + .expect("length checked"), + ); + } + payload_offset = extension_end; + } + Ok(Self { + payload_type: packet[1] & 0x7f, + sequence_number: u16::from_be_bytes([packet[2], packet[3]]), + timestamp: u32::from_be_bytes([packet[4], packet[5], packet[6], packet[7]]), + ssrc: u32::from_be_bytes([packet[8], packet[9], packet[10], packet[11]]), + payload_offset, + has_padding: first & 0x20 != 0, + gs_video_header, + }) + } + + fn payload<'a>(&self, packet: &'a [u8]) -> Result<&'a [u8], RtpParseError> { + if packet.len() < self.payload_offset { + return Err(RtpParseError::TooShort); + } + let mut end = packet.len(); + if self.has_padding { + let padding = usize::from(*packet.last().ok_or(RtpParseError::InvalidPadding)?); + if padding == 0 || padding > end.saturating_sub(self.payload_offset) { + return Err(RtpParseError::InvalidPadding); + } + end -= padding; + } + Ok(&packet[self.payload_offset..end]) + } +} + +#[derive(Debug, Clone)] +struct RtpPacket { + index: u64, + header: RtpHeader, + plaintext: Vec, + origin: RtpPacketOrigin, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RtpPacketOrigin { + Authenticated, + RecoveredData, +} + +#[derive(Clone)] +struct SrtpReceiver { + cipher: SrtpCipher, + replay: ReplayWindow, +} + +#[derive(Clone)] +enum SrtpCipher { + AeadAes128Gcm { + encryption_key: [u8; 16], + session_salt: [u8; 12], + authentication_tag_len: usize, + }, + AeadAes256Gcm { + encryption_key: [u8; 32], + session_salt: [u8; 12], + authentication_tag_len: usize, + }, + AesCm128HmacSha1 { + encryption_key: [u8; 16], + authentication_key: [u8; 20], + session_salt: [u8; 14], + authentication_tag_len: usize, + }, + AesCm256HmacSha1 { + encryption_key: [u8; 32], + authentication_key: [u8; 20], + session_salt: [u8; 14], + authentication_tag_len: usize, + }, +} + +impl SrtpReceiver { + fn from_material(material: &NvstSrtpMaterial) -> Self { + let cipher = match material { + NvstSrtpMaterial::AeadAes128Gcm { + master_key, + master_salt, + authentication_tag_len, + } => SrtpCipher::AeadAes128Gcm { + encryption_key: derive_aes128_cm_key::<16>( + master_key, + master_salt, + GFN_SRTP_KEY_LABEL, + ), + session_salt: derive_aes128_cm_key::<12>( + master_key, + master_salt, + GFN_SRTP_SALT_LABEL, + ), + authentication_tag_len: *authentication_tag_len, + }, + NvstSrtpMaterial::AeadAes256Gcm { + master_key, + master_salt, + authentication_tag_len, + } => SrtpCipher::AeadAes256Gcm { + encryption_key: derive_aes_cm_key::<32>( + master_key, + master_salt, + GFN_SRTP_KEY_LABEL, + ), + session_salt: derive_aes_cm_key::<12>(master_key, master_salt, GFN_SRTP_SALT_LABEL), + authentication_tag_len: *authentication_tag_len, + }, + NvstSrtpMaterial::AesCm128HmacSha1 { + master_key, + master_salt, + authentication_tag_len, + } => SrtpCipher::AesCm128HmacSha1 { + encryption_key: derive_aes128_cm_key::<16>(master_key, master_salt, 0x00), + authentication_key: derive_aes128_cm_key::<20>(master_key, master_salt, 0x01), + session_salt: derive_aes128_cm_key::<14>(master_key, master_salt, 0x02), + authentication_tag_len: *authentication_tag_len, + }, + NvstSrtpMaterial::AesCm256HmacSha1 { + master_key, + master_salt, + authentication_tag_len, + } => SrtpCipher::AesCm256HmacSha1 { + encryption_key: derive_aes_cm_key::<32>(master_key, master_salt, 0x00), + authentication_key: derive_aes_cm_key::<20>(master_key, master_salt, 0x01), + session_salt: derive_aes_cm_key::<14>(master_key, master_salt, 0x02), + authentication_tag_len: *authentication_tag_len, + }, + }; + Self { + cipher, + replay: ReplayWindow::default(), + } + } + + fn unprotect(&mut self, datagram: &[u8]) -> Result { + let header = RtpHeader::parse(datagram).map_err(NvstDropReason::MalformedRtp)?; + let packet_index = self.replay.guess_packet_index(header.sequence_number)?; + let roc = u32::try_from(packet_index >> 16).map_err(|_| NvstDropReason::ReplayRejected)?; + let plaintext = match &self.cipher { + SrtpCipher::AeadAes128Gcm { + encryption_key, + session_salt, + authentication_tag_len, + } => unprotect_aes_gcm( + datagram, + header, + roc, + encryption_key, + session_salt, + *authentication_tag_len, + )?, + SrtpCipher::AeadAes256Gcm { + encryption_key, + session_salt, + authentication_tag_len, + } => unprotect_aes_gcm( + datagram, + header, + roc, + encryption_key, + session_salt, + *authentication_tag_len, + )?, + SrtpCipher::AesCm128HmacSha1 { + encryption_key, + authentication_key, + session_salt, + authentication_tag_len, + } => unprotect_aes_cm_hmac_sha1( + datagram, + header, + packet_index, + roc, + encryption_key, + authentication_key, + session_salt, + *authentication_tag_len, + )?, + SrtpCipher::AesCm256HmacSha1 { + encryption_key, + authentication_key, + session_salt, + authentication_tag_len, + } => unprotect_aes_cm_hmac_sha1( + datagram, + header, + packet_index, + roc, + encryption_key, + authentication_key, + session_salt, + *authentication_tag_len, + )?, + }; + self.replay.check(packet_index)?; + header + .payload(&plaintext) + .map_err(NvstDropReason::MalformedRtp)?; + self.replay.commit(packet_index); + Ok(RtpPacket { + index: packet_index, + header, + plaintext, + origin: RtpPacketOrigin::Authenticated, + }) + } +} + +/// AES-GCM with a truncated tag. `aes-gcm` 0.10 only seals 12–16 byte tags; +/// NVIDIA's Mjolnir path uses the 8-byte `aes_gcm_*_8_auth` libsrtp policy. +fn unprotect_aes_gcm( + datagram: &[u8], + header: RtpHeader, + roc: u32, + encryption_key: &[u8], + session_salt: &[u8; 12], + tag_len: usize, +) -> Result, NvstDropReason> { + let ciphertext_end = + datagram + .len() + .checked_sub(tag_len) + .ok_or(NvstDropReason::MalformedRtp( + RtpParseError::MissingAuthenticationTag, + ))?; + if ciphertext_end < header.payload_offset { + return Err(NvstDropReason::MalformedRtp( + RtpParseError::MissingAuthenticationTag, + )); + } + let iv = srtp_gcm_iv(*session_salt, header.ssrc, roc, header.sequence_number); + let aad = &datagram[..header.payload_offset]; + let ciphertext = &datagram[header.payload_offset..ciphertext_end]; + let received_tag = &datagram[ciphertext_end..]; + let (expected_tag, mut ctr) = aes_gcm_tag_and_ctr(encryption_key, &iv, aad, ciphertext); + if expected_tag[..tag_len].ct_eq(received_tag).unwrap_u8() != 1 { + return Err(NvstDropReason::AuthenticationFailed); + } + let mut plaintext = datagram[..ciphertext_end].to_vec(); + ctr.apply_keystream(&mut plaintext[header.payload_offset..]); + Ok(plaintext) +} + +enum GcmCtr { + Aes128(Box>), + Aes256(Box>), +} + +impl GcmCtr { + fn apply_keystream(&mut self, buffer: &mut [u8]) { + match self { + Self::Aes128(cipher) => cipher.apply_keystream(buffer), + Self::Aes256(cipher) => cipher.apply_keystream(buffer), + } + } +} + +fn aes_gcm_tag_and_ctr( + encryption_key: &[u8], + iv: &[u8; 12], + aad: &[u8], + ciphertext: &[u8], +) -> ([u8; 16], GcmCtr) { + let mut j0 = [0_u8; 16]; + j0[..12].copy_from_slice(iv); + j0[15] = 1; + let mut hash_key = [0_u8; 16]; + let mut tag_mask = [0_u8; 16]; + let ctr = match encryption_key.len() { + 16 => { + let key: &[u8; 16] = encryption_key.try_into().expect("16-byte GCM key"); + let aes = ::new(key.into()); + aes.encrypt_block((&mut hash_key).into()); + let mut ctr = Ctr32BE::::new(key.into(), (&j0).into()); + ctr.apply_keystream(&mut tag_mask); + GcmCtr::Aes128(Box::new(ctr)) + } + 32 => { + let key: &[u8; 32] = encryption_key.try_into().expect("32-byte GCM key"); + let aes = ::new(key.into()); + aes.encrypt_block((&mut hash_key).into()); + let mut ctr = Ctr32BE::::new(key.into(), (&j0).into()); + ctr.apply_keystream(&mut tag_mask); + GcmCtr::Aes256(Box::new(ctr)) + } + _ => unreachable!("AES-GCM key is 16 or 32 bytes"), + }; + let mut hasher = ::new((&hash_key).into()); + hasher.update_padded(aad); + hasher.update_padded(ciphertext); + let mut len_block = ghash::Block::default(); + len_block[..8].copy_from_slice(&((aad.len() as u64) * 8).to_be_bytes()); + len_block[8..].copy_from_slice(&((ciphertext.len() as u64) * 8).to_be_bytes()); + hasher.update(&[len_block]); + let mut tag = hasher.finalize(); + for (byte, mask) in tag.iter_mut().zip(tag_mask) { + *byte ^= mask; + } + let mut expected = [0_u8; 16]; + expected.copy_from_slice(&tag); + (expected, ctr) +} + +#[cfg(test)] +fn protect_aes_gcm( + packet: &mut Vec, + payload_offset: usize, + encryption_key: &[u8], + iv: &[u8; 12], + tag_len: usize, +) { + let aad_owned = packet[..payload_offset].to_vec(); + let (_, mut ctr) = aes_gcm_tag_and_ctr(encryption_key, iv, &aad_owned, &[]); + ctr.apply_keystream(&mut packet[payload_offset..]); + let (tag, _) = aes_gcm_tag_and_ctr(encryption_key, iv, &aad_owned, &packet[payload_offset..]); + packet.extend_from_slice(&tag[..tag_len]); +} + +#[allow(clippy::too_many_arguments)] +fn unprotect_aes_cm_hmac_sha1( + datagram: &[u8], + header: RtpHeader, + packet_index: u64, + roc: u32, + encryption_key: &[u8], + authentication_key: &[u8; 20], + session_salt: &[u8; 14], + authentication_tag_len: usize, +) -> Result, NvstDropReason> { + let authenticated_len = + datagram + .len() + .checked_sub(authentication_tag_len) + .ok_or(NvstDropReason::MalformedRtp( + RtpParseError::MissingAuthenticationTag, + ))?; + if authenticated_len < header.payload_offset { + return Err(NvstDropReason::MalformedRtp( + RtpParseError::MissingAuthenticationTag, + )); + } + let mut mac = + HmacSha1::new_from_slice(authentication_key).expect("HMAC-SHA1 accepts a fixed-size key"); + mac.update(&datagram[..authenticated_len]); + mac.update(&roc.to_be_bytes()); + let expected_tag = mac.finalize().into_bytes(); + let received_tag = &datagram[authenticated_len..]; + if expected_tag[..authentication_tag_len] + .ct_eq(received_tag) + .unwrap_u8() + != 1 + { + return Err(NvstDropReason::AuthenticationFailed); + } + let mut plaintext = datagram[..authenticated_len].to_vec(); + let iv = srtp_aes_cm_iv(session_salt, header.ssrc, packet_index); + match encryption_key.len() { + 16 => { + let key: &[u8; 16] = encryption_key.try_into().expect("16-byte AES-CM key"); + let mut cipher = Aes128Ctr::new(key.into(), (&iv).into()); + cipher.apply_keystream(&mut plaintext[header.payload_offset..]); + } + 32 => { + let key: &[u8; 32] = encryption_key.try_into().expect("32-byte AES-CM key"); + let mut cipher = Aes256Ctr::new(key.into(), (&iv).into()); + cipher.apply_keystream(&mut plaintext[header.payload_offset..]); + } + _ => unreachable!("AES-CM key is 16 or 32 bytes"), + } + Ok(plaintext) +} + +fn derive_aes_cm_key( + master_key: &[u8; 32], + master_salt: &[u8], + label: u8, +) -> [u8; N] { + let mut iv = [0_u8; 16]; + iv[..master_salt.len()].copy_from_slice(master_salt); + // RFC 3711 key_id = label * 2^48, then x = master_salt * 2^16 XOR key_id * 2^16. + iv[7] ^= label; + let mut output = [0_u8; N]; + let mut cipher = Aes256Ctr::new(master_key.into(), (&iv).into()); + cipher.apply_keystream(&mut output); + output +} + +fn derive_aes128_cm_key( + master_key: &[u8; 16], + master_salt: &[u8], + label: u8, +) -> [u8; N] { + let mut iv = [0_u8; 16]; + iv[..master_salt.len()].copy_from_slice(master_salt); + iv[7] ^= label; + let mut output = [0_u8; N]; + let mut cipher = Aes128Ctr::new(master_key.into(), (&iv).into()); + cipher.apply_keystream(&mut output); + output +} + +fn srtp_aes_cm_iv(session_salt: &[u8; 14], ssrc: u32, packet_index: u64) -> [u8; 16] { + let mut iv = [0_u8; 16]; + iv[..14].copy_from_slice(session_salt); + for (target, source) in iv[4..8].iter_mut().zip(ssrc.to_be_bytes()) { + *target ^= source; + } + let index_bytes = packet_index.to_be_bytes(); + for (target, source) in iv[8..14].iter_mut().zip(&index_bytes[2..]) { + *target ^= source; + } + iv +} + +fn srtp_gcm_iv(session_salt: [u8; 12], ssrc: u32, roc: u32, sequence_number: u16) -> [u8; 12] { + let mut iv = session_salt; + for (target, source) in iv[2..6].iter_mut().zip(ssrc.to_be_bytes()) { + *target ^= source; + } + for (target, source) in iv[6..10].iter_mut().zip(roc.to_be_bytes()) { + *target ^= source; + } + for (target, source) in iv[10..].iter_mut().zip(sequence_number.to_be_bytes()) { + *target ^= source; + } + iv +} + +/// RFC 7714 §9.2 SRTCP GCM IV: salt XOR (SSRC at bytes 2..6, SRTCP index at 6..10). +fn srtcp_gcm_iv(session_salt: [u8; 12], ssrc: u32, srtcp_index: u32) -> [u8; 12] { + let mut iv = session_salt; + for (target, source) in iv[2..6].iter_mut().zip(ssrc.to_be_bytes()) { + *target ^= source; + } + for (target, source) in iv[6..10].iter_mut().zip(srtcp_index.to_be_bytes()) { + *target ^= source; + } + iv +} + +/// Builds and SRTCP-protects an RTCP Receiver Report carrying one report block. +/// The raw Mjolnir transport keeps the fixed eight-byte ONOW receiver header clear, +/// sets the report payload E/S bit before encryption, and appends the eight-byte GCM +/// tag before E|index. AAD is the clear header followed by the first eight ciphertext +/// bytes; the sender SSRC in that clear header selects the nonce context. +#[allow(clippy::too_many_arguments)] +fn protect_srtcp_receiver_report_gcm( + report: RtcpReportBlock, + srtcp_index: u32, + encryption_key: &[u8], + session_salt: &[u8; 12], + tag_len: usize, +) -> Vec { + let mut packet = Vec::with_capacity(32 + 4 + tag_len); + packet.push(0x81); // V=2, P=0, RC=1 (one report block) + packet.push(201); // PT=RR + packet.extend_from_slice(&7_u16.to_be_bytes()); // length: 8 words - 1 + packet.extend_from_slice(&RTCP_SENDER_SSRC.to_be_bytes()); + packet.extend_from_slice(&report.media_ssrc.to_be_bytes()); + packet.push(report.fraction_lost); + packet.extend_from_slice(&(report.cumulative_lost as u32).to_be_bytes()[1..]); + packet.extend_from_slice(&report.highest_sequence.to_be_bytes()); + packet.extend_from_slice(&report.jitter.to_be_bytes()); + packet.extend_from_slice(&0_u32.to_be_bytes()); // LSR + packet.extend_from_slice(&0_u32.to_be_bytes()); // DLSR + + let e_index = SRTCP_ENCRYPTED_FLAG | (srtcp_index & !SRTCP_ENCRYPTED_FLAG); + let iv = srtcp_gcm_iv(*session_salt, RTCP_SENDER_SSRC, srtcp_index); + packet[8] |= 0x80; + let (_, mut ctr) = aes_gcm_tag_and_ctr(encryption_key, &iv, &[], &[]); + ctr.apply_keystream(&mut packet[8..]); + let mut aad = packet[..8].to_vec(); + aad.extend_from_slice(&packet[8..16]); + let (tag, _) = aes_gcm_tag_and_ctr(encryption_key, &iv, &aad, &packet[8..]); + packet.extend_from_slice(&tag[..tag_len]); + packet.extend_from_slice(&e_index.to_be_bytes()); + packet +} + +/// Builds a plain (unencrypted) RTCP Receiver Report (RFC 3550 §6.4.1) with one +/// report block for `media_ssrc`. Sent over the `rtcp1` SCTP data channel, which +/// is already encrypted by DTLS, so no SRTCP layer is applied. +fn build_rtcp_receiver_report(sender_ssrc: u32, report: RtcpReportBlock) -> Vec { + let mut packet = Vec::with_capacity(32); + packet.push(0x81); // V=2, P=0, RC=1 (one report block) + packet.push(201); // PT=RR + packet.extend_from_slice(&7_u16.to_be_bytes()); // length: 8 words - 1 + packet.extend_from_slice(&sender_ssrc.to_be_bytes()); + packet.extend_from_slice(&report.media_ssrc.to_be_bytes()); + packet.push(report.fraction_lost); + packet.extend_from_slice(&(report.cumulative_lost as u32).to_be_bytes()[1..]); + packet.extend_from_slice(&report.highest_sequence.to_be_bytes()); + packet.extend_from_slice(&report.jitter.to_be_bytes()); + packet.extend_from_slice(&0_u32.to_be_bytes()); // LSR + packet.extend_from_slice(&0_u32.to_be_bytes()); // DLSR + packet +} + +/// Builds a plain RTCP Picture Loss Indication (RFC 4585 §6.3.1) asking the +/// sender of `media_ssrc` for a fresh keyframe. +fn build_rtcp_pli(sender_ssrc: u32, media_ssrc: u32) -> Vec { + let mut packet = Vec::with_capacity(12); + packet.push(0x81); // V=2, P=0, FMT=1 (PLI) + packet.push(206); // PT=PSFB (payload-specific feedback) + packet.extend_from_slice(&2_u16.to_be_bytes()); // length: 3 words - 1 + packet.extend_from_slice(&sender_ssrc.to_be_bytes()); + packet.extend_from_slice(&media_ssrc.to_be_bytes()); + packet +} + +/// Builds an RFC 4585 §6.2.1 generic NACK for a continuous range of missing +/// extended RTP sequence numbers. Each FCI entry represents its PID plus up to +/// 16 following packets in the bitmask; the packet is bounded to avoid turning +/// a malicious sequence jump into an unbounded control message. +fn build_rtcp_nack( + sender_ssrc: u32, + media_ssrc: u32, + first_missing_index: u64, + last_missing_index: u64, +) -> Vec { + let mut entries = Vec::with_capacity(MAX_NACK_FCI_ENTRIES); + let mut current = first_missing_index; + let effective_last = last_missing_index + .min(first_missing_index.saturating_add((MAX_NACK_PACKET_COUNT.saturating_sub(1)) as u64)); + while current <= effective_last && entries.len() < MAX_NACK_FCI_ENTRIES { + let pid = current as u16; + let represented_following = (effective_last - current).min(16) as u16; + let blp = if represented_following == 16 { + u16::MAX + } else { + ((1_u32 << represented_following) - 1) as u16 + }; + entries.push((pid, blp)); + current = current.saturating_add(17); + } + + let mut packet = Vec::with_capacity(12 + entries.len() * 4); + packet.push(0x81); // V=2, P=0, FMT=1 (generic NACK) + packet.push(205); // PT=RTPFB (transport-layer feedback) + packet.extend_from_slice(&(2_u16 + entries.len() as u16).to_be_bytes()); + packet.extend_from_slice(&sender_ssrc.to_be_bytes()); + packet.extend_from_slice(&media_ssrc.to_be_bytes()); + for (pid, blp) in entries { + packet.extend_from_slice(&pid.to_be_bytes()); + packet.extend_from_slice(&blp.to_be_bytes()); + } + packet +} + +fn send_pending_nack( + feedback: &NvstFeedbackState, + now: Instant, + rtc: &mut Rtc, + channels: NvstInputChannels, + mjolnir: bool, + sender_ssrc: u32, + media_ssrc: u32, +) { + let Some(channel) = (if mjolnir { + Some(channels.control_partial) + } else { + channels.rtcp + }) else { + return; + }; + if rtc.channel(channel).is_none() { + return; + } + let retry_rtt = mjolnir.then(|| feedback.ping(now).unwrap_or(DEFAULT_NACK_RTT)); + let Some((first, last)) = feedback.take_nack(now, retry_rtt) else { + return; + }; + let admitted = if mjolnir { + let missing: Vec = (first..=last).map(|index| index as u16).collect(); + nack_v2(0, &missing) + .is_some_and(|command| channels.send_partial_control(rtc, &command.encoded())) + } else { + channels.send_rtcp(rtc, &build_rtcp_nack(sender_ssrc, media_ssrc, first, last)) + }; + if admitted { + let format = if mjolnir { + "private-v2" + } else { + "rtcp-generic" + }; + eprintln!( + "NVST NACK sent format={format} channel={} mediaSsrc={media_ssrc} missing={first}..={last}", + channels.label(channel), + ); + } else { + feedback.mark_nack_send_failed(first, last); + } +} + +fn send_pending_frame_invalidation( + feedback: &NvstFeedbackState, + now: Instant, + rtc: &mut Rtc, + channels: NvstInputChannels, +) { + let Some((first, last)) = feedback.frame_invalidation.take() else { + return; + }; + if !channels.send_control(rtc, &frame_invalidation_range(first, last, 0).encoded()) { + feedback.frame_invalidation.restore((first, last)); + return; + } + feedback.frame_invalidation.record_sent(now); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-rfi", + &format!("invalidation sent frames={first}..={last}"), + ); +} + +fn uses_rtcp_video_feedback(mjolnir: bool, rtcp_channel_open: bool) -> bool { + !mjolnir && rtcp_channel_open +} + +#[derive(Clone)] +struct ReplayWindow { + highest_index: Option, + // Bit N records `highest_index - N`. Words are little-endian by packet age. + seen: [u64; SRTP_REPLAY_WINDOW_WORDS], +} + +impl Default for ReplayWindow { + fn default() -> Self { + Self { + highest_index: None, + seen: [0; SRTP_REPLAY_WINDOW_WORDS], + } + } +} + +impl ReplayWindow { + fn guess_packet_index(&self, sequence_number: u16) -> Result { + let Some(highest_index) = self.highest_index else { + return Ok(u64::from(sequence_number)); + }; + let roc = highest_index >> 16; + let highest_sequence = (highest_index & 0xffff) as u16; + let delta = i32::from(sequence_number) - i32::from(highest_sequence); + let guessed_roc = if delta < -32_768 { + roc.checked_add(1).ok_or(NvstDropReason::ReplayRejected)? + } else if delta > 32_768 { + roc.checked_sub(1).ok_or(NvstDropReason::ReplayRejected)? + } else { + roc + }; + Ok((guessed_roc << 16) | u64::from(sequence_number)) + } + + fn check(&self, index: u64) -> Result<(), NvstDropReason> { + let Some(highest_index) = self.highest_index else { + return Ok(()); + }; + if index > highest_index { + return Ok(()); + } + let age = highest_index - index; + let Ok(age) = usize::try_from(age) else { + return Err(NvstDropReason::ReplayRejected); + }; + if age >= SRTP_REPLAY_WINDOW_PACKETS + || self.seen[age / u64::BITS as usize] & (1_u64 << (age % u64::BITS as usize)) != 0 + { + return Err(NvstDropReason::ReplayRejected); + } + Ok(()) + } + + fn commit(&mut self, index: u64) { + match self.highest_index { + None => { + self.highest_index = Some(index); + self.seen.fill(0); + self.seen[0] = 1; + } + Some(highest_index) if index > highest_index => { + let advance = usize::try_from(index - highest_index).unwrap_or(usize::MAX); + if advance >= SRTP_REPLAY_WINDOW_PACKETS { + self.seen.fill(0); + } else { + let word_shift = advance / u64::BITS as usize; + let bit_shift = advance % u64::BITS as usize; + for destination in (0..SRTP_REPLAY_WINDOW_WORDS).rev() { + self.seen[destination] = if destination < word_shift { + 0 + } else { + let source = destination - word_shift; + let mut shifted = self.seen[source] << bit_shift; + if bit_shift != 0 && source > 0 { + shifted |= + self.seen[source - 1] >> (u64::BITS as usize - bit_shift); + } + shifted + }; + } + } + self.seen[0] |= 1; + self.highest_index = Some(index); + } + Some(highest_index) => { + let age = usize::try_from(highest_index - index) + .expect("replay age was validated before commit"); + self.seen[age / u64::BITS as usize] |= 1_u64 << (age % u64::BITS as usize); + } + } + } +} + +#[derive(Debug, Clone, Copy)] +struct NvVideoPacket { + stream_packet_index: u32, + frame_index: u32, + flags: u8, + fec_current_block: u8, + fec_last_block: u8, + is_fec: bool, +} + +impl NvVideoPacket { + /// Reads the Mjolnir video metadata from the `0x4753` ("GS") RTP extension: + /// a 16-byte little-endian block holding the stream packet index, frame index, + /// the packet-type nibble (picture data / SOF / EOF), and FEC group + /// coordinates. The RTP payload starts with the GameStream frame header on an + /// SOF packet and otherwise continues the H.264 access-unit data directly. + fn parse<'a>(header: &RtpHeader, payload: &'a [u8]) -> Result<(Self, &'a [u8]), RtpParseError> { + let Some(extension) = header.gs_video_header else { + return Err(RtpParseError::MissingNvVideoHeader); + }; + let packet_word = u32::from_le_bytes(extension[0..4].try_into().expect("length checked")); + let flags_word = u32::from_le_bytes(extension[8..12].try_into().expect("length checked")); + let fec_word = u32::from_le_bytes(extension[12..16].try_into().expect("length checked")); + let multi_fec_blocks = extension[11]; + let fec_percentage = (fec_word >> 4) & 0xff; + let fec_index = (fec_word >> 12) & 0x3ff; + let fec_source_packets = (fec_word >> 22) & 0x3ff; + let packet = Self { + stream_packet_index: (packet_word >> 8) & STREAM_PACKET_INDEX_MASK, + frame_index: u32::from_le_bytes(extension[4..8].try_into().expect("length checked")), + flags: (flags_word & 0x0f) as u8, + fec_current_block: (multi_fec_blocks >> 4) & 0x03, + fec_last_block: (multi_fec_blocks >> 6) & 0x03, + is_fec: fec_percentage != 0 && fec_index >= fec_source_packets, + }; + Ok((packet, payload)) + } + + fn is_start_of_frame(self) -> bool { + self.flags & FLAG_SOF != 0 && self.fec_current_block == 0 + } + + fn is_end_of_frame(self) -> bool { + self.flags & FLAG_EOF != 0 && self.fec_current_block == self.fec_last_block + } +} + +struct VideoAccessUnitAssembler { + codec: NvstVideoCodec, + current_frame: Option, + first_stream_packet_index: Option, + frame_type: Option, + last_packet_payload_length: Option, + expected_access_unit_length: Option, + invalid_av1_length_logged: bool, + first_start_payload_logged: bool, + first_access_unit_logged: bool, + bytes: Vec, + max_access_unit_bytes: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct GsFrameHeader { + frame_type: NvstFrameType, +} + +impl GsFrameHeader { + fn parse(payload: &[u8], header_size: usize) -> Option { + let frame_type = match payload { + [0x47, 0x53, _, value, ..] if header_size >= 4 => { + NvstFrameType::from_wire(value & 0x0f) + } + [0x01 | 0x81, _, _, value, ..] if header_size > 0 => NvstFrameType::from_wire(*value), + _ => return None, + }; + Some(Self { frame_type }) + } +} + +impl VideoAccessUnitAssembler { + fn new(codec: NvstVideoCodec, max_access_unit_bytes: usize) -> Self { + Self { + codec, + current_frame: None, + first_stream_packet_index: None, + frame_type: None, + last_packet_payload_length: None, + expected_access_unit_length: None, + invalid_av1_length_logged: false, + first_start_payload_logged: false, + first_access_unit_logged: false, + bytes: Vec::new(), + max_access_unit_bytes, + } + } + + fn reset(&mut self) { + self.current_frame = None; + self.first_stream_packet_index = None; + self.frame_type = None; + self.last_packet_payload_length = None; + self.expected_access_unit_length = None; + self.bytes.clear(); + } + + fn push( + &mut self, + header: NvVideoPacket, + timestamp: u32, + payload: &[u8], + ) -> Result, NvstDropReason> { + if header.is_fec { + return Err(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )); + } + let mut frame_header_size = 0; + let mut picture_payload = payload; + if header.is_start_of_frame() { + self.reset(); + picture_payload = video_picture_payload(self.codec, payload) + .ok_or(NvstDropReason::MissingAnnexBStartCode)?; + frame_header_size = payload.len() - picture_payload.len(); + self.frame_type = + GsFrameHeader::parse(payload, frame_header_size).map(|header| header.frame_type); + if !self.first_start_payload_logged { + self.first_start_payload_logged = true; + eprintln!( + "NVST first {} start payload: frame={} bytes={} frameHeader={} raw={}", + self.codec.label(), + header.frame_index, + payload.len(), + frame_header_size, + diagnostic_hex(payload, 64), + ); + } + if self.codec == NvstVideoCodec::Av1 { + match payload.first() { + Some(0x01) => { + self.last_packet_payload_length = payload + .get(4..6) + .map(|length| usize::from(u16::from_le_bytes([length[0], length[1]]))); + } + Some(0x81) => { + self.expected_access_unit_length = payload.get(16..20).map(|length| { + u32::from_le_bytes([length[0], length[1], length[2], length[3]]) + as usize + }); + } + _ => {} + } + } + self.current_frame = Some(header.frame_index); + self.first_stream_packet_index = Some(header.stream_packet_index); + } else if self.current_frame != Some(header.frame_index) { + self.reset(); + return Err(NvstDropReason::AwaitingStartOfFrame); + } + + // GFN pads the final packet to its FEC block size. AVC/HEVC Annex B + // decoders tolerate those trailing zeroes, while AV1 decoders require + // exact access units. The short header's bytes 4..6 describe the final + // packet; the cloud 0x81 header instead advertises the complete AV1 + // access-unit size in bytes 16..20 and is handled after assembly below. + if self.codec == NvstVideoCodec::Av1 + && header.is_end_of_frame() + && let Some(reported) = self.last_packet_payload_length + { + let exact_payload_length = reported.checked_sub(frame_header_size); + if let Some(exact_payload_length) = exact_payload_length + && exact_payload_length > 0 + && exact_payload_length <= picture_payload.len() + { + picture_payload = &picture_payload[..exact_payload_length]; + } else if !self.invalid_av1_length_logged { + self.invalid_av1_length_logged = true; + eprintln!( + "NVST AV1 final-packet length hint ignored: reported={reported} frameHeader={frame_header_size} available={}", + picture_payload.len() + ); + } + } + + let remaining = self.max_access_unit_bytes.saturating_sub(self.bytes.len()); + if picture_payload.len() > remaining { + self.reset(); + return Err(NvstDropReason::AccessUnitTooLarge { + limit: self.max_access_unit_bytes, + }); + } + self.bytes.extend_from_slice(picture_payload); + if !header.is_end_of_frame() { + return Ok(None); + } + + let mut bytes = std::mem::take(&mut self.bytes); + if self.codec == NvstVideoCodec::Av1 + && let Some(reported) = self.expected_access_unit_length + { + if reported > 0 && reported <= bytes.len() { + bytes.truncate(reported); + } else { + let available = bytes.len(); + self.reset(); + return Err(NvstDropReason::InvalidLastPacketPayloadLength { + reported, + frame_header_size, + available, + }); + } + } + strip_trailing_access_unit_delimiter(self.codec, &mut bytes); + if !self.first_access_unit_logged { + self.first_access_unit_logged = true; + eprintln!( + "NVST first {} access unit: frame={} bytes={} frameType={:?} raw={}", + self.codec.label(), + header.frame_index, + bytes.len(), + self.frame_type, + diagnostic_hex(&bytes, 64), + ); + } + self.current_frame = None; + let first_stream_packet_index = self + .first_stream_packet_index + .take() + .expect("start-of-frame initializes the packet index"); + Ok(Some(EncodedVideoAccessUnit { + codec: self.codec, + timestamp, + frame_index: header.frame_index, + first_stream_packet_index, + frame_type: self.frame_type, + keyframe: video_access_unit_is_keyframe( + self.codec, + &bytes, + self.frame_type == Some(NvstFrameType::I), + ), + contiguous: true, + bytes, + })) + } +} + +fn strip_trailing_access_unit_delimiter(codec: NvstVideoCodec, bytes: &mut Vec) { + if codec == NvstVideoCodec::Av1 { + return; + } + let mut offset = 0; + let mut last_nal = None; + while let Some((start, prefix_len)) = find_annex_b_start_code(&bytes[offset..]) { + let start = offset + start; + let nal_start = start + prefix_len; + last_nal = Some((start, nal_start)); + offset = nal_start; + } + if let Some((start, nal_start)) = last_nal + && bytes.get(nal_start).is_some_and(|header| match codec { + NvstVideoCodec::H264 => header & 0x1f == 9, + NvstVideoCodec::H265 => (header >> 1) & 0x3f == 35, + NvstVideoCodec::Av1 => false, + }) + { + bytes.truncate(start); + } +} + +fn video_picture_payload(codec: NvstVideoCodec, payload: &[u8]) -> Option<&[u8]> { + match codec { + NvstVideoCodec::H264 | NvstVideoCodec::H265 => { + let search_len = payload.len().min(MAX_GS_FRAME_HEADER_BYTES + 4); + let (offset, _) = find_annex_b_start_code(&payload[..search_len])?; + Some(&payload[offset..]) + } + NvstVideoCodec::Av1 => match payload.first()? { + // Unlike AVC/HEVC, AV1 has no Annex-B start code that can be used + // to discover this boundary. Use the GFN cloud header sizes seen + // on the same NVST video track instead of the similarly named but + // different consumer GameStream extended-header layout. + 0x01 => payload.get(GS_SHORT_FRAME_HEADER_BYTES..), + 0x81 => payload.get(GFN_EXTENDED_FRAME_HEADER_BYTES..), + _ => None, + }, + } +} + +fn video_access_unit_is_keyframe( + codec: NvstVideoCodec, + bytes: &[u8], + frame_header_hint: bool, +) -> bool { + if codec == NvstVideoCodec::Av1 { + return frame_header_hint; + } + let mut offset = 0; + while let Some((start, prefix_len)) = find_annex_b_start_code(&bytes[offset..]) { + let nal_start = offset + start + prefix_len; + if let Some(nal_header) = bytes.get(nal_start) + && match codec { + NvstVideoCodec::H264 => nal_header & 0x1f == 5, + NvstVideoCodec::H265 => matches!((nal_header >> 1) & 0x3f, 16..=21), + NvstVideoCodec::Av1 => false, + } + { + return true; + } + offset = nal_start; + } + false +} + +fn find_annex_b_start_code(bytes: &[u8]) -> Option<(usize, usize)> { + for (offset, prefix) in bytes.windows(3).enumerate() { + if prefix == [0, 0, 1] { + return Some((offset, 3)); + } + if prefix == [0, 0, 0] && bytes.get(offset + 3) == Some(&1) { + return Some((offset, 4)); + } + } + None +} + +struct RtpReorderBuffer { + next_index: Option, + packets: BTreeMap, + max_packets: usize, + gap_wait: Option<(u64, Instant)>, +} + +struct ReorderResult { + ready: Vec, + nack: Option<(u64, u64)>, + recovery: Option, + dropped: Option, + fec_repaired: usize, +} + +impl RtpReorderBuffer { + fn new(max_packets: usize) -> Self { + Self { + next_index: None, + packets: BTreeMap::new(), + max_packets, + gap_wait: None, + } + } + + fn reset(&mut self) { + self.next_index = None; + self.packets.clear(); + self.gap_wait = None; + } + + fn push(&mut self, packet: RtpPacket, now: Instant) -> ReorderResult { + let index = packet.index; + let next_index = *self.next_index.get_or_insert(index); + if index < next_index { + return ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped: Some(NvstDropReason::StaleRtpPacket { index }), + fec_repaired: 0, + }; + } + if self.packets.contains_key(&index) { + return ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped: Some(NvstDropReason::DuplicateRtpPacket { index }), + fec_repaired: 0, + }; + } + + let buffered_before = self.packets.len(); + let mut recovery = None; + if buffered_before == 0 && index.saturating_sub(next_index) >= self.max_packets as u64 { + recovery = Some(NvstRecovery::PacketGap { + first_missing_index: next_index, + last_missing_index: index - 1, + }); + self.next_index = Some(index); + self.gap_wait = None; + } + self.packets.insert(index, packet); + self.drain(now, recovery) + } + + fn poll_deadline(&mut self, now: Instant) -> ReorderResult { + self.drain(now, None) + } + + fn drain(&mut self, now: Instant, mut recovery: Option) -> ReorderResult { + let mut ready = Vec::new(); + let mut nack = None; + loop { + while let Some(next) = self.next_index { + let Some(packet) = self.packets.remove(&next) else { + break; + }; + ready.push(packet); + self.next_index = Some(next + 1); + } + + let Some((&first_available, _)) = self.packets.first_key_value() else { + self.gap_wait = None; + break; + }; + let expected = self.next_index.expect("next index remains initialized"); + debug_assert!(first_available > expected); + + // Only NACK sequence numbers that are genuinely absent. Using the + // newest packet index as the range end also requested every packet + // already held in the reorder map, causing duplicate retransmission + // bursts that competed with live video traffic. + nack = Some((expected, first_available.saturating_sub(1))); + let gap_started = match self.gap_wait { + Some((waiting_for, started)) if waiting_for == expected => started, + _ => { + self.gap_wait = Some((expected, now)); + now + } + }; + let gap_exceeded_window = first_available.saturating_sub(expected) + >= self.max_packets as u64 + || self.packets.len() >= self.max_packets; + let gap_timed_out = + now.saturating_duration_since(gap_started) >= MJOLNIR_REORDER_DEQUEUE_TIMEOUT; + if !gap_exceeded_window && !gap_timed_out { + break; + } + + recovery = Some(NvstRecovery::PacketGap { + first_missing_index: expected, + last_missing_index: first_available - 1, + }); + ready.clear(); + self.next_index = Some(first_available); + self.gap_wait = None; + nack = None; + } + ReorderResult { + ready, + nack, + recovery, + dropped: None, + fec_repaired: 0, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FecPacketLayout { + frame_index: u32, + block_index: u8, + last_block_index: u8, + shard_index: usize, + data_shards: usize, + parity_shards: usize, +} + +impl FecPacketLayout { + fn from_packet(packet: &RtpPacket) -> Option { + let extension = packet.header.gs_video_header?; + let frame_index = u32::from_le_bytes(extension[4..8].try_into().ok()?); + let fec_word = u32::from_le_bytes(extension[12..16].try_into().ok()?); + let repair_percent = usize::try_from((fec_word >> 4) & 0xff).ok()?; + let shard_index = usize::try_from((fec_word >> 12) & 0x3ff).ok()?; + let data_shards = usize::try_from((fec_word >> 22) & 0x3ff).ok()?; + if data_shards == 0 || repair_percent == 0 { + return None; + } + let parity_shards = data_shards.saturating_mul(repair_percent).div_ceil(100); + let total_shards = data_shards.checked_add(parity_shards)?; + if parity_shards == 0 || shard_index >= total_shards || total_shards > 1_023 { + return None; + } + Some(Self { + frame_index, + block_index: (extension[11] >> 4) & 0x03, + last_block_index: (extension[11] >> 6) & 0x03, + shard_index, + data_shards, + parity_shards, + }) + } +} + +#[derive(Clone)] +struct FecBlock { + layout: FecPacketLayout, + base_index: u64, + shards: Vec>, + repair_failed: bool, +} + +struct Gf256Tables { + exponent: [u8; 512], + logarithm: [u8; 256], +} + +fn gf256_tables() -> &'static Gf256Tables { + static TABLES: OnceLock = OnceLock::new(); + TABLES.get_or_init(|| { + // NVIDIA's GameStream FEC uses GF(2^8) with primitive polynomial 0x11d. + let mut exponent = [0_u8; 512]; + let mut logarithm = [0_u8; 256]; + let mut value = 1_u16; + for (power, slot) in exponent.iter_mut().take(255).enumerate() { + *slot = value as u8; + logarithm[value as usize] = power as u8; + value <<= 1; + if value & 0x100 != 0 { + value ^= 0x11d; + } + } + let mut power = 255; + while power < exponent.len() { + exponent[power] = exponent[power - 255]; + power += 1; + } + Gf256Tables { + exponent, + logarithm, + } + }) +} + +fn gf256_multiply(left: u8, right: u8) -> u8 { + if left == 0 || right == 0 { + return 0; + } + let tables = gf256_tables(); + tables.exponent[usize::from(tables.logarithm[left as usize]) + + usize::from(tables.logarithm[right as usize])] +} + +fn gf256_inverse(value: u8) -> Option { + (value != 0).then(|| { + let tables = gf256_tables(); + tables.exponent[255 - usize::from(tables.logarithm[value as usize])] + }) +} + +fn gf256_axpy(destination: &mut [u8], source: &[u8], coefficient: u8) { + debug_assert_eq!(destination.len(), source.len()); + if coefficient == 0 { + return; + } + if coefficient == 1 { + for (destination, source) in destination.iter_mut().zip(source) { + *destination ^= *source; + } + return; + } + for (destination, source) in destination.iter_mut().zip(source) { + *destination ^= gf256_multiply(coefficient, *source); + } +} + +fn gf256_scale(bytes: &mut [u8], coefficient: u8) { + if coefficient == 1 { + return; + } + for byte in bytes { + *byte = gf256_multiply(*byte, coefficient); + } +} + +fn nvst_cauchy_coefficient( + data_index: usize, + parity_index: usize, + parity_shards: usize, +) -> Option { + let data_coordinate = u8::try_from(parity_shards.checked_add(data_index)?).ok()?; + let parity_coordinate = u8::try_from(parity_index).ok()?; + gf256_inverse(data_coordinate ^ parity_coordinate) +} + +/// Reconstructs GameStream's systematic Cauchy Reed-Solomon data shards. Common generic Reed- +/// Solomon crates use a different Vandermonde generator matrix; feeding NVIDIA parity to one +/// produces mathematically valid but corrupt H.264 packets. +fn reconstruct_nvst_cauchy_data( + shards: &mut [Option>], + data_shards: usize, + parity_shards: usize, + shard_len: usize, +) -> Result<(), NvstDropReason> { + if data_shards == 0 + || parity_shards == 0 + || data_shards + parity_shards > u8::MAX as usize + || shards.len() != data_shards + parity_shards + { + return Err(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )); + } + let missing = shards[..data_shards] + .iter() + .enumerate() + .filter_map(|(index, shard)| shard.is_none().then_some(index)) + .collect::>(); + if missing.is_empty() { + return Ok(()); + } + let parity_rows = shards[data_shards..] + .iter() + .enumerate() + .filter_map(|(index, shard)| shard.is_some().then_some(index)) + .take(missing.len()) + .collect::>(); + if parity_rows.len() != missing.len() { + return Err(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )); + } + + let mut matrix = parity_rows + .iter() + .map(|&parity_index| { + missing + .iter() + .map(|&data_index| { + nvst_cauchy_coefficient(data_index, parity_index, parity_shards).ok_or( + NvstDropReason::Unsupported(NvstUnsupportedFeature::FecRepair), + ) + }) + .collect::, _>>() + }) + .collect::, _>>()?; + let mut recovered = parity_rows + .iter() + .map(|&parity_index| { + let mut bytes = shards[data_shards + parity_index] + .as_ref() + .expect("selected parity shard is present") + .clone(); + bytes.resize(shard_len, 0); + bytes + }) + .collect::>(); + + for (row, &parity_index) in parity_rows.iter().enumerate() { + for (data_index, known) in shards[..data_shards].iter().enumerate() { + let Some(known) = known.as_ref() else { + continue; + }; + let coefficient = nvst_cauchy_coefficient(data_index, parity_index, parity_shards) + .ok_or(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))?; + let known_len = known.len().min(shard_len); + gf256_axpy( + &mut recovered[row][..known_len], + &known[..known_len], + coefficient, + ); + } + } + + // Reduce the missing-data coefficient matrix to identity while applying the same row + // operations to whole packet shards. + for column in 0..missing.len() { + let pivot = (column..missing.len()) + .find(|&row| matrix[row][column] != 0) + .ok_or(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))?; + matrix.swap(column, pivot); + recovered.swap(column, pivot); + let inverse = gf256_inverse(matrix[column][column]).ok_or(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))?; + gf256_scale(&mut matrix[column][column..], inverse); + gf256_scale(&mut recovered[column], inverse); + for row in 0..missing.len() { + if row == column { + continue; + } + let coefficient = matrix[row][column]; + if coefficient == 0 { + continue; + } + let (target_matrix, pivot_matrix) = if row < column { + let (before, after) = matrix.split_at_mut(column); + (&mut before[row], &after[0]) + } else { + let (before, after) = matrix.split_at_mut(row); + (&mut after[0], &before[column]) + }; + gf256_axpy( + &mut target_matrix[column..], + &pivot_matrix[column..], + coefficient, + ); + let (target_shard, pivot_shard) = if row < column { + let (before, after) = recovered.split_at_mut(column); + (&mut before[row], &after[0]) + } else { + let (before, after) = recovered.split_at_mut(row); + (&mut after[0], &before[column]) + }; + gf256_axpy(target_shard, pivot_shard, coefficient); + } + } + + for (data_index, bytes) in missing.into_iter().zip(recovered) { + shards[data_index] = Some(bytes); + } + Ok(()) +} + +impl FecBlock { + fn new(layout: FecPacketLayout, packet_index: u64) -> Self { + let total = layout.data_shards + layout.parity_shards; + Self { + layout, + base_index: packet_index.saturating_sub(layout.shard_index as u64), + shards: (0..total).map(|_| None).collect(), + repair_failed: false, + } + } + + fn matches(&self, layout: FecPacketLayout, packet_index: u64) -> bool { + self.layout.frame_index == layout.frame_index + && self.layout.block_index == layout.block_index + && self.layout.last_block_index == layout.last_block_index + && self.layout.data_shards == layout.data_shards + && self.layout.parity_shards == layout.parity_shards + && self.base_index == packet_index.saturating_sub(layout.shard_index as u64) + } + + fn missing_data_range(&self) -> Option<(u64, u64)> { + let first = self.shards[..self.layout.data_shards] + .iter() + .position(Option::is_none)?; + let last = self.shards[..self.layout.data_shards] + .iter() + .rposition(Option::is_none) + .unwrap_or(first); + Some(( + self.base_index + first as u64, + self.base_index + last as u64, + )) + } + + fn first_missing_data_run(&self) -> Option<(u64, u64)> { + let first = self.shards[..self.layout.data_shards] + .iter() + .position(Option::is_none)?; + let length = self.shards[first..self.layout.data_shards] + .iter() + .take_while(|shard| shard.is_none()) + .count(); + Some(( + self.base_index + first as u64, + self.base_index + first as u64 + length.saturating_sub(1) as u64, + )) + } + + fn insert(&mut self, layout: FecPacketLayout, packet: RtpPacket) -> Option { + let slot = &mut self.shards[layout.shard_index]; + if slot.is_some() { + return Some(NvstDropReason::DuplicateRtpPacket { + index: packet.index, + }); + } + *slot = Some(packet); + // A newly arrived data or parity shard can make a previously failed reconstruction + // solvable. Permit one new attempt, but do not spin on every unrelated packet. + self.repair_failed = false; + None + } + + fn has_enough_shards(&self) -> bool { + self.shards.iter().filter(|shard| shard.is_some()).count() >= self.layout.data_shards + } + + fn finish(mut self, shard_len: usize) -> Result<(Vec, usize), NvstDropReason> { + let missing_data = self.shards[..self.layout.data_shards] + .iter() + .filter(|shard| shard.is_none()) + .count(); + if missing_data == 0 { + return Ok(( + self.shards[..self.layout.data_shards] + .iter_mut() + .filter_map(Option::take) + .collect(), + 0, + )); + } + + if self + .shards + .iter() + .flatten() + .any(|packet| packet.plaintext.len() > shard_len) + { + return Err(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )); + } + // The RTP envelope is transport framing, not protected video data. A received parity + // packet therefore provides an equally valid envelope template. This matters for the + // common 1-data/2-parity layout: if that one data packet is lost, parity is all we have + // and the block is still fully recoverable. + let template = self + .shards + .iter() + .flatten() + .next() + .ok_or(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))?; + let template_header = template.header; + let mut shard_bytes = self + .shards + .iter() + .map(|shard| { + shard.as_ref().map(|packet| { + let mut bytes = packet.plaintext.clone(); + bytes.resize(shard_len, 0); + bytes + }) + }) + .collect::>(); + reconstruct_nvst_cauchy_data( + &mut shard_bytes, + self.layout.data_shards, + self.layout.parity_shards, + shard_len, + )?; + + let extension_start = template_header + .payload_offset + .checked_sub(NV_VIDEO_PACKET_LEN) + .ok_or(NvstDropReason::MalformedRtp( + RtpParseError::InvalidExtensionLength, + ))?; + let template_rtp_envelope = template.plaintext[..extension_start].to_vec(); + for (index, shard) in shard_bytes[..self.layout.data_shards] + .iter_mut() + .enumerate() + { + if self.shards[index].is_some() { + continue; + } + let plaintext = shard.as_mut().ok_or(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))?; + if plaintext.len() < extension_start + NV_VIDEO_PACKET_LEN { + return Err(NvstDropReason::MalformedRtp( + RtpParseError::InvalidExtensionLength, + )); + } + let sequence = (self.base_index + index as u64) as u16; + // A parity packet has its own valid RTP envelope, so those bytes are not necessarily + // the raw Reed-Solomon parity of the data-packet envelopes. Moonlight restores the + // recovered RTP header from a received packet before consuming NV_VIDEO_PACKET. Do + // the equivalent here, including the GS extension profile/length that our strict RTP + // parser validates. The NVIDIA metadata and encoded payload remain FEC output. + plaintext[..extension_start].copy_from_slice(&template_rtp_envelope); + plaintext[2..4].copy_from_slice(&sequence.to_be_bytes()); + plaintext[extension_start + 4..extension_start + 8] + .copy_from_slice(&self.layout.frame_index.to_le_bytes()); + plaintext[extension_start + 11] = + (self.layout.last_block_index << 6) | (self.layout.block_index << 4); + let header = RtpHeader::parse(plaintext).map_err(NvstDropReason::MalformedRtp)?; + let flags = header.gs_video_header.ok_or(NvstDropReason::MalformedRtp( + RtpParseError::MissingNvVideoHeader, + ))?[8] + & 0x0f; + let invalid_flags = flags & !(FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA) != 0; + if invalid_flags { + return Err(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )); + } + let recovered = RtpPacket { + index: self.base_index + index as u64, + header, + plaintext: std::mem::take(plaintext), + origin: RtpPacketOrigin::RecoveredData, + }; + // Do not validate the reconstructed FEC word. NVIDIA's reference-compatible + // receivers explicitly exclude fecInfo from recovered-packet validation because + // it is transport metadata and can differ in a valid repaired data shard. The + // frame and multi-FEC block coordinates above are restored from the already + // authenticated block, just like the RTP envelope. Requiring fecInfo to describe + // a data shard caused every real cloud repair to be rejected even though the H.264 + // payload had been reconstructed successfully. + self.shards[index] = Some(recovered); + } + Ok(( + self.shards[..self.layout.data_shards] + .iter_mut() + .filter_map(Option::take) + .collect(), + missing_data, + )) + } +} + +struct FecReorderBuffer { + blocks: BTreeMap, + completed_through: Option, + gap_wait: Option<(u64, Instant)>, + shard_len: usize, +} + +impl FecReorderBuffer { + fn new(video_packet_size: usize) -> Self { + Self { + blocks: BTreeMap::new(), + completed_through: None, + gap_wait: None, + shard_len: video_packet_size + NVST_FEC_RTP_HEADER_ALLOWANCE, + } + } + + fn reset(&mut self) { + self.blocks.clear(); + self.completed_through = None; + self.gap_wait = None; + } + + fn push(&mut self, packet: RtpPacket, layout: FecPacketLayout, now: Instant) -> ReorderResult { + let packet_base = packet.index.saturating_sub(layout.shard_index as u64); + if self + .completed_through + .is_some_and(|completed_through| packet.index < completed_through) + { + return ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped: None, + fec_repaired: 0, + }; + } + let block = self + .blocks + .entry(packet_base) + .or_insert_with(|| FecBlock::new(layout, packet.index)); + if !block.matches(layout, packet.index) { + return ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped: Some(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + )), + fec_repaired: 0, + }; + } + let dropped = block.insert(layout, packet); + if dropped.is_some() { + return ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped, + fec_repaired: 0, + }; + } + + self.poll_deadline(now) + } + + fn poll_deadline(&mut self, now: Instant) -> ReorderResult { + let mut result = ReorderResult { + ready: Vec::new(), + nack: None, + recovery: None, + dropped: None, + fec_repaired: 0, + }; + loop { + let Some((&base, head)) = self.blocks.first_key_value() else { + self.gap_wait = None; + break; + }; + + if let Some(expected) = self.completed_through + && base > expected + { + result.nack = Some((expected, base - 1)); + let started = match self.gap_wait { + Some((waiting_for, started)) if waiting_for == expected => started, + _ => { + self.gap_wait = Some((expected, now)); + now + } + }; + if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + && base.saturating_sub(expected) < MAX_REORDER_WINDOW as u64 + && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + { + break; + } + result.recovery = Some(NvstRecovery::PacketGap { + first_missing_index: expected, + last_missing_index: base - 1, + }); + result.ready.clear(); + result.nack = None; + self.completed_through = Some(base); + self.gap_wait = None; + continue; + } + + if head.has_enough_shards() && !head.repair_failed { + let completed = self + .blocks + .remove(&base) + .expect("head FEC block is present"); + let completed_through = + base + (completed.layout.data_shards + completed.layout.parity_shards) as u64; + // Only reconstruction can fail. Healthy blocks move their RTP + // buffers straight to assembly without copying the video payload. + let repair_backup = completed.missing_data_range().map(|_| completed.clone()); + match completed.finish(self.shard_len) { + Ok((mut ready, repaired)) => { + result.ready.append(&mut ready); + result.fec_repaired += repaired; + self.completed_through = Some(completed_through); + self.gap_wait = None; + continue; + } + Err(reason) => { + let mut pending = repair_backup.expect("failed repair has missing data"); + pending.repair_failed = true; + let (first_missing, last_missing) = pending + .first_missing_data_run() + .expect("failed FEC reconstruction has missing data"); + eprintln!( + "NVST FEC reconstruction deferred to NACK: frame={} block={}/{} data={} parity={} missing={}..={} reason={reason:?}", + pending.layout.frame_index, + pending.layout.block_index, + pending.layout.last_block_index, + pending.layout.data_shards, + pending.layout.parity_shards, + first_missing, + last_missing, + ); + self.blocks.insert(base, pending); + result.nack = Some((first_missing, last_missing)); + self.gap_wait = Some((first_missing, now)); + break; + } + } + } + + // A later block proves that the head block has stopped arriving normally. Ask for + // only the first contiguous missing run and retain all subsequent blocks while the + // retransmission is in flight. + if self.blocks.len() == 1 && self.gap_wait.is_none() { + break; + } + let (first_missing, last_missing) = head + .first_missing_data_run() + .expect("an incomplete FEC block has missing data"); + result.nack = Some((first_missing, last_missing)); + let started = match self.gap_wait { + Some((waiting_for, started)) if waiting_for == first_missing => started, + _ => { + self.gap_wait = Some((first_missing, now)); + now + } + }; + let block_span = self + .blocks + .last_key_value() + .map_or(0, |(&last_base, _)| last_base.saturating_sub(base)); + if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + && block_span < MAX_REORDER_WINDOW as u64 + && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + { + break; + } + + let failed = self + .blocks + .remove(&base) + .expect("head FEC block is present"); + let (first_missing_index, last_missing_index) = failed + .missing_data_range() + .expect("an incomplete FEC block has missing data"); + result.recovery = Some(NvstRecovery::PacketGap { + first_missing_index, + last_missing_index, + }); + result.ready.clear(); + result.nack = None; + self.completed_through = + Some(base + (failed.layout.data_shards + failed.layout.parity_shards) as u64); + self.gap_wait = None; + // Emit any complete successors after the assembler has been marked discontinuous. + } + result + } +} + +/// Sends periodic SRTCP Receiver Reports so the peer keeps the video flowing. +/// The official client maintains an SRTCP session (hook captures show the 0x03/0x05 +/// KDF labels); without any receiver feedback the server stops video after an +/// initial burst. Only the GCM profiles are supported (the active Mjolnir policy). +struct SrtcpSender { + encryption_key: SrtcpKey, + session_salt: [u8; 12], + tag_len: usize, + next_index: u32, + last_sent: Option, +} + +enum SrtcpKey { + Aes128([u8; 16]), + Aes256([u8; 32]), +} + +impl SrtcpKey { + fn as_bytes(&self) -> &[u8] { + match self { + Self::Aes128(key) => key, + Self::Aes256(key) => key, + } + } +} + +impl SrtcpSender { + fn from_material(material: &NvstSrtpMaterial) -> Option { + let (encryption_key, session_salt, tag_len) = match material { + NvstSrtpMaterial::AeadAes128Gcm { + master_key, + master_salt, + authentication_tag_len, + } => ( + SrtcpKey::Aes128(derive_aes128_cm_key::<16>( + master_key, + master_salt, + GFN_SRTCP_KEY_LABEL, + )), + derive_aes128_cm_key::<12>(master_key, master_salt, GFN_SRTCP_SALT_LABEL), + *authentication_tag_len, + ), + NvstSrtpMaterial::AeadAes256Gcm { + master_key, + master_salt, + authentication_tag_len, + } => ( + SrtcpKey::Aes256(derive_aes_cm_key::<32>( + master_key, + master_salt, + GFN_SRTCP_KEY_LABEL, + )), + derive_aes_cm_key::<12>(master_key, master_salt, GFN_SRTCP_SALT_LABEL), + *authentication_tag_len, + ), + _ => return None, + }; + Some(Self { + encryption_key, + session_salt, + tag_len, + next_index: 0, + last_sent: None, + }) + } + + /// Returns an SRTCP Receiver Report to send once per interval, after the media + /// SSRC is known. Returns `None` when it is not yet time or nothing to report on. + fn poll_receiver_report( + &mut self, + report: Option, + now: Instant, + ) -> Option> { + let report = report?; + if let Some(last) = self.last_sent + && now.duration_since(last) < SRTCP_RR_INTERVAL + { + return None; + } + self.last_sent = Some(now); + let index = self.next_index; + self.next_index = self.next_index.wrapping_add(1); + Some(protect_srtcp_receiver_report_gcm( + report, + index, + self.encryption_key.as_bytes(), + &self.session_salt, + self.tag_len, + )) + } +} + +/// Stateful, non-blocking NVST video receiver. `process_datagram` is deterministic and testable; +/// `spawn_nvst_udp_receiver` below is a thin UDP/thread wrapper for the production path. +pub struct NvstVideoReceiver { + config: NvstVideoConfig, + srtp: SrtpReceiver, + srtcp: Option, + reorder: RtpReorderBuffer, + fec_reorder: FecReorderBuffer, + assembler: VideoAccessUnitAssembler, + last_stream_packet_index: Option, + next_frame_contiguous: bool, + state: NvstReceiverState, + bound_ssrc: Option, + highest_sequence_received: u32, + authenticated_packets: u64, + fec_packets: u64, + fec_repaired_packets: u64, + frames_emitted: u64, + replay_rejections: u64, + stale_packets: u64, + recovered_retransmissions: u64, + packet_gap_recoveries: u64, + timeout_origin: Instant, + last_authenticated_packet: Option, + initial_timeout_pending: bool, + frame_progress: FrameProgressWatchdog, + frame_invalidation: FrameInvalidation, +} + +impl NvstVideoReceiver { + pub fn new(config: NvstVideoConfig) -> Self { + let srtp = SrtpReceiver::from_material(&config.srtp); + let srtcp = (!config.rtcp_on_sctp) + .then(|| SrtcpSender::from_material(&config.srtp)) + .flatten(); + let reorder = RtpReorderBuffer::new(config.reorder_window_packets); + let fec_reorder = FecReorderBuffer::new(config.video_packet_size); + let assembler = VideoAccessUnitAssembler::new(config.codec, config.max_access_unit_bytes); + Self { + config, + srtp, + srtcp, + reorder, + fec_reorder, + assembler, + last_stream_packet_index: None, + next_frame_contiguous: true, + state: NvstReceiverState::Running, + bound_ssrc: None, + highest_sequence_received: 0, + authenticated_packets: 0, + fec_packets: 0, + fec_repaired_packets: 0, + frames_emitted: 0, + replay_rejections: 0, + stale_packets: 0, + recovered_retransmissions: 0, + packet_gap_recoveries: 0, + timeout_origin: Instant::now(), + last_authenticated_packet: None, + initial_timeout_pending: true, + frame_progress: FrameProgressWatchdog::default(), + frame_invalidation: FrameInvalidation::default(), + } + } + + pub fn state(&self) -> NvstReceiverState { + self.state + } + + pub fn pause(&mut self) -> Option { + if self.state != NvstReceiverState::Running { + return None; + } + self.reset_media_state(); + self.state = NvstReceiverState::Paused; + Some(NvstReceiveEvent::Lifecycle(self.state)) + } + + pub fn resume(&mut self) -> Option { + if self.state != NvstReceiverState::Paused { + return None; + } + self.reset_media_state(); + self.last_authenticated_packet = None; + self.timeout_origin = Instant::now(); + self.initial_timeout_pending = false; + self.state = NvstReceiverState::Running; + Some(NvstReceiveEvent::Lifecycle(self.state)) + } + + pub fn recover(&mut self) -> Option { + if self.state != NvstReceiverState::RecoveryRequired { + return None; + } + self.reset_media_state(); + self.last_authenticated_packet = None; + self.timeout_origin = Instant::now(); + self.initial_timeout_pending = false; + self.state = NvstReceiverState::Running; + Some(NvstReceiveEvent::Lifecycle(self.state)) + } + + pub fn stop(&mut self, cause: NvstStopCause) -> Option { + if matches!(self.state, NvstReceiverState::Stopped(_)) { + return None; + } + self.reset_media_state(); + self.state = NvstReceiverState::Stopped(cause); + Some(NvstReceiveEvent::Lifecycle(self.state)) + } + + /// Returns a typed timeout only once. The caller must deliberately call `recover` before + /// more media is accepted, preventing a stale stream from silently resuming. + pub fn poll_timeout(&mut self, now: Instant) -> Option { + if self.state != NvstReceiverState::Running { + return None; + } + let last_packet = self.config.feedback.video_timeout_origin( + self.last_authenticated_packet + .unwrap_or(self.timeout_origin), + )?; + let idle_for = now.saturating_duration_since(last_packet); + let timeout = if self.initial_timeout_pending { + self.config.startup_timeout + } else { + self.config.timeout + }; + if idle_for < timeout { + return None; + } + self.reset_media_state(); + self.state = NvstReceiverState::RecoveryRequired; + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { + idle_for, + })) + } + + pub fn frame_progress(&self) -> NvstFrameProgress { + let stage = self.config.feedback().frame_stage_timings(); + NvstFrameProgress { + authenticated_packets: self.authenticated_packets, + last_authenticated_packet: self.last_authenticated_packet, + assembled_frames_total: stage.assembled_frames_total, + last_assembled_frame_index: self.frame_progress.last_assembled_frame_index, + last_assembled_at: stage.last_assembled_at, + stage: self.frame_progress.stage, + } + } + + pub fn poll_frame_progress( + &mut self, + now: Instant, + policy: NvstFrameProgressPolicy, + ) -> Option { + if self.state != NvstReceiverState::Running || self.initial_timeout_pending { + return None; + } + let assembled_at = self.config.feedback().last_assembled_frame_at(); + let epoch_at = self.frame_progress.first_authenticated_at; + let base = if !self.frame_progress.first_keyframe_assembled { + epoch_at? + } else { + match (assembled_at, epoch_at) { + (Some(assembled), Some(epoch)) => assembled.max(epoch), + (Some(assembled), None) => assembled, + (None, epoch) => epoch?, + } + }; + let base = self.config.feedback.video_timeout_origin(base)?; + let idle_for = now.saturating_duration_since(base); + match self.frame_progress.stage { + NvstFrameProgressStage::Tracking => { + if idle_for < policy.stall { + return None; + } + self.frame_progress.stage = NvstFrameProgressStage::KeyframePending; + self.frame_progress.keyframe_requested_at = Some(now); + self.config.feedback.request_keyframe(); + Some(NvstFrameProgressEvent::KeyframeRequested { + idle_for, + last_assembled_frame_index: self.frame_progress.last_assembled_frame_index, + }) + } + NvstFrameProgressStage::KeyframePending => { + let grace_started = self.frame_progress.keyframe_requested_at.unwrap_or(now); + if now.saturating_duration_since(grace_started) < policy.keyframe_grace { + return None; + } + self.frame_progress.stage = NvstFrameProgressStage::RecoveryRequired; + self.state = NvstReceiverState::RecoveryRequired; + Some(NvstFrameProgressEvent::RecoveryNeeded { + idle_for, + last_assembled_frame_index: self.frame_progress.last_assembled_frame_index, + }) + } + NvstFrameProgressStage::RecoveryRequired => None, + } + } + + pub fn process_datagram( + &mut self, + source: SocketAddr, + datagram: &[u8], + now: Instant, + ) -> Vec { + if !self.config.accepts_video_source(source) { + return vec![NvstReceiveEvent::Dropped( + NvstDropReason::UnexpectedSource { + expected: self.config.video_peer, + actual: source, + }, + )]; + } + match self.state { + NvstReceiverState::Paused => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::Paused)]; + } + NvstReceiverState::Stopped(_) => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::Stopped)]; + } + NvstReceiverState::RecoveryRequired => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::RecoveryRequired)]; + } + NvstReceiverState::Running => {} + } + let packet = match self.srtp.unprotect(datagram) { + Ok(packet) => packet, + Err(reason) => { + if reason == NvstDropReason::ReplayRejected { + self.replay_rejections += 1; + } + return vec![NvstReceiveEvent::Dropped(reason)]; + } + }; + if let Some(expected) = self.config.expected_payload_type + && packet.header.payload_type != expected + { + return vec![NvstReceiveEvent::Dropped( + NvstDropReason::UnexpectedPayloadType { + expected, + actual: packet.header.payload_type, + }, + )]; + } + let expected_ssrc = self.config.expected_ssrc.or(self.bound_ssrc); + if let Some(expected) = expected_ssrc + && packet.header.ssrc != expected + { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::UnexpectedSsrc { + expected, + actual: packet.header.ssrc, + })]; + } + if let Some(extension) = packet.header.gs_video_header { + let frame_number = + u32::from_le_bytes(extension[4..8].try_into().expect("GS header length")); + self.config.feedback.record_frame_packet(frame_number, now); + } + self.bound_ssrc.get_or_insert(packet.header.ssrc); + self.last_authenticated_packet = Some(now); + if !self.initial_timeout_pending || self.config.feedback.video_timeout_origin(now).is_some() + { + self.initial_timeout_pending = false; + self.frame_progress.authenticated(now); + } + self.frame_invalidation + .poll_backstop(now, RFI_BACKSTOP_TIMEOUT, &self.config.feedback); + self.authenticated_packets += 1; + let sequence = u32::try_from(packet.index & 0xffff_ffff).unwrap_or(u32::MAX); + let reordered = + self.authenticated_packets > 1 && sequence <= self.highest_sequence_received; + self.highest_sequence_received = self.highest_sequence_received.max(sequence); + let retransmitted = self.config.feedback.resolve_nack(packet.index); + if retransmitted { + self.recovered_retransmissions += 1; + } + let video = (!retransmitted && !reordered) + .then(|| packet.header.payload(&packet.plaintext).ok()) + .flatten() + .and_then(|payload| NvVideoPacket::parse(&packet.header, payload).ok()) + .map(|(video, payload)| { + ( + video.frame_index, + sequence, + payload.len(), + video.is_start_of_frame(), + video.is_end_of_frame(), + video.is_fec, + ) + }); + self.config.feedback.publish_stream_packet( + packet.header.ssrc, + self.highest_sequence_received, + packet.header.timestamp, + now, + video, + ); + + let result = if let Some(layout) = FecPacketLayout::from_packet(&packet) { + if layout.shard_index >= layout.data_shards { + self.fec_packets += 1; + } + let result = self.fec_reorder.push(packet, layout, now); + if layout.shard_index < layout.data_shards + && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) + { + self.invalidate_picture(now); + } + result + } else { + self.reorder.push(packet, now) + }; + self.consume_reorder_result(result, now) + } + + pub fn poll_recovery(&mut self, now: Instant) -> Vec { + if self.state != NvstReceiverState::Running { + return Vec::new(); + } + self.frame_invalidation + .poll_backstop(now, RFI_BACKSTOP_TIMEOUT, &self.config.feedback); + let reordered = self.reorder.poll_deadline(now); + let mut events = self.consume_reorder_result(reordered, now); + let fec = self.fec_reorder.poll_deadline(now); + events.extend(self.consume_reorder_result(fec, now)); + events + } + + fn consume_reorder_result( + &mut self, + result: ReorderResult, + now: Instant, + ) -> Vec { + self.fec_repaired_packets += result.fec_repaired as u64; + let mut events = Vec::new(); + if let Some((first_missing_index, last_missing_index)) = result.nack { + self.config + .feedback + .request_nack(first_missing_index, last_missing_index, now); + } + if let Some(reason) = result.dropped { + if matches!(reason, NvstDropReason::StaleRtpPacket { .. }) { + self.stale_packets += 1; + } + events.push(NvstReceiveEvent::Dropped(reason)); + } + if let Some(recovery) = result.recovery { + self.packet_gap_recoveries += 1; + self.invalidate_picture(now); + self.config.feedback.clear_nacks(); + events.push(NvstReceiveEvent::RecoveryNeeded(recovery)); + } + for packet in result.ready { + let payload = match packet.header.payload(&packet.plaintext) { + Ok(payload) => payload, + Err(error) => { + self.invalidate_picture(now); + events.push(NvstReceiveEvent::Dropped(NvstDropReason::MalformedRtp( + error, + ))); + continue; + } + }; + let (mut nv_packet, media) = match NvVideoPacket::parse(&packet.header, payload) { + Ok(value) => value, + Err(error) => { + self.invalidate_picture(now); + events.push(NvstReceiveEvent::Dropped(NvstDropReason::MalformedRtp( + error, + ))); + continue; + } + }; + if packet.origin == RtpPacketOrigin::RecoveredData { + nv_packet.is_fec = false; + } + if nv_packet.is_fec { + self.fec_packets += 1; + events.push(NvstReceiveEvent::Dropped(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair, + ))); + continue; + } + if nv_packet.is_start_of_frame() + && self + .assembler + .current_frame + .is_some_and(|current| current != nv_packet.frame_index) + { + // A new SOF before EOF means the previous reference frame was + // incomplete even when RTP sequence numbers were continuous. + // Never let the following P-frame look contiguous to the decoder. + self.invalidate_picture(now); + self.config.feedback.clear_nacks(); + events.push(NvstReceiveEvent::Dropped( + NvstDropReason::FrameDiscontinuity, + )); + } + let same_frame_continuation = !nv_packet.is_start_of_frame() + && self.assembler.current_frame == Some(nv_packet.frame_index); + if same_frame_continuation + && self.last_stream_packet_index.is_some_and(|last| { + last.wrapping_add(1) & STREAM_PACKET_INDEX_MASK != nv_packet.stream_packet_index + }) + { + self.invalidate_picture(now); + events.push(NvstReceiveEvent::Dropped( + NvstDropReason::FrameDiscontinuity, + )); + continue; + } + self.last_stream_packet_index = Some(nv_packet.stream_packet_index); + match self + .assembler + .push(nv_packet, packet.header.timestamp, media) + { + Ok(Some(mut frame)) => { + let loss_before = !self.next_frame_contiguous; + self.next_frame_contiguous = true; + if !self.frame_invalidation.admit( + &mut frame, + loss_before, + now, + &self.config.feedback, + ) { + self.config.feedback.publish_completed_frame(&frame); + self.config + .feedback + .retire_undelivered_frame(frame.frame_index); + continue; + } + self.frames_emitted += 1; + if self + .frame_progress + .assembled(frame.frame_index, frame.keyframe) + { + events.push(NvstReceiveEvent::FrameProgressResumed); + } + self.config.feedback.publish_completed_frame(&frame); + self.config + .feedback + .complete_frame_packets(frame.frame_index); + events.push(NvstReceiveEvent::Frame(frame)); + } + Ok(None) => {} + Err(reason) => { + self.invalidate_picture(now); + events.push(NvstReceiveEvent::Dropped(reason)); + } + } + } + events + } + + fn invalidate_picture(&mut self, now: Instant) { + self.assembler.reset(); + self.last_stream_packet_index = None; + self.next_frame_contiguous = false; + if !self.frame_invalidation.lose(now) { + self.config.feedback.request_keyframe(); + } + } + + /// Returns an SRTCP Receiver Report to send to the video peer, at most once per + /// interval, once the media SSRC is known. Keeps the peer's video flowing. + pub fn poll_receiver_report(&mut self, now: Instant) -> Option> { + if self.state != NvstReceiverState::Running { + return None; + } + self.srtcp + .as_mut()? + .poll_receiver_report(self.config.feedback.report_snapshot(true), now) + } + + /// Elapsed-since-start receive counters for ground-truth timing that does not + /// depend on when buffered log lines happen to flush. + pub fn stats_line(&self, origin: Instant) -> String { + format!( + "elapsed={:.1}s auth={} fec={} fecRecovered={} frames={} replay={} stale={} nackRecovered={} nackPending={} nackOverflow={} gaps={} {} ssrc={:?}", + origin.elapsed().as_secs_f64(), + self.authenticated_packets, + self.fec_packets, + self.fec_repaired_packets, + self.frames_emitted, + self.replay_rejections, + self.stale_packets, + self.recovered_retransmissions, + self.config + .feedback + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len(), + self.config.feedback.nack_overflows.load(Ordering::Relaxed), + self.packet_gap_recoveries, + self.config.feedback.frame_invalidation.stats(), + self.bound_ssrc, + ) + } + + fn process_mjolnir_payload( + &mut self, + ssrc: u32, + rtp_timestamp: u32, + _payload: &[u8], + now: Instant, + ) -> Vec { + match self.state { + NvstReceiverState::Paused => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::Paused)]; + } + NvstReceiverState::Stopped(_) => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::Stopped)]; + } + NvstReceiverState::RecoveryRequired => { + return vec![NvstReceiveEvent::Dropped(NvstDropReason::RecoveryRequired)]; + } + NvstReceiverState::Running => {} + } + self.bound_ssrc.get_or_insert(ssrc); + self.last_authenticated_packet = Some(now); + if !self.initial_timeout_pending || self.config.feedback.video_timeout_origin(now).is_some() + { + self.initial_timeout_pending = false; + } + // The bundle path cannot assemble video: the Mjolnir frame metadata lives + // in the `0x4753` RTP extension, which str0m does not surface. The official + // cloud path delivers video exclusively on the raw Mjolnir socket, so bundle + // RTP (audio/control) is intentionally ignored here. + let _ = rtp_timestamp; + Vec::new() + } + + fn reset_media_state(&mut self) { + self.reorder.reset(); + self.fec_reorder.reset(); + self.assembler.reset(); + self.last_stream_packet_index = None; + self.next_frame_contiguous = false; + self.config + .feedback + .frame_packet_timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.config.feedback().reset_frame_stage_epoch(); + self.frame_progress.reset(); + self.frame_invalidation.reset(&self.config.feedback); + } +} + +enum StunDatagram { + NotStun, + Invalid, + Handled(Option>), +} + +fn append_stun_attribute(packet: &mut Vec, attribute_type: u16, value: &[u8]) { + packet.extend_from_slice(&attribute_type.to_be_bytes()); + packet.extend_from_slice(&(value.len() as u16).to_be_bytes()); + packet.extend_from_slice(value); + packet.resize(packet.len().next_multiple_of(4), 0); +} + +fn build_authenticated_stun_packet( + message_type: u16, + transaction_id: &[u8; 12], + key: &[u8], + attributes: &[(u16, Vec)], +) -> Vec { + let mut packet = Vec::with_capacity(128); + packet.extend_from_slice(&message_type.to_be_bytes()); + packet.extend_from_slice(&0_u16.to_be_bytes()); + packet.extend_from_slice(&STUN_MAGIC_COOKIE.to_be_bytes()); + packet.extend_from_slice(transaction_id); + for (attribute_type, value) in attributes { + append_stun_attribute(&mut packet, *attribute_type, value); + } + + let length_through_integrity = packet.len() - STUN_HEADER_LEN + 24; + packet[2..4].copy_from_slice(&(length_through_integrity as u16).to_be_bytes()); + let mut mac = HmacSha1::new_from_slice(key).expect("HMAC accepts variable-size ICE passwords"); + mac.update(&packet); + append_stun_attribute( + &mut packet, + STUN_ATTR_MESSAGE_INTEGRITY, + &mac.finalize().into_bytes(), + ); + + let final_length = packet.len() - STUN_HEADER_LEN + 8; + packet[2..4].copy_from_slice(&(final_length as u16).to_be_bytes()); + let fingerprint = crc32(&packet) ^ STUN_FINGERPRINT_XOR; + append_stun_attribute( + &mut packet, + STUN_ATTR_FINGERPRINT, + &fingerprint.to_be_bytes(), + ); + packet +} + +fn build_stun_binding_request( + credentials: &NvstStunCredentials, + transaction_id: &[u8; 12], +) -> Vec { + let username = format!( + "{}:{}", + credentials.remote_username_fragment, credentials.local_username_fragment + ); + build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + transaction_id, + credentials.remote_password.as_bytes(), + &[(STUN_ATTR_USERNAME, username.into_bytes())], + ) +} + +/// Official `NattHolePunch::SendPing` (pingVersion=6) encodes +/// `SetStunCredentials(local, pingPayload, …, describePassword)` as +/// USERNAME `pingPayload:localUfrag` and HMAC-SHA1 with the DESCRIBE password. +/// WebRtcTransport ICE uses the V2 ufrag via [`build_stun_binding_request`]. +fn build_natt_hole_punch_request( + local_username_fragment: &str, + ping_payload: &[u8], + remote_password: &str, + transaction_id: &[u8; 12], +) -> Vec { + let username = format!( + "{}:{local_username_fragment}", + String::from_utf8_lossy(ping_payload) + ); + build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + transaction_id, + remote_password.as_bytes(), + &[(STUN_ATTR_USERNAME, username.into_bytes())], + ) +} + +fn xor_mapped_address(source: SocketAddr, transaction_id: &[u8; 12]) -> Vec { + let mut value = Vec::with_capacity(20); + value.push(0); + value.push(if source.is_ipv4() { 1 } else { 2 }); + value.extend_from_slice(&(source.port() ^ ((STUN_MAGIC_COOKIE >> 16) as u16)).to_be_bytes()); + match source.ip() { + IpAddr::V4(ip) => { + for (octet, mask) in ip.octets().into_iter().zip(STUN_MAGIC_COOKIE.to_be_bytes()) { + value.push(octet ^ mask); + } + } + IpAddr::V6(ip) => { + let mut mask = [0_u8; 16]; + mask[..4].copy_from_slice(&STUN_MAGIC_COOKIE.to_be_bytes()); + mask[4..].copy_from_slice(transaction_id); + for (octet, mask) in ip.octets().into_iter().zip(mask) { + value.push(octet ^ mask); + } + } + } + value +} + +pub(crate) fn find_stun_attribute(packet: &[u8], wanted_type: u16) -> Option<(usize, &[u8])> { + let mut offset = STUN_HEADER_LEN; + while offset + 4 <= packet.len() { + let attribute_type = u16::from_be_bytes([packet[offset], packet[offset + 1]]); + let length = usize::from(u16::from_be_bytes([packet[offset + 2], packet[offset + 3]])); + let value_start = offset + 4; + let value_end = value_start.checked_add(length)?; + if value_end > packet.len() { + return None; + } + if attribute_type == wanted_type { + return Some((offset, &packet[value_start..value_end])); + } + offset = value_end.next_multiple_of(4); + } + None +} + +fn valid_stun_fingerprint(packet: &[u8]) -> bool { + let Some((offset, value)) = find_stun_attribute(packet, STUN_ATTR_FINGERPRINT) else { + return false; + }; + if value.len() != 4 || offset + 8 != packet.len() { + return false; + } + let expected = crc32(&packet[..offset]) ^ STUN_FINGERPRINT_XOR; + expected.to_be_bytes().ct_eq(value).into() +} + +fn valid_stun_message_integrity(packet: &[u8], key: &[u8]) -> bool { + let Some((integrity_offset, integrity)) = + find_stun_attribute(packet, STUN_ATTR_MESSAGE_INTEGRITY) + else { + return false; + }; + if integrity.len() != 20 { + return false; + } + let fingerprint_bytes = if find_stun_attribute(packet, STUN_ATTR_FINGERPRINT).is_some() { + 8 + } else { + 0 + }; + let Some(adjusted_length) = packet + .len() + .checked_sub(STUN_HEADER_LEN + fingerprint_bytes) + else { + return false; + }; + let mut authenticated = packet[..integrity_offset].to_vec(); + authenticated[2..4].copy_from_slice(&(adjusted_length as u16).to_be_bytes()); + let mut mac = HmacSha1::new_from_slice(key).expect("HMAC accepts variable-size ICE passwords"); + mac.update(&authenticated); + mac.finalize().into_bytes().ct_eq(integrity).into() +} + +fn handle_stun_datagram( + datagram: &[u8], + source: SocketAddr, + credentials: &NvstStunCredentials, +) -> StunDatagram { + if datagram.len() < STUN_HEADER_LEN + || datagram[0] & 0xc0 != 0 + || u32::from_be_bytes([datagram[4], datagram[5], datagram[6], datagram[7]]) + != STUN_MAGIC_COOKIE + { + return StunDatagram::NotStun; + } + let message_length = usize::from(u16::from_be_bytes([datagram[2], datagram[3]])); + if STUN_HEADER_LEN + message_length != datagram.len() || !valid_stun_fingerprint(datagram) { + return StunDatagram::Invalid; + } + let message_type = u16::from_be_bytes([datagram[0], datagram[1]]); + let transaction_id: [u8; 12] = datagram[8..20] + .try_into() + .expect("STUN header length checked above"); + match message_type { + STUN_BINDING_REQUEST => { + let Some((_, username)) = find_stun_attribute(datagram, STUN_ATTR_USERNAME) else { + return StunDatagram::Invalid; + }; + let expected_username = format!( + "{}:{}", + credentials.local_username_fragment, credentials.remote_username_fragment + ); + if !bool::from(expected_username.as_bytes().ct_eq(username)) + || !valid_stun_message_integrity(datagram, credentials.local_password.as_bytes()) + { + return StunDatagram::Invalid; + } + let mapped_address = xor_mapped_address(source, &transaction_id); + StunDatagram::Handled(Some(build_authenticated_stun_packet( + STUN_BINDING_SUCCESS_RESPONSE, + &transaction_id, + credentials.local_password.as_bytes(), + &[(STUN_ATTR_XOR_MAPPED_ADDRESS, mapped_address)], + ))) + } + STUN_BINDING_SUCCESS_RESPONSE => { + if valid_stun_message_integrity(datagram, credentials.remote_password.as_bytes()) { + StunDatagram::Handled(None) + } else { + StunDatagram::Invalid + } + } + _ => StunDatagram::Invalid, + } +} + +enum UdpReceiverCommand { + SendText { + text: opennow_streamer_protocol::text_input::UnicodeText, + timestamp_us: u64, + }, + Pause, + Resume, + Recover, + SendInput { + bytes: Vec, + reply: Option>>, + }, + Stop, +} + +/// Owns the bounded UDP receive worker. Frames go through the same bounded `MediaConsumer` used +/// by WebRTC, so a slow decoder cannot make UDP receive unbounded. +pub struct NvstUdpReceiverSession { + commands: Sender, + join: Option>, + input_ready: Arc, + microphone: Arc>, +} + +#[derive(Clone)] +pub struct NvstUdpReceiverControl { + commands: Sender, + input_ready: Arc, + microphone: Arc>, +} + +#[derive(Debug, Error)] +pub enum NvstUdpReceiverError { + #[error("failed to bind NVST UDP socket: {0}")] + Bind(#[source] std::io::Error), + #[error("failed to configure NVST UDP socket: {0}")] + Configure(#[source] std::io::Error), + #[error("failed to start NVST receive worker: {0}")] + Spawn(#[source] std::io::Error), + #[error("NVST receive worker is no longer running")] + Closed, + #[error("failed to prepare NVST control/audio bundle: {0}")] + WebrtcBundle(String), + #[error("the server did not negotiate native bundled microphone audio")] + MicrophoneUnavailable, + #[error("microphone transmission is disabled")] + MicrophoneDisabled, + #[error("microphone Opus payload must contain 1 to 1275 bytes")] + InvalidMicrophoneFrame, +} + +impl NvstUdpReceiverSession { + pub fn control(&self) -> NvstUdpReceiverControl { + NvstUdpReceiverControl { + commands: self.commands.clone(), + input_ready: Arc::clone(&self.input_ready), + microphone: Arc::clone(&self.microphone), + } + } + + pub fn pause(&self) -> Result<(), NvstUdpReceiverError> { + self.control().pause() + } + + pub fn resume(&self) -> Result<(), NvstUdpReceiverError> { + self.control().resume() + } + + pub fn recover(&self) -> Result<(), NvstUdpReceiverError> { + self.control().recover() + } + + pub fn send_input( + &self, + bytes: Vec, + partially_reliable: bool, + ) -> Result<(), TransportError> { + self.control().send_input(bytes, partially_reliable) + } + + pub fn stop(mut self) { + if let Ok(mut queue) = self.microphone.lock() { + queue.close(); + } + let _ = self.commands.send(UdpReceiverCommand::Stop); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +impl NvstUdpReceiverControl { + pub fn queue_text( + &self, + text: opennow_streamer_protocol::text_input::UnicodeText, + timestamp_us: u64, + ) -> Result<(), TransportError> { + if !self.input_ready.load(Ordering::Acquire) { + return Err(TransportError::InputNotReady); + } + self.commands + .send(UdpReceiverCommand::SendText { text, timestamp_us }) + .map_err(|_| TransportError::Closed) + } + + pub fn set_microphone_enabled(&self, enabled: bool) -> Result<(), NvstUdpReceiverError> { + self.microphone + .lock() + .map_err(|_| NvstUdpReceiverError::Closed)? + .set_enabled(enabled) + } + + pub fn send_microphone_opus( + &self, + payload: Vec, + rtp_timestamp: u32, + ) -> Result<(), NvstUdpReceiverError> { + self.microphone + .lock() + .map_err(|_| NvstUdpReceiverError::Closed)? + .push(payload, rtp_timestamp, Instant::now()) + } + + pub fn microphone_dropped_frames(&self) -> u64 { + self.microphone.lock().map_or(0, |queue| queue.dropped()) + } + + pub fn pause(&self) -> Result<(), NvstUdpReceiverError> { + self.send(UdpReceiverCommand::Pause) + } + + pub fn resume(&self) -> Result<(), NvstUdpReceiverError> { + self.send(UdpReceiverCommand::Resume) + } + + pub fn recover(&self) -> Result<(), NvstUdpReceiverError> { + self.send(UdpReceiverCommand::Recover) + } + + pub fn send_input( + &self, + bytes: Vec, + _partially_reliable: bool, + ) -> Result<(), TransportError> { + if !self.input_ready.load(Ordering::Acquire) { + return Err(TransportError::InputNotReady); + } + let (reply, result) = mpsc::sync_channel(1); + self.commands + .send(UdpReceiverCommand::SendInput { + bytes, + reply: Some(reply), + }) + .map_err(|_| TransportError::Closed)?; + result + .recv_timeout(Duration::from_millis(500)) + .map_err(|_| TransportError::Closed)? + } + + /// Queues latency-sensitive locally captured input without waiting for the + /// receive worker to round-trip an acknowledgement. Readiness is still + /// checked before enqueueing and the ordered command channel preserves the + /// exact keyboard/button/motion sequence. + pub fn queue_input( + &self, + bytes: Vec, + _partially_reliable: bool, + ) -> Result<(), TransportError> { + if !self.input_ready.load(Ordering::Acquire) { + return Err(TransportError::InputNotReady); + } + self.commands + .send(UdpReceiverCommand::SendInput { bytes, reply: None }) + .map_err(|_| TransportError::Closed) + } + + pub fn stop(&self) -> Result<(), NvstUdpReceiverError> { + self.microphone + .lock() + .map_err(|_| NvstUdpReceiverError::Closed)? + .close(); + self.send(UdpReceiverCommand::Stop) + } + + fn send(&self, command: UdpReceiverCommand) -> Result<(), NvstUdpReceiverError> { + self.commands + .send(command) + .map_err(|_| NvstUdpReceiverError::Closed) + } +} + +impl Drop for NvstUdpReceiverSession { + fn drop(&mut self) { + if let Ok(mut queue) = self.microphone.lock() { + queue.close(); + } + let _ = self.commands.send(UdpReceiverCommand::Stop); + } +} + +fn discover_routed_ipv4() -> Option { + let probe = UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)).ok()?; + probe + .connect(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 9)) + .ok()?; + let addr = probe.local_addr().ok()?; + if addr.ip().is_unspecified() || addr.ip().is_loopback() { + return None; + } + Some(addr.ip()) +} + +pub fn reserve_nvst_udp_socket() -> std::io::Result { + // Official binds the bundle sockets on 0.0.0.0 and advertises the routed + // NIC IPv4 separately via clientPorts.localAddress + host a=candidate. + bind_nvst_udp(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0) +} + +/// IPv4 OpenNOW should put in ANNOUNCE `localAddress` / host candidate. +/// Independent of the bind address: official listens on 0.0.0.0. +pub fn advertised_nvst_ipv4() -> Option { + discover_routed_ipv4() +} + +pub fn nvst_video_packet_size(peer: IpAddr) -> std::io::Result { + let (interface, route_mtu) = match mtu::interface_and_mtu(peer) { + Ok(route) => route, + Err(error) => { + opennow_streamer_protocol::log::log_line( + "WARN", + "nvst-udp", + &format!( + "video route lookup failed: {error}; VPN unverified, retaining packet_size={DEFAULT_NVST_VIDEO_PACKET_SIZE}" + ), + ); + return Ok(DEFAULT_NVST_VIDEO_PACKET_SIZE); + } + }; + let vpn_detected = is_vpn_interface(&interface); + let packet_size = video_packet_size_for_vpn_mtu(peer, vpn_detected.then_some(route_mtu))?; + opennow_streamer_protocol::log::log_line( + "INFO", + "nvst-udp", + &format!( + "video route peer={peer} interface_mtu={route_mtu} vpn_detected={vpn_detected} packet_size={packet_size}", + ), + ); + Ok(packet_size) +} + +fn wire_overhead_bytes(peer: IpAddr) -> usize { + let ip_header_bytes = if peer.is_ipv4() { 20 } else { 40 }; + ip_header_bytes + UDP_HEADER_BYTES + NVST_FEC_RTP_HEADER_ALLOWANCE + SRTP_AEAD_AES_GCM_TAG_LEN +} + +pub fn measured_video_packet_size(datagram_size: usize, peer: IpAddr) -> Option { + let ip_header_bytes = if peer.is_ipv4() { 20 } else { 40 }; + let route_mtu = datagram_size + .checked_add(UDP_HEADER_BYTES)? + .checked_add(ip_header_bytes)?; + video_packet_size_for_vpn_mtu(peer, Some(route_mtu)).ok() +} + +fn video_packet_size_for_vpn_mtu(peer: IpAddr, route_mtu: Option) -> std::io::Result { + let Some(route_mtu) = route_mtu else { + return Ok(DEFAULT_NVST_VIDEO_PACKET_SIZE); + }; + let packet_size = route_mtu + .saturating_sub(wire_overhead_bytes(peer)) + .min(DEFAULT_NVST_VIDEO_PACKET_SIZE) + / 16 + * 16; + if packet_size < MIN_NVST_VIDEO_PACKET_SIZE { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "video route MTU is too small for an NVST video packet", + )); + } + Ok(packet_size) +} + +/// ICE + DTLS identity that already owns the reserved bundle socket. +#[derive(Debug, Clone)] +pub struct NvstBundleIdentity { + pub ice_username_fragment: String, + pub ice_password: String, + pub dtls_fingerprint: String, +} + +/// UDP socket plus the `Rtc` that will speak ICE/DTLS on it, plus the dedicated +/// NATT-only video (Mjolnir) socket the official two-socket cloud model uses for +/// raw-SRTP video. +pub struct ReservedNvstBundle { + socket: UdpSocket, + rtc: Rtc, + mjolnir_socket: UdpSocket, +} + +impl ReservedNvstBundle { + pub fn reserve() -> Result { + // Bifrost's legacy dedicated-socket layout assigns video to N and the + // native ICE/DTLS bundle to N+1. The server still relies on this pairing + // even though ANNOUNCE advertises clientPorts.video=0 and routes video + // after the Mjolnir NATT ping. Reserving the bundle first reverses those + // ports and causes some seats to keep all video off the client entirely. + let (socket, mjolnir_socket) = + reserve_nvst_socket_pair().map_err(NvstUdpReceiverError::Bind)?; + let rtc = create_nvst_bundle_rtc(&socket)?; + Ok(Self { + socket, + rtc, + mjolnir_socket, + }) + } + + pub fn local_addr(&self) -> std::io::Result { + self.socket.local_addr() + } + + pub fn mjolnir_local_addr(&self) -> std::io::Result { + self.mjolnir_socket.local_addr() + } + + pub fn advertised_local_address(&self) -> Option { + match self.socket.local_addr().ok()?.ip() { + IpAddr::V4(ip) if !ip.is_unspecified() && !ip.is_loopback() => Some(ip.to_string()), + _ => advertised_nvst_ipv4().map(|ip| ip.to_string()), + } + } + + /// Use the route to the negotiated media peer, not a public DNS service. + /// Split-tunnel VPNs can send those destinations through different NICs. + pub fn advertised_local_address_for(&self, peer: SocketAddr) -> std::io::Result { + let local = self.socket.local_addr()?; + let probe = UdpSocket::bind(SocketAddr::new(local.ip(), 0))?; + probe.connect(peer)?; + let address = probe.local_addr()?.ip(); + if address.is_unspecified() || !address.is_ipv4() { + return Err(std::io::Error::new( + std::io::ErrorKind::AddrNotAvailable, + "no local IPv4 route to the NVST media peer", + )); + } + Ok(address.to_string()) + } + + pub fn identity(&mut self) -> NvstBundleIdentity { + nvst_local_bundle_identity(&mut self.rtc) + } + + pub fn send_to(&self, payload: &[u8], host: &str, port: u16) -> std::io::Result { + self.socket.send_to(payload, (host, port)) + } + + pub fn try_clone_socket(&self) -> std::io::Result { + self.socket.try_clone() + } + + pub fn into_parts(self) -> (UdpSocket, Rtc, UdpSocket) { + (self.socket, self.rtc, self.mjolnir_socket) + } +} + +fn generate_gfn_local_ice_credentials() -> Result { + const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"; + let mut random = [0_u8; 26]; + getrandom::fill(&mut random)?; + let encode = |start: usize, length: usize| { + random[start..start + length] + .iter() + .map(|value| ALPHABET[usize::from(*value) & 0x3f] as char) + .collect::() + }; + Ok(IceCreds { + ufrag: encode(0, 4), + pass: encode(4, 22), + }) +} + +fn create_nvst_bundle_rtc(socket: &UdpSocket) -> Result { + install_crypto(); + let _ = socket; + // Official GenerateIceCredentials() is 4-char ufrag / 22-char password. + // str0m's default 16-char ufrag is rejected by Bifrost length checks. + let mut rtc_config = RtcConfig::new() + .set_rtp_mode(true) + .set_stats_interval(Some(Duration::from_secs(1))) + .set_send_buffer_audio(MICROPHONE_QUEUE_CAPACITY); + rtc_config.codec_config().add_config( + GFN_RED_PAYLOAD_TYPE.into(), + None, + Codec::Opus, + Frequency::FORTY_EIGHT_KHZ, + Some(2), + FormatParams::default(), + ); + let mut rtc = rtc_config.build(Instant::now()); + let credentials = generate_gfn_local_ice_credentials() + .map_err(|error| NvstUdpReceiverError::WebrtcBundle(error.to_string()))?; + rtc.direct_api().set_local_ice_credentials(credentials); + Ok(rtc) +} + +fn nvst_local_bundle_identity(rtc: &mut Rtc) -> NvstBundleIdentity { + let creds = rtc.direct_api().local_ice_credentials(); + let fingerprint = rtc.direct_api().local_dtls_fingerprint().clone(); + NvstBundleIdentity { + ice_username_fragment: creds.ufrag, + ice_password: creds.pass, + dtls_fingerprint: nvst_fingerprint_hex(&fingerprint), + } +} + +fn nvst_fingerprint_hex(fingerprint: &Fingerprint) -> String { + fingerprint + .bytes + .iter() + .map(|byte| format!("{byte:02X}")) + .collect::>() + .join(":") +} + +fn routed_host_addr(peer: Option, local: SocketAddr) -> SocketAddr { + if !local.ip().is_unspecified() && !local.ip().is_loopback() { + return local; + } + if let Some(peer) = peer + && let Ok(probe) = UdpSocket::bind(SocketAddr::new( + if peer.is_ipv4() { + IpAddr::V4(Ipv4Addr::UNSPECIFIED) + } else { + IpAddr::V6(Ipv6Addr::UNSPECIFIED) + }, + 0, + )) + { + let _ = probe.connect(peer); + if let Ok(addr) = probe.local_addr() + && !addr.ip().is_unspecified() + && !addr.ip().is_loopback() + { + return SocketAddr::new(addr.ip(), local.port()); + } + } + local +} + +pub fn logical_ice_addr(addr: SocketAddr, loopback_octet: u8) -> SocketAddr { + match addr.ip() { + IpAddr::V4(ip) if ip.is_link_local() => SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, loopback_octet)), + addr.port(), + ), + _ => addr, + } +} + +fn parse_nvst_fingerprint(value: &str) -> Result { + let trimmed = value.trim(); + let sdp = if trimmed.contains(' ') { + trimmed.to_owned() + } else { + format!("sha-256 {trimmed}") + }; + sdp.parse() +} + +fn bind_nvst_udp(bind_ip: IpAddr, port: u16) -> std::io::Result { + #[cfg(unix)] + if let Ok(fd_text) = std::env::var("OPENNOW_NVST_VIDEO_UDP_FD") { + if let Ok(fd) = fd_text.parse::() { + use std::os::unix::io::FromRawFd; + // A legacy parent can dup the probe socket onto this fd so native never rebinds. + eprintln!("NVST inheriting video UDP socket from fd {fd}"); + return Ok(unsafe { UdpSocket::from_raw_fd(fd) }); + } + } + bind_nvst_udp_socket(bind_ip, port) +} + +fn bind_nvst_udp_socket(bind_ip: IpAddr, port: u16) -> std::io::Result { + let domain = if bind_ip.is_ipv4() { + Domain::IPV4 + } else { + Domain::IPV6 + }; + let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; + // Reservation is the availability check: never probe, close, then rebind. + // SO_REUSEADDR on Windows can steal an occupied UDP port and makes packet + // delivery indeterminate. Keep this exact exclusive socket through ANNOUNCE. + #[cfg(windows)] + set_exclusive_udp_address(&socket)?; + // ICMP port unreachable must not become WSAECONNRESET. Hole-punch pings + // run before the peer socket exists; that reset was exiting the receiver + // and closing the HID endpoint. + #[cfg(windows)] + if let Err(error) = disable_udp_connreset(&socket) { + log_udp_error("disable-connreset", port, &error); + } + if let Err(error) = socket.set_recv_buffer_size(NVST_UDP_RECEIVE_BUFFER_BYTES) { + log_udp_error("receive-buffer", port, &error); + eprintln!( + "NVST could not enlarge UDP receive buffer to {NVST_UDP_RECEIVE_BUFFER_BYTES} bytes: {error}" + ); + } + if let Err(error) = socket.bind(&SocketAddr::new(bind_ip, port).into()) { + log_udp_error("bind-unavailable", port, &error); + return Err(error); + } + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-udp", + &format!( + "bound local_port={} ipv4={} exclusive={} receive_buffer_bytes={} reachability=unverified", + socket + .local_addr()? + .as_socket() + .map_or(0, |addr| addr.port()), + bind_ip.is_ipv4(), + cfg!(windows), + socket.recv_buffer_size().unwrap_or(0) + ), + ); + Ok(socket.into()) +} + +#[cfg(windows)] +fn set_exclusive_udp_address(socket: &Socket) -> std::io::Result<()> { + use std::os::windows::io::AsRawSocket; + use windows_sys::Win32::Networking::WinSock::{ + SO_EXCLUSIVEADDRUSE, SOCKET_ERROR, SOL_SOCKET, WSAGetLastError, setsockopt, + }; + let enabled: u32 = 1; + // SAFETY: socket is live; enabled is a valid DWORD for the duration of the + // synchronous setsockopt call. This runs before bind and before publication. + let result = unsafe { + setsockopt( + socket.as_raw_socket() as _, + SOL_SOCKET, + SO_EXCLUSIVEADDRUSE, + (&enabled as *const u32).cast(), + std::mem::size_of_val(&enabled) as i32, + ) + }; + if result == SOCKET_ERROR { + // SAFETY: read the calling thread's Winsock error immediately on failure. + return Err(std::io::Error::from_raw_os_error(unsafe { + WSAGetLastError() + })); + } + Ok(()) +} + +#[cfg(windows)] +pub(crate) fn disable_udp_connreset(socket: &Socket) -> std::io::Result<()> { + use std::os::windows::io::AsRawSocket; + use windows_sys::Win32::Networking::WinSock::{ + SIO_UDP_CONNRESET, SOCKET_ERROR, WSAGetLastError, WSAIoctl, + }; + + let disabled: windows_sys::core::BOOL = 0; + let mut bytes_returned = 0_u32; + // SAFETY: socket is live and `disabled` outlives this synchronous ioctl. + // FALSE restores the pre-Windows 2000 behavior: ICMP port unreachable is + // discarded instead of failing the next recv/send with WSAECONNRESET. + let result = unsafe { + WSAIoctl( + socket.as_raw_socket() as _, + SIO_UDP_CONNRESET, + (&disabled as *const windows_sys::core::BOOL).cast(), + std::mem::size_of_val(&disabled) as u32, + std::ptr::null_mut(), + 0, + &mut bytes_returned, + std::ptr::null_mut(), + None, + ) + }; + if result == SOCKET_ERROR { + // SAFETY: read the calling thread's Winsock error immediately on failure. + return Err(std::io::Error::from_raw_os_error(unsafe { + WSAGetLastError() + })); + } + Ok(()) +} + +fn idle_ms(last: Option, now: Instant) -> String { + last.map_or_else( + || "none".to_owned(), + |last| now.saturating_duration_since(last).as_millis().to_string(), + ) +} + +/// Failure-time bundle state: when each direction last moved, so a report can tell whether +/// ingress stopped at the socket, the SCTP association, or downstream of both. +#[derive(Debug, Default)] +struct BundleLiveness { + last_inbound: Option, + last_peer_inbound: Option, + last_outbound: Option, + last_channel_data: Option, + ice_state: Option, + port_unreachable: u64, + channels_closed: u64, +} + +impl BundleLiveness { + fn snapshot(&self, now: Instant) -> String { + format!( + "rx_idle_ms={} peer_rx_idle_ms={} tx_idle_ms={} sctp_rx_idle_ms={} ice_state={} port_unreachable={} channels_closed={}", + idle_ms(self.last_inbound, now), + idle_ms(self.last_peer_inbound, now), + idle_ms(self.last_outbound, now), + idle_ms(self.last_channel_data, now), + self.ice_state + .map_or_else(|| "none".to_owned(), |state| format!("{state:?}")), + self.port_unreachable, + self.channels_closed, + ) + } +} + +fn log_udp_error(operation: &str, local_port: u16, error: &std::io::Error) { + // Structured OS codes, never endpoints, payloads, or ICE credentials. + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-udp", + &format!( + "operation={operation} local_port={local_port} kind={:?} os_code={:?}", + error.kind(), + error.raw_os_error() + ), + ); +} + +/// Windows reports ICMP port unreachable for an earlier UDP send as +/// `WSAECONNRESET` (`ErrorKind::ConnectionReset`). A hole-punch ping toward a +/// peer that is not listening yet is expected. Stopping the receiver on that +/// error exits the thread and closes the HID endpoint. +fn udp_icmp_port_unreachable(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::ConnectionReset +} + +fn udp_receive_is_idle(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::Interrupted + | std::io::ErrorKind::ConnectionReset + ) +} + +#[derive(Default)] +struct VideoKeepaliveState { + pings_sent: u64, + ping_tracker: StreamPingTracker, + first_failure: Option, + failed_attempts: u64, + deferred_total: u64, +} + +impl VideoKeepaliveState { + fn record_send( + &mut self, + result: std::io::Result, + transaction_id: Option<[u8; 12]>, + now: Instant, + budget: Duration, + operation: &'static str, + local_port: u16, + ) -> std::io::Result<()> { + const WSAENOBUFS: i32 = 10055; + match result { + Ok(_) => { + if let Some(first_failure) = self.first_failure.take() { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-udp", + &format!( + "operation={operation} local_port={local_port} keepalive_send=recovered failed_attempts={} failure_ms={}", + self.failed_attempts, + now.saturating_duration_since(first_failure).as_millis(), + ), + ); + } + self.failed_attempts = 0; + self.pings_sent += 1; + if let Some(transaction_id) = transaction_id { + self.ping_tracker.sent(transaction_id, now); + } + Ok(()) + } + Err(error) => { + if let Some(first_failure) = self.first_failure + && now.saturating_duration_since(first_failure) >= budget + { + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-udp", + &format!( + "operation={operation} local_port={local_port} keepalive_send=exhausted failed_attempts={} failure_ms={} budget_ms={}", + self.failed_attempts.saturating_add(1), + now.saturating_duration_since(first_failure).as_millis(), + budget.as_millis(), + ), + ); + return Err(error); + } + if udp_icmp_port_unreachable(&error) { + if self.first_failure.is_some() { + self.failed_attempts = self.failed_attempts.saturating_add(1); + } + return Ok(()); + } + if error.raw_os_error() != Some(WSAENOBUFS) + && !matches!( + error.kind(), + std::io::ErrorKind::Interrupted | std::io::ErrorKind::WouldBlock + ) + { + return Err(error); + } + self.failed_attempts = self.failed_attempts.saturating_add(1); + self.first_failure.get_or_insert(now); + self.deferred_total = self.deferred_total.saturating_add(1); + if self.failed_attempts == 1 { + log_udp_error(operation, local_port, &error); + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-udp", + &format!( + "operation={operation} local_port={local_port} keepalive_send=deferred budget_ms={} retry=next-scheduled-keepalive", + budget.as_millis(), + ), + ); + } + Ok(()) + } + } + } +} + +/// Reserves the dedicated NATT-only video (Mjolnir) socket. The native streamer +/// always owns this socket outright, so it never inherits a parent-owned fd. +pub fn reserve_nvst_mjolnir_udp_socket() -> std::io::Result { + bind_nvst_udp_socket(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0) +} + +fn reserve_nvst_socket_pair() -> std::io::Result<(UdpSocket, UdpSocket)> { + reserve_nvst_socket_pair_from(49_005) +} + +fn reserve_nvst_socket_pair_from( + preferred_video_port: u16, +) -> std::io::Result<(UdpSocket, UdpSocket)> { + // Bifrost's Windows client reserves this exact pair first + // (`general.clientPorts.useReserved=1`) and only falls back to dynamic + // ports when it is unavailable. Some cloud seats do not route the + // dedicated Mjolnir video flow back to an arbitrary source port even + // though the version-6 NATT request is otherwise valid. Match the + // official preference while preserving a non-fatal fallback for users + // that already have either port occupied. + let preferred_bundle_port = preferred_video_port.checked_add(1).ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "NVST port pair overflows") + })?; + reserve_nvst_socket_pair_preferred(preferred_video_port, preferred_bundle_port) +} + +fn reserve_nvst_socket_pair_preferred( + video_port: u16, + bundle_port: u16, +) -> std::io::Result<(UdpSocket, UdpSocket)> { + const MAX_PAIR_ATTEMPTS: usize = 32; + let bind_ip = IpAddr::V4(Ipv4Addr::UNSPECIFIED); + let mut last_error = match bind_nvst_udp_socket(bind_ip, video_port) { + Ok(mjolnir) => match bind_nvst_udp_socket(bind_ip, bundle_port) { + Ok(bundle) => { + log_reserved_pair("preferred", &bundle, &mjolnir); + return Ok((bundle, mjolnir)); + } + Err(error) => error, + }, + Err(error) => error, + }; + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-udp", + "preferred-pair-unavailable falling_back=dynamic-adjacent", + ); + + opennow_streamer_protocol::log::log_async( + "INFO", + "transport", + &format!( + "NVST preferred UDP pair unavailable; reserving an exclusive dynamic pair: {last_error}" + ), + ); + + for _ in 0..MAX_PAIR_ATTEMPTS { + let mjolnir = reserve_nvst_mjolnir_udp_socket()?; + let video_port = mjolnir.local_addr()?.port(); + let Some(bundle_port) = video_port.checked_add(1) else { + continue; + }; + match bind_nvst_udp_socket(bind_ip, bundle_port) { + Ok(bundle) => { + log_reserved_pair("dynamic", &bundle, &mjolnir); + return Ok((bundle, mjolnir)); + } + Err(error) => last_error = error, + } + } + Err(last_error) +} + +fn log_reserved_pair(selection: &str, bundle: &UdpSocket, video: &UdpSocket) { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-udp", + &format!( + "reserved selection={selection} video_local_port={} bundle_local_port={} sockets_retained=true reachability=unverified", + video.local_addr().map_or(0, |addr| addr.port()), + bundle.local_addr().map_or(0, |addr| addr.port()) + ), + ); +} + +fn log_udp_receiver_start( + role: &str, + socket: &UdpSocket, + peer: SocketAddr, + config: &NvstVideoConfig, + ping_payload: &[u8], +) { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-udp", + &format!( + "receiver-start role={role} local_port={} peer_port={} ipv4={} ping_version={:?} ping_bytes={} authenticated_natt={} same_video_bundle_host={} reachability=awaiting-inbound", + socket.local_addr().map_or(0, |addr| addr.port()), + peer.port(), + peer.is_ipv4(), + config.ping_version, + ping_payload.len(), + config.stun_credentials.is_some(), + config.video_peer.ip() == config.bundle_peer().ip() + ), + ); +} + +fn log_udp_first_inbound( + role: &str, + local_port: u16, + expected_peer: bool, + source: SocketAddr, + bytes: usize, + origin: Instant, +) { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-udp", + &format!( + "first-inbound role={role} local_port={local_port} source_port={} expected_peer={expected_peer} bytes={bytes} elapsed_ms={} authentication=not-yet-checked", + source.port(), + origin.elapsed().as_millis() + ), + ); +} + +pub fn spawn_nvst_udp_receiver( + config: NvstVideoConfig, + media_consumer: MediaConsumer, + event_sender: Sender, + hid_runtime: Arc, +) -> Result { + spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + None, + None, + hid_runtime, + None, + ) +} + +pub fn spawn_nvst_udp_receiver_with_socket( + config: NvstVideoConfig, + media_consumer: MediaConsumer, + event_sender: Sender, + reserved_socket: Option, + reserved_rtc: Option, + hid_runtime: Arc, + upstream_ready: Option>, +) -> Result { + let bind_ip = match config.video_peer.ip() { + IpAddr::V4(_) => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + IpAddr::V6(_) => IpAddr::V6(Ipv6Addr::UNSPECIFIED), + }; + let socket = match reserved_socket { + Some(socket) => { + eprintln!( + "NVST using UDP socket reserved before ANNOUNCE ({})", + socket + .local_addr() + .map(|addr| addr.to_string()) + .unwrap_or_else(|_| "unknown".to_owned()) + ); + socket + } + None => { + bind_nvst_udp(bind_ip, config.client_udp_port).map_err(NvstUdpReceiverError::Bind)? + } + }; + socket + .set_read_timeout(Some(UDP_RECEIVE_POLL_INTERVAL)) + .map_err(NvstUdpReceiverError::Configure)?; + let rtc = if config.remote_dtls_fingerprint().is_some() { + let rtc = match reserved_rtc { + Some(rtc) => rtc, + None => create_nvst_bundle_rtc(&socket)?, + }; + Some(prepare_nvst_webrtc_bundle(&socket, &config, rtc)?) + } else { + None + }; + spawn_receiver_thread( + "opennow-nvst-video", + socket, + config, + media_consumer, + event_sender, + NvstWorkerSetup { + rtc, + hid_runtime: Some(hid_runtime), + upstream_ready, + }, + ) +} + +/// Spawns the raw-SRTP NATT video receiver on the reserved Mjolnir socket. +/// +/// Official GFN cloud (`nativeRtcOnBundlePort=1`) delivers video RTP/SRTP to this +/// dedicated NATT-only socket while the ICE/DTLS bundle socket carries +/// control/audio. The raw receiver's NATT keepalive pings are what route video +/// here, and it decrypts with the runtime encryptionKey sent in ANNOUNCE. +pub fn spawn_nvst_mjolnir_receiver( + socket: UdpSocket, + config: NvstVideoConfig, + media_consumer: MediaConsumer, + event_sender: Sender, +) -> Result { + opennow_streamer_protocol::log::diagnostic( + "INFO", + "transport", + &format!( + "NVST Mjolnir raw-SRTP video receiver arming on {} peer={}", + socket + .local_addr() + .map(|addr| addr.to_string()) + .unwrap_or_else(|_| "unknown".to_owned()), + config.video_peer, + ), + ); + spawn_receiver_thread( + "opennow-nvst-mjolnir", + socket, + config, + media_consumer, + event_sender, + NvstWorkerSetup { + rtc: None, + hid_runtime: None, + upstream_ready: None, + }, + ) +} + +struct NvstReceiverOutputs { + media_consumer: MediaConsumer, + event_sender: Sender, + input_ready: Arc, + microphone: Arc>, +} + +struct NvstWorkerSetup { + rtc: Option, + hid_runtime: Option>, + upstream_ready: Option>, +} + +fn spawn_receiver_thread( + name: &str, + socket: UdpSocket, + config: NvstVideoConfig, + media_consumer: MediaConsumer, + event_sender: Sender, + setup: NvstWorkerSetup, +) -> Result { + if setup.rtc.is_some() && setup.upstream_ready.is_some() { + config.feedback.wait_for_play(); + } + socket + .set_read_timeout(Some(UDP_RECEIVE_POLL_INTERVAL)) + .map_err(NvstUdpReceiverError::Configure)?; + let (commands, receiver) = mpsc::channel(); + let input_ready = Arc::new(AtomicBool::new(false)); + let worker_input_ready = input_ready.clone(); + let microphone = Arc::new(Mutex::new(MicrophoneQueue::new( + config.microphone_available() && setup.rtc.is_some(), + ))); + let worker_microphone = microphone.clone(); + let transport_origin = Instant::now(); + let endpoint = setup + .hid_runtime + .as_ref() + .map(|runtime| (Arc::clone(runtime), runtime.open_endpoint())); + let worker_endpoint = endpoint.clone(); + let join = thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let _endpoint_guard = + worker_endpoint.map(|(runtime, token)| EndpointGuard { runtime, token }); + run_nvst_udp_receiver( + socket, + config, + receiver, + NvstReceiverOutputs { + media_consumer, + event_sender, + input_ready: worker_input_ready.clone(), + microphone: worker_microphone.clone(), + }, + transport_origin, + setup, + ); + worker_input_ready.store(false, Ordering::Release); + if let Ok(mut queue) = worker_microphone.lock() { + queue.close(); + } + }) + .map_err(|error| { + if let Some((runtime, token)) = endpoint { + runtime.close_endpoint(token); + } + NvstUdpReceiverError::Spawn(error) + })?; + Ok(NvstUdpReceiverSession { + commands, + join: Some(join), + input_ready, + microphone, + }) +} + +fn prepare_nvst_webrtc_bundle( + socket: &UdpSocket, + config: &NvstVideoConfig, + mut rtc: Rtc, +) -> Result { + let fingerprint = config.remote_dtls_fingerprint().ok_or_else(|| { + NvstUdpReceiverError::WebrtcBundle("missing remote DTLS fingerprint".into()) + })?; + let remote_fingerprint = + parse_nvst_fingerprint(fingerprint).map_err(NvstUdpReceiverError::WebrtcBundle)?; + let credentials = config.stun_credentials.as_ref().ok_or_else(|| { + NvstUdpReceiverError::WebrtcBundle( + "DTLS bundle requires local and remote ICE credentials".into(), + ) + })?; + let local_addr = socket + .local_addr() + .map_err(NvstUdpReceiverError::Configure)?; + let bundle_peer = config.bundle_peer(); + let physical_local = routed_host_addr(Some(bundle_peer), local_addr); + let local_candidate = Candidate::host(logical_ice_addr(physical_local, 1), "udp") + .map_err(|error| NvstUdpReceiverError::WebrtcBundle(error.to_string()))?; + let remote_candidate = Candidate::host(logical_ice_addr(bundle_peer, 2), "udp") + .map_err(|error| NvstUdpReceiverError::WebrtcBundle(error.to_string()))?; + rtc.add_local_candidate(local_candidate); + rtc.add_remote_candidate(remote_candidate); + { + let mut api = rtc.direct_api(); + api.set_ice_controlling(true); + api.set_local_ice_credentials(IceCreds { + ufrag: credentials.local_username_fragment.clone(), + pass: credentials.local_password.clone(), + }); + api.set_remote_ice_credentials(IceCreds { + ufrag: credentials.remote_username_fragment.clone(), + pass: credentials.remote_password.clone(), + }); + api.set_remote_fingerprint(remote_fingerprint); + if let Some(audio) = config.audio_track() { + api.declare_media(Mid::from(audio.mid.as_str()), MediaKind::Audio); + } + if config.microphone_available() { + for payload_type in [ + Some(config.microphone_rtp.opus_payload_type), + config.microphone_rtp.red_payload_type, + ] + .into_iter() + .flatten() + { + api.register_payload_type( + payload_type.into(), + Codec::Opus, + Frequency::FORTY_EIGHT_KHZ, + Some(1), + FormatParams::default(), + ); + } + api.declare_media(Mid::from("2"), MediaKind::Audio); + api.declare_stream_tx(Ssrc::from(1), None, Mid::from("2"), None); + } + api.start_dtls(true) + .map_err(|error| NvstUdpReceiverError::WebrtcBundle(error.to_string()))?; + } + eprintln!( + "NVST control/audio bundle armed (local={}, peer={}, remoteFingerprintBytes={})", + physical_local, + bundle_peer, + fingerprint.len() + ); + Ok(rtc) +} + +fn looks_like_rtp(datagram: &[u8]) -> bool { + datagram.len() >= RTP_FIXED_HEADER_LEN + && datagram[0] >> 6 == 2 + && !looks_like_stun(datagram) + && !looks_like_dtls(datagram) + && !looks_like_rtcp(datagram) +} + +fn looks_like_rtcp(datagram: &[u8]) -> bool { + datagram.len() >= 4 && datagram[0] >> 6 == 2 && matches!(datagram[1], 192..=223) +} + +fn looks_like_stun(datagram: &[u8]) -> bool { + datagram.len() >= STUN_HEADER_LEN && datagram[4..8] == STUN_MAGIC_COOKIE.to_be_bytes() +} + +#[cfg_attr(not(test), allow(dead_code))] +fn synthesize_ice_binding_success( + transaction_id: &[u8; 12], + mapped: SocketAddr, + remote_password: &str, +) -> Vec { + build_authenticated_stun_packet( + STUN_BINDING_SUCCESS_RESPONSE, + transaction_id, + remote_password.as_bytes(), + &[( + STUN_ATTR_XOR_MAPPED_ADDRESS, + xor_mapped_address(mapped, transaction_id), + )], + ) +} + +fn looks_like_dtls(datagram: &[u8]) -> bool { + matches!(datagram.first().copied(), Some(20..=63)) +} + +fn peek_rtp_ssrc(datagram: &[u8]) -> Option { + looks_like_rtp(datagram) + .then(|| u32::from_be_bytes([datagram[8], datagram[9], datagram[10], datagram[11]])) +} + +fn peek_rtp_payload_type(datagram: &[u8]) -> Option { + looks_like_rtp(datagram).then(|| datagram[1] & 0x7f) +} + +fn matches_audio_track(track: &NvstAudioTrack, payload_type: u8, ssrc: u32) -> bool { + let matches_payload_type = payload_type == track.payload_type + || (track.payload_type == GFN_OPUS_PAYLOAD_TYPE && payload_type == GFN_RED_PAYLOAD_TYPE); + matches_payload_type && track.ssrc.is_none_or(|expected| expected == ssrc) +} + +#[derive(Default)] +struct NvstAudioStreams { + authenticated: VecDeque, + provisional: Option, +} + +impl NvstAudioStreams { + fn admit(&mut self, rtc: &mut Rtc, track: &NvstAudioTrack, datagram: &[u8]) -> bool { + let Some(ssrc) = peek_rtp_ssrc(datagram) else { + return false; + }; + if ssrc == 0 + || !peek_rtp_payload_type(datagram) + .is_some_and(|payload_type| matches_audio_track(track, payload_type, ssrc)) + { + return false; + } + self.finish_poll(rtc); + if !self.authenticated.contains(&ssrc) { + rtc.direct_api().expect_stream_rx( + Ssrc::from(ssrc), + None, + Mid::from(track.mid.as_str()), + None, + ); + self.provisional = Some(ssrc); + } + true + } + + fn authenticated(&mut self, rtc: &mut Rtc, ssrc: u32) { + if self.provisional != Some(ssrc) { + return; + } + self.provisional = None; + if self.authenticated.len() == 2 + && let Some(retired) = self.authenticated.pop_front() + { + rtc.direct_api().remove_stream_rx(Ssrc::from(retired)); + } + self.authenticated.push_back(ssrc); + } + + fn finish_poll(&mut self, rtc: &mut Rtc) { + if let Some(rejected) = self.provisional.take() { + rtc.direct_api().remove_stream_rx(Ssrc::from(rejected)); + } + } +} + +#[derive(Default)] +struct NvstAudioReceiver { + last_sequence: Option<(u32, u16)>, + discontinuity_pending: bool, +} + +impl NvstAudioReceiver { + fn depacketize( + &mut self, + track: &NvstAudioTrack, + header: &BundleRtpHeader, + payload: Arc<[u8]>, + received_at_us: u64, + ) -> Result, NvstDropReason> { + let mut source_changed = false; + let missing_packets = match self.last_sequence { + Some((ssrc, previous)) if ssrc == *header.ssrc => { + let delta = header.sequence_number.wrapping_sub(previous); + if delta == 0 || delta >= 0x8000 { + return Ok(Vec::new()); + } + usize::from(delta - 1) + } + Some(_) => { + source_changed = true; + 0 + } + None => 0, + }; + let mut frames = Vec::with_capacity(missing_packets.min(MAX_REDUNDANT_AUDIO_BLOCKS) + 1); + let mut append = |payload: Arc<[u8]>, timestamp: u32| { + frames.push(EncodedMediaFrame { + mid: track.mid.clone(), + codec: "opus".to_owned(), + payload, + frame_index: None, + rtp_timestamp: u64::from(timestamp), + clock_rate_hz: track.clock_rate_hz, + channels: Some(track.channels), + received_at_us, + keyframe: false, + contiguous: true, + ssrc: Some(*header.ssrc), + }); + }; + let recovered_packets = if *header.payload_type == GFN_RED_PAYLOAD_TYPE { + let red = parse_red_opus(&payload).ok_or(NvstDropReason::MalformedRedAudio)?; + let recover_count = missing_packets.min(red.redundant.len()); + for block in red + .redundant + .iter() + .skip(red.redundant.len() - recover_count) + { + append( + Arc::from(block.payload), + header + .timestamp + .wrapping_sub(u32::from(block.timestamp_offset)), + ); + } + append(Arc::from(red.primary), header.timestamp); + recover_count + } else { + append(payload, header.timestamp); + 0 + }; + frames[0].contiguous = !source_changed && missing_packets == recovered_packets; + self.last_sequence = Some((*header.ssrc, header.sequence_number)); + Ok(frames) + } + + fn deliver( + &mut self, + consumer: &MediaConsumer, + mut frame: EncodedMediaFrame, + ) -> Result<(), TransportError> { + frame.contiguous &= !self.discontinuity_pending; + let result = deliver_media_frame(consumer, frame); + self.discontinuity_pending = result.is_err(); + result + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RedOpusBlock<'a> { + timestamp_offset: u16, + payload: &'a [u8], +} + +#[derive(Debug, PartialEq, Eq)] +struct RedOpusPayload<'a> { + redundant: Vec>, + primary: &'a [u8], +} + +fn parse_red_opus(payload: &[u8]) -> Option> { + let mut header_offset = 0_usize; + let mut headers = Vec::new(); + loop { + let header = *payload.get(header_offset)?; + let payload_type = header & 0x7f; + if header & 0x80 == 0 { + if payload_type != GFN_OPUS_PAYLOAD_TYPE { + return None; + } + let mut block_offset = header_offset.checked_add(1)?; + let mut redundant = Vec::with_capacity(headers.len()); + for (timestamp_offset, block_len) in headers { + let block_end = block_offset.checked_add(block_len)?; + let block = payload.get(block_offset..block_end)?; + if block.is_empty() || block.len() > MAX_OPUS_PACKET_BYTES { + return None; + } + redundant.push(RedOpusBlock { + timestamp_offset, + payload: block, + }); + block_offset = block_end; + } + let primary = payload.get(block_offset..)?; + return (!primary.is_empty() && primary.len() <= MAX_OPUS_PACKET_BYTES) + .then_some(RedOpusPayload { redundant, primary }); + } + if payload_type != GFN_OPUS_PAYLOAD_TYPE || headers.len() == MAX_REDUNDANT_AUDIO_BLOCKS { + return None; + } + let header_bytes = payload.get(header_offset..header_offset.checked_add(4)?)?; + let timestamp_offset = + (u16::from(header_bytes[1]) << 6) | (u16::from(header_bytes[2]) >> 2); + let block_len = (usize::from(header_bytes[2] & 0x03) << 8) | usize::from(header_bytes[3]); + headers.push((timestamp_offset, block_len)); + header_offset = header_offset.checked_add(4)?; + } +} + +#[cfg(test)] +fn extract_red_opus_primary(payload: &[u8]) -> Option<&[u8]> { + parse_red_opus(payload).map(|red| red.primary) +} + +fn finish_nvst_input_handshake( + input_state: &mut NvstInputChannelState, + channels: NvstInputChannels, + rtc: &mut Rtc, + transport_origin: Instant, + input_ready: &AtomicBool, + event_sender: &Sender, + version: u16, +) -> bool { + if !input_state.activation_sent() { + let timestamp_us = transport_origin + .elapsed() + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + if !channels.send_activation(rtc, timestamp_us) { + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "input activation could not be queued", + )); + return false; + } + eprintln!( + "NVST cursor capture tx: channel={} command=0x0308 enabled=true reason=input-activation", + channels.label(channels.control_reliable), + ); + eprintln!( + "NVST remote cursor tracking tx: channel={} command=0x030d enabled=true reason=input-activation", + channels.label(channels.control_reliable), + ); + eprintln!( + "NVST client state tx: channel={} window_state=19 system_state=0 frame=0", + channels.label(channels.control_reliable), + ); + input_state.mark_activation_sent(); + } + input_ready.store(true, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputReady(version)); + true +} + +fn classify_nvst_unready_input( + rtc: &mut Rtc, + channels: Option, + message: &NvstEncodedInput, +) -> NvstInputWrite { + if !rtc.is_alive() + || channels.is_some_and(|channels| !channels.encoded_channel_is_open(rtc, message)) + { + NvstInputWrite::Closed + } else { + NvstInputWrite::Deferred + } +} + +#[allow(clippy::too_many_arguments)] +fn send_nvst_captured_input( + codec: &mut NvstInputCodec, + delivery: &mut NvstInputDelivery, + bytes: &[u8], + timestamp_us: u64, + reply: Option>>, + input_ready: &AtomicBool, + event_sender: &Sender, + idle_warning_active: &AtomicBool, + mut send: impl FnMut(&NvstEncodedInput) -> NvstInputWrite, +) -> bool { + let result = delivery.submit(codec, bytes, timestamp_us, |message| { + let outcome = send(message); + report_nvst_user_input(message, outcome, idle_warning_active, event_sender); + outcome + }); + if let Err(error) = &result { + match error { + NvstInputDeliveryError::Encoding(error) => { + eprintln!("NVST input packet rejected: {error}") + } + NvstInputDeliveryError::Exhausted => eprintln!("NVST pending input queue exhausted"), + NvstInputDeliveryError::Closed => eprintln!("NVST input channel closed"), + } + } + let failed = result.is_err(); + if let Some(reply) = reply { + let _ = reply.send(result.map_err(|_| TransportError::Closed)); + } + if !failed { + return true; + } + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "discrete input delivery failed; stopping to prevent stuck input", + )); + false +} + +fn report_nvst_user_input( + message: &NvstEncodedInput, + outcome: NvstInputWrite, + idle_warning_active: &AtomicBool, + event_sender: &Sender, +) { + if outcome.is_sent() + && message.is_meaningful() + && idle_warning_active.swap(false, Ordering::AcqRel) + { + let _ = event_sender.send(NvstReceiveEvent::UserInput); + } +} + +struct EndpointGuard { + runtime: Arc, + token: EndpointToken, +} + +impl Drop for EndpointGuard { + fn drop(&mut self) { + self.runtime.close_endpoint(self.token); + } +} + +const SONY_SHUTDOWN_FLUSH_BUDGET: Duration = Duration::from_millis(50); + +fn queue_hid_shutdown( + hid_session: &mut HidSession, + channels: NvstInputChannels, + rtc: &mut Rtc, +) -> bool { + let mut queued = true; + for outbound in hid_session.retire_all() { + let command = match outbound { + HidOutbound::Release { low_id, bytes } => sony_report_command(low_id, &bytes), + HidOutbound::Removal { low_id } => { + sony_device_change_command(SonyDeviceControl::Removal, low_id) + } + HidOutbound::Attach { .. } | HidOutbound::Report { .. } => None, + }; + if let Some(command) = command { + queued &= channels.send_control_teardown(rtc, &command); + } + } + queued +} + +fn send_generic_hid_actions( + session: &mut GenericSession, + actions: Vec, + mut send: impl FnMut(&[u8], bool) -> bool, +) -> bool { + for action in actions { + let teardown = matches!( + action, + GenericOutbound::Detach { .. } | GenericOutbound::Report { release: true, .. } + ); + let Some(command) = generic_hid_command(&action) else { + return false; + }; + if !send(&command, teardown) { + return false; + } + session.sent(&action); + } + true +} + +fn pump_generic_hid_pending( + runtime: &GenericRuntime, + session: &mut GenericSession, + release_epoch: &mut u64, + mut send: impl FnMut(&[u8], bool) -> bool, +) -> bool { + let (capture, current_epoch) = runtime.capture_state(); + if *release_epoch != current_epoch { + let releases = session.set_capture(false); + if !send_generic_hid_actions(session, releases, &mut send) { + return false; + } + *release_epoch = current_epoch; + } + let (inventory, _) = runtime.inventory_snapshot(); + let actions = session.reconcile(&inventory, capture); + if !send_generic_hid_actions(session, actions, &mut send) { + return false; + } + for item in runtime.drain(32) { + let actions = match item { + GenericIngress::Report(report) => session.report(report).into_iter().collect(), + GenericIngress::Fault { slot, incarnation } => session.fault(slot, incarnation), + GenericIngress::Release { .. } => Vec::new(), + }; + if !send_generic_hid_actions(session, actions, &mut send) { + return false; + } + } + runtime.publish_admissions(session); + true +} + +fn store_generic_hid_message( + session: &mut GenericSession, + runtime: &GenericRuntime, + message: GenericHidMessage<'_>, + send: impl FnMut(&[u8], bool) -> bool, +) -> bool { + match message { + GenericHidMessage::Response { + request_id, + control, + opaque, + device_id, + } => { + session.handle_response(request_id, control, opaque, device_id); + runtime.publish_admissions(session); + } + GenericHidMessage::Output { + device_id, + interface, + operation, + bytes, + } => { + if let Some(output) = session.output(device_id, interface, operation, bytes) { + let (slot, incarnation) = (output.slot, output.incarnation); + match runtime.push_output(output) { + Ok(()) + | Err( + GenericError::Inactive | GenericError::Unbound | GenericError::StaleSource, + ) => {} + Err(_) => { + let actions = session.fault(slot, incarnation); + runtime.publish_admissions(session); + return send_generic_hid_actions(session, actions, send); + } + } + } + } + } + true +} + +#[cfg(test)] +mod generic_hid_tests { + use super::*; + use opennow_streamer_hid::generic::{GenericDevice, GenericInterface, GenericReport}; + + fn device(incarnation: u64) -> GenericDevice { + GenericDevice { + slot: 0, + incarnation, + vendor: 0x044f, + product: 0x0402, + revision: 0x0100, + interfaces: vec![GenericInterface { + number: 2, + descriptor: vec![ + 0x05, 0x01, 0x09, 0x04, 0xa1, 0x01, 0x05, 0x09, 0x19, 0x01, 0x29, 0x08, 0x15, + 0x00, 0x25, 0x01, 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, 0x05, 0x01, 0x09, 0x30, + 0x26, 0xff, 0x00, 0x75, 0x08, 0x95, 0x01, 0x81, 0x02, 0xc0, + ], + }], + } + } + + fn report(axis: u8) -> GenericReport { + GenericReport { + slot: 0, + incarnation: 1, + interface: 2, + operation: 0, + error: false, + bytes: vec![0xff, axis], + } + } + + fn runtime() -> GenericRuntime { + let runtime = GenericRuntime::new(); + runtime.replace_inventory(vec![device(1)]).unwrap(); + runtime.bind_session(1); + runtime.set_capture(true); + runtime + } + + fn admit(runtime: &GenericRuntime, session: &mut GenericSession) { + assert!(pump_generic_hid_pending( + runtime, + session, + &mut 0, + |_, teardown| { + assert!(!teardown); + true + } + )); + session.handle_response(10, 1, 0, 42); + assert!(session.output(42, 2, 0, &[1]).is_some()); + } + + #[test] + fn generic_admission_publication_tracks_server_response_and_pause_detach() { + use opennow_streamer_hid::generic::GenericAdmission; + for control in [0, 1, 2, 255] { + let runtime = runtime(); + let mut source = device(1); + source.interfaces[0].descriptor.clear(); + runtime.replace_inventory(vec![source]).unwrap(); + let mut session = GenericSession::new(1); + let mut release_epoch = 0; + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut release_epoch, + |_, _| true + )); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Pending + ); + assert!(store_generic_hid_message( + &mut session, + &runtime, + GenericHidMessage::Response { + request_id: 10, + control, + opaque: u32::MAX, + device_id: 42, + }, + |_, _| true + )); + let expected = if control == 1 { + GenericAdmission::Admitted + } else { + GenericAdmission::Declined + }; + assert_eq!(runtime.admissions_snapshot()[0].admission, expected); + if control != 1 { + continue; + } + runtime.set_capture(false); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut release_epoch, + |_, teardown| teardown + )); + let status = runtime.admissions_snapshot()[0]; + assert_eq!(status.incarnation, 1); + assert_eq!(status.admission, GenericAdmission::Pending); + runtime.set_capture(true); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut release_epoch, + |_, teardown| !teardown + )); + assert_eq!( + runtime.admissions_snapshot()[0].admission, + GenericAdmission::Pending + ); + } + } + + #[test] + fn generic_registration_accepts_only_pending_sent_control_one_and_rejects_stale_ack() { + let runtime = runtime(); + let mut session = GenericSession::new(1); + assert!(!pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| false + )); + session.handle_response(10, 1, 0, 42); + assert!(session.output(42, 2, 0, &[1]).is_none()); + for (control, opaque, returned) in [(3, 0, 42), (1, 0, 255), (1, 0, 9)] { + let mut session = GenericSession::new(1); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + session.handle_response(99, 1, 0, 42); + assert!(session.output(42, 2, 0, &[1]).is_none()); + session.handle_response(10, control, opaque, returned); + assert!(session.output(returned, 2, 0, &[1]).is_none()); + } + let mut session = GenericSession::new(1); + admit(&runtime, &mut session); + let actions = session.retire_all(); + assert!(send_generic_hid_actions( + &mut session, + actions, + |_, teardown| teardown + )); + runtime.replace_inventory(vec![device(2)]).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + session.handle_response(10, 1, 0, 43); + assert!(session.output(43, 2, 0, &[1]).is_none()); + session.handle_response(11, 1, 0xf, 44); + assert_eq!(session.output(44, 2, 0, &[1]).unwrap().incarnation, 2); + } + + #[test] + fn generic_release_uses_last_successful_send_and_precedes_collapsed_capture_reports() { + let runtime = runtime(); + let mut session = GenericSession::new(1); + admit(&runtime, &mut session); + runtime.submit_report(report(0xa5)).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + runtime.submit_report(report(0x55)).unwrap(); + assert!(!pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| false + )); + runtime.set_capture(false); + runtime.set_capture(true); + runtime.submit_report(report(0x66)).unwrap(); + let mut sent = vec![]; + let mut epoch = 0; + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut epoch, + |command, teardown| { + sent.push((command.to_vec(), teardown)); + true + } + )); + assert_eq!(epoch, 1); + assert_eq!(sent.len(), 2); + assert_eq!(&sent[0].0[12..], &[0, 0xa5]); + assert!(sent[0].1); + assert_eq!(&sent[1].0[12..], &[0xff, 0x66]); + assert!(!sent[1].1); + runtime.set_capture(false); + assert!(!pump_generic_hid_pending( + &runtime, + &mut session, + &mut epoch, + |_, _| false + )); + assert_eq!(epoch, 1); + } + + #[test] + fn generic_wire_response_and_empty_get_error_round_trip_without_sony_haptics() { + let runtime = runtime(); + let mut session = GenericSession::new(1); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + let response = [0x06, 0x02, 0x0b, 0, 0x1a, 0, 0, 0, 10, 1, 0, 0, 0, 0xf, 42]; + for_each_generic_hid_message(&response, |message| { + assert!(store_generic_hid_message( + &mut session, + &runtime, + message, + |_, _| true + )); + }); + assert!(session.output(42, 2, 0, &[1]).is_some()); + let get = [0x06, 0x02, 0x0a, 0, 0x11, 0, 0, 0, 42, 2, 3, 1, 0, 1]; + for_each_generic_hid_message(&get, |message| { + assert!(store_generic_hid_message( + &mut session, + &runtime, + message, + |_, _| true + )); + }); + let output = runtime.take_output().unwrap(); + assert_eq!(output.bytes, [1, 0, 1]); + runtime + .submit_report(GenericReport { + slot: output.slot, + incarnation: output.incarnation, + interface: output.interface, + operation: output.operation, + error: true, + bytes: vec![], + }) + .unwrap(); + let mut sent = vec![]; + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |command, teardown| { + assert!(!teardown); + sent.push(command.to_vec()); + true + } + )); + assert_eq!( + sent, + vec![vec![0x0d, 0x02, 8, 0, 0x22, 0x11, 0, 0, 0, 42, 1, 3]] + ); + runtime.submit_report(report(0xa5)).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + runtime.set_capture(false); + sent.clear(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |command, teardown| { + assert!(teardown); + sent.push(command.to_vec()); + true + } + )); + assert_eq!(sent.len(), 1); + assert_eq!(&sent[0][12..], &[0, 0xa5]); + } + + #[test] + fn generic_drain_is_bounded_and_fault_or_inventory_removal_releases_before_detach() { + let runtime = runtime(); + let mut session = GenericSession::new(1); + admit(&runtime, &mut session); + for _ in 0..40 { + runtime.submit_report(report(0x77)).unwrap(); + } + let mut count = 0; + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| { + count += 1; + true + } + )); + assert_eq!(count, 32); + assert_eq!(runtime.drain(32).len(), 8); + for _ in 0..64 { + runtime.submit_report(report(0x77)).unwrap(); + } + assert_eq!( + runtime.submit_report(report(0x77)), + Err(GenericError::Overflow) + ); + let mut sent = vec![]; + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |command, teardown| { + sent.push(command.to_vec()); + assert!(teardown); + true + } + )); + assert_eq!(sent.len(), 2); + assert_eq!(&sent[0][12..], &[0, 0x77]); + assert_eq!(&sent[1][12..14], &[42, 3]); + assert!(session.output(42, 2, 0, &[1]).is_none()); + runtime.replace_inventory(vec![device(2)]).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + session.handle_response(11, 1, 0, 43); + runtime.replace_inventory(vec![]).unwrap(); + sent.clear(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |command, teardown| { + sent.push(command.to_vec()); + assert!(teardown); + true + } + )); + assert_eq!(sent.len(), 1); + assert_eq!(&sent[0][12..14], &[43, 3]); + } + + #[test] + fn generic_outputs_are_typed_and_overflow_retires_source_or_reports_queue_failure() { + for queues_release in [true, false] { + let runtime = runtime(); + let mut session = GenericSession::new(1); + admit(&runtime, &mut session); + runtime.submit_report(report(0xa5)).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + for _ in 0..32 { + assert!(store_generic_hid_message( + &mut session, + &runtime, + GenericHidMessage::Output { + device_id: 42, + interface: 2, + operation: 3, + bytes: &[1, 0, 1], + }, + |_, _| panic!("valid output should not send transport response") + )); + } + let mut sent = vec![]; + assert_eq!( + store_generic_hid_message( + &mut session, + &runtime, + GenericHidMessage::Output { + device_id: 42, + interface: 2, + operation: 3, + bytes: &[1, 0, 1], + }, + |command, teardown| { + sent.push(command.to_vec()); + assert!(teardown); + queues_release + } + ), + queues_release + ); + assert!(session.output(42, 2, 0, &[1]).is_none()); + assert_eq!(sent.len(), if queues_release { 2 } else { 1 }); + let output = runtime.take_output().unwrap(); + assert_eq!( + ( + output.slot, + output.incarnation, + output.interface, + output.operation + ), + (0, 1, 2, 3) + ); + assert_eq!(output.bytes, [1, 0, 1]); + } + } + + #[test] + fn generic_shutdown_queues_release_and_detach_and_invalid_descriptor_uses_detach() { + for valid_descriptor in [true, false] { + let runtime = runtime(); + if !valid_descriptor { + let mut entry = device(1); + entry.interfaces[0].descriptor.clear(); + runtime.replace_inventory(vec![entry]).unwrap(); + } + let mut session = GenericSession::new(1); + admit(&runtime, &mut session); + runtime.submit_report(report(0xa5)).unwrap(); + assert!(pump_generic_hid_pending( + &runtime, + &mut session, + &mut 0, + |_, _| true + )); + let mut sent = vec![]; + let actions = session.retire_all(); + assert!(send_generic_hid_actions( + &mut session, + actions, + |command, teardown| { + sent.push(command.to_vec()); + assert!(teardown); + true + } + )); + assert_eq!(sent.len(), if valid_descriptor { 2 } else { 1 }); + assert_eq!(&sent.last().unwrap()[12..14], &[42, 3]); + runtime.unbind_session(1); + assert!(runtime.take_output().is_none()); + assert!(session.retire_all().is_empty()); + } + } +} + +fn forward_rtc_outputs(rtc: &mut Rtc, mut transmit: F, budget: Duration) { + let deadline = Instant::now() + budget; + while Instant::now() < deadline { + match rtc.poll_output() { + Ok(output @ Output::Transmit(_)) => transmit(&output), + Ok(_) | Err(_) => break, + } + } +} + +fn pump_hid_pending( + hid_runtime: &HidRuntime, + hid_session: &mut HidSession, + input_codec: &mut NvstInputCodec, + channels: NvstInputChannels, + rtc: &mut Rtc, + published_mask: &mut u16, + timestamp_us: u64, +) -> bool { + if hid_runtime.session_generation().is_none() { + return true; + } + let (inventory, inventory_epoch) = hid_runtime.inventory_snapshot(); + let mut attaches = Vec::new(); + for item in hid_session.reconcile(&inventory) { + match item { + HidOutbound::Release { low_id, bytes } => { + if let Some(command) = sony_report_command(low_id, &bytes) + && !channels.send_control_teardown(rtc, &command) + { + return false; + } + } + HidOutbound::Removal { low_id } => { + if let Some(command) = + sony_device_change_command(SonyDeviceControl::Removal, low_id) + && !channels.send_control_teardown(rtc, &command) + { + return false; + } + } + HidOutbound::Attach { low_id } => attaches.push(low_id), + HidOutbound::Report { .. } => {} + } + } + let mask = hid_session.rich_slot_mask(); + hid_runtime.set_session_admission(mask, inventory_epoch); + if mask != *published_mask { + let retirement = input_codec.set_rich_slot_mask(mask, timestamp_us); + let mut failed = false; + for message in &retirement { + if !channels.send_encoded(rtc, message).is_sent() { + failed = true; + break; + } + } + if !failed { + *published_mask = mask; + } else { + return false; + } + } + for low_id in attaches { + if let Some(command) = sony_device_change_command(SonyDeviceControl::Attach, low_id) + && !channels.send_control(rtc, &command) + { + return false; + } + } + let mut published = true; + for item in hid_runtime.drain(DRAIN_PER_ITERATION) { + let (slot, faulted, observed_at_us) = match item { + IngressItem::Snapshot(snapshot) => { + if let Some((low_id, bytes)) = hid_session.build_report(&snapshot) + && let Some(command) = sony_report_command(low_id, &bytes) + && !channels.send_control(rtc, &command) + { + published = false; + } + continue; + } + IngressItem::Release { + slot, + observed_at_us, + .. + } => (slot, false, observed_at_us), + IngressItem::Fault { + slot, + observed_at_us, + .. + } => (slot, true, observed_at_us), + }; + hid_session.observe_slot(slot, observed_at_us); + if let Some((low_id, bytes)) = hid_session.release_snapshot(slot) + && let Some(command) = sony_report_command(low_id, &bytes) + && !channels.send_control_teardown(rtc, &command) + { + published = false; + } + if faulted { + eprintln!( + "Sony source in slot {slot} overflowed its bounded mailbox and was neutralized" + ); + } + } + published +} + +fn store_sony_output( + hid_session: &mut HidSession, + hid_runtime: &HidRuntime, + low_id: u8, + operation: &[u8], + haptics: &NvstHaptics, +) { + let Some(rumble) = hid_session.take_output(operation, low_id) else { + return; + }; + if !hid_runtime.output_admitted(rumble.slot) { + return; + } + haptics.store_sony(rumble); +} + +fn run_nvst_webrtc_bundle( + socket: UdpSocket, + config: NvstVideoConfig, + commands: Receiver, + outputs: NvstReceiverOutputs, + transport_origin: Instant, + setup: NvstWorkerSetup, +) { + let NvstWorkerSetup { + rtc: Some(mut rtc), + hid_runtime, + upstream_ready, + } = setup + else { + unreachable!("WebRTC bundle requires an RTC instance"); + }; + let NvstReceiverOutputs { + media_consumer, + event_sender, + input_ready, + microphone, + } = outputs; + let bundle_peer = config.bundle_peer(); + let local_port = socket.local_addr().map_or(0, |addr| addr.port()); + let ping_payload = config + .bundle_natt_remote_username + .clone() + .unwrap_or_else(|| "PING".to_owned()); + log_udp_receiver_start( + "bundle", + &socket, + bundle_peer, + &config, + ping_payload.as_bytes(), + ); + let stun_credentials = config.stun_credentials.clone(); + let feedback = config.feedback(); + let audio_track = config.audio_track().cloned(); + let mut microphone_packetizer = MicrophonePacketizer::new(config.microphone_rtp); + let frame_time_us = config.frame_time_us; + // With a dedicated Mjolnir video socket the bundle only carries + // control/audio keepalive traffic; the Mjolnir receiver owns the media + // timeout, so the bundle must not raise a spurious media recovery. + let unified_socket = config.unified_socket; + let owns_media_timeout = config.mjolnir_udp_port.is_none(); + let mjolnir = config.carries_mjolnir_video(); + let hid_device_mask = config.hid_device_mask(); + let physical_local = socket.local_addr().ok().map_or_else( + || bundle_peer, + |local| routed_host_addr(Some(bundle_peer), local), + ); + let receive_destination = logical_ice_addr(physical_local, 1); + let receive_source = logical_ice_addr(bundle_peer, 2); + let frame_pacing_feedback = config.frame_pacing_feedback; + let max_control_report_bytes = config.max_control_report_bytes; + let mut receiver = NvstVideoReceiver::new(config); + let mut video_delivery_gap = false; + let mut datagram = vec![0_u8; 65_536]; + let mut inbound_datagrams = 0_u64; + let mut outbound_datagrams = 0_u64; + let mut hole_punch_pings = 0_u64; + let mut ping_tracker = StreamPingTracker::default(); + let mut ice_responses = IceResponseTracker::default(); + let mut ice_ping_responses = 0; + let mut last_hole_punch = Instant::now() - PING_INTERVAL_BEFORE_CONNECTION; + let mut audio_streams = NvstAudioStreams::default(); + let mut dtls_ready = false; + let mut microphone_generation = 0; + let mut microphone_sequence = 0_u64; + // RTCP-over-SCTP (`rtcp1`) feedback channel state. + let mut sctp_started = false; + let mut rtcp_channel: Option = None; + let mut rtcp_channel_open = false; + let mut control_partial_open = false; + let rtcp_sender_ssrc = RTCP_SENDER_SSRC; + let mut last_rtcp_send = Instant::now() - SRTCP_RR_INTERVAL; + let mut last_recovery_send = Instant::now(); + let mut last_keyframe_attempt = Instant::now() - IDR_RETRY_INTERVAL; + let mut keyframe_attempts = 0_u64; + let mut last_keyframe_attempt_log: Option = None; + let mut rtcp_reports_sent = 0_u64; + let mut control_reports = ControlReportBatch::new(Instant::now(), max_control_report_bytes); + let mut last_frame_pacing_send = Instant::now() - FRAME_PACING_INTERVAL; + let control_stats_origin = Instant::now(); + let mut last_control_stats_log = Instant::now(); + let mut frame_pacing_reports_sent = 0_u64; + let mut sctp_started_at: Option = None; + let mut input_channels: Option = None; + let mut hid_session = hid_runtime.as_ref().map(|_| { + HidSession::new(SonyCapability { + server_mask: hid_device_mask, + ..SonyCapability::default() + }) + }); + let mut hid_published_mask = 0_u16; + let mut generic_session = hid_runtime + .as_ref() + .map(|runtime| GenericSession::new(runtime.session_generation().unwrap_or(0))); + let mut generic_release_epoch = 0; + let mut input_state = NvstInputChannelState::default(); + let mut input_codec = NvstInputCodec::default(); + let mut input_delivery = NvstInputDelivery::default(); + let mut last_input_types = Vec::new(); + let mut mouse_motion_packets = 0_u64; + let mut cursor_capture = NvstCursorCapture::default(); + let mut control_keepalive_at = next_control_keepalive(Instant::now()); + let mut control_keepalive_health = ControlKeepaliveHealth::default(); + let mut server_control_diagnostics = ServerControlDiagnostics::default(); + let mut input_timeout_reported = false; + let mut upstream_ready = upstream_ready; + let mut upstream_allowed = upstream_ready.is_none(); + let mut pending_input_version = None; + let mut audio_receiver = NvstAudioReceiver::default(); + let mut audio_jitter = audio_track + .as_ref() + .map(|track| AudioJitterBuffer::new(track.clock_rate_hz)); + let mut last_audio_jitter_log = Instant::now(); + let mut liveness = BundleLiveness::default(); + 'bundle: loop { + let now = Instant::now(); + if let Some(ready) = upstream_ready.as_ref() { + match ready.try_recv() { + Ok(()) => { + feedback.complete_play(now); + upstream_allowed = true; + upstream_ready = None; + control_keepalive_at = now; + if sctp_started { + sctp_started_at = Some(now); + } + } + Err(TryRecvError::Disconnected) => { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::UpstreamGateClosed), + ); + break 'bundle; + } + Err(TryRecvError::Empty) => {} + } + } + if upstream_allowed + && let Some(channels) = input_channels + && let Some(version) = pending_input_version.take() + { + if !input_state.activation_sent() { + cursor_capture.activate(now); + let _ = event_sender.send(NvstReceiveEvent::CursorCapture(true)); + } + finish_nvst_input_handshake( + &mut input_state, + channels, + &mut rtc, + transport_origin, + &input_ready, + &event_sender, + version, + ); + } + loop { + match commands.try_recv() { + Ok(UdpReceiverCommand::Pause) => forward_optional(&event_sender, receiver.pause()), + Ok(UdpReceiverCommand::Resume) => { + forward_optional(&event_sender, receiver.resume()) + } + Ok(UdpReceiverCommand::Recover) => { + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-bundle", + &format!( + "recovery-snapshot {} inbound={inbound_datagrams} outbound={outbound_datagrams} dtls_ready={dtls_ready} sctp_started={sctp_started} input_ready={}", + liveness.snapshot(Instant::now()), + input_ready.load(Ordering::Acquire), + ), + ); + forward_optional(&event_sender, receiver.recover()) + } + Ok(UdpReceiverCommand::SendText { text, timestamp_us }) => { + let result = input_delivery.submit_text(text, timestamp_us, |message| { + let outcome = if upstream_allowed + && input_state.is_ready() + && let Some(channels) = input_channels + { + channels.send_encoded(&mut rtc, message) + } else { + classify_nvst_unready_input(&mut rtc, input_channels, message) + }; + report_nvst_user_input( + message, + outcome, + &feedback.idle_warning_active, + &event_sender, + ); + outcome + }); + match result { + Ok(()) => {} + Err(NvstInputDeliveryError::Exhausted) => { + let _ = event_sender + .send(NvstReceiveEvent::TextRejected("paste-queue-full")); + } + Err(_) => { + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "text input channel closed", + )); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::InputDeliveryFailed), + ); + break 'bundle; + } + } + } + Ok(UdpReceiverCommand::SendInput { bytes, reply }) => { + if upstream_allowed + && input_state.is_ready() + && let Some(channels) = input_channels + { + let input_types = native_input_types(&bytes); + let should_log = verbose_diagnostics_enabled() + && match input_types.as_ref() { + Ok(types) => { + let only_motion = !types.is_empty() + && types.iter().all(|input_type| { + native_input_type_is_motion(*input_type) + }); + if only_motion { + mouse_motion_packets = mouse_motion_packets.wrapping_add(1); + } + let changed = *types != last_input_types; + if changed { + last_input_types.clone_from(types); + } + changed || !only_motion || mouse_motion_packets % 120 == 1 + } + Err(_) => true, + }; + if should_log { + match input_types.as_ref() { + Ok(types) => { + let names = types + .iter() + .map(|input_type| { + format!( + "{}({input_type})", + native_input_type_name(*input_type) + ) + }) + .collect::>() + .join(","); + eprintln!( + "NVST native input rx: events=[{names}] bytes={} raw={}", + bytes.len(), + diagnostic_hex(&bytes, 128), + ); + } + Err(error) => eprintln!( + "NVST native input rx undecodable: error={error} bytes={} raw={}", + bytes.len(), + diagnostic_hex(&bytes, 128), + ), + } + } + let timestamp_us = transport_origin + .elapsed() + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + if !send_nvst_captured_input( + &mut input_codec, + &mut input_delivery, + &bytes, + timestamp_us, + reply, + &input_ready, + &event_sender, + &feedback.idle_warning_active, + |message| { + if should_log { + eprintln!( + "NVST encoded input tx: route={:?} bytes={} raw={}", + message.route, + message.bytes.len(), + diagnostic_hex(&message.bytes, 128), + ); + } + channels.send_encoded(&mut rtc, message) + }, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::InputDeliveryFailed), + ); + break 'bundle; + } + } else { + let timestamp_us = transport_origin + .elapsed() + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + if !send_nvst_captured_input( + &mut input_codec, + &mut input_delivery, + &bytes, + timestamp_us, + reply, + &input_ready, + &event_sender, + &feedback.idle_warning_active, + |message| { + classify_nvst_unready_input(&mut rtc, input_channels, message) + }, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::InputDeliveryFailed), + ); + break 'bundle; + } + } + } + command @ (Ok(UdpReceiverCommand::Stop) | Err(TryRecvError::Disconnected)) => { + let cause = if command.is_ok() { + NvstStopCause::Requested + } else { + NvstStopCause::ControllerClosed + }; + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-bundle", + &format!( + "stop-snapshot {cause} {} inbound={inbound_datagrams} outbound={outbound_datagrams} dtls_ready={dtls_ready} sctp_started={sctp_started}", + liveness.snapshot(Instant::now()), + ), + ); + forward_optional(&event_sender, receiver.stop(cause)); + break 'bundle; + } + Err(TryRecvError::Empty) => break, + } + } + + if upstream_allowed + && let Some(channels) = input_channels + && input_delivery + .flush(|message| { + let outcome = if input_state.is_ready() { + channels.send_encoded(&mut rtc, message) + } else { + classify_nvst_unready_input(&mut rtc, Some(channels), message) + }; + report_nvst_user_input( + message, + outcome, + &feedback.idle_warning_active, + &event_sender, + ); + outcome + }) + .is_err() + { + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "discrete input delivery failed; stopping to prevent stuck input", + )); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::InputDeliveryFailed), + ); + break 'bundle; + } + + // Official first burst is three ICE Binding Requests, plus NATT + // ping-string PING. After DTLS they keep pinging at 100ms. + if upstream_allowed && input_state.control_is_open() && now >= control_keepalive_at { + if let Some(channels) = input_channels { + control_keepalive_health.record(channels.send_keepalive(&mut rtc, 0)); + } + control_keepalive_at = next_control_keepalive(now); + } + if upstream_allowed + && let Some(channels) = input_channels + && cursor_capture.update(now, |command| match command { + CursorCommand::Capture(enabled) => { + channels.send_mouse_cursor_capture(&mut rtc, enabled) + } + CursorCommand::Tracking => channels.send_remote_cursor_tracking(&mut rtc, true), + }) + { + let _ = event_sender.send(NvstReceiveEvent::CursorCapture(false)); + } + if upstream_allowed + && input_state.is_ready() + && input_delivery.is_empty() + && let (Some(hid_runtime), Some(hid_session), Some(channels)) = + (hid_runtime.as_ref(), hid_session.as_mut(), input_channels) + { + let timestamp_us = now + .duration_since(transport_origin) + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + if !pump_hid_pending( + hid_runtime, + hid_session, + &mut input_codec, + channels, + &mut rtc, + &mut hid_published_mask, + timestamp_us, + ) { + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "sony hid output could not be queued; stopping to prevent stuck input", + )); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::HidOutputBlocked), + ); + break 'bundle; + } + } + if upstream_allowed + && input_state.is_ready() + && let (Some(runtime), Some(session), Some(channels)) = ( + hid_runtime.as_ref(), + generic_session.as_mut(), + input_channels, + ) + && !pump_generic_hid_pending( + &runtime.generic, + session, + &mut generic_release_epoch, + |command, teardown| { + if teardown { + channels.send_control_teardown(&mut rtc, command) + } else { + channels.send_control(&mut rtc, command) + } + }, + ) + { + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "generic hid output could not be queued; stopping to prevent stuck input", + )); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::HidOutputBlocked), + ); + break 'bundle; + } + if upstream_allowed + && !input_timeout_reported + && input_state.handshake_timed_out(sctp_started_at, now) + { + input_timeout_reported = true; + let _ = event_sender.send(NvstReceiveEvent::InputUnavailable( + "input handshake timed out", + )); + } + let ping_interval = if dtls_ready { + PING_INTERVAL_AFTER_CONNECTION + } else { + PING_INTERVAL_BEFORE_CONNECTION + }; + if now.duration_since(last_hole_punch) >= ping_interval + && let Some(credentials) = stun_credentials.as_ref() + { + let mut ice_bytes = 0_usize; + if !dtls_ready { + for _ in 0..3 { + let mut ice_tid = [0_u8; 12]; + if getrandom::fill(&mut ice_tid).is_ok() { + let ice = build_stun_binding_request(credentials, &ice_tid); + ice_bytes = ice.len(); + if let Err(error) = socket.send_to(&ice, bundle_peer) { + if udp_icmp_port_unreachable(&error) { + continue; + } + eprintln!("NVST ICE send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-ice-send", &error)), + ); + break 'bundle; + } + } + } + } + let mut natt_tid = [0_u8; 12]; + let natt = if getrandom::fill(&mut natt_tid).is_ok() { + let natt = build_natt_hole_punch_request( + &credentials.local_username_fragment, + ping_payload.as_bytes(), + &credentials.remote_password, + &natt_tid, + ); + let sent_at = Instant::now(); + if let Err(error) = socket.send_to(&natt, bundle_peer) { + if !udp_icmp_port_unreachable(&error) { + eprintln!("NVST NATT send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-natt-send", &error)), + ); + break 'bundle; + } + None + } else { + ping_tracker.sent(natt_tid, sent_at); + Some(natt) + } + } else { + None + }; + hole_punch_pings += 1; + if hole_punch_pings == 1 || hole_punch_pings % 50 == 0 { + eprintln!( + "NVST hole-punch ping={hole_punch_pings} dest={bundle_peer} iceBurst={} iceBytes={ice_bytes} nattBytes={} credentials=redacted", + if dtls_ready { 0 } else { 3 }, + natt.as_ref().map_or(0, Vec::len), + ); + } + last_hole_punch = now; + } + + if upstream_allowed + && control_partial_open + && let Some(channels) = input_channels + { + let mut writable = !control_reports.flush_due(now) + || control_reports.flush(now, &feedback, |bytes| { + channels.send_partial_control(&mut rtc, bytes) + }); + while writable { + if !control_reports.fits(FRAME_ACK_RECORD_LEN) { + writable = control_reports.flush(now, &feedback, |bytes| { + channels.send_partial_control(&mut rtc, bytes) + }); + if !writable { + break; + } + } + let Some(frame) = feedback.take_completed_frame() else { + break; + }; + if frame_pacing_feedback + && feedback.frame_pacing_enabled.load(Ordering::Acquire) + && now.duration_since(last_frame_pacing_send) >= FRAME_PACING_INTERVAL + { + // Packet-completion intervals are intentionally bursty and are not display + // pacing error. Feeding that network jitter into the server PID made its + // encoder cadence oscillate. Until a real vsync timestamp is available, + // report a neutral error exactly as the unavailable ACK stage metrics do. + let pacing = + frame_pacing_report(frame.frame_number, frame_time_us, 0).encoded(); + if channels.send_partial_control(&mut rtc, &pacing) { + frame_pacing_reports_sent = frame_pacing_reports_sent.saturating_add(1); + } + last_frame_pacing_send = now; + } + let ack = frame.command(transport_origin).encoded(); + control_reports.append_ack(frame, &ack, now); + if !control_reports.fits(FRAME_ACK_RECORD_LEN) { + writable = control_reports.flush(now, &feedback, |bytes| { + channels.send_partial_control(&mut rtc, bytes) + }); + } + } + if writable + && control_reports.qos.is_none() + && now.duration_since(control_reports.last_qos_send) >= QOS_REPORT_INTERVAL + { + let report = feedback.qos_report_at( + &control_reports.last_qos_report, + now.saturating_duration_since(transport_origin), + now, + ); + let command = report.command().encoded(); + if !control_reports.fits(command.len()) { + writable = control_reports.flush(now, &feedback, |bytes| { + channels.send_partial_control(&mut rtc, bytes) + }); + } + if writable { + control_reports.append_qos(report, &command, now); + let _ = control_reports.flush(now, &feedback, |bytes| { + channels.send_partial_control(&mut rtc, bytes) + }); + } + } + } + + if now.duration_since(last_control_stats_log) >= Duration::from_secs(10) { + let input_stats = input_delivery.snapshot(); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-input", + &format!( + "pendingMessages={} pendingBytes={} pendingTextMessages={} pendingTextBytes={} deferredWrites={} droppedMotion={}", + input_stats.pending_messages, + input_stats.pending_bytes, + input_stats.pending_text_messages, + input_stats.pending_text_bytes, + input_stats.deferred_writes, + input_stats.dropped_motion, + ), + ); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-control", + &format!( + "keepaliveFailures={} keepaliveConsecutiveFailures={} keepaliveRecoveries={} unknownCommands={} lastUnknownCommand={:?} malformedCommands={} invalidStreamCommands={}", + control_keepalive_health.failures, + control_keepalive_health.consecutive_failures, + control_keepalive_health.recoveries, + server_control_diagnostics.unknown, + server_control_diagnostics.last_unknown, + server_control_diagnostics.malformed, + server_control_diagnostics.invalid_stream, + ), + ); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-bundle", + &format!( + "{} local_port={local_port} peer_port={} inbound={inbound_datagrams} outbound={outbound_datagrams} pings={hole_punch_pings} dtls_ready={dtls_ready} sctp_started={sctp_started} frame_ack={} pacing={frame_pacing_reports_sent} qos={} videoBytesAssembled={} qosBweKbps={} qosUtilPercent={} qosOwdUs={} qosJitterUs={}", + receiver.stats_line(control_stats_origin), + bundle_peer.port(), + control_reports.frame_acks_sent, + control_reports.qos_reports_sent, + feedback.completed_frame_bytes.load(Ordering::Acquire), + control_reports.last_qos_report.bandwidth.estimate_kbps, + control_reports + .last_qos_report + .bandwidth + .utilization_percent, + control_reports.last_qos_report.bandwidth.queue_delay_us, + control_reports.last_qos_report.bandwidth.jitter_us, + ), + ); + eprintln!( + "NVST control-stats elapsed={:.1}s frameAck={} lastAck={:?} pacing={frame_pacing_reports_sent} qos={} assembledBytesAtLastQos={} qosBweKbps={} qosUtilPercent={} qosOwdUs={} qosJitterUs={}", + now.saturating_duration_since(control_stats_origin) + .as_secs_f64(), + control_reports.frame_acks_sent, + control_reports.last_ack_frame, + control_reports.qos_reports_sent, + control_reports.last_qos_report.bytes_received, + control_reports.last_qos_report.bandwidth.estimate_kbps, + control_reports + .last_qos_report + .bandwidth + .utilization_percent, + control_reports.last_qos_report.bandwidth.queue_delay_us, + control_reports.last_qos_report.bandwidth.jitter_us, + ); + eprintln!( + "NVST frame-stage-timings {}", + feedback.frame_stage_timings().log_line() + ); + last_control_stats_log = now; + } + + if upstream_allowed + && uses_rtcp_video_feedback(mjolnir, rtcp_channel_open) + && now.duration_since(last_rtcp_send) >= SRTCP_RR_INTERVAL + && let Some(report_block) = feedback.report_snapshot(true) + { + let report = build_rtcp_receiver_report(rtcp_sender_ssrc, report_block); + let admitted = + input_channels.is_some_and(|channels| channels.send_rtcp(&mut rtc, &report)); + if admitted { + rtcp_reports_sent += 1; + if rtcp_reports_sent == 1 || rtcp_reports_sent % 10 == 0 { + eprintln!( + "NVST rtcp1 RR sent={rtcp_reports_sent} mediaSsrc={} highestSeq={}", + report_block.media_ssrc, report_block.highest_sequence, + ); + } + } + last_rtcp_send = now; + } + + // Loss feedback cannot wait for the one-second Receiver Report cadence: + // request retransmission while the reorder buffer still holds later + // packets, then request a keyframe if bounded recovery was exhausted. + if upstream_allowed + && now.duration_since(last_recovery_send) >= RTCP_RECOVERY_INTERVAL + && let Some((media_ssrc, _)) = feedback.stream_snapshot() + { + if let Some(channels) = input_channels { + send_pending_nack( + &feedback, + now, + &mut rtc, + channels, + mjolnir, + rtcp_sender_ssrc, + media_ssrc, + ); + } + last_recovery_send = now; + } + + if upstream_allowed + && input_state.control_is_open() + && let Some(channels) = input_channels + { + send_pending_frame_invalidation(&feedback, now, &mut rtc, channels); + } + + // Do not gate control IDR on stream_snapshot(): it is needed most when + // the server has not sent any video yet. Bound failed sends as well as + // successful ones, and leave the request armed until a keyframe arrives. + let (try_pli, try_idr) = feedback.keyframe_request_routes( + now, + last_keyframe_attempt, + uses_rtcp_video_feedback(mjolnir, rtcp_channel_open), + input_state.control_is_open(), + ); + if upstream_allowed && (try_pli || try_idr) { + let mut pli_queued = false; + if try_pli && let Some((media_ssrc, _)) = feedback.stream_snapshot() { + let pli = build_rtcp_pli(rtcp_sender_ssrc, media_ssrc); + pli_queued = + input_channels.is_some_and(|channels| channels.send_rtcp(&mut rtc, &pli)); + } + let idr_queued = try_idr + && input_channels.is_some_and(|channels| { + channels.send_control(&mut rtc, &idr_request().encoded()) + }); + last_keyframe_attempt = now; + keyframe_attempts = keyframe_attempts.saturating_add(1); + if last_keyframe_attempt_log + .is_none_or(|last| now.duration_since(last) >= Duration::from_secs(10)) + { + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-keyframe", + &format!( + "attempt={keyframe_attempts} pli_queued={pli_queued} idr_queued={idr_queued} stream_known={} awaiting_assembled_keyframe=true", + feedback.stream_snapshot().is_some() + ), + ); + last_keyframe_attempt_log = Some(now); + } + } else if !feedback.keyframe_needed.load(Ordering::Acquire) { + last_keyframe_attempt_log = None; + } + + let mut microphone_queue = microphone.lock().unwrap_or_else(|error| error.into_inner()); + if upstream_allowed && microphone_generation != microphone_queue.generation { + microphone_generation = microphone_queue.generation; + rtc.direct_api().remove_stream_tx(Ssrc::from(1)); + microphone_packetizer.clear(); + } + if upstream_allowed && dtls_ready { + while let Some(frame) = microphone_queue.pop(Instant::now()) { + let timestamp = frame.timestamp; + let queued_at = frame.queued_at; + let (payload_type, payload) = microphone_packetizer.packetize(frame); + rtc.direct_api() + .declare_stream_tx(Ssrc::from(1), None, Mid::from("2"), None) + .write_rtp(str0m::rtp::RtpWrite::new( + payload_type.into(), + microphone_sequence.into(), + timestamp, + queued_at, + payload, + )); + microphone_sequence += 1; + } + } + let timeout = loop { + match rtc.poll_output() { + Ok(Output::Timeout(timeout)) => { + audio_streams.finish_poll(&mut rtc); + break timeout; + } + Ok(Output::Transmit(transmit)) => { + outbound_datagrams += 1; + liveness.last_outbound = Some(Instant::now()); + let kind = if looks_like_stun(&transmit.contents) { + "stun" + } else if looks_like_dtls(&transmit.contents) { + "dtls" + } else { + "other" + }; + if outbound_datagrams <= 8 + || (verbose_diagnostics_enabled() && outbound_datagrams % 50 == 0) + { + eprintln!( + "NVST WebRTC outbound={outbound_datagrams} kind={kind} dest={} bytes={}", + bundle_peer, + transmit.contents.len() + ); + } + if let Err(error) = socket.send_to(&transmit.contents, bundle_peer) { + if udp_icmp_port_unreachable(&error) { + continue; + } + if peek_rtp_ssrc(&transmit.contents) == Some(1) { + microphone_queue.close(); + microphone_generation = microphone_queue.generation; + rtc.direct_api().remove_stream_tx(Ssrc::from(1)); + let _ = event_sender + .send(NvstReceiveEvent::MicrophoneError(error.to_string())); + continue; + } + eprintln!("NVST WebRTC send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-webrtc-send", &error)), + ); + break 'bundle; + } + ice_responses.sent(&transmit.contents); + // Official ICE-on WebRtcTransport skips setupDtls until a real + // inbound STUN. Do not synthesize Binding Success — that only + // unblocks str0m and sends ClientHello before GFN has a pair. + } + Ok(Output::Event(event)) => match event { + Event::PeerStats(stats) => feedback.update_ice_ping( + stats.selected_candidate_pair.as_ref(), + &mut ice_ping_responses, + Instant::now(), + ), + Event::IceConnectionStateChange(state) => { + liveness.ice_state = Some(state); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-ice", + &format!("state={state:?}"), + ); + eprintln!("NVST ICE state: {state:?}"); + // Official GFN treats hole-punch / ICE receive failure as + // non-fatal. Media is gated on DTLS, not ICE success. + } + Event::Connected => { + dtls_ready = true; + feedback + .authenticated_transport_ready + .store(true, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::TransportReady("dtls")); + eprintln!("NVST DTLS handshake complete; waiting for SRTP/Mjolnir"); + if !sctp_started { + sctp_started = true; + sctp_started_at = Some(Instant::now()); + rtc.direct_api().start_sctp(true); + let channels = NvstInputChannels::create(&mut rtc, !mjolnir); + rtcp_channel = channels.rtcp; + input_channels = Some(channels); + let _ = event_sender.send(NvstReceiveEvent::TransportReady("sctp")); + eprintln!( + "NVST SCTP started with {} data channels", + 6 + usize::from(channels.cursor.is_some()) + + usize::from(channels.rtcp.is_some()) + ); + } + } + Event::ChannelOpen(id, label) => { + eprintln!("NVST data channel open: id={id:?} label={label}"); + if let Some(channels) = input_channels { + if id == channels.control_reliable { + control_keepalive_at = Instant::now(); + } + if id == channels.control_partial { + control_partial_open = true; + } + if let Some(version) = input_state.channel_opened(channels, id) { + pending_input_version = Some(version); + } + } + if Some(id) == rtcp_channel { + rtcp_channel_open = true; + } + } + Event::ChannelData(data) => { + liveness.last_channel_data = Some(Instant::now()); + if let Some(channels) = input_channels + && channels.contains(data.id) + { + let label = channels.label(data.id); + if data.id == channels.control_reliable + || data.id == channels.control_partial + { + if feedback.receive_server_control( + &data.data, + &mut server_control_diagnostics, + &event_sender, + ) { + input_ready.store(false, Ordering::Release); + break 'bundle; + } + feedback.haptics.receive(&data.data); + } + if data.id == channels.control_reliable + && let (Some(hid_session), Some(hid_runtime)) = + (hid_session.as_mut(), hid_runtime.as_ref()) + { + for_each_sony_output(&data.data, |low_id, operation| { + store_sony_output( + hid_session, + hid_runtime, + low_id, + operation, + &feedback.haptics, + ); + }); + } + if data.id == channels.control_reliable + && let (Some(session), Some(runtime)) = + (generic_session.as_mut(), hid_runtime.as_ref()) + { + let mut queued = true; + for_each_generic_hid_message(&data.data, |message| { + if queued { + queued = store_generic_hid_message( + session, + &runtime.generic, + message, + |command, teardown| { + if teardown { + channels + .send_control_teardown(&mut rtc, command) + } else { + channels.send_control(&mut rtc, command) + } + }, + ); + } + }); + if !queued { + input_ready.store(false, Ordering::Release); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::HidOutputBlocked), + ); + break 'bundle; + } + } + let cursor_messages = if Some(data.id) == channels.cursor { + Vec::new() + } else { + server_cursor_messages(&data.data) + }; + for message in cursor_messages { + cursor_capture.notify(Instant::now()); + eprintln!( + "NVST cursor wire rx: channel={label} id={:?} command=0x{:04x} offset={} cursorId={:?} position={:?} visible={:?} bytes={} raw={}", + data.id, + message.command, + message.offset, + message.cursor_id, + message.position, + message.visible, + message.raw.len(), + diagnostic_hex(&message.raw, 512), + ); + if let Some(cursor) = message.normalized { + eprintln!( + "NVST cursor dispatch: source={label} type={} id={} bytes={} raw={}", + cursor[0], + cursor[1], + cursor.len(), + diagnostic_hex(&cursor, 512), + ); + let _ = event_sender.send(NvstReceiveEvent::Cursor(cursor)); + } + } + + if Some(data.id) == channels.cursor { + if !valid_cursor_channel_message(&data.data) { + eprintln!( + "NVST malformed cursor-channel notification ignored: bytes={}", + data.data.len(), + ); + continue; + } + cursor_capture.notify(Instant::now()); + eprintln!( + "NVST cursor-channel raw rx: id={:?} bytes={} type={:?} cursorId={:?} raw={}", + data.id, + data.data.len(), + data.data.first(), + data.data.get(1), + diagnostic_hex(&data.data, 512), + ); + let _ = + event_sender.send(NvstReceiveEvent::Cursor(data.data.to_vec())); + continue; + } + + if verbose_diagnostics_enabled() { + eprintln!( + "NVST data channel rx: channel={label} id={:?} bytes={} raw={}", + data.id, + data.data.len(), + diagnostic_hex(&data.data, 128), + ); + } + } + if let Some(channels) = input_channels + && let Some(version) = + input_state.channel_data(channels, data.id, &data.data) + { + pending_input_version = Some(version); + } else if Some(data.id) == rtcp_channel { + eprintln!( + "NVST rtcp1 inbound: id={:?} binary={} bytes={}", + data.id, + data.binary, + data.data.len(), + ); + } + } + Event::ChannelClose(id) => { + liveness.channels_closed += 1; + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-sctp", + &format!( + "channel-close label={} {}", + input_channels.map_or("unknown", |channels| channels.label(id)), + liveness.snapshot(Instant::now()), + ), + ); + if let Some(channels) = input_channels { + if id == channels.control_reliable { + cursor_capture.reset(); + } + let input_channel_closed = id == channels.input_partial; + if input_state.channel_closed(channels, id) || input_channel_closed { + pending_input_version = None; + input_ready.store(false, Ordering::Release); + let reason = if input_channel_closed { + "partially reliable input data channel closed" + } else { + "reliable input data channel closed" + }; + let _ = + event_sender.send(NvstReceiveEvent::InputUnavailable(reason)); + } + } + if input_channels.is_some_and(|channels| id == channels.control_partial) { + control_partial_open = false; + } + if Some(id) == rtcp_channel { + rtcp_channel_open = false; + rtcp_channel = None; + } + } + Event::RtpPacket(packet) => { + let outer_payload_type = *packet.header.payload_type; + let is_audio = audio_track.as_ref().is_some_and(|audio| { + matches_audio_track(audio, outer_payload_type, *packet.header.ssrc) + }); + if is_audio { + audio_streams.authenticated(&mut rtc, *packet.header.ssrc); + let received_at_us = packet + .timestamp + .saturating_duration_since(transport_origin) + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + if let Some(jitter) = audio_jitter.as_mut() { + jitter.push( + AudioPacket { + header: packet.header, + payload: packet.payload, + received_at_us, + }, + Instant::now(), + ); + } + } else { + let received_at = Instant::now(); + for event in receiver.process_mjolnir_payload( + *packet.header.ssrc, + packet.header.timestamp, + &packet.payload, + received_at, + ) { + if !forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + transport_origin, + received_at, + &mut video_delivery_gap, + event, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + break 'bundle; + } + } + } + } + _ => {} + }, + Err(error) => { + eprintln!("NVST WebRTC bundle failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::webrtc("poll-output", &error)), + ); + break 'bundle; + } + } + }; + + drop(microphone_queue); + if let (Some(jitter), Some(track)) = (audio_jitter.as_mut(), audio_track.as_ref()) { + while let Some(packet) = jitter.pop(Instant::now()) { + let frames = match audio_receiver.depacketize( + track, + &packet.header, + packet.payload, + packet.received_at_us, + ) { + Ok(frames) => frames, + Err(reason) => { + let _ = event_sender.send(NvstReceiveEvent::Dropped(reason)); + continue; + } + }; + for frame in frames { + match audio_receiver.deliver(&media_consumer, frame) { + Ok(()) | Err(TransportError::MediaConsumerBackpressured) => {} + Err(_) => { + let _ = event_sender.send(NvstReceiveEvent::Dropped( + NvstDropReason::MediaConsumerClosed, + )); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + break 'bundle; + } + } + } + } + if last_audio_jitter_log.elapsed() >= Duration::from_secs(1) { + opennow_streamer_protocol::log::log_async( + "INFO", + "audio-jitter", + &format!( + "target_ms={} queued={} stats={:?}", + jitter.target_ms(), + jitter.len(), + jitter.stats + ), + ); + last_audio_jitter_log = Instant::now(); + } + } + let wait = timeout + .saturating_duration_since(Instant::now()) + .min(CONTROL_RECEIVE_POLL_INTERVAL); + if wait.is_zero() { + if let Err(error) = rtc.handle_input(Input::Timeout(Instant::now())) { + eprintln!("NVST WebRTC timer failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::webrtc("timer", &error)), + ); + break 'bundle; + } + } else { + if let Err(error) = socket.set_read_timeout(Some(wait)) { + eprintln!("NVST UDP timeout configuration failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-read-timeout", &error)), + ); + break 'bundle; + } + match socket.recv_from(&mut datagram) { + Ok((length, source)) => { + inbound_datagrams += 1; + liveness.last_inbound = Some(Instant::now()); + if inbound_datagrams == 1 { + log_udp_first_inbound( + "bundle", + local_port, + source == bundle_peer, + source, + length, + transport_origin, + ); + } + if inbound_datagrams == 1 + || (verbose_diagnostics_enabled() && inbound_datagrams % 50 == 0) + { + eprintln!( + "NVST WebRTC inbound={inbound_datagrams} source={source} bytes={length} dtlsReady={dtls_ready}" + ); + } + if source != bundle_peer { + continue; + } + liveness.last_peer_inbound = liveness.last_inbound; + feedback.record_socket_receive(StreamSocket::Bundle, true, dtls_ready, length); + if let Some(credentials) = stun_credentials.as_ref() { + let received_at = Instant::now(); + if let Some(elapsed) = ping_tracker.receive( + &datagram[..length], + source, + credentials, + received_at, + ) { + feedback.publish_ping(false, received_at, elapsed); + } + } + if datagram[..length] == *b"PING" { + if let Err(error) = socket.send_to(b"PONG", source) + && !udp_icmp_port_unreachable(&error) + { + eprintln!("NVST PONG send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-pong-send", &error)), + ); + break 'bundle; + } + continue; + } + if looks_like_rtp(&datagram[..length]) + && (!dtls_ready + || !audio_track.as_ref().is_some_and(|audio| { + audio_streams.admit(&mut rtc, audio, &datagram[..length]) + })) + { + if unified_socket { + let received_at = Instant::now(); + for event in + receiver.process_datagram(source, &datagram[..length], received_at) + { + if !forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + transport_origin, + received_at, + &mut video_delivery_gap, + event, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + break 'bundle; + } + } + } + continue; + } + let destination = receive_destination; + let contents = match datagram[..length].try_into() { + Ok(value) => value, + Err(error) => { + eprintln!("NVST dropping oversized UDP packet: {error}"); + continue; + } + }; + if let Some(credentials) = stun_credentials.as_ref() + && !ice_responses.accept(&datagram[..length], &credentials.remote_password) + { + continue; + } + if let Err(error) = rtc.handle_input(Input::Receive( + Instant::now(), + Receive { + proto: RtcProtocol::Udp, + source: receive_source, + destination, + contents, + }, + )) { + eprintln!("NVST WebRTC handle_input failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::webrtc("receive", &error)), + ); + break 'bundle; + } + } + Err(error) if udp_receive_is_idle(&error) => { + if udp_icmp_port_unreachable(&error) { + liveness.port_unreachable += 1; + if liveness.port_unreachable == 1 { + log_udp_error("bundle-receive-port-unreachable", local_port, &error); + } + thread::sleep(CONTROL_RECEIVE_POLL_INTERVAL); + } + let _ = rtc.handle_input(Input::Timeout(Instant::now())); + } + Err(error) => { + log_udp_error("bundle-receive", local_port, &error); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("bundle-receive", &error)), + ); + break 'bundle; + } + } + } + + let now = Instant::now(); + if unified_socket { + for event in receiver.poll_recovery(now) { + if !forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + transport_origin, + now, + &mut video_delivery_gap, + event, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + break 'bundle; + } + } + } + let timeout = if owns_media_timeout { + receiver.poll_timeout(now) + } else { + None + }; + if timeout.is_some() { + eprintln!( + "NVST WebRTC timeout counters: inbound={inbound_datagrams}, dtlsReady={dtls_ready}" + ); + } + let packets_stalled = timeout.is_some(); + forward_optional(&event_sender, timeout); + if unified_socket && !packets_stalled { + forward_frame_progress(&mut receiver, &event_sender, now); + } + } + let mut shutdown_queued = true; + if let Some(hid_runtime) = hid_runtime.as_ref() { + if let (Some(hid_session), Some(channels)) = (hid_session.as_mut(), input_channels) { + shutdown_queued &= queue_hid_shutdown(hid_session, channels, &mut rtc); + } + if let (Some(session), Some(channels)) = (generic_session.as_mut(), input_channels) { + let actions = session.retire_all(); + shutdown_queued &= send_generic_hid_actions(session, actions, |command, _| { + channels.send_control_teardown(&mut rtc, command) + }); + } + forward_rtc_outputs( + &mut rtc, + |output| { + if let Output::Transmit(transmit) = output { + let _ = socket.send_to(&transmit.contents, bundle_peer); + } + }, + SONY_SHUTDOWN_FLUSH_BUDGET, + ); + if let Some(generation) = hid_runtime.session_generation() { + hid_runtime.unbind_session(generation); + } + } + if !shutdown_queued { + input_ready.store(false, Ordering::Release); + let _ = event_sender.send(NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped( + NvstStopCause::HidOutputBlocked, + ))); + } + rtc.disconnect(); +} + +fn run_nvst_udp_receiver( + socket: UdpSocket, + config: NvstVideoConfig, + commands: Receiver, + outputs: NvstReceiverOutputs, + transport_origin: Instant, + setup: NvstWorkerSetup, +) { + if setup.rtc.is_some() { + run_nvst_webrtc_bundle(socket, config, commands, outputs, transport_origin, setup); + return; + } + run_nvst_video_receiver( + socket, + config, + commands, + outputs, + transport_origin, + |socket, packet, peer| socket.send_to(packet, peer), + ); +} + +fn run_nvst_video_receiver( + socket: UdpSocket, + config: NvstVideoConfig, + commands: Receiver, + outputs: NvstReceiverOutputs, + transport_origin: Instant, + mut send_keepalive: impl FnMut(&UdpSocket, &[u8], SocketAddr) -> std::io::Result, +) { + let local_port = socket.local_addr().map_or(0, |addr| addr.port()); + log_udp_receiver_start( + "video", + &socket, + config.video_peer, + &config, + &config.ping_payload, + ); + let NvstReceiverOutputs { + media_consumer, + event_sender, + .. + } = outputs; + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut video_delivery_gap = false; + let stun_credentials = receiver.config.stun_credentials.clone(); + let mut datagram = vec![0_u8; 65_536]; + let mut peer_seen = false; + let mut last_ping = Instant::now() - PING_INTERVAL_BEFORE_CONNECTION; + let mut keepalive = VideoKeepaliveState::default(); + let mut inbound_datagrams = 0_u64; + let mut handled_stun = 0_u64; + let mut invalid_stun = 0_u64; + let mut non_stun = 0_u64; + let mut wrong_source = 0_u64; + let mut receiver_reports_sent = 0_u64; + let stats_origin = Instant::now(); + let mut last_stats_log = Instant::now(); + let mut last_inbound_at = None; + let mut port_unreachable = 0_u64; + loop { + loop { + match commands.try_recv() { + Ok(UdpReceiverCommand::Pause) => forward_optional(&event_sender, receiver.pause()), + Ok(UdpReceiverCommand::Resume) => { + forward_optional(&event_sender, receiver.resume()) + } + Ok(UdpReceiverCommand::Recover) => { + forward_optional(&event_sender, receiver.recover()) + } + Ok(UdpReceiverCommand::SendText { .. }) => {} + Ok(UdpReceiverCommand::SendInput { reply, .. }) => { + if let Some(reply) = reply { + let _ = reply.send(Err(TransportError::InputNotReady)); + } + } + command @ (Ok(UdpReceiverCommand::Stop) | Err(TryRecvError::Disconnected)) => { + let cause = if command.is_ok() { + NvstStopCause::Requested + } else { + NvstStopCause::ControllerClosed + }; + forward_optional(&event_sender, receiver.stop(cause)); + return; + } + Err(TryRecvError::Empty) => break, + } + } + + let now = Instant::now(); + let ping_interval = if peer_seen { + PING_INTERVAL_AFTER_CONNECTION + } else { + PING_INTERVAL_BEFORE_CONNECTION + }; + if now.duration_since(last_ping) >= ping_interval { + if let Some(credentials) = stun_credentials.as_ref() { + // Ping version 6 is an authenticated STUN Binding request. The official + // NattHolePunch::SendPing path does not prepend a legacy raw `PING`; doing so + // can make the relay retain its legacy route instead of publishing Mjolnir. + let mut transaction_id = [0_u8; 12]; + if let Err(error) = getrandom::fill(&mut transaction_id) { + eprintln!("NVST NATT transaction generation failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::RandomnessUnavailable), + ); + return; + } + let ping = build_natt_hole_punch_request( + &credentials.local_username_fragment, + &receiver.config.ping_payload, + &credentials.remote_password, + &transaction_id, + ); + let sent_at = Instant::now(); + if let Err(error) = keepalive.record_send( + send_keepalive(&socket, &ping, receiver.config.video_peer), + Some(transaction_id), + sent_at, + receiver.config.timeout, + "video-natt-send", + local_port, + ) { + log_udp_error("video-natt-send", local_port, &error); + eprintln!("NVST NATT send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("video-natt-send", &error)), + ); + return; + } + } else { + if let Err(error) = keepalive.record_send( + send_keepalive( + &socket, + &receiver.config.ping_payload, + receiver.config.video_peer, + ), + None, + now, + receiver.config.timeout, + "video-ping-send", + local_port, + ) { + log_udp_error("video-ping-send", local_port, &error); + eprintln!("NVST ping send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("video-ping-send", &error)), + ); + return; + } + } + last_ping = now; + } + + let inbound_before = inbound_datagrams; + match socket.recv_from(&mut datagram) { + Ok((length, source)) => 'datagram: { + inbound_datagrams += 1; + if inbound_datagrams == 1 { + log_udp_first_inbound( + "video", + local_port, + receiver.config.accepts_video_source(source), + source, + length, + transport_origin, + ); + eprintln!( + "NVST raw-SRTP inbound first datagram: source={source} bytes={length} peer={}", + receiver.config.video_peer + ); + } + let expected_source = receiver.config.accepts_video_source(source); + let authenticated_before = receiver.last_authenticated_packet.is_some(); + feedback.record_socket_receive( + StreamSocket::Video, + expected_source, + authenticated_before, + length, + ); + if !expected_source { + wrong_source += 1; + } + if expected_source && let Some(credentials) = stun_credentials.as_ref() { + let received_at = Instant::now(); + if let Some(elapsed) = keepalive.ping_tracker.receive( + &datagram[..length], + source, + credentials, + received_at, + ) { + receiver + .config + .feedback() + .publish_ping(true, received_at, elapsed); + } + match handle_stun_datagram(&datagram[..length], source, credentials) { + StunDatagram::Handled(response) => { + handled_stun += 1; + peer_seen = true; + if let Some(response) = response + && let Err(error) = socket.send_to(&response, source) + && !udp_icmp_port_unreachable(&error) + { + eprintln!("NVST STUN response send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket( + "video-stun-response-send", + &error, + )), + ); + return; + } + break 'datagram; + } + StunDatagram::Invalid => { + invalid_stun += 1; + break 'datagram; + } + StunDatagram::NotStun => non_stun += 1, + } + } + let received_at = Instant::now(); + let events = receiver.process_datagram(source, &datagram[..length], received_at); + if !authenticated_before { + feedback.record_socket_receive( + StreamSocket::Video, + expected_source, + receiver.last_authenticated_packet.is_some(), + length, + ); + } + for event in events { + if !forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + transport_origin, + received_at, + &mut video_delivery_gap, + event, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + return; + } + } + } + Err(error) if udp_receive_is_idle(&error) => { + if udp_icmp_port_unreachable(&error) { + port_unreachable += 1; + if port_unreachable == 1 { + log_udp_error("video-receive-port-unreachable", local_port, &error); + } + thread::sleep(UDP_RECEIVE_POLL_INTERVAL); + } + } + Err(error) => { + log_udp_error("video-receive", local_port, &error); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("video-receive", &error)), + ); + return; + } + } + let now = Instant::now(); + if inbound_datagrams != inbound_before { + last_inbound_at = Some(now); + } + if let Some(report) = receiver.poll_receiver_report(now) { + if let Err(error) = socket.send_to(&report, receiver.config.video_peer) { + if !udp_icmp_port_unreachable(&error) { + eprintln!("NVST receiver report send failed: {error}"); + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::socket("video-receiver-report-send", &error)), + ); + return; + } + } else { + receiver_reports_sent += 1; + } + } + if now.duration_since(last_stats_log) >= Duration::from_secs(10) { + last_stats_log = now; + let pings_sent = keepalive.pings_sent; + let keepalive_deferred = keepalive.deferred_total; + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-video", + &format!( + "{} local_port={local_port} peer_port={} inbound={inbound_datagrams} pings={pings_sent} keepalive_deferred={keepalive_deferred} rr={receiver_reports_sent} stun_ok={handled_stun} stun_invalid={invalid_stun} wrong_source={wrong_source}", + receiver.stats_line(stats_origin), + receiver.config.video_peer.port() + ), + ); + eprintln!( + "NVST rx-stats {} inbound={inbound_datagrams} pings={pings_sent} rr={receiver_reports_sent}", + receiver.stats_line(stats_origin), + ); + } + for event in receiver.poll_recovery(now) { + if !forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + transport_origin, + now, + &mut video_delivery_gap, + event, + ) { + forward_optional( + &event_sender, + receiver.stop(NvstStopCause::MediaConsumerClosed), + ); + return; + } + } + let timeout = receiver.poll_timeout(now); + if timeout.is_some() { + let pings_sent = keepalive.pings_sent; + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-video", + &format!( + "media-timeout local_port={local_port} peer_port={} inbound={inbound_datagrams} rx_idle_ms={} port_unreachable={port_unreachable} pings={pings_sent} stun_ok={handled_stun} wrong_source={wrong_source} {} firewall_status=unknown", + receiver.config.video_peer.port(), + idle_ms(last_inbound_at, now), + receiver.stats_line(stats_origin) + ), + ); + eprintln!( + "NVST transport timeout counters: pings={pings_sent}, inbound={inbound_datagrams}, stunHandled={handled_stun}, stunInvalid={invalid_stun}, nonStun={non_stun}, wrongSource={wrong_source}" + ); + if inbound_datagrams == 0 { + opennow_streamer_protocol::log::diagnostic( + "WARN", + "transport", + "NVST video UDP path received no datagrams; audio/control may still work. Check firewall/VPN routing and UDP port ownership. No video has reached SRTP authentication or decoding.", + ); + } + } + let packets_stalled = timeout.is_some(); + forward_optional(&event_sender, timeout); + if !packets_stalled { + forward_frame_progress(&mut receiver, &event_sender, now); + } + } +} + +fn forward_frame_progress( + receiver: &mut NvstVideoReceiver, + event_sender: &Sender, + now: Instant, +) { + if let Some(event) = receiver.poll_frame_progress( + now, + NvstFrameProgressPolicy { + stall: receiver.config.timeout, + keyframe_grace: receiver.config.timeout, + }, + ) { + match event { + NvstFrameProgressEvent::KeyframeRequested { + idle_for, + last_assembled_frame_index, + } => { + opennow_streamer_protocol::log::log_async( + "WARN", + "nvst-video", + &format!("produced-frame stall idle_for={idle_for:?}; requested a keyframe"), + ); + let _ = event_sender.send(NvstReceiveEvent::FrameProgressStall { + idle_for, + last_assembled_frame_index, + recovery_required: false, + }); + } + NvstFrameProgressEvent::RecoveryNeeded { + idle_for, + last_assembled_frame_index, + } => { + let _ = event_sender.send(NvstReceiveEvent::FrameProgressStall { + idle_for, + last_assembled_frame_index, + recovery_required: true, + }); + let _ = event_sender.send(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::FrameProgress { + idle_for, + last_assembled_frame_index, + }, + )); + } + } + } +} + +fn forward_optional(sender: &Sender, event: Option) { + if let Some(event) = event { + let _ = sender.send(event); + } +} + +fn forward_receive_event( + media_consumer: &MediaConsumer, + event_sender: &Sender, + feedback: &SharedNvstFeedback, + transport_origin: Instant, + delivered_at: Instant, + delivery_gap: &mut bool, + event: NvstReceiveEvent, +) -> bool { + if let NvstReceiveEvent::Frame(frame) = event { + let media_frame = EncodedMediaFrame { + mid: "nvst-video-0".to_owned(), + codec: frame.codec.label().to_owned(), + payload: Arc::from(frame.bytes), + frame_index: Some(frame.frame_index), + rtp_timestamp: u64::from(frame.timestamp), + clock_rate_hz: 90_000, + channels: None, + received_at_us: transport_origin + .elapsed() + .as_micros() + .try_into() + .unwrap_or(u64::MAX), + keyframe: frame.keyframe, + contiguous: frame.contiguous && !*delivery_gap, + ssrc: None, + }; + let frame_index = frame.frame_index; + feedback.publish_assembled_frame(frame_index, delivered_at); + let (reason, keep_running) = match deliver_media_frame(media_consumer, media_frame) { + Ok(()) => { + *delivery_gap = false; + return true; + } + Err(TransportError::MediaConsumerBackpressured) => { + feedback.retire_undelivered_frame(frame_index); + // This is loss AFTER assembly. Preserve it until a frame really + // reaches the decoder; requesting an IDR alone leaves dependent + // frames decoding against a missing reference in the meantime. + *delivery_gap = true; + (NvstDropReason::MediaConsumerBackpressured, true) + } + Err(TransportError::MediaConsumerClosed) => { + feedback.retire_undelivered_frame(frame_index); + (NvstDropReason::MediaConsumerClosed, false) + } + Err(_) => { + feedback.retire_undelivered_frame(frame_index); + (NvstDropReason::MediaConsumerClosed, false) + } + }; + let _ = event_sender.send(NvstReceiveEvent::Dropped(reason)); + return keep_running; + } + let _ = event_sender.send(event); + true +} + +#[cfg(test)] +mod tests { + static PREFERRED_NVST_PORTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + #[test] + fn stop_causes_report_operation_and_os_code_without_endpoints() { + use super::NvstStopCause; + + let error = std::io::Error::from_raw_os_error(10054); + let cause = NvstStopCause::socket("bundle-receive", &error); + assert_eq!( + cause.to_string(), + format!( + "cause=socket operation=bundle-receive kind={:?} os_code=10054", + error.kind() + ) + ); + let error = std::io::Error::new( + std::io::ErrorKind::ConnectionAborted, + "203.0.113.9:5004 private", + ); + let cause = NvstStopCause::socket("video-ping-send", &error).to_string(); + assert_eq!( + cause, + "cause=socket operation=video-ping-send kind=ConnectionAborted os_code=none" + ); + assert!(!cause.contains("203.0.113.9") && !cause.contains("private")); + assert_eq!( + NvstStopCause::webrtc( + "receive", + &str0m::RtcError::RemoteSdp("203.0.113.9 private".to_owned()) + ) + .to_string(), + "cause=webrtc operation=receive error=sdp" + ); + assert_eq!( + NvstStopCause::InputDeliveryFailed.to_string(), + "cause=input-delivery-failed" + ); + } + + #[test] + fn bundle_liveness_snapshot_reports_idle_ages_per_direction() { + use super::BundleLiveness; + use std::time::{Duration, Instant}; + + let now = Instant::now() + Duration::from_secs(30); + assert_eq!( + BundleLiveness::default().snapshot(now), + "rx_idle_ms=none peer_rx_idle_ms=none tx_idle_ms=none sctp_rx_idle_ms=none ice_state=none port_unreachable=0 channels_closed=0" + ); + let liveness = BundleLiveness { + last_inbound: Some(now - Duration::from_millis(2_100)), + last_peer_inbound: Some(now - Duration::from_millis(2_100)), + last_outbound: Some(now - Duration::from_millis(40)), + last_channel_data: Some(now - Duration::from_millis(9_500)), + ice_state: Some(str0m::IceConnectionState::Completed), + port_unreachable: 3, + channels_closed: 2, + }; + assert_eq!( + liveness.snapshot(now), + "rx_idle_ms=2100 peer_rx_idle_ms=2100 tx_idle_ms=40 sctp_rx_idle_ms=9500 ice_state=Completed port_unreachable=3 channels_closed=2" + ); + } + + #[test] + fn stream_socket_receive_byte_counts_saturate_without_losing_updates() { + use super::{NvstFeedbackState, StreamSocket}; + use std::sync::atomic::Ordering; + + let feedback = NvstFeedbackState::default(); + std::thread::scope(|scope| { + for _ in 0..8 { + let feedback = &feedback; + scope.spawn(move || { + for _ in 0..1_000 { + feedback.record_socket_receive(StreamSocket::Bundle, true, true, 7); + } + }); + } + }); + assert_eq!(feedback.socket_receive_bytes(), 56_000); + + feedback + .bundle_receive_bytes + .store(u64::MAX - 1, Ordering::Relaxed); + feedback.record_socket_receive(StreamSocket::Bundle, true, true, 0); + assert_eq!(feedback.socket_receive_bytes(), u64::MAX - 1); + feedback.record_socket_receive(StreamSocket::Bundle, true, true, 2); + assert_eq!(feedback.socket_receive_bytes(), u64::MAX); + feedback.record_socket_receive(StreamSocket::Bundle, true, true, 1); + assert_eq!(feedback.socket_receive_bytes(), u64::MAX); + } + + #[test] + fn stream_socket_receive_counts_peer_datagram_lengths_once_across_sockets() { + use super::{NvstFeedbackState, StreamSocket}; + + let feedback = NvstFeedbackState::default(); + feedback.record_socket_receive(StreamSocket::Bundle, false, true, 99_000); + feedback.record_socket_receive(StreamSocket::Bundle, true, false, 88_000); + assert_eq!(feedback.socket_receive_bytes(), 0); + + let bundle_control = 1_300; + let bundle_audio = 800; + let video_data = 1_400; + let video_fec = 300; + let video_retransmission = 1_400; + feedback.record_socket_receive(StreamSocket::Bundle, true, true, bundle_control); + feedback.record_socket_receive(StreamSocket::Bundle, true, true, bundle_audio); + feedback.record_socket_receive(StreamSocket::Video, true, true, video_data); + feedback.record_socket_receive(StreamSocket::Video, true, true, video_fec); + feedback.record_socket_receive(StreamSocket::Video, true, true, video_retransmission); + feedback.record_socket_receive(StreamSocket::Video, false, true, 77_000); + assert_eq!(feedback.socket_receive_bytes(), 5_200); + assert_eq!(NvstFeedbackState::default().socket_receive_bytes(), 0); + } + + #[test] + fn typed_text_queue_preserves_order_reservation_and_readiness() { + use opennow_streamer_protocol::text_input::{TextInputError, TextInputSlot}; + let (commands, receiver) = std::sync::mpsc::channel(); + let control = super::NvstUdpReceiverControl { + commands, + input_ready: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + microphone: std::sync::Arc::new(std::sync::Mutex::new(super::MicrophoneQueue::new( + false, + ))), + }; + let slot = TextInputSlot::default(); + assert!(matches!( + control.queue_text(slot.submit(b"paste").unwrap(), 42), + Err(super::TransportError::InputNotReady) + )); + assert!(receiver.try_recv().is_err()); + control + .input_ready + .store(true, std::sync::atomic::Ordering::Release); + control.queue_input(vec![1, 2], false).unwrap(); + control + .queue_text(slot.submit("世界".as_bytes()).unwrap(), 42) + .unwrap(); + control.queue_input(vec![3, 4], false).unwrap(); + assert_eq!(slot.submit(b"again"), Err(TextInputError::Busy)); + assert!( + matches!(receiver.try_recv().unwrap(), super::UdpReceiverCommand::SendInput { bytes, .. } if bytes == [1, 2]) + ); + let super::UdpReceiverCommand::SendText { text, timestamp_us } = + receiver.try_recv().unwrap() + else { + panic!("missing text command") + }; + assert_eq!(text.as_str(), "世界"); + assert_eq!(timestamp_us, 42); + slot.cancel(); + assert!(text.is_cancelled()); + drop(text); + assert!( + matches!(receiver.try_recv().unwrap(), super::UdpReceiverCommand::SendInput { bytes, .. } if bytes == [3, 4]) + ); + drop(receiver); + assert!(matches!( + control.queue_text(slot.submit(b"paste").unwrap(), 42), + Err(super::TransportError::Closed) + )); + assert!(slot.submit(b"released").is_ok()); + } + + mod audio_tests { + include!("nvst_audio_tests.rs"); + } + + mod ping_tests { + include!("nvst_ping_tests.rs"); + } + mod recovery_tests { + include!("nvst_recovery_tests.rs"); + } + mod progress_tests { + include!("nvst_progress_tests.rs"); + } + mod qos_tests { + include!("nvst_qos_tests.rs"); + } + mod budget_tests { + include!("nvst_budget_tests.rs"); + } + mod server_control_tests { + include!("nvst_server_control_tests.rs"); + } + mod nack_tests { + include!("nvst_nack_tests.rs"); + } + use super::*; + use serde_json::json; + + const TEST_KEY: &str = "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"; + const TEST_SALT: &str = "000102030405060708090A0B0C0D"; + const TEST_PEER: &str = "192.0.2.20"; + + #[test] + fn asynchronous_discrete_input_write_rejection_requests_terminal_shutdown() { + for input_type in [3_u32, 4, 8, 9, 10] { + let mut packet = vec![0; if input_type == 10 { 22 } else { 18 }]; + packet[..4].copy_from_slice(&input_type.to_le_bytes()); + let mut codec = NvstInputCodec::default(); + let mut delivery = NvstInputDelivery::default(); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut writes = 0; + assert!(send_nvst_captured_input( + &mut codec, + &mut delivery, + &packet, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |message| { + writes += 1; + assert_eq!( + message.route, + super::super::nvst_input::NvstInputRoute::ControlReliable + ); + NvstInputWrite::Deferred + }, + )); + assert_eq!(writes, 1); + assert!(ready.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + let mut retry_writes = 0; + delivery + .flush(|_| { + retry_writes += 1; + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(retry_writes, 1); + } + } + + #[test] + fn asynchronous_motion_write_rejection_remains_nonterminal() { + for (input_type, length) in [(5_u32, 26), (7, 22)] { + let mut packet = vec![0; length]; + packet[..4].copy_from_slice(&input_type.to_le_bytes()); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut writes = 0; + assert!(send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &packet, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| { + writes += 1; + NvstInputWrite::Deferred + }, + )); + assert_eq!(writes, 1); + assert!(ready.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + } + } + + #[test] + fn synchronous_input_write_rejection_preserves_error_reply() { + let mut packet = vec![0; 18]; + packet[..4].copy_from_slice(&4_u32.to_le_bytes()); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let (reply, result) = mpsc::sync_channel(1); + assert!(!send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &packet, + 1, + Some(reply), + &ready, + &events, + &AtomicBool::new(false), + |_| NvstInputWrite::Closed, + )); + assert!(matches!( + result.try_recv().unwrap(), + Err(TransportError::Closed) + )); + assert!(!ready.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::InputUnavailable(_) + )); + } + + #[test] + fn asynchronous_discrete_input_success_keeps_session_ready() { + let mut packet = vec![0; 18]; + packet[..4].copy_from_slice(&4_u32.to_le_bytes()); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut writes = 0; + let warning = AtomicBool::new(true); + assert!(send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &packet, + 1, + None, + &ready, + &events, + &warning, + |_| { + writes += 1; + NvstInputWrite::Sent + }, + )); + assert_eq!(writes, 1); + assert!(ready.load(Ordering::Acquire)); + assert!(!warning.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::UserInput + )); + assert!(received.try_recv().is_err()); + } + + #[test] + fn deferred_input_clears_idle_warning_only_when_it_is_written_and_only_once() { + let mut packet = vec![0; 18]; + packet[..4].copy_from_slice(&4_u32.to_le_bytes()); + let mut delivery = NvstInputDelivery::default(); + let ready = AtomicBool::new(true); + let warning = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let (reply, result) = mpsc::sync_channel(1); + assert!(send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut delivery, + &packet, + 1, + Some(reply), + &ready, + &events, + &warning, + |_| NvstInputWrite::Deferred, + )); + assert!(result.try_recv().unwrap().is_ok()); + assert!(warning.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + delivery + .flush(|message| { + report_nvst_user_input(message, NvstInputWrite::Sent, &warning, &events); + NvstInputWrite::Sent + }) + .unwrap(); + assert!(!warning.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::UserInput + )); + assert!(send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut delivery, + &packet, + 2, + None, + &ready, + &events, + &warning, + |_| NvstInputWrite::Sent, + )); + assert!(received.try_recv().is_err()); + } + + #[test] + fn heartbeat_empty_gamepad_and_nonmeaningful_success_do_not_clear_idle_warning() { + let warning = AtomicBool::new(true); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&12_u32.to_le_bytes()); + let mut haptics = vec![0; 6]; + haptics[..4].copy_from_slice(&13_u32.to_le_bytes()); + let mut locks = vec![0; 5]; + locks[..4].copy_from_slice(&19_u32.to_le_bytes()); + for packet in [2_u32.to_le_bytes().to_vec(), gamepad, haptics, locks] { + assert!(send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &packet, + 1, + None, + &ready, + &events, + &warning, + |_| NvstInputWrite::Sent, + )); + assert!(warning.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + } + } + + #[test] + fn exhausted_pending_input_requests_shutdown_even_with_a_synchronous_reply() { + let mut packet = vec![0; 18]; + packet[..4].copy_from_slice(&4_u32.to_le_bytes()); + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + for _ in 0..256 { + assert!(send_nvst_captured_input( + &mut codec, + &mut delivery, + &packet, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| NvstInputWrite::Deferred, + )); + } + let (reply, result) = mpsc::sync_channel(1); + assert!(!send_nvst_captured_input( + &mut codec, + &mut delivery, + &packet, + 1, + Some(reply), + &ready, + &events, + &AtomicBool::new(false), + |_| NvstInputWrite::Deferred, + )); + assert!(matches!( + result.try_recv().unwrap(), + Err(TransportError::Closed) + )); + assert!(!ready.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::InputUnavailable(_) + )); + } + + #[test] + fn motion_does_not_hide_a_closed_input_transport() { + let mut motion = vec![0; 22]; + motion[..4].copy_from_slice(&7_u32.to_le_bytes()); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + assert!(!send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &motion, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| NvstInputWrite::Closed, + )); + assert!(!ready.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::InputUnavailable(_) + )); + } + + #[test] + fn asynchronous_malformed_input_requests_shutdown_without_channel_write() { + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + assert!(!send_nvst_captured_input( + &mut NvstInputCodec::default(), + &mut NvstInputDelivery::default(), + &[4, 0, 0, 0], + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| panic!("malformed input must not reach the channel"), + )); + assert!(!ready.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::InputUnavailable(_) + )); + } + + #[test] + fn input_write_rejection_restores_gamepad_codec_after_partial_batch() { + let mut packet = vec![0; 38]; + packet[..4].copy_from_slice(&12_u32.to_le_bytes()); + packet[4..6].copy_from_slice(&26_u16.to_le_bytes()); + packet[8..10].copy_from_slice(&0x0101_u16.to_le_bytes()); + let expected = NvstInputCodec::default().encode(&packet, 1).unwrap(); + assert_eq!(expected.len(), 2); + let mut codec = NvstInputCodec::default(); + let mut delivery = NvstInputDelivery::default(); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut writes = 0; + assert!(send_nvst_captured_input( + &mut codec, + &mut delivery, + &packet, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| { + writes += 1; + if writes == 1 { + NvstInputWrite::Sent + } else { + NvstInputWrite::Deferred + } + }, + )); + assert_eq!(writes, 2); + assert!(ready.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + let mut retried = Vec::new(); + delivery + .flush(|message| { + retried.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(retried, expected[1..]); + let next = codec.encode(&packet, 1).unwrap(); + assert_eq!(next.len(), 1); + assert_ne!(next[0], expected[1]); + } + + #[test] + fn input_encoding_rejection_restores_gamepad_codec_after_partial_mutation() { + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&12_u32.to_le_bytes()); + gamepad[4..6].copy_from_slice(&26_u16.to_le_bytes()); + gamepad[8..10].copy_from_slice(&0x0101_u16.to_le_bytes()); + let expected = NvstInputCodec::default().encode(&gamepad, 1).unwrap(); + let mut packet = vec![0x23]; + packet.extend_from_slice(&1_u64.to_be_bytes()); + for event in [&gamepad[..], &4_u32.to_le_bytes()[..]] { + packet.push(0x21); + packet.extend_from_slice(&(event.len() as u16).to_be_bytes()); + packet.extend_from_slice(event); + } + let mut codec = NvstInputCodec::default(); + let ready = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + assert!(!send_nvst_captured_input( + &mut codec, + &mut NvstInputDelivery::default(), + &packet, + 1, + None, + &ready, + &events, + &AtomicBool::new(false), + |_| panic!("invalid batch must not reach the channel"), + )); + assert!(!ready.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::InputUnavailable(_) + )); + assert_eq!(codec.encode(&gamepad, 1).unwrap(), expected); + } + + #[test] + fn startup_keyframe_control_request_does_not_wait_for_video_ssrc() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + let previous = now - IDR_RETRY_INTERVAL; + feedback.request_keyframe(); + assert_eq!(feedback.stream_snapshot(), None); + assert_eq!( + feedback.keyframe_request_routes(now, previous, false, false), + (false, false) + ); + assert!(feedback.keyframe_request_pending()); + assert_eq!( + feedback.keyframe_request_routes(now, previous, true, false), + (false, false) + ); + assert_eq!( + feedback.keyframe_request_routes(now, previous, false, true), + (false, true) + ); + assert_eq!( + feedback.keyframe_request_routes(now, previous, true, true), + (false, true) + ); + feedback.publish_stream(7, 10, 90_000, now); + assert_eq!( + feedback.keyframe_request_routes(now, previous, true, true), + (true, true) + ); + assert_eq!( + feedback.keyframe_request_routes(now, previous, true, false), + (true, false) + ); + } + + #[test] + fn healthy_startup_does_not_request_idr() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + assert!(!feedback.keyframe_request_pending()); + assert_eq!( + feedback.keyframe_request_routes(now, now - IDR_RETRY_INTERVAL, true, true,), + (false, false) + ); + } + + #[test] + fn mjolnir_recovery_uses_control_idr_without_rtcp_pli() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.publish_stream(7, 10, 90_000, now); + feedback.request_keyframe(); + let previous = now - IDR_RETRY_INTERVAL; + assert_eq!( + feedback.keyframe_request_routes( + now, + previous, + uses_rtcp_video_feedback(true, true), + true, + ), + (false, true) + ); + assert_eq!( + feedback.keyframe_request_routes( + now, + previous, + uses_rtcp_video_feedback(false, true), + true, + ), + (true, true) + ); + } + + #[test] + fn network_telemetry_preserves_rtcp_interval_and_reports_real_loss() { + let feedback = NvstFeedbackState::default(); + assert_eq!(feedback.network_metrics(), None); + let now = Instant::now(); + feedback.publish_stream(7, 100, 90_000, now); + feedback.publish_stream(7, 102, 93_000, now + Duration::from_millis(40)); + let first = feedback.network_metrics().unwrap(); + assert!(first.0 > 0.0); + assert!((first.1 - 100.0 / 3.0).abs() < 0.001); + assert_eq!(feedback.network_metrics(), Some(first)); + let report = feedback.report_snapshot(true).unwrap(); + assert_eq!(report.cumulative_lost, 1); + assert!(report.fraction_lost > 0); + } + + fn network_counters(highest: u32, received: u32) -> NetworkCounters { + NetworkCounters { + ssrc: 7, + base: 100, + highest, + received, + } + } + + #[test] + fn recent_network_loss_recovers_on_the_next_healthy_interval() { + let mut recent = RecentNetworkMetrics::default(); + let now = Instant::now(); + assert_eq!(recent.sample(now, network_counters(100, 1)), None); + assert_eq!( + recent.sample(now + Duration::from_secs(1), network_counters(110, 6)), + Some(50.0) + ); + for second in 2..=4 { + assert_eq!( + recent.sample( + now + Duration::from_secs(u64::from(second)), + network_counters(100 + second * 10, second * 10 - 4), + ), + Some(0.0) + ); + } + } + + #[test] + fn recent_network_sampling_is_bounded_and_does_not_manufacture_intervals() { + let mut recent = RecentNetworkMetrics::default(); + let now = Instant::now(); + for millis in 0..=10_000 { + let result = recent.sample( + now + Duration::from_millis(u64::from(millis)), + network_counters(100 + millis, 1 + millis), + ); + assert_eq!(result, (millis > 0 && millis % 1000 == 0).then_some(0.0)); + assert_eq!( + recent.previous.unwrap().0, + now + Duration::from_secs(u64::from(millis / 1000)) + ); + } + let previous = recent.previous; + for second in 10..=20 { + assert_eq!( + recent.sample( + now + Duration::from_secs(second), + network_counters(10_100, 10_001) + ), + None + ); + assert_eq!(recent.previous, previous); + } + assert_eq!( + recent.sample( + now + Duration::from_secs(21), + network_counters(10_110, 10_011) + ), + None + ); + assert_eq!(recent.previous.unwrap().0, now + Duration::from_secs(21)); + assert_eq!( + recent.sample( + now + Duration::from_secs(22), + network_counters(10_120, 10_021) + ), + Some(0.0) + ); + } + + #[test] + fn recent_network_sampling_rejects_stale_and_nonadvancing_times() { + let mut recent = RecentNetworkMetrics::default(); + let now = Instant::now(); + assert_eq!(recent.sample(now, network_counters(100, 1)), None); + assert_eq!(recent.sample(now, network_counters(110, 11)), None); + assert_eq!( + recent.sample(now - Duration::from_secs(1), network_counters(110, 11)), + None + ); + assert_eq!(recent.previous, Some((now, network_counters(100, 1)))); + assert_eq!( + recent.sample( + now + RECENT_NETWORK_MAX_SAMPLE_GAP + Duration::from_millis(1), + network_counters(110, 11) + ), + None + ); + assert_eq!(recent.previous.unwrap().1, network_counters(110, 11)); + } + + #[test] + fn recent_network_loss_clamps_late_recovery_and_requires_traffic() { + let mut recent = RecentNetworkMetrics::default(); + let now = Instant::now(); + assert_eq!(recent.sample(now, network_counters(110, 6)), None); + assert_eq!( + recent.sample(now + Duration::from_secs(1), network_counters(110, 11)), + None + ); + assert_eq!( + recent.sample(now + Duration::from_secs(2), network_counters(120, 26)), + Some(0.0) + ); + let previous = recent.previous; + assert_eq!( + recent.sample(now + Duration::from_secs(3), network_counters(130, 26)), + None + ); + assert_eq!(recent.previous, previous); + } + + #[test] + fn recent_network_sampling_resets_on_stream_and_counter_epoch_changes() { + let now = Instant::now(); + for replacement in [ + NetworkCounters { + ssrc: 8, + ..network_counters(120, 16) + }, + NetworkCounters { + base: 101, + ..network_counters(120, 16) + }, + network_counters(105, 16), + network_counters(120, 2), + ] { + let mut recent = RecentNetworkMetrics::default(); + recent.sample(now, network_counters(100, 1)); + assert_eq!( + recent.sample(now + Duration::from_secs(1), network_counters(110, 6)), + Some(50.0) + ); + assert_eq!( + recent.sample(now + Duration::from_secs(2), replacement), + None + ); + assert_eq!( + recent.previous, + Some((now + Duration::from_secs(2), replacement)) + ); + assert_eq!( + recent.sample( + now + Duration::from_secs(3), + NetworkCounters { + highest: replacement.highest + 10, + received: replacement.received + 10, + ..replacement + } + ), + Some(0.0) + ); + } + } + + #[test] + fn recent_network_sampling_clears_unbound_and_invalid_counters() { + let now = Instant::now(); + for invalid in [ + NetworkCounters { + ssrc: 0, + ..network_counters(110, 6) + }, + NetworkCounters { + base: u32::MAX, + ..network_counters(110, 6) + }, + network_counters(99, 6), + ] { + let mut recent = RecentNetworkMetrics::default(); + recent.sample(now, network_counters(100, 1)); + assert_eq!(recent.sample(now + Duration::from_secs(1), invalid), None); + assert!(recent.previous.is_none()); + } + } + + #[test] + fn recent_network_telemetry_preserves_cumulative_metrics_and_rtcp_intervals() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + assert_eq!(feedback.recent_network_metrics(now), None); + feedback.publish_stream(7, 100, 90_000, now); + assert_eq!(feedback.recent_network_metrics(now), None); + assert_eq!(feedback.report_snapshot(true).unwrap().fraction_lost, 0); + feedback.publish_stream(7, 102, 93_000, now + Duration::from_secs(1)); + let recent = feedback + .recent_network_metrics(now + Duration::from_secs(1)) + .unwrap(); + assert!(recent.0 > 0.0); + assert_eq!(recent.1, 50.0); + assert_eq!(*feedback.report_prior.lock().unwrap(), (1, 1)); + assert_eq!( + feedback.recent_network_metrics(now + Duration::from_secs(1)), + None + ); + let cumulative = feedback.network_metrics().unwrap(); + assert_eq!(recent.0, cumulative.0); + assert!((cumulative.1 - 100.0 / 3.0).abs() < 0.001); + let report = feedback.report_snapshot(true).unwrap(); + assert_eq!(report.fraction_lost, 128); + assert_eq!(report.cumulative_lost, 1); + for second in 2..=4 { + let at = now + Duration::from_secs(u64::from(second)); + feedback.publish_stream(7, 101 + second, 90_000 + second * 3_000, at); + let recent = feedback.recent_network_metrics(at).unwrap(); + assert_eq!(recent.1, 0.0); + } + assert!(feedback.network_metrics().unwrap().1 > 0.0); + assert_eq!(*feedback.report_prior.lock().unwrap(), (3, 2)); + assert_eq!(feedback.report_snapshot(true).unwrap().fraction_lost, 0); + } + + #[test] + fn keyframe_retries_are_bounded_and_stop_only_on_assembled_keyframe() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_keyframe(); + feedback.publish_stream(7, 10, 90_000, now); + assert_eq!( + feedback.keyframe_request_routes(now, now - IDR_RETRY_INTERVAL, true, true), + (true, true) + ); + // Queue success is not completion. The same cooldown applies if the + // channel refused the write; neither outcome consumes the request. + assert!(feedback.keyframe_request_pending()); + assert_eq!( + feedback.keyframe_request_routes(now, now, true, true), + (false, false) + ); + assert_eq!( + feedback.keyframe_request_routes( + now + IDR_RETRY_INTERVAL - Duration::from_millis(1), + now, + true, + true + ), + (false, false) + ); + let retry_at = now + IDR_RETRY_INTERVAL; + assert_eq!( + feedback.keyframe_request_routes(retry_at, now, true, true), + (true, true) + ); + let mut frame = EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 90_000, + frame_index: 42, + first_stream_packet_index: 10, + frame_type: None, + keyframe: false, + contiguous: true, + bytes: vec![0, 0, 0, 1, 0x41], + }; + feedback.publish_completed_frame(&frame); + assert!(feedback.keyframe_request_pending()); + frame.keyframe = true; + frame.bytes[4] = 0x65; + feedback.publish_completed_frame(&frame); + assert!(!feedback.keyframe_request_pending()); + assert_eq!( + feedback.keyframe_request_routes(retry_at, now, true, true), + (false, false) + ); + // Subsequent loss or decoder rejection must re-arm recovery. + feedback.request_keyframe(); + assert_eq!( + feedback.keyframe_request_routes(retry_at, now, true, true), + (true, true) + ); + } + + #[test] + fn video_delivery_preserves_the_server_frame_index_for_feedback() { + let (media_consumer, media_receiver) = std::sync::mpsc::sync_channel(1); + let (event_sender, _event_receiver) = std::sync::mpsc::channel(); + let feedback: SharedNvstFeedback = Arc::new(NvstFeedbackState::default()); + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut false, + NvstReceiveEvent::Frame(EncodedVideoAccessUnit { + codec: NvstVideoCodec::Av1, + timestamp: 90_000, + frame_index: 2_417, + first_stream_packet_index: 90, + frame_type: None, + keyframe: false, + contiguous: true, + bytes: vec![0x12, 0], + }), + )); + let delivered = media_receiver.recv().expect("video frame"); + assert_eq!(delivered.frame_index, Some(2_417)); + assert_eq!(feedback.frame_stage_timings().pending_deliveries, 1); + } + + #[test] + fn consumer_admission_before_the_assembly_call_returns_still_matches() { + const FRAMES: u32 = 64; + let (media_consumer, media_receiver) = + std::sync::mpsc::sync_channel(usize::try_from(FRAMES).expect("frame count")); + let (event_sender, _event_receiver) = std::sync::mpsc::channel(); + let feedback: SharedNvstFeedback = Arc::new(NvstFeedbackState::default()); + let consumer_feedback = Arc::clone(&feedback); + std::thread::scope(|scope| { + scope.spawn(move || { + for _ in 0..FRAMES { + let frame: EncodedMediaFrame = media_receiver.recv().expect("video frame"); + consumer_feedback.publish_accepted_frame( + frame.frame_index.expect("sender frame index"), + frame.payload.len() as u32, + Instant::now(), + ); + } + }); + for frame_index in 1..=FRAMES { + let mut delivery_gap = false; + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut delivery_gap, + NvstReceiveEvent::Frame(EncodedVideoAccessUnit { + codec: NvstVideoCodec::H265, + timestamp: frame_index * 3_000, + frame_index, + first_stream_packet_index: frame_index, + frame_type: None, + keyframe: frame_index == 1, + contiguous: true, + bytes: vec![0, 0, 0, 1, 0x26], + }), + )); + } + }); + let timings = feedback.frame_stage_timings(); + assert_eq!( + timings.unmatched_admissions, 0, + "an admission that lands before the assembly call returns must still match" + ); + assert_eq!(timings.admitted_frames_total, u64::from(FRAMES)); + assert_eq!(timings.pending_deliveries, 0); + let delivery = timings + .delivery_to_admission + .expect("every delivery produced a matched sample"); + assert!(delivery.max_ms >= 0.0); + } + + #[test] + fn transient_video_consumer_backpressure_does_not_stop_receiver() { + let (media_consumer, media_receiver) = std::sync::mpsc::sync_channel(1); + let (event_sender, event_receiver) = std::sync::mpsc::channel(); + let feedback: SharedNvstFeedback = Arc::new(NvstFeedbackState::default()); + let mut delivery_gap = false; + let frame = |frame_index: u32| { + NvstReceiveEvent::Frame(EncodedVideoAccessUnit { + codec: NvstVideoCodec::H265, + timestamp: frame_index * 3_000, + frame_index, + first_stream_packet_index: frame_index, + frame_type: None, + keyframe: true, + contiguous: true, + bytes: vec![0, 0, 0, 1, 0x26], + }) + }; + + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut delivery_gap, + frame(1), + )); + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut delivery_gap, + frame(2), + )); + assert!(matches!( + event_receiver.recv().expect("backpressure event"), + NvstReceiveEvent::Dropped(NvstDropReason::MediaConsumerBackpressured) + )); + assert!(delivery_gap); + assert!(media_receiver.recv().unwrap().contiguous); + let timings = feedback.frame_stage_timings(); + assert_eq!( + timings.pending_deliveries, 1, + "the delivered frame is still awaiting admission feedback" + ); + assert_eq!(timings.assembled_frames_total, 2); + assert_eq!( + timings.undelivered_frames_total, 1, + "an assembled frame the consumer refused is counted, not left pending" + ); + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut delivery_gap, + frame(3), + )); + assert!(!media_receiver.recv().unwrap().contiguous); + assert!(!delivery_gap); + assert!(forward_receive_event( + &media_consumer, + &event_sender, + &feedback, + Instant::now(), + Instant::now(), + &mut delivery_gap, + frame(4), + )); + assert!(media_receiver.recv().unwrap().contiguous); + } + + fn legacy_handoff() -> Value { + json!({ + "clientUdpPort": 49005, + "videoPeerIp": TEST_PEER, + "videoPeerPort": 5004, + "srtpAesKeyHex": TEST_KEY, + "srtpSaltHex": "00000000000000009ECA935E", + "codec": "H264", + "rtpPayloadType": 96, + "rtpSsrc": 0x11223344u32, + "reorderWindowPackets": 4, + "maxAccessUnitBytes": 4096, + "timeoutMs": 500 + }) + } + + fn config() -> NvstVideoConfig { + NvstVideoConfig::from_legacy_handoff(&legacy_handoff(), None).expect("valid config") + } + + #[test] + fn recovery_policy_handoff_is_optional_typed_and_preserved() { + assert_eq!( + config().recovery_policy, + crate::NvstRecoveryPolicy::default() + ); + let mut handoff = legacy_handoff(); + handoff["recoveryPolicy"] = json!({"refPicInvalidation":false,"streamRecoveryType":2,"framesWithInvalidRefThreshold":4}); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.recovery_policy.ref_pic_invalidation, Some(false)); + assert_eq!(config.recovery_policy.stream_recovery_type, Some(2)); + handoff["recoveryPolicy"] = json!({"refPicInvalidation":"1"}); + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + } + + #[test] + fn readiness_requires_dtls_or_authenticated_peer_media() { + let feedback = NvstFeedbackState::default(); + assert!(!feedback.authenticated_transport_ready()); + feedback.record_socket_receive(StreamSocket::Video, false, true, 10); + feedback.record_socket_receive(StreamSocket::Video, true, false, 10); + assert!(!feedback.authenticated_transport_ready()); + feedback.record_socket_receive(StreamSocket::Video, true, true, 10); + assert!(feedback.authenticated_transport_ready()); + let feedback = NvstFeedbackState::default(); + feedback + .authenticated_transport_ready + .store(true, Ordering::Release); + assert!(feedback.authenticated_transport_ready()); + } + + #[test] + fn qos_handoff_rejects_versions_without_encoders_and_selects_v5_timings() { + let mut handoff = legacy_handoff(); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .qos_timings_v5 + ); + + for (field, unsupported) in [ + ("qosFeedbackVersion", 6), + ("qosTimingsVersion", 4), + ("qosBlobStatsVersion", 8), + ] { + handoff[field] = serde_json::json!(unsupported); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { field: rejected }) if rejected == field + )); + handoff.as_object_mut().unwrap().remove(field); + } + handoff["qosFeedbackVersion"] = serde_json::json!(7); + handoff["qosTimingsVersion"] = serde_json::json!(5); + handoff["qosBlobStatsVersion"] = serde_json::json!(9); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .qos_timings_v5 + ); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .frame_pacing_feedback + ); + handoff["framePacingFeedbackMode"] = serde_json::json!(1); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .frame_pacing_feedback + ); + handoff["framePacingFeedbackMode"] = serde_json::json!(0); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .frame_pacing_feedback + ); + handoff["framePacingFeedbackMode"] = serde_json::json!(2); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { + field: "framePacingFeedbackMode" + }) + )); + handoff + .as_object_mut() + .unwrap() + .remove("framePacingFeedbackMode"); + handoff["qosFeedbackVersion"] = serde_json::json!(8); + handoff["qosTimingsVersion"] = serde_json::json!(6); + handoff["qosBlobStatsVersion"] = serde_json::json!(10); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .qos_timings_v5 + ); + handoff["qosBlobStatsVersion"] = serde_json::json!("9"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidFieldType { + field: "qosBlobStatsVersion", + .. + }) + )); + } + + #[test] + fn qos_message_size_handoff_defaults_clamps_and_rejects_unusable_values() { + let mut handoff = legacy_handoff(); + assert_eq!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .max_control_report_bytes, + MAX_CONTROL_REPORT_BYTES + ); + for (offered, selected) in [(106, 106), (212, 212), (1071, 1071), (4000, 1071)] { + handoff["maxQosMessagesSize"] = serde_json::json!(offered); + assert_eq!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .max_control_report_bytes, + selected + ); + } + for offered in [0, 56, 105] { + handoff["maxQosMessagesSize"] = serde_json::json!(offered); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { + field: "maxQosMessagesSize" + }) + )); + } + handoff["maxQosMessagesSize"] = serde_json::json!("212"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidFieldType { + field: "maxQosMessagesSize", + .. + }) + )); + } + + fn peer() -> SocketAddr { + SocketAddr::new(TEST_PEER.parse().expect("test IP"), 5004) + } + + fn stun_credentials() -> NvstStunCredentials { + NvstStunCredentials { + local_username_fragment: "loc1".to_owned(), + local_password: "local-password-value-01".to_owned(), + remote_username_fragment: "remote01".to_owned(), + remote_password: "remote-password-with-36-byte-value-001".to_owned(), + } + } + + fn build_plaintext_rtp_with_fec_blocks( + sequence: u16, + flags: u8, + frame_index: u32, + fec_current_block: u8, + fec_last_block: u8, + media: &[u8], + ) -> Vec { + let mut packet = vec![0x90, 0xe0]; + packet.extend_from_slice(&sequence.to_be_bytes()); + packet.extend_from_slice(&0x01020304u32.to_be_bytes()); + packet.extend_from_slice(&0x11223344u32.to_be_bytes()); + packet.extend_from_slice(&GS_VIDEO_EXTENSION_PROFILE.to_be_bytes()); + packet.extend_from_slice(&4_u16.to_be_bytes()); + packet.extend_from_slice(&(u32::from(sequence) << 8).to_le_bytes()); + packet.extend_from_slice(&frame_index.to_le_bytes()); + packet.push(flags); + packet.extend_from_slice(&[ + 0, + 0, + (fec_current_block & 0x03) << 4 | (fec_last_block & 0x03) << 6, + ]); + packet.extend_from_slice(&[0; 4]); + packet.extend_from_slice(media); + packet + } + + fn build_plaintext_rtp(sequence: u16, flags: u8, frame_index: u32, media: &[u8]) -> Vec { + build_plaintext_rtp_with_fec_blocks(sequence, flags, frame_index, 0, 0, media) + } + + fn test_srtp(config: &NvstVideoConfig) -> SrtpReceiver { + SrtpReceiver::from_material(&config.srtp) + } + + fn protect_for_test(crypto: &SrtpReceiver, mut packet: Vec, roc: u32) -> Vec { + let header = RtpHeader::parse(&packet).expect("RTP header"); + let packet_index = (u64::from(roc) << 16) | u64::from(header.sequence_number); + match &crypto.cipher { + SrtpCipher::AeadAes128Gcm { + encryption_key, + session_salt, + authentication_tag_len, + } => { + let iv = srtp_gcm_iv(*session_salt, header.ssrc, roc, header.sequence_number); + protect_aes_gcm( + &mut packet, + header.payload_offset, + encryption_key, + &iv, + *authentication_tag_len, + ); + } + SrtpCipher::AeadAes256Gcm { + encryption_key, + session_salt, + authentication_tag_len, + } => { + let iv = srtp_gcm_iv(*session_salt, header.ssrc, roc, header.sequence_number); + protect_aes_gcm( + &mut packet, + header.payload_offset, + encryption_key, + &iv, + *authentication_tag_len, + ); + } + SrtpCipher::AesCm128HmacSha1 { + encryption_key, + authentication_key, + session_salt, + authentication_tag_len, + } => { + let iv = srtp_aes_cm_iv(session_salt, header.ssrc, packet_index); + let mut cipher = Aes128Ctr::new(encryption_key.into(), (&iv).into()); + cipher.apply_keystream(&mut packet[header.payload_offset..]); + let mut mac = HmacSha1::new_from_slice(authentication_key).expect("fixed key"); + mac.update(&packet); + mac.update(&roc.to_be_bytes()); + packet.extend_from_slice(&mac.finalize().into_bytes()[..*authentication_tag_len]); + } + SrtpCipher::AesCm256HmacSha1 { + encryption_key, + authentication_key, + session_salt, + authentication_tag_len, + } => { + let iv = srtp_aes_cm_iv(session_salt, header.ssrc, packet_index); + let mut cipher = Aes256Ctr::new(encryption_key.into(), (&iv).into()); + cipher.apply_keystream(&mut packet[header.payload_offset..]); + let mut mac = HmacSha1::new_from_slice(authentication_key).expect("fixed key"); + mac.update(&packet); + mac.update(&roc.to_be_bytes()); + packet.extend_from_slice(&mac.finalize().into_bytes()[..*authentication_tag_len]); + } + } + packet + } + + #[test] + fn legacy_schema_defaults_to_aes_256_gcm_8_with_explicit_salt() { + let config = config(); + assert_eq!(config.video_peer(), peer()); + assert_eq!(config.bundle_peer(), peer()); + assert_eq!(config.srtp_profile(), NvstSrtpProfile::AeadAes256Gcm8); + let NvstSrtpMaterial::AeadAes256Gcm { master_salt, .. } = config.srtp else { + panic!("legacy handoff must choose AES-256-GCM"); + }; + assert_eq!( + master_salt, + decode_fixed_hex::<12>("00000000000000009ECA935E", NvstConfigError::InvalidSrtpSalt) + .expect("salt"), + ); + } + + #[test] + fn handoff_carries_the_negotiated_hid_device_mask() { + let mut handoff = legacy_handoff(); + assert_eq!(config().hid_device_mask(), 0); + handoff["hidDeviceMask"] = json!(0x5); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + assert_eq!(config.hid_device_mask(), 0x5); + handoff["hidDeviceMask"] = json!(u32::MAX); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + assert_eq!(config.hid_device_mask(), u32::MAX); + handoff["hidDeviceMask"] = json!(-1); + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + handoff["hidDeviceMask"] = json!(u64::from(u32::MAX) + 1); + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + handoff["hidDeviceMask"] = json!("5"); + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + } + + #[test] + fn handoff_uses_distinct_routable_bundle_peer_when_supplied() { + let mut handoff = legacy_handoff(); + handoff["videoPeerIp"] = json!("169.254.0.21"); + handoff["bundlePeerIp"] = json!("198.51.100.20"); + handoff["bundlePeerPort"] = json!(5006); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid peers"); + assert_eq!(config.video_peer(), "169.254.0.21:5004".parse().unwrap()); + assert_eq!(config.bundle_peer(), "198.51.100.20:5006".parse().unwrap()); + } + + #[test] + fn handoff_accepts_only_explicit_standard_opus_track_metadata() { + let mut handoff = legacy_handoff(); + handoff["audioTrack"] = json!({ + "payloadType": 97, + "codec": "opus", + "clockRateHz": 48_000, + "channels": 2, + "mid": "audio-main", + "ssrc": 424242, + }); + let config = + NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("standard Opus track"); + let audio = config.audio_track().expect("audio track"); + assert_eq!(audio.payload_type, 97); + assert_eq!(audio.clock_rate_hz, 48_000); + assert_eq!(audio.channels, 2); + assert_eq!(audio.mid, "audio-main"); + assert!(matches_audio_track(audio, 97, 424242)); + assert!(!matches_audio_track(audio, 96, 424242)); + assert!(!matches_audio_track(audio, 97, 7)); + + handoff["audioTrack"]["codec"] = json!("proprietary"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidAudioTrack) + )); + } + + #[test] + fn red_pt63_routes_to_negotiated_opus_pt111_and_extracts_the_primary_block() { + let track = NvstAudioTrack { + payload_type: GFN_OPUS_PAYLOAD_TYPE, + clock_rate_hz: 48_000, + channels: 2, + mid: "audio".to_owned(), + ssrc: Some(42), + }; + assert!(matches_audio_track(&track, GFN_OPUS_PAYLOAD_TYPE, 42)); + assert!(matches_audio_track(&track, GFN_RED_PAYLOAD_TYPE, 42)); + assert!(!matches_audio_track(&track, GFN_RED_PAYLOAD_TYPE, 7)); + + let red_payload = [ + 0x80 | GFN_OPUS_PAYLOAD_TYPE, + 0x0f, + 0x00, + 0x03, + GFN_OPUS_PAYLOAD_TYPE, + 0x11, + 0x22, + 0x33, + 0xf8, + 0xff, + ]; + assert_eq!( + extract_red_opus_primary(&red_payload), + Some(&[0xf8, 0xff][..]) + ); + let red = parse_red_opus(&red_payload).expect("RED payload"); + assert_eq!( + red.redundant, + [RedOpusBlock { + timestamp_offset: 960, + payload: &[0x11, 0x22, 0x33], + }] + ); + assert_eq!(red.primary, &[0xf8, 0xff]); + } + + #[test] + fn red_primary_extraction_rejects_malformed_or_unbounded_payloads() { + assert_eq!(extract_red_opus_primary(&[]), None); + assert_eq!(extract_red_opus_primary(&[110, 0xf8]), None); + assert_eq!(extract_red_opus_primary(&[GFN_OPUS_PAYLOAD_TYPE]), None); + + let mut too_many_blocks = Vec::new(); + for _ in 0..=MAX_REDUNDANT_AUDIO_BLOCKS { + too_many_blocks.extend_from_slice(&[0x80 | GFN_OPUS_PAYLOAD_TYPE, 0, 0, 0]); + } + too_many_blocks.extend_from_slice(&[GFN_OPUS_PAYLOAD_TYPE, 0xf8]); + assert_eq!(extract_red_opus_primary(&too_many_blocks), None); + + let mut oversized_primary = vec![GFN_OPUS_PAYLOAD_TYPE]; + oversized_primary.resize(MAX_OPUS_PACKET_BYTES + 2, 0xf8); + assert_eq!(extract_red_opus_primary(&oversized_primary), None); + } + + #[test] + fn ping_version_six_requires_all_ice_credentials() { + let mut handoff = legacy_handoff(); + handoff["pingVersion"] = json!(6); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::MissingField("localIceUsernameFragment")) + )); + + handoff["localIceUsernameFragment"] = json!("loc1"); + handoff["localIcePassword"] = json!("local-password-value-01"); + handoff["remoteIceUsernameFragment"] = json!("remote01"); + handoff["remoteIcePassword"] = json!("remote-password-with-36-byte-value-001"); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None) + .expect("complete version-six credentials"); + assert_eq!(config.ping_version, Some(6)); + assert!(config.stun_credentials.is_some()); + assert!(!format!("{config:?}").contains("local-password-value-01")); + } + + #[test] + fn bundle_natt_identity_is_optional_bounded_and_role_specific() { + let mut handoff = legacy_handoff(); + handoff["pingVersion"] = json!(6); + handoff["localIceUsernameFragment"] = json!("loc1"); + handoff["localIcePassword"] = json!("local-password-value-01"); + handoff["remoteIceUsernameFragment"] = json!("remote01"); + handoff["remoteIcePassword"] = json!("remote-password-with-36-byte-value-001"); + let previous = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(previous.bundle_natt_remote_username, None); + assert_eq!(previous.ping_payload, b"PING"); + + handoff["bundleNattRemoteUsername"] = json!("server47999"); + let negotiated = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!( + negotiated.bundle_natt_remote_username.as_deref(), + Some("server47999") + ); + assert_eq!(negotiated.ping_payload, previous.ping_payload); + assert!(!format!("{negotiated:?}").contains("server47999")); + for invalid in [ + json!(""), + json!("server:47999"), + json!("server\n47999"), + json!("a".repeat(MAX_ICE_CREDENTIAL_BYTES + 1)), + ] { + handoff["bundleNattRemoteUsername"] = invalid; + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { + field: "bundleNattRemoteUsername" + }) + )); + } + handoff["bundleNattRemoteUsername"] = json!(123); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidFieldType { + field: "bundleNattRemoteUsername", + .. + }) + )); + handoff["bundleNattRemoteUsername"] = json!("server47999"); + handoff["pingVersion"] = json!(5); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { + field: "bundleNattRemoteUsername" + }) + )); + } + + #[test] + fn negotiated_bundle_natt_identity_reaches_udp_without_replacing_video_identity() { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let bundle = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut remote = create_nvst_bundle_rtc(&server).unwrap(); + let mut config = config(); + config.client_udp_port = bundle.local_addr().unwrap().port(); + config.video_peer = server.local_addr().unwrap(); + config.remote_dtls_fingerprint = + Some(nvst_local_bundle_identity(&mut remote).dtls_fingerprint); + config.stun_credentials = Some(stun_credentials()); + config.ping_payload = b"video47998".to_vec(); + config.bundle_natt_remote_username = Some("server47999".to_owned()); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + Some(bundle), + None, + Arc::new(HidRuntime::new()), + None, + ) + .unwrap(); + let mut datagram = [0_u8; 2048]; + let deadline = Instant::now() + Duration::from_secs(2); + loop { + assert!(Instant::now() < deadline, "negotiated bundle NATT probe"); + let (length, _) = server.recv_from(&mut datagram).unwrap(); + let (_, username) = + find_stun_attribute(&datagram[..length], STUN_ATTR_USERNAME).unwrap(); + if username == b"remote01:loc1" { + continue; + } + assert_eq!(username, b"server47999:loc1"); + assert!(valid_stun_fingerprint(&datagram[..length])); + assert!(valid_stun_message_integrity( + &datagram[..length], + stun_credentials().remote_password.as_bytes() + )); + break; + } + session.stop(); + } + + #[test] + fn microphone_requires_explicit_bundle_negotiation_and_dtls() { + let mut handoff = legacy_handoff(); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + handoff["microphoneOnBundle"] = json!(true); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + handoff["pingVersion"] = json!(6); + handoff["localIceUsernameFragment"] = json!("loc1"); + handoff["localIcePassword"] = json!("local-password-value-01"); + handoff["remoteIceUsernameFragment"] = json!("remote01"); + handoff["remoteIcePassword"] = json!("remote-password-with-36-byte-value-001"); + handoff["remoteDtlsFingerprint"] = json!("AA:BB"); + assert!( + NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + handoff["microphoneOnBundle"] = json!(false); + assert!( + !NvstVideoConfig::from_legacy_handoff(&handoff, None) + .unwrap() + .microphone_available() + ); + } + + #[test] + fn microphone_direct_rtp_preserves_downlink_and_sequence_rollover() { + fn exchange( + from: &mut Rtc, + to: &mut Rtc, + now: Instant, + received: &mut Vec, + ) { + loop { + match from.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) => { + to.handle_input(Input::Receive( + now, + Receive { + proto: packet.proto, + source: packet.source, + destination: packet.destination, + contents: packet.contents.as_ref().try_into().unwrap(), + }, + )) + .unwrap(); + } + Output::Event(Event::RtpPacket(packet)) => received.push(packet), + _ => {} + } + } + } + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut local = create_nvst_bundle_rtc(&socket).unwrap(); + let mut remote = create_nvst_bundle_rtc(&socket).unwrap(); + let local_candidate = Candidate::host("192.0.2.1:1000".parse().unwrap(), "udp").unwrap(); + let remote_candidate = Candidate::host("192.0.2.2:2000".parse().unwrap(), "udp").unwrap(); + local.add_local_candidate(local_candidate.clone()); + local.add_remote_candidate(remote_candidate.clone()); + remote.add_local_candidate(remote_candidate); + remote.add_remote_candidate(local_candidate); + let local_identity = local.direct_api().local_dtls_fingerprint().clone(); + let remote_identity = remote.direct_api().local_dtls_fingerprint().clone(); + let local_ice = local.direct_api().local_ice_credentials(); + let remote_ice = remote.direct_api().local_ice_credentials(); + local.direct_api().set_remote_fingerprint(remote_identity); + remote.direct_api().set_remote_fingerprint(local_identity); + local.direct_api().set_remote_ice_credentials(remote_ice); + remote.direct_api().set_remote_ice_credentials(local_ice); + local.direct_api().set_ice_controlling(true); + remote.direct_api().set_ice_controlling(false); + local.direct_api().start_dtls(true).unwrap(); + remote.direct_api().start_dtls(false).unwrap(); + for rtc in [&mut local, &mut remote] { + rtc.direct_api() + .declare_media(Mid::from("0"), MediaKind::Audio); + rtc.direct_api() + .declare_media(Mid::from("2"), MediaKind::Audio); + } + local + .direct_api() + .expect_stream_rx(1.into(), None, Mid::from("0"), None); + remote + .direct_api() + .expect_stream_rx(1.into(), None, Mid::from("2"), None); + let origin = Instant::now(); + let mut microphone_packets = Vec::new(); + let mut downlink_packets = Vec::new(); + for tick in 0..2000 { + let now = origin + Duration::from_millis(tick); + local.handle_input(Input::Timeout(now)).unwrap(); + remote.handle_input(Input::Timeout(now)).unwrap(); + if tick == 1250 { + local + .direct_api() + .stream_tx(&Ssrc::from(1)) + .unwrap() + .write_rtp(str0m::rtp::RtpWrite::new( + 111.into(), + 65536_u64.into(), + 0, + now, + vec![0x00], + )); + local.direct_api().remove_stream_tx(1.into()); + } + if tick == 1000 || tick == 1500 { + assert!(local.is_connected() && remote.is_connected()); + let sequence = if tick == 1000 { 65535_u64 } else { 65537_u64 }; + let timestamp = if tick == 1000 { u32::MAX - 959 } else { 0 }; + local.direct_api().remove_stream_tx(1.into()); + local + .direct_api() + .declare_stream_tx(1.into(), None, Mid::from("2"), None) + .write_rtp(str0m::rtp::RtpWrite::new( + 111.into(), + sequence.into(), + timestamp, + now, + vec![0xf8, 0xff, 0xfe], + )); + remote + .direct_api() + .declare_stream_tx(1.into(), None, Mid::from("0"), None) + .write_rtp(str0m::rtp::RtpWrite::new( + 111.into(), + sequence.into(), + timestamp, + now, + vec![0xf8, 0xff, 0xfe], + )); + } + exchange(&mut local, &mut remote, now, &mut downlink_packets); + exchange(&mut remote, &mut local, now, &mut microphone_packets); + } + for packets in [µphone_packets, &downlink_packets] { + assert_eq!(packets.len(), 2); + assert_eq!(packets[0].header.sequence_number, 65535); + assert_eq!(packets[1].header.sequence_number, 1); + assert_eq!(packets[0].header.timestamp, u32::MAX - 959); + assert_eq!(packets[1].header.timestamp, 0); + for packet in packets { + assert_eq!(*packet.header.ssrc, 1); + assert_eq!(*packet.header.payload_type, 111); + assert_eq!(packet.payload.as_ref(), &[0xf8, 0xff, 0xfe]); + } + } + } + + #[test] + fn reserved_bundle_socket_binds_unspecified_ipv4() { + let socket = reserve_nvst_udp_socket().expect("reserve"); + let addr = socket.local_addr().expect("local"); + assert!( + addr.ip().is_unspecified(), + "official binds 0.0.0.0, advertised NIC IPv4 is separate" + ); + assert_ne!(addr.port(), 0); + } + + #[test] + fn reserved_nvst_pair_places_bundle_immediately_after_video() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + let (bundle, video) = reserve_nvst_socket_pair().expect("reserve pair"); + let bundle_addr = bundle.local_addr().expect("bundle address"); + let video_addr = video.local_addr().expect("video address"); + + assert!(bundle_addr.ip().is_unspecified()); + assert!(video_addr.ip().is_unspecified()); + assert_eq!(bundle_addr.port(), video_addr.port() + 1); + } + + #[test] + fn udp_receive_keeps_timeouts_and_windows_port_unreachable_alive() { + let timeout = std::io::Error::new(std::io::ErrorKind::TimedOut, "poll"); + let interrupted = std::io::Error::new(std::io::ErrorKind::Interrupted, "signal"); + let reset = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "icmp"); + let refused = std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "fatal"); + assert!(udp_receive_is_idle(&timeout)); + assert!(udp_receive_is_idle(&interrupted)); + assert!(udp_receive_is_idle(&reset)); + assert!(!udp_receive_is_idle(&refused)); + assert_eq!(reset.kind(), std::io::ErrorKind::ConnectionReset); + } + + #[test] + fn nvst_udp_socket_survives_a_ping_to_a_closed_peer() { + let socket = bind_nvst_udp_socket(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0).expect("bind"); + socket + .set_read_timeout(Some(Duration::from_millis(50))) + .expect("timeout"); + let closed = { + let probe = UdpSocket::bind("127.0.0.1:0").expect("probe"); + let port = probe.local_addr().expect("probe address").port(); + drop(probe); + SocketAddr::from(([127, 0, 0, 1], port)) + }; + socket.send_to(b"ping", closed).expect("send"); + let mut buffer = [0_u8; 64]; + let error = socket + .recv_from(&mut buffer) + .expect_err("closed peer has no datagram"); + assert!( + matches!( + error.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ), + "ICMP port unreachable must not reset the NVST UDP socket: {error}" + ); + } + + #[cfg(windows)] + #[test] + fn windows_udp_reservation_rejects_port_sharing() { + let owner = reserve_nvst_mjolnir_udp_socket().unwrap(); + let address = owner.local_addr().unwrap(); + let contender = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP)).unwrap(); + contender.set_reuse_address(true).unwrap(); + assert!( + contender.bind(&address.into()).is_err(), + "a sharing socket must not steal the reserved port" + ); + assert!(bind_nvst_udp_socket(address.ip(), address.port()).is_err()); + } + + #[test] + fn icmp_port_unreachable_is_an_idle_udp_receive() { + let reset = + std::io::Error::new(std::io::ErrorKind::ConnectionReset, "icmp port unreachable"); + assert!(udp_icmp_port_unreachable(&reset)); + assert!(udp_receive_is_idle(&reset)); + assert!(udp_receive_is_idle(&std::io::Error::new( + std::io::ErrorKind::TimedOut, + "timeout", + ))); + assert!(udp_receive_is_idle(&std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "would block", + ))); + assert!(!udp_receive_is_idle(&std::io::Error::new( + std::io::ErrorKind::ConnectionAborted, + "aborted", + ))); + assert!(!udp_icmp_port_unreachable(&std::io::Error::new( + std::io::ErrorKind::TimedOut, + "timeout", + ))); + } + + fn spawn_video_with_keepalive_fault( + socket: UdpSocket, + config: NvstVideoConfig, + media_consumer: MediaConsumer, + event_sender: Sender, + send_keepalive: impl FnMut(&UdpSocket, &[u8], SocketAddr) -> std::io::Result + + Send + + 'static, + ) -> NvstUdpReceiverSession { + socket + .set_read_timeout(Some(UDP_RECEIVE_POLL_INTERVAL)) + .unwrap(); + let (commands, receiver) = mpsc::channel(); + let input_ready = Arc::new(AtomicBool::new(false)); + let microphone = Arc::new(Mutex::new(MicrophoneQueue::new(false))); + let outputs = NvstReceiverOutputs { + media_consumer, + event_sender, + input_ready: input_ready.clone(), + microphone: microphone.clone(), + }; + let join = thread::spawn(move || { + run_nvst_video_receiver( + socket, + config, + receiver, + outputs, + Instant::now(), + send_keepalive, + ); + }); + NvstUdpReceiverSession { + commands, + join: Some(join), + input_ready, + microphone, + } + } + + #[test] + fn video_keepalive_single_resource_failure_preserves_receive_and_cadence() { + for (error, authenticated_ping) in [ + (std::io::Error::from_raw_os_error(10055), true), + (std::io::Error::from(std::io::ErrorKind::Interrupted), true), + (std::io::Error::from(std::io::ErrorKind::WouldBlock), true), + (std::io::Error::from_raw_os_error(10055), false), + (std::io::Error::from(std::io::ErrorKind::Interrupted), false), + (std::io::Error::from(std::io::ErrorKind::WouldBlock), false), + ] { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client_address = client.local_addr().unwrap(); + let mut config = config(); + config.video_peer = server.local_addr().unwrap(); + config.stun_credentials = authenticated_ping.then(stun_credentials); + let ping_payload = config.ping_payload.clone(); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 42, + &[0, 0, 1, 0x65], + ), + 0, + ); + server.send_to(&packet, client_address).unwrap(); + let (media_consumer, media_receiver) = mpsc::sync_channel(1); + let (event_sender, event_receiver) = mpsc::channel(); + let attempts = Arc::new(Mutex::new(Vec::new())); + let worker_attempts = attempts.clone(); + let mut first_error = Some(error); + let session = spawn_video_with_keepalive_fault( + client, + config, + media_consumer, + event_sender, + move |socket, packet, peer| { + worker_attempts.lock().unwrap().push(Instant::now()); + if let Some(error) = first_error.take() { + Err(error) + } else { + socket.send_to(packet, peer) + } + }, + ); + let frame = media_receiver.recv_timeout(Duration::from_secs(2)); + assert!( + frame.is_ok(), + "a failed keepalive must not discard queued video: {frame:?}; events={:?}", + event_receiver.try_iter().collect::>() + ); + assert_eq!(frame.unwrap().frame_index, Some(42)); + let mut datagram = [0; 512]; + let deadline = Instant::now() + Duration::from_secs(2); + let (length, source) = loop { + assert!( + Instant::now() < deadline, + "next scheduled keepalive succeeds" + ); + let received = server + .recv_from(&mut datagram) + .expect("next scheduled keepalive succeeds"); + if (authenticated_ping && looks_like_stun(&datagram[..received.0])) + || (!authenticated_ping && datagram[..received.0] == ping_payload) + { + break received; + } + }; + assert_eq!(source, client_address); + if authenticated_ping { + assert!(valid_stun_fingerprint(&datagram[..length])); + } else { + assert_eq!(datagram[..length], ping_payload); + } + session.stop(); + let attempts = attempts.lock().unwrap(); + assert!(attempts.len() >= 2); + assert!( + attempts[1].duration_since(attempts[0]) + >= PING_INTERVAL_BEFORE_CONNECTION - UDP_RECEIVE_POLL_INTERVAL + ); + } + } + + #[test] + fn video_keepalive_persistent_resource_failure_stops_despite_inbound_video() { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client_address = client.local_addr().unwrap(); + let mut config = config(); + config.video_peer = server.local_addr().unwrap(); + config.stun_credentials = Some(stun_credentials()); + config.timeout = MIN_TIMEOUT; + let budget = config.timeout; + let srtp = test_srtp(&config); + let binding = build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + &[0x11; 12], + stun_credentials().local_password.as_bytes(), + &[(STUN_ATTR_USERNAME, b"loc1:remote01".to_vec())], + ); + server.send_to(&binding, client_address).unwrap(); + let (media_consumer, media_receiver) = mpsc::sync_channel(64); + let (event_sender, event_receiver) = mpsc::channel(); + let attempts = Arc::new(Mutex::new(Vec::new())); + let worker_attempts = attempts.clone(); + let mut sequence = 1; + let session = spawn_video_with_keepalive_fault( + client, + config, + media_consumer, + event_sender, + move |_, _, _| { + let packet = protect_for_test( + &srtp, + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + u32::from(sequence), + &[0, 0, 1, 0x65], + ), + 0, + ); + server.send_to(&packet, client_address).unwrap(); + sequence += 1; + worker_attempts.lock().unwrap().push(Instant::now()); + Err(std::io::Error::from_raw_os_error(10055)) + }, + ); + let deadline = Instant::now() + Duration::from_secs(2); + thread::sleep(budget + PING_INTERVAL_AFTER_CONNECTION); + let mut frames = 0; + loop { + assert!( + Instant::now() < deadline, + "persistent failures must terminate" + ); + if let Ok(event) = event_receiver.recv_timeout(PING_INTERVAL_BEFORE_CONNECTION) { + assert!( + matches!( + event, + NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped( + NvstStopCause::Socket { + operation: "video-natt-send", + os_code: Some(10055), + .. + } + )) + ), + "unexpected event: {event:?}" + ); + break; + } + for frame in media_receiver.try_iter() { + frames += 1; + assert_eq!(frame.frame_index, Some(frames)); + } + } + session.stop(); + for frame in media_receiver.try_iter() { + frames += 1; + assert_eq!(frame.frame_index, Some(frames)); + } + let attempts = attempts.lock().unwrap(); + assert!( + frames > 0, + "receive must continue during deferred keepalives" + ); + assert!(attempts.len() > 1); + assert!(attempts.last().unwrap().duration_since(attempts[0]) >= budget); + assert!( + attempts + .windows(2) + .all(|pair| pair[1].duration_since(pair[0]) + >= PING_INTERVAL_AFTER_CONNECTION - UDP_RECEIVE_POLL_INTERVAL) + ); + } + + #[test] + fn video_keepalive_permanent_send_failures_remain_terminal() { + for error in [ + std::io::Error::from_raw_os_error(10051), + std::io::Error::from_raw_os_error(10065), + std::io::Error::from_raw_os_error(10022), + std::io::Error::from_raw_os_error(10038), + std::io::Error::from_raw_os_error(1), + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + std::io::Error::from(std::io::ErrorKind::ConnectionAborted), + std::io::Error::from(std::io::ErrorKind::TimedOut), + ] { + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.stun_credentials = Some(stun_credentials()); + let code = error.raw_os_error(); + let kind = error.kind(); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, event_receiver) = mpsc::channel(); + let mut error = Some(error); + let session = spawn_video_with_keepalive_fault( + client, + config, + media_consumer, + event_sender, + move |_, _, _| Err(error.take().expect("permanent failure must not retry")), + ); + let event = event_receiver.recv_timeout(Duration::from_secs(2)).unwrap(); + session.stop(); + assert!( + matches!(event, NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped(NvstStopCause::Socket { operation: "video-natt-send", kind: actual_kind, os_code, })) if actual_kind == kind && os_code == code) + ); + } + } + + #[test] + fn video_keepalive_records_only_successful_sends_and_resets_failure_budget() { + let origin = Instant::now(); + let budget = MIN_TIMEOUT; + let mut state = VideoKeepaliveState::default(); + for (index, error) in [ + std::io::Error::from_raw_os_error(10055), + std::io::Error::from(std::io::ErrorKind::Interrupted), + std::io::Error::from(std::io::ErrorKind::WouldBlock), + std::io::Error::from(std::io::ErrorKind::ConnectionReset), + ] + .into_iter() + .enumerate() + { + state + .record_send( + Err(error), + Some([index as u8; 12]), + origin, + budget, + "video-natt-send", + 0, + ) + .unwrap(); + assert_eq!(state.pings_sent, 0); + assert!(state.ping_tracker.pending.is_empty()); + } + assert_eq!(state.first_failure, Some(origin)); + assert_eq!(state.failed_attempts, 4); + assert_eq!(state.deferred_total, 3); + let sent_at = origin + budget - Duration::from_millis(1); + state + .record_send( + Ok(128), + Some([9; 12]), + sent_at, + budget, + "video-natt-send", + 0, + ) + .unwrap(); + assert_eq!(state.first_failure, None); + assert_eq!(state.failed_attempts, 0); + assert_eq!(state.pings_sent, 1); + assert_eq!( + state + .ping_tracker + .pending + .iter() + .copied() + .collect::>(), + vec![([9; 12], sent_at)] + ); + let new_failure = origin + budget; + state + .record_send( + Err(std::io::Error::from_raw_os_error(10055)), + Some([10; 12]), + new_failure, + budget, + "video-natt-send", + 0, + ) + .unwrap(); + assert_eq!(state.first_failure, Some(new_failure)); + assert!( + state + .record_send( + Err(std::io::Error::from_raw_os_error(10055)), + Some([11; 12]), + new_failure + budget, + budget, + "video-natt-send", + 0 + ) + .is_err() + ); + assert_eq!(state.pings_sent, 1); + assert_eq!(state.ping_tracker.pending.len(), 1); + } + + #[test] + fn video_keepalive_icmp_errors_do_not_create_tracking_or_extend_resource_deadline() { + let origin = Instant::now(); + let mut state = VideoKeepaliveState::default(); + state + .record_send( + Err(std::io::Error::from(std::io::ErrorKind::ConnectionReset)), + Some([1; 12]), + origin, + MIN_TIMEOUT, + "video-natt-send", + 0, + ) + .unwrap(); + assert_eq!(state.first_failure, None); + assert_eq!(state.failed_attempts, 0); + assert_eq!(state.pings_sent, 0); + assert!(state.ping_tracker.pending.is_empty()); + state + .record_send( + Err(std::io::Error::from_raw_os_error(10055)), + Some([2; 12]), + origin, + MIN_TIMEOUT, + "video-natt-send", + 0, + ) + .unwrap(); + assert!( + state + .record_send( + Err(std::io::Error::from(std::io::ErrorKind::ConnectionReset)), + Some([3; 12]), + origin + MIN_TIMEOUT, + MIN_TIMEOUT, + "video-natt-send", + 0 + ) + .is_err() + ); + assert!(state.ping_tracker.pending.is_empty()); + } + + #[cfg(windows)] + #[test] + fn windows_ping_to_a_closed_udp_port_does_not_reset_the_socket() { + let socket = bind_nvst_udp_socket(IpAddr::V4(Ipv4Addr::LOCALHOST), 0).expect("bind"); + socket + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("timeout"); + let closed_port = { + let ephemeral = UdpSocket::bind("127.0.0.1:0").expect("ephemeral"); + let port = ephemeral.local_addr().expect("ephemeral addr").port(); + drop(ephemeral); + port + }; + socket + .send_to(b"PING", SocketAddr::from(([127, 0, 0, 1], closed_port))) + .expect("ping closed port"); + let mut buffer = [0_u8; 64]; + let error = socket + .recv_from(&mut buffer) + .expect_err("closed peer produces no datagram"); + assert!( + matches!( + error.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ), + "ICMP port unreachable must not surface as {error}" + ); + } + + #[cfg(windows)] + #[test] + fn occupied_video_port_selects_distinct_retained_pair() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + let (occupied_bundle, occupied_video) = reserve_nvst_socket_pair().unwrap(); + let video_port = occupied_video.local_addr().unwrap().port(); + let bundle_port = occupied_bundle.local_addr().unwrap().port(); + let (bundle, video) = reserve_nvst_socket_pair_preferred(video_port, bundle_port).unwrap(); + assert_ne!(video.local_addr().unwrap().port(), video_port); + assert_ne!(bundle.local_addr().unwrap().port(), bundle_port); + assert_eq!( + bundle.local_addr().unwrap().port(), + video.local_addr().unwrap().port() + 1 + ); + assert!(UdpSocket::bind(video.local_addr().unwrap()).is_err()); + assert!(UdpSocket::bind(bundle.local_addr().unwrap()).is_err()); + } + + #[cfg(windows)] + #[test] + fn occupied_bundle_releases_partial_video_reservation() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + let (occupied_bundle, available_video) = reserve_nvst_socket_pair().unwrap(); + let address = available_video.local_addr().unwrap(); + let bundle_port = occupied_bundle.local_addr().unwrap().port(); + drop(available_video); + let (bundle, video) = + reserve_nvst_socket_pair_preferred(address.port(), bundle_port).unwrap(); + assert_ne!(bundle.local_addr().unwrap().port(), bundle_port); + assert_ne!(video.local_addr().unwrap().port(), address.port()); + let reclaimed = bind_nvst_udp_socket(address.ip(), address.port()) + .expect("failed pair must release video reservation"); + assert_eq!(reclaimed.local_addr().unwrap(), address); + } + + #[test] + fn nvst_unicast_socket_cannot_share_an_existing_port() { + let owner = reserve_nvst_mjolnir_udp_socket().unwrap(); + let address = owner.local_addr().unwrap(); + assert!(bind_nvst_udp_socket(address.ip(), address.port()).is_err()); + let competitor = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP)).unwrap(); + competitor.set_reuse_address(true).unwrap(); + assert!(competitor.bind(&address.into()).is_err()); + } + + #[test] + fn nvst_pair_falls_back_when_either_preferred_port_is_occupied() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + let (bundle, video) = reserve_nvst_socket_pair().unwrap(); + let preferred = video.local_addr().unwrap().port(); + let (other_bundle, other_video) = reserve_nvst_socket_pair_from(preferred).unwrap(); + assert_ne!(other_video.local_addr().unwrap().port(), preferred); + assert_eq!( + other_bundle.local_addr().unwrap().port(), + other_video.local_addr().unwrap().port() + 1 + ); + drop(video); + let (third_bundle, third_video) = reserve_nvst_socket_pair_from(preferred).unwrap(); + assert_ne!(third_video.local_addr().unwrap().port(), preferred); + assert_eq!( + third_bundle.local_addr().unwrap().port(), + third_video.local_addr().unwrap().port() + 1 + ); + // The failed pair attempt must release its first socket. + assert!(bind_nvst_udp_socket(IpAddr::V4(Ipv4Addr::UNSPECIFIED), preferred).is_ok()); + drop(bundle); + } + + #[test] + fn advertised_media_address_uses_the_requested_peer_route() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + // No packets or Internet access: UDP connect only asks the OS for a route. + let bundle = ReservedNvstBundle::reserve().unwrap(); + assert_eq!( + bundle + .advertised_local_address_for("127.0.0.1:5004".parse().unwrap()) + .unwrap(), + "127.0.0.1" + ); + } + + #[test] + fn video_packet_sizing_reserves_wire_overhead_on_tunnel_routes() { + for (address, mtu, expected) in [ + ("192.0.2.1", Some(1500), 1280), + ("192.0.2.1", Some(1280), 1216), + ("192.0.2.1", Some(1200), 1136), + ("192.0.2.1", None, 1280), + ("2001:db8::1", Some(1280), 1200), + ("2001:db8::1", None, 1280), + ("127.0.0.1", Some(65536), 1280), + ] { + let peer = address.parse().unwrap(); + let packet_size = video_packet_size_for_vpn_mtu(peer, mtu).unwrap(); + assert_eq!(packet_size, expected); + let ip_header_bytes = if peer.is_ipv4() { 20 } else { 40 }; + if let Some(mtu) = mtu { + assert!(packet_size + 16 + 16 + 8 + ip_header_bytes <= mtu); + } + } + for mtu in [0, 59, 60, 300] { + assert!( + video_packet_size_for_vpn_mtu("192.0.2.1".parse().unwrap(), Some(mtu)).is_err() + ); + } + } + + #[test] + fn measured_datagram_sizes_map_to_wire_budget_packet_sizes() { + for (address, datagram_size, expected) in [ + ("192.0.2.1", 1_340_usize, Some(1_280)), + ("192.0.2.1", 1_300, Some(1_264)), + ("192.0.2.1", 1_216, Some(1_184)), + ("2001:db8::1", 1_340, Some(1_280)), + ("2001:db8::1", 1_300, Some(1_264)), + ("192.0.2.1", 300, Some(256)), + ("192.0.2.1", 65_535, Some(1_280)), + ("192.0.2.1", 0, None), + ("2001:db8::1", 0, None), + ] { + let peer: IpAddr = address.parse().unwrap(); + assert_eq!( + measured_video_packet_size(datagram_size, peer), + expected, + "{address} {datagram_size}" + ); + } + } + + #[test] + fn measured_datagrams_never_exceed_the_route_derived_packet_size() { + for peer in ["192.0.2.1", "2001:db8::1"] { + let peer: IpAddr = peer.parse().unwrap(); + let route_derived = video_packet_size_for_vpn_mtu(peer, Some(1_500)).unwrap(); + for datagram_size in [320_usize, 700, 1_000, 1_216, 1_300, 1_340, 1_400] { + let measured = measured_video_packet_size(datagram_size, peer); + assert_eq!( + measured, + Some(((datagram_size - 32) / 16 * 16).min(1_280)), + "{peer} {datagram_size}" + ); + assert!(measured.unwrap() <= route_derived); + } + } + } + + #[test] + fn tunnel_packet_size_is_preserved_by_the_handoff_and_fec_receiver() { + let mut handoff = legacy_handoff(); + let packet_size = video_packet_size_for_vpn_mtu(peer().ip(), Some(1280)).unwrap(); + handoff["packetSize"] = serde_json::json!(packet_size); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.video_packet_size, packet_size); + let receiver = NvstVideoReceiver::new(config.clone()); + assert_eq!(receiver.fec_reorder.shard_len, packet_size + 16); + let plaintext = build_plaintext_rtp(1, FLAG_SOF | FLAG_EOF, 1, &vec![0; packet_size - 16]); + assert_eq!(plaintext.len(), receiver.fec_reorder.shard_len); + let encrypted = protect_for_test(&test_srtp(&config), plaintext, 0); + assert!(encrypted.len() + 20 + 8 <= 1280); + } + + #[test] + fn link_local_interfaces_use_distinct_logical_ice_addresses() { + let local: SocketAddr = "169.254.0.21:49000".parse().unwrap(); + let remote: SocketAddr = "169.254.0.22:5006".parse().unwrap(); + assert_eq!( + logical_ice_addr(local, 1), + "127.0.0.1:49000".parse().unwrap() + ); + assert_eq!( + logical_ice_addr(remote, 2), + "127.0.0.2:5006".parse().unwrap() + ); + assert!(Candidate::host(logical_ice_addr(local, 1), "udp").is_ok()); + assert!(Candidate::host(logical_ice_addr(remote, 2), "udp").is_ok()); + } + + #[test] + fn natt_hole_punch_uses_setup_ping_payload_not_v2_ufrag() { + let credentials = stun_credentials(); + let packet = build_natt_hole_punch_request( + &credentials.local_username_fragment, + b"srv1", + &credentials.remote_password, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + ); + let (_, username) = find_stun_attribute(&packet, STUN_ATTR_USERNAME).expect("USERNAME"); + assert_eq!(username, b"srv1:loc1"); + assert_ne!( + username, + format!( + "{}:{}", + credentials.remote_username_fragment, credentials.local_username_fragment + ) + .as_bytes() + ); + assert!(valid_stun_fingerprint(&packet)); + assert!(valid_stun_message_integrity( + &packet, + credentials.remote_password.as_bytes() + )); + } + + #[test] + fn version_six_binding_request_matches_known_answer() { + let packet = build_stun_binding_request( + &stun_credentials(), + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + ); + assert_eq!( + packet, + hex_bytes( + "000100342112A442000102030405060708090A0B0006000D72656D6F746530313A6C6F633100000000080014B276DC1C7949494C7EF7EB226BE8BB5E0EE5AABD802800045A8349EF" + ), + ); + assert!(valid_stun_fingerprint(&packet)); + assert!(valid_stun_message_integrity( + &packet, + b"remote-password-with-36-byte-value-001" + )); + } + + #[test] + fn version_six_validates_requests_and_builds_authenticated_responses() { + let credentials = stun_credentials(); + let transaction_id = [0x11; 12]; + let request = build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + &transaction_id, + credentials.local_password.as_bytes(), + &[(STUN_ATTR_USERNAME, b"loc1:remote01".to_vec())], + ); + let source: SocketAddr = "192.0.2.20:5004".parse().expect("source"); + let StunDatagram::Handled(Some(response)) = + handle_stun_datagram(&request, source, &credentials) + else { + panic!("authenticated binding request must produce a response"); + }; + assert_eq!( + u16::from_be_bytes([response[0], response[1]]), + STUN_BINDING_SUCCESS_RESPONSE + ); + assert_eq!(&response[8..20], &transaction_id); + assert!(valid_stun_fingerprint(&response)); + assert!(valid_stun_message_integrity( + &response, + credentials.local_password.as_bytes() + )); + let (_, mapped) = find_stun_attribute(&response, STUN_ATTR_XOR_MAPPED_ADDRESS) + .expect("XOR-MAPPED-ADDRESS"); + assert_eq!(mapped[1], 1); + assert_eq!( + u16::from_be_bytes([mapped[2], mapped[3]]) ^ ((STUN_MAGIC_COOKIE >> 16) as u16), + 5004 + ); + + let mut tampered = request; + tampered[24] ^= 1; + assert!(matches!( + handle_stun_datagram(&tampered, source, &credentials), + StunDatagram::Invalid + )); + } + + #[test] + fn gfn_local_ice_credentials_match_official_lengths() { + let creds = generate_gfn_local_ice_credentials().expect("OS randomness"); + assert_eq!(creds.ufrag.len(), 4); + assert_eq!(creds.pass.len(), 22); + } + + #[test] + fn synthesized_ice_success_matches_the_request_transaction() { + let credentials = stun_credentials(); + let transaction_id = [7_u8; 12]; + let mapped = "192.168.1.104:54454".parse().expect("mapped"); + let response = + synthesize_ice_binding_success(&transaction_id, mapped, &credentials.remote_password); + assert_eq!( + u16::from_be_bytes([response[0], response[1]]), + STUN_BINDING_SUCCESS_RESPONSE + ); + assert_eq!(&response[8..20], &transaction_id); + assert!(valid_stun_fingerprint(&response)); + assert!(valid_stun_message_integrity( + &response, + credentials.remote_password.as_bytes() + )); + } + + #[test] + fn legacy_gcm_key_derivation_matches_known_answer() { + let config = config(); + let receiver = test_srtp(&config); + let SrtpCipher::AeadAes256Gcm { + encryption_key, + session_salt, + .. + } = receiver.cipher + else { + panic!("legacy handoff must derive an AES-256-GCM session"); + }; + assert_eq!( + encryption_key, + decode_fixed_hex::<32>( + "0E44A0B0E7F1BDBB298CBEE52C9F8AC1C37726768C946F59BDAAA099608CBF66", + NvstConfigError::InvalidAesKey, + ) + .expect("key"), + ); + assert_eq!( + session_salt, + decode_fixed_hex::<12>("78B9D97B7FFB37CAD539A29D", NvstConfigError::InvalidSrtpSalt) + .expect("salt"), + ); + } + + #[test] + fn parses_valid_legacy_nvst_and_rejects_invalid_handoffs() { + let context = json!({ "nvstVideo": legacy_handoff() }); + assert!(parse_nvst_video_handoff(&context).unwrap().is_some()); + let configured = parse_nvst_video_handoff(&json!({ + "nvstVideo": legacy_handoff(), + "settings": { "fps": 60 }, + "session": { "negotiatedStreamProfile": { "fps": 120 } } + })) + .expect("valid NVST context") + .expect("NVST handoff"); + assert_eq!(configured.frame_time_us, DEFAULT_FRAME_TIME_US); + let capped = parse_nvst_video_handoff(&json!({ + "nvstVideo": legacy_handoff(), + "settings": { "fps": 360 } + })) + .expect("valid high-FPS NVST context") + .expect("NVST handoff"); + assert_eq!(capped.frame_time_us, DEFAULT_FRAME_TIME_US); + assert!(matches!( + parse_nvst_video_handoff(&json!({ "nvstTransport": { "tracks": [] } })), + Err(NvstConfigError::MissingNvstVideoHandoff) + )); + + let mut missing_gcm_salt = legacy_handoff(); + missing_gcm_salt + .as_object_mut() + .expect("handoff object") + .remove("srtpSaltHex"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&missing_gcm_salt, None), + Err(NvstConfigError::MissingField("srtpSaltHex")) + )); + + let mut invalid = legacy_handoff(); + invalid["codec"] = json!("VP9"); + let context = json!({ "nvstVideo": invalid }); + assert!(matches!( + parse_nvst_video_handoff(&context), + Err(NvstConfigError::UnsupportedCodec(_)) + )); + + let mut missing_cm_salt = legacy_handoff(); + missing_cm_salt + .as_object_mut() + .expect("handoff object") + .remove("srtpSaltHex"); + missing_cm_salt["srtpProfile"] = json!("AES_CM_128_HMAC_SHA1_80"); + missing_cm_salt["srtpAesKeyHex"] = json!("000102030405060708090A0B0C0D0E0F"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&missing_cm_salt, None), + Err(NvstConfigError::MissingField("srtpSaltHex")) + )); + missing_cm_salt["srtpSaltHex"] = json!(TEST_SALT); + let explicit_cm = NvstVideoConfig::from_legacy_handoff(&missing_cm_salt, None) + .expect("explicit AES-CM profile"); + assert_eq!( + explicit_cm.srtp_profile(), + NvstSrtpProfile::AesCm128HmacSha1_80 + ); + + let mut gcm_128 = legacy_handoff(); + gcm_128["srtpProfile"] = json!("AEAD_AES_128_GCM"); + gcm_128["srtpAesKeyHex"] = json!("000102030405060708090A0B0C0D0E0F"); + gcm_128["srtpSaltHex"] = json!("00000000000000009ECA935E"); + assert_eq!( + NvstVideoConfig::from_legacy_handoff(&gcm_128, None) + .expect("explicit AES-128-GCM profile") + .srtp_profile(), + NvstSrtpProfile::AeadAes128Gcm + ); + + let mut cm_32 = gcm_128; + cm_32["srtpProfile"] = json!("AES_CM_128_HMAC_SHA1_32"); + cm_32["srtpSaltHex"] = json!(TEST_SALT); + assert_eq!( + NvstVideoConfig::from_legacy_handoff(&cm_32, None) + .expect("explicit AES-CM-32 profile") + .srtp_profile(), + NvstSrtpProfile::AesCm128HmacSha1_32 + ); + } + + #[test] + fn rejects_invalid_peer_and_secret_material() { + let mut handoff = legacy_handoff(); + handoff["videoPeerIp"] = json!("0.0.0.0"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidPeerIp(_)) + )); + let mut handoff = legacy_handoff(); + handoff["srtpAesKeyHex"] = json!("not-a-key"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidAesKey) + )); + + let mut aes_128 = legacy_handoff(); + aes_128["srtpProfile"] = json!("AEAD_AES_128_GCM"); + aes_128["srtpAesKeyHex"] = json!("000102030405060708090A0B0C0D0E0G"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&aes_128, None), + Err(NvstConfigError::InvalidAesKey) + )); + } + + #[test] + fn srtp_aes_cm_hmac_sha1_known_answer_unprotects_packet_when_explicitly_selected() { + let key = decode_fixed_hex::<16>( + "000102030405060708090A0B0C0D0E0F", + NvstConfigError::InvalidAesKey, + ) + .expect("key"); + let salt = + decode_fixed_hex::<14>(TEST_SALT, NvstConfigError::InvalidSrtpSalt).expect("salt"); + let material = NvstSrtpMaterial::AesCm128HmacSha1 { + master_key: key, + master_salt: salt, + authentication_tag_len: SRTP_AES_CM_HMAC_SHA1_80_TAG_LEN, + }; + let crypto = SrtpReceiver::from_material(&material); + // Frozen plaintext so the crypto known-answer vector stays independent of + // the RTP packet-layout helper: 12-byte header + 16-byte inline metadata + // + 6 bytes of media. + let plaintext = + hex_bytes("80E01234010203041122334434120000070000000700000000000000000000016588"); + let protected = protect_for_test(&crypto, plaintext.clone(), 0); + assert_eq!( + protected, + hex_bytes( + "80E01234010203041122334408BA995FCB62EA430BE6EDEF8D4DE06268F0D6D702702082DDFADA13AE83402B" + ), + "independent AES-CTR/HMAC known-answer vector", + ); + let mut receiver = SrtpReceiver::from_material(&material); + let unprotected = receiver.unprotect(&protected).expect("authenticated SRTP"); + assert_eq!(unprotected.plaintext, plaintext); + assert_eq!(unprotected.index, 0x1234); + + let mut tampered = protected; + let last = tampered.last_mut().expect("authentication tag"); + *last ^= 0x01; + let mut receiver = SrtpReceiver::from_material(&material); + assert!(matches!( + receiver.unprotect(&tampered), + Err(NvstDropReason::AuthenticationFailed) + )); + } + + #[test] + fn srtp_aead_aes_256_gcm_rfc_known_answer_unprotects_and_rejects_tampering_and_replay() { + let session_key = + decode_fixed_hex::<32>(TEST_KEY, NvstConfigError::InvalidAesKey).expect("key"); + let session_salt = + decode_fixed_hex::<12>("517569642070726F2071756F", NvstConfigError::InvalidSrtpSalt) + .expect("salt"); + let mut receiver = SrtpReceiver { + cipher: SrtpCipher::AeadAes256Gcm { + encryption_key: session_key, + session_salt, + authentication_tag_len: SRTP_AEAD_AES_GCM_TAG_LEN, + }, + replay: ReplayWindow::default(), + }; + let protected = hex_bytes( + "8040F17B8041F8D35501A0B232B1DE78A822FE12EF9F78FA332E33AAB18012389A58E2F3B50B2A0276FFAE0F1BA63799B87B7AA3DB36DFFFD6B0F9BB7878D7A76C13", + ); + let expected = hex_bytes( + "8040F17B8041F8D35501A0B247616C6C696120657374206F6D6E69732064697669736120696E207061727465732074726573", + ); + let unprotected = receiver.unprotect(&protected).expect("RFC 7714 packet"); + assert_eq!(unprotected.plaintext, expected); + assert!(matches!( + receiver.unprotect(&protected), + Err(NvstDropReason::ReplayRejected) + )); + + let mut tampered = protected; + tampered[20] ^= 0x01; + let mut receiver = SrtpReceiver { + cipher: SrtpCipher::AeadAes256Gcm { + encryption_key: session_key, + session_salt, + authentication_tag_len: SRTP_AEAD_AES_GCM_TAG_LEN, + }, + replay: ReplayWindow::default(), + }; + assert!(matches!( + receiver.unprotect(&tampered), + Err(NvstDropReason::AuthenticationFailed) + )); + } + + #[test] + fn srtp_aead_aes_128_gcm_rfc_known_answer_unprotects_and_rejects_tampering_and_replay() { + let session_key = decode_fixed_hex::<16>( + "000102030405060708090A0B0C0D0E0F", + NvstConfigError::InvalidAesKey, + ) + .expect("key"); + let session_salt = + decode_fixed_hex::<12>("517569642070726F2071756F", NvstConfigError::InvalidSrtpSalt) + .expect("salt"); + let mut receiver = SrtpReceiver { + cipher: SrtpCipher::AeadAes128Gcm { + encryption_key: session_key, + session_salt, + authentication_tag_len: SRTP_AEAD_AES_GCM_TAG_LEN, + }, + replay: ReplayWindow::default(), + }; + let protected = hex_bytes( + "8040F17B8041F8D35501A0B2F24DE3A3FB34DE6CACBA861C9D7E4BCABE633BD50D294E6F42A5F47A51C7D19B36DE3ADF8833899D7F27BEB16A9152CF765EE4390CCE", + ); + let expected = hex_bytes( + "8040F17B8041F8D35501A0B247616C6C696120657374206F6D6E69732064697669736120696E207061727465732074726573", + ); + let unprotected = receiver.unprotect(&protected).expect("RFC 7714 packet"); + assert_eq!(unprotected.plaintext, expected); + assert!(matches!( + receiver.unprotect(&protected), + Err(NvstDropReason::ReplayRejected) + )); + + let mut tampered = protected; + tampered[20] ^= 0x01; + let mut receiver = SrtpReceiver { + cipher: SrtpCipher::AeadAes128Gcm { + encryption_key: session_key, + session_salt, + authentication_tag_len: SRTP_AEAD_AES_GCM_TAG_LEN, + }, + replay: ReplayWindow::default(), + }; + assert!(matches!( + receiver.unprotect(&tampered), + Err(NvstDropReason::AuthenticationFailed) + )); + } + + #[test] + fn parses_rtp_extensions_and_padding_after_srtp_unprotect() { + let mut packet = vec![0xb0, 0x60, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1]; + packet.extend_from_slice(&[0xbe, 0xde, 0, 1, 0xaa, 0xbb, 0xcc, 0xdd]); + packet.extend_from_slice(&[1, 2, 3, 4, 2, 2]); + let header = RtpHeader::parse(&packet).expect("extension parsed"); + assert_eq!(header.payload_offset, 20); + assert_eq!( + header.payload(&packet).expect("padding removed"), + &[1, 2, 3, 4] + ); + } + + #[test] + fn gs_extension_requires_exactly_four_words() { + let exact = build_plaintext_rtp(1, FLAG_SOF, 7, &[0, 0, 1, 0x65]); + assert!(RtpHeader::parse(&exact).is_ok()); + + let mut oversized = exact.clone(); + oversized[14..16].copy_from_slice(&5_u16.to_be_bytes()); + oversized.splice(32..32, [0_u8; 4]); + assert!(matches!( + RtpHeader::parse(&oversized), + Err(RtpParseError::InvalidExtensionLength) + )); + + let mut truncated = exact; + truncated[14..16].copy_from_slice(&5_u16.to_be_bytes()); + assert!(matches!( + RtpHeader::parse(&truncated), + Err(RtpParseError::InvalidExtensionLength) + )); + } + + #[test] + fn parses_the_wire_stream_packet_index_as_a_24_bit_value() { + let packet = build_plaintext_rtp(1, FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, 7, &[]); + // Point the packet index at a 24-bit value to verify the wire shift. + let mut packet = packet; + packet[16..20].copy_from_slice(&(0x12_34_56_u32 << 8).to_le_bytes()); + + let header = RtpHeader::parse(&packet).expect("RTP header"); + let payload = header.payload(&packet).expect("payload"); + let (video, media) = NvVideoPacket::parse(&header, payload).expect("NV video header"); + assert_eq!(video.stream_packet_index, 0x12_34_56); + assert_eq!(video.frame_index, 7); + assert!(!video.is_fec); + assert!(media.is_empty()); + } + + #[test] + fn classifies_fec_packets_from_the_extension_group_coordinates() { + let mut packet = build_plaintext_rtp(3, FLAG_CONTAINS_PIC_DATA, 7, &[0xaa]); + // FecId=3, SrcPkts=3 with the FEC-group marker bits set => correction packet. + packet[28..32].copy_from_slice(&0x00c0_3420_u32.to_le_bytes()); + let header = RtpHeader::parse(&packet).expect("RTP header"); + let payload = header.payload(&packet).expect("payload"); + let (video, _) = NvVideoPacket::parse(&header, payload).expect("NV video header"); + assert!(video.is_fec); + + // Source packet: FecId=2 < SrcPkts=3. + let mut packet = build_plaintext_rtp(2, FLAG_CONTAINS_PIC_DATA, 7, &[0xaa]); + packet[28..32].copy_from_slice(&0x00c0_2420_u32.to_le_bytes()); + let header = RtpHeader::parse(&packet).expect("RTP header"); + let payload = header.payload(&packet).expect("payload"); + let (video, _) = NvVideoPacket::parse(&header, payload).expect("NV video header"); + assert!(!video.is_fec); + } + + #[test] + fn strips_the_gamestream_frame_header_before_annex_b_video() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 9, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let mut payload = vec![0x01, 0, 0, 2, 0, 0, 0, 0]; + payload.extend_from_slice(&[ + 0, 0, 1, 0x67, 0xaa, 0, 0, 0, 1, 0x68, 0xbb, 0, 0, 1, 0x65, 0xcc, + ]); + + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::H264, 4096); + let frame = assembler + .push(header, 90_000, &payload) + .expect("valid frame") + .expect("complete frame"); + + assert_eq!( + frame.bytes, + [ + 0, 0, 0, 1, 0x67, 0xaa, 0, 0, 0, 1, 0x68, 0xbb, 0, 0, 1, 0x65, 0xcc, + ] + ); + assert!(frame.keyframe); + } + + #[test] + fn assembles_h265_annex_b_and_detects_irap_keyframes() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 10, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let mut payload = vec![0x01, 0, 0, 2, 0, 0, 0, 0]; + payload.extend_from_slice(&[0, 0, 0, 1, 19 << 1, 1, 0xaa]); + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::H265, 4096); + + let frame = assembler + .push(header, 90_000, &payload) + .expect("valid H.265 frame") + .expect("complete frame"); + + assert_eq!(frame.codec, NvstVideoCodec::H265); + assert_eq!(frame.bytes, [0, 0, 0, 1, 19 << 1, 1, 0xaa]); + assert!(frame.keyframe); + } + + #[test] + fn assembles_av1_payload_after_the_gamestream_frame_header() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 11, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let payload = [0x01, 0, 0, 2, 11, 0, 0, 0, 0x12, 0x34, 0x56]; + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::Av1, 4096); + + let frame = assembler + .push(header, 90_000, &payload) + .expect("valid AV1 frame") + .expect("complete frame"); + + assert_eq!(frame.codec, NvstVideoCodec::Av1); + assert_eq!(frame.bytes, [0x12, 0x34, 0x56]); + assert!(frame.keyframe); + } + + #[test] + fn strips_the_gfn_cloud_extended_av1_frame_header() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 12, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let mut payload = vec![0_u8; GFN_EXTENDED_FRAME_HEADER_BYTES]; + payload[0] = 0x81; + payload[3] = 2; + payload[16..20].copy_from_slice(&3_u32.to_le_bytes()); + // A temporal-unit length byte is not necessarily a valid OBU header. + payload.extend_from_slice(&[0x81, 0x01, 0x12]); + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::Av1, 4096); + + let frame = assembler + .push(header, 90_000, &payload) + .expect("valid extended AV1 frame") + .expect("complete frame"); + + assert_eq!(frame.bytes, [0x81, 0x01, 0x12]); + assert!(frame.keyframe); + } + + #[test] + fn trims_gfn_cloud_av1_padding_to_the_advertised_access_unit_size() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 13, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let mut payload = vec![0_u8; GFN_EXTENDED_FRAME_HEADER_BYTES]; + payload[0] = 0x81; + payload[3] = 2; + payload[16..20].copy_from_slice(&5_u32.to_le_bytes()); + payload.extend_from_slice(&[0x12, 0x00, 0x32, 0x01, 0xaa, 0, 0, 0]); + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::Av1, 4096); + + let frame = assembler + .push(header, 90_000, &payload) + .expect("valid padded AV1 frame") + .expect("complete frame"); + + assert_eq!(frame.bytes, [0x12, 0x00, 0x32, 0x01, 0xaa]); + assert!(frame.keyframe); + } + + #[test] + fn trims_av1_fec_padding_from_the_final_packet() { + let first = NvVideoPacket { + stream_packet_index: 1, + frame_index: 12, + flags: FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let last = NvVideoPacket { + stream_packet_index: 2, + frame_index: 12, + flags: FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let first_payload = [0x01, 0, 0, 2, 3, 0, 0, 0, 0x12, 0x00]; + let padded_last_payload = [0x32, 0x01, 0xaa, 0, 0, 0, 0]; + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::Av1, 4096); + + assert!( + assembler + .push(first, 90_000, &first_payload) + .expect("valid AV1 first packet") + .is_none() + ); + let frame = assembler + .push(last, 90_000, &padded_last_payload) + .expect("valid AV1 final packet") + .expect("complete frame"); + + assert_eq!(frame.bytes, [0x12, 0x00, 0x32, 0x01, 0xaa]); + assert!(frame.keyframe); + } + + #[test] + fn parses_all_supported_nvst_video_codec_names() { + assert_eq!( + NvstVideoCodec::parse("AVC").expect("AVC"), + NvstVideoCodec::H264 + ); + assert_eq!( + NvstVideoCodec::parse("HEVC").expect("HEVC"), + NvstVideoCodec::H265 + ); + assert_eq!( + NvstVideoCodec::parse("AV1").expect("AV1"), + NvstVideoCodec::Av1 + ); + } + + #[test] + fn annex_b_scanner_returns_the_earliest_mixed_width_start_code() { + assert_eq!( + find_annex_b_start_code(&[0, 0, 1, 0x67, 0, 0, 0, 1, 0x68]), + Some((0, 3)) + ); + assert_eq!( + find_annex_b_start_code(&[0, 0, 0, 1, 0x67, 0, 0, 1, 0x68]), + Some((0, 4)) + ); + } + + #[test] + fn strips_a_terminal_access_unit_delimiter_before_decoder_submission() { + let mut bytes = vec![0, 0, 0, 1, 0x65, 0xaa, 0, 0, 0, 1, 0x09, 0xf0]; + strip_trailing_access_unit_delimiter(NvstVideoCodec::H264, &mut bytes); + assert_eq!(bytes, [0, 0, 0, 1, 0x65, 0xaa]); + + let mut bytes = vec![0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x61, 0xbb]; + strip_trailing_access_unit_delimiter(NvstVideoCodec::H264, &mut bytes); + assert_eq!(bytes, [0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x61, 0xbb]); + } + + #[test] + fn annex_b_terminal_start_code_does_not_panic() { + for codec in [NvstVideoCodec::H264, NvstVideoCodec::H265] { + for suffix in [&[0, 0, 1][..], &[0, 0, 0, 1][..]] { + let mut bytes = vec![0, 0, 1, 0x61, 0xaa]; + bytes.extend_from_slice(suffix); + let expected = bytes.clone(); + strip_trailing_access_unit_delimiter(codec, &mut bytes); + assert_eq!(bytes, expected); + assert!(!video_access_unit_is_keyframe(codec, &bytes, false)); + } + } + } + + #[test] + fn assembles_a_single_packet_frame_without_the_picture_data_flag() { + let header = NvVideoPacket { + stream_packet_index: 25, + frame_index: 12, + flags: FLAG_SOF | FLAG_EOF, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::H264, 4096); + let frame = assembler + .push(header, 90_000, &[0x81, 0x08, 0, 0, 0, 1, 0x61, 0xaa]) + .expect("source packet") + .expect("complete frame"); + assert_eq!(frame.bytes, [0, 0, 0, 1, 0x61, 0xaa]); + } + + #[test] + fn rejects_a_start_packet_without_nearby_annex_b_video() { + let header = NvVideoPacket { + stream_packet_index: 1, + frame_index: 9, + flags: FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + fec_current_block: 0, + fec_last_block: 0, + is_fec: false, + }; + let payload = vec![0x81; MAX_GS_FRAME_HEADER_BYTES + 8]; + let mut assembler = VideoAccessUnitAssembler::new(NvstVideoCodec::H264, 4096); + + assert!(matches!( + assembler.push(header, 90_000, &payload), + Err(NvstDropReason::MissingAnnexBStartCode) + )); + } + + #[test] + fn receiver_reorders_authenticated_packets_and_emits_annex_b_frame() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp( + 10, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + 9, + &[0, 0, 0, 1, 0x65], + ), + 0, + ); + let middle = protect_for_test( + &crypto, + build_plaintext_rtp(11, FLAG_CONTAINS_PIC_DATA, 9, &[0xaa]), + 0, + ); + let last = protect_for_test( + &crypto, + build_plaintext_rtp(12, FLAG_EOF | FLAG_CONTAINS_PIC_DATA, 9, &[0xbb]), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + let origin = Instant::now(); + assert!( + receiver + .process_datagram(peer(), &first, origin + Duration::from_millis(5)) + .is_empty() + ); + assert!( + receiver + .process_datagram(peer(), &last, origin + Duration::from_millis(13)) + .is_empty() + ); + let events = receiver.process_datagram(peer(), &middle, origin + Duration::from_millis(11)); + assert_eq!(events.len(), 1); + let NvstReceiveEvent::Frame(frame) = &events[0] else { + panic!("expected frame, got {events:?}"); + }; + assert_eq!(frame.frame_index, 9); + assert!(frame.keyframe); + assert_eq!(frame.bytes, [0, 0, 0, 1, 0x65, 0xaa, 0xbb]); + assert_eq!(feedback.take_nack(Instant::now(), None), None); + assert_eq!(feedback.completed_frame_snapshot(), (frame.frame_index, 7)); + assert!(feedback.take_completed_frame().is_none()); + feedback.publish_accepted_frame(frame.frame_index, 7, origin + Duration::from_millis(30)); + let pending_ack = feedback + .take_completed_frame() + .expect("accepted frame acknowledgment"); + assert_eq!(pending_ack.frame_number, 9); + assert_eq!(pending_ack.bytes, 7); + let ack = pending_ack.command(origin); + assert_eq!(&ack.payload[12..20], &5.0_f64.to_le_bytes()); + assert_eq!(&ack.payload[56..60], &[0; 4]); + assert_eq!(&ack.payload[72..76], &7_u32.to_le_bytes()); + } + + #[test] + fn accepted_frame_queue_discards_stale_feedback_and_preserves_sequence_numbers() { + let feedback = NvstFeedbackState::default(); + for frame_number in 1..=u32::try_from(MAX_PENDING_FRAME_ACKS + 2).unwrap() { + feedback.publish_accepted_frame(frame_number, frame_number, Instant::now()); + } + + let first = feedback.take_completed_frame().expect("newest queue head"); + assert_eq!(first.frame_number, 3); + assert_eq!(first.bytes, 3); + + let mut last = first; + while let Some(frame) = feedback.take_completed_frame() { + last = frame; + } + assert_eq!(last.frame_number, 514); + assert_eq!(last.bytes, 514); + } + + #[test] + fn frame_ack_timing_ignores_unauthenticated_packets_and_unknown_frames() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let packet = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF | FLAG_EOF, 42, &[0, 0, 0, 1, 0x65, 0xaa]), + 0, + ); + let mut invalid = packet.clone(); + *invalid.last_mut().unwrap() ^= 1; + let mut receiver = NvstVideoReceiver::new(config); + let origin = Instant::now(); + assert!(matches!( + receiver.process_datagram(peer(), &invalid, origin + Duration::from_millis(1))[..], + [NvstReceiveEvent::Dropped(_)] + )); + let events = receiver.process_datagram(peer(), &packet, origin + Duration::from_millis(7)); + assert!(matches!(events[..], [NvstReceiveEvent::Frame(_)])); + feedback.publish_accepted_frame(42, 6, origin + Duration::from_millis(20)); + let ack = feedback.take_completed_frame().unwrap().command(origin); + assert_eq!(&ack.payload[12..20], &7.0_f64.to_le_bytes()); + assert_eq!(&ack.payload[56..60], &[0; 4]); + + feedback.publish_accepted_frame(43, 6, origin + Duration::from_millis(21)); + let unknown = feedback.take_completed_frame().unwrap().command(origin); + assert_eq!(&unknown.payload[12..20], &[0; 8]); + assert_eq!(&unknown.payload[56..60], &[0; 4]); + } + + #[test] + fn frame_ack_packet_timing_is_bounded_and_retired_on_drop() { + let feedback = NvstFeedbackState::default(); + let origin = Instant::now(); + for frame in 0..=MAX_PENDING_FRAME_ACKS as u32 { + feedback.record_frame_packet(frame, origin + Duration::from_millis(u64::from(frame))); + } + let timings = feedback.frame_packet_timings.lock().unwrap(); + assert_eq!(timings.len(), MAX_PENDING_FRAME_ACKS); + assert_eq!(timings.front().unwrap().frame_number, 1); + drop(timings); + feedback.complete_frame_packets(1); + feedback.retire_undelivered_frame(1); + assert!( + feedback + .frame_packet_timings + .lock() + .unwrap() + .iter() + .all(|entry| entry.frame_number != 1) + ); + } + + #[test] + fn control_report_batch_concatenates_existing_records_within_1071_bytes() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + let mut batch = ControlReportBatch::new(now, MAX_CONTROL_REPORT_BYTES); + let mut expected = Vec::new(); + for number in 1..=9 { + let frame = CompletedFrameFeedback { + frame_number: number, + bytes: number, + accepted_at: now, + assembled_at: None, + packet_timing: None, + }; + let record = frame_ack(number, Some(12.0), number).encoded(); + assert_eq!(record.len(), FRAME_ACK_RECORD_LEN); + batch.append_ack(frame, &record, now); + expected.extend_from_slice(&record); + } + let report = feedback.qos_report(&QosReport::default(), Duration::from_millis(50)); + let qos_record = report.command().encoded(); + assert!(batch.fits(qos_record.len())); + batch.append_qos(report, &qos_record, now); + expected.extend_from_slice(&qos_record); + assert!(expected.len() <= MAX_CONTROL_REPORT_BYTES); + assert!(!batch.fits(FRAME_ACK_RECORD_LEN)); + assert!(batch.flush(now, &feedback, |bytes| bytes == expected)); + assert_eq!(batch.frame_acks_sent, 9); + assert_eq!(batch.qos_reports_sent, 1); + assert_eq!(batch.last_ack_frame, Some(9)); + assert_eq!(batch.last_qos_report.sequence, 1); + assert!(batch.bytes.is_empty()); + } + + #[test] + fn full_control_batch_flushes_before_collecting_another_ack() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + let mut batch = ControlReportBatch::new(now, MAX_CONTROL_REPORT_BYTES); + for number in 1..=10 { + let frame = CompletedFrameFeedback { + frame_number: number, + bytes: 1, + accepted_at: now, + assembled_at: None, + packet_timing: None, + }; + batch.append_ack(frame, &frame_ack(number, Some(0.0), 1).encoded(), now); + } + assert_eq!(batch.bytes.len(), 10 * FRAME_ACK_RECORD_LEN); + assert!(!batch.fits(FRAME_ACK_RECORD_LEN)); + assert!(!batch.fits(QosReport::default().command().encoded().len())); + assert!(batch.flush(now, &feedback, |bytes| bytes.len() == 1_060)); + assert!(batch.fits(FRAME_ACK_RECORD_LEN)); + assert_eq!(batch.frame_acks_sent, 10); + } + + #[test] + fn lower_control_cap_flushes_at_boundary_and_preserves_failed_batch_for_retry() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + let mut batch = ControlReportBatch::new(now, 2 * FRAME_ACK_RECORD_LEN); + for number in 1..=2 { + batch.append_ack( + CompletedFrameFeedback { + frame_number: number, + bytes: 1, + accepted_at: now, + assembled_at: None, + packet_timing: None, + }, + &frame_ack(number, Some(0.0), 1).encoded(), + now, + ); + } + assert_eq!(batch.bytes.len(), batch.max_bytes); + assert!(!batch.fits(FRAME_ACK_RECORD_LEN)); + let pending = batch.bytes.clone(); + assert!(!batch.flush(now, &feedback, |bytes| { + assert_eq!(bytes, pending); + false + })); + assert_eq!(batch.bytes, pending); + assert_eq!(batch.frames.len(), 2); + assert_eq!(batch.frame_acks_sent, 0); + assert!(batch.flush(now, &feedback, |bytes| { + assert_eq!(bytes, pending); + true + })); + assert_eq!(batch.frame_acks_sent, 2); + assert_eq!(batch.last_ack_frame, Some(2)); + assert!(batch.fits(FRAME_ACK_RECORD_LEN)); + + let mut minimum = ControlReportBatch::new(now, MIN_CONTROL_REPORT_BYTES); + let record = frame_ack(3, Some(0.0), 1).encoded(); + assert!(minimum.fits(record.len())); + minimum.append_ack( + CompletedFrameFeedback { + frame_number: 3, + bytes: 1, + accepted_at: now, + assembled_at: None, + packet_timing: None, + }, + &record, + now, + ); + assert!(!minimum.fits(1)); + assert!(minimum.flush(now, &feedback, |bytes| bytes == record)); + assert_eq!(minimum.last_ack_frame, Some(3)); + + let report = feedback.qos_report(&minimum.last_qos_report, QOS_REPORT_INTERVAL); + let qos_bytes = report.command().encoded(); + assert!(minimum.fits(qos_bytes.len())); + minimum.append_qos(report, &qos_bytes, now + QOS_REPORT_INTERVAL); + assert!( + !minimum.flush(now + QOS_REPORT_INTERVAL, &feedback, |bytes| { + assert_eq!(bytes, qos_bytes); + false + }) + ); + assert_eq!(minimum.qos_reports_sent, 0); + assert_eq!(minimum.last_qos_report.sequence, 0); + assert_eq!(minimum.bytes, qos_bytes); + assert!( + minimum.flush(now + QOS_REPORT_INTERVAL, &feedback, |bytes| { + assert_eq!(bytes, qos_bytes); + true + }) + ); + assert_eq!(minimum.qos_reports_sent, 1); + assert_eq!(minimum.last_qos_report.sequence, 1); + } + + #[test] + fn failed_control_batch_retries_without_committing_ack_or_qos() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + let mut batch = ControlReportBatch::new(now, MAX_CONTROL_REPORT_BYTES); + let frame = CompletedFrameFeedback { + frame_number: 42, + bytes: 7, + accepted_at: now, + assembled_at: None, + packet_timing: None, + }; + let ack = frame_ack(42, Some(0.0), 7).encoded(); + batch.append_ack(frame, &ack, now); + assert!(!batch.flush_due(now + QOS_REPORT_INTERVAL - Duration::from_nanos(1))); + assert!(batch.flush_due(now + QOS_REPORT_INTERVAL)); + let qos = feedback.qos_report(&batch.last_qos_report, QOS_REPORT_INTERVAL); + let qos_bytes = qos.command().encoded(); + batch.append_qos(qos, &qos_bytes, now + QOS_REPORT_INTERVAL); + let mut expected = ack; + expected.extend_from_slice(&qos_bytes); + assert!(!batch.flush(now + QOS_REPORT_INTERVAL, &feedback, |bytes| { + assert_eq!(bytes, expected); + false + })); + assert_eq!(batch.frame_acks_sent, 0); + assert_eq!(batch.qos_reports_sent, 0); + assert_eq!(batch.last_qos_report.sequence, 0); + assert!(batch.flush_due(now + QOS_REPORT_INTERVAL)); + assert!(batch.flush( + now + QOS_REPORT_INTERVAL + Duration::from_millis(1), + &feedback, + |bytes| { + assert_eq!(bytes, expected); + true + } + )); + assert_eq!(batch.frame_acks_sent, 1); + assert_eq!(batch.qos_reports_sent, 1); + assert_eq!(batch.last_qos_report.sequence, 1); + assert_eq!( + batch.last_qos_send, + now + QOS_REPORT_INTERVAL + Duration::from_millis(1) + ); + assert!(!batch.flush_due(batch.last_qos_send)); + assert_eq!( + feedback + .qos_report(&batch.last_qos_report, QOS_REPORT_INTERVAL * 2) + .sequence, + 2 + ); + } + + #[test] + fn failed_batch_stays_bounded_while_new_frame_queue_evicts_oldest() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + let mut batch = ControlReportBatch::new(now, MAX_CONTROL_REPORT_BYTES); + feedback.publish_accepted_frame(1, 1, now); + let first = feedback.take_completed_frame().unwrap(); + batch.append_ack(first, &frame_ack(1, Some(0.0), 1).encoded(), now); + assert!(!batch.flush(now, &feedback, |_| false)); + for number in 2..=u32::try_from(MAX_PENDING_FRAME_ACKS + 3).unwrap() { + feedback.publish_accepted_frame(number, 1, now); + } + assert_eq!(batch.frames.len(), 1); + assert_eq!(batch.bytes.len(), FRAME_ACK_RECORD_LEN); + assert_eq!(feedback.take_completed_frame().unwrap().frame_number, 4); + assert!(batch.flush(now, &feedback, |_| true)); + assert_eq!(batch.last_ack_frame, Some(1)); + } + + #[test] + fn continuous_rtp_with_a_gs_index_gap_discards_the_frame_and_requests_pli() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF, 9, &[0, 0, 1, 0x65]), + 0, + ); + let mut skipped_gs_index = build_plaintext_rtp(11, FLAG_EOF, 9, &[0xaa]); + skipped_gs_index[16..20].copy_from_slice(&(12_u32 << 8).to_le_bytes()); + let skipped_gs_index = protect_for_test(&crypto, skipped_gs_index, 0); + let mut receiver = NvstVideoReceiver::new(config); + + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .is_empty() + ); + assert_eq!( + receiver.process_datagram(peer(), &skipped_gs_index, Instant::now()), + [NvstReceiveEvent::Dropped( + NvstDropReason::FrameDiscontinuity + )] + ); + assert!(feedback.keyframe_request_pending()); + } + + #[test] + fn a_new_frame_start_recovers_after_a_gs_index_gap() { + let config = config(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp( + 10, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 9, + &[0, 0, 1, 0x65], + ), + 0, + ); + let mut next_frame = build_plaintext_rtp( + 11, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 10, + &[0, 0, 1, 0x65], + ); + next_frame[16..20].copy_from_slice(&(30_u32 << 8).to_le_bytes()); + let next_frame = protect_for_test(&crypto, next_frame, 0); + let mut receiver = NvstVideoReceiver::new(config); + + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .iter() + .any(|event| matches!(event, NvstReceiveEvent::Frame(_))) + ); + assert!( + receiver + .process_datagram(peer(), &next_frame, Instant::now()) + .iter() + .any(|event| matches!(event, NvstReceiveEvent::Frame(_))) + ); + } + + #[test] + fn a_new_sof_after_missing_eof_marks_the_reference_chain_discontinuous() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let incomplete = protect_for_test( + &crypto, + build_plaintext_rtp( + 10, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + 9, + &[0, 0, 1, 0x41, 0xaa], + ), + 0, + ); + let following = protect_for_test( + &crypto, + build_plaintext_rtp( + 11, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 10, + &[0, 0, 1, 0x41, 0xbb], + ), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + + assert!( + receiver + .process_datagram(peer(), &incomplete, Instant::now()) + .is_empty() + ); + let events = receiver.process_datagram(peer(), &following, Instant::now()); + assert!(events.iter().any(|event| matches!( + event, + NvstReceiveEvent::Dropped(NvstDropReason::FrameDiscontinuity) + ))); + let frame = events + .iter() + .find_map(|event| match event { + NvstReceiveEvent::Frame(frame) => Some(frame), + _ => None, + }) + .expect("following frame"); + assert!(!frame.contiguous); + assert!(feedback.keyframe_request_pending()); + } + + #[test] + fn fec_repair_packets_are_not_forwarded_as_video_payload() { + let config = config(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF, 9, &[0, 0, 1, 0x65]), + 0, + ); + let mut repair = build_plaintext_rtp(11, FLAG_CONTAINS_PIC_DATA, 9, &[0xee]); + repair[28..32].copy_from_slice(&0x00c0_3420_u32.to_le_bytes()); + let repair = protect_for_test(&crypto, repair, 0); + let mut receiver = NvstVideoReceiver::new(config); + + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .is_empty() + ); + assert!( + receiver + .process_datagram(peer(), &repair, Instant::now()) + .is_empty(), + "a parity shard must stay in the FEC buffer rather than reaching H.264 assembly", + ); + } + + #[test] + fn authenticated_packet_without_gs_metadata_discards_reassembly_and_requests_pli() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF, 9, &[0, 0, 1, 0x65]), + 0, + ); + let mut missing_gs = build_plaintext_rtp(11, FLAG_EOF, 9, &[0xaa]); + missing_gs[12..14].copy_from_slice(&0xbede_u16.to_be_bytes()); + let missing_gs = protect_for_test(&crypto, missing_gs, 0); + let mut receiver = NvstVideoReceiver::new(config); + + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .is_empty() + ); + assert_eq!( + receiver.process_datagram(peer(), &missing_gs, Instant::now()), + [NvstReceiveEvent::Dropped(NvstDropReason::MalformedRtp( + RtpParseError::MissingNvVideoHeader + ))] + ); + assert!(feedback.keyframe_request_pending()); + } + + #[test] + fn receiver_assembles_one_frame_across_multiple_fec_blocks() { + let config = config(); + let crypto = test_srtp(&config); + let block_zero_start = protect_for_test( + &crypto, + build_plaintext_rtp_with_fec_blocks( + 20, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + 9, + 0, + 1, + &[0, 0, 0, 1, 0x65], + ), + 0, + ); + let block_zero_end = protect_for_test( + &crypto, + build_plaintext_rtp_with_fec_blocks( + 21, + FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 9, + 0, + 1, + &[0xaa], + ), + 0, + ); + let block_one_start = protect_for_test( + &crypto, + build_plaintext_rtp_with_fec_blocks( + 22, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + 9, + 1, + 1, + &[0xbb], + ), + 0, + ); + let block_one_end = protect_for_test( + &crypto, + build_plaintext_rtp_with_fec_blocks( + 23, + FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 9, + 1, + 1, + &[0xcc], + ), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + for packet in [&block_zero_start, &block_zero_end, &block_one_start] { + assert!( + receiver + .process_datagram(peer(), packet, Instant::now()) + .is_empty() + ); + } + let events = receiver.process_datagram(peer(), &block_one_end, Instant::now()); + let [NvstReceiveEvent::Frame(frame)] = events.as_slice() else { + panic!("expected one complete frame, got {events:?}"); + }; + assert_eq!(frame.frame_index, 9); + assert!(frame.keyframe); + assert_eq!(frame.bytes, [0, 0, 0, 1, 0x65, 0xaa, 0xbb, 0xcc]); + } + + #[test] + fn receiver_rejects_wrong_peer_and_duplicate_authenticated_packet() { + let config = config(); + let crypto = test_srtp(&config); + let packet = protect_for_test( + &crypto, + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 1, + &[0, 0, 1, 0x65], + ), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + let wrong_peer = SocketAddr::new("192.0.2.21".parse().expect("IP"), 5004); + assert!(matches!( + receiver + .process_datagram(wrong_peer, &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::UnexpectedSource { .. } + )] + )); + assert!(matches!( + receiver + .process_datagram(peer(), &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Frame(_)] + )); + assert!(matches!( + receiver + .process_datagram(peer(), &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped(NvstDropReason::ReplayRejected)] + )); + } + + #[test] + fn replay_window_accepts_late_unseen_packets_across_the_full_reorder_window() { + let mut replay = ReplayWindow::default(); + replay.check(10).expect("first packet"); + replay.commit(10); + replay.check(1_500).expect("new highest packet"); + replay.commit(1_500); + + // This packet is far beyond the old 64-entry filter but is still inside the bounded + // 2,048-packet reorder/NACK envelope and has never been authenticated before. + replay.check(11).expect("late retransmission"); + replay.commit(11); + assert_eq!(replay.check(11), Err(NvstDropReason::ReplayRejected)); + + replay.check(2_058).expect("advance to replay edge"); + replay.commit(2_058); + assert_eq!(replay.check(10), Err(NvstDropReason::ReplayRejected)); + assert_eq!(replay.check(11), Err(NvstDropReason::ReplayRejected)); + replay + .check(12) + .expect("old unseen packet inside replay edge"); + assert_eq!(replay.check(2_057), Ok(())); + } + + #[test] + fn replay_window_shift_preserves_seen_bits_across_word_boundaries() { + let mut replay = ReplayWindow::default(); + for index in [100, 99, 63, 1] { + replay.check(index).expect("unseen packet"); + replay.commit(index); + } + replay.check(165).expect("advance across a bitmap word"); + replay.commit(165); + for index in [100, 99, 63, 1, 165] { + assert_eq!(replay.check(index), Err(NvstDropReason::ReplayRejected)); + } + replay + .check(98) + .expect("nearby unseen packet remains admissible"); + } + + #[test] + fn fec_block_reconstructs_a_missing_shard_inside_a_multiblock_frame() { + let base_index = 4_000_u64; + let mut data = (0..3_u16) + .map(|offset| { + // This is block 1 of 0..=2, so neither its first nor last shard carries the + // whole-frame SOF/EOF markers. + let mut packet = build_plaintext_rtp_with_fec_blocks( + (base_index as u16).wrapping_add(offset), + FLAG_CONTAINS_PIC_DATA, + 77, + 1, + 2, + &[offset as u8; 32], + ); + let fec_word = (33_u32 << 4) | (u32::from(offset) << 12) | (3_u32 << 22); + let extension_start = RtpHeader::parse(&packet) + .expect("RTP header") + .payload_offset + - NV_VIDEO_PACKET_LEN; + packet[extension_start + 12..extension_start + 16] + .copy_from_slice(&fec_word.to_le_bytes()); + packet + }) + .collect::>(); + let expected_middle = data[1].clone(); + let mut parity = vec![0; data[0].len()]; + for (data_index, shard) in data.iter().enumerate() { + let coefficient = + nvst_cauchy_coefficient(data_index, 0, 1).expect("Cauchy parity coefficient"); + gf256_axpy(&mut parity, shard, coefficient); + } + // Parity is transported inside its own parseable RTP envelope. These bytes are not part + // of the useful recovered NV payload and must be restored before strict RTP parsing. + parity[..16].copy_from_slice(&data[0][..16]); + parity[2..4].copy_from_slice(&(base_index as u16 + 3).to_be_bytes()); + // NVIDIA's fecInfo bytes are transport metadata and are not guaranteed to reconstruct + // to the original data-shard value. Moonlight deliberately excludes them from its + // recovered-packet checks. Model that cloud behavior while keeping the encoded video + // bytes themselves recoverable. + let fec_info_start = RtpHeader::parse(&expected_middle) + .expect("RTP header") + .payload_offset; + parity[fec_info_start - 4..fec_info_start].fill(0xa5); + data.push(parity); + + let layout = |shard_index| FecPacketLayout { + frame_index: 77, + block_index: 1, + last_block_index: 2, + shard_index, + data_shards: 3, + parity_shards: 1, + }; + let packet = |shard_index: usize, plaintext: Vec| RtpPacket { + index: base_index + shard_index as u64, + header: RtpHeader::parse(&expected_middle).expect("template header"), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let mut block = FecBlock::new(layout(0), base_index); + assert_eq!(block.insert(layout(0), packet(0, data[0].clone())), None); + assert_eq!(block.insert(layout(2), packet(2, data[2].clone())), None); + assert_eq!(block.insert(layout(3), packet(3, data[3].clone())), None); + assert!(block.has_enough_shards()); + + let (repaired, repaired_count) = block.finish(expected_middle.len()).expect("FEC repair"); + assert_eq!(repaired_count, 1); + assert_eq!(repaired.len(), 3); + assert_eq!(repaired[1].index, base_index + 1); + assert_eq!( + &repaired[1].plaintext[..fec_info_start - 4], + &expected_middle[..fec_info_start - 4] + ); + assert_ne!( + &repaired[1].plaintext[fec_info_start - 4..fec_info_start], + &expected_middle[fec_info_start - 4..fec_info_start] + ); + assert_eq!( + &repaired[1].plaintext[fec_info_start..], + &expected_middle[fec_info_start..] + ); + + let mut reorder = FecReorderBuffer::new(data[0].len() - NVST_FEC_RTP_HEADER_ALLOWANCE); + assert!( + reorder + .push(packet(0, data[0].clone()), layout(0), Instant::now()) + .ready + .is_empty() + ); + assert!( + reorder + .push(packet(1, data[1].clone()), layout(1), Instant::now()) + .ready + .is_empty() + ); + assert_eq!( + reorder + .push(packet(2, data[2].clone()), layout(2), Instant::now()) + .ready + .len(), + 3 + ); + let late_parity = reorder.push(packet(3, data[3].clone()), layout(3), Instant::now()); + assert!(late_parity.ready.is_empty()); + assert!(late_parity.recovery.is_none()); + } + + #[test] + fn fec_block_reconstructs_single_data_shard_from_parity_only() { + let base_index = 17_392_u64; + let mut data = build_plaintext_rtp_with_fec_blocks( + base_index as u16, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 1_466, + 0, + 0, + &[0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21], + ); + let extension_start = RtpHeader::parse(&data) + .expect("data RTP header") + .payload_offset + - NV_VIDEO_PACKET_LEN; + let fec_word = (data.len() as u32) | (1_u32 << 22) | (2_u32 << 26); + data[extension_start + 12..extension_start + 16].copy_from_slice(&fec_word.to_le_bytes()); + + let mut parity = Vec::new(); + for parity_index in 0..2 { + let coefficient = nvst_cauchy_coefficient(0, parity_index, 2) + .expect("single-data Cauchy coefficient"); + let mut shard = vec![0_u8; data.len()]; + gf256_axpy(&mut shard, &data, coefficient); + // A parity shard is carried in its own valid RTP envelope. FEC recovery restores + // these bytes from the received template before parsing the recovered data packet. + shard[..extension_start].copy_from_slice(&data[..extension_start]); + shard[2..4] + .copy_from_slice(&(base_index as u16 + 1 + parity_index as u16).to_be_bytes()); + parity.push(shard); + } + + let layout = |shard_index| FecPacketLayout { + frame_index: 1_466, + block_index: 0, + last_block_index: 0, + shard_index, + data_shards: 1, + parity_shards: 2, + }; + let packet = |shard_index: usize, plaintext: Vec| RtpPacket { + index: base_index + shard_index as u64, + header: RtpHeader::parse(&plaintext).expect("parity RTP header"), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let mut block = FecBlock::new(layout(0), base_index); + assert_eq!(block.insert(layout(1), packet(1, parity[0].clone())), None); + assert!(block.has_enough_shards()); + + let (repaired, repaired_count) = block.finish(data.len()).expect("parity-only FEC repair"); + assert_eq!(repaired_count, 1); + assert_eq!(repaired.len(), 1); + assert_eq!(repaired[0].index, base_index); + assert_eq!(repaired[0].header.sequence_number, base_index as u16); + assert_eq!( + &repaired[0].plaintext[extension_start..], + &data[extension_start..] + ); + } + + #[test] + fn nvst_cauchy_fec_reconstructs_multiple_missing_data_shards() { + let data = (0..5_usize) + .map(|shard| { + (0..97_usize) + .map(|byte| (shard * 41 + byte * 17) as u8) + .collect::>() + }) + .collect::>(); + let mut shards = data.iter().cloned().map(Some).collect::>(); + for parity_index in 0..2 { + let mut parity = vec![0_u8; data[0].len()]; + for (data_index, source) in data.iter().enumerate() { + let coefficient = nvst_cauchy_coefficient(data_index, parity_index, 2) + .expect("Cauchy coefficient"); + gf256_axpy(&mut parity, source, coefficient); + } + shards.push(Some(parity)); + } + shards[1] = None; + shards[3] = None; + + reconstruct_nvst_cauchy_data(&mut shards, 5, 2, data[0].len()).expect("two-shard repair"); + assert_eq!(shards[1].as_deref(), Some(data[1].as_slice())); + assert_eq!(shards[3].as_deref(), Some(data[3].as_slice())); + } + + #[test] + fn fec_reorder_retains_following_blocks_while_a_retransmission_is_pending() { + fn packet(index: u64) -> RtpPacket { + let mut plaintext = vec![0_u8; RTP_FIXED_HEADER_LEN]; + plaintext[0] = 0x80; + plaintext[1] = 96; + plaintext[2..4].copy_from_slice(&(index as u16).to_be_bytes()); + plaintext[4..8].copy_from_slice(&90_000_u32.to_be_bytes()); + plaintext[8..12].copy_from_slice(&1_u32.to_be_bytes()); + RtpPacket { + index, + header: RtpHeader::parse(&plaintext).expect("RTP header"), + plaintext, + origin: RtpPacketOrigin::Authenticated, + } + } + fn layout(block_index: u8, shard_index: usize) -> FecPacketLayout { + FecPacketLayout { + frame_index: 7, + block_index, + last_block_index: 1, + shard_index, + data_shards: 2, + parity_shards: 1, + } + } + + let started = Instant::now(); + let mut reorder = FecReorderBuffer::new(48); + let first = packet(100); + let first_buffer = first.plaintext.as_ptr(); + assert!(reorder.push(first, layout(0, 0), started).ready.is_empty()); + + let successor = reorder.push( + packet(103), + layout(1, 0), + started + Duration::from_millis(1), + ); + assert_eq!(successor.nack, Some((101, 101))); + assert!(successor.ready.is_empty()); + assert!(successor.recovery.is_none()); + + let retransmitted = reorder.push( + packet(101), + layout(0, 1), + started + Duration::from_millis(20), + ); + assert_eq!( + retransmitted + .ready + .iter() + .map(|packet| packet.index) + .collect::>(), + [100, 101] + ); + assert!(retransmitted.recovery.is_none()); + assert_eq!( + retransmitted.ready[0].plaintext.as_ptr(), + first_buffer, + "a healthy completed block must retain the original packet allocation" + ); + assert_eq!(reorder.blocks.len(), 1, "the successor stays buffered"); + } + + #[test] + fn reorder_nack_excludes_packets_already_buffered_after_the_gap() { + let config = config(); + let feedback = Arc::clone(&config.feedback); + let crypto = test_srtp(&config); + let packets = [1_u16, 3, 4].map(|sequence| { + protect_for_test( + &crypto, + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + u32::from(sequence), + &[0, 0, 1, 0x65], + ), + 0, + ) + }); + let mut receiver = NvstVideoReceiver::new(config); + for packet in &packets { + let _ = receiver.process_datagram(peer(), packet, Instant::now()); + } + assert_eq!(feedback.take_nack(Instant::now(), None), Some((2, 2))); + assert_eq!(feedback.take_nack(Instant::now(), None), None); + } + + #[test] + fn reorder_gap_uses_mjolnir_dequeue_deadline_instead_of_filling_the_window() { + fn packet(index: u64) -> RtpPacket { + RtpPacket { + index, + header: RtpHeader { + payload_type: 96, + sequence_number: index as u16, + timestamp: 90_000, + ssrc: 1, + payload_offset: RTP_FIXED_HEADER_LEN, + has_padding: false, + gs_video_header: None, + }, + plaintext: vec![0; RTP_FIXED_HEADER_LEN], + origin: RtpPacketOrigin::Authenticated, + } + } + + let started = Instant::now(); + let mut reorder = RtpReorderBuffer::new(DEFAULT_REORDER_WINDOW); + assert_eq!(reorder.push(packet(1), started).ready.len(), 1); + + let gap_started = started + Duration::from_millis(1); + let waiting = reorder.push(packet(3), gap_started); + assert_eq!(waiting.nack, Some((2, 2))); + assert!(waiting.ready.is_empty()); + assert!(waiting.recovery.is_none()); + + let still_waiting = reorder.push( + packet(4), + gap_started + MJOLNIR_REORDER_DEQUEUE_TIMEOUT - Duration::from_millis(1), + ); + assert_eq!(still_waiting.nack, Some((2, 2))); + assert!(still_waiting.ready.is_empty()); + assert!(still_waiting.recovery.is_none()); + + let recovered = reorder.push(packet(5), gap_started + MJOLNIR_REORDER_DEQUEUE_TIMEOUT); + assert_eq!( + recovered.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 2, + last_missing_index: 2, + }) + ); + assert_eq!( + recovered + .ready + .iter() + .map(|packet| packet.index) + .collect::>(), + [3, 4, 5] + ); + assert_eq!(recovered.nack, None); + } + + #[test] + fn gaps_require_recovery_and_never_emit_an_incomplete_frame() { + let config = config(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(1, FLAG_SOF | FLAG_CONTAINS_PIC_DATA, 1, &[0, 0, 1, 0x65]), + 0, + ); + let far = protect_for_test( + &crypto, + build_plaintext_rtp(6, FLAG_EOF | FLAG_CONTAINS_PIC_DATA, 1, &[0xaa]), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + let _ = receiver.process_datagram(peer(), &first, Instant::now()); + let events = receiver.process_datagram(peer(), &far, Instant::now()); + assert!(events.iter().any(|event| matches!( + event, + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { .. }) + ))); + assert!( + !events + .iter() + .any(|event| matches!(event, NvstReceiveEvent::Frame(_))) + ); + + let fresh = protect_for_test( + &crypto, + build_plaintext_rtp( + 7, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 2, + &[0, 0, 1, 0x65, 0xbb], + ), + 0, + ); + let fresh_events = receiver.process_datagram(peer(), &fresh, Instant::now()); + let Some(NvstReceiveEvent::Frame(frame)) = fresh_events + .iter() + .find(|event| matches!(event, NvstReceiveEvent::Frame(_))) + else { + panic!("expected a fresh frame after the gap, got {fresh_events:?}"); + }; + assert!(!frame.contiguous); + } + + #[test] + fn generic_nack_encodes_wrapping_sequence_range() { + let packet = build_rtcp_nack(0x0102_0304, 0x0506_0708, 65_534, 65_536); + assert_eq!( + packet, + [0x81, 205, 0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 0xff, 0xfe, 0, 3,] + ); + } + + #[test] + fn picture_loss_indication_uses_rfc_4585_psfb_layout() { + assert_eq!( + build_rtcp_pli(0x0102_0304, 0x0506_0708), + [0x81, 206, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8] + ); + } + + #[test] + fn raw_srtcp_gcm8_matches_the_macforce_pr66_layout() { + let key = decode_fixed_hex::<32>( + "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + NvstConfigError::InvalidAesKey, + ) + .expect("key"); + let salt = + decode_fixed_hex::<12>("A0A1A2A3A4A5A6A7A8A9AAAB", NvstConfigError::InvalidSrtpSalt) + .expect("salt"); + let packet = protect_srtcp_receiver_report_gcm( + RtcpReportBlock { + media_ssrc: 0x1122_3344, + fraction_lost: 0x55, + cumulative_lost: -2, + highest_sequence: 0x0102_0304, + jitter: 0x0506_0708, + }, + 0x0123_4567, + &key, + &salt, + SRTP_AEAD_AES_GCM_8_TAG_LEN, + ); + + assert_eq!( + packet, + hex_bytes( + "81C900074F4E4F57ECBBE4F812433B089B30BF8BA442E7BB3C3C706A5BC548980601CFF1E416E06F81234567" + ) + ); + assert_eq!(&packet[..8], &[0x81, 201, 0, 7, b'O', b'N', b'O', b'W']); + assert_eq!(&packet[packet.len() - 4..], &0x8123_4567_u32.to_be_bytes()); + } + + #[test] + fn receiver_report_tracks_loss_interval_and_late_recovery() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.publish_stream(7, 10, 1_000, now); + feedback.publish_stream(7, 12, 2_800, now + Duration::from_millis(20)); + + let report = feedback.report_snapshot(true).expect("report"); + assert_eq!(report.media_ssrc, 7); + assert_eq!(report.highest_sequence, 12); + assert_eq!(report.cumulative_lost, 1); + assert_eq!(report.fraction_lost, 85); + assert_eq!(report.jitter, 0); + + feedback.publish_stream(7, 12, 1_900, now + Duration::from_millis(10)); + let recovered = feedback.report_snapshot(true).expect("report"); + assert_eq!(recovered.cumulative_lost, 0); + assert_eq!(recovered.fraction_lost, 0); + } + + #[test] + fn raw_receiver_report_advances_interval_loss_counters() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF | FLAG_EOF, 1, &[0, 0, 1, 0x65]), + 0, + ); + let later = protect_for_test( + &crypto, + build_plaintext_rtp(12, FLAG_SOF | FLAG_EOF, 2, &[0, 0, 1, 0x65]), + 0, + ); + let now = Instant::now(); + let mut receiver = NvstVideoReceiver::new(config); + let _ = receiver.process_datagram(peer(), &first, now); + let _ = receiver.process_datagram(peer(), &later, now + Duration::from_millis(20)); + + assert!(receiver.poll_receiver_report(now).is_some()); + assert_eq!( + *feedback + .report_prior + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + (3, 2) + ); + } + + #[test] + fn raw_receiver_report_is_disabled_when_rtcp_is_on_sctp() { + let mut handoff = legacy_handoff(); + handoff["rtcpOnSctp"] = json!(true); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + assert!(config.rtcp_on_sctp()); + let crypto = test_srtp(&config); + let packet = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF | FLAG_EOF, 1, &[0, 0, 1, 0x65]), + 0, + ); + let now = Instant::now(); + let mut receiver = NvstVideoReceiver::new(config); + let _ = receiver.process_datagram(peer(), &packet, now); + + assert!(receiver.poll_receiver_report(now).is_none()); + } + + #[test] + fn negotiated_rtcp_flag_does_not_strand_bundle_loss_feedback() { + for rtcp_on_sctp in [false, true] { + let mut handoff = legacy_handoff(); + handoff["rtcpOnSctp"] = json!(rtcp_on_sctp); + let config = + NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + assert_eq!(config.rtcp_on_sctp(), rtcp_on_sctp); + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, config.mjolnir_udp_port.is_none()); + assert!(channels.rtcp.is_some()); + + handoff["mjolnirUdpPort"] = json!(49006); + let config = + NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, config.mjolnir_udp_port.is_none()); + assert!(channels.rtcp.is_none()); + assert!(channels.cursor.is_none()); + } + } + + #[test] + fn duplicate_rtp_timestamps_do_not_inflate_interarrival_jitter() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.publish_stream(7, 10, 1_000, now); + feedback.publish_stream(7, 11, 1_000, now + Duration::from_millis(10)); + feedback.publish_stream(7, 12, 2_800, now + Duration::from_millis(20)); + + assert_eq!(feedback.report_snapshot(false).expect("report").jitter, 0); + } + + #[test] + fn rtp_classifier_does_not_bind_rtcp_or_dtls_as_media_streams() { + let rtcp = build_rtcp_receiver_report( + 1, + RtcpReportBlock { + media_ssrc: 2, + fraction_lost: 0, + cumulative_lost: 0, + highest_sequence: 3, + jitter: 0, + }, + ); + assert!(looks_like_rtcp(&rtcp)); + assert!(!looks_like_rtp(&rtcp)); + assert_eq!(peek_rtp_ssrc(&rtcp), None); + + let mut dtls = vec![0_u8; RTP_FIXED_HEADER_LEN]; + dtls[0] = 23; + assert!(looks_like_dtls(&dtls)); + assert!(!looks_like_rtp(&dtls)); + } + + #[test] + fn generic_nack_is_bounded() { + let packet = build_rtcp_nack(1, 2, 10, 10_000); + assert_eq!(packet.len(), 12 + MAX_NACK_FCI_ENTRIES * 4); + assert_eq!( + &packet[2..4], + &(2_u16 + MAX_NACK_FCI_ENTRIES as u16).to_be_bytes() + ); + assert_eq!(&packet[packet.len() - 4..], &[0, 61, 0x0f, 0xff]); + } + + #[test] + fn pending_nack_ranges_are_chunked_without_discarding_tail() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(10, 100, now); + feedback.resolve_nack(12); + assert_eq!(feedback.take_nack(now, None), Some((10, 11))); + assert_eq!(feedback.take_nack(now, None), Some((13, 76))); + assert_eq!(feedback.take_nack(now, None), Some((77, 100))); + assert_eq!(feedback.take_nack(now, None), None); + + feedback.clear_nacks(); + feedback.request_nack(10, 100, now); + assert_eq!(feedback.take_nack(now, None), Some((10, 73))); + assert_eq!(feedback.take_nack(now, None), Some((74, 100))); + assert_eq!(feedback.take_nack(now, None), None); + } + + #[test] + fn sent_nacks_remain_outstanding_for_resolution_without_immediate_resend() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(41, 41, now); + assert!(!feedback.resolve_nack(41)); + + feedback.request_nack(42, 42, now); + + assert_eq!(feedback.take_nack(now, None), Some((42, 42))); + feedback.request_nack(42, 42, now + Duration::from_millis(1)); + assert_eq!( + feedback.take_nack(now + NACK_RETRY_INTERVAL / 2, None), + None + ); + assert!(feedback.resolve_nack(42)); + assert_eq!(feedback.take_nack(now + NACK_RETRY_INTERVAL, None), None); + } + + #[test] + fn nack_retries_are_bounded_and_expire() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(7, 7, now); + + for attempt in 0..MAX_NACK_ATTEMPTS { + assert_eq!( + feedback.take_nack(now + NACK_RETRY_INTERVAL * u32::from(attempt), None), + Some((7, 7)) + ); + } + assert_eq!( + feedback.take_nack(now + NACK_RETRY_INTERVAL * 4, None), + None + ); + assert_eq!(feedback.take_nack(now + NACK_TRACKING_TIMEOUT, None), None); + } + + #[test] + fn pause_timeout_recovery_and_stop_fail_closed() { + let mut receiver = NvstVideoReceiver::new(config()); + assert_eq!( + receiver.pause(), + Some(NvstReceiveEvent::Lifecycle(NvstReceiverState::Paused)) + ); + assert!(matches!( + receiver + .process_datagram(peer(), &[], Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped(NvstDropReason::Paused)] + )); + assert_eq!( + receiver.resume(), + Some(NvstReceiveEvent::Lifecycle(NvstReceiverState::Running)) + ); + receiver.last_authenticated_packet = Some(Instant::now() - Duration::from_secs(1)); + assert!(matches!( + receiver.poll_timeout(Instant::now()), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + assert_eq!(receiver.state(), NvstReceiverState::RecoveryRequired); + assert_eq!( + receiver.recover(), + Some(NvstReceiveEvent::Lifecycle(NvstReceiverState::Running)) + ); + assert_eq!( + receiver.stop(NvstStopCause::Requested), + Some(NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped( + NvstStopCause::Requested + ))) + ); + assert_eq!(receiver.stop(NvstStopCause::MediaConsumerClosed), None); + assert_eq!( + receiver.state(), + NvstReceiverState::Stopped(NvstStopCause::Requested), + "the first stop cause must survive later teardown paths" + ); + assert!(matches!( + receiver + .process_datagram(peer(), &[], Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped(NvstDropReason::Stopped)] + )); + } + + #[test] + fn startup_without_authenticated_packets_times_out() { + let mut receiver = NvstVideoReceiver::new(config()); + let feedback = receiver.config.feedback(); + let origin = receiver.timeout_origin; + let deadline = origin + receiver.config.startup_timeout; + + assert!(!feedback.keyframe_request_pending()); + assert_eq!( + feedback.keyframe_request_routes(deadline, origin, true, true), + (false, false) + ); + assert_eq!( + receiver.poll_frame_progress( + deadline, + NvstFrameProgressPolicy { + stall: receiver.config.timeout, + keyframe_grace: receiver.config.timeout, + }, + ), + None + ); + assert_eq!( + receiver.poll_timeout(deadline - Duration::from_millis(1)), + None + ); + + assert!(matches!( + receiver.poll_timeout(deadline), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + assert_eq!(receiver.state(), NvstReceiverState::RecoveryRequired); + assert!(!feedback.keyframe_request_pending()); + } + + #[test] + fn recovery_and_resume_start_a_fresh_media_timeout() { + for paused in [false, true] { + let mut receiver = NvstVideoReceiver::new(config()); + receiver.last_authenticated_packet = Some(Instant::now() - Duration::from_secs(1)); + if paused { + receiver.pause().expect("pause running receiver"); + receiver.resume().expect("resume paused receiver"); + } else { + receiver + .poll_timeout(Instant::now()) + .expect("initial timeout"); + receiver.recover().expect("recover timed out receiver"); + } + assert_eq!(receiver.poll_timeout(Instant::now()), None); + assert!(matches!( + receiver.poll_timeout(receiver.timeout_origin + receiver.config.timeout), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + } + + #[test] + fn startup_timeout_is_optional_and_bounded_by_idle_timeout_and_maximum() { + assert_eq!(config().startup_timeout, config().timeout); + for milliseconds in [500, 60_000, 90_000] { + let mut handoff = legacy_handoff(); + handoff["startupTimeoutMs"] = json!(milliseconds); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.startup_timeout, Duration::from_millis(milliseconds)); + } + for invalid in [ + json!(0), + json!(499), + json!(90_001), + json!(-1), + json!("60000"), + ] { + let mut handoff = legacy_handoff(); + handoff["startupTimeoutMs"] = invalid; + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + } + } + + #[test] + fn startup_grace_waits_for_firewall_without_spending_recovery_budget() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + let mut receiver = NvstVideoReceiver::new(config); + let origin = receiver.timeout_origin; + + for seconds in [8, 16, 30, 59] { + assert_eq!( + receiver.poll_timeout(origin + Duration::from_secs(seconds)), + None + ); + assert_eq!(receiver.state(), NvstReceiverState::Running); + } + assert!(matches!( + receiver.poll_timeout(origin + Duration::from_secs(60)), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + assert_eq!( + receiver.poll_timeout(origin + Duration::from_secs(61)), + None + ); + receiver + .recover() + .expect("bounded recovery after startup grace"); + assert!(matches!( + receiver.poll_timeout(receiver.timeout_origin + receiver.config.timeout), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + + #[test] + fn authenticated_video_ends_startup_grace_and_keeps_idle_recovery_short() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 1, + &[0, 0, 1, 0x65], + ), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + let received_at = receiver.timeout_origin + Duration::from_secs(30); + assert_eq!(receiver.poll_timeout(received_at), None); + let events = receiver.process_datagram(peer(), &packet, received_at); + assert!( + events + .iter() + .any(|event| matches!(event, NvstReceiveEvent::Frame(_))) + ); + assert_eq!(receiver.poll_timeout(received_at), None); + assert!(matches!( + receiver.poll_timeout(received_at + receiver.config.timeout), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + receiver.recover().expect("recover established stream"); + assert!(matches!( + receiver.poll_timeout(receiver.timeout_origin + receiver.config.timeout), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + + #[test] + fn invalid_datagrams_do_not_extend_or_end_startup_grace() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + let mut receiver = NvstVideoReceiver::new(config); + let origin = receiver.timeout_origin; + for seconds in [8, 16, 30, 59] { + let now = origin + Duration::from_secs(seconds); + assert!(matches!( + receiver.process_datagram(peer(), &[], now).as_slice(), + [NvstReceiveEvent::Dropped(_)] + )); + assert_eq!(receiver.poll_timeout(now), None); + } + assert!(matches!( + receiver.poll_timeout(origin + Duration::from_secs(60)), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + + #[test] + fn resume_does_not_regrant_startup_grace() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + let mut receiver = NvstVideoReceiver::new(config); + receiver.pause().expect("pause during startup"); + receiver.resume().expect("resume during startup"); + assert!(matches!( + receiver.poll_timeout(receiver.timeout_origin + receiver.config.timeout), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + + #[test] + fn video_peer_range_handoff_is_bounded_and_defaults_to_one_port() { + let mut handoff = legacy_handoff(); + let single = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(single.video_peer_ports(), 5004..=5004); + for end in [5004, 5005, 5019] { + handoff["videoPeerPortEnd"] = json!(end); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.video_peer_ports(), 5004..=end); + } + for invalid in [ + json!(0), + json!(5003), + json!(5020), + json!(65536), + json!(-1), + json!("5005"), + ] { + handoff["videoPeerPortEnd"] = invalid; + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_err()); + } + handoff["videoPeerPort"] = json!(65535); + handoff["videoPeerPortEnd"] = json!(65535); + assert!(NvstVideoConfig::from_legacy_handoff(&handoff, None).is_ok()); + } + + #[test] + fn video_peer_range_preserves_source_authentication_and_replay_checks() { + let mut handoff = legacy_handoff(); + handoff["videoPeerPortEnd"] = json!(5005); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 1, + &[0, 0, 1, 0x65], + ), + 0, + ); + let mut receiver = NvstVideoReceiver::new(config); + for source in [ + SocketAddr::new(peer().ip(), 5003), + SocketAddr::new(peer().ip(), 5006), + "192.0.2.21:5005".parse().unwrap(), + ] { + assert!(matches!( + receiver + .process_datagram(source, &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::UnexpectedSource { .. } + )] + )); + } + let alternate = SocketAddr::new(peer().ip(), 5005); + let mut corrupted = packet.clone(); + *corrupted.last_mut().unwrap() ^= 1; + assert!(matches!( + receiver + .process_datagram(alternate, &corrupted, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::AuthenticationFailed + )] + )); + assert!(matches!( + receiver + .process_datagram(alternate, &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Frame(_)] + )); + assert!(matches!( + receiver + .process_datagram(peer(), &packet, Instant::now()) + .as_slice(), + [NvstReceiveEvent::Dropped(NvstDropReason::ReplayRejected)] + )); + } + + #[test] + fn mjolnir_punches_negotiated_ports_before_receiving_video_at_normal_and_vpn_sizes() { + let _ports = PREFERRED_NVST_PORTS.lock().unwrap(); + for packet_size in [1280, 1232] { + for use_second_port in [false, true] { + let (second, first) = reserve_nvst_socket_pair().expect("server port pair"); + let first_port = first.local_addr().unwrap().port(); + let second_port = second.local_addr().unwrap().port(); + let server = if use_second_port { &second } else { &first }; + first + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut handoff = legacy_handoff(); + handoff["clientUdpPort"] = json!(client.local_addr().unwrap().port()); + handoff["videoPeerIp"] = json!("127.0.0.1"); + handoff["videoPeerPort"] = json!(first_port); + if use_second_port { + handoff["videoPeerPortEnd"] = json!(second_port); + } + handoff["packetSize"] = json!(packet_size); + let mut config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + config.stun_credentials = Some(stun_credentials()); + config.ping_payload = b"setup-ping".to_vec(); + let feedback = config.feedback(); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 42, + &[0, 0, 1, 0x65], + ), + 0, + ); + let (media_consumer, media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = + spawn_nvst_mjolnir_receiver(client, config, media_consumer, event_sender) + .unwrap(); + let mut datagram = [0_u8; 512]; + let (length, client_address) = first + .recv_from(&mut datagram) + .expect("NAT probe to first server port"); + let (_, username) = + find_stun_attribute(&datagram[..length], STUN_ATTR_USERNAME).unwrap(); + assert_eq!(username, b"setup-ping:loc1"); + assert!(valid_stun_fingerprint(&datagram[..length])); + assert!(valid_stun_message_integrity( + &datagram[..length], + stun_credentials().remote_password.as_bytes() + )); + let transaction_id = [0x11; 12]; + let request = build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + &transaction_id, + stun_credentials().local_password.as_bytes(), + &[(STUN_ATTR_USERNAME, b"loc1:remote01".to_vec())], + ); + server.send_to(&request, client_address).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + loop { + assert!(Instant::now() < deadline, "STUN reply from negotiated port"); + let (length, source) = server.recv_from(&mut datagram).unwrap(); + assert_eq!(source, client_address); + if datagram[..2] == STUN_BINDING_SUCCESS_RESPONSE.to_be_bytes() { + assert_eq!(&datagram[8..20], &transaction_id); + assert!(valid_stun_fingerprint(&datagram[..length])); + assert!(valid_stun_message_integrity( + &datagram[..length], + stun_credentials().local_password.as_bytes() + )); + break; + } + } + server.send_to(&packet, client_address).unwrap(); + let frame = media_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("authenticated video from punched port"); + assert_eq!(frame.frame_index, Some(42)); + assert!(frame.keyframe); + session.stop(); + assert_eq!(feedback.socket_receive_bytes(), packet.len() as u64); + } + } + } + + #[test] + fn bundle_keepalives_do_not_claim_the_video_routing_identity() { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let bundle = UdpSocket::bind("127.0.0.1:0").unwrap(); + let video = UdpSocket::bind("127.0.0.1:0").unwrap(); + let bundle_address = bundle.local_addr().unwrap(); + let video_address = video.local_addr().unwrap(); + let mut remote = create_nvst_bundle_rtc(&server).unwrap(); + let mut config = config(); + config.client_udp_port = bundle_address.port(); + config.mjolnir_udp_port = Some(video_address.port()); + config.video_peer = server.local_addr().unwrap(); + config.remote_dtls_fingerprint = + Some(nvst_local_bundle_identity(&mut remote).dtls_fingerprint); + config.stun_credentials = Some(stun_credentials()); + config.ping_payload = b"setup-ping".to_vec(); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 42, + &[0, 0, 1, 0x65], + ), + 0, + ); + let (media_consumer, media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let bundle_session = spawn_nvst_udp_receiver_with_socket( + config.clone(), + media_consumer.clone(), + event_sender.clone(), + Some(bundle), + None, + Arc::new(HidRuntime::new()), + None, + ) + .unwrap(); + let video_session = + spawn_nvst_mjolnir_receiver(video, config, media_consumer, event_sender).unwrap(); + let mut datagram = [0_u8; 2048]; + let mut bundle_keepalives = 0; + let mut bundle_ice_checks = 0; + let mut video_keepalives = 0; + let deadline = Instant::now() + Duration::from_secs(2); + while bundle_keepalives < 3 || bundle_ice_checks < 3 || video_keepalives < 3 { + assert!( + Instant::now() < deadline, + "both socket roles must send probes" + ); + let (length, source) = server.recv_from(&mut datagram).unwrap(); + let packet = &datagram[..length]; + assert!(valid_stun_fingerprint(packet)); + assert!(valid_stun_message_integrity( + packet, + stun_credentials().remote_password.as_bytes() + )); + let (_, username) = find_stun_attribute(packet, STUN_ATTR_USERNAME).unwrap(); + if source == video_address { + assert_eq!(username, b"setup-ping:loc1"); + video_keepalives += 1; + } else { + assert_eq!(source, bundle_address); + assert_ne!(username, b"setup-ping:loc1", "bundle must not claim video"); + match username { + b"remote01:loc1" => bundle_ice_checks += 1, + b"PING:loc1" => bundle_keepalives += 1, + _ => panic!("unexpected bundle probe identity"), + } + } + } + server.send_to(&packet, video_address).unwrap(); + let frame = media_receiver.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(frame.frame_index, Some(42)); + assert!(frame.keyframe); + video_session.stop(); + bundle_session.stop(); + } + + fn unified_bundle_frame(unified_socket: bool) -> Option { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + let bundle = UdpSocket::bind("127.0.0.1:0").unwrap(); + let bundle_address = bundle.local_addr().unwrap(); + let mut remote = create_nvst_bundle_rtc(&server).unwrap(); + let mut config = config(); + config.client_udp_port = bundle_address.port(); + config.mjolnir_udp_port = None; + config.unified_socket = unified_socket; + config.video_peer = server.local_addr().unwrap(); + config.remote_dtls_fingerprint = + Some(nvst_local_bundle_identity(&mut remote).dtls_fingerprint); + config.stun_credentials = Some(stun_credentials()); + let packet = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 42, + &[0, 0, 1, 0x65], + ), + 0, + ); + let (media_consumer, media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let bundle_session = spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + Some(bundle), + None, + Arc::new(HidRuntime::new()), + None, + ) + .unwrap(); + server.send_to(&packet, bundle_address).unwrap(); + let frame = media_receiver + .recv_timeout(Duration::from_millis(500)) + .ok() + .and_then(|frame| frame.frame_index); + bundle_session.stop(); + frame + } + + #[test] + fn unified_socket_bundle_demuxes_raw_mjolnir_video() { + assert_eq!(unified_bundle_frame(true), Some(42)); + assert_eq!(unified_bundle_frame(false), None); + } + + #[test] + fn unified_socket_handoff_rejects_a_dedicated_video_socket() { + let mut handoff = legacy_handoff(); + handoff["unifiedSocket"] = json!(true); + handoff["mjolnirUdpPort"] = json!(49005); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::OutOfRange { + field: "unifiedSocket" + }) + )); + handoff["mjolnirUdpPort"] = Value::Null; + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert!(config.unified_socket); + assert_eq!(config.mjolnir_udp_port(), None); + assert!(config.carries_mjolnir_video()); + handoff["unifiedSocket"] = json!(false); + let bundle_only = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert!(!bundle_only.carries_mjolnir_video()); + handoff["unifiedSocket"] = json!(true); + handoff["unifiedSocket"] = json!("yes"); + assert!(matches!( + NvstVideoConfig::from_legacy_handoff(&handoff, None), + Err(NvstConfigError::InvalidFieldType { + field: "unifiedSocket", + .. + }) + )); + } + + fn play_gate_loopback(accept_play: bool) { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_millis(10))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client_addr = client.local_addr().unwrap(); + let server_addr = server.local_addr().unwrap(); + let mut client_rtc = create_nvst_bundle_rtc(&client).unwrap(); + let mut server_rtc = create_nvst_bundle_rtc(&server).unwrap(); + let client_identity = nvst_local_bundle_identity(&mut client_rtc); + let server_identity = nvst_local_bundle_identity(&mut server_rtc); + let credentials = stun_credentials(); + server_rtc + .add_local_candidate(Candidate::host(logical_ice_addr(server_addr, 2), "udp").unwrap()); + server_rtc.add_remote_candidate( + Candidate::host(logical_ice_addr(client_addr, 1), "udp").unwrap(), + ); + { + let mut api = server_rtc.direct_api(); + api.set_ice_controlling(false); + api.set_local_ice_credentials(IceCreds { + ufrag: credentials.remote_username_fragment.clone(), + pass: credentials.remote_password.clone(), + }); + api.set_remote_ice_credentials(IceCreds { + ufrag: credentials.local_username_fragment.clone(), + pass: credentials.local_password.clone(), + }); + api.set_remote_fingerprint( + parse_nvst_fingerprint(&client_identity.dtls_fingerprint).unwrap(), + ); + api.start_dtls(false).unwrap(); + } + let mut config = config(); + config.client_udp_port = client_addr.port(); + config.video_peer = server_addr; + config.remote_dtls_fingerprint = Some(server_identity.dtls_fingerprint); + config.stun_credentials = Some(credentials); + let feedback = config.feedback(); + assert!(!feedback.authenticated_transport_ready()); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, event_receiver) = mpsc::channel(); + let (play_sender, play_receiver) = mpsc::sync_channel(1); + let session = spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + Some(client), + Some(client_rtc), + Arc::new(HidRuntime::new()), + Some(play_receiver), + ) + .unwrap(); + + let mut version_sent = false; + let mut app_messages = Vec::new(); + let mut input_ready = false; + let deadline = Instant::now() + Duration::from_secs(5); + let mut packet = [0_u8; 65_536]; + while Instant::now() < deadline { + let now = Instant::now(); + if let Ok((length, source)) = server.recv_from(&mut packet) { + assert_eq!(source, client_addr); + if !looks_like_stun(&packet[..length]) + || find_stun_attribute(&packet[..length], STUN_ATTR_PRIORITY).is_some() + { + server_rtc + .handle_input(Input::Receive( + now, + Receive { + proto: RtcProtocol::Udp, + source: logical_ice_addr(client_addr, 1), + destination: logical_ice_addr(server_addr, 2), + contents: packet[..length].try_into().unwrap(), + }, + )) + .unwrap(); + } + } + server_rtc.handle_input(Input::Timeout(now)).unwrap(); + loop { + match server_rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(transmit) => { + server.send_to(&transmit.contents, client_addr).unwrap(); + } + Output::Event(Event::Connected) => server_rtc.direct_api().start_sctp(false), + Output::Event(Event::ChannelOpen(id, label)) + if label == "control_channel_reliable" => + { + version_sent = server_rtc + .channel(id) + .unwrap() + .write(true, &[0x0e, 0x02, 0x02, 0x00]) + .unwrap(); + } + Output::Event(Event::ChannelData(data)) => app_messages.push(data.data), + _ => {} + } + } + while let Ok(event) = event_receiver.try_recv() { + if matches!(event, NvstReceiveEvent::InputReady(2)) { + input_ready = true; + } + } + if version_sent { + break; + } + } + assert!(version_sent, "SCTP version exchange must precede PLAY"); + assert!( + feedback.authenticated_transport_ready(), + "DTLS readiness must precede PLAY" + ); + assert!( + feedback.video_timeout_origin(Instant::now()).is_none(), + "DTLS readiness must not start the video watchdog before PLAY" + ); + let until = Instant::now() + Duration::from_millis(300); + while Instant::now() < until { + let now = Instant::now(); + if let Ok((length, _)) = server.recv_from(&mut packet) { + if !looks_like_stun(&packet[..length]) && !looks_like_dtls(&packet[..length]) { + server_rtc + .handle_input(Input::Receive( + now, + Receive { + proto: RtcProtocol::Udp, + source: logical_ice_addr(client_addr, 1), + destination: logical_ice_addr(server_addr, 2), + contents: packet[..length].try_into().unwrap(), + }, + )) + .unwrap(); + } + } + server_rtc.handle_input(Input::Timeout(now)).unwrap(); + loop { + match server_rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(transmit) => { + server.send_to(&transmit.contents, client_addr).unwrap(); + } + Output::Event(Event::ChannelData(data)) => app_messages.push(data.data), + _ => {} + } + } + while let Ok(event) = event_receiver.try_recv() { + assert!(!matches!(event, NvstReceiveEvent::InputReady(_))); + } + } + assert!(app_messages.is_empty(), "application data preceded PLAY"); + assert!(!input_ready, "input-ready event preceded PLAY"); + assert!(!session.input_ready.load(Ordering::Acquire)); + if accept_play { + play_sender.try_send(()).unwrap(); + let until = Instant::now() + Duration::from_secs(2); + while Instant::now() < until && (!input_ready || app_messages.is_empty()) { + let now = Instant::now(); + if let Ok((length, _)) = server.recv_from(&mut packet) + && !looks_like_stun(&packet[..length]) + { + server_rtc + .handle_input(Input::Receive( + now, + Receive { + proto: RtcProtocol::Udp, + source: logical_ice_addr(client_addr, 1), + destination: logical_ice_addr(server_addr, 2), + contents: packet[..length].try_into().unwrap(), + }, + )) + .unwrap(); + } + server_rtc.handle_input(Input::Timeout(now)).unwrap(); + loop { + match server_rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(transmit) => { + server.send_to(&transmit.contents, client_addr).unwrap(); + } + Output::Event(Event::ChannelData(data)) => app_messages.push(data.data), + _ => {} + } + } + while let Ok(event) = event_receiver.try_recv() { + input_ready |= matches!(event, NvstReceiveEvent::InputReady(2)); + } + } + assert!(input_ready, "accepted PLAY must publish input readiness"); + assert!( + feedback.video_timeout_origin(Instant::now()).is_some(), + "accepted PLAY must start the shared video watchdog clock" + ); + assert!( + !app_messages.is_empty(), + "accepted PLAY must send activation" + ); + } else { + drop(play_sender); + let until = Instant::now() + Duration::from_secs(1); + let mut stopped = false; + while Instant::now() < until { + if let Ok(event) = event_receiver.recv_timeout(Duration::from_millis(50)) { + stopped |= matches!( + event, + NvstReceiveEvent::Lifecycle(NvstReceiverState::Stopped( + NvstStopCause::UpstreamGateClosed + )) + ); + assert!(!matches!(event, NvstReceiveEvent::InputReady(_))); + if stopped { + break; + } + } + } + assert!(stopped, "failed PLAY must stop the bundle worker"); + assert!(!session.input_ready.load(Ordering::Acquire)); + assert!(app_messages.is_empty()); + } + session.stop(); + } + + #[test] + fn delayed_play_gates_sctp_application_messages_and_input_activation() { + play_gate_loopback(true); + } + + #[test] + fn failed_play_closes_the_gate_without_input_activation() { + play_gate_loopback(false); + } + + #[test] + fn udp_receiver_repeats_the_negotiated_ping_before_media_arrives() { + let server = UdpSocket::bind("127.0.0.1:0").expect("server socket"); + server + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("server timeout"); + let client_reservation = UdpSocket::bind("127.0.0.1:0").expect("client reservation"); + let client_port = client_reservation + .local_addr() + .expect("client address") + .port(); + let mut config = config(); + config.client_udp_port = client_port; + config.video_peer = server.local_addr().expect("server address"); + config.ping_payload = b"negotiated-ping".to_vec(); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + // Keep the reservation alive: parallel socket tests can otherwise claim + // this ephemeral port between the probe and the receiver's bind. + let session = spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + Some(client_reservation), + None, + Arc::new(HidRuntime::new()), + None, + ) + .expect("UDP receiver"); + + let mut datagram = [0_u8; 64]; + let (first_len, _) = server.recv_from(&mut datagram).expect("first ping"); + assert_eq!(&datagram[..first_len], b"negotiated-ping"); + let (second_len, _) = server.recv_from(&mut datagram).expect("repeated ping"); + assert_eq!(&datagram[..second_len], b"negotiated-ping"); + + session.stop(); + } + + #[test] + fn mjolnir_receiver_sends_only_version_six_stun_with_the_setup_identity() { + let server = UdpSocket::bind("127.0.0.1:0").expect("server socket"); + server + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("server timeout"); + let client = UdpSocket::bind("127.0.0.1:0").expect("client socket"); + + let mut config = config(); + config.client_udp_port = client.local_addr().expect("client address").port(); + config.video_peer = server.local_addr().expect("server address"); + config.ping_payload = b"setup-ping".to_vec(); + config.stun_credentials = Some(stun_credentials()); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = spawn_nvst_mjolnir_receiver(client, config, media_consumer, event_sender) + .expect("Mjolnir receiver"); + + let mut datagram = [0_u8; 512]; + let (setup_len, _) = server + .recv_from(&mut datagram) + .expect("SETUP identity probe"); + let (_, setup_username) = + find_stun_attribute(&datagram[..setup_len], STUN_ATTR_USERNAME).expect("USERNAME"); + assert_eq!(setup_username, b"setup-ping:loc1"); + + let (repeated_len, _) = server + .recv_from(&mut datagram) + .expect("repeated SETUP identity probe"); + let (_, repeated_username) = + find_stun_attribute(&datagram[..repeated_len], STUN_ATTR_USERNAME).expect("USERNAME"); + assert_eq!(repeated_username, b"setup-ping:loc1"); + + session.stop(); + } + + #[test] + fn video_natt_targets_first_port_while_media_accepts_the_range() { + for authenticated in [false, true] { + let (first, second) = (0..64) + .find_map(|_| { + let first = UdpSocket::bind("127.0.0.1:0").ok()?; + let next = first.local_addr().ok()?.port().checked_add(1)?; + let second = UdpSocket::bind(("127.0.0.1", next)).ok()?; + Some((first, second)) + }) + .expect("consecutive UDP ports"); + first + .set_read_timeout(Some(Duration::from_millis(250))) + .unwrap(); + second + .set_read_timeout(Some(Duration::from_millis(100))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.client_udp_port = client.local_addr().unwrap().port(); + config.video_peer = first.local_addr().unwrap(); + config.video_peer_port_end = Some(second.local_addr().unwrap().port()); + config.ping_payload = b"setup-ping".to_vec(); + config.stun_credentials = authenticated.then(stun_credentials); + assert!(config.accepts_video_source(second.local_addr().unwrap())); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = + spawn_nvst_mjolnir_receiver(client, config, media_consumer, event_sender).unwrap(); + + let mut datagram = [0_u8; 512]; + let (length, _) = first.recv_from(&mut datagram).expect("first port NATT"); + if authenticated { + let (_, username) = find_stun_attribute(&datagram[..length], STUN_ATTR_USERNAME) + .expect("NATT identity"); + assert_eq!(username, b"setup-ping:loc1"); + } else { + assert_eq!(&datagram[..length], b"setup-ping"); + } + assert!( + second.recv_from(&mut datagram).is_err(), + "no NATT to later ports" + ); + session.stop(); + } + } + + #[test] + fn h264_frame_queue_is_bounded_and_prefers_current_frames() { + let mut queue = BoundedFrameQueue::new(2); + for frame_index in 1..=3 { + queue.push(EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: frame_index, + frame_index, + first_stream_packet_index: frame_index, + frame_type: None, + keyframe: false, + contiguous: true, + bytes: vec![frame_index as u8], + }); + } + assert_eq!(queue.dropped_frames(), 1); + assert_eq!(queue.pop().expect("frame").frame_index, 2); + assert_eq!(queue.pop().expect("frame").frame_index, 3); + } + + fn hex_bytes(value: &str) -> Vec { + value + .as_bytes() + .chunks(2) + .map(|chunk| { + u8::from_str_radix(std::str::from_utf8(chunk).expect("hex"), 16).expect("hex") + }) + .collect() + } +} + +/// In-process bounded queue for callers that drive `NvstVideoReceiver` directly rather than +/// using the UDP worker. It drops the oldest frame to keep interactive latency bounded. +#[derive(Debug)] +pub struct BoundedFrameQueue { + frames: VecDeque, + capacity: usize, + dropped_frames: u64, +} + +impl BoundedFrameQueue { + pub fn new(capacity: usize) -> Self { + Self { + frames: VecDeque::with_capacity(capacity), + capacity: capacity.max(1), + dropped_frames: 0, + } + } + + pub fn push(&mut self, frame: EncodedVideoAccessUnit) { + if self.frames.len() == self.capacity { + let _ = self.frames.pop_front(); + self.dropped_frames += 1; + } + self.frames.push_back(frame); + } + + pub fn pop(&mut self) -> Option { + self.frames.pop_front() + } + + pub fn dropped_frames(&self) -> u64 { + self.dropped_frames + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_jitter.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_jitter.rs new file mode 100644 index 000000000..d2fd31bc1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_jitter.rs @@ -0,0 +1,473 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use str0m::rtp::RtpHeader; + +const INITIAL_TARGET_MS: u64 = 45; +const MAX_TARGET_MS: u64 = 200; +const UNDERFLOW_INCREASE_MS: u64 = 20; +const MAX_PACKETS: usize = 80; +const MAX_PAYLOAD_BYTES: usize = 9 * 1275 + 33; + +pub(crate) struct AudioPacket { + pub header: RtpHeader, + pub payload: Arc<[u8]>, + pub received_at_us: u64, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct AudioJitterStats { + pub reordered: u64, + pub stale: u64, + pub overflow: u64, + pub invalid: u64, + pub source_resets: u64, + pub underflows: u64, + pub playout_resyncs: u64, +} + +pub(crate) struct AudioJitterBuffer { + clock_rate_hz: u32, + anchor: Option<(u32, u32, Instant)>, + packets: VecDeque, + last_played: Option<(u32, u16)>, + target_ms: u64, + late_burst: bool, + pub stats: AudioJitterStats, +} + +impl AudioJitterBuffer { + pub fn new(clock_rate_hz: u32) -> Self { + assert!(clock_rate_hz > 0); + Self { + clock_rate_hz, + anchor: None, + packets: VecDeque::with_capacity(MAX_PACKETS), + last_played: None, + target_ms: INITIAL_TARGET_MS, + late_burst: false, + stats: AudioJitterStats::default(), + } + } + + pub fn target_ms(&self) -> u64 { + self.target_ms + } + + pub fn len(&self) -> usize { + self.packets.len() + } + + fn deadline(&self, timestamp: u32) -> Instant { + let (_, base_timestamp, base_time) = self.anchor.expect("packet has an anchor"); + let delta = timestamp.wrapping_sub(base_timestamp) as i32; + let duration = Duration::from_micros( + u64::from(delta.unsigned_abs()) * 1_000_000 / u64::from(self.clock_rate_hz), + ); + let base_time = base_time + Duration::from_millis(self.target_ms); + if delta < 0 { + base_time.checked_sub(duration).unwrap_or(base_time) + } else { + base_time + duration + } + } + + pub fn push(&mut self, packet: AudioPacket, now: Instant) { + if packet.payload.is_empty() || packet.payload.len() > MAX_PAYLOAD_BYTES { + self.stats.invalid += 1; + return; + } + let ssrc = *packet.header.ssrc; + let timestamp = packet.header.timestamp; + if self.anchor.is_none_or(|(source, _, _)| source != ssrc) { + if self.anchor.is_some() { + self.stats.source_resets += 1; + } + self.packets.clear(); + self.last_played = None; + self.anchor = Some((ssrc, timestamp, now)); + self.target_ms = INITIAL_TARGET_MS; + self.late_burst = false; + } + if self + .last_played + .is_some_and(|(played_timestamp, sequence)| { + let delta = timestamp.wrapping_sub(played_timestamp) as i32; + delta < 0 + || packet.header.sequence_number.wrapping_sub(sequence) >= 0x8000 + || packet.header.sequence_number == sequence + }) + || self + .packets + .iter() + .any(|queued| queued.header.sequence_number == packet.header.sequence_number) + { + self.stats.stale += 1; + return; + } + let late = self.last_played.is_some() && now > self.deadline(timestamp); + if late && !self.late_burst { + self.stats.underflows += 1; + self.target_ms = (self.target_ms + UNDERFLOW_INCREASE_MS).min(MAX_TARGET_MS); + } + self.late_burst = late; + if self.deadline(timestamp).saturating_duration_since(now) + > Duration::from_millis(self.target_ms + MAX_TARGET_MS) + { + self.stats.overflow += self.packets.len() as u64; + self.packets.clear(); + self.anchor = Some((ssrc, timestamp, now)); + } + let index = self + .packets + .iter() + .position(|queued| { + let delta = timestamp.wrapping_sub(queued.header.timestamp) as i32; + delta < 0 + || (delta == 0 + && (packet + .header + .sequence_number + .wrapping_sub(queued.header.sequence_number) + as i16) + < 0) + }) + .unwrap_or(self.packets.len()); + if index < self.packets.len() { + self.stats.reordered += 1; + } + self.packets.insert(index, packet); + let queued = self.packets.len(); + while self.packets.len() > MAX_PACKETS + || self + .packets + .front() + .zip(self.packets.back()) + .is_some_and(|(first, last)| { + u64::from(last.header.timestamp.wrapping_sub(first.header.timestamp)) * 1_000 + > u64::from(self.clock_rate_hz) * MAX_TARGET_MS + }) + { + self.packets.pop_front(); + self.stats.overflow += 1; + } + if self.packets.len() < queued { + let first = self.packets.front().expect("bounded queue retains audio"); + if now < self.deadline(first.header.timestamp) { + let next = self.packets.get(1).unwrap_or(first); + self.anchor = Some(( + ssrc, + next.header.timestamp, + now - Duration::from_millis(self.target_ms), + )); + self.stats.playout_resyncs += 1; + } + } + } + + pub fn pop(&mut self, now: Instant) -> Option { + let packet = self.packets.front()?; + if now < self.deadline(packet.header.timestamp) { + return None; + } + let packet = self.packets.pop_front()?; + self.last_played = Some((packet.header.timestamp, packet.header.sequence_number)); + Some(packet) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn packet(sequence: u16, timestamp: u32) -> AudioPacket { + AudioPacket { + header: RtpHeader { + sequence_number: sequence, + timestamp, + ssrc: 42.into(), + ..RtpHeader::default() + }, + payload: Arc::from([sequence as u8]), + received_at_us: 1234, + } + } + + #[test] + fn timestamp_ordering_preserves_sequence_payload_and_arrival_metadata() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(10, 2400), origin); + buffer.push(packet(12, 2880), origin); + buffer.push(packet(11, 2640), origin + Duration::from_millis(20)); + assert!(buffer.pop(origin + Duration::from_millis(44)).is_none()); + for (offset, sequence) in [(45, 10), (50, 11), (55, 12)] { + let frame = buffer.pop(origin + Duration::from_millis(offset)).unwrap(); + assert_eq!(frame.header.sequence_number, sequence); + assert_eq!(frame.header.timestamp, u32::from(sequence) * 240); + assert_eq!(frame.received_at_us, 1234); + assert_eq!(&*frame.payload, &[sequence as u8]); + } + assert_eq!(buffer.stats.reordered, 1); + } + + #[test] + fn wraps_duplicates_and_source_change_remain_bounded() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(65535, u32::MAX - 239), origin); + buffer.push(packet(1, 240), origin); + buffer.push(packet(0, 0), origin); + buffer.push(packet(0, 0), origin); + for sequence in [65535, 0, 1] { + assert_eq!( + buffer + .pop(origin + Duration::from_millis(55)) + .unwrap() + .header + .sequence_number, + sequence + ); + } + buffer.push(packet(0, 0), origin); + assert_eq!(buffer.stats.stale, 2); + let mut replacement = packet(1, 0); + replacement.header.ssrc = 43.into(); + buffer.push(replacement, origin); + assert_eq!(buffer.stats.source_resets, 1); + assert!(buffer.pop(origin + Duration::from_millis(44)).is_none()); + assert_eq!( + *buffer + .pop(origin + Duration::from_millis(45)) + .unwrap() + .header + .ssrc, + 43 + ); + } + + #[test] + fn late_bursts_raise_target_in_twenty_millisecond_steps_up_to_ceiling() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(0, 0), origin); + buffer.pop(origin + Duration::from_millis(45)).unwrap(); + for step in 1..=20 { + let sequence = step * 2 - 1; + let timestamp = u32::from(step) * 48_000; + let late = + origin + Duration::from_millis(u64::from(step) * 1_000 + buffer.target_ms() + 1); + buffer.push(packet(sequence, timestamp), late); + assert_eq!(buffer.target_ms(), (45 + u64::from(step) * 20).min(200)); + buffer.pop(late + Duration::from_millis(20)).unwrap(); + let next = packet(sequence + 1, timestamp + 24_000); + let timely = origin + Duration::from_millis(u64::from(step) * 1_000 + 500); + buffer.push(next, timely); + buffer + .pop(timely + Duration::from_millis(buffer.target_ms())) + .unwrap(); + } + assert_eq!(buffer.target_ms(), 200); + assert_eq!(buffer.stats.underflows, 20); + } + + #[test] + fn clean_network_keeps_the_forty_five_millisecond_floor() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + let mut delivered = 0; + for tick in 0..=1050_u64 { + let now = origin + Duration::from_millis(tick); + if tick <= 1000 && tick % 5 == 0 { + buffer.push(packet((tick / 5) as u16, tick as u32 * 48), now); + } + while let Some(packet) = buffer.pop(now) { + assert_eq!(tick, u64::from(packet.header.timestamp) / 48 + 45); + delivered += 1; + } + assert_eq!(buffer.target_ms(), 45); + } + assert_eq!(delivered, 201); + assert_eq!(buffer.stats.underflows, 0); + assert_eq!(buffer.len(), 0); + } + + #[test] + fn full_timestamp_span_bursts_preserve_the_anchor_at_floor_and_ceiling() { + let origin = Instant::now(); + for target_ms in [45, 200] { + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(0, 0), origin); + buffer.target_ms = target_ms; + for sequence in (1..=40).rev() { + buffer.push(packet(sequence, u32::from(sequence) * 240), origin); + } + assert_eq!(buffer.anchor, Some((42, 0, origin))); + assert_eq!(buffer.len(), 41); + assert_eq!(buffer.stats.overflow, 0); + assert_eq!(buffer.stats.underflows, 0); + for sequence in 0..=40 { + let deadline = origin + Duration::from_millis(target_ms + u64::from(sequence) * 5); + assert!(buffer.pop(deadline - Duration::from_micros(1)).is_none()); + let frame = buffer.pop(deadline).unwrap(); + assert_eq!(frame.header.sequence_number, sequence); + assert_eq!(frame.header.timestamp, u32::from(sequence) * 240); + } + } + } + + #[test] + fn ceiling_target_preserves_queued_audio_on_a_one_millisecond_early_arrival() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(0, 0), origin); + buffer.target_ms = 200; + for tick in 1..=998_u64 { + let now = origin + Duration::from_millis(tick); + if tick % 5 == 0 { + buffer.push(packet((tick / 5) as u16, tick as u32 * 48), now); + } + while buffer.pop(now).is_some() {} + } + assert_eq!(buffer.len(), 40); + buffer.push(packet(200, 48_000), origin + Duration::from_millis(999)); + assert_eq!(buffer.anchor, Some((42, 0, origin))); + assert_eq!(buffer.len(), 41); + assert_eq!(buffer.stats.overflow, 0); + assert_eq!(buffer.stats.underflows, 0); + assert_eq!( + buffer + .pop(origin + Duration::from_millis(1000)) + .unwrap() + .header + .sequence_number, + 160 + ); + } + + #[test] + fn media_time_jumps_beyond_the_span_limit_still_reanchor() { + let origin = Instant::now(); + for target_ms in [45, 200] { + let mut buffer = AudioJitterBuffer::new(48_000); + buffer.push(packet(0, 0), origin); + buffer.target_ms = target_ms; + buffer.push(packet(1, 2400), origin); + buffer.push(packet(2, 201 * 48), origin); + assert_eq!(buffer.anchor, Some((42, 201 * 48, origin))); + assert_eq!(buffer.len(), 1); + assert_eq!(buffer.stats.overflow, 2); + let deadline = origin + Duration::from_millis(target_ms); + assert!(buffer.pop(deadline - Duration::from_micros(1)).is_none()); + let frame = buffer.pop(deadline).unwrap(); + assert_eq!(frame.header.sequence_number, 2); + assert_eq!(frame.header.timestamp, 201 * 48); + } + } + + #[test] + fn clock_drift_keeps_playout_moving_across_targets_packet_sizes_and_timestamp_wrap() { + let origin = Instant::now(); + let start_timestamp = u32::MAX - 47_999; + for target_ms in [45, 165, 200] { + for packet_us in [2_500_u64, 5_000] { + for clock_ratio in [9_999_u64, 10_001] { + let mut buffer = AudioJitterBuffer::new(48_000); + let mut last_delivery = None; + let mut delivered = 0; + for index in 0..600_000_000 / packet_us { + let media_us = index * packet_us; + let now = origin + Duration::from_micros(media_us * clock_ratio / 10_000); + let timestamp = + start_timestamp.wrapping_add((media_us * 48_000 / 1_000_000) as u32); + let mut frame = packet(index as u16, timestamp); + frame.received_at_us = media_us; + buffer.push(frame, now); + if index == 0 { + buffer.target_ms = target_ms; + } + while let Some(frame) = buffer.pop(now) { + let media_us = frame.received_at_us; + assert_eq!(frame.header.sequence_number, (media_us / packet_us) as u16); + assert_eq!( + frame.header.timestamp, + start_timestamp + .wrapping_add((media_us * 48_000 / 1_000_000) as u32) + ); + assert_eq!(&*frame.payload, &[frame.header.sequence_number as u8]); + last_delivery = Some(now); + delivered += 1; + } + if let Some(last) = last_delivery { + assert!( + now.duration_since(last) + <= Duration::from_millis(MAX_TARGET_MS + 5), + "playout stalled: target={target_ms} packet_us={packet_us} clock_ratio={clock_ratio}" + ); + } + assert!(buffer.len() <= MAX_PACKETS); + } + assert!( + delivered > 100_000, + "target={target_ms} packet_us={packet_us} clock_ratio={clock_ratio} delivered={delivered} stats={:?}", + buffer.stats + ); + assert_eq!(buffer.stats.stale, 0); + assert!(buffer.stats.overflow < 100); + if clock_ratio < 10_000 && target_ms >= 165 { + assert!(buffer.stats.playout_resyncs > 0); + } + } + } + } + } + + #[test] + fn raising_target_with_early_arrivals_does_not_starve_playout() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + let mut last_delivery = None; + let mut delivered = 0; + for tick in 0..=10_000_u64 { + let now = origin + Duration::from_millis(tick); + if tick % 5 == 0 { + let timestamp = if tick == 0 { 0 } else { (tick + 50) * 48 }; + buffer.push(packet((tick / 5) as u16, timestamp as u32), now); + if tick == 0 { + buffer.target_ms = 145; + } + } + if tick == 5_000 { + buffer.target_ms += UNDERFLOW_INCREASE_MS; + } + while buffer.pop(now).is_some() { + last_delivery = Some(now); + delivered += 1; + } + if let Some(last) = last_delivery { + assert!(now.duration_since(last) <= Duration::from_millis(MAX_TARGET_MS + 5)); + } + } + assert!(delivered > 1_900); + assert_eq!(buffer.stats.underflows, 0); + assert_eq!(buffer.stats.playout_resyncs, 1); + } + + #[test] + fn packet_count_timestamp_span_and_payload_size_are_bounded() { + let origin = Instant::now(); + let mut buffer = AudioJitterBuffer::new(48_000); + for sequence in 0..200 { + buffer.push(packet(sequence, u32::from(sequence) * 120), origin); + assert!(buffer.len() <= MAX_PACKETS); + } + assert!(buffer.stats.overflow > 0); + buffer.push(packet(200, 48_000), origin); + assert_eq!(buffer.len(), 1); + let mut invalid = packet(201, 48_120); + invalid.payload = Arc::from(vec![0; MAX_PAYLOAD_BYTES + 1]); + buffer.push(invalid, origin); + assert_eq!(buffer.stats.invalid, 1); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_tests.rs new file mode 100644 index 000000000..bef540d51 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_audio_tests.rs @@ -0,0 +1,575 @@ +use super::super::*; + +fn track() -> NvstAudioTrack { + NvstAudioTrack { + payload_type: GFN_OPUS_PAYLOAD_TYPE, + clock_rate_hz: 48_000, + channels: 2, + mid: "audio".to_owned(), + ssrc: None, + } +} + +fn header(sequence_number: u16) -> BundleRtpHeader { + BundleRtpHeader { + payload_type: GFN_OPUS_PAYLOAD_TYPE.into(), + sequence_number, + timestamp: u32::from(sequence_number) * 240, + ssrc: 42.into(), + ..BundleRtpHeader::default() + } +} + +#[test] +fn microphone_handoff_defaults_and_negotiated_red_are_validated() { + let mut handoff = super::legacy_handoff(); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.microphone_rtp.opus_payload_type, 111); + assert_eq!(config.microphone_rtp.red_payload_type, None); + assert_eq!(config.microphone_rtp.redundancy_level, 0); + handoff["microphoneOpusPayloadType"] = serde_json::json!(109); + handoff["microphoneRedPayloadType"] = serde_json::json!(112); + handoff["microphoneRedundancyLevel"] = serde_json::json!(3); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).unwrap(); + assert_eq!(config.microphone_rtp.opus_payload_type, 109); + assert_eq!(config.microphone_rtp.red_payload_type, Some(112)); + assert_eq!(config.microphone_rtp.redundancy_level, 3); + for (field, invalid) in [ + ("microphoneOpusPayloadType", serde_json::json!(128)), + ("microphoneRedPayloadType", serde_json::json!(80)), + ("microphoneRedPayloadType", serde_json::json!(109)), + ("microphoneRedundancyLevel", serde_json::json!(4)), + ("microphoneRedundancyLevel", serde_json::json!("3")), + ] { + let mut bad = handoff.clone(); + bad[field] = invalid; + assert!(NvstVideoConfig::from_legacy_handoff(&bad, None).is_err(), "{field}"); + } +} + +#[test] +fn microphone_payload_registration_replaces_default_pt_collisions_once() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut rtc = create_nvst_bundle_rtc(&socket).unwrap(); + for _ in 0..2 { + rtc.direct_api().register_payload_type(96.into(), Codec::Opus, Frequency::FORTY_EIGHT_KHZ, Some(1), FormatParams::default()); + } + let registered = rtc.codec_config().iter().filter(|params| params.pt() == 96.into()).collect::>(); + assert_eq!(registered.len(), 1); + assert_eq!(registered[0].spec().codec, Codec::Opus); + assert_eq!(registered[0].spec().clock_rate, Frequency::FORTY_EIGHT_KHZ); + assert!(!rtc.codec_config().iter().any(|params| params.resend() == Some(96.into()))); +} + +#[test] +fn registered_microphone_payload_type_survives_direct_rtp_poll() { + fn exchange(from: &mut Rtc, to: &mut Rtc, now: Instant, received: &mut Vec) { + loop { + match from.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) => { + to.handle_input(Input::Receive(now, Receive { + proto: packet.proto, + source: packet.source, + destination: packet.destination, + contents: packet.contents.as_ref().try_into().unwrap(), + })).unwrap(); + } + Output::Event(Event::RtpPacket(packet)) => received.push(packet), + _ => {} + } + } + } + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut local = create_nvst_bundle_rtc(&socket).unwrap(); + let mut remote = create_nvst_bundle_rtc(&socket).unwrap(); + let local_candidate = Candidate::host("192.0.2.1:1000".parse().unwrap(), "udp").unwrap(); + let remote_candidate = Candidate::host("192.0.2.2:2000".parse().unwrap(), "udp").unwrap(); + local.add_local_candidate(local_candidate.clone()); + local.add_remote_candidate(remote_candidate.clone()); + remote.add_local_candidate(remote_candidate); + remote.add_remote_candidate(local_candidate); + let local_identity = local.direct_api().local_dtls_fingerprint().clone(); + let remote_identity = remote.direct_api().local_dtls_fingerprint().clone(); + let local_ice = local.direct_api().local_ice_credentials(); + let remote_ice = remote.direct_api().local_ice_credentials(); + local.direct_api().set_remote_fingerprint(remote_identity); + remote.direct_api().set_remote_fingerprint(local_identity); + local.direct_api().set_remote_ice_credentials(remote_ice); + remote.direct_api().set_remote_ice_credentials(local_ice); + local.direct_api().set_ice_controlling(true); + remote.direct_api().set_ice_controlling(false); + for rtc in [&mut local, &mut remote] { + rtc.direct_api().register_payload_type(112.into(), Codec::Opus, Frequency::FORTY_EIGHT_KHZ, Some(1), FormatParams::default()); + rtc.direct_api().declare_media(Mid::from("2"), MediaKind::Audio); + } + local.direct_api().start_dtls(true).unwrap(); + remote.direct_api().start_dtls(false).unwrap(); + remote.direct_api().expect_stream_rx(1.into(), None, Mid::from("2"), None); + let origin = Instant::now(); + let mut received = Vec::new(); + for tick in 0..2000 { + let now = origin + Duration::from_millis(tick); + local.handle_input(Input::Timeout(now)).unwrap(); + remote.handle_input(Input::Timeout(now)).unwrap(); + if tick == 1000 || tick == 1500 { + assert!(local.is_connected() && remote.is_connected()); + let pt = if tick == 1000 { 112 } else { 113 }; + local.direct_api().declare_stream_tx(1.into(), None, Mid::from("2"), None) + .write_rtp(str0m::rtp::RtpWrite::new(pt.into(), tick.into(), tick as u32 * 48, now, vec![111, 0xf8, 0xff, 0xfe])); + } + exchange(&mut local, &mut remote, now, &mut Vec::new()); + exchange(&mut remote, &mut local, now, &mut received); + } + assert_eq!(received.len(), 1); + assert_eq!(*received[0].header.payload_type, 112); + assert_eq!(received[0].header.timestamp, 48_000); + assert_eq!(&*received[0].payload, &[111, 0xf8, 0xff, 0xfe]); +} + +#[test] +fn jitter_reorders_before_red_loss_recovery_without_replaying_originals() { + let origin = Instant::now(); + let mut jitter = AudioJitterBuffer::new(48_000); + let mut receiver = NvstAudioReceiver::default(); + for sequence in [10, 12, 11] { + jitter.push(AudioPacket { header: header(sequence), payload: Arc::from([sequence as u8]), received_at_us: 1234 }, origin); + } + let mut frames = Vec::new(); + while let Some(packet) = jitter.pop(origin + Duration::from_millis(100)) { + frames.extend(receiver.depacketize(&track(), &packet.header, packet.payload, packet.received_at_us).unwrap()); + } + assert_eq!(frames.iter().map(|frame| frame.rtp_timestamp).collect::>(), [2400, 2640, 2880]); + assert!(frames.iter().all(|frame| frame.contiguous)); + jitter.push(AudioPacket { header: header(11), payload: Arc::from([11]), received_at_us: 1234 }, origin); + assert!(jitter.pop(origin + Duration::from_millis(100)).is_none()); +} + +#[test] +fn rejected_audio_ssrcs_do_not_fill_admission_or_evict_authenticated_streams() { + install_crypto(); + let mut rtc = RtcConfig::new().set_rtp_mode(true).build(Instant::now()); + let track = track(); + rtc.direct_api() + .declare_media(Mid::from(track.mid.as_str()), MediaKind::Audio); + let mut streams = NvstAudioStreams::default(); + let mut datagram = [0x80, GFN_OPUS_PAYLOAD_TYPE, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + for ssrc in 1_u32..=10_000 { + datagram[8..12].copy_from_slice(&ssrc.to_be_bytes()); + assert!(streams.admit(&mut rtc, &track, &datagram)); + streams.finish_poll(&mut rtc); + assert!(rtc.direct_api().stream_rx(&ssrc.into()).is_none()); + } + assert!(streams.authenticated.is_empty()); + let mut pinned = track.clone(); + pinned.ssrc = Some(42); + assert!(!streams.admit(&mut rtc, &pinned, &datagram)); + datagram[8..12].copy_from_slice(&42_u32.to_be_bytes()); + datagram[1] = 96; + assert!(!streams.admit(&mut rtc, &pinned, &datagram)); + datagram[1] = GFN_RED_PAYLOAD_TYPE; + assert!(streams.admit(&mut rtc, &pinned, &datagram)); + streams.finish_poll(&mut rtc); +} + +#[test] +fn audio_ssrc_admission_recovers_after_invalid_tags_and_authenticated_source_changes() { + fn exchange( + from: &mut Rtc, + to: &mut Rtc, + now: Instant, + inbound: bool, + forged_ssrc: Option, + streams: &mut NvstAudioStreams, + received: &mut Vec, + ) { + loop { + match from.poll_output().unwrap() { + Output::Timeout(_) => { + if !inbound { + streams.finish_poll(from); + } + break; + } + Output::Transmit(packet) => { + let mut bytes = packet.contents.to_vec(); + if inbound && looks_like_rtp(&bytes) { + if let Some(ssrc) = forged_ssrc { + bytes[8..12].copy_from_slice(&ssrc.to_be_bytes()); + } + assert!(streams.admit(to, &track(), &bytes)); + } + to.handle_input(Input::Receive( + now, + Receive { + proto: packet.proto, + source: packet.source, + destination: packet.destination, + contents: bytes.as_slice().try_into().unwrap(), + }, + )) + .unwrap(); + } + Output::Event(Event::RtpPacket(packet)) if !inbound => { + let ssrc = *packet.header.ssrc; + streams.authenticated(from, ssrc); + received.push(ssrc); + } + _ => {} + } + } + } + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut local = create_nvst_bundle_rtc(&socket).unwrap(); + let mut remote = create_nvst_bundle_rtc(&socket).unwrap(); + let local_candidate = Candidate::host("192.0.2.1:1000".parse().unwrap(), "udp").unwrap(); + let remote_candidate = Candidate::host("192.0.2.2:2000".parse().unwrap(), "udp").unwrap(); + local.add_local_candidate(local_candidate.clone()); + local.add_remote_candidate(remote_candidate.clone()); + remote.add_local_candidate(remote_candidate); + remote.add_remote_candidate(local_candidate); + let local_identity = local.direct_api().local_dtls_fingerprint().clone(); + let remote_identity = remote.direct_api().local_dtls_fingerprint().clone(); + let local_ice = local.direct_api().local_ice_credentials(); + let remote_ice = remote.direct_api().local_ice_credentials(); + local.direct_api().set_remote_fingerprint(remote_identity); + remote.direct_api().set_remote_fingerprint(local_identity); + local.direct_api().set_remote_ice_credentials(remote_ice); + remote.direct_api().set_remote_ice_credentials(local_ice); + local.direct_api().set_ice_controlling(true); + remote.direct_api().set_ice_controlling(false); + local.direct_api().start_dtls(true).unwrap(); + remote.direct_api().start_dtls(false).unwrap(); + let mid = Mid::from("audio"); + for rtc in [&mut local, &mut remote] { + rtc.direct_api().declare_media(mid, MediaKind::Audio); + } + let mut streams = NvstAudioStreams::default(); + let mut received = Vec::new(); + let mut previous_sender = None; + let origin = Instant::now(); + for tick in 0..1800_u64 { + let now = origin + Duration::from_millis(tick); + local.handle_input(Input::Timeout(now)).unwrap(); + remote.handle_input(Input::Timeout(now)).unwrap(); + let send_ssrc = match tick { + 1200 | 1250..=1349 | 1400 | 1600 => Some(1), + 1450 => Some(2), + 1500 => Some(3), + _ => None, + }; + if let Some(ssrc) = send_ssrc { + if let Some(previous) = previous_sender.replace(ssrc) + && previous != ssrc + { + remote.direct_api().remove_stream_tx(previous.into()); + } + remote + .direct_api() + .declare_stream_tx(ssrc.into(), None, mid, None) + .write_rtp(str0m::rtp::RtpWrite::new( + GFN_OPUS_PAYLOAD_TYPE.into(), + tick.into(), + (tick * 48) as u32, + now, + vec![0xf8, 0xff, 0xfe], + )); + } + exchange( + &mut local, + &mut remote, + now, + false, + None, + &mut streams, + &mut received, + ); + let forged = (1250..=1349).contains(&tick).then_some(tick as u32); + exchange( + &mut remote, + &mut local, + now, + true, + forged, + &mut streams, + &mut received, + ); + if (1251..=1350).contains(&tick) { + assert!(local.direct_api().stream_rx(&1.into()).is_some()); + assert_eq!(streams.authenticated.len(), 1); + } + assert!(streams.authenticated.len() <= 2); + } + assert_eq!(received, [1, 1, 2, 3, 1]); + assert_eq!( + streams.authenticated.iter().copied().collect::>(), + [3, 1] + ); + assert!(streams.provisional.is_none()); + for ssrc in 1250..=1349 { + assert!(local.direct_api().stream_rx(&ssrc.into()).is_none()); + } +} + +fn red_packet( + receiver: &mut NvstAudioReceiver, + sequence: u16, + timestamp: u32, + redundant_blocks: usize, +) -> Vec { + let mut header = header(sequence); + header.payload_type = GFN_RED_PAYLOAD_TYPE.into(); + header.timestamp = timestamp; + let mut payload = Vec::new(); + for index in (1..=redundant_blocks).rev() { + let offset = index as u16 * 240; + payload.extend_from_slice(&[ + 0x80 | GFN_OPUS_PAYLOAD_TYPE, + (offset >> 6) as u8, + ((offset & 0x3f) << 2) as u8, + 1, + ]); + } + payload.push(GFN_OPUS_PAYLOAD_TYPE); + payload.extend((0..=redundant_blocks).map(|index| index as u8)); + receiver + .depacketize(&track(), &header, payload.into(), 123_456) + .unwrap() +} + +#[test] +fn ordered_audio_preserves_payload_and_sender_metadata() { + let mut receiver = NvstAudioReceiver::default(); + let payload: Arc<[u8]> = Arc::from([0xf8, 0xff, 0xfe]); + for sequence in 10..13 { + let frames = receiver + .depacketize(&track(), &header(sequence), Arc::clone(&payload), 123_456) + .unwrap(); + assert_eq!(frames.len(), 1); + let frame = &frames[0]; + assert!(Arc::ptr_eq(&frame.payload, &payload)); + assert_eq!(frame.mid, "audio"); + assert_eq!(frame.codec, "opus"); + assert_eq!(frame.frame_index, None); + assert_eq!(frame.rtp_timestamp, u64::from(sequence) * 240); + assert_eq!(frame.clock_rate_hz, 48_000); + assert_eq!(frame.channels, Some(2)); + assert_eq!(frame.received_at_us, 123_456); + assert!(frame.contiguous); + assert_eq!(frame.ssrc, Some(42)); + } +} + +#[test] +fn late_originals_and_duplicates_never_replay_recovered_audio_or_rewind_sequence() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, 2_400, 1); + let recovered = red_packet(&mut receiver, 12, 2_880, 1); + assert_eq!(recovered.len(), 2); + assert!(recovered.iter().all(|frame| frame.contiguous)); + assert!(recovered.iter().all(|frame| frame.ssrc == Some(42))); + for sequence in [11, 12, 10] { + assert!(red_packet(&mut receiver, sequence, u32::from(sequence) * 240, 1).is_empty()); + } + let next = red_packet(&mut receiver, 13, 3_120, 1); + assert_eq!(next.len(), 1); + assert!(next[0].contiguous); + assert_eq!(next[0].ssrc, Some(42)); +} + +#[test] +fn partial_red_recovery_marks_the_gap_before_the_first_recovered_packet() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, 2_400, 1); + let frames = red_packet(&mut receiver, 14, 3_360, 2); + assert_eq!(frames.len(), 3); + assert_eq!( + frames + .iter() + .map(|frame| frame.contiguous) + .collect::>(), + [false, true, true] + ); + assert_eq!( + frames + .iter() + .map(|frame| frame.rtp_timestamp) + .collect::>(), + [2_880, 3_120, 3_360] + ); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); + for (index, frame) in frames.iter().enumerate() { + assert_eq!(&*frame.payload, &[index as u8]); + } +} + +#[test] +fn gaps_beyond_red_capacity_remain_discontinuous_and_bounded() { + for redundant_blocks in [0, MAX_REDUNDANT_AUDIO_BLOCKS] { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, 2_400, 0); + let frames = red_packet(&mut receiver, 20_000, 4_800_000, redundant_blocks); + assert_eq!(frames.len(), redundant_blocks + 1); + assert!(!frames[0].contiguous); + assert!(frames[1..].iter().all(|frame| frame.contiguous)); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); + } +} + +#[test] +fn burst_losses_stay_measurable_from_the_preserved_timestamps() { + let mut receiver = NvstAudioReceiver::default(); + let mut sequence = 1_000_u16; + red_packet(&mut receiver, sequence, u32::from(sequence) * 240, 0); + for jump in [2_u16, 4, 6, 20] { + sequence = sequence.wrapping_add(jump); + let frames = receiver + .depacketize(&track(), &header(sequence), Arc::from([0xf8]), 123_456) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(!frames[0].contiguous); + assert_eq!(frames[0].ssrc, Some(42)); + assert_eq!(frames[0].rtp_timestamp, u64::from(sequence) * 240); + } + let frames = receiver + .depacketize(&track(), &header(sequence + 1), Arc::from([0xf8]), 123_456) + .unwrap(); + assert!(frames[0].contiguous); + assert_eq!(frames[0].ssrc, Some(42)); +} + +#[test] +fn redundantly_recovered_packets_are_never_flagged_for_concealment() { + let mut fully_recovered = NvstAudioReceiver::default(); + red_packet(&mut fully_recovered, 40, 9_600, 0); + let frames = red_packet(&mut fully_recovered, 44, 10_560, 3); + assert_eq!(frames.len(), 4); + assert!(frames.iter().all(|frame| frame.contiguous)); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); + + let mut partly_recovered = NvstAudioReceiver::default(); + red_packet(&mut partly_recovered, 40, 9_600, 0); + let frames = red_packet(&mut partly_recovered, 44, 10_560, 2); + assert_eq!(frames.len(), 3); + assert_eq!( + frames + .iter() + .map(|frame| (frame.contiguous, frame.ssrc)) + .collect::>(), + [(false, Some(42)), (true, Some(42)), (true, Some(42))] + ); + assert_eq!( + frames + .iter() + .map(|frame| frame.rtp_timestamp) + .collect::>(), + [10_080, 10_320, 10_560] + ); +} + +#[test] +fn plain_opus_gaps_are_marked_without_fabricating_packets() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, 2_400, 0); + let frames = receiver + .depacketize(&track(), &header(30), Arc::from([0xf8]), 0) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(!frames[0].contiguous); + assert_eq!(frames[0].ssrc, Some(42)); +} + +#[test] +fn sequence_and_timestamp_wrap_preserve_recovery_order() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, u16::MAX - 1, u32::MAX - 239, 0); + let frames = red_packet(&mut receiver, 1, 480, 2); + assert_eq!(frames.len(), 3); + assert!(frames.iter().all(|frame| frame.contiguous)); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); + assert_eq!( + frames + .iter() + .map(|frame| frame.rtp_timestamp) + .collect::>(), + [0, 240, 480] + ); + assert!(red_packet(&mut receiver, u16::MAX, 0, 0).is_empty()); + assert_eq!(red_packet(&mut receiver, 2, 720, 1).len(), 1); + + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, u32::MAX - 479, 0); + let frames = red_packet(&mut receiver, 12, 0, 1); + assert_eq!(frames[0].rtp_timestamp, u64::from(u32::MAX - 239)); + assert_eq!(frames[1].rtp_timestamp, 0); + assert!(frames.iter().all(|frame| frame.contiguous)); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); +} + +#[test] +fn malformed_red_does_not_prevent_recovery_from_the_next_packet() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 10, 2_400, 0); + let mut malformed_header = header(11); + malformed_header.payload_type = GFN_RED_PAYLOAD_TYPE.into(); + assert!(matches!( + receiver.depacketize(&track(), &malformed_header, Arc::from([0xff]), 0), + Err(NvstDropReason::MalformedRedAudio) + )); + let frames = red_packet(&mut receiver, 12, 2_880, 1); + assert_eq!(frames.len(), 2); + assert!(frames.iter().all(|frame| frame.contiguous)); + assert_eq!(frames[0].rtp_timestamp, 2_640); + assert!(frames.iter().all(|frame| frame.ssrc == Some(42))); +} + +#[test] +fn consumer_backpressure_marks_the_next_recovered_frame_not_the_primary() { + let mut receiver = NvstAudioReceiver::default(); + let (consumer, delivered) = mpsc::sync_channel(1); + for sequence in [10, 11] { + let frame = red_packet(&mut receiver, sequence, u32::from(sequence) * 240, 0) + .pop() + .unwrap(); + let result = receiver.deliver(&consumer, frame); + if sequence == 10 { + assert!(result.is_ok()); + } else { + assert!(matches!( + result, + Err(TransportError::MediaConsumerBackpressured) + )); + } + } + assert!(delivered.recv().unwrap().contiguous); + let frames = red_packet(&mut receiver, 13, 3_120, 1); + for (index, frame) in frames.into_iter().enumerate() { + receiver.deliver(&consumer, frame).unwrap(); + let frame = delivered.recv().unwrap(); + assert_eq!(frame.contiguous, index != 0); + assert_eq!(frame.ssrc, Some(42)); + } + drop(delivered); + let frame = red_packet(&mut receiver, 14, 3_360, 0).pop().unwrap(); + assert!(matches!( + receiver.deliver(&consumer, frame), + Err(TransportError::MediaConsumerClosed) + )); +} + +#[test] +fn a_new_source_starts_a_discontinuous_sequence_baseline() { + let mut receiver = NvstAudioReceiver::default(); + red_packet(&mut receiver, 100, 24_000, 0); + for sequence in [1, 2] { + let mut header = header(sequence); + header.ssrc = 43.into(); + let frames = receiver + .depacketize(&track(), &header, Arc::from([0xf8]), 0) + .unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].contiguous, sequence == 2); + assert_eq!(frames[0].ssrc, Some(43)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_bandwidth.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_bandwidth.rs new file mode 100644 index 000000000..30343c351 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_bandwidth.rs @@ -0,0 +1,417 @@ +use std::time::Instant; + +const TRANSIT_SAMPLES: usize = 16; +const MIN_MEASURABLE_TRAIN_MS: f64 = 1.0; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct PacketBandwidthSample { + pub(crate) frame_number: u32, + pub(crate) sequence: u32, + pub(crate) timestamp: u32, + pub(crate) bytes: usize, + pub(crate) started: bool, + pub(crate) ended: bool, + pub(crate) received_at: Instant, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct BandwidthFeedback { + pub(crate) minimum_server_time: u32, + pub(crate) median_server_time: u32, + pub(crate) queue_delay_us: u32, + pub(crate) jitter_us: u16, + pub(crate) utilization_percent: u8, + pub(crate) estimate_kbps: u32, + pub(crate) lossy_frames: u32, +} + +#[derive(Debug, Clone, Copy)] +struct FrameSample { + number: u32, + first_at: Instant, + last_at: Instant, + first_timestamp: u64, + last_timestamp: u64, + last_sequence: u32, + bytes: u64, + sender_gap_ms: f64, + started: bool, + ended: bool, + contiguous: bool, +} + +#[derive(Debug)] +pub(crate) struct BandwidthEstimator { + first_arrival: Option, + last_timestamp: Option<(u32, u64)>, + minimum_transit_ms: Option, + previous_transit_ms: Option, + jitter_ms: f64, + transits: [f64; TRANSIT_SAMPLES], + transit_count: usize, + transit_next: usize, + latest_transit_ms: f64, + active_frame: Option, + frames_observed: u32, + lossy_frames: u32, + average_frame_ms: Option, + average_receive_kbps: Option, + average_utilization_percent: Option, +} + +impl Default for BandwidthEstimator { + fn default() -> Self { + Self { + first_arrival: None, + last_timestamp: None, + minimum_transit_ms: None, + previous_transit_ms: None, + jitter_ms: 0.0, + transits: [0.0; TRANSIT_SAMPLES], + transit_count: 0, + transit_next: 0, + latest_transit_ms: 0.0, + active_frame: None, + frames_observed: 0, + lossy_frames: 0, + average_frame_ms: None, + average_receive_kbps: None, + average_utilization_percent: None, + } + } +} + +impl BandwidthEstimator { + pub(crate) fn observe_parity(&mut self, frame_number: u32, sequence: u32) { + if let Some(frame) = self.active_frame.as_mut() + && frame.number == frame_number + && sequence > frame.last_sequence + { + if sequence != frame.last_sequence + 1 { + frame.contiguous = false; + } + frame.last_sequence = sequence; + } + } + + pub(crate) fn observe(&mut self, sample: PacketBandwidthSample) { + let PacketBandwidthSample { + frame_number, + sequence, + timestamp, + bytes, + started, + ended, + received_at: at, + } = sample; + let first = *self.first_arrival.get_or_insert(at); + let extended = match self.last_timestamp { + Some((last, extended)) => { + let delta = timestamp.wrapping_sub(last) as i32; + if delta < 0 { + return; + } + extended + delta as u64 + } + None => u64::from(timestamp), + }; + self.last_timestamp = Some((timestamp, extended)); + let transit_ms = + at.saturating_duration_since(first).as_secs_f64() * 1_000.0 - extended as f64 / 90.0; + let minimum = self.minimum_transit_ms.get_or_insert(transit_ms); + *minimum = minimum.min(transit_ms); + self.latest_transit_ms = transit_ms; + if let Some(previous) = self.previous_transit_ms { + self.jitter_ms += ((transit_ms - previous).abs() - self.jitter_ms) / 16.0; + } + self.previous_transit_ms = Some(transit_ms); + self.transits[self.transit_next] = transit_ms; + self.transit_next = (self.transit_next + 1) % TRANSIT_SAMPLES; + self.transit_count = (self.transit_count + 1).min(TRANSIT_SAMPLES); + + if let Some(frame) = self.active_frame.as_mut() + && frame.number == frame_number + { + if sequence <= frame.last_sequence { + frame.contiguous = false; + return; + } + if sequence != frame.last_sequence + 1 { + frame.contiguous = false; + } + let sender_gap_ms = extended.saturating_sub(frame.last_timestamp) as f64 / 90.0; + let receive_gap_ms = + at.saturating_duration_since(frame.last_at).as_secs_f64() * 1_000.0; + if sender_gap_ms >= 1.0 { + frame.sender_gap_ms += sender_gap_ms.min(receive_gap_ms); + } + frame.last_at = at; + frame.last_timestamp = extended; + frame.last_sequence = sequence; + frame.bytes = frame.bytes.saturating_add(bytes as u64); + frame.ended |= ended; + return; + } + + if let Some(previous) = self.active_frame.take() { + let distance = frame_number.wrapping_sub(previous.number); + if distance == 0 || distance >= 0x8000_0000 { + self.active_frame = Some(previous); + return; + } + self.lossy_frames = self.lossy_frames.saturating_add(distance.saturating_sub(1)); + if previous.started && previous.ended && previous.contiguous { + let frame_ms = extended.saturating_sub(previous.first_timestamp) as f64 / 90.0; + let received_ms = (previous + .last_at + .saturating_duration_since(previous.first_at) + .as_secs_f64() + * 1_000.0 + - previous.sender_gap_ms) + .max(0.5); + if frame_ms > 0.0 && frame_ms < 1_000.0 { + self.average_frame_ms = Some(smooth(self.average_frame_ms, frame_ms)); + let receive_kbps = previous.bytes as f64 * 8.0 / received_ms; + // A train shorter than the receive clock can resolve, such as a + // single-packet frame in a static scene, only bounds capacity from + // below, so it may raise the estimate but must not drag it down. + if received_ms >= MIN_MEASURABLE_TRAIN_MS + || self + .average_receive_kbps + .is_none_or(|average| receive_kbps > average) + { + self.average_receive_kbps = + Some(smooth(self.average_receive_kbps, receive_kbps)); + } + self.average_utilization_percent = Some(smooth( + self.average_utilization_percent, + (100.0 * received_ms / self.average_frame_ms.unwrap()).min(100.0), + )); + self.frames_observed = self.frames_observed.saturating_add(1); + if self.frames_observed % 60 == 0 { + self.minimum_transit_ms = + self.minimum_transit_ms.map(|minimum| minimum + 0.5); + } + } + } else { + self.lossy_frames = self.lossy_frames.saturating_add(1); + } + } + self.active_frame = Some(FrameSample { + number: frame_number, + first_at: at, + last_at: at, + first_timestamp: extended, + last_timestamp: extended, + last_sequence: sequence, + bytes: bytes as u64, + sender_gap_ms: 0.0, + started, + ended, + contiguous: true, + }); + } + + pub(crate) fn report(&self, at: Instant) -> BandwidthFeedback { + let (Some(first), Some(minimum)) = (self.first_arrival, self.minimum_transit_ms) else { + return BandwidthFeedback::default(); + }; + let mut samples = self.transits; + samples[..self.transit_count].sort_by(f64::total_cmp); + let median = samples[(self.transit_count - 1) / 2]; + let now_ms = at.saturating_duration_since(first).as_secs_f64() * 1_000.0; + let server_time = + |transit: f64| (((now_ms - transit).max(0.0) * 90.0) as u64 & 0x7fff_ffff) as u32; + BandwidthFeedback { + minimum_server_time: server_time(minimum), + median_server_time: server_time(median), + queue_delay_us: ((self.latest_transit_ms - minimum).max(0.0) * 1_000.0) as u32, + jitter_us: (self.jitter_ms * 1_000.0).min(f64::from(u16::MAX)) as u16, + utilization_percent: if self.frames_observed >= 100 { + self.average_utilization_percent.unwrap_or_default() as u8 + } else { + 0 + }, + estimate_kbps: if self.frames_observed >= 100 { + self.average_receive_kbps.unwrap_or_default() as u32 + } else { + 0 + }, + lossy_frames: self.lossy_frames, + } + } +} + +fn smooth(previous: Option, value: f64) -> f64 { + previous.map_or(value, |previous| previous * 0.875 + value * 0.125) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn packet( + frame_number: u32, + sequence: u32, + timestamp: u32, + bytes: usize, + flags: (bool, bool), + received_at: Instant, + ) -> PacketBandwidthSample { + PacketBandwidthSample { + frame_number, + sequence, + timestamp, + bytes, + started: flags.0, + ended: flags.1, + received_at, + } + } + + #[test] + fn packet_train_reports_measured_capacity_after_one_hundred_frames() { + let mut estimator = BandwidthEstimator::default(); + let origin = Instant::now(); + for frame in 0..=100_u32 { + let at = origin + Duration::from_millis(u64::from(frame) * 20); + let timestamp = 90_000 + frame * 1_800; + for part in 0..3_u32 { + estimator.observe(packet( + frame, + frame * 3 + part, + timestamp, + 200, + (part == 0, part == 2), + at + Duration::from_micros(u64::from(part) * 500), + )); + } + } + let report = estimator.report(origin + Duration::from_millis(2_001)); + assert_eq!(report.minimum_server_time, 270_090); + assert_eq!(report.estimate_kbps, 4_800); + assert_eq!(report.utilization_percent, 5); + assert_eq!(report.queue_delay_us, 1_000); + assert!(report.jitter_us > 0); + assert_eq!(report.lossy_frames, 0); + assert_ne!(report.minimum_server_time, report.median_server_time); + } + + #[test] + fn unresolvable_single_packet_frames_do_not_lower_the_estimate() { + let mut estimator = BandwidthEstimator::default(); + let origin = Instant::now(); + for frame in 0..=100_u32 { + let at = origin + Duration::from_millis(u64::from(frame) * 20); + let timestamp = 90_000 + frame * 1_800; + for part in 0..3_u32 { + estimator.observe(packet( + frame, + frame * 3 + part, + timestamp, + 200, + (part == 0, part == 2), + at + Duration::from_micros(u64::from(part) * 500), + )); + } + } + let measured = estimator.average_receive_kbps.unwrap(); + for frame in 101..=200_u32 { + estimator.observe(packet( + frame, + 202 + frame, + 90_000 + frame * 1_800, + 200, + (true, true), + origin + Duration::from_millis(u64::from(frame) * 20), + )); + } + assert_eq!(estimator.average_receive_kbps, Some(measured)); + assert_eq!(estimator.frames_observed, 200); + } + + #[test] + fn incomplete_and_reordered_frames_do_not_inflate_the_estimate() { + let mut estimator = BandwidthEstimator::default(); + let origin = Instant::now(); + estimator.observe(packet(1, 100, 90_000, 1_200, (true, false), origin)); + estimator.observe(packet( + 1, + 102, + 90_000, + 1_200, + (false, true), + origin + Duration::from_millis(1), + )); + estimator.observe(packet( + 3, + 103, + 93_600, + 1_200, + (true, true), + origin + Duration::from_millis(40), + )); + estimator.observe(packet( + 2, + 101, + 91_800, + 1_200, + (true, true), + origin + Duration::from_millis(41), + )); + assert_eq!( + estimator + .report(origin + Duration::from_millis(50)) + .lossy_frames, + 2 + ); + assert_eq!(estimator.frames_observed, 0); + assert!(estimator.average_receive_kbps.is_none()); + } + + #[test] + fn sender_timestamp_wrap_preserves_estimated_server_clock() { + let mut estimator = BandwidthEstimator::default(); + let origin = Instant::now(); + estimator.observe(packet(1, 10, u32::MAX - 899, 300, (true, true), origin)); + estimator.observe(packet( + 2, + 11, + 901, + 300, + (true, true), + origin + Duration::from_millis(20), + )); + let report = estimator.report(origin + Duration::from_millis(20)); + assert!(report.minimum_server_time.abs_diff(900) <= 1); + assert_eq!(estimator.frames_observed, 1); + } + + #[test] + fn parity_between_video_shards_does_not_invalidate_a_frame() { + let mut estimator = BandwidthEstimator::default(); + let origin = Instant::now(); + estimator.observe(packet(1, 10, 90_000, 400, (true, false), origin)); + estimator.observe_parity(1, 11); + estimator.observe(packet( + 1, + 12, + 90_000, + 400, + (false, true), + origin + Duration::from_millis(1), + )); + estimator.observe(packet( + 2, + 13, + 91_800, + 400, + (true, true), + origin + Duration::from_millis(20), + )); + assert_eq!(estimator.frames_observed, 1); + assert_eq!(estimator.lossy_frames, 0); + assert_eq!(estimator.average_receive_kbps, Some(6_400.0)); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget.rs new file mode 100644 index 000000000..04256242e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget.rs @@ -0,0 +1,146 @@ +pub(crate) const TRANSPORT_AGGREGATE_CAPACITY: usize = 128 * 1024; +pub(crate) const TEARDOWN_RESERVE: usize = 64 * 1024; +pub(crate) const NORMAL_BUDGET: usize = TRANSPORT_AGGREGATE_CAPACITY - TEARDOWN_RESERVE; +pub(crate) const TEXT_BATCH_GUARD: usize = 128 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WriteClass { + Normal, + Teardown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WriteAdmission { + Admitted, + OverBudget, + EmptyWrite, +} + +impl WriteAdmission { + pub(crate) const fn is_admitted(self) -> bool { + matches!(self, Self::Admitted) + } +} + +pub(crate) fn admit_write( + buffered_total: usize, + candidate: usize, + class: WriteClass, +) -> WriteAdmission { + if candidate == 0 { + return WriteAdmission::EmptyWrite; + } + let limit = match class { + WriteClass::Normal => NORMAL_BUDGET, + WriteClass::Teardown => TRANSPORT_AGGREGATE_CAPACITY, + }; + if buffered_total.saturating_add(candidate) <= limit { + WriteAdmission::Admitted + } else { + WriteAdmission::OverBudget + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_effective_ceiling_is_the_actual_transport_capacity() { + assert_eq!(TRANSPORT_AGGREGATE_CAPACITY, 128 * 1024); + assert_eq!( + NORMAL_BUDGET, + TRANSPORT_AGGREGATE_CAPACITY - TEARDOWN_RESERVE + ); + assert_eq!( + admit_write(TRANSPORT_AGGREGATE_CAPACITY - 1, 1, WriteClass::Teardown), + WriteAdmission::Admitted + ); + assert_eq!( + admit_write(TRANSPORT_AGGREGATE_CAPACITY, 1, WriteClass::Teardown), + WriteAdmission::OverBudget + ); + } + + #[test] + fn normal_admission_stops_at_the_shared_budget() { + assert_eq!( + admit_write(0, 1, WriteClass::Normal), + WriteAdmission::Admitted + ); + assert_eq!( + admit_write(NORMAL_BUDGET - 1, 1, WriteClass::Normal), + WriteAdmission::Admitted + ); + assert_eq!( + admit_write(NORMAL_BUDGET, 1, WriteClass::Normal), + WriteAdmission::OverBudget + ); + } + + #[test] + fn normal_admission_always_leaves_the_teardown_reservation_free() { + for buffered in 0..=NORMAL_BUDGET { + for candidate in [1_usize, 64, 1024, 4096] { + if admit_write(buffered, candidate, WriteClass::Normal).is_admitted() { + let headroom = NORMAL_BUDGET - (buffered + candidate); + assert!( + TRANSPORT_AGGREGATE_CAPACITY - (buffered + candidate) >= TEARDOWN_RESERVE, + "normal write at buffered={buffered} candidate={candidate} left headroom {}", + TRANSPORT_AGGREGATE_CAPACITY - (buffered + candidate) + ); + let _ = headroom; + assert_eq!( + admit_write(buffered + candidate, 1, WriteClass::Teardown), + WriteAdmission::Admitted + ); + } + } + } + } + + #[test] + fn teardown_admission_still_works_after_a_normal_flood_fills_its_budget() { + assert_eq!( + admit_write(NORMAL_BUDGET, 1, WriteClass::Normal), + WriteAdmission::OverBudget + ); + assert_eq!( + admit_write(NORMAL_BUDGET, 1, WriteClass::Teardown), + WriteAdmission::Admitted + ); + assert_eq!( + admit_write(NORMAL_BUDGET, TEARDOWN_RESERVE, WriteClass::Teardown), + WriteAdmission::Admitted + ); + assert_eq!( + admit_write(NORMAL_BUDGET, TEARDOWN_RESERVE + 1, WriteClass::Teardown), + WriteAdmission::OverBudget + ); + } + + #[test] + fn the_text_guard_is_never_stricter_than_the_shared_ceiling() { + assert_eq!(TEXT_BATCH_GUARD, 128 * 1024); + assert_eq!( + admit_write(0, TEXT_BATCH_GUARD, WriteClass::Normal), + WriteAdmission::OverBudget + ); + assert_eq!( + admit_write(0, TEXT_BATCH_GUARD, WriteClass::Teardown), + WriteAdmission::Admitted + ); + } + + #[test] + fn empty_writes_are_rejected_before_touching_the_transport() { + assert_eq!( + admit_write(0, 0, WriteClass::Normal), + WriteAdmission::EmptyWrite + ); + assert_eq!( + admit_write(0, 0, WriteClass::Teardown), + WriteAdmission::EmptyWrite + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget_tests.rs new file mode 100644 index 000000000..880c24a6a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_budget_tests.rs @@ -0,0 +1,632 @@ +use super::*; + +use std::net::UdpSocket; +use std::time::Duration; + +use str0m::net::Receive; +use str0m::{Candidate, Input, Output}; + +use crate::nvst_budget::{ + NORMAL_BUDGET, TEARDOWN_RESERVE, TEXT_BATCH_GUARD, TRANSPORT_AGGREGATE_CAPACITY, WriteClass, + admit_write, +}; +use crate::nvst_input::{ + NvstEncodedInput, NvstInputRoute, NvstInputWrite, SonyDeviceControl, + sony_device_change_command, sony_report_command, +}; + +fn exchange(local: &mut Rtc, remote: &mut Rtc, now: Instant) { + loop { + match local.poll_output() { + Ok(Output::Timeout(_)) => break, + Ok(Output::Transmit(packet)) => { + remote + .handle_input(Input::Receive( + now, + Receive { + proto: packet.proto, + source: packet.source, + destination: packet.destination, + contents: packet.contents.as_ref().try_into().unwrap(), + }, + )) + .unwrap(); + } + Ok(_) => {} + Err(error) => panic!("bundle poll failed: {error}"), + } + } +} + +pub(super) fn connect_bundle_pair() -> (Rtc, Rtc, NvstInputChannels, NvstInputChannels) { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut local = create_nvst_bundle_rtc(&socket).unwrap(); + let mut remote = create_nvst_bundle_rtc(&socket).unwrap(); + let local_candidate = Candidate::host("192.0.2.1:1000".parse().unwrap(), "udp").unwrap(); + let remote_candidate = Candidate::host("192.0.2.2:2000".parse().unwrap(), "udp").unwrap(); + local.add_local_candidate(local_candidate.clone()); + local.add_remote_candidate(remote_candidate.clone()); + remote.add_local_candidate(remote_candidate); + remote.add_remote_candidate(local_candidate); + let local_identity = local.direct_api().local_dtls_fingerprint().clone(); + let remote_identity = remote.direct_api().local_dtls_fingerprint().clone(); + let local_ice = local.direct_api().local_ice_credentials(); + let remote_ice = remote.direct_api().local_ice_credentials(); + local.direct_api().set_remote_fingerprint(remote_identity); + remote.direct_api().set_remote_fingerprint(local_identity); + local.direct_api().set_remote_ice_credentials(remote_ice); + remote.direct_api().set_remote_ice_credentials(local_ice); + local.direct_api().set_ice_controlling(true); + remote.direct_api().set_ice_controlling(false); + local.direct_api().start_dtls(true).unwrap(); + remote.direct_api().start_dtls(false).unwrap(); + let local_channels = NvstInputChannels::create(&mut local, true); + let remote_channels = NvstInputChannels::create(&mut remote, true); + local.direct_api().start_sctp(true); + remote.direct_api().start_sctp(false); + let origin = Instant::now(); + let mut established = false; + for tick in 0..4000 { + let now = origin + Duration::from_millis(tick); + local.handle_input(Input::Timeout(now)).unwrap(); + remote.handle_input(Input::Timeout(now)).unwrap(); + exchange(&mut local, &mut remote, now); + exchange(&mut remote, &mut local, now); + if tick % 16 == 0 + && local.is_connected() + && remote.is_connected() + && local_channels.send_keepalive(&mut local, 1) + { + established = true; + break; + } + } + assert!( + established, + "the bundle pair never produced an established SCTP association" + ); + (local, remote, local_channels, remote_channels) +} + +fn sony_teardown_commands() -> Vec> { + let report = [0_u8; 64]; + let mut commands = Vec::new(); + for low_id in 6..=9 { + commands.push(sony_report_command(low_id, &report).unwrap()); + commands.push(sony_device_change_command(SonyDeviceControl::Removal, low_id).unwrap()); + } + commands +} + +#[test] +fn encoded_input_classifies_open_full_and_closed_channels() { + let (mut local, _remote, channels, _) = connect_bundle_pair(); + let message = NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: vec![1; 64], + meaningful: true, + }; + assert_eq!( + channels.send_encoded(&mut local, &message), + NvstInputWrite::Sent + ); + while channels.send_control(&mut local, &[1; 1024]) {} + assert_eq!( + channels.send_encoded( + &mut local, + &NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: vec![1; 1024], + meaningful: true, + } + ), + NvstInputWrite::Deferred, + ); + local.disconnect(); + assert_eq!( + channels.send_encoded(&mut local, &message), + NvstInputWrite::Closed + ); + let mut unestablished = Rtc::new(Instant::now()); + let unopened = NvstInputChannels::create(&mut unestablished, true); + assert_eq!( + unopened.send_encoded(&mut unestablished, &message), + NvstInputWrite::Closed + ); +} + +#[test] +fn input_readiness_races_defer_without_writing_but_dead_channels_are_closed() { + let message = NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: vec![1; 64], + meaningful: true, + }; + let mut waiting = Rtc::new(Instant::now()); + assert_eq!( + classify_nvst_unready_input(&mut waiting, None, &message), + NvstInputWrite::Deferred + ); + let (mut local, _remote, channels, _) = connect_bundle_pair(); + let buffered = channels.buffered_total(&mut local); + assert_eq!( + classify_nvst_unready_input(&mut local, Some(channels), &message), + NvstInputWrite::Deferred + ); + assert_eq!(channels.buffered_total(&mut local), buffered); + local.disconnect(); + assert_eq!( + classify_nvst_unready_input(&mut local, Some(channels), &message), + NvstInputWrite::Closed + ); + waiting.disconnect(); + assert_eq!( + classify_nvst_unready_input(&mut waiting, None, &message), + NvstInputWrite::Closed + ); +} + +#[test] +fn typed_unicode_wire_success_clears_idle_warning_but_cancelled_text_does_not() { + let (mut local, _remote, channels, _) = connect_bundle_pair(); + let slot = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let text = slot.submit("é世界".as_bytes()).unwrap(); + let warning = AtomicBool::new(true); + let (events, received) = mpsc::channel(); + let mut delivery = NvstInputDelivery::default(); + delivery.submit_text(text.clone(), 1, |message| { + let outcome = channels.send_encoded(&mut local, message); + report_nvst_user_input(message, outcome, &warning, &events); + outcome + }).unwrap(); + assert!(!warning.load(Ordering::Acquire)); + assert!(matches!( + received.try_recv().unwrap(), + NvstReceiveEvent::UserInput + )); + assert!(received.try_recv().is_err()); + warning.store(true, Ordering::Release); + slot.cancel(); + delivery.submit_text(text.clone(), 1, |_| panic!("cancelled text must not write")).unwrap(); + assert!(warning.load(Ordering::Acquire)); +} + +#[test] +fn teardown_budget_covers_the_admitted_sony_bounds_with_full_framing() { + let commands = sony_teardown_commands(); + assert_eq!(commands.len(), 8); + let total: usize = commands.iter().map(Vec::len).sum(); + assert_eq!(total, 4 * (76 + 26)); + assert!(total <= TEARDOWN_RESERVE); + assert!(total < NORMAL_BUDGET); +} + +#[test] +fn actual_writers_reserve_teardown_capacity_under_a_normal_flood() { + let (mut local, _remote, channels, _remote_channels) = connect_bundle_pair(); + + let report_command = sony_report_command(6, &[0_u8; 64]).unwrap(); + let removal_command = sony_device_change_command(SonyDeviceControl::Removal, 6).unwrap(); + + let mut normal_written = 0_usize; + let mut accepted = 0_usize; + while channels.send_control(&mut local, &report_command) { + normal_written += report_command.len(); + accepted += 1; + assert!(accepted <= TRANSPORT_AGGREGATE_CAPACITY / 8); + } + assert!( + normal_written <= NORMAL_BUDGET, + "normal writers spent {normal_written} bytes beyond the shared budget" + ); + assert!(normal_written + report_command.len() > NORMAL_BUDGET); + + let buffered = channels.buffered_total(&mut local); + assert!(buffered <= NORMAL_BUDGET); + assert!(buffered >= NORMAL_BUDGET - report_command.len()); + assert!( + TRANSPORT_AGGREGATE_CAPACITY - buffered >= TEARDOWN_RESERVE, + "the normal flood consumed the teardown reservation" + ); + + assert!( + !channels.send_control(&mut local, &report_command), + "a normal write must not spend the teardown reservation" + ); + assert_eq!( + channels.send_encoded( + &mut local, + &NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: vec![0_u8; report_command.len()], + meaningful: true, + } + ), + NvstInputWrite::Deferred, + "reliable encoded input must be normal traffic" + ); + + assert!( + channels.send_control_teardown(&mut local, &removal_command), + "teardown traffic must still be admitted at the normal budget edge" + ); + + let mut teardown_written = removal_command.len(); + let mut teardown_commands = 0_usize; + loop { + let command = sony_teardown_commands() + .into_iter() + .nth(teardown_commands % 8) + .unwrap(); + if !channels.send_control_teardown(&mut local, &command) { + break; + } + teardown_written += command.len(); + teardown_commands += 1; + assert!(teardown_commands <= TRANSPORT_AGGREGATE_CAPACITY); + } + + let worst_case_batch: usize = sony_teardown_commands().iter().map(Vec::len).sum(); + assert!( + teardown_written >= worst_case_batch, + "the reservation must cover the worst-case teardown batch" + ); + assert!( + teardown_written <= TEARDOWN_RESERVE, + "teardown traffic spent {teardown_written} bytes beyond the reservation" + ); + assert!( + !channels.send_control(&mut local, &report_command), + "normal traffic must not resume while the reservation is in use" + ); + let buffered = channels.buffered_total(&mut local); + assert!(buffered <= TRANSPORT_AGGREGATE_CAPACITY); +} + +#[test] +fn rtcp_writer_is_normal_traffic() { + let (mut local, _remote, channels, _remote_channels) = connect_bundle_pair(); + + let rtcp = build_rtcp_receiver_report( + 1, + RtcpReportBlock { + media_ssrc: 2, + fraction_lost: 0, + cumulative_lost: 0, + highest_sequence: 0, + jitter: 0, + }, + ); + + let mut rtcp_written = 0_usize; + let mut iterations = 0_usize; + while channels.send_rtcp(&mut local, &rtcp) { + rtcp_written += rtcp.len(); + iterations += 1; + assert!(iterations <= TRANSPORT_AGGREGATE_CAPACITY); + } + assert!( + rtcp_written <= NORMAL_BUDGET, + "RTCP writes spent {rtcp_written} bytes beyond the shared budget" + ); + let buffered = channels.buffered_total(&mut local); + assert!( + TRANSPORT_AGGREGATE_CAPACITY - buffered >= TEARDOWN_RESERVE, + "RTCP writes consumed the teardown reservation" + ); +} + +#[test] +fn the_text_guard_still_rejects_a_batch_that_the_shared_budget_cannot_hold() { + const { assert!(TEXT_BATCH_GUARD > NORMAL_BUDGET) }; + assert_eq!( + admit_write(0, TEXT_BATCH_GUARD, WriteClass::Normal), + crate::nvst_budget::WriteAdmission::OverBudget + ); +} + +fn sony_review_runtime( + incarnation: u64, + server_mask: u32, +) -> (HidRuntime, HidSession, NvstInputCodec) { + let runtime = HidRuntime::new(); + let claim = opennow_streamer_hid::SdlDeviceClaim::new(0, incarnation, 0x054c, 0x05c4).unwrap(); + runtime.replace_inventory(&[Some(claim)]); + runtime.open_endpoint(); + assert!(runtime.bind_session(1).is_some()); + ( + runtime, + HidSession::new(SonyCapability { + server_mask, + ..SonyCapability::default() + }), + NvstInputCodec::default(), + ) +} + +#[test] +fn parent_review_capture_pause_retains_sony_attachment() { + let (mut rtc, _peer, channels, _peer_channels) = connect_bundle_pair(); + let (runtime, mut session, mut codec) = sony_review_runtime(11, 5); + runtime.set_active(true); + let mut mask = 0; + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 1 + )); + assert_eq!(session.attached(0), Some(11)); + runtime.set_active(false); + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 2 + )); + assert_eq!( + session.attached(0), + Some(11), + "capture closure must retain the Sony attachment" + ); + assert!(!session.is_tombstoned(0)); +} + +#[test] +fn parent_review_resume_preserves_undrained_release() { + let (runtime, _session, _codec) = sony_review_runtime(11, 5); + runtime.set_active(true); + let mut snapshot = opennow_streamer_hid::SonySnapshot::neutral(0, 11, 1); + snapshot.buttons = 1; + runtime.submit_snapshot(snapshot); + runtime.drain(32); + runtime.set_active(false); + runtime.set_active(true); + assert!( + runtime + .drain(32) + .iter() + .any(|item| matches!(item, IngressItem::Release { .. })), + "resume must not erase a release before the bundle writer consumes it" + ); +} + +#[test] +fn unbound_capture_leaves_the_bundle_endpoint_inactive() { + let (mut rtc, _peer, channels, _peer_channels) = connect_bundle_pair(); + let runtime = HidRuntime::new(); + let claim = opennow_streamer_hid::SdlDeviceClaim::new(0, 11, 0x054c, 0x05c4).unwrap(); + runtime.replace_inventory(&[Some(claim)]); + runtime.set_active(true); + let mut session = HidSession::new(SonyCapability { + server_mask: 5, + ..SonyCapability::default() + }); + let mut codec = NvstInputCodec::default(); + let mut mask = 0; + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 1 + )); + assert_eq!(session.attached(0), None); + assert_eq!(mask, 0); + let mut snapshot = opennow_streamer_hid::SonySnapshot::neutral(0, 11, 1); + snapshot.buttons = 0x1000; + assert_eq!( + runtime.submit_snapshot(snapshot), + opennow_streamer_hid::SnapshotAdmission::Unbound + ); + assert!(runtime.drain(8).is_empty()); +} + +#[test] +fn repeated_capture_pause_cycles_send_one_neutral_report_per_source() { + let (mut rtc, _peer, channels, _peer_channels) = connect_bundle_pair(); + let (runtime, mut session, mut codec) = sony_review_runtime(11, 5); + runtime.set_active(true); + let mut mask = 0; + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 1 + )); + assert_eq!(mask, 0b0001); + let neutral_report = sony_report_command(6, &[0_u8; 64]).unwrap(); + let neutral_len = neutral_report.len(); + assert_eq!(neutral_len, 76); + for cycle in 0..8_u64 { + let before = channels.buffered_total(&mut rtc); + runtime.set_active(false); + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 100 + cycle + )); + let after = channels.buffered_total(&mut rtc); + assert_eq!( + after - before, + neutral_len, + "one capture pause must spend exactly one neutral report" + ); + runtime.set_active(true); + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut rtc, + &mut mask, + 200 + cycle + )); + assert_eq!( + channels.buffered_total(&mut rtc), + after, + "resuming capture must not queue anything before new input arrives" + ); + } + assert_eq!(session.attached(0), Some(11)); + assert!(!session.is_tombstoned(0)); + assert_eq!(mask, 0b0001); + let buffered = channels.buffered_total(&mut rtc); + assert!( + TRANSPORT_AGGREGATE_CAPACITY - buffered >= TEARDOWN_RESERVE, + "repeated capture cycles drained the teardown reservation" + ); + assert!(channels.send_control(&mut rtc, &neutral_report)); +} + +fn drain_peer_events(peer: &mut Rtc, now: Instant) -> Vec { + let mut events = Vec::new(); + loop { + match peer.poll_output() { + Ok(Output::Timeout(_)) => break, + Ok(Output::Event(event)) => events.push(event), + Ok(Output::Transmit(_)) => {} + Err(_) => break, + } + } + let _ = now; + events +} + +#[test] +fn sony_shutdown_flush_reaches_the_receiving_peer() { + let (mut local, mut peer, channels, _peer_channels) = connect_bundle_pair(); + let mut session = HidSession::new(SonyCapability { + server_mask: 5, + ..SonyCapability::default() + }); + session + .adopt(opennow_streamer_hid::SdlDeviceClaim::new(0, 11, 0x054c, 0x05c4).unwrap()) + .unwrap(); + let mut snapshot = opennow_streamer_hid::SonySnapshot::neutral(0, 11, 1_000_000); + snapshot.buttons = 0x1000; + let (_, prior) = session.build_report(&snapshot).unwrap(); + queue_hid_shutdown(&mut session, channels, &mut local); + forward_rtc_outputs( + &mut local, + |output| { + if let Output::Transmit(transmit) = output { + peer.handle_input(Input::Receive( + Instant::now(), + Receive { + proto: transmit.proto, + source: transmit.source, + destination: transmit.destination, + contents: transmit.contents.as_ref().try_into().unwrap(), + }, + )) + .unwrap(); + } + }, + Duration::from_millis(200), + ); + let mut payloads = Vec::new(); + for event in drain_peer_events(&mut peer, Instant::now()) { + if let str0m::Event::ChannelData(data) = event { + payloads.push(data.data); + } + } + assert!( + !payloads.is_empty(), + "the shutdown release must reach the receiving peer" + ); + let removal = sony_device_change_command(SonyDeviceControl::Removal, 6).unwrap(); + let release = sony_report_command(6, &[0_u8; 64]).unwrap(); + assert!( + payloads.iter().any(|payload| payload == &removal), + "the receiving peer must observe the Sony removal" + ); + let observed_release = payloads + .iter() + .find(|payload| payload.len() == release.len()) + .expect("the receiving peer must observe the Sony release report"); + let report = &observed_release[observed_release.len() - 64..]; + assert_eq!(&report[10..12], &prior[10..12]); + assert_eq!(report[0], 0x01); +} + +#[test] +fn sony_rich_reports_reach_the_receiving_peer_after_admission() { + let (mut local, mut peer, channels, _peer_channels) = connect_bundle_pair(); + let (runtime, mut session, mut codec) = sony_review_runtime(11, 5); + runtime.set_active(true); + let mut mask = 0; + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut local, + &mut mask, + 1 + )); + assert_eq!(mask, 0b0001); + let mut snapshot = opennow_streamer_hid::SonySnapshot::neutral(0, 11, 2_000_000); + snapshot.buttons = 0x1000; + snapshot.left_stick_x = 12_000; + assert_eq!( + runtime.submit_snapshot(snapshot), + opennow_streamer_hid::SnapshotAdmission::Admitted + ); + assert!(pump_hid_pending( + &runtime, + &mut session, + &mut codec, + channels, + &mut local, + &mut mask, + 2 + )); + forward_rtc_outputs( + &mut local, + |output| { + if let Output::Transmit(transmit) = output { + peer.handle_input(Input::Receive( + Instant::now(), + Receive { + proto: transmit.proto, + source: transmit.source, + destination: transmit.destination, + contents: transmit.contents.as_ref().try_into().unwrap(), + }, + )) + .unwrap(); + } + }, + Duration::from_millis(200), + ); + let mut payloads = Vec::new(); + for event in drain_peer_events(&mut peer, Instant::now()) { + if let str0m::Event::ChannelData(data) = event { + payloads.push(data.data); + } + } + let report = payloads + .iter() + .find(|payload| payload.len() == 76) + .expect("the peer must receive the live Sony report"); + assert_eq!(report[report.len() - 64], 0x01); + let buttons = + u16::from_le_bytes([report[report.len() - 64 + 5], report[report.len() - 64 + 6]]); + assert_eq!(buttons & 0x20, 0x20); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_control.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_control.rs new file mode 100644 index 000000000..254e7d6ad --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_control.rs @@ -0,0 +1,414 @@ +use std::time::Duration; + +pub(crate) const FRAME_ACK_CODE: u16 = 0x204; +pub(crate) const FRAME_PACING_CODE: u16 = 0x203; +pub(crate) const QOS_REPORT_CODE: u16 = 0x207; +pub(crate) const FRAME_INVALIDATION_CODE: u16 = 0x301; +pub(crate) const IDR_REQUEST_CODE: u16 = 0x302; +pub(crate) const NACK_V2_CODE: u16 = 0x317; +pub(crate) const MAX_NACK_PACKET_COUNT: usize = 64; + +pub(crate) const FRAME_ACK_PAYLOAD_LEN: usize = 102; +pub(crate) const FRAME_PACING_PAYLOAD_LEN: usize = 28; +pub(crate) const QOS_REPORT_PAYLOAD_LEN: usize = 52; +pub(crate) const FRAME_INVALIDATION_PAYLOAD_LEN: usize = 24; +// The official GFN client keeps the frame-pacing PID target at 16.666 ms even for a +// 120 FPS encoded stream. This is a renderer/feedback target, not the encoded-frame interval. +pub(crate) const DEFAULT_FRAME_TIME_US: u32 = 16_666; +pub(crate) const FRAME_PACING_INTERVAL: Duration = Duration::from_micros(55_556); +pub(crate) const QOS_REPORT_INTERVAL: Duration = Duration::from_millis(50); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NvstControlCommand { + pub(crate) code: u16, + pub(crate) payload: Vec, +} + +impl NvstControlCommand { + pub(crate) fn encoded(&self) -> Vec { + let Ok(payload_len) = u16::try_from(self.payload.len()) else { + return Vec::new(); + }; + let mut encoded = Vec::with_capacity(4 + self.payload.len()); + encoded.extend_from_slice(&self.code.to_le_bytes()); + encoded.extend_from_slice(&payload_len.to_le_bytes()); + encoded.extend_from_slice(&self.payload); + encoded + } +} + +pub(crate) fn nack_v2(stream_index: u8, missing: &[u16]) -> Option { + if missing.is_empty() || missing.len() > MAX_NACK_PACKET_COUNT { + return None; + } + let mut payload = Vec::with_capacity(3 + missing.len() * 10); + payload.extend_from_slice(&[2, stream_index, 0]); + let mut base = missing[0]; + let mut bitmap = 0_u64; + for &sequence in &missing[1..] { + let distance = sequence.wrapping_sub(base); + if distance == 0 { + continue; + } + if distance <= 64 { + bitmap |= 1_u64 << (distance - 1); + } else { + payload.extend_from_slice(&base.to_le_bytes()); + payload.extend_from_slice(&bitmap.to_le_bytes()); + payload[2] += 1; + base = sequence; + bitmap = 0; + } + } + payload.extend_from_slice(&base.to_le_bytes()); + payload.extend_from_slice(&bitmap.to_le_bytes()); + payload[2] += 1; + Some(NvstControlCommand { + code: NACK_V2_CODE, + payload, + }) +} + +pub(crate) fn frame_ack( + frame_number: u32, + first_packet_time_ms: Option, + frame_bytes: u32, +) -> NvstControlCommand { + let mut payload = vec![0; FRAME_ACK_PAYLOAD_LEN]; + put_u16(&mut payload, 0, 1); + put_u16(&mut payload, 2, 9); + put_u32(&mut payload, 4, frame_number); + if let Some(first_packet_time_ms) = first_packet_time_ms { + put_u64(&mut payload, 12, first_packet_time_ms.to_bits()); + } + for offset in (20..=48).step_by(4) { + put_u32(&mut payload, offset, (-1.0_f32).to_bits()); + } + put_u32(&mut payload, 72, frame_bytes); + put_u32(&mut payload, 94, (-1.0_f32).to_bits()); + put_u32(&mut payload, 98, (-1.0_f32).to_bits()); + NvstControlCommand { + code: FRAME_ACK_CODE, + payload, + } +} + +pub(crate) fn frame_pacing_report( + frame_number: u32, + target_frame_time_us: u32, + pacing_error_us: u32, +) -> NvstControlCommand { + let mut payload = vec![0; FRAME_PACING_PAYLOAD_LEN]; + put_u32(&mut payload, 0, 5); + put_u32(&mut payload, 8, 2); + put_u32(&mut payload, 12, frame_number); + put_u32(&mut payload, 16, target_frame_time_us); + put_u32(&mut payload, 20, pacing_error_us.min(target_frame_time_us)); + put_u32(&mut payload, 24, 0x341a); + NvstControlCommand { + code: FRAME_PACING_CODE, + payload, + } +} + +#[derive(Debug, Default)] +pub(crate) struct QosReport { + pub(crate) sequence: u32, + pub(crate) sender_frame_number: u32, + pub(crate) bytes_received: u32, + pub(crate) loss_per_ten_thousand: u16, + pub(crate) client_time_90khz: u32, + pub(crate) packet_snapshot: Option, + pub(crate) bandwidth: crate::nvst_bandwidth::BandwidthFeedback, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct QosPacketSnapshot { + pub(crate) ssrc: u32, + pub(crate) base: u32, + pub(crate) highest: u32, + pub(crate) received: u32, +} + +impl QosReport { + pub(crate) fn command(&self) -> NvstControlCommand { + let mut payload = vec![0; QOS_REPORT_PAYLOAD_LEN]; + put_u32(&mut payload, 0, 7); + put_u32(&mut payload, 8, self.sequence); + put_u32(&mut payload, 12, self.sender_frame_number); + put_u32(&mut payload, 16, self.bandwidth.minimum_server_time); + put_u32(&mut payload, 20, self.bandwidth.queue_delay_us); + put_u16(&mut payload, 24, self.bandwidth.jitter_us); + put_u16(&mut payload, 26, self.loss_per_ten_thousand); + payload[28] = self.bandwidth.utilization_percent; + put_u16(&mut payload, 30, 1_000); + put_u16(&mut payload, 32, 1_000); + put_u32(&mut payload, 36, self.client_time_90khz); + put_u32(&mut payload, 40, self.bandwidth.lossy_frames); + put_u32(&mut payload, 44, self.bandwidth.estimate_kbps); + put_u32(&mut payload, 48, self.bandwidth.median_server_time); + NvstControlCommand { + code: QOS_REPORT_CODE, + payload, + } + } +} + +pub(crate) fn idr_request() -> NvstControlCommand { + NvstControlCommand { + code: IDR_REQUEST_CODE, + payload: 0_u16.to_le_bytes().to_vec(), + } +} + +/// `ServerControl::sendFrameInvalidationRange`: inclusive sender frame range, then the +/// zero-extended stream index, matching GameStream `IDX_INVALIDATE_REF_FRAMES`. +pub(crate) fn frame_invalidation_range( + first_frame: u64, + last_frame: u64, + stream_index: u16, +) -> NvstControlCommand { + let mut payload = vec![0; FRAME_INVALIDATION_PAYLOAD_LEN]; + put_u64(&mut payload, 0, first_frame); + put_u64(&mut payload, 8, last_frame); + put_u64(&mut payload, 16, u64::from(stream_index)); + NvstControlCommand { + code: FRAME_INVALIDATION_CODE, + payload, + } +} + +fn put_u16(payload: &mut [u8], offset: usize, value: u16) { + payload[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(payload: &mut [u8], offset: usize, value: u32) { + payload[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(payload: &mut [u8], offset: usize, value: u64) { + payload[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hex(value: &str) -> Vec { + value + .as_bytes() + .chunks_exact(2) + .map(|pair| { + u8::from_str_radix(std::str::from_utf8(pair).expect("ASCII hex"), 16) + .expect("valid hex") + }) + .collect() + } + + #[test] + fn command_header_is_little_endian_and_byte_exact() { + let command = NvstControlCommand { + code: 0x207, + payload: vec![1, 2, 3], + }; + assert_eq!(command.encoded(), [0x07, 0x02, 0x03, 0x00, 1, 2, 3]); + } + + #[test] + fn nack_v2_encodes_the_implicit_base_without_a_bitmap_bit() { + let command = nack_v2(0, &[0x1234]).unwrap(); + assert_eq!(command.code, NACK_V2_CODE); + assert_eq!(command.encoded(), hex("17030d0002000134120000000000000000")); + } + + #[test] + fn nack_v2_uses_all_64_bitmap_bits_before_starting_another_record() { + let command = nack_v2(7, &[0x1234, 0x1235, 0x1274, 0x1275]).unwrap(); + assert_eq!( + command.encoded(), + hex("170317000207023412010000000000008075120000000000000000") + ); + } + + #[test] + fn nack_v2_groups_across_sequence_wrap_without_requesting_the_holes() { + let command = nack_v2(0, &[65534, 65535, 0, 62, 63]).unwrap(); + assert_eq!( + command.encoded(), + hex("17031700020002feff03000000000000803f000000000000000000") + ); + } + + #[test] + fn nack_v2_bounds_missing_packets_not_sequence_span_or_record_count() { + assert!(nack_v2(0, &[]).is_none()); + assert!(nack_v2(0, &[0; MAX_NACK_PACKET_COUNT + 1]).is_none()); + let contiguous: Vec = (0..64).collect(); + assert_eq!( + nack_v2(0, &contiguous).unwrap().encoded(), + hex("17030d000200010000ffffffffffffff7f") + ); + let sparse: Vec = (0..64).map(|index| index * 65).collect(); + let command = nack_v2(0, &sparse).unwrap(); + assert_eq!(command.payload.len(), 643); + assert_eq!(&command.encoded()[..7], &[0x17, 3, 0x83, 2, 2, 0, 64]); + for (record, sequence) in command.payload[3..].chunks_exact(10).zip(sparse) { + assert_eq!(&record[..2], &sequence.to_le_bytes()); + assert_eq!(&record[2..], &[0; 8]); + } + } + + #[test] + fn nack_v2_round_trip_preserves_only_requested_sequences() { + use std::collections::BTreeSet; + + for base in [0_u16, 1, 32767, 65534, 65535] { + for step in [0_u16, 1, 2, 64, 65, 127, 4095, 65535] { + for count in 1..=MAX_NACK_PACKET_COUNT { + let missing: Vec = (0..count) + .map(|index| base.wrapping_add((index as u16).wrapping_mul(step))) + .collect(); + let command = nack_v2(3, &missing).unwrap(); + assert_eq!( + command.payload.len(), + 3 + usize::from(command.payload[2]) * 10 + ); + let mut decoded = BTreeSet::new(); + for record in command.payload[3..].chunks_exact(10) { + let base = u16::from_le_bytes(record[..2].try_into().unwrap()); + let bitmap = u64::from_le_bytes(record[2..].try_into().unwrap()); + decoded.insert(base); + for bit in 0..64 { + if bitmap & (1_u64 << bit) != 0 { + decoded.insert(base.wrapping_add(bit + 1)); + } + } + } + assert_eq!(decoded, missing.into_iter().collect()); + } + } + } + } + + #[test] + fn frame_pacing_report_matches_the_source_test_vector() { + let command = frame_pacing_report(1, 16_000, 16_000); + assert_eq!(command.code, FRAME_PACING_CODE); + assert_eq!( + command.payload, + hex("05000000000000000200000001000000803e0000803e00001a340000") + ); + assert_eq!(command.encoded().len(), 4 + FRAME_PACING_PAYLOAD_LEN); + } + + #[test] + fn frame_ack_places_only_source_pinned_fields() { + let command = frame_ack(42, Some(20_320.16), 15_168); + assert_eq!(command.code, FRAME_ACK_CODE); + assert_eq!(command.payload.len(), FRAME_ACK_PAYLOAD_LEN); + assert_eq!(&command.payload[0..4], &[1, 0, 9, 0]); + assert_eq!( + u32::from_le_bytes(command.payload[4..8].try_into().unwrap()), + 42 + ); + assert_eq!( + u64::from_le_bytes(command.payload[12..20].try_into().unwrap()), + 20_320.16_f64.to_bits() + ); + for offset in (20..=48).step_by(4) { + assert_eq!( + &command.payload[offset..offset + 4], + &(-1.0_f32).to_le_bytes() + ); + } + assert_eq!(&command.payload[56..60], &[0; 4]); + assert_eq!( + u32::from_le_bytes(command.payload[72..76].try_into().unwrap()), + 15_168 + ); + assert_eq!(&command.payload[84..88], &[0; 4]); + assert_eq!(&command.payload[94..98], &(-1.0_f32).to_le_bytes()); + assert_eq!(&command.payload[98..102], &(-1.0_f32).to_le_bytes()); + } + + #[test] + fn frame_ack_never_writes_a_measured_value_into_unpinned_stage_slots() { + let command = frame_ack(1, None, 7); + assert_eq!(&command.payload[12..20], &[0; 8]); + assert_eq!(&command.payload[56..60], &[0; 4]); + for offset in (20..=48).step_by(4) { + assert_eq!( + &command.payload[offset..offset + 4], + &(-1.0_f32).to_le_bytes() + ); + } + } + + #[test] + fn qos_report_matches_the_source_test_layout() { + let command = QosReport { + sequence: 6, + sender_frame_number: 2, + bytes_received: 244_808, + client_time_90khz: 1_818_674, + ..QosReport::default() + } + .command(); + assert_eq!(command.code, QOS_REPORT_CODE); + assert_eq!( + command.payload, + hex( + "070000000000000006000000020000000000000000000000000000000000e803e803000032c01b00000000000000000000000000" + ) + ); + } + + #[test] + fn qos_report_writes_sender_frame_loss_and_client_clock_at_their_wire_offsets() { + let report = QosReport { + sender_frame_number: 0x1234_5678, + loss_per_ten_thousand: 2_500, + client_time_90khz: 0x8765_4321, + ..QosReport::default() + }; + let payload = report.command().payload; + assert_eq!(&payload[12..16], &0x1234_5678_u32.to_le_bytes()); + assert_eq!(&payload[26..28], &2_500_u16.to_le_bytes()); + assert_eq!(&payload[34..36], &[0, 0]); + assert_eq!(&payload[36..40], &0x8765_4321_u32.to_le_bytes()); + for range in [16..26, 28..30, 40..52] { + assert!(payload[range].iter().all(|byte| *byte == 0)); + } + assert_eq!(QOS_REPORT_INTERVAL, Duration::from_millis(50)); + assert_eq!(FRAME_PACING_INTERVAL, Duration::from_micros(55_556)); + } + + #[test] + fn idr_request_matches_the_source_layout() { + assert_eq!(idr_request().encoded(), [0x02, 0x03, 0x02, 0x00, 0, 0]); + } + + #[test] + fn frame_invalidation_range_matches_the_bifrost_layout() { + let command = frame_invalidation_range(0x0102_0304_0506_0708, 0x1122_3344, 7); + assert_eq!(command.code, FRAME_INVALIDATION_CODE); + assert_eq!( + command.encoded(), + hex(concat!( + "01031800", + "0807060504030201", + "4433221100000000", + "0700000000000000" + )) + ); + assert_eq!( + frame_invalidation_range(41, 43, 0).encoded(), + hex(concat!( + "01031800", + "2900000000000000", + "2b00000000000000", + "0000000000000000" + )) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_cursor.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_cursor.rs new file mode 100644 index 000000000..7c40be7b7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_cursor.rs @@ -0,0 +1,297 @@ +use std::time::{Duration, Instant}; + +const RETRY_INTERVAL: Duration = Duration::from_millis(250); +const MAX_ATTEMPTS: u8 = 8; +const SILENT_HOST_TIMEOUT: Duration = Duration::from_secs(3); + +pub(crate) fn valid_cursor_channel_message(bytes: &[u8]) -> bool { + if bytes.len() < 7 || !matches!(bytes[0], 0 | 1) { + return false; + } + let image_length_offset = 5 + usize::from(bytes[4]); + let Some(length) = bytes.get(image_length_offset..image_length_offset + 2) else { + return false; + }; + let image_length = usize::from(u16::from_le_bytes([length[0], length[1]])); + let image_end = image_length_offset + 2 + image_length; + bytes.len() >= image_end + && matches!(bytes.len() - image_end, 0 | 4 | 6) + && (bytes[0] != 0 || (bytes[4] == 0 && image_length == 0)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CursorCommand { + Capture(bool), + Tracking, +} + +#[derive(Debug, Default)] +enum CaptureState { + #[default] + Inactive, + Waiting { + deadline: Instant, + retry_at: Instant, + attempts: u8, + }, + Disabling { + retry_at: Instant, + attempts: u8, + reason: &'static str, + }, + Local, + Exhausted, +} + +#[derive(Debug, Default)] +pub(crate) struct NvstCursorCapture { + state: CaptureState, +} + +impl NvstCursorCapture { + pub(crate) fn activate(&mut self, now: Instant) { + self.state = CaptureState::Waiting { + deadline: now + SILENT_HOST_TIMEOUT, + retry_at: now + RETRY_INTERVAL, + attempts: 1, + }; + } + + pub(crate) fn reset(&mut self) { + self.state = CaptureState::Inactive; + } + + pub(crate) fn notify(&mut self, now: Instant) { + if matches!(self.state, CaptureState::Waiting { .. }) { + self.state = CaptureState::Disabling { + retry_at: now, + attempts: 0, + reason: "cursor-notification", + }; + } + } + + pub(crate) fn update( + &mut self, + now: Instant, + mut send: impl FnMut(CursorCommand) -> bool, + ) -> bool { + if let CaptureState::Waiting { deadline, .. } = self.state + && now >= deadline + { + self.state = CaptureState::Disabling { + retry_at: now, + attempts: 0, + reason: "silent-host-timeout", + }; + } + match &mut self.state { + CaptureState::Waiting { + retry_at, attempts, .. + } if *attempts < MAX_ATTEMPTS && now >= *retry_at => { + *attempts += 1; + let capture_sent = send(CursorCommand::Capture(true)); + let tracking_sent = send(CursorCommand::Tracking); + eprintln!( + "NVST cursor feature retry: attempt={attempts} captureSent={capture_sent} trackingSent={tracking_sent}" + ); + *retry_at = now + RETRY_INTERVAL; + } + CaptureState::Disabling { + retry_at, + attempts, + reason, + } if now >= *retry_at => { + *attempts += 1; + let sent = send(CursorCommand::Capture(false)); + eprintln!( + "NVST cursor capture tx: command=0x0308 enabled=false reason={reason} attempt={attempts} queued={sent}" + ); + if sent { + self.state = CaptureState::Local; + return true; + } + if *attempts == MAX_ATTEMPTS { + eprintln!( + "NVST cursor capture disable retries exhausted; retaining server-composited cursor until reactivation" + ); + self.state = CaptureState::Exhausted; + } else { + *retry_at = now + RETRY_INTERVAL; + } + } + _ => {} + } + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cursor_channel_requires_complete_supported_framing() { + assert!(valid_cursor_channel_message(&[0, 0, 0, 0, 0, 0, 0])); + assert!(valid_cursor_channel_message(&[ + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ])); + let custom = [ + 1, 0, 1, 2, 3, b'p', b'n', b'g', 4, 0, b'a', b'b', b'c', b'd', + ]; + assert!(valid_cursor_channel_message(&custom)); + for len in 0..custom.len() { + assert!(!valid_cursor_channel_message(&custom[..len])); + } + assert!(!valid_cursor_channel_message(&[2, 1, 0, 0, 0, 0, 0])); + assert!(!valid_cursor_channel_message(&[0, 1, 0, 0, 0, 1, 0, 42])); + assert!(!valid_cursor_channel_message(&[0, 1, 0, 0, 0, 0, 0, 0])); + } + + #[test] + fn truncated_native_bitmaps_do_not_trigger_handoff() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + for length in 0..8_u8 { + let mut bytes = vec![0x10, 0x01, length, 0]; + bytes.resize(4 + usize::from(length), 0); + assert!(crate::nvst_input::server_cursor_messages(&bytes).is_empty()); + } + assert!(!capture.update(now, |_| panic!("no valid notification"))); + } + + #[test] + fn notification_hands_off_once_and_stops_enable_retries() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + capture.notify(now); + let mut commands = Vec::new(); + assert!(capture.update(now, |command| { + commands.push(command); + true + })); + capture.notify(now + RETRY_INTERVAL); + assert!(!capture.update(now + SILENT_HOST_TIMEOUT, |command| { + commands.push(command); + true + })); + assert_eq!(commands, [CursorCommand::Capture(false)]); + } + + #[test] + fn bitmap_notification_hands_off_without_synthesizing_shape_or_mode() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + let messages = crate::nvst_input::server_cursor_messages(&[ + 0x10, 0x01, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef, 0, 0, 0, 0, + ]); + assert_eq!(messages.len(), 1); + for message in messages { + assert_eq!(message.normalized, None); + assert_eq!(message.visible, None); + capture.notify(now); + } + assert!(capture.update(now, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + true + })); + } + + #[test] + fn silent_host_falls_back_after_bounded_enable_retries() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + let mut commands = Vec::new(); + for tick in 0..12 { + assert!(!capture.update(now + RETRY_INTERVAL * tick, |command| { + commands.push(command); + true + })); + } + assert_eq!(commands.len(), usize::from(MAX_ATTEMPTS - 1) * 2); + assert!( + commands + .chunks_exact(2) + .all(|pair| { pair == [CursorCommand::Capture(true), CursorCommand::Tracking] }) + ); + assert!(capture.update(now + SILENT_HOST_TIMEOUT, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + true + })); + } + + #[test] + fn failed_disable_is_paced_and_reports_handoff_only_on_success() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + capture.notify(now); + assert!(!capture.update(now, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + false + })); + capture.notify(now); + assert!(!capture.update(now, |_| panic!("retry must be paced"))); + assert!(capture.update(now + RETRY_INTERVAL, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + true + })); + } + + #[test] + fn exhausted_disable_retries_do_not_report_local_composition() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + capture.notify(now); + let mut attempts = 0; + for tick in 0..20 { + capture.notify(now + RETRY_INTERVAL * tick); + assert!(!capture.update(now + RETRY_INTERVAL * tick, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + attempts += 1; + false + })); + } + assert_eq!(attempts, MAX_ATTEMPTS); + capture.activate(now); + capture.notify(now); + assert!(capture.update(now, |_| true)); + } + + #[test] + fn reset_cancels_retries_and_reactivation_rearms_local_handoff() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.notify(now); + assert!(!capture.update(now, |_| panic!("inactive capture"))); + capture.activate(now); + capture.notify(now); + capture.reset(); + assert!(!capture.update(now + SILENT_HOST_TIMEOUT, |_| panic!("closed channel"))); + capture.activate(now); + capture.notify(now); + assert!(capture.update(now, |_| true)); + capture.activate(now); + assert!(capture.update(now + SILENT_HOST_TIMEOUT, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + true + })); + } + + #[test] + fn failed_enable_retries_do_not_cancel_silent_host_deadline() { + let now = Instant::now(); + let mut capture = NvstCursorCapture::default(); + capture.activate(now); + assert!(!capture.update(now + RETRY_INTERVAL, |_| false)); + assert!(capture.update(now + SILENT_HOST_TIMEOUT, |command| { + assert_eq!(command, CursorCommand::Capture(false)); + true + })); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_haptics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_haptics.rs new file mode 100644 index 000000000..aecafae31 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_haptics.rs @@ -0,0 +1,231 @@ +use std::sync::Mutex; + +pub(crate) const HAPTIC_COMMAND_CODE: u16 = 0x010b; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NvstControllerRumble { + pub controller_id: u8, + pub low_frequency: u16, + pub high_frequency: u16, + pub duration_ms: u32, + pub source_incarnation: Option, +} + +#[derive(Debug, Default)] +pub struct NvstHaptics { + pending: Mutex<([Option; 4], usize)>, +} + +impl NvstHaptics { + pub(crate) fn receive(&self, mut bytes: &[u8]) { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while bytes.len() >= 4 { + let code = u16::from_le_bytes([bytes[0], bytes[1]]); + let length = usize::from(u16::from_le_bytes([bytes[2], bytes[3]])); + let Some(payload) = bytes.get(4..4 + length) else { + break; + }; + bytes = &bytes[4 + length..]; + if code != HAPTIC_COMMAND_CODE || payload.len() < 4 { + continue; + } + let record_length = match u16::from_le_bytes([payload[0], payload[1]]) { + 1 => 6, + 2 => 8, + _ => continue, + }; + let available = + usize::from(u16::from_le_bytes([payload[2], payload[3]])).min(payload.len() - 4); + for record in payload[4..4 + available].chunks_exact(record_length) { + let controller_id = u16::from_le_bytes([record[0], record[1]]); + if controller_id >= 4 { + continue; + } + let duration = if record_length == 8 { + u16::from_le_bytes([record[6], record[7]]) + } else { + 0 + }; + let command = NvstControllerRumble { + controller_id: controller_id as u8, + low_frequency: u16::from_le_bytes([record[2], record[3]]), + high_frequency: u16::from_le_bytes([record[4], record[5]]), + duration_ms: if duration == 0 { + 1000 + } else { + u32::from(duration) + }, + source_incarnation: None, + }; + if pending.0[usize::from(controller_id)] + .replace(command) + .is_some() + { + pending.1 = pending.1.saturating_add(1); + } + } + } + } + + pub fn take(&self) -> ([Option; 4], usize) { + std::mem::take( + &mut *self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ) + } + + pub(super) fn store_sony(&self, rumble: opennow_streamer_hid::SonyRumble) { + let command = NvstControllerRumble { + controller_id: rumble.slot, + low_frequency: rumble.low_frequency, + high_frequency: rumble.high_frequency, + duration_ms: 0, + source_incarnation: Some(rumble.incarnation), + }; + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if pending.0[usize::from(rumble.slot)] + .replace(command) + .is_some() + { + pending.1 = pending.1.saturating_add(1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command(kind: u16, declared_length: u16, records: &[u16]) -> Vec { + let mut payload = Vec::new(); + for value in [kind, declared_length].iter().chain(records) { + payload.extend_from_slice(&value.to_le_bytes()); + } + super::super::nvst_control::NvstControlCommand { + code: 0x010b, + payload, + } + .encoded() + } + + #[test] + fn reference_pulse_vector_is_little_endian_and_framed() { + let haptics = NvstHaptics::default(); + haptics.receive(&[ + 0x0b, 0x01, 0x0c, 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 0x00, 0xff, 0xff, 0x00, 0x00, + 0xfa, 0x00, + ]); + assert_eq!( + haptics.take().0[2], + Some(NvstControllerRumble { + controller_id: 2, + low_frequency: 65535, + high_frequency: 0, + duration_ms: 250, + source_incarnation: None, + }) + ); + } + + #[test] + fn state_and_pulse_preserve_slots_motors_and_effective_duration() { + let haptics = NvstHaptics::default(); + let mut bytes = command(1, 12, &[0, 0x1234, 0xabcd, 3, 0xffff, 0]); + bytes.extend(command(2, 16, &[1, 9, 8, 321, 2, 7, 6, 0])); + haptics.receive(&bytes); + let (commands, coalesced) = haptics.take(); + assert_eq!(coalesced, 0); + assert_eq!( + commands[0], + Some(NvstControllerRumble { + controller_id: 0, + low_frequency: 0x1234, + high_frequency: 0xabcd, + duration_ms: 1000, + source_incarnation: None, + }) + ); + assert_eq!(commands[1].unwrap().duration_ms, 321); + assert_eq!(commands[2].unwrap().duration_ms, 1000); + assert_eq!(commands[3].unwrap().low_frequency, 65535); + assert_eq!(haptics.take(), ([None; 4], 0)); + } + + #[test] + fn sony_output_uses_the_existing_bounded_slot_with_the_source_incarnation() { + let haptics = NvstHaptics::default(); + haptics.store_sony(opennow_streamer_hid::SonyRumble { + slot: 3, + incarnation: 77, + low_frequency: 0x4000, + high_frequency: 0x8000, + }); + haptics.store_sony(opennow_streamer_hid::SonyRumble { + slot: 3, + incarnation: 77, + low_frequency: 0x1000, + high_frequency: 0x2000, + }); + let (commands, coalesced) = haptics.take(); + assert_eq!(coalesced, 1); + assert_eq!( + commands[3], + Some(NvstControllerRumble { + controller_id: 3, + low_frequency: 0x1000, + high_frequency: 0x2000, + duration_ms: 0, + source_incarnation: Some(77), + }) + ); + assert_eq!(commands[0], None); + } + + #[test] + fn malformed_frames_and_unknown_kinds_or_slots_do_not_create_rumble() { + let haptics = NvstHaptics::default(); + let valid = command(2, 8, &[0, 1, 2, 3]); + for length in 0..valid.len() { + haptics.receive(&valid[..length]); + } + haptics.receive(&command(3, 8, &[0, 1, 2, 3])); + haptics.receive(&command(1, 6, &[256, 1, 2])); + haptics.receive(&command(1, 5, &[0, 1, 2])); + assert_eq!(haptics.take(), ([None; 4], 0)); + } + + #[test] + fn blob_lengths_limit_records_and_complete_short_blob_records_survive() { + let haptics = NvstHaptics::default(); + haptics.receive(&command(1, 6, &[0, 1, 2, 1, 3, 4])); + let (commands, _) = haptics.take(); + assert!(commands[0].is_some()); + assert!(commands[1].is_none()); + haptics.receive(&command(1, 12, &[2, 5, 6, 3])); + let (commands, _) = haptics.take(); + assert!(commands[2].is_some()); + assert!(commands[3].is_none()); + } + + #[test] + fn backlog_is_four_latest_states_including_stop() { + let haptics = NvstHaptics::default(); + for _ in 0..1000 { + haptics.receive(&command(1, 6, &[0, 65535, 65535])); + } + haptics.receive(&command(1, 6, &[0, 0, 0])); + let (commands, coalesced) = haptics.take(); + assert_eq!(coalesced, 1000); + assert_eq!(commands.iter().flatten().count(), 1); + assert_eq!(commands[0].unwrap().low_frequency, 0); + assert_eq!(commands[0].unwrap().high_frequency, 0); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input.rs new file mode 100644 index 000000000..1882193f8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input.rs @@ -0,0 +1,2442 @@ +use std::fmt; +use std::time::{Duration, Instant}; + +use str0m::Rtc; +use str0m::channel::{ChannelConfig, ChannelId, Reliability}; + +#[cfg(test)] +use crate::nvst_budget::TEXT_BATCH_GUARD; +use crate::nvst_budget::{WriteClass, admit_write}; + +pub(crate) const CONTROL_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(3); +pub(crate) const INPUT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); + +const CONTROL_RELIABLE_LABEL: &str = "control_channel_reliable"; +const CUSTOM_RELIABLE_LABEL: &str = "custom_message_on_sctp_private_reliable"; +const CUSTOM_PARTIAL_LABEL: &str = "custom_message_on_sctp_private_partially_reliable"; +const CONTROL_PARTIAL_LABEL: &str = "control_channel_partially_reliable"; +const CONTROL_UNRELIABLE_LABEL: &str = "control_channel_unreliable"; +const INPUT_PARTIAL_LABEL: &str = "input_channel_partially_reliable"; +const CURSOR_LABEL: &str = "cursor_channel"; +const RTCP_ON_SCTP_LABEL: &str = "rtcp_on_sctp_private"; +const PARTIAL_RELIABLE_LIFETIME_MS: u16 = 300; +const CUSTOM_PARTIAL_MAX_RETRANSMITS: u16 = 2; + +const COMMAND_SYSTEM_CURSOR: u16 = 0x010f; +const COMMAND_BITMAP_CURSOR: u16 = 0x0110; +const COMMAND_KEEPALIVE: u16 = 0x0200; +const COMMAND_REMOTE_INPUT: u16 = 0x0206; +const COMMAND_ENABLE_INPUT: u16 = 0x020b; +const COMMAND_GAMEPAD: u16 = 0x020d; +const COMMAND_INPUT_VERSION: u16 = 0x020e; +// Controls whether the server composites its cursor into the video. The +// official client enables this for startup, waits for the first ServerControl +// cursor notification, then disables it and renders the reported cursor +// locally. Notifications continue after the server-rendered cursor is hidden. +const COMMAND_MOUSE_CURSOR_CAPTURE: u16 = 0x0308; +// NVB feature type 8 ("track remote cursor image") is distinct from cursor +// capture. Bifrost keeps this enabled after it turns capture back off so the +// server continues publishing cursor shape and mode changes. +const COMMAND_TRACK_REMOTE_CURSOR_IMAGE: u16 = 0x030d; +const COMMAND_WINDOW_STATE: u16 = 0x0320; +const COMMAND_SYSTEM_STATE: u16 = 0x0321; + +const INPUT_KEY_DOWN: u32 = 3; +const INPUT_KEY_UP: u32 = 4; +const INPUT_HEARTBEAT: u32 = 2; +const INPUT_MOUSE_ABSOLUTE: u32 = 5; +const INPUT_MOUSE_RELATIVE: u32 = 7; +const INPUT_MOUSE_BUTTON_DOWN: u32 = 8; +const INPUT_MOUSE_BUTTON_UP: u32 = 9; +const INPUT_MOUSE_WHEEL: u32 = 10; +const INPUT_GAMEPAD: u32 = 12; +const INPUT_HAPTICS_ENABLED: u32 = 13; +const INPUT_LOCK_KEYS_SYNC: u32 = 19; +const INPUT_TEXT: u32 = 23; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NvstChannelReliability { + Reliable, + Lifetime(u16), + MaxRetransmits(u16), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NvstChannelDefinition { + pub(crate) sid: u16, + pub(crate) label: &'static str, + pub(crate) ordered: bool, + pub(crate) reliability: NvstChannelReliability, +} + +pub(crate) const NVST_CHANNEL_PROFILE: [NvstChannelDefinition; 8] = [ + NvstChannelDefinition { + sid: 0, + label: CONTROL_RELIABLE_LABEL, + ordered: true, + reliability: NvstChannelReliability::Reliable, + }, + NvstChannelDefinition { + sid: 2, + label: CUSTOM_RELIABLE_LABEL, + ordered: true, + reliability: NvstChannelReliability::Reliable, + }, + NvstChannelDefinition { + sid: 4, + label: CUSTOM_PARTIAL_LABEL, + ordered: true, + reliability: NvstChannelReliability::MaxRetransmits(CUSTOM_PARTIAL_MAX_RETRANSMITS), + }, + NvstChannelDefinition { + sid: 6, + label: CONTROL_PARTIAL_LABEL, + // Mouse motion uses this stream. It must be unordered: with ordered + // PR-SCTP, one lost report blocks every newer report until its 300 ms + // lifetime expires, producing delayed motion followed by catch-up. + ordered: false, + reliability: NvstChannelReliability::Lifetime(PARTIAL_RELIABLE_LIFETIME_MS), + }, + NvstChannelDefinition { + sid: 8, + label: CONTROL_UNRELIABLE_LABEL, + ordered: false, + reliability: NvstChannelReliability::MaxRetransmits(0), + }, + NvstChannelDefinition { + sid: 10, + label: INPUT_PARTIAL_LABEL, + ordered: false, + reliability: NvstChannelReliability::Lifetime(PARTIAL_RELIABLE_LIFETIME_MS), + }, + NvstChannelDefinition { + sid: 12, + label: CURSOR_LABEL, + ordered: true, + reliability: NvstChannelReliability::Reliable, + }, + NvstChannelDefinition { + sid: 14, + label: RTCP_ON_SCTP_LABEL, + ordered: true, + reliability: NvstChannelReliability::Reliable, + }, +]; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct NvstInputChannels { + pub(crate) control_reliable: ChannelId, + pub(crate) custom_reliable: ChannelId, + pub(crate) custom_partial: ChannelId, + pub(crate) control_partial: ChannelId, + pub(crate) control_unreliable: ChannelId, + pub(crate) input_partial: ChannelId, + pub(crate) cursor: Option, + pub(crate) rtcp: Option, +} + +impl NvstInputChannels { + pub(crate) fn create(rtc: &mut Rtc, bundle_video: bool) -> Self { + let mut ids = Vec::with_capacity(6); + for definition in &NVST_CHANNEL_PROFILE[..6] { + ids.push( + rtc.direct_api() + .create_data_channel(channel_config(*definition)), + ); + } + let cursor = bundle_video.then(|| { + rtc.direct_api() + .create_data_channel(channel_config(NVST_CHANNEL_PROFILE[6])) + }); + let rtcp = bundle_video.then(|| { + rtc.direct_api() + .create_data_channel(channel_config(NVST_CHANNEL_PROFILE[7])) + }); + Self { + control_reliable: ids[0], + custom_reliable: ids[1], + custom_partial: ids[2], + control_partial: ids[3], + control_unreliable: ids[4], + input_partial: ids[5], + cursor, + rtcp, + } + } + + pub(crate) fn contains(self, id: ChannelId) -> bool { + self.rtcp != Some(id) && self.all().any(|channel| channel == id) + } + + pub(crate) fn label(self, id: ChannelId) -> &'static str { + if id == self.control_reliable { + CONTROL_RELIABLE_LABEL + } else if id == self.custom_reliable { + CUSTOM_RELIABLE_LABEL + } else if id == self.custom_partial { + CUSTOM_PARTIAL_LABEL + } else if id == self.control_partial { + CONTROL_PARTIAL_LABEL + } else if id == self.control_unreliable { + CONTROL_UNRELIABLE_LABEL + } else if id == self.input_partial { + INPUT_PARTIAL_LABEL + } else if Some(id) == self.cursor { + CURSOR_LABEL + } else if Some(id) == self.rtcp { + RTCP_ON_SCTP_LABEL + } else { + "unknown" + } + } + + pub(crate) fn send_control(self, rtc: &mut Rtc, bytes: &[u8]) -> bool { + self.write(rtc, self.control_reliable, bytes, WriteClass::Normal) + } + + pub(crate) fn send_control_teardown(self, rtc: &mut Rtc, bytes: &[u8]) -> bool { + self.write(rtc, self.control_reliable, bytes, WriteClass::Teardown) + } + + pub(crate) fn send_rtcp(self, rtc: &mut Rtc, bytes: &[u8]) -> bool { + self.rtcp + .is_some_and(|id| self.write(rtc, id, bytes, WriteClass::Normal)) + } + + pub(crate) fn send_partial_control(self, rtc: &mut Rtc, bytes: &[u8]) -> bool { + self.write(rtc, self.control_partial, bytes, WriteClass::Normal) + } + + pub(crate) fn send_keepalive(self, rtc: &mut Rtc, stream_value: u32) -> bool { + self.send_control(rtc, &control_keepalive(stream_value)) + } + + pub(crate) fn send_activation(self, rtc: &mut Rtc, timestamp_us: u64) -> bool { + activation_chain(timestamp_us) + .iter() + .all(|bytes| self.send_control(rtc, bytes)) + } + + pub(crate) fn send_mouse_cursor_capture(self, rtc: &mut Rtc, enabled: bool) -> bool { + self.send_control(rtc, &mouse_cursor_capture(enabled)) + } + + pub(crate) fn send_remote_cursor_tracking(self, rtc: &mut Rtc, enabled: bool) -> bool { + self.send_control(rtc, &remote_cursor_tracking(enabled)) + } + + pub(crate) fn send_encoded(self, rtc: &mut Rtc, message: &NvstEncodedInput) -> NvstInputWrite { + self.write_classified( + rtc, + self.encoded_channel(message), + &message.bytes, + WriteClass::Normal, + ) + } + + pub(crate) fn encoded_channel_is_open(self, rtc: &mut Rtc, message: &NvstEncodedInput) -> bool { + rtc.channel(self.encoded_channel(message)).is_some() + } + + fn encoded_channel(self, message: &NvstEncodedInput) -> ChannelId { + match message.route { + NvstInputRoute::ControlReliable => self.control_reliable, + NvstInputRoute::ControlPartial => self.control_partial, + NvstInputRoute::InputPartial => self.input_partial, + } + } + + fn write(self, rtc: &mut Rtc, id: ChannelId, bytes: &[u8], class: WriteClass) -> bool { + self.write_classified(rtc, id, bytes, class).is_sent() + } + + fn write_classified( + self, + rtc: &mut Rtc, + id: ChannelId, + bytes: &[u8], + class: WriteClass, + ) -> NvstInputWrite { + if rtc.channel(id).is_none() { + return NvstInputWrite::Closed; + } + if !admit_write(self.buffered_total(rtc), bytes.len(), class).is_admitted() { + return NvstInputWrite::Deferred; + } + match rtc + .channel(id) + .map(|mut channel| channel.write(true, bytes)) + { + Some(Ok(true)) => NvstInputWrite::Sent, + Some(Ok(false)) => NvstInputWrite::Deferred, + Some(Err(_)) | None => NvstInputWrite::Closed, + } + } + + pub(crate) fn buffered_total(self, rtc: &mut Rtc) -> usize { + self.all() + .filter_map(|id| rtc.channel(id).map(|mut channel| channel.buffered_amount())) + .sum() + } + + fn all(self) -> impl Iterator { + [ + Some(self.control_reliable), + Some(self.custom_reliable), + Some(self.custom_partial), + Some(self.control_partial), + Some(self.control_unreliable), + Some(self.input_partial), + self.cursor, + self.rtcp, + ] + .into_iter() + .flatten() + } +} + +pub(crate) fn text_batch_fits(buffered: usize, batch_bytes: usize, guard: usize) -> bool { + buffered.saturating_add(batch_bytes) <= guard +} + +fn channel_config(definition: NvstChannelDefinition) -> ChannelConfig { + let reliability = match definition.reliability { + NvstChannelReliability::Reliable => Reliability::Reliable, + NvstChannelReliability::Lifetime(lifetime) => Reliability::MaxPacketLifetime { lifetime }, + NvstChannelReliability::MaxRetransmits(retransmits) => { + Reliability::MaxRetransmits { retransmits } + } + }; + ChannelConfig { + label: definition.label.to_owned(), + ordered: definition.ordered, + reliability, + negotiated: None, + protocol: String::new(), + } +} + +#[derive(Debug, Default)] +pub(crate) struct NvstInputChannelState { + control_open: bool, + negotiated_version: Option, + ready_reported: bool, + activation_sent: bool, +} + +impl NvstInputChannelState { + pub(crate) fn channel_opened( + &mut self, + channels: NvstInputChannels, + id: ChannelId, + ) -> Option { + if id == channels.control_reliable { + self.control_open = true; + } + self.take_new_ready_version() + } + + pub(crate) fn channel_data( + &mut self, + channels: NvstInputChannels, + id: ChannelId, + bytes: &[u8], + ) -> Option { + if id == channels.control_reliable + && let Some(version) = input_protocol_version(bytes) + { + self.negotiated_version = Some(version); + } + self.take_new_ready_version() + } + + pub(crate) fn is_ready(&self) -> bool { + self.control_open && self.negotiated_version.is_some() + } + + pub(crate) fn control_is_open(&self) -> bool { + self.control_open + } + + pub(crate) fn handshake_timed_out(&self, started_at: Option, now: Instant) -> bool { + !self.is_ready() + && started_at.is_some_and(|started| { + now.saturating_duration_since(started) >= INPUT_HANDSHAKE_TIMEOUT + }) + } + + pub(crate) fn activation_sent(&self) -> bool { + self.activation_sent + } + + pub(crate) fn mark_activation_sent(&mut self) { + self.activation_sent = true; + } + + pub(crate) fn channel_closed(&mut self, channels: NvstInputChannels, id: ChannelId) -> bool { + if id == channels.input_partial { + return self.is_ready(); + } + if id != channels.control_reliable { + return false; + } + let was_ready = self.is_ready(); + self.control_open = false; + self.negotiated_version = None; + self.ready_reported = false; + self.activation_sent = false; + was_ready + } + + fn take_new_ready_version(&mut self) -> Option { + if self.ready_reported || !self.is_ready() { + return None; + } + self.ready_reported = true; + self.negotiated_version + } +} + +pub(crate) fn next_control_keepalive(now: Instant) -> Instant { + now + CONTROL_KEEPALIVE_INTERVAL +} + +pub(crate) fn input_protocol_version(bytes: &[u8]) -> Option { + if bytes.len() < 2 { + return None; + } + let mut cursor = 0; + while bytes.len().saturating_sub(cursor) >= 4 { + let code = u16::from_le_bytes([bytes[cursor], bytes[cursor + 1]]); + let payload_len = usize::from(u16::from_le_bytes([bytes[cursor + 2], bytes[cursor + 3]])); + let payload_start = cursor + 4; + let payload_end = payload_start.checked_add(payload_len)?; + if payload_end > bytes.len() { + break; + } + if code == COMMAND_INPUT_VERSION && payload_len >= 2 { + return Some(u16::from_le_bytes([ + bytes[payload_start], + bytes[payload_start + 1], + ])); + } + cursor = payload_end; + } + let first = u16::from_le_bytes([bytes[0], bytes[1]]); + if first == COMMAND_INPUT_VERSION { + return Some(if bytes.len() >= 4 { + u16::from_le_bytes([bytes[2], bytes[3]]) + } else { + 2 + }); + } + (bytes[0] == 0x0e).then_some(first) +} + +/// Extracts server cursor notifications from the reliable control stream and +/// normalizes them for the platform output layer. +/// +/// Bifrost's current Windows client dispatches 0x010f (system cursor) and +/// 0x0110 (bitmap cursor) through ServerControl. The separately negotiated +/// `cursor_channel` is not used for these notifications by current GFN hosts. +#[cfg(test)] +pub(crate) fn server_cursor_updates(bytes: &[u8]) -> Vec> { + server_cursor_messages(bytes) + .into_iter() + .filter_map(|message| message.normalized) + .collect() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NvstServerCursorMessage { + pub(crate) command: u16, + pub(crate) offset: usize, + pub(crate) raw: Vec, + pub(crate) cursor_id: Option, + pub(crate) position: Option<(u16, u16)>, + pub(crate) visible: Option, + pub(crate) normalized: Option>, +} + +pub(crate) fn server_cursor_messages(bytes: &[u8]) -> Vec { + let mut updates = Vec::new(); + let mut offset = 0_usize; + while bytes.len().saturating_sub(offset) >= 4 { + let code = u16::from_le_bytes([bytes[offset], bytes[offset + 1]]); + let payload_len = usize::from(u16::from_le_bytes([bytes[offset + 2], bytes[offset + 3]])); + let payload_start = offset + 4; + let Some(payload_end) = payload_start.checked_add(payload_len) else { + break; + }; + if payload_end > bytes.len() { + break; + } + let payload = &bytes[payload_start..payload_end]; + match code { + COMMAND_SYSTEM_CURSOR if payload.len() >= 4 => { + let cursor_id = + u32::from_le_bytes([payload[0], payload[1], payload[2], payload[3]]); + let position = (payload.len() >= 8).then(|| { + ( + u16::from_le_bytes([payload[4], payload[5]]), + u16::from_le_bytes([payload[6], payload[7]]), + ) + }); + let visible = payload.get(8).map(|value| *value != 0); + let normalized = u8::try_from(cursor_id).ok().map(|cursor_id| { + // Platform cursor wire shape: + // type, id, hotspot x/y, empty MIME/image, optional x/y. + // + // The optional byte after x/y is cursor visibility metadata, + // not the relative-mouse sentinel. Geronimo preserves the + // cursor ID verbatim; predefined ID 0 is what selects locked + // input in the official SDL cursor manager. + let mut normalized = vec![0, cursor_id, 0, 0, 0, 0, 0]; + if payload.len() >= 8 { + normalized.extend_from_slice(&payload[4..8]); + } + normalized + }); + updates.push(NvstServerCursorMessage { + command: code, + offset, + raw: bytes[offset..payload_end].to_vec(), + cursor_id: Some(cursor_id), + position, + visible, + normalized, + }); + } + COMMAND_BITMAP_CURSOR if payload.len() >= 8 => { + // Bitmap cursor payloads have a distinct native pixel layout. + // Keep detecting them explicitly so they cannot be mistaken for + // input/control commands while raw bitmap support is added. + updates.push(NvstServerCursorMessage { + command: code, + offset, + raw: bytes[offset..payload_end].to_vec(), + cursor_id: None, + position: None, + visible: None, + normalized: None, + }); + } + _ => {} + } + offset = payload_end; + } + updates +} + +pub(crate) fn native_input_types(packet: &[u8]) -> Result, NvstInputCodecError> { + native_events(packet, 0).map(|events| { + events + .into_iter() + .filter_map(|event| read_u32_le(event.bytes, 0)) + .collect() + }) +} + +pub(crate) fn native_input_type_name(input_type: u32) -> &'static str { + match input_type { + INPUT_HEARTBEAT => "heartbeat", + INPUT_KEY_DOWN => "key-down", + INPUT_KEY_UP => "key-up", + INPUT_MOUSE_ABSOLUTE => "mouse-absolute", + INPUT_MOUSE_RELATIVE => "mouse-relative", + INPUT_MOUSE_BUTTON_DOWN => "mouse-button-down", + INPUT_MOUSE_BUTTON_UP => "mouse-button-up", + INPUT_MOUSE_WHEEL => "mouse-wheel", + INPUT_GAMEPAD => "gamepad", + INPUT_HAPTICS_ENABLED => "haptics-enabled", + INPUT_LOCK_KEYS_SYNC => "lock-keys-sync", + INPUT_TEXT => "text", + _ => "unknown", + } +} + +pub(crate) fn native_input_type_is_motion(input_type: u32) -> bool { + matches!(input_type, INPUT_MOUSE_ABSOLUTE | INPUT_MOUSE_RELATIVE) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NvstInputRoute { + ControlReliable, + ControlPartial, + InputPartial, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NvstEncodedInput { + pub(crate) route: NvstInputRoute, + pub(crate) bytes: Vec, + pub(crate) meaningful: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NvstInputWrite { + Sent, + Deferred, + Closed, +} + +impl NvstInputWrite { + pub(crate) fn is_sent(self) -> bool { + self == Self::Sent + } +} + +impl NvstEncodedInput { + pub(crate) fn is_meaningful(&self) -> bool { + self.meaningful + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NvstInputCodecError { + Malformed(&'static str), + UnsupportedType(u32), + UnsupportedText, +} + +impl fmt::Display for NvstInputCodecError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Malformed(reason) => write!(formatter, "malformed native input: {reason}"), + Self::UnsupportedType(input_type) => { + write!(formatter, "unsupported native input type {input_type}") + } + Self::UnsupportedText => { + formatter.write_str("native NVST text contains an unmappable character") + } + } + } +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct NvstInputCodec { + gamepad_sequences: [u16; 4], + gamepad_bitmap: Option, + rich_slot_mask: u16, + last_gamepad_report: Option<[u8; 38]>, +} + +impl NvstInputCodec { + pub(crate) fn set_rich_slot_mask( + &mut self, + mask: u16, + timestamp_us: u64, + ) -> Vec { + if self.rich_slot_mask == mask { + return Vec::new(); + } + self.rich_slot_mask = mask; + let mut encoded = Vec::new(); + let Some(previous) = self.gamepad_bitmap else { + return encoded; + }; + let filtered = previous & !mask; + if filtered != previous { + self.emit_topology_transition(previous, filtered, timestamp_us, &mut encoded); + } + encoded + } + + fn emit_topology_transition( + &mut self, + previous: u16, + filtered: u16, + timestamp_us: u64, + encoded: &mut Vec, + ) { + for id in 0..self.gamepad_sequences.len() { + if previous & (1 << id) != 0 && filtered & (1 << id) == 0 { + let mut neutral = self.last_gamepad_report.unwrap_or([0_u8; 38]); + neutral[6..8].copy_from_slice(&(id as u16).to_le_bytes()); + neutral[8..10].copy_from_slice(&previous.to_le_bytes()); + neutral[12..24].fill(0); + neutral[30..38].copy_from_slice(×tamp_us.to_be_bytes()); + let mut payload = Vec::with_capacity(48); + payload.push(0x23); + payload.extend_from_slice(×tamp_us.to_be_bytes()); + payload.push(0x22); + payload.extend_from_slice(&neutral); + encoded.push(NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: control_command(COMMAND_GAMEPAD, &payload), + meaningful: false, + }); + } + } + for id in 0..self.gamepad_sequences.len() { + if filtered & (1 << id) == 0 { + self.gamepad_sequences[id] = 0; + } + } + self.gamepad_bitmap = Some(filtered); + encoded.push(NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: device_descriptor(timestamp_us, filtered), + meaningful: false, + }); + } + + pub(crate) fn encode( + &mut self, + packet: &[u8], + fallback_timestamp_us: u64, + ) -> Result, NvstInputCodecError> { + let events = native_events(packet, fallback_timestamp_us)?; + let mut encoded = Vec::new(); + for event in events { + let input_type = read_u32_le(event.bytes, 0) + .ok_or(NvstInputCodecError::Malformed("missing input type"))?; + match input_type { + // The native bundle has its own control-channel keepalive. Renderer heartbeat + // packets are transport keepalives, not remote HID events, so consuming them is + // correct and prevents a benign packet from disabling all native input. + INPUT_HEARTBEAT => {} + INPUT_KEY_DOWN | INPUT_KEY_UP => { + require_len(event.bytes, 18, "short keyboard packet")?; + let key = read_u16_be(event.bytes, 4).expect("keyboard length checked"); + let modifiers = read_u16_be(event.bytes, 6).expect("keyboard length checked"); + let timestamp = read_u64_be(event.bytes, 10).unwrap_or(event.timestamp_us); + encoded.push(remote_input_message( + remote_input_packet( + input_type, + &[ + (key >> 8) as u8, + key as u8, + (modifiers >> 8) as u8, + modifiers as u8, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ], + ), + timestamp, + )); + } + INPUT_MOUSE_RELATIVE => { + require_len(event.bytes, 22, "short relative mouse packet")?; + let body = [ + event.bytes[4], + event.bytes[5], + event.bytes[6], + event.bytes[7], + 0, + 0, + ]; + encoded.push(remote_input_message( + remote_input_packet(input_type, &body), + read_u64_be(event.bytes, 14).unwrap_or(event.timestamp_us), + )); + } + INPUT_MOUSE_ABSOLUTE => { + require_len(event.bytes, 26, "short absolute mouse packet")?; + let body = [ + event.bytes[4], + event.bytes[5], + event.bytes[6], + event.bytes[7], + 0x08, + 0x00, + event.bytes[10], + event.bytes[11], + event.bytes[12], + event.bytes[13], + ]; + encoded.push(remote_input_message( + remote_input_packet(input_type, &body), + read_u64_be(event.bytes, 18).unwrap_or(event.timestamp_us), + )); + } + INPUT_MOUSE_BUTTON_DOWN | INPUT_MOUSE_BUTTON_UP => { + require_len(event.bytes, 18, "short mouse button packet")?; + encoded.push(remote_input_message( + remote_input_packet(input_type, &[event.bytes[4], 0]), + read_u64_be(event.bytes, 10).unwrap_or(event.timestamp_us), + )); + } + INPUT_MOUSE_WHEEL => { + require_len(event.bytes, 22, "short mouse wheel packet")?; + let body = [ + event.bytes[4], + event.bytes[5], + event.bytes[6], + event.bytes[7], + 0, + 0, + ]; + encoded.push(remote_input_message( + remote_input_packet(input_type, &body), + read_u64_be(event.bytes, 14).unwrap_or(event.timestamp_us), + )); + } + INPUT_GAMEPAD => { + require_len(event.bytes, 38, "short gamepad packet")?; + let timestamp = read_u64_le(event.bytes, 30).unwrap_or(event.timestamp_us); + let controller_id = usize::from( + read_u16_le(event.bytes, 6) + .ok_or(NvstInputCodecError::Malformed("missing gamepad id"))?, + ); + if controller_id >= self.gamepad_sequences.len() { + return Err(NvstInputCodecError::Malformed( + "gamepad id is outside the supported range", + )); + } + let bitmap = read_u16_le(event.bytes, 8).expect("gamepad length checked"); + self.last_gamepad_report = Some( + event.bytes[..38] + .try_into() + .expect("gamepad length checked"), + ); + let filtered = bitmap & !self.rich_slot_mask; + if self.gamepad_bitmap != Some(filtered) { + let previous_bitmap = self.gamepad_bitmap.unwrap_or_default(); + self.emit_topology_transition( + previous_bitmap, + filtered, + timestamp, + &mut encoded, + ); + } + if filtered & (1 << controller_id) == 0 { + continue; + } + self.gamepad_sequences[controller_id] = + self.gamepad_sequences[controller_id].wrapping_add(1); + encoded.push(NvstEncodedInput { + route: NvstInputRoute::InputPartial, + bytes: gamepad_command( + event.bytes, + timestamp, + self.gamepad_sequences[controller_id], + ), + meaningful: true, + }); + } + INPUT_HAPTICS_ENABLED => { + require_len(event.bytes, 6, "short haptics packet")?; + encoded.push(haptics_state( + read_u16_be(event.bytes, 4).unwrap_or_default() != 0, + event.timestamp_us, + )); + } + INPUT_LOCK_KEYS_SYNC => { + require_len(event.bytes, 5, "short lock-key sync packet")?; + encoded.push(remote_input_message( + remote_input_packet(input_type, &[event.bytes[4]]), + event.timestamp_us, + )); + } + INPUT_TEXT => encoded.extend(text_messages(&event)?), + other => return Err(NvstInputCodecError::UnsupportedType(other)), + } + } + Ok(encoded) + } +} + +#[derive(Debug, Clone, Copy)] +struct NativeEvent<'a> { + bytes: &'a [u8], + timestamp_us: u64, +} + +fn native_events( + packet: &[u8], + fallback_timestamp_us: u64, +) -> Result>, NvstInputCodecError> { + if packet.is_empty() { + return Err(NvstInputCodecError::Malformed("empty packet")); + } + if packet[0] == 0x22 { + if packet.len() < 5 || read_u32_le(packet, 1) != Some(INPUT_TEXT) { + return Err(NvstInputCodecError::Malformed( + "unknown leading 0x22 packet", + )); + } + return Ok(vec![NativeEvent { + bytes: &packet[1..], + timestamp_us: fallback_timestamp_us, + }]); + } + if packet[0] != 0x23 { + return Ok(vec![NativeEvent { + bytes: packet, + timestamp_us: fallback_timestamp_us, + }]); + } + require_len(packet, 10, "short protocol-v3 wrapper")?; + let timestamp_us = read_u64_be(packet, 1).expect("wrapper length checked"); + let mut cursor = 9; + let mut events = Vec::new(); + while cursor < packet.len() { + match packet[cursor] { + 0x22 => { + let start = cursor + 1; + let input_type = read_u32_le(packet, start) + .ok_or(NvstInputCodecError::Malformed("short single-event wrapper"))?; + let length = native_event_length(input_type, packet.len() - start)?; + let end = start + length; + if end > packet.len() { + return Err(NvstInputCodecError::Malformed("truncated single event")); + } + events.push(NativeEvent { + bytes: &packet[start..end], + timestamp_us, + }); + cursor = end; + } + 0x21 => { + require_remaining(packet, cursor, 3, "short length-prefixed wrapper")?; + let length = + usize::from(u16::from_be_bytes([packet[cursor + 1], packet[cursor + 2]])); + let start = cursor + 3; + let end = start + length; + if end > packet.len() { + return Err(NvstInputCodecError::Malformed( + "truncated length-prefixed event", + )); + } + events.push(NativeEvent { + bytes: &packet[start..end], + timestamp_us, + }); + cursor = end; + } + 0x26 => { + require_remaining(packet, cursor, 7, "short partially-reliable wrapper")?; + if packet[cursor + 4] != 0x21 { + return Err(NvstInputCodecError::Malformed( + "missing gamepad payload marker", + )); + } + let length = + usize::from(u16::from_be_bytes([packet[cursor + 5], packet[cursor + 6]])); + let start = cursor + 7; + let end = start + length; + if end > packet.len() { + return Err(NvstInputCodecError::Malformed("truncated gamepad event")); + } + events.push(NativeEvent { + bytes: &packet[start..end], + timestamp_us, + }); + cursor = end; + } + _ => return Err(NvstInputCodecError::Malformed("unknown protocol-v3 marker")), + } + } + Ok(events) +} + +fn native_event_length(input_type: u32, remaining: usize) -> Result { + match input_type { + INPUT_HEARTBEAT => Ok(4), + INPUT_KEY_DOWN | INPUT_KEY_UP | INPUT_MOUSE_BUTTON_DOWN | INPUT_MOUSE_BUTTON_UP => Ok(18), + INPUT_MOUSE_RELATIVE | INPUT_MOUSE_WHEEL => Ok(22), + INPUT_MOUSE_ABSOLUTE => Ok(26), + INPUT_GAMEPAD => Ok(38), + INPUT_HAPTICS_ENABLED => Ok(6), + INPUT_LOCK_KEYS_SYNC => Ok(5), + INPUT_TEXT => Ok(remaining), + other => Err(NvstInputCodecError::UnsupportedType(other)), + } +} + +fn remote_input_message(inner: Vec, timestamp_us: u64) -> NvstEncodedInput { + let input_type = read_u32_le(&inner, 4).expect("remote input packet has type"); + let mut body = inner; + let padded = 24.max(body.len().div_ceil(8) * 8); + body.resize(padded, 0); + let timestamp_count = usize::from(matches!( + input_type, + INPUT_KEY_DOWN | INPUT_KEY_UP | INPUT_MOUSE_ABSOLUTE + )) + 1; + for _ in 0..timestamp_count { + body.extend_from_slice(×tamp_us.to_le_bytes()); + } + NvstEncodedInput { + route: if matches!(input_type, INPUT_MOUSE_RELATIVE | INPUT_MOUSE_ABSOLUTE) { + NvstInputRoute::ControlPartial + } else { + NvstInputRoute::ControlReliable + }, + bytes: control_command(COMMAND_REMOTE_INPUT, &remote_input_packet(0x0e, &body)), + meaningful: matches!( + input_type, + INPUT_KEY_DOWN + | INPUT_KEY_UP + | INPUT_MOUSE_ABSOLUTE + | INPUT_MOUSE_RELATIVE + | INPUT_MOUSE_BUTTON_DOWN + | INPUT_MOUSE_BUTTON_UP + | INPUT_MOUSE_WHEEL + | INPUT_TEXT + ), + } +} + +fn remote_input_packet(input_type: u32, body: &[u8]) -> Vec { + let mut packet = Vec::with_capacity(8 + body.len()); + packet.extend_from_slice(&(4_u32 + body.len() as u32).to_be_bytes()); + packet.extend_from_slice(&input_type.to_le_bytes()); + packet.extend_from_slice(body); + packet +} + +pub(crate) fn unicode_text_messages( + text: &opennow_streamer_protocol::text_input::UnicodeText, + timestamp_us: u64, +) -> Vec { + let mut remaining = text.as_str(); + let mut messages = Vec::new(); + while !remaining.is_empty() { + let mut end = remaining.len().min(0x3f8); + while !remaining.is_char_boundary(end) { + end -= 1; + } + let mut packet = remote_input_packet(INPUT_TEXT, &remaining.as_bytes()[..end]); + if packet.len() < 0x7e * 8 { + packet.resize(((packet.len() + 8) / 8) * 8 + 8, 0); + packet.extend_from_slice(×tamp_us.to_le_bytes()); + packet = remote_input_packet(0x0e, &packet); + } + messages.push(NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: control_command(COMMAND_REMOTE_INPUT, &packet), + meaningful: true, + }); + remaining = &remaining[end..]; + } + messages +} + +fn gamepad_command(packet: &[u8], timestamp_us: u64, sequence: u16) -> Vec { + let mut payload = Vec::with_capacity(54); + payload.push(0x23); + payload.extend_from_slice(×tamp_us.to_be_bytes()); + payload.extend_from_slice(&[0x26, packet[6]]); + payload.extend_from_slice(&sequence.to_be_bytes()); + payload.push(0x21); + payload.extend_from_slice(&38_u16.to_be_bytes()); + payload.extend_from_slice(&packet[..38]); + control_command(COMMAND_GAMEPAD, &payload) +} + +pub(crate) const RI_TYPE_DEVICE_CHANGE: u32 = 0x12; +pub(crate) const RI_TYPE_HID_REPORT: u32 = 0x11; +pub(crate) const HID_REPORT_MARKER: u8 = 0x22; +pub(crate) const SONY_DEVICE_CHANGE_FLAGS: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SonyDeviceControl { + Attach, + Removal, +} + +impl SonyDeviceControl { + const fn code(self) -> u8 { + match self { + Self::Attach => 1, + Self::Removal => 3, + } + } +} + +pub(crate) fn sony_low_id_in_range(low_id: u8) -> Option<()> { + let offset = low_id.checked_sub(opennow_streamer_hid::ds4::DS4_LOW_ID_BASE)?; + (usize::from(offset) < opennow_streamer_hid::MAX_SOURCES).then_some(()) +} + +pub(crate) fn sony_device_change_command( + control: SonyDeviceControl, + low_id: u8, +) -> Option> { + sony_low_id_in_range(low_id)?; + let mut body = Vec::with_capacity(14); + body.push(low_id); + body.push(control.code()); + body.extend_from_slice(&SONY_DEVICE_CHANGE_FLAGS.to_be_bytes()); + body.extend_from_slice(&0_u16.to_be_bytes()); + body.extend_from_slice(&0_u16.to_be_bytes()); + body.extend_from_slice(&0_u16.to_be_bytes()); + body.extend_from_slice(&0_u16.to_be_bytes()); + Some(control_command( + COMMAND_REMOTE_INPUT, + &remote_input_packet(RI_TYPE_DEVICE_CHANGE, &body), + )) +} + +pub(crate) fn sony_report_command( + low_id: u8, + report: &[u8; opennow_streamer_hid::ds4::DS4_REPORT_BYTES], +) -> Option> { + sony_low_id_in_range(low_id)?; + let mut payload = Vec::with_capacity(72); + payload.push(HID_REPORT_MARKER); + payload.extend_from_slice(&RI_TYPE_HID_REPORT.to_le_bytes()); + payload.push(low_id); + payload.push(opennow_streamer_hid::ds4::DS4_INTERFACE); + payload.push(0); + payload.extend_from_slice(report); + Some(control_command(COMMAND_GAMEPAD, &payload)) +} + +pub(crate) fn parse_sony_output(payload: &[u8]) -> Option<(u8, &[u8])> { + if payload.len() < 13 { + return None; + } + if u32::from_le_bytes(payload[0..4].try_into().ok()?) != RI_TYPE_HID_REPORT { + return None; + } + let low_id = payload[4]; + sony_low_id_in_range(low_id)?; + Some((low_id, &payload[7..])) +} + +pub(crate) fn for_each_sony_output(mut bytes: &[u8], mut handle: impl FnMut(u8, &[u8])) { + while bytes.len() >= 4 { + let code = u16::from_le_bytes([bytes[0], bytes[1]]); + let length = usize::from(u16::from_le_bytes([bytes[2], bytes[3]])); + let Some(payload) = bytes.get(4..4 + length) else { + return; + }; + bytes = &bytes[4 + length..]; + if code != COMMAND_REMOTE_INPUT { + continue; + } + if let Some((low_id, operation)) = parse_sony_output(payload) { + handle(low_id, operation); + } + } +} + +pub(crate) fn generic_hid_command( + outbound: &opennow_streamer_hid::generic::GenericOutbound, +) -> Option> { + use opennow_streamer_hid::generic::{GenericOutbound, MAX_REPORT_BYTES}; + + let (id, control, identity) = match outbound { + GenericOutbound::Attach { request_id, device } => ( + *request_id, + 1, + [device.vendor, device.product, device.revision, 0], + ), + GenericOutbound::Detach { device_id } => (*device_id, 3, [0; 4]), + GenericOutbound::Report { + device_id, + interface, + operation, + error, + bytes, + .. + } => { + if !(10..=254).contains(device_id) + || *operation > 4 + || bytes.len() > MAX_REPORT_BYTES + || (bytes.is_empty() && !(matches!(*operation, 1..=4) && *error)) + || (*operation == 0 && *error) + { + return None; + } + let mut payload = Vec::with_capacity(8 + bytes.len()); + payload.push(HID_REPORT_MARKER); + payload.extend_from_slice(&RI_TYPE_HID_REPORT.to_le_bytes()); + payload.push(*device_id); + payload.push(if *operation == 0 { + *interface + } else { + u8::from(*error) + }); + payload.push(*operation); + payload.extend_from_slice(bytes); + return Some(control_command(COMMAND_GAMEPAD, &payload)); + } + }; + if !(10..=254).contains(&id) { + return None; + } + let mut body = Vec::with_capacity(14); + body.extend_from_slice(&[id, control]); + body.extend_from_slice(&0xf_u32.to_be_bytes()); + for word in identity { + body.extend_from_slice(&word.to_be_bytes()); + } + Some(control_command( + COMMAND_REMOTE_INPUT, + &remote_input_packet(RI_TYPE_DEVICE_CHANGE, &body), + )) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GenericHidMessage<'a> { + Response { + request_id: u8, + control: u8, + opaque: u32, + device_id: u8, + }, + Output { + device_id: u8, + interface: u8, + operation: u8, + bytes: &'a [u8], + }, +} + +fn parse_generic_hid_message(payload: &[u8]) -> Option> { + use opennow_streamer_hid::generic::MAX_REPORT_BYTES; + + match u32::from_le_bytes(payload.get(..4)?.try_into().ok()?) { + 0x1a if payload.len() == 11 => Some(GenericHidMessage::Response { + request_id: payload[4], + control: payload[5], + opaque: u32::from_be_bytes(payload[6..10].try_into().ok()?), + device_id: payload[10], + }), + RI_TYPE_HID_REPORT if (7..=7 + MAX_REPORT_BYTES).contains(&payload.len()) => { + let device_id = payload[4]; + let operation = payload[6]; + let bytes = &payload[7..]; + if !(10..=254).contains(&device_id) || operation > 4 { + return None; + } + if matches!(operation, 2 | 3) + && (bytes.len() != 3 + || !(1..=MAX_REPORT_BYTES) + .contains(&usize::from(u16::from_le_bytes([bytes[1], bytes[2]])))) + { + return None; + } + Some(GenericHidMessage::Output { + device_id, + interface: payload[5], + operation, + bytes, + }) + } + _ => None, + } +} + +pub(crate) fn for_each_generic_hid_message<'a>( + mut bytes: &'a [u8], + mut handle: impl FnMut(GenericHidMessage<'a>), +) { + while bytes.len() >= 4 { + let code = u16::from_le_bytes([bytes[0], bytes[1]]); + let length = usize::from(u16::from_le_bytes([bytes[2], bytes[3]])); + let Some(payload) = bytes.get(4..4 + length) else { + return; + }; + bytes = &bytes[4 + length..]; + if code == COMMAND_REMOTE_INPUT + && let Some(message) = parse_generic_hid_message(payload) + { + handle(message); + } + } +} + +fn text_messages(event: &NativeEvent<'_>) -> Result, NvstInputCodecError> { + require_len(event.bytes, 5, "short text packet")?; + let text = + std::str::from_utf8(&event.bytes[5..]).map_err(|_| NvstInputCodecError::UnsupportedText)?; + let mut messages = Vec::with_capacity(text.chars().count() * 2); + for character in text.chars() { + let (virtual_key, modifiers) = + text_keystroke(character).ok_or(NvstInputCodecError::UnsupportedText)?; + for input_type in [INPUT_KEY_DOWN, INPUT_KEY_UP] { + let mut body = Vec::with_capacity(14); + body.extend_from_slice(&virtual_key.to_be_bytes()); + body.extend_from_slice(&modifiers.to_be_bytes()); + body.resize(14, 0); + messages.push(remote_input_message( + remote_input_packet(input_type, &body), + event.timestamp_us, + )); + } + } + Ok(messages) +} + +fn text_keystroke(character: char) -> Option<(u16, u16)> { + let virtual_key = match character { + 'A'..='Z' => return Some((character as u16, 1)), + 'a'..='z' => return Some((character.to_ascii_uppercase() as u16, 0)), + '0'..='9' => return Some((character as u16, 0)), + ' ' => return Some((0x20, 0)), + '\t' => return Some((0x09, 0)), + '\n' | '\r' => return Some((0x0d, 0)), + '!' => 0x31, + '@' => 0x32, + '#' => 0x33, + '$' => 0x34, + '%' => 0x35, + '^' => 0x36, + '&' => 0x37, + '*' => 0x38, + '(' => 0x39, + ')' => 0x30, + '_' => 0xbd, + '+' => 0xbb, + '{' => 0xdb, + '}' => 0xdd, + '|' => 0xdc, + ':' => 0xba, + '"' => 0xde, + '<' => 0xbc, + '>' => 0xbe, + '?' => 0xbf, + '~' => 0xc0, + '-' => return Some((0xbd, 0)), + '=' => return Some((0xbb, 0)), + '[' => return Some((0xdb, 0)), + ']' => return Some((0xdd, 0)), + '\\' => return Some((0xdc, 0)), + ';' => return Some((0xba, 0)), + '\'' => return Some((0xde, 0)), + ',' => return Some((0xbc, 0)), + '.' => return Some((0xbe, 0)), + '/' => return Some((0xbf, 0)), + '`' => return Some((0xc0, 0)), + _ => return None, + }; + Some((virtual_key, 1)) +} + +fn activation_chain(timestamp_us: u64) -> [Vec; 8] { + [ + enable_input(1, false), + device_descriptor(timestamp_us, 0), + mouse_cursor_capture(true), + remote_cursor_tracking(true), + state_change(COMMAND_WINDOW_STATE, 19, 0), + state_change(COMMAND_SYSTEM_STATE, 0, 0), + enable_input(1, true), + haptics_state(true, timestamp_us).bytes, + ] +} + +fn mouse_cursor_capture(enabled: bool) -> Vec { + control_command(COMMAND_MOUSE_CURSOR_CAPTURE, &[u8::from(enabled)]) +} + +fn remote_cursor_tracking(enabled: bool) -> Vec { + control_command(COMMAND_TRACK_REMOTE_CURSOR_IMAGE, &[u8::from(enabled)]) +} + +fn haptics_state(enabled: bool, timestamp_us: u64) -> NvstEncodedInput { + remote_input_message( + remote_input_packet(INPUT_HAPTICS_ENABLED, &u16::from(enabled).to_le_bytes()), + timestamp_us, + ) +} + +fn control_keepalive(stream_value: u32) -> Vec { + control_command(COMMAND_KEEPALIVE, &stream_value.to_le_bytes()) +} + +fn enable_input(counter: u32, enabled: bool) -> Vec { + let mut payload = Vec::with_capacity(12); + payload.extend_from_slice(&0_u32.to_le_bytes()); + payload.extend_from_slice(&counter.to_le_bytes()); + payload.extend_from_slice(&u32::from(enabled).to_le_bytes()); + control_command(COMMAND_ENABLE_INPUT, &payload) +} + +fn device_descriptor(timestamp_us: u64, bitmap: u16) -> Vec { + let mut payload = Vec::with_capacity(48); + payload.push(0x23); + payload.extend_from_slice(×tamp_us.to_be_bytes()); + let mut body = [ + 0x22, 0x0c, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x02, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + body[9..11].copy_from_slice(&bitmap.to_le_bytes()); + payload.extend_from_slice(&body); + control_command(COMMAND_GAMEPAD, &payload) +} + +fn state_change(code: u16, state: u32, frame_number: u32) -> Vec { + // Bifrost serializes these as three little-endian u32 fields: stream + // index, requested state, and frame number. The desktop client announces + // window state 19 at frame zero once input is activated. Advertising the + // previous all-zero window state leaves the session looking inactive and + // prevents the server from sending its system-cursor mode updates. + let mut payload = Vec::with_capacity(12); + payload.extend_from_slice(&0_u32.to_le_bytes()); + payload.extend_from_slice(&state.to_le_bytes()); + payload.extend_from_slice(&frame_number.to_le_bytes()); + control_command(code, &payload) +} + +fn control_command(code: u16, payload: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(4 + payload.len()); + bytes.extend_from_slice(&code.to_le_bytes()); + bytes.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + bytes.extend_from_slice(payload); + bytes +} + +fn require_len( + bytes: &[u8], + minimum: usize, + reason: &'static str, +) -> Result<(), NvstInputCodecError> { + if bytes.len() < minimum { + return Err(NvstInputCodecError::Malformed(reason)); + } + Ok(()) +} + +fn require_remaining( + bytes: &[u8], + offset: usize, + minimum: usize, + reason: &'static str, +) -> Result<(), NvstInputCodecError> { + if bytes.len().saturating_sub(offset) < minimum { + return Err(NvstInputCodecError::Malformed(reason)); + } + Ok(()) +} + +fn read_u16_be(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_be_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u16_le(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_le_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32_le(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64_be(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_be_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn read_u64_le(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_le_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unicode_text_uses_verified_type_23_bytes_and_timestamp_envelope() { + let text = opennow_streamer_protocol::text_input::TextInputSlot::default() + .submit("é世🦫".as_bytes()) + .unwrap(); + let messages = unicode_text_messages(&text, 0x0102030405060708); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].route, NvstInputRoute::ControlReliable); + assert_eq!( + messages[0].bytes, + hex(concat!( + "06023000", + "0000002c0e000000", + "0000000d17000000", + "c3a9e4b896f09fa6ab", + "000000000000000000000000000000", + "0807060504030201" + )) + ); + } + + #[test] + fn unicode_text_chunk_boundaries_preserve_every_utf8_byte() { + use opennow_streamer_protocol::text_input::{MAX_TEXT_BYTES, TextInputSlot}; + for length in [1, 991, 992, 999, 1000, 1015, 1016, 1017, MAX_TEXT_BYTES] { + let source = "a".repeat(length); + let text = TextInputSlot::default().submit(source.as_bytes()).unwrap(); + let messages = unicode_text_messages(&text, 7); + let mut reconstructed = Vec::new(); + for message in &messages { + assert_eq!(message.route, NvstInputRoute::ControlReliable); + assert_eq!(&message.bytes[..2], &[6, 2]); + assert_eq!( + usize::from(u16::from_le_bytes(message.bytes[2..4].try_into().unwrap())), + message.bytes.len() - 4 + ); + let mut packet = &message.bytes[4..]; + if read_u32_le(packet, 4) == Some(14) { + assert_eq!(&packet[packet.len() - 8..], &7_u64.to_le_bytes()); + packet = &packet[8..]; + } + assert_eq!(read_u32_le(packet, 4), Some(23)); + let body_len = u32::from_be_bytes(packet[..4].try_into().unwrap()) as usize - 4; + assert!(body_len <= 1016); + assert_eq!( + read_u32_le(&message.bytes[4..], 4) == Some(14), + body_len < 1000 + ); + let body = &packet[8..8 + body_len]; + assert!(std::str::from_utf8(body).is_ok()); + reconstructed.extend_from_slice(body); + } + assert_eq!(reconstructed, source.as_bytes()); + } + for prefix in 1013..=1016 { + for suffix in ["é", "世", "🦫"] { + let source = format!("{}{suffix}tail", "x".repeat(prefix)); + let text = TextInputSlot::default().submit(source.as_bytes()).unwrap(); + let messages = unicode_text_messages(&text, 0); + assert_eq!(messages.len(), 2); + let first = &messages[0].bytes[4..]; + let first_len = u32::from_be_bytes(first[..4].try_into().unwrap()) as usize - 4; + assert!(source.is_char_boundary(first_len)); + assert!(first_len <= 1016); + let mut reconstructed = first[8..8 + first_len].to_vec(); + let tail = &messages[1].bytes[12..]; + let tail_len = u32::from_be_bytes(tail[..4].try_into().unwrap()) as usize - 4; + reconstructed.extend_from_slice(&tail[8..8 + tail_len]); + assert_eq!(reconstructed, source.as_bytes()); + } + } + } + + #[test] + fn unicode_text_batch_admission_rejects_whole_batch_before_capacity_overflow() { + assert!(text_batch_fits(0, 70_000, TEXT_BATCH_GUARD)); + assert!(text_batch_fits( + 128 * 1024 - 70_000, + 70_000, + TEXT_BATCH_GUARD + )); + assert!(!text_batch_fits( + 128 * 1024 - 70_000 + 1, + 70_000, + TEXT_BATCH_GUARD + )); + assert!(!text_batch_fits(usize::MAX, 70_000, TEXT_BATCH_GUARD)); + } + + #[test] + fn text_admission_is_normal_traffic_bounded_by_the_shared_budget() { + let guard = crate::nvst_budget::TEXT_BATCH_GUARD; + assert!(guard <= crate::nvst_budget::TRANSPORT_AGGREGATE_CAPACITY); + assert!(guard > crate::nvst_budget::NORMAL_BUDGET); + assert_eq!( + crate::nvst_budget::admit_write(guard, 0, WriteClass::Normal), + crate::nvst_budget::WriteAdmission::EmptyWrite + ); + assert_eq!( + crate::nvst_budget::admit_write(0, guard, WriteClass::Normal), + crate::nvst_budget::WriteAdmission::OverBudget + ); + } + + fn hex(value: &str) -> Vec { + assert_eq!(value.len() % 2, 0, "hex input must contain whole bytes"); + value + .as_bytes() + .chunks_exact(2) + .map(|pair| { + u8::from_str_radix(std::str::from_utf8(pair).expect("hex utf8"), 16) + .expect("hex byte") + }) + .collect() + } + + #[test] + fn channel_profile_matches_official_sids_labels_and_reliability() { + assert_eq!( + NVST_CHANNEL_PROFILE.map(|definition| definition.sid), + [0, 2, 4, 6, 8, 10, 12, 14] + ); + assert_eq!( + NVST_CHANNEL_PROFILE.map(|definition| definition.label), + [ + CONTROL_RELIABLE_LABEL, + CUSTOM_RELIABLE_LABEL, + CUSTOM_PARTIAL_LABEL, + CONTROL_PARTIAL_LABEL, + CONTROL_UNRELIABLE_LABEL, + INPUT_PARTIAL_LABEL, + CURSOR_LABEL, + RTCP_ON_SCTP_LABEL, + ] + ); + assert_eq!( + NVST_CHANNEL_PROFILE.map(|definition| definition.ordered), + [true, true, true, false, false, false, true, true] + ); + assert_eq!( + NVST_CHANNEL_PROFILE.map(|definition| definition.reliability), + [ + NvstChannelReliability::Reliable, + NvstChannelReliability::Reliable, + NvstChannelReliability::MaxRetransmits(2), + NvstChannelReliability::Lifetime(300), + NvstChannelReliability::MaxRetransmits(0), + NvstChannelReliability::Lifetime(300), + NvstChannelReliability::Reliable, + NvstChannelReliability::Reliable, + ] + ); + let configs = NVST_CHANNEL_PROFILE.map(channel_config); + assert_eq!( + configs.clone().map(|config| config.ordered), + [true, true, true, false, false, false, true, true] + ); + assert_eq!( + configs.map(|config| config.reliability), + [ + Reliability::Reliable, + Reliability::Reliable, + Reliability::MaxRetransmits { retransmits: 2 }, + Reliability::MaxPacketLifetime { lifetime: 300 }, + Reliability::MaxRetransmits { retransmits: 0 }, + Reliability::MaxPacketLifetime { lifetime: 300 }, + Reliability::Reliable, + Reliability::Reliable, + ] + ); + + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, true); + let rtcp = channels.rtcp.expect("negotiated RTCP channel"); + assert_eq!(channels.label(rtcp), RTCP_ON_SCTP_LABEL); + assert!(!channels.contains(rtcp)); + } + + #[test] + fn custom_channel_retransmits_while_mouse_motion_stays_unordered() { + let custom = channel_config(NVST_CHANNEL_PROFILE[2]); + assert!(custom.ordered); + assert_eq!( + custom.reliability, + Reliability::MaxRetransmits { retransmits: 2 } + ); + + let motion_channel = channel_config(NVST_CHANNEL_PROFILE[3]); + assert!(!motion_channel.ordered); + assert_eq!( + motion_channel.reliability, + Reliability::MaxPacketLifetime { lifetime: 300 } + ); + let mut motion = [0; 22]; + motion[..4].copy_from_slice(&INPUT_MOUSE_RELATIVE.to_le_bytes()); + let encoded = NvstInputCodec::default().encode(&motion, 0).unwrap(); + assert_eq!(encoded[0].route, NvstInputRoute::ControlPartial); + } + + #[test] + fn optional_channels_follow_video_route_without_dropping_bundle_feedback() { + for (bundle_video, expected_count) in [(false, 6), (true, 8)] { + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, bundle_video); + assert_eq!(channels.all().count(), expected_count); + assert_eq!(channels.cursor.is_some(), bundle_video); + assert_eq!(channels.rtcp.is_some(), bundle_video); + for id in channels.all() { + assert_eq!(channels.contains(id), Some(id) != channels.rtcp); + assert_ne!(channels.label(id), "unknown"); + } + if let Some(cursor) = channels.cursor { + assert_eq!(channels.label(cursor), CURSOR_LABEL); + } + if let Some(rtcp) = channels.rtcp { + assert_eq!(channels.label(rtcp), RTCP_ON_SCTP_LABEL); + assert!(channels.all().any(|id| id == rtcp)); + assert!(!channels.contains(rtcp)); + } else { + assert!(!channels.send_rtcp(&mut rtc, &[0x80, 0xc9])); + } + } + } + + #[test] + fn optional_channel_closure_preserves_input_until_control_shutdown() { + for bundle_video in [false, true] { + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, bundle_video); + let mut state = NvstInputChannelState::default(); + state.channel_opened(channels, channels.control_reliable); + state.channel_data( + channels, + channels.control_reliable, + &[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00], + ); + assert!(state.is_ready()); + for id in [channels.cursor, channels.rtcp].into_iter().flatten() { + assert!(!state.channel_closed(channels, id)); + assert!(state.is_ready()); + } + assert!(state.channel_closed(channels, channels.control_reliable)); + assert!(!state.is_ready()); + assert!(!state.channel_closed(channels, channels.control_reliable)); + } + } + + #[test] + fn parses_full_control_handshake_and_requires_only_control_plus_version() { + assert_eq!( + input_protocol_version(&[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00]), + Some(3) + ); + assert_eq!(input_protocol_version(&[0x0e, 0x02, 0x03, 0x00]), Some(3)); + assert_eq!(input_protocol_version(&[0x0e, 0x03]), Some(0x030e)); + assert_eq!(input_protocol_version(&[0x0e, 0x02]), Some(2)); + assert_eq!(input_protocol_version(&[1]), None); + assert_eq!( + input_protocol_version(&[0x01, 0x01, 0x00, 0x00, 0x0e, 0x02, 0x02, 0x00, 0x03, 0x00,]), + Some(3) + ); + + let mut rtc = Rtc::new(Instant::now()); + let channels = NvstInputChannels::create(&mut rtc, false); + let mut state = NvstInputChannelState::default(); + assert_eq!( + state.channel_opened(channels, channels.control_reliable), + None + ); + assert!(!state.is_ready()); + assert_eq!( + state.channel_data( + channels, + channels.input_partial, + &[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00], + ), + None + ); + assert!(!state.is_ready()); + assert_eq!( + state.channel_data( + channels, + channels.control_reliable, + &[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00], + ), + Some(3) + ); + assert!(state.is_ready()); + } + + #[test] + fn extracts_system_cursor_notifications_from_server_control() { + let visible = hex("0f010900010000000c80168001"); + assert_eq!( + server_cursor_updates(&visible), + vec![hex("000100000000000c801680")] + ); + + let mut concatenated = visible; + concatenated.extend_from_slice(&hex("0f0109000c0000000000000000")); + assert_eq!( + server_cursor_updates(&concatenated), + vec![hex("000100000000000c801680"), hex("000c000000000000000000")] + ); + + assert_eq!( + server_cursor_updates(&hex("0f010800010000000c801680")), + vec![hex("000100000000000c801680")] + ); + assert_eq!( + server_cursor_updates(&hex("0f01040000000000")), + vec![hex("00000000000000")] + ); + assert!(server_cursor_updates(&hex("0f010400010000")).is_empty()); + assert!(server_cursor_updates(&hex("0a0102007b7d")).is_empty()); + } + + #[test] + fn control_keepalive_and_activation_are_byte_exact() { + assert_eq!(control_keepalive(0), hex("0002040000000000")); + assert_eq!(mouse_cursor_capture(false), hex("0803010000")); + assert_eq!(mouse_cursor_capture(true), hex("0803010001")); + assert_eq!(remote_cursor_tracking(false), hex("0d03010000")); + assert_eq!(remote_cursor_tracking(true), hex("0d03010001")); + let now = Instant::now(); + assert_eq!( + next_control_keepalive(now).duration_since(now), + Duration::from_secs(3) + ); + + let chain = activation_chain(20_102_193); + assert_eq!(chain[0], hex("0b020c00000000000100000000000000")); + assert_eq!( + chain[1], + hex( + "0d02300023000000000132bc31220c0000001a000000000014000000000000000000000000000000550000000000000000000000" + ) + ); + assert_eq!(chain[2], hex("0803010001")); + assert_eq!(chain[3], hex("0d03010001")); + assert_eq!(chain[4], hex("20030c00000000001300000000000000")); + assert_eq!(chain[5], hex("21030c00000000000000000000000000")); + assert_eq!(chain[6], hex("0b020c00000000000100000001000000")); + assert_eq!( + chain[7], + hex( + "06022800000000240e000000000000060d0000000100000000000000000000000000000031bc320100000000" + ) + ); + } + + #[test] + fn haptic_motor_values_are_not_scanned_as_cursor_commands() { + let bytes = hex("0b010c000200080000000f010400fa000f01040001000000"); + let cursors = server_cursor_messages(&bytes); + assert_eq!(cursors.len(), 1); + assert_eq!(cursors[0].offset, 16); + assert_eq!(cursors[0].cursor_id, Some(1)); + } + + #[test] + fn unrelated_control_payloads_cannot_toggle_cursor_lock() { + for code in [0x010a, 0x0111, 0x0200, 0xffff] { + let mut bytes = control_command(COMMAND_SYSTEM_CURSOR, &0_u32.to_le_bytes()); + let unrelated = control_command( + code, + &hex("0f010400010000000f01040000000000100108000000000000000000"), + ); + bytes.extend_from_slice(&unrelated); + let visible_offset = bytes.len(); + bytes.extend_from_slice(&control_command( + COMMAND_SYSTEM_CURSOR, + &12_u32.to_le_bytes(), + )); + + let cursors = server_cursor_messages(&bytes); + assert_eq!(cursors.len(), 2, "outer command {code:#06x}"); + assert_eq!(cursors[0].offset, 0); + assert_eq!(cursors[0].normalized, Some(hex("00000000000000"))); + assert_eq!(cursors[1].offset, visible_offset); + assert_eq!(cursors[1].normalized, Some(hex("000c0000000000"))); + } + } + + #[test] + fn truncated_control_payloads_cannot_invent_cursor_notifications() { + for code in [0x010a, COMMAND_SYSTEM_CURSOR, COMMAND_BITMAP_CURSOR] { + let mut bytes = control_command(COMMAND_SYSTEM_CURSOR, &0_u32.to_le_bytes()); + bytes.extend_from_slice(&code.to_le_bytes()); + bytes.extend_from_slice(&100_u16.to_le_bytes()); + bytes.extend_from_slice(&hex("0f01040001000000100108000000000000000000")); + + let cursors = server_cursor_messages(&bytes); + assert_eq!(cursors.len(), 1, "truncated command {code:#06x}"); + assert_eq!(cursors[0].offset, 0); + assert_eq!(cursors[0].normalized, Some(hex("00000000000000"))); + } + } + + #[test] + fn short_cursor_payloads_do_not_consume_following_commands() { + for (code, minimum_length) in [(COMMAND_SYSTEM_CURSOR, 4), (COMMAND_BITMAP_CURSOR, 8)] { + for length in 0..minimum_length { + let mut bytes = control_command(code, &vec![0x0f; length]); + let visible_offset = bytes.len(); + bytes.extend_from_slice(&control_command( + COMMAND_SYSTEM_CURSOR, + &1_u32.to_le_bytes(), + )); + let cursors = server_cursor_messages(&bytes); + assert_eq!(cursors.len(), 1); + assert_eq!(cursors[0].offset, visible_offset); + assert_eq!(cursors[0].normalized, Some(hex("00010000000000"))); + } + } + } + + #[test] + fn keyboard_and_mouse_translate_to_byte_exact_control_commands() { + let mut codec = NvstInputCodec::default(); + let mut key = vec![0; 18]; + key[..4].copy_from_slice(&INPUT_KEY_DOWN.to_le_bytes()); + key[4..6].copy_from_slice(&0x41_u16.to_be_bytes()); + key[6..8].copy_from_slice(&1_u16.to_be_bytes()); + key[10..18].copy_from_slice(&0x015b_15b1_u64.to_be_bytes()); + let mut wrapped_key = vec![0x23]; + wrapped_key.extend_from_slice(&99_u64.to_be_bytes()); + wrapped_key.push(0x22); + wrapped_key.extend_from_slice(&key); + let key = codec.encode(&wrapped_key, 0).expect("key"); + assert_eq!(key.len(), 1); + assert_eq!(key[0].route, NvstInputRoute::ControlReliable); + assert_eq!( + key[0].bytes, + hex( + "060230000000002c0e000000000000120300000000410001000000000000000000000000b1155b0100000000b1155b0100000000" + ) + ); + + let mut mouse = vec![0; 22]; + mouse[..4].copy_from_slice(&INPUT_MOUSE_RELATIVE.to_le_bytes()); + mouse[4..6].copy_from_slice(&24_i16.to_be_bytes()); + mouse[6..8].copy_from_slice(&24_i16.to_be_bytes()); + mouse[14..22].copy_from_slice(&0x0186_1330_u64.to_be_bytes()); + let mut wrapped_mouse = vec![0x23]; + wrapped_mouse.extend_from_slice(&100_u64.to_be_bytes()); + wrapped_mouse.push(0x21); + wrapped_mouse.extend_from_slice(&(mouse.len() as u16).to_be_bytes()); + wrapped_mouse.extend_from_slice(&mouse); + let mouse = codec.encode(&wrapped_mouse, 0).expect("mouse"); + assert_eq!(mouse[0].route, NvstInputRoute::ControlPartial); + assert_eq!( + mouse[0].bytes, + hex( + "06022800000000240e0000000000000a07000000001800180000000000000000000000003013860100000000" + ) + ); + } + + #[test] + fn standard_gamepad_translates_to_input_partial_and_registers_once() { + let mut codec = NvstInputCodec::default(); + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&INPUT_GAMEPAD.to_le_bytes()); + gamepad[4..6].copy_from_slice(&26_u16.to_le_bytes()); + gamepad[8..10].copy_from_slice(&0x0101_u16.to_le_bytes()); + gamepad[10..12].copy_from_slice(&20_u16.to_le_bytes()); + gamepad[12..14].copy_from_slice(&0x1000_u16.to_le_bytes()); + gamepad[26] = 0x55; + gamepad[30..38].copy_from_slice(&0x015b_171a_u64.to_le_bytes()); + let mut wrapped_gamepad = vec![0x23]; + wrapped_gamepad.extend_from_slice(&101_u64.to_be_bytes()); + wrapped_gamepad.extend_from_slice(&[0x26, 0, 0, 1, 0x21]); + wrapped_gamepad.extend_from_slice(&(gamepad.len() as u16).to_be_bytes()); + wrapped_gamepad.extend_from_slice(&gamepad); + + let first = codec.encode(&wrapped_gamepad, 0).expect("gamepad"); + assert_eq!(first.len(), 2); + assert_eq!(first[0].route, NvstInputRoute::ControlReliable); + assert_eq!(first[1].route, NvstInputRoute::InputPartial); + assert_eq!( + first[1].bytes, + hex( + "0d0236002300000000015b171a260000012100260c0000001a000000010114000010000000000000000000000000550000001a175b0100000000" + ) + ); + + let second = codec.encode(&wrapped_gamepad, 0).expect("second gamepad"); + assert_eq!(second.len(), 1); + assert_eq!(second[0].route, NvstInputRoute::InputPartial); + assert_eq!(second[0].bytes[16], 2); + } + + #[test] + fn gamepads_keep_independent_descriptor_identity_and_sequences() { + let mut codec = NvstInputCodec::default(); + let packet = |controller_id: u16, timestamp: u64| { + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&INPUT_GAMEPAD.to_le_bytes()); + gamepad[6..8].copy_from_slice(&controller_id.to_le_bytes()); + let bitmap = if timestamp == 1 { + 0x0101_u16 + } else { + 0x0303_u16 + }; + gamepad[8..10].copy_from_slice(&bitmap.to_le_bytes()); + gamepad[30..38].copy_from_slice(×tamp.to_le_bytes()); + gamepad + }; + + let first = codec.encode(&packet(0, 1), 0).expect("first pad"); + let second = codec.encode(&packet(1, 2), 0).expect("second pad"); + let first_again = codec.encode(&packet(0, 3), 0).expect("first pad again"); + + assert_eq!(first.len(), 2); + assert_eq!(second.len(), 2); + assert_eq!(first_again.len(), 1); + assert_eq!(&first[0].bytes[22..24], &0x0101_u16.to_le_bytes()); + assert_eq!(&second[0].bytes[22..24], &0x0303_u16.to_le_bytes()); + assert_eq!(first[1].bytes[26], 0); + assert_eq!(second[1].bytes[26], 1); + assert_eq!(first[1].bytes[14], 0); + assert_eq!(second[1].bytes[14], 1); + assert_eq!(first[1].bytes[16], 1); + assert_eq!(second[1].bytes[16], 1); + assert_eq!(first_again[0].bytes[16], 2); + assert_eq!( + codec.encode(&packet(4, 4), 0), + Err(NvstInputCodecError::Malformed( + "gamepad id is outside the supported range" + )) + ); + } + + fn gamepad_fixture(controller_id: u16, bitmap: u16) -> Vec { + let mut packet = + hex("0c0000001a00000001011400015111e7c7cfa05bd08a31750000550000000807060504030201"); + packet[6..8].copy_from_slice(&controller_id.to_le_bytes()); + packet[8..10].copy_from_slice(&bitmap.to_le_bytes()); + packet + } + + #[test] + fn gamepad_envelope_preserves_every_event_field_for_all_four_slots() { + let mut codec = NvstInputCodec::default(); + for id in 0..4_u16 { + let packet = gamepad_fixture(id, 0x0f0f); + assert_eq!(packet.len(), 38); + let messages = codec.encode(&packet, 0).unwrap(); + assert_eq!(messages.len(), if id == 0 { 2 } else { 1 }); + if id == 0 { + assert_eq!(&messages[0].bytes[22..24], &0x0f0f_u16.to_le_bytes()); + } + let state = messages.last().unwrap(); + assert_eq!(state.route, NvstInputRoute::InputPartial); + assert_eq!(&state.bytes[..4], &hex("0d023600")); + assert_eq!(&state.bytes[4..13], &hex("230102030405060708")); + assert_eq!(&state.bytes[13..20], &[0x26, id as u8, 0, 1, 0x21, 0, 38]); + assert_eq!(&state.bytes[20..], packet.as_slice()); + let events = native_events(&state.bytes[4..], 0).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].bytes, packet.as_slice()); + } + } + + #[test] + fn gamepad_sequences_use_all_sixteen_bits_and_wrap_independently() { + let mut codec = NvstInputCodec::default(); + let packet = gamepad_fixture(2, 0x0f0f); + codec.encode(&packet, 0).unwrap(); + for (previous, next) in [(255, 256_u16), (65535, 0), (0, 1)] { + codec.gamepad_sequences[2] = previous; + let messages = codec.encode(&packet, 0).unwrap(); + assert_eq!(messages.len(), 1); + assert_eq!(&messages[0].bytes[15..17], &next.to_be_bytes()); + assert_eq!(codec.gamepad_sequences, [0, 0, next, 0]); + } + } + + #[test] + fn gamepad_disconnect_releases_before_topology_change_and_reconnect_registers() { + let mut codec = NvstInputCodec::default(); + codec.encode(&gamepad_fixture(0, 0x0303), 0).unwrap(); + codec.encode(&gamepad_fixture(1, 0x0303), 0).unwrap(); + + let removed = codec.encode(&gamepad_fixture(1, 0x0101), 0).unwrap(); + assert_eq!(removed.len(), 2); + assert_eq!(removed[0].route, NvstInputRoute::ControlReliable); + assert_eq!(removed[1].route, NvstInputRoute::ControlReliable); + assert_eq!(&removed[0].bytes[..4], &hex("0d023000")); + assert_eq!(removed[0].bytes[13], 0x22); + assert_eq!(&removed[0].bytes[20..24], &hex("01000303")); + assert_eq!(&removed[0].bytes[26..38], &[0; 12]); + assert_eq!(&removed[1].bytes[22..24], &0x0101_u16.to_le_bytes()); + assert_eq!(codec.gamepad_sequences, [1, 0, 0, 0]); + assert!( + codec + .encode(&gamepad_fixture(1, 0x0101), 0) + .unwrap() + .is_empty() + ); + + let reconnected = codec.encode(&gamepad_fixture(1, 0x0303), 0).unwrap(); + assert_eq!(reconnected.len(), 2); + assert_eq!(&reconnected[0].bytes[22..24], &0x0303_u16.to_le_bytes()); + assert_eq!(&reconnected[1].bytes[15..17], &1_u16.to_be_bytes()); + + let empty = codec.encode(&gamepad_fixture(0, 0), 0).unwrap(); + assert_eq!(empty.len(), 3); + assert!( + empty + .iter() + .all(|message| message.route == NvstInputRoute::ControlReliable) + ); + assert_eq!(&empty[2].bytes[22..24], &[0, 0]); + assert_eq!(codec.gamepad_sequences, [0; 4]); + } + + #[test] + fn gamepad_registration_is_replayed_after_send_rollback_or_session_restart() { + let mut codec = NvstInputCodec::default(); + let packet = gamepad_fixture(0, 0x0101); + let checkpoint = codec.clone(); + let attempted = codec.encode(&packet, 0).unwrap(); + codec = checkpoint; + let retried = codec.encode(&packet, 0).unwrap(); + assert_eq!(retried.len(), 2); + assert_eq!(retried[0].bytes, attempted[0].bytes); + assert_eq!(retried[1].bytes, attempted[1].bytes); + assert_eq!(codec.encode(&packet, 0).unwrap().len(), 1); + let restarted = NvstInputCodec::default().encode(&packet, 0).unwrap(); + assert_eq!(restarted.len(), 2); + assert_eq!(restarted[0].bytes, attempted[0].bytes); + assert_eq!(restarted[1].bytes, attempted[1].bytes); + } + + #[test] + fn transport_control_inputs_do_not_disable_native_input() { + let mut codec = NvstInputCodec::default(); + + assert!( + codec + .encode(&INPUT_HEARTBEAT.to_le_bytes(), 0) + .unwrap() + .is_empty() + ); + + let mut haptics = vec![0x23]; + haptics.extend_from_slice(&11_u64.to_be_bytes()); + haptics.push(0x22); + haptics.extend_from_slice(&INPUT_HAPTICS_ENABLED.to_le_bytes()); + haptics.extend_from_slice(&1_u16.to_be_bytes()); + let haptics = codec.encode(&haptics, 0).unwrap(); + assert_eq!(haptics.len(), 1); + assert_eq!(haptics[0].route, NvstInputRoute::ControlReliable); + assert_eq!( + haptics[0].bytes, + hex( + "06022800000000240e000000000000060d000000010000000000000000000000000000000b00000000000000" + ) + ); + + let mut lock_keys = vec![0x23]; + lock_keys.extend_from_slice(&12_u64.to_be_bytes()); + lock_keys.push(0x22); + lock_keys.extend_from_slice(&INPUT_LOCK_KEYS_SYNC.to_le_bytes()); + lock_keys.push(0b011); + let encoded = codec.encode(&lock_keys, 0).expect("lock-key sync"); + assert_eq!(encoded.len(), 1); + assert_eq!(encoded[0].route, NvstInputRoute::ControlReliable); + let lock_keys_type = INPUT_LOCK_KEYS_SYNC.to_le_bytes(); + assert!( + encoded[0] + .bytes + .windows(lock_keys_type.len()) + .any(|bytes| bytes == lock_keys_type) + ); + assert!(encoded[0].bytes.contains(&0b011)); + } + + #[test] + fn haptics_disable_uses_remote_input_envelope_and_preserves_timestamp() { + let mut codec = NvstInputCodec::default(); + let disabled = hex("0d0000000000"); + let expected = hex( + "06022800000000240e000000000000060d000000000000000000000000000000000000000c00000000000000", + ); + let encoded = codec.encode(&disabled, 12).unwrap(); + assert_eq!(encoded.len(), 1); + assert_eq!(encoded[0].route, NvstInputRoute::ControlReliable); + assert_eq!(encoded[0].bytes, expected); + + let mut wrapped = hex("23000000000000000c22"); + wrapped.extend_from_slice(&disabled); + assert_eq!(codec.encode(&wrapped, 999).unwrap(), encoded); + assert!(codec.encode(&disabled[..5], 12).is_err()); + assert!(codec.encode(&wrapped[..wrapped.len() - 1], 999).is_err()); + } + + #[test] + fn unsupported_packets_fail_closed() { + let mut codec = NvstInputCodec::default(); + assert_eq!( + codec.encode(&99_u32.to_le_bytes(), 0), + Err(NvstInputCodecError::UnsupportedType(99)) + ); + let mut text = vec![0x22]; + text.extend_from_slice(&INPUT_TEXT.to_le_bytes()); + text.extend_from_slice("é".as_bytes()); + assert_eq!( + codec.encode(&text, 0), + Err(NvstInputCodecError::UnsupportedText) + ); + } + + #[test] + fn closure_and_timeout_state_are_predictable() { + let now = Instant::now(); + let mut rtc = Rtc::new(now); + let channels = NvstInputChannels::create(&mut rtc, false); + let mut state = NvstInputChannelState::default(); + state.channel_opened(channels, channels.control_reliable); + assert!(!state.handshake_timed_out(Some(now), now + Duration::from_millis(4_999))); + assert!(state.handshake_timed_out(Some(now), now + INPUT_HANDSHAKE_TIMEOUT)); + state.channel_data( + channels, + channels.control_reliable, + &[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00], + ); + assert!(!state.handshake_timed_out(Some(now), now + INPUT_HANDSHAKE_TIMEOUT)); + state.mark_activation_sent(); + assert_eq!( + state.channel_data( + channels, + channels.control_reliable, + &[0x0e, 0x02, 0x02, 0x00, 0x03, 0x00], + ), + None + ); + assert!(state.activation_sent()); + assert!(state.channel_closed(channels, channels.input_partial)); + assert!(state.is_ready()); + assert!(state.channel_closed(channels, channels.control_reliable)); + assert!(!state.is_ready()); + assert!(!state.channel_closed(channels, channels.input_partial)); + assert!(!state.is_ready()); + } + + #[test] + fn sony_device_change_attaches_and_removes_with_the_proved_envelope() { + let attach = sony_device_change_command(SonyDeviceControl::Attach, 6).unwrap(); + let mut expected = Vec::new(); + expected.extend_from_slice(&COMMAND_REMOTE_INPUT.to_le_bytes()); + expected.extend_from_slice(&22_u16.to_le_bytes()); + expected.extend_from_slice(&18_u32.to_be_bytes()); + expected.extend_from_slice(&RI_TYPE_DEVICE_CHANGE.to_le_bytes()); + expected.push(6); + expected.push(1); + expected.extend_from_slice(&SONY_DEVICE_CHANGE_FLAGS.to_be_bytes()); + expected.extend_from_slice(&[0; 8]); + assert_eq!(attach, expected); + assert_eq!(attach.len(), 26); + + let removal = sony_device_change_command(SonyDeviceControl::Removal, 9).unwrap(); + let mut expected = Vec::new(); + expected.extend_from_slice(&COMMAND_REMOTE_INPUT.to_le_bytes()); + expected.extend_from_slice(&22_u16.to_le_bytes()); + expected.extend_from_slice(&18_u32.to_be_bytes()); + expected.extend_from_slice(&RI_TYPE_DEVICE_CHANGE.to_le_bytes()); + expected.push(9); + expected.push(3); + expected.extend_from_slice(&SONY_DEVICE_CHANGE_FLAGS.to_be_bytes()); + expected.extend_from_slice(&[0; 8]); + assert_eq!(removal, expected); + assert_eq!(removal.len(), 26); + } + + #[test] + fn sony_device_change_rejects_ids_outside_the_reserved_domain() { + for low_id in [0, 1, 5, 10, 255] { + assert!(sony_device_change_command(SonyDeviceControl::Attach, low_id).is_none()); + } + for low_id in [6, 7, 8, 9] { + assert!(sony_device_change_command(SonyDeviceControl::Attach, low_id).is_some()); + } + } + + #[test] + fn sony_report_command_matches_the_single_report_wire_shape() { + let mut report = [0_u8; 64]; + report[0] = 0x01; + report[35] = 0x01; + let command = sony_report_command(7, &report).unwrap(); + assert_eq!(&command[0..2], &0x020d_u16.to_le_bytes()); + assert_eq!(&command[2..4], &72_u16.to_le_bytes()); + assert_eq!(command[4], 0x22); + assert_eq!(&command[5..9], &0x11_u32.to_le_bytes()); + assert_eq!(command[9], 7); + assert_eq!(command[10], 4); + assert_eq!(command[11], 0); + assert_eq!(&command[12..76], &report); + assert_eq!(command.len(), 76); + } + + #[test] + fn sony_report_command_rejects_ids_outside_the_reserved_domain() { + let report = [0_u8; 64]; + assert!(sony_report_command(5, &report).is_none()); + assert!(sony_report_command(10, &report).is_none()); + assert!(sony_report_command(9, &report).is_some()); + } + + #[test] + fn sony_output_parser_extracts_the_low_id_and_operation_payload() { + let mut payload = vec![0x11, 0x00, 0x00, 0x00, 0x08, 0x04, 0x00]; + payload.extend_from_slice(&[5, 0x01, 0, 0, 0x40, 0x80]); + let (low_id, operation) = parse_sony_output(&payload).unwrap(); + assert_eq!(low_id, 8); + assert_eq!(operation, &[5, 0x01, 0, 0, 0x40, 0x80]); + } + + #[test] + fn sony_output_parser_rejects_short_unknown_and_high_id_payloads() { + let mut payload = vec![0x11, 0x00, 0x00, 0x00, 0x06, 0x04, 0x00]; + payload.extend_from_slice(&[5, 0x01, 0, 0, 0x40, 0x80]); + assert!(parse_sony_output(&payload).is_some()); + assert!(parse_sony_output(&payload[..12]).is_none()); + let mut wrong_type = payload.clone(); + wrong_type[0] = 0x12; + assert!(parse_sony_output(&wrong_type).is_none()); + let mut high_id = payload.clone(); + high_id[4] = 10; + assert!(parse_sony_output(&high_id).is_none()); + let mut low_id = payload; + low_id[4] = 5; + assert!(parse_sony_output(&low_id).is_none()); + } + + #[test] + fn sony_report_route_uses_the_reliable_control_channel_not_the_partial_one() { + let report = [0_u8; 64]; + let command = sony_report_command(6, &report).unwrap(); + let encoded = NvstEncodedInput { + route: NvstInputRoute::ControlReliable, + bytes: command, + meaningful: true, + }; + assert_eq!(encoded.route, NvstInputRoute::ControlReliable); + assert_ne!(encoded.route, NvstInputRoute::InputPartial); + } + + fn generic_report( + operation: u8, + error: bool, + bytes: Vec, + ) -> opennow_streamer_hid::generic::GenericOutbound { + opennow_streamer_hid::generic::GenericOutbound::Report { + device_id: 10, + interface: 2, + operation, + error, + bytes, + slot: 0, + incarnation: 1, + release: false, + } + } + + #[test] + fn generic_attach_detach_and_report_match_exact_wire_golden_bytes() { + use opennow_streamer_hid::generic::{GenericDevice, GenericOutbound}; + let attach = GenericOutbound::Attach { + request_id: 10, + device: GenericDevice { + slot: 0, + incarnation: 1, + vendor: 0x044f, + product: 0x0402, + revision: 0x0100, + interfaces: vec![], + }, + }; + assert_eq!( + generic_hid_command(&attach).unwrap(), + hex("0602160000000012120000000a010000000f044f040201000000") + ); + assert_eq!( + generic_hid_command(&GenericOutbound::Detach { device_id: 42 }).unwrap(), + hex("0602160000000012120000002a030000000f0000000000000000") + ); + assert_eq!( + generic_hid_command(&generic_report(0, false, hex("01aabb"))).unwrap(), + hex("0d020b0022110000000a020001aabb") + ); + assert_eq!( + generic_hid_command(&generic_report(3, true, vec![])).unwrap(), + hex("0d02080022110000000a0103") + ); + assert_eq!( + generic_hid_command(&generic_report(2, false, hex("010203"))).unwrap(), + hex("0d020b0022110000000a0002010203") + ); + } + + #[test] + fn generic_outbound_bounds_do_not_rewrite_reports() { + for op in 0..=4 { + let bytes = vec![0xa5; 256]; + let command = generic_hid_command(&generic_report(op, false, bytes.clone())).unwrap(); + assert_eq!(&command[12..], &bytes); + assert!(generic_hid_command(&generic_report(op, false, vec![0; 257])).is_none()); + } + assert!(generic_hid_command(&generic_report(5, false, vec![1])).is_none()); + assert!(generic_hid_command(&generic_report(0, false, vec![])).is_none()); + assert!(generic_hid_command(&generic_report(0, true, vec![1])).is_none()); + assert!(generic_hid_command(&generic_report(3, false, vec![])).is_none()); + assert!(generic_hid_command(&generic_report(1, false, vec![])).is_none()); + assert!(generic_hid_command(&generic_report(1, true, vec![])).is_some()); + } + + #[test] + fn generic_response_parser_uses_eleven_byte_payload_and_big_endian_opaque_word() { + let packet = hex("06020b001a0000000a01123456782a"); + let mut parsed = vec![]; + for_each_generic_hid_message(&packet, |message| parsed.push(message)); + assert_eq!( + parsed, + vec![GenericHidMessage::Response { + request_id: 10, + control: 1, + opaque: 0x12345678, + device_id: 42, + }] + ); + for length in 0..11 { + let short = control_command(COMMAND_REMOTE_INPUT, &packet[4..4 + length]); + for_each_generic_hid_message(&short, |_| panic!("short response accepted")); + } + let mut oversized = packet[4..].to_vec(); + oversized.push(0); + for_each_generic_hid_message(&control_command(COMMAND_REMOTE_INPUT, &oversized), |_| { + panic!("oversized response accepted") + }); + } + + #[test] + fn generic_output_parser_validates_operation_get_and_command_boundaries() { + let output = hex("110000000a020001aabb"); + assert_eq!( + parse_generic_hid_message(&output), + Some(GenericHidMessage::Output { + device_id: 10, + interface: 2, + operation: 0, + bytes: &output[7..], + }) + ); + for op in 2..=3 { + let mut get = hex("110000000a0202010001"); + get[6] = op; + assert!(parse_generic_hid_message(&get).is_some()); + get.push(0); + assert!(parse_generic_hid_message(&get).is_none()); + get.truncate(10); + get[8..].copy_from_slice(&0_u16.to_le_bytes()); + assert!(parse_generic_hid_message(&get).is_none()); + get[8..].copy_from_slice(&257_u16.to_le_bytes()); + assert!(parse_generic_hid_message(&get).is_none()); + } + for length in 0..7 { + assert!(parse_generic_hid_message(&output[..length]).is_none()); + } + assert!(parse_generic_hid_message(&output[..7]).is_some()); + let mut invalid = output.clone(); + invalid[6] = 5; + assert!(parse_generic_hid_message(&invalid).is_none()); + invalid = output.clone(); + invalid.resize(264, 0); + assert!(parse_generic_hid_message(&invalid).is_none()); + invalid = output.clone(); + invalid[4] = 6; + assert!(parse_generic_hid_message(&invalid).is_none()); + let mut packets = control_command(0xffff, &output); + packets.extend(control_command(COMMAND_REMOTE_INPUT, &output)); + packets.extend(hex("0602ffff110000000a020001")); + let mut count = 0; + for_each_generic_hid_message(&packets, |_| count += 1); + assert_eq!(count, 1); + let mut sony_count = 0; + for_each_sony_output(&packets, |_, _| sony_count += 1); + assert_eq!(sony_count, 0); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input_delivery.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input_delivery.rs new file mode 100644 index 000000000..09e39bf2c --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_input_delivery.rs @@ -0,0 +1,607 @@ +use opennow_streamer_protocol::text_input::UnicodeText; +use std::collections::VecDeque; + +use crate::nvst_budget::TEXT_BATCH_GUARD; +use crate::nvst_input::{ + NvstEncodedInput, NvstInputCodec, NvstInputCodecError, NvstInputWrite, + native_input_type_is_motion, native_input_types, text_batch_fits, unicode_text_messages, +}; + +const PENDING_INPUT_MESSAGES: usize = 256; +const PENDING_INPUT_BYTES: usize = 64 * 1024; + +#[derive(Debug)] +pub(crate) enum NvstInputDeliveryError { + Encoding(NvstInputCodecError), + Exhausted, + Closed, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(pressed: bool) -> Vec { + let mut bytes = vec![0; 18]; + bytes[..4].copy_from_slice(&(if pressed { 3_u32 } else { 4_u32 }).to_le_bytes()); + bytes + } + + #[test] + fn releases_wait_behind_deferred_presses_and_successful_prefixes_are_not_replayed() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let press = key(true); + let release = key(false); + let mut expected = codec.clone().encode(&press, 1).unwrap(); + expected.extend(codec.clone().encode(&release, 2).unwrap()); + delivery + .submit(&mut codec, &press, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit(&mut codec, &release, 2, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!(delivery.pending.len(), 2); + let mut delivered = Vec::new(); + delivery + .flush(|message| { + if delivered.is_empty() { + delivered.push(message.clone()); + NvstInputWrite::Sent + } else { + NvstInputWrite::Deferred + } + }) + .unwrap(); + assert_eq!(delivery.pending.len(), 1); + delivery + .flush(|message| { + delivered.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + delivery + .flush(|_| panic!("delivered input must not be retried")) + .unwrap(); + assert_eq!(delivered, expected); + assert_eq!(delivery.pending_bytes, 0); + } + + #[test] + fn repeated_rejections_keep_streaming_until_the_bounded_queue_is_exhausted() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + for _ in 0..PENDING_INPUT_MESSAGES { + delivery + .submit(&mut codec, &key(false), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + } + assert!(matches!( + delivery.submit(&mut codec, &key(false), 1, |_| NvstInputWrite::Deferred), + Err(NvstInputDeliveryError::Exhausted) + )); + assert_eq!(delivery.pending.len(), PENDING_INPUT_MESSAGES); + assert!(delivery.pending_bytes <= PENDING_INPUT_BYTES); + } + + #[test] + fn encoded_batch_admission_is_atomic_and_rejected_codec_state_is_restored() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + for _ in 0..PENDING_INPUT_MESSAGES - 1 { + delivery + .submit(&mut codec, &key(false), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + } + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&12_u32.to_le_bytes()); + gamepad[8..10].copy_from_slice(&1_u16.to_le_bytes()); + let expected = codec.clone().encode(&gamepad, 1).unwrap(); + assert_eq!(expected.len(), 2); + assert!(matches!( + delivery.submit(&mut codec, &gamepad, 1, |_| NvstInputWrite::Deferred), + Err(NvstInputDeliveryError::Exhausted) + )); + assert_eq!(delivery.pending.len(), PENDING_INPUT_MESSAGES - 1); + assert_eq!(codec.encode(&gamepad, 1).unwrap(), expected); + } + + #[test] + fn byte_capacity_rejects_a_batch_before_any_write() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + delivery + .pending + .push_back(PendingInput::Discrete(NvstEncodedInput { + route: crate::nvst_input::NvstInputRoute::ControlReliable, + bytes: vec![0; PENDING_INPUT_BYTES - 1], + meaningful: false, + })); + delivery.pending_messages = 1; + delivery.pending_bytes = PENDING_INPUT_BYTES - 1; + let mut writes = 0; + assert!(matches!( + delivery.submit(&mut codec, &key(false), 1, |message| { + writes += 1; + assert_eq!(message.bytes.len(), PENDING_INPUT_BYTES - 1); + NvstInputWrite::Deferred + }), + Err(NvstInputDeliveryError::Exhausted) + )); + assert_eq!(writes, 1); + assert_eq!(delivery.pending.len(), 1); + assert_eq!(delivery.pending_bytes, PENDING_INPUT_BYTES - 1); + } + + #[test] + fn motion_is_dropped_instead_of_accumulating_or_overtaking_pending_discrete_input() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let mut motion = vec![0; 22]; + motion[..4].copy_from_slice(&7_u32.to_le_bytes()); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + assert!(delivery.pending.is_empty()); + delivery + .submit(&mut codec, &key(true), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + for _ in 0..1000 { + delivery + .submit(&mut codec, &motion, 1, |message| { + assert_eq!( + message.route, + crate::nvst_input::NvstInputRoute::ControlReliable + ); + NvstInputWrite::Deferred + }) + .unwrap(); + } + assert_eq!(delivery.pending.len(), 1); + } + + #[test] + fn pending_release_reports_a_closed_transport_instead_of_retrying_forever() { + let mut delivery = NvstInputDelivery::default(); + delivery + .submit(&mut NvstInputCodec::default(), &key(false), 1, |_| { + NvstInputWrite::Deferred + }) + .unwrap(); + assert!(matches!( + delivery.flush(|_| NvstInputWrite::Closed), + Err(NvstInputDeliveryError::Closed) + )); + assert_eq!(delivery.pending.len(), 1); + } + + #[test] + fn diagnostics_track_pending_bytes_deferrals_and_motion_drops_without_resetting_history() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let packet = key(false); + let bytes = codec.clone().encode(&packet, 1).unwrap()[0].bytes.len(); + delivery + .submit(&mut codec, &packet, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!( + delivery.snapshot(), + NvstInputDeliverySnapshot { + pending_messages: 1, + pending_bytes: bytes, + pending_text_messages: 0, + pending_text_bytes: 0, + deferred_writes: 1, + dropped_motion: 0, + } + ); + let mut motion = vec![0; 22]; + motion[..4].copy_from_slice(&7_u32.to_le_bytes()); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!( + delivery.snapshot(), + NvstInputDeliverySnapshot { + pending_messages: 1, + pending_bytes: bytes, + pending_text_messages: 0, + pending_text_bytes: 0, + deferred_writes: 2, + dropped_motion: 1, + } + ); + delivery.flush(|_| NvstInputWrite::Sent).unwrap(); + assert_eq!( + delivery.snapshot(), + NvstInputDeliverySnapshot { + pending_messages: 0, + pending_bytes: 0, + pending_text_messages: 0, + pending_text_bytes: 0, + deferred_writes: 2, + dropped_motion: 1, + } + ); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!( + delivery.snapshot(), + NvstInputDeliverySnapshot { + pending_messages: 0, + pending_bytes: 0, + pending_text_messages: 0, + pending_text_bytes: 0, + deferred_writes: 3, + dropped_motion: 2, + } + ); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Sent) + .unwrap(); + assert_eq!(delivery.snapshot().deferred_writes, 3); + assert_eq!(delivery.snapshot().dropped_motion, 2); + } + + #[test] + fn diagnostic_counters_saturate_under_prolonged_pressure() { + let mut delivery = NvstInputDelivery { + deferred_writes: u64::MAX, + dropped_motion: u64::MAX, + ..NvstInputDelivery::default() + }; + let mut codec = NvstInputCodec::default(); + let mut motion = vec![0; 22]; + motion[..4].copy_from_slice(&7_u32.to_le_bytes()); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit(&mut codec, &key(false), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit(&mut codec, &motion, 1, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!(delivery.snapshot().deferred_writes, u64::MAX); + assert_eq!(delivery.snapshot().dropped_motion, u64::MAX); + } + + #[test] + fn paste_waits_behind_deferred_press_and_before_subsequent_release() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let slot = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let text = slot.submit("café 世界".as_bytes()).unwrap(); + let mut expected = codec.clone().encode(&key(true), 1).unwrap(); + expected.extend(unicode_text_messages(&text, 2)); + expected.extend(codec.clone().encode(&key(false), 3).unwrap()); + delivery + .submit(&mut codec, &key(true), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit_text(text, 2, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit(&mut codec, &key(false), 3, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!(delivery.snapshot().pending_messages, 2); + assert_eq!(delivery.snapshot().pending_text_messages, 1); + let mut sent = Vec::new(); + delivery + .flush(|message| { + sent.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(sent, expected); + assert!(delivery.is_empty()); + assert_eq!(delivery.snapshot().pending_bytes, 0); + assert_eq!(delivery.snapshot().pending_text_bytes, 0); + } + + #[test] + fn partial_paste_keeps_single_flight_reservation_and_never_replays_sent_prefix() { + let mut delivery = NvstInputDelivery::default(); + let slot = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let text = slot.submit("é世界🦫\r\n".repeat(400).as_bytes()).unwrap(); + let expected = unicode_text_messages(&text, 7); + assert!(expected.len() > 2); + let mut sent = Vec::new(); + delivery + .submit_text(text, 7, |message| { + if sent.is_empty() { + sent.push(message.clone()); + NvstInputWrite::Sent + } else { + NvstInputWrite::Deferred + } + }) + .unwrap(); + assert!(matches!( + slot.submit(b"next"), + Err(opennow_streamer_protocol::text_input::TextInputError::Busy) + )); + assert_eq!( + delivery.snapshot().pending_text_messages, + expected.len() - 1 + ); + delivery + .flush(|message| { + sent.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(sent, expected); + assert_eq!(delivery.snapshot().pending_text_messages, 0); + assert!(slot.submit(b"next").is_ok()); + } + + #[test] + fn cancellation_discards_remaining_text_without_discarding_key_releases() { + let mut delivery = NvstInputDelivery::default(); + let mut codec = NvstInputCodec::default(); + let slot = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let text = slot.submit(b"paste").unwrap(); + let mut expected = codec.clone().encode(&key(true), 1).unwrap(); + expected.extend(codec.clone().encode(&key(false), 3).unwrap()); + delivery + .submit(&mut codec, &key(true), 1, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit_text(text, 2, |_| NvstInputWrite::Deferred) + .unwrap(); + delivery + .submit(&mut codec, &key(false), 3, |_| NvstInputWrite::Deferred) + .unwrap(); + slot.cancel(); + let mut sent = Vec::new(); + delivery + .flush(|message| { + sent.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(sent, expected); + assert_eq!(delivery.snapshot().pending_text_bytes, 0); + assert_eq!(delivery.snapshot().pending_text_messages, 0); + assert!(slot.submit(b"next").is_ok()); + } + + #[test] + fn maximum_paste_is_bounded_and_survives_a_full_sctp_normal_budget() { + let mut delivery = NvstInputDelivery::default(); + let slot = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let text = slot + .submit(&vec![ + b'a'; + opennow_streamer_protocol::text_input::MAX_TEXT_BYTES + ]) + .unwrap(); + let expected = unicode_text_messages(&text, 9); + let total: usize = expected.iter().map(|message| message.bytes.len()).sum(); + assert!(total > crate::nvst_budget::NORMAL_BUDGET); + assert!(total <= TEXT_BATCH_GUARD); + delivery + .submit_text(text, 9, |_| NvstInputWrite::Deferred) + .unwrap(); + assert_eq!(delivery.snapshot().pending_text_bytes, total); + let mut budget = crate::nvst_budget::NORMAL_BUDGET; + let mut sent = Vec::new(); + delivery + .flush(|message| { + if message.bytes.len() > budget { + return NvstInputWrite::Deferred; + } + budget -= message.bytes.len(); + sent.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert!(!delivery.is_empty()); + delivery + .flush(|message| { + sent.push(message.clone()); + NvstInputWrite::Sent + }) + .unwrap(); + assert_eq!(sent, expected); + assert!(delivery.is_empty()); + } + + #[test] + fn only_one_bounded_paste_batch_can_wait_at_a_time() { + let mut delivery = NvstInputDelivery::default(); + let first = opennow_streamer_protocol::text_input::TextInputSlot::default(); + let second = opennow_streamer_protocol::text_input::TextInputSlot::default(); + delivery + .submit_text(first.submit(b"first").unwrap(), 1, |_| { + NvstInputWrite::Deferred + }) + .unwrap(); + let snapshot = delivery.snapshot(); + assert!(matches!( + delivery.submit_text(second.submit(b"second").unwrap(), 2, |_| { + NvstInputWrite::Deferred + }), + Err(NvstInputDeliveryError::Exhausted) + )); + assert_eq!( + delivery.snapshot().pending_text_messages, + snapshot.pending_text_messages + ); + assert_eq!( + delivery.snapshot().pending_text_bytes, + snapshot.pending_text_bytes + ); + assert!(second.submit(b"retry").is_ok()); + } +} + +#[derive(Default)] +pub(crate) struct NvstInputDelivery { + pending: VecDeque, + pending_messages: usize, + pending_bytes: usize, + pending_text_messages: usize, + pending_text_bytes: usize, + deferred_writes: u64, + dropped_motion: u64, +} + +enum PendingInput { + Discrete(NvstEncodedInput), + Text { + reservation: UnicodeText, + messages: VecDeque, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NvstInputDeliverySnapshot { + pub(crate) pending_messages: usize, + pub(crate) pending_bytes: usize, + pub(crate) pending_text_messages: usize, + pub(crate) pending_text_bytes: usize, + pub(crate) deferred_writes: u64, + pub(crate) dropped_motion: u64, +} + +impl NvstInputDelivery { + pub(crate) fn snapshot(&self) -> NvstInputDeliverySnapshot { + NvstInputDeliverySnapshot { + pending_messages: self.pending_messages, + pending_bytes: self.pending_bytes, + pending_text_messages: self.pending_text_messages, + pending_text_bytes: self.pending_text_bytes, + deferred_writes: self.deferred_writes, + dropped_motion: self.dropped_motion, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.pending.is_empty() + } + + pub(crate) fn submit( + &mut self, + codec: &mut NvstInputCodec, + bytes: &[u8], + timestamp_us: u64, + mut send: impl FnMut(&NvstEncodedInput) -> NvstInputWrite, + ) -> Result<(), NvstInputDeliveryError> { + self.flush(&mut send)?; + let previous_codec = codec.clone(); + let messages = match codec.encode(bytes, timestamp_us) { + Ok(messages) => messages, + Err(error) => { + *codec = previous_codec; + return Err(NvstInputDeliveryError::Encoding(error)); + } + }; + let only_motion = native_input_types(bytes).is_ok_and(|types| { + !types.is_empty() && types.into_iter().all(native_input_type_is_motion) + }); + if only_motion { + if self.pending.is_empty() { + for message in messages { + match send(&message) { + NvstInputWrite::Sent => {} + NvstInputWrite::Deferred => { + self.deferred_writes = self.deferred_writes.saturating_add(1); + self.dropped_motion = self.dropped_motion.saturating_add(1); + } + NvstInputWrite::Closed => return Err(NvstInputDeliveryError::Closed), + } + } + } else { + self.dropped_motion = self.dropped_motion.saturating_add(messages.len() as u64); + } + return Ok(()); + } + let total_bytes: usize = messages.iter().map(|message| message.bytes.len()).sum(); + if self.pending_messages.saturating_add(messages.len()) > PENDING_INPUT_MESSAGES + || self.pending_bytes.saturating_add(total_bytes) > PENDING_INPUT_BYTES + { + *codec = previous_codec; + return Err(NvstInputDeliveryError::Exhausted); + } + self.pending_bytes += total_bytes; + self.pending_messages += messages.len(); + self.pending + .extend(messages.into_iter().map(PendingInput::Discrete)); + self.flush(send) + } + + pub(crate) fn submit_text( + &mut self, + text: UnicodeText, + timestamp_us: u64, + mut send: impl FnMut(&NvstEncodedInput) -> NvstInputWrite, + ) -> Result<(), NvstInputDeliveryError> { + self.flush(&mut send)?; + if text.is_cancelled() { + return Ok(()); + } + if self.pending_text_messages != 0 { + return Err(NvstInputDeliveryError::Exhausted); + } + let messages = unicode_text_messages(&text, timestamp_us); + let bytes = messages.iter().map(|message| message.bytes.len()).sum(); + if !text_batch_fits(0, bytes, TEXT_BATCH_GUARD) { + return Err(NvstInputDeliveryError::Exhausted); + } + self.pending_text_messages = messages.len(); + self.pending_text_bytes = bytes; + self.pending.push_back(PendingInput::Text { + reservation: text, + messages: messages.into(), + }); + self.flush(send) + } + + pub(crate) fn flush( + &mut self, + mut send: impl FnMut(&NvstEncodedInput) -> NvstInputWrite, + ) -> Result<(), NvstInputDeliveryError> { + while let Some(pending) = self.pending.front_mut() { + if let PendingInput::Text { reservation, .. } = pending + && reservation.is_cancelled() + { + self.pending_text_messages = 0; + self.pending_text_bytes = 0; + self.pending.pop_front(); + continue; + } + let message = match pending { + PendingInput::Discrete(message) => message, + PendingInput::Text { messages, .. } => { + messages.front().expect("pending text message") + } + }; + match send(message) { + NvstInputWrite::Sent => match pending { + PendingInput::Discrete(message) => { + self.pending_bytes -= message.bytes.len(); + self.pending_messages -= 1; + self.pending.pop_front(); + } + PendingInput::Text { messages, .. } => { + self.pending_text_bytes -= + messages.pop_front().expect("sent text message").bytes.len(); + self.pending_text_messages -= 1; + if messages.is_empty() { + self.pending.pop_front(); + } + } + }, + NvstInputWrite::Deferred => { + self.deferred_writes = self.deferred_writes.saturating_add(1); + return Ok(()); + } + NvstInputWrite::Closed => return Err(NvstInputDeliveryError::Closed), + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs new file mode 100644 index 000000000..a5e8a41db --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs @@ -0,0 +1,275 @@ +//! Reference-picture invalidation (RFI) loss recovery, following the GFN client's +//! `NvstQosManager::updateInvalidationStatus` policy: after unrecoverable loss the +//! receiver reports the lost sender frame range with control command 0x301, withholds +//! every frame that may reference it, and resumes on the first I, intra-refresh, or +//! reference-picture-invalidation frame. An IDR request remains the bounded backstop. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use super::nvst::{EncodedVideoAccessUnit, NvstFeedbackState}; + +const RFI_RETRY_INTERVAL: Duration = Duration::from_millis(100); + +/// `GsFrameDescriptor.frameType`, carried in byte 3 of the GameStream frame header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvstFrameType { + P, + I, + Header, + IntraRefresh, + RefPicI, + NonRefP, + Unknown(u8), +} + +impl NvstFrameType { + pub fn from_wire(value: u8) -> Self { + match value { + 1 => Self::P, + 2 => Self::I, + 3 => Self::Header, + 4 => Self::IntraRefresh, + 5 => Self::RefPicI, + 6 => Self::NonRefP, + other => Self::Unknown(other), + } + } + + fn clears_invalidation(self) -> bool { + matches!(self, Self::I | Self::IntraRefresh | Self::RefPicI) + } +} + +#[derive(Debug, Default)] +struct UnsentInvalidation { + range: Option<(u64, u64)>, + in_flight: bool, + last_sent_at: Option, +} + +#[derive(Debug, Default)] +pub(crate) struct FrameInvalidationFeedback { + unsent: Mutex, + sent: AtomicU64, + dropped_frames: AtomicU64, + recovered_i: AtomicU64, + recovered_intra_refresh: AtomicU64, + recovered_ref_pic_i: AtomicU64, + idr_backstops: AtomicU64, +} + +impl FrameInvalidationFeedback { + fn unsent(&self) -> std::sync::MutexGuard<'_, UnsentInvalidation> { + self.unsent + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// A newer range always covers the unsent one because ranges only grow by merging. + pub(crate) fn queue(&self, range: (u64, u64)) { + self.unsent().range = Some(range); + } + + pub(crate) fn take(&self) -> Option<(u64, u64)> { + let mut unsent = self.unsent(); + let range = unsent.range.take(); + unsent.in_flight = range.is_some(); + range + } + + /// Requeues a rejected send unless recovery discarded the range while it was in flight. + pub(crate) fn restore(&self, range: (u64, u64)) { + let mut unsent = self.unsent(); + if std::mem::take(&mut unsent.in_flight) { + unsent.range.get_or_insert(range); + } + } + + pub(crate) fn record_sent(&self, now: Instant) { + let mut unsent = self.unsent(); + if !std::mem::take(&mut unsent.in_flight) { + return; + } + unsent.last_sent_at = Some(now); + self.sent.fetch_add(1, Ordering::Relaxed); + } + + fn last_sent_at(&self) -> Option { + self.unsent().last_sent_at + } + + fn discard(&self) { + *self.unsent() = UnsentInvalidation::default(); + } + + pub(crate) fn stats(&self) -> String { + let load = |counter: &AtomicU64| counter.load(Ordering::Relaxed); + format!( + "rfiSent={} rfiDropped={} rfiRecoveredI={} rfiRecoveredIntraRefresh={} rfiRecoveredRefPicI={} rfiIdrBackstop={}", + load(&self.sent), + load(&self.dropped_frames), + load(&self.recovered_i), + load(&self.recovered_intra_refresh), + load(&self.recovered_ref_pic_i), + load(&self.idr_backstops), + ) + } +} + +#[derive(Debug, Default)] +pub(crate) struct FrameInvalidation { + last_frame: Option, + frame_types_known: bool, + pending: Option<(u64, u64)>, + active_since: Option, + last_retry_at: Option, + backstop_requested: bool, +} + +impl FrameInvalidation { + pub(crate) fn reset(&mut self, feedback: &NvstFeedbackState) { + feedback.frame_invalidation.discard(); + *self = Self::default(); + } + + /// Returns whether invalidation owns recovery for this loss. Streams without a frame + /// type keep the immediate keyframe request. + pub(crate) fn lose(&mut self, now: Instant) -> bool { + if !self.frame_types_known { + return false; + } + self.active_since.get_or_insert(now); + true + } + + pub(crate) fn poll_backstop( + &mut self, + now: Instant, + cooldown: Duration, + feedback: &NvstFeedbackState, + ) { + if let Some(range) = self.pending + && self + .last_retry_at + .max(feedback.frame_invalidation.last_sent_at()) + .is_none_or(|last| now.saturating_duration_since(last) >= RFI_RETRY_INTERVAL) + { + feedback.frame_invalidation.queue(range); + self.last_retry_at = Some(now); + } + if self + .active_since + .is_some_and(|since| now.saturating_duration_since(since) >= cooldown) + { + self.request_idr_backstop(feedback); + } + } + + /// Decides whether an assembled frame may reach the decoder and sets its contiguity. + /// Every assembled frame counts as received, including frames withheld here. + pub(crate) fn admit( + &mut self, + frame: &mut EncodedVideoAccessUnit, + loss_before: bool, + now: Instant, + feedback: &NvstFeedbackState, + ) -> bool { + let previous = self.last_frame.replace(frame.frame_index); + self.frame_types_known = frame.frame_type.is_some(); + let frame_gap = + previous.is_some_and(|previous| frame.frame_index > previous.saturating_add(1)); + if frame_gap { + self.lose(now); + } + let invalidating = self.active_since.is_some(); + let Some(frame_type) = frame.frame_type else { + if invalidating { + feedback.request_keyframe(); + } + self.clear(feedback); + frame.contiguous = !loss_before && !invalidating; + return true; + }; + if frame.keyframe || frame_type.clears_invalidation() { + if invalidating { + let counters = &feedback.frame_invalidation; + let recovered = match frame_type { + NvstFrameType::IntraRefresh => &counters.recovered_intra_refresh, + NvstFrameType::RefPicI => &counters.recovered_ref_pic_i, + _ => &counters.recovered_i, + }; + recovered.fetch_add(1, Ordering::Relaxed); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-rfi", + &format!( + "recovered frame={} type={frame_type:?} keyframe={} pending={:?}", + frame.frame_index, frame.keyframe, self.pending + ), + ); + self.clear(feedback); + } + frame.contiguous = invalidating || !loss_before; + return true; + } + if !invalidating { + frame.contiguous = !loss_before; + return true; + } + if loss_before || frame_gap { + self.invalidate(previous, frame.frame_index, now, feedback); + } + if frame_type == NvstFrameType::Header { + frame.contiguous = true; + return true; + } + feedback + .frame_invalidation + .dropped_frames + .fetch_add(1, Ordering::Relaxed); + false + } + + fn invalidate( + &mut self, + previous: Option, + frame_index: u32, + now: Instant, + feedback: &NvstFeedbackState, + ) { + let Some(previous) = previous.filter(|previous| frame_index > previous.saturating_add(1)) + else { + self.request_idr_backstop(feedback); + return; + }; + let lost = (u64::from(previous) + 1, u64::from(frame_index) - 1); + let range = self.pending.map_or(lost, |pending| { + (pending.0.min(lost.0), pending.1.max(lost.1)) + }); + self.pending = Some(range); + feedback.frame_invalidation.queue(range); + self.last_retry_at = Some(now); + } + + fn request_idr_backstop(&mut self, feedback: &NvstFeedbackState) { + if self.backstop_requested { + return; + } + self.backstop_requested = true; + feedback + .frame_invalidation + .idr_backstops + .fetch_add(1, Ordering::Relaxed); + feedback.request_keyframe(); + } + + fn clear(&mut self, feedback: &NvstFeedbackState) { + feedback.frame_invalidation.discard(); + self.pending = None; + self.active_since = None; + self.last_retry_at = None; + self.backstop_requested = false; + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_microphone.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_microphone.rs new file mode 100644 index 000000000..bdfba8a9c --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_microphone.rs @@ -0,0 +1,362 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use crate::nvst::NvstUdpReceiverError; + +pub(crate) const MICROPHONE_QUEUE_CAPACITY: usize = 5; +const MAX_OPUS_BYTES: usize = 1275; +const MAX_FRAME_AGE: Duration = Duration::from_millis(100); + +#[derive(Debug, Clone, Copy)] +pub(crate) struct MicrophoneRtpConfig { + pub opus_payload_type: u8, + pub red_payload_type: Option, + pub redundancy_level: u8, +} + +impl Default for MicrophoneRtpConfig { + fn default() -> Self { + Self { + opus_payload_type: 111, + red_payload_type: None, + redundancy_level: 0, + } + } +} + +pub(crate) struct MicrophonePacketizer { + config: MicrophoneRtpConfig, + history: VecDeque<(u32, Vec)>, +} + +impl MicrophonePacketizer { + pub fn new(config: MicrophoneRtpConfig) -> Self { + Self { + config, + history: VecDeque::with_capacity(3), + } + } + + pub fn clear(&mut self) { + self.history.clear(); + } + + pub fn packetize(&mut self, frame: MicrophoneFrame) -> (u8, Vec) { + let Some(red_payload_type) = self + .config + .red_payload_type + .filter(|_| self.config.redundancy_level > 0) + else { + return (self.config.opus_payload_type, frame.payload); + }; + self.history.retain(|(timestamp, payload)| { + let offset = frame.timestamp.wrapping_sub(*timestamp); + (1..=0x3fff).contains(&offset) && payload.len() <= 0x3ff + }); + while 1 + + frame.payload.len() + + self + .history + .iter() + .map(|(_, bytes)| 4 + bytes.len()) + .sum::() + > MAX_OPUS_BYTES + 1 + { + self.history.pop_front(); + } + let mut payload = Vec::with_capacity( + 1 + frame.payload.len() + + self + .history + .iter() + .map(|(_, bytes)| 4 + bytes.len()) + .sum::(), + ); + for (timestamp, bytes) in &self.history { + let offset = frame.timestamp.wrapping_sub(*timestamp) as u16; + let length = bytes.len() as u16; + payload.push(0x80 | self.config.opus_payload_type); + payload.push((offset >> 6) as u8); + payload.push(((offset & 0x3f) << 2) as u8 | (length >> 8) as u8); + payload.push(length as u8); + } + payload.push(self.config.opus_payload_type); + for (_, bytes) in &self.history { + payload.extend_from_slice(bytes); + } + payload.extend_from_slice(&frame.payload); + if frame.payload.len() <= 0x3ff { + self.history.push_back((frame.timestamp, frame.payload)); + while self.history.len() > usize::from(self.config.redundancy_level) { + self.history.pop_front(); + } + } + (red_payload_type, payload) + } +} + +pub(crate) struct MicrophoneFrame { + pub payload: Vec, + pub timestamp: u32, + pub queued_at: Instant, +} + +pub(crate) struct MicrophoneQueue { + available: bool, + enabled: bool, + closed: bool, + pub generation: u64, + frames: VecDeque, + dropped: u64, +} + +impl MicrophoneQueue { + pub fn new(available: bool) -> Self { + Self { + available, + enabled: false, + closed: false, + generation: 0, + frames: VecDeque::with_capacity(MICROPHONE_QUEUE_CAPACITY), + dropped: 0, + } + } + + fn clear(&mut self) { + self.dropped += self.frames.len() as u64; + self.frames.clear(); + } + + pub fn set_enabled(&mut self, enabled: bool) -> Result<(), NvstUdpReceiverError> { + self.clear(); + self.generation = self.generation.wrapping_add(1); + if self.closed { + return Err(NvstUdpReceiverError::Closed); + } + if enabled && !self.available { + return Err(NvstUdpReceiverError::MicrophoneUnavailable); + } + self.enabled = enabled; + Ok(()) + } + + pub fn push( + &mut self, + payload: Vec, + timestamp: u32, + now: Instant, + ) -> Result<(), NvstUdpReceiverError> { + if self.closed { + return Err(NvstUdpReceiverError::Closed); + } + if !self.available { + return Err(NvstUdpReceiverError::MicrophoneUnavailable); + } + if !self.enabled { + return Err(NvstUdpReceiverError::MicrophoneDisabled); + } + if payload.is_empty() || payload.len() > MAX_OPUS_BYTES { + return Err(NvstUdpReceiverError::InvalidMicrophoneFrame); + } + if self.frames.len() == MICROPHONE_QUEUE_CAPACITY { + self.frames.pop_front(); + self.dropped += 1; + } + self.frames.push_back(MicrophoneFrame { + payload, + timestamp, + queued_at: now, + }); + Ok(()) + } + + pub fn pop(&mut self, now: Instant) -> Option { + while let Some(frame) = self.frames.pop_front() { + if now.saturating_duration_since(frame.queued_at) < MAX_FRAME_AGE { + return Some(frame); + } + self.dropped += 1; + } + None + } + + pub fn dropped(&self) -> u64 { + self.dropped + } + + pub fn close(&mut self) { + self.clear(); + self.enabled = false; + self.closed = true; + self.generation = self.generation.wrapping_add(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(timestamp: u32, value: u8) -> MicrophoneFrame { + MicrophoneFrame { + payload: vec![value], + timestamp, + queued_at: Instant::now(), + } + } + + #[test] + fn red_encodes_three_prior_frames_with_rfc2198_headers_and_timestamp_wrap() { + let mut sender = MicrophonePacketizer::new(MicrophoneRtpConfig { + opus_payload_type: 111, + red_payload_type: Some(63), + redundancy_level: 3, + }); + let first = u32::MAX - 479; + assert_eq!(sender.packetize(frame(first, 1)), (63, vec![111, 1])); + sender.packetize(frame(0, 2)); + sender.packetize(frame(480, 3)); + let (pt, payload) = sender.packetize(frame(960, 4)); + assert_eq!(pt, 63); + assert_eq!( + payload, + [ + 239, 22, 128, 1, 239, 15, 0, 1, 239, 7, 128, 1, 111, 1, 2, 3, 4 + ] + ); + assert_eq!(sender.history.len(), 3); + sender.clear(); + assert_eq!(sender.packetize(frame(1440, 5)), (63, vec![111, 5])); + } + + #[test] + fn negotiated_payload_types_and_disabled_red_are_honored() { + for config in [ + MicrophoneRtpConfig::default(), + MicrophoneRtpConfig { + opus_payload_type: 109, + red_payload_type: Some(112), + redundancy_level: 0, + }, + ] { + let mut sender = MicrophonePacketizer::new(config); + assert_eq!( + sender.packetize(frame(0, 1)), + (config.opus_payload_type, vec![1]) + ); + assert!(sender.history.is_empty()); + } + let mut sender = MicrophonePacketizer::new(MicrophoneRtpConfig { + opus_payload_type: 109, + red_payload_type: Some(112), + redundancy_level: 3, + }); + assert_eq!(sender.packetize(frame(0, 1)), (112, vec![109, 1])); + assert_eq!( + sender.packetize(frame(480, 2)), + (112, vec![237, 7, 128, 1, 109, 1, 2]) + ); + } + + #[test] + fn red_history_excludes_unrepresentable_lengths_offsets_and_timestamp_restarts() { + let mut sender = MicrophonePacketizer::new(MicrophoneRtpConfig { + opus_payload_type: 111, + red_payload_type: Some(63), + redundancy_level: 3, + }); + let mut oversized = frame(0, 1); + oversized.payload = vec![1; 1275]; + assert_eq!(sender.packetize(oversized).1.len(), 1276); + assert!(sender.history.is_empty()); + sender.packetize(frame(480, 2)); + assert_eq!(sender.packetize(frame(480 + 0x4000, 3)).1, [111, 3]); + assert_eq!(sender.packetize(frame(0, 4)).1, [111, 4]); + } + + #[test] + fn red_payload_budget_drops_oldest_redundancy_not_primary_audio() { + let mut sender = MicrophonePacketizer::new(MicrophoneRtpConfig { + opus_payload_type: 111, + red_payload_type: Some(63), + redundancy_level: 3, + }); + for timestamp in [0, 480, 960, 1440] { + let mut frame = frame(timestamp, (timestamp / 480) as u8); + frame.payload.resize(500, frame.payload[0]); + let expected = frame.payload.clone(); + let (_, payload) = sender.packetize(frame); + assert!(payload.len() <= MAX_OPUS_BYTES + 1); + assert_eq!(&payload[payload.len() - 500..], &expected); + } + } + + #[test] + fn microphone_queue_bounds_payload_and_drops_oldest() { + let mut queue = MicrophoneQueue::new(true); + queue.set_enabled(true).unwrap(); + let now = Instant::now(); + assert!(queue.push(vec![], 0, now).is_err()); + assert!(queue.push(vec![0; MAX_OPUS_BYTES + 1], 0, now).is_err()); + for timestamp in 0..8 { + queue.push(vec![1], timestamp, now).unwrap(); + } + assert_eq!(queue.frames.len(), MICROPHONE_QUEUE_CAPACITY); + assert_eq!(queue.dropped(), 3); + assert_eq!(queue.pop(now).unwrap().timestamp, 3); + assert!(queue.pop(now + MAX_FRAME_AGE).is_none()); + assert_eq!(queue.dropped(), 7); + } + + #[test] + fn microphone_queue_flushes_every_explicit_toggle_and_shutdown() { + let mut queue = MicrophoneQueue::new(true); + let now = Instant::now(); + assert!(matches!( + queue.push(vec![1], 0, now), + Err(NvstUdpReceiverError::MicrophoneDisabled) + )); + queue.set_enabled(true).unwrap(); + queue.push(vec![1], u32::MAX, now).unwrap(); + queue.set_enabled(true).unwrap(); + assert!(queue.pop(now).is_none()); + queue.push(vec![1], 0, now).unwrap(); + queue.set_enabled(false).unwrap(); + queue.set_enabled(true).unwrap(); + assert!(queue.pop(now).is_none()); + queue.close(); + assert!(matches!( + queue.set_enabled(true), + Err(NvstUdpReceiverError::Closed) + )); + assert!(matches!( + queue.push(vec![1], 0, now), + Err(NvstUdpReceiverError::Closed) + )); + } + + #[test] + fn microphone_frames_expire_at_one_hundred_milliseconds() { + let mut queue = MicrophoneQueue::new(true); + queue.set_enabled(true).unwrap(); + let now = Instant::now(); + queue.push(vec![1], 0, now).unwrap(); + assert!(queue.pop(now + Duration::from_millis(99)).is_some()); + queue.push(vec![1], 960, now).unwrap(); + assert!(queue.pop(now + Duration::from_millis(100)).is_none()); + assert_eq!(queue.dropped(), 1); + } + + #[test] + fn unsupported_microphone_remains_disabled() { + let mut queue = MicrophoneQueue::new(false); + assert!(matches!( + queue.set_enabled(true), + Err(NvstUdpReceiverError::MicrophoneUnavailable) + )); + queue.set_enabled(false).unwrap(); + assert!(matches!( + queue.push(vec![1], 0, Instant::now()), + Err(NvstUdpReceiverError::MicrophoneUnavailable) + )); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs new file mode 100644 index 000000000..c5b72b343 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs @@ -0,0 +1,450 @@ +use super::*; +use str0m::channel::Reliability; + +use super::budget_tests::connect_bundle_pair; +use crate::nvst_budget::{NORMAL_BUDGET, TRANSPORT_AGGREGATE_CAPACITY}; + +fn receive_messages(local: &mut Rtc, remote: &mut Rtc, now: Instant) -> Vec<(String, Vec)> { + let mut messages = Vec::new(); + for _ in 0..4 { + for collect in [false, true] { + let (source, destination) = if collect { + (&mut *remote, &mut *local) + } else { + (&mut *local, &mut *remote) + }; + loop { + match source.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) => destination + .handle_input(Input::Receive( + now, + Receive { + proto: packet.proto, + source: packet.source, + destination: packet.destination, + contents: packet.contents.as_ref().try_into().unwrap(), + }, + )) + .unwrap(), + Output::Event(Event::ChannelData(data)) if collect => { + assert!(data.binary); + let channel = source.channel(data.id).unwrap(); + messages.push((channel.config().unwrap().label.clone(), data.data)); + } + _ => {} + } + } + } + } + messages +} + +#[test] +fn mjolnir_nack_uses_partial_control_and_preserves_wrapping_rtp_sequences() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let partial = local.channel(channels.control_partial).unwrap(); + let config = partial.config().unwrap(); + assert!(!config.ordered); + assert_eq!( + config.reliability, + Reliability::MaxPacketLifetime { lifetime: 300 } + ); + let rtcp = channels.rtcp.expect("bundle RTCP channel"); + local.direct_api().close_data_channel(rtcp); + receive_messages(&mut local, &mut remote, now); + assert!(local.channel(rtcp).is_none()); + + let feedback = NvstFeedbackState::default(); + feedback.request_nack(65534, 65537, now); + send_pending_nack(&feedback, now, &mut local, channels, true, 1, 0x12345678); + let messages = receive_messages(&mut local, &mut remote, now); + assert_eq!( + messages, + vec![( + "control_channel_partially_reliable".to_owned(), + vec![0x17, 3, 13, 0, 2, 0, 1, 0xfe, 0xff, 7, 0, 0, 0, 0, 0, 0, 0], + )] + ); + assert_eq!(feedback.take_nack(now, Some(DEFAULT_NACK_RTT)), None); + assert!(feedback.resolve_nack(65536)); + assert!(!feedback.keyframe_request_pending()); +} + +#[test] +fn non_mjolnir_nack_remains_rtcp_generic_on_the_rtcp_channel() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + local + .direct_api() + .close_data_channel(channels.control_partial); + receive_messages(&mut local, &mut remote, now); + assert!(local.channel(channels.control_partial).is_none()); + let feedback = NvstFeedbackState::default(); + feedback.request_nack(65534, 65537, now); + send_pending_nack(&feedback, now, &mut local, channels, false, 1, 2); + assert_eq!( + receive_messages(&mut local, &mut remote, now), + vec![( + "rtcp_on_sctp_private".to_owned(), + vec![0x81, 205, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0xff, 0xfe, 0, 7], + )] + ); + assert!(feedback.resolve_nack(65536)); +} + +#[test] +fn a_closed_nack_channel_does_not_spend_an_attempt_or_fall_back() { + for mjolnir in [true, false] { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + local.direct_api().close_data_channel(if mjolnir { + channels.control_partial + } else { + channels.rtcp.expect("bundle RTCP channel") + }); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + feedback.request_nack(42, 42, now); + for _ in 0..MAX_NACK_ATTEMPTS + 1 { + send_pending_nack(&feedback, now, &mut local, channels, mjolnir, 1, 2); + } + assert!(receive_messages(&mut local, &mut remote, now).is_empty()); + assert!(!feedback.resolve_nack(42)); + } +} + +#[test] +fn rejected_nacks_preserve_the_shared_budget_and_retry_attempts() { + for mjolnir in [true, false] { + let (mut local, _remote, channels, _) = connect_bundle_pair(); + let now = Instant::now(); + let buffered = channels.buffered_total(&mut local); + assert!(channels.send_control(&mut local, &vec![0; NORMAL_BUDGET - buffered])); + let feedback = NvstFeedbackState::default(); + feedback.request_nack(42, 42, now); + for _ in 0..MAX_NACK_ATTEMPTS + 1 { + send_pending_nack(&feedback, now, &mut local, channels, mjolnir, 1, 2); + } + let buffered = channels.buffered_total(&mut local); + assert_eq!(buffered, NORMAL_BUDGET); + assert!(buffered < TRANSPORT_AGGREGATE_CAPACITY); + assert!(!feedback.resolve_nack(42)); + feedback.request_nack(43, 43, now); + send_pending_nack(&feedback, now, &mut local, channels, mjolnir, 1, 2); + assert_eq!( + feedback.take_nack(now, Some(DEFAULT_NACK_RTT)), + Some((43, 43)) + ); + } +} + +#[test] +fn mjolnir_nacks_keep_the_existing_batch_retry_and_expiry_limits() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + let rtt = Duration::from_millis(5); + let retry_interval = rtt + NACK_RETRY_INTERVAL; + feedback.publish_ping(true, now, rtt); + feedback.request_nack(u64::MAX - 64, u64::MAX, now); + for _ in 0..2 { + send_pending_nack(&feedback, now, &mut local, channels, true, 1, 2); + } + let messages = receive_messages(&mut local, &mut remote, now); + assert_eq!(messages.len(), 2); + assert_eq!( + messages[0].1, + nack_v2(0, &(65471..65535).collect::>()) + .unwrap() + .encoded() + ); + assert_eq!(messages[1].1, nack_v2(0, &[65535]).unwrap().encoded()); + assert_eq!(feedback.take_nack(now, Some(rtt)), None); + for attempt in 1..MAX_NACK_ATTEMPTS { + let retry_at = now + retry_interval * u32::from(attempt); + send_pending_nack( + &feedback, + retry_at - Duration::from_nanos(1), + &mut local, + channels, + true, + 1, + 2, + ); + assert!(receive_messages(&mut local, &mut remote, retry_at).is_empty()); + for _ in 0..2 { + send_pending_nack(&feedback, retry_at, &mut local, channels, true, 1, 2); + } + assert_eq!(receive_messages(&mut local, &mut remote, retry_at).len(), 2); + } + send_pending_nack( + &feedback, + now + retry_interval * 3, + &mut local, + channels, + true, + 1, + 2, + ); + assert!(receive_messages(&mut local, &mut remote, now + retry_interval * 3).is_empty()); + assert_eq!( + feedback.take_nack(now + NACK_TRACKING_TIMEOUT, Some(rtt)), + None + ); + assert!(!feedback.resolve_nack(u64::MAX)); + assert!(!feedback.keyframe_request_pending()); +} + +#[test] +fn rtt_aware_nack_spacing_and_send_caps_match_the_wait_budget() { + let cases: &[(Duration, &[u64])] = &[ + (Duration::ZERO, &[0, 4, 8]), + (Duration::from_nanos(1), &[0, 5, 10]), + (Duration::from_millis(5), &[0, 9, 18]), + (Duration::from_millis(10), &[0, 14, 28]), + (Duration::from_millis(17), &[0, 21, 42]), + (Duration::from_micros(17_333), &[0, 22, 44]), + (Duration::from_micros(17_334), &[0, 22]), + (Duration::from_millis(20), &[0, 24]), + (Duration::from_millis(26), &[0, 30]), + (Duration::from_micros(26_001), &[0]), + (Duration::from_millis(30), &[0]), + (Duration::from_millis(52), &[0]), + (Duration::from_millis(100), &[0]), + (Duration::MAX, &[0]), + ]; + for &(rtt, expected) in cases { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(42, 42, now); + let mut sent_at = Vec::new(); + for millis in 0..=52 { + if let Some(range) = feedback.take_nack(now + Duration::from_millis(millis), Some(rtt)) + { + assert_eq!(range, (42, 42)); + sent_at.push(millis); + } + } + assert_eq!(sent_at, expected, "RTT {rtt:?}"); + assert!(!feedback.resolve_nack(42)); + assert!(!feedback.keyframe_request_pending()); + } +} + +#[test] +fn rtt_aware_nack_cap_keeps_sent_packets_resolvable_until_the_deadline() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + let rtt = Some(Duration::from_millis(40)); + feedback.request_nack(42, 43, now); + assert_eq!(feedback.take_nack(now, rtt), Some((42, 43))); + assert_eq!( + feedback.take_nack(now + Duration::from_millis(44), rtt), + None + ); + assert!(feedback.resolve_nack(42)); + assert_eq!(feedback.take_nack(now + NACK_TRACKING_TIMEOUT, rtt), None); + assert!(!feedback.resolve_nack(43)); +} + +#[test] +fn mjolnir_nack_sender_uses_fresh_rtt_with_bounded_unknown_and_stale_fallbacks() { + for (video_rtt, stale, expected) in [ + (None, false, vec![0]), + (Some(Duration::from_millis(5)), true, vec![0]), + (Some(Duration::ZERO), false, vec![0, 4, 8]), + (Some(Duration::from_millis(20)), false, vec![0, 24]), + ] { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + if let Some(rtt) = video_rtt { + feedback.publish_ping( + true, + if stale { + now - STREAM_PING_TIMEOUT + } else { + now + }, + rtt, + ); + } + feedback.request_nack(42, 42, now); + let mut sent_at = Vec::new(); + for millis in 0..=52 { + let at = now + Duration::from_millis(millis); + send_pending_nack(&feedback, at, &mut local, channels, true, 1, 2); + let messages = receive_messages(&mut local, &mut remote, at); + assert!(messages.len() <= 1); + if !messages.is_empty() { + sent_at.push(millis); + } + } + assert_eq!(sent_at, expected, "RTT {video_rtt:?}, stale={stale}"); + } +} + +#[test] +fn mjolnir_nack_sender_adapts_to_new_rtt_without_resetting_attempts() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + feedback.publish_ping(false, now, Duration::from_millis(5)); + feedback.request_nack(42, 42, now); + for (millis, new_video_rtt, expected) in [ + (0, None, 1), + (8, None, 0), + (9, Some(20), 0), + (23, None, 0), + (24, None, 1), + (28, Some(1), 0), + (29, None, 1), + (34, None, 0), + ] { + let at = now + Duration::from_millis(millis); + if let Some(rtt) = new_video_rtt { + feedback.publish_ping(true, at, Duration::from_millis(rtt)); + } + send_pending_nack(&feedback, at, &mut local, channels, true, 1, 2); + assert_eq!( + receive_messages(&mut local, &mut remote, at).len(), + expected, + "at {millis} ms" + ); + } + assert!(feedback.resolve_nack(42)); +} + +#[test] +fn generic_nack_sender_keeps_fixed_retries_even_with_a_live_rtt() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + feedback.publish_ping(true, now, Duration::from_millis(40)); + feedback.request_nack(42, 42, now); + let mut sent_at = Vec::new(); + for millis in 0..=52 { + let at = now + Duration::from_millis(millis); + send_pending_nack(&feedback, at, &mut local, channels, false, 1, 2); + for (channel, _) in receive_messages(&mut local, &mut remote, at) { + assert_eq!(channel, "rtcp_on_sctp_private"); + sent_at.push(millis); + } + } + assert_eq!(sent_at, [0, 4, 8]); +} + +#[test] +fn frame_invalidation_uses_the_idr_control_channel_once_and_retries_rejected_sends() { + let (mut local, mut remote, channels, _) = connect_bundle_pair(); + let now = Instant::now() + Duration::from_secs(1); + receive_messages(&mut local, &mut remote, now); + let feedback = NvstFeedbackState::default(); + feedback.frame_invalidation.queue((41, 43)); + send_pending_frame_invalidation(&feedback, now, &mut local, channels); + send_pending_frame_invalidation(&feedback, now, &mut local, channels); + assert_eq!( + receive_messages(&mut local, &mut remote, now), + vec![( + "control_channel_reliable".to_owned(), + frame_invalidation_range(41, 43, 0).encoded(), + )] + ); + assert!(feedback.frame_invalidation.stats().starts_with("rfiSent=1 ")); + assert!(!feedback.keyframe_request_pending()); + + local + .direct_api() + .close_data_channel(channels.control_reliable); + receive_messages(&mut local, &mut remote, now); + feedback.frame_invalidation.queue((44, 50)); + send_pending_frame_invalidation(&feedback, now, &mut local, channels); + assert!(receive_messages(&mut local, &mut remote, now).is_empty()); + assert_eq!(feedback.frame_invalidation.take(), Some((44, 50))); + assert!(feedback.frame_invalidation.stats().starts_with("rfiSent=1 ")); +} + +#[test] +fn extending_a_nack_keeps_each_packets_original_deadline_and_attempts() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(10, 11, now); + assert_eq!(feedback.take_nack(now, None), Some((10, 11))); + feedback.request_nack(11, 12, now + Duration::from_millis(40)); + assert_eq!( + feedback.take_nack(now + Duration::from_millis(40), None), + Some((10, 12)) + ); + assert!(feedback.resolve_nack(11)); + assert_eq!( + feedback.take_nack(now + NACK_TRACKING_TIMEOUT, None), + Some((12, 12)) + ); + assert!(!feedback.resolve_nack(10)); +} + +#[test] +fn nack_overflow_requests_idr_without_evicting_older_losses() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(0, u64::MAX, now); + assert_eq!( + feedback.pending_nacks.lock().unwrap().len(), + MAX_PENDING_NACK_PACKETS + ); + assert!(feedback.keyframe_request_pending()); + assert_eq!(feedback.nack_overflows.load(Ordering::Relaxed), 1); + assert_eq!(feedback.take_nack(now, None), Some((0, 63))); + for index in (1..MAX_PENDING_NACK_PACKETS as u64).step_by(2) { + feedback.resolve_nack(index); + } + assert_eq!( + feedback.pending_nacks.lock().unwrap().len(), + MAX_PENDING_NACK_PACKETS / 2 + ); + feedback.request_nack( + MAX_PENDING_NACK_PACKETS as u64, + MAX_PENDING_NACK_PACKETS as u64 + 2048, + now, + ); + assert_eq!( + feedback.pending_nacks.lock().unwrap().len(), + MAX_PENDING_NACK_PACKETS + ); +} + +#[test] +fn resolving_holes_and_readding_ranges_does_not_reset_nack_attempts() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.request_nack(0, 128, now); + for attempt in 0..MAX_NACK_ATTEMPTS { + let at = now + NACK_RETRY_INTERVAL * u32::from(attempt); + assert_eq!(feedback.take_nack(at, None), Some((0, 63))); + assert_eq!(feedback.take_nack(at, None), Some((64, 127))); + assert_eq!(feedback.take_nack(at, None), Some((128, 128))); + } + assert!(feedback.resolve_nack(65)); + feedback.request_nack(64, 129, now + Duration::from_millis(20)); + assert_eq!( + feedback.take_nack(now + Duration::from_millis(20), None), + Some((65, 65)) + ); + assert_eq!( + feedback.take_nack(now + Duration::from_millis(20), None), + Some((129, 129)) + ); + assert_eq!( + feedback.take_nack(now + Duration::from_millis(20), None), + None + ); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_network.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_network.rs new file mode 100644 index 000000000..20aa7aff3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_network.rs @@ -0,0 +1,112 @@ +fn known_vpn_adapter(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + [ + "cloudflarewarp", + "cloudflare warp", + "wireguard", + "wintun", + "tap-windows", + "openvpn", + "tailscale", + "zerotier", + "nordlynx", + ] + .iter() + .any(|marker| name.contains(marker)) +} + +#[cfg(any(target_os = "linux", target_os = "macos", test))] +fn numbered_interface(name: &str, prefix: &str) -> bool { + name.strip_prefix(prefix).is_some_and(|suffix| { + !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) + }) +} + +#[cfg(target_os = "linux")] +pub(super) fn is_vpn_interface(name: &str) -> bool { + known_vpn_adapter(name) + || numbered_interface(name, "wg") + || std::path::Path::new("/sys/class/net") + .join(name) + .join("tun_flags") + .is_file() +} + +#[cfg(target_os = "macos")] +pub(super) fn is_vpn_interface(name: &str) -> bool { + known_vpn_adapter(name) + || ["utun", "tun", "tap", "wg"] + .iter() + .any(|prefix| numbered_interface(name, prefix)) +} + +#[cfg(windows)] +pub(super) fn is_vpn_interface(name: &str) -> bool { + use windows_sys::Win32::NetworkManagement::IpHelper::{ + ConvertInterfaceNameToLuidA, GetIfEntry2, MIB_IF_ROW2, + }; + + let Ok(name) = std::ffi::CString::new(name) else { + return false; + }; + let mut row = MIB_IF_ROW2::default(); + if unsafe { ConvertInterfaceNameToLuidA(name.as_ptr().cast(), &mut row.InterfaceLuid) } != 0 + || unsafe { GetIfEntry2(&mut row) } != 0 + { + return false; + } + let end = row + .Description + .iter() + .position(|value| *value == 0) + .unwrap_or(row.Description.len()); + known_vpn_adapter(&String::from_utf16_lossy(&row.Description[..end])) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +pub(super) fn is_vpn_interface(_name: &str) -> bool { + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vpn_adapter_detection_recognizes_tunnel_drivers_not_generic_virtual_adapters() { + for name in [ + "CloudflareWARP", + "Cloudflare WARP Interface Tunnel", + "WireGuard Tunnel", + "Wintun Userspace Tunnel", + "TAP-Windows Adapter V9", + "OpenVPN Data Channel Offload", + "Tailscale Tunnel", + "ZeroTier Virtual Port", + "NordLynx Tunnel", + ] { + assert!(known_vpn_adapter(name), "{name}"); + } + for name in [ + "Ethernet", + "Wi-Fi", + "Intel Ethernet Controller", + "Hyper-V Virtual Ethernet Adapter", + "lo", + "en0", + "eth0", + "wg-home-ethernet", + ] { + assert!(!known_vpn_adapter(name), "{name}"); + } + } + + #[test] + fn numbered_tunnel_names_require_an_exact_prefix_and_numeric_suffix() { + assert!(numbered_interface("utun4", "utun")); + assert!(numbered_interface("wg0", "wg")); + assert!(!numbered_interface("wg", "wg")); + assert!(!numbered_interface("wg-office", "wg")); + assert!(!numbered_interface("ethernet0", "wg")); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_ping_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_ping_tests.rs new file mode 100644 index 000000000..1100997cf --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_ping_tests.rs @@ -0,0 +1,530 @@ +use super::*; + +fn continuous_stun_media_deadline(valid: bool, first_frame: bool) { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let client_address = client.local_addr().unwrap(); + let mut config = config(); + config.client_udp_port = client_address.port(); + config.video_peer = server.local_addr().unwrap(); + config.timeout = Duration::from_millis(100); + config.startup_timeout = config.timeout; + config.stun_credentials = Some(stun_credentials()); + let first = protect_for_test( + &test_srtp(&config), + build_plaintext_rtp( + 1, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 42, + &[0, 0, 1, 0x65], + ), + 0, + ); + let (media, frames) = mpsc::sync_channel(8); + let (events, received) = mpsc::channel(); + let session = spawn_nvst_mjolnir_receiver(client, config, media, events).unwrap(); + if first_frame { + server.send_to(&first, client_address).unwrap(); + assert_eq!( + frames.recv_timeout(Duration::from_secs(1)).unwrap().frame_index, + Some(42) + ); + } + let mut stun = build_authenticated_stun_packet( + STUN_BINDING_REQUEST, + &[0x11; 12], + stun_credentials().local_password.as_bytes(), + &[(STUN_ATTR_USERNAME, b"loc1:remote01".to_vec())], + ); + if !valid { + *stun.last_mut().unwrap() ^= 1; + assert!(matches!( + handle_stun_datagram(&stun, server.local_addr().unwrap(), &stun_credentials()), + StunDatagram::Invalid + )); + } + let deadline = Instant::now() + Duration::from_millis(450); + while Instant::now() < deadline { + server.send_to(&stun, client_address).unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert!(matches!( + received.try_recv().expect("media timeout during continuous STUN"), + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { .. }) + )); + assert!(received.try_recv().is_err()); + assert!(frames.try_recv().is_err()); + session.stop(); +} + +#[test] +fn valid_stun_does_not_starve_startup_media_timeout() { + continuous_stun_media_deadline(true, false); +} + +#[test] +fn invalid_stun_does_not_starve_startup_media_timeout() { + continuous_stun_media_deadline(false, false); +} + +#[test] +fn valid_stun_does_not_starve_media_timeout_after_first_frame() { + continuous_stun_media_deadline(true, true); +} + +#[test] +fn invalid_stun_does_not_starve_media_timeout_after_first_frame() { + continuous_stun_media_deadline(false, true); +} + +#[test] +fn selected_ice_ping_does_not_count_duplicate_replies_as_new_round_trips() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut rtc = create_nvst_bundle_rtc(&socket).unwrap(); + let local: SocketAddr = "192.0.2.1:1000".parse().unwrap(); + let remote: SocketAddr = "192.0.2.2:2000".parse().unwrap(); + let credentials = stun_credentials(); + rtc.add_local_candidate(Candidate::host(local, "udp").unwrap()); + rtc.add_remote_candidate(Candidate::host(remote, "udp").unwrap()); + rtc.direct_api().set_remote_ice_credentials(IceCreds { + ufrag: credentials.remote_username_fragment.clone(), + pass: credentials.remote_password.clone(), + }); + rtc.direct_api().set_ice_controlling(true); + let feedback = NvstFeedbackState::default(); + let mut previous_responses = 0; + let start = Instant::now(); + let mut last_response = None; + let mut samples = Vec::new(); + let mut ice_responses = IceResponseTracker::default(); + + for tick in 0..=350 { + let now = start + Duration::from_millis(tick * 10); + rtc.handle_input(Input::Timeout(now)).unwrap(); + loop { + match rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) + if packet.contents.get(..2) == Some(&STUN_BINDING_REQUEST.to_be_bytes()) => + { + ice_responses.sent(&packet.contents); + if tick >= 50 { + continue; + } + let transaction_id = packet.contents[8..20].try_into().unwrap(); + let response = synthesize_ice_binding_success( + &transaction_id, + local, + &credentials.remote_password, + ); + assert!(ice_responses.accept(&response, &credentials.remote_password)); + rtc.handle_input(Input::Receive( + now + Duration::from_millis(5), + Receive { + proto: RtcProtocol::Udp, + source: remote, + destination: local, + contents: response.as_slice().try_into().unwrap(), + }, + )) + .unwrap(); + last_response = Some(response); + } + Output::Event(Event::PeerStats(stats)) => { + feedback.update_ice_ping( + stats.selected_candidate_pair.as_ref(), + &mut previous_responses, + now, + ); + if let Some(sample) = feedback.ping_ms(now) { + samples.push(sample); + } + } + _ => {} + } + } + if tick == 100 || tick == 200 { + let response = last_response.as_ref().unwrap(); + if !ice_responses.accept(response, &credentials.remote_password) { + continue; + } + rtc.handle_input(Input::Receive( + now, + Receive { + proto: RtcProtocol::Udp, + source: remote, + destination: local, + contents: response.as_slice().try_into().unwrap(), + }, + )) + .unwrap(); + } + } + assert_eq!(samples.len(), 3); + assert!(samples.iter().all(|sample| *sample <= 5.0), "{samples:?}"); +} + +#[test] +fn ice_response_tracking_ignores_only_authenticated_duplicates_of_sent_requests() { + let credentials = stun_credentials(); + let request = build_stun_binding_request(&credentials, &[1; 12]); + let response = success([1; 12]); + let mut tracker = IceResponseTracker::default(); + assert!(tracker.accept(&response, &credentials.remote_password)); + tracker.sent(&request); + let invalid_integrity = build_authenticated_stun_packet( + STUN_BINDING_SUCCESS_RESPONSE, + &[1; 12], + b"wrong-password", + &[], + ); + let mut invalid_fingerprint = response.clone(); + *invalid_fingerprint.last_mut().unwrap() ^= 1; + for packet in [vec![], request.clone(), invalid_integrity, invalid_fingerprint] { + assert!(tracker.accept(&packet, &credentials.remote_password)); + assert!(!tracker.requests.front().unwrap().1); + } + assert!(tracker.accept(&response, &credentials.remote_password)); + assert!(!tracker.accept(&response, &credentials.remote_password)); + tracker.sent(&request); + assert!(!tracker.accept(&response, &credentials.remote_password)); + assert_eq!(tracker.requests.len(), 1); + assert!(tracker.accept(&success([2; 12]), &credentials.remote_password)); +} + +#[test] +fn ice_response_tracking_handles_optional_fingerprints_and_resets_per_session() { + let credentials = stun_credentials(); + let mut tracker = IceResponseTracker::default(); + tracker.sent(&build_stun_binding_request(&credentials, &[1; 12])); + let mut response = success([1; 12]); + response.truncate(response.len() - 8); + let length = (response.len() - STUN_HEADER_LEN) as u16; + response[2..4].copy_from_slice(&length.to_be_bytes()); + assert!(tracker.accept(&response, &credentials.remote_password)); + assert!(!tracker.accept(&response, &credentials.remote_password)); + let mut restarted = IceResponseTracker::default(); + restarted.sent(&build_stun_binding_request(&credentials, &[1; 12])); + assert!(restarted.accept(&response, &credentials.remote_password)); +} + +#[test] +fn ice_response_tracking_is_bounded_without_tracking_non_ice_traffic() { + let credentials = stun_credentials(); + let mut tracker = IceResponseTracker::default(); + for packet in [vec![], b"PING".to_vec(), success([0; 12])] { + tracker.sent(&packet); + } + assert!(tracker.requests.is_empty()); + for id in 0..=MAX_PENDING_STREAM_PINGS { + tracker.sent(&build_stun_binding_request(&credentials, &[id as u8; 12])); + } + assert_eq!(tracker.requests.len(), MAX_PENDING_STREAM_PINGS); + assert_eq!(tracker.requests.front().unwrap().0, [1; 12]); + assert!(tracker.accept(&success([1; 12]), &credentials.remote_password)); + assert!(!tracker.accept(&success([1; 12]), &credentials.remote_password)); +} + +#[test] +fn ice_response_tracking_preserves_reordering_and_genuinely_slow_first_replies() { + let credentials = stun_credentials(); + let mut ice_responses = IceResponseTracker::default(); + let mut ping = StreamPingTracker::default(); + let start = Instant::now(); + for id in [1, 2] { + ice_responses.sent(&build_stun_binding_request(&credentials, &[id; 12])); + ping.sent([id; 12], start); + } + for (id, elapsed) in [(2, Duration::from_millis(5)), (1, Duration::from_secs(2))] { + let response = success([id; 12]); + assert!(ice_responses.accept(&response, &credentials.remote_password)); + assert_eq!( + ping.receive(&response, peer(), &credentials, start + elapsed), + Some(elapsed) + ); + assert!(!ice_responses.accept(&response, &credentials.remote_password)); + } +} + +fn success(transaction_id: [u8; 12]) -> Vec { + build_authenticated_stun_packet( + STUN_BINDING_SUCCESS_RESPONSE, + &transaction_id, + stun_credentials().remote_password.as_bytes(), + &[], + ) +} + +#[test] +fn stream_ping_matches_authenticated_transactions_once_and_handles_reordering() { + let mut tracker = StreamPingTracker::default(); + let start = Instant::now(); + let credentials = stun_credentials(); + tracker.sent([1; 12], start); + tracker.sent([2; 12], start + Duration::from_millis(100)); + let received = start + Duration::from_micros(125_500); + assert_eq!( + tracker.receive(&success([3; 12]), peer(), &credentials, received), + None + ); + assert_eq!( + tracker.receive(&success([2; 12]), peer(), &credentials, received), + Some(Duration::from_micros(25_500)) + ); + assert_eq!( + tracker.receive(&success([2; 12]), peer(), &credentials, received), + None + ); + assert_eq!( + tracker.receive(&success([1; 12]), peer(), &credentials, received), + Some(Duration::from_micros(125_500)) + ); + assert!(tracker.pending.is_empty()); +} + +#[test] +fn stream_ping_rejects_invalid_packets_without_consuming_the_request() { + let mut tracker = StreamPingTracker::default(); + let start = Instant::now(); + let credentials = stun_credentials(); + tracker.sent([1; 12], start); + let mut bad_fingerprint = success([1; 12]); + *bad_fingerprint.last_mut().unwrap() ^= 1; + let bad_integrity = + build_authenticated_stun_packet(STUN_BINDING_SUCCESS_RESPONSE, &[1; 12], b"wrong-key", &[]); + let request = build_stun_binding_request(&credentials, &[1; 12]); + let error_response = build_authenticated_stun_packet( + 0x0111, + &[1; 12], + credentials.remote_password.as_bytes(), + &[], + ); + for packet in [ + vec![], + vec![1; 19], + bad_fingerprint, + bad_integrity, + request, + error_response, + ] { + assert_eq!(tracker.receive(&packet, peer(), &credentials, start), None); + assert_eq!(tracker.pending.len(), 1); + } + assert_eq!( + tracker.receive(&success([1; 12]), peer(), &credentials, start), + Some(Duration::ZERO) + ); +} + +#[test] +fn stream_ping_bounds_pending_requests_and_expires_late_replies() { + let mut tracker = StreamPingTracker::default(); + let start = Instant::now(); + let credentials = stun_credentials(); + for id in 0..=MAX_PENDING_STREAM_PINGS { + tracker.sent([id as u8; 12], start); + } + assert_eq!(tracker.pending.len(), MAX_PENDING_STREAM_PINGS); + assert_eq!( + tracker.receive(&success([0; 12]), peer(), &credentials, start), + None + ); + assert_eq!( + tracker.receive( + &success([1; 12]), + peer(), + &credentials, + start + STREAM_PING_TIMEOUT + ), + None + ); + tracker.sent([99; 12], start + STREAM_PING_TIMEOUT); + assert_eq!(tracker.pending.len(), 1); + assert_eq!(tracker.pending.front().unwrap().0, [99; 12]); +} + +#[test] +fn stream_ping_keeps_fresh_source_priority_when_other_samples_arrive_later() { + let feedback = NvstFeedbackState::default(); + let start = Instant::now(); + *feedback.ice_ping.lock().unwrap() = Some((start, Duration::from_millis(10))); + feedback.publish_ping( + true, + start + Duration::from_secs(1), + Duration::from_millis(30), + ); + feedback.publish_ping( + false, + start + Duration::from_secs(2), + Duration::from_millis(50), + ); + assert_eq!(feedback.ping_ms(start + Duration::from_secs(2)), Some(10.0)); + assert_eq!(feedback.ping_ms(start + STREAM_PING_TIMEOUT), Some(30.0)); + feedback.publish_ping( + false, + start + STREAM_PING_TIMEOUT, + Duration::from_millis(60), + ); + assert_eq!(feedback.ping_ms(start + STREAM_PING_TIMEOUT), Some(30.0)); + assert_eq!( + feedback.ping_ms(start + STREAM_PING_TIMEOUT + Duration::from_secs(1)), + Some(60.0) + ); +} + +#[test] +fn stream_ping_prefers_video_with_fresh_bundle_fallback_and_resets_per_session() { + let feedback = NvstFeedbackState::default(); + let start = Instant::now(); + assert_eq!(feedback.ping_ms(start), None); + feedback.publish_ping(false, start, Duration::from_micros(22_500)); + assert_eq!(feedback.ping_ms(start), Some(22.5)); + feedback.publish_ping(true, start, Duration::from_millis(30)); + assert_eq!(feedback.ping_ms(start), Some(30.0)); + feedback.publish_ping( + false, + start + Duration::from_secs(1), + Duration::from_millis(24), + ); + assert_eq!(feedback.ping_ms(start + STREAM_PING_TIMEOUT), Some(24.0)); + assert_eq!( + feedback.ping_ms(start + STREAM_PING_TIMEOUT + Duration::from_secs(1)), + None + ); + assert_eq!(NvstFeedbackState::default().ping_ms(start), None); +} + +#[test] +fn mjolnir_receiver_publishes_ping_from_its_real_udp_keepalive_reply() { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.video_peer = server.local_addr().unwrap(); + config.stun_credentials = Some(stun_credentials()); + let feedback = config.feedback(); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = + spawn_nvst_mjolnir_receiver(client, config, media_consumer, event_sender).unwrap(); + let mut packet = [0; 512]; + let (length, source) = server.recv_from(&mut packet).unwrap(); + assert!(length >= STUN_HEADER_LEN); + let transaction_id = packet[8..20].try_into().unwrap(); + let response = success(transaction_id); + let unexpected_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); + unexpected_peer.send_to(&response, source).unwrap(); + server.recv_from(&mut packet).unwrap(); + assert_eq!(feedback.ping_ms(Instant::now()), None); + server.send_to(&response, source).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while feedback.ping_ms(Instant::now()).is_none() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + let measured = feedback.ping_ms(Instant::now()); + session.stop(); + assert!(measured.is_some_and(|value| value > 0.0 && value < 2000.0)); +} + +#[test] +fn bundle_receiver_publishes_ping_from_its_real_udp_keepalive_reply() { + let server = UdpSocket::bind("127.0.0.1:0").unwrap(); + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.video_peer = server.local_addr().unwrap(); + config.stun_credentials = Some(stun_credentials()); + config.remote_dtls_fingerprint = Some(["00"; 32].join(":")); + config.ping_payload = b"ping-fixture".to_vec(); + let feedback = config.feedback(); + let (media_consumer, _media_receiver) = mpsc::sync_channel(1); + let (event_sender, _event_receiver) = mpsc::channel(); + let session = spawn_nvst_udp_receiver_with_socket( + config, + media_consumer, + event_sender, + Some(client), + None, + Arc::new(HidRuntime::new()), + None, + ) + .unwrap(); + let mut packet = [0; 2048]; + let deadline = Instant::now() + Duration::from_secs(2); + let mut replied = false; + while Instant::now() < deadline { + let (length, source) = server.recv_from(&mut packet).unwrap(); + if find_stun_attribute(&packet[..length], STUN_ATTR_USERNAME) + .is_some_and(|(_, username)| username == b"PING:loc1") + { + let transaction_id = packet[8..20].try_into().unwrap(); + server.send_to(&success(transaction_id), source).unwrap(); + replied = true; + break; + } + } + while feedback.ping_ms(Instant::now()).is_none() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + let measured = feedback.ping_ms(Instant::now()); + session.stop(); + assert!(replied); + assert!(measured.is_some_and(|value| (0.0..2000.0).contains(&value))); + assert!(feedback.video_ping.lock().unwrap().is_none()); +} + +#[test] +fn selected_ice_pair_ping_takes_priority_without_refreshing_old_samples() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + let mut previous_responses = 0; + let mut pair = CandidatePairStats { + protocol: RtcProtocol::Udp, + local: str0m::stats::CandidateStats { + addr: "127.0.0.1:5000".parse().unwrap(), + }, + remote: str0m::stats::CandidateStats { addr: peer() }, + current_round_trip_time: None, + total_round_trip_time: Duration::ZERO, + responses_received: 0, + }; + feedback.update_ice_ping(Some(&pair), &mut previous_responses, now); + assert_eq!(feedback.ping_ms(now), None); + feedback.publish_ping(true, now, Duration::from_millis(40)); + pair.responses_received = 1; + pair.current_round_trip_time = Some(Duration::from_micros(25_500)); + feedback.update_ice_ping(Some(&pair), &mut previous_responses, now); + assert_eq!(feedback.ping_ms(now), Some(25.5)); + feedback.publish_ping( + true, + now + Duration::from_secs(1), + Duration::from_millis(30), + ); + feedback.update_ice_ping( + Some(&pair), + &mut previous_responses, + now + STREAM_PING_TIMEOUT, + ); + assert_eq!(feedback.ping_ms(now + STREAM_PING_TIMEOUT), Some(30.0)); + pair.responses_received = 2; + pair.current_round_trip_time = Some(Duration::ZERO); + feedback.update_ice_ping( + Some(&pair), + &mut previous_responses, + now + STREAM_PING_TIMEOUT, + ); + assert_eq!(feedback.ping_ms(now + STREAM_PING_TIMEOUT), Some(0.0)); + feedback.update_ice_ping(None, &mut previous_responses, now + STREAM_PING_TIMEOUT); + assert_eq!(feedback.ping_ms(now + STREAM_PING_TIMEOUT), Some(30.0)); + assert_eq!(previous_responses, 0); + feedback.update_ice_ping( + Some(&pair), + &mut previous_responses, + now + STREAM_PING_TIMEOUT, + ); + assert_eq!(feedback.ping_ms(now + STREAM_PING_TIMEOUT), Some(0.0)); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_progress_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_progress_tests.rs new file mode 100644 index 000000000..7e3c3ccb8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_progress_tests.rs @@ -0,0 +1,939 @@ +use super::*; + +use std::sync::mpsc::Sender; + +fn progress_policy() -> NvstFrameProgressPolicy { + NvstFrameProgressPolicy { + stall: Duration::from_secs(2), + keyframe_grace: Duration::from_secs(1), + } +} + +struct DeliveryHarness { + media: MediaConsumer, + events: Sender, + feedback: SharedNvstFeedback, + gaps: Vec, + resumed: usize, +} + +impl DeliveryHarness { + fn new(feedback: SharedNvstFeedback) -> Self { + let (media, _media_receiver) = std::sync::mpsc::sync_channel(512); + let (events, _event_receiver) = std::sync::mpsc::channel(); + Self { + media, + events, + feedback, + gaps: Vec::new(), + resumed: 0, + } + } + + fn deliver( + &mut self, + receiver: &mut NvstVideoReceiver, + source: SocketAddr, + datagram: &[u8], + now: Instant, + ) { + self.gaps.push(false); + let gap = self.gaps.last_mut().expect("gap slot"); + for event in receiver.process_datagram(source, datagram, now) { + if matches!(event, NvstReceiveEvent::FrameProgressResumed) { + self.resumed += 1; + } + let _ = forward_receive_event( + &self.media, + &self.events, + &self.feedback, + now, + now, + gap, + event, + ); + } + } +} + +fn protect_partial_frame(crypto: &SrtpReceiver, index: u16, frame_index: u32) -> Vec { + let mut flags = FLAG_CONTAINS_PIC_DATA; + if index == 0 { + flags |= FLAG_SOF; + } + let mut payload = vec![0x55_u8; 64]; + if index == 0 { + payload[..5].copy_from_slice(&[0, 0, 0, 1, 0x61]); + } + protect_for_test( + crypto, + build_plaintext_rtp(8_000 + index, flags, frame_index, &payload), + 0, + ) +} + +fn protect_complete_frame(crypto: &SrtpReceiver, sequence: u16, frame_index: u32) -> Vec { + protect_for_test( + crypto, + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + frame_index, + &[0, 0, 0, 1, 0x65, 0x11, 0x22], + ), + 0, + ) +} + +#[test] +fn first_keyframe_deadline_survives_continuing_non_keyframes() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + let mut events = Vec::new(); + + for index in 0..9_u16 { + let now = origin + Duration::from_millis(400) * u32::from(index); + let packet = protect_for_test( + &crypto, + build_plaintext_rtp( + 400 + index, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + u32::from(index), + &[0, 0, 0, 1, 0x61, 0x11], + ), + 0, + ); + harness.deliver(&mut receiver, peer(), &packet, now); + if let Some(event) = receiver.poll_frame_progress(now, policy) { + events.push((now.saturating_duration_since(origin), event)); + } + } + + assert_eq!(feedback.frame_stage_timings().assembled_frames_total, 9); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, Duration::from_secs(2)); + assert!(matches!(events[0].1, NvstFrameProgressEvent::KeyframeRequested { .. })); + assert_eq!(events[1].0, Duration::from_millis(3_200)); + assert!(matches!(events[1].1, NvstFrameProgressEvent::RecoveryNeeded { .. })); + assert!(feedback.keyframe_request_pending()); +} + +#[test] +fn first_keyframe_inside_grace_clears_pending_recovery() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + for index in 0..7_u16 { + let now = origin + Duration::from_millis(400) * u32::from(index); + let packet = protect_for_test( + &crypto, + build_plaintext_rtp( + 500 + index, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + u32::from(index), + &[0, 0, 0, 1, 0x61, 0x11], + ), + 0, + ); + harness.deliver(&mut receiver, peer(), &packet, now); + let result = receiver.poll_frame_progress(now, policy); + if index == 5 { + assert!(matches!(result, Some(NvstFrameProgressEvent::KeyframeRequested { .. }))); + } else { + assert_eq!(result, None); + } + } + assert!(feedback.keyframe_request_pending()); + let keyframe_at = origin + Duration::from_millis(2_800); + let keyframe = protect_complete_frame(&crypto, 507, 7); + harness.deliver(&mut receiver, peer(), &keyframe, keyframe_at); + assert_eq!(harness.resumed, 1); + assert!(!feedback.keyframe_request_pending()); + assert_eq!(receiver.poll_frame_progress(keyframe_at + Duration::from_millis(400), policy), None); + assert_eq!(receiver.frame_progress().stage, NvstFrameProgressStage::Tracking); +} + +#[test] +fn incomplete_initial_idr_does_not_close_grace_before_a_complete_reference() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + let partial = protect_for_test( + &crypto, + build_plaintext_rtp(900, FLAG_SOF | FLAG_CONTAINS_PIC_DATA, 1, &[0, 0, 0, 1, 0x65]), + 0, + ); + assert!(receiver.process_datagram(peer(), &partial, origin).is_empty()); + assert_eq!(receiver.poll_frame_progress(origin, policy), None); + assert!(matches!( + receiver.poll_frame_progress(origin + Duration::from_secs(2), policy), + Some(NvstFrameProgressEvent::KeyframeRequested { .. }) + )); + assert!(feedback.keyframe_request_pending()); + + let incomplete_at = origin + Duration::from_millis(2_400); + let mut missing_middle = build_plaintext_rtp(901, FLAG_EOF, 1, &[0xaa]); + missing_middle[16..20].copy_from_slice(&(902_u32 << 8).to_le_bytes()); + let missing_middle = protect_for_test(&crypto, missing_middle, 0); + assert_eq!( + receiver.process_datagram(peer(), &missing_middle, incomplete_at), + [NvstReceiveEvent::Dropped(NvstDropReason::FrameDiscontinuity)] + ); + assert_eq!(receiver.poll_frame_progress(incomplete_at, policy), None); + assert_eq!(receiver.frame_progress().stage, NvstFrameProgressStage::KeyframePending); + assert!(feedback.keyframe_request_pending()); + + let complete_at = origin + Duration::from_millis(2_800); + let next_idr = protect_complete_frame(&crypto, 902, 2); + assert!(matches!( + receiver.process_datagram(peer(), &next_idr, complete_at).as_slice(), + [NvstReceiveEvent::FrameProgressResumed, NvstReceiveEvent::Frame(frame)] + if frame.keyframe && !frame.contiguous + )); + feedback.publish_assembled_frame(2, complete_at); + assert!(!feedback.keyframe_request_pending()); + assert_eq!(receiver.poll_frame_progress(complete_at, policy), None); +} + +#[test] +fn partial_packets_that_never_complete_reach_keyframe_then_recovery() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let step = Duration::from_millis(400); + let policy = progress_policy(); + + let mut events = Vec::new(); + for index in 0..25_u16 { + let now = origin + step * u32::from(index); + let protected = protect_partial_frame(&crypto, index, u32::from(index) / 3); + harness.deliver(&mut receiver, peer(), &protected, now); + if let Some(event) = receiver.poll_frame_progress(now, policy) { + events.push((now.saturating_duration_since(origin), event)); + } + } + + assert_eq!(feedback.frame_stage_timings().assembled_frames_total, 0); + assert_eq!( + events.len(), + 2, + "escalation must be bounded to one keyframe stage and one recovery stage" + ); + assert!(matches!( + events[0].1, + NvstFrameProgressEvent::KeyframeRequested { .. } + )); + assert_eq!(events[0].0, Duration::from_millis(2_000)); + assert!(matches!( + events[1].1, + NvstFrameProgressEvent::RecoveryNeeded { .. } + )); + assert_eq!(events[1].0, Duration::from_millis(3_200)); + assert!(feedback.keyframe_request_pending()); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::RecoveryRequired + ); +} + +#[test] +fn later_packet_arrivals_do_not_postpone_the_produced_frame_deadline() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + let mut events = Vec::new(); + for index in 0..45_u16 { + let now = origin + Duration::from_millis(100) * u32::from(index); + let protected = protect_partial_frame(&crypto, index, u32::from(index) / 3); + harness.deliver(&mut receiver, peer(), &protected, now); + if let Some(event) = receiver.poll_frame_progress(now, policy) { + events.push((now.saturating_duration_since(origin), event)); + } + } + + assert_eq!( + events, + [ + ( + Duration::from_secs(2), + NvstFrameProgressEvent::KeyframeRequested { + idle_for: Duration::from_secs(2), + last_assembled_frame_index: None, + } + ), + ( + Duration::from_secs(3), + NvstFrameProgressEvent::RecoveryNeeded { + idle_for: Duration::from_secs(3), + last_assembled_frame_index: None, + } + ), + ], + "the deadline is anchored at the first authenticated packet, so continuing packets \ + must not push it forward" + ); +} + +#[test] +fn watchdog_stays_silent_before_the_first_authenticated_packet() { + let config = config(); + let mut receiver = NvstVideoReceiver::new(config); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + assert_eq!( + receiver.poll_frame_progress(origin + Duration::from_secs(30), policy), + None, + "startup before any authenticated video belongs to the packet timeout" + ); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking + ); +} + +#[test] +fn assembled_frames_inside_the_threshold_keep_the_watchdog_tracking() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let step = Duration::from_millis(400); + let policy = progress_policy(); + + for index in 0..20_u16 { + let now = origin + step * u32::from(index); + let protected = protect_complete_frame(&crypto, 500 + index, u32::from(index)); + harness.deliver(&mut receiver, peer(), &protected, now); + assert_eq!(receiver.poll_frame_progress(now, policy), None); + } + assert_eq!( + feedback.frame_stage_timings().assembled_frames_total, + 20, + "every healthy frame must be assembled" + ); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking + ); + assert_eq!(receiver.frame_progress().last_assembled_frame_index, Some(19)); + assert_eq!(receiver.frame_progress().last_assembled_at, Some(origin + step * 19)); +} + +#[test] +fn keyframe_inside_the_grace_restores_progress_without_recovery() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback); + let origin = receiver.timeout_origin; + let step = Duration::from_millis(400); + let policy = progress_policy(); + + let mut events = Vec::new(); + for index in 0..9_u16 { + let now = origin + step * u32::from(index); + let protected = if index == 7 { + protect_complete_frame(&crypto, 9_000, 42) + } else { + protect_partial_frame(&crypto, index, u32::from(index) / 3) + }; + harness.deliver(&mut receiver, peer(), &protected, now); + if let Some(event) = receiver.poll_frame_progress(now, policy) { + events.push(event); + } + } + + assert_eq!( + events, + [NvstFrameProgressEvent::KeyframeRequested { + idle_for: Duration::from_millis(2_000), + last_assembled_frame_index: None, + }] + ); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking + ); + assert_eq!(receiver.frame_progress().last_assembled_frame_index, Some(42)); + assert_eq!( + receiver.frame_progress().last_assembled_at, + Some(origin + step * 7) + ); +} + +#[test] +fn packet_silence_stays_with_the_packet_timeout_and_not_the_progress_watchdog() { + let config = config(); + let mut receiver = NvstVideoReceiver::new(config); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + let now = origin + receiver.config.timeout; + + assert!(matches!( + receiver.poll_timeout(now), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { .. })) + )); + assert_eq!(receiver.poll_frame_progress(now, policy), None); +} + +#[test] +fn pause_and_stop_never_report_a_progress_stall() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + let protected = protect_partial_frame(&crypto, 0, 0); + harness.deliver(&mut receiver, peer(), &protected, origin); + + receiver.pause(); + assert_eq!( + receiver.poll_frame_progress(origin + Duration::from_secs(30), policy), + None + ); + + receiver.resume(); + let resumed_origin = receiver.timeout_origin; + assert_eq!( + receiver.poll_frame_progress(resumed_origin + Duration::from_millis(1_900), policy), + None, + "resume must open a fresh stall window instead of inheriting the paused interval" + ); + + receiver.stop(NvstStopCause::Requested); + assert_eq!( + receiver.poll_frame_progress(origin + Duration::from_secs(60), policy), + None + ); +} + +#[test] +fn recovery_is_reported_once_per_episode_and_reset_by_recover() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + let mut index = 0_u16; + let mut now = origin; + while now < origin + Duration::from_secs(10) { + now = origin + Duration::from_millis(400) * u32::from(index); + let protected = protect_partial_frame(&crypto, index, u32::from(index) / 3); + harness.deliver(&mut receiver, peer(), &protected, now); + receiver.poll_frame_progress(now, policy); + index += 1; + } + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::RecoveryRequired + ); + assert_eq!( + receiver.poll_frame_progress(now + Duration::from_secs(60), policy), + None, + "a terminal episode must not re-fire until the media state is reset" + ); + + assert!(receiver.recover().is_some()); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking + ); + let recovered_origin = receiver.timeout_origin; + assert_eq!(receiver.poll_frame_progress(recovered_origin, policy), None); + assert_eq!( + receiver.poll_frame_progress(recovered_origin + Duration::from_secs(2), policy), + None, + "recovery clears the authenticated marker, so the packet timeout owns the next window" + ); +} + +#[test] +fn packet_timeout_supersedes_a_pending_keyframe_grace_without_double_reporting() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + + for index in 0..7_u16 { + let now = origin + Duration::from_millis(400) * u32::from(index); + let protected = protect_partial_frame(&crypto, index, u32::from(index) / 3); + harness.deliver(&mut receiver, peer(), &protected, now); + receiver.poll_frame_progress(now, policy); + } + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::KeyframePending + ); + + let timeout_at = origin + Duration::from_millis(2_400) + receiver.config.timeout; + assert!(matches!( + receiver.poll_timeout(timeout_at), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { .. })) + )); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking, + "the packet timeout must clear the pending progress episode" + ); + assert_eq!( + receiver.poll_frame_progress(timeout_at + Duration::from_secs(60), policy), + None + ); +} + +#[test] +fn udp_receive_loop_reports_frame_progress_recovery_while_partial_packets_continue() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").expect("receiver socket"); + let receiver_port = socket.local_addr().expect("local addr").port(); + let peer_socket = std::net::UdpSocket::bind("127.0.0.1:0").expect("peer socket"); + let peer_port = peer_socket.local_addr().expect("peer addr").port(); + + let handoff = json!({ + "clientUdpPort": receiver_port, + "videoPeerIp": "127.0.0.1", + "videoPeerPort": peer_port, + "srtpAesKeyHex": "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F", + "srtpSaltHex": "00000000000000009ECA935E", + "codec": "H264", + "rtpPayloadType": 96, + "rtpSsrc": 0x11223344u32, + "reorderWindowPackets": 4, + "maxAccessUnitBytes": 4096, + "timeoutMs": 1_000 + }); + let config = NvstVideoConfig::from_legacy_handoff(&handoff, None).expect("valid config"); + let crypto = test_srtp(&config); + + let (media_sender, _media_receiver) = std::sync::mpsc::sync_channel(64); + let (event_sender, event_receiver) = std::sync::mpsc::channel(); + let session = + spawn_nvst_udp_receiver_with_socket( + config, + media_sender, + event_sender, + Some(socket), + None, + Arc::new(HidRuntime::new()), + None, + ) + .expect("receiver spawns"); + + let destination = format!("127.0.0.1:{receiver_port}"); + let feeder = std::thread::spawn(move || { + for index in 0..50_u16 { + let mut flags = FLAG_CONTAINS_PIC_DATA; + if index == 0 { + flags |= FLAG_SOF; + } + let mut payload = vec![0x55_u8; 64]; + if index == 0 { + payload[..5].copy_from_slice(&[0, 0, 0, 1, 0x61]); + } + let packet = protect_for_test( + &crypto, + build_plaintext_rtp(1_000 + index, flags, 7, &payload), + 0, + ); + if peer_socket.send_to(&packet, &destination).is_err() { + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + }); + + let deadline = Instant::now() + Duration::from_secs(20); + let mut recovery = None; + while Instant::now() < deadline { + if let Ok(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::FrameProgress { + idle_for, + last_assembled_frame_index, + })) = event_receiver.recv_timeout(Duration::from_millis(500)) + { + recovery = Some((idle_for, last_assembled_frame_index)); + break; + } + } + + session.stop(); + let _ = feeder.join(); + let (idle_for, last_assembled_frame_index) = + recovery.expect("the receive loop must report a produced-frame stall"); + assert!( + idle_for >= Duration::from_millis(900), + "idle_for={idle_for:?}" + ); + assert_eq!(last_assembled_frame_index, None); +} + +#[test] +fn resumed_progress_is_reported_once_when_an_episode_closes_on_assembly() { + let config = config(); + let crypto = test_srtp(&config); + let feedback = config.feedback(); + let mut receiver = NvstVideoReceiver::new(config); + let mut harness = DeliveryHarness::new(feedback.clone()); + let origin = receiver.timeout_origin; + let policy = progress_policy(); + let mut sequence = 6_000_u16; + + for index in 0..6_u16 { + let now = origin + Duration::from_millis(400) * u32::from(index); + let mut flags = FLAG_CONTAINS_PIC_DATA; + if index == 0 { + flags |= FLAG_SOF; + } + let mut payload = vec![0x55_u8; 64]; + if index == 0 { + payload[..5].copy_from_slice(&[0, 0, 0, 1, 0x61]); + } + let protected = protect_for_test( + &crypto, + build_plaintext_rtp(sequence, flags, 1, &payload), + 0, + ); + sequence += 1; + harness.deliver(&mut receiver, peer(), &protected, now); + receiver.poll_frame_progress(now, policy); + } + assert_eq!(harness.resumed, 0, "no episode may close before one opens"); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::KeyframePending, + "the partial stream must open an episode inside its grace window" + ); + + let mut closed_at = None; + for index in 6..20_u16 { + let now = origin + Duration::from_millis(400) * u32::from(index); + let protected = protect_for_test( + &crypto, + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + 1, + &[0, 0, 0, 1, 0x65, 0x11], + ), + 0, + ); + sequence += 1; + let before = harness.resumed; + harness.deliver(&mut receiver, peer(), &protected, now); + receiver.poll_frame_progress(now, policy); + if harness.resumed > before { + assert!(closed_at.is_none(), "only one episode may close in this run"); + closed_at = Some(index); + } + } + + assert_eq!( + harness.resumed, 1, + "exactly one resumed transition for one closed episode" + ); + assert_eq!(closed_at, Some(6)); + assert_eq!( + feedback.frame_stage_timings().assembled_frames_total, + 14, + "every complete frame in the second phase must be assembled" + ); + assert_eq!( + receiver.frame_progress().stage, + NvstFrameProgressStage::Tracking + ); +} + +#[test] +fn unified_bundle_reports_the_same_frame_stall_events_as_dedicated_video() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let destination = socket.local_addr().unwrap(); + let peer_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.video_peer = peer_socket.local_addr().unwrap(); + config.bundle_peer = Some(config.video_peer); + config.unified_socket = true; + config.timeout = Duration::from_millis(200); + let crypto = test_srtp(&config); + let (media_sender, _media_receiver) = mpsc::sync_channel(64); + let (event_sender, event_receiver) = mpsc::channel(); + let session = spawn_receiver_thread( + "opennow-test-unified-recovery", + socket, + config, + media_sender, + event_sender, + NvstWorkerSetup { + rtc: Some(Rtc::new(Instant::now())), + hid_runtime: None, + upstream_ready: None, + }, + ) + .unwrap(); + let feeder = thread::spawn(move || { + for index in 0..20_u16 { + let packet = protect_partial_frame(&crypto, index, 7); + peer_socket.send_to(&packet, destination).unwrap(); + thread::sleep(Duration::from_millis(40)); + } + }); + let deadline = Instant::now() + Duration::from_secs(3); + let mut states = Vec::new(); + let mut recovered = false; + while Instant::now() < deadline { + match event_receiver.recv_timeout(Duration::from_millis(100)) { + Ok(NvstReceiveEvent::FrameProgressStall { + recovery_required, .. + }) => states.push(recovery_required), + Ok(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::FrameProgress { + last_assembled_frame_index, + .. + })) => { + assert_eq!(last_assembled_frame_index, None); + recovered = true; + break; + } + Ok(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { .. })) => { + panic!("authenticated partial packets must keep media liveness healthy") + } + _ => {} + } + } + session.stop(); + feeder.join().unwrap(); + assert!( + recovered, + "unified Mjolnir must run the assembly-progress watchdog" + ); + assert_eq!(states, [false, true]); +} + +#[test] +fn delayed_play_does_not_consume_the_dedicated_first_video_timeout() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(8); + let feedback = config.feedback(); + feedback.wait_for_play(); + let mut receiver = NvstVideoReceiver::new(config); + let started = receiver.timeout_origin; + let played = started + Duration::from_secs(19); + assert_eq!(receiver.poll_timeout(played), None); + assert_eq!(receiver.state(), NvstReceiverState::Running); + feedback.complete_play(played); + assert_eq!( + receiver.poll_timeout(played + Duration::from_secs(8) - Duration::from_nanos(1)), + None + ); + assert_eq!( + receiver.poll_timeout(played + Duration::from_secs(8)), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { + idle_for: Duration::from_secs(8) + })) + ); + assert_eq!(receiver.poll_timeout(played + Duration::from_secs(9)), None); +} + +#[test] +fn pre_play_video_preserves_the_windows_first_video_allowance() { + for raw in [true, false] { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + config.timeout = Duration::from_secs(8); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + feedback.wait_for_play(); + let mut receiver = NvstVideoReceiver::new(config); + let started = receiver.timeout_origin; + if raw { + receiver.process_datagram(peer(), &protect_partial_frame(&crypto, 0, 7), started); + } else { + receiver.process_mjolnir_payload(1, 0, &[], started); + } + let played = started + Duration::from_secs(19); + feedback.complete_play(played); + assert_eq!(receiver.poll_timeout(played + Duration::from_secs(8)), None); + assert_eq!( + receiver.poll_frame_progress(played + Duration::from_secs(8), progress_policy()), + None + ); + assert_eq!( + receiver.poll_timeout(played + Duration::from_secs(60) - Duration::from_nanos(1)), + None + ); + assert_eq!( + receiver.poll_timeout(played + Duration::from_secs(60)), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { + idle_for: Duration::from_secs(60) + })) + ); + } +} + +#[test] +fn early_partial_video_cannot_start_a_frame_watchdog_before_play() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + feedback.wait_for_play(); + let mut receiver = NvstVideoReceiver::new(config); + let started = receiver.timeout_origin; + let packet = protect_partial_frame(&crypto, 0, 7); + receiver.process_datagram(peer(), &packet, started); + let played = started + Duration::from_secs(19); + assert_eq!(receiver.poll_timeout(played), None); + assert_eq!( + receiver.poll_frame_progress(played, progress_policy()), + None + ); + feedback.complete_play(played); + assert_eq!( + receiver.poll_frame_progress(played, progress_policy()), + None + ); + receiver.process_datagram(peer(), &protect_partial_frame(&crypto, 1, 7), played); + assert_eq!( + receiver.poll_frame_progress( + played + Duration::from_secs(2) - Duration::from_nanos(1), + progress_policy() + ), + None + ); + assert!(matches!( + receiver.poll_frame_progress(played + Duration::from_secs(2), progress_policy()), + Some(NvstFrameProgressEvent::KeyframeRequested { .. }) + )); +} + +#[test] +fn bundle_play_completion_arms_the_shared_dedicated_video_clock() { + let bundle_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let video_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let peer_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut config = config(); + config.video_peer = peer_socket.local_addr().unwrap(); + config.bundle_peer = Some(config.video_peer); + config.mjolnir_udp_port = Some(video_socket.local_addr().unwrap().port()); + config.startup_timeout = Duration::from_millis(120); + let feedback = config.feedback(); + let (play_sender, play_receiver) = mpsc::channel(); + let (media_sender, _media_receiver) = mpsc::sync_channel(64); + let (event_sender, event_receiver) = mpsc::channel(); + let bundle = spawn_receiver_thread( + "opennow-test-play-clock", + bundle_socket, + config.clone(), + media_sender.clone(), + event_sender.clone(), + NvstWorkerSetup { + rtc: Some(Rtc::new(Instant::now())), + hid_runtime: None, + upstream_ready: Some(play_receiver), + }, + ) + .unwrap(); + assert_eq!(feedback.video_timeout_origin(Instant::now()), None); + let video = + spawn_nvst_mjolnir_receiver(video_socket, config, media_sender, event_sender).unwrap(); + let before_play = Instant::now() + Duration::from_millis(300); + while Instant::now() < before_play { + assert!(!matches!( + event_receiver.recv_timeout(Duration::from_millis(20)), + Ok(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::Timeout { .. } + )) + )); + } + let played = Instant::now(); + play_sender.send(()).unwrap(); + let deadline = played + Duration::from_secs(2); + let mut timeout = None; + while Instant::now() < deadline { + if let Ok(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { idle_for })) = + event_receiver.recv_timeout(Duration::from_millis(20)) + { + timeout = Some(idle_for); + break; + } + } + video.stop(); + bundle.stop(); + assert!(timeout.is_some_and(|idle| idle >= Duration::from_millis(120))); + assert!( + matches!(*feedback.video_startup.lock().unwrap(), VideoStartup::PlayComplete(at) if at >= played) + ); +} + +#[test] +fn first_post_play_packet_starts_the_idle_budget_and_frame_watchdog() { + let mut config = config(); + config.startup_timeout = Duration::from_secs(60); + config.timeout = Duration::from_secs(8); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + feedback.wait_for_play(); + let mut receiver = NvstVideoReceiver::new(config); + let started = receiver.timeout_origin; + receiver.process_datagram(peer(), &protect_partial_frame(&crypto, 0, 7), started); + let played = started + Duration::from_secs(19); + feedback.complete_play(played); + let received = played + Duration::from_secs(30); + assert_eq!(receiver.poll_timeout(received), None); + receiver.process_datagram(peer(), &protect_partial_frame(&crypto, 1, 7), received); + assert!(!receiver.initial_timeout_pending); + assert_eq!( + receiver.frame_progress.first_authenticated_at, + Some(received) + ); + assert_eq!( + receiver.poll_frame_progress( + received + Duration::from_secs(2) - Duration::from_nanos(1), + progress_policy() + ), + None + ); + assert!(matches!( + receiver.poll_frame_progress(received + Duration::from_secs(2), progress_policy()), + Some(NvstFrameProgressEvent::KeyframeRequested { .. }) + )); + assert_eq!( + receiver.poll_timeout(received + Duration::from_secs(8) - Duration::from_nanos(1)), + None + ); + assert_eq!( + receiver.poll_timeout(received + Duration::from_secs(8)), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::Timeout { + idle_for: Duration::from_secs(8) + })) + ); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_qos_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_qos_tests.rs new file mode 100644 index 000000000..eb221c0ab --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_qos_tests.rs @@ -0,0 +1,449 @@ +use super::*; + +fn word(report: &QosReport, offset: usize) -> u32 { + u32::from_le_bytes( + report.command().payload[offset..offset + 4] + .try_into() + .unwrap(), + ) +} + +fn halfword(report: &QosReport, offset: usize) -> u16 { + u16::from_le_bytes( + report.command().payload[offset..offset + 2] + .try_into() + .unwrap(), + ) +} + +#[test] +fn qos_reports_measure_successive_intervals_at_50_and_75_mbps() { + for bitrate_mbps in [50_u32, 75] { + let mut config = config(); + config.max_access_unit_bytes = DEFAULT_MAX_ACCESS_UNIT_BYTES; + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let mut previous = QosReport::default(); + let mut sequence = 10_u16; + let interval_bytes = bitrate_mbps * 1_000_000 / 8 / 20; + let mut media = vec![0x55; interval_bytes as usize]; + media[..5].copy_from_slice(&[0, 0, 0, 1, 0x65]); + let origin = Instant::now(); + + for frame_index in 1..=4 { + let sender_frame = frame_index + 100; + let timestamp = 90_000 + frame_index * 5_000; + let now = origin + QOS_REPORT_INTERVAL * frame_index; + let mut emitted = 0; + let chunk_count = media.len().div_ceil(1_200); + for (index, chunk) in media.chunks(1_200).enumerate() { + let mut flags = FLAG_CONTAINS_PIC_DATA; + if index == 0 { + flags |= FLAG_SOF; + } + if index + 1 == chunk_count { + flags |= FLAG_EOF; + } + let mut packet = build_plaintext_rtp(sequence, flags, sender_frame, chunk); + packet[4..8].copy_from_slice(×tamp.to_be_bytes()); + let encrypted = protect_for_test(&crypto, packet, 0); + for event in receiver.process_datagram(peer(), &encrypted, now) { + let NvstReceiveEvent::Frame(frame) = event else { + panic!("unexpected receive event: {event:?}"); + }; + assert_eq!(frame.bytes, media); + emitted += 1; + } + sequence += 1; + } + assert_eq!(emitted, 1); + let report = feedback.qos_report_at(&previous, QOS_REPORT_INTERVAL * frame_index, now); + assert_eq!(report.sequence, frame_index); + assert_eq!(report.sender_frame_number, sender_frame); + assert_eq!(word(&report, 12), sender_frame); + assert_eq!(report.bytes_received, frame_index * interval_bytes); + assert_eq!(previous.bytes_received, (frame_index - 1) * interval_bytes); + assert!(word(&report, 16) > 0); + assert!(word(&report, 48) > 0); + assert!(word(&report, 16) >= word(&report, 48)); + assert_eq!(word(&report, 44), 0); + assert_eq!(word(&report, 36), frame_index * 4_500); + assert_eq!(report.command().encoded().len(), 56); + previous = report; + } + + let idle = feedback.qos_report_at( + &previous, + QOS_REPORT_INTERVAL * 5, + origin + QOS_REPORT_INTERVAL * 5, + ); + assert_eq!(idle.bytes_received, previous.bytes_received); + assert!(word(&idle, 16) > word(&previous, 16)); + assert_eq!(word(&idle, 44), 0); + assert_eq!(idle.sender_frame_number, previous.sender_frame_number); + } +} + +#[test] +fn authenticated_packet_train_populates_capacity_feedback_after_warmup() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let origin = Instant::now(); + for frame in 1..=101_u32 { + let timestamp = 90_000 + frame * 1_800; + let start = origin + Duration::from_millis(u64::from(frame) * 20); + for packet in 0..3_u16 { + let flags = FLAG_CONTAINS_PIC_DATA + | if packet == 0 { FLAG_SOF } else { 0 } + | if packet == 2 { FLAG_EOF } else { 0 }; + let media = if packet == 0 { + vec![0, 0, 0, 1, 0x65] + } else { + vec![0x55; 200] + }; + let mut plaintext = + build_plaintext_rtp((frame as u16 - 1) * 3 + packet, flags, frame, &media); + plaintext[4..8].copy_from_slice(×tamp.to_be_bytes()); + let received_at = start + Duration::from_micros(u64::from(packet) * 500); + receiver.process_datagram(peer(), &protect_for_test(&crypto, plaintext, 0), received_at); + } + } + let report = feedback.qos_report_at( + &QosReport::default(), + Duration::from_millis(2_041), + origin + Duration::from_millis(2_041), + ); + assert_eq!(report.bandwidth.lossy_frames, 0); + assert!(report.bandwidth.estimate_kbps > 0); + assert!(report.bandwidth.utilization_percent > 0); + assert_eq!(word(&report, 44), report.bandwidth.estimate_kbps); + assert_eq!(report.command().payload[28], report.bandwidth.utilization_percent); + assert!(word(&report, 16) >= word(&report, 48)); +} + +#[test] +fn qos_receive_progress_advances_before_frame_completion_but_not_on_invalid_packets() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let mut first = build_plaintext_rtp(10, FLAG_SOF, 1, &[0, 0, 0, 1, 0x65]); + first[4..8].copy_from_slice(&90_000_u32.to_be_bytes()); + let first = protect_for_test(&crypto, first, 0); + let now = Instant::now(); + assert!(receiver.process_datagram(peer(), &first, now).is_empty()); + let first_report = feedback.qos_report(&QosReport::default(), Duration::from_millis(50)); + assert_eq!(first_report.sender_frame_number, 0); + assert_eq!(word(&first_report, 36), 4_500); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(90_000) + ); + assert_eq!(first_report.bytes_received, 0); + assert_eq!(halfword(&first_report, 26), 0); + + let mut later = build_plaintext_rtp(12, FLAG_SOF, 2, &[0, 0, 0, 1, 0x61]); + later[4..8].copy_from_slice(&93_000_u32.to_be_bytes()); + let later = protect_for_test(&crypto, later, 0); + let mut corrupted = later.clone(); + corrupted[4] ^= 1; + assert!(matches!( + receiver + .process_datagram(peer(), &corrupted, now) + .as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::AuthenticationFailed + )] + )); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(90_000) + ); + let wrong_peer = SocketAddr::new(peer().ip(), peer().port() + 1); + assert!(matches!( + receiver + .process_datagram(wrong_peer, &later, now) + .as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::UnexpectedSource { .. } + )] + )); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(90_000) + ); + assert!(receiver.process_datagram(peer(), &later, now).is_empty()); + let later_report = feedback.qos_report(&first_report, Duration::from_millis(100)); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(93_000) + ); + assert_eq!(later_report.sender_frame_number, 0); + assert_eq!(word(&later_report, 36), 9_000); + assert_eq!(halfword(&later_report, 26), 5_000); + assert_eq!(word(&later_report, 44), 0); + assert!(matches!( + receiver.process_datagram(peer(), &first, now).as_slice(), + [NvstReceiveEvent::Dropped(NvstDropReason::ReplayRejected)] + )); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(93_000) + ); +} + +#[test] +fn qos_receive_timestamp_handles_reordering_and_sender_clock_wrap() { + let feedback = NvstFeedbackState::default(); + let previous = QosReport::default(); + let now = Instant::now(); + feedback.publish_stream(7, 10, u32::MAX - 2_000, now); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(u32::MAX - 2_000) + ); + feedback.publish_stream(7, 12, 1_000, now + Duration::from_millis(33)); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(1_000) + ); + feedback.publish_stream(7, 12, u32::MAX - 500, now + Duration::from_millis(34)); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(1_000) + ); + feedback.publish_stream(7, 13, 4_000, now + Duration::from_millis(66)); + assert_eq!( + feedback + .reception_timing + .lock() + .unwrap() + .latest_rtp_timestamp, + Some(4_000) + ); + assert_eq!( + word( + &feedback.qos_report(&previous, Duration::from_millis(125)), + 36 + ), + 11_250 + ); +} + +#[test] +fn qos_loss_tracks_only_authenticated_packets_since_the_last_successful_report() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let now = Instant::now(); + let packet = |sequence| { + protect_for_test( + &crypto, + build_plaintext_rtp(sequence, FLAG_SOF, u32::from(sequence), &[0, 0, 0, 1, 0x65]), + 0, + ) + }; + + assert!( + receiver + .process_datagram(peer(), &packet(10), now) + .is_empty() + ); + let sent = feedback.qos_report(&QosReport::default(), Duration::from_millis(50)); + assert_eq!(halfword(&sent, 26), 0); + + let mut rejected = packet(11); + rejected[4] ^= 1; + assert!(matches!( + receiver.process_datagram(peer(), &rejected, now).as_slice(), + [NvstReceiveEvent::Dropped( + NvstDropReason::AuthenticationFailed + )] + )); + assert!( + receiver + .process_datagram(peer(), &packet(12), now) + .is_empty() + ); + let unsent = feedback.qos_report(&sent, Duration::from_millis(100)); + assert_eq!(halfword(&unsent, 26), 5_000); + + assert!( + receiver + .process_datagram(peer(), &packet(14), now) + .is_empty() + ); + let retried = feedback.qos_report(&sent, Duration::from_millis(150)); + assert_eq!(retried.sequence, unsent.sequence); + assert_eq!(halfword(&retried, 26), 5_000); + + let _ = receiver.process_datagram(peer(), &packet(11), now); + assert_eq!(feedback.received_packets.load(Ordering::Acquire), 4); + let recovered = feedback.qos_report(&retried, Duration::from_millis(200)); + assert_eq!(halfword(&recovered, 26), 0); + assert_eq!(word(&recovered, 36), 18_000); +} + +#[test] +fn qos_loss_resets_on_stream_and_counter_epoch_changes() { + let feedback = NvstFeedbackState::default(); + let now = Instant::now(); + feedback.publish_stream(7, 10, 90_000, now); + let baseline = feedback.qos_report(&QosReport::default(), Duration::ZERO); + feedback.publish_stream(7, 12, 93_000, now); + let loss = feedback.qos_report(&baseline, Duration::from_millis(50)); + assert_eq!(halfword(&loss, 26), 5_000); + + feedback.publish_stream(8, 13, 94_000, now); + let new_stream = feedback.qos_report(&loss, Duration::from_millis(100)); + assert_eq!(halfword(&new_stream, 26), 0); + feedback.publish_stream(8, 14, 95_000, now); + let healthy = feedback.qos_report(&new_stream, Duration::from_millis(150)); + assert_eq!(halfword(&healthy, 26), 0); + + feedback + .reception_timing + .lock() + .unwrap() + .qos_packet_snapshot + .as_mut() + .unwrap() + .received = 0; + let reset = feedback.qos_report(&healthy, Duration::from_millis(200)); + assert_eq!(halfword(&reset, 26), 0); +} + +#[test] +fn qos_client_clock_truncates_to_milliseconds_and_wraps_at_u32_ticks() { + let feedback = NvstFeedbackState::default(); + let baseline = QosReport::default(); + assert_eq!( + word( + &feedback.qos_report(&baseline, Duration::from_micros(1_999)), + 36 + ), + 90 + ); + let elapsed = Duration::from_millis(u64::from(u32::MAX) / 90 + 1); + assert_eq!( + word(&feedback.qos_report(&baseline, elapsed), 36), + 14 + ); +} + +#[test] +fn qos_unsent_samples_do_not_advance_the_successful_report_baseline() { + let feedback = NvstFeedbackState::default(); + feedback + .completed_frame_bytes + .store(1_000, Ordering::Release); + let sent = feedback.qos_report(&QosReport::default(), Duration::ZERO); + feedback + .completed_frame_bytes + .store(2_000, Ordering::Release); + let unsent = feedback.qos_report(&sent, Duration::ZERO); + assert_eq!(sent.bytes_received, 1_000); + assert_eq!(unsent.bytes_received, 2_000); + assert_eq!(word(&unsent, 44), 0); + feedback + .completed_frame_bytes + .store(3_000, Ordering::Release); + let retry = feedback.qos_report(&sent, Duration::ZERO); + assert_eq!(retry.sequence, unsent.sequence); + assert_eq!(sent.bytes_received, 1_000); + assert_eq!(retry.bytes_received, 3_000); + assert_eq!(word(&retry, 48), 0); + assert_eq!(word(&retry, 44), 0); + let next = feedback.qos_report(&retry, Duration::ZERO); + assert_eq!(next.sequence, retry.sequence + 1); + assert_eq!(word(&next, 44), 0); +} + +#[test] +fn qos_counters_wrap_without_zeroing_or_overflowing_interval_bits() { + let feedback = NvstFeedbackState::default(); + feedback + .completed_frame_bytes + .store(u64::from(u32::MAX) + 101, Ordering::Release); + let previous = QosReport { + sequence: u32::MAX, + bytes_received: u32::MAX - 99, + ..QosReport::default() + }; + let report = feedback.qos_report(&previous, Duration::ZERO); + assert_eq!(report.sequence, 0); + assert_eq!(report.bytes_received, 100); + assert_eq!(previous.bytes_received, u32::MAX - 99); + assert_eq!(word(&report, 16), 0); + assert_eq!(word(&report, 48), 0); + assert_eq!(word(&report, 44), 0); + + feedback + .completed_frame_bytes + .store(u64::from(u32::MAX), Ordering::Release); + let saturated = feedback.qos_report(&QosReport::default(), Duration::ZERO); + assert_eq!(saturated.bytes_received, u32::MAX); + assert_eq!(word(&saturated, 44), 0); +} + +#[test] +fn qos_warmup_and_new_sessions_keep_independent_baselines() { + let feedback = NvstFeedbackState::default(); + feedback + .completed_frame_bytes + .store(500_000, Ordering::Release); + let warmup = feedback.qos_report(&QosReport::default(), Duration::ZERO); + assert_eq!(word(&warmup, 44), 0); + feedback + .completed_frame_bytes + .store(1_000_000, Ordering::Release); + let warmed_up = feedback.qos_report(&warmup, Duration::ZERO); + assert_eq!(warmup.bytes_received, 500_000); + assert_eq!(word(&warmed_up, 48), 0); + assert_eq!(word(&warmed_up, 44), 0); + + let next_session = NvstFeedbackState::default(); + let report = next_session.qos_report(&QosReport::default(), Duration::ZERO); + assert_eq!(report.sequence, 1); + assert_eq!(report.sender_frame_number, 0); + assert_eq!(word(&report, 16), 0); + assert_eq!(word(&report, 48), 0); + assert_eq!(word(&report, 44), 0); + assert_eq!(word(&report, 36), 0); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_policy.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_policy.rs new file mode 100644 index 000000000..41c34c175 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_policy.rs @@ -0,0 +1,48 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct NvstRecoveryPolicy { + pub ref_pic_invalidation: Option, + pub stream_recovery_type: Option, + pub frames_with_invalid_ref_threshold: Option, + pub enable_recovery_mode: Option, + pub recovery_mode_trigger_threshold_ms: Option, + pub enable_intra_refresh_in_recovery_mode: Option, + pub encoder_ltr_feature_setting: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn recovery_policy_preserves_offered_values_without_defaults() { + let policy: NvstRecoveryPolicy = serde_json::from_value(json!({ + "refPicInvalidation":true,"streamRecoveryType":2, + "framesWithInvalidRefThreshold":4,"enableRecoveryMode":false + })) + .unwrap(); + assert_eq!(policy.stream_recovery_type, Some(2)); + assert_eq!(policy.ref_pic_invalidation, Some(true)); + assert_eq!(policy.enable_recovery_mode, Some(false)); + assert_eq!(policy.encoder_ltr_feature_setting, None); + assert_eq!( + serde_json::from_value::(json!({})).unwrap(), + NvstRecoveryPolicy::default() + ); + } + + #[test] + fn recovery_policy_rejects_malformed_handoff_fields() { + for value in [ + json!({"refPicInvalidation":1}), + json!({"streamRecoveryType":-1}), + json!({"framesWithInvalidRefThreshold":4294967296_u64}), + json!({"unknown":1}), + ] { + assert!(serde_json::from_value::(value).is_err()); + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs new file mode 100644 index 000000000..1055dd1ec --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs @@ -0,0 +1,955 @@ +use super::*; + +#[test] +fn multiple_reorder_gaps_do_not_revalidate_post_gap_references() { + for fec in [false, true] { + let mut config = config(); + config.reorder_window_packets = DEFAULT_REORDER_WINDOW; + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let now = Instant::now(); + let sequences = if fec { + [0, 4, 2055, 2057] + } else { + [0, 2, 1028, 1029] + }; + let mut result = Vec::new(); + for (index, (sequence, frame, keyframe)) in sequences + .into_iter() + .zip([0, 2, 100, 101]) + .zip([true, true, false, false]) + .map(|((sequence, frame), keyframe)| (sequence, frame, keyframe)) + .enumerate() + { + let plaintext = if fec { + reference_fec_block(sequence, frame, 1, 100, keyframe).remove(0) + } else { + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + frame, + &[0, 0, 0, 1, if keyframe { 0x65 } else { 0x61 }, 0x88], + ) + }; + let at = if index == 3 { + now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1) + } else { + now + }; + let events = + receiver.process_datagram(peer(), &protect_for_test(&crypto, plaintext, 0), at); + if index == 3 { + result = events; + } + } + assert!(matches!( + result.first(), + Some(NvstReceiveEvent::RecoveryNeeded( + NvstRecovery::PacketGap { .. } + )) + )); + let frames: Vec<_> = result + .into_iter() + .filter_map(|event| match event { + NvstReceiveEvent::Frame(frame) => { + Some((frame.frame_index, frame.keyframe, frame.contiguous)) + } + _ => None, + }) + .collect(); + assert_eq!( + frames, + [(100, false, false), (101, false, true)], + "fec={fec}" + ); + assert!(receiver.config.feedback.keyframe_request_pending()); + } +} + +fn reference_fec_block( + base: u16, + frame: u32, + count: usize, + percent: u32, + keyframe: bool, +) -> Vec> { + const NANORS_INVERSE: [u8; 256] = [ + 0, 1, 142, 244, 71, 167, 122, 186, 173, 157, 221, 152, 61, 170, 93, 150, 216, 114, 192, 88, + 224, 62, 76, 102, 144, 222, 85, 128, 160, 131, 75, 42, 108, 237, 57, 81, 96, 86, 44, 138, + 112, 208, 31, 74, 38, 139, 51, 110, 72, 137, 111, 46, 164, 195, 64, 94, 80, 34, 207, 169, + 171, 12, 21, 225, 54, 95, 248, 213, 146, 78, 166, 4, 48, 136, 43, 30, 22, 103, 69, 147, 56, + 35, 104, 140, 129, 26, 37, 97, 19, 193, 203, 99, 151, 14, 55, 65, 36, 87, 202, 91, 185, + 196, 23, 77, 82, 141, 239, 179, 32, 236, 47, 50, 40, 209, 17, 217, 233, 251, 218, 121, 219, + 119, 6, 187, 132, 205, 254, 252, 27, 84, 161, 29, 124, 204, 228, 176, 73, 49, 39, 45, 83, + 105, 2, 245, 24, 223, 68, 79, 155, 188, 15, 92, 11, 220, 189, 148, 172, 9, 199, 162, 28, + 130, 159, 198, 52, 194, 70, 5, 206, 59, 13, 60, 156, 8, 190, 183, 135, 229, 238, 107, 235, + 242, 191, 175, 197, 100, 7, 123, 149, 154, 174, 182, 18, 89, 165, 53, 101, 184, 163, 158, + 210, 247, 98, 90, 133, 125, 168, 58, 41, 113, 200, 246, 249, 67, 215, 214, 16, 115, 118, + 120, 153, 10, 25, 145, 20, 63, 230, 240, 134, 177, 226, 241, 250, 116, 243, 180, 109, 33, + 178, 106, 227, 231, 181, 234, 3, 143, 211, 201, 66, 212, 232, 117, 127, 255, 126, 253, + ]; + let parity_count = (count * percent as usize).div_ceil(100); + let mut packets = Vec::new(); + for index in 0..count { + let mut payload = vec![0x55; 1264]; + let mut flags = FLAG_CONTAINS_PIC_DATA; + if index == 0 { + flags |= FLAG_SOF; + payload[..5].copy_from_slice(&[0, 0, 0, 1, if keyframe { 0x65 } else { 0x61 }]); + } + if index == count - 1 { + flags |= FLAG_EOF; + } + let mut packet = build_plaintext_rtp(base + index as u16, flags, frame, &payload); + let fec_word = (percent << 4) | ((index as u32) << 12) | ((count as u32) << 22); + packet[28..32].copy_from_slice(&fec_word.to_le_bytes()); + packets.push(packet); + } + for parity_index in 0..parity_count { + let mut parity = vec![0; 1296]; + for (data_index, data) in packets[..count].iter().enumerate() { + let coefficient = NANORS_INVERSE[(parity_count + data_index) ^ parity_index]; + for (target, source) in parity.iter_mut().zip(data) { + let mut value = *source; + let mut multiplier = coefficient; + while multiplier != 0 { + if multiplier & 1 != 0 { + *target ^= value; + } + let high_bit = value & 0x80 != 0; + value <<= 1; + if high_bit { + value ^= 0x1d; + } + multiplier >>= 1; + } + } + } + parity[..16].copy_from_slice(&packets[0][..16]); + parity[2..4].copy_from_slice(&(base + (count + parity_index) as u16).to_be_bytes()); + parity[20..24].copy_from_slice(&frame.to_le_bytes()); + parity[27] = 0; + let fec_word = + (percent << 4) | (((count + parity_index) as u32) << 12) | ((count as u32) << 22); + parity[28..32].copy_from_slice(&fec_word.to_le_bytes()); + packets.push(parity); + } + packets +} + +#[test] +fn recovered_sof_is_delivered_as_data_before_its_late_original() { + for (count, percent) in [(1, 200), (2, 100), (3, 66), (100, 20)] { + let mut config = config(); + config.max_access_unit_bytes = DEFAULT_MAX_ACCESS_UNIT_BYTES; + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let mut frames = Vec::new(); + let initial = reference_fec_block(10, 5, 1, 200, true); + let reference = reference_fec_block(13, 6, count, percent, false); + let expected_reference = reference[..count] + .iter() + .flat_map(|packet| packet[32..].iter().copied()) + .collect::>(); + let successor = reference_fec_block(13 + reference.len() as u16, 7, 1, 200, false); + let reordered_reference = reference[1..=count] + .iter() + .chain(std::iter::once(&reference[0])) + .chain(reference[count + 1..].iter()); + for packet in initial.iter().chain(reordered_reference).chain(&successor) { + let encrypted = protect_for_test(&crypto, packet.clone(), 0); + for event in receiver.process_datagram(peer(), &encrypted, Instant::now()) { + let NvstReceiveEvent::Frame(frame) = event else { + panic!("unexpected {count}-data FEC event: {event:?}"); + }; + assert!(frame.contiguous); + frames.push(frame); + } + } + assert_eq!( + frames + .iter() + .map(|frame| frame.frame_index) + .collect::>(), + [5, 6, 7] + ); + assert_eq!(frames[1].bytes, expected_reference); + assert_eq!(receiver.fec_repaired_packets, 1); + assert!(!feedback.keyframe_request_pending()); + assert_eq!(feedback.network_metrics().unwrap().1, 0.0); + } +} + +#[test] +fn rejected_picture_requires_a_new_keyframe_and_marks_the_next_frame_discontinuous() { + let mut oversized = vec![0x55; 4097]; + oversized[..5].copy_from_slice(&[0, 0, 0, 1, 0x61]); + for (flags, payload, reason) in [ + ( + FLAG_SOF | FLAG_EOF, + oversized, + NvstDropReason::AccessUnitTooLarge { limit: 4096 }, + ), + ( + FLAG_SOF | FLAG_EOF, + vec![0x55; 20], + NvstDropReason::MissingAnnexBStartCode, + ), + ( + FLAG_EOF, + vec![0x55; 20], + NvstDropReason::AwaitingStartOfFrame, + ), + ] { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let initial = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF | FLAG_EOF, 5, &[0, 0, 0, 1, 0x65]), + 0, + ); + assert!( + matches!(receiver.process_datagram(peer(), &initial, Instant::now()).as_slice(), [NvstReceiveEvent::Frame(frame)] if frame.keyframe) + ); + let rejected = protect_for_test(&crypto, build_plaintext_rtp(11, flags, 6, &payload), 0); + assert_eq!( + receiver.process_datagram(peer(), &rejected, Instant::now()), + [NvstReceiveEvent::Dropped(reason)] + ); + assert!(receiver.assembler.current_frame.is_none()); + assert!(feedback.keyframe_request_pending()); + let successor = protect_for_test( + &crypto, + build_plaintext_rtp(12, FLAG_SOF | FLAG_EOF, 7, &[0, 0, 0, 1, 0x61]), + 0, + ); + assert!( + matches!(receiver.process_datagram(peer(), &successor, Instant::now()).as_slice(), [NvstReceiveEvent::Frame(frame)] if !frame.contiguous && !frame.keyframe) + ); + assert!(feedback.keyframe_request_pending()); + assert_eq!(feedback.network_metrics().unwrap().1, 0.0); + let recovery = protect_for_test( + &crypto, + build_plaintext_rtp(13, FLAG_SOF | FLAG_EOF, 8, &[0, 0, 0, 1, 0x65]), + 0, + ); + assert!( + matches!(receiver.process_datagram(peer(), &recovery, Instant::now()).as_slice(), [NvstReceiveEvent::Frame(frame)] if frame.keyframe) + ); + assert!(!feedback.keyframe_request_pending()); + } +} + +#[test] +fn authenticated_unsupported_parity_and_replays_do_not_invalidate_picture_data() { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let first = protect_for_test( + &crypto, + build_plaintext_rtp(10, FLAG_SOF, 5, &[0, 0, 0, 1, 0x65]), + 0, + ); + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .is_empty() + ); + assert_eq!( + receiver.process_datagram(peer(), &first, Instant::now()), + [NvstReceiveEvent::Dropped(NvstDropReason::ReplayRejected)] + ); + let mut parity = build_plaintext_rtp(11, FLAG_CONTAINS_PIC_DATA, 5, &[0x55]); + parity[28..32].copy_from_slice(&((100_u32 << 4) | (2_u32 << 12) | (1_u32 << 22)).to_le_bytes()); + let parity = protect_for_test(&crypto, parity, 0); + assert_eq!( + receiver.process_datagram(peer(), &parity, Instant::now()), + [NvstReceiveEvent::Dropped(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair + ))] + ); + assert_eq!(receiver.assembler.current_frame, Some(5)); + assert!(!feedback.keyframe_request_pending()); +} + +#[test] +fn conflicting_fec_layout_invalidates_only_discarded_picture_data() { + for parity in [false, true] { + let config = config(); + let feedback = config.feedback(); + let crypto = test_srtp(&config); + let mut receiver = NvstVideoReceiver::new(config); + let block = reference_fec_block(10, 5, 2, 100, false); + let first = protect_for_test(&crypto, block[0].clone(), 0); + assert!( + receiver + .process_datagram(peer(), &first, Instant::now()) + .is_empty() + ); + let conflicting = reference_fec_block(10, 5, 3, 66, false); + let packet = protect_for_test(&crypto, conflicting[if parity { 3 } else { 1 }].clone(), 0); + assert_eq!( + receiver.process_datagram(peer(), &packet, Instant::now()), + [NvstReceiveEvent::Dropped(NvstDropReason::Unsupported( + NvstUnsupportedFeature::FecRepair + ))] + ); + assert_eq!(feedback.keyframe_request_pending(), !parity); + assert_eq!(receiver.next_frame_contiguous, parity); + } +} + +fn typed_frame(sequence: u16, frame: u32, frame_type: u8, nal: u8) -> Vec { + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + frame, + &[0x01, 0, 0, frame_type, 0, 0, 0, 0, 0, 0, 0, 1, nal, 0x88], + ) +} + +struct TypedStream { + receiver: NvstVideoReceiver, + crypto: SrtpReceiver, + feedback: SharedNvstFeedback, + origin: Instant, +} + +impl TypedStream { + fn new() -> Self { + let mut config = config(); + config.reorder_window_packets = DEFAULT_REORDER_WINDOW; + Self { + feedback: config.feedback(), + crypto: test_srtp(&config), + receiver: NvstVideoReceiver::new(config), + origin: Instant::now(), + } + } + + fn push(&mut self, packet: Vec, after: Duration) -> Vec { + let packet = protect_for_test(&self.crypto, packet, 0); + self.receiver + .process_datagram(peer(), &packet, self.origin + after) + } + + fn frames(&mut self, packet: Vec, after: Duration) -> Vec<(u32, bool, bool)> { + delivered(self.push(packet, after)) + } + + fn stats(&self) -> String { + self.feedback.frame_invalidation.stats() + } + + /// Frames 1 (I) and 2 (P) arrive, frame 3 is lost beyond NACK recovery, and frames 4 and + /// 5 are released by the reorder timeout. + fn lose_frame_three(&mut self) -> Vec { + assert_eq!( + self.frames(typed_frame(0, 1, 2, 0x65), Duration::ZERO), + [(1, true, true)] + ); + assert_eq!( + self.frames(typed_frame(1, 2, 1, 0x41), Duration::ZERO), + [(2, false, true)] + ); + assert!(self.push(typed_frame(3, 4, 1, 0x41), Duration::ZERO).is_empty()); + assert_eq!(self.feedback.take_nack(self.origin, None), Some((2, 2))); + self.push( + typed_frame(4, 5, 1, 0x41), + MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1), + ) + } +} + +fn delivered(events: Vec) -> Vec<(u32, bool, bool)> { + events + .into_iter() + .filter_map(|event| match event { + NvstReceiveEvent::Frame(frame) => { + Some((frame.frame_index, frame.keyframe, frame.contiguous)) + } + _ => None, + }) + .collect() +} + +#[test] +fn assembled_frames_carry_the_gs_frame_type_and_sender_frame_index() { + for (wire, expected) in [ + (1, NvstFrameType::P), + (2, NvstFrameType::I), + (3, NvstFrameType::Header), + (4, NvstFrameType::IntraRefresh), + (5, NvstFrameType::RefPicI), + (6, NvstFrameType::NonRefP), + (9, NvstFrameType::Unknown(9)), + ] { + let mut stream = TypedStream::new(); + let [NvstReceiveEvent::Frame(frame)] = + &stream.push(typed_frame(0, 77, wire, 0x41), Duration::ZERO)[..] + else { + panic!("frame type {wire} was not delivered"); + }; + assert_eq!(frame.frame_type, Some(expected)); + assert_eq!(frame.frame_index, 77); + } + let mut stream = TypedStream::new(); + let [NvstReceiveEvent::Frame(frame)] = &stream.push( + build_plaintext_rtp(0, FLAG_SOF | FLAG_EOF, 78, &[0, 0, 0, 1, 0x65]), + Duration::ZERO, + )[..] else { + panic!("headerless frame was not delivered"); + }; + assert_eq!(frame.frame_type, None); +} + +#[test] +fn unrecoverable_loss_invalidates_the_lost_range_and_resumes_on_a_clearing_frame() { + for (frame_type, nal, keyframe) in [(5, 0x41, false), (4, 0x41, false), (2, 0x65, true)] { + let mut stream = TypedStream::new(); + let events = stream.lose_frame_three(); + assert!(matches!( + events.first(), + Some(NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { .. })) + )); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 3))); + assert!(!stream.feedback.keyframe_request_pending()); + assert!( + stream + .frames(typed_frame(5, 6, 1, 0x41), Duration::from_millis(60)) + .is_empty() + ); + assert_eq!( + stream.frames( + typed_frame(6, 7, frame_type, nal), + Duration::from_millis(70) + ), + [(7, keyframe, true)], + "frame type {frame_type}" + ); + assert_eq!( + stream.frames(typed_frame(7, 8, 1, 0x41), Duration::from_millis(80)), + [(8, false, true)] + ); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + assert!(!stream.feedback.keyframe_request_pending()); + let recovered = match frame_type { + 5 => "rfiRecoveredI=0 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=1", + 4 => "rfiRecoveredI=0 rfiRecoveredIntraRefresh=1 rfiRecoveredRefPicI=0", + _ => "rfiRecoveredI=1 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0", + }; + assert_eq!( + stream.stats(), + format!("rfiSent=0 rfiDropped=3 {recovered} rfiIdrBackstop=0") + ); + assert_eq!(stream.receiver.frames_emitted, 4); + } +} + +#[test] +fn further_loss_during_invalidation_extends_the_pending_range() { + let mut stream = TypedStream::new(); + stream.lose_frame_three(); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 3))); + assert!(stream.push(typed_frame(6, 8, 1, 0x41), Duration::from_millis(60)).is_empty()); + let events = stream.push( + typed_frame(7, 9, 6, 0x01), + Duration::from_millis(60) + MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1), + ); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 7))); + assert!(stream.stats().contains("rfiDropped=4")); + assert!(!stream.feedback.keyframe_request_pending()); +} + +#[test] +fn header_frames_pass_during_invalidation() { + let mut stream = TypedStream::new(); + stream.lose_frame_three(); + assert_eq!( + stream.frames(typed_frame(5, 6, 3, 0x67), Duration::from_millis(60)), + [(6, false, true)] + ); + assert!( + stream + .frames(typed_frame(6, 7, 1, 0x41), Duration::from_millis(61)) + .is_empty() + ); + assert!(stream.stats().contains("rfiDropped=3")); +} + +#[test] +fn a_lost_frame_followed_by_a_header_frame_is_still_invalidated() { + let mut stream = TypedStream::new(); + stream.frames(typed_frame(0, 1, 2, 0x65), Duration::ZERO); + stream.push(typed_frame(2, 3, 3, 0x67), Duration::ZERO); + let events = stream.push( + typed_frame(3, 4, 1, 0x41), + MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1), + ); + assert_eq!(delivered(events), [(3, false, true)]); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + assert!(stream.stats().contains("rfiDropped=1")); +} + +#[test] +fn invalidation_without_a_clearing_frame_falls_back_to_idr_after_the_cooldown() { + let mut stream = TypedStream::new(); + stream.lose_frame_three(); + let lost_at = MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1); + assert!( + stream + .frames( + typed_frame(5, 6, 1, 0x41), + lost_at + RFI_BACKSTOP_TIMEOUT - Duration::from_millis(1) + ) + .is_empty() + ); + assert!(!stream.feedback.keyframe_request_pending()); + assert!( + stream + .frames(typed_frame(6, 7, 1, 0x41), lost_at + RFI_BACKSTOP_TIMEOUT) + .is_empty() + ); + assert!(stream.feedback.keyframe_request_pending()); + assert!( + stream + .frames( + typed_frame(7, 8, 1, 0x41), + lost_at + RFI_BACKSTOP_TIMEOUT * 2 + ) + .is_empty() + ); + assert_eq!( + stream.frames( + typed_frame(8, 9, 2, 0x65), + lost_at + RFI_BACKSTOP_TIMEOUT * 3 + ), + [(9, true, true)] + ); + assert!(!stream.feedback.keyframe_request_pending()); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=5 rfiRecoveredI=1 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=1" + ); +} + +#[test] +fn an_incomplete_frame_is_invalidated_once_the_next_frame_assembles() { + let mut stream = TypedStream::new(); + stream.frames(typed_frame(0, 1, 2, 0x65), Duration::ZERO); + let mut truncated = typed_frame(1, 2, 1, 0x41); + truncated[24] = FLAG_SOF | FLAG_CONTAINS_PIC_DATA; + assert!(stream.push(truncated, Duration::ZERO).is_empty()); + let mut skipped_index = build_plaintext_rtp(2, FLAG_EOF, 2, &[0x55]); + skipped_index[16..20].copy_from_slice(&(5_u32 << 8).to_le_bytes()); + assert_eq!( + stream.push(skipped_index, Duration::ZERO), + [NvstReceiveEvent::Dropped(NvstDropReason::FrameDiscontinuity)] + ); + assert!(!stream.feedback.keyframe_request_pending()); + assert!( + stream + .frames(typed_frame(3, 3, 1, 0x41), Duration::ZERO) + .is_empty() + ); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + assert!(!stream.feedback.keyframe_request_pending()); +} + +#[test] +fn loss_without_a_frame_number_gap_requests_an_idr_immediately() { + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + let now = Instant::now(); + let frame = |frame_index, frame_type| EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 0, + frame_index, + first_stream_packet_index: 0, + frame_type: Some(frame_type), + keyframe: false, + contiguous: true, + bytes: Vec::new(), + }; + assert!(invalidation.admit(&mut frame(10, NvstFrameType::P), false, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(11, NvstFrameType::P), true, Instant::now(), &feedback)); + assert_eq!(feedback.frame_invalidation.take(), None); + assert!(feedback.keyframe_request_pending()); + invalidation.poll_backstop(now + RFI_BACKSTOP_TIMEOUT, RFI_BACKSTOP_TIMEOUT, &feedback); + assert!(feedback.frame_invalidation.stats().ends_with("rfiIdrBackstop=1")); + let mut recovery = frame(12, NvstFrameType::RefPicI); + assert!(invalidation.admit(&mut recovery, false, Instant::now(), &feedback)); + assert!(recovery.contiguous); +} + +#[test] +fn frames_without_a_type_end_invalidation_through_the_keyframe_path() { + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + let frame = |frame_index, frame_type| EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 0, + frame_index, + first_stream_packet_index: 0, + frame_type, + keyframe: false, + contiguous: true, + bytes: Vec::new(), + }; + assert!(!invalidation.lose(Instant::now())); + assert!(invalidation.admit(&mut frame(1, Some(NvstFrameType::P)), false, Instant::now(), &feedback)); + assert!(invalidation.lose(Instant::now())); + let mut untyped = frame(3, None); + assert!(invalidation.admit(&mut untyped, true, Instant::now(), &feedback)); + assert!(!untyped.contiguous); + assert!(feedback.keyframe_request_pending()); + assert_eq!(feedback.frame_invalidation.take(), None); + assert!(!invalidation.lose(Instant::now())); +} + +#[test] +fn nack_recovered_frames_are_never_invalidated() { + let mut stream = TypedStream::new(); + stream.frames(typed_frame(0, 1, 2, 0x65), Duration::ZERO); + stream.frames(typed_frame(1, 2, 1, 0x41), Duration::ZERO); + assert!(stream.push(typed_frame(3, 4, 1, 0x41), Duration::ZERO).is_empty()); + assert_eq!(stream.feedback.take_nack(stream.origin, None), Some((2, 2))); + assert_eq!( + stream.frames(typed_frame(2, 3, 1, 0x41), Duration::from_millis(20)), + [(3, false, true), (4, false, true)] + ); + assert_eq!(stream.receiver.recovered_retransmissions, 1); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + assert!(!stream.feedback.keyframe_request_pending()); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=0 rfiRecoveredI=0 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=0" + ); +} + +#[test] +fn headerless_streams_keep_the_immediate_keyframe_recovery() { + let mut stream = TypedStream::new(); + let headerless = |sequence, frame, nal| { + build_plaintext_rtp( + sequence, + FLAG_SOF | FLAG_EOF | FLAG_CONTAINS_PIC_DATA, + frame, + &[0, 0, 0, 1, nal, 0x88], + ) + }; + stream.frames(headerless(0, 1, 0x65), Duration::ZERO); + stream.push(headerless(2, 3, 0x41), Duration::ZERO); + let events = stream.push( + headerless(3, 4, 0x41), + MJOLNIR_REORDER_DEQUEUE_TIMEOUT + Duration::from_millis(1), + ); + assert_eq!(delivered(events), [(3, false, false), (4, false, true)]); + assert!(stream.feedback.keyframe_request_pending()); + assert_eq!(stream.feedback.frame_invalidation.take(), None); +} + +#[test] +fn an_unanswered_invalidation_does_not_disable_future_rfi_episodes() { + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + let now = Instant::now(); + let frame = |frame_index, frame_type| EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 0, + frame_index, + first_stream_packet_index: 0, + frame_type: Some(frame_type), + keyframe: false, + contiguous: true, + bytes: Vec::new(), + }; + assert!(invalidation.admit(&mut frame(1, NvstFrameType::P), false, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(3, NvstFrameType::P), true, Instant::now(), &feedback)); + assert_eq!(feedback.frame_invalidation.take(), Some((2, 2))); + feedback.frame_invalidation.record_sent(now); + invalidation.poll_backstop(now + RFI_BACKSTOP_TIMEOUT, RFI_BACKSTOP_TIMEOUT, &feedback); + assert!(feedback.keyframe_request_pending()); + assert!(invalidation.admit(&mut frame(5, NvstFrameType::I), true, Instant::now(), &feedback)); + assert!(invalidation.lose(now + RFI_BACKSTOP_TIMEOUT)); + + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + assert!(invalidation.admit(&mut frame(1, NvstFrameType::P), false, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + assert!(invalidation.admit(&mut frame(3, NvstFrameType::RefPicI), true, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + invalidation.poll_backstop(now + RFI_BACKSTOP_TIMEOUT, RFI_BACKSTOP_TIMEOUT, &feedback); + assert!(invalidation.admit(&mut frame(9, NvstFrameType::I), true, Instant::now(), &feedback)); + assert!(invalidation.lose(now + RFI_BACKSTOP_TIMEOUT)); +} + +#[test] +fn an_invalidation_that_was_never_sent_keeps_rfi_enabled() { + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + let now = Instant::now(); + let frame = |frame_index, frame_type| EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 0, + frame_index, + first_stream_packet_index: 0, + frame_type: Some(frame_type), + keyframe: false, + contiguous: true, + bytes: Vec::new(), + }; + assert!(invalidation.admit(&mut frame(1, NvstFrameType::P), false, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(3, NvstFrameType::P), true, Instant::now(), &feedback)); + assert_eq!(feedback.frame_invalidation.take(), Some((2, 2))); + feedback.frame_invalidation.restore((2, 2)); + invalidation.poll_backstop(now + RFI_BACKSTOP_TIMEOUT, RFI_BACKSTOP_TIMEOUT, &feedback); + assert!(feedback.keyframe_request_pending()); + assert!(feedback.frame_invalidation.stats().ends_with("rfiIdrBackstop=1")); + assert!(invalidation.admit(&mut frame(5, NvstFrameType::I), true, Instant::now(), &feedback)); + assert!(invalidation.lose(now + RFI_BACKSTOP_TIMEOUT)); +} + +#[test] +fn recovery_discards_an_invalidation_that_was_not_sent_yet() { + let feedback = NvstFeedbackState::default(); + let mut invalidation = crate::nvst_invalidation::FrameInvalidation::default(); + let now = Instant::now(); + let frame = |frame_index, frame_type| EncodedVideoAccessUnit { + codec: NvstVideoCodec::H264, + timestamp: 0, + frame_index, + first_stream_packet_index: 0, + frame_type: Some(frame_type), + keyframe: false, + contiguous: true, + bytes: Vec::new(), + }; + assert!(invalidation.admit(&mut frame(1, NvstFrameType::P), false, Instant::now(), &feedback)); + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(3, NvstFrameType::P), true, Instant::now(), &feedback)); + assert!(invalidation.admit(&mut frame(4, NvstFrameType::RefPicI), false, Instant::now(), &feedback)); + assert_eq!(feedback.frame_invalidation.take(), None); + + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(6, NvstFrameType::P), true, Instant::now(), &feedback)); + assert_eq!(feedback.frame_invalidation.take(), Some((5, 5))); + assert!(invalidation.admit(&mut frame(7, NvstFrameType::IntraRefresh), false, Instant::now(), &feedback)); + feedback.frame_invalidation.restore((5, 5)); + assert_eq!(feedback.frame_invalidation.take(), None); + + assert!(invalidation.lose(now)); + assert!(!invalidation.admit(&mut frame(9, NvstFrameType::P), true, Instant::now(), &feedback)); + invalidation.reset(&feedback); + assert_eq!(feedback.frame_invalidation.take(), None); +} + +#[test] +fn a_sender_frame_gap_without_packet_loss_starts_invalidation() { + let mut stream = TypedStream::new(); + assert_eq!( + stream.frames(typed_frame(0, 10, 2, 0x65), Duration::ZERO), + [(10, true, true)] + ); + assert!( + stream + .frames(typed_frame(1, 12, 1, 0x41), Duration::ZERO) + .is_empty() + ); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((11, 11))); + assert!(!stream.feedback.keyframe_request_pending()); +} + +#[test] +fn canonical_gs_headers_mask_priority_and_reject_bad_signatures() { + for priority in 0..16 { + for (wire, expected) in [ + (2, NvstFrameType::I), + (4, NvstFrameType::IntraRefresh), + (5, NvstFrameType::RefPicI), + ] { + let header = [0x47, 0x53, 1, (priority << 4) | wire]; + assert_eq!( + GsFrameHeader::parse(&header, header.len()) + .unwrap() + .frame_type, + expected + ); + let mut stream = TypedStream::new(); + let mut payload = header.to_vec(); + payload.extend_from_slice(&[0, 0, 0, 1, 0x41, 0x88]); + let [NvstReceiveEvent::Frame(frame)] = &stream.push( + build_plaintext_rtp(0, FLAG_SOF | FLAG_EOF, 7, &payload), + Duration::ZERO, + )[..] else { + panic!("prioritized canonical GS frame was not delivered"); + }; + assert_eq!(frame.frame_type, Some(expected)); + } + } + for header in [ + &[0x47, 0x52, 1, 2][..], + &[0x46, 0x53, 1, 2], + &[0x47, 0x53, 1], + ] { + assert_eq!(GsFrameHeader::parse(header, header.len()), None); + } + for version in [0x01, 0x81] { + assert_eq!( + GsFrameHeader::parse(&[version, 0, 0, 0x52], 4) + .unwrap() + .frame_type, + NvstFrameType::Unknown(0x52) + ); + } + let mut stream = TypedStream::new(); + let [NvstReceiveEvent::Frame(frame)] = &stream.push( + build_plaintext_rtp(0, FLAG_SOF | FLAG_EOF, 7, &[0x47, 0x53, 0, 0, 0, 1, 0x65]), + Duration::ZERO, + )[..] else { + panic!("frame with truncated GS metadata was not delivered"); + }; + assert_eq!(frame.frame_type, None); +} + +#[test] +fn timer_expiry_drains_a_buffered_successor_without_another_packet() { + let mut stream = TypedStream::new(); + stream.frames(typed_frame(0, 10, 2, 0x65), Duration::ZERO); + assert!( + stream + .push(typed_frame(2, 12, 1, 0x41), Duration::ZERO) + .is_empty() + ); + assert!( + stream + .receiver + .poll_recovery( + stream.origin + MJOLNIR_REORDER_DEQUEUE_TIMEOUT - Duration::from_nanos(1) + ) + .is_empty() + ); + let events = stream + .receiver + .poll_recovery(stream.origin + MJOLNIR_REORDER_DEQUEUE_TIMEOUT); + assert_eq!( + events, + [NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { + first_missing_index: 1, + last_missing_index: 1 + })] + ); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((11, 11))); + assert!( + stream + .receiver + .poll_recovery(stream.origin + MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + .is_empty() + ); +} + +#[test] +fn fec_timer_expiry_drains_complete_successors_without_another_packet() { + let now = Instant::now(); + let mut reorder = FecReorderBuffer::new(48); + let packet = |index: u64| { + let plaintext = build_plaintext_rtp( + index as u16, + FLAG_SOF | FLAG_EOF, + index as u32, + &[0, 0, 0, 1, 0x65], + ); + RtpPacket { + index, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + } + }; + let layout = |frame_index, shard_index| FecPacketLayout { + frame_index, + block_index: 0, + last_block_index: 0, + shard_index, + data_shards: 2, + parity_shards: 0, + }; + assert!(reorder.push(packet(0), layout(1, 0), now).ready.is_empty()); + assert_eq!( + reorder.push(packet(2), layout(2, 0), now).nack, + Some((1, 1)) + ); + assert!(reorder.push(packet(3), layout(2, 1), now).ready.is_empty()); + assert!( + reorder + .poll_deadline(now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT - Duration::from_nanos(1)) + .ready + .is_empty() + ); + let result = reorder.poll_deadline(now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT); + assert_eq!( + result.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 1, + last_missing_index: 1 + }) + ); + assert_eq!( + result + .ready + .iter() + .map(|packet| packet.index) + .collect::>(), + [2, 3] + ); + assert!(reorder.blocks.is_empty()); +} + +#[test] +fn rfi_retries_preserve_the_episode_deadline_and_stop_on_recovery() { + let mut stream = TypedStream::new(); + stream.frames(typed_frame(0, 10, 2, 0x65), Duration::ZERO); + stream.frames(typed_frame(1, 12, 1, 0x41), Duration::ZERO); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((11, 11))); + stream + .feedback + .frame_invalidation + .record_sent(stream.origin + Duration::from_millis(50)); + stream + .receiver + .poll_recovery(stream.origin + Duration::from_millis(149)); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + stream + .receiver + .poll_recovery(stream.origin + Duration::from_millis(150)); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((11, 11))); + stream + .feedback + .frame_invalidation + .record_sent(stream.origin + Duration::from_millis(150)); + stream.frames(typed_frame(2, 14, 1, 0x41), Duration::from_millis(200)); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((11, 13))); + stream + .feedback + .frame_invalidation + .record_sent(stream.origin + Duration::from_millis(200)); + stream + .receiver + .poll_recovery(stream.origin + RFI_BACKSTOP_TIMEOUT); + assert!(stream.feedback.keyframe_request_pending()); + stream.frames(typed_frame(3, 15, 2, 0x65), Duration::from_millis(260)); + stream + .receiver + .poll_recovery(stream.origin + Duration::from_secs(1)); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + assert!(!stream.feedback.keyframe_request_pending()); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control.rs new file mode 100644 index 000000000..758476d5a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control.rs @@ -0,0 +1,385 @@ +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct ControlKeepaliveHealth { + pub(crate) failures: u64, + pub(crate) consecutive_failures: u32, + pub(crate) recoveries: u64, +} + +impl ControlKeepaliveHealth { + pub(crate) fn record(&mut self, admitted: bool) { + if !admitted { + self.failures = self.failures.saturating_add(1); + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + return; + } + if self.consecutive_failures != 0 { + self.recoveries = self.recoveries.saturating_add(1); + self.consecutive_failures = 0; + } + } +} + +pub(crate) fn session_resumable(reason: Option, extended_code: Option) -> Option { + match reason? { + 0x37 => Some(!matches!( + extended_code?, + 0x8004_0003 | 0x8004_000b..=0x8004_000f | 0x8004_0011..=0x8004_0016 + )), + 0x3b => Some(matches!( + extended_code?, + 0x8004_0004 | 0x8004_0007 | 0x8004_0010 + )), + _ => Some(false), + } +} + +pub(crate) const MAX_SERVER_CONTROL_BYTES: usize = 4 + u16::MAX as usize; +pub(crate) const MAX_SERVER_CONTROL_COMMANDS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ServerControlParseError { + Oversized, + TooManyCommands, + Truncated, + InvalidLength, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NvstSessionTimer { + pub timer_type: u8, + pub seconds_left: u32, + pub active: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NvstServerTermination { + pub wire_code: u32, + pub reason: Option, + pub extended_code: Option, +} + +impl NvstServerTermination { + pub(crate) fn from_wire(wire_code: u32, _extra: &[u8]) -> Self { + let (reason, extended_code) = match wire_code { + 0x8003_0023 => (Some(0x38), Some(0x8003_0007)), + 0x8004_0010 => (Some(0x37), Some(0x8004_0010)), + _ => (None, None), + }; + Self { + wire_code, + reason, + extended_code, + } + } + + pub fn resumable(self) -> Option { + session_resumable(self.reason, self.extended_code) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FramePacingFeedbackMode { + Never, + Interval, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ServerControlMessage<'a> { + Timer(NvstSessionTimer), + Termination { + wire_code: u32, + extra: &'a [u8], + }, + FramePacingMode { + stream_index: u32, + mode: FramePacingFeedbackMode, + }, + Unhandled, + Unknown(u16), +} + +pub(crate) struct ServerControlMessages<'a> { + bytes: &'a [u8], + remaining: usize, + error: Option, +} + +pub(crate) fn server_control_messages(bytes: &[u8]) -> ServerControlMessages<'_> { + ServerControlMessages { + bytes, + remaining: MAX_SERVER_CONTROL_COMMANDS, + error: (bytes.len() > MAX_SERVER_CONTROL_BYTES) + .then_some(ServerControlParseError::Oversized), + } +} + +impl<'a> Iterator for ServerControlMessages<'a> { + type Item = Result, ServerControlParseError>; + + fn next(&mut self) -> Option { + if let Some(error) = self.error.take() { + self.bytes = &[]; + return Some(Err(error)); + } + if self.bytes.is_empty() { + return None; + } + let error = if self.remaining == 0 { + Some(ServerControlParseError::TooManyCommands) + } else if self.bytes.len() < 4 { + Some(ServerControlParseError::Truncated) + } else { + None + }; + if let Some(error) = error { + self.bytes = &[]; + return Some(Err(error)); + } + let code = u16::from_le_bytes([self.bytes[0], self.bytes[1]]); + let length = usize::from(u16::from_le_bytes([self.bytes[2], self.bytes[3]])); + let Some(payload) = self.bytes.get(4..4 + length) else { + self.bytes = &[]; + return Some(Err(ServerControlParseError::Truncated)); + }; + self.bytes = &self.bytes[4 + length..]; + self.remaining -= 1; + Some(decode_control_message(code, payload)) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct ServerControlDiagnostics { + pub(crate) unknown: u64, + pub(crate) last_unknown: Option, + pub(crate) malformed: u64, + pub(crate) invalid_stream: u64, +} + +fn decode_control_message( + code: u16, + payload: &[u8], +) -> Result, ServerControlParseError> { + let word = |offset| { + payload + .get(offset..offset + 4) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_le_bytes) + .ok_or(ServerControlParseError::Truncated) + }; + Ok(match code { + 0x104 | 0x105 => ServerControlMessage::Timer(NvstSessionTimer { + timer_type: if code == 0x104 { 4 } else { 5 }, + seconds_left: word(0)?, + active: code == 0x104, + }), + 0x109 => ServerControlMessage::Termination { + wire_code: u32::from_be_bytes( + payload + .get(..4) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or(ServerControlParseError::Truncated)?, + ), + extra: &payload[4..], + }, + 0x112 => { + if payload.len() != 8 { + return Err(ServerControlParseError::InvalidLength); + } + ServerControlMessage::FramePacingMode { + stream_index: word(0)?, + mode: if word(4)? == 1 { + FramePacingFeedbackMode::Interval + } else { + FramePacingFeedbackMode::Never + }, + } + } + 0x101..=0x103 | 0x106 | 0x10a..=0x111 | 0x113 | 0x206 | 0x325 | 0x408 => { + ServerControlMessage::Unhandled + } + _ => ServerControlMessage::Unknown(code), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn server_control_envelopes_are_walked_without_searching_payload_bytes() { + let bytes = [ + 0x99, 0x01, 4, 0, 0x09, 0x01, 0, 0, 0x05, 0x01, 4, 0, 0, 0, 0, 0, + ]; + assert_eq!( + server_control_messages(&bytes).collect::>(), + vec![ + Ok(ServerControlMessage::Unknown(0x199)), + Ok(ServerControlMessage::Timer(NvstSessionTimer { + timer_type: 5, + seconds_left: 0, + active: false, + })) + ] + ); + } + + #[test] + fn server_control_envelopes_reject_truncation_without_partial_payloads() { + for bytes in [&[4_u8, 1][..], &[4, 1, 4, 0, 60][..]] { + let mut messages = server_control_messages(bytes); + assert_eq!( + messages.next(), + Some(Err(ServerControlParseError::Truncated)) + ); + assert_eq!(messages.next(), None); + } + } + + #[test] + fn server_control_work_and_bytes_are_bounded() { + let maximum = command(0x199, &vec![0; usize::from(u16::MAX)]); + assert_eq!( + server_control_messages(&maximum).next(), + Some(Ok(ServerControlMessage::Unknown(0x199))) + ); + assert_eq!( + server_control_messages(&vec![0; MAX_SERVER_CONTROL_BYTES + 1]).next(), + Some(Err(ServerControlParseError::Oversized)) + ); + let bytes = [0x00, 0x01, 0, 0].repeat(MAX_SERVER_CONTROL_COMMANDS + 1); + let messages = server_control_messages(&bytes).collect::>(); + assert_eq!(messages.len(), MAX_SERVER_CONTROL_COMMANDS + 1); + assert_eq!( + messages.last(), + Some(&Err(ServerControlParseError::TooManyCommands)) + ); + assert!( + messages[..MAX_SERVER_CONTROL_COMMANDS] + .iter() + .all(Result::is_ok) + ); + } + + fn command(code: u16, payload: &[u8]) -> Vec { + let mut bytes = code.to_le_bytes().to_vec(); + bytes.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + bytes.extend_from_slice(payload); + bytes + } + + #[test] + fn idle_timers_preserve_little_endian_seconds_and_timer_type() { + for (code, timer_type) in [(0x104, 4), (0x105, 5)] { + let bytes = command(code, &60_u32.to_le_bytes()); + assert_eq!( + server_control_messages(&bytes).next(), + Some(Ok(ServerControlMessage::Timer(NvstSessionTimer { + timer_type, + seconds_left: 60, + active: timer_type == 4, + }))) + ); + for length in 0..4 { + assert_eq!( + server_control_messages(&command(code, &vec![0; length])).next(), + Some(Err(ServerControlParseError::Truncated)) + ); + } + } + } + + #[test] + fn termination_preserves_network_order_result_and_opaque_tail() { + let bytes = command(0x109, &[0x80, 0x03, 0x00, 0x23, 1, 2, 3]); + assert_eq!( + server_control_messages(&bytes).next(), + Some(Ok(ServerControlMessage::Termination { + wire_code: 0x8003_0023, + extra: &bytes[8..], + })) + ); + assert_eq!( + server_control_messages(&command(0x109, &[0; 3])).next(), + Some(Err(ServerControlParseError::Truncated)) + ); + } + + #[test] + fn server_feedback_mode_requires_exact_payload_and_normalizes_invalid_modes() { + for (raw, mode) in [ + (0_u32, FramePacingFeedbackMode::Never), + (1, FramePacingFeedbackMode::Interval), + (2, FramePacingFeedbackMode::Never), + (u32::MAX, FramePacingFeedbackMode::Never), + ] { + let payload = [2_u32.to_le_bytes(), raw.to_le_bytes()].concat(); + assert_eq!( + server_control_messages(&command(0x112, &payload)).next(), + Some(Ok(ServerControlMessage::FramePacingMode { + stream_index: 2, + mode + })) + ); + } + for length in [0, 4, 7, 9, 12] { + assert_eq!( + server_control_messages(&command(0x112, &vec![0; length])).next(), + Some(Err(ServerControlParseError::InvalidLength)) + ); + } + } + + #[test] + fn control_keepalive_failures_are_counted_and_reset_on_admission() { + let mut health = ControlKeepaliveHealth::default(); + health.record(false); + health.record(false); + assert_eq!(health.failures, 2); + assert_eq!(health.consecutive_failures, 2); + health.record(true); + assert_eq!(health.failures, 2); + assert_eq!(health.consecutive_failures, 0); + assert_eq!(health.recoveries, 1); + health.record(true); + assert_eq!(health.recoveries, 1); + } + + #[test] + fn control_keepalive_counters_saturate() { + let mut health = ControlKeepaliveHealth { + failures: u64::MAX, + consecutive_failures: u32::MAX, + recoveries: u64::MAX, + }; + health.record(false); + assert_eq!(health.failures, u64::MAX); + assert_eq!(health.consecutive_failures, u32::MAX); + health.record(true); + assert_eq!(health.recoveries, u64::MAX); + assert_eq!(health.consecutive_failures, 0); + } + + #[test] + fn geronimo_resumability_matches_reason_and_extended_code_predicate() { + for code in 0x8004_0000..=0x8004_0017 { + assert_eq!( + session_resumable(Some(0x37), Some(code)), + Some(!matches!( + code, + 0x8004_0003 | 0x8004_000b..=0x8004_000f | 0x8004_0011..=0x8004_0016 + )) + ); + assert_eq!( + session_resumable(Some(0x3b), Some(code)), + Some(matches!(code, 0x8004_0004 | 0x8004_0007 | 0x8004_0010)) + ); + } + assert_eq!( + session_resumable(Some(0x38), Some(0x8003_0007)), + Some(false) + ); + assert_eq!(session_resumable(None, Some(0x8004_0010)), None); + assert_eq!(session_resumable(Some(0x37), None), None); + assert_eq!(session_resumable(Some(0x3b), None), None); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control_tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control_tests.rs new file mode 100644 index 000000000..e609e62ff --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_server_control_tests.rs @@ -0,0 +1,132 @@ +use super::*; + +fn server_command(code: u16, payload: &[u8]) -> Vec { + let mut bytes = code.to_le_bytes().to_vec(); + bytes.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + bytes.extend_from_slice(payload); + bytes +} + +#[test] +fn inbound_idle_warning_and_expiry_update_state_without_ending_media() { + let feedback = NvstFeedbackState::default(); + let mut diagnostics = ServerControlDiagnostics::default(); + let (events, received) = mpsc::channel(); + assert!(!feedback.receive_server_control( + &server_command(0x104, &60_u32.to_le_bytes()), + &mut diagnostics, + &events, + )); + assert!(feedback.idle_warning_active.load(Ordering::Acquire)); + assert_eq!( + received.try_recv().unwrap(), + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 4, + seconds_left: 60, + active: true, + }) + ); + assert!(!feedback.receive_server_control( + &server_command(0x105, &0_u32.to_le_bytes()), + &mut diagnostics, + &events, + )); + assert!(!feedback.idle_warning_active.load(Ordering::Acquire)); + assert_eq!( + received.try_recv().unwrap(), + NvstReceiveEvent::SessionTimer(NvstSessionTimer { + timer_type: 5, + seconds_left: 0, + active: false, + }) + ); + assert_eq!(diagnostics, ServerControlDiagnostics::default()); +} + +#[test] +fn zero_second_warning_remains_active_until_clear_or_expiry() { + let feedback = NvstFeedbackState::default(); + let (events, received) = mpsc::channel(); + assert!(!feedback.receive_server_control( + &server_command(0x104, &0_u32.to_le_bytes()), + &mut ServerControlDiagnostics::default(), + &events, + )); + assert!(feedback.idle_warning_active.load(Ordering::Acquire)); + assert!(matches!(received.try_recv().unwrap(), NvstReceiveEvent::SessionTimer(NvstSessionTimer { active: true, seconds_left: 0, .. }))); +} + +#[test] +fn server_termination_preserves_wire_code_and_is_emitted_before_shutdown() { + for (wire_code, resumable) in [ + (0x8003_0023_u32, Some(false)), + (0x8004_0010, Some(true)), + (0xdead_beef, None), + ] { + let feedback = NvstFeedbackState::default(); + feedback.idle_warning_active.store(true, Ordering::Release); + let (events, received) = mpsc::channel(); + assert!(feedback.receive_server_control( + &server_command(0x109, &wire_code.to_be_bytes()), + &mut ServerControlDiagnostics::default(), + &events + )); + let NvstReceiveEvent::ServerTermination(termination) = received.try_recv().unwrap() else { + panic!("termination event required") + }; + assert_eq!(termination.wire_code, wire_code); + assert_eq!(termination.resumable(), resumable); + assert!(!feedback.idle_warning_active.load(Ordering::Acquire)); + } +} + +#[test] +fn server_feedback_mode_updates_only_the_owned_video_stream() { + let feedback = NvstFeedbackState::default(); + let mut diagnostics = ServerControlDiagnostics::default(); + let (events, received) = mpsc::channel(); + for (stream, mode, enabled) in [ + (0_u32, 0_u32, false), + (0, 1, true), + (1, 0, true), + (0, 99, false), + ] { + assert!(!feedback.receive_server_control( + &server_command(0x112, &[stream.to_le_bytes(), mode.to_le_bytes()].concat()), + &mut diagnostics, + &events + )); + assert_eq!( + feedback.frame_pacing_enabled.load(Ordering::Acquire), + enabled + ); + } + assert_eq!(diagnostics.invalid_stream, 1); + assert!(received.try_recv().is_err()); +} + +#[test] +fn unknown_and_malformed_server_commands_are_counted_not_terminal() { + let feedback = NvstFeedbackState::default(); + let mut diagnostics = ServerControlDiagnostics::default(); + let (events, received) = mpsc::channel(); + let bytes = [ + server_command(0x999, &[1, 2, 3]), + server_command(0x104, &[0]), + server_command(0x112, &[0; 8]), + ] + .concat(); + assert!(!feedback.receive_server_control(&bytes, &mut diagnostics, &events)); + assert_eq!(diagnostics.unknown, 1); + assert_eq!(diagnostics.last_unknown, Some(0x999)); + assert_eq!(diagnostics.malformed, 1); + assert!(!feedback.frame_pacing_enabled.load(Ordering::Acquire)); + assert!(received.try_recv().is_err()); + diagnostics.unknown = u64::MAX; + assert!(!feedback.receive_server_control( + &server_command(0x999, &[]), + &mut diagnostics, + &events + )); + assert_eq!(diagnostics.unknown, u64::MAX); +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/diagnostics.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/diagnostics.rs new file mode 100644 index 000000000..58f0ee63e --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/diagnostics.rs @@ -0,0 +1,185 @@ +use std::collections::HashSet; +use std::net::IpAddr; + +use serde::Serialize; +use serde_json::{Value, json}; +use str0m::Candidate; + +#[derive(Default, Serialize)] +pub(super) struct PacketCounts { + stun: u64, + dtls: u64, + rtp: u64, + other: u64, +} + +impl PacketCounts { + pub(super) fn record(&mut self, bytes: &[u8]) { + let count = match bytes.first().copied() { + Some(0..=3) if bytes.get(4..8) == Some(&[0x21, 0x12, 0xa4, 0x42]) => &mut self.stun, + Some(20..=63) => &mut self.dtls, + Some(128..=191) => &mut self.rtp, + _ => &mut self.other, + }; + *count = count.saturating_add(1); + } +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct VideoAssembly { + pub(super) frames: u64, + pub(super) keyframes: u64, + pub(super) discontinuities: u64, + pub(super) delivered: u64, + pub(super) awaiting_reference: u64, + pub(super) consumer_full: u64, + pub(super) keyframe_requests: u64, +} + +impl VideoAssembly { + pub(super) fn assembled(&mut self, keyframe: bool, contiguous: bool) { + self.frames = self.frames.saturating_add(1); + self.keyframes = self.keyframes.saturating_add(u64::from(keyframe)); + self.discontinuities = self.discontinuities.saturating_add(u64::from(!contiguous)); + } +} + +#[derive(Default, Serialize)] +pub(super) struct Traffic { + pub(super) transmitted: PacketCounts, + pub(super) received: PacketCounts, + pub(super) routed: PacketCounts, + pub(super) video: VideoAssembly, + pub(super) send_errors: u64, + pub(super) tx_to_known_peer: u64, + pub(super) rx_from_known_peer: u64, + pub(super) last_tx_port: Option, + pub(super) last_rx_port: Option, +} + +pub(super) fn address_class(ip: IpAddr) -> &'static str { + if ip.is_unspecified() { + "unspecified" + } else if ip.is_loopback() { + "loopback" + } else if ip.is_multicast() { + "multicast" + } else { + match ip { + IpAddr::V4(ip) if ip.is_private() => "private", + IpAddr::V4(ip) if ip.is_link_local() => "link-local", + IpAddr::V6(ip) if ip.is_unique_local() => "private", + IpAddr::V6(ip) if ip.is_unicast_link_local() => "link-local", + _ => "public", + } + } +} + +pub(super) fn candidate_summary(raw: &str, parsed: Option<&Candidate>) -> Value { + let parts: Vec<_> = raw.split_ascii_whitespace().take(8).collect(); + let advertised = parts.get(4).and_then(|value| value.parse::().ok()); + let protocol = match parts + .get(2) + .map(|value| value.to_ascii_lowercase()) + .as_deref() + { + Some("udp") => "udp", + Some("tcp") => "tcp", + _ => "other", + }; + let kind = match parts.get(7).copied() { + Some("host") => "host", + Some("srflx") => "srflx", + Some("prflx") => "prflx", + Some("relay") => "relay", + _ => "other", + }; + json!({"protocol":protocol,"kind":kind, + "advertisedClass":advertised.map(address_class).unwrap_or("hostname"), + "advertisedPort":parts.get(5).and_then(|value| value.parse::().ok()), + "accepted":parsed.is_some(), + "selectedClass":parsed.map(|candidate| address_class(candidate.addr().ip())), + "selectedPort":parsed.map(|candidate| candidate.addr().port())}) +} + +pub(super) fn sdp_summary(sdp: &str) -> Value { + let values = |prefix: &str| { + sdp.lines() + .filter_map(|line| line.strip_prefix(prefix)) + .take(16) + .collect::>() + }; + let ufrags = values("a=ice-ufrag:"); + let passwords = values("a=ice-pwd:"); + let roles = values("a=setup:") + .into_iter() + .map(|role| match role { + "active" => "active", + "passive" => "passive", + "actpass" => "actpass", + _ => "other", + }) + .collect::>(); + let rtcp_feedback = |kind: &str| { + sdp.lines() + .filter_map(|line| line.strip_prefix("a=rtcp-fb:")) + .filter(|line| { + line.split_once(' ') + .is_some_and(|(_, value)| value.trim() == kind) + }) + .count() + }; + json!({"iceLite":sdp.lines().any(|line| line == "a=ice-lite"), + "rtxPayloads":sdp.lines().filter(|line| line.starts_with("a=fmtp:") && line.contains("apt=")).count(), + "nackFeedback":rtcp_feedback("nack"), + "pliFeedback":rtcp_feedback("nack pli"), + "bundleSize":sdp.lines().find_map(|line| line.strip_prefix("a=group:BUNDLE ")).map(|mids| mids.split_whitespace().count()), + "candidateCount":sdp.lines().filter(|line| line.starts_with("a=candidate:")).count(), + "ufragLengths":ufrags.iter().map(|value| value.len()).collect::>(), + "passwordLengths":passwords.iter().map(|value| value.len()).collect::>(), + "ufragVariants":ufrags.iter().collect::>().len(), + "passwordVariants":passwords.iter().collect::>().len(), + "setupRoles":roles}) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trace_summaries_exclude_credentials_addresses_and_payloads() { + let sdp = "a=ice-ufrag:private-user\na=ice-pwd:private-password\na=setup:private-role\na=group:BUNDLE private-mid\na=candidate:private-foundation 1 udp 123 192.0.2.42 5555 typ host\na=rtcp-fb:96 nack\na=rtcp-fb:96 nack pli\na=fmtp:97 apt=96\n"; + let summary = sdp_summary(sdp); + assert_eq!(summary["candidateCount"], 1); + assert_eq!(summary["rtxPayloads"], 1); + assert_eq!(summary["nackFeedback"], 1); + assert_eq!(summary["pliFeedback"], 1); + assert_eq!(summary["bundleSize"], 1); + assert!(!summary.to_string().contains("private")); + let summary = candidate_summary( + "candidate:private-foundation 1 udp 123 private-host 5555 typ host ufrag private-user", + None, + ); + assert_eq!(summary["advertisedClass"], "hostname"); + assert_eq!(summary["advertisedPort"], 5555); + assert!(!summary.to_string().contains("private")); + let mut packets = PacketCounts::default(); + packets.record(&[0, 1, 0, 0, 0x21, 0x12, 0xa4, 0x42]); + packets.record(&[22, 0, 0]); + packets.record(&[128, 0, 0]); + packets.record(b"private-payload"); + assert_eq!( + serde_json::to_value(packets).unwrap(), + json!({"stun":1,"dtls":1,"rtp":1,"other":1}) + ); + let mut video = VideoAssembly::default(); + video.assembled(true, true); + video.assembled(false, false); + assert_eq!( + serde_json::to_value(video).unwrap(), + json!({"frames":2,"keyframes":1,"discontinuities":1,"delivered":0, + "awaitingReference":0,"consumerFull":0,"keyframeRequests":0}) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/input.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/input.rs new file mode 100644 index 000000000..4e3d6412a --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/input.rs @@ -0,0 +1,258 @@ +use std::time::Instant; + +use str0m::Rtc; +use str0m::channel::{ChannelConfig, ChannelId, Reliability}; + +use super::TransportError; + +const CHANNEL_BUFFER_BYTES: usize = 64 * 1024; + +pub(super) fn validate_packet(bytes: &[u8]) -> Result { + let kind = bytes + .get(..4) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_le_bytes) + .ok_or(TransportError::MalformedInput)?; + let expected = match kind { + 2 => 4, + 3 | 4 | 8 | 9 => 18, + 5 => 26, + 7 | 10 => 22, + 12 => 38, + 13 => 6, + 19 => 5, + _ => return Err(TransportError::UnsupportedInput), + }; + if bytes.len() != expected { + return Err(TransportError::MalformedInput); + } + Ok(kind) +} + +#[derive(Clone, Copy)] +pub(super) struct InputChannels { + reliable: ChannelId, + partial: ChannelId, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn versions_and_event_classes_use_verified_wire_wrappers() { + let start = Instant::now(); + for version in [1, 2, 3, 4] { + let state = InputChannelState { + reliable_open: true, + partial_open: true, + handshake: Some((version, start)), + reported: true, + }; + for (kind, length, batched) in [ + (3_u32, 18, false), + (4, 18, false), + (5, 26, true), + (7, 22, true), + (8, 18, false), + (9, 18, false), + (10, 22, false), + (12, 38, true), + ] { + let mut body = vec![0; length]; + body[..4].copy_from_slice(&kind.to_le_bytes()); + let encoded = state + .encode(&body, start + Duration::from_micros(1234)) + .unwrap(); + if version <= 2 { + assert_eq!(encoded, body); + continue; + } + assert_eq!(encoded[0], 0x23); + assert_eq!(&encoded[1..9], &1234_u64.to_be_bytes()); + assert_eq!(encoded[9], if batched { 0x21 } else { 0x22 }); + if batched { + assert_eq!(&encoded[10..12], &(length as u16).to_be_bytes()); + assert_eq!(&encoded[12..], body); + } else { + assert_eq!(&encoded[10..], body); + } + } + assert_eq!(state.encode(&[2, 0, 0, 0], start).unwrap(), [2, 0, 0, 0]); + } + } + + #[test] + fn handshake_requires_supported_layout_and_both_input_channels() { + crate::install_crypto(); + let mut rtc = Rtc::new(Instant::now()); + let channels = InputChannels::create(&mut rtc, 300); + let mut state = InputChannelState::default(); + assert_eq!(state.channel_opened(channels, channels.reliable), None); + assert_eq!( + state.channel_data(channels, channels.reliable, &[0x0d, 2]), + None + ); + assert_eq!( + state.channel_data(channels, channels.partial, &[0x0e, 2, 3, 0]), + None + ); + assert_eq!( + state.channel_data(channels, channels.reliable, &[0x0e, 2, 3, 0]), + None + ); + assert_eq!(state.channel_opened(channels, channels.partial), Some(3)); + assert!(state.is_ready()); + assert!(state.channel_closed(channels, channels.partial)); + assert!(!state.is_ready()); + } + + #[test] + fn handshake_accepts_android_and_historical_legacy_layouts() { + crate::install_crypto(); + for (packet, version) in [ + (&[0x0e, 2][..], 2), + (&[0x0e, 2, 3, 0][..], 3), + (&[0x0e, 3][..], 0x030e), + ] { + let mut rtc = Rtc::new(Instant::now()); + let channels = InputChannels::create(&mut rtc, 300); + let mut state = InputChannelState::default(); + state.channel_opened(channels, channels.reliable); + state.channel_opened(channels, channels.partial); + assert_eq!( + state.channel_data(channels, channels.reliable, packet), + Some(version) + ); + assert!(state.is_ready()); + } + } +} + +impl InputChannels { + pub(super) fn create(rtc: &mut Rtc, lifetime: u16) -> Self { + rtc.direct_api().create_data_channel(ChannelConfig { + label: "stats_channel".to_owned(), + ordered: false, + reliability: Reliability::MaxRetransmits { retransmits: 0 }, + ..Default::default() + }); + let reliable = rtc.direct_api().create_data_channel(ChannelConfig { + label: "input_channel_v1".to_owned(), + ..Default::default() + }); + let partial = rtc.direct_api().create_data_channel(ChannelConfig { + label: "input_channel_partially_reliable".to_owned(), + ordered: false, + reliability: Reliability::MaxPacketLifetime { lifetime }, + ..Default::default() + }); + Self { reliable, partial } + } + + pub(super) fn send(self, rtc: &mut Rtc, bytes: &[u8], partially_reliable: bool) -> bool { + let id = if partially_reliable { + self.partial + } else { + self.reliable + }; + rtc.channel(id).is_some_and(|mut channel| { + channel.buffered_amount().saturating_add(bytes.len()) <= CHANNEL_BUFFER_BYTES + && channel.write(true, bytes).unwrap_or(false) + }) + } +} + +#[derive(Default)] +pub(super) struct InputChannelState { + reliable_open: bool, + partial_open: bool, + handshake: Option<(u16, Instant)>, + reported: bool, +} + +impl InputChannelState { + pub(super) fn channel_opened(&mut self, channels: InputChannels, id: ChannelId) -> Option { + if id == channels.reliable { + self.reliable_open = true; + } + if id == channels.partial { + self.partial_open = true; + } + self.ready_version() + } + + pub(super) fn channel_data( + &mut self, + channels: InputChannels, + id: ChannelId, + bytes: &[u8], + ) -> Option { + if id == channels.reliable && bytes.len() >= 2 && self.handshake.is_none() { + let first = u16::from_le_bytes([bytes[0], bytes[1]]); + let version = if first == 526 { + Some(if bytes.len() >= 4 { + u16::from_le_bytes([bytes[2], bytes[3]]) + } else { + 2 + }) + } else { + (bytes[0] == 0x0e).then_some(first) + }; + if let Some(version) = version.filter(|version| *version != 0) { + self.handshake = Some((version, Instant::now())); + } + } + self.ready_version() + } + + pub(super) fn channel_closed(&mut self, channels: InputChannels, id: ChannelId) -> bool { + if id != channels.reliable && id != channels.partial { + return false; + } + self.reliable_open = false; + self.partial_open = false; + self.handshake = None; + self.reported = false; + true + } + + pub(super) fn is_ready(&self) -> bool { + self.reliable_open && self.partial_open && self.handshake.is_some() + } + + fn ready_version(&mut self) -> Option { + if self.reported || !self.is_ready() { + return None; + } + self.reported = true; + self.handshake.map(|(version, _)| version) + } + + pub(super) fn encode(&self, bytes: &[u8], now: Instant) -> Result, String> { + let Some((version, start)) = self.handshake else { + return Err("input handshake is incomplete".to_owned()); + }; + let kind = validate_packet(bytes).map_err(|error| error.to_string())?; + if version <= 2 || kind == 2 { + return Ok(bytes.to_vec()); + } + let mut framed = Vec::with_capacity(bytes.len() + 12); + framed.push(0x23); + let timestamp: u64 = now + .saturating_duration_since(start) + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + framed.extend_from_slice(×tamp.to_be_bytes()); + if matches!(kind, 5 | 7 | 12) { + framed.push(0x21); + framed.extend_from_slice(&(bytes.len() as u16).to_be_bytes()); + } else { + framed.push(0x22); + } + framed.extend_from_slice(bytes); + Ok(framed) + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/mod.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/mod.rs new file mode 100644 index 000000000..9ec10d01f --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/mod.rs @@ -0,0 +1,1155 @@ +use std::io::ErrorKind; +use std::net::{IpAddr, SocketAddr, UdpSocket}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TrySendError}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use opennow_streamer_protocol::Session; +use serde::{Deserialize, Serialize}; +use str0m::change::SdpOffer; +use str0m::format::{Codec, PayloadParams}; +use str0m::media::{Direction, Frequency, KeyframeRequestKind, MediaData, MediaKind, Mid}; +use str0m::net::{Protocol, Receive}; +use str0m::{Candidate, Event, IceConnectionState, Input, Output, Rtc, RtcConfig}; +use thiserror::Error; + +use crate::{EncodedMediaFrame, MediaConsumer, install_crypto}; + +mod diagnostics; +mod input; +mod stats; +#[cfg(test)] +mod tests; + +use input::{InputChannelState, InputChannels}; +use stats::Stats; +pub use stats::TransportStats; + +const COMMAND_CAPACITY: usize = 64; +const MAX_INPUT_BYTES: usize = 4096; +const MAX_SDP_BYTES: usize = 256 * 1024; +const MAX_CANDIDATE_BYTES: usize = 4096; +const MAX_CANDIDATES: usize = 64; +const MAX_MEDIA_SECTIONS: usize = 8; +const MAX_VIDEO_BYTES: usize = 16 * 1024 * 1024; +const POLL_INTERVAL: Duration = Duration::from_millis(10); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const INPUT_TIMEOUT: Duration = Duration::from_secs(5); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(2); +const KEYFRAME_INTERVAL: Duration = Duration::from_millis(500); +const MAX_FLUSH_TIME: Duration = Duration::from_millis(150); +const NACK_REPAIRABLE_FRAMES: usize = 101; + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct IceCandidate { + pub candidate: String, + #[serde(default)] + pub sdp_mid: Option, + #[serde(default)] + pub sdp_m_line_index: Option, + #[serde(default)] + pub username_fragment: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NegotiatedVideoCodec { + H264, + H265Main, + H265Main10, +} + +impl NegotiatedVideoCodec { + fn matches(self, params: &PayloadParams) -> bool { + let spec = params.spec(); + match self { + Self::H264 => spec.codec == Codec::H264, + Self::H265Main | Self::H265Main10 => { + let profile = spec + .format + .h265_profile_tier_level + .map(|ptl| u32::from(ptl.profile().to_id())) + .or(spec.format.profile_id) + .unwrap_or(1); + spec.codec == Codec::H265 && profile == if self == Self::H265Main10 { 2 } else { 1 } + } + } + } +} + +#[derive(Debug, Error)] +pub enum TransportError { + #[error("invalid WebRTC offer: {0}")] + Offer(String), + #[error("invalid WebRTC endpoint: {0}")] + Endpoint(String), + #[error("WebRTC socket failed: {0}")] + Io(#[from] std::io::Error), + #[error("invalid remote ICE candidate: {0}")] + RemoteCandidate(String), + #[error( + "native WebRTC currently supports direct UDP only; TURN relay gathering is unavailable" + )] + RelayUnsupported, + #[error("input channel is not ready")] + InputNotReady, + #[error("WebRTC command queue is full")] + Backpressured, + #[error("WebRTC command exceeds its size limit")] + OversizedCommand, + #[error("input mode or opcode is unsupported in WebRTC compatibility mode")] + UnsupportedInput, + #[error("input packet has an invalid length")] + MalformedInput, + #[error("WebRTC transport is closed")] + Closed, + #[error("WebRTC input flush timed out")] + FlushTimeout, +} + +impl TransportError { + pub const fn code(&self) -> &'static str { + match self { + Self::Offer(_) => "invalid-offer", + Self::Endpoint(_) => "invalid-media-endpoint", + Self::Io(_) => "local-transport-failed", + Self::RemoteCandidate(_) => "invalid-remote-candidate", + Self::RelayUnsupported => "webrtc-relay-unsupported", + Self::InputNotReady => "input-not-ready", + Self::Backpressured => "transport-backpressured", + Self::OversizedCommand => "transport-command-too-large", + Self::UnsupportedInput => "webrtc-input-unsupported", + Self::MalformedInput => "invalid-input", + Self::Closed => "transport-closed", + Self::FlushTimeout => "transport-flush-timeout", + } + } +} + +#[derive(Debug)] +pub enum TransportEvent { + Connected, + Disconnected(String), + InputReady(u16), + InputUnavailable(String), + Log(String), + Stats(TransportStats), +} + +enum TransportCommand { + AddRemoteCandidate(Candidate), + SendInput { + bytes: Vec, + partially_reliable: bool, + }, + RequestKeyframe(Mid), + FlushInput(SyncSender>), +} + +#[derive(Clone)] +pub struct TransportControl { + commands: SyncSender, + cancelled: Arc, + input_ready: Arc, +} + +impl TransportControl { + fn enqueue(&self, command: TransportCommand) -> Result<(), TransportError> { + if self.cancelled.load(Ordering::Acquire) { + return Err(TransportError::Closed); + } + self.commands + .try_send(command) + .map_err(|error| match error { + TrySendError::Full(_) => TransportError::Backpressured, + TrySendError::Disconnected(_) => TransportError::Closed, + }) + } + + pub fn send_input( + &self, + bytes: Vec, + partially_reliable: bool, + ) -> Result<(), TransportError> { + if partially_reliable { + return Err(TransportError::UnsupportedInput); + } + if bytes.len() > MAX_INPUT_BYTES { + return Err(TransportError::OversizedCommand); + } + input::validate_packet(&bytes)?; + if !self.input_ready.load(Ordering::Acquire) { + return Err(TransportError::InputNotReady); + } + self.enqueue(TransportCommand::SendInput { + bytes, + partially_reliable, + }) + } + + pub fn request_keyframe(&self, mid: impl Into) -> Result<(), TransportError> { + let mid = mid.into(); + if mid.is_empty() || mid.len() > 16 || !mid.is_ascii() { + return Err(TransportError::OversizedCommand); + } + self.enqueue(TransportCommand::RequestKeyframe(Mid::from(mid.as_str()))) + } + + pub fn flush_input(&self, timeout: Duration) -> Result<(), TransportError> { + let deadline = Instant::now() + timeout.min(MAX_FLUSH_TIME); + let (acknowledge, completed) = mpsc::sync_channel(1); + let mut command = TransportCommand::FlushInput(acknowledge); + loop { + if self.cancelled.load(Ordering::Acquire) { + return Err(TransportError::Closed); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(TransportError::FlushTimeout); + } + match self.commands.try_send(command) { + Ok(()) => break, + Err(TrySendError::Disconnected(_)) => return Err(TransportError::Closed), + Err(TrySendError::Full(pending)) => command = pending, + } + thread::sleep(remaining.min(Duration::from_millis(1))); + } + loop { + if self.cancelled.load(Ordering::Acquire) { + return Err(TransportError::Closed); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(TransportError::FlushTimeout); + } + match completed.recv_timeout(remaining.min(POLL_INTERVAL)) { + Ok(result) => return result, + Err(mpsc::RecvTimeoutError::Disconnected) => return Err(TransportError::Closed), + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } + } + + pub fn stop(&self) { + self.input_ready.store(false, Ordering::Release); + self.cancelled.store(true, Ordering::Release); + } +} + +pub struct TransportSession { + control: TransportControl, + worker: Option>, + media_endpoint: Option, +} + +impl TransportSession { + pub fn control(&self) -> TransportControl { + self.control.clone() + } + + pub fn add_remote_candidate(&self, candidate: &IceCandidate) -> Result<(), TransportError> { + if let Some(candidate) = parse_candidate(&candidate.candidate, self.media_endpoint)? { + self.control + .enqueue(TransportCommand::AddRemoteCandidate(candidate))?; + } + Ok(()) + } + + pub fn stop(self) { + drop(self); + } +} + +impl Drop for TransportSession { + fn drop(&mut self) { + self.control.stop(); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +pub struct NegotiatedTransport { + pub answer_sdp: String, + pub video_mid: Mid, + pub local_candidate: IceCandidate, + pub session: TransportSession, +} + +pub fn negotiate( + offer_sdp: &str, + session: &Session, + codec: NegotiatedVideoCodec, + partial_reliable_lifetime_ms: u16, + events: SyncSender, + media_consumer: MediaConsumer, +) -> Result { + install_crypto(); + if offer_sdp.len() > MAX_SDP_BYTES { + return Err(TransportError::Offer("SDP exceeds 256 KiB".to_owned())); + } + if session + .extra + .get("iceTransportPolicy") + .and_then(serde_json::Value::as_str) + == Some("relay") + { + return Err(TransportError::RelayUnsupported); + } + let media_endpoint = media_endpoint(session)?; + let normalized = normalize_offer(offer_sdp, media_endpoint)?; + let peer = match media_endpoint { + Some(endpoint) => endpoint.ip(), + None => resolve_host(&session.server_ip)?, + }; + let socket = bind_routed_socket(peer)?; + opennow_streamer_protocol::log::log_line("INFO", "webrtc-local", &serde_json::json!({ + "peerClass":diagnostics::address_class(peer), + "localClass":diagnostics::address_class(socket.local_addr()?.ip()), + "localPort":socket.local_addr()?.port(), + "mediaEndpointPresent":media_endpoint.is_some(), + "configuredIceServerCount":session.extra.get("iceServers").and_then(serde_json::Value::as_array).map_or(0, Vec::len) + }).to_string()); + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-remote-sdp", + &diagnostics::sdp_summary(offer_sdp).to_string(), + ); + socket.set_write_timeout(Some(POLL_INTERVAL))?; + let local = Candidate::host(socket.local_addr()?, "udp") + .map_err(|error| TransportError::Endpoint(error.to_string()))?; + let mut builder = RtcConfig::new() + .clear_codecs() + .enable_opus(true) + .set_reordering_size_audio(32) + .set_reordering_size_video(NACK_REPAIRABLE_FRAMES) + .set_send_buffer_audio(1) + .set_send_buffer_video(1) + .set_stats_interval(Some(Duration::from_secs(1))); + match codec { + NegotiatedVideoCodec::H264 => { + for (payload, profile) in [ + (96_u8, 0x42002a), + (98, 0x42e02a), + (100, 0x4d002a), + (102, 0x64002a), + ] { + builder.codec_config().add_h264( + payload.into(), + Some((payload + 1).into()), + true, + profile, + ); + } + } + NegotiatedVideoCodec::H265Main | NegotiatedVideoCodec::H265Main10 => { + let profile = if codec == NegotiatedVideoCodec::H265Main10 { + 2 + } else { + 1 + }; + for (payload, tier) in [(96_u8, 0), (98, 1)] { + builder.codec_config().add_h265( + payload.into(), + Some((payload + 1).into()), + profile, + tier, + 180, + ); + } + } + } + let mut rtc = builder.build(Instant::now()); + rtc.add_local_candidate(local.clone()); + let parsed_offer = SdpOffer::from_sdp_string(&normalized) + .map_err(|error| TransportError::Offer(error.to_string()))?; + let mut offer = (*parsed_offer).clone(); + let session_direction = normalized + .lines() + .take_while(|line| !line.starts_with("m=")) + .find_map(|line| match line { + "a=sendonly" => Some(Direction::SendOnly), + "a=recvonly" => Some(Direction::RecvOnly), + "a=inactive" => Some(Direction::Inactive), + "a=sendrecv" => Some(Direction::SendRecv), + _ => None, + }) + .unwrap_or(Direction::SendRecv); + let mut data_sections = 0; + let mut unsupported_media = false; + let mut sections = Vec::new(); + for media in &mut offer.media_lines { + let kind = match media.typ.to_string().as_str() { + "video" => "video", + "audio" => "audio", + "application" => "application", + _ => "other", + }; + let offered_direction = if media.attrs.iter().any(|attribute| attribute.is_direction()) { + media.direction() + } else { + session_direction + }; + sections.push(serde_json::json!({"kind":kind,"direction":offered_direction.to_string(),"portZero":media.disabled})); + if media.typ.is_channel() { + data_sections += 1; + } else if media.typ.is_media() { + let direction = if offered_direction.is_sending() { + Direction::SendOnly + } else { + Direction::Inactive + }; + media.attrs.retain(|attribute| !attribute.is_direction()); + media.attrs.push(direction.into()); + } else { + unsupported_media = true; + } + } + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-offer", + &serde_json::json!({"sections":sections}).to_string(), + ); + if unsupported_media { + return Err(TransportError::Offer( + "unsupported media section kind".to_owned(), + )); + } + if data_sections > 1 { + return Err(TransportError::Offer( + "multiple data transports are unsupported".to_owned(), + )); + } + if normalized + .lines() + .find_map(|line| line.strip_prefix("a=setup:")) + .is_some_and(|setup| setup.trim() == "actpass") + { + rtc.direct_api() + .set_ice_controlling(offer.session.ice_lite()); + rtc.direct_api() + .start_dtls(true) + .map_err(|error| TransportError::Offer(error.to_string()))?; + } + let answer = rtc + .sdp_api() + .accept_offer(offer.into()) + .map_err(|error| TransportError::Offer(error.to_string()))?; + let mut answer = (*answer).clone(); + for section in &mut answer.media_lines { + if section.typ.is_media() && section.disabled && section.pts.is_empty() { + let offered = parsed_offer + .media_lines + .iter() + .find(|offered| offered.mid() == section.mid()) + .ok_or_else(|| { + TransportError::Offer("rejected media section has no matching offer".to_owned()) + })?; + section.pts.clone_from(&offered.pts); + } + } + let mut video_mid = None; + let mut audio_mid = None; + for section in &answer.media_lines { + if !section.typ.is_media() { + continue; + } + let mid = section.mid(); + let Some(media) = rtc + .media(mid) + .filter(|media| !media.disabled() && media.direction().is_receiving()) + else { + continue; + }; + match media.kind() { + MediaKind::Video => { + if video_mid.is_some() { + return Err(TransportError::Offer( + "multiple incoming video tracks are unsupported".to_owned(), + )); + } + let Some(clock_rate) = section + .rtp_params() + .iter() + .find(|params| codec.matches(params)) + .map(|params| params.spec().clock_rate) + else { + return Err(TransportError::Offer(format!( + "peer did not negotiate {codec:?} video" + ))); + }; + video_mid = Some((mid, clock_rate)); + } + MediaKind::Audio => { + if audio_mid.is_some() { + return Err(TransportError::Offer( + "multiple incoming audio tracks are unsupported".to_owned(), + )); + } + if !section + .rtp_params() + .iter() + .any(|params| params.spec().codec == Codec::Opus) + { + return Err(TransportError::Offer( + "peer did not negotiate Opus audio".to_owned(), + )); + } + audio_mid = Some(mid); + } + } + } + let (video_mid, video_clock_rate) = video_mid + .ok_or_else(|| TransportError::Offer(format!("peer did not negotiate {codec:?} video")))?; + let answer_sdp = answer.to_string(); + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-local-sdp", + &diagnostics::sdp_summary(&answer_sdp).to_string(), + ); + let (candidate_mid, candidate_index) = candidate_target(&answer_sdp)?; + let active_track = |kind: &str| { + answer_sdp.lines().any(|line| { + let mut tokens = line.split_ascii_whitespace(); + tokens.next() == Some(kind) && tokens.next().is_some_and(|port| port != "0") + }) + }; + if !active_track("m=application") { + return Err(TransportError::Offer( + "peer did not negotiate input data channels".to_owned(), + )); + } + let channels = InputChannels::create(&mut rtc, partial_reliable_lifetime_ms); + let (commands, receiver) = mpsc::sync_channel(COMMAND_CAPACITY); + let control = TransportControl { + commands, + cancelled: Arc::new(AtomicBool::new(false)), + input_ready: Arc::new(AtomicBool::new(false)), + }; + let cancelled = Arc::clone(&control.cancelled); + let input_ready = Arc::clone(&control.input_ready); + let worker = thread::Builder::new() + .name("opennow-webrtc".to_owned()) + .spawn(move || { + let mut runtime = TransportWorker { + rtc, + socket, + commands: receiver, + events, + media_consumer, + video_codec: codec, + stats: Stats::new(video_mid, video_clock_rate), + channels, + input: InputChannelState::default(), + input_ready, + cancelled, + origin: Instant::now(), + connected_at: None, + next_heartbeat: Instant::now() + HEARTBEAT_INTERVAL, + video: VideoRecovery::default(), + remote_candidates: normalized + .lines() + .filter(|line| line.starts_with("a=candidate:")) + .count(), + video_drops: 0, + audio_drops: 0, + last_drop_report: Instant::now(), + traffic: diagnostics::Traffic::default(), + peer, + last_connection_trace: Instant::now(), + }; + let result = runtime.run(); + runtime.trace_connection("finished"); + runtime.input_ready.store(false, Ordering::Release); + runtime.rtc.disconnect(); + drop(runtime.commands); + drop(runtime.socket); + drop(runtime.rtc); + let mut event = + TransportEvent::Disconnected(result.err().unwrap_or_else(|| "stopped".to_owned())); + loop { + match runtime.events.try_send(event) { + Ok(()) | Err(TrySendError::Disconnected(_)) => break, + Err(TrySendError::Full(pending)) => event = pending, + } + if runtime.cancelled.load(Ordering::Acquire) { + break; + } + thread::sleep(POLL_INTERVAL); + } + })?; + Ok(NegotiatedTransport { + answer_sdp, + video_mid, + local_candidate: IceCandidate { + candidate: local.to_sdp_string(), + sdp_mid: Some(candidate_mid), + sdp_m_line_index: Some(candidate_index), + username_fragment: None, + }, + session: TransportSession { + control, + worker: Some(worker), + media_endpoint, + }, + }) +} + +fn resolve_host(host: &str) -> Result { + if host.is_empty() || host.len() > 253 { + return Err(TransportError::Endpoint( + "invalid hostname length".to_owned(), + )); + } + if let Ok(ip) = host.parse() { + return Ok(ip); + } + if let Some(label) = host.split('.').next() { + let octets = label + .split('-') + .map(str::parse::) + .collect::, _>>(); + if let Ok(octets) = octets { + if let Ok(octets) = <[u8; 4]>::try_from(octets) { + return Ok(IpAddr::V4(octets.into())); + } + } + } + Err(TransportError::Endpoint( + "caller must resolve media hostname before starting WebRTC".to_owned(), + )) +} + +fn media_endpoint(session: &Session) -> Result, TransportError> { + let Some(endpoint) = session + .media_connection_info + .as_ref() + .filter(|endpoint| matches!(endpoint.usage, Some(2 | 14 | 15 | 17))) + else { + return Ok(None); + }; + let port = u16::try_from(endpoint.port) + .ok() + .filter(|port| *port != 0) + .ok_or_else(|| TransportError::Endpoint("media port is outside 1..=65535".to_owned()))?; + Ok(Some(SocketAddr::new(resolve_host(&endpoint.ip)?, port))) +} + +fn normalize_offer(sdp: &str, endpoint: Option) -> Result { + let mut lines = Vec::new(); + let mut candidates = 0; + let mut direct_candidates = 0; + let mut relay_candidates = 0; + let mut media_lines = 0; + for line in sdp.lines() { + if line.starts_with("a=candidate:") { + candidates += 1; + if candidates > MAX_CANDIDATES { + return Err(TransportError::Offer("too many ICE candidates".to_owned())); + } + if let Some(candidate) = parse_candidate(line, endpoint)? { + if line.split_ascii_whitespace().nth(7) == Some("relay") { + relay_candidates += 1; + } else { + direct_candidates += 1; + } + lines.push(format!("a={}", candidate.to_sdp_string())); + } + } else { + if line.starts_with("m=") { + media_lines += 1; + if media_lines > MAX_MEDIA_SECTIONS { + return Err(TransportError::Offer( + "too many SDP media sections".to_owned(), + )); + } + } + lines.push(line.to_owned()); + } + } + if relay_candidates > 0 && direct_candidates == 0 { + return Err(TransportError::RelayUnsupported); + } + Ok(lines.join("\r\n") + "\r\n") +} + +fn parse_candidate( + text: &str, + endpoint: Option, +) -> Result, TransportError> { + if text.len() > MAX_CANDIDATE_BYTES { + return Err(TransportError::OversizedCommand); + } + let text = text.trim(); + let text = text.strip_prefix("a=").unwrap_or(text); + if text.is_empty() || text == "end-of-candidates" { + return Ok(None); + } + let mut tokens: Vec<_> = text.split_ascii_whitespace().map(str::to_owned).collect(); + if tokens.len() < 8 || !tokens[0].starts_with("candidate:") { + return Err(TransportError::RemoteCandidate( + "malformed candidate".to_owned(), + )); + } + if !tokens[2].eq_ignore_ascii_case("udp") { + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-candidate", + &diagnostics::candidate_summary(text, None).to_string(), + ); + return Ok(None); + } + if let Some(endpoint) = endpoint { + let needs_rewrite = match tokens[4].parse::() { + Ok(IpAddr::V4(ip)) => { + ip.is_private() || ip.is_loopback() || ip.is_unspecified() || ip.is_link_local() + } + Ok(IpAddr::V6(ip)) => { + ip.is_unique_local() + || ip.is_loopback() + || ip.is_unspecified() + || ip.is_unicast_link_local() + } + Err(_) => true, + }; + if tokens[7] == "host" && needs_rewrite { + tokens[4] = endpoint.ip().to_string(); + tokens[5] = endpoint.port().to_string(); + } + } + let candidate = Candidate::from_sdp_string(&tokens.join(" ")) + .map_err(|error| TransportError::RemoteCandidate(error.to_string()))?; + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-candidate", + &diagnostics::candidate_summary(text, Some(&candidate)).to_string(), + ); + Ok(Some(candidate)) +} + +fn candidate_target(sdp: &str) -> Result<(String, u16), TransportError> { + let bundled = sdp + .lines() + .find_map(|line| line.strip_prefix("a=group:BUNDLE ")) + .and_then(|mids| mids.split_ascii_whitespace().next()); + let mut index = None; + let mut active = false; + let mut first = None; + for line in sdp.lines() { + if line.starts_with("m=") { + index = Some(index.map_or(0_u16, |index| index + 1)); + active = line.split_ascii_whitespace().nth(1) != Some("0"); + } else if active { + if let Some(mid) = line.strip_prefix("a=mid:") { + let target = (mid.to_owned(), index.unwrap_or(0)); + if bundled == Some(mid) { + return Ok(target); + } + first.get_or_insert(target); + } + } + } + first.ok_or_else(|| TransportError::Offer("answer has no active media id".to_owned())) +} + +fn bind_routed_socket(peer: IpAddr) -> Result { + let unspecified = if peer.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }; + let probe = UdpSocket::bind(unspecified)?; + probe.connect(SocketAddr::new(peer, 9))?; + let socket = UdpSocket::bind(SocketAddr::new(probe.local_addr()?.ip(), 0))?; + #[cfg(windows)] + let socket = { + let socket = socket2::Socket::from(socket); + crate::nvst::disable_udp_connreset(&socket)?; + UdpSocket::from(socket) + }; + Ok(socket) +} + +fn transient_udp_error(error: &std::io::Error) -> bool { + matches!( + error.kind(), + ErrorKind::ConnectionReset | ErrorKind::ConnectionRefused | ErrorKind::Interrupted + ) +} + +#[derive(Default)] +struct VideoRecovery { + mid: Option, + ssrc: Option, + reference_ready: bool, + last_request: Option, +} + +struct TransportWorker { + rtc: Rtc, + socket: UdpSocket, + commands: Receiver, + events: SyncSender, + media_consumer: MediaConsumer, + video_codec: NegotiatedVideoCodec, + stats: Stats, + channels: InputChannels, + input: InputChannelState, + input_ready: Arc, + cancelled: Arc, + origin: Instant, + connected_at: Option, + next_heartbeat: Instant, + video: VideoRecovery, + remote_candidates: usize, + video_drops: u64, + audio_drops: u64, + last_drop_report: Instant, + traffic: diagnostics::Traffic, + peer: IpAddr, + last_connection_trace: Instant, +} + +impl TransportWorker { + fn trace_connection(&self, phase: &str) { + opennow_streamer_protocol::log::log_line("INFO", "webrtc-connection", &serde_json::json!({ + "phase":phase,"elapsedMs":self.origin.elapsed().as_millis(), + "iceAndDtlsConnected":self.connected_at.is_some(), "remoteCandidates":self.remote_candidates, + "traffic":self.traffic + }).to_string()); + } + + fn emit(&self, event: TransportEvent) -> Result<(), String> { + self.events + .try_send(event) + .map_err(|_| "WebRTC event consumer is unavailable or backpressured".to_owned()) + } + + fn request_keyframe(&mut self, mid: Mid) { + let now = Instant::now(); + if self + .video + .last_request + .is_some_and(|last| now.duration_since(last) < KEYFRAME_INTERVAL) + { + return; + } + if let Some(mut writer) = self.rtc.writer(mid) { + let kind = [KeyframeRequestKind::Pli, KeyframeRequestKind::Fir] + .into_iter() + .find(|kind| writer.is_request_keyframe_possible(*kind)); + if let Some(kind) = kind { + if writer.request_keyframe(None, kind).is_ok() { + self.traffic.video.keyframe_requests = + self.traffic.video.keyframe_requests.saturating_add(1); + self.video.last_request = Some(now); + } + } + } + } + + fn media(&mut self, data: MediaData) -> Result<(), String> { + let codec = data.params.spec().codec; + let is_video = self.video_codec.matches(&data.params); + if !is_video && codec != Codec::Opus { + return Err("received an unnegotiated media codec".to_owned()); + } + let keyframe = data.is_keyframe(); + let ssrc = self + .rtc + .direct_api() + .stream_rx_by_mid(data.mid, data.rid) + .map(|stream| *stream.ssrc()); + if is_video { + self.traffic.video.assembled(keyframe, data.contiguous); + if self.video.mid.is_some_and(|mid| mid != data.mid) { + return Err("multiple video tracks are unsupported".to_owned()); + } + self.video.mid = Some(data.mid); + if self.video.ssrc != ssrc || !data.contiguous || data.data.len() > MAX_VIDEO_BYTES { + self.video.reference_ready = false; + } + self.video.ssrc = ssrc; + if data.data.len() > MAX_VIDEO_BYTES || (!self.video.reference_ready && !keyframe) { + self.traffic.video.awaiting_reference = + self.traffic.video.awaiting_reference.saturating_add(1); + self.video_drops = self.video_drops.saturating_add(1); + self.request_keyframe(data.mid); + return Ok(()); + } + } + let frame = EncodedMediaFrame { + mid: data.mid.to_string(), + codec: if is_video { + codec.to_string() + } else { + "Opus".to_owned() + }, + payload: data.data, + frame_index: None, + rtp_timestamp: data.time.numer() & u64::from(u32::MAX), + clock_rate_hz: data.time.denom(), + channels: data.params.spec().channels, + received_at_us: data + .network_time + .saturating_duration_since(self.origin) + .as_micros() + .try_into() + .unwrap_or(u64::MAX), + keyframe, + contiguous: data.contiguous && (!is_video || self.video.reference_ready), + ssrc, + }; + match self.media_consumer.try_send(frame) { + Ok(()) => { + if is_video { + self.traffic.video.delivered = self.traffic.video.delivered.saturating_add(1); + self.video.reference_ready |= keyframe; + } + Ok(()) + } + Err(TrySendError::Full(_)) => { + if is_video { + self.traffic.video.consumer_full = + self.traffic.video.consumer_full.saturating_add(1); + self.video_drops = self.video_drops.saturating_add(1); + self.video.reference_ready = false; + self.request_keyframe(data.mid); + } else { + self.audio_drops = self.audio_drops.saturating_add(1); + } + Ok(()) + } + Err(TrySendError::Disconnected(_)) => Err("encoded media consumer closed".to_owned()), + } + } + + fn event(&mut self, event: Event) -> Result<(), String> { + match event { + Event::PeerStats(stats) => { + let stats = self.stats.peer(stats); + self.emit(TransportEvent::Stats(stats))?; + } + Event::MediaIngressStats(stats) => { + if let Some(stats) = self.stats.media(stats) { + self.emit(TransportEvent::Stats(stats))?; + } + } + Event::Connected => { + self.connected_at = Some(Instant::now()); + self.trace_connection("connected"); + self.emit(TransportEvent::Connected)?; + } + Event::IceConnectionStateChange(IceConnectionState::Disconnected) => { + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-ice", + "state=Disconnected", + ); + return Err( + "ICE disconnected (direct UDP only; TURN relay gathering is unavailable)" + .to_owned(), + ); + } + Event::IceConnectionStateChange(state) => { + opennow_streamer_protocol::log::log_line( + "INFO", + "webrtc-ice", + &format!("state={state:?}"), + ); + } + Event::ChannelOpen(id, _) => { + if let Some(version) = self.input.channel_opened(self.channels, id) { + self.input_ready.store(true, Ordering::Release); + self.emit(TransportEvent::InputReady(version))?; + } + } + Event::ChannelData(data) => { + if let Some(version) = self.input.channel_data(self.channels, data.id, &data.data) { + self.input_ready.store(true, Ordering::Release); + self.emit(TransportEvent::InputReady(version))?; + } + } + Event::ChannelClose(id) if self.input.channel_closed(self.channels, id) => { + self.input_ready.store(false, Ordering::Release); + self.emit(TransportEvent::InputUnavailable( + "input data channel closed".to_owned(), + ))?; + return Err("input data channel closed".to_owned()); + } + Event::MediaData(data) => self.media(data)?, + _ => {} + } + Ok(()) + } + + fn run(&mut self) -> Result<(), String> { + let mut bytes = [0_u8; 65_536]; + let destination = self + .socket + .local_addr() + .map_err(|error| error.to_string())?; + loop { + if self.cancelled.load(Ordering::Acquire) { + return Ok(()); + } + let now = Instant::now(); + if now.duration_since(self.last_connection_trace) >= Duration::from_secs(5) + && (self.connected_at.is_none() + || now.duration_since(self.origin) < Duration::from_secs(20)) + { + self.trace_connection("progress"); + self.last_connection_trace = now; + } + match self.connected_at { + None if now.duration_since(self.origin) >= CONNECT_TIMEOUT => return Err("WebRTC connection timed out (direct UDP only; TURN relay gathering is unavailable)".to_owned()), + Some(connected) if !self.input.is_ready() && now.duration_since(connected) >= INPUT_TIMEOUT => return Err("WebRTC input handshake timed out".to_owned()), + _ => {} + } + let mut flush = None; + for _ in 0..COMMAND_CAPACITY { + match self.commands.try_recv() { + Ok(TransportCommand::AddRemoteCandidate(candidate)) => { + self.remote_candidates += 1; + if self.remote_candidates > MAX_CANDIDATES { + return Err("too many trickled ICE candidates".to_owned()); + } + self.rtc.add_remote_candidate(candidate); + } + Ok(TransportCommand::SendInput { + bytes, + partially_reliable, + }) => { + let bytes = self.input.encode(&bytes, Instant::now())?; + if !self.input.is_ready() + || !self + .channels + .send(&mut self.rtc, &bytes, partially_reliable) + { + return Err("input channel is unavailable or backpressured".to_owned()); + } + } + Ok(TransportCommand::RequestKeyframe(mid)) => { + self.video.reference_ready = false; + self.request_keyframe(mid); + } + Ok(TransportCommand::FlushInput(acknowledge)) => { + flush = Some(acknowledge); + self.rtc + .handle_input(Input::Timeout(Instant::now())) + .map_err(|error| error.to_string())?; + break; + } + Err(mpsc::TryRecvError::Empty) => break, + Err(mpsc::TryRecvError::Disconnected) => return Ok(()), + } + } + if self.input.is_ready() && now >= self.next_heartbeat { + if !self.channels.send(&mut self.rtc, &[2, 0, 0, 0], false) { + return Err("input heartbeat is backpressured".to_owned()); + } + self.next_heartbeat = now + HEARTBEAT_INTERVAL; + } + if !self.video.reference_ready { + if let Some(mid) = self.video.mid { + self.request_keyframe(mid); + } + } + if now.duration_since(self.last_drop_report) >= Duration::from_secs(1) + && (self.video_drops > 0 || self.audio_drops > 0) + { + if self + .events + .try_send(TransportEvent::Log(format!( + "WebRTC media dropped video={} audio={} awaiting_keyframe={}", + self.video_drops, self.audio_drops, !self.video.reference_ready, + ))) + .is_ok() + { + self.video_drops = 0; + self.audio_drops = 0; + } + self.last_drop_report = now; + } + let mut flush_error = None; + let timeout = loop { + if self.cancelled.load(Ordering::Acquire) { + return Ok(()); + } + match self.rtc.poll_output().map_err(|error| error.to_string())? { + Output::Timeout(timeout) => break timeout, + Output::Transmit(packet) => { + self.traffic.transmitted.record(&packet.contents); + self.traffic.last_tx_port = Some(packet.destination.port()); + if packet.destination.ip() == self.peer { + self.traffic.tx_to_known_peer += 1; + } + if let Err(error) = + self.socket.send_to(&packet.contents, packet.destination) + { + self.traffic.send_errors += 1; + if !transient_udp_error(&error) { + return Err(error.to_string()); + } + flush_error = Some(error); + } + } + Output::Event(event) => self.event(event)?, + } + }; + if let Some(acknowledge) = flush { + let _ = acknowledge + .try_send(flush_error.map_or(Ok(()), |error| Err(TransportError::Io(error)))); + } + let now = Instant::now(); + let wait = timeout.saturating_duration_since(now).min(POLL_INTERVAL); + if wait.is_zero() { + self.rtc + .handle_input(Input::Timeout(now)) + .map_err(|error| error.to_string())?; + continue; + } + self.socket + .set_read_timeout(Some(wait)) + .map_err(|error| error.to_string())?; + match self.socket.recv_from(&mut bytes) { + Ok((length, source)) => { + self.traffic.received.record(&bytes[..length]); + self.traffic.last_rx_port = Some(source.port()); + if source.ip() == self.peer { + self.traffic.rx_from_known_peer += 1; + } + let Ok(contents) = (&bytes[..length]).try_into() else { + continue; + }; + let input = Input::Receive( + Instant::now(), + Receive { + proto: Protocol::Udp, + source, + destination, + contents, + }, + ); + if self.rtc.accepts(&input) { + self.traffic.routed.record(&bytes[..length]); + self.rtc + .handle_input(input) + .map_err(|error| error.to_string())?; + } + } + Err(error) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) + || transient_udp_error(&error) => + { + self.rtc + .handle_input(Input::Timeout(Instant::now())) + .map_err(|error| error.to_string())?; + } + Err(error) => return Err(error.to_string()), + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/stats.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/stats.rs new file mode 100644 index 000000000..497a02c34 --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/stats.rs @@ -0,0 +1,77 @@ +use super::*; +use str0m::stats::{MediaIngressStats, PeerStats}; + +const PING_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone, Copy, Default)] +pub struct TransportStats { + pub ping_ms: Option, + pub jitter_ms: Option, + pub packet_loss_percent: Option, + pub socket_receive_bytes: Option, +} + +pub(super) struct Stats { + video_mid: Mid, + video_clock_rate: Frequency, + previous_responses: u64, + ice_ping: Option<(Instant, Duration)>, + peer_loss: Option, + video_loss: Option, + latest: TransportStats, +} + +impl Stats { + pub(super) fn new(video_mid: Mid, video_clock_rate: Frequency) -> Self { + Self { + video_mid, + video_clock_rate, + previous_responses: 0, + ice_ping: None, + peer_loss: None, + video_loss: None, + latest: TransportStats::default(), + } + } + + pub(super) fn peer(&mut self, stats: PeerStats) -> TransportStats { + match stats.selected_candidate_pair { + None => { + self.previous_responses = 0; + self.ice_ping = None; + } + Some(pair) if pair.responses_received != self.previous_responses => { + self.previous_responses = pair.responses_received; + self.ice_ping = pair + .current_round_trip_time + .map(|rtt| (stats.timestamp, rtt)); + } + Some(_) => {} + } + self.latest.ping_ms = self + .ice_ping + .filter(|(at, _)| stats.timestamp.saturating_duration_since(*at) < PING_TIMEOUT) + .map(|(_, rtt)| rtt.as_secs_f64() * 1_000.0); + self.latest.socket_receive_bytes = Some(stats.peer_bytes_rx); + self.peer_loss = stats.ingress_loss_fraction; + self.latest.packet_loss_percent = self + .video_loss + .or(self.peer_loss) + .map(|loss| f64::from(loss) * 100.0); + self.latest + } + + pub(super) fn media(&mut self, stats: MediaIngressStats) -> Option { + if stats.mid != self.video_mid { + return None; + } + self.latest.jitter_ms = + Some(f64::from(stats.jitter) * 1_000.0 / f64::from(self.video_clock_rate.get())); + self.video_loss = stats.loss; + self.latest.packet_loss_percent = self + .video_loss + .or(self.peer_loss) + .map(|loss| f64::from(loss) * 100.0); + Some(self.latest) + } +} diff --git a/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/tests.rs b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/tests.rs new file mode 100644 index 000000000..d2e5c45de --- /dev/null +++ b/external/opennow/native/opennow-streamer/crates/opennow-streamer-transport/src/webrtc/tests.rs @@ -0,0 +1,1385 @@ +use super::*; +use str0m::change::SdpAnswer; +use str0m::channel::ChannelId; +use str0m::media::{Direction, Frequency, MediaKind, MediaTime}; +use str0m::stats::{CandidatePairStats, CandidateStats, MediaIngressStats, PeerStats}; + +fn peer_stats(now: Instant) -> PeerStats { + PeerStats { + peer_bytes_rx: 1_000_000, + peer_bytes_tx: 0, + bytes_rx: 900_000, + bytes_tx: 0, + timestamp: now, + bwe_tx: None, + egress_loss_fraction: None, + ingress_loss_fraction: Some(0.025), + rtt: Some(Duration::from_millis(99)), + selected_candidate_pair: Some(CandidatePairStats { + protocol: Protocol::Udp, + local: CandidateStats { + addr: "127.0.0.1:1000".parse().unwrap(), + }, + remote: CandidateStats { + addr: "127.0.0.1:2000".parse().unwrap(), + }, + current_round_trip_time: Some(Duration::from_millis(12)), + total_round_trip_time: Duration::from_millis(12), + responses_received: 1, + }), + } +} + +#[test] +fn transport_stats_translate_video_jitter_and_ingress_loss_without_inventing_rtt() { + let now = Instant::now(); + let video = Mid::from("video"); + let mut stats = Stats::new(video, Frequency::NINETY_KHZ); + let translated = stats.peer(peer_stats(now)); + assert_eq!(translated.socket_receive_bytes, Some(1_000_000)); + assert_eq!(translated.ping_ms, Some(12.0)); + assert!((translated.packet_loss_percent.unwrap() - 2.5).abs() < 0.000_001); + assert!(translated.jitter_ms.is_none()); + let mut media = MediaIngressStats { + mid: video, + rid: None, + bytes: 900_000, + packets: 100, + firs: 0, + plis: 0, + nacks: 0, + jitter: 225, + rtt: Some(Duration::from_millis(99)), + loss: None, + timestamp: now, + remote: None, + }; + assert_eq!(stats.media(media.clone()).unwrap().jitter_ms, Some(2.5)); + media.mid = Mid::from("audio"); + media.jitter = 48_000; + assert!(stats.media(media.clone()).is_none()); + let mut peer = peer_stats(now + Duration::from_secs(1)); + peer.selected_candidate_pair + .as_mut() + .unwrap() + .current_round_trip_time = Some(Duration::from_millis(50)); + assert_eq!(stats.peer(peer.clone()).ping_ms, Some(12.0)); + peer.timestamp = now + Duration::from_secs(5); + assert!(stats.peer(peer.clone()).ping_ms.is_none()); + peer.selected_candidate_pair + .as_mut() + .unwrap() + .responses_received = 2; + assert_eq!(stats.peer(peer.clone()).ping_ms, Some(50.0)); + peer.selected_candidate_pair + .as_mut() + .unwrap() + .responses_received = 3; + peer.selected_candidate_pair + .as_mut() + .unwrap() + .current_round_trip_time = None; + assert!(stats.peer(peer.clone()).ping_ms.is_none()); + peer.selected_candidate_pair = None; + peer.ingress_loss_fraction = None; + let translated = stats.peer(peer); + assert!(translated.ping_ms.is_none()); + assert!(translated.packet_loss_percent.is_none()); + let mut alternate_clock = Stats::new(video, Frequency::FORTY_EIGHT_KHZ); + media.mid = video; + media.jitter = 120; + assert_eq!( + alternate_clock.media(media.clone()).unwrap().jitter_ms, + Some(2.5) + ); + media.loss = Some(0.125); + let video_only = alternate_clock.media(media.clone()).unwrap(); + assert_eq!(video_only.packet_loss_percent, Some(12.5)); + assert!(video_only.ping_ms.is_none()); + assert!(video_only.socket_receive_bytes.is_none()); + let mut no_twcc = peer_stats(now); + no_twcc.ingress_loss_fraction = None; + assert_eq!( + alternate_clock.peer(no_twcc.clone()).packet_loss_percent, + Some(12.5) + ); + media.loss = Some(0.0); + assert_eq!( + alternate_clock + .media(media.clone()) + .unwrap() + .packet_loss_percent, + Some(0.0) + ); + media.loss = None; + assert!( + alternate_clock + .media(media.clone()) + .unwrap() + .packet_loss_percent + .is_none() + ); + media.jitter = 0; + assert_eq!(alternate_clock.media(media).unwrap().jitter_ms, Some(0.0)); +} + +struct Peer { + rtc: Rtc, + socket: UdpSocket, + video: Mid, + audio: Mid, + transport: Option, + events: Receiver, + event_sender: SyncSender, + drain_events: bool, + media: Receiver, + connected: bool, + input_ready: bool, + received: Vec>, + stats: Option, + keyframe_requests: usize, + input_channel: Option, + client_address: SocketAddr, + answer_sdp: String, + rtp_transmits: usize, + drop_rtp_number: Option, + dropped_rtp: usize, + drop_payload_types: Vec, + hold_rtp_number: Option, + held_rtp: Option<(Vec, SocketAddr)>, + controlling_checks: bool, + controlled_checks: bool, +} + +impl Peer { + fn connect(capacity: usize) -> Self { + Self::connect_with_profile(capacity, None) + } + + fn connect_with_profile(capacity: usize, profile: Option) -> Self { + Self::connect_with_extra_media(capacity, profile, None) + } + + fn connect_with_extra_media( + capacity: usize, + profile: Option, + extra: Option<(MediaKind, Direction)>, + ) -> Self { + Self::connect_with_ice_lite_peer(capacity, profile, extra, false) + } + + fn connect_with_ice_lite_peer( + capacity: usize, + profile: Option, + extra: Option<(MediaKind, Direction)>, + passive_ice_lite: bool, + ) -> Self { + Self::connect_with_codec( + capacity, + profile, + extra, + passive_ice_lite, + NegotiatedVideoCodec::H264, + ) + } + + fn connect_with_codec( + capacity: usize, + profile: Option, + extra: Option<(MediaKind, Direction)>, + passive_ice_lite: bool, + codec: NegotiatedVideoCodec, + ) -> Self { + install_crypto(); + let socket = bind_routed_socket("127.0.0.1".parse().unwrap()).unwrap(); + socket.set_nonblocking(true).unwrap(); + let address = socket.local_addr().unwrap(); + let mut config = RtcConfig::new().clear_codecs().enable_opus(true); + let pcmu_microphone = extra == Some((MediaKind::Audio, Direction::RecvOnly)); + if pcmu_microphone { + config.codec_config().enable_pcmu(true); + } + if codec != NegotiatedVideoCodec::H264 { + for (payload, profile, tier) in [(100_u8, 1, 0), (102, 2, 0), (104, 2, 1)] { + config.codec_config().add_h265( + payload.into(), + Some((payload + 1).into()), + profile, + tier, + 153, + ); + } + config = config.enable_h264(true); + } else if let Some(profile) = profile { + config + .codec_config() + .add_h264(96.into(), Some(97.into()), true, profile); + } else { + config = config.enable_h264(true); + } + let mut rtc = config.set_ice_lite(passive_ice_lite).build(Instant::now()); + if passive_ice_lite { + rtc.direct_api().start_dtls(false).unwrap(); + } + rtc.add_local_candidate(Candidate::host(address, "udp").unwrap()); + let mut changes = rtc.sdp_api(); + let extra_mid = + extra.map(|(kind, direction)| changes.add_media(kind, direction, None, None, None)); + let video = changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + let audio = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + changes.add_channel("server-bootstrap".to_owned()); + let (offer, pending) = changes.apply().unwrap(); + let offer_sdp = if pcmu_microphone { + let mut offer = (*offer).clone(); + let microphone = &mut offer.media_lines[0]; + microphone.pts = vec![0.into()]; + microphone.attrs.retain(|attribute| { + let line = attribute.to_string(); + !line.starts_with("a=rtpmap:") + && !line.starts_with("a=fmtp:") + && !line.starts_with("a=rtcp-fb:") + }); + offer.to_string() + } else { + offer.to_sdp_string() + }; + let session: Session = serde_json::from_value(serde_json::json!({ + "sessionId": "local-peer", + "serverIp": "127.0.0.1", + "mediaConnectionInfo": {"ip":"127.0.0.1", "port":address.port(), "usage":15}, + "iceServers": [{"urls":["turn:unused.invalid"],"username":"test","credential":"test"}] + })) + .unwrap(); + let (events, event_rx) = mpsc::sync_channel(8); + let (media, media_rx) = mpsc::sync_channel(capacity); + let negotiated = + negotiate(&offer_sdp, &session, codec, 300, events.clone(), media).unwrap(); + let video_codec = if codec == NegotiatedVideoCodec::H264 { + "H264" + } else { + "H265" + }; + assert!( + negotiated + .answer_sdp + .contains(&format!("{video_codec}/90000")) + ); + assert_eq!(negotiated.video_mid, video); + assert!(negotiated.answer_sdp.contains("opus/48000/2")); + assert!(!negotiated.answer_sdp.contains("VP8/90000")); + if passive_ice_lite { + assert!(negotiated.answer_sdp.contains("a=setup:active")); + } + rtc.sdp_api() + .accept_answer( + pending, + SdpAnswer::from_sdp_string(&negotiated.answer_sdp).unwrap(), + ) + .unwrap(); + if let Some(mid) = extra_mid { + assert_eq!(rtc.media(mid).unwrap().direction(), Direction::Inactive); + if pcmu_microphone { + assert!(rtc.media(mid).unwrap().disabled()); + } + assert_eq!( + negotiated + .answer_sdp + .lines() + .filter(|line| line.starts_with("m=")) + .count(), + 4 + ); + } + let mut peer = Self { + rtc, + socket, + video, + audio, + transport: Some(negotiated.session), + events: event_rx, + event_sender: events, + drain_events: true, + media: media_rx, + connected: false, + input_ready: false, + received: Vec::new(), + stats: None, + keyframe_requests: 0, + input_channel: None, + client_address: parse_candidate(&negotiated.local_candidate.candidate, None) + .unwrap() + .unwrap() + .addr(), + answer_sdp: negotiated.answer_sdp, + rtp_transmits: 0, + drop_rtp_number: None, + dropped_rtp: 0, + drop_payload_types: Vec::new(), + hold_rtp_number: None, + held_rtp: None, + controlling_checks: false, + controlled_checks: false, + }; + peer.until(|peer| peer.connected && peer.input_ready); + peer + } + + fn control(&self) -> TransportControl { + self.transport.as_ref().unwrap().control() + } + + fn tick(&mut self) { + self.rtc + .handle_input(Input::Timeout(Instant::now())) + .unwrap(); + loop { + match self.rtc.poll_output().unwrap() { + Output::Timeout(_) => break, + Output::Transmit(packet) => { + if packet.contents.len() >= 12 + && packet.contents[0] & 0xc0 == 0x80 + && !(192..=223).contains(&packet.contents[1]) + { + if self + .drop_payload_types + .contains(&(packet.contents[1] & 0x7f)) + { + self.dropped_rtp += 1; + continue; + } + self.rtp_transmits += 1; + if self.hold_rtp_number == Some(self.rtp_transmits) { + self.hold_rtp_number = None; + self.held_rtp = Some((packet.contents.to_vec(), packet.destination)); + continue; + } + if self.drop_rtp_number == Some(self.rtp_transmits) { + self.drop_rtp_number = None; + self.dropped_rtp += 1; + continue; + } + } + match self.socket.send_to(&packet.contents, packet.destination) { + Ok(_) => {} + Err(error) if transient_udp_error(&error) => {} + Err(error) => panic!("peer send: {error}"), + } + } + Output::Event(Event::ChannelOpen(id, label)) if label == "input_channel_v1" => { + self.input_channel = Some(id); + assert!( + self.rtc + .channel(id) + .unwrap() + .write(true, &[0x0e, 0x02, 3, 0]) + .unwrap() + ); + } + Output::Event(Event::ChannelData(data)) => self.received.push(data.data), + Output::Event(Event::KeyframeRequest(_)) => self.keyframe_requests += 1, + _ => {} + } + } + let mut buffer = [0; 65_536]; + loop { + match self.socket.recv_from(&mut buffer) { + Ok((length, source)) => { + if buffer.get(..2) == Some(&[0, 1]) + && buffer.get(4..8) == Some(&[0x21, 0x12, 0xa4, 0x42]) + { + self.controlling_checks |= + crate::nvst::find_stun_attribute(&buffer[..length], 0x802a).is_some(); + self.controlled_checks |= + crate::nvst::find_stun_attribute(&buffer[..length], 0x8029).is_some(); + } + let contents = (&buffer[..length]).try_into().unwrap(); + self.rtc + .handle_input(Input::Receive( + Instant::now(), + Receive { + proto: Protocol::Udp, + source, + destination: self.socket.local_addr().unwrap(), + contents, + }, + )) + .unwrap(); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => break, + Err(error) if transient_udp_error(&error) => continue, + Err(error) => panic!("peer receive: {error}"), + } + } + if self.drain_events { + for event in self.events.try_iter() { + match event { + TransportEvent::Connected => self.connected = true, + TransportEvent::Stats(stats) => self.stats = Some(stats), + TransportEvent::InputReady(version) => { + assert_eq!(version, 3); + self.input_ready = true; + } + TransportEvent::Disconnected(reason) => { + panic!("unexpected disconnect: {reason}") + } + _ => {} + } + } + } + thread::sleep(Duration::from_millis(1)); + } + + fn until(&mut self, condition: impl Fn(&Self) -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !condition(self) { + assert!(Instant::now() < deadline, "local peer timed out"); + self.tick(); + } + } + + fn release_held_rtp(&mut self) { + let (bytes, destination) = self.held_rtp.take().unwrap(); + self.socket.send_to(&bytes, destination).unwrap(); + } + + fn retransmission_payload_types(&mut self) -> Vec { + self.rtc + .writer(self.video) + .unwrap() + .payload_params() + .filter_map(|params| params.resend()) + .map(|pt| *pt) + .collect() + } + + fn pump(&mut self) { + for _ in 0..30 { + self.tick(); + } + } + + fn write(&mut self, video: bool, timestamp: u64, data: &[u8]) -> u32 { + let mid = if video { self.video } else { self.audio }; + let pt = self + .rtc + .writer(mid) + .unwrap() + .payload_params() + .find(|params| { + if video { + matches!(params.spec().codec, Codec::H264 | Codec::H265) + } else { + params.spec().codec == Codec::Opus + } + }) + .unwrap() + .pt(); + let time = if video { + MediaTime::from_90khz(timestamp) + } else { + MediaTime::new(timestamp, Frequency::FORTY_EIGHT_KHZ) + }; + self.rtc + .writer(mid) + .unwrap() + .write(pt, Instant::now(), time, data.to_vec()) + .unwrap(); + *self + .rtc + .direct_api() + .stream_tx_by_mid(mid, None) + .unwrap() + .ssrc() + } + + fn frame(&mut self) -> EncodedMediaFrame { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Ok(frame) = self.media.try_recv() { + return frame; + } + assert!(Instant::now() < deadline, "no compressed media delivered"); + self.tick(); + } + } +} + +#[test] +fn refuses_unresolved_hostname_without_dns() { + assert!(matches!( + resolve_host("not-a-real-host.invalid"), + Err(TransportError::Endpoint(_)) + )); + assert_eq!( + resolve_host("192-0-2-10.example.com").unwrap(), + "192.0.2.10".parse::().unwrap() + ); +} + +#[test] +fn local_hevc_main10_peer_produces_transport_stats() { + let mut peer = Peer::connect_with_codec(4, None, None, false, NegotiatedVideoCodec::H265Main10); + let video = [0, 0, 0, 1, 0x26, 0x01, 0xaf]; + peer.write(true, 90_123, &video); + assert_eq!(peer.frame().codec, "H265"); + peer.until(|peer| { + peer.stats.is_some_and(|stats| { + stats.ping_ms.is_some() + && stats.jitter_ms.is_some() + && stats.packet_loss_percent.is_some() + && stats.socket_receive_bytes.is_some_and(|bytes| bytes > 0) + }) + }); + let stats = peer.stats.unwrap(); + assert_eq!(stats.packet_loss_percent, Some(0.0)); + assert!(stats.ping_ms.unwrap() >= 0.0); + assert!(stats.jitter_ms.unwrap() >= 0.0); +} + +#[test] +fn local_peer_delivers_h264_opus_and_versioned_input() { + let mut peer = Peer::connect(8); + let video = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + let video_ssrc = peer.write(true, 900_123, &video); + let frame = peer.frame(); + assert_eq!(frame.codec, "H264"); + assert_eq!(frame.payload.as_ref(), video); + assert_eq!(frame.frame_index, None); + assert_eq!(frame.rtp_timestamp, 900_123); + assert_eq!(frame.clock_rate_hz, 90_000); + assert_eq!(frame.ssrc, Some(video_ssrc)); + assert!(frame.keyframe); + let opus = [0xf8, 0xff, 0xfe]; + let audio_ssrc = peer.write(false, 48_321, &opus); + let frame = peer.frame(); + assert_eq!(frame.codec, "Opus"); + assert_eq!(frame.payload.as_ref(), opus); + assert_eq!(frame.rtp_timestamp, 48_321); + assert_eq!(frame.clock_rate_hz, 48_000); + assert_eq!(frame.channels, Some(2)); + assert_eq!(frame.ssrc, Some(audio_ssrc)); + let mut key = vec![0; 18]; + key[..4].copy_from_slice(&3_u32.to_le_bytes()); + key[4..6].copy_from_slice(&65_u16.to_be_bytes()); + peer.control().send_input(key.clone(), false).unwrap(); + peer.until(|peer| { + peer.received + .iter() + .any(|packet| packet.first() == Some(&0x23)) + }); + let packet = peer + .received + .iter() + .find(|packet| packet.first() == Some(&0x23)) + .unwrap(); + assert_eq!(packet[9], 0x22); + assert_eq!(&packet[10..], key); + let send_timestamp = u64::from_be_bytes(packet[1..9].try_into().unwrap()); + assert!(send_timestamp < 5_000_000); + assert!( + !peer + .received + .iter() + .any(|packet| packet == &[0x0e, 0x02, 3, 0]) + ); + let started = Instant::now(); + peer.transport.take().unwrap().stop(); + assert!(started.elapsed() < Duration::from_secs(1)); +} + +#[test] +fn media_overload_drops_delta_frames_until_reference_and_requests_pli() { + let mut peer = Peer::connect(1); + let key = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + let delta = [0, 0, 0, 1, 0x41, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &key); + peer.pump(); + peer.write(true, 93_000, &delta); + peer.until(|peer| peer.keyframe_requests > 0); + assert_eq!(peer.frame().rtp_timestamp, 90_000); + peer.write(true, 96_000, &delta); + peer.pump(); + assert!(matches!( + peer.media.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + peer.write(true, 99_000, &key); + let recovered = peer.frame(); + assert_eq!(recovered.rtp_timestamp, 99_000); + assert!(recovered.keyframe); + assert!(!recovered.contiguous); + peer.write(true, 102_000, &delta); + assert_eq!(peer.frame().rtp_timestamp, 102_000); +} + +#[test] +fn local_peer_delivers_signed_relative_mouse_motion() { + let mut peer = Peer::connect(1); + let mut motion = vec![0; 22]; + motion[..4].copy_from_slice(&7_u32.to_le_bytes()); + motion[4..6].copy_from_slice(&(-12_i16).to_be_bytes()); + motion[6..8].copy_from_slice(&34_i16.to_be_bytes()); + motion[14..22].copy_from_slice(&1234_u64.to_be_bytes()); + peer.control().send_input(motion.clone(), false).unwrap(); + peer.until(|peer| { + peer.received + .iter() + .any(|packet| packet.first() == Some(&0x23) && packet.get(9) == Some(&0x21)) + }); + let packet = peer + .received + .iter() + .find(|packet| packet.first() == Some(&0x23) && packet.get(9) == Some(&0x21)) + .unwrap(); + assert_eq!(&packet[10..12], &22_u16.to_be_bytes()); + assert_eq!(&packet[12..], motion); +} + +#[test] +fn command_bounds_and_stop_do_not_depend_on_queue_space() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = TransportControl { + commands, + cancelled: Arc::new(AtomicBool::new(false)), + input_ready: Arc::new(AtomicBool::new(true)), + }; + control.request_keyframe("video").unwrap(); + assert!(matches!( + control.request_keyframe("video"), + Err(TransportError::Backpressured) + )); + assert!(matches!( + control.send_input(vec![0; MAX_INPUT_BYTES + 1], false), + Err(TransportError::OversizedCommand) + )); + control.stop(); + assert!(control.cancelled.load(Ordering::Acquire)); + assert!(!control.input_ready.load(Ordering::Acquire)); + assert!(matches!( + control.request_keyframe("video"), + Err(TransportError::Closed) + )); + assert!(receiver.try_recv().is_ok()); +} + +#[test] +fn candidate_normalization_preserves_relay_addresses_and_uses_usage_15() { + let endpoint: SocketAddr = "192.0.2.42:4444".parse().unwrap(); + let host = parse_candidate( + "candidate:1 1 udp 2130706431 127.0.0.1 9 typ host", + Some(endpoint), + ) + .unwrap() + .unwrap(); + assert!(host.to_sdp_string().contains("192.0.2.42 4444 typ host")); + let relay = parse_candidate( + "candidate:2 1 udp 16777215 192.0.2.50 5000 typ relay raddr 192.0.2.51 rport 5001", + Some(endpoint), + ) + .unwrap() + .unwrap(); + assert!(relay.to_sdp_string().contains("192.0.2.50 5000 typ relay")); + let session: Session = serde_json::from_value(serde_json::json!({ + "sessionId":"test", "serverIp":"unresolved.invalid", "mediaConnectionInfo": {"ip":"192.0.2.42","port":4444,"usage":15} + })).unwrap(); + assert_eq!(media_endpoint(&session).unwrap(), Some(endpoint)); +} + +#[test] +fn candidates_keep_public_endpoints_and_ignore_tcp_in_mixed_offers() { + let endpoint = "192.0.2.42:4444".parse().unwrap(); + let public = "candidate:1 1 udp 2130706431 203.0.113.8 5555 typ host"; + assert!( + parse_candidate(public, Some(endpoint)) + .unwrap() + .unwrap() + .to_sdp_string() + .contains("203.0.113.8 5555 typ host") + ); + let tcp = "candidate:2 1 tcp 2130706431 203.0.113.8 9 typ host tcptype active"; + assert!(parse_candidate(tcp, Some(endpoint)).unwrap().is_none()); + let offer = format!("a={tcp}\r\na={public}\r\n"); + let normalized = normalize_offer(&offer, Some(endpoint)).unwrap(); + assert!(!normalized.contains(" tcp ")); + assert!(normalized.contains("203.0.113.8 5555 typ host")); + assert!(matches!( + normalize_offer( + "a=candidate:2 1 udp 16777215 192.0.2.50 5000 typ relay\r\n", + None + ), + Err(TransportError::RelayUnsupported) + )); +} + +#[test] +fn local_candidate_targets_negotiated_mid_instead_of_assuming_zero() { + let sdp = "a=group:BUNDLE audio-main video-main\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=mid:video-main\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=mid:audio-main\r\n"; + assert_eq!(candidate_target(sdp).unwrap(), ("audio-main".to_owned(), 1)); + let rejected = "m=video 0 UDP/TLS/RTP/SAVPF 96\r\na=mid:rejected\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=mid:active\r\n"; + assert_eq!( + candidate_target(rejected).unwrap(), + ("active".to_owned(), 1) + ); +} + +#[test] +fn relay_only_policy_is_explicitly_unsupported() { + let session: Session = serde_json::from_value(serde_json::json!({ + "sessionId":"test", "serverIp":"127.0.0.1", "iceTransportPolicy":"relay" + })) + .unwrap(); + let (events, _) = mpsc::sync_channel(1); + let (media, _) = mpsc::sync_channel(1); + assert!(matches!( + negotiate("", &session, NegotiatedVideoCodec::H264, 300, events, media), + Err(TransportError::RelayUnsupported) + )); +} + +#[test] +fn full_event_queue_cannot_block_session_drop_or_accept_new_input() { + let mut peer = Peer::connect(1); + peer.drain_events = false; + for _ in 0..8 { + peer.event_sender + .try_send(TransportEvent::Log("full".to_owned())) + .unwrap(); + } + peer.rtc + .direct_api() + .close_data_channel(peer.input_channel.unwrap()); + peer.pump(); + let control = peer.control(); + assert!(!control.input_ready.load(Ordering::Acquire)); + let started = Instant::now(); + drop(peer.transport.take().unwrap()); + assert!(started.elapsed() < Duration::from_secs(1)); + assert!(matches!( + control.request_keyframe("video"), + Err(TransportError::Closed) + )); +} + +#[test] +fn terminal_event_survives_event_backpressure_when_consumer_resumes() { + let mut peer = Peer::connect(1); + peer.drain_events = false; + for _ in 0..8 { + peer.event_sender + .try_send(TransportEvent::Log("full".to_owned())) + .unwrap(); + } + peer.rtc + .direct_api() + .close_data_channel(peer.input_channel.unwrap()); + peer.pump(); + for _ in 0..8 { + assert!(matches!( + peer.events.try_recv().unwrap(), + TransportEvent::Log(_) + )); + } + let terminal = peer.events.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(matches!(terminal, TransportEvent::Disconnected(_))); +} + +#[test] +fn sender_rtp_timestamp_wraps_without_becoming_a_local_frame_counter() { + let mut peer = Peer::connect(4); + let key = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + let delta = [0, 0, 0, 1, 0x41, 0x88, 0x84, 0x21]; + let first = u64::from(u32::MAX) - 2999; + peer.write(true, first, &key); + assert_eq!(peer.frame().rtp_timestamp, first); + peer.write(true, first + 3000, &delta); + assert_eq!(peer.frame().rtp_timestamp, 0); + peer.write(true, first + 6000, &delta); + assert_eq!(peer.frame().rtp_timestamp, 3000); +} + +#[test] +fn audio_rtp_wrap_preserves_wire_timestamps_accepted_by_native_audio() { + let mut peer = Peer::connect(4); + let opus = [0xf8, 0xff, 0xfe]; + let first = u64::from(u32::MAX) - 959; + let source = peer.write(false, first, &opus); + let frame = peer.frame(); + assert_eq!(frame.rtp_timestamp, first); + assert_eq!(frame.ssrc, Some(source)); + for (sent, expected) in [(first + 960, 0_u32), (first + 1920, 960)] { + peer.write(false, sent, &opus); + let frame = peer.frame(); + assert_eq!(u32::try_from(frame.rtp_timestamp).unwrap(), expected); + assert_eq!(frame.clock_rate_hz, 48_000); + assert_eq!(frame.ssrc, Some(source)); + } +} + +#[test] +fn unsolicited_udp_cannot_disconnect_an_authenticated_media_peer() { + let mut peer = Peer::connect(4); + let outsider = UdpSocket::bind("127.0.0.1:0").unwrap(); + for packet in [ + &[0x80, 0x66, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1][..], + &[0xff, 1, 2][..], + ] { + outsider.send_to(packet, peer.client_address).unwrap(); + } + peer.pump(); + let key = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &key); + assert_eq!(peer.frame().payload.as_ref(), key); +} + +#[test] +fn flush_sends_queued_neutral_input_before_session_teardown() { + let mut peer = Peer::connect(1); + let control = peer.control(); + let mut down = vec![0; 18]; + down[..4].copy_from_slice(&3_u32.to_le_bytes()); + down[4..6].copy_from_slice(&65_u16.to_be_bytes()); + for _ in 0..32 { + control.send_input(down.clone(), false).unwrap(); + } + let mut key_up = down.clone(); + key_up[..4].copy_from_slice(&4_u32.to_le_bytes()); + let mut mouse_up = vec![0; 18]; + mouse_up[..4].copy_from_slice(&9_u32.to_le_bytes()); + mouse_up[4] = 1; + let mut gamepad = vec![0; 38]; + gamepad[..4].copy_from_slice(&12_u32.to_le_bytes()); + for body in [&key_up, &mouse_up, &gamepad] { + control.send_input(body.clone(), false).unwrap(); + } + control.flush_input(Duration::from_millis(150)).unwrap(); + peer.drain_events = false; + peer.transport.take().unwrap().stop(); + peer.until(|peer| { + [&key_up, &mouse_up, &gamepad] + .iter() + .all(|body| peer.received.iter().any(|packet| packet.ends_with(body))) + }); +} + +#[test] +fn flush_is_bounded_when_command_consumer_is_stalled_or_cancelled() { + let (commands, _receiver) = mpsc::sync_channel(1); + let control = TransportControl { + commands, + cancelled: Arc::new(AtomicBool::new(false)), + input_ready: Arc::new(AtomicBool::new(true)), + }; + control.request_keyframe("video").unwrap(); + let start = Instant::now(); + assert!(matches!( + control.flush_input(Duration::from_secs(30)), + Err(TransportError::FlushTimeout) + )); + assert!(start.elapsed() < Duration::from_millis(500)); + control.stop(); + let start = Instant::now(); + assert!(matches!( + control.flush_input(Duration::from_secs(30)), + Err(TransportError::Closed) + )); + assert!(start.elapsed() < Duration::from_millis(100)); +} + +#[test] +fn full_event_consumer_cannot_block_input_flush() { + let mut peer = Peer::connect(1); + peer.drain_events = false; + for _ in 0..8 { + peer.event_sender + .try_send(TransportEvent::Log("full".to_owned())) + .unwrap(); + } + let start = Instant::now(); + peer.control() + .flush_input(Duration::from_millis(150)) + .unwrap(); + assert!(start.elapsed() < Duration::from_millis(500)); +} + +#[test] +fn cancellation_interrupts_a_flush_waiting_for_its_local_acknowledgement() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = TransportControl { + commands, + cancelled: Arc::new(AtomicBool::new(false)), + input_ready: Arc::new(AtomicBool::new(true)), + }; + let waiting = control.clone(); + let worker = thread::spawn(move || waiting.flush_input(Duration::from_secs(30))); + let barrier = receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(matches!(barrier, TransportCommand::FlushInput(_))); + let start = Instant::now(); + control.stop(); + assert!(matches!( + worker.join().unwrap(), + Err(TransportError::Closed) + )); + assert!(start.elapsed() < Duration::from_millis(100)); + drop(barrier); +} + +#[test] +fn end_of_candidates_is_a_nonfatal_noop() { + for value in ["", " ", "end-of-candidates", "a=end-of-candidates\r\n"] { + assert!(parse_candidate(value, None).unwrap().is_none()); + } + let peer = Peer::connect(1); + peer.transport + .as_ref() + .unwrap() + .add_remote_candidate(&IceCandidate { + candidate: String::new(), + sdp_mid: None, + sdp_m_line_index: None, + username_fragment: None, + }) + .unwrap(); + peer.control() + .flush_input(Duration::from_millis(150)) + .unwrap(); +} + +#[test] +fn icmp_and_interruption_are_transient_but_device_errors_are_not() { + for kind in [ + ErrorKind::ConnectionReset, + ErrorKind::ConnectionRefused, + ErrorKind::Interrupted, + ] { + assert!(transient_udp_error(&std::io::Error::from(kind))); + } + assert!(!transient_udp_error(&std::io::Error::from( + ErrorKind::PermissionDenied + ))); +} + +#[cfg(windows)] +#[test] +fn windows_closed_udp_port_does_not_poison_later_receives() { + let closed = UdpSocket::bind("127.0.0.1:0").unwrap(); + let closed_address = closed.local_addr().unwrap(); + drop(closed); + let socket = bind_routed_socket("127.0.0.1".parse().unwrap()).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + socket.send_to(b"probe", closed_address).unwrap(); + thread::sleep(Duration::from_millis(50)); + let sender = UdpSocket::bind("127.0.0.1:0").unwrap(); + sender + .send_to(b"valid", socket.local_addr().unwrap()) + .unwrap(); + let mut bytes = [0; 32]; + let (length, _) = socket.recv_from(&mut bytes).unwrap(); + assert_eq!(&bytes[..length], b"valid"); +} + +#[test] +fn ice_lite_server_waiting_for_client_hello_connects_and_delivers_media() { + let mut peer = Peer::connect_with_ice_lite_peer(4, None, None, true); + assert!(peer.controlling_checks); + assert!(!peer.controlled_checks); + let video = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &video); + assert_eq!(peer.frame().payload.as_ref(), video); + peer.write(false, 48_000, &[0xf8, 0xff, 0xfe]); + assert_eq!(peer.frame().codec, "Opus"); + let mut key = vec![0; 18]; + key[..4].copy_from_slice(&3_u32.to_le_bytes()); + peer.control().send_input(key.clone(), false).unwrap(); + peer.until(|peer| { + peer.received + .iter() + .any(|packet| packet.get(10..) == Some(key.as_slice())) + }); +} + +#[test] +fn extra_microphone_section_does_not_block_media_or_input() { + let mut peer = + Peer::connect_with_extra_media(4, None, Some((MediaKind::Audio, Direction::RecvOnly))); + let video = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &video); + assert_eq!(peer.frame().payload.as_ref(), video); + peer.write(false, 48_000, &[0xf8, 0xff, 0xfe]); + assert_eq!(peer.frame().codec, "Opus"); + let mut key = vec![0; 18]; + key[..4].copy_from_slice(&3_u32.to_le_bytes()); + peer.control().send_input(key.clone(), false).unwrap(); + peer.until(|peer| { + peer.received + .iter() + .any(|packet| packet.get(10..) == Some(key.as_slice())) + }); +} + +#[test] +fn inactive_video_section_does_not_replace_the_receiving_video_mid() { + let mut peer = + Peer::connect_with_extra_media(4, None, Some((MediaKind::Video, Direction::Inactive))); + let video = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &video); + assert_eq!(peer.frame().payload.as_ref(), video); +} + +#[test] +fn extra_incoming_tracks_are_rejected_by_role_not_section_count() { + install_crypto(); + for kind in [MediaKind::Video, MediaKind::Audio] { + let mut remote = RtcConfig::new() + .clear_codecs() + .enable_h264(true) + .enable_opus(true) + .build(Instant::now()); + remote.add_local_candidate( + Candidate::host("127.0.0.1:49153".parse().unwrap(), "udp").unwrap(), + ); + let mut changes = remote.sdp_api(); + changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + changes.add_media(kind, Direction::SendOnly, None, None, None); + changes.add_channel("data".to_owned()); + let (offer, _) = changes.apply().unwrap(); + let session: Session = serde_json::from_value( + serde_json::json!({"sessionId":"fixture","serverIp":"127.0.0.1"}), + ) + .unwrap(); + let (events, _) = mpsc::sync_channel(8); + let (media, _) = mpsc::sync_channel(8); + let error = negotiate( + &offer.to_sdp_string(), + &session, + NegotiatedVideoCodec::H264, + 300, + events, + media, + ) + .err() + .unwrap(); + let expected = if kind == MediaKind::Video { + "multiple incoming video" + } else { + "multiple incoming audio" + }; + assert!(error.to_string().contains(expected), "{error}"); + } +} + +#[test] +fn bidirectional_playback_offer_is_answered_receive_only() { + install_crypto(); + for (offered_role, answer_role) in [ + ("actpass", "active"), + ("active", "passive"), + ("passive", "active"), + ] { + let mut remote = RtcConfig::new() + .clear_codecs() + .enable_h264(true) + .enable_opus(true) + .build(Instant::now()); + remote.add_local_candidate( + Candidate::host("127.0.0.1:49153".parse().unwrap(), "udp").unwrap(), + ); + let mut changes = remote.sdp_api(); + let video = changes.add_media(MediaKind::Video, Direction::SendRecv, None, None, None); + let audio = changes.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + changes.add_channel("data".to_owned()); + let (offer, pending) = changes.apply().unwrap(); + let session: Session = serde_json::from_value( + serde_json::json!({"sessionId":"fixture","serverIp":"127.0.0.1"}), + ) + .unwrap(); + let (events, _events) = mpsc::sync_channel(8); + let (media, _media) = mpsc::sync_channel(8); + let negotiated = negotiate( + &offer + .to_sdp_string() + .replace("a=setup:actpass", &format!("a=setup:{offered_role}")), + &session, + NegotiatedVideoCodec::H264, + 300, + events, + media, + ) + .unwrap(); + let roles: Vec<_> = negotiated + .answer_sdp + .lines() + .filter_map(|line| line.strip_prefix("a=setup:")) + .collect(); + assert!(!roles.is_empty()); + assert!(roles.iter().all(|role| *role == answer_role)); + remote + .sdp_api() + .accept_answer( + pending, + SdpAnswer::from_sdp_string(&negotiated.answer_sdp).unwrap(), + ) + .unwrap(); + assert_eq!( + remote.media(video).unwrap().direction(), + Direction::SendOnly + ); + assert_eq!( + remote.media(audio).unwrap().direction(), + Direction::SendOnly + ); + negotiated.session.stop(); + } +} + +#[test] +fn media_section_limit_remains_bounded() { + let section = "m=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=inactive\r\n"; + assert!(normalize_offer(§ion.repeat(MAX_MEDIA_SECTIONS), None).is_ok()); + assert!( + normalize_offer(§ion.repeat(MAX_MEDIA_SECTIONS + 1), None) + .unwrap_err() + .to_string() + .contains("too many SDP media sections") + ); +} + +#[test] +fn hevc_profiles_negotiate_exactly_and_deliver_fragmented_reference() { + for (codec, profile) in [ + (NegotiatedVideoCodec::H265Main, 1), + (NegotiatedVideoCodec::H265Main10, 2), + ] { + let mut peer = Peer::connect_with_codec(4, None, None, false, codec); + assert!(!peer.answer_sdp.contains("H264/90000")); + let answer = SdpAnswer::from_sdp_string(&peer.answer_sdp).unwrap(); + let video = answer + .media_lines + .iter() + .find(|line| line.mid() == peer.video) + .unwrap(); + let params = video.rtp_params(); + assert!(!params.is_empty()); + for params in params { + assert!(codec.matches(¶ms)); + assert_eq!( + params + .spec() + .format + .h265_profile_tier_level + .unwrap() + .profile() + .to_id(), + profile + ); + } + let mut access_unit = vec![0, 0, 0, 1, 0x26, 0x01, 0xaf]; + access_unit.extend(std::iter::repeat_n(0x55, 6000)); + let ssrc = peer.write(true, 90_123, &access_unit); + let frame = peer.frame(); + assert_eq!(frame.codec, "H265"); + assert!(frame.keyframe); + assert_eq!(frame.payload.as_ref(), access_unit); + assert_eq!(frame.ssrc, Some(ssrc)); + assert_eq!(frame.rtp_timestamp, 90_123); + assert_eq!(frame.clock_rate_hz, 90_000); + assert_eq!(frame.frame_index, None); + assert!(peer.rtp_transmits > 2); + let delta = [0, 0, 0, 1, 0x02, 0x01, 0x55]; + peer.write(true, 93_123, &delta); + let frame = peer.frame(); + assert!(!frame.keyframe); + assert!(frame.contiguous); + assert_eq!(frame.payload.as_ref(), delta); + } +} + +#[test] +fn incompatible_video_codec_or_hevc_profile_is_rejected_before_media() { + install_crypto(); + for (offered, requested) in [ + (NegotiatedVideoCodec::H264, NegotiatedVideoCodec::H265Main10), + ( + NegotiatedVideoCodec::H265Main, + NegotiatedVideoCodec::H265Main10, + ), + ( + NegotiatedVideoCodec::H265Main10, + NegotiatedVideoCodec::H265Main, + ), + (NegotiatedVideoCodec::H265Main10, NegotiatedVideoCodec::H264), + ] { + let mut config = RtcConfig::new().clear_codecs().enable_opus(true); + match offered { + NegotiatedVideoCodec::H264 => config = config.enable_h264(true), + NegotiatedVideoCodec::H265Main | NegotiatedVideoCodec::H265Main10 => { + config.codec_config().add_h265( + 96.into(), + Some(97.into()), + if offered == NegotiatedVideoCodec::H265Main10 { + 2 + } else { + 1 + }, + 0, + 153, + ); + } + } + let mut remote = config.build(Instant::now()); + remote.add_local_candidate( + Candidate::host("127.0.0.1:49153".parse().unwrap(), "udp").unwrap(), + ); + let mut changes = remote.sdp_api(); + changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + changes.add_channel("data".to_owned()); + let (offer, _) = changes.apply().unwrap(); + let session: Session = serde_json::from_value( + serde_json::json!({"sessionId":"fixture","serverIp":"127.0.0.1"}), + ) + .unwrap(); + let (events, event_rx) = mpsc::sync_channel(8); + let (media, media_rx) = mpsc::sync_channel(8); + let error = negotiate( + &offer.to_sdp_string(), + &session, + requested, + 300, + events, + media, + ) + .err() + .unwrap(); + assert!(matches!(error, TransportError::Offer(_))); + assert!( + error + .to_string() + .contains(&format!("did not negotiate {requested:?}")) + ); + assert!(event_rx.try_recv().is_err()); + assert!(media_rx.try_recv().is_err()); + } +} + +#[test] +fn high_profile_h264_offer_negotiates_and_delivers_fragmented_reference() { + let mut peer = Peer::connect_with_profile(4, Some(0x640032)); + assert!( + peer.answer_sdp.contains("profile-level-id=64002a"), + "answered fmtp: {:?}", + peer.answer_sdp + .lines() + .filter(|line| line.starts_with("a=fmtp:")) + .collect::>() + ); + assert!( + peer.answer_sdp + .lines() + .any(|line| line.starts_with("a=fmtp:") + && line.contains("profile-level-id=64002a") + && line.contains("packetization-mode=1")) + ); + let mut access_unit = vec![ + 0, 0, 0, 1, 0x67, 0x64, 0, 0x2a, 0xac, 0xd9, 0x40, 0, 0, 0, 1, 0x68, 0xeb, 0xec, 0xb2, + 0x2c, 0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21, + ]; + access_unit.extend(std::iter::repeat_n(0x55, 6000)); + peer.write(true, 90_000, &access_unit); + let frame = peer.frame(); + assert!(frame.keyframe); + assert_eq!(frame.payload.as_ref(), access_unit); + assert!(peer.rtp_transmits > 2); +} + +#[test] +fn lost_idr_fragment_is_retransmitted_before_complete_reference_delivery() { + let mut peer = Peer::connect(4); + let mut access_unit = vec![ + 0, 0, 0, 1, 0x67, 0x42, 0, 0x1f, 0xac, 0xd9, 0x40, 0, 0, 0, 1, 0x68, 0xce, 0x06, 0xe2, 0, + 0, 0, 1, 0x65, 0x88, 0x84, 0x21, + ]; + access_unit.extend(std::iter::repeat_n(0x55, 6000)); + peer.drop_rtp_number = Some(3); + peer.write(true, 90_000, &access_unit); + let frame = peer.frame(); + assert_eq!(peer.dropped_rtp, 1); + assert!(frame.keyframe); + assert_eq!(frame.payload.as_ref(), access_unit); + assert!(peer.media.try_recv().is_err()); +} + +#[test] +fn historical_usage_14_media_endpoint_remains_authoritative() { + let session: Session = serde_json::from_value(serde_json::json!({ + "sessionId":"test", "serverIp":"unresolved.invalid", "mediaConnectionInfo": {"ip":"192.0.2.42","port":4444,"usage":14} + })).unwrap(); + assert_eq!( + media_endpoint(&session).unwrap(), + Some("192.0.2.42:4444".parse().unwrap()) + ); +} + +#[test] +fn unrecoverable_delta_loss_releases_the_next_keyframe_once_nack_can_no_longer_repair_it() { + let mut peer = Peer::connect(64); + peer.drop_payload_types = peer.retransmission_payload_types(); + assert!(!peer.drop_payload_types.is_empty()); + let key = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + let delta = [0, 0, 0, 1, 0x41, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &key); + assert_eq!(peer.frame().rtp_timestamp, 90_000); + peer.drop_rtp_number = Some(peer.rtp_transmits + 1); + let mut timestamp = 93_000; + for _ in 0..NACK_REPAIRABLE_FRAMES { + peer.write(true, timestamp, &delta); + peer.tick(); + timestamp += 1_500; + } + peer.pump(); + assert!( + peer.dropped_rtp >= 2, + "the loss must be NACKed and stay unrecovered" + ); + assert!(peer.media.try_recv().is_err()); + peer.write(true, timestamp, &key); + let recovered = peer.frame(); + assert_eq!(recovered.rtp_timestamp, timestamp); + assert!(recovered.keyframe); + assert!(!recovered.contiguous); + peer.pump(); + assert!(peer.media.try_recv().is_err()); + assert!(peer.keyframe_requests > 0); +} + +#[test] +fn repairable_delta_loss_waits_for_retransmission_and_stays_contiguous() { + let mut peer = Peer::connect(64); + peer.drop_payload_types = peer.retransmission_payload_types(); + let key = [0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]; + peer.write(true, 90_000, &key); + assert_eq!(peer.frame().rtp_timestamp, 90_000); + peer.hold_rtp_number = Some(peer.rtp_transmits + 1); + for index in 1..=20_u8 { + peer.write( + true, + 90_000 + u64::from(index) * 1_500, + &[0, 0, 0, 1, 0x41, 0x88, 0x84, index], + ); + peer.tick(); + } + peer.pump(); + assert!(peer.held_rtp.is_some()); + assert!(peer.media.try_recv().is_err()); + peer.release_held_rtp(); + for index in 1..=20_u8 { + let frame = peer.frame(); + assert_eq!(frame.rtp_timestamp, 90_000 + u64::from(index) * 1_500); + assert_eq!(frame.payload.last(), Some(&index)); + assert!(frame.contiguous); + } + peer.pump(); + assert!(peer.media.try_recv().is_err()); + assert_eq!(peer.keyframe_requests, 0); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.github/workflows/ci.yml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.github/workflows/ci.yml new file mode 100644 index 000000000..76d11ae6e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: [push, pull_request] + +jobs: + test: + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + + strategy: + fail-fast: false + matrix: + name: + - stable + - beta + - nightly + - macOS + - Windows + + include: + - name: beta + toolchain: beta + - name: nightly + toolchain: nightly + - name: macOS + os: macOS-latest + - name: Windows + os: windows-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Install toolchain + id: tc + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain || 'stable' }} + profile: minimal + override: true + + - name: Install dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libopus-dev + + - name: Setup cache + if: runner.os != 'macOS' + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ matrix.os }}-test-${{ steps.tc.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.toml') }} + + - name: Build static + run: cargo build --features "static" + + - name: Build dynamic + run: cargo build --features "dynamic" + + # TODO: Fix for CI environment. + #- name: Generate bindings + # run: cargo build --features "generate_binding" + + - name: Test all features + # TODO: Once "generate_binding" is fixed, replace with `--all-features` + # again. + run: cargo test --features "static dynamic" diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitignore b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitignore new file mode 100644 index 000000000..693699042 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitmodules b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitmodules new file mode 100644 index 000000000..ba5e9dacc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/.gitmodules @@ -0,0 +1,3 @@ +[submodule "opus"] + path = opus + url = https://github.com/xiph/opus/ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CHANGELOG.md b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CHANGELOG.md new file mode 100644 index 000000000..b32c749ab --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CHANGELOG.md @@ -0,0 +1,60 @@ +# Change Log + +An overview of changes: + +## [0.2.0] + +* Now requires `cmake`. +* Windows will build via `cmake` too. +* Windows pre-built binaries have been removed. +* Updated `bindgen` to version `0.58`. + +## [0.1.8] + +This release adds build support for FreeBSD. + +## [0.1.7] + +Add missing `opus`-folder. + +## [0.1.6] + +This release removes the `bindgen`-dependency from the default features. +Additionally, the `bindgen`-feature has been added in order to generate a new binding. + +## [0.1.4 and 0.1.5] + +v0.1.4: +This release fixes a problem where `audiopus_sys` could not find the +Opus folder. + +v0.1.5: +Convert Unix-relevant files' EOLs from CRLF to LF inside the opus-folder. + +### **Fix** +* Bundle the Opus project again. +* Added missing `cfg` on `find_via_pkg_config`. + +## [0.1.3] + +Fixes build-issues related to `pkg-config`. + +## [0.1.2] + +This release adds the ability to bypass `pkg-config`. + +### **Added:** + +* Ignore `pkg-config` when `LIBOPUS_NO_PKG` or `OPUS_NO_PKG` is set. +* Print the dynamic/static build cause via `cargo:info`. +* Add missing repository-link in `Cargo.toml`. + +## [0.1.1] + +### **Added:** + +* Copy Opus' source to `OUT_DIR` before building to avoid modifying and generating files outside of `OUT_DIR`. + +### **Fixed:** +* Convert Unix-relevant files' EOLs from `CRLF` to `LF` inside the `opus`-folder. +* Resolve unused import warnings when building with Unix. diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CONTRIBUTING.md b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CONTRIBUTING.md new file mode 100644 index 000000000..e61f23f9d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/CONTRIBUTING.md @@ -0,0 +1,62 @@ +# Contributing + +Everyone is welcome to get involved, may it be a pull request, suggestion, bug +report, or a textual improvement! : ) + +The language applied in this repository is British English. + +## Contributions + +Contributions to `audiopus_sys` should be first discussed up via an issue and then +implemented via pull request. +Issues display development-plans or required brainstorming, feel free to ask, +suggest, and discuss! +The `master`-branch contains the latest release. + +## Comments & Documentation Style + +- Comments are placed the lines before the related code line, not on the same +line. + +- Write full sentences in British English. + +- `unsafe` must always be reasoned and their soundness must be proven via a +comment. + +- Use Rust intra-doc-links paths to refer Rust items in documentation: +`[name](crate::module::struct::method)`. + +- If code ends up difficult, try to simplify it, if unavoidable, explain code +with comments. Prefer explicit variable naming instead of abbreviations. + +## Commit Style + +Write full sentences in British English. + +Commits should describe the action being peformed. + +Example: +- *Fix deadlock for events.* +- *Correct grammar in `command`-example.* + +## Pull Request Checklist + +- Make sure to open an issue prior working on a problem or ask on existing +issue be assigned. + +- If a pull requests breaks the current API, use the `breaking-changes`-branch, +otherwise `stable-changes`. + +- Commits shall be as small as possible, compile, and pass all tests. + +- Make sure your code is formatted with `rustfmt` and free of lints, +run `cargo fmt` and `cargo clippy`. + +- If you fixed a bug, add a test for that bug. Unit tests belong inside the +same file's `mod` named `tests`, integrational tests belong inside the +`tests`-folder. + +- Last but not least, make sure your planned pull request merges cleanly, +if it does not, rebase your changes. + +If you have any questions left, please reach out via the issue system : ) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml new file mode 100644 index 000000000..2eb3d64d9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml @@ -0,0 +1,44 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +edition = "2018" +name = "audiopus_sys" +version = "0.2.2" +authors = ["Lakelezz "] +description = "FFI-Binding to Opus, dynamically or statically linked for Windows and UNIX." +documentation = "https://docs.rs/audiopus_sys" +readme = "README.md" +keywords = ["audio", "opus", "codec"] +categories = ["api-bindings", "compression", "encoding", "multimedia::audio", "multimedia::encoding"] +license = "ISC" +repository = "https://github.com/lakelezz/audiopus_sys.git" + +[dependencies] +[build-dependencies.bindgen] +version = "0.58" +optional = true + +[build-dependencies.cmake] +version = "0.1" + +[build-dependencies.log] +version = "0.4" + +[build-dependencies.pkg-config] +version = "0.3" + +[features] +default = [] +dynamic = [] +generate_binding = ["bindgen"] +static = [] diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml.orig b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml.orig new file mode 100644 index 000000000..227d110eb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/Cargo.toml.orig @@ -0,0 +1,30 @@ +[package] +name = "audiopus_sys" +version = "0.2.2" +license = "ISC" +repository = "https://github.com/lakelezz/audiopus_sys.git" +authors = ["Lakelezz "] +keywords = ["audio", "opus", "codec"] +categories = ["api-bindings", "compression", "encoding", + "multimedia::audio", "multimedia::encoding"] +description = "FFI-Binding to Opus, dynamically or statically linked for Windows and UNIX." +readme = "README.md" +documentation = "https://docs.rs/audiopus_sys" +edition = "2018" + +[dependencies] + +[build-dependencies] +log = "0.4" +pkg-config = "0.3" +cmake = "0.1" + +[build-dependencies.bindgen] +version = "0.58" +optional = true + +[features] +default = [] +dynamic = [] +static = [] +generate_binding = ["bindgen"] diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/LICENSE.md b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/LICENSE.md new file mode 100644 index 000000000..a861f02de --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/LICENSE.md @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2019, Lakelezz + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/README.md b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/README.md new file mode 100644 index 000000000..c60b256e8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/README.md @@ -0,0 +1,81 @@ +[![ci-badge][]][ci] [![docs-badge][]][docs] [![rust version badge]][rust version link] [![crates.io version]][crates.io link] + +# About + +`audiopus_sys` is an FFI-Rust-binding to [`Opus`] version 1.3. + +Orginally, this sys-crate was made to empower the [`serenity`]-crate to build audio features on Windows, Linux, and Mac. However, it's not limited to that. + +Everyone is welcome to contribute, +check out the [`CONTRIBUTING.md`](CONTRIBUTING.md) for further guidance. + +# Building + +## Requirements +If you want to build Opus, you will need `cmake`. + +If you have `pkg-config`, it will attempt to use that before building. + +You can also link a pre-installed Opus, see [**Pre-installed Opus**](#Pre-installed-Opus) +below. + +This crate provides a pre-built binding. In case you want to generate the +binding yourself, you will need [`Clang`](https://rust-lang.github.io/rust-bindgen/requirements.html#clang), +see [**Pre-installed Opus**](#Generating-The-Binding) below for further +instructions. + +## Linking +`audiopus_sys` links to Opus 1.3 and supports Windows, Linux, and MacOS +By default, we statically link to Windows, MacOS, and if you use the +`musl`-environment. We will link dynamically for Linux except when using +mentioned `musl`. + +This can be altered by compiling with the `static` or `dynamic` feature having +effects respective to their names. If both features are enabled, +we will pick your system's default. + +Environment variables named `LIBOPUS_STATIC` or `OPUS_STATIC` will take +precedence over features thus overriding the behaviour. The value of these +environment variables have no influence of the result: If one of them is set, +statically linking will be picked. + +## Pkg-Config +By default, `audiopus_sys` will use `pkg-config` on Unix or GNU. +Setting the environment variable `LIBOPUS_NO_PKG` or `OPUS_NO_PKG` will bypass +probing for Opus via `pkg-config`. + +## Pre-installed Opus +If you have Opus pre-installed, you can set `LIBOPUS_LIB_DIR` or +`OPUS_LIB_DIR` to the directory containing Opus. + +Be aware that using an Opus other than version 1.3 may not work. + +# Generating The Binding +If you want to generate the binding yourself, you can use the +`generate_binding`-feature. + +Be aware, `bindgen` requires Clang and its `LIBCLANG_PATH` +environment variable to be specified. + +# Installation +Add this to your `Cargo.toml`: + +```toml +[dependencies] +audiopus_sys = "0.2" +``` +[`serenity`]: https://crates.io/crates/serenity + +[`Opus`]: https://www.opus-codec.org/ + +[ci-badge]: https://img.shields.io/github/workflow/status/Lakelezz/audiopus_sys/CI?style=flat-square +[ci]: https://github.com/Lakelezz/audiopus_sys/actions + +[docs-badge]: https://img.shields.io/badge/docs-online-5023dd.svg?style=flat-square&colorB=32b6b7 +[docs]: https://docs.rs/audiopus_sys + +[rust version badge]: https://img.shields.io/badge/rust-1.51+-93450a.svg?style=flat-square&colorB=ff9a0d +[rust version link]: https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html + +[crates.io link]: https://crates.io/crates/audiopus_sys +[crates.io version]: https://img.shields.io/crates/v/audiopus_sys.svg?style=flat-square&colorB=b73732 diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/build.rs b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/build.rs new file mode 100644 index 000000000..1035e145c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/build.rs @@ -0,0 +1,167 @@ +#![deny(rust_2018_idioms)] + +#[cfg(feature = "generate_binding")] +use std::path::PathBuf; +use std::{env, fmt::Display, path::Path}; + +/// Outputs the library-file's prefix as word usable for actual arguments on +/// commands or paths. +const fn rustc_linking_word(is_static_link: bool) -> &'static str { + if is_static_link { + "static" + } else { + "dylib" + } +} + +/// Generates a new binding at `src/lib.rs` using `src/wrapper.h`. +#[cfg(feature = "generate_binding")] +fn generate_binding() { + const ALLOW_UNCONVENTIONALS: &'static str = "#![allow(non_upper_case_globals)]\n\ + #![allow(non_camel_case_types)]\n\ + #![allow(non_snake_case)]\n"; + + let bindings = bindgen::Builder::default() + .header("src/wrapper.h") + .raw_line(ALLOW_UNCONVENTIONALS) + .generate() + .expect("Unable to generate binding"); + + let binding_target_path = PathBuf::new().join("src").join("lib.rs"); + + bindings + .write_to_file(binding_target_path) + .expect("Could not write binding to the file at `src/lib.rs`"); + + println!("cargo:info=Successfully generated binding."); +} + +fn build_opus(is_static: bool) { + let opus_path = Path::new("opus"); + + println!( + "cargo:info=Opus source path used: {:?}.", + opus_path + .canonicalize() + .expect("Could not canonicalise to absolute path") + ); + + println!("cargo:info=Building Opus via CMake."); + // Rust's MSVC targets use the release CRT in debug and release profiles. + // A CMake Debug build defines _DEBUG and links Opus assertions against + // MSVCRTD, which makes every Rust test binary fail on _CrtDbgReportW. + // Opus is a bundled runtime dependency, so keep its C ABI/profile stable + // and optimized regardless of the surrounding Rust profile. + let mut config = cmake::Config::new(opus_path); + config.profile("Release"); + if cfg!(target_env = "msvc") { + config.define("CMAKE_MSVC_RUNTIME_LIBRARY", "MultiThreadedDLL"); + } + let opus_build_dir = config.build(); + link_opus(is_static, opus_build_dir.display()) +} + +fn link_opus(is_static: bool, opus_build_dir: impl Display) { + let is_static_text = rustc_linking_word(is_static); + + println!( + "cargo:info=Linking Opus as {} lib: {}", + is_static_text, opus_build_dir + ); + println!("cargo:rustc-link-lib={}=opus", is_static_text); + println!("cargo:rustc-link-search=native={}/lib", opus_build_dir); +} + +#[cfg(any(unix, target_env = "gnu"))] +fn find_via_pkg_config(is_static: bool) -> bool { + pkg_config::Config::new() + .statik(is_static) + .probe("opus") + .is_ok() +} + +/// Based on the OS or target environment we are building for, +/// this function will return an expected default library linking method. +/// +/// If we build for Windows, MacOS, or Linux with musl, we will link statically. +/// However, if you build for Linux without musl, we will link dynamically. +/// +/// **Info**: +/// This is a helper-function and may not be called if +/// if the `static`-feature is enabled, the environment variable +/// `LIBOPUS_STATIC` or `OPUS_STATIC` is set. +fn default_library_linking() -> bool { + #[cfg(any(windows, target_os = "macos", target_env = "musl"))] + { + true + } + #[cfg(any(target_os = "freebsd", all(unix, target_env = "gnu")))] + { + false + } +} + +fn find_installed_opus() -> Option { + if let Ok(lib_directory) = env::var("LIBOPUS_LIB_DIR") { + Some(lib_directory) + } else if let Ok(lib_directory) = env::var("OPUS_LIB_DIR") { + Some(lib_directory) + } else { + None + } +} + +fn is_static_build() -> bool { + if cfg!(feature = "static") && cfg!(feature = "dynamic") { + default_library_linking() + } else if cfg!(feature = "static") + || env::var("LIBOPUS_STATIC").is_ok() + || env::var("OPUS_STATIC").is_ok() + { + println!("cargo:info=Static feature or environment variable found."); + + true + } else if cfg!(feature = "dynamic") { + println!("cargo:info=Dynamic feature enabled."); + + false + } else { + println!("cargo:info=No feature or environment variable found, linking by default."); + + default_library_linking() + } +} + +fn main() { + #[cfg(feature = "generate_binding")] + generate_binding(); + + let is_static = is_static_build(); + + // The streamer is distributed as a self-contained executable. When the + // static feature is selected, always compile the vendored Opus sources so + // pkg-config cannot silently introduce a runtime dependency on libopus. + if is_static { + build_opus(true); + return; + } + + #[cfg(any(unix, target_env = "gnu"))] + { + if std::env::var("LIBOPUS_NO_PKG").is_ok() || std::env::var("OPUS_NO_PKG").is_ok() { + println!("cargo:info=Bypassed `pkg-config`."); + } else if find_via_pkg_config(is_static) { + println!("cargo:info=Found `Opus` via `pkg_config`."); + + return; + } else { + println!("cargo:info=`pkg_config` could not find `Opus`."); + } + } + + if let Some(installed_opus) = find_installed_opus() { + link_opus(is_static, installed_opus); + } else { + build_opus(is_static); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.appveyor.yml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.appveyor.yml new file mode 100644 index 000000000..a0f4a776e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.appveyor.yml @@ -0,0 +1,37 @@ +image: Visual Studio 2015 +configuration: +- Debug +- DebugDLL +- DebugDLL_fixed +- Release +- ReleaseDLL +- ReleaseDLL_fixed + +platform: +- Win32 +- x64 + +environment: + api_key: + secure: kR3Ac0NjGwFnTmXdFrR8d6VXjdk5F7L4F/BilC4nvaM= + +build: + project: win32\VS2015\opus.sln + parallel: true + verbosity: minimal + +after_build: +- cd %APPVEYOR_BUILD_FOLDER% +- 7z a opus.zip win32\VS2015\%PLATFORM%\%CONFIGURATION%\opus.??? include\*.h + +test_script: +- cd %APPVEYOR_BUILD_FOLDER%\win32\VS2015\%PLATFORM%\%CONFIGURATION% +- test_opus_api.exe +- test_opus_decode.exe +- test_opus_encode.exe + +artifacts: +- path: opus.zip + +on_success: +- ps: if ($env:api_key -and "$env:configuration/$env:platform" -eq "ReleaseDLL_fixed/x64") { Start-AppveyorBuild -ApiKey $env:api_key -ProjectSlug 'opus-tools' } diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.gitlab-ci.yml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.gitlab-ci.yml new file mode 100644 index 000000000..01b033fda --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.gitlab-ci.yml @@ -0,0 +1,61 @@ +include: + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + +default: + tags: + - docker + # Image from https://hub.docker.com/_/gcc/ based on Debian + image: gcc:9 + +whitespace: + stage: test + script: + - git diff-tree --check origin/master HEAD + +autoconf: + stage: build + before_script: + - apt-get update && + apt-get install -y zip doxygen + script: + - ./autogen.sh + - ./configure + - make -j4 + - make distcheck + cache: + paths: + - "src/*.o" + - "src/.libs/*.o" + - "silk/*.o" + - "silk/.libs/*.o" + - "celt/*.o" + - "celt/.libs/*.o" + +cmake: + stage: build + before_script: + - apt-get update && + apt-get install -y cmake ninja-build + script: + - mkdir build + - cmake -S . -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DOPUS_BUILD_TESTING=ON -DOPUS_BUILD_PROGRAMS=ON + - cmake --build build + - cd build && ctest --output-on-failure + +meson: + stage: build + before_script: + - apt-get update && + apt-get install -y python3-pip ninja-build doxygen + - export XDG_CACHE_HOME=$PWD/pip-cache + - pip3 install --user meson + script: + - export PATH=$PATH:$HOME/.local/bin + - mkdir builddir + - meson setup --werror -Dtests=enabled -Ddocs=enabled -Dbuildtype=release builddir + - meson compile -C builddir + - meson test -C builddir + #- meson dist --no-tests -C builddir + cache: + paths: + - 'pip-cache/*' diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.travis.yml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.travis.yml new file mode 100644 index 000000000..821c813ec --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/.travis.yml @@ -0,0 +1,21 @@ +language: c + +compiler: + - gcc + - clang + +os: + - linux + - osx + +env: + - CONFIG="" + - CONFIG="--enable-assertions" + - CONFIG="--enable-fixed-point" + - CONFIG="--enable-fixed-point --disable-float-api" + - CONFIG="--enable-fixed-point --enable-assertions" + +script: + - ./autogen.sh + - ./configure $CONFIG + - make distcheck diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/AUTHORS b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/AUTHORS new file mode 100644 index 000000000..b3d22a20c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/AUTHORS @@ -0,0 +1,6 @@ +Jean-Marc Valin (jmvalin@jmvalin.ca) +Koen Vos (koenvos74@gmail.com) +Timothy Terriberry (tterribe@xiph.org) +Karsten Vandborg Sorensen (karsten.vandborg.sorensen@skype.net) +Soren Skak Jensen (ssjensen@gn.com) +Gregory Maxwell (greg@xiph.org) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/CMakeLists.txt b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/CMakeLists.txt new file mode 100644 index 000000000..94a5880a3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/CMakeLists.txt @@ -0,0 +1,618 @@ +cmake_minimum_required(VERSION 3.5...4.0) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +include(OpusPackageVersion) +get_package_version(PACKAGE_VERSION PROJECT_VERSION) + +project(Opus LANGUAGES C VERSION ${PROJECT_VERSION}) + +include(OpusFunctions) +include(OpusBuildtype) +include(OpusConfig) +include(OpusSources) +include(GNUInstallDirs) +include(CMakeDependentOption) +include(FeatureSummary) + +set(OPUS_BUILD_SHARED_LIBRARY_HELP_STR "build shared library.") +option(OPUS_BUILD_SHARED_LIBRARY ${OPUS_BUILD_SHARED_LIBRARY_HELP_STR} OFF) +if(OPUS_BUILD_SHARED_LIBRARY OR BUILD_SHARED_LIBS OR OPUS_BUILD_FRAMEWORK) + # Global flag to cause add_library() to create shared libraries if on. + set(BUILD_SHARED_LIBS ON) + set(OPUS_BUILD_SHARED_LIBRARY ON) +endif() +add_feature_info(OPUS_BUILD_SHARED_LIBRARY OPUS_BUILD_SHARED_LIBRARY ${OPUS_BUILD_SHARED_LIBRARY_HELP_STR}) + +set(OPUS_BUILD_TESTING_HELP_STR "build tests.") +option(OPUS_BUILD_TESTING ${OPUS_BUILD_TESTING_HELP_STR} OFF) +if(OPUS_BUILD_TESTING OR BUILD_TESTING) + set(OPUS_BUILD_TESTING ON) + set(BUILD_TESTING ON) +endif() +add_feature_info(OPUS_BUILD_TESTING OPUS_BUILD_TESTING ${OPUS_BUILD_TESTING_HELP_STR}) + +set(OPUS_CUSTOM_MODES_HELP_STR "enable non-Opus modes, e.g. 44.1 kHz & 2^n frames.") +option(OPUS_CUSTOM_MODES ${OPUS_CUSTOM_MODES_HELP_STR} OFF) +add_feature_info(OPUS_CUSTOM_MODES OPUS_CUSTOM_MODES ${OPUS_CUSTOM_MODES_HELP_STR}) + +set(OPUS_BUILD_PROGRAMS_HELP_STR "build programs.") +option(OPUS_BUILD_PROGRAMS ${OPUS_BUILD_PROGRAMS_HELP_STR} OFF) +add_feature_info(OPUS_BUILD_PROGRAMS OPUS_BUILD_PROGRAMS ${OPUS_BUILD_PROGRAMS_HELP_STR}) + +set(OPUS_DISABLE_INTRINSICS_HELP_STR "disable all intrinsics optimizations.") +option(OPUS_DISABLE_INTRINSICS ${OPUS_DISABLE_INTRINSICS_HELP_STR} OFF) +add_feature_info(OPUS_DISABLE_INTRINSICS OPUS_DISABLE_INTRINSICS ${OPUS_DISABLE_INTRINSICS_HELP_STR}) + +set(OPUS_FIXED_POINT_HELP_STR "compile as fixed-point (for machines without a fast enough FPU).") +option(OPUS_FIXED_POINT ${OPUS_FIXED_POINT_HELP_STR} OFF) +add_feature_info(OPUS_FIXED_POINT OPUS_FIXED_POINT ${OPUS_FIXED_POINT_HELP_STR}) + +set(OPUS_ENABLE_FLOAT_API_HELP_STR "compile with the floating point API (for machines with float library).") +option(OPUS_ENABLE_FLOAT_API ${OPUS_ENABLE_FLOAT_API_HELP_STR} ON) +add_feature_info(OPUS_ENABLE_FLOAT_API OPUS_ENABLE_FLOAT_API ${OPUS_ENABLE_FLOAT_API_HELP_STR}) + +set(OPUS_FLOAT_APPROX_HELP_STR "enable floating point approximations (Ensure your platform supports IEEE 754 before enabling).") +option(OPUS_FLOAT_APPROX ${OPUS_FLOAT_APPROX_HELP_STR} OFF) +add_feature_info(OPUS_FLOAT_APPROX OPUS_FLOAT_APPROX ${OPUS_FLOAT_APPROX_HELP_STR}) + +set(OPUS_ASSERTIONS_HELP_STR "additional software error checking.") +option(OPUS_ASSERTIONS ${OPUS_ASSERTIONS_HELP_STR} OFF) +add_feature_info(OPUS_ASSERTIONS OPUS_ASSERTIONS ${OPUS_ASSERTIONS_HELP_STR}) + +set(OPUS_HARDENING_HELP_STR "run-time checks that are cheap and safe for use in production.") +option(OPUS_HARDENING ${OPUS_HARDENING_HELP_STR} ON) +add_feature_info(OPUS_HARDENING OPUS_HARDENING ${OPUS_HARDENING_HELP_STR}) + +set(OPUS_FUZZING_HELP_STR "causes the encoder to make random decisions (do not use in production).") +option(OPUS_FUZZING ${OPUS_FUZZING_HELP_STR} OFF) +add_feature_info(OPUS_FUZZING OPUS_FUZZING ${OPUS_FUZZING_HELP_STR}) + +set(OPUS_CHECK_ASM_HELP_STR "enable bit-exactness checks between optimized and c implementations.") +option(OPUS_CHECK_ASM ${OPUS_CHECK_ASM_HELP_STR} OFF) +add_feature_info(OPUS_CHECK_ASM OPUS_CHECK_ASM ${OPUS_CHECK_ASM_HELP_STR}) + +set(OPUS_INSTALL_PKG_CONFIG_MODULE_HELP_STR "install pkg-config module.") +option(OPUS_INSTALL_PKG_CONFIG_MODULE ${OPUS_INSTALL_PKG_CONFIG_MODULE_HELP_STR} ON) +add_feature_info(OPUS_INSTALL_PKG_CONFIG_MODULE OPUS_INSTALL_PKG_CONFIG_MODULE ${OPUS_INSTALL_PKG_CONFIG_MODULE_HELP_STR}) + +set(OPUS_INSTALL_CMAKE_CONFIG_MODULE_HELP_STR "install CMake package config module.") +option(OPUS_INSTALL_CMAKE_CONFIG_MODULE ${OPUS_INSTALL_CMAKE_CONFIG_MODULE_HELP_STR} ON) +add_feature_info(OPUS_INSTALL_CMAKE_CONFIG_MODULE OPUS_INSTALL_CMAKE_CONFIG_MODULE ${OPUS_INSTALL_CMAKE_CONFIG_MODULE_HELP_STR}) + +if(APPLE) + set(OPUS_BUILD_FRAMEWORK_HELP_STR "build Framework bundle for Apple systems.") + option(OPUS_BUILD_FRAMEWORK ${OPUS_BUILD_FRAMEWORK_HELP_STR} OFF) + add_feature_info(OPUS_BUILD_FRAMEWORK OPUS_BUILD_FRAMEWORK ${OPUS_BUILD_FRAMEWORK_HELP_STR}) +endif() + +set(OPUS_FIXED_POINT_DEBUG_HELP_STR "debug fixed-point implementation.") +cmake_dependent_option(OPUS_FIXED_POINT_DEBUG + ${OPUS_FIXED_POINT_DEBUG_HELP_STR} + ON + "OPUS_FIXED_POINT; OPUS_FIXED_POINT_DEBUG" + OFF) +add_feature_info(OPUS_FIXED_POINT_DEBUG OPUS_FIXED_POINT_DEBUG ${OPUS_FIXED_POINT_DEBUG_HELP_STR}) + +set(OPUS_VAR_ARRAYS_HELP_STR "use variable length arrays for stack arrays.") +cmake_dependent_option(OPUS_VAR_ARRAYS + ${OPUS_VAR_ARRAYS_HELP_STR} + ON + "VLA_SUPPORTED; NOT OPUS_USE_ALLOCA; NOT OPUS_NONTHREADSAFE_PSEUDOSTACK" + OFF) +add_feature_info(OPUS_VAR_ARRAYS OPUS_VAR_ARRAYS ${OPUS_VAR_ARRAYS_HELP_STR}) + +set(OPUS_USE_ALLOCA_HELP_STR "use alloca for stack arrays (on non-C99 compilers).") +cmake_dependent_option(OPUS_USE_ALLOCA + ${OPUS_USE_ALLOCA_HELP_STR} + ON + "USE_ALLOCA_SUPPORTED; NOT OPUS_VAR_ARRAYS; NOT OPUS_NONTHREADSAFE_PSEUDOSTACK" + OFF) +add_feature_info(OPUS_USE_ALLOCA OPUS_USE_ALLOCA ${OPUS_USE_ALLOCA_HELP_STR}) + +set(OPUS_NONTHREADSAFE_PSEUDOSTACK_HELP_STR "use a non threadsafe pseudostack when neither variable length arrays or alloca is supported.") +cmake_dependent_option(OPUS_NONTHREADSAFE_PSEUDOSTACK + ${OPUS_NONTHREADSAFE_PSEUDOSTACK_HELP_STR} + ON + "NOT OPUS_VAR_ARRAYS; NOT OPUS_USE_ALLOCA" + OFF) +add_feature_info(OPUS_NONTHREADSAFE_PSEUDOSTACK OPUS_NONTHREADSAFE_PSEUDOSTACK ${OPUS_NONTHREADSAFE_PSEUDOSTACK_HELP_STR}) + +set(OPUS_FAST_MATH_HELP_STR "enable fast math (unsupported and discouraged use, as code is not well tested with this build option).") +cmake_dependent_option(OPUS_FAST_MATH + ${OPUS_FAST_MATH_HELP_STR} + ON + "OPUS_FLOAT_APPROX; OPUS_FAST_MATH; FAST_MATH_SUPPORTED" + OFF) +add_feature_info(OPUS_FAST_MATH OPUS_FAST_MATH ${OPUS_FAST_MATH_HELP_STR}) + +set(OPUS_STACK_PROTECTOR_HELP_STR "use stack protection.") +cmake_dependent_option(OPUS_STACK_PROTECTOR + ${OPUS_STACK_PROTECTOR_HELP_STR} + ON + "STACK_PROTECTOR_SUPPORTED" + OFF) +add_feature_info(OPUS_STACK_PROTECTOR OPUS_STACK_PROTECTOR ${OPUS_STACK_PROTECTOR_HELP_STR}) + +if(NOT MSVC) + set(OPUS_FORTIFY_SOURCE_HELP_STR "add protection against buffer overflows.") + cmake_dependent_option(OPUS_FORTIFY_SOURCE + ${OPUS_FORTIFY_SOURCE_HELP_STR} + ON + "FORTIFY_SOURCE_SUPPORTED" + OFF) + add_feature_info(OPUS_FORTIFY_SOURCE OPUS_FORTIFY_SOURCE ${OPUS_FORTIFY_SOURCE_HELP_STR}) +endif() + +if(MINGW AND (OPUS_FORTIFY_SOURCE OR OPUS_STACK_PROTECTOR)) + # ssp lib is needed for security features for MINGW + list(APPEND OPUS_REQUIRED_LIBRARIES ssp) +endif() + +if(OPUS_CPU_X86 OR OPUS_CPU_X64) + set(OPUS_X86_MAY_HAVE_SSE_HELP_STR "does runtime check for SSE1 support.") + cmake_dependent_option(OPUS_X86_MAY_HAVE_SSE + ${OPUS_X86_MAY_HAVE_SSE_HELP_STR} + ON + "SSE1_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_MAY_HAVE_SSE OPUS_X86_MAY_HAVE_SSE ${OPUS_X86_MAY_HAVE_SSE_HELP_STR}) + + set(OPUS_X86_MAY_HAVE_SSE2_HELP_STR "does runtime check for SSE2 support.") + cmake_dependent_option(OPUS_X86_MAY_HAVE_SSE2 + ${OPUS_X86_MAY_HAVE_SSE2_HELP_STR} + ON + "SSE2_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_MAY_HAVE_SSE2 OPUS_X86_MAY_HAVE_SSE2 ${OPUS_X86_MAY_HAVE_SSE2_HELP_STR}) + + set(OPUS_X86_MAY_HAVE_SSE4_1_HELP_STR "does runtime check for SSE4.1 support.") + cmake_dependent_option(OPUS_X86_MAY_HAVE_SSE4_1 + ${OPUS_X86_MAY_HAVE_SSE4_1_HELP_STR} + ON + "SSE4_1_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_MAY_HAVE_SSE4_1 OPUS_X86_MAY_HAVE_SSE4_1 ${OPUS_X86_MAY_HAVE_SSE4_1_HELP_STR}) + + set(OPUS_X86_MAY_HAVE_AVX_HELP_STR "does runtime check for AVX support.") + cmake_dependent_option(OPUS_X86_MAY_HAVE_AVX + ${OPUS_X86_MAY_HAVE_AVX_HELP_STR} + ON + "AVX_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_MAY_HAVE_AVX OPUS_X86_MAY_HAVE_AVX ${OPUS_X86_MAY_HAVE_AVX_HELP_STR}) + + # PRESUME depends on MAY HAVE, but PRESUME will override runtime detection + set(OPUS_X86_PRESUME_SSE_HELP_STR "assume target CPU has SSE1 support (override runtime check).") + set(OPUS_X86_PRESUME_SSE2_HELP_STR "assume target CPU has SSE2 support (override runtime check).") + if(OPUS_CPU_X64) # Assume x86_64 has up to SSE2 support + cmake_dependent_option(OPUS_X86_PRESUME_SSE + ${OPUS_X86_PRESUME_SSE_HELP_STR} + ON + "OPUS_X86_MAY_HAVE_SSE; NOT OPUS_DISABLE_INTRINSICS" + OFF) + + cmake_dependent_option(OPUS_X86_PRESUME_SSE2 + ${OPUS_X86_PRESUME_SSE2_HELP_STR} + ON + "OPUS_X86_MAY_HAVE_SSE2; NOT OPUS_DISABLE_INTRINSICS" + OFF) + else() + cmake_dependent_option(OPUS_X86_PRESUME_SSE + ${OPUS_X86_PRESUME_SSE_HELP_STR} + OFF + "OPUS_X86_MAY_HAVE_SSE; NOT OPUS_DISABLE_INTRINSICS" + OFF) + + cmake_dependent_option(OPUS_X86_PRESUME_SSE2 + ${OPUS_X86_PRESUME_SSE2_HELP_STR} + OFF + "OPUS_X86_MAY_HAVE_SSE2; NOT OPUS_DISABLE_INTRINSICS" + OFF) + endif() + add_feature_info(OPUS_X86_PRESUME_SSE OPUS_X86_PRESUME_SSE ${OPUS_X86_PRESUME_SSE_HELP_STR}) + add_feature_info(OPUS_X86_PRESUME_SSE2 OPUS_X86_PRESUME_SSE2 ${OPUS_X86_PRESUME_SSE2_HELP_STR}) + + set(OPUS_X86_PRESUME_SSE4_1_HELP_STR "assume target CPU has SSE4.1 support (override runtime check).") + cmake_dependent_option(OPUS_X86_PRESUME_SSE4_1 + ${OPUS_X86_PRESUME_SSE4_1_HELP_STR} + OFF + "OPUS_X86_MAY_HAVE_SSE4_1; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_PRESUME_SSE4_1 OPUS_X86_PRESUME_SSE4_1 ${OPUS_X86_PRESUME_SSE4_1_HELP_STR}) + + set(OPUS_X86_PRESUME_AVX_HELP_STR "assume target CPU has AVX support (override runtime check).") + cmake_dependent_option(OPUS_X86_PRESUME_AVX + ${OPUS_X86_PRESUME_AVX_HELP_STR} + OFF + "OPUS_X86_MAY_HAVE_AVX; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_PRESUME_AVX OPUS_X86_PRESUME_AVX ${OPUS_X86_PRESUME_AVX_HELP_STR}) +endif() + +feature_summary(WHAT ALL) + +set_package_properties(Git + PROPERTIES + TYPE + REQUIRED + DESCRIPTION + "fast, scalable, distributed revision control system" + URL + "https://git-scm.com/" + PURPOSE + "required to set up package version") + +set(Opus_PUBLIC_HEADER + ${CMAKE_CURRENT_SOURCE_DIR}/include/opus.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_defines.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_multistream.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_projection.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_types.h) + +if(OPUS_CUSTOM_MODES) + list(APPEND Opus_PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_custom.h) +endif() + +add_library(opus ${opus_headers} ${opus_sources} ${opus_sources_float} ${Opus_PUBLIC_HEADER}) +add_library(Opus::opus ALIAS opus) + +get_library_version(OPUS_LIBRARY_VERSION OPUS_LIBRARY_VERSION_MAJOR) +message(DEBUG "Opus library version: ${OPUS_LIBRARY_VERSION}") + +set_target_properties(opus + PROPERTIES SOVERSION + ${OPUS_LIBRARY_VERSION_MAJOR} + VERSION + ${OPUS_LIBRARY_VERSION} + PUBLIC_HEADER + "${Opus_PUBLIC_HEADER}") + +target_include_directories( + opus + PUBLIC $ + $ + $ + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} + celt + silk) + +target_link_libraries(opus PRIVATE ${OPUS_REQUIRED_LIBRARIES}) +target_compile_definitions(opus PRIVATE OPUS_BUILD) + +if(OPUS_FIXED_POINT_DEBUG) + target_compile_definitions(opus PRIVATE FIXED_DEBUG) +endif() + +if(OPUS_FORTIFY_SOURCE AND NOT MSVC) + target_compile_definitions(opus PRIVATE + $<$>:_FORTIFY_SOURCE=2>) +endif() + +if(OPUS_FLOAT_APPROX) + target_compile_definitions(opus PRIVATE FLOAT_APPROX) +endif() + +if(OPUS_ASSERTIONS) + target_compile_definitions(opus PRIVATE ENABLE_ASSERTIONS) +endif() + +if(OPUS_HARDENING) + target_compile_definitions(opus PRIVATE ENABLE_HARDENING) +endif() + +if(OPUS_FUZZING) + target_compile_definitions(opus PRIVATE FUZZING) +endif() + +if(OPUS_CHECK_ASM) + target_compile_definitions(opus PRIVATE OPUS_CHECK_ASM) +endif() + +if(OPUS_VAR_ARRAYS) + target_compile_definitions(opus PRIVATE VAR_ARRAYS) +elseif(OPUS_USE_ALLOCA) + target_compile_definitions(opus PRIVATE USE_ALLOCA) +elseif(OPUS_NONTHREADSAFE_PSEUDOSTACK) + target_compile_definitions(opus PRIVATE NONTHREADSAFE_PSEUDOSTACK) +else() + message(ERROR "Need to set a define for stack allocation") +endif() + +if(OPUS_CUSTOM_MODES) + target_compile_definitions(opus PRIVATE CUSTOM_MODES) +endif() + +if(OPUS_FAST_MATH) + if(MSVC) + target_compile_options(opus PRIVATE /fp:fast) + else() + target_compile_options(opus PRIVATE -ffast-math) + endif() +endif() + +if(OPUS_STACK_PROTECTOR) + if(MSVC) + target_compile_options(opus PRIVATE /GS) + else() + target_compile_options(opus PRIVATE -fstack-protector-strong) + endif() +elseif(STACK_PROTECTOR_DISABLED_SUPPORTED) + target_compile_options(opus PRIVATE /GS-) +endif() + +if(BUILD_SHARED_LIBS) + if(WIN32) + target_compile_definitions(opus PRIVATE DLL_EXPORT) + elseif(HIDDEN_VISIBILITY_SUPPORTED) + set_target_properties(opus PROPERTIES C_VISIBILITY_PRESET hidden) + endif() +endif() + +add_sources_group(opus silk ${silk_headers} ${silk_sources}) +add_sources_group(opus celt ${celt_headers} ${celt_sources}) + +if(OPUS_FIXED_POINT) + add_sources_group(opus silk ${silk_sources_fixed}) + target_include_directories(opus PRIVATE silk/fixed) + target_compile_definitions(opus PRIVATE FIXED_POINT=1) +else() + add_sources_group(opus silk ${silk_sources_float}) + target_include_directories(opus PRIVATE silk/float) +endif() + +if(NOT OPUS_ENABLE_FLOAT_API) + target_compile_definitions(opus PRIVATE DISABLE_FLOAT_API) +endif() + +if(NOT OPUS_DISABLE_INTRINSICS) + if((OPUS_X86_MAY_HAVE_SSE AND NOT OPUS_X86_PRESUME_SSE) OR + (OPUS_X86_MAY_HAVE_SSE2 AND NOT OPUS_X86_PRESUME_SSE2) OR + (OPUS_X86_MAY_HAVE_SSE4_1 AND NOT OPUS_X86_PRESUME_SSE4_1) OR + (OPUS_X86_MAY_HAVE_AVX AND NOT OPUS_X86_PRESUME_AVX)) + target_compile_definitions(opus PRIVATE OPUS_HAVE_RTCD) + endif() + + if(SSE1_SUPPORTED) + if(OPUS_X86_MAY_HAVE_SSE) + add_sources_group(opus celt ${celt_sources_sse}) + target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_SSE) + if(NOT MSVC) + set_source_files_properties(${celt_sources_sse} PROPERTIES COMPILE_FLAGS -msse) + endif() + endif() + if(OPUS_X86_PRESUME_SSE) + target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_SSE) + if(NOT MSVC) + target_compile_options(opus PRIVATE -msse) + endif() + endif() + endif() + + if(SSE2_SUPPORTED) + if(OPUS_X86_MAY_HAVE_SSE2) + add_sources_group(opus celt ${celt_sources_sse2}) + target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_SSE2) + if(NOT MSVC) + set_source_files_properties(${celt_sources_sse2} PROPERTIES COMPILE_FLAGS -msse2) + endif() + endif() + if(OPUS_X86_PRESUME_SSE2) + target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_SSE2) + if(NOT MSVC) + target_compile_options(opus PRIVATE -msse2) + endif() + endif() + endif() + + if(SSE4_1_SUPPORTED) + if(OPUS_X86_MAY_HAVE_SSE4_1) + add_sources_group(opus celt ${celt_sources_sse4_1}) + add_sources_group(opus silk ${silk_sources_sse4_1}) + target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_SSE4_1) + if(NOT MSVC) + set_source_files_properties(${celt_sources_sse4_1} ${silk_sources_sse4_1} PROPERTIES COMPILE_FLAGS -msse4.1) + endif() + + if(OPUS_FIXED_POINT) + add_sources_group(opus silk ${silk_sources_fixed_sse4_1}) + if(NOT MSVC) + set_source_files_properties(${silk_sources_fixed_sse4_1} PROPERTIES COMPILE_FLAGS -msse4.1) + endif() + endif() + endif() + if(OPUS_X86_PRESUME_SSE4_1) + target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_SSE4_1) + if(NOT MSVC) + target_compile_options(opus PRIVATE -msse4.1) + endif() + endif() + endif() + + if(AVX_SUPPORTED) + # mostly placeholder in case of avx intrinsics is added + if(OPUS_X86_MAY_HAVE_AVX) + target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_AVX) + endif() + if(OPUS_X86_PRESUME_AVX) + target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_AVX) + if(NOT MSVC) + target_compile_options(opus PRIVATE -mavx) + endif() + endif() + endif() + + if(MSVC) + if(AVX_SUPPORTED AND OPUS_X86_PRESUME_AVX) # on 64 bit and 32 bits + add_definitions(/arch:AVX) + elseif(OPUS_CPU_X86) # if AVX not supported then set SSE flag + if((SSE4_1_SUPPORTED AND OPUS_X86_PRESUME_SSE4_1) + OR (SSE2_SUPPORTED AND OPUS_X86_PRESUME_SSE2)) + target_compile_definitions(opus PRIVATE /arch:SSE2) + elseif(SSE1_SUPPORTED AND OPUS_X86_PRESUME_SSE) + target_compile_definitions(opus PRIVATE /arch:SSE) + endif() + endif() + endif() + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm|aarch64)") + add_sources_group(opus celt ${celt_sources_arm}) + endif() + + if(COMPILER_SUPPORT_NEON) + if(OPUS_MAY_HAVE_NEON) + if(RUNTIME_CPU_CAPABILITY_DETECTION) + message(STATUS "OPUS_MAY_HAVE_NEON enabling runtime detection") + target_compile_definitions(opus PRIVATE OPUS_HAVE_RTCD) + else() + message(ERROR "Runtime cpu capability detection needed for MAY_HAVE_NEON") + endif() + # Do runtime check for NEON + target_compile_definitions(opus + PRIVATE + OPUS_ARM_MAY_HAVE_NEON + OPUS_ARM_MAY_HAVE_NEON_INTR) + endif() + + add_sources_group(opus celt ${celt_sources_arm_neon_intr}) + add_sources_group(opus silk ${silk_sources_arm_neon_intr}) + + # silk arm neon depends on main_Fix.h + target_include_directories(opus PRIVATE silk/fixed) + + if(OPUS_FIXED_POINT) + add_sources_group(opus silk ${silk_sources_fixed_arm_neon_intr}) + endif() + + if(OPUS_PRESUME_NEON) + target_compile_definitions(opus + PRIVATE + OPUS_ARM_PRESUME_NEON + OPUS_ARM_PRESUME_NEON_INTR) + endif() + endif() +endif() + +target_compile_definitions(opus + PRIVATE + $<$:HAVE_LRINT> + $<$:HAVE_LRINTF>) + +if(OPUS_BUILD_FRAMEWORK) + set_target_properties(opus PROPERTIES + FRAMEWORK TRUE + FRAMEWORK_VERSION ${PROJECT_VERSION} + MACOSX_FRAMEWORK_IDENTIFIER org.xiph.opus + MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${PROJECT_VERSION} + MACOSX_FRAMEWORK_BUNDLE_VERSION ${PROJECT_VERSION} + XCODE_ATTRIBUTE_INSTALL_PATH "@rpath" + OUTPUT_NAME Opus) +endif() + +install(TARGETS opus + EXPORT OpusTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + FRAMEWORK DESTINATION ${CMAKE_INSTALL_PREFIX} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/opus) + +if(OPUS_INSTALL_PKG_CONFIG_MODULE) + set(prefix ${CMAKE_INSTALL_PREFIX}) + set(exec_prefix ${CMAKE_INSTALL_PREFIX}) + set(libdir ${CMAKE_INSTALL_FULL_LIBDIR}) + set(includedir ${CMAKE_INSTALL_FULL_INCLUDEDIR}) + set(VERSION ${PACKAGE_VERSION}) + if(HAVE_LIBM) + set(LIBM "-lm") + endif() + configure_file(opus.pc.in opus.pc) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/opus.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +endif() + +if(OPUS_INSTALL_CMAKE_CONFIG_MODULE) + set(CPACK_GENERATOR TGZ) + include(CPack) + set(CMAKE_INSTALL_PACKAGEDIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) + install(EXPORT OpusTargets + NAMESPACE Opus:: + DESTINATION ${CMAKE_INSTALL_PACKAGEDIR}) + + include(CMakePackageConfigHelpers) + + set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}) + configure_package_config_file(${PROJECT_SOURCE_DIR}/cmake/OpusConfig.cmake.in + OpusConfig.cmake + INSTALL_DESTINATION + ${CMAKE_INSTALL_PACKAGEDIR} + PATH_VARS + INCLUDE_INSTALL_DIR + INSTALL_PREFIX + ${CMAKE_INSTALL_PREFIX}) + write_basic_package_version_file(OpusConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/OpusConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/OpusConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_PACKAGEDIR}) +endif() + +if(OPUS_BUILD_PROGRAMS) + # demo + if(OPUS_CUSTOM_MODES) + add_executable(opus_custom_demo ${opus_custom_demo_sources}) + target_include_directories(opus_custom_demo + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + target_link_libraries(opus_custom_demo PRIVATE opus) + endif() + + add_executable(opus_demo ${opus_demo_sources}) + target_include_directories(opus_demo PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + target_include_directories(opus_demo PRIVATE silk) # debug.h + target_include_directories(opus_demo PRIVATE celt) # arch.h + target_link_libraries(opus_demo PRIVATE opus ${OPUS_REQUIRED_LIBRARIES}) + + # compare + add_executable(opus_compare ${opus_compare_sources}) + target_include_directories(opus_compare PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + target_link_libraries(opus_compare PRIVATE opus ${OPUS_REQUIRED_LIBRARIES}) +endif() + +if(BUILD_TESTING) + enable_testing() + + # tests + add_executable(test_opus_decode ${test_opus_decode_sources}) + target_include_directories(test_opus_decode + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + target_link_libraries(test_opus_decode PRIVATE opus) + if(OPUS_FIXED_POINT) + target_compile_definitions(test_opus_decode PRIVATE DISABLE_FLOAT_API) + endif() + add_test(NAME test_opus_decode COMMAND $ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + add_executable(test_opus_padding ${test_opus_padding_sources}) + target_include_directories(test_opus_padding + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + target_link_libraries(test_opus_padding PRIVATE opus) + add_test(NAME test_opus_padding COMMAND $ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + if(NOT BUILD_SHARED_LIBS) + # disable tests that depends on private API when building shared lib + add_executable(test_opus_api ${test_opus_api_sources}) + target_include_directories(test_opus_api + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} celt) + target_link_libraries(test_opus_api PRIVATE opus) + if(OPUS_FIXED_POINT) + target_compile_definitions(test_opus_api PRIVATE DISABLE_FLOAT_API) + endif() + add_test(NAME test_opus_api COMMAND $ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + add_executable(test_opus_encode ${test_opus_encode_sources}) + target_include_directories(test_opus_encode + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} celt) + target_link_libraries(test_opus_encode PRIVATE opus) + add_test(NAME test_opus_encode COMMAND $ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + endif() +endif() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/COPYING b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/COPYING new file mode 100644 index 000000000..9c739c34a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/COPYING @@ -0,0 +1,44 @@ +Copyright 2001-2011 Xiph.Org, Skype Limited, Octasic, + Jean-Marc Valin, Timothy B. Terriberry, + CSIRO, Gregory Maxwell, Mark Borgerding, + Erik de Castro Lopo + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses which are +specified at: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/ChangeLog b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/ChangeLog new file mode 100644 index 000000000..e69de29bb diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/LICENSE_PLEASE_READ.txt b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/LICENSE_PLEASE_READ.txt new file mode 100644 index 000000000..bc88efa6c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/LICENSE_PLEASE_READ.txt @@ -0,0 +1,22 @@ +Contributions to the collaboration shall not be considered confidential. + +Each contributor represents and warrants that it has the right and +authority to license copyright in its contributions to the collaboration. + +Each contributor agrees to license the copyright in the contributions +under the Modified (2-clause or 3-clause) BSD License or the Clear BSD License. + +Please see the IPR statements submitted to the IETF for the complete +patent licensing details: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Skype Limited: +https://datatracker.ietf.org/ipr/1602/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.am b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.am new file mode 100644 index 000000000..83beaa3f1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.am @@ -0,0 +1,371 @@ +# Provide the full test output for failed tests when using the parallel +# test suite (which is enabled by default with automake 1.13+). +export VERBOSE = yes + +AUTOMAKE_OPTIONS = subdir-objects +ACLOCAL_AMFLAGS = -I m4 + +lib_LTLIBRARIES = libopus.la + +DIST_SUBDIRS = doc + +AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/celt -I$(top_srcdir)/silk \ + -I$(top_srcdir)/silk/float -I$(top_srcdir)/silk/fixed $(NE10_CFLAGS) + +include celt_sources.mk +include silk_sources.mk +include opus_sources.mk + +if FIXED_POINT +SILK_SOURCES += $(SILK_SOURCES_FIXED) +if HAVE_SSE4_1 +SILK_SOURCES += $(SILK_SOURCES_SSE4_1) $(SILK_SOURCES_FIXED_SSE4_1) +endif +if HAVE_ARM_NEON_INTR +SILK_SOURCES += $(SILK_SOURCES_FIXED_ARM_NEON_INTR) +endif +else +SILK_SOURCES += $(SILK_SOURCES_FLOAT) +if HAVE_SSE4_1 +SILK_SOURCES += $(SILK_SOURCES_SSE4_1) +endif +endif + +if DISABLE_FLOAT_API +else +OPUS_SOURCES += $(OPUS_SOURCES_FLOAT) +endif + +if HAVE_SSE +CELT_SOURCES += $(CELT_SOURCES_SSE) +endif +if HAVE_SSE2 +CELT_SOURCES += $(CELT_SOURCES_SSE2) +endif +if HAVE_SSE4_1 +CELT_SOURCES += $(CELT_SOURCES_SSE4_1) +endif + +if CPU_ARM +CELT_SOURCES += $(CELT_SOURCES_ARM) +SILK_SOURCES += $(SILK_SOURCES_ARM) + +if HAVE_ARM_NEON_INTR +CELT_SOURCES += $(CELT_SOURCES_ARM_NEON_INTR) +SILK_SOURCES += $(SILK_SOURCES_ARM_NEON_INTR) +endif + +if HAVE_ARM_NE10 +CELT_SOURCES += $(CELT_SOURCES_ARM_NE10) +endif + +if OPUS_ARM_EXTERNAL_ASM +noinst_LTLIBRARIES = libarmasm.la +libarmasm_la_SOURCES = $(CELT_SOURCES_ARM_ASM:.s=-gnu.S) +BUILT_SOURCES = $(CELT_SOURCES_ARM_ASM:.s=-gnu.S) \ + $(CELT_AM_SOURCES_ARM_ASM:.s.in=.s) \ + $(CELT_AM_SOURCES_ARM_ASM:.s.in=-gnu.S) +endif +endif + +CLEANFILES = $(CELT_SOURCES_ARM_ASM:.s=-gnu.S) \ + $(CELT_AM_SOURCES_ARM_ASM:.s.in=-gnu.S) + +include celt_headers.mk +include silk_headers.mk +include opus_headers.mk + +libopus_la_SOURCES = $(CELT_SOURCES) $(SILK_SOURCES) $(OPUS_SOURCES) +libopus_la_LDFLAGS = -no-undefined -version-info @OPUS_LT_CURRENT@:@OPUS_LT_REVISION@:@OPUS_LT_AGE@ +libopus_la_LIBADD = $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +libopus_la_LIBADD += libarmasm.la +endif + +pkginclude_HEADERS = include/opus.h include/opus_multistream.h include/opus_types.h include/opus_defines.h include/opus_projection.h + +noinst_HEADERS = $(OPUS_HEAD) $(SILK_HEAD) $(CELT_HEAD) + +if EXTRA_PROGRAMS +noinst_PROGRAMS = celt/tests/test_unit_cwrs32 \ + celt/tests/test_unit_dft \ + celt/tests/test_unit_entropy \ + celt/tests/test_unit_laplace \ + celt/tests/test_unit_mathops \ + celt/tests/test_unit_mdct \ + celt/tests/test_unit_rotation \ + celt/tests/test_unit_types \ + opus_compare \ + opus_demo \ + repacketizer_demo \ + silk/tests/test_unit_LPC_inv_pred_gain \ + tests/test_opus_api \ + tests/test_opus_decode \ + tests/test_opus_encode \ + tests/test_opus_padding \ + tests/test_opus_projection \ + trivial_example + +TESTS = celt/tests/test_unit_cwrs32 \ + celt/tests/test_unit_dft \ + celt/tests/test_unit_entropy \ + celt/tests/test_unit_laplace \ + celt/tests/test_unit_mathops \ + celt/tests/test_unit_mdct \ + celt/tests/test_unit_rotation \ + celt/tests/test_unit_types \ + silk/tests/test_unit_LPC_inv_pred_gain \ + tests/test_opus_api \ + tests/test_opus_decode \ + tests/test_opus_encode \ + tests/test_opus_padding \ + tests/test_opus_projection + +opus_demo_SOURCES = src/opus_demo.c + +opus_demo_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +repacketizer_demo_SOURCES = src/repacketizer_demo.c + +repacketizer_demo_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +opus_compare_SOURCES = src/opus_compare.c +opus_compare_LDADD = $(LIBM) + +trivial_example_SOURCES = doc/trivial_example.c +trivial_example_LDADD = libopus.la $(LIBM) + +tests_test_opus_api_SOURCES = tests/test_opus_api.c tests/test_opus_common.h +tests_test_opus_api_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +tests_test_opus_encode_SOURCES = tests/test_opus_encode.c tests/opus_encode_regressions.c tests/test_opus_common.h +tests_test_opus_encode_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +tests_test_opus_decode_SOURCES = tests/test_opus_decode.c tests/test_opus_common.h +tests_test_opus_decode_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +tests_test_opus_padding_SOURCES = tests/test_opus_padding.c tests/test_opus_common.h +tests_test_opus_padding_LDADD = libopus.la $(NE10_LIBS) $(LIBM) + +CELT_OBJ = $(CELT_SOURCES:.c=.lo) +SILK_OBJ = $(SILK_SOURCES:.c=.lo) +OPUS_OBJ = $(OPUS_SOURCES:.c=.lo) + +tests_test_opus_projection_SOURCES = tests/test_opus_projection.c tests/test_opus_common.h +tests_test_opus_projection_LDADD = $(OPUS_OBJ) $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +tests_test_opus_projection_LDADD += libarmasm.la +endif + +silk_tests_test_unit_LPC_inv_pred_gain_SOURCES = silk/tests/test_unit_LPC_inv_pred_gain.c +silk_tests_test_unit_LPC_inv_pred_gain_LDADD = $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +silk_tests_test_unit_LPC_inv_pred_gain_LDADD += libarmasm.la +endif + +celt_tests_test_unit_cwrs32_SOURCES = celt/tests/test_unit_cwrs32.c +celt_tests_test_unit_cwrs32_LDADD = $(LIBM) + +celt_tests_test_unit_dft_SOURCES = celt/tests/test_unit_dft.c +celt_tests_test_unit_dft_LDADD = $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +celt_tests_test_unit_dft_LDADD += libarmasm.la +endif + +celt_tests_test_unit_entropy_SOURCES = celt/tests/test_unit_entropy.c +celt_tests_test_unit_entropy_LDADD = $(LIBM) + +celt_tests_test_unit_laplace_SOURCES = celt/tests/test_unit_laplace.c +celt_tests_test_unit_laplace_LDADD = $(LIBM) + +celt_tests_test_unit_mathops_SOURCES = celt/tests/test_unit_mathops.c +celt_tests_test_unit_mathops_LDADD = $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +celt_tests_test_unit_mathops_LDADD += libarmasm.la +endif + +celt_tests_test_unit_mdct_SOURCES = celt/tests/test_unit_mdct.c +celt_tests_test_unit_mdct_LDADD = $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +celt_tests_test_unit_mdct_LDADD += libarmasm.la +endif + +celt_tests_test_unit_rotation_SOURCES = celt/tests/test_unit_rotation.c +celt_tests_test_unit_rotation_LDADD = $(CELT_OBJ) $(NE10_LIBS) $(LIBM) +if OPUS_ARM_EXTERNAL_ASM +celt_tests_test_unit_rotation_LDADD += libarmasm.la +endif + +celt_tests_test_unit_types_SOURCES = celt/tests/test_unit_types.c +celt_tests_test_unit_types_LDADD = $(LIBM) +endif + +if CUSTOM_MODES +pkginclude_HEADERS += include/opus_custom.h +if EXTRA_PROGRAMS +noinst_PROGRAMS += opus_custom_demo +opus_custom_demo_SOURCES = celt/opus_custom_demo.c +opus_custom_demo_LDADD = libopus.la $(LIBM) +endif +endif + +EXTRA_DIST = opus.pc.in \ + opus-uninstalled.pc.in \ + opus.m4 \ + Makefile.mips \ + Makefile.unix \ + CMakeLists.txt \ + cmake/CFeatureCheck.cmake \ + cmake/OpusBuildtype.cmake \ + cmake/OpusConfig.cmake \ + cmake/OpusConfig.cmake.in \ + cmake/OpusFunctions.cmake \ + cmake/OpusPackageVersion.cmake \ + cmake/OpusSources.cmake \ + cmake/config.h.cmake.in \ + cmake/vla.c \ + meson/get-version.py \ + meson/read-sources-list.py \ + meson.build \ + meson_options.txt \ + include/meson.build \ + celt/meson.build \ + celt/tests/meson.build \ + silk/meson.build \ + silk/tests/meson.build \ + src/meson.build \ + tests/meson.build \ + doc/meson.build \ + tests/run_vectors.sh \ + celt/arm/arm2gnu.pl \ + celt/arm/celt_pitch_xcorr_arm.s \ + win32/VS2015/opus.vcxproj \ + win32/VS2015/test_opus_encode.vcxproj.filters \ + win32/VS2015/test_opus_encode.vcxproj \ + win32/VS2015/opus_demo.vcxproj \ + win32/VS2015/test_opus_api.vcxproj.filters \ + win32/VS2015/test_opus_api.vcxproj \ + win32/VS2015/test_opus_decode.vcxproj.filters \ + win32/VS2015/opus_demo.vcxproj.filters \ + win32/VS2015/opus.vcxproj.filters \ + win32/VS2015/test_opus_decode.vcxproj \ + win32/VS2015/opus.sln \ + win32/VS2015/common.props \ + win32/genversion.bat \ + win32/config.h + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = opus.pc + +m4datadir = $(datadir)/aclocal +m4data_DATA = opus.m4 + +# Targets to build and install just the library without the docs +opus check-opus install-opus: export NO_DOXYGEN = 1 + +opus: all +check-opus: check +install-opus: install + + +# Or just the docs +docs: + ( cd doc && $(MAKE) $(AM_MAKEFLAGS) ) + +install-docs: + ( cd doc && $(MAKE) $(AM_MAKEFLAGS) install ) + + +# Or everything (by default) +all-local: + @[ -n "$(NO_DOXYGEN)" ] || ( cd doc && $(MAKE) $(AM_MAKEFLAGS) ) + +install-data-local: + @[ -n "$(NO_DOXYGEN)" ] || ( cd doc && $(MAKE) $(AM_MAKEFLAGS) install ) + +clean-local: + -( cd doc && $(MAKE) $(AM_MAKEFLAGS) clean ) + +uninstall-local: + ( cd doc && $(MAKE) $(AM_MAKEFLAGS) uninstall ) + + +# We check this every time make is run, with configure.ac being touched to +# trigger an update of the build system files if update_version changes the +# current PACKAGE_VERSION (or if package_version was modified manually by a +# user with either AUTO_UPDATE=no or no update_version script present - the +# latter being the normal case for tarball releases). +# +# We can't just add the package_version file to CONFIGURE_DEPENDENCIES since +# simply running autoconf will not actually regenerate configure for us when +# the content of that file changes (due to autoconf dependency checking not +# knowing about that without us creating yet another file for it to include). +# +# The MAKECMDGOALS check is a gnu-make'ism, but will degrade 'gracefully' for +# makes that don't support it. The only loss of functionality is not forcing +# an update of package_version for `make dist` if AUTO_UPDATE=no, but that is +# unlikely to be a real problem for any real user. +$(top_srcdir)/configure.ac: force + @case "$(MAKECMDGOALS)" in \ + dist-hook) exit 0 ;; \ + dist-* | dist | distcheck | distclean) _arg=release ;; \ + esac; \ + if ! $(top_srcdir)/update_version $$_arg 2> /dev/null; then \ + if [ ! -e $(top_srcdir)/package_version ]; then \ + echo 'PACKAGE_VERSION="unknown"' > $(top_srcdir)/package_version; \ + fi; \ + . $(top_srcdir)/package_version || exit 1; \ + [ "$(PACKAGE_VERSION)" != "$$PACKAGE_VERSION" ] || exit 0; \ + fi; \ + touch $@ + +force: + +# Create a minimal package_version file when make dist is run. +dist-hook: + echo 'PACKAGE_VERSION="$(PACKAGE_VERSION)"' > $(top_distdir)/package_version + + +.PHONY: opus check-opus install-opus docs install-docs + +# automake doesn't do dependency tracking for asm files, that I can tell +$(CELT_SOURCES_ARM_ASM:%.s=%-gnu.S): celt/arm/armopts-gnu.S +$(CELT_SOURCES_ARM_ASM:%.s=%-gnu.S): $(top_srcdir)/celt/arm/arm2gnu.pl + +# convert ARM asm to GNU as format +%-gnu.S: $(top_srcdir)/%.s + $(top_srcdir)/celt/arm/arm2gnu.pl @ARM2GNU_PARAMS@ < $< > $@ +# For autoconf-modified sources (e.g., armopts.s) +%-gnu.S: %.s + $(top_srcdir)/celt/arm/arm2gnu.pl @ARM2GNU_PARAMS@ < $< > $@ + +OPT_UNIT_TEST_OBJ = $(celt_tests_test_unit_mathops_SOURCES:.c=.o) \ + $(celt_tests_test_unit_rotation_SOURCES:.c=.o) \ + $(celt_tests_test_unit_mdct_SOURCES:.c=.o) \ + $(celt_tests_test_unit_dft_SOURCES:.c=.o) \ + $(silk_tests_test_unit_LPC_inv_pred_gain_SOURCES:.c=.o) + +if HAVE_SSE +SSE_OBJ = $(CELT_SOURCES_SSE:.c=.lo) +$(SSE_OBJ): CFLAGS += $(OPUS_X86_SSE_CFLAGS) +endif + +if HAVE_SSE2 +SSE2_OBJ = $(CELT_SOURCES_SSE2:.c=.lo) +$(SSE2_OBJ): CFLAGS += $(OPUS_X86_SSE2_CFLAGS) +endif + +if HAVE_SSE4_1 +SSE4_1_OBJ = $(CELT_SOURCES_SSE4_1:.c=.lo) \ + $(SILK_SOURCES_SSE4_1:.c=.lo) \ + $(SILK_SOURCES_FIXED_SSE4_1:.c=.lo) +$(SSE4_1_OBJ): CFLAGS += $(OPUS_X86_SSE4_1_CFLAGS) +endif + +if HAVE_ARM_NEON_INTR +ARM_NEON_INTR_OBJ = $(CELT_SOURCES_ARM_NEON_INTR:.c=.lo) \ + $(SILK_SOURCES_ARM_NEON_INTR:.c=.lo) \ + $(SILK_SOURCES_FIXED_ARM_NEON_INTR:.c=.lo) +$(ARM_NEON_INTR_OBJ): CFLAGS += \ + $(OPUS_ARM_NEON_INTR_CFLAGS) $(NE10_CFLAGS) +endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.mips b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.mips new file mode 100644 index 000000000..e9bfc22e8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.mips @@ -0,0 +1,161 @@ +#################### COMPILE OPTIONS ####################### + +# Uncomment this for fixed-point build +FIXED_POINT=1 + +# It is strongly recommended to uncomment one of these +# VAR_ARRAYS: Use C99 variable-length arrays for stack allocation +# USE_ALLOCA: Use alloca() for stack allocation +# If none is defined, then the fallback is a non-threadsafe global array +CFLAGS := -DUSE_ALLOCA $(CFLAGS) +#CFLAGS := -DVAR_ARRAYS $(CFLAGS) + +# These options affect performance +# HAVE_LRINTF: Use C99 intrinsics to speed up float-to-int conversion +CFLAGS := -DHAVE_LRINTF $(CFLAGS) + +###################### END OF OPTIONS ###################### + +-include package_version + +include silk_sources.mk +include celt_sources.mk +include opus_sources.mk + +ifdef FIXED_POINT +SILK_SOURCES += $(SILK_SOURCES_FIXED) +else +SILK_SOURCES += $(SILK_SOURCES_FLOAT) +OPUS_SOURCES += $(OPUS_SOURCES_FLOAT) +endif + +EXESUFFIX = +LIBPREFIX = lib +LIBSUFFIX = .a +OBJSUFFIX = .o + +CC = $(TOOLCHAIN_PREFIX)cc$(TOOLCHAIN_SUFFIX) +AR = $(TOOLCHAIN_PREFIX)ar +RANLIB = $(TOOLCHAIN_PREFIX)ranlib +CP = $(TOOLCHAIN_PREFIX)cp + +cppflags-from-defines = $(addprefix -D,$(1)) +cppflags-from-includes = $(addprefix -I,$(1)) +ldflags-from-ldlibdirs = $(addprefix -L,$(1)) +ldlibs-from-libs = $(addprefix -l,$(1)) + +WARNINGS = -Wall -W -Wstrict-prototypes -Wextra -Wcast-align -Wnested-externs -Wshadow + +CFLAGS += -mips32r2 -mno-mips16 -std=gnu99 -O2 -g $(WARNINGS) -DENABLE_ASSERTIONS -DMIPSr1_ASM -DOPUS_BUILD -mdspr2 -march=74kc -mtune=74kc -mmt -mgp32 + +CINCLUDES = include silk celt + +ifdef FIXED_POINT +CFLAGS += -DFIXED_POINT=1 -DDISABLE_FLOAT_API +CINCLUDES += silk/fixed +else +CINCLUDES += silk/float +endif + + +LIBS = m + +LDLIBDIRS = ./ + +CFLAGS += $(call cppflags-from-defines,$(CDEFINES)) +CFLAGS += $(call cppflags-from-includes,$(CINCLUDES)) +LDFLAGS += $(call ldflags-from-ldlibdirs,$(LDLIBDIRS)) +LDLIBS += $(call ldlibs-from-libs,$(LIBS)) + +COMPILE.c.cmdline = $(CC) -c $(CFLAGS) -o $@ $< +LINK.o = $(CC) $(LDPREFLAGS) $(LDFLAGS) +LINK.o.cmdline = $(LINK.o) $^ $(LDLIBS) -o $@$(EXESUFFIX) + +ARCHIVE.cmdline = $(AR) $(ARFLAGS) $@ $^ && $(RANLIB) $@ + +%$(OBJSUFFIX):%.c + $(COMPILE.c.cmdline) + +%$(OBJSUFFIX):%.cpp + $(COMPILE.cpp.cmdline) + +# Directives + + +# Variable definitions +LIB_NAME = opus +TARGET = $(LIBPREFIX)$(LIB_NAME)$(LIBSUFFIX) + +SRCS_C = $(SILK_SOURCES) $(CELT_SOURCES) $(OPUS_SOURCES) + +OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(SRCS_C)) + +OPUSDEMO_SRCS_C = src/opus_demo.c +OPUSDEMO_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(OPUSDEMO_SRCS_C)) + +TESTOPUSAPI_SRCS_C = tests/test_opus_api.c +TESTOPUSAPI_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSAPI_SRCS_C)) + +TESTOPUSDECODE_SRCS_C = tests/test_opus_decode.c +TESTOPUSDECODE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSDECODE_SRCS_C)) + +TESTOPUSENCODE_SRCS_C = tests/test_opus_encode.c tests/opus_encode_regressions.c +TESTOPUSENCODE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSENCODE_SRCS_C)) + +TESTOPUSPADDING_SRCS_C = tests/test_opus_padding.c +TESTOPUSPADDING_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSPADDING_SRCS_C)) + +OPUSCOMPARE_SRCS_C = src/opus_compare.c +OPUSCOMPARE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(OPUSCOMPARE_SRCS_C)) + +TESTS := test_opus_api test_opus_decode test_opus_encode test_opus_padding + +# Rules +all: lib opus_demo opus_compare $(TESTS) + +lib: $(TARGET) + +check: all + for test in $(TESTS); do ./$$test; done + +$(TARGET): $(OBJS) + $(ARCHIVE.cmdline) + +opus_demo$(EXESUFFIX): $(OPUSDEMO_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_api$(EXESUFFIX): $(TESTOPUSAPI_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_decode$(EXESUFFIX): $(TESTOPUSDECODE_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_encode$(EXESUFFIX): $(TESTOPUSENCODE_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_padding$(EXESUFFIX): $(TESTOPUSPADDING_OBJS) $(TARGET) + $(LINK.o.cmdline) + +opus_compare$(EXESUFFIX): $(OPUSCOMPARE_OBJS) + $(LINK.o.cmdline) + +celt/celt.o: CFLAGS += -DPACKAGE_VERSION='$(PACKAGE_VERSION)' +celt/celt.o: package_version + +package_version: force + @if [ -x ./update_version ]; then \ + ./update_version || true; \ + elif [ ! -e ./package_version ]; then \ + echo 'PACKAGE_VERSION="unknown"' > ./package_version; \ + fi + +force: + +clean: + rm -f opus_demo$(EXESUFFIX) opus_compare$(EXESUFFIX) $(TARGET) \ + test_opus_api$(EXESUFFIX) test_opus_decode$(EXESUFFIX) \ + test_opus_encode$(EXESUFFIX) test_opus_padding$(EXESUFFIX) \ + $(OBJS) $(OPUSDEMO_OBJS) $(OPUSCOMPARE_OBJS) $(TESTOPUSAPI_OBJS) \ + $(TESTOPUSDECODE_OBJS) $(TESTOPUSENCODE_OBJS) $(TESTOPUSPADDING_OBJS) + +.PHONY: all lib clean force check diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.unix b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.unix new file mode 100644 index 000000000..90a48f0cc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/Makefile.unix @@ -0,0 +1,159 @@ +#################### COMPILE OPTIONS ####################### + +# Uncomment this for fixed-point build +#FIXED_POINT=1 + +# It is strongly recommended to uncomment one of these +# VAR_ARRAYS: Use C99 variable-length arrays for stack allocation +# USE_ALLOCA: Use alloca() for stack allocation +# If none is defined, then the fallback is a non-threadsafe global array +CFLAGS := -DUSE_ALLOCA $(CFLAGS) +#CFLAGS := -DVAR_ARRAYS $(CFLAGS) + +# These options affect performance +# HAVE_LRINTF: Use C99 intrinsics to speed up float-to-int conversion +#CFLAGS := -DHAVE_LRINTF $(CFLAGS) + +###################### END OF OPTIONS ###################### + +-include package_version + +include silk_sources.mk +include celt_sources.mk +include opus_sources.mk + +ifdef FIXED_POINT +SILK_SOURCES += $(SILK_SOURCES_FIXED) +else +SILK_SOURCES += $(SILK_SOURCES_FLOAT) +OPUS_SOURCES += $(OPUS_SOURCES_FLOAT) +endif + +EXESUFFIX = +LIBPREFIX = lib +LIBSUFFIX = .a +OBJSUFFIX = .o + +CC = $(TOOLCHAIN_PREFIX)cc$(TOOLCHAIN_SUFFIX) +AR = $(TOOLCHAIN_PREFIX)ar +RANLIB = $(TOOLCHAIN_PREFIX)ranlib +CP = $(TOOLCHAIN_PREFIX)cp + +cppflags-from-defines = $(addprefix -D,$(1)) +cppflags-from-includes = $(addprefix -I,$(1)) +ldflags-from-ldlibdirs = $(addprefix -L,$(1)) +ldlibs-from-libs = $(addprefix -l,$(1)) + +WARNINGS = -Wall -W -Wstrict-prototypes -Wextra -Wcast-align -Wnested-externs -Wshadow +CFLAGS += -O2 -g $(WARNINGS) -DOPUS_BUILD +CINCLUDES = include silk celt + +ifdef FIXED_POINT +CFLAGS += -DFIXED_POINT=1 -DDISABLE_FLOAT_API +CINCLUDES += silk/fixed +else +CINCLUDES += silk/float +endif + + +LIBS = m + +LDLIBDIRS = ./ + +CFLAGS += $(call cppflags-from-defines,$(CDEFINES)) +CFLAGS += $(call cppflags-from-includes,$(CINCLUDES)) +LDFLAGS += $(call ldflags-from-ldlibdirs,$(LDLIBDIRS)) +LDLIBS += $(call ldlibs-from-libs,$(LIBS)) + +COMPILE.c.cmdline = $(CC) -c $(CFLAGS) -o $@ $< +LINK.o = $(CC) $(LDPREFLAGS) $(LDFLAGS) +LINK.o.cmdline = $(LINK.o) $^ $(LDLIBS) -o $@$(EXESUFFIX) + +ARCHIVE.cmdline = $(AR) $(ARFLAGS) $@ $^ && $(RANLIB) $@ + +%$(OBJSUFFIX):%.c + $(COMPILE.c.cmdline) + +%$(OBJSUFFIX):%.cpp + $(COMPILE.cpp.cmdline) + +# Directives + + +# Variable definitions +LIB_NAME = opus +TARGET = $(LIBPREFIX)$(LIB_NAME)$(LIBSUFFIX) + +SRCS_C = $(SILK_SOURCES) $(CELT_SOURCES) $(OPUS_SOURCES) + +OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(SRCS_C)) + +OPUSDEMO_SRCS_C = src/opus_demo.c +OPUSDEMO_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(OPUSDEMO_SRCS_C)) + +TESTOPUSAPI_SRCS_C = tests/test_opus_api.c +TESTOPUSAPI_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSAPI_SRCS_C)) + +TESTOPUSDECODE_SRCS_C = tests/test_opus_decode.c +TESTOPUSDECODE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSDECODE_SRCS_C)) + +TESTOPUSENCODE_SRCS_C = tests/test_opus_encode.c tests/opus_encode_regressions.c +TESTOPUSENCODE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSENCODE_SRCS_C)) + +TESTOPUSPADDING_SRCS_C = tests/test_opus_padding.c +TESTOPUSPADDING_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(TESTOPUSPADDING_SRCS_C)) + +OPUSCOMPARE_SRCS_C = src/opus_compare.c +OPUSCOMPARE_OBJS := $(patsubst %.c,%$(OBJSUFFIX),$(OPUSCOMPARE_SRCS_C)) + +TESTS := test_opus_api test_opus_decode test_opus_encode test_opus_padding + +# Rules +all: lib opus_demo opus_compare $(TESTS) + +lib: $(TARGET) + +check: all + for test in $(TESTS); do ./$$test; done + +$(TARGET): $(OBJS) + $(ARCHIVE.cmdline) + +opus_demo$(EXESUFFIX): $(OPUSDEMO_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_api$(EXESUFFIX): $(TESTOPUSAPI_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_decode$(EXESUFFIX): $(TESTOPUSDECODE_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_encode$(EXESUFFIX): $(TESTOPUSENCODE_OBJS) $(TARGET) + $(LINK.o.cmdline) + +test_opus_padding$(EXESUFFIX): $(TESTOPUSPADDING_OBJS) $(TARGET) + $(LINK.o.cmdline) + +opus_compare$(EXESUFFIX): $(OPUSCOMPARE_OBJS) + $(LINK.o.cmdline) + +celt/celt.o: CFLAGS += -DPACKAGE_VERSION='$(PACKAGE_VERSION)' +celt/celt.o: package_version + +package_version: force + @if [ -x ./update_version ]; then \ + ./update_version || true; \ + elif [ ! -e ./package_version ]; then \ + echo 'PACKAGE_VERSION="unknown"' > ./package_version; \ + fi + +force: + +clean: + rm -f opus_demo$(EXESUFFIX) opus_compare$(EXESUFFIX) $(TARGET) \ + test_opus_api$(EXESUFFIX) test_opus_decode$(EXESUFFIX) \ + test_opus_encode$(EXESUFFIX) test_opus_padding$(EXESUFFIX) \ + $(OBJS) $(OPUSDEMO_OBJS) $(OPUSCOMPARE_OBJS) $(TESTOPUSAPI_OBJS) \ + $(TESTOPUSDECODE_OBJS) $(TESTOPUSENCODE_OBJS) $(TESTOPUSPADDING_OBJS) + +.PHONY: all lib clean force check diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/NEWS b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/NEWS new file mode 100644 index 000000000..e69de29bb diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README new file mode 100644 index 000000000..4b13076fa --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README @@ -0,0 +1,161 @@ +== Opus audio codec == + +Opus is a codec for interactive speech and audio transmission over the Internet. + + Opus can handle a wide range of interactive audio applications, including +Voice over IP, videoconferencing, in-game chat, and even remote live music +performances. It can scale from low bit-rate narrowband speech to very high +quality stereo music. + + Opus, when coupled with an appropriate container format, is also suitable +for non-realtime stored-file applications such as music distribution, game +soundtracks, portable music players, jukeboxes, and other applications that +have historically used high latency formats such as MP3, AAC, or Vorbis. + + Opus is specified by IETF RFC 6716: + https://tools.ietf.org/html/rfc6716 + + The Opus format and this implementation of it are subject to the royalty- +free patent and copyright licenses specified in the file COPYING. + +This package implements a shared library for encoding and decoding raw Opus +bitstreams. Raw Opus bitstreams should be used over RTP according to + https://tools.ietf.org/html/rfc7587 + +The package also includes a number of test tools used for testing the +correct operation of the library. The bitstreams read/written by these +tools should not be used for Opus file distribution: They include +additional debugging data and cannot support seeking. + +Opus stored in files should use the Ogg encapsulation for Opus which is +described at: + https://tools.ietf.org/html/rfc7845 + +An opus-tools package is available which provides encoding and decoding of +Ogg encapsulated Opus files and includes a number of useful features. + +Opus-tools can be found at: + https://gitlab.xiph.org/xiph/opus-tools.git +or on the main Opus website: + https://opus-codec.org/ + +== Compiling libopus == + +To build from a distribution tarball, you only need to do the following: + + % ./configure + % make + +To build from the git repository, the following steps are necessary: + +0) Set up a development environment: + +On an Ubuntu or Debian family Linux distribution: + + % sudo apt-get install git autoconf automake libtool gcc make + +On a Fedora/Redhat based Linux: + + % sudo dnf install git autoconf automake libtool gcc make + +Or for older Redhat/Centos Linux releases: + + % sudo yum install git autoconf automake libtool gcc make + +On Apple macOS, install Xcode and brew.sh, then in the Terminal enter: + + % brew install autoconf automake libtool + +1) Clone the repository: + + % git clone https://gitlab.xiph.org/xiph/opus.git + % cd opus + +2) Compiling the source + + % ./autogen.sh + % ./configure + % make + +3) Install the codec libraries (optional) + + % sudo make install + +Once you have compiled the codec, there will be a opus_demo executable +in the top directory. + +Usage: opus_demo [-e] + [options] + opus_demo -d [options] + + +mode: voip | audio | restricted-lowdelay +options: + -e : only runs the encoder (output the bit-stream) + -d : only runs the decoder (reads the bit-stream as input) + -cbr : enable constant bitrate; default: variable bitrate + -cvbr : enable constrained variable bitrate; default: + unconstrained + -bandwidth + : audio bandwidth (from narrowband to fullband); + default: sampling rate + -framesize <2.5|5|10|20|40|60> + : frame size in ms; default: 20 + -max_payload + : maximum payload size in bytes, default: 1024 + -complexity + : complexity, 0 (lowest) ... 10 (highest); default: 10 + -inbandfec : enable SILK inband FEC + -forcemono : force mono encoding, even for stereo input + -dtx : enable SILK DTX + -loss : simulate packet loss, in percent (0-100); default: 0 + +input and output are little-endian signed 16-bit PCM files or opus +bitstreams with simple opus_demo proprietary framing. + +== Testing == + +This package includes a collection of automated unit and system tests +which SHOULD be run after compiling the package especially the first +time it is run on a new platform. + +To run the integrated tests: + + % make check + +There is also collection of standard test vectors which are not +included in this package for size reasons but can be obtained from: +https://opus-codec.org/docs/opus_testvectors-rfc8251.tar.gz + +To run compare the code to these test vectors: + + % curl -OL https://opus-codec.org/docs/opus_testvectors-rfc8251.tar.gz + % tar -zxf opus_testvectors-rfc8251.tar.gz + % ./tests/run_vectors.sh ./ opus_newvectors 48000 + +== Portability notes == + +This implementation uses floating-point by default but can be compiled to +use only fixed-point arithmetic by setting --enable-fixed-point (if using +autoconf) or by defining the FIXED_POINT macro (if building manually). +The fixed point implementation has somewhat lower audio quality and is +slower on platforms with fast FPUs, it is normally only used in embedded +environments. + +The implementation can be compiled with either a C89 or a C99 compiler. +While it does not rely on any _undefined behavior_ as defined by C89 or +C99, it relies on common _implementation-defined behavior_ for two's +complement architectures: + +o Right shifts of negative values are consistent with two's + complement arithmetic, so that a>>b is equivalent to + floor(a/(2^b)), + +o For conversion to a signed integer of N bits, the value is reduced + modulo 2^N to be within range of the type, + +o The result of integer division of a negative value is truncated + towards zero, and + +o The compiler provides a 64-bit integer type (a C99 requirement + which is supported by most C89 compilers). diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README.draft b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README.draft new file mode 100644 index 000000000..9c31bd023 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/README.draft @@ -0,0 +1,54 @@ +To build this source code, simply type: + +% make + +If this does not work, or if you want to change the default configuration +(e.g., to compile for a fixed-point architecture), simply edit the options +in the Makefile. + +An up-to-date implementation conforming to this standard is available in a +Git repository at https://gitlab.xiph.org/xiph/opus.git or on a website at: +https://opus-codec.org/ +However, although that implementation is expected to remain conformant +with the standard, it is the code in this RFC that shall remain normative. +To build from the git repository instead of using this RFC, follow these +steps: + +1) Clone the repository (latest implementation of this standard at the time +of publication) + +% git clone https://gitlab.xiph.org/xiph/opus.git +% cd opus + +2) Compile + +% ./autogen.sh +% ./configure +% make + +Once you have compiled the codec, there will be a opus_demo executable in +the top directory. + +Usage: opus_demo [-e] + [options] + opus_demo -d [options] + + +mode: voip | audio | restricted-lowdelay +options: +-e : only runs the encoder (output the bit-stream) +-d : only runs the decoder (reads the bit-stream as input) +-cbr : enable constant bitrate; default: variable bitrate +-cvbr : enable constrained variable bitrate; default: unconstrained +-bandwidth : audio bandwidth (from narrowband to fullband); + default: sampling rate +-framesize <2.5|5|10|20|40|60> : frame size in ms; default: 20 +-max_payload : maximum payload size in bytes, default: 1024 +-complexity : complexity, 0 (lowest) ... 10 (highest); default: 10 +-inbandfec : enable SILK inband FEC +-forcemono : force mono encoding, even for stereo input +-dtx : enable SILK DTX +-loss : simulate packet loss, in percent (0-100); default: 0 + +input and output are little endian signed 16-bit PCM files or opus bitstreams +with simple opus_demo proprietary framing. diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/autogen.sh b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/autogen.sh new file mode 100755 index 000000000..380d1f3ca --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/autogen.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Copyright (c) 2010-2015 Xiph.Org Foundation and contributors. +# Use of this source code is governed by a BSD-style license that can be +# found in the COPYING file. + +# Run this to set up the build system: configure, makefiles, etc. +set -e + +srcdir=`dirname $0` +test -n "$srcdir" && cd "$srcdir" + +echo "Updating build configuration files, please wait...." + +autoreconf -isf diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/_kiss_fft_guts.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/_kiss_fft_guts.h new file mode 100644 index 000000000..17392b3e9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/_kiss_fft_guts.h @@ -0,0 +1,182 @@ +/*Copyright (c) 2003-2004, Mark Borgerding + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.*/ + +#ifndef KISS_FFT_GUTS_H +#define KISS_FFT_GUTS_H + +#define MIN(a,b) ((a)<(b) ? (a):(b)) +#define MAX(a,b) ((a)>(b) ? (a):(b)) + +/* kiss_fft.h + defines kiss_fft_scalar as either short or a float type + and defines + typedef struct { kiss_fft_scalar r; kiss_fft_scalar i; }kiss_fft_cpx; */ +#include "kiss_fft.h" + +/* + Explanation of macros dealing with complex math: + + C_MUL(m,a,b) : m = a*b + C_FIXDIV( c , div ) : if a fixed point impl., c /= div. noop otherwise + C_SUB( res, a,b) : res = a - b + C_SUBFROM( res , a) : res -= a + C_ADDTO( res , a) : res += a + * */ +#ifdef FIXED_POINT +#include "arch.h" + + +#define SAMP_MAX 2147483647 +#define TWID_MAX 32767 +#define TRIG_UPSCALE 1 + +#define SAMP_MIN -SAMP_MAX + + +# define S_MUL(a,b) MULT16_32_Q15(b, a) + +# define C_MUL(m,a,b) \ + do{ (m).r = SUB32_ovflw(S_MUL((a).r,(b).r) , S_MUL((a).i,(b).i)); \ + (m).i = ADD32_ovflw(S_MUL((a).r,(b).i) , S_MUL((a).i,(b).r)); }while(0) + +# define C_MULC(m,a,b) \ + do{ (m).r = ADD32_ovflw(S_MUL((a).r,(b).r) , S_MUL((a).i,(b).i)); \ + (m).i = SUB32_ovflw(S_MUL((a).i,(b).r) , S_MUL((a).r,(b).i)); }while(0) + +# define C_MULBYSCALAR( c, s ) \ + do{ (c).r = S_MUL( (c).r , s ) ;\ + (c).i = S_MUL( (c).i , s ) ; }while(0) + +# define DIVSCALAR(x,k) \ + (x) = S_MUL( x, (TWID_MAX-((k)>>1))/(k)+1 ) + +# define C_FIXDIV(c,div) \ + do { DIVSCALAR( (c).r , div); \ + DIVSCALAR( (c).i , div); }while (0) + +#define C_ADD( res, a,b)\ + do {(res).r=ADD32_ovflw((a).r,(b).r); (res).i=ADD32_ovflw((a).i,(b).i); \ + }while(0) +#define C_SUB( res, a,b)\ + do {(res).r=SUB32_ovflw((a).r,(b).r); (res).i=SUB32_ovflw((a).i,(b).i); \ + }while(0) +#define C_ADDTO( res , a)\ + do {(res).r = ADD32_ovflw((res).r, (a).r); (res).i = ADD32_ovflw((res).i,(a).i);\ + }while(0) + +#define C_SUBFROM( res , a)\ + do {(res).r = ADD32_ovflw((res).r,(a).r); (res).i = SUB32_ovflw((res).i,(a).i); \ + }while(0) + +#if defined(OPUS_ARM_INLINE_ASM) +#include "arm/kiss_fft_armv4.h" +#endif + +#if defined(OPUS_ARM_INLINE_EDSP) +#include "arm/kiss_fft_armv5e.h" +#endif +#if defined(MIPSr1_ASM) +#include "mips/kiss_fft_mipsr1.h" +#endif + +#else /* not FIXED_POINT*/ + +# define S_MUL(a,b) ( (a)*(b) ) +#define C_MUL(m,a,b) \ + do{ (m).r = (a).r*(b).r - (a).i*(b).i;\ + (m).i = (a).r*(b).i + (a).i*(b).r; }while(0) +#define C_MULC(m,a,b) \ + do{ (m).r = (a).r*(b).r + (a).i*(b).i;\ + (m).i = (a).i*(b).r - (a).r*(b).i; }while(0) + +#define C_MUL4(m,a,b) C_MUL(m,a,b) + +# define C_FIXDIV(c,div) /* NOOP */ +# define C_MULBYSCALAR( c, s ) \ + do{ (c).r *= (s);\ + (c).i *= (s); }while(0) +#endif + +#ifndef CHECK_OVERFLOW_OP +# define CHECK_OVERFLOW_OP(a,op,b) /* noop */ +#endif + +#ifndef C_ADD +#define C_ADD( res, a,b)\ + do { \ + CHECK_OVERFLOW_OP((a).r,+,(b).r)\ + CHECK_OVERFLOW_OP((a).i,+,(b).i)\ + (res).r=(a).r+(b).r; (res).i=(a).i+(b).i; \ + }while(0) +#define C_SUB( res, a,b)\ + do { \ + CHECK_OVERFLOW_OP((a).r,-,(b).r)\ + CHECK_OVERFLOW_OP((a).i,-,(b).i)\ + (res).r=(a).r-(b).r; (res).i=(a).i-(b).i; \ + }while(0) +#define C_ADDTO( res , a)\ + do { \ + CHECK_OVERFLOW_OP((res).r,+,(a).r)\ + CHECK_OVERFLOW_OP((res).i,+,(a).i)\ + (res).r += (a).r; (res).i += (a).i;\ + }while(0) + +#define C_SUBFROM( res , a)\ + do {\ + CHECK_OVERFLOW_OP((res).r,-,(a).r)\ + CHECK_OVERFLOW_OP((res).i,-,(a).i)\ + (res).r -= (a).r; (res).i -= (a).i; \ + }while(0) +#endif /* C_ADD defined */ + +#ifdef FIXED_POINT +/*# define KISS_FFT_COS(phase) TRIG_UPSCALE*floor(MIN(32767,MAX(-32767,.5+32768 * cos (phase)))) +# define KISS_FFT_SIN(phase) TRIG_UPSCALE*floor(MIN(32767,MAX(-32767,.5+32768 * sin (phase))))*/ +# define KISS_FFT_COS(phase) floor(.5+TWID_MAX*cos (phase)) +# define KISS_FFT_SIN(phase) floor(.5+TWID_MAX*sin (phase)) +# define HALF_OF(x) ((x)>>1) +#elif defined(USE_SIMD) +# define KISS_FFT_COS(phase) _mm_set1_ps( cos(phase) ) +# define KISS_FFT_SIN(phase) _mm_set1_ps( sin(phase) ) +# define HALF_OF(x) ((x)*_mm_set1_ps(.5f)) +#else +# define KISS_FFT_COS(phase) (kiss_fft_scalar) cos(phase) +# define KISS_FFT_SIN(phase) (kiss_fft_scalar) sin(phase) +# define HALF_OF(x) ((x)*.5f) +#endif + +#define kf_cexp(x,phase) \ + do{ \ + (x)->r = KISS_FFT_COS(phase);\ + (x)->i = KISS_FFT_SIN(phase);\ + }while(0) + +#define kf_cexp2(x,phase) \ + do{ \ + (x)->r = TRIG_UPSCALE*celt_cos_norm((phase));\ + (x)->i = TRIG_UPSCALE*celt_cos_norm((phase)-32768);\ +}while(0) + +#endif /* KISS_FFT_GUTS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arch.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arch.h new file mode 100644 index 000000000..3845c3a08 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arch.h @@ -0,0 +1,291 @@ +/* Copyright (c) 2003-2008 Jean-Marc Valin + Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/** + @file arch.h + @brief Various architecture definitions for CELT +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef ARCH_H +#define ARCH_H + +#include "opus_types.h" +#include "opus_defines.h" + +# if !defined(__GNUC_PREREQ) +# if defined(__GNUC__)&&defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(_maj,_min) \ + ((__GNUC__<<16)+__GNUC_MINOR__>=((_maj)<<16)+(_min)) +# else +# define __GNUC_PREREQ(_maj,_min) 0 +# endif +# endif + +#if OPUS_GNUC_PREREQ(3, 0) +#define opus_likely(x) (__builtin_expect(!!(x), 1)) +#define opus_unlikely(x) (__builtin_expect(!!(x), 0)) +#else +#define opus_likely(x) (!!(x)) +#define opus_unlikely(x) (!!(x)) +#endif + +#define CELT_SIG_SCALE 32768.f + +#define CELT_FATAL(str) celt_fatal(str, __FILE__, __LINE__); + +#if defined(ENABLE_ASSERTIONS) || defined(ENABLE_HARDENING) +#ifdef __GNUC__ +__attribute__((noreturn)) +#endif +void celt_fatal(const char *str, const char *file, int line); + +#if defined(CELT_C) && !defined(OVERRIDE_celt_fatal) +#include +#include +#ifdef __GNUC__ +__attribute__((noreturn)) +#endif +void celt_fatal(const char *str, const char *file, int line) +{ + fprintf (stderr, "Fatal (internal) error in %s, line %d: %s\n", file, line, str); +#if defined(_MSC_VER) + _set_abort_behavior( 0, _WRITE_ABORT_MSG); +#endif + abort(); +} +#endif + +#define celt_assert(cond) {if (!(cond)) {CELT_FATAL("assertion failed: " #cond);}} +#define celt_assert2(cond, message) {if (!(cond)) {CELT_FATAL("assertion failed: " #cond "\n" message);}} +#define MUST_SUCCEED(call) celt_assert((call) == OPUS_OK) +#else +#define celt_assert(cond) +#define celt_assert2(cond, message) +#define MUST_SUCCEED(call) do {if((call) != OPUS_OK) {RESTORE_STACK; return OPUS_INTERNAL_ERROR;} } while (0) +#endif + +#if defined(ENABLE_ASSERTIONS) +#define celt_sig_assert(cond) {if (!(cond)) {CELT_FATAL("signal assertion failed: " #cond);}} +#else +#define celt_sig_assert(cond) +#endif + +#define IMUL32(a,b) ((a)*(b)) + +#define MIN16(a,b) ((a) < (b) ? (a) : (b)) /**< Minimum 16-bit value. */ +#define MAX16(a,b) ((a) > (b) ? (a) : (b)) /**< Maximum 16-bit value. */ +#define MIN32(a,b) ((a) < (b) ? (a) : (b)) /**< Minimum 32-bit value. */ +#define MAX32(a,b) ((a) > (b) ? (a) : (b)) /**< Maximum 32-bit value. */ +#define IMIN(a,b) ((a) < (b) ? (a) : (b)) /**< Minimum int value. */ +#define IMAX(a,b) ((a) > (b) ? (a) : (b)) /**< Maximum int value. */ +#define UADD32(a,b) ((a)+(b)) +#define USUB32(a,b) ((a)-(b)) + +/* Set this if opus_int64 is a native type of the CPU. */ +/* Assume that all LP64 architectures have fast 64-bit types; also x86_64 + (which can be ILP32 for x32) and Win64 (which is LLP64). */ +#if defined(__x86_64__) || defined(__LP64__) || defined(_WIN64) +#define OPUS_FAST_INT64 1 +#else +#define OPUS_FAST_INT64 0 +#endif + +#define PRINT_MIPS(file) + +#ifdef FIXED_POINT + +typedef opus_int16 opus_val16; +typedef opus_int32 opus_val32; +typedef opus_int64 opus_val64; + +typedef opus_val32 celt_sig; +typedef opus_val16 celt_norm; +typedef opus_val32 celt_ener; + +#define celt_isnan(x) 0 + +#define Q15ONE 32767 + +#define SIG_SHIFT 12 +/* Safe saturation value for 32-bit signals. Should be less than + 2^31*(1-0.85) to avoid blowing up on DC at deemphasis.*/ +#define SIG_SAT (300000000) + +#define NORM_SCALING 16384 + +#define DB_SHIFT 10 + +#define EPSILON 1 +#define VERY_SMALL 0 +#define VERY_LARGE16 ((opus_val16)32767) +#define Q15_ONE ((opus_val16)32767) + +#define SCALEIN(a) (a) +#define SCALEOUT(a) (a) + +#define ABS16(x) ((x) < 0 ? (-(x)) : (x)) +#define ABS32(x) ((x) < 0 ? (-(x)) : (x)) + +static OPUS_INLINE opus_int16 SAT16(opus_int32 x) { + return x > 32767 ? 32767 : x < -32768 ? -32768 : (opus_int16)x; +} + +#ifdef FIXED_DEBUG +#include "fixed_debug.h" +#else + +#include "fixed_generic.h" + +#ifdef OPUS_ARM_PRESUME_AARCH64_NEON_INTR +#include "arm/fixed_arm64.h" +#elif defined (OPUS_ARM_INLINE_EDSP) +#include "arm/fixed_armv5e.h" +#elif defined (OPUS_ARM_INLINE_ASM) +#include "arm/fixed_armv4.h" +#elif defined (BFIN_ASM) +#include "fixed_bfin.h" +#elif defined (TI_C5X_ASM) +#include "fixed_c5x.h" +#elif defined (TI_C6X_ASM) +#include "fixed_c6x.h" +#endif + +#endif + +#else /* FIXED_POINT */ + +typedef float opus_val16; +typedef float opus_val32; +typedef float opus_val64; + +typedef float celt_sig; +typedef float celt_norm; +typedef float celt_ener; + +#ifdef FLOAT_APPROX +/* This code should reliably detect NaN/inf even when -ffast-math is used. + Assumes IEEE 754 format. */ +static OPUS_INLINE int celt_isnan(float x) +{ + union {float f; opus_uint32 i;} in; + in.f = x; + return ((in.i>>23)&0xFF)==0xFF && (in.i&0x007FFFFF)!=0; +} +#else +#ifdef __FAST_MATH__ +#error Cannot build libopus with -ffast-math unless FLOAT_APPROX is defined. This could result in crashes on extreme (e.g. NaN) input +#endif +#define celt_isnan(x) ((x)!=(x)) +#endif + +#define Q15ONE 1.0f + +#define NORM_SCALING 1.f + +#define EPSILON 1e-15f +#define VERY_SMALL 1e-30f +#define VERY_LARGE16 1e15f +#define Q15_ONE ((opus_val16)1.f) + +/* This appears to be the same speed as C99's fabsf() but it's more portable. */ +#define ABS16(x) ((float)fabs(x)) +#define ABS32(x) ((float)fabs(x)) + +#define QCONST16(x,bits) (x) +#define QCONST32(x,bits) (x) + +#define NEG16(x) (-(x)) +#define NEG32(x) (-(x)) +#define NEG32_ovflw(x) (-(x)) +#define EXTRACT16(x) (x) +#define EXTEND32(x) (x) +#define SHR16(a,shift) (a) +#define SHL16(a,shift) (a) +#define SHR32(a,shift) (a) +#define SHL32(a,shift) (a) +#define PSHR32(a,shift) (a) +#define VSHR32(a,shift) (a) + +#define PSHR(a,shift) (a) +#define SHR(a,shift) (a) +#define SHL(a,shift) (a) +#define SATURATE(x,a) (x) +#define SATURATE16(x) (x) + +#define ROUND16(a,shift) (a) +#define SROUND16(a,shift) (a) +#define HALF16(x) (.5f*(x)) +#define HALF32(x) (.5f*(x)) + +#define ADD16(a,b) ((a)+(b)) +#define SUB16(a,b) ((a)-(b)) +#define ADD32(a,b) ((a)+(b)) +#define SUB32(a,b) ((a)-(b)) +#define ADD32_ovflw(a,b) ((a)+(b)) +#define SUB32_ovflw(a,b) ((a)-(b)) +#define MULT16_16_16(a,b) ((a)*(b)) +#define MULT16_16(a,b) ((opus_val32)(a)*(opus_val32)(b)) +#define MAC16_16(c,a,b) ((c)+(opus_val32)(a)*(opus_val32)(b)) + +#define MULT16_32_Q15(a,b) ((a)*(b)) +#define MULT16_32_Q16(a,b) ((a)*(b)) + +#define MULT32_32_Q31(a,b) ((a)*(b)) + +#define MAC16_32_Q15(c,a,b) ((c)+(a)*(b)) +#define MAC16_32_Q16(c,a,b) ((c)+(a)*(b)) + +#define MULT16_16_Q11_32(a,b) ((a)*(b)) +#define MULT16_16_Q11(a,b) ((a)*(b)) +#define MULT16_16_Q13(a,b) ((a)*(b)) +#define MULT16_16_Q14(a,b) ((a)*(b)) +#define MULT16_16_Q15(a,b) ((a)*(b)) +#define MULT16_16_P15(a,b) ((a)*(b)) +#define MULT16_16_P13(a,b) ((a)*(b)) +#define MULT16_16_P14(a,b) ((a)*(b)) +#define MULT16_32_P16(a,b) ((a)*(b)) + +#define DIV32_16(a,b) (((opus_val32)(a))/(opus_val16)(b)) +#define DIV32(a,b) (((opus_val32)(a))/(opus_val32)(b)) + +#define SCALEIN(a) ((a)*CELT_SIG_SCALE) +#define SCALEOUT(a) ((a)*(1/CELT_SIG_SCALE)) + +#define SIG2WORD16(x) (x) + +#endif /* !FIXED_POINT */ + +#ifndef GLOBAL_STACK_SIZE +#ifdef FIXED_POINT +#define GLOBAL_STACK_SIZE 120000 +#else +#define GLOBAL_STACK_SIZE 120000 +#endif +#endif + +#endif /* ARCH_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm2gnu.pl b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm2gnu.pl new file mode 100755 index 000000000..a2895f744 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm2gnu.pl @@ -0,0 +1,353 @@ +#!/usr/bin/perl +# Copyright (C) 2002-2013 Xiph.org Foundation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +my $bigend; # little/big endian +my $nxstack; +my $apple = 0; +my $symprefix = ""; + +$nxstack = 0; + +eval 'exec /usr/local/bin/perl -S $0 ${1+"$@"}' + if $running_under_some_shell; + +while ($ARGV[0] =~ /^-/) { + $_ = shift; + last if /^--$/; + if (/^-n$/) { + $nflag++; + next; + } + if (/^--apple$/) { + $apple = 1; + $symprefix = "_"; + next; + } + die "I don't recognize this switch: $_\\n"; +} +$printit++ unless $nflag; + +$\ = "\n"; # automatically add newline on print +$n=0; + +$thumb = 0; # ARM mode by default, not Thumb. +@proc_stack = (); + +printf (" .syntax unified\n"); + +LINE: +while (<>) { + + # For ADRLs we need to add a new line after the substituted one. + $addPadding = 0; + + # First, we do not dare to touch *anything* inside double quotes, do we? + # Second, if you want a dollar character in the string, + # insert two of them -- that's how ARM C and assembler treat strings. + s/^([A-Za-z_]\w*)[ \t]+DCB[ \t]*\"/$1: .ascii \"/ && do { s/\$\$/\$/g; next }; + s/\bDCB\b[ \t]*\"/.ascii \"/ && do { s/\$\$/\$/g; next }; + s/^(\S+)\s+RN\s+(\S+)/$1 .req r$2/ && do { s/\$\$/\$/g; next }; + # If there's nothing on a line but a comment, don't try to apply any further + # substitutions (this is a cheap hack to avoid mucking up the license header) + s/^([ \t]*);/$1@/ && do { s/\$\$/\$/g; next }; + # If substituted -- leave immediately ! + + s/@/,:/; + s/;/@/; + while ( /@.*'/ ) { + s/(@.*)'/$1/g; + } + s/\{FALSE\}/0/g; + s/\{TRUE\}/1/g; + s/\{(\w\w\w\w+)\}/$1/g; + s/\bINCLUDE[ \t]*([^ \t\n]+)/.include \"$1\"/; + s/\bGET[ \t]*([^ \t\n]+)/.include \"${ my $x=$1; $x =~ s|\.s|-gnu.S|; \$x }\"/; + s/\bIMPORT\b/.extern/; + s/\bEXPORT\b\s*/.global $symprefix/; + s/^(\s+)\[/$1IF/; + s/^(\s+)\|/$1ELSE/; + s/^(\s+)\]/$1ENDIF/; + s/IF *:DEF:/ .ifdef/; + s/IF *:LNOT: *:DEF:/ .ifndef/; + s/ELSE/ .else/; + s/ENDIF/ .endif/; + + if( /\bIF\b/ ) { + s/\bIF\b/ .if/; + s/=/==/; + } + if ( $n == 2) { + s/\$/\\/g; + } + if ($n == 1) { + s/\$//g; + s/label//g; + $n = 2; + } + if ( /MACRO/ ) { + s/MACRO *\n/.macro/; + $n=1; + } + if ( /\bMEND\b/ ) { + s/\bMEND\b/.endm/; + $n=0; + } + + # ".rdata" doesn't work in 'as' version 2.13.2, as it is ".rodata" there. + # + if ( /\bAREA\b/ ) { + my $align; + $align = "2"; + if ( /ALIGN=(\d+)/ ) { + $align = $1; + } + if ( /CODE/ ) { + $nxstack = 1; + } + s/^(.+)CODE(.+)READONLY(.*)/ .text/; + s/^(.+)DATA(.+)READONLY(.*)/ .section .rdata/; + s/^(.+)\|\|\.data\|\|(.+)/ .data/; + s/^(.+)\|\|\.bss\|\|(.+)/ .bss/; + s/$/; .p2align $align/; + # Enable NEON instructions but don't produce a binary that requires + # ARMv7. RVCT does not have equivalent directives, so we just do this + # for all CODE areas. + if ( /.text/ ) { + # Separating .arch, .fpu, etc., by semicolons does not work (gas + # thinks the semicolon is part of the arch name, even when there's + # whitespace separating them). Sadly this means our line numbers + # won't match the original source file (we could use the .line + # directive, which is documented to be obsolete, but then gdb will + # show the wrong line in the translated source file). + s/$/; .arch armv7-a\n .fpu neon\n .object_arch armv4t/ unless ($apple); + } + } + + s/\|\|\.constdata\$(\d+)\|\|/.L_CONST$1/; # ||.constdata$3|| + s/\|\|\.bss\$(\d+)\|\|/.L_BSS$1/; # ||.bss$2|| + s/\|\|\.data\$(\d+)\|\|/.L_DATA$1/; # ||.data$2|| + s/\|\|([a-zA-Z0-9_]+)\@([a-zA-Z0-9_]+)\|\|/@ $&/; + s/^(\s+)\%(\s)/ .space $1/; + + s/\|(.+)\.(\d+)\|/\.$1_$2/; # |L80.123| -> .L80_123 + s/\bCODE32\b/.code 32/ && do {$thumb = 0}; + s/\bCODE16\b/.code 16/ && do {$thumb = 1}; + if (/\bPROC\b/) + { + my $prefix; + my $proc; + /^([A-Za-z_\.]\w+)\b/; + $proc = $1; + $prefix = ""; + if ($proc) + { + $prefix = $prefix.sprintf("\t.type\t%s, %%function", $proc) unless ($apple); + # Make sure we $prefix isn't empty here (for the $apple case). + # We handle mangling the label here, make sure it doesn't match + # the label handling below (if $prefix would be empty). + $prefix = $prefix."; "; + push(@proc_stack, $proc); + s/^[A-Za-z_\.]\w+/$symprefix$&:/; + } + $prefix = $prefix."\t.thumb_func; " if ($thumb); + s/\bPROC\b/@ $&/; + $_ = $prefix.$_; + } + s/^(\s*)(S|Q|SH|U|UQ|UH)ASX\b/$1$2ADDSUBX/; + s/^(\s*)(S|Q|SH|U|UQ|UH)SAX\b/$1$2SUBADDX/; + if (/\bENDP\b/) + { + my $proc; + s/\bENDP\b/@ $&/; + $proc = pop(@proc_stack); + $_ = "\t.size $proc, .-$proc".$_ if ($proc && !$apple); + } + s/\bSUBT\b/@ $&/; + s/\bDATA\b/@ $&/; # DATA directive is deprecated -- Asm guide, p.7-25 + s/\bKEEP\b/@ $&/; + s/\bEXPORTAS\b/@ $&/; + s/\|\|(.)+\bEQU\b/@ $&/; + s/\|\|([\w\$]+)\|\|/$1/; + s/\bENTRY\b/@ $&/; + s/\bASSERT\b/@ $&/; + s/\bGBLL\b/@ $&/; + s/\bGBLA\b/@ $&/; + s/^\W+OPT\b/@ $&/; + s/:OR:/|/g; + s/:SHL:/<>/g; + s/:AND:/&/g; + s/:LAND:/&&/g; + s/CPSR/cpsr/; + s/SPSR/spsr/; + s/ALIGN$/.balign 4/; + s/ALIGN\s+([0-9x]+)$/.balign $1/; + s/psr_cxsf/psr_all/; + s/LTORG/.ltorg/; + s/^([A-Za-z_]\w*)[ \t]+EQU/ .set $1,/; + s/^([A-Za-z_]\w*)[ \t]+SETL/ .set $1,/; + s/^([A-Za-z_]\w*)[ \t]+SETA/ .set $1,/; + s/^([A-Za-z_]\w*)[ \t]+\*/ .set $1,/; + + # {PC} + 0xdeadfeed --> . + 0xdeadfeed + s/\{PC\} \+/ \. +/; + + # Single hex constant on the line ! + # + # >>> NOTE <<< + # Double-precision floats in gcc are always mixed-endian, which means + # bytes in two words are little-endian, but words are big-endian. + # So, 0x0000deadfeed0000 would be stored as 0x0000dead at low address + # and 0xfeed0000 at high address. + # + s/\bDCFD\b[ \t]+0x([a-fA-F0-9]{8})([a-fA-F0-9]{8})/.long 0x$1, 0x$2/; + # Only decimal constants on the line, no hex ! + s/\bDCFD\b[ \t]+([0-9\.\-]+)/.double $1/; + + # Single hex constant on the line ! +# s/\bDCFS\b[ \t]+0x([a-f0-9]{8})([a-f0-9]{8})/.long 0x$1, 0x$2/; + # Only decimal constants on the line, no hex ! +# s/\bDCFS\b[ \t]+([0-9\.\-]+)/.double $1/; + s/\bDCFS[ \t]+0x/.word 0x/; + s/\bDCFS\b/.float/; + + s/^([A-Za-z_]\w*)[ \t]+DCD/$1 .word/; + s/\bDCD\b/.word/; + s/^([A-Za-z_]\w*)[ \t]+DCW/$1 .short/; + s/\bDCW\b/.short/; + s/^([A-Za-z_]\w*)[ \t]+DCB/$1 .byte/; + s/\bDCB\b/.byte/; + s/^([A-Za-z_]\w*)[ \t]+\%/.comm $1,/; + s/^[A-Za-z_\.]\w+/$&:/; + s/^(\d+)/$1:/; + s/\%(\d+)/$1b_or_f/; + s/\%[Bb](\d+)/$1b/; + s/\%[Ff](\d+)/$1f/; + s/\%[Ff][Tt](\d+)/$1f/; + s/&([\dA-Fa-f]+)/0x$1/; + if ( /\b2_[01]+\b/ ) { + s/\b2_([01]+)\b/conv$1&&&&/g; + while ( /[01][01][01][01]&&&&/ ) { + s/0000&&&&/&&&&0/g; + s/0001&&&&/&&&&1/g; + s/0010&&&&/&&&&2/g; + s/0011&&&&/&&&&3/g; + s/0100&&&&/&&&&4/g; + s/0101&&&&/&&&&5/g; + s/0110&&&&/&&&&6/g; + s/0111&&&&/&&&&7/g; + s/1000&&&&/&&&&8/g; + s/1001&&&&/&&&&9/g; + s/1010&&&&/&&&&A/g; + s/1011&&&&/&&&&B/g; + s/1100&&&&/&&&&C/g; + s/1101&&&&/&&&&D/g; + s/1110&&&&/&&&&E/g; + s/1111&&&&/&&&&F/g; + } + s/000&&&&/&&&&0/g; + s/001&&&&/&&&&1/g; + s/010&&&&/&&&&2/g; + s/011&&&&/&&&&3/g; + s/100&&&&/&&&&4/g; + s/101&&&&/&&&&5/g; + s/110&&&&/&&&&6/g; + s/111&&&&/&&&&7/g; + s/00&&&&/&&&&0/g; + s/01&&&&/&&&&1/g; + s/10&&&&/&&&&2/g; + s/11&&&&/&&&&3/g; + s/0&&&&/&&&&0/g; + s/1&&&&/&&&&1/g; + s/conv&&&&/0x/g; + } + + if ( /commandline/) + { + if( /-bigend/) + { + $bigend=1; + } + } + + if ( /\bDCDU\b/ ) + { + my $cmd=$_; + my $value; + my $prefix; + my $w1; + my $w2; + my $w3; + my $w4; + + s/\s+DCDU\b/@ $&/; + + $cmd =~ /\bDCDU\b\s+0x(\d+)/; + $value = $1; + $value =~ /(\w\w)(\w\w)(\w\w)(\w\w)/; + $w1 = $1; + $w2 = $2; + $w3 = $3; + $w4 = $4; + + if( $bigend ne "") + { + # big endian + $prefix = "\t.byte\t0x".$w1.";". + "\t.byte\t0x".$w2.";". + "\t.byte\t0x".$w3.";". + "\t.byte\t0x".$w4."; "; + } + else + { + # little endian + $prefix = "\t.byte\t0x".$w4.";". + "\t.byte\t0x".$w3.";". + "\t.byte\t0x".$w2.";". + "\t.byte\t0x".$w1."; "; + } + $_=$prefix.$_; + } + + if ( /\badrl\b/i ) + { + s/\badrl\s+(\w+)\s*,\s*(\w+)/ldr $1,=$2/i; + $addPadding = 1; + } + s/\bEND\b/@ END/; +} continue { + printf ("%s", $_) if $printit; + if ($addPadding != 0) + { + printf (" mov r0,r0\n"); + $addPadding = 0; + } +} +#If we had a code section, mark that this object doesn't need an executable +# stack. +if ($nxstack && !$apple) { + printf (" .section\t.note.GNU-stack,\"\",\%\%progbits\n"); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm_celt_map.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm_celt_map.c new file mode 100644 index 000000000..ca988b66f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/arm_celt_map.c @@ -0,0 +1,160 @@ +/* Copyright (c) 2010 Xiph.Org Foundation + * Copyright (c) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pitch.h" +#include "kiss_fft.h" +#include "mdct.h" + +#if defined(OPUS_HAVE_RTCD) + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR) +opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *x, const opus_val16 *y, int N) = { + celt_inner_prod_c, /* ARMv4 */ + celt_inner_prod_c, /* EDSP */ + celt_inner_prod_c, /* Media */ + celt_inner_prod_neon /* NEON */ +}; + +void (*const DUAL_INNER_PROD_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) = { + dual_inner_prod_c, /* ARMv4 */ + dual_inner_prod_c, /* EDSP */ + dual_inner_prod_c, /* Media */ + dual_inner_prod_neon /* NEON */ +}; +# endif + +# if defined(FIXED_POINT) +# if ((defined(OPUS_ARM_MAY_HAVE_NEON) && !defined(OPUS_ARM_PRESUME_NEON)) || \ + (defined(OPUS_ARM_MAY_HAVE_MEDIA) && !defined(OPUS_ARM_PRESUME_MEDIA)) || \ + (defined(OPUS_ARM_MAY_HAVE_EDSP) && !defined(OPUS_ARM_PRESUME_EDSP))) +opus_val32 (*const CELT_PITCH_XCORR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *, + const opus_val16 *, opus_val32 *, int, int, int) = { + celt_pitch_xcorr_c, /* ARMv4 */ + MAY_HAVE_EDSP(celt_pitch_xcorr), /* EDSP */ + MAY_HAVE_MEDIA(celt_pitch_xcorr), /* Media */ + MAY_HAVE_NEON(celt_pitch_xcorr) /* NEON */ +}; + +# endif +# else /* !FIXED_POINT */ +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR) +void (*const CELT_PITCH_XCORR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *, + const opus_val16 *, opus_val32 *, int, int, int) = { + celt_pitch_xcorr_c, /* ARMv4 */ + celt_pitch_xcorr_c, /* EDSP */ + celt_pitch_xcorr_c, /* Media */ + celt_pitch_xcorr_float_neon /* Neon */ +}; +# endif +# endif /* FIXED_POINT */ + +#if defined(FIXED_POINT) && defined(OPUS_HAVE_RTCD) && \ + defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR) + +void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])( + const opus_val16 *x, + const opus_val16 *y, + opus_val32 sum[4], + int len +) = { + xcorr_kernel_c, /* ARMv4 */ + xcorr_kernel_c, /* EDSP */ + xcorr_kernel_c, /* Media */ + xcorr_kernel_neon_fixed, /* Neon */ +}; + +#endif + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +# if defined(HAVE_ARM_NE10) +# if defined(CUSTOM_MODES) +int (*const OPUS_FFT_ALLOC_ARCH_IMPL[OPUS_ARCHMASK+1])(kiss_fft_state *st) = { + opus_fft_alloc_arch_c, /* ARMv4 */ + opus_fft_alloc_arch_c, /* EDSP */ + opus_fft_alloc_arch_c, /* Media */ + opus_fft_alloc_arm_neon /* Neon with NE10 library support */ +}; + +void (*const OPUS_FFT_FREE_ARCH_IMPL[OPUS_ARCHMASK+1])(kiss_fft_state *st) = { + opus_fft_free_arch_c, /* ARMv4 */ + opus_fft_free_arch_c, /* EDSP */ + opus_fft_free_arch_c, /* Media */ + opus_fft_free_arm_neon /* Neon with NE10 */ +}; +# endif /* CUSTOM_MODES */ + +void (*const OPUS_FFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout) = { + opus_fft_c, /* ARMv4 */ + opus_fft_c, /* EDSP */ + opus_fft_c, /* Media */ + opus_fft_neon /* Neon with NE10 */ +}; + +void (*const OPUS_IFFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout) = { + opus_ifft_c, /* ARMv4 */ + opus_ifft_c, /* EDSP */ + opus_ifft_c, /* Media */ + opus_ifft_neon /* Neon with NE10 */ +}; + +void (*const CLT_MDCT_FORWARD_IMPL[OPUS_ARCHMASK+1])(const mdct_lookup *l, + kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, + int overlap, int shift, + int stride, int arch) = { + clt_mdct_forward_c, /* ARMv4 */ + clt_mdct_forward_c, /* EDSP */ + clt_mdct_forward_c, /* Media */ + clt_mdct_forward_neon /* Neon with NE10 */ +}; + +void (*const CLT_MDCT_BACKWARD_IMPL[OPUS_ARCHMASK+1])(const mdct_lookup *l, + kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, + int overlap, int shift, + int stride, int arch) = { + clt_mdct_backward_c, /* ARMv4 */ + clt_mdct_backward_c, /* EDSP */ + clt_mdct_backward_c, /* Media */ + clt_mdct_backward_neon /* Neon with NE10 */ +}; + +# endif /* HAVE_ARM_NE10 */ +# endif /* OPUS_ARM_MAY_HAVE_NEON_INTR */ + +#endif /* OPUS_HAVE_RTCD */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.c new file mode 100644 index 000000000..cce3ae3a9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.c @@ -0,0 +1,187 @@ +/* Copyright (c) 2010 Xiph.Org Foundation + * Copyright (c) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Original code from libtheora modified to suit to Opus */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef OPUS_HAVE_RTCD + +#include "armcpu.h" +#include "cpu_support.h" +#include "os_support.h" +#include "opus_types.h" +#include "arch.h" + +#define OPUS_CPU_ARM_V4_FLAG (1< + +static OPUS_INLINE opus_uint32 opus_cpu_capabilities(void){ + opus_uint32 flags; + flags=0; + /* MSVC has no OPUS_INLINE __asm support for ARM, but it does let you __emit + * instructions via their assembled hex code. + * All of these instructions should be essentially nops. */ +# if defined(OPUS_ARM_MAY_HAVE_EDSP) || defined(OPUS_ARM_MAY_HAVE_MEDIA) \ + || defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + __try{ + /*PLD [r13]*/ + __emit(0xF5DDF000); + flags|=OPUS_CPU_ARM_EDSP_FLAG; + } + __except(GetExceptionCode()==EXCEPTION_ILLEGAL_INSTRUCTION){ + /*Ignore exception.*/ + } +# if defined(OPUS_ARM_MAY_HAVE_MEDIA) \ + || defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + __try{ + /*SHADD8 r3,r3,r3*/ + __emit(0xE6333F93); + flags|=OPUS_CPU_ARM_MEDIA_FLAG; + } + __except(GetExceptionCode()==EXCEPTION_ILLEGAL_INSTRUCTION){ + /*Ignore exception.*/ + } +# if defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + __try{ + /*VORR q0,q0,q0*/ + __emit(0xF2200150); + flags|=OPUS_CPU_ARM_NEON_FLAG; + } + __except(GetExceptionCode()==EXCEPTION_ILLEGAL_INSTRUCTION){ + /*Ignore exception.*/ + } +# endif +# endif +# endif + return flags; +} + +#elif defined(__linux__) +/* Linux based */ +#include + +opus_uint32 opus_cpu_capabilities(void) +{ + opus_uint32 flags = 0; + FILE *cpuinfo; + + /* Reading /proc/self/auxv would be easier, but that doesn't work reliably on + * Android */ + cpuinfo = fopen("/proc/cpuinfo", "r"); + + if(cpuinfo != NULL) + { + /* 512 should be enough for anybody (it's even enough for all the flags that + * x86 has accumulated... so far). */ + char buf[512]; + + while(fgets(buf, 512, cpuinfo) != NULL) + { +# if defined(OPUS_ARM_MAY_HAVE_EDSP) || defined(OPUS_ARM_MAY_HAVE_MEDIA) \ + || defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + /* Search for edsp and neon flag */ + if(memcmp(buf, "Features", 8) == 0) + { + char *p; + p = strstr(buf, " edsp"); + if(p != NULL && (p[5] == ' ' || p[5] == '\n')) + flags |= OPUS_CPU_ARM_EDSP_FLAG; + +# if defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + p = strstr(buf, " neon"); + if(p != NULL && (p[5] == ' ' || p[5] == '\n')) + flags |= OPUS_CPU_ARM_NEON_FLAG; +# endif + } +# endif + +# if defined(OPUS_ARM_MAY_HAVE_MEDIA) \ + || defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + /* Search for media capabilities (>= ARMv6) */ + if(memcmp(buf, "CPU architecture:", 17) == 0) + { + int version; + version = atoi(buf+17); + + if(version >= 6) + flags |= OPUS_CPU_ARM_MEDIA_FLAG; + } +# endif + } + + fclose(cpuinfo); + } + return flags; +} +#else +/* The feature registers which can tell us what the processor supports are + * accessible in priveleged modes only, so we can't have a general user-space + * detection method like on x86.*/ +# error "Configured to use ARM asm but no CPU detection method available for " \ + "your platform. Reconfigure with --disable-rtcd (or send patches)." +#endif + +int opus_select_arch(void) +{ + opus_uint32 flags = opus_cpu_capabilities(); + int arch = 0; + + if(!(flags & OPUS_CPU_ARM_EDSP_FLAG)) { + /* Asserts ensure arch values are sequential */ + celt_assert(arch == OPUS_ARCH_ARM_V4); + return arch; + } + arch++; + + if(!(flags & OPUS_CPU_ARM_MEDIA_FLAG)) { + celt_assert(arch == OPUS_ARCH_ARM_EDSP); + return arch; + } + arch++; + + if(!(flags & OPUS_CPU_ARM_NEON_FLAG)) { + celt_assert(arch == OPUS_ARCH_ARM_MEDIA); + return arch; + } + arch++; + + celt_assert(arch == OPUS_ARCH_ARM_NEON); + return arch; +} + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.h new file mode 100644 index 000000000..820262ff5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armcpu.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2010 Xiph.Org Foundation + * Copyright (c) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(ARMCPU_H) +# define ARMCPU_H + +# if defined(OPUS_ARM_MAY_HAVE_EDSP) +# define MAY_HAVE_EDSP(name) name ## _edsp +# else +# define MAY_HAVE_EDSP(name) name ## _c +# endif + +# if defined(OPUS_ARM_MAY_HAVE_MEDIA) +# define MAY_HAVE_MEDIA(name) name ## _media +# else +# define MAY_HAVE_MEDIA(name) MAY_HAVE_EDSP(name) +# endif + +# if defined(OPUS_ARM_MAY_HAVE_NEON) +# define MAY_HAVE_NEON(name) name ## _neon +# else +# define MAY_HAVE_NEON(name) MAY_HAVE_MEDIA(name) +# endif + +# if defined(OPUS_ARM_PRESUME_EDSP) +# define PRESUME_EDSP(name) name ## _edsp +# else +# define PRESUME_EDSP(name) name ## _c +# endif + +# if defined(OPUS_ARM_PRESUME_MEDIA) +# define PRESUME_MEDIA(name) name ## _media +# else +# define PRESUME_MEDIA(name) PRESUME_EDSP(name) +# endif + +# if defined(OPUS_ARM_PRESUME_NEON) +# define PRESUME_NEON(name) name ## _neon +# else +# define PRESUME_NEON(name) PRESUME_MEDIA(name) +# endif + +# if defined(OPUS_HAVE_RTCD) +int opus_select_arch(void); + +#define OPUS_ARCH_ARM_V4 (0) +#define OPUS_ARCH_ARM_EDSP (1) +#define OPUS_ARCH_ARM_MEDIA (2) +#define OPUS_ARCH_ARM_NEON (3) + +# endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armopts.s.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armopts.s.in new file mode 100644 index 000000000..3d8aaf275 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/armopts.s.in @@ -0,0 +1,37 @@ +/* Copyright (C) 2013 Mozilla Corporation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +; Set the following to 1 if we have EDSP instructions +; (LDRD/STRD, etc., ARMv5E and later). +OPUS_ARM_MAY_HAVE_EDSP * @OPUS_ARM_MAY_HAVE_EDSP@ + +; Set the following to 1 if we have ARMv6 media instructions. +OPUS_ARM_MAY_HAVE_MEDIA * @OPUS_ARM_MAY_HAVE_MEDIA@ + +; Set the following to 1 if we have NEON (some ARMv7) +OPUS_ARM_MAY_HAVE_NEON * @OPUS_ARM_MAY_HAVE_NEON@ + +END diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_fft_ne10.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_fft_ne10.c new file mode 100644 index 000000000..ea5fd7808 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_fft_ne10.c @@ -0,0 +1,173 @@ +/* Copyright (c) 2015 Xiph.Org Foundation + Written by Viswanath Puttagunta */ +/** + @file celt_fft_ne10.c + @brief ARM Neon optimizations for fft using NE10 library + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SKIP_CONFIG_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#endif + +#include +#include "os_support.h" +#include "kiss_fft.h" +#include "stack_alloc.h" + +#if !defined(FIXED_POINT) +# define NE10_FFT_ALLOC_C2C_TYPE_NEON ne10_fft_alloc_c2c_float32_neon +# define NE10_FFT_CFG_TYPE_T ne10_fft_cfg_float32_t +# define NE10_FFT_STATE_TYPE_T ne10_fft_state_float32_t +# define NE10_FFT_DESTROY_C2C_TYPE ne10_fft_destroy_c2c_float32 +# define NE10_FFT_CPX_TYPE_T ne10_fft_cpx_float32_t +# define NE10_FFT_C2C_1D_TYPE_NEON ne10_fft_c2c_1d_float32_neon +#else +# define NE10_FFT_ALLOC_C2C_TYPE_NEON(nfft) ne10_fft_alloc_c2c_int32_neon(nfft) +# define NE10_FFT_CFG_TYPE_T ne10_fft_cfg_int32_t +# define NE10_FFT_STATE_TYPE_T ne10_fft_state_int32_t +# define NE10_FFT_DESTROY_C2C_TYPE ne10_fft_destroy_c2c_int32 +# define NE10_FFT_DESTROY_C2C_TYPE ne10_fft_destroy_c2c_int32 +# define NE10_FFT_CPX_TYPE_T ne10_fft_cpx_int32_t +# define NE10_FFT_C2C_1D_TYPE_NEON ne10_fft_c2c_1d_int32_neon +#endif + +#if defined(CUSTOM_MODES) + +/* nfft lengths in NE10 that support scaled fft */ +# define NE10_FFTSCALED_SUPPORT_MAX 4 +static const int ne10_fft_scaled_support[NE10_FFTSCALED_SUPPORT_MAX] = { + 480, 240, 120, 60 +}; + +int opus_fft_alloc_arm_neon(kiss_fft_state *st) +{ + int i; + size_t memneeded = sizeof(struct arch_fft_state); + + st->arch_fft = (arch_fft_state *)opus_alloc(memneeded); + if (!st->arch_fft) + return -1; + + for (i = 0; i < NE10_FFTSCALED_SUPPORT_MAX; i++) { + if(st->nfft == ne10_fft_scaled_support[i]) + break; + } + if (i == NE10_FFTSCALED_SUPPORT_MAX) { + /* This nfft length (scaled fft) is not supported in NE10 */ + st->arch_fft->is_supported = 0; + st->arch_fft->priv = NULL; + } + else { + st->arch_fft->is_supported = 1; + st->arch_fft->priv = (void *)NE10_FFT_ALLOC_C2C_TYPE_NEON(st->nfft); + if (st->arch_fft->priv == NULL) { + return -1; + } + } + return 0; +} + +void opus_fft_free_arm_neon(kiss_fft_state *st) +{ + NE10_FFT_CFG_TYPE_T cfg; + + if (!st->arch_fft) + return; + + cfg = (NE10_FFT_CFG_TYPE_T)st->arch_fft->priv; + if (cfg) + NE10_FFT_DESTROY_C2C_TYPE(cfg); + opus_free(st->arch_fft); +} +#endif + +void opus_fft_neon(const kiss_fft_state *st, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout) +{ + NE10_FFT_STATE_TYPE_T state; + NE10_FFT_CFG_TYPE_T cfg = &state; + VARDECL(NE10_FFT_CPX_TYPE_T, buffer); + SAVE_STACK; + ALLOC(buffer, st->nfft, NE10_FFT_CPX_TYPE_T); + + if (!st->arch_fft->is_supported) { + /* This nfft length (scaled fft) not supported in NE10 */ + opus_fft_c(st, fin, fout); + } + else { + memcpy((void *)cfg, st->arch_fft->priv, sizeof(NE10_FFT_STATE_TYPE_T)); + state.buffer = (NE10_FFT_CPX_TYPE_T *)&buffer[0]; +#if !defined(FIXED_POINT) + state.is_forward_scaled = 1; + + NE10_FFT_C2C_1D_TYPE_NEON((NE10_FFT_CPX_TYPE_T *)fout, + (NE10_FFT_CPX_TYPE_T *)fin, + cfg, 0); +#else + NE10_FFT_C2C_1D_TYPE_NEON((NE10_FFT_CPX_TYPE_T *)fout, + (NE10_FFT_CPX_TYPE_T *)fin, + cfg, 0, 1); +#endif + } + RESTORE_STACK; +} + +void opus_ifft_neon(const kiss_fft_state *st, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout) +{ + NE10_FFT_STATE_TYPE_T state; + NE10_FFT_CFG_TYPE_T cfg = &state; + VARDECL(NE10_FFT_CPX_TYPE_T, buffer); + SAVE_STACK; + ALLOC(buffer, st->nfft, NE10_FFT_CPX_TYPE_T); + + if (!st->arch_fft->is_supported) { + /* This nfft length (scaled fft) not supported in NE10 */ + opus_ifft_c(st, fin, fout); + } + else { + memcpy((void *)cfg, st->arch_fft->priv, sizeof(NE10_FFT_STATE_TYPE_T)); + state.buffer = (NE10_FFT_CPX_TYPE_T *)&buffer[0]; +#if !defined(FIXED_POINT) + state.is_backward_scaled = 0; + + NE10_FFT_C2C_1D_TYPE_NEON((NE10_FFT_CPX_TYPE_T *)fout, + (NE10_FFT_CPX_TYPE_T *)fin, + cfg, 1); +#else + NE10_FFT_C2C_1D_TYPE_NEON((NE10_FFT_CPX_TYPE_T *)fout, + (NE10_FFT_CPX_TYPE_T *)fin, + cfg, 1, 0); +#endif + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_mdct_ne10.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_mdct_ne10.c new file mode 100644 index 000000000..3531d02d1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_mdct_ne10.c @@ -0,0 +1,258 @@ +/* Copyright (c) 2015 Xiph.Org Foundation + Written by Viswanath Puttagunta */ +/** + @file celt_mdct_ne10.c + @brief ARM Neon optimizations for mdct using NE10 library + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SKIP_CONFIG_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#endif + +#include "kiss_fft.h" +#include "_kiss_fft_guts.h" +#include "mdct.h" +#include "stack_alloc.h" + +void clt_mdct_forward_neon(const mdct_lookup *l, + kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, + int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + VARDECL(kiss_fft_scalar, f); + VARDECL(kiss_fft_cpx, f2); + const kiss_fft_state *st = l->kfft[shift]; + const kiss_twiddle_scalar *trig; + + SAVE_STACK; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + ALLOC(f, N2, kiss_fft_scalar); + ALLOC(f2, N4, kiss_fft_cpx); + + /* Consider the input to be composed of four blocks: [a, b, c, d] */ + /* Window, shuffle, fold */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in+(overlap>>1); + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+N2-1+(overlap>>1); + kiss_fft_scalar * OPUS_RESTRICT yp = f; + const opus_val16 * OPUS_RESTRICT wp1 = window+(overlap>>1); + const opus_val16 * OPUS_RESTRICT wp2 = window+(overlap>>1)-1; + for(i=0;i<((overlap+3)>>2);i++) + { + /* Real part arranged as -d-cR, Imag part arranged as -b+aR*/ + *yp++ = MULT16_32_Q15(*wp2, xp1[N2]) + MULT16_32_Q15(*wp1,*xp2); + *yp++ = MULT16_32_Q15(*wp1, *xp1) - MULT16_32_Q15(*wp2, xp2[-N2]); + xp1+=2; + xp2-=2; + wp1+=2; + wp2-=2; + } + wp1 = window; + wp2 = window+overlap-1; + for(;i>2);i++) + { + /* Real part arranged as a-bR, Imag part arranged as -c-dR */ + *yp++ = *xp2; + *yp++ = *xp1; + xp1+=2; + xp2-=2; + } + for(;ii,t[N4+i]) - S_MUL(fp->r,t[i]); + yi = S_MUL(fp->r,t[N4+i]) + S_MUL(fp->i,t[i]); + *yp1 = yr; + *yp2 = yi; + fp++; + yp1 += 2*stride; + yp2 -= 2*stride; + } + } + RESTORE_STACK; +} + +void clt_mdct_backward_neon(const mdct_lookup *l, + kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 * OPUS_RESTRICT window, + int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + VARDECL(kiss_fft_scalar, f); + const kiss_twiddle_scalar *trig; + const kiss_fft_state *st = l->kfft[shift]; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + ALLOC(f, N2, kiss_fft_scalar); + + /* Pre-rotate */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in; + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+stride*(N2-1); + kiss_fft_scalar * OPUS_RESTRICT yp = f; + const kiss_twiddle_scalar * OPUS_RESTRICT t = &trig[0]; + for(i=0;i>1)), arch); + + /* Post-rotate and de-shuffle from both ends of the buffer at once to make + it in-place. */ + { + kiss_fft_scalar * yp0 = out+(overlap>>1); + kiss_fft_scalar * yp1 = out+(overlap>>1)+N2-2; + const kiss_twiddle_scalar *t = &trig[0]; + /* Loop to (N4+1)>>1 to handle odd N4. When N4 is odd, the + middle pair will be computed twice. */ + for(i=0;i<(N4+1)>>1;i++) + { + kiss_fft_scalar re, im, yr, yi; + kiss_twiddle_scalar t0, t1; + re = yp0[0]; + im = yp0[1]; + t0 = t[i]; + t1 = t[N4+i]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = S_MUL(re,t0) + S_MUL(im,t1); + yi = S_MUL(re,t1) - S_MUL(im,t0); + re = yp1[0]; + im = yp1[1]; + yp0[0] = yr; + yp1[1] = yi; + + t0 = t[(N4-i-1)]; + t1 = t[(N2-i-1)]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = S_MUL(re,t0) + S_MUL(im,t1); + yi = S_MUL(re,t1) - S_MUL(im,t0); + yp1[0] = yr; + yp0[1] = yi; + yp0 += 2; + yp1 -= 2; + } + } + + /* Mirror on both sides for TDAC */ + { + kiss_fft_scalar * OPUS_RESTRICT xp1 = out+overlap-1; + kiss_fft_scalar * OPUS_RESTRICT yp1 = out; + const opus_val16 * OPUS_RESTRICT wp1 = window; + const opus_val16 * OPUS_RESTRICT wp2 = window+overlap-1; + + for(i = 0; i < overlap/2; i++) + { + kiss_fft_scalar x1, x2; + x1 = *xp1; + x2 = *yp1; + *yp1++ = MULT16_32_Q15(*wp2, x2) - MULT16_32_Q15(*wp1, x1); + *xp1-- = MULT16_32_Q15(*wp1, x2) + MULT16_32_Q15(*wp2, x1); + wp1++; + wp2--; + } + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_neon_intr.c new file mode 100644 index 000000000..effda769d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_neon_intr.c @@ -0,0 +1,211 @@ +/* Copyright (c) 2014-2015 Xiph.Org Foundation + Written by Viswanath Puttagunta */ +/** + @file celt_neon_intr.c + @brief ARM Neon Intrinsic optimizations for celt + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "../pitch.h" + +#if defined(FIXED_POINT) +void xcorr_kernel_neon_fixed(const opus_val16 * x, const opus_val16 * y, opus_val32 sum[4], int len) +{ + int j; + int32x4_t a = vld1q_s32(sum); + /* Load y[0...3] */ + /* This requires len>0 to always be valid (which we assert in the C code). */ + int16x4_t y0 = vld1_s16(y); + y += 4; + + for (j = 0; j + 8 <= len; j += 8) + { + /* Load x[0...7] */ + int16x8_t xx = vld1q_s16(x); + int16x4_t x0 = vget_low_s16(xx); + int16x4_t x4 = vget_high_s16(xx); + /* Load y[4...11] */ + int16x8_t yy = vld1q_s16(y); + int16x4_t y4 = vget_low_s16(yy); + int16x4_t y8 = vget_high_s16(yy); + int32x4_t a0 = vmlal_lane_s16(a, y0, x0, 0); + int32x4_t a1 = vmlal_lane_s16(a0, y4, x4, 0); + + int16x4_t y1 = vext_s16(y0, y4, 1); + int16x4_t y5 = vext_s16(y4, y8, 1); + int32x4_t a2 = vmlal_lane_s16(a1, y1, x0, 1); + int32x4_t a3 = vmlal_lane_s16(a2, y5, x4, 1); + + int16x4_t y2 = vext_s16(y0, y4, 2); + int16x4_t y6 = vext_s16(y4, y8, 2); + int32x4_t a4 = vmlal_lane_s16(a3, y2, x0, 2); + int32x4_t a5 = vmlal_lane_s16(a4, y6, x4, 2); + + int16x4_t y3 = vext_s16(y0, y4, 3); + int16x4_t y7 = vext_s16(y4, y8, 3); + int32x4_t a6 = vmlal_lane_s16(a5, y3, x0, 3); + int32x4_t a7 = vmlal_lane_s16(a6, y7, x4, 3); + + y0 = y8; + a = a7; + x += 8; + y += 8; + } + + for (; j < len; j++) + { + int16x4_t x0 = vld1_dup_s16(x); /* load next x */ + int32x4_t a0 = vmlal_s16(a, y0, x0); + + int16x4_t y4 = vld1_dup_s16(y); /* load next y */ + y0 = vext_s16(y0, y4, 1); + a = a0; + x++; + y++; + } + + vst1q_s32(sum, a); +} + +#else +/* + * Function: xcorr_kernel_neon_float + * --------------------------------- + * Computes 4 correlation values and stores them in sum[4] + */ +static void xcorr_kernel_neon_float(const float32_t *x, const float32_t *y, + float32_t sum[4], int len) { + float32x4_t YY[3]; + float32x4_t YEXT[3]; + float32x4_t XX[2]; + float32x2_t XX_2; + float32x4_t SUMM; + const float32_t *xi = x; + const float32_t *yi = y; + + celt_assert(len>0); + + YY[0] = vld1q_f32(yi); + SUMM = vdupq_n_f32(0); + + /* Consume 8 elements in x vector and 12 elements in y + * vector. However, the 12'th element never really gets + * touched in this loop. So, if len == 8, then we only + * must access y[0] to y[10]. y[11] must not be accessed + * hence make sure len > 8 and not len >= 8 + */ + while (len > 8) { + yi += 4; + YY[1] = vld1q_f32(yi); + yi += 4; + YY[2] = vld1q_f32(yi); + + XX[0] = vld1q_f32(xi); + xi += 4; + XX[1] = vld1q_f32(xi); + xi += 4; + + SUMM = vmlaq_lane_f32(SUMM, YY[0], vget_low_f32(XX[0]), 0); + YEXT[0] = vextq_f32(YY[0], YY[1], 1); + SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[0]), 1); + YEXT[1] = vextq_f32(YY[0], YY[1], 2); + SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[0]), 0); + YEXT[2] = vextq_f32(YY[0], YY[1], 3); + SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[0]), 1); + + SUMM = vmlaq_lane_f32(SUMM, YY[1], vget_low_f32(XX[1]), 0); + YEXT[0] = vextq_f32(YY[1], YY[2], 1); + SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[1]), 1); + YEXT[1] = vextq_f32(YY[1], YY[2], 2); + SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[1]), 0); + YEXT[2] = vextq_f32(YY[1], YY[2], 3); + SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[1]), 1); + + YY[0] = YY[2]; + len -= 8; + } + + /* Consume 4 elements in x vector and 8 elements in y + * vector. However, the 8'th element in y never really gets + * touched in this loop. So, if len == 4, then we only + * must access y[0] to y[6]. y[7] must not be accessed + * hence make sure len>4 and not len>=4 + */ + if (len > 4) { + yi += 4; + YY[1] = vld1q_f32(yi); + + XX[0] = vld1q_f32(xi); + xi += 4; + + SUMM = vmlaq_lane_f32(SUMM, YY[0], vget_low_f32(XX[0]), 0); + YEXT[0] = vextq_f32(YY[0], YY[1], 1); + SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[0]), 1); + YEXT[1] = vextq_f32(YY[0], YY[1], 2); + SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[0]), 0); + YEXT[2] = vextq_f32(YY[0], YY[1], 3); + SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[0]), 1); + + YY[0] = YY[1]; + len -= 4; + } + + while (--len > 0) { + XX_2 = vld1_dup_f32(xi++); + SUMM = vmlaq_lane_f32(SUMM, YY[0], XX_2, 0); + YY[0]= vld1q_f32(++yi); + } + + XX_2 = vld1_dup_f32(xi); + SUMM = vmlaq_lane_f32(SUMM, YY[0], XX_2, 0); + + vst1q_f32(sum, SUMM); +} + +void celt_pitch_xcorr_float_neon(const opus_val16 *_x, const opus_val16 *_y, + opus_val32 *xcorr, int len, int max_pitch, int arch) { + int i; + (void)arch; + celt_assert(max_pitch > 0); + celt_sig_assert((((unsigned char *)_x-(unsigned char *)NULL)&3)==0); + + for (i = 0; i < (max_pitch-3); i += 4) { + xcorr_kernel_neon_float((const float32_t *)_x, (const float32_t *)_y+i, + (float32_t *)xcorr+i, len); + } + + /* In case max_pitch isn't a multiple of 4, do non-unrolled version. */ + for (; i < max_pitch; i++) { + xcorr[i] = celt_inner_prod_neon(_x, _y+i, len); + } +} +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_pitch_xcorr_arm.s b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_pitch_xcorr_arm.s new file mode 100644 index 000000000..6e873afc3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/celt_pitch_xcorr_arm.s @@ -0,0 +1,551 @@ +; Copyright (c) 2007-2008 CSIRO +; Copyright (c) 2007-2009 Xiph.Org Foundation +; Copyright (c) 2013 Parrot +; Written by Aurélien Zanelli +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; +; - Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; +; - Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in the +; documentation and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +; OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + AREA |.text|, CODE, READONLY + + GET celt/arm/armopts.s + +IF OPUS_ARM_MAY_HAVE_EDSP + EXPORT celt_pitch_xcorr_edsp +ENDIF + +IF OPUS_ARM_MAY_HAVE_NEON + EXPORT celt_pitch_xcorr_neon +ENDIF + +IF OPUS_ARM_MAY_HAVE_NEON + +; Compute sum[k]=sum(x[j]*y[j+k],j=0...len-1), k=0...3 +xcorr_kernel_neon PROC +xcorr_kernel_neon_start + ; input: + ; r3 = int len + ; r4 = opus_val16 *x + ; r5 = opus_val16 *y + ; q0 = opus_val32 sum[4] + ; output: + ; q0 = opus_val32 sum[4] + ; preserved: r0-r3, r6-r11, d2, q4-q7, q9-q15 + ; internal usage: + ; r12 = int j + ; d3 = y_3|y_2|y_1|y_0 + ; q2 = y_B|y_A|y_9|y_8|y_7|y_6|y_5|y_4 + ; q3 = x_7|x_6|x_5|x_4|x_3|x_2|x_1|x_0 + ; q8 = scratch + ; + ; Load y[0...3] + ; This requires len>0 to always be valid (which we assert in the C code). + VLD1.16 {d5}, [r5]! + SUBS r12, r3, #8 + BLE xcorr_kernel_neon_process4 +; Process 8 samples at a time. +; This loop loads one y value more than we actually need. Therefore we have to +; stop as soon as there are 8 or fewer samples left (instead of 7), to avoid +; reading past the end of the array. +xcorr_kernel_neon_process8 + ; This loop has 19 total instructions (10 cycles to issue, minimum), with + ; - 2 cycles of ARM insrtuctions, + ; - 10 cycles of load/store/byte permute instructions, and + ; - 9 cycles of data processing instructions. + ; On a Cortex A8, we dual-issue the maximum amount (9 cycles) between the + ; latter two categories, meaning the whole loop should run in 10 cycles per + ; iteration, barring cache misses. + ; + ; Load x[0...7] + VLD1.16 {d6, d7}, [r4]! + ; Unlike VMOV, VAND is a data processsing instruction (and doesn't get + ; assembled to VMOV, like VORR would), so it dual-issues with the prior VLD1. + VAND d3, d5, d5 + SUBS r12, r12, #8 + ; Load y[4...11] + VLD1.16 {d4, d5}, [r5]! + VMLAL.S16 q0, d3, d6[0] + VEXT.16 d16, d3, d4, #1 + VMLAL.S16 q0, d4, d7[0] + VEXT.16 d17, d4, d5, #1 + VMLAL.S16 q0, d16, d6[1] + VEXT.16 d16, d3, d4, #2 + VMLAL.S16 q0, d17, d7[1] + VEXT.16 d17, d4, d5, #2 + VMLAL.S16 q0, d16, d6[2] + VEXT.16 d16, d3, d4, #3 + VMLAL.S16 q0, d17, d7[2] + VEXT.16 d17, d4, d5, #3 + VMLAL.S16 q0, d16, d6[3] + VMLAL.S16 q0, d17, d7[3] + BGT xcorr_kernel_neon_process8 +; Process 4 samples here if we have > 4 left (still reading one extra y value). +xcorr_kernel_neon_process4 + ADDS r12, r12, #4 + BLE xcorr_kernel_neon_process2 + ; Load x[0...3] + VLD1.16 d6, [r4]! + ; Use VAND since it's a data processing instruction again. + VAND d4, d5, d5 + SUB r12, r12, #4 + ; Load y[4...7] + VLD1.16 d5, [r5]! + VMLAL.S16 q0, d4, d6[0] + VEXT.16 d16, d4, d5, #1 + VMLAL.S16 q0, d16, d6[1] + VEXT.16 d16, d4, d5, #2 + VMLAL.S16 q0, d16, d6[2] + VEXT.16 d16, d4, d5, #3 + VMLAL.S16 q0, d16, d6[3] +; Process 2 samples here if we have > 2 left (still reading one extra y value). +xcorr_kernel_neon_process2 + ADDS r12, r12, #2 + BLE xcorr_kernel_neon_process1 + ; Load x[0...1] + VLD2.16 {d6[],d7[]}, [r4]! + ; Use VAND since it's a data processing instruction again. + VAND d4, d5, d5 + SUB r12, r12, #2 + ; Load y[4...5] + VLD1.32 {d5[]}, [r5]! + VMLAL.S16 q0, d4, d6 + VEXT.16 d16, d4, d5, #1 + ; Replace bottom copy of {y5,y4} in d5 with {y3,y2} from d4, using VSRI + ; instead of VEXT, since it's a data-processing instruction. + VSRI.64 d5, d4, #32 + VMLAL.S16 q0, d16, d7 +; Process 1 sample using the extra y value we loaded above. +xcorr_kernel_neon_process1 + ; Load next *x + VLD1.16 {d6[]}, [r4]! + ADDS r12, r12, #1 + ; y[0...3] are left in d5 from prior iteration(s) (if any) + VMLAL.S16 q0, d5, d6 + MOVLE pc, lr +; Now process 1 last sample, not reading ahead. + ; Load last *y + VLD1.16 {d4[]}, [r5]! + VSRI.64 d4, d5, #16 + ; Load last *x + VLD1.16 {d6[]}, [r4]! + VMLAL.S16 q0, d4, d6 + MOV pc, lr + ENDP + +; opus_val32 celt_pitch_xcorr_neon(opus_val16 *_x, opus_val16 *_y, +; opus_val32 *xcorr, int len, int max_pitch, int arch) +celt_pitch_xcorr_neon PROC + ; input: + ; r0 = opus_val16 *_x + ; r1 = opus_val16 *_y + ; r2 = opus_val32 *xcorr + ; r3 = int len + ; output: + ; r0 = int maxcorr + ; internal usage: + ; r4 = opus_val16 *x (for xcorr_kernel_neon()) + ; r5 = opus_val16 *y (for xcorr_kernel_neon()) + ; r6 = int max_pitch + ; r12 = int j + ; q15 = int maxcorr[4] (q15 is not used by xcorr_kernel_neon()) + ; ignored: + ; int arch + STMFD sp!, {r4-r6, lr} + LDR r6, [sp, #16] + VMOV.S32 q15, #1 + ; if (max_pitch < 4) goto celt_pitch_xcorr_neon_process4_done + SUBS r6, r6, #4 + BLT celt_pitch_xcorr_neon_process4_done +celt_pitch_xcorr_neon_process4 + ; xcorr_kernel_neon parameters: + ; r3 = len, r4 = _x, r5 = _y, q0 = {0, 0, 0, 0} + MOV r4, r0 + MOV r5, r1 + VEOR q0, q0, q0 + ; xcorr_kernel_neon only modifies r4, r5, r12, and q0...q3. + ; So we don't save/restore any other registers. + BL xcorr_kernel_neon_start + SUBS r6, r6, #4 + VST1.32 {q0}, [r2]! + ; _y += 4 + ADD r1, r1, #8 + VMAX.S32 q15, q15, q0 + ; if (max_pitch < 4) goto celt_pitch_xcorr_neon_process4_done + BGE celt_pitch_xcorr_neon_process4 +; We have less than 4 sums left to compute. +celt_pitch_xcorr_neon_process4_done + ADDS r6, r6, #4 + ; Reduce maxcorr to a single value + VMAX.S32 d30, d30, d31 + VPMAX.S32 d30, d30, d30 + ; if (max_pitch <= 0) goto celt_pitch_xcorr_neon_done + BLE celt_pitch_xcorr_neon_done +; Now compute each remaining sum one at a time. +celt_pitch_xcorr_neon_process_remaining + MOV r4, r0 + MOV r5, r1 + VMOV.I32 q0, #0 + SUBS r12, r3, #8 + BLT celt_pitch_xcorr_neon_process_remaining4 +; Sum terms 8 at a time. +celt_pitch_xcorr_neon_process_remaining_loop8 + ; Load x[0...7] + VLD1.16 {q1}, [r4]! + ; Load y[0...7] + VLD1.16 {q2}, [r5]! + SUBS r12, r12, #8 + VMLAL.S16 q0, d4, d2 + VMLAL.S16 q0, d5, d3 + BGE celt_pitch_xcorr_neon_process_remaining_loop8 +; Sum terms 4 at a time. +celt_pitch_xcorr_neon_process_remaining4 + ADDS r12, r12, #4 + BLT celt_pitch_xcorr_neon_process_remaining4_done + ; Load x[0...3] + VLD1.16 {d2}, [r4]! + ; Load y[0...3] + VLD1.16 {d3}, [r5]! + SUB r12, r12, #4 + VMLAL.S16 q0, d3, d2 +celt_pitch_xcorr_neon_process_remaining4_done + ; Reduce the sum to a single value. + VADD.S32 d0, d0, d1 + VPADDL.S32 d0, d0 + ADDS r12, r12, #4 + BLE celt_pitch_xcorr_neon_process_remaining_loop_done +; Sum terms 1 at a time. +celt_pitch_xcorr_neon_process_remaining_loop1 + VLD1.16 {d2[]}, [r4]! + VLD1.16 {d3[]}, [r5]! + SUBS r12, r12, #1 + VMLAL.S16 q0, d2, d3 + BGT celt_pitch_xcorr_neon_process_remaining_loop1 +celt_pitch_xcorr_neon_process_remaining_loop_done + VST1.32 {d0[0]}, [r2]! + VMAX.S32 d30, d30, d0 + SUBS r6, r6, #1 + ; _y++ + ADD r1, r1, #2 + ; if (--max_pitch > 0) goto celt_pitch_xcorr_neon_process_remaining + BGT celt_pitch_xcorr_neon_process_remaining +celt_pitch_xcorr_neon_done + VMOV.32 r0, d30[0] + LDMFD sp!, {r4-r6, pc} + ENDP + +ENDIF + +IF OPUS_ARM_MAY_HAVE_EDSP + +; This will get used on ARMv7 devices without NEON, so it has been optimized +; to take advantage of dual-issuing where possible. +xcorr_kernel_edsp PROC +xcorr_kernel_edsp_start + ; input: + ; r3 = int len + ; r4 = opus_val16 *_x (must be 32-bit aligned) + ; r5 = opus_val16 *_y (must be 32-bit aligned) + ; r6...r9 = opus_val32 sum[4] + ; output: + ; r6...r9 = opus_val32 sum[4] + ; preserved: r0-r5 + ; internal usage + ; r2 = int j + ; r12,r14 = opus_val16 x[4] + ; r10,r11 = opus_val16 y[4] + STMFD sp!, {r2,r4,r5,lr} + LDR r10, [r5], #4 ; Load y[0...1] + SUBS r2, r3, #4 ; j = len-4 + LDR r11, [r5], #4 ; Load y[2...3] + BLE xcorr_kernel_edsp_process4_done + LDR r12, [r4], #4 ; Load x[0...1] + ; Stall +xcorr_kernel_edsp_process4 + ; The multiplies must issue from pipeline 0, and can't dual-issue with each + ; other. Every other instruction here dual-issues with a multiply, and is + ; thus "free". There should be no stalls in the body of the loop. + SMLABB r6, r12, r10, r6 ; sum[0] = MAC16_16(sum[0],x_0,y_0) + LDR r14, [r4], #4 ; Load x[2...3] + SMLABT r7, r12, r10, r7 ; sum[1] = MAC16_16(sum[1],x_0,y_1) + SUBS r2, r2, #4 ; j-=4 + SMLABB r8, r12, r11, r8 ; sum[2] = MAC16_16(sum[2],x_0,y_2) + SMLABT r9, r12, r11, r9 ; sum[3] = MAC16_16(sum[3],x_0,y_3) + SMLATT r6, r12, r10, r6 ; sum[0] = MAC16_16(sum[0],x_1,y_1) + LDR r10, [r5], #4 ; Load y[4...5] + SMLATB r7, r12, r11, r7 ; sum[1] = MAC16_16(sum[1],x_1,y_2) + SMLATT r8, r12, r11, r8 ; sum[2] = MAC16_16(sum[2],x_1,y_3) + SMLATB r9, r12, r10, r9 ; sum[3] = MAC16_16(sum[3],x_1,y_4) + LDRGT r12, [r4], #4 ; Load x[0...1] + SMLABB r6, r14, r11, r6 ; sum[0] = MAC16_16(sum[0],x_2,y_2) + SMLABT r7, r14, r11, r7 ; sum[1] = MAC16_16(sum[1],x_2,y_3) + SMLABB r8, r14, r10, r8 ; sum[2] = MAC16_16(sum[2],x_2,y_4) + SMLABT r9, r14, r10, r9 ; sum[3] = MAC16_16(sum[3],x_2,y_5) + SMLATT r6, r14, r11, r6 ; sum[0] = MAC16_16(sum[0],x_3,y_3) + LDR r11, [r5], #4 ; Load y[6...7] + SMLATB r7, r14, r10, r7 ; sum[1] = MAC16_16(sum[1],x_3,y_4) + SMLATT r8, r14, r10, r8 ; sum[2] = MAC16_16(sum[2],x_3,y_5) + SMLATB r9, r14, r11, r9 ; sum[3] = MAC16_16(sum[3],x_3,y_6) + BGT xcorr_kernel_edsp_process4 +xcorr_kernel_edsp_process4_done + ADDS r2, r2, #4 + BLE xcorr_kernel_edsp_done + LDRH r12, [r4], #2 ; r12 = *x++ + SUBS r2, r2, #1 ; j-- + ; Stall + SMLABB r6, r12, r10, r6 ; sum[0] = MAC16_16(sum[0],x,y_0) + LDRHGT r14, [r4], #2 ; r14 = *x++ + SMLABT r7, r12, r10, r7 ; sum[1] = MAC16_16(sum[1],x,y_1) + SMLABB r8, r12, r11, r8 ; sum[2] = MAC16_16(sum[2],x,y_2) + SMLABT r9, r12, r11, r9 ; sum[3] = MAC16_16(sum[3],x,y_3) + BLE xcorr_kernel_edsp_done + SMLABT r6, r14, r10, r6 ; sum[0] = MAC16_16(sum[0],x,y_1) + SUBS r2, r2, #1 ; j-- + SMLABB r7, r14, r11, r7 ; sum[1] = MAC16_16(sum[1],x,y_2) + LDRH r10, [r5], #2 ; r10 = y_4 = *y++ + SMLABT r8, r14, r11, r8 ; sum[2] = MAC16_16(sum[2],x,y_3) + LDRHGT r12, [r4], #2 ; r12 = *x++ + SMLABB r9, r14, r10, r9 ; sum[3] = MAC16_16(sum[3],x,y_4) + BLE xcorr_kernel_edsp_done + SMLABB r6, r12, r11, r6 ; sum[0] = MAC16_16(sum[0],tmp,y_2) + CMP r2, #1 ; j-- + SMLABT r7, r12, r11, r7 ; sum[1] = MAC16_16(sum[1],tmp,y_3) + LDRH r2, [r5], #2 ; r2 = y_5 = *y++ + SMLABB r8, r12, r10, r8 ; sum[2] = MAC16_16(sum[2],tmp,y_4) + LDRHGT r14, [r4] ; r14 = *x + SMLABB r9, r12, r2, r9 ; sum[3] = MAC16_16(sum[3],tmp,y_5) + BLE xcorr_kernel_edsp_done + SMLABT r6, r14, r11, r6 ; sum[0] = MAC16_16(sum[0],tmp,y_3) + LDRH r11, [r5] ; r11 = y_6 = *y + SMLABB r7, r14, r10, r7 ; sum[1] = MAC16_16(sum[1],tmp,y_4) + SMLABB r8, r14, r2, r8 ; sum[2] = MAC16_16(sum[2],tmp,y_5) + SMLABB r9, r14, r11, r9 ; sum[3] = MAC16_16(sum[3],tmp,y_6) +xcorr_kernel_edsp_done + LDMFD sp!, {r2,r4,r5,pc} + ENDP + +celt_pitch_xcorr_edsp PROC + ; input: + ; r0 = opus_val16 *_x (must be 32-bit aligned) + ; r1 = opus_val16 *_y (only needs to be 16-bit aligned) + ; r2 = opus_val32 *xcorr + ; r3 = int len + ; output: + ; r0 = maxcorr + ; internal usage + ; r4 = opus_val16 *x + ; r5 = opus_val16 *y + ; r6 = opus_val32 sum0 + ; r7 = opus_val32 sum1 + ; r8 = opus_val32 sum2 + ; r9 = opus_val32 sum3 + ; r1 = int max_pitch + ; r12 = int j + ; ignored: + ; int arch + STMFD sp!, {r4-r11, lr} + MOV r5, r1 + LDR r1, [sp, #36] + MOV r4, r0 + TST r5, #3 + ; maxcorr = 1 + MOV r0, #1 + BEQ celt_pitch_xcorr_edsp_process1u_done +; Compute one sum at the start to make y 32-bit aligned. + SUBS r12, r3, #4 + ; r14 = sum = 0 + MOV r14, #0 + LDRH r8, [r5], #2 + BLE celt_pitch_xcorr_edsp_process1u_loop4_done + LDR r6, [r4], #4 + MOV r8, r8, LSL #16 +celt_pitch_xcorr_edsp_process1u_loop4 + LDR r9, [r5], #4 + SMLABT r14, r6, r8, r14 ; sum = MAC16_16(sum, x_0, y_0) + LDR r7, [r4], #4 + SMLATB r14, r6, r9, r14 ; sum = MAC16_16(sum, x_1, y_1) + LDR r8, [r5], #4 + SMLABT r14, r7, r9, r14 ; sum = MAC16_16(sum, x_2, y_2) + SUBS r12, r12, #4 ; j-=4 + SMLATB r14, r7, r8, r14 ; sum = MAC16_16(sum, x_3, y_3) + LDRGT r6, [r4], #4 + BGT celt_pitch_xcorr_edsp_process1u_loop4 + MOV r8, r8, LSR #16 +celt_pitch_xcorr_edsp_process1u_loop4_done + ADDS r12, r12, #4 +celt_pitch_xcorr_edsp_process1u_loop1 + LDRHGE r6, [r4], #2 + ; Stall + SMLABBGE r14, r6, r8, r14 ; sum = MAC16_16(sum, *x, *y) + SUBSGE r12, r12, #1 + LDRHGT r8, [r5], #2 + BGT celt_pitch_xcorr_edsp_process1u_loop1 + ; Restore _x + SUB r4, r4, r3, LSL #1 + ; Restore and advance _y + SUB r5, r5, r3, LSL #1 + ; maxcorr = max(maxcorr, sum) + CMP r0, r14 + ADD r5, r5, #2 + MOVLT r0, r14 + SUBS r1, r1, #1 + ; xcorr[i] = sum + STR r14, [r2], #4 + BLE celt_pitch_xcorr_edsp_done +celt_pitch_xcorr_edsp_process1u_done + ; if (max_pitch < 4) goto celt_pitch_xcorr_edsp_process2 + SUBS r1, r1, #4 + BLT celt_pitch_xcorr_edsp_process2 +celt_pitch_xcorr_edsp_process4 + ; xcorr_kernel_edsp parameters: + ; r3 = len, r4 = _x, r5 = _y, r6...r9 = sum[4] = {0, 0, 0, 0} + MOV r6, #0 + MOV r7, #0 + MOV r8, #0 + MOV r9, #0 + BL xcorr_kernel_edsp_start ; xcorr_kernel_edsp(_x, _y+i, xcorr+i, len) + ; maxcorr = max(maxcorr, sum0, sum1, sum2, sum3) + CMP r0, r6 + ; _y+=4 + ADD r5, r5, #8 + MOVLT r0, r6 + CMP r0, r7 + MOVLT r0, r7 + CMP r0, r8 + MOVLT r0, r8 + CMP r0, r9 + MOVLT r0, r9 + STMIA r2!, {r6-r9} + SUBS r1, r1, #4 + BGE celt_pitch_xcorr_edsp_process4 +celt_pitch_xcorr_edsp_process2 + ADDS r1, r1, #2 + BLT celt_pitch_xcorr_edsp_process1a + SUBS r12, r3, #4 + ; {r10, r11} = {sum0, sum1} = {0, 0} + MOV r10, #0 + MOV r11, #0 + LDR r8, [r5], #4 + BLE celt_pitch_xcorr_edsp_process2_loop_done + LDR r6, [r4], #4 + LDR r9, [r5], #4 +celt_pitch_xcorr_edsp_process2_loop4 + SMLABB r10, r6, r8, r10 ; sum0 = MAC16_16(sum0, x_0, y_0) + LDR r7, [r4], #4 + SMLABT r11, r6, r8, r11 ; sum1 = MAC16_16(sum1, x_0, y_1) + SUBS r12, r12, #4 ; j-=4 + SMLATT r10, r6, r8, r10 ; sum0 = MAC16_16(sum0, x_1, y_1) + LDR r8, [r5], #4 + SMLATB r11, r6, r9, r11 ; sum1 = MAC16_16(sum1, x_1, y_2) + LDRGT r6, [r4], #4 + SMLABB r10, r7, r9, r10 ; sum0 = MAC16_16(sum0, x_2, y_2) + SMLABT r11, r7, r9, r11 ; sum1 = MAC16_16(sum1, x_2, y_3) + SMLATT r10, r7, r9, r10 ; sum0 = MAC16_16(sum0, x_3, y_3) + LDRGT r9, [r5], #4 + SMLATB r11, r7, r8, r11 ; sum1 = MAC16_16(sum1, x_3, y_4) + BGT celt_pitch_xcorr_edsp_process2_loop4 +celt_pitch_xcorr_edsp_process2_loop_done + ADDS r12, r12, #2 + BLE celt_pitch_xcorr_edsp_process2_1 + LDR r6, [r4], #4 + ; Stall + SMLABB r10, r6, r8, r10 ; sum0 = MAC16_16(sum0, x_0, y_0) + LDR r9, [r5], #4 + SMLABT r11, r6, r8, r11 ; sum1 = MAC16_16(sum1, x_0, y_1) + SUB r12, r12, #2 + SMLATT r10, r6, r8, r10 ; sum0 = MAC16_16(sum0, x_1, y_1) + MOV r8, r9 + SMLATB r11, r6, r9, r11 ; sum1 = MAC16_16(sum1, x_1, y_2) +celt_pitch_xcorr_edsp_process2_1 + LDRH r6, [r4], #2 + ADDS r12, r12, #1 + ; Stall + SMLABB r10, r6, r8, r10 ; sum0 = MAC16_16(sum0, x_0, y_0) + LDRHGT r7, [r4], #2 + SMLABT r11, r6, r8, r11 ; sum1 = MAC16_16(sum1, x_0, y_1) + BLE celt_pitch_xcorr_edsp_process2_done + LDRH r9, [r5], #2 + SMLABT r10, r7, r8, r10 ; sum0 = MAC16_16(sum0, x_0, y_1) + SMLABB r11, r7, r9, r11 ; sum1 = MAC16_16(sum1, x_0, y_2) +celt_pitch_xcorr_edsp_process2_done + ; Restore _x + SUB r4, r4, r3, LSL #1 + ; Restore and advance _y + SUB r5, r5, r3, LSL #1 + ; maxcorr = max(maxcorr, sum0) + CMP r0, r10 + ADD r5, r5, #2 + MOVLT r0, r10 + SUB r1, r1, #2 + ; maxcorr = max(maxcorr, sum1) + CMP r0, r11 + ; xcorr[i] = sum + STR r10, [r2], #4 + MOVLT r0, r11 + STR r11, [r2], #4 +celt_pitch_xcorr_edsp_process1a + ADDS r1, r1, #1 + BLT celt_pitch_xcorr_edsp_done + SUBS r12, r3, #4 + ; r14 = sum = 0 + MOV r14, #0 + BLT celt_pitch_xcorr_edsp_process1a_loop_done + LDR r6, [r4], #4 + LDR r8, [r5], #4 + LDR r7, [r4], #4 + LDR r9, [r5], #4 +celt_pitch_xcorr_edsp_process1a_loop4 + SMLABB r14, r6, r8, r14 ; sum = MAC16_16(sum, x_0, y_0) + SUBS r12, r12, #4 ; j-=4 + SMLATT r14, r6, r8, r14 ; sum = MAC16_16(sum, x_1, y_1) + LDRGE r6, [r4], #4 + SMLABB r14, r7, r9, r14 ; sum = MAC16_16(sum, x_2, y_2) + LDRGE r8, [r5], #4 + SMLATT r14, r7, r9, r14 ; sum = MAC16_16(sum, x_3, y_3) + LDRGE r7, [r4], #4 + LDRGE r9, [r5], #4 + BGE celt_pitch_xcorr_edsp_process1a_loop4 +celt_pitch_xcorr_edsp_process1a_loop_done + ADDS r12, r12, #2 + LDRGE r6, [r4], #4 + LDRGE r8, [r5], #4 + ; Stall + SMLABBGE r14, r6, r8, r14 ; sum = MAC16_16(sum, x_0, y_0) + SUBGE r12, r12, #2 + SMLATTGE r14, r6, r8, r14 ; sum = MAC16_16(sum, x_1, y_1) + ADDS r12, r12, #1 + LDRHGE r6, [r4], #2 + LDRHGE r8, [r5], #2 + ; Stall + SMLABBGE r14, r6, r8, r14 ; sum = MAC16_16(sum, *x, *y) + ; maxcorr = max(maxcorr, sum) + CMP r0, r14 + ; xcorr[i] = sum + STR r14, [r2], #4 + MOVLT r0, r14 +celt_pitch_xcorr_edsp_done + LDMFD sp!, {r4-r11, pc} + ENDP + +ENDIF + +END diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fft_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fft_arm.h new file mode 100644 index 000000000..0b78175f3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fft_arm.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2015 Xiph.Org Foundation + Written by Viswanath Puttagunta */ +/** + @file fft_arm.h + @brief ARM Neon Intrinsic optimizations for fft using NE10 library + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#if !defined(FFT_ARM_H) +#define FFT_ARM_H + +#include "kiss_fft.h" + +#if defined(HAVE_ARM_NE10) + +int opus_fft_alloc_arm_neon(kiss_fft_state *st); +void opus_fft_free_arm_neon(kiss_fft_state *st); + +void opus_fft_neon(const kiss_fft_state *st, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout); + +void opus_ifft_neon(const kiss_fft_state *st, + const kiss_fft_cpx *fin, + kiss_fft_cpx *fout); + +#if !defined(OPUS_HAVE_RTCD) +#define OVERRIDE_OPUS_FFT (1) + +#define opus_fft_alloc_arch(_st, arch) \ + ((void)(arch), opus_fft_alloc_arm_neon(_st)) + +#define opus_fft_free_arch(_st, arch) \ + ((void)(arch), opus_fft_free_arm_neon(_st)) + +#define opus_fft(_st, _fin, _fout, arch) \ + ((void)(arch), opus_fft_neon(_st, _fin, _fout)) + +#define opus_ifft(_st, _fin, _fout, arch) \ + ((void)(arch), opus_ifft_neon(_st, _fin, _fout)) + +#endif /* OPUS_HAVE_RTCD */ + +#endif /* HAVE_ARM_NE10 */ + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_arm64.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_arm64.h new file mode 100644 index 000000000..c6fbd3db2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_arm64.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2015 Vidyo */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_ARM64_H +#define FIXED_ARM64_H + +#include + +#undef SIG2WORD16 +#define SIG2WORD16(x) (vqmovns_s32(PSHR32((x), SIG_SHIFT))) + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv4.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv4.h new file mode 100644 index 000000000..d84888a77 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv4.h @@ -0,0 +1,80 @@ +/* Copyright (C) 2013 Xiph.Org Foundation and contributors */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_ARMv4_H +#define FIXED_ARMv4_H + +/** 16x32 multiplication, followed by a 16-bit shift right. Results fits in 32 bits */ +#undef MULT16_32_Q16 +static OPUS_INLINE opus_val32 MULT16_32_Q16_armv4(opus_val16 a, opus_val32 b) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#MULT16_32_Q16\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(b),"r"(SHL32(a,16)) + ); + return rd_hi; +} +#define MULT16_32_Q16(a, b) (MULT16_32_Q16_armv4(a, b)) + + +/** 16x32 multiplication, followed by a 15-bit shift right. Results fits in 32 bits */ +#undef MULT16_32_Q15 +static OPUS_INLINE opus_val32 MULT16_32_Q15_armv4(opus_val16 a, opus_val32 b) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#MULT16_32_Q15\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(b), "r"(SHL32(a,16)) + ); + /*We intentionally don't OR in the high bit of rd_lo for speed.*/ + return SHL32(rd_hi,1); +} +#define MULT16_32_Q15(a, b) (MULT16_32_Q15_armv4(a, b)) + + +/** 16x32 multiply, followed by a 15-bit shift right and 32-bit add. + b must fit in 31 bits. + Result fits in 32 bits. */ +#undef MAC16_32_Q15 +#define MAC16_32_Q15(c, a, b) ADD32(c, MULT16_32_Q15(a, b)) + +/** 16x32 multiply, followed by a 16-bit shift right and 32-bit add. + Result fits in 32 bits. */ +#undef MAC16_32_Q16 +#define MAC16_32_Q16(c, a, b) ADD32(c, MULT16_32_Q16(a, b)) + +/** 32x32 multiplication, followed by a 31-bit shift right. Results fits in 32 bits */ +#undef MULT32_32_Q31 +#define MULT32_32_Q31(a,b) (opus_val32)((((opus_int64)(a)) * ((opus_int64)(b)))>>31) + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv5e.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv5e.h new file mode 100644 index 000000000..6bf73cbac --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/fixed_armv5e.h @@ -0,0 +1,151 @@ +/* Copyright (C) 2007-2009 Xiph.Org Foundation + Copyright (C) 2003-2008 Jean-Marc Valin + Copyright (C) 2007-2008 CSIRO + Copyright (C) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_ARMv5E_H +#define FIXED_ARMv5E_H + +#include "fixed_armv4.h" + +/** 16x32 multiplication, followed by a 16-bit shift right. Results fits in 32 bits */ +#undef MULT16_32_Q16 +static OPUS_INLINE opus_val32 MULT16_32_Q16_armv5e(opus_val16 a, opus_val32 b) +{ + int res; + __asm__( + "#MULT16_32_Q16\n\t" + "smulwb %0, %1, %2\n\t" + : "=r"(res) + : "r"(b),"r"(a) + ); + return res; +} +#define MULT16_32_Q16(a, b) (MULT16_32_Q16_armv5e(a, b)) + + +/** 16x32 multiplication, followed by a 15-bit shift right. Results fits in 32 bits */ +#undef MULT16_32_Q15 +static OPUS_INLINE opus_val32 MULT16_32_Q15_armv5e(opus_val16 a, opus_val32 b) +{ + int res; + __asm__( + "#MULT16_32_Q15\n\t" + "smulwb %0, %1, %2\n\t" + : "=r"(res) + : "r"(b), "r"(a) + ); + return SHL32(res,1); +} +#define MULT16_32_Q15(a, b) (MULT16_32_Q15_armv5e(a, b)) + + +/** 16x32 multiply, followed by a 15-bit shift right and 32-bit add. + b must fit in 31 bits. + Result fits in 32 bits. */ +#undef MAC16_32_Q15 +static OPUS_INLINE opus_val32 MAC16_32_Q15_armv5e(opus_val32 c, opus_val16 a, + opus_val32 b) +{ + int res; + __asm__( + "#MAC16_32_Q15\n\t" + "smlawb %0, %1, %2, %3;\n" + : "=r"(res) + : "r"(SHL32(b,1)), "r"(a), "r"(c) + ); + return res; +} +#define MAC16_32_Q15(c, a, b) (MAC16_32_Q15_armv5e(c, a, b)) + +/** 16x32 multiply, followed by a 16-bit shift right and 32-bit add. + Result fits in 32 bits. */ +#undef MAC16_32_Q16 +static OPUS_INLINE opus_val32 MAC16_32_Q16_armv5e(opus_val32 c, opus_val16 a, + opus_val32 b) +{ + int res; + __asm__( + "#MAC16_32_Q16\n\t" + "smlawb %0, %1, %2, %3;\n" + : "=r"(res) + : "r"(b), "r"(a), "r"(c) + ); + return res; +} +#define MAC16_32_Q16(c, a, b) (MAC16_32_Q16_armv5e(c, a, b)) + +/** 16x16 multiply-add where the result fits in 32 bits */ +#undef MAC16_16 +static OPUS_INLINE opus_val32 MAC16_16_armv5e(opus_val32 c, opus_val16 a, + opus_val16 b) +{ + int res; + __asm__( + "#MAC16_16\n\t" + "smlabb %0, %1, %2, %3;\n" + : "=r"(res) + : "r"(a), "r"(b), "r"(c) + ); + return res; +} +#define MAC16_16(c, a, b) (MAC16_16_armv5e(c, a, b)) + +/** 16x16 multiplication where the result fits in 32 bits */ +#undef MULT16_16 +static OPUS_INLINE opus_val32 MULT16_16_armv5e(opus_val16 a, opus_val16 b) +{ + int res; + __asm__( + "#MULT16_16\n\t" + "smulbb %0, %1, %2;\n" + : "=r"(res) + : "r"(a), "r"(b) + ); + return res; +} +#define MULT16_16(a, b) (MULT16_16_armv5e(a, b)) + +#ifdef OPUS_ARM_INLINE_MEDIA + +#undef SIG2WORD16 +static OPUS_INLINE opus_val16 SIG2WORD16_armv6(opus_val32 x) +{ + celt_sig res; + __asm__( + "#SIG2WORD16\n\t" + "ssat %0, #16, %1, ASR #12\n\t" + : "=r"(res) + : "r"(x+2048) + ); + return EXTRACT16(res); +} +#define SIG2WORD16(x) (SIG2WORD16_armv6(x)) + +#endif /* OPUS_ARM_INLINE_MEDIA */ + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv4.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv4.h new file mode 100644 index 000000000..e4faad6f2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv4.h @@ -0,0 +1,121 @@ +/*Copyright (c) 2013, Xiph.Org Foundation and contributors. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.*/ + +#ifndef KISS_FFT_ARMv4_H +#define KISS_FFT_ARMv4_H + +#if !defined(KISS_FFT_GUTS_H) +#error "This file should only be included from _kiss_fft_guts.h" +#endif + +#ifdef FIXED_POINT + +#undef C_MUL +#define C_MUL(m,a,b) \ + do{ \ + int br__; \ + int bi__; \ + int tt__; \ + __asm__ __volatile__( \ + "#C_MUL\n\t" \ + "ldrsh %[br], [%[bp], #0]\n\t" \ + "ldm %[ap], {r0,r1}\n\t" \ + "ldrsh %[bi], [%[bp], #2]\n\t" \ + "smull %[tt], %[mi], r1, %[br]\n\t" \ + "smlal %[tt], %[mi], r0, %[bi]\n\t" \ + "rsb %[bi], %[bi], #0\n\t" \ + "smull %[br], %[mr], r0, %[br]\n\t" \ + "mov %[tt], %[tt], lsr #15\n\t" \ + "smlal %[br], %[mr], r1, %[bi]\n\t" \ + "orr %[mi], %[tt], %[mi], lsl #17\n\t" \ + "mov %[br], %[br], lsr #15\n\t" \ + "orr %[mr], %[br], %[mr], lsl #17\n\t" \ + : [mr]"=r"((m).r), [mi]"=r"((m).i), \ + [br]"=&r"(br__), [bi]"=r"(bi__), [tt]"=r"(tt__) \ + : [ap]"r"(&(a)), [bp]"r"(&(b)) \ + : "r0", "r1" \ + ); \ + } \ + while(0) + +#undef C_MUL4 +#define C_MUL4(m,a,b) \ + do{ \ + int br__; \ + int bi__; \ + int tt__; \ + __asm__ __volatile__( \ + "#C_MUL4\n\t" \ + "ldrsh %[br], [%[bp], #0]\n\t" \ + "ldm %[ap], {r0,r1}\n\t" \ + "ldrsh %[bi], [%[bp], #2]\n\t" \ + "smull %[tt], %[mi], r1, %[br]\n\t" \ + "smlal %[tt], %[mi], r0, %[bi]\n\t" \ + "rsb %[bi], %[bi], #0\n\t" \ + "smull %[br], %[mr], r0, %[br]\n\t" \ + "mov %[tt], %[tt], lsr #17\n\t" \ + "smlal %[br], %[mr], r1, %[bi]\n\t" \ + "orr %[mi], %[tt], %[mi], lsl #15\n\t" \ + "mov %[br], %[br], lsr #17\n\t" \ + "orr %[mr], %[br], %[mr], lsl #15\n\t" \ + : [mr]"=r"((m).r), [mi]"=r"((m).i), \ + [br]"=&r"(br__), [bi]"=r"(bi__), [tt]"=r"(tt__) \ + : [ap]"r"(&(a)), [bp]"r"(&(b)) \ + : "r0", "r1" \ + ); \ + } \ + while(0) + +#undef C_MULC +#define C_MULC(m,a,b) \ + do{ \ + int br__; \ + int bi__; \ + int tt__; \ + __asm__ __volatile__( \ + "#C_MULC\n\t" \ + "ldrsh %[br], [%[bp], #0]\n\t" \ + "ldm %[ap], {r0,r1}\n\t" \ + "ldrsh %[bi], [%[bp], #2]\n\t" \ + "smull %[tt], %[mr], r0, %[br]\n\t" \ + "smlal %[tt], %[mr], r1, %[bi]\n\t" \ + "rsb %[bi], %[bi], #0\n\t" \ + "smull %[br], %[mi], r1, %[br]\n\t" \ + "mov %[tt], %[tt], lsr #15\n\t" \ + "smlal %[br], %[mi], r0, %[bi]\n\t" \ + "orr %[mr], %[tt], %[mr], lsl #17\n\t" \ + "mov %[br], %[br], lsr #15\n\t" \ + "orr %[mi], %[br], %[mi], lsl #17\n\t" \ + : [mr]"=r"((m).r), [mi]"=r"((m).i), \ + [br]"=&r"(br__), [bi]"=r"(bi__), [tt]"=r"(tt__) \ + : [ap]"r"(&(a)), [bp]"r"(&(b)) \ + : "r0", "r1" \ + ); \ + } \ + while(0) + +#endif /* FIXED_POINT */ + +#endif /* KISS_FFT_ARMv4_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv5e.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv5e.h new file mode 100644 index 000000000..9eca183d7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/kiss_fft_armv5e.h @@ -0,0 +1,118 @@ +/*Copyright (c) 2013, Xiph.Org Foundation and contributors. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.*/ + +#ifndef KISS_FFT_ARMv5E_H +#define KISS_FFT_ARMv5E_H + +#if !defined(KISS_FFT_GUTS_H) +#error "This file should only be included from _kiss_fft_guts.h" +#endif + +#ifdef FIXED_POINT + +#if defined(__thumb__)||defined(__thumb2__) +#define LDRD_CONS "Q" +#else +#define LDRD_CONS "Uq" +#endif + +#undef C_MUL +#define C_MUL(m,a,b) \ + do{ \ + int mr1__; \ + int mr2__; \ + int mi__; \ + long long aval__; \ + int bval__; \ + __asm__( \ + "#C_MUL\n\t" \ + "ldrd %[aval], %H[aval], %[ap]\n\t" \ + "ldr %[bval], %[bp]\n\t" \ + "smulwb %[mi], %H[aval], %[bval]\n\t" \ + "smulwb %[mr1], %[aval], %[bval]\n\t" \ + "smulwt %[mr2], %H[aval], %[bval]\n\t" \ + "smlawt %[mi], %[aval], %[bval], %[mi]\n\t" \ + : [mr1]"=r"(mr1__), [mr2]"=r"(mr2__), [mi]"=r"(mi__), \ + [aval]"=&r"(aval__), [bval]"=r"(bval__) \ + : [ap]LDRD_CONS(a), [bp]"m"(b) \ + ); \ + (m).r = SHL32(SUB32(mr1__, mr2__), 1); \ + (m).i = SHL32(mi__, 1); \ + } \ + while(0) + +#undef C_MUL4 +#define C_MUL4(m,a,b) \ + do{ \ + int mr1__; \ + int mr2__; \ + int mi__; \ + long long aval__; \ + int bval__; \ + __asm__( \ + "#C_MUL4\n\t" \ + "ldrd %[aval], %H[aval], %[ap]\n\t" \ + "ldr %[bval], %[bp]\n\t" \ + "smulwb %[mi], %H[aval], %[bval]\n\t" \ + "smulwb %[mr1], %[aval], %[bval]\n\t" \ + "smulwt %[mr2], %H[aval], %[bval]\n\t" \ + "smlawt %[mi], %[aval], %[bval], %[mi]\n\t" \ + : [mr1]"=r"(mr1__), [mr2]"=r"(mr2__), [mi]"=r"(mi__), \ + [aval]"=&r"(aval__), [bval]"=r"(bval__) \ + : [ap]LDRD_CONS(a), [bp]"m"(b) \ + ); \ + (m).r = SHR32(SUB32(mr1__, mr2__), 1); \ + (m).i = SHR32(mi__, 1); \ + } \ + while(0) + +#undef C_MULC +#define C_MULC(m,a,b) \ + do{ \ + int mr__; \ + int mi1__; \ + int mi2__; \ + long long aval__; \ + int bval__; \ + __asm__( \ + "#C_MULC\n\t" \ + "ldrd %[aval], %H[aval], %[ap]\n\t" \ + "ldr %[bval], %[bp]\n\t" \ + "smulwb %[mr], %[aval], %[bval]\n\t" \ + "smulwb %[mi1], %H[aval], %[bval]\n\t" \ + "smulwt %[mi2], %[aval], %[bval]\n\t" \ + "smlawt %[mr], %H[aval], %[bval], %[mr]\n\t" \ + : [mr]"=r"(mr__), [mi1]"=r"(mi1__), [mi2]"=r"(mi2__), \ + [aval]"=&r"(aval__), [bval]"=r"(bval__) \ + : [ap]LDRD_CONS(a), [bp]"m"(b) \ + ); \ + (m).r = SHL32(mr__, 1); \ + (m).i = SHL32(SUB32(mi1__, mi2__), 1); \ + } \ + while(0) + +#endif /* FIXED_POINT */ + +#endif /* KISS_FFT_GUTS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/mdct_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/mdct_arm.h new file mode 100644 index 000000000..14200bac4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/mdct_arm.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2015 Xiph.Org Foundation + Written by Viswanath Puttagunta */ +/** + @file arm_mdct.h + @brief ARM Neon Intrinsic optimizations for mdct using NE10 library + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(MDCT_ARM_H) +#define MDCT_ARM_H + +#include "mdct.h" + +#if defined(HAVE_ARM_NE10) +/** Compute a forward MDCT and scale by 4/N, trashes the input array */ +void clt_mdct_forward_neon(const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, int overlap, + int shift, int stride, int arch); + +void clt_mdct_backward_neon(const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, int overlap, + int shift, int stride, int arch); + +#if !defined(OPUS_HAVE_RTCD) +#define OVERRIDE_OPUS_MDCT (1) +#define clt_mdct_forward(_l, _in, _out, _window, _int, _shift, _stride, _arch) \ + clt_mdct_forward_neon(_l, _in, _out, _window, _int, _shift, _stride, _arch) +#define clt_mdct_backward(_l, _in, _out, _window, _int, _shift, _stride, _arch) \ + clt_mdct_backward_neon(_l, _in, _out, _window, _int, _shift, _stride, _arch) +#endif /* OPUS_HAVE_RTCD */ +#endif /* HAVE_ARM_NE10 */ + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_arm.h new file mode 100644 index 000000000..bed8b04ea --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_arm.h @@ -0,0 +1,160 @@ +/* Copyright (c) 2010 Xiph.Org Foundation + * Copyright (c) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(PITCH_ARM_H) +# define PITCH_ARM_H + +# include "armcpu.h" + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +opus_val32 celt_inner_prod_neon(const opus_val16 *x, const opus_val16 *y, int N); +void dual_inner_prod_neon(const opus_val16 *x, const opus_val16 *y01, + const opus_val16 *y02, int N, opus_val32 *xy1, opus_val32 *xy2); + +# if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) +# define OVERRIDE_CELT_INNER_PROD (1) +# define OVERRIDE_DUAL_INNER_PROD (1) +# define celt_inner_prod(x, y, N, arch) ((void)(arch), PRESUME_NEON(celt_inner_prod)(x, y, N)) +# define dual_inner_prod(x, y01, y02, N, xy1, xy2, arch) ((void)(arch), PRESUME_NEON(dual_inner_prod)(x, y01, y02, N, xy1, xy2)) +# endif +# endif + +# if !defined(OVERRIDE_CELT_INNER_PROD) +# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *x, const opus_val16 *y, int N); +# define OVERRIDE_CELT_INNER_PROD (1) +# define celt_inner_prod(x, y, N, arch) ((*CELT_INNER_PROD_IMPL[(arch)&OPUS_ARCHMASK])(x, y, N)) +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_CELT_INNER_PROD (1) +# define celt_inner_prod(x, y, N, arch) ((void)(arch), celt_inner_prod_neon(x, y, N)) +# endif +# endif + +# if !defined(OVERRIDE_DUAL_INNER_PROD) +# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern void (*const DUAL_INNER_PROD_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *x, + const opus_val16 *y01, const opus_val16 *y02, int N, opus_val32 *xy1, opus_val32 *xy2); +# define OVERRIDE_DUAL_INNER_PROD (1) +# define dual_inner_prod(x, y01, y02, N, xy1, xy2, arch) ((*DUAL_INNER_PROD_IMPL[(arch)&OPUS_ARCHMASK])(x, y01, y02, N, xy1, xy2)) +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_DUAL_INNER_PROD (1) +# define dual_inner_prod(x, y01, y02, N, xy1, xy2, arch) ((void)(arch), dual_inner_prod_neon(x, y01, y02, N, xy1, xy2)) +# endif +# endif + +# if defined(FIXED_POINT) + +# if defined(OPUS_ARM_MAY_HAVE_NEON) +opus_val32 celt_pitch_xcorr_neon(const opus_val16 *_x, const opus_val16 *_y, + opus_val32 *xcorr, int len, int max_pitch, int arch); +# endif + +# if defined(OPUS_ARM_MAY_HAVE_MEDIA) +# define celt_pitch_xcorr_media MAY_HAVE_EDSP(celt_pitch_xcorr) +# endif + +# if defined(OPUS_ARM_MAY_HAVE_EDSP) +opus_val32 celt_pitch_xcorr_edsp(const opus_val16 *_x, const opus_val16 *_y, + opus_val32 *xcorr, int len, int max_pitch, int arch); +# endif + +# if defined(OPUS_HAVE_RTCD) && \ + ((defined(OPUS_ARM_MAY_HAVE_NEON) && !defined(OPUS_ARM_PRESUME_NEON)) || \ + (defined(OPUS_ARM_MAY_HAVE_MEDIA) && !defined(OPUS_ARM_PRESUME_MEDIA)) || \ + (defined(OPUS_ARM_MAY_HAVE_EDSP) && !defined(OPUS_ARM_PRESUME_EDSP))) +extern opus_val32 +(*const CELT_PITCH_XCORR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *, + const opus_val16 *, opus_val32 *, int, int, int); +# define OVERRIDE_PITCH_XCORR (1) +# define celt_pitch_xcorr(_x, _y, xcorr, len, max_pitch, arch) \ + ((*CELT_PITCH_XCORR_IMPL[(arch)&OPUS_ARCHMASK])(_x, _y, \ + xcorr, len, max_pitch, arch)) + +# elif defined(OPUS_ARM_PRESUME_EDSP) || \ + defined(OPUS_ARM_PRESUME_MEDIA) || \ + defined(OPUS_ARM_PRESUME_NEON) +# define OVERRIDE_PITCH_XCORR (1) +# define celt_pitch_xcorr (PRESUME_NEON(celt_pitch_xcorr)) + +# endif + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +void xcorr_kernel_neon_fixed( + const opus_val16 *x, + const opus_val16 *y, + opus_val32 sum[4], + int len); +# endif + +# if defined(OPUS_HAVE_RTCD) && \ + (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) + +extern void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])( + const opus_val16 *x, + const opus_val16 *y, + opus_val32 sum[4], + int len); + +# define OVERRIDE_XCORR_KERNEL (1) +# define xcorr_kernel(x, y, sum, len, arch) \ + ((*XCORR_KERNEL_IMPL[(arch) & OPUS_ARCHMASK])(x, y, sum, len)) + +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_XCORR_KERNEL (1) +# define xcorr_kernel(x, y, sum, len, arch) \ + ((void)arch, xcorr_kernel_neon_fixed(x, y, sum, len)) + +# endif + +#else /* Start !FIXED_POINT */ +/* Float case */ +#if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +void celt_pitch_xcorr_float_neon(const opus_val16 *_x, const opus_val16 *_y, + opus_val32 *xcorr, int len, int max_pitch, int arch); +#endif + +# if defined(OPUS_HAVE_RTCD) && \ + (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern void +(*const CELT_PITCH_XCORR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *, + const opus_val16 *, opus_val32 *, int, int, int); + +# define OVERRIDE_PITCH_XCORR (1) +# define celt_pitch_xcorr(_x, _y, xcorr, len, max_pitch, arch) \ + ((*CELT_PITCH_XCORR_IMPL[(arch)&OPUS_ARCHMASK])(_x, _y, \ + xcorr, len, max_pitch, arch)) + +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) + +# define OVERRIDE_PITCH_XCORR (1) +# define celt_pitch_xcorr celt_pitch_xcorr_float_neon + +# endif + +#endif /* end !FIXED_POINT */ + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_neon_intr.c new file mode 100644 index 000000000..1ac38c433 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/arm/pitch_neon_intr.c @@ -0,0 +1,290 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "pitch.h" + +#ifdef FIXED_POINT + +opus_val32 celt_inner_prod_neon(const opus_val16 *x, const opus_val16 *y, int N) +{ + int i; + opus_val32 xy; + int16x8_t x_s16x8, y_s16x8; + int32x4_t xy_s32x4 = vdupq_n_s32(0); + int64x2_t xy_s64x2; + int64x1_t xy_s64x1; + + for (i = 0; i < N - 7; i += 8) { + x_s16x8 = vld1q_s16(&x[i]); + y_s16x8 = vld1q_s16(&y[i]); + xy_s32x4 = vmlal_s16(xy_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y_s16x8)); + xy_s32x4 = vmlal_s16(xy_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y_s16x8)); + } + + if (N - i >= 4) { + const int16x4_t x_s16x4 = vld1_s16(&x[i]); + const int16x4_t y_s16x4 = vld1_s16(&y[i]); + xy_s32x4 = vmlal_s16(xy_s32x4, x_s16x4, y_s16x4); + i += 4; + } + + xy_s64x2 = vpaddlq_s32(xy_s32x4); + xy_s64x1 = vadd_s64(vget_low_s64(xy_s64x2), vget_high_s64(xy_s64x2)); + xy = vget_lane_s32(vreinterpret_s32_s64(xy_s64x1), 0); + + for (; i < N; i++) { + xy = MAC16_16(xy, x[i], y[i]); + } + +#ifdef OPUS_CHECK_ASM + celt_assert(celt_inner_prod_c(x, y, N) == xy); +#endif + + return xy; +} + +void dual_inner_prod_neon(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) +{ + int i; + opus_val32 xy01, xy02; + int16x8_t x_s16x8, y01_s16x8, y02_s16x8; + int32x4_t xy01_s32x4 = vdupq_n_s32(0); + int32x4_t xy02_s32x4 = vdupq_n_s32(0); + int64x2_t xy01_s64x2, xy02_s64x2; + int64x1_t xy01_s64x1, xy02_s64x1; + + for (i = 0; i < N - 7; i += 8) { + x_s16x8 = vld1q_s16(&x[i]); + y01_s16x8 = vld1q_s16(&y01[i]); + y02_s16x8 = vld1q_s16(&y02[i]); + xy01_s32x4 = vmlal_s16(xy01_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y01_s16x8)); + xy02_s32x4 = vmlal_s16(xy02_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y02_s16x8)); + xy01_s32x4 = vmlal_s16(xy01_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y01_s16x8)); + xy02_s32x4 = vmlal_s16(xy02_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y02_s16x8)); + } + + if (N - i >= 4) { + const int16x4_t x_s16x4 = vld1_s16(&x[i]); + const int16x4_t y01_s16x4 = vld1_s16(&y01[i]); + const int16x4_t y02_s16x4 = vld1_s16(&y02[i]); + xy01_s32x4 = vmlal_s16(xy01_s32x4, x_s16x4, y01_s16x4); + xy02_s32x4 = vmlal_s16(xy02_s32x4, x_s16x4, y02_s16x4); + i += 4; + } + + xy01_s64x2 = vpaddlq_s32(xy01_s32x4); + xy02_s64x2 = vpaddlq_s32(xy02_s32x4); + xy01_s64x1 = vadd_s64(vget_low_s64(xy01_s64x2), vget_high_s64(xy01_s64x2)); + xy02_s64x1 = vadd_s64(vget_low_s64(xy02_s64x2), vget_high_s64(xy02_s64x2)); + xy01 = vget_lane_s32(vreinterpret_s32_s64(xy01_s64x1), 0); + xy02 = vget_lane_s32(vreinterpret_s32_s64(xy02_s64x1), 0); + + for (; i < N; i++) { + xy01 = MAC16_16(xy01, x[i], y01[i]); + xy02 = MAC16_16(xy02, x[i], y02[i]); + } + *xy1 = xy01; + *xy2 = xy02; + +#ifdef OPUS_CHECK_ASM + { + opus_val32 xy1_c, xy2_c; + dual_inner_prod_c(x, y01, y02, N, &xy1_c, &xy2_c); + celt_assert(xy1_c == *xy1); + celt_assert(xy2_c == *xy2); + } +#endif +} + +#else /* !FIXED_POINT */ + +/* ========================================================================== */ + +#ifdef OPUS_CHECK_ASM + +/* This part of code simulates floating-point NEON operations. */ + +/* celt_inner_prod_neon_float_c_simulation() simulates the floating-point */ +/* operations of celt_inner_prod_neon(), and both functions should have bit */ +/* exact output. */ +static opus_val32 celt_inner_prod_neon_float_c_simulation(const opus_val16 *x, const opus_val16 *y, int N) +{ + int i; + opus_val32 xy, xy0 = 0, xy1 = 0, xy2 = 0, xy3 = 0; + for (i = 0; i < N - 3; i += 4) { + xy0 = MAC16_16(xy0, x[i + 0], y[i + 0]); + xy1 = MAC16_16(xy1, x[i + 1], y[i + 1]); + xy2 = MAC16_16(xy2, x[i + 2], y[i + 2]); + xy3 = MAC16_16(xy3, x[i + 3], y[i + 3]); + } + xy0 += xy2; + xy1 += xy3; + xy = xy0 + xy1; + for (; i < N; i++) { + xy = MAC16_16(xy, x[i], y[i]); + } + return xy; +} + +/* dual_inner_prod_neon_float_c_simulation() simulates the floating-point */ +/* operations of dual_inner_prod_neon(), and both functions should have bit */ +/* exact output. */ +static void dual_inner_prod_neon_float_c_simulation(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) +{ + int i; + opus_val32 xy01, xy02, xy01_0 = 0, xy01_1 = 0, xy01_2 = 0, xy01_3 = 0, xy02_0 = 0, xy02_1 = 0, xy02_2 = 0, xy02_3 = 0; + for (i = 0; i < N - 3; i += 4) { + xy01_0 = MAC16_16(xy01_0, x[i + 0], y01[i + 0]); + xy01_1 = MAC16_16(xy01_1, x[i + 1], y01[i + 1]); + xy01_2 = MAC16_16(xy01_2, x[i + 2], y01[i + 2]); + xy01_3 = MAC16_16(xy01_3, x[i + 3], y01[i + 3]); + xy02_0 = MAC16_16(xy02_0, x[i + 0], y02[i + 0]); + xy02_1 = MAC16_16(xy02_1, x[i + 1], y02[i + 1]); + xy02_2 = MAC16_16(xy02_2, x[i + 2], y02[i + 2]); + xy02_3 = MAC16_16(xy02_3, x[i + 3], y02[i + 3]); + } + xy01_0 += xy01_2; + xy02_0 += xy02_2; + xy01_1 += xy01_3; + xy02_1 += xy02_3; + xy01 = xy01_0 + xy01_1; + xy02 = xy02_0 + xy02_1; + for (; i < N; i++) { + xy01 = MAC16_16(xy01, x[i], y01[i]); + xy02 = MAC16_16(xy02, x[i], y02[i]); + } + *xy1 = xy01; + *xy2 = xy02; +} + +#endif /* OPUS_CHECK_ASM */ + +/* ========================================================================== */ + +opus_val32 celt_inner_prod_neon(const opus_val16 *x, const opus_val16 *y, int N) +{ + int i; + opus_val32 xy; + float32x4_t xy_f32x4 = vdupq_n_f32(0); + float32x2_t xy_f32x2; + + for (i = 0; i < N - 7; i += 8) { + float32x4_t x_f32x4, y_f32x4; + x_f32x4 = vld1q_f32(&x[i]); + y_f32x4 = vld1q_f32(&y[i]); + xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); + x_f32x4 = vld1q_f32(&x[i + 4]); + y_f32x4 = vld1q_f32(&y[i + 4]); + xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); + } + + if (N - i >= 4) { + const float32x4_t x_f32x4 = vld1q_f32(&x[i]); + const float32x4_t y_f32x4 = vld1q_f32(&y[i]); + xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); + i += 4; + } + + xy_f32x2 = vadd_f32(vget_low_f32(xy_f32x4), vget_high_f32(xy_f32x4)); + xy_f32x2 = vpadd_f32(xy_f32x2, xy_f32x2); + xy = vget_lane_f32(xy_f32x2, 0); + + for (; i < N; i++) { + xy = MAC16_16(xy, x[i], y[i]); + } + +#ifdef OPUS_CHECK_ASM + celt_assert(ABS32(celt_inner_prod_neon_float_c_simulation(x, y, N) - xy) <= VERY_SMALL); +#endif + + return xy; +} + +void dual_inner_prod_neon(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) +{ + int i; + opus_val32 xy01, xy02; + float32x4_t xy01_f32x4 = vdupq_n_f32(0); + float32x4_t xy02_f32x4 = vdupq_n_f32(0); + float32x2_t xy01_f32x2, xy02_f32x2; + + for (i = 0; i < N - 7; i += 8) { + float32x4_t x_f32x4, y01_f32x4, y02_f32x4; + x_f32x4 = vld1q_f32(&x[i]); + y01_f32x4 = vld1q_f32(&y01[i]); + y02_f32x4 = vld1q_f32(&y02[i]); + xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); + xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); + x_f32x4 = vld1q_f32(&x[i + 4]); + y01_f32x4 = vld1q_f32(&y01[i + 4]); + y02_f32x4 = vld1q_f32(&y02[i + 4]); + xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); + xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); + } + + if (N - i >= 4) { + const float32x4_t x_f32x4 = vld1q_f32(&x[i]); + const float32x4_t y01_f32x4 = vld1q_f32(&y01[i]); + const float32x4_t y02_f32x4 = vld1q_f32(&y02[i]); + xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); + xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); + i += 4; + } + + xy01_f32x2 = vadd_f32(vget_low_f32(xy01_f32x4), vget_high_f32(xy01_f32x4)); + xy02_f32x2 = vadd_f32(vget_low_f32(xy02_f32x4), vget_high_f32(xy02_f32x4)); + xy01_f32x2 = vpadd_f32(xy01_f32x2, xy01_f32x2); + xy02_f32x2 = vpadd_f32(xy02_f32x2, xy02_f32x2); + xy01 = vget_lane_f32(xy01_f32x2, 0); + xy02 = vget_lane_f32(xy02_f32x2, 0); + + for (; i < N; i++) { + xy01 = MAC16_16(xy01, x[i], y01[i]); + xy02 = MAC16_16(xy02, x[i], y02[i]); + } + *xy1 = xy01; + *xy2 = xy02; + +#ifdef OPUS_CHECK_ASM + { + opus_val32 xy1_c, xy2_c; + dual_inner_prod_neon_float_c_simulation(x, y01, y02, N, &xy1_c, &xy2_c); + celt_assert(ABS32(xy1_c - *xy1) <= VERY_SMALL); + celt_assert(ABS32(xy2_c - *xy2) <= VERY_SMALL); + } +#endif +} + +#endif /* FIXED_POINT */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/bands.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/bands.c new file mode 100644 index 000000000..2702963c3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/bands.c @@ -0,0 +1,1672 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2008-2009 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "bands.h" +#include "modes.h" +#include "vq.h" +#include "cwrs.h" +#include "stack_alloc.h" +#include "os_support.h" +#include "mathops.h" +#include "rate.h" +#include "quant_bands.h" +#include "pitch.h" + +int hysteresis_decision(opus_val16 val, const opus_val16 *thresholds, const opus_val16 *hysteresis, int N, int prev) +{ + int i; + for (i=0;iprev && val < thresholds[prev]+hysteresis[prev]) + i=prev; + if (i thresholds[prev-1]-hysteresis[prev-1]) + i=prev; + return i; +} + +opus_uint32 celt_lcg_rand(opus_uint32 seed) +{ + return 1664525 * seed + 1013904223; +} + +/* This is a cos() approximation designed to be bit-exact on any platform. Bit exactness + with this approximation is important because it has an impact on the bit allocation */ +opus_int16 bitexact_cos(opus_int16 x) +{ + opus_int32 tmp; + opus_int16 x2; + tmp = (4096+((opus_int32)(x)*(x)))>>13; + celt_sig_assert(tmp<=32767); + x2 = tmp; + x2 = (32767-x2) + FRAC_MUL16(x2, (-7651 + FRAC_MUL16(x2, (8277 + FRAC_MUL16(-626, x2))))); + celt_sig_assert(x2<=32766); + return 1+x2; +} + +int bitexact_log2tan(int isin,int icos) +{ + int lc; + int ls; + lc=EC_ILOG(icos); + ls=EC_ILOG(isin); + icos<<=15-lc; + isin<<=15-ls; + return (ls-lc)*(1<<11) + +FRAC_MUL16(isin, FRAC_MUL16(isin, -2597) + 7932) + -FRAC_MUL16(icos, FRAC_MUL16(icos, -2597) + 7932); +} + +#ifdef FIXED_POINT +/* Compute the amplitude (sqrt energy) in each of the bands */ +void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch) +{ + int i, c, N; + const opus_int16 *eBands = m->eBands; + (void)arch; + N = m->shortMdctSize< 0) + { + int shift = celt_ilog2(maxval) - 14 + (((m->logN[i]>>BITRES)+LM+1)>>1); + j=eBands[i]<0) + { + do { + sum = MAC16_16(sum, EXTRACT16(SHR32(X[j+c*N],shift)), + EXTRACT16(SHR32(X[j+c*N],shift))); + } while (++jnbEBands] = EPSILON+VSHR32(EXTEND32(celt_sqrt(sum)),-shift); + } else { + bandE[i+c*m->nbEBands] = EPSILON; + } + /*printf ("%f ", bandE[i+c*m->nbEBands]);*/ + } + } while (++ceBands; + N = M*m->shortMdctSize; + c=0; do { + i=0; do { + opus_val16 g; + int j,shift; + opus_val16 E; + shift = celt_zlog2(bandE[i+c*m->nbEBands])-13; + E = VSHR32(bandE[i+c*m->nbEBands], shift); + g = EXTRACT16(celt_rcp(SHL32(E,3))); + j=M*eBands[i]; do { + X[j+c*N] = MULT16_16_Q15(VSHR32(freq[j+c*N],shift-1),g); + } while (++jeBands; + N = m->shortMdctSize<nbEBands] = celt_sqrt(sum); + /*printf ("%f ", bandE[i+c*m->nbEBands]);*/ + } + } while (++ceBands; + N = M*m->shortMdctSize; + c=0; do { + for (i=0;inbEBands]); + for (j=M*eBands[i];jeBands; + N = M*m->shortMdctSize; + bound = M*eBands[end]; + if (downsample!=1) + bound = IMIN(bound, N/downsample); + if (silence) + { + bound = 0; + start = end = 0; + } + f = freq; + x = X+M*eBands[start]; + for (i=0;i>DB_SHIFT); + if (shift>31) + { + shift=0; + g=0; + } else { + /* Handle the fractional part. */ + g = celt_exp2_frac(lg&((1< 16384 we'd be likely to overflow, so we're + capping the gain here, which is equivalent to a cap of 18 on lg. + This shouldn't trigger unless the bitstream is already corrupted. */ + if (shift <= -2) + { + g = 16384; + shift = -2; + } + do { + *f++ = SHL32(MULT16_16(*x++, g), -shift); + } while (++jeBands[i+1]-m->eBands[i]; + /* depth in 1/8 bits */ + celt_sig_assert(pulses[i]>=0); + depth = celt_udiv(1+pulses[i], (m->eBands[i+1]-m->eBands[i]))>>LM; + +#ifdef FIXED_POINT + thresh32 = SHR32(celt_exp2(-SHL16(depth, 10-BITRES)),1); + thresh = MULT16_32_Q15(QCONST16(0.5f, 15), MIN32(32767,thresh32)); + { + opus_val32 t; + t = N0<>1; + t = SHL32(t, (7-shift)<<1); + sqrt_1 = celt_rsqrt_norm(t); + } +#else + thresh = .5f*celt_exp2(-.125f*depth); + sqrt_1 = celt_rsqrt(N0<nbEBands+i]; + prev2 = prev2logE[c*m->nbEBands+i]; + if (C==1) + { + prev1 = MAX16(prev1,prev1logE[m->nbEBands+i]); + prev2 = MAX16(prev2,prev2logE[m->nbEBands+i]); + } + Ediff = EXTEND32(logE[c*m->nbEBands+i])-EXTEND32(MIN16(prev1,prev2)); + Ediff = MAX32(0, Ediff); + +#ifdef FIXED_POINT + if (Ediff < 16384) + { + opus_val32 r32 = SHR32(celt_exp2(-EXTRACT16(Ediff)),1); + r = 2*MIN16(16383,r32); + } else { + r = 0; + } + if (LM==3) + r = MULT16_16_Q14(23170, MIN32(23169, r)); + r = SHR16(MIN16(thresh, r),1); + r = SHR32(MULT16_16_Q15(sqrt_1, r),shift); +#else + /* r needs to be multiplied by 2 or 2*sqrt(2) depending on LM because + short blocks don't have the same energy as long */ + r = 2.f*celt_exp2(-Ediff); + if (LM==3) + r *= 1.41421356f; + r = MIN16(thresh, r); + r = r*sqrt_1; +#endif + X = X_+c*size+(m->eBands[i]<nbEBands]))-13; +#endif + left = VSHR32(bandE[i],shift); + right = VSHR32(bandE[i+m->nbEBands],shift); + norm = EPSILON + celt_sqrt(EPSILON+MULT16_16(left,left)+MULT16_16(right,right)); + a1 = DIV32_16(SHL32(EXTEND32(left),14),norm); + a2 = DIV32_16(SHL32(EXTEND32(right),14),norm); + for (j=0;j>1; + kr = celt_ilog2(Er)>>1; +#endif + t = VSHR32(El, (kl-7)<<1); + lgain = celt_rsqrt_norm(t); + t = VSHR32(Er, (kr-7)<<1); + rgain = celt_rsqrt_norm(t); + +#ifdef FIXED_POINT + if (kl < 7) + kl = 7; + if (kr < 7) + kr = 7; +#endif + + for (j=0;jeBands; + int decision; + int hf_sum=0; + + celt_assert(end>0); + + N0 = M*m->shortMdctSize; + + if (M*(eBands[end]-eBands[end-1]) <= 8) + return SPREAD_NONE; + c=0; do { + for (i=0;im->nbEBands-4) + hf_sum += celt_udiv(32*(tcount[1]+tcount[0]), N); + tmp = (2*tcount[2] >= N) + (2*tcount[1] >= N) + (2*tcount[0] >= N); + sum += tmp*spread_weight[i]; + nbBands+=spread_weight[i]; + } + } while (++cnbEBands+end)); + *hf_average = (*hf_average+hf_sum)>>1; + hf_sum = *hf_average; + if (*tapset_decision==2) + hf_sum += 4; + else if (*tapset_decision==0) + hf_sum -= 4; + if (hf_sum > 22) + *tapset_decision=2; + else if (hf_sum > 18) + *tapset_decision=1; + else + *tapset_decision=0; + } + /*printf("%d %d %d\n", hf_sum, *hf_average, *tapset_decision);*/ + celt_assert(nbBands>0); /* end has to be non-zero */ + celt_assert(sum>=0); + sum = celt_udiv((opus_int32)sum<<8, nbBands); + /* Recursive averaging */ + sum = (sum+*average)>>1; + *average = sum; + /* Hysteresis */ + sum = (3*sum + (((3-last_decision)<<7) + 64) + 2)>>2; + if (sum < 80) + { + decision = SPREAD_AGGRESSIVE; + } else if (sum < 256) + { + decision = SPREAD_NORMAL; + } else if (sum < 384) + { + decision = SPREAD_LIGHT; + } else { + decision = SPREAD_NONE; + } +#ifdef FUZZING + decision = rand()&0x3; + *tapset_decision=rand()%3; +#endif + return decision; +} + +/* Indexing table for converting from natural Hadamard to ordery Hadamard + This is essentially a bit-reversed Gray, on top of which we've added + an inversion of the order because we want the DC at the end rather than + the beginning. The lines are for N=2, 4, 8, 16 */ +static const int ordery_table[] = { + 1, 0, + 3, 0, 2, 1, + 7, 0, 4, 3, 6, 1, 5, 2, + 15, 0, 8, 7, 12, 3, 11, 4, 14, 1, 9, 6, 13, 2, 10, 5, +}; + +static void deinterleave_hadamard(celt_norm *X, int N0, int stride, int hadamard) +{ + int i,j; + VARDECL(celt_norm, tmp); + int N; + SAVE_STACK; + N = N0*stride; + ALLOC(tmp, N, celt_norm); + celt_assert(stride>0); + if (hadamard) + { + const int *ordery = ordery_table+stride-2; + for (i=0;i>= 1; + for (i=0;i>1)) { + qn = 1; + } else { + qn = exp2_table8[qb&0x7]>>(14-(qb>>BITRES)); + qn = (qn+1)>>1<<1; + } + celt_assert(qn <= 256); + return qn; +} + +struct band_ctx { + int encode; + int resynth; + const CELTMode *m; + int i; + int intensity; + int spread; + int tf_change; + ec_ctx *ec; + opus_int32 remaining_bits; + const celt_ener *bandE; + opus_uint32 seed; + int arch; + int theta_round; + int disable_inv; + int avoid_split_noise; +}; + +struct split_ctx { + int inv; + int imid; + int iside; + int delta; + int itheta; + int qalloc; +}; + +static void compute_theta(struct band_ctx *ctx, struct split_ctx *sctx, + celt_norm *X, celt_norm *Y, int N, int *b, int B, int B0, + int LM, + int stereo, int *fill) +{ + int qn; + int itheta=0; + int delta; + int imid, iside; + int qalloc; + int pulse_cap; + int offset; + opus_int32 tell; + int inv=0; + int encode; + const CELTMode *m; + int i; + int intensity; + ec_ctx *ec; + const celt_ener *bandE; + + encode = ctx->encode; + m = ctx->m; + i = ctx->i; + intensity = ctx->intensity; + ec = ctx->ec; + bandE = ctx->bandE; + + /* Decide on the resolution to give to the split parameter theta */ + pulse_cap = m->logN[i]+LM*(1<>1) - (stereo&&N==2 ? QTHETA_OFFSET_TWOPHASE : QTHETA_OFFSET); + qn = compute_qn(N, *b, offset, pulse_cap, stereo); + if (stereo && i>=intensity) + qn = 1; + if (encode) + { + /* theta is the atan() of the ratio between the (normalized) + side and mid. With just that parameter, we can re-scale both + mid and side because we know that 1) they have unit norm and + 2) they are orthogonal. */ + itheta = stereo_itheta(X, Y, stereo, N, ctx->arch); + } + tell = ec_tell_frac(ec); + if (qn!=1) + { + if (encode) + { + if (!stereo || ctx->theta_round == 0) + { + itheta = (itheta*(opus_int32)qn+8192)>>14; + if (!stereo && ctx->avoid_split_noise && itheta > 0 && itheta < qn) + { + /* Check if the selected value of theta will cause the bit allocation + to inject noise on one side. If so, make sure the energy of that side + is zero. */ + int unquantized = celt_udiv((opus_int32)itheta*16384, qn); + imid = bitexact_cos((opus_int16)unquantized); + iside = bitexact_cos((opus_int16)(16384-unquantized)); + delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid)); + if (delta > *b) + itheta = qn; + else if (delta < -*b) + itheta = 0; + } + } else { + int down; + /* Bias quantization towards itheta=0 and itheta=16384. */ + int bias = itheta > 8192 ? 32767/qn : -32767/qn; + down = IMIN(qn-1, IMAX(0, (itheta*(opus_int32)qn + bias)>>14)); + if (ctx->theta_round < 0) + itheta = down; + else + itheta = down+1; + } + } + /* Entropy coding of the angle. We use a uniform pdf for the + time split, a step for stereo, and a triangular one for the rest. */ + if (stereo && N>2) + { + int p0 = 3; + int x = itheta; + int x0 = qn/2; + int ft = p0*(x0+1) + x0; + /* Use a probability of p0 up to itheta=8192 and then use 1 after */ + if (encode) + { + ec_encode(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft); + } else { + int fs; + fs=ec_decode(ec,ft); + if (fs<(x0+1)*p0) + x=fs/p0; + else + x=x0+1+(fs-(x0+1)*p0); + ec_dec_update(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft); + itheta = x; + } + } else if (B0>1 || stereo) { + /* Uniform pdf */ + if (encode) + ec_enc_uint(ec, itheta, qn+1); + else + itheta = ec_dec_uint(ec, qn+1); + } else { + int fs=1, ft; + ft = ((qn>>1)+1)*((qn>>1)+1); + if (encode) + { + int fl; + + fs = itheta <= (qn>>1) ? itheta + 1 : qn + 1 - itheta; + fl = itheta <= (qn>>1) ? itheta*(itheta + 1)>>1 : + ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1); + + ec_encode(ec, fl, fl+fs, ft); + } else { + /* Triangular pdf */ + int fl=0; + int fm; + fm = ec_decode(ec, ft); + + if (fm < ((qn>>1)*((qn>>1) + 1)>>1)) + { + itheta = (isqrt32(8*(opus_uint32)fm + 1) - 1)>>1; + fs = itheta + 1; + fl = itheta*(itheta + 1)>>1; + } + else + { + itheta = (2*(qn + 1) + - isqrt32(8*(opus_uint32)(ft - fm - 1) + 1))>>1; + fs = qn + 1 - itheta; + fl = ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1); + } + + ec_dec_update(ec, fl, fl+fs, ft); + } + } + celt_assert(itheta>=0); + itheta = celt_udiv((opus_int32)itheta*16384, qn); + if (encode && stereo) + { + if (itheta==0) + intensity_stereo(m, X, Y, bandE, i, N); + else + stereo_split(X, Y, N); + } + /* NOTE: Renormalising X and Y *may* help fixed-point a bit at very high rate. + Let's do that at higher complexity */ + } else if (stereo) { + if (encode) + { + inv = itheta > 8192 && !ctx->disable_inv; + if (inv) + { + int j; + for (j=0;j2<remaining_bits > 2<disable_inv) + inv = 0; + itheta = 0; + } + qalloc = ec_tell_frac(ec) - tell; + *b -= qalloc; + + if (itheta == 0) + { + imid = 32767; + iside = 0; + *fill &= (1<inv = inv; + sctx->imid = imid; + sctx->iside = iside; + sctx->delta = delta; + sctx->itheta = itheta; + sctx->qalloc = qalloc; +} +static unsigned quant_band_n1(struct band_ctx *ctx, celt_norm *X, celt_norm *Y, int b, + celt_norm *lowband_out) +{ + int c; + int stereo; + celt_norm *x = X; + int encode; + ec_ctx *ec; + + encode = ctx->encode; + ec = ctx->ec; + + stereo = Y != NULL; + c=0; do { + int sign=0; + if (ctx->remaining_bits>=1<remaining_bits -= 1<resynth) + x[0] = sign ? -NORM_SCALING : NORM_SCALING; + x = Y; + } while (++c<1+stereo); + if (lowband_out) + lowband_out[0] = SHR16(X[0],4); + return 1; +} + +/* This function is responsible for encoding and decoding a mono partition. + It can split the band in two and transmit the energy difference with + the two half-bands. It can be called recursively so bands can end up being + split in 8 parts. */ +static unsigned quant_partition(struct band_ctx *ctx, celt_norm *X, + int N, int b, int B, celt_norm *lowband, + int LM, + opus_val16 gain, int fill) +{ + const unsigned char *cache; + int q; + int curr_bits; + int imid=0, iside=0; + int B0=B; + opus_val16 mid=0, side=0; + unsigned cm=0; + celt_norm *Y=NULL; + int encode; + const CELTMode *m; + int i; + int spread; + ec_ctx *ec; + + encode = ctx->encode; + m = ctx->m; + i = ctx->i; + spread = ctx->spread; + ec = ctx->ec; + + /* If we need 1.5 more bit than we can produce, split the band in two. */ + cache = m->cache.bits + m->cache.index[(LM+1)*m->nbEBands+i]; + if (LM != -1 && b > cache[cache[0]]+12 && N>2) + { + int mbits, sbits, delta; + int itheta; + int qalloc; + struct split_ctx sctx; + celt_norm *next_lowband2=NULL; + opus_int32 rebalance; + + N >>= 1; + Y = X+N; + LM -= 1; + if (B==1) + fill = (fill&1)|(fill<<1); + B = (B+1)>>1; + + compute_theta(ctx, &sctx, X, Y, N, &b, B, B0, LM, 0, &fill); + imid = sctx.imid; + iside = sctx.iside; + delta = sctx.delta; + itheta = sctx.itheta; + qalloc = sctx.qalloc; +#ifdef FIXED_POINT + mid = imid; + side = iside; +#else + mid = (1.f/32768)*imid; + side = (1.f/32768)*iside; +#endif + + /* Give more bits to low-energy MDCTs than they would otherwise deserve */ + if (B0>1 && (itheta&0x3fff)) + { + if (itheta > 8192) + /* Rough approximation for pre-echo masking */ + delta -= delta>>(4-LM); + else + /* Corresponds to a forward-masking slope of 1.5 dB per 10 ms */ + delta = IMIN(0, delta + (N<>(5-LM))); + } + mbits = IMAX(0, IMIN(b, (b-delta)/2)); + sbits = b-mbits; + ctx->remaining_bits -= qalloc; + + if (lowband) + next_lowband2 = lowband+N; /* >32-bit split case */ + + rebalance = ctx->remaining_bits; + if (mbits >= sbits) + { + cm = quant_partition(ctx, X, N, mbits, B, lowband, LM, + MULT16_16_P15(gain,mid), fill); + rebalance = mbits - (rebalance-ctx->remaining_bits); + if (rebalance > 3<>B)<<(B0>>1); + } else { + cm = quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM, + MULT16_16_P15(gain,side), fill>>B)<<(B0>>1); + rebalance = sbits - (rebalance-ctx->remaining_bits); + if (rebalance > 3<remaining_bits -= curr_bits; + + /* Ensures we can never bust the budget */ + while (ctx->remaining_bits < 0 && q > 0) + { + ctx->remaining_bits += curr_bits; + q--; + curr_bits = pulses2bits(m, i, LM, q); + ctx->remaining_bits -= curr_bits; + } + + if (q!=0) + { + int K = get_pulses(q); + + /* Finally do the actual quantization */ + if (encode) + { + cm = alg_quant(X, N, K, spread, B, ec, gain, ctx->resynth, ctx->arch); + } else { + cm = alg_unquant(X, N, K, spread, B, ec, gain); + } + } else { + /* If there's no pulse, fill the band anyway */ + int j; + if (ctx->resynth) + { + unsigned cm_mask; + /* B can be as large as 16, so this shift might overflow an int on a + 16-bit platform; use a long to get defined behavior.*/ + cm_mask = (unsigned)(1UL<seed = celt_lcg_rand(ctx->seed); + X[j] = (celt_norm)((opus_int32)ctx->seed>>20); + } + cm = cm_mask; + } else { + /* Folded spectrum */ + for (j=0;jseed = celt_lcg_rand(ctx->seed); + /* About 48 dB below the "normal" folding level */ + tmp = QCONST16(1.0f/256, 10); + tmp = (ctx->seed)&0x8000 ? tmp : -tmp; + X[j] = lowband[j]+tmp; + } + cm = fill; + } + renormalise_vector(X, N, gain, ctx->arch); + } + } + } + } + + return cm; +} + + +/* This function is responsible for encoding and decoding a band for the mono case. */ +static unsigned quant_band(struct band_ctx *ctx, celt_norm *X, + int N, int b, int B, celt_norm *lowband, + int LM, celt_norm *lowband_out, + opus_val16 gain, celt_norm *lowband_scratch, int fill) +{ + int N0=N; + int N_B=N; + int N_B0; + int B0=B; + int time_divide=0; + int recombine=0; + int longBlocks; + unsigned cm=0; + int k; + int encode; + int tf_change; + + encode = ctx->encode; + tf_change = ctx->tf_change; + + longBlocks = B0==1; + + N_B = celt_udiv(N_B, B); + + /* Special case for one sample */ + if (N==1) + { + return quant_band_n1(ctx, X, NULL, b, lowband_out); + } + + if (tf_change>0) + recombine = tf_change; + /* Band recombining to increase frequency resolution */ + + if (lowband_scratch && lowband && (recombine || ((N_B&1) == 0 && tf_change<0) || B0>1)) + { + OPUS_COPY(lowband_scratch, lowband, N); + lowband = lowband_scratch; + } + + for (k=0;k>k, 1<>k, 1<>4]<<2; + } + B>>=recombine; + N_B<<=recombine; + + /* Increasing the time resolution */ + while ((N_B&1) == 0 && tf_change<0) + { + if (encode) + haar1(X, N_B, B); + if (lowband) + haar1(lowband, N_B, B); + fill |= fill<>= 1; + time_divide++; + tf_change++; + } + B0=B; + N_B0 = N_B; + + /* Reorganize the samples in time order instead of frequency order */ + if (B0>1) + { + if (encode) + deinterleave_hadamard(X, N_B>>recombine, B0<>recombine, B0<resynth) + { + /* Undo the sample reorganization going from time order to frequency order */ + if (B0>1) + interleave_hadamard(X, N_B>>recombine, B0<>= 1; + N_B <<= 1; + cm |= cm>>B; + haar1(X, N_B, B); + } + + for (k=0;k>k, 1<encode; + ec = ctx->ec; + + /* Special case for one sample */ + if (N==1) + { + return quant_band_n1(ctx, X, Y, b, lowband_out); + } + + orig_fill = fill; + + compute_theta(ctx, &sctx, X, Y, N, &b, B, B, LM, 1, &fill); + inv = sctx.inv; + imid = sctx.imid; + iside = sctx.iside; + delta = sctx.delta; + itheta = sctx.itheta; + qalloc = sctx.qalloc; +#ifdef FIXED_POINT + mid = imid; + side = iside; +#else + mid = (1.f/32768)*imid; + side = (1.f/32768)*iside; +#endif + + /* This is a special case for N=2 that only works for stereo and takes + advantage of the fact that mid and side are orthogonal to encode + the side with just one bit. */ + if (N==2) + { + int c; + int sign=0; + celt_norm *x2, *y2; + mbits = b; + sbits = 0; + /* Only need one bit for the side. */ + if (itheta != 0 && itheta != 16384) + sbits = 1< 8192; + ctx->remaining_bits -= qalloc+sbits; + + x2 = c ? Y : X; + y2 = c ? X : Y; + if (sbits) + { + if (encode) + { + /* Here we only need to encode a sign for the side. */ + sign = x2[0]*y2[1] - x2[1]*y2[0] < 0; + ec_enc_bits(ec, sign, 1); + } else { + sign = ec_dec_bits(ec, 1); + } + } + sign = 1-2*sign; + /* We use orig_fill here because we want to fold the side, but if + itheta==16384, we'll have cleared the low bits of fill. */ + cm = quant_band(ctx, x2, N, mbits, B, lowband, LM, lowband_out, Q15ONE, + lowband_scratch, orig_fill); + /* We don't split N=2 bands, so cm is either 1 or 0 (for a fold-collapse), + and there's no need to worry about mixing with the other channel. */ + y2[0] = -sign*x2[1]; + y2[1] = sign*x2[0]; + if (ctx->resynth) + { + celt_norm tmp; + X[0] = MULT16_16_Q15(mid, X[0]); + X[1] = MULT16_16_Q15(mid, X[1]); + Y[0] = MULT16_16_Q15(side, Y[0]); + Y[1] = MULT16_16_Q15(side, Y[1]); + tmp = X[0]; + X[0] = SUB16(tmp,Y[0]); + Y[0] = ADD16(tmp,Y[0]); + tmp = X[1]; + X[1] = SUB16(tmp,Y[1]); + Y[1] = ADD16(tmp,Y[1]); + } + } else { + /* "Normal" split code */ + opus_int32 rebalance; + + mbits = IMAX(0, IMIN(b, (b-delta)/2)); + sbits = b-mbits; + ctx->remaining_bits -= qalloc; + + rebalance = ctx->remaining_bits; + if (mbits >= sbits) + { + /* In stereo mode, we do not apply a scaling to the mid because we need the normalized + mid for folding later. */ + cm = quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q15ONE, + lowband_scratch, fill); + rebalance = mbits - (rebalance-ctx->remaining_bits); + if (rebalance > 3<>B); + } else { + /* For a stereo split, the high bits of fill are always zero, so no + folding will be done to the side. */ + cm = quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B); + rebalance = sbits - (rebalance-ctx->remaining_bits); + if (rebalance > 3<resynth) + { + if (N!=2) + stereo_merge(X, Y, mid, N, ctx->arch); + if (inv) + { + int j; + for (j=0;jeBands; + n1 = M*(eBands[start+1]-eBands[start]); + n2 = M*(eBands[start+2]-eBands[start+1]); + /* Duplicate enough of the first band folding data to be able to fold the second band. + Copies no data for CELT-only mode. */ + OPUS_COPY(&norm[n1], &norm[2*n1 - n2], n2-n1); + if (dual_stereo) + OPUS_COPY(&norm2[n1], &norm2[2*n1 - n2], n2-n1); +} + +void quant_all_bands(int encode, const CELTMode *m, int start, int end, + celt_norm *X_, celt_norm *Y_, unsigned char *collapse_masks, + const celt_ener *bandE, int *pulses, int shortBlocks, int spread, + int dual_stereo, int intensity, int *tf_res, opus_int32 total_bits, + opus_int32 balance, ec_ctx *ec, int LM, int codedBands, + opus_uint32 *seed, int complexity, int arch, int disable_inv) +{ + int i; + opus_int32 remaining_bits; + const opus_int16 * OPUS_RESTRICT eBands = m->eBands; + celt_norm * OPUS_RESTRICT norm, * OPUS_RESTRICT norm2; + VARDECL(celt_norm, _norm); + VARDECL(celt_norm, _lowband_scratch); + VARDECL(celt_norm, X_save); + VARDECL(celt_norm, Y_save); + VARDECL(celt_norm, X_save2); + VARDECL(celt_norm, Y_save2); + VARDECL(celt_norm, norm_save2); + int resynth_alloc; + celt_norm *lowband_scratch; + int B; + int M; + int lowband_offset; + int update_lowband = 1; + int C = Y_ != NULL ? 2 : 1; + int norm_offset; + int theta_rdo = encode && Y_!=NULL && !dual_stereo && complexity>=8; +#ifdef RESYNTH + int resynth = 1; +#else + int resynth = !encode || theta_rdo; +#endif + struct band_ctx ctx; + SAVE_STACK; + + M = 1<nbEBands-1]-norm_offset), celt_norm); + norm = _norm; + norm2 = norm + M*eBands[m->nbEBands-1]-norm_offset; + + /* For decoding, we can use the last band as scratch space because we don't need that + scratch space for the last band and we don't care about the data there until we're + decoding the last band. */ + if (encode && resynth) + resynth_alloc = M*(eBands[m->nbEBands]-eBands[m->nbEBands-1]); + else + resynth_alloc = ALLOC_NONE; + ALLOC(_lowband_scratch, resynth_alloc, celt_norm); + if (encode && resynth) + lowband_scratch = _lowband_scratch; + else + lowband_scratch = X_+M*eBands[m->nbEBands-1]; + ALLOC(X_save, resynth_alloc, celt_norm); + ALLOC(Y_save, resynth_alloc, celt_norm); + ALLOC(X_save2, resynth_alloc, celt_norm); + ALLOC(Y_save2, resynth_alloc, celt_norm); + ALLOC(norm_save2, resynth_alloc, celt_norm); + + lowband_offset = 0; + ctx.bandE = bandE; + ctx.ec = ec; + ctx.encode = encode; + ctx.intensity = intensity; + ctx.m = m; + ctx.seed = *seed; + ctx.spread = spread; + ctx.arch = arch; + ctx.disable_inv = disable_inv; + ctx.resynth = resynth; + ctx.theta_round = 0; + /* Avoid injecting noise in the first band on transients. */ + ctx.avoid_split_noise = B > 1; + for (i=start;i 0); + tell = ec_tell_frac(ec); + + /* Compute how many bits we want to allocate to this band */ + if (i != start) + balance -= tell; + remaining_bits = total_bits-tell-1; + ctx.remaining_bits = remaining_bits; + if (i <= codedBands-1) + { + curr_balance = celt_sudiv(balance, IMIN(3, codedBands-i)); + b = IMAX(0, IMIN(16383, IMIN(remaining_bits+1,pulses[i]+curr_balance))); + } else { + b = 0; + } + +#ifndef DISABLE_UPDATE_DRAFT + if (resynth && (M*eBands[i]-N >= M*eBands[start] || i==start+1) && (update_lowband || lowband_offset==0)) + lowband_offset = i; + if (i == start+1) + special_hybrid_folding(m, norm, norm2, start, M, dual_stereo); +#else + if (resynth && M*eBands[i]-N >= M*eBands[start] && (update_lowband || lowband_offset==0)) + lowband_offset = i; +#endif + + tf_change = tf_res[i]; + ctx.tf_change = tf_change; + if (i>=m->effEBands) + { + X=norm; + if (Y_!=NULL) + Y = norm; + lowband_scratch = NULL; + } + if (last && !theta_rdo) + lowband_scratch = NULL; + + /* Get a conservative estimate of the collapse_mask's for the bands we're + going to be folding from. */ + if (lowband_offset != 0 && (spread!=SPREAD_AGGRESSIVE || B>1 || tf_change<0)) + { + int fold_start; + int fold_end; + int fold_i; + /* This ensures we never repeat spectral content within one band */ + effective_lowband = IMAX(0, M*eBands[lowband_offset]-norm_offset-N); + fold_start = lowband_offset; + while(M*eBands[--fold_start] > effective_lowband+norm_offset); + fold_end = lowband_offset-1; +#ifndef DISABLE_UPDATE_DRAFT + while(++fold_end < i && M*eBands[fold_end] < effective_lowband+norm_offset+N); +#else + while(M*eBands[++fold_end] < effective_lowband+norm_offset+N); +#endif + x_cm = y_cm = 0; + fold_i = fold_start; do { + x_cm |= collapse_masks[fold_i*C+0]; + y_cm |= collapse_masks[fold_i*C+C-1]; + } while (++fold_inbEBands], w); + /* Make a copy. */ + cm = x_cm|y_cm; + ec_save = *ec; + ctx_save = ctx; + OPUS_COPY(X_save, X, N); + OPUS_COPY(Y_save, Y, N); + /* Encode and round down. */ + ctx.theta_round = -1; + x_cm = quant_band_stereo(&ctx, X, Y, N, b, B, + effective_lowband != -1 ? norm+effective_lowband : NULL, LM, + last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm); + dist0 = MULT16_32_Q15(w[0], celt_inner_prod(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod(Y_save, Y, N, arch)); + + /* Save first result. */ + cm2 = x_cm; + ec_save2 = *ec; + ctx_save2 = ctx; + OPUS_COPY(X_save2, X, N); + OPUS_COPY(Y_save2, Y, N); + if (!last) + OPUS_COPY(norm_save2, norm+M*eBands[i]-norm_offset, N); + nstart_bytes = ec_save.offs; + nend_bytes = ec_save.storage; + bytes_buf = ec_save.buf+nstart_bytes; + save_bytes = nend_bytes-nstart_bytes; + OPUS_COPY(bytes_save, bytes_buf, save_bytes); + + /* Restore */ + *ec = ec_save; + ctx = ctx_save; + OPUS_COPY(X, X_save, N); + OPUS_COPY(Y, Y_save, N); +#ifndef DISABLE_UPDATE_DRAFT + if (i == start+1) + special_hybrid_folding(m, norm, norm2, start, M, dual_stereo); +#endif + /* Encode and round up. */ + ctx.theta_round = 1; + x_cm = quant_band_stereo(&ctx, X, Y, N, b, B, + effective_lowband != -1 ? norm+effective_lowband : NULL, LM, + last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm); + dist1 = MULT16_32_Q15(w[0], celt_inner_prod(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod(Y_save, Y, N, arch)); + if (dist0 >= dist1) { + x_cm = cm2; + *ec = ec_save2; + ctx = ctx_save2; + OPUS_COPY(X, X_save2, N); + OPUS_COPY(Y, Y_save2, N); + if (!last) + OPUS_COPY(norm+M*eBands[i]-norm_offset, norm_save2, N); + OPUS_COPY(bytes_buf, bytes_save, save_bytes); + } + } else { + ctx.theta_round = 0; + x_cm = quant_band_stereo(&ctx, X, Y, N, b, B, + effective_lowband != -1 ? norm+effective_lowband : NULL, LM, + last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, x_cm|y_cm); + } + } else { + x_cm = quant_band(&ctx, X, N, b, B, + effective_lowband != -1 ? norm+effective_lowband : NULL, LM, + last?NULL:norm+M*eBands[i]-norm_offset, Q15ONE, lowband_scratch, x_cm|y_cm); + } + y_cm = x_cm; + } + collapse_masks[i*C+0] = (unsigned char)x_cm; + collapse_masks[i*C+C-1] = (unsigned char)y_cm; + balance += pulses[i] + tell; + + /* Update the folding position only as long as we have 1 bit/sample depth. */ + update_lowband = b>(N< +#include "celt.h" +#include "pitch.h" +#include "bands.h" +#include "modes.h" +#include "entcode.h" +#include "quant_bands.h" +#include "rate.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "float_cast.h" +#include +#include "celt_lpc.h" +#include "vq.h" + +#ifndef PACKAGE_VERSION +#define PACKAGE_VERSION "unknown" +#endif + +#if defined(MIPSr1_ASM) +#include "mips/celt_mipsr1.h" +#endif + + +int resampling_factor(opus_int32 rate) +{ + int ret; + switch (rate) + { + case 48000: + ret = 1; + break; + case 24000: + ret = 2; + break; + case 16000: + ret = 3; + break; + case 12000: + ret = 4; + break; + case 8000: + ret = 6; + break; + default: +#ifndef CUSTOM_MODES + celt_assert(0); +#endif + ret = 0; + break; + } + return ret; +} + +#if !defined(OVERRIDE_COMB_FILTER_CONST) || defined(NON_STATIC_COMB_FILTER_CONST_C) +/* This version should be faster on ARM */ +#ifdef OPUS_ARM_ASM +#ifndef NON_STATIC_COMB_FILTER_CONST_C +static +#endif +void comb_filter_const_c(opus_val32 *y, opus_val32 *x, int T, int N, + opus_val16 g10, opus_val16 g11, opus_val16 g12) +{ + opus_val32 x0, x1, x2, x3, x4; + int i; + x4 = SHL32(x[-T-2], 1); + x3 = SHL32(x[-T-1], 1); + x2 = SHL32(x[-T], 1); + x1 = SHL32(x[-T+1], 1); + for (i=0;inbEBands;i++) + { + int N; + N=(m->eBands[i+1]-m->eBands[i])<cache.caps[m->nbEBands*(2*LM+C-1)+i]+64)*C*N>>2; + } +} + + + +const char *opus_strerror(int error) +{ + static const char * const error_strings[8] = { + "success", + "invalid argument", + "buffer too small", + "internal error", + "corrupted stream", + "request not implemented", + "invalid state", + "memory allocation failed" + }; + if (error > 0 || error < -7) + return "unknown error"; + else + return error_strings[-error]; +} + +const char *opus_get_version_string(void) +{ + return "libopus " PACKAGE_VERSION + /* Applications may rely on the presence of this substring in the version + string to determine if they have a fixed-point or floating-point build + at runtime. */ +#ifdef FIXED_POINT + "-fixed" +#endif +#ifdef FUZZING + "-fuzzing" +#endif + ; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt.h new file mode 100644 index 000000000..24b6b2b52 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt.h @@ -0,0 +1,251 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2008 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/** + @file celt.h + @brief Contains all the functions for encoding and decoding audio + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CELT_H +#define CELT_H + +#include "opus_types.h" +#include "opus_defines.h" +#include "opus_custom.h" +#include "entenc.h" +#include "entdec.h" +#include "arch.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define CELTEncoder OpusCustomEncoder +#define CELTDecoder OpusCustomDecoder +#define CELTMode OpusCustomMode + +#define LEAK_BANDS 19 + +typedef struct { + int valid; + float tonality; + float tonality_slope; + float noisiness; + float activity; + float music_prob; + float music_prob_min; + float music_prob_max; + int bandwidth; + float activity_probability; + float max_pitch_ratio; + /* Store as Q6 char to save space. */ + unsigned char leak_boost[LEAK_BANDS]; +} AnalysisInfo; + +typedef struct { + int signalType; + int offset; +} SILKInfo; + +#define __celt_check_mode_ptr_ptr(ptr) ((ptr) + ((ptr) - (const CELTMode**)(ptr))) + +#define __celt_check_analysis_ptr(ptr) ((ptr) + ((ptr) - (const AnalysisInfo*)(ptr))) + +#define __celt_check_silkinfo_ptr(ptr) ((ptr) + ((ptr) - (const SILKInfo*)(ptr))) + +/* Encoder/decoder Requests */ + + +#define CELT_SET_PREDICTION_REQUEST 10002 +/** Controls the use of interframe prediction. + 0=Independent frames + 1=Short term interframe prediction allowed + 2=Long term prediction allowed + */ +#define CELT_SET_PREDICTION(x) CELT_SET_PREDICTION_REQUEST, __opus_check_int(x) + +#define CELT_SET_INPUT_CLIPPING_REQUEST 10004 +#define CELT_SET_INPUT_CLIPPING(x) CELT_SET_INPUT_CLIPPING_REQUEST, __opus_check_int(x) + +#define CELT_GET_AND_CLEAR_ERROR_REQUEST 10007 +#define CELT_GET_AND_CLEAR_ERROR(x) CELT_GET_AND_CLEAR_ERROR_REQUEST, __opus_check_int_ptr(x) + +#define CELT_SET_CHANNELS_REQUEST 10008 +#define CELT_SET_CHANNELS(x) CELT_SET_CHANNELS_REQUEST, __opus_check_int(x) + + +/* Internal */ +#define CELT_SET_START_BAND_REQUEST 10010 +#define CELT_SET_START_BAND(x) CELT_SET_START_BAND_REQUEST, __opus_check_int(x) + +#define CELT_SET_END_BAND_REQUEST 10012 +#define CELT_SET_END_BAND(x) CELT_SET_END_BAND_REQUEST, __opus_check_int(x) + +#define CELT_GET_MODE_REQUEST 10015 +/** Get the CELTMode used by an encoder or decoder */ +#define CELT_GET_MODE(x) CELT_GET_MODE_REQUEST, __celt_check_mode_ptr_ptr(x) + +#define CELT_SET_SIGNALLING_REQUEST 10016 +#define CELT_SET_SIGNALLING(x) CELT_SET_SIGNALLING_REQUEST, __opus_check_int(x) + +#define CELT_SET_TONALITY_REQUEST 10018 +#define CELT_SET_TONALITY(x) CELT_SET_TONALITY_REQUEST, __opus_check_int(x) +#define CELT_SET_TONALITY_SLOPE_REQUEST 10020 +#define CELT_SET_TONALITY_SLOPE(x) CELT_SET_TONALITY_SLOPE_REQUEST, __opus_check_int(x) + +#define CELT_SET_ANALYSIS_REQUEST 10022 +#define CELT_SET_ANALYSIS(x) CELT_SET_ANALYSIS_REQUEST, __celt_check_analysis_ptr(x) + +#define OPUS_SET_LFE_REQUEST 10024 +#define OPUS_SET_LFE(x) OPUS_SET_LFE_REQUEST, __opus_check_int(x) + +#define OPUS_SET_ENERGY_MASK_REQUEST 10026 +#define OPUS_SET_ENERGY_MASK(x) OPUS_SET_ENERGY_MASK_REQUEST, __opus_check_val16_ptr(x) + +#define CELT_SET_SILK_INFO_REQUEST 10028 +#define CELT_SET_SILK_INFO(x) CELT_SET_SILK_INFO_REQUEST, __celt_check_silkinfo_ptr(x) + +/* Encoder stuff */ + +int celt_encoder_get_size(int channels); + +int celt_encode_with_ec(OpusCustomEncoder * OPUS_RESTRICT st, const opus_val16 * pcm, int frame_size, unsigned char *compressed, int nbCompressedBytes, ec_enc *enc); + +int celt_encoder_init(CELTEncoder *st, opus_int32 sampling_rate, int channels, + int arch); + + + +/* Decoder stuff */ + +int celt_decoder_get_size(int channels); + + +int celt_decoder_init(CELTDecoder *st, opus_int32 sampling_rate, int channels); + +int celt_decode_with_ec(OpusCustomDecoder * OPUS_RESTRICT st, const unsigned char *data, + int len, opus_val16 * OPUS_RESTRICT pcm, int frame_size, ec_dec *dec, int accum); + +#define celt_encoder_ctl opus_custom_encoder_ctl +#define celt_decoder_ctl opus_custom_decoder_ctl + + +#ifdef CUSTOM_MODES +#define OPUS_CUSTOM_NOSTATIC +#else +#define OPUS_CUSTOM_NOSTATIC static OPUS_INLINE +#endif + +static const unsigned char trim_icdf[11] = {126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0}; +/* Probs: NONE: 21.875%, LIGHT: 6.25%, NORMAL: 65.625%, AGGRESSIVE: 6.25% */ +static const unsigned char spread_icdf[4] = {25, 23, 2, 0}; + +static const unsigned char tapset_icdf[3]={2,1,0}; + +#ifdef CUSTOM_MODES +static const unsigned char toOpusTable[20] = { + 0xE0, 0xE8, 0xF0, 0xF8, + 0xC0, 0xC8, 0xD0, 0xD8, + 0xA0, 0xA8, 0xB0, 0xB8, + 0x00, 0x00, 0x00, 0x00, + 0x80, 0x88, 0x90, 0x98, +}; + +static const unsigned char fromOpusTable[16] = { + 0x80, 0x88, 0x90, 0x98, + 0x40, 0x48, 0x50, 0x58, + 0x20, 0x28, 0x30, 0x38, + 0x00, 0x08, 0x10, 0x18 +}; + +static OPUS_INLINE int toOpus(unsigned char c) +{ + int ret=0; + if (c<0xA0) + ret = toOpusTable[c>>3]; + if (ret == 0) + return -1; + else + return ret|(c&0x7); +} + +static OPUS_INLINE int fromOpus(unsigned char c) +{ + if (c<0x80) + return -1; + else + return fromOpusTable[(c>>3)-16] | (c&0x7); +} +#endif /* CUSTOM_MODES */ + +#define COMBFILTER_MAXPERIOD 1024 +#define COMBFILTER_MINPERIOD 15 + +extern const signed char tf_select_table[4][8]; + +#if defined(ENABLE_HARDENING) || defined(ENABLE_ASSERTIONS) +void validate_celt_decoder(CELTDecoder *st); +#define VALIDATE_CELT_DECODER(st) validate_celt_decoder(st) +#else +#define VALIDATE_CELT_DECODER(st) +#endif + +int resampling_factor(opus_int32 rate); + +void celt_preemphasis(const opus_val16 * OPUS_RESTRICT pcmp, celt_sig * OPUS_RESTRICT inp, + int N, int CC, int upsample, const opus_val16 *coef, celt_sig *mem, int clip); + +void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, + opus_val16 g0, opus_val16 g1, int tapset0, int tapset1, + const opus_val16 *window, int overlap, int arch); + +#ifdef NON_STATIC_COMB_FILTER_CONST_C +void comb_filter_const_c(opus_val32 *y, opus_val32 *x, int T, int N, + opus_val16 g10, opus_val16 g11, opus_val16 g12); +#endif + +#ifndef OVERRIDE_COMB_FILTER_CONST +# define comb_filter_const(y, x, T, N, g10, g11, g12, arch) \ + ((void)(arch),comb_filter_const_c(y, x, T, N, g10, g11, g12)) +#endif + +void init_caps(const CELTMode *m,int *cap,int LM,int C); + +#ifdef RESYNTH +void deemphasis(celt_sig *in[], opus_val16 *pcm, int N, int C, int downsample, const opus_val16 *coef, celt_sig *mem); +void celt_synthesis(const CELTMode *mode, celt_norm *X, celt_sig * out_syn[], + opus_val16 *oldBandE, int start, int effEnd, int C, int CC, int isTransient, + int LM, int downsample, int silence); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* CELT_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_decoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_decoder.c new file mode 100644 index 000000000..74ca3b740 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_decoder.c @@ -0,0 +1,1378 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2010 Xiph.Org Foundation + Copyright (c) 2008 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define CELT_DECODER_C + +#include "cpu_support.h" +#include "os_support.h" +#include "mdct.h" +#include +#include "celt.h" +#include "pitch.h" +#include "bands.h" +#include "modes.h" +#include "entcode.h" +#include "quant_bands.h" +#include "rate.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "float_cast.h" +#include +#include "celt_lpc.h" +#include "vq.h" + +/* The maximum pitch lag to allow in the pitch-based PLC. It's possible to save + CPU time in the PLC pitch search by making this smaller than MAX_PERIOD. The + current value corresponds to a pitch of 66.67 Hz. */ +#define PLC_PITCH_LAG_MAX (720) +/* The minimum pitch lag to allow in the pitch-based PLC. This corresponds to a + pitch of 480 Hz. */ +#define PLC_PITCH_LAG_MIN (100) + +#if defined(SMALL_FOOTPRINT) && defined(FIXED_POINT) +#define NORM_ALIASING_HACK +#endif +/**********************************************************************/ +/* */ +/* DECODER */ +/* */ +/**********************************************************************/ +#define DECODE_BUFFER_SIZE 2048 + +/** Decoder state + @brief Decoder state + */ +struct OpusCustomDecoder { + const OpusCustomMode *mode; + int overlap; + int channels; + int stream_channels; + + int downsample; + int start, end; + int signalling; + int disable_inv; + int arch; + + /* Everything beyond this point gets cleared on a reset */ +#define DECODER_RESET_START rng + + opus_uint32 rng; + int error; + int last_pitch_index; + int loss_count; + int skip_plc; + int postfilter_period; + int postfilter_period_old; + opus_val16 postfilter_gain; + opus_val16 postfilter_gain_old; + int postfilter_tapset; + int postfilter_tapset_old; + + celt_sig preemph_memD[2]; + + celt_sig _decode_mem[1]; /* Size = channels*(DECODE_BUFFER_SIZE+mode->overlap) */ + /* opus_val16 lpc[], Size = channels*LPC_ORDER */ + /* opus_val16 oldEBands[], Size = 2*mode->nbEBands */ + /* opus_val16 oldLogE[], Size = 2*mode->nbEBands */ + /* opus_val16 oldLogE2[], Size = 2*mode->nbEBands */ + /* opus_val16 backgroundLogE[], Size = 2*mode->nbEBands */ +}; + +#if defined(ENABLE_HARDENING) || defined(ENABLE_ASSERTIONS) +/* Make basic checks on the CELT state to ensure we don't end + up writing all over memory. */ +void validate_celt_decoder(CELTDecoder *st) +{ +#ifndef CUSTOM_MODES + celt_assert(st->mode == opus_custom_mode_create(48000, 960, NULL)); + celt_assert(st->overlap == 120); + celt_assert(st->end <= 21); +#else +/* From Section 4.3 in the spec: "The normal CELT layer uses 21 of those bands, + though Opus Custom (see Section 6.2) may use a different number of bands" + + Check if it's within the maximum number of Bark frequency bands instead */ + celt_assert(st->end <= 25); +#endif + celt_assert(st->channels == 1 || st->channels == 2); + celt_assert(st->stream_channels == 1 || st->stream_channels == 2); + celt_assert(st->downsample > 0); + celt_assert(st->start == 0 || st->start == 17); + celt_assert(st->start < st->end); +#ifdef OPUS_ARCHMASK + celt_assert(st->arch >= 0); + celt_assert(st->arch <= OPUS_ARCHMASK); +#endif + celt_assert(st->last_pitch_index <= PLC_PITCH_LAG_MAX); + celt_assert(st->last_pitch_index >= PLC_PITCH_LAG_MIN || st->last_pitch_index == 0); + celt_assert(st->postfilter_period < MAX_PERIOD); + celt_assert(st->postfilter_period >= COMBFILTER_MINPERIOD || st->postfilter_period == 0); + celt_assert(st->postfilter_period_old < MAX_PERIOD); + celt_assert(st->postfilter_period_old >= COMBFILTER_MINPERIOD || st->postfilter_period_old == 0); + celt_assert(st->postfilter_tapset <= 2); + celt_assert(st->postfilter_tapset >= 0); + celt_assert(st->postfilter_tapset_old <= 2); + celt_assert(st->postfilter_tapset_old >= 0); +} +#endif + +int celt_decoder_get_size(int channels) +{ + const CELTMode *mode = opus_custom_mode_create(48000, 960, NULL); + return opus_custom_decoder_get_size(mode, channels); +} + +OPUS_CUSTOM_NOSTATIC int opus_custom_decoder_get_size(const CELTMode *mode, int channels) +{ + int size = sizeof(struct CELTDecoder) + + (channels*(DECODE_BUFFER_SIZE+mode->overlap)-1)*sizeof(celt_sig) + + channels*LPC_ORDER*sizeof(opus_val16) + + 4*2*mode->nbEBands*sizeof(opus_val16); + return size; +} + +#ifdef CUSTOM_MODES +CELTDecoder *opus_custom_decoder_create(const CELTMode *mode, int channels, int *error) +{ + int ret; + CELTDecoder *st = (CELTDecoder *)opus_alloc(opus_custom_decoder_get_size(mode, channels)); + ret = opus_custom_decoder_init(st, mode, channels); + if (ret != OPUS_OK) + { + opus_custom_decoder_destroy(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} +#endif /* CUSTOM_MODES */ + +int celt_decoder_init(CELTDecoder *st, opus_int32 sampling_rate, int channels) +{ + int ret; + ret = opus_custom_decoder_init(st, opus_custom_mode_create(48000, 960, NULL), channels); + if (ret != OPUS_OK) + return ret; + st->downsample = resampling_factor(sampling_rate); + if (st->downsample==0) + return OPUS_BAD_ARG; + else + return OPUS_OK; +} + +OPUS_CUSTOM_NOSTATIC int opus_custom_decoder_init(CELTDecoder *st, const CELTMode *mode, int channels) +{ + if (channels < 0 || channels > 2) + return OPUS_BAD_ARG; + + if (st==NULL) + return OPUS_ALLOC_FAIL; + + OPUS_CLEAR((char*)st, opus_custom_decoder_get_size(mode, channels)); + + st->mode = mode; + st->overlap = mode->overlap; + st->stream_channels = st->channels = channels; + + st->downsample = 1; + st->start = 0; + st->end = st->mode->effEBands; + st->signalling = 1; +#ifndef DISABLE_UPDATE_DRAFT + st->disable_inv = channels == 1; +#else + st->disable_inv = 0; +#endif + st->arch = opus_select_arch(); + + opus_custom_decoder_ctl(st, OPUS_RESET_STATE); + + return OPUS_OK; +} + +#ifdef CUSTOM_MODES +void opus_custom_decoder_destroy(CELTDecoder *st) +{ + opus_free(st); +} +#endif /* CUSTOM_MODES */ + +#ifndef CUSTOM_MODES +/* Special case for stereo with no downsampling and no accumulation. This is + quite common and we can make it faster by processing both channels in the + same loop, reducing overhead due to the dependency loop in the IIR filter. */ +static void deemphasis_stereo_simple(celt_sig *in[], opus_val16 *pcm, int N, const opus_val16 coef0, + celt_sig *mem) +{ + celt_sig * OPUS_RESTRICT x0; + celt_sig * OPUS_RESTRICT x1; + celt_sig m0, m1; + int j; + x0=in[0]; + x1=in[1]; + m0 = mem[0]; + m1 = mem[1]; + for (j=0;j1) + { + /* Shortcut for the standard (non-custom modes) case */ + for (j=0;joverlap; + nbEBands = mode->nbEBands; + N = mode->shortMdctSize<shortMdctSize; + shift = mode->maxLM; + } else { + B = 1; + NB = mode->shortMdctSize<maxLM-LM; + } + + if (CC==2&&C==1) + { + /* Copying a mono streams to two channels */ + celt_sig *freq2; + denormalise_bands(mode, X, freq, oldBandE, start, effEnd, M, + downsample, silence); + /* Store a temporary copy in the output buffer because the IMDCT destroys its input. */ + freq2 = out_syn[1]+overlap/2; + OPUS_COPY(freq2, freq, N); + for (b=0;bmdct, &freq2[b], out_syn[0]+NB*b, mode->window, overlap, shift, B, arch); + for (b=0;bmdct, &freq[b], out_syn[1]+NB*b, mode->window, overlap, shift, B, arch); + } else if (CC==1&&C==2) + { + /* Downmixing a stereo stream to mono */ + celt_sig *freq2; + freq2 = out_syn[0]+overlap/2; + denormalise_bands(mode, X, freq, oldBandE, start, effEnd, M, + downsample, silence); + /* Use the output buffer as temp array before downmixing. */ + denormalise_bands(mode, X+N, freq2, oldBandE+nbEBands, start, effEnd, M, + downsample, silence); + for (i=0;imdct, &freq[b], out_syn[0]+NB*b, mode->window, overlap, shift, B, arch); + } else { + /* Normal case (mono or stereo) */ + c=0; do { + denormalise_bands(mode, X+c*N, freq, oldBandE+c*nbEBands, start, effEnd, M, + downsample, silence); + for (b=0;bmdct, &freq[b], out_syn[c]+NB*b, mode->window, overlap, shift, B, arch); + } while (++cstorage*8; + tell = ec_tell(dec); + logp = isTransient ? 2 : 4; + tf_select_rsv = LM>0 && tell+logp+1<=budget; + budget -= tf_select_rsv; + tf_changed = curr = 0; + for (i=start;i>1, opus_val16 ); + pitch_downsample(decode_mem, lp_pitch_buf, + DECODE_BUFFER_SIZE, C, arch); + pitch_search(lp_pitch_buf+(PLC_PITCH_LAG_MAX>>1), lp_pitch_buf, + DECODE_BUFFER_SIZE-PLC_PITCH_LAG_MAX, + PLC_PITCH_LAG_MAX-PLC_PITCH_LAG_MIN, &pitch_index, arch); + pitch_index = PLC_PITCH_LAG_MAX-pitch_index; + RESTORE_STACK; + return pitch_index; +} + +static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) +{ + int c; + int i; + const int C = st->channels; + celt_sig *decode_mem[2]; + celt_sig *out_syn[2]; + opus_val16 *lpc; + opus_val16 *oldBandE, *oldLogE, *oldLogE2, *backgroundLogE; + const OpusCustomMode *mode; + int nbEBands; + int overlap; + int start; + int loss_count; + int noise_based; + const opus_int16 *eBands; + SAVE_STACK; + + mode = st->mode; + nbEBands = mode->nbEBands; + overlap = mode->overlap; + eBands = mode->eBands; + + c=0; do { + decode_mem[c] = st->_decode_mem + c*(DECODE_BUFFER_SIZE+overlap); + out_syn[c] = decode_mem[c]+DECODE_BUFFER_SIZE-N; + } while (++c_decode_mem+(DECODE_BUFFER_SIZE+overlap)*C); + oldBandE = lpc+C*LPC_ORDER; + oldLogE = oldBandE + 2*nbEBands; + oldLogE2 = oldLogE + 2*nbEBands; + backgroundLogE = oldLogE2 + 2*nbEBands; + + loss_count = st->loss_count; + start = st->start; + noise_based = loss_count >= 5 || start != 0 || st->skip_plc; + if (noise_based) + { + /* Noise-based PLC/CNG */ +#ifdef NORM_ALIASING_HACK + celt_norm *X; +#else + VARDECL(celt_norm, X); +#endif + opus_uint32 seed; + int end; + int effEnd; + opus_val16 decay; + end = st->end; + effEnd = IMAX(start, IMIN(end, mode->effEBands)); + +#ifdef NORM_ALIASING_HACK + /* This is an ugly hack that breaks aliasing rules and would be easily broken, + but it saves almost 4kB of stack. */ + X = (celt_norm*)(out_syn[C-1]+overlap/2); +#else + ALLOC(X, C*N, celt_norm); /**< Interleaved normalised MDCTs */ +#endif + + /* Energy decay */ + decay = loss_count==0 ? QCONST16(1.5f, DB_SHIFT) : QCONST16(.5f, DB_SHIFT); + c=0; do + { + for (i=start;irng; + for (c=0;c>20); + } + renormalise_vector(X+boffs, blen, Q15ONE, st->arch); + } + } + st->rng = seed; + + c=0; do { + OPUS_MOVE(decode_mem[c], decode_mem[c]+N, + DECODE_BUFFER_SIZE-N+(overlap>>1)); + } while (++cdownsample, 0, st->arch); + } else { + int exc_length; + /* Pitch-based PLC */ + const opus_val16 *window; + opus_val16 *exc; + opus_val16 fade = Q15ONE; + int pitch_index; + VARDECL(opus_val32, etmp); + VARDECL(opus_val16, _exc); + VARDECL(opus_val16, fir_tmp); + + if (loss_count == 0) + { + st->last_pitch_index = pitch_index = celt_plc_pitch_search(decode_mem, C, st->arch); + } else { + pitch_index = st->last_pitch_index; + fade = QCONST16(.8f,15); + } + + /* We want the excitation for 2 pitch periods in order to look for a + decaying signal, but we can't get more than MAX_PERIOD. */ + exc_length = IMIN(2*pitch_index, MAX_PERIOD); + + ALLOC(etmp, overlap, opus_val32); + ALLOC(_exc, MAX_PERIOD+LPC_ORDER, opus_val16); + ALLOC(fir_tmp, exc_length, opus_val16); + exc = _exc+LPC_ORDER; + window = mode->window; + c=0; do { + opus_val16 decay; + opus_val16 attenuation; + opus_val32 S1=0; + celt_sig *buf; + int extrapolation_offset; + int extrapolation_len; + int j; + + buf = decode_mem[c]; + for (i=0;iarch); + /* Add a noise floor of -40 dB. */ +#ifdef FIXED_POINT + ac[0] += SHR32(ac[0],13); +#else + ac[0] *= 1.0001f; +#endif + /* Use lag windowing to stabilize the Levinson-Durbin recursion. */ + for (i=1;i<=LPC_ORDER;i++) + { + /*ac[i] *= exp(-.5*(2*M_PI*.002*i)*(2*M_PI*.002*i));*/ +#ifdef FIXED_POINT + ac[i] -= MULT16_32_Q15(2*i*i, ac[i]); +#else + ac[i] -= ac[i]*(0.008f*0.008f)*i*i; +#endif + } + _celt_lpc(lpc+c*LPC_ORDER, ac, LPC_ORDER); +#ifdef FIXED_POINT + /* For fixed-point, apply bandwidth expansion until we can guarantee that + no overflow can happen in the IIR filter. This means: + 32768*sum(abs(filter)) < 2^31 */ + while (1) { + opus_val16 tmp=Q15ONE; + opus_val32 sum=QCONST16(1., SIG_SHIFT); + for (i=0;iarch); + OPUS_COPY(exc+MAX_PERIOD-exc_length, fir_tmp, exc_length); + } + + /* Check if the waveform is decaying, and if so how fast. + We do this to avoid adding energy when concealing in a segment + with decaying energy. */ + { + opus_val32 E1=1, E2=1; + int decay_length; +#ifdef FIXED_POINT + int shift = IMAX(0,2*celt_zlog2(celt_maxabs16(&exc[MAX_PERIOD-exc_length], exc_length))-20); +#endif + decay_length = exc_length>>1; + for (i=0;i= pitch_index) { + j -= pitch_index; + attenuation = MULT16_16_Q15(attenuation, decay); + } + buf[DECODE_BUFFER_SIZE-N+i] = + SHL32(EXTEND32(MULT16_16_Q15(attenuation, + exc[extrapolation_offset+j])), SIG_SHIFT); + /* Compute the energy of the previously decoded signal whose + excitation we're copying. */ + tmp = ROUND16( + buf[DECODE_BUFFER_SIZE-MAX_PERIOD-N+extrapolation_offset+j], + SIG_SHIFT); + S1 += SHR32(MULT16_16(tmp, tmp), 10); + } + { + opus_val16 lpc_mem[LPC_ORDER]; + /* Copy the last decoded samples (prior to the overlap region) to + synthesis filter memory so we can have a continuous signal. */ + for (i=0;iarch); +#ifdef FIXED_POINT + for (i=0; i < extrapolation_len; i++) + buf[DECODE_BUFFER_SIZE-N+i] = SATURATE(buf[DECODE_BUFFER_SIZE-N+i], SIG_SAT); +#endif + } + + /* Check if the synthesis energy is higher than expected, which can + happen with the signal changes during our window. If so, + attenuate. */ + { + opus_val32 S2=0; + for (i=0;i SHR32(S2,2))) +#else + /* The float test is written this way to catch NaNs in the output + of the IIR filter at the same time. */ + if (!(S1 > 0.2f*S2)) +#endif + { + for (i=0;ipostfilter_period, st->postfilter_period, overlap, + -st->postfilter_gain, -st->postfilter_gain, + st->postfilter_tapset, st->postfilter_tapset, NULL, 0, st->arch); + + /* Simulate TDAC on the concealed audio so that it blends with the + MDCT of the next frame. */ + for (i=0;iloss_count = loss_count+1; + + RESTORE_STACK; +} + +int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, + int len, opus_val16 * OPUS_RESTRICT pcm, int frame_size, ec_dec *dec, int accum) +{ + int c, i, N; + int spread_decision; + opus_int32 bits; + ec_dec _dec; +#ifdef NORM_ALIASING_HACK + celt_norm *X; +#else + VARDECL(celt_norm, X); +#endif + VARDECL(int, fine_quant); + VARDECL(int, pulses); + VARDECL(int, cap); + VARDECL(int, offsets); + VARDECL(int, fine_priority); + VARDECL(int, tf_res); + VARDECL(unsigned char, collapse_masks); + celt_sig *decode_mem[2]; + celt_sig *out_syn[2]; + opus_val16 *lpc; + opus_val16 *oldBandE, *oldLogE, *oldLogE2, *backgroundLogE; + + int shortBlocks; + int isTransient; + int intra_ener; + const int CC = st->channels; + int LM, M; + int start; + int end; + int effEnd; + int codedBands; + int alloc_trim; + int postfilter_pitch; + opus_val16 postfilter_gain; + int intensity=0; + int dual_stereo=0; + opus_int32 total_bits; + opus_int32 balance; + opus_int32 tell; + int dynalloc_logp; + int postfilter_tapset; + int anti_collapse_rsv; + int anti_collapse_on=0; + int silence; + int C = st->stream_channels; + const OpusCustomMode *mode; + int nbEBands; + int overlap; + const opus_int16 *eBands; + ALLOC_STACK; + + VALIDATE_CELT_DECODER(st); + mode = st->mode; + nbEBands = mode->nbEBands; + overlap = mode->overlap; + eBands = mode->eBands; + start = st->start; + end = st->end; + frame_size *= st->downsample; + + lpc = (opus_val16*)(st->_decode_mem+(DECODE_BUFFER_SIZE+overlap)*CC); + oldBandE = lpc+CC*LPC_ORDER; + oldLogE = oldBandE + 2*nbEBands; + oldLogE2 = oldLogE + 2*nbEBands; + backgroundLogE = oldLogE2 + 2*nbEBands; + +#ifdef CUSTOM_MODES + if (st->signalling && data!=NULL) + { + int data0=data[0]; + /* Convert "standard mode" to Opus header */ + if (mode->Fs==48000 && mode->shortMdctSize==120) + { + data0 = fromOpus(data0); + if (data0<0) + return OPUS_INVALID_PACKET; + } + st->end = end = IMAX(1, mode->effEBands-2*(data0>>5)); + LM = (data0>>3)&0x3; + C = 1 + ((data0>>2)&0x1); + data++; + len--; + if (LM>mode->maxLM) + return OPUS_INVALID_PACKET; + if (frame_size < mode->shortMdctSize<shortMdctSize<maxLM;LM++) + if (mode->shortMdctSize<mode->maxLM) + return OPUS_BAD_ARG; + } + M=1<1275 || pcm==NULL) + return OPUS_BAD_ARG; + + N = M*mode->shortMdctSize; + c=0; do { + decode_mem[c] = st->_decode_mem + c*(DECODE_BUFFER_SIZE+overlap); + out_syn[c] = decode_mem[c]+DECODE_BUFFER_SIZE-N; + } while (++c mode->effEBands) + effEnd = mode->effEBands; + + if (data == NULL || len<=1) + { + celt_decode_lost(st, N, LM); + deemphasis(out_syn, pcm, N, CC, st->downsample, mode->preemph, st->preemph_memD, accum); + RESTORE_STACK; + return frame_size/st->downsample; + } + + /* Check if there are at least two packets received consecutively before + * turning on the pitch-based PLC */ + st->skip_plc = st->loss_count != 0; + + if (dec == NULL) + { + ec_dec_init(&_dec,(unsigned char*)data,len); + dec = &_dec; + } + + if (C==1) + { + for (i=0;i= total_bits) + silence = 1; + else if (tell==1) + silence = ec_dec_bit_logp(dec, 15); + else + silence = 0; + if (silence) + { + /* Pretend we've read all the remaining bits */ + tell = len*8; + dec->nbits_total+=tell-ec_tell(dec); + } + + postfilter_gain = 0; + postfilter_pitch = 0; + postfilter_tapset = 0; + if (start==0 && tell+16 <= total_bits) + { + if(ec_dec_bit_logp(dec, 1)) + { + int qg, octave; + octave = ec_dec_uint(dec, 6); + postfilter_pitch = (16< 0 && tell+3 <= total_bits) + { + isTransient = ec_dec_bit_logp(dec, 3); + tell = ec_tell(dec); + } + else + isTransient = 0; + + if (isTransient) + shortBlocks = M; + else + shortBlocks = 0; + + /* Decode the global flags (first symbols in the stream) */ + intra_ener = tell+3<=total_bits ? ec_dec_bit_logp(dec, 3) : 0; + /* Get band energies */ + unquant_coarse_energy(mode, start, end, oldBandE, + intra_ener, dec, C, LM); + + ALLOC(tf_res, nbEBands, int); + tf_decode(start, end, isTransient, tf_res, LM, dec); + + tell = ec_tell(dec); + spread_decision = SPREAD_NORMAL; + if (tell+4 <= total_bits) + spread_decision = ec_dec_icdf(dec, spread_icdf, 5); + + ALLOC(cap, nbEBands, int); + + init_caps(mode,cap,LM,C); + + ALLOC(offsets, nbEBands, int); + + dynalloc_logp = 6; + total_bits<<=BITRES; + tell = ec_tell_frac(dec); + for (i=start;i0) + dynalloc_logp = IMAX(2, dynalloc_logp-1); + } + + ALLOC(fine_quant, nbEBands, int); + alloc_trim = tell+(6<=2&&bits>=((LM+2)<rng, 0, + st->arch, st->disable_inv); + + if (anti_collapse_rsv > 0) + { + anti_collapse_on = ec_dec_bits(dec, 1); + } + + unquant_energy_finalise(mode, start, end, oldBandE, + fine_quant, fine_priority, len*8-ec_tell(dec), dec, C); + + if (anti_collapse_on) + anti_collapse(mode, X, collapse_masks, LM, C, N, + start, end, oldBandE, oldLogE, oldLogE2, pulses, st->rng, st->arch); + + if (silence) + { + for (i=0;idownsample, silence, st->arch); + + c=0; do { + st->postfilter_period=IMAX(st->postfilter_period, COMBFILTER_MINPERIOD); + st->postfilter_period_old=IMAX(st->postfilter_period_old, COMBFILTER_MINPERIOD); + comb_filter(out_syn[c], out_syn[c], st->postfilter_period_old, st->postfilter_period, mode->shortMdctSize, + st->postfilter_gain_old, st->postfilter_gain, st->postfilter_tapset_old, st->postfilter_tapset, + mode->window, overlap, st->arch); + if (LM!=0) + comb_filter(out_syn[c]+mode->shortMdctSize, out_syn[c]+mode->shortMdctSize, st->postfilter_period, postfilter_pitch, N-mode->shortMdctSize, + st->postfilter_gain, postfilter_gain, st->postfilter_tapset, postfilter_tapset, + mode->window, overlap, st->arch); + + } while (++cpostfilter_period_old = st->postfilter_period; + st->postfilter_gain_old = st->postfilter_gain; + st->postfilter_tapset_old = st->postfilter_tapset; + st->postfilter_period = postfilter_pitch; + st->postfilter_gain = postfilter_gain; + st->postfilter_tapset = postfilter_tapset; + if (LM!=0) + { + st->postfilter_period_old = st->postfilter_period; + st->postfilter_gain_old = st->postfilter_gain; + st->postfilter_tapset_old = st->postfilter_tapset; + } + + if (C==1) + OPUS_COPY(&oldBandE[nbEBands], oldBandE, nbEBands); + + /* In case start or end were to change */ + if (!isTransient) + { + opus_val16 max_background_increase; + OPUS_COPY(oldLogE2, oldLogE, 2*nbEBands); + OPUS_COPY(oldLogE, oldBandE, 2*nbEBands); + /* In normal circumstances, we only allow the noise floor to increase by + up to 2.4 dB/second, but when we're in DTX, we allow up to 6 dB + increase for each update.*/ + if (st->loss_count < 10) + max_background_increase = M*QCONST16(0.001f,DB_SHIFT); + else + max_background_increase = QCONST16(1.f,DB_SHIFT); + for (i=0;i<2*nbEBands;i++) + backgroundLogE[i] = MIN16(backgroundLogE[i] + max_background_increase, oldBandE[i]); + } else { + for (i=0;i<2*nbEBands;i++) + oldLogE[i] = MIN16(oldLogE[i], oldBandE[i]); + } + c=0; do + { + for (i=0;irng = dec->rng; + + deemphasis(out_syn, pcm, N, CC, st->downsample, mode->preemph, st->preemph_memD, accum); + st->loss_count = 0; + RESTORE_STACK; + if (ec_tell(dec) > 8*len) + return OPUS_INTERNAL_ERROR; + if(ec_get_error(dec)) + st->error = 1; + return frame_size/st->downsample; +} + + +#ifdef CUSTOM_MODES + +#ifdef FIXED_POINT +int opus_custom_decode(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, int len, opus_int16 * OPUS_RESTRICT pcm, int frame_size) +{ + return celt_decode_with_ec(st, data, len, pcm, frame_size, NULL, 0); +} + +#ifndef DISABLE_FLOAT_API +int opus_custom_decode_float(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, int len, float * OPUS_RESTRICT pcm, int frame_size) +{ + int j, ret, C, N; + VARDECL(opus_int16, out); + ALLOC_STACK; + + if (pcm==NULL) + return OPUS_BAD_ARG; + + C = st->channels; + N = frame_size; + + ALLOC(out, C*N, opus_int16); + ret=celt_decode_with_ec(st, data, len, out, frame_size, NULL, 0); + if (ret>0) + for (j=0;jchannels; + N = frame_size; + ALLOC(out, C*N, celt_sig); + + ret=celt_decode_with_ec(st, data, len, out, frame_size, NULL, 0); + + if (ret>0) + for (j=0;j=st->mode->nbEBands) + goto bad_arg; + st->start = value; + } + break; + case CELT_SET_END_BAND_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<1 || value>st->mode->nbEBands) + goto bad_arg; + st->end = value; + } + break; + case CELT_SET_CHANNELS_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<1 || value>2) + goto bad_arg; + st->stream_channels = value; + } + break; + case CELT_GET_AND_CLEAR_ERROR_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (value==NULL) + goto bad_arg; + *value=st->error; + st->error = 0; + } + break; + case OPUS_GET_LOOKAHEAD_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (value==NULL) + goto bad_arg; + *value = st->overlap/st->downsample; + } + break; + case OPUS_RESET_STATE: + { + int i; + opus_val16 *lpc, *oldBandE, *oldLogE, *oldLogE2; + lpc = (opus_val16*)(st->_decode_mem+(DECODE_BUFFER_SIZE+st->overlap)*st->channels); + oldBandE = lpc+st->channels*LPC_ORDER; + oldLogE = oldBandE + 2*st->mode->nbEBands; + oldLogE2 = oldLogE + 2*st->mode->nbEBands; + OPUS_CLEAR((char*)&st->DECODER_RESET_START, + opus_custom_decoder_get_size(st->mode, st->channels)- + ((char*)&st->DECODER_RESET_START - (char*)st)); + for (i=0;i<2*st->mode->nbEBands;i++) + oldLogE[i]=oldLogE2[i]=-QCONST16(28.f,DB_SHIFT); + st->skip_plc = 1; + } + break; + case OPUS_GET_PITCH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (value==NULL) + goto bad_arg; + *value = st->postfilter_period; + } + break; + case CELT_GET_MODE_REQUEST: + { + const CELTMode ** value = va_arg(ap, const CELTMode**); + if (value==0) + goto bad_arg; + *value=st->mode; + } + break; + case CELT_SET_SIGNALLING_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->signalling = value; + } + break; + case OPUS_GET_FINAL_RANGE_REQUEST: + { + opus_uint32 * value = va_arg(ap, opus_uint32 *); + if (value==0) + goto bad_arg; + *value=st->rng; + } + break; + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->disable_inv = value; + } + break; + case OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->disable_inv; + } + break; + default: + goto bad_request; + } + va_end(ap); + return OPUS_OK; +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +bad_request: + va_end(ap); + return OPUS_UNIMPLEMENTED; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_encoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_encoder.c new file mode 100644 index 000000000..d6f8afc20 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_encoder.c @@ -0,0 +1,2607 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2010 Xiph.Org Foundation + Copyright (c) 2008 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define CELT_ENCODER_C + +#include "cpu_support.h" +#include "os_support.h" +#include "mdct.h" +#include +#include "celt.h" +#include "pitch.h" +#include "bands.h" +#include "modes.h" +#include "entcode.h" +#include "quant_bands.h" +#include "rate.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "float_cast.h" +#include +#include "celt_lpc.h" +#include "vq.h" + + +/** Encoder state + @brief Encoder state + */ +struct OpusCustomEncoder { + const OpusCustomMode *mode; /**< Mode used by the encoder */ + int channels; + int stream_channels; + + int force_intra; + int clip; + int disable_pf; + int complexity; + int upsample; + int start, end; + + opus_int32 bitrate; + int vbr; + int signalling; + int constrained_vbr; /* If zero, VBR can do whatever it likes with the rate */ + int loss_rate; + int lsb_depth; + int lfe; + int disable_inv; + int arch; + + /* Everything beyond this point gets cleared on a reset */ +#define ENCODER_RESET_START rng + + opus_uint32 rng; + int spread_decision; + opus_val32 delayedIntra; + int tonal_average; + int lastCodedBands; + int hf_average; + int tapset_decision; + + int prefilter_period; + opus_val16 prefilter_gain; + int prefilter_tapset; +#ifdef RESYNTH + int prefilter_period_old; + opus_val16 prefilter_gain_old; + int prefilter_tapset_old; +#endif + int consec_transient; + AnalysisInfo analysis; + SILKInfo silk_info; + + opus_val32 preemph_memE[2]; + opus_val32 preemph_memD[2]; + + /* VBR-related parameters */ + opus_int32 vbr_reservoir; + opus_int32 vbr_drift; + opus_int32 vbr_offset; + opus_int32 vbr_count; + opus_val32 overlap_max; + opus_val16 stereo_saving; + int intensity; + opus_val16 *energy_mask; + opus_val16 spec_avg; + +#ifdef RESYNTH + /* +MAX_PERIOD/2 to make space for overlap */ + celt_sig syn_mem[2][2*MAX_PERIOD+MAX_PERIOD/2]; +#endif + + celt_sig in_mem[1]; /* Size = channels*mode->overlap */ + /* celt_sig prefilter_mem[], Size = channels*COMBFILTER_MAXPERIOD */ + /* opus_val16 oldBandE[], Size = channels*mode->nbEBands */ + /* opus_val16 oldLogE[], Size = channels*mode->nbEBands */ + /* opus_val16 oldLogE2[], Size = channels*mode->nbEBands */ + /* opus_val16 energyError[], Size = channels*mode->nbEBands */ +}; + +int celt_encoder_get_size(int channels) +{ + CELTMode *mode = opus_custom_mode_create(48000, 960, NULL); + return opus_custom_encoder_get_size(mode, channels); +} + +OPUS_CUSTOM_NOSTATIC int opus_custom_encoder_get_size(const CELTMode *mode, int channels) +{ + int size = sizeof(struct CELTEncoder) + + (channels*mode->overlap-1)*sizeof(celt_sig) /* celt_sig in_mem[channels*mode->overlap]; */ + + channels*COMBFILTER_MAXPERIOD*sizeof(celt_sig) /* celt_sig prefilter_mem[channels*COMBFILTER_MAXPERIOD]; */ + + 4*channels*mode->nbEBands*sizeof(opus_val16); /* opus_val16 oldBandE[channels*mode->nbEBands]; */ + /* opus_val16 oldLogE[channels*mode->nbEBands]; */ + /* opus_val16 oldLogE2[channels*mode->nbEBands]; */ + /* opus_val16 energyError[channels*mode->nbEBands]; */ + return size; +} + +#ifdef CUSTOM_MODES +CELTEncoder *opus_custom_encoder_create(const CELTMode *mode, int channels, int *error) +{ + int ret; + CELTEncoder *st = (CELTEncoder *)opus_alloc(opus_custom_encoder_get_size(mode, channels)); + /* init will handle the NULL case */ + ret = opus_custom_encoder_init(st, mode, channels); + if (ret != OPUS_OK) + { + opus_custom_encoder_destroy(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} +#endif /* CUSTOM_MODES */ + +static int opus_custom_encoder_init_arch(CELTEncoder *st, const CELTMode *mode, + int channels, int arch) +{ + if (channels < 0 || channels > 2) + return OPUS_BAD_ARG; + + if (st==NULL || mode==NULL) + return OPUS_ALLOC_FAIL; + + OPUS_CLEAR((char*)st, opus_custom_encoder_get_size(mode, channels)); + + st->mode = mode; + st->stream_channels = st->channels = channels; + + st->upsample = 1; + st->start = 0; + st->end = st->mode->effEBands; + st->signalling = 1; + st->arch = arch; + + st->constrained_vbr = 1; + st->clip = 1; + + st->bitrate = OPUS_BITRATE_MAX; + st->vbr = 0; + st->force_intra = 0; + st->complexity = 5; + st->lsb_depth=24; + + opus_custom_encoder_ctl(st, OPUS_RESET_STATE); + + return OPUS_OK; +} + +#ifdef CUSTOM_MODES +int opus_custom_encoder_init(CELTEncoder *st, const CELTMode *mode, int channels) +{ + return opus_custom_encoder_init_arch(st, mode, channels, opus_select_arch()); +} +#endif + +int celt_encoder_init(CELTEncoder *st, opus_int32 sampling_rate, int channels, + int arch) +{ + int ret; + ret = opus_custom_encoder_init_arch(st, + opus_custom_mode_create(48000, 960, NULL), channels, arch); + if (ret != OPUS_OK) + return ret; + st->upsample = resampling_factor(sampling_rate); + return OPUS_OK; +} + +#ifdef CUSTOM_MODES +void opus_custom_encoder_destroy(CELTEncoder *st) +{ + opus_free(st); +} +#endif /* CUSTOM_MODES */ + + +static int transient_analysis(const opus_val32 * OPUS_RESTRICT in, int len, int C, + opus_val16 *tf_estimate, int *tf_chan, int allow_weak_transients, + int *weak_transient) +{ + int i; + VARDECL(opus_val16, tmp); + opus_val32 mem0,mem1; + int is_transient = 0; + opus_int32 mask_metric = 0; + int c; + opus_val16 tf_max; + int len2; + /* Forward masking: 6.7 dB/ms. */ +#ifdef FIXED_POINT + int forward_shift = 4; +#else + opus_val16 forward_decay = QCONST16(.0625f,15); +#endif + /* Table of 6*64/x, trained on real data to minimize the average error */ + static const unsigned char inv_table[128] = { + 255,255,156,110, 86, 70, 59, 51, 45, 40, 37, 33, 31, 28, 26, 25, + 23, 22, 21, 20, 19, 18, 17, 16, 16, 15, 15, 14, 13, 13, 12, 12, + 12, 12, 11, 11, 11, 10, 10, 10, 9, 9, 9, 9, 9, 9, 8, 8, + 8, 8, 8, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, + }; + SAVE_STACK; + ALLOC(tmp, len, opus_val16); + + *weak_transient = 0; + /* For lower bitrates, let's be more conservative and have a forward masking + decay of 3.3 dB/ms. This avoids having to code transients at very low + bitrate (mostly for hybrid), which can result in unstable energy and/or + partial collapse. */ + if (allow_weak_transients) + { +#ifdef FIXED_POINT + forward_shift = 5; +#else + forward_decay = QCONST16(.03125f,15); +#endif + } + len2=len/2; + for (c=0;c=0;i--) + { + /* Backward masking: 13.9 dB/ms. */ +#ifdef FIXED_POINT + /* FIXME: Use PSHR16() instead */ + tmp[i] = mem0 + PSHR32(tmp[i]-mem0,3); +#else + tmp[i] = mem0 + MULT16_16_P15(QCONST16(0.125f,15),tmp[i]-mem0); +#endif + mem0 = tmp[i]; + maxE = MAX16(maxE, mem0); + } + /*for (i=0;i>1))); +#else + mean = celt_sqrt(mean * maxE*.5*len2); +#endif + /* Inverse of the mean energy in Q15+6 */ + norm = SHL32(EXTEND32(len2),6+14)/ADD32(EPSILON,SHR32(mean,1)); + /* Compute harmonic mean discarding the unreliable boundaries + The data is smooth, so we only take 1/4th of the samples */ + unmask=0; + /* We should never see NaNs here. If we find any, then something really bad happened and we better abort + before it does any damage later on. If these asserts are disabled (no hardening), then the table + lookup a few lines below (id = ...) is likely to crash dur to an out-of-bounds read. DO NOT FIX + that crash on NaN since it could result in a worse issue later on. */ + celt_assert(!celt_isnan(tmp[0])); + celt_assert(!celt_isnan(norm)); + for (i=12;imask_metric) + { + *tf_chan = c; + mask_metric = unmask; + } + } + is_transient = mask_metric>200; + /* For low bitrates, define "weak transients" that need to be + handled differently to avoid partial collapse. */ + if (allow_weak_transients && is_transient && mask_metric<600) { + is_transient = 0; + *weak_transient = 1; + } + /* Arbitrary metric for VBR boost */ + tf_max = MAX16(0,celt_sqrt(27*mask_metric)-42); + /* *tf_estimate = 1 + MIN16(1, sqrt(MAX16(0, tf_max-30))/20); */ + *tf_estimate = celt_sqrt(MAX32(0, SHL32(MULT16_16(QCONST16(0.0069,14),MIN16(163,tf_max)),14)-QCONST32(0.139,28))); + /*printf("%d %f\n", tf_max, mask_metric);*/ + RESTORE_STACK; +#ifdef FUZZING + is_transient = rand()&0x1; +#endif + /*printf("%d %f %d\n", is_transient, (float)*tf_estimate, tf_max);*/ + return is_transient; +} + +/* Looks for sudden increases of energy to decide whether we need to patch + the transient decision */ +static int patch_transient_decision(opus_val16 *newE, opus_val16 *oldE, int nbEBands, + int start, int end, int C) +{ + int i, c; + opus_val32 mean_diff=0; + opus_val16 spread_old[26]; + /* Apply an aggressive (-6 dB/Bark) spreading function to the old frame to + avoid false detection caused by irrelevant bands */ + if (C==1) + { + spread_old[start] = oldE[start]; + for (i=start+1;i=start;i--) + spread_old[i] = MAX16(spread_old[i], spread_old[i+1]-QCONST16(1.0f, DB_SHIFT)); + /* Compute mean increase */ + c=0; do { + for (i=IMAX(2,start);i QCONST16(1.f, DB_SHIFT); +} + +/** Apply window and compute the MDCT for all sub-frames and + all channels in a frame */ +static void compute_mdcts(const CELTMode *mode, int shortBlocks, celt_sig * OPUS_RESTRICT in, + celt_sig * OPUS_RESTRICT out, int C, int CC, int LM, int upsample, + int arch) +{ + const int overlap = mode->overlap; + int N; + int B; + int shift; + int i, b, c; + if (shortBlocks) + { + B = shortBlocks; + N = mode->shortMdctSize; + shift = mode->maxLM; + } else { + B = 1; + N = mode->shortMdctSize<maxLM-LM; + } + c=0; do { + for (b=0;bmdct, in+c*(B*N+overlap)+b*N, + &out[b+c*N*B], mode->window, overlap, shift, B, + arch); + } + } while (++ceBands[len]-m->eBands[len-1])<eBands[len]-m->eBands[len-1])<eBands[i+1]-m->eBands[i])<eBands[i+1]-m->eBands[i])==1; + OPUS_COPY(tmp, &X[tf_chan*N0 + (m->eBands[i]<eBands[i]<>LM, 1<>k, 1<=0;i--) + { + if (tf_res[i+1] == 1) + tf_res[i] = path1[i+1]; + else + tf_res[i] = path0[i+1]; + } + /*printf("%d %f\n", *tf_sum, tf_estimate);*/ + RESTORE_STACK; +#ifdef FUZZING + tf_select = rand()&0x1; + tf_res[0] = rand()&0x1; + for (i=1;istorage*8; + tell = ec_tell(enc); + logp = isTransient ? 2 : 4; + /* Reserve space to code the tf_select decision. */ + tf_select_rsv = LM>0 && tell+logp+1 <= budget; + budget -= tf_select_rsv; + curr = tf_changed = 0; + for (i=start;i> 10; + trim = QCONST16(4.f, 8) + QCONST16(1.f/16.f, 8)*frac; + } + if (C==2) + { + opus_val16 sum = 0; /* Q10 */ + opus_val16 minXC; /* Q10 */ + /* Compute inter-channel correlation for low frequencies */ + for (i=0;i<8;i++) + { + opus_val32 partial; + partial = celt_inner_prod(&X[m->eBands[i]<eBands[i]<eBands[i+1]-m->eBands[i])<eBands[i]<eBands[i]<eBands[i+1]-m->eBands[i])<nbEBands]*(opus_int32)(2+2*i-end); + } + } while (++cvalid) + { + trim -= MAX16(-QCONST16(2.f, 8), MIN16(QCONST16(2.f, 8), + (opus_val16)(QCONST16(2.f, 8)*(analysis->tonality_slope+.05f)))); + } +#else + (void)analysis; +#endif + +#ifdef FIXED_POINT + trim_index = PSHR32(trim, 8); +#else + trim_index = (int)floor(.5f+trim); +#endif + trim_index = IMAX(0, IMIN(10, trim_index)); + /*printf("%d\n", trim_index);*/ +#ifdef FUZZING + trim_index = rand()%11; +#endif + return trim_index; +} + +static int stereo_analysis(const CELTMode *m, const celt_norm *X, + int LM, int N0) +{ + int i; + int thetas; + opus_val32 sumLR = EPSILON, sumMS = EPSILON; + + /* Use the L1 norm to model the entropy of the L/R signal vs the M/S signal */ + for (i=0;i<13;i++) + { + int j; + for (j=m->eBands[i]<eBands[i+1]<eBands[13]<<(LM+1))+thetas, sumMS) + > MULT16_32_Q15(m->eBands[13]<<(LM+1), sumLR); +} + +#define MSWAP(a,b) do {opus_val16 tmp = a;a=b;b=tmp;} while(0) +static opus_val16 median_of_5(const opus_val16 *x) +{ + opus_val16 t0, t1, t2, t3, t4; + t2 = x[2]; + if (x[0] > x[1]) + { + t0 = x[1]; + t1 = x[0]; + } else { + t0 = x[0]; + t1 = x[1]; + } + if (x[3] > x[4]) + { + t3 = x[4]; + t4 = x[3]; + } else { + t3 = x[3]; + t4 = x[4]; + } + if (t0 > t3) + { + MSWAP(t0, t3); + MSWAP(t1, t4); + } + if (t2 > t1) + { + if (t1 < t3) + return MIN16(t2, t3); + else + return MIN16(t4, t1); + } else { + if (t2 < t3) + return MIN16(t1, t3); + else + return MIN16(t2, t4); + } +} + +static opus_val16 median_of_3(const opus_val16 *x) +{ + opus_val16 t0, t1, t2; + if (x[0] > x[1]) + { + t0 = x[1]; + t1 = x[0]; + } else { + t0 = x[0]; + t1 = x[1]; + } + t2 = x[2]; + if (t1 < t2) + return t1; + else if (t0 < t2) + return t2; + else + return t0; +} + +static opus_val16 dynalloc_analysis(const opus_val16 *bandLogE, const opus_val16 *bandLogE2, + int nbEBands, int start, int end, int C, int *offsets, int lsb_depth, const opus_int16 *logN, + int isTransient, int vbr, int constrained_vbr, const opus_int16 *eBands, int LM, + int effectiveBytes, opus_int32 *tot_boost_, int lfe, opus_val16 *surround_dynalloc, + AnalysisInfo *analysis, int *importance, int *spread_weight) +{ + int i, c; + opus_int32 tot_boost=0; + opus_val16 maxDepth; + VARDECL(opus_val16, follower); + VARDECL(opus_val16, noise_floor); + SAVE_STACK; + ALLOC(follower, C*nbEBands, opus_val16); + ALLOC(noise_floor, C*nbEBands, opus_val16); + OPUS_CLEAR(offsets, nbEBands); + /* Dynamic allocation code */ + maxDepth=-QCONST16(31.9f, DB_SHIFT); + for (i=0;i=0;i--) + mask[i] = MAX16(mask[i], mask[i+1] - QCONST16(3.f, DB_SHIFT)); + for (i=0;i> shift; + } + /*for (i=0;i 50 && LM>=1 && !lfe) + { + int last=0; + c=0;do + { + opus_val16 offset; + opus_val16 tmp; + opus_val16 *f; + f = &follower[c*nbEBands]; + f[0] = bandLogE2[c*nbEBands]; + for (i=1;i bandLogE2[c*nbEBands+i-1]+QCONST16(.5f,DB_SHIFT)) + last=i; + f[i] = MIN16(f[i-1]+QCONST16(1.5f,DB_SHIFT), bandLogE2[c*nbEBands+i]); + } + for (i=last-1;i>=0;i--) + f[i] = MIN16(f[i], MIN16(f[i+1]+QCONST16(2.f,DB_SHIFT), bandLogE2[c*nbEBands+i])); + + /* Combine with a median filter to avoid dynalloc triggering unnecessarily. + The "offset" value controls how conservative we are -- a higher offset + reduces the impact of the median filter and makes dynalloc use more bits. */ + offset = QCONST16(1.f, DB_SHIFT); + for (i=2;i=12) + follower[i] = HALF16(follower[i]); + } +#ifdef DISABLE_FLOAT_API + (void)analysis; +#else + if (analysis->valid) + { + for (i=start;ileak_boost[i]; + } +#endif + for (i=start;i 48) { + boost = (int)SHR32(EXTEND32(follower[i])*8,DB_SHIFT); + boost_bits = (boost*width<>BITRES>>3 > 2*effectiveBytes/3) + { + opus_int32 cap = ((2*effectiveBytes/3)<mode; + overlap = mode->overlap; + ALLOC(_pre, CC*(N+COMBFILTER_MAXPERIOD), celt_sig); + + pre[0] = _pre; + pre[1] = _pre + (N+COMBFILTER_MAXPERIOD); + + + c=0; do { + OPUS_COPY(pre[c], prefilter_mem+c*COMBFILTER_MAXPERIOD, COMBFILTER_MAXPERIOD); + OPUS_COPY(pre[c]+COMBFILTER_MAXPERIOD, in+c*(N+overlap)+overlap, N); + } while (++c>1, opus_val16); + + pitch_downsample(pre, pitch_buf, COMBFILTER_MAXPERIOD+N, CC, st->arch); + /* Don't search for the fir last 1.5 octave of the range because + there's too many false-positives due to short-term correlation */ + pitch_search(pitch_buf+(COMBFILTER_MAXPERIOD>>1), pitch_buf, N, + COMBFILTER_MAXPERIOD-3*COMBFILTER_MINPERIOD, &pitch_index, + st->arch); + pitch_index = COMBFILTER_MAXPERIOD-pitch_index; + + gain1 = remove_doubling(pitch_buf, COMBFILTER_MAXPERIOD, COMBFILTER_MINPERIOD, + N, &pitch_index, st->prefilter_period, st->prefilter_gain, st->arch); + if (pitch_index > COMBFILTER_MAXPERIOD-2) + pitch_index = COMBFILTER_MAXPERIOD-2; + gain1 = MULT16_16_Q15(QCONST16(.7f,15),gain1); + /*printf("%d %d %f %f\n", pitch_change, pitch_index, gain1, st->analysis.tonality);*/ + if (st->loss_rate>2) + gain1 = HALF32(gain1); + if (st->loss_rate>4) + gain1 = HALF32(gain1); + if (st->loss_rate>8) + gain1 = 0; + } else { + gain1 = 0; + pitch_index = COMBFILTER_MINPERIOD; + } +#ifndef DISABLE_FLOAT_API + if (analysis->valid) + gain1 = (opus_val16)(gain1 * analysis->max_pitch_ratio); +#else + (void)analysis; +#endif + /* Gain threshold for enabling the prefilter/postfilter */ + pf_threshold = QCONST16(.2f,15); + + /* Adjusting the threshold based on rate and continuity */ + if (abs(pitch_index-st->prefilter_period)*10>pitch_index) + pf_threshold += QCONST16(.2f,15); + if (nbAvailableBytes<25) + pf_threshold += QCONST16(.1f,15); + if (nbAvailableBytes<35) + pf_threshold += QCONST16(.1f,15); + if (st->prefilter_gain > QCONST16(.4f,15)) + pf_threshold -= QCONST16(.1f,15); + if (st->prefilter_gain > QCONST16(.55f,15)) + pf_threshold -= QCONST16(.1f,15); + + /* Hard threshold at 0.2 */ + pf_threshold = MAX16(pf_threshold, QCONST16(.2f,15)); + if (gain1prefilter_gain)prefilter_gain; + +#ifdef FIXED_POINT + qg = ((gain1+1536)>>10)/3-1; +#else + qg = (int)floor(.5f+gain1*32/3)-1; +#endif + qg = IMAX(0, IMIN(7, qg)); + gain1 = QCONST16(0.09375f,15)*(qg+1); + pf_on = 1; + } + /*printf("%d %f\n", pitch_index, gain1);*/ + + c=0; do { + int offset = mode->shortMdctSize-overlap; + st->prefilter_period=IMAX(st->prefilter_period, COMBFILTER_MINPERIOD); + OPUS_COPY(in+c*(N+overlap), st->in_mem+c*(overlap), overlap); + if (offset) + comb_filter(in+c*(N+overlap)+overlap, pre[c]+COMBFILTER_MAXPERIOD, + st->prefilter_period, st->prefilter_period, offset, -st->prefilter_gain, -st->prefilter_gain, + st->prefilter_tapset, st->prefilter_tapset, NULL, 0, st->arch); + + comb_filter(in+c*(N+overlap)+overlap+offset, pre[c]+COMBFILTER_MAXPERIOD+offset, + st->prefilter_period, pitch_index, N-offset, -st->prefilter_gain, -gain1, + st->prefilter_tapset, prefilter_tapset, mode->window, overlap, st->arch); + OPUS_COPY(st->in_mem+c*(overlap), in+c*(N+overlap)+N, overlap); + + if (N>COMBFILTER_MAXPERIOD) + { + OPUS_COPY(prefilter_mem+c*COMBFILTER_MAXPERIOD, pre[c]+N, COMBFILTER_MAXPERIOD); + } else { + OPUS_MOVE(prefilter_mem+c*COMBFILTER_MAXPERIOD, prefilter_mem+c*COMBFILTER_MAXPERIOD+N, COMBFILTER_MAXPERIOD-N); + OPUS_COPY(prefilter_mem+c*COMBFILTER_MAXPERIOD+COMBFILTER_MAXPERIOD-N, pre[c]+COMBFILTER_MAXPERIOD, N); + } + } while (++cnbEBands; + eBands = mode->eBands; + + coded_bands = lastCodedBands ? lastCodedBands : nbEBands; + coded_bins = eBands[coded_bands]<analysis.activity, st->analysis.tonality, tf_estimate, st->stereo_saving, tot_boost, coded_bands);*/ +#ifndef DISABLE_FLOAT_API + if (analysis->valid && analysis->activity<.4) + target -= (opus_int32)((coded_bins<activity)); +#endif + /* Stereo savings */ + if (C==2) + { + int coded_stereo_bands; + int coded_stereo_dof; + opus_val16 max_frac; + coded_stereo_bands = IMIN(intensity, coded_bands); + coded_stereo_dof = (eBands[coded_stereo_bands]<valid && !lfe) + { + opus_int32 tonal_target; + float tonal; + + /* Tonality boost (compensating for the average). */ + tonal = MAX16(0.f,analysis->tonality-.15f)-0.12f; + tonal_target = target + (opus_int32)((coded_bins<tonality, tonal);*/ + target = tonal_target; + } +#else + (void)analysis; + (void)pitch_change; +#endif + + if (has_surround_mask&&!lfe) + { + opus_int32 surround_target = target + (opus_int32)SHR32(MULT16_16(surround_masking,coded_bins<end, st->intensity, surround_target, target, st->bitrate);*/ + target = IMAX(target/4, surround_target); + } + + { + opus_int32 floor_depth; + int bins; + bins = eBands[nbEBands-2]<>2); + target = IMIN(target, floor_depth); + /*printf("%f %d\n", maxDepth, floor_depth);*/ + } + + /* Make VBR less aggressive for constrained VBR because we can't keep a higher bitrate + for long. Needs tuning. */ + if ((!has_surround_mask||lfe) && constrained_vbr) + { + target = base_target + (opus_int32)MULT16_32_Q15(QCONST16(0.67f, 15), target-base_target); + } + + if (!has_surround_mask && tf_estimate < QCONST16(.2f, 14)) + { + opus_val16 amount; + opus_val16 tvbr_factor; + amount = MULT16_16_Q15(QCONST16(.0000031f, 30), IMAX(0, IMIN(32000, 96000-bitrate))); + tvbr_factor = SHR32(MULT16_16(temporal_vbr, amount), DB_SHIFT); + target += (opus_int32)MULT16_32_Q15(tvbr_factor, target); + } + + /* Don't allow more than doubling the rate */ + target = IMIN(2*base_target, target); + + return target; +} + +int celt_encode_with_ec(CELTEncoder * OPUS_RESTRICT st, const opus_val16 * pcm, int frame_size, unsigned char *compressed, int nbCompressedBytes, ec_enc *enc) +{ + int i, c, N; + opus_int32 bits; + ec_enc _enc; + VARDECL(celt_sig, in); + VARDECL(celt_sig, freq); + VARDECL(celt_norm, X); + VARDECL(celt_ener, bandE); + VARDECL(opus_val16, bandLogE); + VARDECL(opus_val16, bandLogE2); + VARDECL(int, fine_quant); + VARDECL(opus_val16, error); + VARDECL(int, pulses); + VARDECL(int, cap); + VARDECL(int, offsets); + VARDECL(int, importance); + VARDECL(int, spread_weight); + VARDECL(int, fine_priority); + VARDECL(int, tf_res); + VARDECL(unsigned char, collapse_masks); + celt_sig *prefilter_mem; + opus_val16 *oldBandE, *oldLogE, *oldLogE2, *energyError; + int shortBlocks=0; + int isTransient=0; + const int CC = st->channels; + const int C = st->stream_channels; + int LM, M; + int tf_select; + int nbFilledBytes, nbAvailableBytes; + int start; + int end; + int effEnd; + int codedBands; + int alloc_trim; + int pitch_index=COMBFILTER_MINPERIOD; + opus_val16 gain1 = 0; + int dual_stereo=0; + int effectiveBytes; + int dynalloc_logp; + opus_int32 vbr_rate; + opus_int32 total_bits; + opus_int32 total_boost; + opus_int32 balance; + opus_int32 tell; + opus_int32 tell0_frac; + int prefilter_tapset=0; + int pf_on; + int anti_collapse_rsv; + int anti_collapse_on=0; + int silence=0; + int tf_chan = 0; + opus_val16 tf_estimate; + int pitch_change=0; + opus_int32 tot_boost; + opus_val32 sample_max; + opus_val16 maxDepth; + const OpusCustomMode *mode; + int nbEBands; + int overlap; + const opus_int16 *eBands; + int secondMdct; + int signalBandwidth; + int transient_got_disabled=0; + opus_val16 surround_masking=0; + opus_val16 temporal_vbr=0; + opus_val16 surround_trim = 0; + opus_int32 equiv_rate; + int hybrid; + int weak_transient = 0; + int enable_tf_analysis; + VARDECL(opus_val16, surround_dynalloc); + ALLOC_STACK; + + mode = st->mode; + nbEBands = mode->nbEBands; + overlap = mode->overlap; + eBands = mode->eBands; + start = st->start; + end = st->end; + hybrid = start != 0; + tf_estimate = 0; + if (nbCompressedBytes<2 || pcm==NULL) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + frame_size *= st->upsample; + for (LM=0;LM<=mode->maxLM;LM++) + if (mode->shortMdctSize<mode->maxLM) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + M=1<shortMdctSize; + + prefilter_mem = st->in_mem+CC*(overlap); + oldBandE = (opus_val16*)(st->in_mem+CC*(overlap+COMBFILTER_MAXPERIOD)); + oldLogE = oldBandE + CC*nbEBands; + oldLogE2 = oldLogE + CC*nbEBands; + energyError = oldLogE2 + CC*nbEBands; + + if (enc==NULL) + { + tell0_frac=tell=1; + nbFilledBytes=0; + } else { + tell0_frac=ec_tell_frac(enc); + tell=ec_tell(enc); + nbFilledBytes=(tell+4)>>3; + } + +#ifdef CUSTOM_MODES + if (st->signalling && enc==NULL) + { + int tmp = (mode->effEBands-end)>>1; + end = st->end = IMAX(1, mode->effEBands-tmp); + compressed[0] = tmp<<5; + compressed[0] |= LM<<3; + compressed[0] |= (C==2)<<2; + /* Convert "standard mode" to Opus header */ + if (mode->Fs==48000 && mode->shortMdctSize==120) + { + int c0 = toOpus(compressed[0]); + if (c0<0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + compressed[0] = c0; + } + compressed++; + nbCompressedBytes--; + } +#else + celt_assert(st->signalling==0); +#endif + + /* Can't produce more than 1275 output bytes */ + nbCompressedBytes = IMIN(nbCompressedBytes,1275); + nbAvailableBytes = nbCompressedBytes - nbFilledBytes; + + if (st->vbr && st->bitrate!=OPUS_BITRATE_MAX) + { + opus_int32 den=mode->Fs>>BITRES; + vbr_rate=(st->bitrate*frame_size+(den>>1))/den; +#ifdef CUSTOM_MODES + if (st->signalling) + vbr_rate -= 8<>(3+BITRES); + } else { + opus_int32 tmp; + vbr_rate = 0; + tmp = st->bitrate*frame_size; + if (tell>1) + tmp += tell; + if (st->bitrate!=OPUS_BITRATE_MAX) + nbCompressedBytes = IMAX(2, IMIN(nbCompressedBytes, + (tmp+4*mode->Fs)/(8*mode->Fs)-!!st->signalling)); + effectiveBytes = nbCompressedBytes - nbFilledBytes; + } + equiv_rate = ((opus_int32)nbCompressedBytes*8*50 << (3-LM)) - (40*C+20)*((400>>LM) - 50); + if (st->bitrate != OPUS_BITRATE_MAX) + equiv_rate = IMIN(equiv_rate, st->bitrate - (40*C+20)*((400>>LM) - 50)); + + if (enc==NULL) + { + ec_enc_init(&_enc, compressed, nbCompressedBytes); + enc = &_enc; + } + + if (vbr_rate>0) + { + /* Computes the max bit-rate allowed in VBR mode to avoid violating the + target rate and buffering. + We must do this up front so that bust-prevention logic triggers + correctly if we don't have enough bits. */ + if (st->constrained_vbr) + { + opus_int32 vbr_bound; + opus_int32 max_allowed; + /* We could use any multiple of vbr_rate as bound (depending on the + delay). + This is clamped to ensure we use at least two bytes if the encoder + was entirely empty, but to allow 0 in hybrid mode. */ + vbr_bound = vbr_rate; + max_allowed = IMIN(IMAX(tell==1?2:0, + (vbr_rate+vbr_bound-st->vbr_reservoir)>>(BITRES+3)), + nbAvailableBytes); + if(max_allowed < nbAvailableBytes) + { + nbCompressedBytes = nbFilledBytes+max_allowed; + nbAvailableBytes = max_allowed; + ec_enc_shrink(enc, nbCompressedBytes); + } + } + } + total_bits = nbCompressedBytes*8; + + effEnd = end; + if (effEnd > mode->effEBands) + effEnd = mode->effEBands; + + ALLOC(in, CC*(N+overlap), celt_sig); + + sample_max=MAX32(st->overlap_max, celt_maxabs16(pcm, C*(N-overlap)/st->upsample)); + st->overlap_max=celt_maxabs16(pcm+C*(N-overlap)/st->upsample, C*overlap/st->upsample); + sample_max=MAX32(sample_max, st->overlap_max); +#ifdef FIXED_POINT + silence = (sample_max==0); +#else + silence = (sample_max <= (opus_val16)1/(1<lsb_depth)); +#endif +#ifdef FUZZING + if ((rand()&0x3F)==0) + silence = 1; +#endif + if (tell==1) + ec_enc_bit_logp(enc, silence, 15); + else + silence=0; + if (silence) + { + /*In VBR mode there is no need to send more than the minimum. */ + if (vbr_rate>0) + { + effectiveBytes=nbCompressedBytes=IMIN(nbCompressedBytes, nbFilledBytes+2); + total_bits=nbCompressedBytes*8; + nbAvailableBytes=2; + ec_enc_shrink(enc, nbCompressedBytes); + } + /* Pretend we've filled all the remaining bits with zeros + (that's what the initialiser did anyway) */ + tell = nbCompressedBytes*8; + enc->nbits_total+=tell-ec_tell(enc); + } + c=0; do { + int need_clip=0; +#ifndef FIXED_POINT + need_clip = st->clip && sample_max>65536.f; +#endif + celt_preemphasis(pcm+c, in+c*(N+overlap)+overlap, N, CC, st->upsample, + mode->preemph, st->preemph_memE+c, need_clip); + } while (++clfe&&nbAvailableBytes>3) || nbAvailableBytes>12*C) && !hybrid && !silence && !st->disable_pf + && st->complexity >= 5; + + prefilter_tapset = st->tapset_decision; + pf_on = run_prefilter(st, in, prefilter_mem, CC, N, prefilter_tapset, &pitch_index, &gain1, &qg, enabled, nbAvailableBytes, &st->analysis); + if ((gain1 > QCONST16(.4f,15) || st->prefilter_gain > QCONST16(.4f,15)) && (!st->analysis.valid || st->analysis.tonality > .3) + && (pitch_index > 1.26*st->prefilter_period || pitch_index < .79*st->prefilter_period)) + pitch_change = 1; + if (pf_on==0) + { + if(!hybrid && tell+16<=total_bits) + ec_enc_bit_logp(enc, 0, 1); + } else { + /*This block is not gated by a total bits check only because + of the nbAvailableBytes check above.*/ + int octave; + ec_enc_bit_logp(enc, 1, 1); + pitch_index += 1; + octave = EC_ILOG(pitch_index)-5; + ec_enc_uint(enc, octave, 6); + ec_enc_bits(enc, pitch_index-(16<complexity >= 1 && !st->lfe) + { + /* Reduces the likelihood of energy instability on fricatives at low bitrate + in hybrid mode. It seems like we still want to have real transients on vowels + though (small SILK quantization offset value). */ + int allow_weak_transients = hybrid && effectiveBytes<15 && st->silk_info.signalType != 2; + isTransient = transient_analysis(in, N+overlap, CC, + &tf_estimate, &tf_chan, allow_weak_transients, &weak_transient); + } + if (LM>0 && ec_tell(enc)+3<=total_bits) + { + if (isTransient) + shortBlocks = M; + } else { + isTransient = 0; + transient_got_disabled=1; + } + + ALLOC(freq, CC*N, celt_sig); /**< Interleaved signal MDCTs */ + ALLOC(bandE,nbEBands*CC, celt_ener); + ALLOC(bandLogE,nbEBands*CC, opus_val16); + + secondMdct = shortBlocks && st->complexity>=8; + ALLOC(bandLogE2, C*nbEBands, opus_val16); + if (secondMdct) + { + compute_mdcts(mode, 0, in, freq, C, CC, LM, st->upsample, st->arch); + compute_band_energies(mode, freq, bandE, effEnd, C, LM, st->arch); + amp2Log2(mode, effEnd, end, bandE, bandLogE2, C); + for (i=0;iupsample, st->arch); + /* This should catch any NaN in the CELT input. Since we're not supposed to see any (they're filtered + at the Opus layer), just abort. */ + celt_assert(!celt_isnan(freq[0]) && (C==1 || !celt_isnan(freq[N]))); + if (CC==2&&C==1) + tf_chan = 0; + compute_band_energies(mode, freq, bandE, effEnd, C, LM, st->arch); + + if (st->lfe) + { + for (i=2;ienergy_mask&&!st->lfe) + { + int mask_end; + int midband; + int count_dynalloc; + opus_val32 mask_avg=0; + opus_val32 diff=0; + int count=0; + mask_end = IMAX(2,st->lastCodedBands); + for (c=0;cenergy_mask[nbEBands*c+i], + QCONST16(.25f, DB_SHIFT)), -QCONST16(2.0f, DB_SHIFT)); + if (mask > 0) + mask = HALF16(mask); + mask_avg += MULT16_16(mask, eBands[i+1]-eBands[i]); + count += eBands[i+1]-eBands[i]; + diff += MULT16_16(mask, 1+2*i-mask_end); + } + } + celt_assert(count>0); + mask_avg = DIV32_16(mask_avg,count); + mask_avg += QCONST16(.2f, DB_SHIFT); + diff = diff*6/(C*(mask_end-1)*(mask_end+1)*mask_end); + /* Again, being conservative */ + diff = HALF32(diff); + diff = MAX32(MIN32(diff, QCONST32(.031f, DB_SHIFT)), -QCONST32(.031f, DB_SHIFT)); + /* Find the band that's in the middle of the coded spectrum */ + for (midband=0;eBands[midband+1] < eBands[mask_end]/2;midband++); + count_dynalloc=0; + for(i=0;ienergy_mask[i], st->energy_mask[nbEBands+i]); + else + unmask = st->energy_mask[i]; + unmask = MIN16(unmask, QCONST16(.0f, DB_SHIFT)); + unmask -= lin; + if (unmask > QCONST16(.25f, DB_SHIFT)) + { + surround_dynalloc[i] = unmask - QCONST16(.25f, DB_SHIFT); + count_dynalloc++; + } + } + if (count_dynalloc>=3) + { + /* If we need dynalloc in many bands, it's probably because our + initial masking rate was too low. */ + mask_avg += QCONST16(.25f, DB_SHIFT); + if (mask_avg>0) + { + /* Something went really wrong in the original calculations, + disabling masking. */ + mask_avg = 0; + diff = 0; + OPUS_CLEAR(surround_dynalloc, mask_end); + } else { + for(i=0;ilfe) + { + opus_val16 follow=-QCONST16(10.0f,DB_SHIFT); + opus_val32 frame_avg=0; + opus_val16 offset = shortBlocks?HALF16(SHL16(LM, DB_SHIFT)):0; + for(i=start;ispec_avg); + temporal_vbr = MIN16(QCONST16(3.f, DB_SHIFT), MAX16(-QCONST16(1.5f, DB_SHIFT), temporal_vbr)); + st->spec_avg += MULT16_16_Q15(QCONST16(.02f, 15), temporal_vbr); + } + /*for (i=0;i<21;i++) + printf("%f ", bandLogE[i]); + printf("\n");*/ + + if (!secondMdct) + { + OPUS_COPY(bandLogE2, bandLogE, C*nbEBands); + } + + /* Last chance to catch any transient we might have missed in the + time-domain analysis */ + if (LM>0 && ec_tell(enc)+3<=total_bits && !isTransient && st->complexity>=5 && !st->lfe && !hybrid) + { + if (patch_transient_decision(bandLogE, oldBandE, nbEBands, start, end, C)) + { + isTransient = 1; + shortBlocks = M; + compute_mdcts(mode, shortBlocks, in, freq, C, CC, LM, st->upsample, st->arch); + compute_band_energies(mode, freq, bandE, effEnd, C, LM, st->arch); + amp2Log2(mode, effEnd, end, bandE, bandLogE, C); + /* Compensate for the scaling of short vs long mdcts */ + for (i=0;i0 && ec_tell(enc)+3<=total_bits) + ec_enc_bit_logp(enc, isTransient, 3); + + ALLOC(X, C*N, celt_norm); /**< Interleaved normalised MDCTs */ + + /* Band normalisation */ + normalise_bands(mode, freq, X, bandE, effEnd, C, M); + + enable_tf_analysis = effectiveBytes>=15*C && !hybrid && st->complexity>=2 && !st->lfe; + + ALLOC(offsets, nbEBands, int); + ALLOC(importance, nbEBands, int); + ALLOC(spread_weight, nbEBands, int); + + maxDepth = dynalloc_analysis(bandLogE, bandLogE2, nbEBands, start, end, C, offsets, + st->lsb_depth, mode->logN, isTransient, st->vbr, st->constrained_vbr, + eBands, LM, effectiveBytes, &tot_boost, st->lfe, surround_dynalloc, &st->analysis, importance, spread_weight); + + ALLOC(tf_res, nbEBands, int); + /* Disable variable tf resolution for hybrid and at very low bitrate */ + if (enable_tf_analysis) + { + int lambda; + lambda = IMAX(80, 20480/effectiveBytes + 2); + tf_select = tf_analysis(mode, effEnd, isTransient, tf_res, lambda, X, N, LM, tf_estimate, tf_chan, importance); + for (i=effEnd;isilk_info.signalType != 2) + { + /* For low bitrate hybrid, we force temporal resolution to 5 ms rather than 2.5 ms. */ + for (i=0;iforce_intra, + &st->delayedIntra, st->complexity >= 4, st->loss_rate, st->lfe); + + tf_encode(start, end, isTransient, tf_res, LM, tf_select, enc); + + if (ec_tell(enc)+4<=total_bits) + { + if (st->lfe) + { + st->tapset_decision = 0; + st->spread_decision = SPREAD_NORMAL; + } else if (hybrid) + { + if (st->complexity == 0) + st->spread_decision = SPREAD_NONE; + else if (isTransient) + st->spread_decision = SPREAD_NORMAL; + else + st->spread_decision = SPREAD_AGGRESSIVE; + } else if (shortBlocks || st->complexity < 3 || nbAvailableBytes < 10*C) + { + if (st->complexity == 0) + st->spread_decision = SPREAD_NONE; + else + st->spread_decision = SPREAD_NORMAL; + } else { + /* Disable new spreading+tapset estimator until we can show it works + better than the old one. So far it seems like spreading_decision() + works best. */ +#if 0 + if (st->analysis.valid) + { + static const opus_val16 spread_thresholds[3] = {-QCONST16(.6f, 15), -QCONST16(.2f, 15), -QCONST16(.07f, 15)}; + static const opus_val16 spread_histeresis[3] = {QCONST16(.15f, 15), QCONST16(.07f, 15), QCONST16(.02f, 15)}; + static const opus_val16 tapset_thresholds[2] = {QCONST16(.0f, 15), QCONST16(.15f, 15)}; + static const opus_val16 tapset_histeresis[2] = {QCONST16(.1f, 15), QCONST16(.05f, 15)}; + st->spread_decision = hysteresis_decision(-st->analysis.tonality, spread_thresholds, spread_histeresis, 3, st->spread_decision); + st->tapset_decision = hysteresis_decision(st->analysis.tonality_slope, tapset_thresholds, tapset_histeresis, 2, st->tapset_decision); + } else +#endif + { + st->spread_decision = spreading_decision(mode, X, + &st->tonal_average, st->spread_decision, &st->hf_average, + &st->tapset_decision, pf_on&&!shortBlocks, effEnd, C, M, spread_weight); + } + /*printf("%d %d\n", st->tapset_decision, st->spread_decision);*/ + /*printf("%f %d %f %d\n\n", st->analysis.tonality, st->spread_decision, st->analysis.tonality_slope, st->tapset_decision);*/ + } + ec_enc_icdf(enc, st->spread_decision, spread_icdf, 5); + } + + /* For LFE, everything interesting is in the first band */ + if (st->lfe) + offsets[0] = IMIN(8, effectiveBytes/3); + ALLOC(cap, nbEBands, int); + init_caps(mode,cap,LM,C); + + dynalloc_logp = 6; + total_bits<<=BITRES; + total_boost = 0; + tell = ec_tell_frac(enc); + for (i=start;iintensity = hysteresis_decision((opus_val16)(equiv_rate/1000), + intensity_thresholds, intensity_histeresis, 21, st->intensity); + st->intensity = IMIN(end,IMAX(start, st->intensity)); + } + + alloc_trim = 5; + if (tell+(6< 0 || st->lfe) + { + st->stereo_saving = 0; + alloc_trim = 5; + } else { + alloc_trim = alloc_trim_analysis(mode, X, bandLogE, + end, LM, C, N, &st->analysis, &st->stereo_saving, tf_estimate, + st->intensity, surround_trim, equiv_rate, st->arch); + } + ec_enc_icdf(enc, alloc_trim, trim_icdf, 7); + tell = ec_tell_frac(enc); + } + + /* Variable bitrate */ + if (vbr_rate>0) + { + opus_val16 alpha; + opus_int32 delta; + /* The target rate in 8th bits per frame */ + opus_int32 target, base_target; + opus_int32 min_allowed; + int lm_diff = mode->maxLM - LM; + + /* Don't attempt to use more than 510 kb/s, even for frames smaller than 20 ms. + The CELT allocator will just not be able to use more than that anyway. */ + nbCompressedBytes = IMIN(nbCompressedBytes,1275>>(3-LM)); + if (!hybrid) + { + base_target = vbr_rate - ((40*C+20)<constrained_vbr) + base_target += (st->vbr_offset>>lm_diff); + + if (!hybrid) + { + target = compute_vbr(mode, &st->analysis, base_target, LM, equiv_rate, + st->lastCodedBands, C, st->intensity, st->constrained_vbr, + st->stereo_saving, tot_boost, tf_estimate, pitch_change, maxDepth, + st->lfe, st->energy_mask!=NULL, surround_masking, + temporal_vbr); + } else { + target = base_target; + /* Tonal frames (offset<100) need more bits than noisy (offset>100) ones. */ + if (st->silk_info.offset < 100) target += 12 << BITRES >> (3-LM); + if (st->silk_info.offset > 100) target -= 18 << BITRES >> (3-LM); + /* Boosting bitrate on transients and vowels with significant temporal + spikes. */ + target += (opus_int32)MULT16_16_Q14(tf_estimate-QCONST16(.25f,14), (50< QCONST16(.7f,14)) + target = IMAX(target, 50<>(BITRES+3)) + 2; + /* Take into account the 37 bits we need to have left in the packet to + signal a redundant frame in hybrid mode. Creating a shorter packet would + create an entropy coder desync. */ + if (hybrid) + min_allowed = IMAX(min_allowed, (tell0_frac+(37<>(BITRES+3)); + + nbAvailableBytes = (target+(1<<(BITRES+2)))>>(BITRES+3); + nbAvailableBytes = IMAX(min_allowed,nbAvailableBytes); + nbAvailableBytes = IMIN(nbCompressedBytes,nbAvailableBytes); + + /* By how much did we "miss" the target on that frame */ + delta = target - vbr_rate; + + target=nbAvailableBytes<<(BITRES+3); + + /*If the frame is silent we don't adjust our drift, otherwise + the encoder will shoot to very high rates after hitting a + span of silence, but we do allow the bitres to refill. + This means that we'll undershoot our target in CVBR/VBR modes + on files with lots of silence. */ + if(silence) + { + nbAvailableBytes = 2; + target = 2*8<vbr_count < 970) + { + st->vbr_count++; + alpha = celt_rcp(SHL32(EXTEND32(st->vbr_count+20),16)); + } else + alpha = QCONST16(.001f,15); + /* How many bits have we used in excess of what we're allowed */ + if (st->constrained_vbr) + st->vbr_reservoir += target - vbr_rate; + /*printf ("%d\n", st->vbr_reservoir);*/ + + /* Compute the offset we need to apply in order to reach the target */ + if (st->constrained_vbr) + { + st->vbr_drift += (opus_int32)MULT16_32_Q15(alpha,(delta*(1<vbr_offset-st->vbr_drift); + st->vbr_offset = -st->vbr_drift; + } + /*printf ("%d\n", st->vbr_drift);*/ + + if (st->constrained_vbr && st->vbr_reservoir < 0) + { + /* We're under the min value -- increase rate */ + int adjust = (-st->vbr_reservoir)/(8<vbr_reservoir = 0; + /*printf ("+%d\n", adjust);*/ + } + nbCompressedBytes = IMIN(nbCompressedBytes,nbAvailableBytes); + /*printf("%d\n", nbCompressedBytes*50*8);*/ + /* This moves the raw bits to take into account the new compressed size */ + ec_enc_shrink(enc, nbCompressedBytes); + } + + /* Bit allocation */ + ALLOC(fine_quant, nbEBands, int); + ALLOC(pulses, nbEBands, int); + ALLOC(fine_priority, nbEBands, int); + + /* bits = packet size - where we are - safety*/ + bits = (((opus_int32)nbCompressedBytes*8)<=2&&bits>=((LM+2)<analysis.valid) + { + int min_bandwidth; + if (equiv_rate < (opus_int32)32000*C) + min_bandwidth = 13; + else if (equiv_rate < (opus_int32)48000*C) + min_bandwidth = 16; + else if (equiv_rate < (opus_int32)60000*C) + min_bandwidth = 18; + else if (equiv_rate < (opus_int32)80000*C) + min_bandwidth = 19; + else + min_bandwidth = 20; + signalBandwidth = IMAX(st->analysis.bandwidth, min_bandwidth); + } +#endif + if (st->lfe) + signalBandwidth = 1; + codedBands = clt_compute_allocation(mode, start, end, offsets, cap, + alloc_trim, &st->intensity, &dual_stereo, bits, &balance, pulses, + fine_quant, fine_priority, C, LM, enc, 1, st->lastCodedBands, signalBandwidth); + if (st->lastCodedBands) + st->lastCodedBands = IMIN(st->lastCodedBands+1,IMAX(st->lastCodedBands-1,codedBands)); + else + st->lastCodedBands = codedBands; + + quant_fine_energy(mode, start, end, oldBandE, error, fine_quant, enc, C); + + /* Residual quantisation */ + ALLOC(collapse_masks, C*nbEBands, unsigned char); + quant_all_bands(1, mode, start, end, X, C==2 ? X+N : NULL, collapse_masks, + bandE, pulses, shortBlocks, st->spread_decision, + dual_stereo, st->intensity, tf_res, nbCompressedBytes*(8<rng, st->complexity, st->arch, st->disable_inv); + + if (anti_collapse_rsv > 0) + { + anti_collapse_on = st->consec_transient<2; +#ifdef FUZZING + anti_collapse_on = rand()&0x1; +#endif + ec_enc_bits(enc, anti_collapse_on, 1); + } + quant_energy_finalise(mode, start, end, oldBandE, error, fine_quant, fine_priority, nbCompressedBytes*8-ec_tell(enc), enc, C); + OPUS_CLEAR(energyError, nbEBands*CC); + c=0; + do { + for (i=start;irng); + } + + c=0; do { + OPUS_MOVE(st->syn_mem[c], st->syn_mem[c]+N, 2*MAX_PERIOD-N+overlap/2); + } while (++csyn_mem[c]+2*MAX_PERIOD-N; + } while (++cupsample, silence, st->arch); + + c=0; do { + st->prefilter_period=IMAX(st->prefilter_period, COMBFILTER_MINPERIOD); + st->prefilter_period_old=IMAX(st->prefilter_period_old, COMBFILTER_MINPERIOD); + comb_filter(out_mem[c], out_mem[c], st->prefilter_period_old, st->prefilter_period, mode->shortMdctSize, + st->prefilter_gain_old, st->prefilter_gain, st->prefilter_tapset_old, st->prefilter_tapset, + mode->window, overlap); + if (LM!=0) + comb_filter(out_mem[c]+mode->shortMdctSize, out_mem[c]+mode->shortMdctSize, st->prefilter_period, pitch_index, N-mode->shortMdctSize, + st->prefilter_gain, gain1, st->prefilter_tapset, prefilter_tapset, + mode->window, overlap); + } while (++cupsample, mode->preemph, st->preemph_memD); + st->prefilter_period_old = st->prefilter_period; + st->prefilter_gain_old = st->prefilter_gain; + st->prefilter_tapset_old = st->prefilter_tapset; + } +#endif + + st->prefilter_period = pitch_index; + st->prefilter_gain = gain1; + st->prefilter_tapset = prefilter_tapset; +#ifdef RESYNTH + if (LM!=0) + { + st->prefilter_period_old = st->prefilter_period; + st->prefilter_gain_old = st->prefilter_gain; + st->prefilter_tapset_old = st->prefilter_tapset; + } +#endif + + if (CC==2&&C==1) { + OPUS_COPY(&oldBandE[nbEBands], oldBandE, nbEBands); + } + + if (!isTransient) + { + OPUS_COPY(oldLogE2, oldLogE, CC*nbEBands); + OPUS_COPY(oldLogE, oldBandE, CC*nbEBands); + } else { + for (i=0;iconsec_transient++; + else + st->consec_transient=0; + st->rng = enc->rng; + + /* If there's any room left (can only happen for very high rates), + it's already filled with zeros */ + ec_enc_done(enc); + +#ifdef CUSTOM_MODES + if (st->signalling) + nbCompressedBytes++; +#endif + + RESTORE_STACK; + if (ec_get_error(enc)) + return OPUS_INTERNAL_ERROR; + else + return nbCompressedBytes; +} + + +#ifdef CUSTOM_MODES + +#ifdef FIXED_POINT +int opus_custom_encode(CELTEncoder * OPUS_RESTRICT st, const opus_int16 * pcm, int frame_size, unsigned char *compressed, int nbCompressedBytes) +{ + return celt_encode_with_ec(st, pcm, frame_size, compressed, nbCompressedBytes, NULL); +} + +#ifndef DISABLE_FLOAT_API +int opus_custom_encode_float(CELTEncoder * OPUS_RESTRICT st, const float * pcm, int frame_size, unsigned char *compressed, int nbCompressedBytes) +{ + int j, ret, C, N; + VARDECL(opus_int16, in); + ALLOC_STACK; + + if (pcm==NULL) + return OPUS_BAD_ARG; + + C = st->channels; + N = frame_size; + ALLOC(in, C*N, opus_int16); + + for (j=0;jchannels; + N=frame_size; + ALLOC(in, C*N, celt_sig); + for (j=0;j10) + goto bad_arg; + st->complexity = value; + } + break; + case CELT_SET_START_BAND_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<0 || value>=st->mode->nbEBands) + goto bad_arg; + st->start = value; + } + break; + case CELT_SET_END_BAND_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<1 || value>st->mode->nbEBands) + goto bad_arg; + st->end = value; + } + break; + case CELT_SET_PREDICTION_REQUEST: + { + int value = va_arg(ap, opus_int32); + if (value<0 || value>2) + goto bad_arg; + st->disable_pf = value<=1; + st->force_intra = value==0; + } + break; + case OPUS_SET_PACKET_LOSS_PERC_REQUEST: + { + int value = va_arg(ap, opus_int32); + if (value<0 || value>100) + goto bad_arg; + st->loss_rate = value; + } + break; + case OPUS_SET_VBR_CONSTRAINT_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->constrained_vbr = value; + } + break; + case OPUS_SET_VBR_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->vbr = value; + } + break; + case OPUS_SET_BITRATE_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<=500 && value!=OPUS_BITRATE_MAX) + goto bad_arg; + value = IMIN(value, 260000*st->channels); + st->bitrate = value; + } + break; + case CELT_SET_CHANNELS_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<1 || value>2) + goto bad_arg; + st->stream_channels = value; + } + break; + case OPUS_SET_LSB_DEPTH_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<8 || value>24) + goto bad_arg; + st->lsb_depth=value; + } + break; + case OPUS_GET_LSB_DEPTH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + *value=st->lsb_depth; + } + break; + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->disable_inv = value; + } + break; + case OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->disable_inv; + } + break; + case OPUS_RESET_STATE: + { + int i; + opus_val16 *oldBandE, *oldLogE, *oldLogE2; + oldBandE = (opus_val16*)(st->in_mem+st->channels*(st->mode->overlap+COMBFILTER_MAXPERIOD)); + oldLogE = oldBandE + st->channels*st->mode->nbEBands; + oldLogE2 = oldLogE + st->channels*st->mode->nbEBands; + OPUS_CLEAR((char*)&st->ENCODER_RESET_START, + opus_custom_encoder_get_size(st->mode, st->channels)- + ((char*)&st->ENCODER_RESET_START - (char*)st)); + for (i=0;ichannels*st->mode->nbEBands;i++) + oldLogE[i]=oldLogE2[i]=-QCONST16(28.f,DB_SHIFT); + st->vbr_offset = 0; + st->delayedIntra = 1; + st->spread_decision = SPREAD_NORMAL; + st->tonal_average = 256; + st->hf_average = 0; + st->tapset_decision = 0; + } + break; +#ifdef CUSTOM_MODES + case CELT_SET_INPUT_CLIPPING_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->clip = value; + } + break; +#endif + case CELT_SET_SIGNALLING_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->signalling = value; + } + break; + case CELT_SET_ANALYSIS_REQUEST: + { + AnalysisInfo *info = va_arg(ap, AnalysisInfo *); + if (info) + OPUS_COPY(&st->analysis, info, 1); + } + break; + case CELT_SET_SILK_INFO_REQUEST: + { + SILKInfo *info = va_arg(ap, SILKInfo *); + if (info) + OPUS_COPY(&st->silk_info, info, 1); + } + break; + case CELT_GET_MODE_REQUEST: + { + const CELTMode ** value = va_arg(ap, const CELTMode**); + if (value==0) + goto bad_arg; + *value=st->mode; + } + break; + case OPUS_GET_FINAL_RANGE_REQUEST: + { + opus_uint32 * value = va_arg(ap, opus_uint32 *); + if (value==0) + goto bad_arg; + *value=st->rng; + } + break; + case OPUS_SET_LFE_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->lfe = value; + } + break; + case OPUS_SET_ENERGY_MASK_REQUEST: + { + opus_val16 *value = va_arg(ap, opus_val16*); + st->energy_mask = value; + } + break; + default: + goto bad_request; + } + va_end(ap); + return OPUS_OK; +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +bad_request: + va_end(ap); + return OPUS_UNIMPLEMENTED; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.c new file mode 100644 index 000000000..457e7ed0d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.c @@ -0,0 +1,340 @@ +/* Copyright (c) 2009-2010 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" + +void _celt_lpc( + opus_val16 *_lpc, /* out: [0...p-1] LPC coefficients */ +const opus_val32 *ac, /* in: [0...p] autocorrelation values */ +int p +) +{ + int i, j; + opus_val32 r; + opus_val32 error = ac[0]; +#ifdef FIXED_POINT + opus_val32 lpc[LPC_ORDER]; +#else + float *lpc = _lpc; +#endif + + OPUS_CLEAR(lpc, p); + if (ac[0] != 0) + { + for (i = 0; i < p; i++) { + /* Sum up this iteration's reflection coefficient */ + opus_val32 rr = 0; + for (j = 0; j < i; j++) + rr += MULT32_32_Q31(lpc[j],ac[i - j]); + rr += SHR32(ac[i + 1],6); + r = -frac_div32(SHL32(rr,6), error); + /* Update LPC coefficients and total error */ + lpc[i] = SHR32(r,6); + for (j = 0; j < (i+1)>>1; j++) + { + opus_val32 tmp1, tmp2; + tmp1 = lpc[j]; + tmp2 = lpc[i-1-j]; + lpc[j] = tmp1 + MULT32_32_Q31(r,tmp2); + lpc[i-1-j] = tmp2 + MULT32_32_Q31(r,tmp1); + } + + error = error - MULT32_32_Q31(MULT32_32_Q31(r,r),error); + /* Bail out once we get 30 dB gain */ +#ifdef FIXED_POINT + if (error maxabs) { + maxabs = absval; + idx = i; + } + } + maxabs = PSHR32(maxabs, 13); /* Q25->Q12 */ + + if (maxabs > 32767) { + maxabs = MIN32(maxabs, 163838); + chirp_Q16 = QCONST32(0.999, 16) - DIV32(SHL32(maxabs - 32767, 14), + SHR32(MULT32_32_32(maxabs, idx + 1), 2)); + chirp_minus_one_Q16 = chirp_Q16 - 65536; + + /* Apply bandwidth expansion. */ + for (i = 0; i < p - 1; i++) { + lpc[i] = MULT32_32_Q16(chirp_Q16, lpc[i]); + chirp_Q16 += PSHR32(MULT32_32_32(chirp_Q16, chirp_minus_one_Q16), 16); + } + lpc[p - 1] = MULT32_32_Q16(chirp_Q16, lpc[p - 1]); + } else { + break; + } + } + + if (iter == 10) { + /* If the coeffs still do not fit into the 16 bit range after 10 iterations, + fall back to the A(z)=1 filter. */ + OPUS_CLEAR(lpc, p); + _lpc[0] = 4096; /* Q12 */ + } else { + for (i = 0; i < p; i++) { + _lpc[i] = EXTRACT16(PSHR32(lpc[i], 13)); /* Q25->Q12 */ + } + } + } +#endif +} + + +void celt_fir_c( + const opus_val16 *x, + const opus_val16 *num, + opus_val16 *y, + int N, + int ord, + int arch) +{ + int i,j; + VARDECL(opus_val16, rnum); + SAVE_STACK; + celt_assert(x != y); + ALLOC(rnum, ord, opus_val16); + for(i=0;i=1;j--) + { + mem[j]=mem[j-1]; + } + mem[0] = SROUND16(sum, SIG_SHIFT); + _y[i] = sum; + } +#else + int i,j; + VARDECL(opus_val16, rden); + VARDECL(opus_val16, y); + SAVE_STACK; + + celt_assert((ord&3)==0); + ALLOC(rden, ord, opus_val16); + ALLOC(y, N+ord, opus_val16); + for(i=0;i0); + celt_assert(overlap>=0); + if (overlap == 0) + { + xptr = x; + } else { + for (i=0;i0) + { + for(i=0;i= 536870912) + { + int shift2=1; + if (ac[0] >= 1073741824) + shift2++; + for (i=0;i<=lag;i++) + ac[i] = SHR32(ac[i], shift2); + shift += shift2; + } +#endif + + RESTORE_STACK; + return shift; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.h new file mode 100644 index 000000000..a4c5fd6ea --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/celt_lpc.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2009-2010 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef PLC_H +#define PLC_H + +#include "arch.h" +#include "cpu_support.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#include "x86/celt_lpc_sse.h" +#endif + +#define LPC_ORDER 24 + +void _celt_lpc(opus_val16 *_lpc, const opus_val32 *ac, int p); + +void celt_fir_c( + const opus_val16 *x, + const opus_val16 *num, + opus_val16 *y, + int N, + int ord, + int arch); + +#if !defined(OVERRIDE_CELT_FIR) +#define celt_fir(x, num, y, N, ord, arch) \ + (celt_fir_c(x, num, y, N, ord, arch)) +#endif + +void celt_iir(const opus_val32 *x, + const opus_val16 *den, + opus_val32 *y, + int N, + int ord, + opus_val16 *mem, + int arch); + +int _celt_autocorr(const opus_val16 *x, opus_val32 *ac, + const opus_val16 *window, int overlap, int lag, int n, int arch); + +#endif /* PLC_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cpu_support.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cpu_support.h new file mode 100644 index 000000000..68fc60678 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cpu_support.h @@ -0,0 +1,70 @@ +/* Copyright (c) 2010 Xiph.Org Foundation + * Copyright (c) 2013 Parrot */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CPU_SUPPORT_H +#define CPU_SUPPORT_H + +#include "opus_types.h" +#include "opus_defines.h" + +#if defined(OPUS_HAVE_RTCD) && \ + (defined(OPUS_ARM_ASM) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)) +#include "arm/armcpu.h" + +/* We currently support 4 ARM variants: + * arch[0] -> ARMv4 + * arch[1] -> ARMv5E + * arch[2] -> ARMv6 + * arch[3] -> NEON + */ +#define OPUS_ARCHMASK 3 + +#elif (defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \ + (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \ + (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \ + (defined(OPUS_X86_MAY_HAVE_AVX) && !defined(OPUS_X86_PRESUME_AVX)) + +#include "x86/x86cpu.h" +/* We currently support 5 x86 variants: + * arch[0] -> non-sse + * arch[1] -> sse + * arch[2] -> sse2 + * arch[3] -> sse4.1 + * arch[4] -> avx + */ +#define OPUS_ARCHMASK 7 +int opus_select_arch(void); + +#else +#define OPUS_ARCHMASK 0 + +static OPUS_INLINE int opus_select_arch(void) +{ + return 0; +} +#endif +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.c new file mode 100644 index 000000000..a552e4f0f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.c @@ -0,0 +1,715 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2007-2009 Timothy B. Terriberry + Written by Timothy B. Terriberry and Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "os_support.h" +#include "cwrs.h" +#include "mathops.h" +#include "arch.h" + +#ifdef CUSTOM_MODES + +/*Guaranteed to return a conservatively large estimate of the binary logarithm + with frac bits of fractional precision. + Tested for all possible 32-bit inputs with frac=4, where the maximum + overestimation is 0.06254243 bits.*/ +int log2_frac(opus_uint32 val, int frac) +{ + int l; + l=EC_ILOG(val); + if(val&(val-1)){ + /*This is (val>>l-16), but guaranteed to round up, even if adding a bias + before the shift would cause overflow (e.g., for 0xFFFFxxxx). + Doesn't work for val=0, but that case fails the test above.*/ + if(l>16)val=((val-1)>>(l-16))+1; + else val<<=16-l; + l=(l-1)<>16); + l+=b<>b; + val=(val*val+0x7FFF)>>15; + } + while(frac-->0); + /*If val is not exactly 0x8000, then we have to round up the remainder.*/ + return l+(val>0x8000); + } + /*Exact powers of two require no rounding.*/ + else return (l-1)<0 ? sum(k=1...K,2**k*choose(N,k)*choose(K-1,k-1)) : 1, + where choose() is the binomial function. + A table of values for N<10 and K<10 looks like: + V[10][10] = { + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {1, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {1, 4, 8, 12, 16, 20, 24, 28, 32, 36}, + {1, 6, 18, 38, 66, 102, 146, 198, 258, 326}, + {1, 8, 32, 88, 192, 360, 608, 952, 1408, 1992}, + {1, 10, 50, 170, 450, 1002, 1970, 3530, 5890, 9290}, + {1, 12, 72, 292, 912, 2364, 5336, 10836, 20256, 35436}, + {1, 14, 98, 462, 1666, 4942, 12642, 28814, 59906, 115598}, + {1, 16, 128, 688, 2816, 9424, 27008, 68464, 157184, 332688}, + {1, 18, 162, 978, 4482, 16722, 53154, 148626, 374274, 864146} + }; + + U(N,K) = the number of such combinations wherein N-1 objects are taken at + most K-1 at a time. + This is given by + U(N,K) = sum(k=0...K-1,V(N-1,k)) + = K>0 ? (V(N-1,K-1) + V(N,K-1))/2 : 0. + The latter expression also makes clear that U(N,K) is half the number of such + combinations wherein the first object is taken at least once. + Although it may not be clear from either of these definitions, U(N,K) is the + natural function to work with when enumerating the pulse vector codebooks, + not V(N,K). + U(N,K) is not well-defined for N=0, but with the extension + U(0,K) = K>0 ? 0 : 1, + the function becomes symmetric: U(N,K) = U(K,N), with a similar table: + U[10][10] = { + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {0, 1, 3, 5, 7, 9, 11, 13, 15, 17}, + {0, 1, 5, 13, 25, 41, 61, 85, 113, 145}, + {0, 1, 7, 25, 63, 129, 231, 377, 575, 833}, + {0, 1, 9, 41, 129, 321, 681, 1289, 2241, 3649}, + {0, 1, 11, 61, 231, 681, 1683, 3653, 7183, 13073}, + {0, 1, 13, 85, 377, 1289, 3653, 8989, 19825, 40081}, + {0, 1, 15, 113, 575, 2241, 7183, 19825, 48639, 108545}, + {0, 1, 17, 145, 833, 3649, 13073, 40081, 108545, 265729} + }; + + With this extension, V(N,K) may be written in terms of U(N,K): + V(N,K) = U(N,K) + U(N,K+1) + for all N>=0, K>=0. + Thus U(N,K+1) represents the number of combinations where the first element + is positive or zero, and U(N,K) represents the number of combinations where + it is negative. + With a large enough table of U(N,K) values, we could write O(N) encoding + and O(min(N*log(K),N+K)) decoding routines, but such a table would be + prohibitively large for small embedded devices (K may be as large as 32767 + for small N, and N may be as large as 200). + + Both functions obey the same recurrence relation: + V(N,K) = V(N-1,K) + V(N,K-1) + V(N-1,K-1), + U(N,K) = U(N-1,K) + U(N,K-1) + U(N-1,K-1), + for all N>0, K>0, with different initial conditions at N=0 or K=0. + This allows us to construct a row of one of the tables above given the + previous row or the next row. + Thus we can derive O(NK) encoding and decoding routines with O(K) memory + using only addition and subtraction. + + When encoding, we build up from the U(2,K) row and work our way forwards. + When decoding, we need to start at the U(N,K) row and work our way backwards, + which requires a means of computing U(N,K). + U(N,K) may be computed from two previous values with the same N: + U(N,K) = ((2*N-1)*U(N,K-1) - U(N,K-2))/(K-1) + U(N,K-2) + for all N>1, and since U(N,K) is symmetric, a similar relation holds for two + previous values with the same K: + U(N,K>1) = ((2*K-1)*U(N-1,K) - U(N-2,K))/(N-1) + U(N-2,K) + for all K>1. + This allows us to construct an arbitrary row of the U(N,K) table by starting + with the first two values, which are constants. + This saves roughly 2/3 the work in our O(NK) decoding routine, but costs O(K) + multiplications. + Similar relations can be derived for V(N,K), but are not used here. + + For N>0 and K>0, U(N,K) and V(N,K) take on the form of an (N-1)-degree + polynomial for fixed N. + The first few are + U(1,K) = 1, + U(2,K) = 2*K-1, + U(3,K) = (2*K-2)*K+1, + U(4,K) = (((4*K-6)*K+8)*K-3)/3, + U(5,K) = ((((2*K-4)*K+10)*K-8)*K+3)/3, + and + V(1,K) = 2, + V(2,K) = 4*K, + V(3,K) = 4*K*K+2, + V(4,K) = 8*(K*K+2)*K/3, + V(5,K) = ((4*K*K+20)*K*K+6)/3, + for all K>0. + This allows us to derive O(N) encoding and O(N*log(K)) decoding routines for + small N (and indeed decoding is also O(N) for N<3). + + @ARTICLE{Fis86, + author="Thomas R. Fischer", + title="A Pyramid Vector Quantizer", + journal="IEEE Transactions on Information Theory", + volume="IT-32", + number=4, + pages="568--583", + month=Jul, + year=1986 + }*/ + +#if !defined(SMALL_FOOTPRINT) + +/*U(N,K) = U(K,N) := N>0?K>0?U(N-1,K)+U(N,K-1)+U(N-1,K-1):0:K>0?1:0*/ +# define CELT_PVQ_U(_n,_k) (CELT_PVQ_U_ROW[IMIN(_n,_k)][IMAX(_n,_k)]) +/*V(N,K) := U(N,K)+U(N,K+1) = the number of PVQ codewords for a band of size N + with K pulses allocated to it.*/ +# define CELT_PVQ_V(_n,_k) (CELT_PVQ_U(_n,_k)+CELT_PVQ_U(_n,(_k)+1)) + +/*For each V(N,K) supported, we will access element U(min(N,K+1),max(N,K+1)). + Thus, the number of entries in row I is the larger of the maximum number of + pulses we will ever allocate for a given N=I (K=128, or however many fit in + 32 bits, whichever is smaller), plus one, and the maximum N for which + K=I-1 pulses fit in 32 bits. + The largest band size in an Opus Custom mode is 208. + Otherwise, we can limit things to the set of N which can be achieved by + splitting a band from a standard Opus mode: 176, 144, 96, 88, 72, 64, 48, + 44, 36, 32, 24, 22, 18, 16, 8, 4, 2).*/ +#if defined(CUSTOM_MODES) +static const opus_uint32 CELT_PVQ_U_DATA[1488]={ +#else +static const opus_uint32 CELT_PVQ_U_DATA[1272]={ +#endif + /*N=0, K=0...176:*/ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +#if defined(CUSTOM_MODES) + /*...208:*/ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, +#endif + /*N=1, K=1...176:*/ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +#if defined(CUSTOM_MODES) + /*...208:*/ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, +#endif + /*N=2, K=2...176:*/ + 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, + 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, + 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 101, 103, 105, 107, 109, 111, 113, + 115, 117, 119, 121, 123, 125, 127, 129, 131, 133, 135, 137, 139, 141, 143, + 145, 147, 149, 151, 153, 155, 157, 159, 161, 163, 165, 167, 169, 171, 173, + 175, 177, 179, 181, 183, 185, 187, 189, 191, 193, 195, 197, 199, 201, 203, + 205, 207, 209, 211, 213, 215, 217, 219, 221, 223, 225, 227, 229, 231, 233, + 235, 237, 239, 241, 243, 245, 247, 249, 251, 253, 255, 257, 259, 261, 263, + 265, 267, 269, 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, 291, 293, + 295, 297, 299, 301, 303, 305, 307, 309, 311, 313, 315, 317, 319, 321, 323, + 325, 327, 329, 331, 333, 335, 337, 339, 341, 343, 345, 347, 349, 351, +#if defined(CUSTOM_MODES) + /*...208:*/ + 353, 355, 357, 359, 361, 363, 365, 367, 369, 371, 373, 375, 377, 379, 381, + 383, 385, 387, 389, 391, 393, 395, 397, 399, 401, 403, 405, 407, 409, 411, + 413, 415, +#endif + /*N=3, K=3...176:*/ + 13, 25, 41, 61, 85, 113, 145, 181, 221, 265, 313, 365, 421, 481, 545, 613, + 685, 761, 841, 925, 1013, 1105, 1201, 1301, 1405, 1513, 1625, 1741, 1861, + 1985, 2113, 2245, 2381, 2521, 2665, 2813, 2965, 3121, 3281, 3445, 3613, 3785, + 3961, 4141, 4325, 4513, 4705, 4901, 5101, 5305, 5513, 5725, 5941, 6161, 6385, + 6613, 6845, 7081, 7321, 7565, 7813, 8065, 8321, 8581, 8845, 9113, 9385, 9661, + 9941, 10225, 10513, 10805, 11101, 11401, 11705, 12013, 12325, 12641, 12961, + 13285, 13613, 13945, 14281, 14621, 14965, 15313, 15665, 16021, 16381, 16745, + 17113, 17485, 17861, 18241, 18625, 19013, 19405, 19801, 20201, 20605, 21013, + 21425, 21841, 22261, 22685, 23113, 23545, 23981, 24421, 24865, 25313, 25765, + 26221, 26681, 27145, 27613, 28085, 28561, 29041, 29525, 30013, 30505, 31001, + 31501, 32005, 32513, 33025, 33541, 34061, 34585, 35113, 35645, 36181, 36721, + 37265, 37813, 38365, 38921, 39481, 40045, 40613, 41185, 41761, 42341, 42925, + 43513, 44105, 44701, 45301, 45905, 46513, 47125, 47741, 48361, 48985, 49613, + 50245, 50881, 51521, 52165, 52813, 53465, 54121, 54781, 55445, 56113, 56785, + 57461, 58141, 58825, 59513, 60205, 60901, 61601, +#if defined(CUSTOM_MODES) + /*...208:*/ + 62305, 63013, 63725, 64441, 65161, 65885, 66613, 67345, 68081, 68821, 69565, + 70313, 71065, 71821, 72581, 73345, 74113, 74885, 75661, 76441, 77225, 78013, + 78805, 79601, 80401, 81205, 82013, 82825, 83641, 84461, 85285, 86113, +#endif + /*N=4, K=4...176:*/ + 63, 129, 231, 377, 575, 833, 1159, 1561, 2047, 2625, 3303, 4089, 4991, 6017, + 7175, 8473, 9919, 11521, 13287, 15225, 17343, 19649, 22151, 24857, 27775, + 30913, 34279, 37881, 41727, 45825, 50183, 54809, 59711, 64897, 70375, 76153, + 82239, 88641, 95367, 102425, 109823, 117569, 125671, 134137, 142975, 152193, + 161799, 171801, 182207, 193025, 204263, 215929, 228031, 240577, 253575, + 267033, 280959, 295361, 310247, 325625, 341503, 357889, 374791, 392217, + 410175, 428673, 447719, 467321, 487487, 508225, 529543, 551449, 573951, + 597057, 620775, 645113, 670079, 695681, 721927, 748825, 776383, 804609, + 833511, 863097, 893375, 924353, 956039, 988441, 1021567, 1055425, 1090023, + 1125369, 1161471, 1198337, 1235975, 1274393, 1313599, 1353601, 1394407, + 1436025, 1478463, 1521729, 1565831, 1610777, 1656575, 1703233, 1750759, + 1799161, 1848447, 1898625, 1949703, 2001689, 2054591, 2108417, 2163175, + 2218873, 2275519, 2333121, 2391687, 2451225, 2511743, 2573249, 2635751, + 2699257, 2763775, 2829313, 2895879, 2963481, 3032127, 3101825, 3172583, + 3244409, 3317311, 3391297, 3466375, 3542553, 3619839, 3698241, 3777767, + 3858425, 3940223, 4023169, 4107271, 4192537, 4278975, 4366593, 4455399, + 4545401, 4636607, 4729025, 4822663, 4917529, 5013631, 5110977, 5209575, + 5309433, 5410559, 5512961, 5616647, 5721625, 5827903, 5935489, 6044391, + 6154617, 6266175, 6379073, 6493319, 6608921, 6725887, 6844225, 6963943, + 7085049, 7207551, +#if defined(CUSTOM_MODES) + /*...208:*/ + 7331457, 7456775, 7583513, 7711679, 7841281, 7972327, 8104825, 8238783, + 8374209, 8511111, 8649497, 8789375, 8930753, 9073639, 9218041, 9363967, + 9511425, 9660423, 9810969, 9963071, 10116737, 10271975, 10428793, 10587199, + 10747201, 10908807, 11072025, 11236863, 11403329, 11571431, 11741177, + 11912575, +#endif + /*N=5, K=5...176:*/ + 321, 681, 1289, 2241, 3649, 5641, 8361, 11969, 16641, 22569, 29961, 39041, + 50049, 63241, 78889, 97281, 118721, 143529, 172041, 204609, 241601, 283401, + 330409, 383041, 441729, 506921, 579081, 658689, 746241, 842249, 947241, + 1061761, 1186369, 1321641, 1468169, 1626561, 1797441, 1981449, 2179241, + 2391489, 2618881, 2862121, 3121929, 3399041, 3694209, 4008201, 4341801, + 4695809, 5071041, 5468329, 5888521, 6332481, 6801089, 7295241, 7815849, + 8363841, 8940161, 9545769, 10181641, 10848769, 11548161, 12280841, 13047849, + 13850241, 14689089, 15565481, 16480521, 17435329, 18431041, 19468809, + 20549801, 21675201, 22846209, 24064041, 25329929, 26645121, 28010881, + 29428489, 30899241, 32424449, 34005441, 35643561, 37340169, 39096641, + 40914369, 42794761, 44739241, 46749249, 48826241, 50971689, 53187081, + 55473921, 57833729, 60268041, 62778409, 65366401, 68033601, 70781609, + 73612041, 76526529, 79526721, 82614281, 85790889, 89058241, 92418049, + 95872041, 99421961, 103069569, 106816641, 110664969, 114616361, 118672641, + 122835649, 127107241, 131489289, 135983681, 140592321, 145317129, 150160041, + 155123009, 160208001, 165417001, 170752009, 176215041, 181808129, 187533321, + 193392681, 199388289, 205522241, 211796649, 218213641, 224775361, 231483969, + 238341641, 245350569, 252512961, 259831041, 267307049, 274943241, 282741889, + 290705281, 298835721, 307135529, 315607041, 324252609, 333074601, 342075401, + 351257409, 360623041, 370174729, 379914921, 389846081, 399970689, 410291241, + 420810249, 431530241, 442453761, 453583369, 464921641, 476471169, 488234561, + 500214441, 512413449, 524834241, 537479489, 550351881, 563454121, 576788929, + 590359041, 604167209, 618216201, 632508801, +#if defined(CUSTOM_MODES) + /*...208:*/ + 647047809, 661836041, 676876329, 692171521, 707724481, 723538089, 739615241, + 755958849, 772571841, 789457161, 806617769, 824056641, 841776769, 859781161, + 878072841, 896654849, 915530241, 934702089, 954173481, 973947521, 994027329, + 1014416041, 1035116809, 1056132801, 1077467201, 1099123209, 1121104041, + 1143412929, 1166053121, 1189027881, 1212340489, 1235994241, +#endif + /*N=6, K=6...96:*/ + 1683, 3653, 7183, 13073, 22363, 36365, 56695, 85305, 124515, 177045, 246047, + 335137, 448427, 590557, 766727, 982729, 1244979, 1560549, 1937199, 2383409, + 2908411, 3522221, 4235671, 5060441, 6009091, 7095093, 8332863, 9737793, + 11326283, 13115773, 15124775, 17372905, 19880915, 22670725, 25765455, + 29189457, 32968347, 37129037, 41699767, 46710137, 52191139, 58175189, + 64696159, 71789409, 79491819, 87841821, 96879431, 106646281, 117185651, + 128542501, 140763503, 153897073, 167993403, 183104493, 199284183, 216588185, + 235074115, 254801525, 275831935, 298228865, 322057867, 347386557, 374284647, + 402823977, 433078547, 465124549, 499040399, 534906769, 572806619, 612825229, + 655050231, 699571641, 746481891, 795875861, 847850911, 902506913, 959946283, + 1020274013, 1083597703, 1150027593, 1219676595, 1292660325, 1369097135, + 1449108145, 1532817275, 1620351277, 1711839767, 1807415257, 1907213187, + 2011371957, 2120032959, +#if defined(CUSTOM_MODES) + /*...109:*/ + 2233340609U, 2351442379U, 2474488829U, 2602633639U, 2736033641U, 2874848851U, + 3019242501U, 3169381071U, 3325434321U, 3487575323U, 3655980493U, 3830829623U, + 4012305913U, +#endif + /*N=7, K=7...54*/ + 8989, 19825, 40081, 75517, 134245, 227305, 369305, 579125, 880685, 1303777, + 1884961, 2668525, 3707509, 5064793, 6814249, 9041957, 11847485, 15345233, + 19665841, 24957661, 31388293, 39146185, 48442297, 59511829, 72616013, + 88043969, 106114625, 127178701, 151620757, 179861305, 212358985, 249612805, + 292164445, 340600625, 395555537, 457713341, 527810725, 606639529, 695049433, + 793950709, 904317037, 1027188385, 1163673953, 1314955181, 1482288821, + 1667010073, 1870535785, 2094367717, +#if defined(CUSTOM_MODES) + /*...60:*/ + 2340095869U, 2609401873U, 2904062449U, 3225952925U, 3577050821U, 3959439497U, +#endif + /*N=8, K=8...37*/ + 48639, 108545, 224143, 433905, 795455, 1392065, 2340495, 3800305, 5984767, + 9173505, 13726991, 20103025, 28875327, 40754369, 56610575, 77500017, + 104692735, 139703809, 184327311, 240673265, 311207743, 398796225, 506750351, + 638878193, 799538175, 993696769, 1226990095, 1505789553, 1837271615, + 2229491905U, +#if defined(CUSTOM_MODES) + /*...40:*/ + 2691463695U, 3233240945U, 3866006015U, +#endif + /*N=9, K=9...28:*/ + 265729, 598417, 1256465, 2485825, 4673345, 8405905, 14546705, 24331777, + 39490049, 62390545, 96220561, 145198913, 214828609, 312193553, 446304145, + 628496897, 872893441, 1196924561, 1621925137, 2173806145U, +#if defined(CUSTOM_MODES) + /*...29:*/ + 2883810113U, +#endif + /*N=10, K=10...24:*/ + 1462563, 3317445, 7059735, 14218905, 27298155, 50250765, 89129247, 152951073, + 254831667, 413442773, 654862247, 1014889769, 1541911931, 2300409629U, + 3375210671U, + /*N=11, K=11...19:*/ + 8097453, 18474633, 39753273, 81270333, 158819253, 298199265, 540279585, + 948062325, 1616336765, +#if defined(CUSTOM_MODES) + /*...20:*/ + 2684641785U, +#endif + /*N=12, K=12...18:*/ + 45046719, 103274625, 224298231, 464387817, 921406335, 1759885185, + 3248227095U, + /*N=13, K=13...16:*/ + 251595969, 579168825, 1267854873, 2653649025U, + /*N=14, K=14:*/ + 1409933619 +}; + +#if defined(CUSTOM_MODES) +static const opus_uint32 *const CELT_PVQ_U_ROW[15]={ + CELT_PVQ_U_DATA+ 0,CELT_PVQ_U_DATA+ 208,CELT_PVQ_U_DATA+ 415, + CELT_PVQ_U_DATA+ 621,CELT_PVQ_U_DATA+ 826,CELT_PVQ_U_DATA+1030, + CELT_PVQ_U_DATA+1233,CELT_PVQ_U_DATA+1336,CELT_PVQ_U_DATA+1389, + CELT_PVQ_U_DATA+1421,CELT_PVQ_U_DATA+1441,CELT_PVQ_U_DATA+1455, + CELT_PVQ_U_DATA+1464,CELT_PVQ_U_DATA+1470,CELT_PVQ_U_DATA+1473 +}; +#else +static const opus_uint32 *const CELT_PVQ_U_ROW[15]={ + CELT_PVQ_U_DATA+ 0,CELT_PVQ_U_DATA+ 176,CELT_PVQ_U_DATA+ 351, + CELT_PVQ_U_DATA+ 525,CELT_PVQ_U_DATA+ 698,CELT_PVQ_U_DATA+ 870, + CELT_PVQ_U_DATA+1041,CELT_PVQ_U_DATA+1131,CELT_PVQ_U_DATA+1178, + CELT_PVQ_U_DATA+1207,CELT_PVQ_U_DATA+1226,CELT_PVQ_U_DATA+1240, + CELT_PVQ_U_DATA+1248,CELT_PVQ_U_DATA+1254,CELT_PVQ_U_DATA+1257 +}; +#endif + +#if defined(CUSTOM_MODES) +void get_required_bits(opus_int16 *_bits,int _n,int _maxk,int _frac){ + int k; + /*_maxk==0 => there's nothing to do.*/ + celt_assert(_maxk>0); + _bits[0]=0; + for(k=1;k<=_maxk;k++)_bits[k]=log2_frac(CELT_PVQ_V(_n,k),_frac); +} +#endif + +static opus_uint32 icwrs(int _n,const int *_y){ + opus_uint32 i; + int j; + int k; + celt_assert(_n>=2); + j=_n-1; + i=_y[j]<0; + k=abs(_y[j]); + do{ + j--; + i+=CELT_PVQ_U(_n-j,k); + k+=abs(_y[j]); + if(_y[j]<0)i+=CELT_PVQ_U(_n-j,k+1); + } + while(j>0); + return i; +} + +void encode_pulses(const int *_y,int _n,int _k,ec_enc *_enc){ + celt_assert(_k>0); + ec_enc_uint(_enc,icwrs(_n,_y),CELT_PVQ_V(_n,_k)); +} + +static opus_val32 cwrsi(int _n,int _k,opus_uint32 _i,int *_y){ + opus_uint32 p; + int s; + int k0; + opus_int16 val; + opus_val32 yy=0; + celt_assert(_k>0); + celt_assert(_n>1); + while(_n>2){ + opus_uint32 q; + /*Lots of pulses case:*/ + if(_k>=_n){ + const opus_uint32 *row; + row=CELT_PVQ_U_ROW[_n]; + /*Are the pulses in this dimension negative?*/ + p=row[_k+1]; + s=-(_i>=p); + _i-=p&s; + /*Count how many pulses were placed in this dimension.*/ + k0=_k; + q=row[_n]; + if(q>_i){ + celt_sig_assert(p>q); + _k=_n; + do p=CELT_PVQ_U_ROW[--_k][_n]; + while(p>_i); + } + else for(p=row[_k];p>_i;p=row[_k])_k--; + _i-=p; + val=(k0-_k+s)^s; + *_y++=val; + yy=MAC16_16(yy,val,val); + } + /*Lots of dimensions case:*/ + else{ + /*Are there any pulses in this dimension at all?*/ + p=CELT_PVQ_U_ROW[_k][_n]; + q=CELT_PVQ_U_ROW[_k+1][_n]; + if(p<=_i&&_i=q); + _i-=q&s; + /*Count how many pulses were placed in this dimension.*/ + k0=_k; + do p=CELT_PVQ_U_ROW[--_k][_n]; + while(p>_i); + _i-=p; + val=(k0-_k+s)^s; + *_y++=val; + yy=MAC16_16(yy,val,val); + } + } + _n--; + } + /*_n==2*/ + p=2*_k+1; + s=-(_i>=p); + _i-=p&s; + k0=_k; + _k=(_i+1)>>1; + if(_k)_i-=2*_k-1; + val=(k0-_k+s)^s; + *_y++=val; + yy=MAC16_16(yy,val,val); + /*_n==1*/ + s=-(int)_i; + val=(_k+s)^s; + *_y=val; + yy=MAC16_16(yy,val,val); + return yy; +} + +opus_val32 decode_pulses(int *_y,int _n,int _k,ec_dec *_dec){ + return cwrsi(_n,_k,ec_dec_uint(_dec,CELT_PVQ_V(_n,_k)),_y); +} + +#else /* SMALL_FOOTPRINT */ + +/*Computes the next row/column of any recurrence that obeys the relation + u[i][j]=u[i-1][j]+u[i][j-1]+u[i-1][j-1]. + _ui0 is the base case for the new row/column.*/ +static OPUS_INLINE void unext(opus_uint32 *_ui,unsigned _len,opus_uint32 _ui0){ + opus_uint32 ui1; + unsigned j; + /*This do-while will overrun the array if we don't have storage for at least + 2 values.*/ + j=1; do { + ui1=UADD32(UADD32(_ui[j],_ui[j-1]),_ui0); + _ui[j-1]=_ui0; + _ui0=ui1; + } while (++j<_len); + _ui[j-1]=_ui0; +} + +/*Computes the previous row/column of any recurrence that obeys the relation + u[i-1][j]=u[i][j]-u[i][j-1]-u[i-1][j-1]. + _ui0 is the base case for the new row/column.*/ +static OPUS_INLINE void uprev(opus_uint32 *_ui,unsigned _n,opus_uint32 _ui0){ + opus_uint32 ui1; + unsigned j; + /*This do-while will overrun the array if we don't have storage for at least + 2 values.*/ + j=1; do { + ui1=USUB32(USUB32(_ui[j],_ui[j-1]),_ui0); + _ui[j-1]=_ui0; + _ui0=ui1; + } while (++j<_n); + _ui[j-1]=_ui0; +} + +/*Compute V(_n,_k), as well as U(_n,0..._k+1). + _u: On exit, _u[i] contains U(_n,i) for i in [0..._k+1].*/ +static opus_uint32 ncwrs_urow(unsigned _n,unsigned _k,opus_uint32 *_u){ + opus_uint32 um2; + unsigned len; + unsigned k; + len=_k+2; + /*We require storage at least 3 values (e.g., _k>0).*/ + celt_assert(len>=3); + _u[0]=0; + _u[1]=um2=1; + /*If _n==0, _u[0] should be 1 and the rest should be 0.*/ + /*If _n==1, _u[i] should be 1 for i>1.*/ + celt_assert(_n>=2); + /*If _k==0, the following do-while loop will overflow the buffer.*/ + celt_assert(_k>0); + k=2; + do _u[k]=(k<<1)-1; + while(++k0); + j=0; + do{ + opus_uint32 p; + int s; + int yj; + p=_u[_k+1]; + s=-(_i>=p); + _i-=p&s; + yj=_k; + p=_u[_k]; + while(p>_i)p=_u[--_k]; + _i-=p; + yj-=_k; + val=(yj+s)^s; + _y[j]=val; + yy=MAC16_16(yy,val,val); + uprev(_u,_k+2,0); + } + while(++j<_n); + return yy; +} + +/*Returns the index of the given combination of K elements chosen from a set + of size 1 with associated sign bits. + _y: The vector of pulses, whose sum of absolute values is K. + _k: Returns K.*/ +static OPUS_INLINE opus_uint32 icwrs1(const int *_y,int *_k){ + *_k=abs(_y[0]); + return _y[0]<0; +} + +/*Returns the index of the given combination of K elements chosen from a set + of size _n with associated sign bits. + _y: The vector of pulses, whose sum of absolute values must be _k. + _nc: Returns V(_n,_k).*/ +static OPUS_INLINE opus_uint32 icwrs(int _n,int _k,opus_uint32 *_nc,const int *_y, + opus_uint32 *_u){ + opus_uint32 i; + int j; + int k; + /*We can't unroll the first two iterations of the loop unless _n>=2.*/ + celt_assert(_n>=2); + _u[0]=0; + for(k=1;k<=_k+1;k++)_u[k]=(k<<1)-1; + i=icwrs1(_y+_n-1,&k); + j=_n-2; + i+=_u[k]; + k+=abs(_y[j]); + if(_y[j]<0)i+=_u[k+1]; + while(j-->0){ + unext(_u,_k+2,0); + i+=_u[k]; + k+=abs(_y[j]); + if(_y[j]<0)i+=_u[k+1]; + } + *_nc=_u[k]+_u[k+1]; + return i; +} + +#ifdef CUSTOM_MODES +void get_required_bits(opus_int16 *_bits,int _n,int _maxk,int _frac){ + int k; + /*_maxk==0 => there's nothing to do.*/ + celt_assert(_maxk>0); + _bits[0]=0; + if (_n==1) + { + for (k=1;k<=_maxk;k++) + _bits[k] = 1<<_frac; + } + else { + VARDECL(opus_uint32,u); + SAVE_STACK; + ALLOC(u,_maxk+2U,opus_uint32); + ncwrs_urow(_n,_maxk,u); + for(k=1;k<=_maxk;k++) + _bits[k]=log2_frac(u[k]+u[k+1],_frac); + RESTORE_STACK; + } +} +#endif /* CUSTOM_MODES */ + +void encode_pulses(const int *_y,int _n,int _k,ec_enc *_enc){ + opus_uint32 i; + VARDECL(opus_uint32,u); + opus_uint32 nc; + SAVE_STACK; + celt_assert(_k>0); + ALLOC(u,_k+2U,opus_uint32); + i=icwrs(_n,_k,&nc,_y,u); + ec_enc_uint(_enc,i,nc); + RESTORE_STACK; +} + +opus_val32 decode_pulses(int *_y,int _n,int _k,ec_dec *_dec){ + VARDECL(opus_uint32,u); + int ret; + SAVE_STACK; + celt_assert(_k>0); + ALLOC(u,_k+2U,opus_uint32); + ret = cwrsi(_n,_k,ec_dec_uint(_dec,ncwrs_urow(_n,_k,u)),_y,u); + RESTORE_STACK; + return ret; +} + +#endif /* SMALL_FOOTPRINT */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.h new file mode 100644 index 000000000..7cd471745 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/cwrs.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2007-2009 Timothy B. Terriberry + Written by Timothy B. Terriberry and Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CWRS_H +#define CWRS_H + +#include "arch.h" +#include "stack_alloc.h" +#include "entenc.h" +#include "entdec.h" + +#ifdef CUSTOM_MODES +int log2_frac(opus_uint32 val, int frac); +#endif + +void get_required_bits(opus_int16 *bits, int N, int K, int frac); + +void encode_pulses(const int *_y, int N, int K, ec_enc *enc); + +opus_val32 decode_pulses(int *_y, int N, int K, ec_dec *dec); + +#endif /* CWRS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes.c new file mode 100644 index 000000000..9105a5344 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/dump_modes/dump_modes.c @@ -0,0 +1,353 @@ +/* Copyright (c) 2008 CSIRO + Copyright (c) 2008-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "modes.h" +#include "celt.h" +#include "rate.h" +#include "dump_modes_arch.h" + +#define INT16 "%d" +#define INT32 "%d" +#define FLOAT "%#0.8gf" + +#ifdef FIXED_POINT +#define WORD16 INT16 +#define WORD32 INT32 +#else +#define WORD16 FLOAT +#define WORD32 FLOAT +#endif + +void dump_modes(FILE *file, CELTMode **modes, int nb_modes) +{ + int i, j, k; + int mdct_twiddles_size; + fprintf(file, "/* The contents of this file was automatically generated by dump_modes.c\n"); + fprintf(file, " with arguments:"); + for (i=0;iFs,mode->shortMdctSize*mode->nbShortMdcts); + } + fprintf(file, "\n It contains static definitions for some pre-defined modes. */\n"); + fprintf(file, "#include \"modes.h\"\n"); + fprintf(file, "#include \"rate.h\"\n"); + fprintf(file, "\n#ifdef HAVE_ARM_NE10\n"); + fprintf(file, "#define OVERRIDE_FFT 1\n"); + fprintf(file, "#include \"%s\"\n", ARM_NE10_ARCH_FILE_NAME); + fprintf(file, "#endif\n"); + + fprintf(file, "\n"); + + for (i=0;ishortMdctSize*mode->nbShortMdcts; + standard = (mode->Fs == 400*(opus_int32)mode->shortMdctSize); + framerate = mode->Fs/mode->shortMdctSize; + + if (!standard) + { + fprintf(file, "#ifndef DEF_EBANDS%d_%d\n", mode->Fs, mdctSize); + fprintf(file, "#define DEF_EBANDS%d_%d\n", mode->Fs, mdctSize); + fprintf (file, "static const opus_int16 eBands%d_%d[%d] = {\n", mode->Fs, mdctSize, mode->nbEBands+2); + for (j=0;jnbEBands+2;j++) + fprintf (file, "%d, ", mode->eBands[j]); + fprintf (file, "};\n"); + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + } + + fprintf(file, "#ifndef DEF_WINDOW%d\n", mode->overlap); + fprintf(file, "#define DEF_WINDOW%d\n", mode->overlap); + fprintf (file, "static const opus_val16 window%d[%d] = {\n", mode->overlap, mode->overlap); + for (j=0;joverlap;j++) + fprintf (file, WORD16 ",%c", mode->window[j],(j+6)%5==0?'\n':' '); + fprintf (file, "};\n"); + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + + if (!standard) + { + fprintf(file, "#ifndef DEF_ALLOC_VECTORS%d_%d\n", mode->Fs, mdctSize); + fprintf(file, "#define DEF_ALLOC_VECTORS%d_%d\n", mode->Fs, mdctSize); + fprintf (file, "static const unsigned char allocVectors%d_%d[%d] = {\n", mode->Fs, mdctSize, mode->nbEBands*mode->nbAllocVectors); + for (j=0;jnbAllocVectors;j++) + { + for (k=0;knbEBands;k++) + fprintf (file, "%2d, ", mode->allocVectors[j*mode->nbEBands+k]); + fprintf (file, "\n"); + } + fprintf (file, "};\n"); + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + } + + fprintf(file, "#ifndef DEF_LOGN%d\n", framerate); + fprintf(file, "#define DEF_LOGN%d\n", framerate); + fprintf (file, "static const opus_int16 logN%d[%d] = {\n", framerate, mode->nbEBands); + for (j=0;jnbEBands;j++) + fprintf (file, "%d, ", mode->logN[j]); + fprintf (file, "};\n"); + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + + /* Pulse cache */ + fprintf(file, "#ifndef DEF_PULSE_CACHE%d\n", mode->Fs/mdctSize); + fprintf(file, "#define DEF_PULSE_CACHE%d\n", mode->Fs/mdctSize); + fprintf (file, "static const opus_int16 cache_index%d[%d] = {\n", mode->Fs/mdctSize, (mode->maxLM+2)*mode->nbEBands); + for (j=0;jnbEBands*(mode->maxLM+2);j++) + fprintf (file, "%d,%c", mode->cache.index[j],(j+16)%15==0?'\n':' '); + fprintf (file, "};\n"); + fprintf (file, "static const unsigned char cache_bits%d[%d] = {\n", mode->Fs/mdctSize, mode->cache.size); + for (j=0;jcache.size;j++) + fprintf (file, "%d,%c", mode->cache.bits[j],(j+16)%15==0?'\n':' '); + fprintf (file, "};\n"); + fprintf (file, "static const unsigned char cache_caps%d[%d] = {\n", mode->Fs/mdctSize, (mode->maxLM+1)*2*mode->nbEBands); + for (j=0;j<(mode->maxLM+1)*2*mode->nbEBands;j++) + fprintf (file, "%d,%c", mode->cache.caps[j],(j+16)%15==0?'\n':' '); + fprintf (file, "};\n"); + + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + + /* FFT twiddles */ + fprintf(file, "#ifndef FFT_TWIDDLES%d_%d\n", mode->Fs, mdctSize); + fprintf(file, "#define FFT_TWIDDLES%d_%d\n", mode->Fs, mdctSize); + fprintf (file, "static const kiss_twiddle_cpx fft_twiddles%d_%d[%d] = {\n", + mode->Fs, mdctSize, mode->mdct.kfft[0]->nfft); + for (j=0;jmdct.kfft[0]->nfft;j++) + fprintf (file, "{" WORD16 ", " WORD16 "},%c", mode->mdct.kfft[0]->twiddles[j].r, mode->mdct.kfft[0]->twiddles[j].i,(j+3)%2==0?'\n':' '); + fprintf (file, "};\n"); + +#ifdef OVERRIDE_FFT + dump_mode_arch(mode); +#endif + /* FFT Bitrev tables */ + for (k=0;k<=mode->mdct.maxshift;k++) + { + fprintf(file, "#ifndef FFT_BITREV%d\n", mode->mdct.kfft[k]->nfft); + fprintf(file, "#define FFT_BITREV%d\n", mode->mdct.kfft[k]->nfft); + fprintf (file, "static const opus_int16 fft_bitrev%d[%d] = {\n", + mode->mdct.kfft[k]->nfft, mode->mdct.kfft[k]->nfft); + for (j=0;jmdct.kfft[k]->nfft;j++) + fprintf (file, "%d,%c", mode->mdct.kfft[k]->bitrev[j],(j+16)%15==0?'\n':' '); + fprintf (file, "};\n"); + + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + } + + /* FFT States */ + for (k=0;k<=mode->mdct.maxshift;k++) + { + fprintf(file, "#ifndef FFT_STATE%d_%d_%d\n", mode->Fs, mdctSize, k); + fprintf(file, "#define FFT_STATE%d_%d_%d\n", mode->Fs, mdctSize, k); + fprintf (file, "static const kiss_fft_state fft_state%d_%d_%d = {\n", + mode->Fs, mdctSize, k); + fprintf (file, "%d, /* nfft */\n", mode->mdct.kfft[k]->nfft); + fprintf (file, WORD16 ", /* scale */\n", mode->mdct.kfft[k]->scale); +#ifdef FIXED_POINT + fprintf (file, "%d, /* scale_shift */\n", mode->mdct.kfft[k]->scale_shift); +#endif + fprintf (file, "%d, /* shift */\n", mode->mdct.kfft[k]->shift); + fprintf (file, "{"); + for (j=0;j<2*MAXFACTORS;j++) + fprintf (file, "%d, ", mode->mdct.kfft[k]->factors[j]); + fprintf (file, "}, /* factors */\n"); + fprintf (file, "fft_bitrev%d, /* bitrev */\n", mode->mdct.kfft[k]->nfft); + fprintf (file, "fft_twiddles%d_%d, /* bitrev */\n", mode->Fs, mdctSize); + + fprintf (file, "#ifdef OVERRIDE_FFT\n"); + fprintf (file, "(arch_fft_state *)&cfg_arch_%d,\n", mode->mdct.kfft[k]->nfft); + fprintf (file, "#else\n"); + fprintf (file, "NULL,\n"); + fprintf(file, "#endif\n"); + + fprintf (file, "};\n"); + + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + } + + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + + /* MDCT twiddles */ + mdct_twiddles_size = mode->mdct.n-(mode->mdct.n/2>>mode->mdct.maxshift); + fprintf(file, "#ifndef MDCT_TWIDDLES%d\n", mdctSize); + fprintf(file, "#define MDCT_TWIDDLES%d\n", mdctSize); + fprintf (file, "static const opus_val16 mdct_twiddles%d[%d] = {\n", + mdctSize, mdct_twiddles_size); + for (j=0;jmdct.trig[j],(j+6)%5==0?'\n':' '); + fprintf (file, "};\n"); + + fprintf(file, "#endif\n"); + fprintf(file, "\n"); + + + /* Print the actual mode data */ + fprintf(file, "static const CELTMode mode%d_%d_%d = {\n", mode->Fs, mdctSize, mode->overlap); + fprintf(file, INT32 ", /* Fs */\n", mode->Fs); + fprintf(file, "%d, /* overlap */\n", mode->overlap); + fprintf(file, "%d, /* nbEBands */\n", mode->nbEBands); + fprintf(file, "%d, /* effEBands */\n", mode->effEBands); + fprintf(file, "{"); + for (j=0;j<4;j++) + fprintf(file, WORD16 ", ", mode->preemph[j]); + fprintf(file, "}, /* preemph */\n"); + if (standard) + fprintf(file, "eband5ms, /* eBands */\n"); + else + fprintf(file, "eBands%d_%d, /* eBands */\n", mode->Fs, mdctSize); + + fprintf(file, "%d, /* maxLM */\n", mode->maxLM); + fprintf(file, "%d, /* nbShortMdcts */\n", mode->nbShortMdcts); + fprintf(file, "%d, /* shortMdctSize */\n", mode->shortMdctSize); + + fprintf(file, "%d, /* nbAllocVectors */\n", mode->nbAllocVectors); + if (standard) + fprintf(file, "band_allocation, /* allocVectors */\n"); + else + fprintf(file, "allocVectors%d_%d, /* allocVectors */\n", mode->Fs, mdctSize); + + fprintf(file, "logN%d, /* logN */\n", framerate); + fprintf(file, "window%d, /* window */\n", mode->overlap); + fprintf(file, "{%d, %d, {", mode->mdct.n, mode->mdct.maxshift); + for (k=0;k<=mode->mdct.maxshift;k++) + fprintf(file, "&fft_state%d_%d_%d, ", mode->Fs, mdctSize, k); + fprintf (file, "}, mdct_twiddles%d}, /* mdct */\n", mdctSize); + + fprintf(file, "{%d, cache_index%d, cache_bits%d, cache_caps%d}, /* cache */\n", + mode->cache.size, mode->Fs/mdctSize, mode->Fs/mdctSize, mode->Fs/mdctSize); + fprintf(file, "};\n"); + } + fprintf(file, "\n"); + fprintf(file, "/* List of all the available modes */\n"); + fprintf(file, "#define TOTAL_MODES %d\n", nb_modes); + fprintf(file, "static const CELTMode * const static_mode_list[TOTAL_MODES] = {\n"); + for (i=0;ishortMdctSize*mode->nbShortMdcts; + fprintf(file, "&mode%d_%d_%d,\n", mode->Fs, mdctSize, mode->overlap); + } + fprintf(file, "};\n"); +} + +void dump_header(FILE *file, CELTMode **modes, int nb_modes) +{ + int i; + int channels = 0; + int frame_size = 0; + int overlap = 0; + fprintf (file, "/* This header file is generated automatically*/\n"); + for (i=0;ishortMdctSize*mode->nbShortMdcts; + else if (frame_size != mode->shortMdctSize*mode->nbShortMdcts) + frame_size = -1; + if (overlap==0) + overlap = mode->overlap; + else if (overlap != mode->overlap) + overlap = -1; + } + if (channels>0) + { + fprintf (file, "#define CHANNELS(mode) %d\n", channels); + if (channels==1) + fprintf (file, "#define DISABLE_STEREO\n"); + } + if (frame_size>0) + { + fprintf (file, "#define FRAMESIZE(mode) %d\n", frame_size); + } + if (overlap>0) + { + fprintf (file, "#define OVERLAP(mode) %d\n", overlap); + } +} + +#ifdef FIXED_POINT +#define BASENAME "static_modes_fixed" +#else +#define BASENAME "static_modes_float" +#endif + +int main(int argc, char **argv) +{ + int i, nb; + FILE *file; + CELTMode **m; + if (argc%2 != 1 || argc<3) + { + fprintf (stderr, "Usage: %s rate frame_size [rate frame_size] [rate frame_size]...\n",argv[0]); + return 1; + } + nb = (argc-1)/2; + m = malloc(nb*sizeof(CELTMode*)); + for (i=0;i +#include +#include "modes.h" +#include "dump_modes_arch.h" +#include + +#if !defined(FIXED_POINT) +# define NE10_FFT_CFG_TYPE_T ne10_fft_cfg_float32_t +# define NE10_FFT_CPX_TYPE_T_STR "ne10_fft_cpx_float32_t" +# define NE10_FFT_STATE_TYPE_T_STR "ne10_fft_state_float32_t" +#else +# define NE10_FFT_CFG_TYPE_T ne10_fft_cfg_int32_t +# define NE10_FFT_CPX_TYPE_T_STR "ne10_fft_cpx_int32_t" +# define NE10_FFT_STATE_TYPE_T_STR "ne10_fft_state_int32_t" +#endif + +static FILE *file; + +void dump_modes_arch_init(CELTMode **modes, int nb_modes) +{ + int i; + + file = fopen(ARM_NE10_ARCH_FILE_NAME, "w"); + fprintf(file, "/* The contents of this file was automatically generated by\n"); + fprintf(file, " * dump_mode_arm_ne10.c with arguments:"); + for (i=0;iFs,mode->shortMdctSize*mode->nbShortMdcts); + } + fprintf(file, "\n * It contains static definitions for some pre-defined modes. */\n"); + fprintf(file, "#include \n\n"); +} + +void dump_modes_arch_finalize() +{ + fclose(file); +} + +void dump_mode_arch(CELTMode *mode) +{ + int k, j; + int mdctSize; + + mdctSize = mode->shortMdctSize*mode->nbShortMdcts; + + fprintf(file, "#ifndef NE10_FFT_PARAMS%d_%d\n", mode->Fs, mdctSize); + fprintf(file, "#define NE10_FFT_PARAMS%d_%d\n", mode->Fs, mdctSize); + /* cfg->factors */ + for(k=0;k<=mode->mdct.maxshift;k++) { + NE10_FFT_CFG_TYPE_T cfg; + cfg = (NE10_FFT_CFG_TYPE_T)mode->mdct.kfft[k]->arch_fft->priv; + if (!cfg) + continue; + fprintf(file, "static const ne10_int32_t ne10_factors_%d[%d] = {\n", + mode->mdct.kfft[k]->nfft, (NE10_MAXFACTORS * 2)); + for(j=0;j<(NE10_MAXFACTORS * 2);j++) { + fprintf(file, "%d,%c", cfg->factors[j],(j+16)%15==0?'\n':' '); + } + fprintf (file, "};\n"); + } + + /* cfg->twiddles */ + for(k=0;k<=mode->mdct.maxshift;k++) { + NE10_FFT_CFG_TYPE_T cfg; + cfg = (NE10_FFT_CFG_TYPE_T)mode->mdct.kfft[k]->arch_fft->priv; + if (!cfg) + continue; + fprintf(file, "static const %s ne10_twiddles_%d[%d] = {\n", + NE10_FFT_CPX_TYPE_T_STR, mode->mdct.kfft[k]->nfft, + mode->mdct.kfft[k]->nfft); + for(j=0;jmdct.kfft[k]->nfft;j++) { +#if !defined(FIXED_POINT) + fprintf(file, "{%#0.8gf,%#0.8gf},%c", + cfg->twiddles[j].r, cfg->twiddles[j].i,(j+4)%3==0?'\n':' '); +#else + fprintf(file, "{%d,%d},%c", + cfg->twiddles[j].r, cfg->twiddles[j].i,(j+4)%3==0?'\n':' '); +#endif + } + fprintf (file, "};\n"); + } + + for(k=0;k<=mode->mdct.maxshift;k++) { + NE10_FFT_CFG_TYPE_T cfg; + cfg = (NE10_FFT_CFG_TYPE_T)mode->mdct.kfft[k]->arch_fft->priv; + if (!cfg) { + fprintf(file, "/* Ne10 does not support scaled FFT for length = %d */\n", + mode->mdct.kfft[k]->nfft); + fprintf(file, "static const arch_fft_state cfg_arch_%d = {\n", mode->mdct.kfft[k]->nfft); + fprintf(file, "0,\n"); + fprintf(file, "NULL\n"); + fprintf(file, "};\n"); + continue; + } + fprintf(file, "static const %s %s_%d = {\n", NE10_FFT_STATE_TYPE_T_STR, + NE10_FFT_STATE_TYPE_T_STR, mode->mdct.kfft[k]->nfft); + fprintf(file, "%d,\n", cfg->nfft); + fprintf(file, "(ne10_int32_t *)ne10_factors_%d,\n", mode->mdct.kfft[k]->nfft); + fprintf(file, "(%s *)ne10_twiddles_%d,\n", + NE10_FFT_CPX_TYPE_T_STR, mode->mdct.kfft[k]->nfft); + fprintf(file, "NULL,\n"); /* buffer */ + fprintf(file, "(%s *)&ne10_twiddles_%d[%d],\n", + NE10_FFT_CPX_TYPE_T_STR, mode->mdct.kfft[k]->nfft, cfg->nfft); +#if !defined(FIXED_POINT) + fprintf(file, "/* is_forward_scaled = true */\n"); + fprintf(file, "(ne10_int32_t) 1,\n"); + fprintf(file, "/* is_backward_scaled = false */\n"); + fprintf(file, "(ne10_int32_t) 0,\n"); +#endif + fprintf(file, "};\n"); + + fprintf(file, "static const arch_fft_state cfg_arch_%d = {\n", + mode->mdct.kfft[k]->nfft); + fprintf(file, "1,\n"); + fprintf(file, "(void *)&%s_%d,\n", + NE10_FFT_STATE_TYPE_T_STR, mode->mdct.kfft[k]->nfft); + fprintf(file, "};\n\n"); + } + fprintf(file, "#endif /* end NE10_FFT_PARAMS%d_%d */\n", mode->Fs, mdctSize); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/ecintrin.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/ecintrin.h new file mode 100644 index 000000000..66a4c36ea --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/ecintrin.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2003-2008 Timothy B. Terriberry + Copyright (c) 2008 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/*Some common macros for potential platform-specific optimization.*/ +#include "opus_types.h" +#include +#include +#include "arch.h" +#if !defined(_ecintrin_H) +# define _ecintrin_H (1) + +/*Some specific platforms may have optimized intrinsic or OPUS_INLINE assembly + versions of these functions which can substantially improve performance. + We define macros for them to allow easy incorporation of these non-ANSI + features.*/ + +/*Modern gcc (4.x) can compile the naive versions of min and max with cmov if + given an appropriate architecture, but the branchless bit-twiddling versions + are just as fast, and do not require any special target architecture. + Earlier gcc versions (3.x) compiled both code to the same assembly + instructions, because of the way they represented ((_b)>(_a)) internally.*/ +# define EC_MINI(_a,_b) ((_a)+(((_b)-(_a))&-((_b)<(_a)))) + +/*Count leading zeros. + This macro should only be used for implementing ec_ilog(), if it is defined. + All other code should use EC_ILOG() instead.*/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#if defined(_MSC_VER) && (_MSC_VER >= 1910) +# include /* Improve compiler throughput. */ +#else +# include +#endif +/*In _DEBUG mode this is not an intrinsic by default.*/ +# pragma intrinsic(_BitScanReverse) + +static __inline int ec_bsr(unsigned long _x){ + unsigned long ret; + _BitScanReverse(&ret,_x); + return (int)ret; +} +# define EC_CLZ0 (1) +# define EC_CLZ(_x) (-ec_bsr(_x)) +#elif defined(ENABLE_TI_DSPLIB) +# include "dsplib.h" +# define EC_CLZ0 (31) +# define EC_CLZ(_x) (_lnorm(_x)) +#elif __GNUC_PREREQ(3,4) +# if INT_MAX>=2147483647 +# define EC_CLZ0 ((int)sizeof(unsigned)*CHAR_BIT) +# define EC_CLZ(_x) (__builtin_clz(_x)) +# elif LONG_MAX>=2147483647L +# define EC_CLZ0 ((int)sizeof(unsigned long)*CHAR_BIT) +# define EC_CLZ(_x) (__builtin_clzl(_x)) +# endif +#endif + +#if defined(EC_CLZ) +/*Note that __builtin_clz is not defined when _x==0, according to the gcc + documentation (and that of the BSR instruction that implements it on x86). + The majority of the time we can never pass it zero. + When we need to, it can be special cased.*/ +# define EC_ILOG(_x) (EC_CLZ0-EC_CLZ(_x)) +#else +int ec_ilog(opus_uint32 _v); +# define EC_ILOG(_x) (ec_ilog(_x)) +#endif +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.c new file mode 100644 index 000000000..70f32016e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.c @@ -0,0 +1,153 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "entcode.h" +#include "arch.h" + +#if !defined(EC_CLZ) +/*This is a fallback for systems where we don't know how to access + a BSR or CLZ instruction (see ecintrin.h). + If you are optimizing Opus on a new platform and it has a native CLZ or + BZR (e.g. cell, MIPS, x86, etc) then making it available to Opus will be + an easy performance win.*/ +int ec_ilog(opus_uint32 _v){ + /*On a Pentium M, this branchless version tested as the fastest on + 1,000,000,000 random 32-bit integers, edging out a similar version with + branches, and a 256-entry LUT version.*/ + int ret; + int m; + ret=!!_v; + m=!!(_v&0xFFFF0000)<<4; + _v>>=m; + ret|=m; + m=!!(_v&0xFF00)<<3; + _v>>=m; + ret|=m; + m=!!(_v&0xF0)<<2; + _v>>=m; + ret|=m; + m=!!(_v&0xC)<<1; + _v>>=m; + ret|=m; + ret+=!!(_v&0x2); + return ret; +} +#endif + +#if 1 +/* This is a faster version of ec_tell_frac() that takes advantage + of the low (1/8 bit) resolution to use just a linear function + followed by a lookup to determine the exact transition thresholds. */ +opus_uint32 ec_tell_frac(ec_ctx *_this){ + static const unsigned correction[8] = + {35733, 38967, 42495, 46340, + 50535, 55109, 60097, 65535}; + opus_uint32 nbits; + opus_uint32 r; + int l; + unsigned b; + nbits=_this->nbits_total<rng); + r=_this->rng>>(l-16); + b = (r>>12)-8; + b += r>correction[b]; + l = (l<<3)+b; + return nbits-l; +} +#else +opus_uint32 ec_tell_frac(ec_ctx *_this){ + opus_uint32 nbits; + opus_uint32 r; + int l; + int i; + /*To handle the non-integral number of bits still left in the encoder/decoder + state, we compute the worst-case number of bits of val that must be + encoded to ensure that the value is inside the range for any possible + subsequent bits. + The computation here is independent of val itself (the decoder does not + even track that value), even though the real number of bits used after + ec_enc_done() may be 1 smaller if rng is a power of two and the + corresponding trailing bits of val are all zeros. + If we did try to track that special case, then coding a value with a + probability of 1/(1<nbits_total<rng); + r=_this->rng>>(l-16); + for(i=BITRES;i-->0;){ + int b; + r=r*r>>15; + b=(int)(r>>16); + l=l<<1|b; + r>>=b; + } + return nbits-l; +} +#endif + +#ifdef USE_SMALL_DIV_TABLE +/* Result of 2^32/(2*i+1), except for i=0. */ +const opus_uint32 SMALL_DIV_TABLE[129] = { + 0xFFFFFFFF, 0x55555555, 0x33333333, 0x24924924, + 0x1C71C71C, 0x1745D174, 0x13B13B13, 0x11111111, + 0x0F0F0F0F, 0x0D79435E, 0x0C30C30C, 0x0B21642C, + 0x0A3D70A3, 0x097B425E, 0x08D3DCB0, 0x08421084, + 0x07C1F07C, 0x07507507, 0x06EB3E45, 0x06906906, + 0x063E7063, 0x05F417D0, 0x05B05B05, 0x0572620A, + 0x05397829, 0x05050505, 0x04D4873E, 0x04A7904A, + 0x047DC11F, 0x0456C797, 0x04325C53, 0x04104104, + 0x03F03F03, 0x03D22635, 0x03B5CC0E, 0x039B0AD1, + 0x0381C0E0, 0x0369D036, 0x03531DEC, 0x033D91D2, + 0x0329161F, 0x03159721, 0x03030303, 0x02F14990, + 0x02E05C0B, 0x02D02D02, 0x02C0B02C, 0x02B1DA46, + 0x02A3A0FD, 0x0295FAD4, 0x0288DF0C, 0x027C4597, + 0x02702702, 0x02647C69, 0x02593F69, 0x024E6A17, + 0x0243F6F0, 0x0239E0D5, 0x02302302, 0x0226B902, + 0x021D9EAD, 0x0214D021, 0x020C49BA, 0x02040810, + 0x01FC07F0, 0x01F44659, 0x01ECC07B, 0x01E573AC, + 0x01DE5D6E, 0x01D77B65, 0x01D0CB58, 0x01CA4B30, + 0x01C3F8F0, 0x01BDD2B8, 0x01B7D6C3, 0x01B20364, + 0x01AC5701, 0x01A6D01A, 0x01A16D3F, 0x019C2D14, + 0x01970E4F, 0x01920FB4, 0x018D3018, 0x01886E5F, + 0x0183C977, 0x017F405F, 0x017AD220, 0x01767DCE, + 0x01724287, 0x016E1F76, 0x016A13CD, 0x01661EC6, + 0x01623FA7, 0x015E75BB, 0x015AC056, 0x01571ED3, + 0x01539094, 0x01501501, 0x014CAB88, 0x0149539E, + 0x01460CBC, 0x0142D662, 0x013FB013, 0x013C995A, + 0x013991C2, 0x013698DF, 0x0133AE45, 0x0130D190, + 0x012E025C, 0x012B404A, 0x01288B01, 0x0125E227, + 0x01234567, 0x0120B470, 0x011E2EF3, 0x011BB4A4, + 0x01194538, 0x0116E068, 0x011485F0, 0x0112358E, + 0x010FEF01, 0x010DB20A, 0x010B7E6E, 0x010953F3, + 0x01073260, 0x0105197F, 0x0103091B, 0x01010101 +}; +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.h new file mode 100644 index 000000000..3763e3f28 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entcode.h @@ -0,0 +1,152 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "opus_types.h" +#include "opus_defines.h" + +#if !defined(_entcode_H) +# define _entcode_H (1) +# include +# include +# include "ecintrin.h" + +extern const opus_uint32 SMALL_DIV_TABLE[129]; + +#ifdef OPUS_ARM_ASM +#define USE_SMALL_DIV_TABLE +#endif + +/*OPT: ec_window must be at least 32 bits, but if you have fast arithmetic on a + larger type, you can speed up the decoder by using it here.*/ +typedef opus_uint32 ec_window; +typedef struct ec_ctx ec_ctx; +typedef struct ec_ctx ec_enc; +typedef struct ec_ctx ec_dec; + +# define EC_WINDOW_SIZE ((int)sizeof(ec_window)*CHAR_BIT) + +/*The number of bits to use for the range-coded part of unsigned integers.*/ +# define EC_UINT_BITS (8) + +/*The resolution of fractional-precision bit usage measurements, i.e., + 3 => 1/8th bits.*/ +# define BITRES 3 + +/*The entropy encoder/decoder context. + We use the same structure for both, so that common functions like ec_tell() + can be used on either one.*/ +struct ec_ctx{ + /*Buffered input/output.*/ + unsigned char *buf; + /*The size of the buffer.*/ + opus_uint32 storage; + /*The offset at which the last byte containing raw bits was read/written.*/ + opus_uint32 end_offs; + /*Bits that will be read from/written at the end.*/ + ec_window end_window; + /*Number of valid bits in end_window.*/ + int nend_bits; + /*The total number of whole bits read/written. + This does not include partial bits currently in the range coder.*/ + int nbits_total; + /*The offset at which the next range coder byte will be read/written.*/ + opus_uint32 offs; + /*The number of values in the current range.*/ + opus_uint32 rng; + /*In the decoder: the difference between the top of the current range and + the input value, minus one. + In the encoder: the low end of the current range.*/ + opus_uint32 val; + /*In the decoder: the saved normalization factor from ec_decode(). + In the encoder: the number of oustanding carry propagating symbols.*/ + opus_uint32 ext; + /*A buffered input/output symbol, awaiting carry propagation.*/ + int rem; + /*Nonzero if an error occurred.*/ + int error; +}; + +static OPUS_INLINE opus_uint32 ec_range_bytes(ec_ctx *_this){ + return _this->offs; +} + +static OPUS_INLINE unsigned char *ec_get_buffer(ec_ctx *_this){ + return _this->buf; +} + +static OPUS_INLINE int ec_get_error(ec_ctx *_this){ + return _this->error; +} + +/*Returns the number of bits "used" by the encoded or decoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Return: The number of bits. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +static OPUS_INLINE int ec_tell(ec_ctx *_this){ + return _this->nbits_total-EC_ILOG(_this->rng); +} + +/*Returns the number of bits "used" by the encoded or decoded symbols so far. + This same number can be computed in either the encoder or the decoder, and is + suitable for making coding decisions. + Return: The number of bits scaled by 2**BITRES. + This will always be slightly larger than the exact value (e.g., all + rounding error is in the positive direction).*/ +opus_uint32 ec_tell_frac(ec_ctx *_this); + +/* Tested exhaustively for all n and for 1<=d<=256 */ +static OPUS_INLINE opus_uint32 celt_udiv(opus_uint32 n, opus_uint32 d) { + celt_sig_assert(d>0); +#ifdef USE_SMALL_DIV_TABLE + if (d>256) + return n/d; + else { + opus_uint32 t, q; + t = EC_ILOG(d&-d); + q = (opus_uint64)SMALL_DIV_TABLE[d>>t]*(n>>(t-1))>>32; + return q+(n-q*d >= d); + } +#else + return n/d; +#endif +} + +static OPUS_INLINE opus_int32 celt_sudiv(opus_int32 n, opus_int32 d) { + celt_sig_assert(d>0); +#ifdef USE_SMALL_DIV_TABLE + if (n<0) + return -(opus_int32)celt_udiv(-n, d); + else + return celt_udiv(n, d); +#else + return n/d; +#endif +} + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.c new file mode 100644 index 000000000..0b3433ed8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.c @@ -0,0 +1,245 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "os_support.h" +#include "arch.h" +#include "entdec.h" +#include "mfrngcod.h" + +/*A range decoder. + This is an entropy decoder based upon \cite{Mar79}, which is itself a + rediscovery of the FIFO arithmetic code introduced by \cite{Pas76}. + It is very similar to arithmetic encoding, except that encoding is done with + digits in any base, instead of with bits, and so it is faster when using + larger bases (i.e.: a byte). + The author claims an average waste of $\frac{1}{2}\log_b(2b)$ bits, where $b$ + is the base, longer than the theoretical optimum, but to my knowledge there + is no published justification for this claim. + This only seems true when using near-infinite precision arithmetic so that + the process is carried out with no rounding errors. + + An excellent description of implementation details is available at + http://www.arturocampos.com/ac_range.html + A recent work \cite{MNW98} which proposes several changes to arithmetic + encoding for efficiency actually re-discovers many of the principles + behind range encoding, and presents a good theoretical analysis of them. + + End of stream is handled by writing out the smallest number of bits that + ensures that the stream will be correctly decoded regardless of the value of + any subsequent bits. + ec_tell() can be used to determine how many bits were needed to decode + all the symbols thus far; other data can be packed in the remaining bits of + the input buffer. + @PHDTHESIS{Pas76, + author="Richard Clark Pasco", + title="Source coding algorithms for fast data compression", + school="Dept. of Electrical Engineering, Stanford University", + address="Stanford, CA", + month=May, + year=1976 + } + @INPROCEEDINGS{Mar79, + author="Martin, G.N.N.", + title="Range encoding: an algorithm for removing redundancy from a digitised + message", + booktitle="Video & Data Recording Conference", + year=1979, + address="Southampton", + month=Jul + } + @ARTICLE{MNW98, + author="Alistair Moffat and Radford Neal and Ian H. Witten", + title="Arithmetic Coding Revisited", + journal="{ACM} Transactions on Information Systems", + year=1998, + volume=16, + number=3, + pages="256--294", + month=Jul, + URL="http://www.stanford.edu/class/ee398a/handouts/papers/Moffat98ArithmCoding.pdf" + }*/ + +static int ec_read_byte(ec_dec *_this){ + return _this->offs<_this->storage?_this->buf[_this->offs++]:0; +} + +static int ec_read_byte_from_end(ec_dec *_this){ + return _this->end_offs<_this->storage? + _this->buf[_this->storage-++(_this->end_offs)]:0; +} + +/*Normalizes the contents of val and rng so that rng lies entirely in the + high-order symbol.*/ +static void ec_dec_normalize(ec_dec *_this){ + /*If the range is too small, rescale it and input some bits.*/ + while(_this->rng<=EC_CODE_BOT){ + int sym; + _this->nbits_total+=EC_SYM_BITS; + _this->rng<<=EC_SYM_BITS; + /*Use up the remaining bits from our last symbol.*/ + sym=_this->rem; + /*Read the next value from the input.*/ + _this->rem=ec_read_byte(_this); + /*Take the rest of the bits we need from this new symbol.*/ + sym=(sym<rem)>>(EC_SYM_BITS-EC_CODE_EXTRA); + /*And subtract them from val, capped to be less than EC_CODE_TOP.*/ + _this->val=((_this->val<buf=_buf; + _this->storage=_storage; + _this->end_offs=0; + _this->end_window=0; + _this->nend_bits=0; + /*This is the offset from which ec_tell() will subtract partial bits. + The final value after the ec_dec_normalize() call will be the same as in + the encoder, but we have to compensate for the bits that are added there.*/ + _this->nbits_total=EC_CODE_BITS+1 + -((EC_CODE_BITS-EC_CODE_EXTRA)/EC_SYM_BITS)*EC_SYM_BITS; + _this->offs=0; + _this->rng=1U<rem=ec_read_byte(_this); + _this->val=_this->rng-1-(_this->rem>>(EC_SYM_BITS-EC_CODE_EXTRA)); + _this->error=0; + /*Normalize the interval.*/ + ec_dec_normalize(_this); +} + +unsigned ec_decode(ec_dec *_this,unsigned _ft){ + unsigned s; + _this->ext=celt_udiv(_this->rng,_ft); + s=(unsigned)(_this->val/_this->ext); + return _ft-EC_MINI(s+1,_ft); +} + +unsigned ec_decode_bin(ec_dec *_this,unsigned _bits){ + unsigned s; + _this->ext=_this->rng>>_bits; + s=(unsigned)(_this->val/_this->ext); + return (1U<<_bits)-EC_MINI(s+1U,1U<<_bits); +} + +void ec_dec_update(ec_dec *_this,unsigned _fl,unsigned _fh,unsigned _ft){ + opus_uint32 s; + s=IMUL32(_this->ext,_ft-_fh); + _this->val-=s; + _this->rng=_fl>0?IMUL32(_this->ext,_fh-_fl):_this->rng-s; + ec_dec_normalize(_this); +} + +/*The probability of having a "one" is 1/(1<<_logp).*/ +int ec_dec_bit_logp(ec_dec *_this,unsigned _logp){ + opus_uint32 r; + opus_uint32 d; + opus_uint32 s; + int ret; + r=_this->rng; + d=_this->val; + s=r>>_logp; + ret=dval=d-s; + _this->rng=ret?s:r-s; + ec_dec_normalize(_this); + return ret; +} + +int ec_dec_icdf(ec_dec *_this,const unsigned char *_icdf,unsigned _ftb){ + opus_uint32 r; + opus_uint32 d; + opus_uint32 s; + opus_uint32 t; + int ret; + s=_this->rng; + d=_this->val; + r=s>>_ftb; + ret=-1; + do{ + t=s; + s=IMUL32(r,_icdf[++ret]); + } + while(dval=d-s; + _this->rng=t-s; + ec_dec_normalize(_this); + return ret; +} + +opus_uint32 ec_dec_uint(ec_dec *_this,opus_uint32 _ft){ + unsigned ft; + unsigned s; + int ftb; + /*In order to optimize EC_ILOG(), it is undefined for the value 0.*/ + celt_assert(_ft>1); + _ft--; + ftb=EC_ILOG(_ft); + if(ftb>EC_UINT_BITS){ + opus_uint32 t; + ftb-=EC_UINT_BITS; + ft=(unsigned)(_ft>>ftb)+1; + s=ec_decode(_this,ft); + ec_dec_update(_this,s,s+1,ft); + t=(opus_uint32)s<error=1; + return _ft; + } + else{ + _ft++; + s=ec_decode(_this,(unsigned)_ft); + ec_dec_update(_this,s,s+1,(unsigned)_ft); + return s; + } +} + +opus_uint32 ec_dec_bits(ec_dec *_this,unsigned _bits){ + ec_window window; + int available; + opus_uint32 ret; + window=_this->end_window; + available=_this->nend_bits; + if((unsigned)available<_bits){ + do{ + window|=(ec_window)ec_read_byte_from_end(_this)<>=_bits; + available-=_bits; + _this->end_window=window; + _this->nend_bits=available; + _this->nbits_total+=_bits; + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.h new file mode 100644 index 000000000..025fc1870 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entdec.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(_entdec_H) +# define _entdec_H (1) +# include +# include "entcode.h" + +/*Initializes the decoder. + _buf: The input buffer to use. + Return: 0 on success, or a negative value on error.*/ +void ec_dec_init(ec_dec *_this,unsigned char *_buf,opus_uint32 _storage); + +/*Calculates the cumulative frequency for the next symbol. + This can then be fed into the probability model to determine what that + symbol is, and the additional frequency information required to advance to + the next symbol. + This function cannot be called more than once without a corresponding call to + ec_dec_update(), or decoding will not proceed correctly. + _ft: The total frequency of the symbols in the alphabet the next symbol was + encoded with. + Return: A cumulative frequency representing the encoded symbol. + If the cumulative frequency of all the symbols before the one that + was encoded was fl, and the cumulative frequency of all the symbols + up to and including the one encoded is fh, then the returned value + will fall in the range [fl,fh).*/ +unsigned ec_decode(ec_dec *_this,unsigned _ft); + +/*Equivalent to ec_decode() with _ft==1<<_bits.*/ +unsigned ec_decode_bin(ec_dec *_this,unsigned _bits); + +/*Advance the decoder past the next symbol using the frequency information the + symbol was encoded with. + Exactly one call to ec_decode() must have been made so that all necessary + intermediate calculations are performed. + _fl: The cumulative frequency of all symbols that come before the symbol + decoded. + _fh: The cumulative frequency of all symbols up to and including the symbol + decoded. + Together with _fl, this defines the range [_fl,_fh) in which the value + returned above must fall. + _ft: The total frequency of the symbols in the alphabet the symbol decoded + was encoded in. + This must be the same as passed to the preceding call to ec_decode().*/ +void ec_dec_update(ec_dec *_this,unsigned _fl,unsigned _fh,unsigned _ft); + +/* Decode a bit that has a 1/(1<<_logp) probability of being a one */ +int ec_dec_bit_logp(ec_dec *_this,unsigned _logp); + +/*Decodes a symbol given an "inverse" CDF table. + No call to ec_dec_update() is necessary after this call. + _icdf: The "inverse" CDF, such that symbol s falls in the range + [s>0?ft-_icdf[s-1]:0,ft-_icdf[s]), where ft=1<<_ftb. + The values must be monotonically non-increasing, and the last value + must be 0. + _ftb: The number of bits of precision in the cumulative distribution. + Return: The decoded symbol s.*/ +int ec_dec_icdf(ec_dec *_this,const unsigned char *_icdf,unsigned _ftb); + +/*Extracts a raw unsigned integer with a non-power-of-2 range from the stream. + The bits must have been encoded with ec_enc_uint(). + No call to ec_dec_update() is necessary after this call. + _ft: The number of integers that can be decoded (one more than the max). + This must be at least 2, and no more than 2**32-1. + Return: The decoded bits.*/ +opus_uint32 ec_dec_uint(ec_dec *_this,opus_uint32 _ft); + +/*Extracts a sequence of raw bits from the stream. + The bits must have been encoded with ec_enc_bits(). + No call to ec_dec_update() is necessary after this call. + _ftb: The number of bits to extract. + This must be between 0 and 25, inclusive. + Return: The decoded bits.*/ +opus_uint32 ec_dec_bits(ec_dec *_this,unsigned _ftb); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.c new file mode 100644 index 000000000..f1750d25b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.c @@ -0,0 +1,294 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if defined(HAVE_CONFIG_H) +# include "config.h" +#endif +#include "os_support.h" +#include "arch.h" +#include "entenc.h" +#include "mfrngcod.h" + +/*A range encoder. + See entdec.c and the references for implementation details \cite{Mar79,MNW98}. + + @INPROCEEDINGS{Mar79, + author="Martin, G.N.N.", + title="Range encoding: an algorithm for removing redundancy from a digitised + message", + booktitle="Video \& Data Recording Conference", + year=1979, + address="Southampton", + month=Jul + } + @ARTICLE{MNW98, + author="Alistair Moffat and Radford Neal and Ian H. Witten", + title="Arithmetic Coding Revisited", + journal="{ACM} Transactions on Information Systems", + year=1998, + volume=16, + number=3, + pages="256--294", + month=Jul, + URL="http://www.stanford.edu/class/ee398/handouts/papers/Moffat98ArithmCoding.pdf" + }*/ + +static int ec_write_byte(ec_enc *_this,unsigned _value){ + if(_this->offs+_this->end_offs>=_this->storage)return -1; + _this->buf[_this->offs++]=(unsigned char)_value; + return 0; +} + +static int ec_write_byte_at_end(ec_enc *_this,unsigned _value){ + if(_this->offs+_this->end_offs>=_this->storage)return -1; + _this->buf[_this->storage-++(_this->end_offs)]=(unsigned char)_value; + return 0; +} + +/*Outputs a symbol, with a carry bit. + If there is a potential to propagate a carry over several symbols, they are + buffered until it can be determined whether or not an actual carry will + occur. + If the counter for the buffered symbols overflows, then the stream becomes + undecodable. + This gives a theoretical limit of a few billion symbols in a single packet on + 32-bit systems. + The alternative is to truncate the range in order to force a carry, but + requires similar carry tracking in the decoder, needlessly slowing it down.*/ +static void ec_enc_carry_out(ec_enc *_this,int _c){ + if(_c!=EC_SYM_MAX){ + /*No further carry propagation possible, flush buffer.*/ + int carry; + carry=_c>>EC_SYM_BITS; + /*Don't output a byte on the first write. + This compare should be taken care of by branch-prediction thereafter.*/ + if(_this->rem>=0)_this->error|=ec_write_byte(_this,_this->rem+carry); + if(_this->ext>0){ + unsigned sym; + sym=(EC_SYM_MAX+carry)&EC_SYM_MAX; + do _this->error|=ec_write_byte(_this,sym); + while(--(_this->ext)>0); + } + _this->rem=_c&EC_SYM_MAX; + } + else _this->ext++; +} + +static OPUS_INLINE void ec_enc_normalize(ec_enc *_this){ + /*If the range is too small, output some bits and rescale it.*/ + while(_this->rng<=EC_CODE_BOT){ + ec_enc_carry_out(_this,(int)(_this->val>>EC_CODE_SHIFT)); + /*Move the next-to-high-order symbol into the high-order position.*/ + _this->val=(_this->val<rng<<=EC_SYM_BITS; + _this->nbits_total+=EC_SYM_BITS; + } +} + +void ec_enc_init(ec_enc *_this,unsigned char *_buf,opus_uint32 _size){ + _this->buf=_buf; + _this->end_offs=0; + _this->end_window=0; + _this->nend_bits=0; + /*This is the offset from which ec_tell() will subtract partial bits.*/ + _this->nbits_total=EC_CODE_BITS+1; + _this->offs=0; + _this->rng=EC_CODE_TOP; + _this->rem=-1; + _this->val=0; + _this->ext=0; + _this->storage=_size; + _this->error=0; +} + +void ec_encode(ec_enc *_this,unsigned _fl,unsigned _fh,unsigned _ft){ + opus_uint32 r; + r=celt_udiv(_this->rng,_ft); + if(_fl>0){ + _this->val+=_this->rng-IMUL32(r,(_ft-_fl)); + _this->rng=IMUL32(r,(_fh-_fl)); + } + else _this->rng-=IMUL32(r,(_ft-_fh)); + ec_enc_normalize(_this); +} + +void ec_encode_bin(ec_enc *_this,unsigned _fl,unsigned _fh,unsigned _bits){ + opus_uint32 r; + r=_this->rng>>_bits; + if(_fl>0){ + _this->val+=_this->rng-IMUL32(r,((1U<<_bits)-_fl)); + _this->rng=IMUL32(r,(_fh-_fl)); + } + else _this->rng-=IMUL32(r,((1U<<_bits)-_fh)); + ec_enc_normalize(_this); +} + +/*The probability of having a "one" is 1/(1<<_logp).*/ +void ec_enc_bit_logp(ec_enc *_this,int _val,unsigned _logp){ + opus_uint32 r; + opus_uint32 s; + opus_uint32 l; + r=_this->rng; + l=_this->val; + s=r>>_logp; + r-=s; + if(_val)_this->val=l+r; + _this->rng=_val?s:r; + ec_enc_normalize(_this); +} + +void ec_enc_icdf(ec_enc *_this,int _s,const unsigned char *_icdf,unsigned _ftb){ + opus_uint32 r; + r=_this->rng>>_ftb; + if(_s>0){ + _this->val+=_this->rng-IMUL32(r,_icdf[_s-1]); + _this->rng=IMUL32(r,_icdf[_s-1]-_icdf[_s]); + } + else _this->rng-=IMUL32(r,_icdf[_s]); + ec_enc_normalize(_this); +} + +void ec_enc_uint(ec_enc *_this,opus_uint32 _fl,opus_uint32 _ft){ + unsigned ft; + unsigned fl; + int ftb; + /*In order to optimize EC_ILOG(), it is undefined for the value 0.*/ + celt_assert(_ft>1); + _ft--; + ftb=EC_ILOG(_ft); + if(ftb>EC_UINT_BITS){ + ftb-=EC_UINT_BITS; + ft=(_ft>>ftb)+1; + fl=(unsigned)(_fl>>ftb); + ec_encode(_this,fl,fl+1,ft); + ec_enc_bits(_this,_fl&(((opus_uint32)1<end_window; + used=_this->nend_bits; + celt_assert(_bits>0); + if(used+_bits>EC_WINDOW_SIZE){ + do{ + _this->error|=ec_write_byte_at_end(_this,(unsigned)window&EC_SYM_MAX); + window>>=EC_SYM_BITS; + used-=EC_SYM_BITS; + } + while(used>=EC_SYM_BITS); + } + window|=(ec_window)_fl<end_window=window; + _this->nend_bits=used; + _this->nbits_total+=_bits; +} + +void ec_enc_patch_initial_bits(ec_enc *_this,unsigned _val,unsigned _nbits){ + int shift; + unsigned mask; + celt_assert(_nbits<=EC_SYM_BITS); + shift=EC_SYM_BITS-_nbits; + mask=((1<<_nbits)-1)<offs>0){ + /*The first byte has been finalized.*/ + _this->buf[0]=(unsigned char)((_this->buf[0]&~mask)|_val<rem>=0){ + /*The first byte is still awaiting carry propagation.*/ + _this->rem=(_this->rem&~mask)|_val<rng<=(EC_CODE_TOP>>_nbits)){ + /*The renormalization loop has never been run.*/ + _this->val=(_this->val&~((opus_uint32)mask<error=-1; +} + +void ec_enc_shrink(ec_enc *_this,opus_uint32 _size){ + celt_assert(_this->offs+_this->end_offs<=_size); + OPUS_MOVE(_this->buf+_size-_this->end_offs, + _this->buf+_this->storage-_this->end_offs,_this->end_offs); + _this->storage=_size; +} + +void ec_enc_done(ec_enc *_this){ + ec_window window; + int used; + opus_uint32 msk; + opus_uint32 end; + int l; + /*We output the minimum number of bits that ensures that the symbols encoded + thus far will be decoded correctly regardless of the bits that follow.*/ + l=EC_CODE_BITS-EC_ILOG(_this->rng); + msk=(EC_CODE_TOP-1)>>l; + end=(_this->val+msk)&~msk; + if((end|msk)>=_this->val+_this->rng){ + l++; + msk>>=1; + end=(_this->val+msk)&~msk; + } + while(l>0){ + ec_enc_carry_out(_this,(int)(end>>EC_CODE_SHIFT)); + end=(end<rem>=0||_this->ext>0)ec_enc_carry_out(_this,0); + /*If we have buffered extra bits, flush them as well.*/ + window=_this->end_window; + used=_this->nend_bits; + while(used>=EC_SYM_BITS){ + _this->error|=ec_write_byte_at_end(_this,(unsigned)window&EC_SYM_MAX); + window>>=EC_SYM_BITS; + used-=EC_SYM_BITS; + } + /*Clear any excess space and add any remaining extra bits to the last byte.*/ + if(!_this->error){ + OPUS_CLEAR(_this->buf+_this->offs, + _this->storage-_this->offs-_this->end_offs); + if(used>0){ + /*If there's no range coder data at all, give up.*/ + if(_this->end_offs>=_this->storage)_this->error=-1; + else{ + l=-l; + /*If we've busted, don't add too many extra bits to the last byte; it + would corrupt the range coder data, and that's more important.*/ + if(_this->offs+_this->end_offs>=_this->storage&&lerror=-1; + } + _this->buf[_this->storage-_this->end_offs-1]|=(unsigned char)window; + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.h new file mode 100644 index 000000000..f502eaf66 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/entenc.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2001-2011 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(_entenc_H) +# define _entenc_H (1) +# include +# include "entcode.h" + +/*Initializes the encoder. + _buf: The buffer to store output bytes in. + _size: The size of the buffer, in chars.*/ +void ec_enc_init(ec_enc *_this,unsigned char *_buf,opus_uint32 _size); +/*Encodes a symbol given its frequency information. + The frequency information must be discernable by the decoder, assuming it + has read only the previous symbols from the stream. + It is allowable to change the frequency information, or even the entire + source alphabet, so long as the decoder can tell from the context of the + previously encoded information that it is supposed to do so as well. + _fl: The cumulative frequency of all symbols that come before the one to be + encoded. + _fh: The cumulative frequency of all symbols up to and including the one to + be encoded. + Together with _fl, this defines the range [_fl,_fh) in which the + decoded value will fall. + _ft: The sum of the frequencies of all the symbols*/ +void ec_encode(ec_enc *_this,unsigned _fl,unsigned _fh,unsigned _ft); + +/*Equivalent to ec_encode() with _ft==1<<_bits.*/ +void ec_encode_bin(ec_enc *_this,unsigned _fl,unsigned _fh,unsigned _bits); + +/* Encode a bit that has a 1/(1<<_logp) probability of being a one */ +void ec_enc_bit_logp(ec_enc *_this,int _val,unsigned _logp); + +/*Encodes a symbol given an "inverse" CDF table. + _s: The index of the symbol to encode. + _icdf: The "inverse" CDF, such that symbol _s falls in the range + [_s>0?ft-_icdf[_s-1]:0,ft-_icdf[_s]), where ft=1<<_ftb. + The values must be monotonically non-increasing, and the last value + must be 0. + _ftb: The number of bits of precision in the cumulative distribution.*/ +void ec_enc_icdf(ec_enc *_this,int _s,const unsigned char *_icdf,unsigned _ftb); + +/*Encodes a raw unsigned integer in the stream. + _fl: The integer to encode. + _ft: The number of integers that can be encoded (one more than the max). + This must be at least 2, and no more than 2**32-1.*/ +void ec_enc_uint(ec_enc *_this,opus_uint32 _fl,opus_uint32 _ft); + +/*Encodes a sequence of raw bits in the stream. + _fl: The bits to encode. + _ftb: The number of bits to encode. + This must be between 1 and 25, inclusive.*/ +void ec_enc_bits(ec_enc *_this,opus_uint32 _fl,unsigned _ftb); + +/*Overwrites a few bits at the very start of an existing stream, after they + have already been encoded. + This makes it possible to have a few flags up front, where it is easy for + decoders to access them without parsing the whole stream, even if their + values are not determined until late in the encoding process, without having + to buffer all the intermediate symbols in the encoder. + In order for this to work, at least _nbits bits must have already been + encoded using probabilities that are an exact power of two. + The encoder can verify the number of encoded bits is sufficient, but cannot + check this latter condition. + _val: The bits to encode (in the least _nbits significant bits). + They will be decoded in order from most-significant to least. + _nbits: The number of bits to overwrite. + This must be no more than 8.*/ +void ec_enc_patch_initial_bits(ec_enc *_this,unsigned _val,unsigned _nbits); + +/*Compacts the data to fit in the target size. + This moves up the raw bits at the end of the current buffer so they are at + the end of the new buffer size. + The caller must ensure that the amount of data that's already been written + will fit in the new size. + _size: The number of bytes in the new buffer. + This must be large enough to contain the bits already written, and + must be no larger than the existing size.*/ +void ec_enc_shrink(ec_enc *_this,opus_uint32 _size); + +/*Indicates that there are no more symbols to encode. + All reamining output bytes are flushed to the output buffer. + ec_enc_init() must be called before the encoder can be used again.*/ +void ec_enc_done(ec_enc *_this); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c5x.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c5x.h new file mode 100644 index 000000000..ea95a998c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c5x.h @@ -0,0 +1,79 @@ +/* Copyright (C) 2003 Jean-Marc Valin */ +/** + @file fixed_c5x.h + @brief Fixed-point operations for the TI C5x DSP family +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_C5X_H +#define FIXED_C5X_H + +#include "dsplib.h" + +#undef IMUL32 +static OPUS_INLINE long IMUL32(long i, long j) +{ + long ac0, ac1; + ac0 = _lmpy(i>>16,j); + ac1 = ac0 + _lmpy(i,j>>16); + return _lmpyu(i,j) + (ac1<<16); +} + +#undef MAX16 +#define MAX16(a,b) _max(a,b) + +#undef MIN16 +#define MIN16(a,b) _min(a,b) + +#undef MAX32 +#define MAX32(a,b) _lmax(a,b) + +#undef MIN32 +#define MIN32(a,b) _lmin(a,b) + +#undef VSHR32 +#define VSHR32(a, shift) _lshl(a,-(shift)) + +#undef MULT16_16_Q15 +#define MULT16_16_Q15(a,b) (_smpy(a,b)) + +#undef MULT16_16SU +#define MULT16_16SU(a,b) _lmpysu(a,b) + +#undef MULT_16_16 +#define MULT_16_16(a,b) _lmpy(a,b) + +/* FIXME: This is technically incorrect and is bound to cause problems. Is there any cleaner solution? */ +#undef MULT16_32_Q15 +#define MULT16_32_Q15(a,b) ADD32(SHL(MULT16_16((a),SHR((b),16)),1), SHR(MULT16_16SU((a),(b)),15)) + +#define celt_ilog2(x) (30 - _lnorm(x)) +#define OVERRIDE_CELT_ILOG2 + +#define celt_maxabs16(x, len) MAX32(EXTEND32(maxval((DATA *)x, len)),-EXTEND32(minval((DATA *)x, len))) +#define OVERRIDE_CELT_MAXABS16 + +#endif /* FIXED_C5X_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c6x.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c6x.h new file mode 100644 index 000000000..bb6ad9278 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_c6x.h @@ -0,0 +1,70 @@ +/* Copyright (C) 2008 CSIRO */ +/** + @file fixed_c6x.h + @brief Fixed-point operations for the TI C6x DSP family +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_C6X_H +#define FIXED_C6X_H + +#undef MULT16_16SU +#define MULT16_16SU(a,b) _mpysu(a,b) + +#undef MULT_16_16 +#define MULT_16_16(a,b) _mpy(a,b) + +#define celt_ilog2(x) (30 - _norm(x)) +#define OVERRIDE_CELT_ILOG2 + +#undef MULT16_32_Q15 +#define MULT16_32_Q15(a,b) (_mpylill(a, b) >> 15) + +#if 0 +#include "dsplib.h" + +#undef MAX16 +#define MAX16(a,b) _max(a,b) + +#undef MIN16 +#define MIN16(a,b) _min(a,b) + +#undef MAX32 +#define MAX32(a,b) _lmax(a,b) + +#undef MIN32 +#define MIN32(a,b) _lmin(a,b) + +#undef VSHR32 +#define VSHR32(a, shift) _lshl(a,-(shift)) + +#undef MULT16_16_Q15 +#define MULT16_16_Q15(a,b) (_smpy(a,b)) + +#define celt_maxabs16(x, len) MAX32(EXTEND32(maxval((DATA *)x, len)),-EXTEND32(minval((DATA *)x, len))) +#define OVERRIDE_CELT_MAXABS16 + +#endif /* FIXED_C6X_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_debug.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_debug.h new file mode 100644 index 000000000..3765baa60 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_debug.h @@ -0,0 +1,836 @@ +/* Copyright (C) 2003-2008 Jean-Marc Valin + Copyright (C) 2007-2012 Xiph.Org Foundation */ +/** + @file fixed_debug.h + @brief Fixed-point operations with debugging +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_DEBUG_H +#define FIXED_DEBUG_H + +#include +#include "opus_defines.h" + +#ifdef CELT_C +OPUS_EXPORT opus_int64 celt_mips=0; +#else +extern opus_int64 celt_mips; +#endif + +#define MULT16_16SU(a,b) ((opus_val32)(opus_val16)(a)*(opus_val32)(opus_uint16)(b)) +#define MULT32_32_Q31(a,b) ADD32(ADD32(SHL32(MULT16_16(SHR32((a),16),SHR((b),16)),1), SHR32(MULT16_16SU(SHR32((a),16),((b)&0x0000ffff)),15)), SHR32(MULT16_16SU(SHR32((b),16),((a)&0x0000ffff)),15)) + +/** 16x32 multiplication, followed by a 16-bit shift right. Results fits in 32 bits */ +#define MULT16_32_Q16(a,b) ADD32(MULT16_16((a),SHR32((b),16)), SHR32(MULT16_16SU((a),((b)&0x0000ffff)),16)) + +#define MULT16_32_P16(a,b) MULT16_32_PX(a,b,16) + +#define QCONST16(x,bits) ((opus_val16)(.5+(x)*(((opus_val32)1)<<(bits)))) +#define QCONST32(x,bits) ((opus_val32)(.5+(x)*(((opus_val32)1)<<(bits)))) + +#define VERIFY_SHORT(x) ((x)<=32767&&(x)>=-32768) +#define VERIFY_INT(x) ((x)<=2147483647LL&&(x)>=-2147483648LL) +#define VERIFY_UINT(x) ((x)<=(2147483647LLU<<1)) + +#define SHR(a,b) SHR32(a,b) +#define PSHR(a,b) PSHR32(a,b) + +/** Add two 32-bit values, ignore any overflows */ +#define ADD32_ovflw(a,b) (celt_mips+=2,(opus_val32)((opus_uint32)(a)+(opus_uint32)(b))) +/** Subtract two 32-bit values, ignore any overflows */ +#define SUB32_ovflw(a,b) (celt_mips+=2,(opus_val32)((opus_uint32)(a)-(opus_uint32)(b))) +/* Avoid MSVC warning C4146: unary minus operator applied to unsigned type */ +/** Negate 32-bit value, ignore any overflows */ +#define NEG32_ovflw(a) (celt_mips+=2,(opus_val32)(0-(opus_uint32)(a))) + +static OPUS_INLINE short NEG16(int x) +{ + int res; + if (!VERIFY_SHORT(x)) + { + fprintf (stderr, "NEG16: input is not short: %d\n", (int)x); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = -x; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "NEG16: output is not short: %d\n", (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips++; + return res; +} +static OPUS_INLINE int NEG32(opus_int64 x) +{ + opus_int64 res; + if (!VERIFY_INT(x)) + { + fprintf (stderr, "NEG16: input is not int: %d\n", (int)x); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = -x; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "NEG16: output is not int: %d\n", (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} + +#define EXTRACT16(x) EXTRACT16_(x, __FILE__, __LINE__) +static OPUS_INLINE short EXTRACT16_(int x, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(x)) + { + fprintf (stderr, "EXTRACT16: input is not short: %d in %s: line %d\n", x, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = x; + celt_mips++; + return res; +} + +#define EXTEND32(x) EXTEND32_(x, __FILE__, __LINE__) +static OPUS_INLINE int EXTEND32_(int x, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(x)) + { + fprintf (stderr, "EXTEND32: input is not short: %d in %s: line %d\n", x, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = x; + celt_mips++; + return res; +} + +#define SHR16(a, shift) SHR16_(a, shift, __FILE__, __LINE__) +static OPUS_INLINE short SHR16_(int a, int shift, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(shift)) + { + fprintf (stderr, "SHR16: inputs are not short: %d >> %d in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a>>shift; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "SHR16: output is not short: %d in %s: line %d\n", res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips++; + return res; +} +#define SHL16(a, shift) SHL16_(a, shift, __FILE__, __LINE__) +static OPUS_INLINE short SHL16_(int a, int shift, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(shift)) + { + fprintf (stderr, "SHL16: inputs are not short: %d %d in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a<>shift; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "SHR32: output is not int: %d\n", (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} +#define SHL32(a, shift) SHL32_(a, shift, __FILE__, __LINE__) +static OPUS_INLINE int SHL32_(opus_int64 a, int shift, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_INT(a) || !VERIFY_SHORT(shift)) + { + fprintf (stderr, "SHL32: inputs are not int: %lld %d in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a<>1))),shift)) +#define VSHR32(a, shift) (((shift)>0) ? SHR32(a, shift) : SHL32(a, -(shift))) + +#define ROUND16(x,a) (celt_mips--,EXTRACT16(PSHR32((x),(a)))) +#define SROUND16(x,a) (celt_mips--,EXTRACT16(SATURATE(PSHR32(x,a), 32767))); + +#define HALF16(x) (SHR16(x,1)) +#define HALF32(x) (SHR32(x,1)) + +#define ADD16(a, b) ADD16_(a, b, __FILE__, __LINE__) +static OPUS_INLINE short ADD16_(int a, int b, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "ADD16: inputs are not short: %d %d in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a+b; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "ADD16: output is not short: %d+%d=%d in %s: line %d\n", a,b,res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips++; + return res; +} + +#define SUB16(a, b) SUB16_(a, b, __FILE__, __LINE__) +static OPUS_INLINE short SUB16_(int a, int b, char *file, int line) +{ + int res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "SUB16: inputs are not short: %d %d in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a-b; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "SUB16: output is not short: %d in %s: line %d\n", res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips++; + return res; +} + +#define ADD32(a, b) ADD32_(a, b, __FILE__, __LINE__) +static OPUS_INLINE int ADD32_(opus_int64 a, opus_int64 b, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_INT(a) || !VERIFY_INT(b)) + { + fprintf (stderr, "ADD32: inputs are not int: %d %d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a+b; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "ADD32: output is not int: %d in %s: line %d\n", (int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} + +#define SUB32(a, b) SUB32_(a, b, __FILE__, __LINE__) +static OPUS_INLINE int SUB32_(opus_int64 a, opus_int64 b, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_INT(a) || !VERIFY_INT(b)) + { + fprintf (stderr, "SUB32: inputs are not int: %d %d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a-b; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "SUB32: output is not int: %d in %s: line %d\n", (int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} + +#undef UADD32 +#define UADD32(a, b) UADD32_(a, b, __FILE__, __LINE__) +static OPUS_INLINE unsigned int UADD32_(opus_uint64 a, opus_uint64 b, char *file, int line) +{ + opus_uint64 res; + if (!VERIFY_UINT(a) || !VERIFY_UINT(b)) + { + fprintf (stderr, "UADD32: inputs are not uint32: %llu %llu in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a+b; + if (!VERIFY_UINT(res)) + { + fprintf (stderr, "UADD32: output is not uint32: %llu in %s: line %d\n", res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} + +#undef USUB32 +#define USUB32(a, b) USUB32_(a, b, __FILE__, __LINE__) +static OPUS_INLINE unsigned int USUB32_(opus_uint64 a, opus_uint64 b, char *file, int line) +{ + opus_uint64 res; + if (!VERIFY_UINT(a) || !VERIFY_UINT(b)) + { + fprintf (stderr, "USUB32: inputs are not uint32: %llu %llu in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + if (a> 16; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT32_32_Q16: output is not int: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=5; + return res; +} + +#define MULT16_16(a, b) MULT16_16_(a, b, __FILE__, __LINE__) +static OPUS_INLINE int MULT16_16_(int a, int b, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16: inputs are not short: %d %d in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_16: output is not int: %d in %s: line %d\n", (int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips++; + return res; +} + +#define MAC16_16(c,a,b) (celt_mips-=2,ADD32((c),MULT16_16((a),(b)))) + +#define MULT16_32_QX(a, b, Q) MULT16_32_QX_(a, b, Q, __FILE__, __LINE__) +static OPUS_INLINE int MULT16_32_QX_(int a, opus_int64 b, int Q, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_INT(b)) + { + fprintf (stderr, "MULT16_32_Q%d: inputs are not short+int: %d %d in %s: line %d\n", Q, (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + if (ABS32(b)>=((opus_val32)(1)<<(15+Q))) + { + fprintf (stderr, "MULT16_32_Q%d: second operand too large: %d %d in %s: line %d\n", Q, (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = (((opus_int64)a)*(opus_int64)b) >> Q; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_32_Q%d: output is not int: %d*%d=%d in %s: line %d\n", Q, (int)a, (int)b,(int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + if (Q==15) + celt_mips+=3; + else + celt_mips+=4; + return res; +} + +#define MULT16_32_PX(a, b, Q) MULT16_32_PX_(a, b, Q, __FILE__, __LINE__) +static OPUS_INLINE int MULT16_32_PX_(int a, opus_int64 b, int Q, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_INT(b)) + { + fprintf (stderr, "MULT16_32_P%d: inputs are not short+int: %d %d in %s: line %d\n\n", Q, (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + if (ABS32(b)>=((opus_int64)(1)<<(15+Q))) + { + fprintf (stderr, "MULT16_32_Q%d: second operand too large: %d %d in %s: line %d\n\n", Q, (int)a, (int)b,file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((((opus_int64)a)*(opus_int64)b) + (((opus_val32)(1)<>1))>> Q; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_32_P%d: output is not int: %d*%d=%d in %s: line %d\n\n", Q, (int)a, (int)b,(int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + if (Q==15) + celt_mips+=4; + else + celt_mips+=5; + return res; +} + +#define MULT16_32_Q15(a,b) MULT16_32_QX(a,b,15) +#define MAC16_32_Q15(c,a,b) (celt_mips-=2,ADD32((c),MULT16_32_Q15((a),(b)))) +#define MAC16_32_Q16(c,a,b) (celt_mips-=2,ADD32((c),MULT16_32_Q16((a),(b)))) + +static OPUS_INLINE int SATURATE(int a, int b) +{ + if (a>b) + a=b; + if (a<-b) + a = -b; + celt_mips+=3; + return a; +} + +static OPUS_INLINE opus_int16 SATURATE16(opus_int32 a) +{ + celt_mips+=3; + if (a>32767) + return 32767; + else if (a<-32768) + return -32768; + else return a; +} + +static OPUS_INLINE int MULT16_16_Q11_32(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_Q11: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res >>= 11; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_16_Q11: output is not short: %d*%d=%d\n", (int)a, (int)b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=3; + return res; +} +static OPUS_INLINE short MULT16_16_Q13(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_Q13: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res >>= 13; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_Q13: output is not short: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=3; + return res; +} +static OPUS_INLINE short MULT16_16_Q14(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_Q14: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res >>= 14; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_Q14: output is not short: %d\n", (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=3; + return res; +} + +#define MULT16_16_Q15(a, b) MULT16_16_Q15_(a, b, __FILE__, __LINE__) +static OPUS_INLINE short MULT16_16_Q15_(int a, int b, char *file, int line) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_Q15: inputs are not short: %d %d in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res >>= 15; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_Q15: output is not short: %d in %s: line %d\n", (int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=1; + return res; +} + +static OPUS_INLINE short MULT16_16_P13(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_P13: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res += 4096; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_16_P13: overflow: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res >>= 13; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_P13: output is not short: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=4; + return res; +} +static OPUS_INLINE short MULT16_16_P14(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_P14: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res += 8192; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_16_P14: overflow: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res >>= 14; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_P14: output is not short: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=4; + return res; +} +static OPUS_INLINE short MULT16_16_P15(int a, int b) +{ + opus_int64 res; + if (!VERIFY_SHORT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "MULT16_16_P15: inputs are not short: %d %d\n", a, b); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = ((opus_int64)a)*b; + res += 16384; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "MULT16_16_P15: overflow: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res >>= 15; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "MULT16_16_P15: output is not short: %d*%d=%d\n", a, b, (int)res); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=2; + return res; +} + +#define DIV32_16(a, b) DIV32_16_(a, b, __FILE__, __LINE__) + +static OPUS_INLINE int DIV32_16_(opus_int64 a, opus_int64 b, char *file, int line) +{ + opus_int64 res; + if (b==0) + { + fprintf(stderr, "DIV32_16: divide by zero: %d/%d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + return 0; + } + if (!VERIFY_INT(a) || !VERIFY_SHORT(b)) + { + fprintf (stderr, "DIV32_16: inputs are not int/short: %d %d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a/b; + if (!VERIFY_SHORT(res)) + { + fprintf (stderr, "DIV32_16: output is not short: %d / %d = %d in %s: line %d\n", (int)a,(int)b,(int)res, file, line); + if (res>32767) + res = 32767; + if (res<-32768) + res = -32768; +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=35; + return res; +} + +#define DIV32(a, b) DIV32_(a, b, __FILE__, __LINE__) +static OPUS_INLINE int DIV32_(opus_int64 a, opus_int64 b, char *file, int line) +{ + opus_int64 res; + if (b==0) + { + fprintf(stderr, "DIV32: divide by zero: %d/%d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + return 0; + } + + if (!VERIFY_INT(a) || !VERIFY_INT(b)) + { + fprintf (stderr, "DIV32: inputs are not int/short: %d %d in %s: line %d\n", (int)a, (int)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + res = a/b; + if (!VERIFY_INT(res)) + { + fprintf (stderr, "DIV32: output is not int: %d in %s: line %d\n", (int)res, file, line); +#ifdef FIXED_DEBUG_ASSERT + celt_assert(0); +#endif + } + celt_mips+=70; + return res; +} + +static OPUS_INLINE opus_val16 SIG2WORD16_generic(celt_sig x) +{ + x = PSHR32(x, SIG_SHIFT); + x = MAX32(x, -32768); + x = MIN32(x, 32767); + return EXTRACT16(x); +} +#define SIG2WORD16(x) (SIG2WORD16_generic(x)) + + +#undef PRINT_MIPS +#define PRINT_MIPS(file) do {fprintf (file, "total complexity = %llu MIPS\n", celt_mips);} while (0); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_generic.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_generic.h new file mode 100644 index 000000000..8f29d46bb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/fixed_generic.h @@ -0,0 +1,188 @@ +/* Copyright (C) 2007-2009 Xiph.Org Foundation + Copyright (C) 2003-2008 Jean-Marc Valin + Copyright (C) 2007-2008 CSIRO */ +/** + @file fixed_generic.h + @brief Generic fixed-point operations +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef FIXED_GENERIC_H +#define FIXED_GENERIC_H + +/** Multiply a 16-bit signed value by a 16-bit unsigned value. The result is a 32-bit signed value */ +#define MULT16_16SU(a,b) ((opus_val32)(opus_val16)(a)*(opus_val32)(opus_uint16)(b)) + +/** 16x32 multiplication, followed by a 16-bit shift right. Results fits in 32 bits */ +#if OPUS_FAST_INT64 +#define MULT16_32_Q16(a,b) ((opus_val32)SHR((opus_int64)((opus_val16)(a))*(b),16)) +#else +#define MULT16_32_Q16(a,b) ADD32(MULT16_16((a),SHR((b),16)), SHR(MULT16_16SU((a),((b)&0x0000ffff)),16)) +#endif + +/** 16x32 multiplication, followed by a 16-bit shift right (round-to-nearest). Results fits in 32 bits */ +#if OPUS_FAST_INT64 +#define MULT16_32_P16(a,b) ((opus_val32)PSHR((opus_int64)((opus_val16)(a))*(b),16)) +#else +#define MULT16_32_P16(a,b) ADD32(MULT16_16((a),SHR((b),16)), PSHR(MULT16_16SU((a),((b)&0x0000ffff)),16)) +#endif + +/** 16x32 multiplication, followed by a 15-bit shift right. Results fits in 32 bits */ +#if OPUS_FAST_INT64 +#define MULT16_32_Q15(a,b) ((opus_val32)SHR((opus_int64)((opus_val16)(a))*(b),15)) +#else +#define MULT16_32_Q15(a,b) ADD32(SHL(MULT16_16((a),SHR((b),16)),1), SHR(MULT16_16SU((a),((b)&0x0000ffff)),15)) +#endif + +/** 32x32 multiplication, followed by a 16-bit shift right. Results fits in 32 bits */ +#if OPUS_FAST_INT64 +#define MULT32_32_Q16(a,b) ((opus_val32)SHR((opus_int64)(a)*(opus_int64)(b),16)) +#else +#define MULT32_32_Q16(a,b) (ADD32(ADD32(ADD32((opus_val32)(SHR32(((opus_uint32)((a)&0x0000ffff)*(opus_uint32)((b)&0x0000ffff)),16)), MULT16_16SU(SHR32(a,16),((b)&0x0000ffff))), MULT16_16SU(SHR32(b,16),((a)&0x0000ffff))), SHL32(MULT16_16(SHR32(a,16),SHR32(b,16)),16))) +#endif + +/** 32x32 multiplication, followed by a 31-bit shift right. Results fits in 32 bits */ +#if OPUS_FAST_INT64 +#define MULT32_32_Q31(a,b) ((opus_val32)SHR((opus_int64)(a)*(opus_int64)(b),31)) +#else +#define MULT32_32_Q31(a,b) ADD32(ADD32(SHL(MULT16_16(SHR((a),16),SHR((b),16)),1), SHR(MULT16_16SU(SHR((a),16),((b)&0x0000ffff)),15)), SHR(MULT16_16SU(SHR((b),16),((a)&0x0000ffff)),15)) +#endif + +/** Compile-time conversion of float constant to 16-bit value */ +#define QCONST16(x,bits) ((opus_val16)(.5+(x)*(((opus_val32)1)<<(bits)))) + +/** Compile-time conversion of float constant to 32-bit value */ +#define QCONST32(x,bits) ((opus_val32)(.5+(x)*(((opus_val32)1)<<(bits)))) + +/** Negate a 16-bit value */ +#define NEG16(x) (-(x)) +/** Negate a 32-bit value */ +#define NEG32(x) (-(x)) + +/** Change a 32-bit value into a 16-bit value. The value is assumed to fit in 16-bit, otherwise the result is undefined */ +#define EXTRACT16(x) ((opus_val16)(x)) +/** Change a 16-bit value into a 32-bit value */ +#define EXTEND32(x) ((opus_val32)(x)) + +/** Arithmetic shift-right of a 16-bit value */ +#define SHR16(a,shift) ((a) >> (shift)) +/** Arithmetic shift-left of a 16-bit value */ +#define SHL16(a,shift) ((opus_int16)((opus_uint16)(a)<<(shift))) +/** Arithmetic shift-right of a 32-bit value */ +#define SHR32(a,shift) ((a) >> (shift)) +/** Arithmetic shift-left of a 32-bit value */ +#define SHL32(a,shift) ((opus_int32)((opus_uint32)(a)<<(shift))) + +/** 32-bit arithmetic shift right with rounding-to-nearest instead of rounding down */ +#define PSHR32(a,shift) (SHR32((a)+((EXTEND32(1)<<((shift))>>1)),shift)) +/** 32-bit arithmetic shift right where the argument can be negative */ +#define VSHR32(a, shift) (((shift)>0) ? SHR32(a, shift) : SHL32(a, -(shift))) + +/** "RAW" macros, should not be used outside of this header file */ +#define SHR(a,shift) ((a) >> (shift)) +#define SHL(a,shift) SHL32(a,shift) +#define PSHR(a,shift) (SHR((a)+((EXTEND32(1)<<((shift))>>1)),shift)) +#define SATURATE(x,a) (((x)>(a) ? (a) : (x)<-(a) ? -(a) : (x))) + +#define SATURATE16(x) (EXTRACT16((x)>32767 ? 32767 : (x)<-32768 ? -32768 : (x))) + +/** Shift by a and round-to-nearest 32-bit value. Result is a 16-bit value */ +#define ROUND16(x,a) (EXTRACT16(PSHR32((x),(a)))) +/** Shift by a and round-to-nearest 32-bit value. Result is a saturated 16-bit value */ +#define SROUND16(x,a) EXTRACT16(SATURATE(PSHR32(x,a), 32767)); + +/** Divide by two */ +#define HALF16(x) (SHR16(x,1)) +#define HALF32(x) (SHR32(x,1)) + +/** Add two 16-bit values */ +#define ADD16(a,b) ((opus_val16)((opus_val16)(a)+(opus_val16)(b))) +/** Subtract two 16-bit values */ +#define SUB16(a,b) ((opus_val16)(a)-(opus_val16)(b)) +/** Add two 32-bit values */ +#define ADD32(a,b) ((opus_val32)(a)+(opus_val32)(b)) +/** Subtract two 32-bit values */ +#define SUB32(a,b) ((opus_val32)(a)-(opus_val32)(b)) + +/** Add two 32-bit values, ignore any overflows */ +#define ADD32_ovflw(a,b) ((opus_val32)((opus_uint32)(a)+(opus_uint32)(b))) +/** Subtract two 32-bit values, ignore any overflows */ +#define SUB32_ovflw(a,b) ((opus_val32)((opus_uint32)(a)-(opus_uint32)(b))) +/* Avoid MSVC warning C4146: unary minus operator applied to unsigned type */ +/** Negate 32-bit value, ignore any overflows */ +#define NEG32_ovflw(a) ((opus_val32)(0-(opus_uint32)(a))) + +/** 16x16 multiplication where the result fits in 16 bits */ +#define MULT16_16_16(a,b) ((((opus_val16)(a))*((opus_val16)(b)))) + +/** 32x32 multiplication where the result fits in 32 bits */ +#define MULT32_32_32(a,b) ((((opus_val32)(a))*((opus_val32)(b)))) + +/* (opus_val32)(opus_val16) gives TI compiler a hint that it's 16x16->32 multiply */ +/** 16x16 multiplication where the result fits in 32 bits */ +#define MULT16_16(a,b) (((opus_val32)(opus_val16)(a))*((opus_val32)(opus_val16)(b))) + +/** 16x16 multiply-add where the result fits in 32 bits */ +#define MAC16_16(c,a,b) (ADD32((c),MULT16_16((a),(b)))) +/** 16x32 multiply, followed by a 15-bit shift right and 32-bit add. + b must fit in 31 bits. + Result fits in 32 bits. */ +#define MAC16_32_Q15(c,a,b) ADD32((c),ADD32(MULT16_16((a),SHR((b),15)), SHR(MULT16_16((a),((b)&0x00007fff)),15))) + +/** 16x32 multiplication, followed by a 16-bit shift right and 32-bit add. + Results fits in 32 bits */ +#define MAC16_32_Q16(c,a,b) ADD32((c),ADD32(MULT16_16((a),SHR((b),16)), SHR(MULT16_16SU((a),((b)&0x0000ffff)),16))) + +#define MULT16_16_Q11_32(a,b) (SHR(MULT16_16((a),(b)),11)) +#define MULT16_16_Q11(a,b) (SHR(MULT16_16((a),(b)),11)) +#define MULT16_16_Q13(a,b) (SHR(MULT16_16((a),(b)),13)) +#define MULT16_16_Q14(a,b) (SHR(MULT16_16((a),(b)),14)) +#define MULT16_16_Q15(a,b) (SHR(MULT16_16((a),(b)),15)) + +#define MULT16_16_P13(a,b) (SHR(ADD32(4096,MULT16_16((a),(b))),13)) +#define MULT16_16_P14(a,b) (SHR(ADD32(8192,MULT16_16((a),(b))),14)) +#define MULT16_16_P15(a,b) (SHR(ADD32(16384,MULT16_16((a),(b))),15)) + +/** Divide a 32-bit value by a 16-bit value. Result fits in 16 bits */ +#define DIV32_16(a,b) ((opus_val16)(((opus_val32)(a))/((opus_val16)(b)))) + +/** Divide a 32-bit value by a 32-bit value. Result fits in 32 bits */ +#define DIV32(a,b) (((opus_val32)(a))/((opus_val32)(b))) + +#if defined(MIPSr1_ASM) +#include "mips/fixed_generic_mipsr1.h" +#endif + +static OPUS_INLINE opus_val16 SIG2WORD16_generic(celt_sig x) +{ + x = PSHR32(x, SIG_SHIFT); + x = MAX32(x, -32768); + x = MIN32(x, 32767); + return EXTRACT16(x); +} +#define SIG2WORD16(x) (SIG2WORD16_generic(x)) + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/float_cast.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/float_cast.h new file mode 100644 index 000000000..9d34976ee --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/float_cast.h @@ -0,0 +1,152 @@ +/* Copyright (C) 2001 Erik de Castro Lopo */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Version 1.1 */ + +#ifndef FLOAT_CAST_H +#define FLOAT_CAST_H + + +#include "arch.h" + +/*============================================================================ +** On Intel Pentium processors (especially PIII and probably P4), converting +** from float to int is very slow. To meet the C specs, the code produced by +** most C compilers targeting Pentium needs to change the FPU rounding mode +** before the float to int conversion is performed. +** +** Changing the FPU rounding mode causes the FPU pipeline to be flushed. It +** is this flushing of the pipeline which is so slow. +** +** Fortunately the ISO C99 specifications define the functions lrint, lrintf, +** llrint and llrintf which fix this problem as a side effect. +** +** On Unix-like systems, the configure process should have detected the +** presence of these functions. If they weren't found we have to replace them +** here with a standard C cast. +*/ + +/* +** The C99 prototypes for lrint and lrintf are as follows: +** +** long int lrintf (float x) ; +** long int lrint (double x) ; +*/ + +/* The presence of the required functions are detected during the configure +** process and the values HAVE_LRINT and HAVE_LRINTF are set accordingly in +** the config.h file. +*/ + +/* With GCC, when SSE is available, the fastest conversion is cvtss2si. */ +#if defined(__GNUC__) && defined(__SSE__) + +#include +static OPUS_INLINE opus_int32 float2int(float x) {return _mm_cvt_ss2si(_mm_set_ss(x));} + +#elif (defined(_MSC_VER) && _MSC_VER >= 1400) && (defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)) + + #include + static OPUS_INLINE opus_int32 float2int(float value) + { + /* _mm_load_ss will generate same code as _mm_set_ss + ** in _MSC_VER >= 1914 /02 so keep __mm_load__ss + ** for backward compatibility. + */ + return _mm_cvtss_si32(_mm_load_ss(&value)); + } + +#elif (defined(_MSC_VER) && _MSC_VER >= 1400) && defined (_M_IX86) + + #include + + /* Win32 doesn't seem to have these functions. + ** Therefore implement OPUS_INLINE versions of these functions here. + */ + + static OPUS_INLINE opus_int32 + float2int (float flt) + { int intgr; + + _asm + { fld flt + fistp intgr + } ; + + return intgr ; + } + +#elif defined(HAVE_LRINTF) + +/* These defines enable functionality introduced with the 1999 ISO C +** standard. They must be defined before the inclusion of math.h to +** engage them. If optimisation is enabled, these functions will be +** inlined. With optimisation switched off, you have to link in the +** maths library using -lm. +*/ + +#define _ISOC9X_SOURCE 1 +#define _ISOC99_SOURCE 1 + +#define __USE_ISOC9X 1 +#define __USE_ISOC99 1 + +#include +#define float2int(x) lrintf(x) + +#elif (defined(HAVE_LRINT)) + +#define _ISOC9X_SOURCE 1 +#define _ISOC99_SOURCE 1 + +#define __USE_ISOC9X 1 +#define __USE_ISOC99 1 + +#include +#define float2int(x) lrint(x) + +#else + +#if (defined(__GNUC__) && defined(__STDC__) && __STDC__ && __STDC_VERSION__ >= 199901L) + /* supported by gcc in C99 mode, but not by all other compilers */ + #warning "Don't have the functions lrint() and lrintf ()." + #warning "Replacing these functions with a standard C cast." +#endif /* __STDC_VERSION__ >= 199901L */ + #include + #define float2int(flt) ((int)(floor(.5+flt))) +#endif + +#ifndef DISABLE_FLOAT_API +static OPUS_INLINE opus_int16 FLOAT2INT16(float x) +{ + x = x*CELT_SIG_SCALE; + x = MAX32(x, -32768); + x = MIN32(x, 32767); + return (opus_int16)float2int(x); +} +#endif /* DISABLE_FLOAT_API */ + +#endif /* FLOAT_CAST_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.c new file mode 100644 index 000000000..83775165d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.c @@ -0,0 +1,604 @@ +/*Copyright (c) 2003-2004, Mark Borgerding + Lots of modifications by Jean-Marc Valin + Copyright (c) 2005-2007, Xiph.Org Foundation + Copyright (c) 2008, Xiph.Org Foundation, CSIRO + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.*/ + +/* This code is originally from Mark Borgerding's KISS-FFT but has been + heavily modified to better suit Opus */ + +#ifndef SKIP_CONFIG_H +# ifdef HAVE_CONFIG_H +# include "config.h" +# endif +#endif + +#include "_kiss_fft_guts.h" +#include "arch.h" +#include "os_support.h" +#include "mathops.h" +#include "stack_alloc.h" + +/* The guts header contains all the multiplication and addition macros that are defined for + complex numbers. It also delares the kf_ internal functions. +*/ + +static void kf_bfly2( + kiss_fft_cpx * Fout, + int m, + int N + ) +{ + kiss_fft_cpx * Fout2; + int i; + (void)m; +#ifdef CUSTOM_MODES + if (m==1) + { + celt_assert(m==1); + for (i=0;itwiddles; + /* m is guaranteed to be a multiple of 4. */ + for (j=0;jtwiddles[fstride*m]; +#endif + for (i=0;itwiddles; + /* For non-custom modes, m is guaranteed to be a multiple of 4. */ + k=m; + do { + + C_MUL(scratch[1],Fout[m] , *tw1); + C_MUL(scratch[2],Fout[m2] , *tw2); + + C_ADD(scratch[3],scratch[1],scratch[2]); + C_SUB(scratch[0],scratch[1],scratch[2]); + tw1 += fstride; + tw2 += fstride*2; + + Fout[m].r = SUB32_ovflw(Fout->r, HALF_OF(scratch[3].r)); + Fout[m].i = SUB32_ovflw(Fout->i, HALF_OF(scratch[3].i)); + + C_MULBYSCALAR( scratch[0] , epi3.i ); + + C_ADDTO(*Fout,scratch[3]); + + Fout[m2].r = ADD32_ovflw(Fout[m].r, scratch[0].i); + Fout[m2].i = SUB32_ovflw(Fout[m].i, scratch[0].r); + + Fout[m].r = SUB32_ovflw(Fout[m].r, scratch[0].i); + Fout[m].i = ADD32_ovflw(Fout[m].i, scratch[0].r); + + ++Fout; + } while(--k); + } +} + + +#ifndef OVERRIDE_kf_bfly5 +static void kf_bfly5( + kiss_fft_cpx * Fout, + const size_t fstride, + const kiss_fft_state *st, + int m, + int N, + int mm + ) +{ + kiss_fft_cpx *Fout0,*Fout1,*Fout2,*Fout3,*Fout4; + int i, u; + kiss_fft_cpx scratch[13]; + const kiss_twiddle_cpx *tw; + kiss_twiddle_cpx ya,yb; + kiss_fft_cpx * Fout_beg = Fout; + +#ifdef FIXED_POINT + ya.r = 10126; + ya.i = -31164; + yb.r = -26510; + yb.i = -19261; +#else + ya = st->twiddles[fstride*m]; + yb = st->twiddles[fstride*2*m]; +#endif + tw=st->twiddles; + + for (i=0;ir = ADD32_ovflw(Fout0->r, ADD32_ovflw(scratch[7].r, scratch[8].r)); + Fout0->i = ADD32_ovflw(Fout0->i, ADD32_ovflw(scratch[7].i, scratch[8].i)); + + scratch[5].r = ADD32_ovflw(scratch[0].r, ADD32_ovflw(S_MUL(scratch[7].r,ya.r), S_MUL(scratch[8].r,yb.r))); + scratch[5].i = ADD32_ovflw(scratch[0].i, ADD32_ovflw(S_MUL(scratch[7].i,ya.r), S_MUL(scratch[8].i,yb.r))); + + scratch[6].r = ADD32_ovflw(S_MUL(scratch[10].i,ya.i), S_MUL(scratch[9].i,yb.i)); + scratch[6].i = NEG32_ovflw(ADD32_ovflw(S_MUL(scratch[10].r,ya.i), S_MUL(scratch[9].r,yb.i))); + + C_SUB(*Fout1,scratch[5],scratch[6]); + C_ADD(*Fout4,scratch[5],scratch[6]); + + scratch[11].r = ADD32_ovflw(scratch[0].r, ADD32_ovflw(S_MUL(scratch[7].r,yb.r), S_MUL(scratch[8].r,ya.r))); + scratch[11].i = ADD32_ovflw(scratch[0].i, ADD32_ovflw(S_MUL(scratch[7].i,yb.r), S_MUL(scratch[8].i,ya.r))); + scratch[12].r = SUB32_ovflw(S_MUL(scratch[9].i,ya.i), S_MUL(scratch[10].i,yb.i)); + scratch[12].i = SUB32_ovflw(S_MUL(scratch[10].r,yb.i), S_MUL(scratch[9].r,ya.i)); + + C_ADD(*Fout2,scratch[11],scratch[12]); + C_SUB(*Fout3,scratch[11],scratch[12]); + + ++Fout0;++Fout1;++Fout2;++Fout3;++Fout4; + } + } +} +#endif /* OVERRIDE_kf_bfly5 */ + + +#endif + + +#ifdef CUSTOM_MODES + +static +void compute_bitrev_table( + int Fout, + opus_int16 *f, + const size_t fstride, + int in_stride, + opus_int16 * factors, + const kiss_fft_state *st + ) +{ + const int p=*factors++; /* the radix */ + const int m=*factors++; /* stage's fft length/p */ + + /*printf ("fft %d %d %d %d %d %d\n", p*m, m, p, s2, fstride*in_stride, N);*/ + if (m==1) + { + int j; + for (j=0;j32000 || (opus_int32)p*(opus_int32)p > n) + p = n; /* no more factors, skip to end */ + } + n /= p; +#ifdef RADIX_TWO_ONLY + if (p!=2 && p != 4) +#else + if (p>5) +#endif + { + return 0; + } + facbuf[2*stages] = p; + if (p==2 && stages > 1) + { + facbuf[2*stages] = 4; + facbuf[2] = 2; + } + stages++; + } while (n > 1); + n = nbak; + /* Reverse the order to get the radix 4 at the end, so we can use the + fast degenerate case. It turns out that reversing the order also + improves the noise behaviour. */ + for (i=0;i= memneeded) + st = (kiss_fft_state*)mem; + *lenmem = memneeded; + } + if (st) { + opus_int16 *bitrev; + kiss_twiddle_cpx *twiddles; + + st->nfft=nfft; +#ifdef FIXED_POINT + st->scale_shift = celt_ilog2(st->nfft); + if (st->nfft == 1<scale_shift) + st->scale = Q15ONE; + else + st->scale = (1073741824+st->nfft/2)/st->nfft>>(15-st->scale_shift); +#else + st->scale = 1.f/nfft; +#endif + if (base != NULL) + { + st->twiddles = base->twiddles; + st->shift = 0; + while (st->shift < 32 && nfft<shift != base->nfft) + st->shift++; + if (st->shift>=32) + goto fail; + } else { + st->twiddles = twiddles = (kiss_twiddle_cpx*)KISS_FFT_MALLOC(sizeof(kiss_twiddle_cpx)*nfft); + compute_twiddles(twiddles, nfft); + st->shift = -1; + } + if (!kf_factor(nfft,st->factors)) + { + goto fail; + } + + /* bitrev */ + st->bitrev = bitrev = (opus_int16*)KISS_FFT_MALLOC(sizeof(opus_int16)*nfft); + if (st->bitrev==NULL) + goto fail; + compute_bitrev_table(0, bitrev, 1,1, st->factors,st); + + /* Initialize architecture specific fft parameters */ + if (opus_fft_alloc_arch(st, arch)) + goto fail; + } + return st; +fail: + opus_fft_free(st, arch); + return NULL; +} + +kiss_fft_state *opus_fft_alloc(int nfft,void * mem,size_t * lenmem, int arch) +{ + return opus_fft_alloc_twiddles(nfft, mem, lenmem, NULL, arch); +} + +void opus_fft_free_arch_c(kiss_fft_state *st) { + (void)st; +} + +void opus_fft_free(const kiss_fft_state *cfg, int arch) +{ + if (cfg) + { + opus_fft_free_arch((kiss_fft_state *)cfg, arch); + opus_free((opus_int16*)cfg->bitrev); + if (cfg->shift < 0) + opus_free((kiss_twiddle_cpx*)cfg->twiddles); + opus_free((kiss_fft_state*)cfg); + } +} + +#endif /* CUSTOM_MODES */ + +void opus_fft_impl(const kiss_fft_state *st,kiss_fft_cpx *fout) +{ + int m2, m; + int p; + int L; + int fstride[MAXFACTORS]; + int i; + int shift; + + /* st->shift can be -1 */ + shift = st->shift>0 ? st->shift : 0; + + fstride[0] = 1; + L=0; + do { + p = st->factors[2*L]; + m = st->factors[2*L+1]; + fstride[L+1] = fstride[L]*p; + L++; + } while(m!=1); + m = st->factors[2*L-1]; + for (i=L-1;i>=0;i--) + { + if (i!=0) + m2 = st->factors[2*i-1]; + else + m2 = 1; + switch (st->factors[2*i]) + { + case 2: + kf_bfly2(fout, m, fstride[i]); + break; + case 4: + kf_bfly4(fout,fstride[i]<scale_shift-1; +#endif + scale = st->scale; + + celt_assert2 (fin != fout, "In-place FFT not supported"); + /* Bit-reverse the input */ + for (i=0;infft;i++) + { + kiss_fft_cpx x = fin[i]; + fout[st->bitrev[i]].r = SHR32(MULT16_32_Q16(scale, x.r), scale_shift); + fout[st->bitrev[i]].i = SHR32(MULT16_32_Q16(scale, x.i), scale_shift); + } + opus_fft_impl(st, fout); +} + + +void opus_ifft_c(const kiss_fft_state *st,const kiss_fft_cpx *fin,kiss_fft_cpx *fout) +{ + int i; + celt_assert2 (fin != fout, "In-place FFT not supported"); + /* Bit-reverse the input */ + for (i=0;infft;i++) + fout[st->bitrev[i]] = fin[i]; + for (i=0;infft;i++) + fout[i].i = -fout[i].i; + opus_fft_impl(st, fout); + for (i=0;infft;i++) + fout[i].i = -fout[i].i; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.h new file mode 100644 index 000000000..bffa2bfad --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/kiss_fft.h @@ -0,0 +1,200 @@ +/*Copyright (c) 2003-2004, Mark Borgerding + Lots of modifications by Jean-Marc Valin + Copyright (c) 2005-2007, Xiph.Org Foundation + Copyright (c) 2008, Xiph.Org Foundation, CSIRO + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.*/ + +#ifndef KISS_FFT_H +#define KISS_FFT_H + +#include +#include +#include "arch.h" +#include "cpu_support.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef USE_SIMD +# include +# define kiss_fft_scalar __m128 +#define KISS_FFT_MALLOC(nbytes) memalign(16,nbytes) +#else +#define KISS_FFT_MALLOC opus_alloc +#endif + +#ifdef FIXED_POINT +#include "arch.h" + +# define kiss_fft_scalar opus_int32 +# define kiss_twiddle_scalar opus_int16 + + +#else +# ifndef kiss_fft_scalar +/* default is float */ +# define kiss_fft_scalar float +# define kiss_twiddle_scalar float +# define KF_SUFFIX _celt_single +# endif +#endif + +typedef struct { + kiss_fft_scalar r; + kiss_fft_scalar i; +}kiss_fft_cpx; + +typedef struct { + kiss_twiddle_scalar r; + kiss_twiddle_scalar i; +}kiss_twiddle_cpx; + +#define MAXFACTORS 8 +/* e.g. an fft of length 128 has 4 factors + as far as kissfft is concerned + 4*4*4*2 + */ + +typedef struct arch_fft_state{ + int is_supported; + void *priv; +} arch_fft_state; + +typedef struct kiss_fft_state{ + int nfft; + opus_val16 scale; +#ifdef FIXED_POINT + int scale_shift; +#endif + int shift; + opus_int16 factors[2*MAXFACTORS]; + const opus_int16 *bitrev; + const kiss_twiddle_cpx *twiddles; + arch_fft_state *arch_fft; +} kiss_fft_state; + +#if defined(HAVE_ARM_NE10) +#include "arm/fft_arm.h" +#endif + +/*typedef struct kiss_fft_state* kiss_fft_cfg;*/ + +/** + * opus_fft_alloc + * + * Initialize a FFT (or IFFT) algorithm's cfg/state buffer. + * + * typical usage: kiss_fft_cfg mycfg=opus_fft_alloc(1024,0,NULL,NULL); + * + * The return value from fft_alloc is a cfg buffer used internally + * by the fft routine or NULL. + * + * If lenmem is NULL, then opus_fft_alloc will allocate a cfg buffer using malloc. + * The returned value should be free()d when done to avoid memory leaks. + * + * The state can be placed in a user supplied buffer 'mem': + * If lenmem is not NULL and mem is not NULL and *lenmem is large enough, + * then the function places the cfg in mem and the size used in *lenmem + * and returns mem. + * + * If lenmem is not NULL and ( mem is NULL or *lenmem is not large enough), + * then the function returns NULL and places the minimum cfg + * buffer size in *lenmem. + * */ + +kiss_fft_state *opus_fft_alloc_twiddles(int nfft,void * mem,size_t * lenmem, const kiss_fft_state *base, int arch); + +kiss_fft_state *opus_fft_alloc(int nfft,void * mem,size_t * lenmem, int arch); + +/** + * opus_fft(cfg,in_out_buf) + * + * Perform an FFT on a complex input buffer. + * for a forward FFT, + * fin should be f[0] , f[1] , ... ,f[nfft-1] + * fout will be F[0] , F[1] , ... ,F[nfft-1] + * Note that each element is complex and can be accessed like + f[k].r and f[k].i + * */ +void opus_fft_c(const kiss_fft_state *cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); +void opus_ifft_c(const kiss_fft_state *cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); + +void opus_fft_impl(const kiss_fft_state *st,kiss_fft_cpx *fout); +void opus_ifft_impl(const kiss_fft_state *st,kiss_fft_cpx *fout); + +void opus_fft_free(const kiss_fft_state *cfg, int arch); + + +void opus_fft_free_arch_c(kiss_fft_state *st); +int opus_fft_alloc_arch_c(kiss_fft_state *st); + +#if !defined(OVERRIDE_OPUS_FFT) +/* Is run-time CPU detection enabled on this platform? */ +#if defined(OPUS_HAVE_RTCD) && (defined(HAVE_ARM_NE10)) + +extern int (*const OPUS_FFT_ALLOC_ARCH_IMPL[OPUS_ARCHMASK+1])( + kiss_fft_state *st); + +#define opus_fft_alloc_arch(_st, arch) \ + ((*OPUS_FFT_ALLOC_ARCH_IMPL[(arch)&OPUS_ARCHMASK])(_st)) + +extern void (*const OPUS_FFT_FREE_ARCH_IMPL[OPUS_ARCHMASK+1])( + kiss_fft_state *st); +#define opus_fft_free_arch(_st, arch) \ + ((*OPUS_FFT_FREE_ARCH_IMPL[(arch)&OPUS_ARCHMASK])(_st)) + +extern void (*const OPUS_FFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg, + const kiss_fft_cpx *fin, kiss_fft_cpx *fout); +#define opus_fft(_cfg, _fin, _fout, arch) \ + ((*OPUS_FFT[(arch)&OPUS_ARCHMASK])(_cfg, _fin, _fout)) + +extern void (*const OPUS_IFFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg, + const kiss_fft_cpx *fin, kiss_fft_cpx *fout); +#define opus_ifft(_cfg, _fin, _fout, arch) \ + ((*OPUS_IFFT[(arch)&OPUS_ARCHMASK])(_cfg, _fin, _fout)) + +#else /* else for if defined(OPUS_HAVE_RTCD) && (defined(HAVE_ARM_NE10)) */ + +#define opus_fft_alloc_arch(_st, arch) \ + ((void)(arch), opus_fft_alloc_arch_c(_st)) + +#define opus_fft_free_arch(_st, arch) \ + ((void)(arch), opus_fft_free_arch_c(_st)) + +#define opus_fft(_cfg, _fin, _fout, arch) \ + ((void)(arch), opus_fft_c(_cfg, _fin, _fout)) + +#define opus_ifft(_cfg, _fin, _fout, arch) \ + ((void)(arch), opus_ifft_c(_cfg, _fin, _fout)) + +#endif /* end if defined(OPUS_HAVE_RTCD) && (defined(HAVE_ARM_NE10)) */ +#endif /* end if !defined(OVERRIDE_OPUS_FFT) */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/laplace.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/laplace.c new file mode 100644 index 000000000..a7bca874b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/laplace.c @@ -0,0 +1,134 @@ +/* Copyright (c) 2007 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "laplace.h" +#include "mathops.h" + +/* The minimum probability of an energy delta (out of 32768). */ +#define LAPLACE_LOG_MINP (0) +#define LAPLACE_MINP (1<>15; +} + +void ec_laplace_encode(ec_enc *enc, int *value, unsigned fs, int decay) +{ + unsigned fl; + int val = *value; + fl = 0; + if (val) + { + int s; + int i; + s = -(val<0); + val = (val+s)^s; + fl = fs; + fs = ec_laplace_get_freq1(fs, decay); + /* Search the decaying part of the PDF.*/ + for (i=1; fs > 0 && i < val; i++) + { + fs *= 2; + fl += fs+2*LAPLACE_MINP; + fs = (fs*(opus_int32)decay)>>15; + } + /* Everything beyond that has probability LAPLACE_MINP. */ + if (!fs) + { + int di; + int ndi_max; + ndi_max = (32768-fl+LAPLACE_MINP-1)>>LAPLACE_LOG_MINP; + ndi_max = (ndi_max-s)>>1; + di = IMIN(val - i, ndi_max - 1); + fl += (2*di+1+s)*LAPLACE_MINP; + fs = IMIN(LAPLACE_MINP, 32768-fl); + *value = (i+di+s)^s; + } + else + { + fs += LAPLACE_MINP; + fl += fs&~s; + } + celt_assert(fl+fs<=32768); + celt_assert(fs>0); + } + ec_encode_bin(enc, fl, fl+fs, 15); +} + +int ec_laplace_decode(ec_dec *dec, unsigned fs, int decay) +{ + int val=0; + unsigned fl; + unsigned fm; + fm = ec_decode_bin(dec, 15); + fl = 0; + if (fm >= fs) + { + val++; + fl = fs; + fs = ec_laplace_get_freq1(fs, decay)+LAPLACE_MINP; + /* Search the decaying part of the PDF.*/ + while(fs > LAPLACE_MINP && fm >= fl+2*fs) + { + fs *= 2; + fl += fs; + fs = ((fs-2*LAPLACE_MINP)*(opus_int32)decay)>>15; + fs += LAPLACE_MINP; + val++; + } + /* Everything beyond that has probability LAPLACE_MINP. */ + if (fs <= LAPLACE_MINP) + { + int di; + di = (fm-fl)>>(LAPLACE_LOG_MINP+1); + val += di; + fl += 2*di*LAPLACE_MINP; + } + if (fm < fl+fs) + val = -val; + else + fl += fs; + } + celt_assert(fl<32768); + celt_assert(fs>0); + celt_assert(fl<=fm); + celt_assert(fm>1; + b=1U<>=1; + bshift--; + } + while(bshift>=0); + return g; +} + +#ifdef FIXED_POINT + +opus_val32 frac_div32(opus_val32 a, opus_val32 b) +{ + opus_val16 rcp; + opus_val32 result, rem; + int shift = celt_ilog2(b)-29; + a = VSHR32(a,shift); + b = VSHR32(b,shift); + /* 16-bit reciprocal */ + rcp = ROUND16(celt_rcp(ROUND16(b,16)),3); + result = MULT16_32_Q15(rcp, a); + rem = PSHR32(a,2)-MULT32_32_Q31(result, b); + result = ADD32(result, SHL32(MULT16_32_Q15(rcp, rem),2)); + if (result >= 536870912) /* 2^29 */ + return 2147483647; /* 2^31 - 1 */ + else if (result <= -536870912) /* -2^29 */ + return -2147483647; /* -2^31 */ + else + return SHL32(result, 2); +} + +/** Reciprocal sqrt approximation in the range [0.25,1) (Q16 in, Q14 out) */ +opus_val16 celt_rsqrt_norm(opus_val32 x) +{ + opus_val16 n; + opus_val16 r; + opus_val16 r2; + opus_val16 y; + /* Range of n is [-16384,32767] ([-0.5,1) in Q15). */ + n = x-32768; + /* Get a rough initial guess for the root. + The optimal minimax quadratic approximation (using relative error) is + r = 1.437799046117536+n*(-0.823394375837328+n*0.4096419668459485). + Coefficients here, and the final result r, are Q14.*/ + r = ADD16(23557, MULT16_16_Q15(n, ADD16(-13490, MULT16_16_Q15(n, 6713)))); + /* We want y = x*r*r-1 in Q15, but x is 32-bit Q16 and r is Q14. + We can compute the result from n and r using Q15 multiplies with some + adjustment, carefully done to avoid overflow. + Range of y is [-1564,1594]. */ + r2 = MULT16_16_Q15(r, r); + y = SHL16(SUB16(ADD16(MULT16_16_Q15(r2, n), r2), 16384), 1); + /* Apply a 2nd-order Householder iteration: r += r*y*(y*0.375-0.5). + This yields the Q14 reciprocal square root of the Q16 x, with a maximum + relative error of 1.04956E-4, a (relative) RMSE of 2.80979E-5, and a + peak absolute error of 2.26591/16384. */ + return ADD16(r, MULT16_16_Q15(r, MULT16_16_Q15(y, + SUB16(MULT16_16_Q15(y, 12288), 16384)))); +} + +/** Sqrt approximation (QX input, QX/2 output) */ +opus_val32 celt_sqrt(opus_val32 x) +{ + int k; + opus_val16 n; + opus_val32 rt; + static const opus_val16 C[5] = {23175, 11561, -3011, 1699, -664}; + if (x==0) + return 0; + else if (x>=1073741824) + return 32767; + k = (celt_ilog2(x)>>1)-7; + x = VSHR32(x, 2*k); + n = x-32768; + rt = ADD16(C[0], MULT16_16_Q15(n, ADD16(C[1], MULT16_16_Q15(n, ADD16(C[2], + MULT16_16_Q15(n, ADD16(C[3], MULT16_16_Q15(n, (C[4]))))))))); + rt = VSHR32(rt,7-k); + return rt; +} + +#define L1 32767 +#define L2 -7651 +#define L3 8277 +#define L4 -626 + +static OPUS_INLINE opus_val16 _celt_cos_pi_2(opus_val16 x) +{ + opus_val16 x2; + + x2 = MULT16_16_P15(x,x); + return ADD16(1,MIN16(32766,ADD32(SUB16(L1,x2), MULT16_16_P15(x2, ADD32(L2, MULT16_16_P15(x2, ADD32(L3, MULT16_16_P15(L4, x2 + )))))))); +} + +#undef L1 +#undef L2 +#undef L3 +#undef L4 + +opus_val16 celt_cos_norm(opus_val32 x) +{ + x = x&0x0001ffff; + if (x>SHL32(EXTEND32(1), 16)) + x = SUB32(SHL32(EXTEND32(1), 17),x); + if (x&0x00007fff) + { + if (x0); + i = celt_ilog2(x); + /* n is Q15 with range [0,1). */ + n = VSHR32(x,i-15)-32768; + /* Start with a linear approximation: + r = 1.8823529411764706-0.9411764705882353*n. + The coefficients and the result are Q14 in the range [15420,30840].*/ + r = ADD16(30840, MULT16_16_Q15(-15420, n)); + /* Perform two Newton iterations: + r -= r*((r*n)-1.Q15) + = r*((r*n)+(r-1.Q15)). */ + r = SUB16(r, MULT16_16_Q15(r, + ADD16(MULT16_16_Q15(r, n), ADD16(r, -32768)))); + /* We subtract an extra 1 in the second iteration to avoid overflow; it also + neatly compensates for truncation error in the rest of the process. */ + r = SUB16(r, ADD16(1, MULT16_16_Q15(r, + ADD16(MULT16_16_Q15(r, n), ADD16(r, -32768))))); + /* r is now the Q15 solution to 2/(n+1), with a maximum relative error + of 7.05346E-5, a (relative) RMSE of 2.14418E-5, and a peak absolute + error of 1.24665/32768. */ + return VSHR32(EXTEND32(r),i-16); +} + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mathops.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mathops.h new file mode 100644 index 000000000..fe29dac1c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mathops.h @@ -0,0 +1,290 @@ +/* Copyright (c) 2002-2008 Jean-Marc Valin + Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/** + @file mathops.h + @brief Various math functions +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef MATHOPS_H +#define MATHOPS_H + +#include "arch.h" +#include "entcode.h" +#include "os_support.h" + +#define PI 3.141592653f + +/* Multiplies two 16-bit fractional values. Bit-exactness of this macro is important */ +#define FRAC_MUL16(a,b) ((16384+((opus_int32)(opus_int16)(a)*(opus_int16)(b)))>>15) + +unsigned isqrt32(opus_uint32 _val); + +/* CELT doesn't need it for fixed-point, by analysis.c does. */ +#if !defined(FIXED_POINT) || defined(ANALYSIS_C) +#define cA 0.43157974f +#define cB 0.67848403f +#define cC 0.08595542f +#define cE ((float)PI/2) +static OPUS_INLINE float fast_atan2f(float y, float x) { + float x2, y2; + x2 = x*x; + y2 = y*y; + /* For very small values, we don't care about the answer, so + we can just return 0. */ + if (x2 + y2 < 1e-18f) + { + return 0; + } + if(x2>23)-127; + in.i -= (opus_uint32)integer<<23; + frac = in.f - 1.5f; + frac = -0.41445418f + frac*(0.95909232f + + frac*(-0.33951290f + frac*0.16541097f)); + return 1+integer+frac; +} + +/** Base-2 exponential approximation (2^x). */ +static OPUS_INLINE float celt_exp2(float x) +{ + int integer; + float frac; + union { + float f; + opus_uint32 i; + } res; + integer = floor(x); + if (integer < -50) + return 0; + frac = x-integer; + /* K0 = 1, K1 = log(2), K2 = 3-4*log(2), K3 = 3*log(2) - 2 */ + res.f = 0.99992522f + frac * (0.69583354f + + frac * (0.22606716f + 0.078024523f*frac)); + res.i = (res.i + ((opus_uint32)integer<<23)) & 0x7fffffff; + return res.f; +} + +#else +#define celt_log2(x) ((float)(1.442695040888963387*log(x))) +#define celt_exp2(x) ((float)exp(0.6931471805599453094*(x))) +#endif + +#endif + +#ifdef FIXED_POINT + +#include "os_support.h" + +#ifndef OVERRIDE_CELT_ILOG2 +/** Integer log in base2. Undefined for zero and negative numbers */ +static OPUS_INLINE opus_int16 celt_ilog2(opus_int32 x) +{ + celt_sig_assert(x>0); + return EC_ILOG(x)-1; +} +#endif + + +/** Integer log in base2. Defined for zero, but not for negative numbers */ +static OPUS_INLINE opus_int16 celt_zlog2(opus_val32 x) +{ + return x <= 0 ? 0 : celt_ilog2(x); +} + +opus_val16 celt_rsqrt_norm(opus_val32 x); + +opus_val32 celt_sqrt(opus_val32 x); + +opus_val16 celt_cos_norm(opus_val32 x); + +/** Base-2 logarithm approximation (log2(x)). (Q14 input, Q10 output) */ +static OPUS_INLINE opus_val16 celt_log2(opus_val32 x) +{ + int i; + opus_val16 n, frac; + /* -0.41509302963303146, 0.9609890551383969, -0.31836011537636605, + 0.15530808010959576, -0.08556153059057618 */ + static const opus_val16 C[5] = {-6801+(1<<(13-DB_SHIFT)), 15746, -5217, 2545, -1401}; + if (x==0) + return -32767; + i = celt_ilog2(x); + n = VSHR32(x,i-15)-32768-16384; + frac = ADD16(C[0], MULT16_16_Q15(n, ADD16(C[1], MULT16_16_Q15(n, ADD16(C[2], MULT16_16_Q15(n, ADD16(C[3], MULT16_16_Q15(n, C[4])))))))); + return SHL16(i-13,DB_SHIFT)+SHR16(frac,14-DB_SHIFT); +} + +/* + K0 = 1 + K1 = log(2) + K2 = 3-4*log(2) + K3 = 3*log(2) - 2 +*/ +#define D0 16383 +#define D1 22804 +#define D2 14819 +#define D3 10204 + +static OPUS_INLINE opus_val32 celt_exp2_frac(opus_val16 x) +{ + opus_val16 frac; + frac = SHL16(x, 4); + return ADD16(D0, MULT16_16_Q15(frac, ADD16(D1, MULT16_16_Q15(frac, ADD16(D2 , MULT16_16_Q15(D3,frac)))))); +} +/** Base-2 exponential approximation (2^x). (Q10 input, Q16 output) */ +static OPUS_INLINE opus_val32 celt_exp2(opus_val16 x) +{ + int integer; + opus_val16 frac; + integer = SHR16(x,10); + if (integer>14) + return 0x7f000000; + else if (integer < -15) + return 0; + frac = celt_exp2_frac(x-SHL16(integer,10)); + return VSHR32(EXTEND32(frac), -integer-2); +} + +opus_val32 celt_rcp(opus_val32 x); + +#define celt_div(a,b) MULT32_32_Q31((opus_val32)(a),celt_rcp(b)) + +opus_val32 frac_div32(opus_val32 a, opus_val32 b); + +#define M1 32767 +#define M2 -21 +#define M3 -11943 +#define M4 4936 + +/* Atan approximation using a 4th order polynomial. Input is in Q15 format + and normalized by pi/4. Output is in Q15 format */ +static OPUS_INLINE opus_val16 celt_atan01(opus_val16 x) +{ + return MULT16_16_P15(x, ADD32(M1, MULT16_16_P15(x, ADD32(M2, MULT16_16_P15(x, ADD32(M3, MULT16_16_P15(M4, x))))))); +} + +#undef M1 +#undef M2 +#undef M3 +#undef M4 + +/* atan2() approximation valid for positive input values */ +static OPUS_INLINE opus_val16 celt_atan2p(opus_val16 y, opus_val16 x) +{ + if (y < x) + { + opus_val32 arg; + arg = celt_div(SHL32(EXTEND32(y),15),x); + if (arg >= 32767) + arg = 32767; + return SHR16(celt_atan01(EXTRACT16(arg)),1); + } else { + opus_val32 arg; + arg = celt_div(SHL32(EXTEND32(x),15),y); + if (arg >= 32767) + arg = 32767; + return 25736-SHR16(celt_atan01(EXTRACT16(arg)),1); + } +} + +#endif /* FIXED_POINT */ +#endif /* MATHOPS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.c new file mode 100644 index 000000000..5c6dab5b7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.c @@ -0,0 +1,343 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2008 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This is a simple MDCT implementation that uses a N/4 complex FFT + to do most of the work. It should be relatively straightforward to + plug in pretty much and FFT here. + + This replaces the Vorbis FFT (and uses the exact same API), which + was a bit too messy and that was ending up duplicating code + (might as well use the same FFT everywhere). + + The algorithm is similar to (and inspired from) Fabrice Bellard's + MDCT implementation in FFMPEG, but has differences in signs, ordering + and scaling in many places. +*/ + +#ifndef SKIP_CONFIG_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#endif + +#include "mdct.h" +#include "kiss_fft.h" +#include "_kiss_fft_guts.h" +#include +#include "os_support.h" +#include "mathops.h" +#include "stack_alloc.h" + +#if defined(MIPSr1_ASM) +#include "mips/mdct_mipsr1.h" +#endif + + +#ifdef CUSTOM_MODES + +int clt_mdct_init(mdct_lookup *l,int N, int maxshift, int arch) +{ + int i; + kiss_twiddle_scalar *trig; + int shift; + int N2=N>>1; + l->n = N; + l->maxshift = maxshift; + for (i=0;i<=maxshift;i++) + { + if (i==0) + l->kfft[i] = opus_fft_alloc(N>>2>>i, 0, 0, arch); + else + l->kfft[i] = opus_fft_alloc_twiddles(N>>2>>i, 0, 0, l->kfft[0], arch); +#ifndef ENABLE_TI_DSPLIB55 + if (l->kfft[i]==NULL) + return 0; +#endif + } + l->trig = trig = (kiss_twiddle_scalar*)opus_alloc((N-(N2>>maxshift))*sizeof(kiss_twiddle_scalar)); + if (l->trig==NULL) + return 0; + for (shift=0;shift<=maxshift;shift++) + { + /* We have enough points that sine isn't necessary */ +#if defined(FIXED_POINT) +#if 1 + for (i=0;i>= 1; + N >>= 1; + } + return 1; +} + +void clt_mdct_clear(mdct_lookup *l, int arch) +{ + int i; + for (i=0;i<=l->maxshift;i++) + opus_fft_free(l->kfft[i], arch); + opus_free((kiss_twiddle_scalar*)l->trig); +} + +#endif /* CUSTOM_MODES */ + +/* Forward MDCT trashes the input array */ +#ifndef OVERRIDE_clt_mdct_forward +void clt_mdct_forward_c(const mdct_lookup *l, kiss_fft_scalar *in, kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + VARDECL(kiss_fft_scalar, f); + VARDECL(kiss_fft_cpx, f2); + const kiss_fft_state *st = l->kfft[shift]; + const kiss_twiddle_scalar *trig; + opus_val16 scale; +#ifdef FIXED_POINT + /* Allows us to scale with MULT16_32_Q16(), which is faster than + MULT16_32_Q15() on ARM. */ + int scale_shift = st->scale_shift-1; +#endif + SAVE_STACK; + (void)arch; + scale = st->scale; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + ALLOC(f, N2, kiss_fft_scalar); + ALLOC(f2, N4, kiss_fft_cpx); + + /* Consider the input to be composed of four blocks: [a, b, c, d] */ + /* Window, shuffle, fold */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in+(overlap>>1); + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+N2-1+(overlap>>1); + kiss_fft_scalar * OPUS_RESTRICT yp = f; + const opus_val16 * OPUS_RESTRICT wp1 = window+(overlap>>1); + const opus_val16 * OPUS_RESTRICT wp2 = window+(overlap>>1)-1; + for(i=0;i<((overlap+3)>>2);i++) + { + /* Real part arranged as -d-cR, Imag part arranged as -b+aR*/ + *yp++ = MULT16_32_Q15(*wp2, xp1[N2]) + MULT16_32_Q15(*wp1,*xp2); + *yp++ = MULT16_32_Q15(*wp1, *xp1) - MULT16_32_Q15(*wp2, xp2[-N2]); + xp1+=2; + xp2-=2; + wp1+=2; + wp2-=2; + } + wp1 = window; + wp2 = window+overlap-1; + for(;i>2);i++) + { + /* Real part arranged as a-bR, Imag part arranged as -c-dR */ + *yp++ = *xp2; + *yp++ = *xp1; + xp1+=2; + xp2-=2; + } + for(;ibitrev[i]] = yc; + } + } + + /* N/4 complex FFT, does not downscale anymore */ + opus_fft_impl(st, f2); + + /* Post-rotate */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_cpx * OPUS_RESTRICT fp = f2; + kiss_fft_scalar * OPUS_RESTRICT yp1 = out; + kiss_fft_scalar * OPUS_RESTRICT yp2 = out+stride*(N2-1); + const kiss_twiddle_scalar *t = &trig[0]; + /* Temp pointers to make it really clear to the compiler what we're doing */ + for(i=0;ii,t[N4+i]) - S_MUL(fp->r,t[i]); + yi = S_MUL(fp->r,t[N4+i]) + S_MUL(fp->i,t[i]); + *yp1 = yr; + *yp2 = yi; + fp++; + yp1 += 2*stride; + yp2 -= 2*stride; + } + } + RESTORE_STACK; +} +#endif /* OVERRIDE_clt_mdct_forward */ + +#ifndef OVERRIDE_clt_mdct_backward +void clt_mdct_backward_c(const mdct_lookup *l, kiss_fft_scalar *in, kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 * OPUS_RESTRICT window, int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + const kiss_twiddle_scalar *trig; + (void) arch; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + /* Pre-rotate */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in; + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+stride*(N2-1); + kiss_fft_scalar * OPUS_RESTRICT yp = out+(overlap>>1); + const kiss_twiddle_scalar * OPUS_RESTRICT t = &trig[0]; + const opus_int16 * OPUS_RESTRICT bitrev = l->kfft[shift]->bitrev; + for(i=0;ikfft[shift], (kiss_fft_cpx*)(out+(overlap>>1))); + + /* Post-rotate and de-shuffle from both ends of the buffer at once to make + it in-place. */ + { + kiss_fft_scalar * yp0 = out+(overlap>>1); + kiss_fft_scalar * yp1 = out+(overlap>>1)+N2-2; + const kiss_twiddle_scalar *t = &trig[0]; + /* Loop to (N4+1)>>1 to handle odd N4. When N4 is odd, the + middle pair will be computed twice. */ + for(i=0;i<(N4+1)>>1;i++) + { + kiss_fft_scalar re, im, yr, yi; + kiss_twiddle_scalar t0, t1; + /* We swap real and imag because we're using an FFT instead of an IFFT. */ + re = yp0[1]; + im = yp0[0]; + t0 = t[i]; + t1 = t[N4+i]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = ADD32_ovflw(S_MUL(re,t0), S_MUL(im,t1)); + yi = SUB32_ovflw(S_MUL(re,t1), S_MUL(im,t0)); + /* We swap real and imag because we're using an FFT instead of an IFFT. */ + re = yp1[1]; + im = yp1[0]; + yp0[0] = yr; + yp1[1] = yi; + + t0 = t[(N4-i-1)]; + t1 = t[(N2-i-1)]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = ADD32_ovflw(S_MUL(re,t0), S_MUL(im,t1)); + yi = SUB32_ovflw(S_MUL(re,t1), S_MUL(im,t0)); + yp1[0] = yr; + yp0[1] = yi; + yp0 += 2; + yp1 -= 2; + } + } + + /* Mirror on both sides for TDAC */ + { + kiss_fft_scalar * OPUS_RESTRICT xp1 = out+overlap-1; + kiss_fft_scalar * OPUS_RESTRICT yp1 = out; + const opus_val16 * OPUS_RESTRICT wp1 = window; + const opus_val16 * OPUS_RESTRICT wp2 = window+overlap-1; + + for(i = 0; i < overlap/2; i++) + { + kiss_fft_scalar x1, x2; + x1 = *xp1; + x2 = *yp1; + *yp1++ = SUB32_ovflw(MULT16_32_Q15(*wp2, x2), MULT16_32_Q15(*wp1, x1)); + *xp1-- = ADD32_ovflw(MULT16_32_Q15(*wp1, x2), MULT16_32_Q15(*wp2, x1)); + wp1++; + wp2--; + } + } +} +#endif /* OVERRIDE_clt_mdct_backward */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.h new file mode 100644 index 000000000..160ae4e0f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mdct.h @@ -0,0 +1,112 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2008 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This is a simple MDCT implementation that uses a N/4 complex FFT + to do most of the work. It should be relatively straightforward to + plug in pretty much and FFT here. + + This replaces the Vorbis FFT (and uses the exact same API), which + was a bit too messy and that was ending up duplicating code + (might as well use the same FFT everywhere). + + The algorithm is similar to (and inspired from) Fabrice Bellard's + MDCT implementation in FFMPEG, but has differences in signs, ordering + and scaling in many places. +*/ + +#ifndef MDCT_H +#define MDCT_H + +#include "opus_defines.h" +#include "kiss_fft.h" +#include "arch.h" + +typedef struct { + int n; + int maxshift; + const kiss_fft_state *kfft[4]; + const kiss_twiddle_scalar * OPUS_RESTRICT trig; +} mdct_lookup; + +#if defined(HAVE_ARM_NE10) +#include "arm/mdct_arm.h" +#endif + + +int clt_mdct_init(mdct_lookup *l,int N, int maxshift, int arch); +void clt_mdct_clear(mdct_lookup *l, int arch); + +/** Compute a forward MDCT and scale by 4/N, trashes the input array */ +void clt_mdct_forward_c(const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, int overlap, + int shift, int stride, int arch); + +/** Compute a backward MDCT (no scaling) and performs weighted overlap-add + (scales implicitly by 1/2) */ +void clt_mdct_backward_c(const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 * OPUS_RESTRICT window, + int overlap, int shift, int stride, int arch); + +#if !defined(OVERRIDE_OPUS_MDCT) +/* Is run-time CPU detection enabled on this platform? */ +#if defined(OPUS_HAVE_RTCD) && defined(HAVE_ARM_NE10) + +extern void (*const CLT_MDCT_FORWARD_IMPL[OPUS_ARCHMASK+1])( + const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, const opus_val16 *window, + int overlap, int shift, int stride, int arch); + +#define clt_mdct_forward(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) \ + ((*CLT_MDCT_FORWARD_IMPL[(arch)&OPUS_ARCHMASK])(_l, _in, _out, \ + _window, _overlap, _shift, \ + _stride, _arch)) + +extern void (*const CLT_MDCT_BACKWARD_IMPL[OPUS_ARCHMASK+1])( + const mdct_lookup *l, kiss_fft_scalar *in, + kiss_fft_scalar * OPUS_RESTRICT out, const opus_val16 *window, + int overlap, int shift, int stride, int arch); + +#define clt_mdct_backward(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) \ + (*CLT_MDCT_BACKWARD_IMPL[(arch)&OPUS_ARCHMASK])(_l, _in, _out, \ + _window, _overlap, _shift, \ + _stride, _arch) + +#else /* if defined(OPUS_HAVE_RTCD) && defined(HAVE_ARM_NE10) */ + +#define clt_mdct_forward(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) \ + clt_mdct_forward_c(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) + +#define clt_mdct_backward(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) \ + clt_mdct_backward_c(_l, _in, _out, _window, _overlap, _shift, _stride, _arch) + +#endif /* end if defined(OPUS_HAVE_RTCD) && defined(HAVE_ARM_NE10) && !defined(FIXED_POINT) */ +#endif /* end if !defined(OVERRIDE_OPUS_MDCT) */ + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/meson.build new file mode 100644 index 000000000..370ea1fe0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/meson.build @@ -0,0 +1,63 @@ +celt_sources = sources['CELT_SOURCES'] + +celt_sse_sources = sources['CELT_SOURCES_SSE'] + +celt_sse2_sources = sources['CELT_SOURCES_SSE2'] + +celt_sse4_1_sources = sources['CELT_SOURCES_SSE4_1'] + +celt_neon_intr_sources = sources['CELT_SOURCES_ARM_NEON_INTR'] + +celt_static_libs = [] + +foreach intr_name : ['sse', 'sse2', 'sse4_1', 'neon_intr'] + have_intr = get_variable('have_' + intr_name) + if not have_intr + continue + endif + + intr_sources = get_variable('celt_@0@_sources'.format(intr_name)) + intr_args = get_variable('opus_@0@_args'.format(intr_name), []) + celt_static_libs += static_library('celt_' + intr_name, intr_sources, + c_args: intr_args, + include_directories: opus_includes, + install: false) +endforeach + +have_arm_intrinsics_or_asm = have_arm_ne10 +if (intrinsics_support.length() + asm_optimization.length() + inline_optimization.length()) > 0 + have_arm_intrinsics_or_asm = true +endif + +if host_cpu_family in ['arm', 'aarch64'] and have_arm_intrinsics_or_asm + celt_sources += sources['CELT_SOURCES_ARM'] + if have_arm_ne10 + celt_sources += sources['CELT_SOURCES_ARM_NE10'] + endif + if opus_arm_external_asm + arm2gnu = [find_program('arm/arm2gnu.pl')] + arm2gnu_args + celt_sources_arm_asm = configure_file(input: 'arm/celt_pitch_xcorr_arm.s', + output: '@BASENAME@-gnu.S', + command: arm2gnu + ['@INPUT@'], + capture: true) + celt_arm_armopts_s = configure_file(input: 'arm/armopts.s.in', + output: 'arm/armopts.s', + configuration: opus_conf) + celt_static_libs += static_library('celt-armasm', + celt_arm_armopts_s, celt_sources_arm_asm, + install: false) + endif +endif + +celt_c_args = [] +if host_system == 'windows' + celt_c_args += ['-DDLL_EXPORT'] +endif + +celt_lib = static_library('opus-celt', + celt_sources, + c_args: celt_c_args, + include_directories: opus_includes, + link_whole: celt_static_libs, + dependencies: libm, + install: false) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mfrngcod.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mfrngcod.h new file mode 100644 index 000000000..809152a59 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mfrngcod.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001-2008 Timothy B. Terriberry + Copyright (c) 2008-2009 Xiph.Org Foundation */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(_mfrngcode_H) +# define _mfrngcode_H (1) +# include "entcode.h" + +/*Constants used by the entropy encoder/decoder.*/ + +/*The number of bits to output at a time.*/ +# define EC_SYM_BITS (8) +/*The total number of bits in each of the state registers.*/ +# define EC_CODE_BITS (32) +/*The maximum symbol value.*/ +# define EC_SYM_MAX ((1U<>EC_SYM_BITS) +/*The number of bits available for the last, partial symbol in the code field.*/ +# define EC_CODE_EXTRA ((EC_CODE_BITS-2)%EC_SYM_BITS+1) +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/celt_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/celt_mipsr1.h new file mode 100644 index 000000000..c332fe047 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/celt_mipsr1.h @@ -0,0 +1,152 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2010 Xiph.Org Foundation + Copyright (c) 2008 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __CELT_MIPSR1_H__ +#define __CELT_MIPSR1_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define CELT_C + +#include "os_support.h" +#include "mdct.h" +#include +#include "celt.h" +#include "pitch.h" +#include "bands.h" +#include "modes.h" +#include "entcode.h" +#include "quant_bands.h" +#include "rate.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "float_cast.h" +#include +#include "celt_lpc.h" +#include "vq.h" + +#define OVERRIDE_COMB_FILTER_CONST +#define OVERRIDE_comb_filter +void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, + opus_val16 g0, opus_val16 g1, int tapset0, int tapset1, + const opus_val16 *window, int overlap, int arch) +{ + int i; + opus_val32 x0, x1, x2, x3, x4; + + (void)arch; + + /* printf ("%d %d %f %f\n", T0, T1, g0, g1); */ + opus_val16 g00, g01, g02, g10, g11, g12; + static const opus_val16 gains[3][3] = { + {QCONST16(0.3066406250f, 15), QCONST16(0.2170410156f, 15), QCONST16(0.1296386719f, 15)}, + {QCONST16(0.4638671875f, 15), QCONST16(0.2680664062f, 15), QCONST16(0.f, 15)}, + {QCONST16(0.7998046875f, 15), QCONST16(0.1000976562f, 15), QCONST16(0.f, 15)}}; + + if (g0==0 && g1==0) + { + /* OPT: Happens to work without the OPUS_MOVE(), but only because the current encoder already copies x to y */ + if (x!=y) + OPUS_MOVE(y, x, N); + return; + } + + g00 = MULT16_16_P15(g0, gains[tapset0][0]); + g01 = MULT16_16_P15(g0, gains[tapset0][1]); + g02 = MULT16_16_P15(g0, gains[tapset0][2]); + g10 = MULT16_16_P15(g1, gains[tapset1][0]); + g11 = MULT16_16_P15(g1, gains[tapset1][1]); + g12 = MULT16_16_P15(g1, gains[tapset1][2]); + x1 = x[-T1+1]; + x2 = x[-T1 ]; + x3 = x[-T1-1]; + x4 = x[-T1-2]; + /* If the filter didn't change, we don't need the overlap */ + if (g0==g1 && T0==T1 && tapset0==tapset1) + overlap=0; + + for (i=0;itwiddles[fstride*m]; + yb = st->twiddles[fstride*2*m]; +#endif + + tw=st->twiddles; + + for (i=0;ir += scratch[7].r + scratch[8].r; + Fout0->i += scratch[7].i + scratch[8].i; + scratch[5].r = scratch[0].r + S_MUL_ADD(scratch[7].r,ya.r,scratch[8].r,yb.r); + scratch[5].i = scratch[0].i + S_MUL_ADD(scratch[7].i,ya.r,scratch[8].i,yb.r); + + scratch[6].r = S_MUL_ADD(scratch[10].i,ya.i,scratch[9].i,yb.i); + scratch[6].i = -S_MUL_ADD(scratch[10].r,ya.i,scratch[9].r,yb.i); + + C_SUB(*Fout1,scratch[5],scratch[6]); + C_ADD(*Fout4,scratch[5],scratch[6]); + + scratch[11].r = scratch[0].r + S_MUL_ADD(scratch[7].r,yb.r,scratch[8].r,ya.r); + scratch[11].i = scratch[0].i + S_MUL_ADD(scratch[7].i,yb.r,scratch[8].i,ya.r); + + scratch[12].r = S_MUL_SUB(scratch[9].i,ya.i,scratch[10].i,yb.i); + scratch[12].i = S_MUL_SUB(scratch[10].r,yb.i,scratch[9].r,ya.i); + + C_ADD(*Fout2,scratch[11],scratch[12]); + C_SUB(*Fout3,scratch[11],scratch[12]); + + ++Fout0;++Fout1;++Fout2;++Fout3;++Fout4; + } + } +} + + +#endif /* KISS_FFT_MIPSR1_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/mdct_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/mdct_mipsr1.h new file mode 100644 index 000000000..2934dab77 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/mdct_mipsr1.h @@ -0,0 +1,288 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2008 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This is a simple MDCT implementation that uses a N/4 complex FFT + to do most of the work. It should be relatively straightforward to + plug in pretty much and FFT here. + + This replaces the Vorbis FFT (and uses the exact same API), which + was a bit too messy and that was ending up duplicating code + (might as well use the same FFT everywhere). + + The algorithm is similar to (and inspired from) Fabrice Bellard's + MDCT implementation in FFMPEG, but has differences in signs, ordering + and scaling in many places. +*/ +#ifndef __MDCT_MIPSR1_H__ +#define __MDCT_MIPSR1_H__ + +#ifndef SKIP_CONFIG_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#endif + +#include "mdct.h" +#include "kiss_fft.h" +#include "_kiss_fft_guts.h" +#include +#include "os_support.h" +#include "mathops.h" +#include "stack_alloc.h" + +/* Forward MDCT trashes the input array */ +#define OVERRIDE_clt_mdct_forward +void clt_mdct_forward(const mdct_lookup *l, kiss_fft_scalar *in, kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 *window, int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + VARDECL(kiss_fft_scalar, f); + VARDECL(kiss_fft_cpx, f2); + const kiss_fft_state *st = l->kfft[shift]; + const kiss_twiddle_scalar *trig; + opus_val16 scale; +#ifdef FIXED_POINT + /* Allows us to scale with MULT16_32_Q16(), which is faster than + MULT16_32_Q15() on ARM. */ + int scale_shift = st->scale_shift-1; +#endif + + (void)arch; + + SAVE_STACK; + scale = st->scale; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + ALLOC(f, N2, kiss_fft_scalar); + ALLOC(f2, N4, kiss_fft_cpx); + + /* Consider the input to be composed of four blocks: [a, b, c, d] */ + /* Window, shuffle, fold */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in+(overlap>>1); + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+N2-1+(overlap>>1); + kiss_fft_scalar * OPUS_RESTRICT yp = f; + const opus_val16 * OPUS_RESTRICT wp1 = window+(overlap>>1); + const opus_val16 * OPUS_RESTRICT wp2 = window+(overlap>>1)-1; + for(i=0;i<((overlap+3)>>2);i++) + { + /* Real part arranged as -d-cR, Imag part arranged as -b+aR*/ + *yp++ = S_MUL_ADD(*wp2, xp1[N2],*wp1,*xp2); + *yp++ = S_MUL_SUB(*wp1, *xp1,*wp2, xp2[-N2]); + xp1+=2; + xp2-=2; + wp1+=2; + wp2-=2; + } + wp1 = window; + wp2 = window+overlap-1; + for(;i>2);i++) + { + /* Real part arranged as a-bR, Imag part arranged as -c-dR */ + *yp++ = *xp2; + *yp++ = *xp1; + xp1+=2; + xp2-=2; + } + for(;ibitrev[i]] = yc; + } + } + + /* N/4 complex FFT, does not downscale anymore */ + opus_fft_impl(st, f2); + + /* Post-rotate */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_cpx * OPUS_RESTRICT fp = f2; + kiss_fft_scalar * OPUS_RESTRICT yp1 = out; + kiss_fft_scalar * OPUS_RESTRICT yp2 = out+stride*(N2-1); + const kiss_twiddle_scalar *t = &trig[0]; + /* Temp pointers to make it really clear to the compiler what we're doing */ + for(i=0;ii,t[N4+i] , fp->r,t[i]); + yi = S_MUL_ADD(fp->r,t[N4+i] ,fp->i,t[i]); + *yp1 = yr; + *yp2 = yi; + fp++; + yp1 += 2*stride; + yp2 -= 2*stride; + } + } + RESTORE_STACK; +} + +#define OVERRIDE_clt_mdct_backward +void clt_mdct_backward(const mdct_lookup *l, kiss_fft_scalar *in, kiss_fft_scalar * OPUS_RESTRICT out, + const opus_val16 * OPUS_RESTRICT window, int overlap, int shift, int stride, int arch) +{ + int i; + int N, N2, N4; + const kiss_twiddle_scalar *trig; + + (void)arch; + + N = l->n; + trig = l->trig; + for (i=0;i>= 1; + trig += N; + } + N2 = N>>1; + N4 = N>>2; + + /* Pre-rotate */ + { + /* Temp pointers to make it really clear to the compiler what we're doing */ + const kiss_fft_scalar * OPUS_RESTRICT xp1 = in; + const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+stride*(N2-1); + kiss_fft_scalar * OPUS_RESTRICT yp = out+(overlap>>1); + const kiss_twiddle_scalar * OPUS_RESTRICT t = &trig[0]; + const opus_int16 * OPUS_RESTRICT bitrev = l->kfft[shift]->bitrev; + for(i=0;ikfft[shift], (kiss_fft_cpx*)(out+(overlap>>1))); + + /* Post-rotate and de-shuffle from both ends of the buffer at once to make + it in-place. */ + { + kiss_fft_scalar * OPUS_RESTRICT yp0 = out+(overlap>>1); + kiss_fft_scalar * OPUS_RESTRICT yp1 = out+(overlap>>1)+N2-2; + const kiss_twiddle_scalar *t = &trig[0]; + /* Loop to (N4+1)>>1 to handle odd N4. When N4 is odd, the + middle pair will be computed twice. */ + for(i=0;i<(N4+1)>>1;i++) + { + kiss_fft_scalar re, im, yr, yi; + kiss_twiddle_scalar t0, t1; + /* We swap real and imag because we're using an FFT instead of an IFFT. */ + re = yp0[1]; + im = yp0[0]; + t0 = t[i]; + t1 = t[N4+i]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = S_MUL_ADD(re,t0 , im,t1); + yi = S_MUL_SUB(re,t1 , im,t0); + /* We swap real and imag because we're using an FFT instead of an IFFT. */ + re = yp1[1]; + im = yp1[0]; + yp0[0] = yr; + yp1[1] = yi; + + t0 = t[(N4-i-1)]; + t1 = t[(N2-i-1)]; + /* We'd scale up by 2 here, but instead it's done when mixing the windows */ + yr = S_MUL_ADD(re,t0,im,t1); + yi = S_MUL_SUB(re,t1,im,t0); + yp1[0] = yr; + yp0[1] = yi; + yp0 += 2; + yp1 -= 2; + } + } + + /* Mirror on both sides for TDAC */ + { + kiss_fft_scalar * OPUS_RESTRICT xp1 = out+overlap-1; + kiss_fft_scalar * OPUS_RESTRICT yp1 = out; + const opus_val16 * OPUS_RESTRICT wp1 = window; + const opus_val16 * OPUS_RESTRICT wp2 = window+overlap-1; + + for(i = 0; i < overlap/2; i++) + { + kiss_fft_scalar x1, x2; + x1 = *xp1; + x2 = *yp1; + *yp1++ = MULT16_32_Q15(*wp2, x2) - MULT16_32_Q15(*wp1, x1); + *xp1-- = MULT16_32_Q15(*wp1, x2) + MULT16_32_Q15(*wp2, x1); + wp1++; + wp2--; + } + } +} +#endif /* __MDCT_MIPSR1_H__ */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/pitch_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/pitch_mipsr1.h new file mode 100644 index 000000000..a9500aff5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/mips/pitch_mipsr1.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/** + @file pitch.h + @brief Pitch analysis + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef PITCH_MIPSR1_H +#define PITCH_MIPSR1_H + +#define OVERRIDE_DUAL_INNER_PROD +static inline void dual_inner_prod(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2, int arch) +{ + int j; + opus_val32 xy01=0; + opus_val32 xy02=0; + + (void)arch; + + asm volatile("MULT $ac1, $0, $0"); + asm volatile("MULT $ac2, $0, $0"); + /* Compute the norm of X+Y and X-Y as |X|^2 + |Y|^2 +/- sum(xy) */ + for (j=0;j=0;i--) + { + celt_norm x1, x2; + x1 = Xptr[0]; + x2 = Xptr[stride]; + Xptr[stride] = EXTRACT16(PSHR32(MAC16_16(MULT16_16(c, x2), s, x1), 15)); + *Xptr-- = EXTRACT16(PSHR32(MAC16_16(MULT16_16(c, x1), ms, x2), 15)); + } +} + +#define OVERRIDE_renormalise_vector +void renormalise_vector(celt_norm *X, int N, opus_val16 gain, int arch) +{ + int i; +#ifdef FIXED_POINT + int k; +#endif + opus_val32 E = EPSILON; + opus_val16 g; + opus_val32 t; + celt_norm *xptr = X; + int X0, X1; + + (void)arch; + + asm volatile("mult $ac1, $0, $0"); + asm volatile("MTLO %0, $ac1" : :"r" (E)); + /*if(N %4) + printf("error");*/ + for (i=0;i>1; +#endif + t = VSHR32(E, 2*(k-7)); + g = MULT16_16_P15(celt_rsqrt_norm(t),gain); + + xptr = X; + for (i=0;i= Fs) + break; + + /* Find where the linear part ends (i.e. where the spacing is more than min_width */ + for (lin=0;lin= res) + break; + + low = (bark_freq[lin]+res/2)/res; + high = nBark-lin; + *nbEBands = low+high; + eBands = opus_alloc(sizeof(opus_int16)*(*nbEBands+2)); + + if (eBands==NULL) + return NULL; + + /* Linear spacing (min_width) */ + for (i=0;i0) + offset = eBands[low-1]*res - bark_freq[lin-1]; + /* Spacing follows critical bands */ + for (i=0;i frame_size) + eBands[*nbEBands] = frame_size; + for (i=1;i<*nbEBands-1;i++) + { + if (eBands[i+1]-eBands[i] < eBands[i]-eBands[i-1]) + { + eBands[i] -= (2*eBands[i]-eBands[i-1]-eBands[i+1])/2; + } + } + /* Remove any empty bands. */ + for (i=j=0;i<*nbEBands;i++) + if(eBands[i+1]>eBands[j]) + eBands[++j]=eBands[i+1]; + *nbEBands=j; + + for (i=1;i<*nbEBands;i++) + { + /* Every band must be smaller than the last band. */ + celt_assert(eBands[i]-eBands[i-1]<=eBands[*nbEBands]-eBands[*nbEBands-1]); + /* Each band must be no larger than twice the size of the previous one. */ + celt_assert(eBands[i+1]-eBands[i]<=2*(eBands[i]-eBands[i-1])); + } + + return eBands; +} + +static void compute_allocation_table(CELTMode *mode) +{ + int i, j; + unsigned char *allocVectors; + int maxBands = sizeof(eband5ms)/sizeof(eband5ms[0])-1; + + mode->nbAllocVectors = BITALLOC_SIZE; + allocVectors = opus_alloc(sizeof(unsigned char)*(BITALLOC_SIZE*mode->nbEBands)); + if (allocVectors==NULL) + return; + + /* Check for standard mode */ + if (mode->Fs == 400*(opus_int32)mode->shortMdctSize) + { + for (i=0;inbEBands;i++) + allocVectors[i] = band_allocation[i]; + mode->allocVectors = allocVectors; + return; + } + /* If not the standard mode, interpolate */ + /* Compute per-codec-band allocation from per-critical-band matrix */ + for (i=0;inbEBands;j++) + { + int k; + for (k=0;k mode->eBands[j]*(opus_int32)mode->Fs/mode->shortMdctSize) + break; + } + if (k>maxBands-1) + allocVectors[i*mode->nbEBands+j] = band_allocation[i*maxBands + maxBands-1]; + else { + opus_int32 a0, a1; + a1 = mode->eBands[j]*(opus_int32)mode->Fs/mode->shortMdctSize - 400*(opus_int32)eband5ms[k-1]; + a0 = 400*(opus_int32)eband5ms[k] - mode->eBands[j]*(opus_int32)mode->Fs/mode->shortMdctSize; + allocVectors[i*mode->nbEBands+j] = (a0*band_allocation[i*maxBands+k-1] + + a1*band_allocation[i*maxBands+k])/(a0+a1); + } + } + } + + /*printf ("\n"); + for (i=0;inbEBands;j++) + printf ("%d ", allocVectors[i*mode->nbEBands+j]); + printf ("\n"); + } + exit(0);*/ + + mode->allocVectors = allocVectors; +} + +#endif /* CUSTOM_MODES */ + +CELTMode *opus_custom_mode_create(opus_int32 Fs, int frame_size, int *error) +{ + int i; +#ifdef CUSTOM_MODES + CELTMode *mode=NULL; + int res; + opus_val16 *window; + opus_int16 *logN; + int LM; + int arch = opus_select_arch(); + ALLOC_STACK; +#if !defined(VAR_ARRAYS) && !defined(USE_ALLOCA) + if (global_stack==NULL) + goto failure; +#endif +#endif + +#ifndef CUSTOM_MODES_ONLY + for (i=0;iFs && + (frame_size<shortMdctSize*static_mode_list[i]->nbShortMdcts) + { + if (error) + *error = OPUS_OK; + return (CELTMode*)static_mode_list[i]; + } + } + } +#endif /* CUSTOM_MODES_ONLY */ + +#ifndef CUSTOM_MODES + if (error) + *error = OPUS_BAD_ARG; + return NULL; +#else + + /* The good thing here is that permutation of the arguments will automatically be invalid */ + + if (Fs < 8000 || Fs > 96000) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + if (frame_size < 40 || frame_size > 1024 || frame_size%2!=0) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + /* Frames of less than 1ms are not supported. */ + if ((opus_int32)frame_size*1000 < Fs) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + + if ((opus_int32)frame_size*75 >= Fs && (frame_size%16)==0) + { + LM = 3; + } else if ((opus_int32)frame_size*150 >= Fs && (frame_size%8)==0) + { + LM = 2; + } else if ((opus_int32)frame_size*300 >= Fs && (frame_size%4)==0) + { + LM = 1; + } else + { + LM = 0; + } + + /* Shorts longer than 3.3ms are not supported. */ + if ((opus_int32)(frame_size>>LM)*300 > Fs) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + + mode = opus_alloc(sizeof(CELTMode)); + if (mode==NULL) + goto failure; + mode->Fs = Fs; + + /* Pre/de-emphasis depends on sampling rate. The "standard" pre-emphasis + is defined as A(z) = 1 - 0.85*z^-1 at 48 kHz. Other rates should + approximate that. */ + if(Fs < 12000) /* 8 kHz */ + { + mode->preemph[0] = QCONST16(0.3500061035f, 15); + mode->preemph[1] = -QCONST16(0.1799926758f, 15); + mode->preemph[2] = QCONST16(0.2719968125f, SIG_SHIFT); /* exact 1/preemph[3] */ + mode->preemph[3] = QCONST16(3.6765136719f, 13); + } else if(Fs < 24000) /* 16 kHz */ + { + mode->preemph[0] = QCONST16(0.6000061035f, 15); + mode->preemph[1] = -QCONST16(0.1799926758f, 15); + mode->preemph[2] = QCONST16(0.4424998650f, SIG_SHIFT); /* exact 1/preemph[3] */ + mode->preemph[3] = QCONST16(2.2598876953f, 13); + } else if(Fs < 40000) /* 32 kHz */ + { + mode->preemph[0] = QCONST16(0.7799987793f, 15); + mode->preemph[1] = -QCONST16(0.1000061035f, 15); + mode->preemph[2] = QCONST16(0.7499771125f, SIG_SHIFT); /* exact 1/preemph[3] */ + mode->preemph[3] = QCONST16(1.3333740234f, 13); + } else /* 48 kHz */ + { + mode->preemph[0] = QCONST16(0.8500061035f, 15); + mode->preemph[1] = QCONST16(0.0f, 15); + mode->preemph[2] = QCONST16(1.f, SIG_SHIFT); + mode->preemph[3] = QCONST16(1.f, 13); + } + + mode->maxLM = LM; + mode->nbShortMdcts = 1<shortMdctSize = frame_size/mode->nbShortMdcts; + res = (mode->Fs+mode->shortMdctSize)/(2*mode->shortMdctSize); + + mode->eBands = compute_ebands(Fs, mode->shortMdctSize, res, &mode->nbEBands); + if (mode->eBands==NULL) + goto failure; +#if !defined(SMALL_FOOTPRINT) + /* Make sure we don't allocate a band larger than our PVQ table. + 208 should be enough, but let's be paranoid. */ + if ((mode->eBands[mode->nbEBands] - mode->eBands[mode->nbEBands-1])< + 208) { + goto failure; + } +#endif + + mode->effEBands = mode->nbEBands; + while (mode->eBands[mode->effEBands] > mode->shortMdctSize) + mode->effEBands--; + + /* Overlap must be divisible by 4 */ + mode->overlap = ((mode->shortMdctSize>>2)<<2); + + compute_allocation_table(mode); + if (mode->allocVectors==NULL) + goto failure; + + window = (opus_val16*)opus_alloc(mode->overlap*sizeof(opus_val16)); + if (window==NULL) + goto failure; + +#ifndef FIXED_POINT + for (i=0;ioverlap;i++) + window[i] = Q15ONE*sin(.5*M_PI* sin(.5*M_PI*(i+.5)/mode->overlap) * sin(.5*M_PI*(i+.5)/mode->overlap)); +#else + for (i=0;ioverlap;i++) + window[i] = MIN32(32767,floor(.5+32768.*sin(.5*M_PI* sin(.5*M_PI*(i+.5)/mode->overlap) * sin(.5*M_PI*(i+.5)/mode->overlap)))); +#endif + mode->window = window; + + logN = (opus_int16*)opus_alloc(mode->nbEBands*sizeof(opus_int16)); + if (logN==NULL) + goto failure; + + for (i=0;inbEBands;i++) + logN[i] = log2_frac(mode->eBands[i+1]-mode->eBands[i], BITRES); + mode->logN = logN; + + compute_pulse_cache(mode, mode->maxLM); + + if (clt_mdct_init(&mode->mdct, 2*mode->shortMdctSize*mode->nbShortMdcts, + mode->maxLM, arch) == 0) + goto failure; + + if (error) + *error = OPUS_OK; + + return mode; +failure: + if (error) + *error = OPUS_ALLOC_FAIL; + if (mode!=NULL) + opus_custom_mode_destroy(mode); + return NULL; +#endif /* !CUSTOM_MODES */ +} + +#ifdef CUSTOM_MODES +void opus_custom_mode_destroy(CELTMode *mode) +{ + int arch = opus_select_arch(); + + if (mode == NULL) + return; +#ifndef CUSTOM_MODES_ONLY + { + int i; + for (i=0;ieBands); + opus_free((unsigned char*)mode->allocVectors); + + opus_free((opus_val16*)mode->window); + opus_free((opus_int16*)mode->logN); + + opus_free((opus_int16*)mode->cache.index); + opus_free((unsigned char*)mode->cache.bits); + opus_free((unsigned char*)mode->cache.caps); + clt_mdct_clear(&mode->mdct, arch); + + opus_free((CELTMode *)mode); +} +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/modes.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/modes.h new file mode 100644 index 000000000..be813ccc8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/modes.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2008 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef MODES_H +#define MODES_H + +#include "opus_types.h" +#include "celt.h" +#include "arch.h" +#include "mdct.h" +#include "entenc.h" +#include "entdec.h" + +#define MAX_PERIOD 1024 + +typedef struct { + int size; + const opus_int16 *index; + const unsigned char *bits; + const unsigned char *caps; +} PulseCache; + +/** Mode definition (opaque) + @brief Mode definition + */ +struct OpusCustomMode { + opus_int32 Fs; + int overlap; + + int nbEBands; + int effEBands; + opus_val16 preemph[4]; + const opus_int16 *eBands; /**< Definition for each "pseudo-critical band" */ + + int maxLM; + int nbShortMdcts; + int shortMdctSize; + + int nbAllocVectors; /**< Number of lines in the matrix below */ + const unsigned char *allocVectors; /**< Number of bits in each band for several rates */ + const opus_int16 *logN; + + const opus_val16 *window; + mdct_lookup mdct; + PulseCache cache; +}; + + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/opus_custom_demo.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/opus_custom_demo.c new file mode 100644 index 000000000..ae41c0de5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/opus_custom_demo.c @@ -0,0 +1,210 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_custom.h" +#include "arch.h" +#include +#include +#include +#include + +#define MAX_PACKET 1275 + +int main(int argc, char *argv[]) +{ + int err; + char *inFile, *outFile; + FILE *fin, *fout; + OpusCustomMode *mode=NULL; + OpusCustomEncoder *enc; + OpusCustomDecoder *dec; + int len; + opus_int32 frame_size, channels, rate; + int bytes_per_packet; + unsigned char data[MAX_PACKET]; + int complexity; +#if !(defined (FIXED_POINT) && !defined(CUSTOM_MODES)) && defined(RESYNTH) + int i; + double rmsd = 0; +#endif + int count = 0; + opus_int32 skip; + opus_int16 *in, *out; + if (argc != 9 && argc != 8 && argc != 7) + { + fprintf (stderr, "Usage: test_opus_custom " + " [ [packet loss rate]] " + " \n"); + return 1; + } + + rate = (opus_int32)atol(argv[1]); + channels = atoi(argv[2]); + frame_size = atoi(argv[3]); + mode = opus_custom_mode_create(rate, frame_size, NULL); + if (mode == NULL) + { + fprintf(stderr, "failed to create a mode\n"); + return 1; + } + + bytes_per_packet = atoi(argv[4]); + if (bytes_per_packet < 0 || bytes_per_packet > MAX_PACKET) + { + fprintf (stderr, "bytes per packet must be between 0 and %d\n", + MAX_PACKET); + return 1; + } + + inFile = argv[argc-2]; + fin = fopen(inFile, "rb"); + if (!fin) + { + fprintf (stderr, "Could not open input file %s\n", argv[argc-2]); + return 1; + } + outFile = argv[argc-1]; + fout = fopen(outFile, "wb+"); + if (!fout) + { + fprintf (stderr, "Could not open output file %s\n", argv[argc-1]); + fclose(fin); + return 1; + } + + enc = opus_custom_encoder_create(mode, channels, &err); + if (err != 0) + { + fprintf(stderr, "Failed to create the encoder: %s\n", opus_strerror(err)); + fclose(fin); + fclose(fout); + return 1; + } + dec = opus_custom_decoder_create(mode, channels, &err); + if (err != 0) + { + fprintf(stderr, "Failed to create the decoder: %s\n", opus_strerror(err)); + fclose(fin); + fclose(fout); + return 1; + } + opus_custom_decoder_ctl(dec, OPUS_GET_LOOKAHEAD(&skip)); + + if (argc>7) + { + complexity=atoi(argv[5]); + opus_custom_encoder_ctl(enc,OPUS_SET_COMPLEXITY(complexity)); + } + + in = (opus_int16*)malloc(frame_size*channels*sizeof(opus_int16)); + out = (opus_int16*)malloc(frame_size*channels*sizeof(opus_int16)); + + while (!feof(fin)) + { + int ret; + err = fread(in, sizeof(short), frame_size*channels, fin); + if (feof(fin)) + break; + len = opus_custom_encode(enc, in, frame_size, data, bytes_per_packet); + if (len <= 0) + fprintf (stderr, "opus_custom_encode() failed: %s\n", opus_strerror(len)); + + /* This is for simulating bit errors */ +#if 0 + int errors = 0; + int eid = 0; + /* This simulates random bit error */ + for (i=0;i 0) + { + rmsd = sqrt(rmsd/(1.0*frame_size*channels*count)); + fprintf (stderr, "Error: encoder doesn't match decoder\n"); + fprintf (stderr, "RMS mismatch is %f\n", rmsd); + return 1; + } else { + fprintf (stderr, "Encoder matches decoder!!\n"); + } +#endif + return 0; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/os_support.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/os_support.h new file mode 100644 index 000000000..009bf861d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/os_support.h @@ -0,0 +1,91 @@ +/* Copyright (C) 2007 Jean-Marc Valin + + File: os_support.h + This is the (tiny) OS abstraction layer. Aside from math.h, this is the + only place where system headers are allowed. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef OS_SUPPORT_H +#define OS_SUPPORT_H + +#ifdef CUSTOM_SUPPORT +# include "custom_support.h" +#endif + +#include "opus_types.h" +#include "opus_defines.h" + +#include +#include + +/** Opus wrapper for malloc(). To do your own dynamic allocation, all you need to do is replace this function and opus_free */ +#ifndef OVERRIDE_OPUS_ALLOC +static OPUS_INLINE void *opus_alloc (size_t size) +{ + return malloc(size); +} +#endif + +/** Same as celt_alloc(), except that the area is only needed inside a CELT call (might cause problem with wideband though) */ +#ifndef OVERRIDE_OPUS_ALLOC_SCRATCH +static OPUS_INLINE void *opus_alloc_scratch (size_t size) +{ + /* Scratch space doesn't need to be cleared */ + return opus_alloc(size); +} +#endif + +/** Opus wrapper for free(). To do your own dynamic allocation, all you need to do is replace this function and opus_alloc */ +#ifndef OVERRIDE_OPUS_FREE +static OPUS_INLINE void opus_free (void *ptr) +{ + free(ptr); +} +#endif + +/** Copy n elements from src to dst. The 0* term provides compile-time type checking */ +#ifndef OVERRIDE_OPUS_COPY +#define OPUS_COPY(dst, src, n) (memcpy((dst), (src), (n)*sizeof(*(dst)) + 0*((dst)-(src)) )) +#endif + +/** Copy n elements from src to dst, allowing overlapping regions. The 0* term + provides compile-time type checking */ +#ifndef OVERRIDE_OPUS_MOVE +#define OPUS_MOVE(dst, src, n) (memmove((dst), (src), (n)*sizeof(*(dst)) + 0*((dst)-(src)) )) +#endif + +/** Set n elements of dst to zero */ +#ifndef OVERRIDE_OPUS_CLEAR +#define OPUS_CLEAR(dst, n) (memset((dst), 0, (n)*sizeof(*(dst)))) +#endif + +/*#ifdef __GNUC__ +#pragma GCC poison printf sprintf +#pragma GCC poison malloc free realloc calloc +#endif*/ + +#endif /* OS_SUPPORT_H */ + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/pitch.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/pitch.c new file mode 100644 index 000000000..872582a48 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/pitch.c @@ -0,0 +1,537 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/** + @file pitch.c + @brief Pitch analysis + */ + +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pitch.h" +#include "os_support.h" +#include "modes.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "celt_lpc.h" + +static void find_best_pitch(opus_val32 *xcorr, opus_val16 *y, int len, + int max_pitch, int *best_pitch +#ifdef FIXED_POINT + , int yshift, opus_val32 maxcorr +#endif + ) +{ + int i, j; + opus_val32 Syy=1; + opus_val16 best_num[2]; + opus_val32 best_den[2]; +#ifdef FIXED_POINT + int xshift; + + xshift = celt_ilog2(maxcorr)-14; +#endif + + best_num[0] = -1; + best_num[1] = -1; + best_den[0] = 0; + best_den[1] = 0; + best_pitch[0] = 0; + best_pitch[1] = 1; + for (j=0;j0) + { + opus_val16 num; + opus_val32 xcorr16; + xcorr16 = EXTRACT16(VSHR32(xcorr[i], xshift)); +#ifndef FIXED_POINT + /* Considering the range of xcorr16, this should avoid both underflows + and overflows (inf) when squaring xcorr16 */ + xcorr16 *= 1e-12f; +#endif + num = MULT16_16_Q15(xcorr16,xcorr16); + if (MULT16_32_Q15(num,best_den[1]) > MULT16_32_Q15(best_num[1],Syy)) + { + if (MULT16_32_Q15(num,best_den[0]) > MULT16_32_Q15(best_num[0],Syy)) + { + best_num[1] = best_num[0]; + best_den[1] = best_den[0]; + best_pitch[1] = best_pitch[0]; + best_num[0] = num; + best_den[0] = Syy; + best_pitch[0] = i; + } else { + best_num[1] = num; + best_den[1] = Syy; + best_pitch[1] = i; + } + } + } + Syy += SHR32(MULT16_16(y[i+len],y[i+len]),yshift) - SHR32(MULT16_16(y[i],y[i]),yshift); + Syy = MAX32(1, Syy); + } +} + +static void celt_fir5(opus_val16 *x, + const opus_val16 *num, + int N) +{ + int i; + opus_val16 num0, num1, num2, num3, num4; + opus_val32 mem0, mem1, mem2, mem3, mem4; + num0=num[0]; + num1=num[1]; + num2=num[2]; + num3=num[3]; + num4=num[4]; + mem0=0; + mem1=0; + mem2=0; + mem3=0; + mem4=0; + for (i=0;i>1;i++) + x_lp[i] = SHR32(HALF32(HALF32(x[0][(2*i-1)]+x[0][(2*i+1)])+x[0][2*i]), shift); + x_lp[0] = SHR32(HALF32(HALF32(x[0][1])+x[0][0]), shift); + if (C==2) + { + for (i=1;i>1;i++) + x_lp[i] += SHR32(HALF32(HALF32(x[1][(2*i-1)]+x[1][(2*i+1)])+x[1][2*i]), shift); + x_lp[0] += SHR32(HALF32(HALF32(x[1][1])+x[1][0]), shift); + } + + _celt_autocorr(x_lp, ac, NULL, 0, + 4, len>>1, arch); + + /* Noise floor -40 dB */ +#ifdef FIXED_POINT + ac[0] += SHR32(ac[0],13); +#else + ac[0] *= 1.0001f; +#endif + /* Lag windowing */ + for (i=1;i<=4;i++) + { + /*ac[i] *= exp(-.5*(2*M_PI*.002*i)*(2*M_PI*.002*i));*/ +#ifdef FIXED_POINT + ac[i] -= MULT16_32_Q15(2*i*i, ac[i]); +#else + ac[i] -= ac[i]*(.008f*i)*(.008f*i); +#endif + } + + _celt_lpc(lpc, ac, 4); + for (i=0;i<4;i++) + { + tmp = MULT16_16_Q15(QCONST16(.9f,15), tmp); + lpc[i] = MULT16_16_Q15(lpc[i], tmp); + } + /* Add a zero */ + lpc2[0] = lpc[0] + QCONST16(.8f,SIG_SHIFT); + lpc2[1] = lpc[1] + MULT16_16_Q15(c1,lpc[0]); + lpc2[2] = lpc[2] + MULT16_16_Q15(c1,lpc[1]); + lpc2[3] = lpc[3] + MULT16_16_Q15(c1,lpc[2]); + lpc2[4] = MULT16_16_Q15(c1,lpc[3]); + celt_fir5(x_lp, lpc2, len>>1); +} + +/* Pure C implementation. */ +#ifdef FIXED_POINT +opus_val32 +#else +void +#endif +celt_pitch_xcorr_c(const opus_val16 *_x, const opus_val16 *_y, + opus_val32 *xcorr, int len, int max_pitch, int arch) +{ + +#if 0 /* This is a simple version of the pitch correlation that should work + well on DSPs like Blackfin and TI C5x/C6x */ + int i, j; +#ifdef FIXED_POINT + opus_val32 maxcorr=1; +#endif +#if !defined(OVERRIDE_PITCH_XCORR) + (void)arch; +#endif + for (i=0;i0); + celt_sig_assert((((unsigned char *)_x-(unsigned char *)NULL)&3)==0); + for (i=0;i0); + celt_assert(max_pitch>0); + lag = len+max_pitch; + + ALLOC(x_lp4, len>>2, opus_val16); + ALLOC(y_lp4, lag>>2, opus_val16); + ALLOC(xcorr, max_pitch>>1, opus_val32); + + /* Downsample by 2 again */ + for (j=0;j>2;j++) + x_lp4[j] = x_lp[2*j]; + for (j=0;j>2;j++) + y_lp4[j] = y[2*j]; + +#ifdef FIXED_POINT + xmax = celt_maxabs16(x_lp4, len>>2); + ymax = celt_maxabs16(y_lp4, lag>>2); + shift = celt_ilog2(MAX32(1, MAX32(xmax, ymax)))-11; + if (shift>0) + { + for (j=0;j>2;j++) + x_lp4[j] = SHR16(x_lp4[j], shift); + for (j=0;j>2;j++) + y_lp4[j] = SHR16(y_lp4[j], shift); + /* Use double the shift for a MAC */ + shift *= 2; + } else { + shift = 0; + } +#endif + + /* Coarse search with 4x decimation */ + +#ifdef FIXED_POINT + maxcorr = +#endif + celt_pitch_xcorr(x_lp4, y_lp4, xcorr, len>>2, max_pitch>>2, arch); + + find_best_pitch(xcorr, y_lp4, len>>2, max_pitch>>2, best_pitch +#ifdef FIXED_POINT + , 0, maxcorr +#endif + ); + + /* Finer search with 2x decimation */ +#ifdef FIXED_POINT + maxcorr=1; +#endif + for (i=0;i>1;i++) + { + opus_val32 sum; + xcorr[i] = 0; + if (abs(i-2*best_pitch[0])>2 && abs(i-2*best_pitch[1])>2) + continue; +#ifdef FIXED_POINT + sum = 0; + for (j=0;j>1;j++) + sum += SHR32(MULT16_16(x_lp[j],y[i+j]), shift); +#else + sum = celt_inner_prod(x_lp, y+i, len>>1, arch); +#endif + xcorr[i] = MAX32(-1, sum); +#ifdef FIXED_POINT + maxcorr = MAX32(maxcorr, sum); +#endif + } + find_best_pitch(xcorr, y, len>>1, max_pitch>>1, best_pitch +#ifdef FIXED_POINT + , shift+1, maxcorr +#endif + ); + + /* Refine by pseudo-interpolation */ + if (best_pitch[0]>0 && best_pitch[0]<(max_pitch>>1)-1) + { + opus_val32 a, b, c; + a = xcorr[best_pitch[0]-1]; + b = xcorr[best_pitch[0]]; + c = xcorr[best_pitch[0]+1]; + if ((c-a) > MULT16_32_Q15(QCONST16(.7f,15),b-a)) + offset = 1; + else if ((a-c) > MULT16_32_Q15(QCONST16(.7f,15),b-c)) + offset = -1; + else + offset = 0; + } else { + offset = 0; + } + *pitch = 2*best_pitch[0]-offset; + + RESTORE_STACK; +} + +#ifdef FIXED_POINT +static opus_val16 compute_pitch_gain(opus_val32 xy, opus_val32 xx, opus_val32 yy) +{ + opus_val32 x2y2; + int sx, sy, shift; + opus_val32 g; + opus_val16 den; + if (xy == 0 || xx == 0 || yy == 0) + return 0; + sx = celt_ilog2(xx)-14; + sy = celt_ilog2(yy)-14; + shift = sx + sy; + x2y2 = SHR32(MULT16_16(VSHR32(xx, sx), VSHR32(yy, sy)), 14); + if (shift & 1) { + if (x2y2 < 32768) + { + x2y2 <<= 1; + shift--; + } else { + x2y2 >>= 1; + shift++; + } + } + den = celt_rsqrt_norm(x2y2); + g = MULT16_32_Q15(den, xy); + g = VSHR32(g, (shift>>1)-1); + return EXTRACT16(MIN32(g, Q15ONE)); +} +#else +static opus_val16 compute_pitch_gain(opus_val32 xy, opus_val32 xx, opus_val32 yy) +{ + return xy/celt_sqrt(1+xx*yy); +} +#endif + +static const int second_check[16] = {0, 0, 3, 2, 3, 2, 5, 2, 3, 2, 3, 2, 5, 2, 3, 2}; +opus_val16 remove_doubling(opus_val16 *x, int maxperiod, int minperiod, + int N, int *T0_, int prev_period, opus_val16 prev_gain, int arch) +{ + int k, i, T, T0; + opus_val16 g, g0; + opus_val16 pg; + opus_val32 xy,xx,yy,xy2; + opus_val32 xcorr[3]; + opus_val32 best_xy, best_yy; + int offset; + int minperiod0; + VARDECL(opus_val32, yy_lookup); + SAVE_STACK; + + minperiod0 = minperiod; + maxperiod /= 2; + minperiod /= 2; + *T0_ /= 2; + prev_period /= 2; + N /= 2; + x += maxperiod; + if (*T0_>=maxperiod) + *T0_=maxperiod-1; + + T = T0 = *T0_; + ALLOC(yy_lookup, maxperiod+1, opus_val32); + dual_inner_prod(x, x, x-T0, N, &xx, &xy, arch); + yy_lookup[0] = xx; + yy=xx; + for (i=1;i<=maxperiod;i++) + { + yy = yy+MULT16_16(x[-i],x[-i])-MULT16_16(x[N-i],x[N-i]); + yy_lookup[i] = MAX32(0, yy); + } + yy = yy_lookup[T0]; + best_xy = xy; + best_yy = yy; + g = g0 = compute_pitch_gain(xy, xx, yy); + /* Look for any pitch at T/k */ + for (k=2;k<=15;k++) + { + int T1, T1b; + opus_val16 g1; + opus_val16 cont=0; + opus_val16 thresh; + T1 = celt_udiv(2*T0+k, 2*k); + if (T1 < minperiod) + break; + /* Look for another strong correlation at T1b */ + if (k==2) + { + if (T1+T0>maxperiod) + T1b = T0; + else + T1b = T0+T1; + } else + { + T1b = celt_udiv(2*second_check[k]*T0+k, 2*k); + } + dual_inner_prod(x, &x[-T1], &x[-T1b], N, &xy, &xy2, arch); + xy = HALF32(xy + xy2); + yy = HALF32(yy_lookup[T1] + yy_lookup[T1b]); + g1 = compute_pitch_gain(xy, xx, yy); + if (abs(T1-prev_period)<=1) + cont = prev_gain; + else if (abs(T1-prev_period)<=2 && 5*k*k < T0) + cont = HALF16(prev_gain); + else + cont = 0; + thresh = MAX16(QCONST16(.3f,15), MULT16_16_Q15(QCONST16(.7f,15),g0)-cont); + /* Bias against very high pitch (very short period) to avoid false-positives + due to short-term correlation */ + if (T1<3*minperiod) + thresh = MAX16(QCONST16(.4f,15), MULT16_16_Q15(QCONST16(.85f,15),g0)-cont); + else if (T1<2*minperiod) + thresh = MAX16(QCONST16(.5f,15), MULT16_16_Q15(QCONST16(.9f,15),g0)-cont); + if (g1 > thresh) + { + best_xy = xy; + best_yy = yy; + T = T1; + g = g1; + } + } + best_xy = MAX32(0, best_xy); + if (best_yy <= best_xy) + pg = Q15ONE; + else + pg = SHR32(frac_div32(best_xy,best_yy+1),16); + + for (k=0;k<3;k++) + xcorr[k] = celt_inner_prod(x, x-(T+k-1), N, arch); + if ((xcorr[2]-xcorr[0]) > MULT16_32_Q15(QCONST16(.7f,15),xcorr[1]-xcorr[0])) + offset = 1; + else if ((xcorr[0]-xcorr[2]) > MULT16_32_Q15(QCONST16(.7f,15),xcorr[1]-xcorr[2])) + offset = -1; + else + offset = 0; + if (pg > g) + pg = g; + *T0_ = 2*T+offset; + + if (*T0_=3); + y_3=0; /* gcc doesn't realize that y_3 can't be used uninitialized */ + y_0=*y++; + y_1=*y++; + y_2=*y++; + for (j=0;j +#include "os_support.h" +#include "arch.h" +#include "mathops.h" +#include "stack_alloc.h" +#include "rate.h" + +#ifdef FIXED_POINT +/* Mean energy in each band quantized in Q4 */ +const signed char eMeans[25] = { + 103,100, 92, 85, 81, + 77, 72, 70, 78, 75, + 73, 71, 78, 74, 69, + 72, 70, 74, 76, 71, + 60, 60, 60, 60, 60 +}; +#else +/* Mean energy in each band quantized in Q4 and converted back to float */ +const opus_val16 eMeans[25] = { + 6.437500f, 6.250000f, 5.750000f, 5.312500f, 5.062500f, + 4.812500f, 4.500000f, 4.375000f, 4.875000f, 4.687500f, + 4.562500f, 4.437500f, 4.875000f, 4.625000f, 4.312500f, + 4.500000f, 4.375000f, 4.625000f, 4.750000f, 4.437500f, + 3.750000f, 3.750000f, 3.750000f, 3.750000f, 3.750000f +}; +#endif +/* prediction coefficients: 0.9, 0.8, 0.65, 0.5 */ +#ifdef FIXED_POINT +static const opus_val16 pred_coef[4] = {29440, 26112, 21248, 16384}; +static const opus_val16 beta_coef[4] = {30147, 22282, 12124, 6554}; +static const opus_val16 beta_intra = 4915; +#else +static const opus_val16 pred_coef[4] = {29440/32768., 26112/32768., 21248/32768., 16384/32768.}; +static const opus_val16 beta_coef[4] = {30147/32768., 22282/32768., 12124/32768., 6554/32768.}; +static const opus_val16 beta_intra = 4915/32768.; +#endif + +/*Parameters of the Laplace-like probability models used for the coarse energy. + There is one pair of parameters for each frame size, prediction type + (inter/intra), and band number. + The first number of each pair is the probability of 0, and the second is the + decay rate, both in Q8 precision.*/ +static const unsigned char e_prob_model[4][2][42] = { + /*120 sample frames.*/ + { + /*Inter*/ + { + 72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, + 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, + 114, 40, 132, 26, 132, 26, 145, 17, 161, 12, 176, 10, 177, 11 + }, + /*Intra*/ + { + 24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, + 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, + 91, 67, 100, 59, 108, 50, 120, 40, 122, 37, 97, 43, 78, 50 + } + }, + /*240 sample frames.*/ + { + /*Inter*/ + { + 83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, + 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, + 146, 19, 162, 12, 165, 10, 178, 7, 189, 6, 190, 8, 177, 9 + }, + /*Intra*/ + { + 23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, + 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, + 104, 60, 117, 52, 123, 44, 138, 35, 133, 31, 97, 38, 77, 45 + } + }, + /*480 sample frames.*/ + { + /*Inter*/ + { + 61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, + 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, + 158, 18, 170, 13, 177, 10, 187, 8, 192, 6, 175, 9, 159, 10 + }, + /*Intra*/ + { + 21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, + 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, + 112, 56, 129, 51, 132, 40, 150, 33, 140, 29, 98, 35, 77, 42 + } + }, + /*960 sample frames.*/ + { + /*Inter*/ + { + 42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, + 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, + 154, 26, 166, 21, 173, 16, 184, 13, 184, 10, 150, 13, 139, 15 + }, + /*Intra*/ + { + 22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, + 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, + 117, 55, 135, 49, 137, 39, 157, 32, 145, 29, 97, 33, 77, 40 + } + } +}; + +static const unsigned char small_energy_icdf[3]={2,1,0}; + +static opus_val32 loss_distortion(const opus_val16 *eBands, opus_val16 *oldEBands, int start, int end, int len, int C) +{ + int c, i; + opus_val32 dist = 0; + c=0; do { + for (i=start;inbEBands]; + oldE = MAX16(-QCONST16(9.f,DB_SHIFT), oldEBands[i+c*m->nbEBands]); +#ifdef FIXED_POINT + f = SHL32(EXTEND32(x),7) - PSHR32(MULT16_16(coef,oldE), 8) - prev[c]; + /* Rounding to nearest integer here is really important! */ + qi = (f+QCONST32(.5f,DB_SHIFT+7))>>(DB_SHIFT+7); + decay_bound = EXTRACT16(MAX32(-QCONST16(28.f,DB_SHIFT), + SUB32((opus_val32)oldEBands[i+c*m->nbEBands],max_decay))); +#else + f = x-coef*oldE-prev[c]; + /* Rounding to nearest integer here is really important! */ + qi = (int)floor(.5f+f); + decay_bound = MAX16(-QCONST16(28.f,DB_SHIFT), oldEBands[i+c*m->nbEBands]) - max_decay; +#endif + /* Prevent the energy from going down too quickly (e.g. for bands + that have just one bin) */ + if (qi < 0 && x < decay_bound) + { + qi += (int)SHR16(SUB16(decay_bound,x), DB_SHIFT); + if (qi > 0) + qi = 0; + } + qi0 = qi; + /* If we don't have enough bits to encode all the energy, just assume + something safe. */ + tell = ec_tell(enc); + bits_left = budget-tell-3*C*(end-i); + if (i!=start && bits_left < 30) + { + if (bits_left < 24) + qi = IMIN(1, qi); + if (bits_left < 16) + qi = IMAX(-1, qi); + } + if (lfe && i>=2) + qi = IMIN(qi, 0); + if (budget-tell >= 15) + { + int pi; + pi = 2*IMIN(i,20); + ec_laplace_encode(enc, &qi, + prob_model[pi]<<7, prob_model[pi+1]<<6); + } + else if(budget-tell >= 2) + { + qi = IMAX(-1, IMIN(qi, 1)); + ec_enc_icdf(enc, 2*qi^-(qi<0), small_energy_icdf, 2); + } + else if(budget-tell >= 1) + { + qi = IMIN(0, qi); + ec_enc_bit_logp(enc, -qi, 1); + } + else + qi = -1; + error[i+c*m->nbEBands] = PSHR32(f,7) - SHL16(qi,DB_SHIFT); + badness += abs(qi0-qi); + q = (opus_val32)SHL32(EXTEND32(qi),DB_SHIFT); + + tmp = PSHR32(MULT16_16(coef,oldE),8) + prev[c] + SHL32(q,7); +#ifdef FIXED_POINT + tmp = MAX32(-QCONST32(28.f, DB_SHIFT+7), tmp); +#endif + oldEBands[i+c*m->nbEBands] = PSHR32(tmp, 7); + prev[c] = prev[c] + SHL32(q,7) - MULT16_16(beta,PSHR32(q,8)); + } while (++c < C); + } + return lfe ? 0 : badness; +} + +void quant_coarse_energy(const CELTMode *m, int start, int end, int effEnd, + const opus_val16 *eBands, opus_val16 *oldEBands, opus_uint32 budget, + opus_val16 *error, ec_enc *enc, int C, int LM, int nbAvailableBytes, + int force_intra, opus_val32 *delayedIntra, int two_pass, int loss_rate, int lfe) +{ + int intra; + opus_val16 max_decay; + VARDECL(opus_val16, oldEBands_intra); + VARDECL(opus_val16, error_intra); + ec_enc enc_start_state; + opus_uint32 tell; + int badness1=0; + opus_int32 intra_bias; + opus_val32 new_distortion; + SAVE_STACK; + + intra = force_intra || (!two_pass && *delayedIntra>2*C*(end-start) && nbAvailableBytes > (end-start)*C); + intra_bias = (opus_int32)((budget**delayedIntra*loss_rate)/(C*512)); + new_distortion = loss_distortion(eBands, oldEBands, start, effEnd, m->nbEBands, C); + + tell = ec_tell(enc); + if (tell+3 > budget) + two_pass = intra = 0; + + max_decay = QCONST16(16.f,DB_SHIFT); + if (end-start>10) + { +#ifdef FIXED_POINT + max_decay = MIN32(max_decay, SHL32(EXTEND32(nbAvailableBytes),DB_SHIFT-3)); +#else + max_decay = MIN32(max_decay, .125f*nbAvailableBytes); +#endif + } + if (lfe) + max_decay = QCONST16(3.f,DB_SHIFT); + enc_start_state = *enc; + + ALLOC(oldEBands_intra, C*m->nbEBands, opus_val16); + ALLOC(error_intra, C*m->nbEBands, opus_val16); + OPUS_COPY(oldEBands_intra, oldEBands, C*m->nbEBands); + + if (two_pass || intra) + { + badness1 = quant_coarse_energy_impl(m, start, end, eBands, oldEBands_intra, budget, + tell, e_prob_model[LM][1], error_intra, enc, C, LM, 1, max_decay, lfe); + } + + if (!intra) + { + unsigned char *intra_buf; + ec_enc enc_intra_state; + opus_int32 tell_intra; + opus_uint32 nstart_bytes; + opus_uint32 nintra_bytes; + opus_uint32 save_bytes; + int badness2; + VARDECL(unsigned char, intra_bits); + + tell_intra = ec_tell_frac(enc); + + enc_intra_state = *enc; + + nstart_bytes = ec_range_bytes(&enc_start_state); + nintra_bytes = ec_range_bytes(&enc_intra_state); + intra_buf = ec_get_buffer(&enc_intra_state) + nstart_bytes; + save_bytes = nintra_bytes-nstart_bytes; + if (save_bytes == 0) + save_bytes = ALLOC_NONE; + ALLOC(intra_bits, save_bytes, unsigned char); + /* Copy bits from intra bit-stream */ + OPUS_COPY(intra_bits, intra_buf, nintra_bytes - nstart_bytes); + + *enc = enc_start_state; + + badness2 = quant_coarse_energy_impl(m, start, end, eBands, oldEBands, budget, + tell, e_prob_model[LM][intra], error, enc, C, LM, 0, max_decay, lfe); + + if (two_pass && (badness1 < badness2 || (badness1 == badness2 && ((opus_int32)ec_tell_frac(enc))+intra_bias > tell_intra))) + { + *enc = enc_intra_state; + /* Copy intra bits to bit-stream */ + OPUS_COPY(intra_buf, intra_bits, nintra_bytes - nstart_bytes); + OPUS_COPY(oldEBands, oldEBands_intra, C*m->nbEBands); + OPUS_COPY(error, error_intra, C*m->nbEBands); + intra = 1; + } + } else { + OPUS_COPY(oldEBands, oldEBands_intra, C*m->nbEBands); + OPUS_COPY(error, error_intra, C*m->nbEBands); + } + + if (intra) + *delayedIntra = new_distortion; + else + *delayedIntra = ADD32(MULT16_32_Q15(MULT16_16_Q15(pred_coef[LM], pred_coef[LM]),*delayedIntra), + new_distortion); + + RESTORE_STACK; +} + +void quant_fine_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, opus_val16 *error, int *fine_quant, ec_enc *enc, int C) +{ + int i, c; + + /* Encode finer resolution */ + for (i=start;inbEBands]+QCONST16(.5f,DB_SHIFT))>>(DB_SHIFT-fine_quant[i]); +#else + q2 = (int)floor((error[i+c*m->nbEBands]+.5f)*frac); +#endif + if (q2 > frac-1) + q2 = frac-1; + if (q2<0) + q2 = 0; + ec_enc_bits(enc, q2, fine_quant[i]); +#ifdef FIXED_POINT + offset = SUB16(SHR32(SHL32(EXTEND32(q2),DB_SHIFT)+QCONST16(.5f,DB_SHIFT),fine_quant[i]),QCONST16(.5f,DB_SHIFT)); +#else + offset = (q2+.5f)*(1<<(14-fine_quant[i]))*(1.f/16384) - .5f; +#endif + oldEBands[i+c*m->nbEBands] += offset; + error[i+c*m->nbEBands] -= offset; + /*printf ("%f ", error[i] - offset);*/ + } while (++c < C); + } +} + +void quant_energy_finalise(const CELTMode *m, int start, int end, opus_val16 *oldEBands, opus_val16 *error, int *fine_quant, int *fine_priority, int bits_left, ec_enc *enc, int C) +{ + int i, prio, c; + + /* Use up the remaining bits */ + for (prio=0;prio<2;prio++) + { + for (i=start;i=C ;i++) + { + if (fine_quant[i] >= MAX_FINE_BITS || fine_priority[i]!=prio) + continue; + c=0; + do { + int q2; + opus_val16 offset; + q2 = error[i+c*m->nbEBands]<0 ? 0 : 1; + ec_enc_bits(enc, q2, 1); +#ifdef FIXED_POINT + offset = SHR16(SHL16(q2,DB_SHIFT)-QCONST16(.5f,DB_SHIFT),fine_quant[i]+1); +#else + offset = (q2-.5f)*(1<<(14-fine_quant[i]-1))*(1.f/16384); +#endif + oldEBands[i+c*m->nbEBands] += offset; + error[i+c*m->nbEBands] -= offset; + bits_left--; + } while (++c < C); + } + } +} + +void unquant_coarse_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int intra, ec_dec *dec, int C, int LM) +{ + const unsigned char *prob_model = e_prob_model[LM][intra]; + int i, c; + opus_val32 prev[2] = {0, 0}; + opus_val16 coef; + opus_val16 beta; + opus_int32 budget; + opus_int32 tell; + + if (intra) + { + coef = 0; + beta = beta_intra; + } else { + beta = beta_coef[LM]; + coef = pred_coef[LM]; + } + + budget = dec->storage*8; + + /* Decode at a fixed coarse resolution */ + for (i=start;i=15) + { + int pi; + pi = 2*IMIN(i,20); + qi = ec_laplace_decode(dec, + prob_model[pi]<<7, prob_model[pi+1]<<6); + } + else if(budget-tell>=2) + { + qi = ec_dec_icdf(dec, small_energy_icdf, 2); + qi = (qi>>1)^-(qi&1); + } + else if(budget-tell>=1) + { + qi = -ec_dec_bit_logp(dec, 1); + } + else + qi = -1; + q = (opus_val32)SHL32(EXTEND32(qi),DB_SHIFT); + + oldEBands[i+c*m->nbEBands] = MAX16(-QCONST16(9.f,DB_SHIFT), oldEBands[i+c*m->nbEBands]); + tmp = PSHR32(MULT16_16(coef,oldEBands[i+c*m->nbEBands]),8) + prev[c] + SHL32(q,7); +#ifdef FIXED_POINT + tmp = MAX32(-QCONST32(28.f, DB_SHIFT+7), tmp); +#endif + oldEBands[i+c*m->nbEBands] = PSHR32(tmp, 7); + prev[c] = prev[c] + SHL32(q,7) - MULT16_16(beta,PSHR32(q,8)); + } while (++c < C); + } +} + +void unquant_fine_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int *fine_quant, ec_dec *dec, int C) +{ + int i, c; + /* Decode finer resolution */ + for (i=start;inbEBands] += offset; + } while (++c < C); + } +} + +void unquant_energy_finalise(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int *fine_quant, int *fine_priority, int bits_left, ec_dec *dec, int C) +{ + int i, prio, c; + + /* Use up the remaining bits */ + for (prio=0;prio<2;prio++) + { + for (i=start;i=C ;i++) + { + if (fine_quant[i] >= MAX_FINE_BITS || fine_priority[i]!=prio) + continue; + c=0; + do { + int q2; + opus_val16 offset; + q2 = ec_dec_bits(dec, 1); +#ifdef FIXED_POINT + offset = SHR16(SHL16(q2,DB_SHIFT)-QCONST16(.5f,DB_SHIFT),fine_quant[i]+1); +#else + offset = (q2-.5f)*(1<<(14-fine_quant[i]-1))*(1.f/16384); +#endif + oldEBands[i+c*m->nbEBands] += offset; + bits_left--; + } while (++c < C); + } + } +} + +void amp2Log2(const CELTMode *m, int effEnd, int end, + celt_ener *bandE, opus_val16 *bandLogE, int C) +{ + int c, i; + c=0; + do { + for (i=0;inbEBands] = + celt_log2(bandE[i+c*m->nbEBands]) + - SHL16((opus_val16)eMeans[i],6); +#ifdef FIXED_POINT + /* Compensate for bandE[] being Q12 but celt_log2() taking a Q14 input. */ + bandLogE[i+c*m->nbEBands] += QCONST16(2.f, DB_SHIFT); +#endif + } + for (i=effEnd;inbEBands+i] = -QCONST16(14.f,DB_SHIFT); + } while (++c < C); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/quant_bands.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/quant_bands.h new file mode 100644 index 000000000..0490bca4b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/quant_bands.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef QUANT_BANDS +#define QUANT_BANDS + +#include "arch.h" +#include "modes.h" +#include "entenc.h" +#include "entdec.h" +#include "mathops.h" + +#ifdef FIXED_POINT +extern const signed char eMeans[25]; +#else +extern const opus_val16 eMeans[25]; +#endif + +void amp2Log2(const CELTMode *m, int effEnd, int end, + celt_ener *bandE, opus_val16 *bandLogE, int C); + +void log2Amp(const CELTMode *m, int start, int end, + celt_ener *eBands, const opus_val16 *oldEBands, int C); + +void quant_coarse_energy(const CELTMode *m, int start, int end, int effEnd, + const opus_val16 *eBands, opus_val16 *oldEBands, opus_uint32 budget, + opus_val16 *error, ec_enc *enc, int C, int LM, + int nbAvailableBytes, int force_intra, opus_val32 *delayedIntra, + int two_pass, int loss_rate, int lfe); + +void quant_fine_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, opus_val16 *error, int *fine_quant, ec_enc *enc, int C); + +void quant_energy_finalise(const CELTMode *m, int start, int end, opus_val16 *oldEBands, opus_val16 *error, int *fine_quant, int *fine_priority, int bits_left, ec_enc *enc, int C); + +void unquant_coarse_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int intra, ec_dec *dec, int C, int LM); + +void unquant_fine_energy(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int *fine_quant, ec_dec *dec, int C); + +void unquant_energy_finalise(const CELTMode *m, int start, int end, opus_val16 *oldEBands, int *fine_quant, int *fine_priority, int bits_left, ec_dec *dec, int C); + +#endif /* QUANT_BANDS */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.c new file mode 100644 index 000000000..465e1ba26 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.c @@ -0,0 +1,644 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "modes.h" +#include "cwrs.h" +#include "arch.h" +#include "os_support.h" + +#include "entcode.h" +#include "rate.h" + +static const unsigned char LOG2_FRAC_TABLE[24]={ + 0, + 8,13, + 16,19,21,23, + 24,26,27,28,29,30,31,32, + 32,33,34,34,35,36,36,37,37 +}; + +#ifdef CUSTOM_MODES + +/*Determines if V(N,K) fits in a 32-bit unsigned integer. + N and K are themselves limited to 15 bits.*/ +static int fits_in32(int _n, int _k) +{ + static const opus_int16 maxN[15] = { + 32767, 32767, 32767, 1476, 283, 109, 60, 40, + 29, 24, 20, 18, 16, 14, 13}; + static const opus_int16 maxK[15] = { + 32767, 32767, 32767, 32767, 1172, 238, 95, 53, + 36, 27, 22, 18, 16, 15, 13}; + if (_n>=14) + { + if (_k>=14) + return 0; + else + return _n <= maxN[_k]; + } else { + return _k <= maxK[_n]; + } +} + +void compute_pulse_cache(CELTMode *m, int LM) +{ + int C; + int i; + int j; + int curr=0; + int nbEntries=0; + int entryN[100], entryK[100], entryI[100]; + const opus_int16 *eBands = m->eBands; + PulseCache *cache = &m->cache; + opus_int16 *cindex; + unsigned char *bits; + unsigned char *cap; + + cindex = (opus_int16 *)opus_alloc(sizeof(cache->index[0])*m->nbEBands*(LM+2)); + cache->index = cindex; + + /* Scan for all unique band sizes */ + for (i=0;i<=LM+1;i++) + { + for (j=0;jnbEBands;j++) + { + int k; + int N = (eBands[j+1]-eBands[j])<>1; + cindex[i*m->nbEBands+j] = -1; + /* Find other bands that have the same size */ + for (k=0;k<=i;k++) + { + int n; + for (n=0;nnbEBands && (k!=i || n>1) + { + cindex[i*m->nbEBands+j] = cindex[k*m->nbEBands+n]; + break; + } + } + } + if (cache->index[i*m->nbEBands+j] == -1 && N!=0) + { + int K; + entryN[nbEntries] = N; + K = 0; + while (fits_in32(N,get_pulses(K+1)) && KnbEBands+j] = curr; + entryI[nbEntries] = curr; + + curr += K+1; + nbEntries++; + } + } + } + bits = (unsigned char *)opus_alloc(sizeof(unsigned char)*curr); + cache->bits = bits; + cache->size = curr; + /* Compute the cache for all unique sizes */ + for (i=0;icaps = cap = (unsigned char *)opus_alloc(sizeof(cache->caps[0])*(LM+1)*2*m->nbEBands); + for (i=0;i<=LM;i++) + { + for (C=1;C<=2;C++) + { + for (j=0;jnbEBands;j++) + { + int N0; + int max_bits; + N0 = m->eBands[j+1]-m->eBands[j]; + /* N=1 bands only have a sign bit and fine bits. */ + if (N0<1 are even, including custom modes.*/ + if (N0 > 2) + { + N0>>=1; + LM0--; + } + /* N0=1 bands can't be split down to N<2. */ + else if (N0 <= 1) + { + LM0=IMIN(i,1); + N0<<=LM0; + } + /* Compute the cost for the lowest-level PVQ of a fully split + band. */ + pcache = bits + cindex[(LM0+1)*m->nbEBands+j]; + max_bits = pcache[pcache[0]]+1; + /* Add in the cost of coding regular splits. */ + N = N0; + for(k=0;klogN[j]+((LM0+k)<>1)-QTHETA_OFFSET; + /* The number of qtheta bits we'll allocate if the remainder + is to be max_bits. + The average measured cost for theta is 0.89701 times qb, + approximated here as 459/512. */ + num=459*(opus_int32)((2*N-1)*offset+max_bits); + den=((opus_int32)(2*N-1)<<9)-459; + qb = IMIN((num+(den>>1))/den, 57); + celt_assert(qb >= 0); + max_bits += qb; + N <<= 1; + } + /* Add in the cost of a stereo split, if necessary. */ + if (C==2) + { + max_bits <<= 1; + offset = ((m->logN[j]+(i<>1)-(N==2?QTHETA_OFFSET_TWOPHASE:QTHETA_OFFSET); + ndof = 2*N-1-(N==2); + /* The average measured cost for theta with the step PDF is + 0.95164 times qb, approximated here as 487/512. */ + num = (N==2?512:487)*(opus_int32)(max_bits+ndof*offset); + den = ((opus_int32)ndof<<9)-(N==2?512:487); + qb = IMIN((num+(den>>1))/den, (N==2?64:61)); + celt_assert(qb >= 0); + max_bits += qb; + } + /* Add the fine bits we'll use. */ + /* Compensate for the extra DoF in stereo */ + ndof = C*N + ((C==2 && N>2) ? 1 : 0); + /* Offset the number of fine bits by log2(N)/2 + FINE_OFFSET + compared to their "fair share" of total/N */ + offset = ((m->logN[j] + (i<>1)-FINE_OFFSET; + /* N=2 is the only point that doesn't match the curve */ + if (N==2) + offset += 1<>2; + /* The number of fine bits we'll allocate if the remainder is + to be max_bits. */ + num = max_bits+ndof*offset; + den = (ndof-1)<>1))/den, MAX_FINE_BITS); + celt_assert(qb >= 0); + max_bits += C*qb<eBands[j+1]-m->eBands[j])<= 0); + celt_assert(max_bits < 256); + *cap++ = (unsigned char)max_bits; + } + } + } +} + +#endif /* CUSTOM_MODES */ + +#define ALLOC_STEPS 6 + +static OPUS_INLINE int interp_bits2pulses(const CELTMode *m, int start, int end, int skip_start, + const int *bits1, const int *bits2, const int *thresh, const int *cap, opus_int32 total, opus_int32 *_balance, + int skip_rsv, int *intensity, int intensity_rsv, int *dual_stereo, int dual_stereo_rsv, int *bits, + int *ebits, int *fine_priority, int C, int LM, ec_ctx *ec, int encode, int prev, int signalBandwidth) +{ + opus_int32 psum; + int lo, hi; + int i, j; + int logM; + int stereo; + int codedBands=-1; + int alloc_floor; + opus_int32 left, percoeff; + int done; + opus_int32 balance; + SAVE_STACK; + + alloc_floor = C<1; + + logM = LM<>1; + psum = 0; + done = 0; + for (j=end;j-->start;) + { + int tmp = bits1[j] + (mid*(opus_int32)bits2[j]>>ALLOC_STEPS); + if (tmp >= thresh[j] || done) + { + done = 1; + /* Don't allocate more than we can actually use */ + psum += IMIN(tmp, cap[j]); + } else { + if (tmp >= alloc_floor) + psum += alloc_floor; + } + } + if (psum > total) + hi = mid; + else + lo = mid; + } + psum = 0; + /*printf ("interp bisection gave %d\n", lo);*/ + done = 0; + for (j=end;j-->start;) + { + int tmp = bits1[j] + ((opus_int32)lo*bits2[j]>>ALLOC_STEPS); + if (tmp < thresh[j] && !done) + { + if (tmp >= alloc_floor) + tmp = alloc_floor; + else + tmp = 0; + } else + done = 1; + /* Don't allocate more than we can actually use */ + tmp = IMIN(tmp, cap[j]); + bits[j] = tmp; + psum += tmp; + } + + /* Decide which bands to skip, working backwards from the end. */ + for (codedBands=end;;codedBands--) + { + int band_width; + int band_bits; + int rem; + j = codedBands-1; + /* Never skip the first band, nor a band that has been boosted by + dynalloc. + In the first case, we'd be coding a bit to signal we're going to waste + all the other bits. + In the second case, we'd be coding a bit to redistribute all the bits + we just signaled should be cocentrated in this band. */ + if (j<=skip_start) + { + /* Give the bit we reserved to end skipping back. */ + total += skip_rsv; + break; + } + /*Figure out how many left-over bits we would be adding to this band. + This can include bits we've stolen back from higher, skipped bands.*/ + left = total-psum; + percoeff = celt_udiv(left, m->eBands[codedBands]-m->eBands[start]); + left -= (m->eBands[codedBands]-m->eBands[start])*percoeff; + rem = IMAX(left-(m->eBands[j]-m->eBands[start]),0); + band_width = m->eBands[codedBands]-m->eBands[j]; + band_bits = (int)(bits[j] + percoeff*band_width + rem); + /*Only code a skip decision if we're above the threshold for this band. + Otherwise it is force-skipped. + This ensures that we have enough bits to code the skip flag.*/ + if (band_bits >= IMAX(thresh[j], alloc_floor+(1< 17) + depth_threshold = j (depth_threshold*band_width<>4 && j<=signalBandwidth)) +#endif + { + ec_enc_bit_logp(ec, 1, 1); + break; + } + ec_enc_bit_logp(ec, 0, 1); + } else if (ec_dec_bit_logp(ec, 1)) { + break; + } + /*We used a bit to skip this band.*/ + psum += 1< 0) + intensity_rsv = LOG2_FRAC_TABLE[j-start]; + psum += intensity_rsv; + if (band_bits >= alloc_floor) + { + /*If we have enough for a fine energy bit per channel, use it.*/ + psum += alloc_floor; + bits[j] = alloc_floor; + } else { + /*Otherwise this band gets nothing at all.*/ + bits[j] = 0; + } + } + + celt_assert(codedBands > start); + /* Code the intensity and dual stereo parameters. */ + if (intensity_rsv > 0) + { + if (encode) + { + *intensity = IMIN(*intensity, codedBands); + ec_enc_uint(ec, *intensity-start, codedBands+1-start); + } + else + *intensity = start+ec_dec_uint(ec, codedBands+1-start); + } + else + *intensity = 0; + if (*intensity <= start) + { + total += dual_stereo_rsv; + dual_stereo_rsv = 0; + } + if (dual_stereo_rsv > 0) + { + if (encode) + ec_enc_bit_logp(ec, *dual_stereo, 1); + else + *dual_stereo = ec_dec_bit_logp(ec, 1); + } + else + *dual_stereo = 0; + + /* Allocate the remaining bits */ + left = total-psum; + percoeff = celt_udiv(left, m->eBands[codedBands]-m->eBands[start]); + left -= (m->eBands[codedBands]-m->eBands[start])*percoeff; + for (j=start;jeBands[j+1]-m->eBands[j])); + for (j=start;jeBands[j+1]-m->eBands[j]); + bits[j] += tmp; + left -= tmp; + } + /*for (j=0;j= 0); + N0 = m->eBands[j+1]-m->eBands[j]; + N=N0<1) + { + excess = MAX32(bit-cap[j],0); + bits[j] = bit-excess; + + /* Compensate for the extra DoF in stereo */ + den=(C*N+ ((C==2 && N>2 && !*dual_stereo && j<*intensity) ? 1 : 0)); + + NClogN = den*(m->logN[j] + logM); + + /* Offset for the number of fine bits by log2(N)/2 + FINE_OFFSET + compared to their "fair share" of total/N */ + offset = (NClogN>>1)-den*FINE_OFFSET; + + /* N=2 is the only point that doesn't match the curve */ + if (N==2) + offset += den<>2; + + /* Changing the offset for allocating the second and third + fine energy bit */ + if (bits[j] + offset < den*2<>2; + else if (bits[j] + offset < den*3<>3; + + /* Divide with rounding */ + ebits[j] = IMAX(0, (bits[j] + offset + (den<<(BITRES-1)))); + ebits[j] = celt_udiv(ebits[j], den)>>BITRES; + + /* Make sure not to bust */ + if (C*ebits[j] > (bits[j]>>BITRES)) + ebits[j] = bits[j] >> stereo >> BITRES; + + /* More than that is useless because that's about as far as PVQ can go */ + ebits[j] = IMIN(ebits[j], MAX_FINE_BITS); + + /* If we rounded down or capped this band, make it a candidate for the + final fine energy pass */ + fine_priority[j] = ebits[j]*(den<= bits[j]+offset; + + /* Remove the allocated fine bits; the rest are assigned to PVQ */ + bits[j] -= C*ebits[j]< 0) + { + int extra_fine; + int extra_bits; + extra_fine = IMIN(excess>>(stereo+BITRES),MAX_FINE_BITS-ebits[j]); + ebits[j] += extra_fine; + extra_bits = extra_fine*C<= excess-balance; + excess -= extra_bits; + } + balance = excess; + + celt_assert(bits[j] >= 0); + celt_assert(ebits[j] >= 0); + } + /* Save any remaining bits over the cap for the rebalancing in + quant_all_bands(). */ + *_balance = balance; + + /* The skipped bands use all their bits for fine energy. */ + for (;j> stereo >> BITRES; + celt_assert(C*ebits[j]<nbEBands; + skip_start = start; + /* Reserve a bit to signal the end of manually skipped bands. */ + skip_rsv = total >= 1<total) + intensity_rsv = 0; + else + { + total -= intensity_rsv; + dual_stereo_rsv = total>=1<eBands[j+1]-m->eBands[j])<>4); + /* Tilt of the allocation curve */ + trim_offset[j] = C*(m->eBands[j+1]-m->eBands[j])*(alloc_trim-5-LM)*(end-j-1) + *(1<<(LM+BITRES))>>6; + /* Giving less resolution to single-coefficient bands because they get + more benefit from having one coarse value per coefficient*/ + if ((m->eBands[j+1]-m->eBands[j])<nbAllocVectors - 1; + do + { + int done = 0; + int psum = 0; + int mid = (lo+hi) >> 1; + for (j=end;j-->start;) + { + int bitsj; + int N = m->eBands[j+1]-m->eBands[j]; + bitsj = C*N*m->allocVectors[mid*len+j]<>2; + if (bitsj > 0) + bitsj = IMAX(0, bitsj + trim_offset[j]); + bitsj += offsets[j]; + if (bitsj >= thresh[j] || done) + { + done = 1; + /* Don't allocate more than we can actually use */ + psum += IMIN(bitsj, cap[j]); + } else { + if (bitsj >= C< total) + hi = mid - 1; + else + lo = mid + 1; + /*printf ("lo = %d, hi = %d\n", lo, hi);*/ + } + while (lo <= hi); + hi = lo--; + /*printf ("interp between %d and %d\n", lo, hi);*/ + for (j=start;jeBands[j+1]-m->eBands[j]; + bits1j = C*N*m->allocVectors[lo*len+j]<>2; + bits2j = hi>=m->nbAllocVectors ? + cap[j] : C*N*m->allocVectors[hi*len+j]<>2; + if (bits1j > 0) + bits1j = IMAX(0, bits1j + trim_offset[j]); + if (bits2j > 0) + bits2j = IMAX(0, bits2j + trim_offset[j]); + if (lo > 0) + bits1j += offsets[j]; + bits2j += offsets[j]; + if (offsets[j]>0) + skip_start = j; + bits2j = IMAX(0,bits2j-bits1j); + bits1[j] = bits1j; + bits2[j] = bits2j; + } + codedBands = interp_bits2pulses(m, start, end, skip_start, bits1, bits2, thresh, cap, + total, balance, skip_rsv, intensity, intensity_rsv, dual_stereo, dual_stereo_rsv, + pulses, ebits, fine_priority, C, LM, ec, encode, prev, signalBandwidth); + RESTORE_STACK; + return codedBands; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.h new file mode 100644 index 000000000..fad5e412d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/rate.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef RATE_H +#define RATE_H + +#define MAX_PSEUDO 40 +#define LOG_MAX_PSEUDO 6 + +#define CELT_MAX_PULSES 128 + +#define MAX_FINE_BITS 8 + +#define FINE_OFFSET 21 +#define QTHETA_OFFSET 4 +#define QTHETA_OFFSET_TWOPHASE 16 + +#include "cwrs.h" +#include "modes.h" + +void compute_pulse_cache(CELTMode *m, int LM); + +static OPUS_INLINE int get_pulses(int i) +{ + return i<8 ? i : (8 + (i&7)) << ((i>>3)-1); +} + +static OPUS_INLINE int bits2pulses(const CELTMode *m, int band, int LM, int bits) +{ + int i; + int lo, hi; + const unsigned char *cache; + + LM++; + cache = m->cache.bits + m->cache.index[LM*m->nbEBands+band]; + + lo = 0; + hi = cache[0]; + bits--; + for (i=0;i>1; + /* OPT: Make sure this is implemented with a conditional move */ + if ((int)cache[mid] >= bits) + hi = mid; + else + lo = mid; + } + if (bits- (lo == 0 ? -1 : (int)cache[lo]) <= (int)cache[hi]-bits) + return lo; + else + return hi; +} + +static OPUS_INLINE int pulses2bits(const CELTMode *m, int band, int LM, int pulses) +{ + const unsigned char *cache; + + LM++; + cache = m->cache.bits + m->cache.index[LM*m->nbEBands+band]; + return pulses == 0 ? 0 : cache[pulses]+1; +} + +/** Compute the pulse allocation, i.e. how many pulses will go in each + * band. + @param m mode + @param offsets Requested increase or decrease in the number of bits for + each band + @param total Number of bands + @param pulses Number of pulses per band (returned) + @return Total number of bits allocated +*/ +int clt_compute_allocation(const CELTMode *m, int start, int end, const int *offsets, const int *cap, int alloc_trim, int *intensity, int *dual_stereo, + opus_int32 total, opus_int32 *balance, int *pulses, int *ebits, int *fine_priority, int C, int LM, ec_ctx *ec, int encode, int prev, int signalBandwidth); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/stack_alloc.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/stack_alloc.h new file mode 100644 index 000000000..ae40e2a16 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/stack_alloc.h @@ -0,0 +1,184 @@ +/* Copyright (C) 2002-2003 Jean-Marc Valin + Copyright (C) 2007-2009 Xiph.Org Foundation */ +/** + @file stack_alloc.h + @brief Temporary memory allocation on stack +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef STACK_ALLOC_H +#define STACK_ALLOC_H + +#include "opus_types.h" +#include "opus_defines.h" + +#if (!defined (VAR_ARRAYS) && !defined (USE_ALLOCA) && !defined (NONTHREADSAFE_PSEUDOSTACK)) +#error "Opus requires one of VAR_ARRAYS, USE_ALLOCA, or NONTHREADSAFE_PSEUDOSTACK be defined to select the temporary allocation mode." +#endif + +#ifdef USE_ALLOCA +# ifdef _WIN32 +# include +# else +# ifdef HAVE_ALLOCA_H +# include +# else +# include +# endif +# endif +#endif + +/** + * @def ALIGN(stack, size) + * + * Aligns the stack to a 'size' boundary + * + * @param stack Stack + * @param size New size boundary + */ + +/** + * @def PUSH(stack, size, type) + * + * Allocates 'size' elements of type 'type' on the stack + * + * @param stack Stack + * @param size Number of elements + * @param type Type of element + */ + +/** + * @def VARDECL(var) + * + * Declare variable on stack + * + * @param var Variable to declare + */ + +/** + * @def ALLOC(var, size, type) + * + * Allocate 'size' elements of 'type' on stack + * + * @param var Name of variable to allocate + * @param size Number of elements + * @param type Type of element + */ + +#if defined(VAR_ARRAYS) + +#define VARDECL(type, var) +#define ALLOC(var, size, type) type var[size] +#define SAVE_STACK +#define RESTORE_STACK +#define ALLOC_STACK +/* C99 does not allow VLAs of size zero */ +#define ALLOC_NONE 1 + +#elif defined(USE_ALLOCA) + +#define VARDECL(type, var) type *var + +# ifdef _WIN32 +# define ALLOC(var, size, type) var = ((type*)_alloca(sizeof(type)*(size))) +# else +# define ALLOC(var, size, type) var = ((type*)alloca(sizeof(type)*(size))) +# endif + +#define SAVE_STACK +#define RESTORE_STACK +#define ALLOC_STACK +#define ALLOC_NONE 0 + +#else + +#ifdef CELT_C +char *scratch_ptr=0; +char *global_stack=0; +#else +extern char *global_stack; +extern char *scratch_ptr; +#endif /* CELT_C */ + +#ifdef ENABLE_VALGRIND + +#include + +#ifdef CELT_C +char *global_stack_top=0; +#else +extern char *global_stack_top; +#endif /* CELT_C */ + +#define ALIGN(stack, size) ((stack) += ((size) - (long)(stack)) & ((size) - 1)) +#define PUSH(stack, size, type) (VALGRIND_MAKE_MEM_NOACCESS(stack, global_stack_top-stack),ALIGN((stack),sizeof(type)/sizeof(char)),VALGRIND_MAKE_MEM_UNDEFINED(stack, ((size)*sizeof(type)/sizeof(char))),(stack)+=(2*(size)*sizeof(type)/sizeof(char)),(type*)((stack)-(2*(size)*sizeof(type)/sizeof(char)))) +#define RESTORE_STACK ((global_stack = _saved_stack),VALGRIND_MAKE_MEM_NOACCESS(global_stack, global_stack_top-global_stack)) +#define ALLOC_STACK char *_saved_stack; ((global_stack = (global_stack==0) ? ((global_stack_top=opus_alloc_scratch(GLOBAL_STACK_SIZE*2)+(GLOBAL_STACK_SIZE*2))-(GLOBAL_STACK_SIZE*2)) : global_stack),VALGRIND_MAKE_MEM_NOACCESS(global_stack, global_stack_top-global_stack)); _saved_stack = global_stack; + +#else + +#define ALIGN(stack, size) ((stack) += ((size) - (long)(stack)) & ((size) - 1)) +#define PUSH(stack, size, type) (ALIGN((stack),sizeof(type)/sizeof(char)),(stack)+=(size)*(sizeof(type)/sizeof(char)),(type*)((stack)-(size)*(sizeof(type)/sizeof(char)))) +#if 0 /* Set this to 1 to instrument pseudostack usage */ +#define RESTORE_STACK (printf("%ld %s:%d\n", global_stack-scratch_ptr, __FILE__, __LINE__),global_stack = _saved_stack) +#else +#define RESTORE_STACK (global_stack = _saved_stack) +#endif +#define ALLOC_STACK char *_saved_stack; (global_stack = (global_stack==0) ? (scratch_ptr=opus_alloc_scratch(GLOBAL_STACK_SIZE)) : global_stack); _saved_stack = global_stack; + +#endif /* ENABLE_VALGRIND */ + +#include "os_support.h" +#define VARDECL(type, var) type *var +#define ALLOC(var, size, type) var = PUSH(global_stack, size, type) +#define SAVE_STACK char *_saved_stack = global_stack; +#define ALLOC_NONE 0 + +#endif /* VAR_ARRAYS */ + + +#ifdef ENABLE_VALGRIND + +#include +#define OPUS_CHECK_ARRAY(ptr, len) VALGRIND_CHECK_MEM_IS_DEFINED(ptr, len*sizeof(*ptr)) +#define OPUS_CHECK_VALUE(value) VALGRIND_CHECK_VALUE_IS_DEFINED(value) +#define OPUS_CHECK_ARRAY_COND(ptr, len) VALGRIND_CHECK_MEM_IS_DEFINED(ptr, len*sizeof(*ptr)) +#define OPUS_CHECK_VALUE_COND(value) VALGRIND_CHECK_VALUE_IS_DEFINED(value) +#define OPUS_PRINT_INT(value) do {fprintf(stderr, #value " = %d at %s:%d\n", value, __FILE__, __LINE__);}while(0) +#define OPUS_FPRINTF fprintf + +#else + +static OPUS_INLINE int _opus_false(void) {return 0;} +#define OPUS_CHECK_ARRAY(ptr, len) _opus_false() +#define OPUS_CHECK_VALUE(value) _opus_false() +#define OPUS_PRINT_INT(value) do{}while(0) +#define OPUS_FPRINTF (void) + +#endif + + +#endif /* STACK_ALLOC_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed.h new file mode 100644 index 000000000..8717d626c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed.h @@ -0,0 +1,892 @@ +/* The contents of this file was automatically generated by dump_modes.c + with arguments: 48000 960 + It contains static definitions for some pre-defined modes. */ +#include "modes.h" +#include "rate.h" + +#ifdef HAVE_ARM_NE10 +#define OVERRIDE_FFT 1 +#include "static_modes_fixed_arm_ne10.h" +#endif + +#ifndef DEF_WINDOW120 +#define DEF_WINDOW120 +static const opus_val16 window120[120] = { +2, 20, 55, 108, 178, +266, 372, 494, 635, 792, +966, 1157, 1365, 1590, 1831, +2089, 2362, 2651, 2956, 3276, +3611, 3961, 4325, 4703, 5094, +5499, 5916, 6346, 6788, 7241, +7705, 8179, 8663, 9156, 9657, +10167, 10684, 11207, 11736, 12271, +12810, 13353, 13899, 14447, 14997, +15547, 16098, 16648, 17197, 17744, +18287, 18827, 19363, 19893, 20418, +20936, 21447, 21950, 22445, 22931, +23407, 23874, 24330, 24774, 25208, +25629, 26039, 26435, 26819, 27190, +27548, 27893, 28224, 28541, 28845, +29135, 29411, 29674, 29924, 30160, +30384, 30594, 30792, 30977, 31151, +31313, 31463, 31602, 31731, 31849, +31958, 32057, 32148, 32229, 32303, +32370, 32429, 32481, 32528, 32568, +32604, 32634, 32661, 32683, 32701, +32717, 32729, 32740, 32748, 32754, +32758, 32762, 32764, 32766, 32767, +32767, 32767, 32767, 32767, 32767, +}; +#endif + +#ifndef DEF_LOGN400 +#define DEF_LOGN400 +static const opus_int16 logN400[21] = { +0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 16, 16, 16, 21, 21, 24, 29, 34, 36, }; +#endif + +#ifndef DEF_PULSE_CACHE50 +#define DEF_PULSE_CACHE50 +static const opus_int16 cache_index50[105] = { +-1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 41, 41, 41, +82, 82, 123, 164, 200, 222, 0, 0, 0, 0, 0, 0, 0, 0, 41, +41, 41, 41, 123, 123, 123, 164, 164, 240, 266, 283, 295, 41, 41, 41, +41, 41, 41, 41, 41, 123, 123, 123, 123, 240, 240, 240, 266, 266, 305, +318, 328, 336, 123, 123, 123, 123, 123, 123, 123, 123, 240, 240, 240, 240, +305, 305, 305, 318, 318, 343, 351, 358, 364, 240, 240, 240, 240, 240, 240, +240, 240, 305, 305, 305, 305, 343, 343, 343, 351, 351, 370, 376, 382, 387, +}; +static const unsigned char cache_bits50[392] = { +40, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 40, 15, 23, 28, +31, 34, 36, 38, 39, 41, 42, 43, 44, 45, 46, 47, 47, 49, 50, +51, 52, 53, 54, 55, 55, 57, 58, 59, 60, 61, 62, 63, 63, 65, +66, 67, 68, 69, 70, 71, 71, 40, 20, 33, 41, 48, 53, 57, 61, +64, 66, 69, 71, 73, 75, 76, 78, 80, 82, 85, 87, 89, 91, 92, +94, 96, 98, 101, 103, 105, 107, 108, 110, 112, 114, 117, 119, 121, 123, +124, 126, 128, 40, 23, 39, 51, 60, 67, 73, 79, 83, 87, 91, 94, +97, 100, 102, 105, 107, 111, 115, 118, 121, 124, 126, 129, 131, 135, 139, +142, 145, 148, 150, 153, 155, 159, 163, 166, 169, 172, 174, 177, 179, 35, +28, 49, 65, 78, 89, 99, 107, 114, 120, 126, 132, 136, 141, 145, 149, +153, 159, 165, 171, 176, 180, 185, 189, 192, 199, 205, 211, 216, 220, 225, +229, 232, 239, 245, 251, 21, 33, 58, 79, 97, 112, 125, 137, 148, 157, +166, 174, 182, 189, 195, 201, 207, 217, 227, 235, 243, 251, 17, 35, 63, +86, 106, 123, 139, 152, 165, 177, 187, 197, 206, 214, 222, 230, 237, 250, +25, 31, 55, 75, 91, 105, 117, 128, 138, 146, 154, 161, 168, 174, 180, +185, 190, 200, 208, 215, 222, 229, 235, 240, 245, 255, 16, 36, 65, 89, +110, 128, 144, 159, 173, 185, 196, 207, 217, 226, 234, 242, 250, 11, 41, +74, 103, 128, 151, 172, 191, 209, 225, 241, 255, 9, 43, 79, 110, 138, +163, 186, 207, 227, 246, 12, 39, 71, 99, 123, 144, 164, 182, 198, 214, +228, 241, 253, 9, 44, 81, 113, 142, 168, 192, 214, 235, 255, 7, 49, +90, 127, 160, 191, 220, 247, 6, 51, 95, 134, 170, 203, 234, 7, 47, +87, 123, 155, 184, 212, 237, 6, 52, 97, 137, 174, 208, 240, 5, 57, +106, 151, 192, 231, 5, 59, 111, 158, 202, 243, 5, 55, 103, 147, 187, +224, 5, 60, 113, 161, 206, 248, 4, 65, 122, 175, 224, 4, 67, 127, +182, 234, }; +static const unsigned char cache_caps50[168] = { +224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, +178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, +240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, 160, 160, 160, +160, 160, 160, 160, 160, 185, 185, 185, 185, 193, 193, 193, 183, 183, 172, +138, 64, 38, 240, 240, 240, 240, 240, 240, 240, 240, 207, 207, 207, 207, +204, 204, 204, 193, 193, 180, 143, 66, 40, 185, 185, 185, 185, 185, 185, +185, 185, 193, 193, 193, 193, 193, 193, 193, 183, 183, 172, 138, 65, 39, +207, 207, 207, 207, 207, 207, 207, 207, 204, 204, 204, 204, 201, 201, 201, +188, 188, 176, 141, 66, 40, 193, 193, 193, 193, 193, 193, 193, 193, 193, +193, 193, 193, 194, 194, 194, 184, 184, 173, 139, 65, 39, 204, 204, 204, +204, 204, 204, 204, 204, 201, 201, 201, 201, 198, 198, 198, 187, 187, 175, +140, 66, 40, }; +#endif + +#ifndef FFT_TWIDDLES48000_960 +#define FFT_TWIDDLES48000_960 +static const kiss_twiddle_cpx fft_twiddles48000_960[480] = { +{32767, 0}, {32766, -429}, +{32757, -858}, {32743, -1287}, +{32724, -1715}, {32698, -2143}, +{32667, -2570}, {32631, -2998}, +{32588, -3425}, {32541, -3851}, +{32488, -4277}, {32429, -4701}, +{32364, -5125}, {32295, -5548}, +{32219, -5971}, {32138, -6393}, +{32051, -6813}, {31960, -7231}, +{31863, -7650}, {31760, -8067}, +{31652, -8481}, {31539, -8895}, +{31419, -9306}, {31294, -9716}, +{31165, -10126}, {31030, -10532}, +{30889, -10937}, {30743, -11340}, +{30592, -11741}, {30436, -12141}, +{30274, -12540}, {30107, -12935}, +{29936, -13328}, {29758, -13718}, +{29577, -14107}, {29390, -14493}, +{29197, -14875}, {29000, -15257}, +{28797, -15635}, {28590, -16010}, +{28379, -16384}, {28162, -16753}, +{27940, -17119}, {27714, -17484}, +{27482, -17845}, {27246, -18205}, +{27006, -18560}, {26760, -18911}, +{26510, -19260}, {26257, -19606}, +{25997, -19947}, {25734, -20286}, +{25466, -20621}, {25194, -20952}, +{24918, -21281}, {24637, -21605}, +{24353, -21926}, {24063, -22242}, +{23770, -22555}, {23473, -22865}, +{23171, -23171}, {22866, -23472}, +{22557, -23769}, {22244, -24063}, +{21927, -24352}, {21606, -24636}, +{21282, -24917}, {20954, -25194}, +{20622, -25465}, {20288, -25733}, +{19949, -25997}, {19607, -26255}, +{19261, -26509}, {18914, -26760}, +{18561, -27004}, {18205, -27246}, +{17846, -27481}, {17485, -27713}, +{17122, -27940}, {16755, -28162}, +{16385, -28378}, {16012, -28590}, +{15636, -28797}, {15258, -28999}, +{14878, -29197}, {14494, -29389}, +{14108, -29576}, {13720, -29757}, +{13329, -29934}, {12937, -30107}, +{12540, -30274}, {12142, -30435}, +{11744, -30592}, {11342, -30743}, +{10939, -30889}, {10534, -31030}, +{10127, -31164}, {9718, -31294}, +{9307, -31418}, {8895, -31537}, +{8482, -31652}, {8067, -31759}, +{7650, -31862}, {7233, -31960}, +{6815, -32051}, {6393, -32138}, +{5973, -32219}, {5549, -32294}, +{5127, -32364}, {4703, -32429}, +{4278, -32487}, {3852, -32541}, +{3426, -32588}, {2999, -32630}, +{2572, -32667}, {2144, -32698}, +{1716, -32724}, {1287, -32742}, +{860, -32757}, {430, -32766}, +{0, -32767}, {-429, -32766}, +{-858, -32757}, {-1287, -32743}, +{-1715, -32724}, {-2143, -32698}, +{-2570, -32667}, {-2998, -32631}, +{-3425, -32588}, {-3851, -32541}, +{-4277, -32488}, {-4701, -32429}, +{-5125, -32364}, {-5548, -32295}, +{-5971, -32219}, {-6393, -32138}, +{-6813, -32051}, {-7231, -31960}, +{-7650, -31863}, {-8067, -31760}, +{-8481, -31652}, {-8895, -31539}, +{-9306, -31419}, {-9716, -31294}, +{-10126, -31165}, {-10532, -31030}, +{-10937, -30889}, {-11340, -30743}, +{-11741, -30592}, {-12141, -30436}, +{-12540, -30274}, {-12935, -30107}, +{-13328, -29936}, {-13718, -29758}, +{-14107, -29577}, {-14493, -29390}, +{-14875, -29197}, {-15257, -29000}, +{-15635, -28797}, {-16010, -28590}, +{-16384, -28379}, {-16753, -28162}, +{-17119, -27940}, {-17484, -27714}, +{-17845, -27482}, {-18205, -27246}, +{-18560, -27006}, {-18911, -26760}, +{-19260, -26510}, {-19606, -26257}, +{-19947, -25997}, {-20286, -25734}, +{-20621, -25466}, {-20952, -25194}, +{-21281, -24918}, {-21605, -24637}, +{-21926, -24353}, {-22242, -24063}, +{-22555, -23770}, {-22865, -23473}, +{-23171, -23171}, {-23472, -22866}, +{-23769, -22557}, {-24063, -22244}, +{-24352, -21927}, {-24636, -21606}, +{-24917, -21282}, {-25194, -20954}, +{-25465, -20622}, {-25733, -20288}, +{-25997, -19949}, {-26255, -19607}, +{-26509, -19261}, {-26760, -18914}, +{-27004, -18561}, {-27246, -18205}, +{-27481, -17846}, {-27713, -17485}, +{-27940, -17122}, {-28162, -16755}, +{-28378, -16385}, {-28590, -16012}, +{-28797, -15636}, {-28999, -15258}, +{-29197, -14878}, {-29389, -14494}, +{-29576, -14108}, {-29757, -13720}, +{-29934, -13329}, {-30107, -12937}, +{-30274, -12540}, {-30435, -12142}, +{-30592, -11744}, {-30743, -11342}, +{-30889, -10939}, {-31030, -10534}, +{-31164, -10127}, {-31294, -9718}, +{-31418, -9307}, {-31537, -8895}, +{-31652, -8482}, {-31759, -8067}, +{-31862, -7650}, {-31960, -7233}, +{-32051, -6815}, {-32138, -6393}, +{-32219, -5973}, {-32294, -5549}, +{-32364, -5127}, {-32429, -4703}, +{-32487, -4278}, {-32541, -3852}, +{-32588, -3426}, {-32630, -2999}, +{-32667, -2572}, {-32698, -2144}, +{-32724, -1716}, {-32742, -1287}, +{-32757, -860}, {-32766, -430}, +{-32767, 0}, {-32766, 429}, +{-32757, 858}, {-32743, 1287}, +{-32724, 1715}, {-32698, 2143}, +{-32667, 2570}, {-32631, 2998}, +{-32588, 3425}, {-32541, 3851}, +{-32488, 4277}, {-32429, 4701}, +{-32364, 5125}, {-32295, 5548}, +{-32219, 5971}, {-32138, 6393}, +{-32051, 6813}, {-31960, 7231}, +{-31863, 7650}, {-31760, 8067}, +{-31652, 8481}, {-31539, 8895}, +{-31419, 9306}, {-31294, 9716}, +{-31165, 10126}, {-31030, 10532}, +{-30889, 10937}, {-30743, 11340}, +{-30592, 11741}, {-30436, 12141}, +{-30274, 12540}, {-30107, 12935}, +{-29936, 13328}, {-29758, 13718}, +{-29577, 14107}, {-29390, 14493}, +{-29197, 14875}, {-29000, 15257}, +{-28797, 15635}, {-28590, 16010}, +{-28379, 16384}, {-28162, 16753}, +{-27940, 17119}, {-27714, 17484}, +{-27482, 17845}, {-27246, 18205}, +{-27006, 18560}, {-26760, 18911}, +{-26510, 19260}, {-26257, 19606}, +{-25997, 19947}, {-25734, 20286}, +{-25466, 20621}, {-25194, 20952}, +{-24918, 21281}, {-24637, 21605}, +{-24353, 21926}, {-24063, 22242}, +{-23770, 22555}, {-23473, 22865}, +{-23171, 23171}, {-22866, 23472}, +{-22557, 23769}, {-22244, 24063}, +{-21927, 24352}, {-21606, 24636}, +{-21282, 24917}, {-20954, 25194}, +{-20622, 25465}, {-20288, 25733}, +{-19949, 25997}, {-19607, 26255}, +{-19261, 26509}, {-18914, 26760}, +{-18561, 27004}, {-18205, 27246}, +{-17846, 27481}, {-17485, 27713}, +{-17122, 27940}, {-16755, 28162}, +{-16385, 28378}, {-16012, 28590}, +{-15636, 28797}, {-15258, 28999}, +{-14878, 29197}, {-14494, 29389}, +{-14108, 29576}, {-13720, 29757}, +{-13329, 29934}, {-12937, 30107}, +{-12540, 30274}, {-12142, 30435}, +{-11744, 30592}, {-11342, 30743}, +{-10939, 30889}, {-10534, 31030}, +{-10127, 31164}, {-9718, 31294}, +{-9307, 31418}, {-8895, 31537}, +{-8482, 31652}, {-8067, 31759}, +{-7650, 31862}, {-7233, 31960}, +{-6815, 32051}, {-6393, 32138}, +{-5973, 32219}, {-5549, 32294}, +{-5127, 32364}, {-4703, 32429}, +{-4278, 32487}, {-3852, 32541}, +{-3426, 32588}, {-2999, 32630}, +{-2572, 32667}, {-2144, 32698}, +{-1716, 32724}, {-1287, 32742}, +{-860, 32757}, {-430, 32766}, +{0, 32767}, {429, 32766}, +{858, 32757}, {1287, 32743}, +{1715, 32724}, {2143, 32698}, +{2570, 32667}, {2998, 32631}, +{3425, 32588}, {3851, 32541}, +{4277, 32488}, {4701, 32429}, +{5125, 32364}, {5548, 32295}, +{5971, 32219}, {6393, 32138}, +{6813, 32051}, {7231, 31960}, +{7650, 31863}, {8067, 31760}, +{8481, 31652}, {8895, 31539}, +{9306, 31419}, {9716, 31294}, +{10126, 31165}, {10532, 31030}, +{10937, 30889}, {11340, 30743}, +{11741, 30592}, {12141, 30436}, +{12540, 30274}, {12935, 30107}, +{13328, 29936}, {13718, 29758}, +{14107, 29577}, {14493, 29390}, +{14875, 29197}, {15257, 29000}, +{15635, 28797}, {16010, 28590}, +{16384, 28379}, {16753, 28162}, +{17119, 27940}, {17484, 27714}, +{17845, 27482}, {18205, 27246}, +{18560, 27006}, {18911, 26760}, +{19260, 26510}, {19606, 26257}, +{19947, 25997}, {20286, 25734}, +{20621, 25466}, {20952, 25194}, +{21281, 24918}, {21605, 24637}, +{21926, 24353}, {22242, 24063}, +{22555, 23770}, {22865, 23473}, +{23171, 23171}, {23472, 22866}, +{23769, 22557}, {24063, 22244}, +{24352, 21927}, {24636, 21606}, +{24917, 21282}, {25194, 20954}, +{25465, 20622}, {25733, 20288}, +{25997, 19949}, {26255, 19607}, +{26509, 19261}, {26760, 18914}, +{27004, 18561}, {27246, 18205}, +{27481, 17846}, {27713, 17485}, +{27940, 17122}, {28162, 16755}, +{28378, 16385}, {28590, 16012}, +{28797, 15636}, {28999, 15258}, +{29197, 14878}, {29389, 14494}, +{29576, 14108}, {29757, 13720}, +{29934, 13329}, {30107, 12937}, +{30274, 12540}, {30435, 12142}, +{30592, 11744}, {30743, 11342}, +{30889, 10939}, {31030, 10534}, +{31164, 10127}, {31294, 9718}, +{31418, 9307}, {31537, 8895}, +{31652, 8482}, {31759, 8067}, +{31862, 7650}, {31960, 7233}, +{32051, 6815}, {32138, 6393}, +{32219, 5973}, {32294, 5549}, +{32364, 5127}, {32429, 4703}, +{32487, 4278}, {32541, 3852}, +{32588, 3426}, {32630, 2999}, +{32667, 2572}, {32698, 2144}, +{32724, 1716}, {32742, 1287}, +{32757, 860}, {32766, 430}, +}; +#ifndef FFT_BITREV480 +#define FFT_BITREV480 +static const opus_int16 fft_bitrev480[480] = { +0, 96, 192, 288, 384, 32, 128, 224, 320, 416, 64, 160, 256, 352, 448, +8, 104, 200, 296, 392, 40, 136, 232, 328, 424, 72, 168, 264, 360, 456, +16, 112, 208, 304, 400, 48, 144, 240, 336, 432, 80, 176, 272, 368, 464, +24, 120, 216, 312, 408, 56, 152, 248, 344, 440, 88, 184, 280, 376, 472, +4, 100, 196, 292, 388, 36, 132, 228, 324, 420, 68, 164, 260, 356, 452, +12, 108, 204, 300, 396, 44, 140, 236, 332, 428, 76, 172, 268, 364, 460, +20, 116, 212, 308, 404, 52, 148, 244, 340, 436, 84, 180, 276, 372, 468, +28, 124, 220, 316, 412, 60, 156, 252, 348, 444, 92, 188, 284, 380, 476, +1, 97, 193, 289, 385, 33, 129, 225, 321, 417, 65, 161, 257, 353, 449, +9, 105, 201, 297, 393, 41, 137, 233, 329, 425, 73, 169, 265, 361, 457, +17, 113, 209, 305, 401, 49, 145, 241, 337, 433, 81, 177, 273, 369, 465, +25, 121, 217, 313, 409, 57, 153, 249, 345, 441, 89, 185, 281, 377, 473, +5, 101, 197, 293, 389, 37, 133, 229, 325, 421, 69, 165, 261, 357, 453, +13, 109, 205, 301, 397, 45, 141, 237, 333, 429, 77, 173, 269, 365, 461, +21, 117, 213, 309, 405, 53, 149, 245, 341, 437, 85, 181, 277, 373, 469, +29, 125, 221, 317, 413, 61, 157, 253, 349, 445, 93, 189, 285, 381, 477, +2, 98, 194, 290, 386, 34, 130, 226, 322, 418, 66, 162, 258, 354, 450, +10, 106, 202, 298, 394, 42, 138, 234, 330, 426, 74, 170, 266, 362, 458, +18, 114, 210, 306, 402, 50, 146, 242, 338, 434, 82, 178, 274, 370, 466, +26, 122, 218, 314, 410, 58, 154, 250, 346, 442, 90, 186, 282, 378, 474, +6, 102, 198, 294, 390, 38, 134, 230, 326, 422, 70, 166, 262, 358, 454, +14, 110, 206, 302, 398, 46, 142, 238, 334, 430, 78, 174, 270, 366, 462, +22, 118, 214, 310, 406, 54, 150, 246, 342, 438, 86, 182, 278, 374, 470, +30, 126, 222, 318, 414, 62, 158, 254, 350, 446, 94, 190, 286, 382, 478, +3, 99, 195, 291, 387, 35, 131, 227, 323, 419, 67, 163, 259, 355, 451, +11, 107, 203, 299, 395, 43, 139, 235, 331, 427, 75, 171, 267, 363, 459, +19, 115, 211, 307, 403, 51, 147, 243, 339, 435, 83, 179, 275, 371, 467, +27, 123, 219, 315, 411, 59, 155, 251, 347, 443, 91, 187, 283, 379, 475, +7, 103, 199, 295, 391, 39, 135, 231, 327, 423, 71, 167, 263, 359, 455, +15, 111, 207, 303, 399, 47, 143, 239, 335, 431, 79, 175, 271, 367, 463, +23, 119, 215, 311, 407, 55, 151, 247, 343, 439, 87, 183, 279, 375, 471, +31, 127, 223, 319, 415, 63, 159, 255, 351, 447, 95, 191, 287, 383, 479, +}; +#endif + +#ifndef FFT_BITREV240 +#define FFT_BITREV240 +static const opus_int16 fft_bitrev240[240] = { +0, 48, 96, 144, 192, 16, 64, 112, 160, 208, 32, 80, 128, 176, 224, +4, 52, 100, 148, 196, 20, 68, 116, 164, 212, 36, 84, 132, 180, 228, +8, 56, 104, 152, 200, 24, 72, 120, 168, 216, 40, 88, 136, 184, 232, +12, 60, 108, 156, 204, 28, 76, 124, 172, 220, 44, 92, 140, 188, 236, +1, 49, 97, 145, 193, 17, 65, 113, 161, 209, 33, 81, 129, 177, 225, +5, 53, 101, 149, 197, 21, 69, 117, 165, 213, 37, 85, 133, 181, 229, +9, 57, 105, 153, 201, 25, 73, 121, 169, 217, 41, 89, 137, 185, 233, +13, 61, 109, 157, 205, 29, 77, 125, 173, 221, 45, 93, 141, 189, 237, +2, 50, 98, 146, 194, 18, 66, 114, 162, 210, 34, 82, 130, 178, 226, +6, 54, 102, 150, 198, 22, 70, 118, 166, 214, 38, 86, 134, 182, 230, +10, 58, 106, 154, 202, 26, 74, 122, 170, 218, 42, 90, 138, 186, 234, +14, 62, 110, 158, 206, 30, 78, 126, 174, 222, 46, 94, 142, 190, 238, +3, 51, 99, 147, 195, 19, 67, 115, 163, 211, 35, 83, 131, 179, 227, +7, 55, 103, 151, 199, 23, 71, 119, 167, 215, 39, 87, 135, 183, 231, +11, 59, 107, 155, 203, 27, 75, 123, 171, 219, 43, 91, 139, 187, 235, +15, 63, 111, 159, 207, 31, 79, 127, 175, 223, 47, 95, 143, 191, 239, +}; +#endif + +#ifndef FFT_BITREV120 +#define FFT_BITREV120 +static const opus_int16 fft_bitrev120[120] = { +0, 24, 48, 72, 96, 8, 32, 56, 80, 104, 16, 40, 64, 88, 112, +4, 28, 52, 76, 100, 12, 36, 60, 84, 108, 20, 44, 68, 92, 116, +1, 25, 49, 73, 97, 9, 33, 57, 81, 105, 17, 41, 65, 89, 113, +5, 29, 53, 77, 101, 13, 37, 61, 85, 109, 21, 45, 69, 93, 117, +2, 26, 50, 74, 98, 10, 34, 58, 82, 106, 18, 42, 66, 90, 114, +6, 30, 54, 78, 102, 14, 38, 62, 86, 110, 22, 46, 70, 94, 118, +3, 27, 51, 75, 99, 11, 35, 59, 83, 107, 19, 43, 67, 91, 115, +7, 31, 55, 79, 103, 15, 39, 63, 87, 111, 23, 47, 71, 95, 119, +}; +#endif + +#ifndef FFT_BITREV60 +#define FFT_BITREV60 +static const opus_int16 fft_bitrev60[60] = { +0, 12, 24, 36, 48, 4, 16, 28, 40, 52, 8, 20, 32, 44, 56, +1, 13, 25, 37, 49, 5, 17, 29, 41, 53, 9, 21, 33, 45, 57, +2, 14, 26, 38, 50, 6, 18, 30, 42, 54, 10, 22, 34, 46, 58, +3, 15, 27, 39, 51, 7, 19, 31, 43, 55, 11, 23, 35, 47, 59, +}; +#endif + +#ifndef FFT_STATE48000_960_0 +#define FFT_STATE48000_960_0 +static const kiss_fft_state fft_state48000_960_0 = { +480, /* nfft */ +17476, /* scale */ +8, /* scale_shift */ +-1, /* shift */ +{5, 96, 3, 32, 4, 8, 2, 4, 4, 1, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev480, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_480, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_1 +#define FFT_STATE48000_960_1 +static const kiss_fft_state fft_state48000_960_1 = { +240, /* nfft */ +17476, /* scale */ +7, /* scale_shift */ +1, /* shift */ +{5, 48, 3, 16, 4, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev240, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_240, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_2 +#define FFT_STATE48000_960_2 +static const kiss_fft_state fft_state48000_960_2 = { +120, /* nfft */ +17476, /* scale */ +6, /* scale_shift */ +2, /* shift */ +{5, 24, 3, 8, 2, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev120, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_120, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_3 +#define FFT_STATE48000_960_3 +static const kiss_fft_state fft_state48000_960_3 = { +60, /* nfft */ +17476, /* scale */ +5, /* scale_shift */ +3, /* shift */ +{5, 12, 3, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev60, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_60, +#else +NULL, +#endif +}; +#endif + +#endif + +#ifndef MDCT_TWIDDLES960 +#define MDCT_TWIDDLES960 +static const opus_val16 mdct_twiddles960[1800] = { +32767, 32767, 32767, 32766, 32765, +32763, 32761, 32759, 32756, 32753, +32750, 32746, 32742, 32738, 32733, +32728, 32722, 32717, 32710, 32704, +32697, 32690, 32682, 32674, 32666, +32657, 32648, 32639, 32629, 32619, +32609, 32598, 32587, 32576, 32564, +32552, 32539, 32526, 32513, 32500, +32486, 32472, 32457, 32442, 32427, +32411, 32395, 32379, 32362, 32345, +32328, 32310, 32292, 32274, 32255, +32236, 32217, 32197, 32177, 32157, +32136, 32115, 32093, 32071, 32049, +32027, 32004, 31981, 31957, 31933, +31909, 31884, 31859, 31834, 31809, +31783, 31756, 31730, 31703, 31676, +31648, 31620, 31592, 31563, 31534, +31505, 31475, 31445, 31415, 31384, +31353, 31322, 31290, 31258, 31226, +31193, 31160, 31127, 31093, 31059, +31025, 30990, 30955, 30920, 30884, +30848, 30812, 30775, 30738, 30701, +30663, 30625, 30587, 30548, 30509, +30470, 30430, 30390, 30350, 30309, +30269, 30227, 30186, 30144, 30102, +30059, 30016, 29973, 29930, 29886, +29842, 29797, 29752, 29707, 29662, +29616, 29570, 29524, 29477, 29430, +29383, 29335, 29287, 29239, 29190, +29142, 29092, 29043, 28993, 28943, +28892, 28842, 28791, 28739, 28688, +28636, 28583, 28531, 28478, 28425, +28371, 28317, 28263, 28209, 28154, +28099, 28044, 27988, 27932, 27876, +27820, 27763, 27706, 27648, 27591, +27533, 27474, 27416, 27357, 27298, +27238, 27178, 27118, 27058, 26997, +26936, 26875, 26814, 26752, 26690, +26628, 26565, 26502, 26439, 26375, +26312, 26247, 26183, 26119, 26054, +25988, 25923, 25857, 25791, 25725, +25658, 25592, 25524, 25457, 25389, +25322, 25253, 25185, 25116, 25047, +24978, 24908, 24838, 24768, 24698, +24627, 24557, 24485, 24414, 24342, +24270, 24198, 24126, 24053, 23980, +23907, 23834, 23760, 23686, 23612, +23537, 23462, 23387, 23312, 23237, +23161, 23085, 23009, 22932, 22856, +22779, 22701, 22624, 22546, 22468, +22390, 22312, 22233, 22154, 22075, +21996, 21916, 21836, 21756, 21676, +21595, 21515, 21434, 21352, 21271, +21189, 21107, 21025, 20943, 20860, +20777, 20694, 20611, 20528, 20444, +20360, 20276, 20192, 20107, 20022, +19937, 19852, 19767, 19681, 19595, +19509, 19423, 19336, 19250, 19163, +19076, 18988, 18901, 18813, 18725, +18637, 18549, 18460, 18372, 18283, +18194, 18104, 18015, 17925, 17835, +17745, 17655, 17565, 17474, 17383, +17292, 17201, 17110, 17018, 16927, +16835, 16743, 16650, 16558, 16465, +16372, 16279, 16186, 16093, 15999, +15906, 15812, 15718, 15624, 15529, +15435, 15340, 15245, 15150, 15055, +14960, 14864, 14769, 14673, 14577, +14481, 14385, 14288, 14192, 14095, +13998, 13901, 13804, 13706, 13609, +13511, 13414, 13316, 13218, 13119, +13021, 12923, 12824, 12725, 12626, +12527, 12428, 12329, 12230, 12130, +12030, 11930, 11831, 11730, 11630, +11530, 11430, 11329, 11228, 11128, +11027, 10926, 10824, 10723, 10622, +10520, 10419, 10317, 10215, 10113, +10011, 9909, 9807, 9704, 9602, +9499, 9397, 9294, 9191, 9088, +8985, 8882, 8778, 8675, 8572, +8468, 8364, 8261, 8157, 8053, +7949, 7845, 7741, 7637, 7532, +7428, 7323, 7219, 7114, 7009, +6905, 6800, 6695, 6590, 6485, +6380, 6274, 6169, 6064, 5958, +5853, 5747, 5642, 5536, 5430, +5325, 5219, 5113, 5007, 4901, +4795, 4689, 4583, 4476, 4370, +4264, 4157, 4051, 3945, 3838, +3732, 3625, 3518, 3412, 3305, +3198, 3092, 2985, 2878, 2771, +2664, 2558, 2451, 2344, 2237, +2130, 2023, 1916, 1809, 1702, +1594, 1487, 1380, 1273, 1166, +1059, 952, 844, 737, 630, +523, 416, 308, 201, 94, +-13, -121, -228, -335, -442, +-550, -657, -764, -871, -978, +-1086, -1193, -1300, -1407, -1514, +-1621, -1728, -1835, -1942, -2049, +-2157, -2263, -2370, -2477, -2584, +-2691, -2798, -2905, -3012, -3118, +-3225, -3332, -3439, -3545, -3652, +-3758, -3865, -3971, -4078, -4184, +-4290, -4397, -4503, -4609, -4715, +-4821, -4927, -5033, -5139, -5245, +-5351, -5457, -5562, -5668, -5774, +-5879, -5985, -6090, -6195, -6301, +-6406, -6511, -6616, -6721, -6826, +-6931, -7036, -7140, -7245, -7349, +-7454, -7558, -7663, -7767, -7871, +-7975, -8079, -8183, -8287, -8390, +-8494, -8597, -8701, -8804, -8907, +-9011, -9114, -9217, -9319, -9422, +-9525, -9627, -9730, -9832, -9934, +-10037, -10139, -10241, -10342, -10444, +-10546, -10647, -10748, -10850, -10951, +-11052, -11153, -11253, -11354, -11455, +-11555, -11655, -11756, -11856, -11955, +-12055, -12155, -12254, -12354, -12453, +-12552, -12651, -12750, -12849, -12947, +-13046, -13144, -13242, -13340, -13438, +-13536, -13633, -13731, -13828, -13925, +-14022, -14119, -14216, -14312, -14409, +-14505, -14601, -14697, -14793, -14888, +-14984, -15079, -15174, -15269, -15364, +-15459, -15553, -15647, -15741, -15835, +-15929, -16023, -16116, -16210, -16303, +-16396, -16488, -16581, -16673, -16766, +-16858, -16949, -17041, -17133, -17224, +-17315, -17406, -17497, -17587, -17678, +-17768, -17858, -17948, -18037, -18127, +-18216, -18305, -18394, -18483, -18571, +-18659, -18747, -18835, -18923, -19010, +-19098, -19185, -19271, -19358, -19444, +-19531, -19617, -19702, -19788, -19873, +-19959, -20043, -20128, -20213, -20297, +-20381, -20465, -20549, -20632, -20715, +-20798, -20881, -20963, -21046, -21128, +-21210, -21291, -21373, -21454, -21535, +-21616, -21696, -21776, -21856, -21936, +-22016, -22095, -22174, -22253, -22331, +-22410, -22488, -22566, -22643, -22721, +-22798, -22875, -22951, -23028, -23104, +-23180, -23256, -23331, -23406, -23481, +-23556, -23630, -23704, -23778, -23852, +-23925, -23998, -24071, -24144, -24216, +-24288, -24360, -24432, -24503, -24574, +-24645, -24716, -24786, -24856, -24926, +-24995, -25064, -25133, -25202, -25270, +-25339, -25406, -25474, -25541, -25608, +-25675, -25742, -25808, -25874, -25939, +-26005, -26070, -26135, -26199, -26264, +-26327, -26391, -26455, -26518, -26581, +-26643, -26705, -26767, -26829, -26891, +-26952, -27013, -27073, -27133, -27193, +-27253, -27312, -27372, -27430, -27489, +-27547, -27605, -27663, -27720, -27777, +-27834, -27890, -27946, -28002, -28058, +-28113, -28168, -28223, -28277, -28331, +-28385, -28438, -28491, -28544, -28596, +-28649, -28701, -28752, -28803, -28854, +-28905, -28955, -29006, -29055, -29105, +-29154, -29203, -29251, -29299, -29347, +-29395, -29442, -29489, -29535, -29582, +-29628, -29673, -29719, -29764, -29808, +-29853, -29897, -29941, -29984, -30027, +-30070, -30112, -30154, -30196, -30238, +-30279, -30320, -30360, -30400, -30440, +-30480, -30519, -30558, -30596, -30635, +-30672, -30710, -30747, -30784, -30821, +-30857, -30893, -30929, -30964, -30999, +-31033, -31068, -31102, -31135, -31168, +-31201, -31234, -31266, -31298, -31330, +-31361, -31392, -31422, -31453, -31483, +-31512, -31541, -31570, -31599, -31627, +-31655, -31682, -31710, -31737, -31763, +-31789, -31815, -31841, -31866, -31891, +-31915, -31939, -31963, -31986, -32010, +-32032, -32055, -32077, -32099, -32120, +-32141, -32162, -32182, -32202, -32222, +-32241, -32260, -32279, -32297, -32315, +-32333, -32350, -32367, -32383, -32399, +-32415, -32431, -32446, -32461, -32475, +-32489, -32503, -32517, -32530, -32542, +-32555, -32567, -32579, -32590, -32601, +-32612, -32622, -32632, -32641, -32651, +-32659, -32668, -32676, -32684, -32692, +-32699, -32706, -32712, -32718, -32724, +-32729, -32734, -32739, -32743, -32747, +-32751, -32754, -32757, -32760, -32762, +-32764, -32765, -32767, -32767, -32767, +32767, 32767, 32765, 32761, 32756, +32750, 32742, 32732, 32722, 32710, +32696, 32681, 32665, 32647, 32628, +32608, 32586, 32562, 32538, 32512, +32484, 32455, 32425, 32393, 32360, +32326, 32290, 32253, 32214, 32174, +32133, 32090, 32046, 32001, 31954, +31906, 31856, 31805, 31753, 31700, +31645, 31588, 31530, 31471, 31411, +31349, 31286, 31222, 31156, 31089, +31020, 30951, 30880, 30807, 30733, +30658, 30582, 30504, 30425, 30345, +30263, 30181, 30096, 30011, 29924, +29836, 29747, 29656, 29564, 29471, +29377, 29281, 29184, 29086, 28987, +28886, 28784, 28681, 28577, 28471, +28365, 28257, 28147, 28037, 27925, +27812, 27698, 27583, 27467, 27349, +27231, 27111, 26990, 26868, 26744, +26620, 26494, 26367, 26239, 26110, +25980, 25849, 25717, 25583, 25449, +25313, 25176, 25038, 24900, 24760, +24619, 24477, 24333, 24189, 24044, +23898, 23751, 23602, 23453, 23303, +23152, 22999, 22846, 22692, 22537, +22380, 22223, 22065, 21906, 21746, +21585, 21423, 21261, 21097, 20933, +20767, 20601, 20434, 20265, 20096, +19927, 19756, 19584, 19412, 19239, +19065, 18890, 18714, 18538, 18361, +18183, 18004, 17824, 17644, 17463, +17281, 17098, 16915, 16731, 16546, +16361, 16175, 15988, 15800, 15612, +15423, 15234, 15043, 14852, 14661, +14469, 14276, 14083, 13889, 13694, +13499, 13303, 13107, 12910, 12713, +12515, 12317, 12118, 11918, 11718, +11517, 11316, 11115, 10913, 10710, +10508, 10304, 10100, 9896, 9691, +9486, 9281, 9075, 8869, 8662, +8455, 8248, 8040, 7832, 7623, +7415, 7206, 6996, 6787, 6577, +6366, 6156, 5945, 5734, 5523, +5311, 5100, 4888, 4675, 4463, +4251, 4038, 3825, 3612, 3399, +3185, 2972, 2758, 2544, 2330, +2116, 1902, 1688, 1474, 1260, +1045, 831, 617, 402, 188, +-27, -241, -456, -670, -885, +-1099, -1313, -1528, -1742, -1956, +-2170, -2384, -2598, -2811, -3025, +-3239, -3452, -3665, -3878, -4091, +-4304, -4516, -4728, -4941, -5153, +-5364, -5576, -5787, -5998, -6209, +-6419, -6629, -6839, -7049, -7258, +-7467, -7676, -7884, -8092, -8300, +-8507, -8714, -8920, -9127, -9332, +-9538, -9743, -9947, -10151, -10355, +-10558, -10761, -10963, -11165, -11367, +-11568, -11768, -11968, -12167, -12366, +-12565, -12762, -12960, -13156, -13352, +-13548, -13743, -13937, -14131, -14324, +-14517, -14709, -14900, -15091, -15281, +-15470, -15659, -15847, -16035, -16221, +-16407, -16593, -16777, -16961, -17144, +-17326, -17508, -17689, -17869, -18049, +-18227, -18405, -18582, -18758, -18934, +-19108, -19282, -19455, -19627, -19799, +-19969, -20139, -20308, -20475, -20642, +-20809, -20974, -21138, -21301, -21464, +-21626, -21786, -21946, -22105, -22263, +-22420, -22575, -22730, -22884, -23037, +-23189, -23340, -23490, -23640, -23788, +-23935, -24080, -24225, -24369, -24512, +-24654, -24795, -24934, -25073, -25211, +-25347, -25482, -25617, -25750, -25882, +-26013, -26143, -26272, -26399, -26526, +-26651, -26775, -26898, -27020, -27141, +-27260, -27379, -27496, -27612, -27727, +-27841, -27953, -28065, -28175, -28284, +-28391, -28498, -28603, -28707, -28810, +-28911, -29012, -29111, -29209, -29305, +-29401, -29495, -29587, -29679, -29769, +-29858, -29946, -30032, -30118, -30201, +-30284, -30365, -30445, -30524, -30601, +-30677, -30752, -30825, -30897, -30968, +-31038, -31106, -31172, -31238, -31302, +-31365, -31426, -31486, -31545, -31602, +-31658, -31713, -31766, -31818, -31869, +-31918, -31966, -32012, -32058, -32101, +-32144, -32185, -32224, -32262, -32299, +-32335, -32369, -32401, -32433, -32463, +-32491, -32518, -32544, -32568, -32591, +-32613, -32633, -32652, -32669, -32685, +-32700, -32713, -32724, -32735, -32744, +-32751, -32757, -32762, -32766, -32767, +32767, 32764, 32755, 32741, 32720, +32694, 32663, 32626, 32583, 32535, +32481, 32421, 32356, 32286, 32209, +32128, 32041, 31948, 31850, 31747, +31638, 31523, 31403, 31278, 31148, +31012, 30871, 30724, 30572, 30415, +30253, 30086, 29913, 29736, 29553, +29365, 29172, 28974, 28771, 28564, +28351, 28134, 27911, 27684, 27452, +27216, 26975, 26729, 26478, 26223, +25964, 25700, 25432, 25159, 24882, +24601, 24315, 24026, 23732, 23434, +23133, 22827, 22517, 22204, 21886, +21565, 21240, 20912, 20580, 20244, +19905, 19563, 19217, 18868, 18516, +18160, 17802, 17440, 17075, 16708, +16338, 15964, 15588, 15210, 14829, +14445, 14059, 13670, 13279, 12886, +12490, 12093, 11693, 11291, 10888, +10482, 10075, 9666, 9255, 8843, +8429, 8014, 7597, 7180, 6760, +6340, 5919, 5496, 5073, 4649, +4224, 3798, 3372, 2945, 2517, +2090, 1661, 1233, 804, 375, +-54, -483, -911, -1340, -1768, +-2197, -2624, -3052, -3479, -3905, +-4330, -4755, -5179, -5602, -6024, +-6445, -6865, -7284, -7702, -8118, +-8533, -8946, -9358, -9768, -10177, +-10584, -10989, -11392, -11793, -12192, +-12589, -12984, -13377, -13767, -14155, +-14541, -14924, -15305, -15683, -16058, +-16430, -16800, -17167, -17531, -17892, +-18249, -18604, -18956, -19304, -19649, +-19990, -20329, -20663, -20994, -21322, +-21646, -21966, -22282, -22595, -22904, +-23208, -23509, -23806, -24099, -24387, +-24672, -24952, -25228, -25499, -25766, +-26029, -26288, -26541, -26791, -27035, +-27275, -27511, -27741, -27967, -28188, +-28405, -28616, -28823, -29024, -29221, +-29412, -29599, -29780, -29957, -30128, +-30294, -30455, -30611, -30761, -30906, +-31046, -31181, -31310, -31434, -31552, +-31665, -31773, -31875, -31972, -32063, +-32149, -32229, -32304, -32373, -32437, +-32495, -32547, -32594, -32635, -32671, +-32701, -32726, -32745, -32758, -32766, +32767, 32754, 32717, 32658, 32577, +32473, 32348, 32200, 32029, 31837, +31624, 31388, 31131, 30853, 30553, +30232, 29891, 29530, 29148, 28746, +28324, 27883, 27423, 26944, 26447, +25931, 25398, 24847, 24279, 23695, +23095, 22478, 21846, 21199, 20538, +19863, 19174, 18472, 17757, 17030, +16291, 15541, 14781, 14010, 13230, +12441, 11643, 10837, 10024, 9204, +8377, 7545, 6708, 5866, 5020, +4171, 3319, 2464, 1608, 751, +-107, -965, -1822, -2678, -3532, +-4383, -5232, -6077, -6918, -7754, +-8585, -9409, -10228, -11039, -11843, +-12639, -13426, -14204, -14972, -15730, +-16477, -17213, -17937, -18648, -19347, +-20033, -20705, -21363, -22006, -22634, +-23246, -23843, -24423, -24986, -25533, +-26062, -26573, -27066, -27540, -27995, +-28431, -28848, -29245, -29622, -29979, +-30315, -30630, -30924, -31197, -31449, +-31679, -31887, -32074, -32239, -32381, +-32501, -32600, -32675, -32729, -32759, +}; +#endif + +static const CELTMode mode48000_960_120 = { +48000, /* Fs */ +120, /* overlap */ +21, /* nbEBands */ +21, /* effEBands */ +{27853, 0, 4096, 8192, }, /* preemph */ +eband5ms, /* eBands */ +3, /* maxLM */ +8, /* nbShortMdcts */ +120, /* shortMdctSize */ +11, /* nbAllocVectors */ +band_allocation, /* allocVectors */ +logN400, /* logN */ +window120, /* window */ +{1920, 3, {&fft_state48000_960_0, &fft_state48000_960_1, &fft_state48000_960_2, &fft_state48000_960_3, }, mdct_twiddles960}, /* mdct */ +{392, cache_index50, cache_bits50, cache_caps50}, /* cache */ +}; + +/* List of all the available modes */ +#define TOTAL_MODES 1 +static const CELTMode * const static_mode_list[TOTAL_MODES] = { +&mode48000_960_120, +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed_arm_ne10.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed_arm_ne10.h new file mode 100644 index 000000000..762309219 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_fixed_arm_ne10.h @@ -0,0 +1,388 @@ +/* The contents of this file was automatically generated by + * dump_mode_arm_ne10.c with arguments: 48000 960 + * It contains static definitions for some pre-defined modes. */ +#include + +#ifndef NE10_FFT_PARAMS48000_960 +#define NE10_FFT_PARAMS48000_960 +static const ne10_int32_t ne10_factors_480[64] = { +4, 40, 4, 30, 2, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_240[64] = { +3, 20, 4, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_120[64] = { +3, 10, 2, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_60[64] = { +2, 5, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_fft_cpx_int32_t ne10_twiddles_480[480] = { +{0,0}, {2147483647,0}, {2147483647,0}, +{2147483647,0}, {1961823921,-873460313}, {1436946998,-1595891394}, +{2147483647,0}, {1436946998,-1595891394}, {-224473265,-2135719496}, +{2147483647,0}, {663608871,-2042378339}, {-1737350854,-1262259096}, +{2147483647,0}, {-224473265,-2135719496}, {-2100555935,446487152}, +{2147483647,0}, {2100555974,-446486968}, {1961823921,-873460313}, +{1737350743,-1262259248}, {1436946998,-1595891394}, {1073741769,-1859775424}, +{663608871,-2042378339}, {224473078,-2135719516}, {-224473265,-2135719496}, +{-663609049,-2042378281}, {-1073741932,-1859775330}, {-1436947137,-1595891268}, +{-1737350854,-1262259096}, {-1961823997,-873460141}, {-2100556013,-446486785}, +{2147483647,0}, {2144540595,-112390613}, {2135719506,-224473172}, +{2121044558,-335940465}, {2100555974,-446486968}, {2074309912,-555809682}, +{2042378310,-663608960}, {2004848691,-769589332}, {1961823921,-873460313}, +{1913421927,-974937199}, {1859775377,-1073741851}, {1801031311,-1169603450}, +{1737350743,-1262259248}, {1668908218,-1351455280}, {1595891331,-1436947067}, +{1518500216,-1518500282}, {1436946998,-1595891394}, {1351455207,-1668908277}, +{1262259172,-1737350799}, {1169603371,-1801031362}, {1073741769,-1859775424}, +{974937230,-1913421912}, {873460227,-1961823959}, {769589125,-2004848771}, +{663608871,-2042378339}, {555809715,-2074309903}, {446486876,-2100555994}, +{335940246,-2121044593}, {224473078,-2135719516}, {112390647,-2144540593}, +{2147483647,0}, {2135719506,-224473172}, {2100555974,-446486968}, +{2042378310,-663608960}, {1961823921,-873460313}, {1859775377,-1073741851}, +{1737350743,-1262259248}, {1595891331,-1436947067}, {1436946998,-1595891394}, +{1262259172,-1737350799}, {1073741769,-1859775424}, {873460227,-1961823959}, +{663608871,-2042378339}, {446486876,-2100555994}, {224473078,-2135719516}, +{-94,-2147483647}, {-224473265,-2135719496}, {-446487060,-2100555955}, +{-663609049,-2042378281}, {-873460398,-1961823883}, {-1073741932,-1859775330}, +{-1262259116,-1737350839}, {-1436947137,-1595891268}, {-1595891628,-1436946738}, +{-1737350854,-1262259096}, {-1859775343,-1073741910}, {-1961823997,-873460141}, +{-2042378447,-663608538}, {-2100556013,-446486785}, {-2135719499,-224473240}, +{2147483647,0}, {2121044558,-335940465}, {2042378310,-663608960}, +{1913421927,-974937199}, {1737350743,-1262259248}, {1518500216,-1518500282}, +{1262259172,-1737350799}, {974937230,-1913421912}, {663608871,-2042378339}, +{335940246,-2121044593}, {-94,-2147483647}, {-335940431,-2121044564}, +{-663609049,-2042378281}, {-974937397,-1913421827}, {-1262259116,-1737350839}, +{-1518500258,-1518500240}, {-1737350854,-1262259096}, {-1913422071,-974936918}, +{-2042378447,-663608538}, {-2121044568,-335940406}, {-2147483647,188}, +{-2121044509,335940777}, {-2042378331,663608895}, {-1913421900,974937252}, +{-1737350633,1262259400}, {-1518499993,1518500506}, {-1262258813,1737351059}, +{-974936606,1913422229}, {-663609179,2042378239}, {-335940566,2121044542}, +{2147483647,0}, {2147299667,-28109693}, {2146747758,-56214570}, +{2145828015,-84309815}, {2144540595,-112390613}, {2142885719,-140452154}, +{2140863671,-168489630}, {2138474797,-196498235}, {2135719506,-224473172}, +{2132598271,-252409646}, {2129111626,-280302871}, {2125260168,-308148068}, +{2121044558,-335940465}, {2116465518,-363675300}, {2111523833,-391347822}, +{2106220349,-418953288}, {2100555974,-446486968}, {2094531681,-473944146}, +{2088148500,-501320115}, {2081407525,-528610186}, {2074309912,-555809682}, +{2066856885,-582913912}, {2059049696,-609918325}, {2050889698,-636818231}, +{2042378310,-663608960}, {2033516972,-690285983}, {2024307180,-716844791}, +{2014750533,-743280770}, {2004848691,-769589332}, {1994603329,-795766029}, +{1984016179,-821806435}, {1973089077,-847706028}, {1961823921,-873460313}, +{1950222618,-899064934}, {1938287127,-924515564}, {1926019520,-949807783}, +{1913421927,-974937199}, {1900496481,-999899565}, {1887245364,-1024690661}, +{1873670877,-1049306180}, {1859775377,-1073741851}, {1845561215,-1097993541}, +{1831030826,-1122057097}, {1816186632,-1145928502}, {1801031311,-1169603450}, +{1785567394,-1193077993}, {1769797456,-1216348214}, {1753724345,-1239409914}, +{1737350743,-1262259248}, {1720679456,-1284892300}, {1703713340,-1307305194}, +{1686455222,-1329494189}, {1668908218,-1351455280}, {1651075255,-1373184807}, +{1632959307,-1394679144}, {1614563642,-1415934412}, {1595891331,-1436947067}, +{1576945572,-1457713510}, {1557729613,-1478230181}, {1538246655,-1498493658}, +{1518500216,-1518500282}, {1498493590,-1538246721}, {1478230113,-1557729677}, +{1457713441,-1576945636}, {1436946998,-1595891394}, {1415934341,-1614563704}, +{1394679073,-1632959368}, {1373184735,-1651075315}, {1351455207,-1668908277}, +{1329494115,-1686455280}, {1307305120,-1703713397}, {1284892225,-1720679512}, +{1262259172,-1737350799}, {1239409837,-1753724400}, {1216348136,-1769797510}, +{1193077915,-1785567446}, {1169603371,-1801031362}, {1145928423,-1816186682}, +{1122057017,-1831030875}, {1097993571,-1845561197}, {1073741769,-1859775424}, +{1049305987,-1873670985}, {1024690635,-1887245378}, {999899482,-1900496524}, +{974937230,-1913421912}, {949807699,-1926019561}, {924515422,-1938287195}, +{899064965,-1950222603}, {873460227,-1961823959}, {847705824,-1973089164}, +{821806407,-1984016190}, {795765941,-1994603364}, {769589125,-2004848771}, +{743280682,-2014750566}, {716844642,-2024307233}, {690286016,-2033516961}, +{663608871,-2042378339}, {636818019,-2050889764}, {609918296,-2059049705}, +{582913822,-2066856911}, {555809715,-2074309903}, {528610126,-2081407540}, +{501319962,-2088148536}, {473944148,-2094531680}, {446486876,-2100555994}, +{418953102,-2106220386}, {391347792,-2111523838}, {363675176,-2116465540}, +{335940246,-2121044593}, {308148006,-2125260177}, {280302715,-2129111646}, +{252409648,-2132598271}, {224473078,-2135719516}, {196498046,-2138474814}, +{168489600,-2140863674}, {140452029,-2142885728}, {112390647,-2144540593}, +{84309753,-2145828017}, {56214412,-2146747762}, {28109695,-2147299667}, +{2147483647,0}, {2146747758,-56214570}, {2144540595,-112390613}, +{2140863671,-168489630}, {2135719506,-224473172}, {2129111626,-280302871}, +{2121044558,-335940465}, {2111523833,-391347822}, {2100555974,-446486968}, +{2088148500,-501320115}, {2074309912,-555809682}, {2059049696,-609918325}, +{2042378310,-663608960}, {2024307180,-716844791}, {2004848691,-769589332}, +{1984016179,-821806435}, {1961823921,-873460313}, {1938287127,-924515564}, +{1913421927,-974937199}, {1887245364,-1024690661}, {1859775377,-1073741851}, +{1831030826,-1122057097}, {1801031311,-1169603450}, {1769797456,-1216348214}, +{1737350743,-1262259248}, {1703713340,-1307305194}, {1668908218,-1351455280}, +{1632959307,-1394679144}, {1595891331,-1436947067}, {1557729613,-1478230181}, +{1518500216,-1518500282}, {1478230113,-1557729677}, {1436946998,-1595891394}, +{1394679073,-1632959368}, {1351455207,-1668908277}, {1307305120,-1703713397}, +{1262259172,-1737350799}, {1216348136,-1769797510}, {1169603371,-1801031362}, +{1122057017,-1831030875}, {1073741769,-1859775424}, {1024690635,-1887245378}, +{974937230,-1913421912}, {924515422,-1938287195}, {873460227,-1961823959}, +{821806407,-1984016190}, {769589125,-2004848771}, {716844642,-2024307233}, +{663608871,-2042378339}, {609918296,-2059049705}, {555809715,-2074309903}, +{501319962,-2088148536}, {446486876,-2100555994}, {391347792,-2111523838}, +{335940246,-2121044593}, {280302715,-2129111646}, {224473078,-2135719516}, +{168489600,-2140863674}, {112390647,-2144540593}, {56214412,-2146747762}, +{-94,-2147483647}, {-56214600,-2146747757}, {-112390835,-2144540584}, +{-168489787,-2140863659}, {-224473265,-2135719496}, {-280302901,-2129111622}, +{-335940431,-2121044564}, {-391347977,-2111523804}, {-446487060,-2100555955}, +{-501320144,-2088148493}, {-555809896,-2074309855}, {-609918476,-2059049651}, +{-663609049,-2042378281}, {-716844819,-2024307170}, {-769589300,-2004848703}, +{-821806581,-1984016118}, {-873460398,-1961823883}, {-924515591,-1938287114}, +{-974937397,-1913421827}, {-1024690575,-1887245411}, {-1073741932,-1859775330}, +{-1122057395,-1831030643}, {-1169603421,-1801031330}, {-1216348291,-1769797403}, +{-1262259116,-1737350839}, {-1307305268,-1703713283}, {-1351455453,-1668908078}, +{-1394679021,-1632959413}, {-1436947137,-1595891268}, {-1478230435,-1557729372}, +{-1518500258,-1518500240}, {-1557729742,-1478230045}, {-1595891628,-1436946738}, +{-1632959429,-1394679001}, {-1668908417,-1351455035}, {-1703713298,-1307305248}, +{-1737350854,-1262259096}, {-1769797708,-1216347848}, {-1801031344,-1169603400}, +{-1831030924,-1122056937}, {-1859775343,-1073741910}, {-1887245423,-1024690552}, +{-1913422071,-974936918}, {-1938287125,-924515568}, {-1961823997,-873460141}, +{-1984016324,-821806084}, {-2004848713,-769589276}, {-2024307264,-716844553}, +{-2042378447,-663608538}, {-2059049731,-609918206}, {-2074309994,-555809377}, +{-2088148499,-501320119}, {-2100556013,-446486785}, {-2111523902,-391347448}, +{-2121044568,-335940406}, {-2129111659,-280302621}, {-2135719499,-224473240}, +{-2140863681,-168489506}, {-2144540612,-112390298}, {-2146747758,-56214574}, +{2147483647,0}, {2145828015,-84309815}, {2140863671,-168489630}, +{2132598271,-252409646}, {2121044558,-335940465}, {2106220349,-418953288}, +{2088148500,-501320115}, {2066856885,-582913912}, {2042378310,-663608960}, +{2014750533,-743280770}, {1984016179,-821806435}, {1950222618,-899064934}, +{1913421927,-974937199}, {1873670877,-1049306180}, {1831030826,-1122057097}, +{1785567394,-1193077993}, {1737350743,-1262259248}, {1686455222,-1329494189}, +{1632959307,-1394679144}, {1576945572,-1457713510}, {1518500216,-1518500282}, +{1457713441,-1576945636}, {1394679073,-1632959368}, {1329494115,-1686455280}, +{1262259172,-1737350799}, {1193077915,-1785567446}, {1122057017,-1831030875}, +{1049305987,-1873670985}, {974937230,-1913421912}, {899064965,-1950222603}, +{821806407,-1984016190}, {743280682,-2014750566}, {663608871,-2042378339}, +{582913822,-2066856911}, {501319962,-2088148536}, {418953102,-2106220386}, +{335940246,-2121044593}, {252409648,-2132598271}, {168489600,-2140863674}, +{84309753,-2145828017}, {-94,-2147483647}, {-84309940,-2145828010}, +{-168489787,-2140863659}, {-252409834,-2132598249}, {-335940431,-2121044564}, +{-418953286,-2106220349}, {-501320144,-2088148493}, {-582914003,-2066856860}, +{-663609049,-2042378281}, {-743280858,-2014750501}, {-821806581,-1984016118}, +{-899065136,-1950222525}, {-974937397,-1913421827}, {-1049306374,-1873670768}, +{-1122057395,-1831030643}, {-1193078284,-1785567199}, {-1262259116,-1737350839}, +{-1329494061,-1686455323}, {-1394679021,-1632959413}, {-1457713485,-1576945595}, +{-1518500258,-1518500240}, {-1576945613,-1457713466}, {-1632959429,-1394679001}, +{-1686455338,-1329494041}, {-1737350854,-1262259096}, {-1785567498,-1193077837}, +{-1831030924,-1122056937}, {-1873671031,-1049305905}, {-1913422071,-974936918}, +{-1950222750,-899064648}, {-1984016324,-821806084}, {-2014750687,-743280354}, +{-2042378447,-663608538}, {-2066856867,-582913978}, {-2088148499,-501320119}, +{-2106220354,-418953261}, {-2121044568,-335940406}, {-2132598282,-252409555}, +{-2140863681,-168489506}, {-2145828021,-84309659}, {-2147483647,188}, +{-2145828006,84310034}, {-2140863651,168489881}, {-2132598237,252409928}, +{-2121044509,335940777}, {-2106220281,418953629}, {-2088148411,501320484}, +{-2066856765,582914339}, {-2042378331,663608895}, {-2014750557,743280706}, +{-1984016181,821806431}, {-1950222593,899064989}, {-1913421900,974937252}, +{-1873670848,1049306232}, {-1831030728,1122057257}, {-1785567289,1193078149}, +{-1737350633,1262259400}, {-1686455106,1329494336}, {-1632959185,1394679287}, +{-1576945358,1457713742}, {-1518499993,1518500506}, {-1457713209,1576945850}, +{-1394678735,1632959656}, {-1329493766,1686455555}, {-1262258813,1737351059}, +{-1193077546,1785567692}, {-1122056638,1831031107}, {-1049305599,1873671202}, +{-974936606,1913422229}, {-899064330,1950222896}, {-821805761,1984016458}, +{-743280025,2014750808}, {-663609179,2042378239}, {-582914134,2066856823}, +{-501320277,2088148461}, {-418953420,2106220322}, {-335940566,2121044542}, +{-252409716,2132598263}, {-168489668,2140863668}, {-84309821,2145828015}, +}; +static const ne10_fft_cpx_int32_t ne10_twiddles_240[240] = { +{0,0}, {2147483647,0}, {2147483647,0}, +{2147483647,0}, {1961823921,-873460313}, {1436946998,-1595891394}, +{2147483647,0}, {1436946998,-1595891394}, {-224473265,-2135719496}, +{2147483647,0}, {663608871,-2042378339}, {-1737350854,-1262259096}, +{2147483647,0}, {-224473265,-2135719496}, {-2100555935,446487152}, +{2147483647,0}, {2135719506,-224473172}, {2100555974,-446486968}, +{2042378310,-663608960}, {1961823921,-873460313}, {1859775377,-1073741851}, +{1737350743,-1262259248}, {1595891331,-1436947067}, {1436946998,-1595891394}, +{1262259172,-1737350799}, {1073741769,-1859775424}, {873460227,-1961823959}, +{663608871,-2042378339}, {446486876,-2100555994}, {224473078,-2135719516}, +{2147483647,0}, {2100555974,-446486968}, {1961823921,-873460313}, +{1737350743,-1262259248}, {1436946998,-1595891394}, {1073741769,-1859775424}, +{663608871,-2042378339}, {224473078,-2135719516}, {-224473265,-2135719496}, +{-663609049,-2042378281}, {-1073741932,-1859775330}, {-1436947137,-1595891268}, +{-1737350854,-1262259096}, {-1961823997,-873460141}, {-2100556013,-446486785}, +{2147483647,0}, {2042378310,-663608960}, {1737350743,-1262259248}, +{1262259172,-1737350799}, {663608871,-2042378339}, {-94,-2147483647}, +{-663609049,-2042378281}, {-1262259116,-1737350839}, {-1737350854,-1262259096}, +{-2042378447,-663608538}, {-2147483647,188}, {-2042378331,663608895}, +{-1737350633,1262259400}, {-1262258813,1737351059}, {-663609179,2042378239}, +{2147483647,0}, {2146747758,-56214570}, {2144540595,-112390613}, +{2140863671,-168489630}, {2135719506,-224473172}, {2129111626,-280302871}, +{2121044558,-335940465}, {2111523833,-391347822}, {2100555974,-446486968}, +{2088148500,-501320115}, {2074309912,-555809682}, {2059049696,-609918325}, +{2042378310,-663608960}, {2024307180,-716844791}, {2004848691,-769589332}, +{1984016179,-821806435}, {1961823921,-873460313}, {1938287127,-924515564}, +{1913421927,-974937199}, {1887245364,-1024690661}, {1859775377,-1073741851}, +{1831030826,-1122057097}, {1801031311,-1169603450}, {1769797456,-1216348214}, +{1737350743,-1262259248}, {1703713340,-1307305194}, {1668908218,-1351455280}, +{1632959307,-1394679144}, {1595891331,-1436947067}, {1557729613,-1478230181}, +{1518500216,-1518500282}, {1478230113,-1557729677}, {1436946998,-1595891394}, +{1394679073,-1632959368}, {1351455207,-1668908277}, {1307305120,-1703713397}, +{1262259172,-1737350799}, {1216348136,-1769797510}, {1169603371,-1801031362}, +{1122057017,-1831030875}, {1073741769,-1859775424}, {1024690635,-1887245378}, +{974937230,-1913421912}, {924515422,-1938287195}, {873460227,-1961823959}, +{821806407,-1984016190}, {769589125,-2004848771}, {716844642,-2024307233}, +{663608871,-2042378339}, {609918296,-2059049705}, {555809715,-2074309903}, +{501319962,-2088148536}, {446486876,-2100555994}, {391347792,-2111523838}, +{335940246,-2121044593}, {280302715,-2129111646}, {224473078,-2135719516}, +{168489600,-2140863674}, {112390647,-2144540593}, {56214412,-2146747762}, +{2147483647,0}, {2144540595,-112390613}, {2135719506,-224473172}, +{2121044558,-335940465}, {2100555974,-446486968}, {2074309912,-555809682}, +{2042378310,-663608960}, {2004848691,-769589332}, {1961823921,-873460313}, +{1913421927,-974937199}, {1859775377,-1073741851}, {1801031311,-1169603450}, +{1737350743,-1262259248}, {1668908218,-1351455280}, {1595891331,-1436947067}, +{1518500216,-1518500282}, {1436946998,-1595891394}, {1351455207,-1668908277}, +{1262259172,-1737350799}, {1169603371,-1801031362}, {1073741769,-1859775424}, +{974937230,-1913421912}, {873460227,-1961823959}, {769589125,-2004848771}, +{663608871,-2042378339}, {555809715,-2074309903}, {446486876,-2100555994}, +{335940246,-2121044593}, {224473078,-2135719516}, {112390647,-2144540593}, +{-94,-2147483647}, {-112390835,-2144540584}, {-224473265,-2135719496}, +{-335940431,-2121044564}, {-446487060,-2100555955}, {-555809896,-2074309855}, +{-663609049,-2042378281}, {-769589300,-2004848703}, {-873460398,-1961823883}, +{-974937397,-1913421827}, {-1073741932,-1859775330}, {-1169603421,-1801031330}, +{-1262259116,-1737350839}, {-1351455453,-1668908078}, {-1436947137,-1595891268}, +{-1518500258,-1518500240}, {-1595891628,-1436946738}, {-1668908417,-1351455035}, +{-1737350854,-1262259096}, {-1801031344,-1169603400}, {-1859775343,-1073741910}, +{-1913422071,-974936918}, {-1961823997,-873460141}, {-2004848713,-769589276}, +{-2042378447,-663608538}, {-2074309994,-555809377}, {-2100556013,-446486785}, +{-2121044568,-335940406}, {-2135719499,-224473240}, {-2144540612,-112390298}, +{2147483647,0}, {2140863671,-168489630}, {2121044558,-335940465}, +{2088148500,-501320115}, {2042378310,-663608960}, {1984016179,-821806435}, +{1913421927,-974937199}, {1831030826,-1122057097}, {1737350743,-1262259248}, +{1632959307,-1394679144}, {1518500216,-1518500282}, {1394679073,-1632959368}, +{1262259172,-1737350799}, {1122057017,-1831030875}, {974937230,-1913421912}, +{821806407,-1984016190}, {663608871,-2042378339}, {501319962,-2088148536}, +{335940246,-2121044593}, {168489600,-2140863674}, {-94,-2147483647}, +{-168489787,-2140863659}, {-335940431,-2121044564}, {-501320144,-2088148493}, +{-663609049,-2042378281}, {-821806581,-1984016118}, {-974937397,-1913421827}, +{-1122057395,-1831030643}, {-1262259116,-1737350839}, {-1394679021,-1632959413}, +{-1518500258,-1518500240}, {-1632959429,-1394679001}, {-1737350854,-1262259096}, +{-1831030924,-1122056937}, {-1913422071,-974936918}, {-1984016324,-821806084}, +{-2042378447,-663608538}, {-2088148499,-501320119}, {-2121044568,-335940406}, +{-2140863681,-168489506}, {-2147483647,188}, {-2140863651,168489881}, +{-2121044509,335940777}, {-2088148411,501320484}, {-2042378331,663608895}, +{-1984016181,821806431}, {-1913421900,974937252}, {-1831030728,1122057257}, +{-1737350633,1262259400}, {-1632959185,1394679287}, {-1518499993,1518500506}, +{-1394678735,1632959656}, {-1262258813,1737351059}, {-1122056638,1831031107}, +{-974936606,1913422229}, {-821805761,1984016458}, {-663609179,2042378239}, +{-501320277,2088148461}, {-335940566,2121044542}, {-168489668,2140863668}, +}; +static const ne10_fft_cpx_int32_t ne10_twiddles_120[120] = { +{0,0}, {2147483647,0}, {2147483647,0}, +{2147483647,0}, {1961823921,-873460313}, {1436946998,-1595891394}, +{2147483647,0}, {1436946998,-1595891394}, {-224473265,-2135719496}, +{2147483647,0}, {663608871,-2042378339}, {-1737350854,-1262259096}, +{2147483647,0}, {-224473265,-2135719496}, {-2100555935,446487152}, +{2147483647,0}, {2100555974,-446486968}, {1961823921,-873460313}, +{1737350743,-1262259248}, {1436946998,-1595891394}, {1073741769,-1859775424}, +{663608871,-2042378339}, {224473078,-2135719516}, {-224473265,-2135719496}, +{-663609049,-2042378281}, {-1073741932,-1859775330}, {-1436947137,-1595891268}, +{-1737350854,-1262259096}, {-1961823997,-873460141}, {-2100556013,-446486785}, +{2147483647,0}, {2144540595,-112390613}, {2135719506,-224473172}, +{2121044558,-335940465}, {2100555974,-446486968}, {2074309912,-555809682}, +{2042378310,-663608960}, {2004848691,-769589332}, {1961823921,-873460313}, +{1913421927,-974937199}, {1859775377,-1073741851}, {1801031311,-1169603450}, +{1737350743,-1262259248}, {1668908218,-1351455280}, {1595891331,-1436947067}, +{1518500216,-1518500282}, {1436946998,-1595891394}, {1351455207,-1668908277}, +{1262259172,-1737350799}, {1169603371,-1801031362}, {1073741769,-1859775424}, +{974937230,-1913421912}, {873460227,-1961823959}, {769589125,-2004848771}, +{663608871,-2042378339}, {555809715,-2074309903}, {446486876,-2100555994}, +{335940246,-2121044593}, {224473078,-2135719516}, {112390647,-2144540593}, +{2147483647,0}, {2135719506,-224473172}, {2100555974,-446486968}, +{2042378310,-663608960}, {1961823921,-873460313}, {1859775377,-1073741851}, +{1737350743,-1262259248}, {1595891331,-1436947067}, {1436946998,-1595891394}, +{1262259172,-1737350799}, {1073741769,-1859775424}, {873460227,-1961823959}, +{663608871,-2042378339}, {446486876,-2100555994}, {224473078,-2135719516}, +{-94,-2147483647}, {-224473265,-2135719496}, {-446487060,-2100555955}, +{-663609049,-2042378281}, {-873460398,-1961823883}, {-1073741932,-1859775330}, +{-1262259116,-1737350839}, {-1436947137,-1595891268}, {-1595891628,-1436946738}, +{-1737350854,-1262259096}, {-1859775343,-1073741910}, {-1961823997,-873460141}, +{-2042378447,-663608538}, {-2100556013,-446486785}, {-2135719499,-224473240}, +{2147483647,0}, {2121044558,-335940465}, {2042378310,-663608960}, +{1913421927,-974937199}, {1737350743,-1262259248}, {1518500216,-1518500282}, +{1262259172,-1737350799}, {974937230,-1913421912}, {663608871,-2042378339}, +{335940246,-2121044593}, {-94,-2147483647}, {-335940431,-2121044564}, +{-663609049,-2042378281}, {-974937397,-1913421827}, {-1262259116,-1737350839}, +{-1518500258,-1518500240}, {-1737350854,-1262259096}, {-1913422071,-974936918}, +{-2042378447,-663608538}, {-2121044568,-335940406}, {-2147483647,188}, +{-2121044509,335940777}, {-2042378331,663608895}, {-1913421900,974937252}, +{-1737350633,1262259400}, {-1518499993,1518500506}, {-1262258813,1737351059}, +{-974936606,1913422229}, {-663609179,2042378239}, {-335940566,2121044542}, +}; +static const ne10_fft_cpx_int32_t ne10_twiddles_60[60] = { +{0,0}, {2147483647,0}, {2147483647,0}, +{2147483647,0}, {1961823921,-873460313}, {1436946998,-1595891394}, +{2147483647,0}, {1436946998,-1595891394}, {-224473265,-2135719496}, +{2147483647,0}, {663608871,-2042378339}, {-1737350854,-1262259096}, +{2147483647,0}, {-224473265,-2135719496}, {-2100555935,446487152}, +{2147483647,0}, {2135719506,-224473172}, {2100555974,-446486968}, +{2042378310,-663608960}, {1961823921,-873460313}, {1859775377,-1073741851}, +{1737350743,-1262259248}, {1595891331,-1436947067}, {1436946998,-1595891394}, +{1262259172,-1737350799}, {1073741769,-1859775424}, {873460227,-1961823959}, +{663608871,-2042378339}, {446486876,-2100555994}, {224473078,-2135719516}, +{2147483647,0}, {2100555974,-446486968}, {1961823921,-873460313}, +{1737350743,-1262259248}, {1436946998,-1595891394}, {1073741769,-1859775424}, +{663608871,-2042378339}, {224473078,-2135719516}, {-224473265,-2135719496}, +{-663609049,-2042378281}, {-1073741932,-1859775330}, {-1436947137,-1595891268}, +{-1737350854,-1262259096}, {-1961823997,-873460141}, {-2100556013,-446486785}, +{2147483647,0}, {2042378310,-663608960}, {1737350743,-1262259248}, +{1262259172,-1737350799}, {663608871,-2042378339}, {-94,-2147483647}, +{-663609049,-2042378281}, {-1262259116,-1737350839}, {-1737350854,-1262259096}, +{-2042378447,-663608538}, {-2147483647,188}, {-2042378331,663608895}, +{-1737350633,1262259400}, {-1262258813,1737351059}, {-663609179,2042378239}, +}; +static const ne10_fft_state_int32_t ne10_fft_state_int32_t_480 = { +120, +(ne10_int32_t *)ne10_factors_480, +(ne10_fft_cpx_int32_t *)ne10_twiddles_480, +NULL, +(ne10_fft_cpx_int32_t *)&ne10_twiddles_480[120], +}; +static const arch_fft_state cfg_arch_480 = { +1, +(void *)&ne10_fft_state_int32_t_480, +}; + +static const ne10_fft_state_int32_t ne10_fft_state_int32_t_240 = { +60, +(ne10_int32_t *)ne10_factors_240, +(ne10_fft_cpx_int32_t *)ne10_twiddles_240, +NULL, +(ne10_fft_cpx_int32_t *)&ne10_twiddles_240[60], +}; +static const arch_fft_state cfg_arch_240 = { +1, +(void *)&ne10_fft_state_int32_t_240, +}; + +static const ne10_fft_state_int32_t ne10_fft_state_int32_t_120 = { +30, +(ne10_int32_t *)ne10_factors_120, +(ne10_fft_cpx_int32_t *)ne10_twiddles_120, +NULL, +(ne10_fft_cpx_int32_t *)&ne10_twiddles_120[30], +}; +static const arch_fft_state cfg_arch_120 = { +1, +(void *)&ne10_fft_state_int32_t_120, +}; + +static const ne10_fft_state_int32_t ne10_fft_state_int32_t_60 = { +15, +(ne10_int32_t *)ne10_factors_60, +(ne10_fft_cpx_int32_t *)ne10_twiddles_60, +NULL, +(ne10_fft_cpx_int32_t *)&ne10_twiddles_60[15], +}; +static const arch_fft_state cfg_arch_60 = { +1, +(void *)&ne10_fft_state_int32_t_60, +}; + +#endif /* end NE10_FFT_PARAMS48000_960 */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float.h new file mode 100644 index 000000000..e102a3839 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float.h @@ -0,0 +1,888 @@ +/* The contents of this file was automatically generated by dump_modes.c + with arguments: 48000 960 + It contains static definitions for some pre-defined modes. */ +#include "modes.h" +#include "rate.h" + +#ifdef HAVE_ARM_NE10 +#define OVERRIDE_FFT 1 +#include "static_modes_float_arm_ne10.h" +#endif + +#ifndef DEF_WINDOW120 +#define DEF_WINDOW120 +static const opus_val16 window120[120] = { +6.7286966e-05f, 0.00060551348f, 0.0016815970f, 0.0032947962f, 0.0054439943f, +0.0081276923f, 0.011344001f, 0.015090633f, 0.019364886f, 0.024163635f, +0.029483315f, 0.035319905f, 0.041668911f, 0.048525347f, 0.055883718f, +0.063737999f, 0.072081616f, 0.080907428f, 0.090207705f, 0.099974111f, +0.11019769f, 0.12086883f, 0.13197729f, 0.14351214f, 0.15546177f, +0.16781389f, 0.18055550f, 0.19367290f, 0.20715171f, 0.22097682f, +0.23513243f, 0.24960208f, 0.26436860f, 0.27941419f, 0.29472040f, +0.31026818f, 0.32603788f, 0.34200931f, 0.35816177f, 0.37447407f, +0.39092462f, 0.40749142f, 0.42415215f, 0.44088423f, 0.45766484f, +0.47447104f, 0.49127978f, 0.50806798f, 0.52481261f, 0.54149077f, +0.55807973f, 0.57455701f, 0.59090049f, 0.60708841f, 0.62309951f, +0.63891306f, 0.65450896f, 0.66986776f, 0.68497077f, 0.69980010f, +0.71433873f, 0.72857055f, 0.74248043f, 0.75605424f, 0.76927895f, +0.78214257f, 0.79463430f, 0.80674445f, 0.81846456f, 0.82978733f, +0.84070669f, 0.85121779f, 0.86131698f, 0.87100183f, 0.88027111f, +0.88912479f, 0.89756398f, 0.90559094f, 0.91320904f, 0.92042270f, +0.92723738f, 0.93365955f, 0.93969656f, 0.94535671f, 0.95064907f, +0.95558353f, 0.96017067f, 0.96442171f, 0.96834849f, 0.97196334f, +0.97527906f, 0.97830883f, 0.98106616f, 0.98356480f, 0.98581869f, +0.98784191f, 0.98964856f, 0.99125274f, 0.99266849f, 0.99390969f, +0.99499004f, 0.99592297f, 0.99672162f, 0.99739874f, 0.99796667f, +0.99843728f, 0.99882195f, 0.99913147f, 0.99937606f, 0.99956527f, +0.99970802f, 0.99981248f, 0.99988613f, 0.99993565f, 0.99996697f, +0.99998518f, 0.99999457f, 0.99999859f, 0.99999982f, 1.0000000f, +}; +#endif + +#ifndef DEF_LOGN400 +#define DEF_LOGN400 +static const opus_int16 logN400[21] = { +0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 16, 16, 16, 21, 21, 24, 29, 34, 36, }; +#endif + +#ifndef DEF_PULSE_CACHE50 +#define DEF_PULSE_CACHE50 +static const opus_int16 cache_index50[105] = { +-1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 41, 41, 41, +82, 82, 123, 164, 200, 222, 0, 0, 0, 0, 0, 0, 0, 0, 41, +41, 41, 41, 123, 123, 123, 164, 164, 240, 266, 283, 295, 41, 41, 41, +41, 41, 41, 41, 41, 123, 123, 123, 123, 240, 240, 240, 266, 266, 305, +318, 328, 336, 123, 123, 123, 123, 123, 123, 123, 123, 240, 240, 240, 240, +305, 305, 305, 318, 318, 343, 351, 358, 364, 240, 240, 240, 240, 240, 240, +240, 240, 305, 305, 305, 305, 343, 343, 343, 351, 351, 370, 376, 382, 387, +}; +static const unsigned char cache_bits50[392] = { +40, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 40, 15, 23, 28, +31, 34, 36, 38, 39, 41, 42, 43, 44, 45, 46, 47, 47, 49, 50, +51, 52, 53, 54, 55, 55, 57, 58, 59, 60, 61, 62, 63, 63, 65, +66, 67, 68, 69, 70, 71, 71, 40, 20, 33, 41, 48, 53, 57, 61, +64, 66, 69, 71, 73, 75, 76, 78, 80, 82, 85, 87, 89, 91, 92, +94, 96, 98, 101, 103, 105, 107, 108, 110, 112, 114, 117, 119, 121, 123, +124, 126, 128, 40, 23, 39, 51, 60, 67, 73, 79, 83, 87, 91, 94, +97, 100, 102, 105, 107, 111, 115, 118, 121, 124, 126, 129, 131, 135, 139, +142, 145, 148, 150, 153, 155, 159, 163, 166, 169, 172, 174, 177, 179, 35, +28, 49, 65, 78, 89, 99, 107, 114, 120, 126, 132, 136, 141, 145, 149, +153, 159, 165, 171, 176, 180, 185, 189, 192, 199, 205, 211, 216, 220, 225, +229, 232, 239, 245, 251, 21, 33, 58, 79, 97, 112, 125, 137, 148, 157, +166, 174, 182, 189, 195, 201, 207, 217, 227, 235, 243, 251, 17, 35, 63, +86, 106, 123, 139, 152, 165, 177, 187, 197, 206, 214, 222, 230, 237, 250, +25, 31, 55, 75, 91, 105, 117, 128, 138, 146, 154, 161, 168, 174, 180, +185, 190, 200, 208, 215, 222, 229, 235, 240, 245, 255, 16, 36, 65, 89, +110, 128, 144, 159, 173, 185, 196, 207, 217, 226, 234, 242, 250, 11, 41, +74, 103, 128, 151, 172, 191, 209, 225, 241, 255, 9, 43, 79, 110, 138, +163, 186, 207, 227, 246, 12, 39, 71, 99, 123, 144, 164, 182, 198, 214, +228, 241, 253, 9, 44, 81, 113, 142, 168, 192, 214, 235, 255, 7, 49, +90, 127, 160, 191, 220, 247, 6, 51, 95, 134, 170, 203, 234, 7, 47, +87, 123, 155, 184, 212, 237, 6, 52, 97, 137, 174, 208, 240, 5, 57, +106, 151, 192, 231, 5, 59, 111, 158, 202, 243, 5, 55, 103, 147, 187, +224, 5, 60, 113, 161, 206, 248, 4, 65, 122, 175, 224, 4, 67, 127, +182, 234, }; +static const unsigned char cache_caps50[168] = { +224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, +178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, +240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, 160, 160, 160, +160, 160, 160, 160, 160, 185, 185, 185, 185, 193, 193, 193, 183, 183, 172, +138, 64, 38, 240, 240, 240, 240, 240, 240, 240, 240, 207, 207, 207, 207, +204, 204, 204, 193, 193, 180, 143, 66, 40, 185, 185, 185, 185, 185, 185, +185, 185, 193, 193, 193, 193, 193, 193, 193, 183, 183, 172, 138, 65, 39, +207, 207, 207, 207, 207, 207, 207, 207, 204, 204, 204, 204, 201, 201, 201, +188, 188, 176, 141, 66, 40, 193, 193, 193, 193, 193, 193, 193, 193, 193, +193, 193, 193, 194, 194, 194, 184, 184, 173, 139, 65, 39, 204, 204, 204, +204, 204, 204, 204, 204, 201, 201, 201, 201, 198, 198, 198, 187, 187, 175, +140, 66, 40, }; +#endif + +#ifndef FFT_TWIDDLES48000_960 +#define FFT_TWIDDLES48000_960 +static const kiss_twiddle_cpx fft_twiddles48000_960[480] = { +{1.0000000f, -0.0000000f}, {0.99991433f, -0.013089596f}, +{0.99965732f, -0.026176948f}, {0.99922904f, -0.039259816f}, +{0.99862953f, -0.052335956f}, {0.99785892f, -0.065403129f}, +{0.99691733f, -0.078459096f}, {0.99580493f, -0.091501619f}, +{0.99452190f, -0.10452846f}, {0.99306846f, -0.11753740f}, +{0.99144486f, -0.13052619f}, {0.98965139f, -0.14349262f}, +{0.98768834f, -0.15643447f}, {0.98555606f, -0.16934950f}, +{0.98325491f, -0.18223553f}, {0.98078528f, -0.19509032f}, +{0.97814760f, -0.20791169f}, {0.97534232f, -0.22069744f}, +{0.97236992f, -0.23344536f}, {0.96923091f, -0.24615329f}, +{0.96592583f, -0.25881905f}, {0.96245524f, -0.27144045f}, +{0.95881973f, -0.28401534f}, {0.95501994f, -0.29654157f}, +{0.95105652f, -0.30901699f}, {0.94693013f, -0.32143947f}, +{0.94264149f, -0.33380686f}, {0.93819134f, -0.34611706f}, +{0.93358043f, -0.35836795f}, {0.92880955f, -0.37055744f}, +{0.92387953f, -0.38268343f}, {0.91879121f, -0.39474386f}, +{0.91354546f, -0.40673664f}, {0.90814317f, -0.41865974f}, +{0.90258528f, -0.43051110f}, {0.89687274f, -0.44228869f}, +{0.89100652f, -0.45399050f}, {0.88498764f, -0.46561452f}, +{0.87881711f, -0.47715876f}, {0.87249601f, -0.48862124f}, +{0.86602540f, -0.50000000f}, {0.85940641f, -0.51129309f}, +{0.85264016f, -0.52249856f}, {0.84572782f, -0.53361452f}, +{0.83867057f, -0.54463904f}, {0.83146961f, -0.55557023f}, +{0.82412619f, -0.56640624f}, {0.81664156f, -0.57714519f}, +{0.80901699f, -0.58778525f}, {0.80125381f, -0.59832460f}, +{0.79335334f, -0.60876143f}, {0.78531693f, -0.61909395f}, +{0.77714596f, -0.62932039f}, {0.76884183f, -0.63943900f}, +{0.76040597f, -0.64944805f}, {0.75183981f, -0.65934582f}, +{0.74314483f, -0.66913061f}, {0.73432251f, -0.67880075f}, +{0.72537437f, -0.68835458f}, {0.71630194f, -0.69779046f}, +{0.70710678f, -0.70710678f}, {0.69779046f, -0.71630194f}, +{0.68835458f, -0.72537437f}, {0.67880075f, -0.73432251f}, +{0.66913061f, -0.74314483f}, {0.65934582f, -0.75183981f}, +{0.64944805f, -0.76040597f}, {0.63943900f, -0.76884183f}, +{0.62932039f, -0.77714596f}, {0.61909395f, -0.78531693f}, +{0.60876143f, -0.79335334f}, {0.59832460f, -0.80125381f}, +{0.58778525f, -0.80901699f}, {0.57714519f, -0.81664156f}, +{0.56640624f, -0.82412619f}, {0.55557023f, -0.83146961f}, +{0.54463904f, -0.83867057f}, {0.53361452f, -0.84572782f}, +{0.52249856f, -0.85264016f}, {0.51129309f, -0.85940641f}, +{0.50000000f, -0.86602540f}, {0.48862124f, -0.87249601f}, +{0.47715876f, -0.87881711f}, {0.46561452f, -0.88498764f}, +{0.45399050f, -0.89100652f}, {0.44228869f, -0.89687274f}, +{0.43051110f, -0.90258528f}, {0.41865974f, -0.90814317f}, +{0.40673664f, -0.91354546f}, {0.39474386f, -0.91879121f}, +{0.38268343f, -0.92387953f}, {0.37055744f, -0.92880955f}, +{0.35836795f, -0.93358043f}, {0.34611706f, -0.93819134f}, +{0.33380686f, -0.94264149f}, {0.32143947f, -0.94693013f}, +{0.30901699f, -0.95105652f}, {0.29654157f, -0.95501994f}, +{0.28401534f, -0.95881973f}, {0.27144045f, -0.96245524f}, +{0.25881905f, -0.96592583f}, {0.24615329f, -0.96923091f}, +{0.23344536f, -0.97236992f}, {0.22069744f, -0.97534232f}, +{0.20791169f, -0.97814760f}, {0.19509032f, -0.98078528f}, +{0.18223553f, -0.98325491f}, {0.16934950f, -0.98555606f}, +{0.15643447f, -0.98768834f}, {0.14349262f, -0.98965139f}, +{0.13052619f, -0.99144486f}, {0.11753740f, -0.99306846f}, +{0.10452846f, -0.99452190f}, {0.091501619f, -0.99580493f}, +{0.078459096f, -0.99691733f}, {0.065403129f, -0.99785892f}, +{0.052335956f, -0.99862953f}, {0.039259816f, -0.99922904f}, +{0.026176948f, -0.99965732f}, {0.013089596f, -0.99991433f}, +{6.1230318e-17f, -1.0000000f}, {-0.013089596f, -0.99991433f}, +{-0.026176948f, -0.99965732f}, {-0.039259816f, -0.99922904f}, +{-0.052335956f, -0.99862953f}, {-0.065403129f, -0.99785892f}, +{-0.078459096f, -0.99691733f}, {-0.091501619f, -0.99580493f}, +{-0.10452846f, -0.99452190f}, {-0.11753740f, -0.99306846f}, +{-0.13052619f, -0.99144486f}, {-0.14349262f, -0.98965139f}, +{-0.15643447f, -0.98768834f}, {-0.16934950f, -0.98555606f}, +{-0.18223553f, -0.98325491f}, {-0.19509032f, -0.98078528f}, +{-0.20791169f, -0.97814760f}, {-0.22069744f, -0.97534232f}, +{-0.23344536f, -0.97236992f}, {-0.24615329f, -0.96923091f}, +{-0.25881905f, -0.96592583f}, {-0.27144045f, -0.96245524f}, +{-0.28401534f, -0.95881973f}, {-0.29654157f, -0.95501994f}, +{-0.30901699f, -0.95105652f}, {-0.32143947f, -0.94693013f}, +{-0.33380686f, -0.94264149f}, {-0.34611706f, -0.93819134f}, +{-0.35836795f, -0.93358043f}, {-0.37055744f, -0.92880955f}, +{-0.38268343f, -0.92387953f}, {-0.39474386f, -0.91879121f}, +{-0.40673664f, -0.91354546f}, {-0.41865974f, -0.90814317f}, +{-0.43051110f, -0.90258528f}, {-0.44228869f, -0.89687274f}, +{-0.45399050f, -0.89100652f}, {-0.46561452f, -0.88498764f}, +{-0.47715876f, -0.87881711f}, {-0.48862124f, -0.87249601f}, +{-0.50000000f, -0.86602540f}, {-0.51129309f, -0.85940641f}, +{-0.52249856f, -0.85264016f}, {-0.53361452f, -0.84572782f}, +{-0.54463904f, -0.83867057f}, {-0.55557023f, -0.83146961f}, +{-0.56640624f, -0.82412619f}, {-0.57714519f, -0.81664156f}, +{-0.58778525f, -0.80901699f}, {-0.59832460f, -0.80125381f}, +{-0.60876143f, -0.79335334f}, {-0.61909395f, -0.78531693f}, +{-0.62932039f, -0.77714596f}, {-0.63943900f, -0.76884183f}, +{-0.64944805f, -0.76040597f}, {-0.65934582f, -0.75183981f}, +{-0.66913061f, -0.74314483f}, {-0.67880075f, -0.73432251f}, +{-0.68835458f, -0.72537437f}, {-0.69779046f, -0.71630194f}, +{-0.70710678f, -0.70710678f}, {-0.71630194f, -0.69779046f}, +{-0.72537437f, -0.68835458f}, {-0.73432251f, -0.67880075f}, +{-0.74314483f, -0.66913061f}, {-0.75183981f, -0.65934582f}, +{-0.76040597f, -0.64944805f}, {-0.76884183f, -0.63943900f}, +{-0.77714596f, -0.62932039f}, {-0.78531693f, -0.61909395f}, +{-0.79335334f, -0.60876143f}, {-0.80125381f, -0.59832460f}, +{-0.80901699f, -0.58778525f}, {-0.81664156f, -0.57714519f}, +{-0.82412619f, -0.56640624f}, {-0.83146961f, -0.55557023f}, +{-0.83867057f, -0.54463904f}, {-0.84572782f, -0.53361452f}, +{-0.85264016f, -0.52249856f}, {-0.85940641f, -0.51129309f}, +{-0.86602540f, -0.50000000f}, {-0.87249601f, -0.48862124f}, +{-0.87881711f, -0.47715876f}, {-0.88498764f, -0.46561452f}, +{-0.89100652f, -0.45399050f}, {-0.89687274f, -0.44228869f}, +{-0.90258528f, -0.43051110f}, {-0.90814317f, -0.41865974f}, +{-0.91354546f, -0.40673664f}, {-0.91879121f, -0.39474386f}, +{-0.92387953f, -0.38268343f}, {-0.92880955f, -0.37055744f}, +{-0.93358043f, -0.35836795f}, {-0.93819134f, -0.34611706f}, +{-0.94264149f, -0.33380686f}, {-0.94693013f, -0.32143947f}, +{-0.95105652f, -0.30901699f}, {-0.95501994f, -0.29654157f}, +{-0.95881973f, -0.28401534f}, {-0.96245524f, -0.27144045f}, +{-0.96592583f, -0.25881905f}, {-0.96923091f, -0.24615329f}, +{-0.97236992f, -0.23344536f}, {-0.97534232f, -0.22069744f}, +{-0.97814760f, -0.20791169f}, {-0.98078528f, -0.19509032f}, +{-0.98325491f, -0.18223553f}, {-0.98555606f, -0.16934950f}, +{-0.98768834f, -0.15643447f}, {-0.98965139f, -0.14349262f}, +{-0.99144486f, -0.13052619f}, {-0.99306846f, -0.11753740f}, +{-0.99452190f, -0.10452846f}, {-0.99580493f, -0.091501619f}, +{-0.99691733f, -0.078459096f}, {-0.99785892f, -0.065403129f}, +{-0.99862953f, -0.052335956f}, {-0.99922904f, -0.039259816f}, +{-0.99965732f, -0.026176948f}, {-0.99991433f, -0.013089596f}, +{-1.0000000f, -1.2246064e-16f}, {-0.99991433f, 0.013089596f}, +{-0.99965732f, 0.026176948f}, {-0.99922904f, 0.039259816f}, +{-0.99862953f, 0.052335956f}, {-0.99785892f, 0.065403129f}, +{-0.99691733f, 0.078459096f}, {-0.99580493f, 0.091501619f}, +{-0.99452190f, 0.10452846f}, {-0.99306846f, 0.11753740f}, +{-0.99144486f, 0.13052619f}, {-0.98965139f, 0.14349262f}, +{-0.98768834f, 0.15643447f}, {-0.98555606f, 0.16934950f}, +{-0.98325491f, 0.18223553f}, {-0.98078528f, 0.19509032f}, +{-0.97814760f, 0.20791169f}, {-0.97534232f, 0.22069744f}, +{-0.97236992f, 0.23344536f}, {-0.96923091f, 0.24615329f}, +{-0.96592583f, 0.25881905f}, {-0.96245524f, 0.27144045f}, +{-0.95881973f, 0.28401534f}, {-0.95501994f, 0.29654157f}, +{-0.95105652f, 0.30901699f}, {-0.94693013f, 0.32143947f}, +{-0.94264149f, 0.33380686f}, {-0.93819134f, 0.34611706f}, +{-0.93358043f, 0.35836795f}, {-0.92880955f, 0.37055744f}, +{-0.92387953f, 0.38268343f}, {-0.91879121f, 0.39474386f}, +{-0.91354546f, 0.40673664f}, {-0.90814317f, 0.41865974f}, +{-0.90258528f, 0.43051110f}, {-0.89687274f, 0.44228869f}, +{-0.89100652f, 0.45399050f}, {-0.88498764f, 0.46561452f}, +{-0.87881711f, 0.47715876f}, {-0.87249601f, 0.48862124f}, +{-0.86602540f, 0.50000000f}, {-0.85940641f, 0.51129309f}, +{-0.85264016f, 0.52249856f}, {-0.84572782f, 0.53361452f}, +{-0.83867057f, 0.54463904f}, {-0.83146961f, 0.55557023f}, +{-0.82412619f, 0.56640624f}, {-0.81664156f, 0.57714519f}, +{-0.80901699f, 0.58778525f}, {-0.80125381f, 0.59832460f}, +{-0.79335334f, 0.60876143f}, {-0.78531693f, 0.61909395f}, +{-0.77714596f, 0.62932039f}, {-0.76884183f, 0.63943900f}, +{-0.76040597f, 0.64944805f}, {-0.75183981f, 0.65934582f}, +{-0.74314483f, 0.66913061f}, {-0.73432251f, 0.67880075f}, +{-0.72537437f, 0.68835458f}, {-0.71630194f, 0.69779046f}, +{-0.70710678f, 0.70710678f}, {-0.69779046f, 0.71630194f}, +{-0.68835458f, 0.72537437f}, {-0.67880075f, 0.73432251f}, +{-0.66913061f, 0.74314483f}, {-0.65934582f, 0.75183981f}, +{-0.64944805f, 0.76040597f}, {-0.63943900f, 0.76884183f}, +{-0.62932039f, 0.77714596f}, {-0.61909395f, 0.78531693f}, +{-0.60876143f, 0.79335334f}, {-0.59832460f, 0.80125381f}, +{-0.58778525f, 0.80901699f}, {-0.57714519f, 0.81664156f}, +{-0.56640624f, 0.82412619f}, {-0.55557023f, 0.83146961f}, +{-0.54463904f, 0.83867057f}, {-0.53361452f, 0.84572782f}, +{-0.52249856f, 0.85264016f}, {-0.51129309f, 0.85940641f}, +{-0.50000000f, 0.86602540f}, {-0.48862124f, 0.87249601f}, +{-0.47715876f, 0.87881711f}, {-0.46561452f, 0.88498764f}, +{-0.45399050f, 0.89100652f}, {-0.44228869f, 0.89687274f}, +{-0.43051110f, 0.90258528f}, {-0.41865974f, 0.90814317f}, +{-0.40673664f, 0.91354546f}, {-0.39474386f, 0.91879121f}, +{-0.38268343f, 0.92387953f}, {-0.37055744f, 0.92880955f}, +{-0.35836795f, 0.93358043f}, {-0.34611706f, 0.93819134f}, +{-0.33380686f, 0.94264149f}, {-0.32143947f, 0.94693013f}, +{-0.30901699f, 0.95105652f}, {-0.29654157f, 0.95501994f}, +{-0.28401534f, 0.95881973f}, {-0.27144045f, 0.96245524f}, +{-0.25881905f, 0.96592583f}, {-0.24615329f, 0.96923091f}, +{-0.23344536f, 0.97236992f}, {-0.22069744f, 0.97534232f}, +{-0.20791169f, 0.97814760f}, {-0.19509032f, 0.98078528f}, +{-0.18223553f, 0.98325491f}, {-0.16934950f, 0.98555606f}, +{-0.15643447f, 0.98768834f}, {-0.14349262f, 0.98965139f}, +{-0.13052619f, 0.99144486f}, {-0.11753740f, 0.99306846f}, +{-0.10452846f, 0.99452190f}, {-0.091501619f, 0.99580493f}, +{-0.078459096f, 0.99691733f}, {-0.065403129f, 0.99785892f}, +{-0.052335956f, 0.99862953f}, {-0.039259816f, 0.99922904f}, +{-0.026176948f, 0.99965732f}, {-0.013089596f, 0.99991433f}, +{-1.8369095e-16f, 1.0000000f}, {0.013089596f, 0.99991433f}, +{0.026176948f, 0.99965732f}, {0.039259816f, 0.99922904f}, +{0.052335956f, 0.99862953f}, {0.065403129f, 0.99785892f}, +{0.078459096f, 0.99691733f}, {0.091501619f, 0.99580493f}, +{0.10452846f, 0.99452190f}, {0.11753740f, 0.99306846f}, +{0.13052619f, 0.99144486f}, {0.14349262f, 0.98965139f}, +{0.15643447f, 0.98768834f}, {0.16934950f, 0.98555606f}, +{0.18223553f, 0.98325491f}, {0.19509032f, 0.98078528f}, +{0.20791169f, 0.97814760f}, {0.22069744f, 0.97534232f}, +{0.23344536f, 0.97236992f}, {0.24615329f, 0.96923091f}, +{0.25881905f, 0.96592583f}, {0.27144045f, 0.96245524f}, +{0.28401534f, 0.95881973f}, {0.29654157f, 0.95501994f}, +{0.30901699f, 0.95105652f}, {0.32143947f, 0.94693013f}, +{0.33380686f, 0.94264149f}, {0.34611706f, 0.93819134f}, +{0.35836795f, 0.93358043f}, {0.37055744f, 0.92880955f}, +{0.38268343f, 0.92387953f}, {0.39474386f, 0.91879121f}, +{0.40673664f, 0.91354546f}, {0.41865974f, 0.90814317f}, +{0.43051110f, 0.90258528f}, {0.44228869f, 0.89687274f}, +{0.45399050f, 0.89100652f}, {0.46561452f, 0.88498764f}, +{0.47715876f, 0.87881711f}, {0.48862124f, 0.87249601f}, +{0.50000000f, 0.86602540f}, {0.51129309f, 0.85940641f}, +{0.52249856f, 0.85264016f}, {0.53361452f, 0.84572782f}, +{0.54463904f, 0.83867057f}, {0.55557023f, 0.83146961f}, +{0.56640624f, 0.82412619f}, {0.57714519f, 0.81664156f}, +{0.58778525f, 0.80901699f}, {0.59832460f, 0.80125381f}, +{0.60876143f, 0.79335334f}, {0.61909395f, 0.78531693f}, +{0.62932039f, 0.77714596f}, {0.63943900f, 0.76884183f}, +{0.64944805f, 0.76040597f}, {0.65934582f, 0.75183981f}, +{0.66913061f, 0.74314483f}, {0.67880075f, 0.73432251f}, +{0.68835458f, 0.72537437f}, {0.69779046f, 0.71630194f}, +{0.70710678f, 0.70710678f}, {0.71630194f, 0.69779046f}, +{0.72537437f, 0.68835458f}, {0.73432251f, 0.67880075f}, +{0.74314483f, 0.66913061f}, {0.75183981f, 0.65934582f}, +{0.76040597f, 0.64944805f}, {0.76884183f, 0.63943900f}, +{0.77714596f, 0.62932039f}, {0.78531693f, 0.61909395f}, +{0.79335334f, 0.60876143f}, {0.80125381f, 0.59832460f}, +{0.80901699f, 0.58778525f}, {0.81664156f, 0.57714519f}, +{0.82412619f, 0.56640624f}, {0.83146961f, 0.55557023f}, +{0.83867057f, 0.54463904f}, {0.84572782f, 0.53361452f}, +{0.85264016f, 0.52249856f}, {0.85940641f, 0.51129309f}, +{0.86602540f, 0.50000000f}, {0.87249601f, 0.48862124f}, +{0.87881711f, 0.47715876f}, {0.88498764f, 0.46561452f}, +{0.89100652f, 0.45399050f}, {0.89687274f, 0.44228869f}, +{0.90258528f, 0.43051110f}, {0.90814317f, 0.41865974f}, +{0.91354546f, 0.40673664f}, {0.91879121f, 0.39474386f}, +{0.92387953f, 0.38268343f}, {0.92880955f, 0.37055744f}, +{0.93358043f, 0.35836795f}, {0.93819134f, 0.34611706f}, +{0.94264149f, 0.33380686f}, {0.94693013f, 0.32143947f}, +{0.95105652f, 0.30901699f}, {0.95501994f, 0.29654157f}, +{0.95881973f, 0.28401534f}, {0.96245524f, 0.27144045f}, +{0.96592583f, 0.25881905f}, {0.96923091f, 0.24615329f}, +{0.97236992f, 0.23344536f}, {0.97534232f, 0.22069744f}, +{0.97814760f, 0.20791169f}, {0.98078528f, 0.19509032f}, +{0.98325491f, 0.18223553f}, {0.98555606f, 0.16934950f}, +{0.98768834f, 0.15643447f}, {0.98965139f, 0.14349262f}, +{0.99144486f, 0.13052619f}, {0.99306846f, 0.11753740f}, +{0.99452190f, 0.10452846f}, {0.99580493f, 0.091501619f}, +{0.99691733f, 0.078459096f}, {0.99785892f, 0.065403129f}, +{0.99862953f, 0.052335956f}, {0.99922904f, 0.039259816f}, +{0.99965732f, 0.026176948f}, {0.99991433f, 0.013089596f}, +}; +#ifndef FFT_BITREV480 +#define FFT_BITREV480 +static const opus_int16 fft_bitrev480[480] = { +0, 96, 192, 288, 384, 32, 128, 224, 320, 416, 64, 160, 256, 352, 448, +8, 104, 200, 296, 392, 40, 136, 232, 328, 424, 72, 168, 264, 360, 456, +16, 112, 208, 304, 400, 48, 144, 240, 336, 432, 80, 176, 272, 368, 464, +24, 120, 216, 312, 408, 56, 152, 248, 344, 440, 88, 184, 280, 376, 472, +4, 100, 196, 292, 388, 36, 132, 228, 324, 420, 68, 164, 260, 356, 452, +12, 108, 204, 300, 396, 44, 140, 236, 332, 428, 76, 172, 268, 364, 460, +20, 116, 212, 308, 404, 52, 148, 244, 340, 436, 84, 180, 276, 372, 468, +28, 124, 220, 316, 412, 60, 156, 252, 348, 444, 92, 188, 284, 380, 476, +1, 97, 193, 289, 385, 33, 129, 225, 321, 417, 65, 161, 257, 353, 449, +9, 105, 201, 297, 393, 41, 137, 233, 329, 425, 73, 169, 265, 361, 457, +17, 113, 209, 305, 401, 49, 145, 241, 337, 433, 81, 177, 273, 369, 465, +25, 121, 217, 313, 409, 57, 153, 249, 345, 441, 89, 185, 281, 377, 473, +5, 101, 197, 293, 389, 37, 133, 229, 325, 421, 69, 165, 261, 357, 453, +13, 109, 205, 301, 397, 45, 141, 237, 333, 429, 77, 173, 269, 365, 461, +21, 117, 213, 309, 405, 53, 149, 245, 341, 437, 85, 181, 277, 373, 469, +29, 125, 221, 317, 413, 61, 157, 253, 349, 445, 93, 189, 285, 381, 477, +2, 98, 194, 290, 386, 34, 130, 226, 322, 418, 66, 162, 258, 354, 450, +10, 106, 202, 298, 394, 42, 138, 234, 330, 426, 74, 170, 266, 362, 458, +18, 114, 210, 306, 402, 50, 146, 242, 338, 434, 82, 178, 274, 370, 466, +26, 122, 218, 314, 410, 58, 154, 250, 346, 442, 90, 186, 282, 378, 474, +6, 102, 198, 294, 390, 38, 134, 230, 326, 422, 70, 166, 262, 358, 454, +14, 110, 206, 302, 398, 46, 142, 238, 334, 430, 78, 174, 270, 366, 462, +22, 118, 214, 310, 406, 54, 150, 246, 342, 438, 86, 182, 278, 374, 470, +30, 126, 222, 318, 414, 62, 158, 254, 350, 446, 94, 190, 286, 382, 478, +3, 99, 195, 291, 387, 35, 131, 227, 323, 419, 67, 163, 259, 355, 451, +11, 107, 203, 299, 395, 43, 139, 235, 331, 427, 75, 171, 267, 363, 459, +19, 115, 211, 307, 403, 51, 147, 243, 339, 435, 83, 179, 275, 371, 467, +27, 123, 219, 315, 411, 59, 155, 251, 347, 443, 91, 187, 283, 379, 475, +7, 103, 199, 295, 391, 39, 135, 231, 327, 423, 71, 167, 263, 359, 455, +15, 111, 207, 303, 399, 47, 143, 239, 335, 431, 79, 175, 271, 367, 463, +23, 119, 215, 311, 407, 55, 151, 247, 343, 439, 87, 183, 279, 375, 471, +31, 127, 223, 319, 415, 63, 159, 255, 351, 447, 95, 191, 287, 383, 479, +}; +#endif + +#ifndef FFT_BITREV240 +#define FFT_BITREV240 +static const opus_int16 fft_bitrev240[240] = { +0, 48, 96, 144, 192, 16, 64, 112, 160, 208, 32, 80, 128, 176, 224, +4, 52, 100, 148, 196, 20, 68, 116, 164, 212, 36, 84, 132, 180, 228, +8, 56, 104, 152, 200, 24, 72, 120, 168, 216, 40, 88, 136, 184, 232, +12, 60, 108, 156, 204, 28, 76, 124, 172, 220, 44, 92, 140, 188, 236, +1, 49, 97, 145, 193, 17, 65, 113, 161, 209, 33, 81, 129, 177, 225, +5, 53, 101, 149, 197, 21, 69, 117, 165, 213, 37, 85, 133, 181, 229, +9, 57, 105, 153, 201, 25, 73, 121, 169, 217, 41, 89, 137, 185, 233, +13, 61, 109, 157, 205, 29, 77, 125, 173, 221, 45, 93, 141, 189, 237, +2, 50, 98, 146, 194, 18, 66, 114, 162, 210, 34, 82, 130, 178, 226, +6, 54, 102, 150, 198, 22, 70, 118, 166, 214, 38, 86, 134, 182, 230, +10, 58, 106, 154, 202, 26, 74, 122, 170, 218, 42, 90, 138, 186, 234, +14, 62, 110, 158, 206, 30, 78, 126, 174, 222, 46, 94, 142, 190, 238, +3, 51, 99, 147, 195, 19, 67, 115, 163, 211, 35, 83, 131, 179, 227, +7, 55, 103, 151, 199, 23, 71, 119, 167, 215, 39, 87, 135, 183, 231, +11, 59, 107, 155, 203, 27, 75, 123, 171, 219, 43, 91, 139, 187, 235, +15, 63, 111, 159, 207, 31, 79, 127, 175, 223, 47, 95, 143, 191, 239, +}; +#endif + +#ifndef FFT_BITREV120 +#define FFT_BITREV120 +static const opus_int16 fft_bitrev120[120] = { +0, 24, 48, 72, 96, 8, 32, 56, 80, 104, 16, 40, 64, 88, 112, +4, 28, 52, 76, 100, 12, 36, 60, 84, 108, 20, 44, 68, 92, 116, +1, 25, 49, 73, 97, 9, 33, 57, 81, 105, 17, 41, 65, 89, 113, +5, 29, 53, 77, 101, 13, 37, 61, 85, 109, 21, 45, 69, 93, 117, +2, 26, 50, 74, 98, 10, 34, 58, 82, 106, 18, 42, 66, 90, 114, +6, 30, 54, 78, 102, 14, 38, 62, 86, 110, 22, 46, 70, 94, 118, +3, 27, 51, 75, 99, 11, 35, 59, 83, 107, 19, 43, 67, 91, 115, +7, 31, 55, 79, 103, 15, 39, 63, 87, 111, 23, 47, 71, 95, 119, +}; +#endif + +#ifndef FFT_BITREV60 +#define FFT_BITREV60 +static const opus_int16 fft_bitrev60[60] = { +0, 12, 24, 36, 48, 4, 16, 28, 40, 52, 8, 20, 32, 44, 56, +1, 13, 25, 37, 49, 5, 17, 29, 41, 53, 9, 21, 33, 45, 57, +2, 14, 26, 38, 50, 6, 18, 30, 42, 54, 10, 22, 34, 46, 58, +3, 15, 27, 39, 51, 7, 19, 31, 43, 55, 11, 23, 35, 47, 59, +}; +#endif + +#ifndef FFT_STATE48000_960_0 +#define FFT_STATE48000_960_0 +static const kiss_fft_state fft_state48000_960_0 = { +480, /* nfft */ +0.002083333f, /* scale */ +-1, /* shift */ +{5, 96, 3, 32, 4, 8, 2, 4, 4, 1, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev480, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_480, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_1 +#define FFT_STATE48000_960_1 +static const kiss_fft_state fft_state48000_960_1 = { +240, /* nfft */ +0.004166667f, /* scale */ +1, /* shift */ +{5, 48, 3, 16, 4, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev240, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_240, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_2 +#define FFT_STATE48000_960_2 +static const kiss_fft_state fft_state48000_960_2 = { +120, /* nfft */ +0.008333333f, /* scale */ +2, /* shift */ +{5, 24, 3, 8, 2, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev120, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_120, +#else +NULL, +#endif +}; +#endif + +#ifndef FFT_STATE48000_960_3 +#define FFT_STATE48000_960_3 +static const kiss_fft_state fft_state48000_960_3 = { +60, /* nfft */ +0.016666667f, /* scale */ +3, /* shift */ +{5, 12, 3, 4, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, /* factors */ +fft_bitrev60, /* bitrev */ +fft_twiddles48000_960, /* bitrev */ +#ifdef OVERRIDE_FFT +(arch_fft_state *)&cfg_arch_60, +#else +NULL, +#endif +}; +#endif + +#endif + +#ifndef MDCT_TWIDDLES960 +#define MDCT_TWIDDLES960 +static const opus_val16 mdct_twiddles960[1800] = { +0.99999994f, 0.99999321f, 0.99997580f, 0.99994773f, 0.99990886f, +0.99985933f, 0.99979913f, 0.99972820f, 0.99964654f, 0.99955416f, +0.99945110f, 0.99933738f, 0.99921292f, 0.99907774f, 0.99893188f, +0.99877530f, 0.99860805f, 0.99843007f, 0.99824142f, 0.99804211f, +0.99783206f, 0.99761140f, 0.99737996f, 0.99713790f, 0.99688518f, +0.99662173f, 0.99634761f, 0.99606287f, 0.99576741f, 0.99546129f, +0.99514455f, 0.99481714f, 0.99447906f, 0.99413031f, 0.99377096f, +0.99340093f, 0.99302030f, 0.99262899f, 0.99222708f, 0.99181455f, +0.99139136f, 0.99095762f, 0.99051321f, 0.99005818f, 0.98959261f, +0.98911643f, 0.98862964f, 0.98813224f, 0.98762429f, 0.98710573f, +0.98657662f, 0.98603696f, 0.98548669f, 0.98492593f, 0.98435456f, +0.98377270f, 0.98318028f, 0.98257732f, 0.98196387f, 0.98133987f, +0.98070538f, 0.98006040f, 0.97940493f, 0.97873890f, 0.97806245f, +0.97737551f, 0.97667813f, 0.97597027f, 0.97525197f, 0.97452319f, +0.97378403f, 0.97303438f, 0.97227436f, 0.97150391f, 0.97072303f, +0.96993178f, 0.96913016f, 0.96831810f, 0.96749574f, 0.96666300f, +0.96581990f, 0.96496642f, 0.96410263f, 0.96322852f, 0.96234411f, +0.96144938f, 0.96054435f, 0.95962906f, 0.95870346f, 0.95776761f, +0.95682150f, 0.95586514f, 0.95489854f, 0.95392174f, 0.95293468f, +0.95193744f, 0.95093000f, 0.94991243f, 0.94888461f, 0.94784665f, +0.94679856f, 0.94574034f, 0.94467193f, 0.94359344f, 0.94250488f, +0.94140619f, 0.94029742f, 0.93917859f, 0.93804967f, 0.93691075f, +0.93576175f, 0.93460274f, 0.93343377f, 0.93225473f, 0.93106574f, +0.92986679f, 0.92865789f, 0.92743903f, 0.92621022f, 0.92497152f, +0.92372292f, 0.92246443f, 0.92119598f, 0.91991776f, 0.91862965f, +0.91733170f, 0.91602397f, 0.91470635f, 0.91337901f, 0.91204184f, +0.91069490f, 0.90933824f, 0.90797186f, 0.90659571f, 0.90520984f, +0.90381432f, 0.90240908f, 0.90099424f, 0.89956969f, 0.89813554f, +0.89669174f, 0.89523834f, 0.89377540f, 0.89230281f, 0.89082074f, +0.88932908f, 0.88782793f, 0.88631725f, 0.88479710f, 0.88326746f, +0.88172835f, 0.88017982f, 0.87862182f, 0.87705445f, 0.87547767f, +0.87389153f, 0.87229604f, 0.87069118f, 0.86907703f, 0.86745358f, +0.86582077f, 0.86417878f, 0.86252749f, 0.86086690f, 0.85919720f, +0.85751826f, 0.85583007f, 0.85413277f, 0.85242635f, 0.85071075f, +0.84898609f, 0.84725231f, 0.84550947f, 0.84375757f, 0.84199661f, +0.84022665f, 0.83844769f, 0.83665979f, 0.83486289f, 0.83305705f, +0.83124226f, 0.82941860f, 0.82758605f, 0.82574469f, 0.82389444f, +0.82203537f, 0.82016748f, 0.81829083f, 0.81640542f, 0.81451124f, +0.81260836f, 0.81069672f, 0.80877650f, 0.80684757f, 0.80490994f, +0.80296379f, 0.80100900f, 0.79904562f, 0.79707366f, 0.79509324f, +0.79310423f, 0.79110676f, 0.78910083f, 0.78708643f, 0.78506362f, +0.78303236f, 0.78099275f, 0.77894479f, 0.77688843f, 0.77482378f, +0.77275085f, 0.77066964f, 0.76858020f, 0.76648247f, 0.76437658f, +0.76226246f, 0.76014024f, 0.75800985f, 0.75587130f, 0.75372469f, +0.75157005f, 0.74940729f, 0.74723655f, 0.74505776f, 0.74287105f, +0.74067634f, 0.73847371f, 0.73626316f, 0.73404479f, 0.73181850f, +0.72958434f, 0.72734243f, 0.72509271f, 0.72283524f, 0.72057003f, +0.71829706f, 0.71601641f, 0.71372813f, 0.71143216f, 0.70912862f, +0.70681745f, 0.70449871f, 0.70217246f, 0.69983864f, 0.69749737f, +0.69514859f, 0.69279242f, 0.69042879f, 0.68805778f, 0.68567938f, +0.68329364f, 0.68090063f, 0.67850029f, 0.67609268f, 0.67367786f, +0.67125577f, 0.66882652f, 0.66639012f, 0.66394657f, 0.66149592f, +0.65903819f, 0.65657341f, 0.65410155f, 0.65162271f, 0.64913690f, +0.64664418f, 0.64414448f, 0.64163786f, 0.63912445f, 0.63660413f, +0.63407701f, 0.63154310f, 0.62900239f, 0.62645501f, 0.62390089f, +0.62134010f, 0.61877263f, 0.61619854f, 0.61361790f, 0.61103064f, +0.60843682f, 0.60583651f, 0.60322970f, 0.60061646f, 0.59799677f, +0.59537065f, 0.59273821f, 0.59009939f, 0.58745426f, 0.58480281f, +0.58214509f, 0.57948118f, 0.57681108f, 0.57413477f, 0.57145232f, +0.56876373f, 0.56606907f, 0.56336832f, 0.56066155f, 0.55794877f, +0.55523002f, 0.55250537f, 0.54977477f, 0.54703826f, 0.54429591f, +0.54154772f, 0.53879374f, 0.53603399f, 0.53326851f, 0.53049731f, +0.52772039f, 0.52493787f, 0.52214974f, 0.51935595f, 0.51655668f, +0.51375180f, 0.51094145f, 0.50812566f, 0.50530440f, 0.50247771f, +0.49964568f, 0.49680826f, 0.49396557f, 0.49111754f, 0.48826426f, +0.48540577f, 0.48254207f, 0.47967321f, 0.47679919f, 0.47392011f, +0.47103590f, 0.46814668f, 0.46525243f, 0.46235323f, 0.45944905f, +0.45653993f, 0.45362595f, 0.45070711f, 0.44778344f, 0.44485497f, +0.44192174f, 0.43898380f, 0.43604112f, 0.43309379f, 0.43014181f, +0.42718524f, 0.42422408f, 0.42125839f, 0.41828820f, 0.41531351f, +0.41233435f, 0.40935081f, 0.40636289f, 0.40337059f, 0.40037400f, +0.39737311f, 0.39436796f, 0.39135858f, 0.38834500f, 0.38532731f, +0.38230544f, 0.37927949f, 0.37624949f, 0.37321547f, 0.37017745f, +0.36713544f, 0.36408952f, 0.36103970f, 0.35798600f, 0.35492846f, +0.35186714f, 0.34880206f, 0.34573323f, 0.34266070f, 0.33958447f, +0.33650464f, 0.33342120f, 0.33033419f, 0.32724363f, 0.32414958f, +0.32105204f, 0.31795108f, 0.31484672f, 0.31173897f, 0.30862790f, +0.30551350f, 0.30239585f, 0.29927495f, 0.29615086f, 0.29302359f, +0.28989318f, 0.28675964f, 0.28362307f, 0.28048345f, 0.27734083f, +0.27419522f, 0.27104670f, 0.26789525f, 0.26474094f, 0.26158381f, +0.25842386f, 0.25526115f, 0.25209570f, 0.24892756f, 0.24575676f, +0.24258332f, 0.23940729f, 0.23622867f, 0.23304754f, 0.22986393f, +0.22667783f, 0.22348931f, 0.22029841f, 0.21710514f, 0.21390954f, +0.21071166f, 0.20751151f, 0.20430915f, 0.20110460f, 0.19789790f, +0.19468907f, 0.19147816f, 0.18826519f, 0.18505022f, 0.18183327f, +0.17861435f, 0.17539354f, 0.17217083f, 0.16894630f, 0.16571994f, +0.16249183f, 0.15926196f, 0.15603039f, 0.15279715f, 0.14956227f, +0.14632578f, 0.14308774f, 0.13984816f, 0.13660708f, 0.13336454f, +0.13012058f, 0.12687522f, 0.12362850f, 0.12038045f, 0.11713112f, +0.11388054f, 0.11062872f, 0.10737573f, 0.10412160f, 0.10086634f, +0.097609997f, 0.094352618f, 0.091094226f, 0.087834857f, 0.084574550f, +0.081313334f, 0.078051247f, 0.074788325f, 0.071524605f, 0.068260118f, +0.064994894f, 0.061728980f, 0.058462404f, 0.055195201f, 0.051927410f, +0.048659060f, 0.045390189f, 0.042120833f, 0.038851023f, 0.035580799f, +0.032310195f, 0.029039243f, 0.025767982f, 0.022496443f, 0.019224664f, +0.015952680f, 0.012680525f, 0.0094082337f, 0.0061358409f, 0.0028633832f, +-0.00040910527f, -0.0036815894f, -0.0069540343f, -0.010226404f, -0.013498665f, +-0.016770782f, -0.020042717f, -0.023314439f, -0.026585912f, -0.029857099f, +-0.033127967f, -0.036398482f, -0.039668605f, -0.042938303f, -0.046207540f, +-0.049476285f, -0.052744497f, -0.056012146f, -0.059279196f, -0.062545612f, +-0.065811358f, -0.069076397f, -0.072340697f, -0.075604223f, -0.078866936f, +-0.082128808f, -0.085389800f, -0.088649876f, -0.091909006f, -0.095167145f, +-0.098424271f, -0.10168034f, -0.10493532f, -0.10818918f, -0.11144188f, +-0.11469338f, -0.11794366f, -0.12119267f, -0.12444039f, -0.12768677f, +-0.13093179f, -0.13417540f, -0.13741758f, -0.14065829f, -0.14389749f, +-0.14713514f, -0.15037122f, -0.15360570f, -0.15683852f, -0.16006967f, +-0.16329910f, -0.16652679f, -0.16975269f, -0.17297678f, -0.17619900f, +-0.17941935f, -0.18263777f, -0.18585424f, -0.18906870f, -0.19228116f, +-0.19549155f, -0.19869985f, -0.20190603f, -0.20511003f, -0.20831184f, +-0.21151142f, -0.21470875f, -0.21790376f, -0.22109644f, -0.22428675f, +-0.22747467f, -0.23066014f, -0.23384315f, -0.23702365f, -0.24020162f, +-0.24337701f, -0.24654980f, -0.24971995f, -0.25288740f, -0.25605217f, +-0.25921419f, -0.26237345f, -0.26552987f, -0.26868346f, -0.27183419f, +-0.27498198f, -0.27812684f, -0.28126872f, -0.28440759f, -0.28754342f, +-0.29067615f, -0.29380578f, -0.29693225f, -0.30005556f, -0.30317566f, +-0.30629250f, -0.30940607f, -0.31251630f, -0.31562322f, -0.31872672f, +-0.32182685f, -0.32492352f, -0.32801670f, -0.33110636f, -0.33419248f, +-0.33727503f, -0.34035397f, -0.34342924f, -0.34650084f, -0.34956875f, +-0.35263291f, -0.35569328f, -0.35874987f, -0.36180258f, -0.36485144f, +-0.36789638f, -0.37093741f, -0.37397444f, -0.37700745f, -0.38003644f, +-0.38306138f, -0.38608220f, -0.38909888f, -0.39211139f, -0.39511973f, +-0.39812380f, -0.40112361f, -0.40411916f, -0.40711036f, -0.41009718f, +-0.41307965f, -0.41605768f, -0.41903123f, -0.42200032f, -0.42496487f, +-0.42792490f, -0.43088034f, -0.43383113f, -0.43677729f, -0.43971881f, +-0.44265559f, -0.44558764f, -0.44851488f, -0.45143735f, -0.45435500f, +-0.45726776f, -0.46017563f, -0.46307856f, -0.46597654f, -0.46886954f, +-0.47175750f, -0.47464043f, -0.47751826f, -0.48039100f, -0.48325855f, +-0.48612097f, -0.48897815f, -0.49183011f, -0.49467680f, -0.49751821f, +-0.50035429f, -0.50318497f, -0.50601029f, -0.50883019f, -0.51164466f, +-0.51445359f, -0.51725709f, -0.52005500f, -0.52284735f, -0.52563411f, +-0.52841520f, -0.53119069f, -0.53396046f, -0.53672451f, -0.53948283f, +-0.54223537f, -0.54498214f, -0.54772300f, -0.55045801f, -0.55318713f, +-0.55591035f, -0.55862761f, -0.56133890f, -0.56404412f, -0.56674337f, +-0.56943649f, -0.57212353f, -0.57480448f, -0.57747924f, -0.58014780f, +-0.58281022f, -0.58546633f, -0.58811617f, -0.59075975f, -0.59339696f, +-0.59602785f, -0.59865236f, -0.60127044f, -0.60388207f, -0.60648727f, +-0.60908598f, -0.61167812f, -0.61426371f, -0.61684275f, -0.61941516f, +-0.62198097f, -0.62454009f, -0.62709254f, -0.62963831f, -0.63217729f, +-0.63470948f, -0.63723493f, -0.63975352f, -0.64226526f, -0.64477009f, +-0.64726806f, -0.64975911f, -0.65224314f, -0.65472025f, -0.65719032f, +-0.65965337f, -0.66210932f, -0.66455823f, -0.66700000f, -0.66943461f, +-0.67186207f, -0.67428231f, -0.67669535f, -0.67910111f, -0.68149966f, +-0.68389088f, -0.68627477f, -0.68865126f, -0.69102043f, -0.69338220f, +-0.69573659f, -0.69808346f, -0.70042288f, -0.70275480f, -0.70507920f, +-0.70739603f, -0.70970529f, -0.71200693f, -0.71430099f, -0.71658736f, +-0.71886611f, -0.72113711f, -0.72340041f, -0.72565591f, -0.72790372f, +-0.73014367f, -0.73237586f, -0.73460019f, -0.73681659f, -0.73902518f, +-0.74122584f, -0.74341851f, -0.74560326f, -0.74778003f, -0.74994880f, +-0.75210953f, -0.75426215f, -0.75640678f, -0.75854325f, -0.76067162f, +-0.76279181f, -0.76490390f, -0.76700771f, -0.76910341f, -0.77119076f, +-0.77326995f, -0.77534080f, -0.77740335f, -0.77945763f, -0.78150350f, +-0.78354102f, -0.78557014f, -0.78759086f, -0.78960317f, -0.79160696f, +-0.79360235f, -0.79558921f, -0.79756755f, -0.79953730f, -0.80149853f, +-0.80345118f, -0.80539525f, -0.80733067f, -0.80925739f, -0.81117553f, +-0.81308490f, -0.81498563f, -0.81687760f, -0.81876087f, -0.82063532f, +-0.82250100f, -0.82435787f, -0.82620591f, -0.82804507f, -0.82987541f, +-0.83169687f, -0.83350939f, -0.83531296f, -0.83710766f, -0.83889335f, +-0.84067005f, -0.84243774f, -0.84419644f, -0.84594607f, -0.84768665f, +-0.84941816f, -0.85114056f, -0.85285389f, -0.85455805f, -0.85625303f, +-0.85793889f, -0.85961550f, -0.86128294f, -0.86294121f, -0.86459017f, +-0.86622989f, -0.86786032f, -0.86948150f, -0.87109333f, -0.87269586f, +-0.87428904f, -0.87587279f, -0.87744725f, -0.87901229f, -0.88056785f, +-0.88211405f, -0.88365078f, -0.88517809f, -0.88669586f, -0.88820416f, +-0.88970292f, -0.89119220f, -0.89267188f, -0.89414203f, -0.89560264f, +-0.89705360f, -0.89849502f, -0.89992678f, -0.90134889f, -0.90276134f, +-0.90416414f, -0.90555727f, -0.90694070f, -0.90831441f, -0.90967834f, +-0.91103262f, -0.91237706f, -0.91371179f, -0.91503674f, -0.91635185f, +-0.91765714f, -0.91895264f, -0.92023826f, -0.92151409f, -0.92277998f, +-0.92403603f, -0.92528218f, -0.92651838f, -0.92774469f, -0.92896110f, +-0.93016750f, -0.93136400f, -0.93255049f, -0.93372697f, -0.93489349f, +-0.93604994f, -0.93719643f, -0.93833286f, -0.93945926f, -0.94057560f, +-0.94168180f, -0.94277799f, -0.94386405f, -0.94494003f, -0.94600588f, +-0.94706154f, -0.94810712f, -0.94914252f, -0.95016778f, -0.95118284f, +-0.95218778f, -0.95318246f, -0.95416695f, -0.95514119f, -0.95610523f, +-0.95705903f, -0.95800257f, -0.95893586f, -0.95985889f, -0.96077162f, +-0.96167403f, -0.96256620f, -0.96344805f, -0.96431959f, -0.96518075f, +-0.96603161f, -0.96687216f, -0.96770233f, -0.96852213f, -0.96933156f, +-0.97013056f, -0.97091925f, -0.97169751f, -0.97246534f, -0.97322279f, +-0.97396982f, -0.97470641f, -0.97543252f, -0.97614825f, -0.97685349f, +-0.97754824f, -0.97823256f, -0.97890645f, -0.97956979f, -0.98022264f, +-0.98086500f, -0.98149687f, -0.98211825f, -0.98272908f, -0.98332942f, +-0.98391914f, -0.98449844f, -0.98506713f, -0.98562527f, -0.98617285f, +-0.98670989f, -0.98723638f, -0.98775226f, -0.98825759f, -0.98875231f, +-0.98923647f, -0.98971003f, -0.99017298f, -0.99062532f, -0.99106705f, +-0.99149817f, -0.99191868f, -0.99232858f, -0.99272782f, -0.99311644f, +-0.99349445f, -0.99386179f, -0.99421853f, -0.99456459f, -0.99489999f, +-0.99522477f, -0.99553883f, -0.99584228f, -0.99613506f, -0.99641716f, +-0.99668860f, -0.99694937f, -0.99719942f, -0.99743885f, -0.99766755f, +-0.99788558f, -0.99809295f, -0.99828959f, -0.99847561f, -0.99865085f, +-0.99881548f, -0.99896932f, -0.99911255f, -0.99924499f, -0.99936682f, +-0.99947786f, -0.99957830f, -0.99966794f, -0.99974692f, -0.99981517f, +-0.99987274f, -0.99991959f, -0.99995571f, -0.99998116f, -0.99999589f, +0.99999964f, 0.99997288f, 0.99990326f, 0.99979085f, 0.99963558f, +0.99943751f, 0.99919659f, 0.99891287f, 0.99858636f, 0.99821711f, +0.99780506f, 0.99735034f, 0.99685282f, 0.99631262f, 0.99572974f, +0.99510419f, 0.99443603f, 0.99372530f, 0.99297196f, 0.99217612f, +0.99133772f, 0.99045694f, 0.98953366f, 0.98856801f, 0.98756003f, +0.98650974f, 0.98541719f, 0.98428243f, 0.98310548f, 0.98188645f, +0.98062533f, 0.97932225f, 0.97797716f, 0.97659022f, 0.97516143f, +0.97369087f, 0.97217858f, 0.97062469f, 0.96902919f, 0.96739221f, +0.96571374f, 0.96399397f, 0.96223283f, 0.96043050f, 0.95858705f, +0.95670253f, 0.95477700f, 0.95281059f, 0.95080340f, 0.94875544f, +0.94666684f, 0.94453770f, 0.94236809f, 0.94015813f, 0.93790787f, +0.93561745f, 0.93328691f, 0.93091643f, 0.92850608f, 0.92605597f, +0.92356616f, 0.92103678f, 0.91846794f, 0.91585976f, 0.91321236f, +0.91052586f, 0.90780038f, 0.90503591f, 0.90223277f, 0.89939094f, +0.89651060f, 0.89359182f, 0.89063478f, 0.88763964f, 0.88460642f, +0.88153529f, 0.87842643f, 0.87527996f, 0.87209594f, 0.86887461f, +0.86561602f, 0.86232042f, 0.85898781f, 0.85561842f, 0.85221243f, +0.84876984f, 0.84529096f, 0.84177583f, 0.83822471f, 0.83463764f, +0.83101481f, 0.82735640f, 0.82366252f, 0.81993335f, 0.81616908f, +0.81236988f, 0.80853581f, 0.80466717f, 0.80076402f, 0.79682660f, +0.79285502f, 0.78884947f, 0.78481019f, 0.78073722f, 0.77663082f, +0.77249116f, 0.76831841f, 0.76411277f, 0.75987434f, 0.75560343f, +0.75130010f, 0.74696463f, 0.74259710f, 0.73819780f, 0.73376691f, +0.72930455f, 0.72481096f, 0.72028631f, 0.71573079f, 0.71114463f, +0.70652801f, 0.70188117f, 0.69720417f, 0.69249737f, 0.68776089f, +0.68299496f, 0.67819971f, 0.67337549f, 0.66852236f, 0.66364062f, +0.65873051f, 0.65379208f, 0.64882571f, 0.64383155f, 0.63880974f, +0.63376063f, 0.62868434f, 0.62358117f, 0.61845124f, 0.61329484f, +0.60811216f, 0.60290343f, 0.59766883f, 0.59240872f, 0.58712316f, +0.58181250f, 0.57647687f, 0.57111657f, 0.56573176f, 0.56032276f, +0.55488980f, 0.54943299f, 0.54395270f, 0.53844911f, 0.53292239f, +0.52737290f, 0.52180082f, 0.51620632f, 0.51058978f, 0.50495136f, +0.49929130f, 0.49360985f, 0.48790723f, 0.48218375f, 0.47643960f, +0.47067502f, 0.46489030f, 0.45908567f, 0.45326138f, 0.44741765f, +0.44155475f, 0.43567297f, 0.42977250f, 0.42385364f, 0.41791660f, +0.41196167f, 0.40598908f, 0.39999911f, 0.39399201f, 0.38796803f, +0.38192743f, 0.37587047f, 0.36979741f, 0.36370850f, 0.35760403f, +0.35148421f, 0.34534934f, 0.33919969f, 0.33303553f, 0.32685706f, +0.32066461f, 0.31445843f, 0.30823877f, 0.30200592f, 0.29576012f, +0.28950164f, 0.28323078f, 0.27694780f, 0.27065292f, 0.26434645f, +0.25802869f, 0.25169984f, 0.24536023f, 0.23901010f, 0.23264973f, +0.22627939f, 0.21989937f, 0.21350993f, 0.20711134f, 0.20070387f, +0.19428782f, 0.18786344f, 0.18143101f, 0.17499080f, 0.16854310f, +0.16208819f, 0.15562633f, 0.14915779f, 0.14268288f, 0.13620184f, +0.12971498f, 0.12322257f, 0.11672486f, 0.11022217f, 0.10371475f, +0.097202882f, 0.090686858f, 0.084166944f, 0.077643424f, 0.071116582f, +0.064586692f, 0.058054037f, 0.051518895f, 0.044981543f, 0.038442269f, +0.031901345f, 0.025359053f, 0.018815678f, 0.012271495f, 0.0057267868f, +-0.00081816671f, -0.0073630852f, -0.013907688f, -0.020451695f, -0.026994826f, +-0.033536803f, -0.040077340f, -0.046616159f, -0.053152986f, -0.059687532f, +-0.066219524f, -0.072748676f, -0.079274714f, -0.085797355f, -0.092316322f, +-0.098831341f, -0.10534211f, -0.11184838f, -0.11834986f, -0.12484626f, +-0.13133731f, -0.13782275f, -0.14430228f, -0.15077563f, -0.15724251f, +-0.16370267f, -0.17015581f, -0.17660165f, -0.18303993f, -0.18947038f, +-0.19589271f, -0.20230664f, -0.20871192f, -0.21510825f, -0.22149536f, +-0.22787298f, -0.23424086f, -0.24059868f, -0.24694622f, -0.25328314f, +-0.25960925f, -0.26592422f, -0.27222782f, -0.27851975f, -0.28479972f, +-0.29106751f, -0.29732284f, -0.30356544f, -0.30979502f, -0.31601134f, +-0.32221413f, -0.32840309f, -0.33457801f, -0.34073856f, -0.34688455f, +-0.35301566f, -0.35913166f, -0.36523229f, -0.37131724f, -0.37738630f, +-0.38343921f, -0.38947567f, -0.39549544f, -0.40149832f, -0.40748394f, +-0.41345215f, -0.41940263f, -0.42533514f, -0.43124944f, -0.43714526f, +-0.44302234f, -0.44888046f, -0.45471936f, -0.46053877f, -0.46633846f, +-0.47211814f, -0.47787762f, -0.48361665f, -0.48933494f, -0.49503228f, +-0.50070840f, -0.50636309f, -0.51199609f, -0.51760709f, -0.52319598f, +-0.52876246f, -0.53430629f, -0.53982723f, -0.54532504f, -0.55079949f, +-0.55625033f, -0.56167740f, -0.56708032f, -0.57245898f, -0.57781315f, +-0.58314258f, -0.58844697f, -0.59372622f, -0.59897995f, -0.60420811f, +-0.60941035f, -0.61458647f, -0.61973625f, -0.62485951f, -0.62995601f, +-0.63502556f, -0.64006782f, -0.64508271f, -0.65007001f, -0.65502942f, +-0.65996075f, -0.66486382f, -0.66973841f, -0.67458433f, -0.67940134f, +-0.68418926f, -0.68894786f, -0.69367695f, -0.69837630f, -0.70304573f, +-0.70768511f, -0.71229410f, -0.71687263f, -0.72142041f, -0.72593731f, +-0.73042315f, -0.73487765f, -0.73930067f, -0.74369204f, -0.74805158f, +-0.75237900f, -0.75667429f, -0.76093709f, -0.76516730f, -0.76936477f, +-0.77352923f, -0.77766061f, -0.78175867f, -0.78582323f, -0.78985411f, +-0.79385114f, -0.79781419f, -0.80174309f, -0.80563760f, -0.80949765f, +-0.81332302f, -0.81711352f, -0.82086903f, -0.82458937f, -0.82827437f, +-0.83192390f, -0.83553779f, -0.83911592f, -0.84265804f, -0.84616417f, +-0.84963393f, -0.85306740f, -0.85646427f, -0.85982448f, -0.86314780f, +-0.86643422f, -0.86968350f, -0.87289548f, -0.87607014f, -0.87920725f, +-0.88230664f, -0.88536829f, -0.88839203f, -0.89137769f, -0.89432514f, +-0.89723432f, -0.90010506f, -0.90293723f, -0.90573072f, -0.90848541f, +-0.91120118f, -0.91387796f, -0.91651553f, -0.91911387f, -0.92167282f, +-0.92419231f, -0.92667222f, -0.92911243f, -0.93151283f, -0.93387336f, +-0.93619382f, -0.93847424f, -0.94071442f, -0.94291431f, -0.94507378f, +-0.94719279f, -0.94927126f, -0.95130903f, -0.95330608f, -0.95526224f, +-0.95717752f, -0.95905179f, -0.96088499f, -0.96267700f, -0.96442777f, +-0.96613729f, -0.96780539f, -0.96943200f, -0.97101706f, -0.97256058f, +-0.97406244f, -0.97552258f, -0.97694093f, -0.97831738f, -0.97965199f, +-0.98094457f, -0.98219514f, -0.98340368f, -0.98457009f, -0.98569429f, +-0.98677629f, -0.98781598f, -0.98881340f, -0.98976845f, -0.99068111f, +-0.99155134f, -0.99237907f, -0.99316430f, -0.99390697f, -0.99460709f, +-0.99526459f, -0.99587947f, -0.99645168f, -0.99698120f, -0.99746799f, +-0.99791211f, -0.99831343f, -0.99867201f, -0.99898779f, -0.99926084f, +-0.99949104f, -0.99967843f, -0.99982297f, -0.99992472f, -0.99998361f, +0.99999869f, 0.99989158f, 0.99961317f, 0.99916345f, 0.99854255f, +0.99775058f, 0.99678761f, 0.99565387f, 0.99434954f, 0.99287480f, +0.99122995f, 0.98941529f, 0.98743105f, 0.98527765f, 0.98295540f, +0.98046476f, 0.97780609f, 0.97497988f, 0.97198665f, 0.96882683f, +0.96550101f, 0.96200979f, 0.95835376f, 0.95453346f, 0.95054960f, +0.94640291f, 0.94209403f, 0.93762374f, 0.93299282f, 0.92820197f, +0.92325211f, 0.91814411f, 0.91287869f, 0.90745693f, 0.90187967f, +0.89614785f, 0.89026248f, 0.88422459f, 0.87803519f, 0.87169534f, +0.86520612f, 0.85856867f, 0.85178405f, 0.84485358f, 0.83777827f, +0.83055943f, 0.82319832f, 0.81569612f, 0.80805415f, 0.80027372f, +0.79235619f, 0.78430289f, 0.77611518f, 0.76779449f, 0.75934225f, +0.75075996f, 0.74204898f, 0.73321080f, 0.72424710f, 0.71515924f, +0.70594883f, 0.69661748f, 0.68716675f, 0.67759830f, 0.66791373f, +0.65811473f, 0.64820296f, 0.63818014f, 0.62804794f, 0.61780810f, +0.60746247f, 0.59701276f, 0.58646071f, 0.57580817f, 0.56505698f, +0.55420899f, 0.54326600f, 0.53222996f, 0.52110273f, 0.50988621f, +0.49858227f, 0.48719296f, 0.47572014f, 0.46416581f, 0.45253196f, +0.44082057f, 0.42903364f, 0.41717321f, 0.40524128f, 0.39323992f, +0.38117120f, 0.36903715f, 0.35683987f, 0.34458145f, 0.33226398f, +0.31988961f, 0.30746040f, 0.29497850f, 0.28244606f, 0.26986524f, +0.25723818f, 0.24456702f, 0.23185398f, 0.21910121f, 0.20631088f, +0.19348522f, 0.18062639f, 0.16773662f, 0.15481812f, 0.14187308f, +0.12890373f, 0.11591230f, 0.10290100f, 0.089872077f, 0.076827750f, +0.063770257f, 0.050701842f, 0.037624735f, 0.024541186f, 0.011453429f, +-0.0016362892f, -0.014725727f, -0.027812643f, -0.040894791f, -0.053969935f, +-0.067035832f, -0.080090240f, -0.093130924f, -0.10615565f, -0.11916219f, +-0.13214831f, -0.14511178f, -0.15805040f, -0.17096193f, -0.18384418f, +-0.19669491f, -0.20951195f, -0.22229309f, -0.23503613f, -0.24773891f, +-0.26039925f, -0.27301496f, -0.28558388f, -0.29810387f, -0.31057280f, +-0.32298848f, -0.33534884f, -0.34765175f, -0.35989508f, -0.37207675f, +-0.38419467f, -0.39624676f, -0.40823093f, -0.42014518f, -0.43198743f, +-0.44375566f, -0.45544785f, -0.46706200f, -0.47859612f, -0.49004826f, +-0.50141639f, -0.51269865f, -0.52389306f, -0.53499764f, -0.54601061f, +-0.55693001f, -0.56775403f, -0.57848072f, -0.58910829f, -0.59963489f, +-0.61005878f, -0.62037814f, -0.63059121f, -0.64069623f, -0.65069145f, +-0.66057515f, -0.67034572f, -0.68000144f, -0.68954057f, -0.69896162f, +-0.70826286f, -0.71744281f, -0.72649974f, -0.73543227f, -0.74423873f, +-0.75291771f, -0.76146764f, -0.76988715f, -0.77817470f, -0.78632891f, +-0.79434842f, -0.80223179f, -0.80997771f, -0.81758487f, -0.82505190f, +-0.83237761f, -0.83956063f, -0.84659988f, -0.85349399f, -0.86024189f, +-0.86684239f, -0.87329435f, -0.87959671f, -0.88574833f, -0.89174819f, +-0.89759529f, -0.90328854f, -0.90882701f, -0.91420978f, -0.91943592f, +-0.92450452f, -0.92941469f, -0.93416560f, -0.93875647f, -0.94318646f, +-0.94745487f, -0.95156091f, -0.95550388f, -0.95928317f, -0.96289814f, +-0.96634805f, -0.96963239f, -0.97275060f, -0.97570217f, -0.97848648f, +-0.98110318f, -0.98355180f, -0.98583186f, -0.98794299f, -0.98988485f, +-0.99165714f, -0.99325943f, -0.99469161f, -0.99595332f, -0.99704438f, +-0.99796462f, -0.99871385f, -0.99929196f, -0.99969882f, -0.99993443f, +0.99999464f, 0.99956632f, 0.99845290f, 0.99665523f, 0.99417448f, +0.99101239f, 0.98717111f, 0.98265326f, 0.97746199f, 0.97160077f, +0.96507365f, 0.95788515f, 0.95004016f, 0.94154406f, 0.93240267f, +0.92262226f, 0.91220951f, 0.90117162f, 0.88951606f, 0.87725091f, +0.86438453f, 0.85092574f, 0.83688372f, 0.82226819f, 0.80708915f, +0.79135692f, 0.77508235f, 0.75827658f, 0.74095112f, 0.72311783f, +0.70478898f, 0.68597710f, 0.66669506f, 0.64695615f, 0.62677377f, +0.60616189f, 0.58513457f, 0.56370622f, 0.54189157f, 0.51970547f, +0.49716324f, 0.47428027f, 0.45107225f, 0.42755505f, 0.40374488f, +0.37965798f, 0.35531086f, 0.33072025f, 0.30590299f, 0.28087607f, +0.25565663f, 0.23026201f, 0.20470956f, 0.17901683f, 0.15320139f, +0.12728097f, 0.10127331f, 0.075196236f, 0.049067631f, 0.022905400f, +-0.0032725304f, -0.029448219f, -0.055603724f, -0.081721120f, -0.10778251f, +-0.13377003f, -0.15966587f, -0.18545228f, -0.21111161f, -0.23662624f, +-0.26197869f, -0.28715160f, -0.31212771f, -0.33688989f, -0.36142120f, +-0.38570482f, -0.40972409f, -0.43346253f, -0.45690393f, -0.48003218f, +-0.50283146f, -0.52528608f, -0.54738069f, -0.56910020f, -0.59042966f, +-0.61135447f, -0.63186026f, -0.65193301f, -0.67155898f, -0.69072473f, +-0.70941705f, -0.72762316f, -0.74533063f, -0.76252723f, -0.77920127f, +-0.79534131f, -0.81093621f, -0.82597536f, -0.84044844f, -0.85434550f, +-0.86765707f, -0.88037395f, -0.89248747f, -0.90398932f, -0.91487163f, +-0.92512697f, -0.93474823f, -0.94372886f, -0.95206273f, -0.95974404f, +-0.96676767f, -0.97312868f, -0.97882277f, -0.98384601f, -0.98819500f, +-0.99186671f, -0.99485862f, -0.99716878f, -0.99879545f, -0.99973762f, +}; +#endif + +static const CELTMode mode48000_960_120 = { +48000, /* Fs */ +120, /* overlap */ +21, /* nbEBands */ +21, /* effEBands */ +{0.85000610f, 0.0000000f, 1.0000000f, 1.0000000f, }, /* preemph */ +eband5ms, /* eBands */ +3, /* maxLM */ +8, /* nbShortMdcts */ +120, /* shortMdctSize */ +11, /* nbAllocVectors */ +band_allocation, /* allocVectors */ +logN400, /* logN */ +window120, /* window */ +{1920, 3, {&fft_state48000_960_0, &fft_state48000_960_1, &fft_state48000_960_2, &fft_state48000_960_3, }, mdct_twiddles960}, /* mdct */ +{392, cache_index50, cache_bits50, cache_caps50}, /* cache */ +}; + +/* List of all the available modes */ +#define TOTAL_MODES 1 +static const CELTMode * const static_mode_list[TOTAL_MODES] = { +&mode48000_960_120, +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float_arm_ne10.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float_arm_ne10.h new file mode 100644 index 000000000..66e1abb10 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/static_modes_float_arm_ne10.h @@ -0,0 +1,404 @@ +/* The contents of this file was automatically generated by + * dump_mode_arm_ne10.c with arguments: 48000 960 + * It contains static definitions for some pre-defined modes. */ +#include + +#ifndef NE10_FFT_PARAMS48000_960 +#define NE10_FFT_PARAMS48000_960 +static const ne10_int32_t ne10_factors_480[64] = { +4, 40, 4, 30, 2, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_240[64] = { +3, 20, 4, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_120[64] = { +3, 10, 2, 15, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_int32_t ne10_factors_60[64] = { +2, 5, 5, 3, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, }; +static const ne10_fft_cpx_float32_t ne10_twiddles_480[480] = { +{1.0000000f,0.0000000f}, {1.0000000f,-0.0000000f}, {1.0000000f,-0.0000000f}, +{1.0000000f,-0.0000000f}, {0.91354543f,-0.40673664f}, {0.66913056f,-0.74314487f}, +{1.0000000f,-0.0000000f}, {0.66913056f,-0.74314487f}, {-0.10452851f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.30901697f,-0.95105654f}, {-0.80901700f,-0.58778518f}, +{1.0000000f,-0.0000000f}, {-0.10452851f,-0.99452192f}, {-0.97814757f,0.20791179f}, +{1.0000000f,-0.0000000f}, {0.97814763f,-0.20791170f}, {0.91354543f,-0.40673664f}, +{0.80901700f,-0.58778524f}, {0.66913056f,-0.74314487f}, {0.49999997f,-0.86602545f}, +{0.30901697f,-0.95105654f}, {0.10452842f,-0.99452192f}, {-0.10452851f,-0.99452192f}, +{-0.30901703f,-0.95105648f}, {-0.50000006f,-0.86602533f}, {-0.66913068f,-0.74314475f}, +{-0.80901700f,-0.58778518f}, {-0.91354549f,-0.40673658f}, {-0.97814763f,-0.20791161f}, +{1.0000000f,-0.0000000f}, {0.99862951f,-0.052335959f}, {0.99452192f,-0.10452846f}, +{0.98768836f,-0.15643448f}, {0.97814763f,-0.20791170f}, {0.96592581f,-0.25881904f}, +{0.95105648f,-0.30901700f}, {0.93358040f,-0.35836795f}, {0.91354543f,-0.40673664f}, +{0.89100653f,-0.45399052f}, {0.86602545f,-0.50000000f}, {0.83867055f,-0.54463905f}, +{0.80901700f,-0.58778524f}, {0.77714598f,-0.62932038f}, {0.74314475f,-0.66913062f}, +{0.70710677f,-0.70710683f}, {0.66913056f,-0.74314487f}, {0.62932038f,-0.77714598f}, +{0.58778524f,-0.80901700f}, {0.54463899f,-0.83867055f}, {0.49999997f,-0.86602545f}, +{0.45399052f,-0.89100653f}, {0.40673661f,-0.91354549f}, {0.35836786f,-0.93358046f}, +{0.30901697f,-0.95105654f}, {0.25881907f,-0.96592581f}, {0.20791166f,-0.97814763f}, +{0.15643437f,-0.98768836f}, {0.10452842f,-0.99452192f}, {0.052335974f,-0.99862951f}, +{1.0000000f,-0.0000000f}, {0.99452192f,-0.10452846f}, {0.97814763f,-0.20791170f}, +{0.95105648f,-0.30901700f}, {0.91354543f,-0.40673664f}, {0.86602545f,-0.50000000f}, +{0.80901700f,-0.58778524f}, {0.74314475f,-0.66913062f}, {0.66913056f,-0.74314487f}, +{0.58778524f,-0.80901700f}, {0.49999997f,-0.86602545f}, {0.40673661f,-0.91354549f}, +{0.30901697f,-0.95105654f}, {0.20791166f,-0.97814763f}, {0.10452842f,-0.99452192f}, +{-4.3711388e-08f,-1.0000000f}, {-0.10452851f,-0.99452192f}, {-0.20791174f,-0.97814757f}, +{-0.30901703f,-0.95105648f}, {-0.40673670f,-0.91354543f}, {-0.50000006f,-0.86602533f}, +{-0.58778518f,-0.80901700f}, {-0.66913068f,-0.74314475f}, {-0.74314493f,-0.66913044f}, +{-0.80901700f,-0.58778518f}, {-0.86602539f,-0.50000006f}, {-0.91354549f,-0.40673658f}, +{-0.95105654f,-0.30901679f}, {-0.97814763f,-0.20791161f}, {-0.99452192f,-0.10452849f}, +{1.0000000f,-0.0000000f}, {0.98768836f,-0.15643448f}, {0.95105648f,-0.30901700f}, +{0.89100653f,-0.45399052f}, {0.80901700f,-0.58778524f}, {0.70710677f,-0.70710683f}, +{0.58778524f,-0.80901700f}, {0.45399052f,-0.89100653f}, {0.30901697f,-0.95105654f}, +{0.15643437f,-0.98768836f}, {-4.3711388e-08f,-1.0000000f}, {-0.15643445f,-0.98768836f}, +{-0.30901703f,-0.95105648f}, {-0.45399061f,-0.89100647f}, {-0.58778518f,-0.80901700f}, +{-0.70710677f,-0.70710677f}, {-0.80901700f,-0.58778518f}, {-0.89100659f,-0.45399037f}, +{-0.95105654f,-0.30901679f}, {-0.98768836f,-0.15643445f}, {-1.0000000f,8.7422777e-08f}, +{-0.98768830f,0.15643461f}, {-0.95105654f,0.30901697f}, {-0.89100653f,0.45399055f}, +{-0.80901694f,0.58778536f}, {-0.70710665f,0.70710689f}, {-0.58778507f,0.80901712f}, +{-0.45399022f,0.89100665f}, {-0.30901709f,0.95105648f}, {-0.15643452f,0.98768830f}, +{1.0000000f,-0.0000000f}, {0.99991435f,-0.013089596f}, {0.99965733f,-0.026176950f}, +{0.99922901f,-0.039259817f}, {0.99862951f,-0.052335959f}, {0.99785894f,-0.065403134f}, +{0.99691731f,-0.078459099f}, {0.99580491f,-0.091501623f}, {0.99452192f,-0.10452846f}, +{0.99306846f,-0.11753740f}, {0.99144489f,-0.13052620f}, {0.98965138f,-0.14349262f}, +{0.98768836f,-0.15643448f}, {0.98555607f,-0.16934951f}, {0.98325491f,-0.18223552f}, +{0.98078525f,-0.19509032f}, {0.97814763f,-0.20791170f}, {0.97534233f,-0.22069745f}, +{0.97236991f,-0.23344538f}, {0.96923089f,-0.24615330f}, {0.96592581f,-0.25881904f}, +{0.96245521f,-0.27144045f}, {0.95881975f,-0.28401536f}, {0.95501995f,-0.29654160f}, +{0.95105648f,-0.30901700f}, {0.94693011f,-0.32143945f}, {0.94264150f,-0.33380687f}, +{0.93819129f,-0.34611708f}, {0.93358040f,-0.35836795f}, {0.92880952f,-0.37055743f}, +{0.92387956f,-0.38268346f}, {0.91879117f,-0.39474389f}, {0.91354543f,-0.40673664f}, +{0.90814316f,-0.41865975f}, {0.90258527f,-0.43051112f}, {0.89687270f,-0.44228873f}, +{0.89100653f,-0.45399052f}, {0.88498765f,-0.46561453f}, {0.87881708f,-0.47715878f}, +{0.87249601f,-0.48862126f}, {0.86602545f,-0.50000000f}, {0.85940641f,-0.51129311f}, +{0.85264015f,-0.52249855f}, {0.84572786f,-0.53361452f}, {0.83867055f,-0.54463905f}, +{0.83146960f,-0.55557024f}, {0.82412618f,-0.56640625f}, {0.81664151f,-0.57714522f}, +{0.80901700f,-0.58778524f}, {0.80125380f,-0.59832460f}, {0.79335332f,-0.60876143f}, +{0.78531694f,-0.61909395f}, {0.77714598f,-0.62932038f}, {0.76884180f,-0.63943899f}, +{0.76040596f,-0.64944810f}, {0.75183982f,-0.65934587f}, {0.74314475f,-0.66913062f}, +{0.73432249f,-0.67880076f}, {0.72537434f,-0.68835455f}, {0.71630192f,-0.69779050f}, +{0.70710677f,-0.70710683f}, {0.69779044f,-0.71630198f}, {0.68835455f,-0.72537440f}, +{0.67880070f,-0.73432255f}, {0.66913056f,-0.74314487f}, {0.65934581f,-0.75183982f}, +{0.64944804f,-0.76040596f}, {0.63943899f,-0.76884186f}, {0.62932038f,-0.77714598f}, +{0.61909395f,-0.78531694f}, {0.60876137f,-0.79335338f}, {0.59832460f,-0.80125386f}, +{0.58778524f,-0.80901700f}, {0.57714516f,-0.81664151f}, {0.56640625f,-0.82412618f}, +{0.55557019f,-0.83146960f}, {0.54463899f,-0.83867055f}, {0.53361452f,-0.84572786f}, +{0.52249849f,-0.85264015f}, {0.51129311f,-0.85940641f}, {0.49999997f,-0.86602545f}, +{0.48862118f,-0.87249601f}, {0.47715876f,-0.87881708f}, {0.46561447f,-0.88498765f}, +{0.45399052f,-0.89100653f}, {0.44228867f,-0.89687276f}, {0.43051103f,-0.90258533f}, +{0.41865975f,-0.90814316f}, {0.40673661f,-0.91354549f}, {0.39474380f,-0.91879129f}, +{0.38268343f,-0.92387956f}, {0.37055740f,-0.92880958f}, {0.35836786f,-0.93358046f}, +{0.34611705f,-0.93819135f}, {0.33380681f,-0.94264150f}, {0.32143947f,-0.94693011f}, +{0.30901697f,-0.95105654f}, {0.29654151f,-0.95501995f}, {0.28401533f,-0.95881975f}, +{0.27144039f,-0.96245527f}, {0.25881907f,-0.96592581f}, {0.24615327f,-0.96923089f}, +{0.23344530f,-0.97236991f}, {0.22069745f,-0.97534233f}, {0.20791166f,-0.97814763f}, +{0.19509023f,-0.98078531f}, {0.18223552f,-0.98325491f}, {0.16934945f,-0.98555607f}, +{0.15643437f,-0.98768836f}, {0.14349259f,-0.98965138f}, {0.13052613f,-0.99144489f}, +{0.11753740f,-0.99306846f}, {0.10452842f,-0.99452192f}, {0.091501534f,-0.99580491f}, +{0.078459084f,-0.99691731f}, {0.065403074f,-0.99785894f}, {0.052335974f,-0.99862951f}, +{0.039259788f,-0.99922901f}, {0.026176875f,-0.99965733f}, {0.013089597f,-0.99991435f}, +{1.0000000f,-0.0000000f}, {0.99965733f,-0.026176950f}, {0.99862951f,-0.052335959f}, +{0.99691731f,-0.078459099f}, {0.99452192f,-0.10452846f}, {0.99144489f,-0.13052620f}, +{0.98768836f,-0.15643448f}, {0.98325491f,-0.18223552f}, {0.97814763f,-0.20791170f}, +{0.97236991f,-0.23344538f}, {0.96592581f,-0.25881904f}, {0.95881975f,-0.28401536f}, +{0.95105648f,-0.30901700f}, {0.94264150f,-0.33380687f}, {0.93358040f,-0.35836795f}, +{0.92387956f,-0.38268346f}, {0.91354543f,-0.40673664f}, {0.90258527f,-0.43051112f}, +{0.89100653f,-0.45399052f}, {0.87881708f,-0.47715878f}, {0.86602545f,-0.50000000f}, +{0.85264015f,-0.52249855f}, {0.83867055f,-0.54463905f}, {0.82412618f,-0.56640625f}, +{0.80901700f,-0.58778524f}, {0.79335332f,-0.60876143f}, {0.77714598f,-0.62932038f}, +{0.76040596f,-0.64944810f}, {0.74314475f,-0.66913062f}, {0.72537434f,-0.68835455f}, +{0.70710677f,-0.70710683f}, {0.68835455f,-0.72537440f}, {0.66913056f,-0.74314487f}, +{0.64944804f,-0.76040596f}, {0.62932038f,-0.77714598f}, {0.60876137f,-0.79335338f}, +{0.58778524f,-0.80901700f}, {0.56640625f,-0.82412618f}, {0.54463899f,-0.83867055f}, +{0.52249849f,-0.85264015f}, {0.49999997f,-0.86602545f}, {0.47715876f,-0.87881708f}, +{0.45399052f,-0.89100653f}, {0.43051103f,-0.90258533f}, {0.40673661f,-0.91354549f}, +{0.38268343f,-0.92387956f}, {0.35836786f,-0.93358046f}, {0.33380681f,-0.94264150f}, +{0.30901697f,-0.95105654f}, {0.28401533f,-0.95881975f}, {0.25881907f,-0.96592581f}, +{0.23344530f,-0.97236991f}, {0.20791166f,-0.97814763f}, {0.18223552f,-0.98325491f}, +{0.15643437f,-0.98768836f}, {0.13052613f,-0.99144489f}, {0.10452842f,-0.99452192f}, +{0.078459084f,-0.99691731f}, {0.052335974f,-0.99862951f}, {0.026176875f,-0.99965733f}, +{-4.3711388e-08f,-1.0000000f}, {-0.026176963f,-0.99965733f}, {-0.052336060f,-0.99862951f}, +{-0.078459173f,-0.99691731f}, {-0.10452851f,-0.99452192f}, {-0.13052621f,-0.99144489f}, +{-0.15643445f,-0.98768836f}, {-0.18223560f,-0.98325491f}, {-0.20791174f,-0.97814757f}, +{-0.23344538f,-0.97236991f}, {-0.25881916f,-0.96592581f}, {-0.28401542f,-0.95881969f}, +{-0.30901703f,-0.95105648f}, {-0.33380687f,-0.94264150f}, {-0.35836795f,-0.93358040f}, +{-0.38268352f,-0.92387950f}, {-0.40673670f,-0.91354543f}, {-0.43051112f,-0.90258527f}, +{-0.45399061f,-0.89100647f}, {-0.47715873f,-0.87881708f}, {-0.50000006f,-0.86602533f}, +{-0.52249867f,-0.85264009f}, {-0.54463905f,-0.83867055f}, {-0.56640631f,-0.82412612f}, +{-0.58778518f,-0.80901700f}, {-0.60876143f,-0.79335332f}, {-0.62932050f,-0.77714586f}, +{-0.64944804f,-0.76040596f}, {-0.66913068f,-0.74314475f}, {-0.68835467f,-0.72537428f}, +{-0.70710677f,-0.70710677f}, {-0.72537446f,-0.68835449f}, {-0.74314493f,-0.66913044f}, +{-0.76040596f,-0.64944804f}, {-0.77714604f,-0.62932026f}, {-0.79335332f,-0.60876143f}, +{-0.80901700f,-0.58778518f}, {-0.82412624f,-0.56640613f}, {-0.83867055f,-0.54463899f}, +{-0.85264021f,-0.52249849f}, {-0.86602539f,-0.50000006f}, {-0.87881714f,-0.47715873f}, +{-0.89100659f,-0.45399037f}, {-0.90258527f,-0.43051112f}, {-0.91354549f,-0.40673658f}, +{-0.92387956f,-0.38268328f}, {-0.93358040f,-0.35836792f}, {-0.94264150f,-0.33380675f}, +{-0.95105654f,-0.30901679f}, {-0.95881975f,-0.28401530f}, {-0.96592587f,-0.25881892f}, +{-0.97236991f,-0.23344538f}, {-0.97814763f,-0.20791161f}, {-0.98325491f,-0.18223536f}, +{-0.98768836f,-0.15643445f}, {-0.99144489f,-0.13052608f}, {-0.99452192f,-0.10452849f}, +{-0.99691737f,-0.078459039f}, {-0.99862957f,-0.052335810f}, {-0.99965733f,-0.026176952f}, +{1.0000000f,-0.0000000f}, {0.99922901f,-0.039259817f}, {0.99691731f,-0.078459099f}, +{0.99306846f,-0.11753740f}, {0.98768836f,-0.15643448f}, {0.98078525f,-0.19509032f}, +{0.97236991f,-0.23344538f}, {0.96245521f,-0.27144045f}, {0.95105648f,-0.30901700f}, +{0.93819129f,-0.34611708f}, {0.92387956f,-0.38268346f}, {0.90814316f,-0.41865975f}, +{0.89100653f,-0.45399052f}, {0.87249601f,-0.48862126f}, {0.85264015f,-0.52249855f}, +{0.83146960f,-0.55557024f}, {0.80901700f,-0.58778524f}, {0.78531694f,-0.61909395f}, +{0.76040596f,-0.64944810f}, {0.73432249f,-0.67880076f}, {0.70710677f,-0.70710683f}, +{0.67880070f,-0.73432255f}, {0.64944804f,-0.76040596f}, {0.61909395f,-0.78531694f}, +{0.58778524f,-0.80901700f}, {0.55557019f,-0.83146960f}, {0.52249849f,-0.85264015f}, +{0.48862118f,-0.87249601f}, {0.45399052f,-0.89100653f}, {0.41865975f,-0.90814316f}, +{0.38268343f,-0.92387956f}, {0.34611705f,-0.93819135f}, {0.30901697f,-0.95105654f}, +{0.27144039f,-0.96245527f}, {0.23344530f,-0.97236991f}, {0.19509023f,-0.98078531f}, +{0.15643437f,-0.98768836f}, {0.11753740f,-0.99306846f}, {0.078459084f,-0.99691731f}, +{0.039259788f,-0.99922901f}, {-4.3711388e-08f,-1.0000000f}, {-0.039259877f,-0.99922901f}, +{-0.078459173f,-0.99691731f}, {-0.11753749f,-0.99306846f}, {-0.15643445f,-0.98768836f}, +{-0.19509032f,-0.98078525f}, {-0.23344538f,-0.97236991f}, {-0.27144048f,-0.96245521f}, +{-0.30901703f,-0.95105648f}, {-0.34611711f,-0.93819129f}, {-0.38268352f,-0.92387950f}, +{-0.41865984f,-0.90814310f}, {-0.45399061f,-0.89100647f}, {-0.48862135f,-0.87249595f}, +{-0.52249867f,-0.85264009f}, {-0.55557036f,-0.83146954f}, {-0.58778518f,-0.80901700f}, +{-0.61909389f,-0.78531694f}, {-0.64944804f,-0.76040596f}, {-0.67880076f,-0.73432249f}, +{-0.70710677f,-0.70710677f}, {-0.73432249f,-0.67880070f}, {-0.76040596f,-0.64944804f}, +{-0.78531694f,-0.61909389f}, {-0.80901700f,-0.58778518f}, {-0.83146966f,-0.55557019f}, +{-0.85264021f,-0.52249849f}, {-0.87249607f,-0.48862115f}, {-0.89100659f,-0.45399037f}, +{-0.90814322f,-0.41865960f}, {-0.92387956f,-0.38268328f}, {-0.93819135f,-0.34611690f}, +{-0.95105654f,-0.30901679f}, {-0.96245521f,-0.27144048f}, {-0.97236991f,-0.23344538f}, +{-0.98078531f,-0.19509031f}, {-0.98768836f,-0.15643445f}, {-0.99306846f,-0.11753736f}, +{-0.99691737f,-0.078459039f}, {-0.99922901f,-0.039259743f}, {-1.0000000f,8.7422777e-08f}, +{-0.99922901f,0.039259918f}, {-0.99691731f,0.078459218f}, {-0.99306846f,0.11753753f}, +{-0.98768830f,0.15643461f}, {-0.98078525f,0.19509049f}, {-0.97236985f,0.23344554f}, +{-0.96245515f,0.27144065f}, {-0.95105654f,0.30901697f}, {-0.93819135f,0.34611705f}, +{-0.92387956f,0.38268346f}, {-0.90814316f,0.41865975f}, {-0.89100653f,0.45399055f}, +{-0.87249601f,0.48862129f}, {-0.85264015f,0.52249861f}, {-0.83146960f,0.55557030f}, +{-0.80901694f,0.58778536f}, {-0.78531688f,0.61909401f}, {-0.76040590f,0.64944816f}, +{-0.73432243f,0.67880082f}, {-0.70710665f,0.70710689f}, {-0.67880058f,0.73432261f}, +{-0.64944792f,0.76040608f}, {-0.61909378f,0.78531706f}, {-0.58778507f,0.80901712f}, +{-0.55557001f,0.83146977f}, {-0.52249837f,0.85264033f}, {-0.48862100f,0.87249613f}, +{-0.45399022f,0.89100665f}, {-0.41865945f,0.90814328f}, {-0.38268313f,0.92387968f}, +{-0.34611672f,0.93819147f}, {-0.30901709f,0.95105648f}, {-0.27144054f,0.96245521f}, +{-0.23344545f,0.97236991f}, {-0.19509038f,0.98078525f}, {-0.15643452f,0.98768830f}, +{-0.11753743f,0.99306846f}, {-0.078459114f,0.99691731f}, {-0.039259821f,0.99922901f}, +}; +static const ne10_fft_cpx_float32_t ne10_twiddles_240[240] = { +{1.0000000f,0.0000000f}, {1.0000000f,-0.0000000f}, {1.0000000f,-0.0000000f}, +{1.0000000f,-0.0000000f}, {0.91354543f,-0.40673664f}, {0.66913056f,-0.74314487f}, +{1.0000000f,-0.0000000f}, {0.66913056f,-0.74314487f}, {-0.10452851f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.30901697f,-0.95105654f}, {-0.80901700f,-0.58778518f}, +{1.0000000f,-0.0000000f}, {-0.10452851f,-0.99452192f}, {-0.97814757f,0.20791179f}, +{1.0000000f,-0.0000000f}, {0.99452192f,-0.10452846f}, {0.97814763f,-0.20791170f}, +{0.95105648f,-0.30901700f}, {0.91354543f,-0.40673664f}, {0.86602545f,-0.50000000f}, +{0.80901700f,-0.58778524f}, {0.74314475f,-0.66913062f}, {0.66913056f,-0.74314487f}, +{0.58778524f,-0.80901700f}, {0.49999997f,-0.86602545f}, {0.40673661f,-0.91354549f}, +{0.30901697f,-0.95105654f}, {0.20791166f,-0.97814763f}, {0.10452842f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.97814763f,-0.20791170f}, {0.91354543f,-0.40673664f}, +{0.80901700f,-0.58778524f}, {0.66913056f,-0.74314487f}, {0.49999997f,-0.86602545f}, +{0.30901697f,-0.95105654f}, {0.10452842f,-0.99452192f}, {-0.10452851f,-0.99452192f}, +{-0.30901703f,-0.95105648f}, {-0.50000006f,-0.86602533f}, {-0.66913068f,-0.74314475f}, +{-0.80901700f,-0.58778518f}, {-0.91354549f,-0.40673658f}, {-0.97814763f,-0.20791161f}, +{1.0000000f,-0.0000000f}, {0.95105648f,-0.30901700f}, {0.80901700f,-0.58778524f}, +{0.58778524f,-0.80901700f}, {0.30901697f,-0.95105654f}, {-4.3711388e-08f,-1.0000000f}, +{-0.30901703f,-0.95105648f}, {-0.58778518f,-0.80901700f}, {-0.80901700f,-0.58778518f}, +{-0.95105654f,-0.30901679f}, {-1.0000000f,8.7422777e-08f}, {-0.95105654f,0.30901697f}, +{-0.80901694f,0.58778536f}, {-0.58778507f,0.80901712f}, {-0.30901709f,0.95105648f}, +{1.0000000f,-0.0000000f}, {0.99965733f,-0.026176950f}, {0.99862951f,-0.052335959f}, +{0.99691731f,-0.078459099f}, {0.99452192f,-0.10452846f}, {0.99144489f,-0.13052620f}, +{0.98768836f,-0.15643448f}, {0.98325491f,-0.18223552f}, {0.97814763f,-0.20791170f}, +{0.97236991f,-0.23344538f}, {0.96592581f,-0.25881904f}, {0.95881975f,-0.28401536f}, +{0.95105648f,-0.30901700f}, {0.94264150f,-0.33380687f}, {0.93358040f,-0.35836795f}, +{0.92387956f,-0.38268346f}, {0.91354543f,-0.40673664f}, {0.90258527f,-0.43051112f}, +{0.89100653f,-0.45399052f}, {0.87881708f,-0.47715878f}, {0.86602545f,-0.50000000f}, +{0.85264015f,-0.52249855f}, {0.83867055f,-0.54463905f}, {0.82412618f,-0.56640625f}, +{0.80901700f,-0.58778524f}, {0.79335332f,-0.60876143f}, {0.77714598f,-0.62932038f}, +{0.76040596f,-0.64944810f}, {0.74314475f,-0.66913062f}, {0.72537434f,-0.68835455f}, +{0.70710677f,-0.70710683f}, {0.68835455f,-0.72537440f}, {0.66913056f,-0.74314487f}, +{0.64944804f,-0.76040596f}, {0.62932038f,-0.77714598f}, {0.60876137f,-0.79335338f}, +{0.58778524f,-0.80901700f}, {0.56640625f,-0.82412618f}, {0.54463899f,-0.83867055f}, +{0.52249849f,-0.85264015f}, {0.49999997f,-0.86602545f}, {0.47715876f,-0.87881708f}, +{0.45399052f,-0.89100653f}, {0.43051103f,-0.90258533f}, {0.40673661f,-0.91354549f}, +{0.38268343f,-0.92387956f}, {0.35836786f,-0.93358046f}, {0.33380681f,-0.94264150f}, +{0.30901697f,-0.95105654f}, {0.28401533f,-0.95881975f}, {0.25881907f,-0.96592581f}, +{0.23344530f,-0.97236991f}, {0.20791166f,-0.97814763f}, {0.18223552f,-0.98325491f}, +{0.15643437f,-0.98768836f}, {0.13052613f,-0.99144489f}, {0.10452842f,-0.99452192f}, +{0.078459084f,-0.99691731f}, {0.052335974f,-0.99862951f}, {0.026176875f,-0.99965733f}, +{1.0000000f,-0.0000000f}, {0.99862951f,-0.052335959f}, {0.99452192f,-0.10452846f}, +{0.98768836f,-0.15643448f}, {0.97814763f,-0.20791170f}, {0.96592581f,-0.25881904f}, +{0.95105648f,-0.30901700f}, {0.93358040f,-0.35836795f}, {0.91354543f,-0.40673664f}, +{0.89100653f,-0.45399052f}, {0.86602545f,-0.50000000f}, {0.83867055f,-0.54463905f}, +{0.80901700f,-0.58778524f}, {0.77714598f,-0.62932038f}, {0.74314475f,-0.66913062f}, +{0.70710677f,-0.70710683f}, {0.66913056f,-0.74314487f}, {0.62932038f,-0.77714598f}, +{0.58778524f,-0.80901700f}, {0.54463899f,-0.83867055f}, {0.49999997f,-0.86602545f}, +{0.45399052f,-0.89100653f}, {0.40673661f,-0.91354549f}, {0.35836786f,-0.93358046f}, +{0.30901697f,-0.95105654f}, {0.25881907f,-0.96592581f}, {0.20791166f,-0.97814763f}, +{0.15643437f,-0.98768836f}, {0.10452842f,-0.99452192f}, {0.052335974f,-0.99862951f}, +{-4.3711388e-08f,-1.0000000f}, {-0.052336060f,-0.99862951f}, {-0.10452851f,-0.99452192f}, +{-0.15643445f,-0.98768836f}, {-0.20791174f,-0.97814757f}, {-0.25881916f,-0.96592581f}, +{-0.30901703f,-0.95105648f}, {-0.35836795f,-0.93358040f}, {-0.40673670f,-0.91354543f}, +{-0.45399061f,-0.89100647f}, {-0.50000006f,-0.86602533f}, {-0.54463905f,-0.83867055f}, +{-0.58778518f,-0.80901700f}, {-0.62932050f,-0.77714586f}, {-0.66913068f,-0.74314475f}, +{-0.70710677f,-0.70710677f}, {-0.74314493f,-0.66913044f}, {-0.77714604f,-0.62932026f}, +{-0.80901700f,-0.58778518f}, {-0.83867055f,-0.54463899f}, {-0.86602539f,-0.50000006f}, +{-0.89100659f,-0.45399037f}, {-0.91354549f,-0.40673658f}, {-0.93358040f,-0.35836792f}, +{-0.95105654f,-0.30901679f}, {-0.96592587f,-0.25881892f}, {-0.97814763f,-0.20791161f}, +{-0.98768836f,-0.15643445f}, {-0.99452192f,-0.10452849f}, {-0.99862957f,-0.052335810f}, +{1.0000000f,-0.0000000f}, {0.99691731f,-0.078459099f}, {0.98768836f,-0.15643448f}, +{0.97236991f,-0.23344538f}, {0.95105648f,-0.30901700f}, {0.92387956f,-0.38268346f}, +{0.89100653f,-0.45399052f}, {0.85264015f,-0.52249855f}, {0.80901700f,-0.58778524f}, +{0.76040596f,-0.64944810f}, {0.70710677f,-0.70710683f}, {0.64944804f,-0.76040596f}, +{0.58778524f,-0.80901700f}, {0.52249849f,-0.85264015f}, {0.45399052f,-0.89100653f}, +{0.38268343f,-0.92387956f}, {0.30901697f,-0.95105654f}, {0.23344530f,-0.97236991f}, +{0.15643437f,-0.98768836f}, {0.078459084f,-0.99691731f}, {-4.3711388e-08f,-1.0000000f}, +{-0.078459173f,-0.99691731f}, {-0.15643445f,-0.98768836f}, {-0.23344538f,-0.97236991f}, +{-0.30901703f,-0.95105648f}, {-0.38268352f,-0.92387950f}, {-0.45399061f,-0.89100647f}, +{-0.52249867f,-0.85264009f}, {-0.58778518f,-0.80901700f}, {-0.64944804f,-0.76040596f}, +{-0.70710677f,-0.70710677f}, {-0.76040596f,-0.64944804f}, {-0.80901700f,-0.58778518f}, +{-0.85264021f,-0.52249849f}, {-0.89100659f,-0.45399037f}, {-0.92387956f,-0.38268328f}, +{-0.95105654f,-0.30901679f}, {-0.97236991f,-0.23344538f}, {-0.98768836f,-0.15643445f}, +{-0.99691737f,-0.078459039f}, {-1.0000000f,8.7422777e-08f}, {-0.99691731f,0.078459218f}, +{-0.98768830f,0.15643461f}, {-0.97236985f,0.23344554f}, {-0.95105654f,0.30901697f}, +{-0.92387956f,0.38268346f}, {-0.89100653f,0.45399055f}, {-0.85264015f,0.52249861f}, +{-0.80901694f,0.58778536f}, {-0.76040590f,0.64944816f}, {-0.70710665f,0.70710689f}, +{-0.64944792f,0.76040608f}, {-0.58778507f,0.80901712f}, {-0.52249837f,0.85264033f}, +{-0.45399022f,0.89100665f}, {-0.38268313f,0.92387968f}, {-0.30901709f,0.95105648f}, +{-0.23344545f,0.97236991f}, {-0.15643452f,0.98768830f}, {-0.078459114f,0.99691731f}, +}; +static const ne10_fft_cpx_float32_t ne10_twiddles_120[120] = { +{1.0000000f,0.0000000f}, {1.0000000f,-0.0000000f}, {1.0000000f,-0.0000000f}, +{1.0000000f,-0.0000000f}, {0.91354543f,-0.40673664f}, {0.66913056f,-0.74314487f}, +{1.0000000f,-0.0000000f}, {0.66913056f,-0.74314487f}, {-0.10452851f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.30901697f,-0.95105654f}, {-0.80901700f,-0.58778518f}, +{1.0000000f,-0.0000000f}, {-0.10452851f,-0.99452192f}, {-0.97814757f,0.20791179f}, +{1.0000000f,-0.0000000f}, {0.97814763f,-0.20791170f}, {0.91354543f,-0.40673664f}, +{0.80901700f,-0.58778524f}, {0.66913056f,-0.74314487f}, {0.49999997f,-0.86602545f}, +{0.30901697f,-0.95105654f}, {0.10452842f,-0.99452192f}, {-0.10452851f,-0.99452192f}, +{-0.30901703f,-0.95105648f}, {-0.50000006f,-0.86602533f}, {-0.66913068f,-0.74314475f}, +{-0.80901700f,-0.58778518f}, {-0.91354549f,-0.40673658f}, {-0.97814763f,-0.20791161f}, +{1.0000000f,-0.0000000f}, {0.99862951f,-0.052335959f}, {0.99452192f,-0.10452846f}, +{0.98768836f,-0.15643448f}, {0.97814763f,-0.20791170f}, {0.96592581f,-0.25881904f}, +{0.95105648f,-0.30901700f}, {0.93358040f,-0.35836795f}, {0.91354543f,-0.40673664f}, +{0.89100653f,-0.45399052f}, {0.86602545f,-0.50000000f}, {0.83867055f,-0.54463905f}, +{0.80901700f,-0.58778524f}, {0.77714598f,-0.62932038f}, {0.74314475f,-0.66913062f}, +{0.70710677f,-0.70710683f}, {0.66913056f,-0.74314487f}, {0.62932038f,-0.77714598f}, +{0.58778524f,-0.80901700f}, {0.54463899f,-0.83867055f}, {0.49999997f,-0.86602545f}, +{0.45399052f,-0.89100653f}, {0.40673661f,-0.91354549f}, {0.35836786f,-0.93358046f}, +{0.30901697f,-0.95105654f}, {0.25881907f,-0.96592581f}, {0.20791166f,-0.97814763f}, +{0.15643437f,-0.98768836f}, {0.10452842f,-0.99452192f}, {0.052335974f,-0.99862951f}, +{1.0000000f,-0.0000000f}, {0.99452192f,-0.10452846f}, {0.97814763f,-0.20791170f}, +{0.95105648f,-0.30901700f}, {0.91354543f,-0.40673664f}, {0.86602545f,-0.50000000f}, +{0.80901700f,-0.58778524f}, {0.74314475f,-0.66913062f}, {0.66913056f,-0.74314487f}, +{0.58778524f,-0.80901700f}, {0.49999997f,-0.86602545f}, {0.40673661f,-0.91354549f}, +{0.30901697f,-0.95105654f}, {0.20791166f,-0.97814763f}, {0.10452842f,-0.99452192f}, +{-4.3711388e-08f,-1.0000000f}, {-0.10452851f,-0.99452192f}, {-0.20791174f,-0.97814757f}, +{-0.30901703f,-0.95105648f}, {-0.40673670f,-0.91354543f}, {-0.50000006f,-0.86602533f}, +{-0.58778518f,-0.80901700f}, {-0.66913068f,-0.74314475f}, {-0.74314493f,-0.66913044f}, +{-0.80901700f,-0.58778518f}, {-0.86602539f,-0.50000006f}, {-0.91354549f,-0.40673658f}, +{-0.95105654f,-0.30901679f}, {-0.97814763f,-0.20791161f}, {-0.99452192f,-0.10452849f}, +{1.0000000f,-0.0000000f}, {0.98768836f,-0.15643448f}, {0.95105648f,-0.30901700f}, +{0.89100653f,-0.45399052f}, {0.80901700f,-0.58778524f}, {0.70710677f,-0.70710683f}, +{0.58778524f,-0.80901700f}, {0.45399052f,-0.89100653f}, {0.30901697f,-0.95105654f}, +{0.15643437f,-0.98768836f}, {-4.3711388e-08f,-1.0000000f}, {-0.15643445f,-0.98768836f}, +{-0.30901703f,-0.95105648f}, {-0.45399061f,-0.89100647f}, {-0.58778518f,-0.80901700f}, +{-0.70710677f,-0.70710677f}, {-0.80901700f,-0.58778518f}, {-0.89100659f,-0.45399037f}, +{-0.95105654f,-0.30901679f}, {-0.98768836f,-0.15643445f}, {-1.0000000f,8.7422777e-08f}, +{-0.98768830f,0.15643461f}, {-0.95105654f,0.30901697f}, {-0.89100653f,0.45399055f}, +{-0.80901694f,0.58778536f}, {-0.70710665f,0.70710689f}, {-0.58778507f,0.80901712f}, +{-0.45399022f,0.89100665f}, {-0.30901709f,0.95105648f}, {-0.15643452f,0.98768830f}, +}; +static const ne10_fft_cpx_float32_t ne10_twiddles_60[60] = { +{1.0000000f,0.0000000f}, {1.0000000f,-0.0000000f}, {1.0000000f,-0.0000000f}, +{1.0000000f,-0.0000000f}, {0.91354543f,-0.40673664f}, {0.66913056f,-0.74314487f}, +{1.0000000f,-0.0000000f}, {0.66913056f,-0.74314487f}, {-0.10452851f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.30901697f,-0.95105654f}, {-0.80901700f,-0.58778518f}, +{1.0000000f,-0.0000000f}, {-0.10452851f,-0.99452192f}, {-0.97814757f,0.20791179f}, +{1.0000000f,-0.0000000f}, {0.99452192f,-0.10452846f}, {0.97814763f,-0.20791170f}, +{0.95105648f,-0.30901700f}, {0.91354543f,-0.40673664f}, {0.86602545f,-0.50000000f}, +{0.80901700f,-0.58778524f}, {0.74314475f,-0.66913062f}, {0.66913056f,-0.74314487f}, +{0.58778524f,-0.80901700f}, {0.49999997f,-0.86602545f}, {0.40673661f,-0.91354549f}, +{0.30901697f,-0.95105654f}, {0.20791166f,-0.97814763f}, {0.10452842f,-0.99452192f}, +{1.0000000f,-0.0000000f}, {0.97814763f,-0.20791170f}, {0.91354543f,-0.40673664f}, +{0.80901700f,-0.58778524f}, {0.66913056f,-0.74314487f}, {0.49999997f,-0.86602545f}, +{0.30901697f,-0.95105654f}, {0.10452842f,-0.99452192f}, {-0.10452851f,-0.99452192f}, +{-0.30901703f,-0.95105648f}, {-0.50000006f,-0.86602533f}, {-0.66913068f,-0.74314475f}, +{-0.80901700f,-0.58778518f}, {-0.91354549f,-0.40673658f}, {-0.97814763f,-0.20791161f}, +{1.0000000f,-0.0000000f}, {0.95105648f,-0.30901700f}, {0.80901700f,-0.58778524f}, +{0.58778524f,-0.80901700f}, {0.30901697f,-0.95105654f}, {-4.3711388e-08f,-1.0000000f}, +{-0.30901703f,-0.95105648f}, {-0.58778518f,-0.80901700f}, {-0.80901700f,-0.58778518f}, +{-0.95105654f,-0.30901679f}, {-1.0000000f,8.7422777e-08f}, {-0.95105654f,0.30901697f}, +{-0.80901694f,0.58778536f}, {-0.58778507f,0.80901712f}, {-0.30901709f,0.95105648f}, +}; +static const ne10_fft_state_float32_t ne10_fft_state_float32_t_480 = { +120, +(ne10_int32_t *)ne10_factors_480, +(ne10_fft_cpx_float32_t *)ne10_twiddles_480, +NULL, +(ne10_fft_cpx_float32_t *)&ne10_twiddles_480[120], +/* is_forward_scaled = true */ +(ne10_int32_t) 1, +/* is_backward_scaled = false */ +(ne10_int32_t) 0, +}; +static const arch_fft_state cfg_arch_480 = { +1, +(void *)&ne10_fft_state_float32_t_480, +}; + +static const ne10_fft_state_float32_t ne10_fft_state_float32_t_240 = { +60, +(ne10_int32_t *)ne10_factors_240, +(ne10_fft_cpx_float32_t *)ne10_twiddles_240, +NULL, +(ne10_fft_cpx_float32_t *)&ne10_twiddles_240[60], +/* is_forward_scaled = true */ +(ne10_int32_t) 1, +/* is_backward_scaled = false */ +(ne10_int32_t) 0, +}; +static const arch_fft_state cfg_arch_240 = { +1, +(void *)&ne10_fft_state_float32_t_240, +}; + +static const ne10_fft_state_float32_t ne10_fft_state_float32_t_120 = { +30, +(ne10_int32_t *)ne10_factors_120, +(ne10_fft_cpx_float32_t *)ne10_twiddles_120, +NULL, +(ne10_fft_cpx_float32_t *)&ne10_twiddles_120[30], +/* is_forward_scaled = true */ +(ne10_int32_t) 1, +/* is_backward_scaled = false */ +(ne10_int32_t) 0, +}; +static const arch_fft_state cfg_arch_120 = { +1, +(void *)&ne10_fft_state_float32_t_120, +}; + +static const ne10_fft_state_float32_t ne10_fft_state_float32_t_60 = { +15, +(ne10_int32_t *)ne10_factors_60, +(ne10_fft_cpx_float32_t *)ne10_twiddles_60, +NULL, +(ne10_fft_cpx_float32_t *)&ne10_twiddles_60[15], +/* is_forward_scaled = true */ +(ne10_int32_t) 1, +/* is_backward_scaled = false */ +(ne10_int32_t) 0, +}; +static const arch_fft_state cfg_arch_60 = { +1, +(void *)&ne10_fft_state_float32_t_60, +}; + +#endif /* end NE10_FFT_PARAMS48000_960 */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/meson.build new file mode 100644 index 000000000..0e6d2e627 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/meson.build @@ -0,0 +1,19 @@ +tests = [ + 'test_unit_types', + 'test_unit_mathops', + 'test_unit_entropy', + 'test_unit_laplace', + 'test_unit_dft', + 'test_unit_mdct', + 'test_unit_rotation', + 'test_unit_cwrs32', +] + +foreach test_name : tests + exe = executable(test_name, '@0@.c'.format(test_name), + include_directories : opus_includes, + link_with : [celt_lib, celt_static_libs], + dependencies : libm, + install : false) + test(test_name, exe) +endforeach diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_cwrs32.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_cwrs32.c new file mode 100644 index 000000000..36dd8af5f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_cwrs32.c @@ -0,0 +1,161 @@ +/* Copyright (c) 2008-2011 Xiph.Org Foundation, Mozilla Corporation, + Gregory Maxwell + Written by Jean-Marc Valin, Gregory Maxwell, and Timothy B. Terriberry */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#ifndef CUSTOM_MODES +#define CUSTOM_MODES +#else +#define TEST_CUSTOM_MODES +#endif + +#define CELT_C +#include "stack_alloc.h" +#include "entenc.c" +#include "entdec.c" +#include "entcode.c" +#include "cwrs.c" +#include "mathops.c" +#include "rate.h" + +#define NMAX (240) +#define KMAX (128) + +#ifdef TEST_CUSTOM_MODES + +#define NDIMS (44) +static const int pn[NDIMS]={ + 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 18, 20, 22, + 24, 26, 28, 30, 32, 36, 40, 44, 48, + 52, 56, 60, 64, 72, 80, 88, 96, 104, + 112, 120, 128, 144, 160, 176, 192, 208 +}; +static const int pkmax[NDIMS]={ + 128, 128, 128, 128, 88, 52, 36, 26, 22, + 18, 16, 15, 13, 12, 12, 11, 10, 9, + 9, 8, 8, 7, 7, 7, 7, 6, 6, + 6, 6, 6, 5, 5, 5, 5, 5, 5, + 4, 4, 4, 4, 4, 4, 4, 4 +}; + +#else /* TEST_CUSTOM_MODES */ + +#define NDIMS (22) +static const int pn[NDIMS]={ + 2, 3, 4, 6, 8, 9, 11, 12, 16, + 18, 22, 24, 32, 36, 44, 48, 64, 72, + 88, 96, 144, 176 +}; +static const int pkmax[NDIMS]={ + 128, 128, 128, 88, 36, 26, 18, 16, 12, + 11, 9, 9, 7, 7, 6, 6, 5, 5, + 5, 5, 4, 4 +}; + +#endif + +int main(void){ + int t; + int n; + ALLOC_STACK; + for(t=0;tpkmax[t])break; + printf("Testing CWRS with N=%i, K=%i...\n",n,k); +#if defined(SMALL_FOOTPRINT) + nc=ncwrs_urow(n,k,uu); +#else + nc=CELT_PVQ_V(n,k); +#endif + inc=nc/20000; + if(inc<1)inc=1; + for(i=0;i");*/ +#if defined(SMALL_FOOTPRINT) + ii=icwrs(n,k,&v,y,u); +#else + ii=icwrs(n,y); + v=CELT_PVQ_V(n,k); +#endif + if(ii!=i){ + fprintf(stderr,"Combination-index mismatch (%lu!=%lu).\n", + (long)ii,(long)i); + return 1; + } + if(v!=nc){ + fprintf(stderr,"Combination count mismatch (%lu!=%lu).\n", + (long)v,(long)nc); + return 2; + } + /*printf(" %6u\n",i);*/ + } + /*printf("\n");*/ + } + } + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_dft.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_dft.c new file mode 100644 index 000000000..70f8f4937 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_dft.c @@ -0,0 +1,179 @@ +/* Copyright (c) 2008 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "stack_alloc.h" +#include "kiss_fft.h" +#include "mathops.h" +#include "modes.h" + +#ifndef M_PI +#define M_PI 3.141592653 +#endif + +int ret = 0; + +void check(kiss_fft_cpx * in,kiss_fft_cpx * out,int nfft,int isinverse) +{ + int bin,k; + double errpow=0,sigpow=0, snr; + + for (bin=0;binmdct.kfft[id]; +#endif + + in = (kiss_fft_cpx*)malloc(buflen); + out = (kiss_fft_cpx*)malloc(buflen); + + for (k=0;k1) { + int k; + for (k=1;k +#include +#include +#include +#define CELT_C +#include "entcode.h" +#include "entenc.h" +#include "entdec.h" +#include + +#include "entenc.c" +#include "entdec.c" +#include "entcode.c" + +#ifndef M_LOG2E +# define M_LOG2E 1.4426950408889634074 +#endif +#define DATA_SIZE 10000000 +#define DATA_SIZE2 10000 + +int main(int _argc,char **_argv){ + ec_enc enc; + ec_dec dec; + long nbits; + long nbits2; + double entropy; + int ft; + int ftb; + int sz; + int i; + int ret; + unsigned int sym; + unsigned int seed; + unsigned char *ptr; + const char *env_seed; + ret=0; + entropy=0; + if (_argc > 2) { + fprintf(stderr, "Usage: %s []\n", _argv[0]); + return 1; + } + env_seed = getenv("SEED"); + if (_argc > 1) + seed = atoi(_argv[1]); + else if (env_seed) + seed = atoi(env_seed); + else + seed = time(NULL); + /*Testing encoding of raw bit values.*/ + ptr = (unsigned char *)malloc(DATA_SIZE); + ec_enc_init(&enc,ptr, DATA_SIZE); + for(ft=2;ft<1024;ft++){ + for(i=0;i>(rand()%11U))+1U)+10; + sz=rand()/((RAND_MAX>>(rand()%9U))+1U); + data=(unsigned *)malloc(sz*sizeof(*data)); + tell=(unsigned *)malloc((sz+1)*sizeof(*tell)); + ec_enc_init(&enc,ptr,DATA_SIZE2); + zeros = rand()%13==0; + tell[0]=ec_tell_frac(&enc); + for(j=0;j>(rand()%9U))+1U); + logp1=(unsigned *)malloc(sz*sizeof(*logp1)); + data=(unsigned *)malloc(sz*sizeof(*data)); + tell=(unsigned *)malloc((sz+1)*sizeof(*tell)); + enc_method=(unsigned *)malloc(sz*sizeof(*enc_method)); + ec_enc_init(&enc,ptr,DATA_SIZE2); + tell[0]=ec_tell_frac(&enc); + for(j=0;j>1)+1); + logp1[j]=(rand()%15)+1; + enc_method[j]=rand()/((RAND_MAX>>2)+1); + switch(enc_method[j]){ + case 0:{ + ec_encode(&enc,data[j]?(1<>2)+1); + switch(dec_method){ + case 0:{ + fs=ec_decode(&dec,1<=(1<=(1< +#include +#define CELT_C +#include "laplace.h" +#include "stack_alloc.h" + +#include "entenc.c" +#include "entdec.c" +#include "entcode.c" +#include "laplace.c" + +#define DATA_SIZE 40000 + +int ec_laplace_get_start_freq(int decay) +{ + opus_uint32 ft = 32768 - LAPLACE_MINP*(2*LAPLACE_NMIN+1); + int fs = (ft*(16384-decay))/(16384+decay); + return fs+LAPLACE_MINP; +} + +int main(void) +{ + int i; + int ret = 0; + ec_enc enc; + ec_dec dec; + unsigned char *ptr; + int val[10000], decay[10000]; + ALLOC_STACK; + ptr = (unsigned char *)malloc(DATA_SIZE); + ec_enc_init(&enc,ptr,DATA_SIZE); + + val[0] = 3; decay[0] = 6000; + val[1] = 0; decay[1] = 5800; + val[2] = -1; decay[2] = 5600; + for (i=3;i<10000;i++) + { + val[i] = rand()%15-7; + decay[i] = rand()%11000+5000; + } + for (i=0;i<10000;i++) + ec_laplace_encode(&enc, &val[i], + ec_laplace_get_start_freq(decay[i]), decay[i]); + + ec_enc_done(&enc); + + ec_dec_init(&dec,ec_get_buffer(&enc),ec_range_bytes(&enc)); + + for (i=0;i<10000;i++) + { + int d = ec_laplace_decode(&dec, + ec_laplace_get_start_freq(decay[i]), decay[i]); + if (d != val[i]) + { + fprintf (stderr, "Got %d instead of %d\n", d, val[i]); + ret = 1; + } + } + + free(ptr); + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mathops.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mathops.c new file mode 100644 index 000000000..0615448db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mathops.c @@ -0,0 +1,266 @@ +/* Copyright (c) 2008-2011 Xiph.Org Foundation, Mozilla Corporation, + Gregory Maxwell + Written by Jean-Marc Valin, Gregory Maxwell, and Timothy B. Terriberry */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifndef CUSTOM_MODES +#define CUSTOM_MODES +#endif + +#include +#include +#include "mathops.h" +#include "bands.h" + +#ifdef FIXED_POINT +#define WORD "%d" +#else +#define WORD "%f" +#endif + +int ret = 0; + +void testdiv(void) +{ + opus_int32 i; + for (i=1;i<=327670;i++) + { + double prod; + opus_val32 val; + val = celt_rcp(i); +#ifdef FIXED_POINT + prod = (1./32768./65526.)*val*i; +#else + prod = val*i; +#endif + if (fabs(prod-1) > .00025) + { + fprintf (stderr, "div failed: 1/%d="WORD" (product = %f)\n", i, val, prod); + ret = 1; + } + } +} + +void testsqrt(void) +{ + opus_int32 i; + for (i=1;i<=1000000000;i++) + { + double ratio; + opus_val16 val; + val = celt_sqrt(i); + ratio = val/sqrt(i); + if (fabs(ratio - 1) > .0005 && fabs(val-sqrt(i)) > 2) + { + fprintf (stderr, "sqrt failed: sqrt(%d)="WORD" (ratio = %f)\n", i, val, ratio); + ret = 1; + } + i+= i>>10; + } +} + +void testbitexactcos(void) +{ + int i; + opus_int32 min_d,max_d,last,chk; + chk=max_d=0; + last=min_d=32767; + for(i=64;i<=16320;i++) + { + opus_int32 d; + opus_int32 q=bitexact_cos(i); + chk ^= q*i; + d = last - q; + if (d>max_d)max_d=d; + if (dmax_d)max_d=d; + if (d0.0009) + { + fprintf (stderr, "celt_log2 failed: fabs((1.442695040888963387*log(x))-celt_log2(x))>0.001 (x = %f, error = %f)\n", x,error); + ret = 1; + } + } +} + +void testexp2(void) +{ + float x; + for (x=-11.0;x<24.0;x+=0.0007f) + { + float error = fabs(x-(1.442695040888963387*log(celt_exp2(x)))); + if (error>0.0002) + { + fprintf (stderr, "celt_exp2 failed: fabs(x-(1.442695040888963387*log(celt_exp2(x))))>0.0005 (x = %f, error = %f)\n", x,error); + ret = 1; + } + } +} + +void testexp2log2(void) +{ + float x; + for (x=-11.0;x<24.0;x+=0.0007f) + { + float error = fabs(x-(celt_log2(celt_exp2(x)))); + if (error>0.001) + { + fprintf (stderr, "celt_log2/celt_exp2 failed: fabs(x-(celt_log2(celt_exp2(x))))>0.001 (x = %f, error = %f)\n", x,error); + ret = 1; + } + } +} +#else +void testlog2(void) +{ + opus_val32 x; + for (x=8;x<1073741824;x+=(x>>3)) + { + float error = fabs((1.442695040888963387*log(x/16384.0))-celt_log2(x)/1024.0); + if (error>0.003) + { + fprintf (stderr, "celt_log2 failed: x = %ld, error = %f\n", (long)x,error); + ret = 1; + } + } +} + +void testexp2(void) +{ + opus_val16 x; + for (x=-32768;x<15360;x++) + { + float error1 = fabs(x/1024.0-(1.442695040888963387*log(celt_exp2(x)/65536.0))); + float error2 = fabs(exp(0.6931471805599453094*x/1024.0)-celt_exp2(x)/65536.0); + if (error1>0.0002&&error2>0.00004) + { + fprintf (stderr, "celt_exp2 failed: x = "WORD", error1 = %f, error2 = %f\n", x,error1,error2); + ret = 1; + } + } +} + +void testexp2log2(void) +{ + opus_val32 x; + for (x=8;x<65536;x+=(x>>3)) + { + float error = fabs(x-0.25*celt_exp2(celt_log2(x)))/16384; + if (error>0.004) + { + fprintf (stderr, "celt_log2/celt_exp2 failed: fabs(x-(celt_exp2(celt_log2(x))))>0.001 (x = %ld, error = %f)\n", (long)x,error); + ret = 1; + } + } +} + +void testilog2(void) +{ + opus_val32 x; + for (x=1;x<=268435455;x+=127) + { + opus_val32 lg; + opus_val32 y; + + lg = celt_ilog2(x); + if (lg<0 || lg>=31) + { + printf("celt_ilog2 failed: 0<=celt_ilog2(x)<31 (x = %d, celt_ilog2(x) = %d)\n",x,lg); + ret = 1; + } + y = 1<>1)>=y) + { + printf("celt_ilog2 failed: 2**celt_ilog2(x)<=x<2**(celt_ilog2(x)+1) (x = %d, 2**celt_ilog2(x) = %d)\n",x,y); + ret = 1; + } + } +} +#endif + +int main(void) +{ + testbitexactcos(); + testbitexactlog2tan(); + testdiv(); + testsqrt(); + testlog2(); + testexp2(); + testexp2log2(); +#ifdef FIXED_POINT + testilog2(); +#endif + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mdct.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mdct.c new file mode 100644 index 000000000..4a563ccfe --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_mdct.c @@ -0,0 +1,227 @@ +/* Copyright (c) 2008-2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "mdct.h" +#include "stack_alloc.h" +#include "kiss_fft.h" +#include "mdct.h" +#include "modes.h" + +#ifndef M_PI +#define M_PI 3.141592653 +#endif + +int ret = 0; +void check(kiss_fft_scalar * in,kiss_fft_scalar * out,int nfft,int isinverse) +{ + int bin,k; + double errpow=0,sigpow=0; + double snr; + for (bin=0;binmdct; +#endif + + in = (kiss_fft_scalar*)malloc(buflen); + in_copy = (kiss_fft_scalar*)malloc(buflen); + out = (kiss_fft_scalar*)malloc(buflen); + window = (opus_val16*)malloc(sizeof(opus_val16)*nfft/2); + + for (k=0;k1) { + int k; + for (k=1;k +#include +#include "vq.h" +#include "bands.h" +#include "stack_alloc.h" +#include + + +#define MAX_SIZE 100 + +int ret=0; +void test_rotation(int N, int K) +{ + int i; + double err = 0, ener = 0, snr, snr0; + opus_val16 x0[MAX_SIZE]; + opus_val16 x1[MAX_SIZE]; + for (i=0;i 20) + { + fprintf(stderr, "FAIL!\n"); + ret = 1; + } +} + +int main(void) +{ + ALLOC_STACK; + test_rotation(15, 3); + test_rotation(23, 5); + test_rotation(50, 3); + test_rotation(80, 1); + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_types.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_types.c new file mode 100644 index 000000000..67a0fb8ed --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/tests/test_unit_types.c @@ -0,0 +1,50 @@ +/* Copyright (c) 2008-2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_types.h" +#include + +int main(void) +{ + opus_int16 i = 1; + i <<= 14; + if (i>>14 != 1) + { + fprintf(stderr, "opus_int16 isn't 16 bits\n"); + return 1; + } + if (sizeof(opus_int16)*2 != sizeof(opus_int32)) + { + fprintf(stderr, "16*2 != 32\n"); + return 1; + } + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/vq.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/vq.c new file mode 100644 index 000000000..8011e2254 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/vq.c @@ -0,0 +1,442 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mathops.h" +#include "cwrs.h" +#include "vq.h" +#include "arch.h" +#include "os_support.h" +#include "bands.h" +#include "rate.h" +#include "pitch.h" + +#if defined(MIPSr1_ASM) +#include "mips/vq_mipsr1.h" +#endif + +#ifndef OVERRIDE_vq_exp_rotation1 +static void exp_rotation1(celt_norm *X, int len, int stride, opus_val16 c, opus_val16 s) +{ + int i; + opus_val16 ms; + celt_norm *Xptr; + Xptr = X; + ms = NEG16(s); + for (i=0;i=0;i--) + { + celt_norm x1, x2; + x1 = Xptr[0]; + x2 = Xptr[stride]; + Xptr[stride] = EXTRACT16(PSHR32(MAC16_16(MULT16_16(c, x2), s, x1), 15)); + *Xptr-- = EXTRACT16(PSHR32(MAC16_16(MULT16_16(c, x1), ms, x2), 15)); + } +} +#endif /* OVERRIDE_vq_exp_rotation1 */ + +void exp_rotation(celt_norm *X, int len, int dir, int stride, int K, int spread) +{ + static const int SPREAD_FACTOR[3]={15,10,5}; + int i; + opus_val16 c, s; + opus_val16 gain, theta; + int stride2=0; + int factor; + + if (2*K>=len || spread==SPREAD_NONE) + return; + factor = SPREAD_FACTOR[spread-1]; + + gain = celt_div((opus_val32)MULT16_16(Q15_ONE,len),(opus_val32)(len+factor*K)); + theta = HALF16(MULT16_16_Q15(gain,gain)); + + c = celt_cos_norm(EXTEND32(theta)); + s = celt_cos_norm(EXTEND32(SUB16(Q15ONE,theta))); /* sin(theta) */ + + if (len>=8*stride) + { + stride2 = 1; + /* This is just a simple (equivalent) way of computing sqrt(len/stride) with rounding. + It's basically incrementing long as (stride2+0.5)^2 < len/stride. */ + while ((stride2*stride2+stride2)*stride + (stride>>2) < len) + stride2++; + } + /*NOTE: As a minor optimization, we could be passing around log2(B), not B, for both this and for + extract_collapse_mask().*/ + len = celt_udiv(len, stride); + for (i=0;i>1; +#endif + t = VSHR32(Ryy, 2*(k-7)); + g = MULT16_16_P15(celt_rsqrt_norm(t),gain); + + i=0; + do + X[i] = EXTRACT16(PSHR32(MULT16_16(g, iy[i]), k+1)); + while (++i < N); +} + +static unsigned extract_collapse_mask(int *iy, int N, int B) +{ + unsigned collapse_mask; + int N0; + int i; + if (B<=1) + return 1; + /*NOTE: As a minor optimization, we could be passing around log2(B), not B, for both this and for + exp_rotation().*/ + N0 = celt_udiv(N, B); + collapse_mask = 0; + i=0; do { + int j; + unsigned tmp=0; + j=0; do { + tmp |= iy[i*N0+j]; + } while (++j (N>>1)) + { + opus_val16 rcp; + j=0; do { + sum += X[j]; + } while (++j EPSILON && sum < 64)) +#endif + { + X[0] = QCONST16(1.f,14); + j=1; do + X[j]=0; + while (++j=0); + + /* This should never happen, but just in case it does (e.g. on silence) + we fill the first bin with pulses. */ +#ifdef FIXED_POINT_DEBUG + celt_sig_assert(pulsesLeft<=N+3); +#endif + if (pulsesLeft > N+3) + { + opus_val16 tmp = (opus_val16)pulsesLeft; + yy = MAC16_16(yy, tmp, tmp); + yy = MAC16_16(yy, tmp, y[0]); + iy[0] += pulsesLeft; + pulsesLeft=0; + } + + for (i=0;i= best_num/best_den, but that way + we can do it without any division */ + /* OPT: It's not clear whether a cmov is faster than a branch here + since the condition is more often false than true and using + a cmov introduces data dependencies across iterations. The optimal + choice may be architecture-dependent. */ + if (opus_unlikely(MULT16_16(best_den, Rxy) > MULT16_16(Ryy, best_num))) + { + best_den = Ryy; + best_num = Rxy; + best_id = j; + } + } while (++j0, "alg_quant() needs at least one pulse"); + celt_assert2(N>1, "alg_quant() needs at least two dimensions"); + + /* Covers vectorization by up to 4. */ + ALLOC(iy, N+3, int); + + exp_rotation(X, N, 1, B, K, spread); + + yy = op_pvq_search(X, iy, K, N, arch); + + encode_pulses(iy, N, K, enc); + + if (resynth) + { + normalise_residual(iy, X, N, yy, gain); + exp_rotation(X, N, -1, B, K, spread); + } + + collapse_mask = extract_collapse_mask(iy, N, B); + RESTORE_STACK; + return collapse_mask; +} + +/** Decode pulse vector and combine the result with the pitch vector to produce + the final normalised signal in the current band. */ +unsigned alg_unquant(celt_norm *X, int N, int K, int spread, int B, + ec_dec *dec, opus_val16 gain) +{ + opus_val32 Ryy; + unsigned collapse_mask; + VARDECL(int, iy); + SAVE_STACK; + + celt_assert2(K>0, "alg_unquant() needs at least one pulse"); + celt_assert2(N>1, "alg_unquant() needs at least two dimensions"); + ALLOC(iy, N, int); + Ryy = decode_pulses(iy, N, K, dec); + normalise_residual(iy, X, N, Ryy, gain); + exp_rotation(X, N, -1, B, K, spread); + collapse_mask = extract_collapse_mask(iy, N, B); + RESTORE_STACK; + return collapse_mask; +} + +#ifndef OVERRIDE_renormalise_vector +void renormalise_vector(celt_norm *X, int N, opus_val16 gain, int arch) +{ + int i; +#ifdef FIXED_POINT + int k; +#endif + opus_val32 E; + opus_val16 g; + opus_val32 t; + celt_norm *xptr; + E = EPSILON + celt_inner_prod(X, X, N, arch); +#ifdef FIXED_POINT + k = celt_ilog2(E)>>1; +#endif + t = VSHR32(E, 2*(k-7)); + g = MULT16_16_P15(celt_rsqrt_norm(t),gain); + + xptr = X; + for (i=0;i +#include +#include +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" +#include "x86cpu.h" + +#if defined(FIXED_POINT) + +void celt_fir_sse4_1(const opus_val16 *x, + const opus_val16 *num, + opus_val16 *y, + int N, + int ord, + int arch) +{ + int i,j; + VARDECL(opus_val16, rnum); + + __m128i vecNoA; + opus_int32 noA ; + SAVE_STACK; + + ALLOC(rnum, ord, opus_val16); + for(i=0;i> 1; + vecNoA = _mm_set_epi32(noA, noA, noA, noA); + + for (i=0;i +#include "arch.h" + +void xcorr_kernel_sse(const opus_val16 *x, const opus_val16 *y, opus_val32 sum[4], int len) +{ + int j; + __m128 xsum1, xsum2; + xsum1 = _mm_loadu_ps(sum); + xsum2 = _mm_setzero_ps(); + + for (j = 0; j < len-3; j += 4) + { + __m128 x0 = _mm_loadu_ps(x+j); + __m128 yj = _mm_loadu_ps(y+j); + __m128 y3 = _mm_loadu_ps(y+j+3); + + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x00),yj)); + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x55), + _mm_shuffle_ps(yj,y3,0x49))); + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xaa), + _mm_shuffle_ps(yj,y3,0x9e))); + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xff),y3)); + } + if (j < len) + { + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + if (++j < len) + { + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + if (++j < len) + { + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + } + } + } + _mm_storeu_ps(sum,_mm_add_ps(xsum1,xsum2)); +} + + +void dual_inner_prod_sse(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) +{ + int i; + __m128 xsum1, xsum2; + xsum1 = _mm_setzero_ps(); + xsum2 = _mm_setzero_ps(); + for (i=0;i +#include + +#include "macros.h" +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE2) && defined(FIXED_POINT) +opus_val32 celt_inner_prod_sse2(const opus_val16 *x, const opus_val16 *y, + int N) +{ + opus_int i, dataSize16; + opus_int32 sum; + + __m128i inVec1_76543210, inVec1_FEDCBA98, acc1; + __m128i inVec2_76543210, inVec2_FEDCBA98, acc2; + + sum = 0; + dataSize16 = N & ~15; + + acc1 = _mm_setzero_si128(); + acc2 = _mm_setzero_si128(); + + for (i=0;i= 8) + { + inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); + inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); + + inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); + + acc1 = _mm_add_epi32(acc1, inVec1_76543210); + i += 8; + } + + acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64( acc1, acc1)); + acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16( acc1, 0x0E)); + sum += _mm_cvtsi128_si32(acc1); + + for (;i +#include + +#include "macros.h" +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) && defined(FIXED_POINT) +#include +#include "x86cpu.h" + +opus_val32 celt_inner_prod_sse4_1(const opus_val16 *x, const opus_val16 *y, + int N) +{ + opus_int i, dataSize16; + opus_int32 sum; + __m128i inVec1_76543210, inVec1_FEDCBA98, acc1; + __m128i inVec2_76543210, inVec2_FEDCBA98, acc2; + __m128i inVec1_3210, inVec2_3210; + + sum = 0; + dataSize16 = N & ~15; + + acc1 = _mm_setzero_si128(); + acc2 = _mm_setzero_si128(); + + for (i=0;i= 8) + { + inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); + inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); + + inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); + + acc1 = _mm_add_epi32(acc1, inVec1_76543210); + i += 8; + } + + if (N - i >= 4) + { + inVec1_3210 = OP_CVTEPI16_EPI32_M64(&x[i + 0]); + inVec2_3210 = OP_CVTEPI16_EPI32_M64(&y[i + 0]); + + inVec1_3210 = _mm_mullo_epi32(inVec1_3210, inVec2_3210); + + acc1 = _mm_add_epi32(acc1, inVec1_3210); + i += 4; + } + + acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64(acc1, acc1)); + acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16(acc1, 0x0E)); + + sum += _mm_cvtsi128_si32(acc1); + + for (;i= 3); + + sum0 = _mm_setzero_si128(); + sum1 = _mm_setzero_si128(); + sum2 = _mm_setzero_si128(); + sum3 = _mm_setzero_si128(); + + for (j=0;j<(len-7);j+=8) + { + vecX = _mm_loadu_si128((__m128i *)(&x[j + 0])); + vecY0 = _mm_loadu_si128((__m128i *)(&y[j + 0])); + vecY1 = _mm_loadu_si128((__m128i *)(&y[j + 1])); + vecY2 = _mm_loadu_si128((__m128i *)(&y[j + 2])); + vecY3 = _mm_loadu_si128((__m128i *)(&y[j + 3])); + + sum0 = _mm_add_epi32(sum0, _mm_madd_epi16(vecX, vecY0)); + sum1 = _mm_add_epi32(sum1, _mm_madd_epi16(vecX, vecY1)); + sum2 = _mm_add_epi32(sum2, _mm_madd_epi16(vecX, vecY2)); + sum3 = _mm_add_epi32(sum3, _mm_madd_epi16(vecX, vecY3)); + } + + sum0 = _mm_add_epi32(sum0, _mm_unpackhi_epi64( sum0, sum0)); + sum0 = _mm_add_epi32(sum0, _mm_shufflelo_epi16( sum0, 0x0E)); + + sum1 = _mm_add_epi32(sum1, _mm_unpackhi_epi64( sum1, sum1)); + sum1 = _mm_add_epi32(sum1, _mm_shufflelo_epi16( sum1, 0x0E)); + + sum2 = _mm_add_epi32(sum2, _mm_unpackhi_epi64( sum2, sum2)); + sum2 = _mm_add_epi32(sum2, _mm_shufflelo_epi16( sum2, 0x0E)); + + sum3 = _mm_add_epi32(sum3, _mm_unpackhi_epi64( sum3, sum3)); + sum3 = _mm_add_epi32(sum3, _mm_shufflelo_epi16( sum3, 0x0E)); + + vecSum = _mm_unpacklo_epi64(_mm_unpacklo_epi32(sum0, sum1), + _mm_unpacklo_epi32(sum2, sum3)); + + for (;j<(len-3);j+=4) + { + vecX = OP_CVTEPI16_EPI32_M64(&x[j + 0]); + vecX0 = _mm_shuffle_epi32(vecX, 0x00); + vecX1 = _mm_shuffle_epi32(vecX, 0x55); + vecX2 = _mm_shuffle_epi32(vecX, 0xaa); + vecX3 = _mm_shuffle_epi32(vecX, 0xff); + + vecY0 = OP_CVTEPI16_EPI32_M64(&y[j + 0]); + vecY1 = OP_CVTEPI16_EPI32_M64(&y[j + 1]); + vecY2 = OP_CVTEPI16_EPI32_M64(&y[j + 2]); + vecY3 = OP_CVTEPI16_EPI32_M64(&y[j + 3]); + + sum0 = _mm_mullo_epi32(vecX0, vecY0); + sum1 = _mm_mullo_epi32(vecX1, vecY1); + sum2 = _mm_mullo_epi32(vecX2, vecY2); + sum3 = _mm_mullo_epi32(vecX3, vecY3); + + sum0 = _mm_add_epi32(sum0, sum1); + sum2 = _mm_add_epi32(sum2, sum3); + vecSum = _mm_add_epi32(vecSum, sum0); + vecSum = _mm_add_epi32(vecSum, sum2); + } + + for (;j +#include +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "vq.h" +#include "x86cpu.h" + + +#ifndef FIXED_POINT + +opus_val16 op_pvq_search_sse2(celt_norm *_X, int *iy, int K, int N, int arch) +{ + int i, j; + int pulsesLeft; + float xy, yy; + VARDECL(celt_norm, y); + VARDECL(celt_norm, X); + VARDECL(float, signy); + __m128 signmask; + __m128 sums; + __m128i fours; + SAVE_STACK; + + (void)arch; + /* All bits set to zero, except for the sign bit. */ + signmask = _mm_set_ps1(-0.f); + fours = _mm_set_epi32(4, 4, 4, 4); + ALLOC(y, N+3, celt_norm); + ALLOC(X, N+3, celt_norm); + ALLOC(signy, N+3, float); + + OPUS_COPY(X, _X, N); + X[N] = X[N+1] = X[N+2] = 0; + sums = _mm_setzero_ps(); + for (j=0;j (N>>1)) + { + __m128i pulses_sum; + __m128 yy4, xy4; + __m128 rcp4; + opus_val32 sum = _mm_cvtss_f32(sums); + /* If X is too small, just replace it with a pulse at 0 */ + /* Prevents infinities and NaNs from causing too many pulses + to be allocated. 64 is an approximation of infinity here. */ + if (!(sum > EPSILON && sum < 64)) + { + X[0] = QCONST16(1.f,14); + j=1; do + X[j]=0; + while (++j=0); + + /* This should never happen, but just in case it does (e.g. on silence) + we fill the first bin with pulses. */ + if (pulsesLeft > N+3) + { + opus_val16 tmp = (opus_val16)pulsesLeft; + yy = MAC16_16(yy, tmp, tmp); + yy = MAC16_16(yy, tmp, y[0]); + iy[0] += pulsesLeft; + pulsesLeft=0; + } + + for (i=0;i +static _inline void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) +{ + __cpuid((int*)CPUInfo, InfoType); +} + +#else + +#if defined(CPU_INFO_BY_C) +#include +#endif + +static void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) +{ +#if defined(CPU_INFO_BY_ASM) +#if defined(__i386__) && defined(__PIC__) +/* %ebx is PIC register in 32-bit, so mustn't clobber it. */ + __asm__ __volatile__ ( + "xchg %%ebx, %1\n" + "cpuid\n" + "xchg %%ebx, %1\n": + "=a" (CPUInfo[0]), + "=r" (CPUInfo[1]), + "=c" (CPUInfo[2]), + "=d" (CPUInfo[3]) : + "0" (InfoType) + ); +#else + __asm__ __volatile__ ( + "cpuid": + "=a" (CPUInfo[0]), + "=b" (CPUInfo[1]), + "=c" (CPUInfo[2]), + "=d" (CPUInfo[3]) : + "0" (InfoType) + ); +#endif +#elif defined(CPU_INFO_BY_C) + __get_cpuid(InfoType, &(CPUInfo[0]), &(CPUInfo[1]), &(CPUInfo[2]), &(CPUInfo[3])); +#endif +} + +#endif + +typedef struct CPU_Feature{ + /* SIMD: 128-bit */ + int HW_SSE; + int HW_SSE2; + int HW_SSE41; + /* SIMD: 256-bit */ + int HW_AVX; +} CPU_Feature; + +static void opus_cpu_feature_check(CPU_Feature *cpu_feature) +{ + unsigned int info[4] = {0}; + unsigned int nIds = 0; + + cpuid(info, 0); + nIds = info[0]; + + if (nIds >= 1){ + cpuid(info, 1); + cpu_feature->HW_SSE = (info[3] & (1 << 25)) != 0; + cpu_feature->HW_SSE2 = (info[3] & (1 << 26)) != 0; + cpu_feature->HW_SSE41 = (info[2] & (1 << 19)) != 0; + cpu_feature->HW_AVX = (info[2] & (1 << 28)) != 0; + } + else { + cpu_feature->HW_SSE = 0; + cpu_feature->HW_SSE2 = 0; + cpu_feature->HW_SSE41 = 0; + cpu_feature->HW_AVX = 0; + } +} + +int opus_select_arch(void) +{ + CPU_Feature cpu_feature; + int arch; + + opus_cpu_feature_check(&cpu_feature); + + arch = 0; + if (!cpu_feature.HW_SSE) + { + return arch; + } + arch++; + + if (!cpu_feature.HW_SSE2) + { + return arch; + } + arch++; + + if (!cpu_feature.HW_SSE41) + { + return arch; + } + arch++; + + if (!cpu_feature.HW_AVX) + { + return arch; + } + arch++; + + return arch; +} + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86cpu.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86cpu.h new file mode 100644 index 000000000..1e2bf17b9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt/x86/x86cpu.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if !defined(X86CPU_H) +# define X86CPU_H + +# if defined(OPUS_X86_MAY_HAVE_SSE) +# define MAY_HAVE_SSE(name) name ## _sse +# else +# define MAY_HAVE_SSE(name) name ## _c +# endif + +# if defined(OPUS_X86_MAY_HAVE_SSE2) +# define MAY_HAVE_SSE2(name) name ## _sse2 +# else +# define MAY_HAVE_SSE2(name) name ## _c +# endif + +# if defined(OPUS_X86_MAY_HAVE_SSE4_1) +# define MAY_HAVE_SSE4_1(name) name ## _sse4_1 +# else +# define MAY_HAVE_SSE4_1(name) name ## _c +# endif + +# if defined(OPUS_X86_MAY_HAVE_AVX) +# define MAY_HAVE_AVX(name) name ## _avx +# else +# define MAY_HAVE_AVX(name) name ## _c +# endif + +# if defined(OPUS_HAVE_RTCD) +int opus_select_arch(void); +# endif + +/*gcc appears to emit MOVDQA's to load the argument of an _mm_cvtepi8_epi32() + or _mm_cvtepi16_epi32() when optimizations are disabled, even though the + actual PMOVSXWD instruction takes an m32 or m64. Unlike a normal memory + reference, these require 16-byte alignment and load a full 16 bytes (instead + of 4 or 8), possibly reading out of bounds. + + We can insert an explicit MOVD or MOVQ using _mm_cvtsi32_si128() or + _mm_loadl_epi64(), which should have the same semantics as an m32 or m64 + reference in the PMOVSXWD instruction itself, but gcc is not smart enough to + optimize this out when optimizations ARE enabled. + + Clang, in contrast, requires us to do this always for _mm_cvtepi8_epi32 + (which is fair, since technically the compiler is always allowed to do the + dereference before invoking the function implementing the intrinsic). + However, it is smart enough to eliminate the extra MOVD instruction. + For _mm_cvtepi16_epi32, it does the right thing, though does *not* optimize out + the extra MOVQ if it's specified explicitly */ + +# if defined(__clang__) || !defined(__OPTIMIZE__) +# define OP_CVTEPI8_EPI32_M32(x) \ + (_mm_cvtepi8_epi32(_mm_cvtsi32_si128(*(int *)(x)))) +# else +# define OP_CVTEPI8_EPI32_M32(x) \ + (_mm_cvtepi8_epi32(*(__m128i *)(x))) +#endif + +/* similar reasoning about the instruction sequence as in the 32-bit macro above, + */ +# if defined(__clang__) || !defined(__OPTIMIZE__) +# define OP_CVTEPI16_EPI32_M64(x) \ + (_mm_cvtepi16_epi32(_mm_loadl_epi64((__m128i *)(x)))) +# else +# define OP_CVTEPI16_EPI32_M64(x) \ + (_mm_cvtepi16_epi32(*(__m128i *)(x))) +# endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_headers.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_headers.mk new file mode 100644 index 000000000..706185da4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_headers.mk @@ -0,0 +1,53 @@ +CELT_HEAD = \ +celt/arch.h \ +celt/bands.h \ +celt/celt.h \ +celt/cpu_support.h \ +include/opus_types.h \ +include/opus_defines.h \ +include/opus_custom.h \ +celt/cwrs.h \ +celt/ecintrin.h \ +celt/entcode.h \ +celt/entdec.h \ +celt/entenc.h \ +celt/fixed_debug.h \ +celt/fixed_generic.h \ +celt/float_cast.h \ +celt/_kiss_fft_guts.h \ +celt/kiss_fft.h \ +celt/laplace.h \ +celt/mathops.h \ +celt/mdct.h \ +celt/mfrngcod.h \ +celt/modes.h \ +celt/os_support.h \ +celt/pitch.h \ +celt/celt_lpc.h \ +celt/x86/celt_lpc_sse.h \ +celt/quant_bands.h \ +celt/rate.h \ +celt/stack_alloc.h \ +celt/vq.h \ +celt/static_modes_float.h \ +celt/static_modes_fixed.h \ +celt/static_modes_float_arm_ne10.h \ +celt/static_modes_fixed_arm_ne10.h \ +celt/arm/armcpu.h \ +celt/arm/fixed_armv4.h \ +celt/arm/fixed_armv5e.h \ +celt/arm/fixed_arm64.h \ +celt/arm/kiss_fft_armv4.h \ +celt/arm/kiss_fft_armv5e.h \ +celt/arm/pitch_arm.h \ +celt/arm/fft_arm.h \ +celt/arm/mdct_arm.h \ +celt/mips/celt_mipsr1.h \ +celt/mips/fixed_generic_mipsr1.h \ +celt/mips/kiss_fft_mipsr1.h \ +celt/mips/mdct_mipsr1.h \ +celt/mips/pitch_mipsr1.h \ +celt/mips/vq_mipsr1.h \ +celt/x86/pitch_sse.h \ +celt/x86/vq_sse.h \ +celt/x86/x86cpu.h diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_sources.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_sources.mk new file mode 100644 index 000000000..c9dab06ec --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/celt_sources.mk @@ -0,0 +1,50 @@ +CELT_SOURCES = \ +celt/bands.c \ +celt/celt.c \ +celt/celt_encoder.c \ +celt/celt_decoder.c \ +celt/cwrs.c \ +celt/entcode.c \ +celt/entdec.c \ +celt/entenc.c \ +celt/kiss_fft.c \ +celt/laplace.c \ +celt/mathops.c \ +celt/mdct.c \ +celt/modes.c \ +celt/pitch.c \ +celt/celt_lpc.c \ +celt/quant_bands.c \ +celt/rate.c \ +celt/vq.c + +CELT_SOURCES_SSE = \ +celt/x86/x86cpu.c \ +celt/x86/x86_celt_map.c \ +celt/x86/pitch_sse.c + +CELT_SOURCES_SSE2 = \ +celt/x86/pitch_sse2.c \ +celt/x86/vq_sse2.c + +CELT_SOURCES_SSE4_1 = \ +celt/x86/celt_lpc_sse4_1.c \ +celt/x86/pitch_sse4_1.c + +CELT_SOURCES_ARM = \ +celt/arm/armcpu.c \ +celt/arm/arm_celt_map.c + +CELT_SOURCES_ARM_ASM = \ +celt/arm/celt_pitch_xcorr_arm.s + +CELT_AM_SOURCES_ARM_ASM = \ +celt/arm/armopts.s.in + +CELT_SOURCES_ARM_NEON_INTR = \ +celt/arm/celt_neon_intr.c \ +celt/arm/pitch_neon_intr.c + +CELT_SOURCES_ARM_NE10 = \ +celt/arm/celt_fft_ne10.c \ +celt/arm/celt_mdct_ne10.c diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/CFeatureCheck.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/CFeatureCheck.cmake new file mode 100644 index 000000000..08828f585 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/CFeatureCheck.cmake @@ -0,0 +1,39 @@ +# - Compile and run code to check for C features +# +# This functions compiles a source file under the `cmake` folder +# and adds the corresponding `HAVE_[FILENAME]` flag to the CMake +# environment +# +# c_feature_check( []) +# +# - Example +# +# include(CFeatureCheck) +# c_feature_check(VLA) + +if(__c_feature_check) + return() +endif() +set(__c_feature_check INCLUDED) + +function(c_feature_check FILE) + string(TOLOWER ${FILE} FILE) + string(TOUPPER ${FILE} VAR) + string(TOUPPER "${VAR}_SUPPORTED" FEATURE) + if (DEFINED ${VAR}_SUPPORTED) + set(${VAR}_SUPPORTED 1 PARENT_SCOPE) + return() + endif() + + if (NOT DEFINED COMPILE_${FEATURE}) + message(STATUS "Performing Test ${FEATURE}") + try_compile(COMPILE_${FEATURE} ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/${FILE}.c) + endif() + + if(COMPILE_${FEATURE}) + message(STATUS "Performing Test ${FEATURE} -- success") + set(${VAR}_SUPPORTED 1 PARENT_SCOPE) + else() + message(STATUS "Performing Test ${FEATURE} -- failed to compile") + endif() +endfunction() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusBuildtype.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusBuildtype.cmake new file mode 100644 index 000000000..557cc89b3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusBuildtype.cmake @@ -0,0 +1,27 @@ +# Set a default build type if none was specified +if(__opus_buildtype) + return() +endif() +set(__opus_buildtype INCLUDED) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + if(CMAKE_C_FLAGS) + message(STATUS "CMAKE_C_FLAGS: " ${CMAKE_C_FLAGS}) + else() + set(default_build_type "Release") + message( + STATUS + "Setting build type to '${default_build_type}' as none was specified and no CFLAGS was exported." + ) + set(CMAKE_BUILD_TYPE "${default_build_type}" + CACHE STRING "Choose the type of build." + FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE + PROPERTY STRINGS + "Debug" + "Release" + "MinSizeRel" + "RelWithDebInfo") + endif() +endif() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake new file mode 100644 index 000000000..8d19a5357 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake @@ -0,0 +1,104 @@ +if(__opus_config) + return() +endif() +set(__opus_config INCLUDED) + +include(OpusFunctions) + +configure_file(cmake/config.h.cmake.in config.h @ONLY) +add_definitions(-DHAVE_CONFIG_H) + +set_property(GLOBAL PROPERTY USE_FOLDERS ON) +set_property(GLOBAL PROPERTY C_STANDARD 99) + +if(MSVC) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) +endif() + +include(CheckLibraryExists) +check_library_exists(m floor "" HAVE_LIBM) +if(HAVE_LIBM) + list(APPEND OPUS_REQUIRED_LIBRARIES m) +endif() + +include(CFeatureCheck) +c_feature_check(VLA) + +include(CheckIncludeFile) +check_include_file(alloca.h HAVE_ALLOCA_H) + +include(CheckSymbolExists) +if(HAVE_ALLOCA_H) + add_definitions(-DHAVE_ALLOCA_H) + check_symbol_exists(alloca "alloca.h" USE_ALLOCA_SUPPORTED) +else() + check_symbol_exists(alloca "stdlib.h;malloc.h" USE_ALLOCA_SUPPORTED) +endif() + +include(CheckFunctionExists) +check_function_exists(lrintf HAVE_LRINTF) +check_function_exists(lrint HAVE_LRINT) + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "(i[0-9]86|x86|X86|amd64|AMD64|x86_64)") + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(OPUS_CPU_X64 1) + else() + set(OPUS_CPU_X86 1) + endif() +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm|aarch64)") + set(OPUS_CPU_ARM 1) +endif() + +if(NOT OPUS_DISABLE_INTRINSICS) + opus_supports_cpu_detection(RUNTIME_CPU_CAPABILITY_DETECTION) +endif() + +if(OPUS_CPU_X86 OR OPUS_CPU_X64 AND NOT OPUS_DISABLE_INTRINSICS) + opus_detect_sse(COMPILER_SUPPORT_SIMD) +elseif(OPUS_CPU_ARM AND NOT OPUS_DISABLE_INTRINSICS) + opus_detect_neon(COMPILER_SUPPORT_NEON) + if(COMPILER_SUPPORT_NEON) + option(OPUS_USE_NEON "Option to enable NEON" ON) + option(OPUS_MAY_HAVE_NEON "Does runtime check for neon support" ON) + option(OPUS_PRESUME_NEON "Assume target CPU has NEON support" OFF) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + set(OPUS_PRESUME_NEON ON) + elseif(CMAKE_SYSTEM_NAME MATCHES "iOS") + set(OPUS_PRESUME_NEON ON) + endif() + endif() +endif() + +if(MSVC) + check_flag(FAST_MATH /fp:fast) + check_flag(STACK_PROTECTOR /GS) + check_flag(STACK_PROTECTOR_DISABLED /GS-) +else() + check_flag(FAST_MATH -ffast-math) + check_flag(STACK_PROTECTOR -fstack-protector-strong) + check_flag(HIDDEN_VISIBILITY -fvisibility=hidden) + set(FORTIFY_SOURCE_SUPPORTED 1) +endif() + +if(MINGW) + # For MINGW we need to link ssp lib for security features such as + # stack protector and fortify_sources + check_library_exists(ssp __stack_chk_fail "" HAVE_LIBSSP) + if(NOT HAVE_LIBSSP) + message(WARNING "Could not find libssp in MinGW, disabling STACK_PROTECTOR and FORTIFY_SOURCE") + set(STACK_PROTECTOR_SUPPORTED 0) + set(FORTIFY_SOURCE_SUPPORTED 0) + endif() +endif() + +if(NOT MSVC) + set(WARNING_LIST -Wall -W -Wstrict-prototypes -Wextra -Wcast-align -Wnested-externs -Wshadow) + include(CheckCCompilerFlag) + foreach(WARNING_FLAG ${WARNING_LIST}) + string(REPLACE - "" WARNING_VAR ${WARNING_FLAG}) + check_c_compiler_flag(${WARNING_FLAG} ${WARNING_VAR}_SUPPORTED) + if(${WARNING_VAR}_SUPPORTED) + add_compile_options(${WARNING_FLAG}) + endif() + endforeach() +endif() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake.in new file mode 100644 index 000000000..0b21231d4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusConfig.cmake.in @@ -0,0 +1,20 @@ +set(OPUS_VERSION @PROJECT_VERSION@) +set(OPUS_VERSION_STRING @PROJECT_VERSION@) +set(OPUS_VERSION_MAJOR @PROJECT_VERSION_MAJOR@) +set(OPUS_VERSION_MINOR @PROJECT_VERSION_MINOR@) +set(OPUS_VERSION_PATCH @PROJECT_VERSION_PATCH@) + +@PACKAGE_INIT@ + +set_and_check(OPUS_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@") +set(OPUS_INCLUDE_DIR ${OPUS_INCLUDE_DIR};${OPUS_INCLUDE_DIR}/opus) +set(OPUS_INCLUDE_DIRS "@PACKAGE_INCLUDE_INSTALL_DIR@;@PACKAGE_INCLUDE_INSTALL_DIR@/opus") + +include(${CMAKE_CURRENT_LIST_DIR}/OpusTargets.cmake) + +set(OPUS_LIBRARY Opus::opus) +set(OPUS_LIBRARIES Opus::opus) + +check_required_components(Opus) + +set(OPUS_FOUND 1) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusFunctions.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusFunctions.cmake new file mode 100644 index 000000000..fcf3351fb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusFunctions.cmake @@ -0,0 +1,215 @@ +if(__opus_functions) + return() +endif() +set(__opus_functions INCLUDED) + +function(get_library_version OPUS_LIBRARY_VERSION OPUS_LIBRARY_VERSION_MAJOR) + file(STRINGS configure.ac opus_lt_current_string + LIMIT_COUNT 1 + REGEX "OPUS_LT_CURRENT=") + string(REGEX MATCH + "OPUS_LT_CURRENT=([0-9]*)" + _ + ${opus_lt_current_string}) + set(OPUS_LT_CURRENT ${CMAKE_MATCH_1}) + + file(STRINGS configure.ac opus_lt_revision_string + LIMIT_COUNT 1 + REGEX "OPUS_LT_REVISION=") + string(REGEX MATCH + "OPUS_LT_REVISION=([0-9]*)" + _ + ${opus_lt_revision_string}) + set(OPUS_LT_REVISION ${CMAKE_MATCH_1}) + + file(STRINGS configure.ac opus_lt_age_string + LIMIT_COUNT 1 + REGEX "OPUS_LT_AGE=") + string(REGEX MATCH + "OPUS_LT_AGE=([0-9]*)" + _ + ${opus_lt_age_string}) + set(OPUS_LT_AGE ${CMAKE_MATCH_1}) + + math(EXPR OPUS_LIBRARY_VERSION_MAJOR "${OPUS_LT_CURRENT} - ${OPUS_LT_AGE}") + set(OPUS_LIBRARY_VERSION_MINOR ${OPUS_LT_AGE}) + set(OPUS_LIBRARY_VERSION_PATCH ${OPUS_LT_REVISION}) + set( + OPUS_LIBRARY_VERSION + "${OPUS_LIBRARY_VERSION_MAJOR}.${OPUS_LIBRARY_VERSION_MINOR}.${OPUS_LIBRARY_VERSION_PATCH}" + PARENT_SCOPE) + set(OPUS_LIBRARY_VERSION_MAJOR ${OPUS_LIBRARY_VERSION_MAJOR} PARENT_SCOPE) +endfunction() + +function(check_flag NAME FLAG) + include(CheckCCompilerFlag) + check_c_compiler_flag(${FLAG} ${NAME}_SUPPORTED) +endfunction() + +include(CheckIncludeFile) +# function to check if compiler supports SSE, SSE2, SSE4.1 and AVX if target +# systems may not have SSE support then use OPUS_MAY_HAVE_SSE option if target +# system is guaranteed to have SSE support then OPUS_PRESUME_SSE can be used to +# skip SSE runtime check +function(opus_detect_sse COMPILER_SUPPORT_SIMD) + message(STATUS "Check SIMD support by compiler") + check_include_file(xmmintrin.h HAVE_XMMINTRIN_H) # SSE1 + if(HAVE_XMMINTRIN_H) + if(MSVC) + # different arch options for 32 and 64 bit target for MSVC + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + check_flag(SSE1 /arch:SSE) + else() + set(SSE1_SUPPORTED + 1 + PARENT_SCOPE) + endif() + else() + check_flag(SSE1 -msse) + endif() + else() + set(SSE1_SUPPORTED + 0 + PARENT_SCOPE) + endif() + + check_include_file(emmintrin.h HAVE_EMMINTRIN_H) # SSE2 + if(HAVE_EMMINTRIN_H) + if(MSVC) + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + check_flag(SSE2 /arch:SSE2) + else() + set(SSE2_SUPPORTED + 1 + PARENT_SCOPE) + endif() + else() + check_flag(SSE2 -msse2) + endif() + else() + set(SSE2_SUPPORTED + 0 + PARENT_SCOPE) + endif() + + check_include_file(smmintrin.h HAVE_SMMINTRIN_H) # SSE4.1 + if(HAVE_SMMINTRIN_H) + if(MSVC) + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + check_flag(SSE4_1 /arch:SSE2) # SSE2 and above + else() + set(SSE4_1_SUPPORTED + 1 + PARENT_SCOPE) + endif() + else() + check_flag(SSE4_1 -msse4.1) + endif() + else() + set(SSE4_1_SUPPORTED + 0 + PARENT_SCOPE) + endif() + + check_include_file(immintrin.h HAVE_IMMINTRIN_H) # AVX + if(HAVE_IMMINTRIN_H) + if(MSVC) + check_flag(AVX /arch:AVX) + else() + check_flag(AVX -mavx) + endif() + else() + set(AVX_SUPPORTED + 0 + PARENT_SCOPE) + endif() + + if(SSE1_SUPPORTED OR SSE2_SUPPORTED OR SSE4_1_SUPPORTED OR AVX_SUPPORTED) + set(COMPILER_SUPPORT_SIMD 1 PARENT_SCOPE) + else() + message(STATUS "No SIMD support in compiler") + endif() +endfunction() + +function(opus_detect_neon COMPILER_SUPPORT_NEON) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm|aarch64)") + message(STATUS "Check NEON support by compiler") + check_include_file(arm_neon.h HAVE_ARM_NEON_H) + if(HAVE_ARM_NEON_H) + set(COMPILER_SUPPORT_NEON ${HAVE_ARM_NEON_H} PARENT_SCOPE) + endif() + endif() +endfunction() + +function(opus_supports_cpu_detection RUNTIME_CPU_CAPABILITY_DETECTION) + if(MSVC) + check_include_file(intrin.h HAVE_INTRIN_H) + else() + check_include_file(cpuid.h HAVE_CPUID_H) + endif() + if(HAVE_INTRIN_H OR HAVE_CPUID_H) + set(RUNTIME_CPU_CAPABILITY_DETECTION 1 PARENT_SCOPE) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm|aarch64)") + # ARM cpu detection is implemented for Windows and anything + # using a Linux kernel (such as Android). + if (CMAKE_SYSTEM_NAME MATCHES "(Windows|Linux|Android)") + set(RUNTIME_CPU_CAPABILITY_DETECTION 1 PARENT_SCOPE) + endif () + else() + set(RUNTIME_CPU_CAPABILITY_DETECTION 0 PARENT_SCOPE) + endif() +endfunction() + +function(add_sources_group target group) + target_sources(${target} PRIVATE ${ARGN}) + source_group(${group} FILES ${ARGN}) +endfunction() + +function(get_opus_sources SOURCE_GROUP MAKE_FILE SOURCES) + # read file, each item in list is one group + file(STRINGS ${MAKE_FILE} opus_sources) + + # add wildcard for regex match + string(CONCAT SOURCE_GROUP ${SOURCE_GROUP} ".*$") + + # find group + foreach(val IN LISTS opus_sources) + if(val MATCHES ${SOURCE_GROUP}) + list(LENGTH val list_length) + if(${list_length} EQUAL 1) + # for tests split by '=' and clean up the rest into a list + string(FIND ${val} "=" index) + math(EXPR index "${index} + 1") + string(SUBSTRING ${val} + ${index} + -1 + sources) + string(REPLACE " " + ";" + sources + ${sources}) + else() + # discard the group + list(REMOVE_AT val 0) + set(sources ${val}) + endif() + break() + endif() + endforeach() + + list(LENGTH sources list_length) + if(${list_length} LESS 1) + message( + FATAL_ERROR + "No files parsed succesfully from ${SOURCE_GROUP} in ${MAKE_FILE}") + endif() + + # remove trailing whitespaces + set(list_var "") + foreach(source ${sources}) + string(STRIP "${source}" source) + list(APPEND list_var "${source}") + endforeach() + + set(${SOURCES} ${list_var} PARENT_SCOPE) +endfunction() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusPackageVersion.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusPackageVersion.cmake new file mode 100644 index 000000000..447ce3b17 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusPackageVersion.cmake @@ -0,0 +1,70 @@ +if(__opus_version) + return() +endif() +set(__opus_version INCLUDED) + +function(get_package_version PACKAGE_VERSION PROJECT_VERSION) + + find_package(Git) + if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/.git") + execute_process(COMMAND ${GIT_EXECUTABLE} + --git-dir=${CMAKE_CURRENT_LIST_DIR}/.git describe + --tags --match "v*" OUTPUT_VARIABLE OPUS_PACKAGE_VERSION) + if(OPUS_PACKAGE_VERSION) + string(STRIP ${OPUS_PACKAGE_VERSION}, OPUS_PACKAGE_VERSION) + string(REPLACE \n + "" + OPUS_PACKAGE_VERSION + ${OPUS_PACKAGE_VERSION}) + string(REPLACE , + "" + OPUS_PACKAGE_VERSION + ${OPUS_PACKAGE_VERSION}) + + string(SUBSTRING ${OPUS_PACKAGE_VERSION} + 1 + -1 + OPUS_PACKAGE_VERSION) + message(STATUS "Opus package version from git repo: ${OPUS_PACKAGE_VERSION}") + endif() + + elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/package_version" + AND NOT OPUS_PACKAGE_VERSION) + # Not a git repo, lets' try to parse it from package_version file if exists + file(STRINGS package_version OPUS_PACKAGE_VERSION + LIMIT_COUNT 1 + REGEX "PACKAGE_VERSION=") + string(REPLACE "PACKAGE_VERSION=" + "" + OPUS_PACKAGE_VERSION + ${OPUS_PACKAGE_VERSION}) + string(REPLACE "\"" + "" + OPUS_PACKAGE_VERSION + ${OPUS_PACKAGE_VERSION}) + # In case we have a unknown dist here we just replace it with 0 + string(REPLACE "unknown" + "0" + OPUS_PACKAGE_VERSION + ${OPUS_PACKAGE_VERSION}) + message(STATUS "Opus package version from package_version file: ${OPUS_PACKAGE_VERSION}") + endif() + + if(OPUS_PACKAGE_VERSION) + string(REGEX + REPLACE "^([0-9]+.[0-9]+\\.?([0-9]+)?).*" + "\\1" + OPUS_PROJECT_VERSION + ${OPUS_PACKAGE_VERSION}) + else() + # fail to parse version from git and package version + message(WARNING "Could not get package version.") + set(OPUS_PACKAGE_VERSION 0) + set(OPUS_PROJECT_VERSION 0) + endif() + + message(STATUS "Opus project version: ${OPUS_PROJECT_VERSION}") + + set(PACKAGE_VERSION ${OPUS_PACKAGE_VERSION} PARENT_SCOPE) + set(PROJECT_VERSION ${OPUS_PROJECT_VERSION} PARENT_SCOPE) +endfunction() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusSources.cmake b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusSources.cmake new file mode 100644 index 000000000..01e75d1a3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/OpusSources.cmake @@ -0,0 +1,46 @@ +if(__opus_sources) + return() +endif() +set(__opus_sources INCLUDED) + +include(OpusFunctions) + +get_opus_sources(SILK_HEAD silk_headers.mk silk_headers) +get_opus_sources(SILK_SOURCES silk_sources.mk silk_sources) +get_opus_sources(SILK_SOURCES_FLOAT silk_sources.mk silk_sources_float) +get_opus_sources(SILK_SOURCES_FIXED silk_sources.mk silk_sources_fixed) +get_opus_sources(SILK_SOURCES_SSE4_1 silk_sources.mk silk_sources_sse4_1) +get_opus_sources(SILK_SOURCES_FIXED_SSE4_1 silk_sources.mk + silk_sources_fixed_sse4_1) +get_opus_sources(SILK_SOURCES_ARM_NEON_INTR silk_sources.mk + silk_sources_arm_neon_intr) +get_opus_sources(SILK_SOURCES_FIXED_ARM_NEON_INTR silk_sources.mk + silk_sources_fixed_arm_neon_intr) + +get_opus_sources(OPUS_HEAD opus_headers.mk opus_headers) +get_opus_sources(OPUS_SOURCES opus_sources.mk opus_sources) +get_opus_sources(OPUS_SOURCES_FLOAT opus_sources.mk opus_sources_float) + +get_opus_sources(CELT_HEAD celt_headers.mk celt_headers) +get_opus_sources(CELT_SOURCES celt_sources.mk celt_sources) +get_opus_sources(CELT_SOURCES_SSE celt_sources.mk celt_sources_sse) +get_opus_sources(CELT_SOURCES_SSE2 celt_sources.mk celt_sources_sse2) +get_opus_sources(CELT_SOURCES_SSE4_1 celt_sources.mk celt_sources_sse4_1) +get_opus_sources(CELT_SOURCES_ARM celt_sources.mk celt_sources_arm) +get_opus_sources(CELT_SOURCES_ARM_ASM celt_sources.mk celt_sources_arm_asm) +get_opus_sources(CELT_AM_SOURCES_ARM_ASM celt_sources.mk + celt_am_sources_arm_asm) +get_opus_sources(CELT_SOURCES_ARM_NEON_INTR celt_sources.mk + celt_sources_arm_neon_intr) +get_opus_sources(CELT_SOURCES_ARM_NE10 celt_sources.mk celt_sources_arm_ne10) + +get_opus_sources(opus_demo_SOURCES Makefile.am opus_demo_sources) +get_opus_sources(opus_custom_demo_SOURCES Makefile.am opus_custom_demo_sources) +get_opus_sources(opus_compare_SOURCES Makefile.am opus_compare_sources) +get_opus_sources(tests_test_opus_api_SOURCES Makefile.am test_opus_api_sources) +get_opus_sources(tests_test_opus_encode_SOURCES Makefile.am + test_opus_encode_sources) +get_opus_sources(tests_test_opus_decode_SOURCES Makefile.am + test_opus_decode_sources) +get_opus_sources(tests_test_opus_padding_SOURCES Makefile.am + test_opus_padding_sources) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/config.h.cmake.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/config.h.cmake.in new file mode 100644 index 000000000..5550842c5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/config.h.cmake.in @@ -0,0 +1 @@ +#cmakedefine PACKAGE_VERSION "@PACKAGE_VERSION@" \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/vla.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/vla.c new file mode 100644 index 000000000..05b211979 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/cmake/vla.c @@ -0,0 +1,7 @@ +int main() { + static int x; + char a[++x]; + a[sizeof a - 1] = 0; + int N; + return a[0]; +} \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/configure.ac b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/configure.ac new file mode 100644 index 000000000..f12f0aa9a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/configure.ac @@ -0,0 +1,946 @@ +dnl Process this file with autoconf to produce a configure script. -*-m4-*- + +dnl The package_version file will be automatically synced to the git revision +dnl by the update_version script when configured in the repository, but will +dnl remain constant in tarball releases unless it is manually edited. +m4_define([CURRENT_VERSION], + m4_esyscmd([ ./update_version 2>/dev/null || true + if test -e package_version; then + . ./package_version + printf "$PACKAGE_VERSION" + else + printf "unknown" + fi ])) + +AC_INIT([opus],[CURRENT_VERSION],[opus@xiph.org]) + +AC_CONFIG_SRCDIR(src/opus_encoder.c) +AC_CONFIG_MACRO_DIR([m4]) + +dnl enable silent rules on automake 1.11 and later +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + +# For libtool. +dnl Please update these for releases. +OPUS_LT_CURRENT=8 +OPUS_LT_REVISION=0 +OPUS_LT_AGE=8 + +AC_SUBST(OPUS_LT_CURRENT) +AC_SUBST(OPUS_LT_REVISION) +AC_SUBST(OPUS_LT_AGE) + +AM_INIT_AUTOMAKE([no-define]) +AM_MAINTAINER_MODE([enable]) + +AC_CANONICAL_HOST +AC_MINGW32 +AM_PROG_LIBTOOL +AM_PROG_CC_C_O + +AC_PROG_CC_C99 +AC_C_CONST +AC_C_INLINE + +AM_PROG_AS + +AC_DEFINE([OPUS_BUILD], [], [This is a build of OPUS]) + +#Use a hacked up version of autoconf's AC_C_RESTRICT because it's not +#strong enough a test to detect old buggy versions of GCC (e.g. 2.95.3) +#Note: Both this and the test for variable-size arrays below are also +# done by AC_PROG_CC_C99, but not thoroughly enough apparently. +AC_CACHE_CHECK([for C/C++ restrict keyword], ac_cv_c_restrict, + [ac_cv_c_restrict=no + # The order here caters to the fact that C++ does not require restrict. + for ac_kw in __restrict __restrict__ _Restrict restrict; do + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[typedef int * int_ptr; + int foo (int_ptr $ac_kw ip, int * $ac_kw baz[]) { + return ip[0]; + }]], + [[int s[1]; + int * $ac_kw t = s; + t[0] = 0; + return foo(t, (void *)0)]])], + [ac_cv_c_restrict=$ac_kw]) + test "$ac_cv_c_restrict" != no && break + done + ]) + +AH_VERBATIM([restrict], +[/* Define to the equivalent of the C99 'restrict' keyword, or to + nothing if this is not supported. Do not define if restrict is + supported directly. */ +#undef restrict +/* Work around a bug in Sun C++: it does not support _Restrict or + __restrict__, even though the corresponding Sun C compiler ends up with + "#define restrict _Restrict" or "#define restrict __restrict__" in the + previous line. Perhaps some future version of Sun C++ will work with + restrict; if so, hopefully it defines __RESTRICT like Sun C does. */ +#if defined __SUNPRO_CC && !defined __RESTRICT +# define _Restrict +# define __restrict__ +#endif]) + +case $ac_cv_c_restrict in + restrict) ;; + no) AC_DEFINE([restrict], []) ;; + *) AC_DEFINE_UNQUOTED([restrict], [$ac_cv_c_restrict]) ;; +esac + +AC_MSG_CHECKING(for C99 variable-size arrays) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], + [[static int x; char a[++x]; a[sizeof a - 1] = 0; int N; return a[0];]])], + [ has_var_arrays=yes + use_alloca="no (using var arrays)" + AC_DEFINE([VAR_ARRAYS], [1], [Use C99 variable-size arrays]) + ],[ + has_var_arrays=no + ]) +AC_MSG_RESULT([$has_var_arrays]) + +AS_IF([test "$has_var_arrays" = "no"], + [ + AC_CHECK_HEADERS([alloca.h]) + AC_MSG_CHECKING(for alloca) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[int foo=10; int *array = alloca(foo);]])], + [ use_alloca=yes; + AC_DEFINE([USE_ALLOCA], [], [Make use of alloca]) + ],[ + use_alloca=no + ]) + AC_MSG_RESULT([$use_alloca]) + ]) + +LT_LIB_M + +AC_ARG_ENABLE([fixed-point], + [AS_HELP_STRING([--enable-fixed-point], + [compile without floating point (for machines without a fast enough FPU)])],, + [enable_fixed_point=no]) + +AS_IF([test "$enable_fixed_point" = "yes"],[ + enable_float="no" + AC_DEFINE([FIXED_POINT], [1], [Compile as fixed-point (for machines without a fast enough FPU)]) + PC_BUILD="fixed-point" +],[ + enable_float="yes"; + PC_BUILD="floating-point" +]) + +AM_CONDITIONAL([FIXED_POINT], [test "$enable_fixed_point" = "yes"]) + +AC_ARG_ENABLE([fixed-point-debug], + [AS_HELP_STRING([--enable-fixed-point-debug], [debug fixed-point implementation])],, + [enable_fixed_point_debug=no]) + +AS_IF([test "$enable_fixed_point_debug" = "yes"],[ + AC_DEFINE([FIXED_DEBUG], [1], [Debug fixed-point implementation]) +]) + +AC_ARG_ENABLE([float_api], + [AS_HELP_STRING([--disable-float-api], + [compile without the floating point API (for machines with no float library)])],, + [enable_float_api=yes]) + +AM_CONDITIONAL([DISABLE_FLOAT_API], [test "$enable_float_api" = "no"]) + +AS_IF([test "$enable_float_api" = "no"],[ + AC_DEFINE([DISABLE_FLOAT_API], [1], [Do not build the float API]) +]) + +AC_ARG_ENABLE([custom-modes], + [AS_HELP_STRING([--enable-custom-modes], [enable non-Opus modes, e.g. 44.1 kHz & 2^n frames])],, + [enable_custom_modes=no]) + +AS_IF([test "$enable_custom_modes" = "yes"],[ + AC_DEFINE([CUSTOM_MODES], [1], [Custom modes]) + PC_BUILD="$PC_BUILD, custom modes" +]) + +AM_CONDITIONAL([CUSTOM_MODES], [test "$enable_custom_modes" = "yes"]) + +has_float_approx=no +#case "$host_cpu" in +#i[[3456]]86 | x86_64 | powerpc64 | powerpc32 | ia64) +# has_float_approx=yes +# ;; +#esac + +AC_ARG_ENABLE([float-approx], + [AS_HELP_STRING([--enable-float-approx], [enable fast approximations for floating point])], + [if test "$enable_float_approx" = "yes"; then + AC_WARN([Floating point approximations are not supported on all platforms.]) + fi + ], + [enable_float_approx=$has_float_approx]) + +AS_IF([test "$enable_float_approx" = "yes"],[ + AC_DEFINE([FLOAT_APPROX], [1], [Float approximations]) +]) + +AC_ARG_ENABLE([asm], + [AS_HELP_STRING([--disable-asm], [Disable assembly optimizations])],, + [enable_asm=yes]) + +AC_ARG_ENABLE([rtcd], + [AS_HELP_STRING([--disable-rtcd], [Disable run-time CPU capabilities detection])],, + [enable_rtcd=yes]) + +AC_ARG_ENABLE([intrinsics], + [AS_HELP_STRING([--disable-intrinsics], [Disable intrinsics optimizations])],, + [enable_intrinsics=yes]) + +rtcd_support=no +cpu_arm=no + +AS_IF([test x"${enable_asm}" = x"yes"],[ + inline_optimization="No inline ASM for your platform, please send patches" + case $host_cpu in + arm*) + dnl Currently we only have asm for fixed-point + AS_IF([test "$enable_float" != "yes"],[ + cpu_arm=yes + AC_DEFINE([OPUS_ARM_ASM], [], [Make use of ARM asm optimization]) + AS_GCC_INLINE_ASSEMBLY( + [inline_optimization="ARM"], + [inline_optimization="disabled"] + ) + AS_ASM_ARM_EDSP([OPUS_ARM_INLINE_EDSP=1],[OPUS_ARM_INLINE_EDSP=0]) + AS_ASM_ARM_MEDIA([OPUS_ARM_INLINE_MEDIA=1], + [OPUS_ARM_INLINE_MEDIA=0]) + AS_ASM_ARM_NEON([OPUS_ARM_INLINE_NEON=1],[OPUS_ARM_INLINE_NEON=0]) + AS_IF([test x"$inline_optimization" = x"ARM"],[ + AM_CONDITIONAL([OPUS_ARM_INLINE_ASM],[true]) + AC_DEFINE([OPUS_ARM_INLINE_ASM], 1, + [Use generic ARMv4 inline asm optimizations]) + AS_IF([test x"$OPUS_ARM_INLINE_EDSP" = x"1"],[ + AC_DEFINE([OPUS_ARM_INLINE_EDSP], [1], + [Use ARMv5E inline asm optimizations]) + inline_optimization="$inline_optimization (EDSP)" + ]) + AS_IF([test x"$OPUS_ARM_INLINE_MEDIA" = x"1"],[ + AC_DEFINE([OPUS_ARM_INLINE_MEDIA], [1], + [Use ARMv6 inline asm optimizations]) + inline_optimization="$inline_optimization (Media)" + ]) + AS_IF([test x"$OPUS_ARM_INLINE_NEON" = x"1"],[ + AC_DEFINE([OPUS_ARM_INLINE_NEON], 1, + [Use ARM NEON inline asm optimizations]) + inline_optimization="$inline_optimization (NEON)" + ]) + ]) + dnl We need Perl to translate RVCT-syntax asm to gas syntax. + AC_CHECK_PROG([HAVE_PERL], perl, yes, no) + AS_IF([test x"$HAVE_PERL" = x"yes"],[ + AM_CONDITIONAL([OPUS_ARM_EXTERNAL_ASM],[true]) + asm_optimization="ARM" + AS_IF([test x"$OPUS_ARM_INLINE_EDSP" = x"1"], [ + OPUS_ARM_PRESUME_EDSP=1 + OPUS_ARM_MAY_HAVE_EDSP=1 + ], + [ + OPUS_ARM_PRESUME_EDSP=0 + OPUS_ARM_MAY_HAVE_EDSP=0 + ]) + AS_IF([test x"$OPUS_ARM_INLINE_MEDIA" = x"1"], [ + OPUS_ARM_PRESUME_MEDIA=1 + OPUS_ARM_MAY_HAVE_MEDIA=1 + ], + [ + OPUS_ARM_PRESUME_MEDIA=0 + OPUS_ARM_MAY_HAVE_MEDIA=0 + ]) + AS_IF([test x"$OPUS_ARM_INLINE_NEON" = x"1"], [ + OPUS_ARM_PRESUME_NEON=1 + OPUS_ARM_MAY_HAVE_NEON=1 + ], + [ + OPUS_ARM_PRESUME_NEON=0 + OPUS_ARM_MAY_HAVE_NEON=0 + ]) + AS_IF([test x"$enable_rtcd" = x"yes"],[ + AS_IF([test x"$OPUS_ARM_MAY_HAVE_EDSP" != x"1"],[ + AC_MSG_NOTICE( + [Trying to force-enable armv5e EDSP instructions...]) + AS_ASM_ARM_EDSP_FORCE([OPUS_ARM_MAY_HAVE_EDSP=1]) + ]) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_MEDIA" != x"1"],[ + AC_MSG_NOTICE( + [Trying to force-enable ARMv6 media instructions...]) + AS_ASM_ARM_MEDIA_FORCE([OPUS_ARM_MAY_HAVE_MEDIA=1]) + ]) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON" != x"1"],[ + AC_MSG_NOTICE( + [Trying to force-enable NEON instructions...]) + AS_ASM_ARM_NEON_FORCE([OPUS_ARM_MAY_HAVE_NEON=1]) + ]) + ]) + rtcd_support= + AS_IF([test x"$OPUS_ARM_MAY_HAVE_EDSP" = x"1"],[ + AC_DEFINE(OPUS_ARM_MAY_HAVE_EDSP, 1, + [Define if assembler supports EDSP instructions]) + AS_IF([test x"$OPUS_ARM_PRESUME_EDSP" = x"1"],[ + AC_DEFINE(OPUS_ARM_PRESUME_EDSP, 1, + [Define if binary requires EDSP instruction support]) + asm_optimization="$asm_optimization (EDSP)" + ], + [rtcd_support="$rtcd_support (EDSP)"] + ) + ]) + AC_SUBST(OPUS_ARM_MAY_HAVE_EDSP) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_MEDIA" = x"1"],[ + AC_DEFINE(OPUS_ARM_MAY_HAVE_MEDIA, 1, + [Define if assembler supports ARMv6 media instructions]) + AS_IF([test x"$OPUS_ARM_PRESUME_MEDIA" = x"1"],[ + AC_DEFINE(OPUS_ARM_PRESUME_MEDIA, 1, + [Define if binary requires ARMv6 media instruction support]) + asm_optimization="$asm_optimization (Media)" + ], + [rtcd_support="$rtcd_support (Media)"] + ) + ]) + AC_SUBST(OPUS_ARM_MAY_HAVE_MEDIA) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON" = x"1"],[ + AC_DEFINE(OPUS_ARM_MAY_HAVE_NEON, 1, + [Define if compiler supports NEON instructions]) + AS_IF([test x"$OPUS_ARM_PRESUME_NEON" = x"1"], [ + AC_DEFINE(OPUS_ARM_PRESUME_NEON, 1, + [Define if binary requires NEON instruction support]) + asm_optimization="$asm_optimization (NEON)" + ], + [rtcd_support="$rtcd_support (NEON)"] + ) + ]) + AC_SUBST(OPUS_ARM_MAY_HAVE_NEON) + dnl Make sure turning on RTCD gets us at least one + dnl instruction set. + AS_IF([test x"$rtcd_support" != x""], + [rtcd_support=ARM"$rtcd_support"], + [rtcd_support="no"] + ) + AC_MSG_CHECKING([for apple style tools]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([ +#ifndef __APPLE__ +#error 1 +#endif],[])], + [AC_MSG_RESULT([yes]); ARM2GNU_PARAMS="--apple"], + [AC_MSG_RESULT([no]); ARM2GNU_PARAMS=""]) + AC_SUBST(ARM2GNU_PARAMS) + ], + [ + AC_MSG_WARN( + [*** ARM assembly requires perl -- disabling optimizations]) + asm_optimization="(missing perl dependency for ARM)" + ]) + ]) + ;; + esac +],[ + inline_optimization="disabled" + asm_optimization="disabled" +]) + +AM_CONDITIONAL([OPUS_ARM_INLINE_ASM], + [test x"${inline_optimization%% *}" = x"ARM"]) +AM_CONDITIONAL([OPUS_ARM_EXTERNAL_ASM], + [test x"${asm_optimization%% *}" = x"ARM"]) + +AM_CONDITIONAL([HAVE_SSE], [false]) +AM_CONDITIONAL([HAVE_SSE2], [false]) +AM_CONDITIONAL([HAVE_SSE4_1], [false]) +AM_CONDITIONAL([HAVE_AVX], [false]) + +m4_define([DEFAULT_X86_SSE_CFLAGS], [-msse]) +m4_define([DEFAULT_X86_SSE2_CFLAGS], [-msse2]) +m4_define([DEFAULT_X86_SSE4_1_CFLAGS], [-msse4.1]) +m4_define([DEFAULT_X86_AVX_CFLAGS], [-mavx]) +m4_define([DEFAULT_ARM_NEON_INTR_CFLAGS], [-mfpu=neon]) +# With GCC on ARM32 softfp architectures (e.g. Android, or older Ubuntu) you need to specify +# -mfloat-abi=softfp for -mfpu=neon to work. However, on ARM32 hardfp architectures (e.g. newer Ubuntu), +# this option will break things. + +# As a heuristic, if host matches arm*eabi* but not arm*hf*, it's probably soft-float. +m4_define([DEFAULT_ARM_NEON_SOFTFP_INTR_CFLAGS], [-mfpu=neon -mfloat-abi=softfp]) + +AS_CASE([$host], + [arm*hf*], [AS_VAR_SET([RESOLVED_DEFAULT_ARM_NEON_INTR_CFLAGS], "DEFAULT_ARM_NEON_INTR_CFLAGS")], + [arm*eabi*], [AS_VAR_SET([RESOLVED_DEFAULT_ARM_NEON_INTR_CFLAGS], "DEFAULT_ARM_NEON_SOFTFP_INTR_CFLAGS")], + [AS_VAR_SET([RESOLVED_DEFAULT_ARM_NEON_INTR_CFLAGS], "DEFAULT_ARM_NEON_INTR_CFLAGS")]) + +AC_ARG_VAR([X86_SSE_CFLAGS], [C compiler flags to compile SSE intrinsics @<:@default=]DEFAULT_X86_SSE_CFLAGS[@:>@]) +AC_ARG_VAR([X86_SSE2_CFLAGS], [C compiler flags to compile SSE2 intrinsics @<:@default=]DEFAULT_X86_SSE2_CFLAGS[@:>@]) +AC_ARG_VAR([X86_SSE4_1_CFLAGS], [C compiler flags to compile SSE4.1 intrinsics @<:@default=]DEFAULT_X86_SSE4_1_CFLAGS[@:>@]) +AC_ARG_VAR([X86_AVX_CFLAGS], [C compiler flags to compile AVX intrinsics @<:@default=]DEFAULT_X86_AVX_CFLAGS[@:>@]) +AC_ARG_VAR([ARM_NEON_INTR_CFLAGS], [C compiler flags to compile ARM NEON intrinsics @<:@default=]DEFAULT_ARM_NEON_INTR_CFLAGS / DEFAULT_ARM_NEON_SOFTFP_INTR_CFLAGS[@:>@]) + +AS_VAR_SET_IF([X86_SSE_CFLAGS], [], [AS_VAR_SET([X86_SSE_CFLAGS], "DEFAULT_X86_SSE_CFLAGS")]) +AS_VAR_SET_IF([X86_SSE2_CFLAGS], [], [AS_VAR_SET([X86_SSE2_CFLAGS], "DEFAULT_X86_SSE2_CFLAGS")]) +AS_VAR_SET_IF([X86_SSE4_1_CFLAGS], [], [AS_VAR_SET([X86_SSE4_1_CFLAGS], "DEFAULT_X86_SSE4_1_CFLAGS")]) +AS_VAR_SET_IF([X86_AVX_CFLAGS], [], [AS_VAR_SET([X86_AVX_CFLAGS], "DEFAULT_X86_AVX_CFLAGS")]) +AS_VAR_SET_IF([ARM_NEON_INTR_CFLAGS], [], [AS_VAR_SET([ARM_NEON_INTR_CFLAGS], ["$RESOLVED_DEFAULT_ARM_NEON_INTR_CFLAGS"])]) + +AC_DEFUN([OPUS_PATH_NE10], + [ + AC_ARG_WITH(NE10, + AC_HELP_STRING([--with-NE10=PFX],[Prefix where libNE10 is installed (optional)]), + NE10_prefix="$withval", NE10_prefix="") + AC_ARG_WITH(NE10-libraries, + AC_HELP_STRING([--with-NE10-libraries=DIR], + [Directory where libNE10 library is installed (optional)]), + NE10_libraries="$withval", NE10_libraries="") + AC_ARG_WITH(NE10-includes, + AC_HELP_STRING([--with-NE10-includes=DIR], + [Directory where libNE10 header files are installed (optional)]), + NE10_includes="$withval", NE10_includes="") + + if test "x$NE10_libraries" != "x" ; then + NE10_LIBS="-L$NE10_libraries" + elif test "x$NE10_prefix" = "xno" || test "x$NE10_prefix" = "xyes" ; then + NE10_LIBS="" + elif test "x$NE10_prefix" != "x" ; then + NE10_LIBS="-L$NE10_prefix/lib" + elif test "x$prefix" != "xNONE" ; then + NE10_LIBS="-L$prefix/lib" + fi + + if test "x$NE10_prefix" != "xno" ; then + NE10_LIBS="$NE10_LIBS -lNE10" + fi + + if test "x$NE10_includes" != "x" ; then + NE10_CFLAGS="-I$NE10_includes" + elif test "x$NE10_prefix" = "xno" || test "x$NE10_prefix" = "xyes" ; then + NE10_CFLAGS="" + elif test "x$NE10_prefix" != "x" ; then + NE10_CFLAGS="-I$NE10_prefix/include" + elif test "x$prefix" != "xNONE"; then + NE10_CFLAGS="-I$prefix/include" + fi + + AC_MSG_CHECKING(for NE10) + save_CFLAGS="$CFLAGS"; CFLAGS="$CFLAGS $NE10_CFLAGS" + save_LIBS="$LIBS"; LIBS="$LIBS $NE10_LIBS $LIBM" + AC_LINK_IFELSE( + [ + AC_LANG_PROGRAM( + [[#include + ]], + [[ + ne10_fft_cfg_float32_t cfg; + cfg = ne10_fft_alloc_c2c_float32_neon(480); + ]] + ) + ],[ + HAVE_ARM_NE10=1 + AC_MSG_RESULT([yes]) + ],[ + HAVE_ARM_NE10=0 + AC_MSG_RESULT([no]) + NE10_CFLAGS="" + NE10_LIBS="" + ] + ) + CFLAGS="$save_CFLAGS"; LIBS="$save_LIBS" + #Now we know if libNE10 is installed or not + AS_IF([test x"$HAVE_ARM_NE10" = x"1"], + [ + AC_DEFINE([HAVE_ARM_NE10], 1, [NE10 library is installed on host. Make sure it is on target!]) + AC_SUBST(HAVE_ARM_NE10) + AC_SUBST(NE10_CFLAGS) + AC_SUBST(NE10_LIBS) + ] + ) + ] +) + +AS_IF([test x"$enable_intrinsics" = x"yes"],[ + intrinsics_support="" + AS_CASE([$host_cpu], + [arm*|aarch64*], + [ + cpu_arm=yes + OPUS_CHECK_INTRINSICS( + [ARM Neon], + [$ARM_NEON_INTR_CFLAGS], + [OPUS_ARM_MAY_HAVE_NEON_INTR], + [OPUS_ARM_PRESUME_NEON_INTR], + [[#include + ]], + [[ + static float32x4_t A0, A1, SUMM; + SUMM = vmlaq_f32(SUMM, A0, A1); + return (int)vgetq_lane_f32(SUMM, 0); + ]] + ) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1" && test x"$OPUS_ARM_PRESUME_NEON_INTR" != x"1"], + [ + OPUS_ARM_NEON_INTR_CFLAGS="$ARM_NEON_INTR_CFLAGS" + AC_SUBST([OPUS_ARM_NEON_INTR_CFLAGS]) + ] + ) + + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1"], + [ + AC_DEFINE([OPUS_ARM_MAY_HAVE_NEON_INTR], 1, [Compiler supports ARMv7/Aarch64 Neon Intrinsics]) + intrinsics_support="$intrinsics_support (NEON)" + + AS_IF([test x"$enable_rtcd" != x"no" && test x"$OPUS_ARM_PRESUME_NEON_INTR" != x"1"], + [AS_IF([test x"$rtcd_support" = x"no"], + [rtcd_support="ARM (NEON Intrinsics)"], + [rtcd_support="$rtcd_support (NEON Intrinsics)"])]) + + AS_IF([test x"$OPUS_ARM_PRESUME_NEON_INTR" = x"1"], + [AC_DEFINE([OPUS_ARM_PRESUME_NEON_INTR], 1, [Define if binary requires NEON intrinsics support])]) + + OPUS_PATH_NE10() + AS_IF([test x"$NE10_LIBS" != x""], + [ + intrinsics_support="$intrinsics_support (NE10)" + AS_IF([test x"enable_rtcd" != x"" \ + && test x"$OPUS_ARM_PRESUME_NEON_INTR" != x"1"], + [rtcd_support="$rtcd_support (NE10)"]) + ]) + + OPUS_CHECK_INTRINSICS( + [Aarch64 Neon], + [$ARM_NEON_INTR_CFLAGS], + [OPUS_ARM_MAY_HAVE_AARCH64_NEON_INTR], + [OPUS_ARM_PRESUME_AARCH64_NEON_INTR], + [[#include + ]], + [[ + static int32_t IN; + static int16_t OUT; + OUT = vqmovns_s32(IN); + ]] + ) + + AS_IF([test x"$OPUS_ARM_PRESUME_AARCH64_NEON_INTR" = x"1"], + [ + AC_DEFINE([OPUS_ARM_PRESUME_AARCH64_NEON_INTR], 1, [Define if binary requires Aarch64 Neon Intrinsics]) + intrinsics_support="$intrinsics_support (NEON [Aarch64])" + ]) + + AS_IF([test x"$intrinsics_support" = x""], + [intrinsics_support=no], + [intrinsics_support="ARM$intrinsics_support"]) + ], + [ + AC_MSG_WARN([Compiler does not support ARM intrinsics]) + intrinsics_support=no + ]) + ], + [i?86|x86_64], + [ + OPUS_CHECK_INTRINSICS( + [SSE], + [$X86_SSE_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE], + [OPUS_X86_PRESUME_SSE], + [[#include + #include + ]], + [[ + __m128 mtest; + mtest = _mm_set1_ps((float)time(NULL)); + mtest = _mm_mul_ps(mtest, mtest); + return _mm_cvtss_si32(mtest); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE" = x"1" && test x"$OPUS_X86_PRESUME_SSE" != x"1"], + [ + OPUS_X86_SSE_CFLAGS="$X86_SSE_CFLAGS" + AC_SUBST([OPUS_X86_SSE_CFLAGS]) + ] + ) + OPUS_CHECK_INTRINSICS( + [SSE2], + [$X86_SSE2_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE2], + [OPUS_X86_PRESUME_SSE2], + [[#include + #include + ]], + [[ + __m128i mtest; + mtest = _mm_set1_epi32((int)time(NULL)); + mtest = _mm_mul_epu32(mtest, mtest); + return _mm_cvtsi128_si32(mtest); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1" && test x"$OPUS_X86_PRESUME_SSE2" != x"1"], + [ + OPUS_X86_SSE2_CFLAGS="$X86_SSE2_CFLAGS" + AC_SUBST([OPUS_X86_SSE2_CFLAGS]) + ] + ) + OPUS_CHECK_INTRINSICS( + [SSE4.1], + [$X86_SSE4_1_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE4_1], + [OPUS_X86_PRESUME_SSE4_1], + [[#include + #include + ]], + [[ + __m128i mtest; + mtest = _mm_set1_epi32((int)time(NULL)); + mtest = _mm_mul_epi32(mtest, mtest); + return _mm_cvtsi128_si32(mtest); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1" && test x"$OPUS_X86_PRESUME_SSE4_1" != x"1"], + [ + OPUS_X86_SSE4_1_CFLAGS="$X86_SSE4_1_CFLAGS" + AC_SUBST([OPUS_X86_SSE4_1_CFLAGS]) + ] + ) + OPUS_CHECK_INTRINSICS( + [AVX], + [$X86_AVX_CFLAGS], + [OPUS_X86_MAY_HAVE_AVX], + [OPUS_X86_PRESUME_AVX], + [[#include + #include + ]], + [[ + __m256 mtest; + mtest = _mm256_set1_ps((float)time(NULL)); + mtest = _mm256_addsub_ps(mtest, mtest); + return _mm_cvtss_si32(_mm256_extractf128_ps(mtest, 0)); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_AVX" = x"1" && test x"$OPUS_X86_PRESUME_AVX" != x"1"], + [ + OPUS_X86_AVX_CFLAGS="$X86_AVX_CFLAGS" + AC_SUBST([OPUS_X86_AVX_CFLAGS]) + ] + ) + AS_IF([test x"$rtcd_support" = x"no"], [rtcd_support=""]) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE], 1, [Compiler supports X86 SSE Intrinsics]) + intrinsics_support="$intrinsics_support SSE" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE], 1, [Define if binary requires SSE intrinsics support])], + [rtcd_support="$rtcd_support SSE"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE intrinsics]) + ]) + + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE2], 1, [Compiler supports X86 SSE2 Intrinsics]) + intrinsics_support="$intrinsics_support SSE2" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE2" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE2], 1, [Define if binary requires SSE2 intrinsics support])], + [rtcd_support="$rtcd_support SSE2"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE2 intrinsics]) + ]) + + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE4_1], 1, [Compiler supports X86 SSE4.1 Intrinsics]) + intrinsics_support="$intrinsics_support SSE4.1" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE4_1" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE4_1], 1, [Define if binary requires SSE4.1 intrinsics support])], + [rtcd_support="$rtcd_support SSE4.1"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE4.1 intrinsics]) + ]) + AS_IF([test x"$OPUS_X86_MAY_HAVE_AVX" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_AVX], 1, [Compiler supports X86 AVX Intrinsics]) + intrinsics_support="$intrinsics_support AVX" + + AS_IF([test x"$OPUS_X86_PRESUME_AVX" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_AVX], 1, [Define if binary requires AVX intrinsics support])], + [rtcd_support="$rtcd_support AVX"]) + ], + [ + AC_MSG_WARN([Compiler does not support AVX intrinsics]) + ]) + + AS_IF([test x"$intrinsics_support" = x""], + [intrinsics_support=no], + [intrinsics_support="x86$intrinsics_support"] + ) + AS_IF([test x"$rtcd_support" = x""], + [rtcd_support=no], + [rtcd_support="x86$rtcd_support"], + ) + + AS_IF([test x"$enable_rtcd" = x"yes" && test x"$rtcd_support" != x""],[ + get_cpuid_by_asm="no" + AC_MSG_CHECKING([How to get X86 CPU Info]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + ]],[[ + unsigned int CPUInfo0; + unsigned int CPUInfo1; + unsigned int CPUInfo2; + unsigned int CPUInfo3; + unsigned int InfoType; + #if defined(__i386__) && defined(__PIC__) + __asm__ __volatile__ ( + "xchg %%ebx, %1\n" + "cpuid\n" + "xchg %%ebx, %1\n": + "=a" (CPUInfo0), + "=r" (CPUInfo1), + "=c" (CPUInfo2), + "=d" (CPUInfo3) : + "a" (InfoType), "c" (0) + ); + #else + __asm__ __volatile__ ( + "cpuid": + "=a" (CPUInfo0), + "=b" (CPUInfo1), + "=c" (CPUInfo2), + "=d" (CPUInfo3) : + "a" (InfoType), "c" (0) + ); + #endif + ]])], + [get_cpuid_by_asm="yes" + AC_MSG_RESULT([Inline Assembly]) + AC_DEFINE([CPU_INFO_BY_ASM], [1], [Get CPU Info by asm method])], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + ]],[[ + unsigned int CPUInfo0; + unsigned int CPUInfo1; + unsigned int CPUInfo2; + unsigned int CPUInfo3; + unsigned int InfoType; + __get_cpuid(InfoType, &CPUInfo0, &CPUInfo1, &CPUInfo2, &CPUInfo3); + ]])], + [AC_MSG_RESULT([C method]) + AC_DEFINE([CPU_INFO_BY_C], [1], [Get CPU Info by c method])], + [AC_MSG_ERROR([no supported Get CPU Info method, please disable run-time CPU capabilities detection or intrinsics])])])]) + ], + [ + AC_MSG_WARN([No intrinsics support for your architecture]) + intrinsics_support="no" + ]) +], +[ + intrinsics_support="no" +]) + +AM_CONDITIONAL([CPU_ARM], [test "$cpu_arm" = "yes"]) +AM_CONDITIONAL([HAVE_ARM_NEON_INTR], + [test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1"]) +AM_CONDITIONAL([HAVE_ARM_NE10], + [test x"$HAVE_ARM_NE10" = x"1"]) +AM_CONDITIONAL([HAVE_SSE], + [test x"$OPUS_X86_MAY_HAVE_SSE" = x"1"]) +AM_CONDITIONAL([HAVE_SSE2], + [test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1"]) +AM_CONDITIONAL([HAVE_SSE4_1], + [test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1"]) +AM_CONDITIONAL([HAVE_AVX], + [test x"$OPUS_X86_MAY_HAVE_AVX" = x"1"]) + +AS_IF([test x"$enable_rtcd" = x"yes"],[ + AS_IF([test x"$rtcd_support" != x"no"],[ + AC_DEFINE([OPUS_HAVE_RTCD], [1], + [Use run-time CPU capabilities detection]) + OPUS_HAVE_RTCD=1 + AC_SUBST(OPUS_HAVE_RTCD) + ]) +],[ + rtcd_support="disabled" +]) + +AC_ARG_ENABLE([assertions], + [AS_HELP_STRING([--enable-assertions],[enable additional software error checking])],, + [enable_assertions=no]) + +AS_IF([test "$enable_assertions" = "yes"], [ + AC_DEFINE([ENABLE_ASSERTIONS], [1], [Assertions]) +]) + +AC_ARG_ENABLE([hardening], + [AS_HELP_STRING([--disable-hardening],[disable run-time checks that are cheap and safe for use in production])],, + [enable_hardening=yes]) + +AS_IF([test "$enable_hardening" = "yes"], [ + AC_DEFINE([ENABLE_HARDENING], [1], [Hardening]) +]) + +AC_ARG_ENABLE([fuzzing], + [AS_HELP_STRING([--enable-fuzzing],[causes the encoder to make random decisions (do not use in production)])],, + [enable_fuzzing=no]) + +AS_IF([test "$enable_fuzzing" = "yes"], [ + AC_DEFINE([FUZZING], [1], [Fuzzing]) +]) + +AC_ARG_ENABLE([check-asm], + [AS_HELP_STRING([--enable-check-asm], + [enable bit-exactness checks between optimized and c implementations])],, + [enable_check_asm=no]) + +AS_IF([test "$enable_check_asm" = "yes"], [ + AC_DEFINE([OPUS_CHECK_ASM], [1], [Run bit-exactness checks between optimized and c implementations]) +]) + +AC_ARG_ENABLE([doc], + [AS_HELP_STRING([--disable-doc], [Do not build API documentation])],, + [enable_doc=yes]) + +AS_IF([test "$enable_doc" = "yes"], [ + AC_CHECK_PROG(HAVE_DOXYGEN, [doxygen], [yes], [no]) + AC_CHECK_PROG(HAVE_DOT, [dot], [yes], [no]) +],[ + HAVE_DOXYGEN=no +]) + +AM_CONDITIONAL([HAVE_DOXYGEN], [test "$HAVE_DOXYGEN" = "yes"]) + +AC_ARG_ENABLE([extra-programs], + [AS_HELP_STRING([--disable-extra-programs], [Do not build extra programs (demo and tests)])],, + [enable_extra_programs=yes]) + +AM_CONDITIONAL([EXTRA_PROGRAMS], [test "$enable_extra_programs" = "yes"]) + + +AC_ARG_ENABLE([rfc8251], + AS_HELP_STRING([--disable-rfc8251], [Disable bitstream fixes from RFC 8251]),, + [enable_rfc8251=yes]) + +AS_IF([test "$enable_rfc8251" = "no"], [ + AC_DEFINE([DISABLE_UPDATE_DRAFT], [1], [Disable bitstream fixes from RFC 8251]) +]) + + +saved_CFLAGS="$CFLAGS" +CFLAGS="$CFLAGS -fvisibility=hidden" +AC_MSG_CHECKING([if ${CC} supports -fvisibility=hidden]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], + [ AC_MSG_RESULT([yes]) ], + [ AC_MSG_RESULT([no]) + CFLAGS="$saved_CFLAGS" + ]) + +on_x86=no +case "$host_cpu" in +i[[3456]]86 | x86_64) + on_x86=yes + ;; +esac + +on_windows=no +case $host in +*cygwin*|*mingw*) + on_windows=yes + ;; +esac + +dnl Enable stack-protector-all only on x86 where it's well supported. +dnl on some platforms it causes crashes. Hopefully the OS's default's +dnl include this on platforms that work but have been missed here. +AC_ARG_ENABLE([stack-protector], + [AS_HELP_STRING([--disable-stack-protector],[Disable compiler stack hardening])],, + [ + AS_IF([test "$ac_cv_c_compiler_gnu" = "yes" && test "$on_x86" = "yes" && test "$on_windows" = "no"], + [enable_stack_protector=yes],[enable_stack_protector=no]) + ]) + +AS_IF([test "$enable_stack_protector" = "yes"], + [ + saved_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -fstack-protector-strong" + AC_MSG_CHECKING([if ${CC} supports -fstack-protector-strong]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[[char foo;]])], + [ AC_MSG_RESULT([yes]) ], + [ + AC_MSG_RESULT([no]) + enable_stack_protector=no + CFLAGS="$saved_CFLAGS" + ]) + ]) + +AS_IF([test x$ac_cv_c_compiler_gnu = xyes], + [AX_ADD_FORTIFY_SOURCE] +) + +CFLAGS="$CFLAGS -W" + +warn_CFLAGS="-Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes" +saved_CFLAGS="$CFLAGS" +CFLAGS="$CFLAGS $warn_CFLAGS" +AC_MSG_CHECKING([if ${CC} supports ${warn_CFLAGS}]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], + [ AC_MSG_RESULT([yes]) ], + [ AC_MSG_RESULT([no]) + CFLAGS="$saved_CFLAGS" + ]) + +saved_LIBS="$LIBS" +LIBS="$LIBS $LIBM" +AC_CHECK_FUNCS([lrintf]) +AC_CHECK_FUNCS([lrint]) +LIBS="$saved_LIBS" + +AC_CHECK_FUNCS([__malloc_hook]) + +AC_SUBST([PC_BUILD]) + +AC_CONFIG_FILES([ + Makefile + opus.pc + opus-uninstalled.pc + celt/arm/armopts.s + doc/Makefile + doc/Doxyfile +]) +AC_CONFIG_HEADERS([config.h]) + +AC_OUTPUT + +AC_MSG_NOTICE([ +------------------------------------------------------------------------ + $PACKAGE_NAME $PACKAGE_VERSION: Automatic configuration OK. + + Compiler support: + + C99 var arrays: ................ ${has_var_arrays} + C99 lrintf: .................... ${ac_cv_func_lrintf} + Use alloca: .................... ${use_alloca} + + General configuration: + + Floating point support: ........ ${enable_float} + Fast float approximations: ..... ${enable_float_approx} + Fixed point debugging: ......... ${enable_fixed_point_debug} + Inline Assembly Optimizations: . ${inline_optimization} + External Assembly Optimizations: ${asm_optimization} + Intrinsics Optimizations: ...... ${intrinsics_support} + Run-time CPU detection: ........ ${rtcd_support} + Custom modes: .................. ${enable_custom_modes} + Assertion checking: ............ ${enable_assertions} + Hardening: ..................... ${enable_hardening} + Fuzzing: ....................... ${enable_fuzzing} + Check ASM: ..................... ${enable_check_asm} + + API documentation: ............. ${enable_doc} + Extra programs: ................ ${enable_extra_programs} +------------------------------------------------------------------------ + + Type "make; make install" to compile and install + Type "make check" to run the test suite +]) + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Doxyfile.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Doxyfile.in new file mode 100644 index 000000000..6eef650b2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Doxyfile.in @@ -0,0 +1,340 @@ +# Doxyfile 1.8.18 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +# Only non-default options are included below to improve portability +# between doxygen versions. +# +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = Opus + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @VERSION@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Opus audio codec (RFC 6716): API and operations manual" + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = YES + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 8 + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# (including Cygwin) ands Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = YES + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = @top_srcdir@/include/opus.h \ + @top_srcdir@/include/opus_types.h \ + @top_srcdir@/include/opus_defines.h \ + @top_srcdir@/include/opus_multistream.h \ + @top_srcdir@/include/opus_custom.h + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = @top_srcdir@/doc/header.html + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = @top_srcdir@/doc/footer.html + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = @top_srcdir@/doc/customdoxygen.css + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = @top_srcdir@/doc/opus_logo.svg + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 0 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = YES + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://www.mathjax.org/mathjax + +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# The PAPER_TYPE tag can be used to set the paper type that is used by the +# printer. +# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x +# 14 inches) and executive (7.25 x 10.5 inches). +# The default value is: a4. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PAPER_TYPE = letter + +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for +# classes and files. +# The default value is: NO. + +GENERATE_MAN = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = + +# The PREDEFINED tag can be used to specify one or more macro names that are +# defined before the preprocessor is started (similar to the -D option of e.g. +# gcc). The argument of the tag is a list of macros of the form: name or +# name=definition (no spaces). If the definition and the "=" are omitted, "=1" +# is assumed. To prevent a macro definition from being undefined via #undef or +# recursively expanded use the := operator instead of the = operator. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +PREDEFINED = OPUS_EXPORT= \ + OPUS_CUSTOM_EXPORT= \ + OPUS_CUSTOM_EXPORT_STATIC= \ + OPUS_WARN_UNUSED_RESULT= \ + OPUS_ARG_NONNULL(_x)= + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz (see: +# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# Bell Labs. The other options in this section have no effect if this option is +# set to NO +# The default value is: NO. + +# Debian defaults to YES here, while Fedora and Homebrew default to NO. +# So we set this based on whether the graphviz package is available at +# configure time. +# +HAVE_DOT = @HAVE_DOT@ + +# move docs to the correct place +OUTPUT_DIRECTORY = @top_builddir@/doc diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Makefile.am b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Makefile.am new file mode 100644 index 000000000..31fddab49 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/Makefile.am @@ -0,0 +1,45 @@ +## Process this file with automake to produce Makefile.in + +DOCINPUTS = $(top_srcdir)/include/opus.h \ + $(top_srcdir)/include/opus_multistream.h \ + $(top_srcdir)/include/opus_defines.h \ + $(top_srcdir)/include/opus_types.h \ + $(top_srcdir)/include/opus_custom.h \ + $(top_srcdir)/doc/header.html \ + $(top_srcdir)/doc/footer.html \ + $(top_srcdir)/doc/customdoxygen.css + +EXTRA_DIST = customdoxygen.css Doxyfile.in footer.html header.html \ + opus_logo.svg trivial_example.c + + +if HAVE_DOXYGEN + +all-local: doxygen-build.stamp + +doxygen-build.stamp: Doxyfile $(DOCINPUTS) + doxygen + touch $@ + +install-data-local: + $(INSTALL) -d $(DESTDIR)$(docdir)/html/search + for f in `find html -type f \! -name "installdox"`; do \ + $(INSTALL_DATA) $$f $(DESTDIR)$(docdir)/$$f; \ + done + + $(INSTALL) -d $(DESTDIR)$(mandir)/man3 + cd man && find man3 -type f -name opus_*.3 \ + -exec $(INSTALL_DATA) \{} $(DESTDIR)$(mandir)/man3 \; + +clean-local: + $(RM) -r html + $(RM) -r latex + $(RM) -r man + $(RM) doxygen-build.stamp + $(RM) doxygen_sqlite3.db + +uninstall-local: + $(RM) -r $(DESTDIR)$(docdir)/html + $(RM) $(DESTDIR)$(mandir)/man3/opus_*.3 $(DESTDIR)$(mandir)/man3/opus.h.3 + +endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_draft.sh b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_draft.sh new file mode 100755 index 000000000..14ae51dd5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_draft.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +# Copyright (c) 2011-2012 Xiph.Org Foundation and Mozilla Corporation +# +# This file is extracted from RFC6716. Please see that RFC for additional +# information. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# - Neither the name of Internet Society, IETF or IETF Trust, nor the +# names of specific contributors, may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#Stop on errors +set -e +#Set the CWD to the location of this script +[ -n "${0%/*}" ] && cd "${0%/*}" + +toplevel=".." +destdir="opus_source" + +echo packaging source code +rm -rf "${destdir}" +mkdir "${destdir}" +mkdir "${destdir}/src" +mkdir "${destdir}/silk" +mkdir "${destdir}/silk/float" +mkdir "${destdir}/silk/fixed" +mkdir "${destdir}/silk/fixed/x86" +mkdir "${destdir}/silk/fixed/arm" +mkdir "${destdir}/silk/fixed/mips" +mkdir "${destdir}/silk/x86" +mkdir "${destdir}/silk/arm" +mkdir "${destdir}/silk/mips" +mkdir "${destdir}/celt" +mkdir "${destdir}/celt/x86" +mkdir "${destdir}/celt/arm" +mkdir "${destdir}/celt/mips" +mkdir "${destdir}/include" +for f in `cat "${toplevel}"/opus_sources.mk "${toplevel}"/celt_sources.mk \ + "${toplevel}"/silk_sources.mk "${toplevel}"/opus_headers.mk \ + "${toplevel}"/celt_headers.mk "${toplevel}"/silk_headers.mk \ + | grep '\.[ch]' | sed -e 's/^.*=//' -e 's/\\\\//'` ; do + cp -a "${toplevel}/${f}" "${destdir}/${f}" +done +cp -a "${toplevel}"/src/opus_demo.c "${destdir}"/src/ +cp -a "${toplevel}"/src/opus_compare.c "${destdir}"/src/ +cp -a "${toplevel}"/celt/opus_custom_demo.c "${destdir}"/celt/ +cp -a "${toplevel}"/Makefile.unix "${destdir}"/Makefile +cp -a "${toplevel}"/opus_sources.mk "${destdir}"/ +cp -a "${toplevel}"/celt_sources.mk "${destdir}"/ +cp -a "${toplevel}"/silk_sources.mk "${destdir}"/ +cp -a "${toplevel}"/README.draft "${destdir}"/README +cp -a "${toplevel}"/COPYING "${destdir}"/COPYING +cp -a "${toplevel}"/tests/run_vectors.sh "${destdir}"/ + +GZIP=-9 tar --owner=root --group=root --format=v7 -czf opus_source.tar.gz "${destdir}" +echo building base64 version +cat opus_source.tar.gz| base64 | tr -d '\n' | fold -w 64 | \ + sed -e 's/^/\###/' -e 's/$/\<\/spanx\>\/' > \ + opus_source.base64 + + +#echo '
' > opus_compare_escaped.c +#echo '' >> opus_compare_escaped.c +#echo '> opus_compare_escaped.c +#cat opus_compare.c >> opus_compare_escaped.c +#echo ']]>' >> opus_compare_escaped.c +#echo '' >> opus_compare_escaped.c +#echo '
' >> opus_compare_escaped.c + +if [ ! -d ../opus_testvectors ] ; then + echo "Downloading test vectors..." + wget 'http://opus-codec.org/testvectors/opus_testvectors.tar.gz' + tar -C .. -xvzf opus_testvectors.tar.gz +fi +echo '
' > testvectors_sha1 +echo '' >> testvectors_sha1 +echo '> testvectors_sha1 +(cd ../opus_testvectors; sha1sum *.bit *.dec) >> testvectors_sha1 +#cd opus_testvectors +#sha1sum *.bit *.dec >> ../testvectors_sha1 +#cd .. +echo ']]>' >> testvectors_sha1 +echo '' >> testvectors_sha1 +echo '
' >> testvectors_sha1 + +echo running xml2rfc +xml2rfc draft-ietf-codec-opus.xml draft-ietf-codec-opus.html & +xml2rfc draft-ietf-codec-opus.xml +wait diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_isobmff.sh b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_isobmff.sh new file mode 100755 index 000000000..95ea202e4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_isobmff.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +# Copyright (c) 2014 Xiph.Org Foundation and Mozilla Foundation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#Stop on errors +set -e +#Set the CWD to the location of this script +[ -n "${0%/*}" ] && cd "${0%/*}" + +HTML=opus_in_isobmff.html + +echo downloading updates... +CSS=${HTML%%.html}.css +wget -q http://vfrmaniac.fushizen.eu/contents/${HTML} -O ${HTML} +wget -q http://vfrmaniac.fushizen.eu/style.css -O ${CSS} + +echo updating links... +cat ${HTML} | sed -e "s/\\.\\.\\/style.css/${CSS}/" > ${HTML}+ && mv ${HTML}+ ${HTML} + +echo stripping... +cat ${HTML} | sed -e 's///g' > ${HTML}+ && mv ${HTML}+ ${HTML} +cat ${HTML} | sed -e 's/ *$//g' > ${HTML}+ && mv ${HTML}+ ${HTML} +cat ${CSS} | sed -e 's/ *$//g' > ${CSS}+ && mv ${CSS}+ ${CSS} + + +VERSION=$(fgrep Version ${HTML} | sed 's/.*Version \([0-9]\.[0-9]\.[0-9]\).*/\1/') +echo Now at version ${VERSION} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_oggdraft.sh b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_oggdraft.sh new file mode 100755 index 000000000..30ee534b0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/build_oggdraft.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +# Copyright (c) 2012 Xiph.Org Foundation and Mozilla Corporation +# +# This file is extracted from RFC6716. Please see that RFC for additional +# information. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# - Neither the name of Internet Society, IETF or IETF Trust, nor the +# names of specific contributors, may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#Stop on errors +set -e +#Set the CWD to the location of this script +[ -n "${0%/*}" ] && cd "${0%/*}" + +if test -z `which xml2rfc 2> /dev/null`; then + echo "Error: couldn't find xml2rfc." + echo + echo "Please install xml2rfc version 2 or later." + echo "E.g. 'pip install xml2rfc' or follow the instructions" + echo "on http://pypi.python.org/pypi/xml2rfc/ or tools.ietf.org." + exit 1 +fi + +echo running xml2rfc +# version 2 syntax +xml2rfc draft-ietf-codec-oggopus.xml --text --html diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/customdoxygen.css b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/customdoxygen.css new file mode 100644 index 000000000..70047787c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/customdoxygen.css @@ -0,0 +1,1011 @@ +/* The standard CSS for doxygen */ + +body, table, div, p, dl { + font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif; + font-size: 13px; + line-height: 1.3; +} + +/* @group Heading Levels */ + +h1 { + font-size: 150%; +} + +.title { + font-size: 150%; + font-weight: bold; + margin: 10px 2px; +} + +h2 { + font-size: 120%; +} + +h3 { + font-size: 100%; +} + +dt { + font-weight: bold; +} + +div.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; +} + +p.startli, p.startdd, p.starttd { + margin-top: 2px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.qindex, div.navtab{ + background-color: #F1F1F1; + border: 1px solid #BDBDBD; + text-align: center; +} + +div.qindex, div.navpath { + width: 100%; + line-height: 140%; +} + +div.navtab { + margin-right: 15px; +} + +/* @group Link Styling */ + +a { + color: #646464; + font-weight: normal; + text-decoration: none; +} + +.contents a:visited { + color: #747474; +} + +a:hover { + text-decoration: underline; +} + +a.qindex { + font-weight: bold; +} + +a.qindexHL { + font-weight: bold; + background-color: #B8B8B8; + color: #ffffff; + border: 1px double #A8A8A8; +} + +.contents a.qindexHL:visited { + color: #ffffff; +} + +a.el { + font-weight: bold; +} + +a.elRef { +} + +a.code, a.code:visited { + color: #4665A2; +} + +a.codeRef, a.codeRef:visited { + color: #4665A2; +} + +/* @end */ + +dl.el { + margin-left: -1cm; +} + +.fragment { + font-family: monospace, fixed; + font-size: 105%; +} + +pre.fragment { + border: 1px solid #D5D5D5; + background-color: #FCFCFC; + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; +} + +div.ah { + background-color: black; + font-weight: bold; + color: #ffffff; + margin-bottom: 3px; + margin-top: 3px; + padding: 0.2em; + border: solid thin #333; + border-radius: 0.5em; + -webkit-border-radius: .5em; + -moz-border-radius: .5em; + box-shadow: 2px 2px 3px #999; + -webkit-box-shadow: 2px 2px 3px #999; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); + background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000); +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +body { + background-color: white; + color: black; + margin: 0; +} + +div.contents { + margin-top: 10px; + margin-left: 8px; + margin-right: 8px; +} + +td.indexkey { + background-color: #F1F1F1; + font-weight: bold; + border: 1px solid #D5D5D5; + margin: 2px 0px 2px 0; + padding: 2px 10px; + white-space: nowrap; + vertical-align: top; +} + +td.indexvalue { + background-color: #F1F1F1; + border: 1px solid #D5D5D5; + padding: 2px 10px; + margin: 2px 0px; +} + +tr.memlist { + background-color: #F2F2F2; +} + +p.formulaDsp { + text-align: center; +} + +img.formulaDsp { + +} + +img.formulaInl { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; +} + +/* @group Code Colorization */ + +span.keyword { + color: #008000 +} + +span.keywordtype { + color: #604020 +} + +span.keywordflow { + color: #e08000 +} + +span.comment { + color: #800000 +} + +span.preprocessor { + color: #806020 +} + +span.stringliteral { + color: #002080 +} + +span.charliteral { + color: #008080 +} + +span.vhdldigit { + color: #ff00ff +} + +span.vhdlchar { + color: #000000 +} + +span.vhdlkeyword { + color: #700070 +} + +span.vhdllogic { + color: #ff0000 +} + +blockquote { + background-color: #F9F9F9; + border-left: 2px solid #B8B8B8; + margin: 0 24px 0 4px; + padding: 0 12px 0 16px; +} + +/* @end */ + +/* +.search { + color: #003399; + font-weight: bold; +} + +form.search { + margin-bottom: 0px; + margin-top: 0px; +} + +input.search { + font-size: 75%; + color: #000080; + font-weight: normal; + background-color: #e8eef2; +} +*/ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid #BDBDBD; +} + +th.dirtab { + background: #F1F1F1; + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid #7A7A7A; +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: #FAFAFA; + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: #555; +} + +.memItemLeft, .memItemRight, .memTemplParams { + border-top: 1px solid #D5D5D5; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight { + width: 100%; +} + +.memTemplParams { + color: #747474; + white-space: nowrap; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtemplate { + font-size: 80%; + color: #747474; + font-weight: normal; + margin-left: 9px; +} + +.memnav { + background-color: #F1F1F1; + border: 1px solid #BDBDBD; + text-align: center; + margin: 2px; + margin-right: 15px; + padding: 2px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; +} + +.memname { + white-space: nowrap; + font-weight: bold; + margin-left: 6px; +} + +.memproto, dl.reflist dt { + border-top: 1px solid #C0C0C0; + border-left: 1px solid #C0C0C0; + border-right: 1px solid #C0C0C0; + padding: 6px 0px 6px 0px; + color: #3D3D3D; + font-weight: bold; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + /* opera specific markup */ + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 8px; + border-top-left-radius: 8px; + /* firefox specific markup */ + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + -moz-border-radius-topright: 8px; + -moz-border-radius-topleft: 8px; + /* webkit specific markup */ + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + -webkit-border-top-right-radius: 8px; + -webkit-border-top-left-radius: 8px; + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #EAEAEA; + +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid #C0C0C0; + border-left: 1px solid #C0C0C0; + border-right: 1px solid #C0C0C0; + padding: 2px 5px; + background-color: #FCFCFC; + border-top-width: 0; + /* opera specific markup */ + border-bottom-left-radius: 8px; + border-bottom-right-radius: 8px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 8px; + -moz-border-radius-bottomright: 8px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + background-image: -moz-linear-gradient(center top, #FFFFFF 0%, #FFFFFF 60%, #F9F9F9 95%, #F2F2F2); + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 8px; + -webkit-border-bottom-right-radius: 8px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + background-image: -webkit-gradient(linear,center top,center bottom,from(#FFFFFF), color-stop(0.6,#FFFFFF), color-stop(0.60,#FFFFFF), color-stop(0.95,#F9F9F9), to(#F2F2F2)); +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: #602020; + white-space: nowrap; +} +.paramname em { + font-style: normal; +} + +.params, .retval, .exception, .tparams { + border-spacing: 6px 2px; +} + +.params .paramname, .retval .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir { + font-family: "courier new",courier,monospace; + vertical-align: top; +} + + + + +/* @end */ + +/* @group Directory (tree) */ + +/* for the tree view */ + +.ftvtree { + font-family: sans-serif; + margin: 0px; +} + +/* these are for tree view when used as main index */ + +.directory { + font-size: 9pt; + font-weight: bold; + margin: 5px; +} + +.directory h3 { + margin: 0px; + margin-top: 1em; + font-size: 11pt; +} + +/* +The following two styles can be used to replace the root node title +with an image of your choice. Simply uncomment the next two styles, +specify the name of your image and be sure to set 'height' to the +proper pixel height of your image. +*/ + +/* +.directory h3.swap { + height: 61px; + background-repeat: no-repeat; + background-image: url("yourimage.gif"); +} +.directory h3.swap span { + display: none; +} +*/ + +.directory > h3 { + margin-top: 0; +} + +.directory p { + margin: 0px; + white-space: nowrap; +} + +.directory div { + display: none; + margin: 0px; +} + +.directory img { + vertical-align: -30%; +} + +/* these are for tree view when not used as main index */ + +.directory-alt { + font-size: 100%; + font-weight: bold; +} + +.directory-alt h3 { + margin: 0px; + margin-top: 1em; + font-size: 11pt; +} + +.directory-alt > h3 { + margin-top: 0; +} + +.directory-alt p { + margin: 0px; + white-space: nowrap; +} + +.directory-alt div { + display: none; + margin: 0px; +} + +.directory-alt img { + vertical-align: -30%; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; +} + +address { + font-style: normal; + color: #464646; +} + +table.doxtable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.doxtable td, table.doxtable th { + border: 1px solid #4A4A4A; + padding: 3px 7px 2px; +} + +table.doxtable th { + background-color: #5B5B5B; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +table.fieldtable { + width: 100%; + margin-bottom: 10px; + border: 1px solid #C0C0C0; + border-spacing: 0px; + -moz-border-radius: 4px; + -webkit-border-radius: 4px; + border-radius: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid #C0C0C0; + border-bottom: 1px solid #C0C0C0; + vertical-align: top; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid #C0C0C0; + width: 100%; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #EAEAEA; + font-size: 90%; + color: #3D3D3D; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + -moz-border-radius-topleft: 4px; + -moz-border-radius-topright: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid #C0C0C0; +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: url('tab_b.png'); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul +{ + font-size: 11px; + background-image:url('tab_b.png'); + background-repeat:repeat-x; + height:30px; + line-height:30px; + color:#ABABAB; + border:solid 1px #D3D3D3; + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:url('bc_s.png'); + background-repeat:no-repeat; + background-position:right; + color:#595959; +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; +} + +.navpath li.navelem a:hover +{ + color:#929292; +} + +.navpath li.footer +{ + list-style-type:none; + float:right; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color:#595959; + font-size: 8pt; +} + + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +div.ingroups +{ + margin-left: 5px; + font-size: 8pt; + padding-left: 5px; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +div.header +{ + background-image:url('nav_h.png'); + background-repeat:repeat-x; + background-color: #FAFAFA; + margin: 0px; + border-bottom: 1px solid #D5D5D5; +} + +div.headertitle +{ + padding: 5px 5px 5px 7px; +} + +dl +{ + padding: 0 0 0 10px; +} + +/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug */ +dl.section +{ + border-left:4px solid; + padding: 0 0 0 6px; +} + +dl.note +{ + border-color: #D0C000; +} + +dl.warning, dl.attention +{ + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant +{ + border-color: #00D000; +} + +dl.deprecated +{ + border-color: #505050; +} + +dl.todo +{ + border-color: #00C0E0; +} + +dl.test +{ + border-color: #3030E0; +} + +dl.bug +{ + border-color: #C08050; +} + +dl.section dd { + margin-bottom: 6px; +} + + +#projectlogo +{ + text-align: center; + vertical-align: bottom; + border-collapse: separate; +} + +#projectlogo img +{ + border: 0px none; +} + +#projectname +{ + font: 300% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 2px 0px; +} + +#projectbrief +{ + font: 120% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#projectnumber +{ + font: 100% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#titlearea +{ + padding: 0px; + margin: 0px; + width: 100%; + border-bottom: 1px solid #848484; +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +div.zoom +{ + border: 1px solid #AFAFAF; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:#545454; + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; +} + +dl.citelist dd { + margin:2px 0; + padding:5px 0; +} + +div.toc { + padding: 14px 25px; + background-color: #F7F7F7; + border: 1px solid #E3E3E3; + border-radius: 7px 7px 7px 7px; + float: right; + height: auto; + margin: 0 20px 10px 10px; + width: 200px; +} + +div.toc li { + background: url("bdwn.png") no-repeat scroll 0 5px transparent; + font: 10px/1.2 Verdana,DejaVu Sans,Geneva,sans-serif; + margin-top: 5px; + padding-left: 10px; + padding-top: 2px; +} + +div.toc h3 { + font: bold 12px/1.2 Arial,FreeSans,sans-serif; + color: #747474; + border-bottom: 0 none; + margin: 0; +} + +div.toc ul { + list-style: none outside none; + border: medium none; + padding: 0px; +} + +div.toc li.level1 { + margin-left: 0px; +} + +div.toc li.level2 { + margin-left: 15px; +} + +div.toc li.level3 { + margin-left: 30px; +} + +div.toc li.level4 { + margin-left: 45px; +} + + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } + pre.fragment + { + overflow: visible; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-oggopus.xml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-oggopus.xml new file mode 100644 index 000000000..128816e93 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-oggopus.xml @@ -0,0 +1,1873 @@ + + + + + + + + + + + + +]> + + + + + +Ogg Encapsulation for the Opus Audio Codec + +Mozilla Corporation +
+ +650 Castro Street +Mountain View +CA +94041 +USA + ++1 650 903-0800 +tterribe@xiph.org +
+
+ + +Voicetronix +
+ +246 Pulteney Street, Level 1 +Adelaide +SA +5000 +Australia + ++61 8 8232 9112 +ron@debian.org +
+
+ + +Mozilla Corporation +
+ +163 West Hastings Street +Vancouver +BC +V6B 1H5 +Canada + ++1 778 785 1540 +giles@xiph.org +
+
+ + +RAI +codec + + + +This document defines the Ogg encapsulation for the Opus interactive speech and + audio codec. +This allows data encoded in the Opus format to be stored in an Ogg logical + bitstream. + + +
+ + +
+ +The IETF Opus codec is a low-latency audio codec optimized for both voice and + general-purpose audio. +See for technical details. +This document defines the encapsulation of Opus in a continuous, logical Ogg + bitstream . +Ogg encapsulation provides Opus with a long-term storage format supporting + all of the essential features, including metadata, fast and accurate seeking, + corruption detection, recapture after errors, low overhead, and the ability to + multiplex Opus with other codecs (including video) with minimal buffering. +It also provides a live streamable format, capable of delivery over a reliable + stream-oriented transport, without requiring all the data, or even the total + length of the data, up-front, in a form that is identical to the on-disk + storage format. + + +Ogg bitstreams are made up of a series of 'pages', each of which contains data + from one or more 'packets'. +Pages are the fundamental unit of multiplexing in an Ogg stream. +Each page is associated with a particular logical stream and contains a capture + pattern and checksum, flags to mark the beginning and end of the logical + stream, and a 'granule position' that represents an absolute position in the + stream, to aid seeking. +A single page can contain up to 65,025 octets of packet data from up to 255 + different packets. +Packets can be split arbitrarily across pages, and continued from one page to + the next (allowing packets much larger than would fit on a single page). +Each page contains 'lacing values' that indicate how the data is partitioned + into packets, allowing a demultiplexer (demuxer) to recover the packet + boundaries without examining the encoded data. +A packet is said to 'complete' on a page when the page contains the final + lacing value corresponding to that packet. + + +This encapsulation defines the contents of the packet data, including + the necessary headers, the organization of those packets into a logical + stream, and the interpretation of the codec-specific granule position field. +It does not attempt to describe or specify the existing Ogg container format. +Readers unfamiliar with the basic concepts mentioned above are encouraged to + review the details in . + + +
+ +
+ +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", + "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in . + + +
+ +
+ +An Ogg Opus stream is organized as follows (see + for an example). + + +
+ +
+ + +There are two mandatory header packets. +The first packet in the logical Ogg bitstream MUST contain the identification + (ID) header, which uniquely identifies a stream as Opus audio. +The format of this header is defined in . +It is placed alone (without any other packet data) on the first page of + the logical Ogg bitstream, and completes on that page. +This page has its 'beginning of stream' flag set. + + +The second packet in the logical Ogg bitstream MUST contain the comment header, + which contains user-supplied metadata. +The format of this header is defined in . +It MAY span multiple pages, beginning on the second page of the logical + stream. +However many pages it spans, the comment header packet MUST finish the page on + which it completes. + + +All subsequent pages are audio data pages, and the Ogg packets they contain are + audio data packets. +Each audio data packet contains one Opus packet for each of N different + streams, where N is typically one for mono or stereo, but MAY be greater than + one for multichannel audio. +The value N is specified in the ID header (see + ), and is fixed over the entire length of the + logical Ogg bitstream. + + +The first (N - 1) Opus packets, if any, are packed one after another + into the Ogg packet, using the self-delimiting framing from Appendix B of + . +The remaining Opus packet is packed at the end of the Ogg packet using the + regular, undelimited framing from Section 3 of . +All of the Opus packets in a single Ogg packet MUST be constrained to have the + same duration. +An implementation of this specification SHOULD treat any Opus packet whose + duration is different from that of the first Opus packet in an Ogg packet as + if it were a malformed Opus packet with an invalid Table Of Contents (TOC) + sequence. + + +The TOC sequence at the beginning of each Opus packet indicates the coding + mode, audio bandwidth, channel count, duration (frame size), and number of + frames per packet, as described in Section 3.1 + of . +The coding mode is one of SILK, Hybrid, or Constrained Energy Lapped Transform + (CELT). +The combination of coding mode, audio bandwidth, and frame size is referred to + as the configuration of an Opus packet. + + +Packets are placed into Ogg pages in order until the end of stream. +Audio data packets might span page boundaries. +The first audio data page could have the 'continued packet' flag set + (indicating the first audio data packet is continued from a previous page) if, + for example, it was a live stream joined mid-broadcast, with the headers + pasted on the front. +If a page has the 'continued packet' flag set and one of the following + conditions is also true: + +the previous page with packet data does not end in a continued packet (does + not end with a lacing value of 255) OR +the page sequence numbers are not consecutive, + + then a demuxer MUST NOT attempt to decode the data for the first packet on the + page unless the demuxer has some special knowledge that would allow it to + interpret this data despite the missing pieces. +An implementation MUST treat a zero-octet audio data packet as if it were a + malformed Opus packet as described in + Section 3.4 of . + + +A logical stream ends with a page with the 'end of stream' flag set, but + implementations need to be prepared to deal with truncated streams that do not + have a page marked 'end of stream'. +There is no reason for the final packet on the last page to be a continued + packet, i.e., for the final lacing value to be 255. +However, demuxers might encounter such streams, possibly as the result of a + transfer that did not complete or of corruption. +If a packet continues onto a subsequent page (i.e., when the page ends with a + lacing value of 255) and one of the following conditions is also true: + +the next page with packet data does not have the 'continued packet' flag + set OR +there is no next page with packet data OR +the page sequence numbers are not consecutive, + + then a demuxer MUST NOT attempt to decode the data from that packet unless the + demuxer has some special knowledge that would allow it to interpret this data + despite the missing pieces. +There MUST NOT be any more pages in an Opus logical bitstream after a page + marked 'end of stream'. + +
+ +
+ +The granule position MUST be zero for the ID header page and the + page where the comment header completes. +That is, the first page in the logical stream, and the last header + page before the first audio data page both have a granule position of zero. + + +The granule position of an audio data page encodes the total number of PCM + samples in the stream up to and including the last fully-decodable sample from + the last packet completed on that page. +The granule position of the first audio data page will usually be larger than + zero, as described in . + + + +A page that is entirely spanned by a single packet (that completes on a + subsequent page) has no granule position, and the granule position field is + set to the special value '-1' in two's complement. + + + +The granule position of an audio data page is in units of PCM audio samples at + a fixed rate of 48 kHz (per channel; a stereo stream's granule position + does not increment at twice the speed of a mono stream). +It is possible to run an Opus decoder at other sampling rates, + but all Opus packets encode samples at a sampling rate that evenly divides + 48 kHz. +Therefore, the value in the granule position field always counts samples + assuming a 48 kHz decoding rate, and the rest of this specification makes + the same assumption. + + + +The duration of an Opus packet as defined in can be + any multiple of 2.5 ms, up to a maximum of 120 ms. +This duration is encoded in the TOC sequence at the beginning of each packet. +The number of samples returned by a decoder corresponds to this duration + exactly, even for the first few packets. +For example, a 20 ms packet fed to a decoder running at 48 kHz will + always return 960 samples. +A demuxer can parse the TOC sequence at the beginning of each Ogg packet to + work backwards or forwards from a packet with a known granule position (i.e., + the last packet completed on some page) in order to assign granule positions + to every packet, or even every individual sample. +The one exception is the last page in the stream, as described below. + + + +All other pages with completed packets after the first MUST have a granule + position equal to the number of samples contained in packets that complete on + that page plus the granule position of the most recent page with completed + packets. +This guarantees that a demuxer can assign individual packets the same granule + position when working forwards as when working backwards. +For this to work, there cannot be any gaps. + + +
+ +In order to support capturing a real-time stream that has lost or not + transmitted packets, a multiplexer (muxer) SHOULD emit packets that explicitly + request the use of Packet Loss Concealment (PLC) in place of the missing + packets. +Implementations that fail to do so still MUST NOT increment the granule + position for a page by anything other than the number of samples contained in + packets that actually complete on that page. + + +Only gaps that are a multiple of 2.5 ms are repairable, as these are the + only durations that can be created by packet loss or discontinuous + transmission. +Muxers need not handle other gap sizes. +Creating the necessary packets involves synthesizing a TOC byte (defined in +Section 3.1 of )—and whatever + additional internal framing is needed—to indicate the packet duration + for each stream. +The actual length of each missing Opus frame inside the packet is zero bytes, + as defined in Section 3.2.1 of . + + + +Zero-byte frames MAY be packed into packets using any of codes 0, 1, + 2, or 3. +When successive frames have the same configuration, the higher code packings + reduce overhead. +Likewise, if the TOC configuration matches, the muxer MAY further combine the + empty frames with previous or subsequent non-zero-length frames (using + code 2 or VBR code 3). + + + + does not impose any requirements on the PLC, but this + section outlines choices that are expected to have a positive influence on + most PLC implementations, including the reference implementation. +Synthesized TOC sequences SHOULD maintain the same mode, audio bandwidth, + channel count, and frame size as the previous packet (if any). +This is the simplest and usually the most well-tested case for the PLC to + handle and it covers all losses that do not include a configuration switch, + as defined in Section 4.5 of . + + + +When a previous packet is available, keeping the audio bandwidth and channel + count the same allows the PLC to provide maximum continuity in the concealment + data it generates. +However, if the size of the gap is not a multiple of the most recent frame + size, then the frame size will have to change for at least some frames. +Such changes SHOULD be delayed as long as possible to simplify + things for PLC implementations. + + + +As an example, a 95 ms gap could be encoded as nineteen 5 ms frames + in two bytes with a single CBR code 3 packet. +If the previous frame size was 20 ms, using four 20 ms frames + followed by three 5 ms frames requires 4 bytes (plus an extra byte + of Ogg lacing overhead), but allows the PLC to use its well-tested steady + state behavior for as long as possible. +The total bitrate of the latter approach, including Ogg overhead, is about + 0.4 kbps, so the impact on file size is minimal. + + + +Changing modes is discouraged, since this causes some decoder implementations + to reset their PLC state. +However, SILK and Hybrid mode frames cannot fill gaps that are not a multiple + of 10 ms. +If switching to CELT mode is needed to match the gap size, a muxer SHOULD do + so at the end of the gap to allow the PLC to function for as long as possible. + + + +In the example above, if the previous frame was a 20 ms SILK mode frame, + the better solution is to synthesize a packet describing four 20 ms SILK + frames, followed by a packet with a single 10 ms SILK + frame, and finally a packet with a 5 ms CELT frame, to fill the 95 ms + gap. +This also requires four bytes to describe the synthesized packet data (two + bytes for a CBR code 3 and one byte each for two code 0 packets) but three + bytes of Ogg lacing overhead are needed to mark the packet boundaries. +At 0.6 kbps, this is still a minimal bitrate impact over a naive, low quality + solution. + + + +Since medium-band audio is an option only in the SILK mode, wideband frames + SHOULD be generated if switching from that configuration to CELT mode, to + ensure that any PLC implementation which does try to migrate state between + the modes will be able to preserve all of the available audio bandwidth. + + +
+ +
+ +There is some amount of latency introduced during the decoding process, to + allow for overlap in the CELT mode, stereo mixing in the SILK mode, and + resampling. +The encoder might have introduced additional latency through its own resampling + and analysis (though the exact amount is not specified). +Therefore, the first few samples produced by the decoder do not correspond to + real input audio, but are instead composed of padding inserted by the encoder + to compensate for this latency. +These samples need to be stored and decoded, as Opus is an asymptotically + convergent predictive codec, meaning the decoded contents of each frame depend + on the recent history of decoder inputs. +However, a player will want to skip these samples after decoding them. + + + +A 'pre-skip' field in the ID header (see ) signals + the number of samples that SHOULD be skipped (decoded but discarded) at the + beginning of the stream, though some specific applications might have a reason + for looking at that data. +This amount need not be a multiple of 2.5 ms, MAY be smaller than a single + packet, or MAY span the contents of several packets. +These samples are not valid audio. + + + +For example, if the first Opus frame uses the CELT mode, it will always + produce 120 samples of windowed overlap-add data. +However, the overlap data is initially all zeros (since there is no prior + frame), meaning this cannot, in general, accurately represent the original + audio. +The SILK mode requires additional delay to account for its analysis and + resampling latency. +The encoder delays the original audio to avoid this problem. + + + +The pre-skip field MAY also be used to perform sample-accurate cropping of + already encoded streams. +In this case, a value of at least 3840 samples (80 ms) provides + sufficient history to the decoder that it will have converged + before the stream's output begins. + + +
+ +
+ +The PCM sample position is determined from the granule position using the + formula + +
+ +
+ + +For example, if the granule position of the first audio data page is 59,971, + and the pre-skip is 11,971, then the PCM sample position of the last decoded + sample from that page is 48,000. + + +This can be converted into a playback time using the formula + +
+ +
+ + +The initial PCM sample position before any samples are played is normally '0'. +In this case, the PCM sample position of the first audio sample to be played + starts at '1', because it marks the time on the clock + after that sample has been played, and a stream + that is exactly one second long has a final PCM sample position of '48000', + as in the example here. + + + +Vorbis streams use a granule position smaller than the number of audio samples + contained in the first audio data page to indicate that some of those samples + are trimmed from the output (see ). +However, to do so, Vorbis requires that the first audio data page contains + exactly two packets, in order to allow the decoder to perform PCM position + adjustments before needing to return any PCM data. +Opus uses the pre-skip mechanism for this purpose instead, since the encoder + might introduce more than a single packet's worth of latency, and since very + large packets in streams with a very large number of channels might not fit + on a single page. + +
+ +
+ +The page with the 'end of stream' flag set MAY have a granule position that + indicates the page contains less audio data than would normally be returned by + decoding up through the final packet. +This is used to end the stream somewhere other than an even frame boundary. +The granule position of the most recent audio data page with completed packets + is used to make this determination, or '0' is used if there were no previous + audio data pages with a completed packet. +The difference between these granule positions indicates how many samples to + keep after decoding the packets that completed on the final page. +The remaining samples are discarded. +The number of discarded samples SHOULD be no larger than the number decoded + from the last packet. + +
+ +
+ +The granule position of the first audio data page with a completed packet MAY + be larger than the number of samples contained in packets that complete on + that page, however it MUST NOT be smaller, unless that page has the 'end of + stream' flag set. +Allowing a granule position larger than the number of samples allows the + beginning of a stream to be cropped or a live stream to be joined without + rewriting the granule position of all the remaining pages. +This means that the PCM sample position just before the first sample to be + played MAY be larger than '0'. +Synchronization when multiplexing with other logical streams still uses the PCM + sample position relative to '0' to compute sample times. +This does not affect the behavior of pre-skip: exactly 'pre-skip' samples + SHOULD be skipped from the beginning of the decoded output, even if the + initial PCM sample position is greater than zero. + + + +On the other hand, a granule position that is smaller than the number of + decoded samples prevents a demuxer from working backwards to assign each + packet or each individual sample a valid granule position, since granule + positions are non-negative. +An implementation MUST treat any stream as invalid if the granule position + is smaller than the number of samples contained in packets that complete on + the first audio data page with a completed packet, unless that page has the + 'end of stream' flag set. +It MAY defer this action until it decodes the last packet completed on that + page. + + + +If that page has the 'end of stream' flag set, a demuxer MUST treat any stream + as invalid if its granule position is smaller than the 'pre-skip' amount. +This would indicate that there are more samples to be skipped from the initial + decoded output than exist in the stream. +If the granule position is smaller than the number of decoded samples produced + by the packets that complete on that page, then a demuxer MUST use an initial + granule position of '0', and can work forwards from '0' to timestamp + individual packets. +If the granule position is larger than the number of decoded samples available, + then the demuxer MUST still work backwards as described above, even if the + 'end of stream' flag is set, to determine the initial granule position, and + thus the initial PCM sample position. +Both of these will be greater than '0' in this case. + +
+ +
+ +Seeking in Ogg files is best performed using a bisection search for a page + whose granule position corresponds to a PCM position at or before the seek + target. +With appropriately weighted bisection, accurate seeking can be performed in + just one or two bisections on average, even in multi-gigabyte files. +See for an example of general implementation guidance. + + + +When seeking within an Ogg Opus stream, an implementation SHOULD start decoding + (and discarding the output) at least 3840 samples (80 ms) prior to + the seek target in order to ensure that the output audio is correct by the + time it reaches the seek target. +This 'pre-roll' is separate from, and unrelated to, the 'pre-skip' used at the + beginning of the stream. +If the point 80 ms prior to the seek target comes before the initial PCM + sample position, an implementation SHOULD start decoding from the beginning of + the stream, applying pre-skip as normal, regardless of whether the pre-skip is + larger or smaller than 80 ms, and then continue to discard samples + to reach the seek target (if any). + +
+ +
+ +
+ +An Ogg Opus logical stream contains exactly two mandatory header packets: + an identification header and a comment header. + + +
+ +
+ +
+ + +The fields in the identification (ID) header have the following meaning: + +Magic Signature: + +This is an 8-octet (64-bit) field that allows codec identification and is + human-readable. +It contains, in order, the magic numbers: + +0x4F 'O' +0x70 'p' +0x75 'u' +0x73 's' +0x48 'H' +0x65 'e' +0x61 'a' +0x64 'd' + +Starting with "Op" helps distinguish it from audio data packets, as this is an + invalid TOC sequence. + + +Version (8 bits, unsigned): + +The version number MUST always be '1' for this version of the encapsulation + specification. +Implementations SHOULD treat streams where the upper four bits of the version + number match that of a recognized specification as backwards-compatible with + that specification. +That is, the version number can be split into "major" and "minor" version + sub-fields, with changes to the "minor" sub-field (in the lower four bits) + signaling compatible changes. +For example, an implementation of this specification SHOULD accept any stream + with a version number of '15' or less, and SHOULD assume any stream with a + version number '16' or greater is incompatible. +The initial version '1' was chosen to keep implementations from relying on this + octet as a null terminator for the "OpusHead" string. + + +Output Channel Count 'C' (8 bits, unsigned): + +This is the number of output channels. +This might be different than the number of encoded channels, which can change + on a packet-by-packet basis. +This value MUST NOT be zero. +The maximum allowable value depends on the channel mapping family, and might be + as large as 255. +See for details. + + +Pre-skip (16 bits, unsigned, little + endian): + +This is the number of samples (at 48 kHz) to discard from the decoder + output when starting playback, and also the number to subtract from a page's + granule position to calculate its PCM sample position. +When cropping the beginning of existing Ogg Opus streams, a pre-skip of at + least 3,840 samples (80 ms) is RECOMMENDED to ensure complete + convergence in the decoder. + + +Input Sample Rate (32 bits, unsigned, little + endian): + +This is the sample rate of the original input (before encoding), in Hz. +This field is not the sample rate to use for + playback of the encoded data. + +Opus can switch between internal audio bandwidths of 4, 6, 8, 12, and + 20 kHz. +Each packet in the stream can have a different audio bandwidth. +Regardless of the audio bandwidth, the reference decoder supports decoding any + stream at a sample rate of 8, 12, 16, 24, or 48 kHz. +The original sample rate of the audio passed to the encoder is not preserved + by the lossy compression. + +An Ogg Opus player SHOULD select the playback sample rate according to the + following procedure: + +If the hardware supports 48 kHz playback, decode at 48 kHz. +Otherwise, if the hardware's highest available sample rate is a supported + rate, decode at this sample rate. +Otherwise, if the hardware's highest available sample rate is less than + 48 kHz, decode at the next higher Opus supported rate above the highest + available hardware rate and resample. +Otherwise, decode at 48 kHz and resample. + +However, the 'Input Sample Rate' field allows the muxer to pass the sample + rate of the original input stream as metadata. +This is useful when the user requires the output sample rate to match the + input sample rate. +For example, when not playing the output, an implementation writing PCM format + samples to disk might choose to resample the audio back to the original input + sample rate to reduce surprise to the user, who might reasonably expect to get + back a file with the same sample rate. + +A value of zero indicates 'unspecified'. +Muxers SHOULD write the actual input sample rate or zero, but implementations + which do something with this field SHOULD take care to behave sanely if given + crazy values (e.g., do not actually upsample the output to 10 MHz if + requested). +Implementations SHOULD support input sample rates between 8 kHz and + 192 kHz (inclusive). +Rates outside this range MAY be ignored by falling back to the default rate of + 48 kHz instead. + + +Output Gain (16 bits, signed, little endian): + +This is a gain to be applied when decoding. +It is 20*log10 of the factor by which to scale the decoder output to achieve + the desired playback volume, stored in a 16-bit, signed, two's complement + fixed-point value with 8 fractional bits (i.e., + Q7.8 ). + +To apply the gain, an implementation could use +
+ +
+ where output_gain is the raw 16-bit value from the header. + +Players and media frameworks SHOULD apply it by default. +If a player chooses to apply any volume adjustment or gain modification, such + as the R128_TRACK_GAIN (see ), the adjustment + MUST be applied in addition to this output gain in order to achieve playback + at the normalized volume. + +A muxer SHOULD set this field to zero, and instead apply any gain prior to + encoding, when this is possible and does not conflict with the user's wishes. +A nonzero output gain indicates the gain was adjusted after encoding, or that + a user wished to adjust the gain for playback while preserving the ability + to recover the original signal amplitude. + +Although the output gain has enormous range (+/- 128 dB, enough to amplify + inaudible sounds to the threshold of physical pain), most applications can + only reasonably use a small portion of this range around zero. +The large range serves in part to ensure that gain can always be losslessly + transferred between OpusHead and R128 gain tags (see below) without + saturating. + +
+Channel Mapping Family (8 bits, unsigned): + +This octet indicates the order and semantic meaning of the output channels. + +Each currently specified value of this octet indicates a mapping family, which + defines a set of allowed channel counts, and the ordered set of channel names + for each allowed channel count. +The details are described in . + +Channel Mapping Table: +This table defines the mapping from encoded streams to output channels. +Its contents are specified in . + +
+
+ + +All fields in the ID headers are REQUIRED, except for the channel mapping + table, which MUST be omitted when the channel mapping family is 0, but + is REQUIRED otherwise. +Implementations SHOULD treat a stream as invalid if it contains an ID header + that does not have enough data for these fields, even if it contain a valid + Magic Signature. +Future versions of this specification, even backwards-compatible versions, + might include additional fields in the ID header. +If an ID header has a compatible major version, but a larger minor version, + an implementation MUST NOT treat it as invalid for containing additional data + not specified here, provided it still completes on the first page. + + +
+ +An Ogg Opus stream allows mapping one number of Opus streams (N) to a possibly + larger number of decoded channels (M + N) to yet another number of + output channels (C), which might be larger or smaller than the number of + decoded channels. +The order and meaning of these channels are defined by a channel mapping, + which consists of the 'channel mapping family' octet and, for channel mapping + families other than family 0, a channel mapping table, as illustrated in + . + + +
+ +
+ + +The fields in the channel mapping table have the following meaning: + +Stream Count 'N' (8 bits, unsigned): + +This is the total number of streams encoded in each Ogg packet. +This value is necessary to correctly parse the packed Opus packets inside an + Ogg packet, as described in . +This value MUST NOT be zero, as without at least one Opus packet with a valid + TOC sequence, a demuxer cannot recover the duration of an Ogg packet. + +For channel mapping family 0, this value defaults to 1, and is not coded. + + +Coupled Stream Count 'M' (8 bits, unsigned): +This is the number of streams whose decoders are to be configured to produce + two channels (stereo). +This MUST be no larger than the total number of streams, N. + +Each packet in an Opus stream has an internal channel count of 1 or 2, which + can change from packet to packet. +This is selected by the encoder depending on the bitrate and the audio being + encoded. +The original channel count of the audio passed to the encoder is not + necessarily preserved by the lossy compression. + +Regardless of the internal channel count, any Opus stream can be decoded as + mono (a single channel) or stereo (two channels) by appropriate initialization + of the decoder. +The 'coupled stream count' field indicates that the decoders for the first M + Opus streams are to be initialized for stereo (two-channel) output, and the + remaining (N - M) decoders are to be initialized for mono (a single + channel) only. +The total number of decoded channels, (M + N), MUST be no larger than + 255, as there is no way to index more channels than that in the channel + mapping. + +For channel mapping family 0, this value defaults to (C - 1) + (i.e., 0 for mono and 1 for stereo), and is not coded. + + +Channel Mapping (8*C bits): +This contains one octet per output channel, indicating which decoded channel + is to be used for each one. +Let 'index' be the value of this octet for a particular output channel. +This value MUST either be smaller than (M + N), or be the special + value 255. +If 'index' is less than 2*M, the output MUST be taken from decoding stream + ('index'/2) as stereo and selecting the left channel if 'index' is even, and + the right channel if 'index' is odd. +If 'index' is 2*M or larger, but less than 255, the output MUST be taken from + decoding stream ('index' - M) as mono. +If 'index' is 255, the corresponding output channel MUST contain pure silence. + +The number of output channels, C, is not constrained to match the number of + decoded channels (M + N). +A single index value MAY appear multiple times, i.e., the same decoded channel + might be mapped to multiple output channels. +Some decoded channels might not be assigned to any output channel, as well. + +For channel mapping family 0, the first index defaults to 0, and if + C == 2, the second index defaults to 1. +Neither index is coded. + + + + + +After producing the output channels, the channel mapping family determines the + semantic meaning of each one. +There are three defined mapping families in this specification. + + +
+ +Allowed numbers of channels: 1 or 2. +RTP mapping. +This is the same channel interpretation as . + + + +1 channel: monophonic (mono). +2 channels: stereo (left, right). + +Special mapping: This channel mapping value also + indicates that the contents consists of a single Opus stream that is stereo if + and only if C == 2, with stream index 0 mapped to output + channel 0 (mono, or left channel) and stream index 1 mapped to + output channel 1 (right channel) if stereo. +When the 'channel mapping family' octet has this value, the channel mapping + table MUST be omitted from the ID header packet. + +
+ +
+ +Allowed numbers of channels: 1...8. +Vorbis channel order (see below). + + +Each channel is assigned to a speaker location in a conventional surround + arrangement. +Specific locations depend on the number of channels, and are given below + in order of the corresponding channel indices. + + 1 channel: monophonic (mono). + 2 channels: stereo (left, right). + 3 channels: linear surround (left, center, right) + 4 channels: quadraphonic (front left, front right, rear left, rear right). + 5 channels: 5.0 surround (front left, front center, front right, rear left, rear right). + 6 channels: 5.1 surround (front left, front center, front right, rear left, rear right, LFE). + 7 channels: 6.1 surround (front left, front center, front right, side left, side right, rear center, LFE). + 8 channels: 7.1 surround (front left, front center, front right, side left, side right, rear left, rear right, LFE) + + + +This set of surround options and speaker location orderings is the same + as those used by the Vorbis codec . +The ordering is different from the one used by the + WAVE and + Free Lossless Audio Codec (FLAC) formats, + so correct ordering requires permutation of the output channels when decoding + to or encoding from those formats. +'LFE' here refers to a Low Frequency Effects channel, often mapped to a + subwoofer with no particular spatial position. +Implementations SHOULD identify 'side' or 'rear' speaker locations with + 'surround' and 'back' as appropriate when interfacing with audio formats + or systems which prefer that terminology. + +
+ +
+ +Allowed numbers of channels: 1...255. +No defined channel meaning. + + +Channels are unidentified. +General-purpose players SHOULD NOT attempt to play these streams. +Offline implementations MAY deinterleave the output into separate PCM files, + one per channel. +Implementations SHOULD NOT produce output for channels mapped to stream index + 255 (pure silence) unless they have no other way to indicate the index of + non-silent channels. + +
+ +
+ +The remaining channel mapping families (2...254) are reserved. +A demuxer implementation encountering a reserved channel mapping family value + SHOULD act as though the value is 255. + +
+ +
+ +An Ogg Opus player MUST support any valid channel mapping with a channel + mapping family of 0 or 1, even if the number of channels does not match the + physically connected audio hardware. +Players SHOULD perform channel mixing to increase or reduce the number of + channels as needed. + + + +Implementations MAY use the matrices in + Figures  + through  to implement + downmixing from multichannel files using + Channel Mapping Family 1, which are + known to give acceptable results for stereo. +Matrices for 3 and 4 channels are normalized so each coefficient row sums + to 1 to avoid clipping. +For 5 or more channels they are normalized to 2 as a compromise between + clipping and dynamic range reduction. + + +In these matrices the front left and front right channels are generally +passed through directly. +When a surround channel is split between both the left and right stereo + channels, coefficients are chosen so their squares sum to 1, which + helps preserve the perceived intensity. +Rear channels are mixed more diffusely or attenuated to maintain focus + on the front channels. + + +
+ + +Exact coefficient values are 1 and 1/sqrt(2), multiplied by + 1/(1 + 1/sqrt(2)) for normalization. + +
+ +
+ + +Exact coefficient values are 1, sqrt(3)/2 and 1/2, multiplied by + 1/(1 + sqrt(3)/2 + 1/2) for normalization. + +
+ +
+ + +Exact coefficient values are 1, 1/sqrt(2), sqrt(3)/2 and 1/2, multiplied by + 2/(1 + 1/sqrt(2) + sqrt(3)/2 + 1/2) + for normalization. + +
+ +
+ + +Exact coefficient values are 1, 1/sqrt(2), sqrt(3)/2 and 1/2, multiplied by +2/(1 + 1/sqrt(2) + sqrt(3)/2 + 1/2 + 1/sqrt(2)) + for normalization. + +
+ +
+ + +Exact coefficient values are 1, 1/sqrt(2), sqrt(3)/2, 1/2 and + sqrt(3)/2/sqrt(2), multiplied by + 2/(1 + 1/sqrt(2) + sqrt(3)/2 + 1/2 + + sqrt(3)/2/sqrt(2) + 1/sqrt(2)) for normalization. +The coefficients are in the same order as in , + and the matrices above. + +
+ +
+ + +Exact coefficient values are 1, 1/sqrt(2), sqrt(3)/2 and 1/2, multiplied by + 2/(2 + 2/sqrt(2) + sqrt(3)) for normalization. +The coefficients are in the same order as in , + and the matrices above. + +
+ +
+ +
+ +
+ +
+ +
+ +
+ + +The comment header consists of a 64-bit magic signature, followed by data in + the same format as the header used in Ogg + Vorbis, except (like Ogg Theora and Speex) the final "framing bit" specified + in the Vorbis spec is not present. + +Magic Signature: + +This is an 8-octet (64-bit) field that allows codec identification and is + human-readable. +It contains, in order, the magic numbers: + +0x4F 'O' +0x70 'p' +0x75 'u' +0x73 's' +0x54 'T' +0x61 'a' +0x67 'g' +0x73 's' + +Starting with "Op" helps distinguish it from audio data packets, as this is an + invalid TOC sequence. + + +Vendor String Length (32 bits, unsigned, little endian): + +This field gives the length of the following vendor string, in octets. +It MUST NOT indicate that the vendor string is longer than the rest of the + packet. + + +Vendor String (variable length, UTF-8 vector): + +This is a simple human-readable tag for vendor information, encoded as a UTF-8 + string . +No terminating null octet is necessary. + +This tag is intended to identify the codec encoder and encapsulation + implementations, for tracing differences in technical behavior. +User-facing applications can use the 'ENCODER' user comment tag to identify + themselves. + + +User Comment List Length (32 bits, unsigned, little endian): + +This field indicates the number of user-supplied comments. +It MAY indicate there are zero user-supplied comments, in which case there are + no additional fields in the packet. +It MUST NOT indicate that there are so many comments that the comment string + lengths would require more data than is available in the rest of the packet. + + +User Comment #i String Length (32 bits, unsigned, little endian): + +This field gives the length of the following user comment string, in octets. +There is one for each user comment indicated by the 'user comment list length' + field. +It MUST NOT indicate that the string is longer than the rest of the packet. + + +User Comment #i String (variable length, UTF-8 vector): + +This field contains a single user comment encoded as a UTF-8 + string . +There is one for each user comment indicated by the 'user comment list length' + field. + + + + + +The vendor string length and user comment list length are REQUIRED, and + implementations SHOULD treat a stream as invalid if it contains a comment + header that does not have enough data for these fields, or that does not + contain enough data for the corresponding vendor string or user comments they + describe. +Making this check before allocating the associated memory to contain the data + helps prevent a possible Denial-of-Service (DoS) attack from small comment + headers that claim to contain strings longer than the entire packet or more + user comments than than could possibly fit in the packet. + + + +Immediately following the user comment list, the comment header MAY + contain zero-padding or other binary data which is not specified here. +If the least-significant bit of the first byte of this data is 1, then editors + SHOULD preserve the contents of this data when updating the tags, but if this + bit is 0, all such data MAY be treated as padding, and truncated or discarded + as desired. +This allows informal experimentation with the format of this binary data until + it can be specified later. + + + +The comment header can be arbitrarily large and might be spread over a large + number of Ogg pages. +Implementations MUST avoid attempting to allocate excessive amounts of memory + when presented with a very large comment header. +To accomplish this, implementations MAY treat a stream as invalid if it has a + comment header larger than 125,829,120 octets (120 MB), and MAY + ignore individual comments that are not fully contained within the first + 61,440 octets of the comment header. + + +
+ +The user comment strings follow the NAME=value format described by + with the same recommended tag names: + ARTIST, TITLE, DATE, ALBUM, and so on. + + +Two new comment tags are introduced here: + + +First, an optional gain for track normalization: +
+ +
+ + representing the volume shift needed to normalize the track's volume + during isolated playback, in random shuffle, and so on. +The gain is a Q7.8 fixed point number in dB, as in the ID header's 'output + gain' field. +This tag is similar to the REPLAYGAIN_TRACK_GAIN tag in + Vorbis , except that the normal volume + reference is the standard. + +Second, an optional gain for album normalization: +
+ +
+ + representing the volume shift needed to normalize the overall volume when + played as part of a particular collection of tracks. +The gain is also a Q7.8 fixed point number in dB, as in the ID header's + 'output gain' field. +The values '-573' and '111' given here are just examples. + + +An Ogg Opus stream MUST NOT have more than one of each of these tags, and if + present their values MUST be an integer from -32768 to 32767, inclusive, + represented in ASCII as a base 10 number with no whitespace. +A leading '+' or '-' character is valid. +Leading zeros are also permitted, but the value MUST be represented by + no more than 6 characters. +Other non-digit characters MUST NOT be present. + + +If present, R128_TRACK_GAIN and R128_ALBUM_GAIN MUST correctly represent + the R128 normalization gain relative to the 'output gain' field specified + in the ID header. +If a player chooses to make use of the R128_TRACK_GAIN tag or the + R128_ALBUM_GAIN tag, it MUST apply those gains + in addition to the 'output gain' value. +If a tool modifies the ID header's 'output gain' field, it MUST also update or + remove the R128_TRACK_GAIN and R128_ALBUM_GAIN comment tags if present. +A muxer SHOULD place the gain it wants other tools to use by default into the + 'output gain' field, and not the comment tag. + + +To avoid confusion with multiple normalization schemes, an Opus comment header + SHOULD NOT contain any of the REPLAYGAIN_TRACK_GAIN, REPLAYGAIN_TRACK_PEAK, + REPLAYGAIN_ALBUM_GAIN, or REPLAYGAIN_ALBUM_PEAK tags, unless they are only + to be used in some context where there is guaranteed to be no such confusion. + normalization is preferred to the earlier + REPLAYGAIN schemes because of its clear definition and adoption by industry. +Peak normalizations are difficult to calculate reliably for lossy codecs + because of variation in excursion heights due to decoder differences. +In the authors' investigations they were not applied consistently or broadly + enough to merit inclusion here. + +
+
+ +
+ +
+ +Technically, valid Opus packets can be arbitrarily large due to the padding + format, although the amount of non-padding data they can contain is bounded. +These packets might be spread over a similarly enormous number of Ogg pages. +When encoding, implementations SHOULD limit the use of padding in audio data + packets to no more than is necessary to make a variable bitrate (VBR) stream + constant bitrate (CBR), unless they have no reasonable way to determine what + is necessary. +Demuxers SHOULD treat audio data packets as invalid (treat them as if they were + malformed Opus packets with an invalid TOC sequence) if they are larger than + 61,440 octets per Opus stream, unless they have a specific reason for + allowing extra padding. +Such packets necessarily contain more padding than needed to make a stream CBR. +Demuxers MUST avoid attempting to allocate excessive amounts of memory when + presented with a very large packet. +Demuxers MAY treat audio data packets as invalid or partially process them if + they are larger than 61,440 octets in an Ogg Opus stream with channel + mapping families 0 or 1. +Demuxers MAY treat audio data packets as invalid or partially process them in + any Ogg Opus stream if the packet is larger than 61,440 octets and also + larger than 7,680 octets per Opus stream. +The presence of an extremely large packet in the stream could indicate a + memory exhaustion attack or stream corruption. + + +In an Ogg Opus stream, the largest possible valid packet that does not use + padding has a size of (61,298*N - 2) octets. +With 255 streams, this is 15,630,988 octets and can + span up to 61,298 Ogg pages, all but one of which will have a granule + position of -1. +This is of course a very extreme packet, consisting of 255 streams, each + containing 120 ms of audio encoded as 2.5 ms frames, each frame + using the maximum possible number of octets (1275) and stored in the least + efficient manner allowed (a VBR code 3 Opus packet). +Even in such a packet, most of the data will be zeros as 2.5 ms frames + cannot actually use all 1275 octets. + + +The largest packet consisting of entirely useful data is + (15,326*N - 2) octets. +This corresponds to 120 ms of audio encoded as 10 ms frames in either + SILK or Hybrid mode, but at a data rate of over 1 Mbps, which makes little + sense for the quality achieved. + + +A more reasonable limit is (7,664*N - 2) octets. +This corresponds to 120 ms of audio encoded as 20 ms stereo CELT mode + frames, with a total bitrate just under 511 kbps (not counting the Ogg + encapsulation overhead). +For channel mapping family 1, N=8 provides a reasonable upper bound, as it + allows for each of the 8 possible output channels to be decoded from a + separate stereo Opus stream. +This gives a size of 61,310 octets, which is rounded up to a multiple of + 1,024 octets to yield the audio data packet size of 61,440 octets + that any implementation is expected to be able to process successfully. + +
+ +
+ +When encoding Opus streams, Ogg muxers SHOULD take into account the + algorithmic delay of the Opus encoder. + + +In encoders derived from the reference + implementation , the number of samples can be + queried with: + +
+ +
+ +To achieve good quality in the very first samples of a stream, implementations + MAY use linear predictive coding (LPC) extrapolation to generate at least 120 + extra samples at the beginning to avoid the Opus encoder having to encode a + discontinuous signal. +For more information on linear prediction, see + . +For an input file containing 'length' samples, the implementation SHOULD set + the pre-skip header value to (delay_samples + extra_samples), encode + at least (length + delay_samples + extra_samples) + samples, and set the granule position of the last page to + (length + delay_samples + extra_samples). +This ensures that the encoded file has the same duration as the original, with + no time offset. The best way to pad the end of the stream is to also use LPC + extrapolation, but zero-padding is also acceptable. + + +
+ +The first step in LPC extrapolation is to compute linear prediction + coefficients. +When extending the end of the signal, order-N (typically with N ranging from 8 + to 40) LPC analysis is performed on a window near the end of the signal. +The last N samples are used as memory to an infinite impulse response (IIR) + filter. + + +The filter is then applied on a zero input to extrapolate the end of the signal. +Let a(k) be the kth LPC coefficient and x(n) be the nth sample of the signal, + each new sample past the end of the signal is computed as: + +
+ +
+ +The process is repeated independently for each channel. +It is possible to extend the beginning of the signal by applying the same + process backward in time. +When extending the beginning of the signal, it is best to apply a "fade in" to + the extrapolated signal, e.g. by multiplying it by a half-Hanning window + . + + +
+ +
+ +In some applications, such as Internet radio, it is desirable to cut a long + stream into smaller chains, e.g. so the comment header can be updated. +This can be done simply by separating the input streams into segments and + encoding each segment independently. +The drawback of this approach is that it creates a small discontinuity + at the boundary due to the lossy nature of Opus. +A muxer MAY avoid this discontinuity by using the following procedure: + +Encode the last frame of the first segment as an independent frame by + turning off all forms of inter-frame prediction. +De-emphasis is allowed. +Set the granule position of the last page to a point near the end of the + last frame. +Begin the second segment with a copy of the last frame of the first + segment. +Set the pre-skip value of the second stream in such a way as to properly + join the two streams. +Continue the encoding process normally from there, without any reset to + the encoder. + + + +In encoders derived from the reference implementation, inter-frame prediction + can be turned off by calling: + +
+ +
+ +For best results, this implementation requires that prediction be explicitly + enabled again before resuming normal encoding, even after a reset. + + +
+ +
+ +
+ +A brief summary of major implementations of this draft is available + at , + along with their status. + + +[Note to RFC Editor: please remove this entire section before + final publication per , along with + its references.] + +
+ +
+ +Implementations of the Opus codec need to take appropriate security + considerations into account, as outlined in . +This is just as much a problem for the container as it is for the codec itself. +Malicious payloads and/or input streams can be used to attack codec + implementations. +Implementations MUST NOT overrun their allocated memory nor consume excessive + resources when decoding payloads or processing input streams. +Although problems in encoding applications are typically rarer, this still + applies to a muxer, as vulnerabilities would allow an attacker to attack + transcoding gateways. + + + +Header parsing code contains the most likely area for potential overruns. +It is important for implementations to ensure their buffers contain enough + data for all of the required fields before attempting to read it (for example, + for all of the channel map data in the ID header). +Implementations would do well to validate the indices of the channel map, also, + to ensure they meet all of the restrictions outlined in + , in order to avoid attempting to read data + from channels that do not exist. + + + +To avoid excessive resource usage, we advise implementations to be especially + wary of streams that might cause them to process far more data than was + actually transmitted. +For example, a relatively small comment header may contain values for the + string lengths or user comment list length that imply that it is many + gigabytes in size. +Even computing the size of the required buffer could overflow a 32-bit integer, + and actually attempting to allocate such a buffer before verifying it would be + a reasonable size is a bad idea. +After reading the user comment list length, implementations might wish to + verify that the header contains at least the minimum amount of data for that + many comments (4 additional octets per comment, to indicate each has a + length of zero) before proceeding any further, again taking care to avoid + overflow in these calculations. +If allocating an array of pointers to point at these strings, the size of the + pointers may be larger than 4 octets, potentially requiring a separate + overflow check. + + + +Another bug in this class we have observed more than once involves the handling + of invalid data at the end of a stream. +Often, implementations will seek to the end of a stream to locate the last + timestamp in order to compute its total duration. +If they do not find a valid capture pattern and Ogg page from the desired + logical stream, they will back up and try again. +If care is not taken to avoid re-scanning data that was already scanned, this + search can quickly devolve into something with a complexity that is quadratic + in the amount of invalid data. + + + +In general when seeking, implementations will wish to be cautious about the + effects of invalid granule position values, and ensure all algorithms will + continue to make progress and eventually terminate, even if these are missing + or out-of-order. + + + +Like most other container formats, Ogg Opus streams SHOULD NOT be used with + insecure ciphers or cipher modes that are vulnerable to known-plaintext + attacks. +Elements such as the Ogg page capture pattern and the magic signatures in the + ID header and the comment header all have easily predictable values, in + addition to various elements of the codec data itself. + +
+ +
+ +An "Ogg Opus file" consists of one or more sequentially multiplexed segments, + each containing exactly one Ogg Opus stream. +The RECOMMENDED mime-type for Ogg Opus files is "audio/ogg". + + + +If more specificity is desired, one MAY indicate the presence of Opus streams + using the codecs parameter defined in and + , e.g., + +
+ +
+ + for an Ogg Opus file. + + + +The RECOMMENDED filename extension for Ogg Opus files is '.opus'. + + + +When Opus is concurrently multiplexed with other streams in an Ogg container, + one SHOULD use one of the "audio/ogg", "video/ogg", or "application/ogg" + mime-types, as defined in . +Such streams are not strictly "Ogg Opus files" as described above, + since they contain more than a single Opus stream per sequentially + multiplexed segment, e.g. video or multiple audio tracks. +In such cases the the '.opus' filename extension is NOT RECOMMENDED. + + + +In either case, this document updates + to add 'opus' as a codecs parameter value with char[8]: 'OpusHead' + as Codec Identifier. + +
+ +
+ +This document updates the IANA Media Types registry to add .opus + as a file extension for "audio/ogg", and to add itself as a reference + alongside for "audio/ogg", "video/ogg", and + "application/ogg" Media Types. + + +This document defines a new registry "Opus Channel Mapping Families" to + indicate how the semantic meanings of the channels in a multi-channel Opus + stream are described. +IANA is requested to create a new name space of "Opus Channel Mapping + Families". +This will be a new registry on the IANA Matrix, and not a subregistry of an + existing registry. +Modifications to this registry follow the "Specification Required" registration + policy as defined in . +Each registry entry consists of a Channel Mapping Family Number, which is + specified in decimal in the range 0 to 255, inclusive, and a Reference (or + list of references) +Each Reference must point to sufficient documentation to describe what + information is coded in the Opus identification header for this channel + mapping family, how a demuxer determines the Stream Count ('N') and Coupled + Stream Count ('M') from this information, and how it determines the proper + interpretation of each of the decoded channels. + + +This document defines three initial assignments for this registry. + + +ValueReference +0[RFCXXXX] +1[RFCXXXX] +255[RFCXXXX] + + +The designated expert will determine if the Reference points to a specification + that meets the requirements for permanence and ready availability laid out + in  and that it specifies the information + described above with sufficient clarity to allow interoperable + implementations. + +
+ +
+ +Thanks to Ben Campbell, Joel M. Halpern, Mark Harris, Greg Maxwell, + Christopher "Monty" Montgomery, Jean-Marc Valin, Stephan Wenger, and Mo Zanaty + for their valuable contributions to this document. +Additional thanks to Andrew D'Addesio, Greg Maxwell, and Vincent Penquerc'h for + their feedback based on early implementations. + +
+ +
+ +In , "RFCXXXX" is to be replaced with the RFC number + assigned to this draft. + +
+ +
+ + + &rfc2119; + &rfc3533; + &rfc3629; + &rfc5226; + &rfc5334; + &rfc6381; + &rfc6716; + + + + Loudness Recommendation EBU R128 + + EBU Technical Committee + + + + + + + +Ogg Vorbis I Format Specification: Comment Field and Header + Specification + + + + + + + + + + + &rfc4732; + &rfc6982; + &rfc7587; + + + + FLAC - Free Lossless Audio Codec Format Description + + + + + + + + Hann window + + Wikipedia + + + + + + + + Linear Predictive Coding + + Wikipedia + + + + + + + + Autocorrelation LPC coeff generation algorithm + (Vorbis source code) + + + + + + + + +Q (number format) +Wikipedia + + + + + + +VorbisComment: Replay Gain + + + + + + + + +Granulepos Encoding and How Seeking Really Works + + + + + + + + + +The Vorbis I Specification, Section 4.3.9 Output Channel Order + + + + + + + + The Vorbis I Specification, Appendix A: Embedding Vorbis + into an Ogg stream + + + + + + + + Multiple Channel Audio Data and WAVE Files + + Microsoft Corporation + + + + + + + + +
diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus-update.xml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus-update.xml new file mode 100644 index 000000000..3124e22c0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus-update.xml @@ -0,0 +1,513 @@ + + + + + + + + + + + + + + + Updates to the Opus Audio Codec + + +Mozilla Corporation +
+ +331 E. Evelyn Avenue +Mountain View +CA +94041 +USA + ++1 650 903-0800 +jmvalin@jmvalin.ca +
+
+ + +vocTone +
+ + + + + + + + +koenvos74@gmail.com +
+
+ + + + + + + This document addresses minor issues that were found in the specification + of the Opus audio codec in RFC 6716. It updates the normative decoder implementation + included in the appendix of RFC 6716. The changes fixes real and potential security-related + issues, as well minor quality-related issues. + +
+ + +
+ This document addresses minor issues that were discovered in the reference + implementation of the Opus codec. Unlike most IETF specifications, Opus is defined + in RFC 6716 in terms of a normative reference + decoder implementation rather than from the associated text description. + That RFC includes the reference decoder implementation as Appendix A. + That's why only issues affecting the decoder are + listed here. An up-to-date implementation of the Opus encoder can be found at + . + + Some of the changes in this document update normative behaviour in a way that requires + new test vectors. The English text of the specification is unaffected, only + the C implementation is. The updated specification remains fully compatible with + the original specification. + + + + Note: due to RFC formatting conventions, lines exceeding the column width + in the patch are split using a backslash character. The backslashes + at the end of a line and the white space at the beginning + of the following line are not part of the patch. A properly formatted patch + including all changes is available at + + and has a SHA-1 hash of 029e3aa88fc342c91e67a21e7bfbc9458661cd5f. + + +
+ +
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119. +
+ +
+ The reference implementation does not reinitialize the stereo state + during a mode switch. The old stereo memory can produce a brief impulse + (i.e. single sample) in the decoded audio. This can be fixed by changing + silk/dec_API.c at line 72: + +
+ + for( n = 0; n < DECODER_NUM_CHANNELS; n++ ) { + ret = silk_init_decoder( &channel_state[ n ] ); + } ++ silk_memset(&((silk_decoder *)decState)->sStereo, 0, ++ sizeof(((silk_decoder *)decState)->sStereo)); ++ /* Not strictly needed, but it's cleaner that way */ ++ ((silk_decoder *)decState)->prev_decode_only_middle = 0; + + return ret; + } + +]]> +
+ + This change affects the normative output of the decoder, but the + amount of change is within the tolerance and too small to make the testvector check fail. + +
+ +
+ It was discovered that some invalid packets of very large size could trigger + an out-of-bounds read in the Opus packet parsing code responsible for padding. + This is due to an integer overflow if the signaled padding exceeds 2^31-1 bytes + (the actual packet may be smaller). The code can be fixed by decrementing the + (signed) len value, instead of incrementing a separate padding counter. + This is done by applying the following changes at line 596 of src/opus_decoder.c: + +
+ + /* Padding flag is bit 6 */ + if (ch&0x40) + { +- int padding=0; + int p; + do { + if (len<=0) + return OPUS_INVALID_PACKET; + p = *data++; + len--; +- padding += p==255 ? 254: p; ++ len -= p==255 ? 254: p; + } while (p==255); +- len -= padding; + } + +]]> +
+ This packet parsing issue is limited to reading memory up + to about 60 kB beyond the compressed buffer. This can only be triggered + by a compressed packet more than about 16 MB long, so it's not a problem + for RTP. In theory, it could crash a file + decoder (e.g. Opus in Ogg) if the memory just after the incoming packet + is out-of-range, but our attempts to trigger such a crash in a production + application built using an affected version of the Opus decoder failed. +
+ +
+ The SILK resampler had the following issues: + + The calls to memcpy() were using sizeof(opus_int32), but the type of the + local buffer was opus_int16. + Because the size was wrong, this potentially allowed the source + and destination regions of the memcpy() to overlap on the copy from "buf" to "buf". + We believe that nSamplesIn (number of input samples) is at least fs_in_khZ (sampling rate in kHz), + which is at least 8. + Since RESAMPLER_ORDER_FIR_12 is only 8, that should not be a problem once + the type size is fixed. + The size of the buffer used RESAMPLER_MAX_BATCH_SIZE_IN, but the + data stored in it was actually twice the input batch size + (nSamplesIn<<1). + + The code can be fixed by applying the following changes to line 78 of silk/resampler_private_IIR_FIR.c: + +
+ + ) + { + silk_resampler_state_struct *S = \ +(silk_resampler_state_struct *)SS; + opus_int32 nSamplesIn; + opus_int32 max_index_Q16, index_increment_Q16; +- opus_int16 buf[ RESAMPLER_MAX_BATCH_SIZE_IN + \ +RESAMPLER_ORDER_FIR_12 ]; ++ opus_int16 buf[ 2*RESAMPLER_MAX_BATCH_SIZE_IN + \ +RESAMPLER_ORDER_FIR_12 ]; + + /* Copy buffered samples to start of buffer */ +- silk_memcpy( buf, S->sFIR, RESAMPLER_ORDER_FIR_12 \ +* sizeof( opus_int32 ) ); ++ silk_memcpy( buf, S->sFIR, RESAMPLER_ORDER_FIR_12 \ +* sizeof( opus_int16 ) ); + + /* Iterate over blocks of frameSizeIn input samples */ + index_increment_Q16 = S->invRatio_Q16; + while( 1 ) { + nSamplesIn = silk_min( inLen, S->batchSize ); + + /* Upsample 2x */ + silk_resampler_private_up2_HQ( S->sIIR, &buf[ \ +RESAMPLER_ORDER_FIR_12 ], in, nSamplesIn ); + + max_index_Q16 = silk_LSHIFT32( nSamplesIn, 16 + 1 \ +); /* + 1 because 2x upsampling */ + out = silk_resampler_private_IIR_FIR_INTERPOL( out, \ +buf, max_index_Q16, index_increment_Q16 ); + in += nSamplesIn; + inLen -= nSamplesIn; + + if( inLen > 0 ) { + /* More iterations to do; copy last part of \ +filtered signal to beginning of buffer */ +- silk_memcpy( buf, &buf[ nSamplesIn << 1 ], \ +RESAMPLER_ORDER_FIR_12 * sizeof( opus_int32 ) ); ++ silk_memmove( buf, &buf[ nSamplesIn << 1 ], \ +RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + } else { + break; + } + } + + /* Copy last part of filtered signal to the state for \ +the next call */ +- silk_memcpy( S->sFIR, &buf[ nSamplesIn << 1 ], \ +RESAMPLER_ORDER_FIR_12 * sizeof( opus_int32 ) ); ++ silk_memcpy( S->sFIR, &buf[ nSamplesIn << 1 ], \ +RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + } + +]]> +
+
+ +
+ + It was discovered through decoder fuzzing that some bitstreams could produce + integer values exceeding 32-bits in LPC_inverse_pred_gain_QA(), causing + a wrap-around. The C standard considers + this behavior as undefined. The following patch to line 87 of silk/LPC_inv_pred_gain.c + detects values that do not fit in a 32-bit integer and considers the corresponding filters unstable: + +
+ + /* Update AR coefficient */ + for( n = 0; n < k; n++ ) { +- tmp_QA = Aold_QA[ n ] - MUL32_FRAC_Q( \ +Aold_QA[ k - n - 1 ], rc_Q31, 31 ); +- Anew_QA[ n ] = MUL32_FRAC_Q( tmp_QA, rc_mult2 , mult2Q ); ++ opus_int64 tmp64; ++ tmp_QA = silk_SUB_SAT32( Aold_QA[ n ], MUL32_FRAC_Q( \ +Aold_QA[ k - n - 1 ], rc_Q31, 31 ) ); ++ tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( tmp_QA, \ +rc_mult2 ), mult2Q); ++ if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { ++ return 0; ++ } ++ Anew_QA[ n ] = ( opus_int32 )tmp64; + } + +]]> +
+
+ +
+ + It was discovered -- also from decoder fuzzing -- that an integer wrap-around could + occur when decoding bitstreams with extremely large values for the high LSF parameters. + The end result of the wrap-around is an illegal read access on the stack, which + the authors do not believe is exploitable but should nonetheless be fixed. The following + patch to line 137 of silk/NLSF_stabilize.c prevents the problem: + +
+ + /* Keep delta_min distance between the NLSFs */ + for( i = 1; i < L; i++ ) +- NLSF_Q15[i] = silk_max_int( NLSF_Q15[i], \ +NLSF_Q15[i-1] + NDeltaMin_Q15[i] ); ++ NLSF_Q15[i] = silk_max_int( NLSF_Q15[i], \ +silk_ADD_SAT16( NLSF_Q15[i-1], NDeltaMin_Q15[i] ) ); + + /* Last NLSF should be no higher than 1 - NDeltaMin[L] */ + +]]> +
+ +
+ +
+ On extreme bit-streams, it is possible for log-domain band energy levels + to exceed the maximum single-precision floating point value once converted + to a linear scale. This would later cause the decoded values to be NaN (not a number), + possibly causing problems in the software using the PCM values. This can be + avoided with the following patch to line 552 of celt/quant_bands.c: + +
+ + { + opus_val16 lg = ADD16(oldEBands[i+c*m->nbEBands], + SHL16((opus_val16)eMeans[i],6)); ++ lg = MIN32(QCONST32(32.f, 16), lg); + eBands[i+c*m->nbEBands] = PSHR32(celt_exp2(lg),4); + } + for (;inbEBands;i++) + +]]> +
+
+ +
+ When encoding in hybrid mode at low bitrate, we sometimes only have + enough bits to code a single CELT band (8 - 9.6 kHz). When that happens, + the second band (CELT band 18, from 9.6 to 12 kHz) cannot use folding + because it is wider than the amount already coded, and falls back to + white noise. Because it can also happen on transients (e.g. stops), it + can cause audible pre-echo. + + + To address the issue, we change the folding behavior so that it is + never forced to fall back to LCG due to the first band not containing + enough coefficients to fold onto the second band. This + is achieved by simply repeating part of the first band in the folding + of the second band. This changes the code in celt/bands.c around line 1237: + +
+ + b = 0; + } + +- if (resynth && M*eBands[i]-N >= M*eBands[start] && \ +(update_lowband || lowband_offset==0)) ++ if (resynth && (M*eBands[i]-N >= M*eBands[start] || \ +i==start+1) && (update_lowband || lowband_offset==0)) + lowband_offset = i; + ++ if (i == start+1) ++ { ++ int n1, n2; ++ int offset; ++ n1 = M*(eBands[start+1]-eBands[start]); ++ n2 = M*(eBands[start+2]-eBands[start+1]); ++ offset = M*eBands[start]; ++ /* Duplicate enough of the first band folding data to \ +be able to fold the second band. ++ Copies no data for CELT-only mode. */ ++ OPUS_COPY(&norm[offset+n1], &norm[offset+2*n1 - n2], n2-n1); ++ if (C==2) ++ OPUS_COPY(&norm2[offset+n1], &norm2[offset+2*n1 - n2], \ +n2-n1); ++ } ++ + tf_change = tf_res[i]; + if (i>=m->effEBands) + { + +]]> +
+ + + as well as line 1260: + + +
+ + fold_start = lowband_offset; + while(M*eBands[--fold_start] > effective_lowband); + fold_end = lowband_offset-1; +- while(M*eBands[++fold_end] < effective_lowband+N); ++ while(++fold_end < i && M*eBands[fold_end] < \ +effective_lowband+N); + x_cm = y_cm = 0; + fold_i = fold_start; do { + x_cm |= collapse_masks[fold_i*C+0]; + + +]]> +
+ + The fix does not impact compatibility, because the improvement does + not depend on the encoder doing anything special. There is also no + reasonable way for an encoder to use the original behavior to + improve quality over the proposed change. + +
+ +
+ The last issue is not strictly a bug, but it is an issue that has been reported + when downmixing an Opus decoded stream to mono, whether this is done inside the decoder + or as a post-processing step on the stereo decoder output. Opus intensity stereo allows + optionally coding the two channels 180-degrees out of phase on a per-band basis. + This provides better stereo quality than forcing the two channels to be in phase, + but when the output is downmixed to mono, the energy in the affected bands is cancelled + sometimes resulting in audible artifacts. + + As a work-around for this issue, the decoder MAY choose not to apply the 180-degree + phase shift. This can be useful when downmixing to mono inside or + outside of the decoder (e.g. user-controllable). + +
+ + +
+ Changes in and have + sufficient impact on the testvectors to make them fail. For this reason, + this document also updates the Opus test vectors. The new test vectors now + include two decoded outputs for the same bitstream. The outputs with + suffix 'm' do not apply the CELT 180-degree phase shift as allowed in + , while the outputs without the suffix do. An + implementation is compliant as long as it passes either set of vectors. + + + Any Opus implementation + that passes either the original test vectors from RFC 6716 + or one of the new sets of test vectors is compliant with the Opus specification. However, newer implementations + SHOULD be based on the new test vectors rather than the old ones. + + The new test vectors are located at + . + The SHA-1 hashes of the test vectors are: +
+ + + +
+ Note that the decoder input bitstream files (.bit) are unchanged. +
+
+ +
+ This document fixes two security issues reported on Opus and that affect the + reference implementation in RFC 6716: CVE-2013-0899 + + and CVE-2017-0381 . + CVE- 2013-0899 theoretically could have caused an information leak. The leaked + information would have gone through the decoder process before being accessible + to the attacker. It is fixed by . + CVE-2017-0381 could have resulted in a 16-bit out-of-bounds read from a fixed + location. It is fixed in . + Beyond the two fixed CVEs, this document adds no new security considerations on top of + RFC 6716. + +
+ +
+ This document makes no request of IANA. + + Note to RFC Editor: this section may be removed on publication as an + RFC. +
+ +
+ We would like to thank Juri Aedla for reporting the issue with the parsing of + the Opus padding. Thanks to Felicia Lim for reporting the LSF integer overflow issue. + Also, thanks to Tina le Grand, Jonathan Lennox, and Mark Harris for their + feedback on this document. +
+
+ + + + + + + + + +
diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus.xml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus.xml new file mode 100644 index 000000000..334cad97a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-codec-opus.xml @@ -0,0 +1,8276 @@ + + + + + + + +Definition of the Opus Audio Codec + + + +Mozilla Corporation +
+ +650 Castro Street +Mountain View +CA +94041 +USA + ++1 650 903-0800 +jmvalin@jmvalin.ca +
+
+ + +Skype Technologies S.A. +
+ +Soder Malarstrand 43 +Stockholm + +11825 +SE + ++46 73 085 7619 +koen.vos@skype.net +
+
+ + +Mozilla Corporation +
+ +650 Castro Street +Mountain View +CA +94041 +USA + ++1 650 903-0800 +tterriberry@mozilla.com +
+
+ + + +General + + + + + +This document defines the Opus interactive speech and audio codec. +Opus is designed to handle a wide range of interactive audio applications, + including Voice over IP, videoconferencing, in-game chat, and even live, + distributed music performances. +It scales from low bitrate narrowband speech at 6 kb/s to very high quality + stereo music at 510 kb/s. +Opus uses both linear prediction (LP) and the Modified Discrete Cosine + Transform (MDCT) to achieve good compression of both speech and music. + + +
+ + + +
+ +The Opus codec is a real-time interactive audio codec designed to meet the requirements +described in . +It is composed of a linear + prediction (LP)-based layer and a Modified Discrete Cosine Transform + (MDCT)-based layer. +The main idea behind using two layers is that in speech, linear prediction + techniques (such as Code-Excited Linear Prediction, or CELP) code low frequencies more efficiently than transform + (e.g., MDCT) domain techniques, while the situation is reversed for music and + higher speech frequencies. +Thus a codec with both layers available can operate over a wider range than + either one alone and, by combining them, achieve better quality than either + one individually. + + + +The primary normative part of this specification is provided by the source code + in . +Only the decoder portion of this software is normative, though a + significant amount of code is shared by both the encoder and decoder. + provides a decoder conformance test. +The decoder contains a great deal of integer and fixed-point arithmetic which + needs to be performed exactly, including all rounding considerations, so any + useful specification requires domain-specific symbolic language to adequately + define these operations. +Additionally, any +conflict between the symbolic representation and the included reference +implementation must be resolved. For the practical reasons of compatibility and +testability it would be advantageous to give the reference implementation +priority in any disagreement. The C language is also one of the most +widely understood human-readable symbolic representations for machine +behavior. +For these reasons this RFC uses the reference implementation as the sole + symbolic representation of the codec. + + +While the symbolic representation is unambiguous and complete it is not +always the easiest way to understand the codec's operation. For this reason +this document also describes significant parts of the codec in English and +takes the opportunity to explain the rationale behind many of the more +surprising elements of the design. These descriptions are intended to be +accurate and informative, but the limitations of common English sometimes +result in ambiguity, so it is expected that the reader will always read +them alongside the symbolic representation. Numerous references to the +implementation are provided for this purpose. The descriptions sometimes +differ from the reference in ordering or through mathematical simplification +wherever such deviation makes an explanation easier to understand. +For example, the right shift and left shift operations in the reference +implementation are often described using division and multiplication in the text. +In general, the text is focused on the "what" and "why" while the symbolic +representation most clearly provides the "how". + + +
+ +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", + "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be + interpreted as described in RFC 2119 . + + +Various operations in the codec require bit-exact fixed-point behavior, even + when writing a floating point implementation. +The notation "Q<n>", where n is an integer, denotes the number of binary + digits to the right of the decimal point in a fixed-point number. +For example, a signed Q14 value in a 16-bit word can represent values from + -2.0 to 1.99993896484375, inclusive. +This notation is for informational purposes only. +Arithmetic, when described, always operates on the underlying integer. +E.g., the text will explicitly indicate any shifts required after a + multiplication. + + +Expressions, where included in the text, follow C operator rules and + precedence, with the exception that the syntax "x**y" indicates x raised to + the power y. +The text also makes use of the following functions: + + +
+ +The smallest of two values x and y. + +
+ +
+ +The largest of two values x and y. + +
+ +
+
+ +
+ +With this definition, if lo > hi, the lower bound is the one that + is enforced. + +
+ +
+ +The sign of x, i.e., +
+ 0 . +]]> +
+
+
+ +
+ +The absolute value of x, i.e., +
+ +
+
+
+ +
+ +The largest integer z such that z <= f. + +
+ +
+ +The smallest integer z such that z >= f. + +
+ +
+ +The integer z nearest to f, with ties rounded towards negative infinity, + i.e., +
+ +
+
+
+ +
+ +The base-two logarithm of f. + +
+ +
+ +The minimum number of bits required to store a positive integer n in two's + complement notation, or 0 for a non-positive integer n. +
+ 0 +]]> +
+Examples: + +ilog(-1) = 0 +ilog(0) = 0 +ilog(1) = 1 +ilog(2) = 2 +ilog(3) = 2 +ilog(4) = 3 +ilog(7) = 3 + +
+
+ +
+ +
+ +
+ + +The Opus codec scales from 6 kb/s narrowband mono speech to 510 kb/s + fullband stereo music, with algorithmic delays ranging from 5 ms to + 65.2 ms. +At any given time, either the LP layer, the MDCT layer, or both, may be active. +It can seamlessly switch between all of its various operating modes, giving it + a great deal of flexibility to adapt to varying content and network + conditions without renegotiating the current session. +The codec allows input and output of various audio bandwidths, defined as + follows: + + +Abbreviation +Audio Bandwidth +Sample Rate (Effective) +NB (narrowband) 4 kHz 8 kHz +MB (medium-band) 6 kHz 12 kHz +WB (wideband) 8 kHz 16 kHz +SWB (super-wideband) 12 kHz 24 kHz +FB (fullband) 20 kHz (*) 48 kHz + + +(*) Although the sampling theorem allows a bandwidth as large as half the + sampling rate, Opus never codes audio above 20 kHz, as that is the + generally accepted upper limit of human hearing. + + + +Opus defines super-wideband (SWB) with an effective sample rate of 24 kHz, + unlike some other audio coding standards that use 32 kHz. +This was chosen for a number of reasons. +The band layout in the MDCT layer naturally allows skipping coefficients for + frequencies over 12 kHz, but does not allow cleanly dropping just those + frequencies over 16 kHz. +A sample rate of 24 kHz also makes resampling in the MDCT layer easier, + as 24 evenly divides 48, and when 24 kHz is sufficient, it can save + computation in other processing, such as Acoustic Echo Cancellation (AEC). +Experimental changes to the band layout to allow a 16 kHz cutoff + (32 kHz effective sample rate) showed potential quality degradations at + other sample rates, and at typical bitrates the number of bits saved by using + such a cutoff instead of coding in fullband (FB) mode is very small. +Therefore, if an application wishes to process a signal sampled at 32 kHz, + it should just use FB. + + + +The LP layer is based on the SILK codec + . +It supports NB, MB, or WB audio and frame sizes from 10 ms to 60 ms, + and requires an additional 5 ms look-ahead for noise shaping estimation. +A small additional delay (up to 1.5 ms) may be required for sampling rate + conversion. +Like Vorbis and many other modern codecs, SILK is inherently designed for + variable-bitrate (VBR) coding, though the encoder can also produce + constant-bitrate (CBR) streams. +The version of SILK used in Opus is substantially modified from, and not + compatible with, the stand-alone SILK codec previously deployed by Skype. +This document does not serve to define that format, but those interested in the + original SILK codec should see instead. + + + +The MDCT layer is based on the CELT codec . +It supports NB, WB, SWB, or FB audio and frame sizes from 2.5 ms to + 20 ms, and requires an additional 2.5 ms look-ahead due to the + overlapping MDCT windows. +The CELT codec is inherently designed for CBR coding, but unlike many CBR + codecs it is not limited to a set of predetermined rates. +It internally allocates bits to exactly fill any given target budget, and an + encoder can produce a VBR stream by varying the target on a per-frame basis. +The MDCT layer is not used for speech when the audio bandwidth is WB or less, + as it is not useful there. +On the other hand, non-speech signals are not always adequately coded using + linear prediction, so for music only the MDCT layer should be used. + + + +A "Hybrid" mode allows the use of both layers simultaneously with a frame size + of 10 or 20 ms and a SWB or FB audio bandwidth. +The LP layer codes the low frequencies by resampling the signal down to WB. +The MDCT layer follows, coding the high frequency portion of the signal. +The cutoff between the two lies at 8 kHz, the maximum WB audio bandwidth. +In the MDCT layer, all bands below 8 kHz are discarded, so there is no + coding redundancy between the two layers. + + + +The sample rate (in contrast to the actual audio bandwidth) can be chosen + independently on the encoder and decoder side, e.g., a fullband signal can be + decoded as wideband, or vice versa. +This approach ensures a sender and receiver can always interoperate, regardless + of the capabilities of their actual audio hardware. +Internally, the LP layer always operates at a sample rate of twice the audio + bandwidth, up to a maximum of 16 kHz, which it continues to use for SWB + and FB. +The decoder simply resamples its output to support different sample rates. +The MDCT layer always operates internally at a sample rate of 48 kHz. +Since all the supported sample rates evenly divide this rate, and since the + the decoder may easily zero out the high frequency portion of the spectrum in + the frequency domain, it can simply decimate the MDCT layer output to achieve + the other supported sample rates very cheaply. + + + +After conversion to the common, desired output sample rate, the decoder simply + adds the output from the two layers together. +To compensate for the different look-ahead required by each layer, the CELT + encoder input is delayed by an additional 2.7 ms. +This ensures that low frequencies and high frequencies arrive at the same time. +This extra delay may be reduced by an encoder by using less look-ahead for noise + shaping or using a simpler resampler in the LP layer, but this will reduce + quality. +However, the base 2.5 ms look-ahead in the CELT layer cannot be reduced in + the encoder because it is needed for the MDCT overlap, whose size is fixed by + the decoder. + + + +Both layers use the same entropy coder, avoiding any waste from "padding bits" + between them. +The hybrid approach makes it easy to support both CBR and VBR coding. +Although the LP layer is VBR, the bit allocation of the MDCT layer can produce + a final stream that is CBR by using all the bits left unused by the LP layer. + + +
+ +The Opus codec includes a number of control parameters which can be changed dynamically during +regular operation of the codec, without interrupting the audio stream from the encoder to the decoder. +These parameters only affect the encoder since any impact they have on the bit-stream is signaled +in-band such that a decoder can decode any Opus stream without any out-of-band signaling. Any Opus +implementation can add or modify these control parameters without affecting interoperability. The most +important encoder control parameters in the reference encoder are listed below. + + +
+ +Opus supports all bitrates from 6 kb/s to 510 kb/s. All other parameters being +equal, higher bitrate results in higher quality. For a frame size of 20 ms, these +are the bitrate "sweet spots" for Opus in various configurations: + +8-12 kb/s for NB speech, +16-20 kb/s for WB speech, +28-40 kb/s for FB speech, +48-64 kb/s for FB mono music, and +64-128 kb/s for FB stereo music. + + +
+ +
+ +Opus can transmit either mono or stereo frames within a single stream. +When decoding a mono frame in a stereo decoder, the left and right channels are + identical, and when decoding a stereo frame in a mono decoder, the mono output + is the average of the left and right channels. +In some cases, it is desirable to encode a stereo input stream in mono (e.g., + because the bitrate is too low to encode stereo with sufficient quality). +The number of channels encoded can be selected in real-time, but by default the + reference encoder attempts to make the best decision possible given the + current bitrate. + +
+ +
+ +The audio bandwidths supported by Opus are listed in + . +Just like for the number of channels, any decoder can decode audio encoded at + any bandwidth. +For example, any Opus decoder operating at 8 kHz can decode a FB Opus + frame, and any Opus decoder operating at 48 kHz can decode a NB frame. +Similarly, the reference encoder can take a 48 kHz input signal and + encode it as NB. +The higher the audio bandwidth, the higher the required bitrate to achieve + acceptable quality. +The audio bandwidth can be explicitly specified in real-time, but by default + the reference encoder attempts to make the best bandwidth decision possible + given the current bitrate. + +
+ + +
+ +Opus can encode frames of 2.5, 5, 10, 20, 40 or 60 ms. +It can also combine multiple frames into packets of up to 120 ms. +For real-time applications, sending fewer packets per second reduces the + bitrate, since it reduces the overhead from IP, UDP, and RTP headers. +However, it increases latency and sensitivity to packet losses, as losing one + packet constitutes a loss of a bigger chunk of audio. +Increasing the frame duration also slightly improves coding efficiency, but the + gain becomes small for frame sizes above 20 ms. +For this reason, 20 ms frames are a good choice for most applications. + +
+ +
+ +There are various aspects of the Opus encoding process where trade-offs +can be made between CPU complexity and quality/bitrate. In the reference +encoder, the complexity is selected using an integer from 0 to 10, where +0 is the lowest complexity and 10 is the highest. Examples of +computations for which such trade-offs may occur are: + +The order of the pitch analysis whitening filter , +The order of the short-term noise shaping filter, +The number of states in delayed decision quantization of the +residual signal, and +The use of certain bit-stream features such as variable time-frequency +resolution and the pitch post-filter. + + +
+ +
+ +Audio codecs often exploit inter-frame correlations to reduce the +bitrate at a cost in error propagation: after losing one packet +several packets need to be received before the decoder is able to +accurately reconstruct the speech signal. The extent to which Opus +exploits inter-frame dependencies can be adjusted on the fly to +choose a trade-off between bitrate and amount of error propagation. + +
+ +
+ + Another mechanism providing robustness against packet loss is the in-band + Forward Error Correction (FEC). Packets that are determined to + contain perceptually important speech information, such as onsets or + transients, are encoded again at a lower bitrate and this re-encoded + information is added to a subsequent packet. + +
+ +
+ +Opus is more efficient when operating with variable bitrate (VBR), which is +the default. However, in some (rare) applications, constant bitrate (CBR) +is required. There are two main reasons to operate in CBR mode: + +When the transport only supports a fixed size for each compressed frame +When encryption is used for an audio stream that is either highly constrained + (e.g. yes/no, recorded prompts) or highly sensitive + + +When low-latency transmission is required over a relatively slow connection, then +constrained VBR can also be used. This uses VBR in a way that simulates a +"bit reservoir" and is equivalent to what MP3 (MPEG 1, Layer 3) and +AAC (Advanced Audio Coding) call CBR (i.e., not true +CBR due to the bit reservoir). + +
+ +
+ + Discontinuous Transmission (DTX) reduces the bitrate during silence + or background noise. When DTX is enabled, only one frame is encoded + every 400 milliseconds. + +
+ +
+ +
+ +
+ + +The Opus encoder produces "packets", which are each a contiguous set of bytes + meant to be transmitted as a single unit. +The packets described here do not include such things as IP, UDP, or RTP + headers which are normally found in a transport-layer packet. +A single packet may contain multiple audio frames, so long as they share a + common set of parameters, including the operating mode, audio bandwidth, frame + size, and channel count (mono vs. stereo). +This section describes the possible combinations of these parameters and the + internal framing used to pack multiple frames into a single packet. +This framing is not self-delimiting. +Instead, it assumes that a higher layer (such as UDP or RTP +or Ogg or Matroska ) + will communicate the length, in bytes, of the packet, and it uses this + information to reduce the framing overhead in the packet itself. +A decoder implementation MUST support the framing described in this section. +An alternative, self-delimiting variant of the framing is described in + . +Support for that variant is OPTIONAL. + + + +All bit diagrams in this document number the bits so that bit 0 is the most + significant bit of the first byte, and bit 7 is the least significant. +Bit 8 is thus the most significant bit of the second byte, etc. +Well-formed Opus packets obey certain requirements, marked [R1] through [R7] + below. +These are summarized in along with + appropriate means of handling malformed packets. + + +
+ +A well-formed Opus packet MUST contain at least one byte [R1]. +This byte forms a table-of-contents (TOC) header that signals which of the + various modes and configurations a given packet uses. +It is composed of a configuration number, "config", a stereo flag, "s", and a + frame count code, "c", arranged as illustrated in + . +A description of each of these fields follows. + + +
+ +
+ + +The top five bits of the TOC byte, labeled "config", encode one of 32 possible + configurations of operating mode, audio bandwidth, and frame size. +As described, the LP (SILK) layer and MDCT (CELT) layer can be combined in three possible + operating modes: + +A SILK-only mode for use in low bitrate connections with an audio bandwidth + of WB or less, +A Hybrid (SILK+CELT) mode for SWB or FB speech at medium bitrates, and +A CELT-only mode for very low delay speech transmission as well as music + transmission (NB to FB). + +The 32 possible configurations each identify which one of these operating modes + the packet uses, as well as the audio bandwidth and the frame size. + lists the parameters for each configuration. + + +Configuration Number(s) +Mode +Bandwidth +Frame Sizes +0...3 SILK-only NB 10, 20, 40, 60 ms +4...7 SILK-only MB 10, 20, 40, 60 ms +8...11 SILK-only WB 10, 20, 40, 60 ms +12...13 Hybrid SWB 10, 20 ms +14...15 Hybrid FB 10, 20 ms +16...19 CELT-only NB 2.5, 5, 10, 20 ms +20...23 CELT-only WB 2.5, 5, 10, 20 ms +24...27 CELT-only SWB 2.5, 5, 10, 20 ms +28...31 CELT-only FB 2.5, 5, 10, 20 ms + + +The configuration numbers in each range (e.g., 0...3 for NB SILK-only) + correspond to the various choices of frame size, in the same order. +For example, configuration 0 has a 10 ms frame size and configuration 3 + has a 60 ms frame size. + + + +One additional bit, labeled "s", signals mono vs. stereo, with 0 indicating + mono and 1 indicating stereo. + + + +The remaining two bits of the TOC byte, labeled "c", code the number of frames + per packet (codes 0 to 3) as follows: + +0: 1 frame in the packet +1: 2 frames in the packet, each with equal compressed size +2: 2 frames in the packet, with different compressed sizes +3: an arbitrary number of frames in the packet + +This draft refers to a packet as a code 0 packet, code 1 packet, etc., based on + the value of "c". + + +
+ +
+ + +This section describes how frames are packed according to each possible value + of "c" in the TOC byte. + + +
+ +When a packet contains multiple VBR frames (i.e., code 2 or 3), the compressed + length of one or more of these frames is indicated with a one- or two-byte + sequence, with the meaning of the first byte as follows: + +0: No frame (discontinuous transmission (DTX) or lost packet) +1...251: Length of the frame in bytes +252...255: A second byte is needed. The total length is (second_byte*4)+first_byte + + + + +The special length 0 indicates that no frame is available, either because it + was dropped during transmission by some intermediary or because the encoder + chose not to transmit it. +Any Opus frame in any mode MAY have a length of 0. + + + +The maximum representable length is 255*4+255=1275 bytes. +For 20 ms frames, this represents a bitrate of 510 kb/s, which is + approximately the highest useful rate for lossily compressed fullband stereo + music. +Beyond this point, lossless codecs are more appropriate. +It is also roughly the maximum useful rate of the MDCT layer, as shortly + thereafter quality no longer improves with additional bits due to limitations + on the codebook sizes. + + + +No length is transmitted for the last frame in a VBR packet, or for any of the + frames in a CBR packet, as it can be inferred from the total size of the + packet and the size of all other data in the packet. +However, the length of any individual frame MUST NOT exceed + 1275 bytes [R2], to allow for repacketization by gateways, + conference bridges, or other software. + +
+ +
+ + +For code 0 packets, the TOC byte is immediately followed by N-1 bytes + of compressed data for a single frame (where N is the size of the packet), + as illustrated in . + +
+ +
+
+ +
+ +For code 1 packets, the TOC byte is immediately followed by the + (N-1)/2 bytes of compressed data for the first frame, followed by + (N-1)/2 bytes of compressed data for the second frame, as illustrated in + . +The number of payload bytes available for compressed data, N-1, MUST be even + for all code 1 packets [R3]. + +
+ +
+
+ +
+ +For code 2 packets, the TOC byte is followed by a one- or two-byte sequence + indicating the length of the first frame (marked N1 in ), + followed by N1 bytes of compressed data for the first frame. +The remaining N-N1-2 or N-N1-3 bytes are the compressed data for the + second frame. +This is illustrated in . +A code 2 packet MUST contain enough bytes to represent a valid length. +For example, a 1-byte code 2 packet is always invalid, and a 2-byte code 2 + packet whose second byte is in the range 252...255 is also invalid. +The length of the first frame, N1, MUST also be no larger than the size of the + payload remaining after decoding that length for all code 2 packets [R4]. +This makes, for example, a 2-byte code 2 packet with a second byte in the range + 1...251 invalid as well (the only valid 2-byte code 2 packet is one where the + length of both frames is zero). + +
+ +
+
+ +
+ +Code 3 packets signal the number of frames, as well as additional + padding, called "Opus padding" to indicate that this padding is added at the + Opus layer, rather than at the transport layer. +Code 3 packets MUST have at least 2 bytes [R6,R7]. +The TOC byte is followed by a byte encoding the number of frames in the packet + in bits 2 to 7 (marked "M" in ), with bit 1 indicating whether + or not Opus padding is inserted (marked "p" in ), and bit 0 + indicating VBR (marked "v" in ). +M MUST NOT be zero, and the audio duration contained within a packet MUST NOT + exceed 120 ms [R5]. +This limits the maximum frame count for any frame size to 48 (for 2.5 ms + frames), with lower limits for longer frame sizes. + illustrates the layout of the frame count + byte. + +
+ +
+ +When Opus padding is used, the number of bytes of padding is encoded in the + bytes following the frame count byte. +Values from 0...254 indicate that 0...254 bytes of padding are included, + in addition to the byte(s) used to indicate the size of the padding. +If the value is 255, then the size of the additional padding is 254 bytes, + plus the padding value encoded in the next byte. +There MUST be at least one more byte in the packet in this case [R6,R7]. +The additional padding bytes appear at the end of the packet, and MUST be set + to zero by the encoder to avoid creating a covert channel. +The decoder MUST accept any value for the padding bytes, however. + + +Although this encoding provides multiple ways to indicate a given number of + padding bytes, each uses a different number of bytes to indicate the padding + size, and thus will increase the total packet size by a different amount. +For example, to add 255 bytes to a packet, set the padding bit, p, to 1, insert + a single byte after the frame count byte with a value of 254, and append 254 + padding bytes with the value zero to the end of the packet. +To add 256 bytes to a packet, set the padding bit to 1, insert two bytes after + the frame count byte with the values 255 and 0, respectively, and append 254 + padding bytes with the value zero to the end of the packet. +By using the value 255 multiple times, it is possible to create a packet of any + specific, desired size. +Let P be the number of header bytes used to indicate the padding size plus the + number of padding bytes themselves (i.e., P is the total number of bytes added + to the packet). +Then P MUST be no more than N-2 [R6,R7]. + + +In the CBR case, let R=N-2-P be the number of bytes remaining in the packet + after subtracting the (optional) padding. +Then the compressed length of each frame in bytes is equal to R/M. +The value R MUST be a non-negative integer multiple of M [R6]. +The compressed data for all M frames follows, each of size + R/M bytes, as illustrated in . + + +
+ +
+ + +In the VBR case, the (optional) padding length is followed by M-1 frame + lengths (indicated by "N1" to "N[M-1]" in ), each encoded in a + one- or two-byte sequence as described above. +The packet MUST contain enough data for the M-1 lengths after removing the + (optional) padding, and the sum of these lengths MUST be no larger than the + number of bytes remaining in the packet after decoding them [R7]. +The compressed data for all M frames follows, each frame consisting of the + indicated number of bytes, with the final frame consuming any remaining bytes + before the final padding, as illustrated in . +The number of header bytes (TOC byte, frame count byte, padding length bytes, + and frame length bytes), plus the signaled length of the first M-1 frames themselves, + plus the signaled length of the padding MUST be no larger than N, the total size of the + packet. + + +
+ +
+
+
+ +
+ +Simplest case, one NB mono 20 ms SILK frame: + + +
+ +
+ + +Two FB mono 5 ms CELT frames of the same compressed size: + + +
+ +
+ + +Two FB mono 20 ms Hybrid frames of different compressed size: + + +
+ +
+ + +Four FB stereo 20 ms CELT frames of the same compressed size: + + +
+ +
+
+ +
+ +A receiver MUST NOT process packets which violate any of the rules above as + normal Opus packets. +They are reserved for future applications, such as in-band headers (containing + metadata, etc.). +Packets which violate these constraints may cause implementations of + this specification to treat them as malformed, and + discard them. + + +These constraints are summarized here for reference: + +Packets are at least one byte. +No implicit frame length is larger than 1275 bytes. +Code 1 packets have an odd total length, N, so that (N-1)/2 is an + integer. +Code 2 packets have enough bytes after the TOC for a valid frame + length, and that length is no larger than the number of bytes remaining in the + packet. +Code 3 packets contain at least one frame, but no more than 120 ms + of audio total. +The length of a CBR code 3 packet, N, is at least two bytes, the number of + bytes added to indicate the padding size plus the trailing padding bytes + themselves, P, is no more than N-2, and the frame count, M, satisfies + the constraint that (N-2-P) is a non-negative integer multiple of M. +VBR code 3 packets are large enough to contain all the header bytes (TOC + byte, frame count byte, any padding length bytes, and any frame length bytes), + plus the length of the first M-1 frames, plus any trailing padding bytes. + + +
+ +
+ +
+ +The Opus decoder consists of two main blocks: the SILK decoder and the CELT + decoder. +At any given time, one or both of the SILK and CELT decoders may be active. +The output of the Opus decode is the sum of the outputs from the SILK and CELT + decoders with proper sample rate conversion and delay compensation on the SILK + side, and optional decimation (when decoding to sample rates less than + 48 kHz) on the CELT side, as illustrated in the block diagram below. + +
+ +| Decoder |--->| Rate |----+ +Bit- +---------+ | | | | Conversion | v +stream | Range |---+ +---------+ +------------+ /---\ Audio +------->| Decoder | | + |------> + | |---+ +---------+ +------------+ \---/ + +---------+ | | CELT | | Decimation | ^ + +->| Decoder |--->| (Optional) |----+ + | | | | + +---------+ +------------+ +]]> + +
+ +
+ +Opus uses an entropy coder based on range coding +, +which is itself a rediscovery of the FIFO arithmetic code introduced by . +It is very similar to arithmetic encoding, except that encoding is done with +digits in any base instead of with bits, +so it is faster when using larger bases (i.e., a byte). All of the +calculations in the range coder must use bit-exact integer arithmetic. + + +Symbols may also be coded as "raw bits" packed directly into the bitstream, + bypassing the range coder. +These are packed backwards starting at the end of the frame, as illustrated in + . +This reduces complexity and makes the stream more resilient to bit errors, as + corruption in the raw bits will not desynchronize the decoding process, unlike + corruption in the input to the range decoder. +Raw bits are only used in the CELT layer. + + +
+ : ++ + +: : ++ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +: | <- Boundary occurs at an arbitrary bit position : ++-+-+-+ + +: <- Raw bits data (packed LSB to MSB) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +]]> +
+ + +Each symbol coded by the range coder is drawn from a finite alphabet and coded + in a separate "context", which describes the size of the alphabet and the + relative frequency of each symbol in that alphabet. + + +Suppose there is a context with n symbols, identified with an index that ranges + from 0 to n-1. +The parameters needed to encode or decode symbol k in this context are + represented by a three-tuple (fl[k], fh[k], ft), with + 0 <= fl[k] < fh[k] <= ft <= 65535. +The values of this tuple are derived from the probability model for the + symbol, represented by traditional "frequency counts". +Because Opus uses static contexts these are not updated as symbols are decoded. +Let f[i] be the frequency of symbol i. +Then the three-tuple corresponding to symbol k is given by + +
+ +
+ +The range decoder extracts the symbols and integers encoded using the range + encoder in . +The range decoder maintains an internal state vector composed of the two-tuple + (val, rng), representing the difference between the high end of the + current range and the actual coded value, minus one, and the size of the + current range, respectively. +Both val and rng are 32-bit unsigned integer values. + + +
+ +Let b0 be the first input byte (or zero if there are no bytes in this Opus + frame). +The decoder initializes rng to 128 and initializes val to + (127 - (b0>>1)), where (b0>>1) is the top 7 bits of the + first input byte. +It saves the remaining bit, (b0&1), for use in the renormalization + procedure described in , which the + decoder invokes immediately after initialization to read additional bits and + establish the invariant that rng > 2**23. + +
+ +
+ +Decoding a symbol is a two-step process. +The first step determines a 16-bit unsigned value fs, which lies within the + range of some symbol in the current context. +The second step updates the range decoder state with the three-tuple + (fl[k], fh[k], ft) corresponding to that symbol. + + +The first step is implemented by ec_decode() (entdec.c), which computes +
+ +
+The divisions here are integer division. +
+ +The decoder then identifies the symbol in the current context corresponding to + fs; i.e., the value of k whose three-tuple (fl[k], fh[k], ft) + satisfies fl[k] <= fs < fh[k]. +It uses this tuple to update val according to +
+ +
+If fl[k] is greater than zero, then the decoder updates rng using +
+ +
+Otherwise, it updates rng using +
+ +
+
+ +Using a special case for the first symbol (rather than the last symbol, as is + commonly done in other arithmetic coders) ensures that all the truncation + error from the finite precision arithmetic accumulates in symbol 0. +This makes the cost of coding a 0 slightly smaller, on average, than its + estimated probability indicates and makes the cost of coding any other symbol + slightly larger. +When contexts are designed so that 0 is the most probable symbol, which is + often the case, this strategy minimizes the inefficiency introduced by the + finite precision. +It also makes some of the special-case decoding routines in + particularly simple. + + +After the updates, implemented by ec_dec_update() (entdec.c), the decoder + normalizes the range using the procedure in the next section, and returns the + index k. + + +
+ +To normalize the range, the decoder repeats the following process, implemented + by ec_dec_normalize() (entdec.c), until rng > 2**23. +If rng is already greater than 2**23, the entire process is skipped. +First, it sets rng to (rng<<8). +Then it reads the next byte of the Opus frame and forms an 8-bit value sym, + using the left-over bit buffered from the previous byte as the high bit + and the top 7 bits of the byte just read as the other 7 bits of sym. +The remaining bit in the byte just read is buffered for use in the next + iteration. +If no more input bytes remain, it uses zero bits instead. +See for the initialization used to process + the first byte. +Then, it sets +
+ +
+
+ +It is normal and expected that the range decoder will read several bytes + into the raw bits data (if any) at the end of the packet by the time the frame + is completely decoded, as illustrated in . +This same data MUST also be returned as raw bits when requested. +The encoder is expected to terminate the stream in such a way that the decoder + will decode the intended values regardless of the data contained in the raw + bits. + describes a procedure for doing this. +If the range decoder consumes all of the bytes belonging to the current frame, + it MUST continue to use zero when any further input bytes are required, even + if there is additional data in the current packet from padding or other + frames. + + +
+ | : ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ^ ^ + | End of data buffered by the range coder | +...-----------------------------------------------+ + | + | End of data consumed by raw bits + +-------------------------------------------------------... +]]> +
+
+
+ +
+ +The reference implementation uses three additional decoding methods that are + exactly equivalent to the above, but make assumptions and simplifications that + allow for a more efficient implementation. + +
+ +The first is ec_decode_bin() (entdec.c), defined using the parameter ftb + instead of ft. +It is mathematically equivalent to calling ec_decode() with + ft = (1<<ftb), but avoids one of the divisions. + +
+
+ +The next is ec_dec_bit_logp() (entdec.c), which decodes a single binary symbol, + replacing both the ec_decode() and ec_dec_update() steps. +The context is described by a single parameter, logp, which is the absolute + value of the base-2 logarithm of the probability of a "1". +It is mathematically equivalent to calling ec_decode() with + ft = (1<<logp), followed by ec_dec_update() with + the 3-tuple (fl[k] = 0, + fh[k] = (1<<logp) - 1, + ft = (1<<logp)) if the returned value + of fs is less than (1<<logp) - 1 (a "0" was decoded), and with + (fl[k] = (1<<logp) - 1, + fh[k] = ft = (1<<logp)) otherwise (a "1" was + decoded). +The implementation requires no multiplications or divisions. + +
+
+ +The last is ec_dec_icdf() (entdec.c), which decodes a single symbol with a + table-based context of up to 8 bits, also replacing both the ec_decode() and + ec_dec_update() steps, as well as the search for the decoded symbol in between. +The context is described by two parameters, an icdf + ("inverse" cumulative distribution function) table and ftb. +As with ec_decode_bin(), (1<<ftb) is equivalent to ft. +idcf[k], on the other hand, stores (1<<ftb)-fh[k], which is equal to + (1<<ftb) - fl[k+1]. +fl[0] is assumed to be 0, and the table is terminated by a value of 0 (where + fh[k] == ft). + + +The function is mathematically equivalent to calling ec_decode() with + ft = (1<<ftb), using the returned value fs to search the table + for the first entry where fs < (1<<ftb)-icdf[k], and + calling ec_dec_update() with + fl[k] = (1<<ftb) - icdf[k-1] (or 0 + if k == 0), fh[k] = (1<<ftb) - idcf[k], + and ft = (1<<ftb). +Combining the search with the update allows the division to be replaced by a + series of multiplications (which are usually much cheaper), and using an + inverse CDF allows the use of an ftb as large as 8 in an 8-bit table without + any special cases. +This is the primary interface with the range decoder in the SILK layer, though + it is used in a few places in the CELT layer as well. + + +Although icdf[k] is more convenient for the code, the frequency counts, f[k], + are a more natural representation of the probability distribution function + (PDF) for a given symbol. +Therefore this draft lists the latter, not the former, when describing the + context in which a symbol is coded as a list, e.g., {4, 4, 4, 4}/16 for a + uniform context with four possible values and ft = 16. +The value of ft after the slash is always the sum of the entries in the PDF, + but is included for convenience. +Contexts with identical probabilities, f[k]/ft, but different values of ft + (or equivalently, ftb) are not the same, and cannot, in general, be used in + place of one another. +An icdf table is also not capable of representing a PDF where the first symbol + has 0 probability. +In such contexts, ec_dec_icdf() can decode the symbol by using a table that + drops the entries for any initial zero-probability values and adding the + constant offset of the first value with a non-zero probability to its return + value. + +
+
+ +
+ +The raw bits used by the CELT layer are packed at the end of the packet, with + the least significant bit of the first value packed in the least significant + bit of the last byte, filling up to the most significant bit in the last byte, + continuing on to the least significant bit of the penultimate byte, and so on. +The reference implementation reads them using ec_dec_bits() (entdec.c). +Because the range decoder must read several bytes ahead in the stream, as + described in , the input consumed by the + raw bits may overlap with the input consumed by the range coder, and a decoder + MUST allow this. +The format should render it impossible to attempt to read more raw bits than + there are actual bits in the frame, though a decoder may wish to check for + this and report an error. + +
+ +
+ +The function ec_dec_uint() (entdec.c) decodes one of ft equiprobable values in + the range 0 to (ft - 1), inclusive, each with a frequency of 1, + where ft may be as large as (2**32 - 1). +Because ec_decode() is limited to a total frequency of (2**16 - 1), + it splits up the value into a range coded symbol representing up to 8 of the + high bits, and, if necessary, raw bits representing the remainder of the + value. +The limit of 8 bits in the range coded symbol is a trade-off between + implementation complexity, modeling error (since the symbols no longer truly + have equal coding cost), and rounding error introduced by the range coder + itself (which gets larger as more bits are included). +Using raw bits reduces the maximum number of divisions required in the worst + case, but means that it may be possible to decode a value outside the range + 0 to (ft - 1), inclusive. + + + +ec_dec_uint() takes a single, positive parameter, ft, which is not necessarily + a power of two, and returns an integer, t, whose value lies between 0 and + (ft - 1), inclusive. +Let ftb = ilog(ft - 1), i.e., the number of bits required + to store (ft - 1) in two's complement notation. +If ftb is 8 or less, then t is decoded with t = ec_decode(ft), and + the range coder state is updated using the three-tuple (t, t + 1, + ft). + + +If ftb is greater than 8, then the top 8 bits of t are decoded using +
+> (ftb - 8)) + 1) , +]]> +
+ the decoder state is updated using the three-tuple + (t, t + 1, + ((ft - 1) >> (ftb - 8)) + 1), + and the remaining bits are decoded as raw bits, setting +
+ +
+If, at this point, t >= ft, then the current frame is corrupt. +In that case, the decoder should assume there has been an error in the coding, + decoding, or transmission and SHOULD take measures to conceal the + error and/or report to the application that the error has occurred. +
+ +
+ +
+ +The bit allocation routines in the CELT decoder need a conservative upper bound + on the number of bits that have been used from the current frame thus far, + including both range coder bits and raw bits. +This drives allocation decisions that must match those made in the encoder. +The upper bound is computed in the reference implementation to whole-bit + precision by the function ec_tell() (entcode.h) and to fractional 1/8th bit + precision by the function ec_tell_frac() (entcode.c). +Like all operations in the range coder, it must be implemented in a bit-exact + manner, and must produce exactly the same value returned by the same functions + in the encoder after encoding the same symbols. + + +ec_tell() is guaranteed to return ceil(ec_tell_frac()/8.0). +In various places the codec will check to ensure there is enough room to + contain a symbol before attempting to decode it. +In practice, although the number of bits used so far is an upper bound, + decoding a symbol whose probability model suggests it has a worst-case cost of + p 1/8th bits may actually advance the return value of ec_tell_frac() by + p-1, p, or p+1 1/8th bits, due to approximation error in that upper bound, + truncation error in the range coder, and for large values of ft, modeling + error in ec_dec_uint(). + + +However, this error is bounded, and periodic calls to ec_tell() or + ec_tell_frac() at precisely defined points in the decoding process prevent it + from accumulating. +For a range coder symbol that requires a whole number of bits (i.e., + for which ft/(fh[k] - fl[k]) is a power of two), where there are at + least p 1/8th bits available, decoding the symbol will never cause ec_tell() or + ec_tell_frac() to exceed the size of the frame ("bust the budget"). +In this case the return value of ec_tell_frac() will only advance by more than + p 1/8th bits if there was an additional, fractional number of bits remaining, + and it will never advance beyond the next whole-bit boundary, which is safe, + since frames always contain a whole number of bits. +However, when p is not a whole number of bits, an extra 1/8th bit is required + to ensure that decoding the symbol will not bust the budget. + + +The reference implementation keeps track of the total number of whole bits that + have been processed by the decoder so far in the variable nbits_total, + including the (possibly fractional) number of bits that are currently + buffered, but not consumed, inside the range coder. +nbits_total is initialized to 9 just before the initial range renormalization + process completes (or equivalently, it can be initialized to 33 after the + first renormalization). +The extra two bits over the actual amount buffered by the range coder + guarantees that it is an upper bound and that there is enough room for the + encoder to terminate the stream. +Each iteration through the range coder's renormalization loop increases + nbits_total by 8. +Reading raw bits increases nbits_total by the number of raw bits read. + + +
+ +The whole number of bits buffered in rng may be estimated via lg = ilog(rng). +ec_tell() then becomes a simple matter of removing these bits from the total. +It returns (nbits_total - lg). + + +In a newly initialized decoder, before any symbols have been read, this reports + that 1 bit has been used. +This is the bit reserved for termination of the encoder. + +
+ +
+ +ec_tell_frac() estimates the number of bits buffered in rng to fractional + precision. +Since rng must be greater than 2**23 after renormalization, lg must be at least + 24. +Let +
+ +> (lg-16) , +]]> +
+ so that 32768 <= r_Q15 < 65536, an unsigned Q15 value representing the + fractional part of rng. +Then the following procedure can be used to add one bit of precision to lg. +First, update +
+ +> 15 . +]]> +
+Then add the 16th bit of r_Q15 to lg via +
+ +> 16) . +]]> +
+Finally, if this bit was a 1, reduce r_Q15 by a factor of two via +
+ +> 1 , +]]> +
+ so that it once again lies in the range 32768 <= r_Q15 < 65536. +
+ +This procedure is repeated three times to extend lg to 1/8th bit precision. +ec_tell_frac() then returns (nbits_total*8 - lg). + +
+ +
+ +
+ +
+ +The decoder's LP layer uses a modified version of the SILK codec (herein simply + called "SILK"), which runs a decoded excitation signal through adaptive + long-term and short-term prediction synthesis filters. +It runs at NB, MB, and WB sample rates internally. +When used in a SWB or FB Hybrid frame, the LP layer itself still only runs in + WB. + + +
+ +An overview of the decoder is given in . + +
+ +| Range |--->| Decode |---------------------------+ + 1 | Decoder | 2 | Parameters |----------+ 5 | + +---------+ +------------+ 4 | | + 3 | | | + \/ \/ \/ + +------------+ +------------+ +------------+ + | Generate |-->| LTP |-->| LPC | + | Excitation | | Synthesis | | Synthesis | + +------------+ +------------+ +------------+ + ^ | + | | + +-------------------+----------------+ + | 6 + | +------------+ +-------------+ + +-->| Stereo |-->| Sample Rate |--> + | Unmixing | 7 | Conversion | 8 + +------------+ +-------------+ + +1: Range encoded bitstream +2: Coded parameters +3: Pulses, LSBs, and signs +4: Pitch lags, Long-Term Prediction (LTP) coefficients +5: Linear Predictive Coding (LPC) coefficients and gains +6: Decoded signal (mono or mid-side stereo) +7: Unmixed signal (mono or left-right stereo) +8: Resampled signal +]]> + +
+ + +The decoder feeds the bitstream (1) to the range decoder from + , and then decodes the parameters in it (2) + using the procedures detailed in + Sections  + through . +These parameters (3, 4, 5) are used to generate an excitation signal (see + ), which is fed to an optional + long-term prediction (LTP) filter (voiced frames only, see + ) and then a short-term prediction filter + (see ), producing the decoded signal (6). +For stereo streams, the mid-side representation is converted to separate left + and right channels (7). +The result is finally resampled to the desired output sample rate (e.g., + 48 kHz) so that the resampled signal (8) can be mixed with the CELT + layer. + + +
+ +
+ + +Internally, the LP layer of a single Opus frame is composed of either a single + 10 ms regular SILK frame or between one and three 20 ms regular SILK + frames. +A stereo Opus frame may double the number of regular SILK frames (up to a total + of six), since it includes separate frames for a mid channel and, optionally, + a side channel. +Optional Low Bit-Rate Redundancy (LBRR) frames, which are reduced-bitrate + encodings of previous SILK frames, may be included to aid in recovery from + packet loss. +If present, these appear before the regular SILK frames. +They are in most respects identical to regular, active SILK frames, except that + they are usually encoded with a lower bitrate. +This draft uses "SILK frame" to refer to either one and "regular SILK frame" if + it needs to draw a distinction between the two. + + +Logically, each SILK frame is in turn composed of either two or four 5 ms + subframes. +Various parameters, such as the quantization gain of the excitation and the + pitch lag and filter coefficients can vary on a subframe-by-subframe basis. +Physically, the parameters for each subframe are interleaved in the bitstream, + as described in the relevant sections for each parameter. + + +All of these frames and subframes are decoded from the same range coder, with + no padding between them. +Thus packing multiple SILK frames in a single Opus frame saves, on average, + half a byte per SILK frame. +It also allows some parameters to be predicted from prior SILK frames in the + same Opus frame, since this does not degrade packet loss robustness (beyond + any penalty for merely using fewer, larger packets to store multiple frames). + + + +Stereo support in SILK uses a variant of mid-side coding, allowing a mono + decoder to simply decode the mid channel. +However, the data for the two channels is interleaved, so a mono decoder must + still unpack the data for the side channel. +It would be required to do so anyway for Hybrid Opus frames, or to support + decoding individual 20 ms frames. + + + + summarizes the overall grouping of the contents of + the LP layer. +Figures  + and  illustrate + the ordering of the various SILK frames for a 60 ms Opus frame, for both + mono and stereo, respectively. + + + +Symbol(s) +PDF(s) +Condition + +Voice Activity Detection (VAD) flags +{1, 1}/2 + + +LBRR flag +{1, 1}/2 + + +Per-frame LBRR flags + + + +LBRR Frame(s) + + + +Regular SILK Frame(s) + + + + + +
+ +
+ +
+ +
+ +
+ +
+ +The LP layer begins with two to eight header bits, decoded in silk_Decode() + (dec_API.c). +These consist of one Voice Activity Detection (VAD) bit per frame (up to 3), + followed by a single flag indicating the presence of LBRR frames. +For a stereo packet, these first flags correspond to the mid channel, and a + second set of flags is included for the side channel. + + +Because these are the first symbols decoded by the range coder and because they + are coded as binary values with uniform probability, they can be extracted + directly from the most significant bits of the first byte of compressed data. +Thus, a receiver can determine if an Opus frame contains any active SILK frames + without the overhead of using the range decoder. + +
+ +
+ +For Opus frames longer than 20 ms, a set of LBRR flags is + decoded for each channel that has its LBRR flag set. +Each set contains one flag per 20 ms SILK frame. +40 ms Opus frames use the 2-frame LBRR flag PDF from + , and 60 ms Opus frames use the + 3-frame LBRR flag PDF. +For each channel, the resulting 2- or 3-bit integer contains the corresponding + LBRR flag for each frame, packed in order from the LSB to the MSB. + + + +Frame Size +PDF +40 ms {0, 53, 53, 150}/256 +60 ms {0, 41, 20, 29, 41, 15, 28, 82}/256 + + + +A 10 or 20 ms Opus frame does not contain any per-frame LBRR flags, + as there may be at most one LBRR frame per channel. +The global LBRR flag in the header bits (see ) + is already sufficient to indicate the presence of that single LBRR frame. + + +
+ +
+ +The LBRR frames, if present, contain an encoded representation of the signal + immediately prior to the current Opus frame as if it were encoded with the + current mode, frame size, audio bandwidth, and channel count, even if those + differ from the prior Opus frame. +When one of these parameters changes from one Opus frame to the next, this + implies that the LBRR frames of the current Opus frame may not be simple + drop-in replacements for the contents of the previous Opus frame. + + + +For example, when switching from 20 ms to 60 ms, the 60 ms Opus + frame may contain LBRR frames covering up to three prior 20 ms Opus + frames, even if those frames already contained LBRR frames covering some of + the same time periods. +When switching from 20 ms to 10 ms, the 10 ms Opus frame can + contain an LBRR frame covering at most half the prior 20 ms Opus frame, + potentially leaving a hole that needs to be concealed from even a single + packet loss (see ). +When switching from mono to stereo, the LBRR frames in the first stereo Opus + frame MAY contain a non-trivial side channel. + + + +In order to properly produce LBRR frames under all conditions, an encoder might + need to buffer up to 60 ms of audio and re-encode it during these + transitions. +However, the reference implementation opts to disable LBRR frames at the + transition point for simplicity. +Since transitions are relatively infrequent in normal usage, this does not have + a significant impact on packet loss robustness. + + + +The LBRR frames immediately follow the LBRR flags, prior to any regular SILK + frames. + describes their exact contents. +LBRR frames do not include their own separate VAD flags. +LBRR frames are only meant to be transmitted for active speech, thus all LBRR + frames are treated as active. + + + +In a stereo Opus frame longer than 20 ms, although the per-frame LBRR + flags for the mid channel are coded as a unit before the per-frame LBRR flags + for the side channel, the LBRR frames themselves are interleaved. +The decoder parses an LBRR frame for the mid channel of a given 20 ms + interval (if present) and then immediately parses the corresponding LBRR + frame for the side channel (if present), before proceeding to the next + 20 ms interval. + +
+ +
+ +The regular SILK frame(s) follow the LBRR frames (if any). + describes their contents, as well. +Unlike the LBRR frames, a regular SILK frame is coded for each time interval in + an Opus frame, even if the corresponding VAD flags are unset. +For stereo Opus frames longer than 20 ms, the regular mid and side SILK + frames for each 20 ms interval are interleaved, just as with the LBRR + frames. +The side frame may be skipped by coding an appropriate flag, as detailed in + . + +
+ +
+ +Each SILK frame includes a set of side information that encodes + +The frame type and quantization type (), +Quantization gains (), +Short-term prediction filter coefficients (), +A Line Spectral Frequencies (LSF) interpolation weight (), + +Long-term prediction filter lags and gains (), + and + +A linear congruential generator (LCG) seed (). + +The quantized excitation signal (see ) follows + these at the end of the frame. + details the overall organization of a + SILK frame. + + + +Symbol(s) +PDF(s) +Condition + +Stereo Prediction Weights + + + +Mid-only Flag + + + +Frame Type + + + +Subframe Gains + + + +Normalized LSF Stage-1 Index + + + +Normalized LSF Stage-2 Residual + + + +Normalized LSF Interpolation Weight + +20 ms frame + +Primary Pitch Lag + +Voiced frame + +Subframe Pitch Contour + +Voiced frame + +Periodicity Index + +Voiced frame + +LTP Filter + +Voiced frame + +LTP Scaling + + + +LCG Seed + + + +Excitation Rate Level + + + +Excitation Pulse Counts + + + +Excitation Pulse Locations + +Non-zero pulse count + +Excitation LSBs + + + +Excitation Signs + + + + + +
+ +A SILK frame corresponding to the mid channel of a stereo Opus frame begins + with a pair of side channel prediction weights, designed such that zeros + indicate normal mid-side coupling. +Since these weights can change on every frame, the first portion of each frame + linearly interpolates between the previous weights and the current ones, using + zeros for the previous weights if none are available. +These prediction weights are never included in a mono Opus frame, and the + previous weights are reset to zeros on any transition from mono to stereo. +They are also not included in an LBRR frame for the side channel, even if the + LBRR flags indicate the corresponding mid channel was not coded. +In that case, the previous weights are used, again substituting in zeros if no + previous weights are available since the last decoder reset + (see ). + + + +To summarize, these weights are coded if and only if + +This is a stereo Opus frame (), and +The current SILK frame corresponds to the mid channel. + + + + +The prediction weights are coded in three separate pieces, which are decoded + by silk_stereo_decode_pred() (decode_stereo_pred.c). +The first piece jointly codes the high-order part of a table index for both + weights. +The second piece codes the low-order part of each table index. +The third piece codes an offset used to linearly interpolate between table + indices. +The details are as follows. + + + +Let n be an index decoded with the 25-element stage-1 PDF in + . +Then let i0 and i1 be indices decoded with the stage-2 and stage-3 PDFs in + , respectively, and let i2 and i3 + be two more indices decoded with the stage-2 and stage-3 PDFs, all in that + order. + + + +Stage +PDF +Stage 1 +{7, 2, 1, 1, 1, + 10, 24, 8, 1, 1, + 3, 23, 92, 23, 3, + 1, 1, 8, 24, 10, + 1, 1, 1, 2, 7}/256 + +Stage 2 +{85, 86, 85}/256 + +Stage 3 +{51, 51, 52, 51, 51}/256 + + + +Then use n, i0, and i2 to form two table indices, wi0 and wi1, according to +
+ +
+ where the division is integer division. +The range of these indices is 0 to 14, inclusive. +Let w[i] be the i'th weight from . +Then the two prediction weights, w0_Q13 and w1_Q13, are +
+> 16)*(2*i3 + 1) + +w0_Q13 = w_Q13[wi0] + + ((w_Q13[wi0+1] - w_Q13[wi0])*6554) >> 16)*(2*i1 + 1) + - w1_Q13 +]]> +
+N.b., w1_Q13 is computed first here, because w0_Q13 depends on it. +The constant 6554 is approximately 0.1 in Q16. +Although wi0 and wi1 only have 15 possible values, + contains 16 entries to allow + interpolation between entry wi0 and (wi0 + 1) (and likewise for wi1). +
+ + +Index +Weight (Q13) + 0 -13732 + 1 -10050 + 2 -8266 + 3 -7526 + 4 -6500 + 5 -5000 + 6 -2950 + 7 -820 + 8 820 + 9 2950 +10 5000 +11 6500 +12 7526 +13 8266 +14 10050 +15 13732 + + +
+ +
+ +A flag appears after the stereo prediction weights that indicates if only the + mid channel is coded for this time interval. +It appears only when + +This is a stereo Opus frame (see ), +The current SILK frame corresponds to the mid channel, and +Either + +This is a regular SILK frame where the VAD flags + (see ) indicate that the corresponding side + channel is not active. + +This is an LBRR frame where the LBRR flags + (see and ) + indicate that the corresponding side channel is not coded. + + + + +It is omitted when there are no stereo weights, for all of the same reasons. +It is also omitted for a regular SILK frame when the VAD flag of the + corresponding side channel frame is set (indicating it is active). +The side channel must be coded in this case, making the mid-only flag + redundant. +It is also omitted for an LBRR frame when the corresponding LBRR flags + indicate the side channel is coded. + + + +When the flag is present, the decoder reads a single value using the PDF in + , as implemented in + silk_stereo_decode_mid_only() (decode_stereo_pred.c). +If the flag is set, then there is no corresponding SILK frame for the side + channel, the entire decoding process for the side channel is skipped, and + zeros are fed to the stereo unmixing process (see + ) instead. +As stated above, LBRR frames still include this flag when the LBRR flag + indicates that the side channel is not coded. +In that case, if this flag is zero (indicating that there should be a side + channel), then Packet Loss Concealment (PLC, see + ) SHOULD be invoked to recover a + side channel signal. +Otherwise, the stereo image will collapse. + + + +PDF +{192, 64}/256 + + +
+ +
+ +Each SILK frame contains a single "frame type" symbol that jointly codes the + signal type and quantization offset type of the corresponding frame. +If the current frame is a regular SILK frame whose VAD bit was not set (an + "inactive" frame), then the frame type symbol takes on a value of either 0 or + 1 and is decoded using the first PDF in . +If the frame is an LBRR frame or a regular SILK frame whose VAD flag was set + (an "active" frame), then the value of the symbol may range from 2 to 5, + inclusive, and is decoded using the second PDF in + . + translates between the value of the + frame type symbol and the corresponding signal type and quantization offset + type. + + + +VAD Flag +PDF +Inactive {26, 230, 0, 0, 0, 0}/256 +Active {0, 0, 24, 74, 148, 10}/256 + + + +Frame Type +Signal Type +Quantization Offset Type +0 Inactive Low +1 Inactive High +2 Unvoiced Low +3 Unvoiced High +4 Voiced Low +5 Voiced High + + +
+ +
+ +A separate quantization gain is coded for each 5 ms subframe. +These gains control the step size between quantization levels of the excitation + signal and, therefore, the quality of the reconstruction. +They are independent of and unrelated to the pitch contours coded for voiced + frames. +The quantization gains are themselves uniformly quantized to 6 bits on a + log scale, giving them a resolution of approximately 1.369 dB and a range + of approximately 1.94 dB to 88.21 dB. + + +The subframe gains are either coded independently, or relative to the gain from + the most recent coded subframe in the same channel. +Independent coding is used if and only if + + +This is the first subframe in the current SILK frame, and + +Either + + +This is the first SILK frame of its type (LBRR or regular) for this channel in + the current Opus frame, or + + +The previous SILK frame of the same type (LBRR or regular) for this channel in + the same Opus frame was not coded. + + + + + + + +In an independently coded subframe gain, the 3 most significant bits of the + quantization gain are decoded using a PDF selected from + based on the decoded signal + type (see ). + + + +Signal Type +PDF +Inactive {32, 112, 68, 29, 12, 1, 1, 1}/256 +Unvoiced {2, 17, 45, 60, 62, 47, 19, 4}/256 +Voiced {1, 3, 26, 71, 94, 50, 9, 2}/256 + + + +The 3 least significant bits are decoded using a uniform PDF: + + +PDF +{32, 32, 32, 32, 32, 32, 32, 32}/256 + + + +These 6 bits are combined to form a value, gain_index, between 0 and 63. +When the gain for the previous subframe is available, then the current gain is + limited as follows: +
+ +
+This may help some implementations limit the change in precision of their + internal LTP history. +The indices which this clamp applies to cannot simply be removed from the + codebook, because previous_log_gain will not be available after packet loss. +The clamping is skipped after a decoder reset, and in the side channel if the + previous frame in the side channel was not coded, since there is no value for + previous_log_gain available. +It MAY also be skipped after packet loss. +
+ + +For subframes which do not have an independent gain (including the first + subframe of frames not listed as using independent coding above), the + quantization gain is coded relative to the gain from the previous subframe (in + the same channel). +The PDF in yields a delta_gain_index value + between 0 and 40, inclusive. + + +PDF +{6, 5, 11, 31, 132, 21, 8, 4, + 3, 2, 2, 2, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1}/256 + + +The following formula translates this index into a quantization gain for the + current subframe using the gain from the previous subframe: +
+ +
+
+ +silk_gains_dequant() (gain_quant.c) dequantizes log_gain for the k'th subframe + and converts it into a linear Q16 scale factor via +
+>16) + 2090) +]]> +
+
+ +The function silk_log2lin() (log2lin.c) computes an approximation of + 2**(inLog_Q7/128.0), where inLog_Q7 is its Q7 input. +Let i = inLog_Q7>>7 be the integer part of inLogQ7 and + f = inLog_Q7&127 be the fractional part. +Then +
+>16)+f)*((1<>7) +]]> +
+ yields the approximate exponential. +The final Q16 gain values lies between 81920 and 1686110208, inclusive + (representing scale factors of 1.25 to 25728, respectively). +
+
+ +
+ +A set of normalized Line Spectral Frequency (LSF) coefficients follow the + quantization gains in the bitstream, and represent the Linear Predictive + Coding (LPC) coefficients for the current SILK frame. +Once decoded, the normalized LSFs form an increasing list of Q15 values between + 0 and 1. +These represent the interleaved zeros on the upper half of the unit circle + (between 0 and pi, hence "normalized") in the standard decomposition + of the LPC filter into a symmetric part + and an anti-symmetric part (P and Q in ). +Because of non-linear effects in the decoding process, an implementation SHOULD + match the fixed-point arithmetic described in this section exactly. +An encoder SHOULD also use the same process. + + +The normalized LSFs are coded using a two-stage vector quantizer (VQ) + ( and ). +NB and MB frames use an order-10 predictor, while WB frames use an order-16 + predictor, and thus have different sets of tables. +After reconstructing the normalized LSFs + (), the decoder runs them through a + stabilization process (), interpolates + them between frames (), converts them + back into LPC coefficients (), and then runs + them through further processes to limit the range of the coefficients + () and the gain of the filter + (). +All of this is necessary to ensure the reconstruction process is stable. + + +
+ +The first VQ stage uses a 32-element codebook, coded with one of the PDFs in + , depending on the audio bandwidth and + the signal type of the current SILK frame. +This yields a single index, I1, for the entire frame, which + +Indexes an element in a coarse codebook, +Selects the PDFs for the second stage of the VQ, and +Selects the prediction weights used to remove intra-frame redundancy from + the second stage. + +The actual codebook elements are listed in + and + , but they are not needed until the last + stages of reconstructing the LSF coefficients. + + + +Audio Bandwidth +Signal Type +PDF +NB or MB Inactive or unvoiced + +{44, 34, 30, 19, 21, 12, 11, 3, + 3, 2, 16, 2, 2, 1, 5, 2, + 1, 3, 3, 1, 1, 2, 2, 2, + 3, 1, 9, 9, 2, 7, 2, 1}/256 + +NB or MB Voiced + +{1, 10, 1, 8, 3, 8, 8, 14, +13, 14, 1, 14, 12, 13, 11, 11, +12, 11, 10, 10, 11, 8, 9, 8, + 7, 8, 1, 1, 6, 1, 6, 5}/256 + +WB Inactive or unvoiced + +{31, 21, 3, 17, 1, 8, 17, 4, + 1, 18, 16, 4, 2, 3, 1, 10, + 1, 3, 16, 11, 16, 2, 2, 3, + 2, 11, 1, 4, 9, 8, 7, 3}/256 + +WB Voiced + +{1, 4, 16, 5, 18, 11, 5, 14, +15, 1, 3, 12, 13, 14, 14, 6, +14, 12, 2, 6, 1, 12, 12, 11, +10, 3, 10, 5, 1, 1, 1, 3}/256 + + + +
+ +
+ +A total of 16 PDFs are available for the LSF residual in the second stage: the + 8 (a...h) for NB and MB frames given in + , and the 8 (i...p) for WB frames + given in . +Which PDF is used for which coefficient is driven by the index, I1, + decoded in the first stage. + lists the letter of the + corresponding PDF for each normalized LSF coefficient for NB and MB, and + lists the same information for WB. + + + +Codebook +PDF +a {1, 1, 1, 15, 224, 11, 1, 1, 1}/256 +b {1, 1, 2, 34, 183, 32, 1, 1, 1}/256 +c {1, 1, 4, 42, 149, 55, 2, 1, 1}/256 +d {1, 1, 8, 52, 123, 61, 8, 1, 1}/256 +e {1, 3, 16, 53, 101, 74, 6, 1, 1}/256 +f {1, 3, 17, 55, 90, 73, 15, 1, 1}/256 +g {1, 7, 24, 53, 74, 67, 26, 3, 1}/256 +h {1, 1, 18, 63, 78, 58, 30, 6, 1}/256 + + + +Codebook +PDF +i {1, 1, 1, 9, 232, 9, 1, 1, 1}/256 +j {1, 1, 2, 28, 186, 35, 1, 1, 1}/256 +k {1, 1, 3, 42, 152, 53, 2, 1, 1}/256 +l {1, 1, 10, 49, 126, 65, 2, 1, 1}/256 +m {1, 4, 19, 48, 100, 77, 5, 1, 1}/256 +n {1, 1, 14, 54, 100, 72, 12, 1, 1}/256 +o {1, 1, 15, 61, 87, 61, 25, 4, 1}/256 +p {1, 7, 21, 50, 77, 81, 17, 1, 1}/256 + + + +I1 +Coefficient + +0 1 2 3 4 5 6 7 8 9 + 0 +a a a a a a a a a a + 1 +b d b c c b c b b b + 2 +c b b b b b b b b b + 3 +b c c c c b c b b b + 4 +c d d d d c c c c c + 5 +a f d d c c c c b b + g +a c c c c c c c c b + 7 +c d g e e e f e f f + 8 +c e f f e f e g e e + 9 +c e e h e f e f f e +10 +e d d d c d c c c c +11 +b f f g e f e f f f +12 +c h e g f f f f f f +13 +c h f f f f f g f e +14 +d d f e e f e f e e +15 +c d d f f e e e e e +16 +c e e g e f e f f f +17 +c f e g f f f e f e +18 +c h e f e f e f f f +19 +c f e g h g f g f e +20 +d g h e g f f g e f +21 +c h g e e e f e f f +22 +e f f e g g f g f e +23 +c f f g f g e g e e +24 +e f f f d h e f f e +25 +c d e f f g e f f e +26 +c d c d d e c d d d +27 +b b c c c c c d c c +28 +e f f g g g f g e f +29 +d f f e e e e d d c +30 +c f d h f f e e f e +31 +e e f e f g f g f e + + + +I1 +Coefficient + +0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 + 0 +i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i + 1 +k  l  l  l  l  l  k  k  k  k  k  j  j  j  i  l + 2 +k  n  n  l  p  m  m  n  k  n  m  n  n  m  l  l + 3 +i  k  j  k  k  j  j  j  j  j  i  i  i  i  i  j + 4 +i  o  n  m  o  m  p  n  m  m  m  n  n  m  m  l + 5 +i  l  n  n  m  l  l  n  l  l  l  l  l  l  k  m + 6 +i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i + 7 +i  k  o  l  p  k  n  l  m  n  n  m  l  l  k  l + 8 +i  o  k  o  o  m  n  m  o  n  m  m  n  l  l  l + 9 +k  j  i  i  i  i  i  i  i  i  i  i  i  i  i  i +10 +i  j  i  i  i  i  i  i  i  i  i  i  i  i  i  j +11 +k  k  l  m  n  l  l  l  l  l  l  l  k  k  j  l +12 +k  k  l  l  m  l  l  l  l  l  l  l  l  k  j  l +13 +l  m  m  m  o  m  m  n  l  n  m  m  n  m  l  m +14 +i  o  m  n  m  p  n  k  o  n  p  m  m  l  n  l +15 +i  j  i  j  j  j  j  j  j  j  i  i  i  i  j  i +16 +j  o  n  p  n  m  n  l  m  n  m  m  m  l  l  m +17 +j  l  l  m  m  l  l  n  k  l  l  n  n  n  l  m +18 +k  l  l  k  k  k  l  k  j  k  j  k  j  j  j  m +19 +i  k  l  n  l  l  k  k  k  j  j  i  i  i  i  i +20 +l  m  l  n  l  l  k  k  j  j  j  j  j  k  k  m +21 +k  o  l  p  p  m  n  m  n  l  n  l  l  k  l  l +22 +k  l  n  o  o  l  n  l  m  m  l  l  l  l  k  m +23 +j  l  l  m  m  m  m  l  n  n  n  l  j  j  j  j +24 +k  n  l  o  o  m  p  m  m  n  l  m  m  l  l  l +25 +i  o  j  j  i  i  i  i  i  i  i  i  i  i  i  i +26 +i  o  o  l  n  k  n  n  l  m  m  p  p  m  m  m +27 +l  l  p  l  n  m  l  l  l  k  k  l  l  l  k  l +28 +i  i  j  i  i  i  k  j  k  j  j  k  k  k  j  j +29 +i  l  k  n  l  l  k  l  k  j  i  i  j  i  i  j +30 +l  n  n  m  p  n  l  l  k  l  k  k  j  i  j  i +31 +k  l  n  l  m  l  l  l  k  j  k  o  m  i  i  i + + + +Decoding the second stage residual proceeds as follows. +For each coefficient, the decoder reads a symbol using the PDF corresponding to + I1 from either or + , and subtracts 4 from the result + to give an index in the range -4 to 4, inclusive. +If the index is either -4 or 4, it reads a second symbol using the PDF in + , and adds the value of this second symbol + to the index, using the same sign. +This gives the index, I2[k], a total range of -10 to 10, inclusive. + + + +PDF +{156, 60, 24, 9, 4, 2, 1}/256 + + + +The decoded indices from both stages are translated back into normalized LSF + coefficients in silk_NLSF_decode() (NLSF_decode.c). +The stage-2 indices represent residuals after both the first stage of the VQ + and a separate backwards-prediction step. +The backwards prediction process in the encoder subtracts a prediction from + each residual formed by a multiple of the coefficient that follows it. +The decoder must undo this process. + contains lists of prediction weights + for each coefficient. +There are two lists for NB and MB, and another two lists for WB, giving two + possible prediction weights for each coefficient. + + + +Coefficient +A +B +C +D + 0 179 116 175 68 + 1 138 67 148 62 + 2 140 82 160 66 + 3 148 59 176 60 + 4 151 92 178 72 + 5 149 72 173 117 + 6 153 100 174 85 + 7 151 89 164 90 + 8 163 92 177 118 + 9 174 136 +10 196 151 +11 182 142 +12 198 160 +13 192 142 +14 182 155 + + + +The prediction is undone using the procedure implemented in + silk_NLSF_residual_dequant() (NLSF_decode.c), which is as follows. +Each coefficient selects its prediction weight from one of the two lists based + on the stage-1 index, I1. + gives the selections for each + coefficient for NB and MB, and gives + the selections for WB. +Let d_LPC be the order of the codebook, i.e., 10 for NB and MB, and 16 for WB, + and let pred_Q8[k] be the weight for the k'th coefficient selected by this + process for 0 <= k < d_LPC-1. +Then, the stage-2 residual for each coefficient is computed via +
+>8 : 0) + + ((((I2[k]<<10) - sign(I2[k])*102)*qstep)>>16) , +]]> +
+ where qstep is the Q16 quantization step size, which is 11796 for NB and MB + and 9830 for WB (representing step sizes of approximately 0.18 and 0.15, + respectively). +
+ + +I1 +Coefficient + +0 1 2 3 4 5 6 7 8 + 0 +A B A A A A A A A + 1 +B A A A A A A A A + 2 +A A A A A A A A A + 3 +B B B A A A A B A + 4 +A B A A A A A A A + 5 +A B A A A A A A A + 6 +B A B B A A A B A + 7 +A B B A A B B A A + 8 +A A B B A B A B B + 9 +A A B B A A B B B +10 +A A A A A A A A A +11 +A B A B B B B B A +12 +A B A B B B B B A +13 +A B B B B B B B A +14 +B A B B A B B B B +15 +A B B B B B A B A +16 +A A B B A B A B A +17 +A A B B B A B B B +18 +A B B A A B B B A +19 +A A A B B B A B A +20 +A B B A A B A B A +21 +A B B A A A B B A +22 +A A A A A B B B B +23 +A A B B A A A B B +24 +A A A B A B B B B +25 +A B B B B B B B A +26 +A A A A A A A A A +27 +A A A A A A A A A +28 +A A B A B B A B A +29 +B A A B A A A A A +30 +A A A B B A B A B +31 +B A B B A B B B B + + + +I1 +Coefficient + +0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 + 0 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D + 1 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  C + 2 +C  C  D  C  C  D  D  D  C  D  D  D  D  C  C + 3 +C  C  C  C  C  C  C  C  C  C  C  C  D  C  C + 4 +C  D  D  C  D  C  D  D  C  D  D  D  D  D  C + 5 +C  C  D  C  C  C  C  C  C  C  C  C  C  C  C + 6 +D  C  C  C  C  C  C  C  C  C  C  D  C  D  C + 7 +C  D  D  C  C  C  D  C  D  D  D  C  D  C  D + 8 +C  D  C  D  D  C  D  C  D  C  D  D  D  D  D + 9 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D +10 +C  D  C  C  C  C  C  C  C  C  C  C  C  C  C +11 +C  C  D  C  D  D  D  D  D  D  D  C  D  C  C +12 +C  C  D  C  C  D  C  D  C  D  C  C  D  C  C +13 +C  C  C  C  D  D  C  D  C  D  D  D  D  C  C +14 +C  D  C  C  C  D  D  C  D  D  D  C  D  D  D +15 +C  C  D  D  C  C  C  C  C  C  C  C  D  D  C +16 +C  D  D  C  D  C  D  D  D  D  D  C  D  C  C +17 +C  C  D  C  C  C  C  D  C  C  D  D  D  C  C +18 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D +19 +C  C  C  C  C  C  C  C  C  C  C  C  D  C  C +20 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  C +21 +C  D  C  D  C  D  D  C  D  C  D  C  D  D  C +22 +C  C  D  D  D  D  C  D  D  C  C  D  D  C  C +23 +C  D  D  C  D  C  D  C  D  C  C  C  C  D  C +24 +C  C  C  D  D  C  D  C  D  D  D  D  D  D  D +25 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D +26 +C  D  D  C  C  C  D  D  C  C  D  D  D  D  D +27 +C  C  C  C  C  D  C  D  D  D  D  C  D  D  D +28 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D +29 +C  C  C  C  C  C  C  C  C  C  C  C  C  C  D +30 +D  C  C  C  C  C  C  C  C  C  C  D  C  C  C +31 +C  C  D  C  C  D  D  D  C  C  D  C  C  D  C + + +
+ +
+ +Once the stage-1 index I1 and the stage-2 residual res_Q10[] have been decoded, + the final normalized LSF coefficients can be reconstructed. + + +The spectral distortion introduced by the quantization of each LSF coefficient + varies, so the stage-2 residual is weighted accordingly, using the + low-complexity Inverse Harmonic Mean Weighting (IHMW) function proposed in + . +The weights are derived directly from the stage-1 codebook vector. +Let cb1_Q8[k] be the k'th entry of the stage-1 codebook vector from + or + . +Then for 0 <= k < d_LPC the following expression + computes the square of the weight as a Q18 value: +
+ + + +
+ where cb1_Q8[-1] = 0 and cb1_Q8[d_LPC] = 256, and the + division is integer division. +This is reduced to an unsquared, Q9 value using the following square-root + approximation: +
+>(i-8)) & 127 +y = ((i&1) ? 32768 : 46214) >> ((32-i)>>1) +w_Q9[k] = y + ((213*f*y)>>16) +]]> +
+The constant 46214 here is approximately the square root of 2 in Q15. +The cb1_Q8[] vector completely determines these weights, and they may be + tabulated and stored as 13-bit unsigned values (with a range of 1819 to 5227, + inclusive) to avoid computing them when decoding. +The reference implementation already requires code to compute these weights on + unquantized coefficients in the encoder, in silk_NLSF_VQ_weights_laroia() + (NLSF_VQ_weights_laroia.c) and its callers, so it reuses that code in the + decoder instead of using a pre-computed table to reduce the amount of ROM + required. +
+ + +I1 +Codebook (Q8) + + 0   1   2   3   4   5   6   7   8   9 +0 +12  35  60  83 108 132 157 180 206 228 +1 +15  32  55  77 101 125 151 175 201 225 +2 +19  42  66  89 114 137 162 184 209 230 +3 +12  25  50  72  97 120 147 172 200 223 +4 +26  44  69  90 114 135 159 180 205 225 +5 +13  22  53  80 106 130 156 180 205 228 +6 +15  25  44  64  90 115 142 168 196 222 +7 +19  24  62  82 100 120 145 168 190 214 +8 +22  31  50  79 103 120 151 170 203 227 +9 +21  29  45  65 106 124 150 171 196 224 +10 +30  49  75  97 121 142 165 186 209 229 +11 +19  25  52  70  93 116 143 166 192 219 +12 +26  34  62  75  97 118 145 167 194 217 +13 +25  33  56  70  91 113 143 165 196 223 +14 +21  34  51  72  97 117 145 171 196 222 +15 +20  29  50  67  90 117 144 168 197 221 +16 +22  31  48  66  95 117 146 168 196 222 +17 +24  33  51  77 116 134 158 180 200 224 +18 +21  28  70  87 106 124 149 170 194 217 +19 +26  33  53  64  83 117 152 173 204 225 +20 +27  34  65  95 108 129 155 174 210 225 +21 +20  26  72  99 113 131 154 176 200 219 +22 +34  43  61  78  93 114 155 177 205 229 +23 +23  29  54  97 124 138 163 179 209 229 +24 +30  38  56  89 118 129 158 178 200 231 +25 +21  29  49  63  85 111 142 163 193 222 +26 +27  48  77 103 133 158 179 196 215 232 +27 +29  47  74  99 124 151 176 198 220 237 +28 +33  42  61  76  93 121 155 174 207 225 +29 +29  53  87 112 136 154 170 188 208 227 +30 +24  30  52  84 131 150 166 186 203 229 +31 +37  48  64  84 104 118 156 177 201 230 + + + +I1 +Codebook (Q8) + + 0  1  2  3  4   5   6   7   8   9  10  11  12  13  14  15 +0 + 7 23 38 54 69  85 100 116 131 147 162 178 193 208 223 239 +1 +13 25 41 55 69  83  98 112 127 142 157 171 187 203 220 236 +2 +15 21 34 51 61  78  92 106 126 136 152 167 185 205 225 240 +3 +10 21 36 50 63  79  95 110 126 141 157 173 189 205 221 237 +4 +17 20 37 51 59  78  89 107 123 134 150 164 184 205 224 240 +5 +10 15 32 51 67  81  96 112 129 142 158 173 189 204 220 236 +6 + 8 21 37 51 65  79  98 113 126 138 155 168 179 192 209 218 +7 +12 15 34 55 63  78  87 108 118 131 148 167 185 203 219 236 +8 +16 19 32 36 56  79  91 108 118 136 154 171 186 204 220 237 +9 +11 28 43 58 74  89 105 120 135 150 165 180 196 211 226 241 +10 + 6 16 33 46 60  75  92 107 123 137 156 169 185 199 214 225 +11 +11 19 30 44 57  74  89 105 121 135 152 169 186 202 218 234 +12 +12 19 29 46 57  71  88 100 120 132 148 165 182 199 216 233 +13 +17 23 35 46 56  77  92 106 123 134 152 167 185 204 222 237 +14 +14 17 45 53 63  75  89 107 115 132 151 171 188 206 221 240 +15 + 9 16 29 40 56  71  88 103 119 137 154 171 189 205 222 237 +16 +16 19 36 48 57  76  87 105 118 132 150 167 185 202 218 236 +17 +12 17 29 54 71  81  94 104 126 136 149 164 182 201 221 237 +18 +15 28 47 62 79  97 115 129 142 155 168 180 194 208 223 238 +19 + 8 14 30 45 62  78  94 111 127 143 159 175 192 207 223 239 +20 +17 30 49 62 79  92 107 119 132 145 160 174 190 204 220 235 +21 +14 19 36 45 61  76  91 108 121 138 154 172 189 205 222 238 +22 +12 18 31 45 60  76  91 107 123 138 154 171 187 204 221 236 +23 +13 17 31 43 53  70  83 103 114 131 149 167 185 203 220 237 +24 +17 22 35 42 58  78  93 110 125 139 155 170 188 206 224 240 +25 + 8 15 34 50 67  83  99 115 131 146 162 178 193 209 224 239 +26 +13 16 41 66 73  86  95 111 128 137 150 163 183 206 225 241 +27 +17 25 37 52 63  75  92 102 119 132 144 160 175 191 212 231 +28 +19 31 49 65 83 100 117 133 147 161 174 187 200 213 227 242 +29 +18 31 52 68 88 103 117 126 138 149 163 177 192 207 223 239 +30 +16 29 47 61 76  90 106 119 133 147 161 176 193 209 224 240 +31 +15 21 35 50 61  73  86  97 110 119 129 141 175 198 218 237 + + + +Given the stage-1 codebook entry cb1_Q8[], the stage-2 residual res_Q10[], and + their corresponding weights, w_Q9[], the reconstructed normalized LSF + coefficients are +
+ +
+ where the division is integer division. +However, nothing in either the reconstruction process or the + quantization process in the encoder thus far guarantees that the coefficients + are monotonically increasing and separated well enough to ensure a stable + filter . +When using the reference encoder, roughly 2% of frames violate this constraint. +The next section describes a stabilization procedure used to make these + guarantees. +
+ +
+ +
+ +The normalized LSF stabilization procedure is implemented in + silk_NLSF_stabilize() (NLSF_stabilize.c). +This process ensures that consecutive values of the normalized LSF + coefficients, NLSF_Q15[], are spaced some minimum distance apart + (predetermined to be the 0.01 percentile of a large training set). + gives the minimum spacings for NB and MB + and those for WB, where row k is the minimum allowed value of + NLSF_Q[k]-NLSF_Q[k-1]. +For the purposes of computing this spacing for the first and last coefficient, + NLSF_Q15[-1] is taken to be 0, and NLSF_Q15[d_LPC] is taken to be 32768. + + + +Coefficient +NB and MB +WB + 0 250 100 + 1 3 3 + 2 6 40 + 3 3 3 + 4 3 3 + 5 3 3 + 6 4 5 + 7 3 14 + 8 3 14 + 9 3 10 +10 461 11 +11 3 +12 8 +13 9 +14 7 +15 3 +16 347 + + + +The procedure starts off by trying to make small adjustments which attempt to + minimize the amount of distortion introduced. +After 20 such adjustments, it falls back to a more direct method which + guarantees the constraints are enforced but may require large adjustments. + + +Let NDeltaMin_Q15[k] be the minimum required spacing for the current audio + bandwidth from . +First, the procedure finds the index i where + NLSF_Q15[i] - NLSF_Q15[i-1] - NDeltaMin_Q15[i] is the + smallest, breaking ties by using the lower value of i. +If this value is non-negative, then the stabilization stops; the coefficients + satisfy all the constraints. +Otherwise, if i == 0, it sets NLSF_Q15[0] to NDeltaMin_Q15[0], and if + i == d_LPC, it sets NLSF_Q15[d_LPC-1] to + (32768 - NDeltaMin_Q15[d_LPC]). +For all other values of i, both NLSF_Q15[i-1] and NLSF_Q15[i] are updated as + follows: +
+>1) + \ NDeltaMin_Q15[k] + /_ + k=0 + d_LPC + __ + max_center_Q15 = 32768 - (NDeltaMin_Q15[i]>>1) - \ NDeltaMin_Q15[k] + /_ + k=i+1 +center_freq_Q15 = clamp(min_center_Q15[i], + (NLSF_Q15[i-1] + NLSF_Q15[i] + 1)>>1, + max_center_Q15[i]) + + NLSF_Q15[i-1] = center_freq_Q15 - (NDeltaMin_Q15[i]>>1) + + NLSF_Q15[i] = NLSF_Q15[i-1] + NDeltaMin_Q15[i] . +]]> +
+Then the procedure repeats again, until it has either executed 20 times or + has stopped because the coefficients satisfy all the constraints. +
+ +After the 20th repetition of the above procedure, the following fallback + procedure executes once. +First, the values of NLSF_Q15[k] for 0 <= k < d_LPC + are sorted in ascending order. +Then for each value of k from 0 to d_LPC-1, NLSF_Q15[k] is set to +
+ +
+Next, for each value of k from d_LPC-1 down to 0, NLSF_Q15[k] is set to +
+ +
+
+ +
+ +
+ +For 20 ms SILK frames, the first half of the frame (i.e., the first two + subframes) may use normalized LSF coefficients that are interpolated between + the decoded LSFs for the most recent coded frame (in the same channel) and the + current frame. +A Q2 interpolation factor follows the LSF coefficient indices in the bitstream, + which is decoded using the PDF in . +This happens in silk_decode_indices() (decode_indices.c). +After either + +An uncoded regular SILK frame in the side channel, or +A decoder reset (see ), + + the decoder still decodes this factor, but ignores its value and always uses + 4 instead. +For 10 ms SILK frames, this factor is not stored at all. + + + +PDF +{13, 22, 29, 11, 181}/256 + + + +Let n2_Q15[k] be the normalized LSF coefficients decoded by the procedure in + , n0_Q15[k] be the LSF coefficients + decoded for the prior frame, and w_Q2 be the interpolation factor. +Then the normalized LSF coefficients used for the first half of a 20 ms + frame, n1_Q15[k], are +
+> 2) . +]]> +
+This interpolation is performed in silk_decode_parameters() + (decode_parameters.c). +
+
+ +
+ +Any LPC filter A(z) can be split into a symmetric part P(z) and an + anti-symmetric part Q(z) such that +
+ +
+with +
+ +
+The even normalized LSF coefficients correspond to a pair of conjugate roots of + P(z), while the odd coefficients correspond to a pair of conjugate roots of + Q(z), all of which lie on the unit circle. +In addition, P(z) has a root at pi and Q(z) has a root at 0. +Thus, they may be reconstructed mathematically from a set of normalized LSF + coefficients, n[k], as +
+ +
+
+ +However, SILK performs this reconstruction using a fixed-point approximation so + that all decoders can reproduce it in a bit-exact manner to avoid prediction + drift. +The function silk_NLSF2A() (NLSF2A.c) implements this procedure. + + +To start, it approximates cos(pi*n[k]) using a table lookup with linear + interpolation. +The encoder SHOULD use the inverse of this piecewise linear approximation, + rather than the true inverse of the cosine function, when deriving the + normalized LSF coefficients. +These values are also re-ordered to improve numerical accuracy when + constructing the LPC polynomials. + + + +Coefficient +NB and MB +WB + 0 0 0 + 1 9 15 + 2 6 8 + 3 3 7 + 4 4 4 + 5 5 11 + 6 8 12 + 7 1 3 + 8 2 2 + 9 7 13 +10 10 +11 5 +12 6 +13 9 +14 14 +15 1 + + + +The top 7 bits of each normalized LSF coefficient index a value in the table, + and the next 8 bits interpolate between it and the next value. +Let i = (n[k] >> 8) be the integer index and + f = (n[k] & 255) be the fractional part of a given + coefficient. +Then the re-ordered, approximated cosine, c_Q17[ordering[k]], is +
+> 3 , +]]> +
+ where ordering[k] is the k'th entry of the column of + corresponding to the current audio + bandwidth and cos_Q12[i] is the i'th entry of . +
+ + +i ++0 ++1 ++2 ++3 +0 + 4096 4095 4091 4085 +4 + 4076 4065 4052 4036 +8 + 4017 3997 3973 3948 +12 + 3920 3889 3857 3822 +16 + 3784 3745 3703 3659 +20 + 3613 3564 3513 3461 +24 + 3406 3349 3290 3229 +28 + 3166 3102 3035 2967 +32 + 2896 2824 2751 2676 +36 + 2599 2520 2440 2359 +40 + 2276 2191 2106 2019 +44 + 1931 1842 1751 1660 +48 + 1568 1474 1380 1285 +52 + 1189 1093 995 897 +56 + 799 700 601 501 +60 + 401 301 201 101 +64 + 0 -101 -201 -301 +68 + -401 -501 -601 -700 +72 + -799 -897 -995 -1093 +76 +-1189-1285-1380-1474 +80 +-1568-1660-1751-1842 +84 +-1931-2019-2106-2191 +88 +-2276-2359-2440-2520 +92 +-2599-2676-2751-2824 +96 +-2896-2967-3035-3102 +100 +-3166-3229-3290-3349 +104 +-3406-3461-3513-3564 +108 +-3613-3659-3703-3745 +112 +-3784-3822-3857-3889 +116 +-3920-3948-3973-3997 +120 +-4017-4036-4052-4065 +124 +-4076-4085-4091-4095 +128 +-4096 + + + +Given the list of cosine values, silk_NLSF2A_find_poly() (NLSF2A.c) + computes the coefficients of P and Q, described here via a simple recurrence. +Let p_Q16[k][j] and q_Q16[k][j] be the coefficients of the products of the + first (k+1) root pairs for P and Q, with j indexing the coefficient number. +Only the first (k+2) coefficients are needed, as the products are symmetric. +Let p_Q16[0][0] = q_Q16[0][0] = 1<<16, + p_Q16[0][1] = -c_Q17[0], q_Q16[0][1] = -c_Q17[1], and + d2 = d_LPC/2. +As boundary conditions, assume + p_Q16[k][j] = q_Q16[k][j] = 0 for all + j < 0. +Also, assume p_Q16[k][k+2] = p_Q16[k][k] and + q_Q16[k][k+2] = q_Q16[k][k] (because of the symmetry). +Then, for 0 < k < d2 and 0 <= j <= k+1, +
+>16) , + +q_Q16[k][j] = q_Q16[k-1][j] + q_Q16[k-1][j-2] + - ((c_Q17[2*k+1]*q_Q16[k-1][j-1] + 32768)>>16) . +]]> +
+The use of Q17 values for the cosine terms in an otherwise Q16 expression + implicitly scales them by a factor of 2. +The multiplications in this recurrence may require up to 48 bits of precision + in the result to avoid overflow. +In practice, each row of the recurrence only depends on the previous row, so an + implementation does not need to store all of them. +
+ +silk_NLSF2A() uses the values from the last row of this recurrence to + reconstruct a 32-bit version of the LPC filter (without the leading 1.0 + coefficient), a32_Q17[k], 0 <= k < d2: +
+ +
+The sum and difference of two terms from each of the p_Q16 and q_Q16 + coefficient lists reflect the (1 + z**-1) and + (1 - z**-1) factors of P and Q, respectively. +The promotion of the expression from Q16 to Q17 implicitly scales the result + by 1/2. +
+
+ +
+ +The a32_Q17[] coefficients are too large to fit in a 16-bit value, which + significantly increases the cost of applying this filter in fixed-point + decoders. +Reducing them to Q12 precision doesn't incur any significant quality loss, + but still does not guarantee they will fit. +silk_NLSF2A() applies up to 10 rounds of bandwidth expansion to limit + the dynamic range of these coefficients. +Even floating-point decoders SHOULD perform these steps, to avoid mismatch. + + +For each round, the process first finds the index k such that abs(a32_Q17[k]) + is largest, breaking ties by choosing the lowest value of k. +Then, it computes the corresponding Q12 precision value, maxabs_Q12, subject to + an upper bound to avoid overflow in subsequent computations: +
+> 5, 163838) . +]]> +
+If this is larger than 32767, the procedure derives the chirp factor, + sc_Q16[0], to use in the bandwidth expansion as +
+> 2 +]]> +
+ where the division here is integer division. +This is an approximation of the chirp factor needed to reduce the target + coefficient to 32767, though it is both less than 0.999 and, for + k > 0 when maxabs_Q12 is much greater than 32767, still slightly + too large. +The upper bound on maxabs_Q12, 163838, was chosen because it is equal to + ((2**31 - 1) >> 14) + 32767, i.e., the + largest value of maxabs_Q12 that would not overflow the numerator in the + equation above when stored in a signed 32-bit integer. +
+ +silk_bwexpander_32() (bwexpander_32.c) performs the bandwidth expansion (again, + only when maxabs_Q12 is greater than 32767) using the following recurrence: +
+> 16 + +sc_Q16[k+1] = (sc_Q16[0]*sc_Q16[k] + 32768) >> 16 +]]> +
+The first multiply may require up to 48 bits of precision in the result to + avoid overflow. +The second multiply must be unsigned to avoid overflow with only 32 bits of + precision. +The reference implementation uses a slightly more complex formulation that + avoids the 32-bit overflow using signed multiplication, but is otherwise + equivalent. +
+ +After 10 rounds of bandwidth expansion are performed, they are simply saturated + to 16 bits: +
+> 5, 32767) << 5 . +]]> +
+Because this performs the actual saturation in the Q12 domain, but converts the + coefficients back to the Q17 domain for the purposes of prediction gain + limiting, this step must be performed after the 10th round of bandwidth + expansion, regardless of whether or not the Q12 version of any coefficient + still overflows a 16-bit integer. +This saturation is not performed if maxabs_Q12 drops to 32767 or less prior to + the 10th round. +
+
+ +
+ +The prediction gain of an LPC synthesis filter is the square-root of the output + energy when the filter is excited by a unit-energy impulse. +Even if the Q12 coefficients would fit, the resulting filter may still have a + significant gain (especially for voiced sounds), making the filter unstable. +silk_NLSF2A() applies up to 18 additional rounds of bandwidth expansion to + limit the prediction gain. +Instead of controlling the amount of bandwidth expansion using the prediction + gain itself (which may diverge to infinity for an unstable filter), + silk_NLSF2A() uses silk_LPC_inverse_pred_gain_QA() (LPC_inv_pred_gain.c) to + compute the reflection coefficients associated with the filter. +The filter is stable if and only if the magnitude of these coefficients is + sufficiently less than one. +The reflection coefficients, rc[k], can be computed using a simple Levinson + recurrence, initialized with the LPC coefficients + a[d_LPC-1][n] = a[n], and then updated via +
+ +
+
+ +However, silk_LPC_inverse_pred_gain_QA() approximates this using fixed-point + arithmetic to guarantee reproducible results across platforms and + implementations. +Since small changes in the coefficients can make a stable filter unstable, it + takes the real Q12 coefficients that will be used during reconstruction as + input. +Thus, let +
+> 5 +]]> +
+ be the Q12 version of the LPC coefficients that will eventually be used. +As a simple initial check, the decoder computes the DC response as +
+ +
+ and if DC_resp > 4096, the filter is unstable. +
+ +Increasing the precision of these Q12 coefficients to Q24 for intermediate + computations allows more accurate computation of the reflection coefficients, + so the decoder initializes the recurrence via +
+ +
+Then for each k from d_LPC-1 down to 0, if + abs(a32_Q24[k][k]) > 16773022, the filter is unstable and the + recurrence stops. +The constant 16773022 here is approximately 0.99975 in Q24. +Otherwise, row k-1 of a32_Q24 is computed from row k as +
+> 32) , + + b1[k] = ilog(div_Q30[k]) , + + b2[k] = b1[k] - 16 , + + (1<<29) - 1 + inv_Qb2[k] = ----------------------- , + div_Q30[k] >> (b2[k]+1) + + err_Q29[k] = (1<<29) + - ((div_Q30[k]<<(15-b2[k]))*inv_Qb2[k] >> 16) , + + gain_Qb1[k] = ((inv_Qb2[k] << 16) + + (err_Q29[k]*inv_Qb2[k] >> 13)) , + +num_Q24[k-1][n] = a32_Q24[k][n] + - ((a32_Q24[k][k-n-1]*rc_Q31[k] + (1<<30)) >> 31) , + +a32_Q24[k-1][n] = (num_Q24[k-1][n]*gain_Qb1[k] + + (1<<(b1[k]-1))) >> b1[k] , +]]> +
+ where 0 <= n < k. +Here, rc_Q30[k] are the reflection coefficients. +div_Q30[k] is the denominator for each iteration, and gain_Qb1[k] is its + multiplicative inverse (with b1[k] fractional bits, where b1[k] ranges from + 20 to 31). +inv_Qb2[k], which ranges from 16384 to 32767, is a low-precision version of + that inverse (with b2[k] fractional bits). +err_Q29[k] is the residual error, ranging from -32763 to 32392, which is used + to improve the accuracy. +The values t_Q24[k-1][n] for each n are the numerators for the next row of + coefficients in the recursion, and a32_Q24[k-1][n] is the final version of + that row. +Every multiply in this procedure except the one used to compute gain_Qb1[k] + requires more than 32 bits of precision, but otherwise all intermediate + results fit in 32 bits or less. +In practice, because each row only depends on the next one, an implementation + does not need to store them all. +
+ +If abs(a32_Q24[k][k]) <= 16773022 for + 0 <= k < d_LPC, then the filter is considered stable. +However, the problem of determining stability is ill-conditioned when the + filter contains several reflection coefficients whose magnitude is very close + to one. +This fixed-point algorithm is not mathematically guaranteed to correctly + classify filters as stable or unstable in this case, though it does very well + in practice. + + +On round i, 1 <= i <= 18, if the filter passes these + stability checks, then this procedure stops, and the final LPC coefficients to + use for reconstruction in are +
+> 5 . +]]> +
+Otherwise, a round of bandwidth expansion is applied using the same procedure + as in , with +
+ +
+During the 15th round, sc_Q16[0] becomes 0 in the above equation, so a_Q12[k] + is set to 0 for all k, guaranteeing a stable filter. +
+
+ +
+ +
+ +After the normalized LSF indices and, for 20 ms frames, the LSF + interpolation index, voiced frames (see ) + include additional LTP parameters. +There is one primary lag index for each SILK frame, but this is refined to + produce a separate lag index per subframe using a vector quantizer. +Each subframe also gets its own prediction gain coefficient. + + +
+ +The primary lag index is coded either relative to the primary lag of the prior + frame in the same channel, or as an absolute index. +Absolute coding is used if and only if + + +This is the first SILK frame of its type (LBRR or regular) for this channel in + the current Opus frame, + + +The previous SILK frame of the same type (LBRR or regular) for this channel in + the same Opus frame was not coded, or + + +That previous SILK frame was coded, but was not voiced (see + ). + + + + + +With absolute coding, the primary pitch lag may range from 2 ms + (inclusive) up to 18 ms (exclusive), corresponding to pitches from + 500 Hz down to 55.6 Hz, respectively. +It is comprised of a high part and a low part, where the decoder reads the high + part using the 32-entry codebook in + and the low part using the codebook corresponding to the current audio + bandwidth from . +The final primary pitch lag is then +
+ +
+ where lag_high is the high part, lag_low is the low part, and lag_scale + and lag_min are the values from the "Scale" and "Minimum Lag" columns of + , respectively. +
+ + +PDF +{3, 3, 6, 11, 21, 30, 32, 19, + 11, 10, 12, 13, 13, 12, 11, 9, + 8, 7, 6, 4, 2, 2, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1}/256 + + + +Audio Bandwidth +PDF +Scale +Minimum Lag +Maximum Lag +NB {64, 64, 64, 64}/256 4 16 144 +MB {43, 42, 43, 43, 42, 43}/256 6 24 216 +WB {32, 32, 32, 32, 32, 32, 32, 32}/256 8 32 288 + + + +All frames that do not use absolute coding for the primary lag index use + relative coding instead. +The decoder reads a single delta value using the 21-entry PDF in + . +If the resulting value is zero, it falls back to the absolute coding procedure + from the prior paragraph. +Otherwise, the final primary pitch lag is then +
+ +
+ where previous_lag is the primary pitch lag from the most recent frame in the + same channel and delta_lag_index is the value just decoded. +This allows a per-frame change in the pitch lag of -8 to +11 samples. +The decoder does no clamping at this point, so this value can fall outside the + range of 2 ms to 18 ms, and the decoder must use this unclamped + value when using relative coding in the next SILK frame (if any). +However, because an Opus frame can use relative coding for at most two + consecutive SILK frames, integer overflow should not be an issue. +
+ + +PDF +{46, 2, 2, 3, 4, 6, 10, 15, + 26, 38, 30, 22, 15, 10, 7, 6, + 4, 4, 2, 2, 2}/256 + + + +After the primary pitch lag, a "pitch contour", stored as a single entry from + one of four small VQ codebooks, gives lag offsets for each subframe in the + current SILK frame. +The codebook index is decoded using one of the PDFs in + depending on the current frame size + and audio bandwidth. +Tables  + through  + give the corresponding offsets to apply to the primary pitch lag for each + subframe given the decoded codebook index. + + + +Audio Bandwidth +SILK Frame Size +Codebook Size +PDF +NB 10 ms 3 +{143, 50, 63}/256 +NB 20 ms 11 +{68, 12, 21, 17, 19, 22, 30, 24, + 17, 16, 10}/256 +MB or WB 10 ms 12 +{91, 46, 39, 19, 14, 12, 8, 7, + 6, 5, 5, 4}/256 +MB or WB 20 ms 34 +{33, 22, 18, 16, 15, 14, 14, 13, + 13, 10, 9, 9, 8, 6, 6, 6, + 5, 4, 4, 4, 3, 3, 3, 2, + 2, 2, 2, 2, 2, 2, 1, 1, + 1, 1}/256 + + + +Index +Subframe Offsets +0  0  0 +1  1  0 +2  0  1 + + + +Index +Subframe Offsets + 0  0  0  0  0 + 1  2  1  0 -1 + 2 -1  0  1  2 + 3 -1  0  0  1 + 4 -1  0  0  0 + 5  0  0  0  1 + 6  0  0  1  1 + 7  1  1  0  0 + 8  1  0  0  0 + 9  0  0  0 -1 +10  1  0  0 -1 + + + +Index +Subframe Offsets + 0  0  0 + 1  0  1 + 2  1  0 + 3 -1  1 + 4  1 -1 + 5 -1  2 + 6  2 -1 + 7 -2  2 + 8  2 -2 + 9 -2  3 +10  3 -2 +11 -3  3 + + + +Index +Subframe Offsets + 0  0  0  0  0 + 1  0  0  1  1 + 2  1  1  0  0 + 3 -1  0  0  0 + 4  0  0  0  1 + 5  1  0  0  0 + 6 -1  0  0  1 + 7  0  0  0 -1 + 8 -1  0  1  2 + 9  1  0  0 -1 +10 -2 -1  1  2 +11  2  1  0 -1 +12 -2  0  0  2 +13 -2  0  1  3 +14  2  1 -1 -2 +15 -3 -1  1  3 +16  2  0  0 -2 +17  3  1  0 -2 +18 -3 -1  2  4 +19 -4 -1  1  4 +20  3  1 -1 -3 +21 -4 -1  2  5 +22  4  2 -1 -3 +23  4  1 -1 -4 +24 -5 -1  2  6 +25  5  2 -1 -4 +26 -6 -2  2  6 +27 -5 -2  2  5 +28  6  2 -1 -5 +29 -7 -2  3  8 +30  6  2 -2 -6 +31  5  2 -2 -5 +32  8  3 -2 -7 +33 -9 -3  3  9 + + + +The final pitch lag for each subframe is assembled in silk_decode_pitch() + (decode_pitch.c). +Let lag be the primary pitch lag for the current SILK frame, contour_index be + index of the VQ codebook, and lag_cb[contour_index][k] be the corresponding + entry of the codebook from the appropriate table given above for the k'th + subframe. +Then the final pitch lag for that subframe is +
+ +
+ where lag_min and lag_max are the values from the "Minimum Lag" and + "Maximum Lag" columns of , + respectively. +
+ +
+ +
+ +SILK uses a separate 5-tap pitch filter for each subframe, selected from one + of three codebooks. +The three codebooks each represent different rate-distortion trade-offs, with + average rates of 1.61 bits/subframe, 3.68 bits/subframe, and + 4.85 bits/subframe, respectively. + + + +The importance of the filter coefficients generally depends on two factors: the + periodicity of the signal and relative energy between the current subframe and + the signal from one period earlier. +Greater periodicity and decaying energy both lead to more important filter + coefficients, and thus should be coded with lower distortion and higher rate. +These properties are relatively stable over the duration of a single SILK + frame, hence all of the subframes in a SILK frame choose their filter from the + same codebook. +This is signaled with an explicitly-coded "periodicity index". +This immediately follows the subframe pitch lags, and is coded using the + 3-entry PDF from . + + + +PDF +{77, 80, 99}/256 + + + +The indices of the filters for each subframe follow. +They are all coded using the PDF from + corresponding to the periodicity index. +Tables  + through  + contain the corresponding filter taps as signed Q7 integers. + + + +Periodicity Index +Codebook Size +PDF +0 8 {185, 15, 13, 13, 9, 9, 6, 6}/256 +1 16 {57, 34, 21, 20, 15, 13, 12, 13, + 10, 10, 9, 10, 9, 8, 7, 8}/256 +2 32 {15, 16, 14, 12, 12, 12, 11, 11, + 11, 10, 9, 9, 9, 9, 8, 8, + 8, 8, 7, 7, 6, 6, 5, 4, + 5, 4, 4, 4, 3, 4, 3, 2}/256 + + + +Index +Filter Taps (Q7) + 0 +  4   6  24   7   5 + 1 +  0   0   2   0   0 + 2 + 12  28  41  13  -4 + 3 + -9  15  42  25  14 + 4 +  1  -2  62  41  -9 + 5 +-10  37  65  -4   3 + 6 + -6   4  66   7  -8 + 7 + 16  14  38  -3  33 + + + +Index +Filter Taps (Q7) + + 0 + 13  22  39  23  12 + 1 + -1  36  64  27  -6 + 2 + -7  10  55  43  17 + 3 +  1   1   8   1   1 + 4 +  6 -11  74  53  -9 + 5 +-12  55  76 -12   8 + 6 + -3   3  93  27  -4 + 7 + 26  39  59   3  -8 + 8 +  2   0  77  11   9 + 9 + -8  22  44  -6   7 +10 + 40   9  26   3   9 +11 + -7  20 101  -7   4 +12 +  3  -8  42  26   0 +13 +-15  33  68   2  23 +14 + -2  55  46  -2  15 +15 +  3  -1  21  16  41 + + + +Index +Filter Taps (Q7) + 0 + -6  27  61  39   5 + 1 +-11  42  88   4   1 + 2 + -2  60  65   6  -4 + 3 + -1  -5  73  56   1 + 4 + -9  19  94  29  -9 + 5 +  0  12  99   6   4 + 6 +  8 -19 102  46 -13 + 7 +  3   2  13   3   2 + 8 +  9 -21  84  72 -18 + 9 +-11  46 104 -22   8 +10 + 18  38  48  23   0 +11 +-16  70  83 -21  11 +12 +  5 -11 117  22  -8 +13 + -6  23 117 -12   3 +14 +  3  -8  95  28   4 +15 +-10  15  77  60 -15 +16 + -1   4 124   2  -4 +17 +  3  38  84  24 -25 +18 +  2  13  42  13  31 +19 + 21  -4  56  46  -1 +20 + -1  35  79 -13  19 +21 + -7  65  88  -9 -14 +22 + 20   4  81  49 -29 +23 + 20   0  75   3 -17 +24 +  5  -9  44  92  -8 +25 +  1  -3  22  69  31 +26 + -6  95  41 -12   5 +27 + 39  67  16  -4   1 +28 +  0  -6 120  55 -36 +29 +-13  44 122   4 -24 +30 + 81   5  11   3   7 +31 +  2   0   9  10  88 + + +
+ +
+ +An LTP scaling parameter appears after the LTP filter coefficients if and only + if + +This is a voiced frame (see ), and +Either + + +This SILK frame corresponds to the first time interval of the + current Opus frame for its type (LBRR or regular), or + + +This is an LBRR frame where the LBRR flags (see + ) indicate the previous LBRR frame in the same + channel is not coded. + + + + +This allows the encoder to trade off the prediction gain between + packets against the recovery time after packet loss. +Unlike absolute-coding for pitch lags, regular SILK frames that are not at the + start of an Opus frame (i.e., that do not correspond to the first 20 ms + time interval in Opus frames of 40 or 60 ms) do not include this + field, even if the prior frame was not voiced, or (in the case of the side + channel) not even coded. +After an uncoded frame in the side channel, the LTP buffer (see + ) is cleared to zero, and is thus in a + known state. +In contrast, LBRR frames do include this field when the prior frame was not + coded, since the LTP buffer contains the output of the PLC, which is + non-normative. + + +If present, the decoder reads a value using the 3-entry PDF in + . +The three possible values represent Q14 scale factors of 15565, 12288, and + 8192, respectively (corresponding to approximately 0.95, 0.75, and 0.5). +Frames that do not code the scaling parameter use the default factor of 15565 + (approximately 0.95). + + + +PDF +{128, 64, 64}/256 + + +
+ +
+ +
+ +As described in , SILK uses a + linear congruential generator (LCG) to inject pseudorandom noise into the + quantized excitation. +To ensure synchronization of this process between the encoder and decoder, each + SILK frame stores a 2-bit seed after the LTP parameters (if any). +The encoder may consider the choice of seed during quantization, and the + flexibility of this choice lets it reduce distortion, helping to pay for the + bit cost required to signal it. +The decoder reads the seed using the uniform 4-entry PDF in + , yielding a value between 0 and 3, inclusive. + + + +PDF +{64, 64, 64, 64}/256 + + +
+ +
+ +SILK codes the excitation using a modified version of the Pyramid Vector + Quantization (PVQ) codebook . +The PVQ codebook is designed for Laplace-distributed values and consists of all + sums of K signed, unit pulses in a vector of dimension N, where two pulses at + the same position are required to have the same sign. +Thus the codebook includes all integer codevectors y of dimension N that + satisfy +
+ +
+Unlike regular PVQ, SILK uses a variable-length, rather than fixed-length, + encoding. +This encoding is better suited to the more Gaussian-like distribution of the + coefficient magnitudes and the non-uniform distribution of their signs (caused + by the quantization offset described below). +SILK also handles large codebooks by coding the least significant bits (LSBs) + of each coefficient directly. +This adds a small coding efficiency loss, but greatly reduces the computation + time and ROM size required for decoding, as implemented in + silk_decode_pulses() (decode_pulses.c). +
+ + +SILK fixes the dimension of the codebook to N = 16. +The excitation is made up of a number of "shell blocks", each 16 samples in + size. + lists the number of shell blocks + required for a SILK frame for each possible audio bandwidth and frame size. +10 ms MB frames nominally contain 120 samples (10 ms at + 12 kHz), which is not a multiple of 16. +This is handled by coding 8 shell blocks (128 samples) and discarding the final + 8 samples of the last block. +The decoder contains no special case that prevents an encoder from placing + pulses in these samples, and they must be correctly parsed from the bitstream + if present, but they are otherwise ignored. + + + +Audio Bandwidth +Frame Size +Number of Shell Blocks +NB 10 ms 5 +MB 10 ms 8 +WB 10 ms 10 +NB 20 ms 10 +MB 20 ms 15 +WB 20 ms 20 + + +
+ +The first symbol in the excitation is a "rate level", which is an index from 0 + to 8, inclusive, coded using the PDF in + corresponding to the signal type of the current frame (from + ). +The rate level selects the PDF used to decode the number of pulses in + the individual shell blocks. +It does not directly convey any information about the bitrate or the number of + pulses itself, but merely changes the probability of the symbols in + . +Level 0 provides a more efficient encoding at low rates generally, and + level 8 provides a more efficient encoding at high rates generally, + though the most efficient level for a particular SILK frame may depend on the + exact distribution of the coded symbols. +An encoder should, but is not required to, use the most efficient rate level. + + + +Signal Type +PDF +Inactive or Unvoiced +{15, 51, 12, 46, 45, 13, 33, 27, 14}/256 +Voiced +{33, 30, 36, 17, 34, 49, 18, 21, 18}/256 + + +
+ +
+ +The total number of pulses in each of the shell blocks follows the rate level. +The pulse counts for all of the shell blocks are coded consecutively, before + the content of any of the blocks. +Each block may have anywhere from 0 to 16 pulses, inclusive, coded using the + 18-entry PDF in corresponding to the + rate level from . +The special value 17 indicates that this block has one or more additional + LSBs to decode for each coefficient. +If the decoder encounters this value, it decodes another value for the actual + pulse count of the block, but uses the PDF corresponding to the special rate + level 9 instead of the normal rate level. +This process repeats until the decoder reads a value less than 17, and it then + sets the number of extra LSBs used to the number of 17's decoded for that + block. +If it reads the value 17 ten times, then the next iteration uses the special + rate level 10 instead of 9. +The probability of decoding a 17 when using the PDF for rate level 10 is + zero, ensuring that the number of LSBs for a block will not exceed 10. +The cumulative distribution for rate level 10 is just a shifted version of + that for 9 and thus does not require any additional storage. + + + +Rate Level +PDF +0 +{131, 74, 25, 8, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}/256 +1 +{58, 93, 60, 23, 7, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}/256 +2 +{43, 51, 46, 33, 24, 16, 11, 8, 6, 3, 3, 3, 2, 1, 1, 2, 1, 2}/256 +3 +{17, 52, 71, 57, 31, 12, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}/256 +4 +{6, 21, 41, 53, 49, 35, 21, 11, 6, 3, 2, 2, 1, 1, 1, 1, 1, 1}/256 +5 +{7, 14, 22, 28, 29, 28, 25, 20, 17, 13, 11, 9, 7, 5, 4, 4, 3, 10}/256 +6 +{2, 5, 14, 29, 42, 46, 41, 31, 19, 11, 6, 3, 2, 1, 1, 1, 1, 1}/256 +7 +{1, 2, 4, 10, 19, 29, 35, 37, 34, 28, 20, 14, 8, 5, 4, 2, 2, 2}/256 +8 +{1, 2, 2, 5, 9, 14, 20, 24, 27, 28, 26, 23, 20, 15, 11, 8, 6, 15}/256 +9 +{1, 1, 1, 6, 27, 58, 56, 39, 25, 14, 10, 6, 3, 3, 2, 1, 1, 2}/256 +10 +{2, 1, 6, 27, 58, 56, 39, 25, 14, 10, 6, 3, 3, 2, 1, 1, 2, 0}/256 + + +
+ +
+ +The locations of the pulses in each shell block follow the pulse counts, + as decoded by silk_shell_decoder() (shell_coder.c). +As with the pulse counts, these locations are coded for all the shell blocks + before any of the remaining information for each block. +Unlike many other codecs, SILK places no restriction on the distribution of + pulses within a shell block. +All of the pulses may be placed in a single location, or each one in a unique + location, or anything in between. + + + +The location of pulses is coded by recursively partitioning each block into + halves, and coding how many pulses fall on the left side of the split. +All remaining pulses must fall on the right side of the split. +The process then recurses into the left half, and after that returns, the + right half (preorder traversal). +The PDF to use is chosen by the size of the current partition (16, 8, 4, or 2) + and the number of pulses in the partition (1 to 16, inclusive). +Tables  + through  list the + PDFs used for each partition size and pulse count. +This process skips partitions without any pulses, i.e., where the initial pulse + count from was zero, or where the split in + the prior level indicated that all of the pulses fell on the other side. +These partitions have nothing to code, so they require no PDF. + + + +Pulse Count +PDF + 1 {126, 130}/256 + 2 {56, 142, 58}/256 + 3 {25, 101, 104, 26}/256 + 4 {12, 60, 108, 64, 12}/256 + 5 {7, 35, 84, 87, 37, 6}/256 + 6 {4, 20, 59, 86, 63, 21, 3}/256 + 7 {3, 12, 38, 72, 75, 42, 12, 2}/256 + 8 {2, 8, 25, 54, 73, 59, 27, 7, 1}/256 + 9 {2, 5, 17, 39, 63, 65, 42, 18, 4, 1}/256 +10 {1, 4, 12, 28, 49, 63, 54, 30, 11, 3, 1}/256 +11 {1, 4, 8, 20, 37, 55, 57, 41, 22, 8, 2, 1}/256 +12 {1, 3, 7, 15, 28, 44, 53, 48, 33, 16, 6, 1, 1}/256 +13 {1, 2, 6, 12, 21, 35, 47, 48, 40, 25, 12, 5, 1, 1}/256 +14 {1, 1, 4, 10, 17, 27, 37, 47, 43, 33, 21, 9, 4, 1, 1}/256 +15 {1, 1, 1, 8, 14, 22, 33, 40, 43, 38, 28, 16, 8, 1, 1, 1}/256 +16 {1, 1, 1, 1, 13, 18, 27, 36, 41, 41, 34, 24, 14, 1, 1, 1, 1}/256 + + + +Pulse Count +PDF + 1 {127, 129}/256 + 2 {53, 149, 54}/256 + 3 {22, 105, 106, 23}/256 + 4 {11, 61, 111, 63, 10}/256 + 5 {6, 35, 86, 88, 36, 5}/256 + 6 {4, 20, 59, 87, 62, 21, 3}/256 + 7 {3, 13, 40, 71, 73, 41, 13, 2}/256 + 8 {3, 9, 27, 53, 70, 56, 28, 9, 1}/256 + 9 {3, 8, 19, 37, 57, 61, 44, 20, 6, 1}/256 +10 {3, 7, 15, 28, 44, 54, 49, 33, 17, 5, 1}/256 +11 {1, 7, 13, 22, 34, 46, 48, 38, 28, 14, 4, 1}/256 +12 {1, 1, 11, 22, 27, 35, 42, 47, 33, 25, 10, 1, 1}/256 +13 {1, 1, 6, 14, 26, 37, 43, 43, 37, 26, 14, 6, 1, 1}/256 +14 {1, 1, 4, 10, 20, 31, 40, 42, 40, 31, 20, 10, 4, 1, 1}/256 +15 {1, 1, 3, 8, 16, 26, 35, 38, 38, 35, 26, 16, 8, 3, 1, 1}/256 +16 {1, 1, 2, 6, 12, 21, 30, 36, 38, 36, 30, 21, 12, 6, 2, 1, 1}/256 + + + +Pulse Count +PDF + 1 {127, 129}/256 + 2 {49, 157, 50}/256 + 3 {20, 107, 109, 20}/256 + 4 {11, 60, 113, 62, 10}/256 + 5 {7, 36, 84, 87, 36, 6}/256 + 6 {6, 24, 57, 82, 60, 23, 4}/256 + 7 {5, 18, 39, 64, 68, 42, 16, 4}/256 + 8 {6, 14, 29, 47, 61, 52, 30, 14, 3}/256 + 9 {1, 15, 23, 35, 51, 50, 40, 30, 10, 1}/256 +10 {1, 1, 21, 32, 42, 52, 46, 41, 18, 1, 1}/256 +11 {1, 6, 16, 27, 36, 42, 42, 36, 27, 16, 6, 1}/256 +12 {1, 5, 12, 21, 31, 38, 40, 38, 31, 21, 12, 5, 1}/256 +13 {1, 3, 9, 17, 26, 34, 38, 38, 34, 26, 17, 9, 3, 1}/256 +14 {1, 3, 7, 14, 22, 29, 34, 36, 34, 29, 22, 14, 7, 3, 1}/256 +15 {1, 2, 5, 11, 18, 25, 31, 35, 35, 31, 25, 18, 11, 5, 2, 1}/256 +16 {1, 1, 4, 9, 15, 21, 28, 32, 34, 32, 28, 21, 15, 9, 4, 1, 1}/256 + + + +Pulse Count +PDF + 1 {128, 128}/256 + 2 {42, 172, 42}/256 + 3 {21, 107, 107, 21}/256 + 4 {12, 60, 112, 61, 11}/256 + 5 {8, 34, 86, 86, 35, 7}/256 + 6 {8, 23, 55, 90, 55, 20, 5}/256 + 7 {5, 15, 38, 72, 72, 36, 15, 3}/256 + 8 {6, 12, 27, 52, 77, 47, 20, 10, 5}/256 + 9 {6, 19, 28, 35, 40, 40, 35, 28, 19, 6}/256 +10 {4, 14, 22, 31, 37, 40, 37, 31, 22, 14, 4}/256 +11 {3, 10, 18, 26, 33, 38, 38, 33, 26, 18, 10, 3}/256 +12 {2, 8, 13, 21, 29, 36, 38, 36, 29, 21, 13, 8, 2}/256 +13 {1, 5, 10, 17, 25, 32, 38, 38, 32, 25, 17, 10, 5, 1}/256 +14 {1, 4, 7, 13, 21, 29, 35, 36, 35, 29, 21, 13, 7, 4, 1}/256 +15 {1, 2, 5, 10, 17, 25, 32, 36, 36, 32, 25, 17, 10, 5, 2, 1}/256 +16 {1, 2, 4, 7, 13, 21, 28, 34, 36, 34, 28, 21, 13, 7, 4, 2, 1}/256 + + +
+ +
+ +After the decoder reads the pulse locations for all blocks, it reads the LSBs + (if any) for each block in turn. +Inside each block, it reads all the LSBs for each coefficient in turn, even + those where no pulses were allocated, before proceeding to the next one. +For 10 ms MB frames, it reads LSBs even for the extra 8 samples in + the last block. +The LSBs are coded from most significant to least significant, and they all use + the PDF in . + + + +PDF +{136, 120}/256 + + + +The number of LSBs read for each coefficient in a block is determined in + . +The magnitude of the coefficient is initially equal to the number of pulses + placed at that location in . +As each LSB is decoded, the magnitude is doubled, and then the value of the LSB + added to it, to obtain an updated magnitude. + +
+ +
+ +After decoding the pulse locations and the LSBs, the decoder knows the + magnitude of each coefficient in the excitation. +It then decodes a sign for all coefficients with a non-zero magnitude, using + one of the PDFs from . +If the value decoded is 0, then the coefficient magnitude is negated. +Otherwise, it remains positive. + + + +The decoder chooses the PDF for the sign based on the signal type and + quantization offset type (from ) and the + number of pulses in the block (from ). +The number of pulses in the block does not take into account any LSBs. +Most PDFs are skewed towards negative signs because of the quantization offset, + but the PDFs for zero pulses are highly skewed towards positive signs. +If a block contains many positive coefficients, it is sometimes beneficial to + code it solely using LSBs (i.e., with zero pulses), since the encoder may be + able to save enough bits on the signs to justify the less efficient + coefficient magnitude encoding. + + + +Signal Type +Quantization Offset Type +Pulse Count +PDF +Inactive Low 0 {2, 254}/256 +Inactive Low 1 {207, 49}/256 +Inactive Low 2 {189, 67}/256 +Inactive Low 3 {179, 77}/256 +Inactive Low 4 {174, 82}/256 +Inactive Low 5 {163, 93}/256 +Inactive Low 6 or more {157, 99}/256 +Inactive High 0 {58, 198}/256 +Inactive High 1 {245, 11}/256 +Inactive High 2 {238, 18}/256 +Inactive High 3 {232, 24}/256 +Inactive High 4 {225, 31}/256 +Inactive High 5 {220, 36}/256 +Inactive High 6 or more {211, 45}/256 +Unvoiced Low 0 {1, 255}/256 +Unvoiced Low 1 {210, 46}/256 +Unvoiced Low 2 {190, 66}/256 +Unvoiced Low 3 {178, 78}/256 +Unvoiced Low 4 {169, 87}/256 +Unvoiced Low 5 {162, 94}/256 +Unvoiced Low 6 or more {152, 104}/256 +Unvoiced High 0 {48, 208}/256 +Unvoiced High 1 {242, 14}/256 +Unvoiced High 2 {235, 21}/256 +Unvoiced High 3 {224, 32}/256 +Unvoiced High 4 {214, 42}/256 +Unvoiced High 5 {205, 51}/256 +Unvoiced High 6 or more {190, 66}/256 +Voiced Low 0 {1, 255}/256 +Voiced Low 1 {162, 94}/256 +Voiced Low 2 {152, 104}/256 +Voiced Low 3 {147, 109}/256 +Voiced Low 4 {144, 112}/256 +Voiced Low 5 {141, 115}/256 +Voiced Low 6 or more {138, 118}/256 +Voiced High 0 {8, 248}/256 +Voiced High 1 {203, 53}/256 +Voiced High 2 {187, 69}/256 +Voiced High 3 {176, 80}/256 +Voiced High 4 {168, 88}/256 +Voiced High 5 {161, 95}/256 +Voiced High 6 or more {154, 102}/256 + + +
+ +
+ + +After the signs have been read, there is enough information to reconstruct the + complete excitation signal. +This requires adding a constant quantization offset to each non-zero sample, + and then pseudorandomly inverting and offsetting every sample. +The constant quantization offset varies depending on the signal type and + quantization offset type (see ). + + + +Signal Type +Quantization Offset Type +Quantization Offset (Q23) +Inactive Low 25 +Inactive High 60 +Unvoiced Low 25 +Unvoiced High 60 +Voiced Low 8 +Voiced High 25 + + + +Let e_raw[i] be the raw excitation value at position i, with a magnitude + composed of the pulses at that location (see + ) combined with any additional LSBs (see + ), and with the corresponding sign decoded in + . +Additionally, let seed be the current pseudorandom seed, which is initialized + to the value decoded from for the first sample in + the current SILK frame, and updated for each subsequent sample according to + the procedure below. +Finally, let offset_Q23 be the quantization offset from + . +Then the following procedure produces the final reconstructed excitation value, + e_Q23[i]: +
+ +
+When e_raw[i] is zero, sign() returns 0 by the definition in + , so the factor of 20 does not get added. +The final e_Q23[i] value may require more than 16 bits per sample, but will not + require more than 23, including the sign. +
+ +
+ +
+ +
+ + +The remainder of the reconstruction process for the frame does not need to be + bit-exact, as small errors should only introduce proportionally small + distortions. +Although the reference implementation only includes a fixed-point version of + the remaining steps, this section describes them in terms of a floating-point + version for simplicity. +This produces a signal with a nominal range of -1.0 to 1.0. + + + +silk_decode_core() (decode_core.c) contains the code for the main + reconstruction process. +It proceeds subframe-by-subframe, since quantization gains, LTP parameters, and + (in 20 ms SILK frames) LPC coefficients can vary from one to the + next. + + + +Let a_Q12[k] be the LPC coefficients for the current subframe. +If this is the first or second subframe of a 20 ms SILK frame and the LSF + interpolation factor, w_Q2 (see ), is + less than 4, then these correspond to the final LPC coefficients produced by + from the interpolated LSF coefficients, + n1_Q15[k] (computed in ). +Otherwise, they correspond to the final LPC coefficients produced from the + uninterpolated LSF coefficients for the current frame, n2_Q15[k]. + + + +Also, let n be the number of samples in a subframe (40 for NB, 60 for MB, and + 80 for WB), s be the index of the current subframe in this SILK frame (0 or 1 + for 10 ms frames, or 0 to 3 for 20 ms frames), and j be the index of + the first sample in the residual corresponding to the current subframe. + + +
+ +Voiced SILK frames (see ) pass the excitation + through an LTP filter using the parameters decoded in + to produce an LPC residual. +The LTP filter requires LPC residual values from before the current subframe as + input. +However, since the LPC coefficients may have changed, it obtains this residual + by "rewhitening" the corresponding output signal using the LPC coefficients + from the current subframe. +Let out[i] for + (j - pitch_lags[s] - d_LPC - 2) <= i < j + be the fully reconstructed output signal from the last + (pitch_lags[s] + d_LPC + 2) samples of previous subframes + (see ), where pitch_lags[s] is the pitch + lag for the current subframe from . +During reconstruction of the first subframe for this channel after either + +An uncoded regular SILK frame (if this is the side channel), or +A decoder reset (see ), + + out[] is rewhitened into an LPC residual, + res[i], via +
+ +
+This requires storage to buffer up to 306 values of out[i] from previous + subframes. +This corresponds to WB with a maximum pitch lag of + 18 ms * 16 kHz samples, plus 16 samples for d_LPC, plus 2 + samples for the width of the LTP filter. +
+ + +Let e_Q23[i] for j <= i < (j + n) be the + excitation for the current subframe, and b_Q7[k] for + 0 <= k < 5 be the coefficients of the LTP filter + taken from the codebook entry in one of + Tables  + through  + corresponding to the index decoded for the current subframe in + . +Then for i such that j <= i < (j + n), + the LPC residual is +
+ +
+
+ + +For unvoiced frames, the LPC residual for + j <= i < (j + n) is simply a normalized + copy of the excitation signal, i.e., +
+ +
+
+
+ +
+ +LPC synthesis uses the short-term LPC filter to predict the next output + coefficient. +For i such that (j - d_LPC) <= i < j, let + lpc[i] be the result of LPC synthesis from the last d_LPC samples of the + previous subframe, or zeros in the first subframe for this channel after + either + +An uncoded regular SILK frame (if this is the side channel), or +A decoder reset (see ). + +Then for i such that j <= i < (j + n), the + result of LPC synthesis for the current subframe is +
+ +
+The decoder saves the final d_LPC values, i.e., lpc[i] such that + (j + n - d_LPC) <= i < (j + n), + to feed into the LPC synthesis of the next subframe. +This requires storage for up to 16 values of lpc[i] (for WB frames). +
+ + +Then, the signal is clamped into the final nominal range: +
+ +
+This clamping occurs entirely after the LPC synthesis filter has run. +The decoder saves the unclamped values, lpc[i], to feed into the LPC filter for + the next subframe, but saves the clamped values, out[i], for rewhitening in + voiced frames. +
+
+ +
+ +
+ +
+ +For stereo streams, after decoding a frame from each channel, the decoder must + convert the mid-side (MS) representation into a left-right (LR) + representation. +The function silk_stereo_MS_to_LR (stereo_MS_to_LR.c) implements this process. +In it, the decoder predicts the side channel using a) a simple low-passed + version of the mid channel, and b) the unfiltered mid channel, using the + prediction weights decoded in . +This simple low-pass filter imposes a one-sample delay, and the unfiltered +mid channel is also delayed by one sample. +In order to allow seamless switching between stereo and mono, mono streams must + also impose the same one-sample delay. +The encoder requires an additional one-sample delay for both mono and stereo + streams, though an encoder may omit the delay for mono if it knows it will + never switch to stereo. + + + +The unmixing process operates in two phases. +The first phase lasts for 8 ms, during which it interpolates the + prediction weights from the previous frame, prev_w0_Q13 and prev_w1_Q13, to + the values for the current frame, w0_Q13 and w1_Q13. +The second phase simply uses these weights for the remainder of the frame. + + + +Let mid[i] and side[i] be the contents of out[i] (from + ) for the current mid and side channels, + respectively, and let left[i] and right[i] be the corresponding stereo output + channels. +If the side channel is not coded (see ), + then side[i] is set to zero. +Also let j be defined as in , n1 be + the number of samples in phase 1 (64 for NB, 96 for MB, and 128 for WB), + and n2 be the total number of samples in the frame. +Then for i such that j <= i < (j + n2), + the left and right channel output is +
+ +
+These formulas require two samples prior to index j, the start of the + frame, for the mid channel, and one prior sample for the side channel. +For the first frame after a decoder reset, zeros are used instead. +
+ +
+ +
+ +After stereo unmixing (if any), the decoder applies resampling to convert the + decoded SILK output to the sample rate desired by the application. +This is necessary when decoding a Hybrid frame at SWB or FB sample rates, or + whenever the decoder wants the output at a different sample rate than the + internal SILK sampling rate (e.g., to allow a constant sample rate when the + audio bandwidth changes, or to allow mixing with audio from other + applications). +The resampler itself is non-normative, and a decoder can use any method it + wants to perform the resampling. + + + +However, a minimum amount of delay is imposed to allow the resampler to + operate, and this delay is normative, so that the corresponding delay can be + applied to the MDCT layer in the encoder. +A decoder is always free to use a resampler which requires more delay than + allowed for here (e.g., to improve quality), but it must then delay the output + of the MDCT layer by this extra amount. +Keeping as much delay as possible on the encoder side allows an encoder which + knows it will never use any of the SILK or Hybrid modes to skip this delay. +By contrast, if it were all applied by the decoder, then a decoder which + processes audio in fixed-size blocks would be forced to delay the output of + CELT frames just in case of a later switch to a SILK or Hybrid mode. + + + + gives the maximum resampler delay + in samples at 48 kHz for each SILK audio bandwidth. +Because the actual output rate may not be 48 kHz, it may not be possible + to achieve exactly these delays while using a whole number of input or output + samples. +The reference implementation is able to resample to any of the supported + output sampling rates (8, 12, 16, 24, or 48 kHz) within or near this + delay constraint. +Some resampling filters (including those used by the reference implementation) + may add a delay that is not an exact integer, or is not linear-phase, and so + cannot be represented by a single delay at all frequencies. +However, such deviations are unlikely to be perceptible, and the comparison + tool described in is designed to be relatively + insensitive to them. +The delays listed here are the ones that should be targeted by the encoder. + + + +Audio Bandwidth +Delay in millisecond +NB 0.538 +MB 0.692 +WB 0.706 + + + +NB is given a smaller decoder delay allocation than MB and WB to allow a + higher-order filter when resampling to 8 kHz in both the encoder and + decoder. +This implies that the audio content of two SILK frames operating at different + bandwidths are not perfectly aligned in time. +This is not an issue for any transitions described in + , because they all involve a SILK decoder reset. +When the decoder is reset, any samples remaining in the resampling buffer + are discarded, and the resampler is re-initialized with silence. + + +
+ +
+ + +
+ + +The CELT layer of Opus is based on the Modified Discrete Cosine Transform + with partially overlapping windows of 5 to 22.5 ms. +The main principle behind CELT is that the MDCT spectrum is divided into +bands that (roughly) follow the Bark scale, i.e., the scale of the ear's +critical bands . The normal CELT layer uses 21 of those bands, though Opus + Custom (see ) may use a different number of bands. +In Hybrid mode, the first 17 bands (up to 8 kHz) are not coded. +A band can contain as little as one MDCT bin per channel, and as many as 176 +bins per channel, as detailed in . +In each band, the gain (energy) is coded separately from +the shape of the spectrum. Coding the gain explicitly makes it easy to +preserve the spectral envelope of the signal. The remaining unit-norm shape +vector is encoded using a Pyramid Vector Quantizer (PVQ) . + + + +Frame Size: +2.5 ms +5 ms +10 ms +20 ms +Start Frequency +Stop Frequency +Band Bins: + 0 1 2 4 8 0 Hz 200 Hz + 1 1 2 4 8 200 Hz 400 Hz + 2 1 2 4 8 400 Hz 600 Hz + 3 1 2 4 8 600 Hz 800 Hz + 4 1 2 4 8 800 Hz 1000 Hz + 5 1 2 4 8 1000 Hz 1200 Hz + 6 1 2 4 8 1200 Hz 1400 Hz + 7 1 2 4 8 1400 Hz 1600 Hz + 8 2 4 8 16 1600 Hz 2000 Hz + 9 2 4 8 16 2000 Hz 2400 Hz +10 2 4 8 16 2400 Hz 2800 Hz +11 2 4 8 16 2800 Hz 3200 Hz +12 4 8 16 32 3200 Hz 4000 Hz +13 4 8 16 32 4000 Hz 4800 Hz +14 4 8 16 32 4800 Hz 5600 Hz +15 6 12 24 48 5600 Hz 6800 Hz +16 6 12 24 48 6800 Hz 8000 Hz +17 8 16 32 64 8000 Hz 9600 Hz +18 12 24 48 96 9600 Hz 12000 Hz +19 18 36 72 144 12000 Hz 15600 Hz +20 22 44 88 176 15600 Hz 20000 Hz + + + +Transients are notoriously difficult for transform codecs to code. +CELT uses two different strategies for them: + +Using multiple smaller MDCTs instead of a single large MDCT, and +Dynamic time-frequency resolution changes (See ). + +To improve quality on highly tonal and periodic signals, CELT includes +a prefilter/postfilter combination. The prefilter on the encoder side +attenuates the signal's harmonics. The postfilter on the decoder side +restores the original gain of the harmonics, while shaping the coding noise +to roughly follow the harmonics. Such noise shaping reduces the perception +of the noise. + + + +When coding a stereo signal, three coding methods are available: + +mid-side stereo: encodes the mean and the difference of the left and right channels, +intensity stereo: only encodes the mean of the left and right channels (discards the difference), +dual stereo: encodes the left and right channels separately. + + + + +An overview of the decoder is given in . + + +
+| decoder |----+ + | +---------+ | + | | + | +---------+ v + | | Fine | +---+ + +->| decoder |->| + | + | +---------+ +---+ + | ^ | ++---------+ | | | +| Range | | +----------+ v +| Decoder |-+ | Bit | +------+ ++---------+ | |Allocation| | 2**x | + | +----------+ +------+ + | | | + | v v +--------+ + | +---------+ +---+ +-------+ | pitch | + +->| PVQ |->| * |->| IMDCT |->| post- |---> + | | decoder | +---+ +-------+ | filter | + | +---------+ +--------+ + | ^ + +--------------------------------------+ +]]> +
+ + +The decoder is based on the following symbols and sets of symbols: + + + +Symbol(s) +PDF +Condition +silence {32767, 1}/32768 +post-filter {1, 1}/2 +octave uniform (6)post-filter +period raw bits (4+octave)post-filter +gain raw bits (3)post-filter +tapset {2, 1, 1}/4post-filter +transient {7, 1}/8 +intra {7, 1}/8 +coarse energy +tf_change +tf_select {1, 1}/2 +spread {7, 2, 21, 2}/32 +dyn. alloc. +alloc. trim {2, 2, 5, 10, 22, 46, 22, 10, 5, 2, 2}/128 +skip {1, 1}/2 +intensity uniform +dual {1, 1}/2 +fine energy +residual +anti-collapse{1, 1}/2 +finalize + + + +The decoder extracts information from the range-coded bitstream in the order +described in . In some circumstances, it is +possible for a decoded value to be out of range due to a very small amount of redundancy +in the encoding of large integers by the range coder. +In that case, the decoder should assume there has been an error in the coding, +decoding, or transmission and SHOULD take measures to conceal the error and/or report +to the application that a problem has occurred. Such out of range errors cannot occur +in the SILK layer. + + +
+ +The "transient" flag indicates whether the frame uses a single long MDCT or several short MDCTs. +When it is set, then the MDCT coefficients represent multiple +short MDCTs in the frame. When not set, the coefficients represent a single +long MDCT for the frame. The flag is encoded in the bitstream with a probability of 1/8. +In addition to the global transient flag is a per-band +binary flag to change the time-frequency (tf) resolution independently in each band. The +change in tf resolution is defined in tf_select_table[][] in celt.c and depends +on the frame size, whether the transient flag is set, and the value of tf_select. +The tf_select flag uses a 1/2 probability, but is only decoded +if it can have an impact on the result knowing the value of all per-band +tf_change flags. + +
+ +
+ + +It is important to quantize the energy with sufficient resolution because +any energy quantization error cannot be compensated for at a later +stage. Regardless of the resolution used for encoding the spectral shape of a band, +it is perceptually important to preserve the energy in each band. CELT uses a +three-step coarse-fine-fine strategy for encoding the energy in the base-2 log +domain, as implemented in quant_bands.c + +
+ +Coarse quantization of the energy uses a fixed resolution of 6 dB +(integer part of base-2 log). To minimize the bitrate, prediction is applied +both in time (using the previous frame) and in frequency (using the previous +bands). The part of the prediction that is based on the +previous frame can be disabled, creating an "intra" frame where the energy +is coded without reference to prior frames. The decoder first reads the intra flag +to determine what prediction is used. +The 2-D z-transform of +the prediction filter is: +
+ +
+where b is the band index and l is the frame index. The prediction coefficients +applied depend on the frame size in use when not using intra energy and are alpha=0, beta=4915/32768 +when using intra energy. +The time-domain prediction is based on the final fine quantization of the previous +frame, while the frequency domain (within the current frame) prediction is based +on coarse quantization only (because the fine quantization has not been computed +yet). The prediction is clamped internally so that fixed point implementations with +limited dynamic range always remain in the same state as floating point implementations. +We approximate the ideal +probability distribution of the prediction error using a Laplace distribution +with separate parameters for each frame size in intra- and inter-frame modes. These +parameters are held in the e_prob_model table in quant_bands.c. +The +coarse energy quantization is performed by unquant_coarse_energy() and +unquant_coarse_energy_impl() (quant_bands.c). The encoding of the Laplace-distributed values is +implemented in ec_laplace_decode() (laplace.c). +
+ +
+ +
+ +The number of bits assigned to fine energy quantization in each band is determined +by the bit allocation computation described in . +Let B_i be the number of fine energy bits +for band i; the refinement is an integer f in the range [0,2**B_i-1]. The mapping between f +and the correction applied to the coarse energy is equal to (f+1/2)/2**B_i - 1/2. Fine +energy quantization is implemented in quant_fine_energy() (quant_bands.c). + + +When some bits are left "unused" after all other flags have been decoded, these bits +are assigned to a "final" step of fine allocation. In effect, these bits are used +to add one extra fine energy bit per band per channel. The allocation process +determines two "priorities" for the final fine bits. +Any remaining bits are first assigned only to bands of priority 0, starting +from band 0 and going up. If all bands of priority 0 have received one bit per +channel, then bands of priority 1 are assigned an extra bit per channel, +starting from band 0. If any bits are left after this, they are left unused. +This is implemented in unquant_energy_finalise() (quant_bands.c). + + +
+ +
+ +
+ +Because the bit allocation drives the decoding of the range-coder +stream, it MUST be recovered exactly so that identical coding decisions are +made in the encoder and decoder. Any deviation from the reference's resulting +bit allocation will result in corrupted output, though implementers are +free to implement the procedure in any way which produces identical results. + +The per-band gain-shape structure of the CELT layer ensures that using + the same number of bits for the spectral shape of a band in every frame will + result in a roughly constant signal-to-noise ratio in that band. +This results in coding noise that has the same spectral envelope as the signal. +The masking curve produced by a standard psychoacoustic model also closely + follows the spectral envelope of the signal. +This structure means that the ideal allocation is more consistent from frame to + frame than it is for other codecs without an equivalent structure, and that a + fixed allocation provides fairly consistent perceptual + performance . + +Many codecs transmit significant amounts of side information to control the + bit allocation within a frame. +Often this control is only indirect, and must be exercised carefully to + achieve the desired rate constraints. +The CELT layer, however, can adapt over a very wide range of rates, and thus + has a large number of codebook sizes to choose from for each band. +Explicitly signaling the size of each of these codebooks would impose + considerable overhead, even though the allocation is relatively static from + frame to frame. +This is because all of the information required to compute these codebook sizes + must be derived from a single frame by itself, in order to retain robustness + to packet loss, so the signaling cannot take advantage of knowledge of the + allocation in neighboring frames. +This problem is exacerbated in low-latency (small frame size) applications, + which would include this overhead in every frame. + +For this reason, in the MDCT mode Opus uses a primarily implicit bit +allocation. The available bitstream capacity is known in advance to both +the encoder and decoder without additional signaling, ultimately from the +packet sizes expressed by a higher-level protocol. Using this information, +the codec interpolates an allocation from a hard-coded table. + +While the band-energy structure effectively models intra-band masking, +it ignores the weaker inter-band masking, band-temporal masking, and +other less significant perceptual effects. While these effects can +often be ignored, they can become significant for particular samples. One +mechanism available to encoders would be to simply increase the overall +rate for these frames, but this is not possible in a constant rate mode +and can be fairly inefficient. As a result three explicitly signaled +mechanisms are provided to alter the implicit allocation: + + + +Band boost +Allocation trim +Band skipping + + + +The first of these mechanisms, band boost, allows an encoder to boost +the allocation in specific bands. The second, allocation trim, works by +biasing the overall allocation towards higher or lower frequency bands. The third, band +skipping, selects which low-precision high frequency bands +will be allocated no shape bits at all. + +In stereo mode there are two additional parameters +potentially coded as part of the allocation procedure: a parameter to allow the +selective elimination of allocation for the 'side' (i.e., intensity stereo) in jointly coded bands, +and a flag to deactivate joint coding (i.e., dual stereo). These values are not signaled if +they would be meaningless in the overall context of the allocation. + +Because every signaled adjustment increases overhead and implementation +complexity, none were included speculatively: the reference encoder makes use +of all of these mechanisms. While the decision logic in the reference was +found to be effective enough to justify the overhead and complexity, further +analysis techniques may be discovered which increase the effectiveness of these +parameters. As with other signaled parameters, an encoder is free to choose the +values in any manner, but unless a technique is known to deliver superior +perceptual results the methods used by the reference implementation should be +used. + +The allocation process consists of the following steps: determining the per-band +maximum allocation vector, decoding the boosts, decoding the tilt, determining +the remaining capacity of the frame, searching the mode table for the +entry nearest but not exceeding the available space (subject to the tilt, boosts, band +maximums, and band minimums), linear interpolation, reallocation of +unused bits with concurrent skip decoding, determination of the +fine-energy vs. shape split, and final reallocation. This process results +in a per-band shape allocation (in 1/8th bit units), a per-band fine-energy +allocation (in 1 bit per channel units), a set of band priorities for +controlling the use of remaining bits at the end of the frame, and a +remaining balance of unallocated space, which is usually zero except +at very high rates. + + +The "static" bit allocation (in 1/8 bits) for a quality q, excluding the minimums, maximums, +tilt and boosts, is equal to channels*N*alloc[band][q]<<LM>>2, where +alloc[][] is given in and LM=log2(frame_size/120). The allocation +is obtained by linearly interpolating between two values of q (in steps of 1/64) to find the +highest allocation that does not exceed the number of bits remaining. + + + + Rows indicate the MDCT bands, columns are the different quality (q) parameters. The units are 1/32 bit per MDCT bin. +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +090110118126134144152162172200 +080100110119127137145155165200 +07590103112120130138148158200 +0698493104114124132142152200 +063788695103113123133143200 +05671808997107117127137200 +04965758391101111121131200 +0405870788595105115125200 +034516572788898108118198 +029455966728292102112193 +02039536066768696106188 +01832475460708090100183 +0102640475464748494178 +002031394757677787173 +001223324151617181168 +00015253545556575163 +0004172939495969158 +0000122333435363153 +000011626364656148 +000001015203045129 +00000111120104 + + +The maximum allocation vector is an approximation of the maximum space +that can be used by each band for a given mode. The value is +approximate because the shape encoding is variable rate (due +to entropy coding of splitting parameters). Setting the maximum too low reduces the +maximum achievable quality in a band while setting it too high +may result in waste: bitstream capacity available at the end +of the frame which can not be put to any use. The maximums +specified by the codec reflect the average maximum. In the reference +implementation, the maximums in bits/sample are precomputed in a static table +(see cache_caps50[] in static_modes_float.h) for each band, +for each value of LM, and for both mono and stereo. + +Implementations are expected +to simply use the same table data, but the procedure for generating +this table is included in rate.c as part of compute_pulse_cache(). + +To convert the values in cache.caps into the actual maximums: first +set nbBands to the maximum number of bands for this mode, and stereo to +zero if stereo is not in use and one otherwise. For each band set N +to the number of MDCT bins covered by the band (for one channel), set LM +to the shift value for the frame size, +then set i to nbBands*(2*LM+stereo). Then set the maximum for the band to +the i-th index of cache.caps + 64 and multiply by the number of channels +in the current frame (one or two) and by N, then divide the result by 4 +using integer division. The resulting vector will be called +cap[]. The elements fit in signed 16-bit integers but do not fit in 8 bits. +This procedure is implemented in the reference in the function init_caps() in celt.c. + + +The band boosts are represented by a series of binary symbols which +are entropy coded with very low probability. Each band can potentially be boosted +multiple times, subject to the frame actually having enough room to obey +the boost and having enough room to code the boost symbol. The default +coding cost for a boost starts out at six bits (probability p=1/64), but subsequent boosts +in a band cost only a single bit and every time a band is boosted the +initial cost is reduced (down to a minimum of two bits, or p=1/4). Since the initial +cost of coding a boost is 6 bits, the coding cost of the boost symbols when +completely unused is 0.48 bits/frame for a 21 band mode (21*-log2(1-1/2**6)). + +To decode the band boosts: First set 'dynalloc_logp' to 6, the initial +amount of storage required to signal a boost in bits, 'total_bits' to the +size of the frame in 8th bits, 'total_boost' to zero, and 'tell' to the total number +of 8th bits decoded +so far. For each band from the coding start (0 normally, but 17 in Hybrid mode) +to the coding end (which changes depending on the signaled bandwidth), the boost quanta +in units of 1/8 bit is calculated as quanta = min(8*N, max(48, N)). +This represents a boost step size of six bits, subject to a lower limit of +1/8th bit/sample and an upper limit of 1 bit/sample. +Set 'boost' to zero and 'dynalloc_loop_logp' +to dynalloc_logp. While dynalloc_loop_log (the current worst case symbol cost) in +8th bits plus tell is less than total_bits plus total_boost and boost is less than cap[] for this +band: Decode a bit from the bitstream with a with dynalloc_loop_logp as the cost +of a one, update tell to reflect the current used capacity, if the decoded value +is zero break the loop otherwise add quanta to boost and total_boost, subtract quanta from +total_bits, and set dynalloc_loop_log to 1. When the while loop finishes +boost contains the boost for this band. If boost is non-zero and dynalloc_logp +is greater than 2, decrease dynalloc_logp. Once this process has been +executed on all bands, the band boosts have been decoded. This procedure +is implemented around line 2474 of celt.c. + +At very low rates it is possible that there won't be enough available +space to execute the inner loop even once. In these cases band boost +is not possible but its overhead is completely eliminated. Because of the +high cost of band boost when activated, a reasonable encoder should not be +using it at very low rates. The reference implements its dynalloc decision +logic around line 1304 of celt.c. + +The allocation trim is a integer value from 0-10. The default value of +5 indicates no trim. The trim parameter is entropy coded in order to +lower the coding cost of less extreme adjustments. Values lower than +5 bias the allocation towards lower frequencies and values above 5 +bias it towards higher frequencies. Like other signaled parameters, signaling +of the trim is gated so that it is not included if there is insufficient space +available in the bitstream. To decode the trim, first set +the trim value to 5, then if and only if the count of decoded 8th bits so far (ec_tell_frac) +plus 48 (6 bits) is less than or equal to the total frame size in 8th +bits minus total_boost (a product of the above band boost procedure), +decode the trim value using the PDF in . + + +PDF +{1, 1, 2, 5, 10, 22, 46, 22, 10, 5, 2, 2}/128 + + +For 10 ms and 20 ms frames using short blocks and that have at least LM+2 bits left prior to +the allocation process, then one anti-collapse bit is reserved in the allocation process so it can +be decoded later. Following the the anti-collapse reservation, one bit is reserved for skip if available. + +For stereo frames, bits are reserved for intensity stereo and for dual stereo. Intensity stereo +requires ilog2(end-start) bits. Those bits are reserved if there is enough bits left. Following this, one +bit is reserved for dual stereo if available. + + +The allocation computation begins by setting up some initial conditions. +'total' is set to the remaining available 8th bits, computed by taking the +size of the coded frame times 8 and subtracting ec_tell_frac(). From this value, one (8th bit) +is subtracted to ensure that the resulting allocation will be conservative. 'anti_collapse_rsv' +is set to 8 (8th bits) if and only if the frame is a transient, LM is greater than 1, and total is +greater than or equal to (LM+2) * 8. Total is then decremented by anti_collapse_rsv and clamped +to be equal to or greater than zero. 'skip_rsv' is set to 8 (8th bits) if total is greater than +8, otherwise it is zero. Total is then decremented by skip_rsv. This reserves space for the +final skipping flag. + +If the current frame is stereo, intensity_rsv is set to the conservative log2 in 8th bits +of the number of coded bands for this frame (given by the table LOG2_FRAC_TABLE in rate.c). If +intensity_rsv is greater than total then intensity_rsv is set to zero. Otherwise total is +decremented by intensity_rsv, and if total is still greater than 8, dual_stereo_rsv is +set to 8 and total is decremented by dual_stereo_rsv. + +The allocation process then computes a vector representing the hard minimum amounts allocation +any band will receive for shape. This minimum is higher than the technical limit of the PVQ +process, but very low rate allocations produce an excessively sparse spectrum and these bands +are better served by having no allocation at all. For each coded band, set thresh[band] to +twenty-four times the number of MDCT bins in the band and divide by 16. If 8 times the number +of channels is greater, use that instead. This sets the minimum allocation to one bit per channel +or 48 128th bits per MDCT bin, whichever is greater. The band-size dependent part of this +value is not scaled by the channel count, because at the very low rates where this limit is +applicable there will usually be no bits allocated to the side. + +The previously decoded allocation trim is used to derive a vector of per-band adjustments, +'trim_offsets[]'. For each coded band take the alloc_trim and subtract 5 and LM. Then multiply +the result by the number of channels, the number of MDCT bins in the shortest frame size for this mode, +the number of remaining bands, 2**LM, and 8. Then divide this value by 64. Finally, if the +number of MDCT bins in the band per channel is only one, 8 times the number of channels is subtracted +in order to diminish the allocation by one bit, because width 1 bands receive greater benefit +from the coarse energy coding. + + +
+ +
+ +In each band, the normalized "shape" is encoded +using a vector quantization scheme called a "pyramid vector quantizer". + + +In +the simplest case, the number of bits allocated in + is converted to a number of pulses as described +by . Knowing the number of pulses and the +number of samples in the band, the decoder calculates the size of the codebook +as detailed in . The size is used to decode +an unsigned integer (uniform probability model), which is the codeword index. +This index is converted into the corresponding vector as explained in +. This vector is then scaled to unit norm. + + +
+ +Although the allocation is performed in 1/8th bit units, the quantization requires +an integer number of pulses K. To do this, the encoder searches for the value +of K that produces the number of bits nearest to the allocated value +(rounding down if exactly halfway between two values), not to exceed +the total number of bits available. For efficiency reasons, the search is performed against a +precomputed allocation table which only permits some K values for each N. The number of +codebook entries can be computed as explained in . The difference +between the number of bits allocated and the number of bits used is accumulated to a +"balance" (initialized to zero) that helps adjust the +allocation for the next bands. One third of the balance is applied to the +bit allocation of each band to help achieve the target allocation. The only +exceptions are the band before the last and the last band, for which half the balance +and the whole balance are applied, respectively. + +
+ +
+ + +Decoding of PVQ vectors is implemented in decode_pulses() (cwrs.c). +The unique codeword index is decoded as a uniformly-distributed integer value between 0 and +V(N,K)-1, where V(N,K) is the number of possible combinations of K pulses in +N samples. The index is then converted to a vector in the same way specified in +. The indexing is based on the calculation of V(N,K) +(denoted N(L,K) in ). + + + + The number of combinations can be computed recursively as +V(N,K) = V(N-1,K) + V(N,K-1) + V(N-1,K-1), with V(N,0) = 1 and V(0,K) = 0, K != 0. +There are many different ways to compute V(N,K), including precomputed tables and direct +use of the recursive formulation. The reference implementation applies the recursive +formulation one line (or column) at a time to save on memory use, +along with an alternate, +univariate recurrence to initialize an arbitrary line, and direct +polynomial solutions for small N. All of these methods are +equivalent, and have different trade-offs in speed, memory usage, and +code size. Implementations MAY use any methods they like, as long as +they are equivalent to the mathematical definition. + + + +The decoded vector X is recovered as follows. +Let i be the index decoded with the procedure in + with ft = V(N,K), so that 0 <= i < V(N,K). +Let k = K. +Then for j = 0 to (N - 1), inclusive, do: + +Let p = (V(N-j-1,k) + V(N-j,k))/2. + +If i < p, then let sgn = 1, else let sgn = -1 + and set i = i - p. + +Let k0 = k and set p = p - V(N-j-1,k). + +While p > i, set k = k - 1 and + p = p - V(N-j-1,k). + + +Set X[j] = sgn*(k0 - k) and i = i - p. + + + + + +The decoded vector X is then normalized such that its +L2-norm equals one. + +
+ +
+ +The normalized vector decoded in is then rotated +for the purpose of avoiding tonal artifacts. The rotation gain is equal to +
+ +
+ +where N is the number of dimensions, K is the number of pulses, and f_r depends on +the value of the "spread" parameter in the bit-stream. +
+ + +Spread value +f_r + 0 infinite (no rotation) + 1 15 + 2 10 + 3 5 + + + +The rotation angle is then calculated as +
+ +
+A 2-D rotation R(i,j) between points x_i and x_j is defined as: +
+ +
+ +An N-D rotation is then achieved by applying a series of 2-D rotations back and forth, in the +following order: R(x_1, x_2), R(x_2, x_3), ..., R(x_N-2, X_N-1), R(x_N-1, X_N), +R(x_N-2, X_N-1), ..., R(x_1, x_2). +
+ + +If the decoded vector represents more +than one time block, then this spreading process is applied separately on each time block. +Also, if each block represents 8 samples or more, then another N-D rotation, by +(pi/2-theta), is applied before the rotation described above. This +extra rotation is applied in an interleaved manner with a stride equal to round(sqrt(N/nb_blocks)), +i.e., it is applied independently for each set of sample S_k = {stride*n + k}, n=0..N/stride-1. + +
+ +
+ +To avoid the need for multi-precision calculations when decoding PVQ codevectors, +the maximum size allowed for codebooks is 32 bits. When larger codebooks are +needed, the vector is instead split in two sub-vectors of size N/2. +A quantized gain parameter with precision +derived from the current allocation is entropy coded to represent the relative +gains of each side of the split, and the entire decoding process is recursively +applied. Multiple levels of splitting may be applied up to a limit of LM+1 splits. +The same recursive mechanism is applied for the joint coding +of stereo audio. + + +
+ +
+ +The time-frequency (TF) parameters are used to control the time-frequency resolution tradeoff +in each coded band. For each band, there are two possible TF choices. For the first +band coded, the PDF is {3, 1}/4 for frames marked as transient and {15, 1}/16 for +the other frames. For subsequent bands, the TF choice is coded relative to the +previous TF choice with probability {15, 1}/15 for transient frames and {31, 1}/32 +otherwise. The mapping between the decoded TF choices and the adjustment in TF +resolution is shown in the tables below. + + + +Frame size (ms) +0 +1 +2.5 0 -1 +5 0 -1 +10 0 -2 +20 0 -2 + + + +Frame size (ms) +0 +1 +2.5 0 -1 +5 0 -2 +10 0 -3 +20 0 -3 + + + + +Frame size (ms) +0 +1 +2.5 0 -1 +5 1 0 +10 2 0 +20 3 0 + + + +Frame size (ms) +0 +1 +2.5 0 -1 +5 1 -1 +10 1 -1 +20 1 -1 + + + +A negative TF adjustment means that the temporal resolution is increased, +while a positive TF adjustment means that the frequency resolution is increased. +Changes in TF resolution are implemented using the Hadamard transform . To increase +the time resolution by N, N "levels" of the Hadamard transform are applied to the +decoded vector for each interleaved MDCT vector. To increase the frequency resolution +(assumes a transient frame), then N levels of the Hadamard transform are applied +across the interleaved MDCT vector. In the case of increased +time resolution the decoder uses the "sequency order" because the input vector +is sorted in time. + +
+ + +
+ +
+ +The anti-collapse feature is designed to avoid the situation where the use of multiple +short MDCTs causes the energy in one or more of the MDCTs to be zero for +some bands, causing unpleasant artifacts. +When the frame has the transient bit set, an anti-collapse bit is decoded. +When anti-collapse is set, the energy in each small MDCT is prevented +from collapsing to zero. For each band of each MDCT where a collapse is +detected, a pseudo-random signal is inserted with an energy corresponding +to the minimum energy over the two previous frames. A renormalization step is +then required to ensure that the anti-collapse step did not alter the +energy preservation property. + +
+ +
+ +Just as each band was normalized in the encoder, the last step of the decoder before +the inverse MDCT is to denormalize the bands. Each decoded normalized band is +multiplied by the square root of the decoded energy. This is done by denormalise_bands() +(bands.c). + +
+ +
+ + +The inverse MDCT implementation has no special characteristics. The +input is N frequency-domain samples and the output is 2*N time-domain +samples, while scaling by 1/2. A "low-overlap" window reduces the algorithmic delay. +It is derived from a basic (full overlap) 240-sample version of the window used by the Vorbis codec: +
+ +
+The low-overlap window is created by zero-padding the basic window and inserting ones in the +middle, such that the resulting window still satisfies power complementarity . +The IMDCT and +windowing are performed by mdct_backward (mdct.c). +
+ +
+ +The output of the inverse MDCT (after weighted overlap-add) is sent to the +post-filter. Although the post-filter is applied at the end, the post-filter +parameters are encoded at the beginning, just after the silence flag. +The post-filter can be switched on or off using one bit (logp=1). +If the post-filter is enabled, then the octave is decoded as an integer value +between 0 and 6 of uniform probability. Once the octave is known, the fine pitch +within the octave is decoded using 4+octave raw bits. The final pitch period +is equal to (16<<octave)+fine_pitch-1 so it is bounded between 15 and 1022, +inclusively. Next, the gain is decoded as three raw bits and is equal to +G=3*(int_gain+1)/32. The set of post-filter taps is decoded last, using +a pdf equal to {2, 1, 1}/4. Tapset zero corresponds to the filter coefficients +g0 = 0.3066406250, g1 = 0.2170410156, g2 = 0.1296386719. Tapset one +corresponds to the filter coefficients g0 = 0.4638671875, g1 = 0.2680664062, +g2 = 0, and tapset two uses filter coefficients g0 = 0.7998046875, +g1 = 0.1000976562, g2 = 0. + + + +The post-filter response is thus computed as: +
+ + + +
+ +During a transition between different gains, a smooth transition is calculated +using the square of the MDCT window. It is important that values of y(n) be +interpolated one at a time such that the past value of y(n) used is interpolated. +
+
+ +
+ +After the post-filter, +the signal is de-emphasized using the inverse of the pre-emphasis filter +used in the encoder: +
+ +
+where alpha_p=0.8500061035. +
+
+ +
+ +
+ +
+ +Packet loss concealment (PLC) is an optional decoder-side feature that +SHOULD be included when receiving from an unreliable channel. Because +PLC is not part of the bitstream, there are many acceptable ways to +implement PLC with different complexity/quality trade-offs. + + + +The PLC in +the reference implementation depends on the mode of last packet received. +In CELT mode, the PLC finds a periodicity in the decoded +signal and repeats the windowed waveform using the pitch offset. The windowed +waveform is overlapped in such a way as to preserve the time-domain aliasing +cancellation with the previous frame and the next frame. This is implemented +in celt_decode_lost() (mdct.c). In SILK mode, the PLC uses LPC extrapolation +from the previous frame, implemented in silk_PLC() (PLC.c). + + +
+ +Clock drift refers to the gradual desynchronization of two endpoints +whose sample clocks run at different frequencies while they are streaming +live audio. Differences in clock frequencies are generally attributable to +manufacturing variation in the endpoints' clock hardware. For long-lived +streams, the time difference between sender and receiver can grow without +bound. + + + +When the sender's clock runs slower than the receiver's, the effect is similar +to packet loss: too few packets are received. The receiver can distinguish +between drift and loss if the transport provides packet timestamps. A receiver +for live streams SHOULD conceal the effects of drift, and MAY do so by invoking +the PLC. + + + +When the sender's clock runs faster than the receiver's, too many packets will +be received. The receiver MAY respond by skipping any packet (i.e., not +submitting the packet for decoding). This is likely to produce a less severe +artifact than if the frame were dropped after decoding. + + + +A decoder MAY employ a more sophisticated drift compensation method. For +example, the +NetEQ component +of the +Google WebRTC codebase +compensates for drift by adding or removing +one period when the signal is highly periodic. The reference implementation of +Opus allows a caller to learn whether the current frame's signal is highly +periodic, and if so what the period is, using the OPUS_GET_PITCH() request. + +
+ +
+ +
+ + +Switching between the Opus coding modes, audio bandwidths, and channel counts + requires careful consideration to avoid audible glitches. +Switching between any two configurations of the CELT-only mode, any two + configurations of the Hybrid mode, or from WB SILK to Hybrid mode does not + require any special treatment in the decoder, as the MDCT overlap will smooth + the transition. +Switching from Hybrid mode to WB SILK requires adding in the final contents + of the CELT overlap buffer to the first SILK-only packet. +This can be done by decoding a 2.5 ms silence frame with the CELT decoder + using the channel count of the SILK-only packet (and any choice of audio + bandwidth), which will correctly handle the cases when the channel count + changes as well. + + + +When changing the channel count for SILK-only or Hybrid packets, the encoder + can avoid glitches by smoothly varying the stereo width of the input signal + before or after the transition, and SHOULD do so. +However, other transitions between SILK-only packets or between NB or MB SILK + and Hybrid packets may cause glitches, because neither the LSF coefficients + nor the LTP, LPC, stereo unmixing, and resampler buffers are available at the + new sample rate. +These switches SHOULD be delayed by the encoder until quiet periods or + transients, where the inevitable glitches will be less audible. Additionally, + the bit-stream MAY include redundant side information ("redundancy"), in the + form of additional CELT frames embedded in each of the Opus frames around the + transition. + + + +The other transitions that cannot be easily handled are those where the lower + frequencies switch between the SILK LP-based model and the CELT MDCT model. +However, an encoder may not have an opportunity to delay such a switch to a + convenient point. +For example, if the content switches from speech to music, and the encoder does + not have enough latency in its analysis to detect this in advance, there may + be no convenient silence period during which to make the transition for quite + some time. +To avoid or reduce glitches during these problematic mode transitions, and + also between audio bandwidth changes in the SILK-only modes, transitions MAY + include redundant side information ("redundancy"), in the form of an + additional CELT frame embedded in the Opus frame. + + + +A transition between coding the lower frequencies with the LP model and the + MDCT model or a transition that involves changing the SILK bandwidth + is only normatively specified when it includes redundancy. +For those without redundancy, it is RECOMMENDED that the decoder use a + concealment technique (e.g., make use of a PLC algorithm) to "fill in" the + gap or discontinuity caused by the mode transition. +Therefore, PLC MUST NOT be applied during any normative transition, i.e., when + +A packet includes redundancy for this transition (as described below), +The transition is between any WB SILK packet and any Hybrid packet, or vice + versa, +The transition is between any two Hybrid mode packets, or +The transition is between any two CELT mode packets, + + unless there is actual packet loss. + + +
+ +Transitions with side information include an extra 5 ms "redundant" CELT + frame within the Opus frame. +This frame is designed to fill in the gap or discontinuity in the different + layers without requiring the decoder to conceal it. +For transitions from CELT-only to SILK-only or Hybrid, the redundant frame is + inserted in the first Opus frame after the transition (i.e., the first + SILK-only or Hybrid frame). +For transitions from SILK-only or Hybrid to CELT-only, the redundant frame is + inserted in the last Opus frame before the transition (i.e., the last + SILK-only or Hybrid frame). + + +
+ +The presence of redundancy is signaled in all SILK-only and Hybrid frames, not + just those involved in a mode transition. +This allows the frames to be decoded correctly even if an adjacent frame is + lost. +For SILK-only frames, this signaling is implicit, based on the size of the + of the Opus frame and the number of bits consumed decoding the SILK portion of + it. +After decoding the SILK portion of the Opus frame, the decoder uses ec_tell() + (see ) to check if there are at least 17 bits + remaining. +If so, then the frame contains redundancy. + + + +For Hybrid frames, this signaling is explicit. +After decoding the SILK portion of the Opus frame, the decoder uses ec_tell() + (see ) to ensure there are at least 37 bits remaining. +If so, it reads a symbol with the PDF in + , and if the value is 1, then the + frame contains redundancy. +Otherwise (if there were fewer than 37 bits left or the value was 0), the frame + does not contain redundancy. + + + +PDF +{4095, 1}/4096 + +
+ +
+ +Since the current frame is a SILK-only or a Hybrid frame, it must be at least + 10 ms. +Therefore, it needs an additional flag to indicate whether the redundant + 5 ms CELT frame should be mixed into the beginning of the current frame, + or the end. +After determining that a frame contains redundancy, the decoder reads a + 1 bit symbol with a uniform PDF + (). + + + +PDF +{1, 1}/2 + + + +If the value is zero, this is the first frame in the transition, and the + redundancy belongs at the end. +If the value is one, this is the second frame in the transition, and the + redundancy belongs at the beginning. +There is no way to specify that an Opus frame contains separate redundant CELT + frames at both the beginning and the end. + +
+ +
+ +Unlike the CELT portion of a Hybrid frame, the redundant CELT frame does not + use the same entropy coder state as the rest of the Opus frame, because this + would break the CELT bit allocation mechanism in Hybrid frames. +Thus, a redundant CELT frame always starts and ends on a byte boundary, even in + SILK-only frames, where this is not strictly necessary. + + + +For SILK-only frames, the number of bytes in the redundant CELT frame is simply + the number of whole bytes remaining, which must be at least 2, due to the + space check in . +For Hybrid frames, the number of bytes is equal to 2, plus a decoded unsigned + integer less than 256 (see ). +This may be more than the number of whole bytes remaining in the Opus frame, + in which case the frame is invalid. +However, a decoder is not required to ignore the entire frame, as this may be + the result of a bit error that desynchronized the range coder. +There may still be useful data before the error, and a decoder MAY keep any + audio decoded so far instead of invoking the PLC, but it is RECOMMENDED that + the decoder stop decoding and discard the rest of the current Opus frame. + + + +It would have been possible to avoid these invalid states in the design of Opus + by limiting the range of the explicit length decoded from Hybrid frames by the + actual number of whole bytes remaining. +However, this would require an encoder to determine the rate allocation for the + MDCT layer up front, before it began encoding that layer. +By allowing some invalid sizes, the encoder is able to defer that decision + until much later. +When encoding Hybrid frames which do not include redundancy, the encoder must + still decide up-front if it wishes to use the minimum 37 bits required to + trigger encoding of the redundancy flag, but this is a much looser + restriction. + + + +After determining the size of the redundant CELT frame, the decoder reduces + the size of the buffer currently in use by the range coder by that amount. +The CELT layer read any raw bits from the end of this reduced buffer, and all + calculations of the number of bits remaining in the buffer must be done using + this new, reduced size, rather than the original size of the Opus frame. + +
+ +
+ +The redundant frame is decoded like any other CELT-only frame, with the + exception that it does not contain a TOC byte. +The frame size is fixed at 5 ms, the channel count is set to that of the + current frame, and the audio bandwidth is also set to that of the current + frame, with the exception that for MB SILK frames, it is set to WB. + + + +If the redundancy belongs at the beginning (in a CELT-only to SILK-only or + Hybrid transition), the final reconstructed output uses the first 2.5 ms + of audio output by the decoder for the redundant frame as-is, discarding + the corresponding output from the SILK-only or Hybrid portion of the frame. +The remaining 2.5 ms is cross-lapped with the decoded SILK/Hybrid signal + using the CELT's power-complementary MDCT window to ensure a smooth + transition. + + + +If the redundancy belongs at the end (in a SILK-only or Hybrid to CELT-only + transition), only the second half (2.5 ms) of the audio output by the + decoder for the redundant frame is used. +In that case, the second half of the redundant frame is cross-lapped with the + end of the SILK/Hybrid signal, again using CELT's power-complementary MDCT + window to ensure a smooth transition. + +
+ +
+ +
+ +When a transition occurs, the state of the SILK or the CELT decoder (or both) + may need to be reset before decoding a frame in the new mode. +This avoids reusing "out of date" memory, which may not have been updated in + some time or may not be in a well-defined state due to, e.g., PLC. +The SILK state is reset before every SILK-only or Hybrid frame where the + previous frame was CELT-only. +The CELT state is reset every time the operating mode changes and the new mode + is either Hybrid or CELT-only, except when the transition uses redundancy as + described above. +When switching from SILK-only or Hybrid to CELT-only with redundancy, the CELT + state is reset before decoding the redundant CELT frame embedded in the + SILK-only or Hybrid frame, but it is not reset before decoding the following + CELT-only frame. +When switching from CELT-only mode to SILK-only or Hybrid mode with redundancy, + the CELT decoder is not reset for decoding the redundant CELT frame. + +
+ +
+ + + illustrates all of the normative + transitions involving a mode change, an audio bandwidth change, or both. +Each one uses an S, H, or C to represent an Opus frame in the corresponding + mode. +In addition, an R indicates the presence of redundancy in the Opus frame it is + cross-lapped with. +Its location in the first or last 5 ms is assumed to correspond to whether + it is the frame before or after the transition. +Other uses of redundancy are non-normative. +Finally, a c indicates the contents of the CELT overlap buffer after the + previously decoded frame (i.e., as extracted by decoding a silence frame). +
+ S -> S + & + !R -> R + & + ;S -> S -> S + +NB or MB SILK to Hybrid with Redundancy: S -> S -> S + & + !R ->;H -> H -> H + +WB SILK to Hybrid: S -> S -> S ->!H -> H -> H + +SILK to CELT with Redundancy: S -> S -> S + & + !R -> C -> C -> C + +Hybrid to NB or MB SILK with Redundancy: H -> H -> H + & + !R -> R + & + ;S -> S -> S + +Hybrid to WB SILK: H -> H -> H -> c + \ + + > S -> S -> S + +Hybrid to CELT with Redundancy: H -> H -> H + & + !R -> C -> C -> C + +CELT to SILK with Redundancy: C -> C -> C -> R + & + ;S -> S -> S + +CELT to Hybrid with Redundancy: C -> C -> C -> R + & + |H -> H -> H + +Key: +S SILK-only frame ; SILK decoder reset +H Hybrid frame | CELT and SILK decoder resets +C CELT-only frame ! CELT decoder reset +c CELT overlap + Direct mixing +R Redundant CELT frame & Windowed cross-lap +]]> +
+The first two and the last two Opus frames in each example are illustrative, + i.e., there is no requirement that a stream remain in the same configuration + for three consecutive frames before or after a switch. +
+ + +The behavior of transitions without redundancy where PLC is allowed is non-normative. +An encoder might still wish to use these transitions if, for example, it + doesn't want to add the extra bitrate required for redundancy or if it makes + a decision to switch after it has already transmitted the frame that would + have had to contain the redundancy. + illustrates the recommended + cross-lapping and decoder resets for these transitions. +
+ S -> S ;S -> S -> S + +NB or MB SILK to Hybrid: S -> S -> S |H -> H -> H + +SILK to CELT without Redundancy: S -> S -> S -> P + & + !C -> C -> C + +Hybrid to NB or MB SILK: H -> H -> H -> c + + + ;S -> S -> S + +Hybrid to CELT without Redundancy: H -> H -> H -> P + & + !C -> C -> C + +CELT to SILK without Redundancy: C -> C -> C -> P + & + ;S -> S -> S + +CELT to Hybrid without Redundancy: C -> C -> C -> P + & + |H -> H -> H + +Key: +S SILK-only frame ; SILK decoder reset +H Hybrid frame | CELT and SILK decoder resets +C CELT-only frame ! CELT decoder reset +c CELT overlap + Direct mixing +P Packet Loss Concealment & Windowed cross-lap +]]> +
+Encoders SHOULD NOT use other transitions, e.g., those that involve redundancy + in ways not illustrated in . +
+ +
+ +
+ +
+ + + + + + +
+ +Just like the decoder, the Opus encoder also normally consists of two main blocks: the +SILK encoder and the CELT encoder. However, unlike the case of the decoder, a valid +(though potentially suboptimal) Opus encoder is not required to support all modes and +may thus only include a SILK encoder module or a CELT encoder module. +The output bit-stream of the Opus encoding contains bits from the SILK and CELT + encoders, though these are not separable due to the use of a range coder. +A block diagram of the encoder is illustrated below. + +
+ +| Rate |--->| Encoder | V + +-----------+ | | Conversion | | | +---------+ + | Optional | | +------------+ +---------+ | Range | +->| High-pass |--+ | Encoder |----> + | Filter | | +--------------+ +---------+ | | Bit- + +-----------+ | | Delay | | CELT | +---------+ stream + +->| Compensation |->| Encoder | ^ + | | | |------+ + +--------------+ +---------+ +]]> + +
+
+ + +For a normal encoder where both the SILK and the CELT modules are included, an optimal +encoder should select which coding mode to use at run-time depending on the conditions. +In the reference implementation, the frame size is selected by the application, but the +other configuration parameters (number of channels, bandwidth, mode) are automatically +selected (unless explicitly overridden by the application) depend on the following: + +Requested bitrate +Input sampling rate +Type of signal (speech vs music) +Frame size in use + + +The type of signal currently needs to be provided by the application (though it can be +changed in real-time). An Opus encoder implementation could also do automatic detection, +but since Opus is an interactive codec, such an implementation would likely have to either +delay the signal (for non-interactive applications) or delay the mode switching decisions (for +interactive applications). + + + +When the encoder is configured for voice over IP applications, the input signal is +filtered by a high-pass filter to remove the lowest part of the spectrum +that contains little speech energy and may contain background noise. This is a second order +Auto Regressive Moving Average (i.e., with poles and zeros) filter with a cut-off frequency around 50 Hz. +In the future, a music detector may also be used to lower the cut-off frequency when the +input signal is detected to be music rather than speech. + + +
+ +The range coder acts as the bit-packer for Opus. +It is used in three different ways: to encode + + +Entropy-coded symbols with a fixed probability model using ec_encode() + (entenc.c), + + +Integers from 0 to (2**M - 1) using ec_enc_uint() or ec_enc_bits() + (entenc.c), + +Integers from 0 to (ft - 1) (where ft is not a power of two) using + ec_enc_uint() (entenc.c). + + + + + +The range encoder maintains an internal state vector composed of the four-tuple + (val, rng, rem, ext) representing the low end of the current + range, the size of the current range, a single buffered output byte, and a + count of additional carry-propagating output bytes. +Both val and rng are 32-bit unsigned integer values, rem is a byte value or + less than 255 or the special value -1, and ext is an unsigned integer with at + least 11 bits. +This state vector is initialized at the start of each each frame to the value + (0, 2**31, -1, 0). +After encoding a sequence of symbols, the value of rng in the encoder should + exactly match the value of rng in the decoder after decoding the same sequence + of symbols. +This is a powerful tool for detecting errors in either an encoder or decoder + implementation. +The value of val, on the other hand, represents different things in the encoder + and decoder, and is not expected to match. + + + +The decoder has no analog for rem and ext. +These are used to perform carry propagation in the renormalization loop below. +Each iteration of this loop produces 9 bits of output, consisting of 8 data + bits and a carry flag. +The encoder cannot determine the final value of the output bytes until it + propagates these carry flags. +Therefore the reference implementation buffers a single non-propagating output + byte (i.e., one less than 255) in rem and keeps a count of additional + propagating (i.e., 255) output bytes in ext. +An implementation may choose to use any mathematically equivalent scheme to + perform carry propagation. + + +
+ +The main encoding function is ec_encode() (entenc.c), which encodes symbol k in + the current context using the same three-tuple (fl[k], fh[k], ft) + as the decoder to describe the range of the symbol (see + ). + + +ec_encode() updates the state of the encoder as follows. +If fl[k] is greater than zero, then +
+ +
+Otherwise, val is unchanged and +
+ +
+The divisions here are integer division. +
+ +
+ +After this update, the range is normalized using a procedure very similar to + that of , implemented by + ec_enc_normalize() (entenc.c). +The following process is repeated until rng > 2**23. +First, the top 9 bits of val, (val>>23), are sent to the carry buffer, + described in . +Then, the encoder sets +
+ +
+
+
+ +
+ +The function ec_enc_carry_out() (entenc.c) implements carry propagation and + output buffering. +It takes as input a 9-bit value, c, consisting of 8 data bits and an additional + carry bit. +If c is equal to the value 255, then ext is simply incremented, and no other + state updates are performed. +Otherwise, let b = (c>>8) be the carry bit. +Then, + + +If the buffered byte rem contains a value other than -1, the encoder outputs + the byte (rem + b). +Otherwise, if rem is -1, no byte is output. + + +If ext is non-zero, then the encoder outputs ext bytes---all with a value of 0 + if b is set, or 255 if b is unset---and sets ext to 0. + + +rem is set to the 8 data bits: +
+ +
+
+
+
+
+ +
+ +
+ +The reference implementation uses three additional encoding methods that are + exactly equivalent to the above, but make assumptions and simplifications that + allow for a more efficient implementation. + + +
+ +The first is ec_encode_bin() (entenc.c), defined using the parameter ftb + instead of ft. +It is mathematically equivalent to calling ec_encode() with + ft = (1<<ftb), but avoids using division. + +
+ +
+ +The next is ec_enc_bit_logp() (entenc.c), which encodes a single binary symbol. +The context is described by a single parameter, logp, which is the absolute + value of the base-2 logarithm of the probability of a "1". +It is mathematically equivalent to calling ec_encode() with the 3-tuple + (fl[k] = 0, fh[k] = (1<<logp) - 1, + ft = (1<<logp)) if k is 0 and with + (fl[k] = (1<<logp) - 1, + fh[k] = ft = (1<<logp)) if k is 1. +The implementation requires no multiplications or divisions. + +
+ +
+ +The last is ec_enc_icdf() (entenc.c), which encodes a single binary symbol with + a table-based context of up to 8 bits. +This uses the same icdf table as ec_dec_icdf() from + . +The function is mathematically equivalent to calling ec_encode() with + fl[k] = (1<<ftb) - icdf[k-1] (or 0 if + k == 0), fh[k] = (1<<ftb) - icdf[k], and + ft = (1<<ftb). +This only saves a few arithmetic operations over ec_encode_bin(), but allows + the encoder to use the same icdf tables as the decoder. + +
+ +
+ +
+ +The raw bits used by the CELT layer are packed at the end of the buffer using + ec_enc_bits() (entenc.c). +Because the raw bits may continue into the last byte output by the range coder + if there is room in the low-order bits, the encoder must be prepared to merge + these values into a single byte. +The procedure in does this in a way that + ensures both the range coded data and the raw bits can be decoded + successfully. + +
+ +
+ +The function ec_enc_uint() (entenc.c) encodes one of ft equiprobable symbols in + the range 0 to (ft - 1), inclusive, each with a frequency of 1, + where ft may be as large as (2**32 - 1). +Like the decoder (see ), it splits up the + value into a range coded symbol representing up to 8 of the high bits, and, if + necessary, raw bits representing the remainder of the value. + + +ec_enc_uint() takes a two-tuple (t, ft), where t is the value to be + encoded, 0 <= t < ft, and ft is not necessarily a + power of two. +Let ftb = ilog(ft - 1), i.e., the number of bits required + to store (ft - 1) in two's complement notation. +If ftb is 8 or less, then t is encoded directly using ec_encode() with the + three-tuple (t, t + 1, ft). + + +If ftb is greater than 8, then the top 8 bits of t are encoded using the + three-tuple (t>>(ftb - 8), + (t>>(ftb - 8)) + 1, + ((ft - 1)>>(ftb - 8)) + 1), and the + remaining bits, + (t & ((1<<(ftb - 8)) - 1), + are encoded as raw bits with ec_enc_bits(). + +
+ +
+ +After all symbols are encoded, the stream must be finalized by outputting a + value inside the current range. +Let end be the integer in the interval [val, val + rng) with the + largest number of trailing zero bits, b, such that + (end + (1<<b) - 1) is also in the interval + [val, val + rng). +This choice of end allows the maximum number of trailing bits to be set to + arbitrary values while still ensuring the range coded part of the buffer can + be decoded correctly. +Then, while end is not zero, the top 9 bits of end, i.e., (end>>23), are + passed to the carry buffer in accordance with the procedure in + , and end is updated via +
+ +
+Finally, if the buffered output byte, rem, is neither zero nor the special + value -1, or the carry count, ext, is greater than zero, then 9 zero bits are + sent to the carry buffer to flush it to the output buffer. +When outputting the final byte from the range coder, if it would overlap any + raw bits already packed into the end of the output buffer, they should be ORed + into the same byte. +The bit allocation routines in the CELT layer should ensure that this can be + done without corrupting the range coder data so long as end is chosen as + described above. +If there is any space between the end of the range coder data and the end of + the raw bits, it is padded with zero bits. +This entire process is implemented by ec_enc_done() (entenc.c). +
+
+ +
+ + The bit allocation routines in Opus need to be able to determine a + conservative upper bound on the number of bits that have been used + to encode the current frame thus far. This drives allocation + decisions and ensures that the range coder and raw bits will not + overflow the output buffer. This is computed in the + reference implementation to whole-bit precision by + the function ec_tell() (entcode.h) and to fractional 1/8th bit + precision by the function ec_tell_frac() (entcode.c). + Like all operations in the range coder, it must be implemented in a + bit-exact manner, and must produce exactly the same value returned by + the same functions in the decoder after decoding the same symbols. + +
+ +
+ +
+ + In many respects the SILK encoder mirrors the SILK decoder described + in . + Details such as the quantization and range coder tables can be found + there, while this section describes the high-level design choices that + were made. + The diagram below shows the basic modules of the SILK encoder. +
+ +| Rate |--->| Mixing |--->| Core |----------> +Input |Conversion| | | | Encoder | Bitstream + +----------+ +--------+ +---------+ +]]> + +
+
+ +
+ +The input signal's sampling rate is adjusted by a sample rate conversion +module so that it matches the SILK internal sampling rate. +The input to the sample rate converter is delayed by a number of samples +depending on the sample rate ratio, such that the overall delay is constant +for all input and output sample rates. + +
+ +
+ +The stereo mixer is only used for stereo input signals. +It converts a stereo left/right signal into an adaptive +mid/side representation. +The first step is to compute non-adaptive mid/side signals +as half the sum and difference between left and right signals. +The side signal is then minimized in energy by subtracting a +prediction of it based on the mid signal. +This prediction works well when the left and right signals +exhibit linear dependency, for instance for an amplitude-panned +input signal. +Like in the decoder, the prediction coefficients are linearly +interpolated during the first 8 ms of the frame. + The mid signal is always encoded, whereas the residual + side signal is only encoded if it has sufficient + energy compared to the mid signal's energy. + If it has not, + the "mid_only_flag" is set without encoding the side signal. + + +The predictor coefficients are coded regardless of whether +the side signal is encoded. +For each frame, two predictor coefficients are computed, one +that predicts between low-passed mid and side channels, and +one that predicts between high-passed mid and side channels. +The low-pass filter is a simple three-tap filter +and creates a delay of one sample. +The high-pass filtered signal is the difference between +the mid signal delayed by one sample and the low-passed +signal. Instead of explicitly computing the high-passed +signal, it is computationally more efficient to transform +the prediction coefficients before applying them to the +filtered mid signal, as follows +
+ + + +
+where w0 and w1 are the low-pass and high-pass prediction +coefficients, mid(n-1) is the mid signal delayed by one sample, +LP(n) and HP(n) are the low-passed and high-passed +signals and pred(n) is the prediction signal that is subtracted +from the side signal. +
+
+ +
+ +What follows is a description of the core encoder and its components. +For simplicity, the core encoder is referred to simply as the encoder in +the remainder of this section. An overview of the encoder is given in +. + +
+ +| | + +---------+ | +---------+ | | + |Voice | | |LTP |12 | | + +-->|Activity |--+ +----->|Scaling |-----------+---->| | + | |Detector |3 | | |Control |<--+ | | | + | +---------+ | | +---------+ | | | | + | | | +---------+ | | | | + | | | |Gains | | | | | + | | | +-->|Processor|---|---+---|---->| R | + | | | | | |11 | | | | a | + | \/ | | +---------+ | | | | n | + | +---------+ | | +---------+ | | | | g | + | |Pitch | | | |LSF | | | | | e | + | +->|Analysis |---+ | |Quantizer|---|---|---|---->| | + | | | |4 | | | |8 | | | | E |--> + | | +---------+ | | +---------+ | | | | n | 2 + | | | | 9/\ 10| | | | | c | + | | | | | \/ | | | | o | + | | +---------+ | | +----------+ | | | | d | + | | |Noise | +--|-->|Prediction|--+---|---|---->| e | + | +->|Shaping |---|--+ |Analysis |7 | | | | r | + | | |Analysis |5 | | | | | | | | | + | | +---------+ | | +----------+ | | | | | + | | | | /\ | | | | | + | | +----------|--|--------+ | | | | | + | | | \/ \/ \/ \/ \/ | | + | | | +---------+ +------------+ | | + | | | | | |Noise | | | +-+-------+-----+------>|Prefilter|--------->|Shaping |-->| | +1 | | 6 |Quantization|13 | | + +---------+ +------------+ +---+ + +1: Input speech signal +2: Range encoded bitstream +3: Voice activity estimate +4: Pitch lags (per 5 ms) and voicing decision (per 20 ms) +5: Noise shaping quantization coefficients + - Short term synthesis and analysis + noise shaping coefficients (per 5 ms) + - Long term synthesis and analysis noise + shaping coefficients (per 5 ms and for voiced speech only) + - Noise shaping tilt (per 5 ms) + - Quantizer gain/step size (per 5 ms) +6: Input signal filtered with analysis noise shaping filters +7: Short and long term prediction coefficients + LTP (per 5 ms) and LPC (per 20 ms) +8: LSF quantization indices +9: LSF coefficients +10: Quantized LSF coefficients +11: Processed gains, and synthesis noise shape coefficients +12: LTP state scaling coefficient. Controlling error propagation + / prediction gain trade-off +13: Quantized signal +]]> + +
+ +
+ +The input signal is processed by a Voice Activity Detector (VAD) to produce +a measure of voice activity, spectral tilt, and signal-to-noise estimates for +each frame. The VAD uses a sequence of half-band filterbanks to split the +signal into four subbands: 0...Fs/16, Fs/16...Fs/8, Fs/8...Fs/4, and +Fs/4...Fs/2, where Fs is the sampling frequency (8, 12, 16, or 24 kHz). +The lowest subband, from 0 - Fs/16, is high-pass filtered with a first-order +moving average (MA) filter (with transfer function H(z) = 1-z**(-1)) to +reduce the energy at the lowest frequencies. For each frame, the signal +energy per subband is computed. +In each subband, a noise level estimator tracks the background noise level +and a Signal-to-Noise Ratio (SNR) value is computed as the logarithm of the +ratio of energy to noise level. +Using these intermediate variables, the following parameters are calculated +for use in other SILK modules: + + +Average SNR. The average of the subband SNR values. + + + +Smoothed subband SNRs. Temporally smoothed subband SNR values. + + + +Speech activity level. Based on the average SNR and a weighted average of the +subband energies. + + + +Spectral tilt. A weighted average of the subband SNRs, with positive weights +for the low subbands and negative weights for the high subbands. + + + +
+ +
+ +The input signal is processed by the open loop pitch estimator shown in +. +
+ +|sampling|->|Correlator| | + | | | | | |4 + | +--------+ +----------+ \/ + | | 2 +-------+ + | | +-->|Speech |5 + +---------+ +--------+ | \/ | |Type |-> + |LPC | |Down | | +----------+ | | + +->|Analysis | +->|sample |-+------------->|Time- | +-------+ + | | | | |to 8 kHz| |Correlator|-----------> + | +---------+ | +--------+ |__________| 6 + | | | |3 + | \/ | \/ + | +---------+ | +----------+ + | |Whitening| | |Time- | +-+->|Filter |-+--------------------------->|Correlator|-----------> +1 | | | | 7 + +---------+ +----------+ + +1: Input signal +2: Lag candidates from stage 1 +3: Lag candidates from stage 2 +4: Correlation threshold +5: Voiced/unvoiced flag +6: Pitch correlation +7: Pitch lags +]]> + +
+The pitch analysis finds a binary voiced/unvoiced classification, and, for +frames classified as voiced, four pitch lags per frame - one for each +5 ms subframe - and a pitch correlation indicating the periodicity of +the signal. +The input is first whitened using a Linear Prediction (LP) whitening filter, +where the coefficients are computed through standard Linear Prediction Coding +(LPC) analysis. The order of the whitening filter is 16 for best results, but +is reduced to 12 for medium complexity and 8 for low complexity modes. +The whitened signal is analyzed to find pitch lags for which the time +correlation is high. +The analysis consists of three stages for reducing the complexity: + +In the first stage, the whitened signal is downsampled to 4 kHz +(from 8 kHz) and the current frame is correlated to a signal delayed +by a range of lags, starting from a shortest lag corresponding to +500 Hz, to a longest lag corresponding to 56 Hz. + + +The second stage operates on an 8 kHz signal (downsampled from 12, 16, +or 24 kHz) and measures time correlations only near the lags +corresponding to those that had sufficiently high correlations in the first +stage. The resulting correlations are adjusted for a small bias towards +short lags to avoid ending up with a multiple of the true pitch lag. +The highest adjusted correlation is compared to a threshold depending on: + + +Whether the previous frame was classified as voiced + + +The speech activity level + + +The spectral tilt. + + +If the threshold is exceeded, the current frame is classified as voiced and +the lag with the highest adjusted correlation is stored for a final pitch +analysis of the highest precision in the third stage. + + +The last stage operates directly on the whitened input signal to compute time +correlations for each of the four subframes independently in a narrow range +around the lag with highest correlation from the second stage. + + +
+
+ +
+ +The noise shaping analysis finds gains and filter coefficients used in the +prefilter and noise shaping quantizer. These parameters are chosen such that +they will fulfill several requirements: + + +Balancing quantization noise and bitrate. +The quantization gains determine the step size between reconstruction levels +of the excitation signal. Therefore, increasing the quantization gain +amplifies quantization noise, but also reduces the bitrate by lowering +the entropy of the quantization indices. + + +Spectral shaping of the quantization noise; the noise shaping quantizer is +capable of reducing quantization noise in some parts of the spectrum at the +cost of increased noise in other parts without substantially changing the +bitrate. +By shaping the noise such that it follows the signal spectrum, it becomes +less audible. In practice, best results are obtained by making the shape +of the noise spectrum slightly flatter than the signal spectrum. + + +De-emphasizing spectral valleys; by using different coefficients in the +analysis and synthesis part of the prefilter and noise shaping quantizer, +the levels of the spectral valleys can be decreased relative to the levels +of the spectral peaks such as speech formants and harmonics. +This reduces the entropy of the signal, which is the difference between the +coded signal and the quantization noise, thus lowering the bitrate. + + +Matching the levels of the decoded speech formants to the levels of the +original speech formants; an adjustment gain and a first order tilt +coefficient are computed to compensate for the effect of the noise +shaping quantization on the level and spectral tilt. + + + + +
+ + + Frequency + +1: Input signal spectrum +2: De-emphasized and level matched spectrum +3: Quantization noise spectrum +]]> + +
+ shows an example of an +input signal spectrum (1). +After de-emphasis and level matching, the spectrum has deeper valleys (2). +The quantization noise spectrum (3) more or less follows the input signal +spectrum, while having slightly less pronounced peaks. +The entropy, which provides a lower bound on the bitrate for encoding the +excitation signal, is proportional to the area between the de-emphasized +spectrum (2) and the quantization noise spectrum (3). Without de-emphasis, +the entropy is proportional to the area between input spectrum (1) and +quantization noise (3) - clearly higher. +
+ + +The transformation from input signal to de-emphasized signal can be +described as a filtering operation with a filter +
+ + + +
+having an adjustment gain G, a first order tilt adjustment filter with +tilt coefficient c_tilt, and where +
+ + + +
+is the analysis part of the de-emphasis filter, consisting of the short-term +shaping filter with coefficients a_ana(k), and the long-term shaping filter +with coefficients b_ana(k) and pitch lag L. +The parameter d determines the number of long-term shaping filter taps. +
+ + +Similarly, but without the tilt adjustment, the synthesis part can be written as +
+ + + +
+
+ +All noise shaping parameters are computed and applied per subframe of 5 ms. +First, an LPC analysis is performed on a windowed signal block of 15 ms. +The signal block has a look-ahead of 5 ms relative to the current subframe, +and the window is an asymmetric sine window. The LPC analysis is done with the +autocorrelation method, with an order of between 8, in lowest-complexity mode, +and 16, for best quality. + + +Optionally the LPC analysis and noise shaping filters are warped by replacing +the delay elements by first-order allpass filters. +This increases the frequency resolution at low frequencies and reduces it at +high ones, which better matches the human auditory system and improves +quality. +The warped analysis and filtering comes at a cost in complexity +and is therefore only done in higher complexity modes. + + +The quantization gain is found by taking the square root of the residual energy +from the LPC analysis and multiplying it by a value inversely proportional +to the coding quality control parameter and the pitch correlation. + + +Next the two sets of short-term noise shaping coefficients a_ana(k) and +a_syn(k) are obtained by applying different amounts of bandwidth expansion to the +coefficients found in the LPC analysis. +This bandwidth expansion moves the roots of the LPC polynomial towards the +origin, using the formulas +
+ + + +
+where a(k) is the k'th LPC coefficient, and the bandwidth expansion factors +g_ana and g_syn are calculated as +
+ + + +
+where C is the coding quality control parameter between 0 and 1. +Applying more bandwidth expansion to the analysis part than to the synthesis +part gives the desired de-emphasis of spectral valleys in between formants. +
+ + +The long-term shaping is applied only during voiced frames. +It uses three filter taps, described by +
+ + + +
+For unvoiced frames these coefficients are set to 0. The multiplication factors +F_ana and F_syn are chosen between 0 and 1, depending on the coding quality +control parameter, as well as the calculated pitch correlation and smoothed +subband SNR of the lowest subband. By having F_ana less than F_syn, +the pitch harmonics are emphasized relative to the valleys in between the +harmonics. +
+ + +The tilt coefficient c_tilt is for unvoiced frames chosen as +
+ + + +
+and as +
+ + + +
+for voiced frames, where V is the voice activity level between 0 and 1. +
+ +The adjustment gain G serves to correct any level mismatch between the original +and decoded signals that might arise from the noise shaping and de-emphasis. +This gain is computed as the ratio of the prediction gain of the short-term +analysis and synthesis filter coefficients. The prediction gain of an LPC +synthesis filter is the square root of the output energy when the filter is +excited by a unit-energy impulse on the input. +An efficient way to compute the prediction gain is by first computing the +reflection coefficients from the LPC coefficients through the step-down +algorithm, and extracting the prediction gain from the reflection coefficients +as +
+ + + +
+where r_k is the k'th reflection coefficient. +
+ + +Initial values for the quantization gains are computed as the square-root of +the residual energy of the LPC analysis, adjusted by the coding quality control +parameter. +These quantization gains are later adjusted based on the results of the +prediction analysis. + +
+ +
+ +The prediction analysis is performed in one of two ways depending on how +the pitch estimator classified the frame. +The processing for voiced and unvoiced speech is described in + and + , respectively. + Inputs to this function include the pre-whitened signal from the + pitch estimator (see ). + + +
+ + For a frame of voiced speech the pitch pulses will remain dominant in the + pre-whitened input signal. + Further whitening is desirable as it leads to higher quality at the same + available bitrate. + To achieve this, a Long-Term Prediction (LTP) analysis is carried out to + estimate the coefficients of a fifth-order LTP filter for each of four + subframes. + The LTP coefficients are quantized using the method described in + , and the quantized LTP + coefficients are used to compute the LTP residual signal. + This LTP residual signal is the input to an LPC analysis where the LPC coefficients are + estimated using Burg's method , such that the residual energy is minimized. + The estimated LPC coefficients are converted to a Line Spectral Frequency (LSF) vector + and quantized as described in . +After quantization, the quantized LSF vector is converted back to LPC +coefficients using the full procedure in . +By using quantized LTP coefficients and LPC coefficients derived from the +quantized LSF coefficients, the encoder remains fully synchronized with the +decoder. +The quantized LPC and LTP coefficients are also used to filter the input +signal and measure residual energy for each of the four subframes. + +
+
+ +For a speech signal that has been classified as unvoiced, there is no need +for LTP filtering, as it has already been determined that the pre-whitened +input signal is not periodic enough within the allowed pitch period range +for LTP analysis to be worth the cost in terms of complexity and bitrate. +The pre-whitened input signal is therefore discarded, and instead the input +signal is used for LPC analysis using Burg's method. +The resulting LPC coefficients are converted to an LSF vector and quantized +as described in the following section. +They are then transformed back to obtain quantized LPC coefficients, which +are then used to filter the input signal and measure residual energy for +each of the four subframes. + +
+ +The main purpose of linear prediction in SILK is to reduce the bitrate by +minimizing the residual energy. +At least at high bitrates, perceptual aspects are handled +independently by the noise shaping filter. +Burg's method is used because it provides higher prediction gain +than the autocorrelation method and, unlike the covariance method, +produces stable filters (assuming numerical errors don't spoil +that). SILK's implementation of Burg's method is also computationally +faster than the autocovariance method. +The implementation of Burg's method differs from traditional +implementations in two aspects. +The first difference is that it +operates on autocorrelations, similar to the Schur algorithm , but +with a simple update to the autocorrelations after finding each +reflection coefficient to make the result identical to Burg's method. +This brings down the complexity of Burg's method to near that of +the autocorrelation method. +The second difference is that the signal in each subframe is scaled +by the inverse of the residual quantization step size. Subframes with +a small quantization step size will on average spend more bits for a +given amount of residual energy than subframes with a large step size. +Without scaling, Burg's method minimizes the total residual energy in +all subframes, which doesn't necessarily minimize the total number of +bits needed for coding the quantized residual. The residual energy +of the scaled subframes is a better measure for that number of +bits. + +
+
+
+ +
+ +Unlike many other speech codecs, SILK uses variable bitrate coding +for the LSFs. +This improves the average rate-distortion (R-D) tradeoff and reduces outliers. +The variable bitrate coding minimizes a linear combination of the weighted +quantization errors and the bitrate. +The weights for the quantization errors are the Inverse +Harmonic Mean Weighting (IHMW) function proposed by Laroia et al. +(see ). +These weights are referred to here as Laroia weights. + + +The LSF quantizer consists of two stages. +The first stage is an (unweighted) vector quantizer (VQ), with a +codebook size of 32 vectors. +The quantization errors for the codebook vector are sorted, and +for the N best vectors a second stage quantizer is run. +By varying the number N a tradeoff is made between R-D performance +and computational efficiency. +For each of the N codebook vectors the Laroia weights corresponding +to that vector (and not to the input vector) are calculated. +Then the residual between the input LSF vector and the codebook +vector is scaled by the square roots of these Laroia weights. +This scaling partially normalizes error sensitivity for the +residual vector, so that a uniform quantizer with fixed +step sizes can be used in the second stage without too much +performance loss. +And by scaling with Laroia weights determined from the first-stage +codebook vector, the process can be reversed in the decoder. + + +The second stage uses predictive delayed decision scalar +quantization. +The quantization error is weighted by Laroia weights determined +from the LSF input vector. +The predictor multiplies the previous quantized residual value +by a prediction coefficient that depends on the vector index from the +first stage VQ and on the location in the LSF vector. +The prediction is subtracted from the LSF residual value before +quantizing the result, and added back afterwards. +This subtraction can be interpreted as shifting the quantization levels +of the scalar quantizer, and as a result the quantization error of +each value depends on the quantization decision of the previous value. +This dependency is exploited by the delayed decision mechanism to +search for a quantization sequency with best R-D performance +with a Viterbi-like algorithm . +The quantizer processes the residual LSF vector in reverse order +(i.e., it starts with the highest residual LSF value). +This is done because the prediction works slightly +better in the reverse direction. + + +The quantization index of the first stage is entropy coded. +The quantization sequence from the second stage is also entropy +coded, where for each element the probability table is chosen +depending on the vector index from the first stage and the location +of that element in the LSF vector. + + +
+ +If the input is stable, finding the best candidate usually results in a +quantized vector that is also stable. Because of the two-stage approach, +however, it is possible that the best quantization candidate is unstable. +The encoder applies the same stabilization procedure applied by the decoder + (see to ensure the LSF parameters + are within their valid range, increasingly sorted, and have minimum + distances between each other and the border values. + +
+
+ +
+ +For voiced frames, the prediction analysis described in + resulted in four sets +(one set per subframe) of five LTP coefficients, plus four weighting matrices. +The LTP coefficients for each subframe are quantized using entropy constrained +vector quantization. +A total of three vector codebooks are available for quantization, with +different rate-distortion trade-offs. The three codebooks have 10, 20, and +40 vectors and average rates of about 3, 4, and 5 bits per vector, respectively. +Consequently, the first codebook has larger average quantization distortion at +a lower rate, whereas the last codebook has smaller average quantization +distortion at a higher rate. +Given the weighting matrix W_ltp and LTP vector b, the weighted rate-distortion +measure for a codebook vector cb_i with rate r_i is give by +
+ + + +
+where u is a fixed, heuristically-determined parameter balancing the distortion +and rate. +Which codebook gives the best performance for a given LTP vector depends on the +weighting matrix for that LTP vector. +For example, for a low valued W_ltp, it is advantageous to use the codebook +with 10 vectors as it has a lower average rate. +For a large W_ltp, on the other hand, it is often better to use the codebook +with 40 vectors, as it is more likely to contain the best codebook vector. +The weighting matrix W_ltp depends mostly on two aspects of the input signal. +The first is the periodicity of the signal; the more periodic, the larger W_ltp. +The second is the change in signal energy in the current subframe, relative to +the signal one pitch lag earlier. +A decaying energy leads to a larger W_ltp than an increasing energy. +Both aspects fluctuate relatively slowly, which causes the W_ltp matrices for +different subframes of one frame often to be similar. +Because of this, one of the three codebooks typically gives good performance +for all subframes, and therefore the codebook search for the subframe LTP +vectors is constrained to only allow codebook vectors to be chosen from the +same codebook, resulting in a rate reduction. +
+ + +To find the best codebook, each of the three vector codebooks is +used to quantize all subframe LTP vectors and produce a combined +weighted rate-distortion measure for each vector codebook. +The vector codebook with the lowest combined rate-distortion +over all subframes is chosen. The quantized LTP vectors are used +in the noise shaping quantizer, and the index of the codebook +plus the four indices for the four subframe codebook vectors +are passed on to the range encoder. + +
+ +
+ +In the prefilter the input signal is filtered using the spectral valley +de-emphasis filter coefficients from the noise shaping analysis +(see ). +By applying only the noise shaping analysis filter to the input signal, +it provides the input to the noise shaping quantizer. + +
+ +
+ +The noise shaping quantizer independently shapes the signal and coding noise +spectra to obtain a perceptually higher quality at the same bitrate. + + +The prefilter output signal is multiplied with a compensation gain G computed +in the noise shaping analysis. Then the output of a synthesis shaping filter +is added, and the output of a prediction filter is subtracted to create a +residual signal. +The residual signal is multiplied by the inverse quantized quantization gain +from the noise shaping analysis, and input to a scalar quantizer. +The quantization indices of the scalar quantizer represent a signal of pulses +that is input to the pyramid range encoder. +The scalar quantizer also outputs a quantization signal, which is multiplied +by the quantized quantization gain from the noise shaping analysis to create +an excitation signal. +The output of the prediction filter is added to the excitation signal to form +the quantized output signal y(n). +The quantized output signal y(n) is input to the synthesis shaping and +prediction filters. + + +Optionally the noise shaping quantizer operates in a delayed decision +mode. +In this mode it uses a Viterbi algorithm to keep track of +multiple rounding choices in the quantizer and select the best +one after a delay of 32 samples. This improves the rate/distortion +performance of the quantizer. + +
+ +
+ + SILK was designed to run in Variable Bitrate (VBR) mode. However + the reference implementation also has a Constant Bitrate (CBR) mode + for SILK. In CBR mode SILK will attempt to encode each packet with + no more than the allowed number of bits. The Opus wrapper code + then pads the bitstream if any unused bits are left in SILK mode, or + encodes the high band with the remaining number of bits in Hybrid mode. + The number of payload bits is adjusted by changing + the quantization gains and the rate/distortion tradeoff in the noise + shaping quantizer, in an iterative loop + around the noise shaping quantizer and entropy coding. + Compared to the SILK VBR mode, the CBR mode has lower + audio quality at a given average bitrate, and also has higher + computational complexity. + +
+ +
+ +
+ + +
+ +Most of the aspects of the CELT encoder can be directly derived from the description +of the decoder. For example, the filters and rotations in the encoder are simply the +inverse of the operation performed by the decoder. Similarly, the quantizers generally +optimize for the mean square error (because noise shaping is part of the bit-stream itself), +so no special search is required. For this reason, only the less straightforward aspects of the +encoder are described here. + + +
+The pitch prefilter is applied after the pre-emphasis. It is applied +in such a way as to be the inverse of the decoder's post-filter. The main non-obvious aspect of the +prefilter is the selection of the pitch period. The pitch search should be optimized for the +following criteria: + +continuity: it is important that the pitch period +does not change abruptly between frames; and +avoidance of pitch multiples: when the period used is a multiple of the real period +(lower frequency fundamental), the post-filter loses most of its ability to reduce noise + + +
+ +
+ +The MDCT output is divided into bands that are designed to match the ear's critical +bands for the smallest (2.5 ms) frame size. The larger frame sizes use integer +multiples of the 2.5 ms layout. For each band, the encoder +computes the energy that will later be encoded. Each band is then normalized by the +square root of the unquantized energy, such that each band now forms a unit vector X. +The energy and the normalization are computed by compute_band_energies() +and normalise_bands() (bands.c), respectively. + +
+ +
+ + +Energy quantization (both coarse and fine) can be easily understood from the decoding process. +For all useful bitrates, the coarse quantizer always chooses the quantized log energy value that +minimizes the error for each band. Only at very low rate does the encoder allow larger errors to +minimize the rate and avoid using more bits than are available. When the +available CPU requirements allow it, it is best to try encoding the coarse energy both with and without +inter-frame prediction such that the best prediction mode can be selected. The optimal mode depends on +the coding rate, the available bitrate, and the current rate of packet loss. + + +The fine energy quantizer always chooses the quantized log energy value that +minimizes the error for each band because the rate of the fine quantization depends only +on the bit allocation and not on the values that are coded. + +
+ +
+The encoder must use exactly the same bit allocation process as used by the decoder +and described in . The three mechanisms that can be used by the +encoder to adjust the bitrate on a frame-by-frame basis are band boost, allocation trim, +and band skipping. + + +
+The reference encoder makes a decision to boost a band when the energy of that band is significantly +higher than that of the neighboring bands. Let E_j be the log-energy of band j, we define + +D_j = 2*E_j - E_j-1 - E_j+1 + + +The allocation of band j is boosted once if D_j > t1 and twice if D_j > t2. For LM>=1, t1=2 and t2=4, +while for LM<1, t1=3 and t2=5. + + +
+ +
+The allocation trim is a value between 0 and 10 (inclusively) that controls the allocation +balance between the low and high frequencies. The encoder starts with a safe "default" of 5 +and deviates from that default in two different ways. First the trim can deviate by +/- 2 +depending on the spectral tilt of the input signal. For signals with more low frequencies, the +trim is increased by up to 2, while for signals with more high frequencies, the trim is +decreased by up to 2. +For stereo inputs, the trim value can +be decreased by up to 4 when the inter-channel correlation at low frequency (first 8 bands) +is high. +
+ +
+The encoder uses band skipping to ensure that the shape of the bands is only coded +if there is at least 1/2 bit per sample available for the PVQ. If not, then no bit is allocated +and folding is used instead. To ensure continuity in the allocation, some amount of hysteresis is +added to the process, such that a band that received PVQ bits in the previous frame only needs 7/16 +bit/sample to be coded for the current frame, while a band that did not receive PVQ bits in the +previous frames needs at least 9/16 bit/sample to be coded. +
+ +
+ +
+Because CELT applies mid-side stereo coupling in the normalized domain, it does not suffer from +important stereo image problems even when the two channels are completely uncorrelated. For this reason +it is always safe to use stereo coupling on any audio frame. That being said, there are some frames +for which dual (independent) stereo is still more efficient. This decision is made by comparing the estimated +entropy with and without coupling over the first 13 bands, taking into account the fact that all bands with +more than two MDCT bins require one extra degree of freedom when coded in mid-side. Let L1_ms and L1_lr +be the L1-norm of the mid-side vector and the L1-norm of the left-right vector, respectively. The decision +to use mid-side is made if and only if +
+ +
+where bins is the number of MDCT bins in the first 13 bands and E is the number of extra degrees of +freedom for mid-side coding. For LM>1, E=13, otherwise E=5. +
+ +The reference encoder decides on the intensity stereo threshold based on the bitrate alone. After +taking into account the frame size by subtracting 80 bits per frame for coarse energy, the first +band using intensity coding is as follows: + + + +bitrate (kb/s) +start band +<35 8 +35-50 12 +50-68 16 +84-84 18 +84-102 19 +102-130 20 +>130 disabled + + + +
+ +
+ +The choice of time-frequency resolution used in is based on +R-D optimization. The distortion is the L1-norm (sum of absolute values) of each band +after each TF resolution under consideration. The L1 norm is used because it represents the entropy +for a Laplacian source. The number of bits required to code a change in TF resolution between +two bands is higher than the cost of having those two bands use the same resolution, which is +what requires the R-D optimization. The optimal decision is computed using the Viterbi algorithm. +See tf_analysis() in celt/celt.c. + +
+ +
+ +The choice of the spreading value in has an +impact on the nature of the coding noise introduced by CELT. The larger the f_r value, the +lower the impact of the rotation, and the more tonal the coding noise. The +more tonal the signal, the more tonal the noise should be, so the CELT encoder determines +the optimal value for f_r by estimating how tonal the signal is. The tonality estimate +is based on discrete pdf (4-bin histogram) of each band. Bands that have a large number of small +values are considered more tonal and a decision is made by combining all bands with more than +8 samples. See spreading_decision() in celt/bands.c. + +
+ +
+CELT uses a Pyramid Vector Quantization (PVQ) +codebook for quantizing the details of the spectrum in each band that have not +been predicted by the pitch predictor. The PVQ codebook consists of all sums +of K signed pulses in a vector of N samples, where two pulses at the same position +are required to have the same sign. Thus the codebook includes +all integer codevectors y of N dimensions that satisfy sum(abs(y(j))) = K. + + + +In bands where there are sufficient bits allocated PVQ is used to encode +the unit vector that results from the normalization in + directly. Given a PVQ codevector y, +the unit vector X is obtained as X = y/||y||, where ||.|| denotes the +L2 norm. + + + +
+ + +The search for the best codevector y is performed by alg_quant() +(vq.c). There are several possible approaches to the +search, with a trade-off between quality and complexity. The method used in the reference +implementation computes an initial codeword y1 by projecting the normalized spectrum +X onto the codebook pyramid of K-1 pulses: + + +y0 = truncate_towards_zero( (K-1) * X / sum(abs(X))) + + + +Depending on N, K and the input data, the initial codeword y0 may contain from +0 to K-1 non-zero values. All the remaining pulses, with the exception of the last one, +are found iteratively with a greedy search that minimizes the normalized correlation +between y and X: +
+ +
+
+ + +The search described above is considered to be a good trade-off between quality +and computational cost. However, there are other possible ways to search the PVQ +codebook and the implementers MAY use any other search methods. See alg_quant() in celt/vq.c. + +
+ +
+ + +The vector to encode, X, is converted into an index i such that + 0 <= i < V(N,K) as follows. +Let i = 0 and k = 0. +Then for j = (N - 1) down to 0, inclusive, do: + + +If k > 0, set + i = i + (V(N-j-1,k-1) + V(N-j,k-1))/2. + +Set k = k + abs(X[j]). + +If X[j] < 0, set + i = i + (V(N-j-1,k) + V(N-j,k))/2. + + + + + +The index i is then encoded using the procedure in + with ft = V(N,K). + + +
+ +
+ + + + + +
+ +
+ + +
+ + +It is our intention to allow the greatest possible choice of freedom in +implementing the specification. For this reason, outside of the exceptions +noted in this section, conformance is defined through the reference +implementation of the decoder provided in . +Although this document includes an English description of the codec, should +the description contradict the source code of the reference implementation, +the latter shall take precedence. + + + +Compliance with this specification means that in addition to following the normative keywords in this document, + a decoder's output MUST also be + within the thresholds specified by the opus_compare.c tool (included + with the code) when compared to the reference implementation for each of the + test vectors provided (see ) and for each output + sampling rate and channel count supported. In addition, a compliant + decoder implementation MUST have the same final range decoder state as that of the + reference decoder. It is therefore RECOMMENDED that the + decoder implement the same functional behavior as the reference. + + A decoder implementation is not required to support all output sampling + rates or all output channel counts. + + +
+ +Using the reference code provided in , +a test vector can be decoded with + +opus_demo -d <rate> <channels> testvectorX.bit testX.out + +where <rate> is the sampling rate and can be 8000, 12000, 16000, 24000, or 48000, and +<channels> is 1 for mono or 2 for stereo. + + + +If the range decoder state is incorrect for one of the frames, the decoder will exit with +"Error: Range coder state mismatch between encoder and decoder". If the decoder succeeds, then +the output can be compared with the "reference" output with + +opus_compare -s -r <rate> testvectorX.dec testX.out + +for stereo or + +opus_compare -r <rate> testvectorX.dec testX.out + +for mono. + + +In addition to indicating whether the test vector comparison passes, the opus_compare tool +outputs an "Opus quality metric" that indicates how well the tested decoder matches the +reference implementation. A quality of 0 corresponds to the passing threshold, while +a quality of 100 is the highest possible value and means that the output of the tested decoder is identical to the reference +implementation. The passing threshold (quality 0) was calibrated in such a way that it corresponds to +additive white noise with a 48 dB SNR (similar to what can be obtained on a cassette deck). +It is still possible for an implementation to sound very good with such a low quality measure +(e.g. if the deviation is due to inaudible phase distortion), but unless this is verified by +listening tests, it is RECOMMENDED that implementations achieve a quality above 90 for 48 kHz +decoding. For other sampling rates, it is normal for the quality metric to be lower +(typically as low as 50 even for a good implementation) because of harmless mismatch with +the delay and phase of the internal sampling rate conversion. + + + +On POSIX environments, the run_vectors.sh script can be used to verify all test +vectors. This can be done with + +run_vectors.sh <exec path> <vector path> <rate> + +where <exec path> is the directory where the opus_demo and opus_compare executables +are built and <vector path> is the directory containing the test vectors. + +
+ +
+ +Opus Custom is an OPTIONAL part of the specification that is defined to +handle special sample rates and frame rates that are not supported by the +main Opus specification. Use of Opus Custom is discouraged for all but very +special applications for which a frame size different from 2.5, 5, 10, or 20 ms is +needed (for either complexity or latency reasons). Because Opus Custom is +optional, streams encoded using Opus Custom cannot be expected to be decodable by all Opus +implementations. Also, because no in-band mechanism exists for specifying the sampling +rate and frame size of Opus Custom streams, out-of-band signaling is required. +In Opus Custom operation, only the CELT layer is available, using the opus_custom_* function +calls in opus_custom.h. + +
+ +
+ +
+ + +Implementations of the Opus codec need to take appropriate security considerations +into account, as outlined in . +It is extremely important for the decoder to be robust against malicious +payloads. +Malicious payloads must not cause the decoder to overrun its allocated memory + or to take an excessive amount of resources to decode. +Although problems +in encoders are typically rarer, the same applies to the encoder. Malicious +audio streams must not cause the encoder to misbehave because this would +allow an attacker to attack transcoding gateways. + + +The reference implementation contains no known buffer overflow or cases where + a specially crafted packet or audio segment could cause a significant increase + in CPU load. +However, on certain CPU architectures where denormalized floating-point + operations are much slower than normal floating-point operations, it is + possible for some audio content (e.g., silence or near-silence) to cause an + increase in CPU load. +Denormals can be introduced by reordering operations in the compiler and depend + on the target architecture, so it is difficult to guarantee that an implementation + avoids them. +For architectures on which denormals are problematic, adding very small + floating-point offsets to the affected signals to prevent significant numbers + of denormalized operations is RECOMMENDED. +Alternatively, it is often possible to configure the hardware to treat + denormals as zero (DAZ). +No such issue exists for the fixed-point reference implementation. + +The reference implementation was validated in the following conditions: + + +Sending the decoder valid packets generated by the reference encoder and + verifying that the decoder's final range coder state matches that of the + encoder. + + +Sending the decoder packets generated by the reference encoder and then + subjected to random corruption. + +Sending the decoder random packets. + +Sending the decoder packets generated by a version of the reference encoder + modified to make random coding decisions (internal fuzzing), including mode + switching, and verifying that the range coder final states match. + + +In all of the conditions above, both the encoder and the decoder were run + inside the Valgrind memory + debugger, which tracks reads and writes to invalid memory regions as well as + the use of uninitialized memory. +There were no errors reported on any of the tested conditions. + +
+ + +
+ +This document has no actions for IANA. + +
+ +
+ +Thanks to all other developers, including Raymond Chen, Soeren Skak Jensen, Gregory Maxwell, +Christopher Montgomery, and Karsten Vandborg Soerensen. We would also +like to thank Igor Dyakonov, Jan Skoglund, and Christian Hoene for their help with subjective testing of the +Opus codec. Thanks to Ralph Giles, John Ridges, Ben Schwartz, Keith Yan, Christian Hoene, Kat Walsh, and many others on the Opus and CELT mailing lists +for their bug reports and feedback. + +
+ +
+The authors agree to grant third parties the irrevocable right to copy, use and distribute +the work (excluding Code Components available under the simplified BSD license), with or +without modification, in any medium, without royalty, provided that, unless separate +permission is granted, redistributed modified works do not contain misleading author, version, +name of work, or endorsement information. +
+ +
+ + + + + + + +Key words for use in RFCs to Indicate Requirement Levels + + + + + + + + + + + +Requirements for an Internet Audio Codec + + + + + +IETF + + +This document provides specific requirements for an Internet audio + codec. These requirements address quality, sample rate, bitrate, + and packet-loss robustness, as well as other desirable properties. + + + + + + + + + + +SILK Speech Codec + + + + + + + + + + + + + + + + + +Robust and Efficient Quantization of Speech LSP Parameters Using Structured Vector Quantization + + + + + + + + + + + + + + + + +Constrained-Energy Lapped Transform (CELT) Codec + + + + + + + + + + + + + + + + + + +Guidelines for the use of Variable Bit Rate Audio with Secure RTP + + + + + + + + + + + + + + +Internet Denial-of-Service Considerations + + + + + +IAB + + +This document provides an overview of possible avenues for denial-of-service (DoS) attack on Internet systems. The aim is to encourage protocol designers and network engineers towards designs that are more robust. We discuss partial solutions that reduce the effectiveness of attacks, and how some solutions might inadvertently open up alternative vulnerabilities. This memo provides information for the Internet community. + + + + + + +Range encoding: An algorithm for removing redundancy from a digitised message + + + + + + + + +Source coding algorithms for fast data compression + + + + + + + + +A Pyramid Vector Quantizer + + + + + + + + +The Computation of Line Spectral Frequencies Using Chebyshev Polynomials + + + + + + + + + + +Valgrind website + + + + + + +Google NetEQ code + + + + + + +Google WebRTC code + + + + + + + +Opus Git Repository + + + + + + +Opus website + + + + + + +Vorbis website + + + + + + +Matroska website + + + + + + +Opus Testvectors (webside) + + + + + + +Opus Testvectors (proceedings) + + + + + + +Line Spectral Pairs +Wikipedia + + + + + +Range Coding +Wikipedia + + + + + +Hadamard Transform +Wikipedia + + + + + +Viterbi Algorithm +Wikipedia + + + + + +White Noise +Wikipedia + + + + + +Linear Prediction +Wikipedia + + + + + +Modified Discrete Cosine Transform +Wikipedia + + + + + +Fast Fourier Transform +Wikipedia + + + + + +Z-transform +Wikipedia + + + + + + +Maximum Entropy Spectral Analysis + + + + + + +A fixed point computation of partial correlation coefficients + + + + + + + + +Analysis/synthesis filter bank design based on time domain aliasing cancellation + + + + + + + + +A High-Quality Speech and Audio Codec With Less Than 10 ms delay + + + + + + + + + + + + +Subdivision of the audible frequency range into critical bands + + + + + + + + + +
+ +This appendix contains the complete source code for the +reference implementation of the Opus codec written in C. By default, +this implementation relies on floating-point arithmetic, but it can be +compiled to use only fixed-point arithmetic by defining the FIXED_POINT +macro. Information on building and using the reference implementation is +available in the README file. + + +The implementation can be compiled with either a C89 or a C99 +compiler. It is reasonably optimized for most platforms such that +only architecture-specific optimizations are likely to be useful. +The FFT used is a slightly modified version of the KISS-FFT library, +but it is easy to substitute any other FFT library. + + + +While the reference implementation does not rely on any +undefined behavior as defined by C89 or C99, +it relies on common implementation-defined behavior +for two's complement architectures: + +Right shifts of negative values are consistent with two's complement arithmetic, so that a>>b is equivalent to floor(a/(2**b)), +For conversion to a signed integer of N bits, the value is reduced modulo 2**N to be within range of the type, +The result of integer division of a negative value is truncated towards zero, and +The compiler provides a 64-bit integer type (a C99 requirement which is supported by most C89 compilers). + + + + +In its current form, the reference implementation also requires the following +architectural characteristics to obtain acceptable performance: + +Two's complement arithmetic, +At least a 16 bit by 16 bit integer multiplier (32-bit result), and +At least a 32-bit adder/accumulator. + + + + +
+ +The complete source code can be extracted from this draft, by running the +following command line: + + + opus_source.tar.gz +]]> + +tar xzvf opus_source.tar.gz + +cd opus_source +make + +On systems where the provided Makefile does not work, the following command line may be used to compile +the source code: + + + + + +On systems where the base64 utility is not present, the following commands can be used instead: + + opus.b64 +]]> +openssl base64 -d -in opus.b64 > opus_source.tar.gz + + + +
+ +
+ +As of the time of publication of this memo, an up-to-date implementation conforming to +this standard is available in a + Git repository. +Releases and other resources are available at + . However, although that implementation is expected to + remain conformant with the standard, it is the code in this document that shall + remain normative. + +
+ +
+ + + +
+ +
+ +Because of size constraints, the Opus test vectors are not distributed in this +draft. They are available in the proceedings of the 83th IETF meeting (Paris) and from the Opus codec website at +. These test vectors were created specifically to exercise +all aspects of the decoder and therefore the audio quality of the decoded output is +significantly lower than what Opus can achieve in normal operation. + + + +The SHA1 hash of the files in the test vector package are + + + +
+ +
+ +
+ +To use the internal framing described in , the decoder + must know the total length of the Opus packet, in bytes. +This section describes a simple variation of that framing which can be used + when the total length of the packet is not known. +Nothing in the encoding of the packet itself allows a decoder to distinguish + between the regular, undelimited framing and the self-delimiting framing + described in this appendix. +Which one is used and where must be established by context at the transport + layer. +It is RECOMMENDED that a transport layer choose exactly one framing scheme, + rather than allowing an encoder to signal which one it wants to use. + + + +For example, although a regular Opus stream does not support more than two + channels, a multi-channel Opus stream may be formed from several one- and + two-channel streams. +To pack an Opus packet from each of these streams together in a single packet + at the transport layer, one could use the self-delimiting framing for all but + the last stream, and then the regular, undelimited framing for the last one. +Reverting to the undelimited framing for the last stream saves overhead + (because the total size of the transport-layer packet will still be known), + and ensures that a "multi-channel" stream which only has a single Opus stream + uses the same framing as a regular Opus stream does. +This avoids the need for signaling to distinguish these two cases. + + + +The self-delimiting framing is identical to the regular, undelimited framing + from , except that each Opus packet contains one extra + length field, encoded using the same one- or two-byte scheme from + . +This extra length immediately precedes the compressed data of the first Opus + frame in the packet, and is interpreted in the various modes as follows: + + +Code 0 packets: It is the length of the single Opus frame (see + ). + + +Code 1 packets: It is the length used for both of the Opus frames (see + ). + + +Code 2 packets: It is the length of the second Opus frame (see + ). + +CBR Code 3 packets: It is the length used for all of the Opus frames (see + ). + +VBR Code 3 packets: It is the length of the last Opus frame (see + ). + + + + +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-payload-rtp-opus.xml b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-payload-rtp-opus.xml new file mode 100644 index 000000000..c4eb21077 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/draft-ietf-payload-rtp-opus.xml @@ -0,0 +1,960 @@ + + + + + + + + + + + + + + + + + + + + + + ]> + + + + + + + + + + + + + + + + + + RTP Payload Format for the Opus Speech and Audio Codec + + + +
+ jspittka@gmail.com +
+
+ + + vocTone +
+ + + + + + + + koenvos74@gmail.com +
+
+ + + Mozilla +
+ + 331 E. Evelyn Avenue + Mountain View + CA + 94041 + USA + + jmvalin@jmvalin.ca +
+
+ + + + + + This document defines the Real-time Transport Protocol (RTP) payload + format for packetization of Opus encoded + speech and audio data necessary to integrate the codec in the + most compatible way. It also provides an applicability statement + for the use of Opus over RTP. Further, it describes media type registrations + for the RTP payload format. + + +
+ + +
+ + Opus is a speech and audio codec developed within the + IETF Internet Wideband Audio Codec working group. The codec + has a very low algorithmic delay and it + is highly scalable in terms of audio bandwidth, bitrate, and + complexity. Further, it provides different modes to efficiently encode speech signals + as well as music signals, thus making it the codec of choice for + various applications using the Internet or similar networks. + + + This document defines the Real-time Transport Protocol (RTP) + payload format for packetization + of Opus encoded speech and audio data necessary to + integrate Opus in the + most compatible way. It also provides an applicability statement + for the use of Opus over RTP. + Further, it describes media type registrations for + the RTP payload format. + +
+ +
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in . + + + The range of audio frequecies being coded + Constant bitrate + Central Processing Unit + Discontinuous transmission + Forward error correction + Internet Protocol + Speech or audio samples (per channel) + Session Description Protocol + Variable bitrate + + + + Throughout this document, we refer to the following definitions: + + + Abbreviation + Name + Audio Bandwidth (Hz) + Sampling Rate (Hz) + NB + Narrowband + 0 - 4000 + 8000 + + MB + Mediumband + 0 - 6000 + 12000 + + WB + Wideband + 0 - 8000 + 16000 + + SWB + Super-wideband + 0 - 12000 + 24000 + + FB + Fullband + 0 - 20000 + 48000 + + + Audio bandwidth naming + + +
+ +
+ + Opus encodes speech + signals as well as general audio signals. Two different modes can be + chosen, a voice mode or an audio mode, to allow the most efficient coding + depending on the type of the input signal, the sampling frequency of the + input signal, and the intended application. + + + + The voice mode allows efficient encoding of voice signals at lower bit + rates while the audio mode is optimized for general audio signals at medium and + higher bitrates. + + + + Opus is highly scalable in terms of audio + bandwidth, bitrate, and complexity. Further, Opus allows + transmitting stereo signals with in-band signaling in the bit-stream. + + +
+ + Opus supports bitrates from 6 kb/s to 510 kb/s. + The bitrate can be changed dynamically within that range. + All + other parameters being + equal, higher bitrates result in higher audio quality. + +
+ + For a frame size of + 20 ms, these + are the bitrate "sweet spots" for Opus in various configurations: + + + 8-12 kb/s for NB speech, + 16-20 kb/s for WB speech, + 28-40 kb/s for FB speech, + 48-64 kb/s for FB mono music, and + 64-128 kb/s for FB stereo music. + + +
+
+ + For the same average bitrate, variable bitrate (VBR) can achieve higher audio quality + than constant bitrate (CBR). For the majority of voice transmission applications, VBR + is the best choice. One reason for choosing CBR is the potential + information leak that might occur when encrypting the + compressed stream. See for guidelines on when VBR is + appropriate for encrypted audio communications. In the case where an existing + VBR stream needs to be converted to CBR for security reasons, then the Opus padding + mechanism described in is the RECOMMENDED way to achieve padding + because the RTP padding bit is unencrypted. + + + The bitrate can be adjusted at any point in time. To avoid congestion, + the average bitrate SHOULD NOT exceed the available + network bandwidth. If no target bitrate is specified, the bitrates specified in + are RECOMMENDED. + + +
+ +
+ + + Opus can, as described in , + be operated with a variable bitrate. In that case, the encoder will + automatically reduce the bitrate for certain input signals, like periods + of silence. When using continuous transmission, it will reduce the + bitrate when the characteristics of the input signal permit, but + will never interrupt the transmission to the receiver. Therefore, the + received signal will maintain the same high level of audio quality over the + full duration of a transmission while minimizing the average bit + rate over time. + + + + In cases where the bitrate of Opus needs to be reduced even + further or in cases where only constant bitrate is available, + the Opus encoder can use discontinuous + transmission (DTX), where parts of the encoded signal that + correspond to periods of silence in the input speech or audio signal + are not transmitted to the receiver. A receiver can distinguish + between DTX and packet loss by looking for gaps in the sequence + number, as described by Section 4.1 + of . + + + + On the receiving side, the non-transmitted parts will be handled by a + frame loss concealment unit in the Opus decoder which generates a + comfort noise signal to replace the non transmitted parts of the + speech or audio signal. Use of Comfort + Noise (CN) with Opus is discouraged. + The transmitter MUST drop whole frames only, + based on the size of the last transmitted frame, + to ensure successive RTP timestamps differ by a multiple of 120 and + to allow the receiver to use whole frames for concealment. + + + + DTX can be used with both variable and constant bitrate. + It will have a slightly lower speech or audio + quality than continuous transmission. Therefore, using continuous + transmission is RECOMMENDED unless constraints on available network bandwidth + are severe. + + +
+ +
+ +
+ + + Complexity of the encoder can be scaled to optimize for CPU resources in real-time, mostly as + a trade-off between audio quality and bitrate. Also, different modes of Opus have different complexity. + + +
+ +
+ + + The voice mode of Opus allows for embedding "in-band" forward error correction (FEC) + data into the Opus bit stream. This FEC scheme adds + redundant information about the previous packet (N-1) to the current + output packet N. For + each frame, the encoder decides whether to use FEC based on (1) an + externally-provided estimate of the channel's packet loss rate; (2) an + externally-provided estimate of the channel's capacity; (3) the + sensitivity of the audio or speech signal to packet loss; (4) whether + the receiving decoder has indicated it can take advantage of "in-band" + FEC information. The decision to send "in-band" FEC information is + entirely controlled by the encoder and therefore no special precautions + for the payload have to be taken. + + + + On the receiving side, the decoder can take advantage of this + additional information when it loses a packet and the next packet + is available. In order to use the FEC data, the jitter buffer needs + to provide access to payloads with the FEC data. + Instead of performing loss concealment for a missing packet, the + receiver can then configure its decoder to decode the FEC data from the next packet. + + + + Any compliant Opus decoder is capable of ignoring + FEC information when it is not needed, so encoding with FEC cannot cause + interoperability problems. + However, if FEC cannot be used on the receiving side, then FEC + SHOULD NOT be used, as it leads to an inefficient usage of network + resources. Decoder support for FEC SHOULD be indicated at the time a + session is set up. + + +
+ +
+ + + Opus allows for transmission of stereo audio signals. This operation + is signaled in-band in the Opus bit-stream and no special arrangement + is needed in the payload format. An + Opus decoder is capable of handling a stereo encoding, but an + application might only be capable of consuming a single audio + channel. + + + If a decoder cannot take advantage of the benefits of a stereo signal + this SHOULD be indicated at the time a session is set up. In that case + the sending side SHOULD NOT send stereo signals as it leads to an + inefficient usage of network resources. + + +
+ +
+ +
+ The payload format for Opus consists of the RTP header and Opus payload + data. +
+ The format of the RTP header is specified in . + The use of the fields of the RTP header by the Opus payload format is + consistent with that specification. + + The payload length of Opus is an integer number of octets and + therefore no padding is necessary. The payload MAY be padded by an + integer number of octets according to , + although the Opus internal padding is preferred. + + The timestamp, sequence number, and marker bit (M) of the RTP header + are used in accordance with Section 4.1 + of . + + The RTP payload type for Opus is to be assigned dynamically. + + The receiving side MUST be prepared to receive duplicate RTP + packets. The receiver MUST provide at most one of those payloads to the + Opus decoder for decoding, and MUST discard the others. + + Opus supports 5 different audio bandwidths, which can be adjusted during + a stream. + The RTP timestamp is incremented with a 48000 Hz clock rate + for all modes of Opus and all sampling rates. + The unit + for the timestamp is samples per single (mono) channel. The RTP timestamp corresponds to the + sample time of the first encoded sample in the encoded frame. + For data encoded with sampling rates other than 48000 Hz, + the sampling rate has to be adjusted to 48000 Hz. + +
+ +
+ + The Opus encoder can output encoded frames representing 2.5, 5, 10, 20, + 40, or 60 ms of speech or audio data. Further, an arbitrary number of frames can be + combined into a packet, up to a maximum packet duration representing + 120 ms of speech or audio data. The grouping of one or more Opus + frames into a single Opus packet is defined in Section 3 of + . An RTP payload MUST contain exactly one + Opus packet as defined by that document. + + + shows the structure combined with the RTP header. + +
+ + + +
+ + + shows supported frame sizes in + milliseconds of encoded speech or audio data for the speech and audio modes + (Mode) and sampling rates (fs) of Opus and shows how the timestamp is + incremented for packetization (ts incr). If the Opus encoder + outputs multiple encoded frames into a single packet, the timestamp + increment is the sum of the increments for the individual frames. + + + + Mode + fs + 2.5 + 5 + 10 + 20 + 40 + 60 + ts incr + all + 120 + 240 + 480 + 960 + 1920 + 2880 + voice + NB/MB/WB/SWB/FB + x + x + o + o + o + o + audio + NB/WB/SWB/FB + o + o + o + o + x + x + + +
+ +
+ +
+ + The target bitrate of Opus can be adjusted at any point in time, thus + allowing efficient congestion control. Furthermore, the amount + of encoded speech or audio data encoded in a + single packet can be used for congestion control, since the transmission + rate is inversely proportional to the packet duration. A lower packet + transmission rate reduces the amount of header overhead, but at the same + time increases latency and loss sensitivity, so it ought to be used with + care. + + Since UDP does not provide congestion control, applications that use + RTP over UDP SHOULD implement their own congestion control above the + UDP layer . Work in the rmcat working group + describes the + interactions and conceptual interfaces necessary between the application + components that relate to congestion control, including the RTP layer, + the higher-level media codec control layer, and the lower-level + transport interface, as well as components dedicated to congestion + control functions. +
+ +
+ One media subtype (audio/opus) has been defined and registered as + described in the following section. + +
+ Media type registration is done according to and . + + Type name: audio + Subtype name: opus + + Required parameters: + + the RTP timestamp is incremented with a + 48000 Hz clock rate for all modes of Opus and all sampling + rates. For data encoded with sampling rates other than 48000 Hz, + the sampling rate has to be adjusted to 48000 Hz. + + + + Optional parameters: + + + + a hint about the maximum output sampling rate that the receiver is + capable of rendering in Hz. + The decoder MUST be capable of decoding + any audio bandwidth but due to hardware limitations only signals + up to the specified sampling rate can be played back. Sending signals + with higher audio bandwidth results in higher than necessary network + usage and encoding complexity, so an encoder SHOULD NOT encode + frequencies above the audio bandwidth specified by maxplaybackrate. + This parameter can take any value between 8000 and 48000, although + commonly the value will match one of the Opus bandwidths + (). + By default, the receiver is assumed to have no limitations, i.e. 48000. + + + + + a hint about the maximum input sampling rate that the sender is likely to produce. + This is not a guarantee that the sender will never send any higher bandwidth + (e.g. it could send a pre-recorded prompt that uses a higher bandwidth), but it + indicates to the receiver that frequencies above this maximum can safely be discarded. + This parameter is useful to avoid wasting receiver resources by operating the audio + processing pipeline (e.g. echo cancellation) at a higher rate than necessary. + This parameter can take any value between 8000 and 48000, although + commonly the value will match one of the Opus bandwidths + (). + By default, the sender is assumed to have no limitations, i.e. 48000. + + + + the maximum duration of media represented + by a packet (according to Section 6 of + ) that a decoder wants to receive, in + milliseconds rounded up to the next full integer value. + Possible values are 3, 5, 10, 20, 40, 60, or an arbitrary + multiple of an Opus frame size rounded up to the next full integer + value, up to a maximum value of 120, as + defined in . If no value is + specified, the default is 120. + + + the preferred duration of media represented + by a packet (according to Section 6 of + ) that a decoder wants to receive, in + milliseconds rounded up to the next full integer value. + Possible values are 3, 5, 10, 20, 40, 60, or an arbitrary + multiple of an Opus frame size rounded up to the next full integer + value, up to a maximum value of 120, as defined in . If no value is + specified, the default is 20. + + + specifies the maximum average + receive bitrate of a session in bits per second (b/s). The actual + value of the bitrate can vary, as it is dependent on the + characteristics of the media in a packet. Note that the maximum + average bitrate MAY be modified dynamically during a session. Any + positive integer is allowed, but values outside the range + 6000 to 510000 SHOULD be ignored. If no value is specified, the + maximum value specified in + for the corresponding mode of Opus and corresponding maxplaybackrate + is the default. + + + specifies whether the decoder prefers receiving stereo or mono signals. + Possible values are 1 and 0 where 1 specifies that stereo signals are preferred, + and 0 specifies that only mono signals are preferred. + Independent of the stereo parameter every receiver MUST be able to receive and + decode stereo signals but sending stereo signals to a receiver that signaled a + preference for mono signals may result in higher than necessary network + utilization and encoding complexity. If no value is specified, + the default is 0 (mono). + + + + specifies whether the sender is likely to produce stereo audio. + Possible values are 1 and 0, where 1 specifies that stereo signals are likely to + be sent, and 0 specifies that the sender will likely only send mono. + This is not a guarantee that the sender will never send stereo audio + (e.g. it could send a pre-recorded prompt that uses stereo), but it + indicates to the receiver that the received signal can be safely downmixed to mono. + This parameter is useful to avoid wasting receiver resources by operating the audio + processing pipeline (e.g. echo cancellation) in stereo when not necessary. + If no value is specified, the default is 0 + (mono). + + + + specifies if the decoder prefers the use of a constant bitrate versus + variable bitrate. Possible values are 1 and 0, where 1 specifies constant + bitrate and 0 specifies variable bitrate. If no value is specified, + the default is 0 (vbr). When cbr is 1, the maximum average bitrate can still + change, e.g. to adapt to changing network conditions. + + + specifies that the decoder has the capability to + take advantage of the Opus in-band FEC. Possible values are 1 and 0. + Providing 0 when FEC cannot be used on the receiving side is + RECOMMENDED. If no + value is specified, useinbandfec is assumed to be 0. + This parameter is only a preference and the receiver MUST be able to process + packets that include FEC information, even if it means the FEC part is discarded. + + + specifies if the decoder prefers the use of + DTX. Possible values are 1 and 0. If no value is specified, the + default is 0. + + + Encoding considerations: + + The Opus media type is framed and consists of binary data according + to Section 4.8 in . + + + Security considerations: + + See of this document. + + + Interoperability considerations: none + Published specification: RFC [XXXX] + Note to the RFC Editor: Replace [XXXX] with the number of the published + RFC. + + Applications that use this media type: + + Any application that requires the transport of + speech or audio data can use this media type. Some examples are, + but not limited to, audio and video conferencing, Voice over IP, + media streaming. + + + Fragment identifier considerations: N/A + + Person & email address to contact for further information: + + SILK Support silksupport@skype.net + Jean-Marc Valin jmvalin@jmvalin.ca + + + Intended usage: COMMON + + Restrictions on usage: + + + For transfer over RTP, the RTP payload format ( of this document) SHALL be + used. + + + Author: + + Julian Spittka jspittka@gmail.com + Koen Vos koenvos74@gmail.com + Jean-Marc Valin jmvalin@jmvalin.ca + + + Change controller: IETF Payload Working Group delegated from the IESG +
+
+ +
+ The information described in the media type specification has a + specific mapping to fields in the Session Description Protocol (SDP) + , which is commonly used to describe RTP + sessions. When SDP is used to specify sessions employing Opus, + the mapping is as follows: + + + + The media type ("audio") goes in SDP "m=" as the media name. + + The media subtype ("opus") goes in SDP "a=rtpmap" as the encoding + name. The RTP clock rate in "a=rtpmap" MUST be 48000 and the number of + channels MUST be 2. + + The OPTIONAL media type parameters "ptime" and "maxptime" are + mapped to "a=ptime" and "a=maxptime" attributes, respectively, in the + SDP. + + The OPTIONAL media type parameters "maxaveragebitrate", + "maxplaybackrate", "stereo", "cbr", "useinbandfec", and + "usedtx", when present, MUST be included in the "a=fmtp" attribute + in the SDP, expressed as a media type string in the form of a + semicolon-separated list of parameter=value pairs (e.g., + maxplaybackrate=48000). They MUST NOT be specified in an + SSRC-specific "fmtp" source-level attribute (as defined in + Section 6.3 of ). + + The OPTIONAL media type parameters "sprop-maxcapturerate", + and "sprop-stereo" MAY be mapped to the "a=fmtp" SDP attribute by + copying them directly from the media type parameter string as part + of the semicolon-separated list of parameter=value pairs (e.g., + sprop-stereo=1). These same OPTIONAL media type parameters MAY also + be specified using an SSRC-specific "fmtp" source-level attribute + as described in Section 6.3 of . + They MAY be specified in both places, in which case the parameter + in the source-level attribute overrides the one found on the + "a=fmtp" line. The value of any parameter which is not specified in + a source-level source attribute MUST be taken from the "a=fmtp" + line, if it is present there. + + + + + Below are some examples of SDP session descriptions for Opus: + + Example 1: Standard mono session with 48000 Hz clock rate +
+ + + +
+ + + Example 2: 16000 Hz clock rate, maximum packet size of 40 ms, + recommended packet size of 40 ms, maximum average bitrate of 20000 bps, + prefers to receive stereo but only plans to send mono, FEC is desired, + DTX is not desired + +
+ + + +
+ + Example 3: Two-way full-band stereo preferred + +
+ + + +
+ + +
+ + When using the offer-answer procedure described in to negotiate the use of Opus, the following + considerations apply: + + + + Opus supports several clock rates. For signaling purposes only + the highest, i.e. 48000, is used. The actual clock rate of the + corresponding media is signaled inside the payload and is not + restricted by this payload format description. The decoder MUST be + capable of decoding every received clock rate. An example + is shown below: + +
+ + + +
+
+ + The "ptime" and "maxptime" parameters are unidirectional + receive-only parameters and typically will not compromise + interoperability; however, some values might cause application + performance to suffer. defines the SDP offer-answer handling of the + "ptime" parameter. The "maxptime" parameter MUST be handled in the + same way. + + + The "maxplaybackrate" parameter is a unidirectional receive-only + parameter that reflects limitations of the local receiver. When + sending to a single destination, a sender MUST NOT use an audio + bandwidth higher than necessary to make full use of audio sampled at + a sampling rate of "maxplaybackrate". Gateways or senders that + are sending the same encoded audio to multiple destinations + SHOULD NOT use an audio bandwidth higher than necessary to + represent audio sampled at "maxplaybackrate", as this would lead + to inefficient use of network resources. + The "maxplaybackrate" parameter does not + affect interoperability. Also, this parameter SHOULD NOT be used + to adjust the audio bandwidth as a function of the bitrate, as this + is the responsibility of the Opus encoder implementation. + + + The "maxaveragebitrate" parameter is a unidirectional receive-only + parameter that reflects limitations of the local receiver. The sender + of the other side MUST NOT send with an average bitrate higher than + "maxaveragebitrate" as it might overload the network and/or + receiver. The "maxaveragebitrate" parameter typically will not + compromise interoperability; however, some values might cause + application performance to suffer, and ought to be set with + care. + + The "sprop-maxcapturerate" and "sprop-stereo" parameters are + unidirectional sender-only parameters that reflect limitations of + the sender side. + They allow the receiver to set up a reduced-complexity audio + processing pipeline if the sender is not planning to use the full + range of Opus's capabilities. + Neither "sprop-maxcapturerate" nor "sprop-stereo" affect + interoperability and the receiver MUST be capable of receiving any signal. + + + + The "stereo" parameter is a unidirectional receive-only + parameter. When sending to a single destination, a sender MUST + NOT use stereo when "stereo" is 0. Gateways or senders that are + sending the same encoded audio to multiple destinations SHOULD + NOT use stereo when "stereo" is 0, as this would lead to + inefficient use of network resources. The "stereo" parameter does + not affect interoperability. + + + + The "cbr" parameter is a unidirectional receive-only + parameter. + + + The "useinbandfec" parameter is a unidirectional receive-only + parameter. + + The "usedtx" parameter is a unidirectional receive-only + parameter. + + Any unknown parameter in an offer MUST be ignored by the receiver + and MUST be removed from the answer. + +
+ + + The Opus parameters in an SDP Offer/Answer exchange are completely + orthogonal, and there is no relationship between the SDP Offer and + the Answer. + +
+ +
+ + For declarative use of SDP such as in Session Announcement Protocol + (SAP), , and RTSP, , for + Opus, the following needs to be considered: + + + + The values for "maxptime", "ptime", "maxplaybackrate", and + "maxaveragebitrate" ought to be selected carefully to ensure that a + reasonable performance can be achieved for the participants of a session. + + + The values for "maxptime", "ptime", and of the payload + format configuration are recommendations by the decoding side to ensure + the best performance for the decoder. + + + All other parameters of the payload format configuration are declarative + and a participant MUST use the configurations that are provided for + the session. More than one configuration can be provided if necessary + by declaring multiple RTP payload types; however, the number of types + ought to be kept small. + +
+
+ +
+ + Use of variable bitrate (VBR) is subject to the security considerations in + . + + RTP packets using the payload format defined in this specification + are subject to the security considerations discussed in the RTP + specification , and in any applicable RTP profile such as + RTP/AVP , RTP/AVPF , + RTP/SAVP or RTP/SAVPF . + However, as "Securing the RTP Protocol Framework: + Why RTP Does Not Mandate a Single Media Security Solution" + discusses, it is not an RTP payload + format's responsibility to discuss or mandate what solutions are used + to meet the basic security goals like confidentiality, integrity and + source authenticity for RTP in general. This responsibility lays on + anyone using RTP in an application. They can find guidance on + available security mechanisms and important considerations in Options + for Securing RTP Sessions [I-D.ietf-avtcore-rtp-security-options]. + Applications SHOULD use one or more appropriate strong security + mechanisms. + + This payload format and the Opus encoding do not exhibit any + significant non-uniformity in the receiver-end computational load and thus + are unlikely to pose a denial-of-service threat due to the receipt of + pathological datagrams. +
+ +
+ Many people have made useful comments and suggestions contributing to this document. + In particular, we would like to thank + Tina le Grand, Cullen Jennings, Jonathan Lennox, Gregory Maxwell, Colin Perkins, Jan Skoglund, + Timothy B. Terriberry, Martin Thompson, Justin Uberti, Magnus Westerlund, and Mo Zanaty. +
+
+ + + + &rfc2119; + &rfc3389; + &rfc3550; + &rfc3711; + &rfc3551; + &rfc6838; + &rfc4855; + &rfc4566; + &rfc3264; + &rfc2326; + &rfc5576; + &rfc6562; + &rfc6716; + + + + &rfc2974; + &rfc4585; + &rfc5124; + &rfc5405; + &rfc7202; + + + + rmcat documents + + + + + + + + + + + +
diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/footer.html b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/footer.html new file mode 100644 index 000000000..346c40aba --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/footer.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + +
+For more information visit the Opus Website. + + +
+ + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/header.html b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/header.html new file mode 100644 index 000000000..babdcf6d7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/header.html @@ -0,0 +1,61 @@ + + + + + + + + +$projectname: $title +$title + + + +$treeview +$search +$mathjax + +$extrastylesheet + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
Opus
+
+ + +
+
$projectbrief
+
$projectnumber +
+
+
$projectbrief
+
$searchbox
+
+ + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/meson.build new file mode 100644 index 000000000..ee42a2b02 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/meson.build @@ -0,0 +1,33 @@ +have_dot = find_program('dot', required: false).found() + +doxyfile_conf = configuration_data() +doxyfile_conf.set('VERSION', opus_version) +doxyfile_conf.set('HAVE_DOT', have_dot) +doxyfile_conf.set('top_srcdir', top_srcdir) +doxyfile_conf.set('top_builddir', top_builddir) + +doxyfile = configure_file(input: 'Doxyfile.in', + output: 'Doxyfile', + configuration: doxyfile_conf, + install: false) + +docdir = join_paths(get_option('datadir'), get_option('docdir')) + +doc_inputs = [ + 'customdoxygen.css', + 'footer.html', + 'header.html', + 'opus_logo.svg', + top_srcdir + '/include/opus.h', + top_srcdir + '/include/opus_multistream.h', + top_srcdir + '/include/opus_defines.h', + top_srcdir + '/include/opus_types.h', + top_srcdir + '/include/opus_custom.h', +] + +custom_target('doc', + input: [ doxyfile ] + doc_inputs, + output: [ 'html' ], + command: [ doxygen, doxyfile ], + install_dir: docdir, + install: true) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.css b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.css new file mode 100644 index 000000000..bffe8f45a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.css @@ -0,0 +1,60 @@ +/* Normal links */ +.normal_link a:link +{ + color : yellow; +} +.normal_link a:visited +{ + color : green; +} + +/* Boxes */ +.pre +{ + white-space: pre; /* CSS 2.0 */ + white-space: pre-wrap; /* CSS 2.1 */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: -moz-pre-wrap; /* Mozilla */ + white-space: -hp-pre-wrap; /* HP Printers */ + word-wrap : break-word; /* IE 5+ */ +} + +.title_box +{ + width : 470px; + height : 70px; + margin : 2px 50px 2px 2px; + padding : 10px; + border : 1px solid black; + background-color : #666666; + white-space : pre; + float : left; + text-align : center; + color : #C0C0C0; + font-size : 50pt; + font-style : italic; +} + +.subindex_box +{ + margin : 5px; + padding : 14px 22px; + border : 1px solid black; + background-color : #778877; + float : left; + text-align : center; + color : #115555; + font-size : 32pt; +} + +.frame_box +{ + margin : 10px; + padding : 10px; + border : 0px; + background-color : #084040; + text-align : left; + color : #C0C0C0; + font-family : monospace; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.html b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.html new file mode 100644 index 000000000..38aefbf2a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_in_isobmff.html @@ -0,0 +1,692 @@ + + + + + + Encapsulation of Opus in ISO Base Media File Format + + + Encapsulation of Opus in ISO Base Media File Format
+ last updated: August 28, 2018
+
+ + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_logo.svg b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_logo.svg new file mode 100644 index 000000000..db2879efc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_logo.svg @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_update.patch b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_update.patch new file mode 100644 index 000000000..11f066c70 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/opus_update.patch @@ -0,0 +1,244 @@ +diff --git a/celt/bands.c b/celt/bands.c +index 6962587..32e1de6 100644 +--- a/celt/bands.c ++++ b/celt/bands.c +@@ -1234,9 +1234,23 @@ void quant_all_bands(int encode, const CELTMode *m, int start, int end, + b = 0; + } + +- if (resynth && M*eBands[i]-N >= M*eBands[start] && (update_lowband || lowband_offset==0)) ++ if (resynth && (M*eBands[i]-N >= M*eBands[start] || i==start+1) && (update_lowband || lowband_offset==0)) + lowband_offset = i; + ++ if (i == start+1) ++ { ++ int n1, n2; ++ int offset; ++ n1 = M*(eBands[start+1]-eBands[start]); ++ n2 = M*(eBands[start+2]-eBands[start+1]); ++ offset = M*eBands[start]; ++ /* Duplicate enough of the first band folding data to be able to fold the second band. ++ Copies no data for CELT-only mode. */ ++ OPUS_COPY(&norm[offset+n1], &norm[offset+2*n1 - n2], n2-n1); ++ if (C==2) ++ OPUS_COPY(&norm2[offset+n1], &norm2[offset+2*n1 - n2], n2-n1); ++ } ++ + tf_change = tf_res[i]; + if (i>=m->effEBands) + { +@@ -1257,7 +1271,7 @@ void quant_all_bands(int encode, const CELTMode *m, int start, int end, + fold_start = lowband_offset; + while(M*eBands[--fold_start] > effective_lowband); + fold_end = lowband_offset-1; +- while(M*eBands[++fold_end] < effective_lowband+N); ++ while(++fold_end < i && M*eBands[fold_end] < effective_lowband+N); + x_cm = y_cm = 0; + fold_i = fold_start; do { + x_cm |= collapse_masks[fold_i*C+0]; +diff --git a/celt/quant_bands.c b/celt/quant_bands.c +index e5ed9ef..82fb823 100644 +--- a/celt/quant_bands.c ++++ b/celt/quant_bands.c +@@ -552,6 +552,7 @@ void log2Amp(const CELTMode *m, int start, int end, + { + opus_val16 lg = ADD16(oldEBands[i+c*m->nbEBands], + SHL16((opus_val16)eMeans[i],6)); ++ lg = MIN32(QCONST32(32.f, 16), lg); + eBands[i+c*m->nbEBands] = PSHR32(celt_exp2(lg),4); + } + for (;inbEBands;i++) +diff --git a/silk/LPC_inv_pred_gain.c b/silk/LPC_inv_pred_gain.c +index 60c439b..6c301da 100644 +--- a/silk/LPC_inv_pred_gain.c ++++ b/silk/LPC_inv_pred_gain.c +@@ -84,8 +84,13 @@ static opus_int32 LPC_inverse_pred_gain_QA( /* O Returns inver + + /* Update AR coefficient */ + for( n = 0; n < k; n++ ) { +- tmp_QA = Aold_QA[ n ] - MUL32_FRAC_Q( Aold_QA[ k - n - 1 ], rc_Q31, 31 ); +- Anew_QA[ n ] = MUL32_FRAC_Q( tmp_QA, rc_mult2 , mult2Q ); ++ opus_int64 tmp64; ++ tmp_QA = silk_SUB_SAT32( Aold_QA[ n ], MUL32_FRAC_Q( Aold_QA[ k - n - 1 ], rc_Q31, 31 ) ); ++ tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( tmp_QA, rc_mult2 ), mult2Q); ++ if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { ++ return 0; ++ } ++ Anew_QA[ n ] = ( opus_int32 )tmp64; + } + } + +diff --git a/silk/NLSF_stabilize.c b/silk/NLSF_stabilize.c +index 979aaba..2ef2398 100644 +--- a/silk/NLSF_stabilize.c ++++ b/silk/NLSF_stabilize.c +@@ -134,7 +134,7 @@ void silk_NLSF_stabilize( + + /* Keep delta_min distance between the NLSFs */ + for( i = 1; i < L; i++ ) +- NLSF_Q15[i] = silk_max_int( NLSF_Q15[i], NLSF_Q15[i-1] + NDeltaMin_Q15[i] ); ++ NLSF_Q15[i] = silk_max_int( NLSF_Q15[i], silk_ADD_SAT16( NLSF_Q15[i-1], NDeltaMin_Q15[i] ) ); + + /* Last NLSF should be no higher than 1 - NDeltaMin[L] */ + NLSF_Q15[L-1] = silk_min_int( NLSF_Q15[L-1], (1<<15) - NDeltaMin_Q15[L] ); +diff --git a/silk/dec_API.c b/silk/dec_API.c +index efd7918..21bb7e0 100644 +--- a/silk/dec_API.c ++++ b/silk/dec_API.c +@@ -72,6 +72,9 @@ opus_int silk_InitDecoder( /* O Returns error co + for( n = 0; n < DECODER_NUM_CHANNELS; n++ ) { + ret = silk_init_decoder( &channel_state[ n ] ); + } ++ silk_memset(&((silk_decoder *)decState)->sStereo, 0, sizeof(((silk_decoder *)decState)->sStereo)); ++ /* Not strictly needed, but it's cleaner that way */ ++ ((silk_decoder *)decState)->prev_decode_only_middle = 0; + + return ret; + } +diff --git a/silk/resampler_private_IIR_FIR.c b/silk/resampler_private_IIR_FIR.c +index dbd6d9a..91a43aa 100644 +--- a/silk/resampler_private_IIR_FIR.c ++++ b/silk/resampler_private_IIR_FIR.c +@@ -75,10 +75,10 @@ void silk_resampler_private_IIR_FIR( + silk_resampler_state_struct *S = (silk_resampler_state_struct *)SS; + opus_int32 nSamplesIn; + opus_int32 max_index_Q16, index_increment_Q16; +- opus_int16 buf[ RESAMPLER_MAX_BATCH_SIZE_IN + RESAMPLER_ORDER_FIR_12 ]; ++ opus_int16 buf[ 2*RESAMPLER_MAX_BATCH_SIZE_IN + RESAMPLER_ORDER_FIR_12 ]; + + /* Copy buffered samples to start of buffer */ +- silk_memcpy( buf, S->sFIR, RESAMPLER_ORDER_FIR_12 * sizeof( opus_int32 ) ); ++ silk_memcpy( buf, S->sFIR, RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + + /* Iterate over blocks of frameSizeIn input samples */ + index_increment_Q16 = S->invRatio_Q16; +@@ -95,13 +95,13 @@ void silk_resampler_private_IIR_FIR( + + if( inLen > 0 ) { + /* More iterations to do; copy last part of filtered signal to beginning of buffer */ +- silk_memcpy( buf, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int32 ) ); ++ silk_memmove( buf, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + } else { + break; + } + } + + /* Copy last part of filtered signal to the state for the next call */ +- silk_memcpy( S->sFIR, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int32 ) ); ++ silk_memcpy( S->sFIR, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + } + +diff --git a/src/opus_decoder.c b/src/opus_decoder.c +index 0cc56f8..8a30fbc 100644 +--- a/src/opus_decoder.c ++++ b/src/opus_decoder.c +@@ -595,16 +595,14 @@ static int opus_packet_parse_impl(const unsigned char *data, int len, + /* Padding flag is bit 6 */ + if (ch&0x40) + { +- int padding=0; + int p; + do { + if (len<=0) + return OPUS_INVALID_PACKET; + p = *data++; + len--; +- padding += p==255 ? 254: p; ++ len -= p==255 ? 254: p; + } while (p==255); +- len -= padding; + } + if (len<0) + return OPUS_INVALID_PACKET; +diff --git a/run_vectors.sh b/run_vectors.sh +index 7cd23ed..4841b0a 100755 +--- a/run_vectors.sh ++++ b/run_vectors.sh +@@ -1,3 +1,5 @@ ++#!/bin/sh ++# + # Copyright (c) 2011-2012 IETF Trust, Jean-Marc Valin. All rights reserved. + # + # This file is extracted from RFC6716. Please see that RFC for additional +@@ -31,10 +33,8 @@ + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-#!/bin/sh +- +-rm logs_mono.txt +-rm logs_stereo.txt ++rm -f logs_mono.txt logs_mono2.txt ++rm -f logs_stereo.txt logs_stereo2.txt + + if [ "$#" -ne "3" ]; then + echo "usage: run_vectors.sh " +@@ -45,18 +45,23 @@ CMD_PATH=$1 + VECTOR_PATH=$2 + RATE=$3 + +-OPUS_DEMO=$CMD_PATH/opus_demo +-OPUS_COMPARE=$CMD_PATH/opus_compare ++: ${OPUS_DEMO:=$CMD_PATH/opus_demo} ++: ${OPUS_COMPARE:=$CMD_PATH/opus_compare} + + if [ -d $VECTOR_PATH ]; then + echo Test vectors found in $VECTOR_PATH + else + echo No test vectors found +- #Don't make the test fail here because the test vectors will be +- #distributed separately ++ #Don't make the test fail here because the test vectors ++ #will be distributed separately + exit 0 + fi + ++if [ ! -x $OPUS_COMPARE ]; then ++ echo ERROR: Compare program not found: $OPUS_COMPARE ++ exit 1 ++fi ++ + if [ -x $OPUS_DEMO ]; then + echo Decoding with $OPUS_DEMO + else +@@ -82,9 +87,11 @@ do + echo ERROR: decoding failed + exit 1 + fi +- $OPUS_COMPARE -r $RATE $VECTOR_PATH/testvector$file.dec tmp.out >> logs_mono.txt 2>&1 ++ $OPUS_COMPARE -r $RATE $VECTOR_PATH/testvector${file}.dec tmp.out >> logs_mono.txt 2>&1 + float_ret=$? +- if [ "$float_ret" -eq "0" ]; then ++ $OPUS_COMPARE -r $RATE $VECTOR_PATH/testvector${file}m.dec tmp.out >> logs_mono2.txt 2>&1 ++ float_ret2=$? ++ if [ "$float_ret" -eq "0" ] || [ "$float_ret2" -eq "0" ]; then + echo output matches reference + else + echo ERROR: output does not match reference +@@ -111,9 +118,11 @@ do + echo ERROR: decoding failed + exit 1 + fi +- $OPUS_COMPARE -s -r $RATE $VECTOR_PATH/testvector$file.dec tmp.out >> logs_stereo.txt 2>&1 ++ $OPUS_COMPARE -s -r $RATE $VECTOR_PATH/testvector${file}.dec tmp.out >> logs_stereo.txt 2>&1 + float_ret=$? +- if [ "$float_ret" -eq "0" ]; then ++ $OPUS_COMPARE -s -r $RATE $VECTOR_PATH/testvector${file}m.dec tmp.out >> logs_stereo2.txt 2>&1 ++ float_ret2=$? ++ if [ "$float_ret" -eq "0" ] || [ "$float_ret2" -eq "0" ]; then + echo output matches reference + else + echo ERROR: output does not match reference +@@ -125,5 +134,10 @@ done + + + echo All tests have passed successfully +-grep quality logs_mono.txt | awk '{sum+=$4}END{print "Average mono quality is", sum/NR, "%"}' +-grep quality logs_stereo.txt | awk '{sum+=$4}END{print "Average stereo quality is", sum/NR, "%"}' ++mono1=`grep quality logs_mono.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` ++mono2=`grep quality logs_mono2.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` ++echo $mono1 $mono2 | awk '{if ($2 > $1) $1 = $2; print "Average mono quality is", $1, "%"}' ++ ++stereo1=`grep quality logs_stereo.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` ++stereo2=`grep quality logs_stereo2.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` ++echo $stereo1 $stereo2 | awk '{if ($2 > $1) $1 = $2; print "Average stereo quality is", $1, "%"}' diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/release.txt b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/release.txt new file mode 100644 index 000000000..411ab7f4b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/release.txt @@ -0,0 +1,43 @@ += Release checklist = + +== Source release == + +- Check for uncommitted changes to master. +- Update OPUS_LT_* API versioning in configure.ac. +- Tag the release commit with 'git tag -s vN.M'. + - Include release notes in the tag annotation. +- Verify 'make distcheck' produces a tarball with + the desired name. +- Push tag to public repo. +- Upload source package 'opus-${version}.tar.gz' + - Add to https://svn.xiph.org/releases/opus/ + - Update checksum files + - svn commit + - Copy to archive.mozilla.org/pub/opus/ + - Update checksum files there as well. +- Add release notes to https://gitlab.xiph.org/xiph/opus-website.git +- Update links and checksums on the downloads page. +- Add a copy of the documentation to + and update the links. +- Update /topic in #opus IRC channel. + +Releases are commited to https://svn.xiph.org/releases/opus/ +which propagates to downloads.xiph.org, and copied manually +to https://archive.mozilla.org/pub/opus/ + +Website updates are committed to https://gitlab.xiph.org/xiph/opus-website.git +which propagates to https://opus-codec.org/ + +== Binary release == + +We usually build opus-tools binaries for MacOS and Windows. + +Binary releases are copied manually to +https://archive.mozilla.org/pub/opus/win32/ + +For Mac, submit a pull request to homebrew. + +== Website updates == + +For major releases, recreate the files on https://opus-codec.org/examples/ +with the next encoder. diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/trivial_example.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/trivial_example.c new file mode 100644 index 000000000..9cf435b4a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/doc/trivial_example.c @@ -0,0 +1,171 @@ +/* Copyright (c) 2013 Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This is meant to be a simple example of encoding and decoding audio + using Opus. It should make it easy to understand how the Opus API + works. For more information, see the full API documentation at: + https://www.opus-codec.org/docs/ */ + +#include +#include +#include +#include +#include + +/*The frame size is hardcoded for this sample code but it doesn't have to be*/ +#define FRAME_SIZE 960 +#define SAMPLE_RATE 48000 +#define CHANNELS 2 +#define APPLICATION OPUS_APPLICATION_AUDIO +#define BITRATE 64000 + +#define MAX_FRAME_SIZE 6*960 +#define MAX_PACKET_SIZE (3*1276) + +int main(int argc, char **argv) +{ + char *inFile; + FILE *fin; + char *outFile; + FILE *fout; + opus_int16 in[FRAME_SIZE*CHANNELS]; + opus_int16 out[MAX_FRAME_SIZE*CHANNELS]; + unsigned char cbits[MAX_PACKET_SIZE]; + int nbBytes; + /*Holds the state of the encoder and decoder */ + OpusEncoder *encoder; + OpusDecoder *decoder; + int err; + + if (argc != 3) + { + fprintf(stderr, "usage: trivial_example input.pcm output.pcm\n"); + fprintf(stderr, "input and output are 16-bit little-endian raw files\n"); + return EXIT_FAILURE; + } + + /*Create a new encoder state */ + encoder = opus_encoder_create(SAMPLE_RATE, CHANNELS, APPLICATION, &err); + if (err<0) + { + fprintf(stderr, "failed to create an encoder: %s\n", opus_strerror(err)); + return EXIT_FAILURE; + } + /* Set the desired bit-rate. You can also set other parameters if needed. + The Opus library is designed to have good defaults, so only set + parameters you know you need. Doing otherwise is likely to result + in worse quality, but better. */ + err = opus_encoder_ctl(encoder, OPUS_SET_BITRATE(BITRATE)); + if (err<0) + { + fprintf(stderr, "failed to set bitrate: %s\n", opus_strerror(err)); + return EXIT_FAILURE; + } + inFile = argv[1]; + fin = fopen(inFile, "rb"); + if (fin==NULL) + { + fprintf(stderr, "failed to open input file: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + + + /* Create a new decoder state. */ + decoder = opus_decoder_create(SAMPLE_RATE, CHANNELS, &err); + if (err<0) + { + fprintf(stderr, "failed to create decoder: %s\n", opus_strerror(err)); + return EXIT_FAILURE; + } + outFile = argv[2]; + fout = fopen(outFile, "wb"); + if (fout==NULL) + { + fprintf(stderr, "failed to open output file: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + + while (1) + { + int i; + unsigned char pcm_bytes[MAX_FRAME_SIZE*CHANNELS*2]; + int frame_size; + size_t samples; + + /* Read a 16 bits/sample audio frame. */ + samples = fread(pcm_bytes, sizeof(short)*CHANNELS, FRAME_SIZE, fin); + + /* For simplicity, only read whole frames. In a real application, + * we'd pad the final partial frame with zeroes, record the exact + * duration, and trim the decoded audio to match. + */ + if (samples != FRAME_SIZE) + { + break; + } + + /* Convert from little-endian ordering. */ + for (i=0;i>8)&0xFF; + } + /* Write the decoded audio to file. */ + fwrite(pcm_bytes, sizeof(short), frame_size*CHANNELS, fout); + } + /*Destroy the encoder state*/ + opus_encoder_destroy(encoder); + opus_decoder_destroy(decoder); + fclose(fin); + fclose(fout); + return EXIT_SUCCESS; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/meson.build new file mode 100644 index 000000000..c1fb0b76f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/meson.build @@ -0,0 +1,13 @@ +opus_headers = [ + 'opus.h', + 'opus_multistream.h', + 'opus_projection.h', + 'opus_types.h', + 'opus_defines.h', +] + +if opt_custom_modes + opus_headers += ['opus_custom.h'] +endif + +install_headers(opus_headers, subdir: 'opus') diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus.h new file mode 100644 index 000000000..d282f21d2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus.h @@ -0,0 +1,981 @@ +/* Copyright (c) 2010-2011 Xiph.Org Foundation, Skype Limited + Written by Jean-Marc Valin and Koen Vos */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + * @file opus.h + * @brief Opus reference implementation API + */ + +#ifndef OPUS_H +#define OPUS_H + +#include "opus_types.h" +#include "opus_defines.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @mainpage Opus + * + * The Opus codec is designed for interactive speech and audio transmission over the Internet. + * It is designed by the IETF Codec Working Group and incorporates technology from + * Skype's SILK codec and Xiph.Org's CELT codec. + * + * The Opus codec is designed to handle a wide range of interactive audio applications, + * including Voice over IP, videoconferencing, in-game chat, and even remote live music + * performances. It can scale from low bit-rate narrowband speech to very high quality + * stereo music. Its main features are: + + * @li Sampling rates from 8 to 48 kHz + * @li Bit-rates from 6 kb/s to 510 kb/s + * @li Support for both constant bit-rate (CBR) and variable bit-rate (VBR) + * @li Audio bandwidth from narrowband to full-band + * @li Support for speech and music + * @li Support for mono and stereo + * @li Support for multichannel (up to 255 channels) + * @li Frame sizes from 2.5 ms to 60 ms + * @li Good loss robustness and packet loss concealment (PLC) + * @li Floating point and fixed-point implementation + * + * Documentation sections: + * @li @ref opus_encoder + * @li @ref opus_decoder + * @li @ref opus_repacketizer + * @li @ref opus_multistream + * @li @ref opus_libinfo + * @li @ref opus_custom + */ + +/** @defgroup opus_encoder Opus Encoder + * @{ + * + * @brief This page describes the process and functions used to encode Opus. + * + * Since Opus is a stateful codec, the encoding process starts with creating an encoder + * state. This can be done with: + * + * @code + * int error; + * OpusEncoder *enc; + * enc = opus_encoder_create(Fs, channels, application, &error); + * @endcode + * + * From this point, @c enc can be used for encoding an audio stream. An encoder state + * @b must @b not be used for more than one stream at the same time. Similarly, the encoder + * state @b must @b not be re-initialized for each frame. + * + * While opus_encoder_create() allocates memory for the state, it's also possible + * to initialize pre-allocated memory: + * + * @code + * int size; + * int error; + * OpusEncoder *enc; + * size = opus_encoder_get_size(channels); + * enc = malloc(size); + * error = opus_encoder_init(enc, Fs, channels, application); + * @endcode + * + * where opus_encoder_get_size() returns the required size for the encoder state. Note that + * future versions of this code may change the size, so no assuptions should be made about it. + * + * The encoder state is always continuous in memory and only a shallow copy is sufficient + * to copy it (e.g. memcpy()) + * + * It is possible to change some of the encoder's settings using the opus_encoder_ctl() + * interface. All these settings already default to the recommended value, so they should + * only be changed when necessary. The most common settings one may want to change are: + * + * @code + * opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate)); + * opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(complexity)); + * opus_encoder_ctl(enc, OPUS_SET_SIGNAL(signal_type)); + * @endcode + * + * where + * + * @arg bitrate is in bits per second (b/s) + * @arg complexity is a value from 1 to 10, where 1 is the lowest complexity and 10 is the highest + * @arg signal_type is either OPUS_AUTO (default), OPUS_SIGNAL_VOICE, or OPUS_SIGNAL_MUSIC + * + * See @ref opus_encoderctls and @ref opus_genericctls for a complete list of parameters that can be set or queried. Most parameters can be set or changed at any time during a stream. + * + * To encode a frame, opus_encode() or opus_encode_float() must be called with exactly one frame (2.5, 5, 10, 20, 40 or 60 ms) of audio data: + * @code + * len = opus_encode(enc, audio_frame, frame_size, packet, max_packet); + * @endcode + * + * where + *
    + *
  • audio_frame is the audio data in opus_int16 (or float for opus_encode_float())
  • + *
  • frame_size is the duration of the frame in samples (per channel)
  • + *
  • packet is the byte array to which the compressed data is written
  • + *
  • max_packet is the maximum number of bytes that can be written in the packet (4000 bytes is recommended). + * Do not use max_packet to control VBR target bitrate, instead use the #OPUS_SET_BITRATE CTL.
  • + *
+ * + * opus_encode() and opus_encode_float() return the number of bytes actually written to the packet. + * The return value can be negative, which indicates that an error has occurred. If the return value + * is 2 bytes or less, then the packet does not need to be transmitted (DTX). + * + * Once the encoder state if no longer needed, it can be destroyed with + * + * @code + * opus_encoder_destroy(enc); + * @endcode + * + * If the encoder was created with opus_encoder_init() rather than opus_encoder_create(), + * then no action is required aside from potentially freeing the memory that was manually + * allocated for it (calling free(enc) for the example above) + * + */ + +/** Opus encoder state. + * This contains the complete state of an Opus encoder. + * It is position independent and can be freely copied. + * @see opus_encoder_create,opus_encoder_init + */ +typedef struct OpusEncoder OpusEncoder; + +/** Gets the size of an OpusEncoder structure. + * @param[in] channels int: Number of channels. + * This must be 1 or 2. + * @returns The size in bytes. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_encoder_get_size(int channels); + +/** + */ + +/** Allocates and initializes an encoder state. + * There are three coding modes: + * + * @ref OPUS_APPLICATION_VOIP gives best quality at a given bitrate for voice + * signals. It enhances the input signal by high-pass filtering and + * emphasizing formants and harmonics. Optionally it includes in-band + * forward error correction to protect against packet loss. Use this + * mode for typical VoIP applications. Because of the enhancement, + * even at high bitrates the output may sound different from the input. + * + * @ref OPUS_APPLICATION_AUDIO gives best quality at a given bitrate for most + * non-voice signals like music. Use this mode for music and mixed + * (music/voice) content, broadcast, and applications requiring less + * than 15 ms of coding delay. + * + * @ref OPUS_APPLICATION_RESTRICTED_LOWDELAY configures low-delay mode that + * disables the speech-optimized mode in exchange for slightly reduced delay. + * This mode can only be set on an newly initialized or freshly reset encoder + * because it changes the codec delay. + * + * This is useful when the caller knows that the speech-optimized modes will not be needed (use with caution). + * @param [in] Fs opus_int32: Sampling rate of input signal (Hz) + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param [in] channels int: Number of channels (1 or 2) in input signal + * @param [in] application int: Coding mode (@ref OPUS_APPLICATION_VOIP/@ref OPUS_APPLICATION_AUDIO/@ref OPUS_APPLICATION_RESTRICTED_LOWDELAY) + * @param [out] error int*: @ref opus_errorcodes + * @note Regardless of the sampling rate and number channels selected, the Opus encoder + * can switch to a lower audio bandwidth or number of channels if the bitrate + * selected is too low. This also means that it is safe to always use 48 kHz stereo input + * and let the encoder optimize the encoding. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusEncoder *opus_encoder_create( + opus_int32 Fs, + int channels, + int application, + int *error +); + +/** Initializes a previously allocated encoder state + * The memory pointed to by st must be at least the size returned by opus_encoder_get_size(). + * This is intended for applications which use their own allocator instead of malloc. + * @see opus_encoder_create(),opus_encoder_get_size() + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @param [in] st OpusEncoder*: Encoder state + * @param [in] Fs opus_int32: Sampling rate of input signal (Hz) + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param [in] channels int: Number of channels (1 or 2) in input signal + * @param [in] application int: Coding mode (OPUS_APPLICATION_VOIP/OPUS_APPLICATION_AUDIO/OPUS_APPLICATION_RESTRICTED_LOWDELAY) + * @retval #OPUS_OK Success or @ref opus_errorcodes + */ +OPUS_EXPORT int opus_encoder_init( + OpusEncoder *st, + opus_int32 Fs, + int channels, + int application +) OPUS_ARG_NONNULL(1); + +/** Encodes an Opus frame. + * @param [in] st OpusEncoder*: Encoder state + * @param [in] pcm opus_int16*: Input signal (interleaved if 2 channels). length is frame_size*channels*sizeof(opus_int16) + * @param [in] frame_size int: Number of samples per channel in the + * input signal. + * This must be an Opus frame size for + * the encoder's sampling rate. + * For example, at 48 kHz the permitted + * values are 120, 240, 480, 960, 1920, + * and 2880. + * Passing in a duration of less than + * 10 ms (480 samples at 48 kHz) will + * prevent the encoder from using the LPC + * or hybrid modes. + * @param [out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_encode( + OpusEncoder *st, + const opus_int16 *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Encodes an Opus frame from floating point input. + * @param [in] st OpusEncoder*: Encoder state + * @param [in] pcm float*: Input in float format (interleaved if 2 channels), with a normal range of +/-1.0. + * Samples with a range beyond +/-1.0 are supported but will + * be clipped by decoders using the integer API and should + * only be used if it is known that the far end supports + * extended dynamic range. + * length is frame_size*channels*sizeof(float) + * @param [in] frame_size int: Number of samples per channel in the + * input signal. + * This must be an Opus frame size for + * the encoder's sampling rate. + * For example, at 48 kHz the permitted + * values are 120, 240, 480, 960, 1920, + * and 2880. + * Passing in a duration of less than + * 10 ms (480 samples at 48 kHz) will + * prevent the encoder from using the LPC + * or hybrid modes. + * @param [out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_encode_float( + OpusEncoder *st, + const float *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Frees an OpusEncoder allocated by opus_encoder_create(). + * @param[in] st OpusEncoder*: State to be freed. + */ +OPUS_EXPORT void opus_encoder_destroy(OpusEncoder *st); + +/** Perform a CTL function on an Opus encoder. + * + * Generally the request and subsequent arguments are generated + * by a convenience macro. + * @param st OpusEncoder*: Encoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls or + * @ref opus_encoderctls. + * @see opus_genericctls + * @see opus_encoderctls + */ +OPUS_EXPORT int opus_encoder_ctl(OpusEncoder *st, int request, ...) OPUS_ARG_NONNULL(1); +/**@}*/ + +/** @defgroup opus_decoder Opus Decoder + * @{ + * + * @brief This page describes the process and functions used to decode Opus. + * + * The decoding process also starts with creating a decoder + * state. This can be done with: + * @code + * int error; + * OpusDecoder *dec; + * dec = opus_decoder_create(Fs, channels, &error); + * @endcode + * where + * @li Fs is the sampling rate and must be 8000, 12000, 16000, 24000, or 48000 + * @li channels is the number of channels (1 or 2) + * @li error will hold the error code in case of failure (or #OPUS_OK on success) + * @li the return value is a newly created decoder state to be used for decoding + * + * While opus_decoder_create() allocates memory for the state, it's also possible + * to initialize pre-allocated memory: + * @code + * int size; + * int error; + * OpusDecoder *dec; + * size = opus_decoder_get_size(channels); + * dec = malloc(size); + * error = opus_decoder_init(dec, Fs, channels); + * @endcode + * where opus_decoder_get_size() returns the required size for the decoder state. Note that + * future versions of this code may change the size, so no assuptions should be made about it. + * + * The decoder state is always continuous in memory and only a shallow copy is sufficient + * to copy it (e.g. memcpy()) + * + * To decode a frame, opus_decode() or opus_decode_float() must be called with a packet of compressed audio data: + * @code + * frame_size = opus_decode(dec, packet, len, decoded, max_size, 0); + * @endcode + * where + * + * @li packet is the byte array containing the compressed data + * @li len is the exact number of bytes contained in the packet + * @li decoded is the decoded audio data in opus_int16 (or float for opus_decode_float()) + * @li max_size is the max duration of the frame in samples (per channel) that can fit into the decoded_frame array + * + * opus_decode() and opus_decode_float() return the number of samples (per channel) decoded from the packet. + * If that value is negative, then an error has occurred. This can occur if the packet is corrupted or if the audio + * buffer is too small to hold the decoded audio. + * + * Opus is a stateful codec with overlapping blocks and as a result Opus + * packets are not coded independently of each other. Packets must be + * passed into the decoder serially and in the correct order for a correct + * decode. Lost packets can be replaced with loss concealment by calling + * the decoder with a null pointer and zero length for the missing packet. + * + * A single codec state may only be accessed from a single thread at + * a time and any required locking must be performed by the caller. Separate + * streams must be decoded with separate decoder states and can be decoded + * in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK + * defined. + * + */ + +/** Opus decoder state. + * This contains the complete state of an Opus decoder. + * It is position independent and can be freely copied. + * @see opus_decoder_create,opus_decoder_init + */ +typedef struct OpusDecoder OpusDecoder; + +/** Gets the size of an OpusDecoder structure. + * @param [in] channels int: Number of channels. + * This must be 1 or 2. + * @returns The size in bytes. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decoder_get_size(int channels); + +/** Allocates and initializes a decoder state. + * @param [in] Fs opus_int32: Sample rate to decode at (Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param [in] channels int: Number of channels (1 or 2) to decode + * @param [out] error int*: #OPUS_OK Success or @ref opus_errorcodes + * + * Internally Opus stores data at 48000 Hz, so that should be the default + * value for Fs. However, the decoder can efficiently decode to buffers + * at 8, 12, 16, and 24 kHz so if for some reason the caller cannot use + * data at the full sample rate, or knows the compressed data doesn't + * use the full frequency range, it can request decoding at a reduced + * rate. Likewise, the decoder is capable of filling in either mono or + * interleaved stereo pcm buffers, at the caller's request. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusDecoder *opus_decoder_create( + opus_int32 Fs, + int channels, + int *error +); + +/** Initializes a previously allocated decoder state. + * The state must be at least the size returned by opus_decoder_get_size(). + * This is intended for applications which use their own allocator instead of malloc. @see opus_decoder_create,opus_decoder_get_size + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @param [in] st OpusDecoder*: Decoder state. + * @param [in] Fs opus_int32: Sampling rate to decode to (Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param [in] channels int: Number of channels (1 or 2) to decode + * @retval #OPUS_OK Success or @ref opus_errorcodes + */ +OPUS_EXPORT int opus_decoder_init( + OpusDecoder *st, + opus_int32 Fs, + int channels +) OPUS_ARG_NONNULL(1); + +/** Decode an Opus packet. + * @param [in] st OpusDecoder*: Decoder state + * @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss + * @param [in] len opus_int32: Number of bytes in payload* + * @param [out] pcm opus_int16*: Output signal (interleaved if 2 channels). length + * is frame_size*channels*sizeof(opus_int16) + * @param [in] frame_size Number of samples per channel of available space in \a pcm. + * If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will + * not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1), + * then frame_size needs to be exactly the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the next incoming packet. For the PLC and + * FEC cases, frame_size must be a multiple of 2.5 ms. + * @param [in] decode_fec int: Flag (0 or 1) to request that any in-band forward error correction data be + * decoded. If no such data is available, the frame is decoded as if it were lost. + * @returns Number of decoded samples or @ref opus_errorcodes + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decode( + OpusDecoder *st, + const unsigned char *data, + opus_int32 len, + opus_int16 *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Decode an Opus packet with floating point output. + * @param [in] st OpusDecoder*: Decoder state + * @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss + * @param [in] len opus_int32: Number of bytes in payload + * @param [out] pcm float*: Output signal (interleaved if 2 channels). length + * is frame_size*channels*sizeof(float) + * @param [in] frame_size Number of samples per channel of available space in \a pcm. + * If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will + * not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1), + * then frame_size needs to be exactly the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the next incoming packet. For the PLC and + * FEC cases, frame_size must be a multiple of 2.5 ms. + * @param [in] decode_fec int: Flag (0 or 1) to request that any in-band forward error correction data be + * decoded. If no such data is available the frame is decoded as if it were lost. + * @returns Number of decoded samples or @ref opus_errorcodes + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decode_float( + OpusDecoder *st, + const unsigned char *data, + opus_int32 len, + float *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Perform a CTL function on an Opus decoder. + * + * Generally the request and subsequent arguments are generated + * by a convenience macro. + * @param st OpusDecoder*: Decoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls or + * @ref opus_decoderctls. + * @see opus_genericctls + * @see opus_decoderctls + */ +OPUS_EXPORT int opus_decoder_ctl(OpusDecoder *st, int request, ...) OPUS_ARG_NONNULL(1); + +/** Frees an OpusDecoder allocated by opus_decoder_create(). + * @param[in] st OpusDecoder*: State to be freed. + */ +OPUS_EXPORT void opus_decoder_destroy(OpusDecoder *st); + +/** Parse an opus packet into one or more frames. + * Opus_decode will perform this operation internally so most applications do + * not need to use this function. + * This function does not copy the frames, the returned pointers are pointers into + * the input packet. + * @param [in] data char*: Opus packet to be parsed + * @param [in] len opus_int32: size of data + * @param [out] out_toc char*: TOC pointer + * @param [out] frames char*[48] encapsulated frames + * @param [out] size opus_int16[48] sizes of the encapsulated frames + * @param [out] payload_offset int*: returns the position of the payload within the packet (in bytes) + * @returns number of frames + */ +OPUS_EXPORT int opus_packet_parse( + const unsigned char *data, + opus_int32 len, + unsigned char *out_toc, + const unsigned char *frames[48], + opus_int16 size[48], + int *payload_offset +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(5); + +/** Gets the bandwidth of an Opus packet. + * @param [in] data char*: Opus packet + * @retval OPUS_BANDWIDTH_NARROWBAND Narrowband (4kHz bandpass) + * @retval OPUS_BANDWIDTH_MEDIUMBAND Mediumband (6kHz bandpass) + * @retval OPUS_BANDWIDTH_WIDEBAND Wideband (8kHz bandpass) + * @retval OPUS_BANDWIDTH_SUPERWIDEBAND Superwideband (12kHz bandpass) + * @retval OPUS_BANDWIDTH_FULLBAND Fullband (20kHz bandpass) + * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_bandwidth(const unsigned char *data) OPUS_ARG_NONNULL(1); + +/** Gets the number of samples per frame from an Opus packet. + * @param [in] data char*: Opus packet. + * This must contain at least one byte of + * data. + * @param [in] Fs opus_int32: Sampling rate in Hz. + * This must be a multiple of 400, or + * inaccurate results will be returned. + * @returns Number of samples per frame. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_samples_per_frame(const unsigned char *data, opus_int32 Fs) OPUS_ARG_NONNULL(1); + +/** Gets the number of channels from an Opus packet. + * @param [in] data char*: Opus packet + * @returns Number of channels + * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_channels(const unsigned char *data) OPUS_ARG_NONNULL(1); + +/** Gets the number of frames in an Opus packet. + * @param [in] packet char*: Opus packet + * @param [in] len opus_int32: Length of packet + * @returns Number of frames + * @retval OPUS_BAD_ARG Insufficient data was passed to the function + * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_frames(const unsigned char packet[], opus_int32 len) OPUS_ARG_NONNULL(1); + +/** Gets the number of samples of an Opus packet. + * @param [in] packet char*: Opus packet + * @param [in] len opus_int32: Length of packet + * @param [in] Fs opus_int32: Sampling rate in Hz. + * This must be a multiple of 400, or + * inaccurate results will be returned. + * @returns Number of samples + * @retval OPUS_BAD_ARG Insufficient data was passed to the function + * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_samples(const unsigned char packet[], opus_int32 len, opus_int32 Fs) OPUS_ARG_NONNULL(1); + +/** Gets the number of samples of an Opus packet. + * @param [in] dec OpusDecoder*: Decoder state + * @param [in] packet char*: Opus packet + * @param [in] len opus_int32: Length of packet + * @returns Number of samples + * @retval OPUS_BAD_ARG Insufficient data was passed to the function + * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decoder_get_nb_samples(const OpusDecoder *dec, const unsigned char packet[], opus_int32 len) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2); + +/** Applies soft-clipping to bring a float signal within the [-1,1] range. If + * the signal is already in that range, nothing is done. If there are values + * outside of [-1,1], then the signal is clipped as smoothly as possible to + * both fit in the range and avoid creating excessive distortion in the + * process. + * @param [in,out] pcm float*: Input PCM and modified PCM + * @param [in] frame_size int Number of samples per channel to process + * @param [in] channels int: Number of channels + * @param [in,out] softclip_mem float*: State memory for the soft clipping process (one float per channel, initialized to zero) + */ +OPUS_EXPORT void opus_pcm_soft_clip(float *pcm, int frame_size, int channels, float *softclip_mem); + + +/**@}*/ + +/** @defgroup opus_repacketizer Repacketizer + * @{ + * + * The repacketizer can be used to merge multiple Opus packets into a single + * packet or alternatively to split Opus packets that have previously been + * merged. Splitting valid Opus packets is always guaranteed to succeed, + * whereas merging valid packets only succeeds if all frames have the same + * mode, bandwidth, and frame size, and when the total duration of the merged + * packet is no more than 120 ms. The 120 ms limit comes from the + * specification and limits decoder memory requirements at a point where + * framing overhead becomes negligible. + * + * The repacketizer currently only operates on elementary Opus + * streams. It will not manipualte multistream packets successfully, except in + * the degenerate case where they consist of data from a single stream. + * + * The repacketizing process starts with creating a repacketizer state, either + * by calling opus_repacketizer_create() or by allocating the memory yourself, + * e.g., + * @code + * OpusRepacketizer *rp; + * rp = (OpusRepacketizer*)malloc(opus_repacketizer_get_size()); + * if (rp != NULL) + * opus_repacketizer_init(rp); + * @endcode + * + * Then the application should submit packets with opus_repacketizer_cat(), + * extract new packets with opus_repacketizer_out() or + * opus_repacketizer_out_range(), and then reset the state for the next set of + * input packets via opus_repacketizer_init(). + * + * For example, to split a sequence of packets into individual frames: + * @code + * unsigned char *data; + * int len; + * while (get_next_packet(&data, &len)) + * { + * unsigned char out[1276]; + * opus_int32 out_len; + * int nb_frames; + * int err; + * int i; + * err = opus_repacketizer_cat(rp, data, len); + * if (err != OPUS_OK) + * { + * release_packet(data); + * return err; + * } + * nb_frames = opus_repacketizer_get_nb_frames(rp); + * for (i = 0; i < nb_frames; i++) + * { + * out_len = opus_repacketizer_out_range(rp, i, i+1, out, sizeof(out)); + * if (out_len < 0) + * { + * release_packet(data); + * return (int)out_len; + * } + * output_next_packet(out, out_len); + * } + * opus_repacketizer_init(rp); + * release_packet(data); + * } + * @endcode + * + * Alternatively, to combine a sequence of frames into packets that each + * contain up to TARGET_DURATION_MS milliseconds of data: + * @code + * // The maximum number of packets with duration TARGET_DURATION_MS occurs + * // when the frame size is 2.5 ms, for a total of (TARGET_DURATION_MS*2/5) + * // packets. + * unsigned char *data[(TARGET_DURATION_MS*2/5)+1]; + * opus_int32 len[(TARGET_DURATION_MS*2/5)+1]; + * int nb_packets; + * unsigned char out[1277*(TARGET_DURATION_MS*2/2)]; + * opus_int32 out_len; + * int prev_toc; + * nb_packets = 0; + * while (get_next_packet(data+nb_packets, len+nb_packets)) + * { + * int nb_frames; + * int err; + * nb_frames = opus_packet_get_nb_frames(data[nb_packets], len[nb_packets]); + * if (nb_frames < 1) + * { + * release_packets(data, nb_packets+1); + * return nb_frames; + * } + * nb_frames += opus_repacketizer_get_nb_frames(rp); + * // If adding the next packet would exceed our target, or it has an + * // incompatible TOC sequence, output the packets we already have before + * // submitting it. + * // N.B., The nb_packets > 0 check ensures we've submitted at least one + * // packet since the last call to opus_repacketizer_init(). Otherwise a + * // single packet longer than TARGET_DURATION_MS would cause us to try to + * // output an (invalid) empty packet. It also ensures that prev_toc has + * // been set to a valid value. Additionally, len[nb_packets] > 0 is + * // guaranteed by the call to opus_packet_get_nb_frames() above, so the + * // reference to data[nb_packets][0] should be valid. + * if (nb_packets > 0 && ( + * ((prev_toc & 0xFC) != (data[nb_packets][0] & 0xFC)) || + * opus_packet_get_samples_per_frame(data[nb_packets], 48000)*nb_frames > + * TARGET_DURATION_MS*48)) + * { + * out_len = opus_repacketizer_out(rp, out, sizeof(out)); + * if (out_len < 0) + * { + * release_packets(data, nb_packets+1); + * return (int)out_len; + * } + * output_next_packet(out, out_len); + * opus_repacketizer_init(rp); + * release_packets(data, nb_packets); + * data[0] = data[nb_packets]; + * len[0] = len[nb_packets]; + * nb_packets = 0; + * } + * err = opus_repacketizer_cat(rp, data[nb_packets], len[nb_packets]); + * if (err != OPUS_OK) + * { + * release_packets(data, nb_packets+1); + * return err; + * } + * prev_toc = data[nb_packets][0]; + * nb_packets++; + * } + * // Output the final, partial packet. + * if (nb_packets > 0) + * { + * out_len = opus_repacketizer_out(rp, out, sizeof(out)); + * release_packets(data, nb_packets); + * if (out_len < 0) + * return (int)out_len; + * output_next_packet(out, out_len); + * } + * @endcode + * + * An alternate way of merging packets is to simply call opus_repacketizer_cat() + * unconditionally until it fails. At that point, the merged packet can be + * obtained with opus_repacketizer_out() and the input packet for which + * opus_repacketizer_cat() needs to be re-added to a newly reinitialized + * repacketizer state. + */ + +typedef struct OpusRepacketizer OpusRepacketizer; + +/** Gets the size of an OpusRepacketizer structure. + * @returns The size in bytes. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_repacketizer_get_size(void); + +/** (Re)initializes a previously allocated repacketizer state. + * The state must be at least the size returned by opus_repacketizer_get_size(). + * This can be used for applications which use their own allocator instead of + * malloc(). + * It must also be called to reset the queue of packets waiting to be + * repacketized, which is necessary if the maximum packet duration of 120 ms + * is reached or if you wish to submit packets with a different Opus + * configuration (coding mode, audio bandwidth, frame size, or channel count). + * Failure to do so will prevent a new packet from being added with + * opus_repacketizer_cat(). + * @see opus_repacketizer_create + * @see opus_repacketizer_get_size + * @see opus_repacketizer_cat + * @param rp OpusRepacketizer*: The repacketizer state to + * (re)initialize. + * @returns A pointer to the same repacketizer state that was passed in. + */ +OPUS_EXPORT OpusRepacketizer *opus_repacketizer_init(OpusRepacketizer *rp) OPUS_ARG_NONNULL(1); + +/** Allocates memory and initializes the new repacketizer with + * opus_repacketizer_init(). + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusRepacketizer *opus_repacketizer_create(void); + +/** Frees an OpusRepacketizer allocated by + * opus_repacketizer_create(). + * @param[in] rp OpusRepacketizer*: State to be freed. + */ +OPUS_EXPORT void opus_repacketizer_destroy(OpusRepacketizer *rp); + +/** Add a packet to the current repacketizer state. + * This packet must match the configuration of any packets already submitted + * for repacketization since the last call to opus_repacketizer_init(). + * This means that it must have the same coding mode, audio bandwidth, frame + * size, and channel count. + * This can be checked in advance by examining the top 6 bits of the first + * byte of the packet, and ensuring they match the top 6 bits of the first + * byte of any previously submitted packet. + * The total duration of audio in the repacketizer state also must not exceed + * 120 ms, the maximum duration of a single packet, after adding this packet. + * + * The contents of the current repacketizer state can be extracted into new + * packets using opus_repacketizer_out() or opus_repacketizer_out_range(). + * + * In order to add a packet with a different configuration or to add more + * audio beyond 120 ms, you must clear the repacketizer state by calling + * opus_repacketizer_init(). + * If a packet is too large to add to the current repacketizer state, no part + * of it is added, even if it contains multiple frames, some of which might + * fit. + * If you wish to be able to add parts of such packets, you should first use + * another repacketizer to split the packet into pieces and add them + * individually. + * @see opus_repacketizer_out_range + * @see opus_repacketizer_out + * @see opus_repacketizer_init + * @param rp OpusRepacketizer*: The repacketizer state to which to + * add the packet. + * @param[in] data const unsigned char*: The packet data. + * The application must ensure + * this pointer remains valid + * until the next call to + * opus_repacketizer_init() or + * opus_repacketizer_destroy(). + * @param len opus_int32: The number of bytes in the packet data. + * @returns An error code indicating whether or not the operation succeeded. + * @retval #OPUS_OK The packet's contents have been added to the repacketizer + * state. + * @retval #OPUS_INVALID_PACKET The packet did not have a valid TOC sequence, + * the packet's TOC sequence was not compatible + * with previously submitted packets (because + * the coding mode, audio bandwidth, frame size, + * or channel count did not match), or adding + * this packet would increase the total amount of + * audio stored in the repacketizer state to more + * than 120 ms. + */ +OPUS_EXPORT int opus_repacketizer_cat(OpusRepacketizer *rp, const unsigned char *data, opus_int32 len) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2); + + +/** Construct a new packet from data previously submitted to the repacketizer + * state via opus_repacketizer_cat(). + * @param rp OpusRepacketizer*: The repacketizer state from which to + * construct the new packet. + * @param begin int: The index of the first frame in the current + * repacketizer state to include in the output. + * @param end int: One past the index of the last frame in the + * current repacketizer state to include in the + * output. + * @param[out] data const unsigned char*: The buffer in which to + * store the output packet. + * @param maxlen opus_int32: The maximum number of bytes to store in + * the output buffer. In order to guarantee + * success, this should be at least + * 1276 for a single frame, + * or for multiple frames, + * 1277*(end-begin). + * However, 1*(end-begin) plus + * the size of all packet data submitted to + * the repacketizer since the last call to + * opus_repacketizer_init() or + * opus_repacketizer_create() is also + * sufficient, and possibly much smaller. + * @returns The total size of the output packet on success, or an error code + * on failure. + * @retval #OPUS_BAD_ARG [begin,end) was an invalid range of + * frames (begin < 0, begin >= end, or end > + * opus_repacketizer_get_nb_frames()). + * @retval #OPUS_BUFFER_TOO_SMALL \a maxlen was insufficient to contain the + * complete output packet. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_repacketizer_out_range(OpusRepacketizer *rp, int begin, int end, unsigned char *data, opus_int32 maxlen) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Return the total number of frames contained in packet data submitted to + * the repacketizer state so far via opus_repacketizer_cat() since the last + * call to opus_repacketizer_init() or opus_repacketizer_create(). + * This defines the valid range of packets that can be extracted with + * opus_repacketizer_out_range() or opus_repacketizer_out(). + * @param rp OpusRepacketizer*: The repacketizer state containing the + * frames. + * @returns The total number of frames contained in the packet data submitted + * to the repacketizer state. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_repacketizer_get_nb_frames(OpusRepacketizer *rp) OPUS_ARG_NONNULL(1); + +/** Construct a new packet from data previously submitted to the repacketizer + * state via opus_repacketizer_cat(). + * This is a convenience routine that returns all the data submitted so far + * in a single packet. + * It is equivalent to calling + * @code + * opus_repacketizer_out_range(rp, 0, opus_repacketizer_get_nb_frames(rp), + * data, maxlen) + * @endcode + * @param rp OpusRepacketizer*: The repacketizer state from which to + * construct the new packet. + * @param[out] data const unsigned char*: The buffer in which to + * store the output packet. + * @param maxlen opus_int32: The maximum number of bytes to store in + * the output buffer. In order to guarantee + * success, this should be at least + * 1277*opus_repacketizer_get_nb_frames(rp). + * However, + * 1*opus_repacketizer_get_nb_frames(rp) + * plus the size of all packet data + * submitted to the repacketizer since the + * last call to opus_repacketizer_init() or + * opus_repacketizer_create() is also + * sufficient, and possibly much smaller. + * @returns The total size of the output packet on success, or an error code + * on failure. + * @retval #OPUS_BUFFER_TOO_SMALL \a maxlen was insufficient to contain the + * complete output packet. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_repacketizer_out(OpusRepacketizer *rp, unsigned char *data, opus_int32 maxlen) OPUS_ARG_NONNULL(1); + +/** Pads a given Opus packet to a larger size (possibly changing the TOC sequence). + * @param[in,out] data const unsigned char*: The buffer containing the + * packet to pad. + * @param len opus_int32: The size of the packet. + * This must be at least 1. + * @param new_len opus_int32: The desired size of the packet after padding. + * This must be at least as large as len. + * @returns an error code + * @retval #OPUS_OK \a on success. + * @retval #OPUS_BAD_ARG \a len was less than 1 or new_len was less than len. + * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet. + */ +OPUS_EXPORT int opus_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len); + +/** Remove all padding from a given Opus packet and rewrite the TOC sequence to + * minimize space usage. + * @param[in,out] data const unsigned char*: The buffer containing the + * packet to strip. + * @param len opus_int32: The size of the packet. + * This must be at least 1. + * @returns The new size of the output packet on success, or an error code + * on failure. + * @retval #OPUS_BAD_ARG \a len was less than 1. + * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_packet_unpad(unsigned char *data, opus_int32 len); + +/** Pads a given Opus multi-stream packet to a larger size (possibly changing the TOC sequence). + * @param[in,out] data const unsigned char*: The buffer containing the + * packet to pad. + * @param len opus_int32: The size of the packet. + * This must be at least 1. + * @param new_len opus_int32: The desired size of the packet after padding. + * This must be at least 1. + * @param nb_streams opus_int32: The number of streams (not channels) in the packet. + * This must be at least as large as len. + * @returns an error code + * @retval #OPUS_OK \a on success. + * @retval #OPUS_BAD_ARG \a len was less than 1. + * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet. + */ +OPUS_EXPORT int opus_multistream_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len, int nb_streams); + +/** Remove all padding from a given Opus multi-stream packet and rewrite the TOC sequence to + * minimize space usage. + * @param[in,out] data const unsigned char*: The buffer containing the + * packet to strip. + * @param len opus_int32: The size of the packet. + * This must be at least 1. + * @param nb_streams opus_int32: The number of streams (not channels) in the packet. + * This must be at least 1. + * @returns The new size of the output packet on success, or an error code + * on failure. + * @retval #OPUS_BAD_ARG \a len was less than 1 or new_len was less than len. + * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_multistream_packet_unpad(unsigned char *data, opus_int32 len, int nb_streams); + +/**@}*/ + +#ifdef __cplusplus +} +#endif + +#endif /* OPUS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_custom.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_custom.h new file mode 100644 index 000000000..2227be011 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_custom.h @@ -0,0 +1,342 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Copyright (c) 2008-2012 Gregory Maxwell + Written by Jean-Marc Valin and Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + @file opus_custom.h + @brief Opus-Custom reference implementation API + */ + +#ifndef OPUS_CUSTOM_H +#define OPUS_CUSTOM_H + +#include "opus_defines.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef CUSTOM_MODES +# define OPUS_CUSTOM_EXPORT OPUS_EXPORT +# define OPUS_CUSTOM_EXPORT_STATIC OPUS_EXPORT +#else +# define OPUS_CUSTOM_EXPORT +# ifdef OPUS_BUILD +# define OPUS_CUSTOM_EXPORT_STATIC static OPUS_INLINE +# else +# define OPUS_CUSTOM_EXPORT_STATIC +# endif +#endif + +/** @defgroup opus_custom Opus Custom + * @{ + * Opus Custom is an optional part of the Opus specification and + * reference implementation which uses a distinct API from the regular + * API and supports frame sizes that are not normally supported.\ Use + * of Opus Custom is discouraged for all but very special applications + * for which a frame size different from 2.5, 5, 10, or 20 ms is needed + * (for either complexity or latency reasons) and where interoperability + * is less important. + * + * In addition to the interoperability limitations the use of Opus custom + * disables a substantial chunk of the codec and generally lowers the + * quality available at a given bitrate. Normally when an application needs + * a different frame size from the codec it should buffer to match the + * sizes but this adds a small amount of delay which may be important + * in some very low latency applications. Some transports (especially + * constant rate RF transports) may also work best with frames of + * particular durations. + * + * Libopus only supports custom modes if they are enabled at compile time. + * + * The Opus Custom API is similar to the regular API but the + * @ref opus_encoder_create and @ref opus_decoder_create calls take + * an additional mode parameter which is a structure produced by + * a call to @ref opus_custom_mode_create. Both the encoder and decoder + * must create a mode using the same sample rate (fs) and frame size + * (frame size) so these parameters must either be signaled out of band + * or fixed in a particular implementation. + * + * Similar to regular Opus the custom modes support on the fly frame size + * switching, but the sizes available depend on the particular frame size in + * use. For some initial frame sizes on a single on the fly size is available. + */ + +/** Contains the state of an encoder. One encoder state is needed + for each stream. It is initialized once at the beginning of the + stream. Do *not* re-initialize the state for every frame. + @brief Encoder state + */ +typedef struct OpusCustomEncoder OpusCustomEncoder; + +/** State of the decoder. One decoder state is needed for each stream. + It is initialized once at the beginning of the stream. Do *not* + re-initialize the state for every frame. + @brief Decoder state + */ +typedef struct OpusCustomDecoder OpusCustomDecoder; + +/** The mode contains all the information necessary to create an + encoder. Both the encoder and decoder need to be initialized + with exactly the same mode, otherwise the output will be + corrupted. + @brief Mode configuration + */ +typedef struct OpusCustomMode OpusCustomMode; + +/** Creates a new mode struct. This will be passed to an encoder or + * decoder. The mode MUST NOT BE DESTROYED until the encoders and + * decoders that use it are destroyed as well. + * @param [in] Fs int: Sampling rate (8000 to 96000 Hz) + * @param [in] frame_size int: Number of samples (per channel) to encode in each + * packet (64 - 1024, prime factorization must contain zero or more 2s, 3s, or 5s and no other primes) + * @param [out] error int*: Returned error code (if NULL, no error will be returned) + * @return A newly created mode + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomMode *opus_custom_mode_create(opus_int32 Fs, int frame_size, int *error); + +/** Destroys a mode struct. Only call this after all encoders and + * decoders using this mode are destroyed as well. + * @param [in] mode OpusCustomMode*: Mode to be freed. + */ +OPUS_CUSTOM_EXPORT void opus_custom_mode_destroy(OpusCustomMode *mode); + + +#if !defined(OPUS_BUILD) || defined(CELT_ENCODER_C) + +/* Encoder */ +/** Gets the size of an OpusCustomEncoder structure. + * @param [in] mode OpusCustomMode *: Mode configuration + * @param [in] channels int: Number of channels + * @returns size + */ +OPUS_CUSTOM_EXPORT_STATIC OPUS_WARN_UNUSED_RESULT int opus_custom_encoder_get_size( + const OpusCustomMode *mode, + int channels +) OPUS_ARG_NONNULL(1); + +# ifdef CUSTOM_MODES +/** Initializes a previously allocated encoder state + * The memory pointed to by st must be the size returned by opus_custom_encoder_get_size. + * This is intended for applications which use their own allocator instead of malloc. + * @see opus_custom_encoder_create(),opus_custom_encoder_get_size() + * To reset a previously initialized state use the OPUS_RESET_STATE CTL. + * @param [in] st OpusCustomEncoder*: Encoder state + * @param [in] mode OpusCustomMode *: Contains all the information about the characteristics of + * the stream (must be the same characteristics as used for the + * decoder) + * @param [in] channels int: Number of channels + * @return OPUS_OK Success or @ref opus_errorcodes + */ +OPUS_CUSTOM_EXPORT int opus_custom_encoder_init( + OpusCustomEncoder *st, + const OpusCustomMode *mode, + int channels +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2); +# endif +#endif + + +/** Creates a new encoder state. Each stream needs its own encoder + * state (can't be shared across simultaneous streams). + * @param [in] mode OpusCustomMode*: Contains all the information about the characteristics of + * the stream (must be the same characteristics as used for the + * decoder) + * @param [in] channels int: Number of channels + * @param [out] error int*: Returns an error code + * @return Newly created encoder state. +*/ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomEncoder *opus_custom_encoder_create( + const OpusCustomMode *mode, + int channels, + int *error +) OPUS_ARG_NONNULL(1); + + +/** Destroys an encoder state. + * @param[in] st OpusCustomEncoder*: State to be freed. + */ +OPUS_CUSTOM_EXPORT void opus_custom_encoder_destroy(OpusCustomEncoder *st); + +/** Encodes a frame of audio. + * @param [in] st OpusCustomEncoder*: Encoder state + * @param [in] pcm float*: PCM audio in float format, with a normal range of +/-1.0. + * Samples with a range beyond +/-1.0 are supported but will + * be clipped by decoders using the integer API and should + * only be used if it is known that the far end supports + * extended dynamic range. There must be exactly + * frame_size samples per channel. + * @param [in] frame_size int: Number of samples per frame of input signal + * @param [out] compressed char *: The compressed data is written here. This may not alias pcm and must be at least maxCompressedBytes long. + * @param [in] maxCompressedBytes int: Maximum number of bytes to use for compressing the frame + * (can change from one frame to another) + * @return Number of bytes written to "compressed". + * If negative, an error has occurred (see error codes). It is IMPORTANT that + * the length returned be somehow transmitted to the decoder. Otherwise, no + * decoding is possible. + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_encode_float( + OpusCustomEncoder *st, + const float *pcm, + int frame_size, + unsigned char *compressed, + int maxCompressedBytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Encodes a frame of audio. + * @param [in] st OpusCustomEncoder*: Encoder state + * @param [in] pcm opus_int16*: PCM audio in signed 16-bit format (native endian). + * There must be exactly frame_size samples per channel. + * @param [in] frame_size int: Number of samples per frame of input signal + * @param [out] compressed char *: The compressed data is written here. This may not alias pcm and must be at least maxCompressedBytes long. + * @param [in] maxCompressedBytes int: Maximum number of bytes to use for compressing the frame + * (can change from one frame to another) + * @return Number of bytes written to "compressed". + * If negative, an error has occurred (see error codes). It is IMPORTANT that + * the length returned be somehow transmitted to the decoder. Otherwise, no + * decoding is possible. + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_encode( + OpusCustomEncoder *st, + const opus_int16 *pcm, + int frame_size, + unsigned char *compressed, + int maxCompressedBytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Perform a CTL function on an Opus custom encoder. + * + * Generally the request and subsequent arguments are generated + * by a convenience macro. + * @see opus_encoderctls + */ +OPUS_CUSTOM_EXPORT int opus_custom_encoder_ctl(OpusCustomEncoder * OPUS_RESTRICT st, int request, ...) OPUS_ARG_NONNULL(1); + + +#if !defined(OPUS_BUILD) || defined(CELT_DECODER_C) +/* Decoder */ + +/** Gets the size of an OpusCustomDecoder structure. + * @param [in] mode OpusCustomMode *: Mode configuration + * @param [in] channels int: Number of channels + * @returns size + */ +OPUS_CUSTOM_EXPORT_STATIC OPUS_WARN_UNUSED_RESULT int opus_custom_decoder_get_size( + const OpusCustomMode *mode, + int channels +) OPUS_ARG_NONNULL(1); + +/** Initializes a previously allocated decoder state + * The memory pointed to by st must be the size returned by opus_custom_decoder_get_size. + * This is intended for applications which use their own allocator instead of malloc. + * @see opus_custom_decoder_create(),opus_custom_decoder_get_size() + * To reset a previously initialized state use the OPUS_RESET_STATE CTL. + * @param [in] st OpusCustomDecoder*: Decoder state + * @param [in] mode OpusCustomMode *: Contains all the information about the characteristics of + * the stream (must be the same characteristics as used for the + * encoder) + * @param [in] channels int: Number of channels + * @return OPUS_OK Success or @ref opus_errorcodes + */ +OPUS_CUSTOM_EXPORT_STATIC int opus_custom_decoder_init( + OpusCustomDecoder *st, + const OpusCustomMode *mode, + int channels +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2); + +#endif + + +/** Creates a new decoder state. Each stream needs its own decoder state (can't + * be shared across simultaneous streams). + * @param [in] mode OpusCustomMode: Contains all the information about the characteristics of the + * stream (must be the same characteristics as used for the encoder) + * @param [in] channels int: Number of channels + * @param [out] error int*: Returns an error code + * @return Newly created decoder state. + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomDecoder *opus_custom_decoder_create( + const OpusCustomMode *mode, + int channels, + int *error +) OPUS_ARG_NONNULL(1); + +/** Destroys a decoder state. + * @param[in] st OpusCustomDecoder*: State to be freed. + */ +OPUS_CUSTOM_EXPORT void opus_custom_decoder_destroy(OpusCustomDecoder *st); + +/** Decode an opus custom frame with floating point output + * @param [in] st OpusCustomDecoder*: Decoder state + * @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss + * @param [in] len int: Number of bytes in payload + * @param [out] pcm float*: Output signal (interleaved if 2 channels). length + * is frame_size*channels*sizeof(float) + * @param [in] frame_size Number of samples per channel of available space in *pcm. + * @returns Number of decoded samples or @ref opus_errorcodes + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_decode_float( + OpusCustomDecoder *st, + const unsigned char *data, + int len, + float *pcm, + int frame_size +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Decode an opus custom frame + * @param [in] st OpusCustomDecoder*: Decoder state + * @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss + * @param [in] len int: Number of bytes in payload + * @param [out] pcm opus_int16*: Output signal (interleaved if 2 channels). length + * is frame_size*channels*sizeof(opus_int16) + * @param [in] frame_size Number of samples per channel of available space in *pcm. + * @returns Number of decoded samples or @ref opus_errorcodes + */ +OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_decode( + OpusCustomDecoder *st, + const unsigned char *data, + int len, + opus_int16 *pcm, + int frame_size +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Perform a CTL function on an Opus custom decoder. + * + * Generally the request and subsequent arguments are generated + * by a convenience macro. + * @see opus_genericctls + */ +OPUS_CUSTOM_EXPORT int opus_custom_decoder_ctl(OpusCustomDecoder * OPUS_RESTRICT st, int request, ...) OPUS_ARG_NONNULL(1); + +/**@}*/ + +#ifdef __cplusplus +} +#endif + +#endif /* OPUS_CUSTOM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_defines.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_defines.h new file mode 100644 index 000000000..ceee5b840 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_defines.h @@ -0,0 +1,799 @@ +/* Copyright (c) 2010-2011 Xiph.Org Foundation, Skype Limited + Written by Jean-Marc Valin and Koen Vos */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + * @file opus_defines.h + * @brief Opus reference implementation constants + */ + +#ifndef OPUS_DEFINES_H +#define OPUS_DEFINES_H + +#include "opus_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup opus_errorcodes Error codes + * @{ + */ +/** No error @hideinitializer*/ +#define OPUS_OK 0 +/** One or more invalid/out of range arguments @hideinitializer*/ +#define OPUS_BAD_ARG -1 +/** Not enough bytes allocated in the buffer @hideinitializer*/ +#define OPUS_BUFFER_TOO_SMALL -2 +/** An internal error was detected @hideinitializer*/ +#define OPUS_INTERNAL_ERROR -3 +/** The compressed data passed is corrupted @hideinitializer*/ +#define OPUS_INVALID_PACKET -4 +/** Invalid/unsupported request number @hideinitializer*/ +#define OPUS_UNIMPLEMENTED -5 +/** An encoder or decoder structure is invalid or already freed @hideinitializer*/ +#define OPUS_INVALID_STATE -6 +/** Memory allocation has failed @hideinitializer*/ +#define OPUS_ALLOC_FAIL -7 +/**@}*/ + +/** @cond OPUS_INTERNAL_DOC */ +/**Export control for opus functions */ + +#ifndef OPUS_EXPORT +# if defined(_WIN32) +# if defined(OPUS_BUILD) && defined(DLL_EXPORT) +# define OPUS_EXPORT __declspec(dllexport) +# else +# define OPUS_EXPORT +# endif +# elif defined(__GNUC__) && defined(OPUS_BUILD) +# define OPUS_EXPORT __attribute__ ((visibility ("default"))) +# else +# define OPUS_EXPORT +# endif +#endif + +# if !defined(OPUS_GNUC_PREREQ) +# if defined(__GNUC__)&&defined(__GNUC_MINOR__) +# define OPUS_GNUC_PREREQ(_maj,_min) \ + ((__GNUC__<<16)+__GNUC_MINOR__>=((_maj)<<16)+(_min)) +# else +# define OPUS_GNUC_PREREQ(_maj,_min) 0 +# endif +# endif + +#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) ) +# if OPUS_GNUC_PREREQ(3,0) +# define OPUS_RESTRICT __restrict__ +# elif (defined(_MSC_VER) && _MSC_VER >= 1400) +# define OPUS_RESTRICT __restrict +# else +# define OPUS_RESTRICT +# endif +#else +# define OPUS_RESTRICT restrict +#endif + +#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) ) +# if OPUS_GNUC_PREREQ(2,7) +# define OPUS_INLINE __inline__ +# elif (defined(_MSC_VER)) +# define OPUS_INLINE __inline +# else +# define OPUS_INLINE +# endif +#else +# define OPUS_INLINE inline +#endif + +/**Warning attributes for opus functions + * NONNULL is not used in OPUS_BUILD to avoid the compiler optimizing out + * some paranoid null checks. */ +#if defined(__GNUC__) && OPUS_GNUC_PREREQ(3, 4) +# define OPUS_WARN_UNUSED_RESULT __attribute__ ((__warn_unused_result__)) +#else +# define OPUS_WARN_UNUSED_RESULT +#endif +#if !defined(OPUS_BUILD) && defined(__GNUC__) && OPUS_GNUC_PREREQ(3, 4) +# define OPUS_ARG_NONNULL(_x) __attribute__ ((__nonnull__(_x))) +#else +# define OPUS_ARG_NONNULL(_x) +#endif + +/** These are the actual Encoder CTL ID numbers. + * They should not be used directly by applications. + * In general, SETs should be even and GETs should be odd.*/ +#define OPUS_SET_APPLICATION_REQUEST 4000 +#define OPUS_GET_APPLICATION_REQUEST 4001 +#define OPUS_SET_BITRATE_REQUEST 4002 +#define OPUS_GET_BITRATE_REQUEST 4003 +#define OPUS_SET_MAX_BANDWIDTH_REQUEST 4004 +#define OPUS_GET_MAX_BANDWIDTH_REQUEST 4005 +#define OPUS_SET_VBR_REQUEST 4006 +#define OPUS_GET_VBR_REQUEST 4007 +#define OPUS_SET_BANDWIDTH_REQUEST 4008 +#define OPUS_GET_BANDWIDTH_REQUEST 4009 +#define OPUS_SET_COMPLEXITY_REQUEST 4010 +#define OPUS_GET_COMPLEXITY_REQUEST 4011 +#define OPUS_SET_INBAND_FEC_REQUEST 4012 +#define OPUS_GET_INBAND_FEC_REQUEST 4013 +#define OPUS_SET_PACKET_LOSS_PERC_REQUEST 4014 +#define OPUS_GET_PACKET_LOSS_PERC_REQUEST 4015 +#define OPUS_SET_DTX_REQUEST 4016 +#define OPUS_GET_DTX_REQUEST 4017 +#define OPUS_SET_VBR_CONSTRAINT_REQUEST 4020 +#define OPUS_GET_VBR_CONSTRAINT_REQUEST 4021 +#define OPUS_SET_FORCE_CHANNELS_REQUEST 4022 +#define OPUS_GET_FORCE_CHANNELS_REQUEST 4023 +#define OPUS_SET_SIGNAL_REQUEST 4024 +#define OPUS_GET_SIGNAL_REQUEST 4025 +#define OPUS_GET_LOOKAHEAD_REQUEST 4027 +/* #define OPUS_RESET_STATE 4028 */ +#define OPUS_GET_SAMPLE_RATE_REQUEST 4029 +#define OPUS_GET_FINAL_RANGE_REQUEST 4031 +#define OPUS_GET_PITCH_REQUEST 4033 +#define OPUS_SET_GAIN_REQUEST 4034 +#define OPUS_GET_GAIN_REQUEST 4045 /* Should have been 4035 */ +#define OPUS_SET_LSB_DEPTH_REQUEST 4036 +#define OPUS_GET_LSB_DEPTH_REQUEST 4037 +#define OPUS_GET_LAST_PACKET_DURATION_REQUEST 4039 +#define OPUS_SET_EXPERT_FRAME_DURATION_REQUEST 4040 +#define OPUS_GET_EXPERT_FRAME_DURATION_REQUEST 4041 +#define OPUS_SET_PREDICTION_DISABLED_REQUEST 4042 +#define OPUS_GET_PREDICTION_DISABLED_REQUEST 4043 +/* Don't use 4045, it's already taken by OPUS_GET_GAIN_REQUEST */ +#define OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST 4046 +#define OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST 4047 +#define OPUS_GET_IN_DTX_REQUEST 4049 + +/** Defines for the presence of extended APIs. */ +#define OPUS_HAVE_OPUS_PROJECTION_H + +/* Macros to trigger compilation errors when the wrong types are provided to a CTL */ +#define __opus_check_int(x) (((void)((x) == (opus_int32)0)), (opus_int32)(x)) +#define __opus_check_int_ptr(ptr) ((ptr) + ((ptr) - (opus_int32*)(ptr))) +#define __opus_check_uint_ptr(ptr) ((ptr) + ((ptr) - (opus_uint32*)(ptr))) +#define __opus_check_val16_ptr(ptr) ((ptr) + ((ptr) - (opus_val16*)(ptr))) +/** @endcond */ + +/** @defgroup opus_ctlvalues Pre-defined values for CTL interface + * @see opus_genericctls, opus_encoderctls + * @{ + */ +/* Values for the various encoder CTLs */ +#define OPUS_AUTO -1000 /**opus_int32: Allowed values: 0-10, inclusive. + * + * @hideinitializer */ +#define OPUS_SET_COMPLEXITY(x) OPUS_SET_COMPLEXITY_REQUEST, __opus_check_int(x) +/** Gets the encoder's complexity configuration. + * @see OPUS_SET_COMPLEXITY + * @param[out] x opus_int32 *: Returns a value in the range 0-10, + * inclusive. + * @hideinitializer */ +#define OPUS_GET_COMPLEXITY(x) OPUS_GET_COMPLEXITY_REQUEST, __opus_check_int_ptr(x) + +/** Configures the bitrate in the encoder. + * Rates from 500 to 512000 bits per second are meaningful, as well as the + * special values #OPUS_AUTO and #OPUS_BITRATE_MAX. + * The value #OPUS_BITRATE_MAX can be used to cause the codec to use as much + * rate as it can, which is useful for controlling the rate by adjusting the + * output buffer size. + * @see OPUS_GET_BITRATE + * @param[in] x opus_int32: Bitrate in bits per second. The default + * is determined based on the number of + * channels and the input sampling rate. + * @hideinitializer */ +#define OPUS_SET_BITRATE(x) OPUS_SET_BITRATE_REQUEST, __opus_check_int(x) +/** Gets the encoder's bitrate configuration. + * @see OPUS_SET_BITRATE + * @param[out] x opus_int32 *: Returns the bitrate in bits per second. + * The default is determined based on the + * number of channels and the input + * sampling rate. + * @hideinitializer */ +#define OPUS_GET_BITRATE(x) OPUS_GET_BITRATE_REQUEST, __opus_check_int_ptr(x) + +/** Enables or disables variable bitrate (VBR) in the encoder. + * The configured bitrate may not be met exactly because frames must + * be an integer number of bytes in length. + * @see OPUS_GET_VBR + * @see OPUS_SET_VBR_CONSTRAINT + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Hard CBR. For LPC/hybrid modes at very low bit-rate, this can + * cause noticeable quality degradation.
+ *
1
VBR (default). The exact type of VBR is controlled by + * #OPUS_SET_VBR_CONSTRAINT.
+ *
+ * @hideinitializer */ +#define OPUS_SET_VBR(x) OPUS_SET_VBR_REQUEST, __opus_check_int(x) +/** Determine if variable bitrate (VBR) is enabled in the encoder. + * @see OPUS_SET_VBR + * @see OPUS_GET_VBR_CONSTRAINT + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
Hard CBR.
+ *
1
VBR (default). The exact type of VBR may be retrieved via + * #OPUS_GET_VBR_CONSTRAINT.
+ *
+ * @hideinitializer */ +#define OPUS_GET_VBR(x) OPUS_GET_VBR_REQUEST, __opus_check_int_ptr(x) + +/** Enables or disables constrained VBR in the encoder. + * This setting is ignored when the encoder is in CBR mode. + * @warning Only the MDCT mode of Opus currently heeds the constraint. + * Speech mode ignores it completely, hybrid mode may fail to obey it + * if the LPC layer uses more bitrate than the constraint would have + * permitted. + * @see OPUS_GET_VBR_CONSTRAINT + * @see OPUS_SET_VBR + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Unconstrained VBR.
+ *
1
Constrained VBR (default). This creates a maximum of one + * frame of buffering delay assuming a transport with a + * serialization speed of the nominal bitrate.
+ *
+ * @hideinitializer */ +#define OPUS_SET_VBR_CONSTRAINT(x) OPUS_SET_VBR_CONSTRAINT_REQUEST, __opus_check_int(x) +/** Determine if constrained VBR is enabled in the encoder. + * @see OPUS_SET_VBR_CONSTRAINT + * @see OPUS_GET_VBR + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
Unconstrained VBR.
+ *
1
Constrained VBR (default).
+ *
+ * @hideinitializer */ +#define OPUS_GET_VBR_CONSTRAINT(x) OPUS_GET_VBR_CONSTRAINT_REQUEST, __opus_check_int_ptr(x) + +/** Configures mono/stereo forcing in the encoder. + * This can force the encoder to produce packets encoded as either mono or + * stereo, regardless of the format of the input audio. This is useful when + * the caller knows that the input signal is currently a mono source embedded + * in a stereo stream. + * @see OPUS_GET_FORCE_CHANNELS + * @param[in] x opus_int32: Allowed values: + *
+ *
#OPUS_AUTO
Not forced (default)
+ *
1
Forced mono
+ *
2
Forced stereo
+ *
+ * @hideinitializer */ +#define OPUS_SET_FORCE_CHANNELS(x) OPUS_SET_FORCE_CHANNELS_REQUEST, __opus_check_int(x) +/** Gets the encoder's forced channel configuration. + * @see OPUS_SET_FORCE_CHANNELS + * @param[out] x opus_int32 *: + *
+ *
#OPUS_AUTO
Not forced (default)
+ *
1
Forced mono
+ *
2
Forced stereo
+ *
+ * @hideinitializer */ +#define OPUS_GET_FORCE_CHANNELS(x) OPUS_GET_FORCE_CHANNELS_REQUEST, __opus_check_int_ptr(x) + +/** Configures the maximum bandpass that the encoder will select automatically. + * Applications should normally use this instead of #OPUS_SET_BANDWIDTH + * (leaving that set to the default, #OPUS_AUTO). This allows the + * application to set an upper bound based on the type of input it is + * providing, but still gives the encoder the freedom to reduce the bandpass + * when the bitrate becomes too low, for better overall quality. + * @see OPUS_GET_MAX_BANDWIDTH + * @param[in] x opus_int32: Allowed values: + *
+ *
OPUS_BANDWIDTH_NARROWBAND
4 kHz passband
+ *
OPUS_BANDWIDTH_MEDIUMBAND
6 kHz passband
+ *
OPUS_BANDWIDTH_WIDEBAND
8 kHz passband
+ *
OPUS_BANDWIDTH_SUPERWIDEBAND
12 kHz passband
+ *
OPUS_BANDWIDTH_FULLBAND
20 kHz passband (default)
+ *
+ * @hideinitializer */ +#define OPUS_SET_MAX_BANDWIDTH(x) OPUS_SET_MAX_BANDWIDTH_REQUEST, __opus_check_int(x) + +/** Gets the encoder's configured maximum allowed bandpass. + * @see OPUS_SET_MAX_BANDWIDTH + * @param[out] x opus_int32 *: Allowed values: + *
+ *
#OPUS_BANDWIDTH_NARROWBAND
4 kHz passband
+ *
#OPUS_BANDWIDTH_MEDIUMBAND
6 kHz passband
+ *
#OPUS_BANDWIDTH_WIDEBAND
8 kHz passband
+ *
#OPUS_BANDWIDTH_SUPERWIDEBAND
12 kHz passband
+ *
#OPUS_BANDWIDTH_FULLBAND
20 kHz passband (default)
+ *
+ * @hideinitializer */ +#define OPUS_GET_MAX_BANDWIDTH(x) OPUS_GET_MAX_BANDWIDTH_REQUEST, __opus_check_int_ptr(x) + +/** Sets the encoder's bandpass to a specific value. + * This prevents the encoder from automatically selecting the bandpass based + * on the available bitrate. If an application knows the bandpass of the input + * audio it is providing, it should normally use #OPUS_SET_MAX_BANDWIDTH + * instead, which still gives the encoder the freedom to reduce the bandpass + * when the bitrate becomes too low, for better overall quality. + * @see OPUS_GET_BANDWIDTH + * @param[in] x opus_int32: Allowed values: + *
+ *
#OPUS_AUTO
(default)
+ *
#OPUS_BANDWIDTH_NARROWBAND
4 kHz passband
+ *
#OPUS_BANDWIDTH_MEDIUMBAND
6 kHz passband
+ *
#OPUS_BANDWIDTH_WIDEBAND
8 kHz passband
+ *
#OPUS_BANDWIDTH_SUPERWIDEBAND
12 kHz passband
+ *
#OPUS_BANDWIDTH_FULLBAND
20 kHz passband
+ *
+ * @hideinitializer */ +#define OPUS_SET_BANDWIDTH(x) OPUS_SET_BANDWIDTH_REQUEST, __opus_check_int(x) + +/** Configures the type of signal being encoded. + * This is a hint which helps the encoder's mode selection. + * @see OPUS_GET_SIGNAL + * @param[in] x opus_int32: Allowed values: + *
+ *
#OPUS_AUTO
(default)
+ *
#OPUS_SIGNAL_VOICE
Bias thresholds towards choosing LPC or Hybrid modes.
+ *
#OPUS_SIGNAL_MUSIC
Bias thresholds towards choosing MDCT modes.
+ *
+ * @hideinitializer */ +#define OPUS_SET_SIGNAL(x) OPUS_SET_SIGNAL_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured signal type. + * @see OPUS_SET_SIGNAL + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
#OPUS_AUTO
(default)
+ *
#OPUS_SIGNAL_VOICE
Bias thresholds towards choosing LPC or Hybrid modes.
+ *
#OPUS_SIGNAL_MUSIC
Bias thresholds towards choosing MDCT modes.
+ *
+ * @hideinitializer */ +#define OPUS_GET_SIGNAL(x) OPUS_GET_SIGNAL_REQUEST, __opus_check_int_ptr(x) + + +/** Configures the encoder's intended application. + * The initial value is a mandatory argument to the encoder_create function. + * @see OPUS_GET_APPLICATION + * @param[in] x opus_int32: Returns one of the following values: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @hideinitializer */ +#define OPUS_SET_APPLICATION(x) OPUS_SET_APPLICATION_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured application. + * @see OPUS_SET_APPLICATION + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @hideinitializer */ +#define OPUS_GET_APPLICATION(x) OPUS_GET_APPLICATION_REQUEST, __opus_check_int_ptr(x) + +/** Gets the total samples of delay added by the entire codec. + * This can be queried by the encoder and then the provided number of samples can be + * skipped on from the start of the decoder's output to provide time aligned input + * and output. From the perspective of a decoding application the real data begins this many + * samples late. + * + * The decoder contribution to this delay is identical for all decoders, but the + * encoder portion of the delay may vary from implementation to implementation, + * version to version, or even depend on the encoder's initial configuration. + * Applications needing delay compensation should call this CTL rather than + * hard-coding a value. + * @param[out] x opus_int32 *: Number of lookahead samples + * @hideinitializer */ +#define OPUS_GET_LOOKAHEAD(x) OPUS_GET_LOOKAHEAD_REQUEST, __opus_check_int_ptr(x) + +/** Configures the encoder's use of inband forward error correction (FEC). + * @note This is only applicable to the LPC layer + * @see OPUS_GET_INBAND_FEC + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Disable inband FEC (default).
+ *
1
Enable inband FEC.
+ *
+ * @hideinitializer */ +#define OPUS_SET_INBAND_FEC(x) OPUS_SET_INBAND_FEC_REQUEST, __opus_check_int(x) +/** Gets encoder's configured use of inband forward error correction. + * @see OPUS_SET_INBAND_FEC + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
Inband FEC disabled (default).
+ *
1
Inband FEC enabled.
+ *
+ * @hideinitializer */ +#define OPUS_GET_INBAND_FEC(x) OPUS_GET_INBAND_FEC_REQUEST, __opus_check_int_ptr(x) + +/** Configures the encoder's expected packet loss percentage. + * Higher values trigger progressively more loss resistant behavior in the encoder + * at the expense of quality at a given bitrate in the absence of packet loss, but + * greater quality under loss. + * @see OPUS_GET_PACKET_LOSS_PERC + * @param[in] x opus_int32: Loss percentage in the range 0-100, inclusive (default: 0). + * @hideinitializer */ +#define OPUS_SET_PACKET_LOSS_PERC(x) OPUS_SET_PACKET_LOSS_PERC_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured packet loss percentage. + * @see OPUS_SET_PACKET_LOSS_PERC + * @param[out] x opus_int32 *: Returns the configured loss percentage + * in the range 0-100, inclusive (default: 0). + * @hideinitializer */ +#define OPUS_GET_PACKET_LOSS_PERC(x) OPUS_GET_PACKET_LOSS_PERC_REQUEST, __opus_check_int_ptr(x) + +/** Configures the encoder's use of discontinuous transmission (DTX). + * @note This is only applicable to the LPC layer + * @see OPUS_GET_DTX + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Disable DTX (default).
+ *
1
Enabled DTX.
+ *
+ * @hideinitializer */ +#define OPUS_SET_DTX(x) OPUS_SET_DTX_REQUEST, __opus_check_int(x) +/** Gets encoder's configured use of discontinuous transmission. + * @see OPUS_SET_DTX + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
DTX disabled (default).
+ *
1
DTX enabled.
+ *
+ * @hideinitializer */ +#define OPUS_GET_DTX(x) OPUS_GET_DTX_REQUEST, __opus_check_int_ptr(x) +/** Configures the depth of signal being encoded. + * + * This is a hint which helps the encoder identify silence and near-silence. + * It represents the number of significant bits of linear intensity below + * which the signal contains ignorable quantization or other noise. + * + * For example, OPUS_SET_LSB_DEPTH(14) would be an appropriate setting + * for G.711 u-law input. OPUS_SET_LSB_DEPTH(16) would be appropriate + * for 16-bit linear pcm input with opus_encode_float(). + * + * When using opus_encode() instead of opus_encode_float(), or when libopus + * is compiled for fixed-point, the encoder uses the minimum of the value + * set here and the value 16. + * + * @see OPUS_GET_LSB_DEPTH + * @param[in] x opus_int32: Input precision in bits, between 8 and 24 + * (default: 24). + * @hideinitializer */ +#define OPUS_SET_LSB_DEPTH(x) OPUS_SET_LSB_DEPTH_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured signal depth. + * @see OPUS_SET_LSB_DEPTH + * @param[out] x opus_int32 *: Input precision in bits, between 8 and + * 24 (default: 24). + * @hideinitializer */ +#define OPUS_GET_LSB_DEPTH(x) OPUS_GET_LSB_DEPTH_REQUEST, __opus_check_int_ptr(x) + +/** Configures the encoder's use of variable duration frames. + * When variable duration is enabled, the encoder is free to use a shorter frame + * size than the one requested in the opus_encode*() call. + * It is then the user's responsibility + * to verify how much audio was encoded by checking the ToC byte of the encoded + * packet. The part of the audio that was not encoded needs to be resent to the + * encoder for the next call. Do not use this option unless you really + * know what you are doing. + * @see OPUS_GET_EXPERT_FRAME_DURATION + * @param[in] x opus_int32: Allowed values: + *
+ *
OPUS_FRAMESIZE_ARG
Select frame size from the argument (default).
+ *
OPUS_FRAMESIZE_2_5_MS
Use 2.5 ms frames.
+ *
OPUS_FRAMESIZE_5_MS
Use 5 ms frames.
+ *
OPUS_FRAMESIZE_10_MS
Use 10 ms frames.
+ *
OPUS_FRAMESIZE_20_MS
Use 20 ms frames.
+ *
OPUS_FRAMESIZE_40_MS
Use 40 ms frames.
+ *
OPUS_FRAMESIZE_60_MS
Use 60 ms frames.
+ *
OPUS_FRAMESIZE_80_MS
Use 80 ms frames.
+ *
OPUS_FRAMESIZE_100_MS
Use 100 ms frames.
+ *
OPUS_FRAMESIZE_120_MS
Use 120 ms frames.
+ *
+ * @hideinitializer */ +#define OPUS_SET_EXPERT_FRAME_DURATION(x) OPUS_SET_EXPERT_FRAME_DURATION_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured use of variable duration frames. + * @see OPUS_SET_EXPERT_FRAME_DURATION + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
OPUS_FRAMESIZE_ARG
Select frame size from the argument (default).
+ *
OPUS_FRAMESIZE_2_5_MS
Use 2.5 ms frames.
+ *
OPUS_FRAMESIZE_5_MS
Use 5 ms frames.
+ *
OPUS_FRAMESIZE_10_MS
Use 10 ms frames.
+ *
OPUS_FRAMESIZE_20_MS
Use 20 ms frames.
+ *
OPUS_FRAMESIZE_40_MS
Use 40 ms frames.
+ *
OPUS_FRAMESIZE_60_MS
Use 60 ms frames.
+ *
OPUS_FRAMESIZE_80_MS
Use 80 ms frames.
+ *
OPUS_FRAMESIZE_100_MS
Use 100 ms frames.
+ *
OPUS_FRAMESIZE_120_MS
Use 120 ms frames.
+ *
+ * @hideinitializer */ +#define OPUS_GET_EXPERT_FRAME_DURATION(x) OPUS_GET_EXPERT_FRAME_DURATION_REQUEST, __opus_check_int_ptr(x) + +/** If set to 1, disables almost all use of prediction, making frames almost + * completely independent. This reduces quality. + * @see OPUS_GET_PREDICTION_DISABLED + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Enable prediction (default).
+ *
1
Disable prediction.
+ *
+ * @hideinitializer */ +#define OPUS_SET_PREDICTION_DISABLED(x) OPUS_SET_PREDICTION_DISABLED_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured prediction status. + * @see OPUS_SET_PREDICTION_DISABLED + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
Prediction enabled (default).
+ *
1
Prediction disabled.
+ *
+ * @hideinitializer */ +#define OPUS_GET_PREDICTION_DISABLED(x) OPUS_GET_PREDICTION_DISABLED_REQUEST, __opus_check_int_ptr(x) + +/**@}*/ + +/** @defgroup opus_genericctls Generic CTLs + * + * These macros are used with the \c opus_decoder_ctl and + * \c opus_encoder_ctl calls to generate a particular + * request. + * + * When called on an \c OpusDecoder they apply to that + * particular decoder instance. When called on an + * \c OpusEncoder they apply to the corresponding setting + * on that encoder instance, if present. + * + * Some usage examples: + * + * @code + * int ret; + * opus_int32 pitch; + * ret = opus_decoder_ctl(dec_ctx, OPUS_GET_PITCH(&pitch)); + * if (ret == OPUS_OK) return ret; + * + * opus_encoder_ctl(enc_ctx, OPUS_RESET_STATE); + * opus_decoder_ctl(dec_ctx, OPUS_RESET_STATE); + * + * opus_int32 enc_bw, dec_bw; + * opus_encoder_ctl(enc_ctx, OPUS_GET_BANDWIDTH(&enc_bw)); + * opus_decoder_ctl(dec_ctx, OPUS_GET_BANDWIDTH(&dec_bw)); + * if (enc_bw != dec_bw) { + * printf("packet bandwidth mismatch!\n"); + * } + * @endcode + * + * @see opus_encoder, opus_decoder_ctl, opus_encoder_ctl, opus_decoderctls, opus_encoderctls + * @{ + */ + +/** Resets the codec state to be equivalent to a freshly initialized state. + * This should be called when switching streams in order to prevent + * the back to back decoding from giving different results from + * one at a time decoding. + * @hideinitializer */ +#define OPUS_RESET_STATE 4028 + +/** Gets the final state of the codec's entropy coder. + * This is used for testing purposes, + * The encoder and decoder state should be identical after coding a payload + * (assuming no data corruption or software bugs) + * + * @param[out] x opus_uint32 *: Entropy coder state + * + * @hideinitializer */ +#define OPUS_GET_FINAL_RANGE(x) OPUS_GET_FINAL_RANGE_REQUEST, __opus_check_uint_ptr(x) + +/** Gets the encoder's configured bandpass or the decoder's last bandpass. + * @see OPUS_SET_BANDWIDTH + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
#OPUS_AUTO
(default)
+ *
#OPUS_BANDWIDTH_NARROWBAND
4 kHz passband
+ *
#OPUS_BANDWIDTH_MEDIUMBAND
6 kHz passband
+ *
#OPUS_BANDWIDTH_WIDEBAND
8 kHz passband
+ *
#OPUS_BANDWIDTH_SUPERWIDEBAND
12 kHz passband
+ *
#OPUS_BANDWIDTH_FULLBAND
20 kHz passband
+ *
+ * @hideinitializer */ +#define OPUS_GET_BANDWIDTH(x) OPUS_GET_BANDWIDTH_REQUEST, __opus_check_int_ptr(x) + +/** Gets the sampling rate the encoder or decoder was initialized with. + * This simply returns the Fs value passed to opus_encoder_init() + * or opus_decoder_init(). + * @param[out] x opus_int32 *: Sampling rate of encoder or decoder. + * @hideinitializer + */ +#define OPUS_GET_SAMPLE_RATE(x) OPUS_GET_SAMPLE_RATE_REQUEST, __opus_check_int_ptr(x) + +/** If set to 1, disables the use of phase inversion for intensity stereo, + * improving the quality of mono downmixes, but slightly reducing normal + * stereo quality. Disabling phase inversion in the decoder does not comply + * with RFC 6716, although it does not cause any interoperability issue and + * is expected to become part of the Opus standard once RFC 6716 is updated + * by draft-ietf-codec-opus-update. + * @see OPUS_GET_PHASE_INVERSION_DISABLED + * @param[in] x opus_int32: Allowed values: + *
+ *
0
Enable phase inversion (default).
+ *
1
Disable phase inversion.
+ *
+ * @hideinitializer */ +#define OPUS_SET_PHASE_INVERSION_DISABLED(x) OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured phase inversion status. + * @see OPUS_SET_PHASE_INVERSION_DISABLED + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
Stereo phase inversion enabled (default).
+ *
1
Stereo phase inversion disabled.
+ *
+ * @hideinitializer */ +#define OPUS_GET_PHASE_INVERSION_DISABLED(x) OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST, __opus_check_int_ptr(x) +/** Gets the DTX state of the encoder. + * Returns whether the last encoded frame was either a comfort noise update + * during DTX or not encoded because of DTX. + * @param[out] x opus_int32 *: Returns one of the following values: + *
+ *
0
The encoder is not in DTX.
+ *
1
The encoder is in DTX.
+ *
+ * @hideinitializer */ +#define OPUS_GET_IN_DTX(x) OPUS_GET_IN_DTX_REQUEST, __opus_check_int_ptr(x) + +/**@}*/ + +/** @defgroup opus_decoderctls Decoder related CTLs + * @see opus_genericctls, opus_encoderctls, opus_decoder + * @{ + */ + +/** Configures decoder gain adjustment. + * Scales the decoded output by a factor specified in Q8 dB units. + * This has a maximum range of -32768 to 32767 inclusive, and returns + * OPUS_BAD_ARG otherwise. The default is zero indicating no adjustment. + * This setting survives decoder reset. + * + * gain = pow(10, x/(20.0*256)) + * + * @param[in] x opus_int32: Amount to scale PCM signal by in Q8 dB units. + * @hideinitializer */ +#define OPUS_SET_GAIN(x) OPUS_SET_GAIN_REQUEST, __opus_check_int(x) +/** Gets the decoder's configured gain adjustment. @see OPUS_SET_GAIN + * + * @param[out] x opus_int32 *: Amount to scale PCM signal by in Q8 dB units. + * @hideinitializer */ +#define OPUS_GET_GAIN(x) OPUS_GET_GAIN_REQUEST, __opus_check_int_ptr(x) + +/** Gets the duration (in samples) of the last packet successfully decoded or concealed. + * @param[out] x opus_int32 *: Number of samples (at current sampling rate). + * @hideinitializer */ +#define OPUS_GET_LAST_PACKET_DURATION(x) OPUS_GET_LAST_PACKET_DURATION_REQUEST, __opus_check_int_ptr(x) + +/** Gets the pitch of the last decoded frame, if available. + * This can be used for any post-processing algorithm requiring the use of pitch, + * e.g. time stretching/shortening. If the last frame was not voiced, or if the + * pitch was not coded in the frame, then zero is returned. + * + * This CTL is only implemented for decoder instances. + * + * @param[out] x opus_int32 *: pitch period at 48 kHz (or 0 if not available) + * + * @hideinitializer */ +#define OPUS_GET_PITCH(x) OPUS_GET_PITCH_REQUEST, __opus_check_int_ptr(x) + +/**@}*/ + +/** @defgroup opus_libinfo Opus library information functions + * @{ + */ + +/** Converts an opus error code into a human readable string. + * + * @param[in] error int: Error number + * @returns Error string + */ +OPUS_EXPORT const char *opus_strerror(int error); + +/** Gets the libopus version string. + * + * Applications may look for the substring "-fixed" in the version string to + * determine whether they have a fixed-point or floating-point build at + * runtime. + * + * @returns Version string + */ +OPUS_EXPORT const char *opus_get_version_string(void); +/**@}*/ + +#ifdef __cplusplus +} +#endif + +#endif /* OPUS_DEFINES_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_multistream.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_multistream.h new file mode 100644 index 000000000..babcee690 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_multistream.h @@ -0,0 +1,660 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + * @file opus_multistream.h + * @brief Opus reference implementation multistream API + */ + +#ifndef OPUS_MULTISTREAM_H +#define OPUS_MULTISTREAM_H + +#include "opus.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** @cond OPUS_INTERNAL_DOC */ + +/** Macros to trigger compilation errors when the wrong types are provided to a + * CTL. */ +/**@{*/ +#define __opus_check_encstate_ptr(ptr) ((ptr) + ((ptr) - (OpusEncoder**)(ptr))) +#define __opus_check_decstate_ptr(ptr) ((ptr) + ((ptr) - (OpusDecoder**)(ptr))) +/**@}*/ + +/** These are the actual encoder and decoder CTL ID numbers. + * They should not be used directly by applications. + * In general, SETs should be even and GETs should be odd.*/ +/**@{*/ +#define OPUS_MULTISTREAM_GET_ENCODER_STATE_REQUEST 5120 +#define OPUS_MULTISTREAM_GET_DECODER_STATE_REQUEST 5122 +/**@}*/ + +/** @endcond */ + +/** @defgroup opus_multistream_ctls Multistream specific encoder and decoder CTLs + * + * These are convenience macros that are specific to the + * opus_multistream_encoder_ctl() and opus_multistream_decoder_ctl() + * interface. + * The CTLs from @ref opus_genericctls, @ref opus_encoderctls, and + * @ref opus_decoderctls may be applied to a multistream encoder or decoder as + * well. + * In addition, you may retrieve the encoder or decoder state for an specific + * stream via #OPUS_MULTISTREAM_GET_ENCODER_STATE or + * #OPUS_MULTISTREAM_GET_DECODER_STATE and apply CTLs to it individually. + */ +/**@{*/ + +/** Gets the encoder state for an individual stream of a multistream encoder. + * @param[in] x opus_int32: The index of the stream whose encoder you + * wish to retrieve. + * This must be non-negative and less than + * the streams parameter used + * to initialize the encoder. + * @param[out] y OpusEncoder**: Returns a pointer to the given + * encoder state. + * @retval OPUS_BAD_ARG The index of the requested stream was out of range. + * @hideinitializer + */ +#define OPUS_MULTISTREAM_GET_ENCODER_STATE(x,y) OPUS_MULTISTREAM_GET_ENCODER_STATE_REQUEST, __opus_check_int(x), __opus_check_encstate_ptr(y) + +/** Gets the decoder state for an individual stream of a multistream decoder. + * @param[in] x opus_int32: The index of the stream whose decoder you + * wish to retrieve. + * This must be non-negative and less than + * the streams parameter used + * to initialize the decoder. + * @param[out] y OpusDecoder**: Returns a pointer to the given + * decoder state. + * @retval OPUS_BAD_ARG The index of the requested stream was out of range. + * @hideinitializer + */ +#define OPUS_MULTISTREAM_GET_DECODER_STATE(x,y) OPUS_MULTISTREAM_GET_DECODER_STATE_REQUEST, __opus_check_int(x), __opus_check_decstate_ptr(y) + +/**@}*/ + +/** @defgroup opus_multistream Opus Multistream API + * @{ + * + * The multistream API allows individual Opus streams to be combined into a + * single packet, enabling support for up to 255 channels. Unlike an + * elementary Opus stream, the encoder and decoder must negotiate the channel + * configuration before the decoder can successfully interpret the data in the + * packets produced by the encoder. Some basic information, such as packet + * duration, can be computed without any special negotiation. + * + * The format for multistream Opus packets is defined in + * RFC 7845 + * and is based on the self-delimited Opus framing described in Appendix B of + * RFC 6716. + * Normal Opus packets are just a degenerate case of multistream Opus packets, + * and can be encoded or decoded with the multistream API by setting + * streams to 1 when initializing the encoder or + * decoder. + * + * Multistream Opus streams can contain up to 255 elementary Opus streams. + * These may be either "uncoupled" or "coupled", indicating that the decoder + * is configured to decode them to either 1 or 2 channels, respectively. + * The streams are ordered so that all coupled streams appear at the + * beginning. + * + * A mapping table defines which decoded channel i + * should be used for each input/output (I/O) channel j. This table is + * typically provided as an unsigned char array. + * Let i = mapping[j] be the index for I/O channel j. + * If i < 2*coupled_streams, then I/O channel j is + * encoded as the left channel of stream (i/2) if i + * is even, or as the right channel of stream (i/2) if + * i is odd. Otherwise, I/O channel j is encoded as + * mono in stream (i - coupled_streams), unless it has the special + * value 255, in which case it is omitted from the encoding entirely (the + * decoder will reproduce it as silence). Each value i must either + * be the special value 255 or be less than streams + coupled_streams. + * + * The output channels specified by the encoder + * should use the + * Vorbis + * channel ordering. A decoder may wish to apply an additional permutation + * to the mapping the encoder used to achieve a different output channel + * order (e.g. for outputing in WAV order). + * + * Each multistream packet contains an Opus packet for each stream, and all of + * the Opus packets in a single multistream packet must have the same + * duration. Therefore the duration of a multistream packet can be extracted + * from the TOC sequence of the first stream, which is located at the + * beginning of the packet, just like an elementary Opus stream: + * + * @code + * int nb_samples; + * int nb_frames; + * nb_frames = opus_packet_get_nb_frames(data, len); + * if (nb_frames < 1) + * return nb_frames; + * nb_samples = opus_packet_get_samples_per_frame(data, 48000) * nb_frames; + * @endcode + * + * The general encoding and decoding process proceeds exactly the same as in + * the normal @ref opus_encoder and @ref opus_decoder APIs. + * See their documentation for an overview of how to use the corresponding + * multistream functions. + */ + +/** Opus multistream encoder state. + * This contains the complete state of a multistream Opus encoder. + * It is position independent and can be freely copied. + * @see opus_multistream_encoder_create + * @see opus_multistream_encoder_init + */ +typedef struct OpusMSEncoder OpusMSEncoder; + +/** Opus multistream decoder state. + * This contains the complete state of a multistream Opus decoder. + * It is position independent and can be freely copied. + * @see opus_multistream_decoder_create + * @see opus_multistream_decoder_init + */ +typedef struct OpusMSDecoder OpusMSDecoder; + +/**\name Multistream encoder functions */ +/**@{*/ + +/** Gets the size of an OpusMSEncoder structure. + * @param streams int: The total number of streams to encode from the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number of coupled (2 channel) streams + * to encode. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * encoded channels (streams + + * coupled_streams) must be no + * more than 255. + * @returns The size in bytes on success, or a negative error code + * (see @ref opus_errorcodes) on error. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_multistream_encoder_get_size( + int streams, + int coupled_streams +); + +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_multistream_surround_encoder_get_size( + int channels, + int mapping_family +); + + +/** Allocates and initializes a multistream encoder state. + * Call opus_multistream_encoder_destroy() to release + * this object when finished. + * @param Fs opus_int32: Sampling rate of the input signal (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels in the input signal. + * This must be at most 255. + * It may be greater than the number of + * coded channels (streams + + * coupled_streams). + * @param streams int: The total number of streams to encode from the + * input. + * This must be no more than the number of channels. + * @param coupled_streams int: Number of coupled (2 channel) streams + * to encode. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * encoded channels (streams + + * coupled_streams) must be no + * more than the number of input channels. + * @param[in] mapping const unsigned char[channels]: Mapping from + * encoded channels to input channels, as described in + * @ref opus_multistream. As an extra constraint, the + * multistream encoder does not allow encoding coupled + * streams for which one channel is unused since this + * is never a good idea. + * @param application int: The target encoder application. + * This must be one of the following: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @param[out] error int *: Returns #OPUS_OK on success, or an error + * code (see @ref opus_errorcodes) on + * failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusMSEncoder *opus_multistream_encoder_create( + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int application, + int *error +) OPUS_ARG_NONNULL(5); + +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusMSEncoder *opus_multistream_surround_encoder_create( + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + unsigned char *mapping, + int application, + int *error +) OPUS_ARG_NONNULL(4) OPUS_ARG_NONNULL(5) OPUS_ARG_NONNULL(6); + +/** Initialize a previously allocated multistream encoder state. + * The memory pointed to by \a st must be at least the size returned by + * opus_multistream_encoder_get_size(). + * This is intended for applications which use their own allocator instead of + * malloc. + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @see opus_multistream_encoder_create + * @see opus_multistream_encoder_get_size + * @param st OpusMSEncoder*: Multistream encoder state to initialize. + * @param Fs opus_int32: Sampling rate of the input signal (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels in the input signal. + * This must be at most 255. + * It may be greater than the number of + * coded channels (streams + + * coupled_streams). + * @param streams int: The total number of streams to encode from the + * input. + * This must be no more than the number of channels. + * @param coupled_streams int: Number of coupled (2 channel) streams + * to encode. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * encoded channels (streams + + * coupled_streams) must be no + * more than the number of input channels. + * @param[in] mapping const unsigned char[channels]: Mapping from + * encoded channels to input channels, as described in + * @ref opus_multistream. As an extra constraint, the + * multistream encoder does not allow encoding coupled + * streams for which one channel is unused since this + * is never a good idea. + * @param application int: The target encoder application. + * This must be one of the following: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes) + * on failure. + */ +OPUS_EXPORT int opus_multistream_encoder_init( + OpusMSEncoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int application +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(6); + +OPUS_EXPORT int opus_multistream_surround_encoder_init( + OpusMSEncoder *st, + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + unsigned char *mapping, + int application +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(5) OPUS_ARG_NONNULL(6) OPUS_ARG_NONNULL(7); + +/** Encodes a multistream Opus frame. + * @param st OpusMSEncoder*: Multistream encoder state. + * @param[in] pcm const opus_int16*: The input signal as interleaved + * samples. + * This must contain + * frame_size*channels + * samples. + * @param frame_size int: Number of samples per channel in the input + * signal. + * This must be an Opus frame size for the + * encoder's sampling rate. + * For example, at 48 kHz the permitted values + * are 120, 240, 480, 960, 1920, and 2880. + * Passing in a duration of less than 10 ms + * (480 samples at 48 kHz) will prevent the + * encoder from using the LPC or hybrid modes. + * @param[out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_multistream_encode( + OpusMSEncoder *st, + const opus_int16 *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Encodes a multistream Opus frame from floating point input. + * @param st OpusMSEncoder*: Multistream encoder state. + * @param[in] pcm const float*: The input signal as interleaved + * samples with a normal range of + * +/-1.0. + * Samples with a range beyond +/-1.0 + * are supported but will be clipped by + * decoders using the integer API and + * should only be used if it is known + * that the far end supports extended + * dynamic range. + * This must contain + * frame_size*channels + * samples. + * @param frame_size int: Number of samples per channel in the input + * signal. + * This must be an Opus frame size for the + * encoder's sampling rate. + * For example, at 48 kHz the permitted values + * are 120, 240, 480, 960, 1920, and 2880. + * Passing in a duration of less than 10 ms + * (480 samples at 48 kHz) will prevent the + * encoder from using the LPC or hybrid modes. + * @param[out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_multistream_encode_float( + OpusMSEncoder *st, + const float *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + +/** Frees an OpusMSEncoder allocated by + * opus_multistream_encoder_create(). + * @param st OpusMSEncoder*: Multistream encoder state to be freed. + */ +OPUS_EXPORT void opus_multistream_encoder_destroy(OpusMSEncoder *st); + +/** Perform a CTL function on a multistream Opus encoder. + * + * Generally the request and subsequent arguments are generated by a + * convenience macro. + * @param st OpusMSEncoder*: Multistream encoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls, + * @ref opus_encoderctls, or @ref opus_multistream_ctls. + * @see opus_genericctls + * @see opus_encoderctls + * @see opus_multistream_ctls + */ +OPUS_EXPORT int opus_multistream_encoder_ctl(OpusMSEncoder *st, int request, ...) OPUS_ARG_NONNULL(1); + +/**@}*/ + +/**\name Multistream decoder functions */ +/**@{*/ + +/** Gets the size of an OpusMSDecoder structure. + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @returns The size in bytes on success, or a negative error code + * (see @ref opus_errorcodes) on error. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_multistream_decoder_get_size( + int streams, + int coupled_streams +); + +/** Allocates and initializes a multistream decoder state. + * Call opus_multistream_decoder_destroy() to release + * this object when finished. + * @param Fs opus_int32: Sampling rate to decode at (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels to output. + * This must be at most 255. + * It may be different from the number of coded + * channels (streams + + * coupled_streams). + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number of streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @param[in] mapping const unsigned char[channels]: Mapping from + * coded channels to output channels, as described in + * @ref opus_multistream. + * @param[out] error int *: Returns #OPUS_OK on success, or an error + * code (see @ref opus_errorcodes) on + * failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusMSDecoder *opus_multistream_decoder_create( + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int *error +) OPUS_ARG_NONNULL(5); + +/** Intialize a previously allocated decoder state object. + * The memory pointed to by \a st must be at least the size returned by + * opus_multistream_encoder_get_size(). + * This is intended for applications which use their own allocator instead of + * malloc. + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @see opus_multistream_decoder_create + * @see opus_multistream_deocder_get_size + * @param st OpusMSEncoder*: Multistream encoder state to initialize. + * @param Fs opus_int32: Sampling rate to decode at (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels to output. + * This must be at most 255. + * It may be different from the number of coded + * channels (streams + + * coupled_streams). + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number of streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @param[in] mapping const unsigned char[channels]: Mapping from + * coded channels to output channels, as described in + * @ref opus_multistream. + * @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes) + * on failure. + */ +OPUS_EXPORT int opus_multistream_decoder_init( + OpusMSDecoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(6); + +/** Decode a multistream Opus packet. + * @param st OpusMSDecoder*: Multistream decoder state. + * @param[in] data const unsigned char*: Input payload. + * Use a NULL + * pointer to indicate packet + * loss. + * @param len opus_int32: Number of bytes in payload. + * @param[out] pcm opus_int16*: Output signal, with interleaved + * samples. + * This must contain room for + * frame_size*channels + * samples. + * @param frame_size int: The number of samples per channel of + * available space in \a pcm. + * If this is less than the maximum packet duration + * (120 ms; 5760 for 48kHz), this function will not be capable + * of decoding some packets. In the case of PLC (data==NULL) + * or FEC (decode_fec=1), then frame_size needs to be exactly + * the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the + * next incoming packet. For the PLC and FEC cases, frame_size + * must be a multiple of 2.5 ms. + * @param decode_fec int: Flag (0 or 1) to request that any in-band + * forward error correction data be decoded. + * If no such data is available, the frame is + * decoded as if it were lost. + * @returns Number of samples decoded on success or a negative error code + * (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_multistream_decode( + OpusMSDecoder *st, + const unsigned char *data, + opus_int32 len, + opus_int16 *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Decode a multistream Opus packet with floating point output. + * @param st OpusMSDecoder*: Multistream decoder state. + * @param[in] data const unsigned char*: Input payload. + * Use a NULL + * pointer to indicate packet + * loss. + * @param len opus_int32: Number of bytes in payload. + * @param[out] pcm opus_int16*: Output signal, with interleaved + * samples. + * This must contain room for + * frame_size*channels + * samples. + * @param frame_size int: The number of samples per channel of + * available space in \a pcm. + * If this is less than the maximum packet duration + * (120 ms; 5760 for 48kHz), this function will not be capable + * of decoding some packets. In the case of PLC (data==NULL) + * or FEC (decode_fec=1), then frame_size needs to be exactly + * the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the + * next incoming packet. For the PLC and FEC cases, frame_size + * must be a multiple of 2.5 ms. + * @param decode_fec int: Flag (0 or 1) to request that any in-band + * forward error correction data be decoded. + * If no such data is available, the frame is + * decoded as if it were lost. + * @returns Number of samples decoded on success or a negative error code + * (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_multistream_decode_float( + OpusMSDecoder *st, + const unsigned char *data, + opus_int32 len, + float *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + +/** Perform a CTL function on a multistream Opus decoder. + * + * Generally the request and subsequent arguments are generated by a + * convenience macro. + * @param st OpusMSDecoder*: Multistream decoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls, + * @ref opus_decoderctls, or @ref opus_multistream_ctls. + * @see opus_genericctls + * @see opus_decoderctls + * @see opus_multistream_ctls + */ +OPUS_EXPORT int opus_multistream_decoder_ctl(OpusMSDecoder *st, int request, ...) OPUS_ARG_NONNULL(1); + +/** Frees an OpusMSDecoder allocated by + * opus_multistream_decoder_create(). + * @param st OpusMSDecoder: Multistream decoder state to be freed. + */ +OPUS_EXPORT void opus_multistream_decoder_destroy(OpusMSDecoder *st); + +/**@}*/ + +/**@}*/ + +#ifdef __cplusplus +} +#endif + +#endif /* OPUS_MULTISTREAM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_projection.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_projection.h new file mode 100644 index 000000000..9dabf4e85 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_projection.h @@ -0,0 +1,568 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + * @file opus_projection.h + * @brief Opus projection reference API + */ + +#ifndef OPUS_PROJECTION_H +#define OPUS_PROJECTION_H + +#include "opus_multistream.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** @cond OPUS_INTERNAL_DOC */ + +/** These are the actual encoder and decoder CTL ID numbers. + * They should not be used directly by applications.c + * In general, SETs should be even and GETs should be odd.*/ +/**@{*/ +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX_GAIN_REQUEST 6001 +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE_REQUEST 6003 +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX_REQUEST 6005 +/**@}*/ + + +/** @endcond */ + +/** @defgroup opus_projection_ctls Projection specific encoder and decoder CTLs + * + * These are convenience macros that are specific to the + * opus_projection_encoder_ctl() and opus_projection_decoder_ctl() + * interface. + * The CTLs from @ref opus_genericctls, @ref opus_encoderctls, + * @ref opus_decoderctls, and @ref opus_multistream_ctls may be applied to a + * projection encoder or decoder as well. + */ +/**@{*/ + +/** Gets the gain (in dB. S7.8-format) of the demixing matrix from the encoder. + * @param[out] x opus_int32 *: Returns the gain (in dB. S7.8-format) + * of the demixing matrix. + * @hideinitializer + */ +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX_GAIN(x) OPUS_PROJECTION_GET_DEMIXING_MATRIX_GAIN_REQUEST, __opus_check_int_ptr(x) + + +/** Gets the size in bytes of the demixing matrix from the encoder. + * @param[out] x opus_int32 *: Returns the size in bytes of the + * demixing matrix. + * @hideinitializer + */ +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE(x) OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE_REQUEST, __opus_check_int_ptr(x) + + +/** Copies the demixing matrix to the supplied pointer location. + * @param[out] x unsigned char *: Returns the demixing matrix to the + * supplied pointer location. + * @param y opus_int32: The size in bytes of the reserved memory at the + * pointer location. + * @hideinitializer + */ +#define OPUS_PROJECTION_GET_DEMIXING_MATRIX(x,y) OPUS_PROJECTION_GET_DEMIXING_MATRIX_REQUEST, x, __opus_check_int(y) + + +/**@}*/ + +/** Opus projection encoder state. + * This contains the complete state of a projection Opus encoder. + * It is position independent and can be freely copied. + * @see opus_projection_ambisonics_encoder_create + */ +typedef struct OpusProjectionEncoder OpusProjectionEncoder; + + +/** Opus projection decoder state. + * This contains the complete state of a projection Opus decoder. + * It is position independent and can be freely copied. + * @see opus_projection_decoder_create + * @see opus_projection_decoder_init + */ +typedef struct OpusProjectionDecoder OpusProjectionDecoder; + + +/**\name Projection encoder functions */ +/**@{*/ + +/** Gets the size of an OpusProjectionEncoder structure. + * @param channels int: The total number of input channels to encode. + * This must be no more than 255. + * @param mapping_family int: The mapping family to use for selecting + * the appropriate projection. + * @returns The size in bytes on success, or a negative error code + * (see @ref opus_errorcodes) on error. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_projection_ambisonics_encoder_get_size( + int channels, + int mapping_family +); + + +/** Allocates and initializes a projection encoder state. + * Call opus_projection_encoder_destroy() to release + * this object when finished. + * @param Fs opus_int32: Sampling rate of the input signal (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels in the input signal. + * This must be at most 255. + * It may be greater than the number of + * coded channels (streams + + * coupled_streams). + * @param mapping_family int: The mapping family to use for selecting + * the appropriate projection. + * @param[out] streams int *: The total number of streams that will + * be encoded from the input. + * @param[out] coupled_streams int *: Number of coupled (2 channel) + * streams that will be encoded from the input. + * @param application int: The target encoder application. + * This must be one of the following: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @param[out] error int *: Returns #OPUS_OK on success, or an error + * code (see @ref opus_errorcodes) on + * failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusProjectionEncoder *opus_projection_ambisonics_encoder_create( + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + int application, + int *error +) OPUS_ARG_NONNULL(4) OPUS_ARG_NONNULL(5); + + +/** Initialize a previously allocated projection encoder state. + * The memory pointed to by \a st must be at least the size returned by + * opus_projection_ambisonics_encoder_get_size(). + * This is intended for applications which use their own allocator instead of + * malloc. + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @see opus_projection_ambisonics_encoder_create + * @see opus_projection_ambisonics_encoder_get_size + * @param st OpusProjectionEncoder*: Projection encoder state to initialize. + * @param Fs opus_int32: Sampling rate of the input signal (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels in the input signal. + * This must be at most 255. + * It may be greater than the number of + * coded channels (streams + + * coupled_streams). + * @param streams int: The total number of streams to encode from the + * input. + * This must be no more than the number of channels. + * @param coupled_streams int: Number of coupled (2 channel) streams + * to encode. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * encoded channels (streams + + * coupled_streams) must be no + * more than the number of input channels. + * @param application int: The target encoder application. + * This must be one of the following: + *
+ *
#OPUS_APPLICATION_VOIP
+ *
Process signal for improved speech intelligibility.
+ *
#OPUS_APPLICATION_AUDIO
+ *
Favor faithfulness to the original input.
+ *
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
+ *
Configure the minimum possible coding delay by disabling certain modes + * of operation.
+ *
+ * @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes) + * on failure. + */ +OPUS_EXPORT int opus_projection_ambisonics_encoder_init( + OpusProjectionEncoder *st, + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + int application +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(5) OPUS_ARG_NONNULL(6); + + +/** Encodes a projection Opus frame. + * @param st OpusProjectionEncoder*: Projection encoder state. + * @param[in] pcm const opus_int16*: The input signal as interleaved + * samples. + * This must contain + * frame_size*channels + * samples. + * @param frame_size int: Number of samples per channel in the input + * signal. + * This must be an Opus frame size for the + * encoder's sampling rate. + * For example, at 48 kHz the permitted values + * are 120, 240, 480, 960, 1920, and 2880. + * Passing in a duration of less than 10 ms + * (480 samples at 48 kHz) will prevent the + * encoder from using the LPC or hybrid modes. + * @param[out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_projection_encode( + OpusProjectionEncoder *st, + const opus_int16 *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + + +/** Encodes a projection Opus frame from floating point input. + * @param st OpusProjectionEncoder*: Projection encoder state. + * @param[in] pcm const float*: The input signal as interleaved + * samples with a normal range of + * +/-1.0. + * Samples with a range beyond +/-1.0 + * are supported but will be clipped by + * decoders using the integer API and + * should only be used if it is known + * that the far end supports extended + * dynamic range. + * This must contain + * frame_size*channels + * samples. + * @param frame_size int: Number of samples per channel in the input + * signal. + * This must be an Opus frame size for the + * encoder's sampling rate. + * For example, at 48 kHz the permitted values + * are 120, 240, 480, 960, 1920, and 2880. + * Passing in a duration of less than 10 ms + * (480 samples at 48 kHz) will prevent the + * encoder from using the LPC or hybrid modes. + * @param[out] data unsigned char*: Output payload. + * This must contain storage for at + * least \a max_data_bytes. + * @param [in] max_data_bytes opus_int32: Size of the allocated + * memory for the output + * payload. This may be + * used to impose an upper limit on + * the instant bitrate, but should + * not be used as the only bitrate + * control. Use #OPUS_SET_BITRATE to + * control the bitrate. + * @returns The length of the encoded packet (in bytes) on success or a + * negative error code (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_projection_encode_float( + OpusProjectionEncoder *st, + const float *pcm, + int frame_size, + unsigned char *data, + opus_int32 max_data_bytes +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4); + + +/** Frees an OpusProjectionEncoder allocated by + * opus_projection_ambisonics_encoder_create(). + * @param st OpusProjectionEncoder*: Projection encoder state to be freed. + */ +OPUS_EXPORT void opus_projection_encoder_destroy(OpusProjectionEncoder *st); + + +/** Perform a CTL function on a projection Opus encoder. + * + * Generally the request and subsequent arguments are generated by a + * convenience macro. + * @param st OpusProjectionEncoder*: Projection encoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls, + * @ref opus_encoderctls, @ref opus_multistream_ctls, or + * @ref opus_projection_ctls + * @see opus_genericctls + * @see opus_encoderctls + * @see opus_multistream_ctls + * @see opus_projection_ctls + */ +OPUS_EXPORT int opus_projection_encoder_ctl(OpusProjectionEncoder *st, int request, ...) OPUS_ARG_NONNULL(1); + + +/**@}*/ + +/**\name Projection decoder functions */ +/**@{*/ + +/** Gets the size of an OpusProjectionDecoder structure. + * @param channels int: The total number of output channels. + * This must be no more than 255. + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @returns The size in bytes on success, or a negative error code + * (see @ref opus_errorcodes) on error. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_projection_decoder_get_size( + int channels, + int streams, + int coupled_streams +); + + +/** Allocates and initializes a projection decoder state. + * Call opus_projection_decoder_destroy() to release + * this object when finished. + * @param Fs opus_int32: Sampling rate to decode at (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels to output. + * This must be at most 255. + * It may be different from the number of coded + * channels (streams + + * coupled_streams). + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number of streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @param[in] demixing_matrix const unsigned char[demixing_matrix_size]: Demixing matrix + * that mapping from coded channels to output channels, + * as described in @ref opus_projection and + * @ref opus_projection_ctls. + * @param demixing_matrix_size opus_int32: The size in bytes of the + * demixing matrix, as + * described in @ref + * opus_projection_ctls. + * @param[out] error int *: Returns #OPUS_OK on success, or an error + * code (see @ref opus_errorcodes) on + * failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusProjectionDecoder *opus_projection_decoder_create( + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + unsigned char *demixing_matrix, + opus_int32 demixing_matrix_size, + int *error +) OPUS_ARG_NONNULL(5); + + +/** Intialize a previously allocated projection decoder state object. + * The memory pointed to by \a st must be at least the size returned by + * opus_projection_decoder_get_size(). + * This is intended for applications which use their own allocator instead of + * malloc. + * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL. + * @see opus_projection_decoder_create + * @see opus_projection_deocder_get_size + * @param st OpusProjectionDecoder*: Projection encoder state to initialize. + * @param Fs opus_int32: Sampling rate to decode at (in Hz). + * This must be one of 8000, 12000, 16000, + * 24000, or 48000. + * @param channels int: Number of channels to output. + * This must be at most 255. + * It may be different from the number of coded + * channels (streams + + * coupled_streams). + * @param streams int: The total number of streams coded in the + * input. + * This must be no more than 255. + * @param coupled_streams int: Number of streams to decode as coupled + * (2 channel) streams. + * This must be no larger than the total + * number of streams. + * Additionally, The total number of + * coded channels (streams + + * coupled_streams) must be no + * more than 255. + * @param[in] demixing_matrix const unsigned char[demixing_matrix_size]: Demixing matrix + * that mapping from coded channels to output channels, + * as described in @ref opus_projection and + * @ref opus_projection_ctls. + * @param demixing_matrix_size opus_int32: The size in bytes of the + * demixing matrix, as + * described in @ref + * opus_projection_ctls. + * @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes) + * on failure. + */ +OPUS_EXPORT int opus_projection_decoder_init( + OpusProjectionDecoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + unsigned char *demixing_matrix, + opus_int32 demixing_matrix_size +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(6); + + +/** Decode a projection Opus packet. + * @param st OpusProjectionDecoder*: Projection decoder state. + * @param[in] data const unsigned char*: Input payload. + * Use a NULL + * pointer to indicate packet + * loss. + * @param len opus_int32: Number of bytes in payload. + * @param[out] pcm opus_int16*: Output signal, with interleaved + * samples. + * This must contain room for + * frame_size*channels + * samples. + * @param frame_size int: The number of samples per channel of + * available space in \a pcm. + * If this is less than the maximum packet duration + * (120 ms; 5760 for 48kHz), this function will not be capable + * of decoding some packets. In the case of PLC (data==NULL) + * or FEC (decode_fec=1), then frame_size needs to be exactly + * the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the + * next incoming packet. For the PLC and FEC cases, frame_size + * must be a multiple of 2.5 ms. + * @param decode_fec int: Flag (0 or 1) to request that any in-band + * forward error correction data be decoded. + * If no such data is available, the frame is + * decoded as if it were lost. + * @returns Number of samples decoded on success or a negative error code + * (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_projection_decode( + OpusProjectionDecoder *st, + const unsigned char *data, + opus_int32 len, + opus_int16 *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + + +/** Decode a projection Opus packet with floating point output. + * @param st OpusProjectionDecoder*: Projection decoder state. + * @param[in] data const unsigned char*: Input payload. + * Use a NULL + * pointer to indicate packet + * loss. + * @param len opus_int32: Number of bytes in payload. + * @param[out] pcm opus_int16*: Output signal, with interleaved + * samples. + * This must contain room for + * frame_size*channels + * samples. + * @param frame_size int: The number of samples per channel of + * available space in \a pcm. + * If this is less than the maximum packet duration + * (120 ms; 5760 for 48kHz), this function will not be capable + * of decoding some packets. In the case of PLC (data==NULL) + * or FEC (decode_fec=1), then frame_size needs to be exactly + * the duration of audio that is missing, otherwise the + * decoder will not be in the optimal state to decode the + * next incoming packet. For the PLC and FEC cases, frame_size + * must be a multiple of 2.5 ms. + * @param decode_fec int: Flag (0 or 1) to request that any in-band + * forward error correction data be decoded. + * If no such data is available, the frame is + * decoded as if it were lost. + * @returns Number of samples decoded on success or a negative error code + * (see @ref opus_errorcodes) on failure. + */ +OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_projection_decode_float( + OpusProjectionDecoder *st, + const unsigned char *data, + opus_int32 len, + float *pcm, + int frame_size, + int decode_fec +) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4); + + +/** Perform a CTL function on a projection Opus decoder. + * + * Generally the request and subsequent arguments are generated by a + * convenience macro. + * @param st OpusProjectionDecoder*: Projection decoder state. + * @param request This and all remaining parameters should be replaced by one + * of the convenience macros in @ref opus_genericctls, + * @ref opus_decoderctls, @ref opus_multistream_ctls, or + * @ref opus_projection_ctls. + * @see opus_genericctls + * @see opus_decoderctls + * @see opus_multistream_ctls + * @see opus_projection_ctls + */ +OPUS_EXPORT int opus_projection_decoder_ctl(OpusProjectionDecoder *st, int request, ...) OPUS_ARG_NONNULL(1); + + +/** Frees an OpusProjectionDecoder allocated by + * opus_projection_decoder_create(). + * @param st OpusProjectionDecoder: Projection decoder state to be freed. + */ +OPUS_EXPORT void opus_projection_decoder_destroy(OpusProjectionDecoder *st); + + +/**@}*/ + +/**@}*/ + +#ifdef __cplusplus +} +#endif + +#endif /* OPUS_PROJECTION_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_types.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_types.h new file mode 100644 index 000000000..7cf675580 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/include/opus_types.h @@ -0,0 +1,166 @@ +/* (C) COPYRIGHT 1994-2002 Xiph.Org Foundation */ +/* Modified by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +/* opus_types.h based on ogg_types.h from libogg */ + +/** + @file opus_types.h + @brief Opus reference implementation types +*/ +#ifndef OPUS_TYPES_H +#define OPUS_TYPES_H + +#define opus_int int /* used for counters etc; at least 16 bits */ +#define opus_int64 long long +#define opus_int8 signed char + +#define opus_uint unsigned int /* used for counters etc; at least 16 bits */ +#define opus_uint64 unsigned long long +#define opus_uint8 unsigned char + +/* Use the real stdint.h if it's there (taken from Paul Hsieh's pstdint.h) */ +#if (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (defined(__GNUC__) && (defined(_STDINT_H) || defined(_STDINT_H_)) || defined (HAVE_STDINT_H)) +#include +# undef opus_int64 +# undef opus_int8 +# undef opus_uint64 +# undef opus_uint8 + typedef int8_t opus_int8; + typedef uint8_t opus_uint8; + typedef int16_t opus_int16; + typedef uint16_t opus_uint16; + typedef int32_t opus_int32; + typedef uint32_t opus_uint32; + typedef int64_t opus_int64; + typedef uint64_t opus_uint64; +#elif defined(_WIN32) + +# if defined(__CYGWIN__) +# include <_G_config.h> + typedef _G_int32_t opus_int32; + typedef _G_uint32_t opus_uint32; + typedef _G_int16 opus_int16; + typedef _G_uint16 opus_uint16; +# elif defined(__MINGW32__) + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef int opus_int32; + typedef unsigned int opus_uint32; +# elif defined(__MWERKS__) + typedef int opus_int32; + typedef unsigned int opus_uint32; + typedef short opus_int16; + typedef unsigned short opus_uint16; +# else + /* MSVC/Borland */ + typedef __int32 opus_int32; + typedef unsigned __int32 opus_uint32; + typedef __int16 opus_int16; + typedef unsigned __int16 opus_uint16; +# endif + +#elif defined(__MACOS__) + +# include + typedef SInt16 opus_int16; + typedef UInt16 opus_uint16; + typedef SInt32 opus_int32; + typedef UInt32 opus_uint32; + +#elif (defined(__APPLE__) && defined(__MACH__)) /* MacOS X Framework build */ + +# include + typedef int16_t opus_int16; + typedef u_int16_t opus_uint16; + typedef int32_t opus_int32; + typedef u_int32_t opus_uint32; + +#elif defined(__BEOS__) + + /* Be */ +# include + typedef int16 opus_int16; + typedef u_int16 opus_uint16; + typedef int32_t opus_int32; + typedef u_int32_t opus_uint32; + +#elif defined (__EMX__) + + /* OS/2 GCC */ + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef int opus_int32; + typedef unsigned int opus_uint32; + +#elif defined (DJGPP) + + /* DJGPP */ + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef int opus_int32; + typedef unsigned int opus_uint32; + +#elif defined(R5900) + + /* PS2 EE */ + typedef int opus_int32; + typedef unsigned opus_uint32; + typedef short opus_int16; + typedef unsigned short opus_uint16; + +#elif defined(__SYMBIAN32__) + + /* Symbian GCC */ + typedef signed short opus_int16; + typedef unsigned short opus_uint16; + typedef signed int opus_int32; + typedef unsigned int opus_uint32; + +#elif defined(CONFIG_TI_C54X) || defined (CONFIG_TI_C55X) + + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef long opus_int32; + typedef unsigned long opus_uint32; + +#elif defined(CONFIG_TI_C6X) + + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef int opus_int32; + typedef unsigned int opus_uint32; + +#else + + /* Give up, take a reasonable guess */ + typedef short opus_int16; + typedef unsigned short opus_uint16; + typedef int opus_int32; + typedef unsigned int opus_uint32; + +#endif + +#endif /* OPUS_TYPES_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/as-gcc-inline-assembly.m4 b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/as-gcc-inline-assembly.m4 new file mode 100644 index 000000000..b0d2da414 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/as-gcc-inline-assembly.m4 @@ -0,0 +1,98 @@ +dnl as-gcc-inline-assembly.m4 0.1.0 + +dnl autostars m4 macro for detection of gcc inline assembly + +dnl David Schleef + +dnl AS_COMPILER_FLAG(ACTION-IF-ACCEPTED, [ACTION-IF-NOT-ACCEPTED]) +dnl Tries to compile with the given CFLAGS. +dnl Runs ACTION-IF-ACCEPTED if the compiler can compile with the flags, +dnl and ACTION-IF-NOT-ACCEPTED otherwise. + +AC_DEFUN([AS_GCC_INLINE_ASSEMBLY], +[ + AC_MSG_CHECKING([if compiler supports gcc-style inline assembly]) + + AC_TRY_COMPILE([], [ +#ifdef __GNUC_MINOR__ +#if (__GNUC__ * 1000 + __GNUC_MINOR__) < 3004 +#error GCC before 3.4 has critical bugs compiling inline assembly +#endif +#endif +__asm__ (""::) ], [flag_ok=yes], [flag_ok=no]) + + if test "X$flag_ok" = Xyes ; then + $1 + true + else + $2 + true + fi + AC_MSG_RESULT([$flag_ok]) +]) + +AC_DEFUN([AS_ASM_ARM_NEON], +[ + AC_MSG_CHECKING([if assembler supports NEON instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__("vorr d0,d0,d0")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) + +AC_DEFUN([AS_ASM_ARM_NEON_FORCE], +[ + AC_MSG_CHECKING([if assembler supports NEON instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__(".arch armv7-a\n.fpu neon\n.object_arch armv4t\nvorr d0,d0,d0")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) + +AC_DEFUN([AS_ASM_ARM_MEDIA], +[ + AC_MSG_CHECKING([if assembler supports ARMv6 media instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__("shadd8 r3,r3,r3")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) + +AC_DEFUN([AS_ASM_ARM_MEDIA_FORCE], +[ + AC_MSG_CHECKING([if assembler supports ARMv6 media instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__(".arch armv6\n.object_arch armv4t\nshadd8 r3,r3,r3")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) + +AC_DEFUN([AS_ASM_ARM_EDSP], +[ + AC_MSG_CHECKING([if assembler supports EDSP instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__("qadd r3,r3,r3")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) + +AC_DEFUN([AS_ASM_ARM_EDSP_FORCE], +[ + AC_MSG_CHECKING([if assembler supports EDSP instructions on ARM]) + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[__asm__(".arch armv5te\n.object_arch armv4t\nqadd r3,r3,r3")])], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) +]) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/ax_add_fortify_source.m4 b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/ax_add_fortify_source.m4 new file mode 100644 index 000000000..1c89e4108 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/ax_add_fortify_source.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# Modified from https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_ADD_FORTIFY_SOURCE +# +# DESCRIPTION +# +# Check whether -D_FORTIFY_SOURCE=2 can be added to CFLAGS without macro +# redefinition warnings. Some distributions (such as Gentoo Linux) enable +# _FORTIFY_SOURCE globally in their compilers, leading to unnecessary +# warnings in the form of +# +# :0:0: error: "_FORTIFY_SOURCE" redefined [-Werror] +# : note: this is the location of the previous definition +# +# which is a problem if -Werror is enabled. This macro checks whether +# _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2 +# to CFLAGS. +# +# LICENSE +# +# Copyright (c) 2017 David Seifert +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 1 + +AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[ + AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CFLAGS]) + AC_LINK_IFELSE([ + AC_LANG_SOURCE( + [[ + int main() { + #ifndef _FORTIFY_SOURCE + return 0; + #else + this_is_an_error; + #endif + } + ]] + )], [ + AC_MSG_RESULT([yes]) + CFLAGS="$CFLAGS -D_FORTIFY_SOURCE=2" + ], [ + AC_MSG_RESULT([no]) + ]) +]) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/opus-intrinsics.m4 b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/opus-intrinsics.m4 new file mode 100644 index 000000000..a262ca181 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/m4/opus-intrinsics.m4 @@ -0,0 +1,29 @@ +dnl opus-intrinsics.m4 +dnl macro for testing for support for compiler intrinsics, either by default or with a compiler flag + +dnl OPUS_CHECK_INTRINSICS(NAME-OF-INTRINSICS, COMPILER-FLAG-FOR-INTRINSICS, VAR-IF-PRESENT, VAR-IF-DEFAULT, TEST-PROGRAM-HEADER, TEST-PROGRAM-BODY) +AC_DEFUN([OPUS_CHECK_INTRINSICS], +[ + AC_MSG_CHECKING([if compiler supports $1 intrinsics]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM($5, $6)], + [ + $3=1 + $4=1 + AC_MSG_RESULT([yes]) + ],[ + $4=0 + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([if compiler supports $1 intrinsics with $2]) + save_CFLAGS="$CFLAGS"; CFLAGS="$CFLAGS $2" + AC_LINK_IFELSE([AC_LANG_PROGRAM($5, $6)], + [ + AC_MSG_RESULT([yes]) + $3=1 + ],[ + AC_MSG_RESULT([no]) + $3=0 + ]) + CFLAGS="$save_CFLAGS" + ]) +]) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson.build new file mode 100644 index 000000000..41f69353f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson.build @@ -0,0 +1,675 @@ +project('opus', 'c', + version: run_command('meson/get-version.py', '--package-version', check: true).stdout().strip(), + meson_version: '>=0.54.0', + default_options: ['warning_level=2', + 'c_std=gnu99', + 'buildtype=debugoptimized']) + +libversion = run_command('meson/get-version.py', '--libtool-version', check: true).stdout().strip() +macosversion = run_command('meson/get-version.py', '--darwin-version', check: true).stdout().strip() + +cc = meson.get_compiler('c') +host_system = host_machine.system() +host_cpu_family = host_machine.cpu_family() +top_srcdir = meson.current_source_dir() +top_builddir = meson.current_build_dir() + +opus_includes = include_directories('.', 'include', 'celt', 'silk') +opus_public_includes = include_directories('include') + +add_project_arguments('-DOPUS_BUILD', language: 'c') +add_project_arguments('-DHAVE_CONFIG_H', language: 'c') + +if host_system == 'windows' + if cc.get_argument_syntax() == 'msvc' + add_project_arguments('-D_CRT_SECURE_NO_WARNINGS', language: 'c') + endif +endif + +if cc.get_argument_syntax() == 'gnu' + add_project_arguments('-D_FORTIFY_SOURCE=2', language: 'c') +endif + +# Check for extra compiler args +additional_c_args = [] +if cc.get_argument_syntax() != 'msvc' + additional_c_args += [ + '-fvisibility=hidden', + '-Wcast-align', + '-Wnested-externs', + '-Wshadow', + '-Wstrict-prototypes', + ] + + # On Windows, -fstack-protector-strong adds a libssp-0.dll dependency and + # prevents static linking + if host_system != 'windows' + additional_c_args += ['-fstack-protector-strong'] + endif +endif + +foreach arg : additional_c_args + if cc.has_argument(arg) + add_project_arguments(arg, language: 'c') + endif +endforeach + +# Windows MSVC warnings +if cc.get_id() == 'msvc' + # Ignore several spurious warnings. + # If a warning is completely useless and spammy, use '/wdXXXX' to suppress it + # If a warning is harmless but hard to fix, use '/woXXXX' so it's shown once + # NOTE: Only add warnings here if you are sure they're spurious + add_project_arguments('/wd4035', '/wd4715', '/wd4116', '/wd4046', '/wd4068', + '/wd4820', '/wd4244', '/wd4255', '/wd4668', + language : 'c') +endif + +opus_version = meson.project_version() + +opus_conf = configuration_data() +opus_conf.set('PACKAGE_BUGREPORT', '"opus@xiph.org"') +opus_conf.set('PACKAGE_NAME', '"opus"') +opus_conf.set('PACKAGE_STRING', '"opus @0@"'.format(opus_version)) +opus_conf.set('PACKAGE_TARNAME', '"opus"') +opus_conf.set('PACKAGE_URL', '""') +opus_conf.set('PACKAGE_VERSION', '"@0@"'.format(opus_version)) + +# FIXME: optional Ne10 dependency +have_arm_ne10 = false + +libm = cc.find_library('m', required : false) + +opus_conf.set('HAVE_LRINTF', cc.has_function('lrintf', prefix: '#include ', dependencies: libm)) +opus_conf.set('HAVE_LRINT', cc.has_function('lrint', prefix: '#include ', dependencies: libm)) +opus_conf.set('HAVE___MALLOC_HOOK', cc.has_function('__malloc_hook', prefix: '#include ')) +opus_conf.set('HAVE_STDINT_H', cc.check_header('stdint.h')) + +# Check for restrict keyword +restrict_tmpl = ''' +typedef int * int_ptr; +int foo (int_ptr @0@ ip, int * @0@ baz[]) { + return ip[0]; +} +int main (int argc, char ** argv) { + int s[1]; + int * @0@ t = s; + t[0] = 0; + return foo(t, (void *)0); +}''' +# Define restrict to the equivalent of the C99 restrict keyword, or to +# nothing if this is not supported. Do not define if restrict is +# supported directly. +if not cc.compiles(restrict_tmpl.format('restrict'), name : 'restrict keyword') + if cc.compiles(restrict_tmpl.format('__restrict'), name : '__restrict') + opus_conf.set('restrict', '__restrict') + elif cc.compiles(restrict_tmpl.format('__restrict__'), name : '__restrict__') + opus_conf.set('restrict', '__restrict') + elif cc.compiles(restrict_tmpl.format('_Restrict'), name : '_Restrict') + opus_conf.set('restrict', '_Restrict') + else + opus_conf.set('restrict', '/**/') + endif +endif + +# Check for C99 variable-size arrays, or alloca() as fallback +msg_use_alloca = false +if cc.compiles('''static int x; + char some_func (void) { + char a[++x]; + a[sizeof a - 1] = 0; + int N; + return a[0]; + }''', name : 'C99 variable-size arrays') + opus_conf.set('VAR_ARRAYS', 1) + msg_use_alloca = 'NO (using C99 variable-size arrays instead)' +elif cc.compiles('''#include + void some_func (void) { + int foo=10; + int * array = alloca(foo); + }''', name : 'alloca (alloca.h)') + opus_conf.set('USE_ALLOCA', true) + opus_conf.set('HAVE_ALLOCA_H', true) + msg_use_alloca = true +elif cc.compiles('''#include + #include + void some_func (void) { + int foo=10; + int * array = alloca(foo); + }''', name : 'alloca (std)') + opus_conf.set('USE_ALLOCA', true) + msg_use_alloca = true +endif + +opts = [ + [ 'fixed-point', 'FIXED_POINT' ], + [ 'fixed-point-debug', 'FIXED_DEBUG' ], + [ 'custom-modes', 'CUSTOM_MODES' ], + [ 'float-approx', 'FLOAT_APPROX' ], + [ 'assertions', 'ENABLE_ASSERTIONS' ], + [ 'hardening', 'ENABLE_HARDENING' ], + [ 'fuzzing', 'FUZZING' ], + [ 'check-asm', 'OPUS_CHECK_ASM' ], +] + +foreach opt : opts + # we assume these are all boolean options + opt_foo = get_option(opt[0]) + if opt_foo + opus_conf.set(opt[1], 1) + endif + set_variable('opt_' + opt[0].underscorify(), opt_foo) +endforeach + +opt_asm = get_option('asm') +opt_rtcd = get_option('rtcd') +opt_intrinsics = get_option('intrinsics') +extra_programs = get_option('extra-programs') +opt_tests = get_option('tests') + +disable_float_api = not get_option('float-api') +if disable_float_api + opus_conf.set('DISABLE_FLOAT_API', 1) +endif + +# This is for the description in the pkg-config .pc file +if opt_fixed_point + pc_build = 'fixed-point' +else + pc_build = 'floating-point' +endif +if opt_custom_modes + pc_build = pc_build + ', custom modes' +endif + +rtcd_support = [] +# With GCC, Clang, ICC, etc, we differentiate between 'may support this SIMD' +# and 'presume we have this SIMD' by checking whether the SIMD / intrinsics can +# be compiled by the compiler as-is (presume) or with SIMD cflags (may have). +# With MSVC, the compiler will always build SIMD/intrinsics targeting all +# specific instruction sets supported by that version of the compiler. No +# special arguments are ever needed. If runtime CPU detection is not disabled, +# we must always assume that we only 'may have' it. +opus_can_presume_simd = true +if cc.get_argument_syntax() == 'msvc' + if opt_rtcd.disabled() + warning('Building with an MSVC-like compiler and runtime CPU detection is disabled. Outputs may not run on all @0@ CPUs.'.format(host_cpu_family)) + else + opus_can_presume_simd = false + endif +endif + +opus_arm_external_asm = false + +asm_tmpl = ''' +int main (int argc, char ** argv) { + __asm__("@0@"); + return 0; +}''' + +asm_optimization = [] +inline_optimization = [] +if not opt_asm.disabled() + # Currently we only have inline asm for fixed-point + if host_cpu_family == 'arm' and opt_fixed_point + opus_conf.set('OPUS_ARM_ASM', true) + + # Check if compiler supports gcc-style inline assembly + if cc.compiles('''#ifdef __GNUC_MINOR__ + #if (__GNUC__ * 1000 + __GNUC_MINOR__) < 3004 + #error GCC before 3.4 has critical bugs compiling inline assembly + #endif + #endif + __asm__ (""::)''', + name : 'compiler supports gcc-style inline assembly') + + opus_conf.set('OPUS_ARM_INLINE_ASM', 1) + + # AS_ASM_ARM_EDSP + if cc.compiles(asm_tmpl.format('qadd r3,r3,r3'), + name : 'assembler supports EDSP instructions on ARM') + opus_conf.set('OPUS_ARM_INLINE_EDSP', 1) + inline_optimization += ['ESDP'] + endif + + # AS_ASM_ARM_MEDIA + if cc.compiles(asm_tmpl.format('shadd8 r3,r3,r3'), + name : 'assembler supports ARMv6 media instructions on ARM') + opus_conf.set('OPUS_ARM_INLINE_MEDIA', 1) + inline_optimization += ['Media'] + endif + + # AS_ASM_ARM_NEON + if cc.compiles(asm_tmpl.format('vorr d0,d0,d0'), + name : 'assembler supports NEON instructions on ARM') + opus_conf.set('OPUS_ARM_INLINE_NEON', 1) + inline_optimization += ['NEON'] + endif + endif + + # We need Perl to translate RVCT-syntax asm to gas syntax + perl = find_program('perl', required: get_option('asm')) + if perl.found() + opus_arm_external_asm = true + # opus_arm_presume_* mean we can and will use those instructions + # directly without doing runtime CPU detection. + # opus_arm_may_have_* mean we can emit those instructions, but we can + # only use them after runtime detection. + # The same rules apply for x86 assembly and intrinsics. + + opus_arm_may_have_edsp = opus_conf.has('OPUS_ARM_INLINE_EDSP') + opus_arm_presume_edsp = opus_arm_may_have_edsp and opus_can_presume_simd + + opus_arm_may_have_media = opus_conf.has('OPUS_ARM_INLINE_MEDIA') + opus_arm_presume_media = opus_arm_may_have_media and opus_can_presume_simd + + opus_arm_may_have_neon = opus_conf.has('OPUS_ARM_INLINE_NEON') + opus_arm_presume_neon = opus_arm_may_have_neon and opus_can_presume_simd + + if not opt_rtcd.disabled() + if not opus_arm_may_have_edsp + message('Trying to force-enable armv5e EDSP instructions...') + # AS_ASM_ARM_EDSP_FORCE + opus_arm_may_have_edsp = cc.compiles(asm_tmpl.format('.arch armv5te\n.object_arch armv4t\nqadd r3,r3,r3'), + name : 'Assembler supports EDSP instructions on ARM (forced)') + endif + if not opus_arm_may_have_media + message('Trying to force-enable ARMv6 media instructions...') + opus_arm_may_have_media = cc.compiles(asm_tmpl.format('.arch armv6\n.object_arch armv4t\nshadd8 r3,r3,r3'), + name : 'Assembler supports ARMv6 media instructions on ARM (forced)') + endif + if not opus_arm_may_have_neon + message('Trying to force-enable NEON instructions...') + opus_arm_may_have_neon = cc.compiles(asm_tmpl.format('.arch armv7-a\n.fpu neon\n.object_arch armv4t\nvorr d0,d0,d0'), + name : 'Assembler supports NEON instructions on ARM (forced)') + endif + endif + + if opus_arm_may_have_edsp + opus_conf.set('OPUS_ARM_MAY_HAVE_EDSP', 1) + if opus_arm_presume_edsp + opus_conf.set('OPUS_ARM_PRESUME_EDSP', 1) + asm_optimization += ['EDSP'] + else + rtcd_support += ['EDSP'] + endif + endif + if opus_arm_may_have_media + opus_conf.set('OPUS_ARM_MAY_HAVE_MEDIA', 1) + if opus_arm_presume_media + opus_conf.set('OPUS_ARM_PRESUME_MEDIA', 1) + asm_optimization += ['Media'] + else + rtcd_support += ['Media'] + endif + endif + if opus_arm_may_have_neon + opus_conf.set('OPUS_ARM_MAY_HAVE_NEON', 1) + if opus_arm_presume_neon + opus_conf.set('OPUS_ARM_PRESUME_NEON', 1) + asm_optimization += ['NEON'] + else + rtcd_support += ['NEON'] + endif + endif + + if cc.get_define('__APPLE__') + arm2gnu_args = ['--apple'] + else + arm2gnu_args = [] + endif + endif # found perl + else # arm + enable fixed point + if opt_asm.enabled() + error('asm option is enabled, but no assembly support for ' + host_cpu_family) + endif + endif +endif # enable asm + +# Check whether we require assembly and we support assembly on this arch, +# but none were detected. Can happen because of incorrect compiler flags, such +# as missing -mfloat-abi=softfp on ARM32 softfp architectures. +if opt_asm.enabled() and (asm_optimization.length() + inline_optimization.length()) == 0 + error('asm option was enabled, but no assembly support was detected') +endif + +# XXX: NEON has hardfp vs softfp compiler configuration issues +# When targeting ARM32 softfp, we sometimes need to explicitly pass +# -mfloat-abi=softfp to enable NEON. F.ex., on Android. It should +# be set in the cross file. +arm_neon_intr_link_args = ['-mfpu=neon'] + +have_sse = false +have_sse2 = false +have_sse4_1 = false +have_avx = false # no avx opus code yet +have_neon_intr = false + +intrinsics_support = [] +if not opt_intrinsics.disabled() + if host_cpu_family in ['arm', 'aarch64'] + # Check for ARMv7/AArch64 neon intrinsics + intrin_check = ''' + #include + int main (void) { + static float32x4_t A0, A1, SUMM; + SUMM = vmlaq_f32(SUMM, A0, A1); + return (int)vgetq_lane_f32(SUMM, 0); + }''' + intrin_name = 'ARMv7/AArch64 NEON' + if cc.links(intrin_check, + name: 'compiler supports @0@ intrinsics'.format(intrin_name)) + opus_arm_presume_neon_intr = opus_can_presume_simd + opus_arm_may_have_neon_intr = true + else + opus_arm_presume_neon_intr = false + if cc.links(intrin_check, + args: arm_neon_intr_link_args, + name: 'compiler supports @0@ intrinsics with @1@'.format(intrin_name, ' '.join(arm_neon_intr_link_args))) + opus_arm_may_have_neon_intr = true + else + opus_arm_may_have_neon_intr = false + endif + endif + + if opus_arm_may_have_neon_intr + have_neon_intr = true + intrinsics_support += [intrin_name] + opus_conf.set('OPUS_ARM_MAY_HAVE_NEON_INTR', 1) + if opus_arm_presume_neon_intr + opus_conf.set('OPUS_ARM_PRESUME_NEON_INTR', 1) + else + rtcd_support += [intrin_name] + opus_neon_intr_args = arm_neon_intr_link_args + endif + else + message('Compiler does not support @0@ intrinsics'.format(intrin_name)) + endif + + # Check for aarch64 neon intrinsics + intrin_check = ''' + #include + int main (void) { + static int32_t IN; + static int16_t OUT; + OUT = vqmovns_s32(IN); + }''' + intrin_name = 'AArch64 NEON' + if cc.links(intrin_check, + name: 'compiler supports @0@ intrinsics'.format(intrin_name)) + opus_arm_presume_aarch64_neon_intr = opus_can_presume_simd + opus_arm_may_have_aarch64_neon_intr = true + else + opus_arm_presume_aarch64_neon_intr = false + if cc.links(intrin_check, + args: arm_neon_intr_link_args, + name: 'compiler supports @0@ intrinsics with @1@'.format(intrin_name, ' '.join(arm_neon_intr_link_args))) + opus_arm_may_have_aarch64_neon_intr = true + else + opus_arm_may_have_aarch64_neon_intr = false + endif + endif + + if opus_arm_may_have_aarch64_neon_intr + intrinsics_support += [intrin_name] + opus_conf.set('OPUS_X86_MAY_HAVE_AARCH64_NEON_INTR', 1) + if opus_arm_presume_aarch64_neon_intr + opus_conf.set('OPUS_X86_PRESUME_AARCH64_NEON_INTR', 1) + endif + else + message('Compiler does not support @0@ intrinsics'.format(intrin_name)) + endif + elif host_cpu_family in ['x86', 'x86_64'] + # XXX: allow external override/specification of the flags + x86_intrinsics = [ + [ 'SSE', 'xmmintrin.h', '__m128', '_mm_setzero_ps()', ['-msse'] ], + [ 'SSE2', 'emmintrin.h', '__m128i', '_mm_setzero_si128()', ['-msse2'] ], + [ 'SSE4.1', 'smmintrin.h', '__m128i', '_mm_setzero_si128(); mtest = _mm_cmpeq_epi64(mtest, mtest)', ['-msse4.1'] ], + [ 'AVX', 'immintrin.h', '__m256', '_mm256_setzero_ps()', ['-mavx'] ], + ] + + foreach intrin : x86_intrinsics + intrin_check = '''#include <@0@> + int main (int argc, char ** argv) { + static @1@ mtest; + mtest = @2@; + return *((unsigned char *) &mtest) != 0; + }'''.format(intrin[1],intrin[2],intrin[3]) + intrin_name = intrin[0] + # Intrinsics arguments are not available with MSVC-like compilers + intrin_args = cc.get_argument_syntax() == 'msvc' ? [] : intrin[4] + if cc.links(intrin_check, name : 'compiler supports @0@ intrinsics'.format(intrin_name)) + may_have_intrin = true + presume_intrin = opus_can_presume_simd + elif intrin_args.length() > 0 + presume_intrin = false + if cc.links(intrin_check, + args : intrin_args, + name : 'compiler supports @0@ intrinsics with @1@'.format(intrin_name, ' '.join(intrin_args))) + may_have_intrin = true + else + may_have_intrin = false + endif + endif + if may_have_intrin + intrinsics_support += [intrin_name] + intrin_lower_name = intrin_name.to_lower().underscorify() + set_variable('have_' + intrin_lower_name, true) + opus_conf.set('OPUS_X86_MAY_HAVE_' + intrin_name.underscorify(), 1) + if presume_intrin + opus_conf.set('OPUS_X86_PRESUME_' + intrin_name.underscorify(), 1) + else + rtcd_support += [intrin_name] + set_variable('opus_@0@_args'.format(intrin_lower_name), intrin_args) + endif + else + message('Compiler does not support @0@ intrinsics'.format(intrin_name)) + endif + endforeach + + if not opt_rtcd.disabled() + get_cpuid_by_asm = false + cpuid_asm_code = ''' + #include + int main (int argc, char ** argv) { + unsigned int CPUInfo0; + unsigned int CPUInfo1; + unsigned int CPUInfo2; + unsigned int CPUInfo3; + unsigned int InfoType; + #if defined(__i386__) && defined(__PIC__) + __asm__ __volatile__ ( + "xchg %%ebx, %1\n" + "cpuid\n" + "xchg %%ebx, %1\n": + "=a" (CPUInfo0), + "=r" (CPUInfo1), + "=c" (CPUInfo2), + "=d" (CPUInfo3) : + "a" (InfoType), "c" (0) + ); + #else + __asm__ __volatile__ ( + "cpuid": + "=a" (CPUInfo0), + "=b" (CPUInfo1), + "=c" (CPUInfo2), + "=d" (CPUInfo3) : + "a" (InfoType), "c" (0) + ); + #endif + return 0; + }''' + cpuid_c_code = ''' + #include + int main (int argc, char ** argv) { + unsigned int CPUInfo0; + unsigned int CPUInfo1; + unsigned int CPUInfo2; + unsigned int CPUInfo3; + unsigned int InfoType; + __get_cpuid(InfoType, &CPUInfo0, &CPUInfo1, &CPUInfo2, &CPUInfo3); + return 0; + }''' + cpuid_msvc_code = ''' + #include + int main (void) { + int CPUInfo, InfoType; + __cpuid(&CPUInfo, InfoType); + }''' + if cc.links(cpuid_asm_code, name : 'Get X86 CPU info via inline assembly') + opus_conf.set('CPU_INFO_BY_ASM', 1) + elif cc.links(cpuid_c_code, name : 'Get X86 CPU info via C method') + opus_conf.set('CPU_INFO_BY_C', 1) + elif cc.get_define('_MSC_VER') != '' and cc.links(cpuid_msvc_code) + message('Getting X86 CPU info via __cpuid') + else + if opt_intrinsics.enabled() and opt_rtcd.enabled() + error('intrinsics and rtcd options are enabled, but no Get CPU Info method detected') + endif + warning('Get CPU Info method not detected, no rtcd for intrinsics') + endif + endif # opt_rtcd + else + if opt_intrinsics.enabled() + error('intrinsics option enabled, but no intrinsics support for ' + host_machine.get_cpu()) + endif + warning('No intrinsics support for ' + host_machine.get_cpu()) + endif +endif + +# Check whether we require intrinsics and we support intrinsics on this arch, +# but none were detected. Can happen because of incorrect compiler flags, such +# as missing -mfloat-abi=softfp on ARM32 softfp architectures. +if opt_intrinsics.enabled() and intrinsics_support.length() == 0 + error('intrinsics option was enabled, but none were detected') +endif + +if opt_rtcd.disabled() + rtcd_support = 'disabled' +else + if rtcd_support.length() > 0 + opus_conf.set('OPUS_HAVE_RTCD', 1) + else + if intrinsics_support.length() == 0 + rtcd_support = 'none' + if opt_rtcd.enabled() + error('rtcd option is enabled, but no support for intrinsics or assembly is available') + endif + else + rtcd_support = 'not needed' + endif + endif +endif + +# extract source file lists from .mk files +mk_files = ['silk_sources.mk', 'opus_headers.mk', 'opus_sources.mk', 'silk_headers.mk', 'celt_sources.mk', 'celt_headers.mk'] +lines = run_command('meson/read-sources-list.py', mk_files, check: true).stdout().strip().split('\n') +sources = {} +foreach l : lines + a = l.split(' = ') + var_name = a[0] + file_list = a[1].split() + sources += {var_name: files(file_list)} +endforeach + +subdir('include') +subdir('silk') +subdir('celt') +subdir('src') + +configure_file(output: 'config.h', configuration: opus_conf) + +if not opt_tests.disabled() + subdir('celt/tests') + subdir('silk/tests') + subdir('tests') +endif + +# pkg-config files (not using pkg module so we can use the existing .pc.in file) +pkgconf = configuration_data() + +pkgconf.set('prefix', join_paths(get_option('prefix'))) +pkgconf.set('exec_prefix', '${prefix}') +pkgconf.set('libdir', '${prefix}/@0@'.format(get_option('libdir'))) +pkgconf.set('includedir', '${prefix}/@0@'.format(get_option('includedir'))) +pkgconf.set('VERSION', opus_version) +pkgconf.set('PC_BUILD', pc_build) +pkgconf.set('LIBM', libm.found() ? '-lm' : '') + +pkg_install_dir = '@0@/pkgconfig'.format(get_option('libdir')) + +configure_file(input : 'opus.pc.in', + output : 'opus.pc', + configuration : pkgconf, + install_dir : pkg_install_dir) + +# The uninstalled one has hardcoded libtool + static lib stuff, skip it for now +#configure_file(input : 'opus-uninstalled.pc.in', +# output : 'opus-uninstalled.pc', +# configuration : pkgconf, +# install : false) + +doxygen = find_program('doxygen', required: get_option('docs')) +if doxygen.found() + subdir('doc') +endif + +summary( + { + 'C99 var arrays': opus_conf.has('VAR_ARRAYS'), + 'C99 lrintf': opus_conf.has('HAVE_LRINTF'), + 'Use alloca': msg_use_alloca, + }, + section: 'Compiler support', + bool_yn: true, + list_sep: ', ', +) + +# Parse optimization status +foreach status : [['inline_optimization', opt_asm], + ['asm_optimization', opt_asm], + ['intrinsics_support', opt_intrinsics]] + res = status[0] + opt = status[1] + resval = get_variable(res) + if opt.disabled() + set_variable(res, 'disabled') + elif resval.length() == 0 + if host_cpu_family not in ['arm', 'aarch64', 'x86', 'x86_64'] + set_variable(res, 'No optimizations for your platform, please send patches') + else + set_variable(res, 'none') + endif + endif +endforeach + +summary( + { + 'Floating point support': not opt_fixed_point, + 'Fast float approximations': opt_float_approx, + 'Fixed point debugging': opt_fixed_point_debug, + 'Inline assembly optimizations': inline_optimization, + 'External assembly optimizations': asm_optimization, + 'Intrinsics optimizations': intrinsics_support, + 'Run-time CPU detection': rtcd_support, + }, + section: 'Optimizations', + bool_yn: true, + list_sep: ', ', +) +summary( + { + 'Custom modes': opt_custom_modes, + 'Assertions': opt_assertions, + 'Hardening': opt_hardening, + 'Fuzzing': opt_fuzzing, + 'Check ASM': opt_check_asm, + 'API documentation': doxygen.found(), + 'Extra programs': not extra_programs.disabled(), + 'Tests': not opt_tests.disabled(), + }, + section: 'General configuration', + bool_yn: true, + list_sep: ', ', +) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/get-version.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/get-version.py new file mode 100755 index 000000000..0e8b8623a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/get-version.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Opus get-version.py +# +# Extracts versions for build: +# - Opus package version based on 'git describe' or $srcroot/package_version +# - libtool version based on configure.ac +# - macos lib version based on configure.ac +# +# Usage: +# get-version.py [--package-version | --libtool-version | --darwin-version] +import argparse +import subprocess +import os +import sys +import shutil + +if __name__ == '__main__': + arg_parser = argparse.ArgumentParser(description='Extract Opus package version or libtool version') + group = arg_parser.add_mutually_exclusive_group(required=True) + group.add_argument('--libtool-version', action='store_true') + group.add_argument('--package-version', action='store_true') + group.add_argument('--darwin-version', action='store_true') + args = arg_parser.parse_args() + + srcroot = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) + + # package version + if args.package_version: + package_version = None + + # check if git checkout + git_dir = os.path.join(srcroot, '.git') + is_git = os.path.isdir(git_dir) + have_git = shutil.which('git') is not None + + if is_git and have_git: + git_cmd = subprocess.run(['git', '--git-dir=' + git_dir, 'describe', 'HEAD'], stdout=subprocess.PIPE) + if git_cmd.returncode: + print('ERROR: Could not extract package version via `git describe` in', srcroot, file=sys.stderr) + sys.exit(-1) + package_version = git_cmd.stdout.decode('ascii').strip().lstrip('v') + else: + with open(os.path.join(srcroot, 'package_version'), 'r') as f: + for line in f: + if line.startswith('PACKAGE_VERSION="'): + package_version = line[17:].strip().lstrip('v').rstrip('"') + if package_version: + break + + if not package_version: + print('ERROR: Could not extract package version from package_version file in', srcroot, file=sys.stderr) + sys.exit(-1) + + print(package_version) + sys.exit(0) + + # libtool version + darwin version + elif args.libtool_version or args.darwin_version: + opus_lt_cur = None + opus_lt_rev = None + opus_lt_age = None + + with open(os.path.join(srcroot, 'configure.ac'), 'r') as f: + for line in f: + if line.strip().startswith('OPUS_LT_CURRENT='): + opus_lt_cur = line[16:].strip() + elif line.strip().startswith('OPUS_LT_REVISION='): + opus_lt_rev = line[17:].strip() + elif line.strip().startswith('OPUS_LT_AGE='): + opus_lt_age = line[12:].strip() + + if opus_lt_cur and opus_lt_rev and opus_lt_age: + opus_lt_cur = int(opus_lt_cur) + opus_lt_rev = int(opus_lt_rev) + opus_lt_age = int(opus_lt_age) + if args.libtool_version: + print('{}.{}.{}'.format(opus_lt_cur - opus_lt_age, opus_lt_age, opus_lt_rev)) + elif args.darwin_version: + print('{}.{}.{}'.format(opus_lt_cur + 1, 0, 0)) + sys.exit(0) + else: + print('ERROR: Could not extract libtool version from configure.ac file in', srcroot, file=sys.stderr) + sys.exit(-1) + else: + sys.exit(-1) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/read-sources-list.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/read-sources-list.py new file mode 100755 index 000000000..fcbec5019 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson/read-sources-list.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# opus/read-sources-list.py +# +# Parses .mk files and extracts list of source files. +# Prints one line per source file list, with filenames space-separated. + +import sys + +if len(sys.argv) < 2: + sys.exit('Usage: {} sources_foo.mk [sources_bar.mk...]'.format(sys.argv[0])) + +for input_fn in sys.argv[1:]: + with open(input_fn, 'r', encoding='utf8') as f: + text = f.read() + text = text.replace('\\\n', '') + + # Remove empty lines + lines = [line for line in text.split('\n') if line.strip()] + + # Print SOURCES_XYZ = file1.c file2.c + for line in lines: + values = line.strip().split('=', maxsplit=2) + if len(values) != 2: + raise RuntimeError('Unable to parse line "{}" from file "{}"'.format(line, input_fn)) + var, files = values + sources_list = [f for f in files.split(' ') if f] + print(var.strip(), '=', ' '.join(sources_list)) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson_options.txt b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson_options.txt new file mode 100644 index 000000000..360ff2635 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/meson_options.txt @@ -0,0 +1,22 @@ +# Optimizations +option('fixed-point', type : 'boolean', value : false, description : 'Compile without floating point (for machines without a fast enough FPU') +option('fixed-point-debug', type : 'boolean', value : false, description : 'Debug fixed-point implementation') +option('float-api', type : 'boolean', value : true, description : 'Compile with or without the floating point API (for machines with no float library') +option('float-approx', type : 'boolean', value : false, description : 'Enable fast approximations for floating point (not supported on all platforms)') +option('rtcd', type : 'feature', value : 'auto', description : 'Run-time CPU capabilities detection') +option('asm', type : 'feature', value : 'auto', description : 'Assembly optimizations for ARM (fixed-point)') +option('intrinsics', type : 'feature', value : 'auto', description : 'Intrinsics optimizations for ARM NEON or x86') + +option('custom-modes', type : 'boolean', value : false, description : 'Enable non-Opus modes, e.g. 44.1 kHz & 2^n frames') +option('extra-programs', type : 'feature', value : 'auto', description : 'Extra programs (demo and tests)') +option('assertions', type : 'boolean', value : false, description : 'Additional software error checking') +option('hardening', type : 'boolean', value : true, description : 'Run-time checks that are cheap and safe for use in production') +option('fuzzing', type : 'boolean', value : false, description : 'Causes the encoder to make random decisions') +option('check-asm', type : 'boolean', value : false, description : 'Run bit-exactness checks between optimized and c implementations') + +# common feature options +option('tests', type : 'feature', value : 'auto', description : 'Build tests') +option('docs', type: 'feature', value: 'auto', description: 'Build API documentation') + +# other options +option('docdir', type: 'string', value: 'doc/opus', description: 'Directory to install documentation into (default: DATADIR/doc/opus') diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus-uninstalled.pc.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus-uninstalled.pc.in new file mode 100644 index 000000000..19f5c933d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus-uninstalled.pc.in @@ -0,0 +1,12 @@ +# Opus codec reference implementation uninstalled pkg-config file + +libdir=${pcfiledir}/.libs +includedir=${pcfiledir} + +Name: opus uninstalled +Description: Opus IETF audio codec (not installed, @PC_BUILD@) +Version: @VERSION@ +Requires: +Conflicts: +Libs: ${libdir}/libopus.la @LIBM@ +Cflags: -I${pcfiledir}/@top_srcdir@/include diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.m4 b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.m4 new file mode 100644 index 000000000..47f5ec496 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.m4 @@ -0,0 +1,117 @@ +# Configure paths for libopus +# Gregory Maxwell 08-30-2012 +# Shamelessly stolen from Jack Moffitt (libogg) who +# Shamelessly stole from Owen Taylor and Manish Singh + +dnl XIPH_PATH_OPUS([ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND]]) +dnl Test for libopus, and define OPUS_CFLAGS and OPUS_LIBS +dnl +AC_DEFUN([XIPH_PATH_OPUS], +[dnl +dnl Get the cflags and libraries +dnl +AC_ARG_WITH(opus,AC_HELP_STRING([--with-opus=PFX],[Prefix where opus is installed (optional)]), opus_prefix="$withval", opus_prefix="") +AC_ARG_WITH(opus-libraries,AC_HELP_STRING([--with-opus-libraries=DIR],[Directory where the opus library is installed (optional)]), opus_libraries="$withval", opus_libraries="") +AC_ARG_WITH(opus-includes,AC_HELP_STRING([--with-opus-includes=DIR],[Directory where the opus header files are installed (optional)]), opus_includes="$withval", opus_includes="") +AC_ARG_ENABLE(opustest,AC_HELP_STRING([--disable-opustest],[Do not try to compile and run a test opus program]),, enable_opustest=yes) + + if test "x$opus_libraries" != "x" ; then + OPUS_LIBS="-L$opus_libraries" + elif test "x$opus_prefix" = "xno" || test "x$opus_prefix" = "xyes" ; then + OPUS_LIBS="" + elif test "x$opus_prefix" != "x" ; then + OPUS_LIBS="-L$opus_prefix/lib" + elif test "x$prefix" != "xNONE" ; then + OPUS_LIBS="-L$prefix/lib" + fi + + if test "x$opus_prefix" != "xno" ; then + OPUS_LIBS="$OPUS_LIBS -lopus" + fi + + if test "x$opus_includes" != "x" ; then + OPUS_CFLAGS="-I$opus_includes" + elif test "x$opus_prefix" = "xno" || test "x$opus_prefix" = "xyes" ; then + OPUS_CFLAGS="" + elif test "x$opus_prefix" != "x" ; then + OPUS_CFLAGS="-I$opus_prefix/include" + elif test "x$prefix" != "xNONE"; then + OPUS_CFLAGS="-I$prefix/include" + fi + + AC_MSG_CHECKING(for Opus) + if test "x$opus_prefix" = "xno" ; then + no_opus="disabled" + enable_opustest="no" + else + no_opus="" + fi + + + if test "x$enable_opustest" = "xyes" ; then + ac_save_CFLAGS="$CFLAGS" + ac_save_LIBS="$LIBS" + CFLAGS="$CFLAGS $OPUS_CFLAGS" + LIBS="$LIBS $OPUS_LIBS" +dnl +dnl Now check if the installed Opus is sufficiently new. +dnl + rm -f conf.opustest + AC_TRY_RUN([ +#include +#include +#include +#include + +int main () +{ + system("touch conf.opustest"); + return 0; +} + +],, no_opus=yes,[echo $ac_n "cross compiling; assumed OK... $ac_c"]) + CFLAGS="$ac_save_CFLAGS" + LIBS="$ac_save_LIBS" + fi + + if test "x$no_opus" = "xdisabled" ; then + AC_MSG_RESULT(no) + ifelse([$2], , :, [$2]) + elif test "x$no_opus" = "x" ; then + AC_MSG_RESULT(yes) + ifelse([$1], , :, [$1]) + else + AC_MSG_RESULT(no) + if test -f conf.opustest ; then + : + else + echo "*** Could not run Opus test program, checking why..." + CFLAGS="$CFLAGS $OPUS_CFLAGS" + LIBS="$LIBS $OPUS_LIBS" + AC_TRY_LINK([ +#include +#include +], [ return 0; ], + [ echo "*** The test program compiled, but did not run. This usually means" + echo "*** that the run-time linker is not finding Opus or finding the wrong" + echo "*** version of Opus. If it is not finding Opus, you'll need to set your" + echo "*** LD_LIBRARY_PATH environment variable, or edit /etc/ld.so.conf to point" + echo "*** to the installed location Also, make sure you have run ldconfig if that" + echo "*** is required on your system" + echo "***" + echo "*** If you have an old version installed, it is best to remove it, although" + echo "*** you may also be able to get things to work by modifying LD_LIBRARY_PATH"], + [ echo "*** The test program failed to compile or link. See the file config.log for the" + echo "*** exact error that occurred. This usually means Opus was incorrectly installed" + echo "*** or that you have moved Opus since it was installed." ]) + CFLAGS="$ac_save_CFLAGS" + LIBS="$ac_save_LIBS" + fi + OPUS_CFLAGS="" + OPUS_LIBS="" + ifelse([$2], , :, [$2]) + fi + AC_SUBST(OPUS_CFLAGS) + AC_SUBST(OPUS_LIBS) + rm -f conf.opustest +]) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.pc.in b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.pc.in new file mode 100644 index 000000000..6946e7de5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus.pc.in @@ -0,0 +1,16 @@ +# Opus codec reference implementation pkg-config file + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Opus +Description: Opus IETF audio codec (@PC_BUILD@ build) +URL: https://opus-codec.org/ +Version: @VERSION@ +Requires: +Conflicts: +Libs: -L${libdir} -lopus +Libs.private: @LIBM@ +Cflags: -I${includedir}/opus diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_headers.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_headers.mk new file mode 100644 index 000000000..27596f2a4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_headers.mk @@ -0,0 +1,9 @@ +OPUS_HEAD = \ +include/opus.h \ +include/opus_multistream.h \ +include/opus_projection.h \ +src/opus_private.h \ +src/analysis.h \ +src/mapping_matrix.h \ +src/mlp.h \ +src/tansig_table.h diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_sources.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_sources.mk new file mode 100644 index 000000000..44153b570 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/opus_sources.mk @@ -0,0 +1,16 @@ +OPUS_SOURCES = \ +src/opus.c \ +src/opus_decoder.c \ +src/opus_encoder.c \ +src/opus_multistream.c \ +src/opus_multistream_encoder.c \ +src/opus_multistream_decoder.c \ +src/repacketizer.c \ +src/opus_projection_encoder.c \ +src/opus_projection_decoder.c \ +src/mapping_matrix.c + +OPUS_SOURCES_FLOAT = \ +src/analysis.c \ +src/mlp.c \ +src/mlp_data.c diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/releases.sha2 b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/releases.sha2 new file mode 100644 index 000000000..334976b14 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/releases.sha2 @@ -0,0 +1,79 @@ +b2f75c4ac5ab837845eb028413fae2a28754bfb0a6d76416e2af1441ef447649 opus-0.9.0.tar.gz +4e379a98ba95bbbfe9087ef10fdd05c8ac9060b6d695f587ea82a7b43a0df4fe opus-0.9.10.tar.gz +b1cad6846a8f819a141009fe3f8f10c946e8eff7e9c2339cd517bb136cc59eae opus-0.9.14.tar.gz +206221afc47b87496588013bd4523e1e9f556336c0813f4372773fc536dd4293 opus-0.9.1.tar.gz +6e85c1b57e1d7b7dfe2928bf92586b96b73a9067e054ede45bd8e6d24bd30582 opus-0.9.2.tar.gz +d916e34c18a396eb7dffc47af754f441af52a290b761e20db9aedb65928c699e opus-0.9.3.tar.gz +53801066fa97329768e7b871fd1495740269ec46802e1c9051aa7e78c6edee5b opus-0.9.5.tar.gz +3bfaeb25f4b4a625a0bc994d6fc6f6776a05193f60099e0a99f7530c6b256309 opus-0.9.6.tar.gz +1b69772c31c5cbaa43d1dfa5b1c495fc29712e8e0ff69d6f8ad46459e5c6715f opus-0.9.7.tar.gz +4aa30d2e0652ffb4a7a22cc8a29c4ce78267626f560a2d9213b1d2d4e618cf36 opus-0.9.8.tar.gz +2f62359f09151fa3b242040dc9b4c5b6bda15557c5daea59c8420f1a2ff328b7 opus-0.9.9.tar.gz +43bcea51afa531f32a6a5fdd9cba4bd496993e26a141217db3cccce6caa7cd74 opus-1.0.0-rc.tar.gz +9250fcc74472d45c1e14745542ec9c8d09982538aefed56962495614be3e0d2d opus-1.0.0.tar.gz +76bc0a31502a51dae9ab737b4db043b9ecfcd0b5861f0bfda41b662bd5b92227 opus-1.0.1-rc2.tar.gz +3de8d6809dac38971ebb305532d4ea532519d3bed08985f25d6c557f9ce5e8ff opus-1.0.1-rc3.tar.gz +8044397a6365a07117b08cbe8f9818bf7c93746908806ba74a2917187bbdda5f opus-1.0.1-rc.tar.gz +80fa5c3caf2ac0fd68f8a22cce1564fc46b368c773a17554887d0066fe1841ef opus-1.0.1.tar.gz +da615edbee5d019c1833071d69a4782c19f178cf9ca1401375036ecef25cd78a opus-1.0.2.tar.gz +191a089c92dbc403de6980463dd3604b65beb12d283c607e246c8076363cb49c opus-1.0.3.tar.gz +a8d40efe87f6c3e76725391457d46277878c7a816ae1642843261463133fa5c8 opus-1.1-alpha.tar.gz +ec1784287f385aef994b64734aaecae04860e61aa50fc6eef6643fa7e40dd193 opus-1.1-beta.tar.gz +8aa16360f59a94d3e38f38f28d24039f7663179682cbae82aa42f1dd9e52e6ed opus-1.1-rc.tar.gz +ebc87a086d4fe677c5e42d56888b1fd25af858e4179eae4f8656270410dffac3 opus-1.1-rc2.tar.gz +cbfd09c58cc10a4d3fcb727ad5d46d7bb549f8185ac922ee28b4581b52a7bee9 opus-1.1-rc3.tar.gz +b9727015a58affcf3db527322bf8c4d2fcf39f5f6b8f15dbceca20206cbe1d95 opus-1.1.tar.gz +0c668639dcd16b14709fc9dc49e6686606f5a256f2eaa1ebaa2f39a66f8626cd opus-1.1.1-beta.tar.gz +66f2a5877c8803dc9a5a44b4f3d0bdc8f06bd066324222d144eb255612b68152 opus-1.1.1-rc.tar.gz +9b84ff56bd7720d5554103c557664efac2b8b18acc4bbcc234cb881ab9a3371e opus-1.1.1.tar.gz +0e290078e31211baa7b5886bcc8ab6bc048b9fc83882532da4a1a45e58e907fd opus-1.1.2.tar.gz +58b6fe802e7e30182e95d0cde890c0ace40b6f125cffc50635f0ad2eef69b633 opus-1.1.3.tar.gz +9122b6b380081dd2665189f97bfd777f04f92dc3ab6698eea1dbb27ad59d8692 opus-1.1.4.tar.gz +eb84981ca0f40a3e5d5e58d2e8582cb2fee05a022825a6dfe14d14b04eb563e4 opus-1.1.5.tar.gz +654a9bebb73266271a28edcfff431e4cfd9bfcde71f42849a0cdd73bece803a7 opus-1.2-alpha.tar.gz +c0e90507259cf21ce7b2c82fb9ac55367d8543dae91cc3d4d2c59afd37f44023 opus-1.2-alpha2.tar.gz +291e979a8a2fb679ed35a5dff5d761a9d9a5e22586fd07934ed94461e2636c7a opus-1.2-beta.tar.gz +85343fdaed96529d94c1e1f3a210fa51240d04ca62fa01e97ef02f88020c2ce9 opus-1.2-rc1.tar.gz +77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz +cfafd339ccd9c5ef8d6ab15d7e1a412c054bf4cb4ecbbbcc78c12ef2def70732 opus-1.2.1.tar.gz +7f56e058c9549d03ae35511ad9e16ef6d1eb257836830d54abff0f495f17e187 opus-1.3-beta.tar.gz +96fa28598e8ccd558b297277ad59a045c551ba0e06d65a9675938e084f837669 opus-1.3-rc.tar.gz +f6bab321fb81db984766f1e4d340a9e71a5ca2c5d4d53f4ee072e84afda271ca opus-1.3-rc2.tar.gz +4f3d69aefdf2dbaf9825408e452a8a414ffc60494c70633560700398820dc550 opus-1.3.tar.gz +65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d opus-1.3.1.tar.gz +94ac78ca4f74c4e43bc9fe4ec1ad0aa36f38ab90f45b0727c40dd1e96096e767 opus_testvectors-draft11.tar.gz +94ac78ca4f74c4e43bc9fe4ec1ad0aa36f38ab90f45b0727c40dd1e96096e767 opus_testvectors.tar.gz +6b26a22f9ba87b2b836906a9bb7afec5f8e54d49553b1200382520ee6fedfa55 opus_testvectors-rfc8251.tar.gz +5d2b99757bcb628bab2611f3ed27af6f35276ce3abc96c0ed4399d6c6463dda5 opus-tools-0.1.2.tar.gz +008317297d6ce84f84992abf8cc948a048a4fa135e1d1caf429fafde8965a792 opus-tools-0.1.3.tar.gz +de80485c5afa1fd83c0e16a0dd4860470c872997a7dd0a58e99b2ee8a93e5168 opus-tools-0.1.4.tar.gz +76678d0eb7a9b3d793bd0243f9ced9ab0ecdab263f5232ed940c8f5795fb0405 opus-tools-0.1.5.tar.gz +cc86dbc2a4d76da7e1ed9afee85448c8f798c465a5412233f178783220f3a2c1 opus-tools-0.1.6.tar.gz +e0f08d301555dffc417604269b5a85d2bd197f259c7d6c957f370ffd33d6d9cd opus-tools-0.1.7.tar.gz +e4e188579ea1c4e4d5066460d4a7214a7eafe3539e9a4466fdc98af41ba4a2f6 opus-tools-0.1.8.tar.gz +b1873dd78c7fbc98cf65d6e10cfddb5c2c03b3af93f922139a2104baedb4643a opus-tools-0.1.9.tar.gz +a2357532d19471b70666e0e0ec17d514246d8b3cb2eb168f68bb0f6fd372b28c opus-tools-0.1.10.tar.gz +b4e56cb00d3e509acfba9a9b627ffd8273b876b4e2408642259f6da28fa0ff86 opus-tools-0.2.tar.gz +bd6d14e8897a2f80065ef34a516c70e74f8e00060abdbc238e79e5f99bca3e96 libopusenc-0.1.tar.gz +02e6e0b14cbbe0569d948a46420f9c9a81d93bba32dc576a4007cbf96da68ef3 libopusenc-0.1.1.tar.gz +c79e95eeee43a0b965e9b2c59a243763a8f8b0a7e71441df2aa9084f6171c73a libopusenc-0.2.tar.gz +8298db61a8d3d63e41c1a80705baa8ce9ff3f50452ea7ec1c19a564fe106cbb9 libopusenc-0.2.1.tar.gz +8071b968475c1a17f54b6840d6de9d9ee20f930e827b0401abe3c4cf4f3bf30a opusfile-0.1.tar.gz +b4a678b3b6c4adfb6aff1f67ef658becfe146ea7c7ff228e99543762171557f9 opusfile-0.2.tar.gz +4248927f2c4e316ea5b84fb02bd100bfec8fa4624a6910d77f0af7f0c6cb8baa opusfile-0.3.tar.gz +9836ea11706c44f36de92c4c9b1248e03a4c521e7fb2cff18a0cb4f8b0e79140 opusfile-0.4.tar.gz +f187906b1b35f7f0d7de6a759b4aab512a9279d23adb35d8009e7e33bd6a922a opusfile-0.4.zip +2ce52d006aeeec9f10260dbe3073c4636954a1ab19c82b8baafefe0180aa4a39 opusfile-0.5.tar.gz +b940d62beb15b5974764574b9f265481fe5b6ee16902fb705727546caf956261 opusfile-0.5.zip +2428717b356e139f18ed2fdb5ad990b5654a238907a0058200b39c46a7d03ea6 opusfile-0.6.tar.gz +753339225193df605372944889023b9b3c5378d672e8784d69fa241cd465278c opusfile-0.6.zip +9e2bed13bc729058591a0f1cab2505e8cfd8e7ac460bf10a78bcc3b125e7c301 opusfile-0.7.tar.gz +346967d7989bb83b05949483b76bd0f69a12c59bd8b4457e864902b52bb0ac34 opusfile-0.7.zip +2c231ed3cfaa1b3173f52d740e5bbd77d51b9dfecb87014b404917fba4b855a4 opusfile-0.8.tar.gz +89dff4342c3b789574cbea5c57f11b96d4ebe4d28ab90248c1783ea569b1e9e3 opusfile-0.8.zip +f75fb500e40b122775ac1a71ad80c4477698842a8fe9da4a1b4a1a9f16e4e979 opusfile-0.9.tar.gz +e9591da4d4c9e857436c2d46a28a9e470fa5355ea5a76d4d582f137d18755d36 opusfile-0.9.zip +48e03526ba87ef9cf5f1c47b5ebe3aa195bd89b912a57060c36184a6cd19412f opusfile-0.10.tar.gz +9d9e95d01817ecf48bf6daaea8f071f9b45bd1751ca1fc8ce50e5075eb2bc3c8 opusfile-0.10.zip +74ce9b6cf4da103133e7b5c95df810ceb7195471e1162ed57af415fabf5603bf opusfile-0.11.tar.gz +23c5168026c4f1fc34843650135b409d0fc8cf452508163b4ece8077256ac6ff opusfile-0.11.zip diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/dump_rnn.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/dump_rnn.py new file mode 100755 index 000000000..dd66403b5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/dump_rnn.py @@ -0,0 +1,57 @@ +#!/usr/bin/python + +from __future__ import print_function + +from keras.models import Sequential +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.models import load_model +from keras import backend as K + +import numpy as np + +def printVector(f, vector, name): + v = np.reshape(vector, (-1)); + #print('static const float ', name, '[', len(v), '] = \n', file=f) + f.write('static const opus_int16 {}[{}] = {{\n '.format(name, len(v))) + for i in range(0, len(v)): + f.write('{}'.format(int(round(8192*v[i])))) + if (i!=len(v)-1): + f.write(',') + else: + break; + if (i%8==7): + f.write("\n ") + else: + f.write(" ") + #print(v, file=f) + f.write('\n};\n\n') + return; + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1) + + +model = load_model("weights.hdf5", custom_objects={'binary_crossentrop2': binary_crossentrop2}) + +weights = model.get_weights() + +f = open('rnn_weights.c', 'w') + +f.write('/*This file is automatically generated from a Keras model*/\n\n') +f.write('#ifdef HAVE_CONFIG_H\n#include "config.h"\n#endif\n\n#include "mlp.h"\n\n') + +printVector(f, weights[0], 'layer0_weights') +printVector(f, weights[1], 'layer0_bias') +printVector(f, weights[2], 'layer1_weights') +printVector(f, weights[3], 'layer1_recur_weights') +printVector(f, weights[4], 'layer1_bias') +printVector(f, weights[5], 'layer2_weights') +printVector(f, weights[6], 'layer2_bias') + +f.write('const DenseLayer layer0 = {\n layer0_bias,\n layer0_weights,\n 25, 16, 0\n};\n\n') +f.write('const GRULayer layer1 = {\n layer1_bias,\n layer1_weights,\n layer1_recur_weights,\n 16, 12\n};\n\n') +f.write('const DenseLayer layer2 = {\n layer2_bias,\n layer2_weights,\n 12, 2, 1\n};\n\n') + +f.close() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/rnn_train.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/rnn_train.py new file mode 100755 index 000000000..ffdaa1e7d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/scripts/rnn_train.py @@ -0,0 +1,67 @@ +#!/usr/bin/python + +from __future__ import print_function + +from keras.models import Sequential +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.layers import SimpleRNN +from keras.layers import Dropout +from keras import losses +import h5py + +from keras import backend as K +import numpy as np + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1) + +print('Build model...') +#model = Sequential() +#model.add(Dense(16, activation='tanh', input_shape=(None, 25))) +#model.add(GRU(12, dropout=0.0, recurrent_dropout=0.0, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)) +#model.add(Dense(2, activation='sigmoid')) + +main_input = Input(shape=(None, 25), name='main_input') +x = Dense(16, activation='tanh')(main_input) +x = GRU(12, dropout=0.1, recurrent_dropout=0.1, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x) +x = Dense(2, activation='sigmoid')(x) +model = Model(inputs=main_input, outputs=x) + +batch_size = 64 + +print('Loading data...') +with h5py.File('features.h5', 'r') as hf: + all_data = hf['features'][:] +print('done.') + +window_size = 1500 + +nb_sequences = len(all_data)/window_size +print(nb_sequences, ' sequences') +x_train = all_data[:nb_sequences*window_size, :-2] +x_train = np.reshape(x_train, (nb_sequences, window_size, 25)) + +y_train = np.copy(all_data[:nb_sequences*window_size, -2:]) +y_train = np.reshape(y_train, (nb_sequences, window_size, 2)) + +all_data = 0; +x_train = x_train.astype('float32') +y_train = y_train.astype('float32') + +print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape) + +# try using different optimizers and different optimizer configs +model.compile(loss=binary_crossentrop2, + optimizer='adam', + metrics=['binary_accuracy']) + +print('Train...') +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=200, + validation_data=(x_train, y_train)) +model.save("newweights.hdf5") diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/A2NLSF.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/A2NLSF.c new file mode 100644 index 000000000..b487686ff --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/A2NLSF.c @@ -0,0 +1,267 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +/* Conversion between prediction filter coefficients and NLSFs */ +/* Requires the order to be an even number */ +/* A piecewise linear approximation maps LSF <-> cos(LSF) */ +/* Therefore the result is not accurate NLSFs, but the two */ +/* functions are accurate inverses of each other */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "tables.h" + +/* Number of binary divisions, when not in low complexity mode */ +#define BIN_DIV_STEPS_A2NLSF_FIX 3 /* must be no higher than 16 - log2( LSF_COS_TAB_SZ_FIX ) */ +#define MAX_ITERATIONS_A2NLSF_FIX 16 + +/* Helper function for A2NLSF(..) */ +/* Transforms polynomials from cos(n*f) to cos(f)^n */ +static OPUS_INLINE void silk_A2NLSF_trans_poly( + opus_int32 *p, /* I/O Polynomial */ + const opus_int dd /* I Polynomial order (= filter order / 2 ) */ +) +{ + opus_int k, n; + + for( k = 2; k <= dd; k++ ) { + for( n = dd; n > k; n-- ) { + p[ n - 2 ] -= p[ n ]; + } + p[ k - 2 ] -= silk_LSHIFT( p[ k ], 1 ); + } +} +/* Helper function for A2NLSF(..) */ +/* Polynomial evaluation */ +static OPUS_INLINE opus_int32 silk_A2NLSF_eval_poly( /* return the polynomial evaluation, in Q16 */ + opus_int32 *p, /* I Polynomial, Q16 */ + const opus_int32 x, /* I Evaluation point, Q12 */ + const opus_int dd /* I Order */ +) +{ + opus_int n; + opus_int32 x_Q16, y32; + + y32 = p[ dd ]; /* Q16 */ + x_Q16 = silk_LSHIFT( x, 4 ); + + if ( opus_likely( 8 == dd ) ) + { + y32 = silk_SMLAWW( p[ 7 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 6 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 5 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 4 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 3 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 2 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 1 ], y32, x_Q16 ); + y32 = silk_SMLAWW( p[ 0 ], y32, x_Q16 ); + } + else + { + for( n = dd - 1; n >= 0; n-- ) { + y32 = silk_SMLAWW( p[ n ], y32, x_Q16 ); /* Q16 */ + } + } + return y32; +} + +static OPUS_INLINE void silk_A2NLSF_init( + const opus_int32 *a_Q16, + opus_int32 *P, + opus_int32 *Q, + const opus_int dd +) +{ + opus_int k; + + /* Convert filter coefs to even and odd polynomials */ + P[dd] = silk_LSHIFT( 1, 16 ); + Q[dd] = silk_LSHIFT( 1, 16 ); + for( k = 0; k < dd; k++ ) { + P[ k ] = -a_Q16[ dd - k - 1 ] - a_Q16[ dd + k ]; /* Q16 */ + Q[ k ] = -a_Q16[ dd - k - 1 ] + a_Q16[ dd + k ]; /* Q16 */ + } + + /* Divide out zeros as we have that for even filter orders, */ + /* z = 1 is always a root in Q, and */ + /* z = -1 is always a root in P */ + for( k = dd; k > 0; k-- ) { + P[ k - 1 ] -= P[ k ]; + Q[ k - 1 ] += Q[ k ]; + } + + /* Transform polynomials from cos(n*f) to cos(f)^n */ + silk_A2NLSF_trans_poly( P, dd ); + silk_A2NLSF_trans_poly( Q, dd ); +} + +/* Compute Normalized Line Spectral Frequencies (NLSFs) from whitening filter coefficients */ +/* If not all roots are found, the a_Q16 coefficients are bandwidth expanded until convergence. */ +void silk_A2NLSF( + opus_int16 *NLSF, /* O Normalized Line Spectral Frequencies in Q15 (0..2^15-1) [d] */ + opus_int32 *a_Q16, /* I/O Monic whitening filter coefficients in Q16 [d] */ + const opus_int d /* I Filter order (must be even) */ +) +{ + opus_int i, k, m, dd, root_ix, ffrac; + opus_int32 xlo, xhi, xmid; + opus_int32 ylo, yhi, ymid, thr; + opus_int32 nom, den; + opus_int32 P[ SILK_MAX_ORDER_LPC / 2 + 1 ]; + opus_int32 Q[ SILK_MAX_ORDER_LPC / 2 + 1 ]; + opus_int32 *PQ[ 2 ]; + opus_int32 *p; + + /* Store pointers to array */ + PQ[ 0 ] = P; + PQ[ 1 ] = Q; + + dd = silk_RSHIFT( d, 1 ); + + silk_A2NLSF_init( a_Q16, P, Q, dd ); + + /* Find roots, alternating between P and Q */ + p = P; /* Pointer to polynomial */ + + xlo = silk_LSFCosTab_FIX_Q12[ 0 ]; /* Q12*/ + ylo = silk_A2NLSF_eval_poly( p, xlo, dd ); + + if( ylo < 0 ) { + /* Set the first NLSF to zero and move on to the next */ + NLSF[ 0 ] = 0; + p = Q; /* Pointer to polynomial */ + ylo = silk_A2NLSF_eval_poly( p, xlo, dd ); + root_ix = 1; /* Index of current root */ + } else { + root_ix = 0; /* Index of current root */ + } + k = 1; /* Loop counter */ + i = 0; /* Counter for bandwidth expansions applied */ + thr = 0; + while( 1 ) { + /* Evaluate polynomial */ + xhi = silk_LSFCosTab_FIX_Q12[ k ]; /* Q12 */ + yhi = silk_A2NLSF_eval_poly( p, xhi, dd ); + + /* Detect zero crossing */ + if( ( ylo <= 0 && yhi >= thr ) || ( ylo >= 0 && yhi <= -thr ) ) { + if( yhi == 0 ) { + /* If the root lies exactly at the end of the current */ + /* interval, look for the next root in the next interval */ + thr = 1; + } else { + thr = 0; + } + /* Binary division */ + ffrac = -256; + for( m = 0; m < BIN_DIV_STEPS_A2NLSF_FIX; m++ ) { + /* Evaluate polynomial */ + xmid = silk_RSHIFT_ROUND( xlo + xhi, 1 ); + ymid = silk_A2NLSF_eval_poly( p, xmid, dd ); + + /* Detect zero crossing */ + if( ( ylo <= 0 && ymid >= 0 ) || ( ylo >= 0 && ymid <= 0 ) ) { + /* Reduce frequency */ + xhi = xmid; + yhi = ymid; + } else { + /* Increase frequency */ + xlo = xmid; + ylo = ymid; + ffrac = silk_ADD_RSHIFT( ffrac, 128, m ); + } + } + + /* Interpolate */ + if( silk_abs( ylo ) < 65536 ) { + /* Avoid dividing by zero */ + den = ylo - yhi; + nom = silk_LSHIFT( ylo, 8 - BIN_DIV_STEPS_A2NLSF_FIX ) + silk_RSHIFT( den, 1 ); + if( den != 0 ) { + ffrac += silk_DIV32( nom, den ); + } + } else { + /* No risk of dividing by zero because abs(ylo - yhi) >= abs(ylo) >= 65536 */ + ffrac += silk_DIV32( ylo, silk_RSHIFT( ylo - yhi, 8 - BIN_DIV_STEPS_A2NLSF_FIX ) ); + } + NLSF[ root_ix ] = (opus_int16)silk_min_32( silk_LSHIFT( (opus_int32)k, 8 ) + ffrac, silk_int16_MAX ); + + silk_assert( NLSF[ root_ix ] >= 0 ); + + root_ix++; /* Next root */ + if( root_ix >= d ) { + /* Found all roots */ + break; + } + /* Alternate pointer to polynomial */ + p = PQ[ root_ix & 1 ]; + + /* Evaluate polynomial */ + xlo = silk_LSFCosTab_FIX_Q12[ k - 1 ]; /* Q12*/ + ylo = silk_LSHIFT( 1 - ( root_ix & 2 ), 12 ); + } else { + /* Increment loop counter */ + k++; + xlo = xhi; + ylo = yhi; + thr = 0; + + if( k > LSF_COS_TAB_SZ_FIX ) { + i++; + if( i > MAX_ITERATIONS_A2NLSF_FIX ) { + /* Set NLSFs to white spectrum and exit */ + NLSF[ 0 ] = (opus_int16)silk_DIV32_16( 1 << 15, d + 1 ); + for( k = 1; k < d; k++ ) { + NLSF[ k ] = (opus_int16)silk_ADD16( NLSF[ k-1 ], NLSF[ 0 ] ); + } + return; + } + + /* Error: Apply progressively more bandwidth expansion and run again */ + silk_bwexpander_32( a_Q16, d, 65536 - silk_LSHIFT( 1, i ) ); + + silk_A2NLSF_init( a_Q16, P, Q, dd ); + p = P; /* Pointer to polynomial */ + xlo = silk_LSFCosTab_FIX_Q12[ 0 ]; /* Q12*/ + ylo = silk_A2NLSF_eval_poly( p, xlo, dd ); + if( ylo < 0 ) { + /* Set the first NLSF to zero and move on to the next */ + NLSF[ 0 ] = 0; + p = Q; /* Pointer to polynomial */ + ylo = silk_A2NLSF_eval_poly( p, xlo, dd ); + root_ix = 1; /* Index of current root */ + } else { + root_ix = 0; /* Index of current root */ + } + k = 1; /* Reset loop counter */ + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/API.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/API.h new file mode 100644 index 000000000..4d90ff9aa --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/API.h @@ -0,0 +1,135 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_API_H +#define SILK_API_H + +#include "control.h" +#include "typedef.h" +#include "errors.h" +#include "entenc.h" +#include "entdec.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define SILK_MAX_FRAMES_PER_PACKET 3 + +/* Struct for TOC (Table of Contents) */ +typedef struct { + opus_int VADFlag; /* Voice activity for packet */ + opus_int VADFlags[ SILK_MAX_FRAMES_PER_PACKET ]; /* Voice activity for each frame in packet */ + opus_int inbandFECFlag; /* Flag indicating if packet contains in-band FEC */ +} silk_TOC_struct; + +/****************************************/ +/* Encoder functions */ +/****************************************/ + +/***********************************************/ +/* Get size in bytes of the Silk encoder state */ +/***********************************************/ +opus_int silk_Get_Encoder_Size( /* O Returns error code */ + opus_int *encSizeBytes /* O Number of bytes in SILK encoder state */ +); + +/*************************/ +/* Init or reset encoder */ +/*************************/ +opus_int silk_InitEncoder( /* O Returns error code */ + void *encState, /* I/O State */ + int arch, /* I Run-time architecture */ + silk_EncControlStruct *encStatus /* O Encoder Status */ +); + +/**************************/ +/* Encode frame with Silk */ +/**************************/ +/* Note: if prefillFlag is set, the input must contain 10 ms of audio, irrespective of what */ +/* encControl->payloadSize_ms is set to */ +opus_int silk_Encode( /* O Returns error code */ + void *encState, /* I/O State */ + silk_EncControlStruct *encControl, /* I Control status */ + const opus_int16 *samplesIn, /* I Speech sample input vector */ + opus_int nSamplesIn, /* I Number of samples in input vector */ + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int32 *nBytesOut, /* I/O Number of bytes in payload (input: Max bytes) */ + const opus_int prefillFlag, /* I Flag to indicate prefilling buffers no coding */ + int activity /* I Decision of Opus voice activity detector */ +); + +/****************************************/ +/* Decoder functions */ +/****************************************/ + +/***********************************************/ +/* Get size in bytes of the Silk decoder state */ +/***********************************************/ +opus_int silk_Get_Decoder_Size( /* O Returns error code */ + opus_int *decSizeBytes /* O Number of bytes in SILK decoder state */ +); + +/*************************/ +/* Init or Reset decoder */ +/*************************/ +opus_int silk_InitDecoder( /* O Returns error code */ + void *decState /* I/O State */ +); + +/******************/ +/* Decode a frame */ +/******************/ +opus_int silk_Decode( /* O Returns error code */ + void* decState, /* I/O State */ + silk_DecControlStruct* decControl, /* I/O Control Structure */ + opus_int lostFlag, /* I 0: no loss, 1 loss, 2 decode fec */ + opus_int newPacketFlag, /* I Indicates first decoder call for this packet */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 *samplesOut, /* O Decoded output speech vector */ + opus_int32 *nSamplesOut, /* O Number of samples decoded */ + int arch /* I Run-time architecture */ +); + +#if 0 +/**************************************/ +/* Get table of contents for a packet */ +/**************************************/ +opus_int silk_get_TOC( + const opus_uint8 *payload, /* I Payload data */ + const opus_int nBytesIn, /* I Number of input bytes */ + const opus_int nFramesPerPayload, /* I Number of SILK frames per payload */ + silk_TOC_struct *Silk_TOC /* O Type of content */ +); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/CNG.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/CNG.c new file mode 100644 index 000000000..2a910099e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/CNG.c @@ -0,0 +1,188 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/* Generates excitation for CNG LPC synthesis */ +static OPUS_INLINE void silk_CNG_exc( + opus_int32 exc_Q14[], /* O CNG excitation signal Q10 */ + opus_int32 exc_buf_Q14[], /* I Random samples buffer Q10 */ + opus_int length, /* I Length */ + opus_int32 *rand_seed /* I/O Seed to random index generator */ +) +{ + opus_int32 seed; + opus_int i, idx, exc_mask; + + exc_mask = CNG_BUF_MASK_MAX; + while( exc_mask > length ) { + exc_mask = silk_RSHIFT( exc_mask, 1 ); + } + + seed = *rand_seed; + for( i = 0; i < length; i++ ) { + seed = silk_RAND( seed ); + idx = (opus_int)( silk_RSHIFT( seed, 24 ) & exc_mask ); + silk_assert( idx >= 0 ); + silk_assert( idx <= CNG_BUF_MASK_MAX ); + exc_Q14[ i ] = exc_buf_Q14[ idx ]; + } + *rand_seed = seed; +} + +void silk_CNG_Reset( + silk_decoder_state *psDec /* I/O Decoder state */ +) +{ + opus_int i, NLSF_step_Q15, NLSF_acc_Q15; + + NLSF_step_Q15 = silk_DIV32_16( silk_int16_MAX, psDec->LPC_order + 1 ); + NLSF_acc_Q15 = 0; + for( i = 0; i < psDec->LPC_order; i++ ) { + NLSF_acc_Q15 += NLSF_step_Q15; + psDec->sCNG.CNG_smth_NLSF_Q15[ i ] = NLSF_acc_Q15; + } + psDec->sCNG.CNG_smth_Gain_Q16 = 0; + psDec->sCNG.rand_seed = 3176576; +} + +/* Updates CNG estimate, and applies the CNG when packet was lost */ +void silk_CNG( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* I/O Signal */ + opus_int length /* I Length of residual */ +) +{ + opus_int i, subfr; + opus_int32 LPC_pred_Q10, max_Gain_Q16, gain_Q16, gain_Q10; + opus_int16 A_Q12[ MAX_LPC_ORDER ]; + silk_CNG_struct *psCNG = &psDec->sCNG; + SAVE_STACK; + + if( psDec->fs_kHz != psCNG->fs_kHz ) { + /* Reset state */ + silk_CNG_Reset( psDec ); + + psCNG->fs_kHz = psDec->fs_kHz; + } + if( psDec->lossCnt == 0 && psDec->prevSignalType == TYPE_NO_VOICE_ACTIVITY ) { + /* Update CNG parameters */ + + /* Smoothing of LSF's */ + for( i = 0; i < psDec->LPC_order; i++ ) { + psCNG->CNG_smth_NLSF_Q15[ i ] += silk_SMULWB( (opus_int32)psDec->prevNLSF_Q15[ i ] - (opus_int32)psCNG->CNG_smth_NLSF_Q15[ i ], CNG_NLSF_SMTH_Q16 ); + } + /* Find the subframe with the highest gain */ + max_Gain_Q16 = 0; + subfr = 0; + for( i = 0; i < psDec->nb_subfr; i++ ) { + if( psDecCtrl->Gains_Q16[ i ] > max_Gain_Q16 ) { + max_Gain_Q16 = psDecCtrl->Gains_Q16[ i ]; + subfr = i; + } + } + /* Update CNG excitation buffer with excitation from this subframe */ + silk_memmove( &psCNG->CNG_exc_buf_Q14[ psDec->subfr_length ], psCNG->CNG_exc_buf_Q14, ( psDec->nb_subfr - 1 ) * psDec->subfr_length * sizeof( opus_int32 ) ); + silk_memcpy( psCNG->CNG_exc_buf_Q14, &psDec->exc_Q14[ subfr * psDec->subfr_length ], psDec->subfr_length * sizeof( opus_int32 ) ); + + /* Smooth gains */ + for( i = 0; i < psDec->nb_subfr; i++ ) { + psCNG->CNG_smth_Gain_Q16 += silk_SMULWB( psDecCtrl->Gains_Q16[ i ] - psCNG->CNG_smth_Gain_Q16, CNG_GAIN_SMTH_Q16 ); + /* If the smoothed gain is 3 dB greater than this subframe's gain, use this subframe's gain to adapt faster. */ + if( silk_SMULWW( psCNG->CNG_smth_Gain_Q16, CNG_GAIN_SMTH_THRESHOLD_Q16 ) > psDecCtrl->Gains_Q16[ i ] ) { + psCNG->CNG_smth_Gain_Q16 = psDecCtrl->Gains_Q16[ i ]; + } + } + } + + /* Add CNG when packet is lost or during DTX */ + if( psDec->lossCnt ) { + VARDECL( opus_int32, CNG_sig_Q14 ); + ALLOC( CNG_sig_Q14, length + MAX_LPC_ORDER, opus_int32 ); + + /* Generate CNG excitation */ + gain_Q16 = silk_SMULWW( psDec->sPLC.randScale_Q14, psDec->sPLC.prevGain_Q16[1] ); + if( gain_Q16 >= (1 << 21) || psCNG->CNG_smth_Gain_Q16 > (1 << 23) ) { + gain_Q16 = silk_SMULTT( gain_Q16, gain_Q16 ); + gain_Q16 = silk_SUB_LSHIFT32(silk_SMULTT( psCNG->CNG_smth_Gain_Q16, psCNG->CNG_smth_Gain_Q16 ), gain_Q16, 5 ); + gain_Q16 = silk_LSHIFT32( silk_SQRT_APPROX( gain_Q16 ), 16 ); + } else { + gain_Q16 = silk_SMULWW( gain_Q16, gain_Q16 ); + gain_Q16 = silk_SUB_LSHIFT32(silk_SMULWW( psCNG->CNG_smth_Gain_Q16, psCNG->CNG_smth_Gain_Q16 ), gain_Q16, 5 ); + gain_Q16 = silk_LSHIFT32( silk_SQRT_APPROX( gain_Q16 ), 8 ); + } + gain_Q10 = silk_RSHIFT( gain_Q16, 6 ); + + silk_CNG_exc( CNG_sig_Q14 + MAX_LPC_ORDER, psCNG->CNG_exc_buf_Q14, length, &psCNG->rand_seed ); + + /* Convert CNG NLSF to filter representation */ + silk_NLSF2A( A_Q12, psCNG->CNG_smth_NLSF_Q15, psDec->LPC_order, psDec->arch ); + + /* Generate CNG signal, by synthesis filtering */ + silk_memcpy( CNG_sig_Q14, psCNG->CNG_synth_state, MAX_LPC_ORDER * sizeof( opus_int32 ) ); + celt_assert( psDec->LPC_order == 10 || psDec->LPC_order == 16 ); + for( i = 0; i < length; i++ ) { + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q10 = silk_RSHIFT( psDec->LPC_order, 1 ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 1 ], A_Q12[ 0 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 2 ], A_Q12[ 1 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 3 ], A_Q12[ 2 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 4 ], A_Q12[ 3 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 5 ], A_Q12[ 4 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 6 ], A_Q12[ 5 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 7 ], A_Q12[ 6 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 8 ], A_Q12[ 7 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 9 ], A_Q12[ 8 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 10 ], A_Q12[ 9 ] ); + if( psDec->LPC_order == 16 ) { + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 11 ], A_Q12[ 10 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 12 ], A_Q12[ 11 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 13 ], A_Q12[ 12 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 14 ], A_Q12[ 13 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 15 ], A_Q12[ 14 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, CNG_sig_Q14[ MAX_LPC_ORDER + i - 16 ], A_Q12[ 15 ] ); + } + + /* Update states */ + CNG_sig_Q14[ MAX_LPC_ORDER + i ] = silk_ADD_SAT32( CNG_sig_Q14[ MAX_LPC_ORDER + i ], silk_LSHIFT_SAT32( LPC_pred_Q10, 4 ) ); + + /* Scale with Gain and add to input signal */ + frame[ i ] = (opus_int16)silk_ADD_SAT16( frame[ i ], silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( CNG_sig_Q14[ MAX_LPC_ORDER + i ], gain_Q10 ), 8 ) ) ); + + } + silk_memcpy( psCNG->CNG_synth_state, &CNG_sig_Q14[ length ], MAX_LPC_ORDER * sizeof( opus_int32 ) ); + } else { + silk_memset( psCNG->CNG_synth_state, 0, psDec->LPC_order * sizeof( opus_int32 ) ); + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/HP_variable_cutoff.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/HP_variable_cutoff.c new file mode 100644 index 000000000..bbe10f04c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/HP_variable_cutoff.c @@ -0,0 +1,77 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef FIXED_POINT +#include "main_FIX.h" +#else +#include "main_FLP.h" +#endif +#include "tuning_parameters.h" + +/* High-pass filter with cutoff frequency adaptation based on pitch lag statistics */ +void silk_HP_variable_cutoff( + silk_encoder_state_Fxx state_Fxx[] /* I/O Encoder states */ +) +{ + opus_int quality_Q15; + opus_int32 pitch_freq_Hz_Q16, pitch_freq_log_Q7, delta_freq_Q7; + silk_encoder_state *psEncC1 = &state_Fxx[ 0 ].sCmn; + + /* Adaptive cutoff frequency: estimate low end of pitch frequency range */ + if( psEncC1->prevSignalType == TYPE_VOICED ) { + /* difference, in log domain */ + pitch_freq_Hz_Q16 = silk_DIV32_16( silk_LSHIFT( silk_MUL( psEncC1->fs_kHz, 1000 ), 16 ), psEncC1->prevLag ); + pitch_freq_log_Q7 = silk_lin2log( pitch_freq_Hz_Q16 ) - ( 16 << 7 ); + + /* adjustment based on quality */ + quality_Q15 = psEncC1->input_quality_bands_Q15[ 0 ]; + pitch_freq_log_Q7 = silk_SMLAWB( pitch_freq_log_Q7, silk_SMULWB( silk_LSHIFT( -quality_Q15, 2 ), quality_Q15 ), + pitch_freq_log_Q7 - ( silk_lin2log( SILK_FIX_CONST( VARIABLE_HP_MIN_CUTOFF_HZ, 16 ) ) - ( 16 << 7 ) ) ); + + /* delta_freq = pitch_freq_log - psEnc->variable_HP_smth1; */ + delta_freq_Q7 = pitch_freq_log_Q7 - silk_RSHIFT( psEncC1->variable_HP_smth1_Q15, 8 ); + if( delta_freq_Q7 < 0 ) { + /* less smoothing for decreasing pitch frequency, to track something close to the minimum */ + delta_freq_Q7 = silk_MUL( delta_freq_Q7, 3 ); + } + + /* limit delta, to reduce impact of outliers in pitch estimation */ + delta_freq_Q7 = silk_LIMIT_32( delta_freq_Q7, -SILK_FIX_CONST( VARIABLE_HP_MAX_DELTA_FREQ, 7 ), SILK_FIX_CONST( VARIABLE_HP_MAX_DELTA_FREQ, 7 ) ); + + /* update smoother */ + psEncC1->variable_HP_smth1_Q15 = silk_SMLAWB( psEncC1->variable_HP_smth1_Q15, + silk_SMULBB( psEncC1->speech_activity_Q8, delta_freq_Q7 ), SILK_FIX_CONST( VARIABLE_HP_SMTH_COEF1, 16 ) ); + + /* limit frequency range */ + psEncC1->variable_HP_smth1_Q15 = silk_LIMIT_32( psEncC1->variable_HP_smth1_Q15, + silk_LSHIFT( silk_lin2log( VARIABLE_HP_MIN_CUTOFF_HZ ), 8 ), + silk_LSHIFT( silk_lin2log( VARIABLE_HP_MAX_CUTOFF_HZ ), 8 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/Inlines.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/Inlines.h new file mode 100644 index 000000000..ec986cdfd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/Inlines.h @@ -0,0 +1,188 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +/*! \file silk_Inlines.h + * \brief silk_Inlines.h defines OPUS_INLINE signal processing functions. + */ + +#ifndef SILK_FIX_INLINES_H +#define SILK_FIX_INLINES_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* count leading zeros of opus_int64 */ +static OPUS_INLINE opus_int32 silk_CLZ64( opus_int64 in ) +{ + opus_int32 in_upper; + + in_upper = (opus_int32)silk_RSHIFT64(in, 32); + if (in_upper == 0) { + /* Search in the lower 32 bits */ + return 32 + silk_CLZ32( (opus_int32) in ); + } else { + /* Search in the upper 32 bits */ + return silk_CLZ32( in_upper ); + } +} + +/* get number of leading zeros and fractional part (the bits right after the leading one */ +static OPUS_INLINE void silk_CLZ_FRAC( + opus_int32 in, /* I input */ + opus_int32 *lz, /* O number of leading zeros */ + opus_int32 *frac_Q7 /* O the 7 bits right after the leading one */ +) +{ + opus_int32 lzeros = silk_CLZ32(in); + + * lz = lzeros; + * frac_Q7 = silk_ROR32(in, 24 - lzeros) & 0x7f; +} + +/* Approximation of square root */ +/* Accuracy: < +/- 10% for output values > 15 */ +/* < +/- 2.5% for output values > 120 */ +static OPUS_INLINE opus_int32 silk_SQRT_APPROX( opus_int32 x ) +{ + opus_int32 y, lz, frac_Q7; + + if( x <= 0 ) { + return 0; + } + + silk_CLZ_FRAC(x, &lz, &frac_Q7); + + if( lz & 1 ) { + y = 32768; + } else { + y = 46214; /* 46214 = sqrt(2) * 32768 */ + } + + /* get scaling right */ + y >>= silk_RSHIFT(lz, 1); + + /* increment using fractional part of input */ + y = silk_SMLAWB(y, y, silk_SMULBB(213, frac_Q7)); + + return y; +} + +/* Divide two int32 values and return result as int32 in a given Q-domain */ +static OPUS_INLINE opus_int32 silk_DIV32_varQ( /* O returns a good approximation of "(a32 << Qres) / b32" */ + const opus_int32 a32, /* I numerator (Q0) */ + const opus_int32 b32, /* I denominator (Q0) */ + const opus_int Qres /* I Q-domain of result (>= 0) */ +) +{ + opus_int a_headrm, b_headrm, lshift; + opus_int32 b32_inv, a32_nrm, b32_nrm, result; + + silk_assert( b32 != 0 ); + silk_assert( Qres >= 0 ); + + /* Compute number of bits head room and normalize inputs */ + a_headrm = silk_CLZ32( silk_abs(a32) ) - 1; + a32_nrm = silk_LSHIFT(a32, a_headrm); /* Q: a_headrm */ + b_headrm = silk_CLZ32( silk_abs(b32) ) - 1; + b32_nrm = silk_LSHIFT(b32, b_headrm); /* Q: b_headrm */ + + /* Inverse of b32, with 14 bits of precision */ + b32_inv = silk_DIV32_16( silk_int32_MAX >> 2, silk_RSHIFT(b32_nrm, 16) ); /* Q: 29 + 16 - b_headrm */ + + /* First approximation */ + result = silk_SMULWB(a32_nrm, b32_inv); /* Q: 29 + a_headrm - b_headrm */ + + /* Compute residual by subtracting product of denominator and first approximation */ + /* It's OK to overflow because the final value of a32_nrm should always be small */ + a32_nrm = silk_SUB32_ovflw(a32_nrm, silk_LSHIFT_ovflw( silk_SMMUL(b32_nrm, result), 3 )); /* Q: a_headrm */ + + /* Refinement */ + result = silk_SMLAWB(result, a32_nrm, b32_inv); /* Q: 29 + a_headrm - b_headrm */ + + /* Convert to Qres domain */ + lshift = 29 + a_headrm - b_headrm - Qres; + if( lshift < 0 ) { + return silk_LSHIFT_SAT32(result, -lshift); + } else { + if( lshift < 32){ + return silk_RSHIFT(result, lshift); + } else { + /* Avoid undefined result */ + return 0; + } + } +} + +/* Invert int32 value and return result as int32 in a given Q-domain */ +static OPUS_INLINE opus_int32 silk_INVERSE32_varQ( /* O returns a good approximation of "(1 << Qres) / b32" */ + const opus_int32 b32, /* I denominator (Q0) */ + const opus_int Qres /* I Q-domain of result (> 0) */ +) +{ + opus_int b_headrm, lshift; + opus_int32 b32_inv, b32_nrm, err_Q32, result; + + silk_assert( b32 != 0 ); + silk_assert( Qres > 0 ); + + /* Compute number of bits head room and normalize input */ + b_headrm = silk_CLZ32( silk_abs(b32) ) - 1; + b32_nrm = silk_LSHIFT(b32, b_headrm); /* Q: b_headrm */ + + /* Inverse of b32, with 14 bits of precision */ + b32_inv = silk_DIV32_16( silk_int32_MAX >> 2, silk_RSHIFT(b32_nrm, 16) ); /* Q: 29 + 16 - b_headrm */ + + /* First approximation */ + result = silk_LSHIFT(b32_inv, 16); /* Q: 61 - b_headrm */ + + /* Compute residual by subtracting product of denominator and first approximation from one */ + err_Q32 = silk_LSHIFT( ((opus_int32)1<<29) - silk_SMULWB(b32_nrm, b32_inv), 3 ); /* Q32 */ + + /* Refinement */ + result = silk_SMLAWW(result, err_Q32, b32_inv); /* Q: 61 - b_headrm */ + + /* Convert to Qres domain */ + lshift = 61 - b_headrm - Qres; + if( lshift <= 0 ) { + return silk_LSHIFT_SAT32(result, -lshift); + } else { + if( lshift < 32){ + return silk_RSHIFT(result, lshift); + }else{ + /* Avoid undefined result */ + return 0; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_FIX_INLINES_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_analysis_filter.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_analysis_filter.c new file mode 100644 index 000000000..d34b5eb70 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_analysis_filter.c @@ -0,0 +1,111 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "celt_lpc.h" + +/*******************************************/ +/* LPC analysis filter */ +/* NB! State is kept internally and the */ +/* filter always starts with zero state */ +/* first d output samples are set to zero */ +/*******************************************/ + +/* OPT: Using celt_fir() for this function should be faster, but it may cause + integer overflows in intermediate values (not final results), which the + current implementation silences by casting to unsigned. Enabling + this should be safe in pretty much all cases, even though it is not technically + C89-compliant. */ +#define USE_CELT_FIR 0 + +void silk_LPC_analysis_filter( + opus_int16 *out, /* O Output signal */ + const opus_int16 *in, /* I Input signal */ + const opus_int16 *B, /* I MA prediction coefficients, Q12 [order] */ + const opus_int32 len, /* I Signal length */ + const opus_int32 d, /* I Filter order */ + int arch /* I Run-time architecture */ +) +{ + opus_int j; +#if defined(FIXED_POINT) && USE_CELT_FIR + opus_int16 num[SILK_MAX_ORDER_LPC]; +#else + int ix; + opus_int32 out32_Q12, out32; + const opus_int16 *in_ptr; +#endif + + celt_assert( d >= 6 ); + celt_assert( (d & 1) == 0 ); + celt_assert( d <= len ); + +#if defined(FIXED_POINT) && USE_CELT_FIR + celt_assert( d <= SILK_MAX_ORDER_LPC ); + for ( j = 0; j < d; j++ ) { + num[ j ] = -B[ j ]; + } + celt_fir( in + d, num, out + d, len - d, d, arch ); + for ( j = 0; j < d; j++ ) { + out[ j ] = 0; + } +#else + (void)arch; + for( ix = d; ix < len; ix++ ) { + in_ptr = &in[ ix - 1 ]; + + out32_Q12 = silk_SMULBB( in_ptr[ 0 ], B[ 0 ] ); + /* Allowing wrap around so that two wraps can cancel each other. The rare + cases where the result wraps around can only be triggered by invalid streams*/ + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -1 ], B[ 1 ] ); + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -2 ], B[ 2 ] ); + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -3 ], B[ 3 ] ); + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -4 ], B[ 4 ] ); + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -5 ], B[ 5 ] ); + for( j = 6; j < d; j += 2 ) { + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -j ], B[ j ] ); + out32_Q12 = silk_SMLABB_ovflw( out32_Q12, in_ptr[ -j - 1 ], B[ j + 1 ] ); + } + + /* Subtract prediction */ + out32_Q12 = silk_SUB32_ovflw( silk_LSHIFT( (opus_int32)in_ptr[ 1 ], 12 ), out32_Q12 ); + + /* Scale to Q0 */ + out32 = silk_RSHIFT_ROUND( out32_Q12, 12 ); + + /* Saturate output */ + out[ ix ] = (opus_int16)silk_SAT16( out32 ); + } + + /* Set first d output samples to zero */ + silk_memset( out, 0, d * sizeof( opus_int16 ) ); +#endif +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_fit.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_fit.c new file mode 100644 index 000000000..c0690a1fc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_fit.c @@ -0,0 +1,82 @@ +/*********************************************************************** +Copyright (c) 2013, Koen Vos. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Convert int32 coefficients to int16 coefs and make sure there's no wrap-around. + This logic is reused in _celt_lpc(). Any bug fixes should also be applied there. */ +void silk_LPC_fit( + opus_int16 *a_QOUT, /* O Output signal */ + opus_int32 *a_QIN, /* I/O Input signal */ + const opus_int QOUT, /* I Input Q domain */ + const opus_int QIN, /* I Input Q domain */ + const opus_int d /* I Filter order */ +) +{ + opus_int i, k, idx = 0; + opus_int32 maxabs, absval, chirp_Q16; + + /* Limit the maximum absolute value of the prediction coefficients, so that they'll fit in int16 */ + for( i = 0; i < 10; i++ ) { + /* Find maximum absolute value and its index */ + maxabs = 0; + for( k = 0; k < d; k++ ) { + absval = silk_abs( a_QIN[k] ); + if( absval > maxabs ) { + maxabs = absval; + idx = k; + } + } + maxabs = silk_RSHIFT_ROUND( maxabs, QIN - QOUT ); + + if( maxabs > silk_int16_MAX ) { + /* Reduce magnitude of prediction coefficients */ + maxabs = silk_min( maxabs, 163838 ); /* ( silk_int32_MAX >> 14 ) + silk_int16_MAX = 163838 */ + chirp_Q16 = SILK_FIX_CONST( 0.999, 16 ) - silk_DIV32( silk_LSHIFT( maxabs - silk_int16_MAX, 14 ), + silk_RSHIFT32( silk_MUL( maxabs, idx + 1), 2 ) ); + silk_bwexpander_32( a_QIN, d, chirp_Q16 ); + } else { + break; + } + } + + if( i == 10 ) { + /* Reached the last iteration, clip the coefficients */ + for( k = 0; k < d; k++ ) { + a_QOUT[ k ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( a_QIN[ k ], QIN - QOUT ) ); + a_QIN[ k ] = silk_LSHIFT( (opus_int32)a_QOUT[ k ], QIN - QOUT ); + } + } else { + for( k = 0; k < d; k++ ) { + a_QOUT[ k ] = (opus_int16)silk_RSHIFT_ROUND( a_QIN[ k ], QIN - QOUT ); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_inv_pred_gain.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_inv_pred_gain.c new file mode 100644 index 000000000..a3746a6ef --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LPC_inv_pred_gain.c @@ -0,0 +1,141 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "define.h" + +#define QA 24 +#define A_LIMIT SILK_FIX_CONST( 0.99975, QA ) + +#define MUL32_FRAC_Q(a32, b32, Q) ((opus_int32)(silk_RSHIFT_ROUND64(silk_SMULL(a32, b32), Q))) + +/* Compute inverse of LPC prediction gain, and */ +/* test if LPC coefficients are stable (all poles within unit circle) */ +static opus_int32 LPC_inverse_pred_gain_QA_c( /* O Returns inverse prediction gain in energy domain, Q30 */ + opus_int32 A_QA[ SILK_MAX_ORDER_LPC ], /* I Prediction coefficients */ + const opus_int order /* I Prediction order */ +) +{ + opus_int k, n, mult2Q; + opus_int32 invGain_Q30, rc_Q31, rc_mult1_Q30, rc_mult2, tmp1, tmp2; + + invGain_Q30 = SILK_FIX_CONST( 1, 30 ); + for( k = order - 1; k > 0; k-- ) { + /* Check for stability */ + if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { + return 0; + } + + /* Set RC equal to negated AR coef */ + rc_Q31 = -silk_LSHIFT( A_QA[ k ], 31 - QA ); + + /* rc_mult1_Q30 range: [ 1 : 2^30 ] */ + rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); + silk_assert( rc_mult1_Q30 > ( 1 << 15 ) ); /* reduce A_LIMIT if fails */ + silk_assert( rc_mult1_Q30 <= ( 1 << 30 ) ); + + /* Update inverse gain */ + /* invGain_Q30 range: [ 0 : 2^30 ] */ + invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); + silk_assert( invGain_Q30 >= 0 ); + silk_assert( invGain_Q30 <= ( 1 << 30 ) ); + if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { + return 0; + } + + /* rc_mult2 range: [ 2^30 : silk_int32_MAX ] */ + mult2Q = 32 - silk_CLZ32( silk_abs( rc_mult1_Q30 ) ); + rc_mult2 = silk_INVERSE32_varQ( rc_mult1_Q30, mult2Q + 30 ); + + /* Update AR coefficient */ + for( n = 0; n < (k + 1) >> 1; n++ ) { + opus_int64 tmp64; + tmp1 = A_QA[ n ]; + tmp2 = A_QA[ k - n - 1 ]; + tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp1, + MUL32_FRAC_Q( tmp2, rc_Q31, 31 ) ), rc_mult2 ), mult2Q); + if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { + return 0; + } + A_QA[ n ] = ( opus_int32 )tmp64; + tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp2, + MUL32_FRAC_Q( tmp1, rc_Q31, 31 ) ), rc_mult2), mult2Q); + if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { + return 0; + } + A_QA[ k - n - 1 ] = ( opus_int32 )tmp64; + } + } + + /* Check for stability */ + if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { + return 0; + } + + /* Set RC equal to negated AR coef */ + rc_Q31 = -silk_LSHIFT( A_QA[ 0 ], 31 - QA ); + + /* Range: [ 1 : 2^30 ] */ + rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); + + /* Update inverse gain */ + /* Range: [ 0 : 2^30 ] */ + invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); + silk_assert( invGain_Q30 >= 0 ); + silk_assert( invGain_Q30 <= ( 1 << 30 ) ); + if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { + return 0; + } + + return invGain_Q30; +} + +/* For input in Q12 domain */ +opus_int32 silk_LPC_inverse_pred_gain_c( /* O Returns inverse prediction gain in energy domain, Q30 */ + const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ + const opus_int order /* I Prediction order */ +) +{ + opus_int k; + opus_int32 Atmp_QA[ SILK_MAX_ORDER_LPC ]; + opus_int32 DC_resp = 0; + + /* Increase Q domain of the AR coefficients */ + for( k = 0; k < order; k++ ) { + DC_resp += (opus_int32)A_Q12[ k ]; + Atmp_QA[ k ] = silk_LSHIFT32( (opus_int32)A_Q12[ k ], QA - 12 ); + } + /* If the DC is unstable, we don't even need to do the full calculations */ + if( DC_resp >= 4096 ) { + return 0; + } + return LPC_inverse_pred_gain_QA_c( Atmp_QA, order ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LP_variable_cutoff.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LP_variable_cutoff.c new file mode 100644 index 000000000..79112ad35 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/LP_variable_cutoff.c @@ -0,0 +1,135 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* + Elliptic/Cauer filters designed with 0.1 dB passband ripple, + 80 dB minimum stopband attenuation, and + [0.95 : 0.15 : 0.35] normalized cut off frequencies. +*/ + +#include "main.h" + +/* Helper function, interpolates the filter taps */ +static OPUS_INLINE void silk_LP_interpolate_filter_taps( + opus_int32 B_Q28[ TRANSITION_NB ], + opus_int32 A_Q28[ TRANSITION_NA ], + const opus_int ind, + const opus_int32 fac_Q16 +) +{ + opus_int nb, na; + + if( ind < TRANSITION_INT_NUM - 1 ) { + if( fac_Q16 > 0 ) { + if( fac_Q16 < 32768 ) { /* fac_Q16 is in range of a 16-bit int */ + /* Piece-wise linear interpolation of B and A */ + for( nb = 0; nb < TRANSITION_NB; nb++ ) { + B_Q28[ nb ] = silk_SMLAWB( + silk_Transition_LP_B_Q28[ ind ][ nb ], + silk_Transition_LP_B_Q28[ ind + 1 ][ nb ] - + silk_Transition_LP_B_Q28[ ind ][ nb ], + fac_Q16 ); + } + for( na = 0; na < TRANSITION_NA; na++ ) { + A_Q28[ na ] = silk_SMLAWB( + silk_Transition_LP_A_Q28[ ind ][ na ], + silk_Transition_LP_A_Q28[ ind + 1 ][ na ] - + silk_Transition_LP_A_Q28[ ind ][ na ], + fac_Q16 ); + } + } else { /* ( fac_Q16 - ( 1 << 16 ) ) is in range of a 16-bit int */ + silk_assert( fac_Q16 - ( 1 << 16 ) == silk_SAT16( fac_Q16 - ( 1 << 16 ) ) ); + /* Piece-wise linear interpolation of B and A */ + for( nb = 0; nb < TRANSITION_NB; nb++ ) { + B_Q28[ nb ] = silk_SMLAWB( + silk_Transition_LP_B_Q28[ ind + 1 ][ nb ], + silk_Transition_LP_B_Q28[ ind + 1 ][ nb ] - + silk_Transition_LP_B_Q28[ ind ][ nb ], + fac_Q16 - ( (opus_int32)1 << 16 ) ); + } + for( na = 0; na < TRANSITION_NA; na++ ) { + A_Q28[ na ] = silk_SMLAWB( + silk_Transition_LP_A_Q28[ ind + 1 ][ na ], + silk_Transition_LP_A_Q28[ ind + 1 ][ na ] - + silk_Transition_LP_A_Q28[ ind ][ na ], + fac_Q16 - ( (opus_int32)1 << 16 ) ); + } + } + } else { + silk_memcpy( B_Q28, silk_Transition_LP_B_Q28[ ind ], TRANSITION_NB * sizeof( opus_int32 ) ); + silk_memcpy( A_Q28, silk_Transition_LP_A_Q28[ ind ], TRANSITION_NA * sizeof( opus_int32 ) ); + } + } else { + silk_memcpy( B_Q28, silk_Transition_LP_B_Q28[ TRANSITION_INT_NUM - 1 ], TRANSITION_NB * sizeof( opus_int32 ) ); + silk_memcpy( A_Q28, silk_Transition_LP_A_Q28[ TRANSITION_INT_NUM - 1 ], TRANSITION_NA * sizeof( opus_int32 ) ); + } +} + +/* Low-pass filter with variable cutoff frequency based on */ +/* piece-wise linear interpolation between elliptic filters */ +/* Start by setting psEncC->mode <> 0; */ +/* Deactivate by setting psEncC->mode = 0; */ +void silk_LP_variable_cutoff( + silk_LP_state *psLP, /* I/O LP filter state */ + opus_int16 *frame, /* I/O Low-pass filtered output signal */ + const opus_int frame_length /* I Frame length */ +) +{ + opus_int32 B_Q28[ TRANSITION_NB ], A_Q28[ TRANSITION_NA ], fac_Q16 = 0; + opus_int ind = 0; + + silk_assert( psLP->transition_frame_no >= 0 && psLP->transition_frame_no <= TRANSITION_FRAMES ); + + /* Run filter if needed */ + if( psLP->mode != 0 ) { + /* Calculate index and interpolation factor for interpolation */ +#if( TRANSITION_INT_STEPS == 64 ) + fac_Q16 = silk_LSHIFT( TRANSITION_FRAMES - psLP->transition_frame_no, 16 - 6 ); +#else + fac_Q16 = silk_DIV32_16( silk_LSHIFT( TRANSITION_FRAMES - psLP->transition_frame_no, 16 ), TRANSITION_FRAMES ); +#endif + ind = silk_RSHIFT( fac_Q16, 16 ); + fac_Q16 -= silk_LSHIFT( ind, 16 ); + + silk_assert( ind >= 0 ); + silk_assert( ind < TRANSITION_INT_NUM ); + + /* Interpolate filter coefficients */ + silk_LP_interpolate_filter_taps( B_Q28, A_Q28, ind, fac_Q16 ); + + /* Update transition frame number for next frame */ + psLP->transition_frame_no = silk_LIMIT( psLP->transition_frame_no + psLP->mode, 0, TRANSITION_FRAMES ); + + /* ARMA low-pass filtering */ + silk_assert( TRANSITION_NB == 3 && TRANSITION_NA == 2 ); + silk_biquad_alt_stride1( frame, B_Q28, A_Q28, psLP->In_LP_State, frame, frame_length); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroCount.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroCount.h new file mode 100644 index 000000000..dab2f57a6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroCount.h @@ -0,0 +1,710 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SIGPROCFIX_API_MACROCOUNT_H +#define SIGPROCFIX_API_MACROCOUNT_H + +#ifdef silk_MACRO_COUNT +#include +#define varDefine opus_int64 ops_count = 0; + +extern opus_int64 ops_count; + +static OPUS_INLINE opus_int64 silk_SaveCount(){ + return(ops_count); +} + +static OPUS_INLINE opus_int64 silk_SaveResetCount(){ + opus_int64 ret; + + ret = ops_count; + ops_count = 0; + return(ret); +} + +static OPUS_INLINE silk_PrintCount(){ + printf("ops_count = %d \n ", (opus_int32)ops_count); +} + +#undef silk_MUL +static OPUS_INLINE opus_int32 silk_MUL(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + ops_count += 4; + ret = a32 * b32; + return ret; +} + +#undef silk_MUL_uint +static OPUS_INLINE opus_uint32 silk_MUL_uint(opus_uint32 a32, opus_uint32 b32){ + opus_uint32 ret; + ops_count += 4; + ret = a32 * b32; + return ret; +} +#undef silk_MLA +static OPUS_INLINE opus_int32 silk_MLA(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 4; + ret = a32 + b32 * c32; + return ret; +} + +#undef silk_MLA_uint +static OPUS_INLINE opus_int32 silk_MLA_uint(opus_uint32 a32, opus_uint32 b32, opus_uint32 c32){ + opus_uint32 ret; + ops_count += 4; + ret = a32 + b32 * c32; + return ret; +} + +#undef silk_SMULWB +static OPUS_INLINE opus_int32 silk_SMULWB(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + ops_count += 5; + ret = (a32 >> 16) * (opus_int32)((opus_int16)b32) + (((a32 & 0x0000FFFF) * (opus_int32)((opus_int16)b32)) >> 16); + return ret; +} +#undef silk_SMLAWB +static OPUS_INLINE opus_int32 silk_SMLAWB(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 5; + ret = ((a32) + ((((b32) >> 16) * (opus_int32)((opus_int16)(c32))) + ((((b32) & 0x0000FFFF) * (opus_int32)((opus_int16)(c32))) >> 16))); + return ret; +} + +#undef silk_SMULWT +static OPUS_INLINE opus_int32 silk_SMULWT(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + ops_count += 4; + ret = (a32 >> 16) * (b32 >> 16) + (((a32 & 0x0000FFFF) * (b32 >> 16)) >> 16); + return ret; +} +#undef silk_SMLAWT +static OPUS_INLINE opus_int32 silk_SMLAWT(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 4; + ret = a32 + ((b32 >> 16) * (c32 >> 16)) + (((b32 & 0x0000FFFF) * ((c32 >> 16)) >> 16)); + return ret; +} + +#undef silk_SMULBB +static OPUS_INLINE opus_int32 silk_SMULBB(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + ops_count += 1; + ret = (opus_int32)((opus_int16)a32) * (opus_int32)((opus_int16)b32); + return ret; +} +#undef silk_SMLABB +static OPUS_INLINE opus_int32 silk_SMLABB(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 1; + ret = a32 + (opus_int32)((opus_int16)b32) * (opus_int32)((opus_int16)c32); + return ret; +} + +#undef silk_SMULBT +static OPUS_INLINE opus_int32 silk_SMULBT(opus_int32 a32, opus_int32 b32 ){ + opus_int32 ret; + ops_count += 4; + ret = ((opus_int32)((opus_int16)a32)) * (b32 >> 16); + return ret; +} + +#undef silk_SMLABT +static OPUS_INLINE opus_int32 silk_SMLABT(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 1; + ret = a32 + ((opus_int32)((opus_int16)b32)) * (c32 >> 16); + return ret; +} + +#undef silk_SMULTT +static OPUS_INLINE opus_int32 silk_SMULTT(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + ops_count += 1; + ret = (a32 >> 16) * (b32 >> 16); + return ret; +} + +#undef silk_SMLATT +static OPUS_INLINE opus_int32 silk_SMLATT(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + ops_count += 1; + ret = a32 + (b32 >> 16) * (c32 >> 16); + return ret; +} + + +/* multiply-accumulate macros that allow overflow in the addition (ie, no asserts in debug mode)*/ +#undef silk_MLA_ovflw +#define silk_MLA_ovflw silk_MLA + +#undef silk_SMLABB_ovflw +#define silk_SMLABB_ovflw silk_SMLABB + +#undef silk_SMLABT_ovflw +#define silk_SMLABT_ovflw silk_SMLABT + +#undef silk_SMLATT_ovflw +#define silk_SMLATT_ovflw silk_SMLATT + +#undef silk_SMLAWB_ovflw +#define silk_SMLAWB_ovflw silk_SMLAWB + +#undef silk_SMLAWT_ovflw +#define silk_SMLAWT_ovflw silk_SMLAWT + +#undef silk_SMULL +static OPUS_INLINE opus_int64 silk_SMULL(opus_int32 a32, opus_int32 b32){ + opus_int64 ret; + ops_count += 8; + ret = ((opus_int64)(a32) * /*(opus_int64)*/(b32)); + return ret; +} + +#undef silk_SMLAL +static OPUS_INLINE opus_int64 silk_SMLAL(opus_int64 a64, opus_int32 b32, opus_int32 c32){ + opus_int64 ret; + ops_count += 8; + ret = a64 + ((opus_int64)(b32) * /*(opus_int64)*/(c32)); + return ret; +} +#undef silk_SMLALBB +static OPUS_INLINE opus_int64 silk_SMLALBB(opus_int64 a64, opus_int16 b16, opus_int16 c16){ + opus_int64 ret; + ops_count += 4; + ret = a64 + ((opus_int64)(b16) * /*(opus_int64)*/(c16)); + return ret; +} + +#undef SigProcFIX_CLZ16 +static OPUS_INLINE opus_int32 SigProcFIX_CLZ16(opus_int16 in16) +{ + opus_int32 out32 = 0; + ops_count += 10; + if( in16 == 0 ) { + return 16; + } + /* test nibbles */ + if( in16 & 0xFF00 ) { + if( in16 & 0xF000 ) { + in16 >>= 12; + } else { + out32 += 4; + in16 >>= 8; + } + } else { + if( in16 & 0xFFF0 ) { + out32 += 8; + in16 >>= 4; + } else { + out32 += 12; + } + } + /* test bits and return */ + if( in16 & 0xC ) { + if( in16 & 0x8 ) + return out32 + 0; + else + return out32 + 1; + } else { + if( in16 & 0xE ) + return out32 + 2; + else + return out32 + 3; + } +} + +#undef SigProcFIX_CLZ32 +static OPUS_INLINE opus_int32 SigProcFIX_CLZ32(opus_int32 in32) +{ + /* test highest 16 bits and convert to opus_int16 */ + ops_count += 2; + if( in32 & 0xFFFF0000 ) { + return SigProcFIX_CLZ16((opus_int16)(in32 >> 16)); + } else { + return SigProcFIX_CLZ16((opus_int16)in32) + 16; + } +} + +#undef silk_DIV32 +static OPUS_INLINE opus_int32 silk_DIV32(opus_int32 a32, opus_int32 b32){ + ops_count += 64; + return a32 / b32; +} + +#undef silk_DIV32_16 +static OPUS_INLINE opus_int32 silk_DIV32_16(opus_int32 a32, opus_int32 b32){ + ops_count += 32; + return a32 / b32; +} + +#undef silk_SAT8 +static OPUS_INLINE opus_int8 silk_SAT8(opus_int64 a){ + opus_int8 tmp; + ops_count += 1; + tmp = (opus_int8)((a) > silk_int8_MAX ? silk_int8_MAX : \ + ((a) < silk_int8_MIN ? silk_int8_MIN : (a))); + return(tmp); +} + +#undef silk_SAT16 +static OPUS_INLINE opus_int16 silk_SAT16(opus_int64 a){ + opus_int16 tmp; + ops_count += 1; + tmp = (opus_int16)((a) > silk_int16_MAX ? silk_int16_MAX : \ + ((a) < silk_int16_MIN ? silk_int16_MIN : (a))); + return(tmp); +} +#undef silk_SAT32 +static OPUS_INLINE opus_int32 silk_SAT32(opus_int64 a){ + opus_int32 tmp; + ops_count += 1; + tmp = (opus_int32)((a) > silk_int32_MAX ? silk_int32_MAX : \ + ((a) < silk_int32_MIN ? silk_int32_MIN : (a))); + return(tmp); +} +#undef silk_POS_SAT32 +static OPUS_INLINE opus_int32 silk_POS_SAT32(opus_int64 a){ + opus_int32 tmp; + ops_count += 1; + tmp = (opus_int32)((a) > silk_int32_MAX ? silk_int32_MAX : (a)); + return(tmp); +} + +#undef silk_ADD_POS_SAT8 +static OPUS_INLINE opus_int8 silk_ADD_POS_SAT8(opus_int64 a, opus_int64 b){ + opus_int8 tmp; + ops_count += 1; + tmp = (opus_int8)((((a)+(b)) & 0x80) ? silk_int8_MAX : ((a)+(b))); + return(tmp); +} +#undef silk_ADD_POS_SAT16 +static OPUS_INLINE opus_int16 silk_ADD_POS_SAT16(opus_int64 a, opus_int64 b){ + opus_int16 tmp; + ops_count += 1; + tmp = (opus_int16)((((a)+(b)) & 0x8000) ? silk_int16_MAX : ((a)+(b))); + return(tmp); +} + +#undef silk_ADD_POS_SAT32 +static OPUS_INLINE opus_int32 silk_ADD_POS_SAT32(opus_int64 a, opus_int64 b){ + opus_int32 tmp; + ops_count += 1; + tmp = (opus_int32)((((a)+(b)) & 0x80000000) ? silk_int32_MAX : ((a)+(b))); + return(tmp); +} + +#undef silk_LSHIFT8 +static OPUS_INLINE opus_int8 silk_LSHIFT8(opus_int8 a, opus_int32 shift){ + opus_int8 ret; + ops_count += 1; + ret = a << shift; + return ret; +} +#undef silk_LSHIFT16 +static OPUS_INLINE opus_int16 silk_LSHIFT16(opus_int16 a, opus_int32 shift){ + opus_int16 ret; + ops_count += 1; + ret = a << shift; + return ret; +} +#undef silk_LSHIFT32 +static OPUS_INLINE opus_int32 silk_LSHIFT32(opus_int32 a, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a << shift; + return ret; +} +#undef silk_LSHIFT64 +static OPUS_INLINE opus_int64 silk_LSHIFT64(opus_int64 a, opus_int shift){ + ops_count += 1; + return a << shift; +} + +#undef silk_LSHIFT_ovflw +static OPUS_INLINE opus_int32 silk_LSHIFT_ovflw(opus_int32 a, opus_int32 shift){ + ops_count += 1; + return a << shift; +} + +#undef silk_LSHIFT_uint +static OPUS_INLINE opus_uint32 silk_LSHIFT_uint(opus_uint32 a, opus_int32 shift){ + opus_uint32 ret; + ops_count += 1; + ret = a << shift; + return ret; +} + +#undef silk_RSHIFT8 +static OPUS_INLINE opus_int8 silk_RSHIFT8(opus_int8 a, opus_int32 shift){ + ops_count += 1; + return a >> shift; +} +#undef silk_RSHIFT16 +static OPUS_INLINE opus_int16 silk_RSHIFT16(opus_int16 a, opus_int32 shift){ + ops_count += 1; + return a >> shift; +} +#undef silk_RSHIFT32 +static OPUS_INLINE opus_int32 silk_RSHIFT32(opus_int32 a, opus_int32 shift){ + ops_count += 1; + return a >> shift; +} +#undef silk_RSHIFT64 +static OPUS_INLINE opus_int64 silk_RSHIFT64(opus_int64 a, opus_int64 shift){ + ops_count += 1; + return a >> shift; +} + +#undef silk_RSHIFT_uint +static OPUS_INLINE opus_uint32 silk_RSHIFT_uint(opus_uint32 a, opus_int32 shift){ + ops_count += 1; + return a >> shift; +} + +#undef silk_ADD_LSHIFT +static OPUS_INLINE opus_int32 silk_ADD_LSHIFT(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a + (b << shift); + return ret; /* shift >= 0*/ +} +#undef silk_ADD_LSHIFT32 +static OPUS_INLINE opus_int32 silk_ADD_LSHIFT32(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a + (b << shift); + return ret; /* shift >= 0*/ +} +#undef silk_ADD_LSHIFT_uint +static OPUS_INLINE opus_uint32 silk_ADD_LSHIFT_uint(opus_uint32 a, opus_uint32 b, opus_int32 shift){ + opus_uint32 ret; + ops_count += 1; + ret = a + (b << shift); + return ret; /* shift >= 0*/ +} +#undef silk_ADD_RSHIFT +static OPUS_INLINE opus_int32 silk_ADD_RSHIFT(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a + (b >> shift); + return ret; /* shift > 0*/ +} +#undef silk_ADD_RSHIFT32 +static OPUS_INLINE opus_int32 silk_ADD_RSHIFT32(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a + (b >> shift); + return ret; /* shift > 0*/ +} +#undef silk_ADD_RSHIFT_uint +static OPUS_INLINE opus_uint32 silk_ADD_RSHIFT_uint(opus_uint32 a, opus_uint32 b, opus_int32 shift){ + opus_uint32 ret; + ops_count += 1; + ret = a + (b >> shift); + return ret; /* shift > 0*/ +} +#undef silk_SUB_LSHIFT32 +static OPUS_INLINE opus_int32 silk_SUB_LSHIFT32(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a - (b << shift); + return ret; /* shift >= 0*/ +} +#undef silk_SUB_RSHIFT32 +static OPUS_INLINE opus_int32 silk_SUB_RSHIFT32(opus_int32 a, opus_int32 b, opus_int32 shift){ + opus_int32 ret; + ops_count += 1; + ret = a - (b >> shift); + return ret; /* shift > 0*/ +} + +#undef silk_RSHIFT_ROUND +static OPUS_INLINE opus_int32 silk_RSHIFT_ROUND(opus_int32 a, opus_int32 shift){ + opus_int32 ret; + ops_count += 3; + ret = shift == 1 ? (a >> 1) + (a & 1) : ((a >> (shift - 1)) + 1) >> 1; + return ret; +} + +#undef silk_RSHIFT_ROUND64 +static OPUS_INLINE opus_int64 silk_RSHIFT_ROUND64(opus_int64 a, opus_int32 shift){ + opus_int64 ret; + ops_count += 6; + ret = shift == 1 ? (a >> 1) + (a & 1) : ((a >> (shift - 1)) + 1) >> 1; + return ret; +} + +#undef silk_abs_int64 +static OPUS_INLINE opus_int64 silk_abs_int64(opus_int64 a){ + ops_count += 1; + return (((a) > 0) ? (a) : -(a)); /* Be careful, silk_abs returns wrong when input equals to silk_intXX_MIN*/ +} + +#undef silk_abs_int32 +static OPUS_INLINE opus_int32 silk_abs_int32(opus_int32 a){ + ops_count += 1; + return silk_abs(a); +} + + +#undef silk_min +static silk_min(a, b){ + ops_count += 1; + return (((a) < (b)) ? (a) : (b)); +} +#undef silk_max +static silk_max(a, b){ + ops_count += 1; + return (((a) > (b)) ? (a) : (b)); +} +#undef silk_sign +static silk_sign(a){ + ops_count += 1; + return ((a) > 0 ? 1 : ( (a) < 0 ? -1 : 0 )); +} + +#undef silk_ADD16 +static OPUS_INLINE opus_int16 silk_ADD16(opus_int16 a, opus_int16 b){ + opus_int16 ret; + ops_count += 1; + ret = a + b; + return ret; +} + +#undef silk_ADD32 +static OPUS_INLINE opus_int32 silk_ADD32(opus_int32 a, opus_int32 b){ + opus_int32 ret; + ops_count += 1; + ret = a + b; + return ret; +} + +#undef silk_ADD64 +static OPUS_INLINE opus_int64 silk_ADD64(opus_int64 a, opus_int64 b){ + opus_int64 ret; + ops_count += 2; + ret = a + b; + return ret; +} + +#undef silk_SUB16 +static OPUS_INLINE opus_int16 silk_SUB16(opus_int16 a, opus_int16 b){ + opus_int16 ret; + ops_count += 1; + ret = a - b; + return ret; +} + +#undef silk_SUB32 +static OPUS_INLINE opus_int32 silk_SUB32(opus_int32 a, opus_int32 b){ + opus_int32 ret; + ops_count += 1; + ret = a - b; + return ret; +} + +#undef silk_SUB64 +static OPUS_INLINE opus_int64 silk_SUB64(opus_int64 a, opus_int64 b){ + opus_int64 ret; + ops_count += 2; + ret = a - b; + return ret; +} + +#undef silk_ADD_SAT16 +static OPUS_INLINE opus_int16 silk_ADD_SAT16( opus_int16 a16, opus_int16 b16 ) { + opus_int16 res; + /* Nb will be counted in AKP_add32 and silk_SAT16*/ + res = (opus_int16)silk_SAT16( silk_ADD32( (opus_int32)(a16), (b16) ) ); + return res; +} + +#undef silk_ADD_SAT32 +static OPUS_INLINE opus_int32 silk_ADD_SAT32(opus_int32 a32, opus_int32 b32){ + opus_int32 res; + ops_count += 1; + res = ((((a32) + (b32)) & 0x80000000) == 0 ? \ + ((((a32) & (b32)) & 0x80000000) != 0 ? silk_int32_MIN : (a32)+(b32)) : \ + ((((a32) | (b32)) & 0x80000000) == 0 ? silk_int32_MAX : (a32)+(b32)) ); + return res; +} + +#undef silk_ADD_SAT64 +static OPUS_INLINE opus_int64 silk_ADD_SAT64( opus_int64 a64, opus_int64 b64 ) { + opus_int64 res; + ops_count += 1; + res = ((((a64) + (b64)) & 0x8000000000000000LL) == 0 ? \ + ((((a64) & (b64)) & 0x8000000000000000LL) != 0 ? silk_int64_MIN : (a64)+(b64)) : \ + ((((a64) | (b64)) & 0x8000000000000000LL) == 0 ? silk_int64_MAX : (a64)+(b64)) ); + return res; +} + +#undef silk_SUB_SAT16 +static OPUS_INLINE opus_int16 silk_SUB_SAT16( opus_int16 a16, opus_int16 b16 ) { + opus_int16 res; + silk_assert(0); + /* Nb will be counted in sub-macros*/ + res = (opus_int16)silk_SAT16( silk_SUB32( (opus_int32)(a16), (b16) ) ); + return res; +} + +#undef silk_SUB_SAT32 +static OPUS_INLINE opus_int32 silk_SUB_SAT32( opus_int32 a32, opus_int32 b32 ) { + opus_int32 res; + ops_count += 1; + res = ((((a32)-(b32)) & 0x80000000) == 0 ? \ + (( (a32) & ((b32)^0x80000000) & 0x80000000) ? silk_int32_MIN : (a32)-(b32)) : \ + ((((a32)^0x80000000) & (b32) & 0x80000000) ? silk_int32_MAX : (a32)-(b32)) ); + return res; +} + +#undef silk_SUB_SAT64 +static OPUS_INLINE opus_int64 silk_SUB_SAT64( opus_int64 a64, opus_int64 b64 ) { + opus_int64 res; + ops_count += 1; + res = ((((a64)-(b64)) & 0x8000000000000000LL) == 0 ? \ + (( (a64) & ((b64)^0x8000000000000000LL) & 0x8000000000000000LL) ? silk_int64_MIN : (a64)-(b64)) : \ + ((((a64)^0x8000000000000000LL) & (b64) & 0x8000000000000000LL) ? silk_int64_MAX : (a64)-(b64)) ); + + return res; +} + +#undef silk_SMULWW +static OPUS_INLINE opus_int32 silk_SMULWW(opus_int32 a32, opus_int32 b32){ + opus_int32 ret; + /* Nb will be counted in sub-macros*/ + ret = silk_MLA(silk_SMULWB((a32), (b32)), (a32), silk_RSHIFT_ROUND((b32), 16)); + return ret; +} + +#undef silk_SMLAWW +static OPUS_INLINE opus_int32 silk_SMLAWW(opus_int32 a32, opus_int32 b32, opus_int32 c32){ + opus_int32 ret; + /* Nb will be counted in sub-macros*/ + ret = silk_MLA(silk_SMLAWB((a32), (b32), (c32)), (b32), silk_RSHIFT_ROUND((c32), 16)); + return ret; +} + +#undef silk_min_int +static OPUS_INLINE opus_int silk_min_int(opus_int a, opus_int b) +{ + ops_count += 1; + return (((a) < (b)) ? (a) : (b)); +} + +#undef silk_min_16 +static OPUS_INLINE opus_int16 silk_min_16(opus_int16 a, opus_int16 b) +{ + ops_count += 1; + return (((a) < (b)) ? (a) : (b)); +} +#undef silk_min_32 +static OPUS_INLINE opus_int32 silk_min_32(opus_int32 a, opus_int32 b) +{ + ops_count += 1; + return (((a) < (b)) ? (a) : (b)); +} +#undef silk_min_64 +static OPUS_INLINE opus_int64 silk_min_64(opus_int64 a, opus_int64 b) +{ + ops_count += 1; + return (((a) < (b)) ? (a) : (b)); +} + +/* silk_min() versions with typecast in the function call */ +#undef silk_max_int +static OPUS_INLINE opus_int silk_max_int(opus_int a, opus_int b) +{ + ops_count += 1; + return (((a) > (b)) ? (a) : (b)); +} +#undef silk_max_16 +static OPUS_INLINE opus_int16 silk_max_16(opus_int16 a, opus_int16 b) +{ + ops_count += 1; + return (((a) > (b)) ? (a) : (b)); +} +#undef silk_max_32 +static OPUS_INLINE opus_int32 silk_max_32(opus_int32 a, opus_int32 b) +{ + ops_count += 1; + return (((a) > (b)) ? (a) : (b)); +} + +#undef silk_max_64 +static OPUS_INLINE opus_int64 silk_max_64(opus_int64 a, opus_int64 b) +{ + ops_count += 1; + return (((a) > (b)) ? (a) : (b)); +} + + +#undef silk_LIMIT_int +static OPUS_INLINE opus_int silk_LIMIT_int(opus_int a, opus_int limit1, opus_int limit2) +{ + opus_int ret; + ops_count += 6; + + ret = ((limit1) > (limit2) ? ((a) > (limit1) ? (limit1) : ((a) < (limit2) ? (limit2) : (a))) \ + : ((a) > (limit2) ? (limit2) : ((a) < (limit1) ? (limit1) : (a)))); + + return(ret); +} + +#undef silk_LIMIT_16 +static OPUS_INLINE opus_int16 silk_LIMIT_16(opus_int16 a, opus_int16 limit1, opus_int16 limit2) +{ + opus_int16 ret; + ops_count += 6; + + ret = ((limit1) > (limit2) ? ((a) > (limit1) ? (limit1) : ((a) < (limit2) ? (limit2) : (a))) \ + : ((a) > (limit2) ? (limit2) : ((a) < (limit1) ? (limit1) : (a)))); + +return(ret); +} + + +#undef silk_LIMIT_32 +static OPUS_INLINE opus_int32 silk_LIMIT_32(opus_int32 a, opus_int32 limit1, opus_int32 limit2) +{ + opus_int32 ret; + ops_count += 6; + + ret = ((limit1) > (limit2) ? ((a) > (limit1) ? (limit1) : ((a) < (limit2) ? (limit2) : (a))) \ + : ((a) > (limit2) ? (limit2) : ((a) < (limit1) ? (limit1) : (a)))); + return(ret); +} + +#else +#define varDefine +#define silk_SaveCount() + +#endif +#endif + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroDebug.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroDebug.h new file mode 100644 index 000000000..8dd4ce2ee --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/MacroDebug.h @@ -0,0 +1,951 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Copyright (C) 2012 Xiph.Org Foundation +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef MACRO_DEBUG_H +#define MACRO_DEBUG_H + +/* Redefine macro functions with extensive assertion in DEBUG mode. + As functions can't be undefined, this file can't work with SigProcFIX_MacroCount.h */ + +#if ( defined (FIXED_DEBUG) || ( 0 && defined (_DEBUG) ) ) && !defined (silk_MACRO_COUNT) + +#undef silk_ADD16 +#define silk_ADD16(a,b) silk_ADD16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_ADD16_(opus_int16 a, opus_int16 b, char *file, int line){ + opus_int16 ret; + + ret = a + b; + if ( ret != silk_ADD_SAT16( a, b ) ) + { + fprintf (stderr, "silk_ADD16(%d, %d) in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_ADD32 +#define silk_ADD32(a,b) silk_ADD32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_ADD32_(opus_int32 a, opus_int32 b, char *file, int line){ + opus_int32 ret; + + ret = a + b; + if ( ret != silk_ADD_SAT32( a, b ) ) + { + fprintf (stderr, "silk_ADD32(%d, %d) in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_ADD64 +#define silk_ADD64(a,b) silk_ADD64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_ADD64_(opus_int64 a, opus_int64 b, char *file, int line){ + opus_int64 ret; + + ret = a + b; + if ( ret != silk_ADD_SAT64( a, b ) ) + { + fprintf (stderr, "silk_ADD64(%lld, %lld) in %s: line %d\n", (long long)a, (long long)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SUB16 +#define silk_SUB16(a,b) silk_SUB16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_SUB16_(opus_int16 a, opus_int16 b, char *file, int line){ + opus_int16 ret; + + ret = a - b; + if ( ret != silk_SUB_SAT16( a, b ) ) + { + fprintf (stderr, "silk_SUB16(%d, %d) in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SUB32 +#define silk_SUB32(a,b) silk_SUB32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SUB32_(opus_int32 a, opus_int32 b, char *file, int line){ + opus_int32 ret; + + ret = a - b; + if ( ret != silk_SUB_SAT32( a, b ) ) + { + fprintf (stderr, "silk_SUB32(%d, %d) in %s: line %d\n", a, b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SUB64 +#define silk_SUB64(a,b) silk_SUB64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_SUB64_(opus_int64 a, opus_int64 b, char *file, int line){ + opus_int64 ret; + + ret = a - b; + if ( ret != silk_SUB_SAT64( a, b ) ) + { + fprintf (stderr, "silk_SUB64(%lld, %lld) in %s: line %d\n", (long long)a, (long long)b, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_ADD_SAT16 +#define silk_ADD_SAT16(a,b) silk_ADD_SAT16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_ADD_SAT16_( opus_int16 a16, opus_int16 b16, char *file, int line) { + opus_int16 res; + res = (opus_int16)silk_SAT16( silk_ADD32( (opus_int32)(a16), (b16) ) ); + if ( res != silk_SAT16( (opus_int32)a16 + (opus_int32)b16 ) ) + { + fprintf (stderr, "silk_ADD_SAT16(%d, %d) in %s: line %d\n", a16, b16, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_ADD_SAT32 +#define silk_ADD_SAT32(a,b) silk_ADD_SAT32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_ADD_SAT32_(opus_int32 a32, opus_int32 b32, char *file, int line){ + opus_int32 res; + res = ((((opus_uint32)(a32) + (opus_uint32)(b32)) & 0x80000000) == 0 ? \ + ((((a32) & (b32)) & 0x80000000) != 0 ? silk_int32_MIN : (a32)+(b32)) : \ + ((((a32) | (b32)) & 0x80000000) == 0 ? silk_int32_MAX : (a32)+(b32)) ); + if ( res != silk_SAT32( (opus_int64)a32 + (opus_int64)b32 ) ) + { + fprintf (stderr, "silk_ADD_SAT32(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_ADD_SAT64 +#define silk_ADD_SAT64(a,b) silk_ADD_SAT64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_ADD_SAT64_( opus_int64 a64, opus_int64 b64, char *file, int line) { + opus_int64 res; + int fail = 0; + res = ((((a64) + (b64)) & 0x8000000000000000LL) == 0 ? \ + ((((a64) & (b64)) & 0x8000000000000000LL) != 0 ? silk_int64_MIN : (a64)+(b64)) : \ + ((((a64) | (b64)) & 0x8000000000000000LL) == 0 ? silk_int64_MAX : (a64)+(b64)) ); + if( res != a64 + b64 ) { + /* Check that we saturated to the correct extreme value */ + if ( !(( res == silk_int64_MAX && ( ( a64 >> 1 ) + ( b64 >> 1 ) > ( silk_int64_MAX >> 3 ) ) ) || + ( res == silk_int64_MIN && ( ( a64 >> 1 ) + ( b64 >> 1 ) < ( silk_int64_MIN >> 3 ) ) ) ) ) + { + fail = 1; + } + } else { + /* Saturation not necessary */ + fail = res != a64 + b64; + } + if ( fail ) + { + fprintf (stderr, "silk_ADD_SAT64(%lld, %lld) in %s: line %d\n", (long long)a64, (long long)b64, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_SUB_SAT16 +#define silk_SUB_SAT16(a,b) silk_SUB_SAT16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_SUB_SAT16_( opus_int16 a16, opus_int16 b16, char *file, int line ) { + opus_int16 res; + res = (opus_int16)silk_SAT16( silk_SUB32( (opus_int32)(a16), (b16) ) ); + if ( res != silk_SAT16( (opus_int32)a16 - (opus_int32)b16 ) ) + { + fprintf (stderr, "silk_SUB_SAT16(%d, %d) in %s: line %d\n", a16, b16, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_SUB_SAT32 +#define silk_SUB_SAT32(a,b) silk_SUB_SAT32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SUB_SAT32_( opus_int32 a32, opus_int32 b32, char *file, int line ) { + opus_int32 res; + res = ((((opus_uint32)(a32)-(opus_uint32)(b32)) & 0x80000000) == 0 ? \ + (( (a32) & ((b32)^0x80000000) & 0x80000000) ? silk_int32_MIN : (a32)-(b32)) : \ + ((((a32)^0x80000000) & (b32) & 0x80000000) ? silk_int32_MAX : (a32)-(b32)) ); + if ( res != silk_SAT32( (opus_int64)a32 - (opus_int64)b32 ) ) + { + fprintf (stderr, "silk_SUB_SAT32(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_SUB_SAT64 +#define silk_SUB_SAT64(a,b) silk_SUB_SAT64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_SUB_SAT64_( opus_int64 a64, opus_int64 b64, char *file, int line ) { + opus_int64 res; + int fail = 0; + res = ((((a64)-(b64)) & 0x8000000000000000LL) == 0 ? \ + (( (a64) & ((b64)^0x8000000000000000LL) & 0x8000000000000000LL) ? silk_int64_MIN : (a64)-(b64)) : \ + ((((a64)^0x8000000000000000LL) & (b64) & 0x8000000000000000LL) ? silk_int64_MAX : (a64)-(b64)) ); + if( res != a64 - b64 ) { + /* Check that we saturated to the correct extreme value */ + if( !(( res == silk_int64_MAX && ( ( a64 >> 1 ) + ( b64 >> 1 ) > ( silk_int64_MAX >> 3 ) ) ) || + ( res == silk_int64_MIN && ( ( a64 >> 1 ) + ( b64 >> 1 ) < ( silk_int64_MIN >> 3 ) ) ) )) + { + fail = 1; + } + } else { + /* Saturation not necessary */ + fail = res != a64 - b64; + } + if ( fail ) + { + fprintf (stderr, "silk_SUB_SAT64(%lld, %lld) in %s: line %d\n", (long long)a64, (long long)b64, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return res; +} + +#undef silk_MUL +#define silk_MUL(a,b) silk_MUL_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_MUL_(opus_int32 a32, opus_int32 b32, char *file, int line){ + opus_int32 ret; + opus_int64 ret64; + ret = a32 * b32; + ret64 = (opus_int64)a32 * (opus_int64)b32; + if ( (opus_int64)ret != ret64 ) + { + fprintf (stderr, "silk_MUL(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_MUL_uint +#define silk_MUL_uint(a,b) silk_MUL_uint_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_uint32 silk_MUL_uint_(opus_uint32 a32, opus_uint32 b32, char *file, int line){ + opus_uint32 ret; + ret = a32 * b32; + if ( (opus_uint64)ret != (opus_uint64)a32 * (opus_uint64)b32 ) + { + fprintf (stderr, "silk_MUL_uint(%u, %u) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_MLA +#define silk_MLA(a,b,c) silk_MLA_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_MLA_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = a32 + b32 * c32; + if ( (opus_int64)ret != (opus_int64)a32 + (opus_int64)b32 * (opus_int64)c32 ) + { + fprintf (stderr, "silk_MLA(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_MLA_uint +#define silk_MLA_uint(a,b,c) silk_MLA_uint_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_MLA_uint_(opus_uint32 a32, opus_uint32 b32, opus_uint32 c32, char *file, int line){ + opus_uint32 ret; + ret = a32 + b32 * c32; + if ( (opus_int64)ret != (opus_int64)a32 + (opus_int64)b32 * (opus_int64)c32 ) + { + fprintf (stderr, "silk_MLA_uint(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMULWB +#define silk_SMULWB(a,b) silk_SMULWB_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMULWB_(opus_int32 a32, opus_int32 b32, char *file, int line){ + opus_int32 ret; + ret = (a32 >> 16) * (opus_int32)((opus_int16)b32) + (((a32 & 0x0000FFFF) * (opus_int32)((opus_int16)b32)) >> 16); + if ( (opus_int64)ret != ((opus_int64)a32 * (opus_int16)b32) >> 16 ) + { + fprintf (stderr, "silk_SMULWB(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMLAWB +#define silk_SMLAWB(a,b,c) silk_SMLAWB_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLAWB_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = silk_ADD32( a32, silk_SMULWB( b32, c32 ) ); + if ( silk_ADD32( a32, silk_SMULWB( b32, c32 ) ) != silk_ADD_SAT32( a32, silk_SMULWB( b32, c32 ) ) ) + { + fprintf (stderr, "silk_SMLAWB(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMULWT +#define silk_SMULWT(a,b) silk_SMULWT_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMULWT_(opus_int32 a32, opus_int32 b32, char *file, int line){ + opus_int32 ret; + ret = (a32 >> 16) * (b32 >> 16) + (((a32 & 0x0000FFFF) * (b32 >> 16)) >> 16); + if ( (opus_int64)ret != ((opus_int64)a32 * (b32 >> 16)) >> 16 ) + { + fprintf (stderr, "silk_SMULWT(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMLAWT +#define silk_SMLAWT(a,b,c) silk_SMLAWT_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLAWT_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = a32 + ((b32 >> 16) * (c32 >> 16)) + (((b32 & 0x0000FFFF) * ((c32 >> 16)) >> 16)); + if ( (opus_int64)ret != (opus_int64)a32 + (((opus_int64)b32 * (c32 >> 16)) >> 16) ) + { + fprintf (stderr, "silk_SMLAWT(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMULL +#define silk_SMULL(a,b) silk_SMULL_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_SMULL_(opus_int64 a64, opus_int64 b64, char *file, int line){ + opus_int64 ret64; + int fail = 0; + ret64 = a64 * b64; + if( b64 != 0 ) { + fail = a64 != (ret64 / b64); + } else if( a64 != 0 ) { + fail = b64 != (ret64 / a64); + } + if ( fail ) + { + fprintf (stderr, "silk_SMULL(%lld, %lld) in %s: line %d\n", (long long)a64, (long long)b64, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret64; +} + +/* no checking needed for silk_SMULBB */ +#undef silk_SMLABB +#define silk_SMLABB(a,b,c) silk_SMLABB_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLABB_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = a32 + (opus_int32)((opus_int16)b32) * (opus_int32)((opus_int16)c32); + if ( (opus_int64)ret != (opus_int64)a32 + (opus_int64)b32 * (opus_int16)c32 ) + { + fprintf (stderr, "silk_SMLABB(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +/* no checking needed for silk_SMULBT */ +#undef silk_SMLABT +#define silk_SMLABT(a,b,c) silk_SMLABT_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLABT_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = a32 + ((opus_int32)((opus_int16)b32)) * (c32 >> 16); + if ( (opus_int64)ret != (opus_int64)a32 + (opus_int64)b32 * (c32 >> 16) ) + { + fprintf (stderr, "silk_SMLABT(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +/* no checking needed for silk_SMULTT */ +#undef silk_SMLATT +#define silk_SMLATT(a,b,c) silk_SMLATT_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLATT_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret; + ret = a32 + (b32 >> 16) * (c32 >> 16); + if ( (opus_int64)ret != (opus_int64)a32 + (b32 >> 16) * (c32 >> 16) ) + { + fprintf (stderr, "silk_SMLATT(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_SMULWW +#define silk_SMULWW(a,b) silk_SMULWW_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMULWW_(opus_int32 a32, opus_int32 b32, char *file, int line){ + opus_int32 ret, tmp1, tmp2; + opus_int64 ret64; + int fail = 0; + + ret = silk_SMULWB( a32, b32 ); + tmp1 = silk_RSHIFT_ROUND( b32, 16 ); + tmp2 = silk_MUL( a32, tmp1 ); + + fail |= (opus_int64)tmp2 != (opus_int64) a32 * (opus_int64) tmp1; + + tmp1 = ret; + ret = silk_ADD32( tmp1, tmp2 ); + fail |= silk_ADD32( tmp1, tmp2 ) != silk_ADD_SAT32( tmp1, tmp2 ); + + ret64 = silk_RSHIFT64( silk_SMULL( a32, b32 ), 16 ); + fail |= (opus_int64)ret != ret64; + + if ( fail ) + { + fprintf (stderr, "silk_SMULWT(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + + return ret; +} + +#undef silk_SMLAWW +#define silk_SMLAWW(a,b,c) silk_SMLAWW_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SMLAWW_(opus_int32 a32, opus_int32 b32, opus_int32 c32, char *file, int line){ + opus_int32 ret, tmp; + + tmp = silk_SMULWW( b32, c32 ); + ret = silk_ADD32( a32, tmp ); + if ( ret != silk_ADD_SAT32( a32, tmp ) ) + { + fprintf (stderr, "silk_SMLAWW(%d, %d, %d) in %s: line %d\n", a32, b32, c32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +/* Multiply-accumulate macros that allow overflow in the addition (ie, no asserts in debug mode) */ +#undef silk_MLA_ovflw +#define silk_MLA_ovflw(a32, b32, c32) ((a32) + ((b32) * (c32))) +#undef silk_SMLABB_ovflw +#define silk_SMLABB_ovflw(a32, b32, c32) ((a32) + ((opus_int32)((opus_int16)(b32))) * (opus_int32)((opus_int16)(c32))) + +/* no checking needed for silk_SMULL + no checking needed for silk_SMLAL + no checking needed for silk_SMLALBB + no checking needed for SigProcFIX_CLZ16 + no checking needed for SigProcFIX_CLZ32*/ + +#undef silk_DIV32 +#define silk_DIV32(a,b) silk_DIV32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_DIV32_(opus_int32 a32, opus_int32 b32, char *file, int line){ + if ( b32 == 0 ) + { + fprintf (stderr, "silk_DIV32(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a32 / b32; +} + +#undef silk_DIV32_16 +#define silk_DIV32_16(a,b) silk_DIV32_16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_DIV32_16_(opus_int32 a32, opus_int32 b32, char *file, int line){ + int fail = 0; + fail |= b32 == 0; + fail |= b32 > silk_int16_MAX; + fail |= b32 < silk_int16_MIN; + if ( fail ) + { + fprintf (stderr, "silk_DIV32_16(%d, %d) in %s: line %d\n", a32, b32, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a32 / b32; +} + +/* no checking needed for silk_SAT8 + no checking needed for silk_SAT16 + no checking needed for silk_SAT32 + no checking needed for silk_POS_SAT32 + no checking needed for silk_ADD_POS_SAT8 + no checking needed for silk_ADD_POS_SAT16 + no checking needed for silk_ADD_POS_SAT32 */ + +#undef silk_LSHIFT8 +#define silk_LSHIFT8(a,b) silk_LSHIFT8_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int8 silk_LSHIFT8_(opus_int8 a, opus_int32 shift, char *file, int line){ + opus_int8 ret; + int fail = 0; + ret = a << shift; + fail |= shift < 0; + fail |= shift >= 8; + fail |= (opus_int64)ret != ((opus_int64)a) << shift; + if ( fail ) + { + fprintf (stderr, "silk_LSHIFT8(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_LSHIFT16 +#define silk_LSHIFT16(a,b) silk_LSHIFT16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_LSHIFT16_(opus_int16 a, opus_int32 shift, char *file, int line){ + opus_int16 ret; + int fail = 0; + ret = a << shift; + fail |= shift < 0; + fail |= shift >= 16; + fail |= (opus_int64)ret != ((opus_int64)a) << shift; + if ( fail ) + { + fprintf (stderr, "silk_LSHIFT16(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_LSHIFT32 +#define silk_LSHIFT32(a,b) silk_LSHIFT32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_LSHIFT32_(opus_int32 a, opus_int32 shift, char *file, int line){ + opus_int32 ret; + int fail = 0; + ret = a << shift; + fail |= shift < 0; + fail |= shift >= 32; + fail |= (opus_int64)ret != ((opus_int64)a) << shift; + if ( fail ) + { + fprintf (stderr, "silk_LSHIFT32(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_LSHIFT64 +#define silk_LSHIFT64(a,b) silk_LSHIFT64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_LSHIFT64_(opus_int64 a, opus_int shift, char *file, int line){ + opus_int64 ret; + int fail = 0; + ret = a << shift; + fail |= shift < 0; + fail |= shift >= 64; + fail |= (ret>>shift) != ((opus_int64)a); + if ( fail ) + { + fprintf (stderr, "silk_LSHIFT64(%lld, %d) in %s: line %d\n", (long long)a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_LSHIFT_ovflw +#define silk_LSHIFT_ovflw(a,b) silk_LSHIFT_ovflw_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_LSHIFT_ovflw_(opus_int32 a, opus_int32 shift, char *file, int line){ + if ( (shift < 0) || (shift >= 32) ) /* no check for overflow */ + { + fprintf (stderr, "silk_LSHIFT_ovflw(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a << shift; +} + +#undef silk_LSHIFT_uint +#define silk_LSHIFT_uint(a,b) silk_LSHIFT_uint_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_uint32 silk_LSHIFT_uint_(opus_uint32 a, opus_int32 shift, char *file, int line){ + opus_uint32 ret; + ret = a << shift; + if ( (shift < 0) || ((opus_int64)ret != ((opus_int64)a) << shift)) + { + fprintf (stderr, "silk_LSHIFT_uint(%u, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_RSHIFT8 +#define silk_RSHITF8(a,b) silk_RSHIFT8_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int8 silk_RSHIFT8_(opus_int8 a, opus_int32 shift, char *file, int line){ + if ( (shift < 0) || (shift>=8) ) + { + fprintf (stderr, "silk_RSHITF8(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a >> shift; +} + +#undef silk_RSHIFT16 +#define silk_RSHITF16(a,b) silk_RSHIFT16_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_RSHIFT16_(opus_int16 a, opus_int32 shift, char *file, int line){ + if ( (shift < 0) || (shift>=16) ) + { + fprintf (stderr, "silk_RSHITF16(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a >> shift; +} + +#undef silk_RSHIFT32 +#define silk_RSHIFT32(a,b) silk_RSHIFT32_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_RSHIFT32_(opus_int32 a, opus_int32 shift, char *file, int line){ + if ( (shift < 0) || (shift>=32) ) + { + fprintf (stderr, "silk_RSHITF32(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a >> shift; +} + +#undef silk_RSHIFT64 +#define silk_RSHIFT64(a,b) silk_RSHIFT64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_RSHIFT64_(opus_int64 a, opus_int64 shift, char *file, int line){ + if ( (shift < 0) || (shift>=64) ) + { + fprintf (stderr, "silk_RSHITF64(%lld, %lld) in %s: line %d\n", (long long)a, (long long)shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a >> shift; +} + +#undef silk_RSHIFT_uint +#define silk_RSHIFT_uint(a,b) silk_RSHIFT_uint_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_uint32 silk_RSHIFT_uint_(opus_uint32 a, opus_int32 shift, char *file, int line){ + if ( (shift < 0) || (shift>32) ) + { + fprintf (stderr, "silk_RSHIFT_uint(%u, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return a >> shift; +} + +#undef silk_ADD_LSHIFT +#define silk_ADD_LSHIFT(a,b,c) silk_ADD_LSHIFT_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE int silk_ADD_LSHIFT_(int a, int b, int shift, char *file, int line){ + opus_int16 ret; + ret = a + (b << shift); + if ( (shift < 0) || (shift>15) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) << shift)) ) + { + fprintf (stderr, "silk_ADD_LSHIFT(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift >= 0 */ +} + +#undef silk_ADD_LSHIFT32 +#define silk_ADD_LSHIFT32(a,b,c) silk_ADD_LSHIFT32_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_ADD_LSHIFT32_(opus_int32 a, opus_int32 b, opus_int32 shift, char *file, int line){ + opus_int32 ret; + ret = a + (b << shift); + if ( (shift < 0) || (shift>31) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) << shift)) ) + { + fprintf (stderr, "silk_ADD_LSHIFT32(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift >= 0 */ +} + +#undef silk_ADD_LSHIFT_uint +#define silk_ADD_LSHIFT_uint(a,b,c) silk_ADD_LSHIFT_uint_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_uint32 silk_ADD_LSHIFT_uint_(opus_uint32 a, opus_uint32 b, opus_int32 shift, char *file, int line){ + opus_uint32 ret; + ret = a + (b << shift); + if ( (shift < 0) || (shift>32) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) << shift)) ) + { + fprintf (stderr, "silk_ADD_LSHIFT_uint(%u, %u, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift >= 0 */ +} + +#undef silk_ADD_RSHIFT +#define silk_ADD_RSHIFT(a,b,c) silk_ADD_RSHIFT_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE int silk_ADD_RSHIFT_(int a, int b, int shift, char *file, int line){ + opus_int16 ret; + ret = a + (b >> shift); + if ( (shift < 0) || (shift>15) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) >> shift)) ) + { + fprintf (stderr, "silk_ADD_RSHIFT(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift > 0 */ +} + +#undef silk_ADD_RSHIFT32 +#define silk_ADD_RSHIFT32(a,b,c) silk_ADD_RSHIFT32_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_ADD_RSHIFT32_(opus_int32 a, opus_int32 b, opus_int32 shift, char *file, int line){ + opus_int32 ret; + ret = a + (b >> shift); + if ( (shift < 0) || (shift>31) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) >> shift)) ) + { + fprintf (stderr, "silk_ADD_RSHIFT32(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift > 0 */ +} + +#undef silk_ADD_RSHIFT_uint +#define silk_ADD_RSHIFT_uint(a,b,c) silk_ADD_RSHIFT_uint_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_uint32 silk_ADD_RSHIFT_uint_(opus_uint32 a, opus_uint32 b, opus_int32 shift, char *file, int line){ + opus_uint32 ret; + ret = a + (b >> shift); + if ( (shift < 0) || (shift>32) || ((opus_int64)ret != (opus_int64)a + (((opus_int64)b) >> shift)) ) + { + fprintf (stderr, "silk_ADD_RSHIFT_uint(%u, %u, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift > 0 */ +} + +#undef silk_SUB_LSHIFT32 +#define silk_SUB_LSHIFT32(a,b,c) silk_SUB_LSHIFT32_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SUB_LSHIFT32_(opus_int32 a, opus_int32 b, opus_int32 shift, char *file, int line){ + opus_int32 ret; + ret = a - (b << shift); + if ( (shift < 0) || (shift>31) || ((opus_int64)ret != (opus_int64)a - (((opus_int64)b) << shift)) ) + { + fprintf (stderr, "silk_SUB_LSHIFT32(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift >= 0 */ +} + +#undef silk_SUB_RSHIFT32 +#define silk_SUB_RSHIFT32(a,b,c) silk_SUB_RSHIFT32_((a), (b), (c), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_SUB_RSHIFT32_(opus_int32 a, opus_int32 b, opus_int32 shift, char *file, int line){ + opus_int32 ret; + ret = a - (b >> shift); + if ( (shift < 0) || (shift>31) || ((opus_int64)ret != (opus_int64)a - (((opus_int64)b) >> shift)) ) + { + fprintf (stderr, "silk_SUB_RSHIFT32(%d, %d, %d) in %s: line %d\n", a, b, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; /* shift > 0 */ +} + +#undef silk_RSHIFT_ROUND +#define silk_RSHIFT_ROUND(a,b) silk_RSHIFT_ROUND_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_RSHIFT_ROUND_(opus_int32 a, opus_int32 shift, char *file, int line){ + opus_int32 ret; + ret = shift == 1 ? (a >> 1) + (a & 1) : ((a >> (shift - 1)) + 1) >> 1; + /* the marco definition can't handle a shift of zero */ + if ( (shift <= 0) || (shift>31) || ((opus_int64)ret != ((opus_int64)a + ((opus_int64)1 << (shift - 1))) >> shift) ) + { + fprintf (stderr, "silk_RSHIFT_ROUND(%d, %d) in %s: line %d\n", a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return ret; +} + +#undef silk_RSHIFT_ROUND64 +#define silk_RSHIFT_ROUND64(a,b) silk_RSHIFT_ROUND64_((a), (b), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_RSHIFT_ROUND64_(opus_int64 a, opus_int32 shift, char *file, int line){ + opus_int64 ret; + /* the marco definition can't handle a shift of zero */ + if ( (shift <= 0) || (shift>=64) ) + { + fprintf (stderr, "silk_RSHIFT_ROUND64(%lld, %d) in %s: line %d\n", (long long)a, shift, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + ret = shift == 1 ? (a >> 1) + (a & 1) : ((a >> (shift - 1)) + 1) >> 1; + return ret; +} + +/* silk_abs is used on floats also, so doesn't work... */ +/*#undef silk_abs +static OPUS_INLINE opus_int32 silk_abs(opus_int32 a){ + silk_assert(a != 0x80000000); + return (((a) > 0) ? (a) : -(a)); // Be careful, silk_abs returns wrong when input equals to silk_intXX_MIN +}*/ + +#undef silk_abs_int64 +#define silk_abs_int64(a) silk_abs_int64_((a), __FILE__, __LINE__) +static OPUS_INLINE opus_int64 silk_abs_int64_(opus_int64 a, char *file, int line){ + if ( a == silk_int64_MIN ) + { + fprintf (stderr, "silk_abs_int64(%lld) in %s: line %d\n", (long long)a, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return (((a) > 0) ? (a) : -(a)); /* Be careful, silk_abs returns wrong when input equals to silk_intXX_MIN */ +} + +#undef silk_abs_int32 +#define silk_abs_int32(a) silk_abs_int32_((a), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_abs_int32_(opus_int32 a, char *file, int line){ + if ( a == silk_int32_MIN ) + { + fprintf (stderr, "silk_abs_int32(%d) in %s: line %d\n", a, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return silk_abs(a); +} + +#undef silk_CHECK_FIT8 +#define silk_CHECK_FIT8(a) silk_CHECK_FIT8_((a), __FILE__, __LINE__) +static OPUS_INLINE opus_int8 silk_CHECK_FIT8_( opus_int64 a, char *file, int line ){ + opus_int8 ret; + ret = (opus_int8)a; + if ( (opus_int64)ret != a ) + { + fprintf (stderr, "silk_CHECK_FIT8(%lld) in %s: line %d\n", (long long)a, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return( ret ); +} + +#undef silk_CHECK_FIT16 +#define silk_CHECK_FIT16(a) silk_CHECK_FIT16_((a), __FILE__, __LINE__) +static OPUS_INLINE opus_int16 silk_CHECK_FIT16_( opus_int64 a, char *file, int line ){ + opus_int16 ret; + ret = (opus_int16)a; + if ( (opus_int64)ret != a ) + { + fprintf (stderr, "silk_CHECK_FIT16(%lld) in %s: line %d\n", (long long)a, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return( ret ); +} + +#undef silk_CHECK_FIT32 +#define silk_CHECK_FIT32(a) silk_CHECK_FIT32_((a), __FILE__, __LINE__) +static OPUS_INLINE opus_int32 silk_CHECK_FIT32_( opus_int64 a, char *file, int line ){ + opus_int32 ret; + ret = (opus_int32)a; + if ( (opus_int64)ret != a ) + { + fprintf (stderr, "silk_CHECK_FIT32(%lld) in %s: line %d\n", (long long)a, file, line); +#ifdef FIXED_DEBUG_ASSERT + silk_assert( 0 ); +#endif + } + return( ret ); +} + +/* no checking for silk_NSHIFT_MUL_32_32 + no checking for silk_NSHIFT_MUL_16_16 + no checking needed for silk_min + no checking needed for silk_max + no checking needed for silk_sign +*/ + +#endif +#endif /* MACRO_DEBUG_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF2A.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF2A.c new file mode 100644 index 000000000..d5b773063 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF2A.c @@ -0,0 +1,141 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* conversion between prediction filter coefficients and LSFs */ +/* order should be even */ +/* a piecewise linear approximation maps LSF <-> cos(LSF) */ +/* therefore the result is not accurate LSFs, but the two */ +/* functions are accurate inverses of each other */ + +#include "SigProc_FIX.h" +#include "tables.h" + +#define QA 16 + +/* helper function for NLSF2A(..) */ +static OPUS_INLINE void silk_NLSF2A_find_poly( + opus_int32 *out, /* O intermediate polynomial, QA [dd+1] */ + const opus_int32 *cLSF, /* I vector of interleaved 2*cos(LSFs), QA [d] */ + opus_int dd /* I polynomial order (= 1/2 * filter order) */ +) +{ + opus_int k, n; + opus_int32 ftmp; + + out[0] = silk_LSHIFT( 1, QA ); + out[1] = -cLSF[0]; + for( k = 1; k < dd; k++ ) { + ftmp = cLSF[2*k]; /* QA*/ + out[k+1] = silk_LSHIFT( out[k-1], 1 ) - (opus_int32)silk_RSHIFT_ROUND64( silk_SMULL( ftmp, out[k] ), QA ); + for( n = k; n > 1; n-- ) { + out[n] += out[n-2] - (opus_int32)silk_RSHIFT_ROUND64( silk_SMULL( ftmp, out[n-1] ), QA ); + } + out[1] -= ftmp; + } +} + +/* compute whitening filter coefficients from normalized line spectral frequencies */ +void silk_NLSF2A( + opus_int16 *a_Q12, /* O monic whitening filter coefficients in Q12, [ d ] */ + const opus_int16 *NLSF, /* I normalized line spectral frequencies in Q15, [ d ] */ + const opus_int d, /* I filter order (should be even) */ + int arch /* I Run-time architecture */ +) +{ + /* This ordering was found to maximize quality. It improves numerical accuracy of + silk_NLSF2A_find_poly() compared to "standard" ordering. */ + static const unsigned char ordering16[16] = { + 0, 15, 8, 7, 4, 11, 12, 3, 2, 13, 10, 5, 6, 9, 14, 1 + }; + static const unsigned char ordering10[10] = { + 0, 9, 6, 3, 4, 5, 8, 1, 2, 7 + }; + const unsigned char *ordering; + opus_int k, i, dd; + opus_int32 cos_LSF_QA[ SILK_MAX_ORDER_LPC ]; + opus_int32 P[ SILK_MAX_ORDER_LPC / 2 + 1 ], Q[ SILK_MAX_ORDER_LPC / 2 + 1 ]; + opus_int32 Ptmp, Qtmp, f_int, f_frac, cos_val, delta; + opus_int32 a32_QA1[ SILK_MAX_ORDER_LPC ]; + + silk_assert( LSF_COS_TAB_SZ_FIX == 128 ); + celt_assert( d==10 || d==16 ); + + /* convert LSFs to 2*cos(LSF), using piecewise linear curve from table */ + ordering = d == 16 ? ordering16 : ordering10; + for( k = 0; k < d; k++ ) { + silk_assert( NLSF[k] >= 0 ); + + /* f_int on a scale 0-127 (rounded down) */ + f_int = silk_RSHIFT( NLSF[k], 15 - 7 ); + + /* f_frac, range: 0..255 */ + f_frac = NLSF[k] - silk_LSHIFT( f_int, 15 - 7 ); + + silk_assert(f_int >= 0); + silk_assert(f_int < LSF_COS_TAB_SZ_FIX ); + + /* Read start and end value from table */ + cos_val = silk_LSFCosTab_FIX_Q12[ f_int ]; /* Q12 */ + delta = silk_LSFCosTab_FIX_Q12[ f_int + 1 ] - cos_val; /* Q12, with a range of 0..200 */ + + /* Linear interpolation */ + cos_LSF_QA[ordering[k]] = silk_RSHIFT_ROUND( silk_LSHIFT( cos_val, 8 ) + silk_MUL( delta, f_frac ), 20 - QA ); /* QA */ + } + + dd = silk_RSHIFT( d, 1 ); + + /* generate even and odd polynomials using convolution */ + silk_NLSF2A_find_poly( P, &cos_LSF_QA[ 0 ], dd ); + silk_NLSF2A_find_poly( Q, &cos_LSF_QA[ 1 ], dd ); + + /* convert even and odd polynomials to opus_int32 Q12 filter coefs */ + for( k = 0; k < dd; k++ ) { + Ptmp = P[ k+1 ] + P[ k ]; + Qtmp = Q[ k+1 ] - Q[ k ]; + + /* the Ptmp and Qtmp values at this stage need to fit in int32 */ + a32_QA1[ k ] = -Qtmp - Ptmp; /* QA+1 */ + a32_QA1[ d-k-1 ] = Qtmp - Ptmp; /* QA+1 */ + } + + /* Convert int32 coefficients to Q12 int16 coefs */ + silk_LPC_fit( a_Q12, a32_QA1, 12, QA + 1, d ); + + for( i = 0; silk_LPC_inverse_pred_gain( a_Q12, d, arch ) == 0 && i < MAX_LPC_STABILIZE_ITERATIONS; i++ ) { + /* Prediction coefficients are (too close to) unstable; apply bandwidth expansion */ + /* on the unscaled coefficients, convert to Q12 and measure again */ + silk_bwexpander_32( a32_QA1, d, 65536 - silk_LSHIFT( 2, i ) ); + for( k = 0; k < d; k++ ) { + a_Q12[ k ] = (opus_int16)silk_RSHIFT_ROUND( a32_QA1[ k ], QA + 1 - 12 ); /* QA+1 -> Q12 */ + } + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ.c new file mode 100644 index 000000000..b83182a79 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ.c @@ -0,0 +1,76 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Compute quantization errors for an LPC_order element input vector for a VQ codebook */ +void silk_NLSF_VQ( + opus_int32 err_Q24[], /* O Quantization errors [K] */ + const opus_int16 in_Q15[], /* I Input vectors to be quantized [LPC_order] */ + const opus_uint8 pCB_Q8[], /* I Codebook vectors [K*LPC_order] */ + const opus_int16 pWght_Q9[], /* I Codebook weights [K*LPC_order] */ + const opus_int K, /* I Number of codebook vectors */ + const opus_int LPC_order /* I Number of LPCs */ +) +{ + opus_int i, m; + opus_int32 diff_Q15, diffw_Q24, sum_error_Q24, pred_Q24; + const opus_int16 *w_Q9_ptr; + const opus_uint8 *cb_Q8_ptr; + + celt_assert( ( LPC_order & 1 ) == 0 ); + + /* Loop over codebook */ + cb_Q8_ptr = pCB_Q8; + w_Q9_ptr = pWght_Q9; + for( i = 0; i < K; i++ ) { + sum_error_Q24 = 0; + pred_Q24 = 0; + for( m = LPC_order-2; m >= 0; m -= 2 ) { + /* Compute weighted absolute predictive quantization error for index m + 1 */ + diff_Q15 = silk_SUB_LSHIFT32( in_Q15[ m + 1 ], (opus_int32)cb_Q8_ptr[ m + 1 ], 7 ); /* range: [ -32767 : 32767 ]*/ + diffw_Q24 = silk_SMULBB( diff_Q15, w_Q9_ptr[ m + 1 ] ); + sum_error_Q24 = silk_ADD32( sum_error_Q24, silk_abs( silk_SUB_RSHIFT32( diffw_Q24, pred_Q24, 1 ) ) ); + pred_Q24 = diffw_Q24; + + /* Compute weighted absolute predictive quantization error for index m */ + diff_Q15 = silk_SUB_LSHIFT32( in_Q15[ m ], (opus_int32)cb_Q8_ptr[ m ], 7 ); /* range: [ -32767 : 32767 ]*/ + diffw_Q24 = silk_SMULBB( diff_Q15, w_Q9_ptr[ m ] ); + sum_error_Q24 = silk_ADD32( sum_error_Q24, silk_abs( silk_SUB_RSHIFT32( diffw_Q24, pred_Q24, 1 ) ) ); + pred_Q24 = diffw_Q24; + + silk_assert( sum_error_Q24 >= 0 ); + } + err_Q24[ i ] = sum_error_Q24; + cb_Q8_ptr += LPC_order; + w_Q9_ptr += LPC_order; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ_weights_laroia.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ_weights_laroia.c new file mode 100644 index 000000000..9873bcde1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_VQ_weights_laroia.c @@ -0,0 +1,80 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "define.h" +#include "SigProc_FIX.h" + +/* +R. Laroia, N. Phamdo and N. Farvardin, "Robust and Efficient Quantization of Speech LSP +Parameters Using Structured Vector Quantization", Proc. IEEE Int. Conf. Acoust., Speech, +Signal Processing, pp. 641-644, 1991. +*/ + +/* Laroia low complexity NLSF weights */ +void silk_NLSF_VQ_weights_laroia( + opus_int16 *pNLSFW_Q_OUT, /* O Pointer to input vector weights [D] */ + const opus_int16 *pNLSF_Q15, /* I Pointer to input vector [D] */ + const opus_int D /* I Input vector dimension (even) */ +) +{ + opus_int k; + opus_int32 tmp1_int, tmp2_int; + + celt_assert( D > 0 ); + celt_assert( ( D & 1 ) == 0 ); + + /* First value */ + tmp1_int = silk_max_int( pNLSF_Q15[ 0 ], 1 ); + tmp1_int = silk_DIV32_16( (opus_int32)1 << ( 15 + NLSF_W_Q ), tmp1_int ); + tmp2_int = silk_max_int( pNLSF_Q15[ 1 ] - pNLSF_Q15[ 0 ], 1 ); + tmp2_int = silk_DIV32_16( (opus_int32)1 << ( 15 + NLSF_W_Q ), tmp2_int ); + pNLSFW_Q_OUT[ 0 ] = (opus_int16)silk_min_int( tmp1_int + tmp2_int, silk_int16_MAX ); + silk_assert( pNLSFW_Q_OUT[ 0 ] > 0 ); + + /* Main loop */ + for( k = 1; k < D - 1; k += 2 ) { + tmp1_int = silk_max_int( pNLSF_Q15[ k + 1 ] - pNLSF_Q15[ k ], 1 ); + tmp1_int = silk_DIV32_16( (opus_int32)1 << ( 15 + NLSF_W_Q ), tmp1_int ); + pNLSFW_Q_OUT[ k ] = (opus_int16)silk_min_int( tmp1_int + tmp2_int, silk_int16_MAX ); + silk_assert( pNLSFW_Q_OUT[ k ] > 0 ); + + tmp2_int = silk_max_int( pNLSF_Q15[ k + 2 ] - pNLSF_Q15[ k + 1 ], 1 ); + tmp2_int = silk_DIV32_16( (opus_int32)1 << ( 15 + NLSF_W_Q ), tmp2_int ); + pNLSFW_Q_OUT[ k + 1 ] = (opus_int16)silk_min_int( tmp1_int + tmp2_int, silk_int16_MAX ); + silk_assert( pNLSFW_Q_OUT[ k + 1 ] > 0 ); + } + + /* Last value */ + tmp1_int = silk_max_int( ( 1 << 15 ) - pNLSF_Q15[ D - 1 ], 1 ); + tmp1_int = silk_DIV32_16( (opus_int32)1 << ( 15 + NLSF_W_Q ), tmp1_int ); + pNLSFW_Q_OUT[ D - 1 ] = (opus_int16)silk_min_int( tmp1_int + tmp2_int, silk_int16_MAX ); + silk_assert( pNLSFW_Q_OUT[ D - 1 ] > 0 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_decode.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_decode.c new file mode 100644 index 000000000..eeb0ba8c9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_decode.c @@ -0,0 +1,93 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Predictive dequantizer for NLSF residuals */ +static OPUS_INLINE void silk_NLSF_residual_dequant( /* O Returns RD value in Q30 */ + opus_int16 x_Q10[], /* O Output [ order ] */ + const opus_int8 indices[], /* I Quantization indices [ order ] */ + const opus_uint8 pred_coef_Q8[], /* I Backward predictor coefs [ order ] */ + const opus_int quant_step_size_Q16, /* I Quantization step size */ + const opus_int16 order /* I Number of input values */ +) +{ + opus_int i, out_Q10, pred_Q10; + + out_Q10 = 0; + for( i = order-1; i >= 0; i-- ) { + pred_Q10 = silk_RSHIFT( silk_SMULBB( out_Q10, (opus_int16)pred_coef_Q8[ i ] ), 8 ); + out_Q10 = silk_LSHIFT( indices[ i ], 10 ); + if( out_Q10 > 0 ) { + out_Q10 = silk_SUB16( out_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } else if( out_Q10 < 0 ) { + out_Q10 = silk_ADD16( out_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } + out_Q10 = silk_SMLAWB( pred_Q10, (opus_int32)out_Q10, quant_step_size_Q16 ); + x_Q10[ i ] = out_Q10; + } +} + + +/***********************/ +/* NLSF vector decoder */ +/***********************/ +void silk_NLSF_decode( + opus_int16 *pNLSF_Q15, /* O Quantized NLSF vector [ LPC_ORDER ] */ + opus_int8 *NLSFIndices, /* I Codebook path vector [ LPC_ORDER + 1 ] */ + const silk_NLSF_CB_struct *psNLSF_CB /* I Codebook object */ +) +{ + opus_int i; + opus_uint8 pred_Q8[ MAX_LPC_ORDER ]; + opus_int16 ec_ix[ MAX_LPC_ORDER ]; + opus_int16 res_Q10[ MAX_LPC_ORDER ]; + opus_int32 NLSF_Q15_tmp; + const opus_uint8 *pCB_element; + const opus_int16 *pCB_Wght_Q9; + + /* Unpack entropy table indices and predictor for current CB1 index */ + silk_NLSF_unpack( ec_ix, pred_Q8, psNLSF_CB, NLSFIndices[ 0 ] ); + + /* Predictive residual dequantizer */ + silk_NLSF_residual_dequant( res_Q10, &NLSFIndices[ 1 ], pred_Q8, psNLSF_CB->quantStepSize_Q16, psNLSF_CB->order ); + + /* Apply inverse square-rooted weights to first stage and add to output */ + pCB_element = &psNLSF_CB->CB1_NLSF_Q8[ NLSFIndices[ 0 ] * psNLSF_CB->order ]; + pCB_Wght_Q9 = &psNLSF_CB->CB1_Wght_Q9[ NLSFIndices[ 0 ] * psNLSF_CB->order ]; + for( i = 0; i < psNLSF_CB->order; i++ ) { + NLSF_Q15_tmp = silk_ADD_LSHIFT32( silk_DIV32_16( silk_LSHIFT( (opus_int32)res_Q10[ i ], 14 ), pCB_Wght_Q9[ i ] ), (opus_int16)pCB_element[ i ], 7 ); + pNLSF_Q15[ i ] = (opus_int16)silk_LIMIT( NLSF_Q15_tmp, 0, 32767 ); + } + + /* NLSF stabilization */ + silk_NLSF_stabilize( pNLSF_Q15, psNLSF_CB->deltaMin_Q15, psNLSF_CB->order ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_del_dec_quant.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_del_dec_quant.c new file mode 100644 index 000000000..44a16acd0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_del_dec_quant.c @@ -0,0 +1,215 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Delayed-decision quantizer for NLSF residuals */ +opus_int32 silk_NLSF_del_dec_quant( /* O Returns RD value in Q25 */ + opus_int8 indices[], /* O Quantization indices [ order ] */ + const opus_int16 x_Q10[], /* I Input [ order ] */ + const opus_int16 w_Q5[], /* I Weights [ order ] */ + const opus_uint8 pred_coef_Q8[], /* I Backward predictor coefs [ order ] */ + const opus_int16 ec_ix[], /* I Indices to entropy coding tables [ order ] */ + const opus_uint8 ec_rates_Q5[], /* I Rates [] */ + const opus_int quant_step_size_Q16, /* I Quantization step size */ + const opus_int16 inv_quant_step_size_Q6, /* I Inverse quantization step size */ + const opus_int32 mu_Q20, /* I R/D tradeoff */ + const opus_int16 order /* I Number of input values */ +) +{ + opus_int i, j, nStates, ind_tmp, ind_min_max, ind_max_min, in_Q10, res_Q10; + opus_int pred_Q10, diff_Q10, rate0_Q5, rate1_Q5; + opus_int16 out0_Q10, out1_Q10; + opus_int32 RD_tmp_Q25, min_Q25, min_max_Q25, max_min_Q25; + opus_int ind_sort[ NLSF_QUANT_DEL_DEC_STATES ]; + opus_int8 ind[ NLSF_QUANT_DEL_DEC_STATES ][ MAX_LPC_ORDER ]; + opus_int16 prev_out_Q10[ 2 * NLSF_QUANT_DEL_DEC_STATES ]; + opus_int32 RD_Q25[ 2 * NLSF_QUANT_DEL_DEC_STATES ]; + opus_int32 RD_min_Q25[ NLSF_QUANT_DEL_DEC_STATES ]; + opus_int32 RD_max_Q25[ NLSF_QUANT_DEL_DEC_STATES ]; + const opus_uint8 *rates_Q5; + + opus_int out0_Q10_table[2 * NLSF_QUANT_MAX_AMPLITUDE_EXT]; + opus_int out1_Q10_table[2 * NLSF_QUANT_MAX_AMPLITUDE_EXT]; + + for (i = -NLSF_QUANT_MAX_AMPLITUDE_EXT; i <= NLSF_QUANT_MAX_AMPLITUDE_EXT-1; i++) + { + out0_Q10 = silk_LSHIFT( i, 10 ); + out1_Q10 = silk_ADD16( out0_Q10, 1024 ); + if( i > 0 ) { + out0_Q10 = silk_SUB16( out0_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + out1_Q10 = silk_SUB16( out1_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } else if( i == 0 ) { + out1_Q10 = silk_SUB16( out1_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } else if( i == -1 ) { + out0_Q10 = silk_ADD16( out0_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } else { + out0_Q10 = silk_ADD16( out0_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + out1_Q10 = silk_ADD16( out1_Q10, SILK_FIX_CONST( NLSF_QUANT_LEVEL_ADJ, 10 ) ); + } + out0_Q10_table[ i + NLSF_QUANT_MAX_AMPLITUDE_EXT ] = silk_RSHIFT( silk_SMULBB( out0_Q10, quant_step_size_Q16 ), 16 ); + out1_Q10_table[ i + NLSF_QUANT_MAX_AMPLITUDE_EXT ] = silk_RSHIFT( silk_SMULBB( out1_Q10, quant_step_size_Q16 ), 16 ); + } + + silk_assert( (NLSF_QUANT_DEL_DEC_STATES & (NLSF_QUANT_DEL_DEC_STATES-1)) == 0 ); /* must be power of two */ + + nStates = 1; + RD_Q25[ 0 ] = 0; + prev_out_Q10[ 0 ] = 0; + for( i = order - 1; i >= 0; i-- ) { + rates_Q5 = &ec_rates_Q5[ ec_ix[ i ] ]; + in_Q10 = x_Q10[ i ]; + for( j = 0; j < nStates; j++ ) { + pred_Q10 = silk_RSHIFT( silk_SMULBB( (opus_int16)pred_coef_Q8[ i ], prev_out_Q10[ j ] ), 8 ); + res_Q10 = silk_SUB16( in_Q10, pred_Q10 ); + ind_tmp = silk_RSHIFT( silk_SMULBB( inv_quant_step_size_Q6, res_Q10 ), 16 ); + ind_tmp = silk_LIMIT( ind_tmp, -NLSF_QUANT_MAX_AMPLITUDE_EXT, NLSF_QUANT_MAX_AMPLITUDE_EXT-1 ); + ind[ j ][ i ] = (opus_int8)ind_tmp; + + /* compute outputs for ind_tmp and ind_tmp + 1 */ + out0_Q10 = out0_Q10_table[ ind_tmp + NLSF_QUANT_MAX_AMPLITUDE_EXT ]; + out1_Q10 = out1_Q10_table[ ind_tmp + NLSF_QUANT_MAX_AMPLITUDE_EXT ]; + + out0_Q10 = silk_ADD16( out0_Q10, pred_Q10 ); + out1_Q10 = silk_ADD16( out1_Q10, pred_Q10 ); + prev_out_Q10[ j ] = out0_Q10; + prev_out_Q10[ j + nStates ] = out1_Q10; + + /* compute RD for ind_tmp and ind_tmp + 1 */ + if( ind_tmp + 1 >= NLSF_QUANT_MAX_AMPLITUDE ) { + if( ind_tmp + 1 == NLSF_QUANT_MAX_AMPLITUDE ) { + rate0_Q5 = rates_Q5[ ind_tmp + NLSF_QUANT_MAX_AMPLITUDE ]; + rate1_Q5 = 280; + } else { + rate0_Q5 = silk_SMLABB( 280 - 43 * NLSF_QUANT_MAX_AMPLITUDE, 43, ind_tmp ); + rate1_Q5 = silk_ADD16( rate0_Q5, 43 ); + } + } else if( ind_tmp <= -NLSF_QUANT_MAX_AMPLITUDE ) { + if( ind_tmp == -NLSF_QUANT_MAX_AMPLITUDE ) { + rate0_Q5 = 280; + rate1_Q5 = rates_Q5[ ind_tmp + 1 + NLSF_QUANT_MAX_AMPLITUDE ]; + } else { + rate0_Q5 = silk_SMLABB( 280 - 43 * NLSF_QUANT_MAX_AMPLITUDE, -43, ind_tmp ); + rate1_Q5 = silk_SUB16( rate0_Q5, 43 ); + } + } else { + rate0_Q5 = rates_Q5[ ind_tmp + NLSF_QUANT_MAX_AMPLITUDE ]; + rate1_Q5 = rates_Q5[ ind_tmp + 1 + NLSF_QUANT_MAX_AMPLITUDE ]; + } + RD_tmp_Q25 = RD_Q25[ j ]; + diff_Q10 = silk_SUB16( in_Q10, out0_Q10 ); + RD_Q25[ j ] = silk_SMLABB( silk_MLA( RD_tmp_Q25, silk_SMULBB( diff_Q10, diff_Q10 ), w_Q5[ i ] ), mu_Q20, rate0_Q5 ); + diff_Q10 = silk_SUB16( in_Q10, out1_Q10 ); + RD_Q25[ j + nStates ] = silk_SMLABB( silk_MLA( RD_tmp_Q25, silk_SMULBB( diff_Q10, diff_Q10 ), w_Q5[ i ] ), mu_Q20, rate1_Q5 ); + } + + if( nStates <= NLSF_QUANT_DEL_DEC_STATES/2 ) { + /* double number of states and copy */ + for( j = 0; j < nStates; j++ ) { + ind[ j + nStates ][ i ] = ind[ j ][ i ] + 1; + } + nStates = silk_LSHIFT( nStates, 1 ); + for( j = nStates; j < NLSF_QUANT_DEL_DEC_STATES; j++ ) { + ind[ j ][ i ] = ind[ j - nStates ][ i ]; + } + } else { + /* sort lower and upper half of RD_Q25, pairwise */ + for( j = 0; j < NLSF_QUANT_DEL_DEC_STATES; j++ ) { + if( RD_Q25[ j ] > RD_Q25[ j + NLSF_QUANT_DEL_DEC_STATES ] ) { + RD_max_Q25[ j ] = RD_Q25[ j ]; + RD_min_Q25[ j ] = RD_Q25[ j + NLSF_QUANT_DEL_DEC_STATES ]; + RD_Q25[ j ] = RD_min_Q25[ j ]; + RD_Q25[ j + NLSF_QUANT_DEL_DEC_STATES ] = RD_max_Q25[ j ]; + /* swap prev_out values */ + out0_Q10 = prev_out_Q10[ j ]; + prev_out_Q10[ j ] = prev_out_Q10[ j + NLSF_QUANT_DEL_DEC_STATES ]; + prev_out_Q10[ j + NLSF_QUANT_DEL_DEC_STATES ] = out0_Q10; + ind_sort[ j ] = j + NLSF_QUANT_DEL_DEC_STATES; + } else { + RD_min_Q25[ j ] = RD_Q25[ j ]; + RD_max_Q25[ j ] = RD_Q25[ j + NLSF_QUANT_DEL_DEC_STATES ]; + ind_sort[ j ] = j; + } + } + /* compare the highest RD values of the winning half with the lowest one in the losing half, and copy if necessary */ + /* afterwards ind_sort[] will contain the indices of the NLSF_QUANT_DEL_DEC_STATES winning RD values */ + while( 1 ) { + min_max_Q25 = silk_int32_MAX; + max_min_Q25 = 0; + ind_min_max = 0; + ind_max_min = 0; + for( j = 0; j < NLSF_QUANT_DEL_DEC_STATES; j++ ) { + if( min_max_Q25 > RD_max_Q25[ j ] ) { + min_max_Q25 = RD_max_Q25[ j ]; + ind_min_max = j; + } + if( max_min_Q25 < RD_min_Q25[ j ] ) { + max_min_Q25 = RD_min_Q25[ j ]; + ind_max_min = j; + } + } + if( min_max_Q25 >= max_min_Q25 ) { + break; + } + /* copy ind_min_max to ind_max_min */ + ind_sort[ ind_max_min ] = ind_sort[ ind_min_max ] ^ NLSF_QUANT_DEL_DEC_STATES; + RD_Q25[ ind_max_min ] = RD_Q25[ ind_min_max + NLSF_QUANT_DEL_DEC_STATES ]; + prev_out_Q10[ ind_max_min ] = prev_out_Q10[ ind_min_max + NLSF_QUANT_DEL_DEC_STATES ]; + RD_min_Q25[ ind_max_min ] = 0; + RD_max_Q25[ ind_min_max ] = silk_int32_MAX; + silk_memcpy( ind[ ind_max_min ], ind[ ind_min_max ], MAX_LPC_ORDER * sizeof( opus_int8 ) ); + } + /* increment index if it comes from the upper half */ + for( j = 0; j < NLSF_QUANT_DEL_DEC_STATES; j++ ) { + ind[ j ][ i ] += silk_RSHIFT( ind_sort[ j ], NLSF_QUANT_DEL_DEC_STATES_LOG2 ); + } + } + } + + /* last sample: find winner, copy indices and return RD value */ + ind_tmp = 0; + min_Q25 = silk_int32_MAX; + for( j = 0; j < 2 * NLSF_QUANT_DEL_DEC_STATES; j++ ) { + if( min_Q25 > RD_Q25[ j ] ) { + min_Q25 = RD_Q25[ j ]; + ind_tmp = j; + } + } + for( j = 0; j < order; j++ ) { + indices[ j ] = ind[ ind_tmp & ( NLSF_QUANT_DEL_DEC_STATES - 1 ) ][ j ]; + silk_assert( indices[ j ] >= -NLSF_QUANT_MAX_AMPLITUDE_EXT ); + silk_assert( indices[ j ] <= NLSF_QUANT_MAX_AMPLITUDE_EXT ); + } + indices[ 0 ] += silk_RSHIFT( ind_tmp, NLSF_QUANT_DEL_DEC_STATES_LOG2 ); + silk_assert( indices[ 0 ] <= NLSF_QUANT_MAX_AMPLITUDE_EXT ); + silk_assert( min_Q25 >= 0 ); + return min_Q25; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_encode.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_encode.c new file mode 100644 index 000000000..01ac7db78 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_encode.c @@ -0,0 +1,124 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/***********************/ +/* NLSF vector encoder */ +/***********************/ +opus_int32 silk_NLSF_encode( /* O Returns RD value in Q25 */ + opus_int8 *NLSFIndices, /* I Codebook path vector [ LPC_ORDER + 1 ] */ + opus_int16 *pNLSF_Q15, /* I/O (Un)quantized NLSF vector [ LPC_ORDER ] */ + const silk_NLSF_CB_struct *psNLSF_CB, /* I Codebook object */ + const opus_int16 *pW_Q2, /* I NLSF weight vector [ LPC_ORDER ] */ + const opus_int NLSF_mu_Q20, /* I Rate weight for the RD optimization */ + const opus_int nSurvivors, /* I Max survivors after first stage */ + const opus_int signalType /* I Signal type: 0/1/2 */ +) +{ + opus_int i, s, ind1, bestIndex, prob_Q8, bits_q7; + opus_int32 W_tmp_Q9, ret; + VARDECL( opus_int32, err_Q24 ); + VARDECL( opus_int32, RD_Q25 ); + VARDECL( opus_int, tempIndices1 ); + VARDECL( opus_int8, tempIndices2 ); + opus_int16 res_Q10[ MAX_LPC_ORDER ]; + opus_int16 NLSF_tmp_Q15[ MAX_LPC_ORDER ]; + opus_int16 W_adj_Q5[ MAX_LPC_ORDER ]; + opus_uint8 pred_Q8[ MAX_LPC_ORDER ]; + opus_int16 ec_ix[ MAX_LPC_ORDER ]; + const opus_uint8 *pCB_element, *iCDF_ptr; + const opus_int16 *pCB_Wght_Q9; + SAVE_STACK; + + celt_assert( signalType >= 0 && signalType <= 2 ); + silk_assert( NLSF_mu_Q20 <= 32767 && NLSF_mu_Q20 >= 0 ); + + /* NLSF stabilization */ + silk_NLSF_stabilize( pNLSF_Q15, psNLSF_CB->deltaMin_Q15, psNLSF_CB->order ); + + /* First stage: VQ */ + ALLOC( err_Q24, psNLSF_CB->nVectors, opus_int32 ); + silk_NLSF_VQ( err_Q24, pNLSF_Q15, psNLSF_CB->CB1_NLSF_Q8, psNLSF_CB->CB1_Wght_Q9, psNLSF_CB->nVectors, psNLSF_CB->order ); + + /* Sort the quantization errors */ + ALLOC( tempIndices1, nSurvivors, opus_int ); + silk_insertion_sort_increasing( err_Q24, tempIndices1, psNLSF_CB->nVectors, nSurvivors ); + + ALLOC( RD_Q25, nSurvivors, opus_int32 ); + ALLOC( tempIndices2, nSurvivors * MAX_LPC_ORDER, opus_int8 ); + + /* Loop over survivors */ + for( s = 0; s < nSurvivors; s++ ) { + ind1 = tempIndices1[ s ]; + + /* Residual after first stage */ + pCB_element = &psNLSF_CB->CB1_NLSF_Q8[ ind1 * psNLSF_CB->order ]; + pCB_Wght_Q9 = &psNLSF_CB->CB1_Wght_Q9[ ind1 * psNLSF_CB->order ]; + for( i = 0; i < psNLSF_CB->order; i++ ) { + NLSF_tmp_Q15[ i ] = silk_LSHIFT16( (opus_int16)pCB_element[ i ], 7 ); + W_tmp_Q9 = pCB_Wght_Q9[ i ]; + res_Q10[ i ] = (opus_int16)silk_RSHIFT( silk_SMULBB( pNLSF_Q15[ i ] - NLSF_tmp_Q15[ i ], W_tmp_Q9 ), 14 ); + W_adj_Q5[ i ] = silk_DIV32_varQ( (opus_int32)pW_Q2[ i ], silk_SMULBB( W_tmp_Q9, W_tmp_Q9 ), 21 ); + } + + /* Unpack entropy table indices and predictor for current CB1 index */ + silk_NLSF_unpack( ec_ix, pred_Q8, psNLSF_CB, ind1 ); + + /* Trellis quantizer */ + RD_Q25[ s ] = silk_NLSF_del_dec_quant( &tempIndices2[ s * MAX_LPC_ORDER ], res_Q10, W_adj_Q5, pred_Q8, ec_ix, + psNLSF_CB->ec_Rates_Q5, psNLSF_CB->quantStepSize_Q16, psNLSF_CB->invQuantStepSize_Q6, NLSF_mu_Q20, psNLSF_CB->order ); + + /* Add rate for first stage */ + iCDF_ptr = &psNLSF_CB->CB1_iCDF[ ( signalType >> 1 ) * psNLSF_CB->nVectors ]; + if( ind1 == 0 ) { + prob_Q8 = 256 - iCDF_ptr[ ind1 ]; + } else { + prob_Q8 = iCDF_ptr[ ind1 - 1 ] - iCDF_ptr[ ind1 ]; + } + bits_q7 = ( 8 << 7 ) - silk_lin2log( prob_Q8 ); + RD_Q25[ s ] = silk_SMLABB( RD_Q25[ s ], bits_q7, silk_RSHIFT( NLSF_mu_Q20, 2 ) ); + } + + /* Find the lowest rate-distortion error */ + silk_insertion_sort_increasing( RD_Q25, &bestIndex, nSurvivors, 1 ); + + NLSFIndices[ 0 ] = (opus_int8)tempIndices1[ bestIndex ]; + silk_memcpy( &NLSFIndices[ 1 ], &tempIndices2[ bestIndex * MAX_LPC_ORDER ], psNLSF_CB->order * sizeof( opus_int8 ) ); + + /* Decode */ + silk_NLSF_decode( pNLSF_Q15, NLSFIndices, psNLSF_CB ); + + ret = RD_Q25[ 0 ]; + RESTORE_STACK; + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_stabilize.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_stabilize.c new file mode 100644 index 000000000..8f3426b91 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_stabilize.c @@ -0,0 +1,142 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* NLSF stabilizer: */ +/* */ +/* - Moves NLSFs further apart if they are too close */ +/* - Moves NLSFs away from borders if they are too close */ +/* - High effort to achieve a modification with minimum */ +/* Euclidean distance to input vector */ +/* - Output are sorted NLSF coefficients */ +/* */ + +#include "SigProc_FIX.h" + +/* Constant Definitions */ +#define MAX_LOOPS 20 + +/* NLSF stabilizer, for a single input data vector */ +void silk_NLSF_stabilize( + opus_int16 *NLSF_Q15, /* I/O Unstable/stabilized normalized LSF vector in Q15 [L] */ + const opus_int16 *NDeltaMin_Q15, /* I Min distance vector, NDeltaMin_Q15[L] must be >= 1 [L+1] */ + const opus_int L /* I Number of NLSF parameters in the input vector */ +) +{ + opus_int i, I=0, k, loops; + opus_int16 center_freq_Q15; + opus_int32 diff_Q15, min_diff_Q15, min_center_Q15, max_center_Q15; + + /* This is necessary to ensure an output within range of a opus_int16 */ + silk_assert( NDeltaMin_Q15[L] >= 1 ); + + for( loops = 0; loops < MAX_LOOPS; loops++ ) { + /**************************/ + /* Find smallest distance */ + /**************************/ + /* First element */ + min_diff_Q15 = NLSF_Q15[0] - NDeltaMin_Q15[0]; + I = 0; + /* Middle elements */ + for( i = 1; i <= L-1; i++ ) { + diff_Q15 = NLSF_Q15[i] - ( NLSF_Q15[i-1] + NDeltaMin_Q15[i] ); + if( diff_Q15 < min_diff_Q15 ) { + min_diff_Q15 = diff_Q15; + I = i; + } + } + /* Last element */ + diff_Q15 = ( 1 << 15 ) - ( NLSF_Q15[L-1] + NDeltaMin_Q15[L] ); + if( diff_Q15 < min_diff_Q15 ) { + min_diff_Q15 = diff_Q15; + I = L; + } + + /***************************************************/ + /* Now check if the smallest distance non-negative */ + /***************************************************/ + if( min_diff_Q15 >= 0 ) { + return; + } + + if( I == 0 ) { + /* Move away from lower limit */ + NLSF_Q15[0] = NDeltaMin_Q15[0]; + + } else if( I == L) { + /* Move away from higher limit */ + NLSF_Q15[L-1] = ( 1 << 15 ) - NDeltaMin_Q15[L]; + + } else { + /* Find the lower extreme for the location of the current center frequency */ + min_center_Q15 = 0; + for( k = 0; k < I; k++ ) { + min_center_Q15 += NDeltaMin_Q15[k]; + } + min_center_Q15 += silk_RSHIFT( NDeltaMin_Q15[I], 1 ); + + /* Find the upper extreme for the location of the current center frequency */ + max_center_Q15 = 1 << 15; + for( k = L; k > I; k-- ) { + max_center_Q15 -= NDeltaMin_Q15[k]; + } + max_center_Q15 -= silk_RSHIFT( NDeltaMin_Q15[I], 1 ); + + /* Move apart, sorted by value, keeping the same center frequency */ + center_freq_Q15 = (opus_int16)silk_LIMIT_32( silk_RSHIFT_ROUND( (opus_int32)NLSF_Q15[I-1] + (opus_int32)NLSF_Q15[I], 1 ), + min_center_Q15, max_center_Q15 ); + NLSF_Q15[I-1] = center_freq_Q15 - silk_RSHIFT( NDeltaMin_Q15[I], 1 ); + NLSF_Q15[I] = NLSF_Q15[I-1] + NDeltaMin_Q15[I]; + } + } + + /* Safe and simple fall back method, which is less ideal than the above */ + if( loops == MAX_LOOPS ) + { + /* Insertion sort (fast for already almost sorted arrays): */ + /* Best case: O(n) for an already sorted array */ + /* Worst case: O(n^2) for an inversely sorted array */ + silk_insertion_sort_increasing_all_values_int16( &NLSF_Q15[0], L ); + + /* First NLSF should be no less than NDeltaMin[0] */ + NLSF_Q15[0] = silk_max_int( NLSF_Q15[0], NDeltaMin_Q15[0] ); + + /* Keep delta_min distance between the NLSFs */ + for( i = 1; i < L; i++ ) + NLSF_Q15[i] = silk_max_int( NLSF_Q15[i], silk_ADD_SAT16( NLSF_Q15[i-1], NDeltaMin_Q15[i] ) ); + + /* Last NLSF should be no higher than 1 - NDeltaMin[L] */ + NLSF_Q15[L-1] = silk_min_int( NLSF_Q15[L-1], (1<<15) - NDeltaMin_Q15[L] ); + + /* Keep NDeltaMin distance between the NLSFs */ + for( i = L-2; i >= 0; i-- ) + NLSF_Q15[i] = silk_min_int( NLSF_Q15[i], NLSF_Q15[i+1] - NDeltaMin_Q15[i+1] ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_unpack.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_unpack.c new file mode 100644 index 000000000..17bd23f75 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NLSF_unpack.c @@ -0,0 +1,55 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Unpack predictor values and indices for entropy coding tables */ +void silk_NLSF_unpack( + opus_int16 ec_ix[], /* O Indices to entropy tables [ LPC_ORDER ] */ + opus_uint8 pred_Q8[], /* O LSF predictor [ LPC_ORDER ] */ + const silk_NLSF_CB_struct *psNLSF_CB, /* I Codebook object */ + const opus_int CB1_index /* I Index of vector in first LSF codebook */ +) +{ + opus_int i; + opus_uint8 entry; + const opus_uint8 *ec_sel_ptr; + + ec_sel_ptr = &psNLSF_CB->ec_sel[ CB1_index * psNLSF_CB->order / 2 ]; + for( i = 0; i < psNLSF_CB->order; i += 2 ) { + entry = *ec_sel_ptr++; + ec_ix [ i ] = silk_SMULBB( silk_RSHIFT( entry, 1 ) & 7, 2 * NLSF_QUANT_MAX_AMPLITUDE + 1 ); + pred_Q8[ i ] = psNLSF_CB->pred_Q8[ i + ( entry & 1 ) * ( psNLSF_CB->order - 1 ) ]; + ec_ix [ i + 1 ] = silk_SMULBB( silk_RSHIFT( entry, 5 ) & 7, 2 * NLSF_QUANT_MAX_AMPLITUDE + 1 ); + pred_Q8[ i + 1 ] = psNLSF_CB->pred_Q8[ i + ( silk_RSHIFT( entry, 4 ) & 1 ) * ( psNLSF_CB->order - 1 ) + 1 ]; + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.c new file mode 100644 index 000000000..1d64d8e25 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.c @@ -0,0 +1,437 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" +#include "NSQ.h" + + +static OPUS_INLINE void silk_nsq_scale_states( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + const opus_int16 x16[], /* I input */ + opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */ + const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I subframe number */ + const opus_int LTP_scale_Q14, /* I */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type /* I Signal type */ +); + +#if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +static OPUS_INLINE void silk_noise_shape_quantizer( + silk_nsq_state *NSQ, /* I/O NSQ state */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_sc_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP state */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int shapingLPCOrder, /* I Noise shaping AR filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + int arch /* I Architecture */ +); +#endif + +void silk_NSQ_c +( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) +{ + opus_int k, lag, start_idx, LSF_interpolation_flag; + const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; + opus_int16 *pxq; + VARDECL( opus_int32, sLTP_Q15 ); + VARDECL( opus_int16, sLTP ); + opus_int32 HarmShapeFIRPacked_Q14; + opus_int offset_Q10; + VARDECL( opus_int32, x_sc_Q10 ); + SAVE_STACK; + + NSQ->rand_seed = psIndices->Seed; + + /* Set unvoiced lag to the previous one, overwrite later for voiced */ + lag = NSQ->lagPrev; + + silk_assert( NSQ->prev_gain_Q16 != 0 ); + + offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; + + if( psIndices->NLSFInterpCoef_Q2 == 4 ) { + LSF_interpolation_flag = 0; + } else { + LSF_interpolation_flag = 1; + } + + ALLOC( sLTP_Q15, psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); + ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); + ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); + /* Set up pointers to start of sub frame */ + NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; + for( k = 0; k < psEncC->nb_subfr; k++ ) { + A_Q12 = &PredCoef_Q12[ (( k >> 1 ) | ( 1 - LSF_interpolation_flag )) * MAX_LPC_ORDER ]; + B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; + AR_shp_Q13 = &AR_Q13[ k * MAX_SHAPE_LPC_ORDER ]; + + /* Noise shape parameters */ + silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); + HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); + HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); + + NSQ->rewhite_flag = 0; + if( psIndices->signalType == TYPE_VOICED ) { + /* Voiced */ + lag = pitchL[ k ]; + + /* Re-whitening */ + if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { + /* Rewhiten with new A coefs */ + start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; + celt_assert( start_idx > 0 ); + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], + A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); + + NSQ->rewhite_flag = 1; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + } + } + + silk_nsq_scale_states( psEncC, NSQ, x16, x_sc_Q10, sLTP, sLTP_Q15, k, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType ); + + silk_noise_shape_quantizer( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14, + AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ], Lambda_Q10, + offset_Q10, psEncC->subfr_length, psEncC->shapingLPCOrder, psEncC->predictLPCOrder, psEncC->arch ); + + x16 += psEncC->subfr_length; + pulses += psEncC->subfr_length; + pxq += psEncC->subfr_length; + } + + /* Update lagPrev for next frame */ + NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; + + /* Save quantized speech and noise shaping signals */ + silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); + silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); + RESTORE_STACK; +} + +/***********************************/ +/* silk_noise_shape_quantizer */ +/***********************************/ + +#if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +static OPUS_INLINE +#endif +void silk_noise_shape_quantizer( + silk_nsq_state *NSQ, /* I/O NSQ state */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_sc_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP state */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int shapingLPCOrder, /* I Noise shaping AR filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + int arch /* I Architecture */ +) +{ + opus_int i; + opus_int32 LTP_pred_Q13, LPC_pred_Q10, n_AR_Q12, n_LTP_Q13; + opus_int32 n_LF_Q12, r_Q10, rr_Q10, q1_Q0, q1_Q10, q2_Q10, rd1_Q20, rd2_Q20; + opus_int32 exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10; + opus_int32 tmp1, tmp2, sLF_AR_shp_Q14; + opus_int32 *psLPC_Q14, *shp_lag_ptr, *pred_lag_ptr; +#ifdef silk_short_prediction_create_arch_coef + opus_int32 a_Q12_arch[MAX_LPC_ORDER]; +#endif + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + + /* Set up short term AR state */ + psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 ]; + +#ifdef silk_short_prediction_create_arch_coef + silk_short_prediction_create_arch_coef(a_Q12_arch, a_Q12, predictLPCOrder); +#endif + + for( i = 0; i < length; i++ ) { + /* Generate dither */ + NSQ->rand_seed = silk_RAND( NSQ->rand_seed ); + + /* Short-term prediction */ + LPC_pred_Q10 = silk_noise_shape_quantizer_short_prediction(psLPC_Q14, a_Q12, a_Q12_arch, predictLPCOrder, arch); + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q13 = 2; + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ 0 ], b_Q14[ 0 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -1 ], b_Q14[ 1 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -2 ], b_Q14[ 2 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -3 ], b_Q14[ 3 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); + pred_lag_ptr++; + } else { + LTP_pred_Q13 = 0; + } + + /* Noise shape feedback */ + celt_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ + n_AR_Q12 = silk_NSQ_noise_shape_feedback_loop(&NSQ->sDiff_shp_Q14, NSQ->sAR2_Q14, AR_shp_Q13, shapingLPCOrder, arch); + + n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sLF_AR_shp_Q14, Tilt_Q14 ); + + n_LF_Q12 = silk_SMULWB( NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - 1 ], LF_shp_Q14 ); + n_LF_Q12 = silk_SMLAWT( n_LF_Q12, NSQ->sLF_AR_shp_Q14, LF_shp_Q14 ); + + celt_assert( lag > 0 || signalType != TYPE_VOICED ); + + /* Combine prediction and noise shaping signals */ + tmp1 = silk_SUB32( silk_LSHIFT32( LPC_pred_Q10, 2 ), n_AR_Q12 ); /* Q12 */ + tmp1 = silk_SUB32( tmp1, n_LF_Q12 ); /* Q12 */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q13 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q13 = silk_SMLAWT( n_LTP_Q13, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q13 = silk_LSHIFT( n_LTP_Q13, 1 ); + shp_lag_ptr++; + + tmp2 = silk_SUB32( LTP_pred_Q13, n_LTP_Q13 ); /* Q13 */ + tmp1 = silk_ADD_LSHIFT32( tmp2, tmp1, 1 ); /* Q13 */ + tmp1 = silk_RSHIFT_ROUND( tmp1, 3 ); /* Q10 */ + } else { + tmp1 = silk_RSHIFT_ROUND( tmp1, 2 ); /* Q10 */ + } + + r_Q10 = silk_SUB32( x_sc_Q10[ i ], tmp1 ); /* residual error Q10 */ + + /* Flip sign depending on dither */ + if( NSQ->rand_seed < 0 ) { + r_Q10 = -r_Q10; + } + r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + q1_Q10 = silk_SUB32( r_Q10, offset_Q10 ); + q1_Q0 = silk_RSHIFT( q1_Q10, 10 ); + if (Lambda_Q10 > 2048) { + /* For aggressive RDO, the bias becomes more than one pulse. */ + int rdo_offset = Lambda_Q10/2 - 512; + if (q1_Q10 > rdo_offset) { + q1_Q0 = silk_RSHIFT( q1_Q10 - rdo_offset, 10 ); + } else if (q1_Q10 < -rdo_offset) { + q1_Q0 = silk_RSHIFT( q1_Q10 + rdo_offset, 10 ); + } else if (q1_Q10 < 0) { + q1_Q0 = -1; + } else { + q1_Q0 = 0; + } + } + if( q1_Q0 > 0 ) { + q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q20 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q20 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == 0 ) { + q1_Q10 = offset_Q10; + q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q20 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q20 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == -1 ) { + q2_Q10 = offset_Q10; + q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q20 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q20 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else { /* Q1_Q0 < -1 */ + q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q20 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q20 = silk_SMULBB( -q2_Q10, Lambda_Q10 ); + } + rr_Q10 = silk_SUB32( r_Q10, q1_Q10 ); + rd1_Q20 = silk_SMLABB( rd1_Q20, rr_Q10, rr_Q10 ); + rr_Q10 = silk_SUB32( r_Q10, q2_Q10 ); + rd2_Q20 = silk_SMLABB( rd2_Q20, rr_Q10, rr_Q10 ); + + if( rd2_Q20 < rd1_Q20 ) { + q1_Q10 = q2_Q10; + } + + pulses[ i ] = (opus_int8)silk_RSHIFT_ROUND( q1_Q10, 10 ); + + /* Excitation */ + exc_Q14 = silk_LSHIFT( q1_Q10, 4 ); + if ( NSQ->rand_seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD_LSHIFT32( exc_Q14, LTP_pred_Q13, 1 ); + xq_Q14 = silk_ADD_LSHIFT32( LPC_exc_Q14, LPC_pred_Q10, 4 ); + + /* Scale XQ back to normal level before saving */ + xq[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( xq_Q14, Gain_Q10 ), 8 ) ); + + /* Update states */ + psLPC_Q14++; + *psLPC_Q14 = xq_Q14; + NSQ->sDiff_shp_Q14 = silk_SUB_LSHIFT32( xq_Q14, x_sc_Q10[ i ], 4 ); + sLF_AR_shp_Q14 = silk_SUB_LSHIFT32( NSQ->sDiff_shp_Q14, n_AR_Q12, 2 ); + NSQ->sLF_AR_shp_Q14 = sLF_AR_shp_Q14; + + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx ] = silk_SUB_LSHIFT32( sLF_AR_shp_Q14, n_LF_Q12, 2 ); + sLTP_Q15[ NSQ->sLTP_buf_idx ] = silk_LSHIFT( LPC_exc_Q14, 1 ); + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Make dither dependent on quantized signal */ + NSQ->rand_seed = silk_ADD32_ovflw( NSQ->rand_seed, pulses[ i ] ); + } + + /* Update LPC synth buffer */ + silk_memcpy( NSQ->sLPC_Q14, &NSQ->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); +} + +static OPUS_INLINE void silk_nsq_scale_states( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + const opus_int16 x16[], /* I input */ + opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */ + const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I subframe number */ + const opus_int LTP_scale_Q14, /* I */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type /* I Signal type */ +) +{ + opus_int i, lag; + opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q26; + + lag = pitchL[ subfr ]; + inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); + silk_assert( inv_gain_Q31 != 0 ); + + /* Scale input */ + inv_gain_Q26 = silk_RSHIFT_ROUND( inv_gain_Q31, 5 ); + for( i = 0; i < psEncC->subfr_length; i++ ) { + x_sc_Q10[ i ] = silk_SMULWW( x16[ i ], inv_gain_Q26 ); + } + + /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ + if( NSQ->rewhite_flag ) { + if( subfr == 0 ) { + /* Do LTP downscaling */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); + } + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + silk_assert( i < MAX_FRAME_LENGTH ); + sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] ); + } + } + + /* Adjust for changing gain */ + if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { + gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); + + /* Scale long-term shaping state */ + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + NSQ->sLF_AR_shp_Q14 = silk_SMULWW( gain_adj_Q16, NSQ->sLF_AR_shp_Q14 ); + NSQ->sDiff_shp_Q14 = silk_SMULWW( gain_adj_Q16, NSQ->sDiff_shp_Q14 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + NSQ->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLPC_Q14[ i ] ); + } + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + NSQ->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sAR2_Q14[ i ] ); + } + + /* Save inverse gain */ + NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.h new file mode 100644 index 000000000..971832f66 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ.h @@ -0,0 +1,101 @@ +/*********************************************************************** +Copyright (c) 2014 Vidyo. +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +#ifndef SILK_NSQ_H +#define SILK_NSQ_H + +#include "SigProc_FIX.h" + +#undef silk_short_prediction_create_arch_coef + +static OPUS_INLINE opus_int32 silk_noise_shape_quantizer_short_prediction_c(const opus_int32 *buf32, const opus_int16 *coef16, opus_int order) +{ + opus_int32 out; + silk_assert( order == 10 || order == 16 ); + + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + out = silk_RSHIFT( order, 1 ); + out = silk_SMLAWB( out, buf32[ 0 ], coef16[ 0 ] ); + out = silk_SMLAWB( out, buf32[ -1 ], coef16[ 1 ] ); + out = silk_SMLAWB( out, buf32[ -2 ], coef16[ 2 ] ); + out = silk_SMLAWB( out, buf32[ -3 ], coef16[ 3 ] ); + out = silk_SMLAWB( out, buf32[ -4 ], coef16[ 4 ] ); + out = silk_SMLAWB( out, buf32[ -5 ], coef16[ 5 ] ); + out = silk_SMLAWB( out, buf32[ -6 ], coef16[ 6 ] ); + out = silk_SMLAWB( out, buf32[ -7 ], coef16[ 7 ] ); + out = silk_SMLAWB( out, buf32[ -8 ], coef16[ 8 ] ); + out = silk_SMLAWB( out, buf32[ -9 ], coef16[ 9 ] ); + + if( order == 16 ) + { + out = silk_SMLAWB( out, buf32[ -10 ], coef16[ 10 ] ); + out = silk_SMLAWB( out, buf32[ -11 ], coef16[ 11 ] ); + out = silk_SMLAWB( out, buf32[ -12 ], coef16[ 12 ] ); + out = silk_SMLAWB( out, buf32[ -13 ], coef16[ 13 ] ); + out = silk_SMLAWB( out, buf32[ -14 ], coef16[ 14 ] ); + out = silk_SMLAWB( out, buf32[ -15 ], coef16[ 15 ] ); + } + return out; +} + +#define silk_noise_shape_quantizer_short_prediction(in, coef, coefRev, order, arch) ((void)arch,silk_noise_shape_quantizer_short_prediction_c(in, coef, order)) + +static OPUS_INLINE opus_int32 silk_NSQ_noise_shape_feedback_loop_c(const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, opus_int order) +{ + opus_int32 out; + opus_int32 tmp1, tmp2; + opus_int j; + + tmp2 = data0[0]; + tmp1 = data1[0]; + data1[0] = tmp2; + + out = silk_RSHIFT(order, 1); + out = silk_SMLAWB(out, tmp2, coef[0]); + + for (j = 2; j < order; j += 2) { + tmp2 = data1[j - 1]; + data1[j - 1] = tmp1; + out = silk_SMLAWB(out, tmp1, coef[j - 1]); + tmp1 = data1[j + 0]; + data1[j + 0] = tmp2; + out = silk_SMLAWB(out, tmp2, coef[j]); + } + data1[order - 1] = tmp1; + out = silk_SMLAWB(out, tmp1, coef[order - 1]); + /* Q11 -> Q12 */ + out = silk_LSHIFT32( out, 1 ); + return out; +} + +#define silk_NSQ_noise_shape_feedback_loop(data0, data1, coef, order, arch) ((void)arch,silk_NSQ_noise_shape_feedback_loop_c(data0, data1, coef, order)) + +#if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +#include "arm/NSQ_neon.h" +#endif + +#endif /* SILK_NSQ_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ_del_dec.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ_del_dec.c new file mode 100644 index 000000000..00e749c3b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/NSQ_del_dec.c @@ -0,0 +1,733 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" +#include "NSQ.h" + + +typedef struct { + opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ]; + opus_int32 RandState[ DECISION_DELAY ]; + opus_int32 Q_Q10[ DECISION_DELAY ]; + opus_int32 Xq_Q14[ DECISION_DELAY ]; + opus_int32 Pred_Q15[ DECISION_DELAY ]; + opus_int32 Shape_Q14[ DECISION_DELAY ]; + opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 LF_AR_Q14; + opus_int32 Diff_Q14; + opus_int32 Seed; + opus_int32 SeedInit; + opus_int32 RD_Q10; +} NSQ_del_dec_struct; + +typedef struct { + opus_int32 Q_Q10; + opus_int32 RD_Q10; + opus_int32 xq_Q14; + opus_int32 LF_AR_Q14; + opus_int32 Diff_Q14; + opus_int32 sLTP_shp_Q14; + opus_int32 LPC_exc_Q14; +} NSQ_sample_struct; + +typedef NSQ_sample_struct NSQ_sample_pair[ 2 ]; + +#if defined(MIPSr1_ASM) +#include "mips/NSQ_del_dec_mipsr1.h" +#endif +static OPUS_INLINE void silk_nsq_del_dec_scale_states( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int16 x16[], /* I Input */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + opus_int nStatesDelayedDecision, /* I Number of del dec states */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +); + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay, /* I */ + int arch /* I */ +); + +void silk_NSQ_del_dec_c( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) +{ + opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr; + opus_int last_smple_idx, smpl_buf_idx, decisionDelay; + const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; + opus_int16 *pxq; + VARDECL( opus_int32, sLTP_Q15 ); + VARDECL( opus_int16, sLTP ); + opus_int32 HarmShapeFIRPacked_Q14; + opus_int offset_Q10; + opus_int32 RDmin_Q10, Gain_Q10; + VARDECL( opus_int32, x_sc_Q10 ); + VARDECL( opus_int32, delayedGain_Q10 ); + VARDECL( NSQ_del_dec_struct, psDelDec ); + NSQ_del_dec_struct *psDD; + SAVE_STACK; + + /* Set unvoiced lag to the previous one, overwrite later for voiced */ + lag = NSQ->lagPrev; + + silk_assert( NSQ->prev_gain_Q16 != 0 ); + + /* Initialize delayed decision states */ + ALLOC( psDelDec, psEncC->nStatesDelayedDecision, NSQ_del_dec_struct ); + silk_memset( psDelDec, 0, psEncC->nStatesDelayedDecision * sizeof( NSQ_del_dec_struct ) ); + for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + psDD->Seed = ( k + psIndices->Seed ) & 3; + psDD->SeedInit = psDD->Seed; + psDD->RD_Q10 = 0; + psDD->LF_AR_Q14 = NSQ->sLF_AR_shp_Q14; + psDD->Diff_Q14 = NSQ->sDiff_shp_Q14; + psDD->Shape_Q14[ 0 ] = NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ]; + silk_memcpy( psDD->sLPC_Q14, NSQ->sLPC_Q14, NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + silk_memcpy( psDD->sAR2_Q14, NSQ->sAR2_Q14, sizeof( NSQ->sAR2_Q14 ) ); + } + + offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; + smpl_buf_idx = 0; /* index of oldest samples */ + + decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length ); + + /* For voiced frames limit the decision delay to lower than the pitch lag */ + if( psIndices->signalType == TYPE_VOICED ) { + for( k = 0; k < psEncC->nb_subfr; k++ ) { + decisionDelay = silk_min_int( decisionDelay, pitchL[ k ] - LTP_ORDER / 2 - 1 ); + } + } else { + if( lag > 0 ) { + decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 ); + } + } + + if( psIndices->NLSFInterpCoef_Q2 == 4 ) { + LSF_interpolation_flag = 0; + } else { + LSF_interpolation_flag = 1; + } + + ALLOC( sLTP_Q15, psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); + ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); + ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); + ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 ); + /* Set up pointers to start of sub frame */ + pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; + NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + subfr = 0; + for( k = 0; k < psEncC->nb_subfr; k++ ) { + A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ]; + B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; + AR_shp_Q13 = &AR_Q13[ k * MAX_SHAPE_LPC_ORDER ]; + + /* Noise shape parameters */ + silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); + HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); + HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); + + NSQ->rewhite_flag = 0; + if( psIndices->signalType == TYPE_VOICED ) { + /* Voiced */ + lag = pitchL[ k ]; + + /* Re-whitening */ + if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { + if( k == 2 ) { + /* RESET DELAYED DECISIONS */ + /* Find winner */ + RDmin_Q10 = psDelDec[ 0 ].RD_Q10; + Winner_ind = 0; + for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) { + if( psDelDec[ i ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psDelDec[ i ].RD_Q10; + Winner_ind = i; + } + } + for( i = 0; i < psEncC->nStatesDelayedDecision; i++ ) { + if( i != Winner_ind ) { + psDelDec[ i ].RD_Q10 += ( silk_int32_MAX >> 4 ); + silk_assert( psDelDec[ i ].RD_Q10 >= 0 ); + } + } + + /* Copy final part of signals from winner state to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + last_smple_idx = smpl_buf_idx + decisionDelay; + for( i = 0; i < decisionDelay; i++ ) { + last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY; + if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY; + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gains_Q16[ 1 ] ), 14 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ]; + } + + subfr = 0; + } + + /* Rewhiten with new A coefs */ + start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; + celt_assert( start_idx > 0 ); + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], + A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); + + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + NSQ->rewhite_flag = 1; + } + } + + silk_nsq_del_dec_scale_states( psEncC, NSQ, psDelDec, x16, x_sc_Q10, sLTP, sLTP_Q15, k, + psEncC->nStatesDelayedDecision, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay ); + + silk_noise_shape_quantizer_del_dec( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, + delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], + Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder, + psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay, psEncC->arch ); + + x16 += psEncC->subfr_length; + pulses += psEncC->subfr_length; + pxq += psEncC->subfr_length; + } + + /* Find winner */ + RDmin_Q10 = psDelDec[ 0 ].RD_Q10; + Winner_ind = 0; + for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) { + if( psDelDec[ k ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psDelDec[ k ].RD_Q10; + Winner_ind = k; + } + } + + /* Copy final part of signals from winner state to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + psIndices->Seed = psDD->SeedInit; + last_smple_idx = smpl_buf_idx + decisionDelay; + Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 ); + for( i = 0; i < decisionDelay; i++ ) { + last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY; + if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY; + + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gain_Q10 ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ]; + } + silk_memcpy( NSQ->sLPC_Q14, &psDD->sLPC_Q14[ psEncC->subfr_length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + silk_memcpy( NSQ->sAR2_Q14, psDD->sAR2_Q14, sizeof( psDD->sAR2_Q14 ) ); + + /* Update states */ + NSQ->sLF_AR_shp_Q14 = psDD->LF_AR_Q14; + NSQ->sDiff_shp_Q14 = psDD->Diff_Q14; + NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; + + /* Save quantized speech signal */ + silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); + silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); + RESTORE_STACK; +} + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +#ifndef OVERRIDE_silk_noise_shape_quantizer_del_dec +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay, /* I */ + int arch /* I */ +) +{ + opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx; + opus_int32 Winner_rand_state; + opus_int32 LTP_pred_Q14, LPC_pred_Q14, n_AR_Q14, n_LTP_Q14; + opus_int32 n_LF_Q14, r_Q10, rr_Q10, rd1_Q10, rd2_Q10, RDmin_Q10, RDmax_Q10; + opus_int32 q1_Q0, q1_Q10, q2_Q10, exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10; + opus_int32 tmp1, tmp2, sLF_AR_shp_Q14; + opus_int32 *pred_lag_ptr, *shp_lag_ptr, *psLPC_Q14; +#ifdef silk_short_prediction_create_arch_coef + opus_int32 a_Q12_arch[MAX_LPC_ORDER]; +#endif + + VARDECL( NSQ_sample_pair, psSampleState ); + NSQ_del_dec_struct *psDD; + NSQ_sample_struct *psSS; + SAVE_STACK; + + celt_assert( nStatesDelayedDecision > 0 ); + ALLOC( psSampleState, nStatesDelayedDecision, NSQ_sample_pair ); + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + +#ifdef silk_short_prediction_create_arch_coef + silk_short_prediction_create_arch_coef(a_Q12_arch, a_Q12, predictLPCOrder); +#endif + + for( i = 0; i < length; i++ ) { + /* Perform common calculations used in all states */ + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q14 = 2; + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ 0 ], b_Q14[ 0 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -1 ], b_Q14[ 1 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -2 ], b_Q14[ 2 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -3 ], b_Q14[ 3 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); + LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */ + pred_lag_ptr++; + } else { + LTP_pred_Q14 = 0; + } + + /* Long-term shaping */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q14 = silk_SMULWB( silk_ADD_SAT32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */ + shp_lag_ptr++; + } else { + n_LTP_Q14 = 0; + } + + for( k = 0; k < nStatesDelayedDecision; k++ ) { + /* Delayed decision state */ + psDD = &psDelDec[ k ]; + + /* Sample state */ + psSS = psSampleState[ k ]; + + /* Generate dither */ + psDD->Seed = silk_RAND( psDD->Seed ); + + /* Pointer used in short term prediction and shaping */ + psLPC_Q14 = &psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 + i ]; + /* Short-term prediction */ + LPC_pred_Q14 = silk_noise_shape_quantizer_short_prediction(psLPC_Q14, a_Q12, a_Q12_arch, predictLPCOrder, arch); + LPC_pred_Q14 = silk_LSHIFT( LPC_pred_Q14, 4 ); /* Q10 -> Q14 */ + + /* Noise shape feedback */ + celt_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ + /* Output of lowpass section */ + tmp2 = silk_SMLAWB( psDD->Diff_Q14, psDD->sAR2_Q14[ 0 ], warping_Q16 ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ 0 ], psDD->sAR2_Q14[ 1 ] - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ 0 ] = tmp2; + n_AR_Q14 = silk_RSHIFT( shapingLPCOrder, 1 ); + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ 0 ] ); + /* Loop over allpass sections */ + for( j = 2; j < shapingLPCOrder; j += 2 ) { + /* Output of allpass section */ + tmp2 = silk_SMLAWB( psDD->sAR2_Q14[ j - 1 ], psDD->sAR2_Q14[ j + 0 ] - tmp1, warping_Q16 ); + psDD->sAR2_Q14[ j - 1 ] = tmp1; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ j - 1 ] ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ j + 0 ], psDD->sAR2_Q14[ j + 1 ] - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ j + 0 ] = tmp2; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ j ] ); + } + psDD->sAR2_Q14[ shapingLPCOrder - 1 ] = tmp1; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ shapingLPCOrder - 1 ] ); + + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 1 ); /* Q11 -> Q12 */ + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, psDD->LF_AR_Q14, Tilt_Q14 ); /* Q12 */ + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 2 ); /* Q12 -> Q14 */ + + n_LF_Q14 = silk_SMULWB( psDD->Shape_Q14[ *smpl_buf_idx ], LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_SMLAWT( n_LF_Q14, psDD->LF_AR_Q14, LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_LSHIFT( n_LF_Q14, 2 ); /* Q12 -> Q14 */ + + /* Input minus prediction plus noise feedback */ + /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */ + tmp1 = silk_ADD_SAT32( n_AR_Q14, n_LF_Q14 ); /* Q14 */ + tmp2 = silk_ADD32( n_LTP_Q14, LPC_pred_Q14 ); /* Q13 */ + tmp1 = silk_SUB_SAT32( tmp2, tmp1 ); /* Q13 */ + tmp1 = silk_RSHIFT_ROUND( tmp1, 4 ); /* Q10 */ + + r_Q10 = silk_SUB32( x_Q10[ i ], tmp1 ); /* residual error Q10 */ + + /* Flip sign depending on dither */ + if ( psDD->Seed < 0 ) { + r_Q10 = -r_Q10; + } + r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + q1_Q10 = silk_SUB32( r_Q10, offset_Q10 ); + q1_Q0 = silk_RSHIFT( q1_Q10, 10 ); + if (Lambda_Q10 > 2048) { + /* For aggressive RDO, the bias becomes more than one pulse. */ + int rdo_offset = Lambda_Q10/2 - 512; + if (q1_Q10 > rdo_offset) { + q1_Q0 = silk_RSHIFT( q1_Q10 - rdo_offset, 10 ); + } else if (q1_Q10 < -rdo_offset) { + q1_Q0 = silk_RSHIFT( q1_Q10 + rdo_offset, 10 ); + } else if (q1_Q10 < 0) { + q1_Q0 = -1; + } else { + q1_Q0 = 0; + } + } + if( q1_Q0 > 0 ) { + q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == 0 ) { + q1_Q10 = offset_Q10; + q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == -1 ) { + q2_Q10 = offset_Q10; + q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else { /* q1_Q0 < -1 */ + q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( -q2_Q10, Lambda_Q10 ); + } + rr_Q10 = silk_SUB32( r_Q10, q1_Q10 ); + rd1_Q10 = silk_RSHIFT( silk_SMLABB( rd1_Q10, rr_Q10, rr_Q10 ), 10 ); + rr_Q10 = silk_SUB32( r_Q10, q2_Q10 ); + rd2_Q10 = silk_RSHIFT( silk_SMLABB( rd2_Q10, rr_Q10, rr_Q10 ), 10 ); + + if( rd1_Q10 < rd2_Q10 ) { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 0 ].Q_Q10 = q1_Q10; + psSS[ 1 ].Q_Q10 = q2_Q10; + } else { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 0 ].Q_Q10 = q2_Q10; + psSS[ 1 ].Q_Q10 = q1_Q10; + } + + /* Update states for best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 0 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + psSS[ 0 ].Diff_Q14 = silk_SUB_LSHIFT32( xq_Q14, x_Q10[ i ], 4 ); + sLF_AR_shp_Q14 = silk_SUB32( psSS[ 0 ].Diff_Q14, n_AR_Q14 ); + psSS[ 0 ].sLTP_shp_Q14 = silk_SUB_SAT32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 0 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 0 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 0 ].xq_Q14 = xq_Q14; + + /* Update states for second best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 1 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + psSS[ 1 ].Diff_Q14 = silk_SUB_LSHIFT32( xq_Q14, x_Q10[ i ], 4 ); + sLF_AR_shp_Q14 = silk_SUB32( psSS[ 1 ].Diff_Q14, n_AR_Q14 ); + psSS[ 1 ].sLTP_shp_Q14 = silk_SUB_SAT32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 1 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 1 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 1 ].xq_Q14 = xq_Q14; + } + + *smpl_buf_idx = ( *smpl_buf_idx - 1 ) % DECISION_DELAY; + if( *smpl_buf_idx < 0 ) *smpl_buf_idx += DECISION_DELAY; + last_smple_idx = ( *smpl_buf_idx + decisionDelay ) % DECISION_DELAY; + + /* Find winner */ + RDmin_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + Winner_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + if( psSampleState[ k ][ 0 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + Winner_ind = k; + } + } + + /* Increase RD values of expired states */ + Winner_rand_state = psDelDec[ Winner_ind ].RandState[ last_smple_idx ]; + for( k = 0; k < nStatesDelayedDecision; k++ ) { + if( psDelDec[ k ].RandState[ last_smple_idx ] != Winner_rand_state ) { + psSampleState[ k ][ 0 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 0 ].RD_Q10, silk_int32_MAX >> 4 ); + psSampleState[ k ][ 1 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 1 ].RD_Q10, silk_int32_MAX >> 4 ); + silk_assert( psSampleState[ k ][ 0 ].RD_Q10 >= 0 ); + } + } + + /* Find worst in first set and best in second set */ + RDmax_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + RDmin_Q10 = psSampleState[ 0 ][ 1 ].RD_Q10; + RDmax_ind = 0; + RDmin_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + /* find worst in first set */ + if( psSampleState[ k ][ 0 ].RD_Q10 > RDmax_Q10 ) { + RDmax_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + RDmax_ind = k; + } + /* find best in second set */ + if( psSampleState[ k ][ 1 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 1 ].RD_Q10; + RDmin_ind = k; + } + } + + /* Replace a state if best from second set outperforms worst in first set */ + if( RDmin_Q10 < RDmax_Q10 ) { + silk_memcpy( ( (opus_int32 *)&psDelDec[ RDmax_ind ] ) + i, + ( (opus_int32 *)&psDelDec[ RDmin_ind ] ) + i, sizeof( NSQ_del_dec_struct ) - i * sizeof( opus_int32) ); + silk_memcpy( &psSampleState[ RDmax_ind ][ 0 ], &psSampleState[ RDmin_ind ][ 1 ], sizeof( NSQ_sample_struct ) ); + } + + /* Write samples from winner to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + if( subfr > 0 || i >= decisionDelay ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], delayedGain_Q10[ last_smple_idx ] ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDD->Shape_Q14[ last_smple_idx ]; + sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDD->Pred_Q15[ last_smple_idx ]; + } + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Update states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + psSS = &psSampleState[ k ][ 0 ]; + psDD->LF_AR_Q14 = psSS->LF_AR_Q14; + psDD->Diff_Q14 = psSS->Diff_Q14; + psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ] = psSS->xq_Q14; + psDD->Xq_Q14[ *smpl_buf_idx ] = psSS->xq_Q14; + psDD->Q_Q10[ *smpl_buf_idx ] = psSS->Q_Q10; + psDD->Pred_Q15[ *smpl_buf_idx ] = silk_LSHIFT32( psSS->LPC_exc_Q14, 1 ); + psDD->Shape_Q14[ *smpl_buf_idx ] = psSS->sLTP_shp_Q14; + psDD->Seed = silk_ADD32_ovflw( psDD->Seed, silk_RSHIFT_ROUND( psSS->Q_Q10, 10 ) ); + psDD->RandState[ *smpl_buf_idx ] = psDD->Seed; + psDD->RD_Q10 = psSS->RD_Q10; + } + delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10; + } + /* Update LPC states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + silk_memcpy( psDD->sLPC_Q14, &psDD->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + } + RESTORE_STACK; +} +#endif /* OVERRIDE_silk_noise_shape_quantizer_del_dec */ + +static OPUS_INLINE void silk_nsq_del_dec_scale_states( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int16 x16[], /* I Input */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + opus_int nStatesDelayedDecision, /* I Number of del dec states */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +) +{ + opus_int i, k, lag; + opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q26; + NSQ_del_dec_struct *psDD; + + lag = pitchL[ subfr ]; + inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); + silk_assert( inv_gain_Q31 != 0 ); + + /* Scale input */ + inv_gain_Q26 = silk_RSHIFT_ROUND( inv_gain_Q31, 5 ); + for( i = 0; i < psEncC->subfr_length; i++ ) { + x_sc_Q10[ i ] = silk_SMULWW( x16[ i ], inv_gain_Q26 ); + } + + /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ + if( NSQ->rewhite_flag ) { + if( subfr == 0 ) { + /* Do LTP downscaling */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); + } + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + silk_assert( i < MAX_FRAME_LENGTH ); + sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] ); + } + } + + /* Adjust for changing gain */ + if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { + gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); + + /* Scale long-term shaping state */ + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + + /* Scale scalar states */ + psDD->LF_AR_Q14 = silk_SMULWW( gain_adj_Q16, psDD->LF_AR_Q14 ); + psDD->Diff_Q14 = silk_SMULWW( gain_adj_Q16, psDD->Diff_Q14 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + psDD->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sLPC_Q14[ i ] ); + } + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + psDD->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sAR2_Q14[ i ] ); + } + for( i = 0; i < DECISION_DELAY; i++ ) { + psDD->Pred_Q15[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Pred_Q15[ i ] ); + psDD->Shape_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Shape_Q14[ i ] ); + } + } + + /* Save inverse gain */ + NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.c new file mode 100644 index 000000000..4667440db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.c @@ -0,0 +1,446 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" +#include "PLC.h" + +#define NB_ATT 2 +static const opus_int16 HARM_ATT_Q15[NB_ATT] = { 32440, 31130 }; /* 0.99, 0.95 */ +static const opus_int16 PLC_RAND_ATTENUATE_V_Q15[NB_ATT] = { 31130, 26214 }; /* 0.95, 0.8 */ +static const opus_int16 PLC_RAND_ATTENUATE_UV_Q15[NB_ATT] = { 32440, 29491 }; /* 0.99, 0.9 */ + +static OPUS_INLINE void silk_PLC_update( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl /* I/O Decoder control */ +); + +static OPUS_INLINE void silk_PLC_conceal( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* O LPC residual signal */ + int arch /* I Run-time architecture */ +); + + +void silk_PLC_Reset( + silk_decoder_state *psDec /* I/O Decoder state */ +) +{ + psDec->sPLC.pitchL_Q8 = silk_LSHIFT( psDec->frame_length, 8 - 1 ); + psDec->sPLC.prevGain_Q16[ 0 ] = SILK_FIX_CONST( 1, 16 ); + psDec->sPLC.prevGain_Q16[ 1 ] = SILK_FIX_CONST( 1, 16 ); + psDec->sPLC.subfr_length = 20; + psDec->sPLC.nb_subfr = 2; +} + +void silk_PLC( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* I/O signal */ + opus_int lost, /* I Loss flag */ + int arch /* I Run-time architecture */ +) +{ + /* PLC control function */ + if( psDec->fs_kHz != psDec->sPLC.fs_kHz ) { + silk_PLC_Reset( psDec ); + psDec->sPLC.fs_kHz = psDec->fs_kHz; + } + + if( lost ) { + /****************************/ + /* Generate Signal */ + /****************************/ + silk_PLC_conceal( psDec, psDecCtrl, frame, arch ); + + psDec->lossCnt++; + } else { + /****************************/ + /* Update state */ + /****************************/ + silk_PLC_update( psDec, psDecCtrl ); + } +} + +/**************************************************/ +/* Update state of PLC */ +/**************************************************/ +static OPUS_INLINE void silk_PLC_update( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl /* I/O Decoder control */ +) +{ + opus_int32 LTP_Gain_Q14, temp_LTP_Gain_Q14; + opus_int i, j; + silk_PLC_struct *psPLC; + + psPLC = &psDec->sPLC; + + /* Update parameters used in case of packet loss */ + psDec->prevSignalType = psDec->indices.signalType; + LTP_Gain_Q14 = 0; + if( psDec->indices.signalType == TYPE_VOICED ) { + /* Find the parameters for the last subframe which contains a pitch pulse */ + for( j = 0; j * psDec->subfr_length < psDecCtrl->pitchL[ psDec->nb_subfr - 1 ]; j++ ) { + if( j == psDec->nb_subfr ) { + break; + } + temp_LTP_Gain_Q14 = 0; + for( i = 0; i < LTP_ORDER; i++ ) { + temp_LTP_Gain_Q14 += psDecCtrl->LTPCoef_Q14[ ( psDec->nb_subfr - 1 - j ) * LTP_ORDER + i ]; + } + if( temp_LTP_Gain_Q14 > LTP_Gain_Q14 ) { + LTP_Gain_Q14 = temp_LTP_Gain_Q14; + silk_memcpy( psPLC->LTPCoef_Q14, + &psDecCtrl->LTPCoef_Q14[ silk_SMULBB( psDec->nb_subfr - 1 - j, LTP_ORDER ) ], + LTP_ORDER * sizeof( opus_int16 ) ); + + psPLC->pitchL_Q8 = silk_LSHIFT( psDecCtrl->pitchL[ psDec->nb_subfr - 1 - j ], 8 ); + } + } + + silk_memset( psPLC->LTPCoef_Q14, 0, LTP_ORDER * sizeof( opus_int16 ) ); + psPLC->LTPCoef_Q14[ LTP_ORDER / 2 ] = LTP_Gain_Q14; + + /* Limit LT coefs */ + if( LTP_Gain_Q14 < V_PITCH_GAIN_START_MIN_Q14 ) { + opus_int scale_Q10; + opus_int32 tmp; + + tmp = silk_LSHIFT( V_PITCH_GAIN_START_MIN_Q14, 10 ); + scale_Q10 = silk_DIV32( tmp, silk_max( LTP_Gain_Q14, 1 ) ); + for( i = 0; i < LTP_ORDER; i++ ) { + psPLC->LTPCoef_Q14[ i ] = silk_RSHIFT( silk_SMULBB( psPLC->LTPCoef_Q14[ i ], scale_Q10 ), 10 ); + } + } else if( LTP_Gain_Q14 > V_PITCH_GAIN_START_MAX_Q14 ) { + opus_int scale_Q14; + opus_int32 tmp; + + tmp = silk_LSHIFT( V_PITCH_GAIN_START_MAX_Q14, 14 ); + scale_Q14 = silk_DIV32( tmp, silk_max( LTP_Gain_Q14, 1 ) ); + for( i = 0; i < LTP_ORDER; i++ ) { + psPLC->LTPCoef_Q14[ i ] = silk_RSHIFT( silk_SMULBB( psPLC->LTPCoef_Q14[ i ], scale_Q14 ), 14 ); + } + } + } else { + psPLC->pitchL_Q8 = silk_LSHIFT( silk_SMULBB( psDec->fs_kHz, 18 ), 8 ); + silk_memset( psPLC->LTPCoef_Q14, 0, LTP_ORDER * sizeof( opus_int16 )); + } + + /* Save LPC coeficients */ + silk_memcpy( psPLC->prevLPC_Q12, psDecCtrl->PredCoef_Q12[ 1 ], psDec->LPC_order * sizeof( opus_int16 ) ); + psPLC->prevLTP_scale_Q14 = psDecCtrl->LTP_scale_Q14; + + /* Save last two gains */ + silk_memcpy( psPLC->prevGain_Q16, &psDecCtrl->Gains_Q16[ psDec->nb_subfr - 2 ], 2 * sizeof( opus_int32 ) ); + + psPLC->subfr_length = psDec->subfr_length; + psPLC->nb_subfr = psDec->nb_subfr; +} + +static OPUS_INLINE void silk_PLC_energy(opus_int32 *energy1, opus_int *shift1, opus_int32 *energy2, opus_int *shift2, + const opus_int32 *exc_Q14, const opus_int32 *prevGain_Q10, int subfr_length, int nb_subfr) +{ + int i, k; + VARDECL( opus_int16, exc_buf ); + opus_int16 *exc_buf_ptr; + SAVE_STACK; + ALLOC( exc_buf, 2*subfr_length, opus_int16 ); + /* Find random noise component */ + /* Scale previous excitation signal */ + exc_buf_ptr = exc_buf; + for( k = 0; k < 2; k++ ) { + for( i = 0; i < subfr_length; i++ ) { + exc_buf_ptr[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT( + silk_SMULWW( exc_Q14[ i + ( k + nb_subfr - 2 ) * subfr_length ], prevGain_Q10[ k ] ), 8 ) ); + } + exc_buf_ptr += subfr_length; + } + /* Find the subframe with lowest energy of the last two and use that as random noise generator */ + silk_sum_sqr_shift( energy1, shift1, exc_buf, subfr_length ); + silk_sum_sqr_shift( energy2, shift2, &exc_buf[ subfr_length ], subfr_length ); + RESTORE_STACK; +} + +static OPUS_INLINE void silk_PLC_conceal( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* O LPC residual signal */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, j, k; + opus_int lag, idx, sLTP_buf_idx, shift1, shift2; + opus_int32 rand_seed, harm_Gain_Q15, rand_Gain_Q15, inv_gain_Q30; + opus_int32 energy1, energy2, *rand_ptr, *pred_lag_ptr; + opus_int32 LPC_pred_Q10, LTP_pred_Q12; + opus_int16 rand_scale_Q14; + opus_int16 *B_Q14; + opus_int32 *sLPC_Q14_ptr; + opus_int16 A_Q12[ MAX_LPC_ORDER ]; +#ifdef SMALL_FOOTPRINT + opus_int16 *sLTP; +#else + VARDECL( opus_int16, sLTP ); +#endif + VARDECL( opus_int32, sLTP_Q14 ); + silk_PLC_struct *psPLC = &psDec->sPLC; + opus_int32 prevGain_Q10[2]; + SAVE_STACK; + + ALLOC( sLTP_Q14, psDec->ltp_mem_length + psDec->frame_length, opus_int32 ); +#ifdef SMALL_FOOTPRINT + /* Ugly hack that breaks aliasing rules to save stack: put sLTP at the very end of sLTP_Q14. */ + sLTP = ((opus_int16*)&sLTP_Q14[psDec->ltp_mem_length + psDec->frame_length])-psDec->ltp_mem_length; +#else + ALLOC( sLTP, psDec->ltp_mem_length, opus_int16 ); +#endif + + prevGain_Q10[0] = silk_RSHIFT( psPLC->prevGain_Q16[ 0 ], 6); + prevGain_Q10[1] = silk_RSHIFT( psPLC->prevGain_Q16[ 1 ], 6); + + if( psDec->first_frame_after_reset ) { + silk_memset( psPLC->prevLPC_Q12, 0, sizeof( psPLC->prevLPC_Q12 ) ); + } + + silk_PLC_energy(&energy1, &shift1, &energy2, &shift2, psDec->exc_Q14, prevGain_Q10, psDec->subfr_length, psDec->nb_subfr); + + if( silk_RSHIFT( energy1, shift2 ) < silk_RSHIFT( energy2, shift1 ) ) { + /* First sub-frame has lowest energy */ + rand_ptr = &psDec->exc_Q14[ silk_max_int( 0, ( psPLC->nb_subfr - 1 ) * psPLC->subfr_length - RAND_BUF_SIZE ) ]; + } else { + /* Second sub-frame has lowest energy */ + rand_ptr = &psDec->exc_Q14[ silk_max_int( 0, psPLC->nb_subfr * psPLC->subfr_length - RAND_BUF_SIZE ) ]; + } + + /* Set up Gain to random noise component */ + B_Q14 = psPLC->LTPCoef_Q14; + rand_scale_Q14 = psPLC->randScale_Q14; + + /* Set up attenuation gains */ + harm_Gain_Q15 = HARM_ATT_Q15[ silk_min_int( NB_ATT - 1, psDec->lossCnt ) ]; + if( psDec->prevSignalType == TYPE_VOICED ) { + rand_Gain_Q15 = PLC_RAND_ATTENUATE_V_Q15[ silk_min_int( NB_ATT - 1, psDec->lossCnt ) ]; + } else { + rand_Gain_Q15 = PLC_RAND_ATTENUATE_UV_Q15[ silk_min_int( NB_ATT - 1, psDec->lossCnt ) ]; + } + + /* LPC concealment. Apply BWE to previous LPC */ + silk_bwexpander( psPLC->prevLPC_Q12, psDec->LPC_order, SILK_FIX_CONST( BWE_COEF, 16 ) ); + + /* Preload LPC coeficients to array on stack. Gives small performance gain */ + silk_memcpy( A_Q12, psPLC->prevLPC_Q12, psDec->LPC_order * sizeof( opus_int16 ) ); + + /* First Lost frame */ + if( psDec->lossCnt == 0 ) { + rand_scale_Q14 = 1 << 14; + + /* Reduce random noise Gain for voiced frames */ + if( psDec->prevSignalType == TYPE_VOICED ) { + for( i = 0; i < LTP_ORDER; i++ ) { + rand_scale_Q14 -= B_Q14[ i ]; + } + rand_scale_Q14 = silk_max_16( 3277, rand_scale_Q14 ); /* 0.2 */ + rand_scale_Q14 = (opus_int16)silk_RSHIFT( silk_SMULBB( rand_scale_Q14, psPLC->prevLTP_scale_Q14 ), 14 ); + } else { + /* Reduce random noise for unvoiced frames with high LPC gain */ + opus_int32 invGain_Q30, down_scale_Q30; + + invGain_Q30 = silk_LPC_inverse_pred_gain( psPLC->prevLPC_Q12, psDec->LPC_order, arch ); + + down_scale_Q30 = silk_min_32( silk_RSHIFT( (opus_int32)1 << 30, LOG2_INV_LPC_GAIN_HIGH_THRES ), invGain_Q30 ); + down_scale_Q30 = silk_max_32( silk_RSHIFT( (opus_int32)1 << 30, LOG2_INV_LPC_GAIN_LOW_THRES ), down_scale_Q30 ); + down_scale_Q30 = silk_LSHIFT( down_scale_Q30, LOG2_INV_LPC_GAIN_HIGH_THRES ); + + rand_Gain_Q15 = silk_RSHIFT( silk_SMULWB( down_scale_Q30, rand_Gain_Q15 ), 14 ); + } + } + + rand_seed = psPLC->rand_seed; + lag = silk_RSHIFT_ROUND( psPLC->pitchL_Q8, 8 ); + sLTP_buf_idx = psDec->ltp_mem_length; + + /* Rewhiten LTP state */ + idx = psDec->ltp_mem_length - lag - psDec->LPC_order - LTP_ORDER / 2; + celt_assert( idx > 0 ); + silk_LPC_analysis_filter( &sLTP[ idx ], &psDec->outBuf[ idx ], A_Q12, psDec->ltp_mem_length - idx, psDec->LPC_order, arch ); + /* Scale LTP state */ + inv_gain_Q30 = silk_INVERSE32_varQ( psPLC->prevGain_Q16[ 1 ], 46 ); + inv_gain_Q30 = silk_min( inv_gain_Q30, silk_int32_MAX >> 1 ); + for( i = idx + psDec->LPC_order; i < psDec->ltp_mem_length; i++ ) { + sLTP_Q14[ i ] = silk_SMULWB( inv_gain_Q30, sLTP[ i ] ); + } + + /***************************/ + /* LTP synthesis filtering */ + /***************************/ + for( k = 0; k < psDec->nb_subfr; k++ ) { + /* Set up pointer */ + pred_lag_ptr = &sLTP_Q14[ sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + for( i = 0; i < psDec->subfr_length; i++ ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q12 = 2; + LTP_pred_Q12 = silk_SMLAWB( LTP_pred_Q12, pred_lag_ptr[ 0 ], B_Q14[ 0 ] ); + LTP_pred_Q12 = silk_SMLAWB( LTP_pred_Q12, pred_lag_ptr[ -1 ], B_Q14[ 1 ] ); + LTP_pred_Q12 = silk_SMLAWB( LTP_pred_Q12, pred_lag_ptr[ -2 ], B_Q14[ 2 ] ); + LTP_pred_Q12 = silk_SMLAWB( LTP_pred_Q12, pred_lag_ptr[ -3 ], B_Q14[ 3 ] ); + LTP_pred_Q12 = silk_SMLAWB( LTP_pred_Q12, pred_lag_ptr[ -4 ], B_Q14[ 4 ] ); + pred_lag_ptr++; + + /* Generate LPC excitation */ + rand_seed = silk_RAND( rand_seed ); + idx = silk_RSHIFT( rand_seed, 25 ) & RAND_BUF_MASK; + sLTP_Q14[ sLTP_buf_idx ] = silk_LSHIFT32( silk_SMLAWB( LTP_pred_Q12, rand_ptr[ idx ], rand_scale_Q14 ), 2 ); + sLTP_buf_idx++; + } + + /* Gradually reduce LTP gain */ + for( j = 0; j < LTP_ORDER; j++ ) { + B_Q14[ j ] = silk_RSHIFT( silk_SMULBB( harm_Gain_Q15, B_Q14[ j ] ), 15 ); + } + /* Gradually reduce excitation gain */ + rand_scale_Q14 = silk_RSHIFT( silk_SMULBB( rand_scale_Q14, rand_Gain_Q15 ), 15 ); + + /* Slowly increase pitch lag */ + psPLC->pitchL_Q8 = silk_SMLAWB( psPLC->pitchL_Q8, psPLC->pitchL_Q8, PITCH_DRIFT_FAC_Q16 ); + psPLC->pitchL_Q8 = silk_min_32( psPLC->pitchL_Q8, silk_LSHIFT( silk_SMULBB( MAX_PITCH_LAG_MS, psDec->fs_kHz ), 8 ) ); + lag = silk_RSHIFT_ROUND( psPLC->pitchL_Q8, 8 ); + } + + /***************************/ + /* LPC synthesis filtering */ + /***************************/ + sLPC_Q14_ptr = &sLTP_Q14[ psDec->ltp_mem_length - MAX_LPC_ORDER ]; + + /* Copy LPC state */ + silk_memcpy( sLPC_Q14_ptr, psDec->sLPC_Q14_buf, MAX_LPC_ORDER * sizeof( opus_int32 ) ); + + celt_assert( psDec->LPC_order >= 10 ); /* check that unrolling works */ + for( i = 0; i < psDec->frame_length; i++ ) { + /* partly unrolled */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q10 = silk_RSHIFT( psDec->LPC_order, 1 ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 1 ], A_Q12[ 0 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 2 ], A_Q12[ 1 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 3 ], A_Q12[ 2 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 4 ], A_Q12[ 3 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 5 ], A_Q12[ 4 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 6 ], A_Q12[ 5 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 7 ], A_Q12[ 6 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 8 ], A_Q12[ 7 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 9 ], A_Q12[ 8 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - 10 ], A_Q12[ 9 ] ); + for( j = 10; j < psDec->LPC_order; j++ ) { + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14_ptr[ MAX_LPC_ORDER + i - j - 1 ], A_Q12[ j ] ); + } + + /* Add prediction to LPC excitation */ + sLPC_Q14_ptr[ MAX_LPC_ORDER + i ] = silk_ADD_SAT32( sLPC_Q14_ptr[ MAX_LPC_ORDER + i ], + silk_LSHIFT_SAT32( LPC_pred_Q10, 4 )); + + /* Scale with Gain */ + frame[ i ] = (opus_int16)silk_SAT16( silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( sLPC_Q14_ptr[ MAX_LPC_ORDER + i ], prevGain_Q10[ 1 ] ), 8 ) ) ); + } + + /* Save LPC state */ + silk_memcpy( psDec->sLPC_Q14_buf, &sLPC_Q14_ptr[ psDec->frame_length ], MAX_LPC_ORDER * sizeof( opus_int32 ) ); + + /**************************************/ + /* Update states */ + /**************************************/ + psPLC->rand_seed = rand_seed; + psPLC->randScale_Q14 = rand_scale_Q14; + for( i = 0; i < MAX_NB_SUBFR; i++ ) { + psDecCtrl->pitchL[ i ] = lag; + } + RESTORE_STACK; +} + +/* Glues concealed frames with new good received frames */ +void silk_PLC_glue_frames( + silk_decoder_state *psDec, /* I/O decoder state */ + opus_int16 frame[], /* I/O signal */ + opus_int length /* I length of signal */ +) +{ + opus_int i, energy_shift; + opus_int32 energy; + silk_PLC_struct *psPLC; + psPLC = &psDec->sPLC; + + if( psDec->lossCnt ) { + /* Calculate energy in concealed residual */ + silk_sum_sqr_shift( &psPLC->conc_energy, &psPLC->conc_energy_shift, frame, length ); + + psPLC->last_frame_lost = 1; + } else { + if( psDec->sPLC.last_frame_lost ) { + /* Calculate residual in decoded signal if last frame was lost */ + silk_sum_sqr_shift( &energy, &energy_shift, frame, length ); + + /* Normalize energies */ + if( energy_shift > psPLC->conc_energy_shift ) { + psPLC->conc_energy = silk_RSHIFT( psPLC->conc_energy, energy_shift - psPLC->conc_energy_shift ); + } else if( energy_shift < psPLC->conc_energy_shift ) { + energy = silk_RSHIFT( energy, psPLC->conc_energy_shift - energy_shift ); + } + + /* Fade in the energy difference */ + if( energy > psPLC->conc_energy ) { + opus_int32 frac_Q24, LZ; + opus_int32 gain_Q16, slope_Q16; + + LZ = silk_CLZ32( psPLC->conc_energy ); + LZ = LZ - 1; + psPLC->conc_energy = silk_LSHIFT( psPLC->conc_energy, LZ ); + energy = silk_RSHIFT( energy, silk_max_32( 24 - LZ, 0 ) ); + + frac_Q24 = silk_DIV32( psPLC->conc_energy, silk_max( energy, 1 ) ); + + gain_Q16 = silk_LSHIFT( silk_SQRT_APPROX( frac_Q24 ), 4 ); + slope_Q16 = silk_DIV32_16( ( (opus_int32)1 << 16 ) - gain_Q16, length ); + /* Make slope 4x steeper to avoid missing onsets after DTX */ + slope_Q16 = silk_LSHIFT( slope_Q16, 2 ); + + for( i = 0; i < length; i++ ) { + frame[ i ] = silk_SMULWB( gain_Q16, frame[ i ] ); + gain_Q16 += slope_Q16; + if( gain_Q16 > (opus_int32)1 << 16 ) { + break; + } + } + } + } + psPLC->last_frame_lost = 0; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.h new file mode 100644 index 000000000..6438f5163 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/PLC.h @@ -0,0 +1,62 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_PLC_H +#define SILK_PLC_H + +#include "main.h" + +#define BWE_COEF 0.99 +#define V_PITCH_GAIN_START_MIN_Q14 11469 /* 0.7 in Q14 */ +#define V_PITCH_GAIN_START_MAX_Q14 15565 /* 0.95 in Q14 */ +#define MAX_PITCH_LAG_MS 18 +#define RAND_BUF_SIZE 128 +#define RAND_BUF_MASK ( RAND_BUF_SIZE - 1 ) +#define LOG2_INV_LPC_GAIN_HIGH_THRES 3 /* 2^3 = 8 dB LPC gain */ +#define LOG2_INV_LPC_GAIN_LOW_THRES 8 /* 2^8 = 24 dB LPC gain */ +#define PITCH_DRIFT_FAC_Q16 655 /* 0.01 in Q16 */ + +void silk_PLC_Reset( + silk_decoder_state *psDec /* I/O Decoder state */ +); + +void silk_PLC( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* I/O signal */ + opus_int lost, /* I Loss flag */ + int arch /* I Run-time architecture */ +); + +void silk_PLC_glue_frames( + silk_decoder_state *psDec, /* I/O decoder state */ + opus_int16 frame[], /* I/O signal */ + opus_int length /* I length of signal */ +); + +#endif + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/SigProc_FIX.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/SigProc_FIX.h new file mode 100644 index 000000000..f9ae32632 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/SigProc_FIX.h @@ -0,0 +1,641 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_SIGPROC_FIX_H +#define SILK_SIGPROC_FIX_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/*#define silk_MACRO_COUNT */ /* Used to enable WMOPS counting */ + +#define SILK_MAX_ORDER_LPC 24 /* max order of the LPC analysis in schur() and k2a() */ + +#include /* for memset(), memcpy(), memmove() */ +#include "typedef.h" +#include "resampler_structs.h" +#include "macros.h" +#include "cpu_support.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#include "x86/SigProc_FIX_sse.h" +#endif + +#if (defined(OPUS_ARM_ASM) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)) +#include "arm/biquad_alt_arm.h" +#include "arm/LPC_inv_pred_gain_arm.h" +#endif + +/********************************************************************/ +/* SIGNAL PROCESSING FUNCTIONS */ +/********************************************************************/ + +/*! + * Initialize/reset the resampler state for a given pair of input/output sampling rates +*/ +opus_int silk_resampler_init( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int32 Fs_Hz_in, /* I Input sampling rate (Hz) */ + opus_int32 Fs_Hz_out, /* I Output sampling rate (Hz) */ + opus_int forEnc /* I If 1: encoder; if 0: decoder */ +); + +/*! + * Resampler: convert from one sampling rate to another + */ +opus_int silk_resampler( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +); + +/*! +* Downsample 2x, mediocre quality +*/ +void silk_resampler_down2( + opus_int32 *S, /* I/O State vector [ 2 ] */ + opus_int16 *out, /* O Output signal [ len ] */ + const opus_int16 *in, /* I Input signal [ floor(len/2) ] */ + opus_int32 inLen /* I Number of input samples */ +); + +/*! + * Downsample by a factor 2/3, low quality +*/ +void silk_resampler_down2_3( + opus_int32 *S, /* I/O State vector [ 6 ] */ + opus_int16 *out, /* O Output signal [ floor(2*inLen/3) ] */ + const opus_int16 *in, /* I Input signal [ inLen ] */ + opus_int32 inLen /* I Number of input samples */ +); + +/*! + * second order ARMA filter; + * slower than biquad() but uses more precise coefficients + * can handle (slowly) varying coefficients + */ +void silk_biquad_alt_stride1( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [2] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +); + +void silk_biquad_alt_stride2_c( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +); + +/* Variable order MA prediction error filter. */ +void silk_LPC_analysis_filter( + opus_int16 *out, /* O Output signal */ + const opus_int16 *in, /* I Input signal */ + const opus_int16 *B, /* I MA prediction coefficients, Q12 [order] */ + const opus_int32 len, /* I Signal length */ + const opus_int32 d, /* I Filter order */ + int arch /* I Run-time architecture */ +); + +/* Chirp (bandwidth expand) LP AR filter */ +void silk_bwexpander( + opus_int16 *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I Length of ar */ + opus_int32 chirp_Q16 /* I Chirp factor (typically in the range 0 to 1) */ +); + +/* Chirp (bandwidth expand) LP AR filter */ +void silk_bwexpander_32( + opus_int32 *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I Length of ar */ + opus_int32 chirp_Q16 /* I Chirp factor in Q16 */ +); + +/* Compute inverse of LPC prediction gain, and */ +/* test if LPC coefficients are stable (all poles within unit circle) */ +opus_int32 silk_LPC_inverse_pred_gain_c( /* O Returns inverse prediction gain in energy domain, Q30 */ + const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ + const opus_int order /* I Prediction order */ +); + +/* Split signal in two decimated bands using first-order allpass filters */ +void silk_ana_filt_bank_1( + const opus_int16 *in, /* I Input signal [N] */ + opus_int32 *S, /* I/O State vector [2] */ + opus_int16 *outL, /* O Low band [N/2] */ + opus_int16 *outH, /* O High band [N/2] */ + const opus_int32 N /* I Number of input samples */ +); + +#if !defined(OVERRIDE_silk_biquad_alt_stride2) +#define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((void)(arch), silk_biquad_alt_stride2_c(in, B_Q28, A_Q28, S, out, len)) +#endif + +#if !defined(OVERRIDE_silk_LPC_inverse_pred_gain) +#define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((void)(arch), silk_LPC_inverse_pred_gain_c(A_Q12, order)) +#endif + +/********************************************************************/ +/* SCALAR FUNCTIONS */ +/********************************************************************/ + +/* Approximation of 128 * log2() (exact inverse of approx 2^() below) */ +/* Convert input to a log scale */ +opus_int32 silk_lin2log( + const opus_int32 inLin /* I input in linear scale */ +); + +/* Approximation of a sigmoid function */ +opus_int silk_sigm_Q15( + opus_int in_Q5 /* I */ +); + +/* Approximation of 2^() (exact inverse of approx log2() above) */ +/* Convert input to a linear scale */ +opus_int32 silk_log2lin( + const opus_int32 inLog_Q7 /* I input on log scale */ +); + +/* Compute number of bits to right shift the sum of squares of a vector */ +/* of int16s to make it fit in an int32 */ +void silk_sum_sqr_shift( + opus_int32 *energy, /* O Energy of x, after shifting to the right */ + opus_int *shift, /* O Number of bits right shift applied to energy */ + const opus_int16 *x, /* I Input vector */ + opus_int len /* I Length of input vector */ +); + +/* Calculates the reflection coefficients from the correlation sequence */ +/* Faster than schur64(), but much less accurate. */ +/* uses SMLAWB(), requiring armv5E and higher. */ +opus_int32 silk_schur( /* O Returns residual energy */ + opus_int16 *rc_Q15, /* O reflection coefficients [order] Q15 */ + const opus_int32 *c, /* I correlations [order+1] */ + const opus_int32 order /* I prediction order */ +); + +/* Calculates the reflection coefficients from the correlation sequence */ +/* Slower than schur(), but more accurate. */ +/* Uses SMULL(), available on armv4 */ +opus_int32 silk_schur64( /* O returns residual energy */ + opus_int32 rc_Q16[], /* O Reflection coefficients [order] Q16 */ + const opus_int32 c[], /* I Correlations [order+1] */ + opus_int32 order /* I Prediction order */ +); + +/* Step up function, converts reflection coefficients to prediction coefficients */ +void silk_k2a( + opus_int32 *A_Q24, /* O Prediction coefficients [order] Q24 */ + const opus_int16 *rc_Q15, /* I Reflection coefficients [order] Q15 */ + const opus_int32 order /* I Prediction order */ +); + +/* Step up function, converts reflection coefficients to prediction coefficients */ +void silk_k2a_Q16( + opus_int32 *A_Q24, /* O Prediction coefficients [order] Q24 */ + const opus_int32 *rc_Q16, /* I Reflection coefficients [order] Q16 */ + const opus_int32 order /* I Prediction order */ +); + +/* Apply sine window to signal vector. */ +/* Window types: */ +/* 1 -> sine window from 0 to pi/2 */ +/* 2 -> sine window from pi/2 to pi */ +/* every other sample of window is linearly interpolated, for speed */ +void silk_apply_sine_window( + opus_int16 px_win[], /* O Pointer to windowed signal */ + const opus_int16 px[], /* I Pointer to input signal */ + const opus_int win_type, /* I Selects a window type */ + const opus_int length /* I Window length, multiple of 4 */ +); + +/* Compute autocorrelation */ +void silk_autocorr( + opus_int32 *results, /* O Result (length correlationCount) */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *inputData, /* I Input data to correlate */ + const opus_int inputDataSize, /* I Length of input */ + const opus_int correlationCount, /* I Number of correlation taps to compute */ + int arch /* I Run-time architecture */ +); + +void silk_decode_pitch( + opus_int16 lagIndex, /* I */ + opus_int8 contourIndex, /* O */ + opus_int pitch_lags[], /* O 4 pitch values */ + const opus_int Fs_kHz, /* I sampling frequency (kHz) */ + const opus_int nb_subfr /* I number of sub frames */ +); + +opus_int silk_pitch_analysis_core( /* O Voicing estimate: 0 voiced, 1 unvoiced */ + const opus_int16 *frame, /* I Signal of length PE_FRAME_LENGTH_MS*Fs_kHz */ + opus_int *pitch_out, /* O 4 pitch lag values */ + opus_int16 *lagIndex, /* O Lag Index */ + opus_int8 *contourIndex, /* O Pitch contour Index */ + opus_int *LTPCorr_Q15, /* I/O Normalized correlation; input: value from previous frame */ + opus_int prevLag, /* I Last lag of previous frame; set to zero is unvoiced */ + const opus_int32 search_thres1_Q16, /* I First stage threshold for lag candidates 0 - 1 */ + const opus_int search_thres2_Q13, /* I Final threshold for lag candidates 0 - 1 */ + const opus_int Fs_kHz, /* I Sample frequency (kHz) */ + const opus_int complexity, /* I Complexity setting, 0-2, where 2 is highest */ + const opus_int nb_subfr, /* I number of 5 ms subframes */ + int arch /* I Run-time architecture */ +); + +/* Compute Normalized Line Spectral Frequencies (NLSFs) from whitening filter coefficients */ +/* If not all roots are found, the a_Q16 coefficients are bandwidth expanded until convergence. */ +void silk_A2NLSF( + opus_int16 *NLSF, /* O Normalized Line Spectral Frequencies in Q15 (0..2^15-1) [d] */ + opus_int32 *a_Q16, /* I/O Monic whitening filter coefficients in Q16 [d] */ + const opus_int d /* I Filter order (must be even) */ +); + +/* compute whitening filter coefficients from normalized line spectral frequencies */ +void silk_NLSF2A( + opus_int16 *a_Q12, /* O monic whitening filter coefficients in Q12, [ d ] */ + const opus_int16 *NLSF, /* I normalized line spectral frequencies in Q15, [ d ] */ + const opus_int d, /* I filter order (should be even) */ + int arch /* I Run-time architecture */ +); + +/* Convert int32 coefficients to int16 coefs and make sure there's no wrap-around */ +void silk_LPC_fit( + opus_int16 *a_QOUT, /* O Output signal */ + opus_int32 *a_QIN, /* I/O Input signal */ + const opus_int QOUT, /* I Input Q domain */ + const opus_int QIN, /* I Input Q domain */ + const opus_int d /* I Filter order */ +); + +void silk_insertion_sort_increasing( + opus_int32 *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +); + +void silk_insertion_sort_decreasing_int16( + opus_int16 *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +); + +void silk_insertion_sort_increasing_all_values_int16( + opus_int16 *a, /* I/O Unsorted / Sorted vector */ + const opus_int L /* I Vector length */ +); + +/* NLSF stabilizer, for a single input data vector */ +void silk_NLSF_stabilize( + opus_int16 *NLSF_Q15, /* I/O Unstable/stabilized normalized LSF vector in Q15 [L] */ + const opus_int16 *NDeltaMin_Q15, /* I Min distance vector, NDeltaMin_Q15[L] must be >= 1 [L+1] */ + const opus_int L /* I Number of NLSF parameters in the input vector */ +); + +/* Laroia low complexity NLSF weights */ +void silk_NLSF_VQ_weights_laroia( + opus_int16 *pNLSFW_Q_OUT, /* O Pointer to input vector weights [D] */ + const opus_int16 *pNLSF_Q15, /* I Pointer to input vector [D] */ + const opus_int D /* I Input vector dimension (even) */ +); + +/* Compute reflection coefficients from input signal */ +void silk_burg_modified_c( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */ +); + +/* Copy and multiply a vector by a constant */ +void silk_scale_copy_vector16( + opus_int16 *data_out, + const opus_int16 *data_in, + opus_int32 gain_Q16, /* I Gain in Q16 */ + const opus_int dataSize /* I Length */ +); + +/* Some for the LTP related function requires Q26 to work.*/ +void silk_scale_vector32_Q26_lshift_18( + opus_int32 *data1, /* I/O Q0/Q18 */ + opus_int32 gain_Q26, /* I Q26 */ + opus_int dataSize /* I length */ +); + +/********************************************************************/ +/* INLINE ARM MATH */ +/********************************************************************/ + +/* return sum( inVec1[i] * inVec2[i] ) */ + +opus_int32 silk_inner_prod_aligned( + const opus_int16 *const inVec1, /* I input vector 1 */ + const opus_int16 *const inVec2, /* I input vector 2 */ + const opus_int len, /* I vector lengths */ + int arch /* I Run-time architecture */ +); + + +opus_int32 silk_inner_prod_aligned_scale( + const opus_int16 *const inVec1, /* I input vector 1 */ + const opus_int16 *const inVec2, /* I input vector 2 */ + const opus_int scale, /* I number of bits to shift */ + const opus_int len /* I vector lengths */ +); + +opus_int64 silk_inner_prod16_aligned_64_c( + const opus_int16 *inVec1, /* I input vector 1 */ + const opus_int16 *inVec2, /* I input vector 2 */ + const opus_int len /* I vector lengths */ +); + +/********************************************************************/ +/* MACROS */ +/********************************************************************/ + +/* Rotate a32 right by 'rot' bits. Negative rot values result in rotating + left. Output is 32bit int. + Note: contemporary compilers recognize the C expression below and + compile it into a 'ror' instruction if available. No need for OPUS_INLINE ASM! */ +static OPUS_INLINE opus_int32 silk_ROR32( opus_int32 a32, opus_int rot ) +{ + opus_uint32 x = (opus_uint32) a32; + opus_uint32 r = (opus_uint32) rot; + opus_uint32 m = (opus_uint32) -rot; + if( rot == 0 ) { + return a32; + } else if( rot < 0 ) { + return (opus_int32) ((x << m) | (x >> (32 - m))); + } else { + return (opus_int32) ((x << (32 - r)) | (x >> r)); + } +} + +/* Allocate opus_int16 aligned to 4-byte memory address */ +#if EMBEDDED_ARM +#define silk_DWORD_ALIGN __attribute__((aligned(4))) +#else +#define silk_DWORD_ALIGN +#endif + +/* Useful Macros that can be adjusted to other platforms */ +#define silk_memcpy(dest, src, size) memcpy((dest), (src), (size)) +#define silk_memset(dest, src, size) memset((dest), (src), (size)) +#define silk_memmove(dest, src, size) memmove((dest), (src), (size)) + +/* Fixed point macros */ + +/* (a32 * b32) output have to be 32bit int */ +#define silk_MUL(a32, b32) ((a32) * (b32)) + +/* (a32 * b32) output have to be 32bit uint */ +#define silk_MUL_uint(a32, b32) silk_MUL(a32, b32) + +/* a32 + (b32 * c32) output have to be 32bit int */ +#define silk_MLA(a32, b32, c32) silk_ADD32((a32),((b32) * (c32))) + +/* a32 + (b32 * c32) output have to be 32bit uint */ +#define silk_MLA_uint(a32, b32, c32) silk_MLA(a32, b32, c32) + +/* ((a32 >> 16) * (b32 >> 16)) output have to be 32bit int */ +#define silk_SMULTT(a32, b32) (((a32) >> 16) * ((b32) >> 16)) + +/* a32 + ((a32 >> 16) * (b32 >> 16)) output have to be 32bit int */ +#define silk_SMLATT(a32, b32, c32) silk_ADD32((a32),((b32) >> 16) * ((c32) >> 16)) + +#define silk_SMLALBB(a64, b16, c16) silk_ADD64((a64),(opus_int64)((opus_int32)(b16) * (opus_int32)(c16))) + +/* (a32 * b32) */ +#define silk_SMULL(a32, b32) ((opus_int64)(a32) * /*(opus_int64)*/(b32)) + +/* Adds two signed 32-bit values in a way that can overflow, while not relying on undefined behaviour + (just standard two's complement implementation-specific behaviour) */ +#define silk_ADD32_ovflw(a, b) ((opus_int32)((opus_uint32)(a) + (opus_uint32)(b))) +/* Subtractss two signed 32-bit values in a way that can overflow, while not relying on undefined behaviour + (just standard two's complement implementation-specific behaviour) */ +#define silk_SUB32_ovflw(a, b) ((opus_int32)((opus_uint32)(a) - (opus_uint32)(b))) + +/* Multiply-accumulate macros that allow overflow in the addition (ie, no asserts in debug mode) */ +#define silk_MLA_ovflw(a32, b32, c32) silk_ADD32_ovflw((a32), (opus_uint32)(b32) * (opus_uint32)(c32)) +#define silk_SMLABB_ovflw(a32, b32, c32) (silk_ADD32_ovflw((a32) , ((opus_int32)((opus_int16)(b32))) * (opus_int32)((opus_int16)(c32)))) + +#define silk_DIV32_16(a32, b16) ((opus_int32)((a32) / (b16))) +#define silk_DIV32(a32, b32) ((opus_int32)((a32) / (b32))) + +/* These macros enables checking for overflow in silk_API_Debug.h*/ +#define silk_ADD16(a, b) ((a) + (b)) +#define silk_ADD32(a, b) ((a) + (b)) +#define silk_ADD64(a, b) ((a) + (b)) + +#define silk_SUB16(a, b) ((a) - (b)) +#define silk_SUB32(a, b) ((a) - (b)) +#define silk_SUB64(a, b) ((a) - (b)) + +#define silk_SAT8(a) ((a) > silk_int8_MAX ? silk_int8_MAX : \ + ((a) < silk_int8_MIN ? silk_int8_MIN : (a))) +#define silk_SAT16(a) ((a) > silk_int16_MAX ? silk_int16_MAX : \ + ((a) < silk_int16_MIN ? silk_int16_MIN : (a))) +#define silk_SAT32(a) ((a) > silk_int32_MAX ? silk_int32_MAX : \ + ((a) < silk_int32_MIN ? silk_int32_MIN : (a))) + +#define silk_CHECK_FIT8(a) (a) +#define silk_CHECK_FIT16(a) (a) +#define silk_CHECK_FIT32(a) (a) + +#define silk_ADD_SAT16(a, b) (opus_int16)silk_SAT16( silk_ADD32( (opus_int32)(a), (b) ) ) +#define silk_ADD_SAT64(a, b) ((((a) + (b)) & 0x8000000000000000LL) == 0 ? \ + ((((a) & (b)) & 0x8000000000000000LL) != 0 ? silk_int64_MIN : (a)+(b)) : \ + ((((a) | (b)) & 0x8000000000000000LL) == 0 ? silk_int64_MAX : (a)+(b)) ) + +#define silk_SUB_SAT16(a, b) (opus_int16)silk_SAT16( silk_SUB32( (opus_int32)(a), (b) ) ) +#define silk_SUB_SAT64(a, b) ((((a)-(b)) & 0x8000000000000000LL) == 0 ? \ + (( (a) & ((b)^0x8000000000000000LL) & 0x8000000000000000LL) ? silk_int64_MIN : (a)-(b)) : \ + ((((a)^0x8000000000000000LL) & (b) & 0x8000000000000000LL) ? silk_int64_MAX : (a)-(b)) ) + +/* Saturation for positive input values */ +#define silk_POS_SAT32(a) ((a) > silk_int32_MAX ? silk_int32_MAX : (a)) + +/* Add with saturation for positive input values */ +#define silk_ADD_POS_SAT8(a, b) ((((a)+(b)) & 0x80) ? silk_int8_MAX : ((a)+(b))) +#define silk_ADD_POS_SAT16(a, b) ((((a)+(b)) & 0x8000) ? silk_int16_MAX : ((a)+(b))) +#define silk_ADD_POS_SAT32(a, b) ((((opus_uint32)(a)+(opus_uint32)(b)) & 0x80000000) ? silk_int32_MAX : ((a)+(b))) + +#define silk_LSHIFT8(a, shift) ((opus_int8)((opus_uint8)(a)<<(shift))) /* shift >= 0, shift < 8 */ +#define silk_LSHIFT16(a, shift) ((opus_int16)((opus_uint16)(a)<<(shift))) /* shift >= 0, shift < 16 */ +#define silk_LSHIFT32(a, shift) ((opus_int32)((opus_uint32)(a)<<(shift))) /* shift >= 0, shift < 32 */ +#define silk_LSHIFT64(a, shift) ((opus_int64)((opus_uint64)(a)<<(shift))) /* shift >= 0, shift < 64 */ +#define silk_LSHIFT(a, shift) silk_LSHIFT32(a, shift) /* shift >= 0, shift < 32 */ + +#define silk_RSHIFT8(a, shift) ((a)>>(shift)) /* shift >= 0, shift < 8 */ +#define silk_RSHIFT16(a, shift) ((a)>>(shift)) /* shift >= 0, shift < 16 */ +#define silk_RSHIFT32(a, shift) ((a)>>(shift)) /* shift >= 0, shift < 32 */ +#define silk_RSHIFT64(a, shift) ((a)>>(shift)) /* shift >= 0, shift < 64 */ +#define silk_RSHIFT(a, shift) silk_RSHIFT32(a, shift) /* shift >= 0, shift < 32 */ + +/* saturates before shifting */ +#define silk_LSHIFT_SAT32(a, shift) (silk_LSHIFT32( silk_LIMIT( (a), silk_RSHIFT32( silk_int32_MIN, (shift) ), \ + silk_RSHIFT32( silk_int32_MAX, (shift) ) ), (shift) )) + +#define silk_LSHIFT_ovflw(a, shift) ((opus_int32)((opus_uint32)(a) << (shift))) /* shift >= 0, allowed to overflow */ +#define silk_LSHIFT_uint(a, shift) ((a) << (shift)) /* shift >= 0 */ +#define silk_RSHIFT_uint(a, shift) ((a) >> (shift)) /* shift >= 0 */ + +#define silk_ADD_LSHIFT(a, b, shift) ((a) + silk_LSHIFT((b), (shift))) /* shift >= 0 */ +#define silk_ADD_LSHIFT32(a, b, shift) silk_ADD32((a), silk_LSHIFT32((b), (shift))) /* shift >= 0 */ +#define silk_ADD_LSHIFT_uint(a, b, shift) ((a) + silk_LSHIFT_uint((b), (shift))) /* shift >= 0 */ +#define silk_ADD_RSHIFT(a, b, shift) ((a) + silk_RSHIFT((b), (shift))) /* shift >= 0 */ +#define silk_ADD_RSHIFT32(a, b, shift) silk_ADD32((a), silk_RSHIFT32((b), (shift))) /* shift >= 0 */ +#define silk_ADD_RSHIFT_uint(a, b, shift) ((a) + silk_RSHIFT_uint((b), (shift))) /* shift >= 0 */ +#define silk_SUB_LSHIFT32(a, b, shift) silk_SUB32((a), silk_LSHIFT32((b), (shift))) /* shift >= 0 */ +#define silk_SUB_RSHIFT32(a, b, shift) silk_SUB32((a), silk_RSHIFT32((b), (shift))) /* shift >= 0 */ + +/* Requires that shift > 0 */ +#define silk_RSHIFT_ROUND(a, shift) ((shift) == 1 ? ((a) >> 1) + ((a) & 1) : (((a) >> ((shift) - 1)) + 1) >> 1) +#define silk_RSHIFT_ROUND64(a, shift) ((shift) == 1 ? ((a) >> 1) + ((a) & 1) : (((a) >> ((shift) - 1)) + 1) >> 1) + +/* Number of rightshift required to fit the multiplication */ +#define silk_NSHIFT_MUL_32_32(a, b) ( -(31- (32-silk_CLZ32(silk_abs(a)) + (32-silk_CLZ32(silk_abs(b))))) ) +#define silk_NSHIFT_MUL_16_16(a, b) ( -(15- (16-silk_CLZ16(silk_abs(a)) + (16-silk_CLZ16(silk_abs(b))))) ) + + +#define silk_min(a, b) (((a) < (b)) ? (a) : (b)) +#define silk_max(a, b) (((a) > (b)) ? (a) : (b)) + +/* Macro to convert floating-point constants to fixed-point */ +#define SILK_FIX_CONST( C, Q ) ((opus_int32)((C) * ((opus_int64)1 << (Q)) + 0.5)) + +/* silk_min() versions with typecast in the function call */ +static OPUS_INLINE opus_int silk_min_int(opus_int a, opus_int b) +{ + return (((a) < (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int16 silk_min_16(opus_int16 a, opus_int16 b) +{ + return (((a) < (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int32 silk_min_32(opus_int32 a, opus_int32 b) +{ + return (((a) < (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int64 silk_min_64(opus_int64 a, opus_int64 b) +{ + return (((a) < (b)) ? (a) : (b)); +} + +/* silk_min() versions with typecast in the function call */ +static OPUS_INLINE opus_int silk_max_int(opus_int a, opus_int b) +{ + return (((a) > (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int16 silk_max_16(opus_int16 a, opus_int16 b) +{ + return (((a) > (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int32 silk_max_32(opus_int32 a, opus_int32 b) +{ + return (((a) > (b)) ? (a) : (b)); +} +static OPUS_INLINE opus_int64 silk_max_64(opus_int64 a, opus_int64 b) +{ + return (((a) > (b)) ? (a) : (b)); +} + +#define silk_LIMIT( a, limit1, limit2) ((limit1) > (limit2) ? ((a) > (limit1) ? (limit1) : ((a) < (limit2) ? (limit2) : (a))) \ + : ((a) > (limit2) ? (limit2) : ((a) < (limit1) ? (limit1) : (a)))) + +#define silk_LIMIT_int silk_LIMIT +#define silk_LIMIT_16 silk_LIMIT +#define silk_LIMIT_32 silk_LIMIT + +#define silk_abs(a) (((a) > 0) ? (a) : -(a)) /* Be careful, silk_abs returns wrong when input equals to silk_intXX_MIN */ +#define silk_abs_int(a) (((a) ^ ((a) >> (8 * sizeof(a) - 1))) - ((a) >> (8 * sizeof(a) - 1))) +#define silk_abs_int32(a) (((a) ^ ((a) >> 31)) - ((a) >> 31)) +#define silk_abs_int64(a) (((a) > 0) ? (a) : -(a)) + +#define silk_sign(a) ((a) > 0 ? 1 : ( (a) < 0 ? -1 : 0 )) + +/* PSEUDO-RANDOM GENERATOR */ +/* Make sure to store the result as the seed for the next call (also in between */ +/* frames), otherwise result won't be random at all. When only using some of the */ +/* bits, take the most significant bits by right-shifting. */ +#define RAND_MULTIPLIER 196314165 +#define RAND_INCREMENT 907633515 +#define silk_RAND(seed) (silk_MLA_ovflw((RAND_INCREMENT), (seed), (RAND_MULTIPLIER))) + +/* Add some multiplication functions that can be easily mapped to ARM. */ + +/* silk_SMMUL: Signed top word multiply. + ARMv6 2 instruction cycles. + ARMv3M+ 3 instruction cycles. use SMULL and ignore LSB registers.(except xM)*/ +/*#define silk_SMMUL(a32, b32) (opus_int32)silk_RSHIFT(silk_SMLAL(silk_SMULWB((a32), (b32)), (a32), silk_RSHIFT_ROUND((b32), 16)), 16)*/ +/* the following seems faster on x86 */ +#define silk_SMMUL(a32, b32) (opus_int32)silk_RSHIFT64(silk_SMULL((a32), (b32)), 32) + +#if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +#define silk_burg_modified(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch) \ + ((void)(arch), silk_burg_modified_c(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch)) + +#define silk_inner_prod16_aligned_64(inVec1, inVec2, len, arch) \ + ((void)(arch),silk_inner_prod16_aligned_64_c(inVec1, inVec2, len)) +#endif + +#include "Inlines.h" +#include "MacroCount.h" +#include "MacroDebug.h" + +#ifdef OPUS_ARM_INLINE_ASM +#include "arm/SigProc_FIX_armv4.h" +#endif + +#ifdef OPUS_ARM_INLINE_EDSP +#include "arm/SigProc_FIX_armv5e.h" +#endif + +#if defined(MIPSr1_ASM) +#include "mips/sigproc_fix_mipsr1.h" +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_SIGPROC_FIX_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VAD.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VAD.c new file mode 100644 index 000000000..d0cda5216 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VAD.c @@ -0,0 +1,360 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/* Silk VAD noise level estimation */ +# if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +static OPUS_INLINE void silk_VAD_GetNoiseLevels( + const opus_int32 pX[ VAD_N_BANDS ], /* I subband energies */ + silk_VAD_state *psSilk_VAD /* I/O Pointer to Silk VAD state */ +); +#endif + +/**********************************/ +/* Initialization of the Silk VAD */ +/**********************************/ +opus_int silk_VAD_Init( /* O Return value, 0 if success */ + silk_VAD_state *psSilk_VAD /* I/O Pointer to Silk VAD state */ +) +{ + opus_int b, ret = 0; + + /* reset state memory */ + silk_memset( psSilk_VAD, 0, sizeof( silk_VAD_state ) ); + + /* init noise levels */ + /* Initialize array with approx pink noise levels (psd proportional to inverse of frequency) */ + for( b = 0; b < VAD_N_BANDS; b++ ) { + psSilk_VAD->NoiseLevelBias[ b ] = silk_max_32( silk_DIV32_16( VAD_NOISE_LEVELS_BIAS, b + 1 ), 1 ); + } + + /* Initialize state */ + for( b = 0; b < VAD_N_BANDS; b++ ) { + psSilk_VAD->NL[ b ] = silk_MUL( 100, psSilk_VAD->NoiseLevelBias[ b ] ); + psSilk_VAD->inv_NL[ b ] = silk_DIV32( silk_int32_MAX, psSilk_VAD->NL[ b ] ); + } + psSilk_VAD->counter = 15; + + /* init smoothed energy-to-noise ratio*/ + for( b = 0; b < VAD_N_BANDS; b++ ) { + psSilk_VAD->NrgRatioSmth_Q8[ b ] = 100 * 256; /* 100 * 256 --> 20 dB SNR */ + } + + return( ret ); +} + +/* Weighting factors for tilt measure */ +static const opus_int32 tiltWeights[ VAD_N_BANDS ] = { 30000, 6000, -12000, -12000 }; + +/***************************************/ +/* Get the speech activity level in Q8 */ +/***************************************/ +opus_int silk_VAD_GetSA_Q8_c( /* O Return value, 0 if success */ + silk_encoder_state *psEncC, /* I/O Encoder state */ + const opus_int16 pIn[] /* I PCM input */ +) +{ + opus_int SA_Q15, pSNR_dB_Q7, input_tilt; + opus_int decimated_framelength1, decimated_framelength2; + opus_int decimated_framelength; + opus_int dec_subframe_length, dec_subframe_offset, SNR_Q7, i, b, s; + opus_int32 sumSquared, smooth_coef_Q16; + opus_int16 HPstateTmp; + VARDECL( opus_int16, X ); + opus_int32 Xnrg[ VAD_N_BANDS ]; + opus_int32 NrgToNoiseRatio_Q8[ VAD_N_BANDS ]; + opus_int32 speech_nrg, x_tmp; + opus_int X_offset[ VAD_N_BANDS ]; + opus_int ret = 0; + silk_VAD_state *psSilk_VAD = &psEncC->sVAD; + SAVE_STACK; + + /* Safety checks */ + silk_assert( VAD_N_BANDS == 4 ); + celt_assert( MAX_FRAME_LENGTH >= psEncC->frame_length ); + celt_assert( psEncC->frame_length <= 512 ); + celt_assert( psEncC->frame_length == 8 * silk_RSHIFT( psEncC->frame_length, 3 ) ); + + /***********************/ + /* Filter and Decimate */ + /***********************/ + decimated_framelength1 = silk_RSHIFT( psEncC->frame_length, 1 ); + decimated_framelength2 = silk_RSHIFT( psEncC->frame_length, 2 ); + decimated_framelength = silk_RSHIFT( psEncC->frame_length, 3 ); + /* Decimate into 4 bands: + 0 L 3L L 3L 5L + - -- - -- -- + 8 8 2 4 4 + + [0-1 kHz| temp. |1-2 kHz| 2-4 kHz | 4-8 kHz | + + They're arranged to allow the minimal ( frame_length / 4 ) extra + scratch space during the downsampling process */ + X_offset[ 0 ] = 0; + X_offset[ 1 ] = decimated_framelength + decimated_framelength2; + X_offset[ 2 ] = X_offset[ 1 ] + decimated_framelength; + X_offset[ 3 ] = X_offset[ 2 ] + decimated_framelength2; + ALLOC( X, X_offset[ 3 ] + decimated_framelength1, opus_int16 ); + + /* 0-8 kHz to 0-4 kHz and 4-8 kHz */ + silk_ana_filt_bank_1( pIn, &psSilk_VAD->AnaState[ 0 ], + X, &X[ X_offset[ 3 ] ], psEncC->frame_length ); + + /* 0-4 kHz to 0-2 kHz and 2-4 kHz */ + silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState1[ 0 ], + X, &X[ X_offset[ 2 ] ], decimated_framelength1 ); + + /* 0-2 kHz to 0-1 kHz and 1-2 kHz */ + silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState2[ 0 ], + X, &X[ X_offset[ 1 ] ], decimated_framelength2 ); + + /*********************************************/ + /* HP filter on lowest band (differentiator) */ + /*********************************************/ + X[ decimated_framelength - 1 ] = silk_RSHIFT( X[ decimated_framelength - 1 ], 1 ); + HPstateTmp = X[ decimated_framelength - 1 ]; + for( i = decimated_framelength - 1; i > 0; i-- ) { + X[ i - 1 ] = silk_RSHIFT( X[ i - 1 ], 1 ); + X[ i ] -= X[ i - 1 ]; + } + X[ 0 ] -= psSilk_VAD->HPstate; + psSilk_VAD->HPstate = HPstateTmp; + + /*************************************/ + /* Calculate the energy in each band */ + /*************************************/ + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* Find the decimated framelength in the non-uniformly divided bands */ + decimated_framelength = silk_RSHIFT( psEncC->frame_length, silk_min_int( VAD_N_BANDS - b, VAD_N_BANDS - 1 ) ); + + /* Split length into subframe lengths */ + dec_subframe_length = silk_RSHIFT( decimated_framelength, VAD_INTERNAL_SUBFRAMES_LOG2 ); + dec_subframe_offset = 0; + + /* Compute energy per sub-frame */ + /* initialize with summed energy of last subframe */ + Xnrg[ b ] = psSilk_VAD->XnrgSubfr[ b ]; + for( s = 0; s < VAD_INTERNAL_SUBFRAMES; s++ ) { + sumSquared = 0; + for( i = 0; i < dec_subframe_length; i++ ) { + /* The energy will be less than dec_subframe_length * ( silk_int16_MIN / 8 ) ^ 2. */ + /* Therefore we can accumulate with no risk of overflow (unless dec_subframe_length > 128) */ + x_tmp = silk_RSHIFT( + X[ X_offset[ b ] + i + dec_subframe_offset ], 3 ); + sumSquared = silk_SMLABB( sumSquared, x_tmp, x_tmp ); + + /* Safety check */ + silk_assert( sumSquared >= 0 ); + } + + /* Add/saturate summed energy of current subframe */ + if( s < VAD_INTERNAL_SUBFRAMES - 1 ) { + Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], sumSquared ); + } else { + /* Look-ahead subframe */ + Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], silk_RSHIFT( sumSquared, 1 ) ); + } + + dec_subframe_offset += dec_subframe_length; + } + psSilk_VAD->XnrgSubfr[ b ] = sumSquared; + } + + /********************/ + /* Noise estimation */ + /********************/ + silk_VAD_GetNoiseLevels( &Xnrg[ 0 ], psSilk_VAD ); + + /***********************************************/ + /* Signal-plus-noise to noise ratio estimation */ + /***********************************************/ + sumSquared = 0; + input_tilt = 0; + for( b = 0; b < VAD_N_BANDS; b++ ) { + speech_nrg = Xnrg[ b ] - psSilk_VAD->NL[ b ]; + if( speech_nrg > 0 ) { + /* Divide, with sufficient resolution */ + if( ( Xnrg[ b ] & 0xFF800000 ) == 0 ) { + NrgToNoiseRatio_Q8[ b ] = silk_DIV32( silk_LSHIFT( Xnrg[ b ], 8 ), psSilk_VAD->NL[ b ] + 1 ); + } else { + NrgToNoiseRatio_Q8[ b ] = silk_DIV32( Xnrg[ b ], silk_RSHIFT( psSilk_VAD->NL[ b ], 8 ) + 1 ); + } + + /* Convert to log domain */ + SNR_Q7 = silk_lin2log( NrgToNoiseRatio_Q8[ b ] ) - 8 * 128; + + /* Sum-of-squares */ + sumSquared = silk_SMLABB( sumSquared, SNR_Q7, SNR_Q7 ); /* Q14 */ + + /* Tilt measure */ + if( speech_nrg < ( (opus_int32)1 << 20 ) ) { + /* Scale down SNR value for small subband speech energies */ + SNR_Q7 = silk_SMULWB( silk_LSHIFT( silk_SQRT_APPROX( speech_nrg ), 6 ), SNR_Q7 ); + } + input_tilt = silk_SMLAWB( input_tilt, tiltWeights[ b ], SNR_Q7 ); + } else { + NrgToNoiseRatio_Q8[ b ] = 256; + } + } + + /* Mean-of-squares */ + sumSquared = silk_DIV32_16( sumSquared, VAD_N_BANDS ); /* Q14 */ + + /* Root-mean-square approximation, scale to dBs, and write to output pointer */ + pSNR_dB_Q7 = (opus_int16)( 3 * silk_SQRT_APPROX( sumSquared ) ); /* Q7 */ + + /*********************************/ + /* Speech Probability Estimation */ + /*********************************/ + SA_Q15 = silk_sigm_Q15( silk_SMULWB( VAD_SNR_FACTOR_Q16, pSNR_dB_Q7 ) - VAD_NEGATIVE_OFFSET_Q5 ); + + /**************************/ + /* Frequency Tilt Measure */ + /**************************/ + psEncC->input_tilt_Q15 = silk_LSHIFT( silk_sigm_Q15( input_tilt ) - 16384, 1 ); + + /**************************************************/ + /* Scale the sigmoid output based on power levels */ + /**************************************************/ + speech_nrg = 0; + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* Accumulate signal-without-noise energies, higher frequency bands have more weight */ + speech_nrg += ( b + 1 ) * silk_RSHIFT( Xnrg[ b ] - psSilk_VAD->NL[ b ], 4 ); + } + + if( psEncC->frame_length == 20 * psEncC->fs_kHz ) { + speech_nrg = silk_RSHIFT32( speech_nrg, 1 ); + } + /* Power scaling */ + if( speech_nrg <= 0 ) { + SA_Q15 = silk_RSHIFT( SA_Q15, 1 ); + } else if( speech_nrg < 16384 ) { + speech_nrg = silk_LSHIFT32( speech_nrg, 16 ); + + /* square-root */ + speech_nrg = silk_SQRT_APPROX( speech_nrg ); + SA_Q15 = silk_SMULWB( 32768 + speech_nrg, SA_Q15 ); + } + + /* Copy the resulting speech activity in Q8 */ + psEncC->speech_activity_Q8 = silk_min_int( silk_RSHIFT( SA_Q15, 7 ), silk_uint8_MAX ); + + /***********************************/ + /* Energy Level and SNR estimation */ + /***********************************/ + /* Smoothing coefficient */ + smooth_coef_Q16 = silk_SMULWB( VAD_SNR_SMOOTH_COEF_Q18, silk_SMULWB( (opus_int32)SA_Q15, SA_Q15 ) ); + + if( psEncC->frame_length == 10 * psEncC->fs_kHz ) { + smooth_coef_Q16 >>= 1; + } + + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* compute smoothed energy-to-noise ratio per band */ + psSilk_VAD->NrgRatioSmth_Q8[ b ] = silk_SMLAWB( psSilk_VAD->NrgRatioSmth_Q8[ b ], + NrgToNoiseRatio_Q8[ b ] - psSilk_VAD->NrgRatioSmth_Q8[ b ], smooth_coef_Q16 ); + + /* signal to noise ratio in dB per band */ + SNR_Q7 = 3 * ( silk_lin2log( psSilk_VAD->NrgRatioSmth_Q8[b] ) - 8 * 128 ); + /* quality = sigmoid( 0.25 * ( SNR_dB - 16 ) ); */ + psEncC->input_quality_bands_Q15[ b ] = silk_sigm_Q15( silk_RSHIFT( SNR_Q7 - 16 * 128, 4 ) ); + } + + RESTORE_STACK; + return( ret ); +} + +/**************************/ +/* Noise level estimation */ +/**************************/ +# if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +static OPUS_INLINE +#endif +void silk_VAD_GetNoiseLevels( + const opus_int32 pX[ VAD_N_BANDS ], /* I subband energies */ + silk_VAD_state *psSilk_VAD /* I/O Pointer to Silk VAD state */ +) +{ + opus_int k; + opus_int32 nl, nrg, inv_nrg; + opus_int coef, min_coef; + + /* Initially faster smoothing */ + if( psSilk_VAD->counter < 1000 ) { /* 1000 = 20 sec */ + min_coef = silk_DIV32_16( silk_int16_MAX, silk_RSHIFT( psSilk_VAD->counter, 4 ) + 1 ); + /* Increment frame counter */ + psSilk_VAD->counter++; + } else { + min_coef = 0; + } + + for( k = 0; k < VAD_N_BANDS; k++ ) { + /* Get old noise level estimate for current band */ + nl = psSilk_VAD->NL[ k ]; + silk_assert( nl >= 0 ); + + /* Add bias */ + nrg = silk_ADD_POS_SAT32( pX[ k ], psSilk_VAD->NoiseLevelBias[ k ] ); + silk_assert( nrg > 0 ); + + /* Invert energies */ + inv_nrg = silk_DIV32( silk_int32_MAX, nrg ); + silk_assert( inv_nrg >= 0 ); + + /* Less update when subband energy is high */ + if( nrg > silk_LSHIFT( nl, 3 ) ) { + coef = VAD_NOISE_LEVEL_SMOOTH_COEF_Q16 >> 3; + } else if( nrg < nl ) { + coef = VAD_NOISE_LEVEL_SMOOTH_COEF_Q16; + } else { + coef = silk_SMULWB( silk_SMULWW( inv_nrg, nl ), VAD_NOISE_LEVEL_SMOOTH_COEF_Q16 << 1 ); + } + + /* Initially faster smoothing */ + coef = silk_max_int( coef, min_coef ); + + /* Smooth inverse energies */ + psSilk_VAD->inv_NL[ k ] = silk_SMLAWB( psSilk_VAD->inv_NL[ k ], inv_nrg - psSilk_VAD->inv_NL[ k ], coef ); + silk_assert( psSilk_VAD->inv_NL[ k ] >= 0 ); + + /* Compute noise level by inverting again */ + nl = silk_DIV32( silk_int32_MAX, psSilk_VAD->inv_NL[ k ] ); + silk_assert( nl >= 0 ); + + /* Limit noise levels (guarantee 7 bits of head room) */ + nl = silk_min( nl, 0x00FFFFFF ); + + /* Store as part of state */ + psSilk_VAD->NL[ k ] = nl; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VQ_WMat_EC.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VQ_WMat_EC.c new file mode 100644 index 000000000..0f3d545c4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/VQ_WMat_EC.c @@ -0,0 +1,131 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Entropy constrained matrix-weighted VQ, hard-coded to 5-element vectors, for a single input data vector */ +void silk_VQ_WMat_EC_c( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *res_nrg_Q15, /* O best residual energy */ + opus_int32 *rate_dist_Q8, /* O best total bitrate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int32 *XX_Q17, /* I correlation matrix */ + const opus_int32 *xX_Q17, /* I correlation vector */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int subfr_len, /* I number of samples per subframe */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + const opus_int L /* I number of vectors in codebook */ +) +{ + opus_int k, gain_tmp_Q7; + const opus_int8 *cb_row_Q7; + opus_int32 neg_xX_Q24[ 5 ]; + opus_int32 sum1_Q15, sum2_Q24; + opus_int32 bits_res_Q8, bits_tot_Q8; + + /* Negate and convert to new Q domain */ + neg_xX_Q24[ 0 ] = -silk_LSHIFT32( xX_Q17[ 0 ], 7 ); + neg_xX_Q24[ 1 ] = -silk_LSHIFT32( xX_Q17[ 1 ], 7 ); + neg_xX_Q24[ 2 ] = -silk_LSHIFT32( xX_Q17[ 2 ], 7 ); + neg_xX_Q24[ 3 ] = -silk_LSHIFT32( xX_Q17[ 3 ], 7 ); + neg_xX_Q24[ 4 ] = -silk_LSHIFT32( xX_Q17[ 4 ], 7 ); + + /* Loop over codebook */ + *rate_dist_Q8 = silk_int32_MAX; + *res_nrg_Q15 = silk_int32_MAX; + cb_row_Q7 = cb_Q7; + /* In things go really bad, at least *ind is set to something safe. */ + *ind = 0; + for( k = 0; k < L; k++ ) { + opus_int32 penalty; + gain_tmp_Q7 = cb_gain_Q7[k]; + /* Weighted rate */ + /* Quantization error: 1 - 2 * xX * cb + cb' * XX * cb */ + sum1_Q15 = SILK_FIX_CONST( 1.001, 15 ); + + /* Penalty for too large gain */ + penalty = silk_LSHIFT32( silk_max( silk_SUB32( gain_tmp_Q7, max_gain_Q7 ), 0 ), 11 ); + + /* first row of XX_Q17 */ + sum2_Q24 = silk_MLA( neg_xX_Q24[ 0 ], XX_Q17[ 1 ], cb_row_Q7[ 1 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 2 ], cb_row_Q7[ 2 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 3 ], cb_row_Q7[ 3 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 4 ], cb_row_Q7[ 4 ] ); + sum2_Q24 = silk_LSHIFT32( sum2_Q24, 1 ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 0 ], cb_row_Q7[ 0 ] ); + sum1_Q15 = silk_SMLAWB( sum1_Q15, sum2_Q24, cb_row_Q7[ 0 ] ); + + /* second row of XX_Q17 */ + sum2_Q24 = silk_MLA( neg_xX_Q24[ 1 ], XX_Q17[ 7 ], cb_row_Q7[ 2 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 8 ], cb_row_Q7[ 3 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 9 ], cb_row_Q7[ 4 ] ); + sum2_Q24 = silk_LSHIFT32( sum2_Q24, 1 ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 6 ], cb_row_Q7[ 1 ] ); + sum1_Q15 = silk_SMLAWB( sum1_Q15, sum2_Q24, cb_row_Q7[ 1 ] ); + + /* third row of XX_Q17 */ + sum2_Q24 = silk_MLA( neg_xX_Q24[ 2 ], XX_Q17[ 13 ], cb_row_Q7[ 3 ] ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 14 ], cb_row_Q7[ 4 ] ); + sum2_Q24 = silk_LSHIFT32( sum2_Q24, 1 ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 12 ], cb_row_Q7[ 2 ] ); + sum1_Q15 = silk_SMLAWB( sum1_Q15, sum2_Q24, cb_row_Q7[ 2 ] ); + + /* fourth row of XX_Q17 */ + sum2_Q24 = silk_MLA( neg_xX_Q24[ 3 ], XX_Q17[ 19 ], cb_row_Q7[ 4 ] ); + sum2_Q24 = silk_LSHIFT32( sum2_Q24, 1 ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 18 ], cb_row_Q7[ 3 ] ); + sum1_Q15 = silk_SMLAWB( sum1_Q15, sum2_Q24, cb_row_Q7[ 3 ] ); + + /* last row of XX_Q17 */ + sum2_Q24 = silk_LSHIFT32( neg_xX_Q24[ 4 ], 1 ); + sum2_Q24 = silk_MLA( sum2_Q24, XX_Q17[ 24 ], cb_row_Q7[ 4 ] ); + sum1_Q15 = silk_SMLAWB( sum1_Q15, sum2_Q24, cb_row_Q7[ 4 ] ); + + /* find best */ + if( sum1_Q15 >= 0 ) { + /* Translate residual energy to bits using high-rate assumption (6 dB ==> 1 bit/sample) */ + bits_res_Q8 = silk_SMULBB( subfr_len, silk_lin2log( sum1_Q15 + penalty) - (15 << 7) ); + /* In the following line we reduce the codelength component by half ("-1"); seems to slghtly improve quality */ + bits_tot_Q8 = silk_ADD_LSHIFT32( bits_res_Q8, cl_Q5[ k ], 3-1 ); + if( bits_tot_Q8 <= *rate_dist_Q8 ) { + *rate_dist_Q8 = bits_tot_Q8; + *res_nrg_Q15 = sum1_Q15 + penalty; + *ind = (opus_int8)k; + *gain_Q7 = gain_tmp_Q7; + } + } + + /* Go to next cbk vector */ + cb_row_Q7 += LTP_ORDER; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/ana_filt_bank_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/ana_filt_bank_1.c new file mode 100644 index 000000000..24cfb03fd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/ana_filt_bank_1.c @@ -0,0 +1,74 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Coefficients for 2-band filter bank based on first-order allpass filters */ +static opus_int16 A_fb1_20 = 5394 << 1; +static opus_int16 A_fb1_21 = -24290; /* (opus_int16)(20623 << 1) */ + +/* Split signal into two decimated bands using first-order allpass filters */ +void silk_ana_filt_bank_1( + const opus_int16 *in, /* I Input signal [N] */ + opus_int32 *S, /* I/O State vector [2] */ + opus_int16 *outL, /* O Low band [N/2] */ + opus_int16 *outH, /* O High band [N/2] */ + const opus_int32 N /* I Number of input samples */ +) +{ + opus_int k, N2 = silk_RSHIFT( N, 1 ); + opus_int32 in32, X, Y, out_1, out_2; + + /* Internal variables and state are in Q10 format */ + for( k = 0; k < N2; k++ ) { + /* Convert to Q10 */ + in32 = silk_LSHIFT( (opus_int32)in[ 2 * k ], 10 ); + + /* All-pass section for even input sample */ + Y = silk_SUB32( in32, S[ 0 ] ); + X = silk_SMLAWB( Y, Y, A_fb1_21 ); + out_1 = silk_ADD32( S[ 0 ], X ); + S[ 0 ] = silk_ADD32( in32, X ); + + /* Convert to Q10 */ + in32 = silk_LSHIFT( (opus_int32)in[ 2 * k + 1 ], 10 ); + + /* All-pass section for odd input sample, and add to output of previous section */ + Y = silk_SUB32( in32, S[ 1 ] ); + X = silk_SMULWB( Y, A_fb1_20 ); + out_2 = silk_ADD32( S[ 1 ], X ); + S[ 1 ] = silk_ADD32( in32, X ); + + /* Add/subtract, convert back to int16 and store to output */ + outL[ k ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_ADD32( out_2, out_1 ), 11 ) ); + outH[ k ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SUB32( out_2, out_1 ), 11 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_arm.h new file mode 100644 index 000000000..9895b555c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_arm.h @@ -0,0 +1,57 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_LPC_INV_PRED_GAIN_ARM_H +# define SILK_LPC_INV_PRED_GAIN_ARM_H + +# include "celt/arm/armcpu.h" + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +opus_int32 silk_LPC_inverse_pred_gain_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ + const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ + const opus_int order /* I Prediction order */ +); + +# if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) +# define OVERRIDE_silk_LPC_inverse_pred_gain (1) +# define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((void)(arch), PRESUME_NEON(silk_LPC_inverse_pred_gain)(A_Q12, order)) +# endif +# endif + +# if !defined(OVERRIDE_silk_LPC_inverse_pred_gain) +/*Is run-time CPU detection enabled on this platform?*/ +# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern opus_int32 (*const SILK_LPC_INVERSE_PRED_GAIN_IMPL[OPUS_ARCHMASK+1])(const opus_int16 *A_Q12, const opus_int order); +# define OVERRIDE_silk_LPC_inverse_pred_gain (1) +# define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((*SILK_LPC_INVERSE_PRED_GAIN_IMPL[(arch)&OPUS_ARCHMASK])(A_Q12, order)) +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_silk_LPC_inverse_pred_gain (1) +# define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((void)(arch), silk_LPC_inverse_pred_gain_neon(A_Q12, order)) +# endif +# endif + +#endif /* end SILK_LPC_INV_PRED_GAIN_ARM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_neon_intr.c new file mode 100644 index 000000000..726e6667b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/LPC_inv_pred_gain_neon_intr.c @@ -0,0 +1,288 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "SigProc_FIX.h" +#include "define.h" + +#define QA 24 +#define A_LIMIT SILK_FIX_CONST( 0.99975, QA ) + +#define MUL32_FRAC_Q(a32, b32, Q) ((opus_int32)(silk_RSHIFT_ROUND64(silk_SMULL(a32, b32), Q))) + +/* The difficulty is how to judge a 64-bit signed integer tmp64 is 32-bit overflowed, + * since NEON has no 64-bit min, max or comparison instructions. + * A failed idea is to compare the results of vmovn(tmp64) and vqmovn(tmp64) whether they are equal or not. + * However, this idea fails when the tmp64 is something like 0xFFFFFFF980000000. + * Here we know that mult2Q >= 1, so the highest bit (bit 63, sign bit) of tmp64 must equal to bit 62. + * tmp64 was shifted left by 1 and we got tmp64'. If high_half(tmp64') != 0 and high_half(tmp64') != -1, + * then we know that bit 31 to bit 63 of tmp64 can not all be the sign bit, and therefore tmp64 is 32-bit overflowed. + * That is, we judge if tmp64' > 0x00000000FFFFFFFF, or tmp64' <= 0xFFFFFFFF00000000. + * We use narrowing shift right 31 bits to tmp32' to save data bandwidth and instructions. + * That is, we judge if tmp32' > 0x00000000, or tmp32' <= 0xFFFFFFFF. + */ + +/* Compute inverse of LPC prediction gain, and */ +/* test if LPC coefficients are stable (all poles within unit circle) */ +static OPUS_INLINE opus_int32 LPC_inverse_pred_gain_QA_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ + opus_int32 A_QA[ SILK_MAX_ORDER_LPC ], /* I Prediction coefficients */ + const opus_int order /* I Prediction order */ +) +{ + opus_int k, n, mult2Q; + opus_int32 invGain_Q30, rc_Q31, rc_mult1_Q30, rc_mult2, tmp1, tmp2; + opus_int32 max, min; + int32x4_t max_s32x4, min_s32x4; + int32x2_t max_s32x2, min_s32x2; + + max_s32x4 = vdupq_n_s32( silk_int32_MIN ); + min_s32x4 = vdupq_n_s32( silk_int32_MAX ); + invGain_Q30 = SILK_FIX_CONST( 1, 30 ); + for( k = order - 1; k > 0; k-- ) { + int32x2_t rc_Q31_s32x2, rc_mult2_s32x2; + int64x2_t mult2Q_s64x2; + + /* Check for stability */ + if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { + return 0; + } + + /* Set RC equal to negated AR coef */ + rc_Q31 = -silk_LSHIFT( A_QA[ k ], 31 - QA ); + + /* rc_mult1_Q30 range: [ 1 : 2^30 ] */ + rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); + silk_assert( rc_mult1_Q30 > ( 1 << 15 ) ); /* reduce A_LIMIT if fails */ + silk_assert( rc_mult1_Q30 <= ( 1 << 30 ) ); + + /* Update inverse gain */ + /* invGain_Q30 range: [ 0 : 2^30 ] */ + invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); + silk_assert( invGain_Q30 >= 0 ); + silk_assert( invGain_Q30 <= ( 1 << 30 ) ); + if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { + return 0; + } + + /* rc_mult2 range: [ 2^30 : silk_int32_MAX ] */ + mult2Q = 32 - silk_CLZ32( silk_abs( rc_mult1_Q30 ) ); + rc_mult2 = silk_INVERSE32_varQ( rc_mult1_Q30, mult2Q + 30 ); + + /* Update AR coefficient */ + rc_Q31_s32x2 = vdup_n_s32( rc_Q31 ); + mult2Q_s64x2 = vdupq_n_s64( -mult2Q ); + rc_mult2_s32x2 = vdup_n_s32( rc_mult2 ); + + for( n = 0; n < ( ( k + 1 ) >> 1 ) - 3; n += 4 ) { + /* We always calculate extra elements of A_QA buffer when ( k % 4 ) != 0, to take the advantage of SIMD parallelization. */ + int32x4_t tmp1_s32x4, tmp2_s32x4, t0_s32x4, t1_s32x4, s0_s32x4, s1_s32x4, t_QA0_s32x4, t_QA1_s32x4; + int64x2_t t0_s64x2, t1_s64x2, t2_s64x2, t3_s64x2; + tmp1_s32x4 = vld1q_s32( A_QA + n ); + tmp2_s32x4 = vld1q_s32( A_QA + k - n - 4 ); + tmp2_s32x4 = vrev64q_s32( tmp2_s32x4 ); + tmp2_s32x4 = vcombine_s32( vget_high_s32( tmp2_s32x4 ), vget_low_s32( tmp2_s32x4 ) ); + t0_s32x4 = vqrdmulhq_lane_s32( tmp2_s32x4, rc_Q31_s32x2, 0 ); + t1_s32x4 = vqrdmulhq_lane_s32( tmp1_s32x4, rc_Q31_s32x2, 0 ); + t_QA0_s32x4 = vqsubq_s32( tmp1_s32x4, t0_s32x4 ); + t_QA1_s32x4 = vqsubq_s32( tmp2_s32x4, t1_s32x4 ); + t0_s64x2 = vmull_s32( vget_low_s32 ( t_QA0_s32x4 ), rc_mult2_s32x2 ); + t1_s64x2 = vmull_s32( vget_high_s32( t_QA0_s32x4 ), rc_mult2_s32x2 ); + t2_s64x2 = vmull_s32( vget_low_s32 ( t_QA1_s32x4 ), rc_mult2_s32x2 ); + t3_s64x2 = vmull_s32( vget_high_s32( t_QA1_s32x4 ), rc_mult2_s32x2 ); + t0_s64x2 = vrshlq_s64( t0_s64x2, mult2Q_s64x2 ); + t1_s64x2 = vrshlq_s64( t1_s64x2, mult2Q_s64x2 ); + t2_s64x2 = vrshlq_s64( t2_s64x2, mult2Q_s64x2 ); + t3_s64x2 = vrshlq_s64( t3_s64x2, mult2Q_s64x2 ); + t0_s32x4 = vcombine_s32( vmovn_s64( t0_s64x2 ), vmovn_s64( t1_s64x2 ) ); + t1_s32x4 = vcombine_s32( vmovn_s64( t2_s64x2 ), vmovn_s64( t3_s64x2 ) ); + s0_s32x4 = vcombine_s32( vshrn_n_s64( t0_s64x2, 31 ), vshrn_n_s64( t1_s64x2, 31 ) ); + s1_s32x4 = vcombine_s32( vshrn_n_s64( t2_s64x2, 31 ), vshrn_n_s64( t3_s64x2, 31 ) ); + max_s32x4 = vmaxq_s32( max_s32x4, s0_s32x4 ); + min_s32x4 = vminq_s32( min_s32x4, s0_s32x4 ); + max_s32x4 = vmaxq_s32( max_s32x4, s1_s32x4 ); + min_s32x4 = vminq_s32( min_s32x4, s1_s32x4 ); + t1_s32x4 = vrev64q_s32( t1_s32x4 ); + t1_s32x4 = vcombine_s32( vget_high_s32( t1_s32x4 ), vget_low_s32( t1_s32x4 ) ); + vst1q_s32( A_QA + n, t0_s32x4 ); + vst1q_s32( A_QA + k - n - 4, t1_s32x4 ); + } + for( ; n < (k + 1) >> 1; n++ ) { + opus_int64 tmp64; + tmp1 = A_QA[ n ]; + tmp2 = A_QA[ k - n - 1 ]; + tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp1, + MUL32_FRAC_Q( tmp2, rc_Q31, 31 ) ), rc_mult2 ), mult2Q); + if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { + return 0; + } + A_QA[ n ] = ( opus_int32 )tmp64; + tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp2, + MUL32_FRAC_Q( tmp1, rc_Q31, 31 ) ), rc_mult2), mult2Q); + if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { + return 0; + } + A_QA[ k - n - 1 ] = ( opus_int32 )tmp64; + } + } + + /* Check for stability */ + if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { + return 0; + } + + max_s32x2 = vmax_s32( vget_low_s32( max_s32x4 ), vget_high_s32( max_s32x4 ) ); + min_s32x2 = vmin_s32( vget_low_s32( min_s32x4 ), vget_high_s32( min_s32x4 ) ); + max_s32x2 = vmax_s32( max_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( max_s32x2 ), 32 ) ) ); + min_s32x2 = vmin_s32( min_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( min_s32x2 ), 32 ) ) ); + max = vget_lane_s32( max_s32x2, 0 ); + min = vget_lane_s32( min_s32x2, 0 ); + if( ( max > 0 ) || ( min < -1 ) ) { + return 0; + } + + /* Set RC equal to negated AR coef */ + rc_Q31 = -silk_LSHIFT( A_QA[ 0 ], 31 - QA ); + + /* Range: [ 1 : 2^30 ] */ + rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); + + /* Update inverse gain */ + /* Range: [ 0 : 2^30 ] */ + invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); + silk_assert( invGain_Q30 >= 0 ); + silk_assert( invGain_Q30 <= ( 1 << 30 ) ); + if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { + return 0; + } + + return invGain_Q30; +} + +/* For input in Q12 domain */ +opus_int32 silk_LPC_inverse_pred_gain_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ + const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ + const opus_int order /* I Prediction order */ +) +{ +#ifdef OPUS_CHECK_ASM + const opus_int32 invGain_Q30_c = silk_LPC_inverse_pred_gain_c( A_Q12, order ); +#endif + + opus_int32 invGain_Q30; + if( ( SILK_MAX_ORDER_LPC != 24 ) || ( order & 1 )) { + invGain_Q30 = silk_LPC_inverse_pred_gain_c( A_Q12, order ); + } + else { + opus_int32 Atmp_QA[ SILK_MAX_ORDER_LPC ]; + opus_int32 DC_resp; + int16x8_t t0_s16x8, t1_s16x8, t2_s16x8; + int32x4_t t0_s32x4; + const opus_int leftover = order & 7; + + /* Increase Q domain of the AR coefficients */ + t0_s16x8 = vld1q_s16( A_Q12 + 0 ); + t1_s16x8 = vld1q_s16( A_Q12 + 8 ); + if ( order > 16 ) { + t2_s16x8 = vld1q_s16( A_Q12 + 16 ); + } + t0_s32x4 = vpaddlq_s16( t0_s16x8 ); + + switch( order - leftover ) + { + case 24: + t0_s32x4 = vpadalq_s16( t0_s32x4, t2_s16x8 ); + vst1q_s32( Atmp_QA + 16, vshll_n_s16( vget_low_s16 ( t2_s16x8 ), QA - 12 ) ); + vst1q_s32( Atmp_QA + 20, vshll_n_s16( vget_high_s16( t2_s16x8 ), QA - 12 ) ); + /* FALLTHROUGH */ + + case 16: + t0_s32x4 = vpadalq_s16( t0_s32x4, t1_s16x8 ); + vst1q_s32( Atmp_QA + 8, vshll_n_s16( vget_low_s16 ( t1_s16x8 ), QA - 12 ) ); + vst1q_s32( Atmp_QA + 12, vshll_n_s16( vget_high_s16( t1_s16x8 ), QA - 12 ) ); + /* FALLTHROUGH */ + + case 8: + { + const int32x2_t t_s32x2 = vpadd_s32( vget_low_s32( t0_s32x4 ), vget_high_s32( t0_s32x4 ) ); + const int64x1_t t_s64x1 = vpaddl_s32( t_s32x2 ); + DC_resp = vget_lane_s32( vreinterpret_s32_s64( t_s64x1 ), 0 ); + vst1q_s32( Atmp_QA + 0, vshll_n_s16( vget_low_s16 ( t0_s16x8 ), QA - 12 ) ); + vst1q_s32( Atmp_QA + 4, vshll_n_s16( vget_high_s16( t0_s16x8 ), QA - 12 ) ); + } + break; + + default: + DC_resp = 0; + break; + } + A_Q12 += order - leftover; + + switch( leftover ) + { + case 6: + DC_resp += (opus_int32)A_Q12[ 5 ]; + DC_resp += (opus_int32)A_Q12[ 4 ]; + Atmp_QA[ order - leftover + 5 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 5 ], QA - 12 ); + Atmp_QA[ order - leftover + 4 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 4 ], QA - 12 ); + /* FALLTHROUGH */ + + case 4: + DC_resp += (opus_int32)A_Q12[ 3 ]; + DC_resp += (opus_int32)A_Q12[ 2 ]; + Atmp_QA[ order - leftover + 3 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 3 ], QA - 12 ); + Atmp_QA[ order - leftover + 2 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 2 ], QA - 12 ); + /* FALLTHROUGH */ + + case 2: + DC_resp += (opus_int32)A_Q12[ 1 ]; + DC_resp += (opus_int32)A_Q12[ 0 ]; + Atmp_QA[ order - leftover + 1 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 1 ], QA - 12 ); + Atmp_QA[ order - leftover + 0 ] = silk_LSHIFT32( (opus_int32)A_Q12[ 0 ], QA - 12 ); + /* FALLTHROUGH */ + + default: + break; + } + + /* If the DC is unstable, we don't even need to do the full calculations */ + if( DC_resp >= 4096 ) { + invGain_Q30 = 0; + } else { + invGain_Q30 = LPC_inverse_pred_gain_QA_neon( Atmp_QA, order ); + } + } + +#ifdef OPUS_CHECK_ASM + silk_assert( invGain_Q30_c == invGain_Q30 ); +#endif + + return invGain_Q30; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_arm.h new file mode 100644 index 000000000..9e76e1692 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_arm.h @@ -0,0 +1,100 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_NSQ_DEL_DEC_ARM_H +#define SILK_NSQ_DEL_DEC_ARM_H + +#include "celt/arm/armcpu.h" + +#if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +void silk_NSQ_del_dec_neon( + const silk_encoder_state *psEncC, silk_nsq_state *NSQ, + SideInfoIndices *psIndices, const opus_int16 x16[], opus_int8 pulses[], + const opus_int16 PredCoef_Q12[2 * MAX_LPC_ORDER], + const opus_int16 LTPCoef_Q14[LTP_ORDER * MAX_NB_SUBFR], + const opus_int16 AR_Q13[MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER], + const opus_int HarmShapeGain_Q14[MAX_NB_SUBFR], + const opus_int Tilt_Q14[MAX_NB_SUBFR], + const opus_int32 LF_shp_Q14[MAX_NB_SUBFR], + const opus_int32 Gains_Q16[MAX_NB_SUBFR], + const opus_int pitchL[MAX_NB_SUBFR], const opus_int Lambda_Q10, + const opus_int LTP_scale_Q14); + +#if !defined(OPUS_HAVE_RTCD) +#define OVERRIDE_silk_NSQ_del_dec (1) +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ + LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ + LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ + LTP_scale_Q14, arch) \ + ((void)(arch), \ + PRESUME_NEON(silk_NSQ_del_dec)( \ + psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, \ + AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, \ + Lambda_Q10, LTP_scale_Q14)) +#endif +#endif + +#if !defined(OVERRIDE_silk_NSQ_del_dec) +/*Is run-time CPU detection enabled on this platform?*/ +#if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && \ + !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( + const silk_encoder_state *psEncC, silk_nsq_state *NSQ, + SideInfoIndices *psIndices, const opus_int16 x16[], opus_int8 pulses[], + const opus_int16 PredCoef_Q12[2 * MAX_LPC_ORDER], + const opus_int16 LTPCoef_Q14[LTP_ORDER * MAX_NB_SUBFR], + const opus_int16 AR_Q13[MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER], + const opus_int HarmShapeGain_Q14[MAX_NB_SUBFR], + const opus_int Tilt_Q14[MAX_NB_SUBFR], + const opus_int32 LF_shp_Q14[MAX_NB_SUBFR], + const opus_int32 Gains_Q16[MAX_NB_SUBFR], + const opus_int pitchL[MAX_NB_SUBFR], const opus_int Lambda_Q10, + const opus_int LTP_scale_Q14); +#define OVERRIDE_silk_NSQ_del_dec (1) +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ + LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ + LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ + LTP_scale_Q14, arch) \ + ((*SILK_NSQ_DEL_DEC_IMPL[(arch)&OPUS_ARCHMASK])( \ + psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, \ + AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, \ + Lambda_Q10, LTP_scale_Q14)) +#elif defined(OPUS_ARM_PRESUME_NEON_INTR) +#define OVERRIDE_silk_NSQ_del_dec (1) +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ + LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ + LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ + LTP_scale_Q14, arch) \ + ((void)(arch), \ + silk_NSQ_del_dec_neon(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ + LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ + LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ + LTP_scale_Q14)) +#endif +#endif + +#endif /* end SILK_NSQ_DEL_DEC_ARM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_neon_intr.c new file mode 100644 index 000000000..212410f36 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_del_dec_neon_intr.c @@ -0,0 +1,1124 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#ifdef OPUS_CHECK_ASM +# include +#endif +#include "main.h" +#include "stack_alloc.h" + +/* NEON intrinsics optimization now can only parallelize up to 4 delay decision states. */ +/* If there are more states, C function is called, and this optimization must be expanded. */ +#define NEON_MAX_DEL_DEC_STATES 4 + +typedef struct { + opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 RandState[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Q_Q10[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Xq_Q14[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Pred_Q15[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Shape_Q14[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ][ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 LF_AR_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Diff_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Seed[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 SeedInit[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 RD_Q10[ NEON_MAX_DEL_DEC_STATES ]; +} NSQ_del_decs_struct; + +typedef struct { + opus_int32 Q_Q10[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 RD_Q10[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 xq_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 LF_AR_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 Diff_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 sLTP_shp_Q14[ NEON_MAX_DEL_DEC_STATES ]; + opus_int32 LPC_exc_Q14[ NEON_MAX_DEL_DEC_STATES ]; +} NSQ_samples_struct; + +static OPUS_INLINE void silk_nsq_del_dec_scale_states_neon( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int16 x16[], /* I Input */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +); + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_neon( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay /* I */ +); + +static OPUS_INLINE void copy_winner_state_kernel( + const NSQ_del_decs_struct *psDelDec, + const opus_int offset, + const opus_int last_smple_idx, + const opus_int Winner_ind, + const int32x2_t gain_lo_s32x2, + const int32x2_t gain_hi_s32x2, + const int32x4_t shift_s32x4, + int32x4_t t0_s32x4, + int32x4_t t1_s32x4, + opus_int8 *const pulses, + opus_int16 *pxq, + silk_nsq_state *NSQ +) +{ + int16x8_t t_s16x8; + int32x4_t o0_s32x4, o1_s32x4; + + t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); + t_s16x8 = vcombine_s16( vrshrn_n_s32( t0_s32x4, 10 ), vrshrn_n_s32( t1_s32x4, 10 ) ); + vst1_s8( &pulses[ offset ], vmovn_s16( t_s16x8 ) ); + + t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); + o0_s32x4 = vqdmulhq_lane_s32( t0_s32x4, gain_lo_s32x2, 0 ); + o1_s32x4 = vqdmulhq_lane_s32( t1_s32x4, gain_lo_s32x2, 0 ); + o0_s32x4 = vmlaq_lane_s32( o0_s32x4, t0_s32x4, gain_hi_s32x2, 0 ); + o1_s32x4 = vmlaq_lane_s32( o1_s32x4, t1_s32x4, gain_hi_s32x2, 0 ); + o0_s32x4 = vrshlq_s32( o0_s32x4, shift_s32x4 ); + o1_s32x4 = vrshlq_s32( o1_s32x4, shift_s32x4 ); + vst1_s16( &pxq[ offset + 0 ], vqmovn_s32( o0_s32x4 ) ); + vst1_s16( &pxq[ offset + 4 ], vqmovn_s32( o1_s32x4 ) ); + + t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); + t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); + t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); + vst1q_s32( &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx + offset + 0 ], t0_s32x4 ); + vst1q_s32( &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx + offset + 4 ], t1_s32x4 ); +} + +static OPUS_INLINE void copy_winner_state( + const NSQ_del_decs_struct *psDelDec, + const opus_int decisionDelay, + const opus_int smpl_buf_idx, + const opus_int Winner_ind, + const opus_int32 gain, + const opus_int32 shift, + opus_int8 *const pulses, + opus_int16 *pxq, + silk_nsq_state *NSQ +) +{ + opus_int i, last_smple_idx; + const int32x2_t gain_lo_s32x2 = vdup_n_s32( silk_LSHIFT32( gain & 0x0000FFFF, 15 ) ); + const int32x2_t gain_hi_s32x2 = vdup_n_s32( gain >> 16 ); + const int32x4_t shift_s32x4 = vdupq_n_s32( -shift ); + int32x4_t t0_s32x4, t1_s32x4; + + t0_s32x4 = t1_s32x4 = vdupq_n_s32( 0 ); /* initialization */ + last_smple_idx = smpl_buf_idx + decisionDelay - 1 + DECISION_DELAY; + if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; + if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; + + for( i = 0; ( i < ( decisionDelay - 7 ) ) && ( last_smple_idx >= 7 ); i += 8, last_smple_idx -= 8 ) { + copy_winner_state_kernel( psDelDec, i - decisionDelay, last_smple_idx, Winner_ind, gain_lo_s32x2, gain_hi_s32x2, shift_s32x4, t0_s32x4, t1_s32x4, pulses, pxq, NSQ ); + } + for( ; ( i < decisionDelay ) && ( last_smple_idx >= 0 ); i++, last_smple_idx-- ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], gain ), shift ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; + } + + last_smple_idx += DECISION_DELAY; + for( ; i < ( decisionDelay - 7 ); i++, last_smple_idx-- ) { + copy_winner_state_kernel( psDelDec, i - decisionDelay, last_smple_idx, Winner_ind, gain_lo_s32x2, gain_hi_s32x2, shift_s32x4, t0_s32x4, t1_s32x4, pulses, pxq, NSQ ); + } + for( ; i < decisionDelay; i++, last_smple_idx-- ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], gain ), shift ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; + } +} + +void silk_NSQ_del_dec_neon( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) +{ +#ifdef OPUS_CHECK_ASM + silk_nsq_state NSQ_c; + SideInfoIndices psIndices_c; + opus_int8 pulses_c[ MAX_FRAME_LENGTH ]; + const opus_int8 *const pulses_a = pulses; + + ( void )pulses_a; + silk_memcpy( &NSQ_c, NSQ, sizeof( NSQ_c ) ); + silk_memcpy( &psIndices_c, psIndices, sizeof( psIndices_c ) ); + silk_memcpy( pulses_c, pulses, sizeof( pulses_c ) ); + silk_NSQ_del_dec_c( psEncC, &NSQ_c, &psIndices_c, x16, pulses_c, PredCoef_Q12, LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, + pitchL, Lambda_Q10, LTP_scale_Q14 ); +#endif + + /* The optimization parallelizes the different delay decision states. */ + if(( psEncC->nStatesDelayedDecision > NEON_MAX_DEL_DEC_STATES ) || ( psEncC->nStatesDelayedDecision <= 2 )) { + /* NEON intrinsics optimization now can only parallelize up to 4 delay decision states. */ + /* If there are more states, C function is called, and this optimization must be expanded. */ + /* When the number of delay decision states is less than 3, there are penalties using this */ + /* optimization, and C function is called. */ + /* When the number of delay decision states is 2, it's better to specialize another */ + /* structure NSQ_del_dec2_struct and optimize with shorter NEON registers. (Low priority) */ + silk_NSQ_del_dec_c( psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, + Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14 ); + } else { + opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr; + opus_int smpl_buf_idx, decisionDelay; + const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; + opus_int16 *pxq; + VARDECL( opus_int32, sLTP_Q15 ); + VARDECL( opus_int16, sLTP ); + opus_int32 HarmShapeFIRPacked_Q14; + opus_int offset_Q10; + opus_int32 RDmin_Q10, Gain_Q10; + VARDECL( opus_int32, x_sc_Q10 ); + VARDECL( opus_int32, delayedGain_Q10 ); + VARDECL( NSQ_del_decs_struct, psDelDec ); + int32x4_t t_s32x4; + SAVE_STACK; + + /* Set unvoiced lag to the previous one, overwrite later for voiced */ + lag = NSQ->lagPrev; + + silk_assert( NSQ->prev_gain_Q16 != 0 ); + + /* Initialize delayed decision states */ + ALLOC( psDelDec, 1, NSQ_del_decs_struct ); + /* Only RandState and RD_Q10 need to be initialized to 0. */ + silk_memset( psDelDec->RandState, 0, sizeof( psDelDec->RandState ) ); + vst1q_s32( psDelDec->RD_Q10, vdupq_n_s32( 0 ) ); + + for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) { + psDelDec->SeedInit[ k ] = psDelDec->Seed[ k ] = ( k + psIndices->Seed ) & 3; + } + vst1q_s32( psDelDec->LF_AR_Q14, vld1q_dup_s32( &NSQ->sLF_AR_shp_Q14 ) ); + vst1q_s32( psDelDec->Diff_Q14, vld1q_dup_s32( &NSQ->sDiff_shp_Q14 ) ); + vst1q_s32( psDelDec->Shape_Q14[ 0 ], vld1q_dup_s32( &NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ] ) ); + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + vst1q_s32( psDelDec->sLPC_Q14[ i ], vld1q_dup_s32( &NSQ->sLPC_Q14[ i ] ) ); + } + for( i = 0; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) ); i++ ) { + vst1q_s32( psDelDec->sAR2_Q14[ i ], vld1q_dup_s32( &NSQ->sAR2_Q14[ i ] ) ); + } + + offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; + smpl_buf_idx = 0; /* index of oldest samples */ + + decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length ); + + /* For voiced frames limit the decision delay to lower than the pitch lag */ + if( psIndices->signalType == TYPE_VOICED ) { + opus_int pitch_min = pitchL[ 0 ]; + for( k = 1; k < psEncC->nb_subfr; k++ ) { + pitch_min = silk_min_int( pitch_min, pitchL[ k ] ); + } + decisionDelay = silk_min_int( decisionDelay, pitch_min - LTP_ORDER / 2 - 1 ); + } else { + if( lag > 0 ) { + decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 ); + } + } + + if( psIndices->NLSFInterpCoef_Q2 == 4 ) { + LSF_interpolation_flag = 0; + } else { + LSF_interpolation_flag = 1; + } + + ALLOC( sLTP_Q15, psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); + ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); + ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); + ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 ); + /* Set up pointers to start of sub frame */ + pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; + NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + subfr = 0; + for( k = 0; k < psEncC->nb_subfr; k++ ) { + A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ]; + B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; + AR_shp_Q13 = &AR_Q13[ k * MAX_SHAPE_LPC_ORDER ]; + + /* Noise shape parameters */ + silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); + HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); + HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); + + NSQ->rewhite_flag = 0; + if( psIndices->signalType == TYPE_VOICED ) { + /* Voiced */ + lag = pitchL[ k ]; + + /* Re-whitening */ + if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { + if( k == 2 ) { + /* RESET DELAYED DECISIONS */ + /* Find winner */ + int32x4_t RD_Q10_s32x4; + RDmin_Q10 = psDelDec->RD_Q10[ 0 ]; + Winner_ind = 0; + for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) { + if( psDelDec->RD_Q10[ i ] < RDmin_Q10 ) { + RDmin_Q10 = psDelDec->RD_Q10[ i ]; + Winner_ind = i; + } + } + psDelDec->RD_Q10[ Winner_ind ] -= ( silk_int32_MAX >> 4 ); + RD_Q10_s32x4 = vld1q_s32( psDelDec->RD_Q10 ); + RD_Q10_s32x4 = vaddq_s32( RD_Q10_s32x4, vdupq_n_s32( silk_int32_MAX >> 4 ) ); + vst1q_s32( psDelDec->RD_Q10, RD_Q10_s32x4 ); + + /* Copy final part of signals from winner state to output and long-term filter states */ + copy_winner_state( psDelDec, decisionDelay, smpl_buf_idx, Winner_ind, Gains_Q16[ 1 ], 14, pulses, pxq, NSQ ); + + subfr = 0; + } + + /* Rewhiten with new A coefs */ + start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; + silk_assert( start_idx > 0 ); + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], + A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); + + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + NSQ->rewhite_flag = 1; + } + } + + silk_nsq_del_dec_scale_states_neon( psEncC, NSQ, psDelDec, x16, x_sc_Q10, sLTP, sLTP_Q15, k, + LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay ); + + silk_noise_shape_quantizer_del_dec_neon( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, + delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], + Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder, + psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay ); + + x16 += psEncC->subfr_length; + pulses += psEncC->subfr_length; + pxq += psEncC->subfr_length; + } + + /* Find winner */ + RDmin_Q10 = psDelDec->RD_Q10[ 0 ]; + Winner_ind = 0; + for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) { + if( psDelDec->RD_Q10[ k ] < RDmin_Q10 ) { + RDmin_Q10 = psDelDec->RD_Q10[ k ]; + Winner_ind = k; + } + } + + /* Copy final part of signals from winner state to output and long-term filter states */ + psIndices->Seed = psDelDec->SeedInit[ Winner_ind ]; + Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 ); + copy_winner_state( psDelDec, decisionDelay, smpl_buf_idx, Winner_ind, Gain_Q10, 8, pulses, pxq, NSQ ); + + t_s32x4 = vdupq_n_s32( 0 ); /* initialization */ + for( i = 0; i < ( NSQ_LPC_BUF_LENGTH - 3 ); i += 4 ) { + t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 0 ][ Winner_ind ], t_s32x4, 0 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 1 ][ Winner_ind ], t_s32x4, 1 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 2 ][ Winner_ind ], t_s32x4, 2 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 3 ][ Winner_ind ], t_s32x4, 3 ); + vst1q_s32( &NSQ->sLPC_Q14[ i ], t_s32x4 ); + } + + for( ; i < NSQ_LPC_BUF_LENGTH; i++ ) { + NSQ->sLPC_Q14[ i ] = psDelDec->sLPC_Q14[ i ][ Winner_ind ]; + } + + for( i = 0; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) - 3 ); i += 4 ) { + t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 0 ][ Winner_ind ], t_s32x4, 0 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 1 ][ Winner_ind ], t_s32x4, 1 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 2 ][ Winner_ind ], t_s32x4, 2 ); + t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 3 ][ Winner_ind ], t_s32x4, 3 ); + vst1q_s32( &NSQ->sAR2_Q14[ i ], t_s32x4 ); + } + + for( ; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) ); i++ ) { + NSQ->sAR2_Q14[ i ] = psDelDec->sAR2_Q14[ i ][ Winner_ind ]; + } + + /* Update states */ + NSQ->sLF_AR_shp_Q14 = psDelDec->LF_AR_Q14[ Winner_ind ]; + NSQ->sDiff_shp_Q14 = psDelDec->Diff_Q14[ Winner_ind ]; + NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; + + /* Save quantized speech signal */ + silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); + silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); + RESTORE_STACK; + } + +#ifdef OPUS_CHECK_ASM + silk_assert( !memcmp( &NSQ_c, NSQ, sizeof( NSQ_c ) ) ); + silk_assert( !memcmp( &psIndices_c, psIndices, sizeof( psIndices_c ) ) ); + silk_assert( !memcmp( pulses_c, pulses_a, sizeof( pulses_c ) ) ); +#endif +} + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +/* Note: Function silk_short_prediction_create_arch_coef_neon() defined in NSQ_neon.h is actually a hacking C function. */ +/* Therefore here we append "_local" to the NEON function name to avoid confusion. */ +static OPUS_INLINE void silk_short_prediction_create_arch_coef_neon_local(opus_int32 *out, const opus_int16 *in, opus_int order) +{ + int16x8_t t_s16x8; + int32x4_t t0_s32x4, t1_s32x4, t2_s32x4, t3_s32x4; + silk_assert( order == 10 || order == 16 ); + + t_s16x8 = vld1q_s16( in + 0 ); /* 7 6 5 4 3 2 1 0 */ + t_s16x8 = vrev64q_s16( t_s16x8 ); /* 4 5 6 7 0 1 2 3 */ + t2_s32x4 = vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ); /* 4 5 6 7 */ + t3_s32x4 = vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ); /* 0 1 2 3 */ + + if( order == 16 ) { + t_s16x8 = vld1q_s16( in + 8 ); /* F E D C B A 9 8 */ + t_s16x8 = vrev64q_s16( t_s16x8 ); /* C D E F 8 9 A B */ + t0_s32x4 = vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ); /* C D E F */ + t1_s32x4 = vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ); /* 8 9 A B */ + } else { + int16x4_t t_s16x4; + + t0_s32x4 = vdupq_n_s32( 0 ); /* zero zero zero zero */ + t_s16x4 = vld1_s16( in + 6 ); /* 9 8 7 6 */ + t_s16x4 = vrev64_s16( t_s16x4 ); /* 6 7 8 9 */ + t1_s32x4 = vshll_n_s16( t_s16x4, 15 ); + t1_s32x4 = vcombine_s32( vget_low_s32(t0_s32x4), vget_low_s32( t1_s32x4 ) ); /* 8 9 zero zero */ + } + vst1q_s32( out + 0, t0_s32x4 ); + vst1q_s32( out + 4, t1_s32x4 ); + vst1q_s32( out + 8, t2_s32x4 ); + vst1q_s32( out + 12, t3_s32x4 ); +} + +static OPUS_INLINE int32x4_t silk_SMLAWB_lane0_neon( + const int32x4_t out_s32x4, + const int32x4_t in_s32x4, + const int32x2_t coef_s32x2 +) +{ + return vaddq_s32( out_s32x4, vqdmulhq_lane_s32( in_s32x4, coef_s32x2, 0 ) ); +} + +static OPUS_INLINE int32x4_t silk_SMLAWB_lane1_neon( + const int32x4_t out_s32x4, + const int32x4_t in_s32x4, + const int32x2_t coef_s32x2 +) +{ + return vaddq_s32( out_s32x4, vqdmulhq_lane_s32( in_s32x4, coef_s32x2, 1 ) ); +} + +/* Note: This function has different return value than silk_noise_shape_quantizer_short_prediction_neon(). */ +/* Therefore here we append "_local" to the function name to avoid confusion. */ +static OPUS_INLINE int32x4_t silk_noise_shape_quantizer_short_prediction_neon_local(const opus_int32 *buf32, const opus_int32 *a_Q12_arch, opus_int order) +{ + const int32x4_t a_Q12_arch0_s32x4 = vld1q_s32( a_Q12_arch + 0 ); + const int32x4_t a_Q12_arch1_s32x4 = vld1q_s32( a_Q12_arch + 4 ); + const int32x4_t a_Q12_arch2_s32x4 = vld1q_s32( a_Q12_arch + 8 ); + const int32x4_t a_Q12_arch3_s32x4 = vld1q_s32( a_Q12_arch + 12 ); + int32x4_t LPC_pred_Q14_s32x4; + + silk_assert( order == 10 || order == 16 ); + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q14_s32x4 = vdupq_n_s32( silk_RSHIFT( order, 1 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 0 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch0_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 1 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch0_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 2 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch0_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 3 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch0_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 4 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch1_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 5 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch1_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 6 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch1_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 7 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch1_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 8 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch2_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 9 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch2_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 10 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch2_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 11 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch2_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 12 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch3_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 13 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch3_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 14 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch3_s32x4 ) ); + LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 15 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch3_s32x4 ) ); + + return LPC_pred_Q14_s32x4; +} + +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_neon( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay /* I */ +) +{ + opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx; + opus_int32 Winner_rand_state; + opus_int32 LTP_pred_Q14, n_LTP_Q14; + opus_int32 RDmin_Q10, RDmax_Q10; + opus_int32 Gain_Q10; + opus_int32 *pred_lag_ptr, *shp_lag_ptr; + opus_int32 a_Q12_arch[MAX_LPC_ORDER]; + const int32x2_t warping_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( warping_Q16, 16 ) >> 1 ); + const opus_int32 LF_shp_Q29 = silk_LSHIFT32( LF_shp_Q14, 16 ) >> 1; + opus_int32 AR_shp_Q28[ MAX_SHAPE_LPC_ORDER ]; + const uint32x4_t rand_multiplier_u32x4 = vdupq_n_u32( RAND_MULTIPLIER ); + const uint32x4_t rand_increment_u32x4 = vdupq_n_u32( RAND_INCREMENT ); + + VARDECL( NSQ_samples_struct, psSampleState ); + SAVE_STACK; + + silk_assert( nStatesDelayedDecision > 0 ); + silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ + ALLOC( psSampleState, 2, NSQ_samples_struct ); + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + + for( i = 0; i < ( MAX_SHAPE_LPC_ORDER - 7 ); i += 8 ) { + const int16x8_t t_s16x8 = vld1q_s16( AR_shp_Q13 + i ); + vst1q_s32( AR_shp_Q28 + i + 0, vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ) ); + vst1q_s32( AR_shp_Q28 + i + 4, vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ) ); + } + + for( ; i < MAX_SHAPE_LPC_ORDER; i++ ) { + AR_shp_Q28[i] = silk_LSHIFT32( AR_shp_Q13[i], 15 ); + } + + silk_short_prediction_create_arch_coef_neon_local( a_Q12_arch, a_Q12, predictLPCOrder ); + + for( i = 0; i < length; i++ ) { + int32x4_t Seed_s32x4, LPC_pred_Q14_s32x4; + int32x4_t sign_s32x4, tmp1_s32x4, tmp2_s32x4; + int32x4_t n_AR_Q14_s32x4, n_LF_Q14_s32x4; + int32x2_t AR_shp_Q28_s32x2; + int16x4_t r_Q10_s16x4, rr_Q10_s16x4; + + /* Perform common calculations used in all states */ + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q14 = 2; + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ 0 ], b_Q14[ 0 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -1 ], b_Q14[ 1 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -2 ], b_Q14[ 2 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -3 ], b_Q14[ 3 ] ); + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); + LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */ + pred_lag_ptr++; + } else { + LTP_pred_Q14 = 0; + } + + /* Long-term shaping */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */ + shp_lag_ptr++; + } else { + n_LTP_Q14 = 0; + } + + /* Generate dither */ + Seed_s32x4 = vld1q_s32( psDelDec->Seed ); + Seed_s32x4 = vreinterpretq_s32_u32( vmlaq_u32( rand_increment_u32x4, vreinterpretq_u32_s32( Seed_s32x4 ), rand_multiplier_u32x4 ) ); + vst1q_s32( psDelDec->Seed, Seed_s32x4 ); + + /* Short-term prediction */ + LPC_pred_Q14_s32x4 = silk_noise_shape_quantizer_short_prediction_neon_local(psDelDec->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 16 + i ], a_Q12_arch, predictLPCOrder); + LPC_pred_Q14_s32x4 = vshlq_n_s32( LPC_pred_Q14_s32x4, 4 ); /* Q10 -> Q14 */ + + /* Noise shape feedback */ + /* Output of lowpass section */ + tmp2_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->Diff_Q14 ), vld1q_s32( psDelDec->sAR2_Q14[ 0 ] ), warping_Q16_s32x2 ); + /* Output of allpass section */ + tmp1_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ 1 ] ), tmp2_s32x4 ); + tmp1_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ 0 ] ), tmp1_s32x4, warping_Q16_s32x2 ); + vst1q_s32( psDelDec->sAR2_Q14[ 0 ], tmp2_s32x4 ); + AR_shp_Q28_s32x2 = vld1_s32( AR_shp_Q28 ); + n_AR_Q14_s32x4 = vaddq_s32( vdupq_n_s32( silk_RSHIFT( shapingLPCOrder, 1 ) ), vqdmulhq_lane_s32( tmp2_s32x4, AR_shp_Q28_s32x2, 0 ) ); + + /* Loop over allpass sections */ + for( j = 2; j < shapingLPCOrder; j += 2 ) { + /* Output of allpass section */ + tmp2_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ j + 0 ] ), tmp1_s32x4 ); + tmp2_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ j - 1 ] ), tmp2_s32x4, warping_Q16_s32x2 ); + vst1q_s32( psDelDec->sAR2_Q14[ j - 1 ], tmp1_s32x4 ); + n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp1_s32x4, AR_shp_Q28_s32x2, 1 ) ); + /* Output of allpass section */ + tmp1_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ j + 1 ] ), tmp2_s32x4 ); + tmp1_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ j + 0 ] ), tmp1_s32x4, warping_Q16_s32x2 ); + vst1q_s32( psDelDec->sAR2_Q14[ j + 0 ], tmp2_s32x4 ); + AR_shp_Q28_s32x2 = vld1_s32( &AR_shp_Q28[ j ] ); + n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp2_s32x4, AR_shp_Q28_s32x2, 0 ) ); + } + vst1q_s32( psDelDec->sAR2_Q14[ shapingLPCOrder - 1 ], tmp1_s32x4 ); + n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp1_s32x4, AR_shp_Q28_s32x2, 1 ) ); + n_AR_Q14_s32x4 = vshlq_n_s32( n_AR_Q14_s32x4, 1 ); /* Q11 -> Q12 */ + n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_n_s32( vld1q_s32( psDelDec->LF_AR_Q14 ), silk_LSHIFT32( Tilt_Q14, 16 ) >> 1 ) ); /* Q12 */ + n_AR_Q14_s32x4 = vshlq_n_s32( n_AR_Q14_s32x4, 2 ); /* Q12 -> Q14 */ + n_LF_Q14_s32x4 = vqdmulhq_n_s32( vld1q_s32( psDelDec->Shape_Q14[ *smpl_buf_idx ] ), LF_shp_Q29 ); /* Q12 */ + n_LF_Q14_s32x4 = vaddq_s32( n_LF_Q14_s32x4, vqdmulhq_n_s32( vld1q_s32( psDelDec->LF_AR_Q14 ), silk_LSHIFT32( LF_shp_Q14 >> 16 , 15 ) ) ); /* Q12 */ + n_LF_Q14_s32x4 = vshlq_n_s32( n_LF_Q14_s32x4, 2 ); /* Q12 -> Q14 */ + + /* Input minus prediction plus noise feedback */ + /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */ + tmp1_s32x4 = vaddq_s32( n_AR_Q14_s32x4, n_LF_Q14_s32x4 ); /* Q14 */ + tmp2_s32x4 = vaddq_s32( vdupq_n_s32( n_LTP_Q14 ), LPC_pred_Q14_s32x4 ); /* Q13 */ + tmp1_s32x4 = vsubq_s32( tmp2_s32x4, tmp1_s32x4 ); /* Q13 */ + tmp1_s32x4 = vrshrq_n_s32( tmp1_s32x4, 4 ); /* Q10 */ + tmp1_s32x4 = vsubq_s32( vdupq_n_s32( x_Q10[ i ] ), tmp1_s32x4 ); /* residual error Q10 */ + + /* Flip sign depending on dither */ + sign_s32x4 = vreinterpretq_s32_u32( vcltq_s32( Seed_s32x4, vdupq_n_s32( 0 ) ) ); + tmp1_s32x4 = veorq_s32( tmp1_s32x4, sign_s32x4 ); + tmp1_s32x4 = vsubq_s32( tmp1_s32x4, sign_s32x4 ); + tmp1_s32x4 = vmaxq_s32( tmp1_s32x4, vdupq_n_s32( -( 31 << 10 ) ) ); + tmp1_s32x4 = vminq_s32( tmp1_s32x4, vdupq_n_s32( 30 << 10 ) ); + r_Q10_s16x4 = vmovn_s32( tmp1_s32x4 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + { + int16x4_t q1_Q10_s16x4 = vsub_s16( r_Q10_s16x4, vdup_n_s16( offset_Q10 ) ); + int16x4_t q1_Q0_s16x4 = vshr_n_s16( q1_Q10_s16x4, 10 ); + int16x4_t q2_Q10_s16x4; + int32x4_t rd1_Q10_s32x4, rd2_Q10_s32x4; + uint32x4_t t_u32x4; + + if( Lambda_Q10 > 2048 ) { + /* For aggressive RDO, the bias becomes more than one pulse. */ + const int rdo_offset = Lambda_Q10/2 - 512; + const uint16x4_t greaterThanRdo = vcgt_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ); + const uint16x4_t lessThanMinusRdo = vclt_s16( q1_Q10_s16x4, vdup_n_s16( -rdo_offset ) ); + /* If Lambda_Q10 > 32767, then q1_Q0, q1_Q10 and q2_Q10 must change to 32-bit. */ + silk_assert( Lambda_Q10 <= 32767 ); + + q1_Q0_s16x4 = vreinterpret_s16_u16( vclt_s16( q1_Q10_s16x4, vdup_n_s16( 0 ) ) ); + q1_Q0_s16x4 = vbsl_s16( greaterThanRdo, vsub_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ), q1_Q0_s16x4 ); + q1_Q0_s16x4 = vbsl_s16( lessThanMinusRdo, vadd_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ), q1_Q0_s16x4 ); + q1_Q0_s16x4 = vshr_n_s16( q1_Q0_s16x4, 10 ); + } + { + const uint16x4_t equal0_u16x4 = vceq_s16( q1_Q0_s16x4, vdup_n_s16( 0 ) ); + const uint16x4_t equalMinus1_u16x4 = vceq_s16( q1_Q0_s16x4, vdup_n_s16( -1 ) ); + const uint16x4_t lessThanMinus1_u16x4 = vclt_s16( q1_Q0_s16x4, vdup_n_s16( -1 ) ); + int16x4_t tmp1_s16x4, tmp2_s16x4; + + q1_Q10_s16x4 = vshl_n_s16( q1_Q0_s16x4, 10 ); + tmp1_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( offset_Q10 - QUANT_LEVEL_ADJUST_Q10 ) ); + q1_Q10_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( offset_Q10 + QUANT_LEVEL_ADJUST_Q10 ) ); + q1_Q10_s16x4 = vbsl_s16( lessThanMinus1_u16x4, q1_Q10_s16x4, tmp1_s16x4 ); + q1_Q10_s16x4 = vbsl_s16( equal0_u16x4, vdup_n_s16( offset_Q10 ), q1_Q10_s16x4 ); + q1_Q10_s16x4 = vbsl_s16( equalMinus1_u16x4, vdup_n_s16( offset_Q10 - ( 1024 - QUANT_LEVEL_ADJUST_Q10 ) ), q1_Q10_s16x4 ); + q2_Q10_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( 1024 ) ); + q2_Q10_s16x4 = vbsl_s16( equal0_u16x4, vdup_n_s16( offset_Q10 + 1024 - QUANT_LEVEL_ADJUST_Q10 ), q2_Q10_s16x4 ); + q2_Q10_s16x4 = vbsl_s16( equalMinus1_u16x4, vdup_n_s16( offset_Q10 ), q2_Q10_s16x4 ); + tmp1_s16x4 = q1_Q10_s16x4; + tmp2_s16x4 = q2_Q10_s16x4; + tmp1_s16x4 = vbsl_s16( vorr_u16( equalMinus1_u16x4, lessThanMinus1_u16x4 ), vneg_s16( tmp1_s16x4 ), tmp1_s16x4 ); + tmp2_s16x4 = vbsl_s16( lessThanMinus1_u16x4, vneg_s16( tmp2_s16x4 ), tmp2_s16x4 ); + rd1_Q10_s32x4 = vmull_s16( tmp1_s16x4, vdup_n_s16( Lambda_Q10 ) ); + rd2_Q10_s32x4 = vmull_s16( tmp2_s16x4, vdup_n_s16( Lambda_Q10 ) ); + } + + rr_Q10_s16x4 = vsub_s16( r_Q10_s16x4, q1_Q10_s16x4 ); + rd1_Q10_s32x4 = vmlal_s16( rd1_Q10_s32x4, rr_Q10_s16x4, rr_Q10_s16x4 ); + rd1_Q10_s32x4 = vshrq_n_s32( rd1_Q10_s32x4, 10 ); + + rr_Q10_s16x4 = vsub_s16( r_Q10_s16x4, q2_Q10_s16x4 ); + rd2_Q10_s32x4 = vmlal_s16( rd2_Q10_s32x4, rr_Q10_s16x4, rr_Q10_s16x4 ); + rd2_Q10_s32x4 = vshrq_n_s32( rd2_Q10_s32x4, 10 ); + + tmp2_s32x4 = vld1q_s32( psDelDec->RD_Q10 ); + tmp1_s32x4 = vaddq_s32( tmp2_s32x4, vminq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ) ); + tmp2_s32x4 = vaddq_s32( tmp2_s32x4, vmaxq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ) ); + vst1q_s32( psSampleState[ 0 ].RD_Q10, tmp1_s32x4 ); + vst1q_s32( psSampleState[ 1 ].RD_Q10, tmp2_s32x4 ); + t_u32x4 = vcltq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ); + tmp1_s32x4 = vbslq_s32( t_u32x4, vmovl_s16( q1_Q10_s16x4 ), vmovl_s16( q2_Q10_s16x4 ) ); + tmp2_s32x4 = vbslq_s32( t_u32x4, vmovl_s16( q2_Q10_s16x4 ), vmovl_s16( q1_Q10_s16x4 ) ); + vst1q_s32( psSampleState[ 0 ].Q_Q10, tmp1_s32x4 ); + vst1q_s32( psSampleState[ 1 ].Q_Q10, tmp2_s32x4 ); + } + + { + /* Update states for best quantization */ + int32x4_t exc_Q14_s32x4, LPC_exc_Q14_s32x4, xq_Q14_s32x4, sLF_AR_shp_Q14_s32x4; + + /* Quantized excitation */ + exc_Q14_s32x4 = vshlq_n_s32( tmp1_s32x4, 4 ); + exc_Q14_s32x4 = veorq_s32( exc_Q14_s32x4, sign_s32x4 ); + exc_Q14_s32x4 = vsubq_s32( exc_Q14_s32x4, sign_s32x4 ); + + /* Add predictions */ + LPC_exc_Q14_s32x4 = vaddq_s32( exc_Q14_s32x4, vdupq_n_s32( LTP_pred_Q14 ) ); + xq_Q14_s32x4 = vaddq_s32( LPC_exc_Q14_s32x4, LPC_pred_Q14_s32x4 ); + + /* Update states */ + tmp1_s32x4 = vsubq_s32( xq_Q14_s32x4, vshlq_n_s32( vdupq_n_s32( x_Q10[ i ] ), 4 ) ); + vst1q_s32( psSampleState[ 0 ].Diff_Q14, tmp1_s32x4 ); + sLF_AR_shp_Q14_s32x4 = vsubq_s32( tmp1_s32x4, n_AR_Q14_s32x4 ); + vst1q_s32( psSampleState[ 0 ].sLTP_shp_Q14, vsubq_s32( sLF_AR_shp_Q14_s32x4, n_LF_Q14_s32x4 ) ); + vst1q_s32( psSampleState[ 0 ].LF_AR_Q14, sLF_AR_shp_Q14_s32x4 ); + vst1q_s32( psSampleState[ 0 ].LPC_exc_Q14, LPC_exc_Q14_s32x4 ); + vst1q_s32( psSampleState[ 0 ].xq_Q14, xq_Q14_s32x4 ); + + /* Quantized excitation */ + exc_Q14_s32x4 = vshlq_n_s32( tmp2_s32x4, 4 ); + exc_Q14_s32x4 = veorq_s32( exc_Q14_s32x4, sign_s32x4 ); + exc_Q14_s32x4 = vsubq_s32( exc_Q14_s32x4, sign_s32x4 ); + + /* Add predictions */ + LPC_exc_Q14_s32x4 = vaddq_s32( exc_Q14_s32x4, vdupq_n_s32( LTP_pred_Q14 ) ); + xq_Q14_s32x4 = vaddq_s32( LPC_exc_Q14_s32x4, LPC_pred_Q14_s32x4 ); + + /* Update states */ + tmp1_s32x4 = vsubq_s32( xq_Q14_s32x4, vshlq_n_s32( vdupq_n_s32( x_Q10[ i ] ), 4 ) ); + vst1q_s32( psSampleState[ 1 ].Diff_Q14, tmp1_s32x4 ); + sLF_AR_shp_Q14_s32x4 = vsubq_s32( tmp1_s32x4, n_AR_Q14_s32x4 ); + vst1q_s32( psSampleState[ 1 ].sLTP_shp_Q14, vsubq_s32( sLF_AR_shp_Q14_s32x4, n_LF_Q14_s32x4 ) ); + vst1q_s32( psSampleState[ 1 ].LF_AR_Q14, sLF_AR_shp_Q14_s32x4 ); + vst1q_s32( psSampleState[ 1 ].LPC_exc_Q14, LPC_exc_Q14_s32x4 ); + vst1q_s32( psSampleState[ 1 ].xq_Q14, xq_Q14_s32x4 ); + } + + *smpl_buf_idx = *smpl_buf_idx ? ( *smpl_buf_idx - 1 ) : ( DECISION_DELAY - 1); + last_smple_idx = *smpl_buf_idx + decisionDelay + DECISION_DELAY; + if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; + if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; + + /* Find winner */ + RDmin_Q10 = psSampleState[ 0 ].RD_Q10[ 0 ]; + Winner_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + if( psSampleState[ 0 ].RD_Q10[ k ] < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ 0 ].RD_Q10[ k ]; + Winner_ind = k; + } + } + + /* Increase RD values of expired states */ + { + uint32x4_t t_u32x4; + Winner_rand_state = psDelDec->RandState[ last_smple_idx ][ Winner_ind ]; + t_u32x4 = vceqq_s32( vld1q_s32( psDelDec->RandState[ last_smple_idx ] ), vdupq_n_s32( Winner_rand_state ) ); + t_u32x4 = vmvnq_u32( t_u32x4 ); + t_u32x4 = vshrq_n_u32( t_u32x4, 5 ); + tmp1_s32x4 = vld1q_s32( psSampleState[ 0 ].RD_Q10 ); + tmp2_s32x4 = vld1q_s32( psSampleState[ 1 ].RD_Q10 ); + tmp1_s32x4 = vaddq_s32( tmp1_s32x4, vreinterpretq_s32_u32( t_u32x4 ) ); + tmp2_s32x4 = vaddq_s32( tmp2_s32x4, vreinterpretq_s32_u32( t_u32x4 ) ); + vst1q_s32( psSampleState[ 0 ].RD_Q10, tmp1_s32x4 ); + vst1q_s32( psSampleState[ 1 ].RD_Q10, tmp2_s32x4 ); + + /* Find worst in first set and best in second set */ + RDmax_Q10 = psSampleState[ 0 ].RD_Q10[ 0 ]; + RDmin_Q10 = psSampleState[ 1 ].RD_Q10[ 0 ]; + RDmax_ind = 0; + RDmin_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + /* find worst in first set */ + if( psSampleState[ 0 ].RD_Q10[ k ] > RDmax_Q10 ) { + RDmax_Q10 = psSampleState[ 0 ].RD_Q10[ k ]; + RDmax_ind = k; + } + /* find best in second set */ + if( psSampleState[ 1 ].RD_Q10[ k ] < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ 1 ].RD_Q10[ k ]; + RDmin_ind = k; + } + } + } + + /* Replace a state if best from second set outperforms worst in first set */ + if( RDmin_Q10 < RDmax_Q10 ) { + opus_int32 (*ptr)[NEON_MAX_DEL_DEC_STATES] = psDelDec->RandState; + const int numOthers = (int)( ( sizeof( NSQ_del_decs_struct ) - sizeof( ( (NSQ_del_decs_struct *)0 )->sLPC_Q14 ) ) + / ( NEON_MAX_DEL_DEC_STATES * sizeof( opus_int32 ) ) ); + /* Only ( predictLPCOrder - 1 ) of sLPC_Q14 buffer need to be updated, though the first several */ + /* useless sLPC_Q14[] will be different comparing with C when predictLPCOrder < NSQ_LPC_BUF_LENGTH. */ + /* Here just update constant ( NSQ_LPC_BUF_LENGTH - 1 ) for simplicity. */ + for( j = i + 1; j < i + NSQ_LPC_BUF_LENGTH; j++ ) { + psDelDec->sLPC_Q14[ j ][ RDmax_ind ] = psDelDec->sLPC_Q14[ j ][ RDmin_ind ]; + } + for( j = 0; j < numOthers; j++ ) { + ptr[ j ][ RDmax_ind ] = ptr[ j ][ RDmin_ind ]; + } + + psSampleState[ 0 ].Q_Q10[ RDmax_ind ] = psSampleState[ 1 ].Q_Q10[ RDmin_ind ]; + psSampleState[ 0 ].RD_Q10[ RDmax_ind ] = psSampleState[ 1 ].RD_Q10[ RDmin_ind ]; + psSampleState[ 0 ].xq_Q14[ RDmax_ind ] = psSampleState[ 1 ].xq_Q14[ RDmin_ind ]; + psSampleState[ 0 ].LF_AR_Q14[ RDmax_ind ] = psSampleState[ 1 ].LF_AR_Q14[ RDmin_ind ]; + psSampleState[ 0 ].Diff_Q14[ RDmax_ind ] = psSampleState[ 1 ].Diff_Q14[ RDmin_ind ]; + psSampleState[ 0 ].sLTP_shp_Q14[ RDmax_ind ] = psSampleState[ 1 ].sLTP_shp_Q14[ RDmin_ind ]; + psSampleState[ 0 ].LPC_exc_Q14[ RDmax_ind ] = psSampleState[ 1 ].LPC_exc_Q14[ RDmin_ind ]; + } + + /* Write samples from winner to output and long-term filter states */ + if( subfr > 0 || i >= decisionDelay ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); + xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], delayedGain_Q10[ last_smple_idx ] ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; + sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDelDec->Pred_Q15[ last_smple_idx ][ Winner_ind ]; + } + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Update states */ + vst1q_s32( psDelDec->LF_AR_Q14, vld1q_s32( psSampleState[ 0 ].LF_AR_Q14 ) ); + vst1q_s32( psDelDec->Diff_Q14, vld1q_s32( psSampleState[ 0 ].Diff_Q14 ) ); + vst1q_s32( psDelDec->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ], vld1q_s32( psSampleState[ 0 ].xq_Q14 ) ); + vst1q_s32( psDelDec->Xq_Q14[ *smpl_buf_idx ], vld1q_s32( psSampleState[ 0 ].xq_Q14 ) ); + tmp1_s32x4 = vld1q_s32( psSampleState[ 0 ].Q_Q10 ); + vst1q_s32( psDelDec->Q_Q10[ *smpl_buf_idx ], tmp1_s32x4 ); + vst1q_s32( psDelDec->Pred_Q15[ *smpl_buf_idx ], vshlq_n_s32( vld1q_s32( psSampleState[ 0 ].LPC_exc_Q14 ), 1 ) ); + vst1q_s32( psDelDec->Shape_Q14[ *smpl_buf_idx ], vld1q_s32( psSampleState[ 0 ].sLTP_shp_Q14 ) ); + tmp1_s32x4 = vrshrq_n_s32( tmp1_s32x4, 10 ); + tmp1_s32x4 = vaddq_s32( vld1q_s32( psDelDec->Seed ), tmp1_s32x4 ); + vst1q_s32( psDelDec->Seed, tmp1_s32x4 ); + vst1q_s32( psDelDec->RandState[ *smpl_buf_idx ], tmp1_s32x4 ); + vst1q_s32( psDelDec->RD_Q10, vld1q_s32( psSampleState[ 0 ].RD_Q10 ) ); + delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10; + } + /* Update LPC states */ + silk_memcpy( psDelDec->sLPC_Q14[ 0 ], psDelDec->sLPC_Q14[ length ], NEON_MAX_DEL_DEC_STATES * NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + + RESTORE_STACK; +} + +static OPUS_INLINE void silk_SMULWB_8_neon( + const opus_int16 *a, + const int32x2_t b, + opus_int32 *o +) +{ + const int16x8_t a_s16x8 = vld1q_s16( a ); + int32x4_t o0_s32x4, o1_s32x4; + + o0_s32x4 = vshll_n_s16( vget_low_s16( a_s16x8 ), 15 ); + o1_s32x4 = vshll_n_s16( vget_high_s16( a_s16x8 ), 15 ); + o0_s32x4 = vqdmulhq_lane_s32( o0_s32x4, b, 0 ); + o1_s32x4 = vqdmulhq_lane_s32( o1_s32x4, b, 0 ); + vst1q_s32( o, o0_s32x4 ); + vst1q_s32( o + 4, o1_s32x4 ); +} + +/* Only works when ( b >= -65536 ) && ( b < 65536 ). */ +static OPUS_INLINE void silk_SMULWW_small_b_4_neon( + opus_int32 *a, + const int32x2_t b_s32x2) +{ + int32x4_t o_s32x4; + + o_s32x4 = vld1q_s32( a ); + o_s32x4 = vqdmulhq_lane_s32( o_s32x4, b_s32x2, 0 ); + vst1q_s32( a, o_s32x4 ); +} + +/* Only works when ( b >= -65536 ) && ( b < 65536 ). */ +static OPUS_INLINE void silk_SMULWW_small_b_8_neon( + opus_int32 *a, + const int32x2_t b_s32x2 +) +{ + int32x4_t o0_s32x4, o1_s32x4; + + o0_s32x4 = vld1q_s32( a ); + o1_s32x4 = vld1q_s32( a + 4 ); + o0_s32x4 = vqdmulhq_lane_s32( o0_s32x4, b_s32x2, 0 ); + o1_s32x4 = vqdmulhq_lane_s32( o1_s32x4, b_s32x2, 0 ); + vst1q_s32( a, o0_s32x4 ); + vst1q_s32( a + 4, o1_s32x4 ); +} + +static OPUS_INLINE void silk_SMULWW_4_neon( + opus_int32 *a, + const int32x2_t b_s32x2) +{ + int32x4_t a_s32x4, o_s32x4; + + a_s32x4 = vld1q_s32( a ); + o_s32x4 = vqdmulhq_lane_s32( a_s32x4, b_s32x2, 0 ); + o_s32x4 = vmlaq_lane_s32( o_s32x4, a_s32x4, b_s32x2, 1 ); + vst1q_s32( a, o_s32x4 ); +} + +static OPUS_INLINE void silk_SMULWW_8_neon( + opus_int32 *a, + const int32x2_t b_s32x2 +) +{ + int32x4_t a0_s32x4, a1_s32x4, o0_s32x4, o1_s32x4; + + a0_s32x4 = vld1q_s32( a ); + a1_s32x4 = vld1q_s32( a + 4 ); + o0_s32x4 = vqdmulhq_lane_s32( a0_s32x4, b_s32x2, 0 ); + o1_s32x4 = vqdmulhq_lane_s32( a1_s32x4, b_s32x2, 0 ); + o0_s32x4 = vmlaq_lane_s32( o0_s32x4, a0_s32x4, b_s32x2, 1 ); + o1_s32x4 = vmlaq_lane_s32( o1_s32x4, a1_s32x4, b_s32x2, 1 ); + vst1q_s32( a, o0_s32x4 ); + vst1q_s32( a + 4, o1_s32x4 ); +} + +static OPUS_INLINE void silk_SMULWW_loop_neon( + const opus_int16 *a, + const opus_int32 b, + opus_int32 *o, + const opus_int loop_num +) +{ + opus_int i; + int32x2_t b_s32x2; + + b_s32x2 = vdup_n_s32( b ); + for( i = 0; i < loop_num - 7; i += 8 ) { + silk_SMULWB_8_neon( a + i, b_s32x2, o + i ); + } + for( ; i < loop_num; i++ ) { + o[ i ] = silk_SMULWW( a[ i ], b ); + } +} + +static OPUS_INLINE void silk_nsq_del_dec_scale_states_neon( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int16 x16[], /* I Input */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +) +{ + opus_int i, lag; + opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q26; + + lag = pitchL[ subfr ]; + inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); + silk_assert( inv_gain_Q31 != 0 ); + + /* Scale input */ + inv_gain_Q26 = silk_RSHIFT_ROUND( inv_gain_Q31, 5 ); + silk_SMULWW_loop_neon( x16, inv_gain_Q26, x_sc_Q10, psEncC->subfr_length ); + + /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ + if( NSQ->rewhite_flag ) { + if( subfr == 0 ) { + /* Do LTP downscaling */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); + } + silk_SMULWW_loop_neon( sLTP + NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2, inv_gain_Q31, sLTP_Q15 + NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2, lag + LTP_ORDER / 2 ); + } + + /* Adjust for changing gain */ + if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { + int32x2_t gain_adj_Q16_s32x2; + gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); + + /* Scale long-term shaping state */ + if( ( gain_adj_Q16 >= -65536 ) && ( gain_adj_Q16 < 65536 ) ) { + gain_adj_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( gain_adj_Q16, 15 ) ); + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 7; i += 8 ) { + silk_SMULWW_small_b_8_neon( NSQ->sLTP_shp_Q14 + i, gain_adj_Q16_s32x2 ); + } + for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay - 7; i += 8 ) { + silk_SMULWW_small_b_8_neon( sLTP_Q15 + i, gain_adj_Q16_s32x2 ); + } + for( ; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + /* Scale scalar states */ + silk_SMULWW_small_b_4_neon( psDelDec->LF_AR_Q14, gain_adj_Q16_s32x2 ); + silk_SMULWW_small_b_4_neon( psDelDec->Diff_Q14, gain_adj_Q16_s32x2 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + silk_SMULWW_small_b_4_neon( psDelDec->sLPC_Q14[ i ], gain_adj_Q16_s32x2 ); + } + + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + silk_SMULWW_small_b_4_neon( psDelDec->sAR2_Q14[ i ], gain_adj_Q16_s32x2 ); + } + + for( i = 0; i < DECISION_DELAY; i++ ) { + silk_SMULWW_small_b_4_neon( psDelDec->Pred_Q15[ i ], gain_adj_Q16_s32x2 ); + silk_SMULWW_small_b_4_neon( psDelDec->Shape_Q14[ i ], gain_adj_Q16_s32x2 ); + } + } else { + gain_adj_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( gain_adj_Q16 & 0x0000FFFF, 15 ) ); + gain_adj_Q16_s32x2 = vset_lane_s32( gain_adj_Q16 >> 16, gain_adj_Q16_s32x2, 1 ); + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 7; i += 8 ) { + silk_SMULWW_8_neon( NSQ->sLTP_shp_Q14 + i, gain_adj_Q16_s32x2 ); + } + for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay - 7; i += 8 ) { + silk_SMULWW_8_neon( sLTP_Q15 + i, gain_adj_Q16_s32x2 ); + } + for( ; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + /* Scale scalar states */ + silk_SMULWW_4_neon( psDelDec->LF_AR_Q14, gain_adj_Q16_s32x2 ); + silk_SMULWW_4_neon( psDelDec->Diff_Q14, gain_adj_Q16_s32x2 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + silk_SMULWW_4_neon( psDelDec->sLPC_Q14[ i ], gain_adj_Q16_s32x2 ); + } + + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + silk_SMULWW_4_neon( psDelDec->sAR2_Q14[ i ], gain_adj_Q16_s32x2 ); + } + + for( i = 0; i < DECISION_DELAY; i++ ) { + silk_SMULWW_4_neon( psDelDec->Pred_Q15[ i ], gain_adj_Q16_s32x2 ); + silk_SMULWW_4_neon( psDelDec->Shape_Q14[ i ], gain_adj_Q16_s32x2 ); + } + } + + /* Save inverse gain */ + NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.c new file mode 100644 index 000000000..964252997 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.c @@ -0,0 +1,112 @@ +/*********************************************************************** +Copyright (C) 2014 Vidyo +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "main.h" +#include "stack_alloc.h" +#include "NSQ.h" +#include "celt/cpu_support.h" +#include "celt/arm/armcpu.h" + +opus_int32 silk_noise_shape_quantizer_short_prediction_neon(const opus_int32 *buf32, const opus_int32 *coef32, opus_int order) +{ + int32x4_t coef0 = vld1q_s32(coef32); + int32x4_t coef1 = vld1q_s32(coef32 + 4); + int32x4_t coef2 = vld1q_s32(coef32 + 8); + int32x4_t coef3 = vld1q_s32(coef32 + 12); + + int32x4_t a0 = vld1q_s32(buf32 - 15); + int32x4_t a1 = vld1q_s32(buf32 - 11); + int32x4_t a2 = vld1q_s32(buf32 - 7); + int32x4_t a3 = vld1q_s32(buf32 - 3); + + int32x4_t b0 = vqdmulhq_s32(coef0, a0); + int32x4_t b1 = vqdmulhq_s32(coef1, a1); + int32x4_t b2 = vqdmulhq_s32(coef2, a2); + int32x4_t b3 = vqdmulhq_s32(coef3, a3); + + int32x4_t c0 = vaddq_s32(b0, b1); + int32x4_t c1 = vaddq_s32(b2, b3); + + int32x4_t d = vaddq_s32(c0, c1); + + int64x2_t e = vpaddlq_s32(d); + + int64x1_t f = vadd_s64(vget_low_s64(e), vget_high_s64(e)); + + opus_int32 out = vget_lane_s32(vreinterpret_s32_s64(f), 0); + + out += silk_RSHIFT( order, 1 ); + + return out; +} + + +opus_int32 silk_NSQ_noise_shape_feedback_loop_neon(const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, opus_int order) +{ + opus_int32 out; + if (order == 8) + { + int32x4_t a00 = vdupq_n_s32(data0[0]); + int32x4_t a01 = vld1q_s32(data1); /* data1[0] ... [3] */ + + int32x4_t a0 = vextq_s32 (a00, a01, 3); /* data0[0] data1[0] ...[2] */ + int32x4_t a1 = vld1q_s32(data1 + 3); /* data1[3] ... [6] */ + + /*TODO: Convert these once in advance instead of once per sample, like + silk_noise_shape_quantizer_short_prediction_neon() does.*/ + int16x8_t coef16 = vld1q_s16(coef); + int32x4_t coef0 = vmovl_s16(vget_low_s16(coef16)); + int32x4_t coef1 = vmovl_s16(vget_high_s16(coef16)); + + /*This is not bit-exact with the C version, since we do not drop the + lower 16 bits of each multiply, but wait until the end to truncate + precision. This is an encoder-specific calculation (and unlike + silk_noise_shape_quantizer_short_prediction_neon(), is not meant to + simulate what the decoder will do). We still could use vqdmulhq_s32() + like silk_noise_shape_quantizer_short_prediction_neon() and save + half the multiplies, but the speed difference is not large, since we + then need two extra adds.*/ + int64x2_t b0 = vmull_s32(vget_low_s32(a0), vget_low_s32(coef0)); + int64x2_t b1 = vmlal_s32(b0, vget_high_s32(a0), vget_high_s32(coef0)); + int64x2_t b2 = vmlal_s32(b1, vget_low_s32(a1), vget_low_s32(coef1)); + int64x2_t b3 = vmlal_s32(b2, vget_high_s32(a1), vget_high_s32(coef1)); + + int64x1_t c = vadd_s64(vget_low_s64(b3), vget_high_s64(b3)); + int64x1_t cS = vrshr_n_s64(c, 15); + int32x2_t d = vreinterpret_s32_s64(cS); + + out = vget_lane_s32(d, 0); + vst1q_s32(data1, a0); + vst1q_s32(data1 + 4, a1); + return out; + } + return silk_NSQ_noise_shape_feedback_loop_c(data0, data1, coef, order); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.h new file mode 100644 index 000000000..b31d9442d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/NSQ_neon.h @@ -0,0 +1,114 @@ +/*********************************************************************** +Copyright (C) 2014 Vidyo +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +#ifndef SILK_NSQ_NEON_H +#define SILK_NSQ_NEON_H + +#include "cpu_support.h" +#include "SigProc_FIX.h" + +#undef silk_short_prediction_create_arch_coef +/* For vectorized calc, reverse a_Q12 coefs, convert to 32-bit, and shift for vqdmulhq_s32. */ +static OPUS_INLINE void silk_short_prediction_create_arch_coef_neon(opus_int32 *out, const opus_int16 *in, opus_int order) +{ + out[15] = silk_LSHIFT32(in[0], 15); + out[14] = silk_LSHIFT32(in[1], 15); + out[13] = silk_LSHIFT32(in[2], 15); + out[12] = silk_LSHIFT32(in[3], 15); + out[11] = silk_LSHIFT32(in[4], 15); + out[10] = silk_LSHIFT32(in[5], 15); + out[9] = silk_LSHIFT32(in[6], 15); + out[8] = silk_LSHIFT32(in[7], 15); + out[7] = silk_LSHIFT32(in[8], 15); + out[6] = silk_LSHIFT32(in[9], 15); + + if (order == 16) + { + out[5] = silk_LSHIFT32(in[10], 15); + out[4] = silk_LSHIFT32(in[11], 15); + out[3] = silk_LSHIFT32(in[12], 15); + out[2] = silk_LSHIFT32(in[13], 15); + out[1] = silk_LSHIFT32(in[14], 15); + out[0] = silk_LSHIFT32(in[15], 15); + } + else + { + out[5] = 0; + out[4] = 0; + out[3] = 0; + out[2] = 0; + out[1] = 0; + out[0] = 0; + } +} + +#if defined(OPUS_ARM_PRESUME_NEON_INTR) + +#define silk_short_prediction_create_arch_coef(out, in, order) \ + (silk_short_prediction_create_arch_coef_neon(out, in, order)) + +#elif defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + +#define silk_short_prediction_create_arch_coef(out, in, order) \ + do { if (arch == OPUS_ARCH_ARM_NEON) { silk_short_prediction_create_arch_coef_neon(out, in, order); } } while (0) + +#endif + +opus_int32 silk_noise_shape_quantizer_short_prediction_neon(const opus_int32 *buf32, const opus_int32 *coef32, opus_int order); + +opus_int32 silk_NSQ_noise_shape_feedback_loop_neon(const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, opus_int order); + +#if defined(OPUS_ARM_PRESUME_NEON_INTR) +#undef silk_noise_shape_quantizer_short_prediction +#define silk_noise_shape_quantizer_short_prediction(in, coef, coefRev, order, arch) \ + ((void)arch,silk_noise_shape_quantizer_short_prediction_neon(in, coefRev, order)) + +#undef silk_NSQ_noise_shape_feedback_loop +#define silk_NSQ_noise_shape_feedback_loop(data0, data1, coef, order, arch) ((void)arch,silk_NSQ_noise_shape_feedback_loop_neon(data0, data1, coef, order)) + +#elif defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_MAY_HAVE_NEON_INTR) + +/* silk_noise_shape_quantizer_short_prediction implementations take different parameters based on arch + (coef vs. coefRev) so can't use the usual IMPL table implementation */ +#undef silk_noise_shape_quantizer_short_prediction +#define silk_noise_shape_quantizer_short_prediction(in, coef, coefRev, order, arch) \ + (arch == OPUS_ARCH_ARM_NEON ? \ + silk_noise_shape_quantizer_short_prediction_neon(in, coefRev, order) : \ + silk_noise_shape_quantizer_short_prediction_c(in, coef, order)) + +extern opus_int32 + (*const SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[OPUS_ARCHMASK+1])( + const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, + opus_int order); + +#undef silk_NSQ_noise_shape_feedback_loop +#define silk_NSQ_noise_shape_feedback_loop(data0, data1, coef, order, arch) \ + (SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[(arch)&OPUS_ARCHMASK](data0, data1, \ + coef, order)) + +#endif + +#endif /* SILK_NSQ_NEON_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv4.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv4.h new file mode 100644 index 000000000..ff62b1e5d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv4.h @@ -0,0 +1,47 @@ +/*********************************************************************** +Copyright (C) 2013 Xiph.Org Foundation and contributors +Copyright (c) 2013 Parrot +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_SIGPROC_FIX_ARMv4_H +#define SILK_SIGPROC_FIX_ARMv4_H + +#undef silk_MLA +static OPUS_INLINE opus_int32 silk_MLA_armv4(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + opus_int32 res; + __asm__( + "#silk_MLA\n\t" + "mla %0, %1, %2, %3\n\t" + : "=&r"(res) + : "r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_MLA(a, b, c) (silk_MLA_armv4(a, b, c)) + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv5e.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv5e.h new file mode 100644 index 000000000..617a09cab --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/SigProc_FIX_armv5e.h @@ -0,0 +1,61 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Copyright (c) 2013 Parrot +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_SIGPROC_FIX_ARMv5E_H +#define SILK_SIGPROC_FIX_ARMv5E_H + +#undef silk_SMULTT +static OPUS_INLINE opus_int32 silk_SMULTT_armv5e(opus_int32 a, opus_int32 b) +{ + opus_int32 res; + __asm__( + "#silk_SMULTT\n\t" + "smultt %0, %1, %2\n\t" + : "=r"(res) + : "%r"(a), "r"(b) + ); + return res; +} +#define silk_SMULTT(a, b) (silk_SMULTT_armv5e(a, b)) + +#undef silk_SMLATT +static OPUS_INLINE opus_int32 silk_SMLATT_armv5e(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + opus_int32 res; + __asm__( + "#silk_SMLATT\n\t" + "smlatt %0, %1, %2, %3\n\t" + : "=r"(res) + : "%r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_SMLATT(a, b, c) (silk_SMLATT_armv5e(a, b, c)) + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/arm_silk_map.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/arm_silk_map.c new file mode 100644 index 000000000..0b9bfec2c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/arm_silk_map.c @@ -0,0 +1,123 @@ +/*********************************************************************** +Copyright (C) 2014 Vidyo +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "main_FIX.h" +#include "NSQ.h" +#include "SigProc_FIX.h" + +#if defined(OPUS_HAVE_RTCD) + +# if (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && \ + !defined(OPUS_ARM_PRESUME_NEON_INTR)) + +void (*const SILK_BIQUAD_ALT_STRIDE2_IMPL[OPUS_ARCHMASK + 1])( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +) = { + silk_biquad_alt_stride2_c, /* ARMv4 */ + silk_biquad_alt_stride2_c, /* EDSP */ + silk_biquad_alt_stride2_c, /* Media */ + silk_biquad_alt_stride2_neon, /* Neon */ +}; + +opus_int32 (*const SILK_LPC_INVERSE_PRED_GAIN_IMPL[OPUS_ARCHMASK + 1])( /* O Returns inverse prediction gain in energy domain, Q30 */ + const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ + const opus_int order /* I Prediction order */ +) = { + silk_LPC_inverse_pred_gain_c, /* ARMv4 */ + silk_LPC_inverse_pred_gain_c, /* EDSP */ + silk_LPC_inverse_pred_gain_c, /* Media */ + silk_LPC_inverse_pred_gain_neon, /* Neon */ +}; + +void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) = { + silk_NSQ_del_dec_c, /* ARMv4 */ + silk_NSQ_del_dec_c, /* EDSP */ + silk_NSQ_del_dec_c, /* Media */ + silk_NSQ_del_dec_neon, /* Neon */ +}; + +/*There is no table for silk_noise_shape_quantizer_short_prediction because the + NEON version takes different parameters than the C version. + Instead RTCD is done via if statements at the call sites. + See NSQ_neon.h for details.*/ + +opus_int32 + (*const SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[OPUS_ARCHMASK+1])( + const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, + opus_int order) = { + silk_NSQ_noise_shape_feedback_loop_c, /* ARMv4 */ + silk_NSQ_noise_shape_feedback_loop_c, /* EDSP */ + silk_NSQ_noise_shape_feedback_loop_c, /* Media */ + silk_NSQ_noise_shape_feedback_loop_neon, /* NEON */ +}; + +# endif + +# if defined(FIXED_POINT) && \ + defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR) + +void (*const SILK_WARPED_AUTOCORRELATION_FIX_IMPL[OPUS_ARCHMASK + 1])( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +) = { + silk_warped_autocorrelation_FIX_c, /* ARMv4 */ + silk_warped_autocorrelation_FIX_c, /* EDSP */ + silk_warped_autocorrelation_FIX_c, /* Media */ + silk_warped_autocorrelation_FIX_neon, /* Neon */ +}; + +# endif + +#endif /* OPUS_HAVE_RTCD */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_arm.h new file mode 100644 index 000000000..66ea9f43d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_arm.h @@ -0,0 +1,68 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_BIQUAD_ALT_ARM_H +# define SILK_BIQUAD_ALT_ARM_H + +# include "celt/arm/armcpu.h" + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +void silk_biquad_alt_stride2_neon( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +); + +# if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) +# define OVERRIDE_silk_biquad_alt_stride2 (1) +# define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((void)(arch), PRESUME_NEON(silk_biquad_alt_stride2)(in, B_Q28, A_Q28, S, out, len)) +# endif +# endif + +# if !defined(OVERRIDE_silk_biquad_alt_stride2) +/*Is run-time CPU detection enabled on this platform?*/ +# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern void (*const SILK_BIQUAD_ALT_STRIDE2_IMPL[OPUS_ARCHMASK+1])( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ + ); +# define OVERRIDE_silk_biquad_alt_stride2 (1) +# define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((*SILK_BIQUAD_ALT_STRIDE2_IMPL[(arch)&OPUS_ARCHMASK])(in, B_Q28, A_Q28, S, out, len)) +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_silk_biquad_alt_stride2 (1) +# define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((void)(arch), silk_biquad_alt_stride2_neon(in, B_Q28, A_Q28, S, out, len)) +# endif +# endif + +#endif /* end SILK_BIQUAD_ALT_ARM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_neon_intr.c new file mode 100644 index 000000000..971573318 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/biquad_alt_neon_intr.c @@ -0,0 +1,156 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#ifdef OPUS_CHECK_ASM +# include +# include "stack_alloc.h" +#endif +#include "SigProc_FIX.h" + +static inline void silk_biquad_alt_stride2_kernel( const int32x4_t A_L_s32x4, const int32x4_t A_U_s32x4, const int32x4_t B_Q28_s32x4, const int32x2_t t_s32x2, const int32x4_t in_s32x4, int32x4_t *S_s32x4, int32x2_t *out32_Q14_s32x2 ) +{ + int32x4_t t_s32x4, out32_Q14_s32x4; + + *out32_Q14_s32x2 = vadd_s32( vget_low_s32( *S_s32x4 ), t_s32x2 ); /* silk_SMLAWB( S{0,1}, B_Q28[ 0 ], in{0,1} ) */ + *S_s32x4 = vcombine_s32( vget_high_s32( *S_s32x4 ), vdup_n_s32( 0 ) ); /* S{0,1} = S{2,3}; S{2,3} = 0; */ + *out32_Q14_s32x2 = vshl_n_s32( *out32_Q14_s32x2, 2 ); /* out32_Q14_{0,1} = silk_LSHIFT( silk_SMLAWB( S{0,1}, B_Q28[ 0 ], in{0,1} ), 2 ); */ + out32_Q14_s32x4 = vcombine_s32( *out32_Q14_s32x2, *out32_Q14_s32x2 ); /* out32_Q14_{0,1,0,1} */ + t_s32x4 = vqdmulhq_s32( out32_Q14_s32x4, A_L_s32x4 ); /* silk_SMULWB( out32_Q14_{0,1,0,1}, A{0,0,1,1}_L_Q28 ) */ + *S_s32x4 = vrsraq_n_s32( *S_s32x4, t_s32x4, 14 ); /* S{0,1} = S{2,3} + silk_RSHIFT_ROUND(); S{2,3} = silk_RSHIFT_ROUND(); */ + t_s32x4 = vqdmulhq_s32( out32_Q14_s32x4, A_U_s32x4 ); /* silk_SMULWB( out32_Q14_{0,1,0,1}, A{0,0,1,1}_U_Q28 ) */ + *S_s32x4 = vaddq_s32( *S_s32x4, t_s32x4 ); /* S0 = silk_SMLAWB( S{0,1,2,3}, out32_Q14_{0,1,0,1}, A{0,0,1,1}_U_Q28 ); */ + t_s32x4 = vqdmulhq_s32( in_s32x4, B_Q28_s32x4 ); /* silk_SMULWB( B_Q28[ {1,1,2,2} ], in{0,1,0,1} ) */ + *S_s32x4 = vaddq_s32( *S_s32x4, t_s32x4 ); /* S0 = silk_SMLAWB( S0, B_Q28[ {1,1,2,2} ], in{0,1,0,1} ); */ +} + +void silk_biquad_alt_stride2_neon( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +) +{ + /* DIRECT FORM II TRANSPOSED (uses 2 element state vector) */ + opus_int k = 0; + const int32x2_t offset_s32x2 = vdup_n_s32( (1<<14) - 1 ); + const int32x4_t offset_s32x4 = vcombine_s32( offset_s32x2, offset_s32x2 ); + int16x4_t in_s16x4 = vdup_n_s16( 0 ); + int16x4_t out_s16x4; + int32x2_t A_Q28_s32x2, A_L_s32x2, A_U_s32x2, B_Q28_s32x2, t_s32x2; + int32x4_t A_L_s32x4, A_U_s32x4, B_Q28_s32x4, S_s32x4, out32_Q14_s32x4; + int32x2x2_t t0_s32x2x2, t1_s32x2x2, t2_s32x2x2, S_s32x2x2; + +#ifdef OPUS_CHECK_ASM + opus_int32 S_c[ 4 ]; + VARDECL( opus_int16, out_c ); + SAVE_STACK; + ALLOC( out_c, 2 * len, opus_int16 ); + + silk_memcpy( &S_c, S, sizeof( S_c ) ); + silk_biquad_alt_stride2_c( in, B_Q28, A_Q28, S_c, out_c, len ); +#endif + + /* Negate A_Q28 values and split in two parts */ + A_Q28_s32x2 = vld1_s32( A_Q28 ); + A_Q28_s32x2 = vneg_s32( A_Q28_s32x2 ); + A_L_s32x2 = vshl_n_s32( A_Q28_s32x2, 18 ); /* ( -A_Q28[] & 0x00003FFF ) << 18 */ + A_L_s32x2 = vreinterpret_s32_u32( vshr_n_u32( vreinterpret_u32_s32( A_L_s32x2 ), 3 ) ); /* ( -A_Q28[] & 0x00003FFF ) << 15 */ + A_U_s32x2 = vshr_n_s32( A_Q28_s32x2, 14 ); /* silk_RSHIFT( -A_Q28[], 14 ) */ + A_U_s32x2 = vshl_n_s32( A_U_s32x2, 16 ); /* silk_RSHIFT( -A_Q28[], 14 ) << 16 (Clip two leading bits to conform to C function.) */ + A_U_s32x2 = vshr_n_s32( A_U_s32x2, 1 ); /* silk_RSHIFT( -A_Q28[], 14 ) << 15 */ + + B_Q28_s32x2 = vld1_s32( B_Q28 ); + t_s32x2 = vld1_s32( B_Q28 + 1 ); + t0_s32x2x2 = vzip_s32( A_L_s32x2, A_L_s32x2 ); + t1_s32x2x2 = vzip_s32( A_U_s32x2, A_U_s32x2 ); + t2_s32x2x2 = vzip_s32( t_s32x2, t_s32x2 ); + A_L_s32x4 = vcombine_s32( t0_s32x2x2.val[ 0 ], t0_s32x2x2.val[ 1 ] ); /* A{0,0,1,1}_L_Q28 */ + A_U_s32x4 = vcombine_s32( t1_s32x2x2.val[ 0 ], t1_s32x2x2.val[ 1 ] ); /* A{0,0,1,1}_U_Q28 */ + B_Q28_s32x4 = vcombine_s32( t2_s32x2x2.val[ 0 ], t2_s32x2x2.val[ 1 ] ); /* B_Q28[ {1,1,2,2} ] */ + S_s32x4 = vld1q_s32( S ); /* S0 = S[ 0 ]; S3 = S[ 3 ]; */ + S_s32x2x2 = vtrn_s32( vget_low_s32( S_s32x4 ), vget_high_s32( S_s32x4 ) ); /* S2 = S[ 1 ]; S1 = S[ 2 ]; */ + S_s32x4 = vcombine_s32( S_s32x2x2.val[ 0 ], S_s32x2x2.val[ 1 ] ); + + for( ; k < len - 1; k += 2 ) { + int32x4_t in_s32x4[ 2 ], t_s32x4; + int32x2_t out32_Q14_s32x2[ 2 ]; + + /* S[ 2 * i + 0 ], S[ 2 * i + 1 ], S[ 2 * i + 2 ], S[ 2 * i + 3 ]: Q12 */ + in_s16x4 = vld1_s16( &in[ 2 * k ] ); /* in{0,1,2,3} = in[ 2 * k + {0,1,2,3} ]; */ + in_s32x4[ 0 ] = vshll_n_s16( in_s16x4, 15 ); /* in{0,1,2,3} << 15 */ + t_s32x4 = vqdmulhq_lane_s32( in_s32x4[ 0 ], B_Q28_s32x2, 0 ); /* silk_SMULWB( B_Q28[ 0 ], in{0,1,2,3} ) */ + in_s32x4[ 1 ] = vcombine_s32( vget_high_s32( in_s32x4[ 0 ] ), vget_high_s32( in_s32x4[ 0 ] ) ); /* in{2,3,2,3} << 15 */ + in_s32x4[ 0 ] = vcombine_s32( vget_low_s32 ( in_s32x4[ 0 ] ), vget_low_s32 ( in_s32x4[ 0 ] ) ); /* in{0,1,0,1} << 15 */ + silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, vget_low_s32 ( t_s32x4 ), in_s32x4[ 0 ], &S_s32x4, &out32_Q14_s32x2[ 0 ] ); + silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, vget_high_s32( t_s32x4 ), in_s32x4[ 1 ], &S_s32x4, &out32_Q14_s32x2[ 1 ] ); + + /* Scale back to Q0 and saturate */ + out32_Q14_s32x4 = vcombine_s32( out32_Q14_s32x2[ 0 ], out32_Q14_s32x2[ 1 ] ); /* out32_Q14_{0,1,2,3} */ + out32_Q14_s32x4 = vaddq_s32( out32_Q14_s32x4, offset_s32x4 ); /* out32_Q14_{0,1,2,3} + (1<<14) - 1 */ + out_s16x4 = vqshrn_n_s32( out32_Q14_s32x4, 14 ); /* (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,2,3} + (1<<14) - 1, 14 ) ) */ + vst1_s16( &out[ 2 * k ], out_s16x4 ); /* out[ 2 * k + {0,1,2,3} ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,2,3} + (1<<14) - 1, 14 ) ); */ + } + + /* Process leftover. */ + if( k < len ) { + int32x4_t in_s32x4; + int32x2_t out32_Q14_s32x2; + + /* S[ 2 * i + 0 ], S[ 2 * i + 1 ]: Q12 */ + in_s16x4 = vld1_lane_s16( &in[ 2 * k + 0 ], in_s16x4, 0 ); /* in{0,1} = in[ 2 * k + {0,1} ]; */ + in_s16x4 = vld1_lane_s16( &in[ 2 * k + 1 ], in_s16x4, 1 ); /* in{0,1} = in[ 2 * k + {0,1} ]; */ + in_s32x4 = vshll_n_s16( in_s16x4, 15 ); /* in{0,1} << 15 */ + t_s32x2 = vqdmulh_lane_s32( vget_low_s32( in_s32x4 ), B_Q28_s32x2, 0 ); /* silk_SMULWB( B_Q28[ 0 ], in{0,1} ) */ + in_s32x4 = vcombine_s32( vget_low_s32( in_s32x4 ), vget_low_s32( in_s32x4 ) ); /* in{0,1,0,1} << 15 */ + silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, t_s32x2, in_s32x4, &S_s32x4, &out32_Q14_s32x2 ); + + /* Scale back to Q0 and saturate */ + out32_Q14_s32x2 = vadd_s32( out32_Q14_s32x2, offset_s32x2 ); /* out32_Q14_{0,1} + (1<<14) - 1 */ + out32_Q14_s32x4 = vcombine_s32( out32_Q14_s32x2, out32_Q14_s32x2 ); /* out32_Q14_{0,1,0,1} + (1<<14) - 1 */ + out_s16x4 = vqshrn_n_s32( out32_Q14_s32x4, 14 ); /* (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,0,1} + (1<<14) - 1, 14 ) ) */ + vst1_lane_s16( &out[ 2 * k + 0 ], out_s16x4, 0 ); /* out[ 2 * k + 0 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_0 + (1<<14) - 1, 14 ) ); */ + vst1_lane_s16( &out[ 2 * k + 1 ], out_s16x4, 1 ); /* out[ 2 * k + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_1 + (1<<14) - 1, 14 ) ); */ + } + + vst1q_lane_s32( &S[ 0 ], S_s32x4, 0 ); /* S[ 0 ] = S0; */ + vst1q_lane_s32( &S[ 1 ], S_s32x4, 2 ); /* S[ 1 ] = S2; */ + vst1q_lane_s32( &S[ 2 ], S_s32x4, 1 ); /* S[ 2 ] = S1; */ + vst1q_lane_s32( &S[ 3 ], S_s32x4, 3 ); /* S[ 3 ] = S3; */ + +#ifdef OPUS_CHECK_ASM + silk_assert( !memcmp( S_c, S, sizeof( S_c ) ) ); + silk_assert( !memcmp( out_c, out, 2 * len * sizeof( opus_int16 ) ) ); + RESTORE_STACK; +#endif +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_arm64.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_arm64.h new file mode 100644 index 000000000..ed030413c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_arm64.h @@ -0,0 +1,39 @@ +/*********************************************************************** +Copyright (C) 2015 Vidyo +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MACROS_ARM64_H +#define SILK_MACROS_ARM64_H + +#include + +#undef silk_ADD_SAT32 +#define silk_ADD_SAT32(a, b) (vqadds_s32((a), (b))) + +#undef silk_SUB_SAT32 +#define silk_SUB_SAT32(a, b) (vqsubs_s32((a), (b))) + +#endif /* SILK_MACROS_ARM64_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv4.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv4.h new file mode 100644 index 000000000..877eb18dd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv4.h @@ -0,0 +1,110 @@ +/*********************************************************************** +Copyright (C) 2013 Xiph.Org Foundation and contributors. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MACROS_ARMv4_H +#define SILK_MACROS_ARMv4_H + +/* This macro only avoids the undefined behaviour from a left shift of + a negative value. It should only be used in macros that can't include + SigProc_FIX.h. In other cases, use silk_LSHIFT32(). */ +#define SAFE_SHL(a,b) ((opus_int32)((opus_uint32)(a) << (b))) + +/* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ +#undef silk_SMULWB +static OPUS_INLINE opus_int32 silk_SMULWB_armv4(opus_int32 a, opus_int16 b) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#silk_SMULWB\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(a), "r"(SAFE_SHL(b,16)) + ); + return rd_hi; +} +#define silk_SMULWB(a, b) (silk_SMULWB_armv4(a, b)) + +/* a32 + (b32 * (opus_int32)((opus_int16)(c32))) >> 16 output have to be 32bit int */ +#undef silk_SMLAWB +#define silk_SMLAWB(a, b, c) ((a) + silk_SMULWB(b, c)) + +/* (a32 * (b32 >> 16)) >> 16 */ +#undef silk_SMULWT +static OPUS_INLINE opus_int32 silk_SMULWT_armv4(opus_int32 a, opus_int32 b) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#silk_SMULWT\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(a), "r"(b&~0xFFFF) + ); + return rd_hi; +} +#define silk_SMULWT(a, b) (silk_SMULWT_armv4(a, b)) + +/* a32 + (b32 * (c32 >> 16)) >> 16 */ +#undef silk_SMLAWT +#define silk_SMLAWT(a, b, c) ((a) + silk_SMULWT(b, c)) + +/* (a32 * b32) >> 16 */ +#undef silk_SMULWW +static OPUS_INLINE opus_int32 silk_SMULWW_armv4(opus_int32 a, opus_int32 b) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#silk_SMULWW\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(a), "r"(b) + ); + return SAFE_SHL(rd_hi,16)+(rd_lo>>16); +} +#define silk_SMULWW(a, b) (silk_SMULWW_armv4(a, b)) + +#undef silk_SMLAWW +static OPUS_INLINE opus_int32 silk_SMLAWW_armv4(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + unsigned rd_lo; + int rd_hi; + __asm__( + "#silk_SMLAWW\n\t" + "smull %0, %1, %2, %3\n\t" + : "=&r"(rd_lo), "=&r"(rd_hi) + : "%r"(b), "r"(c) + ); + return a+SAFE_SHL(rd_hi,16)+(rd_lo>>16); +} +#define silk_SMLAWW(a, b, c) (silk_SMLAWW_armv4(a, b, c)) + +#undef SAFE_SHL + +#endif /* SILK_MACROS_ARMv4_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv5e.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv5e.h new file mode 100644 index 000000000..b14ec65dd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/arm/macros_armv5e.h @@ -0,0 +1,220 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Copyright (c) 2013 Parrot +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MACROS_ARMv5E_H +#define SILK_MACROS_ARMv5E_H + +/* This macro only avoids the undefined behaviour from a left shift of + a negative value. It should only be used in macros that can't include + SigProc_FIX.h. In other cases, use silk_LSHIFT32(). */ +#define SAFE_SHL(a,b) ((opus_int32)((opus_uint32)(a) << (b))) + +/* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ +#undef silk_SMULWB +static OPUS_INLINE opus_int32 silk_SMULWB_armv5e(opus_int32 a, opus_int16 b) +{ + int res; + __asm__( + "#silk_SMULWB\n\t" + "smulwb %0, %1, %2\n\t" + : "=r"(res) + : "r"(a), "r"(b) + ); + return res; +} +#define silk_SMULWB(a, b) (silk_SMULWB_armv5e(a, b)) + +/* a32 + (b32 * (opus_int32)((opus_int16)(c32))) >> 16 output have to be 32bit int */ +#undef silk_SMLAWB +static OPUS_INLINE opus_int32 silk_SMLAWB_armv5e(opus_int32 a, opus_int32 b, + opus_int16 c) +{ + int res; + __asm__( + "#silk_SMLAWB\n\t" + "smlawb %0, %1, %2, %3\n\t" + : "=r"(res) + : "r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_SMLAWB(a, b, c) (silk_SMLAWB_armv5e(a, b, c)) + +/* (a32 * (b32 >> 16)) >> 16 */ +#undef silk_SMULWT +static OPUS_INLINE opus_int32 silk_SMULWT_armv5e(opus_int32 a, opus_int32 b) +{ + int res; + __asm__( + "#silk_SMULWT\n\t" + "smulwt %0, %1, %2\n\t" + : "=r"(res) + : "r"(a), "r"(b) + ); + return res; +} +#define silk_SMULWT(a, b) (silk_SMULWT_armv5e(a, b)) + +/* a32 + (b32 * (c32 >> 16)) >> 16 */ +#undef silk_SMLAWT +static OPUS_INLINE opus_int32 silk_SMLAWT_armv5e(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + int res; + __asm__( + "#silk_SMLAWT\n\t" + "smlawt %0, %1, %2, %3\n\t" + : "=r"(res) + : "r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_SMLAWT(a, b, c) (silk_SMLAWT_armv5e(a, b, c)) + +/* (opus_int32)((opus_int16)(a3))) * (opus_int32)((opus_int16)(b32)) output have to be 32bit int */ +#undef silk_SMULBB +static OPUS_INLINE opus_int32 silk_SMULBB_armv5e(opus_int32 a, opus_int32 b) +{ + int res; + __asm__( + "#silk_SMULBB\n\t" + "smulbb %0, %1, %2\n\t" + : "=r"(res) + : "%r"(a), "r"(b) + ); + return res; +} +#define silk_SMULBB(a, b) (silk_SMULBB_armv5e(a, b)) + +/* a32 + (opus_int32)((opus_int16)(b32)) * (opus_int32)((opus_int16)(c32)) output have to be 32bit int */ +#undef silk_SMLABB +static OPUS_INLINE opus_int32 silk_SMLABB_armv5e(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + int res; + __asm__( + "#silk_SMLABB\n\t" + "smlabb %0, %1, %2, %3\n\t" + : "=r"(res) + : "%r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_SMLABB(a, b, c) (silk_SMLABB_armv5e(a, b, c)) + +/* (opus_int32)((opus_int16)(a32)) * (b32 >> 16) */ +#undef silk_SMULBT +static OPUS_INLINE opus_int32 silk_SMULBT_armv5e(opus_int32 a, opus_int32 b) +{ + int res; + __asm__( + "#silk_SMULBT\n\t" + "smulbt %0, %1, %2\n\t" + : "=r"(res) + : "r"(a), "r"(b) + ); + return res; +} +#define silk_SMULBT(a, b) (silk_SMULBT_armv5e(a, b)) + +/* a32 + (opus_int32)((opus_int16)(b32)) * (c32 >> 16) */ +#undef silk_SMLABT +static OPUS_INLINE opus_int32 silk_SMLABT_armv5e(opus_int32 a, opus_int32 b, + opus_int32 c) +{ + int res; + __asm__( + "#silk_SMLABT\n\t" + "smlabt %0, %1, %2, %3\n\t" + : "=r"(res) + : "r"(b), "r"(c), "r"(a) + ); + return res; +} +#define silk_SMLABT(a, b, c) (silk_SMLABT_armv5e(a, b, c)) + +/* add/subtract with output saturated */ +#undef silk_ADD_SAT32 +static OPUS_INLINE opus_int32 silk_ADD_SAT32_armv5e(opus_int32 a, opus_int32 b) +{ + int res; + __asm__( + "#silk_ADD_SAT32\n\t" + "qadd %0, %1, %2\n\t" + : "=r"(res) + : "%r"(a), "r"(b) + ); + return res; +} +#define silk_ADD_SAT32(a, b) (silk_ADD_SAT32_armv5e(a, b)) + +#undef silk_SUB_SAT32 +static OPUS_INLINE opus_int32 silk_SUB_SAT32_armv5e(opus_int32 a, opus_int32 b) +{ + int res; + __asm__( + "#silk_SUB_SAT32\n\t" + "qsub %0, %1, %2\n\t" + : "=r"(res) + : "r"(a), "r"(b) + ); + return res; +} +#define silk_SUB_SAT32(a, b) (silk_SUB_SAT32_armv5e(a, b)) + +#undef silk_CLZ16 +static OPUS_INLINE opus_int32 silk_CLZ16_armv5(opus_int16 in16) +{ + int res; + __asm__( + "#silk_CLZ16\n\t" + "clz %0, %1;\n" + : "=r"(res) + : "r"(SAFE_SHL(in16,16)|0x8000) + ); + return res; +} +#define silk_CLZ16(in16) (silk_CLZ16_armv5(in16)) + +#undef silk_CLZ32 +static OPUS_INLINE opus_int32 silk_CLZ32_armv5(opus_int32 in32) +{ + int res; + __asm__( + "#silk_CLZ32\n\t" + "clz %0, %1\n\t" + : "=r"(res) + : "r"(in32) + ); + return res; +} +#define silk_CLZ32(in32) (silk_CLZ32_armv5(in32)) + +#undef SAFE_SHL + +#endif /* SILK_MACROS_ARMv5E_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/biquad_alt.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/biquad_alt.c new file mode 100644 index 000000000..54566a43c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/biquad_alt.c @@ -0,0 +1,121 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +/* * + * silk_biquad_alt.c * + * * + * Second order ARMA filter * + * Can handle slowly varying filter coefficients * + * */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Second order ARMA filter, alternative implementation */ +void silk_biquad_alt_stride1( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [2] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +) +{ + /* DIRECT FORM II TRANSPOSED (uses 2 element state vector) */ + opus_int k; + opus_int32 inval, A0_U_Q28, A0_L_Q28, A1_U_Q28, A1_L_Q28, out32_Q14; + + /* Negate A_Q28 values and split in two parts */ + A0_L_Q28 = ( -A_Q28[ 0 ] ) & 0x00003FFF; /* lower part */ + A0_U_Q28 = silk_RSHIFT( -A_Q28[ 0 ], 14 ); /* upper part */ + A1_L_Q28 = ( -A_Q28[ 1 ] ) & 0x00003FFF; /* lower part */ + A1_U_Q28 = silk_RSHIFT( -A_Q28[ 1 ], 14 ); /* upper part */ + + for( k = 0; k < len; k++ ) { + /* S[ 0 ], S[ 1 ]: Q12 */ + inval = in[ k ]; + out32_Q14 = silk_LSHIFT( silk_SMLAWB( S[ 0 ], B_Q28[ 0 ], inval ), 2 ); + + S[ 0 ] = S[1] + silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14, A0_L_Q28 ), 14 ); + S[ 0 ] = silk_SMLAWB( S[ 0 ], out32_Q14, A0_U_Q28 ); + S[ 0 ] = silk_SMLAWB( S[ 0 ], B_Q28[ 1 ], inval); + + S[ 1 ] = silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14, A1_L_Q28 ), 14 ); + S[ 1 ] = silk_SMLAWB( S[ 1 ], out32_Q14, A1_U_Q28 ); + S[ 1 ] = silk_SMLAWB( S[ 1 ], B_Q28[ 2 ], inval ); + + /* Scale back to Q0 and saturate */ + out[ k ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14 + (1<<14) - 1, 14 ) ); + } +} + +void silk_biquad_alt_stride2_c( + const opus_int16 *in, /* I input signal */ + const opus_int32 *B_Q28, /* I MA coefficients [3] */ + const opus_int32 *A_Q28, /* I AR coefficients [2] */ + opus_int32 *S, /* I/O State vector [4] */ + opus_int16 *out, /* O output signal */ + const opus_int32 len /* I signal length (must be even) */ +) +{ + /* DIRECT FORM II TRANSPOSED (uses 2 element state vector) */ + opus_int k; + opus_int32 A0_U_Q28, A0_L_Q28, A1_U_Q28, A1_L_Q28, out32_Q14[ 2 ]; + + /* Negate A_Q28 values and split in two parts */ + A0_L_Q28 = ( -A_Q28[ 0 ] ) & 0x00003FFF; /* lower part */ + A0_U_Q28 = silk_RSHIFT( -A_Q28[ 0 ], 14 ); /* upper part */ + A1_L_Q28 = ( -A_Q28[ 1 ] ) & 0x00003FFF; /* lower part */ + A1_U_Q28 = silk_RSHIFT( -A_Q28[ 1 ], 14 ); /* upper part */ + + for( k = 0; k < len; k++ ) { + /* S[ 0 ], S[ 1 ], S[ 2 ], S[ 3 ]: Q12 */ + out32_Q14[ 0 ] = silk_LSHIFT( silk_SMLAWB( S[ 0 ], B_Q28[ 0 ], in[ 2 * k + 0 ] ), 2 ); + out32_Q14[ 1 ] = silk_LSHIFT( silk_SMLAWB( S[ 2 ], B_Q28[ 0 ], in[ 2 * k + 1 ] ), 2 ); + + S[ 0 ] = S[ 1 ] + silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14[ 0 ], A0_L_Q28 ), 14 ); + S[ 2 ] = S[ 3 ] + silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14[ 1 ], A0_L_Q28 ), 14 ); + S[ 0 ] = silk_SMLAWB( S[ 0 ], out32_Q14[ 0 ], A0_U_Q28 ); + S[ 2 ] = silk_SMLAWB( S[ 2 ], out32_Q14[ 1 ], A0_U_Q28 ); + S[ 0 ] = silk_SMLAWB( S[ 0 ], B_Q28[ 1 ], in[ 2 * k + 0 ] ); + S[ 2 ] = silk_SMLAWB( S[ 2 ], B_Q28[ 1 ], in[ 2 * k + 1 ] ); + + S[ 1 ] = silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14[ 0 ], A1_L_Q28 ), 14 ); + S[ 3 ] = silk_RSHIFT_ROUND( silk_SMULWB( out32_Q14[ 1 ], A1_L_Q28 ), 14 ); + S[ 1 ] = silk_SMLAWB( S[ 1 ], out32_Q14[ 0 ], A1_U_Q28 ); + S[ 3 ] = silk_SMLAWB( S[ 3 ], out32_Q14[ 1 ], A1_U_Q28 ); + S[ 1 ] = silk_SMLAWB( S[ 1 ], B_Q28[ 2 ], in[ 2 * k + 0 ] ); + S[ 3 ] = silk_SMLAWB( S[ 3 ], B_Q28[ 2 ], in[ 2 * k + 1 ] ); + + /* Scale back to Q0 and saturate */ + out[ 2 * k + 0 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14[ 0 ] + (1<<14) - 1, 14 ) ); + out[ 2 * k + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14[ 1 ] + (1<<14) - 1, 14 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander.c new file mode 100644 index 000000000..afa97907e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander.c @@ -0,0 +1,51 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Chirp (bandwidth expand) LP AR filter */ +void silk_bwexpander( + opus_int16 *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I Length of ar */ + opus_int32 chirp_Q16 /* I Chirp factor (typically in the range 0 to 1) */ +) +{ + opus_int i; + opus_int32 chirp_minus_one_Q16 = chirp_Q16 - 65536; + + /* NB: Dont use silk_SMULWB, instead of silk_RSHIFT_ROUND( silk_MUL(), 16 ), below. */ + /* Bias in silk_SMULWB can lead to unstable filters */ + for( i = 0; i < d - 1; i++ ) { + ar[ i ] = (opus_int16)silk_RSHIFT_ROUND( silk_MUL( chirp_Q16, ar[ i ] ), 16 ); + chirp_Q16 += silk_RSHIFT_ROUND( silk_MUL( chirp_Q16, chirp_minus_one_Q16 ), 16 ); + } + ar[ d - 1 ] = (opus_int16)silk_RSHIFT_ROUND( silk_MUL( chirp_Q16, ar[ d - 1 ] ), 16 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander_32.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander_32.c new file mode 100644 index 000000000..0f32b9df1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/bwexpander_32.c @@ -0,0 +1,51 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Chirp (bandwidth expand) LP AR filter. + This logic is reused in _celt_lpc(). Any bug fixes should also be applied there. */ +void silk_bwexpander_32( + opus_int32 *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I Length of ar */ + opus_int32 chirp_Q16 /* I Chirp factor in Q16 */ +) +{ + opus_int i; + opus_int32 chirp_minus_one_Q16 = chirp_Q16 - 65536; + + for( i = 0; i < d - 1; i++ ) { + ar[ i ] = silk_SMULWW( chirp_Q16, ar[ i ] ); + chirp_Q16 += silk_RSHIFT_ROUND( silk_MUL( chirp_Q16, chirp_minus_one_Q16 ), 16 ); + } + ar[ d - 1 ] = silk_SMULWW( chirp_Q16, ar[ d - 1 ] ); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/check_control_input.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/check_control_input.c new file mode 100644 index 000000000..739fb01f1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/check_control_input.c @@ -0,0 +1,106 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "control.h" +#include "errors.h" + +/* Check encoder control struct */ +opus_int check_control_input( + silk_EncControlStruct *encControl /* I Control structure */ +) +{ + celt_assert( encControl != NULL ); + + if( ( ( encControl->API_sampleRate != 8000 ) && + ( encControl->API_sampleRate != 12000 ) && + ( encControl->API_sampleRate != 16000 ) && + ( encControl->API_sampleRate != 24000 ) && + ( encControl->API_sampleRate != 32000 ) && + ( encControl->API_sampleRate != 44100 ) && + ( encControl->API_sampleRate != 48000 ) ) || + ( ( encControl->desiredInternalSampleRate != 8000 ) && + ( encControl->desiredInternalSampleRate != 12000 ) && + ( encControl->desiredInternalSampleRate != 16000 ) ) || + ( ( encControl->maxInternalSampleRate != 8000 ) && + ( encControl->maxInternalSampleRate != 12000 ) && + ( encControl->maxInternalSampleRate != 16000 ) ) || + ( ( encControl->minInternalSampleRate != 8000 ) && + ( encControl->minInternalSampleRate != 12000 ) && + ( encControl->minInternalSampleRate != 16000 ) ) || + ( encControl->minInternalSampleRate > encControl->desiredInternalSampleRate ) || + ( encControl->maxInternalSampleRate < encControl->desiredInternalSampleRate ) || + ( encControl->minInternalSampleRate > encControl->maxInternalSampleRate ) ) { + celt_assert( 0 ); + return SILK_ENC_FS_NOT_SUPPORTED; + } + if( encControl->payloadSize_ms != 10 && + encControl->payloadSize_ms != 20 && + encControl->payloadSize_ms != 40 && + encControl->payloadSize_ms != 60 ) { + celt_assert( 0 ); + return SILK_ENC_PACKET_SIZE_NOT_SUPPORTED; + } + if( encControl->packetLossPercentage < 0 || encControl->packetLossPercentage > 100 ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_LOSS_RATE; + } + if( encControl->useDTX < 0 || encControl->useDTX > 1 ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_DTX_SETTING; + } + if( encControl->useCBR < 0 || encControl->useCBR > 1 ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_CBR_SETTING; + } + if( encControl->useInBandFEC < 0 || encControl->useInBandFEC > 1 ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_INBAND_FEC_SETTING; + } + if( encControl->nChannelsAPI < 1 || encControl->nChannelsAPI > ENCODER_NUM_CHANNELS ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_NUMBER_OF_CHANNELS_ERROR; + } + if( encControl->nChannelsInternal < 1 || encControl->nChannelsInternal > ENCODER_NUM_CHANNELS ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_NUMBER_OF_CHANNELS_ERROR; + } + if( encControl->nChannelsInternal > encControl->nChannelsAPI ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_NUMBER_OF_CHANNELS_ERROR; + } + if( encControl->complexity < 0 || encControl->complexity > 10 ) { + celt_assert( 0 ); + return SILK_ENC_INVALID_COMPLEXITY_SETTING; + } + + return SILK_NO_ERROR; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/code_signs.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/code_signs.c new file mode 100644 index 000000000..dfd1dca9a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/code_signs.c @@ -0,0 +1,115 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/*#define silk_enc_map(a) ((a) > 0 ? 1 : 0)*/ +/*#define silk_dec_map(a) ((a) > 0 ? 1 : -1)*/ +/* shifting avoids if-statement */ +#define silk_enc_map(a) ( silk_RSHIFT( (a), 15 ) + 1 ) +#define silk_dec_map(a) ( silk_LSHIFT( (a), 1 ) - 1 ) + +/* Encodes signs of excitation */ +void silk_encode_signs( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + const opus_int8 pulses[], /* I pulse signal */ + opus_int length, /* I length of input */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I Quantization offset type */ + const opus_int sum_pulses[ MAX_NB_SHELL_BLOCKS ] /* I Sum of absolute pulses per block */ +) +{ + opus_int i, j, p; + opus_uint8 icdf[ 2 ]; + const opus_int8 *q_ptr; + const opus_uint8 *icdf_ptr; + + icdf[ 1 ] = 0; + q_ptr = pulses; + i = silk_SMULBB( 7, silk_ADD_LSHIFT( quantOffsetType, signalType, 1 ) ); + icdf_ptr = &silk_sign_iCDF[ i ]; + length = silk_RSHIFT( length + SHELL_CODEC_FRAME_LENGTH/2, LOG2_SHELL_CODEC_FRAME_LENGTH ); + for( i = 0; i < length; i++ ) { + p = sum_pulses[ i ]; + if( p > 0 ) { + icdf[ 0 ] = icdf_ptr[ silk_min( p & 0x1F, 6 ) ]; + for( j = 0; j < SHELL_CODEC_FRAME_LENGTH; j++ ) { + if( q_ptr[ j ] != 0 ) { + ec_enc_icdf( psRangeEnc, silk_enc_map( q_ptr[ j ]), icdf, 8 ); + } + } + } + q_ptr += SHELL_CODEC_FRAME_LENGTH; + } +} + +/* Decodes signs of excitation */ +void silk_decode_signs( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pulses[], /* I/O pulse signal */ + opus_int length, /* I length of input */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I Quantization offset type */ + const opus_int sum_pulses[ MAX_NB_SHELL_BLOCKS ] /* I Sum of absolute pulses per block */ +) +{ + opus_int i, j, p; + opus_uint8 icdf[ 2 ]; + opus_int16 *q_ptr; + const opus_uint8 *icdf_ptr; + + icdf[ 1 ] = 0; + q_ptr = pulses; + i = silk_SMULBB( 7, silk_ADD_LSHIFT( quantOffsetType, signalType, 1 ) ); + icdf_ptr = &silk_sign_iCDF[ i ]; + length = silk_RSHIFT( length + SHELL_CODEC_FRAME_LENGTH/2, LOG2_SHELL_CODEC_FRAME_LENGTH ); + for( i = 0; i < length; i++ ) { + p = sum_pulses[ i ]; + if( p > 0 ) { + icdf[ 0 ] = icdf_ptr[ silk_min( p & 0x1F, 6 ) ]; + for( j = 0; j < SHELL_CODEC_FRAME_LENGTH; j++ ) { + if( q_ptr[ j ] > 0 ) { + /* attach sign */ +#if 0 + /* conditional implementation */ + if( ec_dec_icdf( psRangeDec, icdf, 8 ) == 0 ) { + q_ptr[ j ] = -q_ptr[ j ]; + } +#else + /* implementation with shift, subtraction, multiplication */ + q_ptr[ j ] *= silk_dec_map( ec_dec_icdf( psRangeDec, icdf, 8 ) ); +#endif + } + } + } + q_ptr += SHELL_CODEC_FRAME_LENGTH; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control.h new file mode 100644 index 000000000..b76ec33cd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control.h @@ -0,0 +1,150 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_CONTROL_H +#define SILK_CONTROL_H + +#include "typedef.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Decoder API flags */ +#define FLAG_DECODE_NORMAL 0 +#define FLAG_PACKET_LOST 1 +#define FLAG_DECODE_LBRR 2 + +/***********************************************/ +/* Structure for controlling encoder operation */ +/***********************************************/ +typedef struct { + /* I: Number of channels; 1/2 */ + opus_int32 nChannelsAPI; + + /* I: Number of channels; 1/2 */ + opus_int32 nChannelsInternal; + + /* I: Input signal sampling rate in Hertz; 8000/12000/16000/24000/32000/44100/48000 */ + opus_int32 API_sampleRate; + + /* I: Maximum internal sampling rate in Hertz; 8000/12000/16000 */ + opus_int32 maxInternalSampleRate; + + /* I: Minimum internal sampling rate in Hertz; 8000/12000/16000 */ + opus_int32 minInternalSampleRate; + + /* I: Soft request for internal sampling rate in Hertz; 8000/12000/16000 */ + opus_int32 desiredInternalSampleRate; + + /* I: Number of samples per packet in milliseconds; 10/20/40/60 */ + opus_int payloadSize_ms; + + /* I: Bitrate during active speech in bits/second; internally limited */ + opus_int32 bitRate; + + /* I: Uplink packet loss in percent (0-100) */ + opus_int packetLossPercentage; + + /* I: Complexity mode; 0 is lowest, 10 is highest complexity */ + opus_int complexity; + + /* I: Flag to enable in-band Forward Error Correction (FEC); 0/1 */ + opus_int useInBandFEC; + + /* I: Flag to actually code in-band Forward Error Correction (FEC) in the current packet; 0/1 */ + opus_int LBRR_coded; + + /* I: Flag to enable discontinuous transmission (DTX); 0/1 */ + opus_int useDTX; + + /* I: Flag to use constant bitrate */ + opus_int useCBR; + + /* I: Maximum number of bits allowed for the frame */ + opus_int maxBits; + + /* I: Causes a smooth downmix to mono */ + opus_int toMono; + + /* I: Opus encoder is allowing us to switch bandwidth */ + opus_int opusCanSwitch; + + /* I: Make frames as independent as possible (but still use LPC) */ + opus_int reducedDependency; + + /* O: Internal sampling rate used, in Hertz; 8000/12000/16000 */ + opus_int32 internalSampleRate; + + /* O: Flag that bandwidth switching is allowed (because low voice activity) */ + opus_int allowBandwidthSwitch; + + /* O: Flag that SILK runs in WB mode without variable LP filter (use for switching between WB/SWB/FB) */ + opus_int inWBmodeWithoutVariableLP; + + /* O: Stereo width */ + opus_int stereoWidth_Q14; + + /* O: Tells the Opus encoder we're ready to switch */ + opus_int switchReady; + + /* O: SILK Signal type */ + opus_int signalType; + + /* O: SILK offset (dithering) */ + opus_int offset; +} silk_EncControlStruct; + +/**************************************************************************/ +/* Structure for controlling decoder operation and reading decoder status */ +/**************************************************************************/ +typedef struct { + /* I: Number of channels; 1/2 */ + opus_int32 nChannelsAPI; + + /* I: Number of channels; 1/2 */ + opus_int32 nChannelsInternal; + + /* I: Output signal sampling rate in Hertz; 8000/12000/16000/24000/32000/44100/48000 */ + opus_int32 API_sampleRate; + + /* I: Internal sampling rate used, in Hertz; 8000/12000/16000 */ + opus_int32 internalSampleRate; + + /* I: Number of samples per packet in milliseconds; 10/20/40/60 */ + opus_int payloadSize_ms; + + /* O: Pitch lag of previous frame (0 if unvoiced), measured in samples at 48 kHz */ + opus_int prevPitchLag; +} silk_DecControlStruct; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_SNR.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_SNR.c new file mode 100644 index 000000000..9a6db2754 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_SNR.c @@ -0,0 +1,113 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "tuning_parameters.h" + +/* These tables hold SNR values divided by 21 (so they fit in 8 bits) + for different target bitrates spaced at 400 bps interval. The first + 10 values are omitted (0-4 kb/s) because they're all zeros. + These tables were obtained by running different SNRs through the + encoder and measuring the active bitrate. */ +static const unsigned char silk_TargetRate_NB_21[117 - 10] = { + 0, 15, 39, 52, 61, 68, + 74, 79, 84, 88, 92, 95, 99,102,105,108,111,114,117,119,122,124, + 126,129,131,133,135,137,139,142,143,145,147,149,151,153,155,157, + 158,160,162,163,165,167,168,170,171,173,174,176,177,179,180,182, + 183,185,186,187,189,190,192,193,194,196,197,199,200,201,203,204, + 205,207,208,209,211,212,213,215,216,217,219,220,221,223,224,225, + 227,228,230,231,232,234,235,236,238,239,241,242,243,245,246,248, + 249,250,252,253,255 +}; + +static const unsigned char silk_TargetRate_MB_21[165 - 10] = { + 0, 0, 28, 43, 52, 59, + 65, 70, 74, 78, 81, 85, 87, 90, 93, 95, 98,100,102,105,107,109, + 111,113,115,116,118,120,122,123,125,127,128,130,131,133,134,136, + 137,138,140,141,143,144,145,147,148,149,151,152,153,154,156,157, + 158,159,160,162,163,164,165,166,167,168,169,171,172,173,174,175, + 176,177,178,179,180,181,182,183,184,185,186,187,188,188,189,190, + 191,192,193,194,195,196,197,198,199,200,201,202,203,203,204,205, + 206,207,208,209,210,211,212,213,214,214,215,216,217,218,219,220, + 221,222,223,224,224,225,226,227,228,229,230,231,232,233,234,235, + 236,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250, + 251,252,253,254,255 +}; + +static const unsigned char silk_TargetRate_WB_21[201 - 10] = { + 0, 0, 0, 8, 29, 41, + 49, 56, 62, 66, 70, 74, 77, 80, 83, 86, 88, 91, 93, 95, 97, 99, + 101,103,105,107,108,110,112,113,115,116,118,119,121,122,123,125, + 126,127,129,130,131,132,134,135,136,137,138,140,141,142,143,144, + 145,146,147,148,149,150,151,152,153,154,156,157,158,159,159,160, + 161,162,163,164,165,166,167,168,169,170,171,171,172,173,174,175, + 176,177,177,178,179,180,181,181,182,183,184,185,185,186,187,188, + 189,189,190,191,192,192,193,194,195,195,196,197,198,198,199,200, + 200,201,202,203,203,204,205,206,206,207,208,209,209,210,211,211, + 212,213,214,214,215,216,216,217,218,219,219,220,221,221,222,223, + 224,224,225,226,226,227,228,229,229,230,231,232,232,233,234,234, + 235,236,237,237,238,239,240,240,241,242,243,243,244,245,246,246, + 247,248,249,249,250,251,252,253,255 +}; + +/* Control SNR of redidual quantizer */ +opus_int silk_control_SNR( + silk_encoder_state *psEncC, /* I/O Pointer to Silk encoder state */ + opus_int32 TargetRate_bps /* I Target max bitrate (bps) */ +) +{ + int id; + int bound; + const unsigned char *snr_table; + + psEncC->TargetRate_bps = TargetRate_bps; + if( psEncC->nb_subfr == 2 ) { + TargetRate_bps -= 2000 + psEncC->fs_kHz/16; + } + if( psEncC->fs_kHz == 8 ) { + bound = sizeof(silk_TargetRate_NB_21); + snr_table = silk_TargetRate_NB_21; + } else if( psEncC->fs_kHz == 12 ) { + bound = sizeof(silk_TargetRate_MB_21); + snr_table = silk_TargetRate_MB_21; + } else { + bound = sizeof(silk_TargetRate_WB_21); + snr_table = silk_TargetRate_WB_21; + } + id = (TargetRate_bps+200)/400; + id = silk_min(id - 10, bound-1); + if( id <= 0 ) { + psEncC->SNR_dB_Q7 = 0; + } else { + psEncC->SNR_dB_Q7 = snr_table[id]*21; + } + return SILK_NO_ERROR; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_audio_bandwidth.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_audio_bandwidth.c new file mode 100644 index 000000000..f6d22d839 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_audio_bandwidth.c @@ -0,0 +1,132 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "tuning_parameters.h" + +/* Control internal sampling rate */ +opus_int silk_control_audio_bandwidth( + silk_encoder_state *psEncC, /* I/O Pointer to Silk encoder state */ + silk_EncControlStruct *encControl /* I Control structure */ +) +{ + opus_int fs_kHz; + opus_int orig_kHz; + opus_int32 fs_Hz; + + orig_kHz = psEncC->fs_kHz; + /* Handle a bandwidth-switching reset where we need to be aware what the last sampling rate was. */ + if( orig_kHz == 0 ) { + orig_kHz = psEncC->sLP.saved_fs_kHz; + } + fs_kHz = orig_kHz; + fs_Hz = silk_SMULBB( fs_kHz, 1000 ); + if( fs_Hz == 0 ) { + /* Encoder has just been initialized */ + fs_Hz = silk_min( psEncC->desiredInternal_fs_Hz, psEncC->API_fs_Hz ); + fs_kHz = silk_DIV32_16( fs_Hz, 1000 ); + } else if( fs_Hz > psEncC->API_fs_Hz || fs_Hz > psEncC->maxInternal_fs_Hz || fs_Hz < psEncC->minInternal_fs_Hz ) { + /* Make sure internal rate is not higher than external rate or maximum allowed, or lower than minimum allowed */ + fs_Hz = psEncC->API_fs_Hz; + fs_Hz = silk_min( fs_Hz, psEncC->maxInternal_fs_Hz ); + fs_Hz = silk_max( fs_Hz, psEncC->minInternal_fs_Hz ); + fs_kHz = silk_DIV32_16( fs_Hz, 1000 ); + } else { + /* State machine for the internal sampling rate switching */ + if( psEncC->sLP.transition_frame_no >= TRANSITION_FRAMES ) { + /* Stop transition phase */ + psEncC->sLP.mode = 0; + } + if( psEncC->allow_bandwidth_switch || encControl->opusCanSwitch ) { + /* Check if we should switch down */ + if( silk_SMULBB( orig_kHz, 1000 ) > psEncC->desiredInternal_fs_Hz ) + { + /* Switch down */ + if( psEncC->sLP.mode == 0 ) { + /* New transition */ + psEncC->sLP.transition_frame_no = TRANSITION_FRAMES; + + /* Reset transition filter state */ + silk_memset( psEncC->sLP.In_LP_State, 0, sizeof( psEncC->sLP.In_LP_State ) ); + } + if( encControl->opusCanSwitch ) { + /* Stop transition phase */ + psEncC->sLP.mode = 0; + + /* Switch to a lower sample frequency */ + fs_kHz = orig_kHz == 16 ? 12 : 8; + } else { + if( psEncC->sLP.transition_frame_no <= 0 ) { + encControl->switchReady = 1; + /* Make room for redundancy */ + encControl->maxBits -= encControl->maxBits * 5 / ( encControl->payloadSize_ms + 5 ); + } else { + /* Direction: down (at double speed) */ + psEncC->sLP.mode = -2; + } + } + } + else + /* Check if we should switch up */ + if( silk_SMULBB( orig_kHz, 1000 ) < psEncC->desiredInternal_fs_Hz ) + { + /* Switch up */ + if( encControl->opusCanSwitch ) { + /* Switch to a higher sample frequency */ + fs_kHz = orig_kHz == 8 ? 12 : 16; + + /* New transition */ + psEncC->sLP.transition_frame_no = 0; + + /* Reset transition filter state */ + silk_memset( psEncC->sLP.In_LP_State, 0, sizeof( psEncC->sLP.In_LP_State ) ); + + /* Direction: up */ + psEncC->sLP.mode = 1; + } else { + if( psEncC->sLP.mode == 0 ) { + encControl->switchReady = 1; + /* Make room for redundancy */ + encControl->maxBits -= encControl->maxBits * 5 / ( encControl->payloadSize_ms + 5 ); + } else { + /* Direction: up */ + psEncC->sLP.mode = 1; + } + } + } else { + if (psEncC->sLP.mode<0) + psEncC->sLP.mode = 1; + } + } + } + + return fs_kHz; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_codec.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_codec.c new file mode 100644 index 000000000..52aa8fded --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/control_codec.c @@ -0,0 +1,423 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef FIXED_POINT +#include "main_FIX.h" +#define silk_encoder_state_Fxx silk_encoder_state_FIX +#else +#include "main_FLP.h" +#define silk_encoder_state_Fxx silk_encoder_state_FLP +#endif +#include "stack_alloc.h" +#include "tuning_parameters.h" +#include "pitch_est_defines.h" + +static opus_int silk_setup_resamplers( + silk_encoder_state_Fxx *psEnc, /* I/O */ + opus_int fs_kHz /* I */ +); + +static opus_int silk_setup_fs( + silk_encoder_state_Fxx *psEnc, /* I/O */ + opus_int fs_kHz, /* I */ + opus_int PacketSize_ms /* I */ +); + +static opus_int silk_setup_complexity( + silk_encoder_state *psEncC, /* I/O */ + opus_int Complexity /* I */ +); + +static OPUS_INLINE opus_int silk_setup_LBRR( + silk_encoder_state *psEncC, /* I/O */ + const silk_EncControlStruct *encControl /* I */ +); + + +/* Control encoder */ +opus_int silk_control_encoder( + silk_encoder_state_Fxx *psEnc, /* I/O Pointer to Silk encoder state */ + silk_EncControlStruct *encControl, /* I Control structure */ + const opus_int allow_bw_switch, /* I Flag to allow switching audio bandwidth */ + const opus_int channelNb, /* I Channel number */ + const opus_int force_fs_kHz +) +{ + opus_int fs_kHz, ret = 0; + + psEnc->sCmn.useDTX = encControl->useDTX; + psEnc->sCmn.useCBR = encControl->useCBR; + psEnc->sCmn.API_fs_Hz = encControl->API_sampleRate; + psEnc->sCmn.maxInternal_fs_Hz = encControl->maxInternalSampleRate; + psEnc->sCmn.minInternal_fs_Hz = encControl->minInternalSampleRate; + psEnc->sCmn.desiredInternal_fs_Hz = encControl->desiredInternalSampleRate; + psEnc->sCmn.useInBandFEC = encControl->useInBandFEC; + psEnc->sCmn.nChannelsAPI = encControl->nChannelsAPI; + psEnc->sCmn.nChannelsInternal = encControl->nChannelsInternal; + psEnc->sCmn.allow_bandwidth_switch = allow_bw_switch; + psEnc->sCmn.channelNb = channelNb; + + if( psEnc->sCmn.controlled_since_last_payload != 0 && psEnc->sCmn.prefillFlag == 0 ) { + if( psEnc->sCmn.API_fs_Hz != psEnc->sCmn.prev_API_fs_Hz && psEnc->sCmn.fs_kHz > 0 ) { + /* Change in API sampling rate in the middle of encoding a packet */ + ret += silk_setup_resamplers( psEnc, psEnc->sCmn.fs_kHz ); + } + return ret; + } + + /* Beyond this point we know that there are no previously coded frames in the payload buffer */ + + /********************************************/ + /* Determine internal sampling rate */ + /********************************************/ + fs_kHz = silk_control_audio_bandwidth( &psEnc->sCmn, encControl ); + if( force_fs_kHz ) { + fs_kHz = force_fs_kHz; + } + /********************************************/ + /* Prepare resampler and buffered data */ + /********************************************/ + ret += silk_setup_resamplers( psEnc, fs_kHz ); + + /********************************************/ + /* Set internal sampling frequency */ + /********************************************/ + ret += silk_setup_fs( psEnc, fs_kHz, encControl->payloadSize_ms ); + + /********************************************/ + /* Set encoding complexity */ + /********************************************/ + ret += silk_setup_complexity( &psEnc->sCmn, encControl->complexity ); + + /********************************************/ + /* Set packet loss rate measured by farend */ + /********************************************/ + psEnc->sCmn.PacketLoss_perc = encControl->packetLossPercentage; + + /********************************************/ + /* Set LBRR usage */ + /********************************************/ + ret += silk_setup_LBRR( &psEnc->sCmn, encControl ); + + psEnc->sCmn.controlled_since_last_payload = 1; + + return ret; +} + +static opus_int silk_setup_resamplers( + silk_encoder_state_Fxx *psEnc, /* I/O */ + opus_int fs_kHz /* I */ +) +{ + opus_int ret = SILK_NO_ERROR; + SAVE_STACK; + + if( psEnc->sCmn.fs_kHz != fs_kHz || psEnc->sCmn.prev_API_fs_Hz != psEnc->sCmn.API_fs_Hz ) + { + if( psEnc->sCmn.fs_kHz == 0 ) { + /* Initialize the resampler for enc_API.c preparing resampling from API_fs_Hz to fs_kHz */ + ret += silk_resampler_init( &psEnc->sCmn.resampler_state, psEnc->sCmn.API_fs_Hz, fs_kHz * 1000, 1 ); + } else { + VARDECL( opus_int16, x_buf_API_fs_Hz ); + VARDECL( silk_resampler_state_struct, temp_resampler_state ); +#ifdef FIXED_POINT + opus_int16 *x_bufFIX = psEnc->x_buf; +#else + VARDECL( opus_int16, x_bufFIX ); + opus_int32 new_buf_samples; +#endif + opus_int32 api_buf_samples; + opus_int32 old_buf_samples; + opus_int32 buf_length_ms; + + buf_length_ms = silk_LSHIFT( psEnc->sCmn.nb_subfr * 5, 1 ) + LA_SHAPE_MS; + old_buf_samples = buf_length_ms * psEnc->sCmn.fs_kHz; + +#ifndef FIXED_POINT + new_buf_samples = buf_length_ms * fs_kHz; + ALLOC( x_bufFIX, silk_max( old_buf_samples, new_buf_samples ), + opus_int16 ); + silk_float2short_array( x_bufFIX, psEnc->x_buf, old_buf_samples ); +#endif + + /* Initialize resampler for temporary resampling of x_buf data to API_fs_Hz */ + ALLOC( temp_resampler_state, 1, silk_resampler_state_struct ); + ret += silk_resampler_init( temp_resampler_state, silk_SMULBB( psEnc->sCmn.fs_kHz, 1000 ), psEnc->sCmn.API_fs_Hz, 0 ); + + /* Calculate number of samples to temporarily upsample */ + api_buf_samples = buf_length_ms * silk_DIV32_16( psEnc->sCmn.API_fs_Hz, 1000 ); + + /* Temporary resampling of x_buf data to API_fs_Hz */ + ALLOC( x_buf_API_fs_Hz, api_buf_samples, opus_int16 ); + ret += silk_resampler( temp_resampler_state, x_buf_API_fs_Hz, x_bufFIX, old_buf_samples ); + + /* Initialize the resampler for enc_API.c preparing resampling from API_fs_Hz to fs_kHz */ + ret += silk_resampler_init( &psEnc->sCmn.resampler_state, psEnc->sCmn.API_fs_Hz, silk_SMULBB( fs_kHz, 1000 ), 1 ); + + /* Correct resampler state by resampling buffered data from API_fs_Hz to fs_kHz */ + ret += silk_resampler( &psEnc->sCmn.resampler_state, x_bufFIX, x_buf_API_fs_Hz, api_buf_samples ); + +#ifndef FIXED_POINT + silk_short2float_array( psEnc->x_buf, x_bufFIX, new_buf_samples); +#endif + } + } + + psEnc->sCmn.prev_API_fs_Hz = psEnc->sCmn.API_fs_Hz; + + RESTORE_STACK; + return ret; +} + +static opus_int silk_setup_fs( + silk_encoder_state_Fxx *psEnc, /* I/O */ + opus_int fs_kHz, /* I */ + opus_int PacketSize_ms /* I */ +) +{ + opus_int ret = SILK_NO_ERROR; + + /* Set packet size */ + if( PacketSize_ms != psEnc->sCmn.PacketSize_ms ) { + if( ( PacketSize_ms != 10 ) && + ( PacketSize_ms != 20 ) && + ( PacketSize_ms != 40 ) && + ( PacketSize_ms != 60 ) ) { + ret = SILK_ENC_PACKET_SIZE_NOT_SUPPORTED; + } + if( PacketSize_ms <= 10 ) { + psEnc->sCmn.nFramesPerPacket = 1; + psEnc->sCmn.nb_subfr = PacketSize_ms == 10 ? 2 : 1; + psEnc->sCmn.frame_length = silk_SMULBB( PacketSize_ms, fs_kHz ); + psEnc->sCmn.pitch_LPC_win_length = silk_SMULBB( FIND_PITCH_LPC_WIN_MS_2_SF, fs_kHz ); + if( psEnc->sCmn.fs_kHz == 8 ) { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_10_ms_NB_iCDF; + } else { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_10_ms_iCDF; + } + } else { + psEnc->sCmn.nFramesPerPacket = silk_DIV32_16( PacketSize_ms, MAX_FRAME_LENGTH_MS ); + psEnc->sCmn.nb_subfr = MAX_NB_SUBFR; + psEnc->sCmn.frame_length = silk_SMULBB( 20, fs_kHz ); + psEnc->sCmn.pitch_LPC_win_length = silk_SMULBB( FIND_PITCH_LPC_WIN_MS, fs_kHz ); + if( psEnc->sCmn.fs_kHz == 8 ) { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_NB_iCDF; + } else { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_iCDF; + } + } + psEnc->sCmn.PacketSize_ms = PacketSize_ms; + psEnc->sCmn.TargetRate_bps = 0; /* trigger new SNR computation */ + } + + /* Set internal sampling frequency */ + celt_assert( fs_kHz == 8 || fs_kHz == 12 || fs_kHz == 16 ); + celt_assert( psEnc->sCmn.nb_subfr == 2 || psEnc->sCmn.nb_subfr == 4 ); + if( psEnc->sCmn.fs_kHz != fs_kHz ) { + /* reset part of the state */ + silk_memset( &psEnc->sShape, 0, sizeof( psEnc->sShape ) ); + silk_memset( &psEnc->sCmn.sNSQ, 0, sizeof( psEnc->sCmn.sNSQ ) ); + silk_memset( psEnc->sCmn.prev_NLSFq_Q15, 0, sizeof( psEnc->sCmn.prev_NLSFq_Q15 ) ); + silk_memset( &psEnc->sCmn.sLP.In_LP_State, 0, sizeof( psEnc->sCmn.sLP.In_LP_State ) ); + psEnc->sCmn.inputBufIx = 0; + psEnc->sCmn.nFramesEncoded = 0; + psEnc->sCmn.TargetRate_bps = 0; /* trigger new SNR computation */ + + /* Initialize non-zero parameters */ + psEnc->sCmn.prevLag = 100; + psEnc->sCmn.first_frame_after_reset = 1; + psEnc->sShape.LastGainIndex = 10; + psEnc->sCmn.sNSQ.lagPrev = 100; + psEnc->sCmn.sNSQ.prev_gain_Q16 = 65536; + psEnc->sCmn.prevSignalType = TYPE_NO_VOICE_ACTIVITY; + + psEnc->sCmn.fs_kHz = fs_kHz; + if( psEnc->sCmn.fs_kHz == 8 ) { + if( psEnc->sCmn.nb_subfr == MAX_NB_SUBFR ) { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_NB_iCDF; + } else { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_10_ms_NB_iCDF; + } + } else { + if( psEnc->sCmn.nb_subfr == MAX_NB_SUBFR ) { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_iCDF; + } else { + psEnc->sCmn.pitch_contour_iCDF = silk_pitch_contour_10_ms_iCDF; + } + } + if( psEnc->sCmn.fs_kHz == 8 || psEnc->sCmn.fs_kHz == 12 ) { + psEnc->sCmn.predictLPCOrder = MIN_LPC_ORDER; + psEnc->sCmn.psNLSF_CB = &silk_NLSF_CB_NB_MB; + } else { + psEnc->sCmn.predictLPCOrder = MAX_LPC_ORDER; + psEnc->sCmn.psNLSF_CB = &silk_NLSF_CB_WB; + } + psEnc->sCmn.subfr_length = SUB_FRAME_LENGTH_MS * fs_kHz; + psEnc->sCmn.frame_length = silk_SMULBB( psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr ); + psEnc->sCmn.ltp_mem_length = silk_SMULBB( LTP_MEM_LENGTH_MS, fs_kHz ); + psEnc->sCmn.la_pitch = silk_SMULBB( LA_PITCH_MS, fs_kHz ); + psEnc->sCmn.max_pitch_lag = silk_SMULBB( 18, fs_kHz ); + if( psEnc->sCmn.nb_subfr == MAX_NB_SUBFR ) { + psEnc->sCmn.pitch_LPC_win_length = silk_SMULBB( FIND_PITCH_LPC_WIN_MS, fs_kHz ); + } else { + psEnc->sCmn.pitch_LPC_win_length = silk_SMULBB( FIND_PITCH_LPC_WIN_MS_2_SF, fs_kHz ); + } + if( psEnc->sCmn.fs_kHz == 16 ) { + psEnc->sCmn.pitch_lag_low_bits_iCDF = silk_uniform8_iCDF; + } else if( psEnc->sCmn.fs_kHz == 12 ) { + psEnc->sCmn.pitch_lag_low_bits_iCDF = silk_uniform6_iCDF; + } else { + psEnc->sCmn.pitch_lag_low_bits_iCDF = silk_uniform4_iCDF; + } + } + + /* Check that settings are valid */ + celt_assert( ( psEnc->sCmn.subfr_length * psEnc->sCmn.nb_subfr ) == psEnc->sCmn.frame_length ); + + return ret; +} + +static opus_int silk_setup_complexity( + silk_encoder_state *psEncC, /* I/O */ + opus_int Complexity /* I */ +) +{ + opus_int ret = 0; + + /* Set encoding complexity */ + celt_assert( Complexity >= 0 && Complexity <= 10 ); + if( Complexity < 1 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MIN_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.8, 16 ); + psEncC->pitchEstimationLPCOrder = 6; + psEncC->shapingLPCOrder = 12; + psEncC->la_shape = 3 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 1; + psEncC->useInterpolatedNLSFs = 0; + psEncC->NLSF_MSVQ_Survivors = 2; + psEncC->warping_Q16 = 0; + } else if( Complexity < 2 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MID_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.76, 16 ); + psEncC->pitchEstimationLPCOrder = 8; + psEncC->shapingLPCOrder = 14; + psEncC->la_shape = 5 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 1; + psEncC->useInterpolatedNLSFs = 0; + psEncC->NLSF_MSVQ_Survivors = 3; + psEncC->warping_Q16 = 0; + } else if( Complexity < 3 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MIN_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.8, 16 ); + psEncC->pitchEstimationLPCOrder = 6; + psEncC->shapingLPCOrder = 12; + psEncC->la_shape = 3 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 2; + psEncC->useInterpolatedNLSFs = 0; + psEncC->NLSF_MSVQ_Survivors = 2; + psEncC->warping_Q16 = 0; + } else if( Complexity < 4 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MID_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.76, 16 ); + psEncC->pitchEstimationLPCOrder = 8; + psEncC->shapingLPCOrder = 14; + psEncC->la_shape = 5 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 2; + psEncC->useInterpolatedNLSFs = 0; + psEncC->NLSF_MSVQ_Survivors = 4; + psEncC->warping_Q16 = 0; + } else if( Complexity < 6 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MID_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.74, 16 ); + psEncC->pitchEstimationLPCOrder = 10; + psEncC->shapingLPCOrder = 16; + psEncC->la_shape = 5 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 2; + psEncC->useInterpolatedNLSFs = 1; + psEncC->NLSF_MSVQ_Survivors = 6; + psEncC->warping_Q16 = psEncC->fs_kHz * SILK_FIX_CONST( WARPING_MULTIPLIER, 16 ); + } else if( Complexity < 8 ) { + psEncC->pitchEstimationComplexity = SILK_PE_MID_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.72, 16 ); + psEncC->pitchEstimationLPCOrder = 12; + psEncC->shapingLPCOrder = 20; + psEncC->la_shape = 5 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = 3; + psEncC->useInterpolatedNLSFs = 1; + psEncC->NLSF_MSVQ_Survivors = 8; + psEncC->warping_Q16 = psEncC->fs_kHz * SILK_FIX_CONST( WARPING_MULTIPLIER, 16 ); + } else { + psEncC->pitchEstimationComplexity = SILK_PE_MAX_COMPLEX; + psEncC->pitchEstimationThreshold_Q16 = SILK_FIX_CONST( 0.7, 16 ); + psEncC->pitchEstimationLPCOrder = 16; + psEncC->shapingLPCOrder = 24; + psEncC->la_shape = 5 * psEncC->fs_kHz; + psEncC->nStatesDelayedDecision = MAX_DEL_DEC_STATES; + psEncC->useInterpolatedNLSFs = 1; + psEncC->NLSF_MSVQ_Survivors = 16; + psEncC->warping_Q16 = psEncC->fs_kHz * SILK_FIX_CONST( WARPING_MULTIPLIER, 16 ); + } + + /* Do not allow higher pitch estimation LPC order than predict LPC order */ + psEncC->pitchEstimationLPCOrder = silk_min_int( psEncC->pitchEstimationLPCOrder, psEncC->predictLPCOrder ); + psEncC->shapeWinLength = SUB_FRAME_LENGTH_MS * psEncC->fs_kHz + 2 * psEncC->la_shape; + psEncC->Complexity = Complexity; + + celt_assert( psEncC->pitchEstimationLPCOrder <= MAX_FIND_PITCH_LPC_ORDER ); + celt_assert( psEncC->shapingLPCOrder <= MAX_SHAPE_LPC_ORDER ); + celt_assert( psEncC->nStatesDelayedDecision <= MAX_DEL_DEC_STATES ); + celt_assert( psEncC->warping_Q16 <= 32767 ); + celt_assert( psEncC->la_shape <= LA_SHAPE_MAX ); + celt_assert( psEncC->shapeWinLength <= SHAPE_LPC_WIN_MAX ); + + return ret; +} + +static OPUS_INLINE opus_int silk_setup_LBRR( + silk_encoder_state *psEncC, /* I/O */ + const silk_EncControlStruct *encControl /* I */ +) +{ + opus_int LBRR_in_previous_packet, ret = SILK_NO_ERROR; + + LBRR_in_previous_packet = psEncC->LBRR_enabled; + psEncC->LBRR_enabled = encControl->LBRR_coded; + if( psEncC->LBRR_enabled ) { + /* Set gain increase for coding LBRR excitation */ + if( LBRR_in_previous_packet == 0 ) { + /* Previous packet did not have LBRR, and was therefore coded at a higher bitrate */ + psEncC->LBRR_GainIncreases = 7; + } else { + psEncC->LBRR_GainIncreases = silk_max_int( 7 - silk_SMULWB( (opus_int32)psEncC->PacketLoss_perc, SILK_FIX_CONST( 0.4, 16 ) ), 2 ); + } + } + + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.c new file mode 100644 index 000000000..eb0c36ef1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.c @@ -0,0 +1,172 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "debug.h" + +#if SILK_DEBUG || SILK_TIC_TOC +#include "SigProc_FIX.h" +#endif + +#if SILK_TIC_TOC + +#if (defined(_WIN32) || defined(_WINCE)) +#include /* timer */ +#else /* Linux or Mac*/ +#include +#endif + +#ifdef _WIN32 +unsigned long silk_GetHighResolutionTime(void) /* O time in usec*/ +{ + /* Returns a time counter in microsec */ + /* the resolution is platform dependent */ + /* but is typically 1.62 us resolution */ + LARGE_INTEGER lpPerformanceCount; + LARGE_INTEGER lpFrequency; + QueryPerformanceCounter(&lpPerformanceCount); + QueryPerformanceFrequency(&lpFrequency); + return (unsigned long)((1000000*(lpPerformanceCount.QuadPart)) / lpFrequency.QuadPart); +} +#else /* Linux or Mac*/ +unsigned long GetHighResolutionTime(void) /* O time in usec*/ +{ + struct timeval tv; + gettimeofday(&tv, 0); + return((tv.tv_sec*1000000)+(tv.tv_usec)); +} +#endif + +int silk_Timer_nTimers = 0; +int silk_Timer_depth_ctr = 0; +char silk_Timer_tags[silk_NUM_TIMERS_MAX][silk_NUM_TIMERS_MAX_TAG_LEN]; +#ifdef _WIN32 +LARGE_INTEGER silk_Timer_start[silk_NUM_TIMERS_MAX]; +#else +unsigned long silk_Timer_start[silk_NUM_TIMERS_MAX]; +#endif +unsigned int silk_Timer_cnt[silk_NUM_TIMERS_MAX]; +opus_int64 silk_Timer_min[silk_NUM_TIMERS_MAX]; +opus_int64 silk_Timer_sum[silk_NUM_TIMERS_MAX]; +opus_int64 silk_Timer_max[silk_NUM_TIMERS_MAX]; +opus_int64 silk_Timer_depth[silk_NUM_TIMERS_MAX]; + +#ifdef _WIN32 +void silk_TimerSave(char *file_name) +{ + if( silk_Timer_nTimers > 0 ) + { + int k; + FILE *fp; + LARGE_INTEGER lpFrequency; + LARGE_INTEGER lpPerformanceCount1, lpPerformanceCount2; + int del = 0x7FFFFFFF; + double avg, sum_avg; + /* estimate overhead of calling performance counters */ + for( k = 0; k < 1000; k++ ) { + QueryPerformanceCounter(&lpPerformanceCount1); + QueryPerformanceCounter(&lpPerformanceCount2); + lpPerformanceCount2.QuadPart -= lpPerformanceCount1.QuadPart; + if( (int)lpPerformanceCount2.LowPart < del ) + del = lpPerformanceCount2.LowPart; + } + QueryPerformanceFrequency(&lpFrequency); + /* print results to file */ + sum_avg = 0.0f; + for( k = 0; k < silk_Timer_nTimers; k++ ) { + if (silk_Timer_depth[k] == 0) { + sum_avg += (1e6 * silk_Timer_sum[k] / silk_Timer_cnt[k] - del) / lpFrequency.QuadPart * silk_Timer_cnt[k]; + } + } + fp = fopen(file_name, "w"); + fprintf(fp, " min avg %% max count\n"); + for( k = 0; k < silk_Timer_nTimers; k++ ) { + if (silk_Timer_depth[k] == 0) { + fprintf(fp, "%-28s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 1) { + fprintf(fp, " %-27s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 2) { + fprintf(fp, " %-26s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 3) { + fprintf(fp, " %-25s", silk_Timer_tags[k]); + } else { + fprintf(fp, " %-24s", silk_Timer_tags[k]); + } + avg = (1e6 * silk_Timer_sum[k] / silk_Timer_cnt[k] - del) / lpFrequency.QuadPart; + fprintf(fp, "%8.2f", (1e6 * (silk_max_64(silk_Timer_min[k] - del, 0))) / lpFrequency.QuadPart); + fprintf(fp, "%12.2f %6.2f", avg, 100.0 * avg / sum_avg * silk_Timer_cnt[k]); + fprintf(fp, "%12.2f", (1e6 * (silk_max_64(silk_Timer_max[k] - del, 0))) / lpFrequency.QuadPart); + fprintf(fp, "%10d\n", silk_Timer_cnt[k]); + } + fprintf(fp, " microseconds\n"); + fclose(fp); + } +} +#else +void silk_TimerSave(char *file_name) +{ + if( silk_Timer_nTimers > 0 ) + { + int k; + FILE *fp; + /* print results to file */ + fp = fopen(file_name, "w"); + fprintf(fp, " min avg max count\n"); + for( k = 0; k < silk_Timer_nTimers; k++ ) + { + if (silk_Timer_depth[k] == 0) { + fprintf(fp, "%-28s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 1) { + fprintf(fp, " %-27s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 2) { + fprintf(fp, " %-26s", silk_Timer_tags[k]); + } else if (silk_Timer_depth[k] == 3) { + fprintf(fp, " %-25s", silk_Timer_tags[k]); + } else { + fprintf(fp, " %-24s", silk_Timer_tags[k]); + } + fprintf(fp, "%d ", silk_Timer_min[k]); + fprintf(fp, "%f ", (double)silk_Timer_sum[k] / (double)silk_Timer_cnt[k]); + fprintf(fp, "%d ", silk_Timer_max[k]); + fprintf(fp, "%10d\n", silk_Timer_cnt[k]); + } + fprintf(fp, " microseconds\n"); + fclose(fp); + } +} +#endif + +#endif /* SILK_TIC_TOC */ + +#if SILK_DEBUG +FILE *silk_debug_store_fp[ silk_NUM_STORES_MAX ]; +int silk_debug_store_count = 0; +#endif /* SILK_DEBUG */ + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.h new file mode 100644 index 000000000..36163e478 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/debug.h @@ -0,0 +1,267 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_DEBUG_H +#define SILK_DEBUG_H + +/* Set to 1 to enable DEBUG_STORE_DATA() macros for dumping + * intermediate signals from the codec. + */ +#define SILK_DEBUG 0 + +/* Flag for using timers */ +#define SILK_TIC_TOC 0 + +#if SILK_DEBUG || SILK_TIC_TOC +#include "typedef.h" +#include /* strcpy, strcmp */ +#include /* file writing */ +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if SILK_TIC_TOC + +unsigned long GetHighResolutionTime(void); /* O time in usec*/ + +#if (defined(_WIN32) || defined(_WINCE)) +#include /* timer */ +#else /* Linux or Mac*/ +#include +#endif + +/*********************************/ +/* timer functions for profiling */ +/*********************************/ +/* example: */ +/* */ +/* TIC(LPC) */ +/* do_LPC(in_vec, order, acoef); // do LPC analysis */ +/* TOC(LPC) */ +/* */ +/* and call the following just before exiting (from main) */ +/* */ +/* silk_TimerSave("silk_TimingData.txt"); */ +/* */ +/* results are now in silk_TimingData.txt */ + +void silk_TimerSave(char *file_name); + +/* max number of timers (in different locations) */ +#define silk_NUM_TIMERS_MAX 50 +/* max length of name tags in TIC(..), TOC(..) */ +#define silk_NUM_TIMERS_MAX_TAG_LEN 30 + +extern int silk_Timer_nTimers; +extern int silk_Timer_depth_ctr; +extern char silk_Timer_tags[silk_NUM_TIMERS_MAX][silk_NUM_TIMERS_MAX_TAG_LEN]; +#ifdef _WIN32 +extern LARGE_INTEGER silk_Timer_start[silk_NUM_TIMERS_MAX]; +#else +extern unsigned long silk_Timer_start[silk_NUM_TIMERS_MAX]; +#endif +extern unsigned int silk_Timer_cnt[silk_NUM_TIMERS_MAX]; +extern opus_int64 silk_Timer_sum[silk_NUM_TIMERS_MAX]; +extern opus_int64 silk_Timer_max[silk_NUM_TIMERS_MAX]; +extern opus_int64 silk_Timer_min[silk_NUM_TIMERS_MAX]; +extern opus_int64 silk_Timer_depth[silk_NUM_TIMERS_MAX]; + +/* WARNING: TIC()/TOC can measure only up to 0.1 seconds at a time */ +#ifdef _WIN32 +#define TIC(TAG_NAME) { \ + static int init = 0; \ + static int ID = -1; \ + if( init == 0 ) \ + { \ + int k; \ + init = 1; \ + for( k = 0; k < silk_Timer_nTimers; k++ ) { \ + if( strcmp(silk_Timer_tags[k], #TAG_NAME) == 0 ) { \ + ID = k; \ + break; \ + } \ + } \ + if (ID == -1) { \ + ID = silk_Timer_nTimers; \ + silk_Timer_nTimers++; \ + silk_Timer_depth[ID] = silk_Timer_depth_ctr; \ + strcpy(silk_Timer_tags[ID], #TAG_NAME); \ + silk_Timer_cnt[ID] = 0; \ + silk_Timer_sum[ID] = 0; \ + silk_Timer_min[ID] = 0xFFFFFFFF; \ + silk_Timer_max[ID] = 0; \ + } \ + } \ + silk_Timer_depth_ctr++; \ + QueryPerformanceCounter(&silk_Timer_start[ID]); \ +} +#else +#define TIC(TAG_NAME) { \ + static int init = 0; \ + static int ID = -1; \ + if( init == 0 ) \ + { \ + int k; \ + init = 1; \ + for( k = 0; k < silk_Timer_nTimers; k++ ) { \ + if( strcmp(silk_Timer_tags[k], #TAG_NAME) == 0 ) { \ + ID = k; \ + break; \ + } \ + } \ + if (ID == -1) { \ + ID = silk_Timer_nTimers; \ + silk_Timer_nTimers++; \ + silk_Timer_depth[ID] = silk_Timer_depth_ctr; \ + strcpy(silk_Timer_tags[ID], #TAG_NAME); \ + silk_Timer_cnt[ID] = 0; \ + silk_Timer_sum[ID] = 0; \ + silk_Timer_min[ID] = 0xFFFFFFFF; \ + silk_Timer_max[ID] = 0; \ + } \ + } \ + silk_Timer_depth_ctr++; \ + silk_Timer_start[ID] = GetHighResolutionTime(); \ +} +#endif + +#ifdef _WIN32 +#define TOC(TAG_NAME) { \ + LARGE_INTEGER lpPerformanceCount; \ + static int init = 0; \ + static int ID = 0; \ + if( init == 0 ) \ + { \ + int k; \ + init = 1; \ + for( k = 0; k < silk_Timer_nTimers; k++ ) { \ + if( strcmp(silk_Timer_tags[k], #TAG_NAME) == 0 ) { \ + ID = k; \ + break; \ + } \ + } \ + } \ + QueryPerformanceCounter(&lpPerformanceCount); \ + lpPerformanceCount.QuadPart -= silk_Timer_start[ID].QuadPart; \ + if((lpPerformanceCount.QuadPart < 100000000) && \ + (lpPerformanceCount.QuadPart >= 0)) { \ + silk_Timer_cnt[ID]++; \ + silk_Timer_sum[ID] += lpPerformanceCount.QuadPart; \ + if( lpPerformanceCount.QuadPart > silk_Timer_max[ID] ) \ + silk_Timer_max[ID] = lpPerformanceCount.QuadPart; \ + if( lpPerformanceCount.QuadPart < silk_Timer_min[ID] ) \ + silk_Timer_min[ID] = lpPerformanceCount.QuadPart; \ + } \ + silk_Timer_depth_ctr--; \ +} +#else +#define TOC(TAG_NAME) { \ + unsigned long endTime; \ + static int init = 0; \ + static int ID = 0; \ + if( init == 0 ) \ + { \ + int k; \ + init = 1; \ + for( k = 0; k < silk_Timer_nTimers; k++ ) { \ + if( strcmp(silk_Timer_tags[k], #TAG_NAME) == 0 ) { \ + ID = k; \ + break; \ + } \ + } \ + } \ + endTime = GetHighResolutionTime(); \ + endTime -= silk_Timer_start[ID]; \ + if((endTime < 100000000) && \ + (endTime >= 0)) { \ + silk_Timer_cnt[ID]++; \ + silk_Timer_sum[ID] += endTime; \ + if( endTime > silk_Timer_max[ID] ) \ + silk_Timer_max[ID] = endTime; \ + if( endTime < silk_Timer_min[ID] ) \ + silk_Timer_min[ID] = endTime; \ + } \ + silk_Timer_depth_ctr--; \ +} +#endif + +#else /* SILK_TIC_TOC */ + +/* define macros as empty strings */ +#define TIC(TAG_NAME) +#define TOC(TAG_NAME) +#define silk_TimerSave(FILE_NAME) + +#endif /* SILK_TIC_TOC */ + + +#if SILK_DEBUG +/************************************/ +/* write data to file for debugging */ +/************************************/ +/* Example: DEBUG_STORE_DATA(testfile.pcm, &RIN[0], 160*sizeof(opus_int16)); */ + +#define silk_NUM_STORES_MAX 100 +extern FILE *silk_debug_store_fp[ silk_NUM_STORES_MAX ]; +extern int silk_debug_store_count; + +/* Faster way of storing the data */ +#define DEBUG_STORE_DATA( FILE_NAME, DATA_PTR, N_BYTES ) { \ + static opus_int init = 0, cnt = 0; \ + static FILE **fp; \ + if (init == 0) { \ + init = 1; \ + cnt = silk_debug_store_count++; \ + silk_debug_store_fp[ cnt ] = fopen(#FILE_NAME, "wb"); \ + } \ + fwrite((DATA_PTR), (N_BYTES), 1, silk_debug_store_fp[ cnt ]); \ +} + +/* Call this at the end of main() */ +#define SILK_DEBUG_STORE_CLOSE_FILES { \ + opus_int i; \ + for( i = 0; i < silk_debug_store_count; i++ ) { \ + fclose( silk_debug_store_fp[ i ] ); \ + } \ +} + +#else /* SILK_DEBUG */ + +/* define macros as empty strings */ +#define DEBUG_STORE_DATA(FILE_NAME, DATA_PTR, N_BYTES) +#define SILK_DEBUG_STORE_CLOSE_FILES + +#endif /* SILK_DEBUG */ + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_DEBUG_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/dec_API.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/dec_API.c new file mode 100644 index 000000000..7d5ca7fb9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/dec_API.c @@ -0,0 +1,419 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "API.h" +#include "main.h" +#include "stack_alloc.h" +#include "os_support.h" + +/************************/ +/* Decoder Super Struct */ +/************************/ +typedef struct { + silk_decoder_state channel_state[ DECODER_NUM_CHANNELS ]; + stereo_dec_state sStereo; + opus_int nChannelsAPI; + opus_int nChannelsInternal; + opus_int prev_decode_only_middle; +} silk_decoder; + +/*********************/ +/* Decoder functions */ +/*********************/ + +opus_int silk_Get_Decoder_Size( /* O Returns error code */ + opus_int *decSizeBytes /* O Number of bytes in SILK decoder state */ +) +{ + opus_int ret = SILK_NO_ERROR; + + *decSizeBytes = sizeof( silk_decoder ); + + return ret; +} + +/* Reset decoder state */ +opus_int silk_InitDecoder( /* O Returns error code */ + void *decState /* I/O State */ +) +{ + opus_int n, ret = SILK_NO_ERROR; + silk_decoder_state *channel_state = ((silk_decoder *)decState)->channel_state; + + for( n = 0; n < DECODER_NUM_CHANNELS; n++ ) { + ret = silk_init_decoder( &channel_state[ n ] ); + } + silk_memset(&((silk_decoder *)decState)->sStereo, 0, sizeof(((silk_decoder *)decState)->sStereo)); + /* Not strictly needed, but it's cleaner that way */ + ((silk_decoder *)decState)->prev_decode_only_middle = 0; + + return ret; +} + +/* Decode a frame */ +opus_int silk_Decode( /* O Returns error code */ + void* decState, /* I/O State */ + silk_DecControlStruct* decControl, /* I/O Control Structure */ + opus_int lostFlag, /* I 0: no loss, 1 loss, 2 decode fec */ + opus_int newPacketFlag, /* I Indicates first decoder call for this packet */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 *samplesOut, /* O Decoded output speech vector */ + opus_int32 *nSamplesOut, /* O Number of samples decoded */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, n, decode_only_middle = 0, ret = SILK_NO_ERROR; + opus_int32 nSamplesOutDec, LBRR_symbol; + opus_int16 *samplesOut1_tmp[ 2 ]; + VARDECL( opus_int16, samplesOut1_tmp_storage1 ); + VARDECL( opus_int16, samplesOut1_tmp_storage2 ); + VARDECL( opus_int16, samplesOut2_tmp ); + opus_int32 MS_pred_Q13[ 2 ] = { 0 }; + opus_int16 *resample_out_ptr; + silk_decoder *psDec = ( silk_decoder * )decState; + silk_decoder_state *channel_state = psDec->channel_state; + opus_int has_side; + opus_int stereo_to_mono; + int delay_stack_alloc; + SAVE_STACK; + + celt_assert( decControl->nChannelsInternal == 1 || decControl->nChannelsInternal == 2 ); + + /**********************************/ + /* Test if first frame in payload */ + /**********************************/ + if( newPacketFlag ) { + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + channel_state[ n ].nFramesDecoded = 0; /* Used to count frames in packet */ + } + } + + /* If Mono -> Stereo transition in bitstream: init state of second channel */ + if( decControl->nChannelsInternal > psDec->nChannelsInternal ) { + ret += silk_init_decoder( &channel_state[ 1 ] ); + } + + stereo_to_mono = decControl->nChannelsInternal == 1 && psDec->nChannelsInternal == 2 && + ( decControl->internalSampleRate == 1000*channel_state[ 0 ].fs_kHz ); + + if( channel_state[ 0 ].nFramesDecoded == 0 ) { + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + opus_int fs_kHz_dec; + if( decControl->payloadSize_ms == 0 ) { + /* Assuming packet loss, use 10 ms */ + channel_state[ n ].nFramesPerPacket = 1; + channel_state[ n ].nb_subfr = 2; + } else if( decControl->payloadSize_ms == 10 ) { + channel_state[ n ].nFramesPerPacket = 1; + channel_state[ n ].nb_subfr = 2; + } else if( decControl->payloadSize_ms == 20 ) { + channel_state[ n ].nFramesPerPacket = 1; + channel_state[ n ].nb_subfr = 4; + } else if( decControl->payloadSize_ms == 40 ) { + channel_state[ n ].nFramesPerPacket = 2; + channel_state[ n ].nb_subfr = 4; + } else if( decControl->payloadSize_ms == 60 ) { + channel_state[ n ].nFramesPerPacket = 3; + channel_state[ n ].nb_subfr = 4; + } else { + celt_assert( 0 ); + RESTORE_STACK; + return SILK_DEC_INVALID_FRAME_SIZE; + } + fs_kHz_dec = ( decControl->internalSampleRate >> 10 ) + 1; + if( fs_kHz_dec != 8 && fs_kHz_dec != 12 && fs_kHz_dec != 16 ) { + celt_assert( 0 ); + RESTORE_STACK; + return SILK_DEC_INVALID_SAMPLING_FREQUENCY; + } + ret += silk_decoder_set_fs( &channel_state[ n ], fs_kHz_dec, decControl->API_sampleRate ); + } + } + + if( decControl->nChannelsAPI == 2 && decControl->nChannelsInternal == 2 && ( psDec->nChannelsAPI == 1 || psDec->nChannelsInternal == 1 ) ) { + silk_memset( psDec->sStereo.pred_prev_Q13, 0, sizeof( psDec->sStereo.pred_prev_Q13 ) ); + silk_memset( psDec->sStereo.sSide, 0, sizeof( psDec->sStereo.sSide ) ); + silk_memcpy( &channel_state[ 1 ].resampler_state, &channel_state[ 0 ].resampler_state, sizeof( silk_resampler_state_struct ) ); + } + psDec->nChannelsAPI = decControl->nChannelsAPI; + psDec->nChannelsInternal = decControl->nChannelsInternal; + + if( decControl->API_sampleRate > (opus_int32)MAX_API_FS_KHZ * 1000 || decControl->API_sampleRate < 8000 ) { + ret = SILK_DEC_INVALID_SAMPLING_FREQUENCY; + RESTORE_STACK; + return( ret ); + } + + if( lostFlag != FLAG_PACKET_LOST && channel_state[ 0 ].nFramesDecoded == 0 ) { + /* First decoder call for this payload */ + /* Decode VAD flags and LBRR flag */ + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + for( i = 0; i < channel_state[ n ].nFramesPerPacket; i++ ) { + channel_state[ n ].VAD_flags[ i ] = ec_dec_bit_logp(psRangeDec, 1); + } + channel_state[ n ].LBRR_flag = ec_dec_bit_logp(psRangeDec, 1); + } + /* Decode LBRR flags */ + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + silk_memset( channel_state[ n ].LBRR_flags, 0, sizeof( channel_state[ n ].LBRR_flags ) ); + if( channel_state[ n ].LBRR_flag ) { + if( channel_state[ n ].nFramesPerPacket == 1 ) { + channel_state[ n ].LBRR_flags[ 0 ] = 1; + } else { + LBRR_symbol = ec_dec_icdf( psRangeDec, silk_LBRR_flags_iCDF_ptr[ channel_state[ n ].nFramesPerPacket - 2 ], 8 ) + 1; + for( i = 0; i < channel_state[ n ].nFramesPerPacket; i++ ) { + channel_state[ n ].LBRR_flags[ i ] = silk_RSHIFT( LBRR_symbol, i ) & 1; + } + } + } + } + + if( lostFlag == FLAG_DECODE_NORMAL ) { + /* Regular decoding: skip all LBRR data */ + for( i = 0; i < channel_state[ 0 ].nFramesPerPacket; i++ ) { + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + if( channel_state[ n ].LBRR_flags[ i ] ) { + opus_int16 pulses[ MAX_FRAME_LENGTH ]; + opus_int condCoding; + + if( decControl->nChannelsInternal == 2 && n == 0 ) { + silk_stereo_decode_pred( psRangeDec, MS_pred_Q13 ); + if( channel_state[ 1 ].LBRR_flags[ i ] == 0 ) { + silk_stereo_decode_mid_only( psRangeDec, &decode_only_middle ); + } + } + /* Use conditional coding if previous frame available */ + if( i > 0 && channel_state[ n ].LBRR_flags[ i - 1 ] ) { + condCoding = CODE_CONDITIONALLY; + } else { + condCoding = CODE_INDEPENDENTLY; + } + silk_decode_indices( &channel_state[ n ], psRangeDec, i, 1, condCoding ); + silk_decode_pulses( psRangeDec, pulses, channel_state[ n ].indices.signalType, + channel_state[ n ].indices.quantOffsetType, channel_state[ n ].frame_length ); + } + } + } + } + } + + /* Get MS predictor index */ + if( decControl->nChannelsInternal == 2 ) { + if( lostFlag == FLAG_DECODE_NORMAL || + ( lostFlag == FLAG_DECODE_LBRR && channel_state[ 0 ].LBRR_flags[ channel_state[ 0 ].nFramesDecoded ] == 1 ) ) + { + silk_stereo_decode_pred( psRangeDec, MS_pred_Q13 ); + /* For LBRR data, decode mid-only flag only if side-channel's LBRR flag is false */ + if( ( lostFlag == FLAG_DECODE_NORMAL && channel_state[ 1 ].VAD_flags[ channel_state[ 0 ].nFramesDecoded ] == 0 ) || + ( lostFlag == FLAG_DECODE_LBRR && channel_state[ 1 ].LBRR_flags[ channel_state[ 0 ].nFramesDecoded ] == 0 ) ) + { + silk_stereo_decode_mid_only( psRangeDec, &decode_only_middle ); + } else { + decode_only_middle = 0; + } + } else { + for( n = 0; n < 2; n++ ) { + MS_pred_Q13[ n ] = psDec->sStereo.pred_prev_Q13[ n ]; + } + } + } + + /* Reset side channel decoder prediction memory for first frame with side coding */ + if( decControl->nChannelsInternal == 2 && decode_only_middle == 0 && psDec->prev_decode_only_middle == 1 ) { + silk_memset( psDec->channel_state[ 1 ].outBuf, 0, sizeof(psDec->channel_state[ 1 ].outBuf) ); + silk_memset( psDec->channel_state[ 1 ].sLPC_Q14_buf, 0, sizeof(psDec->channel_state[ 1 ].sLPC_Q14_buf) ); + psDec->channel_state[ 1 ].lagPrev = 100; + psDec->channel_state[ 1 ].LastGainIndex = 10; + psDec->channel_state[ 1 ].prevSignalType = TYPE_NO_VOICE_ACTIVITY; + psDec->channel_state[ 1 ].first_frame_after_reset = 1; + } + + /* Check if the temp buffer fits into the output PCM buffer. If it fits, + we can delay allocating the temp buffer until after the SILK peak stack + usage. We need to use a < and not a <= because of the two extra samples. */ + delay_stack_alloc = decControl->internalSampleRate*decControl->nChannelsInternal + < decControl->API_sampleRate*decControl->nChannelsAPI; + ALLOC( samplesOut1_tmp_storage1, delay_stack_alloc ? ALLOC_NONE + : decControl->nChannelsInternal*(channel_state[ 0 ].frame_length + 2 ), + opus_int16 ); + if ( delay_stack_alloc ) + { + samplesOut1_tmp[ 0 ] = samplesOut; + samplesOut1_tmp[ 1 ] = samplesOut + channel_state[ 0 ].frame_length + 2; + } else { + samplesOut1_tmp[ 0 ] = samplesOut1_tmp_storage1; + samplesOut1_tmp[ 1 ] = samplesOut1_tmp_storage1 + channel_state[ 0 ].frame_length + 2; + } + + if( lostFlag == FLAG_DECODE_NORMAL ) { + has_side = !decode_only_middle; + } else { + has_side = !psDec->prev_decode_only_middle + || (decControl->nChannelsInternal == 2 && lostFlag == FLAG_DECODE_LBRR && channel_state[1].LBRR_flags[ channel_state[1].nFramesDecoded ] == 1 ); + } + /* Call decoder for one frame */ + for( n = 0; n < decControl->nChannelsInternal; n++ ) { + if( n == 0 || has_side ) { + opus_int FrameIndex; + opus_int condCoding; + + FrameIndex = channel_state[ 0 ].nFramesDecoded - n; + /* Use independent coding if no previous frame available */ + if( FrameIndex <= 0 ) { + condCoding = CODE_INDEPENDENTLY; + } else if( lostFlag == FLAG_DECODE_LBRR ) { + condCoding = channel_state[ n ].LBRR_flags[ FrameIndex - 1 ] ? CODE_CONDITIONALLY : CODE_INDEPENDENTLY; + } else if( n > 0 && psDec->prev_decode_only_middle ) { + /* If we skipped a side frame in this packet, we don't + need LTP scaling; the LTP state is well-defined. */ + condCoding = CODE_INDEPENDENTLY_NO_LTP_SCALING; + } else { + condCoding = CODE_CONDITIONALLY; + } + ret += silk_decode_frame( &channel_state[ n ], psRangeDec, &samplesOut1_tmp[ n ][ 2 ], &nSamplesOutDec, lostFlag, condCoding, arch); + } else { + silk_memset( &samplesOut1_tmp[ n ][ 2 ], 0, nSamplesOutDec * sizeof( opus_int16 ) ); + } + channel_state[ n ].nFramesDecoded++; + } + + if( decControl->nChannelsAPI == 2 && decControl->nChannelsInternal == 2 ) { + /* Convert Mid/Side to Left/Right */ + silk_stereo_MS_to_LR( &psDec->sStereo, samplesOut1_tmp[ 0 ], samplesOut1_tmp[ 1 ], MS_pred_Q13, channel_state[ 0 ].fs_kHz, nSamplesOutDec ); + } else { + /* Buffering */ + silk_memcpy( samplesOut1_tmp[ 0 ], psDec->sStereo.sMid, 2 * sizeof( opus_int16 ) ); + silk_memcpy( psDec->sStereo.sMid, &samplesOut1_tmp[ 0 ][ nSamplesOutDec ], 2 * sizeof( opus_int16 ) ); + } + + /* Number of output samples */ + *nSamplesOut = silk_DIV32( nSamplesOutDec * decControl->API_sampleRate, silk_SMULBB( channel_state[ 0 ].fs_kHz, 1000 ) ); + + /* Set up pointers to temp buffers */ + ALLOC( samplesOut2_tmp, + decControl->nChannelsAPI == 2 ? *nSamplesOut : ALLOC_NONE, opus_int16 ); + if( decControl->nChannelsAPI == 2 ) { + resample_out_ptr = samplesOut2_tmp; + } else { + resample_out_ptr = samplesOut; + } + + ALLOC( samplesOut1_tmp_storage2, delay_stack_alloc + ? decControl->nChannelsInternal*(channel_state[ 0 ].frame_length + 2 ) + : ALLOC_NONE, + opus_int16 ); + if ( delay_stack_alloc ) { + OPUS_COPY(samplesOut1_tmp_storage2, samplesOut, decControl->nChannelsInternal*(channel_state[ 0 ].frame_length + 2)); + samplesOut1_tmp[ 0 ] = samplesOut1_tmp_storage2; + samplesOut1_tmp[ 1 ] = samplesOut1_tmp_storage2 + channel_state[ 0 ].frame_length + 2; + } + for( n = 0; n < silk_min( decControl->nChannelsAPI, decControl->nChannelsInternal ); n++ ) { + + /* Resample decoded signal to API_sampleRate */ + ret += silk_resampler( &channel_state[ n ].resampler_state, resample_out_ptr, &samplesOut1_tmp[ n ][ 1 ], nSamplesOutDec ); + + /* Interleave if stereo output and stereo stream */ + if( decControl->nChannelsAPI == 2 ) { + for( i = 0; i < *nSamplesOut; i++ ) { + samplesOut[ n + 2 * i ] = resample_out_ptr[ i ]; + } + } + } + + /* Create two channel output from mono stream */ + if( decControl->nChannelsAPI == 2 && decControl->nChannelsInternal == 1 ) { + if ( stereo_to_mono ){ + /* Resample right channel for newly collapsed stereo just in case + we weren't doing collapsing when switching to mono */ + ret += silk_resampler( &channel_state[ 1 ].resampler_state, resample_out_ptr, &samplesOut1_tmp[ 0 ][ 1 ], nSamplesOutDec ); + + for( i = 0; i < *nSamplesOut; i++ ) { + samplesOut[ 1 + 2 * i ] = resample_out_ptr[ i ]; + } + } else { + for( i = 0; i < *nSamplesOut; i++ ) { + samplesOut[ 1 + 2 * i ] = samplesOut[ 0 + 2 * i ]; + } + } + } + + /* Export pitch lag, measured at 48 kHz sampling rate */ + if( channel_state[ 0 ].prevSignalType == TYPE_VOICED ) { + int mult_tab[ 3 ] = { 6, 4, 3 }; + decControl->prevPitchLag = channel_state[ 0 ].lagPrev * mult_tab[ ( channel_state[ 0 ].fs_kHz - 8 ) >> 2 ]; + } else { + decControl->prevPitchLag = 0; + } + + if( lostFlag == FLAG_PACKET_LOST ) { + /* On packet loss, remove the gain clamping to prevent having the energy "bounce back" + if we lose packets when the energy is going down */ + for ( i = 0; i < psDec->nChannelsInternal; i++ ) + psDec->channel_state[ i ].LastGainIndex = 10; + } else { + psDec->prev_decode_only_middle = decode_only_middle; + } + RESTORE_STACK; + return ret; +} + +#if 0 +/* Getting table of contents for a packet */ +opus_int silk_get_TOC( + const opus_uint8 *payload, /* I Payload data */ + const opus_int nBytesIn, /* I Number of input bytes */ + const opus_int nFramesPerPayload, /* I Number of SILK frames per payload */ + silk_TOC_struct *Silk_TOC /* O Type of content */ +) +{ + opus_int i, flags, ret = SILK_NO_ERROR; + + if( nBytesIn < 1 ) { + return -1; + } + if( nFramesPerPayload < 0 || nFramesPerPayload > 3 ) { + return -1; + } + + silk_memset( Silk_TOC, 0, sizeof( *Silk_TOC ) ); + + /* For stereo, extract the flags for the mid channel */ + flags = silk_RSHIFT( payload[ 0 ], 7 - nFramesPerPayload ) & ( silk_LSHIFT( 1, nFramesPerPayload + 1 ) - 1 ); + + Silk_TOC->inbandFECFlag = flags & 1; + for( i = nFramesPerPayload - 1; i >= 0 ; i-- ) { + flags = silk_RSHIFT( flags, 1 ); + Silk_TOC->VADFlags[ i ] = flags & 1; + Silk_TOC->VADFlag |= flags & 1; + } + + return ret; +} +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_core.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_core.c new file mode 100644 index 000000000..1c352a652 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_core.c @@ -0,0 +1,237 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/**********************************************************/ +/* Core decoder. Performs inverse NSQ operation LTP + LPC */ +/**********************************************************/ +void silk_decode_core( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I Decoder control */ + opus_int16 xq[], /* O Decoded speech */ + const opus_int16 pulses[ MAX_FRAME_LENGTH ], /* I Pulse signal */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, k, lag = 0, start_idx, sLTP_buf_idx, NLSF_interpolation_flag, signalType; + opus_int16 *A_Q12, *B_Q14, *pxq, A_Q12_tmp[ MAX_LPC_ORDER ]; + VARDECL( opus_int16, sLTP ); + VARDECL( opus_int32, sLTP_Q15 ); + opus_int32 LTP_pred_Q13, LPC_pred_Q10, Gain_Q10, inv_gain_Q31, gain_adj_Q16, rand_seed, offset_Q10; + opus_int32 *pred_lag_ptr, *pexc_Q14, *pres_Q14; + VARDECL( opus_int32, res_Q14 ); + VARDECL( opus_int32, sLPC_Q14 ); + SAVE_STACK; + + silk_assert( psDec->prev_gain_Q16 != 0 ); + + ALLOC( sLTP, psDec->ltp_mem_length, opus_int16 ); + ALLOC( sLTP_Q15, psDec->ltp_mem_length + psDec->frame_length, opus_int32 ); + ALLOC( res_Q14, psDec->subfr_length, opus_int32 ); + ALLOC( sLPC_Q14, psDec->subfr_length + MAX_LPC_ORDER, opus_int32 ); + + offset_Q10 = silk_Quantization_Offsets_Q10[ psDec->indices.signalType >> 1 ][ psDec->indices.quantOffsetType ]; + + if( psDec->indices.NLSFInterpCoef_Q2 < 1 << 2 ) { + NLSF_interpolation_flag = 1; + } else { + NLSF_interpolation_flag = 0; + } + + /* Decode excitation */ + rand_seed = psDec->indices.Seed; + for( i = 0; i < psDec->frame_length; i++ ) { + rand_seed = silk_RAND( rand_seed ); + psDec->exc_Q14[ i ] = silk_LSHIFT( (opus_int32)pulses[ i ], 14 ); + if( psDec->exc_Q14[ i ] > 0 ) { + psDec->exc_Q14[ i ] -= QUANT_LEVEL_ADJUST_Q10 << 4; + } else + if( psDec->exc_Q14[ i ] < 0 ) { + psDec->exc_Q14[ i ] += QUANT_LEVEL_ADJUST_Q10 << 4; + } + psDec->exc_Q14[ i ] += offset_Q10 << 4; + if( rand_seed < 0 ) { + psDec->exc_Q14[ i ] = -psDec->exc_Q14[ i ]; + } + + rand_seed = silk_ADD32_ovflw( rand_seed, pulses[ i ] ); + } + + /* Copy LPC state */ + silk_memcpy( sLPC_Q14, psDec->sLPC_Q14_buf, MAX_LPC_ORDER * sizeof( opus_int32 ) ); + + pexc_Q14 = psDec->exc_Q14; + pxq = xq; + sLTP_buf_idx = psDec->ltp_mem_length; + /* Loop over subframes */ + for( k = 0; k < psDec->nb_subfr; k++ ) { + pres_Q14 = res_Q14; + A_Q12 = psDecCtrl->PredCoef_Q12[ k >> 1 ]; + + /* Preload LPC coeficients to array on stack. Gives small performance gain */ + silk_memcpy( A_Q12_tmp, A_Q12, psDec->LPC_order * sizeof( opus_int16 ) ); + B_Q14 = &psDecCtrl->LTPCoef_Q14[ k * LTP_ORDER ]; + signalType = psDec->indices.signalType; + + Gain_Q10 = silk_RSHIFT( psDecCtrl->Gains_Q16[ k ], 6 ); + inv_gain_Q31 = silk_INVERSE32_varQ( psDecCtrl->Gains_Q16[ k ], 47 ); + + /* Calculate gain adjustment factor */ + if( psDecCtrl->Gains_Q16[ k ] != psDec->prev_gain_Q16 ) { + gain_adj_Q16 = silk_DIV32_varQ( psDec->prev_gain_Q16, psDecCtrl->Gains_Q16[ k ], 16 ); + + /* Scale short term state */ + for( i = 0; i < MAX_LPC_ORDER; i++ ) { + sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, sLPC_Q14[ i ] ); + } + } else { + gain_adj_Q16 = (opus_int32)1 << 16; + } + + /* Save inv_gain */ + silk_assert( inv_gain_Q31 != 0 ); + psDec->prev_gain_Q16 = psDecCtrl->Gains_Q16[ k ]; + + /* Avoid abrupt transition from voiced PLC to unvoiced normal decoding */ + if( psDec->lossCnt && psDec->prevSignalType == TYPE_VOICED && + psDec->indices.signalType != TYPE_VOICED && k < MAX_NB_SUBFR/2 ) { + + silk_memset( B_Q14, 0, LTP_ORDER * sizeof( opus_int16 ) ); + B_Q14[ LTP_ORDER/2 ] = SILK_FIX_CONST( 0.25, 14 ); + + signalType = TYPE_VOICED; + psDecCtrl->pitchL[ k ] = psDec->lagPrev; + } + + if( signalType == TYPE_VOICED ) { + /* Voiced */ + lag = psDecCtrl->pitchL[ k ]; + + /* Re-whitening */ + if( k == 0 || ( k == 2 && NLSF_interpolation_flag ) ) { + /* Rewhiten with new A coefs */ + start_idx = psDec->ltp_mem_length - lag - psDec->LPC_order - LTP_ORDER / 2; + celt_assert( start_idx > 0 ); + + if( k == 2 ) { + silk_memcpy( &psDec->outBuf[ psDec->ltp_mem_length ], xq, 2 * psDec->subfr_length * sizeof( opus_int16 ) ); + } + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &psDec->outBuf[ start_idx + k * psDec->subfr_length ], + A_Q12, psDec->ltp_mem_length - start_idx, psDec->LPC_order, arch ); + + /* After rewhitening the LTP state is unscaled */ + if( k == 0 ) { + /* Do LTP downscaling to reduce inter-packet dependency */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, psDecCtrl->LTP_scale_Q14 ), 2 ); + } + for( i = 0; i < lag + LTP_ORDER/2; i++ ) { + sLTP_Q15[ sLTP_buf_idx - i - 1 ] = silk_SMULWB( inv_gain_Q31, sLTP[ psDec->ltp_mem_length - i - 1 ] ); + } + } else { + /* Update LTP state when Gain changes */ + if( gain_adj_Q16 != (opus_int32)1 << 16 ) { + for( i = 0; i < lag + LTP_ORDER/2; i++ ) { + sLTP_Q15[ sLTP_buf_idx - i - 1 ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ sLTP_buf_idx - i - 1 ] ); + } + } + } + } + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Set up pointer */ + pred_lag_ptr = &sLTP_Q15[ sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + for( i = 0; i < psDec->subfr_length; i++ ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q13 = 2; + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ 0 ], B_Q14[ 0 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -1 ], B_Q14[ 1 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -2 ], B_Q14[ 2 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -3 ], B_Q14[ 3 ] ); + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -4 ], B_Q14[ 4 ] ); + pred_lag_ptr++; + + /* Generate LPC excitation */ + pres_Q14[ i ] = silk_ADD_LSHIFT32( pexc_Q14[ i ], LTP_pred_Q13, 1 ); + + /* Update states */ + sLTP_Q15[ sLTP_buf_idx ] = silk_LSHIFT( pres_Q14[ i ], 1 ); + sLTP_buf_idx++; + } + } else { + pres_Q14 = pexc_Q14; + } + + for( i = 0; i < psDec->subfr_length; i++ ) { + /* Short-term prediction */ + celt_assert( psDec->LPC_order == 10 || psDec->LPC_order == 16 ); + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q10 = silk_RSHIFT( psDec->LPC_order, 1 ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 1 ], A_Q12_tmp[ 0 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 2 ], A_Q12_tmp[ 1 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 3 ], A_Q12_tmp[ 2 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 4 ], A_Q12_tmp[ 3 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 5 ], A_Q12_tmp[ 4 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 6 ], A_Q12_tmp[ 5 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 7 ], A_Q12_tmp[ 6 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 8 ], A_Q12_tmp[ 7 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 9 ], A_Q12_tmp[ 8 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 10 ], A_Q12_tmp[ 9 ] ); + if( psDec->LPC_order == 16 ) { + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 11 ], A_Q12_tmp[ 10 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 12 ], A_Q12_tmp[ 11 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 13 ], A_Q12_tmp[ 12 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 14 ], A_Q12_tmp[ 13 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 15 ], A_Q12_tmp[ 14 ] ); + LPC_pred_Q10 = silk_SMLAWB( LPC_pred_Q10, sLPC_Q14[ MAX_LPC_ORDER + i - 16 ], A_Q12_tmp[ 15 ] ); + } + + /* Add prediction to LPC excitation */ + sLPC_Q14[ MAX_LPC_ORDER + i ] = silk_ADD_SAT32( pres_Q14[ i ], silk_LSHIFT_SAT32( LPC_pred_Q10, 4 ) ); + + /* Scale with gain */ + pxq[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( sLPC_Q14[ MAX_LPC_ORDER + i ], Gain_Q10 ), 8 ) ); + } + + /* Update LPC filter state */ + silk_memcpy( sLPC_Q14, &sLPC_Q14[ psDec->subfr_length ], MAX_LPC_ORDER * sizeof( opus_int32 ) ); + pexc_Q14 += psDec->subfr_length; + pxq += psDec->subfr_length; + } + + /* Save LPC state */ + silk_memcpy( psDec->sLPC_Q14_buf, sLPC_Q14, MAX_LPC_ORDER * sizeof( opus_int32 ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_frame.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_frame.c new file mode 100644 index 000000000..4f36f854c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_frame.c @@ -0,0 +1,129 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" +#include "PLC.h" + +/****************/ +/* Decode frame */ +/****************/ +opus_int silk_decode_frame( + silk_decoder_state *psDec, /* I/O Pointer to Silk decoder state */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pOut[], /* O Pointer to output speech frame */ + opus_int32 *pN, /* O Pointer to size of output frame */ + opus_int lostFlag, /* I 0: no loss, 1 loss, 2 decode fec */ + opus_int condCoding, /* I The type of conditional coding to use */ + int arch /* I Run-time architecture */ +) +{ + VARDECL( silk_decoder_control, psDecCtrl ); + opus_int L, mv_len, ret = 0; + SAVE_STACK; + + L = psDec->frame_length; + ALLOC( psDecCtrl, 1, silk_decoder_control ); + psDecCtrl->LTP_scale_Q14 = 0; + + /* Safety checks */ + celt_assert( L > 0 && L <= MAX_FRAME_LENGTH ); + + if( lostFlag == FLAG_DECODE_NORMAL || + ( lostFlag == FLAG_DECODE_LBRR && psDec->LBRR_flags[ psDec->nFramesDecoded ] == 1 ) ) + { + VARDECL( opus_int16, pulses ); + ALLOC( pulses, (L + SHELL_CODEC_FRAME_LENGTH - 1) & + ~(SHELL_CODEC_FRAME_LENGTH - 1), opus_int16 ); + /*********************************************/ + /* Decode quantization indices of side info */ + /*********************************************/ + silk_decode_indices( psDec, psRangeDec, psDec->nFramesDecoded, lostFlag, condCoding ); + + /*********************************************/ + /* Decode quantization indices of excitation */ + /*********************************************/ + silk_decode_pulses( psRangeDec, pulses, psDec->indices.signalType, + psDec->indices.quantOffsetType, psDec->frame_length ); + + /********************************************/ + /* Decode parameters and pulse signal */ + /********************************************/ + silk_decode_parameters( psDec, psDecCtrl, condCoding ); + + /********************************************************/ + /* Run inverse NSQ */ + /********************************************************/ + silk_decode_core( psDec, psDecCtrl, pOut, pulses, arch ); + + /********************************************************/ + /* Update PLC state */ + /********************************************************/ + silk_PLC( psDec, psDecCtrl, pOut, 0, arch ); + + psDec->lossCnt = 0; + psDec->prevSignalType = psDec->indices.signalType; + celt_assert( psDec->prevSignalType >= 0 && psDec->prevSignalType <= 2 ); + + /* A frame has been decoded without errors */ + psDec->first_frame_after_reset = 0; + } else { + /* Handle packet loss by extrapolation */ + silk_PLC( psDec, psDecCtrl, pOut, 1, arch ); + } + + /*************************/ + /* Update output buffer. */ + /*************************/ + celt_assert( psDec->ltp_mem_length >= psDec->frame_length ); + mv_len = psDec->ltp_mem_length - psDec->frame_length; + silk_memmove( psDec->outBuf, &psDec->outBuf[ psDec->frame_length ], mv_len * sizeof(opus_int16) ); + silk_memcpy( &psDec->outBuf[ mv_len ], pOut, psDec->frame_length * sizeof( opus_int16 ) ); + + /************************************************/ + /* Comfort noise generation / estimation */ + /************************************************/ + silk_CNG( psDec, psDecCtrl, pOut, L ); + + /****************************************************************/ + /* Ensure smooth connection of extrapolated and good frames */ + /****************************************************************/ + silk_PLC_glue_frames( psDec, pOut, L ); + + /* Update some decoder state variables */ + psDec->lagPrev = psDecCtrl->pitchL[ psDec->nb_subfr - 1 ]; + + /* Set output frame length */ + *pN = L; + + RESTORE_STACK; + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_indices.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_indices.c new file mode 100644 index 000000000..0bb4a997a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_indices.c @@ -0,0 +1,151 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Decode side-information parameters from payload */ +void silk_decode_indices( + silk_decoder_state *psDec, /* I/O State */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int FrameIndex, /* I Frame number */ + opus_int decode_LBRR, /* I Flag indicating LBRR data is being decoded */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int i, k, Ix; + opus_int decode_absolute_lagIndex, delta_lagIndex; + opus_int16 ec_ix[ MAX_LPC_ORDER ]; + opus_uint8 pred_Q8[ MAX_LPC_ORDER ]; + + /*******************************************/ + /* Decode signal type and quantizer offset */ + /*******************************************/ + if( decode_LBRR || psDec->VAD_flags[ FrameIndex ] ) { + Ix = ec_dec_icdf( psRangeDec, silk_type_offset_VAD_iCDF, 8 ) + 2; + } else { + Ix = ec_dec_icdf( psRangeDec, silk_type_offset_no_VAD_iCDF, 8 ); + } + psDec->indices.signalType = (opus_int8)silk_RSHIFT( Ix, 1 ); + psDec->indices.quantOffsetType = (opus_int8)( Ix & 1 ); + + /****************/ + /* Decode gains */ + /****************/ + /* First subframe */ + if( condCoding == CODE_CONDITIONALLY ) { + /* Conditional coding */ + psDec->indices.GainsIndices[ 0 ] = (opus_int8)ec_dec_icdf( psRangeDec, silk_delta_gain_iCDF, 8 ); + } else { + /* Independent coding, in two stages: MSB bits followed by 3 LSBs */ + psDec->indices.GainsIndices[ 0 ] = (opus_int8)silk_LSHIFT( ec_dec_icdf( psRangeDec, silk_gain_iCDF[ psDec->indices.signalType ], 8 ), 3 ); + psDec->indices.GainsIndices[ 0 ] += (opus_int8)ec_dec_icdf( psRangeDec, silk_uniform8_iCDF, 8 ); + } + + /* Remaining subframes */ + for( i = 1; i < psDec->nb_subfr; i++ ) { + psDec->indices.GainsIndices[ i ] = (opus_int8)ec_dec_icdf( psRangeDec, silk_delta_gain_iCDF, 8 ); + } + + /**********************/ + /* Decode LSF Indices */ + /**********************/ + psDec->indices.NLSFIndices[ 0 ] = (opus_int8)ec_dec_icdf( psRangeDec, &psDec->psNLSF_CB->CB1_iCDF[ ( psDec->indices.signalType >> 1 ) * psDec->psNLSF_CB->nVectors ], 8 ); + silk_NLSF_unpack( ec_ix, pred_Q8, psDec->psNLSF_CB, psDec->indices.NLSFIndices[ 0 ] ); + celt_assert( psDec->psNLSF_CB->order == psDec->LPC_order ); + for( i = 0; i < psDec->psNLSF_CB->order; i++ ) { + Ix = ec_dec_icdf( psRangeDec, &psDec->psNLSF_CB->ec_iCDF[ ec_ix[ i ] ], 8 ); + if( Ix == 0 ) { + Ix -= ec_dec_icdf( psRangeDec, silk_NLSF_EXT_iCDF, 8 ); + } else if( Ix == 2 * NLSF_QUANT_MAX_AMPLITUDE ) { + Ix += ec_dec_icdf( psRangeDec, silk_NLSF_EXT_iCDF, 8 ); + } + psDec->indices.NLSFIndices[ i+1 ] = (opus_int8)( Ix - NLSF_QUANT_MAX_AMPLITUDE ); + } + + /* Decode LSF interpolation factor */ + if( psDec->nb_subfr == MAX_NB_SUBFR ) { + psDec->indices.NLSFInterpCoef_Q2 = (opus_int8)ec_dec_icdf( psRangeDec, silk_NLSF_interpolation_factor_iCDF, 8 ); + } else { + psDec->indices.NLSFInterpCoef_Q2 = 4; + } + + if( psDec->indices.signalType == TYPE_VOICED ) + { + /*********************/ + /* Decode pitch lags */ + /*********************/ + /* Get lag index */ + decode_absolute_lagIndex = 1; + if( condCoding == CODE_CONDITIONALLY && psDec->ec_prevSignalType == TYPE_VOICED ) { + /* Decode Delta index */ + delta_lagIndex = (opus_int16)ec_dec_icdf( psRangeDec, silk_pitch_delta_iCDF, 8 ); + if( delta_lagIndex > 0 ) { + delta_lagIndex = delta_lagIndex - 9; + psDec->indices.lagIndex = (opus_int16)( psDec->ec_prevLagIndex + delta_lagIndex ); + decode_absolute_lagIndex = 0; + } + } + if( decode_absolute_lagIndex ) { + /* Absolute decoding */ + psDec->indices.lagIndex = (opus_int16)ec_dec_icdf( psRangeDec, silk_pitch_lag_iCDF, 8 ) * silk_RSHIFT( psDec->fs_kHz, 1 ); + psDec->indices.lagIndex += (opus_int16)ec_dec_icdf( psRangeDec, psDec->pitch_lag_low_bits_iCDF, 8 ); + } + psDec->ec_prevLagIndex = psDec->indices.lagIndex; + + /* Get countour index */ + psDec->indices.contourIndex = (opus_int8)ec_dec_icdf( psRangeDec, psDec->pitch_contour_iCDF, 8 ); + + /********************/ + /* Decode LTP gains */ + /********************/ + /* Decode PERIndex value */ + psDec->indices.PERIndex = (opus_int8)ec_dec_icdf( psRangeDec, silk_LTP_per_index_iCDF, 8 ); + + for( k = 0; k < psDec->nb_subfr; k++ ) { + psDec->indices.LTPIndex[ k ] = (opus_int8)ec_dec_icdf( psRangeDec, silk_LTP_gain_iCDF_ptrs[ psDec->indices.PERIndex ], 8 ); + } + + /**********************/ + /* Decode LTP scaling */ + /**********************/ + if( condCoding == CODE_INDEPENDENTLY ) { + psDec->indices.LTP_scaleIndex = (opus_int8)ec_dec_icdf( psRangeDec, silk_LTPscale_iCDF, 8 ); + } else { + psDec->indices.LTP_scaleIndex = 0; + } + } + psDec->ec_prevSignalType = psDec->indices.signalType; + + /***************/ + /* Decode seed */ + /***************/ + psDec->indices.Seed = (opus_int8)ec_dec_icdf( psRangeDec, silk_uniform4_iCDF, 8 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_parameters.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_parameters.c new file mode 100644 index 000000000..a56a40985 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_parameters.c @@ -0,0 +1,115 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Decode parameters from payload */ +void silk_decode_parameters( + silk_decoder_state *psDec, /* I/O State */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int i, k, Ix; + opus_int16 pNLSF_Q15[ MAX_LPC_ORDER ], pNLSF0_Q15[ MAX_LPC_ORDER ]; + const opus_int8 *cbk_ptr_Q7; + + /* Dequant Gains */ + silk_gains_dequant( psDecCtrl->Gains_Q16, psDec->indices.GainsIndices, + &psDec->LastGainIndex, condCoding == CODE_CONDITIONALLY, psDec->nb_subfr ); + + /****************/ + /* Decode NLSFs */ + /****************/ + silk_NLSF_decode( pNLSF_Q15, psDec->indices.NLSFIndices, psDec->psNLSF_CB ); + + /* Convert NLSF parameters to AR prediction filter coefficients */ + silk_NLSF2A( psDecCtrl->PredCoef_Q12[ 1 ], pNLSF_Q15, psDec->LPC_order, psDec->arch ); + + /* If just reset, e.g., because internal Fs changed, do not allow interpolation */ + /* improves the case of packet loss in the first frame after a switch */ + if( psDec->first_frame_after_reset == 1 ) { + psDec->indices.NLSFInterpCoef_Q2 = 4; + } + + if( psDec->indices.NLSFInterpCoef_Q2 < 4 ) { + /* Calculation of the interpolated NLSF0 vector from the interpolation factor, */ + /* the previous NLSF1, and the current NLSF1 */ + for( i = 0; i < psDec->LPC_order; i++ ) { + pNLSF0_Q15[ i ] = psDec->prevNLSF_Q15[ i ] + silk_RSHIFT( silk_MUL( psDec->indices.NLSFInterpCoef_Q2, + pNLSF_Q15[ i ] - psDec->prevNLSF_Q15[ i ] ), 2 ); + } + + /* Convert NLSF parameters to AR prediction filter coefficients */ + silk_NLSF2A( psDecCtrl->PredCoef_Q12[ 0 ], pNLSF0_Q15, psDec->LPC_order, psDec->arch ); + } else { + /* Copy LPC coefficients for first half from second half */ + silk_memcpy( psDecCtrl->PredCoef_Q12[ 0 ], psDecCtrl->PredCoef_Q12[ 1 ], psDec->LPC_order * sizeof( opus_int16 ) ); + } + + silk_memcpy( psDec->prevNLSF_Q15, pNLSF_Q15, psDec->LPC_order * sizeof( opus_int16 ) ); + + /* After a packet loss do BWE of LPC coefs */ + if( psDec->lossCnt ) { + silk_bwexpander( psDecCtrl->PredCoef_Q12[ 0 ], psDec->LPC_order, BWE_AFTER_LOSS_Q16 ); + silk_bwexpander( psDecCtrl->PredCoef_Q12[ 1 ], psDec->LPC_order, BWE_AFTER_LOSS_Q16 ); + } + + if( psDec->indices.signalType == TYPE_VOICED ) { + /*********************/ + /* Decode pitch lags */ + /*********************/ + + /* Decode pitch values */ + silk_decode_pitch( psDec->indices.lagIndex, psDec->indices.contourIndex, psDecCtrl->pitchL, psDec->fs_kHz, psDec->nb_subfr ); + + /* Decode Codebook Index */ + cbk_ptr_Q7 = silk_LTP_vq_ptrs_Q7[ psDec->indices.PERIndex ]; /* set pointer to start of codebook */ + + for( k = 0; k < psDec->nb_subfr; k++ ) { + Ix = psDec->indices.LTPIndex[ k ]; + for( i = 0; i < LTP_ORDER; i++ ) { + psDecCtrl->LTPCoef_Q14[ k * LTP_ORDER + i ] = silk_LSHIFT( cbk_ptr_Q7[ Ix * LTP_ORDER + i ], 7 ); + } + } + + /**********************/ + /* Decode LTP scaling */ + /**********************/ + Ix = psDec->indices.LTP_scaleIndex; + psDecCtrl->LTP_scale_Q14 = silk_LTPScales_table_Q14[ Ix ]; + } else { + silk_memset( psDecCtrl->pitchL, 0, psDec->nb_subfr * sizeof( opus_int ) ); + silk_memset( psDecCtrl->LTPCoef_Q14, 0, LTP_ORDER * psDec->nb_subfr * sizeof( opus_int16 ) ); + psDec->indices.PERIndex = 0; + psDecCtrl->LTP_scale_Q14 = 0; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pitch.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pitch.c new file mode 100644 index 000000000..fd1b6bf55 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pitch.c @@ -0,0 +1,77 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/*********************************************************** +* Pitch analyser function +********************************************************** */ +#include "SigProc_FIX.h" +#include "pitch_est_defines.h" + +void silk_decode_pitch( + opus_int16 lagIndex, /* I */ + opus_int8 contourIndex, /* O */ + opus_int pitch_lags[], /* O 4 pitch values */ + const opus_int Fs_kHz, /* I sampling frequency (kHz) */ + const opus_int nb_subfr /* I number of sub frames */ +) +{ + opus_int lag, k, min_lag, max_lag, cbk_size; + const opus_int8 *Lag_CB_ptr; + + if( Fs_kHz == 8 ) { + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_CB_ptr = &silk_CB_lags_stage2[ 0 ][ 0 ]; + cbk_size = PE_NB_CBKS_STAGE2_EXT; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1 ); + Lag_CB_ptr = &silk_CB_lags_stage2_10_ms[ 0 ][ 0 ]; + cbk_size = PE_NB_CBKS_STAGE2_10MS; + } + } else { + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1 ); + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + } + } + + min_lag = silk_SMULBB( PE_MIN_LAG_MS, Fs_kHz ); + max_lag = silk_SMULBB( PE_MAX_LAG_MS, Fs_kHz ); + lag = min_lag + lagIndex; + + for( k = 0; k < nb_subfr; k++ ) { + pitch_lags[ k ] = lag + matrix_ptr( Lag_CB_ptr, k, contourIndex, cbk_size ); + pitch_lags[ k ] = silk_LIMIT( pitch_lags[ k ], min_lag, max_lag ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pulses.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pulses.c new file mode 100644 index 000000000..a56d2d307 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decode_pulses.c @@ -0,0 +1,115 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/*********************************************/ +/* Decode quantization indices of excitation */ +/*********************************************/ +void silk_decode_pulses( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pulses[], /* O Excitation signal */ + const opus_int signalType, /* I Sigtype */ + const opus_int quantOffsetType, /* I quantOffsetType */ + const opus_int frame_length /* I Frame length */ +) +{ + opus_int i, j, k, iter, abs_q, nLS, RateLevelIndex; + opus_int sum_pulses[ MAX_NB_SHELL_BLOCKS ], nLshifts[ MAX_NB_SHELL_BLOCKS ]; + opus_int16 *pulses_ptr; + const opus_uint8 *cdf_ptr; + + /*********************/ + /* Decode rate level */ + /*********************/ + RateLevelIndex = ec_dec_icdf( psRangeDec, silk_rate_levels_iCDF[ signalType >> 1 ], 8 ); + + /* Calculate number of shell blocks */ + silk_assert( 1 << LOG2_SHELL_CODEC_FRAME_LENGTH == SHELL_CODEC_FRAME_LENGTH ); + iter = silk_RSHIFT( frame_length, LOG2_SHELL_CODEC_FRAME_LENGTH ); + if( iter * SHELL_CODEC_FRAME_LENGTH < frame_length ) { + celt_assert( frame_length == 12 * 10 ); /* Make sure only happens for 10 ms @ 12 kHz */ + iter++; + } + + /***************************************************/ + /* Sum-Weighted-Pulses Decoding */ + /***************************************************/ + cdf_ptr = silk_pulses_per_block_iCDF[ RateLevelIndex ]; + for( i = 0; i < iter; i++ ) { + nLshifts[ i ] = 0; + sum_pulses[ i ] = ec_dec_icdf( psRangeDec, cdf_ptr, 8 ); + + /* LSB indication */ + while( sum_pulses[ i ] == SILK_MAX_PULSES + 1 ) { + nLshifts[ i ]++; + /* When we've already got 10 LSBs, we shift the table to not allow (SILK_MAX_PULSES + 1) */ + sum_pulses[ i ] = ec_dec_icdf( psRangeDec, + silk_pulses_per_block_iCDF[ N_RATE_LEVELS - 1] + ( nLshifts[ i ] == 10 ), 8 ); + } + } + + /***************************************************/ + /* Shell decoding */ + /***************************************************/ + for( i = 0; i < iter; i++ ) { + if( sum_pulses[ i ] > 0 ) { + silk_shell_decoder( &pulses[ silk_SMULBB( i, SHELL_CODEC_FRAME_LENGTH ) ], psRangeDec, sum_pulses[ i ] ); + } else { + silk_memset( &pulses[ silk_SMULBB( i, SHELL_CODEC_FRAME_LENGTH ) ], 0, SHELL_CODEC_FRAME_LENGTH * sizeof( pulses[0] ) ); + } + } + + /***************************************************/ + /* LSB Decoding */ + /***************************************************/ + for( i = 0; i < iter; i++ ) { + if( nLshifts[ i ] > 0 ) { + nLS = nLshifts[ i ]; + pulses_ptr = &pulses[ silk_SMULBB( i, SHELL_CODEC_FRAME_LENGTH ) ]; + for( k = 0; k < SHELL_CODEC_FRAME_LENGTH; k++ ) { + abs_q = pulses_ptr[ k ]; + for( j = 0; j < nLS; j++ ) { + abs_q = silk_LSHIFT( abs_q, 1 ); + abs_q += ec_dec_icdf( psRangeDec, silk_lsb_iCDF, 8 ); + } + pulses_ptr[ k ] = abs_q; + } + /* Mark the number of pulses non-zero for sign decoding. */ + sum_pulses[ i ] |= nLS << 5; + } + } + + /****************************************/ + /* Decode and add signs to pulse signal */ + /****************************************/ + silk_decode_signs( psRangeDec, pulses, frame_length, signalType, quantOffsetType, sum_pulses ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decoder_set_fs.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decoder_set_fs.c new file mode 100644 index 000000000..d9a13d0f0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/decoder_set_fs.c @@ -0,0 +1,108 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Set decoder sampling rate */ +opus_int silk_decoder_set_fs( + silk_decoder_state *psDec, /* I/O Decoder state pointer */ + opus_int fs_kHz, /* I Sampling frequency (kHz) */ + opus_int32 fs_API_Hz /* I API Sampling frequency (Hz) */ +) +{ + opus_int frame_length, ret = 0; + + celt_assert( fs_kHz == 8 || fs_kHz == 12 || fs_kHz == 16 ); + celt_assert( psDec->nb_subfr == MAX_NB_SUBFR || psDec->nb_subfr == MAX_NB_SUBFR/2 ); + + /* New (sub)frame length */ + psDec->subfr_length = silk_SMULBB( SUB_FRAME_LENGTH_MS, fs_kHz ); + frame_length = silk_SMULBB( psDec->nb_subfr, psDec->subfr_length ); + + /* Initialize resampler when switching internal or external sampling frequency */ + if( psDec->fs_kHz != fs_kHz || psDec->fs_API_hz != fs_API_Hz ) { + /* Initialize the resampler for dec_API.c preparing resampling from fs_kHz to API_fs_Hz */ + ret += silk_resampler_init( &psDec->resampler_state, silk_SMULBB( fs_kHz, 1000 ), fs_API_Hz, 0 ); + + psDec->fs_API_hz = fs_API_Hz; + } + + if( psDec->fs_kHz != fs_kHz || frame_length != psDec->frame_length ) { + if( fs_kHz == 8 ) { + if( psDec->nb_subfr == MAX_NB_SUBFR ) { + psDec->pitch_contour_iCDF = silk_pitch_contour_NB_iCDF; + } else { + psDec->pitch_contour_iCDF = silk_pitch_contour_10_ms_NB_iCDF; + } + } else { + if( psDec->nb_subfr == MAX_NB_SUBFR ) { + psDec->pitch_contour_iCDF = silk_pitch_contour_iCDF; + } else { + psDec->pitch_contour_iCDF = silk_pitch_contour_10_ms_iCDF; + } + } + if( psDec->fs_kHz != fs_kHz ) { + psDec->ltp_mem_length = silk_SMULBB( LTP_MEM_LENGTH_MS, fs_kHz ); + if( fs_kHz == 8 || fs_kHz == 12 ) { + psDec->LPC_order = MIN_LPC_ORDER; + psDec->psNLSF_CB = &silk_NLSF_CB_NB_MB; + } else { + psDec->LPC_order = MAX_LPC_ORDER; + psDec->psNLSF_CB = &silk_NLSF_CB_WB; + } + if( fs_kHz == 16 ) { + psDec->pitch_lag_low_bits_iCDF = silk_uniform8_iCDF; + } else if( fs_kHz == 12 ) { + psDec->pitch_lag_low_bits_iCDF = silk_uniform6_iCDF; + } else if( fs_kHz == 8 ) { + psDec->pitch_lag_low_bits_iCDF = silk_uniform4_iCDF; + } else { + /* unsupported sampling rate */ + celt_assert( 0 ); + } + psDec->first_frame_after_reset = 1; + psDec->lagPrev = 100; + psDec->LastGainIndex = 10; + psDec->prevSignalType = TYPE_NO_VOICE_ACTIVITY; + silk_memset( psDec->outBuf, 0, sizeof(psDec->outBuf)); + silk_memset( psDec->sLPC_Q14_buf, 0, sizeof(psDec->sLPC_Q14_buf) ); + } + + psDec->fs_kHz = fs_kHz; + psDec->frame_length = frame_length; + } + + /* Check that settings are valid */ + celt_assert( psDec->frame_length > 0 && psDec->frame_length <= MAX_FRAME_LENGTH ); + + return ret; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/define.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/define.h new file mode 100644 index 000000000..491c86f33 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/define.h @@ -0,0 +1,235 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_DEFINE_H +#define SILK_DEFINE_H + +#include "errors.h" +#include "typedef.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Max number of encoder channels (1/2) */ +#define ENCODER_NUM_CHANNELS 2 +/* Number of decoder channels (1/2) */ +#define DECODER_NUM_CHANNELS 2 + +#define MAX_FRAMES_PER_PACKET 3 + +/* Limits on bitrate */ +#define MIN_TARGET_RATE_BPS 5000 +#define MAX_TARGET_RATE_BPS 80000 + +/* LBRR thresholds */ +#define LBRR_NB_MIN_RATE_BPS 12000 +#define LBRR_MB_MIN_RATE_BPS 14000 +#define LBRR_WB_MIN_RATE_BPS 16000 + +/* DTX settings */ +#define NB_SPEECH_FRAMES_BEFORE_DTX 10 /* eq 200 ms */ +#define MAX_CONSECUTIVE_DTX 20 /* eq 400 ms */ +#define DTX_ACTIVITY_THRESHOLD 0.1f + +/* VAD decision */ +#define VAD_NO_DECISION -1 +#define VAD_NO_ACTIVITY 0 +#define VAD_ACTIVITY 1 + +/* Maximum sampling frequency */ +#define MAX_FS_KHZ 16 +#define MAX_API_FS_KHZ 48 + +/* Signal types */ +#define TYPE_NO_VOICE_ACTIVITY 0 +#define TYPE_UNVOICED 1 +#define TYPE_VOICED 2 + +/* Conditional coding types */ +#define CODE_INDEPENDENTLY 0 +#define CODE_INDEPENDENTLY_NO_LTP_SCALING 1 +#define CODE_CONDITIONALLY 2 + +/* Settings for stereo processing */ +#define STEREO_QUANT_TAB_SIZE 16 +#define STEREO_QUANT_SUB_STEPS 5 +#define STEREO_INTERP_LEN_MS 8 /* must be even */ +#define STEREO_RATIO_SMOOTH_COEF 0.01 /* smoothing coef for signal norms and stereo width */ + +/* Range of pitch lag estimates */ +#define PITCH_EST_MIN_LAG_MS 2 /* 2 ms -> 500 Hz */ +#define PITCH_EST_MAX_LAG_MS 18 /* 18 ms -> 56 Hz */ + +/* Maximum number of subframes */ +#define MAX_NB_SUBFR 4 + +/* Number of samples per frame */ +#define LTP_MEM_LENGTH_MS 20 +#define SUB_FRAME_LENGTH_MS 5 +#define MAX_SUB_FRAME_LENGTH ( SUB_FRAME_LENGTH_MS * MAX_FS_KHZ ) +#define MAX_FRAME_LENGTH_MS ( SUB_FRAME_LENGTH_MS * MAX_NB_SUBFR ) +#define MAX_FRAME_LENGTH ( MAX_FRAME_LENGTH_MS * MAX_FS_KHZ ) + +/* Milliseconds of lookahead for pitch analysis */ +#define LA_PITCH_MS 2 +#define LA_PITCH_MAX ( LA_PITCH_MS * MAX_FS_KHZ ) + +/* Order of LPC used in find pitch */ +#define MAX_FIND_PITCH_LPC_ORDER 16 + +/* Length of LPC window used in find pitch */ +#define FIND_PITCH_LPC_WIN_MS ( 20 + (LA_PITCH_MS << 1) ) +#define FIND_PITCH_LPC_WIN_MS_2_SF ( 10 + (LA_PITCH_MS << 1) ) +#define FIND_PITCH_LPC_WIN_MAX ( FIND_PITCH_LPC_WIN_MS * MAX_FS_KHZ ) + +/* Milliseconds of lookahead for noise shape analysis */ +#define LA_SHAPE_MS 5 +#define LA_SHAPE_MAX ( LA_SHAPE_MS * MAX_FS_KHZ ) + +/* Maximum length of LPC window used in noise shape analysis */ +#define SHAPE_LPC_WIN_MAX ( 15 * MAX_FS_KHZ ) + +/* dB level of lowest gain quantization level */ +#define MIN_QGAIN_DB 2 +/* dB level of highest gain quantization level */ +#define MAX_QGAIN_DB 88 +/* Number of gain quantization levels */ +#define N_LEVELS_QGAIN 64 +/* Max increase in gain quantization index */ +#define MAX_DELTA_GAIN_QUANT 36 +/* Max decrease in gain quantization index */ +#define MIN_DELTA_GAIN_QUANT -4 + +/* Quantization offsets (multiples of 4) */ +#define OFFSET_VL_Q10 32 +#define OFFSET_VH_Q10 100 +#define OFFSET_UVL_Q10 100 +#define OFFSET_UVH_Q10 240 + +#define QUANT_LEVEL_ADJUST_Q10 80 + +/* Maximum numbers of iterations used to stabilize an LPC vector */ +#define MAX_LPC_STABILIZE_ITERATIONS 16 +#define MAX_PREDICTION_POWER_GAIN 1e4f +#define MAX_PREDICTION_POWER_GAIN_AFTER_RESET 1e2f + +#define MAX_LPC_ORDER 16 +#define MIN_LPC_ORDER 10 + +/* Find Pred Coef defines */ +#define LTP_ORDER 5 + +/* LTP quantization settings */ +#define NB_LTP_CBKS 3 + +/* Flag to use harmonic noise shaping */ +#define USE_HARM_SHAPING 1 + +/* Max LPC order of noise shaping filters */ +#define MAX_SHAPE_LPC_ORDER 24 + +#define HARM_SHAPE_FIR_TAPS 3 + +/* Maximum number of delayed decision states */ +#define MAX_DEL_DEC_STATES 4 + +#define LTP_BUF_LENGTH 512 +#define LTP_MASK ( LTP_BUF_LENGTH - 1 ) + +#define DECISION_DELAY 40 + +/* Number of subframes for excitation entropy coding */ +#define SHELL_CODEC_FRAME_LENGTH 16 +#define LOG2_SHELL_CODEC_FRAME_LENGTH 4 +#define MAX_NB_SHELL_BLOCKS ( MAX_FRAME_LENGTH / SHELL_CODEC_FRAME_LENGTH ) + +/* Number of rate levels, for entropy coding of excitation */ +#define N_RATE_LEVELS 10 + +/* Maximum sum of pulses per shell coding frame */ +#define SILK_MAX_PULSES 16 + +#define MAX_MATRIX_SIZE MAX_LPC_ORDER /* Max of LPC Order and LTP order */ + +# define NSQ_LPC_BUF_LENGTH MAX_LPC_ORDER + +/***************************/ +/* Voice activity detector */ +/***************************/ +#define VAD_N_BANDS 4 + +#define VAD_INTERNAL_SUBFRAMES_LOG2 2 +#define VAD_INTERNAL_SUBFRAMES ( 1 << VAD_INTERNAL_SUBFRAMES_LOG2 ) + +#define VAD_NOISE_LEVEL_SMOOTH_COEF_Q16 1024 /* Must be < 4096 */ +#define VAD_NOISE_LEVELS_BIAS 50 + +/* Sigmoid settings */ +#define VAD_NEGATIVE_OFFSET_Q5 128 /* sigmoid is 0 at -128 */ +#define VAD_SNR_FACTOR_Q16 45000 + +/* smoothing for SNR measurement */ +#define VAD_SNR_SMOOTH_COEF_Q18 4096 + +/* Size of the piecewise linear cosine approximation table for the LSFs */ +#define LSF_COS_TAB_SZ_FIX 128 + +/******************/ +/* NLSF quantizer */ +/******************/ +#define NLSF_W_Q 2 +#define NLSF_VQ_MAX_VECTORS 32 +#define NLSF_QUANT_MAX_AMPLITUDE 4 +#define NLSF_QUANT_MAX_AMPLITUDE_EXT 10 +#define NLSF_QUANT_LEVEL_ADJ 0.1 +#define NLSF_QUANT_DEL_DEC_STATES_LOG2 2 +#define NLSF_QUANT_DEL_DEC_STATES ( 1 << NLSF_QUANT_DEL_DEC_STATES_LOG2 ) + +/* Transition filtering for mode switching */ +#define TRANSITION_TIME_MS 5120 /* 5120 = 64 * FRAME_LENGTH_MS * ( TRANSITION_INT_NUM - 1 ) = 64*(20*4)*/ +#define TRANSITION_NB 3 /* Hardcoded in tables */ +#define TRANSITION_NA 2 /* Hardcoded in tables */ +#define TRANSITION_INT_NUM 5 /* Hardcoded in tables */ +#define TRANSITION_FRAMES ( TRANSITION_TIME_MS / MAX_FRAME_LENGTH_MS ) +#define TRANSITION_INT_STEPS ( TRANSITION_FRAMES / ( TRANSITION_INT_NUM - 1 ) ) + +/* BWE factors to apply after packet loss */ +#define BWE_AFTER_LOSS_Q16 63570 + +/* Defines for CN generation */ +#define CNG_BUF_MASK_MAX 255 /* 2^floor(log2(MAX_FRAME_LENGTH))-1 */ +#define CNG_GAIN_SMTH_Q16 4634 /* 0.25^(1/4) */ +#define CNG_GAIN_SMTH_THRESHOLD_Q16 46396 /* -3 dB */ +#define CNG_NLSF_SMTH_Q16 16348 /* 0.25 */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/enc_API.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/enc_API.c new file mode 100644 index 000000000..55a33f37e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/enc_API.c @@ -0,0 +1,576 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "define.h" +#include "API.h" +#include "control.h" +#include "typedef.h" +#include "stack_alloc.h" +#include "structs.h" +#include "tuning_parameters.h" +#ifdef FIXED_POINT +#include "main_FIX.h" +#else +#include "main_FLP.h" +#endif + +/***************************************/ +/* Read control structure from encoder */ +/***************************************/ +static opus_int silk_QueryEncoder( /* O Returns error code */ + const void *encState, /* I State */ + silk_EncControlStruct *encStatus /* O Encoder Status */ +); + +/****************************************/ +/* Encoder functions */ +/****************************************/ + +opus_int silk_Get_Encoder_Size( /* O Returns error code */ + opus_int *encSizeBytes /* O Number of bytes in SILK encoder state */ +) +{ + opus_int ret = SILK_NO_ERROR; + + *encSizeBytes = sizeof( silk_encoder ); + + return ret; +} + +/*************************/ +/* Init or Reset encoder */ +/*************************/ +opus_int silk_InitEncoder( /* O Returns error code */ + void *encState, /* I/O State */ + int arch, /* I Run-time architecture */ + silk_EncControlStruct *encStatus /* O Encoder Status */ +) +{ + silk_encoder *psEnc; + opus_int n, ret = SILK_NO_ERROR; + + psEnc = (silk_encoder *)encState; + + /* Reset encoder */ + silk_memset( psEnc, 0, sizeof( silk_encoder ) ); + for( n = 0; n < ENCODER_NUM_CHANNELS; n++ ) { + if( ret += silk_init_encoder( &psEnc->state_Fxx[ n ], arch ) ) { + celt_assert( 0 ); + } + } + + psEnc->nChannelsAPI = 1; + psEnc->nChannelsInternal = 1; + + /* Read control structure */ + if( ret += silk_QueryEncoder( encState, encStatus ) ) { + celt_assert( 0 ); + } + + return ret; +} + +/***************************************/ +/* Read control structure from encoder */ +/***************************************/ +static opus_int silk_QueryEncoder( /* O Returns error code */ + const void *encState, /* I State */ + silk_EncControlStruct *encStatus /* O Encoder Status */ +) +{ + opus_int ret = SILK_NO_ERROR; + silk_encoder_state_Fxx *state_Fxx; + silk_encoder *psEnc = (silk_encoder *)encState; + + state_Fxx = psEnc->state_Fxx; + + encStatus->nChannelsAPI = psEnc->nChannelsAPI; + encStatus->nChannelsInternal = psEnc->nChannelsInternal; + encStatus->API_sampleRate = state_Fxx[ 0 ].sCmn.API_fs_Hz; + encStatus->maxInternalSampleRate = state_Fxx[ 0 ].sCmn.maxInternal_fs_Hz; + encStatus->minInternalSampleRate = state_Fxx[ 0 ].sCmn.minInternal_fs_Hz; + encStatus->desiredInternalSampleRate = state_Fxx[ 0 ].sCmn.desiredInternal_fs_Hz; + encStatus->payloadSize_ms = state_Fxx[ 0 ].sCmn.PacketSize_ms; + encStatus->bitRate = state_Fxx[ 0 ].sCmn.TargetRate_bps; + encStatus->packetLossPercentage = state_Fxx[ 0 ].sCmn.PacketLoss_perc; + encStatus->complexity = state_Fxx[ 0 ].sCmn.Complexity; + encStatus->useInBandFEC = state_Fxx[ 0 ].sCmn.useInBandFEC; + encStatus->useDTX = state_Fxx[ 0 ].sCmn.useDTX; + encStatus->useCBR = state_Fxx[ 0 ].sCmn.useCBR; + encStatus->internalSampleRate = silk_SMULBB( state_Fxx[ 0 ].sCmn.fs_kHz, 1000 ); + encStatus->allowBandwidthSwitch = state_Fxx[ 0 ].sCmn.allow_bandwidth_switch; + encStatus->inWBmodeWithoutVariableLP = state_Fxx[ 0 ].sCmn.fs_kHz == 16 && state_Fxx[ 0 ].sCmn.sLP.mode == 0; + + return ret; +} + + +/**************************/ +/* Encode frame with Silk */ +/**************************/ +/* Note: if prefillFlag is set, the input must contain 10 ms of audio, irrespective of what */ +/* encControl->payloadSize_ms is set to */ +opus_int silk_Encode( /* O Returns error code */ + void *encState, /* I/O State */ + silk_EncControlStruct *encControl, /* I Control status */ + const opus_int16 *samplesIn, /* I Speech sample input vector */ + opus_int nSamplesIn, /* I Number of samples in input vector */ + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int32 *nBytesOut, /* I/O Number of bytes in payload (input: Max bytes) */ + const opus_int prefillFlag, /* I Flag to indicate prefilling buffers no coding */ + opus_int activity /* I Decision of Opus voice activity detector */ +) +{ + opus_int n, i, nBits, flags, tmp_payloadSize_ms = 0, tmp_complexity = 0, ret = 0; + opus_int nSamplesToBuffer, nSamplesToBufferMax, nBlocksOf10ms; + opus_int nSamplesFromInput = 0, nSamplesFromInputMax; + opus_int speech_act_thr_for_switch_Q8; + opus_int32 TargetRate_bps, MStargetRates_bps[ 2 ], channelRate_bps, LBRR_symbol, sum; + silk_encoder *psEnc = ( silk_encoder * )encState; + VARDECL( opus_int16, buf ); + opus_int transition, curr_block, tot_blocks; + SAVE_STACK; + + if (encControl->reducedDependency) + { + psEnc->state_Fxx[0].sCmn.first_frame_after_reset = 1; + psEnc->state_Fxx[1].sCmn.first_frame_after_reset = 1; + } + psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded = psEnc->state_Fxx[ 1 ].sCmn.nFramesEncoded = 0; + + /* Check values in encoder control structure */ + if( ( ret = check_control_input( encControl ) ) != 0 ) { + celt_assert( 0 ); + RESTORE_STACK; + return ret; + } + + encControl->switchReady = 0; + + if( encControl->nChannelsInternal > psEnc->nChannelsInternal ) { + /* Mono -> Stereo transition: init state of second channel and stereo state */ + ret += silk_init_encoder( &psEnc->state_Fxx[ 1 ], psEnc->state_Fxx[ 0 ].sCmn.arch ); + silk_memset( psEnc->sStereo.pred_prev_Q13, 0, sizeof( psEnc->sStereo.pred_prev_Q13 ) ); + silk_memset( psEnc->sStereo.sSide, 0, sizeof( psEnc->sStereo.sSide ) ); + psEnc->sStereo.mid_side_amp_Q0[ 0 ] = 0; + psEnc->sStereo.mid_side_amp_Q0[ 1 ] = 1; + psEnc->sStereo.mid_side_amp_Q0[ 2 ] = 0; + psEnc->sStereo.mid_side_amp_Q0[ 3 ] = 1; + psEnc->sStereo.width_prev_Q14 = 0; + psEnc->sStereo.smth_width_Q14 = SILK_FIX_CONST( 1, 14 ); + if( psEnc->nChannelsAPI == 2 ) { + silk_memcpy( &psEnc->state_Fxx[ 1 ].sCmn.resampler_state, &psEnc->state_Fxx[ 0 ].sCmn.resampler_state, sizeof( silk_resampler_state_struct ) ); + silk_memcpy( &psEnc->state_Fxx[ 1 ].sCmn.In_HP_State, &psEnc->state_Fxx[ 0 ].sCmn.In_HP_State, sizeof( psEnc->state_Fxx[ 1 ].sCmn.In_HP_State ) ); + } + } + + transition = (encControl->payloadSize_ms != psEnc->state_Fxx[ 0 ].sCmn.PacketSize_ms) || (psEnc->nChannelsInternal != encControl->nChannelsInternal); + + psEnc->nChannelsAPI = encControl->nChannelsAPI; + psEnc->nChannelsInternal = encControl->nChannelsInternal; + + nBlocksOf10ms = silk_DIV32( 100 * nSamplesIn, encControl->API_sampleRate ); + tot_blocks = ( nBlocksOf10ms > 1 ) ? nBlocksOf10ms >> 1 : 1; + curr_block = 0; + if( prefillFlag ) { + silk_LP_state save_LP; + /* Only accept input length of 10 ms */ + if( nBlocksOf10ms != 1 ) { + celt_assert( 0 ); + RESTORE_STACK; + return SILK_ENC_INPUT_INVALID_NO_OF_SAMPLES; + } + if ( prefillFlag == 2 ) { + save_LP = psEnc->state_Fxx[ 0 ].sCmn.sLP; + /* Save the sampling rate so the bandwidth switching code can keep handling transitions. */ + save_LP.saved_fs_kHz = psEnc->state_Fxx[ 0 ].sCmn.fs_kHz; + } + /* Reset Encoder */ + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + ret = silk_init_encoder( &psEnc->state_Fxx[ n ], psEnc->state_Fxx[ n ].sCmn.arch ); + /* Restore the variable LP state. */ + if ( prefillFlag == 2 ) { + psEnc->state_Fxx[ n ].sCmn.sLP = save_LP; + } + celt_assert( !ret ); + } + tmp_payloadSize_ms = encControl->payloadSize_ms; + encControl->payloadSize_ms = 10; + tmp_complexity = encControl->complexity; + encControl->complexity = 0; + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + psEnc->state_Fxx[ n ].sCmn.controlled_since_last_payload = 0; + psEnc->state_Fxx[ n ].sCmn.prefillFlag = 1; + } + } else { + /* Only accept input lengths that are a multiple of 10 ms */ + if( nBlocksOf10ms * encControl->API_sampleRate != 100 * nSamplesIn || nSamplesIn < 0 ) { + celt_assert( 0 ); + RESTORE_STACK; + return SILK_ENC_INPUT_INVALID_NO_OF_SAMPLES; + } + /* Make sure no more than one packet can be produced */ + if( 1000 * (opus_int32)nSamplesIn > encControl->payloadSize_ms * encControl->API_sampleRate ) { + celt_assert( 0 ); + RESTORE_STACK; + return SILK_ENC_INPUT_INVALID_NO_OF_SAMPLES; + } + } + + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + /* Force the side channel to the same rate as the mid */ + opus_int force_fs_kHz = (n==1) ? psEnc->state_Fxx[0].sCmn.fs_kHz : 0; + if( ( ret = silk_control_encoder( &psEnc->state_Fxx[ n ], encControl, psEnc->allowBandwidthSwitch, n, force_fs_kHz ) ) != 0 ) { + silk_assert( 0 ); + RESTORE_STACK; + return ret; + } + if( psEnc->state_Fxx[n].sCmn.first_frame_after_reset || transition ) { + for( i = 0; i < psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket; i++ ) { + psEnc->state_Fxx[ n ].sCmn.LBRR_flags[ i ] = 0; + } + } + psEnc->state_Fxx[ n ].sCmn.inDTX = psEnc->state_Fxx[ n ].sCmn.useDTX; + } + celt_assert( encControl->nChannelsInternal == 1 || psEnc->state_Fxx[ 0 ].sCmn.fs_kHz == psEnc->state_Fxx[ 1 ].sCmn.fs_kHz ); + + /* Input buffering/resampling and encoding */ + nSamplesToBufferMax = + 10 * nBlocksOf10ms * psEnc->state_Fxx[ 0 ].sCmn.fs_kHz; + nSamplesFromInputMax = + silk_DIV32_16( nSamplesToBufferMax * + psEnc->state_Fxx[ 0 ].sCmn.API_fs_Hz, + psEnc->state_Fxx[ 0 ].sCmn.fs_kHz * 1000 ); + ALLOC( buf, nSamplesFromInputMax, opus_int16 ); + while( 1 ) { + nSamplesToBuffer = psEnc->state_Fxx[ 0 ].sCmn.frame_length - psEnc->state_Fxx[ 0 ].sCmn.inputBufIx; + nSamplesToBuffer = silk_min( nSamplesToBuffer, nSamplesToBufferMax ); + nSamplesFromInput = silk_DIV32_16( nSamplesToBuffer * psEnc->state_Fxx[ 0 ].sCmn.API_fs_Hz, psEnc->state_Fxx[ 0 ].sCmn.fs_kHz * 1000 ); + /* Resample and write to buffer */ + if( encControl->nChannelsAPI == 2 && encControl->nChannelsInternal == 2 ) { + opus_int id = psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded; + for( n = 0; n < nSamplesFromInput; n++ ) { + buf[ n ] = samplesIn[ 2 * n ]; + } + /* Making sure to start both resamplers from the same state when switching from mono to stereo */ + if( psEnc->nPrevChannelsInternal == 1 && id==0 ) { + silk_memcpy( &psEnc->state_Fxx[ 1 ].sCmn.resampler_state, &psEnc->state_Fxx[ 0 ].sCmn.resampler_state, sizeof(psEnc->state_Fxx[ 1 ].sCmn.resampler_state)); + } + + ret += silk_resampler( &psEnc->state_Fxx[ 0 ].sCmn.resampler_state, + &psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.inputBufIx + 2 ], buf, nSamplesFromInput ); + psEnc->state_Fxx[ 0 ].sCmn.inputBufIx += nSamplesToBuffer; + + nSamplesToBuffer = psEnc->state_Fxx[ 1 ].sCmn.frame_length - psEnc->state_Fxx[ 1 ].sCmn.inputBufIx; + nSamplesToBuffer = silk_min( nSamplesToBuffer, 10 * nBlocksOf10ms * psEnc->state_Fxx[ 1 ].sCmn.fs_kHz ); + for( n = 0; n < nSamplesFromInput; n++ ) { + buf[ n ] = samplesIn[ 2 * n + 1 ]; + } + ret += silk_resampler( &psEnc->state_Fxx[ 1 ].sCmn.resampler_state, + &psEnc->state_Fxx[ 1 ].sCmn.inputBuf[ psEnc->state_Fxx[ 1 ].sCmn.inputBufIx + 2 ], buf, nSamplesFromInput ); + + psEnc->state_Fxx[ 1 ].sCmn.inputBufIx += nSamplesToBuffer; + } else if( encControl->nChannelsAPI == 2 && encControl->nChannelsInternal == 1 ) { + /* Combine left and right channels before resampling */ + for( n = 0; n < nSamplesFromInput; n++ ) { + sum = samplesIn[ 2 * n ] + samplesIn[ 2 * n + 1 ]; + buf[ n ] = (opus_int16)silk_RSHIFT_ROUND( sum, 1 ); + } + ret += silk_resampler( &psEnc->state_Fxx[ 0 ].sCmn.resampler_state, + &psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.inputBufIx + 2 ], buf, nSamplesFromInput ); + /* On the first mono frame, average the results for the two resampler states */ + if( psEnc->nPrevChannelsInternal == 2 && psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded == 0 ) { + ret += silk_resampler( &psEnc->state_Fxx[ 1 ].sCmn.resampler_state, + &psEnc->state_Fxx[ 1 ].sCmn.inputBuf[ psEnc->state_Fxx[ 1 ].sCmn.inputBufIx + 2 ], buf, nSamplesFromInput ); + for( n = 0; n < psEnc->state_Fxx[ 0 ].sCmn.frame_length; n++ ) { + psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.inputBufIx+n+2 ] = + silk_RSHIFT(psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.inputBufIx+n+2 ] + + psEnc->state_Fxx[ 1 ].sCmn.inputBuf[ psEnc->state_Fxx[ 1 ].sCmn.inputBufIx+n+2 ], 1); + } + } + psEnc->state_Fxx[ 0 ].sCmn.inputBufIx += nSamplesToBuffer; + } else { + celt_assert( encControl->nChannelsAPI == 1 && encControl->nChannelsInternal == 1 ); + silk_memcpy(buf, samplesIn, nSamplesFromInput*sizeof(opus_int16)); + ret += silk_resampler( &psEnc->state_Fxx[ 0 ].sCmn.resampler_state, + &psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.inputBufIx + 2 ], buf, nSamplesFromInput ); + psEnc->state_Fxx[ 0 ].sCmn.inputBufIx += nSamplesToBuffer; + } + + samplesIn += nSamplesFromInput * encControl->nChannelsAPI; + nSamplesIn -= nSamplesFromInput; + + /* Default */ + psEnc->allowBandwidthSwitch = 0; + + /* Silk encoder */ + if( psEnc->state_Fxx[ 0 ].sCmn.inputBufIx >= psEnc->state_Fxx[ 0 ].sCmn.frame_length ) { + /* Enough data in input buffer, so encode */ + celt_assert( psEnc->state_Fxx[ 0 ].sCmn.inputBufIx == psEnc->state_Fxx[ 0 ].sCmn.frame_length ); + celt_assert( encControl->nChannelsInternal == 1 || psEnc->state_Fxx[ 1 ].sCmn.inputBufIx == psEnc->state_Fxx[ 1 ].sCmn.frame_length ); + + /* Deal with LBRR data */ + if( psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded == 0 && !prefillFlag ) { + /* Create space at start of payload for VAD and FEC flags */ + opus_uint8 iCDF[ 2 ] = { 0, 0 }; + iCDF[ 0 ] = 256 - silk_RSHIFT( 256, ( psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket + 1 ) * encControl->nChannelsInternal ); + ec_enc_icdf( psRangeEnc, 0, iCDF, 8 ); + + /* Encode any LBRR data from previous packet */ + /* Encode LBRR flags */ + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + LBRR_symbol = 0; + for( i = 0; i < psEnc->state_Fxx[ n ].sCmn.nFramesPerPacket; i++ ) { + LBRR_symbol |= silk_LSHIFT( psEnc->state_Fxx[ n ].sCmn.LBRR_flags[ i ], i ); + } + psEnc->state_Fxx[ n ].sCmn.LBRR_flag = LBRR_symbol > 0 ? 1 : 0; + if( LBRR_symbol && psEnc->state_Fxx[ n ].sCmn.nFramesPerPacket > 1 ) { + ec_enc_icdf( psRangeEnc, LBRR_symbol - 1, silk_LBRR_flags_iCDF_ptr[ psEnc->state_Fxx[ n ].sCmn.nFramesPerPacket - 2 ], 8 ); + } + } + + /* Code LBRR indices and excitation signals */ + for( i = 0; i < psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket; i++ ) { + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + if( psEnc->state_Fxx[ n ].sCmn.LBRR_flags[ i ] ) { + opus_int condCoding; + + if( encControl->nChannelsInternal == 2 && n == 0 ) { + silk_stereo_encode_pred( psRangeEnc, psEnc->sStereo.predIx[ i ] ); + /* For LBRR data there's no need to code the mid-only flag if the side-channel LBRR flag is set */ + if( psEnc->state_Fxx[ 1 ].sCmn.LBRR_flags[ i ] == 0 ) { + silk_stereo_encode_mid_only( psRangeEnc, psEnc->sStereo.mid_only_flags[ i ] ); + } + } + /* Use conditional coding if previous frame available */ + if( i > 0 && psEnc->state_Fxx[ n ].sCmn.LBRR_flags[ i - 1 ] ) { + condCoding = CODE_CONDITIONALLY; + } else { + condCoding = CODE_INDEPENDENTLY; + } + silk_encode_indices( &psEnc->state_Fxx[ n ].sCmn, psRangeEnc, i, 1, condCoding ); + silk_encode_pulses( psRangeEnc, psEnc->state_Fxx[ n ].sCmn.indices_LBRR[i].signalType, psEnc->state_Fxx[ n ].sCmn.indices_LBRR[i].quantOffsetType, + psEnc->state_Fxx[ n ].sCmn.pulses_LBRR[ i ], psEnc->state_Fxx[ n ].sCmn.frame_length ); + } + } + } + + /* Reset LBRR flags */ + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + silk_memset( psEnc->state_Fxx[ n ].sCmn.LBRR_flags, 0, sizeof( psEnc->state_Fxx[ n ].sCmn.LBRR_flags ) ); + } + + psEnc->nBitsUsedLBRR = ec_tell( psRangeEnc ); + } + + silk_HP_variable_cutoff( psEnc->state_Fxx ); + + /* Total target bits for packet */ + nBits = silk_DIV32_16( silk_MUL( encControl->bitRate, encControl->payloadSize_ms ), 1000 ); + /* Subtract bits used for LBRR */ + if( !prefillFlag ) { + nBits -= psEnc->nBitsUsedLBRR; + } + /* Divide by number of uncoded frames left in packet */ + nBits = silk_DIV32_16( nBits, psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket ); + /* Convert to bits/second */ + if( encControl->payloadSize_ms == 10 ) { + TargetRate_bps = silk_SMULBB( nBits, 100 ); + } else { + TargetRate_bps = silk_SMULBB( nBits, 50 ); + } + /* Subtract fraction of bits in excess of target in previous frames and packets */ + TargetRate_bps -= silk_DIV32_16( silk_MUL( psEnc->nBitsExceeded, 1000 ), BITRESERVOIR_DECAY_TIME_MS ); + if( !prefillFlag && psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded > 0 ) { + /* Compare actual vs target bits so far in this packet */ + opus_int32 bitsBalance = ec_tell( psRangeEnc ) - psEnc->nBitsUsedLBRR - nBits * psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded; + TargetRate_bps -= silk_DIV32_16( silk_MUL( bitsBalance, 1000 ), BITRESERVOIR_DECAY_TIME_MS ); + } + /* Never exceed input bitrate */ + TargetRate_bps = silk_LIMIT( TargetRate_bps, encControl->bitRate, 5000 ); + + /* Convert Left/Right to Mid/Side */ + if( encControl->nChannelsInternal == 2 ) { + silk_stereo_LR_to_MS( &psEnc->sStereo, &psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ 2 ], &psEnc->state_Fxx[ 1 ].sCmn.inputBuf[ 2 ], + psEnc->sStereo.predIx[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ], &psEnc->sStereo.mid_only_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ], + MStargetRates_bps, TargetRate_bps, psEnc->state_Fxx[ 0 ].sCmn.speech_activity_Q8, encControl->toMono, + psEnc->state_Fxx[ 0 ].sCmn.fs_kHz, psEnc->state_Fxx[ 0 ].sCmn.frame_length ); + if( psEnc->sStereo.mid_only_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ] == 0 ) { + /* Reset side channel encoder memory for first frame with side coding */ + if( psEnc->prev_decode_only_middle == 1 ) { + silk_memset( &psEnc->state_Fxx[ 1 ].sShape, 0, sizeof( psEnc->state_Fxx[ 1 ].sShape ) ); + silk_memset( &psEnc->state_Fxx[ 1 ].sCmn.sNSQ, 0, sizeof( psEnc->state_Fxx[ 1 ].sCmn.sNSQ ) ); + silk_memset( psEnc->state_Fxx[ 1 ].sCmn.prev_NLSFq_Q15, 0, sizeof( psEnc->state_Fxx[ 1 ].sCmn.prev_NLSFq_Q15 ) ); + silk_memset( &psEnc->state_Fxx[ 1 ].sCmn.sLP.In_LP_State, 0, sizeof( psEnc->state_Fxx[ 1 ].sCmn.sLP.In_LP_State ) ); + psEnc->state_Fxx[ 1 ].sCmn.prevLag = 100; + psEnc->state_Fxx[ 1 ].sCmn.sNSQ.lagPrev = 100; + psEnc->state_Fxx[ 1 ].sShape.LastGainIndex = 10; + psEnc->state_Fxx[ 1 ].sCmn.prevSignalType = TYPE_NO_VOICE_ACTIVITY; + psEnc->state_Fxx[ 1 ].sCmn.sNSQ.prev_gain_Q16 = 65536; + psEnc->state_Fxx[ 1 ].sCmn.first_frame_after_reset = 1; + } + silk_encode_do_VAD_Fxx( &psEnc->state_Fxx[ 1 ], activity ); + } else { + psEnc->state_Fxx[ 1 ].sCmn.VAD_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ] = 0; + } + if( !prefillFlag ) { + silk_stereo_encode_pred( psRangeEnc, psEnc->sStereo.predIx[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ] ); + if( psEnc->state_Fxx[ 1 ].sCmn.VAD_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ] == 0 ) { + silk_stereo_encode_mid_only( psRangeEnc, psEnc->sStereo.mid_only_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded ] ); + } + } + } else { + /* Buffering */ + silk_memcpy( psEnc->state_Fxx[ 0 ].sCmn.inputBuf, psEnc->sStereo.sMid, 2 * sizeof( opus_int16 ) ); + silk_memcpy( psEnc->sStereo.sMid, &psEnc->state_Fxx[ 0 ].sCmn.inputBuf[ psEnc->state_Fxx[ 0 ].sCmn.frame_length ], 2 * sizeof( opus_int16 ) ); + } + silk_encode_do_VAD_Fxx( &psEnc->state_Fxx[ 0 ], activity ); + + /* Encode */ + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + opus_int maxBits, useCBR; + + /* Handling rate constraints */ + maxBits = encControl->maxBits; + if( tot_blocks == 2 && curr_block == 0 ) { + maxBits = maxBits * 3 / 5; + } else if( tot_blocks == 3 ) { + if( curr_block == 0 ) { + maxBits = maxBits * 2 / 5; + } else if( curr_block == 1 ) { + maxBits = maxBits * 3 / 4; + } + } + useCBR = encControl->useCBR && curr_block == tot_blocks - 1; + + if( encControl->nChannelsInternal == 1 ) { + channelRate_bps = TargetRate_bps; + } else { + channelRate_bps = MStargetRates_bps[ n ]; + if( n == 0 && MStargetRates_bps[ 1 ] > 0 ) { + useCBR = 0; + /* Give mid up to 1/2 of the max bits for that frame */ + maxBits -= encControl->maxBits / ( tot_blocks * 2 ); + } + } + + if( channelRate_bps > 0 ) { + opus_int condCoding; + + silk_control_SNR( &psEnc->state_Fxx[ n ].sCmn, channelRate_bps ); + + /* Use independent coding if no previous frame available */ + if( psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded - n <= 0 ) { + condCoding = CODE_INDEPENDENTLY; + } else if( n > 0 && psEnc->prev_decode_only_middle ) { + /* If we skipped a side frame in this packet, we don't + need LTP scaling; the LTP state is well-defined. */ + condCoding = CODE_INDEPENDENTLY_NO_LTP_SCALING; + } else { + condCoding = CODE_CONDITIONALLY; + } + if( ( ret = silk_encode_frame_Fxx( &psEnc->state_Fxx[ n ], nBytesOut, psRangeEnc, condCoding, maxBits, useCBR ) ) != 0 ) { + silk_assert( 0 ); + } + } + psEnc->state_Fxx[ n ].sCmn.controlled_since_last_payload = 0; + psEnc->state_Fxx[ n ].sCmn.inputBufIx = 0; + psEnc->state_Fxx[ n ].sCmn.nFramesEncoded++; + } + psEnc->prev_decode_only_middle = psEnc->sStereo.mid_only_flags[ psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded - 1 ]; + + /* Insert VAD and FEC flags at beginning of bitstream */ + if( *nBytesOut > 0 && psEnc->state_Fxx[ 0 ].sCmn.nFramesEncoded == psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket) { + flags = 0; + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + for( i = 0; i < psEnc->state_Fxx[ n ].sCmn.nFramesPerPacket; i++ ) { + flags = silk_LSHIFT( flags, 1 ); + flags |= psEnc->state_Fxx[ n ].sCmn.VAD_flags[ i ]; + } + flags = silk_LSHIFT( flags, 1 ); + flags |= psEnc->state_Fxx[ n ].sCmn.LBRR_flag; + } + if( !prefillFlag ) { + ec_enc_patch_initial_bits( psRangeEnc, flags, ( psEnc->state_Fxx[ 0 ].sCmn.nFramesPerPacket + 1 ) * encControl->nChannelsInternal ); + } + + /* Return zero bytes if all channels DTXed */ + if( psEnc->state_Fxx[ 0 ].sCmn.inDTX && ( encControl->nChannelsInternal == 1 || psEnc->state_Fxx[ 1 ].sCmn.inDTX ) ) { + *nBytesOut = 0; + } + + psEnc->nBitsExceeded += *nBytesOut * 8; + psEnc->nBitsExceeded -= silk_DIV32_16( silk_MUL( encControl->bitRate, encControl->payloadSize_ms ), 1000 ); + psEnc->nBitsExceeded = silk_LIMIT( psEnc->nBitsExceeded, 0, 10000 ); + + /* Update flag indicating if bandwidth switching is allowed */ + speech_act_thr_for_switch_Q8 = silk_SMLAWB( SILK_FIX_CONST( SPEECH_ACTIVITY_DTX_THRES, 8 ), + SILK_FIX_CONST( ( 1 - SPEECH_ACTIVITY_DTX_THRES ) / MAX_BANDWIDTH_SWITCH_DELAY_MS, 16 + 8 ), psEnc->timeSinceSwitchAllowed_ms ); + if( psEnc->state_Fxx[ 0 ].sCmn.speech_activity_Q8 < speech_act_thr_for_switch_Q8 ) { + psEnc->allowBandwidthSwitch = 1; + psEnc->timeSinceSwitchAllowed_ms = 0; + } else { + psEnc->allowBandwidthSwitch = 0; + psEnc->timeSinceSwitchAllowed_ms += encControl->payloadSize_ms; + } + } + + if( nSamplesIn == 0 ) { + break; + } + } else { + break; + } + curr_block++; + } + + psEnc->nPrevChannelsInternal = encControl->nChannelsInternal; + + encControl->allowBandwidthSwitch = psEnc->allowBandwidthSwitch; + encControl->inWBmodeWithoutVariableLP = psEnc->state_Fxx[ 0 ].sCmn.fs_kHz == 16 && psEnc->state_Fxx[ 0 ].sCmn.sLP.mode == 0; + encControl->internalSampleRate = silk_SMULBB( psEnc->state_Fxx[ 0 ].sCmn.fs_kHz, 1000 ); + encControl->stereoWidth_Q14 = encControl->toMono ? 0 : psEnc->sStereo.smth_width_Q14; + if( prefillFlag ) { + encControl->payloadSize_ms = tmp_payloadSize_ms; + encControl->complexity = tmp_complexity; + for( n = 0; n < encControl->nChannelsInternal; n++ ) { + psEnc->state_Fxx[ n ].sCmn.controlled_since_last_payload = 0; + psEnc->state_Fxx[ n ].sCmn.prefillFlag = 0; + } + } + + encControl->signalType = psEnc->state_Fxx[0].sCmn.indices.signalType; + encControl->offset = silk_Quantization_Offsets_Q10 + [ psEnc->state_Fxx[0].sCmn.indices.signalType >> 1 ] + [ psEnc->state_Fxx[0].sCmn.indices.quantOffsetType ]; + RESTORE_STACK; + return ret; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_indices.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_indices.c new file mode 100644 index 000000000..4bcbc3347 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_indices.c @@ -0,0 +1,181 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Encode side-information parameters to payload */ +void silk_encode_indices( + silk_encoder_state *psEncC, /* I/O Encoder state */ + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int FrameIndex, /* I Frame number */ + opus_int encode_LBRR, /* I Flag indicating LBRR data is being encoded */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int i, k, typeOffset; + opus_int encode_absolute_lagIndex, delta_lagIndex; + opus_int16 ec_ix[ MAX_LPC_ORDER ]; + opus_uint8 pred_Q8[ MAX_LPC_ORDER ]; + const SideInfoIndices *psIndices; + + if( encode_LBRR ) { + psIndices = &psEncC->indices_LBRR[ FrameIndex ]; + } else { + psIndices = &psEncC->indices; + } + + /*******************************************/ + /* Encode signal type and quantizer offset */ + /*******************************************/ + typeOffset = 2 * psIndices->signalType + psIndices->quantOffsetType; + celt_assert( typeOffset >= 0 && typeOffset < 6 ); + celt_assert( encode_LBRR == 0 || typeOffset >= 2 ); + if( encode_LBRR || typeOffset >= 2 ) { + ec_enc_icdf( psRangeEnc, typeOffset - 2, silk_type_offset_VAD_iCDF, 8 ); + } else { + ec_enc_icdf( psRangeEnc, typeOffset, silk_type_offset_no_VAD_iCDF, 8 ); + } + + /****************/ + /* Encode gains */ + /****************/ + /* first subframe */ + if( condCoding == CODE_CONDITIONALLY ) { + /* conditional coding */ + silk_assert( psIndices->GainsIndices[ 0 ] >= 0 && psIndices->GainsIndices[ 0 ] < MAX_DELTA_GAIN_QUANT - MIN_DELTA_GAIN_QUANT + 1 ); + ec_enc_icdf( psRangeEnc, psIndices->GainsIndices[ 0 ], silk_delta_gain_iCDF, 8 ); + } else { + /* independent coding, in two stages: MSB bits followed by 3 LSBs */ + silk_assert( psIndices->GainsIndices[ 0 ] >= 0 && psIndices->GainsIndices[ 0 ] < N_LEVELS_QGAIN ); + ec_enc_icdf( psRangeEnc, silk_RSHIFT( psIndices->GainsIndices[ 0 ], 3 ), silk_gain_iCDF[ psIndices->signalType ], 8 ); + ec_enc_icdf( psRangeEnc, psIndices->GainsIndices[ 0 ] & 7, silk_uniform8_iCDF, 8 ); + } + + /* remaining subframes */ + for( i = 1; i < psEncC->nb_subfr; i++ ) { + silk_assert( psIndices->GainsIndices[ i ] >= 0 && psIndices->GainsIndices[ i ] < MAX_DELTA_GAIN_QUANT - MIN_DELTA_GAIN_QUANT + 1 ); + ec_enc_icdf( psRangeEnc, psIndices->GainsIndices[ i ], silk_delta_gain_iCDF, 8 ); + } + + /****************/ + /* Encode NLSFs */ + /****************/ + ec_enc_icdf( psRangeEnc, psIndices->NLSFIndices[ 0 ], &psEncC->psNLSF_CB->CB1_iCDF[ ( psIndices->signalType >> 1 ) * psEncC->psNLSF_CB->nVectors ], 8 ); + silk_NLSF_unpack( ec_ix, pred_Q8, psEncC->psNLSF_CB, psIndices->NLSFIndices[ 0 ] ); + celt_assert( psEncC->psNLSF_CB->order == psEncC->predictLPCOrder ); + for( i = 0; i < psEncC->psNLSF_CB->order; i++ ) { + if( psIndices->NLSFIndices[ i+1 ] >= NLSF_QUANT_MAX_AMPLITUDE ) { + ec_enc_icdf( psRangeEnc, 2 * NLSF_QUANT_MAX_AMPLITUDE, &psEncC->psNLSF_CB->ec_iCDF[ ec_ix[ i ] ], 8 ); + ec_enc_icdf( psRangeEnc, psIndices->NLSFIndices[ i+1 ] - NLSF_QUANT_MAX_AMPLITUDE, silk_NLSF_EXT_iCDF, 8 ); + } else if( psIndices->NLSFIndices[ i+1 ] <= -NLSF_QUANT_MAX_AMPLITUDE ) { + ec_enc_icdf( psRangeEnc, 0, &psEncC->psNLSF_CB->ec_iCDF[ ec_ix[ i ] ], 8 ); + ec_enc_icdf( psRangeEnc, -psIndices->NLSFIndices[ i+1 ] - NLSF_QUANT_MAX_AMPLITUDE, silk_NLSF_EXT_iCDF, 8 ); + } else { + ec_enc_icdf( psRangeEnc, psIndices->NLSFIndices[ i+1 ] + NLSF_QUANT_MAX_AMPLITUDE, &psEncC->psNLSF_CB->ec_iCDF[ ec_ix[ i ] ], 8 ); + } + } + + /* Encode NLSF interpolation factor */ + if( psEncC->nb_subfr == MAX_NB_SUBFR ) { + silk_assert( psIndices->NLSFInterpCoef_Q2 >= 0 && psIndices->NLSFInterpCoef_Q2 < 5 ); + ec_enc_icdf( psRangeEnc, psIndices->NLSFInterpCoef_Q2, silk_NLSF_interpolation_factor_iCDF, 8 ); + } + + if( psIndices->signalType == TYPE_VOICED ) + { + /*********************/ + /* Encode pitch lags */ + /*********************/ + /* lag index */ + encode_absolute_lagIndex = 1; + if( condCoding == CODE_CONDITIONALLY && psEncC->ec_prevSignalType == TYPE_VOICED ) { + /* Delta Encoding */ + delta_lagIndex = psIndices->lagIndex - psEncC->ec_prevLagIndex; + if( delta_lagIndex < -8 || delta_lagIndex > 11 ) { + delta_lagIndex = 0; + } else { + delta_lagIndex = delta_lagIndex + 9; + encode_absolute_lagIndex = 0; /* Only use delta */ + } + silk_assert( delta_lagIndex >= 0 && delta_lagIndex < 21 ); + ec_enc_icdf( psRangeEnc, delta_lagIndex, silk_pitch_delta_iCDF, 8 ); + } + if( encode_absolute_lagIndex ) { + /* Absolute encoding */ + opus_int32 pitch_high_bits, pitch_low_bits; + pitch_high_bits = silk_DIV32_16( psIndices->lagIndex, silk_RSHIFT( psEncC->fs_kHz, 1 ) ); + pitch_low_bits = psIndices->lagIndex - silk_SMULBB( pitch_high_bits, silk_RSHIFT( psEncC->fs_kHz, 1 ) ); + silk_assert( pitch_low_bits < psEncC->fs_kHz / 2 ); + silk_assert( pitch_high_bits < 32 ); + ec_enc_icdf( psRangeEnc, pitch_high_bits, silk_pitch_lag_iCDF, 8 ); + ec_enc_icdf( psRangeEnc, pitch_low_bits, psEncC->pitch_lag_low_bits_iCDF, 8 ); + } + psEncC->ec_prevLagIndex = psIndices->lagIndex; + + /* Countour index */ + silk_assert( psIndices->contourIndex >= 0 ); + silk_assert( ( psIndices->contourIndex < 34 && psEncC->fs_kHz > 8 && psEncC->nb_subfr == 4 ) || + ( psIndices->contourIndex < 11 && psEncC->fs_kHz == 8 && psEncC->nb_subfr == 4 ) || + ( psIndices->contourIndex < 12 && psEncC->fs_kHz > 8 && psEncC->nb_subfr == 2 ) || + ( psIndices->contourIndex < 3 && psEncC->fs_kHz == 8 && psEncC->nb_subfr == 2 ) ); + ec_enc_icdf( psRangeEnc, psIndices->contourIndex, psEncC->pitch_contour_iCDF, 8 ); + + /********************/ + /* Encode LTP gains */ + /********************/ + /* PERIndex value */ + silk_assert( psIndices->PERIndex >= 0 && psIndices->PERIndex < 3 ); + ec_enc_icdf( psRangeEnc, psIndices->PERIndex, silk_LTP_per_index_iCDF, 8 ); + + /* Codebook Indices */ + for( k = 0; k < psEncC->nb_subfr; k++ ) { + silk_assert( psIndices->LTPIndex[ k ] >= 0 && psIndices->LTPIndex[ k ] < ( 8 << psIndices->PERIndex ) ); + ec_enc_icdf( psRangeEnc, psIndices->LTPIndex[ k ], silk_LTP_gain_iCDF_ptrs[ psIndices->PERIndex ], 8 ); + } + + /**********************/ + /* Encode LTP scaling */ + /**********************/ + if( condCoding == CODE_INDEPENDENTLY ) { + silk_assert( psIndices->LTP_scaleIndex >= 0 && psIndices->LTP_scaleIndex < 3 ); + ec_enc_icdf( psRangeEnc, psIndices->LTP_scaleIndex, silk_LTPscale_iCDF, 8 ); + } + silk_assert( !condCoding || psIndices->LTP_scaleIndex == 0 ); + } + + psEncC->ec_prevSignalType = psIndices->signalType; + + /***************/ + /* Encode seed */ + /***************/ + silk_assert( psIndices->Seed >= 0 && psIndices->Seed < 4 ); + ec_enc_icdf( psRangeEnc, psIndices->Seed, silk_uniform4_iCDF, 8 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_pulses.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_pulses.c new file mode 100644 index 000000000..8a1999138 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/encode_pulses.c @@ -0,0 +1,206 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/*********************************************/ +/* Encode quantization indices of excitation */ +/*********************************************/ + +static OPUS_INLINE opus_int combine_and_check( /* return ok */ + opus_int *pulses_comb, /* O */ + const opus_int *pulses_in, /* I */ + opus_int max_pulses, /* I max value for sum of pulses */ + opus_int len /* I number of output values */ +) +{ + opus_int k, sum; + + for( k = 0; k < len; k++ ) { + sum = pulses_in[ 2 * k ] + pulses_in[ 2 * k + 1 ]; + if( sum > max_pulses ) { + return 1; + } + pulses_comb[ k ] = sum; + } + + return 0; +} + +/* Encode quantization indices of excitation */ +void silk_encode_pulses( + ec_enc *psRangeEnc, /* I/O compressor data structure */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I quantOffsetType */ + opus_int8 pulses[], /* I quantization indices */ + const opus_int frame_length /* I Frame length */ +) +{ + opus_int i, k, j, iter, bit, nLS, scale_down, RateLevelIndex = 0; + opus_int32 abs_q, minSumBits_Q5, sumBits_Q5; + VARDECL( opus_int, abs_pulses ); + VARDECL( opus_int, sum_pulses ); + VARDECL( opus_int, nRshifts ); + opus_int pulses_comb[ 8 ]; + opus_int *abs_pulses_ptr; + const opus_int8 *pulses_ptr; + const opus_uint8 *cdf_ptr; + const opus_uint8 *nBits_ptr; + SAVE_STACK; + + silk_memset( pulses_comb, 0, 8 * sizeof( opus_int ) ); /* Fixing Valgrind reported problem*/ + + /****************************/ + /* Prepare for shell coding */ + /****************************/ + /* Calculate number of shell blocks */ + silk_assert( 1 << LOG2_SHELL_CODEC_FRAME_LENGTH == SHELL_CODEC_FRAME_LENGTH ); + iter = silk_RSHIFT( frame_length, LOG2_SHELL_CODEC_FRAME_LENGTH ); + if( iter * SHELL_CODEC_FRAME_LENGTH < frame_length ) { + celt_assert( frame_length == 12 * 10 ); /* Make sure only happens for 10 ms @ 12 kHz */ + iter++; + silk_memset( &pulses[ frame_length ], 0, SHELL_CODEC_FRAME_LENGTH * sizeof(opus_int8)); + } + + /* Take the absolute value of the pulses */ + ALLOC( abs_pulses, iter * SHELL_CODEC_FRAME_LENGTH, opus_int ); + silk_assert( !( SHELL_CODEC_FRAME_LENGTH & 3 ) ); + for( i = 0; i < iter * SHELL_CODEC_FRAME_LENGTH; i+=4 ) { + abs_pulses[i+0] = ( opus_int )silk_abs( pulses[ i + 0 ] ); + abs_pulses[i+1] = ( opus_int )silk_abs( pulses[ i + 1 ] ); + abs_pulses[i+2] = ( opus_int )silk_abs( pulses[ i + 2 ] ); + abs_pulses[i+3] = ( opus_int )silk_abs( pulses[ i + 3 ] ); + } + + /* Calc sum pulses per shell code frame */ + ALLOC( sum_pulses, iter, opus_int ); + ALLOC( nRshifts, iter, opus_int ); + abs_pulses_ptr = abs_pulses; + for( i = 0; i < iter; i++ ) { + nRshifts[ i ] = 0; + + while( 1 ) { + /* 1+1 -> 2 */ + scale_down = combine_and_check( pulses_comb, abs_pulses_ptr, silk_max_pulses_table[ 0 ], 8 ); + /* 2+2 -> 4 */ + scale_down += combine_and_check( pulses_comb, pulses_comb, silk_max_pulses_table[ 1 ], 4 ); + /* 4+4 -> 8 */ + scale_down += combine_and_check( pulses_comb, pulses_comb, silk_max_pulses_table[ 2 ], 2 ); + /* 8+8 -> 16 */ + scale_down += combine_and_check( &sum_pulses[ i ], pulses_comb, silk_max_pulses_table[ 3 ], 1 ); + + if( scale_down ) { + /* We need to downscale the quantization signal */ + nRshifts[ i ]++; + for( k = 0; k < SHELL_CODEC_FRAME_LENGTH; k++ ) { + abs_pulses_ptr[ k ] = silk_RSHIFT( abs_pulses_ptr[ k ], 1 ); + } + } else { + /* Jump out of while(1) loop and go to next shell coding frame */ + break; + } + } + abs_pulses_ptr += SHELL_CODEC_FRAME_LENGTH; + } + + /**************/ + /* Rate level */ + /**************/ + /* find rate level that leads to fewest bits for coding of pulses per block info */ + minSumBits_Q5 = silk_int32_MAX; + for( k = 0; k < N_RATE_LEVELS - 1; k++ ) { + nBits_ptr = silk_pulses_per_block_BITS_Q5[ k ]; + sumBits_Q5 = silk_rate_levels_BITS_Q5[ signalType >> 1 ][ k ]; + for( i = 0; i < iter; i++ ) { + if( nRshifts[ i ] > 0 ) { + sumBits_Q5 += nBits_ptr[ SILK_MAX_PULSES + 1 ]; + } else { + sumBits_Q5 += nBits_ptr[ sum_pulses[ i ] ]; + } + } + if( sumBits_Q5 < minSumBits_Q5 ) { + minSumBits_Q5 = sumBits_Q5; + RateLevelIndex = k; + } + } + ec_enc_icdf( psRangeEnc, RateLevelIndex, silk_rate_levels_iCDF[ signalType >> 1 ], 8 ); + + /***************************************************/ + /* Sum-Weighted-Pulses Encoding */ + /***************************************************/ + cdf_ptr = silk_pulses_per_block_iCDF[ RateLevelIndex ]; + for( i = 0; i < iter; i++ ) { + if( nRshifts[ i ] == 0 ) { + ec_enc_icdf( psRangeEnc, sum_pulses[ i ], cdf_ptr, 8 ); + } else { + ec_enc_icdf( psRangeEnc, SILK_MAX_PULSES + 1, cdf_ptr, 8 ); + for( k = 0; k < nRshifts[ i ] - 1; k++ ) { + ec_enc_icdf( psRangeEnc, SILK_MAX_PULSES + 1, silk_pulses_per_block_iCDF[ N_RATE_LEVELS - 1 ], 8 ); + } + ec_enc_icdf( psRangeEnc, sum_pulses[ i ], silk_pulses_per_block_iCDF[ N_RATE_LEVELS - 1 ], 8 ); + } + } + + /******************/ + /* Shell Encoding */ + /******************/ + for( i = 0; i < iter; i++ ) { + if( sum_pulses[ i ] > 0 ) { + silk_shell_encoder( psRangeEnc, &abs_pulses[ i * SHELL_CODEC_FRAME_LENGTH ] ); + } + } + + /****************/ + /* LSB Encoding */ + /****************/ + for( i = 0; i < iter; i++ ) { + if( nRshifts[ i ] > 0 ) { + pulses_ptr = &pulses[ i * SHELL_CODEC_FRAME_LENGTH ]; + nLS = nRshifts[ i ] - 1; + for( k = 0; k < SHELL_CODEC_FRAME_LENGTH; k++ ) { + abs_q = (opus_int8)silk_abs( pulses_ptr[ k ] ); + for( j = nLS; j > 0; j-- ) { + bit = silk_RSHIFT( abs_q, j ) & 1; + ec_enc_icdf( psRangeEnc, bit, silk_lsb_iCDF, 8 ); + } + bit = abs_q & 1; + ec_enc_icdf( psRangeEnc, bit, silk_lsb_iCDF, 8 ); + } + } + } + + /****************/ + /* Encode signs */ + /****************/ + silk_encode_signs( psRangeEnc, pulses, frame_length, signalType, quantOffsetType, sum_pulses ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/errors.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/errors.h new file mode 100644 index 000000000..45070800f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/errors.h @@ -0,0 +1,98 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_ERRORS_H +#define SILK_ERRORS_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/******************/ +/* Error messages */ +/******************/ +#define SILK_NO_ERROR 0 + +/**************************/ +/* Encoder error messages */ +/**************************/ + +/* Input length is not a multiple of 10 ms, or length is longer than the packet length */ +#define SILK_ENC_INPUT_INVALID_NO_OF_SAMPLES -101 + +/* Sampling frequency not 8000, 12000 or 16000 Hertz */ +#define SILK_ENC_FS_NOT_SUPPORTED -102 + +/* Packet size not 10, 20, 40, or 60 ms */ +#define SILK_ENC_PACKET_SIZE_NOT_SUPPORTED -103 + +/* Allocated payload buffer too short */ +#define SILK_ENC_PAYLOAD_BUF_TOO_SHORT -104 + +/* Loss rate not between 0 and 100 percent */ +#define SILK_ENC_INVALID_LOSS_RATE -105 + +/* Complexity setting not valid, use 0...10 */ +#define SILK_ENC_INVALID_COMPLEXITY_SETTING -106 + +/* Inband FEC setting not valid, use 0 or 1 */ +#define SILK_ENC_INVALID_INBAND_FEC_SETTING -107 + +/* DTX setting not valid, use 0 or 1 */ +#define SILK_ENC_INVALID_DTX_SETTING -108 + +/* CBR setting not valid, use 0 or 1 */ +#define SILK_ENC_INVALID_CBR_SETTING -109 + +/* Internal encoder error */ +#define SILK_ENC_INTERNAL_ERROR -110 + +/* Internal encoder error */ +#define SILK_ENC_INVALID_NUMBER_OF_CHANNELS_ERROR -111 + +/**************************/ +/* Decoder error messages */ +/**************************/ + +/* Output sampling frequency lower than internal decoded sampling frequency */ +#define SILK_DEC_INVALID_SAMPLING_FREQUENCY -200 + +/* Payload size exceeded the maximum allowed 1024 bytes */ +#define SILK_DEC_PAYLOAD_TOO_LARGE -201 + +/* Payload has bit errors */ +#define SILK_DEC_PAYLOAD_ERROR -202 + +/* Payload has bit errors */ +#define SILK_DEC_INVALID_FRAME_SIZE -203 + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_analysis_filter_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_analysis_filter_FIX.c new file mode 100644 index 000000000..5574e7069 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_analysis_filter_FIX.c @@ -0,0 +1,90 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +void silk_LTP_analysis_filter_FIX( + opus_int16 *LTP_res, /* O LTP residual signal of length MAX_NB_SUBFR * ( pre_length + subfr_length ) */ + const opus_int16 *x, /* I Pointer to input signal with at least max( pitchL ) preceding samples */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ],/* I LTP_ORDER LTP coefficients for each MAX_NB_SUBFR subframe */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag, one for each subframe */ + const opus_int32 invGains_Q16[ MAX_NB_SUBFR ], /* I Inverse quantization gains, one for each subframe */ + const opus_int subfr_length, /* I Length of each subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + const opus_int pre_length /* I Length of the preceding samples starting at &x[0] for each subframe */ +) +{ + const opus_int16 *x_ptr, *x_lag_ptr; + opus_int16 Btmp_Q14[ LTP_ORDER ]; + opus_int16 *LTP_res_ptr; + opus_int k, i; + opus_int32 LTP_est; + + x_ptr = x; + LTP_res_ptr = LTP_res; + for( k = 0; k < nb_subfr; k++ ) { + + x_lag_ptr = x_ptr - pitchL[ k ]; + + Btmp_Q14[ 0 ] = LTPCoef_Q14[ k * LTP_ORDER ]; + Btmp_Q14[ 1 ] = LTPCoef_Q14[ k * LTP_ORDER + 1 ]; + Btmp_Q14[ 2 ] = LTPCoef_Q14[ k * LTP_ORDER + 2 ]; + Btmp_Q14[ 3 ] = LTPCoef_Q14[ k * LTP_ORDER + 3 ]; + Btmp_Q14[ 4 ] = LTPCoef_Q14[ k * LTP_ORDER + 4 ]; + + /* LTP analysis FIR filter */ + for( i = 0; i < subfr_length + pre_length; i++ ) { + LTP_res_ptr[ i ] = x_ptr[ i ]; + + /* Long-term prediction */ + LTP_est = silk_SMULBB( x_lag_ptr[ LTP_ORDER / 2 ], Btmp_Q14[ 0 ] ); + LTP_est = silk_SMLABB_ovflw( LTP_est, x_lag_ptr[ 1 ], Btmp_Q14[ 1 ] ); + LTP_est = silk_SMLABB_ovflw( LTP_est, x_lag_ptr[ 0 ], Btmp_Q14[ 2 ] ); + LTP_est = silk_SMLABB_ovflw( LTP_est, x_lag_ptr[ -1 ], Btmp_Q14[ 3 ] ); + LTP_est = silk_SMLABB_ovflw( LTP_est, x_lag_ptr[ -2 ], Btmp_Q14[ 4 ] ); + + LTP_est = silk_RSHIFT_ROUND( LTP_est, 14 ); /* round and -> Q0*/ + + /* Subtract long-term prediction */ + LTP_res_ptr[ i ] = (opus_int16)silk_SAT16( (opus_int32)x_ptr[ i ] - LTP_est ); + + /* Scale residual */ + LTP_res_ptr[ i ] = silk_SMULWB( invGains_Q16[ k ], LTP_res_ptr[ i ] ); + + x_lag_ptr++; + } + + /* Update pointers */ + LTP_res_ptr += subfr_length + pre_length; + x_ptr += subfr_length; + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_scale_ctrl_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_scale_ctrl_FIX.c new file mode 100644 index 000000000..3dcedef89 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/LTP_scale_ctrl_FIX.c @@ -0,0 +1,53 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +/* Calculation of LTP state scaling */ +void silk_LTP_scale_ctrl_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int round_loss; + + if( condCoding == CODE_INDEPENDENTLY ) { + /* Only scale if first frame in packet */ + round_loss = psEnc->sCmn.PacketLoss_perc + psEnc->sCmn.nFramesPerPacket; + psEnc->sCmn.indices.LTP_scaleIndex = (opus_int8)silk_LIMIT( + silk_SMULWB( silk_SMULBB( round_loss, psEncCtrl->LTPredCodGain_Q7 ), SILK_FIX_CONST( 0.1, 9 ) ), 0, 2 ); + } else { + /* Default is minimum scaling */ + psEnc->sCmn.indices.LTP_scaleIndex = 0; + } + psEncCtrl->LTP_scale_Q14 = silk_LTPScales_table_Q14[ psEnc->sCmn.indices.LTP_scaleIndex ]; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/apply_sine_window_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/apply_sine_window_FIX.c new file mode 100644 index 000000000..03e088a6d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/apply_sine_window_FIX.c @@ -0,0 +1,101 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Apply sine window to signal vector. */ +/* Window types: */ +/* 1 -> sine window from 0 to pi/2 */ +/* 2 -> sine window from pi/2 to pi */ +/* Every other sample is linearly interpolated, for speed. */ +/* Window length must be between 16 and 120 (incl) and a multiple of 4. */ + +/* Matlab code for table: + for k=16:9*4:16+2*9*4, fprintf(' %7.d,', -round(65536*pi ./ (k:4:k+8*4))); fprintf('\n'); end +*/ +static const opus_int16 freq_table_Q16[ 27 ] = { + 12111, 9804, 8235, 7100, 6239, 5565, 5022, 4575, 4202, + 3885, 3612, 3375, 3167, 2984, 2820, 2674, 2542, 2422, + 2313, 2214, 2123, 2038, 1961, 1889, 1822, 1760, 1702, +}; + +void silk_apply_sine_window( + opus_int16 px_win[], /* O Pointer to windowed signal */ + const opus_int16 px[], /* I Pointer to input signal */ + const opus_int win_type, /* I Selects a window type */ + const opus_int length /* I Window length, multiple of 4 */ +) +{ + opus_int k, f_Q16, c_Q16; + opus_int32 S0_Q16, S1_Q16; + + celt_assert( win_type == 1 || win_type == 2 ); + + /* Length must be in a range from 16 to 120 and a multiple of 4 */ + celt_assert( length >= 16 && length <= 120 ); + celt_assert( ( length & 3 ) == 0 ); + + /* Frequency */ + k = ( length >> 2 ) - 4; + celt_assert( k >= 0 && k <= 26 ); + f_Q16 = (opus_int)freq_table_Q16[ k ]; + + /* Factor used for cosine approximation */ + c_Q16 = silk_SMULWB( (opus_int32)f_Q16, -f_Q16 ); + silk_assert( c_Q16 >= -32768 ); + + /* initialize state */ + if( win_type == 1 ) { + /* start from 0 */ + S0_Q16 = 0; + /* approximation of sin(f) */ + S1_Q16 = f_Q16 + silk_RSHIFT( length, 3 ); + } else { + /* start from 1 */ + S0_Q16 = ( (opus_int32)1 << 16 ); + /* approximation of cos(f) */ + S1_Q16 = ( (opus_int32)1 << 16 ) + silk_RSHIFT( c_Q16, 1 ) + silk_RSHIFT( length, 4 ); + } + + /* Uses the recursive equation: sin(n*f) = 2 * cos(f) * sin((n-1)*f) - sin((n-2)*f) */ + /* 4 samples at a time */ + for( k = 0; k < length; k += 4 ) { + px_win[ k ] = (opus_int16)silk_SMULWB( silk_RSHIFT( S0_Q16 + S1_Q16, 1 ), px[ k ] ); + px_win[ k + 1 ] = (opus_int16)silk_SMULWB( S1_Q16, px[ k + 1] ); + S0_Q16 = silk_SMULWB( S1_Q16, c_Q16 ) + silk_LSHIFT( S1_Q16, 1 ) - S0_Q16 + 1; + S0_Q16 = silk_min( S0_Q16, ( (opus_int32)1 << 16 ) ); + + px_win[ k + 2 ] = (opus_int16)silk_SMULWB( silk_RSHIFT( S0_Q16 + S1_Q16, 1 ), px[ k + 2] ); + px_win[ k + 3 ] = (opus_int16)silk_SMULWB( S0_Q16, px[ k + 3 ] ); + S1_Q16 = silk_SMULWB( S0_Q16, c_Q16 ) + silk_LSHIFT( S0_Q16, 1 ) - S1_Q16; + S1_Q16 = silk_min( S1_Q16, ( (opus_int32)1 << 16 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_arm.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_arm.h new file mode 100644 index 000000000..1992e4328 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_arm.h @@ -0,0 +1,68 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_WARPED_AUTOCORRELATION_FIX_ARM_H +# define SILK_WARPED_AUTOCORRELATION_FIX_ARM_H + +# include "celt/arm/armcpu.h" + +# if defined(FIXED_POINT) + +# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) +void silk_warped_autocorrelation_FIX_neon( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +); + +# if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) +# define OVERRIDE_silk_warped_autocorrelation_FIX (1) +# define silk_warped_autocorrelation_FIX(corr, scale, input, warping_Q16, length, order, arch) \ + ((void)(arch), PRESUME_NEON(silk_warped_autocorrelation_FIX)(corr, scale, input, warping_Q16, length, order)) +# endif +# endif + +# if !defined(OVERRIDE_silk_warped_autocorrelation_FIX) +/*Is run-time CPU detection enabled on this platform?*/ +# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) +extern void (*const SILK_WARPED_AUTOCORRELATION_FIX_IMPL[OPUS_ARCHMASK+1])(opus_int32*, opus_int*, const opus_int16*, const opus_int, const opus_int, const opus_int); +# define OVERRIDE_silk_warped_autocorrelation_FIX (1) +# define silk_warped_autocorrelation_FIX(corr, scale, input, warping_Q16, length, order, arch) \ + ((*SILK_WARPED_AUTOCORRELATION_FIX_IMPL[(arch)&OPUS_ARCHMASK])(corr, scale, input, warping_Q16, length, order)) +# elif defined(OPUS_ARM_PRESUME_NEON_INTR) +# define OVERRIDE_silk_warped_autocorrelation_FIX (1) +# define silk_warped_autocorrelation_FIX(corr, scale, input, warping_Q16, length, order, arch) \ + ((void)(arch), silk_warped_autocorrelation_FIX_neon(corr, scale, input, warping_Q16, length, order)) +# endif +# endif + +# endif /* end FIXED_POINT */ + +#endif /* end SILK_WARPED_AUTOCORRELATION_FIX_ARM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c new file mode 100644 index 000000000..6f3be025c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c @@ -0,0 +1,265 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc., Jean-Marc Valin +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#ifdef OPUS_CHECK_ASM +# include +#endif +#include "stack_alloc.h" +#include "main_FIX.h" + +static OPUS_INLINE void calc_corr( const opus_int32 *const input_QS, opus_int64 *const corr_QC, const opus_int offset, const int32x4_t state_QS_s32x4 ) +{ + int64x2_t corr_QC_s64x2[ 2 ], t_s64x2[ 2 ]; + const int32x4_t input_QS_s32x4 = vld1q_s32( input_QS + offset ); + corr_QC_s64x2[ 0 ] = vld1q_s64( corr_QC + offset + 0 ); + corr_QC_s64x2[ 1 ] = vld1q_s64( corr_QC + offset + 2 ); + t_s64x2[ 0 ] = vmull_s32( vget_low_s32( state_QS_s32x4 ), vget_low_s32( input_QS_s32x4 ) ); + t_s64x2[ 1 ] = vmull_s32( vget_high_s32( state_QS_s32x4 ), vget_high_s32( input_QS_s32x4 ) ); + corr_QC_s64x2[ 0 ] = vsraq_n_s64( corr_QC_s64x2[ 0 ], t_s64x2[ 0 ], 2 * QS - QC ); + corr_QC_s64x2[ 1 ] = vsraq_n_s64( corr_QC_s64x2[ 1 ], t_s64x2[ 1 ], 2 * QS - QC ); + vst1q_s64( corr_QC + offset + 0, corr_QC_s64x2[ 0 ] ); + vst1q_s64( corr_QC + offset + 2, corr_QC_s64x2[ 1 ] ); +} + +static OPUS_INLINE int32x4_t calc_state( const int32x4_t state_QS0_s32x4, const int32x4_t state_QS0_1_s32x4, const int32x4_t state_QS1_1_s32x4, const int32x4_t warping_Q16_s32x4 ) +{ + int32x4_t t_s32x4 = vsubq_s32( state_QS0_s32x4, state_QS0_1_s32x4 ); + t_s32x4 = vqdmulhq_s32( t_s32x4, warping_Q16_s32x4 ); + return vaddq_s32( state_QS1_1_s32x4, t_s32x4 ); +} + +void silk_warped_autocorrelation_FIX_neon( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +) +{ + if( ( MAX_SHAPE_LPC_ORDER > 24 ) || ( order < 6 ) ) { + silk_warped_autocorrelation_FIX_c( corr, scale, input, warping_Q16, length, order ); + } else { + opus_int n, i, lsh; + opus_int64 corr_QC[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; /* In reverse order */ + opus_int64 corr_QC_orderT; + int64x2_t lsh_s64x2; + const opus_int orderT = ( order + 3 ) & ~3; + opus_int64 *corr_QCT; + opus_int32 *input_QS; + VARDECL( opus_int32, input_QST ); + VARDECL( opus_int32, state ); + SAVE_STACK; + + /* Order must be even */ + silk_assert( ( order & 1 ) == 0 ); + silk_assert( 2 * QS - QC >= 0 ); + + /* The additional +4 is to ensure a later vld1q_s32 call does not overflow. */ + /* Strictly, only +3 is needed but +4 simplifies initialization using the 4x32 neon load. */ + ALLOC( input_QST, length + 2 * MAX_SHAPE_LPC_ORDER + 4, opus_int32 ); + + input_QS = input_QST; + /* input_QS has zero paddings in the beginning and end. */ + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + + /* Loop over samples */ + for( n = 0; n < length - 7; n += 8, input_QS += 8 ) { + const int16x8_t t0_s16x4 = vld1q_s16( input + n ); + vst1q_s32( input_QS + 0, vshll_n_s16( vget_low_s16( t0_s16x4 ), QS ) ); + vst1q_s32( input_QS + 4, vshll_n_s16( vget_high_s16( t0_s16x4 ), QS ) ); + } + for( ; n < length; n++, input_QS++ ) { + input_QS[ 0 ] = silk_LSHIFT32( (opus_int32)input[ n ], QS ); + } + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS += 4; + vst1q_s32( input_QS, vdupq_n_s32( 0 ) ); + input_QS = input_QST + MAX_SHAPE_LPC_ORDER - orderT; + + /* The following loop runs ( length + order ) times, with ( order ) extra epilogues. */ + /* The zero paddings in input_QS guarantee corr_QC's correctness even with the extra epilogues. */ + /* The values of state_QS will be polluted by the extra epilogues, however they are temporary values. */ + + /* Keep the C code here to help understand the intrinsics optimization. */ + /* + { + opus_int32 state_QS[ 2 ][ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + opus_int32 *state_QST[ 3 ]; + state_QST[ 0 ] = state_QS[ 0 ]; + state_QST[ 1 ] = state_QS[ 1 ]; + for( n = 0; n < length + order; n++, input_QS++ ) { + state_QST[ 0 ][ orderT ] = input_QS[ orderT ]; + for( i = 0; i < orderT; i++ ) { + corr_QC[ i ] += silk_RSHIFT64( silk_SMULL( state_QST[ 0 ][ i ], input_QS[ i ] ), 2 * QS - QC ); + state_QST[ 1 ][ i ] = silk_SMLAWB( state_QST[ 1 ][ i + 1 ], state_QST[ 0 ][ i ] - state_QST[ 0 ][ i + 1 ], warping_Q16 ); + } + state_QST[ 2 ] = state_QST[ 0 ]; + state_QST[ 0 ] = state_QST[ 1 ]; + state_QST[ 1 ] = state_QST[ 2 ]; + } + } + */ + + { + const int32x4_t warping_Q16_s32x4 = vdupq_n_s32( warping_Q16 << 15 ); + const opus_int32 *in = input_QS + orderT; + opus_int o = orderT; + int32x4_t state_QS_s32x4[ 3 ][ 2 ]; + + /* The additional +4 is to ensure a later vld1q_s32 call does not overflow. */ + ALLOC( state, length + order + 4, opus_int32 ); + state_QS_s32x4[ 2 ][ 1 ] = vdupq_n_s32( 0 ); + + /* Calculate 8 taps of all inputs in each loop. */ + do { + state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 0 ][ 1 ] = + state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 1 ][ 1 ] = vdupq_n_s32( 0 ); + n = 0; + do { + calc_corr( input_QS + n, corr_QC, o - 8, state_QS_s32x4[ 0 ][ 0 ] ); + calc_corr( input_QS + n, corr_QC, o - 4, state_QS_s32x4[ 0 ][ 1 ] ); + state_QS_s32x4[ 2 ][ 1 ] = vld1q_s32( in + n ); + vst1q_lane_s32( state + n, state_QS_s32x4[ 0 ][ 0 ], 0 ); + state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 0 ][ 1 ], 1 ); + state_QS_s32x4[ 2 ][ 1 ] = vextq_s32( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], 1 ); + state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 ); + state_QS_s32x4[ 0 ][ 1 ] = calc_state( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], state_QS_s32x4[ 1 ][ 1 ], warping_Q16_s32x4 ); + state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ]; + state_QS_s32x4[ 1 ][ 1 ] = state_QS_s32x4[ 2 ][ 1 ]; + } while( ++n < ( length + order ) ); + in = state; + o -= 8; + } while( o > 4 ); + + if( o ) { + /* Calculate the last 4 taps of all inputs. */ + opus_int32 *stateT = state; + silk_assert( o == 4 ); + state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 1 ][ 0 ] = vdupq_n_s32( 0 ); + n = length + order; + do { + calc_corr( input_QS, corr_QC, 0, state_QS_s32x4[ 0 ][ 0 ] ); + state_QS_s32x4[ 2 ][ 0 ] = vld1q_s32( stateT ); + vst1q_lane_s32( stateT, state_QS_s32x4[ 0 ][ 0 ], 0 ); + state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], 1 ); + state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 ); + state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ]; + input_QS++; + stateT++; + } while( --n ); + } + } + + { + const opus_int16 *inputT = input; + int32x4_t t_s32x4; + int64x1_t t_s64x1; + int64x2_t t_s64x2 = vdupq_n_s64( 0 ); + for( n = 0; n <= length - 8; n += 8 ) { + int16x8_t input_s16x8 = vld1q_s16( inputT ); + t_s32x4 = vmull_s16( vget_low_s16( input_s16x8 ), vget_low_s16( input_s16x8 ) ); + t_s32x4 = vmlal_s16( t_s32x4, vget_high_s16( input_s16x8 ), vget_high_s16( input_s16x8 ) ); + t_s64x2 = vaddw_s32( t_s64x2, vget_low_s32( t_s32x4 ) ); + t_s64x2 = vaddw_s32( t_s64x2, vget_high_s32( t_s32x4 ) ); + inputT += 8; + } + t_s64x1 = vadd_s64( vget_low_s64( t_s64x2 ), vget_high_s64( t_s64x2 ) ); + corr_QC_orderT = vget_lane_s64( t_s64x1, 0 ); + for( ; n < length; n++ ) { + corr_QC_orderT += silk_SMULL( input[ n ], input[ n ] ); + } + corr_QC_orderT = silk_LSHIFT64( corr_QC_orderT, QC ); + corr_QC[ orderT ] = corr_QC_orderT; + } + + corr_QCT = corr_QC + orderT - order; + lsh = silk_CLZ64( corr_QC_orderT ) - 35; + lsh = silk_LIMIT( lsh, -12 - QC, 30 - QC ); + *scale = -( QC + lsh ); + silk_assert( *scale >= -30 && *scale <= 12 ); + lsh_s64x2 = vdupq_n_s64( lsh ); + for( i = 0; i <= order - 3; i += 4 ) { + int32x4_t corr_s32x4; + int64x2_t corr_QC0_s64x2, corr_QC1_s64x2; + corr_QC0_s64x2 = vld1q_s64( corr_QCT + i ); + corr_QC1_s64x2 = vld1q_s64( corr_QCT + i + 2 ); + corr_QC0_s64x2 = vshlq_s64( corr_QC0_s64x2, lsh_s64x2 ); + corr_QC1_s64x2 = vshlq_s64( corr_QC1_s64x2, lsh_s64x2 ); + corr_s32x4 = vcombine_s32( vmovn_s64( corr_QC1_s64x2 ), vmovn_s64( corr_QC0_s64x2 ) ); + corr_s32x4 = vrev64q_s32( corr_s32x4 ); + vst1q_s32( corr + order - i - 3, corr_s32x4 ); + } + if( lsh >= 0 ) { + for( ; i < order + 1; i++ ) { + corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_LSHIFT64( corr_QCT[ i ], lsh ) ); + } + } else { + for( ; i < order + 1; i++ ) { + corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_RSHIFT64( corr_QCT[ i ], -lsh ) ); + } + } + silk_assert( corr_QCT[ order ] >= 0 ); /* If breaking, decrease QC*/ + RESTORE_STACK; + } + +#ifdef OPUS_CHECK_ASM + { + opus_int32 corr_c[ MAX_SHAPE_LPC_ORDER + 1 ]; + opus_int scale_c; + silk_warped_autocorrelation_FIX_c( corr_c, &scale_c, input, warping_Q16, length, order ); + silk_assert( !memcmp( corr_c, corr, sizeof( corr_c[ 0 ] ) * ( order + 1 ) ) ); + silk_assert( scale_c == *scale ); + } +#endif +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/autocorr_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/autocorr_FIX.c new file mode 100644 index 000000000..de95c9869 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/autocorr_FIX.c @@ -0,0 +1,48 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "celt_lpc.h" + +/* Compute autocorrelation */ +void silk_autocorr( + opus_int32 *results, /* O Result (length correlationCount) */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *inputData, /* I Input data to correlate */ + const opus_int inputDataSize, /* I Length of input */ + const opus_int correlationCount, /* I Number of correlation taps to compute */ + int arch /* I Run-time architecture */ +) +{ + opus_int corrCount; + corrCount = silk_min_int( inputDataSize, correlationCount ); + *scale = _celt_autocorr(inputData, results, NULL, 0, corrCount-1, inputDataSize, arch); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/burg_modified_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/burg_modified_FIX.c new file mode 100644 index 000000000..274d4b28e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/burg_modified_FIX.c @@ -0,0 +1,280 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "define.h" +#include "tuning_parameters.h" +#include "pitch.h" + +#define MAX_FRAME_SIZE 384 /* subfr_length * nb_subfr = ( 0.005 * 16000 + 16 ) * 4 = 384 */ + +#define QA 25 +#define N_BITS_HEAD_ROOM 3 +#define MIN_RSHIFTS -16 +#define MAX_RSHIFTS (32 - QA) + +/* Compute reflection coefficients from input signal */ +void silk_burg_modified_c( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */ +) +{ + opus_int k, n, s, lz, rshifts, reached_max_gain; + opus_int32 C0, num, nrg, rc_Q31, invGain_Q30, Atmp_QA, Atmp1, tmp1, tmp2, x1, x2; + const opus_int16 *x_ptr; + opus_int32 C_first_row[ SILK_MAX_ORDER_LPC ]; + opus_int32 C_last_row[ SILK_MAX_ORDER_LPC ]; + opus_int32 Af_QA[ SILK_MAX_ORDER_LPC ]; + opus_int32 CAf[ SILK_MAX_ORDER_LPC + 1 ]; + opus_int32 CAb[ SILK_MAX_ORDER_LPC + 1 ]; + opus_int32 xcorr[ SILK_MAX_ORDER_LPC ]; + opus_int64 C0_64; + + celt_assert( subfr_length * nb_subfr <= MAX_FRAME_SIZE ); + + /* Compute autocorrelations, added over subframes */ + C0_64 = silk_inner_prod16_aligned_64( x, x, subfr_length*nb_subfr, arch ); + lz = silk_CLZ64(C0_64); + rshifts = 32 + 1 + N_BITS_HEAD_ROOM - lz; + if (rshifts > MAX_RSHIFTS) rshifts = MAX_RSHIFTS; + if (rshifts < MIN_RSHIFTS) rshifts = MIN_RSHIFTS; + + if (rshifts > 0) { + C0 = (opus_int32)silk_RSHIFT64(C0_64, rshifts ); + } else { + C0 = silk_LSHIFT32((opus_int32)C0_64, -rshifts ); + } + + CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */ + silk_memset( C_first_row, 0, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) ); + if( rshifts > 0 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + for( n = 1; n < D + 1; n++ ) { + C_first_row[ n - 1 ] += (opus_int32)silk_RSHIFT64( + silk_inner_prod16_aligned_64( x_ptr, x_ptr + n, subfr_length - n, arch ), rshifts ); + } + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + int i; + opus_int32 d; + x_ptr = x + s * subfr_length; + celt_pitch_xcorr(x_ptr, x_ptr + 1, xcorr, subfr_length - D, D, arch ); + for( n = 1; n < D + 1; n++ ) { + for ( i = n + subfr_length - D, d = 0; i < subfr_length; i++ ) + d = MAC16_16( d, x_ptr[ i ], x_ptr[ i - n ] ); + xcorr[ n - 1 ] += d; + } + for( n = 1; n < D + 1; n++ ) { + C_first_row[ n - 1 ] += silk_LSHIFT32( xcorr[ n - 1 ], -rshifts ); + } + } + } + silk_memcpy( C_last_row, C_first_row, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) ); + + /* Initialize */ + CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */ + + invGain_Q30 = (opus_int32)1 << 30; + reached_max_gain = 0; + for( n = 0; n < D; n++ ) { + /* Update first row of correlation matrix (without first element) */ + /* Update last row of correlation matrix (without last element, stored in reversed order) */ + /* Update C * Af */ + /* Update C * flipud(Af) (stored in reversed order) */ + if( rshifts > -2 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], 16 - rshifts ); /* Q(16-rshifts) */ + x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 16 - rshifts ); /* Q(16-rshifts) */ + tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], QA - 16 ); /* Q(QA-16) */ + tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], QA - 16 ); /* Q(QA-16) */ + for( k = 0; k < n; k++ ) { + C_first_row[ k ] = silk_SMLAWB( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */ + C_last_row[ k ] = silk_SMLAWB( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */ + Atmp_QA = Af_QA[ k ]; + tmp1 = silk_SMLAWB( tmp1, Atmp_QA, x_ptr[ n - k - 1 ] ); /* Q(QA-16) */ + tmp2 = silk_SMLAWB( tmp2, Atmp_QA, x_ptr[ subfr_length - n + k ] ); /* Q(QA-16) */ + } + tmp1 = silk_LSHIFT32( -tmp1, 32 - QA - rshifts ); /* Q(16-rshifts) */ + tmp2 = silk_LSHIFT32( -tmp2, 32 - QA - rshifts ); /* Q(16-rshifts) */ + for( k = 0; k <= n; k++ ) { + CAf[ k ] = silk_SMLAWB( CAf[ k ], tmp1, x_ptr[ n - k ] ); /* Q( -rshift ) */ + CAb[ k ] = silk_SMLAWB( CAb[ k ], tmp2, x_ptr[ subfr_length - n + k - 1 ] ); /* Q( -rshift ) */ + } + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], -rshifts ); /* Q( -rshifts ) */ + x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], -rshifts ); /* Q( -rshifts ) */ + tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], 17 ); /* Q17 */ + tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 17 ); /* Q17 */ + for( k = 0; k < n; k++ ) { + C_first_row[ k ] = silk_MLA( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */ + C_last_row[ k ] = silk_MLA( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */ + Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 17 ); /* Q17 */ + /* We sometimes have get overflows in the multiplications (even beyond +/- 2^32), + but they cancel each other and the real result seems to always fit in a 32-bit + signed integer. This was determined experimentally, not theoretically (unfortunately). */ + tmp1 = silk_MLA_ovflw( tmp1, x_ptr[ n - k - 1 ], Atmp1 ); /* Q17 */ + tmp2 = silk_MLA_ovflw( tmp2, x_ptr[ subfr_length - n + k ], Atmp1 ); /* Q17 */ + } + tmp1 = -tmp1; /* Q17 */ + tmp2 = -tmp2; /* Q17 */ + for( k = 0; k <= n; k++ ) { + CAf[ k ] = silk_SMLAWW( CAf[ k ], tmp1, + silk_LSHIFT32( (opus_int32)x_ptr[ n - k ], -rshifts - 1 ) ); /* Q( -rshift ) */ + CAb[ k ] = silk_SMLAWW( CAb[ k ], tmp2, + silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n + k - 1 ], -rshifts - 1 ) ); /* Q( -rshift ) */ + } + } + } + + /* Calculate nominator and denominator for the next order reflection (parcor) coefficient */ + tmp1 = C_first_row[ n ]; /* Q( -rshifts ) */ + tmp2 = C_last_row[ n ]; /* Q( -rshifts ) */ + num = 0; /* Q( -rshifts ) */ + nrg = silk_ADD32( CAb[ 0 ], CAf[ 0 ] ); /* Q( 1-rshifts ) */ + for( k = 0; k < n; k++ ) { + Atmp_QA = Af_QA[ k ]; + lz = silk_CLZ32( silk_abs( Atmp_QA ) ) - 1; + lz = silk_min( 32 - QA, lz ); + Atmp1 = silk_LSHIFT32( Atmp_QA, lz ); /* Q( QA + lz ) */ + + tmp1 = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( C_last_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + tmp2 = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( C_first_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + num = silk_ADD_LSHIFT32( num, silk_SMMUL( CAb[ n - k ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + nrg = silk_ADD_LSHIFT32( nrg, silk_SMMUL( silk_ADD32( CAb[ k + 1 ], CAf[ k + 1 ] ), + Atmp1 ), 32 - QA - lz ); /* Q( 1-rshifts ) */ + } + CAf[ n + 1 ] = tmp1; /* Q( -rshifts ) */ + CAb[ n + 1 ] = tmp2; /* Q( -rshifts ) */ + num = silk_ADD32( num, tmp2 ); /* Q( -rshifts ) */ + num = silk_LSHIFT32( -num, 1 ); /* Q( 1-rshifts ) */ + + /* Calculate the next order reflection (parcor) coefficient */ + if( silk_abs( num ) < nrg ) { + rc_Q31 = silk_DIV32_varQ( num, nrg, 31 ); + } else { + rc_Q31 = ( num > 0 ) ? silk_int32_MAX : silk_int32_MIN; + } + + /* Update inverse prediction gain */ + tmp1 = ( (opus_int32)1 << 30 ) - silk_SMMUL( rc_Q31, rc_Q31 ); + tmp1 = silk_LSHIFT( silk_SMMUL( invGain_Q30, tmp1 ), 2 ); + if( tmp1 <= minInvGain_Q30 ) { + /* Max prediction gain exceeded; set reflection coefficient such that max prediction gain is exactly hit */ + tmp2 = ( (opus_int32)1 << 30 ) - silk_DIV32_varQ( minInvGain_Q30, invGain_Q30, 30 ); /* Q30 */ + rc_Q31 = silk_SQRT_APPROX( tmp2 ); /* Q15 */ + if( rc_Q31 > 0 ) { + /* Newton-Raphson iteration */ + rc_Q31 = silk_RSHIFT32( rc_Q31 + silk_DIV32( tmp2, rc_Q31 ), 1 ); /* Q15 */ + rc_Q31 = silk_LSHIFT32( rc_Q31, 16 ); /* Q31 */ + if( num < 0 ) { + /* Ensure adjusted reflection coefficients has the original sign */ + rc_Q31 = -rc_Q31; + } + } + invGain_Q30 = minInvGain_Q30; + reached_max_gain = 1; + } else { + invGain_Q30 = tmp1; + } + + /* Update the AR coefficients */ + for( k = 0; k < (n + 1) >> 1; k++ ) { + tmp1 = Af_QA[ k ]; /* QA */ + tmp2 = Af_QA[ n - k - 1 ]; /* QA */ + Af_QA[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* QA */ + Af_QA[ n - k - 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* QA */ + } + Af_QA[ n ] = silk_RSHIFT32( rc_Q31, 31 - QA ); /* QA */ + + if( reached_max_gain ) { + /* Reached max prediction gain; set remaining coefficients to zero and exit loop */ + for( k = n + 1; k < D; k++ ) { + Af_QA[ k ] = 0; + } + break; + } + + /* Update C * Af and C * Ab */ + for( k = 0; k <= n + 1; k++ ) { + tmp1 = CAf[ k ]; /* Q( -rshifts ) */ + tmp2 = CAb[ n - k + 1 ]; /* Q( -rshifts ) */ + CAf[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* Q( -rshifts ) */ + CAb[ n - k + 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* Q( -rshifts ) */ + } + } + + if( reached_max_gain ) { + for( k = 0; k < D; k++ ) { + /* Scale coefficients */ + A_Q16[ k ] = -silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); + } + /* Subtract energy of preceding samples from C0 */ + if( rshifts > 0 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + C0 -= (opus_int32)silk_RSHIFT64( silk_inner_prod16_aligned_64( x_ptr, x_ptr, D, arch ), rshifts ); + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + C0 -= silk_LSHIFT32( silk_inner_prod_aligned( x_ptr, x_ptr, D, arch), -rshifts); + } + } + /* Approximate residual energy */ + *res_nrg = silk_LSHIFT( silk_SMMUL( invGain_Q30, C0 ), 2 ); + *res_nrg_Q = -rshifts; + } else { + /* Return residual energy */ + nrg = CAf[ 0 ]; /* Q( -rshifts ) */ + tmp1 = (opus_int32)1 << 16; /* Q16 */ + for( k = 0; k < D; k++ ) { + Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); /* Q16 */ + nrg = silk_SMLAWW( nrg, CAf[ k + 1 ], Atmp1 ); /* Q( -rshifts ) */ + tmp1 = silk_SMLAWW( tmp1, Atmp1, Atmp1 ); /* Q16 */ + A_Q16[ k ] = -Atmp1; + } + *res_nrg = silk_SMLAWW( nrg, silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ), -tmp1 );/* Q( -rshifts ) */ + *res_nrg_Q = -rshifts; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/corrMatrix_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/corrMatrix_FIX.c new file mode 100644 index 000000000..1b4a29c23 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/corrMatrix_FIX.c @@ -0,0 +1,150 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/********************************************************************** + * Correlation Matrix Computations for LS estimate. + **********************************************************************/ + +#include "main_FIX.h" + +/* Calculates correlation vector X'*t */ +void silk_corrVector_FIX( + const opus_int16 *x, /* I x vector [L + order - 1] used to form data matrix X */ + const opus_int16 *t, /* I Target vector [L] */ + const opus_int L, /* I Length of vectors */ + const opus_int order, /* I Max lag for correlation */ + opus_int32 *Xt, /* O Pointer to X'*t correlation vector [order] */ + const opus_int rshifts, /* I Right shifts of correlations */ + int arch /* I Run-time architecture */ +) +{ + opus_int lag, i; + const opus_int16 *ptr1, *ptr2; + opus_int32 inner_prod; + + ptr1 = &x[ order - 1 ]; /* Points to first sample of column 0 of X: X[:,0] */ + ptr2 = t; + /* Calculate X'*t */ + if( rshifts > 0 ) { + /* Right shifting used */ + for( lag = 0; lag < order; lag++ ) { + inner_prod = 0; + for( i = 0; i < L; i++ ) { + inner_prod = silk_ADD_RSHIFT32( inner_prod, silk_SMULBB( ptr1[ i ], ptr2[i] ), rshifts ); + } + Xt[ lag ] = inner_prod; /* X[:,lag]'*t */ + ptr1--; /* Go to next column of X */ + } + } else { + silk_assert( rshifts == 0 ); + for( lag = 0; lag < order; lag++ ) { + Xt[ lag ] = silk_inner_prod_aligned( ptr1, ptr2, L, arch ); /* X[:,lag]'*t */ + ptr1--; /* Go to next column of X */ + } + } +} + +/* Calculates correlation matrix X'*X */ +void silk_corrMatrix_FIX( + const opus_int16 *x, /* I x vector [L + order - 1] used to form data matrix X */ + const opus_int L, /* I Length of vectors */ + const opus_int order, /* I Max lag for correlation */ + opus_int32 *XX, /* O Pointer to X'*X correlation matrix [ order x order ] */ + opus_int32 *nrg, /* O Energy of x vector */ + opus_int *rshifts, /* O Right shifts of correlations and energy */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, j, lag; + opus_int32 energy; + const opus_int16 *ptr1, *ptr2; + + /* Calculate energy to find shift used to fit in 32 bits */ + silk_sum_sqr_shift( nrg, rshifts, x, L + order - 1 ); + energy = *nrg; + + /* Calculate energy of first column (0) of X: X[:,0]'*X[:,0] */ + /* Remove contribution of first order - 1 samples */ + for( i = 0; i < order - 1; i++ ) { + energy -= silk_RSHIFT32( silk_SMULBB( x[ i ], x[ i ] ), *rshifts ); + } + + /* Calculate energy of remaining columns of X: X[:,j]'*X[:,j] */ + /* Fill out the diagonal of the correlation matrix */ + matrix_ptr( XX, 0, 0, order ) = energy; + silk_assert( energy >= 0 ); + ptr1 = &x[ order - 1 ]; /* First sample of column 0 of X */ + for( j = 1; j < order; j++ ) { + energy = silk_SUB32( energy, silk_RSHIFT32( silk_SMULBB( ptr1[ L - j ], ptr1[ L - j ] ), *rshifts ) ); + energy = silk_ADD32( energy, silk_RSHIFT32( silk_SMULBB( ptr1[ -j ], ptr1[ -j ] ), *rshifts ) ); + matrix_ptr( XX, j, j, order ) = energy; + silk_assert( energy >= 0 ); + } + + ptr2 = &x[ order - 2 ]; /* First sample of column 1 of X */ + /* Calculate the remaining elements of the correlation matrix */ + if( *rshifts > 0 ) { + /* Right shifting used */ + for( lag = 1; lag < order; lag++ ) { + /* Inner product of column 0 and column lag: X[:,0]'*X[:,lag] */ + energy = 0; + for( i = 0; i < L; i++ ) { + energy += silk_RSHIFT32( silk_SMULBB( ptr1[ i ], ptr2[i] ), *rshifts ); + } + /* Calculate remaining off diagonal: X[:,j]'*X[:,j + lag] */ + matrix_ptr( XX, lag, 0, order ) = energy; + matrix_ptr( XX, 0, lag, order ) = energy; + for( j = 1; j < ( order - lag ); j++ ) { + energy = silk_SUB32( energy, silk_RSHIFT32( silk_SMULBB( ptr1[ L - j ], ptr2[ L - j ] ), *rshifts ) ); + energy = silk_ADD32( energy, silk_RSHIFT32( silk_SMULBB( ptr1[ -j ], ptr2[ -j ] ), *rshifts ) ); + matrix_ptr( XX, lag + j, j, order ) = energy; + matrix_ptr( XX, j, lag + j, order ) = energy; + } + ptr2--; /* Update pointer to first sample of next column (lag) in X */ + } + } else { + for( lag = 1; lag < order; lag++ ) { + /* Inner product of column 0 and column lag: X[:,0]'*X[:,lag] */ + energy = silk_inner_prod_aligned( ptr1, ptr2, L, arch ); + matrix_ptr( XX, lag, 0, order ) = energy; + matrix_ptr( XX, 0, lag, order ) = energy; + /* Calculate remaining off diagonal: X[:,j]'*X[:,j + lag] */ + for( j = 1; j < ( order - lag ); j++ ) { + energy = silk_SUB32( energy, silk_SMULBB( ptr1[ L - j ], ptr2[ L - j ] ) ); + energy = silk_SMLABB( energy, ptr1[ -j ], ptr2[ -j ] ); + matrix_ptr( XX, lag + j, j, order ) = energy; + matrix_ptr( XX, j, lag + j, order ) = energy; + } + ptr2--;/* Update pointer to first sample of next column (lag) in X */ + } + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/encode_frame_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/encode_frame_FIX.c new file mode 100644 index 000000000..a02bf87db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/encode_frame_FIX.c @@ -0,0 +1,448 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "main_FIX.h" +#include "stack_alloc.h" +#include "tuning_parameters.h" + +/* Low Bitrate Redundancy (LBRR) encoding. Reuse all parameters but encode with lower bitrate */ +static OPUS_INLINE void silk_LBRR_encode_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Pointer to Silk FIX encoder control struct */ + const opus_int16 x16[], /* I Input signal */ + opus_int condCoding /* I The type of conditional coding used so far for this frame */ +); + +void silk_encode_do_VAD_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + opus_int activity /* I Decision of Opus voice activity detector */ +) +{ + const opus_int activity_threshold = SILK_FIX_CONST( SPEECH_ACTIVITY_DTX_THRES, 8 ); + + /****************************/ + /* Voice Activity Detection */ + /****************************/ + silk_VAD_GetSA_Q8( &psEnc->sCmn, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.arch ); + /* If Opus VAD is inactive and Silk VAD is active: lower Silk VAD to just under the threshold */ + if( activity == VAD_NO_ACTIVITY && psEnc->sCmn.speech_activity_Q8 >= activity_threshold ) { + psEnc->sCmn.speech_activity_Q8 = activity_threshold - 1; + } + + /**************************************************/ + /* Convert speech activity into VAD and DTX flags */ + /**************************************************/ + if( psEnc->sCmn.speech_activity_Q8 < activity_threshold ) { + psEnc->sCmn.indices.signalType = TYPE_NO_VOICE_ACTIVITY; + psEnc->sCmn.noSpeechCounter++; + if( psEnc->sCmn.noSpeechCounter <= NB_SPEECH_FRAMES_BEFORE_DTX ) { + psEnc->sCmn.inDTX = 0; + } else if( psEnc->sCmn.noSpeechCounter > MAX_CONSECUTIVE_DTX + NB_SPEECH_FRAMES_BEFORE_DTX ) { + psEnc->sCmn.noSpeechCounter = NB_SPEECH_FRAMES_BEFORE_DTX; + psEnc->sCmn.inDTX = 0; + } + psEnc->sCmn.VAD_flags[ psEnc->sCmn.nFramesEncoded ] = 0; + } else { + psEnc->sCmn.noSpeechCounter = 0; + psEnc->sCmn.inDTX = 0; + psEnc->sCmn.indices.signalType = TYPE_UNVOICED; + psEnc->sCmn.VAD_flags[ psEnc->sCmn.nFramesEncoded ] = 1; + } +} + +/****************/ +/* Encode frame */ +/****************/ +opus_int silk_encode_frame_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + opus_int32 *pnBytesOut, /* O Pointer to number of payload bytes; */ + ec_enc *psRangeEnc, /* I/O compressor data structure */ + opus_int condCoding, /* I The type of conditional coding to use */ + opus_int maxBits, /* I If > 0: maximum number of output bits */ + opus_int useCBR /* I Flag to force constant-bitrate operation */ +) +{ + silk_encoder_control_FIX sEncCtrl; + opus_int i, iter, maxIter, found_upper, found_lower, ret = 0; + opus_int16 *x_frame; + ec_enc sRangeEnc_copy, sRangeEnc_copy2; + silk_nsq_state sNSQ_copy, sNSQ_copy2; + opus_int32 seed_copy, nBits, nBits_lower, nBits_upper, gainMult_lower, gainMult_upper; + opus_int32 gainsID, gainsID_lower, gainsID_upper; + opus_int16 gainMult_Q8; + opus_int16 ec_prevLagIndex_copy; + opus_int ec_prevSignalType_copy; + opus_int8 LastGainIndex_copy2; + opus_int gain_lock[ MAX_NB_SUBFR ] = {0}; + opus_int16 best_gain_mult[ MAX_NB_SUBFR ]; + opus_int best_sum[ MAX_NB_SUBFR ]; + SAVE_STACK; + + /* This is totally unnecessary but many compilers (including gcc) are too dumb to realise it */ + LastGainIndex_copy2 = nBits_lower = nBits_upper = gainMult_lower = gainMult_upper = 0; + + psEnc->sCmn.indices.Seed = psEnc->sCmn.frameCounter++ & 3; + + /**************************************************************/ + /* Set up Input Pointers, and insert frame in input buffer */ + /*************************************************************/ + /* start of frame to encode */ + x_frame = psEnc->x_buf + psEnc->sCmn.ltp_mem_length; + + /***************************************/ + /* Ensure smooth bandwidth transitions */ + /***************************************/ + silk_LP_variable_cutoff( &psEnc->sCmn.sLP, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.frame_length ); + + /*******************************************/ + /* Copy new frame to front of input buffer */ + /*******************************************/ + silk_memcpy( x_frame + LA_SHAPE_MS * psEnc->sCmn.fs_kHz, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.frame_length * sizeof( opus_int16 ) ); + + if( !psEnc->sCmn.prefillFlag ) { + VARDECL( opus_int16, res_pitch ); + VARDECL( opus_uint8, ec_buf_copy ); + opus_int16 *res_pitch_frame; + + ALLOC( res_pitch, + psEnc->sCmn.la_pitch + psEnc->sCmn.frame_length + + psEnc->sCmn.ltp_mem_length, opus_int16 ); + /* start of pitch LPC residual frame */ + res_pitch_frame = res_pitch + psEnc->sCmn.ltp_mem_length; + + /*****************************************/ + /* Find pitch lags, initial LPC analysis */ + /*****************************************/ + silk_find_pitch_lags_FIX( psEnc, &sEncCtrl, res_pitch, x_frame - psEnc->sCmn.ltp_mem_length, psEnc->sCmn.arch ); + + /************************/ + /* Noise shape analysis */ + /************************/ + silk_noise_shape_analysis_FIX( psEnc, &sEncCtrl, res_pitch_frame, x_frame, psEnc->sCmn.arch ); + + /***************************************************/ + /* Find linear prediction coefficients (LPC + LTP) */ + /***************************************************/ + silk_find_pred_coefs_FIX( psEnc, &sEncCtrl, res_pitch_frame, x_frame, condCoding ); + + /****************************************/ + /* Process gains */ + /****************************************/ + silk_process_gains_FIX( psEnc, &sEncCtrl, condCoding ); + + /****************************************/ + /* Low Bitrate Redundant Encoding */ + /****************************************/ + silk_LBRR_encode_FIX( psEnc, &sEncCtrl, x_frame, condCoding ); + + /* Loop over quantizer and entropy coding to control bitrate */ + maxIter = 6; + gainMult_Q8 = SILK_FIX_CONST( 1, 8 ); + found_lower = 0; + found_upper = 0; + gainsID = silk_gains_ID( psEnc->sCmn.indices.GainsIndices, psEnc->sCmn.nb_subfr ); + gainsID_lower = -1; + gainsID_upper = -1; + /* Copy part of the input state */ + silk_memcpy( &sRangeEnc_copy, psRangeEnc, sizeof( ec_enc ) ); + silk_memcpy( &sNSQ_copy, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + seed_copy = psEnc->sCmn.indices.Seed; + ec_prevLagIndex_copy = psEnc->sCmn.ec_prevLagIndex; + ec_prevSignalType_copy = psEnc->sCmn.ec_prevSignalType; + ALLOC( ec_buf_copy, 1275, opus_uint8 ); + for( iter = 0; ; iter++ ) { + if( gainsID == gainsID_lower ) { + nBits = nBits_lower; + } else if( gainsID == gainsID_upper ) { + nBits = nBits_upper; + } else { + /* Restore part of the input state */ + if( iter > 0 ) { + silk_memcpy( psRangeEnc, &sRangeEnc_copy, sizeof( ec_enc ) ); + silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy, sizeof( silk_nsq_state ) ); + psEnc->sCmn.indices.Seed = seed_copy; + psEnc->sCmn.ec_prevLagIndex = ec_prevLagIndex_copy; + psEnc->sCmn.ec_prevSignalType = ec_prevSignalType_copy; + } + + /*****************************************/ + /* Noise shaping quantization */ + /*****************************************/ + if( psEnc->sCmn.nStatesDelayedDecision > 1 || psEnc->sCmn.warping_Q16 > 0 ) { + silk_NSQ_del_dec( &psEnc->sCmn, &psEnc->sCmn.sNSQ, &psEnc->sCmn.indices, x_frame, psEnc->sCmn.pulses, + sEncCtrl.PredCoef_Q12[ 0 ], sEncCtrl.LTPCoef_Q14, sEncCtrl.AR_Q13, sEncCtrl.HarmShapeGain_Q14, + sEncCtrl.Tilt_Q14, sEncCtrl.LF_shp_Q14, sEncCtrl.Gains_Q16, sEncCtrl.pitchL, sEncCtrl.Lambda_Q10, sEncCtrl.LTP_scale_Q14, + psEnc->sCmn.arch ); + } else { + silk_NSQ( &psEnc->sCmn, &psEnc->sCmn.sNSQ, &psEnc->sCmn.indices, x_frame, psEnc->sCmn.pulses, + sEncCtrl.PredCoef_Q12[ 0 ], sEncCtrl.LTPCoef_Q14, sEncCtrl.AR_Q13, sEncCtrl.HarmShapeGain_Q14, + sEncCtrl.Tilt_Q14, sEncCtrl.LF_shp_Q14, sEncCtrl.Gains_Q16, sEncCtrl.pitchL, sEncCtrl.Lambda_Q10, sEncCtrl.LTP_scale_Q14, + psEnc->sCmn.arch); + } + + if ( iter == maxIter && !found_lower ) { + silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) ); + } + + /****************************************/ + /* Encode Parameters */ + /****************************************/ + silk_encode_indices( &psEnc->sCmn, psRangeEnc, psEnc->sCmn.nFramesEncoded, 0, condCoding ); + + /****************************************/ + /* Encode Excitation Signal */ + /****************************************/ + silk_encode_pulses( psRangeEnc, psEnc->sCmn.indices.signalType, psEnc->sCmn.indices.quantOffsetType, + psEnc->sCmn.pulses, psEnc->sCmn.frame_length ); + + nBits = ec_tell( psRangeEnc ); + + /* If we still bust after the last iteration, do some damage control. */ + if ( iter == maxIter && !found_lower && nBits > maxBits ) { + silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) ); + + /* Keep gains the same as the last frame. */ + psEnc->sShape.LastGainIndex = sEncCtrl.lastGainIndexPrev; + for ( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + psEnc->sCmn.indices.GainsIndices[ i ] = 4; + } + if (condCoding != CODE_CONDITIONALLY) { + psEnc->sCmn.indices.GainsIndices[ 0 ] = sEncCtrl.lastGainIndexPrev; + } + psEnc->sCmn.ec_prevLagIndex = ec_prevLagIndex_copy; + psEnc->sCmn.ec_prevSignalType = ec_prevSignalType_copy; + /* Clear all pulses. */ + for ( i = 0; i < psEnc->sCmn.frame_length; i++ ) { + psEnc->sCmn.pulses[ i ] = 0; + } + + silk_encode_indices( &psEnc->sCmn, psRangeEnc, psEnc->sCmn.nFramesEncoded, 0, condCoding ); + + silk_encode_pulses( psRangeEnc, psEnc->sCmn.indices.signalType, psEnc->sCmn.indices.quantOffsetType, + psEnc->sCmn.pulses, psEnc->sCmn.frame_length ); + + nBits = ec_tell( psRangeEnc ); + } + + if( useCBR == 0 && iter == 0 && nBits <= maxBits ) { + break; + } + } + + if( iter == maxIter ) { + if( found_lower && ( gainsID == gainsID_lower || nBits > maxBits ) ) { + /* Restore output state from earlier iteration that did meet the bitrate budget */ + silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) ); + celt_assert( sRangeEnc_copy2.offs <= 1275 ); + silk_memcpy( psRangeEnc->buf, ec_buf_copy, sRangeEnc_copy2.offs ); + silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy2, sizeof( silk_nsq_state ) ); + psEnc->sShape.LastGainIndex = LastGainIndex_copy2; + } + break; + } + + if( nBits > maxBits ) { + if( found_lower == 0 && iter >= 2 ) { + /* Adjust the quantizer's rate/distortion tradeoff and discard previous "upper" results */ + sEncCtrl.Lambda_Q10 = silk_ADD_RSHIFT32( sEncCtrl.Lambda_Q10, sEncCtrl.Lambda_Q10, 1 ); + found_upper = 0; + gainsID_upper = -1; + } else { + found_upper = 1; + nBits_upper = nBits; + gainMult_upper = gainMult_Q8; + gainsID_upper = gainsID; + } + } else if( nBits < maxBits - 5 ) { + found_lower = 1; + nBits_lower = nBits; + gainMult_lower = gainMult_Q8; + if( gainsID != gainsID_lower ) { + gainsID_lower = gainsID; + /* Copy part of the output state */ + silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) ); + celt_assert( psRangeEnc->offs <= 1275 ); + silk_memcpy( ec_buf_copy, psRangeEnc->buf, psRangeEnc->offs ); + silk_memcpy( &sNSQ_copy2, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + LastGainIndex_copy2 = psEnc->sShape.LastGainIndex; + } + } else { + /* Within 5 bits of budget: close enough */ + break; + } + + if ( !found_lower && nBits > maxBits ) { + int j; + for ( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + int sum=0; + for ( j = i*psEnc->sCmn.subfr_length; j < (i+1)*psEnc->sCmn.subfr_length; j++ ) { + sum += abs( psEnc->sCmn.pulses[j] ); + } + if ( iter == 0 || (sum < best_sum[i] && !gain_lock[i]) ) { + best_sum[i] = sum; + best_gain_mult[i] = gainMult_Q8; + } else { + gain_lock[i] = 1; + } + } + } + if( ( found_lower & found_upper ) == 0 ) { + /* Adjust gain according to high-rate rate/distortion curve */ + if( nBits > maxBits ) { + if (gainMult_Q8 < 16384) { + gainMult_Q8 *= 2; + } else { + gainMult_Q8 = 32767; + } + } else { + opus_int32 gain_factor_Q16; + gain_factor_Q16 = silk_log2lin( silk_LSHIFT( nBits - maxBits, 7 ) / psEnc->sCmn.frame_length + SILK_FIX_CONST( 16, 7 ) ); + gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 ); + } + + } else { + /* Adjust gain by interpolating */ + gainMult_Q8 = gainMult_lower + silk_DIV32_16( silk_MUL( gainMult_upper - gainMult_lower, maxBits - nBits_lower ), nBits_upper - nBits_lower ); + /* New gain multplier must be between 25% and 75% of old range (note that gainMult_upper < gainMult_lower) */ + if( gainMult_Q8 > silk_ADD_RSHIFT32( gainMult_lower, gainMult_upper - gainMult_lower, 2 ) ) { + gainMult_Q8 = silk_ADD_RSHIFT32( gainMult_lower, gainMult_upper - gainMult_lower, 2 ); + } else + if( gainMult_Q8 < silk_SUB_RSHIFT32( gainMult_upper, gainMult_upper - gainMult_lower, 2 ) ) { + gainMult_Q8 = silk_SUB_RSHIFT32( gainMult_upper, gainMult_upper - gainMult_lower, 2 ); + } + } + + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + opus_int16 tmp; + if ( gain_lock[i] ) { + tmp = best_gain_mult[i]; + } else { + tmp = gainMult_Q8; + } + sEncCtrl.Gains_Q16[ i ] = silk_LSHIFT_SAT32( silk_SMULWB( sEncCtrl.GainsUnq_Q16[ i ], tmp ), 8 ); + } + + /* Quantize gains */ + psEnc->sShape.LastGainIndex = sEncCtrl.lastGainIndexPrev; + silk_gains_quant( psEnc->sCmn.indices.GainsIndices, sEncCtrl.Gains_Q16, + &psEnc->sShape.LastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /* Unique identifier of gains vector */ + gainsID = silk_gains_ID( psEnc->sCmn.indices.GainsIndices, psEnc->sCmn.nb_subfr ); + } + } + + /* Update input buffer */ + silk_memmove( psEnc->x_buf, &psEnc->x_buf[ psEnc->sCmn.frame_length ], + ( psEnc->sCmn.ltp_mem_length + LA_SHAPE_MS * psEnc->sCmn.fs_kHz ) * sizeof( opus_int16 ) ); + + /* Exit without entropy coding */ + if( psEnc->sCmn.prefillFlag ) { + /* No payload */ + *pnBytesOut = 0; + RESTORE_STACK; + return ret; + } + + /* Parameters needed for next frame */ + psEnc->sCmn.prevLag = sEncCtrl.pitchL[ psEnc->sCmn.nb_subfr - 1 ]; + psEnc->sCmn.prevSignalType = psEnc->sCmn.indices.signalType; + + /****************************************/ + /* Finalize payload */ + /****************************************/ + psEnc->sCmn.first_frame_after_reset = 0; + /* Payload size */ + *pnBytesOut = silk_RSHIFT( ec_tell( psRangeEnc ) + 7, 3 ); + + RESTORE_STACK; + return ret; +} + +/* Low-Bitrate Redundancy (LBRR) encoding. Reuse all parameters but encode excitation at lower bitrate */ +static OPUS_INLINE void silk_LBRR_encode_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Pointer to Silk FIX encoder control struct */ + const opus_int16 x16[], /* I Input signal */ + opus_int condCoding /* I The type of conditional coding used so far for this frame */ +) +{ + opus_int32 TempGains_Q16[ MAX_NB_SUBFR ]; + SideInfoIndices *psIndices_LBRR = &psEnc->sCmn.indices_LBRR[ psEnc->sCmn.nFramesEncoded ]; + silk_nsq_state sNSQ_LBRR; + + /*******************************************/ + /* Control use of inband LBRR */ + /*******************************************/ + if( psEnc->sCmn.LBRR_enabled && psEnc->sCmn.speech_activity_Q8 > SILK_FIX_CONST( LBRR_SPEECH_ACTIVITY_THRES, 8 ) ) { + psEnc->sCmn.LBRR_flags[ psEnc->sCmn.nFramesEncoded ] = 1; + + /* Copy noise shaping quantizer state and quantization indices from regular encoding */ + silk_memcpy( &sNSQ_LBRR, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + silk_memcpy( psIndices_LBRR, &psEnc->sCmn.indices, sizeof( SideInfoIndices ) ); + + /* Save original gains */ + silk_memcpy( TempGains_Q16, psEncCtrl->Gains_Q16, psEnc->sCmn.nb_subfr * sizeof( opus_int32 ) ); + + if( psEnc->sCmn.nFramesEncoded == 0 || psEnc->sCmn.LBRR_flags[ psEnc->sCmn.nFramesEncoded - 1 ] == 0 ) { + /* First frame in packet or previous frame not LBRR coded */ + psEnc->sCmn.LBRRprevLastGainIndex = psEnc->sShape.LastGainIndex; + + /* Increase Gains to get target LBRR rate */ + psIndices_LBRR->GainsIndices[ 0 ] = psIndices_LBRR->GainsIndices[ 0 ] + psEnc->sCmn.LBRR_GainIncreases; + psIndices_LBRR->GainsIndices[ 0 ] = silk_min_int( psIndices_LBRR->GainsIndices[ 0 ], N_LEVELS_QGAIN - 1 ); + } + + /* Decode to get gains in sync with decoder */ + /* Overwrite unquantized gains with quantized gains */ + silk_gains_dequant( psEncCtrl->Gains_Q16, psIndices_LBRR->GainsIndices, + &psEnc->sCmn.LBRRprevLastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /*****************************************/ + /* Noise shaping quantization */ + /*****************************************/ + if( psEnc->sCmn.nStatesDelayedDecision > 1 || psEnc->sCmn.warping_Q16 > 0 ) { + silk_NSQ_del_dec( &psEnc->sCmn, &sNSQ_LBRR, psIndices_LBRR, x16, + psEnc->sCmn.pulses_LBRR[ psEnc->sCmn.nFramesEncoded ], psEncCtrl->PredCoef_Q12[ 0 ], psEncCtrl->LTPCoef_Q14, + psEncCtrl->AR_Q13, psEncCtrl->HarmShapeGain_Q14, psEncCtrl->Tilt_Q14, psEncCtrl->LF_shp_Q14, + psEncCtrl->Gains_Q16, psEncCtrl->pitchL, psEncCtrl->Lambda_Q10, psEncCtrl->LTP_scale_Q14, psEnc->sCmn.arch ); + } else { + silk_NSQ( &psEnc->sCmn, &sNSQ_LBRR, psIndices_LBRR, x16, + psEnc->sCmn.pulses_LBRR[ psEnc->sCmn.nFramesEncoded ], psEncCtrl->PredCoef_Q12[ 0 ], psEncCtrl->LTPCoef_Q14, + psEncCtrl->AR_Q13, psEncCtrl->HarmShapeGain_Q14, psEncCtrl->Tilt_Q14, psEncCtrl->LF_shp_Q14, + psEncCtrl->Gains_Q16, psEncCtrl->pitchL, psEncCtrl->Lambda_Q10, psEncCtrl->LTP_scale_Q14, psEnc->sCmn.arch ); + } + + /* Restore original gains */ + silk_memcpy( psEncCtrl->Gains_Q16, TempGains_Q16, psEnc->sCmn.nb_subfr * sizeof( opus_int32 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LPC_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LPC_FIX.c new file mode 100644 index 000000000..c762a0f2a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LPC_FIX.c @@ -0,0 +1,151 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" +#include "tuning_parameters.h" + +/* Finds LPC vector from correlations, and converts to NLSF */ +void silk_find_LPC_FIX( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 NLSF_Q15[], /* O NLSFs */ + const opus_int16 x[], /* I Input signal */ + const opus_int32 minInvGain_Q30 /* I Inverse of max prediction gain */ +) +{ + opus_int k, subfr_length; + opus_int32 a_Q16[ MAX_LPC_ORDER ]; + opus_int isInterpLower, shift; + opus_int32 res_nrg0, res_nrg1; + opus_int rshift0, rshift1; + + /* Used only for LSF interpolation */ + opus_int32 a_tmp_Q16[ MAX_LPC_ORDER ], res_nrg_interp, res_nrg, res_tmp_nrg; + opus_int res_nrg_interp_Q, res_nrg_Q, res_tmp_nrg_Q; + opus_int16 a_tmp_Q12[ MAX_LPC_ORDER ]; + opus_int16 NLSF0_Q15[ MAX_LPC_ORDER ]; + SAVE_STACK; + + subfr_length = psEncC->subfr_length + psEncC->predictLPCOrder; + + /* Default: no interpolation */ + psEncC->indices.NLSFInterpCoef_Q2 = 4; + + /* Burg AR analysis for the full frame */ + silk_burg_modified( &res_nrg, &res_nrg_Q, a_Q16, x, minInvGain_Q30, subfr_length, psEncC->nb_subfr, psEncC->predictLPCOrder, psEncC->arch ); + + if( psEncC->useInterpolatedNLSFs && !psEncC->first_frame_after_reset && psEncC->nb_subfr == MAX_NB_SUBFR ) { + VARDECL( opus_int16, LPC_res ); + + /* Optimal solution for last 10 ms */ + silk_burg_modified( &res_tmp_nrg, &res_tmp_nrg_Q, a_tmp_Q16, x + 2 * subfr_length, minInvGain_Q30, subfr_length, 2, psEncC->predictLPCOrder, psEncC->arch ); + + /* subtract residual energy here, as that's easier than adding it to the */ + /* residual energy of the first 10 ms in each iteration of the search below */ + shift = res_tmp_nrg_Q - res_nrg_Q; + if( shift >= 0 ) { + if( shift < 32 ) { + res_nrg = res_nrg - silk_RSHIFT( res_tmp_nrg, shift ); + } + } else { + silk_assert( shift > -32 ); + res_nrg = silk_RSHIFT( res_nrg, -shift ) - res_tmp_nrg; + res_nrg_Q = res_tmp_nrg_Q; + } + + /* Convert to NLSFs */ + silk_A2NLSF( NLSF_Q15, a_tmp_Q16, psEncC->predictLPCOrder ); + + ALLOC( LPC_res, 2 * subfr_length, opus_int16 ); + + /* Search over interpolation indices to find the one with lowest residual energy */ + for( k = 3; k >= 0; k-- ) { + /* Interpolate NLSFs for first half */ + silk_interpolate( NLSF0_Q15, psEncC->prev_NLSFq_Q15, NLSF_Q15, k, psEncC->predictLPCOrder ); + + /* Convert to LPC for residual energy evaluation */ + silk_NLSF2A( a_tmp_Q12, NLSF0_Q15, psEncC->predictLPCOrder, psEncC->arch ); + + /* Calculate residual energy with NLSF interpolation */ + silk_LPC_analysis_filter( LPC_res, x, a_tmp_Q12, 2 * subfr_length, psEncC->predictLPCOrder, psEncC->arch ); + + silk_sum_sqr_shift( &res_nrg0, &rshift0, LPC_res + psEncC->predictLPCOrder, subfr_length - psEncC->predictLPCOrder ); + silk_sum_sqr_shift( &res_nrg1, &rshift1, LPC_res + psEncC->predictLPCOrder + subfr_length, subfr_length - psEncC->predictLPCOrder ); + + /* Add subframe energies from first half frame */ + shift = rshift0 - rshift1; + if( shift >= 0 ) { + res_nrg1 = silk_RSHIFT( res_nrg1, shift ); + res_nrg_interp_Q = -rshift0; + } else { + res_nrg0 = silk_RSHIFT( res_nrg0, -shift ); + res_nrg_interp_Q = -rshift1; + } + res_nrg_interp = silk_ADD32( res_nrg0, res_nrg1 ); + + /* Compare with first half energy without NLSF interpolation, or best interpolated value so far */ + shift = res_nrg_interp_Q - res_nrg_Q; + if( shift >= 0 ) { + if( silk_RSHIFT( res_nrg_interp, shift ) < res_nrg ) { + isInterpLower = silk_TRUE; + } else { + isInterpLower = silk_FALSE; + } + } else { + if( -shift < 32 ) { + if( res_nrg_interp < silk_RSHIFT( res_nrg, -shift ) ) { + isInterpLower = silk_TRUE; + } else { + isInterpLower = silk_FALSE; + } + } else { + isInterpLower = silk_FALSE; + } + } + + /* Determine whether current interpolated NLSFs are best so far */ + if( isInterpLower == silk_TRUE ) { + /* Interpolation has lower residual energy */ + res_nrg = res_nrg_interp; + res_nrg_Q = res_nrg_interp_Q; + psEncC->indices.NLSFInterpCoef_Q2 = (opus_int8)k; + } + } + } + + if( psEncC->indices.NLSFInterpCoef_Q2 == 4 ) { + /* NLSF interpolation is currently inactive, calculate NLSFs from full frame AR coefficients */ + silk_A2NLSF( NLSF_Q15, a_Q16, psEncC->predictLPCOrder ); + } + + celt_assert( psEncC->indices.NLSFInterpCoef_Q2 == 4 || ( psEncC->useInterpolatedNLSFs && !psEncC->first_frame_after_reset && psEncC->nb_subfr == MAX_NB_SUBFR ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LTP_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LTP_FIX.c new file mode 100644 index 000000000..62d4afb25 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_LTP_FIX.c @@ -0,0 +1,99 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "tuning_parameters.h" + +void silk_find_LTP_FIX( + opus_int32 XXLTP_Q17[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* O Correlation matrix */ + opus_int32 xXLTP_Q17[ MAX_NB_SUBFR * LTP_ORDER ], /* O Correlation vector */ + const opus_int16 r_ptr[], /* I Residual signal after LPC */ + const opus_int lag[ MAX_NB_SUBFR ], /* I LTP lags */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, k, extra_shifts; + opus_int xx_shifts, xX_shifts, XX_shifts; + const opus_int16 *lag_ptr; + opus_int32 *XXLTP_Q17_ptr, *xXLTP_Q17_ptr; + opus_int32 xx, nrg, temp; + + xXLTP_Q17_ptr = xXLTP_Q17; + XXLTP_Q17_ptr = XXLTP_Q17; + for( k = 0; k < nb_subfr; k++ ) { + lag_ptr = r_ptr - ( lag[ k ] + LTP_ORDER / 2 ); + + silk_sum_sqr_shift( &xx, &xx_shifts, r_ptr, subfr_length + LTP_ORDER ); /* xx in Q( -xx_shifts ) */ + silk_corrMatrix_FIX( lag_ptr, subfr_length, LTP_ORDER, XXLTP_Q17_ptr, &nrg, &XX_shifts, arch ); /* XXLTP_Q17_ptr and nrg in Q( -XX_shifts ) */ + extra_shifts = xx_shifts - XX_shifts; + if( extra_shifts > 0 ) { + /* Shift XX */ + xX_shifts = xx_shifts; + for( i = 0; i < LTP_ORDER * LTP_ORDER; i++ ) { + XXLTP_Q17_ptr[ i ] = silk_RSHIFT32( XXLTP_Q17_ptr[ i ], extra_shifts ); /* Q( -xX_shifts ) */ + } + nrg = silk_RSHIFT32( nrg, extra_shifts ); /* Q( -xX_shifts ) */ + } else if( extra_shifts < 0 ) { + /* Shift xx */ + xX_shifts = XX_shifts; + xx = silk_RSHIFT32( xx, -extra_shifts ); /* Q( -xX_shifts ) */ + } else { + xX_shifts = xx_shifts; + } + silk_corrVector_FIX( lag_ptr, r_ptr, subfr_length, LTP_ORDER, xXLTP_Q17_ptr, xX_shifts, arch ); /* xXLTP_Q17_ptr in Q( -xX_shifts ) */ + + /* At this point all correlations are in Q(-xX_shifts) */ + temp = silk_SMLAWB( 1, nrg, SILK_FIX_CONST( LTP_CORR_INV_MAX, 16 ) ); + temp = silk_max( temp, xx ); +TIC(div) +#if 0 + for( i = 0; i < LTP_ORDER * LTP_ORDER; i++ ) { + XXLTP_Q17_ptr[ i ] = silk_DIV32_varQ( XXLTP_Q17_ptr[ i ], temp, 17 ); + } + for( i = 0; i < LTP_ORDER; i++ ) { + xXLTP_Q17_ptr[ i ] = silk_DIV32_varQ( xXLTP_Q17_ptr[ i ], temp, 17 ); + } +#else + for( i = 0; i < LTP_ORDER * LTP_ORDER; i++ ) { + XXLTP_Q17_ptr[ i ] = (opus_int32)( silk_LSHIFT64( (opus_int64)XXLTP_Q17_ptr[ i ], 17 ) / temp ); + } + for( i = 0; i < LTP_ORDER; i++ ) { + xXLTP_Q17_ptr[ i ] = (opus_int32)( silk_LSHIFT64( (opus_int64)xXLTP_Q17_ptr[ i ], 17 ) / temp ); + } +#endif +TOC(div) + r_ptr += subfr_length; + XXLTP_Q17_ptr += LTP_ORDER * LTP_ORDER; + xXLTP_Q17_ptr += LTP_ORDER; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pitch_lags_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pitch_lags_FIX.c new file mode 100644 index 000000000..6c3379f2b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pitch_lags_FIX.c @@ -0,0 +1,143 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" +#include "tuning_parameters.h" + +/* Find pitch lags */ +void silk_find_pitch_lags_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + opus_int16 res[], /* O residual */ + const opus_int16 x[], /* I Speech signal */ + int arch /* I Run-time architecture */ +) +{ + opus_int buf_len, i, scale; + opus_int32 thrhld_Q13, res_nrg; + const opus_int16 *x_ptr; + VARDECL( opus_int16, Wsig ); + opus_int16 *Wsig_ptr; + opus_int32 auto_corr[ MAX_FIND_PITCH_LPC_ORDER + 1 ]; + opus_int16 rc_Q15[ MAX_FIND_PITCH_LPC_ORDER ]; + opus_int32 A_Q24[ MAX_FIND_PITCH_LPC_ORDER ]; + opus_int16 A_Q12[ MAX_FIND_PITCH_LPC_ORDER ]; + SAVE_STACK; + + /******************************************/ + /* Set up buffer lengths etc based on Fs */ + /******************************************/ + buf_len = psEnc->sCmn.la_pitch + psEnc->sCmn.frame_length + psEnc->sCmn.ltp_mem_length; + + /* Safety check */ + celt_assert( buf_len >= psEnc->sCmn.pitch_LPC_win_length ); + + /*************************************/ + /* Estimate LPC AR coefficients */ + /*************************************/ + + /* Calculate windowed signal */ + + ALLOC( Wsig, psEnc->sCmn.pitch_LPC_win_length, opus_int16 ); + + /* First LA_LTP samples */ + x_ptr = x + buf_len - psEnc->sCmn.pitch_LPC_win_length; + Wsig_ptr = Wsig; + silk_apply_sine_window( Wsig_ptr, x_ptr, 1, psEnc->sCmn.la_pitch ); + + /* Middle un - windowed samples */ + Wsig_ptr += psEnc->sCmn.la_pitch; + x_ptr += psEnc->sCmn.la_pitch; + silk_memcpy( Wsig_ptr, x_ptr, ( psEnc->sCmn.pitch_LPC_win_length - silk_LSHIFT( psEnc->sCmn.la_pitch, 1 ) ) * sizeof( opus_int16 ) ); + + /* Last LA_LTP samples */ + Wsig_ptr += psEnc->sCmn.pitch_LPC_win_length - silk_LSHIFT( psEnc->sCmn.la_pitch, 1 ); + x_ptr += psEnc->sCmn.pitch_LPC_win_length - silk_LSHIFT( psEnc->sCmn.la_pitch, 1 ); + silk_apply_sine_window( Wsig_ptr, x_ptr, 2, psEnc->sCmn.la_pitch ); + + /* Calculate autocorrelation sequence */ + silk_autocorr( auto_corr, &scale, Wsig, psEnc->sCmn.pitch_LPC_win_length, psEnc->sCmn.pitchEstimationLPCOrder + 1, arch ); + + /* Add white noise, as fraction of energy */ + auto_corr[ 0 ] = silk_SMLAWB( auto_corr[ 0 ], auto_corr[ 0 ], SILK_FIX_CONST( FIND_PITCH_WHITE_NOISE_FRACTION, 16 ) ) + 1; + + /* Calculate the reflection coefficients using schur */ + res_nrg = silk_schur( rc_Q15, auto_corr, psEnc->sCmn.pitchEstimationLPCOrder ); + + /* Prediction gain */ + psEncCtrl->predGain_Q16 = silk_DIV32_varQ( auto_corr[ 0 ], silk_max_int( res_nrg, 1 ), 16 ); + + /* Convert reflection coefficients to prediction coefficients */ + silk_k2a( A_Q24, rc_Q15, psEnc->sCmn.pitchEstimationLPCOrder ); + + /* Convert From 32 bit Q24 to 16 bit Q12 coefs */ + for( i = 0; i < psEnc->sCmn.pitchEstimationLPCOrder; i++ ) { + A_Q12[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT( A_Q24[ i ], 12 ) ); + } + + /* Do BWE */ + silk_bwexpander( A_Q12, psEnc->sCmn.pitchEstimationLPCOrder, SILK_FIX_CONST( FIND_PITCH_BANDWIDTH_EXPANSION, 16 ) ); + + /*****************************************/ + /* LPC analysis filtering */ + /*****************************************/ + silk_LPC_analysis_filter( res, x, A_Q12, buf_len, psEnc->sCmn.pitchEstimationLPCOrder, psEnc->sCmn.arch ); + + if( psEnc->sCmn.indices.signalType != TYPE_NO_VOICE_ACTIVITY && psEnc->sCmn.first_frame_after_reset == 0 ) { + /* Threshold for pitch estimator */ + thrhld_Q13 = SILK_FIX_CONST( 0.6, 13 ); + thrhld_Q13 = silk_SMLABB( thrhld_Q13, SILK_FIX_CONST( -0.004, 13 ), psEnc->sCmn.pitchEstimationLPCOrder ); + thrhld_Q13 = silk_SMLAWB( thrhld_Q13, SILK_FIX_CONST( -0.1, 21 ), psEnc->sCmn.speech_activity_Q8 ); + thrhld_Q13 = silk_SMLABB( thrhld_Q13, SILK_FIX_CONST( -0.15, 13 ), silk_RSHIFT( psEnc->sCmn.prevSignalType, 1 ) ); + thrhld_Q13 = silk_SMLAWB( thrhld_Q13, SILK_FIX_CONST( -0.1, 14 ), psEnc->sCmn.input_tilt_Q15 ); + thrhld_Q13 = silk_SAT16( thrhld_Q13 ); + + /*****************************************/ + /* Call pitch estimator */ + /*****************************************/ + if( silk_pitch_analysis_core( res, psEncCtrl->pitchL, &psEnc->sCmn.indices.lagIndex, &psEnc->sCmn.indices.contourIndex, + &psEnc->LTPCorr_Q15, psEnc->sCmn.prevLag, psEnc->sCmn.pitchEstimationThreshold_Q16, + (opus_int)thrhld_Q13, psEnc->sCmn.fs_kHz, psEnc->sCmn.pitchEstimationComplexity, psEnc->sCmn.nb_subfr, + psEnc->sCmn.arch) == 0 ) + { + psEnc->sCmn.indices.signalType = TYPE_VOICED; + } else { + psEnc->sCmn.indices.signalType = TYPE_UNVOICED; + } + } else { + silk_memset( psEncCtrl->pitchL, 0, sizeof( psEncCtrl->pitchL ) ); + psEnc->sCmn.indices.lagIndex = 0; + psEnc->sCmn.indices.contourIndex = 0; + psEnc->LTPCorr_Q15 = 0; + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pred_coefs_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pred_coefs_FIX.c new file mode 100644 index 000000000..606d86334 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/find_pred_coefs_FIX.c @@ -0,0 +1,145 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" + +void silk_find_pred_coefs_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + const opus_int16 res_pitch[], /* I Residual from pitch analysis */ + const opus_int16 x[], /* I Speech signal */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int i; + opus_int32 invGains_Q16[ MAX_NB_SUBFR ], local_gains[ MAX_NB_SUBFR ]; + opus_int16 NLSF_Q15[ MAX_LPC_ORDER ]; + const opus_int16 *x_ptr; + opus_int16 *x_pre_ptr; + VARDECL( opus_int16, LPC_in_pre ); + opus_int32 min_gain_Q16, minInvGain_Q30; + SAVE_STACK; + + /* weighting for weighted least squares */ + min_gain_Q16 = silk_int32_MAX >> 6; + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + min_gain_Q16 = silk_min( min_gain_Q16, psEncCtrl->Gains_Q16[ i ] ); + } + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + /* Divide to Q16 */ + silk_assert( psEncCtrl->Gains_Q16[ i ] > 0 ); + /* Invert and normalize gains, and ensure that maximum invGains_Q16 is within range of a 16 bit int */ + invGains_Q16[ i ] = silk_DIV32_varQ( min_gain_Q16, psEncCtrl->Gains_Q16[ i ], 16 - 2 ); + + /* Limit inverse */ + invGains_Q16[ i ] = silk_max( invGains_Q16[ i ], 100 ); + + /* Square the inverted gains */ + silk_assert( invGains_Q16[ i ] == silk_SAT16( invGains_Q16[ i ] ) ); + + /* Invert the inverted and normalized gains */ + local_gains[ i ] = silk_DIV32( ( (opus_int32)1 << 16 ), invGains_Q16[ i ] ); + } + + ALLOC( LPC_in_pre, + psEnc->sCmn.nb_subfr * psEnc->sCmn.predictLPCOrder + + psEnc->sCmn.frame_length, opus_int16 ); + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + VARDECL( opus_int32, xXLTP_Q17 ); + VARDECL( opus_int32, XXLTP_Q17 ); + + /**********/ + /* VOICED */ + /**********/ + celt_assert( psEnc->sCmn.ltp_mem_length - psEnc->sCmn.predictLPCOrder >= psEncCtrl->pitchL[ 0 ] + LTP_ORDER / 2 ); + + ALLOC( xXLTP_Q17, psEnc->sCmn.nb_subfr * LTP_ORDER, opus_int32 ); + ALLOC( XXLTP_Q17, psEnc->sCmn.nb_subfr * LTP_ORDER * LTP_ORDER, opus_int32 ); + + /* LTP analysis */ + silk_find_LTP_FIX( XXLTP_Q17, xXLTP_Q17, res_pitch, + psEncCtrl->pitchL, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.arch ); + + /* Quantize LTP gain parameters */ + silk_quant_LTP_gains( psEncCtrl->LTPCoef_Q14, psEnc->sCmn.indices.LTPIndex, &psEnc->sCmn.indices.PERIndex, + &psEnc->sCmn.sum_log_gain_Q7, &psEncCtrl->LTPredCodGain_Q7, XXLTP_Q17, xXLTP_Q17, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.arch ); + + /* Control LTP scaling */ + silk_LTP_scale_ctrl_FIX( psEnc, psEncCtrl, condCoding ); + + /* Create LTP residual */ + silk_LTP_analysis_filter_FIX( LPC_in_pre, x - psEnc->sCmn.predictLPCOrder, psEncCtrl->LTPCoef_Q14, + psEncCtrl->pitchL, invGains_Q16, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.predictLPCOrder ); + + } else { + /************/ + /* UNVOICED */ + /************/ + /* Create signal with prepended subframes, scaled by inverse gains */ + x_ptr = x - psEnc->sCmn.predictLPCOrder; + x_pre_ptr = LPC_in_pre; + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + silk_scale_copy_vector16( x_pre_ptr, x_ptr, invGains_Q16[ i ], + psEnc->sCmn.subfr_length + psEnc->sCmn.predictLPCOrder ); + x_pre_ptr += psEnc->sCmn.subfr_length + psEnc->sCmn.predictLPCOrder; + x_ptr += psEnc->sCmn.subfr_length; + } + + silk_memset( psEncCtrl->LTPCoef_Q14, 0, psEnc->sCmn.nb_subfr * LTP_ORDER * sizeof( opus_int16 ) ); + psEncCtrl->LTPredCodGain_Q7 = 0; + psEnc->sCmn.sum_log_gain_Q7 = 0; + } + + /* Limit on total predictive coding gain */ + if( psEnc->sCmn.first_frame_after_reset ) { + minInvGain_Q30 = SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN_AFTER_RESET, 30 ); + } else { + minInvGain_Q30 = silk_log2lin( silk_SMLAWB( 16 << 7, (opus_int32)psEncCtrl->LTPredCodGain_Q7, SILK_FIX_CONST( 1.0 / 3, 16 ) ) ); /* Q16 */ + minInvGain_Q30 = silk_DIV32_varQ( minInvGain_Q30, + silk_SMULWW( SILK_FIX_CONST( MAX_PREDICTION_POWER_GAIN, 0 ), + silk_SMLAWB( SILK_FIX_CONST( 0.25, 18 ), SILK_FIX_CONST( 0.75, 18 ), psEncCtrl->coding_quality_Q14 ) ), 14 ); + } + + /* LPC_in_pre contains the LTP-filtered input for voiced, and the unfiltered input for unvoiced */ + silk_find_LPC_FIX( &psEnc->sCmn, NLSF_Q15, LPC_in_pre, minInvGain_Q30 ); + + /* Quantize LSFs */ + silk_process_NLSFs( &psEnc->sCmn, psEncCtrl->PredCoef_Q12, NLSF_Q15, psEnc->sCmn.prev_NLSFq_Q15 ); + + /* Calculate residual energy using quantized LPC coefficients */ + silk_residual_energy_FIX( psEncCtrl->ResNrg, psEncCtrl->ResNrgQ, LPC_in_pre, psEncCtrl->PredCoef_Q12, local_gains, + psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.predictLPCOrder, psEnc->sCmn.arch ); + + /* Copy to prediction struct for use in next frame for interpolation */ + silk_memcpy( psEnc->sCmn.prev_NLSFq_Q15, NLSF_Q15, sizeof( psEnc->sCmn.prev_NLSFq_Q15 ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_FIX.c new file mode 100644 index 000000000..549f6eada --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_FIX.c @@ -0,0 +1,54 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Step up function, converts reflection coefficients to prediction coefficients */ +void silk_k2a( + opus_int32 *A_Q24, /* O Prediction coefficients [order] Q24 */ + const opus_int16 *rc_Q15, /* I Reflection coefficients [order] Q15 */ + const opus_int32 order /* I Prediction order */ +) +{ + opus_int k, n; + opus_int32 rc, tmp1, tmp2; + + for( k = 0; k < order; k++ ) { + rc = rc_Q15[ k ]; + for( n = 0; n < (k + 1) >> 1; n++ ) { + tmp1 = A_Q24[ n ]; + tmp2 = A_Q24[ k - n - 1 ]; + A_Q24[ n ] = silk_SMLAWB( tmp1, silk_LSHIFT( tmp2, 1 ), rc ); + A_Q24[ k - n - 1 ] = silk_SMLAWB( tmp2, silk_LSHIFT( tmp1, 1 ), rc ); + } + A_Q24[ k ] = -silk_LSHIFT( (opus_int32)rc_Q15[ k ], 9 ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_Q16_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_Q16_FIX.c new file mode 100644 index 000000000..1595aa621 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/k2a_Q16_FIX.c @@ -0,0 +1,54 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Step up function, converts reflection coefficients to prediction coefficients */ +void silk_k2a_Q16( + opus_int32 *A_Q24, /* O Prediction coefficients [order] Q24 */ + const opus_int32 *rc_Q16, /* I Reflection coefficients [order] Q16 */ + const opus_int32 order /* I Prediction order */ +) +{ + opus_int k, n; + opus_int32 rc, tmp1, tmp2; + + for( k = 0; k < order; k++ ) { + rc = rc_Q16[ k ]; + for( n = 0; n < (k + 1) >> 1; n++ ) { + tmp1 = A_Q24[ n ]; + tmp2 = A_Q24[ k - n - 1 ]; + A_Q24[ n ] = silk_SMLAWW( tmp1, tmp2, rc ); + A_Q24[ k - n - 1 ] = silk_SMLAWW( tmp2, tmp1, rc ); + } + A_Q24[ k ] = -silk_LSHIFT( rc, 8 ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/main_FIX.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/main_FIX.h new file mode 100644 index 000000000..6d2112e51 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/main_FIX.h @@ -0,0 +1,244 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MAIN_FIX_H +#define SILK_MAIN_FIX_H + +#include "SigProc_FIX.h" +#include "structs_FIX.h" +#include "control.h" +#include "main.h" +#include "PLC.h" +#include "debug.h" +#include "entenc.h" + +#if ((defined(OPUS_ARM_ASM) && defined(FIXED_POINT)) \ + || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)) +#include "fixed/arm/warped_autocorrelation_FIX_arm.h" +#endif + +#ifndef FORCE_CPP_BUILD +#ifdef __cplusplus +extern "C" +{ +#endif +#endif + +#define silk_encoder_state_Fxx silk_encoder_state_FIX +#define silk_encode_do_VAD_Fxx silk_encode_do_VAD_FIX +#define silk_encode_frame_Fxx silk_encode_frame_FIX + +#define QC 10 +#define QS 13 + +/*********************/ +/* Encoder Functions */ +/*********************/ + +/* High-pass filter with cutoff frequency adaptation based on pitch lag statistics */ +void silk_HP_variable_cutoff( + silk_encoder_state_Fxx state_Fxx[] /* I/O Encoder states */ +); + +/* Encoder main function */ +void silk_encode_do_VAD_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + opus_int activity /* I Decision of Opus voice activity detector */ +); + +/* Encoder main function */ +opus_int silk_encode_frame_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Pointer to Silk FIX encoder state */ + opus_int32 *pnBytesOut, /* O Pointer to number of payload bytes; */ + ec_enc *psRangeEnc, /* I/O compressor data structure */ + opus_int condCoding, /* I The type of conditional coding to use */ + opus_int maxBits, /* I If > 0: maximum number of output bits */ + opus_int useCBR /* I Flag to force constant-bitrate operation */ +); + +/* Initializes the Silk encoder state */ +opus_int silk_init_encoder( + silk_encoder_state_Fxx *psEnc, /* I/O Pointer to Silk FIX encoder state */ + int arch /* I Run-time architecture */ +); + +/* Control the Silk encoder */ +opus_int silk_control_encoder( + silk_encoder_state_Fxx *psEnc, /* I/O Pointer to Silk encoder state */ + silk_EncControlStruct *encControl, /* I Control structure */ + const opus_int allow_bw_switch, /* I Flag to allow switching audio bandwidth */ + const opus_int channelNb, /* I Channel number */ + const opus_int force_fs_kHz +); + +/**************************/ +/* Noise shaping analysis */ +/**************************/ +/* Compute noise shaping coefficients and initial gain values */ +void silk_noise_shape_analysis_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Encoder state FIX */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Encoder control FIX */ + const opus_int16 *pitch_res, /* I LPC residual from pitch analysis */ + const opus_int16 *x, /* I Input signal [ frame_length + la_shape ] */ + int arch /* I Run-time architecture */ +); + +/* Autocorrelations for a warped frequency axis */ +void silk_warped_autocorrelation_FIX_c( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +); + +#if !defined(OVERRIDE_silk_warped_autocorrelation_FIX) +#define silk_warped_autocorrelation_FIX(corr, scale, input, warping_Q16, length, order, arch) \ + ((void)(arch), silk_warped_autocorrelation_FIX_c(corr, scale, input, warping_Q16, length, order)) +#endif + +/* Calculation of LTP state scaling */ +void silk_LTP_scale_ctrl_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/**********************************************/ +/* Prediction Analysis */ +/**********************************************/ +/* Find pitch lags */ +void silk_find_pitch_lags_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + opus_int16 res[], /* O residual */ + const opus_int16 x[], /* I Speech signal */ + int arch /* I Run-time architecture */ +); + +/* Find LPC and LTP coefficients */ +void silk_find_pred_coefs_FIX( + silk_encoder_state_FIX *psEnc, /* I/O encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O encoder control */ + const opus_int16 res_pitch[], /* I Residual from pitch analysis */ + const opus_int16 x[], /* I Speech signal */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/* LPC analysis */ +void silk_find_LPC_FIX( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 NLSF_Q15[], /* O NLSFs */ + const opus_int16 x[], /* I Input signal */ + const opus_int32 minInvGain_Q30 /* I Inverse of max prediction gain */ +); + +/* LTP analysis */ +void silk_find_LTP_FIX( + opus_int32 XXLTP_Q17[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* O Correlation matrix */ + opus_int32 xXLTP_Q17[ MAX_NB_SUBFR * LTP_ORDER ], /* O Correlation vector */ + const opus_int16 r_lpc[], /* I Residual signal after LPC */ + const opus_int lag[ MAX_NB_SUBFR ], /* I LTP lags */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +); + +void silk_LTP_analysis_filter_FIX( + opus_int16 *LTP_res, /* O LTP residual signal of length MAX_NB_SUBFR * ( pre_length + subfr_length ) */ + const opus_int16 *x, /* I Pointer to input signal with at least max( pitchL ) preceding samples */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ],/* I LTP_ORDER LTP coefficients for each MAX_NB_SUBFR subframe */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag, one for each subframe */ + const opus_int32 invGains_Q16[ MAX_NB_SUBFR ], /* I Inverse quantization gains, one for each subframe */ + const opus_int subfr_length, /* I Length of each subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + const opus_int pre_length /* I Length of the preceding samples starting at &x[0] for each subframe */ +); + +/* Calculates residual energies of input subframes where all subframes have LPC_order */ +/* of preceding samples */ +void silk_residual_energy_FIX( + opus_int32 nrgs[ MAX_NB_SUBFR ], /* O Residual energy per subframe */ + opus_int nrgsQ[ MAX_NB_SUBFR ], /* O Q value per subframe */ + const opus_int16 x[], /* I Input signal */ + opus_int16 a_Q12[ 2 ][ MAX_LPC_ORDER ], /* I AR coefs for each frame half */ + const opus_int32 gains[ MAX_NB_SUBFR ], /* I Quantization gains */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I Number of subframes */ + const opus_int LPC_order, /* I LPC order */ + int arch /* I Run-time architecture */ +); + +/* Residual energy: nrg = wxx - 2 * wXx * c + c' * wXX * c */ +opus_int32 silk_residual_energy16_covar_FIX( + const opus_int16 *c, /* I Prediction vector */ + const opus_int32 *wXX, /* I Correlation matrix */ + const opus_int32 *wXx, /* I Correlation vector */ + opus_int32 wxx, /* I Signal energy */ + opus_int D, /* I Dimension */ + opus_int cQ /* I Q value for c vector 0 - 15 */ +); + +/* Processing of gains */ +void silk_process_gains_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Encoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/******************/ +/* Linear Algebra */ +/******************/ +/* Calculates correlation matrix X'*X */ +void silk_corrMatrix_FIX( + const opus_int16 *x, /* I x vector [L + order - 1] used to form data matrix X */ + const opus_int L, /* I Length of vectors */ + const opus_int order, /* I Max lag for correlation */ + opus_int32 *XX, /* O Pointer to X'*X correlation matrix [ order x order ] */ + opus_int32 *nrg, /* O Energy of x vector */ + opus_int *rshifts, /* O Right shifts of correlations */ + int arch /* I Run-time architecture */ +); + +/* Calculates correlation vector X'*t */ +void silk_corrVector_FIX( + const opus_int16 *x, /* I x vector [L + order - 1] used to form data matrix X */ + const opus_int16 *t, /* I Target vector [L] */ + const opus_int L, /* I Length of vectors */ + const opus_int order, /* I Max lag for correlation */ + opus_int32 *Xt, /* O Pointer to X'*t correlation vector [order] */ + const opus_int rshifts, /* I Right shifts of correlations */ + int arch /* I Run-time architecture */ +); + +#ifndef FORCE_CPP_BUILD +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* FORCE_CPP_BUILD */ +#endif /* SILK_MAIN_FIX_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/noise_shape_analysis_FIX_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/noise_shape_analysis_FIX_mipsr1.h new file mode 100644 index 000000000..3999b5bd0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/noise_shape_analysis_FIX_mipsr1.h @@ -0,0 +1,336 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + + +/**************************************************************/ +/* Compute noise shaping coefficients and initial gain values */ +/**************************************************************/ +#define OVERRIDE_silk_noise_shape_analysis_FIX + +void silk_noise_shape_analysis_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Encoder state FIX */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Encoder control FIX */ + const opus_int16 *pitch_res, /* I LPC residual from pitch analysis */ + const opus_int16 *x, /* I Input signal [ frame_length + la_shape ] */ + int arch /* I Run-time architecture */ +) +{ + silk_shape_state_FIX *psShapeSt = &psEnc->sShape; + opus_int k, i, nSamples, Qnrg, b_Q14, warping_Q16, scale = 0; + opus_int32 SNR_adj_dB_Q7, HarmBoost_Q16, HarmShapeGain_Q16, Tilt_Q16, tmp32; + opus_int32 nrg, pre_nrg_Q30, log_energy_Q7, log_energy_prev_Q7, energy_variation_Q7; + opus_int32 delta_Q16, BWExp1_Q16, BWExp2_Q16, gain_mult_Q16, gain_add_Q16, strength_Q16, b_Q8; + opus_int32 auto_corr[ MAX_SHAPE_LPC_ORDER + 1 ]; + opus_int32 refl_coef_Q16[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 AR1_Q24[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 AR2_Q24[ MAX_SHAPE_LPC_ORDER ]; + VARDECL( opus_int16, x_windowed ); + const opus_int16 *x_ptr, *pitch_res_ptr; + SAVE_STACK; + + /* Point to start of first LPC analysis block */ + x_ptr = x - psEnc->sCmn.la_shape; + + /****************/ + /* GAIN CONTROL */ + /****************/ + SNR_adj_dB_Q7 = psEnc->sCmn.SNR_dB_Q7; + + /* Input quality is the average of the quality in the lowest two VAD bands */ + psEncCtrl->input_quality_Q14 = ( opus_int )silk_RSHIFT( (opus_int32)psEnc->sCmn.input_quality_bands_Q15[ 0 ] + + psEnc->sCmn.input_quality_bands_Q15[ 1 ], 2 ); + + /* Coding quality level, between 0.0_Q0 and 1.0_Q0, but in Q14 */ + psEncCtrl->coding_quality_Q14 = silk_RSHIFT( silk_sigm_Q15( silk_RSHIFT_ROUND( SNR_adj_dB_Q7 - + SILK_FIX_CONST( 20.0, 7 ), 4 ) ), 1 ); + + /* Reduce coding SNR during low speech activity */ + if( psEnc->sCmn.useCBR == 0 ) { + b_Q8 = SILK_FIX_CONST( 1.0, 8 ) - psEnc->sCmn.speech_activity_Q8; + b_Q8 = silk_SMULWB( silk_LSHIFT( b_Q8, 8 ), b_Q8 ); + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, + silk_SMULBB( SILK_FIX_CONST( -BG_SNR_DECR_dB, 7 ) >> ( 4 + 1 ), b_Q8 ), /* Q11*/ + silk_SMULWB( SILK_FIX_CONST( 1.0, 14 ) + psEncCtrl->input_quality_Q14, psEncCtrl->coding_quality_Q14 ) ); /* Q12*/ + } + + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce gains for periodic signals */ + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, SILK_FIX_CONST( HARM_SNR_INCR_dB, 8 ), psEnc->LTPCorr_Q15 ); + } else { + /* For unvoiced signals and low-quality input, adjust the quality slower than SNR_dB setting */ + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, + silk_SMLAWB( SILK_FIX_CONST( 6.0, 9 ), -SILK_FIX_CONST( 0.4, 18 ), psEnc->sCmn.SNR_dB_Q7 ), + SILK_FIX_CONST( 1.0, 14 ) - psEncCtrl->input_quality_Q14 ); + } + + /*************************/ + /* SPARSENESS PROCESSING */ + /*************************/ + /* Set quantizer offset */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Initially set to 0; may be overruled in process_gains(..) */ + psEnc->sCmn.indices.quantOffsetType = 0; + psEncCtrl->sparseness_Q8 = 0; + } else { + /* Sparseness measure, based on relative fluctuations of energy per 2 milliseconds */ + nSamples = silk_LSHIFT( psEnc->sCmn.fs_kHz, 1 ); + energy_variation_Q7 = 0; + log_energy_prev_Q7 = 0; + pitch_res_ptr = pitch_res; + for( k = 0; k < silk_SMULBB( SUB_FRAME_LENGTH_MS, psEnc->sCmn.nb_subfr ) / 2; k++ ) { + silk_sum_sqr_shift( &nrg, &scale, pitch_res_ptr, nSamples ); + nrg += silk_RSHIFT( nSamples, scale ); /* Q(-scale)*/ + + log_energy_Q7 = silk_lin2log( nrg ); + if( k > 0 ) { + energy_variation_Q7 += silk_abs( log_energy_Q7 - log_energy_prev_Q7 ); + } + log_energy_prev_Q7 = log_energy_Q7; + pitch_res_ptr += nSamples; + } + + psEncCtrl->sparseness_Q8 = silk_RSHIFT( silk_sigm_Q15( silk_SMULWB( energy_variation_Q7 - + SILK_FIX_CONST( 5.0, 7 ), SILK_FIX_CONST( 0.1, 16 ) ) ), 7 ); + + /* Set quantization offset depending on sparseness measure */ + if( psEncCtrl->sparseness_Q8 > SILK_FIX_CONST( SPARSENESS_THRESHOLD_QNT_OFFSET, 8 ) ) { + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + psEnc->sCmn.indices.quantOffsetType = 1; + } + + /* Increase coding SNR for sparse signals */ + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, SILK_FIX_CONST( SPARSE_SNR_INCR_dB, 15 ), psEncCtrl->sparseness_Q8 - SILK_FIX_CONST( 0.5, 8 ) ); + } + + /*******************************/ + /* Control bandwidth expansion */ + /*******************************/ + /* More BWE for signals with high prediction gain */ + strength_Q16 = silk_SMULWB( psEncCtrl->predGain_Q16, SILK_FIX_CONST( FIND_PITCH_WHITE_NOISE_FRACTION, 16 ) ); + BWExp1_Q16 = BWExp2_Q16 = silk_DIV32_varQ( SILK_FIX_CONST( BANDWIDTH_EXPANSION, 16 ), + silk_SMLAWW( SILK_FIX_CONST( 1.0, 16 ), strength_Q16, strength_Q16 ), 16 ); + delta_Q16 = silk_SMULWB( SILK_FIX_CONST( 1.0, 16 ) - silk_SMULBB( 3, psEncCtrl->coding_quality_Q14 ), + SILK_FIX_CONST( LOW_RATE_BANDWIDTH_EXPANSION_DELTA, 16 ) ); + BWExp1_Q16 = silk_SUB32( BWExp1_Q16, delta_Q16 ); + BWExp2_Q16 = silk_ADD32( BWExp2_Q16, delta_Q16 ); + /* BWExp1 will be applied after BWExp2, so make it relative */ + BWExp1_Q16 = silk_DIV32_16( silk_LSHIFT( BWExp1_Q16, 14 ), silk_RSHIFT( BWExp2_Q16, 2 ) ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Slightly more warping in analysis will move quantization noise up in frequency, where it's better masked */ + warping_Q16 = silk_SMLAWB( psEnc->sCmn.warping_Q16, (opus_int32)psEncCtrl->coding_quality_Q14, SILK_FIX_CONST( 0.01, 18 ) ); + } else { + warping_Q16 = 0; + } + + /********************************************/ + /* Compute noise shaping AR coefs and gains */ + /********************************************/ + ALLOC( x_windowed, psEnc->sCmn.shapeWinLength, opus_int16 ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + /* Apply window: sine slope followed by flat part followed by cosine slope */ + opus_int shift, slope_part, flat_part; + flat_part = psEnc->sCmn.fs_kHz * 3; + slope_part = silk_RSHIFT( psEnc->sCmn.shapeWinLength - flat_part, 1 ); + + silk_apply_sine_window( x_windowed, x_ptr, 1, slope_part ); + shift = slope_part; + silk_memcpy( x_windowed + shift, x_ptr + shift, flat_part * sizeof(opus_int16) ); + shift += flat_part; + silk_apply_sine_window( x_windowed + shift, x_ptr + shift, 2, slope_part ); + + /* Update pointer: next LPC analysis block */ + x_ptr += psEnc->sCmn.subfr_length; + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Calculate warped auto correlation */ + silk_warped_autocorrelation_FIX( auto_corr, &scale, x_windowed, warping_Q16, psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder, arch ); + } else { + /* Calculate regular auto correlation */ + silk_autocorr( auto_corr, &scale, x_windowed, psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder + 1, arch ); + } + + /* Add white noise, as a fraction of energy */ + auto_corr[0] = silk_ADD32( auto_corr[0], silk_max_32( silk_SMULWB( silk_RSHIFT( auto_corr[ 0 ], 4 ), + SILK_FIX_CONST( SHAPE_WHITE_NOISE_FRACTION, 20 ) ), 1 ) ); + + /* Calculate the reflection coefficients using schur */ + nrg = silk_schur64( refl_coef_Q16, auto_corr, psEnc->sCmn.shapingLPCOrder ); + silk_assert( nrg >= 0 ); + + /* Convert reflection coefficients to prediction coefficients */ + silk_k2a_Q16( AR2_Q24, refl_coef_Q16, psEnc->sCmn.shapingLPCOrder ); + + Qnrg = -scale; /* range: -12...30*/ + silk_assert( Qnrg >= -12 ); + silk_assert( Qnrg <= 30 ); + + /* Make sure that Qnrg is an even number */ + if( Qnrg & 1 ) { + Qnrg -= 1; + nrg >>= 1; + } + + tmp32 = silk_SQRT_APPROX( nrg ); + Qnrg >>= 1; /* range: -6...15*/ + + psEncCtrl->Gains_Q16[ k ] = (silk_LSHIFT32( silk_LIMIT( (tmp32), silk_RSHIFT32( silk_int32_MIN, (16 - Qnrg) ), \ + silk_RSHIFT32( silk_int32_MAX, (16 - Qnrg) ) ), (16 - Qnrg) )); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Adjust gain for warping */ + gain_mult_Q16 = warped_gain( AR2_Q24, warping_Q16, psEnc->sCmn.shapingLPCOrder ); + silk_assert( psEncCtrl->Gains_Q16[ k ] >= 0 ); + if ( silk_SMULWW( silk_RSHIFT_ROUND( psEncCtrl->Gains_Q16[ k ], 1 ), gain_mult_Q16 ) >= ( silk_int32_MAX >> 1 ) ) { + psEncCtrl->Gains_Q16[ k ] = silk_int32_MAX; + } else { + psEncCtrl->Gains_Q16[ k ] = silk_SMULWW( psEncCtrl->Gains_Q16[ k ], gain_mult_Q16 ); + } + } + + /* Bandwidth expansion for synthesis filter shaping */ + silk_bwexpander_32( AR2_Q24, psEnc->sCmn.shapingLPCOrder, BWExp2_Q16 ); + + /* Compute noise shaping filter coefficients */ + silk_memcpy( AR1_Q24, AR2_Q24, psEnc->sCmn.shapingLPCOrder * sizeof( opus_int32 ) ); + + /* Bandwidth expansion for analysis filter shaping */ + silk_assert( BWExp1_Q16 <= SILK_FIX_CONST( 1.0, 16 ) ); + silk_bwexpander_32( AR1_Q24, psEnc->sCmn.shapingLPCOrder, BWExp1_Q16 ); + + /* Ratio of prediction gains, in energy domain */ + pre_nrg_Q30 = silk_LPC_inverse_pred_gain_Q24( AR2_Q24, psEnc->sCmn.shapingLPCOrder, arch ); + nrg = silk_LPC_inverse_pred_gain_Q24( AR1_Q24, psEnc->sCmn.shapingLPCOrder, arch ); + + /*psEncCtrl->GainsPre[ k ] = 1.0f - 0.7f * ( 1.0f - pre_nrg / nrg ) = 0.3f + 0.7f * pre_nrg / nrg;*/ + pre_nrg_Q30 = silk_LSHIFT32( silk_SMULWB( pre_nrg_Q30, SILK_FIX_CONST( 0.7, 15 ) ), 1 ); + psEncCtrl->GainsPre_Q14[ k ] = ( opus_int ) SILK_FIX_CONST( 0.3, 14 ) + silk_DIV32_varQ( pre_nrg_Q30, nrg, 14 ); + + /* Convert to monic warped prediction coefficients and limit absolute values */ + limit_warped_coefs( AR2_Q24, AR1_Q24, warping_Q16, SILK_FIX_CONST( 3.999, 24 ), psEnc->sCmn.shapingLPCOrder ); + + /* Convert from Q24 to Q13 and store in int16 */ + for( i = 0; i < psEnc->sCmn.shapingLPCOrder; i++ ) { + psEncCtrl->AR1_Q13[ k * MAX_SHAPE_LPC_ORDER + i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( AR1_Q24[ i ], 11 ) ); + psEncCtrl->AR2_Q13[ k * MAX_SHAPE_LPC_ORDER + i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( AR2_Q24[ i ], 11 ) ); + } + } + + /*****************/ + /* Gain tweaking */ + /*****************/ + /* Increase gains during low speech activity and put lower limit on gains */ + gain_mult_Q16 = silk_log2lin( -silk_SMLAWB( -SILK_FIX_CONST( 16.0, 7 ), SNR_adj_dB_Q7, SILK_FIX_CONST( 0.16, 16 ) ) ); + gain_add_Q16 = silk_log2lin( silk_SMLAWB( SILK_FIX_CONST( 16.0, 7 ), SILK_FIX_CONST( MIN_QGAIN_DB, 7 ), SILK_FIX_CONST( 0.16, 16 ) ) ); + silk_assert( gain_mult_Q16 > 0 ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains_Q16[ k ] = silk_SMULWW( psEncCtrl->Gains_Q16[ k ], gain_mult_Q16 ); + silk_assert( psEncCtrl->Gains_Q16[ k ] >= 0 ); + psEncCtrl->Gains_Q16[ k ] = silk_ADD_POS_SAT32( psEncCtrl->Gains_Q16[ k ], gain_add_Q16 ); + } + + gain_mult_Q16 = SILK_FIX_CONST( 1.0, 16 ) + silk_RSHIFT_ROUND( silk_MLA( SILK_FIX_CONST( INPUT_TILT, 26 ), + psEncCtrl->coding_quality_Q14, SILK_FIX_CONST( HIGH_RATE_INPUT_TILT, 12 ) ), 10 ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->GainsPre_Q14[ k ] = silk_SMULWB( gain_mult_Q16, psEncCtrl->GainsPre_Q14[ k ] ); + } + + /************************************************/ + /* Control low-frequency shaping and noise tilt */ + /************************************************/ + /* Less low frequency shaping for noisy inputs */ + strength_Q16 = silk_MUL( SILK_FIX_CONST( LOW_FREQ_SHAPING, 4 ), silk_SMLAWB( SILK_FIX_CONST( 1.0, 12 ), + SILK_FIX_CONST( LOW_QUALITY_LOW_FREQ_SHAPING_DECR, 13 ), psEnc->sCmn.input_quality_bands_Q15[ 0 ] - SILK_FIX_CONST( 1.0, 15 ) ) ); + strength_Q16 = silk_RSHIFT( silk_MUL( strength_Q16, psEnc->sCmn.speech_activity_Q8 ), 8 ); + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce low frequencies quantization noise for periodic signals, depending on pitch lag */ + /*f = 400; freqz([1, -0.98 + 2e-4 * f], [1, -0.97 + 7e-4 * f], 2^12, Fs); axis([0, 1000, -10, 1])*/ + opus_int fs_kHz_inv = silk_DIV32_16( SILK_FIX_CONST( 0.2, 14 ), psEnc->sCmn.fs_kHz ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + b_Q14 = fs_kHz_inv + silk_DIV32_16( SILK_FIX_CONST( 3.0, 14 ), psEncCtrl->pitchL[ k ] ); + /* Pack two coefficients in one int32 */ + psEncCtrl->LF_shp_Q14[ k ] = silk_LSHIFT( SILK_FIX_CONST( 1.0, 14 ) - b_Q14 - silk_SMULWB( strength_Q16, b_Q14 ), 16 ); + psEncCtrl->LF_shp_Q14[ k ] |= (opus_uint16)( b_Q14 - SILK_FIX_CONST( 1.0, 14 ) ); + } + silk_assert( SILK_FIX_CONST( HARM_HP_NOISE_COEF, 24 ) < SILK_FIX_CONST( 0.5, 24 ) ); /* Guarantees that second argument to SMULWB() is within range of an opus_int16*/ + Tilt_Q16 = - SILK_FIX_CONST( HP_NOISE_COEF, 16 ) - + silk_SMULWB( SILK_FIX_CONST( 1.0, 16 ) - SILK_FIX_CONST( HP_NOISE_COEF, 16 ), + silk_SMULWB( SILK_FIX_CONST( HARM_HP_NOISE_COEF, 24 ), psEnc->sCmn.speech_activity_Q8 ) ); + } else { + b_Q14 = silk_DIV32_16( 21299, psEnc->sCmn.fs_kHz ); /* 1.3_Q0 = 21299_Q14*/ + /* Pack two coefficients in one int32 */ + psEncCtrl->LF_shp_Q14[ 0 ] = silk_LSHIFT( SILK_FIX_CONST( 1.0, 14 ) - b_Q14 - + silk_SMULWB( strength_Q16, silk_SMULWB( SILK_FIX_CONST( 0.6, 16 ), b_Q14 ) ), 16 ); + psEncCtrl->LF_shp_Q14[ 0 ] |= (opus_uint16)( b_Q14 - SILK_FIX_CONST( 1.0, 14 ) ); + for( k = 1; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->LF_shp_Q14[ k ] = psEncCtrl->LF_shp_Q14[ 0 ]; + } + Tilt_Q16 = -SILK_FIX_CONST( HP_NOISE_COEF, 16 ); + } + + /****************************/ + /* HARMONIC SHAPING CONTROL */ + /****************************/ + /* Control boosting of harmonic frequencies */ + HarmBoost_Q16 = silk_SMULWB( silk_SMULWB( SILK_FIX_CONST( 1.0, 17 ) - silk_LSHIFT( psEncCtrl->coding_quality_Q14, 3 ), + psEnc->LTPCorr_Q15 ), SILK_FIX_CONST( LOW_RATE_HARMONIC_BOOST, 16 ) ); + + /* More harmonic boost for noisy input signals */ + HarmBoost_Q16 = silk_SMLAWB( HarmBoost_Q16, + SILK_FIX_CONST( 1.0, 16 ) - silk_LSHIFT( psEncCtrl->input_quality_Q14, 2 ), SILK_FIX_CONST( LOW_INPUT_QUALITY_HARMONIC_BOOST, 16 ) ); + + if( USE_HARM_SHAPING && psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* More harmonic noise shaping for high bitrates or noisy input */ + HarmShapeGain_Q16 = silk_SMLAWB( SILK_FIX_CONST( HARMONIC_SHAPING, 16 ), + SILK_FIX_CONST( 1.0, 16 ) - silk_SMULWB( SILK_FIX_CONST( 1.0, 18 ) - silk_LSHIFT( psEncCtrl->coding_quality_Q14, 4 ), + psEncCtrl->input_quality_Q14 ), SILK_FIX_CONST( HIGH_RATE_OR_LOW_QUALITY_HARMONIC_SHAPING, 16 ) ); + + /* Less harmonic noise shaping for less periodic signals */ + HarmShapeGain_Q16 = silk_SMULWB( silk_LSHIFT( HarmShapeGain_Q16, 1 ), + silk_SQRT_APPROX( silk_LSHIFT( psEnc->LTPCorr_Q15, 15 ) ) ); + } else { + HarmShapeGain_Q16 = 0; + } + + /*************************/ + /* Smooth over subframes */ + /*************************/ + for( k = 0; k < MAX_NB_SUBFR; k++ ) { + psShapeSt->HarmBoost_smth_Q16 = + silk_SMLAWB( psShapeSt->HarmBoost_smth_Q16, HarmBoost_Q16 - psShapeSt->HarmBoost_smth_Q16, SILK_FIX_CONST( SUBFR_SMTH_COEF, 16 ) ); + psShapeSt->HarmShapeGain_smth_Q16 = + silk_SMLAWB( psShapeSt->HarmShapeGain_smth_Q16, HarmShapeGain_Q16 - psShapeSt->HarmShapeGain_smth_Q16, SILK_FIX_CONST( SUBFR_SMTH_COEF, 16 ) ); + psShapeSt->Tilt_smth_Q16 = + silk_SMLAWB( psShapeSt->Tilt_smth_Q16, Tilt_Q16 - psShapeSt->Tilt_smth_Q16, SILK_FIX_CONST( SUBFR_SMTH_COEF, 16 ) ); + + psEncCtrl->HarmBoost_Q14[ k ] = ( opus_int )silk_RSHIFT_ROUND( psShapeSt->HarmBoost_smth_Q16, 2 ); + psEncCtrl->HarmShapeGain_Q14[ k ] = ( opus_int )silk_RSHIFT_ROUND( psShapeSt->HarmShapeGain_smth_Q16, 2 ); + psEncCtrl->Tilt_Q14[ k ] = ( opus_int )silk_RSHIFT_ROUND( psShapeSt->Tilt_smth_Q16, 2 ); + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/prefilter_FIX_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/prefilter_FIX_mipsr1.h new file mode 100644 index 000000000..21b256885 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/prefilter_FIX_mipsr1.h @@ -0,0 +1,184 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +#ifndef __PREFILTER_FIX_MIPSR1_H__ +#define __PREFILTER_FIX_MIPSR1_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" +#include "tuning_parameters.h" + +#define OVERRIDE_silk_warped_LPC_analysis_filter_FIX +void silk_warped_LPC_analysis_filter_FIX( + opus_int32 state[], /* I/O State [order + 1] */ + opus_int32 res_Q2[], /* O Residual signal [length] */ + const opus_int16 coef_Q13[], /* I Coefficients [order] */ + const opus_int16 input[], /* I Input signal [length] */ + const opus_int16 lambda_Q16, /* I Warping factor */ + const opus_int length, /* I Length of input signal */ + const opus_int order, /* I Filter order (even) */ + int arch +) +{ + opus_int n, i; + opus_int32 acc_Q11, acc_Q22, tmp1, tmp2, tmp3, tmp4; + opus_int32 state_cur, state_next; + + (void)arch; + + /* Order must be even */ + /* Length must be even */ + + silk_assert( ( order & 1 ) == 0 ); + silk_assert( ( length & 1 ) == 0 ); + + for( n = 0; n < length; n+=2 ) { + /* Output of lowpass section */ + tmp2 = silk_SMLAWB( state[ 0 ], state[ 1 ], lambda_Q16 ); + state_cur = silk_LSHIFT( input[ n ], 14 ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( state[ 1 ], state[ 2 ] - tmp2, lambda_Q16 ); + state_next = tmp2; + acc_Q11 = silk_RSHIFT( order, 1 ); + acc_Q11 = silk_SMLAWB( acc_Q11, tmp2, coef_Q13[ 0 ] ); + + + /* Output of lowpass section */ + tmp4 = silk_SMLAWB( state_cur, state_next, lambda_Q16 ); + state[ 0 ] = silk_LSHIFT( input[ n+1 ], 14 ); + /* Output of allpass section */ + tmp3 = silk_SMLAWB( state_next, tmp1 - tmp4, lambda_Q16 ); + state[ 1 ] = tmp4; + acc_Q22 = silk_RSHIFT( order, 1 ); + acc_Q22 = silk_SMLAWB( acc_Q22, tmp4, coef_Q13[ 0 ] ); + + /* Loop over allpass sections */ + for( i = 2; i < order; i += 2 ) { + /* Output of allpass section */ + tmp2 = silk_SMLAWB( state[ i ], state[ i + 1 ] - tmp1, lambda_Q16 ); + state_cur = tmp1; + acc_Q11 = silk_SMLAWB( acc_Q11, tmp1, coef_Q13[ i - 1 ] ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( state[ i + 1 ], state[ i + 2 ] - tmp2, lambda_Q16 ); + state_next = tmp2; + acc_Q11 = silk_SMLAWB( acc_Q11, tmp2, coef_Q13[ i ] ); + + + /* Output of allpass section */ + tmp4 = silk_SMLAWB( state_cur, state_next - tmp3, lambda_Q16 ); + state[ i ] = tmp3; + acc_Q22 = silk_SMLAWB( acc_Q22, tmp3, coef_Q13[ i - 1 ] ); + /* Output of allpass section */ + tmp3 = silk_SMLAWB( state_next, tmp1 - tmp4, lambda_Q16 ); + state[ i + 1 ] = tmp4; + acc_Q22 = silk_SMLAWB( acc_Q22, tmp4, coef_Q13[ i ] ); + } + acc_Q11 = silk_SMLAWB( acc_Q11, tmp1, coef_Q13[ order - 1 ] ); + res_Q2[ n ] = silk_LSHIFT( (opus_int32)input[ n ], 2 ) - silk_RSHIFT_ROUND( acc_Q11, 9 ); + + state[ order ] = tmp3; + acc_Q22 = silk_SMLAWB( acc_Q22, tmp3, coef_Q13[ order - 1 ] ); + res_Q2[ n+1 ] = silk_LSHIFT( (opus_int32)input[ n+1 ], 2 ) - silk_RSHIFT_ROUND( acc_Q22, 9 ); + } +} + + + +/* Prefilter for finding Quantizer input signal */ +#define OVERRIDE_silk_prefilt_FIX +static inline void silk_prefilt_FIX( + silk_prefilter_state_FIX *P, /* I/O state */ + opus_int32 st_res_Q12[], /* I short term residual signal */ + opus_int32 xw_Q3[], /* O prefiltered signal */ + opus_int32 HarmShapeFIRPacked_Q12, /* I Harmonic shaping coeficients */ + opus_int Tilt_Q14, /* I Tilt shaping coeficient */ + opus_int32 LF_shp_Q14, /* I Low-frequancy shaping coeficients */ + opus_int lag, /* I Lag for harmonic shaping */ + opus_int length /* I Length of signals */ +) +{ + opus_int i, idx, LTP_shp_buf_idx; + opus_int32 n_LTP_Q12, n_Tilt_Q10, n_LF_Q10; + opus_int32 sLF_MA_shp_Q12, sLF_AR_shp_Q12; + opus_int16 *LTP_shp_buf; + + /* To speed up use temp variables instead of using the struct */ + LTP_shp_buf = P->sLTP_shp; + LTP_shp_buf_idx = P->sLTP_shp_buf_idx; + sLF_AR_shp_Q12 = P->sLF_AR_shp_Q12; + sLF_MA_shp_Q12 = P->sLF_MA_shp_Q12; + + if( lag > 0 ) { + for( i = 0; i < length; i++ ) { + /* unrolled loop */ + silk_assert( HARM_SHAPE_FIR_TAPS == 3 ); + idx = lag + LTP_shp_buf_idx; + n_LTP_Q12 = silk_SMULBB( LTP_shp_buf[ ( idx - HARM_SHAPE_FIR_TAPS / 2 - 1) & LTP_MASK ], HarmShapeFIRPacked_Q12 ); + n_LTP_Q12 = silk_SMLABT( n_LTP_Q12, LTP_shp_buf[ ( idx - HARM_SHAPE_FIR_TAPS / 2 ) & LTP_MASK ], HarmShapeFIRPacked_Q12 ); + n_LTP_Q12 = silk_SMLABB( n_LTP_Q12, LTP_shp_buf[ ( idx - HARM_SHAPE_FIR_TAPS / 2 + 1) & LTP_MASK ], HarmShapeFIRPacked_Q12 ); + + n_Tilt_Q10 = silk_SMULWB( sLF_AR_shp_Q12, Tilt_Q14 ); + n_LF_Q10 = silk_SMLAWB( silk_SMULWT( sLF_AR_shp_Q12, LF_shp_Q14 ), sLF_MA_shp_Q12, LF_shp_Q14 ); + + sLF_AR_shp_Q12 = silk_SUB32( st_res_Q12[ i ], silk_LSHIFT( n_Tilt_Q10, 2 ) ); + sLF_MA_shp_Q12 = silk_SUB32( sLF_AR_shp_Q12, silk_LSHIFT( n_LF_Q10, 2 ) ); + + LTP_shp_buf_idx = ( LTP_shp_buf_idx - 1 ) & LTP_MASK; + LTP_shp_buf[ LTP_shp_buf_idx ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sLF_MA_shp_Q12, 12 ) ); + + xw_Q3[i] = silk_RSHIFT_ROUND( silk_SUB32( sLF_MA_shp_Q12, n_LTP_Q12 ), 9 ); + } + } + else + { + for( i = 0; i < length; i++ ) { + + n_LTP_Q12 = 0; + + n_Tilt_Q10 = silk_SMULWB( sLF_AR_shp_Q12, Tilt_Q14 ); + n_LF_Q10 = silk_SMLAWB( silk_SMULWT( sLF_AR_shp_Q12, LF_shp_Q14 ), sLF_MA_shp_Q12, LF_shp_Q14 ); + + sLF_AR_shp_Q12 = silk_SUB32( st_res_Q12[ i ], silk_LSHIFT( n_Tilt_Q10, 2 ) ); + sLF_MA_shp_Q12 = silk_SUB32( sLF_AR_shp_Q12, silk_LSHIFT( n_LF_Q10, 2 ) ); + + LTP_shp_buf_idx = ( LTP_shp_buf_idx - 1 ) & LTP_MASK; + LTP_shp_buf[ LTP_shp_buf_idx ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sLF_MA_shp_Q12, 12 ) ); + + xw_Q3[i] = silk_RSHIFT_ROUND( sLF_MA_shp_Q12, 9 ); + } + } + + /* Copy temp variable back to state */ + P->sLF_AR_shp_Q12 = sLF_AR_shp_Q12; + P->sLF_MA_shp_Q12 = sLF_MA_shp_Q12; + P->sLTP_shp_buf_idx = LTP_shp_buf_idx; +} + +#endif /* __PREFILTER_FIX_MIPSR1_H__ */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/warped_autocorrelation_FIX_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/warped_autocorrelation_FIX_mipsr1.h new file mode 100644 index 000000000..66eb2ed26 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/mips/warped_autocorrelation_FIX_mipsr1.h @@ -0,0 +1,165 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef __WARPED_AUTOCORRELATION_FIX_MIPSR1_H__ +#define __WARPED_AUTOCORRELATION_FIX_MIPSR1_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +#undef QC +#define QC 10 + +#undef QS +#define QS 14 + +/* Autocorrelations for a warped frequency axis */ +#define OVERRIDE_silk_warped_autocorrelation_FIX_c +void silk_warped_autocorrelation_FIX_c( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +) +{ + opus_int n, i, lsh; + opus_int32 tmp1_QS=0, tmp2_QS=0, tmp3_QS=0, tmp4_QS=0, tmp5_QS=0, tmp6_QS=0, tmp7_QS=0, tmp8_QS=0, start_1=0, start_2=0, start_3=0; + opus_int32 state_QS[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + opus_int64 corr_QC[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + opus_int64 temp64; + + opus_int32 val; + val = 2 * QS - QC; + + /* Order must be even */ + silk_assert( ( order & 1 ) == 0 ); + silk_assert( 2 * QS - QC >= 0 ); + + /* Loop over samples */ + for( n = 0; n < length; n=n+4 ) { + + tmp1_QS = silk_LSHIFT32( (opus_int32)input[ n ], QS ); + start_1 = tmp1_QS; + tmp3_QS = silk_LSHIFT32( (opus_int32)input[ n+1], QS ); + start_2 = tmp3_QS; + tmp5_QS = silk_LSHIFT32( (opus_int32)input[ n+2], QS ); + start_3 = tmp5_QS; + tmp7_QS = silk_LSHIFT32( (opus_int32)input[ n+3], QS ); + + /* Loop over allpass sections */ + for( i = 0; i < order; i += 2 ) { + /* Output of allpass section */ + tmp2_QS = silk_SMLAWB( state_QS[ i ], state_QS[ i + 1 ] - tmp1_QS, warping_Q16 ); + corr_QC[ i ] = __builtin_mips_madd( corr_QC[ i ], tmp1_QS, start_1); + + tmp4_QS = silk_SMLAWB( tmp1_QS, tmp2_QS - tmp3_QS, warping_Q16 ); + corr_QC[ i ] = __builtin_mips_madd( corr_QC[ i ], tmp3_QS, start_2); + + tmp6_QS = silk_SMLAWB( tmp3_QS, tmp4_QS - tmp5_QS, warping_Q16 ); + corr_QC[ i ] = __builtin_mips_madd( corr_QC[ i ], tmp5_QS, start_3); + + tmp8_QS = silk_SMLAWB( tmp5_QS, tmp6_QS - tmp7_QS, warping_Q16 ); + state_QS[ i ] = tmp7_QS; + corr_QC[ i ] = __builtin_mips_madd( corr_QC[ i ], tmp7_QS, state_QS[0]); + + /* Output of allpass section */ + tmp1_QS = silk_SMLAWB( state_QS[ i + 1 ], state_QS[ i + 2 ] - tmp2_QS, warping_Q16 ); + corr_QC[ i+1 ] = __builtin_mips_madd( corr_QC[ i+1 ], tmp2_QS, start_1); + + tmp3_QS = silk_SMLAWB( tmp2_QS, tmp1_QS - tmp4_QS, warping_Q16 ); + corr_QC[ i+1 ] = __builtin_mips_madd( corr_QC[ i+1 ], tmp4_QS, start_2); + + tmp5_QS = silk_SMLAWB( tmp4_QS, tmp3_QS - tmp6_QS, warping_Q16 ); + corr_QC[ i+1 ] = __builtin_mips_madd( corr_QC[ i+1 ], tmp6_QS, start_3); + + tmp7_QS = silk_SMLAWB( tmp6_QS, tmp5_QS - tmp8_QS, warping_Q16 ); + state_QS[ i + 1 ] = tmp8_QS; + corr_QC[ i+1 ] = __builtin_mips_madd( corr_QC[ i+1 ], tmp8_QS, state_QS[ 0 ]); + + } + state_QS[ order ] = tmp7_QS; + + corr_QC[ order ] = __builtin_mips_madd( corr_QC[ order ], tmp1_QS, start_1); + corr_QC[ order ] = __builtin_mips_madd( corr_QC[ order ], tmp3_QS, start_2); + corr_QC[ order ] = __builtin_mips_madd( corr_QC[ order ], tmp5_QS, start_3); + corr_QC[ order ] = __builtin_mips_madd( corr_QC[ order ], tmp7_QS, state_QS[ 0 ]); + } + + for(;n< length; n++ ) { + + tmp1_QS = silk_LSHIFT32( (opus_int32)input[ n ], QS ); + + /* Loop over allpass sections */ + for( i = 0; i < order; i += 2 ) { + + /* Output of allpass section */ + tmp2_QS = silk_SMLAWB( state_QS[ i ], state_QS[ i + 1 ] - tmp1_QS, warping_Q16 ); + state_QS[ i ] = tmp1_QS; + corr_QC[ i ] = __builtin_mips_madd( corr_QC[ i ], tmp1_QS, state_QS[ 0 ]); + + /* Output of allpass section */ + tmp1_QS = silk_SMLAWB( state_QS[ i + 1 ], state_QS[ i + 2 ] - tmp2_QS, warping_Q16 ); + state_QS[ i + 1 ] = tmp2_QS; + corr_QC[ i+1 ] = __builtin_mips_madd( corr_QC[ i+1 ], tmp2_QS, state_QS[ 0 ]); + } + state_QS[ order ] = tmp1_QS; + corr_QC[ order ] = __builtin_mips_madd( corr_QC[ order ], tmp1_QS, state_QS[ 0 ]); + } + + temp64 = corr_QC[ 0 ]; + temp64 = __builtin_mips_shilo(temp64, val); + + lsh = silk_CLZ64( temp64 ) - 35; + lsh = silk_LIMIT( lsh, -12 - QC, 30 - QC ); + *scale = -( QC + lsh ); + silk_assert( *scale >= -30 && *scale <= 12 ); + if( lsh >= 0 ) { + for( i = 0; i < order + 1; i++ ) { + temp64 = corr_QC[ i ]; + //temp64 = __builtin_mips_shilo(temp64, val); + temp64 = (val >= 0) ? (temp64 >> val) : (temp64 << -val); + corr[ i ] = (opus_int32)silk_CHECK_FIT32( __builtin_mips_shilo( temp64, -lsh ) ); + } + } else { + for( i = 0; i < order + 1; i++ ) { + temp64 = corr_QC[ i ]; + //temp64 = __builtin_mips_shilo(temp64, val); + temp64 = (val >= 0) ? (temp64 >> val) : (temp64 << -val); + corr[ i ] = (opus_int32)silk_CHECK_FIT32( __builtin_mips_shilo( temp64, -lsh ) ); + } + } + + corr_QC[ 0 ] = __builtin_mips_shilo(corr_QC[ 0 ], val); + + silk_assert( corr_QC[ 0 ] >= 0 ); /* If breaking, decrease QC*/ +} +#endif /* __WARPED_AUTOCORRELATION_FIX_MIPSR1_H__ */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/noise_shape_analysis_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/noise_shape_analysis_FIX.c new file mode 100644 index 000000000..85fea0bf0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/noise_shape_analysis_FIX.c @@ -0,0 +1,407 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" +#include "tuning_parameters.h" + +/* Compute gain to make warped filter coefficients have a zero mean log frequency response on a */ +/* non-warped frequency scale. (So that it can be implemented with a minimum-phase monic filter.) */ +/* Note: A monic filter is one with the first coefficient equal to 1.0. In Silk we omit the first */ +/* coefficient in an array of coefficients, for monic filters. */ +static OPUS_INLINE opus_int32 warped_gain( /* gain in Q16*/ + const opus_int32 *coefs_Q24, + opus_int lambda_Q16, + opus_int order +) { + opus_int i; + opus_int32 gain_Q24; + + lambda_Q16 = -lambda_Q16; + gain_Q24 = coefs_Q24[ order - 1 ]; + for( i = order - 2; i >= 0; i-- ) { + gain_Q24 = silk_SMLAWB( coefs_Q24[ i ], gain_Q24, lambda_Q16 ); + } + gain_Q24 = silk_SMLAWB( SILK_FIX_CONST( 1.0, 24 ), gain_Q24, -lambda_Q16 ); + return silk_INVERSE32_varQ( gain_Q24, 40 ); +} + +/* Convert warped filter coefficients to monic pseudo-warped coefficients and limit maximum */ +/* amplitude of monic warped coefficients by using bandwidth expansion on the true coefficients */ +static OPUS_INLINE void limit_warped_coefs( + opus_int32 *coefs_Q24, + opus_int lambda_Q16, + opus_int32 limit_Q24, + opus_int order +) { + opus_int i, iter, ind = 0; + opus_int32 tmp, maxabs_Q24, chirp_Q16, gain_Q16; + opus_int32 nom_Q16, den_Q24; + opus_int32 limit_Q20, maxabs_Q20; + + /* Convert to monic coefficients */ + lambda_Q16 = -lambda_Q16; + for( i = order - 1; i > 0; i-- ) { + coefs_Q24[ i - 1 ] = silk_SMLAWB( coefs_Q24[ i - 1 ], coefs_Q24[ i ], lambda_Q16 ); + } + lambda_Q16 = -lambda_Q16; + nom_Q16 = silk_SMLAWB( SILK_FIX_CONST( 1.0, 16 ), -(opus_int32)lambda_Q16, lambda_Q16 ); + den_Q24 = silk_SMLAWB( SILK_FIX_CONST( 1.0, 24 ), coefs_Q24[ 0 ], lambda_Q16 ); + gain_Q16 = silk_DIV32_varQ( nom_Q16, den_Q24, 24 ); + for( i = 0; i < order; i++ ) { + coefs_Q24[ i ] = silk_SMULWW( gain_Q16, coefs_Q24[ i ] ); + } + limit_Q20 = silk_RSHIFT(limit_Q24, 4); + for( iter = 0; iter < 10; iter++ ) { + /* Find maximum absolute value */ + maxabs_Q24 = -1; + for( i = 0; i < order; i++ ) { + tmp = silk_abs_int32( coefs_Q24[ i ] ); + if( tmp > maxabs_Q24 ) { + maxabs_Q24 = tmp; + ind = i; + } + } + /* Use Q20 to avoid any overflow when multiplying by (ind + 1) later. */ + maxabs_Q20 = silk_RSHIFT(maxabs_Q24, 4); + if( maxabs_Q20 <= limit_Q20 ) { + /* Coefficients are within range - done */ + return; + } + + /* Convert back to true warped coefficients */ + for( i = 1; i < order; i++ ) { + coefs_Q24[ i - 1 ] = silk_SMLAWB( coefs_Q24[ i - 1 ], coefs_Q24[ i ], lambda_Q16 ); + } + gain_Q16 = silk_INVERSE32_varQ( gain_Q16, 32 ); + for( i = 0; i < order; i++ ) { + coefs_Q24[ i ] = silk_SMULWW( gain_Q16, coefs_Q24[ i ] ); + } + + /* Apply bandwidth expansion */ + chirp_Q16 = SILK_FIX_CONST( 0.99, 16 ) - silk_DIV32_varQ( + silk_SMULWB( maxabs_Q20 - limit_Q20, silk_SMLABB( SILK_FIX_CONST( 0.8, 10 ), SILK_FIX_CONST( 0.1, 10 ), iter ) ), + silk_MUL( maxabs_Q20, ind + 1 ), 22 ); + silk_bwexpander_32( coefs_Q24, order, chirp_Q16 ); + + /* Convert to monic warped coefficients */ + lambda_Q16 = -lambda_Q16; + for( i = order - 1; i > 0; i-- ) { + coefs_Q24[ i - 1 ] = silk_SMLAWB( coefs_Q24[ i - 1 ], coefs_Q24[ i ], lambda_Q16 ); + } + lambda_Q16 = -lambda_Q16; + nom_Q16 = silk_SMLAWB( SILK_FIX_CONST( 1.0, 16 ), -(opus_int32)lambda_Q16, lambda_Q16 ); + den_Q24 = silk_SMLAWB( SILK_FIX_CONST( 1.0, 24 ), coefs_Q24[ 0 ], lambda_Q16 ); + gain_Q16 = silk_DIV32_varQ( nom_Q16, den_Q24, 24 ); + for( i = 0; i < order; i++ ) { + coefs_Q24[ i ] = silk_SMULWW( gain_Q16, coefs_Q24[ i ] ); + } + } + silk_assert( 0 ); +} + +/* Disable MIPS version until it's updated. */ +#if 0 && defined(MIPSr1_ASM) +#include "mips/noise_shape_analysis_FIX_mipsr1.h" +#endif + +/**************************************************************/ +/* Compute noise shaping coefficients and initial gain values */ +/**************************************************************/ +#ifndef OVERRIDE_silk_noise_shape_analysis_FIX +void silk_noise_shape_analysis_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Encoder state FIX */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Encoder control FIX */ + const opus_int16 *pitch_res, /* I LPC residual from pitch analysis */ + const opus_int16 *x, /* I Input signal [ frame_length + la_shape ] */ + int arch /* I Run-time architecture */ +) +{ + silk_shape_state_FIX *psShapeSt = &psEnc->sShape; + opus_int k, i, nSamples, nSegs, Qnrg, b_Q14, warping_Q16, scale = 0; + opus_int32 SNR_adj_dB_Q7, HarmShapeGain_Q16, Tilt_Q16, tmp32; + opus_int32 nrg, log_energy_Q7, log_energy_prev_Q7, energy_variation_Q7; + opus_int32 BWExp_Q16, gain_mult_Q16, gain_add_Q16, strength_Q16, b_Q8; + opus_int32 auto_corr[ MAX_SHAPE_LPC_ORDER + 1 ]; + opus_int32 refl_coef_Q16[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 AR_Q24[ MAX_SHAPE_LPC_ORDER ]; + VARDECL( opus_int16, x_windowed ); + const opus_int16 *x_ptr, *pitch_res_ptr; + SAVE_STACK; + + /* Point to start of first LPC analysis block */ + x_ptr = x - psEnc->sCmn.la_shape; + + /****************/ + /* GAIN CONTROL */ + /****************/ + SNR_adj_dB_Q7 = psEnc->sCmn.SNR_dB_Q7; + + /* Input quality is the average of the quality in the lowest two VAD bands */ + psEncCtrl->input_quality_Q14 = ( opus_int )silk_RSHIFT( (opus_int32)psEnc->sCmn.input_quality_bands_Q15[ 0 ] + + psEnc->sCmn.input_quality_bands_Q15[ 1 ], 2 ); + + /* Coding quality level, between 0.0_Q0 and 1.0_Q0, but in Q14 */ + psEncCtrl->coding_quality_Q14 = silk_RSHIFT( silk_sigm_Q15( silk_RSHIFT_ROUND( SNR_adj_dB_Q7 - + SILK_FIX_CONST( 20.0, 7 ), 4 ) ), 1 ); + + /* Reduce coding SNR during low speech activity */ + if( psEnc->sCmn.useCBR == 0 ) { + b_Q8 = SILK_FIX_CONST( 1.0, 8 ) - psEnc->sCmn.speech_activity_Q8; + b_Q8 = silk_SMULWB( silk_LSHIFT( b_Q8, 8 ), b_Q8 ); + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, + silk_SMULBB( SILK_FIX_CONST( -BG_SNR_DECR_dB, 7 ) >> ( 4 + 1 ), b_Q8 ), /* Q11*/ + silk_SMULWB( SILK_FIX_CONST( 1.0, 14 ) + psEncCtrl->input_quality_Q14, psEncCtrl->coding_quality_Q14 ) ); /* Q12*/ + } + + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce gains for periodic signals */ + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, SILK_FIX_CONST( HARM_SNR_INCR_dB, 8 ), psEnc->LTPCorr_Q15 ); + } else { + /* For unvoiced signals and low-quality input, adjust the quality slower than SNR_dB setting */ + SNR_adj_dB_Q7 = silk_SMLAWB( SNR_adj_dB_Q7, + silk_SMLAWB( SILK_FIX_CONST( 6.0, 9 ), -SILK_FIX_CONST( 0.4, 18 ), psEnc->sCmn.SNR_dB_Q7 ), + SILK_FIX_CONST( 1.0, 14 ) - psEncCtrl->input_quality_Q14 ); + } + + /*************************/ + /* SPARSENESS PROCESSING */ + /*************************/ + /* Set quantizer offset */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Initially set to 0; may be overruled in process_gains(..) */ + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + /* Sparseness measure, based on relative fluctuations of energy per 2 milliseconds */ + nSamples = silk_LSHIFT( psEnc->sCmn.fs_kHz, 1 ); + energy_variation_Q7 = 0; + log_energy_prev_Q7 = 0; + pitch_res_ptr = pitch_res; + nSegs = silk_SMULBB( SUB_FRAME_LENGTH_MS, psEnc->sCmn.nb_subfr ) / 2; + for( k = 0; k < nSegs; k++ ) { + silk_sum_sqr_shift( &nrg, &scale, pitch_res_ptr, nSamples ); + nrg += silk_RSHIFT( nSamples, scale ); /* Q(-scale)*/ + + log_energy_Q7 = silk_lin2log( nrg ); + if( k > 0 ) { + energy_variation_Q7 += silk_abs( log_energy_Q7 - log_energy_prev_Q7 ); + } + log_energy_prev_Q7 = log_energy_Q7; + pitch_res_ptr += nSamples; + } + + /* Set quantization offset depending on sparseness measure */ + if( energy_variation_Q7 > SILK_FIX_CONST( ENERGY_VARIATION_THRESHOLD_QNT_OFFSET, 7 ) * (nSegs-1) ) { + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + psEnc->sCmn.indices.quantOffsetType = 1; + } + } + + /*******************************/ + /* Control bandwidth expansion */ + /*******************************/ + /* More BWE for signals with high prediction gain */ + strength_Q16 = silk_SMULWB( psEncCtrl->predGain_Q16, SILK_FIX_CONST( FIND_PITCH_WHITE_NOISE_FRACTION, 16 ) ); + BWExp_Q16 = silk_DIV32_varQ( SILK_FIX_CONST( BANDWIDTH_EXPANSION, 16 ), + silk_SMLAWW( SILK_FIX_CONST( 1.0, 16 ), strength_Q16, strength_Q16 ), 16 ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Slightly more warping in analysis will move quantization noise up in frequency, where it's better masked */ + warping_Q16 = silk_SMLAWB( psEnc->sCmn.warping_Q16, (opus_int32)psEncCtrl->coding_quality_Q14, SILK_FIX_CONST( 0.01, 18 ) ); + } else { + warping_Q16 = 0; + } + + /********************************************/ + /* Compute noise shaping AR coefs and gains */ + /********************************************/ + ALLOC( x_windowed, psEnc->sCmn.shapeWinLength, opus_int16 ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + /* Apply window: sine slope followed by flat part followed by cosine slope */ + opus_int shift, slope_part, flat_part; + flat_part = psEnc->sCmn.fs_kHz * 3; + slope_part = silk_RSHIFT( psEnc->sCmn.shapeWinLength - flat_part, 1 ); + + silk_apply_sine_window( x_windowed, x_ptr, 1, slope_part ); + shift = slope_part; + silk_memcpy( x_windowed + shift, x_ptr + shift, flat_part * sizeof(opus_int16) ); + shift += flat_part; + silk_apply_sine_window( x_windowed + shift, x_ptr + shift, 2, slope_part ); + + /* Update pointer: next LPC analysis block */ + x_ptr += psEnc->sCmn.subfr_length; + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Calculate warped auto correlation */ + silk_warped_autocorrelation_FIX( auto_corr, &scale, x_windowed, warping_Q16, psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder, arch ); + } else { + /* Calculate regular auto correlation */ + silk_autocorr( auto_corr, &scale, x_windowed, psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder + 1, arch ); + } + + /* Add white noise, as a fraction of energy */ + auto_corr[0] = silk_ADD32( auto_corr[0], silk_max_32( silk_SMULWB( silk_RSHIFT( auto_corr[ 0 ], 4 ), + SILK_FIX_CONST( SHAPE_WHITE_NOISE_FRACTION, 20 ) ), 1 ) ); + + /* Calculate the reflection coefficients using schur */ + nrg = silk_schur64( refl_coef_Q16, auto_corr, psEnc->sCmn.shapingLPCOrder ); + silk_assert( nrg >= 0 ); + + /* Convert reflection coefficients to prediction coefficients */ + silk_k2a_Q16( AR_Q24, refl_coef_Q16, psEnc->sCmn.shapingLPCOrder ); + + Qnrg = -scale; /* range: -12...30*/ + silk_assert( Qnrg >= -12 ); + silk_assert( Qnrg <= 30 ); + + /* Make sure that Qnrg is an even number */ + if( Qnrg & 1 ) { + Qnrg -= 1; + nrg >>= 1; + } + + tmp32 = silk_SQRT_APPROX( nrg ); + Qnrg >>= 1; /* range: -6...15*/ + + psEncCtrl->Gains_Q16[ k ] = silk_LSHIFT_SAT32( tmp32, 16 - Qnrg ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Adjust gain for warping */ + gain_mult_Q16 = warped_gain( AR_Q24, warping_Q16, psEnc->sCmn.shapingLPCOrder ); + silk_assert( psEncCtrl->Gains_Q16[ k ] > 0 ); + if( psEncCtrl->Gains_Q16[ k ] < SILK_FIX_CONST( 0.25, 16 ) ) { + psEncCtrl->Gains_Q16[ k ] = silk_SMULWW( psEncCtrl->Gains_Q16[ k ], gain_mult_Q16 ); + } else { + psEncCtrl->Gains_Q16[ k ] = silk_SMULWW( silk_RSHIFT_ROUND( psEncCtrl->Gains_Q16[ k ], 1 ), gain_mult_Q16 ); + if ( psEncCtrl->Gains_Q16[ k ] >= ( silk_int32_MAX >> 1 ) ) { + psEncCtrl->Gains_Q16[ k ] = silk_int32_MAX; + } else { + psEncCtrl->Gains_Q16[ k ] = silk_LSHIFT32( psEncCtrl->Gains_Q16[ k ], 1 ); + } + } + silk_assert( psEncCtrl->Gains_Q16[ k ] > 0 ); + } + + /* Bandwidth expansion */ + silk_bwexpander_32( AR_Q24, psEnc->sCmn.shapingLPCOrder, BWExp_Q16 ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Convert to monic warped prediction coefficients and limit absolute values */ + limit_warped_coefs( AR_Q24, warping_Q16, SILK_FIX_CONST( 3.999, 24 ), psEnc->sCmn.shapingLPCOrder ); + + /* Convert from Q24 to Q13 and store in int16 */ + for( i = 0; i < psEnc->sCmn.shapingLPCOrder; i++ ) { + psEncCtrl->AR_Q13[ k * MAX_SHAPE_LPC_ORDER + i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( AR_Q24[ i ], 11 ) ); + } + } else { + silk_LPC_fit( &psEncCtrl->AR_Q13[ k * MAX_SHAPE_LPC_ORDER ], AR_Q24, 13, 24, psEnc->sCmn.shapingLPCOrder ); + } + } + + /*****************/ + /* Gain tweaking */ + /*****************/ + /* Increase gains during low speech activity and put lower limit on gains */ + gain_mult_Q16 = silk_log2lin( -silk_SMLAWB( -SILK_FIX_CONST( 16.0, 7 ), SNR_adj_dB_Q7, SILK_FIX_CONST( 0.16, 16 ) ) ); + gain_add_Q16 = silk_log2lin( silk_SMLAWB( SILK_FIX_CONST( 16.0, 7 ), SILK_FIX_CONST( MIN_QGAIN_DB, 7 ), SILK_FIX_CONST( 0.16, 16 ) ) ); + silk_assert( gain_mult_Q16 > 0 ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains_Q16[ k ] = silk_SMULWW( psEncCtrl->Gains_Q16[ k ], gain_mult_Q16 ); + silk_assert( psEncCtrl->Gains_Q16[ k ] >= 0 ); + psEncCtrl->Gains_Q16[ k ] = silk_ADD_POS_SAT32( psEncCtrl->Gains_Q16[ k ], gain_add_Q16 ); + } + + + /************************************************/ + /* Control low-frequency shaping and noise tilt */ + /************************************************/ + /* Less low frequency shaping for noisy inputs */ + strength_Q16 = silk_MUL( SILK_FIX_CONST( LOW_FREQ_SHAPING, 4 ), silk_SMLAWB( SILK_FIX_CONST( 1.0, 12 ), + SILK_FIX_CONST( LOW_QUALITY_LOW_FREQ_SHAPING_DECR, 13 ), psEnc->sCmn.input_quality_bands_Q15[ 0 ] - SILK_FIX_CONST( 1.0, 15 ) ) ); + strength_Q16 = silk_RSHIFT( silk_MUL( strength_Q16, psEnc->sCmn.speech_activity_Q8 ), 8 ); + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce low frequencies quantization noise for periodic signals, depending on pitch lag */ + /*f = 400; freqz([1, -0.98 + 2e-4 * f], [1, -0.97 + 7e-4 * f], 2^12, Fs); axis([0, 1000, -10, 1])*/ + opus_int fs_kHz_inv = silk_DIV32_16( SILK_FIX_CONST( 0.2, 14 ), psEnc->sCmn.fs_kHz ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + b_Q14 = fs_kHz_inv + silk_DIV32_16( SILK_FIX_CONST( 3.0, 14 ), psEncCtrl->pitchL[ k ] ); + /* Pack two coefficients in one int32 */ + psEncCtrl->LF_shp_Q14[ k ] = silk_LSHIFT( SILK_FIX_CONST( 1.0, 14 ) - b_Q14 - silk_SMULWB( strength_Q16, b_Q14 ), 16 ); + psEncCtrl->LF_shp_Q14[ k ] |= (opus_uint16)( b_Q14 - SILK_FIX_CONST( 1.0, 14 ) ); + } + silk_assert( SILK_FIX_CONST( HARM_HP_NOISE_COEF, 24 ) < SILK_FIX_CONST( 0.5, 24 ) ); /* Guarantees that second argument to SMULWB() is within range of an opus_int16*/ + Tilt_Q16 = - SILK_FIX_CONST( HP_NOISE_COEF, 16 ) - + silk_SMULWB( SILK_FIX_CONST( 1.0, 16 ) - SILK_FIX_CONST( HP_NOISE_COEF, 16 ), + silk_SMULWB( SILK_FIX_CONST( HARM_HP_NOISE_COEF, 24 ), psEnc->sCmn.speech_activity_Q8 ) ); + } else { + b_Q14 = silk_DIV32_16( 21299, psEnc->sCmn.fs_kHz ); /* 1.3_Q0 = 21299_Q14*/ + /* Pack two coefficients in one int32 */ + psEncCtrl->LF_shp_Q14[ 0 ] = silk_LSHIFT( SILK_FIX_CONST( 1.0, 14 ) - b_Q14 - + silk_SMULWB( strength_Q16, silk_SMULWB( SILK_FIX_CONST( 0.6, 16 ), b_Q14 ) ), 16 ); + psEncCtrl->LF_shp_Q14[ 0 ] |= (opus_uint16)( b_Q14 - SILK_FIX_CONST( 1.0, 14 ) ); + for( k = 1; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->LF_shp_Q14[ k ] = psEncCtrl->LF_shp_Q14[ 0 ]; + } + Tilt_Q16 = -SILK_FIX_CONST( HP_NOISE_COEF, 16 ); + } + + /****************************/ + /* HARMONIC SHAPING CONTROL */ + /****************************/ + if( USE_HARM_SHAPING && psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* More harmonic noise shaping for high bitrates or noisy input */ + HarmShapeGain_Q16 = silk_SMLAWB( SILK_FIX_CONST( HARMONIC_SHAPING, 16 ), + SILK_FIX_CONST( 1.0, 16 ) - silk_SMULWB( SILK_FIX_CONST( 1.0, 18 ) - silk_LSHIFT( psEncCtrl->coding_quality_Q14, 4 ), + psEncCtrl->input_quality_Q14 ), SILK_FIX_CONST( HIGH_RATE_OR_LOW_QUALITY_HARMONIC_SHAPING, 16 ) ); + + /* Less harmonic noise shaping for less periodic signals */ + HarmShapeGain_Q16 = silk_SMULWB( silk_LSHIFT( HarmShapeGain_Q16, 1 ), + silk_SQRT_APPROX( silk_LSHIFT( psEnc->LTPCorr_Q15, 15 ) ) ); + } else { + HarmShapeGain_Q16 = 0; + } + + /*************************/ + /* Smooth over subframes */ + /*************************/ + for( k = 0; k < MAX_NB_SUBFR; k++ ) { + psShapeSt->HarmShapeGain_smth_Q16 = + silk_SMLAWB( psShapeSt->HarmShapeGain_smth_Q16, HarmShapeGain_Q16 - psShapeSt->HarmShapeGain_smth_Q16, SILK_FIX_CONST( SUBFR_SMTH_COEF, 16 ) ); + psShapeSt->Tilt_smth_Q16 = + silk_SMLAWB( psShapeSt->Tilt_smth_Q16, Tilt_Q16 - psShapeSt->Tilt_smth_Q16, SILK_FIX_CONST( SUBFR_SMTH_COEF, 16 ) ); + + psEncCtrl->HarmShapeGain_Q14[ k ] = ( opus_int )silk_RSHIFT_ROUND( psShapeSt->HarmShapeGain_smth_Q16, 2 ); + psEncCtrl->Tilt_Q14[ k ] = ( opus_int )silk_RSHIFT_ROUND( psShapeSt->Tilt_smth_Q16, 2 ); + } + RESTORE_STACK; +} +#endif /* OVERRIDE_silk_noise_shape_analysis_FIX */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/pitch_analysis_core_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/pitch_analysis_core_FIX.c new file mode 100644 index 000000000..14729046d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/pitch_analysis_core_FIX.c @@ -0,0 +1,721 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/*********************************************************** +* Pitch analyser function +********************************************************** */ +#include "SigProc_FIX.h" +#include "pitch_est_defines.h" +#include "stack_alloc.h" +#include "debug.h" +#include "pitch.h" + +#define SCRATCH_SIZE 22 +#define SF_LENGTH_4KHZ ( PE_SUBFR_LENGTH_MS * 4 ) +#define SF_LENGTH_8KHZ ( PE_SUBFR_LENGTH_MS * 8 ) +#define MIN_LAG_4KHZ ( PE_MIN_LAG_MS * 4 ) +#define MIN_LAG_8KHZ ( PE_MIN_LAG_MS * 8 ) +#define MAX_LAG_4KHZ ( PE_MAX_LAG_MS * 4 ) +#define MAX_LAG_8KHZ ( PE_MAX_LAG_MS * 8 - 1 ) +#define CSTRIDE_4KHZ ( MAX_LAG_4KHZ + 1 - MIN_LAG_4KHZ ) +#define CSTRIDE_8KHZ ( MAX_LAG_8KHZ + 3 - ( MIN_LAG_8KHZ - 2 ) ) +#define D_COMP_MIN ( MIN_LAG_8KHZ - 3 ) +#define D_COMP_MAX ( MAX_LAG_8KHZ + 4 ) +#define D_COMP_STRIDE ( D_COMP_MAX - D_COMP_MIN ) + +typedef opus_int32 silk_pe_stage3_vals[ PE_NB_STAGE3_LAGS ]; + +/************************************************************/ +/* Internally used functions */ +/************************************************************/ +static void silk_P_Ana_calc_corr_st3( + silk_pe_stage3_vals cross_corr_st3[], /* O 3 DIM correlation array */ + const opus_int16 frame[], /* I vector to correlate */ + opus_int start_lag, /* I lag offset to search around */ + opus_int sf_length, /* I length of a 5 ms subframe */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +); + +static void silk_P_Ana_calc_energy_st3( + silk_pe_stage3_vals energies_st3[], /* O 3 DIM energy array */ + const opus_int16 frame[], /* I vector to calc energy in */ + opus_int start_lag, /* I lag offset to search around */ + opus_int sf_length, /* I length of one 5 ms subframe */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +); + +/*************************************************************/ +/* FIXED POINT CORE PITCH ANALYSIS FUNCTION */ +/*************************************************************/ +opus_int silk_pitch_analysis_core( /* O Voicing estimate: 0 voiced, 1 unvoiced */ + const opus_int16 *frame_unscaled, /* I Signal of length PE_FRAME_LENGTH_MS*Fs_kHz */ + opus_int *pitch_out, /* O 4 pitch lag values */ + opus_int16 *lagIndex, /* O Lag Index */ + opus_int8 *contourIndex, /* O Pitch contour Index */ + opus_int *LTPCorr_Q15, /* I/O Normalized correlation; input: value from previous frame */ + opus_int prevLag, /* I Last lag of previous frame; set to zero is unvoiced */ + const opus_int32 search_thres1_Q16, /* I First stage threshold for lag candidates 0 - 1 */ + const opus_int search_thres2_Q13, /* I Final threshold for lag candidates 0 - 1 */ + const opus_int Fs_kHz, /* I Sample frequency (kHz) */ + const opus_int complexity, /* I Complexity setting, 0-2, where 2 is highest */ + const opus_int nb_subfr, /* I number of 5 ms subframes */ + int arch /* I Run-time architecture */ +) +{ + VARDECL( opus_int16, frame_8kHz_buf ); + VARDECL( opus_int16, frame_4kHz ); + VARDECL( opus_int16, frame_scaled ); + opus_int32 filt_state[ 6 ]; + const opus_int16 *frame, *frame_8kHz; + opus_int i, k, d, j; + VARDECL( opus_int16, C ); + VARDECL( opus_int32, xcorr32 ); + const opus_int16 *target_ptr, *basis_ptr; + opus_int32 cross_corr, normalizer, energy, energy_basis, energy_target; + opus_int d_srch[ PE_D_SRCH_LENGTH ], Cmax, length_d_srch, length_d_comp, shift; + VARDECL( opus_int16, d_comp ); + opus_int32 sum, threshold, lag_counter; + opus_int CBimax, CBimax_new, CBimax_old, lag, start_lag, end_lag, lag_new; + opus_int32 CC[ PE_NB_CBKS_STAGE2_EXT ], CCmax, CCmax_b, CCmax_new_b, CCmax_new; + VARDECL( silk_pe_stage3_vals, energies_st3 ); + VARDECL( silk_pe_stage3_vals, cross_corr_st3 ); + opus_int frame_length, frame_length_8kHz, frame_length_4kHz; + opus_int sf_length; + opus_int min_lag; + opus_int max_lag; + opus_int32 contour_bias_Q15, diff; + opus_int nb_cbk_search, cbk_size; + opus_int32 delta_lag_log2_sqr_Q7, lag_log2_Q7, prevLag_log2_Q7, prev_lag_bias_Q13; + const opus_int8 *Lag_CB_ptr; + SAVE_STACK; + + /* Check for valid sampling frequency */ + celt_assert( Fs_kHz == 8 || Fs_kHz == 12 || Fs_kHz == 16 ); + + /* Check for valid complexity setting */ + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + silk_assert( search_thres1_Q16 >= 0 && search_thres1_Q16 <= (1<<16) ); + silk_assert( search_thres2_Q13 >= 0 && search_thres2_Q13 <= (1<<13) ); + + /* Set up frame lengths max / min lag for the sampling frequency */ + frame_length = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * Fs_kHz; + frame_length_4kHz = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * 4; + frame_length_8kHz = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * 8; + sf_length = PE_SUBFR_LENGTH_MS * Fs_kHz; + min_lag = PE_MIN_LAG_MS * Fs_kHz; + max_lag = PE_MAX_LAG_MS * Fs_kHz - 1; + + /* Downscale input if necessary */ + silk_sum_sqr_shift( &energy, &shift, frame_unscaled, frame_length ); + shift += 3 - silk_CLZ32( energy ); /* at least two bits headroom */ + ALLOC( frame_scaled, frame_length, opus_int16 ); + if( shift > 0 ) { + shift = silk_RSHIFT( shift + 1, 1 ); + for( i = 0; i < frame_length; i++ ) { + frame_scaled[ i ] = silk_RSHIFT( frame_unscaled[ i ], shift ); + } + frame = frame_scaled; + } else { + frame = frame_unscaled; + } + + ALLOC( frame_8kHz_buf, ( Fs_kHz == 8 ) ? 1 : frame_length_8kHz, opus_int16 ); + /* Resample from input sampled at Fs_kHz to 8 kHz */ + if( Fs_kHz == 16 ) { + silk_memset( filt_state, 0, 2 * sizeof( opus_int32 ) ); + silk_resampler_down2( filt_state, frame_8kHz_buf, frame, frame_length ); + frame_8kHz = frame_8kHz_buf; + } else if( Fs_kHz == 12 ) { + silk_memset( filt_state, 0, 6 * sizeof( opus_int32 ) ); + silk_resampler_down2_3( filt_state, frame_8kHz_buf, frame, frame_length ); + frame_8kHz = frame_8kHz_buf; + } else { + celt_assert( Fs_kHz == 8 ); + frame_8kHz = frame; + } + + /* Decimate again to 4 kHz */ + silk_memset( filt_state, 0, 2 * sizeof( opus_int32 ) );/* Set state to zero */ + ALLOC( frame_4kHz, frame_length_4kHz, opus_int16 ); + silk_resampler_down2( filt_state, frame_4kHz, frame_8kHz, frame_length_8kHz ); + + /* Low-pass filter */ + for( i = frame_length_4kHz - 1; i > 0; i-- ) { + frame_4kHz[ i ] = silk_ADD_SAT16( frame_4kHz[ i ], frame_4kHz[ i - 1 ] ); + } + + + /****************************************************************************** + * FIRST STAGE, operating in 4 khz + ******************************************************************************/ + ALLOC( C, nb_subfr * CSTRIDE_8KHZ, opus_int16 ); + ALLOC( xcorr32, MAX_LAG_4KHZ-MIN_LAG_4KHZ+1, opus_int32 ); + silk_memset( C, 0, (nb_subfr >> 1) * CSTRIDE_4KHZ * sizeof( opus_int16 ) ); + target_ptr = &frame_4kHz[ silk_LSHIFT( SF_LENGTH_4KHZ, 2 ) ]; + for( k = 0; k < nb_subfr >> 1; k++ ) { + /* Check that we are within range of the array */ + celt_assert( target_ptr >= frame_4kHz ); + celt_assert( target_ptr + SF_LENGTH_8KHZ <= frame_4kHz + frame_length_4kHz ); + + basis_ptr = target_ptr - MIN_LAG_4KHZ; + + /* Check that we are within range of the array */ + celt_assert( basis_ptr >= frame_4kHz ); + celt_assert( basis_ptr + SF_LENGTH_8KHZ <= frame_4kHz + frame_length_4kHz ); + + celt_pitch_xcorr( target_ptr, target_ptr - MAX_LAG_4KHZ, xcorr32, SF_LENGTH_8KHZ, MAX_LAG_4KHZ - MIN_LAG_4KHZ + 1, arch ); + + /* Calculate first vector products before loop */ + cross_corr = xcorr32[ MAX_LAG_4KHZ - MIN_LAG_4KHZ ]; + normalizer = silk_inner_prod_aligned( target_ptr, target_ptr, SF_LENGTH_8KHZ, arch ); + normalizer = silk_ADD32( normalizer, silk_inner_prod_aligned( basis_ptr, basis_ptr, SF_LENGTH_8KHZ, arch ) ); + normalizer = silk_ADD32( normalizer, silk_SMULBB( SF_LENGTH_8KHZ, 4000 ) ); + + matrix_ptr( C, k, 0, CSTRIDE_4KHZ ) = + (opus_int16)silk_DIV32_varQ( cross_corr, normalizer, 13 + 1 ); /* Q13 */ + + /* From now on normalizer is computed recursively */ + for( d = MIN_LAG_4KHZ + 1; d <= MAX_LAG_4KHZ; d++ ) { + basis_ptr--; + + /* Check that we are within range of the array */ + silk_assert( basis_ptr >= frame_4kHz ); + silk_assert( basis_ptr + SF_LENGTH_8KHZ <= frame_4kHz + frame_length_4kHz ); + + cross_corr = xcorr32[ MAX_LAG_4KHZ - d ]; + + /* Add contribution of new sample and remove contribution from oldest sample */ + normalizer = silk_ADD32( normalizer, + silk_SMULBB( basis_ptr[ 0 ], basis_ptr[ 0 ] ) - + silk_SMULBB( basis_ptr[ SF_LENGTH_8KHZ ], basis_ptr[ SF_LENGTH_8KHZ ] ) ); + + matrix_ptr( C, k, d - MIN_LAG_4KHZ, CSTRIDE_4KHZ) = + (opus_int16)silk_DIV32_varQ( cross_corr, normalizer, 13 + 1 ); /* Q13 */ + } + /* Update target pointer */ + target_ptr += SF_LENGTH_8KHZ; + } + + /* Combine two subframes into single correlation measure and apply short-lag bias */ + if( nb_subfr == PE_MAX_NB_SUBFR ) { + for( i = MAX_LAG_4KHZ; i >= MIN_LAG_4KHZ; i-- ) { + sum = (opus_int32)matrix_ptr( C, 0, i - MIN_LAG_4KHZ, CSTRIDE_4KHZ ) + + (opus_int32)matrix_ptr( C, 1, i - MIN_LAG_4KHZ, CSTRIDE_4KHZ ); /* Q14 */ + sum = silk_SMLAWB( sum, sum, silk_LSHIFT( -i, 4 ) ); /* Q14 */ + C[ i - MIN_LAG_4KHZ ] = (opus_int16)sum; /* Q14 */ + } + } else { + /* Only short-lag bias */ + for( i = MAX_LAG_4KHZ; i >= MIN_LAG_4KHZ; i-- ) { + sum = silk_LSHIFT( (opus_int32)C[ i - MIN_LAG_4KHZ ], 1 ); /* Q14 */ + sum = silk_SMLAWB( sum, sum, silk_LSHIFT( -i, 4 ) ); /* Q14 */ + C[ i - MIN_LAG_4KHZ ] = (opus_int16)sum; /* Q14 */ + } + } + + /* Sort */ + length_d_srch = silk_ADD_LSHIFT32( 4, complexity, 1 ); + celt_assert( 3 * length_d_srch <= PE_D_SRCH_LENGTH ); + silk_insertion_sort_decreasing_int16( C, d_srch, CSTRIDE_4KHZ, + length_d_srch ); + + /* Escape if correlation is very low already here */ + Cmax = (opus_int)C[ 0 ]; /* Q14 */ + if( Cmax < SILK_FIX_CONST( 0.2, 14 ) ) { + silk_memset( pitch_out, 0, nb_subfr * sizeof( opus_int ) ); + *LTPCorr_Q15 = 0; + *lagIndex = 0; + *contourIndex = 0; + RESTORE_STACK; + return 1; + } + + threshold = silk_SMULWB( search_thres1_Q16, Cmax ); + for( i = 0; i < length_d_srch; i++ ) { + /* Convert to 8 kHz indices for the sorted correlation that exceeds the threshold */ + if( C[ i ] > threshold ) { + d_srch[ i ] = silk_LSHIFT( d_srch[ i ] + MIN_LAG_4KHZ, 1 ); + } else { + length_d_srch = i; + break; + } + } + celt_assert( length_d_srch > 0 ); + + ALLOC( d_comp, D_COMP_STRIDE, opus_int16 ); + for( i = D_COMP_MIN; i < D_COMP_MAX; i++ ) { + d_comp[ i - D_COMP_MIN ] = 0; + } + for( i = 0; i < length_d_srch; i++ ) { + d_comp[ d_srch[ i ] - D_COMP_MIN ] = 1; + } + + /* Convolution */ + for( i = D_COMP_MAX - 1; i >= MIN_LAG_8KHZ; i-- ) { + d_comp[ i - D_COMP_MIN ] += + d_comp[ i - 1 - D_COMP_MIN ] + d_comp[ i - 2 - D_COMP_MIN ]; + } + + length_d_srch = 0; + for( i = MIN_LAG_8KHZ; i < MAX_LAG_8KHZ + 1; i++ ) { + if( d_comp[ i + 1 - D_COMP_MIN ] > 0 ) { + d_srch[ length_d_srch ] = i; + length_d_srch++; + } + } + + /* Convolution */ + for( i = D_COMP_MAX - 1; i >= MIN_LAG_8KHZ; i-- ) { + d_comp[ i - D_COMP_MIN ] += d_comp[ i - 1 - D_COMP_MIN ] + + d_comp[ i - 2 - D_COMP_MIN ] + d_comp[ i - 3 - D_COMP_MIN ]; + } + + length_d_comp = 0; + for( i = MIN_LAG_8KHZ; i < D_COMP_MAX; i++ ) { + if( d_comp[ i - D_COMP_MIN ] > 0 ) { + d_comp[ length_d_comp ] = i - 2; + length_d_comp++; + } + } + + /********************************************************************************** + ** SECOND STAGE, operating at 8 kHz, on lag sections with high correlation + *************************************************************************************/ + + /********************************************************************************* + * Find energy of each subframe projected onto its history, for a range of delays + *********************************************************************************/ + silk_memset( C, 0, nb_subfr * CSTRIDE_8KHZ * sizeof( opus_int16 ) ); + + target_ptr = &frame_8kHz[ PE_LTP_MEM_LENGTH_MS * 8 ]; + for( k = 0; k < nb_subfr; k++ ) { + + /* Check that we are within range of the array */ + celt_assert( target_ptr >= frame_8kHz ); + celt_assert( target_ptr + SF_LENGTH_8KHZ <= frame_8kHz + frame_length_8kHz ); + + energy_target = silk_ADD32( silk_inner_prod_aligned( target_ptr, target_ptr, SF_LENGTH_8KHZ, arch ), 1 ); + for( j = 0; j < length_d_comp; j++ ) { + d = d_comp[ j ]; + basis_ptr = target_ptr - d; + + /* Check that we are within range of the array */ + silk_assert( basis_ptr >= frame_8kHz ); + silk_assert( basis_ptr + SF_LENGTH_8KHZ <= frame_8kHz + frame_length_8kHz ); + + cross_corr = silk_inner_prod_aligned( target_ptr, basis_ptr, SF_LENGTH_8KHZ, arch ); + if( cross_corr > 0 ) { + energy_basis = silk_inner_prod_aligned( basis_ptr, basis_ptr, SF_LENGTH_8KHZ, arch ); + matrix_ptr( C, k, d - ( MIN_LAG_8KHZ - 2 ), CSTRIDE_8KHZ ) = + (opus_int16)silk_DIV32_varQ( cross_corr, + silk_ADD32( energy_target, + energy_basis ), + 13 + 1 ); /* Q13 */ + } else { + matrix_ptr( C, k, d - ( MIN_LAG_8KHZ - 2 ), CSTRIDE_8KHZ ) = 0; + } + } + target_ptr += SF_LENGTH_8KHZ; + } + + /* search over lag range and lags codebook */ + /* scale factor for lag codebook, as a function of center lag */ + + CCmax = silk_int32_MIN; + CCmax_b = silk_int32_MIN; + + CBimax = 0; /* To avoid returning undefined lag values */ + lag = -1; /* To check if lag with strong enough correlation has been found */ + + if( prevLag > 0 ) { + if( Fs_kHz == 12 ) { + prevLag = silk_DIV32_16( silk_LSHIFT( prevLag, 1 ), 3 ); + } else if( Fs_kHz == 16 ) { + prevLag = silk_RSHIFT( prevLag, 1 ); + } + prevLag_log2_Q7 = silk_lin2log( (opus_int32)prevLag ); + } else { + prevLag_log2_Q7 = 0; + } + silk_assert( search_thres2_Q13 == silk_SAT16( search_thres2_Q13 ) ); + /* Set up stage 2 codebook based on number of subframes */ + if( nb_subfr == PE_MAX_NB_SUBFR ) { + cbk_size = PE_NB_CBKS_STAGE2_EXT; + Lag_CB_ptr = &silk_CB_lags_stage2[ 0 ][ 0 ]; + if( Fs_kHz == 8 && complexity > SILK_PE_MIN_COMPLEX ) { + /* If input is 8 khz use a larger codebook here because it is last stage */ + nb_cbk_search = PE_NB_CBKS_STAGE2_EXT; + } else { + nb_cbk_search = PE_NB_CBKS_STAGE2; + } + } else { + cbk_size = PE_NB_CBKS_STAGE2_10MS; + Lag_CB_ptr = &silk_CB_lags_stage2_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE2_10MS; + } + + for( k = 0; k < length_d_srch; k++ ) { + d = d_srch[ k ]; + for( j = 0; j < nb_cbk_search; j++ ) { + CC[ j ] = 0; + for( i = 0; i < nb_subfr; i++ ) { + opus_int d_subfr; + /* Try all codebooks */ + d_subfr = d + matrix_ptr( Lag_CB_ptr, i, j, cbk_size ); + CC[ j ] = CC[ j ] + + (opus_int32)matrix_ptr( C, i, + d_subfr - ( MIN_LAG_8KHZ - 2 ), + CSTRIDE_8KHZ ); + } + } + /* Find best codebook */ + CCmax_new = silk_int32_MIN; + CBimax_new = 0; + for( i = 0; i < nb_cbk_search; i++ ) { + if( CC[ i ] > CCmax_new ) { + CCmax_new = CC[ i ]; + CBimax_new = i; + } + } + + /* Bias towards shorter lags */ + lag_log2_Q7 = silk_lin2log( d ); /* Q7 */ + silk_assert( lag_log2_Q7 == silk_SAT16( lag_log2_Q7 ) ); + silk_assert( nb_subfr * SILK_FIX_CONST( PE_SHORTLAG_BIAS, 13 ) == silk_SAT16( nb_subfr * SILK_FIX_CONST( PE_SHORTLAG_BIAS, 13 ) ) ); + CCmax_new_b = CCmax_new - silk_RSHIFT( silk_SMULBB( nb_subfr * SILK_FIX_CONST( PE_SHORTLAG_BIAS, 13 ), lag_log2_Q7 ), 7 ); /* Q13 */ + + /* Bias towards previous lag */ + silk_assert( nb_subfr * SILK_FIX_CONST( PE_PREVLAG_BIAS, 13 ) == silk_SAT16( nb_subfr * SILK_FIX_CONST( PE_PREVLAG_BIAS, 13 ) ) ); + if( prevLag > 0 ) { + delta_lag_log2_sqr_Q7 = lag_log2_Q7 - prevLag_log2_Q7; + silk_assert( delta_lag_log2_sqr_Q7 == silk_SAT16( delta_lag_log2_sqr_Q7 ) ); + delta_lag_log2_sqr_Q7 = silk_RSHIFT( silk_SMULBB( delta_lag_log2_sqr_Q7, delta_lag_log2_sqr_Q7 ), 7 ); + prev_lag_bias_Q13 = silk_RSHIFT( silk_SMULBB( nb_subfr * SILK_FIX_CONST( PE_PREVLAG_BIAS, 13 ), *LTPCorr_Q15 ), 15 ); /* Q13 */ + prev_lag_bias_Q13 = silk_DIV32( silk_MUL( prev_lag_bias_Q13, delta_lag_log2_sqr_Q7 ), delta_lag_log2_sqr_Q7 + SILK_FIX_CONST( 0.5, 7 ) ); + CCmax_new_b -= prev_lag_bias_Q13; /* Q13 */ + } + + if( CCmax_new_b > CCmax_b && /* Find maximum biased correlation */ + CCmax_new > silk_SMULBB( nb_subfr, search_thres2_Q13 ) && /* Correlation needs to be high enough to be voiced */ + silk_CB_lags_stage2[ 0 ][ CBimax_new ] <= MIN_LAG_8KHZ /* Lag must be in range */ + ) { + CCmax_b = CCmax_new_b; + CCmax = CCmax_new; + lag = d; + CBimax = CBimax_new; + } + } + + if( lag == -1 ) { + /* No suitable candidate found */ + silk_memset( pitch_out, 0, nb_subfr * sizeof( opus_int ) ); + *LTPCorr_Q15 = 0; + *lagIndex = 0; + *contourIndex = 0; + RESTORE_STACK; + return 1; + } + + /* Output normalized correlation */ + *LTPCorr_Q15 = (opus_int)silk_LSHIFT( silk_DIV32_16( CCmax, nb_subfr ), 2 ); + silk_assert( *LTPCorr_Q15 >= 0 ); + + if( Fs_kHz > 8 ) { + /* Search in original signal */ + + CBimax_old = CBimax; + /* Compensate for decimation */ + silk_assert( lag == silk_SAT16( lag ) ); + if( Fs_kHz == 12 ) { + lag = silk_RSHIFT( silk_SMULBB( lag, 3 ), 1 ); + } else if( Fs_kHz == 16 ) { + lag = silk_LSHIFT( lag, 1 ); + } else { + lag = silk_SMULBB( lag, 3 ); + } + + lag = silk_LIMIT_int( lag, min_lag, max_lag ); + start_lag = silk_max_int( lag - 2, min_lag ); + end_lag = silk_min_int( lag + 2, max_lag ); + lag_new = lag; /* to avoid undefined lag */ + CBimax = 0; /* to avoid undefined lag */ + + CCmax = silk_int32_MIN; + /* pitch lags according to second stage */ + for( k = 0; k < nb_subfr; k++ ) { + pitch_out[ k ] = lag + 2 * silk_CB_lags_stage2[ k ][ CBimax_old ]; + } + + /* Set up codebook parameters according to complexity setting and frame length */ + if( nb_subfr == PE_MAX_NB_SUBFR ) { + nb_cbk_search = (opus_int)silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + } else { + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + } + + /* Calculate the correlations and energies needed in stage 3 */ + ALLOC( energies_st3, nb_subfr * nb_cbk_search, silk_pe_stage3_vals ); + ALLOC( cross_corr_st3, nb_subfr * nb_cbk_search, silk_pe_stage3_vals ); + silk_P_Ana_calc_corr_st3( cross_corr_st3, frame, start_lag, sf_length, nb_subfr, complexity, arch ); + silk_P_Ana_calc_energy_st3( energies_st3, frame, start_lag, sf_length, nb_subfr, complexity, arch ); + + lag_counter = 0; + silk_assert( lag == silk_SAT16( lag ) ); + contour_bias_Q15 = silk_DIV32_16( SILK_FIX_CONST( PE_FLATCONTOUR_BIAS, 15 ), lag ); + + target_ptr = &frame[ PE_LTP_MEM_LENGTH_MS * Fs_kHz ]; + energy_target = silk_ADD32( silk_inner_prod_aligned( target_ptr, target_ptr, nb_subfr * sf_length, arch ), 1 ); + for( d = start_lag; d <= end_lag; d++ ) { + for( j = 0; j < nb_cbk_search; j++ ) { + cross_corr = 0; + energy = energy_target; + for( k = 0; k < nb_subfr; k++ ) { + cross_corr = silk_ADD32( cross_corr, + matrix_ptr( cross_corr_st3, k, j, + nb_cbk_search )[ lag_counter ] ); + energy = silk_ADD32( energy, + matrix_ptr( energies_st3, k, j, + nb_cbk_search )[ lag_counter ] ); + silk_assert( energy >= 0 ); + } + if( cross_corr > 0 ) { + CCmax_new = silk_DIV32_varQ( cross_corr, energy, 13 + 1 ); /* Q13 */ + /* Reduce depending on flatness of contour */ + diff = silk_int16_MAX - silk_MUL( contour_bias_Q15, j ); /* Q15 */ + silk_assert( diff == silk_SAT16( diff ) ); + CCmax_new = silk_SMULWB( CCmax_new, diff ); /* Q14 */ + } else { + CCmax_new = 0; + } + + if( CCmax_new > CCmax && ( d + silk_CB_lags_stage3[ 0 ][ j ] ) <= max_lag ) { + CCmax = CCmax_new; + lag_new = d; + CBimax = j; + } + } + lag_counter++; + } + + for( k = 0; k < nb_subfr; k++ ) { + pitch_out[ k ] = lag_new + matrix_ptr( Lag_CB_ptr, k, CBimax, cbk_size ); + pitch_out[ k ] = silk_LIMIT( pitch_out[ k ], min_lag, PE_MAX_LAG_MS * Fs_kHz ); + } + *lagIndex = (opus_int16)( lag_new - min_lag); + *contourIndex = (opus_int8)CBimax; + } else { /* Fs_kHz == 8 */ + /* Save Lags */ + for( k = 0; k < nb_subfr; k++ ) { + pitch_out[ k ] = lag + matrix_ptr( Lag_CB_ptr, k, CBimax, cbk_size ); + pitch_out[ k ] = silk_LIMIT( pitch_out[ k ], MIN_LAG_8KHZ, PE_MAX_LAG_MS * 8 ); + } + *lagIndex = (opus_int16)( lag - MIN_LAG_8KHZ ); + *contourIndex = (opus_int8)CBimax; + } + celt_assert( *lagIndex >= 0 ); + /* return as voiced */ + RESTORE_STACK; + return 0; +} + +/*********************************************************************** + * Calculates the correlations used in stage 3 search. In order to cover + * the whole lag codebook for all the searched offset lags (lag +- 2), + * the following correlations are needed in each sub frame: + * + * sf1: lag range [-8,...,7] total 16 correlations + * sf2: lag range [-4,...,4] total 9 correlations + * sf3: lag range [-3,....4] total 8 correltions + * sf4: lag range [-6,....8] total 15 correlations + * + * In total 48 correlations. The direct implementation computed in worst + * case 4*12*5 = 240 correlations, but more likely around 120. + ***********************************************************************/ +static void silk_P_Ana_calc_corr_st3( + silk_pe_stage3_vals cross_corr_st3[], /* O 3 DIM correlation array */ + const opus_int16 frame[], /* I vector to correlate */ + opus_int start_lag, /* I lag offset to search around */ + opus_int sf_length, /* I length of a 5 ms subframe */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +) +{ + const opus_int16 *target_ptr; + opus_int i, j, k, lag_counter, lag_low, lag_high; + opus_int nb_cbk_search, delta, idx, cbk_size; + VARDECL( opus_int32, scratch_mem ); + VARDECL( opus_int32, xcorr32 ); + const opus_int8 *Lag_range_ptr, *Lag_CB_ptr; + SAVE_STACK; + + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_range_ptr = &silk_Lag_range_stage3[ complexity ][ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + nb_cbk_search = silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1); + Lag_range_ptr = &silk_Lag_range_stage3_10_ms[ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + } + ALLOC( scratch_mem, SCRATCH_SIZE, opus_int32 ); + ALLOC( xcorr32, SCRATCH_SIZE, opus_int32 ); + + target_ptr = &frame[ silk_LSHIFT( sf_length, 2 ) ]; /* Pointer to middle of frame */ + for( k = 0; k < nb_subfr; k++ ) { + lag_counter = 0; + + /* Calculate the correlations for each subframe */ + lag_low = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + lag_high = matrix_ptr( Lag_range_ptr, k, 1, 2 ); + celt_assert(lag_high-lag_low+1 <= SCRATCH_SIZE); + celt_pitch_xcorr( target_ptr, target_ptr - start_lag - lag_high, xcorr32, sf_length, lag_high - lag_low + 1, arch ); + for( j = lag_low; j <= lag_high; j++ ) { + silk_assert( lag_counter < SCRATCH_SIZE ); + scratch_mem[ lag_counter ] = xcorr32[ lag_high - j ]; + lag_counter++; + } + + delta = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + for( i = 0; i < nb_cbk_search; i++ ) { + /* Fill out the 3 dim array that stores the correlations for */ + /* each code_book vector for each start lag */ + idx = matrix_ptr( Lag_CB_ptr, k, i, cbk_size ) - delta; + for( j = 0; j < PE_NB_STAGE3_LAGS; j++ ) { + silk_assert( idx + j < SCRATCH_SIZE ); + silk_assert( idx + j < lag_counter ); + matrix_ptr( cross_corr_st3, k, i, nb_cbk_search )[ j ] = + scratch_mem[ idx + j ]; + } + } + target_ptr += sf_length; + } + RESTORE_STACK; +} + +/********************************************************************/ +/* Calculate the energies for first two subframes. The energies are */ +/* calculated recursively. */ +/********************************************************************/ +static void silk_P_Ana_calc_energy_st3( + silk_pe_stage3_vals energies_st3[], /* O 3 DIM energy array */ + const opus_int16 frame[], /* I vector to calc energy in */ + opus_int start_lag, /* I lag offset to search around */ + opus_int sf_length, /* I length of one 5 ms subframe */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +) +{ + const opus_int16 *target_ptr, *basis_ptr; + opus_int32 energy; + opus_int k, i, j, lag_counter; + opus_int nb_cbk_search, delta, idx, cbk_size, lag_diff; + VARDECL( opus_int32, scratch_mem ); + const opus_int8 *Lag_range_ptr, *Lag_CB_ptr; + SAVE_STACK; + + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_range_ptr = &silk_Lag_range_stage3[ complexity ][ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + nb_cbk_search = silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1); + Lag_range_ptr = &silk_Lag_range_stage3_10_ms[ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + } + ALLOC( scratch_mem, SCRATCH_SIZE, opus_int32 ); + + target_ptr = &frame[ silk_LSHIFT( sf_length, 2 ) ]; + for( k = 0; k < nb_subfr; k++ ) { + lag_counter = 0; + + /* Calculate the energy for first lag */ + basis_ptr = target_ptr - ( start_lag + matrix_ptr( Lag_range_ptr, k, 0, 2 ) ); + energy = silk_inner_prod_aligned( basis_ptr, basis_ptr, sf_length, arch ); + silk_assert( energy >= 0 ); + scratch_mem[ lag_counter ] = energy; + lag_counter++; + + lag_diff = ( matrix_ptr( Lag_range_ptr, k, 1, 2 ) - matrix_ptr( Lag_range_ptr, k, 0, 2 ) + 1 ); + for( i = 1; i < lag_diff; i++ ) { + /* remove part outside new window */ + energy -= silk_SMULBB( basis_ptr[ sf_length - i ], basis_ptr[ sf_length - i ] ); + silk_assert( energy >= 0 ); + + /* add part that comes into window */ + energy = silk_ADD_SAT32( energy, silk_SMULBB( basis_ptr[ -i ], basis_ptr[ -i ] ) ); + silk_assert( energy >= 0 ); + silk_assert( lag_counter < SCRATCH_SIZE ); + scratch_mem[ lag_counter ] = energy; + lag_counter++; + } + + delta = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + for( i = 0; i < nb_cbk_search; i++ ) { + /* Fill out the 3 dim array that stores the correlations for */ + /* each code_book vector for each start lag */ + idx = matrix_ptr( Lag_CB_ptr, k, i, cbk_size ) - delta; + for( j = 0; j < PE_NB_STAGE3_LAGS; j++ ) { + silk_assert( idx + j < SCRATCH_SIZE ); + silk_assert( idx + j < lag_counter ); + matrix_ptr( energies_st3, k, i, nb_cbk_search )[ j ] = + scratch_mem[ idx + j ]; + silk_assert( + matrix_ptr( energies_st3, k, i, nb_cbk_search )[ j ] >= 0 ); + } + } + target_ptr += sf_length; + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/process_gains_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/process_gains_FIX.c new file mode 100644 index 000000000..05aba3178 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/process_gains_FIX.c @@ -0,0 +1,117 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "tuning_parameters.h" + +/* Processing of gains */ +void silk_process_gains_FIX( + silk_encoder_state_FIX *psEnc, /* I/O Encoder state */ + silk_encoder_control_FIX *psEncCtrl, /* I/O Encoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + silk_shape_state_FIX *psShapeSt = &psEnc->sShape; + opus_int k; + opus_int32 s_Q16, InvMaxSqrVal_Q16, gain, gain_squared, ResNrg, ResNrgPart, quant_offset_Q10; + + /* Gain reduction when LTP coding gain is high */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /*s = -0.5f * silk_sigmoid( 0.25f * ( psEncCtrl->LTPredCodGain - 12.0f ) ); */ + s_Q16 = -silk_sigm_Q15( silk_RSHIFT_ROUND( psEncCtrl->LTPredCodGain_Q7 - SILK_FIX_CONST( 12.0, 7 ), 4 ) ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains_Q16[ k ] = silk_SMLAWB( psEncCtrl->Gains_Q16[ k ], psEncCtrl->Gains_Q16[ k ], s_Q16 ); + } + } + + /* Limit the quantized signal */ + /* InvMaxSqrVal = pow( 2.0f, 0.33f * ( 21.0f - SNR_dB ) ) / subfr_length; */ + InvMaxSqrVal_Q16 = silk_DIV32_16( silk_log2lin( + silk_SMULWB( SILK_FIX_CONST( 21 + 16 / 0.33, 7 ) - psEnc->sCmn.SNR_dB_Q7, SILK_FIX_CONST( 0.33, 16 ) ) ), psEnc->sCmn.subfr_length ); + + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + /* Soft limit on ratio residual energy and squared gains */ + ResNrg = psEncCtrl->ResNrg[ k ]; + ResNrgPart = silk_SMULWW( ResNrg, InvMaxSqrVal_Q16 ); + if( psEncCtrl->ResNrgQ[ k ] > 0 ) { + ResNrgPart = silk_RSHIFT_ROUND( ResNrgPart, psEncCtrl->ResNrgQ[ k ] ); + } else { + if( ResNrgPart >= silk_RSHIFT( silk_int32_MAX, -psEncCtrl->ResNrgQ[ k ] ) ) { + ResNrgPart = silk_int32_MAX; + } else { + ResNrgPart = silk_LSHIFT( ResNrgPart, -psEncCtrl->ResNrgQ[ k ] ); + } + } + gain = psEncCtrl->Gains_Q16[ k ]; + gain_squared = silk_ADD_SAT32( ResNrgPart, silk_SMMUL( gain, gain ) ); + if( gain_squared < silk_int16_MAX ) { + /* recalculate with higher precision */ + gain_squared = silk_SMLAWW( silk_LSHIFT( ResNrgPart, 16 ), gain, gain ); + silk_assert( gain_squared > 0 ); + gain = silk_SQRT_APPROX( gain_squared ); /* Q8 */ + gain = silk_min( gain, silk_int32_MAX >> 8 ); + psEncCtrl->Gains_Q16[ k ] = silk_LSHIFT_SAT32( gain, 8 ); /* Q16 */ + } else { + gain = silk_SQRT_APPROX( gain_squared ); /* Q0 */ + gain = silk_min( gain, silk_int32_MAX >> 16 ); + psEncCtrl->Gains_Q16[ k ] = silk_LSHIFT_SAT32( gain, 16 ); /* Q16 */ + } + } + + /* Save unquantized gains and gain Index */ + silk_memcpy( psEncCtrl->GainsUnq_Q16, psEncCtrl->Gains_Q16, psEnc->sCmn.nb_subfr * sizeof( opus_int32 ) ); + psEncCtrl->lastGainIndexPrev = psShapeSt->LastGainIndex; + + /* Quantize gains */ + silk_gains_quant( psEnc->sCmn.indices.GainsIndices, psEncCtrl->Gains_Q16, + &psShapeSt->LastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /* Set quantizer offset for voiced signals. Larger offset when LTP coding gain is low or tilt is high (ie low-pass) */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + if( psEncCtrl->LTPredCodGain_Q7 + silk_RSHIFT( psEnc->sCmn.input_tilt_Q15, 8 ) > SILK_FIX_CONST( 1.0, 7 ) ) { + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + psEnc->sCmn.indices.quantOffsetType = 1; + } + } + + /* Quantizer boundary adjustment */ + quant_offset_Q10 = silk_Quantization_Offsets_Q10[ psEnc->sCmn.indices.signalType >> 1 ][ psEnc->sCmn.indices.quantOffsetType ]; + psEncCtrl->Lambda_Q10 = SILK_FIX_CONST( LAMBDA_OFFSET, 10 ) + + silk_SMULBB( SILK_FIX_CONST( LAMBDA_DELAYED_DECISIONS, 10 ), psEnc->sCmn.nStatesDelayedDecision ) + + silk_SMULWB( SILK_FIX_CONST( LAMBDA_SPEECH_ACT, 18 ), psEnc->sCmn.speech_activity_Q8 ) + + silk_SMULWB( SILK_FIX_CONST( LAMBDA_INPUT_QUALITY, 12 ), psEncCtrl->input_quality_Q14 ) + + silk_SMULWB( SILK_FIX_CONST( LAMBDA_CODING_QUALITY, 12 ), psEncCtrl->coding_quality_Q14 ) + + silk_SMULWB( SILK_FIX_CONST( LAMBDA_QUANT_OFFSET, 16 ), quant_offset_Q10 ); + + silk_assert( psEncCtrl->Lambda_Q10 > 0 ); + silk_assert( psEncCtrl->Lambda_Q10 < SILK_FIX_CONST( 2, 10 ) ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/regularize_correlations_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/regularize_correlations_FIX.c new file mode 100644 index 000000000..a2836b05f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/regularize_correlations_FIX.c @@ -0,0 +1,47 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +/* Add noise to matrix diagonal */ +void silk_regularize_correlations_FIX( + opus_int32 *XX, /* I/O Correlation matrices */ + opus_int32 *xx, /* I/O Correlation values */ + opus_int32 noise, /* I Noise to add */ + opus_int D /* I Dimension of XX */ +) +{ + opus_int i; + for( i = 0; i < D; i++ ) { + matrix_ptr( &XX[ 0 ], i, i, D ) = silk_ADD32( matrix_ptr( &XX[ 0 ], i, i, D ), noise ); + } + xx[ 0 ] += noise; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy16_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy16_FIX.c new file mode 100644 index 000000000..7f130f3d3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy16_FIX.c @@ -0,0 +1,103 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +/* Residual energy: nrg = wxx - 2 * wXx * c + c' * wXX * c */ +opus_int32 silk_residual_energy16_covar_FIX( + const opus_int16 *c, /* I Prediction vector */ + const opus_int32 *wXX, /* I Correlation matrix */ + const opus_int32 *wXx, /* I Correlation vector */ + opus_int32 wxx, /* I Signal energy */ + opus_int D, /* I Dimension */ + opus_int cQ /* I Q value for c vector 0 - 15 */ +) +{ + opus_int i, j, lshifts, Qxtra; + opus_int32 c_max, w_max, tmp, tmp2, nrg; + opus_int cn[ MAX_MATRIX_SIZE ]; + const opus_int32 *pRow; + + /* Safety checks */ + celt_assert( D >= 0 ); + celt_assert( D <= 16 ); + celt_assert( cQ > 0 ); + celt_assert( cQ < 16 ); + + lshifts = 16 - cQ; + Qxtra = lshifts; + + c_max = 0; + for( i = 0; i < D; i++ ) { + c_max = silk_max_32( c_max, silk_abs( (opus_int32)c[ i ] ) ); + } + Qxtra = silk_min_int( Qxtra, silk_CLZ32( c_max ) - 17 ); + + w_max = silk_max_32( wXX[ 0 ], wXX[ D * D - 1 ] ); + Qxtra = silk_min_int( Qxtra, silk_CLZ32( silk_MUL( D, silk_RSHIFT( silk_SMULWB( w_max, c_max ), 4 ) ) ) - 5 ); + Qxtra = silk_max_int( Qxtra, 0 ); + for( i = 0; i < D; i++ ) { + cn[ i ] = silk_LSHIFT( ( opus_int )c[ i ], Qxtra ); + silk_assert( silk_abs(cn[i]) <= ( silk_int16_MAX + 1 ) ); /* Check that silk_SMLAWB can be used */ + } + lshifts -= Qxtra; + + /* Compute wxx - 2 * wXx * c */ + tmp = 0; + for( i = 0; i < D; i++ ) { + tmp = silk_SMLAWB( tmp, wXx[ i ], cn[ i ] ); + } + nrg = silk_RSHIFT( wxx, 1 + lshifts ) - tmp; /* Q: -lshifts - 1 */ + + /* Add c' * wXX * c, assuming wXX is symmetric */ + tmp2 = 0; + for( i = 0; i < D; i++ ) { + tmp = 0; + pRow = &wXX[ i * D ]; + for( j = i + 1; j < D; j++ ) { + tmp = silk_SMLAWB( tmp, pRow[ j ], cn[ j ] ); + } + tmp = silk_SMLAWB( tmp, silk_RSHIFT( pRow[ i ], 1 ), cn[ i ] ); + tmp2 = silk_SMLAWB( tmp2, tmp, cn[ i ] ); + } + nrg = silk_ADD_LSHIFT32( nrg, tmp2, lshifts ); /* Q: -lshifts - 1 */ + + /* Keep one bit free always, because we add them for LSF interpolation */ + if( nrg < 1 ) { + nrg = 1; + } else if( nrg > silk_RSHIFT( silk_int32_MAX, lshifts + 2 ) ) { + nrg = silk_int32_MAX >> 1; + } else { + nrg = silk_LSHIFT( nrg, lshifts + 1 ); /* Q0 */ + } + return nrg; + +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy_FIX.c new file mode 100644 index 000000000..6c7cade9a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/residual_energy_FIX.c @@ -0,0 +1,98 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" +#include "stack_alloc.h" + +/* Calculates residual energies of input subframes where all subframes have LPC_order */ +/* of preceding samples */ +void silk_residual_energy_FIX( + opus_int32 nrgs[ MAX_NB_SUBFR ], /* O Residual energy per subframe */ + opus_int nrgsQ[ MAX_NB_SUBFR ], /* O Q value per subframe */ + const opus_int16 x[], /* I Input signal */ + opus_int16 a_Q12[ 2 ][ MAX_LPC_ORDER ], /* I AR coefs for each frame half */ + const opus_int32 gains[ MAX_NB_SUBFR ], /* I Quantization gains */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I Number of subframes */ + const opus_int LPC_order, /* I LPC order */ + int arch /* I Run-time architecture */ +) +{ + opus_int offset, i, j, rshift, lz1, lz2; + opus_int16 *LPC_res_ptr; + VARDECL( opus_int16, LPC_res ); + const opus_int16 *x_ptr; + opus_int32 tmp32; + SAVE_STACK; + + x_ptr = x; + offset = LPC_order + subfr_length; + + /* Filter input to create the LPC residual for each frame half, and measure subframe energies */ + ALLOC( LPC_res, ( MAX_NB_SUBFR >> 1 ) * offset, opus_int16 ); + celt_assert( ( nb_subfr >> 1 ) * ( MAX_NB_SUBFR >> 1 ) == nb_subfr ); + for( i = 0; i < nb_subfr >> 1; i++ ) { + /* Calculate half frame LPC residual signal including preceding samples */ + silk_LPC_analysis_filter( LPC_res, x_ptr, a_Q12[ i ], ( MAX_NB_SUBFR >> 1 ) * offset, LPC_order, arch ); + + /* Point to first subframe of the just calculated LPC residual signal */ + LPC_res_ptr = LPC_res + LPC_order; + for( j = 0; j < ( MAX_NB_SUBFR >> 1 ); j++ ) { + /* Measure subframe energy */ + silk_sum_sqr_shift( &nrgs[ i * ( MAX_NB_SUBFR >> 1 ) + j ], &rshift, LPC_res_ptr, subfr_length ); + + /* Set Q values for the measured energy */ + nrgsQ[ i * ( MAX_NB_SUBFR >> 1 ) + j ] = -rshift; + + /* Move to next subframe */ + LPC_res_ptr += offset; + } + /* Move to next frame half */ + x_ptr += ( MAX_NB_SUBFR >> 1 ) * offset; + } + + /* Apply the squared subframe gains */ + for( i = 0; i < nb_subfr; i++ ) { + /* Fully upscale gains and energies */ + lz1 = silk_CLZ32( nrgs[ i ] ) - 1; + lz2 = silk_CLZ32( gains[ i ] ) - 1; + + tmp32 = silk_LSHIFT32( gains[ i ], lz2 ); + + /* Find squared gains */ + tmp32 = silk_SMMUL( tmp32, tmp32 ); /* Q( 2 * lz2 - 32 )*/ + + /* Scale energies */ + nrgs[ i ] = silk_SMMUL( tmp32, silk_LSHIFT32( nrgs[ i ], lz1 ) ); /* Q( nrgsQ[ i ] + lz1 + 2 * lz2 - 32 - 32 )*/ + nrgsQ[ i ] += lz1 + 2 * lz2 - 32 - 32; + } + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur64_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur64_FIX.c new file mode 100644 index 000000000..4b7e19ea5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur64_FIX.c @@ -0,0 +1,93 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Slower than schur(), but more accurate. */ +/* Uses SMULL(), available on armv4 */ +opus_int32 silk_schur64( /* O returns residual energy */ + opus_int32 rc_Q16[], /* O Reflection coefficients [order] Q16 */ + const opus_int32 c[], /* I Correlations [order+1] */ + opus_int32 order /* I Prediction order */ +) +{ + opus_int k, n; + opus_int32 C[ SILK_MAX_ORDER_LPC + 1 ][ 2 ]; + opus_int32 Ctmp1_Q30, Ctmp2_Q30, rc_tmp_Q31; + + celt_assert( order >= 0 && order <= SILK_MAX_ORDER_LPC ); + + /* Check for invalid input */ + if( c[ 0 ] <= 0 ) { + silk_memset( rc_Q16, 0, order * sizeof( opus_int32 ) ); + return 0; + } + + k = 0; + do { + C[ k ][ 0 ] = C[ k ][ 1 ] = c[ k ]; + } while( ++k <= order ); + + for( k = 0; k < order; k++ ) { + /* Check that we won't be getting an unstable rc, otherwise stop here. */ + if (silk_abs_int32(C[ k + 1 ][ 0 ]) >= C[ 0 ][ 1 ]) { + if ( C[ k + 1 ][ 0 ] > 0 ) { + rc_Q16[ k ] = -SILK_FIX_CONST( .99f, 16 ); + } else { + rc_Q16[ k ] = SILK_FIX_CONST( .99f, 16 ); + } + k++; + break; + } + + /* Get reflection coefficient: divide two Q30 values and get result in Q31 */ + rc_tmp_Q31 = silk_DIV32_varQ( -C[ k + 1 ][ 0 ], C[ 0 ][ 1 ], 31 ); + + /* Save the output */ + rc_Q16[ k ] = silk_RSHIFT_ROUND( rc_tmp_Q31, 15 ); + + /* Update correlations */ + for( n = 0; n < order - k; n++ ) { + Ctmp1_Q30 = C[ n + k + 1 ][ 0 ]; + Ctmp2_Q30 = C[ n ][ 1 ]; + + /* Multiply and add the highest int32 */ + C[ n + k + 1 ][ 0 ] = Ctmp1_Q30 + silk_SMMUL( silk_LSHIFT( Ctmp2_Q30, 1 ), rc_tmp_Q31 ); + C[ n ][ 1 ] = Ctmp2_Q30 + silk_SMMUL( silk_LSHIFT( Ctmp1_Q30, 1 ), rc_tmp_Q31 ); + } + } + + for(; k < order; k++ ) { + rc_Q16[ k ] = 0; + } + + return silk_max_32( 1, C[ 0 ][ 1 ] ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur_FIX.c new file mode 100644 index 000000000..2840f6b1a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/schur_FIX.c @@ -0,0 +1,107 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Faster than schur64(), but much less accurate. */ +/* uses SMLAWB(), requiring armv5E and higher. */ +opus_int32 silk_schur( /* O Returns residual energy */ + opus_int16 *rc_Q15, /* O reflection coefficients [order] Q15 */ + const opus_int32 *c, /* I correlations [order+1] */ + const opus_int32 order /* I prediction order */ +) +{ + opus_int k, n, lz; + opus_int32 C[ SILK_MAX_ORDER_LPC + 1 ][ 2 ]; + opus_int32 Ctmp1, Ctmp2, rc_tmp_Q15; + + celt_assert( order >= 0 && order <= SILK_MAX_ORDER_LPC ); + + /* Get number of leading zeros */ + lz = silk_CLZ32( c[ 0 ] ); + + /* Copy correlations and adjust level to Q30 */ + k = 0; + if( lz < 2 ) { + /* lz must be 1, so shift one to the right */ + do { + C[ k ][ 0 ] = C[ k ][ 1 ] = silk_RSHIFT( c[ k ], 1 ); + } while( ++k <= order ); + } else if( lz > 2 ) { + /* Shift to the left */ + lz -= 2; + do { + C[ k ][ 0 ] = C[ k ][ 1 ] = silk_LSHIFT( c[ k ], lz ); + } while( ++k <= order ); + } else { + /* No need to shift */ + do { + C[ k ][ 0 ] = C[ k ][ 1 ] = c[ k ]; + } while( ++k <= order ); + } + + for( k = 0; k < order; k++ ) { + /* Check that we won't be getting an unstable rc, otherwise stop here. */ + if (silk_abs_int32(C[ k + 1 ][ 0 ]) >= C[ 0 ][ 1 ]) { + if ( C[ k + 1 ][ 0 ] > 0 ) { + rc_Q15[ k ] = -SILK_FIX_CONST( .99f, 15 ); + } else { + rc_Q15[ k ] = SILK_FIX_CONST( .99f, 15 ); + } + k++; + break; + } + + /* Get reflection coefficient */ + rc_tmp_Q15 = -silk_DIV32_16( C[ k + 1 ][ 0 ], silk_max_32( silk_RSHIFT( C[ 0 ][ 1 ], 15 ), 1 ) ); + + /* Clip (shouldn't happen for properly conditioned inputs) */ + rc_tmp_Q15 = silk_SAT16( rc_tmp_Q15 ); + + /* Store */ + rc_Q15[ k ] = (opus_int16)rc_tmp_Q15; + + /* Update correlations */ + for( n = 0; n < order - k; n++ ) { + Ctmp1 = C[ n + k + 1 ][ 0 ]; + Ctmp2 = C[ n ][ 1 ]; + C[ n + k + 1 ][ 0 ] = silk_SMLAWB( Ctmp1, silk_LSHIFT( Ctmp2, 1 ), rc_tmp_Q15 ); + C[ n ][ 1 ] = silk_SMLAWB( Ctmp2, silk_LSHIFT( Ctmp1, 1 ), rc_tmp_Q15 ); + } + } + + for(; k < order; k++ ) { + rc_Q15[ k ] = 0; + } + + /* return residual energy */ + return silk_max_32( 1, C[ 0 ][ 1 ] ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/structs_FIX.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/structs_FIX.h new file mode 100644 index 000000000..2774a97b2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/structs_FIX.h @@ -0,0 +1,116 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_STRUCTS_FIX_H +#define SILK_STRUCTS_FIX_H + +#include "typedef.h" +#include "main.h" +#include "structs.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/********************************/ +/* Noise shaping analysis state */ +/********************************/ +typedef struct { + opus_int8 LastGainIndex; + opus_int32 HarmBoost_smth_Q16; + opus_int32 HarmShapeGain_smth_Q16; + opus_int32 Tilt_smth_Q16; +} silk_shape_state_FIX; + +/********************************/ +/* Encoder state FIX */ +/********************************/ +typedef struct { + silk_encoder_state sCmn; /* Common struct, shared with floating-point code */ + silk_shape_state_FIX sShape; /* Shape state */ + + /* Buffer for find pitch and noise shape analysis */ + silk_DWORD_ALIGN opus_int16 x_buf[ 2 * MAX_FRAME_LENGTH + LA_SHAPE_MAX ];/* Buffer for find pitch and noise shape analysis */ + opus_int LTPCorr_Q15; /* Normalized correlation from pitch lag estimator */ + opus_int32 resNrgSmth; +} silk_encoder_state_FIX; + +/************************/ +/* Encoder control FIX */ +/************************/ +typedef struct { + /* Prediction and coding parameters */ + opus_int32 Gains_Q16[ MAX_NB_SUBFR ]; + silk_DWORD_ALIGN opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ]; + opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ]; + opus_int LTP_scale_Q14; + opus_int pitchL[ MAX_NB_SUBFR ]; + + /* Noise shaping parameters */ + /* Testing */ + silk_DWORD_ALIGN opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ]; + opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ]; /* Packs two int16 coefficients per int32 value */ + opus_int Tilt_Q14[ MAX_NB_SUBFR ]; + opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ]; + opus_int Lambda_Q10; + opus_int input_quality_Q14; + opus_int coding_quality_Q14; + + /* measures */ + opus_int32 predGain_Q16; + opus_int LTPredCodGain_Q7; + opus_int32 ResNrg[ MAX_NB_SUBFR ]; /* Residual energy per subframe */ + opus_int ResNrgQ[ MAX_NB_SUBFR ]; /* Q domain for the residual energy > 0 */ + + /* Parameters for CBR mode */ + opus_int32 GainsUnq_Q16[ MAX_NB_SUBFR ]; + opus_int8 lastGainIndexPrev; +} silk_encoder_control_FIX; + +/************************/ +/* Encoder Super Struct */ +/************************/ +typedef struct { + silk_encoder_state_FIX state_Fxx[ ENCODER_NUM_CHANNELS ]; + stereo_enc_state sStereo; + opus_int32 nBitsUsedLBRR; + opus_int32 nBitsExceeded; + opus_int nChannelsAPI; + opus_int nChannelsInternal; + opus_int nPrevChannelsInternal; + opus_int timeSinceSwitchAllowed_ms; + opus_int allowBandwidthSwitch; + opus_int prev_decode_only_middle; +} silk_encoder; + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/vector_ops_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/vector_ops_FIX.c new file mode 100644 index 000000000..d94980014 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/vector_ops_FIX.c @@ -0,0 +1,102 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "pitch.h" + +/* Copy and multiply a vector by a constant */ +void silk_scale_copy_vector16( + opus_int16 *data_out, + const opus_int16 *data_in, + opus_int32 gain_Q16, /* I Gain in Q16 */ + const opus_int dataSize /* I Length */ +) +{ + opus_int i; + opus_int32 tmp32; + + for( i = 0; i < dataSize; i++ ) { + tmp32 = silk_SMULWB( gain_Q16, data_in[ i ] ); + data_out[ i ] = (opus_int16)silk_CHECK_FIT16( tmp32 ); + } +} + +/* Multiply a vector by a constant */ +void silk_scale_vector32_Q26_lshift_18( + opus_int32 *data1, /* I/O Q0/Q18 */ + opus_int32 gain_Q26, /* I Q26 */ + opus_int dataSize /* I length */ +) +{ + opus_int i; + + for( i = 0; i < dataSize; i++ ) { + data1[ i ] = (opus_int32)silk_CHECK_FIT32( silk_RSHIFT64( silk_SMULL( data1[ i ], gain_Q26 ), 8 ) ); /* OUTPUT: Q18 */ + } +} + +/* sum = for(i=0;i6, memory access can be reduced by half. */ +opus_int32 silk_inner_prod_aligned( + const opus_int16 *const inVec1, /* I input vector 1 */ + const opus_int16 *const inVec2, /* I input vector 2 */ + const opus_int len, /* I vector lengths */ + int arch /* I Run-time architecture */ +) +{ +#ifdef FIXED_POINT + return celt_inner_prod(inVec1, inVec2, len, arch); +#else + opus_int i; + opus_int32 sum = 0; + for( i = 0; i < len; i++ ) { + sum = silk_SMLABB( sum, inVec1[ i ], inVec2[ i ] ); + } + return sum; +#endif +} + +opus_int64 silk_inner_prod16_aligned_64_c( + const opus_int16 *inVec1, /* I input vector 1 */ + const opus_int16 *inVec2, /* I input vector 2 */ + const opus_int len /* I vector lengths */ +) +{ + opus_int i; + opus_int64 sum = 0; + for( i = 0; i < len; i++ ) { + sum = silk_SMLALBB( sum, inVec1[ i ], inVec2[ i ] ); + } + return sum; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/warped_autocorrelation_FIX.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/warped_autocorrelation_FIX.c new file mode 100644 index 000000000..5c79553bc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/warped_autocorrelation_FIX.c @@ -0,0 +1,92 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FIX.h" + +#if defined(MIPSr1_ASM) +#include "mips/warped_autocorrelation_FIX_mipsr1.h" +#endif + + +/* Autocorrelations for a warped frequency axis */ +#ifndef OVERRIDE_silk_warped_autocorrelation_FIX_c +void silk_warped_autocorrelation_FIX_c( + opus_int32 *corr, /* O Result [order + 1] */ + opus_int *scale, /* O Scaling of the correlation vector */ + const opus_int16 *input, /* I Input data to correlate */ + const opus_int warping_Q16, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +) +{ + opus_int n, i, lsh; + opus_int32 tmp1_QS, tmp2_QS; + opus_int32 state_QS[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + opus_int64 corr_QC[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + + /* Order must be even */ + celt_assert( ( order & 1 ) == 0 ); + silk_assert( 2 * QS - QC >= 0 ); + + /* Loop over samples */ + for( n = 0; n < length; n++ ) { + tmp1_QS = silk_LSHIFT32( (opus_int32)input[ n ], QS ); + /* Loop over allpass sections */ + for( i = 0; i < order; i += 2 ) { + /* Output of allpass section */ + tmp2_QS = silk_SMLAWB( state_QS[ i ], state_QS[ i + 1 ] - tmp1_QS, warping_Q16 ); + state_QS[ i ] = tmp1_QS; + corr_QC[ i ] += silk_RSHIFT64( silk_SMULL( tmp1_QS, state_QS[ 0 ] ), 2 * QS - QC ); + /* Output of allpass section */ + tmp1_QS = silk_SMLAWB( state_QS[ i + 1 ], state_QS[ i + 2 ] - tmp2_QS, warping_Q16 ); + state_QS[ i + 1 ] = tmp2_QS; + corr_QC[ i + 1 ] += silk_RSHIFT64( silk_SMULL( tmp2_QS, state_QS[ 0 ] ), 2 * QS - QC ); + } + state_QS[ order ] = tmp1_QS; + corr_QC[ order ] += silk_RSHIFT64( silk_SMULL( tmp1_QS, state_QS[ 0 ] ), 2 * QS - QC ); + } + + lsh = silk_CLZ64( corr_QC[ 0 ] ) - 35; + lsh = silk_LIMIT( lsh, -12 - QC, 30 - QC ); + *scale = -( QC + lsh ); + silk_assert( *scale >= -30 && *scale <= 12 ); + if( lsh >= 0 ) { + for( i = 0; i < order + 1; i++ ) { + corr[ i ] = (opus_int32)silk_CHECK_FIT32( silk_LSHIFT64( corr_QC[ i ], lsh ) ); + } + } else { + for( i = 0; i < order + 1; i++ ) { + corr[ i ] = (opus_int32)silk_CHECK_FIT32( silk_RSHIFT64( corr_QC[ i ], -lsh ) ); + } + } + silk_assert( corr_QC[ 0 ] >= 0 ); /* If breaking, decrease QC*/ +} +#endif /* OVERRIDE_silk_warped_autocorrelation_FIX_c */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/burg_modified_FIX_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/burg_modified_FIX_sse4_1.c new file mode 100644 index 000000000..bbb1ce0fc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/burg_modified_FIX_sse4_1.c @@ -0,0 +1,377 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "SigProc_FIX.h" +#include "define.h" +#include "tuning_parameters.h" +#include "pitch.h" +#include "celt/x86/x86cpu.h" + +#define MAX_FRAME_SIZE 384 /* subfr_length * nb_subfr = ( 0.005 * 16000 + 16 ) * 4 = 384 */ + +#define QA 25 +#define N_BITS_HEAD_ROOM 2 +#define MIN_RSHIFTS -16 +#define MAX_RSHIFTS (32 - QA) + +/* Compute reflection coefficients from input signal */ +void silk_burg_modified_sse4_1( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */ +) +{ + opus_int k, n, s, lz, rshifts, rshifts_extra, reached_max_gain; + opus_int32 C0, num, nrg, rc_Q31, invGain_Q30, Atmp_QA, Atmp1, tmp1, tmp2, x1, x2; + const opus_int16 *x_ptr; + opus_int32 C_first_row[ SILK_MAX_ORDER_LPC ]; + opus_int32 C_last_row[ SILK_MAX_ORDER_LPC ]; + opus_int32 Af_QA[ SILK_MAX_ORDER_LPC ]; + opus_int32 CAf[ SILK_MAX_ORDER_LPC + 1 ]; + opus_int32 CAb[ SILK_MAX_ORDER_LPC + 1 ]; + opus_int32 xcorr[ SILK_MAX_ORDER_LPC ]; + + __m128i FIRST_3210, LAST_3210, ATMP_3210, TMP1_3210, TMP2_3210, T1_3210, T2_3210, PTR_3210, SUBFR_3210, X1_3210, X2_3210; + __m128i CONST1 = _mm_set1_epi32(1); + + celt_assert( subfr_length * nb_subfr <= MAX_FRAME_SIZE ); + + /* Compute autocorrelations, added over subframes */ + silk_sum_sqr_shift( &C0, &rshifts, x, nb_subfr * subfr_length ); + if( rshifts > MAX_RSHIFTS ) { + C0 = silk_LSHIFT32( C0, rshifts - MAX_RSHIFTS ); + silk_assert( C0 > 0 ); + rshifts = MAX_RSHIFTS; + } else { + lz = silk_CLZ32( C0 ) - 1; + rshifts_extra = N_BITS_HEAD_ROOM - lz; + if( rshifts_extra > 0 ) { + rshifts_extra = silk_min( rshifts_extra, MAX_RSHIFTS - rshifts ); + C0 = silk_RSHIFT32( C0, rshifts_extra ); + } else { + rshifts_extra = silk_max( rshifts_extra, MIN_RSHIFTS - rshifts ); + C0 = silk_LSHIFT32( C0, -rshifts_extra ); + } + rshifts += rshifts_extra; + } + CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */ + silk_memset( C_first_row, 0, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) ); + if( rshifts > 0 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + for( n = 1; n < D + 1; n++ ) { + C_first_row[ n - 1 ] += (opus_int32)silk_RSHIFT64( + silk_inner_prod16_aligned_64( x_ptr, x_ptr + n, subfr_length - n, arch ), rshifts ); + } + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + int i; + opus_int32 d; + x_ptr = x + s * subfr_length; + celt_pitch_xcorr(x_ptr, x_ptr + 1, xcorr, subfr_length - D, D, arch ); + for( n = 1; n < D + 1; n++ ) { + for ( i = n + subfr_length - D, d = 0; i < subfr_length; i++ ) + d = MAC16_16( d, x_ptr[ i ], x_ptr[ i - n ] ); + xcorr[ n - 1 ] += d; + } + for( n = 1; n < D + 1; n++ ) { + C_first_row[ n - 1 ] += silk_LSHIFT32( xcorr[ n - 1 ], -rshifts ); + } + } + } + silk_memcpy( C_last_row, C_first_row, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) ); + + /* Initialize */ + CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */ + + invGain_Q30 = (opus_int32)1 << 30; + reached_max_gain = 0; + for( n = 0; n < D; n++ ) { + /* Update first row of correlation matrix (without first element) */ + /* Update last row of correlation matrix (without last element, stored in reversed order) */ + /* Update C * Af */ + /* Update C * flipud(Af) (stored in reversed order) */ + if( rshifts > -2 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], 16 - rshifts ); /* Q(16-rshifts) */ + x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 16 - rshifts ); /* Q(16-rshifts) */ + tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], QA - 16 ); /* Q(QA-16) */ + tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], QA - 16 ); /* Q(QA-16) */ + for( k = 0; k < n; k++ ) { + C_first_row[ k ] = silk_SMLAWB( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */ + C_last_row[ k ] = silk_SMLAWB( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */ + Atmp_QA = Af_QA[ k ]; + tmp1 = silk_SMLAWB( tmp1, Atmp_QA, x_ptr[ n - k - 1 ] ); /* Q(QA-16) */ + tmp2 = silk_SMLAWB( tmp2, Atmp_QA, x_ptr[ subfr_length - n + k ] ); /* Q(QA-16) */ + } + tmp1 = silk_LSHIFT32( -tmp1, 32 - QA - rshifts ); /* Q(16-rshifts) */ + tmp2 = silk_LSHIFT32( -tmp2, 32 - QA - rshifts ); /* Q(16-rshifts) */ + for( k = 0; k <= n; k++ ) { + CAf[ k ] = silk_SMLAWB( CAf[ k ], tmp1, x_ptr[ n - k ] ); /* Q( -rshift ) */ + CAb[ k ] = silk_SMLAWB( CAb[ k ], tmp2, x_ptr[ subfr_length - n + k - 1 ] ); /* Q( -rshift ) */ + } + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], -rshifts ); /* Q( -rshifts ) */ + x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], -rshifts ); /* Q( -rshifts ) */ + tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], 17 ); /* Q17 */ + tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 17 ); /* Q17 */ + + X1_3210 = _mm_set1_epi32( x1 ); + X2_3210 = _mm_set1_epi32( x2 ); + TMP1_3210 = _mm_setzero_si128(); + TMP2_3210 = _mm_setzero_si128(); + for( k = 0; k < n - 3; k += 4 ) { + PTR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 1 - 3 ] ); + SUBFR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k ] ); + FIRST_3210 = _mm_loadu_si128( (__m128i *)&C_first_row[ k ] ); + PTR_3210 = _mm_shuffle_epi32( PTR_3210, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + LAST_3210 = _mm_loadu_si128( (__m128i *)&C_last_row[ k ] ); + ATMP_3210 = _mm_loadu_si128( (__m128i *)&Af_QA[ k ] ); + + T1_3210 = _mm_mullo_epi32( PTR_3210, X1_3210 ); + T2_3210 = _mm_mullo_epi32( SUBFR_3210, X2_3210 ); + + ATMP_3210 = _mm_srai_epi32( ATMP_3210, 7 ); + ATMP_3210 = _mm_add_epi32( ATMP_3210, CONST1 ); + ATMP_3210 = _mm_srai_epi32( ATMP_3210, 1 ); + + FIRST_3210 = _mm_add_epi32( FIRST_3210, T1_3210 ); + LAST_3210 = _mm_add_epi32( LAST_3210, T2_3210 ); + + PTR_3210 = _mm_mullo_epi32( ATMP_3210, PTR_3210 ); + SUBFR_3210 = _mm_mullo_epi32( ATMP_3210, SUBFR_3210 ); + + _mm_storeu_si128( (__m128i *)&C_first_row[ k ], FIRST_3210 ); + _mm_storeu_si128( (__m128i *)&C_last_row[ k ], LAST_3210 ); + + TMP1_3210 = _mm_add_epi32( TMP1_3210, PTR_3210 ); + TMP2_3210 = _mm_add_epi32( TMP2_3210, SUBFR_3210 ); + } + + TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_unpackhi_epi64(TMP1_3210, TMP1_3210 ) ); + TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_unpackhi_epi64(TMP2_3210, TMP2_3210 ) ); + TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_shufflelo_epi16(TMP1_3210, 0x0E ) ); + TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_shufflelo_epi16(TMP2_3210, 0x0E ) ); + + tmp1 += _mm_cvtsi128_si32( TMP1_3210 ); + tmp2 += _mm_cvtsi128_si32( TMP2_3210 ); + + for( ; k < n; k++ ) { + C_first_row[ k ] = silk_MLA( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */ + C_last_row[ k ] = silk_MLA( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */ + Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 17 ); /* Q17 */ + tmp1 = silk_MLA( tmp1, x_ptr[ n - k - 1 ], Atmp1 ); /* Q17 */ + tmp2 = silk_MLA( tmp2, x_ptr[ subfr_length - n + k ], Atmp1 ); /* Q17 */ + } + + tmp1 = -tmp1; /* Q17 */ + tmp2 = -tmp2; /* Q17 */ + + { + __m128i xmm_tmp1, xmm_tmp2; + __m128i xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1; + __m128i xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1; + + xmm_tmp1 = _mm_set1_epi32( tmp1 ); + xmm_tmp2 = _mm_set1_epi32( tmp2 ); + + for( k = 0; k <= n - 3; k += 4 ) { + xmm_x_ptr_n_k_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 3 ] ); + xmm_x_ptr_sub_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k - 1 ] ); + + xmm_x_ptr_n_k_x2x0 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + + xmm_x_ptr_n_k_x2x0 = _mm_slli_epi32( xmm_x_ptr_n_k_x2x0, -rshifts - 1 ); + xmm_x_ptr_sub_x2x0 = _mm_slli_epi32( xmm_x_ptr_sub_x2x0, -rshifts - 1 ); + + /* equal shift right 4 bytes, xmm_x_ptr_n_k_x3x1 = _mm_srli_si128(xmm_x_ptr_n_k_x2x0, 4)*/ + xmm_x_ptr_n_k_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + xmm_x_ptr_sub_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_sub_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_x_ptr_n_k_x2x0 = _mm_mul_epi32( xmm_x_ptr_n_k_x2x0, xmm_tmp1 ); + xmm_x_ptr_n_k_x3x1 = _mm_mul_epi32( xmm_x_ptr_n_k_x3x1, xmm_tmp1 ); + xmm_x_ptr_sub_x2x0 = _mm_mul_epi32( xmm_x_ptr_sub_x2x0, xmm_tmp2 ); + xmm_x_ptr_sub_x3x1 = _mm_mul_epi32( xmm_x_ptr_sub_x3x1, xmm_tmp2 ); + + xmm_x_ptr_n_k_x2x0 = _mm_srli_epi64( xmm_x_ptr_n_k_x2x0, 16 ); + xmm_x_ptr_n_k_x3x1 = _mm_slli_epi64( xmm_x_ptr_n_k_x3x1, 16 ); + xmm_x_ptr_sub_x2x0 = _mm_srli_epi64( xmm_x_ptr_sub_x2x0, 16 ); + xmm_x_ptr_sub_x3x1 = _mm_slli_epi64( xmm_x_ptr_sub_x3x1, 16 ); + + xmm_x_ptr_n_k_x2x0 = _mm_blend_epi16( xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1, 0xCC ); + xmm_x_ptr_sub_x2x0 = _mm_blend_epi16( xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1, 0xCC ); + + X1_3210 = _mm_loadu_si128( (__m128i *)&CAf[ k ] ); + PTR_3210 = _mm_loadu_si128( (__m128i *)&CAb[ k ] ); + + X1_3210 = _mm_add_epi32( X1_3210, xmm_x_ptr_n_k_x2x0 ); + PTR_3210 = _mm_add_epi32( PTR_3210, xmm_x_ptr_sub_x2x0 ); + + _mm_storeu_si128( (__m128i *)&CAf[ k ], X1_3210 ); + _mm_storeu_si128( (__m128i *)&CAb[ k ], PTR_3210 ); + } + + for( ; k <= n; k++ ) { + CAf[ k ] = silk_SMLAWW( CAf[ k ], tmp1, + silk_LSHIFT32( (opus_int32)x_ptr[ n - k ], -rshifts - 1 ) ); /* Q( -rshift ) */ + CAb[ k ] = silk_SMLAWW( CAb[ k ], tmp2, + silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n + k - 1 ], -rshifts - 1 ) ); /* Q( -rshift ) */ + } + } + } + } + + /* Calculate nominator and denominator for the next order reflection (parcor) coefficient */ + tmp1 = C_first_row[ n ]; /* Q( -rshifts ) */ + tmp2 = C_last_row[ n ]; /* Q( -rshifts ) */ + num = 0; /* Q( -rshifts ) */ + nrg = silk_ADD32( CAb[ 0 ], CAf[ 0 ] ); /* Q( 1-rshifts ) */ + for( k = 0; k < n; k++ ) { + Atmp_QA = Af_QA[ k ]; + lz = silk_CLZ32( silk_abs( Atmp_QA ) ) - 1; + lz = silk_min( 32 - QA, lz ); + Atmp1 = silk_LSHIFT32( Atmp_QA, lz ); /* Q( QA + lz ) */ + + tmp1 = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( C_last_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + tmp2 = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( C_first_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + num = silk_ADD_LSHIFT32( num, silk_SMMUL( CAb[ n - k ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */ + nrg = silk_ADD_LSHIFT32( nrg, silk_SMMUL( silk_ADD32( CAb[ k + 1 ], CAf[ k + 1 ] ), + Atmp1 ), 32 - QA - lz ); /* Q( 1-rshifts ) */ + } + CAf[ n + 1 ] = tmp1; /* Q( -rshifts ) */ + CAb[ n + 1 ] = tmp2; /* Q( -rshifts ) */ + num = silk_ADD32( num, tmp2 ); /* Q( -rshifts ) */ + num = silk_LSHIFT32( -num, 1 ); /* Q( 1-rshifts ) */ + + /* Calculate the next order reflection (parcor) coefficient */ + if( silk_abs( num ) < nrg ) { + rc_Q31 = silk_DIV32_varQ( num, nrg, 31 ); + } else { + rc_Q31 = ( num > 0 ) ? silk_int32_MAX : silk_int32_MIN; + } + + /* Update inverse prediction gain */ + tmp1 = ( (opus_int32)1 << 30 ) - silk_SMMUL( rc_Q31, rc_Q31 ); + tmp1 = silk_LSHIFT( silk_SMMUL( invGain_Q30, tmp1 ), 2 ); + if( tmp1 <= minInvGain_Q30 ) { + /* Max prediction gain exceeded; set reflection coefficient such that max prediction gain is exactly hit */ + tmp2 = ( (opus_int32)1 << 30 ) - silk_DIV32_varQ( minInvGain_Q30, invGain_Q30, 30 ); /* Q30 */ + rc_Q31 = silk_SQRT_APPROX( tmp2 ); /* Q15 */ + if( rc_Q31 > 0 ) { + /* Newton-Raphson iteration */ + rc_Q31 = silk_RSHIFT32( rc_Q31 + silk_DIV32( tmp2, rc_Q31 ), 1 ); /* Q15 */ + rc_Q31 = silk_LSHIFT32( rc_Q31, 16 ); /* Q31 */ + if( num < 0 ) { + /* Ensure adjusted reflection coefficients has the original sign */ + rc_Q31 = -rc_Q31; + } + } + invGain_Q30 = minInvGain_Q30; + reached_max_gain = 1; + } else { + invGain_Q30 = tmp1; + } + + /* Update the AR coefficients */ + for( k = 0; k < (n + 1) >> 1; k++ ) { + tmp1 = Af_QA[ k ]; /* QA */ + tmp2 = Af_QA[ n - k - 1 ]; /* QA */ + Af_QA[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* QA */ + Af_QA[ n - k - 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* QA */ + } + Af_QA[ n ] = silk_RSHIFT32( rc_Q31, 31 - QA ); /* QA */ + + if( reached_max_gain ) { + /* Reached max prediction gain; set remaining coefficients to zero and exit loop */ + for( k = n + 1; k < D; k++ ) { + Af_QA[ k ] = 0; + } + break; + } + + /* Update C * Af and C * Ab */ + for( k = 0; k <= n + 1; k++ ) { + tmp1 = CAf[ k ]; /* Q( -rshifts ) */ + tmp2 = CAb[ n - k + 1 ]; /* Q( -rshifts ) */ + CAf[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* Q( -rshifts ) */ + CAb[ n - k + 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* Q( -rshifts ) */ + } + } + + if( reached_max_gain ) { + for( k = 0; k < D; k++ ) { + /* Scale coefficients */ + A_Q16[ k ] = -silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); + } + /* Subtract energy of preceding samples from C0 */ + if( rshifts > 0 ) { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + C0 -= (opus_int32)silk_RSHIFT64( silk_inner_prod16_aligned_64( x_ptr, x_ptr, D, arch ), rshifts ); + } + } else { + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + C0 -= silk_LSHIFT32( silk_inner_prod_aligned( x_ptr, x_ptr, D, arch ), -rshifts ); + } + } + /* Approximate residual energy */ + *res_nrg = silk_LSHIFT( silk_SMMUL( invGain_Q30, C0 ), 2 ); + *res_nrg_Q = -rshifts; + } else { + /* Return residual energy */ + nrg = CAf[ 0 ]; /* Q( -rshifts ) */ + tmp1 = (opus_int32)1 << 16; /* Q16 */ + for( k = 0; k < D; k++ ) { + Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); /* Q16 */ + nrg = silk_SMLAWW( nrg, CAf[ k + 1 ], Atmp1 ); /* Q( -rshifts ) */ + tmp1 = silk_SMLAWW( tmp1, Atmp1, Atmp1 ); /* Q16 */ + A_Q16[ k ] = -Atmp1; + } + *res_nrg = silk_SMLAWW( nrg, silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ), -tmp1 );/* Q( -rshifts ) */ + *res_nrg_Q = -rshifts; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/prefilter_FIX_sse.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/prefilter_FIX_sse.c new file mode 100644 index 000000000..555432cd9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/prefilter_FIX_sse.c @@ -0,0 +1,160 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "main.h" +#include "celt/x86/x86cpu.h" + +void silk_warped_LPC_analysis_filter_FIX_sse4_1( + opus_int32 state[], /* I/O State [order + 1] */ + opus_int32 res_Q2[], /* O Residual signal [length] */ + const opus_int16 coef_Q13[], /* I Coefficients [order] */ + const opus_int16 input[], /* I Input signal [length] */ + const opus_int16 lambda_Q16, /* I Warping factor */ + const opus_int length, /* I Length of input signal */ + const opus_int order /* I Filter order (even) */ +) +{ + opus_int n, i; + opus_int32 acc_Q11, tmp1, tmp2; + + /* Order must be even */ + celt_assert( ( order & 1 ) == 0 ); + + if (order == 10) + { + if (0 == lambda_Q16) + { + __m128i coef_Q13_3210, coef_Q13_7654; + __m128i coef_Q13_0123, coef_Q13_4567; + __m128i state_0123, state_4567; + __m128i xmm_product1, xmm_product2; + __m128i xmm_tempa, xmm_tempb; + + register opus_int32 sum; + register opus_int32 state_8, state_9, state_a; + register opus_int64 coef_Q13_8, coef_Q13_9; + + celt_assert( length > 0 ); + + coef_Q13_3210 = OP_CVTEPI16_EPI32_M64( &coef_Q13[ 0 ] ); + coef_Q13_7654 = OP_CVTEPI16_EPI32_M64( &coef_Q13[ 4 ] ); + + coef_Q13_0123 = _mm_shuffle_epi32( coef_Q13_3210, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + coef_Q13_4567 = _mm_shuffle_epi32( coef_Q13_7654, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + + coef_Q13_8 = (opus_int64) coef_Q13[ 8 ]; + coef_Q13_9 = (opus_int64) coef_Q13[ 9 ]; + + state_0123 = _mm_loadu_si128( (__m128i *)(&state[ 0 ] ) ); + state_4567 = _mm_loadu_si128( (__m128i *)(&state[ 4 ] ) ); + + state_0123 = _mm_shuffle_epi32( state_0123, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + state_4567 = _mm_shuffle_epi32( state_4567, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + + state_8 = state[ 8 ]; + state_9 = state[ 9 ]; + state_a = 0; + + for( n = 0; n < length; n++ ) + { + xmm_product1 = _mm_mul_epi32( coef_Q13_0123, state_0123 ); /* 64-bit multiply, only 2 pairs */ + xmm_product2 = _mm_mul_epi32( coef_Q13_4567, state_4567 ); + + xmm_tempa = _mm_shuffle_epi32( state_0123, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + xmm_tempb = _mm_shuffle_epi32( state_4567, _MM_SHUFFLE( 0, 1, 2, 3 ) ); + + xmm_product1 = _mm_srli_epi64( xmm_product1, 16 ); /* >> 16, zero extending works */ + xmm_product2 = _mm_srli_epi64( xmm_product2, 16 ); + + xmm_tempa = _mm_mul_epi32( coef_Q13_3210, xmm_tempa ); + xmm_tempb = _mm_mul_epi32( coef_Q13_7654, xmm_tempb ); + + xmm_tempa = _mm_srli_epi64( xmm_tempa, 16 ); + xmm_tempb = _mm_srli_epi64( xmm_tempb, 16 ); + + xmm_tempa = _mm_add_epi32( xmm_tempa, xmm_product1 ); + xmm_tempb = _mm_add_epi32( xmm_tempb, xmm_product2 ); + xmm_tempa = _mm_add_epi32( xmm_tempa, xmm_tempb ); + + sum = (opus_int32)((coef_Q13_8 * state_8) >> 16); + sum += (opus_int32)((coef_Q13_9 * state_9) >> 16); + + xmm_tempa = _mm_add_epi32( xmm_tempa, _mm_shuffle_epi32( xmm_tempa, _MM_SHUFFLE( 0, 0, 0, 2 ) ) ); + sum += _mm_cvtsi128_si32( xmm_tempa); + res_Q2[ n ] = silk_LSHIFT( (opus_int32)input[ n ], 2 ) - silk_RSHIFT_ROUND( ( 5 + sum ), 9); + + /* move right */ + state_a = state_9; + state_9 = state_8; + state_8 = _mm_cvtsi128_si32( state_4567 ); + state_4567 = _mm_alignr_epi8( state_0123, state_4567, 4 ); + + state_0123 = _mm_alignr_epi8( _mm_cvtsi32_si128( silk_LSHIFT( input[ n ], 14 ) ), state_0123, 4 ); + } + + _mm_storeu_si128( (__m128i *)( &state[ 0 ] ), _mm_shuffle_epi32( state_0123, _MM_SHUFFLE( 0, 1, 2, 3 ) ) ); + _mm_storeu_si128( (__m128i *)( &state[ 4 ] ), _mm_shuffle_epi32( state_4567, _MM_SHUFFLE( 0, 1, 2, 3 ) ) ); + state[ 8 ] = state_8; + state[ 9 ] = state_9; + state[ 10 ] = state_a; + + return; + } + } + + for( n = 0; n < length; n++ ) { + /* Output of lowpass section */ + tmp2 = silk_SMLAWB( state[ 0 ], state[ 1 ], lambda_Q16 ); + state[ 0 ] = silk_LSHIFT( input[ n ], 14 ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( state[ 1 ], state[ 2 ] - tmp2, lambda_Q16 ); + state[ 1 ] = tmp2; + acc_Q11 = silk_RSHIFT( order, 1 ); + acc_Q11 = silk_SMLAWB( acc_Q11, tmp2, coef_Q13[ 0 ] ); + /* Loop over allpass sections */ + for( i = 2; i < order; i += 2 ) { + /* Output of allpass section */ + tmp2 = silk_SMLAWB( state[ i ], state[ i + 1 ] - tmp1, lambda_Q16 ); + state[ i ] = tmp1; + acc_Q11 = silk_SMLAWB( acc_Q11, tmp1, coef_Q13[ i - 1 ] ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( state[ i + 1 ], state[ i + 2 ] - tmp2, lambda_Q16 ); + state[ i + 1 ] = tmp2; + acc_Q11 = silk_SMLAWB( acc_Q11, tmp2, coef_Q13[ i ] ); + } + state[ order ] = tmp1; + acc_Q11 = silk_SMLAWB( acc_Q11, tmp1, coef_Q13[ order - 1 ] ); + res_Q2[ n ] = silk_LSHIFT( (opus_int32)input[ n ], 2 ) - silk_RSHIFT_ROUND( acc_Q11, 9 ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/vector_ops_FIX_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/vector_ops_FIX_sse4_1.c new file mode 100644 index 000000000..c1e90564d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/fixed/x86/vector_ops_FIX_sse4_1.c @@ -0,0 +1,88 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "main.h" + +#include "SigProc_FIX.h" +#include "pitch.h" + +opus_int64 silk_inner_prod16_aligned_64_sse4_1( + const opus_int16 *inVec1, /* I input vector 1 */ + const opus_int16 *inVec2, /* I input vector 2 */ + const opus_int len /* I vector lengths */ +) +{ + opus_int i, dataSize8; + opus_int64 sum; + + __m128i xmm_tempa; + __m128i inVec1_76543210, acc1; + __m128i inVec2_76543210, acc2; + + sum = 0; + dataSize8 = len & ~7; + + acc1 = _mm_setzero_si128(); + acc2 = _mm_setzero_si128(); + + for( i = 0; i < dataSize8; i += 8 ) { + inVec1_76543210 = _mm_loadu_si128( (__m128i *)(&inVec1[i + 0] ) ); + inVec2_76543210 = _mm_loadu_si128( (__m128i *)(&inVec2[i + 0] ) ); + + /* only when all 4 operands are -32768 (0x8000), this results in wrap around */ + inVec1_76543210 = _mm_madd_epi16( inVec1_76543210, inVec2_76543210 ); + + xmm_tempa = _mm_cvtepi32_epi64( inVec1_76543210 ); + /* equal shift right 8 bytes */ + inVec1_76543210 = _mm_shuffle_epi32( inVec1_76543210, _MM_SHUFFLE( 0, 0, 3, 2 ) ); + inVec1_76543210 = _mm_cvtepi32_epi64( inVec1_76543210 ); + + acc1 = _mm_add_epi64( acc1, xmm_tempa ); + acc2 = _mm_add_epi64( acc2, inVec1_76543210 ); + } + + acc1 = _mm_add_epi64( acc1, acc2 ); + + /* equal shift right 8 bytes */ + acc2 = _mm_shuffle_epi32( acc1, _MM_SHUFFLE( 0, 0, 3, 2 ) ); + acc1 = _mm_add_epi64( acc1, acc2 ); + + _mm_storel_epi64( (__m128i *)&sum, acc1 ); + + for( ; i < len; i++ ) { + sum = silk_SMLABB( sum, inVec1[ i ], inVec2[ i ] ); + } + + return sum; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_analysis_filter_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_analysis_filter_FLP.c new file mode 100644 index 000000000..0e1a1fed0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_analysis_filter_FLP.c @@ -0,0 +1,249 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "main_FLP.h" + +/************************************************/ +/* LPC analysis filter */ +/* NB! State is kept internally and the */ +/* filter always starts with zero state */ +/* first Order output samples are set to zero */ +/************************************************/ + +/* 16th order LPC analysis filter, does not write first 16 samples */ +static OPUS_INLINE void silk_LPC_analysis_filter16_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length /* I Length of input signal */ +) +{ + opus_int ix; + silk_float LPC_pred; + const silk_float *s_ptr; + + for( ix = 16; ix < length; ix++ ) { + s_ptr = &s[ix - 1]; + + /* short-term prediction */ + LPC_pred = s_ptr[ 0 ] * PredCoef[ 0 ] + + s_ptr[ -1 ] * PredCoef[ 1 ] + + s_ptr[ -2 ] * PredCoef[ 2 ] + + s_ptr[ -3 ] * PredCoef[ 3 ] + + s_ptr[ -4 ] * PredCoef[ 4 ] + + s_ptr[ -5 ] * PredCoef[ 5 ] + + s_ptr[ -6 ] * PredCoef[ 6 ] + + s_ptr[ -7 ] * PredCoef[ 7 ] + + s_ptr[ -8 ] * PredCoef[ 8 ] + + s_ptr[ -9 ] * PredCoef[ 9 ] + + s_ptr[ -10 ] * PredCoef[ 10 ] + + s_ptr[ -11 ] * PredCoef[ 11 ] + + s_ptr[ -12 ] * PredCoef[ 12 ] + + s_ptr[ -13 ] * PredCoef[ 13 ] + + s_ptr[ -14 ] * PredCoef[ 14 ] + + s_ptr[ -15 ] * PredCoef[ 15 ]; + + /* prediction error */ + r_LPC[ix] = s_ptr[ 1 ] - LPC_pred; + } +} + +/* 12th order LPC analysis filter, does not write first 12 samples */ +static OPUS_INLINE void silk_LPC_analysis_filter12_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length /* I Length of input signal */ +) +{ + opus_int ix; + silk_float LPC_pred; + const silk_float *s_ptr; + + for( ix = 12; ix < length; ix++ ) { + s_ptr = &s[ix - 1]; + + /* short-term prediction */ + LPC_pred = s_ptr[ 0 ] * PredCoef[ 0 ] + + s_ptr[ -1 ] * PredCoef[ 1 ] + + s_ptr[ -2 ] * PredCoef[ 2 ] + + s_ptr[ -3 ] * PredCoef[ 3 ] + + s_ptr[ -4 ] * PredCoef[ 4 ] + + s_ptr[ -5 ] * PredCoef[ 5 ] + + s_ptr[ -6 ] * PredCoef[ 6 ] + + s_ptr[ -7 ] * PredCoef[ 7 ] + + s_ptr[ -8 ] * PredCoef[ 8 ] + + s_ptr[ -9 ] * PredCoef[ 9 ] + + s_ptr[ -10 ] * PredCoef[ 10 ] + + s_ptr[ -11 ] * PredCoef[ 11 ]; + + /* prediction error */ + r_LPC[ix] = s_ptr[ 1 ] - LPC_pred; + } +} + +/* 10th order LPC analysis filter, does not write first 10 samples */ +static OPUS_INLINE void silk_LPC_analysis_filter10_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length /* I Length of input signal */ +) +{ + opus_int ix; + silk_float LPC_pred; + const silk_float *s_ptr; + + for( ix = 10; ix < length; ix++ ) { + s_ptr = &s[ix - 1]; + + /* short-term prediction */ + LPC_pred = s_ptr[ 0 ] * PredCoef[ 0 ] + + s_ptr[ -1 ] * PredCoef[ 1 ] + + s_ptr[ -2 ] * PredCoef[ 2 ] + + s_ptr[ -3 ] * PredCoef[ 3 ] + + s_ptr[ -4 ] * PredCoef[ 4 ] + + s_ptr[ -5 ] * PredCoef[ 5 ] + + s_ptr[ -6 ] * PredCoef[ 6 ] + + s_ptr[ -7 ] * PredCoef[ 7 ] + + s_ptr[ -8 ] * PredCoef[ 8 ] + + s_ptr[ -9 ] * PredCoef[ 9 ]; + + /* prediction error */ + r_LPC[ix] = s_ptr[ 1 ] - LPC_pred; + } +} + +/* 8th order LPC analysis filter, does not write first 8 samples */ +static OPUS_INLINE void silk_LPC_analysis_filter8_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length /* I Length of input signal */ +) +{ + opus_int ix; + silk_float LPC_pred; + const silk_float *s_ptr; + + for( ix = 8; ix < length; ix++ ) { + s_ptr = &s[ix - 1]; + + /* short-term prediction */ + LPC_pred = s_ptr[ 0 ] * PredCoef[ 0 ] + + s_ptr[ -1 ] * PredCoef[ 1 ] + + s_ptr[ -2 ] * PredCoef[ 2 ] + + s_ptr[ -3 ] * PredCoef[ 3 ] + + s_ptr[ -4 ] * PredCoef[ 4 ] + + s_ptr[ -5 ] * PredCoef[ 5 ] + + s_ptr[ -6 ] * PredCoef[ 6 ] + + s_ptr[ -7 ] * PredCoef[ 7 ]; + + /* prediction error */ + r_LPC[ix] = s_ptr[ 1 ] - LPC_pred; + } +} + +/* 6th order LPC analysis filter, does not write first 6 samples */ +static OPUS_INLINE void silk_LPC_analysis_filter6_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length /* I Length of input signal */ +) +{ + opus_int ix; + silk_float LPC_pred; + const silk_float *s_ptr; + + for( ix = 6; ix < length; ix++ ) { + s_ptr = &s[ix - 1]; + + /* short-term prediction */ + LPC_pred = s_ptr[ 0 ] * PredCoef[ 0 ] + + s_ptr[ -1 ] * PredCoef[ 1 ] + + s_ptr[ -2 ] * PredCoef[ 2 ] + + s_ptr[ -3 ] * PredCoef[ 3 ] + + s_ptr[ -4 ] * PredCoef[ 4 ] + + s_ptr[ -5 ] * PredCoef[ 5 ]; + + /* prediction error */ + r_LPC[ix] = s_ptr[ 1 ] - LPC_pred; + } +} + +/************************************************/ +/* LPC analysis filter */ +/* NB! State is kept internally and the */ +/* filter always starts with zero state */ +/* first Order output samples are set to zero */ +/************************************************/ +void silk_LPC_analysis_filter_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length, /* I Length of input signal */ + const opus_int Order /* I LPC order */ +) +{ + celt_assert( Order <= length ); + + switch( Order ) { + case 6: + silk_LPC_analysis_filter6_FLP( r_LPC, PredCoef, s, length ); + break; + + case 8: + silk_LPC_analysis_filter8_FLP( r_LPC, PredCoef, s, length ); + break; + + case 10: + silk_LPC_analysis_filter10_FLP( r_LPC, PredCoef, s, length ); + break; + + case 12: + silk_LPC_analysis_filter12_FLP( r_LPC, PredCoef, s, length ); + break; + + case 16: + silk_LPC_analysis_filter16_FLP( r_LPC, PredCoef, s, length ); + break; + + default: + celt_assert( 0 ); + break; + } + + /* Set first Order output samples to zero */ + silk_memset( r_LPC, 0, Order * sizeof( silk_float ) ); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_inv_pred_gain_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_inv_pred_gain_FLP.c new file mode 100644 index 000000000..2be2122d6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LPC_inv_pred_gain_FLP.c @@ -0,0 +1,73 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "SigProc_FLP.h" +#include "define.h" + +/* compute inverse of LPC prediction gain, and */ +/* test if LPC coefficients are stable (all poles within unit circle) */ +/* this code is based on silk_a2k_FLP() */ +silk_float silk_LPC_inverse_pred_gain_FLP( /* O return inverse prediction gain, energy domain */ + const silk_float *A, /* I prediction coefficients [order] */ + opus_int32 order /* I prediction order */ +) +{ + opus_int k, n; + double invGain, rc, rc_mult1, rc_mult2, tmp1, tmp2; + silk_float Atmp[ SILK_MAX_ORDER_LPC ]; + + silk_memcpy( Atmp, A, order * sizeof(silk_float) ); + + invGain = 1.0; + for( k = order - 1; k > 0; k-- ) { + rc = -Atmp[ k ]; + rc_mult1 = 1.0f - rc * rc; + invGain *= rc_mult1; + if( invGain * MAX_PREDICTION_POWER_GAIN < 1.0f ) { + return 0.0f; + } + rc_mult2 = 1.0f / rc_mult1; + for( n = 0; n < (k + 1) >> 1; n++ ) { + tmp1 = Atmp[ n ]; + tmp2 = Atmp[ k - n - 1 ]; + Atmp[ n ] = (silk_float)( ( tmp1 - tmp2 * rc ) * rc_mult2 ); + Atmp[ k - n - 1 ] = (silk_float)( ( tmp2 - tmp1 * rc ) * rc_mult2 ); + } + } + rc = -Atmp[ 0 ]; + rc_mult1 = 1.0f - rc * rc; + invGain *= rc_mult1; + if( invGain * MAX_PREDICTION_POWER_GAIN < 1.0f ) { + return 0.0f; + } + return (silk_float)invGain; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_analysis_filter_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_analysis_filter_FLP.c new file mode 100644 index 000000000..849b7c1c5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_analysis_filter_FLP.c @@ -0,0 +1,75 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +void silk_LTP_analysis_filter_FLP( + silk_float *LTP_res, /* O LTP res MAX_NB_SUBFR*(pre_lgth+subfr_lngth) */ + const silk_float *x, /* I Input signal, with preceding samples */ + const silk_float B[ LTP_ORDER * MAX_NB_SUBFR ], /* I LTP coefficients for each subframe */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const silk_float invGains[ MAX_NB_SUBFR ], /* I Inverse quantization gains */ + const opus_int subfr_length, /* I Length of each subframe */ + const opus_int nb_subfr, /* I number of subframes */ + const opus_int pre_length /* I Preceding samples for each subframe */ +) +{ + const silk_float *x_ptr, *x_lag_ptr; + silk_float Btmp[ LTP_ORDER ]; + silk_float *LTP_res_ptr; + silk_float inv_gain; + opus_int k, i, j; + + x_ptr = x; + LTP_res_ptr = LTP_res; + for( k = 0; k < nb_subfr; k++ ) { + x_lag_ptr = x_ptr - pitchL[ k ]; + inv_gain = invGains[ k ]; + for( i = 0; i < LTP_ORDER; i++ ) { + Btmp[ i ] = B[ k * LTP_ORDER + i ]; + } + + /* LTP analysis FIR filter */ + for( i = 0; i < subfr_length + pre_length; i++ ) { + LTP_res_ptr[ i ] = x_ptr[ i ]; + /* Subtract long-term prediction */ + for( j = 0; j < LTP_ORDER; j++ ) { + LTP_res_ptr[ i ] -= Btmp[ j ] * x_lag_ptr[ LTP_ORDER / 2 - j ]; + } + LTP_res_ptr[ i ] *= inv_gain; + x_lag_ptr++; + } + + /* Update pointers */ + LTP_res_ptr += subfr_length + pre_length; + x_ptr += subfr_length; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_scale_ctrl_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_scale_ctrl_FLP.c new file mode 100644 index 000000000..8dbe29d0f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/LTP_scale_ctrl_FLP.c @@ -0,0 +1,52 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +void silk_LTP_scale_ctrl_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int round_loss; + + if( condCoding == CODE_INDEPENDENTLY ) { + /* Only scale if first frame in packet */ + round_loss = psEnc->sCmn.PacketLoss_perc + psEnc->sCmn.nFramesPerPacket; + psEnc->sCmn.indices.LTP_scaleIndex = (opus_int8)silk_LIMIT( round_loss * psEncCtrl->LTPredCodGain * 0.1f, 0.0f, 2.0f ); + } else { + /* Default is minimum scaling */ + psEnc->sCmn.indices.LTP_scaleIndex = 0; + } + + psEncCtrl->LTP_scale = (silk_float)silk_LTPScales_table_Q14[ psEnc->sCmn.indices.LTP_scaleIndex ] / 16384.0f; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/SigProc_FLP.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/SigProc_FLP.h new file mode 100644 index 000000000..953de8b09 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/SigProc_FLP.h @@ -0,0 +1,197 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_SIGPROC_FLP_H +#define SILK_SIGPROC_FLP_H + +#include "SigProc_FIX.h" +#include "float_cast.h" +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/********************************************************************/ +/* SIGNAL PROCESSING FUNCTIONS */ +/********************************************************************/ + +/* Chirp (bw expand) LP AR filter */ +void silk_bwexpander_FLP( + silk_float *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I length of ar */ + const silk_float chirp /* I chirp factor (typically in range (0..1) ) */ +); + +/* compute inverse of LPC prediction gain, and */ +/* test if LPC coefficients are stable (all poles within unit circle) */ +/* this code is based on silk_FLP_a2k() */ +silk_float silk_LPC_inverse_pred_gain_FLP( /* O return inverse prediction gain, energy domain */ + const silk_float *A, /* I prediction coefficients [order] */ + opus_int32 order /* I prediction order */ +); + +silk_float silk_schur_FLP( /* O returns residual energy */ + silk_float refl_coef[], /* O reflection coefficients (length order) */ + const silk_float auto_corr[], /* I autocorrelation sequence (length order+1) */ + opus_int order /* I order */ +); + +void silk_k2a_FLP( + silk_float *A, /* O prediction coefficients [order] */ + const silk_float *rc, /* I reflection coefficients [order] */ + opus_int32 order /* I prediction order */ +); + +/* compute autocorrelation */ +void silk_autocorrelation_FLP( + silk_float *results, /* O result (length correlationCount) */ + const silk_float *inputData, /* I input data to correlate */ + opus_int inputDataSize, /* I length of input */ + opus_int correlationCount /* I number of correlation taps to compute */ +); + +opus_int silk_pitch_analysis_core_FLP( /* O Voicing estimate: 0 voiced, 1 unvoiced */ + const silk_float *frame, /* I Signal of length PE_FRAME_LENGTH_MS*Fs_kHz */ + opus_int *pitch_out, /* O Pitch lag values [nb_subfr] */ + opus_int16 *lagIndex, /* O Lag Index */ + opus_int8 *contourIndex, /* O Pitch contour Index */ + silk_float *LTPCorr, /* I/O Normalized correlation; input: value from previous frame */ + opus_int prevLag, /* I Last lag of previous frame; set to zero is unvoiced */ + const silk_float search_thres1, /* I First stage threshold for lag candidates 0 - 1 */ + const silk_float search_thres2, /* I Final threshold for lag candidates 0 - 1 */ + const opus_int Fs_kHz, /* I sample frequency (kHz) */ + const opus_int complexity, /* I Complexity setting, 0-2, where 2 is highest */ + const opus_int nb_subfr, /* I Number of 5 ms subframes */ + int arch /* I Run-time architecture */ +); + +void silk_insertion_sort_decreasing_FLP( + silk_float *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +); + +/* Compute reflection coefficients from input signal */ +silk_float silk_burg_modified_FLP( /* O returns residual energy */ + silk_float A[], /* O prediction coefficients (length order) */ + const silk_float x[], /* I input signal, length: nb_subfr*(D+L_sub) */ + const silk_float minInvGain, /* I minimum inverse prediction gain */ + const opus_int subfr_length, /* I input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I number of subframes stacked in x */ + const opus_int D /* I order */ +); + +/* multiply a vector by a constant */ +void silk_scale_vector_FLP( + silk_float *data1, + silk_float gain, + opus_int dataSize +); + +/* copy and multiply a vector by a constant */ +void silk_scale_copy_vector_FLP( + silk_float *data_out, + const silk_float *data_in, + silk_float gain, + opus_int dataSize +); + +/* inner product of two silk_float arrays, with result as double */ +double silk_inner_product_FLP( + const silk_float *data1, + const silk_float *data2, + opus_int dataSize +); + +/* sum of squares of a silk_float array, with result as double */ +double silk_energy_FLP( + const silk_float *data, + opus_int dataSize +); + +/********************************************************************/ +/* MACROS */ +/********************************************************************/ + +#define PI (3.1415926536f) + +#define silk_min_float( a, b ) (((a) < (b)) ? (a) : (b)) +#define silk_max_float( a, b ) (((a) > (b)) ? (a) : (b)) +#define silk_abs_float( a ) ((silk_float)fabs(a)) + +/* sigmoid function */ +static OPUS_INLINE silk_float silk_sigmoid( silk_float x ) +{ + return (silk_float)(1.0 / (1.0 + exp(-x))); +} + +/* floating-point to integer conversion (rounding) */ +static OPUS_INLINE opus_int32 silk_float2int( silk_float x ) +{ + return (opus_int32)float2int( x ); +} + +/* floating-point to integer conversion (rounding) */ +static OPUS_INLINE void silk_float2short_array( + opus_int16 *out, + const silk_float *in, + opus_int32 length +) +{ + opus_int32 k; + for( k = length - 1; k >= 0; k-- ) { + out[k] = silk_SAT16( (opus_int32)float2int( in[k] ) ); + } +} + +/* integer to floating-point conversion */ +static OPUS_INLINE void silk_short2float_array( + silk_float *out, + const opus_int16 *in, + opus_int32 length +) +{ + opus_int32 k; + for( k = length - 1; k >= 0; k-- ) { + out[k] = (silk_float)in[k]; + } +} + +/* using log2() helps the fixed-point conversion */ +static OPUS_INLINE silk_float silk_log2( double x ) +{ + return ( silk_float )( 3.32192809488736 * log10( x ) ); +} + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_SIGPROC_FLP_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/apply_sine_window_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/apply_sine_window_FLP.c new file mode 100644 index 000000000..e49e71799 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/apply_sine_window_FLP.c @@ -0,0 +1,81 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +/* Apply sine window to signal vector */ +/* Window types: */ +/* 1 -> sine window from 0 to pi/2 */ +/* 2 -> sine window from pi/2 to pi */ +void silk_apply_sine_window_FLP( + silk_float px_win[], /* O Pointer to windowed signal */ + const silk_float px[], /* I Pointer to input signal */ + const opus_int win_type, /* I Selects a window type */ + const opus_int length /* I Window length, multiple of 4 */ +) +{ + opus_int k; + silk_float freq, c, S0, S1; + + celt_assert( win_type == 1 || win_type == 2 ); + + /* Length must be multiple of 4 */ + celt_assert( ( length & 3 ) == 0 ); + + freq = PI / ( length + 1 ); + + /* Approximation of 2 * cos(f) */ + c = 2.0f - freq * freq; + + /* Initialize state */ + if( win_type < 2 ) { + /* Start from 0 */ + S0 = 0.0f; + /* Approximation of sin(f) */ + S1 = freq; + } else { + /* Start from 1 */ + S0 = 1.0f; + /* Approximation of cos(f) */ + S1 = 0.5f * c; + } + + /* Uses the recursive equation: sin(n*f) = 2 * cos(f) * sin((n-1)*f) - sin((n-2)*f) */ + /* 4 samples at a time */ + for( k = 0; k < length; k += 4 ) { + px_win[ k + 0 ] = px[ k + 0 ] * 0.5f * ( S0 + S1 ); + px_win[ k + 1 ] = px[ k + 1 ] * S1; + S0 = c * S1 - S0; + px_win[ k + 2 ] = px[ k + 2 ] * 0.5f * ( S1 + S0 ); + px_win[ k + 3 ] = px[ k + 3 ] * S0; + S1 = c * S0 - S1; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/autocorrelation_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/autocorrelation_FLP.c new file mode 100644 index 000000000..8b8a9e659 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/autocorrelation_FLP.c @@ -0,0 +1,52 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "typedef.h" +#include "SigProc_FLP.h" + +/* compute autocorrelation */ +void silk_autocorrelation_FLP( + silk_float *results, /* O result (length correlationCount) */ + const silk_float *inputData, /* I input data to correlate */ + opus_int inputDataSize, /* I length of input */ + opus_int correlationCount /* I number of correlation taps to compute */ +) +{ + opus_int i; + + if( correlationCount > inputDataSize ) { + correlationCount = inputDataSize; + } + + for( i = 0; i < correlationCount; i++ ) { + results[ i ] = (silk_float)silk_inner_product_FLP( inputData, inputData + i, inputDataSize - i ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/burg_modified_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/burg_modified_FLP.c new file mode 100644 index 000000000..756b76a35 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/burg_modified_FLP.c @@ -0,0 +1,186 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" +#include "tuning_parameters.h" +#include "define.h" + +#define MAX_FRAME_SIZE 384 /* subfr_length * nb_subfr = ( 0.005 * 16000 + 16 ) * 4 = 384*/ + +/* Compute reflection coefficients from input signal */ +silk_float silk_burg_modified_FLP( /* O returns residual energy */ + silk_float A[], /* O prediction coefficients (length order) */ + const silk_float x[], /* I input signal, length: nb_subfr*(D+L_sub) */ + const silk_float minInvGain, /* I minimum inverse prediction gain */ + const opus_int subfr_length, /* I input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I number of subframes stacked in x */ + const opus_int D /* I order */ +) +{ + opus_int k, n, s, reached_max_gain; + double C0, invGain, num, nrg_f, nrg_b, rc, Atmp, tmp1, tmp2; + const silk_float *x_ptr; + double C_first_row[ SILK_MAX_ORDER_LPC ], C_last_row[ SILK_MAX_ORDER_LPC ]; + double CAf[ SILK_MAX_ORDER_LPC + 1 ], CAb[ SILK_MAX_ORDER_LPC + 1 ]; + double Af[ SILK_MAX_ORDER_LPC ]; + + celt_assert( subfr_length * nb_subfr <= MAX_FRAME_SIZE ); + + /* Compute autocorrelations, added over subframes */ + C0 = silk_energy_FLP( x, nb_subfr * subfr_length ); + silk_memset( C_first_row, 0, SILK_MAX_ORDER_LPC * sizeof( double ) ); + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + for( n = 1; n < D + 1; n++ ) { + C_first_row[ n - 1 ] += silk_inner_product_FLP( x_ptr, x_ptr + n, subfr_length - n ); + } + } + silk_memcpy( C_last_row, C_first_row, SILK_MAX_ORDER_LPC * sizeof( double ) ); + + /* Initialize */ + CAb[ 0 ] = CAf[ 0 ] = C0 + FIND_LPC_COND_FAC * C0 + 1e-9f; + invGain = 1.0f; + reached_max_gain = 0; + for( n = 0; n < D; n++ ) { + /* Update first row of correlation matrix (without first element) */ + /* Update last row of correlation matrix (without last element, stored in reversed order) */ + /* Update C * Af */ + /* Update C * flipud(Af) (stored in reversed order) */ + for( s = 0; s < nb_subfr; s++ ) { + x_ptr = x + s * subfr_length; + tmp1 = x_ptr[ n ]; + tmp2 = x_ptr[ subfr_length - n - 1 ]; + for( k = 0; k < n; k++ ) { + C_first_row[ k ] -= x_ptr[ n ] * x_ptr[ n - k - 1 ]; + C_last_row[ k ] -= x_ptr[ subfr_length - n - 1 ] * x_ptr[ subfr_length - n + k ]; + Atmp = Af[ k ]; + tmp1 += x_ptr[ n - k - 1 ] * Atmp; + tmp2 += x_ptr[ subfr_length - n + k ] * Atmp; + } + for( k = 0; k <= n; k++ ) { + CAf[ k ] -= tmp1 * x_ptr[ n - k ]; + CAb[ k ] -= tmp2 * x_ptr[ subfr_length - n + k - 1 ]; + } + } + tmp1 = C_first_row[ n ]; + tmp2 = C_last_row[ n ]; + for( k = 0; k < n; k++ ) { + Atmp = Af[ k ]; + tmp1 += C_last_row[ n - k - 1 ] * Atmp; + tmp2 += C_first_row[ n - k - 1 ] * Atmp; + } + CAf[ n + 1 ] = tmp1; + CAb[ n + 1 ] = tmp2; + + /* Calculate nominator and denominator for the next order reflection (parcor) coefficient */ + num = CAb[ n + 1 ]; + nrg_b = CAb[ 0 ]; + nrg_f = CAf[ 0 ]; + for( k = 0; k < n; k++ ) { + Atmp = Af[ k ]; + num += CAb[ n - k ] * Atmp; + nrg_b += CAb[ k + 1 ] * Atmp; + nrg_f += CAf[ k + 1 ] * Atmp; + } + silk_assert( nrg_f > 0.0 ); + silk_assert( nrg_b > 0.0 ); + + /* Calculate the next order reflection (parcor) coefficient */ + rc = -2.0 * num / ( nrg_f + nrg_b ); + silk_assert( rc > -1.0 && rc < 1.0 ); + + /* Update inverse prediction gain */ + tmp1 = invGain * ( 1.0 - rc * rc ); + if( tmp1 <= minInvGain ) { + /* Max prediction gain exceeded; set reflection coefficient such that max prediction gain is exactly hit */ + rc = sqrt( 1.0 - minInvGain / invGain ); + if( num > 0 ) { + /* Ensure adjusted reflection coefficients has the original sign */ + rc = -rc; + } + invGain = minInvGain; + reached_max_gain = 1; + } else { + invGain = tmp1; + } + + /* Update the AR coefficients */ + for( k = 0; k < (n + 1) >> 1; k++ ) { + tmp1 = Af[ k ]; + tmp2 = Af[ n - k - 1 ]; + Af[ k ] = tmp1 + rc * tmp2; + Af[ n - k - 1 ] = tmp2 + rc * tmp1; + } + Af[ n ] = rc; + + if( reached_max_gain ) { + /* Reached max prediction gain; set remaining coefficients to zero and exit loop */ + for( k = n + 1; k < D; k++ ) { + Af[ k ] = 0.0; + } + break; + } + + /* Update C * Af and C * Ab */ + for( k = 0; k <= n + 1; k++ ) { + tmp1 = CAf[ k ]; + CAf[ k ] += rc * CAb[ n - k + 1 ]; + CAb[ n - k + 1 ] += rc * tmp1; + } + } + + if( reached_max_gain ) { + /* Convert to silk_float */ + for( k = 0; k < D; k++ ) { + A[ k ] = (silk_float)( -Af[ k ] ); + } + /* Subtract energy of preceding samples from C0 */ + for( s = 0; s < nb_subfr; s++ ) { + C0 -= silk_energy_FLP( x + s * subfr_length, D ); + } + /* Approximate residual energy */ + nrg_f = C0 * invGain; + } else { + /* Compute residual energy and store coefficients as silk_float */ + nrg_f = CAf[ 0 ]; + tmp1 = 1.0; + for( k = 0; k < D; k++ ) { + Atmp = Af[ k ]; + nrg_f += CAf[ k + 1 ] * Atmp; + tmp1 += Atmp * Atmp; + A[ k ] = (silk_float)(-Atmp); + } + nrg_f -= FIND_LPC_COND_FAC * C0 * tmp1; + } + + /* Return residual energy */ + return (silk_float)nrg_f; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/bwexpander_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/bwexpander_FLP.c new file mode 100644 index 000000000..d55a4d79a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/bwexpander_FLP.c @@ -0,0 +1,49 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* Chirp (bw expand) LP AR filter */ +void silk_bwexpander_FLP( + silk_float *ar, /* I/O AR filter to be expanded (without leading 1) */ + const opus_int d, /* I length of ar */ + const silk_float chirp /* I chirp factor (typically in range (0..1) ) */ +) +{ + opus_int i; + silk_float cfac = chirp; + + for( i = 0; i < d - 1; i++ ) { + ar[ i ] *= cfac; + cfac *= chirp; + } + ar[ d - 1 ] *= cfac; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/corrMatrix_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/corrMatrix_FLP.c new file mode 100644 index 000000000..eae6a1cfc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/corrMatrix_FLP.c @@ -0,0 +1,93 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/********************************************************************** + * Correlation matrix computations for LS estimate. + **********************************************************************/ + +#include "main_FLP.h" + +/* Calculates correlation vector X'*t */ +void silk_corrVector_FLP( + const silk_float *x, /* I x vector [L+order-1] used to create X */ + const silk_float *t, /* I Target vector [L] */ + const opus_int L, /* I Length of vecors */ + const opus_int Order, /* I Max lag for correlation */ + silk_float *Xt /* O X'*t correlation vector [order] */ +) +{ + opus_int lag; + const silk_float *ptr1; + + ptr1 = &x[ Order - 1 ]; /* Points to first sample of column 0 of X: X[:,0] */ + for( lag = 0; lag < Order; lag++ ) { + /* Calculate X[:,lag]'*t */ + Xt[ lag ] = (silk_float)silk_inner_product_FLP( ptr1, t, L ); + ptr1--; /* Next column of X */ + } +} + +/* Calculates correlation matrix X'*X */ +void silk_corrMatrix_FLP( + const silk_float *x, /* I x vector [ L+order-1 ] used to create X */ + const opus_int L, /* I Length of vectors */ + const opus_int Order, /* I Max lag for correlation */ + silk_float *XX /* O X'*X correlation matrix [order x order] */ +) +{ + opus_int j, lag; + double energy; + const silk_float *ptr1, *ptr2; + + ptr1 = &x[ Order - 1 ]; /* First sample of column 0 of X */ + energy = silk_energy_FLP( ptr1, L ); /* X[:,0]'*X[:,0] */ + matrix_ptr( XX, 0, 0, Order ) = ( silk_float )energy; + for( j = 1; j < Order; j++ ) { + /* Calculate X[:,j]'*X[:,j] */ + energy += ptr1[ -j ] * ptr1[ -j ] - ptr1[ L - j ] * ptr1[ L - j ]; + matrix_ptr( XX, j, j, Order ) = ( silk_float )energy; + } + + ptr2 = &x[ Order - 2 ]; /* First sample of column 1 of X */ + for( lag = 1; lag < Order; lag++ ) { + /* Calculate X[:,0]'*X[:,lag] */ + energy = silk_inner_product_FLP( ptr1, ptr2, L ); + matrix_ptr( XX, lag, 0, Order ) = ( silk_float )energy; + matrix_ptr( XX, 0, lag, Order ) = ( silk_float )energy; + /* Calculate X[:,j]'*X[:,j + lag] */ + for( j = 1; j < ( Order - lag ); j++ ) { + energy += ptr1[ -j ] * ptr2[ -j ] - ptr1[ L - j ] * ptr2[ L - j ]; + matrix_ptr( XX, lag + j, j, Order ) = ( silk_float )energy; + matrix_ptr( XX, j, lag + j, Order ) = ( silk_float )energy; + } + ptr2--; /* Next column of X */ + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/encode_frame_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/encode_frame_FLP.c new file mode 100644 index 000000000..b029c3f5c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/encode_frame_FLP.c @@ -0,0 +1,435 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "main_FLP.h" +#include "tuning_parameters.h" + +/* Low Bitrate Redundancy (LBRR) encoding. Reuse all parameters but encode with lower bitrate */ +static OPUS_INLINE void silk_LBRR_encode_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float xfw[], /* I Input signal */ + opus_int condCoding /* I The type of conditional coding used so far for this frame */ +); + +void silk_encode_do_VAD_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + opus_int activity /* I Decision of Opus voice activity detector */ +) +{ + const opus_int activity_threshold = SILK_FIX_CONST( SPEECH_ACTIVITY_DTX_THRES, 8 ); + + /****************************/ + /* Voice Activity Detection */ + /****************************/ + silk_VAD_GetSA_Q8( &psEnc->sCmn, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.arch ); + /* If Opus VAD is inactive and Silk VAD is active: lower Silk VAD to just under the threshold */ + if( activity == VAD_NO_ACTIVITY && psEnc->sCmn.speech_activity_Q8 >= activity_threshold ) { + psEnc->sCmn.speech_activity_Q8 = activity_threshold - 1; + } + + /**************************************************/ + /* Convert speech activity into VAD and DTX flags */ + /**************************************************/ + if( psEnc->sCmn.speech_activity_Q8 < activity_threshold ) { + psEnc->sCmn.indices.signalType = TYPE_NO_VOICE_ACTIVITY; + psEnc->sCmn.noSpeechCounter++; + if( psEnc->sCmn.noSpeechCounter <= NB_SPEECH_FRAMES_BEFORE_DTX ) { + psEnc->sCmn.inDTX = 0; + } else if( psEnc->sCmn.noSpeechCounter > MAX_CONSECUTIVE_DTX + NB_SPEECH_FRAMES_BEFORE_DTX ) { + psEnc->sCmn.noSpeechCounter = NB_SPEECH_FRAMES_BEFORE_DTX; + psEnc->sCmn.inDTX = 0; + } + psEnc->sCmn.VAD_flags[ psEnc->sCmn.nFramesEncoded ] = 0; + } else { + psEnc->sCmn.noSpeechCounter = 0; + psEnc->sCmn.inDTX = 0; + psEnc->sCmn.indices.signalType = TYPE_UNVOICED; + psEnc->sCmn.VAD_flags[ psEnc->sCmn.nFramesEncoded ] = 1; + } +} + +/****************/ +/* Encode frame */ +/****************/ +opus_int silk_encode_frame_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + opus_int32 *pnBytesOut, /* O Number of payload bytes; */ + ec_enc *psRangeEnc, /* I/O compressor data structure */ + opus_int condCoding, /* I The type of conditional coding to use */ + opus_int maxBits, /* I If > 0: maximum number of output bits */ + opus_int useCBR /* I Flag to force constant-bitrate operation */ +) +{ + silk_encoder_control_FLP sEncCtrl; + opus_int i, iter, maxIter, found_upper, found_lower, ret = 0; + silk_float *x_frame, *res_pitch_frame; + silk_float res_pitch[ 2 * MAX_FRAME_LENGTH + LA_PITCH_MAX ]; + ec_enc sRangeEnc_copy, sRangeEnc_copy2; + silk_nsq_state sNSQ_copy, sNSQ_copy2; + opus_int32 seed_copy, nBits, nBits_lower, nBits_upper, gainMult_lower, gainMult_upper; + opus_int32 gainsID, gainsID_lower, gainsID_upper; + opus_int16 gainMult_Q8; + opus_int16 ec_prevLagIndex_copy; + opus_int ec_prevSignalType_copy; + opus_int8 LastGainIndex_copy2; + opus_int32 pGains_Q16[ MAX_NB_SUBFR ]; + opus_uint8 ec_buf_copy[ 1275 ]; + opus_int gain_lock[ MAX_NB_SUBFR ] = {0}; + opus_int16 best_gain_mult[ MAX_NB_SUBFR ]; + opus_int best_sum[ MAX_NB_SUBFR ]; + + /* This is totally unnecessary but many compilers (including gcc) are too dumb to realise it */ + LastGainIndex_copy2 = nBits_lower = nBits_upper = gainMult_lower = gainMult_upper = 0; + + psEnc->sCmn.indices.Seed = psEnc->sCmn.frameCounter++ & 3; + + /**************************************************************/ + /* Set up Input Pointers, and insert frame in input buffer */ + /**************************************************************/ + /* pointers aligned with start of frame to encode */ + x_frame = psEnc->x_buf + psEnc->sCmn.ltp_mem_length; /* start of frame to encode */ + res_pitch_frame = res_pitch + psEnc->sCmn.ltp_mem_length; /* start of pitch LPC residual frame */ + + /***************************************/ + /* Ensure smooth bandwidth transitions */ + /***************************************/ + silk_LP_variable_cutoff( &psEnc->sCmn.sLP, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.frame_length ); + + /*******************************************/ + /* Copy new frame to front of input buffer */ + /*******************************************/ + silk_short2float_array( x_frame + LA_SHAPE_MS * psEnc->sCmn.fs_kHz, psEnc->sCmn.inputBuf + 1, psEnc->sCmn.frame_length ); + + /* Add tiny signal to avoid high CPU load from denormalized floating point numbers */ + for( i = 0; i < 8; i++ ) { + x_frame[ LA_SHAPE_MS * psEnc->sCmn.fs_kHz + i * ( psEnc->sCmn.frame_length >> 3 ) ] += ( 1 - ( i & 2 ) ) * 1e-6f; + } + + if( !psEnc->sCmn.prefillFlag ) { + /*****************************************/ + /* Find pitch lags, initial LPC analysis */ + /*****************************************/ + silk_find_pitch_lags_FLP( psEnc, &sEncCtrl, res_pitch, x_frame, psEnc->sCmn.arch ); + + /************************/ + /* Noise shape analysis */ + /************************/ + silk_noise_shape_analysis_FLP( psEnc, &sEncCtrl, res_pitch_frame, x_frame ); + + /***************************************************/ + /* Find linear prediction coefficients (LPC + LTP) */ + /***************************************************/ + silk_find_pred_coefs_FLP( psEnc, &sEncCtrl, res_pitch_frame, x_frame, condCoding ); + + /****************************************/ + /* Process gains */ + /****************************************/ + silk_process_gains_FLP( psEnc, &sEncCtrl, condCoding ); + + /****************************************/ + /* Low Bitrate Redundant Encoding */ + /****************************************/ + silk_LBRR_encode_FLP( psEnc, &sEncCtrl, x_frame, condCoding ); + + /* Loop over quantizer and entroy coding to control bitrate */ + maxIter = 6; + gainMult_Q8 = SILK_FIX_CONST( 1, 8 ); + found_lower = 0; + found_upper = 0; + gainsID = silk_gains_ID( psEnc->sCmn.indices.GainsIndices, psEnc->sCmn.nb_subfr ); + gainsID_lower = -1; + gainsID_upper = -1; + /* Copy part of the input state */ + silk_memcpy( &sRangeEnc_copy, psRangeEnc, sizeof( ec_enc ) ); + silk_memcpy( &sNSQ_copy, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + seed_copy = psEnc->sCmn.indices.Seed; + ec_prevLagIndex_copy = psEnc->sCmn.ec_prevLagIndex; + ec_prevSignalType_copy = psEnc->sCmn.ec_prevSignalType; + for( iter = 0; ; iter++ ) { + if( gainsID == gainsID_lower ) { + nBits = nBits_lower; + } else if( gainsID == gainsID_upper ) { + nBits = nBits_upper; + } else { + /* Restore part of the input state */ + if( iter > 0 ) { + silk_memcpy( psRangeEnc, &sRangeEnc_copy, sizeof( ec_enc ) ); + silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy, sizeof( silk_nsq_state ) ); + psEnc->sCmn.indices.Seed = seed_copy; + psEnc->sCmn.ec_prevLagIndex = ec_prevLagIndex_copy; + psEnc->sCmn.ec_prevSignalType = ec_prevSignalType_copy; + } + + /*****************************************/ + /* Noise shaping quantization */ + /*****************************************/ + silk_NSQ_wrapper_FLP( psEnc, &sEncCtrl, &psEnc->sCmn.indices, &psEnc->sCmn.sNSQ, psEnc->sCmn.pulses, x_frame ); + + if ( iter == maxIter && !found_lower ) { + silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) ); + } + + /****************************************/ + /* Encode Parameters */ + /****************************************/ + silk_encode_indices( &psEnc->sCmn, psRangeEnc, psEnc->sCmn.nFramesEncoded, 0, condCoding ); + + /****************************************/ + /* Encode Excitation Signal */ + /****************************************/ + silk_encode_pulses( psRangeEnc, psEnc->sCmn.indices.signalType, psEnc->sCmn.indices.quantOffsetType, + psEnc->sCmn.pulses, psEnc->sCmn.frame_length ); + + nBits = ec_tell( psRangeEnc ); + + /* If we still bust after the last iteration, do some damage control. */ + if ( iter == maxIter && !found_lower && nBits > maxBits ) { + silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) ); + + /* Keep gains the same as the last frame. */ + psEnc->sShape.LastGainIndex = sEncCtrl.lastGainIndexPrev; + for ( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + psEnc->sCmn.indices.GainsIndices[ i ] = 4; + } + if (condCoding != CODE_CONDITIONALLY) { + psEnc->sCmn.indices.GainsIndices[ 0 ] = sEncCtrl.lastGainIndexPrev; + } + psEnc->sCmn.ec_prevLagIndex = ec_prevLagIndex_copy; + psEnc->sCmn.ec_prevSignalType = ec_prevSignalType_copy; + /* Clear all pulses. */ + for ( i = 0; i < psEnc->sCmn.frame_length; i++ ) { + psEnc->sCmn.pulses[ i ] = 0; + } + + silk_encode_indices( &psEnc->sCmn, psRangeEnc, psEnc->sCmn.nFramesEncoded, 0, condCoding ); + + silk_encode_pulses( psRangeEnc, psEnc->sCmn.indices.signalType, psEnc->sCmn.indices.quantOffsetType, + psEnc->sCmn.pulses, psEnc->sCmn.frame_length ); + + nBits = ec_tell( psRangeEnc ); + } + + if( useCBR == 0 && iter == 0 && nBits <= maxBits ) { + break; + } + } + + if( iter == maxIter ) { + if( found_lower && ( gainsID == gainsID_lower || nBits > maxBits ) ) { + /* Restore output state from earlier iteration that did meet the bitrate budget */ + silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) ); + celt_assert( sRangeEnc_copy2.offs <= 1275 ); + silk_memcpy( psRangeEnc->buf, ec_buf_copy, sRangeEnc_copy2.offs ); + silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy2, sizeof( silk_nsq_state ) ); + psEnc->sShape.LastGainIndex = LastGainIndex_copy2; + } + break; + } + + if( nBits > maxBits ) { + if( found_lower == 0 && iter >= 2 ) { + /* Adjust the quantizer's rate/distortion tradeoff and discard previous "upper" results */ + sEncCtrl.Lambda = silk_max_float(sEncCtrl.Lambda*1.5f, 1.5f); + /* Reducing dithering can help us hit the target. */ + psEnc->sCmn.indices.quantOffsetType = 0; + found_upper = 0; + gainsID_upper = -1; + } else { + found_upper = 1; + nBits_upper = nBits; + gainMult_upper = gainMult_Q8; + gainsID_upper = gainsID; + } + } else if( nBits < maxBits - 5 ) { + found_lower = 1; + nBits_lower = nBits; + gainMult_lower = gainMult_Q8; + if( gainsID != gainsID_lower ) { + gainsID_lower = gainsID; + /* Copy part of the output state */ + silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) ); + celt_assert( psRangeEnc->offs <= 1275 ); + silk_memcpy( ec_buf_copy, psRangeEnc->buf, psRangeEnc->offs ); + silk_memcpy( &sNSQ_copy2, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + LastGainIndex_copy2 = psEnc->sShape.LastGainIndex; + } + } else { + /* Within 5 bits of budget: close enough */ + break; + } + + if ( !found_lower && nBits > maxBits ) { + int j; + for ( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + int sum=0; + for ( j = i*psEnc->sCmn.subfr_length; j < (i+1)*psEnc->sCmn.subfr_length; j++ ) { + sum += abs( psEnc->sCmn.pulses[j] ); + } + if ( iter == 0 || (sum < best_sum[i] && !gain_lock[i]) ) { + best_sum[i] = sum; + best_gain_mult[i] = gainMult_Q8; + } else { + gain_lock[i] = 1; + } + } + } + if( ( found_lower & found_upper ) == 0 ) { + /* Adjust gain according to high-rate rate/distortion curve */ + if( nBits > maxBits ) { + if (gainMult_Q8 < 16384) { + gainMult_Q8 *= 2; + } else { + gainMult_Q8 = 32767; + } + } else { + opus_int32 gain_factor_Q16; + gain_factor_Q16 = silk_log2lin( silk_LSHIFT( nBits - maxBits, 7 ) / psEnc->sCmn.frame_length + SILK_FIX_CONST( 16, 7 ) ); + gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 ); + } + } else { + /* Adjust gain by interpolating */ + gainMult_Q8 = gainMult_lower + ( ( gainMult_upper - gainMult_lower ) * ( maxBits - nBits_lower ) ) / ( nBits_upper - nBits_lower ); + /* New gain multplier must be between 25% and 75% of old range (note that gainMult_upper < gainMult_lower) */ + if( gainMult_Q8 > silk_ADD_RSHIFT32( gainMult_lower, gainMult_upper - gainMult_lower, 2 ) ) { + gainMult_Q8 = silk_ADD_RSHIFT32( gainMult_lower, gainMult_upper - gainMult_lower, 2 ); + } else + if( gainMult_Q8 < silk_SUB_RSHIFT32( gainMult_upper, gainMult_upper - gainMult_lower, 2 ) ) { + gainMult_Q8 = silk_SUB_RSHIFT32( gainMult_upper, gainMult_upper - gainMult_lower, 2 ); + } + } + + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + opus_int16 tmp; + if ( gain_lock[i] ) { + tmp = best_gain_mult[i]; + } else { + tmp = gainMult_Q8; + } + pGains_Q16[ i ] = silk_LSHIFT_SAT32( silk_SMULWB( sEncCtrl.GainsUnq_Q16[ i ], tmp ), 8 ); + } + + /* Quantize gains */ + psEnc->sShape.LastGainIndex = sEncCtrl.lastGainIndexPrev; + silk_gains_quant( psEnc->sCmn.indices.GainsIndices, pGains_Q16, + &psEnc->sShape.LastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /* Unique identifier of gains vector */ + gainsID = silk_gains_ID( psEnc->sCmn.indices.GainsIndices, psEnc->sCmn.nb_subfr ); + + /* Overwrite unquantized gains with quantized gains and convert back to Q0 from Q16 */ + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + sEncCtrl.Gains[ i ] = pGains_Q16[ i ] / 65536.0f; + } + } + } + + /* Update input buffer */ + silk_memmove( psEnc->x_buf, &psEnc->x_buf[ psEnc->sCmn.frame_length ], + ( psEnc->sCmn.ltp_mem_length + LA_SHAPE_MS * psEnc->sCmn.fs_kHz ) * sizeof( silk_float ) ); + + /* Exit without entropy coding */ + if( psEnc->sCmn.prefillFlag ) { + /* No payload */ + *pnBytesOut = 0; + return ret; + } + + /* Parameters needed for next frame */ + psEnc->sCmn.prevLag = sEncCtrl.pitchL[ psEnc->sCmn.nb_subfr - 1 ]; + psEnc->sCmn.prevSignalType = psEnc->sCmn.indices.signalType; + + /****************************************/ + /* Finalize payload */ + /****************************************/ + psEnc->sCmn.first_frame_after_reset = 0; + /* Payload size */ + *pnBytesOut = silk_RSHIFT( ec_tell( psRangeEnc ) + 7, 3 ); + + return ret; +} + +/* Low-Bitrate Redundancy (LBRR) encoding. Reuse all parameters but encode excitation at lower bitrate */ +static OPUS_INLINE void silk_LBRR_encode_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float xfw[], /* I Input signal */ + opus_int condCoding /* I The type of conditional coding used so far for this frame */ +) +{ + opus_int k; + opus_int32 Gains_Q16[ MAX_NB_SUBFR ]; + silk_float TempGains[ MAX_NB_SUBFR ]; + SideInfoIndices *psIndices_LBRR = &psEnc->sCmn.indices_LBRR[ psEnc->sCmn.nFramesEncoded ]; + silk_nsq_state sNSQ_LBRR; + + /*******************************************/ + /* Control use of inband LBRR */ + /*******************************************/ + if( psEnc->sCmn.LBRR_enabled && psEnc->sCmn.speech_activity_Q8 > SILK_FIX_CONST( LBRR_SPEECH_ACTIVITY_THRES, 8 ) ) { + psEnc->sCmn.LBRR_flags[ psEnc->sCmn.nFramesEncoded ] = 1; + + /* Copy noise shaping quantizer state and quantization indices from regular encoding */ + silk_memcpy( &sNSQ_LBRR, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) ); + silk_memcpy( psIndices_LBRR, &psEnc->sCmn.indices, sizeof( SideInfoIndices ) ); + + /* Save original gains */ + silk_memcpy( TempGains, psEncCtrl->Gains, psEnc->sCmn.nb_subfr * sizeof( silk_float ) ); + + if( psEnc->sCmn.nFramesEncoded == 0 || psEnc->sCmn.LBRR_flags[ psEnc->sCmn.nFramesEncoded - 1 ] == 0 ) { + /* First frame in packet or previous frame not LBRR coded */ + psEnc->sCmn.LBRRprevLastGainIndex = psEnc->sShape.LastGainIndex; + + /* Increase Gains to get target LBRR rate */ + psIndices_LBRR->GainsIndices[ 0 ] += psEnc->sCmn.LBRR_GainIncreases; + psIndices_LBRR->GainsIndices[ 0 ] = silk_min_int( psIndices_LBRR->GainsIndices[ 0 ], N_LEVELS_QGAIN - 1 ); + } + + /* Decode to get gains in sync with decoder */ + silk_gains_dequant( Gains_Q16, psIndices_LBRR->GainsIndices, + &psEnc->sCmn.LBRRprevLastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /* Overwrite unquantized gains with quantized gains and convert back to Q0 from Q16 */ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains[ k ] = Gains_Q16[ k ] * ( 1.0f / 65536.0f ); + } + + /*****************************************/ + /* Noise shaping quantization */ + /*****************************************/ + silk_NSQ_wrapper_FLP( psEnc, psEncCtrl, psIndices_LBRR, &sNSQ_LBRR, + psEnc->sCmn.pulses_LBRR[ psEnc->sCmn.nFramesEncoded ], xfw ); + + /* Restore original gains */ + silk_memcpy( psEncCtrl->Gains, TempGains, psEnc->sCmn.nb_subfr * sizeof( silk_float ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/energy_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/energy_FLP.c new file mode 100644 index 000000000..7bc7173c9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/energy_FLP.c @@ -0,0 +1,59 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* sum of squares of a silk_float array, with result as double */ +double silk_energy_FLP( + const silk_float *data, + opus_int dataSize +) +{ + opus_int i; + double result; + + /* 4x unrolled loop */ + result = 0.0; + for( i = 0; i < dataSize - 3; i += 4 ) { + result += data[ i + 0 ] * (double)data[ i + 0 ] + + data[ i + 1 ] * (double)data[ i + 1 ] + + data[ i + 2 ] * (double)data[ i + 2 ] + + data[ i + 3 ] * (double)data[ i + 3 ]; + } + + /* add any remaining products */ + for( ; i < dataSize; i++ ) { + result += data[ i ] * (double)data[ i ]; + } + + silk_assert( result >= 0.0 ); + return result; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LPC_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LPC_FLP.c new file mode 100644 index 000000000..fa3ffe7f8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LPC_FLP.c @@ -0,0 +1,104 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "define.h" +#include "main_FLP.h" +#include "tuning_parameters.h" + +/* LPC analysis */ +void silk_find_LPC_FLP( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 NLSF_Q15[], /* O NLSFs */ + const silk_float x[], /* I Input signal */ + const silk_float minInvGain /* I Inverse of max prediction gain */ +) +{ + opus_int k, subfr_length; + silk_float a[ MAX_LPC_ORDER ]; + + /* Used only for NLSF interpolation */ + silk_float res_nrg, res_nrg_2nd, res_nrg_interp; + opus_int16 NLSF0_Q15[ MAX_LPC_ORDER ]; + silk_float a_tmp[ MAX_LPC_ORDER ]; + silk_float LPC_res[ MAX_FRAME_LENGTH + MAX_NB_SUBFR * MAX_LPC_ORDER ]; + + subfr_length = psEncC->subfr_length + psEncC->predictLPCOrder; + + /* Default: No interpolation */ + psEncC->indices.NLSFInterpCoef_Q2 = 4; + + /* Burg AR analysis for the full frame */ + res_nrg = silk_burg_modified_FLP( a, x, minInvGain, subfr_length, psEncC->nb_subfr, psEncC->predictLPCOrder ); + + if( psEncC->useInterpolatedNLSFs && !psEncC->first_frame_after_reset && psEncC->nb_subfr == MAX_NB_SUBFR ) { + /* Optimal solution for last 10 ms; subtract residual energy here, as that's easier than */ + /* adding it to the residual energy of the first 10 ms in each iteration of the search below */ + res_nrg -= silk_burg_modified_FLP( a_tmp, x + ( MAX_NB_SUBFR / 2 ) * subfr_length, minInvGain, subfr_length, MAX_NB_SUBFR / 2, psEncC->predictLPCOrder ); + + /* Convert to NLSFs */ + silk_A2NLSF_FLP( NLSF_Q15, a_tmp, psEncC->predictLPCOrder ); + + /* Search over interpolation indices to find the one with lowest residual energy */ + res_nrg_2nd = silk_float_MAX; + for( k = 3; k >= 0; k-- ) { + /* Interpolate NLSFs for first half */ + silk_interpolate( NLSF0_Q15, psEncC->prev_NLSFq_Q15, NLSF_Q15, k, psEncC->predictLPCOrder ); + + /* Convert to LPC for residual energy evaluation */ + silk_NLSF2A_FLP( a_tmp, NLSF0_Q15, psEncC->predictLPCOrder, psEncC->arch ); + + /* Calculate residual energy with LSF interpolation */ + silk_LPC_analysis_filter_FLP( LPC_res, a_tmp, x, 2 * subfr_length, psEncC->predictLPCOrder ); + res_nrg_interp = (silk_float)( + silk_energy_FLP( LPC_res + psEncC->predictLPCOrder, subfr_length - psEncC->predictLPCOrder ) + + silk_energy_FLP( LPC_res + psEncC->predictLPCOrder + subfr_length, subfr_length - psEncC->predictLPCOrder ) ); + + /* Determine whether current interpolated NLSFs are best so far */ + if( res_nrg_interp < res_nrg ) { + /* Interpolation has lower residual energy */ + res_nrg = res_nrg_interp; + psEncC->indices.NLSFInterpCoef_Q2 = (opus_int8)k; + } else if( res_nrg_interp > res_nrg_2nd ) { + /* No reason to continue iterating - residual energies will continue to climb */ + break; + } + res_nrg_2nd = res_nrg_interp; + } + } + + if( psEncC->indices.NLSFInterpCoef_Q2 == 4 ) { + /* NLSF interpolation is currently inactive, calculate NLSFs from full frame AR coefficients */ + silk_A2NLSF_FLP( NLSF_Q15, a, psEncC->predictLPCOrder ); + } + + celt_assert( psEncC->indices.NLSFInterpCoef_Q2 == 4 || + ( psEncC->useInterpolatedNLSFs && !psEncC->first_frame_after_reset && psEncC->nb_subfr == MAX_NB_SUBFR ) ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LTP_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LTP_FLP.c new file mode 100644 index 000000000..f97064930 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_LTP_FLP.c @@ -0,0 +1,64 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" +#include "tuning_parameters.h" + +void silk_find_LTP_FLP( + silk_float XX[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* O Weight for LTP quantization */ + silk_float xX[ MAX_NB_SUBFR * LTP_ORDER ], /* O Weight for LTP quantization */ + const silk_float r_ptr[], /* I LPC residual */ + const opus_int lag[ MAX_NB_SUBFR ], /* I LTP lags */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr /* I number of subframes */ +) +{ + opus_int k; + silk_float *xX_ptr, *XX_ptr; + const silk_float *lag_ptr; + silk_float xx, temp; + + xX_ptr = xX; + XX_ptr = XX; + for( k = 0; k < nb_subfr; k++ ) { + lag_ptr = r_ptr - ( lag[ k ] + LTP_ORDER / 2 ); + silk_corrMatrix_FLP( lag_ptr, subfr_length, LTP_ORDER, XX_ptr ); + silk_corrVector_FLP( lag_ptr, r_ptr, subfr_length, LTP_ORDER, xX_ptr ); + xx = ( silk_float )silk_energy_FLP( r_ptr, subfr_length + LTP_ORDER ); + temp = 1.0f / silk_max( xx, LTP_CORR_INV_MAX * 0.5f * ( XX_ptr[ 0 ] + XX_ptr[ 24 ] ) + 1.0f ); + silk_scale_vector_FLP( XX_ptr, temp, LTP_ORDER * LTP_ORDER ); + silk_scale_vector_FLP( xX_ptr, temp, LTP_ORDER ); + + r_ptr += subfr_length; + XX_ptr += LTP_ORDER * LTP_ORDER; + xX_ptr += LTP_ORDER; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pitch_lags_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pitch_lags_FLP.c new file mode 100644 index 000000000..dedbcd283 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pitch_lags_FLP.c @@ -0,0 +1,132 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "main_FLP.h" +#include "tuning_parameters.h" + +void silk_find_pitch_lags_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + silk_float res[], /* O Residual */ + const silk_float x[], /* I Speech signal */ + int arch /* I Run-time architecture */ +) +{ + opus_int buf_len; + silk_float thrhld, res_nrg; + const silk_float *x_buf_ptr, *x_buf; + silk_float auto_corr[ MAX_FIND_PITCH_LPC_ORDER + 1 ]; + silk_float A[ MAX_FIND_PITCH_LPC_ORDER ]; + silk_float refl_coef[ MAX_FIND_PITCH_LPC_ORDER ]; + silk_float Wsig[ FIND_PITCH_LPC_WIN_MAX ]; + silk_float *Wsig_ptr; + + /******************************************/ + /* Set up buffer lengths etc based on Fs */ + /******************************************/ + buf_len = psEnc->sCmn.la_pitch + psEnc->sCmn.frame_length + psEnc->sCmn.ltp_mem_length; + + /* Safety check */ + celt_assert( buf_len >= psEnc->sCmn.pitch_LPC_win_length ); + + x_buf = x - psEnc->sCmn.ltp_mem_length; + + /******************************************/ + /* Estimate LPC AR coeficients */ + /******************************************/ + + /* Calculate windowed signal */ + + /* First LA_LTP samples */ + x_buf_ptr = x_buf + buf_len - psEnc->sCmn.pitch_LPC_win_length; + Wsig_ptr = Wsig; + silk_apply_sine_window_FLP( Wsig_ptr, x_buf_ptr, 1, psEnc->sCmn.la_pitch ); + + /* Middle non-windowed samples */ + Wsig_ptr += psEnc->sCmn.la_pitch; + x_buf_ptr += psEnc->sCmn.la_pitch; + silk_memcpy( Wsig_ptr, x_buf_ptr, ( psEnc->sCmn.pitch_LPC_win_length - ( psEnc->sCmn.la_pitch << 1 ) ) * sizeof( silk_float ) ); + + /* Last LA_LTP samples */ + Wsig_ptr += psEnc->sCmn.pitch_LPC_win_length - ( psEnc->sCmn.la_pitch << 1 ); + x_buf_ptr += psEnc->sCmn.pitch_LPC_win_length - ( psEnc->sCmn.la_pitch << 1 ); + silk_apply_sine_window_FLP( Wsig_ptr, x_buf_ptr, 2, psEnc->sCmn.la_pitch ); + + /* Calculate autocorrelation sequence */ + silk_autocorrelation_FLP( auto_corr, Wsig, psEnc->sCmn.pitch_LPC_win_length, psEnc->sCmn.pitchEstimationLPCOrder + 1 ); + + /* Add white noise, as a fraction of the energy */ + auto_corr[ 0 ] += auto_corr[ 0 ] * FIND_PITCH_WHITE_NOISE_FRACTION + 1; + + /* Calculate the reflection coefficients using Schur */ + res_nrg = silk_schur_FLP( refl_coef, auto_corr, psEnc->sCmn.pitchEstimationLPCOrder ); + + /* Prediction gain */ + psEncCtrl->predGain = auto_corr[ 0 ] / silk_max_float( res_nrg, 1.0f ); + + /* Convert reflection coefficients to prediction coefficients */ + silk_k2a_FLP( A, refl_coef, psEnc->sCmn.pitchEstimationLPCOrder ); + + /* Bandwidth expansion */ + silk_bwexpander_FLP( A, psEnc->sCmn.pitchEstimationLPCOrder, FIND_PITCH_BANDWIDTH_EXPANSION ); + + /*****************************************/ + /* LPC analysis filtering */ + /*****************************************/ + silk_LPC_analysis_filter_FLP( res, A, x_buf, buf_len, psEnc->sCmn.pitchEstimationLPCOrder ); + + if( psEnc->sCmn.indices.signalType != TYPE_NO_VOICE_ACTIVITY && psEnc->sCmn.first_frame_after_reset == 0 ) { + /* Threshold for pitch estimator */ + thrhld = 0.6f; + thrhld -= 0.004f * psEnc->sCmn.pitchEstimationLPCOrder; + thrhld -= 0.1f * psEnc->sCmn.speech_activity_Q8 * ( 1.0f / 256.0f ); + thrhld -= 0.15f * (psEnc->sCmn.prevSignalType >> 1); + thrhld -= 0.1f * psEnc->sCmn.input_tilt_Q15 * ( 1.0f / 32768.0f ); + + /*****************************************/ + /* Call Pitch estimator */ + /*****************************************/ + if( silk_pitch_analysis_core_FLP( res, psEncCtrl->pitchL, &psEnc->sCmn.indices.lagIndex, + &psEnc->sCmn.indices.contourIndex, &psEnc->LTPCorr, psEnc->sCmn.prevLag, psEnc->sCmn.pitchEstimationThreshold_Q16 / 65536.0f, + thrhld, psEnc->sCmn.fs_kHz, psEnc->sCmn.pitchEstimationComplexity, psEnc->sCmn.nb_subfr, arch ) == 0 ) + { + psEnc->sCmn.indices.signalType = TYPE_VOICED; + } else { + psEnc->sCmn.indices.signalType = TYPE_UNVOICED; + } + } else { + silk_memset( psEncCtrl->pitchL, 0, sizeof( psEncCtrl->pitchL ) ); + psEnc->sCmn.indices.lagIndex = 0; + psEnc->sCmn.indices.contourIndex = 0; + psEnc->LTPCorr = 0; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pred_coefs_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pred_coefs_FLP.c new file mode 100644 index 000000000..dcf7c5202 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/find_pred_coefs_FLP.c @@ -0,0 +1,116 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +/* Find LPC and LTP coefficients */ +void silk_find_pred_coefs_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float res_pitch[], /* I Residual from pitch analysis */ + const silk_float x[], /* I Speech signal */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + opus_int i; + silk_float XXLTP[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ]; + silk_float xXLTP[ MAX_NB_SUBFR * LTP_ORDER ]; + silk_float invGains[ MAX_NB_SUBFR ]; + opus_int16 NLSF_Q15[ MAX_LPC_ORDER ]; + const silk_float *x_ptr; + silk_float *x_pre_ptr, LPC_in_pre[ MAX_NB_SUBFR * MAX_LPC_ORDER + MAX_FRAME_LENGTH ]; + silk_float minInvGain; + + /* Weighting for weighted least squares */ + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + silk_assert( psEncCtrl->Gains[ i ] > 0.0f ); + invGains[ i ] = 1.0f / psEncCtrl->Gains[ i ]; + } + + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /**********/ + /* VOICED */ + /**********/ + celt_assert( psEnc->sCmn.ltp_mem_length - psEnc->sCmn.predictLPCOrder >= psEncCtrl->pitchL[ 0 ] + LTP_ORDER / 2 ); + + /* LTP analysis */ + silk_find_LTP_FLP( XXLTP, xXLTP, res_pitch, psEncCtrl->pitchL, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr ); + + /* Quantize LTP gain parameters */ + silk_quant_LTP_gains_FLP( psEncCtrl->LTPCoef, psEnc->sCmn.indices.LTPIndex, &psEnc->sCmn.indices.PERIndex, + &psEnc->sCmn.sum_log_gain_Q7, &psEncCtrl->LTPredCodGain, XXLTP, xXLTP, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.arch ); + + /* Control LTP scaling */ + silk_LTP_scale_ctrl_FLP( psEnc, psEncCtrl, condCoding ); + + /* Create LTP residual */ + silk_LTP_analysis_filter_FLP( LPC_in_pre, x - psEnc->sCmn.predictLPCOrder, psEncCtrl->LTPCoef, + psEncCtrl->pitchL, invGains, psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.predictLPCOrder ); + } else { + /************/ + /* UNVOICED */ + /************/ + /* Create signal with prepended subframes, scaled by inverse gains */ + x_ptr = x - psEnc->sCmn.predictLPCOrder; + x_pre_ptr = LPC_in_pre; + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + silk_scale_copy_vector_FLP( x_pre_ptr, x_ptr, invGains[ i ], + psEnc->sCmn.subfr_length + psEnc->sCmn.predictLPCOrder ); + x_pre_ptr += psEnc->sCmn.subfr_length + psEnc->sCmn.predictLPCOrder; + x_ptr += psEnc->sCmn.subfr_length; + } + silk_memset( psEncCtrl->LTPCoef, 0, psEnc->sCmn.nb_subfr * LTP_ORDER * sizeof( silk_float ) ); + psEncCtrl->LTPredCodGain = 0.0f; + psEnc->sCmn.sum_log_gain_Q7 = 0; + } + + /* Limit on total predictive coding gain */ + if( psEnc->sCmn.first_frame_after_reset ) { + minInvGain = 1.0f / MAX_PREDICTION_POWER_GAIN_AFTER_RESET; + } else { + minInvGain = (silk_float)pow( 2, psEncCtrl->LTPredCodGain / 3 ) / MAX_PREDICTION_POWER_GAIN; + minInvGain /= 0.25f + 0.75f * psEncCtrl->coding_quality; + } + + /* LPC_in_pre contains the LTP-filtered input for voiced, and the unfiltered input for unvoiced */ + silk_find_LPC_FLP( &psEnc->sCmn, NLSF_Q15, LPC_in_pre, minInvGain ); + + /* Quantize LSFs */ + silk_process_NLSFs_FLP( &psEnc->sCmn, psEncCtrl->PredCoef, NLSF_Q15, psEnc->sCmn.prev_NLSFq_Q15 ); + + /* Calculate residual energy using quantized LPC coefficients */ + silk_residual_energy_FLP( psEncCtrl->ResNrg, LPC_in_pre, psEncCtrl->PredCoef, psEncCtrl->Gains, + psEnc->sCmn.subfr_length, psEnc->sCmn.nb_subfr, psEnc->sCmn.predictLPCOrder ); + + /* Copy to prediction struct for use in next frame for interpolation */ + silk_memcpy( psEnc->sCmn.prev_NLSFq_Q15, NLSF_Q15, sizeof( psEnc->sCmn.prev_NLSFq_Q15 ) ); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/inner_product_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/inner_product_FLP.c new file mode 100644 index 000000000..cdd39d24c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/inner_product_FLP.c @@ -0,0 +1,59 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* inner product of two silk_float arrays, with result as double */ +double silk_inner_product_FLP( + const silk_float *data1, + const silk_float *data2, + opus_int dataSize +) +{ + opus_int i; + double result; + + /* 4x unrolled loop */ + result = 0.0; + for( i = 0; i < dataSize - 3; i += 4 ) { + result += data1[ i + 0 ] * (double)data2[ i + 0 ] + + data1[ i + 1 ] * (double)data2[ i + 1 ] + + data1[ i + 2 ] * (double)data2[ i + 2 ] + + data1[ i + 3 ] * (double)data2[ i + 3 ]; + } + + /* add any remaining products */ + for( ; i < dataSize; i++ ) { + result += data1[ i ] * (double)data2[ i ]; + } + + return result; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/k2a_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/k2a_FLP.c new file mode 100644 index 000000000..1448008db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/k2a_FLP.c @@ -0,0 +1,54 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* step up function, converts reflection coefficients to prediction coefficients */ +void silk_k2a_FLP( + silk_float *A, /* O prediction coefficients [order] */ + const silk_float *rc, /* I reflection coefficients [order] */ + opus_int32 order /* I prediction order */ +) +{ + opus_int k, n; + silk_float rck, tmp1, tmp2; + + for( k = 0; k < order; k++ ) { + rck = rc[ k ]; + for( n = 0; n < (k + 1) >> 1; n++ ) { + tmp1 = A[ n ]; + tmp2 = A[ k - n - 1 ]; + A[ n ] = tmp1 + tmp2 * rck; + A[ k - n - 1 ] = tmp2 + tmp1 * rck; + } + A[ k ] = -rck; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/main_FLP.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/main_FLP.h new file mode 100644 index 000000000..5dc0ccf4a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/main_FLP.h @@ -0,0 +1,286 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MAIN_FLP_H +#define SILK_MAIN_FLP_H + +#include "SigProc_FLP.h" +#include "SigProc_FIX.h" +#include "structs_FLP.h" +#include "main.h" +#include "define.h" +#include "debug.h" +#include "entenc.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define silk_encoder_state_Fxx silk_encoder_state_FLP +#define silk_encode_do_VAD_Fxx silk_encode_do_VAD_FLP +#define silk_encode_frame_Fxx silk_encode_frame_FLP + +/*********************/ +/* Encoder Functions */ +/*********************/ + +/* High-pass filter with cutoff frequency adaptation based on pitch lag statistics */ +void silk_HP_variable_cutoff( + silk_encoder_state_Fxx state_Fxx[] /* I/O Encoder states */ +); + +/* Encoder main function */ +void silk_encode_do_VAD_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + opus_int activity /* I Decision of Opus voice activity detector */ +); + +/* Encoder main function */ +opus_int silk_encode_frame_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + opus_int32 *pnBytesOut, /* O Number of payload bytes; */ + ec_enc *psRangeEnc, /* I/O compressor data structure */ + opus_int condCoding, /* I The type of conditional coding to use */ + opus_int maxBits, /* I If > 0: maximum number of output bits */ + opus_int useCBR /* I Flag to force constant-bitrate operation */ +); + +/* Initializes the Silk encoder state */ +opus_int silk_init_encoder( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + int arch /* I Run-tim architecture */ +); + +/* Control the Silk encoder */ +opus_int silk_control_encoder( + silk_encoder_state_FLP *psEnc, /* I/O Pointer to Silk encoder state FLP */ + silk_EncControlStruct *encControl, /* I Control structure */ + const opus_int allow_bw_switch, /* I Flag to allow switching audio bandwidth */ + const opus_int channelNb, /* I Channel number */ + const opus_int force_fs_kHz +); + +/**************************/ +/* Noise shaping analysis */ +/**************************/ +/* Compute noise shaping coefficients and initial gain values */ +void silk_noise_shape_analysis_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float *pitch_res, /* I LPC residual from pitch analysis */ + const silk_float *x /* I Input signal [frame_length + la_shape] */ +); + +/* Autocorrelations for a warped frequency axis */ +void silk_warped_autocorrelation_FLP( + silk_float *corr, /* O Result [order + 1] */ + const silk_float *input, /* I Input data to correlate */ + const silk_float warping, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +); + +/* Calculation of LTP state scaling */ +void silk_LTP_scale_ctrl_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/**********************************************/ +/* Prediction Analysis */ +/**********************************************/ +/* Find pitch lags */ +void silk_find_pitch_lags_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + silk_float res[], /* O Residual */ + const silk_float x[], /* I Speech signal */ + int arch /* I Run-time architecture */ +); + +/* Find LPC and LTP coefficients */ +void silk_find_pred_coefs_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float res_pitch[], /* I Residual from pitch analysis */ + const silk_float x[], /* I Speech signal */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/* LPC analysis */ +void silk_find_LPC_FLP( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 NLSF_Q15[], /* O NLSFs */ + const silk_float x[], /* I Input signal */ + const silk_float minInvGain /* I Prediction gain from LTP (dB) */ +); + +/* LTP analysis */ +void silk_find_LTP_FLP( + silk_float XX[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* O Weight for LTP quantization */ + silk_float xX[ MAX_NB_SUBFR * LTP_ORDER ], /* O Weight for LTP quantization */ + const silk_float r_ptr[], /* I LPC residual */ + const opus_int lag[ MAX_NB_SUBFR ], /* I LTP lags */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr /* I number of subframes */ +); + +void silk_LTP_analysis_filter_FLP( + silk_float *LTP_res, /* O LTP res MAX_NB_SUBFR*(pre_lgth+subfr_lngth) */ + const silk_float *x, /* I Input signal, with preceding samples */ + const silk_float B[ LTP_ORDER * MAX_NB_SUBFR ], /* I LTP coefficients for each subframe */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const silk_float invGains[ MAX_NB_SUBFR ], /* I Inverse quantization gains */ + const opus_int subfr_length, /* I Length of each subframe */ + const opus_int nb_subfr, /* I number of subframes */ + const opus_int pre_length /* I Preceding samples for each subframe */ +); + +/* Calculates residual energies of input subframes where all subframes have LPC_order */ +/* of preceding samples */ +void silk_residual_energy_FLP( + silk_float nrgs[ MAX_NB_SUBFR ], /* O Residual energy per subframe */ + const silk_float x[], /* I Input signal */ + silk_float a[ 2 ][ MAX_LPC_ORDER ], /* I AR coefs for each frame half */ + const silk_float gains[], /* I Quantization gains */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I number of subframes */ + const opus_int LPC_order /* I LPC order */ +); + +/* 16th order LPC analysis filter */ +void silk_LPC_analysis_filter_FLP( + silk_float r_LPC[], /* O LPC residual signal */ + const silk_float PredCoef[], /* I LPC coefficients */ + const silk_float s[], /* I Input signal */ + const opus_int length, /* I Length of input signal */ + const opus_int Order /* I LPC order */ +); + +/* LTP tap quantizer */ +void silk_quant_LTP_gains_FLP( + silk_float B[ MAX_NB_SUBFR * LTP_ORDER ], /* O Quantized LTP gains */ + opus_int8 cbk_index[ MAX_NB_SUBFR ], /* O Codebook index */ + opus_int8 *periodicity_index, /* O Periodicity index */ + opus_int32 *sum_log_gain_Q7, /* I/O Cumulative max prediction gain */ + silk_float *pred_gain_dB, /* O LTP prediction gain */ + const silk_float XX[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* I Correlation matrix */ + const silk_float xX[ MAX_NB_SUBFR * LTP_ORDER ], /* I Correlation vector */ + const opus_int subfr_len, /* I Number of samples per subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +); + +/* Residual energy: nrg = wxx - 2 * wXx * c + c' * wXX * c */ +silk_float silk_residual_energy_covar_FLP( /* O Weighted residual energy */ + const silk_float *c, /* I Filter coefficients */ + silk_float *wXX, /* I/O Weighted correlation matrix, reg. out */ + const silk_float *wXx, /* I Weighted correlation vector */ + const silk_float wxx, /* I Weighted correlation value */ + const opus_int D /* I Dimension */ +); + +/* Processing of gains */ +void silk_process_gains_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/******************/ +/* Linear Algebra */ +/******************/ +/* Calculates correlation matrix X'*X */ +void silk_corrMatrix_FLP( + const silk_float *x, /* I x vector [ L+order-1 ] used to create X */ + const opus_int L, /* I Length of vectors */ + const opus_int Order, /* I Max lag for correlation */ + silk_float *XX /* O X'*X correlation matrix [order x order] */ +); + +/* Calculates correlation vector X'*t */ +void silk_corrVector_FLP( + const silk_float *x, /* I x vector [L+order-1] used to create X */ + const silk_float *t, /* I Target vector [L] */ + const opus_int L, /* I Length of vecors */ + const opus_int Order, /* I Max lag for correlation */ + silk_float *Xt /* O X'*t correlation vector [order] */ +); + +/* Apply sine window to signal vector. */ +/* Window types: */ +/* 1 -> sine window from 0 to pi/2 */ +/* 2 -> sine window from pi/2 to pi */ +void silk_apply_sine_window_FLP( + silk_float px_win[], /* O Pointer to windowed signal */ + const silk_float px[], /* I Pointer to input signal */ + const opus_int win_type, /* I Selects a window type */ + const opus_int length /* I Window length, multiple of 4 */ +); + +/* Wrapper functions. Call flp / fix code */ + +/* Convert AR filter coefficients to NLSF parameters */ +void silk_A2NLSF_FLP( + opus_int16 *NLSF_Q15, /* O NLSF vector [ LPC_order ] */ + const silk_float *pAR, /* I LPC coefficients [ LPC_order ] */ + const opus_int LPC_order /* I LPC order */ +); + +/* Convert NLSF parameters to AR prediction filter coefficients */ +void silk_NLSF2A_FLP( + silk_float *pAR, /* O LPC coefficients [ LPC_order ] */ + const opus_int16 *NLSF_Q15, /* I NLSF vector [ LPC_order ] */ + const opus_int LPC_order, /* I LPC order */ + int arch /* I Run-time architecture */ +); + +/* Limit, stabilize, and quantize NLSFs */ +void silk_process_NLSFs_FLP( + silk_encoder_state *psEncC, /* I/O Encoder state */ + silk_float PredCoef[ 2 ][ MAX_LPC_ORDER ], /* O Prediction coefficients */ + opus_int16 NLSF_Q15[ MAX_LPC_ORDER ], /* I/O Normalized LSFs (quant out) (0 - (2^15-1)) */ + const opus_int16 prev_NLSF_Q15[ MAX_LPC_ORDER ] /* I Previous Normalized LSFs (0 - (2^15-1)) */ +); + +/* Floating-point Silk NSQ wrapper */ +void silk_NSQ_wrapper_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + SideInfoIndices *psIndices, /* I/O Quantization indices */ + silk_nsq_state *psNSQ, /* I/O Noise Shaping Quantzation state */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const silk_float x[] /* I Prefiltered input signal */ +); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/noise_shape_analysis_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/noise_shape_analysis_FLP.c new file mode 100644 index 000000000..cb3d8a50b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/noise_shape_analysis_FLP.c @@ -0,0 +1,350 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" +#include "tuning_parameters.h" + +/* Compute gain to make warped filter coefficients have a zero mean log frequency response on a */ +/* non-warped frequency scale. (So that it can be implemented with a minimum-phase monic filter.) */ +/* Note: A monic filter is one with the first coefficient equal to 1.0. In Silk we omit the first */ +/* coefficient in an array of coefficients, for monic filters. */ +static OPUS_INLINE silk_float warped_gain( + const silk_float *coefs, + silk_float lambda, + opus_int order +) { + opus_int i; + silk_float gain; + + lambda = -lambda; + gain = coefs[ order - 1 ]; + for( i = order - 2; i >= 0; i-- ) { + gain = lambda * gain + coefs[ i ]; + } + return (silk_float)( 1.0f / ( 1.0f - lambda * gain ) ); +} + +/* Convert warped filter coefficients to monic pseudo-warped coefficients and limit maximum */ +/* amplitude of monic warped coefficients by using bandwidth expansion on the true coefficients */ +static OPUS_INLINE void warped_true2monic_coefs( + silk_float *coefs, + silk_float lambda, + silk_float limit, + opus_int order +) { + opus_int i, iter, ind = 0; + silk_float tmp, maxabs, chirp, gain; + + /* Convert to monic coefficients */ + for( i = order - 1; i > 0; i-- ) { + coefs[ i - 1 ] -= lambda * coefs[ i ]; + } + gain = ( 1.0f - lambda * lambda ) / ( 1.0f + lambda * coefs[ 0 ] ); + for( i = 0; i < order; i++ ) { + coefs[ i ] *= gain; + } + + /* Limit */ + for( iter = 0; iter < 10; iter++ ) { + /* Find maximum absolute value */ + maxabs = -1.0f; + for( i = 0; i < order; i++ ) { + tmp = silk_abs_float( coefs[ i ] ); + if( tmp > maxabs ) { + maxabs = tmp; + ind = i; + } + } + if( maxabs <= limit ) { + /* Coefficients are within range - done */ + return; + } + + /* Convert back to true warped coefficients */ + for( i = 1; i < order; i++ ) { + coefs[ i - 1 ] += lambda * coefs[ i ]; + } + gain = 1.0f / gain; + for( i = 0; i < order; i++ ) { + coefs[ i ] *= gain; + } + + /* Apply bandwidth expansion */ + chirp = 0.99f - ( 0.8f + 0.1f * iter ) * ( maxabs - limit ) / ( maxabs * ( ind + 1 ) ); + silk_bwexpander_FLP( coefs, order, chirp ); + + /* Convert to monic warped coefficients */ + for( i = order - 1; i > 0; i-- ) { + coefs[ i - 1 ] -= lambda * coefs[ i ]; + } + gain = ( 1.0f - lambda * lambda ) / ( 1.0f + lambda * coefs[ 0 ] ); + for( i = 0; i < order; i++ ) { + coefs[ i ] *= gain; + } + } + silk_assert( 0 ); +} + +static OPUS_INLINE void limit_coefs( + silk_float *coefs, + silk_float limit, + opus_int order +) { + opus_int i, iter, ind = 0; + silk_float tmp, maxabs, chirp; + + for( iter = 0; iter < 10; iter++ ) { + /* Find maximum absolute value */ + maxabs = -1.0f; + for( i = 0; i < order; i++ ) { + tmp = silk_abs_float( coefs[ i ] ); + if( tmp > maxabs ) { + maxabs = tmp; + ind = i; + } + } + if( maxabs <= limit ) { + /* Coefficients are within range - done */ + return; + } + + /* Apply bandwidth expansion */ + chirp = 0.99f - ( 0.8f + 0.1f * iter ) * ( maxabs - limit ) / ( maxabs * ( ind + 1 ) ); + silk_bwexpander_FLP( coefs, order, chirp ); + } + silk_assert( 0 ); +} + +/* Compute noise shaping coefficients and initial gain values */ +void silk_noise_shape_analysis_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + const silk_float *pitch_res, /* I LPC residual from pitch analysis */ + const silk_float *x /* I Input signal [frame_length + la_shape] */ +) +{ + silk_shape_state_FLP *psShapeSt = &psEnc->sShape; + opus_int k, nSamples, nSegs; + silk_float SNR_adj_dB, HarmShapeGain, Tilt; + silk_float nrg, log_energy, log_energy_prev, energy_variation; + silk_float BWExp, gain_mult, gain_add, strength, b, warping; + silk_float x_windowed[ SHAPE_LPC_WIN_MAX ]; + silk_float auto_corr[ MAX_SHAPE_LPC_ORDER + 1 ]; + silk_float rc[ MAX_SHAPE_LPC_ORDER + 1 ]; + const silk_float *x_ptr, *pitch_res_ptr; + + /* Point to start of first LPC analysis block */ + x_ptr = x - psEnc->sCmn.la_shape; + + /****************/ + /* GAIN CONTROL */ + /****************/ + SNR_adj_dB = psEnc->sCmn.SNR_dB_Q7 * ( 1 / 128.0f ); + + /* Input quality is the average of the quality in the lowest two VAD bands */ + psEncCtrl->input_quality = 0.5f * ( psEnc->sCmn.input_quality_bands_Q15[ 0 ] + psEnc->sCmn.input_quality_bands_Q15[ 1 ] ) * ( 1.0f / 32768.0f ); + + /* Coding quality level, between 0.0 and 1.0 */ + psEncCtrl->coding_quality = silk_sigmoid( 0.25f * ( SNR_adj_dB - 20.0f ) ); + + if( psEnc->sCmn.useCBR == 0 ) { + /* Reduce coding SNR during low speech activity */ + b = 1.0f - psEnc->sCmn.speech_activity_Q8 * ( 1.0f / 256.0f ); + SNR_adj_dB -= BG_SNR_DECR_dB * psEncCtrl->coding_quality * ( 0.5f + 0.5f * psEncCtrl->input_quality ) * b * b; + } + + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce gains for periodic signals */ + SNR_adj_dB += HARM_SNR_INCR_dB * psEnc->LTPCorr; + } else { + /* For unvoiced signals and low-quality input, adjust the quality slower than SNR_dB setting */ + SNR_adj_dB += ( -0.4f * psEnc->sCmn.SNR_dB_Q7 * ( 1 / 128.0f ) + 6.0f ) * ( 1.0f - psEncCtrl->input_quality ); + } + + /*************************/ + /* SPARSENESS PROCESSING */ + /*************************/ + /* Set quantizer offset */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Initially set to 0; may be overruled in process_gains(..) */ + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + /* Sparseness measure, based on relative fluctuations of energy per 2 milliseconds */ + nSamples = 2 * psEnc->sCmn.fs_kHz; + energy_variation = 0.0f; + log_energy_prev = 0.0f; + pitch_res_ptr = pitch_res; + nSegs = silk_SMULBB( SUB_FRAME_LENGTH_MS, psEnc->sCmn.nb_subfr ) / 2; + for( k = 0; k < nSegs; k++ ) { + nrg = ( silk_float )nSamples + ( silk_float )silk_energy_FLP( pitch_res_ptr, nSamples ); + log_energy = silk_log2( nrg ); + if( k > 0 ) { + energy_variation += silk_abs_float( log_energy - log_energy_prev ); + } + log_energy_prev = log_energy; + pitch_res_ptr += nSamples; + } + + /* Set quantization offset depending on sparseness measure */ + if( energy_variation > ENERGY_VARIATION_THRESHOLD_QNT_OFFSET * (nSegs-1) ) { + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + psEnc->sCmn.indices.quantOffsetType = 1; + } + } + + /*******************************/ + /* Control bandwidth expansion */ + /*******************************/ + /* More BWE for signals with high prediction gain */ + strength = FIND_PITCH_WHITE_NOISE_FRACTION * psEncCtrl->predGain; /* between 0.0 and 1.0 */ + BWExp = BANDWIDTH_EXPANSION / ( 1.0f + strength * strength ); + + /* Slightly more warping in analysis will move quantization noise up in frequency, where it's better masked */ + warping = (silk_float)psEnc->sCmn.warping_Q16 / 65536.0f + 0.01f * psEncCtrl->coding_quality; + + /********************************************/ + /* Compute noise shaping AR coefs and gains */ + /********************************************/ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + /* Apply window: sine slope followed by flat part followed by cosine slope */ + opus_int shift, slope_part, flat_part; + flat_part = psEnc->sCmn.fs_kHz * 3; + slope_part = ( psEnc->sCmn.shapeWinLength - flat_part ) / 2; + + silk_apply_sine_window_FLP( x_windowed, x_ptr, 1, slope_part ); + shift = slope_part; + silk_memcpy( x_windowed + shift, x_ptr + shift, flat_part * sizeof(silk_float) ); + shift += flat_part; + silk_apply_sine_window_FLP( x_windowed + shift, x_ptr + shift, 2, slope_part ); + + /* Update pointer: next LPC analysis block */ + x_ptr += psEnc->sCmn.subfr_length; + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Calculate warped auto correlation */ + silk_warped_autocorrelation_FLP( auto_corr, x_windowed, warping, + psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder ); + } else { + /* Calculate regular auto correlation */ + silk_autocorrelation_FLP( auto_corr, x_windowed, psEnc->sCmn.shapeWinLength, psEnc->sCmn.shapingLPCOrder + 1 ); + } + + /* Add white noise, as a fraction of energy */ + auto_corr[ 0 ] += auto_corr[ 0 ] * SHAPE_WHITE_NOISE_FRACTION + 1.0f; + + /* Convert correlations to prediction coefficients, and compute residual energy */ + nrg = silk_schur_FLP( rc, auto_corr, psEnc->sCmn.shapingLPCOrder ); + silk_k2a_FLP( &psEncCtrl->AR[ k * MAX_SHAPE_LPC_ORDER ], rc, psEnc->sCmn.shapingLPCOrder ); + psEncCtrl->Gains[ k ] = ( silk_float )sqrt( nrg ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Adjust gain for warping */ + psEncCtrl->Gains[ k ] *= warped_gain( &psEncCtrl->AR[ k * MAX_SHAPE_LPC_ORDER ], warping, psEnc->sCmn.shapingLPCOrder ); + } + + /* Bandwidth expansion for synthesis filter shaping */ + silk_bwexpander_FLP( &psEncCtrl->AR[ k * MAX_SHAPE_LPC_ORDER ], psEnc->sCmn.shapingLPCOrder, BWExp ); + + if( psEnc->sCmn.warping_Q16 > 0 ) { + /* Convert to monic warped prediction coefficients and limit absolute values */ + warped_true2monic_coefs( &psEncCtrl->AR[ k * MAX_SHAPE_LPC_ORDER ], warping, 3.999f, psEnc->sCmn.shapingLPCOrder ); + } else { + /* Limit absolute values */ + limit_coefs( &psEncCtrl->AR[ k * MAX_SHAPE_LPC_ORDER ], 3.999f, psEnc->sCmn.shapingLPCOrder ); + } + } + + /*****************/ + /* Gain tweaking */ + /*****************/ + /* Increase gains during low speech activity */ + gain_mult = (silk_float)pow( 2.0f, -0.16f * SNR_adj_dB ); + gain_add = (silk_float)pow( 2.0f, 0.16f * MIN_QGAIN_DB ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains[ k ] *= gain_mult; + psEncCtrl->Gains[ k ] += gain_add; + } + + /************************************************/ + /* Control low-frequency shaping and noise tilt */ + /************************************************/ + /* Less low frequency shaping for noisy inputs */ + strength = LOW_FREQ_SHAPING * ( 1.0f + LOW_QUALITY_LOW_FREQ_SHAPING_DECR * ( psEnc->sCmn.input_quality_bands_Q15[ 0 ] * ( 1.0f / 32768.0f ) - 1.0f ) ); + strength *= psEnc->sCmn.speech_activity_Q8 * ( 1.0f / 256.0f ); + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Reduce low frequencies quantization noise for periodic signals, depending on pitch lag */ + /*f = 400; freqz([1, -0.98 + 2e-4 * f], [1, -0.97 + 7e-4 * f], 2^12, Fs); axis([0, 1000, -10, 1])*/ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + b = 0.2f / psEnc->sCmn.fs_kHz + 3.0f / psEncCtrl->pitchL[ k ]; + psEncCtrl->LF_MA_shp[ k ] = -1.0f + b; + psEncCtrl->LF_AR_shp[ k ] = 1.0f - b - b * strength; + } + Tilt = - HP_NOISE_COEF - + (1 - HP_NOISE_COEF) * HARM_HP_NOISE_COEF * psEnc->sCmn.speech_activity_Q8 * ( 1.0f / 256.0f ); + } else { + b = 1.3f / psEnc->sCmn.fs_kHz; + psEncCtrl->LF_MA_shp[ 0 ] = -1.0f + b; + psEncCtrl->LF_AR_shp[ 0 ] = 1.0f - b - b * strength * 0.6f; + for( k = 1; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->LF_MA_shp[ k ] = psEncCtrl->LF_MA_shp[ 0 ]; + psEncCtrl->LF_AR_shp[ k ] = psEncCtrl->LF_AR_shp[ 0 ]; + } + Tilt = -HP_NOISE_COEF; + } + + /****************************/ + /* HARMONIC SHAPING CONTROL */ + /****************************/ + if( USE_HARM_SHAPING && psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + /* Harmonic noise shaping */ + HarmShapeGain = HARMONIC_SHAPING; + + /* More harmonic noise shaping for high bitrates or noisy input */ + HarmShapeGain += HIGH_RATE_OR_LOW_QUALITY_HARMONIC_SHAPING * + ( 1.0f - ( 1.0f - psEncCtrl->coding_quality ) * psEncCtrl->input_quality ); + + /* Less harmonic noise shaping for less periodic signals */ + HarmShapeGain *= ( silk_float )sqrt( psEnc->LTPCorr ); + } else { + HarmShapeGain = 0.0f; + } + + /*************************/ + /* Smooth over subframes */ + /*************************/ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psShapeSt->HarmShapeGain_smth += SUBFR_SMTH_COEF * ( HarmShapeGain - psShapeSt->HarmShapeGain_smth ); + psEncCtrl->HarmShapeGain[ k ] = psShapeSt->HarmShapeGain_smth; + psShapeSt->Tilt_smth += SUBFR_SMTH_COEF * ( Tilt - psShapeSt->Tilt_smth ); + psEncCtrl->Tilt[ k ] = psShapeSt->Tilt_smth; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/pitch_analysis_core_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/pitch_analysis_core_FLP.c new file mode 100644 index 000000000..f351bc371 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/pitch_analysis_core_FLP.c @@ -0,0 +1,630 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/***************************************************************************** +* Pitch analyser function +******************************************************************************/ +#include "SigProc_FLP.h" +#include "SigProc_FIX.h" +#include "pitch_est_defines.h" +#include "pitch.h" + +#define SCRATCH_SIZE 22 + +/************************************************************/ +/* Internally used functions */ +/************************************************************/ +static void silk_P_Ana_calc_corr_st3( + silk_float cross_corr_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ], /* O 3 DIM correlation array */ + const silk_float frame[], /* I vector to correlate */ + opus_int start_lag, /* I start lag */ + opus_int sf_length, /* I sub frame length */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +); + +static void silk_P_Ana_calc_energy_st3( + silk_float energies_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ], /* O 3 DIM correlation array */ + const silk_float frame[], /* I vector to correlate */ + opus_int start_lag, /* I start lag */ + opus_int sf_length, /* I sub frame length */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity /* I Complexity setting */ +); + +/************************************************************/ +/* CORE PITCH ANALYSIS FUNCTION */ +/************************************************************/ +opus_int silk_pitch_analysis_core_FLP( /* O Voicing estimate: 0 voiced, 1 unvoiced */ + const silk_float *frame, /* I Signal of length PE_FRAME_LENGTH_MS*Fs_kHz */ + opus_int *pitch_out, /* O Pitch lag values [nb_subfr] */ + opus_int16 *lagIndex, /* O Lag Index */ + opus_int8 *contourIndex, /* O Pitch contour Index */ + silk_float *LTPCorr, /* I/O Normalized correlation; input: value from previous frame */ + opus_int prevLag, /* I Last lag of previous frame; set to zero is unvoiced */ + const silk_float search_thres1, /* I First stage threshold for lag candidates 0 - 1 */ + const silk_float search_thres2, /* I Final threshold for lag candidates 0 - 1 */ + const opus_int Fs_kHz, /* I sample frequency (kHz) */ + const opus_int complexity, /* I Complexity setting, 0-2, where 2 is highest */ + const opus_int nb_subfr, /* I Number of 5 ms subframes */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, k, d, j; + silk_float frame_8kHz[ PE_MAX_FRAME_LENGTH_MS * 8 ]; + silk_float frame_4kHz[ PE_MAX_FRAME_LENGTH_MS * 4 ]; + opus_int16 frame_8_FIX[ PE_MAX_FRAME_LENGTH_MS * 8 ]; + opus_int16 frame_4_FIX[ PE_MAX_FRAME_LENGTH_MS * 4 ]; + opus_int32 filt_state[ 6 ]; + silk_float threshold, contour_bias; + silk_float C[ PE_MAX_NB_SUBFR][ (PE_MAX_LAG >> 1) + 5 ]; + opus_val32 xcorr[ PE_MAX_LAG_MS * 4 - PE_MIN_LAG_MS * 4 + 1 ]; + silk_float CC[ PE_NB_CBKS_STAGE2_EXT ]; + const silk_float *target_ptr, *basis_ptr; + double cross_corr, normalizer, energy, energy_tmp; + opus_int d_srch[ PE_D_SRCH_LENGTH ]; + opus_int16 d_comp[ (PE_MAX_LAG >> 1) + 5 ]; + opus_int length_d_srch, length_d_comp; + silk_float Cmax, CCmax, CCmax_b, CCmax_new_b, CCmax_new; + opus_int CBimax, CBimax_new, lag, start_lag, end_lag, lag_new; + opus_int cbk_size; + silk_float lag_log2, prevLag_log2, delta_lag_log2_sqr; + silk_float energies_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ]; + silk_float cross_corr_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ]; + opus_int lag_counter; + opus_int frame_length, frame_length_8kHz, frame_length_4kHz; + opus_int sf_length, sf_length_8kHz, sf_length_4kHz; + opus_int min_lag, min_lag_8kHz, min_lag_4kHz; + opus_int max_lag, max_lag_8kHz, max_lag_4kHz; + opus_int nb_cbk_search; + const opus_int8 *Lag_CB_ptr; + + /* Check for valid sampling frequency */ + celt_assert( Fs_kHz == 8 || Fs_kHz == 12 || Fs_kHz == 16 ); + + /* Check for valid complexity setting */ + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + silk_assert( search_thres1 >= 0.0f && search_thres1 <= 1.0f ); + silk_assert( search_thres2 >= 0.0f && search_thres2 <= 1.0f ); + + /* Set up frame lengths max / min lag for the sampling frequency */ + frame_length = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * Fs_kHz; + frame_length_4kHz = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * 4; + frame_length_8kHz = ( PE_LTP_MEM_LENGTH_MS + nb_subfr * PE_SUBFR_LENGTH_MS ) * 8; + sf_length = PE_SUBFR_LENGTH_MS * Fs_kHz; + sf_length_4kHz = PE_SUBFR_LENGTH_MS * 4; + sf_length_8kHz = PE_SUBFR_LENGTH_MS * 8; + min_lag = PE_MIN_LAG_MS * Fs_kHz; + min_lag_4kHz = PE_MIN_LAG_MS * 4; + min_lag_8kHz = PE_MIN_LAG_MS * 8; + max_lag = PE_MAX_LAG_MS * Fs_kHz - 1; + max_lag_4kHz = PE_MAX_LAG_MS * 4; + max_lag_8kHz = PE_MAX_LAG_MS * 8 - 1; + + /* Resample from input sampled at Fs_kHz to 8 kHz */ + if( Fs_kHz == 16 ) { + /* Resample to 16 -> 8 khz */ + opus_int16 frame_16_FIX[ 16 * PE_MAX_FRAME_LENGTH_MS ]; + silk_float2short_array( frame_16_FIX, frame, frame_length ); + silk_memset( filt_state, 0, 2 * sizeof( opus_int32 ) ); + silk_resampler_down2( filt_state, frame_8_FIX, frame_16_FIX, frame_length ); + silk_short2float_array( frame_8kHz, frame_8_FIX, frame_length_8kHz ); + } else if( Fs_kHz == 12 ) { + /* Resample to 12 -> 8 khz */ + opus_int16 frame_12_FIX[ 12 * PE_MAX_FRAME_LENGTH_MS ]; + silk_float2short_array( frame_12_FIX, frame, frame_length ); + silk_memset( filt_state, 0, 6 * sizeof( opus_int32 ) ); + silk_resampler_down2_3( filt_state, frame_8_FIX, frame_12_FIX, frame_length ); + silk_short2float_array( frame_8kHz, frame_8_FIX, frame_length_8kHz ); + } else { + celt_assert( Fs_kHz == 8 ); + silk_float2short_array( frame_8_FIX, frame, frame_length_8kHz ); + } + + /* Decimate again to 4 kHz */ + silk_memset( filt_state, 0, 2 * sizeof( opus_int32 ) ); + silk_resampler_down2( filt_state, frame_4_FIX, frame_8_FIX, frame_length_8kHz ); + silk_short2float_array( frame_4kHz, frame_4_FIX, frame_length_4kHz ); + + /* Low-pass filter */ + for( i = frame_length_4kHz - 1; i > 0; i-- ) { + frame_4kHz[ i ] = silk_ADD_SAT16( frame_4kHz[ i ], frame_4kHz[ i - 1 ] ); + } + + /****************************************************************************** + * FIRST STAGE, operating in 4 khz + ******************************************************************************/ + silk_memset(C, 0, sizeof(silk_float) * nb_subfr * ((PE_MAX_LAG >> 1) + 5)); + target_ptr = &frame_4kHz[ silk_LSHIFT( sf_length_4kHz, 2 ) ]; + for( k = 0; k < nb_subfr >> 1; k++ ) { + /* Check that we are within range of the array */ + celt_assert( target_ptr >= frame_4kHz ); + celt_assert( target_ptr + sf_length_8kHz <= frame_4kHz + frame_length_4kHz ); + + basis_ptr = target_ptr - min_lag_4kHz; + + /* Check that we are within range of the array */ + celt_assert( basis_ptr >= frame_4kHz ); + celt_assert( basis_ptr + sf_length_8kHz <= frame_4kHz + frame_length_4kHz ); + + celt_pitch_xcorr( target_ptr, target_ptr-max_lag_4kHz, xcorr, sf_length_8kHz, max_lag_4kHz - min_lag_4kHz + 1, arch ); + + /* Calculate first vector products before loop */ + cross_corr = xcorr[ max_lag_4kHz - min_lag_4kHz ]; + normalizer = silk_energy_FLP( target_ptr, sf_length_8kHz ) + + silk_energy_FLP( basis_ptr, sf_length_8kHz ) + + sf_length_8kHz * 4000.0f; + + C[ 0 ][ min_lag_4kHz ] += (silk_float)( 2 * cross_corr / normalizer ); + + /* From now on normalizer is computed recursively */ + for( d = min_lag_4kHz + 1; d <= max_lag_4kHz; d++ ) { + basis_ptr--; + + /* Check that we are within range of the array */ + silk_assert( basis_ptr >= frame_4kHz ); + silk_assert( basis_ptr + sf_length_8kHz <= frame_4kHz + frame_length_4kHz ); + + cross_corr = xcorr[ max_lag_4kHz - d ]; + + /* Add contribution of new sample and remove contribution from oldest sample */ + normalizer += + basis_ptr[ 0 ] * (double)basis_ptr[ 0 ] - + basis_ptr[ sf_length_8kHz ] * (double)basis_ptr[ sf_length_8kHz ]; + C[ 0 ][ d ] += (silk_float)( 2 * cross_corr / normalizer ); + } + /* Update target pointer */ + target_ptr += sf_length_8kHz; + } + + /* Apply short-lag bias */ + for( i = max_lag_4kHz; i >= min_lag_4kHz; i-- ) { + C[ 0 ][ i ] -= C[ 0 ][ i ] * i / 4096.0f; + } + + /* Sort */ + length_d_srch = 4 + 2 * complexity; + celt_assert( 3 * length_d_srch <= PE_D_SRCH_LENGTH ); + silk_insertion_sort_decreasing_FLP( &C[ 0 ][ min_lag_4kHz ], d_srch, max_lag_4kHz - min_lag_4kHz + 1, length_d_srch ); + + /* Escape if correlation is very low already here */ + Cmax = C[ 0 ][ min_lag_4kHz ]; + if( Cmax < 0.2f ) { + silk_memset( pitch_out, 0, nb_subfr * sizeof( opus_int ) ); + *LTPCorr = 0.0f; + *lagIndex = 0; + *contourIndex = 0; + return 1; + } + + threshold = search_thres1 * Cmax; + for( i = 0; i < length_d_srch; i++ ) { + /* Convert to 8 kHz indices for the sorted correlation that exceeds the threshold */ + if( C[ 0 ][ min_lag_4kHz + i ] > threshold ) { + d_srch[ i ] = silk_LSHIFT( d_srch[ i ] + min_lag_4kHz, 1 ); + } else { + length_d_srch = i; + break; + } + } + celt_assert( length_d_srch > 0 ); + + for( i = min_lag_8kHz - 5; i < max_lag_8kHz + 5; i++ ) { + d_comp[ i ] = 0; + } + for( i = 0; i < length_d_srch; i++ ) { + d_comp[ d_srch[ i ] ] = 1; + } + + /* Convolution */ + for( i = max_lag_8kHz + 3; i >= min_lag_8kHz; i-- ) { + d_comp[ i ] += d_comp[ i - 1 ] + d_comp[ i - 2 ]; + } + + length_d_srch = 0; + for( i = min_lag_8kHz; i < max_lag_8kHz + 1; i++ ) { + if( d_comp[ i + 1 ] > 0 ) { + d_srch[ length_d_srch ] = i; + length_d_srch++; + } + } + + /* Convolution */ + for( i = max_lag_8kHz + 3; i >= min_lag_8kHz; i-- ) { + d_comp[ i ] += d_comp[ i - 1 ] + d_comp[ i - 2 ] + d_comp[ i - 3 ]; + } + + length_d_comp = 0; + for( i = min_lag_8kHz; i < max_lag_8kHz + 4; i++ ) { + if( d_comp[ i ] > 0 ) { + d_comp[ length_d_comp ] = (opus_int16)( i - 2 ); + length_d_comp++; + } + } + + /********************************************************************************** + ** SECOND STAGE, operating at 8 kHz, on lag sections with high correlation + *************************************************************************************/ + /********************************************************************************* + * Find energy of each subframe projected onto its history, for a range of delays + *********************************************************************************/ + silk_memset( C, 0, PE_MAX_NB_SUBFR*((PE_MAX_LAG >> 1) + 5) * sizeof(silk_float)); + + if( Fs_kHz == 8 ) { + target_ptr = &frame[ PE_LTP_MEM_LENGTH_MS * 8 ]; + } else { + target_ptr = &frame_8kHz[ PE_LTP_MEM_LENGTH_MS * 8 ]; + } + for( k = 0; k < nb_subfr; k++ ) { + energy_tmp = silk_energy_FLP( target_ptr, sf_length_8kHz ) + 1.0; + for( j = 0; j < length_d_comp; j++ ) { + d = d_comp[ j ]; + basis_ptr = target_ptr - d; + cross_corr = silk_inner_product_FLP( basis_ptr, target_ptr, sf_length_8kHz ); + if( cross_corr > 0.0f ) { + energy = silk_energy_FLP( basis_ptr, sf_length_8kHz ); + C[ k ][ d ] = (silk_float)( 2 * cross_corr / ( energy + energy_tmp ) ); + } else { + C[ k ][ d ] = 0.0f; + } + } + target_ptr += sf_length_8kHz; + } + + /* search over lag range and lags codebook */ + /* scale factor for lag codebook, as a function of center lag */ + + CCmax = 0.0f; /* This value doesn't matter */ + CCmax_b = -1000.0f; + + CBimax = 0; /* To avoid returning undefined lag values */ + lag = -1; /* To check if lag with strong enough correlation has been found */ + + if( prevLag > 0 ) { + if( Fs_kHz == 12 ) { + prevLag = silk_LSHIFT( prevLag, 1 ) / 3; + } else if( Fs_kHz == 16 ) { + prevLag = silk_RSHIFT( prevLag, 1 ); + } + prevLag_log2 = silk_log2( (silk_float)prevLag ); + } else { + prevLag_log2 = 0; + } + + /* Set up stage 2 codebook based on number of subframes */ + if( nb_subfr == PE_MAX_NB_SUBFR ) { + cbk_size = PE_NB_CBKS_STAGE2_EXT; + Lag_CB_ptr = &silk_CB_lags_stage2[ 0 ][ 0 ]; + if( Fs_kHz == 8 && complexity > SILK_PE_MIN_COMPLEX ) { + /* If input is 8 khz use a larger codebook here because it is last stage */ + nb_cbk_search = PE_NB_CBKS_STAGE2_EXT; + } else { + nb_cbk_search = PE_NB_CBKS_STAGE2; + } + } else { + cbk_size = PE_NB_CBKS_STAGE2_10MS; + Lag_CB_ptr = &silk_CB_lags_stage2_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE2_10MS; + } + + for( k = 0; k < length_d_srch; k++ ) { + d = d_srch[ k ]; + for( j = 0; j < nb_cbk_search; j++ ) { + CC[j] = 0.0f; + for( i = 0; i < nb_subfr; i++ ) { + /* Try all codebooks */ + CC[ j ] += C[ i ][ d + matrix_ptr( Lag_CB_ptr, i, j, cbk_size )]; + } + } + /* Find best codebook */ + CCmax_new = -1000.0f; + CBimax_new = 0; + for( i = 0; i < nb_cbk_search; i++ ) { + if( CC[ i ] > CCmax_new ) { + CCmax_new = CC[ i ]; + CBimax_new = i; + } + } + + /* Bias towards shorter lags */ + lag_log2 = silk_log2( (silk_float)d ); + CCmax_new_b = CCmax_new - PE_SHORTLAG_BIAS * nb_subfr * lag_log2; + + /* Bias towards previous lag */ + if( prevLag > 0 ) { + delta_lag_log2_sqr = lag_log2 - prevLag_log2; + delta_lag_log2_sqr *= delta_lag_log2_sqr; + CCmax_new_b -= PE_PREVLAG_BIAS * nb_subfr * (*LTPCorr) * delta_lag_log2_sqr / ( delta_lag_log2_sqr + 0.5f ); + } + + if( CCmax_new_b > CCmax_b && /* Find maximum biased correlation */ + CCmax_new > nb_subfr * search_thres2 /* Correlation needs to be high enough to be voiced */ + ) { + CCmax_b = CCmax_new_b; + CCmax = CCmax_new; + lag = d; + CBimax = CBimax_new; + } + } + + if( lag == -1 ) { + /* No suitable candidate found */ + silk_memset( pitch_out, 0, PE_MAX_NB_SUBFR * sizeof(opus_int) ); + *LTPCorr = 0.0f; + *lagIndex = 0; + *contourIndex = 0; + return 1; + } + + /* Output normalized correlation */ + *LTPCorr = (silk_float)( CCmax / nb_subfr ); + silk_assert( *LTPCorr >= 0.0f ); + + if( Fs_kHz > 8 ) { + /* Search in original signal */ + + /* Compensate for decimation */ + silk_assert( lag == silk_SAT16( lag ) ); + if( Fs_kHz == 12 ) { + lag = silk_RSHIFT_ROUND( silk_SMULBB( lag, 3 ), 1 ); + } else { /* Fs_kHz == 16 */ + lag = silk_LSHIFT( lag, 1 ); + } + + lag = silk_LIMIT_int( lag, min_lag, max_lag ); + start_lag = silk_max_int( lag - 2, min_lag ); + end_lag = silk_min_int( lag + 2, max_lag ); + lag_new = lag; /* to avoid undefined lag */ + CBimax = 0; /* to avoid undefined lag */ + + CCmax = -1000.0f; + + /* Calculate the correlations and energies needed in stage 3 */ + silk_P_Ana_calc_corr_st3( cross_corr_st3, frame, start_lag, sf_length, nb_subfr, complexity, arch ); + silk_P_Ana_calc_energy_st3( energies_st3, frame, start_lag, sf_length, nb_subfr, complexity ); + + lag_counter = 0; + silk_assert( lag == silk_SAT16( lag ) ); + contour_bias = PE_FLATCONTOUR_BIAS / lag; + + /* Set up cbk parameters according to complexity setting and frame length */ + if( nb_subfr == PE_MAX_NB_SUBFR ) { + nb_cbk_search = (opus_int)silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + } else { + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + } + + target_ptr = &frame[ PE_LTP_MEM_LENGTH_MS * Fs_kHz ]; + energy_tmp = silk_energy_FLP( target_ptr, nb_subfr * sf_length ) + 1.0; + for( d = start_lag; d <= end_lag; d++ ) { + for( j = 0; j < nb_cbk_search; j++ ) { + cross_corr = 0.0; + energy = energy_tmp; + for( k = 0; k < nb_subfr; k++ ) { + cross_corr += cross_corr_st3[ k ][ j ][ lag_counter ]; + energy += energies_st3[ k ][ j ][ lag_counter ]; + } + if( cross_corr > 0.0 ) { + CCmax_new = (silk_float)( 2 * cross_corr / energy ); + /* Reduce depending on flatness of contour */ + CCmax_new *= 1.0f - contour_bias * j; + } else { + CCmax_new = 0.0f; + } + + if( CCmax_new > CCmax && ( d + (opus_int)silk_CB_lags_stage3[ 0 ][ j ] ) <= max_lag ) { + CCmax = CCmax_new; + lag_new = d; + CBimax = j; + } + } + lag_counter++; + } + + for( k = 0; k < nb_subfr; k++ ) { + pitch_out[ k ] = lag_new + matrix_ptr( Lag_CB_ptr, k, CBimax, cbk_size ); + pitch_out[ k ] = silk_LIMIT( pitch_out[ k ], min_lag, PE_MAX_LAG_MS * Fs_kHz ); + } + *lagIndex = (opus_int16)( lag_new - min_lag ); + *contourIndex = (opus_int8)CBimax; + } else { /* Fs_kHz == 8 */ + /* Save Lags */ + for( k = 0; k < nb_subfr; k++ ) { + pitch_out[ k ] = lag + matrix_ptr( Lag_CB_ptr, k, CBimax, cbk_size ); + pitch_out[ k ] = silk_LIMIT( pitch_out[ k ], min_lag_8kHz, PE_MAX_LAG_MS * 8 ); + } + *lagIndex = (opus_int16)( lag - min_lag_8kHz ); + *contourIndex = (opus_int8)CBimax; + } + celt_assert( *lagIndex >= 0 ); + /* return as voiced */ + return 0; +} + +/*********************************************************************** + * Calculates the correlations used in stage 3 search. In order to cover + * the whole lag codebook for all the searched offset lags (lag +- 2), + * the following correlations are needed in each sub frame: + * + * sf1: lag range [-8,...,7] total 16 correlations + * sf2: lag range [-4,...,4] total 9 correlations + * sf3: lag range [-3,....4] total 8 correltions + * sf4: lag range [-6,....8] total 15 correlations + * + * In total 48 correlations. The direct implementation computed in worst + * case 4*12*5 = 240 correlations, but more likely around 120. + ***********************************************************************/ +static void silk_P_Ana_calc_corr_st3( + silk_float cross_corr_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ], /* O 3 DIM correlation array */ + const silk_float frame[], /* I vector to correlate */ + opus_int start_lag, /* I start lag */ + opus_int sf_length, /* I sub frame length */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity, /* I Complexity setting */ + int arch /* I Run-time architecture */ +) +{ + const silk_float *target_ptr; + opus_int i, j, k, lag_counter, lag_low, lag_high; + opus_int nb_cbk_search, delta, idx, cbk_size; + silk_float scratch_mem[ SCRATCH_SIZE ]; + opus_val32 xcorr[ SCRATCH_SIZE ]; + const opus_int8 *Lag_range_ptr, *Lag_CB_ptr; + + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_range_ptr = &silk_Lag_range_stage3[ complexity ][ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + nb_cbk_search = silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1); + Lag_range_ptr = &silk_Lag_range_stage3_10_ms[ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + } + + target_ptr = &frame[ silk_LSHIFT( sf_length, 2 ) ]; /* Pointer to middle of frame */ + for( k = 0; k < nb_subfr; k++ ) { + lag_counter = 0; + + /* Calculate the correlations for each subframe */ + lag_low = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + lag_high = matrix_ptr( Lag_range_ptr, k, 1, 2 ); + silk_assert(lag_high-lag_low+1 <= SCRATCH_SIZE); + celt_pitch_xcorr( target_ptr, target_ptr - start_lag - lag_high, xcorr, sf_length, lag_high - lag_low + 1, arch ); + for( j = lag_low; j <= lag_high; j++ ) { + silk_assert( lag_counter < SCRATCH_SIZE ); + scratch_mem[ lag_counter ] = xcorr[ lag_high - j ]; + lag_counter++; + } + + delta = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + for( i = 0; i < nb_cbk_search; i++ ) { + /* Fill out the 3 dim array that stores the correlations for */ + /* each code_book vector for each start lag */ + idx = matrix_ptr( Lag_CB_ptr, k, i, cbk_size ) - delta; + for( j = 0; j < PE_NB_STAGE3_LAGS; j++ ) { + silk_assert( idx + j < SCRATCH_SIZE ); + silk_assert( idx + j < lag_counter ); + cross_corr_st3[ k ][ i ][ j ] = scratch_mem[ idx + j ]; + } + } + target_ptr += sf_length; + } +} + +/********************************************************************/ +/* Calculate the energies for first two subframes. The energies are */ +/* calculated recursively. */ +/********************************************************************/ +static void silk_P_Ana_calc_energy_st3( + silk_float energies_st3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ][ PE_NB_STAGE3_LAGS ], /* O 3 DIM correlation array */ + const silk_float frame[], /* I vector to correlate */ + opus_int start_lag, /* I start lag */ + opus_int sf_length, /* I sub frame length */ + opus_int nb_subfr, /* I number of subframes */ + opus_int complexity /* I Complexity setting */ +) +{ + const silk_float *target_ptr, *basis_ptr; + double energy; + opus_int k, i, j, lag_counter; + opus_int nb_cbk_search, delta, idx, cbk_size, lag_diff; + silk_float scratch_mem[ SCRATCH_SIZE ]; + const opus_int8 *Lag_range_ptr, *Lag_CB_ptr; + + celt_assert( complexity >= SILK_PE_MIN_COMPLEX ); + celt_assert( complexity <= SILK_PE_MAX_COMPLEX ); + + if( nb_subfr == PE_MAX_NB_SUBFR ) { + Lag_range_ptr = &silk_Lag_range_stage3[ complexity ][ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3[ 0 ][ 0 ]; + nb_cbk_search = silk_nb_cbk_searchs_stage3[ complexity ]; + cbk_size = PE_NB_CBKS_STAGE3_MAX; + } else { + celt_assert( nb_subfr == PE_MAX_NB_SUBFR >> 1); + Lag_range_ptr = &silk_Lag_range_stage3_10_ms[ 0 ][ 0 ]; + Lag_CB_ptr = &silk_CB_lags_stage3_10_ms[ 0 ][ 0 ]; + nb_cbk_search = PE_NB_CBKS_STAGE3_10MS; + cbk_size = PE_NB_CBKS_STAGE3_10MS; + } + + target_ptr = &frame[ silk_LSHIFT( sf_length, 2 ) ]; + for( k = 0; k < nb_subfr; k++ ) { + lag_counter = 0; + + /* Calculate the energy for first lag */ + basis_ptr = target_ptr - ( start_lag + matrix_ptr( Lag_range_ptr, k, 0, 2 ) ); + energy = silk_energy_FLP( basis_ptr, sf_length ) + 1e-3; + silk_assert( energy >= 0.0 ); + scratch_mem[lag_counter] = (silk_float)energy; + lag_counter++; + + lag_diff = ( matrix_ptr( Lag_range_ptr, k, 1, 2 ) - matrix_ptr( Lag_range_ptr, k, 0, 2 ) + 1 ); + for( i = 1; i < lag_diff; i++ ) { + /* remove part outside new window */ + energy -= basis_ptr[sf_length - i] * (double)basis_ptr[sf_length - i]; + silk_assert( energy >= 0.0 ); + + /* add part that comes into window */ + energy += basis_ptr[ -i ] * (double)basis_ptr[ -i ]; + silk_assert( energy >= 0.0 ); + silk_assert( lag_counter < SCRATCH_SIZE ); + scratch_mem[lag_counter] = (silk_float)energy; + lag_counter++; + } + + delta = matrix_ptr( Lag_range_ptr, k, 0, 2 ); + for( i = 0; i < nb_cbk_search; i++ ) { + /* Fill out the 3 dim array that stores the correlations for */ + /* each code_book vector for each start lag */ + idx = matrix_ptr( Lag_CB_ptr, k, i, cbk_size ) - delta; + for( j = 0; j < PE_NB_STAGE3_LAGS; j++ ) { + silk_assert( idx + j < SCRATCH_SIZE ); + silk_assert( idx + j < lag_counter ); + energies_st3[ k ][ i ][ j ] = scratch_mem[ idx + j ]; + silk_assert( energies_st3[ k ][ i ][ j ] >= 0.0f ); + } + } + target_ptr += sf_length; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/process_gains_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/process_gains_FLP.c new file mode 100644 index 000000000..c0da0dae4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/process_gains_FLP.c @@ -0,0 +1,103 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" +#include "tuning_parameters.h" + +/* Processing of gains */ +void silk_process_gains_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + opus_int condCoding /* I The type of conditional coding to use */ +) +{ + silk_shape_state_FLP *psShapeSt = &psEnc->sShape; + opus_int k; + opus_int32 pGains_Q16[ MAX_NB_SUBFR ]; + silk_float s, InvMaxSqrVal, gain, quant_offset; + + /* Gain reduction when LTP coding gain is high */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + s = 1.0f - 0.5f * silk_sigmoid( 0.25f * ( psEncCtrl->LTPredCodGain - 12.0f ) ); + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains[ k ] *= s; + } + } + + /* Limit the quantized signal */ + InvMaxSqrVal = ( silk_float )( pow( 2.0f, 0.33f * ( 21.0f - psEnc->sCmn.SNR_dB_Q7 * ( 1 / 128.0f ) ) ) / psEnc->sCmn.subfr_length ); + + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + /* Soft limit on ratio residual energy and squared gains */ + gain = psEncCtrl->Gains[ k ]; + gain = ( silk_float )sqrt( gain * gain + psEncCtrl->ResNrg[ k ] * InvMaxSqrVal ); + psEncCtrl->Gains[ k ] = silk_min_float( gain, 32767.0f ); + } + + /* Prepare gains for noise shaping quantization */ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + pGains_Q16[ k ] = (opus_int32)( psEncCtrl->Gains[ k ] * 65536.0f ); + } + + /* Save unquantized gains and gain Index */ + silk_memcpy( psEncCtrl->GainsUnq_Q16, pGains_Q16, psEnc->sCmn.nb_subfr * sizeof( opus_int32 ) ); + psEncCtrl->lastGainIndexPrev = psShapeSt->LastGainIndex; + + /* Quantize gains */ + silk_gains_quant( psEnc->sCmn.indices.GainsIndices, pGains_Q16, + &psShapeSt->LastGainIndex, condCoding == CODE_CONDITIONALLY, psEnc->sCmn.nb_subfr ); + + /* Overwrite unquantized gains with quantized gains and convert back to Q0 from Q16 */ + for( k = 0; k < psEnc->sCmn.nb_subfr; k++ ) { + psEncCtrl->Gains[ k ] = pGains_Q16[ k ] / 65536.0f; + } + + /* Set quantizer offset for voiced signals. Larger offset when LTP coding gain is low or tilt is high (ie low-pass) */ + if( psEnc->sCmn.indices.signalType == TYPE_VOICED ) { + if( psEncCtrl->LTPredCodGain + psEnc->sCmn.input_tilt_Q15 * ( 1.0f / 32768.0f ) > 1.0f ) { + psEnc->sCmn.indices.quantOffsetType = 0; + } else { + psEnc->sCmn.indices.quantOffsetType = 1; + } + } + + /* Quantizer boundary adjustment */ + quant_offset = silk_Quantization_Offsets_Q10[ psEnc->sCmn.indices.signalType >> 1 ][ psEnc->sCmn.indices.quantOffsetType ] / 1024.0f; + psEncCtrl->Lambda = LAMBDA_OFFSET + + LAMBDA_DELAYED_DECISIONS * psEnc->sCmn.nStatesDelayedDecision + + LAMBDA_SPEECH_ACT * psEnc->sCmn.speech_activity_Q8 * ( 1.0f / 256.0f ) + + LAMBDA_INPUT_QUALITY * psEncCtrl->input_quality + + LAMBDA_CODING_QUALITY * psEncCtrl->coding_quality + + LAMBDA_QUANT_OFFSET * quant_offset; + + silk_assert( psEncCtrl->Lambda > 0.0f ); + silk_assert( psEncCtrl->Lambda < 2.0f ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/regularize_correlations_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/regularize_correlations_FLP.c new file mode 100644 index 000000000..df4612604 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/regularize_correlations_FLP.c @@ -0,0 +1,48 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +/* Add noise to matrix diagonal */ +void silk_regularize_correlations_FLP( + silk_float *XX, /* I/O Correlation matrices */ + silk_float *xx, /* I/O Correlation values */ + const silk_float noise, /* I Noise energy to add */ + const opus_int D /* I Dimension of XX */ +) +{ + opus_int i; + + for( i = 0; i < D; i++ ) { + matrix_ptr( &XX[ 0 ], i, i, D ) += noise; + } + xx[ 0 ] += noise; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/residual_energy_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/residual_energy_FLP.c new file mode 100644 index 000000000..1bd07b33a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/residual_energy_FLP.c @@ -0,0 +1,117 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +#define MAX_ITERATIONS_RESIDUAL_NRG 10 +#define REGULARIZATION_FACTOR 1e-8f + +/* Residual energy: nrg = wxx - 2 * wXx * c + c' * wXX * c */ +silk_float silk_residual_energy_covar_FLP( /* O Weighted residual energy */ + const silk_float *c, /* I Filter coefficients */ + silk_float *wXX, /* I/O Weighted correlation matrix, reg. out */ + const silk_float *wXx, /* I Weighted correlation vector */ + const silk_float wxx, /* I Weighted correlation value */ + const opus_int D /* I Dimension */ +) +{ + opus_int i, j, k; + silk_float tmp, nrg = 0.0f, regularization; + + /* Safety checks */ + celt_assert( D >= 0 ); + + regularization = REGULARIZATION_FACTOR * ( wXX[ 0 ] + wXX[ D * D - 1 ] ); + for( k = 0; k < MAX_ITERATIONS_RESIDUAL_NRG; k++ ) { + nrg = wxx; + + tmp = 0.0f; + for( i = 0; i < D; i++ ) { + tmp += wXx[ i ] * c[ i ]; + } + nrg -= 2.0f * tmp; + + /* compute c' * wXX * c, assuming wXX is symmetric */ + for( i = 0; i < D; i++ ) { + tmp = 0.0f; + for( j = i + 1; j < D; j++ ) { + tmp += matrix_c_ptr( wXX, i, j, D ) * c[ j ]; + } + nrg += c[ i ] * ( 2.0f * tmp + matrix_c_ptr( wXX, i, i, D ) * c[ i ] ); + } + if( nrg > 0 ) { + break; + } else { + /* Add white noise */ + for( i = 0; i < D; i++ ) { + matrix_c_ptr( wXX, i, i, D ) += regularization; + } + /* Increase noise for next run */ + regularization *= 2.0f; + } + } + if( k == MAX_ITERATIONS_RESIDUAL_NRG ) { + silk_assert( nrg == 0 ); + nrg = 1.0f; + } + + return nrg; +} + +/* Calculates residual energies of input subframes where all subframes have LPC_order */ +/* of preceding samples */ +void silk_residual_energy_FLP( + silk_float nrgs[ MAX_NB_SUBFR ], /* O Residual energy per subframe */ + const silk_float x[], /* I Input signal */ + silk_float a[ 2 ][ MAX_LPC_ORDER ], /* I AR coefs for each frame half */ + const silk_float gains[], /* I Quantization gains */ + const opus_int subfr_length, /* I Subframe length */ + const opus_int nb_subfr, /* I number of subframes */ + const opus_int LPC_order /* I LPC order */ +) +{ + opus_int shift; + silk_float *LPC_res_ptr, LPC_res[ ( MAX_FRAME_LENGTH + MAX_NB_SUBFR * MAX_LPC_ORDER ) / 2 ]; + + LPC_res_ptr = LPC_res + LPC_order; + shift = LPC_order + subfr_length; + + /* Filter input to create the LPC residual for each frame half, and measure subframe energies */ + silk_LPC_analysis_filter_FLP( LPC_res, a[ 0 ], x + 0 * shift, 2 * shift, LPC_order ); + nrgs[ 0 ] = ( silk_float )( gains[ 0 ] * gains[ 0 ] * silk_energy_FLP( LPC_res_ptr + 0 * shift, subfr_length ) ); + nrgs[ 1 ] = ( silk_float )( gains[ 1 ] * gains[ 1 ] * silk_energy_FLP( LPC_res_ptr + 1 * shift, subfr_length ) ); + + if( nb_subfr == MAX_NB_SUBFR ) { + silk_LPC_analysis_filter_FLP( LPC_res, a[ 1 ], x + 2 * shift, 2 * shift, LPC_order ); + nrgs[ 2 ] = ( silk_float )( gains[ 2 ] * gains[ 2 ] * silk_energy_FLP( LPC_res_ptr + 0 * shift, subfr_length ) ); + nrgs[ 3 ] = ( silk_float )( gains[ 3 ] * gains[ 3 ] * silk_energy_FLP( LPC_res_ptr + 1 * shift, subfr_length ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_copy_vector_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_copy_vector_FLP.c new file mode 100644 index 000000000..20db32b3b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_copy_vector_FLP.c @@ -0,0 +1,57 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* copy and multiply a vector by a constant */ +void silk_scale_copy_vector_FLP( + silk_float *data_out, + const silk_float *data_in, + silk_float gain, + opus_int dataSize +) +{ + opus_int i, dataSize4; + + /* 4x unrolled loop */ + dataSize4 = dataSize & 0xFFFC; + for( i = 0; i < dataSize4; i += 4 ) { + data_out[ i + 0 ] = gain * data_in[ i + 0 ]; + data_out[ i + 1 ] = gain * data_in[ i + 1 ]; + data_out[ i + 2 ] = gain * data_in[ i + 2 ]; + data_out[ i + 3 ] = gain * data_in[ i + 3 ]; + } + + /* any remaining elements */ + for( ; i < dataSize; i++ ) { + data_out[ i ] = gain * data_in[ i ]; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_vector_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_vector_FLP.c new file mode 100644 index 000000000..108fdcbed --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/scale_vector_FLP.c @@ -0,0 +1,56 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +/* multiply a vector by a constant */ +void silk_scale_vector_FLP( + silk_float *data1, + silk_float gain, + opus_int dataSize +) +{ + opus_int i, dataSize4; + + /* 4x unrolled loop */ + dataSize4 = dataSize & 0xFFFC; + for( i = 0; i < dataSize4; i += 4 ) { + data1[ i + 0 ] *= gain; + data1[ i + 1 ] *= gain; + data1[ i + 2 ] *= gain; + data1[ i + 3 ] *= gain; + } + + /* any remaining elements */ + for( ; i < dataSize; i++ ) { + data1[ i ] *= gain; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/schur_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/schur_FLP.c new file mode 100644 index 000000000..8526c748d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/schur_FLP.c @@ -0,0 +1,70 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FLP.h" + +silk_float silk_schur_FLP( /* O returns residual energy */ + silk_float refl_coef[], /* O reflection coefficients (length order) */ + const silk_float auto_corr[], /* I autocorrelation sequence (length order+1) */ + opus_int order /* I order */ +) +{ + opus_int k, n; + double C[ SILK_MAX_ORDER_LPC + 1 ][ 2 ]; + double Ctmp1, Ctmp2, rc_tmp; + + celt_assert( order >= 0 && order <= SILK_MAX_ORDER_LPC ); + + /* Copy correlations */ + k = 0; + do { + C[ k ][ 0 ] = C[ k ][ 1 ] = auto_corr[ k ]; + } while( ++k <= order ); + + for( k = 0; k < order; k++ ) { + /* Get reflection coefficient */ + rc_tmp = -C[ k + 1 ][ 0 ] / silk_max_float( C[ 0 ][ 1 ], 1e-9f ); + + /* Save the output */ + refl_coef[ k ] = (silk_float)rc_tmp; + + /* Update correlations */ + for( n = 0; n < order - k; n++ ) { + Ctmp1 = C[ n + k + 1 ][ 0 ]; + Ctmp2 = C[ n ][ 1 ]; + C[ n + k + 1 ][ 0 ] = Ctmp1 + Ctmp2 * rc_tmp; + C[ n ][ 1 ] = Ctmp2 + Ctmp1 * rc_tmp; + } + } + + /* Return residual energy */ + return (silk_float)C[ 0 ][ 1 ]; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/sort_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/sort_FLP.c new file mode 100644 index 000000000..0e18f3195 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/sort_FLP.c @@ -0,0 +1,83 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Insertion sort (fast for already almost sorted arrays): */ +/* Best case: O(n) for an already sorted array */ +/* Worst case: O(n^2) for an inversely sorted array */ + +#include "typedef.h" +#include "SigProc_FLP.h" + +void silk_insertion_sort_decreasing_FLP( + silk_float *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +) +{ + silk_float value; + opus_int i, j; + + /* Safety checks */ + celt_assert( K > 0 ); + celt_assert( L > 0 ); + celt_assert( L >= K ); + + /* Write start indices in index vector */ + for( i = 0; i < K; i++ ) { + idx[ i ] = i; + } + + /* Sort vector elements by value, decreasing order */ + for( i = 1; i < K; i++ ) { + value = a[ i ]; + for( j = i - 1; ( j >= 0 ) && ( value > a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + + /* If less than L values are asked check the remaining values, */ + /* but only spend CPU to ensure that the K first values are correct */ + for( i = K; i < L; i++ ) { + value = a[ i ]; + if( value > a[ K - 1 ] ) { + for( j = K - 2; ( j >= 0 ) && ( value > a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/structs_FLP.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/structs_FLP.h new file mode 100644 index 000000000..3150b386e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/structs_FLP.h @@ -0,0 +1,112 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_STRUCTS_FLP_H +#define SILK_STRUCTS_FLP_H + +#include "typedef.h" +#include "main.h" +#include "structs.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/********************************/ +/* Noise shaping analysis state */ +/********************************/ +typedef struct { + opus_int8 LastGainIndex; + silk_float HarmShapeGain_smth; + silk_float Tilt_smth; +} silk_shape_state_FLP; + +/********************************/ +/* Encoder state FLP */ +/********************************/ +typedef struct { + silk_encoder_state sCmn; /* Common struct, shared with fixed-point code */ + silk_shape_state_FLP sShape; /* Noise shaping state */ + + /* Buffer for find pitch and noise shape analysis */ + silk_float x_buf[ 2 * MAX_FRAME_LENGTH + LA_SHAPE_MAX ];/* Buffer for find pitch and noise shape analysis */ + silk_float LTPCorr; /* Normalized correlation from pitch lag estimator */ +} silk_encoder_state_FLP; + +/************************/ +/* Encoder control FLP */ +/************************/ +typedef struct { + /* Prediction and coding parameters */ + silk_float Gains[ MAX_NB_SUBFR ]; + silk_float PredCoef[ 2 ][ MAX_LPC_ORDER ]; /* holds interpolated and final coefficients */ + silk_float LTPCoef[LTP_ORDER * MAX_NB_SUBFR]; + silk_float LTP_scale; + opus_int pitchL[ MAX_NB_SUBFR ]; + + /* Noise shaping parameters */ + silk_float AR[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ]; + silk_float LF_MA_shp[ MAX_NB_SUBFR ]; + silk_float LF_AR_shp[ MAX_NB_SUBFR ]; + silk_float Tilt[ MAX_NB_SUBFR ]; + silk_float HarmShapeGain[ MAX_NB_SUBFR ]; + silk_float Lambda; + silk_float input_quality; + silk_float coding_quality; + + /* Measures */ + silk_float predGain; + silk_float LTPredCodGain; + silk_float ResNrg[ MAX_NB_SUBFR ]; /* Residual energy per subframe */ + + /* Parameters for CBR mode */ + opus_int32 GainsUnq_Q16[ MAX_NB_SUBFR ]; + opus_int8 lastGainIndexPrev; +} silk_encoder_control_FLP; + +/************************/ +/* Encoder Super Struct */ +/************************/ +typedef struct { + silk_encoder_state_FLP state_Fxx[ ENCODER_NUM_CHANNELS ]; + stereo_enc_state sStereo; + opus_int32 nBitsUsedLBRR; + opus_int32 nBitsExceeded; + opus_int nChannelsAPI; + opus_int nChannelsInternal; + opus_int nPrevChannelsInternal; + opus_int timeSinceSwitchAllowed_ms; + opus_int allowBandwidthSwitch; + opus_int prev_decode_only_middle; +} silk_encoder; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/warped_autocorrelation_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/warped_autocorrelation_FLP.c new file mode 100644 index 000000000..09186e73d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/warped_autocorrelation_FLP.c @@ -0,0 +1,73 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +/* Autocorrelations for a warped frequency axis */ +void silk_warped_autocorrelation_FLP( + silk_float *corr, /* O Result [order + 1] */ + const silk_float *input, /* I Input data to correlate */ + const silk_float warping, /* I Warping coefficient */ + const opus_int length, /* I Length of input */ + const opus_int order /* I Correlation order (even) */ +) +{ + opus_int n, i; + double tmp1, tmp2; + double state[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + double C[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; + + /* Order must be even */ + celt_assert( ( order & 1 ) == 0 ); + + /* Loop over samples */ + for( n = 0; n < length; n++ ) { + tmp1 = input[ n ]; + /* Loop over allpass sections */ + for( i = 0; i < order; i += 2 ) { + /* Output of allpass section */ + tmp2 = state[ i ] + warping * ( state[ i + 1 ] - tmp1 ); + state[ i ] = tmp1; + C[ i ] += state[ 0 ] * tmp1; + /* Output of allpass section */ + tmp1 = state[ i + 1 ] + warping * ( state[ i + 2 ] - tmp2 ); + state[ i + 1 ] = tmp2; + C[ i + 1 ] += state[ 0 ] * tmp2; + } + state[ order ] = tmp1; + C[ order ] += state[ 0 ] * tmp1; + } + + /* Copy correlations in silk_float output format */ + for( i = 0; i < order + 1; i++ ) { + corr[ i ] = ( silk_float )C[ i ]; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/wrappers_FLP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/wrappers_FLP.c new file mode 100644 index 000000000..ad90b874a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/float/wrappers_FLP.c @@ -0,0 +1,207 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main_FLP.h" + +/* Wrappers. Calls flp / fix code */ + +/* Convert AR filter coefficients to NLSF parameters */ +void silk_A2NLSF_FLP( + opus_int16 *NLSF_Q15, /* O NLSF vector [ LPC_order ] */ + const silk_float *pAR, /* I LPC coefficients [ LPC_order ] */ + const opus_int LPC_order /* I LPC order */ +) +{ + opus_int i; + opus_int32 a_fix_Q16[ MAX_LPC_ORDER ]; + + for( i = 0; i < LPC_order; i++ ) { + a_fix_Q16[ i ] = silk_float2int( pAR[ i ] * 65536.0f ); + } + + silk_A2NLSF( NLSF_Q15, a_fix_Q16, LPC_order ); +} + +/* Convert LSF parameters to AR prediction filter coefficients */ +void silk_NLSF2A_FLP( + silk_float *pAR, /* O LPC coefficients [ LPC_order ] */ + const opus_int16 *NLSF_Q15, /* I NLSF vector [ LPC_order ] */ + const opus_int LPC_order, /* I LPC order */ + int arch /* I Run-time architecture */ +) +{ + opus_int i; + opus_int16 a_fix_Q12[ MAX_LPC_ORDER ]; + + silk_NLSF2A( a_fix_Q12, NLSF_Q15, LPC_order, arch ); + + for( i = 0; i < LPC_order; i++ ) { + pAR[ i ] = ( silk_float )a_fix_Q12[ i ] * ( 1.0f / 4096.0f ); + } +} + +/******************************************/ +/* Floating-point NLSF processing wrapper */ +/******************************************/ +void silk_process_NLSFs_FLP( + silk_encoder_state *psEncC, /* I/O Encoder state */ + silk_float PredCoef[ 2 ][ MAX_LPC_ORDER ], /* O Prediction coefficients */ + opus_int16 NLSF_Q15[ MAX_LPC_ORDER ], /* I/O Normalized LSFs (quant out) (0 - (2^15-1)) */ + const opus_int16 prev_NLSF_Q15[ MAX_LPC_ORDER ] /* I Previous Normalized LSFs (0 - (2^15-1)) */ +) +{ + opus_int i, j; + opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ]; + + silk_process_NLSFs( psEncC, PredCoef_Q12, NLSF_Q15, prev_NLSF_Q15); + + for( j = 0; j < 2; j++ ) { + for( i = 0; i < psEncC->predictLPCOrder; i++ ) { + PredCoef[ j ][ i ] = ( silk_float )PredCoef_Q12[ j ][ i ] * ( 1.0f / 4096.0f ); + } + } +} + +/****************************************/ +/* Floating-point Silk NSQ wrapper */ +/****************************************/ +void silk_NSQ_wrapper_FLP( + silk_encoder_state_FLP *psEnc, /* I/O Encoder state FLP */ + silk_encoder_control_FLP *psEncCtrl, /* I/O Encoder control FLP */ + SideInfoIndices *psIndices, /* I/O Quantization indices */ + silk_nsq_state *psNSQ, /* I/O Noise Shaping Quantzation state */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const silk_float x[] /* I Prefiltered input signal */ +) +{ + opus_int i, j; + opus_int16 x16[ MAX_FRAME_LENGTH ]; + opus_int32 Gains_Q16[ MAX_NB_SUBFR ]; + silk_DWORD_ALIGN opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ]; + opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ]; + opus_int LTP_scale_Q14; + + /* Noise shaping parameters */ + opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ]; + opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ]; /* Packs two int16 coefficients per int32 value */ + opus_int Lambda_Q10; + opus_int Tilt_Q14[ MAX_NB_SUBFR ]; + opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ]; + + /* Convert control struct to fix control struct */ + /* Noise shape parameters */ + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + for( j = 0; j < psEnc->sCmn.shapingLPCOrder; j++ ) { + AR_Q13[ i * MAX_SHAPE_LPC_ORDER + j ] = silk_float2int( psEncCtrl->AR[ i * MAX_SHAPE_LPC_ORDER + j ] * 8192.0f ); + } + } + + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + LF_shp_Q14[ i ] = silk_LSHIFT32( silk_float2int( psEncCtrl->LF_AR_shp[ i ] * 16384.0f ), 16 ) | + (opus_uint16)silk_float2int( psEncCtrl->LF_MA_shp[ i ] * 16384.0f ); + Tilt_Q14[ i ] = (opus_int)silk_float2int( psEncCtrl->Tilt[ i ] * 16384.0f ); + HarmShapeGain_Q14[ i ] = (opus_int)silk_float2int( psEncCtrl->HarmShapeGain[ i ] * 16384.0f ); + } + Lambda_Q10 = ( opus_int )silk_float2int( psEncCtrl->Lambda * 1024.0f ); + + /* prediction and coding parameters */ + for( i = 0; i < psEnc->sCmn.nb_subfr * LTP_ORDER; i++ ) { + LTPCoef_Q14[ i ] = (opus_int16)silk_float2int( psEncCtrl->LTPCoef[ i ] * 16384.0f ); + } + + for( j = 0; j < 2; j++ ) { + for( i = 0; i < psEnc->sCmn.predictLPCOrder; i++ ) { + PredCoef_Q12[ j ][ i ] = (opus_int16)silk_float2int( psEncCtrl->PredCoef[ j ][ i ] * 4096.0f ); + } + } + + for( i = 0; i < psEnc->sCmn.nb_subfr; i++ ) { + Gains_Q16[ i ] = silk_float2int( psEncCtrl->Gains[ i ] * 65536.0f ); + silk_assert( Gains_Q16[ i ] > 0 ); + } + + if( psIndices->signalType == TYPE_VOICED ) { + LTP_scale_Q14 = silk_LTPScales_table_Q14[ psIndices->LTP_scaleIndex ]; + } else { + LTP_scale_Q14 = 0; + } + + /* Convert input to fix */ + for( i = 0; i < psEnc->sCmn.frame_length; i++ ) { + x16[ i ] = silk_float2int( x[ i ] ); + } + + /* Call NSQ */ + if( psEnc->sCmn.nStatesDelayedDecision > 1 || psEnc->sCmn.warping_Q16 > 0 ) { + silk_NSQ_del_dec( &psEnc->sCmn, psNSQ, psIndices, x16, pulses, PredCoef_Q12[ 0 ], LTPCoef_Q14, + AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, psEncCtrl->pitchL, Lambda_Q10, LTP_scale_Q14, psEnc->sCmn.arch ); + } else { + silk_NSQ( &psEnc->sCmn, psNSQ, psIndices, x16, pulses, PredCoef_Q12[ 0 ], LTPCoef_Q14, + AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, psEncCtrl->pitchL, Lambda_Q10, LTP_scale_Q14, psEnc->sCmn.arch ); + } +} + +/***********************************************/ +/* Floating-point Silk LTP quantiation wrapper */ +/***********************************************/ +void silk_quant_LTP_gains_FLP( + silk_float B[ MAX_NB_SUBFR * LTP_ORDER ], /* O Quantized LTP gains */ + opus_int8 cbk_index[ MAX_NB_SUBFR ], /* O Codebook index */ + opus_int8 *periodicity_index, /* O Periodicity index */ + opus_int32 *sum_log_gain_Q7, /* I/O Cumulative max prediction gain */ + silk_float *pred_gain_dB, /* O LTP prediction gain */ + const silk_float XX[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ], /* I Correlation matrix */ + const silk_float xX[ MAX_NB_SUBFR * LTP_ORDER ], /* I Correlation vector */ + const opus_int subfr_len, /* I Number of samples per subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +) +{ + opus_int i, pred_gain_dB_Q7; + opus_int16 B_Q14[ MAX_NB_SUBFR * LTP_ORDER ]; + opus_int32 XX_Q17[ MAX_NB_SUBFR * LTP_ORDER * LTP_ORDER ]; + opus_int32 xX_Q17[ MAX_NB_SUBFR * LTP_ORDER ]; + + for( i = 0; i < nb_subfr * LTP_ORDER * LTP_ORDER; i++ ) { + XX_Q17[ i ] = (opus_int32)silk_float2int( XX[ i ] * 131072.0f ); + } + for( i = 0; i < nb_subfr * LTP_ORDER; i++ ) { + xX_Q17[ i ] = (opus_int32)silk_float2int( xX[ i ] * 131072.0f ); + } + + silk_quant_LTP_gains( B_Q14, cbk_index, periodicity_index, sum_log_gain_Q7, &pred_gain_dB_Q7, XX_Q17, xX_Q17, subfr_len, nb_subfr, arch ); + + for( i = 0; i < nb_subfr * LTP_ORDER; i++ ) { + B[ i ] = (silk_float)B_Q14[ i ] * ( 1.0f / 16384.0f ); + } + + *pred_gain_dB = (silk_float)pred_gain_dB_Q7 * ( 1.0f / 128.0f ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/gain_quant.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/gain_quant.c new file mode 100644 index 000000000..ee65245aa --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/gain_quant.c @@ -0,0 +1,142 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +#define OFFSET ( ( MIN_QGAIN_DB * 128 ) / 6 + 16 * 128 ) +#define SCALE_Q16 ( ( 65536 * ( N_LEVELS_QGAIN - 1 ) ) / ( ( ( MAX_QGAIN_DB - MIN_QGAIN_DB ) * 128 ) / 6 ) ) +#define INV_SCALE_Q16 ( ( 65536 * ( ( ( MAX_QGAIN_DB - MIN_QGAIN_DB ) * 128 ) / 6 ) ) / ( N_LEVELS_QGAIN - 1 ) ) + +/* Gain scalar quantization with hysteresis, uniform on log scale */ +void silk_gains_quant( + opus_int8 ind[ MAX_NB_SUBFR ], /* O gain indices */ + opus_int32 gain_Q16[ MAX_NB_SUBFR ], /* I/O gains (quantized out) */ + opus_int8 *prev_ind, /* I/O last index in previous frame */ + const opus_int conditional, /* I first gain is delta coded if 1 */ + const opus_int nb_subfr /* I number of subframes */ +) +{ + opus_int k, double_step_size_threshold; + + for( k = 0; k < nb_subfr; k++ ) { + /* Convert to log scale, scale, floor() */ + ind[ k ] = silk_SMULWB( SCALE_Q16, silk_lin2log( gain_Q16[ k ] ) - OFFSET ); + + /* Round towards previous quantized gain (hysteresis) */ + if( ind[ k ] < *prev_ind ) { + ind[ k ]++; + } + ind[ k ] = silk_LIMIT_int( ind[ k ], 0, N_LEVELS_QGAIN - 1 ); + + /* Compute delta indices and limit */ + if( k == 0 && conditional == 0 ) { + /* Full index */ + ind[ k ] = silk_LIMIT_int( ind[ k ], *prev_ind + MIN_DELTA_GAIN_QUANT, N_LEVELS_QGAIN - 1 ); + *prev_ind = ind[ k ]; + } else { + /* Delta index */ + ind[ k ] = ind[ k ] - *prev_ind; + + /* Double the quantization step size for large gain increases, so that the max gain level can be reached */ + double_step_size_threshold = 2 * MAX_DELTA_GAIN_QUANT - N_LEVELS_QGAIN + *prev_ind; + if( ind[ k ] > double_step_size_threshold ) { + ind[ k ] = double_step_size_threshold + silk_RSHIFT( ind[ k ] - double_step_size_threshold + 1, 1 ); + } + + ind[ k ] = silk_LIMIT_int( ind[ k ], MIN_DELTA_GAIN_QUANT, MAX_DELTA_GAIN_QUANT ); + + /* Accumulate deltas */ + if( ind[ k ] > double_step_size_threshold ) { + *prev_ind += silk_LSHIFT( ind[ k ], 1 ) - double_step_size_threshold; + *prev_ind = silk_min_int( *prev_ind, N_LEVELS_QGAIN - 1 ); + } else { + *prev_ind += ind[ k ]; + } + + /* Shift to make non-negative */ + ind[ k ] -= MIN_DELTA_GAIN_QUANT; + } + + /* Scale and convert to linear scale */ + gain_Q16[ k ] = silk_log2lin( silk_min_32( silk_SMULWB( INV_SCALE_Q16, *prev_ind ) + OFFSET, 3967 ) ); /* 3967 = 31 in Q7 */ + } +} + +/* Gains scalar dequantization, uniform on log scale */ +void silk_gains_dequant( + opus_int32 gain_Q16[ MAX_NB_SUBFR ], /* O quantized gains */ + const opus_int8 ind[ MAX_NB_SUBFR ], /* I gain indices */ + opus_int8 *prev_ind, /* I/O last index in previous frame */ + const opus_int conditional, /* I first gain is delta coded if 1 */ + const opus_int nb_subfr /* I number of subframes */ +) +{ + opus_int k, ind_tmp, double_step_size_threshold; + + for( k = 0; k < nb_subfr; k++ ) { + if( k == 0 && conditional == 0 ) { + /* Gain index is not allowed to go down more than 16 steps (~21.8 dB) */ + *prev_ind = silk_max_int( ind[ k ], *prev_ind - 16 ); + } else { + /* Delta index */ + ind_tmp = ind[ k ] + MIN_DELTA_GAIN_QUANT; + + /* Accumulate deltas */ + double_step_size_threshold = 2 * MAX_DELTA_GAIN_QUANT - N_LEVELS_QGAIN + *prev_ind; + if( ind_tmp > double_step_size_threshold ) { + *prev_ind += silk_LSHIFT( ind_tmp, 1 ) - double_step_size_threshold; + } else { + *prev_ind += ind_tmp; + } + } + *prev_ind = silk_LIMIT_int( *prev_ind, 0, N_LEVELS_QGAIN - 1 ); + + /* Scale and convert to linear scale */ + gain_Q16[ k ] = silk_log2lin( silk_min_32( silk_SMULWB( INV_SCALE_Q16, *prev_ind ) + OFFSET, 3967 ) ); /* 3967 = 31 in Q7 */ + } +} + +/* Compute unique identifier of gain indices vector */ +opus_int32 silk_gains_ID( /* O returns unique identifier of gains */ + const opus_int8 ind[ MAX_NB_SUBFR ], /* I gain indices */ + const opus_int nb_subfr /* I number of subframes */ +) +{ + opus_int k; + opus_int32 gainsID; + + gainsID = 0; + for( k = 0; k < nb_subfr; k++ ) { + gainsID = silk_ADD_LSHIFT32( ind[ k ], gainsID, 8 ); + } + + return gainsID; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_decoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_decoder.c new file mode 100644 index 000000000..16c03dcd1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_decoder.c @@ -0,0 +1,57 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/************************/ +/* Init Decoder State */ +/************************/ +opus_int silk_init_decoder( + silk_decoder_state *psDec /* I/O Decoder state pointer */ +) +{ + /* Clear the entire encoder state, except anything copied */ + silk_memset( psDec, 0, sizeof( silk_decoder_state ) ); + + /* Used to deactivate LSF interpolation */ + psDec->first_frame_after_reset = 1; + psDec->prev_gain_Q16 = 65536; + psDec->arch = opus_select_arch(); + + /* Reset CNG state */ + silk_CNG_Reset( psDec ); + + /* Reset PLC state */ + silk_PLC_Reset( psDec ); + + return(0); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_encoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_encoder.c new file mode 100644 index 000000000..65995c33f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/init_encoder.c @@ -0,0 +1,64 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef FIXED_POINT +#include "main_FIX.h" +#else +#include "main_FLP.h" +#endif +#include "tuning_parameters.h" +#include "cpu_support.h" + +/*********************************/ +/* Initialize Silk Encoder state */ +/*********************************/ +opus_int silk_init_encoder( + silk_encoder_state_Fxx *psEnc, /* I/O Pointer to Silk FIX encoder state */ + int arch /* I Run-time architecture */ +) +{ + opus_int ret = 0; + + /* Clear the entire encoder state */ + silk_memset( psEnc, 0, sizeof( silk_encoder_state_Fxx ) ); + + psEnc->sCmn.arch = arch; + + psEnc->sCmn.variable_HP_smth1_Q15 = silk_LSHIFT( silk_lin2log( SILK_FIX_CONST( VARIABLE_HP_MIN_CUTOFF_HZ, 16 ) ) - ( 16 << 7 ), 8 ); + psEnc->sCmn.variable_HP_smth2_Q15 = psEnc->sCmn.variable_HP_smth1_Q15; + + /* Used to deactivate LSF interpolation, pitch prediction */ + psEnc->sCmn.first_frame_after_reset = 1; + + /* Initialize Silk VAD */ + ret += silk_VAD_Init( &psEnc->sCmn.sVAD ); + + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/inner_prod_aligned.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/inner_prod_aligned.c new file mode 100644 index 000000000..257ae9e04 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/inner_prod_aligned.c @@ -0,0 +1,47 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +opus_int32 silk_inner_prod_aligned_scale( + const opus_int16 *const inVec1, /* I input vector 1 */ + const opus_int16 *const inVec2, /* I input vector 2 */ + const opus_int scale, /* I number of bits to shift */ + const opus_int len /* I vector lengths */ +) +{ + opus_int i; + opus_int32 sum = 0; + for( i = 0; i < len; i++ ) { + sum = silk_ADD_RSHIFT32( sum, silk_SMULBB( inVec1[ i ], inVec2[ i ] ), scale ); + } + return sum; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/interpolate.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/interpolate.c new file mode 100644 index 000000000..833c28ef8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/interpolate.c @@ -0,0 +1,51 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Interpolate two vectors */ +void silk_interpolate( + opus_int16 xi[ MAX_LPC_ORDER ], /* O interpolated vector */ + const opus_int16 x0[ MAX_LPC_ORDER ], /* I first vector */ + const opus_int16 x1[ MAX_LPC_ORDER ], /* I second vector */ + const opus_int ifact_Q2, /* I interp. factor, weight on 2nd vector */ + const opus_int d /* I number of parameters */ +) +{ + opus_int i; + + celt_assert( ifact_Q2 >= 0 ); + celt_assert( ifact_Q2 <= 4 ); + + for( i = 0; i < d; i++ ) { + xi[ i ] = (opus_int16)silk_ADD_RSHIFT( x0[ i ], silk_SMULBB( x1[ i ] - x0[ i ], ifact_Q2 ), 2 ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/lin2log.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/lin2log.c new file mode 100644 index 000000000..0d5155aa8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/lin2log.c @@ -0,0 +1,46 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +/* Approximation of 128 * log2() (very close inverse of silk_log2lin()) */ +/* Convert input to a log scale */ +opus_int32 silk_lin2log( + const opus_int32 inLin /* I input in linear scale */ +) +{ + opus_int32 lz, frac_Q7; + + silk_CLZ_FRAC( inLin, &lz, &frac_Q7 ); + + /* Piece-wise parabolic approximation */ + return silk_ADD_LSHIFT32( silk_SMLAWB( frac_Q7, silk_MUL( frac_Q7, 128 - frac_Q7 ), 179 ), 31 - lz, 7 ); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/log2lin.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/log2lin.c new file mode 100644 index 000000000..b7c48e474 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/log2lin.c @@ -0,0 +1,58 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Approximation of 2^() (very close inverse of silk_lin2log()) */ +/* Convert input to a linear scale */ +opus_int32 silk_log2lin( + const opus_int32 inLog_Q7 /* I input on log scale */ +) +{ + opus_int32 out, frac_Q7; + + if( inLog_Q7 < 0 ) { + return 0; + } else if ( inLog_Q7 >= 3967 ) { + return silk_int32_MAX; + } + + out = silk_LSHIFT( 1, silk_RSHIFT( inLog_Q7, 7 ) ); + frac_Q7 = inLog_Q7 & 0x7F; + if( inLog_Q7 < 2048 ) { + /* Piece-wise parabolic approximation */ + out = silk_ADD_RSHIFT32( out, silk_MUL( out, silk_SMLAWB( frac_Q7, silk_SMULBB( frac_Q7, 128 - frac_Q7 ), -174 ) ), 7 ); + } else { + /* Piece-wise parabolic approximation */ + out = silk_MLA( out, silk_RSHIFT( out, 7 ), silk_SMLAWB( frac_Q7, silk_SMULBB( frac_Q7, 128 - frac_Q7 ), -174 ) ); + } + return out; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/macros.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/macros.h new file mode 100644 index 000000000..3c67b6e5d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/macros.h @@ -0,0 +1,151 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MACROS_H +#define SILK_MACROS_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_types.h" +#include "opus_defines.h" +#include "arch.h" + +/* This is an OPUS_INLINE header file for general platform. */ + +/* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ +#if OPUS_FAST_INT64 +#define silk_SMULWB(a32, b32) ((opus_int32)(((a32) * (opus_int64)((opus_int16)(b32))) >> 16)) +#else +#define silk_SMULWB(a32, b32) ((((a32) >> 16) * (opus_int32)((opus_int16)(b32))) + ((((a32) & 0x0000FFFF) * (opus_int32)((opus_int16)(b32))) >> 16)) +#endif + +/* a32 + (b32 * (opus_int32)((opus_int16)(c32))) >> 16 output have to be 32bit int */ +#if OPUS_FAST_INT64 +#define silk_SMLAWB(a32, b32, c32) ((opus_int32)((a32) + (((b32) * (opus_int64)((opus_int16)(c32))) >> 16))) +#else +#define silk_SMLAWB(a32, b32, c32) ((a32) + ((((b32) >> 16) * (opus_int32)((opus_int16)(c32))) + ((((b32) & 0x0000FFFF) * (opus_int32)((opus_int16)(c32))) >> 16))) +#endif + +/* (a32 * (b32 >> 16)) >> 16 */ +#if OPUS_FAST_INT64 +#define silk_SMULWT(a32, b32) ((opus_int32)(((a32) * (opus_int64)((b32) >> 16)) >> 16)) +#else +#define silk_SMULWT(a32, b32) (((a32) >> 16) * ((b32) >> 16) + ((((a32) & 0x0000FFFF) * ((b32) >> 16)) >> 16)) +#endif + +/* a32 + (b32 * (c32 >> 16)) >> 16 */ +#if OPUS_FAST_INT64 +#define silk_SMLAWT(a32, b32, c32) ((opus_int32)((a32) + (((b32) * ((opus_int64)(c32) >> 16)) >> 16))) +#else +#define silk_SMLAWT(a32, b32, c32) ((a32) + (((b32) >> 16) * ((c32) >> 16)) + ((((b32) & 0x0000FFFF) * ((c32) >> 16)) >> 16)) +#endif + +/* (opus_int32)((opus_int16)(a3))) * (opus_int32)((opus_int16)(b32)) output have to be 32bit int */ +#define silk_SMULBB(a32, b32) ((opus_int32)((opus_int16)(a32)) * (opus_int32)((opus_int16)(b32))) + +/* a32 + (opus_int32)((opus_int16)(b32)) * (opus_int32)((opus_int16)(c32)) output have to be 32bit int */ +#define silk_SMLABB(a32, b32, c32) ((a32) + ((opus_int32)((opus_int16)(b32))) * (opus_int32)((opus_int16)(c32))) + +/* (opus_int32)((opus_int16)(a32)) * (b32 >> 16) */ +#define silk_SMULBT(a32, b32) ((opus_int32)((opus_int16)(a32)) * ((b32) >> 16)) + +/* a32 + (opus_int32)((opus_int16)(b32)) * (c32 >> 16) */ +#define silk_SMLABT(a32, b32, c32) ((a32) + ((opus_int32)((opus_int16)(b32))) * ((c32) >> 16)) + +/* a64 + (b32 * c32) */ +#define silk_SMLAL(a64, b32, c32) (silk_ADD64((a64), ((opus_int64)(b32) * (opus_int64)(c32)))) + +/* (a32 * b32) >> 16 */ +#if OPUS_FAST_INT64 +#define silk_SMULWW(a32, b32) ((opus_int32)(((opus_int64)(a32) * (b32)) >> 16)) +#else +#define silk_SMULWW(a32, b32) silk_MLA(silk_SMULWB((a32), (b32)), (a32), silk_RSHIFT_ROUND((b32), 16)) +#endif + +/* a32 + ((b32 * c32) >> 16) */ +#if OPUS_FAST_INT64 +#define silk_SMLAWW(a32, b32, c32) ((opus_int32)((a32) + (((opus_int64)(b32) * (c32)) >> 16))) +#else +#define silk_SMLAWW(a32, b32, c32) silk_MLA(silk_SMLAWB((a32), (b32), (c32)), (b32), silk_RSHIFT_ROUND((c32), 16)) +#endif + +/* add/subtract with output saturated */ +#define silk_ADD_SAT32(a, b) ((((opus_uint32)(a) + (opus_uint32)(b)) & 0x80000000) == 0 ? \ + ((((a) & (b)) & 0x80000000) != 0 ? silk_int32_MIN : (a)+(b)) : \ + ((((a) | (b)) & 0x80000000) == 0 ? silk_int32_MAX : (a)+(b)) ) + +#define silk_SUB_SAT32(a, b) ((((opus_uint32)(a)-(opus_uint32)(b)) & 0x80000000) == 0 ? \ + (( (a) & ((b)^0x80000000) & 0x80000000) ? silk_int32_MIN : (a)-(b)) : \ + ((((a)^0x80000000) & (b) & 0x80000000) ? silk_int32_MAX : (a)-(b)) ) + +#if defined(MIPSr1_ASM) +#include "mips/macros_mipsr1.h" +#endif + +#include "ecintrin.h" +#ifndef OVERRIDE_silk_CLZ16 +static OPUS_INLINE opus_int32 silk_CLZ16(opus_int16 in16) +{ + return 32 - EC_ILOG(in16<<16|0x8000); +} +#endif + +#ifndef OVERRIDE_silk_CLZ32 +static OPUS_INLINE opus_int32 silk_CLZ32(opus_int32 in32) +{ + return in32 ? 32 - EC_ILOG(in32) : 32; +} +#endif + +/* Row based */ +#define matrix_ptr(Matrix_base_adr, row, column, N) \ + (*((Matrix_base_adr) + ((row)*(N)+(column)))) +#define matrix_adr(Matrix_base_adr, row, column, N) \ + ((Matrix_base_adr) + ((row)*(N)+(column))) + +/* Column based */ +#ifndef matrix_c_ptr +# define matrix_c_ptr(Matrix_base_adr, row, column, M) \ + (*((Matrix_base_adr) + ((row)+(M)*(column)))) +#endif + +#ifdef OPUS_ARM_INLINE_ASM +#include "arm/macros_armv4.h" +#endif + +#ifdef OPUS_ARM_INLINE_EDSP +#include "arm/macros_armv5e.h" +#endif + +#ifdef OPUS_ARM_PRESUME_AARCH64_NEON_INTR +#include "arm/macros_arm64.h" +#endif + +#endif /* SILK_MACROS_H */ + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/main.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/main.h new file mode 100644 index 000000000..1a33eed54 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/main.h @@ -0,0 +1,476 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_MAIN_H +#define SILK_MAIN_H + +#include "SigProc_FIX.h" +#include "define.h" +#include "structs.h" +#include "tables.h" +#include "PLC.h" +#include "control.h" +#include "debug.h" +#include "entenc.h" +#include "entdec.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#include "x86/main_sse.h" +#endif + +#if (defined(OPUS_ARM_ASM) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)) +#include "arm/NSQ_del_dec_arm.h" +#endif + +/* Convert Left/Right stereo signal to adaptive Mid/Side representation */ +void silk_stereo_LR_to_MS( + stereo_enc_state *state, /* I/O State */ + opus_int16 x1[], /* I/O Left input signal, becomes mid signal */ + opus_int16 x2[], /* I/O Right input signal, becomes side signal */ + opus_int8 ix[ 2 ][ 3 ], /* O Quantization indices */ + opus_int8 *mid_only_flag, /* O Flag: only mid signal coded */ + opus_int32 mid_side_rates_bps[], /* O Bitrates for mid and side signals */ + opus_int32 total_rate_bps, /* I Total bitrate */ + opus_int prev_speech_act_Q8, /* I Speech activity level in previous frame */ + opus_int toMono, /* I Last frame before a stereo->mono transition */ + opus_int fs_kHz, /* I Sample rate (kHz) */ + opus_int frame_length /* I Number of samples */ +); + +/* Convert adaptive Mid/Side representation to Left/Right stereo signal */ +void silk_stereo_MS_to_LR( + stereo_dec_state *state, /* I/O State */ + opus_int16 x1[], /* I/O Left input signal, becomes mid signal */ + opus_int16 x2[], /* I/O Right input signal, becomes side signal */ + const opus_int32 pred_Q13[], /* I Predictors */ + opus_int fs_kHz, /* I Samples rate (kHz) */ + opus_int frame_length /* I Number of samples */ +); + +/* Find least-squares prediction gain for one signal based on another and quantize it */ +opus_int32 silk_stereo_find_predictor( /* O Returns predictor in Q13 */ + opus_int32 *ratio_Q14, /* O Ratio of residual and mid energies */ + const opus_int16 x[], /* I Basis signal */ + const opus_int16 y[], /* I Target signal */ + opus_int32 mid_res_amp_Q0[], /* I/O Smoothed mid, residual norms */ + opus_int length, /* I Number of samples */ + opus_int smooth_coef_Q16 /* I Smoothing coefficient */ +); + +/* Quantize mid/side predictors */ +void silk_stereo_quant_pred( + opus_int32 pred_Q13[], /* I/O Predictors (out: quantized) */ + opus_int8 ix[ 2 ][ 3 ] /* O Quantization indices */ +); + +/* Entropy code the mid/side quantization indices */ +void silk_stereo_encode_pred( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int8 ix[ 2 ][ 3 ] /* I Quantization indices */ +); + +/* Entropy code the mid-only flag */ +void silk_stereo_encode_mid_only( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int8 mid_only_flag +); + +/* Decode mid/side predictors */ +void silk_stereo_decode_pred( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int32 pred_Q13[] /* O Predictors */ +); + +/* Decode mid-only flag */ +void silk_stereo_decode_mid_only( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int *decode_only_mid /* O Flag that only mid channel has been coded */ +); + +/* Encodes signs of excitation */ +void silk_encode_signs( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + const opus_int8 pulses[], /* I pulse signal */ + opus_int length, /* I length of input */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I Quantization offset type */ + const opus_int sum_pulses[ MAX_NB_SHELL_BLOCKS ] /* I Sum of absolute pulses per block */ +); + +/* Decodes signs of excitation */ +void silk_decode_signs( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pulses[], /* I/O pulse signal */ + opus_int length, /* I length of input */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I Quantization offset type */ + const opus_int sum_pulses[ MAX_NB_SHELL_BLOCKS ] /* I Sum of absolute pulses per block */ +); + +/* Check encoder control struct */ +opus_int check_control_input( + silk_EncControlStruct *encControl /* I Control structure */ +); + +/* Control internal sampling rate */ +opus_int silk_control_audio_bandwidth( + silk_encoder_state *psEncC, /* I/O Pointer to Silk encoder state */ + silk_EncControlStruct *encControl /* I Control structure */ +); + +/* Control SNR of redidual quantizer */ +opus_int silk_control_SNR( + silk_encoder_state *psEncC, /* I/O Pointer to Silk encoder state */ + opus_int32 TargetRate_bps /* I Target max bitrate (bps) */ +); + +/***************/ +/* Shell coder */ +/***************/ + +/* Encode quantization indices of excitation */ +void silk_encode_pulses( + ec_enc *psRangeEnc, /* I/O compressor data structure */ + const opus_int signalType, /* I Signal type */ + const opus_int quantOffsetType, /* I quantOffsetType */ + opus_int8 pulses[], /* I quantization indices */ + const opus_int frame_length /* I Frame length */ +); + +/* Shell encoder, operates on one shell code frame of 16 pulses */ +void silk_shell_encoder( + ec_enc *psRangeEnc, /* I/O compressor data structure */ + const opus_int *pulses0 /* I data: nonnegative pulse amplitudes */ +); + +/* Shell decoder, operates on one shell code frame of 16 pulses */ +void silk_shell_decoder( + opus_int16 *pulses0, /* O data: nonnegative pulse amplitudes */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + const opus_int pulses4 /* I number of pulses per pulse-subframe */ +); + +/* Gain scalar quantization with hysteresis, uniform on log scale */ +void silk_gains_quant( + opus_int8 ind[ MAX_NB_SUBFR ], /* O gain indices */ + opus_int32 gain_Q16[ MAX_NB_SUBFR ], /* I/O gains (quantized out) */ + opus_int8 *prev_ind, /* I/O last index in previous frame */ + const opus_int conditional, /* I first gain is delta coded if 1 */ + const opus_int nb_subfr /* I number of subframes */ +); + +/* Gains scalar dequantization, uniform on log scale */ +void silk_gains_dequant( + opus_int32 gain_Q16[ MAX_NB_SUBFR ], /* O quantized gains */ + const opus_int8 ind[ MAX_NB_SUBFR ], /* I gain indices */ + opus_int8 *prev_ind, /* I/O last index in previous frame */ + const opus_int conditional, /* I first gain is delta coded if 1 */ + const opus_int nb_subfr /* I number of subframes */ +); + +/* Compute unique identifier of gain indices vector */ +opus_int32 silk_gains_ID( /* O returns unique identifier of gains */ + const opus_int8 ind[ MAX_NB_SUBFR ], /* I gain indices */ + const opus_int nb_subfr /* I number of subframes */ +); + +/* Interpolate two vectors */ +void silk_interpolate( + opus_int16 xi[ MAX_LPC_ORDER ], /* O interpolated vector */ + const opus_int16 x0[ MAX_LPC_ORDER ], /* I first vector */ + const opus_int16 x1[ MAX_LPC_ORDER ], /* I second vector */ + const opus_int ifact_Q2, /* I interp. factor, weight on 2nd vector */ + const opus_int d /* I number of parameters */ +); + +/* LTP tap quantizer */ +void silk_quant_LTP_gains( + opus_int16 B_Q14[ MAX_NB_SUBFR * LTP_ORDER ], /* O Quantized LTP gains */ + opus_int8 cbk_index[ MAX_NB_SUBFR ], /* O Codebook Index */ + opus_int8 *periodicity_index, /* O Periodicity Index */ + opus_int32 *sum_gain_dB_Q7, /* I/O Cumulative max prediction gain */ + opus_int *pred_gain_dB_Q7, /* O LTP prediction gain */ + const opus_int32 XX_Q17[ MAX_NB_SUBFR*LTP_ORDER*LTP_ORDER ], /* I Correlation matrix in Q18 */ + const opus_int32 xX_Q17[ MAX_NB_SUBFR*LTP_ORDER ], /* I Correlation vector in Q18 */ + const opus_int subfr_len, /* I Number of samples per subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +); + +/* Entropy constrained matrix-weighted VQ, for a single input data vector */ +void silk_VQ_WMat_EC_c( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *res_nrg_Q15, /* O best residual energy */ + opus_int32 *rate_dist_Q8, /* O best total bitrate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int32 *XX_Q17, /* I correlation matrix */ + const opus_int32 *xX_Q17, /* I correlation vector */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int subfr_len, /* I number of samples per subframe */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + const opus_int L /* I number of vectors in codebook */ +); + +#if !defined(OVERRIDE_silk_VQ_WMat_EC) +#define silk_VQ_WMat_EC(ind, res_nrg_Q15, rate_dist_Q8, gain_Q7, XX_Q17, xX_Q17, cb_Q7, cb_gain_Q7, cl_Q5, subfr_len, max_gain_Q7, L, arch) \ + ((void)(arch),silk_VQ_WMat_EC_c(ind, res_nrg_Q15, rate_dist_Q8, gain_Q7, XX_Q17, xX_Q17, cb_Q7, cb_gain_Q7, cl_Q5, subfr_len, max_gain_Q7, L)) +#endif + +/************************************/ +/* Noise shaping quantization (NSQ) */ +/************************************/ + +void silk_NSQ_c( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +#if !defined(OVERRIDE_silk_NSQ) +#define silk_NSQ(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_c(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) +#endif + +/* Noise shaping using delayed decision */ +void silk_NSQ_del_dec_c( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int16 x16[], /* I Input */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +#if !defined(OVERRIDE_silk_NSQ_del_dec) +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_del_dec_c(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) +#endif + +/************/ +/* Silk VAD */ +/************/ +/* Initialize the Silk VAD */ +opus_int silk_VAD_Init( /* O Return value, 0 if success */ + silk_VAD_state *psSilk_VAD /* I/O Pointer to Silk VAD state */ +); + +/* Get speech activity level in Q8 */ +opus_int silk_VAD_GetSA_Q8_c( /* O Return value, 0 if success */ + silk_encoder_state *psEncC, /* I/O Encoder state */ + const opus_int16 pIn[] /* I PCM input */ +); + +#if !defined(OVERRIDE_silk_VAD_GetSA_Q8) +#define silk_VAD_GetSA_Q8(psEnC, pIn, arch) ((void)(arch),silk_VAD_GetSA_Q8_c(psEnC, pIn)) +#endif + +/* Low-pass filter with variable cutoff frequency based on */ +/* piece-wise linear interpolation between elliptic filters */ +/* Start by setting transition_frame_no = 1; */ +void silk_LP_variable_cutoff( + silk_LP_state *psLP, /* I/O LP filter state */ + opus_int16 *frame, /* I/O Low-pass filtered output signal */ + const opus_int frame_length /* I Frame length */ +); + +/******************/ +/* NLSF Quantizer */ +/******************/ +/* Limit, stabilize, convert and quantize NLSFs */ +void silk_process_NLSFs( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ], /* O Prediction coefficients */ + opus_int16 pNLSF_Q15[ MAX_LPC_ORDER ], /* I/O Normalized LSFs (quant out) (0 - (2^15-1)) */ + const opus_int16 prev_NLSFq_Q15[ MAX_LPC_ORDER ] /* I Previous Normalized LSFs (0 - (2^15-1)) */ +); + +opus_int32 silk_NLSF_encode( /* O Returns RD value in Q25 */ + opus_int8 *NLSFIndices, /* I Codebook path vector [ LPC_ORDER + 1 ] */ + opus_int16 *pNLSF_Q15, /* I/O Quantized NLSF vector [ LPC_ORDER ] */ + const silk_NLSF_CB_struct *psNLSF_CB, /* I Codebook object */ + const opus_int16 *pW_QW, /* I NLSF weight vector [ LPC_ORDER ] */ + const opus_int NLSF_mu_Q20, /* I Rate weight for the RD optimization */ + const opus_int nSurvivors, /* I Max survivors after first stage */ + const opus_int signalType /* I Signal type: 0/1/2 */ +); + +/* Compute quantization errors for an LPC_order element input vector for a VQ codebook */ +void silk_NLSF_VQ( + opus_int32 err_Q26[], /* O Quantization errors [K] */ + const opus_int16 in_Q15[], /* I Input vectors to be quantized [LPC_order] */ + const opus_uint8 pCB_Q8[], /* I Codebook vectors [K*LPC_order] */ + const opus_int16 pWght_Q9[], /* I Codebook weights [K*LPC_order] */ + const opus_int K, /* I Number of codebook vectors */ + const opus_int LPC_order /* I Number of LPCs */ +); + +/* Delayed-decision quantizer for NLSF residuals */ +opus_int32 silk_NLSF_del_dec_quant( /* O Returns RD value in Q25 */ + opus_int8 indices[], /* O Quantization indices [ order ] */ + const opus_int16 x_Q10[], /* I Input [ order ] */ + const opus_int16 w_Q5[], /* I Weights [ order ] */ + const opus_uint8 pred_coef_Q8[], /* I Backward predictor coefs [ order ] */ + const opus_int16 ec_ix[], /* I Indices to entropy coding tables [ order ] */ + const opus_uint8 ec_rates_Q5[], /* I Rates [] */ + const opus_int quant_step_size_Q16, /* I Quantization step size */ + const opus_int16 inv_quant_step_size_Q6, /* I Inverse quantization step size */ + const opus_int32 mu_Q20, /* I R/D tradeoff */ + const opus_int16 order /* I Number of input values */ +); + +/* Unpack predictor values and indices for entropy coding tables */ +void silk_NLSF_unpack( + opus_int16 ec_ix[], /* O Indices to entropy tables [ LPC_ORDER ] */ + opus_uint8 pred_Q8[], /* O LSF predictor [ LPC_ORDER ] */ + const silk_NLSF_CB_struct *psNLSF_CB, /* I Codebook object */ + const opus_int CB1_index /* I Index of vector in first LSF codebook */ +); + +/***********************/ +/* NLSF vector decoder */ +/***********************/ +void silk_NLSF_decode( + opus_int16 *pNLSF_Q15, /* O Quantized NLSF vector [ LPC_ORDER ] */ + opus_int8 *NLSFIndices, /* I Codebook path vector [ LPC_ORDER + 1 ] */ + const silk_NLSF_CB_struct *psNLSF_CB /* I Codebook object */ +); + +/****************************************************/ +/* Decoder Functions */ +/****************************************************/ +opus_int silk_init_decoder( + silk_decoder_state *psDec /* I/O Decoder state pointer */ +); + +/* Set decoder sampling rate */ +opus_int silk_decoder_set_fs( + silk_decoder_state *psDec, /* I/O Decoder state pointer */ + opus_int fs_kHz, /* I Sampling frequency (kHz) */ + opus_int32 fs_API_Hz /* I API Sampling frequency (Hz) */ +); + +/****************/ +/* Decode frame */ +/****************/ +opus_int silk_decode_frame( + silk_decoder_state *psDec, /* I/O Pointer to Silk decoder state */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pOut[], /* O Pointer to output speech frame */ + opus_int32 *pN, /* O Pointer to size of output frame */ + opus_int lostFlag, /* I 0: no loss, 1 loss, 2 decode fec */ + opus_int condCoding, /* I The type of conditional coding to use */ + int arch /* I Run-time architecture */ +); + +/* Decode indices from bitstream */ +void silk_decode_indices( + silk_decoder_state *psDec, /* I/O State */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int FrameIndex, /* I Frame number */ + opus_int decode_LBRR, /* I Flag indicating LBRR data is being decoded */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/* Decode parameters from payload */ +void silk_decode_parameters( + silk_decoder_state *psDec, /* I/O State */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +/* Core decoder. Performs inverse NSQ operation LTP + LPC */ +void silk_decode_core( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I Decoder control */ + opus_int16 xq[], /* O Decoded speech */ + const opus_int16 pulses[ MAX_FRAME_LENGTH ], /* I Pulse signal */ + int arch /* I Run-time architecture */ +); + +/* Decode quantization indices of excitation (Shell coding) */ +void silk_decode_pulses( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int16 pulses[], /* O Excitation signal */ + const opus_int signalType, /* I Sigtype */ + const opus_int quantOffsetType, /* I quantOffsetType */ + const opus_int frame_length /* I Frame length */ +); + +/******************/ +/* CNG */ +/******************/ + +/* Reset CNG */ +void silk_CNG_Reset( + silk_decoder_state *psDec /* I/O Decoder state */ +); + +/* Updates CNG estimate, and applies the CNG when packet was lost */ +void silk_CNG( + silk_decoder_state *psDec, /* I/O Decoder state */ + silk_decoder_control *psDecCtrl, /* I/O Decoder control */ + opus_int16 frame[], /* I/O Signal */ + opus_int length /* I Length of residual */ +); + +/* Encoding of various parameters */ +void silk_encode_indices( + silk_encoder_state *psEncC, /* I/O Encoder state */ + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int FrameIndex, /* I Frame number */ + opus_int encode_LBRR, /* I Flag indicating LBRR data is being encoded */ + opus_int condCoding /* I The type of conditional coding to use */ +); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/meson.build new file mode 100644 index 000000000..706923727 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/meson.build @@ -0,0 +1,53 @@ +silk_sources = sources['SILK_SOURCES'] + +silk_sources_sse4_1 = sources['SILK_SOURCES_SSE4_1'] + +silk_sources_neon_intr = sources['SILK_SOURCES_ARM_NEON_INTR'] + +silk_sources_fixed_neon_intr = sources['SILK_SOURCES_FIXED_ARM_NEON_INTR'] + +silk_sources_fixed = sources['SILK_SOURCES_FIXED'] + +silk_sources_fixed_sse4_1 = sources['SILK_SOURCES_FIXED_SSE4_1'] + +silk_sources_float = sources['SILK_SOURCES_FLOAT'] + +if opt_fixed_point + silk_sources += silk_sources_fixed +else + silk_sources += silk_sources_float +endif + +silk_includes = [opus_includes, include_directories('float', 'fixed')] +silk_static_libs = [] + +foreach intr_name : ['sse4_1', 'neon_intr'] + have_intr = get_variable('have_' + intr_name) + if not have_intr + continue + endif + + intr_sources = get_variable('silk_sources_' + intr_name) + if opt_fixed_point + intr_sources += get_variable('silk_sources_fixed_' + intr_name) + endif + + intr_args = get_variable('opus_@0@_args'.format(intr_name), []) + silk_static_libs += static_library('silk_' + intr_name, intr_sources, + c_args: intr_args, + include_directories: silk_includes, + install: false) +endforeach + +silk_c_args = [] +if host_machine.system() == 'windows' + silk_c_args += ['-DDLL_EXPORT'] +endif + +silk_lib = static_library('opus-silk', + silk_sources, + c_args: silk_c_args, + include_directories: silk_includes, + link_whole: silk_static_libs, + dependencies: libm, + install: false) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/NSQ_del_dec_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/NSQ_del_dec_mipsr1.h new file mode 100644 index 000000000..cd70713a8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/NSQ_del_dec_mipsr1.h @@ -0,0 +1,410 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef __NSQ_DEL_DEC_MIPSR1_H__ +#define __NSQ_DEL_DEC_MIPSR1_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +#define OVERRIDE_silk_noise_shape_quantizer_del_dec +static inline void silk_noise_shape_quantizer_del_dec( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay, /* I */ + int arch /* I */ +) +{ + opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx; + opus_int32 Winner_rand_state; + opus_int32 LTP_pred_Q14, LPC_pred_Q14, n_AR_Q14, n_LTP_Q14; + opus_int32 n_LF_Q14, r_Q10, rr_Q10, rd1_Q10, rd2_Q10, RDmin_Q10, RDmax_Q10; + opus_int32 q1_Q0, q1_Q10, q2_Q10, exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10; + opus_int32 tmp1, tmp2, sLF_AR_shp_Q14; + opus_int32 *pred_lag_ptr, *shp_lag_ptr, *psLPC_Q14; + NSQ_sample_struct psSampleState[ MAX_DEL_DEC_STATES ][ 2 ]; + NSQ_del_dec_struct *psDD; + NSQ_sample_struct *psSS; + opus_int16 b_Q14_0, b_Q14_1, b_Q14_2, b_Q14_3, b_Q14_4; + opus_int16 a_Q12_0, a_Q12_1, a_Q12_2, a_Q12_3, a_Q12_4, a_Q12_5, a_Q12_6; + opus_int16 a_Q12_7, a_Q12_8, a_Q12_9, a_Q12_10, a_Q12_11, a_Q12_12, a_Q12_13; + opus_int16 a_Q12_14, a_Q12_15; + + opus_int32 cur, prev, next; + + /*Unused.*/ + (void)arch; + + //Intialize b_Q14 variables + b_Q14_0 = b_Q14[ 0 ]; + b_Q14_1 = b_Q14[ 1 ]; + b_Q14_2 = b_Q14[ 2 ]; + b_Q14_3 = b_Q14[ 3 ]; + b_Q14_4 = b_Q14[ 4 ]; + + //Intialize a_Q12 variables + a_Q12_0 = a_Q12[0]; + a_Q12_1 = a_Q12[1]; + a_Q12_2 = a_Q12[2]; + a_Q12_3 = a_Q12[3]; + a_Q12_4 = a_Q12[4]; + a_Q12_5 = a_Q12[5]; + a_Q12_6 = a_Q12[6]; + a_Q12_7 = a_Q12[7]; + a_Q12_8 = a_Q12[8]; + a_Q12_9 = a_Q12[9]; + a_Q12_10 = a_Q12[10]; + a_Q12_11 = a_Q12[11]; + a_Q12_12 = a_Q12[12]; + a_Q12_13 = a_Q12[13]; + a_Q12_14 = a_Q12[14]; + a_Q12_15 = a_Q12[15]; + + long long temp64; + + silk_assert( nStatesDelayedDecision > 0 ); + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + + for( i = 0; i < length; i++ ) { + /* Perform common calculations used in all states */ + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + temp64 = __builtin_mips_mult(pred_lag_ptr[ 0 ], b_Q14_0 ); + temp64 = __builtin_mips_madd( temp64, pred_lag_ptr[ -1 ], b_Q14_1 ); + temp64 = __builtin_mips_madd( temp64, pred_lag_ptr[ -2 ], b_Q14_2 ); + temp64 = __builtin_mips_madd( temp64, pred_lag_ptr[ -3 ], b_Q14_3 ); + temp64 = __builtin_mips_madd( temp64, pred_lag_ptr[ -4 ], b_Q14_4 ); + temp64 += 32768; + LTP_pred_Q14 = __builtin_mips_extr_w(temp64, 16); + LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */ + pred_lag_ptr++; + } else { + LTP_pred_Q14 = 0; + } + + /* Long-term shaping */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */ + shp_lag_ptr++; + } else { + n_LTP_Q14 = 0; + } + + for( k = 0; k < nStatesDelayedDecision; k++ ) { + /* Delayed decision state */ + psDD = &psDelDec[ k ]; + + /* Sample state */ + psSS = psSampleState[ k ]; + + /* Generate dither */ + psDD->Seed = silk_RAND( psDD->Seed ); + + /* Pointer used in short term prediction and shaping */ + psLPC_Q14 = &psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 + i ]; + /* Short-term prediction */ + silk_assert( predictLPCOrder == 10 || predictLPCOrder == 16 ); + temp64 = __builtin_mips_mult(psLPC_Q14[ 0 ], a_Q12_0 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -1 ], a_Q12_1 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -2 ], a_Q12_2 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -3 ], a_Q12_3 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -4 ], a_Q12_4 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -5 ], a_Q12_5 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -6 ], a_Q12_6 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -7 ], a_Q12_7 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -8 ], a_Q12_8 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -9 ], a_Q12_9 ); + if( predictLPCOrder == 16 ) { + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -10 ], a_Q12_10 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -11 ], a_Q12_11 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -12 ], a_Q12_12 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -13 ], a_Q12_13 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -14 ], a_Q12_14 ); + temp64 = __builtin_mips_madd( temp64, psLPC_Q14[ -15 ], a_Q12_15 ); + } + temp64 += 32768; + LPC_pred_Q14 = __builtin_mips_extr_w(temp64, 16); + + LPC_pred_Q14 = silk_LSHIFT( LPC_pred_Q14, 4 ); /* Q10 -> Q14 */ + + /* Noise shape feedback */ + silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ + /* Output of lowpass section */ + tmp2 = silk_SMLAWB( psLPC_Q14[ 0 ], psDD->sAR2_Q14[ 0 ], warping_Q16 ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ 0 ], psDD->sAR2_Q14[ 1 ] - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ 0 ] = tmp2; + + temp64 = __builtin_mips_mult(tmp2, AR_shp_Q13[ 0 ] ); + + prev = psDD->sAR2_Q14[ 1 ]; + + /* Loop over allpass sections */ + for( j = 2; j < shapingLPCOrder; j += 2 ) { + cur = psDD->sAR2_Q14[ j ]; + next = psDD->sAR2_Q14[ j+1 ]; + /* Output of allpass section */ + tmp2 = silk_SMLAWB( prev, cur - tmp1, warping_Q16 ); + psDD->sAR2_Q14[ j - 1 ] = tmp1; + temp64 = __builtin_mips_madd( temp64, tmp1, AR_shp_Q13[ j - 1 ] ); + temp64 = __builtin_mips_madd( temp64, tmp2, AR_shp_Q13[ j ] ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( cur, next - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ j + 0 ] = tmp2; + prev = next; + } + psDD->sAR2_Q14[ shapingLPCOrder - 1 ] = tmp1; + temp64 = __builtin_mips_madd( temp64, tmp1, AR_shp_Q13[ shapingLPCOrder - 1 ] ); + temp64 += 32768; + n_AR_Q14 = __builtin_mips_extr_w(temp64, 16); + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 1 ); /* Q11 -> Q12 */ + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, psDD->LF_AR_Q14, Tilt_Q14 ); /* Q12 */ + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 2 ); /* Q12 -> Q14 */ + + n_LF_Q14 = silk_SMULWB( psDD->Shape_Q14[ *smpl_buf_idx ], LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_SMLAWT( n_LF_Q14, psDD->LF_AR_Q14, LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_LSHIFT( n_LF_Q14, 2 ); /* Q12 -> Q14 */ + + /* Input minus prediction plus noise feedback */ + /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */ + tmp1 = silk_ADD32( n_AR_Q14, n_LF_Q14 ); /* Q14 */ + tmp2 = silk_ADD32( n_LTP_Q14, LPC_pred_Q14 ); /* Q13 */ + tmp1 = silk_SUB32( tmp2, tmp1 ); /* Q13 */ + tmp1 = silk_RSHIFT_ROUND( tmp1, 4 ); /* Q10 */ + + r_Q10 = silk_SUB32( x_Q10[ i ], tmp1 ); /* residual error Q10 */ + + /* Flip sign depending on dither */ + if ( psDD->Seed < 0 ) { + r_Q10 = -r_Q10; + } + r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + q1_Q10 = silk_SUB32( r_Q10, offset_Q10 ); + q1_Q0 = silk_RSHIFT( q1_Q10, 10 ); + if( q1_Q0 > 0 ) { + q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == 0 ) { + q1_Q10 = offset_Q10; + q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == -1 ) { + q2_Q10 = offset_Q10; + q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else { /* q1_Q0 < -1 */ + q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( -q2_Q10, Lambda_Q10 ); + } + rr_Q10 = silk_SUB32( r_Q10, q1_Q10 ); + rd1_Q10 = silk_RSHIFT( silk_SMLABB( rd1_Q10, rr_Q10, rr_Q10 ), 10 ); + rr_Q10 = silk_SUB32( r_Q10, q2_Q10 ); + rd2_Q10 = silk_RSHIFT( silk_SMLABB( rd2_Q10, rr_Q10, rr_Q10 ), 10 ); + + if( rd1_Q10 < rd2_Q10 ) { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 0 ].Q_Q10 = q1_Q10; + psSS[ 1 ].Q_Q10 = q2_Q10; + } else { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 0 ].Q_Q10 = q2_Q10; + psSS[ 1 ].Q_Q10 = q1_Q10; + } + + /* Update states for best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 0 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 ); + psSS[ 0 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 0 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 0 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 0 ].xq_Q14 = xq_Q14; + + /* Update states for second best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 1 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 ); + psSS[ 1 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 1 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 1 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 1 ].xq_Q14 = xq_Q14; + } + + *smpl_buf_idx = ( *smpl_buf_idx - 1 ) % DECISION_DELAY; + if( *smpl_buf_idx < 0 ) *smpl_buf_idx += DECISION_DELAY; + last_smple_idx = ( *smpl_buf_idx + decisionDelay ) % DECISION_DELAY; + + /* Find winner */ + RDmin_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + Winner_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + if( psSampleState[ k ][ 0 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + Winner_ind = k; + } + } + + /* Increase RD values of expired states */ + Winner_rand_state = psDelDec[ Winner_ind ].RandState[ last_smple_idx ]; + for( k = 0; k < nStatesDelayedDecision; k++ ) { + if( psDelDec[ k ].RandState[ last_smple_idx ] != Winner_rand_state ) { + psSampleState[ k ][ 0 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 0 ].RD_Q10, silk_int32_MAX >> 4 ); + psSampleState[ k ][ 1 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 1 ].RD_Q10, silk_int32_MAX >> 4 ); + silk_assert( psSampleState[ k ][ 0 ].RD_Q10 >= 0 ); + } + } + + /* Find worst in first set and best in second set */ + RDmax_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + RDmin_Q10 = psSampleState[ 0 ][ 1 ].RD_Q10; + RDmax_ind = 0; + RDmin_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + /* find worst in first set */ + if( psSampleState[ k ][ 0 ].RD_Q10 > RDmax_Q10 ) { + RDmax_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + RDmax_ind = k; + } + /* find best in second set */ + if( psSampleState[ k ][ 1 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 1 ].RD_Q10; + RDmin_ind = k; + } + } + + /* Replace a state if best from second set outperforms worst in first set */ + if( RDmin_Q10 < RDmax_Q10 ) { + silk_memcpy( ( (opus_int32 *)&psDelDec[ RDmax_ind ] ) + i, + ( (opus_int32 *)&psDelDec[ RDmin_ind ] ) + i, sizeof( NSQ_del_dec_struct ) - i * sizeof( opus_int32) ); + silk_memcpy( &psSampleState[ RDmax_ind ][ 0 ], &psSampleState[ RDmin_ind ][ 1 ], sizeof( NSQ_sample_struct ) ); + } + + /* Write samples from winner to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + if( subfr > 0 || i >= decisionDelay ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], delayedGain_Q10[ last_smple_idx ] ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDD->Shape_Q14[ last_smple_idx ]; + sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDD->Pred_Q15[ last_smple_idx ]; + } + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Update states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + psSS = &psSampleState[ k ][ 0 ]; + psDD->LF_AR_Q14 = psSS->LF_AR_Q14; + psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ] = psSS->xq_Q14; + psDD->Xq_Q14[ *smpl_buf_idx ] = psSS->xq_Q14; + psDD->Q_Q10[ *smpl_buf_idx ] = psSS->Q_Q10; + psDD->Pred_Q15[ *smpl_buf_idx ] = silk_LSHIFT32( psSS->LPC_exc_Q14, 1 ); + psDD->Shape_Q14[ *smpl_buf_idx ] = psSS->sLTP_shp_Q14; + psDD->Seed = silk_ADD32_ovflw( psDD->Seed, silk_RSHIFT_ROUND( psSS->Q_Q10, 10 ) ); + psDD->RandState[ *smpl_buf_idx ] = psDD->Seed; + psDD->RD_Q10 = psSS->RD_Q10; + } + delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10; + } + /* Update LPC states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + silk_memcpy( psDD->sLPC_Q14, &psDD->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + } +} + +#endif /* __NSQ_DEL_DEC_MIPSR1_H__ */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/macros_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/macros_mipsr1.h new file mode 100644 index 000000000..12ed981a6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/macros_mipsr1.h @@ -0,0 +1,92 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + + +#ifndef __SILK_MACROS_MIPSR1_H__ +#define __SILK_MACROS_MIPSR1_H__ + +#define mips_clz(x) __builtin_clz(x) + +#undef silk_SMULWB +static inline int silk_SMULWB(int a, int b) +{ + long long ac; + int c; + + ac = __builtin_mips_mult(a, (opus_int32)(opus_int16)b); + c = __builtin_mips_extr_w(ac, 16); + + return c; +} + +#undef silk_SMLAWB +#define silk_SMLAWB(a32, b32, c32) ((a32) + silk_SMULWB(b32, c32)) + +#undef silk_SMULWW +static inline int silk_SMULWW(int a, int b) +{ + long long ac; + int c; + + ac = __builtin_mips_mult(a, b); + c = __builtin_mips_extr_w(ac, 16); + + return c; +} + +#undef silk_SMLAWW +static inline int silk_SMLAWW(int a, int b, int c) +{ + long long ac; + int res; + + ac = __builtin_mips_mult(b, c); + res = __builtin_mips_extr_w(ac, 16); + res += a; + + return res; +} + +#define OVERRIDE_silk_CLZ16 +static inline opus_int32 silk_CLZ16(opus_int16 in16) +{ + int re32; + opus_int32 in32 = (opus_int32 )in16; + re32 = mips_clz(in32); + re32-=16; + return re32; +} + +#define OVERRIDE_silk_CLZ32 +static inline opus_int32 silk_CLZ32(opus_int32 in32) +{ + int re32; + re32 = mips_clz(in32); + return re32; +} + +#endif /* __SILK_MACROS_MIPSR1_H__ */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/sigproc_fix_mipsr1.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/sigproc_fix_mipsr1.h new file mode 100644 index 000000000..51520c0a6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/mips/sigproc_fix_mipsr1.h @@ -0,0 +1,60 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_SIGPROC_FIX_MIPSR1_H +#define SILK_SIGPROC_FIX_MIPSR1_H + +#undef silk_SAT16 +static inline short int silk_SAT16(int a) +{ + int c; + c = __builtin_mips_shll_s_w(a, 16); + c = c>>16; + + return c; +} + +#undef silk_LSHIFT_SAT32 +static inline int silk_LSHIFT_SAT32(int a, int shift) +{ + int r; + + r = __builtin_mips_shll_s_w(a, shift); + + return r; +} + +#undef silk_RSHIFT_ROUND +static inline int silk_RSHIFT_ROUND(int a, int shift) +{ + int r; + + r = __builtin_mips_shra_r_w(a, shift); + return r; +} + +#endif /* SILK_SIGPROC_FIX_MIPSR1_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_defines.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_defines.h new file mode 100644 index 000000000..e1e4b5d76 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_defines.h @@ -0,0 +1,88 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_PE_DEFINES_H +#define SILK_PE_DEFINES_H + +#include "SigProc_FIX.h" + +/********************************************************/ +/* Definitions for pitch estimator */ +/********************************************************/ + +#define PE_MAX_FS_KHZ 16 /* Maximum sampling frequency used */ + +#define PE_MAX_NB_SUBFR 4 +#define PE_SUBFR_LENGTH_MS 5 /* 5 ms */ + +#define PE_LTP_MEM_LENGTH_MS ( 4 * PE_SUBFR_LENGTH_MS ) + +#define PE_MAX_FRAME_LENGTH_MS ( PE_LTP_MEM_LENGTH_MS + PE_MAX_NB_SUBFR * PE_SUBFR_LENGTH_MS ) +#define PE_MAX_FRAME_LENGTH ( PE_MAX_FRAME_LENGTH_MS * PE_MAX_FS_KHZ ) +#define PE_MAX_FRAME_LENGTH_ST_1 ( PE_MAX_FRAME_LENGTH >> 2 ) +#define PE_MAX_FRAME_LENGTH_ST_2 ( PE_MAX_FRAME_LENGTH >> 1 ) + +#define PE_MAX_LAG_MS 18 /* 18 ms -> 56 Hz */ +#define PE_MIN_LAG_MS 2 /* 2 ms -> 500 Hz */ +#define PE_MAX_LAG ( PE_MAX_LAG_MS * PE_MAX_FS_KHZ ) +#define PE_MIN_LAG ( PE_MIN_LAG_MS * PE_MAX_FS_KHZ ) + +#define PE_D_SRCH_LENGTH 24 + +#define PE_NB_STAGE3_LAGS 5 + +#define PE_NB_CBKS_STAGE2 3 +#define PE_NB_CBKS_STAGE2_EXT 11 + +#define PE_NB_CBKS_STAGE3_MAX 34 +#define PE_NB_CBKS_STAGE3_MID 24 +#define PE_NB_CBKS_STAGE3_MIN 16 + +#define PE_NB_CBKS_STAGE3_10MS 12 +#define PE_NB_CBKS_STAGE2_10MS 3 + +#define PE_SHORTLAG_BIAS 0.2f /* for logarithmic weighting */ +#define PE_PREVLAG_BIAS 0.2f /* for logarithmic weighting */ +#define PE_FLATCONTOUR_BIAS 0.05f + +#define SILK_PE_MIN_COMPLEX 0 +#define SILK_PE_MID_COMPLEX 1 +#define SILK_PE_MAX_COMPLEX 2 + +/* Tables for 20 ms frames */ +extern const opus_int8 silk_CB_lags_stage2[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE2_EXT ]; +extern const opus_int8 silk_CB_lags_stage3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ]; +extern const opus_int8 silk_Lag_range_stage3[ SILK_PE_MAX_COMPLEX + 1 ] [ PE_MAX_NB_SUBFR ][ 2 ]; +extern const opus_int8 silk_nb_cbk_searchs_stage3[ SILK_PE_MAX_COMPLEX + 1 ]; + +/* Tables for 10 ms frames */ +extern const opus_int8 silk_CB_lags_stage2_10_ms[ PE_MAX_NB_SUBFR >> 1][ 3 ]; +extern const opus_int8 silk_CB_lags_stage3_10_ms[ PE_MAX_NB_SUBFR >> 1 ][ 12 ]; +extern const opus_int8 silk_Lag_range_stage3_10_ms[ PE_MAX_NB_SUBFR >> 1 ][ 2 ]; + +#endif + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_tables.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_tables.c new file mode 100644 index 000000000..81a8bacac --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/pitch_est_tables.c @@ -0,0 +1,99 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "typedef.h" +#include "pitch_est_defines.h" + +const opus_int8 silk_CB_lags_stage2_10_ms[ PE_MAX_NB_SUBFR >> 1][ PE_NB_CBKS_STAGE2_10MS ] = +{ + {0, 1, 0}, + {0, 0, 1} +}; + +const opus_int8 silk_CB_lags_stage3_10_ms[ PE_MAX_NB_SUBFR >> 1 ][ PE_NB_CBKS_STAGE3_10MS ] = +{ + { 0, 0, 1,-1, 1,-1, 2,-2, 2,-2, 3,-3}, + { 0, 1, 0, 1,-1, 2,-1, 2,-2, 3,-2, 3} +}; + +const opus_int8 silk_Lag_range_stage3_10_ms[ PE_MAX_NB_SUBFR >> 1 ][ 2 ] = +{ + {-3, 7}, + {-2, 7} +}; + +const opus_int8 silk_CB_lags_stage2[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE2_EXT ] = +{ + {0, 2,-1,-1,-1, 0, 0, 1, 1, 0, 1}, + {0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0}, + {0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0}, + {0,-1, 2, 1, 0, 1, 1, 0, 0,-1,-1} +}; + +const opus_int8 silk_CB_lags_stage3[ PE_MAX_NB_SUBFR ][ PE_NB_CBKS_STAGE3_MAX ] = +{ + {0, 0, 1,-1, 0, 1,-1, 0,-1, 1,-2, 2,-2,-2, 2,-3, 2, 3,-3,-4, 3,-4, 4, 4,-5, 5,-6,-5, 6,-7, 6, 5, 8,-9}, + {0, 0, 1, 0, 0, 0, 0, 0, 0, 0,-1, 1, 0, 0, 1,-1, 0, 1,-1,-1, 1,-1, 2, 1,-1, 2,-2,-2, 2,-2, 2, 2, 3,-3}, + {0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1,-1, 1, 0, 0, 2, 1,-1, 2,-1,-1, 2,-1, 2, 2,-1, 3,-2,-2,-2, 3}, + {0, 1, 0, 0, 1, 0, 1,-1, 2,-1, 2,-1, 2, 3,-2, 3,-2,-2, 4, 4,-3, 5,-3,-4, 6,-4, 6, 5,-5, 8,-6,-5,-7, 9} +}; + +const opus_int8 silk_Lag_range_stage3[ SILK_PE_MAX_COMPLEX + 1 ] [ PE_MAX_NB_SUBFR ][ 2 ] = +{ + /* Lags to search for low number of stage3 cbks */ + { + {-5,8}, + {-1,6}, + {-1,6}, + {-4,10} + }, + /* Lags to search for middle number of stage3 cbks */ + { + {-6,10}, + {-2,6}, + {-1,6}, + {-5,10} + }, + /* Lags to search for max number of stage3 cbks */ + { + {-9,12}, + {-3,7}, + {-2,7}, + {-7,13} + } +}; + +const opus_int8 silk_nb_cbk_searchs_stage3[ SILK_PE_MAX_COMPLEX + 1 ] = +{ + PE_NB_CBKS_STAGE3_MIN, + PE_NB_CBKS_STAGE3_MID, + PE_NB_CBKS_STAGE3_MAX +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/process_NLSFs.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/process_NLSFs.c new file mode 100644 index 000000000..d13080954 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/process_NLSFs.c @@ -0,0 +1,107 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Limit, stabilize, convert and quantize NLSFs */ +void silk_process_NLSFs( + silk_encoder_state *psEncC, /* I/O Encoder state */ + opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ], /* O Prediction coefficients */ + opus_int16 pNLSF_Q15[ MAX_LPC_ORDER ], /* I/O Normalized LSFs (quant out) (0 - (2^15-1)) */ + const opus_int16 prev_NLSFq_Q15[ MAX_LPC_ORDER ] /* I Previous Normalized LSFs (0 - (2^15-1)) */ +) +{ + opus_int i, doInterpolate; + opus_int NLSF_mu_Q20; + opus_int16 i_sqr_Q15; + opus_int16 pNLSF0_temp_Q15[ MAX_LPC_ORDER ]; + opus_int16 pNLSFW_QW[ MAX_LPC_ORDER ]; + opus_int16 pNLSFW0_temp_QW[ MAX_LPC_ORDER ]; + + silk_assert( psEncC->speech_activity_Q8 >= 0 ); + silk_assert( psEncC->speech_activity_Q8 <= SILK_FIX_CONST( 1.0, 8 ) ); + celt_assert( psEncC->useInterpolatedNLSFs == 1 || psEncC->indices.NLSFInterpCoef_Q2 == ( 1 << 2 ) ); + + /***********************/ + /* Calculate mu values */ + /***********************/ + /* NLSF_mu = 0.003 - 0.0015 * psEnc->speech_activity; */ + NLSF_mu_Q20 = silk_SMLAWB( SILK_FIX_CONST( 0.003, 20 ), SILK_FIX_CONST( -0.001, 28 ), psEncC->speech_activity_Q8 ); + if( psEncC->nb_subfr == 2 ) { + /* Multiply by 1.5 for 10 ms packets */ + NLSF_mu_Q20 = silk_ADD_RSHIFT( NLSF_mu_Q20, NLSF_mu_Q20, 1 ); + } + + celt_assert( NLSF_mu_Q20 > 0 ); + silk_assert( NLSF_mu_Q20 <= SILK_FIX_CONST( 0.005, 20 ) ); + + /* Calculate NLSF weights */ + silk_NLSF_VQ_weights_laroia( pNLSFW_QW, pNLSF_Q15, psEncC->predictLPCOrder ); + + /* Update NLSF weights for interpolated NLSFs */ + doInterpolate = ( psEncC->useInterpolatedNLSFs == 1 ) && ( psEncC->indices.NLSFInterpCoef_Q2 < 4 ); + if( doInterpolate ) { + /* Calculate the interpolated NLSF vector for the first half */ + silk_interpolate( pNLSF0_temp_Q15, prev_NLSFq_Q15, pNLSF_Q15, + psEncC->indices.NLSFInterpCoef_Q2, psEncC->predictLPCOrder ); + + /* Calculate first half NLSF weights for the interpolated NLSFs */ + silk_NLSF_VQ_weights_laroia( pNLSFW0_temp_QW, pNLSF0_temp_Q15, psEncC->predictLPCOrder ); + + /* Update NLSF weights with contribution from first half */ + i_sqr_Q15 = silk_LSHIFT( silk_SMULBB( psEncC->indices.NLSFInterpCoef_Q2, psEncC->indices.NLSFInterpCoef_Q2 ), 11 ); + for( i = 0; i < psEncC->predictLPCOrder; i++ ) { + pNLSFW_QW[ i ] = silk_ADD16( silk_RSHIFT( pNLSFW_QW[ i ], 1 ), silk_RSHIFT( + silk_SMULBB( pNLSFW0_temp_QW[ i ], i_sqr_Q15 ), 16) ); + silk_assert( pNLSFW_QW[ i ] >= 1 ); + } + } + + silk_NLSF_encode( psEncC->indices.NLSFIndices, pNLSF_Q15, psEncC->psNLSF_CB, pNLSFW_QW, + NLSF_mu_Q20, psEncC->NLSF_MSVQ_Survivors, psEncC->indices.signalType ); + + /* Convert quantized NLSFs back to LPC coefficients */ + silk_NLSF2A( PredCoef_Q12[ 1 ], pNLSF_Q15, psEncC->predictLPCOrder, psEncC->arch ); + + if( doInterpolate ) { + /* Calculate the interpolated, quantized LSF vector for the first half */ + silk_interpolate( pNLSF0_temp_Q15, prev_NLSFq_Q15, pNLSF_Q15, + psEncC->indices.NLSFInterpCoef_Q2, psEncC->predictLPCOrder ); + + /* Convert back to LPC coefficients */ + silk_NLSF2A( PredCoef_Q12[ 0 ], pNLSF0_temp_Q15, psEncC->predictLPCOrder, psEncC->arch ); + + } else { + /* Copy LPC coefficients for first half from second half */ + celt_assert( psEncC->predictLPCOrder <= MAX_LPC_ORDER ); + silk_memcpy( PredCoef_Q12[ 0 ], PredCoef_Q12[ 1 ], psEncC->predictLPCOrder * sizeof( opus_int16 ) ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/quant_LTP_gains.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/quant_LTP_gains.c new file mode 100644 index 000000000..d6b8eff8d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/quant_LTP_gains.c @@ -0,0 +1,132 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "tuning_parameters.h" + +void silk_quant_LTP_gains( + opus_int16 B_Q14[ MAX_NB_SUBFR * LTP_ORDER ], /* O Quantized LTP gains */ + opus_int8 cbk_index[ MAX_NB_SUBFR ], /* O Codebook Index */ + opus_int8 *periodicity_index, /* O Periodicity Index */ + opus_int32 *sum_log_gain_Q7, /* I/O Cumulative max prediction gain */ + opus_int *pred_gain_dB_Q7, /* O LTP prediction gain */ + const opus_int32 XX_Q17[ MAX_NB_SUBFR*LTP_ORDER*LTP_ORDER ], /* I Correlation matrix in Q18 */ + const opus_int32 xX_Q17[ MAX_NB_SUBFR*LTP_ORDER ], /* I Correlation vector in Q18 */ + const opus_int subfr_len, /* I Number of samples per subframe */ + const opus_int nb_subfr, /* I Number of subframes */ + int arch /* I Run-time architecture */ +) +{ + opus_int j, k, cbk_size; + opus_int8 temp_idx[ MAX_NB_SUBFR ]; + const opus_uint8 *cl_ptr_Q5; + const opus_int8 *cbk_ptr_Q7; + const opus_uint8 *cbk_gain_ptr_Q7; + const opus_int32 *XX_Q17_ptr, *xX_Q17_ptr; + opus_int32 res_nrg_Q15_subfr, res_nrg_Q15, rate_dist_Q7_subfr, rate_dist_Q7, min_rate_dist_Q7; + opus_int32 sum_log_gain_tmp_Q7, best_sum_log_gain_Q7, max_gain_Q7; + opus_int gain_Q7; + + /***************************************************/ + /* iterate over different codebooks with different */ + /* rates/distortions, and choose best */ + /***************************************************/ + min_rate_dist_Q7 = silk_int32_MAX; + best_sum_log_gain_Q7 = 0; + for( k = 0; k < 3; k++ ) { + /* Safety margin for pitch gain control, to take into account factors + such as state rescaling/rewhitening. */ + opus_int32 gain_safety = SILK_FIX_CONST( 0.4, 7 ); + + cl_ptr_Q5 = silk_LTP_gain_BITS_Q5_ptrs[ k ]; + cbk_ptr_Q7 = silk_LTP_vq_ptrs_Q7[ k ]; + cbk_gain_ptr_Q7 = silk_LTP_vq_gain_ptrs_Q7[ k ]; + cbk_size = silk_LTP_vq_sizes[ k ]; + + /* Set up pointers to first subframe */ + XX_Q17_ptr = XX_Q17; + xX_Q17_ptr = xX_Q17; + + res_nrg_Q15 = 0; + rate_dist_Q7 = 0; + sum_log_gain_tmp_Q7 = *sum_log_gain_Q7; + for( j = 0; j < nb_subfr; j++ ) { + max_gain_Q7 = silk_log2lin( ( SILK_FIX_CONST( MAX_SUM_LOG_GAIN_DB / 6.0, 7 ) - sum_log_gain_tmp_Q7 ) + + SILK_FIX_CONST( 7, 7 ) ) - gain_safety; + silk_VQ_WMat_EC( + &temp_idx[ j ], /* O index of best codebook vector */ + &res_nrg_Q15_subfr, /* O residual energy */ + &rate_dist_Q7_subfr, /* O best weighted quantization error + mu * rate */ + &gain_Q7, /* O sum of absolute LTP coefficients */ + XX_Q17_ptr, /* I correlation matrix */ + xX_Q17_ptr, /* I correlation vector */ + cbk_ptr_Q7, /* I codebook */ + cbk_gain_ptr_Q7, /* I codebook effective gains */ + cl_ptr_Q5, /* I code length for each codebook vector */ + subfr_len, /* I number of samples per subframe */ + max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + cbk_size, /* I number of vectors in codebook */ + arch /* I Run-time architecture */ + ); + + res_nrg_Q15 = silk_ADD_POS_SAT32( res_nrg_Q15, res_nrg_Q15_subfr ); + rate_dist_Q7 = silk_ADD_POS_SAT32( rate_dist_Q7, rate_dist_Q7_subfr ); + sum_log_gain_tmp_Q7 = silk_max(0, sum_log_gain_tmp_Q7 + + silk_lin2log( gain_safety + gain_Q7 ) - SILK_FIX_CONST( 7, 7 )); + + XX_Q17_ptr += LTP_ORDER * LTP_ORDER; + xX_Q17_ptr += LTP_ORDER; + } + + if( rate_dist_Q7 <= min_rate_dist_Q7 ) { + min_rate_dist_Q7 = rate_dist_Q7; + *periodicity_index = (opus_int8)k; + silk_memcpy( cbk_index, temp_idx, nb_subfr * sizeof( opus_int8 ) ); + best_sum_log_gain_Q7 = sum_log_gain_tmp_Q7; + } + } + + cbk_ptr_Q7 = silk_LTP_vq_ptrs_Q7[ *periodicity_index ]; + for( j = 0; j < nb_subfr; j++ ) { + for( k = 0; k < LTP_ORDER; k++ ) { + B_Q14[ j * LTP_ORDER + k ] = silk_LSHIFT( cbk_ptr_Q7[ cbk_index[ j ] * LTP_ORDER + k ], 7 ); + } + } + + if( nb_subfr == 2 ) { + res_nrg_Q15 = silk_RSHIFT32( res_nrg_Q15, 1 ); + } else { + res_nrg_Q15 = silk_RSHIFT32( res_nrg_Q15, 2 ); + } + + *sum_log_gain_Q7 = best_sum_log_gain_Q7; + *pred_gain_dB_Q7 = (opus_int)silk_SMULBB( -3, silk_lin2log( res_nrg_Q15 ) - ( 15 << 7 ) ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler.c new file mode 100644 index 000000000..1f11e5089 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler.c @@ -0,0 +1,215 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* + * Matrix of resampling methods used: + * Fs_out (kHz) + * 8 12 16 24 48 + * + * 8 C UF U UF UF + * 12 AF C UF U UF + * Fs_in (kHz) 16 D AF C UF UF + * 24 AF D AF C U + * 48 AF AF AF D C + * + * C -> Copy (no resampling) + * D -> Allpass-based 2x downsampling + * U -> Allpass-based 2x upsampling + * UF -> Allpass-based 2x upsampling followed by FIR interpolation + * AF -> AR2 filter followed by FIR interpolation + */ + +#include "resampler_private.h" + +/* Tables with delay compensation values to equalize total delay for different modes */ +static const opus_int8 delay_matrix_enc[ 5 ][ 3 ] = { +/* in \ out 8 12 16 */ +/* 8 */ { 6, 0, 3 }, +/* 12 */ { 0, 7, 3 }, +/* 16 */ { 0, 1, 10 }, +/* 24 */ { 0, 2, 6 }, +/* 48 */ { 18, 10, 12 } +}; + +static const opus_int8 delay_matrix_dec[ 3 ][ 5 ] = { +/* in \ out 8 12 16 24 48 */ +/* 8 */ { 4, 0, 2, 0, 0 }, +/* 12 */ { 0, 9, 4, 7, 4 }, +/* 16 */ { 0, 3, 12, 7, 7 } +}; + +/* Simple way to make [8000, 12000, 16000, 24000, 48000] to [0, 1, 2, 3, 4] */ +#define rateID(R) ( ( ( ((R)>>12) - ((R)>16000) ) >> ((R)>24000) ) - 1 ) + +#define USE_silk_resampler_copy (0) +#define USE_silk_resampler_private_up2_HQ_wrapper (1) +#define USE_silk_resampler_private_IIR_FIR (2) +#define USE_silk_resampler_private_down_FIR (3) + +/* Initialize/reset the resampler state for a given pair of input/output sampling rates */ +opus_int silk_resampler_init( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int32 Fs_Hz_in, /* I Input sampling rate (Hz) */ + opus_int32 Fs_Hz_out, /* I Output sampling rate (Hz) */ + opus_int forEnc /* I If 1: encoder; if 0: decoder */ +) +{ + opus_int up2x; + + /* Clear state */ + silk_memset( S, 0, sizeof( silk_resampler_state_struct ) ); + + /* Input checking */ + if( forEnc ) { + if( ( Fs_Hz_in != 8000 && Fs_Hz_in != 12000 && Fs_Hz_in != 16000 && Fs_Hz_in != 24000 && Fs_Hz_in != 48000 ) || + ( Fs_Hz_out != 8000 && Fs_Hz_out != 12000 && Fs_Hz_out != 16000 ) ) { + celt_assert( 0 ); + return -1; + } + S->inputDelay = delay_matrix_enc[ rateID( Fs_Hz_in ) ][ rateID( Fs_Hz_out ) ]; + } else { + if( ( Fs_Hz_in != 8000 && Fs_Hz_in != 12000 && Fs_Hz_in != 16000 ) || + ( Fs_Hz_out != 8000 && Fs_Hz_out != 12000 && Fs_Hz_out != 16000 && Fs_Hz_out != 24000 && Fs_Hz_out != 48000 ) ) { + celt_assert( 0 ); + return -1; + } + S->inputDelay = delay_matrix_dec[ rateID( Fs_Hz_in ) ][ rateID( Fs_Hz_out ) ]; + } + + S->Fs_in_kHz = silk_DIV32_16( Fs_Hz_in, 1000 ); + S->Fs_out_kHz = silk_DIV32_16( Fs_Hz_out, 1000 ); + + /* Number of samples processed per batch */ + S->batchSize = S->Fs_in_kHz * RESAMPLER_MAX_BATCH_SIZE_MS; + + /* Find resampler with the right sampling ratio */ + up2x = 0; + if( Fs_Hz_out > Fs_Hz_in ) { + /* Upsample */ + if( Fs_Hz_out == silk_MUL( Fs_Hz_in, 2 ) ) { /* Fs_out : Fs_in = 2 : 1 */ + /* Special case: directly use 2x upsampler */ + S->resampler_function = USE_silk_resampler_private_up2_HQ_wrapper; + } else { + /* Default resampler */ + S->resampler_function = USE_silk_resampler_private_IIR_FIR; + up2x = 1; + } + } else if ( Fs_Hz_out < Fs_Hz_in ) { + /* Downsample */ + S->resampler_function = USE_silk_resampler_private_down_FIR; + if( silk_MUL( Fs_Hz_out, 4 ) == silk_MUL( Fs_Hz_in, 3 ) ) { /* Fs_out : Fs_in = 3 : 4 */ + S->FIR_Fracs = 3; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR0; + S->Coefs = silk_Resampler_3_4_COEFS; + } else if( silk_MUL( Fs_Hz_out, 3 ) == silk_MUL( Fs_Hz_in, 2 ) ) { /* Fs_out : Fs_in = 2 : 3 */ + S->FIR_Fracs = 2; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR0; + S->Coefs = silk_Resampler_2_3_COEFS; + } else if( silk_MUL( Fs_Hz_out, 2 ) == Fs_Hz_in ) { /* Fs_out : Fs_in = 1 : 2 */ + S->FIR_Fracs = 1; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR1; + S->Coefs = silk_Resampler_1_2_COEFS; + } else if( silk_MUL( Fs_Hz_out, 3 ) == Fs_Hz_in ) { /* Fs_out : Fs_in = 1 : 3 */ + S->FIR_Fracs = 1; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR2; + S->Coefs = silk_Resampler_1_3_COEFS; + } else if( silk_MUL( Fs_Hz_out, 4 ) == Fs_Hz_in ) { /* Fs_out : Fs_in = 1 : 4 */ + S->FIR_Fracs = 1; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR2; + S->Coefs = silk_Resampler_1_4_COEFS; + } else if( silk_MUL( Fs_Hz_out, 6 ) == Fs_Hz_in ) { /* Fs_out : Fs_in = 1 : 6 */ + S->FIR_Fracs = 1; + S->FIR_Order = RESAMPLER_DOWN_ORDER_FIR2; + S->Coefs = silk_Resampler_1_6_COEFS; + } else { + /* None available */ + celt_assert( 0 ); + return -1; + } + } else { + /* Input and output sampling rates are equal: copy */ + S->resampler_function = USE_silk_resampler_copy; + } + + /* Ratio of input/output samples */ + S->invRatio_Q16 = silk_LSHIFT32( silk_DIV32( silk_LSHIFT32( Fs_Hz_in, 14 + up2x ), Fs_Hz_out ), 2 ); + /* Make sure the ratio is rounded up */ + while( silk_SMULWW( S->invRatio_Q16, Fs_Hz_out ) < silk_LSHIFT32( Fs_Hz_in, up2x ) ) { + S->invRatio_Q16++; + } + + return 0; +} + +/* Resampler: convert from one sampling rate to another */ +/* Input and output sampling rate are at most 48000 Hz */ +opus_int silk_resampler( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +) +{ + opus_int nSamples; + + /* Need at least 1 ms of input data */ + celt_assert( inLen >= S->Fs_in_kHz ); + /* Delay can't exceed the 1 ms of buffering */ + celt_assert( S->inputDelay <= S->Fs_in_kHz ); + + nSamples = S->Fs_in_kHz - S->inputDelay; + + /* Copy to delay buffer */ + silk_memcpy( &S->delayBuf[ S->inputDelay ], in, nSamples * sizeof( opus_int16 ) ); + + switch( S->resampler_function ) { + case USE_silk_resampler_private_up2_HQ_wrapper: + silk_resampler_private_up2_HQ_wrapper( S, out, S->delayBuf, S->Fs_in_kHz ); + silk_resampler_private_up2_HQ_wrapper( S, &out[ S->Fs_out_kHz ], &in[ nSamples ], inLen - S->Fs_in_kHz ); + break; + case USE_silk_resampler_private_IIR_FIR: + silk_resampler_private_IIR_FIR( S, out, S->delayBuf, S->Fs_in_kHz ); + silk_resampler_private_IIR_FIR( S, &out[ S->Fs_out_kHz ], &in[ nSamples ], inLen - S->Fs_in_kHz ); + break; + case USE_silk_resampler_private_down_FIR: + silk_resampler_private_down_FIR( S, out, S->delayBuf, S->Fs_in_kHz ); + silk_resampler_private_down_FIR( S, &out[ S->Fs_out_kHz ], &in[ nSamples ], inLen - S->Fs_in_kHz ); + break; + default: + silk_memcpy( out, S->delayBuf, S->Fs_in_kHz * sizeof( opus_int16 ) ); + silk_memcpy( &out[ S->Fs_out_kHz ], &in[ nSamples ], ( inLen - S->Fs_in_kHz ) * sizeof( opus_int16 ) ); + } + + /* Copy to delay buffer */ + silk_memcpy( S->delayBuf, &in[ inLen - S->inputDelay ], S->inputDelay * sizeof( opus_int16 ) ); + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2.c new file mode 100644 index 000000000..971d7bfd4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2.c @@ -0,0 +1,74 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_rom.h" + +/* Downsample by a factor 2 */ +void silk_resampler_down2( + opus_int32 *S, /* I/O State vector [ 2 ] */ + opus_int16 *out, /* O Output signal [ floor(len/2) ] */ + const opus_int16 *in, /* I Input signal [ len ] */ + opus_int32 inLen /* I Number of input samples */ +) +{ + opus_int32 k, len2 = silk_RSHIFT32( inLen, 1 ); + opus_int32 in32, out32, Y, X; + + celt_assert( silk_resampler_down2_0 > 0 ); + celt_assert( silk_resampler_down2_1 < 0 ); + + /* Internal variables and state are in Q10 format */ + for( k = 0; k < len2; k++ ) { + /* Convert to Q10 */ + in32 = silk_LSHIFT( (opus_int32)in[ 2 * k ], 10 ); + + /* All-pass section for even input sample */ + Y = silk_SUB32( in32, S[ 0 ] ); + X = silk_SMLAWB( Y, Y, silk_resampler_down2_1 ); + out32 = silk_ADD32( S[ 0 ], X ); + S[ 0 ] = silk_ADD32( in32, X ); + + /* Convert to Q10 */ + in32 = silk_LSHIFT( (opus_int32)in[ 2 * k + 1 ], 10 ); + + /* All-pass section for odd input sample, and add to output of previous section */ + Y = silk_SUB32( in32, S[ 1 ] ); + X = silk_SMULWB( Y, silk_resampler_down2_0 ); + out32 = silk_ADD32( out32, S[ 1 ] ); + out32 = silk_ADD32( out32, X ); + S[ 1 ] = silk_ADD32( in32, X ); + + /* Add, convert back to int16 and store to output */ + out[ k ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( out32, 11 ) ); + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2_3.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2_3.c new file mode 100644 index 000000000..4342614dc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_down2_3.c @@ -0,0 +1,103 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_private.h" +#include "stack_alloc.h" + +#define ORDER_FIR 4 + +/* Downsample by a factor 2/3, low quality */ +void silk_resampler_down2_3( + opus_int32 *S, /* I/O State vector [ 6 ] */ + opus_int16 *out, /* O Output signal [ floor(2*inLen/3) ] */ + const opus_int16 *in, /* I Input signal [ inLen ] */ + opus_int32 inLen /* I Number of input samples */ +) +{ + opus_int32 nSamplesIn, counter, res_Q6; + VARDECL( opus_int32, buf ); + opus_int32 *buf_ptr; + SAVE_STACK; + + ALLOC( buf, RESAMPLER_MAX_BATCH_SIZE_IN + ORDER_FIR, opus_int32 ); + + /* Copy buffered samples to start of buffer */ + silk_memcpy( buf, S, ORDER_FIR * sizeof( opus_int32 ) ); + + /* Iterate over blocks of frameSizeIn input samples */ + while( 1 ) { + nSamplesIn = silk_min( inLen, RESAMPLER_MAX_BATCH_SIZE_IN ); + + /* Second-order AR filter (output in Q8) */ + silk_resampler_private_AR2( &S[ ORDER_FIR ], &buf[ ORDER_FIR ], in, + silk_Resampler_2_3_COEFS_LQ, nSamplesIn ); + + /* Interpolate filtered signal */ + buf_ptr = buf; + counter = nSamplesIn; + while( counter > 2 ) { + /* Inner product */ + res_Q6 = silk_SMULWB( buf_ptr[ 0 ], silk_Resampler_2_3_COEFS_LQ[ 2 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 1 ], silk_Resampler_2_3_COEFS_LQ[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 2 ], silk_Resampler_2_3_COEFS_LQ[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 3 ], silk_Resampler_2_3_COEFS_LQ[ 4 ] ); + + /* Scale down, saturate and store in output array */ + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q6, 6 ) ); + + res_Q6 = silk_SMULWB( buf_ptr[ 1 ], silk_Resampler_2_3_COEFS_LQ[ 4 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 2 ], silk_Resampler_2_3_COEFS_LQ[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 3 ], silk_Resampler_2_3_COEFS_LQ[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 4 ], silk_Resampler_2_3_COEFS_LQ[ 2 ] ); + + /* Scale down, saturate and store in output array */ + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q6, 6 ) ); + + buf_ptr += 3; + counter -= 3; + } + + in += nSamplesIn; + inLen -= nSamplesIn; + + if( inLen > 0 ) { + /* More iterations to do; copy last part of filtered signal to beginning of buffer */ + silk_memcpy( buf, &buf[ nSamplesIn ], ORDER_FIR * sizeof( opus_int32 ) ); + } else { + break; + } + } + + /* Copy last part of filtered signal to the state for the next call */ + silk_memcpy( S, &buf[ nSamplesIn ], ORDER_FIR * sizeof( opus_int32 ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private.h new file mode 100644 index 000000000..422a7d9d9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private.h @@ -0,0 +1,88 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_RESAMPLER_PRIVATE_H +#define SILK_RESAMPLER_PRIVATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "SigProc_FIX.h" +#include "resampler_structs.h" +#include "resampler_rom.h" + +/* Number of input samples to process in the inner loop */ +#define RESAMPLER_MAX_BATCH_SIZE_MS 10 +#define RESAMPLER_MAX_FS_KHZ 48 +#define RESAMPLER_MAX_BATCH_SIZE_IN ( RESAMPLER_MAX_BATCH_SIZE_MS * RESAMPLER_MAX_FS_KHZ ) + +/* Description: Hybrid IIR/FIR polyphase implementation of resampling */ +void silk_resampler_private_IIR_FIR( + void *SS, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +); + +/* Description: Hybrid IIR/FIR polyphase implementation of resampling */ +void silk_resampler_private_down_FIR( + void *SS, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +); + +/* Upsample by a factor 2, high quality */ +void silk_resampler_private_up2_HQ_wrapper( + void *SS, /* I/O Resampler state (unused) */ + opus_int16 *out, /* O Output signal [ 2 * len ] */ + const opus_int16 *in, /* I Input signal [ len ] */ + opus_int32 len /* I Number of input samples */ +); + +/* Upsample by a factor 2, high quality */ +void silk_resampler_private_up2_HQ( + opus_int32 *S, /* I/O Resampler state [ 6 ] */ + opus_int16 *out, /* O Output signal [ 2 * len ] */ + const opus_int16 *in, /* I Input signal [ len ] */ + opus_int32 len /* I Number of input samples */ +); + +/* Second order AR filter */ +void silk_resampler_private_AR2( + opus_int32 S[], /* I/O State vector [ 2 ] */ + opus_int32 out_Q8[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + const opus_int16 A_Q14[], /* I AR coefficients, Q14 */ + opus_int32 len /* I Signal length */ +); + +#ifdef __cplusplus +} +#endif +#endif /* SILK_RESAMPLER_PRIVATE_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_AR2.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_AR2.c new file mode 100644 index 000000000..5fff23714 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_AR2.c @@ -0,0 +1,55 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_private.h" + +/* Second order AR filter with single delay elements */ +void silk_resampler_private_AR2( + opus_int32 S[], /* I/O State vector [ 2 ] */ + opus_int32 out_Q8[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + const opus_int16 A_Q14[], /* I AR coefficients, Q14 */ + opus_int32 len /* I Signal length */ +) +{ + opus_int32 k; + opus_int32 out32; + + for( k = 0; k < len; k++ ) { + out32 = silk_ADD_LSHIFT32( S[ 0 ], (opus_int32)in[ k ], 8 ); + out_Q8[ k ] = out32; + out32 = silk_LSHIFT( out32, 2 ); + S[ 0 ] = silk_SMLAWB( S[ 1 ], out32, A_Q14[ 0 ] ); + S[ 1 ] = silk_SMULWB( out32, A_Q14[ 1 ] ); + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_IIR_FIR.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_IIR_FIR.c new file mode 100644 index 000000000..6b2b3a2e1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_IIR_FIR.c @@ -0,0 +1,107 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_private.h" +#include "stack_alloc.h" + +static OPUS_INLINE opus_int16 *silk_resampler_private_IIR_FIR_INTERPOL( + opus_int16 *out, + opus_int16 *buf, + opus_int32 max_index_Q16, + opus_int32 index_increment_Q16 +) +{ + opus_int32 index_Q16, res_Q15; + opus_int16 *buf_ptr; + opus_int32 table_index; + + /* Interpolate upsampled signal and store in output array */ + for( index_Q16 = 0; index_Q16 < max_index_Q16; index_Q16 += index_increment_Q16 ) { + table_index = silk_SMULWB( index_Q16 & 0xFFFF, 12 ); + buf_ptr = &buf[ index_Q16 >> 16 ]; + + res_Q15 = silk_SMULBB( buf_ptr[ 0 ], silk_resampler_frac_FIR_12[ table_index ][ 0 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 1 ], silk_resampler_frac_FIR_12[ table_index ][ 1 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 2 ], silk_resampler_frac_FIR_12[ table_index ][ 2 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 3 ], silk_resampler_frac_FIR_12[ table_index ][ 3 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 4 ], silk_resampler_frac_FIR_12[ 11 - table_index ][ 3 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 5 ], silk_resampler_frac_FIR_12[ 11 - table_index ][ 2 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 6 ], silk_resampler_frac_FIR_12[ 11 - table_index ][ 1 ] ); + res_Q15 = silk_SMLABB( res_Q15, buf_ptr[ 7 ], silk_resampler_frac_FIR_12[ 11 - table_index ][ 0 ] ); + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q15, 15 ) ); + } + return out; +} +/* Upsample using a combination of allpass-based 2x upsampling and FIR interpolation */ +void silk_resampler_private_IIR_FIR( + void *SS, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +) +{ + silk_resampler_state_struct *S = (silk_resampler_state_struct *)SS; + opus_int32 nSamplesIn; + opus_int32 max_index_Q16, index_increment_Q16; + VARDECL( opus_int16, buf ); + SAVE_STACK; + + ALLOC( buf, 2 * S->batchSize + RESAMPLER_ORDER_FIR_12, opus_int16 ); + + /* Copy buffered samples to start of buffer */ + silk_memcpy( buf, S->sFIR.i16, RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + + /* Iterate over blocks of frameSizeIn input samples */ + index_increment_Q16 = S->invRatio_Q16; + while( 1 ) { + nSamplesIn = silk_min( inLen, S->batchSize ); + + /* Upsample 2x */ + silk_resampler_private_up2_HQ( S->sIIR, &buf[ RESAMPLER_ORDER_FIR_12 ], in, nSamplesIn ); + + max_index_Q16 = silk_LSHIFT32( nSamplesIn, 16 + 1 ); /* + 1 because 2x upsampling */ + out = silk_resampler_private_IIR_FIR_INTERPOL( out, buf, max_index_Q16, index_increment_Q16 ); + in += nSamplesIn; + inLen -= nSamplesIn; + + if( inLen > 0 ) { + /* More iterations to do; copy last part of filtered signal to beginning of buffer */ + silk_memcpy( buf, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + } else { + break; + } + } + + /* Copy last part of filtered signal to the state for the next call */ + silk_memcpy( S->sFIR.i16, &buf[ nSamplesIn << 1 ], RESAMPLER_ORDER_FIR_12 * sizeof( opus_int16 ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_down_FIR.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_down_FIR.c new file mode 100644 index 000000000..3e8735a35 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_down_FIR.c @@ -0,0 +1,194 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_private.h" +#include "stack_alloc.h" + +static OPUS_INLINE opus_int16 *silk_resampler_private_down_FIR_INTERPOL( + opus_int16 *out, + opus_int32 *buf, + const opus_int16 *FIR_Coefs, + opus_int FIR_Order, + opus_int FIR_Fracs, + opus_int32 max_index_Q16, + opus_int32 index_increment_Q16 +) +{ + opus_int32 index_Q16, res_Q6; + opus_int32 *buf_ptr; + opus_int32 interpol_ind; + const opus_int16 *interpol_ptr; + + switch( FIR_Order ) { + case RESAMPLER_DOWN_ORDER_FIR0: + for( index_Q16 = 0; index_Q16 < max_index_Q16; index_Q16 += index_increment_Q16 ) { + /* Integer part gives pointer to buffered input */ + buf_ptr = buf + silk_RSHIFT( index_Q16, 16 ); + + /* Fractional part gives interpolation coefficients */ + interpol_ind = silk_SMULWB( index_Q16 & 0xFFFF, FIR_Fracs ); + + /* Inner product */ + interpol_ptr = &FIR_Coefs[ RESAMPLER_DOWN_ORDER_FIR0 / 2 * interpol_ind ]; + res_Q6 = silk_SMULWB( buf_ptr[ 0 ], interpol_ptr[ 0 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 1 ], interpol_ptr[ 1 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 2 ], interpol_ptr[ 2 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 3 ], interpol_ptr[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 4 ], interpol_ptr[ 4 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 5 ], interpol_ptr[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 6 ], interpol_ptr[ 6 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 7 ], interpol_ptr[ 7 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 8 ], interpol_ptr[ 8 ] ); + interpol_ptr = &FIR_Coefs[ RESAMPLER_DOWN_ORDER_FIR0 / 2 * ( FIR_Fracs - 1 - interpol_ind ) ]; + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 17 ], interpol_ptr[ 0 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 16 ], interpol_ptr[ 1 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 15 ], interpol_ptr[ 2 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 14 ], interpol_ptr[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 13 ], interpol_ptr[ 4 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 12 ], interpol_ptr[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 11 ], interpol_ptr[ 6 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 10 ], interpol_ptr[ 7 ] ); + res_Q6 = silk_SMLAWB( res_Q6, buf_ptr[ 9 ], interpol_ptr[ 8 ] ); + + /* Scale down, saturate and store in output array */ + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q6, 6 ) ); + } + break; + case RESAMPLER_DOWN_ORDER_FIR1: + for( index_Q16 = 0; index_Q16 < max_index_Q16; index_Q16 += index_increment_Q16 ) { + /* Integer part gives pointer to buffered input */ + buf_ptr = buf + silk_RSHIFT( index_Q16, 16 ); + + /* Inner product */ + res_Q6 = silk_SMULWB( silk_ADD32( buf_ptr[ 0 ], buf_ptr[ 23 ] ), FIR_Coefs[ 0 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 1 ], buf_ptr[ 22 ] ), FIR_Coefs[ 1 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 2 ], buf_ptr[ 21 ] ), FIR_Coefs[ 2 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 3 ], buf_ptr[ 20 ] ), FIR_Coefs[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 4 ], buf_ptr[ 19 ] ), FIR_Coefs[ 4 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 5 ], buf_ptr[ 18 ] ), FIR_Coefs[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 6 ], buf_ptr[ 17 ] ), FIR_Coefs[ 6 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 7 ], buf_ptr[ 16 ] ), FIR_Coefs[ 7 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 8 ], buf_ptr[ 15 ] ), FIR_Coefs[ 8 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 9 ], buf_ptr[ 14 ] ), FIR_Coefs[ 9 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 10 ], buf_ptr[ 13 ] ), FIR_Coefs[ 10 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 11 ], buf_ptr[ 12 ] ), FIR_Coefs[ 11 ] ); + + /* Scale down, saturate and store in output array */ + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q6, 6 ) ); + } + break; + case RESAMPLER_DOWN_ORDER_FIR2: + for( index_Q16 = 0; index_Q16 < max_index_Q16; index_Q16 += index_increment_Q16 ) { + /* Integer part gives pointer to buffered input */ + buf_ptr = buf + silk_RSHIFT( index_Q16, 16 ); + + /* Inner product */ + res_Q6 = silk_SMULWB( silk_ADD32( buf_ptr[ 0 ], buf_ptr[ 35 ] ), FIR_Coefs[ 0 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 1 ], buf_ptr[ 34 ] ), FIR_Coefs[ 1 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 2 ], buf_ptr[ 33 ] ), FIR_Coefs[ 2 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 3 ], buf_ptr[ 32 ] ), FIR_Coefs[ 3 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 4 ], buf_ptr[ 31 ] ), FIR_Coefs[ 4 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 5 ], buf_ptr[ 30 ] ), FIR_Coefs[ 5 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 6 ], buf_ptr[ 29 ] ), FIR_Coefs[ 6 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 7 ], buf_ptr[ 28 ] ), FIR_Coefs[ 7 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 8 ], buf_ptr[ 27 ] ), FIR_Coefs[ 8 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 9 ], buf_ptr[ 26 ] ), FIR_Coefs[ 9 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 10 ], buf_ptr[ 25 ] ), FIR_Coefs[ 10 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 11 ], buf_ptr[ 24 ] ), FIR_Coefs[ 11 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 12 ], buf_ptr[ 23 ] ), FIR_Coefs[ 12 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 13 ], buf_ptr[ 22 ] ), FIR_Coefs[ 13 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 14 ], buf_ptr[ 21 ] ), FIR_Coefs[ 14 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 15 ], buf_ptr[ 20 ] ), FIR_Coefs[ 15 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 16 ], buf_ptr[ 19 ] ), FIR_Coefs[ 16 ] ); + res_Q6 = silk_SMLAWB( res_Q6, silk_ADD32( buf_ptr[ 17 ], buf_ptr[ 18 ] ), FIR_Coefs[ 17 ] ); + + /* Scale down, saturate and store in output array */ + *out++ = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( res_Q6, 6 ) ); + } + break; + default: + celt_assert( 0 ); + } + return out; +} + +/* Resample with a 2nd order AR filter followed by FIR interpolation */ +void silk_resampler_private_down_FIR( + void *SS, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +) +{ + silk_resampler_state_struct *S = (silk_resampler_state_struct *)SS; + opus_int32 nSamplesIn; + opus_int32 max_index_Q16, index_increment_Q16; + VARDECL( opus_int32, buf ); + const opus_int16 *FIR_Coefs; + SAVE_STACK; + + ALLOC( buf, S->batchSize + S->FIR_Order, opus_int32 ); + + /* Copy buffered samples to start of buffer */ + silk_memcpy( buf, S->sFIR.i32, S->FIR_Order * sizeof( opus_int32 ) ); + + FIR_Coefs = &S->Coefs[ 2 ]; + + /* Iterate over blocks of frameSizeIn input samples */ + index_increment_Q16 = S->invRatio_Q16; + while( 1 ) { + nSamplesIn = silk_min( inLen, S->batchSize ); + + /* Second-order AR filter (output in Q8) */ + silk_resampler_private_AR2( S->sIIR, &buf[ S->FIR_Order ], in, S->Coefs, nSamplesIn ); + + max_index_Q16 = silk_LSHIFT32( nSamplesIn, 16 ); + + /* Interpolate filtered signal */ + out = silk_resampler_private_down_FIR_INTERPOL( out, buf, FIR_Coefs, S->FIR_Order, + S->FIR_Fracs, max_index_Q16, index_increment_Q16 ); + + in += nSamplesIn; + inLen -= nSamplesIn; + + if( inLen > 1 ) { + /* More iterations to do; copy last part of filtered signal to beginning of buffer */ + silk_memcpy( buf, &buf[ nSamplesIn ], S->FIR_Order * sizeof( opus_int32 ) ); + } else { + break; + } + } + + /* Copy last part of filtered signal to the state for the next call */ + silk_memcpy( S->sFIR.i32, &buf[ nSamplesIn ], S->FIR_Order * sizeof( opus_int32 ) ); + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_up2_HQ.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_up2_HQ.c new file mode 100644 index 000000000..c7ec8de36 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_private_up2_HQ.c @@ -0,0 +1,113 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" +#include "resampler_private.h" + +/* Upsample by a factor 2, high quality */ +/* Uses 2nd order allpass filters for the 2x upsampling, followed by a */ +/* notch filter just above Nyquist. */ +void silk_resampler_private_up2_HQ( + opus_int32 *S, /* I/O Resampler state [ 6 ] */ + opus_int16 *out, /* O Output signal [ 2 * len ] */ + const opus_int16 *in, /* I Input signal [ len ] */ + opus_int32 len /* I Number of input samples */ +) +{ + opus_int32 k; + opus_int32 in32, out32_1, out32_2, Y, X; + + silk_assert( silk_resampler_up2_hq_0[ 0 ] > 0 ); + silk_assert( silk_resampler_up2_hq_0[ 1 ] > 0 ); + silk_assert( silk_resampler_up2_hq_0[ 2 ] < 0 ); + silk_assert( silk_resampler_up2_hq_1[ 0 ] > 0 ); + silk_assert( silk_resampler_up2_hq_1[ 1 ] > 0 ); + silk_assert( silk_resampler_up2_hq_1[ 2 ] < 0 ); + + /* Internal variables and state are in Q10 format */ + for( k = 0; k < len; k++ ) { + /* Convert to Q10 */ + in32 = silk_LSHIFT( (opus_int32)in[ k ], 10 ); + + /* First all-pass section for even output sample */ + Y = silk_SUB32( in32, S[ 0 ] ); + X = silk_SMULWB( Y, silk_resampler_up2_hq_0[ 0 ] ); + out32_1 = silk_ADD32( S[ 0 ], X ); + S[ 0 ] = silk_ADD32( in32, X ); + + /* Second all-pass section for even output sample */ + Y = silk_SUB32( out32_1, S[ 1 ] ); + X = silk_SMULWB( Y, silk_resampler_up2_hq_0[ 1 ] ); + out32_2 = silk_ADD32( S[ 1 ], X ); + S[ 1 ] = silk_ADD32( out32_1, X ); + + /* Third all-pass section for even output sample */ + Y = silk_SUB32( out32_2, S[ 2 ] ); + X = silk_SMLAWB( Y, Y, silk_resampler_up2_hq_0[ 2 ] ); + out32_1 = silk_ADD32( S[ 2 ], X ); + S[ 2 ] = silk_ADD32( out32_2, X ); + + /* Apply gain in Q15, convert back to int16 and store to output */ + out[ 2 * k ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( out32_1, 10 ) ); + + /* First all-pass section for odd output sample */ + Y = silk_SUB32( in32, S[ 3 ] ); + X = silk_SMULWB( Y, silk_resampler_up2_hq_1[ 0 ] ); + out32_1 = silk_ADD32( S[ 3 ], X ); + S[ 3 ] = silk_ADD32( in32, X ); + + /* Second all-pass section for odd output sample */ + Y = silk_SUB32( out32_1, S[ 4 ] ); + X = silk_SMULWB( Y, silk_resampler_up2_hq_1[ 1 ] ); + out32_2 = silk_ADD32( S[ 4 ], X ); + S[ 4 ] = silk_ADD32( out32_1, X ); + + /* Third all-pass section for odd output sample */ + Y = silk_SUB32( out32_2, S[ 5 ] ); + X = silk_SMLAWB( Y, Y, silk_resampler_up2_hq_1[ 2 ] ); + out32_1 = silk_ADD32( S[ 5 ], X ); + S[ 5 ] = silk_ADD32( out32_2, X ); + + /* Apply gain in Q15, convert back to int16 and store to output */ + out[ 2 * k + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( out32_1, 10 ) ); + } +} + +void silk_resampler_private_up2_HQ_wrapper( + void *SS, /* I/O Resampler state (unused) */ + opus_int16 *out, /* O Output signal [ 2 * len ] */ + const opus_int16 *in, /* I Input signal [ len ] */ + opus_int32 len /* I Number of input samples */ +) +{ + silk_resampler_state_struct *S = (silk_resampler_state_struct *)SS; + silk_resampler_private_up2_HQ( S->sIIR, out, in, len ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.c new file mode 100644 index 000000000..5e6b04476 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.c @@ -0,0 +1,96 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Filter coefficients for IIR/FIR polyphase resampling * + * Total size: 179 Words (358 Bytes) */ + +#include "resampler_private.h" + +/* Matlab code for the notch filter coefficients: */ +/* B = [1, 0.147, 1]; A = [1, 0.107, 0.89]; G = 0.93; freqz(G * B, A, 2^14, 16e3); axis([0, 8000, -10, 1]) */ +/* fprintf('\t%6d, %6d, %6d, %6d\n', round(B(2)*2^16), round(-A(2)*2^16), round((1-A(3))*2^16), round(G*2^15)) */ +/* const opus_int16 silk_resampler_up2_hq_notch[ 4 ] = { 9634, -7012, 7209, 30474 }; */ + +/* Tables with IIR and FIR coefficients for fractional downsamplers (123 Words) */ +silk_DWORD_ALIGN const opus_int16 silk_Resampler_3_4_COEFS[ 2 + 3 * RESAMPLER_DOWN_ORDER_FIR0 / 2 ] = { + -20694, -13867, + -49, 64, 17, -157, 353, -496, 163, 11047, 22205, + -39, 6, 91, -170, 186, 23, -896, 6336, 19928, + -19, -36, 102, -89, -24, 328, -951, 2568, 15909, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_2_3_COEFS[ 2 + 2 * RESAMPLER_DOWN_ORDER_FIR0 / 2 ] = { + -14457, -14019, + 64, 128, -122, 36, 310, -768, 584, 9267, 17733, + 12, 128, 18, -142, 288, -117, -865, 4123, 14459, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_1_2_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR1 / 2 ] = { + 616, -14323, + -10, 39, 58, -46, -84, 120, 184, -315, -541, 1284, 5380, 9024, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_1_3_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ] = { + 16102, -15162, + -13, 0, 20, 26, 5, -31, -43, -4, 65, 90, 7, -157, -248, -44, 593, 1583, 2612, 3271, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_1_4_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ] = { + 22500, -15099, + 3, -14, -20, -15, 2, 25, 37, 25, -16, -71, -107, -79, 50, 292, 623, 982, 1288, 1464, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_1_6_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ] = { + 27540, -15257, + 17, 12, 8, 1, -10, -22, -30, -32, -22, 3, 44, 100, 168, 243, 317, 381, 429, 455, +}; + +silk_DWORD_ALIGN const opus_int16 silk_Resampler_2_3_COEFS_LQ[ 2 + 2 * 2 ] = { + -2797, -6507, + 4697, 10739, + 1567, 8276, +}; + +/* Table with interplation fractions of 1/24, 3/24, 5/24, ... , 23/24 : 23/24 (46 Words) */ +silk_DWORD_ALIGN const opus_int16 silk_resampler_frac_FIR_12[ 12 ][ RESAMPLER_ORDER_FIR_12 / 2 ] = { + { 189, -600, 617, 30567 }, + { 117, -159, -1070, 29704 }, + { 52, 221, -2392, 28276 }, + { -4, 529, -3350, 26341 }, + { -48, 758, -3956, 23973 }, + { -80, 905, -4235, 21254 }, + { -99, 972, -4222, 18278 }, + { -107, 967, -3957, 15143 }, + { -103, 896, -3487, 11950 }, + { -91, 773, -2865, 8798 }, + { -71, 611, -2143, 5784 }, + { -46, 425, -1375, 2996 }, +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.h new file mode 100644 index 000000000..490b3388d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_rom.h @@ -0,0 +1,68 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_FIX_RESAMPLER_ROM_H +#define SILK_FIX_RESAMPLER_ROM_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "typedef.h" +#include "resampler_structs.h" + +#define RESAMPLER_DOWN_ORDER_FIR0 18 +#define RESAMPLER_DOWN_ORDER_FIR1 24 +#define RESAMPLER_DOWN_ORDER_FIR2 36 +#define RESAMPLER_ORDER_FIR_12 8 + +/* Tables for 2x downsampler */ +static const opus_int16 silk_resampler_down2_0 = 9872; +static const opus_int16 silk_resampler_down2_1 = 39809 - 65536; + +/* Tables for 2x upsampler, high quality */ +static const opus_int16 silk_resampler_up2_hq_0[ 3 ] = { 1746, 14986, 39083 - 65536 }; +static const opus_int16 silk_resampler_up2_hq_1[ 3 ] = { 6854, 25769, 55542 - 65536 }; + +/* Tables with IIR and FIR coefficients for fractional downsamplers */ +extern const opus_int16 silk_Resampler_3_4_COEFS[ 2 + 3 * RESAMPLER_DOWN_ORDER_FIR0 / 2 ]; +extern const opus_int16 silk_Resampler_2_3_COEFS[ 2 + 2 * RESAMPLER_DOWN_ORDER_FIR0 / 2 ]; +extern const opus_int16 silk_Resampler_1_2_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR1 / 2 ]; +extern const opus_int16 silk_Resampler_1_3_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ]; +extern const opus_int16 silk_Resampler_1_4_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ]; +extern const opus_int16 silk_Resampler_1_6_COEFS[ 2 + RESAMPLER_DOWN_ORDER_FIR2 / 2 ]; +extern const opus_int16 silk_Resampler_2_3_COEFS_LQ[ 2 + 2 * 2 ]; + +/* Table with interplation fractions of 1/24, 3/24, ..., 23/24 */ +extern const opus_int16 silk_resampler_frac_FIR_12[ 12 ][ RESAMPLER_ORDER_FIR_12 / 2 ]; + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_FIX_RESAMPLER_ROM_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_structs.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_structs.h new file mode 100644 index 000000000..9e9457d11 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/resampler_structs.h @@ -0,0 +1,60 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_RESAMPLER_STRUCTS_H +#define SILK_RESAMPLER_STRUCTS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define SILK_RESAMPLER_MAX_FIR_ORDER 36 +#define SILK_RESAMPLER_MAX_IIR_ORDER 6 + +typedef struct _silk_resampler_state_struct{ + opus_int32 sIIR[ SILK_RESAMPLER_MAX_IIR_ORDER ]; /* this must be the first element of this struct */ + union{ + opus_int32 i32[ SILK_RESAMPLER_MAX_FIR_ORDER ]; + opus_int16 i16[ SILK_RESAMPLER_MAX_FIR_ORDER ]; + } sFIR; + opus_int16 delayBuf[ 48 ]; + opus_int resampler_function; + opus_int batchSize; + opus_int32 invRatio_Q16; + opus_int FIR_Order; + opus_int FIR_Fracs; + opus_int Fs_in_kHz; + opus_int Fs_out_kHz; + opus_int inputDelay; + const opus_int16 *Coefs; +} silk_resampler_state_struct; + +#ifdef __cplusplus +} +#endif +#endif /* SILK_RESAMPLER_STRUCTS_H */ + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/shell_coder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/shell_coder.c new file mode 100644 index 000000000..4af341474 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/shell_coder.c @@ -0,0 +1,151 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* shell coder; pulse-subframe length is hardcoded */ + +static OPUS_INLINE void combine_pulses( + opus_int *out, /* O combined pulses vector [len] */ + const opus_int *in, /* I input vector [2 * len] */ + const opus_int len /* I number of OUTPUT samples */ +) +{ + opus_int k; + for( k = 0; k < len; k++ ) { + out[ k ] = in[ 2 * k ] + in[ 2 * k + 1 ]; + } +} + +static OPUS_INLINE void encode_split( + ec_enc *psRangeEnc, /* I/O compressor data structure */ + const opus_int p_child1, /* I pulse amplitude of first child subframe */ + const opus_int p, /* I pulse amplitude of current subframe */ + const opus_uint8 *shell_table /* I table of shell cdfs */ +) +{ + if( p > 0 ) { + ec_enc_icdf( psRangeEnc, p_child1, &shell_table[ silk_shell_code_table_offsets[ p ] ], 8 ); + } +} + +static OPUS_INLINE void decode_split( + opus_int16 *p_child1, /* O pulse amplitude of first child subframe */ + opus_int16 *p_child2, /* O pulse amplitude of second child subframe */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + const opus_int p, /* I pulse amplitude of current subframe */ + const opus_uint8 *shell_table /* I table of shell cdfs */ +) +{ + if( p > 0 ) { + p_child1[ 0 ] = ec_dec_icdf( psRangeDec, &shell_table[ silk_shell_code_table_offsets[ p ] ], 8 ); + p_child2[ 0 ] = p - p_child1[ 0 ]; + } else { + p_child1[ 0 ] = 0; + p_child2[ 0 ] = 0; + } +} + +/* Shell encoder, operates on one shell code frame of 16 pulses */ +void silk_shell_encoder( + ec_enc *psRangeEnc, /* I/O compressor data structure */ + const opus_int *pulses0 /* I data: nonnegative pulse amplitudes */ +) +{ + opus_int pulses1[ 8 ], pulses2[ 4 ], pulses3[ 2 ], pulses4[ 1 ]; + + /* this function operates on one shell code frame of 16 pulses */ + silk_assert( SHELL_CODEC_FRAME_LENGTH == 16 ); + + /* tree representation per pulse-subframe */ + combine_pulses( pulses1, pulses0, 8 ); + combine_pulses( pulses2, pulses1, 4 ); + combine_pulses( pulses3, pulses2, 2 ); + combine_pulses( pulses4, pulses3, 1 ); + + encode_split( psRangeEnc, pulses3[ 0 ], pulses4[ 0 ], silk_shell_code_table3 ); + + encode_split( psRangeEnc, pulses2[ 0 ], pulses3[ 0 ], silk_shell_code_table2 ); + + encode_split( psRangeEnc, pulses1[ 0 ], pulses2[ 0 ], silk_shell_code_table1 ); + encode_split( psRangeEnc, pulses0[ 0 ], pulses1[ 0 ], silk_shell_code_table0 ); + encode_split( psRangeEnc, pulses0[ 2 ], pulses1[ 1 ], silk_shell_code_table0 ); + + encode_split( psRangeEnc, pulses1[ 2 ], pulses2[ 1 ], silk_shell_code_table1 ); + encode_split( psRangeEnc, pulses0[ 4 ], pulses1[ 2 ], silk_shell_code_table0 ); + encode_split( psRangeEnc, pulses0[ 6 ], pulses1[ 3 ], silk_shell_code_table0 ); + + encode_split( psRangeEnc, pulses2[ 2 ], pulses3[ 1 ], silk_shell_code_table2 ); + + encode_split( psRangeEnc, pulses1[ 4 ], pulses2[ 2 ], silk_shell_code_table1 ); + encode_split( psRangeEnc, pulses0[ 8 ], pulses1[ 4 ], silk_shell_code_table0 ); + encode_split( psRangeEnc, pulses0[ 10 ], pulses1[ 5 ], silk_shell_code_table0 ); + + encode_split( psRangeEnc, pulses1[ 6 ], pulses2[ 3 ], silk_shell_code_table1 ); + encode_split( psRangeEnc, pulses0[ 12 ], pulses1[ 6 ], silk_shell_code_table0 ); + encode_split( psRangeEnc, pulses0[ 14 ], pulses1[ 7 ], silk_shell_code_table0 ); +} + + +/* Shell decoder, operates on one shell code frame of 16 pulses */ +void silk_shell_decoder( + opus_int16 *pulses0, /* O data: nonnegative pulse amplitudes */ + ec_dec *psRangeDec, /* I/O Compressor data structure */ + const opus_int pulses4 /* I number of pulses per pulse-subframe */ +) +{ + opus_int16 pulses3[ 2 ], pulses2[ 4 ], pulses1[ 8 ]; + + /* this function operates on one shell code frame of 16 pulses */ + silk_assert( SHELL_CODEC_FRAME_LENGTH == 16 ); + + decode_split( &pulses3[ 0 ], &pulses3[ 1 ], psRangeDec, pulses4, silk_shell_code_table3 ); + + decode_split( &pulses2[ 0 ], &pulses2[ 1 ], psRangeDec, pulses3[ 0 ], silk_shell_code_table2 ); + + decode_split( &pulses1[ 0 ], &pulses1[ 1 ], psRangeDec, pulses2[ 0 ], silk_shell_code_table1 ); + decode_split( &pulses0[ 0 ], &pulses0[ 1 ], psRangeDec, pulses1[ 0 ], silk_shell_code_table0 ); + decode_split( &pulses0[ 2 ], &pulses0[ 3 ], psRangeDec, pulses1[ 1 ], silk_shell_code_table0 ); + + decode_split( &pulses1[ 2 ], &pulses1[ 3 ], psRangeDec, pulses2[ 1 ], silk_shell_code_table1 ); + decode_split( &pulses0[ 4 ], &pulses0[ 5 ], psRangeDec, pulses1[ 2 ], silk_shell_code_table0 ); + decode_split( &pulses0[ 6 ], &pulses0[ 7 ], psRangeDec, pulses1[ 3 ], silk_shell_code_table0 ); + + decode_split( &pulses2[ 2 ], &pulses2[ 3 ], psRangeDec, pulses3[ 1 ], silk_shell_code_table2 ); + + decode_split( &pulses1[ 4 ], &pulses1[ 5 ], psRangeDec, pulses2[ 2 ], silk_shell_code_table1 ); + decode_split( &pulses0[ 8 ], &pulses0[ 9 ], psRangeDec, pulses1[ 4 ], silk_shell_code_table0 ); + decode_split( &pulses0[ 10 ], &pulses0[ 11 ], psRangeDec, pulses1[ 5 ], silk_shell_code_table0 ); + + decode_split( &pulses1[ 6 ], &pulses1[ 7 ], psRangeDec, pulses2[ 3 ], silk_shell_code_table1 ); + decode_split( &pulses0[ 12 ], &pulses0[ 13 ], psRangeDec, pulses1[ 6 ], silk_shell_code_table0 ); + decode_split( &pulses0[ 14 ], &pulses0[ 15 ], psRangeDec, pulses1[ 7 ], silk_shell_code_table0 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sigm_Q15.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sigm_Q15.c new file mode 100644 index 000000000..3c507d255 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sigm_Q15.c @@ -0,0 +1,76 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Approximate sigmoid function */ + +#include "SigProc_FIX.h" + +/* fprintf(1, '%d, ', round(1024 * ([1 ./ (1 + exp(-(1:5))), 1] - 1 ./ (1 + exp(-(0:5)))))); */ +static const opus_int32 sigm_LUT_slope_Q10[ 6 ] = { + 237, 153, 73, 30, 12, 7 +}; +/* fprintf(1, '%d, ', round(32767 * 1 ./ (1 + exp(-(0:5))))); */ +static const opus_int32 sigm_LUT_pos_Q15[ 6 ] = { + 16384, 23955, 28861, 31213, 32178, 32548 +}; +/* fprintf(1, '%d, ', round(32767 * 1 ./ (1 + exp((0:5))))); */ +static const opus_int32 sigm_LUT_neg_Q15[ 6 ] = { + 16384, 8812, 3906, 1554, 589, 219 +}; + +opus_int silk_sigm_Q15( + opus_int in_Q5 /* I */ +) +{ + opus_int ind; + + if( in_Q5 < 0 ) { + /* Negative input */ + in_Q5 = -in_Q5; + if( in_Q5 >= 6 * 32 ) { + return 0; /* Clip */ + } else { + /* Linear interpolation of look up table */ + ind = silk_RSHIFT( in_Q5, 5 ); + return( sigm_LUT_neg_Q15[ ind ] - silk_SMULBB( sigm_LUT_slope_Q10[ ind ], in_Q5 & 0x1F ) ); + } + } else { + /* Positive input */ + if( in_Q5 >= 6 * 32 ) { + return 32767; /* clip */ + } else { + /* Linear interpolation of look up table */ + ind = silk_RSHIFT( in_Q5, 5 ); + return( sigm_LUT_pos_Q15[ ind ] + silk_SMULBB( sigm_LUT_slope_Q10[ ind ], in_Q5 & 0x1F ) ); + } + } +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sort.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sort.c new file mode 100644 index 000000000..4fba16f83 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sort.c @@ -0,0 +1,154 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Insertion sort (fast for already almost sorted arrays): */ +/* Best case: O(n) for an already sorted array */ +/* Worst case: O(n^2) for an inversely sorted array */ +/* */ +/* Shell short: https://en.wikipedia.org/wiki/Shell_sort */ + +#include "SigProc_FIX.h" + +void silk_insertion_sort_increasing( + opus_int32 *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +) +{ + opus_int32 value; + opus_int i, j; + + /* Safety checks */ + celt_assert( K > 0 ); + celt_assert( L > 0 ); + celt_assert( L >= K ); + + /* Write start indices in index vector */ + for( i = 0; i < K; i++ ) { + idx[ i ] = i; + } + + /* Sort vector elements by value, increasing order */ + for( i = 1; i < K; i++ ) { + value = a[ i ]; + for( j = i - 1; ( j >= 0 ) && ( value < a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + + /* If less than L values are asked for, check the remaining values, */ + /* but only spend CPU to ensure that the K first values are correct */ + for( i = K; i < L; i++ ) { + value = a[ i ]; + if( value < a[ K - 1 ] ) { + for( j = K - 2; ( j >= 0 ) && ( value < a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + } +} + +#ifdef FIXED_POINT +/* This function is only used by the fixed-point build */ +void silk_insertion_sort_decreasing_int16( + opus_int16 *a, /* I/O Unsorted / Sorted vector */ + opus_int *idx, /* O Index vector for the sorted elements */ + const opus_int L, /* I Vector length */ + const opus_int K /* I Number of correctly sorted positions */ +) +{ + opus_int i, j; + opus_int value; + + /* Safety checks */ + celt_assert( K > 0 ); + celt_assert( L > 0 ); + celt_assert( L >= K ); + + /* Write start indices in index vector */ + for( i = 0; i < K; i++ ) { + idx[ i ] = i; + } + + /* Sort vector elements by value, decreasing order */ + for( i = 1; i < K; i++ ) { + value = a[ i ]; + for( j = i - 1; ( j >= 0 ) && ( value > a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + + /* If less than L values are asked for, check the remaining values, */ + /* but only spend CPU to ensure that the K first values are correct */ + for( i = K; i < L; i++ ) { + value = a[ i ]; + if( value > a[ K - 1 ] ) { + for( j = K - 2; ( j >= 0 ) && ( value > a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + idx[ j + 1 ] = idx[ j ]; /* Shift index */ + } + a[ j + 1 ] = value; /* Write value */ + idx[ j + 1 ] = i; /* Write index */ + } + } +} +#endif + +void silk_insertion_sort_increasing_all_values_int16( + opus_int16 *a, /* I/O Unsorted / Sorted vector */ + const opus_int L /* I Vector length */ +) +{ + opus_int value; + opus_int i, j; + + /* Safety checks */ + celt_assert( L > 0 ); + + /* Sort vector elements by value, increasing order */ + for( i = 1; i < L; i++ ) { + value = a[ i ]; + for( j = i - 1; ( j >= 0 ) && ( value < a[ j ] ); j-- ) { + a[ j + 1 ] = a[ j ]; /* Shift value */ + } + a[ j + 1 ] = value; /* Write value */ + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_LR_to_MS.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_LR_to_MS.c new file mode 100644 index 000000000..c8226663c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_LR_to_MS.c @@ -0,0 +1,229 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" +#include "stack_alloc.h" + +/* Convert Left/Right stereo signal to adaptive Mid/Side representation */ +void silk_stereo_LR_to_MS( + stereo_enc_state *state, /* I/O State */ + opus_int16 x1[], /* I/O Left input signal, becomes mid signal */ + opus_int16 x2[], /* I/O Right input signal, becomes side signal */ + opus_int8 ix[ 2 ][ 3 ], /* O Quantization indices */ + opus_int8 *mid_only_flag, /* O Flag: only mid signal coded */ + opus_int32 mid_side_rates_bps[], /* O Bitrates for mid and side signals */ + opus_int32 total_rate_bps, /* I Total bitrate */ + opus_int prev_speech_act_Q8, /* I Speech activity level in previous frame */ + opus_int toMono, /* I Last frame before a stereo->mono transition */ + opus_int fs_kHz, /* I Sample rate (kHz) */ + opus_int frame_length /* I Number of samples */ +) +{ + opus_int n, is10msFrame, denom_Q16, delta0_Q13, delta1_Q13; + opus_int32 sum, diff, smooth_coef_Q16, pred_Q13[ 2 ], pred0_Q13, pred1_Q13; + opus_int32 LP_ratio_Q14, HP_ratio_Q14, frac_Q16, frac_3_Q16, min_mid_rate_bps, width_Q14, w_Q24, deltaw_Q24; + VARDECL( opus_int16, side ); + VARDECL( opus_int16, LP_mid ); + VARDECL( opus_int16, HP_mid ); + VARDECL( opus_int16, LP_side ); + VARDECL( opus_int16, HP_side ); + opus_int16 *mid = &x1[ -2 ]; + SAVE_STACK; + + ALLOC( side, frame_length + 2, opus_int16 ); + /* Convert to basic mid/side signals */ + for( n = 0; n < frame_length + 2; n++ ) { + sum = x1[ n - 2 ] + (opus_int32)x2[ n - 2 ]; + diff = x1[ n - 2 ] - (opus_int32)x2[ n - 2 ]; + mid[ n ] = (opus_int16)silk_RSHIFT_ROUND( sum, 1 ); + side[ n ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( diff, 1 ) ); + } + + /* Buffering */ + silk_memcpy( mid, state->sMid, 2 * sizeof( opus_int16 ) ); + silk_memcpy( side, state->sSide, 2 * sizeof( opus_int16 ) ); + silk_memcpy( state->sMid, &mid[ frame_length ], 2 * sizeof( opus_int16 ) ); + silk_memcpy( state->sSide, &side[ frame_length ], 2 * sizeof( opus_int16 ) ); + + /* LP and HP filter mid signal */ + ALLOC( LP_mid, frame_length, opus_int16 ); + ALLOC( HP_mid, frame_length, opus_int16 ); + for( n = 0; n < frame_length; n++ ) { + sum = silk_RSHIFT_ROUND( silk_ADD_LSHIFT( mid[ n ] + (opus_int32)mid[ n + 2 ], mid[ n + 1 ], 1 ), 2 ); + LP_mid[ n ] = sum; + HP_mid[ n ] = mid[ n + 1 ] - sum; + } + + /* LP and HP filter side signal */ + ALLOC( LP_side, frame_length, opus_int16 ); + ALLOC( HP_side, frame_length, opus_int16 ); + for( n = 0; n < frame_length; n++ ) { + sum = silk_RSHIFT_ROUND( silk_ADD_LSHIFT( side[ n ] + (opus_int32)side[ n + 2 ], side[ n + 1 ], 1 ), 2 ); + LP_side[ n ] = sum; + HP_side[ n ] = side[ n + 1 ] - sum; + } + + /* Find energies and predictors */ + is10msFrame = frame_length == 10 * fs_kHz; + smooth_coef_Q16 = is10msFrame ? + SILK_FIX_CONST( STEREO_RATIO_SMOOTH_COEF / 2, 16 ) : + SILK_FIX_CONST( STEREO_RATIO_SMOOTH_COEF, 16 ); + smooth_coef_Q16 = silk_SMULWB( silk_SMULBB( prev_speech_act_Q8, prev_speech_act_Q8 ), smooth_coef_Q16 ); + + pred_Q13[ 0 ] = silk_stereo_find_predictor( &LP_ratio_Q14, LP_mid, LP_side, &state->mid_side_amp_Q0[ 0 ], frame_length, smooth_coef_Q16 ); + pred_Q13[ 1 ] = silk_stereo_find_predictor( &HP_ratio_Q14, HP_mid, HP_side, &state->mid_side_amp_Q0[ 2 ], frame_length, smooth_coef_Q16 ); + /* Ratio of the norms of residual and mid signals */ + frac_Q16 = silk_SMLABB( HP_ratio_Q14, LP_ratio_Q14, 3 ); + frac_Q16 = silk_min( frac_Q16, SILK_FIX_CONST( 1, 16 ) ); + + /* Determine bitrate distribution between mid and side, and possibly reduce stereo width */ + total_rate_bps -= is10msFrame ? 1200 : 600; /* Subtract approximate bitrate for coding stereo parameters */ + if( total_rate_bps < 1 ) { + total_rate_bps = 1; + } + min_mid_rate_bps = silk_SMLABB( 2000, fs_kHz, 600 ); + silk_assert( min_mid_rate_bps < 32767 ); + /* Default bitrate distribution: 8 parts for Mid and (5+3*frac) parts for Side. so: mid_rate = ( 8 / ( 13 + 3 * frac ) ) * total_ rate */ + frac_3_Q16 = silk_MUL( 3, frac_Q16 ); + mid_side_rates_bps[ 0 ] = silk_DIV32_varQ( total_rate_bps, SILK_FIX_CONST( 8 + 5, 16 ) + frac_3_Q16, 16+3 ); + /* If Mid bitrate below minimum, reduce stereo width */ + if( mid_side_rates_bps[ 0 ] < min_mid_rate_bps ) { + mid_side_rates_bps[ 0 ] = min_mid_rate_bps; + mid_side_rates_bps[ 1 ] = total_rate_bps - mid_side_rates_bps[ 0 ]; + /* width = 4 * ( 2 * side_rate - min_rate ) / ( ( 1 + 3 * frac ) * min_rate ) */ + width_Q14 = silk_DIV32_varQ( silk_LSHIFT( mid_side_rates_bps[ 1 ], 1 ) - min_mid_rate_bps, + silk_SMULWB( SILK_FIX_CONST( 1, 16 ) + frac_3_Q16, min_mid_rate_bps ), 14+2 ); + width_Q14 = silk_LIMIT( width_Q14, 0, SILK_FIX_CONST( 1, 14 ) ); + } else { + mid_side_rates_bps[ 1 ] = total_rate_bps - mid_side_rates_bps[ 0 ]; + width_Q14 = SILK_FIX_CONST( 1, 14 ); + } + + /* Smoother */ + state->smth_width_Q14 = (opus_int16)silk_SMLAWB( state->smth_width_Q14, width_Q14 - state->smth_width_Q14, smooth_coef_Q16 ); + + /* At very low bitrates or for inputs that are nearly amplitude panned, switch to panned-mono coding */ + *mid_only_flag = 0; + if( toMono ) { + /* Last frame before stereo->mono transition; collapse stereo width */ + width_Q14 = 0; + pred_Q13[ 0 ] = 0; + pred_Q13[ 1 ] = 0; + silk_stereo_quant_pred( pred_Q13, ix ); + } else if( state->width_prev_Q14 == 0 && + ( 8 * total_rate_bps < 13 * min_mid_rate_bps || silk_SMULWB( frac_Q16, state->smth_width_Q14 ) < SILK_FIX_CONST( 0.05, 14 ) ) ) + { + /* Code as panned-mono; previous frame already had zero width */ + /* Scale down and quantize predictors */ + pred_Q13[ 0 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 0 ] ), 14 ); + pred_Q13[ 1 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 1 ] ), 14 ); + silk_stereo_quant_pred( pred_Q13, ix ); + /* Collapse stereo width */ + width_Q14 = 0; + pred_Q13[ 0 ] = 0; + pred_Q13[ 1 ] = 0; + mid_side_rates_bps[ 0 ] = total_rate_bps; + mid_side_rates_bps[ 1 ] = 0; + *mid_only_flag = 1; + } else if( state->width_prev_Q14 != 0 && + ( 8 * total_rate_bps < 11 * min_mid_rate_bps || silk_SMULWB( frac_Q16, state->smth_width_Q14 ) < SILK_FIX_CONST( 0.02, 14 ) ) ) + { + /* Transition to zero-width stereo */ + /* Scale down and quantize predictors */ + pred_Q13[ 0 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 0 ] ), 14 ); + pred_Q13[ 1 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 1 ] ), 14 ); + silk_stereo_quant_pred( pred_Q13, ix ); + /* Collapse stereo width */ + width_Q14 = 0; + pred_Q13[ 0 ] = 0; + pred_Q13[ 1 ] = 0; + } else if( state->smth_width_Q14 > SILK_FIX_CONST( 0.95, 14 ) ) { + /* Full-width stereo coding */ + silk_stereo_quant_pred( pred_Q13, ix ); + width_Q14 = SILK_FIX_CONST( 1, 14 ); + } else { + /* Reduced-width stereo coding; scale down and quantize predictors */ + pred_Q13[ 0 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 0 ] ), 14 ); + pred_Q13[ 1 ] = silk_RSHIFT( silk_SMULBB( state->smth_width_Q14, pred_Q13[ 1 ] ), 14 ); + silk_stereo_quant_pred( pred_Q13, ix ); + width_Q14 = state->smth_width_Q14; + } + + /* Make sure to keep on encoding until the tapered output has been transmitted */ + if( *mid_only_flag == 1 ) { + state->silent_side_len += frame_length - STEREO_INTERP_LEN_MS * fs_kHz; + if( state->silent_side_len < LA_SHAPE_MS * fs_kHz ) { + *mid_only_flag = 0; + } else { + /* Limit to avoid wrapping around */ + state->silent_side_len = 10000; + } + } else { + state->silent_side_len = 0; + } + + if( *mid_only_flag == 0 && mid_side_rates_bps[ 1 ] < 1 ) { + mid_side_rates_bps[ 1 ] = 1; + mid_side_rates_bps[ 0 ] = silk_max_int( 1, total_rate_bps - mid_side_rates_bps[ 1 ]); + } + + /* Interpolate predictors and subtract prediction from side channel */ + pred0_Q13 = -state->pred_prev_Q13[ 0 ]; + pred1_Q13 = -state->pred_prev_Q13[ 1 ]; + w_Q24 = silk_LSHIFT( state->width_prev_Q14, 10 ); + denom_Q16 = silk_DIV32_16( (opus_int32)1 << 16, STEREO_INTERP_LEN_MS * fs_kHz ); + delta0_Q13 = -silk_RSHIFT_ROUND( silk_SMULBB( pred_Q13[ 0 ] - state->pred_prev_Q13[ 0 ], denom_Q16 ), 16 ); + delta1_Q13 = -silk_RSHIFT_ROUND( silk_SMULBB( pred_Q13[ 1 ] - state->pred_prev_Q13[ 1 ], denom_Q16 ), 16 ); + deltaw_Q24 = silk_LSHIFT( silk_SMULWB( width_Q14 - state->width_prev_Q14, denom_Q16 ), 10 ); + for( n = 0; n < STEREO_INTERP_LEN_MS * fs_kHz; n++ ) { + pred0_Q13 += delta0_Q13; + pred1_Q13 += delta1_Q13; + w_Q24 += deltaw_Q24; + sum = silk_LSHIFT( silk_ADD_LSHIFT( mid[ n ] + (opus_int32)mid[ n + 2 ], mid[ n + 1 ], 1 ), 9 ); /* Q11 */ + sum = silk_SMLAWB( silk_SMULWB( w_Q24, side[ n + 1 ] ), sum, pred0_Q13 ); /* Q8 */ + sum = silk_SMLAWB( sum, silk_LSHIFT( (opus_int32)mid[ n + 1 ], 11 ), pred1_Q13 ); /* Q8 */ + x2[ n - 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sum, 8 ) ); + } + + pred0_Q13 = -pred_Q13[ 0 ]; + pred1_Q13 = -pred_Q13[ 1 ]; + w_Q24 = silk_LSHIFT( width_Q14, 10 ); + for( n = STEREO_INTERP_LEN_MS * fs_kHz; n < frame_length; n++ ) { + sum = silk_LSHIFT( silk_ADD_LSHIFT( mid[ n ] + (opus_int32)mid[ n + 2 ], mid[ n + 1 ], 1 ), 9 ); /* Q11 */ + sum = silk_SMLAWB( silk_SMULWB( w_Q24, side[ n + 1 ] ), sum, pred0_Q13 ); /* Q8 */ + sum = silk_SMLAWB( sum, silk_LSHIFT( (opus_int32)mid[ n + 1 ], 11 ), pred1_Q13 ); /* Q8 */ + x2[ n - 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sum, 8 ) ); + } + state->pred_prev_Q13[ 0 ] = (opus_int16)pred_Q13[ 0 ]; + state->pred_prev_Q13[ 1 ] = (opus_int16)pred_Q13[ 1 ]; + state->width_prev_Q14 = (opus_int16)width_Q14; + RESTORE_STACK; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_MS_to_LR.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_MS_to_LR.c new file mode 100644 index 000000000..62521a4f3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_MS_to_LR.c @@ -0,0 +1,85 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Convert adaptive Mid/Side representation to Left/Right stereo signal */ +void silk_stereo_MS_to_LR( + stereo_dec_state *state, /* I/O State */ + opus_int16 x1[], /* I/O Left input signal, becomes mid signal */ + opus_int16 x2[], /* I/O Right input signal, becomes side signal */ + const opus_int32 pred_Q13[], /* I Predictors */ + opus_int fs_kHz, /* I Samples rate (kHz) */ + opus_int frame_length /* I Number of samples */ +) +{ + opus_int n, denom_Q16, delta0_Q13, delta1_Q13; + opus_int32 sum, diff, pred0_Q13, pred1_Q13; + + /* Buffering */ + silk_memcpy( x1, state->sMid, 2 * sizeof( opus_int16 ) ); + silk_memcpy( x2, state->sSide, 2 * sizeof( opus_int16 ) ); + silk_memcpy( state->sMid, &x1[ frame_length ], 2 * sizeof( opus_int16 ) ); + silk_memcpy( state->sSide, &x2[ frame_length ], 2 * sizeof( opus_int16 ) ); + + /* Interpolate predictors and add prediction to side channel */ + pred0_Q13 = state->pred_prev_Q13[ 0 ]; + pred1_Q13 = state->pred_prev_Q13[ 1 ]; + denom_Q16 = silk_DIV32_16( (opus_int32)1 << 16, STEREO_INTERP_LEN_MS * fs_kHz ); + delta0_Q13 = silk_RSHIFT_ROUND( silk_SMULBB( pred_Q13[ 0 ] - state->pred_prev_Q13[ 0 ], denom_Q16 ), 16 ); + delta1_Q13 = silk_RSHIFT_ROUND( silk_SMULBB( pred_Q13[ 1 ] - state->pred_prev_Q13[ 1 ], denom_Q16 ), 16 ); + for( n = 0; n < STEREO_INTERP_LEN_MS * fs_kHz; n++ ) { + pred0_Q13 += delta0_Q13; + pred1_Q13 += delta1_Q13; + sum = silk_LSHIFT( silk_ADD_LSHIFT( x1[ n ] + x1[ n + 2 ], x1[ n + 1 ], 1 ), 9 ); /* Q11 */ + sum = silk_SMLAWB( silk_LSHIFT( (opus_int32)x2[ n + 1 ], 8 ), sum, pred0_Q13 ); /* Q8 */ + sum = silk_SMLAWB( sum, silk_LSHIFT( (opus_int32)x1[ n + 1 ], 11 ), pred1_Q13 ); /* Q8 */ + x2[ n + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sum, 8 ) ); + } + pred0_Q13 = pred_Q13[ 0 ]; + pred1_Q13 = pred_Q13[ 1 ]; + for( n = STEREO_INTERP_LEN_MS * fs_kHz; n < frame_length; n++ ) { + sum = silk_LSHIFT( silk_ADD_LSHIFT( x1[ n ] + x1[ n + 2 ], x1[ n + 1 ], 1 ), 9 ); /* Q11 */ + sum = silk_SMLAWB( silk_LSHIFT( (opus_int32)x2[ n + 1 ], 8 ), sum, pred0_Q13 ); /* Q8 */ + sum = silk_SMLAWB( sum, silk_LSHIFT( (opus_int32)x1[ n + 1 ], 11 ), pred1_Q13 ); /* Q8 */ + x2[ n + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( sum, 8 ) ); + } + state->pred_prev_Q13[ 0 ] = pred_Q13[ 0 ]; + state->pred_prev_Q13[ 1 ] = pred_Q13[ 1 ]; + + /* Convert to left/right signals */ + for( n = 0; n < frame_length; n++ ) { + sum = x1[ n + 1 ] + (opus_int32)x2[ n + 1 ]; + diff = x1[ n + 1 ] - (opus_int32)x2[ n + 1 ]; + x1[ n + 1 ] = (opus_int16)silk_SAT16( sum ); + x2[ n + 1 ] = (opus_int16)silk_SAT16( diff ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_decode_pred.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_decode_pred.c new file mode 100644 index 000000000..56ba3925e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_decode_pred.c @@ -0,0 +1,73 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Decode mid/side predictors */ +void silk_stereo_decode_pred( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int32 pred_Q13[] /* O Predictors */ +) +{ + opus_int n, ix[ 2 ][ 3 ]; + opus_int32 low_Q13, step_Q13; + + /* Entropy decoding */ + n = ec_dec_icdf( psRangeDec, silk_stereo_pred_joint_iCDF, 8 ); + ix[ 0 ][ 2 ] = silk_DIV32_16( n, 5 ); + ix[ 1 ][ 2 ] = n - 5 * ix[ 0 ][ 2 ]; + for( n = 0; n < 2; n++ ) { + ix[ n ][ 0 ] = ec_dec_icdf( psRangeDec, silk_uniform3_iCDF, 8 ); + ix[ n ][ 1 ] = ec_dec_icdf( psRangeDec, silk_uniform5_iCDF, 8 ); + } + + /* Dequantize */ + for( n = 0; n < 2; n++ ) { + ix[ n ][ 0 ] += 3 * ix[ n ][ 2 ]; + low_Q13 = silk_stereo_pred_quant_Q13[ ix[ n ][ 0 ] ]; + step_Q13 = silk_SMULWB( silk_stereo_pred_quant_Q13[ ix[ n ][ 0 ] + 1 ] - low_Q13, + SILK_FIX_CONST( 0.5 / STEREO_QUANT_SUB_STEPS, 16 ) ); + pred_Q13[ n ] = silk_SMLABB( low_Q13, step_Q13, 2 * ix[ n ][ 1 ] + 1 ); + } + + /* Subtract second from first predictor (helps when actually applying these) */ + pred_Q13[ 0 ] -= pred_Q13[ 1 ]; +} + +/* Decode mid-only flag */ +void silk_stereo_decode_mid_only( + ec_dec *psRangeDec, /* I/O Compressor data structure */ + opus_int *decode_only_mid /* O Flag that only mid channel has been coded */ +) +{ + /* Decode flag that only mid channel is coded */ + *decode_only_mid = ec_dec_icdf( psRangeDec, silk_stereo_only_code_mid_iCDF, 8 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_encode_pred.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_encode_pred.c new file mode 100644 index 000000000..03becb673 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_encode_pred.c @@ -0,0 +1,62 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Entropy code the mid/side quantization indices */ +void silk_stereo_encode_pred( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int8 ix[ 2 ][ 3 ] /* I Quantization indices */ +) +{ + opus_int n; + + /* Entropy coding */ + n = 5 * ix[ 0 ][ 2 ] + ix[ 1 ][ 2 ]; + celt_assert( n < 25 ); + ec_enc_icdf( psRangeEnc, n, silk_stereo_pred_joint_iCDF, 8 ); + for( n = 0; n < 2; n++ ) { + celt_assert( ix[ n ][ 0 ] < 3 ); + celt_assert( ix[ n ][ 1 ] < STEREO_QUANT_SUB_STEPS ); + ec_enc_icdf( psRangeEnc, ix[ n ][ 0 ], silk_uniform3_iCDF, 8 ); + ec_enc_icdf( psRangeEnc, ix[ n ][ 1 ], silk_uniform5_iCDF, 8 ); + } +} + +/* Entropy code the mid-only flag */ +void silk_stereo_encode_mid_only( + ec_enc *psRangeEnc, /* I/O Compressor data structure */ + opus_int8 mid_only_flag +) +{ + /* Encode flag that only mid channel is coded */ + ec_enc_icdf( psRangeEnc, mid_only_flag, silk_stereo_only_code_mid_iCDF, 8 ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_find_predictor.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_find_predictor.c new file mode 100644 index 000000000..e30e90bdd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_find_predictor.c @@ -0,0 +1,79 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Find least-squares prediction gain for one signal based on another and quantize it */ +opus_int32 silk_stereo_find_predictor( /* O Returns predictor in Q13 */ + opus_int32 *ratio_Q14, /* O Ratio of residual and mid energies */ + const opus_int16 x[], /* I Basis signal */ + const opus_int16 y[], /* I Target signal */ + opus_int32 mid_res_amp_Q0[], /* I/O Smoothed mid, residual norms */ + opus_int length, /* I Number of samples */ + opus_int smooth_coef_Q16 /* I Smoothing coefficient */ +) +{ + opus_int scale, scale1, scale2; + opus_int32 nrgx, nrgy, corr, pred_Q13, pred2_Q10; + + /* Find predictor */ + silk_sum_sqr_shift( &nrgx, &scale1, x, length ); + silk_sum_sqr_shift( &nrgy, &scale2, y, length ); + scale = silk_max_int( scale1, scale2 ); + scale = scale + ( scale & 1 ); /* make even */ + nrgy = silk_RSHIFT32( nrgy, scale - scale2 ); + nrgx = silk_RSHIFT32( nrgx, scale - scale1 ); + nrgx = silk_max_int( nrgx, 1 ); + corr = silk_inner_prod_aligned_scale( x, y, scale, length ); + pred_Q13 = silk_DIV32_varQ( corr, nrgx, 13 ); + pred_Q13 = silk_LIMIT( pred_Q13, -(1 << 14), 1 << 14 ); + pred2_Q10 = silk_SMULWB( pred_Q13, pred_Q13 ); + + /* Faster update for signals with large prediction parameters */ + smooth_coef_Q16 = (opus_int)silk_max_int( smooth_coef_Q16, silk_abs( pred2_Q10 ) ); + + /* Smoothed mid and residual norms */ + silk_assert( smooth_coef_Q16 < 32768 ); + scale = silk_RSHIFT( scale, 1 ); + mid_res_amp_Q0[ 0 ] = silk_SMLAWB( mid_res_amp_Q0[ 0 ], silk_LSHIFT( silk_SQRT_APPROX( nrgx ), scale ) - mid_res_amp_Q0[ 0 ], + smooth_coef_Q16 ); + /* Residual energy = nrgy - 2 * pred * corr + pred^2 * nrgx */ + nrgy = silk_SUB_LSHIFT32( nrgy, silk_SMULWB( corr, pred_Q13 ), 3 + 1 ); + nrgy = silk_ADD_LSHIFT32( nrgy, silk_SMULWB( nrgx, pred2_Q10 ), 6 ); + mid_res_amp_Q0[ 1 ] = silk_SMLAWB( mid_res_amp_Q0[ 1 ], silk_LSHIFT( silk_SQRT_APPROX( nrgy ), scale ) - mid_res_amp_Q0[ 1 ], + smooth_coef_Q16 ); + + /* Ratio of smoothed residual and mid norms */ + *ratio_Q14 = silk_DIV32_varQ( mid_res_amp_Q0[ 1 ], silk_max( mid_res_amp_Q0[ 0 ], 1 ), 14 ); + *ratio_Q14 = silk_LIMIT( *ratio_Q14, 0, 32767 ); + + return pred_Q13; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_quant_pred.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_quant_pred.c new file mode 100644 index 000000000..d4ced6c3e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/stereo_quant_pred.c @@ -0,0 +1,73 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "main.h" + +/* Quantize mid/side predictors */ +void silk_stereo_quant_pred( + opus_int32 pred_Q13[], /* I/O Predictors (out: quantized) */ + opus_int8 ix[ 2 ][ 3 ] /* O Quantization indices */ +) +{ + opus_int i, j, n; + opus_int32 low_Q13, step_Q13, lvl_Q13, err_min_Q13, err_Q13, quant_pred_Q13 = 0; + + /* Quantize */ + for( n = 0; n < 2; n++ ) { + /* Brute-force search over quantization levels */ + err_min_Q13 = silk_int32_MAX; + for( i = 0; i < STEREO_QUANT_TAB_SIZE - 1; i++ ) { + low_Q13 = silk_stereo_pred_quant_Q13[ i ]; + step_Q13 = silk_SMULWB( silk_stereo_pred_quant_Q13[ i + 1 ] - low_Q13, + SILK_FIX_CONST( 0.5 / STEREO_QUANT_SUB_STEPS, 16 ) ); + for( j = 0; j < STEREO_QUANT_SUB_STEPS; j++ ) { + lvl_Q13 = silk_SMLABB( low_Q13, step_Q13, 2 * j + 1 ); + err_Q13 = silk_abs( pred_Q13[ n ] - lvl_Q13 ); + if( err_Q13 < err_min_Q13 ) { + err_min_Q13 = err_Q13; + quant_pred_Q13 = lvl_Q13; + ix[ n ][ 0 ] = i; + ix[ n ][ 1 ] = j; + } else { + /* Error increasing, so we're past the optimum */ + goto done; + } + } + } + done: + ix[ n ][ 2 ] = silk_DIV32_16( ix[ n ][ 0 ], 3 ); + ix[ n ][ 0 ] -= ix[ n ][ 2 ] * 3; + pred_Q13[ n ] = quant_pred_Q13; + } + + /* Subtract second from first predictor (helps when actually applying these) */ + pred_Q13[ 0 ] -= pred_Q13[ 1 ]; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/structs.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/structs.h new file mode 100644 index 000000000..3380c757b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/structs.h @@ -0,0 +1,329 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_STRUCTS_H +#define SILK_STRUCTS_H + +#include "typedef.h" +#include "SigProc_FIX.h" +#include "define.h" +#include "entenc.h" +#include "entdec.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/************************************/ +/* Noise shaping quantization state */ +/************************************/ +typedef struct { + opus_int16 xq[ 2 * MAX_FRAME_LENGTH ]; /* Buffer for quantized output signal */ + opus_int32 sLTP_shp_Q14[ 2 * MAX_FRAME_LENGTH ]; + opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ]; + opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 sLF_AR_shp_Q14; + opus_int32 sDiff_shp_Q14; + opus_int lagPrev; + opus_int sLTP_buf_idx; + opus_int sLTP_shp_buf_idx; + opus_int32 rand_seed; + opus_int32 prev_gain_Q16; + opus_int rewhite_flag; +} silk_nsq_state; + +/********************************/ +/* VAD state */ +/********************************/ +typedef struct { + opus_int32 AnaState[ 2 ]; /* Analysis filterbank state: 0-8 kHz */ + opus_int32 AnaState1[ 2 ]; /* Analysis filterbank state: 0-4 kHz */ + opus_int32 AnaState2[ 2 ]; /* Analysis filterbank state: 0-2 kHz */ + opus_int32 XnrgSubfr[ VAD_N_BANDS ]; /* Subframe energies */ + opus_int32 NrgRatioSmth_Q8[ VAD_N_BANDS ]; /* Smoothed energy level in each band */ + opus_int16 HPstate; /* State of differentiator in the lowest band */ + opus_int32 NL[ VAD_N_BANDS ]; /* Noise energy level in each band */ + opus_int32 inv_NL[ VAD_N_BANDS ]; /* Inverse noise energy level in each band */ + opus_int32 NoiseLevelBias[ VAD_N_BANDS ]; /* Noise level estimator bias/offset */ + opus_int32 counter; /* Frame counter used in the initial phase */ +} silk_VAD_state; + +/* Variable cut-off low-pass filter state */ +typedef struct { + opus_int32 In_LP_State[ 2 ]; /* Low pass filter state */ + opus_int32 transition_frame_no; /* Counter which is mapped to a cut-off frequency */ + opus_int mode; /* Operating mode, <0: switch down, >0: switch up; 0: do nothing */ + opus_int32 saved_fs_kHz; /* If non-zero, holds the last sampling rate before a bandwidth switching reset. */ +} silk_LP_state; + +/* Structure containing NLSF codebook */ +typedef struct { + const opus_int16 nVectors; + const opus_int16 order; + const opus_int16 quantStepSize_Q16; + const opus_int16 invQuantStepSize_Q6; + const opus_uint8 *CB1_NLSF_Q8; + const opus_int16 *CB1_Wght_Q9; + const opus_uint8 *CB1_iCDF; + const opus_uint8 *pred_Q8; + const opus_uint8 *ec_sel; + const opus_uint8 *ec_iCDF; + const opus_uint8 *ec_Rates_Q5; + const opus_int16 *deltaMin_Q15; +} silk_NLSF_CB_struct; + +typedef struct { + opus_int16 pred_prev_Q13[ 2 ]; + opus_int16 sMid[ 2 ]; + opus_int16 sSide[ 2 ]; + opus_int32 mid_side_amp_Q0[ 4 ]; + opus_int16 smth_width_Q14; + opus_int16 width_prev_Q14; + opus_int16 silent_side_len; + opus_int8 predIx[ MAX_FRAMES_PER_PACKET ][ 2 ][ 3 ]; + opus_int8 mid_only_flags[ MAX_FRAMES_PER_PACKET ]; +} stereo_enc_state; + +typedef struct { + opus_int16 pred_prev_Q13[ 2 ]; + opus_int16 sMid[ 2 ]; + opus_int16 sSide[ 2 ]; +} stereo_dec_state; + +typedef struct { + opus_int8 GainsIndices[ MAX_NB_SUBFR ]; + opus_int8 LTPIndex[ MAX_NB_SUBFR ]; + opus_int8 NLSFIndices[ MAX_LPC_ORDER + 1 ]; + opus_int16 lagIndex; + opus_int8 contourIndex; + opus_int8 signalType; + opus_int8 quantOffsetType; + opus_int8 NLSFInterpCoef_Q2; + opus_int8 PERIndex; + opus_int8 LTP_scaleIndex; + opus_int8 Seed; +} SideInfoIndices; + +/********************************/ +/* Encoder state */ +/********************************/ +typedef struct { + opus_int32 In_HP_State[ 2 ]; /* High pass filter state */ + opus_int32 variable_HP_smth1_Q15; /* State of first smoother */ + opus_int32 variable_HP_smth2_Q15; /* State of second smoother */ + silk_LP_state sLP; /* Low pass filter state */ + silk_VAD_state sVAD; /* Voice activity detector state */ + silk_nsq_state sNSQ; /* Noise Shape Quantizer State */ + opus_int16 prev_NLSFq_Q15[ MAX_LPC_ORDER ]; /* Previously quantized NLSF vector */ + opus_int speech_activity_Q8; /* Speech activity */ + opus_int allow_bandwidth_switch; /* Flag indicating that switching of internal bandwidth is allowed */ + opus_int8 LBRRprevLastGainIndex; + opus_int8 prevSignalType; + opus_int prevLag; + opus_int pitch_LPC_win_length; + opus_int max_pitch_lag; /* Highest possible pitch lag (samples) */ + opus_int32 API_fs_Hz; /* API sampling frequency (Hz) */ + opus_int32 prev_API_fs_Hz; /* Previous API sampling frequency (Hz) */ + opus_int maxInternal_fs_Hz; /* Maximum internal sampling frequency (Hz) */ + opus_int minInternal_fs_Hz; /* Minimum internal sampling frequency (Hz) */ + opus_int desiredInternal_fs_Hz; /* Soft request for internal sampling frequency (Hz) */ + opus_int fs_kHz; /* Internal sampling frequency (kHz) */ + opus_int nb_subfr; /* Number of 5 ms subframes in a frame */ + opus_int frame_length; /* Frame length (samples) */ + opus_int subfr_length; /* Subframe length (samples) */ + opus_int ltp_mem_length; /* Length of LTP memory */ + opus_int la_pitch; /* Look-ahead for pitch analysis (samples) */ + opus_int la_shape; /* Look-ahead for noise shape analysis (samples) */ + opus_int shapeWinLength; /* Window length for noise shape analysis (samples) */ + opus_int32 TargetRate_bps; /* Target bitrate (bps) */ + opus_int PacketSize_ms; /* Number of milliseconds to put in each packet */ + opus_int PacketLoss_perc; /* Packet loss rate measured by farend */ + opus_int32 frameCounter; + opus_int Complexity; /* Complexity setting */ + opus_int nStatesDelayedDecision; /* Number of states in delayed decision quantization */ + opus_int useInterpolatedNLSFs; /* Flag for using NLSF interpolation */ + opus_int shapingLPCOrder; /* Filter order for noise shaping filters */ + opus_int predictLPCOrder; /* Filter order for prediction filters */ + opus_int pitchEstimationComplexity; /* Complexity level for pitch estimator */ + opus_int pitchEstimationLPCOrder; /* Whitening filter order for pitch estimator */ + opus_int32 pitchEstimationThreshold_Q16; /* Threshold for pitch estimator */ + opus_int32 sum_log_gain_Q7; /* Cumulative max prediction gain */ + opus_int NLSF_MSVQ_Survivors; /* Number of survivors in NLSF MSVQ */ + opus_int first_frame_after_reset; /* Flag for deactivating NLSF interpolation, pitch prediction */ + opus_int controlled_since_last_payload; /* Flag for ensuring codec_control only runs once per packet */ + opus_int warping_Q16; /* Warping parameter for warped noise shaping */ + opus_int useCBR; /* Flag to enable constant bitrate */ + opus_int prefillFlag; /* Flag to indicate that only buffers are prefilled, no coding */ + const opus_uint8 *pitch_lag_low_bits_iCDF; /* Pointer to iCDF table for low bits of pitch lag index */ + const opus_uint8 *pitch_contour_iCDF; /* Pointer to iCDF table for pitch contour index */ + const silk_NLSF_CB_struct *psNLSF_CB; /* Pointer to NLSF codebook */ + opus_int input_quality_bands_Q15[ VAD_N_BANDS ]; + opus_int input_tilt_Q15; + opus_int SNR_dB_Q7; /* Quality setting */ + + opus_int8 VAD_flags[ MAX_FRAMES_PER_PACKET ]; + opus_int8 LBRR_flag; + opus_int LBRR_flags[ MAX_FRAMES_PER_PACKET ]; + + SideInfoIndices indices; + opus_int8 pulses[ MAX_FRAME_LENGTH ]; + + int arch; + + /* Input/output buffering */ + opus_int16 inputBuf[ MAX_FRAME_LENGTH + 2 ]; /* Buffer containing input signal */ + opus_int inputBufIx; + opus_int nFramesPerPacket; + opus_int nFramesEncoded; /* Number of frames analyzed in current packet */ + + opus_int nChannelsAPI; + opus_int nChannelsInternal; + opus_int channelNb; + + /* Parameters For LTP scaling Control */ + opus_int frames_since_onset; + + /* Specifically for entropy coding */ + opus_int ec_prevSignalType; + opus_int16 ec_prevLagIndex; + + silk_resampler_state_struct resampler_state; + + /* DTX */ + opus_int useDTX; /* Flag to enable DTX */ + opus_int inDTX; /* Flag to signal DTX period */ + opus_int noSpeechCounter; /* Counts concecutive nonactive frames, used by DTX */ + + /* Inband Low Bitrate Redundancy (LBRR) data */ + opus_int useInBandFEC; /* Saves the API setting for query */ + opus_int LBRR_enabled; /* Depends on useInBandFRC, bitrate and packet loss rate */ + opus_int LBRR_GainIncreases; /* Gains increment for coding LBRR frames */ + SideInfoIndices indices_LBRR[ MAX_FRAMES_PER_PACKET ]; + opus_int8 pulses_LBRR[ MAX_FRAMES_PER_PACKET ][ MAX_FRAME_LENGTH ]; +} silk_encoder_state; + + +/* Struct for Packet Loss Concealment */ +typedef struct { + opus_int32 pitchL_Q8; /* Pitch lag to use for voiced concealment */ + opus_int16 LTPCoef_Q14[ LTP_ORDER ]; /* LTP coeficients to use for voiced concealment */ + opus_int16 prevLPC_Q12[ MAX_LPC_ORDER ]; + opus_int last_frame_lost; /* Was previous frame lost */ + opus_int32 rand_seed; /* Seed for unvoiced signal generation */ + opus_int16 randScale_Q14; /* Scaling of unvoiced random signal */ + opus_int32 conc_energy; + opus_int conc_energy_shift; + opus_int16 prevLTP_scale_Q14; + opus_int32 prevGain_Q16[ 2 ]; + opus_int fs_kHz; + opus_int nb_subfr; + opus_int subfr_length; +} silk_PLC_struct; + +/* Struct for CNG */ +typedef struct { + opus_int32 CNG_exc_buf_Q14[ MAX_FRAME_LENGTH ]; + opus_int16 CNG_smth_NLSF_Q15[ MAX_LPC_ORDER ]; + opus_int32 CNG_synth_state[ MAX_LPC_ORDER ]; + opus_int32 CNG_smth_Gain_Q16; + opus_int32 rand_seed; + opus_int fs_kHz; +} silk_CNG_struct; + +/********************************/ +/* Decoder state */ +/********************************/ +typedef struct { + opus_int32 prev_gain_Q16; + opus_int32 exc_Q14[ MAX_FRAME_LENGTH ]; + opus_int32 sLPC_Q14_buf[ MAX_LPC_ORDER ]; + opus_int16 outBuf[ MAX_FRAME_LENGTH + 2 * MAX_SUB_FRAME_LENGTH ]; /* Buffer for output signal */ + opus_int lagPrev; /* Previous Lag */ + opus_int8 LastGainIndex; /* Previous gain index */ + opus_int fs_kHz; /* Sampling frequency in kHz */ + opus_int32 fs_API_hz; /* API sample frequency (Hz) */ + opus_int nb_subfr; /* Number of 5 ms subframes in a frame */ + opus_int frame_length; /* Frame length (samples) */ + opus_int subfr_length; /* Subframe length (samples) */ + opus_int ltp_mem_length; /* Length of LTP memory */ + opus_int LPC_order; /* LPC order */ + opus_int16 prevNLSF_Q15[ MAX_LPC_ORDER ]; /* Used to interpolate LSFs */ + opus_int first_frame_after_reset; /* Flag for deactivating NLSF interpolation */ + const opus_uint8 *pitch_lag_low_bits_iCDF; /* Pointer to iCDF table for low bits of pitch lag index */ + const opus_uint8 *pitch_contour_iCDF; /* Pointer to iCDF table for pitch contour index */ + + /* For buffering payload in case of more frames per packet */ + opus_int nFramesDecoded; + opus_int nFramesPerPacket; + + /* Specifically for entropy coding */ + opus_int ec_prevSignalType; + opus_int16 ec_prevLagIndex; + + opus_int VAD_flags[ MAX_FRAMES_PER_PACKET ]; + opus_int LBRR_flag; + opus_int LBRR_flags[ MAX_FRAMES_PER_PACKET ]; + + silk_resampler_state_struct resampler_state; + + const silk_NLSF_CB_struct *psNLSF_CB; /* Pointer to NLSF codebook */ + + /* Quantization indices */ + SideInfoIndices indices; + + /* CNG state */ + silk_CNG_struct sCNG; + + /* Stuff used for PLC */ + opus_int lossCnt; + opus_int prevSignalType; + int arch; + + silk_PLC_struct sPLC; + +} silk_decoder_state; + +/************************/ +/* Decoder control */ +/************************/ +typedef struct { + /* Prediction and coding parameters */ + opus_int pitchL[ MAX_NB_SUBFR ]; + opus_int32 Gains_Q16[ MAX_NB_SUBFR ]; + /* Holds interpolated and final coefficients, 4-byte aligned */ + silk_DWORD_ALIGN opus_int16 PredCoef_Q12[ 2 ][ MAX_LPC_ORDER ]; + opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ]; + opus_int LTP_scale_Q14; +} silk_decoder_control; + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sum_sqr_shift.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sum_sqr_shift.c new file mode 100644 index 000000000..4fd0c3d7d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/sum_sqr_shift.c @@ -0,0 +1,83 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "SigProc_FIX.h" + +/* Compute number of bits to right shift the sum of squares of a vector */ +/* of int16s to make it fit in an int32 */ +void silk_sum_sqr_shift( + opus_int32 *energy, /* O Energy of x, after shifting to the right */ + opus_int *shift, /* O Number of bits right shift applied to energy */ + const opus_int16 *x, /* I Input vector */ + opus_int len /* I Length of input vector */ +) +{ + opus_int i, shft; + opus_uint32 nrg_tmp; + opus_int32 nrg; + + /* Do a first run with the maximum shift we could have. */ + shft = 31-silk_CLZ32(len); + /* Let's be conservative with rounding and start with nrg=len. */ + nrg = len; + for( i = 0; i < len - 1; i += 2 ) { + nrg_tmp = silk_SMULBB( x[ i ], x[ i ] ); + nrg_tmp = silk_SMLABB_ovflw( nrg_tmp, x[ i + 1 ], x[ i + 1 ] ); + nrg = (opus_int32)silk_ADD_RSHIFT_uint( nrg, nrg_tmp, shft ); + } + if( i < len ) { + /* One sample left to process */ + nrg_tmp = silk_SMULBB( x[ i ], x[ i ] ); + nrg = (opus_int32)silk_ADD_RSHIFT_uint( nrg, nrg_tmp, shft ); + } + silk_assert( nrg >= 0 ); + /* Make sure the result will fit in a 32-bit signed integer with two bits + of headroom. */ + shft = silk_max_32(0, shft+3 - silk_CLZ32(nrg)); + nrg = 0; + for( i = 0 ; i < len - 1; i += 2 ) { + nrg_tmp = silk_SMULBB( x[ i ], x[ i ] ); + nrg_tmp = silk_SMLABB_ovflw( nrg_tmp, x[ i + 1 ], x[ i + 1 ] ); + nrg = (opus_int32)silk_ADD_RSHIFT_uint( nrg, nrg_tmp, shft ); + } + if( i < len ) { + /* One sample left to process */ + nrg_tmp = silk_SMULBB( x[ i ], x[ i ] ); + nrg = (opus_int32)silk_ADD_RSHIFT_uint( nrg, nrg_tmp, shft ); + } + + silk_assert( nrg >= 0 ); + + /* Output arguments */ + *shift = shft; + *energy = nrg; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/table_LSF_cos.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/table_LSF_cos.c new file mode 100644 index 000000000..ec9dc6392 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/table_LSF_cos.c @@ -0,0 +1,70 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +/* Cosine approximation table for LSF conversion */ +/* Q12 values (even) */ +const opus_int16 silk_LSFCosTab_FIX_Q12[ LSF_COS_TAB_SZ_FIX + 1 ] = { + 8192, 8190, 8182, 8170, + 8152, 8130, 8104, 8072, + 8034, 7994, 7946, 7896, + 7840, 7778, 7714, 7644, + 7568, 7490, 7406, 7318, + 7226, 7128, 7026, 6922, + 6812, 6698, 6580, 6458, + 6332, 6204, 6070, 5934, + 5792, 5648, 5502, 5352, + 5198, 5040, 4880, 4718, + 4552, 4382, 4212, 4038, + 3862, 3684, 3502, 3320, + 3136, 2948, 2760, 2570, + 2378, 2186, 1990, 1794, + 1598, 1400, 1202, 1002, + 802, 602, 402, 202, + 0, -202, -402, -602, + -802, -1002, -1202, -1400, + -1598, -1794, -1990, -2186, + -2378, -2570, -2760, -2948, + -3136, -3320, -3502, -3684, + -3862, -4038, -4212, -4382, + -4552, -4718, -4880, -5040, + -5198, -5352, -5502, -5648, + -5792, -5934, -6070, -6204, + -6332, -6458, -6580, -6698, + -6812, -6922, -7026, -7128, + -7226, -7318, -7406, -7490, + -7568, -7644, -7714, -7778, + -7840, -7896, -7946, -7994, + -8034, -8072, -8104, -8130, + -8152, -8170, -8182, -8190, + -8192 +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables.h new file mode 100644 index 000000000..95230c451 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables.h @@ -0,0 +1,114 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_TABLES_H +#define SILK_TABLES_H + +#include "define.h" +#include "structs.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Entropy coding tables (with size in bytes indicated) */ +extern const opus_uint8 silk_gain_iCDF[ 3 ][ N_LEVELS_QGAIN / 8 ]; /* 24 */ +extern const opus_uint8 silk_delta_gain_iCDF[ MAX_DELTA_GAIN_QUANT - MIN_DELTA_GAIN_QUANT + 1 ]; /* 41 */ + +extern const opus_uint8 silk_pitch_lag_iCDF[ 2 * ( PITCH_EST_MAX_LAG_MS - PITCH_EST_MIN_LAG_MS ) ];/* 32 */ +extern const opus_uint8 silk_pitch_delta_iCDF[ 21 ]; /* 21 */ +extern const opus_uint8 silk_pitch_contour_iCDF[ 34 ]; /* 34 */ +extern const opus_uint8 silk_pitch_contour_NB_iCDF[ 11 ]; /* 11 */ +extern const opus_uint8 silk_pitch_contour_10_ms_iCDF[ 12 ]; /* 12 */ +extern const opus_uint8 silk_pitch_contour_10_ms_NB_iCDF[ 3 ]; /* 3 */ + +extern const opus_uint8 silk_pulses_per_block_iCDF[ N_RATE_LEVELS ][ SILK_MAX_PULSES + 2 ]; /* 180 */ +extern const opus_uint8 silk_pulses_per_block_BITS_Q5[ N_RATE_LEVELS - 1 ][ SILK_MAX_PULSES + 2 ]; /* 162 */ + +extern const opus_uint8 silk_rate_levels_iCDF[ 2 ][ N_RATE_LEVELS - 1 ]; /* 18 */ +extern const opus_uint8 silk_rate_levels_BITS_Q5[ 2 ][ N_RATE_LEVELS - 1 ]; /* 18 */ + +extern const opus_uint8 silk_max_pulses_table[ 4 ]; /* 4 */ + +extern const opus_uint8 silk_shell_code_table0[ 152 ]; /* 152 */ +extern const opus_uint8 silk_shell_code_table1[ 152 ]; /* 152 */ +extern const opus_uint8 silk_shell_code_table2[ 152 ]; /* 152 */ +extern const opus_uint8 silk_shell_code_table3[ 152 ]; /* 152 */ +extern const opus_uint8 silk_shell_code_table_offsets[ SILK_MAX_PULSES + 1 ]; /* 17 */ + +extern const opus_uint8 silk_lsb_iCDF[ 2 ]; /* 2 */ + +extern const opus_uint8 silk_sign_iCDF[ 42 ]; /* 42 */ + +extern const opus_uint8 silk_uniform3_iCDF[ 3 ]; /* 3 */ +extern const opus_uint8 silk_uniform4_iCDF[ 4 ]; /* 4 */ +extern const opus_uint8 silk_uniform5_iCDF[ 5 ]; /* 5 */ +extern const opus_uint8 silk_uniform6_iCDF[ 6 ]; /* 6 */ +extern const opus_uint8 silk_uniform8_iCDF[ 8 ]; /* 8 */ + +extern const opus_uint8 silk_NLSF_EXT_iCDF[ 7 ]; /* 7 */ + +extern const opus_uint8 silk_LTP_per_index_iCDF[ 3 ]; /* 3 */ +extern const opus_uint8 * const silk_LTP_gain_iCDF_ptrs[ NB_LTP_CBKS ]; /* 3 */ +extern const opus_uint8 * const silk_LTP_gain_BITS_Q5_ptrs[ NB_LTP_CBKS ]; /* 3 */ +extern const opus_int8 * const silk_LTP_vq_ptrs_Q7[ NB_LTP_CBKS ]; /* 168 */ +extern const opus_uint8 * const silk_LTP_vq_gain_ptrs_Q7[NB_LTP_CBKS]; +extern const opus_int8 silk_LTP_vq_sizes[ NB_LTP_CBKS ]; /* 3 */ + +extern const opus_uint8 silk_LTPscale_iCDF[ 3 ]; /* 4 */ +extern const opus_int16 silk_LTPScales_table_Q14[ 3 ]; /* 6 */ + +extern const opus_uint8 silk_type_offset_VAD_iCDF[ 4 ]; /* 4 */ +extern const opus_uint8 silk_type_offset_no_VAD_iCDF[ 2 ]; /* 2 */ + +extern const opus_int16 silk_stereo_pred_quant_Q13[ STEREO_QUANT_TAB_SIZE ]; /* 32 */ +extern const opus_uint8 silk_stereo_pred_joint_iCDF[ 25 ]; /* 25 */ +extern const opus_uint8 silk_stereo_only_code_mid_iCDF[ 2 ]; /* 2 */ + +extern const opus_uint8 * const silk_LBRR_flags_iCDF_ptr[ 2 ]; /* 10 */ + +extern const opus_uint8 silk_NLSF_interpolation_factor_iCDF[ 5 ]; /* 5 */ + +extern const silk_NLSF_CB_struct silk_NLSF_CB_WB; /* 1040 */ +extern const silk_NLSF_CB_struct silk_NLSF_CB_NB_MB; /* 728 */ + +/* Quantization offsets */ +extern const opus_int16 silk_Quantization_Offsets_Q10[ 2 ][ 2 ]; /* 8 */ + +/* Interpolation points for filter coefficients used in the bandwidth transition smoother */ +extern const opus_int32 silk_Transition_LP_B_Q28[ TRANSITION_INT_NUM ][ TRANSITION_NB ]; /* 60 */ +extern const opus_int32 silk_Transition_LP_A_Q28[ TRANSITION_INT_NUM ][ TRANSITION_NA ]; /* 60 */ + +/* Rom table with cosine values */ +extern const opus_int16 silk_LSFCosTab_FIX_Q12[ LSF_COS_TAB_SZ_FIX + 1 ]; /* 258 */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_LTP.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_LTP.c new file mode 100644 index 000000000..5e12c8643 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_LTP.c @@ -0,0 +1,294 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +const opus_uint8 silk_LTP_per_index_iCDF[3] = { + 179, 99, 0 +}; + +static const opus_uint8 silk_LTP_gain_iCDF_0[8] = { + 71, 56, 43, 30, 21, 12, 6, 0 +}; + +static const opus_uint8 silk_LTP_gain_iCDF_1[16] = { + 199, 165, 144, 124, 109, 96, 84, 71, + 61, 51, 42, 32, 23, 15, 8, 0 +}; + +static const opus_uint8 silk_LTP_gain_iCDF_2[32] = { + 241, 225, 211, 199, 187, 175, 164, 153, + 142, 132, 123, 114, 105, 96, 88, 80, + 72, 64, 57, 50, 44, 38, 33, 29, + 24, 20, 16, 12, 9, 5, 2, 0 +}; + +static const opus_uint8 silk_LTP_gain_BITS_Q5_0[8] = { + 15, 131, 138, 138, 155, 155, 173, 173 +}; + +static const opus_uint8 silk_LTP_gain_BITS_Q5_1[16] = { + 69, 93, 115, 118, 131, 138, 141, 138, + 150, 150, 155, 150, 155, 160, 166, 160 +}; + +static const opus_uint8 silk_LTP_gain_BITS_Q5_2[32] = { + 131, 128, 134, 141, 141, 141, 145, 145, + 145, 150, 155, 155, 155, 155, 160, 160, + 160, 160, 166, 166, 173, 173, 182, 192, + 182, 192, 192, 192, 205, 192, 205, 224 +}; + +const opus_uint8 * const silk_LTP_gain_iCDF_ptrs[NB_LTP_CBKS] = { + silk_LTP_gain_iCDF_0, + silk_LTP_gain_iCDF_1, + silk_LTP_gain_iCDF_2 +}; + +const opus_uint8 * const silk_LTP_gain_BITS_Q5_ptrs[NB_LTP_CBKS] = { + silk_LTP_gain_BITS_Q5_0, + silk_LTP_gain_BITS_Q5_1, + silk_LTP_gain_BITS_Q5_2 +}; + +static const opus_int8 silk_LTP_gain_vq_0[8][5] = +{ +{ + 4, 6, 24, 7, 5 +}, +{ + 0, 0, 2, 0, 0 +}, +{ + 12, 28, 41, 13, -4 +}, +{ + -9, 15, 42, 25, 14 +}, +{ + 1, -2, 62, 41, -9 +}, +{ + -10, 37, 65, -4, 3 +}, +{ + -6, 4, 66, 7, -8 +}, +{ + 16, 14, 38, -3, 33 +} +}; + +static const opus_int8 silk_LTP_gain_vq_1[16][5] = +{ +{ + 13, 22, 39, 23, 12 +}, +{ + -1, 36, 64, 27, -6 +}, +{ + -7, 10, 55, 43, 17 +}, +{ + 1, 1, 8, 1, 1 +}, +{ + 6, -11, 74, 53, -9 +}, +{ + -12, 55, 76, -12, 8 +}, +{ + -3, 3, 93, 27, -4 +}, +{ + 26, 39, 59, 3, -8 +}, +{ + 2, 0, 77, 11, 9 +}, +{ + -8, 22, 44, -6, 7 +}, +{ + 40, 9, 26, 3, 9 +}, +{ + -7, 20, 101, -7, 4 +}, +{ + 3, -8, 42, 26, 0 +}, +{ + -15, 33, 68, 2, 23 +}, +{ + -2, 55, 46, -2, 15 +}, +{ + 3, -1, 21, 16, 41 +} +}; + +static const opus_int8 silk_LTP_gain_vq_2[32][5] = +{ +{ + -6, 27, 61, 39, 5 +}, +{ + -11, 42, 88, 4, 1 +}, +{ + -2, 60, 65, 6, -4 +}, +{ + -1, -5, 73, 56, 1 +}, +{ + -9, 19, 94, 29, -9 +}, +{ + 0, 12, 99, 6, 4 +}, +{ + 8, -19, 102, 46, -13 +}, +{ + 3, 2, 13, 3, 2 +}, +{ + 9, -21, 84, 72, -18 +}, +{ + -11, 46, 104, -22, 8 +}, +{ + 18, 38, 48, 23, 0 +}, +{ + -16, 70, 83, -21, 11 +}, +{ + 5, -11, 117, 22, -8 +}, +{ + -6, 23, 117, -12, 3 +}, +{ + 3, -8, 95, 28, 4 +}, +{ + -10, 15, 77, 60, -15 +}, +{ + -1, 4, 124, 2, -4 +}, +{ + 3, 38, 84, 24, -25 +}, +{ + 2, 13, 42, 13, 31 +}, +{ + 21, -4, 56, 46, -1 +}, +{ + -1, 35, 79, -13, 19 +}, +{ + -7, 65, 88, -9, -14 +}, +{ + 20, 4, 81, 49, -29 +}, +{ + 20, 0, 75, 3, -17 +}, +{ + 5, -9, 44, 92, -8 +}, +{ + 1, -3, 22, 69, 31 +}, +{ + -6, 95, 41, -12, 5 +}, +{ + 39, 67, 16, -4, 1 +}, +{ + 0, -6, 120, 55, -36 +}, +{ + -13, 44, 122, 4, -24 +}, +{ + 81, 5, 11, 3, 7 +}, +{ + 2, 0, 9, 10, 88 +} +}; + +const opus_int8 * const silk_LTP_vq_ptrs_Q7[NB_LTP_CBKS] = { + (opus_int8 *)&silk_LTP_gain_vq_0[0][0], + (opus_int8 *)&silk_LTP_gain_vq_1[0][0], + (opus_int8 *)&silk_LTP_gain_vq_2[0][0] +}; + +/* Maximum frequency-dependent response of the pitch taps above, + computed as max(abs(freqz(taps))) */ +static const opus_uint8 silk_LTP_gain_vq_0_gain[8] = { + 46, 2, 90, 87, 93, 91, 82, 98 +}; + +static const opus_uint8 silk_LTP_gain_vq_1_gain[16] = { + 109, 120, 118, 12, 113, 115, 117, 119, + 99, 59, 87, 111, 63, 111, 112, 80 +}; + +static const opus_uint8 silk_LTP_gain_vq_2_gain[32] = { + 126, 124, 125, 124, 129, 121, 126, 23, + 132, 127, 127, 127, 126, 127, 122, 133, + 130, 134, 101, 118, 119, 145, 126, 86, + 124, 120, 123, 119, 170, 173, 107, 109 +}; + +const opus_uint8 * const silk_LTP_vq_gain_ptrs_Q7[NB_LTP_CBKS] = { + &silk_LTP_gain_vq_0_gain[0], + &silk_LTP_gain_vq_1_gain[0], + &silk_LTP_gain_vq_2_gain[0] +}; + +const opus_int8 silk_LTP_vq_sizes[NB_LTP_CBKS] = { + 8, 16, 32 +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_NB_MB.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_NB_MB.c new file mode 100644 index 000000000..195d5b95b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_NB_MB.c @@ -0,0 +1,195 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +static const opus_uint8 silk_NLSF_CB1_NB_MB_Q8[ 320 ] = { + 12, 35, 60, 83, 108, 132, 157, 180, + 206, 228, 15, 32, 55, 77, 101, 125, + 151, 175, 201, 225, 19, 42, 66, 89, + 114, 137, 162, 184, 209, 230, 12, 25, + 50, 72, 97, 120, 147, 172, 200, 223, + 26, 44, 69, 90, 114, 135, 159, 180, + 205, 225, 13, 22, 53, 80, 106, 130, + 156, 180, 205, 228, 15, 25, 44, 64, + 90, 115, 142, 168, 196, 222, 19, 24, + 62, 82, 100, 120, 145, 168, 190, 214, + 22, 31, 50, 79, 103, 120, 151, 170, + 203, 227, 21, 29, 45, 65, 106, 124, + 150, 171, 196, 224, 30, 49, 75, 97, + 121, 142, 165, 186, 209, 229, 19, 25, + 52, 70, 93, 116, 143, 166, 192, 219, + 26, 34, 62, 75, 97, 118, 145, 167, + 194, 217, 25, 33, 56, 70, 91, 113, + 143, 165, 196, 223, 21, 34, 51, 72, + 97, 117, 145, 171, 196, 222, 20, 29, + 50, 67, 90, 117, 144, 168, 197, 221, + 22, 31, 48, 66, 95, 117, 146, 168, + 196, 222, 24, 33, 51, 77, 116, 134, + 158, 180, 200, 224, 21, 28, 70, 87, + 106, 124, 149, 170, 194, 217, 26, 33, + 53, 64, 83, 117, 152, 173, 204, 225, + 27, 34, 65, 95, 108, 129, 155, 174, + 210, 225, 20, 26, 72, 99, 113, 131, + 154, 176, 200, 219, 34, 43, 61, 78, + 93, 114, 155, 177, 205, 229, 23, 29, + 54, 97, 124, 138, 163, 179, 209, 229, + 30, 38, 56, 89, 118, 129, 158, 178, + 200, 231, 21, 29, 49, 63, 85, 111, + 142, 163, 193, 222, 27, 48, 77, 103, + 133, 158, 179, 196, 215, 232, 29, 47, + 74, 99, 124, 151, 176, 198, 220, 237, + 33, 42, 61, 76, 93, 121, 155, 174, + 207, 225, 29, 53, 87, 112, 136, 154, + 170, 188, 208, 227, 24, 30, 52, 84, + 131, 150, 166, 186, 203, 229, 37, 48, + 64, 84, 104, 118, 156, 177, 201, 230 +}; + +static const opus_int16 silk_NLSF_CB1_Wght_Q9[ 320 ] = { + 2897, 2314, 2314, 2314, 2287, 2287, 2314, 2300, 2327, 2287, + 2888, 2580, 2394, 2367, 2314, 2274, 2274, 2274, 2274, 2194, + 2487, 2340, 2340, 2314, 2314, 2314, 2340, 2340, 2367, 2354, + 3216, 2766, 2340, 2340, 2314, 2274, 2221, 2207, 2261, 2194, + 2460, 2474, 2367, 2394, 2394, 2394, 2394, 2367, 2407, 2314, + 3479, 3056, 2127, 2207, 2274, 2274, 2274, 2287, 2314, 2261, + 3282, 3141, 2580, 2394, 2247, 2221, 2207, 2194, 2194, 2114, + 4096, 3845, 2221, 2620, 2620, 2407, 2314, 2394, 2367, 2074, + 3178, 3244, 2367, 2221, 2553, 2434, 2340, 2314, 2167, 2221, + 3338, 3488, 2726, 2194, 2261, 2460, 2354, 2367, 2207, 2101, + 2354, 2420, 2327, 2367, 2394, 2420, 2420, 2420, 2460, 2367, + 3779, 3629, 2434, 2527, 2367, 2274, 2274, 2300, 2207, 2048, + 3254, 3225, 2713, 2846, 2447, 2327, 2300, 2300, 2274, 2127, + 3263, 3300, 2753, 2806, 2447, 2261, 2261, 2247, 2127, 2101, + 2873, 2981, 2633, 2367, 2407, 2354, 2194, 2247, 2247, 2114, + 3225, 3197, 2633, 2580, 2274, 2181, 2247, 2221, 2221, 2141, + 3178, 3310, 2740, 2407, 2274, 2274, 2274, 2287, 2194, 2114, + 3141, 3272, 2460, 2061, 2287, 2500, 2367, 2487, 2434, 2181, + 3507, 3282, 2314, 2700, 2647, 2474, 2367, 2394, 2340, 2127, + 3423, 3535, 3038, 3056, 2300, 1950, 2221, 2274, 2274, 2274, + 3404, 3366, 2087, 2687, 2873, 2354, 2420, 2274, 2474, 2540, + 3760, 3488, 1950, 2660, 2897, 2527, 2394, 2367, 2460, 2261, + 3028, 3272, 2740, 2888, 2740, 2154, 2127, 2287, 2234, 2247, + 3695, 3657, 2025, 1969, 2660, 2700, 2580, 2500, 2327, 2367, + 3207, 3413, 2354, 2074, 2888, 2888, 2340, 2487, 2247, 2167, + 3338, 3366, 2846, 2780, 2327, 2154, 2274, 2287, 2114, 2061, + 2327, 2300, 2181, 2167, 2181, 2367, 2633, 2700, 2700, 2553, + 2407, 2434, 2221, 2261, 2221, 2221, 2340, 2420, 2607, 2700, + 3038, 3244, 2806, 2888, 2474, 2074, 2300, 2314, 2354, 2380, + 2221, 2154, 2127, 2287, 2500, 2793, 2793, 2620, 2580, 2367, + 3676, 3713, 2234, 1838, 2181, 2753, 2726, 2673, 2513, 2207, + 2793, 3160, 2726, 2553, 2846, 2513, 2181, 2394, 2221, 2181 +}; + +static const opus_uint8 silk_NLSF_CB1_iCDF_NB_MB[ 64 ] = { + 212, 178, 148, 129, 108, 96, 85, 82, + 79, 77, 61, 59, 57, 56, 51, 49, + 48, 45, 42, 41, 40, 38, 36, 34, + 31, 30, 21, 12, 10, 3, 1, 0, + 255, 245, 244, 236, 233, 225, 217, 203, + 190, 176, 175, 161, 149, 136, 125, 114, + 102, 91, 81, 71, 60, 52, 43, 35, + 28, 20, 19, 18, 12, 11, 5, 0 +}; + +static const opus_uint8 silk_NLSF_CB2_SELECT_NB_MB[ 160 ] = { + 16, 0, 0, 0, 0, 99, 66, 36, + 36, 34, 36, 34, 34, 34, 34, 83, + 69, 36, 52, 34, 116, 102, 70, 68, + 68, 176, 102, 68, 68, 34, 65, 85, + 68, 84, 36, 116, 141, 152, 139, 170, + 132, 187, 184, 216, 137, 132, 249, 168, + 185, 139, 104, 102, 100, 68, 68, 178, + 218, 185, 185, 170, 244, 216, 187, 187, + 170, 244, 187, 187, 219, 138, 103, 155, + 184, 185, 137, 116, 183, 155, 152, 136, + 132, 217, 184, 184, 170, 164, 217, 171, + 155, 139, 244, 169, 184, 185, 170, 164, + 216, 223, 218, 138, 214, 143, 188, 218, + 168, 244, 141, 136, 155, 170, 168, 138, + 220, 219, 139, 164, 219, 202, 216, 137, + 168, 186, 246, 185, 139, 116, 185, 219, + 185, 138, 100, 100, 134, 100, 102, 34, + 68, 68, 100, 68, 168, 203, 221, 218, + 168, 167, 154, 136, 104, 70, 164, 246, + 171, 137, 139, 137, 155, 218, 219, 139 +}; + +static const opus_uint8 silk_NLSF_CB2_iCDF_NB_MB[ 72 ] = { + 255, 254, 253, 238, 14, 3, 2, 1, + 0, 255, 254, 252, 218, 35, 3, 2, + 1, 0, 255, 254, 250, 208, 59, 4, + 2, 1, 0, 255, 254, 246, 194, 71, + 10, 2, 1, 0, 255, 252, 236, 183, + 82, 8, 2, 1, 0, 255, 252, 235, + 180, 90, 17, 2, 1, 0, 255, 248, + 224, 171, 97, 30, 4, 1, 0, 255, + 254, 236, 173, 95, 37, 7, 1, 0 +}; + +static const opus_uint8 silk_NLSF_CB2_BITS_NB_MB_Q5[ 72 ] = { + 255, 255, 255, 131, 6, 145, 255, 255, + 255, 255, 255, 236, 93, 15, 96, 255, + 255, 255, 255, 255, 194, 83, 25, 71, + 221, 255, 255, 255, 255, 162, 73, 34, + 66, 162, 255, 255, 255, 210, 126, 73, + 43, 57, 173, 255, 255, 255, 201, 125, + 71, 48, 58, 130, 255, 255, 255, 166, + 110, 73, 57, 62, 104, 210, 255, 255, + 251, 123, 65, 55, 68, 100, 171, 255 +}; + +static const opus_uint8 silk_NLSF_PRED_NB_MB_Q8[ 18 ] = { + 179, 138, 140, 148, 151, 149, 153, 151, + 163, 116, 67, 82, 59, 92, 72, 100, + 89, 92 +}; + +static const opus_int16 silk_NLSF_DELTA_MIN_NB_MB_Q15[ 11 ] = { + 250, 3, 6, 3, 3, 3, 4, 3, + 3, 3, 461 +}; + +const silk_NLSF_CB_struct silk_NLSF_CB_NB_MB = +{ + 32, + 10, + SILK_FIX_CONST( 0.18, 16 ), + SILK_FIX_CONST( 1.0 / 0.18, 6 ), + silk_NLSF_CB1_NB_MB_Q8, + silk_NLSF_CB1_Wght_Q9, + silk_NLSF_CB1_iCDF_NB_MB, + silk_NLSF_PRED_NB_MB_Q8, + silk_NLSF_CB2_SELECT_NB_MB, + silk_NLSF_CB2_iCDF_NB_MB, + silk_NLSF_CB2_BITS_NB_MB_Q5, + silk_NLSF_DELTA_MIN_NB_MB_Q15, +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_WB.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_WB.c new file mode 100644 index 000000000..5cc9f57bf --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_NLSF_CB_WB.c @@ -0,0 +1,234 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +static const opus_uint8 silk_NLSF_CB1_WB_Q8[ 512 ] = { + 7, 23, 38, 54, 69, 85, 100, 116, + 131, 147, 162, 178, 193, 208, 223, 239, + 13, 25, 41, 55, 69, 83, 98, 112, + 127, 142, 157, 171, 187, 203, 220, 236, + 15, 21, 34, 51, 61, 78, 92, 106, + 126, 136, 152, 167, 185, 205, 225, 240, + 10, 21, 36, 50, 63, 79, 95, 110, + 126, 141, 157, 173, 189, 205, 221, 237, + 17, 20, 37, 51, 59, 78, 89, 107, + 123, 134, 150, 164, 184, 205, 224, 240, + 10, 15, 32, 51, 67, 81, 96, 112, + 129, 142, 158, 173, 189, 204, 220, 236, + 8, 21, 37, 51, 65, 79, 98, 113, + 126, 138, 155, 168, 179, 192, 209, 218, + 12, 15, 34, 55, 63, 78, 87, 108, + 118, 131, 148, 167, 185, 203, 219, 236, + 16, 19, 32, 36, 56, 79, 91, 108, + 118, 136, 154, 171, 186, 204, 220, 237, + 11, 28, 43, 58, 74, 89, 105, 120, + 135, 150, 165, 180, 196, 211, 226, 241, + 6, 16, 33, 46, 60, 75, 92, 107, + 123, 137, 156, 169, 185, 199, 214, 225, + 11, 19, 30, 44, 57, 74, 89, 105, + 121, 135, 152, 169, 186, 202, 218, 234, + 12, 19, 29, 46, 57, 71, 88, 100, + 120, 132, 148, 165, 182, 199, 216, 233, + 17, 23, 35, 46, 56, 77, 92, 106, + 123, 134, 152, 167, 185, 204, 222, 237, + 14, 17, 45, 53, 63, 75, 89, 107, + 115, 132, 151, 171, 188, 206, 221, 240, + 9, 16, 29, 40, 56, 71, 88, 103, + 119, 137, 154, 171, 189, 205, 222, 237, + 16, 19, 36, 48, 57, 76, 87, 105, + 118, 132, 150, 167, 185, 202, 218, 236, + 12, 17, 29, 54, 71, 81, 94, 104, + 126, 136, 149, 164, 182, 201, 221, 237, + 15, 28, 47, 62, 79, 97, 115, 129, + 142, 155, 168, 180, 194, 208, 223, 238, + 8, 14, 30, 45, 62, 78, 94, 111, + 127, 143, 159, 175, 192, 207, 223, 239, + 17, 30, 49, 62, 79, 92, 107, 119, + 132, 145, 160, 174, 190, 204, 220, 235, + 14, 19, 36, 45, 61, 76, 91, 108, + 121, 138, 154, 172, 189, 205, 222, 238, + 12, 18, 31, 45, 60, 76, 91, 107, + 123, 138, 154, 171, 187, 204, 221, 236, + 13, 17, 31, 43, 53, 70, 83, 103, + 114, 131, 149, 167, 185, 203, 220, 237, + 17, 22, 35, 42, 58, 78, 93, 110, + 125, 139, 155, 170, 188, 206, 224, 240, + 8, 15, 34, 50, 67, 83, 99, 115, + 131, 146, 162, 178, 193, 209, 224, 239, + 13, 16, 41, 66, 73, 86, 95, 111, + 128, 137, 150, 163, 183, 206, 225, 241, + 17, 25, 37, 52, 63, 75, 92, 102, + 119, 132, 144, 160, 175, 191, 212, 231, + 19, 31, 49, 65, 83, 100, 117, 133, + 147, 161, 174, 187, 200, 213, 227, 242, + 18, 31, 52, 68, 88, 103, 117, 126, + 138, 149, 163, 177, 192, 207, 223, 239, + 16, 29, 47, 61, 76, 90, 106, 119, + 133, 147, 161, 176, 193, 209, 224, 240, + 15, 21, 35, 50, 61, 73, 86, 97, + 110, 119, 129, 141, 175, 198, 218, 237 +}; + +static const opus_int16 silk_NLSF_CB1_WB_Wght_Q9[ 512 ] = { + 3657, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2925, 2963, 2963, 2925, 2846, + 3216, 3085, 2972, 3056, 3056, 3010, 3010, 3010, 2963, 2963, 3010, 2972, 2888, 2846, 2846, 2726, + 3920, 4014, 2981, 3207, 3207, 2934, 3056, 2846, 3122, 3244, 2925, 2846, 2620, 2553, 2780, 2925, + 3516, 3197, 3010, 3103, 3019, 2888, 2925, 2925, 2925, 2925, 2888, 2888, 2888, 2888, 2888, 2753, + 5054, 5054, 2934, 3573, 3385, 3056, 3085, 2793, 3160, 3160, 2972, 2846, 2513, 2540, 2753, 2888, + 4428, 4149, 2700, 2753, 2972, 3010, 2925, 2846, 2981, 3019, 2925, 2925, 2925, 2925, 2888, 2726, + 3620, 3019, 2972, 3056, 3056, 2873, 2806, 3056, 3216, 3047, 2981, 3291, 3291, 2981, 3310, 2991, + 5227, 5014, 2540, 3338, 3526, 3385, 3197, 3094, 3376, 2981, 2700, 2647, 2687, 2793, 2846, 2673, + 5081, 5174, 4615, 4428, 2460, 2897, 3047, 3207, 3169, 2687, 2740, 2888, 2846, 2793, 2846, 2700, + 3122, 2888, 2963, 2925, 2925, 2925, 2925, 2963, 2963, 2963, 2963, 2925, 2925, 2963, 2963, 2963, + 4202, 3207, 2981, 3103, 3010, 2888, 2888, 2925, 2972, 2873, 2916, 3019, 2972, 3010, 3197, 2873, + 3760, 3760, 3244, 3103, 2981, 2888, 2925, 2888, 2972, 2934, 2793, 2793, 2846, 2888, 2888, 2660, + 3854, 4014, 3207, 3122, 3244, 2934, 3047, 2963, 2963, 3085, 2846, 2793, 2793, 2793, 2793, 2580, + 3845, 4080, 3357, 3516, 3094, 2740, 3010, 2934, 3122, 3085, 2846, 2846, 2647, 2647, 2846, 2806, + 5147, 4894, 3225, 3845, 3441, 3169, 2897, 3413, 3451, 2700, 2580, 2673, 2740, 2846, 2806, 2753, + 4109, 3789, 3291, 3160, 2925, 2888, 2888, 2925, 2793, 2740, 2793, 2740, 2793, 2846, 2888, 2806, + 5081, 5054, 3047, 3545, 3244, 3056, 3085, 2944, 3103, 2897, 2740, 2740, 2740, 2846, 2793, 2620, + 4309, 4309, 2860, 2527, 3207, 3376, 3376, 3075, 3075, 3376, 3056, 2846, 2647, 2580, 2726, 2753, + 3056, 2916, 2806, 2888, 2740, 2687, 2897, 3103, 3150, 3150, 3216, 3169, 3056, 3010, 2963, 2846, + 4375, 3882, 2925, 2888, 2846, 2888, 2846, 2846, 2888, 2888, 2888, 2846, 2888, 2925, 2888, 2846, + 2981, 2916, 2916, 2981, 2981, 3056, 3122, 3216, 3150, 3056, 3010, 2972, 2972, 2972, 2925, 2740, + 4229, 4149, 3310, 3347, 2925, 2963, 2888, 2981, 2981, 2846, 2793, 2740, 2846, 2846, 2846, 2793, + 4080, 4014, 3103, 3010, 2925, 2925, 2925, 2888, 2925, 2925, 2846, 2846, 2846, 2793, 2888, 2780, + 4615, 4575, 3169, 3441, 3207, 2981, 2897, 3038, 3122, 2740, 2687, 2687, 2687, 2740, 2793, 2700, + 4149, 4269, 3789, 3657, 2726, 2780, 2888, 2888, 3010, 2972, 2925, 2846, 2687, 2687, 2793, 2888, + 4215, 3554, 2753, 2846, 2846, 2888, 2888, 2888, 2925, 2925, 2888, 2925, 2925, 2925, 2963, 2888, + 5174, 4921, 2261, 3432, 3789, 3479, 3347, 2846, 3310, 3479, 3150, 2897, 2460, 2487, 2753, 2925, + 3451, 3685, 3122, 3197, 3357, 3047, 3207, 3207, 2981, 3216, 3085, 2925, 2925, 2687, 2540, 2434, + 2981, 3010, 2793, 2793, 2740, 2793, 2846, 2972, 3056, 3103, 3150, 3150, 3150, 3103, 3010, 3010, + 2944, 2873, 2687, 2726, 2780, 3010, 3432, 3545, 3357, 3244, 3056, 3010, 2963, 2925, 2888, 2846, + 3019, 2944, 2897, 3010, 3010, 2972, 3019, 3103, 3056, 3056, 3010, 2888, 2846, 2925, 2925, 2888, + 3920, 3967, 3010, 3197, 3357, 3216, 3291, 3291, 3479, 3704, 3441, 2726, 2181, 2460, 2580, 2607 +}; + +static const opus_uint8 silk_NLSF_CB1_iCDF_WB[ 64 ] = { + 225, 204, 201, 184, 183, 175, 158, 154, + 153, 135, 119, 115, 113, 110, 109, 99, + 98, 95, 79, 68, 52, 50, 48, 45, + 43, 32, 31, 27, 18, 10, 3, 0, + 255, 251, 235, 230, 212, 201, 196, 182, + 167, 166, 163, 151, 138, 124, 110, 104, + 90, 78, 76, 70, 69, 57, 45, 34, + 24, 21, 11, 6, 5, 4, 3, 0 +}; + +static const opus_uint8 silk_NLSF_CB2_SELECT_WB[ 256 ] = { + 0, 0, 0, 0, 0, 0, 0, 1, + 100, 102, 102, 68, 68, 36, 34, 96, + 164, 107, 158, 185, 180, 185, 139, 102, + 64, 66, 36, 34, 34, 0, 1, 32, + 208, 139, 141, 191, 152, 185, 155, 104, + 96, 171, 104, 166, 102, 102, 102, 132, + 1, 0, 0, 0, 0, 16, 16, 0, + 80, 109, 78, 107, 185, 139, 103, 101, + 208, 212, 141, 139, 173, 153, 123, 103, + 36, 0, 0, 0, 0, 0, 0, 1, + 48, 0, 0, 0, 0, 0, 0, 32, + 68, 135, 123, 119, 119, 103, 69, 98, + 68, 103, 120, 118, 118, 102, 71, 98, + 134, 136, 157, 184, 182, 153, 139, 134, + 208, 168, 248, 75, 189, 143, 121, 107, + 32, 49, 34, 34, 34, 0, 17, 2, + 210, 235, 139, 123, 185, 137, 105, 134, + 98, 135, 104, 182, 100, 183, 171, 134, + 100, 70, 68, 70, 66, 66, 34, 131, + 64, 166, 102, 68, 36, 2, 1, 0, + 134, 166, 102, 68, 34, 34, 66, 132, + 212, 246, 158, 139, 107, 107, 87, 102, + 100, 219, 125, 122, 137, 118, 103, 132, + 114, 135, 137, 105, 171, 106, 50, 34, + 164, 214, 141, 143, 185, 151, 121, 103, + 192, 34, 0, 0, 0, 0, 0, 1, + 208, 109, 74, 187, 134, 249, 159, 137, + 102, 110, 154, 118, 87, 101, 119, 101, + 0, 2, 0, 36, 36, 66, 68, 35, + 96, 164, 102, 100, 36, 0, 2, 33, + 167, 138, 174, 102, 100, 84, 2, 2, + 100, 107, 120, 119, 36, 197, 24, 0 +}; + +static const opus_uint8 silk_NLSF_CB2_iCDF_WB[ 72 ] = { + 255, 254, 253, 244, 12, 3, 2, 1, + 0, 255, 254, 252, 224, 38, 3, 2, + 1, 0, 255, 254, 251, 209, 57, 4, + 2, 1, 0, 255, 254, 244, 195, 69, + 4, 2, 1, 0, 255, 251, 232, 184, + 84, 7, 2, 1, 0, 255, 254, 240, + 186, 86, 14, 2, 1, 0, 255, 254, + 239, 178, 91, 30, 5, 1, 0, 255, + 248, 227, 177, 100, 19, 2, 1, 0 +}; + +static const opus_uint8 silk_NLSF_CB2_BITS_WB_Q5[ 72 ] = { + 255, 255, 255, 156, 4, 154, 255, 255, + 255, 255, 255, 227, 102, 15, 92, 255, + 255, 255, 255, 255, 213, 83, 24, 72, + 236, 255, 255, 255, 255, 150, 76, 33, + 63, 214, 255, 255, 255, 190, 121, 77, + 43, 55, 185, 255, 255, 255, 245, 137, + 71, 43, 59, 139, 255, 255, 255, 255, + 131, 66, 50, 66, 107, 194, 255, 255, + 166, 116, 76, 55, 53, 125, 255, 255 +}; + +static const opus_uint8 silk_NLSF_PRED_WB_Q8[ 30 ] = { + 175, 148, 160, 176, 178, 173, 174, 164, + 177, 174, 196, 182, 198, 192, 182, 68, + 62, 66, 60, 72, 117, 85, 90, 118, + 136, 151, 142, 160, 142, 155 +}; + +static const opus_int16 silk_NLSF_DELTA_MIN_WB_Q15[ 17 ] = { + 100, 3, 40, 3, 3, 3, 5, 14, + 14, 10, 11, 3, 8, 9, 7, 3, + 347 +}; + +const silk_NLSF_CB_struct silk_NLSF_CB_WB = +{ + 32, + 16, + SILK_FIX_CONST( 0.15, 16 ), + SILK_FIX_CONST( 1.0 / 0.15, 6 ), + silk_NLSF_CB1_WB_Q8, + silk_NLSF_CB1_WB_Wght_Q9, + silk_NLSF_CB1_iCDF_WB, + silk_NLSF_PRED_WB_Q8, + silk_NLSF_CB2_SELECT_WB, + silk_NLSF_CB2_iCDF_WB, + silk_NLSF_CB2_BITS_WB_Q5, + silk_NLSF_DELTA_MIN_WB_Q15, +}; + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_gain.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_gain.c new file mode 100644 index 000000000..37e41d890 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_gain.c @@ -0,0 +1,63 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +const opus_uint8 silk_gain_iCDF[ 3 ][ N_LEVELS_QGAIN / 8 ] = +{ +{ + 224, 112, 44, 15, 3, 2, 1, 0 +}, +{ + 254, 237, 192, 132, 70, 23, 4, 0 +}, +{ + 255, 252, 226, 155, 61, 11, 2, 0 +} +}; + +const opus_uint8 silk_delta_gain_iCDF[ MAX_DELTA_GAIN_QUANT - MIN_DELTA_GAIN_QUANT + 1 ] = { + 250, 245, 234, 203, 71, 50, 42, 38, + 35, 33, 31, 29, 28, 27, 26, 25, + 24, 23, 22, 21, 20, 19, 18, 17, + 16, 15, 14, 13, 12, 11, 10, 9, + 8, 7, 6, 5, 4, 3, 2, 1, + 0 +}; + +#ifdef __cplusplus +} +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_other.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_other.c new file mode 100644 index 000000000..e34d90777 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_other.c @@ -0,0 +1,124 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "structs.h" +#include "define.h" +#include "tables.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Tables for stereo predictor coding */ +const opus_int16 silk_stereo_pred_quant_Q13[ STEREO_QUANT_TAB_SIZE ] = { + -13732, -10050, -8266, -7526, -6500, -5000, -2950, -820, + 820, 2950, 5000, 6500, 7526, 8266, 10050, 13732 +}; +const opus_uint8 silk_stereo_pred_joint_iCDF[ 25 ] = { + 249, 247, 246, 245, 244, + 234, 210, 202, 201, 200, + 197, 174, 82, 59, 56, + 55, 54, 46, 22, 12, + 11, 10, 9, 7, 0 +}; +const opus_uint8 silk_stereo_only_code_mid_iCDF[ 2 ] = { 64, 0 }; + +/* Tables for LBRR flags */ +static const opus_uint8 silk_LBRR_flags_2_iCDF[ 3 ] = { 203, 150, 0 }; +static const opus_uint8 silk_LBRR_flags_3_iCDF[ 7 ] = { 215, 195, 166, 125, 110, 82, 0 }; +const opus_uint8 * const silk_LBRR_flags_iCDF_ptr[ 2 ] = { + silk_LBRR_flags_2_iCDF, + silk_LBRR_flags_3_iCDF +}; + +/* Table for LSB coding */ +const opus_uint8 silk_lsb_iCDF[ 2 ] = { 120, 0 }; + +/* Tables for LTPScale */ +const opus_uint8 silk_LTPscale_iCDF[ 3 ] = { 128, 64, 0 }; + +/* Tables for signal type and offset coding */ +const opus_uint8 silk_type_offset_VAD_iCDF[ 4 ] = { + 232, 158, 10, 0 +}; +const opus_uint8 silk_type_offset_no_VAD_iCDF[ 2 ] = { + 230, 0 +}; + +/* Tables for NLSF interpolation factor */ +const opus_uint8 silk_NLSF_interpolation_factor_iCDF[ 5 ] = { 243, 221, 192, 181, 0 }; + +/* Quantization offsets */ +const opus_int16 silk_Quantization_Offsets_Q10[ 2 ][ 2 ] = { + { OFFSET_UVL_Q10, OFFSET_UVH_Q10 }, { OFFSET_VL_Q10, OFFSET_VH_Q10 } +}; + +/* Table for LTPScale */ +const opus_int16 silk_LTPScales_table_Q14[ 3 ] = { 15565, 12288, 8192 }; + +/* Uniform entropy tables */ +const opus_uint8 silk_uniform3_iCDF[ 3 ] = { 171, 85, 0 }; +const opus_uint8 silk_uniform4_iCDF[ 4 ] = { 192, 128, 64, 0 }; +const opus_uint8 silk_uniform5_iCDF[ 5 ] = { 205, 154, 102, 51, 0 }; +const opus_uint8 silk_uniform6_iCDF[ 6 ] = { 213, 171, 128, 85, 43, 0 }; +const opus_uint8 silk_uniform8_iCDF[ 8 ] = { 224, 192, 160, 128, 96, 64, 32, 0 }; + +const opus_uint8 silk_NLSF_EXT_iCDF[ 7 ] = { 100, 40, 16, 7, 3, 1, 0 }; + +/* Elliptic/Cauer filters designed with 0.1 dB passband ripple, + 80 dB minimum stopband attenuation, and + [0.95 : 0.15 : 0.35] normalized cut off frequencies. */ + +/* Interpolation points for filter coefficients used in the bandwidth transition smoother */ +const opus_int32 silk_Transition_LP_B_Q28[ TRANSITION_INT_NUM ][ TRANSITION_NB ] = +{ +{ 250767114, 501534038, 250767114 }, +{ 209867381, 419732057, 209867381 }, +{ 170987846, 341967853, 170987846 }, +{ 131531482, 263046905, 131531482 }, +{ 89306658, 178584282, 89306658 } +}; + +/* Interpolation points for filter coefficients used in the bandwidth transition smoother */ +const opus_int32 silk_Transition_LP_A_Q28[ TRANSITION_INT_NUM ][ TRANSITION_NA ] = +{ +{ 506393414, 239854379 }, +{ 411067935, 169683996 }, +{ 306733530, 116694253 }, +{ 185807084, 77959395 }, +{ 35497197, 57401098 } +}; + +#ifdef __cplusplus +} +#endif + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pitch_lag.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pitch_lag.c new file mode 100644 index 000000000..e80cc59a2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pitch_lag.c @@ -0,0 +1,69 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +const opus_uint8 silk_pitch_lag_iCDF[ 2 * ( PITCH_EST_MAX_LAG_MS - PITCH_EST_MIN_LAG_MS ) ] = { + 253, 250, 244, 233, 212, 182, 150, 131, + 120, 110, 98, 85, 72, 60, 49, 40, + 32, 25, 19, 15, 13, 11, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0 +}; + +const opus_uint8 silk_pitch_delta_iCDF[21] = { + 210, 208, 206, 203, 199, 193, 183, 168, + 142, 104, 74, 52, 37, 27, 20, 14, + 10, 6, 4, 2, 0 +}; + +const opus_uint8 silk_pitch_contour_iCDF[34] = { + 223, 201, 183, 167, 152, 138, 124, 111, + 98, 88, 79, 70, 62, 56, 50, 44, + 39, 35, 31, 27, 24, 21, 18, 16, + 14, 12, 10, 8, 6, 4, 3, 2, + 1, 0 +}; + +const opus_uint8 silk_pitch_contour_NB_iCDF[11] = { + 188, 176, 155, 138, 119, 97, 67, 43, + 26, 10, 0 +}; + +const opus_uint8 silk_pitch_contour_10_ms_iCDF[12] = { + 165, 119, 80, 61, 47, 35, 27, 20, + 14, 9, 4, 0 +}; + +const opus_uint8 silk_pitch_contour_10_ms_NB_iCDF[3] = { + 113, 63, 0 +}; + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pulses_per_block.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pulses_per_block.c new file mode 100644 index 000000000..c7c01c889 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tables_pulses_per_block.c @@ -0,0 +1,264 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tables.h" + +const opus_uint8 silk_max_pulses_table[ 4 ] = { + 8, 10, 12, 16 +}; + +const opus_uint8 silk_pulses_per_block_iCDF[ 10 ][ 18 ] = { +{ + 125, 51, 26, 18, 15, 12, 11, 10, + 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0 +}, +{ + 198, 105, 45, 22, 15, 12, 11, 10, + 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0 +}, +{ + 213, 162, 116, 83, 59, 43, 32, 24, + 18, 15, 12, 9, 7, 6, 5, 3, + 2, 0 +}, +{ + 239, 187, 116, 59, 28, 16, 11, 10, + 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0 +}, +{ + 250, 229, 188, 135, 86, 51, 30, 19, + 13, 10, 8, 6, 5, 4, 3, 2, + 1, 0 +}, +{ + 249, 235, 213, 185, 156, 128, 103, 83, + 66, 53, 42, 33, 26, 21, 17, 13, + 10, 0 +}, +{ + 254, 249, 235, 206, 164, 118, 77, 46, + 27, 16, 10, 7, 5, 4, 3, 2, + 1, 0 +}, +{ + 255, 253, 249, 239, 220, 191, 156, 119, + 85, 57, 37, 23, 15, 10, 6, 4, + 2, 0 +}, +{ + 255, 253, 251, 246, 237, 223, 203, 179, + 152, 124, 98, 75, 55, 40, 29, 21, + 15, 0 +}, +{ + 255, 254, 253, 247, 220, 162, 106, 67, + 42, 28, 18, 12, 9, 6, 4, 3, + 2, 0 +} +}; + +const opus_uint8 silk_pulses_per_block_BITS_Q5[ 9 ][ 18 ] = { +{ + 31, 57, 107, 160, 205, 205, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 +}, +{ + 69, 47, 67, 111, 166, 205, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 +}, +{ + 82, 74, 79, 95, 109, 128, 145, 160, + 173, 205, 205, 205, 224, 255, 255, 224, + 255, 224 +}, +{ + 125, 74, 59, 69, 97, 141, 182, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 +}, +{ + 173, 115, 85, 73, 76, 92, 115, 145, + 173, 205, 224, 224, 255, 255, 255, 255, + 255, 255 +}, +{ + 166, 134, 113, 102, 101, 102, 107, 118, + 125, 138, 145, 155, 166, 182, 192, 192, + 205, 150 +}, +{ + 224, 182, 134, 101, 83, 79, 85, 97, + 120, 145, 173, 205, 224, 255, 255, 255, + 255, 255 +}, +{ + 255, 224, 192, 150, 120, 101, 92, 89, + 93, 102, 118, 134, 160, 182, 192, 224, + 224, 224 +}, +{ + 255, 224, 224, 182, 155, 134, 118, 109, + 104, 102, 106, 111, 118, 131, 145, 160, + 173, 131 +} +}; + +const opus_uint8 silk_rate_levels_iCDF[ 2 ][ 9 ] = +{ +{ + 241, 190, 178, 132, 87, 74, 41, 14, + 0 +}, +{ + 223, 193, 157, 140, 106, 57, 39, 18, + 0 +} +}; + +const opus_uint8 silk_rate_levels_BITS_Q5[ 2 ][ 9 ] = +{ +{ + 131, 74, 141, 79, 80, 138, 95, 104, + 134 +}, +{ + 95, 99, 91, 125, 93, 76, 123, 115, + 123 +} +}; + +const opus_uint8 silk_shell_code_table0[ 152 ] = { + 128, 0, 214, 42, 0, 235, 128, 21, + 0, 244, 184, 72, 11, 0, 248, 214, + 128, 42, 7, 0, 248, 225, 170, 80, + 25, 5, 0, 251, 236, 198, 126, 54, + 18, 3, 0, 250, 238, 211, 159, 82, + 35, 15, 5, 0, 250, 231, 203, 168, + 128, 88, 53, 25, 6, 0, 252, 238, + 216, 185, 148, 108, 71, 40, 18, 4, + 0, 253, 243, 225, 199, 166, 128, 90, + 57, 31, 13, 3, 0, 254, 246, 233, + 212, 183, 147, 109, 73, 44, 23, 10, + 2, 0, 255, 250, 240, 223, 198, 166, + 128, 90, 58, 33, 16, 6, 1, 0, + 255, 251, 244, 231, 210, 181, 146, 110, + 75, 46, 25, 12, 5, 1, 0, 255, + 253, 248, 238, 221, 196, 164, 128, 92, + 60, 35, 18, 8, 3, 1, 0, 255, + 253, 249, 242, 229, 208, 180, 146, 110, + 76, 48, 27, 14, 7, 3, 1, 0 +}; + +const opus_uint8 silk_shell_code_table1[ 152 ] = { + 129, 0, 207, 50, 0, 236, 129, 20, + 0, 245, 185, 72, 10, 0, 249, 213, + 129, 42, 6, 0, 250, 226, 169, 87, + 27, 4, 0, 251, 233, 194, 130, 62, + 20, 4, 0, 250, 236, 207, 160, 99, + 47, 17, 3, 0, 255, 240, 217, 182, + 131, 81, 41, 11, 1, 0, 255, 254, + 233, 201, 159, 107, 61, 20, 2, 1, + 0, 255, 249, 233, 206, 170, 128, 86, + 50, 23, 7, 1, 0, 255, 250, 238, + 217, 186, 148, 108, 70, 39, 18, 6, + 1, 0, 255, 252, 243, 226, 200, 166, + 128, 90, 56, 30, 13, 4, 1, 0, + 255, 252, 245, 231, 209, 180, 146, 110, + 76, 47, 25, 11, 4, 1, 0, 255, + 253, 248, 237, 219, 194, 163, 128, 93, + 62, 37, 19, 8, 3, 1, 0, 255, + 254, 250, 241, 226, 205, 177, 145, 111, + 79, 51, 30, 15, 6, 2, 1, 0 +}; + +const opus_uint8 silk_shell_code_table2[ 152 ] = { + 129, 0, 203, 54, 0, 234, 129, 23, + 0, 245, 184, 73, 10, 0, 250, 215, + 129, 41, 5, 0, 252, 232, 173, 86, + 24, 3, 0, 253, 240, 200, 129, 56, + 15, 2, 0, 253, 244, 217, 164, 94, + 38, 10, 1, 0, 253, 245, 226, 189, + 132, 71, 27, 7, 1, 0, 253, 246, + 231, 203, 159, 105, 56, 23, 6, 1, + 0, 255, 248, 235, 213, 179, 133, 85, + 47, 19, 5, 1, 0, 255, 254, 243, + 221, 194, 159, 117, 70, 37, 12, 2, + 1, 0, 255, 254, 248, 234, 208, 171, + 128, 85, 48, 22, 8, 2, 1, 0, + 255, 254, 250, 240, 220, 189, 149, 107, + 67, 36, 16, 6, 2, 1, 0, 255, + 254, 251, 243, 227, 201, 166, 128, 90, + 55, 29, 13, 5, 2, 1, 0, 255, + 254, 252, 246, 234, 213, 183, 147, 109, + 73, 43, 22, 10, 4, 2, 1, 0 +}; + +const opus_uint8 silk_shell_code_table3[ 152 ] = { + 130, 0, 200, 58, 0, 231, 130, 26, + 0, 244, 184, 76, 12, 0, 249, 214, + 130, 43, 6, 0, 252, 232, 173, 87, + 24, 3, 0, 253, 241, 203, 131, 56, + 14, 2, 0, 254, 246, 221, 167, 94, + 35, 8, 1, 0, 254, 249, 232, 193, + 130, 65, 23, 5, 1, 0, 255, 251, + 239, 211, 162, 99, 45, 15, 4, 1, + 0, 255, 251, 243, 223, 186, 131, 74, + 33, 11, 3, 1, 0, 255, 252, 245, + 230, 202, 158, 105, 57, 24, 8, 2, + 1, 0, 255, 253, 247, 235, 214, 179, + 132, 84, 44, 19, 7, 2, 1, 0, + 255, 254, 250, 240, 223, 196, 159, 112, + 69, 36, 15, 6, 2, 1, 0, 255, + 254, 253, 245, 231, 209, 176, 136, 93, + 55, 27, 11, 3, 2, 1, 0, 255, + 254, 253, 252, 239, 221, 194, 158, 117, + 76, 42, 18, 4, 3, 2, 1, 0 +}; + +const opus_uint8 silk_shell_code_table_offsets[ 17 ] = { + 0, 0, 2, 5, 9, 14, 20, 27, + 35, 44, 54, 65, 77, 90, 104, 119, + 135 +}; + +const opus_uint8 silk_sign_iCDF[ 42 ] = { + 254, 49, 67, 77, 82, 93, 99, + 198, 11, 18, 24, 31, 36, 45, + 255, 46, 66, 78, 87, 94, 104, + 208, 14, 21, 32, 42, 51, 66, + 255, 94, 104, 109, 112, 115, 118, + 248, 53, 69, 80, 88, 95, 102 +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/meson.build new file mode 100644 index 000000000..b7c70f75f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/meson.build @@ -0,0 +1,8 @@ +exe = executable('test_unit_LPC_inv_pred_gain', + 'test_unit_LPC_inv_pred_gain.c', '../LPC_inv_pred_gain.c', + include_directories: opus_includes, + link_with: [celt_lib, celt_static_libs, silk_lib, silk_static_libs], + dependencies: libm, + install: false) + +test(test_name, exe) diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/test_unit_LPC_inv_pred_gain.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/test_unit_LPC_inv_pred_gain.c new file mode 100644 index 000000000..67067cead --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tests/test_unit_LPC_inv_pred_gain.c @@ -0,0 +1,129 @@ +/*********************************************************************** +Copyright (c) 2017 Google Inc., Jean-Marc Valin +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "celt/stack_alloc.h" +#include "cpu_support.h" +#include "SigProc_FIX.h" + +/* Computes the impulse response of the filter so we + can catch filters that are definitely unstable. Some + unstable filters may be classified as stable, but not + the other way around. */ +int check_stability(opus_int16 *A_Q12, int order) { + int i; + int j; + int sum_a, sum_abs_a; + sum_a = sum_abs_a = 0; + for( j = 0; j < order; j++ ) { + sum_a += A_Q12[ j ]; + sum_abs_a += silk_abs( A_Q12[ j ] ); + } + /* Check DC stability. */ + if( sum_a >= 4096 ) { + return 0; + } + /* If the sum of absolute values is less than 1, the filter + has to be stable. */ + if( sum_abs_a < 4096 ) { + return 1; + } + double y[SILK_MAX_ORDER_LPC] = {0}; + y[0] = 1; + for( i = 0; i < 10000; i++ ) { + double sum = 0; + for( j = 0; j < order; j++ ) { + sum += y[ j ]*A_Q12[ j ]; + } + for( j = order - 1; j > 0; j-- ) { + y[ j ] = y[ j - 1 ]; + } + y[ 0 ] = sum*(1./4096); + /* If impulse response reaches +/- 10000, the filter + is definitely unstable. */ + if( !(y[ 0 ] < 10000 && y[ 0 ] > -10000) ) { + return 0; + } + /* Test every 8 sample for low amplitude. */ + if( ( i & 0x7 ) == 0 ) { + double amp = 0; + for( j = 0; j < order; j++ ) { + amp += fabs(y[j]); + } + if( amp < 0.00001 ) { + return 1; + } + } + } + return 1; +} + +int main(void) { + const int arch = opus_select_arch(); + /* Set to 10000 so all branches in C function are triggered */ + const int loop_num = 10000; + int count = 0; + ALLOC_STACK; + + /* FIXME: Make the seed random (with option to set it explicitly) + so we get wider coverage. */ + srand(0); + + printf("Testing silk_LPC_inverse_pred_gain() optimization ...\n"); + for( count = 0; count < loop_num; count++ ) { + unsigned int i; + opus_int order; + unsigned int shift; + opus_int16 A_Q12[ SILK_MAX_ORDER_LPC ]; + opus_int32 gain; + + for( order = 2; order <= SILK_MAX_ORDER_LPC; order += 2 ) { /* order must be even. */ + for( shift = 0; shift < 16; shift++ ) { /* Different dynamic range. */ + for( i = 0; i < SILK_MAX_ORDER_LPC; i++ ) { + A_Q12[i] = ((opus_int16)rand()) >> shift; + } + gain = silk_LPC_inverse_pred_gain(A_Q12, order, arch); + /* Look for filters that silk_LPC_inverse_pred_gain() thinks are + stable but definitely aren't. */ + if( gain != 0 && !check_stability(A_Q12, order) ) { + fprintf(stderr, "**Loop %4d failed!**\n", count); + return 1; + } + } + } + if( !(count % 500) ) { + printf("Loop %4d passed\n", count); + } + } + printf("silk_LPC_inverse_pred_gain() optimization passed\n"); + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tuning_parameters.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tuning_parameters.h new file mode 100644 index 000000000..d70275fd8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/tuning_parameters.h @@ -0,0 +1,155 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_TUNING_PARAMETERS_H +#define SILK_TUNING_PARAMETERS_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Decay time for bitreservoir */ +#define BITRESERVOIR_DECAY_TIME_MS 500 + +/*******************/ +/* Pitch estimator */ +/*******************/ + +/* Level of noise floor for whitening filter LPC analysis in pitch analysis */ +#define FIND_PITCH_WHITE_NOISE_FRACTION 1e-3f + +/* Bandwidth expansion for whitening filter in pitch analysis */ +#define FIND_PITCH_BANDWIDTH_EXPANSION 0.99f + +/*********************/ +/* Linear prediction */ +/*********************/ + +/* LPC analysis regularization */ +#define FIND_LPC_COND_FAC 1e-5f + +/* Max cumulative LTP gain */ +#define MAX_SUM_LOG_GAIN_DB 250.0f + +/* LTP analysis defines */ +#define LTP_CORR_INV_MAX 0.03f + +/***********************/ +/* High pass filtering */ +/***********************/ + +/* Smoothing parameters for low end of pitch frequency range estimation */ +#define VARIABLE_HP_SMTH_COEF1 0.1f +#define VARIABLE_HP_SMTH_COEF2 0.015f +#define VARIABLE_HP_MAX_DELTA_FREQ 0.4f + +/* Min and max cut-off frequency values (-3 dB points) */ +#define VARIABLE_HP_MIN_CUTOFF_HZ 60 +#define VARIABLE_HP_MAX_CUTOFF_HZ 100 + +/***********/ +/* Various */ +/***********/ + +/* VAD threshold */ +#define SPEECH_ACTIVITY_DTX_THRES 0.05f + +/* Speech Activity LBRR enable threshold */ +#define LBRR_SPEECH_ACTIVITY_THRES 0.3f + +/*************************/ +/* Perceptual parameters */ +/*************************/ + +/* reduction in coding SNR during low speech activity */ +#define BG_SNR_DECR_dB 2.0f + +/* factor for reducing quantization noise during voiced speech */ +#define HARM_SNR_INCR_dB 2.0f + +/* factor for reducing quantization noise for unvoiced sparse signals */ +#define SPARSE_SNR_INCR_dB 2.0f + +/* threshold for sparseness measure above which to use lower quantization offset during unvoiced */ +#define ENERGY_VARIATION_THRESHOLD_QNT_OFFSET 0.6f + +/* warping control */ +#define WARPING_MULTIPLIER 0.015f + +/* fraction added to first autocorrelation value */ +#define SHAPE_WHITE_NOISE_FRACTION 3e-5f + +/* noise shaping filter chirp factor */ +#define BANDWIDTH_EXPANSION 0.94f + +/* harmonic noise shaping */ +#define HARMONIC_SHAPING 0.3f + +/* extra harmonic noise shaping for high bitrates or noisy input */ +#define HIGH_RATE_OR_LOW_QUALITY_HARMONIC_SHAPING 0.2f + +/* parameter for shaping noise towards higher frequencies */ +#define HP_NOISE_COEF 0.25f + +/* parameter for shaping noise even more towards higher frequencies during voiced speech */ +#define HARM_HP_NOISE_COEF 0.35f + +/* parameter for applying a high-pass tilt to the input signal */ +#define INPUT_TILT 0.05f + +/* parameter for extra high-pass tilt to the input signal at high rates */ +#define HIGH_RATE_INPUT_TILT 0.1f + +/* parameter for reducing noise at the very low frequencies */ +#define LOW_FREQ_SHAPING 4.0f + +/* less reduction of noise at the very low frequencies for signals with low SNR at low frequencies */ +#define LOW_QUALITY_LOW_FREQ_SHAPING_DECR 0.5f + +/* subframe smoothing coefficient for HarmBoost, HarmShapeGain, Tilt (lower -> more smoothing) */ +#define SUBFR_SMTH_COEF 0.4f + +/* parameters defining the R/D tradeoff in the residual quantizer */ +#define LAMBDA_OFFSET 1.2f +#define LAMBDA_SPEECH_ACT -0.2f +#define LAMBDA_DELAYED_DECISIONS -0.05f +#define LAMBDA_INPUT_QUALITY -0.1f +#define LAMBDA_CODING_QUALITY -0.2f +#define LAMBDA_QUANT_OFFSET 0.8f + +/* Compensation in bitrate calculations for 10 ms modes */ +#define REDUCE_BITRATE_10_MS_BPS 2200 + +/* Maximum time before allowing a bandwidth transition */ +#define MAX_BANDWIDTH_SWITCH_DELAY_MS 5000 + +#ifdef __cplusplus +} +#endif + +#endif /* SILK_TUNING_PARAMETERS_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/typedef.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/typedef.h new file mode 100644 index 000000000..793d2c0c1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/typedef.h @@ -0,0 +1,81 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_TYPEDEF_H +#define SILK_TYPEDEF_H + +#include "opus_types.h" +#include "opus_defines.h" + +#ifndef FIXED_POINT +# include +# define silk_float float +# define silk_float_MAX FLT_MAX +#endif + +#define silk_int64_MAX ((opus_int64)0x7FFFFFFFFFFFFFFFLL) /* 2^63 - 1 */ +#define silk_int64_MIN ((opus_int64)0x8000000000000000LL) /* -2^63 */ +#define silk_int32_MAX 0x7FFFFFFF /* 2^31 - 1 = 2147483647 */ +#define silk_int32_MIN ((opus_int32)0x80000000) /* -2^31 = -2147483648 */ +#define silk_int16_MAX 0x7FFF /* 2^15 - 1 = 32767 */ +#define silk_int16_MIN ((opus_int16)0x8000) /* -2^15 = -32768 */ +#define silk_int8_MAX 0x7F /* 2^7 - 1 = 127 */ +#define silk_int8_MIN ((opus_int8)0x80) /* -2^7 = -128 */ +#define silk_uint8_MAX 0xFF /* 2^8 - 1 = 255 */ + +#define silk_TRUE 1 +#define silk_FALSE 0 + +/* assertions */ +#if (defined _WIN32 && !defined _WINCE && !defined(__GNUC__) && !defined(NO_ASSERTS)) +# ifndef silk_assert +# include /* ASSERTE() */ +# define silk_assert(COND) _ASSERTE(COND) +# endif +#else +# ifdef ENABLE_ASSERTIONS +# include +# include +#define silk_fatal(str) _silk_fatal(str, __FILE__, __LINE__); +#ifdef __GNUC__ +__attribute__((noreturn)) +#endif +static OPUS_INLINE void _silk_fatal(const char *str, const char *file, int line) +{ + fprintf (stderr, "Fatal (internal) error in %s, line %d: %s\n", file, line, str); +#if defined(_MSC_VER) + _set_abort_behavior( 0, _WRITE_ABORT_MSG); +#endif + abort(); +} +# define silk_assert(COND) {if (!(COND)) {silk_fatal("assertion failed: " #COND);}} +# else +# define silk_assert(COND) +# endif +#endif + +#endif /* SILK_TYPEDEF_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_del_dec_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_del_dec_sse4_1.c new file mode 100644 index 000000000..2c75ede2d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_del_dec_sse4_1.c @@ -0,0 +1,859 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "main.h" +#include "celt/x86/x86cpu.h" + +#include "stack_alloc.h" + +typedef struct { + opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ]; + opus_int32 RandState[ DECISION_DELAY ]; + opus_int32 Q_Q10[ DECISION_DELAY ]; + opus_int32 Xq_Q14[ DECISION_DELAY ]; + opus_int32 Pred_Q15[ DECISION_DELAY ]; + opus_int32 Shape_Q14[ DECISION_DELAY ]; + opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ]; + opus_int32 LF_AR_Q14; + opus_int32 Seed; + opus_int32 SeedInit; + opus_int32 RD_Q10; +} NSQ_del_dec_struct; + +typedef struct { + opus_int32 Q_Q10; + opus_int32 RD_Q10; + opus_int32 xq_Q14; + opus_int32 LF_AR_Q14; + opus_int32 sLTP_shp_Q14; + opus_int32 LPC_exc_Q14; +} NSQ_sample_struct; + +typedef NSQ_sample_struct NSQ_sample_pair[ 2 ]; + +static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int32 x_Q3[], /* I Input in Q3 */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + opus_int nStatesDelayedDecision, /* I Number of del dec states */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +); + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay /* I */ +); + +void silk_NSQ_del_dec_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) +{ + opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr; + opus_int last_smple_idx, smpl_buf_idx, decisionDelay; + const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; + opus_int16 *pxq; + VARDECL( opus_int32, sLTP_Q15 ); + VARDECL( opus_int16, sLTP ); + opus_int32 HarmShapeFIRPacked_Q14; + opus_int offset_Q10; + opus_int32 RDmin_Q10, Gain_Q10; + VARDECL( opus_int32, x_sc_Q10 ); + VARDECL( opus_int32, delayedGain_Q10 ); + VARDECL( NSQ_del_dec_struct, psDelDec ); + NSQ_del_dec_struct *psDD; + SAVE_STACK; + + /* Set unvoiced lag to the previous one, overwrite later for voiced */ + lag = NSQ->lagPrev; + + silk_assert( NSQ->prev_gain_Q16 != 0 ); + + /* Initialize delayed decision states */ + ALLOC( psDelDec, psEncC->nStatesDelayedDecision, NSQ_del_dec_struct ); + silk_memset( psDelDec, 0, psEncC->nStatesDelayedDecision * sizeof( NSQ_del_dec_struct ) ); + for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + psDD->Seed = ( k + psIndices->Seed ) & 3; + psDD->SeedInit = psDD->Seed; + psDD->RD_Q10 = 0; + psDD->LF_AR_Q14 = NSQ->sLF_AR_shp_Q14; + psDD->Shape_Q14[ 0 ] = NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ]; + silk_memcpy( psDD->sLPC_Q14, NSQ->sLPC_Q14, NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + silk_memcpy( psDD->sAR2_Q14, NSQ->sAR2_Q14, sizeof( NSQ->sAR2_Q14 ) ); + } + + offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; + smpl_buf_idx = 0; /* index of oldest samples */ + + decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length ); + + /* For voiced frames limit the decision delay to lower than the pitch lag */ + if( psIndices->signalType == TYPE_VOICED ) { + for( k = 0; k < psEncC->nb_subfr; k++ ) { + decisionDelay = silk_min_int( decisionDelay, pitchL[ k ] - LTP_ORDER / 2 - 1 ); + } + } else { + if( lag > 0 ) { + decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 ); + } + } + + if( psIndices->NLSFInterpCoef_Q2 == 4 ) { + LSF_interpolation_flag = 0; + } else { + LSF_interpolation_flag = 1; + } + + ALLOC( sLTP_Q15, + psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); + ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); + ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); + ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 ); + /* Set up pointers to start of sub frame */ + pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; + NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + subfr = 0; + for( k = 0; k < psEncC->nb_subfr; k++ ) { + A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ]; + B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; + AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ]; + + /* Noise shape parameters */ + silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); + HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); + HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); + + NSQ->rewhite_flag = 0; + if( psIndices->signalType == TYPE_VOICED ) { + /* Voiced */ + lag = pitchL[ k ]; + + /* Re-whitening */ + if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { + if( k == 2 ) { + /* RESET DELAYED DECISIONS */ + /* Find winner */ + RDmin_Q10 = psDelDec[ 0 ].RD_Q10; + Winner_ind = 0; + for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) { + if( psDelDec[ i ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psDelDec[ i ].RD_Q10; + Winner_ind = i; + } + } + for( i = 0; i < psEncC->nStatesDelayedDecision; i++ ) { + if( i != Winner_ind ) { + psDelDec[ i ].RD_Q10 += ( silk_int32_MAX >> 4 ); + silk_assert( psDelDec[ i ].RD_Q10 >= 0 ); + } + } + + /* Copy final part of signals from winner state to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + last_smple_idx = smpl_buf_idx + decisionDelay; + for( i = 0; i < decisionDelay; i++ ) { + last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY; + if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY; + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gains_Q16[ 1 ] ), 14 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ]; + } + + subfr = 0; + } + + /* Rewhiten with new A coefs */ + start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; + celt_assert( start_idx > 0 ); + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], + A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); + + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + NSQ->rewhite_flag = 1; + } + } + + silk_nsq_del_dec_scale_states_sse4_1( psEncC, NSQ, psDelDec, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k, + psEncC->nStatesDelayedDecision, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay ); + + silk_noise_shape_quantizer_del_dec_sse4_1( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, + delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], + Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder, + psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay ); + + x_Q3 += psEncC->subfr_length; + pulses += psEncC->subfr_length; + pxq += psEncC->subfr_length; + } + + /* Find winner */ + RDmin_Q10 = psDelDec[ 0 ].RD_Q10; + Winner_ind = 0; + for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) { + if( psDelDec[ k ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psDelDec[ k ].RD_Q10; + Winner_ind = k; + } + } + + /* Copy final part of signals from winner state to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + psIndices->Seed = psDD->SeedInit; + last_smple_idx = smpl_buf_idx + decisionDelay; + Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 ); + for( i = 0; i < decisionDelay; i++ ) { + last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY; + if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY; + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gain_Q10 ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ]; + } + silk_memcpy( NSQ->sLPC_Q14, &psDD->sLPC_Q14[ psEncC->subfr_length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + silk_memcpy( NSQ->sAR2_Q14, psDD->sAR2_Q14, sizeof( psDD->sAR2_Q14 ) ); + + /* Update states */ + NSQ->sLF_AR_shp_Q14 = psDD->LF_AR_Q14; + NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; + + /* Save quantized speech signal */ + silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); + silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); + RESTORE_STACK; +} + +/******************************************/ +/* Noise shape quantizer for one subframe */ +/******************************************/ +static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1( + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP filter state */ + opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int subfr, /* I Subframe number */ + opus_int shapingLPCOrder, /* I Shaping LPC filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + opus_int warping_Q16, /* I */ + opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ + opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ + opus_int decisionDelay /* I */ +) +{ + opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx; + opus_int32 Winner_rand_state; + opus_int32 LTP_pred_Q14, LPC_pred_Q14, n_AR_Q14, n_LTP_Q14; + opus_int32 n_LF_Q14, r_Q10, rr_Q10, rd1_Q10, rd2_Q10, RDmin_Q10, RDmax_Q10; + opus_int32 q1_Q0, q1_Q10, q2_Q10, exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10; + opus_int32 tmp1, tmp2, sLF_AR_shp_Q14; + opus_int32 *pred_lag_ptr, *shp_lag_ptr, *psLPC_Q14; + VARDECL( NSQ_sample_pair, psSampleState ); + NSQ_del_dec_struct *psDD; + NSQ_sample_struct *psSS; + + __m128i a_Q12_0123, a_Q12_4567, a_Q12_89AB, a_Q12_CDEF; + __m128i b_Q12_0123, b_sr_Q12_0123; + SAVE_STACK; + + celt_assert( nStatesDelayedDecision > 0 ); + ALLOC( psSampleState, nStatesDelayedDecision, NSQ_sample_pair ); + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + + a_Q12_0123 = OP_CVTEPI16_EPI32_M64( a_Q12 ); + a_Q12_4567 = OP_CVTEPI16_EPI32_M64( a_Q12 + 4 ); + + if( opus_likely( predictLPCOrder == 16 ) ) { + a_Q12_89AB = OP_CVTEPI16_EPI32_M64( a_Q12 + 8 ); + a_Q12_CDEF = OP_CVTEPI16_EPI32_M64( a_Q12 + 12 ); + } + + if( signalType == TYPE_VOICED ){ + b_Q12_0123 = OP_CVTEPI16_EPI32_M64( b_Q14 ); + b_sr_Q12_0123 = _mm_shuffle_epi32( b_Q12_0123, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + } + for( i = 0; i < length; i++ ) { + /* Perform common calculations used in all states */ + + /* Long-term prediction */ + if( signalType == TYPE_VOICED ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q14 = 2; + { + __m128i tmpa, tmpb, pred_lag_ptr_tmp; + pred_lag_ptr_tmp = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) ); + pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, 0x1B ); + tmpa = _mm_mul_epi32( pred_lag_ptr_tmp, b_Q12_0123 ); + tmpa = _mm_srli_si128( tmpa, 2 ); + + pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) );/* equal shift right 4 bytes */ + pred_lag_ptr_tmp = _mm_mul_epi32( pred_lag_ptr_tmp, b_sr_Q12_0123 ); + pred_lag_ptr_tmp = _mm_srli_si128( pred_lag_ptr_tmp, 2 ); + pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpa ); + + tmpb = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 0, 3, 2 ) );/* equal shift right 8 bytes */ + pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpb ); + LTP_pred_Q14 += _mm_cvtsi128_si32( pred_lag_ptr_tmp ); + + LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); + LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */ + pred_lag_ptr++; + } + } else { + LTP_pred_Q14 = 0; + } + + /* Long-term shaping */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */ + shp_lag_ptr++; + } else { + n_LTP_Q14 = 0; + } + { + __m128i tmpa, tmpb, psLPC_Q14_tmp, a_Q12_tmp; + + for( k = 0; k < nStatesDelayedDecision; k++ ) { + /* Delayed decision state */ + psDD = &psDelDec[ k ]; + + /* Sample state */ + psSS = psSampleState[ k ]; + + /* Generate dither */ + psDD->Seed = silk_RAND( psDD->Seed ); + + /* Pointer used in short term prediction and shaping */ + psLPC_Q14 = &psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 + i ]; + /* Short-term prediction */ + silk_assert( predictLPCOrder == 10 || predictLPCOrder == 16 ); + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q14 = silk_RSHIFT( predictLPCOrder, 1 ); + + tmpb = _mm_setzero_si128(); + + /* step 1 */ + psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -3 ] ) ); /* -3, -2 , -1, 0 */ + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); /* 0, -1, -2, -3 */ + tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_0123 ); /* 0, -1, -2, -3 * 0123 -> 0*0, 2*-2 */ + + tmpa = _mm_srli_epi64( tmpa, 16 ); + tmpb = _mm_add_epi32( tmpb, tmpa ); + + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + a_Q12_tmp = _mm_shuffle_epi32( a_Q12_0123, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); /* 1*-1, 3*-3 */ + psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 ); + tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp ); + + /* step 2 */ + psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -7 ] ) ); + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); + tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_4567 ); + tmpa = _mm_srli_epi64( tmpa, 16 ); + tmpb = _mm_add_epi32( tmpb, tmpa ); + + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + a_Q12_tmp = _mm_shuffle_epi32( a_Q12_4567, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); + psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 ); + tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp ); + + if ( opus_likely( predictLPCOrder == 16 ) ) + { + /* step 3 */ + psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -11 ] ) ); + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); + tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_89AB ); + tmpa = _mm_srli_epi64( tmpa, 16 ); + tmpb = _mm_add_epi32( tmpb, tmpa ); + + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + a_Q12_tmp = _mm_shuffle_epi32( a_Q12_89AB, _MM_SHUFFLE(0, 3, 2, 1 ) );/* equal shift right 4 bytes */ + psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); + psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 ); + tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp ); + + /* setp 4 */ + psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -15 ] ) ); + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); + tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_CDEF ); + tmpa = _mm_srli_epi64( tmpa, 16 ); + tmpb = _mm_add_epi32( tmpb, tmpa ); + + psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + a_Q12_tmp = _mm_shuffle_epi32( a_Q12_CDEF, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */ + psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); + psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 ); + tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp ); + + /* add at last */ + /* equal shift right 8 bytes*/ + tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) ); + tmpb = _mm_add_epi32( tmpb, tmpa ); + LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb ); + } + else + { + /* add at last */ + tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) ); /* equal shift right 8 bytes*/ + tmpb = _mm_add_epi32( tmpb, tmpa ); + LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb ); + + LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -8 ], a_Q12[ 8 ] ); + LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -9 ], a_Q12[ 9 ] ); + } + + LPC_pred_Q14 = silk_LSHIFT( LPC_pred_Q14, 4 ); /* Q10 -> Q14 */ + + /* Noise shape feedback */ + silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ + /* Output of lowpass section */ + tmp2 = silk_SMLAWB( psLPC_Q14[ 0 ], psDD->sAR2_Q14[ 0 ], warping_Q16 ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ 0 ], psDD->sAR2_Q14[ 1 ] - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ 0 ] = tmp2; + n_AR_Q14 = silk_RSHIFT( shapingLPCOrder, 1 ); + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ 0 ] ); + /* Loop over allpass sections */ + for( j = 2; j < shapingLPCOrder; j += 2 ) { + /* Output of allpass section */ + tmp2 = silk_SMLAWB( psDD->sAR2_Q14[ j - 1 ], psDD->sAR2_Q14[ j + 0 ] - tmp1, warping_Q16 ); + psDD->sAR2_Q14[ j - 1 ] = tmp1; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ j - 1 ] ); + /* Output of allpass section */ + tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ j + 0 ], psDD->sAR2_Q14[ j + 1 ] - tmp2, warping_Q16 ); + psDD->sAR2_Q14[ j + 0 ] = tmp2; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ j ] ); + } + psDD->sAR2_Q14[ shapingLPCOrder - 1 ] = tmp1; + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ shapingLPCOrder - 1 ] ); + + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 1 ); /* Q11 -> Q12 */ + n_AR_Q14 = silk_SMLAWB( n_AR_Q14, psDD->LF_AR_Q14, Tilt_Q14 ); /* Q12 */ + n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 2 ); /* Q12 -> Q14 */ + + n_LF_Q14 = silk_SMULWB( psDD->Shape_Q14[ *smpl_buf_idx ], LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_SMLAWT( n_LF_Q14, psDD->LF_AR_Q14, LF_shp_Q14 ); /* Q12 */ + n_LF_Q14 = silk_LSHIFT( n_LF_Q14, 2 ); /* Q12 -> Q14 */ + + /* Input minus prediction plus noise feedback */ + /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */ + tmp1 = silk_ADD32( n_AR_Q14, n_LF_Q14 ); /* Q14 */ + tmp2 = silk_ADD32( n_LTP_Q14, LPC_pred_Q14 ); /* Q13 */ + tmp1 = silk_SUB32( tmp2, tmp1 ); /* Q13 */ + tmp1 = silk_RSHIFT_ROUND( tmp1, 4 ); /* Q10 */ + + r_Q10 = silk_SUB32( x_Q10[ i ], tmp1 ); /* residual error Q10 */ + + /* Flip sign depending on dither */ + if ( psDD->Seed < 0 ) { + r_Q10 = -r_Q10; + } + r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + q1_Q10 = silk_SUB32( r_Q10, offset_Q10 ); + q1_Q0 = silk_RSHIFT( q1_Q10, 10 ); + if( q1_Q0 > 0 ) { + q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == 0 ) { + q1_Q10 = offset_Q10; + q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else if( q1_Q0 == -1 ) { + q2_Q10 = offset_Q10; + q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 ); + } else { /* q1_Q0 < -1 */ + q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 ); + q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 ); + q2_Q10 = silk_ADD32( q1_Q10, 1024 ); + rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 ); + rd2_Q10 = silk_SMULBB( -q2_Q10, Lambda_Q10 ); + } + rr_Q10 = silk_SUB32( r_Q10, q1_Q10 ); + rd1_Q10 = silk_RSHIFT( silk_SMLABB( rd1_Q10, rr_Q10, rr_Q10 ), 10 ); + rr_Q10 = silk_SUB32( r_Q10, q2_Q10 ); + rd2_Q10 = silk_RSHIFT( silk_SMLABB( rd2_Q10, rr_Q10, rr_Q10 ), 10 ); + + if( rd1_Q10 < rd2_Q10 ) { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 0 ].Q_Q10 = q1_Q10; + psSS[ 1 ].Q_Q10 = q2_Q10; + } else { + psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 ); + psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 ); + psSS[ 0 ].Q_Q10 = q2_Q10; + psSS[ 1 ].Q_Q10 = q1_Q10; + } + + /* Update states for best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 0 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 ); + psSS[ 0 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 0 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 0 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 0 ].xq_Q14 = xq_Q14; + + /* Update states for second best quantization */ + + /* Quantized excitation */ + exc_Q14 = silk_LSHIFT32( psSS[ 1 ].Q_Q10, 4 ); + if ( psDD->Seed < 0 ) { + exc_Q14 = -exc_Q14; + } + + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 ); + xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 ); + + /* Update states */ + sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 ); + psSS[ 1 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 ); + psSS[ 1 ].LF_AR_Q14 = sLF_AR_shp_Q14; + psSS[ 1 ].LPC_exc_Q14 = LPC_exc_Q14; + psSS[ 1 ].xq_Q14 = xq_Q14; + } + } + *smpl_buf_idx = ( *smpl_buf_idx - 1 ) % DECISION_DELAY; + if( *smpl_buf_idx < 0 ) *smpl_buf_idx += DECISION_DELAY; + last_smple_idx = ( *smpl_buf_idx + decisionDelay ) % DECISION_DELAY; + + /* Find winner */ + RDmin_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + Winner_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + if( psSampleState[ k ][ 0 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + Winner_ind = k; + } + } + + /* Increase RD values of expired states */ + Winner_rand_state = psDelDec[ Winner_ind ].RandState[ last_smple_idx ]; + for( k = 0; k < nStatesDelayedDecision; k++ ) { + if( psDelDec[ k ].RandState[ last_smple_idx ] != Winner_rand_state ) { + psSampleState[ k ][ 0 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 0 ].RD_Q10, silk_int32_MAX >> 4 ); + psSampleState[ k ][ 1 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 1 ].RD_Q10, silk_int32_MAX >> 4 ); + silk_assert( psSampleState[ k ][ 0 ].RD_Q10 >= 0 ); + } + } + + /* Find worst in first set and best in second set */ + RDmax_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10; + RDmin_Q10 = psSampleState[ 0 ][ 1 ].RD_Q10; + RDmax_ind = 0; + RDmin_ind = 0; + for( k = 1; k < nStatesDelayedDecision; k++ ) { + /* find worst in first set */ + if( psSampleState[ k ][ 0 ].RD_Q10 > RDmax_Q10 ) { + RDmax_Q10 = psSampleState[ k ][ 0 ].RD_Q10; + RDmax_ind = k; + } + /* find best in second set */ + if( psSampleState[ k ][ 1 ].RD_Q10 < RDmin_Q10 ) { + RDmin_Q10 = psSampleState[ k ][ 1 ].RD_Q10; + RDmin_ind = k; + } + } + + /* Replace a state if best from second set outperforms worst in first set */ + if( RDmin_Q10 < RDmax_Q10 ) { + silk_memcpy( ( (opus_int32 *)&psDelDec[ RDmax_ind ] ) + i, + ( (opus_int32 *)&psDelDec[ RDmin_ind ] ) + i, sizeof( NSQ_del_dec_struct ) - i * sizeof( opus_int32) ); + silk_memcpy( &psSampleState[ RDmax_ind ][ 0 ], &psSampleState[ RDmin_ind ][ 1 ], sizeof( NSQ_sample_struct ) ); + } + + /* Write samples from winner to output and long-term filter states */ + psDD = &psDelDec[ Winner_ind ]; + if( subfr > 0 || i >= decisionDelay ) { + pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 ); + xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( + silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], delayedGain_Q10[ last_smple_idx ] ), 8 ) ); + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDD->Shape_Q14[ last_smple_idx ]; + sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDD->Pred_Q15[ last_smple_idx ]; + } + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Update states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + psSS = &psSampleState[ k ][ 0 ]; + psDD->LF_AR_Q14 = psSS->LF_AR_Q14; + psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ] = psSS->xq_Q14; + psDD->Xq_Q14[ *smpl_buf_idx ] = psSS->xq_Q14; + psDD->Q_Q10[ *smpl_buf_idx ] = psSS->Q_Q10; + psDD->Pred_Q15[ *smpl_buf_idx ] = silk_LSHIFT32( psSS->LPC_exc_Q14, 1 ); + psDD->Shape_Q14[ *smpl_buf_idx ] = psSS->sLTP_shp_Q14; + psDD->Seed = silk_ADD32_ovflw( psDD->Seed, silk_RSHIFT_ROUND( psSS->Q_Q10, 10 ) ); + psDD->RandState[ *smpl_buf_idx ] = psDD->Seed; + psDD->RD_Q10 = psSS->RD_Q10; + } + delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10; + } + /* Update LPC states */ + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + silk_memcpy( psDD->sLPC_Q14, &psDD->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); + } + RESTORE_STACK; +} + +static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */ + const opus_int32 x_Q3[], /* I Input in Q3 */ + opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ + const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I Subframe number */ + opus_int nStatesDelayedDecision, /* I Number of del dec states */ + const opus_int LTP_scale_Q14, /* I LTP state scaling */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type, /* I Signal type */ + const opus_int decisionDelay /* I Decision delay */ +) +{ + opus_int i, k, lag; + opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23; + NSQ_del_dec_struct *psDD; + __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1; + + lag = pitchL[ subfr ]; + inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); + + silk_assert( inv_gain_Q31 != 0 ); + + /* Calculate gain adjustment factor */ + if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { + gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); + } else { + gain_adj_Q16 = (opus_int32)1 << 16; + } + + /* Scale input */ + inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 ); + + /* prepare inv_gain_Q23 in packed 4 32-bits */ + xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23); + + for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) { + xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) ); + /* equal shift right 4 bytes*/ + xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 ); + xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 ); + + xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 ); + xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 ); + + xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC ); + + _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ])), xmm_x_Q3_x2x0 ); + } + + for( ; i < psEncC->subfr_length; i++ ) { + x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 ); + } + + /* Save inverse gain */ + NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; + + /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ + if( NSQ->rewhite_flag ) { + if( subfr == 0 ) { + /* Do LTP downscaling */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); + } + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + silk_assert( i < MAX_FRAME_LENGTH ); + sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] ); + } + } + + /* Adjust for changing gain */ + if( gain_adj_Q16 != (opus_int32)1 << 16 ) { + /* Scale long-term shaping state */ + { + __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1; + + /* prepare gain_adj_Q16 in packed 4 32-bits */ + xmm_gain_adj_Q16 = _mm_set1_epi32( gain_adj_Q16 ); + + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 ) + { + xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) ); + /* equal shift right 4 bytes*/ + xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 ); + xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 ); + xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC ); + + _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 ); + } + + for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + for( k = 0; k < nStatesDelayedDecision; k++ ) { + psDD = &psDelDec[ k ]; + + /* Scale scalar states */ + psDD->LF_AR_Q14 = silk_SMULWW( gain_adj_Q16, psDD->LF_AR_Q14 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + psDD->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sLPC_Q14[ i ] ); + } + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + psDD->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sAR2_Q14[ i ] ); + } + for( i = 0; i < DECISION_DELAY; i++ ) { + psDD->Pred_Q15[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Pred_Q15[ i ] ); + psDD->Shape_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Shape_Q14[ i ] ); + } + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_sse4_1.c new file mode 100644 index 000000000..b0315e35f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/NSQ_sse4_1.c @@ -0,0 +1,719 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "main.h" +#include "celt/x86/x86cpu.h" +#include "stack_alloc.h" + +static OPUS_INLINE void silk_nsq_scale_states_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + const opus_int32 x_Q3[], /* I input in Q3 */ + opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */ + const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I subframe number */ + const opus_int LTP_scale_Q14, /* I */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type /* I Signal type */ +); + +static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1( + silk_nsq_state *NSQ, /* I/O NSQ state */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_sc_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP state */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int32 table[][4] /* I */ +); + +void silk_NSQ_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) +{ + opus_int k, lag, start_idx, LSF_interpolation_flag; + const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; + opus_int16 *pxq; + VARDECL( opus_int32, sLTP_Q15 ); + VARDECL( opus_int16, sLTP ); + opus_int32 HarmShapeFIRPacked_Q14; + opus_int offset_Q10; + VARDECL( opus_int32, x_sc_Q10 ); + + opus_int32 table[ 64 ][ 4 ]; + opus_int32 tmp1; + opus_int32 q1_Q10, q2_Q10, rd1_Q20, rd2_Q20; + + SAVE_STACK; + + NSQ->rand_seed = psIndices->Seed; + + /* Set unvoiced lag to the previous one, overwrite later for voiced */ + lag = NSQ->lagPrev; + + silk_assert( NSQ->prev_gain_Q16 != 0 ); + + offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; + + /* 0 */ + q1_Q10 = offset_Q10; + q2_Q10 = offset_Q10 + ( 1024 - QUANT_LEVEL_ADJUST_Q10 ); + rd1_Q20 = q1_Q10 * Lambda_Q10; + rd2_Q20 = q2_Q10 * Lambda_Q10; + + table[ 32 ][ 0 ] = q1_Q10; + table[ 32 ][ 1 ] = q2_Q10; + table[ 32 ][ 2 ] = 2 * (q1_Q10 - q2_Q10); + table[ 32 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10); + + /* -1 */ + q1_Q10 = offset_Q10 - ( 1024 - QUANT_LEVEL_ADJUST_Q10 ); + q2_Q10 = offset_Q10; + rd1_Q20 = - q1_Q10 * Lambda_Q10; + rd2_Q20 = q2_Q10 * Lambda_Q10; + + table[ 31 ][ 0 ] = q1_Q10; + table[ 31 ][ 1 ] = q2_Q10; + table[ 31 ][ 2 ] = 2 * (q1_Q10 - q2_Q10); + table[ 31 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10); + + /* > 0 */ + for (k = 1; k <= 31; k++) + { + tmp1 = offset_Q10 + silk_LSHIFT( k, 10 ); + + q1_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10; + q2_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10 + 1024; + rd1_Q20 = q1_Q10 * Lambda_Q10; + rd2_Q20 = q2_Q10 * Lambda_Q10; + + table[ 32 + k ][ 0 ] = q1_Q10; + table[ 32 + k ][ 1 ] = q2_Q10; + table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10); + table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10); + } + + /* < -1 */ + for (k = -32; k <= -2; k++) + { + tmp1 = offset_Q10 + silk_LSHIFT( k, 10 ); + + q1_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10; + q2_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10 + 1024; + rd1_Q20 = - q1_Q10 * Lambda_Q10; + rd2_Q20 = - q2_Q10 * Lambda_Q10; + + table[ 32 + k ][ 0 ] = q1_Q10; + table[ 32 + k ][ 1 ] = q2_Q10; + table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10); + table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10); + } + + if( psIndices->NLSFInterpCoef_Q2 == 4 ) { + LSF_interpolation_flag = 0; + } else { + LSF_interpolation_flag = 1; + } + + ALLOC( sLTP_Q15, + psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); + ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); + ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); + /* Set up pointers to start of sub frame */ + NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; + for( k = 0; k < psEncC->nb_subfr; k++ ) { + A_Q12 = &PredCoef_Q12[ (( k >> 1 ) | ( 1 - LSF_interpolation_flag )) * MAX_LPC_ORDER ]; + B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; + AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ]; + + /* Noise shape parameters */ + silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); + HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); + HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); + + NSQ->rewhite_flag = 0; + if( psIndices->signalType == TYPE_VOICED ) { + /* Voiced */ + lag = pitchL[ k ]; + + /* Re-whitening */ + if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { + /* Rewhiten with new A coefs */ + start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; + celt_assert( start_idx > 0 ); + + silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], + A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); + + NSQ->rewhite_flag = 1; + NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; + } + } + + silk_nsq_scale_states_sse4_1( psEncC, NSQ, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType ); + + if ( opus_likely( ( 10 == psEncC->shapingLPCOrder ) && ( 16 == psEncC->predictLPCOrder) ) ) + { + silk_noise_shape_quantizer_10_16_sse4_1( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14, + AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ], + offset_Q10, psEncC->subfr_length, &(table[32]) ); + } + else + { + silk_noise_shape_quantizer( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14, + AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ], Lambda_Q10, + offset_Q10, psEncC->subfr_length, psEncC->shapingLPCOrder, psEncC->predictLPCOrder, psEncC->arch ); + } + + x_Q3 += psEncC->subfr_length; + pulses += psEncC->subfr_length; + pxq += psEncC->subfr_length; + } + + /* Update lagPrev for next frame */ + NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; + + /* Save quantized speech and noise shaping signals */ + silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); + silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); + RESTORE_STACK; +} + +/***********************************/ +/* silk_noise_shape_quantizer_10_16 */ +/***********************************/ +static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1( + silk_nsq_state *NSQ, /* I/O NSQ state */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_sc_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP state */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int32 table[][4] /* I */ +) +{ + opus_int i; + opus_int32 LTP_pred_Q13, LPC_pred_Q10, n_AR_Q12, n_LTP_Q13; + opus_int32 n_LF_Q12, r_Q10, q1_Q0, q1_Q10, q2_Q10; + opus_int32 exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10; + opus_int32 tmp1, tmp2, sLF_AR_shp_Q14; + opus_int32 *psLPC_Q14, *shp_lag_ptr, *pred_lag_ptr; + + __m128i xmm_tempa, xmm_tempb; + + __m128i xmm_one; + + __m128i psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF; + __m128i psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF; + __m128i a_Q12_01234567, a_Q12_89ABCDEF; + + __m128i sAR2_Q14_hi_76543210, sAR2_Q14_lo_76543210; + __m128i AR_shp_Q13_76543210; + + shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; + pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; + Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); + + /* Set up short term AR state */ + psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 ]; + + sLF_AR_shp_Q14 = NSQ->sLF_AR_shp_Q14; + xq_Q14 = psLPC_Q14[ 0 ]; + LTP_pred_Q13 = 0; + + /* load a_Q12 */ + xmm_one = _mm_set_epi8( 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 ); + + /* load a_Q12[0] - a_Q12[7] */ + a_Q12_01234567 = _mm_loadu_si128( (__m128i *)(&a_Q12[ 0 ] ) ); + /* load a_Q12[ 8 ] - a_Q12[ 15 ] */ + a_Q12_89ABCDEF = _mm_loadu_si128( (__m128i *)(&a_Q12[ 8 ] ) ); + + a_Q12_01234567 = _mm_shuffle_epi8( a_Q12_01234567, xmm_one ); + a_Q12_89ABCDEF = _mm_shuffle_epi8( a_Q12_89ABCDEF, xmm_one ); + + /* load AR_shp_Q13 */ + AR_shp_Q13_76543210 = _mm_loadu_si128( (__m128i *)(&AR_shp_Q13[0] ) ); + + /* load psLPC_Q14 */ + xmm_one = _mm_set_epi8(15, 14, 11, 10, 7, 6, 3, 2, 13, 12, 9, 8, 5, 4, 1, 0 ); + + xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-16]) ); + xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-12]) ); + + xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one ); + xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one ); + + psLPC_Q14_hi_89ABCDEF = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb ); + psLPC_Q14_lo_89ABCDEF = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb ); + + xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -8 ]) ); + xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -4 ]) ); + + xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one ); + xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one ); + + psLPC_Q14_hi_01234567 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb ); + psLPC_Q14_lo_01234567 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb ); + + /* load sAR2_Q14 */ + xmm_tempa = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 0 ]) ) ); + xmm_tempb = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 4 ]) ) ); + + xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one ); + xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one ); + + sAR2_Q14_hi_76543210 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb ); + sAR2_Q14_lo_76543210 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb ); + + /* prepare 1 in 8 * 16bit */ + xmm_one = _mm_set1_epi16(1); + + for( i = 0; i < length; i++ ) + { + /* Short-term prediction */ + __m128i xmm_hi_07, xmm_hi_8F, xmm_lo_07, xmm_lo_8F; + + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LPC_pred_Q10 = 8; /* silk_RSHIFT( predictLPCOrder, 1 ); */ + + /* shift psLPC_Q14 */ + psLPC_Q14_hi_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF, 2 ); + psLPC_Q14_lo_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF, 2 ); + + psLPC_Q14_hi_01234567 = _mm_srli_si128( psLPC_Q14_hi_01234567, 2 ); + psLPC_Q14_lo_01234567 = _mm_srli_si128( psLPC_Q14_lo_01234567, 2 ); + + psLPC_Q14_hi_01234567 = _mm_insert_epi16( psLPC_Q14_hi_01234567, (xq_Q14 >> 16), 7 ); + psLPC_Q14_lo_01234567 = _mm_insert_epi16( psLPC_Q14_lo_01234567, (xq_Q14), 7 ); + + /* high part, use pmaddwd, results in 4 32-bit */ + xmm_hi_07 = _mm_madd_epi16( psLPC_Q14_hi_01234567, a_Q12_01234567 ); + xmm_hi_8F = _mm_madd_epi16( psLPC_Q14_hi_89ABCDEF, a_Q12_89ABCDEF ); + + /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed, _mm_srai_epi16(psLPC_Q14_lo_01234567, 15) */ + xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_01234567 ); + xmm_tempb = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_89ABCDEF ); + + xmm_tempa = _mm_and_si128( xmm_tempa, a_Q12_01234567 ); + xmm_tempb = _mm_and_si128( xmm_tempb, a_Q12_89ABCDEF ); + + xmm_lo_07 = _mm_mulhi_epi16( psLPC_Q14_lo_01234567, a_Q12_01234567 ); + xmm_lo_8F = _mm_mulhi_epi16( psLPC_Q14_lo_89ABCDEF, a_Q12_89ABCDEF ); + + xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa ); + xmm_lo_8F = _mm_add_epi16( xmm_lo_8F, xmm_tempb ); + + xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one ); + xmm_lo_8F = _mm_madd_epi16( xmm_lo_8F, xmm_one ); + + /* accumulate */ + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_hi_8F ); + xmm_lo_07 = _mm_add_epi32( xmm_lo_07, xmm_lo_8F ); + + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 ); + + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) ); + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) ); + + LPC_pred_Q10 += _mm_cvtsi128_si32( xmm_hi_07 ); + + /* Long-term prediction */ + if ( opus_likely( signalType == TYPE_VOICED ) ) { + /* Unrolled loop */ + /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ + LTP_pred_Q13 = 2; + { + __m128i b_Q14_3210, b_Q14_0123, pred_lag_ptr_0123; + + b_Q14_3210 = OP_CVTEPI16_EPI32_M64( b_Q14 ); + b_Q14_0123 = _mm_shuffle_epi32( b_Q14_3210, 0x1B ); + + /* loaded: [0] [-1] [-2] [-3] */ + pred_lag_ptr_0123 = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) ); + /* shuffle to [-3] [-2] [-1] [0] and to new xmm */ + xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, 0x1B ); + /*64-bit multiply, a[2] * b[-2], a[0] * b[0] */ + xmm_tempa = _mm_mul_epi32( xmm_tempa, b_Q14_3210 ); + /* right shift 2 bytes (16 bits), zero extended */ + xmm_tempa = _mm_srli_si128( xmm_tempa, 2 ); + + /* a[1] * b[-1], a[3] * b[-3] */ + pred_lag_ptr_0123 = _mm_mul_epi32( pred_lag_ptr_0123, b_Q14_0123 ); + pred_lag_ptr_0123 = _mm_srli_si128( pred_lag_ptr_0123, 2 ); + + pred_lag_ptr_0123 = _mm_add_epi32( pred_lag_ptr_0123, xmm_tempa ); + /* equal shift right 8 bytes*/ + xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, _MM_SHUFFLE( 0, 0, 3, 2 ) ); + xmm_tempa = _mm_add_epi32( xmm_tempa, pred_lag_ptr_0123 ); + + LTP_pred_Q13 += _mm_cvtsi128_si32( xmm_tempa ); + + LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); + pred_lag_ptr++; + } + } + + /* Noise shape feedback */ + NSQ->sAR2_Q14[ 9 ] = NSQ->sAR2_Q14[ 8 ]; + NSQ->sAR2_Q14[ 8 ] = _mm_cvtsi128_si32( _mm_srli_si128(_mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 ), 12 ) ); + + sAR2_Q14_hi_76543210 = _mm_slli_si128( sAR2_Q14_hi_76543210, 2 ); + sAR2_Q14_lo_76543210 = _mm_slli_si128( sAR2_Q14_lo_76543210, 2 ); + + sAR2_Q14_hi_76543210 = _mm_insert_epi16( sAR2_Q14_hi_76543210, (xq_Q14 >> 16), 0 ); + sAR2_Q14_lo_76543210 = _mm_insert_epi16( sAR2_Q14_lo_76543210, (xq_Q14), 0 ); + + /* high part, use pmaddwd, results in 4 32-bit */ + xmm_hi_07 = _mm_madd_epi16( sAR2_Q14_hi_76543210, AR_shp_Q13_76543210 ); + + /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed,_mm_srai_epi16(sAR2_Q14_lo_76543210, 15) */ + xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), sAR2_Q14_lo_76543210 ); + xmm_tempa = _mm_and_si128( xmm_tempa, AR_shp_Q13_76543210 ); + + xmm_lo_07 = _mm_mulhi_epi16( sAR2_Q14_lo_76543210, AR_shp_Q13_76543210 ); + xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa ); + + xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one ); + + /* accumulate */ + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 ); + + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) ); + xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) ); + + n_AR_Q12 = 5 + _mm_cvtsi128_si32( xmm_hi_07 ); + + n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 8 ], AR_shp_Q13[ 8 ] ); + n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 9 ], AR_shp_Q13[ 9 ] ); + + n_AR_Q12 = silk_LSHIFT32( n_AR_Q12, 1 ); /* Q11 -> Q12 */ + n_AR_Q12 = silk_SMLAWB( n_AR_Q12, sLF_AR_shp_Q14, Tilt_Q14 ); + + n_LF_Q12 = silk_SMULWB( NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - 1 ], LF_shp_Q14 ); + n_LF_Q12 = silk_SMLAWT( n_LF_Q12, sLF_AR_shp_Q14, LF_shp_Q14 ); + + silk_assert( lag > 0 || signalType != TYPE_VOICED ); + + /* Combine prediction and noise shaping signals */ + tmp1 = silk_SUB32( silk_LSHIFT32( LPC_pred_Q10, 2 ), n_AR_Q12 ); /* Q12 */ + tmp1 = silk_SUB32( tmp1, n_LF_Q12 ); /* Q12 */ + if( lag > 0 ) { + /* Symmetric, packed FIR coefficients */ + n_LTP_Q13 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); + n_LTP_Q13 = silk_SMLAWT( n_LTP_Q13, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); + n_LTP_Q13 = silk_LSHIFT( n_LTP_Q13, 1 ); + shp_lag_ptr++; + + tmp2 = silk_SUB32( LTP_pred_Q13, n_LTP_Q13 ); /* Q13 */ + tmp1 = silk_ADD_LSHIFT32( tmp2, tmp1, 1 ); /* Q13 */ + tmp1 = silk_RSHIFT_ROUND( tmp1, 3 ); /* Q10 */ + } else { + tmp1 = silk_RSHIFT_ROUND( tmp1, 2 ); /* Q10 */ + } + + r_Q10 = silk_SUB32( x_sc_Q10[ i ], tmp1 ); /* residual error Q10 */ + + /* Generate dither */ + NSQ->rand_seed = silk_RAND( NSQ->rand_seed ); + + /* Flip sign depending on dither */ + tmp2 = -r_Q10; + if ( NSQ->rand_seed < 0 ) r_Q10 = tmp2; + + r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 ); + + /* Find two quantization level candidates and measure their rate-distortion */ + q1_Q10 = silk_SUB32( r_Q10, offset_Q10 ); + q1_Q0 = silk_RSHIFT( q1_Q10, 10 ); + + q1_Q10 = table[q1_Q0][0]; + q2_Q10 = table[q1_Q0][1]; + + if (r_Q10 * table[q1_Q0][2] - table[q1_Q0][3] < 0) + { + q1_Q10 = q2_Q10; + } + + pulses[ i ] = (opus_int8)silk_RSHIFT_ROUND( q1_Q10, 10 ); + + /* Excitation */ + exc_Q14 = silk_LSHIFT( q1_Q10, 4 ); + + tmp2 = -exc_Q14; + if ( NSQ->rand_seed < 0 ) exc_Q14 = tmp2; + + /* Add predictions */ + LPC_exc_Q14 = silk_ADD_LSHIFT32( exc_Q14, LTP_pred_Q13, 1 ); + xq_Q14 = silk_ADD_LSHIFT32( LPC_exc_Q14, LPC_pred_Q10, 4 ); + + /* Update states */ + psLPC_Q14++; + *psLPC_Q14 = xq_Q14; + sLF_AR_shp_Q14 = silk_SUB_LSHIFT32( xq_Q14, n_AR_Q12, 2 ); + + NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx ] = silk_SUB_LSHIFT32( sLF_AR_shp_Q14, n_LF_Q12, 2 ); + sLTP_Q15[ NSQ->sLTP_buf_idx ] = silk_LSHIFT( LPC_exc_Q14, 1 ); + NSQ->sLTP_shp_buf_idx++; + NSQ->sLTP_buf_idx++; + + /* Make dither dependent on quantized signal */ + NSQ->rand_seed = silk_ADD32_ovflw( NSQ->rand_seed, pulses[ i ] ); + } + + NSQ->sLF_AR_shp_Q14 = sLF_AR_shp_Q14; + + /* Scale XQ back to normal level before saving */ + psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH ]; + + /* write back sAR2_Q14 */ + xmm_tempa = _mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 ); + xmm_tempb = _mm_unpacklo_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 ); + _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 4 ]), xmm_tempa ); + _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 0 ]), xmm_tempb ); + + /* xq[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) ); */ + { + __m128i xmm_Gain_Q10; + __m128i xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, xmm_xq_Q14_7654, xmm_xq_Q14_x7x5; + + /* prepare (1 << 7) in packed 4 32-bits */ + xmm_tempa = _mm_set1_epi32( (1 << 7) ); + + /* prepare Gain_Q10 in packed 4 32-bits */ + xmm_Gain_Q10 = _mm_set1_epi32( Gain_Q10 ); + + /* process xq */ + for (i = 0; i < length - 7; i += 8) + { + xmm_xq_Q14_3210 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 0 ] ) ) ); + xmm_xq_Q14_7654 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 4 ] ) ) ); + + /* equal shift right 4 bytes*/ + xmm_xq_Q14_x3x1 = _mm_shuffle_epi32( xmm_xq_Q14_3210, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + /* equal shift right 4 bytes*/ + xmm_xq_Q14_x7x5 = _mm_shuffle_epi32( xmm_xq_Q14_7654, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_xq_Q14_3210 = _mm_mul_epi32( xmm_xq_Q14_3210, xmm_Gain_Q10 ); + xmm_xq_Q14_x3x1 = _mm_mul_epi32( xmm_xq_Q14_x3x1, xmm_Gain_Q10 ); + xmm_xq_Q14_7654 = _mm_mul_epi32( xmm_xq_Q14_7654, xmm_Gain_Q10 ); + xmm_xq_Q14_x7x5 = _mm_mul_epi32( xmm_xq_Q14_x7x5, xmm_Gain_Q10 ); + + xmm_xq_Q14_3210 = _mm_srli_epi64( xmm_xq_Q14_3210, 16 ); + xmm_xq_Q14_x3x1 = _mm_slli_epi64( xmm_xq_Q14_x3x1, 16 ); + xmm_xq_Q14_7654 = _mm_srli_epi64( xmm_xq_Q14_7654, 16 ); + xmm_xq_Q14_x7x5 = _mm_slli_epi64( xmm_xq_Q14_x7x5, 16 ); + + xmm_xq_Q14_3210 = _mm_blend_epi16( xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, 0xCC ); + xmm_xq_Q14_7654 = _mm_blend_epi16( xmm_xq_Q14_7654, xmm_xq_Q14_x7x5, 0xCC ); + + /* silk_RSHIFT_ROUND(xq, 8) */ + xmm_xq_Q14_3210 = _mm_add_epi32( xmm_xq_Q14_3210, xmm_tempa ); + xmm_xq_Q14_7654 = _mm_add_epi32( xmm_xq_Q14_7654, xmm_tempa ); + + xmm_xq_Q14_3210 = _mm_srai_epi32( xmm_xq_Q14_3210, 8 ); + xmm_xq_Q14_7654 = _mm_srai_epi32( xmm_xq_Q14_7654, 8 ); + + /* silk_SAT16 */ + xmm_xq_Q14_3210 = _mm_packs_epi32( xmm_xq_Q14_3210, xmm_xq_Q14_7654 ); + + /* save to xq */ + _mm_storeu_si128( (__m128i *)(&xq[ i ] ), xmm_xq_Q14_3210 ); + } + } + for ( ; i < length; i++) + { + xq[i] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) ); + } + + /* Update LPC synth buffer */ + silk_memcpy( NSQ->sLPC_Q14, &NSQ->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); +} + +static OPUS_INLINE void silk_nsq_scale_states_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + const opus_int32 x_Q3[], /* I input in Q3 */ + opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */ + const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */ + opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ + opus_int subfr, /* I subframe number */ + const opus_int LTP_scale_Q14, /* I */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ + const opus_int signal_type /* I Signal type */ +) +{ + opus_int i, lag; + opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23; + __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1; + + lag = pitchL[ subfr ]; + inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); + silk_assert( inv_gain_Q31 != 0 ); + + /* Calculate gain adjustment factor */ + if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { + gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); + } else { + gain_adj_Q16 = (opus_int32)1 << 16; + } + + /* Scale input */ + inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 ); + + /* prepare inv_gain_Q23 in packed 4 32-bits */ + xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23); + + for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) { + xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) ); + + /* equal shift right 4 bytes*/ + xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 ); + xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 ); + + xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 ); + xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 ); + + xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC ); + + _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ] ) ), xmm_x_Q3_x2x0 ); + } + + for( ; i < psEncC->subfr_length; i++ ) { + x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 ); + } + + /* Save inverse gain */ + NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; + + /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ + if( NSQ->rewhite_flag ) { + if( subfr == 0 ) { + /* Do LTP downscaling */ + inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); + } + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + silk_assert( i < MAX_FRAME_LENGTH ); + sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] ); + } + } + + /* Adjust for changing gain */ + if( gain_adj_Q16 != (opus_int32)1 << 16 ) { + /* Scale long-term shaping state */ + __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1; + + /* prepare gain_adj_Q16 in packed 4 32-bits */ + xmm_gain_adj_Q16 = _mm_set1_epi32(gain_adj_Q16); + + for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 ) + { + xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) ); + /* equal shift right 4 bytes*/ + xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 ); + xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 ); + xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 ); + + xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC ); + + _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 ); + } + + for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { + NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); + } + + /* Scale long-term prediction state */ + if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { + for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) { + sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); + } + } + + NSQ->sLF_AR_shp_Q14 = silk_SMULWW( gain_adj_Q16, NSQ->sLF_AR_shp_Q14 ); + + /* Scale short-term prediction and shaping states */ + for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { + NSQ->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLPC_Q14[ i ] ); + } + for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { + NSQ->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sAR2_Q14[ i ] ); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/SigProc_FIX_sse.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/SigProc_FIX_sse.h new file mode 100644 index 000000000..61efa8da4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/SigProc_FIX_sse.h @@ -0,0 +1,94 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIGPROC_FIX_SSE_H +#define SIGPROC_FIX_SSE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) +void silk_burg_modified_sse4_1( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */ +); + +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define silk_burg_modified(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch) \ + ((void)(arch), silk_burg_modified_sse4_1(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch)) + +#else + +extern void (*const SILK_BURG_MODIFIED_IMPL[OPUS_ARCHMASK + 1])( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */); + +# define silk_burg_modified(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch) \ + ((*SILK_BURG_MODIFIED_IMPL[(arch) & OPUS_ARCHMASK])(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch)) + +#endif + +opus_int64 silk_inner_prod16_aligned_64_sse4_1( + const opus_int16 *inVec1, + const opus_int16 *inVec2, + const opus_int len +); + + +#if defined(OPUS_X86_PRESUME_SSE4_1) + +#define silk_inner_prod16_aligned_64(inVec1, inVec2, len, arch) \ + ((void)(arch),silk_inner_prod16_aligned_64_sse4_1(inVec1, inVec2, len)) + +#else + +extern opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[OPUS_ARCHMASK + 1])( + const opus_int16 *inVec1, + const opus_int16 *inVec2, + const opus_int len); + +# define silk_inner_prod16_aligned_64(inVec1, inVec2, len, arch) \ + ((*SILK_INNER_PROD16_ALIGNED_64_IMPL[(arch) & OPUS_ARCHMASK])(inVec1, inVec2, len)) + +#endif +#endif +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VAD_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VAD_sse4_1.c new file mode 100644 index 000000000..d02ddf4ad --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VAD_sse4_1.c @@ -0,0 +1,277 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "main.h" +#include "stack_alloc.h" + +/* Weighting factors for tilt measure */ +static const opus_int32 tiltWeights[ VAD_N_BANDS ] = { 30000, 6000, -12000, -12000 }; + +/***************************************/ +/* Get the speech activity level in Q8 */ +/***************************************/ +opus_int silk_VAD_GetSA_Q8_sse4_1( /* O Return value, 0 if success */ + silk_encoder_state *psEncC, /* I/O Encoder state */ + const opus_int16 pIn[] /* I PCM input */ +) +{ + opus_int SA_Q15, pSNR_dB_Q7, input_tilt; + opus_int decimated_framelength1, decimated_framelength2; + opus_int decimated_framelength; + opus_int dec_subframe_length, dec_subframe_offset, SNR_Q7, i, b, s; + opus_int32 sumSquared, smooth_coef_Q16; + opus_int16 HPstateTmp; + VARDECL( opus_int16, X ); + opus_int32 Xnrg[ VAD_N_BANDS ]; + opus_int32 NrgToNoiseRatio_Q8[ VAD_N_BANDS ]; + opus_int32 speech_nrg, x_tmp; + opus_int X_offset[ VAD_N_BANDS ]; + opus_int ret = 0; + silk_VAD_state *psSilk_VAD = &psEncC->sVAD; + + SAVE_STACK; + + /* Safety checks */ + silk_assert( VAD_N_BANDS == 4 ); + celt_assert( MAX_FRAME_LENGTH >= psEncC->frame_length ); + celt_assert( psEncC->frame_length <= 512 ); + celt_assert( psEncC->frame_length == 8 * silk_RSHIFT( psEncC->frame_length, 3 ) ); + + /***********************/ + /* Filter and Decimate */ + /***********************/ + decimated_framelength1 = silk_RSHIFT( psEncC->frame_length, 1 ); + decimated_framelength2 = silk_RSHIFT( psEncC->frame_length, 2 ); + decimated_framelength = silk_RSHIFT( psEncC->frame_length, 3 ); + /* Decimate into 4 bands: + 0 L 3L L 3L 5L + - -- - -- -- + 8 8 2 4 4 + + [0-1 kHz| temp. |1-2 kHz| 2-4 kHz | 4-8 kHz | + + They're arranged to allow the minimal ( frame_length / 4 ) extra + scratch space during the downsampling process */ + X_offset[ 0 ] = 0; + X_offset[ 1 ] = decimated_framelength + decimated_framelength2; + X_offset[ 2 ] = X_offset[ 1 ] + decimated_framelength; + X_offset[ 3 ] = X_offset[ 2 ] + decimated_framelength2; + ALLOC( X, X_offset[ 3 ] + decimated_framelength1, opus_int16 ); + + /* 0-8 kHz to 0-4 kHz and 4-8 kHz */ + silk_ana_filt_bank_1( pIn, &psSilk_VAD->AnaState[ 0 ], + X, &X[ X_offset[ 3 ] ], psEncC->frame_length ); + + /* 0-4 kHz to 0-2 kHz and 2-4 kHz */ + silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState1[ 0 ], + X, &X[ X_offset[ 2 ] ], decimated_framelength1 ); + + /* 0-2 kHz to 0-1 kHz and 1-2 kHz */ + silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState2[ 0 ], + X, &X[ X_offset[ 1 ] ], decimated_framelength2 ); + + /*********************************************/ + /* HP filter on lowest band (differentiator) */ + /*********************************************/ + X[ decimated_framelength - 1 ] = silk_RSHIFT( X[ decimated_framelength - 1 ], 1 ); + HPstateTmp = X[ decimated_framelength - 1 ]; + for( i = decimated_framelength - 1; i > 0; i-- ) { + X[ i - 1 ] = silk_RSHIFT( X[ i - 1 ], 1 ); + X[ i ] -= X[ i - 1 ]; + } + X[ 0 ] -= psSilk_VAD->HPstate; + psSilk_VAD->HPstate = HPstateTmp; + + /*************************************/ + /* Calculate the energy in each band */ + /*************************************/ + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* Find the decimated framelength in the non-uniformly divided bands */ + decimated_framelength = silk_RSHIFT( psEncC->frame_length, silk_min_int( VAD_N_BANDS - b, VAD_N_BANDS - 1 ) ); + + /* Split length into subframe lengths */ + dec_subframe_length = silk_RSHIFT( decimated_framelength, VAD_INTERNAL_SUBFRAMES_LOG2 ); + dec_subframe_offset = 0; + + /* Compute energy per sub-frame */ + /* initialize with summed energy of last subframe */ + Xnrg[ b ] = psSilk_VAD->XnrgSubfr[ b ]; + for( s = 0; s < VAD_INTERNAL_SUBFRAMES; s++ ) { + __m128i xmm_X, xmm_acc; + sumSquared = 0; + + xmm_acc = _mm_setzero_si128(); + + for( i = 0; i < dec_subframe_length - 7; i += 8 ) + { + xmm_X = _mm_loadu_si128( (__m128i *)&(X[ X_offset[ b ] + i + dec_subframe_offset ] ) ); + xmm_X = _mm_srai_epi16( xmm_X, 3 ); + xmm_X = _mm_madd_epi16( xmm_X, xmm_X ); + xmm_acc = _mm_add_epi32( xmm_acc, xmm_X ); + } + + xmm_acc = _mm_add_epi32( xmm_acc, _mm_unpackhi_epi64( xmm_acc, xmm_acc ) ); + xmm_acc = _mm_add_epi32( xmm_acc, _mm_shufflelo_epi16( xmm_acc, 0x0E ) ); + + sumSquared += _mm_cvtsi128_si32( xmm_acc ); + + for( ; i < dec_subframe_length; i++ ) { + /* The energy will be less than dec_subframe_length * ( silk_int16_MIN / 8 ) ^ 2. */ + /* Therefore we can accumulate with no risk of overflow (unless dec_subframe_length > 128) */ + x_tmp = silk_RSHIFT( + X[ X_offset[ b ] + i + dec_subframe_offset ], 3 ); + sumSquared = silk_SMLABB( sumSquared, x_tmp, x_tmp ); + + /* Safety check */ + silk_assert( sumSquared >= 0 ); + } + + /* Add/saturate summed energy of current subframe */ + if( s < VAD_INTERNAL_SUBFRAMES - 1 ) { + Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], sumSquared ); + } else { + /* Look-ahead subframe */ + Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], silk_RSHIFT( sumSquared, 1 ) ); + } + + dec_subframe_offset += dec_subframe_length; + } + psSilk_VAD->XnrgSubfr[ b ] = sumSquared; + } + + /********************/ + /* Noise estimation */ + /********************/ + silk_VAD_GetNoiseLevels( &Xnrg[ 0 ], psSilk_VAD ); + + /***********************************************/ + /* Signal-plus-noise to noise ratio estimation */ + /***********************************************/ + sumSquared = 0; + input_tilt = 0; + for( b = 0; b < VAD_N_BANDS; b++ ) { + speech_nrg = Xnrg[ b ] - psSilk_VAD->NL[ b ]; + if( speech_nrg > 0 ) { + /* Divide, with sufficient resolution */ + if( ( Xnrg[ b ] & 0xFF800000 ) == 0 ) { + NrgToNoiseRatio_Q8[ b ] = silk_DIV32( silk_LSHIFT( Xnrg[ b ], 8 ), psSilk_VAD->NL[ b ] + 1 ); + } else { + NrgToNoiseRatio_Q8[ b ] = silk_DIV32( Xnrg[ b ], silk_RSHIFT( psSilk_VAD->NL[ b ], 8 ) + 1 ); + } + + /* Convert to log domain */ + SNR_Q7 = silk_lin2log( NrgToNoiseRatio_Q8[ b ] ) - 8 * 128; + + /* Sum-of-squares */ + sumSquared = silk_SMLABB( sumSquared, SNR_Q7, SNR_Q7 ); /* Q14 */ + + /* Tilt measure */ + if( speech_nrg < ( (opus_int32)1 << 20 ) ) { + /* Scale down SNR value for small subband speech energies */ + SNR_Q7 = silk_SMULWB( silk_LSHIFT( silk_SQRT_APPROX( speech_nrg ), 6 ), SNR_Q7 ); + } + input_tilt = silk_SMLAWB( input_tilt, tiltWeights[ b ], SNR_Q7 ); + } else { + NrgToNoiseRatio_Q8[ b ] = 256; + } + } + + /* Mean-of-squares */ + sumSquared = silk_DIV32_16( sumSquared, VAD_N_BANDS ); /* Q14 */ + + /* Root-mean-square approximation, scale to dBs, and write to output pointer */ + pSNR_dB_Q7 = (opus_int16)( 3 * silk_SQRT_APPROX( sumSquared ) ); /* Q7 */ + + /*********************************/ + /* Speech Probability Estimation */ + /*********************************/ + SA_Q15 = silk_sigm_Q15( silk_SMULWB( VAD_SNR_FACTOR_Q16, pSNR_dB_Q7 ) - VAD_NEGATIVE_OFFSET_Q5 ); + + /**************************/ + /* Frequency Tilt Measure */ + /**************************/ + psEncC->input_tilt_Q15 = silk_LSHIFT( silk_sigm_Q15( input_tilt ) - 16384, 1 ); + + /**************************************************/ + /* Scale the sigmoid output based on power levels */ + /**************************************************/ + speech_nrg = 0; + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* Accumulate signal-without-noise energies, higher frequency bands have more weight */ + speech_nrg += ( b + 1 ) * silk_RSHIFT( Xnrg[ b ] - psSilk_VAD->NL[ b ], 4 ); + } + + /* Power scaling */ + if( speech_nrg <= 0 ) { + SA_Q15 = silk_RSHIFT( SA_Q15, 1 ); + } else if( speech_nrg < 32768 ) { + if( psEncC->frame_length == 10 * psEncC->fs_kHz ) { + speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 16 ); + } else { + speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 15 ); + } + + /* square-root */ + speech_nrg = silk_SQRT_APPROX( speech_nrg ); + SA_Q15 = silk_SMULWB( 32768 + speech_nrg, SA_Q15 ); + } + + /* Copy the resulting speech activity in Q8 */ + psEncC->speech_activity_Q8 = silk_min_int( silk_RSHIFT( SA_Q15, 7 ), silk_uint8_MAX ); + + /***********************************/ + /* Energy Level and SNR estimation */ + /***********************************/ + /* Smoothing coefficient */ + smooth_coef_Q16 = silk_SMULWB( VAD_SNR_SMOOTH_COEF_Q18, silk_SMULWB( (opus_int32)SA_Q15, SA_Q15 ) ); + + if( psEncC->frame_length == 10 * psEncC->fs_kHz ) { + smooth_coef_Q16 >>= 1; + } + + for( b = 0; b < VAD_N_BANDS; b++ ) { + /* compute smoothed energy-to-noise ratio per band */ + psSilk_VAD->NrgRatioSmth_Q8[ b ] = silk_SMLAWB( psSilk_VAD->NrgRatioSmth_Q8[ b ], + NrgToNoiseRatio_Q8[ b ] - psSilk_VAD->NrgRatioSmth_Q8[ b ], smooth_coef_Q16 ); + + /* signal to noise ratio in dB per band */ + SNR_Q7 = 3 * ( silk_lin2log( psSilk_VAD->NrgRatioSmth_Q8[b] ) - 8 * 128 ); + /* quality = sigmoid( 0.25 * ( SNR_dB - 16 ) ); */ + psEncC->input_quality_bands_Q15[ b ] = silk_sigm_Q15( silk_RSHIFT( SNR_Q7 - 16 * 128, 4 ) ); + } + + RESTORE_STACK; + return( ret ); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VQ_WMat_EC_sse4_1.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VQ_WMat_EC_sse4_1.c new file mode 100644 index 000000000..74d6c6d0e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/VQ_WMat_EC_sse4_1.c @@ -0,0 +1,142 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "main.h" +#include "celt/x86/x86cpu.h" + +/* Entropy constrained matrix-weighted VQ, hard-coded to 5-element vectors, for a single input data vector */ +void silk_VQ_WMat_EC_sse4_1( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int16 *in_Q14, /* I input vector to be quantized */ + const opus_int32 *W_Q18, /* I weighting matrix */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + opus_int L /* I number of vectors in codebook */ +) +{ + opus_int k, gain_tmp_Q7; + const opus_int8 *cb_row_Q7; + opus_int16 diff_Q14[ 5 ]; + opus_int32 sum1_Q14, sum2_Q16; + + __m128i C_tmp1, C_tmp2, C_tmp3, C_tmp4, C_tmp5; + /* Loop over codebook */ + *rate_dist_Q14 = silk_int32_MAX; + cb_row_Q7 = cb_Q7; + for( k = 0; k < L; k++ ) { + gain_tmp_Q7 = cb_gain_Q7[k]; + + diff_Q14[ 0 ] = in_Q14[ 0 ] - silk_LSHIFT( cb_row_Q7[ 0 ], 7 ); + + C_tmp1 = OP_CVTEPI16_EPI32_M64( &in_Q14[ 1 ] ); + C_tmp2 = OP_CVTEPI8_EPI32_M32( &cb_row_Q7[ 1 ] ); + C_tmp2 = _mm_slli_epi32( C_tmp2, 7 ); + C_tmp1 = _mm_sub_epi32( C_tmp1, C_tmp2 ); + + diff_Q14[ 1 ] = _mm_extract_epi16( C_tmp1, 0 ); + diff_Q14[ 2 ] = _mm_extract_epi16( C_tmp1, 2 ); + diff_Q14[ 3 ] = _mm_extract_epi16( C_tmp1, 4 ); + diff_Q14[ 4 ] = _mm_extract_epi16( C_tmp1, 6 ); + + /* Weighted rate */ + sum1_Q14 = silk_SMULBB( mu_Q9, cl_Q5[ k ] ); + + /* Penalty for too large gain */ + sum1_Q14 = silk_ADD_LSHIFT32( sum1_Q14, silk_max( silk_SUB32( gain_tmp_Q7, max_gain_Q7 ), 0 ), 10 ); + + silk_assert( sum1_Q14 >= 0 ); + + /* first row of W_Q18 */ + C_tmp3 = _mm_loadu_si128( (__m128i *)(&W_Q18[ 1 ] ) ); + C_tmp4 = _mm_mul_epi32( C_tmp3, C_tmp1 ); + C_tmp4 = _mm_srli_si128( C_tmp4, 2 ); + + C_tmp1 = _mm_shuffle_epi32( C_tmp1, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */ + C_tmp3 = _mm_shuffle_epi32( C_tmp3, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */ + + C_tmp5 = _mm_mul_epi32( C_tmp3, C_tmp1 ); + C_tmp5 = _mm_srli_si128( C_tmp5, 2 ); + + C_tmp5 = _mm_add_epi32( C_tmp4, C_tmp5 ); + C_tmp5 = _mm_slli_epi32( C_tmp5, 1 ); + + C_tmp5 = _mm_add_epi32( C_tmp5, _mm_shuffle_epi32( C_tmp5, _MM_SHUFFLE( 0, 0, 0, 2 ) ) ); + sum2_Q16 = _mm_cvtsi128_si32( C_tmp5 ); + + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 0 ], diff_Q14[ 0 ] ); + sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 0 ] ); + + /* second row of W_Q18 */ + sum2_Q16 = silk_SMULWB( W_Q18[ 7 ], diff_Q14[ 2 ] ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 8 ], diff_Q14[ 3 ] ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 9 ], diff_Q14[ 4 ] ); + sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 6 ], diff_Q14[ 1 ] ); + sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 1 ] ); + + /* third row of W_Q18 */ + sum2_Q16 = silk_SMULWB( W_Q18[ 13 ], diff_Q14[ 3 ] ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 14 ], diff_Q14[ 4 ] ); + sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 12 ], diff_Q14[ 2 ] ); + sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 2 ] ); + + /* fourth row of W_Q18 */ + sum2_Q16 = silk_SMULWB( W_Q18[ 19 ], diff_Q14[ 4 ] ); + sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 ); + sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 18 ], diff_Q14[ 3 ] ); + sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 3 ] ); + + /* last row of W_Q18 */ + sum2_Q16 = silk_SMULWB( W_Q18[ 24 ], diff_Q14[ 4 ] ); + sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 4 ] ); + + silk_assert( sum1_Q14 >= 0 ); + + /* find best */ + if( sum1_Q14 < *rate_dist_Q14 ) { + *rate_dist_Q14 = sum1_Q14; + *ind = (opus_int8)k; + *gain_Q7 = gain_tmp_Q7; + } + + /* Go to next cbk vector */ + cb_row_Q7 += LTP_ORDER; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/main_sse.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/main_sse.h new file mode 100644 index 000000000..2f15d4486 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/main_sse.h @@ -0,0 +1,248 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef MAIN_SSE_H +#define MAIN_SSE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +# if defined(OPUS_X86_MAY_HAVE_SSE4_1) + +#if 0 /* FIXME: SSE disabled until silk_VQ_WMat_EC_sse4_1() gets updated. */ +# define OVERRIDE_silk_VQ_WMat_EC + +void silk_VQ_WMat_EC_sse4_1( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int16 *in_Q14, /* I input vector to be quantized */ + const opus_int32 *W_Q18, /* I weighting matrix */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + opus_int L /* I number of vectors in codebook */ +); + +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_VQ_WMat_EC(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L, arch) \ + ((void)(arch),silk_VQ_WMat_EC_sse4_1(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L)) + +#else + +extern void (*const SILK_VQ_WMAT_EC_IMPL[OPUS_ARCHMASK + 1])( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int16 *in_Q14, /* I input vector to be quantized */ + const opus_int32 *W_Q18, /* I weighting matrix */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + opus_int L /* I number of vectors in codebook */ +); + +# define silk_VQ_WMat_EC(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L, arch) \ + ((*SILK_VQ_WMAT_EC_IMPL[(arch) & OPUS_ARCHMASK])(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L)) + +#endif +#endif + +#if 0 /* FIXME: SSE disabled until the NSQ code gets updated. */ +# define OVERRIDE_silk_NSQ + +void silk_NSQ_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_NSQ(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_sse4_1(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#else + +extern void (*const SILK_NSQ_IMPL[OPUS_ARCHMASK + 1])( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +# define silk_NSQ(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((*SILK_NSQ_IMPL[(arch) & OPUS_ARCHMASK])(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#endif + +# define OVERRIDE_silk_NSQ_del_dec + +void silk_NSQ_del_dec_sse4_1( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_del_dec_sse4_1(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#else + +extern void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +); + +# define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((*SILK_NSQ_DEL_DEC_IMPL[(arch) & OPUS_ARCHMASK])(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#endif +#endif + +void silk_noise_shape_quantizer( + silk_nsq_state *NSQ, /* I/O NSQ state */ + opus_int signalType, /* I Signal type */ + const opus_int32 x_sc_Q10[], /* I */ + opus_int8 pulses[], /* O */ + opus_int16 xq[], /* O */ + opus_int32 sLTP_Q15[], /* I/O LTP state */ + const opus_int16 a_Q12[], /* I Short term prediction coefs */ + const opus_int16 b_Q14[], /* I Long term prediction coefs */ + const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */ + opus_int lag, /* I Pitch lag */ + opus_int32 HarmShapeFIRPacked_Q14, /* I */ + opus_int Tilt_Q14, /* I Spectral tilt */ + opus_int32 LF_shp_Q14, /* I */ + opus_int32 Gain_Q16, /* I */ + opus_int Lambda_Q10, /* I */ + opus_int offset_Q10, /* I */ + opus_int length, /* I Input length */ + opus_int shapingLPCOrder, /* I Noise shaping AR filter order */ + opus_int predictLPCOrder, /* I Prediction filter order */ + int arch /* I Architecture */ +); + +/**************************/ +/* Noise level estimation */ +/**************************/ +void silk_VAD_GetNoiseLevels( + const opus_int32 pX[ VAD_N_BANDS ], /* I subband energies */ + silk_VAD_state *psSilk_VAD /* I/O Pointer to Silk VAD state */ +); + +# define OVERRIDE_silk_VAD_GetSA_Q8 + +opus_int silk_VAD_GetSA_Q8_sse4_1( + silk_encoder_state *psEnC, + const opus_int16 pIn[] +); + +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define silk_VAD_GetSA_Q8(psEnC, pIn, arch) ((void)(arch),silk_VAD_GetSA_Q8_sse4_1(psEnC, pIn)) + +#else + +# define silk_VAD_GetSA_Q8(psEnC, pIn, arch) \ + ((*SILK_VAD_GETSA_Q8_IMPL[(arch) & OPUS_ARCHMASK])(psEnC, pIn)) + +extern opus_int (*const SILK_VAD_GETSA_Q8_IMPL[OPUS_ARCHMASK + 1])( + silk_encoder_state *psEnC, + const opus_int16 pIn[]); + +#endif + +# endif +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/x86_silk_map.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/x86_silk_map.c new file mode 100644 index 000000000..32dcc3cab --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk/x86/x86_silk_map.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2014, Cisco Systems, INC + Written by XiangMingZhu WeiZhou MinPeng YanWang + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#if defined(HAVE_CONFIG_H) +#include "config.h" +#endif + +#include "celt/x86/x86cpu.h" +#include "structs.h" +#include "SigProc_FIX.h" +#include "pitch.h" +#include "main.h" + +#if !defined(OPUS_X86_PRESUME_SSE4_1) + +#if defined(FIXED_POINT) + +#include "fixed/main_FIX.h" + +opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[ OPUS_ARCHMASK + 1 ] )( + const opus_int16 *inVec1, + const opus_int16 *inVec2, + const opus_int len +) = { + silk_inner_prod16_aligned_64_c, /* non-sse */ + silk_inner_prod16_aligned_64_c, + silk_inner_prod16_aligned_64_c, + MAY_HAVE_SSE4_1( silk_inner_prod16_aligned_64 ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_inner_prod16_aligned_64 ) /* avx */ +}; + +#endif + +opus_int (*const SILK_VAD_GETSA_Q8_IMPL[ OPUS_ARCHMASK + 1 ] )( + silk_encoder_state *psEncC, + const opus_int16 pIn[] +) = { + silk_VAD_GetSA_Q8_c, /* non-sse */ + silk_VAD_GetSA_Q8_c, + silk_VAD_GetSA_Q8_c, + MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ) /* avx */ +}; + +#if 0 /* FIXME: SSE disabled until the NSQ code gets updated. */ +void (*const SILK_NSQ_IMPL[ OPUS_ARCHMASK + 1 ] )( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) = { + silk_NSQ_c, /* non-sse */ + silk_NSQ_c, + silk_NSQ_c, + MAY_HAVE_SSE4_1( silk_NSQ ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_NSQ ) /* avx */ +}; +#endif + +#if 0 /* FIXME: SSE disabled until silk_VQ_WMat_EC_sse4_1() gets updated. */ +void (*const SILK_VQ_WMAT_EC_IMPL[ OPUS_ARCHMASK + 1 ] )( + opus_int8 *ind, /* O index of best codebook vector */ + opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */ + opus_int *gain_Q7, /* O sum of absolute LTP coefficients */ + const opus_int16 *in_Q14, /* I input vector to be quantized */ + const opus_int32 *W_Q18, /* I weighting matrix */ + const opus_int8 *cb_Q7, /* I codebook */ + const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */ + const opus_uint8 *cl_Q5, /* I code length for each codebook vector */ + const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */ + const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */ + opus_int L /* I number of vectors in codebook */ +) = { + silk_VQ_WMat_EC_c, /* non-sse */ + silk_VQ_WMat_EC_c, + silk_VQ_WMat_EC_c, + MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ) /* avx */ +}; +#endif + +#if 0 /* FIXME: SSE disabled until the NSQ code gets updated. */ +void (*const SILK_NSQ_DEL_DEC_IMPL[ OPUS_ARCHMASK + 1 ] )( + const silk_encoder_state *psEncC, /* I Encoder State */ + silk_nsq_state *NSQ, /* I/O NSQ state */ + SideInfoIndices *psIndices, /* I/O Quantization Indices */ + const opus_int32 x_Q3[], /* I Prefiltered input signal */ + opus_int8 pulses[], /* O Quantized pulse signal */ + const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ + const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ + const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ + const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ + const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ + const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ + const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ + const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ + const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ + const opus_int LTP_scale_Q14 /* I LTP state scaling */ +) = { + silk_NSQ_del_dec_c, /* non-sse */ + silk_NSQ_del_dec_c, + silk_NSQ_del_dec_c, + MAY_HAVE_SSE4_1( silk_NSQ_del_dec ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_NSQ_del_dec ) /* avx */ +}; +#endif + +#if defined(FIXED_POINT) + +void (*const SILK_BURG_MODIFIED_IMPL[ OPUS_ARCHMASK + 1 ] )( + opus_int32 *res_nrg, /* O Residual energy */ + opus_int *res_nrg_Q, /* O Residual energy Q value */ + opus_int32 A_Q16[], /* O Prediction coefficients (length order) */ + const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */ + const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */ + const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */ + const opus_int nb_subfr, /* I Number of subframes stacked in x */ + const opus_int D, /* I Order */ + int arch /* I Run-time architecture */ +) = { + silk_burg_modified_c, /* non-sse */ + silk_burg_modified_c, + silk_burg_modified_c, + MAY_HAVE_SSE4_1( silk_burg_modified ), /* sse4.1 */ + MAY_HAVE_SSE4_1( silk_burg_modified ) /* avx */ +}; + +#endif +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_headers.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_headers.mk new file mode 100644 index 000000000..2588067c7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_headers.mk @@ -0,0 +1,44 @@ +SILK_HEAD = \ +silk/debug.h \ +silk/control.h \ +silk/errors.h \ +silk/API.h \ +silk/typedef.h \ +silk/define.h \ +silk/main.h \ +silk/x86/main_sse.h \ +silk/PLC.h \ +silk/structs.h \ +silk/tables.h \ +silk/tuning_parameters.h \ +silk/Inlines.h \ +silk/MacroCount.h \ +silk/MacroDebug.h \ +silk/macros.h \ +silk/NSQ.h \ +silk/pitch_est_defines.h \ +silk/resampler_private.h \ +silk/resampler_rom.h \ +silk/resampler_structs.h \ +silk/SigProc_FIX.h \ +silk/x86/SigProc_FIX_sse.h \ +silk/arm/biquad_alt_arm.h \ +silk/arm/LPC_inv_pred_gain_arm.h \ +silk/arm/macros_armv4.h \ +silk/arm/macros_armv5e.h \ +silk/arm/macros_arm64.h \ +silk/arm/SigProc_FIX_armv4.h \ +silk/arm/SigProc_FIX_armv5e.h \ +silk/arm/NSQ_del_dec_arm.h \ +silk/arm/NSQ_neon.h \ +silk/fixed/main_FIX.h \ +silk/fixed/structs_FIX.h \ +silk/fixed/arm/warped_autocorrelation_FIX_arm.h \ +silk/fixed/mips/noise_shape_analysis_FIX_mipsr1.h \ +silk/fixed/mips/warped_autocorrelation_FIX_mipsr1.h \ +silk/float/main_FLP.h \ +silk/float/structs_FLP.h \ +silk/float/SigProc_FLP.h \ +silk/mips/macros_mipsr1.h \ +silk/mips/NSQ_del_dec_mipsr1.h \ +silk/mips/sigproc_fix_mipsr1.h diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_sources.mk b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_sources.mk new file mode 100644 index 000000000..d2666e665 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/silk_sources.mk @@ -0,0 +1,154 @@ +SILK_SOURCES = \ +silk/CNG.c \ +silk/code_signs.c \ +silk/init_decoder.c \ +silk/decode_core.c \ +silk/decode_frame.c \ +silk/decode_parameters.c \ +silk/decode_indices.c \ +silk/decode_pulses.c \ +silk/decoder_set_fs.c \ +silk/dec_API.c \ +silk/enc_API.c \ +silk/encode_indices.c \ +silk/encode_pulses.c \ +silk/gain_quant.c \ +silk/interpolate.c \ +silk/LP_variable_cutoff.c \ +silk/NLSF_decode.c \ +silk/NSQ.c \ +silk/NSQ_del_dec.c \ +silk/PLC.c \ +silk/shell_coder.c \ +silk/tables_gain.c \ +silk/tables_LTP.c \ +silk/tables_NLSF_CB_NB_MB.c \ +silk/tables_NLSF_CB_WB.c \ +silk/tables_other.c \ +silk/tables_pitch_lag.c \ +silk/tables_pulses_per_block.c \ +silk/VAD.c \ +silk/control_audio_bandwidth.c \ +silk/quant_LTP_gains.c \ +silk/VQ_WMat_EC.c \ +silk/HP_variable_cutoff.c \ +silk/NLSF_encode.c \ +silk/NLSF_VQ.c \ +silk/NLSF_unpack.c \ +silk/NLSF_del_dec_quant.c \ +silk/process_NLSFs.c \ +silk/stereo_LR_to_MS.c \ +silk/stereo_MS_to_LR.c \ +silk/check_control_input.c \ +silk/control_SNR.c \ +silk/init_encoder.c \ +silk/control_codec.c \ +silk/A2NLSF.c \ +silk/ana_filt_bank_1.c \ +silk/biquad_alt.c \ +silk/bwexpander_32.c \ +silk/bwexpander.c \ +silk/debug.c \ +silk/decode_pitch.c \ +silk/inner_prod_aligned.c \ +silk/lin2log.c \ +silk/log2lin.c \ +silk/LPC_analysis_filter.c \ +silk/LPC_inv_pred_gain.c \ +silk/table_LSF_cos.c \ +silk/NLSF2A.c \ +silk/NLSF_stabilize.c \ +silk/NLSF_VQ_weights_laroia.c \ +silk/pitch_est_tables.c \ +silk/resampler.c \ +silk/resampler_down2_3.c \ +silk/resampler_down2.c \ +silk/resampler_private_AR2.c \ +silk/resampler_private_down_FIR.c \ +silk/resampler_private_IIR_FIR.c \ +silk/resampler_private_up2_HQ.c \ +silk/resampler_rom.c \ +silk/sigm_Q15.c \ +silk/sort.c \ +silk/sum_sqr_shift.c \ +silk/stereo_decode_pred.c \ +silk/stereo_encode_pred.c \ +silk/stereo_find_predictor.c \ +silk/stereo_quant_pred.c \ +silk/LPC_fit.c + +SILK_SOURCES_SSE4_1 = \ +silk/x86/NSQ_sse4_1.c \ +silk/x86/NSQ_del_dec_sse4_1.c \ +silk/x86/x86_silk_map.c \ +silk/x86/VAD_sse4_1.c \ +silk/x86/VQ_WMat_EC_sse4_1.c + +SILK_SOURCES_ARM_NEON_INTR = \ +silk/arm/arm_silk_map.c \ +silk/arm/biquad_alt_neon_intr.c \ +silk/arm/LPC_inv_pred_gain_neon_intr.c \ +silk/arm/NSQ_del_dec_neon_intr.c \ +silk/arm/NSQ_neon.c + +SILK_SOURCES_FIXED = \ +silk/fixed/LTP_analysis_filter_FIX.c \ +silk/fixed/LTP_scale_ctrl_FIX.c \ +silk/fixed/corrMatrix_FIX.c \ +silk/fixed/encode_frame_FIX.c \ +silk/fixed/find_LPC_FIX.c \ +silk/fixed/find_LTP_FIX.c \ +silk/fixed/find_pitch_lags_FIX.c \ +silk/fixed/find_pred_coefs_FIX.c \ +silk/fixed/noise_shape_analysis_FIX.c \ +silk/fixed/process_gains_FIX.c \ +silk/fixed/regularize_correlations_FIX.c \ +silk/fixed/residual_energy16_FIX.c \ +silk/fixed/residual_energy_FIX.c \ +silk/fixed/warped_autocorrelation_FIX.c \ +silk/fixed/apply_sine_window_FIX.c \ +silk/fixed/autocorr_FIX.c \ +silk/fixed/burg_modified_FIX.c \ +silk/fixed/k2a_FIX.c \ +silk/fixed/k2a_Q16_FIX.c \ +silk/fixed/pitch_analysis_core_FIX.c \ +silk/fixed/vector_ops_FIX.c \ +silk/fixed/schur64_FIX.c \ +silk/fixed/schur_FIX.c + +SILK_SOURCES_FIXED_SSE4_1 = \ +silk/fixed/x86/vector_ops_FIX_sse4_1.c \ +silk/fixed/x86/burg_modified_FIX_sse4_1.c + +SILK_SOURCES_FIXED_ARM_NEON_INTR = \ +silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c + +SILK_SOURCES_FLOAT = \ +silk/float/apply_sine_window_FLP.c \ +silk/float/corrMatrix_FLP.c \ +silk/float/encode_frame_FLP.c \ +silk/float/find_LPC_FLP.c \ +silk/float/find_LTP_FLP.c \ +silk/float/find_pitch_lags_FLP.c \ +silk/float/find_pred_coefs_FLP.c \ +silk/float/LPC_analysis_filter_FLP.c \ +silk/float/LTP_analysis_filter_FLP.c \ +silk/float/LTP_scale_ctrl_FLP.c \ +silk/float/noise_shape_analysis_FLP.c \ +silk/float/process_gains_FLP.c \ +silk/float/regularize_correlations_FLP.c \ +silk/float/residual_energy_FLP.c \ +silk/float/warped_autocorrelation_FLP.c \ +silk/float/wrappers_FLP.c \ +silk/float/autocorrelation_FLP.c \ +silk/float/burg_modified_FLP.c \ +silk/float/bwexpander_FLP.c \ +silk/float/energy_FLP.c \ +silk/float/inner_product_FLP.c \ +silk/float/k2a_FLP.c \ +silk/float/LPC_inv_pred_gain_FLP.c \ +silk/float/pitch_analysis_core_FLP.c \ +silk/float/scale_copy_vector_FLP.c \ +silk/float/scale_vector_FLP.c \ +silk/float/schur_FLP.c \ +silk/float/sort_FLP.c diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.c new file mode 100644 index 000000000..058328f0f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.c @@ -0,0 +1,983 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ANALYSIS_C + +#ifdef MLP_TRAINING +#include +#endif + +#include "mathops.h" +#include "kiss_fft.h" +#include "celt.h" +#include "modes.h" +#include "arch.h" +#include "quant_bands.h" +#include "analysis.h" +#include "mlp.h" +#include "stack_alloc.h" +#include "float_cast.h" + +#ifndef M_PI +#define M_PI 3.141592653 +#endif + +#ifndef DISABLE_FLOAT_API + +#define TRANSITION_PENALTY 10 + +static const float dct_table[128] = { + 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, + 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, 0.250000f, + 0.351851f, 0.338330f, 0.311806f, 0.273300f, 0.224292f, 0.166664f, 0.102631f, 0.034654f, + -0.034654f,-0.102631f,-0.166664f,-0.224292f,-0.273300f,-0.311806f,-0.338330f,-0.351851f, + 0.346760f, 0.293969f, 0.196424f, 0.068975f,-0.068975f,-0.196424f,-0.293969f,-0.346760f, + -0.346760f,-0.293969f,-0.196424f,-0.068975f, 0.068975f, 0.196424f, 0.293969f, 0.346760f, + 0.338330f, 0.224292f, 0.034654f,-0.166664f,-0.311806f,-0.351851f,-0.273300f,-0.102631f, + 0.102631f, 0.273300f, 0.351851f, 0.311806f, 0.166664f,-0.034654f,-0.224292f,-0.338330f, + 0.326641f, 0.135299f,-0.135299f,-0.326641f,-0.326641f,-0.135299f, 0.135299f, 0.326641f, + 0.326641f, 0.135299f,-0.135299f,-0.326641f,-0.326641f,-0.135299f, 0.135299f, 0.326641f, + 0.311806f, 0.034654f,-0.273300f,-0.338330f,-0.102631f, 0.224292f, 0.351851f, 0.166664f, + -0.166664f,-0.351851f,-0.224292f, 0.102631f, 0.338330f, 0.273300f,-0.034654f,-0.311806f, + 0.293969f,-0.068975f,-0.346760f,-0.196424f, 0.196424f, 0.346760f, 0.068975f,-0.293969f, + -0.293969f, 0.068975f, 0.346760f, 0.196424f,-0.196424f,-0.346760f,-0.068975f, 0.293969f, + 0.273300f,-0.166664f,-0.338330f, 0.034654f, 0.351851f, 0.102631f,-0.311806f,-0.224292f, + 0.224292f, 0.311806f,-0.102631f,-0.351851f,-0.034654f, 0.338330f, 0.166664f,-0.273300f, +}; + +static const float analysis_window[240] = { + 0.000043f, 0.000171f, 0.000385f, 0.000685f, 0.001071f, 0.001541f, 0.002098f, 0.002739f, + 0.003466f, 0.004278f, 0.005174f, 0.006156f, 0.007222f, 0.008373f, 0.009607f, 0.010926f, + 0.012329f, 0.013815f, 0.015385f, 0.017037f, 0.018772f, 0.020590f, 0.022490f, 0.024472f, + 0.026535f, 0.028679f, 0.030904f, 0.033210f, 0.035595f, 0.038060f, 0.040604f, 0.043227f, + 0.045928f, 0.048707f, 0.051564f, 0.054497f, 0.057506f, 0.060591f, 0.063752f, 0.066987f, + 0.070297f, 0.073680f, 0.077136f, 0.080665f, 0.084265f, 0.087937f, 0.091679f, 0.095492f, + 0.099373f, 0.103323f, 0.107342f, 0.111427f, 0.115579f, 0.119797f, 0.124080f, 0.128428f, + 0.132839f, 0.137313f, 0.141849f, 0.146447f, 0.151105f, 0.155823f, 0.160600f, 0.165435f, + 0.170327f, 0.175276f, 0.180280f, 0.185340f, 0.190453f, 0.195619f, 0.200838f, 0.206107f, + 0.211427f, 0.216797f, 0.222215f, 0.227680f, 0.233193f, 0.238751f, 0.244353f, 0.250000f, + 0.255689f, 0.261421f, 0.267193f, 0.273005f, 0.278856f, 0.284744f, 0.290670f, 0.296632f, + 0.302628f, 0.308658f, 0.314721f, 0.320816f, 0.326941f, 0.333097f, 0.339280f, 0.345492f, + 0.351729f, 0.357992f, 0.364280f, 0.370590f, 0.376923f, 0.383277f, 0.389651f, 0.396044f, + 0.402455f, 0.408882f, 0.415325f, 0.421783f, 0.428254f, 0.434737f, 0.441231f, 0.447736f, + 0.454249f, 0.460770f, 0.467298f, 0.473832f, 0.480370f, 0.486912f, 0.493455f, 0.500000f, + 0.506545f, 0.513088f, 0.519630f, 0.526168f, 0.532702f, 0.539230f, 0.545751f, 0.552264f, + 0.558769f, 0.565263f, 0.571746f, 0.578217f, 0.584675f, 0.591118f, 0.597545f, 0.603956f, + 0.610349f, 0.616723f, 0.623077f, 0.629410f, 0.635720f, 0.642008f, 0.648271f, 0.654508f, + 0.660720f, 0.666903f, 0.673059f, 0.679184f, 0.685279f, 0.691342f, 0.697372f, 0.703368f, + 0.709330f, 0.715256f, 0.721144f, 0.726995f, 0.732807f, 0.738579f, 0.744311f, 0.750000f, + 0.755647f, 0.761249f, 0.766807f, 0.772320f, 0.777785f, 0.783203f, 0.788573f, 0.793893f, + 0.799162f, 0.804381f, 0.809547f, 0.814660f, 0.819720f, 0.824724f, 0.829673f, 0.834565f, + 0.839400f, 0.844177f, 0.848895f, 0.853553f, 0.858151f, 0.862687f, 0.867161f, 0.871572f, + 0.875920f, 0.880203f, 0.884421f, 0.888573f, 0.892658f, 0.896677f, 0.900627f, 0.904508f, + 0.908321f, 0.912063f, 0.915735f, 0.919335f, 0.922864f, 0.926320f, 0.929703f, 0.933013f, + 0.936248f, 0.939409f, 0.942494f, 0.945503f, 0.948436f, 0.951293f, 0.954072f, 0.956773f, + 0.959396f, 0.961940f, 0.964405f, 0.966790f, 0.969096f, 0.971321f, 0.973465f, 0.975528f, + 0.977510f, 0.979410f, 0.981228f, 0.982963f, 0.984615f, 0.986185f, 0.987671f, 0.989074f, + 0.990393f, 0.991627f, 0.992778f, 0.993844f, 0.994826f, 0.995722f, 0.996534f, 0.997261f, + 0.997902f, 0.998459f, 0.998929f, 0.999315f, 0.999615f, 0.999829f, 0.999957f, 1.000000f, +}; + +static const int tbands[NB_TBANDS+1] = { + 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 136, 160, 192, 240 +}; + +#define NB_TONAL_SKIP_BANDS 9 + +static opus_val32 silk_resampler_down2_hp( + opus_val32 *S, /* I/O State vector [ 2 ] */ + opus_val32 *out, /* O Output signal [ floor(len/2) ] */ + const opus_val32 *in, /* I Input signal [ len ] */ + int inLen /* I Number of input samples */ +) +{ + int k, len2 = inLen/2; + opus_val32 in32, out32, out32_hp, Y, X; + opus_val64 hp_ener = 0; + /* Internal variables and state are in Q10 format */ + for( k = 0; k < len2; k++ ) { + /* Convert to Q10 */ + in32 = in[ 2 * k ]; + + /* All-pass section for even input sample */ + Y = SUB32( in32, S[ 0 ] ); + X = MULT16_32_Q15(QCONST16(0.6074371f, 15), Y); + out32 = ADD32( S[ 0 ], X ); + S[ 0 ] = ADD32( in32, X ); + out32_hp = out32; + /* Convert to Q10 */ + in32 = in[ 2 * k + 1 ]; + + /* All-pass section for odd input sample, and add to output of previous section */ + Y = SUB32( in32, S[ 1 ] ); + X = MULT16_32_Q15(QCONST16(0.15063f, 15), Y); + out32 = ADD32( out32, S[ 1 ] ); + out32 = ADD32( out32, X ); + S[ 1 ] = ADD32( in32, X ); + + Y = SUB32( -in32, S[ 2 ] ); + X = MULT16_32_Q15(QCONST16(0.15063f, 15), Y); + out32_hp = ADD32( out32_hp, S[ 2 ] ); + out32_hp = ADD32( out32_hp, X ); + S[ 2 ] = ADD32( -in32, X ); + + hp_ener += out32_hp*(opus_val64)out32_hp; + /* Add, convert back to int16 and store to output */ + out[ k ] = HALF32(out32); + } +#ifdef FIXED_POINT + /* len2 can be up to 480, so we shift by 8 more to make it fit. */ + hp_ener = hp_ener >> (2*SIG_SHIFT + 8); +#endif + return (opus_val32)hp_ener; +} + +static opus_val32 downmix_and_resample(downmix_func downmix, const void *_x, opus_val32 *y, opus_val32 S[3], int subframe, int offset, int c1, int c2, int C, int Fs) +{ + VARDECL(opus_val32, tmp); + opus_val32 scale; + int j; + opus_val32 ret = 0; + SAVE_STACK; + + if (subframe==0) return 0; + if (Fs == 48000) + { + subframe *= 2; + offset *= 2; + } else if (Fs == 16000) { + subframe = subframe*2/3; + offset = offset*2/3; + } + ALLOC(tmp, subframe, opus_val32); + + downmix(_x, tmp, subframe, offset, c1, c2, C); +#ifdef FIXED_POINT + scale = (1<-1) + scale /= 2; + for (j=0;jarch = opus_select_arch(); + tonal->Fs = Fs; + /* Clear remaining fields. */ + tonality_analysis_reset(tonal); +} + +void tonality_analysis_reset(TonalityAnalysisState *tonal) +{ + /* Clear non-reusable fields. */ + char *start = (char*)&tonal->TONALITY_ANALYSIS_RESET_START; + OPUS_CLEAR(start, sizeof(TonalityAnalysisState) - (start - (char*)tonal)); +} + +void tonality_get_info(TonalityAnalysisState *tonal, AnalysisInfo *info_out, int len) +{ + int pos; + int curr_lookahead; + float tonality_max; + float tonality_avg; + int tonality_count; + int i; + int pos0; + float prob_avg; + float prob_count; + float prob_min, prob_max; + float vad_prob; + int mpos, vpos; + int bandwidth_span; + + pos = tonal->read_pos; + curr_lookahead = tonal->write_pos-tonal->read_pos; + if (curr_lookahead<0) + curr_lookahead += DETECT_SIZE; + + tonal->read_subframe += len/(tonal->Fs/400); + while (tonal->read_subframe>=8) + { + tonal->read_subframe -= 8; + tonal->read_pos++; + } + if (tonal->read_pos>=DETECT_SIZE) + tonal->read_pos-=DETECT_SIZE; + + /* On long frames, look at the second analysis window rather than the first. */ + if (len > tonal->Fs/50 && pos != tonal->write_pos) + { + pos++; + if (pos==DETECT_SIZE) + pos=0; + } + if (pos == tonal->write_pos) + pos--; + if (pos<0) + pos = DETECT_SIZE-1; + pos0 = pos; + OPUS_COPY(info_out, &tonal->info[pos], 1); + if (!info_out->valid) + return; + tonality_max = tonality_avg = info_out->tonality; + tonality_count = 1; + /* Look at the neighbouring frames and pick largest bandwidth found (to be safe). */ + bandwidth_span = 6; + /* If possible, look ahead for a tone to compensate for the delay in the tone detector. */ + for (i=0;i<3;i++) + { + pos++; + if (pos==DETECT_SIZE) + pos = 0; + if (pos == tonal->write_pos) + break; + tonality_max = MAX32(tonality_max, tonal->info[pos].tonality); + tonality_avg += tonal->info[pos].tonality; + tonality_count++; + info_out->bandwidth = IMAX(info_out->bandwidth, tonal->info[pos].bandwidth); + bandwidth_span--; + } + pos = pos0; + /* Look back in time to see if any has a wider bandwidth than the current frame. */ + for (i=0;iwrite_pos) + break; + info_out->bandwidth = IMAX(info_out->bandwidth, tonal->info[pos].bandwidth); + } + info_out->tonality = MAX32(tonality_avg/tonality_count, tonality_max-.2f); + + mpos = vpos = pos0; + /* If we have enough look-ahead, compensate for the ~5-frame delay in the music prob and + ~1 frame delay in the VAD prob. */ + if (curr_lookahead > 15) + { + mpos += 5; + if (mpos>=DETECT_SIZE) + mpos -= DETECT_SIZE; + vpos += 1; + if (vpos>=DETECT_SIZE) + vpos -= DETECT_SIZE; + } + + /* The following calculations attempt to minimize a "badness function" + for the transition. When switching from speech to music, the badness + of switching at frame k is + b_k = S*v_k + \sum_{i=0}^{k-1} v_i*(p_i - T) + where + v_i is the activity probability (VAD) at frame i, + p_i is the music probability at frame i + T is the probability threshold for switching + S is the penalty for switching during active audio rather than silence + the current frame has index i=0 + + Rather than apply badness to directly decide when to switch, what we compute + instead is the threshold for which the optimal switching point is now. When + considering whether to switch now (frame 0) or at frame k, we have: + S*v_0 = S*v_k + \sum_{i=0}^{k-1} v_i*(p_i - T) + which gives us: + T = ( \sum_{i=0}^{k-1} v_i*p_i + S*(v_k-v_0) ) / ( \sum_{i=0}^{k-1} v_i ) + We take the min threshold across all positive values of k (up to the maximum + amount of lookahead we have) to give us the threshold for which the current + frame is the optimal switch point. + + The last step is that we need to consider whether we want to switch at all. + For that we use the average of the music probability over the entire window. + If the threshold is higher than that average we're not going to + switch, so we compute a min with the average as well. The result of all these + min operations is music_prob_min, which gives the threshold for switching to music + if we're currently encoding for speech. + + We do the exact opposite to compute music_prob_max which is used for switching + from music to speech. + */ + prob_min = 1.f; + prob_max = 0.f; + vad_prob = tonal->info[vpos].activity_probability; + prob_count = MAX16(.1f, vad_prob); + prob_avg = MAX16(.1f, vad_prob)*tonal->info[mpos].music_prob; + while (1) + { + float pos_vad; + mpos++; + if (mpos==DETECT_SIZE) + mpos = 0; + if (mpos == tonal->write_pos) + break; + vpos++; + if (vpos==DETECT_SIZE) + vpos = 0; + if (vpos == tonal->write_pos) + break; + pos_vad = tonal->info[vpos].activity_probability; + prob_min = MIN16((prob_avg - TRANSITION_PENALTY*(vad_prob - pos_vad))/prob_count, prob_min); + prob_max = MAX16((prob_avg + TRANSITION_PENALTY*(vad_prob - pos_vad))/prob_count, prob_max); + prob_count += MAX16(.1f, pos_vad); + prob_avg += MAX16(.1f, pos_vad)*tonal->info[mpos].music_prob; + } + info_out->music_prob = prob_avg/prob_count; + prob_min = MIN16(prob_avg/prob_count, prob_min); + prob_max = MAX16(prob_avg/prob_count, prob_max); + prob_min = MAX16(prob_min, 0.f); + prob_max = MIN16(prob_max, 1.f); + + /* If we don't have enough look-ahead, do our best to make a decent decision. */ + if (curr_lookahead < 10) + { + float pmin, pmax; + pmin = prob_min; + pmax = prob_max; + pos = pos0; + /* Look for min/max in the past. */ + for (i=0;icount-1, 15);i++) + { + pos--; + if (pos < 0) + pos = DETECT_SIZE-1; + pmin = MIN16(pmin, tonal->info[pos].music_prob); + pmax = MAX16(pmax, tonal->info[pos].music_prob); + } + /* Bias against switching on active audio. */ + pmin = MAX16(0.f, pmin - .1f*vad_prob); + pmax = MIN16(1.f, pmax + .1f*vad_prob); + prob_min += (1.f-.1f*curr_lookahead)*(pmin - prob_min); + prob_max += (1.f-.1f*curr_lookahead)*(pmax - prob_max); + } + info_out->music_prob_min = prob_min; + info_out->music_prob_max = prob_max; + + /* printf("%f %f %f %f %f\n", prob_min, prob_max, prob_avg/prob_count, vad_prob, info_out->music_prob); */ +} + +static const float std_feature_bias[9] = { + 5.684947f, 3.475288f, 1.770634f, 1.599784f, 3.773215f, + 2.163313f, 1.260756f, 1.116868f, 1.918795f +}; + +#define LEAKAGE_OFFSET 2.5f +#define LEAKAGE_SLOPE 2.f + +#ifdef FIXED_POINT +/* For fixed-point, the input is +/-2^15 shifted up by SIG_SHIFT, so we need to + compensate for that in the energy. */ +#define SCALE_COMPENS (1.f/((opus_int32)1<<(15+SIG_SHIFT))) +#define SCALE_ENER(e) ((SCALE_COMPENS*SCALE_COMPENS)*(e)) +#else +#define SCALE_ENER(e) (e) +#endif + +#ifdef FIXED_POINT +static int is_digital_silence32(const opus_val32* pcm, int frame_size, int channels, int lsb_depth) +{ + int silence = 0; + opus_val32 sample_max = 0; +#ifdef MLP_TRAINING + return 0; +#endif + sample_max = celt_maxabs32(pcm, frame_size*channels); + + silence = (sample_max == 0); + (void)lsb_depth; + return silence; +} +#else +#define is_digital_silence32(pcm, frame_size, channels, lsb_depth) is_digital_silence(pcm, frame_size, channels, lsb_depth) +#endif + +static void tonality_analysis(TonalityAnalysisState *tonal, const CELTMode *celt_mode, const void *x, int len, int offset, int c1, int c2, int C, int lsb_depth, downmix_func downmix) +{ + int i, b; + const kiss_fft_state *kfft; + VARDECL(kiss_fft_cpx, in); + VARDECL(kiss_fft_cpx, out); + int N = 480, N2=240; + float * OPUS_RESTRICT A = tonal->angle; + float * OPUS_RESTRICT dA = tonal->d_angle; + float * OPUS_RESTRICT d2A = tonal->d2_angle; + VARDECL(float, tonality); + VARDECL(float, noisiness); + float band_tonality[NB_TBANDS]; + float logE[NB_TBANDS]; + float BFCC[8]; + float features[25]; + float frame_tonality; + float max_frame_tonality; + /*float tw_sum=0;*/ + float frame_noisiness; + const float pi4 = (float)(M_PI*M_PI*M_PI*M_PI); + float slope=0; + float frame_stationarity; + float relativeE; + float frame_probs[2]; + float alpha, alphaE, alphaE2; + float frame_loudness; + float bandwidth_mask; + int is_masked[NB_TBANDS+1]; + int bandwidth=0; + float maxE = 0; + float noise_floor; + int remaining; + AnalysisInfo *info; + float hp_ener; + float tonality2[240]; + float midE[8]; + float spec_variability=0; + float band_log2[NB_TBANDS+1]; + float leakage_from[NB_TBANDS+1]; + float leakage_to[NB_TBANDS+1]; + float layer_out[MAX_NEURONS]; + float below_max_pitch; + float above_max_pitch; + int is_silence; + SAVE_STACK; + + if (!tonal->initialized) + { + tonal->mem_fill = 240; + tonal->initialized = 1; + } + alpha = 1.f/IMIN(10, 1+tonal->count); + alphaE = 1.f/IMIN(25, 1+tonal->count); + /* Noise floor related decay for bandwidth detection: -2.2 dB/second */ + alphaE2 = 1.f/IMIN(100, 1+tonal->count); + if (tonal->count <= 1) alphaE2 = 1; + + if (tonal->Fs == 48000) + { + /* len and offset are now at 24 kHz. */ + len/= 2; + offset /= 2; + } else if (tonal->Fs == 16000) { + len = 3*len/2; + offset = 3*offset/2; + } + + kfft = celt_mode->mdct.kfft[0]; + tonal->hp_ener_accum += (float)downmix_and_resample(downmix, x, + &tonal->inmem[tonal->mem_fill], tonal->downmix_state, + IMIN(len, ANALYSIS_BUF_SIZE-tonal->mem_fill), offset, c1, c2, C, tonal->Fs); + if (tonal->mem_fill+len < ANALYSIS_BUF_SIZE) + { + tonal->mem_fill += len; + /* Don't have enough to update the analysis */ + RESTORE_STACK; + return; + } + hp_ener = tonal->hp_ener_accum; + info = &tonal->info[tonal->write_pos++]; + if (tonal->write_pos>=DETECT_SIZE) + tonal->write_pos-=DETECT_SIZE; + + is_silence = is_digital_silence32(tonal->inmem, ANALYSIS_BUF_SIZE, 1, lsb_depth); + + ALLOC(in, 480, kiss_fft_cpx); + ALLOC(out, 480, kiss_fft_cpx); + ALLOC(tonality, 240, float); + ALLOC(noisiness, 240, float); + for (i=0;iinmem[i]); + in[i].i = (kiss_fft_scalar)(w*tonal->inmem[N2+i]); + in[N-i-1].r = (kiss_fft_scalar)(w*tonal->inmem[N-i-1]); + in[N-i-1].i = (kiss_fft_scalar)(w*tonal->inmem[N+N2-i-1]); + } + OPUS_MOVE(tonal->inmem, tonal->inmem+ANALYSIS_BUF_SIZE-240, 240); + remaining = len - (ANALYSIS_BUF_SIZE-tonal->mem_fill); + tonal->hp_ener_accum = (float)downmix_and_resample(downmix, x, + &tonal->inmem[240], tonal->downmix_state, remaining, + offset+ANALYSIS_BUF_SIZE-tonal->mem_fill, c1, c2, C, tonal->Fs); + tonal->mem_fill = 240 + remaining; + if (is_silence) + { + /* On silence, copy the previous analysis. */ + int prev_pos = tonal->write_pos-2; + if (prev_pos < 0) + prev_pos += DETECT_SIZE; + OPUS_COPY(info, &tonal->info[prev_pos], 1); + RESTORE_STACK; + return; + } + opus_fft(kfft, in, out, tonal->arch); +#ifndef FIXED_POINT + /* If there's any NaN on the input, the entire output will be NaN, so we only need to check one value. */ + if (celt_isnan(out[0].r)) + { + info->valid = 0; + RESTORE_STACK; + return; + } +#endif + + for (i=1;iactivity = 0; + frame_noisiness = 0; + frame_stationarity = 0; + if (!tonal->count) + { + for (b=0;blowE[b] = 1e10; + tonal->highE[b] = -1e10; + } + } + relativeE = 0; + frame_loudness = 0; + /* The energy of the very first band is special because of DC. */ + { + float E = 0; + float X1r, X2r; + X1r = 2*(float)out[0].r; + X2r = 2*(float)out[0].i; + E = X1r*X1r + X2r*X2r; + for (i=1;i<4;i++) + { + float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; + E += binE; + } + E = SCALE_ENER(E); + band_log2[0] = .5f*1.442695f*(float)log(E+1e-10f); + } + for (b=0;bvalid = 0; + RESTORE_STACK; + return; + } +#endif + + tonal->E[tonal->E_count][b] = E; + frame_noisiness += nE/(1e-15f+E); + + frame_loudness += (float)sqrt(E+1e-10f); + logE[b] = (float)log(E+1e-10f); + band_log2[b+1] = .5f*1.442695f*(float)log(E+1e-10f); + tonal->logE[tonal->E_count][b] = logE[b]; + if (tonal->count==0) + tonal->highE[b] = tonal->lowE[b] = logE[b]; + if (tonal->highE[b] > tonal->lowE[b] + 7.5) + { + if (tonal->highE[b] - logE[b] > logE[b] - tonal->lowE[b]) + tonal->highE[b] -= .01f; + else + tonal->lowE[b] += .01f; + } + if (logE[b] > tonal->highE[b]) + { + tonal->highE[b] = logE[b]; + tonal->lowE[b] = MAX32(tonal->highE[b]-15, tonal->lowE[b]); + } else if (logE[b] < tonal->lowE[b]) + { + tonal->lowE[b] = logE[b]; + tonal->highE[b] = MIN32(tonal->lowE[b]+15, tonal->highE[b]); + } + relativeE += (logE[b]-tonal->lowE[b])/(1e-5f + (tonal->highE[b]-tonal->lowE[b])); + + L1=L2=0; + for (i=0;iE[i][b]); + L2 += tonal->E[i][b]; + } + + stationarity = MIN16(0.99f,L1/(float)sqrt(1e-15+NB_FRAMES*L2)); + stationarity *= stationarity; + stationarity *= stationarity; + frame_stationarity += stationarity; + /*band_tonality[b] = tE/(1e-15+E)*/; + band_tonality[b] = MAX16(tE/(1e-15f+E), stationarity*tonal->prev_band_tonality[b]); +#if 0 + if (b>=NB_TONAL_SKIP_BANDS) + { + frame_tonality += tweight[b]*band_tonality[b]; + tw_sum += tweight[b]; + } +#else + frame_tonality += band_tonality[b]; + if (b>=NB_TBANDS-NB_TONAL_SKIP_BANDS) + frame_tonality -= band_tonality[b-NB_TBANDS+NB_TONAL_SKIP_BANDS]; +#endif + max_frame_tonality = MAX16(max_frame_tonality, (1.f+.03f*(b-NB_TBANDS))*frame_tonality); + slope += band_tonality[b]*(b-8); + /*printf("%f %f ", band_tonality[b], stationarity);*/ + tonal->prev_band_tonality[b] = band_tonality[b]; + } + + leakage_from[0] = band_log2[0]; + leakage_to[0] = band_log2[0] - LEAKAGE_OFFSET; + for (b=1;b=0;b--) + { + float leak_slope = LEAKAGE_SLOPE*(tbands[b+1]-tbands[b])/4; + leakage_from[b] = MIN16(leakage_from[b+1]+leak_slope, leakage_from[b]); + leakage_to[b] = MAX16(leakage_to[b+1]-leak_slope, leakage_to[b]); + } + celt_assert(NB_TBANDS+1 <= LEAK_BANDS); + for (b=0;bleak_boost[b] = IMIN(255, (int)floor(.5 + 64.f*boost)); + } + for (;bleak_boost[b] = 0; + + for (i=0;ilogE[i][k] - tonal->logE[j][k]; + dist += tmp*tmp; + } + if (j!=i) + mindist = MIN32(mindist, dist); + } + spec_variability += mindist; + } + spec_variability = (float)sqrt(spec_variability/NB_FRAMES/NB_TBANDS); + bandwidth_mask = 0; + bandwidth = 0; + maxE = 0; + noise_floor = 5.7e-4f/(1<<(IMAX(0,lsb_depth-8))); + noise_floor *= noise_floor; + below_max_pitch=0; + above_max_pitch=0; + for (b=0;bmeanE[b] = MAX32((1-alphaE2)*tonal->meanE[b], E); + Em = MAX32(E, tonal->meanE[b]); + /* Consider the band "active" only if all these conditions are met: + 1) less than 90 dB below the peak band (maximal masking possible considering + both the ATH and the loudness-dependent slope of the spreading function) + 2) above the PCM quantization noise floor + We use b+1 because the first CELT band isn't included in tbands[] + */ + if (E*1e9f > maxE && (Em > 3*noise_floor*(band_end-band_start) || E > noise_floor*(band_end-band_start))) + bandwidth = b+1; + /* Check if the band is masked (see below). */ + is_masked[b] = E < (tonal->prev_bandwidth >= b+1 ? .01f : .05f)*bandwidth_mask; + /* Use a simple follower with 13 dB/Bark slope for spreading function. */ + bandwidth_mask = MAX32(.05f*bandwidth_mask, E); + } + /* Special case for the last two bands, for which we don't have spectrum but only + the energy above 12 kHz. The difficulty here is that the high-pass we use + leaks some LF energy, so we need to increase the threshold without accidentally cutting + off the band. */ + if (tonal->Fs == 48000) { + float noise_ratio; + float Em; + float E = hp_ener*(1.f/(60*60)); + noise_ratio = tonal->prev_bandwidth==20 ? 10.f : 30.f; + +#ifdef FIXED_POINT + /* silk_resampler_down2_hp() shifted right by an extra 8 bits. */ + E *= 256.f*(1.f/Q15ONE)*(1.f/Q15ONE); +#endif + above_max_pitch += E; + tonal->meanE[b] = MAX32((1-alphaE2)*tonal->meanE[b], E); + Em = MAX32(E, tonal->meanE[b]); + if (Em > 3*noise_ratio*noise_floor*160 || E > noise_ratio*noise_floor*160) + bandwidth = 20; + /* Check if the band is masked (see below). */ + is_masked[b] = E < (tonal->prev_bandwidth == 20 ? .01f : .05f)*bandwidth_mask; + } + if (above_max_pitch > below_max_pitch) + info->max_pitch_ratio = below_max_pitch/above_max_pitch; + else + info->max_pitch_ratio = 1; + /* In some cases, resampling aliasing can create a small amount of energy in the first band + being cut. So if the last band is masked, we don't include it. */ + if (bandwidth == 20 && is_masked[NB_TBANDS]) + bandwidth-=2; + else if (bandwidth > 0 && bandwidth <= NB_TBANDS && is_masked[bandwidth-1]) + bandwidth--; + if (tonal->count<=2) + bandwidth = 20; + frame_loudness = 20*(float)log10(frame_loudness); + tonal->Etracker = MAX32(tonal->Etracker-.003f, frame_loudness); + tonal->lowECount *= (1-alphaE); + if (frame_loudness < tonal->Etracker-30) + tonal->lowECount += alphaE; + + for (i=0;i<8;i++) + { + float sum=0; + for (b=0;b<16;b++) + sum += dct_table[i*16+b]*logE[b]; + BFCC[i] = sum; + } + for (i=0;i<8;i++) + { + float sum=0; + for (b=0;b<16;b++) + sum += dct_table[i*16+b]*.5f*(tonal->highE[b]+tonal->lowE[b]); + midE[i] = sum; + } + + frame_stationarity /= NB_TBANDS; + relativeE /= NB_TBANDS; + if (tonal->count<10) + relativeE = .5f; + frame_noisiness /= NB_TBANDS; +#if 1 + info->activity = frame_noisiness + (1-frame_noisiness)*relativeE; +#else + info->activity = .5*(1+frame_noisiness-frame_stationarity); +#endif + frame_tonality = (max_frame_tonality/(NB_TBANDS-NB_TONAL_SKIP_BANDS)); + frame_tonality = MAX16(frame_tonality, tonal->prev_tonality*.8f); + tonal->prev_tonality = frame_tonality; + + slope /= 8*8; + info->tonality_slope = slope; + + tonal->E_count = (tonal->E_count+1)%NB_FRAMES; + tonal->count = IMIN(tonal->count+1, ANALYSIS_COUNT_MAX); + info->tonality = frame_tonality; + + for (i=0;i<4;i++) + features[i] = -0.12299f*(BFCC[i]+tonal->mem[i+24]) + 0.49195f*(tonal->mem[i]+tonal->mem[i+16]) + 0.69693f*tonal->mem[i+8] - 1.4349f*tonal->cmean[i]; + + for (i=0;i<4;i++) + tonal->cmean[i] = (1-alpha)*tonal->cmean[i] + alpha*BFCC[i]; + + for (i=0;i<4;i++) + features[4+i] = 0.63246f*(BFCC[i]-tonal->mem[i+24]) + 0.31623f*(tonal->mem[i]-tonal->mem[i+16]); + for (i=0;i<3;i++) + features[8+i] = 0.53452f*(BFCC[i]+tonal->mem[i+24]) - 0.26726f*(tonal->mem[i]+tonal->mem[i+16]) -0.53452f*tonal->mem[i+8]; + + if (tonal->count > 5) + { + for (i=0;i<9;i++) + tonal->std[i] = (1-alpha)*tonal->std[i] + alpha*features[i]*features[i]; + } + for (i=0;i<4;i++) + features[i] = BFCC[i]-midE[i]; + + for (i=0;i<8;i++) + { + tonal->mem[i+24] = tonal->mem[i+16]; + tonal->mem[i+16] = tonal->mem[i+8]; + tonal->mem[i+8] = tonal->mem[i]; + tonal->mem[i] = BFCC[i]; + } + for (i=0;i<9;i++) + features[11+i] = (float)sqrt(tonal->std[i]) - std_feature_bias[i]; + features[18] = spec_variability - 0.78f; + features[20] = info->tonality - 0.154723f; + features[21] = info->activity - 0.724643f; + features[22] = frame_stationarity - 0.743717f; + features[23] = info->tonality_slope + 0.069216f; + features[24] = tonal->lowECount - 0.067930f; + + compute_dense(&layer0, layer_out, features); + compute_gru(&layer1, tonal->rnn_state, layer_out); + compute_dense(&layer2, frame_probs, tonal->rnn_state); + + /* Probability of speech or music vs noise */ + info->activity_probability = frame_probs[1]; + info->music_prob = frame_probs[0]; + + /*printf("%f %f %f\n", frame_probs[0], frame_probs[1], info->music_prob);*/ +#ifdef MLP_TRAINING + for (i=0;i<25;i++) + printf("%f ", features[i]); + printf("\n"); +#endif + + info->bandwidth = bandwidth; + tonal->prev_bandwidth = bandwidth; + /*printf("%d %d\n", info->bandwidth, info->opus_bandwidth);*/ + info->noisiness = frame_noisiness; + info->valid = 1; + RESTORE_STACK; +} + +void run_analysis(TonalityAnalysisState *analysis, const CELTMode *celt_mode, const void *analysis_pcm, + int analysis_frame_size, int frame_size, int c1, int c2, int C, opus_int32 Fs, + int lsb_depth, downmix_func downmix, AnalysisInfo *analysis_info) +{ + int offset; + int pcm_len; + + analysis_frame_size -= analysis_frame_size&1; + if (analysis_pcm != NULL) + { + /* Avoid overflow/wrap-around of the analysis buffer */ + analysis_frame_size = IMIN((DETECT_SIZE-5)*Fs/50, analysis_frame_size); + + pcm_len = analysis_frame_size - analysis->analysis_offset; + offset = analysis->analysis_offset; + while (pcm_len>0) { + tonality_analysis(analysis, celt_mode, analysis_pcm, IMIN(Fs/50, pcm_len), offset, c1, c2, C, lsb_depth, downmix); + offset += Fs/50; + pcm_len -= Fs/50; + } + analysis->analysis_offset = analysis_frame_size; + + analysis->analysis_offset -= frame_size; + } + + tonality_get_info(analysis, analysis_info, frame_size); +} + +#endif /* DISABLE_FLOAT_API */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.h new file mode 100644 index 000000000..0b66555f2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/analysis.h @@ -0,0 +1,103 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef ANALYSIS_H +#define ANALYSIS_H + +#include "celt.h" +#include "opus_private.h" +#include "mlp.h" + +#define NB_FRAMES 8 +#define NB_TBANDS 18 +#define ANALYSIS_BUF_SIZE 720 /* 30 ms at 24 kHz */ + +/* At that point we can stop counting frames because it no longer matters. */ +#define ANALYSIS_COUNT_MAX 10000 + +#define DETECT_SIZE 100 + +/* Uncomment this to print the MLP features on stdout. */ +/*#define MLP_TRAINING*/ + +typedef struct { + int arch; + int application; + opus_int32 Fs; +#define TONALITY_ANALYSIS_RESET_START angle + float angle[240]; + float d_angle[240]; + float d2_angle[240]; + opus_val32 inmem[ANALYSIS_BUF_SIZE]; + int mem_fill; /* number of usable samples in the buffer */ + float prev_band_tonality[NB_TBANDS]; + float prev_tonality; + int prev_bandwidth; + float E[NB_FRAMES][NB_TBANDS]; + float logE[NB_FRAMES][NB_TBANDS]; + float lowE[NB_TBANDS]; + float highE[NB_TBANDS]; + float meanE[NB_TBANDS+1]; + float mem[32]; + float cmean[8]; + float std[9]; + float Etracker; + float lowECount; + int E_count; + int count; + int analysis_offset; + int write_pos; + int read_pos; + int read_subframe; + float hp_ener_accum; + int initialized; + float rnn_state[MAX_NEURONS]; + opus_val32 downmix_state[3]; + AnalysisInfo info[DETECT_SIZE]; +} TonalityAnalysisState; + +/** Initialize a TonalityAnalysisState struct. + * + * This performs some possibly slow initialization steps which should + * not be repeated every analysis step. No allocated memory is retained + * by the state struct, so no cleanup call is required. + */ +void tonality_analysis_init(TonalityAnalysisState *analysis, opus_int32 Fs); + +/** Reset a TonalityAnalysisState stuct. + * + * Call this when there's a discontinuity in the data. + */ +void tonality_analysis_reset(TonalityAnalysisState *analysis); + +void tonality_get_info(TonalityAnalysisState *tonal, AnalysisInfo *info_out, int len); + +void run_analysis(TonalityAnalysisState *analysis, const CELTMode *celt_mode, const void *analysis_pcm, + int analysis_frame_size, int frame_size, int c1, int c2, int C, opus_int32 Fs, + int lsb_depth, downmix_func downmix, AnalysisInfo *analysis_info); + +#endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.c new file mode 100644 index 000000000..31298af05 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.c @@ -0,0 +1,378 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "arch.h" +#include "float_cast.h" +#include "opus_private.h" +#include "opus_defines.h" +#include "mapping_matrix.h" + +#define MATRIX_INDEX(nb_rows, row, col) (nb_rows * col + row) + +opus_int32 mapping_matrix_get_size(int rows, int cols) +{ + opus_int32 size; + + /* Mapping Matrix must only support up to 255 channels in or out. + * Additionally, the total cell count must be <= 65004 octets in order + * for the matrix to be stored in an OGG header. + */ + if (rows > 255 || cols > 255) + return 0; + size = rows * (opus_int32)cols * sizeof(opus_int16); + if (size > 65004) + return 0; + + return align(sizeof(MappingMatrix)) + align(size); +} + +opus_int16 *mapping_matrix_get_data(const MappingMatrix *matrix) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (opus_int16*)(void*)((char*)matrix + align(sizeof(MappingMatrix))); +} + +void mapping_matrix_init(MappingMatrix * const matrix, + int rows, int cols, int gain, const opus_int16 *data, opus_int32 data_size) +{ + int i; + opus_int16 *ptr; + +#if !defined(ENABLE_ASSERTIONS) + (void)data_size; +#endif + celt_assert(align(data_size) == align(rows * cols * sizeof(opus_int16))); + + matrix->rows = rows; + matrix->cols = cols; + matrix->gain = gain; + ptr = mapping_matrix_get_data(matrix); + for (i = 0; i < rows * cols; i++) + { + ptr[i] = data[i]; + } +} + +#ifndef DISABLE_FLOAT_API +void mapping_matrix_multiply_channel_in_float( + const MappingMatrix *matrix, + const float *input, + int input_rows, + opus_val16 *output, + int output_row, + int output_rows, + int frame_size) +{ + /* Matrix data is ordered col-wise. */ + opus_int16* matrix_data; + int i, col; + + celt_assert(input_rows <= matrix->cols && output_rows <= matrix->rows); + + matrix_data = mapping_matrix_get_data(matrix); + + for (i = 0; i < frame_size; i++) + { + float tmp = 0; + for (col = 0; col < input_rows; col++) + { + tmp += + matrix_data[MATRIX_INDEX(matrix->rows, output_row, col)] * + input[MATRIX_INDEX(input_rows, col, i)]; + } +#if defined(FIXED_POINT) + output[output_rows * i] = FLOAT2INT16((1/32768.f)*tmp); +#else + output[output_rows * i] = (1/32768.f)*tmp; +#endif + } +} + +void mapping_matrix_multiply_channel_out_float( + const MappingMatrix *matrix, + const opus_val16 *input, + int input_row, + int input_rows, + float *output, + int output_rows, + int frame_size +) +{ + /* Matrix data is ordered col-wise. */ + opus_int16* matrix_data; + int i, row; + float input_sample; + + celt_assert(input_rows <= matrix->cols && output_rows <= matrix->rows); + + matrix_data = mapping_matrix_get_data(matrix); + + for (i = 0; i < frame_size; i++) + { +#if defined(FIXED_POINT) + input_sample = (1/32768.f)*input[input_rows * i]; +#else + input_sample = input[input_rows * i]; +#endif + for (row = 0; row < output_rows; row++) + { + float tmp = + (1/32768.f)*matrix_data[MATRIX_INDEX(matrix->rows, row, input_row)] * + input_sample; + output[MATRIX_INDEX(output_rows, row, i)] += tmp; + } + } +} +#endif /* DISABLE_FLOAT_API */ + +void mapping_matrix_multiply_channel_in_short( + const MappingMatrix *matrix, + const opus_int16 *input, + int input_rows, + opus_val16 *output, + int output_row, + int output_rows, + int frame_size) +{ + /* Matrix data is ordered col-wise. */ + opus_int16* matrix_data; + int i, col; + + celt_assert(input_rows <= matrix->cols && output_rows <= matrix->rows); + + matrix_data = mapping_matrix_get_data(matrix); + + for (i = 0; i < frame_size; i++) + { + opus_val32 tmp = 0; + for (col = 0; col < input_rows; col++) + { +#if defined(FIXED_POINT) + tmp += + ((opus_int32)matrix_data[MATRIX_INDEX(matrix->rows, output_row, col)] * + (opus_int32)input[MATRIX_INDEX(input_rows, col, i)]) >> 8; +#else + tmp += + matrix_data[MATRIX_INDEX(matrix->rows, output_row, col)] * + input[MATRIX_INDEX(input_rows, col, i)]; +#endif + } +#if defined(FIXED_POINT) + output[output_rows * i] = (opus_int16)((tmp + 64) >> 7); +#else + output[output_rows * i] = (1/(32768.f*32768.f))*tmp; +#endif + } +} + +void mapping_matrix_multiply_channel_out_short( + const MappingMatrix *matrix, + const opus_val16 *input, + int input_row, + int input_rows, + opus_int16 *output, + int output_rows, + int frame_size) +{ + /* Matrix data is ordered col-wise. */ + opus_int16* matrix_data; + int i, row; + opus_int32 input_sample; + + celt_assert(input_rows <= matrix->cols && output_rows <= matrix->rows); + + matrix_data = mapping_matrix_get_data(matrix); + + for (i = 0; i < frame_size; i++) + { +#if defined(FIXED_POINT) + input_sample = (opus_int32)input[input_rows * i]; +#else + input_sample = (opus_int32)FLOAT2INT16(input[input_rows * i]); +#endif + for (row = 0; row < output_rows; row++) + { + opus_int32 tmp = + (opus_int32)matrix_data[MATRIX_INDEX(matrix->rows, row, input_row)] * + input_sample; + output[MATRIX_INDEX(output_rows, row, i)] += (tmp + 16384) >> 15; + } + } +} + +const MappingMatrix mapping_matrix_foa_mixing = { 6, 6, 0 }; +const opus_int16 mapping_matrix_foa_mixing_data[36] = { + 16384, 0, -16384, 23170, 0, 0, 16384, 23170, + 16384, 0, 0, 0, 16384, 0, -16384, -23170, + 0, 0, 16384, -23170, 16384, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 32767 +}; + +const MappingMatrix mapping_matrix_soa_mixing = { 11, 11, 0 }; +const opus_int16 mapping_matrix_soa_mixing_data[121] = { + 10923, 7723, 13377, -13377, 11585, 9459, 7723, -16384, + -6689, 0, 0, 10923, 7723, 13377, 13377, -11585, + 9459, 7723, 16384, -6689, 0, 0, 10923, -15447, + 13377, 0, 0, -18919, 7723, 0, 13377, 0, + 0, 10923, 7723, -13377, -13377, 11585, -9459, 7723, + 16384, -6689, 0, 0, 10923, -7723, 0, 13377, + -16384, 0, -15447, 0, 9459, 0, 0, 10923, + -7723, 0, -13377, 16384, 0, -15447, 0, 9459, + 0, 0, 10923, 15447, 0, 0, 0, 0, + -15447, 0, -18919, 0, 0, 10923, 7723, -13377, + 13377, -11585, -9459, 7723, -16384, -6689, 0, 0, + 10923, -15447, -13377, 0, 0, 18919, 7723, 0, + 13377, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767 +}; + +const MappingMatrix mapping_matrix_toa_mixing = { 18, 18, 0 }; +const opus_int16 mapping_matrix_toa_mixing_data[324] = { + 8208, 0, -881, 14369, 0, 0, -8192, -4163, + 13218, 0, 0, 0, 11095, -8836, -6218, 14833, + 0, 0, 8208, -10161, 881, 10161, -13218, -2944, + -8192, 2944, 0, -10488, -6218, 6248, -11095, -6248, + 0, -10488, 0, 0, 8208, 10161, 881, -10161, + -13218, 2944, -8192, -2944, 0, 10488, -6218, -6248, + -11095, 6248, 0, 10488, 0, 0, 8176, 5566, + -11552, 5566, 9681, -11205, 8192, -11205, 0, 4920, + -15158, 9756, -3334, 9756, 0, -4920, 0, 0, + 8176, 7871, 11552, 0, 0, 15846, 8192, 0, + -9681, -6958, 0, 13797, 3334, 0, -15158, 0, + 0, 0, 8176, 0, 11552, 7871, 0, 0, + 8192, 15846, 9681, 0, 0, 0, 3334, 13797, + 15158, 6958, 0, 0, 8176, 5566, -11552, -5566, + -9681, -11205, 8192, 11205, 0, 4920, 15158, 9756, + -3334, -9756, 0, 4920, 0, 0, 8208, 14369, + -881, 0, 0, -4163, -8192, 0, -13218, -14833, + 0, -8836, 11095, 0, 6218, 0, 0, 0, + 8208, 10161, 881, 10161, 13218, 2944, -8192, 2944, + 0, 10488, 6218, -6248, -11095, -6248, 0, -10488, + 0, 0, 8208, -14369, -881, 0, 0, 4163, + -8192, 0, -13218, 14833, 0, 8836, 11095, 0, + 6218, 0, 0, 0, 8208, 0, -881, -14369, + 0, 0, -8192, 4163, 13218, 0, 0, 0, + 11095, 8836, -6218, -14833, 0, 0, 8176, -5566, + -11552, 5566, -9681, 11205, 8192, -11205, 0, -4920, + 15158, -9756, -3334, 9756, 0, -4920, 0, 0, + 8176, 0, 11552, -7871, 0, 0, 8192, -15846, + 9681, 0, 0, 0, 3334, -13797, 15158, -6958, + 0, 0, 8176, -7871, 11552, 0, 0, -15846, + 8192, 0, -9681, 6958, 0, -13797, 3334, 0, + -15158, 0, 0, 0, 8176, -5566, -11552, -5566, + 9681, 11205, 8192, 11205, 0, -4920, -15158, -9756, + -3334, -9756, 0, 4920, 0, 0, 8208, -10161, + 881, -10161, 13218, -2944, -8192, -2944, 0, -10488, + 6218, 6248, -11095, 6248, 0, 10488, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767 +}; + +const MappingMatrix mapping_matrix_foa_demixing = { 6, 6, 0 }; +const opus_int16 mapping_matrix_foa_demixing_data[36] = { + 16384, 16384, 16384, 16384, 0, 0, 0, 23170, + 0, -23170, 0, 0, -16384, 16384, -16384, 16384, + 0, 0, 23170, 0, -23170, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 32767 +}; + +const MappingMatrix mapping_matrix_soa_demixing = { 11, 11, 3050 }; +const opus_int16 mapping_matrix_soa_demixing_data[121] = { + 2771, 2771, 2771, 2771, 2771, 2771, 2771, 2771, + 2771, 0, 0, 10033, 10033, -20066, 10033, 14189, + 14189, -28378, 10033, -20066, 0, 0, 3393, 3393, + 3393, -3393, 0, 0, 0, -3393, -3393, 0, + 0, -17378, 17378, 0, -17378, -24576, 24576, 0, + 17378, 0, 0, 0, -14189, 14189, 0, -14189, + -28378, 28378, 0, 14189, 0, 0, 0, 2399, + 2399, -4799, -2399, 0, 0, 0, -2399, 4799, + 0, 0, 1959, 1959, 1959, 1959, -3918, -3918, + -3918, 1959, 1959, 0, 0, -4156, 4156, 0, + 4156, 0, 0, 0, -4156, 0, 0, 0, + 8192, 8192, -16384, 8192, 16384, 16384, -32768, 8192, + -16384, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 8312, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8312 +}; + +const MappingMatrix mapping_matrix_toa_demixing = { 18, 18, 0 }; +const opus_int16 mapping_matrix_toa_demixing_data[324] = { + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, + 0, 0, 0, -9779, 9779, 6263, 8857, 0, + 6263, 13829, 9779, -13829, 0, -6263, 0, -8857, + -6263, -9779, 0, 0, -3413, 3413, 3413, -11359, + 11359, 11359, -11359, -3413, 3413, -3413, -3413, -11359, + 11359, 11359, -11359, 3413, 0, 0, 13829, 9779, + -9779, 6263, 0, 8857, -6263, 0, 9779, 0, + -13829, 6263, -8857, 0, -6263, -9779, 0, 0, + 0, -15617, -15617, 6406, 0, 0, -6406, 0, + 15617, 0, 0, -6406, 0, 0, 6406, 15617, + 0, 0, 0, -5003, 5003, -10664, 15081, 0, + -10664, -7075, 5003, 7075, 0, 10664, 0, -15081, + 10664, -5003, 0, 0, -8176, -8176, -8176, 8208, + 8208, 8208, 8208, -8176, -8176, -8176, -8176, 8208, + 8208, 8208, 8208, -8176, 0, 0, -7075, 5003, + -5003, -10664, 0, 15081, 10664, 0, 5003, 0, + 7075, -10664, -15081, 0, 10664, -5003, 0, 0, + 15617, 0, 0, 0, -6406, 6406, 0, -15617, + 0, -15617, 15617, 0, 6406, -6406, 0, 0, + 0, 0, 0, -11393, 11393, 2993, -4233, 0, + 2993, -16112, 11393, 16112, 0, -2993, 0, 4233, + -2993, -11393, 0, 0, 0, -9974, -9974, -13617, + 0, 0, 13617, 0, 9974, 0, 0, 13617, + 0, 0, -13617, 9974, 0, 0, 0, 5579, + -5579, 10185, 14403, 0, 10185, -7890, -5579, 7890, + 0, -10185, 0, -14403, -10185, 5579, 0, 0, + 11826, -11826, -11826, -901, 901, 901, -901, 11826, + -11826, 11826, 11826, -901, 901, 901, -901, -11826, + 0, 0, -7890, -5579, 5579, 10185, 0, 14403, + -10185, 0, -5579, 0, 7890, 10185, -14403, 0, + -10185, 5579, 0, 0, -9974, 0, 0, 0, + -13617, 13617, 0, 9974, 0, 9974, -9974, 0, + 13617, -13617, 0, 0, 0, 0, 16112, -11393, + 11393, -2993, 0, 4233, 2993, 0, -11393, 0, + -16112, -2993, -4233, 0, 2993, 11393, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767 +}; + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.h new file mode 100644 index 000000000..98bc82df3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mapping_matrix.h @@ -0,0 +1,133 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** + * @file mapping_matrix.h + * @brief Opus reference implementation mapping matrix API + */ + +#ifndef MAPPING_MATRIX_H +#define MAPPING_MATRIX_H + +#include "opus_types.h" +#include "opus_projection.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct MappingMatrix +{ + int rows; /* number of channels outputted from matrix. */ + int cols; /* number of channels inputted to matrix. */ + int gain; /* in dB. S7.8-format. */ + /* Matrix cell data goes here using col-wise ordering. */ +} MappingMatrix; + +opus_int32 mapping_matrix_get_size(int rows, int cols); + +opus_int16 *mapping_matrix_get_data(const MappingMatrix *matrix); + +void mapping_matrix_init( + MappingMatrix * const matrix, + int rows, + int cols, + int gain, + const opus_int16 *data, + opus_int32 data_size +); + +#ifndef DISABLE_FLOAT_API +void mapping_matrix_multiply_channel_in_float( + const MappingMatrix *matrix, + const float *input, + int input_rows, + opus_val16 *output, + int output_row, + int output_rows, + int frame_size +); + +void mapping_matrix_multiply_channel_out_float( + const MappingMatrix *matrix, + const opus_val16 *input, + int input_row, + int input_rows, + float *output, + int output_rows, + int frame_size +); +#endif /* DISABLE_FLOAT_API */ + +void mapping_matrix_multiply_channel_in_short( + const MappingMatrix *matrix, + const opus_int16 *input, + int input_rows, + opus_val16 *output, + int output_row, + int output_rows, + int frame_size +); + +void mapping_matrix_multiply_channel_out_short( + const MappingMatrix *matrix, + const opus_val16 *input, + int input_row, + int input_rows, + opus_int16 *output, + int output_rows, + int frame_size +); + +/* Pre-computed mixing and demixing matrices for 1st to 3rd-order ambisonics. + * foa: first-order ambisonics + * soa: second-order ambisonics + * toa: third-order ambisonics + */ +extern const MappingMatrix mapping_matrix_foa_mixing; +extern const opus_int16 mapping_matrix_foa_mixing_data[36]; + +extern const MappingMatrix mapping_matrix_soa_mixing; +extern const opus_int16 mapping_matrix_soa_mixing_data[121]; + +extern const MappingMatrix mapping_matrix_toa_mixing; +extern const opus_int16 mapping_matrix_toa_mixing_data[324]; + +extern const MappingMatrix mapping_matrix_foa_demixing; +extern const opus_int16 mapping_matrix_foa_demixing_data[36]; + +extern const MappingMatrix mapping_matrix_soa_demixing; +extern const opus_int16 mapping_matrix_soa_demixing_data[121]; + +extern const MappingMatrix mapping_matrix_toa_demixing; +extern const opus_int16 mapping_matrix_toa_demixing_data[324]; + +#ifdef __cplusplus +} +#endif + +#endif /* MAPPING_MATRIX_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/meson.build new file mode 100644 index 000000000..cc07ff06e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/meson.build @@ -0,0 +1,45 @@ +opus_sources = sources['OPUS_SOURCES'] + +opus_sources_float = sources['OPUS_SOURCES_FLOAT'] + +if not disable_float_api + opus_sources += opus_sources_float +endif + +opus_lib_c_args = [] +if host_machine.system() == 'windows' + opus_lib_c_args += ['-DDLL_EXPORT'] +endif + +opus_lib = library('opus', + opus_sources, + version: libversion, + darwin_versions: macosversion, + c_args: opus_lib_c_args, + include_directories: opus_includes, + link_with: [celt_lib, silk_lib], + dependencies: libm, + install: true) + +opus_dep = declare_dependency(link_with: opus_lib, + include_directories: opus_public_includes) + +# Extra uninstalled Opus programs +if not extra_programs.disabled() + foreach prog : ['opus_compare', 'opus_demo', 'repacketizer_demo'] + executable(prog, '@0@.c'.format(prog), + include_directories: opus_includes, + link_with: opus_lib, + dependencies: libm, + install: false) + endforeach + + if opt_custom_modes + executable('opus_custom_demo', '../celt/opus_custom_demo.c', + include_directories: opus_includes, + link_with: opus_lib, + dependencies: libm, + install: false) + endif + +endif diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp.c new file mode 100644 index 000000000..964c6a98f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/mlp.c @@ -0,0 +1,144 @@ +/* Copyright (c) 2008-2011 Octasic Inc. + 2012-2017 Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "opus_types.h" +#include "opus_defines.h" +#include "arch.h" +#include "tansig_table.h" +#include "mlp.h" + +static OPUS_INLINE float tansig_approx(float x) +{ + int i; + float y, dy; + float sign=1; + /* Tests are reversed to catch NaNs */ + if (!(x<8)) + return 1; + if (!(x>-8)) + return -1; +#ifndef FIXED_POINT + /* Another check in case of -ffast-math */ + if (celt_isnan(x)) + return 0; +#endif + if (x<0) + { + x=-x; + sign=-1; + } + i = (int)floor(.5f+25*x); + x -= .04f*i; + y = tansig_table[i]; + dy = 1-y*y; + y = y + x*dy*(1 - y*x); + return sign*y; +} + +static OPUS_INLINE float sigmoid_approx(float x) +{ + return .5f + .5f*tansig_approx(.5f*x); +} + +static void gemm_accum(float *out, const opus_int8 *weights, int rows, int cols, int col_stride, const float *x) +{ + int i, j; + for (i=0;inb_inputs; + N = layer->nb_neurons; + stride = N; + for (i=0;ibias[i]; + gemm_accum(output, layer->input_weights, N, M, stride, input); + for (i=0;isigmoid) { + for (i=0;inb_inputs; + N = gru->nb_neurons; + stride = 3*N; + /* Compute update gate. */ + for (i=0;ibias[i]; + gemm_accum(z, gru->input_weights, N, M, stride, input); + gemm_accum(z, gru->recurrent_weights, N, N, stride, state); + for (i=0;ibias[N + i]; + gemm_accum(r, &gru->input_weights[N], N, M, stride, input); + gemm_accum(r, &gru->recurrent_weights[N], N, N, stride, state); + for (i=0;ibias[2*N + i]; + for (i=0;iinput_weights[2*N], N, M, stride, input); + gemm_accum(h, &gru->recurrent_weights[2*N], N, N, stride, tmp); + for (i=0;i=0) + break; + x[i*C] = x[i*C]+a*x[i*C]*x[i*C]; + } + + curr=0; + x0 = x[0]; + while(1) + { + int start, end; + float maxval; + int special=0; + int peak_pos; + for (i=curr;i1 || x[i*C]<-1) + break; + } + if (i==N) + { + a=0; + break; + } + peak_pos = i; + start=end=i; + maxval=ABS16(x[i*C]); + /* Look for first zero crossing before clipping */ + while (start>0 && x[i*C]*x[(start-1)*C]>=0) + start--; + /* Look for first zero crossing after clipping */ + while (end=0) + { + /* Look for other peaks until the next zero-crossing. */ + if (ABS16(x[end*C])>maxval) + { + maxval = ABS16(x[end*C]); + peak_pos = end; + } + end++; + } + /* Detect the special case where we clip before the first zero crossing */ + special = (start==0 && x[i*C]*x[0]>=0); + + /* Compute a such that maxval + a*maxval^2 = 1 */ + a=(maxval-1)/(maxval*maxval); + /* Slightly boost "a" by 2^-22. This is just enough to ensure -ffast-math + does not cause output values larger than +/-1, but small enough not + to matter even for 24-bit output. */ + a += a*2.4e-7f; + if (x[i*C]>0) + a = -a; + /* Apply soft clipping */ + for (i=start;i=2) + { + /* Add a linear ramp from the first sample to the signal peak. + This avoids a discontinuity at the beginning of the frame. */ + float delta; + float offset = x0-x[0]; + delta = offset / peak_pos; + for (i=curr;i>2; + return 2; + } +} + +static int parse_size(const unsigned char *data, opus_int32 len, opus_int16 *size) +{ + if (len<1) + { + *size = -1; + return -1; + } else if (data[0]<252) + { + *size = data[0]; + return 1; + } else if (len<2) + { + *size = -1; + return -1; + } else { + *size = 4*data[1] + data[0]; + return 2; + } +} + +int opus_packet_get_samples_per_frame(const unsigned char *data, + opus_int32 Fs) +{ + int audiosize; + if (data[0]&0x80) + { + audiosize = ((data[0]>>3)&0x3); + audiosize = (Fs<>3)&0x3); + if (audiosize == 3) + audiosize = Fs*60/1000; + else + audiosize = (Fs< len) + return OPUS_INVALID_PACKET; + data += bytes; + last_size = len-size[0]; + break; + /* Multiple CBR/VBR frames (from 0 to 120 ms) */ + default: /*case 3:*/ + if (len<1) + return OPUS_INVALID_PACKET; + /* Number of frames encoded in bits 0 to 5 */ + ch = *data++; + count = ch&0x3F; + if (count <= 0 || framesize*(opus_int32)count > 5760) + return OPUS_INVALID_PACKET; + len--; + /* Padding flag is bit 6 */ + if (ch&0x40) + { + int p; + do { + int tmp; + if (len<=0) + return OPUS_INVALID_PACKET; + p = *data++; + len--; + tmp = p==255 ? 254: p; + len -= tmp; + pad += tmp; + } while (p==255); + } + if (len<0) + return OPUS_INVALID_PACKET; + /* VBR flag is bit 7 */ + cbr = !(ch&0x80); + if (!cbr) + { + /* VBR case */ + last_size = len; + for (i=0;i len) + return OPUS_INVALID_PACKET; + data += bytes; + last_size -= bytes+size[i]; + } + if (last_size<0) + return OPUS_INVALID_PACKET; + } else if (!self_delimited) + { + /* CBR case */ + last_size = len/count; + if (last_size*count!=len) + return OPUS_INVALID_PACKET; + for (i=0;i len) + return OPUS_INVALID_PACKET; + data += bytes; + /* For CBR packets, apply the size to all the frames. */ + if (cbr) + { + if (size[count-1]*count > len) + return OPUS_INVALID_PACKET; + for (i=0;i last_size) + return OPUS_INVALID_PACKET; + } else + { + /* Because it's not encoded explicitly, it's possible the size of the + last packet (or all the packets, for the CBR case) is larger than + 1275. Reject them here.*/ + if (last_size > 1275) + return OPUS_INVALID_PACKET; + size[count-1] = (opus_int16)last_size; + } + + if (payload_offset) + *payload_offset = (int)(data-data0); + + for (i=0;i +#include +#include +#include + +#define OPUS_PI (3.14159265F) + +#define OPUS_COSF(_x) ((float)cos(_x)) +#define OPUS_SINF(_x) ((float)sin(_x)) + +static void *check_alloc(void *_ptr){ + if(_ptr==NULL){ + fprintf(stderr,"Out of memory.\n"); + exit(EXIT_FAILURE); + } + return _ptr; +} + +static void *opus_malloc(size_t _size){ + return check_alloc(malloc(_size)); +} + +static void *opus_realloc(void *_ptr,size_t _size){ + return check_alloc(realloc(_ptr,_size)); +} + +static size_t read_pcm16(float **_samples,FILE *_fin,int _nchannels){ + unsigned char buf[1024]; + float *samples; + size_t nsamples; + size_t csamples; + size_t xi; + size_t nread; + samples=NULL; + nsamples=csamples=0; + for(;;){ + nread=fread(buf,2*_nchannels,1024/(2*_nchannels),_fin); + if(nread<=0)break; + if(nsamples+nread>csamples){ + do csamples=csamples<<1|1; + while(nsamples+nread>csamples); + samples=(float *)opus_realloc(samples, + _nchannels*csamples*sizeof(*samples)); + } + for(xi=0;xi=_window_sz)ti-=_window_sz; + } + re*=_downsample; + im*=_downsample; + _ps[(xi*ps_sz+xj)*_nchannels+ci]=re*re+im*im+100000; + p[ci]+=_ps[(xi*ps_sz+xj)*_nchannels+ci]; + } + } + if(_out){ + _out[(xi*_nbands+bi)*_nchannels]=p[0]/(_bands[bi+1]-_bands[bi]); + if(_nchannels==2){ + _out[(xi*_nbands+bi)*_nchannels+1]=p[1]/(_bands[bi+1]-_bands[bi]); + } + } + } + } + free(window); +} + +#define NBANDS (21) +#define NFREQS (240) + +/*Bands on which we compute the pseudo-NMR (Bark-derived + CELT bands).*/ +static const int BANDS[NBANDS+1]={ + 0,2,4,6,8,10,12,14,16,20,24,28,32,40,48,56,68,80,96,120,156,200 +}; + +#define TEST_WIN_SIZE (480) +#define TEST_WIN_STEP (120) + +int main(int _argc,const char **_argv){ + FILE *fin1; + FILE *fin2; + float *x; + float *y; + float *xb; + float *X; + float *Y; + double err; + float Q; + size_t xlength; + size_t ylength; + size_t nframes; + size_t xi; + int ci; + int xj; + int bi; + int nchannels; + unsigned rate; + int downsample; + int ybands; + int yfreqs; + int max_compare; + if(_argc<3||_argc>6){ + fprintf(stderr,"Usage: %s [-s] [-r rate2] \n", + _argv[0]); + return EXIT_FAILURE; + } + nchannels=1; + if(strcmp(_argv[1],"-s")==0){ + nchannels=2; + _argv++; + } + rate=48000; + ybands=NBANDS; + yfreqs=NFREQS; + downsample=1; + if(strcmp(_argv[1],"-r")==0){ + rate=atoi(_argv[2]); + if(rate!=8000&&rate!=12000&&rate!=16000&&rate!=24000&&rate!=48000){ + fprintf(stderr, + "Sampling rate must be 8000, 12000, 16000, 24000, or 48000\n"); + return EXIT_FAILURE; + } + downsample=48000/rate; + switch(rate){ + case 8000:ybands=13;break; + case 12000:ybands=15;break; + case 16000:ybands=17;break; + case 24000:ybands=19;break; + } + yfreqs=NFREQS/downsample; + _argv+=2; + } + fin1=fopen(_argv[1],"rb"); + if(fin1==NULL){ + fprintf(stderr,"Error opening '%s'.\n",_argv[1]); + return EXIT_FAILURE; + } + fin2=fopen(_argv[2],"rb"); + if(fin2==NULL){ + fprintf(stderr,"Error opening '%s'.\n",_argv[2]); + fclose(fin1); + return EXIT_FAILURE; + } + /*Read in the data and allocate scratch space.*/ + xlength=read_pcm16(&x,fin1,2); + if(nchannels==1){ + for(xi=0;xi0;){ + for(ci=0;ci0){ + /*Temporal masking: -3 dB/2.5ms slope.*/ + for(bi=0;bi=79&&xj<=81)im*=0.1F; + if(xj==80)im*=0.1F; + Eb+=im; + } + } + Eb /= (BANDS[bi+1]-BANDS[bi])*nchannels; + Ef += Eb*Eb; + } + /*Using a fixed normalization value means we're willing to accept slightly + lower quality for lower sampling rates.*/ + Ef/=NBANDS; + Ef*=Ef; + err+=Ef*Ef; + } + free(xb); + free(X); + free(Y); + err=pow(err/nframes,1.0/16); + Q=100*(1-0.5*log(1+err)/log(1.13)); + if(Q<0){ + fprintf(stderr,"Test vector FAILS\n"); + fprintf(stderr,"Internal weighted error is %f\n",err); + return EXIT_FAILURE; + } + else{ + fprintf(stderr,"Test vector PASSES\n"); + fprintf(stderr, + "Opus quality metric: %.1f %% (internal weighted error is %f)\n",Q,err); + return EXIT_SUCCESS; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_decoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_decoder.c new file mode 100644 index 000000000..9113638a0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_decoder.c @@ -0,0 +1,1032 @@ +/* Copyright (c) 2010 Xiph.Org Foundation, Skype Limited + Written by Jean-Marc Valin and Koen Vos */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#ifndef OPUS_BUILD +# error "OPUS_BUILD _MUST_ be defined to build Opus. This probably means you need other defines as well, as in a config.h. See the included build files for details." +#endif + +#if defined(__GNUC__) && (__GNUC__ >= 2) && !defined(__OPTIMIZE__) && !defined(OPUS_WILL_BE_SLOW) +# pragma message "You appear to be compiling without optimization, if so opus will be very slow." +#endif + +#include +#include "celt.h" +#include "opus.h" +#include "entdec.h" +#include "modes.h" +#include "API.h" +#include "stack_alloc.h" +#include "float_cast.h" +#include "opus_private.h" +#include "os_support.h" +#include "structs.h" +#include "define.h" +#include "mathops.h" +#include "cpu_support.h" + +struct OpusDecoder { + int celt_dec_offset; + int silk_dec_offset; + int channels; + opus_int32 Fs; /** Sampling rate (at the API level) */ + silk_DecControlStruct DecControl; + int decode_gain; + int arch; + + /* Everything beyond this point gets cleared on a reset */ +#define OPUS_DECODER_RESET_START stream_channels + int stream_channels; + + int bandwidth; + int mode; + int prev_mode; + int frame_size; + int prev_redundancy; + int last_packet_duration; +#ifndef FIXED_POINT + opus_val16 softclip_mem[2]; +#endif + + opus_uint32 rangeFinal; +}; + +#if defined(ENABLE_HARDENING) || defined(ENABLE_ASSERTIONS) +static void validate_opus_decoder(OpusDecoder *st) +{ + celt_assert(st->channels == 1 || st->channels == 2); + celt_assert(st->Fs == 48000 || st->Fs == 24000 || st->Fs == 16000 || st->Fs == 12000 || st->Fs == 8000); + celt_assert(st->DecControl.API_sampleRate == st->Fs); + celt_assert(st->DecControl.internalSampleRate == 0 || st->DecControl.internalSampleRate == 16000 || st->DecControl.internalSampleRate == 12000 || st->DecControl.internalSampleRate == 8000); + celt_assert(st->DecControl.nChannelsAPI == st->channels); + celt_assert(st->DecControl.nChannelsInternal == 0 || st->DecControl.nChannelsInternal == 1 || st->DecControl.nChannelsInternal == 2); + celt_assert(st->DecControl.payloadSize_ms == 0 || st->DecControl.payloadSize_ms == 10 || st->DecControl.payloadSize_ms == 20 || st->DecControl.payloadSize_ms == 40 || st->DecControl.payloadSize_ms == 60); +#ifdef OPUS_ARCHMASK + celt_assert(st->arch >= 0); + celt_assert(st->arch <= OPUS_ARCHMASK); +#endif + celt_assert(st->stream_channels == 1 || st->stream_channels == 2); +} +#define VALIDATE_OPUS_DECODER(st) validate_opus_decoder(st) +#else +#define VALIDATE_OPUS_DECODER(st) +#endif + +int opus_decoder_get_size(int channels) +{ + int silkDecSizeBytes, celtDecSizeBytes; + int ret; + if (channels<1 || channels > 2) + return 0; + ret = silk_Get_Decoder_Size( &silkDecSizeBytes ); + if(ret) + return 0; + silkDecSizeBytes = align(silkDecSizeBytes); + celtDecSizeBytes = celt_decoder_get_size(channels); + return align(sizeof(OpusDecoder))+silkDecSizeBytes+celtDecSizeBytes; +} + +int opus_decoder_init(OpusDecoder *st, opus_int32 Fs, int channels) +{ + void *silk_dec; + CELTDecoder *celt_dec; + int ret, silkDecSizeBytes; + + if ((Fs!=48000&&Fs!=24000&&Fs!=16000&&Fs!=12000&&Fs!=8000) + || (channels!=1&&channels!=2)) + return OPUS_BAD_ARG; + + OPUS_CLEAR((char*)st, opus_decoder_get_size(channels)); + /* Initialize SILK decoder */ + ret = silk_Get_Decoder_Size(&silkDecSizeBytes); + if (ret) + return OPUS_INTERNAL_ERROR; + + silkDecSizeBytes = align(silkDecSizeBytes); + st->silk_dec_offset = align(sizeof(OpusDecoder)); + st->celt_dec_offset = st->silk_dec_offset+silkDecSizeBytes; + silk_dec = (char*)st+st->silk_dec_offset; + celt_dec = (CELTDecoder*)((char*)st+st->celt_dec_offset); + st->stream_channels = st->channels = channels; + + st->Fs = Fs; + st->DecControl.API_sampleRate = st->Fs; + st->DecControl.nChannelsAPI = st->channels; + + /* Reset decoder */ + ret = silk_InitDecoder( silk_dec ); + if(ret)return OPUS_INTERNAL_ERROR; + + /* Initialize CELT decoder */ + ret = celt_decoder_init(celt_dec, Fs, channels); + if(ret!=OPUS_OK)return OPUS_INTERNAL_ERROR; + + celt_decoder_ctl(celt_dec, CELT_SET_SIGNALLING(0)); + + st->prev_mode = 0; + st->frame_size = Fs/400; + st->arch = opus_select_arch(); + return OPUS_OK; +} + +OpusDecoder *opus_decoder_create(opus_int32 Fs, int channels, int *error) +{ + int ret; + OpusDecoder *st; + if ((Fs!=48000&&Fs!=24000&&Fs!=16000&&Fs!=12000&&Fs!=8000) + || (channels!=1&&channels!=2)) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + st = (OpusDecoder *)opus_alloc(opus_decoder_get_size(channels)); + if (st == NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + ret = opus_decoder_init(st, Fs, channels); + if (error) + *error = ret; + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + return st; +} + +static void smooth_fade(const opus_val16 *in1, const opus_val16 *in2, + opus_val16 *out, int overlap, int channels, + const opus_val16 *window, opus_int32 Fs) +{ + int i, c; + int inc = 48000/Fs; + for (c=0;csilk_dec_offset; + celt_dec = (CELTDecoder*)((char*)st+st->celt_dec_offset); + F20 = st->Fs/50; + F10 = F20>>1; + F5 = F10>>1; + F2_5 = F5>>1; + if (frame_size < F2_5) + { + RESTORE_STACK; + return OPUS_BUFFER_TOO_SMALL; + } + /* Limit frame_size to avoid excessive stack allocations. */ + frame_size = IMIN(frame_size, st->Fs/25*3); + /* Payloads of 1 (2 including ToC) or 0 trigger the PLC/DTX */ + if (len<=1) + { + data = NULL; + /* In that case, don't conceal more than what the ToC says */ + frame_size = IMIN(frame_size, st->frame_size); + } + if (data != NULL) + { + audiosize = st->frame_size; + mode = st->mode; + bandwidth = st->bandwidth; + ec_dec_init(&dec,(unsigned char*)data,len); + } else { + audiosize = frame_size; + mode = st->prev_mode; + bandwidth = 0; + + if (mode == 0) + { + /* If we haven't got any packet yet, all we can do is return zeros */ + for (i=0;ichannels;i++) + pcm[i] = 0; + RESTORE_STACK; + return audiosize; + } + + /* Avoids trying to run the PLC on sizes other than 2.5 (CELT), 5 (CELT), + 10, or 20 (e.g. 12.5 or 30 ms). */ + if (audiosize > F20) + { + do { + int ret = opus_decode_frame(st, NULL, 0, pcm, IMIN(audiosize, F20), 0); + if (ret<0) + { + RESTORE_STACK; + return ret; + } + pcm += ret*st->channels; + audiosize -= ret; + } while (audiosize > 0); + RESTORE_STACK; + return frame_size; + } else if (audiosize < F20) + { + if (audiosize > F10) + audiosize = F10; + else if (mode != MODE_SILK_ONLY && audiosize > F5 && audiosize < F10) + audiosize = F5; + } + } + + /* In fixed-point, we can tell CELT to do the accumulation on top of the + SILK PCM buffer. This saves some stack space. */ +#ifdef FIXED_POINT + celt_accum = (mode != MODE_CELT_ONLY) && (frame_size >= F10); +#else + celt_accum = 0; +#endif + + pcm_transition_silk_size = ALLOC_NONE; + pcm_transition_celt_size = ALLOC_NONE; + if (data!=NULL && st->prev_mode > 0 && ( + (mode == MODE_CELT_ONLY && st->prev_mode != MODE_CELT_ONLY && !st->prev_redundancy) + || (mode != MODE_CELT_ONLY && st->prev_mode == MODE_CELT_ONLY) ) + ) + { + transition = 1; + /* Decide where to allocate the stack memory for pcm_transition */ + if (mode == MODE_CELT_ONLY) + pcm_transition_celt_size = F5*st->channels; + else + pcm_transition_silk_size = F5*st->channels; + } + ALLOC(pcm_transition_celt, pcm_transition_celt_size, opus_val16); + if (transition && mode == MODE_CELT_ONLY) + { + pcm_transition = pcm_transition_celt; + opus_decode_frame(st, NULL, 0, pcm_transition, IMIN(F5, audiosize), 0); + } + if (audiosize > frame_size) + { + /*fprintf(stderr, "PCM buffer too small: %d vs %d (mode = %d)\n", audiosize, frame_size, mode);*/ + RESTORE_STACK; + return OPUS_BAD_ARG; + } else { + frame_size = audiosize; + } + + /* Don't allocate any memory when in CELT-only mode */ + pcm_silk_size = (mode != MODE_CELT_ONLY && !celt_accum) ? IMAX(F10, frame_size)*st->channels : ALLOC_NONE; + ALLOC(pcm_silk, pcm_silk_size, opus_int16); + + /* SILK processing */ + if (mode != MODE_CELT_ONLY) + { + int lost_flag, decoded_samples; + opus_int16 *pcm_ptr; +#ifdef FIXED_POINT + if (celt_accum) + pcm_ptr = pcm; + else +#endif + pcm_ptr = pcm_silk; + + if (st->prev_mode==MODE_CELT_ONLY) + silk_InitDecoder( silk_dec ); + + /* The SILK PLC cannot produce frames of less than 10 ms */ + st->DecControl.payloadSize_ms = IMAX(10, 1000 * audiosize / st->Fs); + + if (data != NULL) + { + st->DecControl.nChannelsInternal = st->stream_channels; + if( mode == MODE_SILK_ONLY ) { + if( bandwidth == OPUS_BANDWIDTH_NARROWBAND ) { + st->DecControl.internalSampleRate = 8000; + } else if( bandwidth == OPUS_BANDWIDTH_MEDIUMBAND ) { + st->DecControl.internalSampleRate = 12000; + } else if( bandwidth == OPUS_BANDWIDTH_WIDEBAND ) { + st->DecControl.internalSampleRate = 16000; + } else { + st->DecControl.internalSampleRate = 16000; + celt_assert( 0 ); + } + } else { + /* Hybrid mode */ + st->DecControl.internalSampleRate = 16000; + } + } + + lost_flag = data == NULL ? 1 : 2 * decode_fec; + decoded_samples = 0; + do { + /* Call SILK decoder */ + int first_frame = decoded_samples == 0; + silk_ret = silk_Decode( silk_dec, &st->DecControl, + lost_flag, first_frame, &dec, pcm_ptr, &silk_frame_size, st->arch ); + if( silk_ret ) { + if (lost_flag) { + /* PLC failure should not be fatal */ + silk_frame_size = frame_size; + for (i=0;ichannels;i++) + pcm_ptr[i] = 0; + } else { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + } + pcm_ptr += silk_frame_size * st->channels; + decoded_samples += silk_frame_size; + } while( decoded_samples < frame_size ); + } + + start_band = 0; + if (!decode_fec && mode != MODE_CELT_ONLY && data != NULL + && ec_tell(&dec)+17+20*(st->mode == MODE_HYBRID) <= 8*len) + { + /* Check if we have a redundant 0-8 kHz band */ + if (mode == MODE_HYBRID) + redundancy = ec_dec_bit_logp(&dec, 12); + else + redundancy = 1; + if (redundancy) + { + celt_to_silk = ec_dec_bit_logp(&dec, 1); + /* redundancy_bytes will be at least two, in the non-hybrid + case due to the ec_tell() check above */ + redundancy_bytes = mode==MODE_HYBRID ? + (opus_int32)ec_dec_uint(&dec, 256)+2 : + len-((ec_tell(&dec)+7)>>3); + len -= redundancy_bytes; + /* This is a sanity check. It should never happen for a valid + packet, so the exact behaviour is not normative. */ + if (len*8 < ec_tell(&dec)) + { + len = 0; + redundancy_bytes = 0; + redundancy = 0; + } + /* Shrink decoder because of raw bits */ + dec.storage -= redundancy_bytes; + } + } + if (mode != MODE_CELT_ONLY) + start_band = 17; + + if (redundancy) + { + transition = 0; + pcm_transition_silk_size=ALLOC_NONE; + } + + ALLOC(pcm_transition_silk, pcm_transition_silk_size, opus_val16); + + if (transition && mode != MODE_CELT_ONLY) + { + pcm_transition = pcm_transition_silk; + opus_decode_frame(st, NULL, 0, pcm_transition, IMIN(F5, audiosize), 0); + } + + + if (bandwidth) + { + int endband=21; + + switch(bandwidth) + { + case OPUS_BANDWIDTH_NARROWBAND: + endband = 13; + break; + case OPUS_BANDWIDTH_MEDIUMBAND: + case OPUS_BANDWIDTH_WIDEBAND: + endband = 17; + break; + case OPUS_BANDWIDTH_SUPERWIDEBAND: + endband = 19; + break; + case OPUS_BANDWIDTH_FULLBAND: + endband = 21; + break; + default: + celt_assert(0); + break; + } + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_END_BAND(endband))); + } + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_CHANNELS(st->stream_channels))); + + /* Only allocation memory for redundancy if/when needed */ + redundant_audio_size = redundancy ? F5*st->channels : ALLOC_NONE; + ALLOC(redundant_audio, redundant_audio_size, opus_val16); + + /* 5 ms redundant frame for CELT->SILK*/ + if (redundancy && celt_to_silk) + { + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_START_BAND(0))); + celt_decode_with_ec(celt_dec, data+len, redundancy_bytes, + redundant_audio, F5, NULL, 0); + MUST_SUCCEED(celt_decoder_ctl(celt_dec, OPUS_GET_FINAL_RANGE(&redundant_rng))); + } + + /* MUST be after PLC */ + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_START_BAND(start_band))); + + if (mode != MODE_SILK_ONLY) + { + int celt_frame_size = IMIN(F20, frame_size); + /* Make sure to discard any previous CELT state */ + if (mode != st->prev_mode && st->prev_mode > 0 && !st->prev_redundancy) + MUST_SUCCEED(celt_decoder_ctl(celt_dec, OPUS_RESET_STATE)); + /* Decode CELT */ + celt_ret = celt_decode_with_ec(celt_dec, decode_fec ? NULL : data, + len, pcm, celt_frame_size, &dec, celt_accum); + } else { + unsigned char silence[2] = {0xFF, 0xFF}; + if (!celt_accum) + { + for (i=0;ichannels;i++) + pcm[i] = 0; + } + /* For hybrid -> SILK transitions, we let the CELT MDCT + do a fade-out by decoding a silence frame */ + if (st->prev_mode == MODE_HYBRID && !(redundancy && celt_to_silk && st->prev_redundancy) ) + { + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_START_BAND(0))); + celt_decode_with_ec(celt_dec, silence, 2, pcm, F2_5, NULL, celt_accum); + } + } + + if (mode != MODE_CELT_ONLY && !celt_accum) + { +#ifdef FIXED_POINT + for (i=0;ichannels;i++) + pcm[i] = SAT16(ADD32(pcm[i], pcm_silk[i])); +#else + for (i=0;ichannels;i++) + pcm[i] = pcm[i] + (opus_val16)((1.f/32768.f)*pcm_silk[i]); +#endif + } + + { + const CELTMode *celt_mode; + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_GET_MODE(&celt_mode))); + window = celt_mode->window; + } + + /* 5 ms redundant frame for SILK->CELT */ + if (redundancy && !celt_to_silk) + { + MUST_SUCCEED(celt_decoder_ctl(celt_dec, OPUS_RESET_STATE)); + MUST_SUCCEED(celt_decoder_ctl(celt_dec, CELT_SET_START_BAND(0))); + + celt_decode_with_ec(celt_dec, data+len, redundancy_bytes, redundant_audio, F5, NULL, 0); + MUST_SUCCEED(celt_decoder_ctl(celt_dec, OPUS_GET_FINAL_RANGE(&redundant_rng))); + smooth_fade(pcm+st->channels*(frame_size-F2_5), redundant_audio+st->channels*F2_5, + pcm+st->channels*(frame_size-F2_5), F2_5, st->channels, window, st->Fs); + } + if (redundancy && celt_to_silk) + { + for (c=0;cchannels;c++) + { + for (i=0;ichannels*i+c] = redundant_audio[st->channels*i+c]; + } + smooth_fade(redundant_audio+st->channels*F2_5, pcm+st->channels*F2_5, + pcm+st->channels*F2_5, F2_5, st->channels, window, st->Fs); + } + if (transition) + { + if (audiosize >= F5) + { + for (i=0;ichannels*F2_5;i++) + pcm[i] = pcm_transition[i]; + smooth_fade(pcm_transition+st->channels*F2_5, pcm+st->channels*F2_5, + pcm+st->channels*F2_5, F2_5, + st->channels, window, st->Fs); + } else { + /* Not enough time to do a clean transition, but we do it anyway + This will not preserve amplitude perfectly and may introduce + a bit of temporal aliasing, but it shouldn't be too bad and + that's pretty much the best we can do. In any case, generating this + transition it pretty silly in the first place */ + smooth_fade(pcm_transition, pcm, + pcm, F2_5, + st->channels, window, st->Fs); + } + } + + if(st->decode_gain) + { + opus_val32 gain; + gain = celt_exp2(MULT16_16_P15(QCONST16(6.48814081e-4f, 25), st->decode_gain)); + for (i=0;ichannels;i++) + { + opus_val32 x; + x = MULT16_32_P16(pcm[i],gain); + pcm[i] = SATURATE(x, 32767); + } + } + + if (len <= 1) + st->rangeFinal = 0; + else + st->rangeFinal = dec.rng ^ redundant_rng; + + st->prev_mode = mode; + st->prev_redundancy = redundancy && !celt_to_silk; + + if (celt_ret>=0) + { + if (OPUS_CHECK_ARRAY(pcm, audiosize*st->channels)) + OPUS_PRINT_INT(audiosize); + } + + RESTORE_STACK; + return celt_ret < 0 ? celt_ret : audiosize; + +} + +int opus_decode_native(OpusDecoder *st, const unsigned char *data, + opus_int32 len, opus_val16 *pcm, int frame_size, int decode_fec, + int self_delimited, opus_int32 *packet_offset, int soft_clip) +{ + int i, nb_samples; + int count, offset; + unsigned char toc; + int packet_frame_size, packet_bandwidth, packet_mode, packet_stream_channels; + /* 48 x 2.5 ms = 120 ms */ + opus_int16 size[48]; + VALIDATE_OPUS_DECODER(st); + if (decode_fec<0 || decode_fec>1) + return OPUS_BAD_ARG; + /* For FEC/PLC, frame_size has to be to have a multiple of 2.5 ms */ + if ((decode_fec || len==0 || data==NULL) && frame_size%(st->Fs/400)!=0) + return OPUS_BAD_ARG; + if (len==0 || data==NULL) + { + int pcm_count=0; + do { + int ret; + ret = opus_decode_frame(st, NULL, 0, pcm+pcm_count*st->channels, frame_size-pcm_count, 0); + if (ret<0) + return ret; + pcm_count += ret; + } while (pcm_count < frame_size); + celt_assert(pcm_count == frame_size); + if (OPUS_CHECK_ARRAY(pcm, pcm_count*st->channels)) + OPUS_PRINT_INT(pcm_count); + st->last_packet_duration = pcm_count; + return pcm_count; + } else if (len<0) + return OPUS_BAD_ARG; + + packet_mode = opus_packet_get_mode(data); + packet_bandwidth = opus_packet_get_bandwidth(data); + packet_frame_size = opus_packet_get_samples_per_frame(data, st->Fs); + packet_stream_channels = opus_packet_get_nb_channels(data); + + count = opus_packet_parse_impl(data, len, self_delimited, &toc, NULL, + size, &offset, packet_offset); + if (count<0) + return count; + + data += offset; + + if (decode_fec) + { + int duration_copy; + int ret; + /* If no FEC can be present, run the PLC (recursive call) */ + if (frame_size < packet_frame_size || packet_mode == MODE_CELT_ONLY || st->mode == MODE_CELT_ONLY) + return opus_decode_native(st, NULL, 0, pcm, frame_size, 0, 0, NULL, soft_clip); + /* Otherwise, run the PLC on everything except the size for which we might have FEC */ + duration_copy = st->last_packet_duration; + if (frame_size-packet_frame_size!=0) + { + ret = opus_decode_native(st, NULL, 0, pcm, frame_size-packet_frame_size, 0, 0, NULL, soft_clip); + if (ret<0) + { + st->last_packet_duration = duration_copy; + return ret; + } + celt_assert(ret==frame_size-packet_frame_size); + } + /* Complete with FEC */ + st->mode = packet_mode; + st->bandwidth = packet_bandwidth; + st->frame_size = packet_frame_size; + st->stream_channels = packet_stream_channels; + ret = opus_decode_frame(st, data, size[0], pcm+st->channels*(frame_size-packet_frame_size), + packet_frame_size, 1); + if (ret<0) + return ret; + else { + if (OPUS_CHECK_ARRAY(pcm, frame_size*st->channels)) + OPUS_PRINT_INT(frame_size); + st->last_packet_duration = frame_size; + return frame_size; + } + } + + if (count*packet_frame_size > frame_size) + return OPUS_BUFFER_TOO_SMALL; + + /* Update the state as the last step to avoid updating it on an invalid packet */ + st->mode = packet_mode; + st->bandwidth = packet_bandwidth; + st->frame_size = packet_frame_size; + st->stream_channels = packet_stream_channels; + + nb_samples=0; + for (i=0;ichannels, frame_size-nb_samples, 0); + if (ret<0) + return ret; + celt_assert(ret==packet_frame_size); + data += size[i]; + nb_samples += ret; + } + st->last_packet_duration = nb_samples; + if (OPUS_CHECK_ARRAY(pcm, nb_samples*st->channels)) + OPUS_PRINT_INT(nb_samples); +#ifndef FIXED_POINT + if (soft_clip) + opus_pcm_soft_clip(pcm, nb_samples, st->channels, st->softclip_mem); + else + st->softclip_mem[0]=st->softclip_mem[1]=0; +#endif + return nb_samples; +} + +#ifdef FIXED_POINT + +int opus_decode(OpusDecoder *st, const unsigned char *data, + opus_int32 len, opus_val16 *pcm, int frame_size, int decode_fec) +{ + if(frame_size<=0) + return OPUS_BAD_ARG; + return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0); +} + +#ifndef DISABLE_FLOAT_API +int opus_decode_float(OpusDecoder *st, const unsigned char *data, + opus_int32 len, float *pcm, int frame_size, int decode_fec) +{ + VARDECL(opus_int16, out); + int ret, i; + int nb_samples; + ALLOC_STACK; + + if(frame_size<=0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + if (data != NULL && len > 0 && !decode_fec) + { + nb_samples = opus_decoder_get_nb_samples(st, data, len); + if (nb_samples>0) + frame_size = IMIN(frame_size, nb_samples); + else + return OPUS_INVALID_PACKET; + } + celt_assert(st->channels == 1 || st->channels == 2); + ALLOC(out, frame_size*st->channels, opus_int16); + + ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 0); + if (ret > 0) + { + for (i=0;ichannels;i++) + pcm[i] = (1.f/32768.f)*(out[i]); + } + RESTORE_STACK; + return ret; +} +#endif + + +#else +int opus_decode(OpusDecoder *st, const unsigned char *data, + opus_int32 len, opus_int16 *pcm, int frame_size, int decode_fec) +{ + VARDECL(float, out); + int ret, i; + int nb_samples; + ALLOC_STACK; + + if(frame_size<=0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + if (data != NULL && len > 0 && !decode_fec) + { + nb_samples = opus_decoder_get_nb_samples(st, data, len); + if (nb_samples>0) + frame_size = IMIN(frame_size, nb_samples); + else + return OPUS_INVALID_PACKET; + } + celt_assert(st->channels == 1 || st->channels == 2); + ALLOC(out, frame_size*st->channels, float); + + ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 1); + if (ret > 0) + { + for (i=0;ichannels;i++) + pcm[i] = FLOAT2INT16(out[i]); + } + RESTORE_STACK; + return ret; +} + +int opus_decode_float(OpusDecoder *st, const unsigned char *data, + opus_int32 len, opus_val16 *pcm, int frame_size, int decode_fec) +{ + if(frame_size<=0) + return OPUS_BAD_ARG; + return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0); +} + +#endif + +int opus_decoder_ctl(OpusDecoder *st, int request, ...) +{ + int ret = OPUS_OK; + va_list ap; + void *silk_dec; + CELTDecoder *celt_dec; + + silk_dec = (char*)st+st->silk_dec_offset; + celt_dec = (CELTDecoder*)((char*)st+st->celt_dec_offset); + + + va_start(ap, request); + + switch (request) + { + case OPUS_GET_BANDWIDTH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->bandwidth; + } + break; + case OPUS_GET_FINAL_RANGE_REQUEST: + { + opus_uint32 *value = va_arg(ap, opus_uint32*); + if (!value) + { + goto bad_arg; + } + *value = st->rangeFinal; + } + break; + case OPUS_RESET_STATE: + { + OPUS_CLEAR((char*)&st->OPUS_DECODER_RESET_START, + sizeof(OpusDecoder)- + ((char*)&st->OPUS_DECODER_RESET_START - (char*)st)); + + celt_decoder_ctl(celt_dec, OPUS_RESET_STATE); + silk_InitDecoder( silk_dec ); + st->stream_channels = st->channels; + st->frame_size = st->Fs/400; + } + break; + case OPUS_GET_SAMPLE_RATE_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->Fs; + } + break; + case OPUS_GET_PITCH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + if (st->prev_mode == MODE_CELT_ONLY) + ret = celt_decoder_ctl(celt_dec, OPUS_GET_PITCH(value)); + else + *value = st->DecControl.prevPitchLag; + } + break; + case OPUS_GET_GAIN_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->decode_gain; + } + break; + case OPUS_SET_GAIN_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<-32768 || value>32767) + { + goto bad_arg; + } + st->decode_gain = value; + } + break; + case OPUS_GET_LAST_PACKET_DURATION_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->last_packet_duration; + } + break; + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + ret = celt_decoder_ctl(celt_dec, OPUS_SET_PHASE_INVERSION_DISABLED(value)); + } + break; + case OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + ret = celt_decoder_ctl(celt_dec, OPUS_GET_PHASE_INVERSION_DISABLED(value)); + } + break; + default: + /*fprintf(stderr, "unknown opus_decoder_ctl() request: %d", request);*/ + ret = OPUS_UNIMPLEMENTED; + break; + } + + va_end(ap); + return ret; +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +} + +void opus_decoder_destroy(OpusDecoder *st) +{ + opus_free(st); +} + + +int opus_packet_get_bandwidth(const unsigned char *data) +{ + int bandwidth; + if (data[0]&0x80) + { + bandwidth = OPUS_BANDWIDTH_MEDIUMBAND + ((data[0]>>5)&0x3); + if (bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) + bandwidth = OPUS_BANDWIDTH_NARROWBAND; + } else if ((data[0]&0x60) == 0x60) + { + bandwidth = (data[0]&0x10) ? OPUS_BANDWIDTH_FULLBAND : + OPUS_BANDWIDTH_SUPERWIDEBAND; + } else { + bandwidth = OPUS_BANDWIDTH_NARROWBAND + ((data[0]>>5)&0x3); + } + return bandwidth; +} + +int opus_packet_get_nb_channels(const unsigned char *data) +{ + return (data[0]&0x4) ? 2 : 1; +} + +int opus_packet_get_nb_frames(const unsigned char packet[], opus_int32 len) +{ + int count; + if (len<1) + return OPUS_BAD_ARG; + count = packet[0]&0x3; + if (count==0) + return 1; + else if (count!=3) + return 2; + else if (len<2) + return OPUS_INVALID_PACKET; + else + return packet[1]&0x3F; +} + +int opus_packet_get_nb_samples(const unsigned char packet[], opus_int32 len, + opus_int32 Fs) +{ + int samples; + int count = opus_packet_get_nb_frames(packet, len); + + if (count<0) + return count; + + samples = count*opus_packet_get_samples_per_frame(packet, Fs); + /* Can't have more than 120 ms */ + if (samples*25 > Fs*3) + return OPUS_INVALID_PACKET; + else + return samples; +} + +int opus_decoder_get_nb_samples(const OpusDecoder *dec, + const unsigned char packet[], opus_int32 len) +{ + return opus_packet_get_nb_samples(packet, len, dec->Fs); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_demo.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_demo.c new file mode 100644 index 000000000..4cc26a6c7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_demo.c @@ -0,0 +1,892 @@ +/* Copyright (c) 2007-2008 CSIRO + Copyright (c) 2007-2009 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include "opus.h" +#include "debug.h" +#include "opus_types.h" +#include "opus_private.h" +#include "opus_multistream.h" + +#define MAX_PACKET 1500 + +void print_usage( char* argv[] ) +{ + fprintf(stderr, "Usage: %s [-e] " + " [options] \n", argv[0]); + fprintf(stderr, " %s -d " + "[options] \n\n", argv[0]); + fprintf(stderr, "application: voip | audio | restricted-lowdelay\n" ); + fprintf(stderr, "options:\n" ); + fprintf(stderr, "-e : only runs the encoder (output the bit-stream)\n" ); + fprintf(stderr, "-d : only runs the decoder (reads the bit-stream as input)\n" ); + fprintf(stderr, "-cbr : enable constant bitrate; default: variable bitrate\n" ); + fprintf(stderr, "-cvbr : enable constrained variable bitrate; default: unconstrained\n" ); + fprintf(stderr, "-delayed-decision : use look-ahead for speech/music detection (experts only); default: disabled\n" ); + fprintf(stderr, "-bandwidth : audio bandwidth (from narrowband to fullband); default: sampling rate\n" ); + fprintf(stderr, "-framesize <2.5|5|10|20|40|60|80|100|120> : frame size in ms; default: 20 \n" ); + fprintf(stderr, "-max_payload : maximum payload size in bytes, default: 1024\n" ); + fprintf(stderr, "-complexity : complexity, 0 (lowest) ... 10 (highest); default: 10\n" ); + fprintf(stderr, "-inbandfec : enable SILK inband FEC\n" ); + fprintf(stderr, "-forcemono : force mono encoding, even for stereo input\n" ); + fprintf(stderr, "-dtx : enable SILK DTX\n" ); + fprintf(stderr, "-loss : simulate packet loss, in percent (0-100); default: 0\n" ); +} + +static void int_to_char(opus_uint32 i, unsigned char ch[4]) +{ + ch[0] = i>>24; + ch[1] = (i>>16)&0xFF; + ch[2] = (i>>8)&0xFF; + ch[3] = i&0xFF; +} + +static opus_uint32 char_to_int(unsigned char ch[4]) +{ + return ((opus_uint32)ch[0]<<24) | ((opus_uint32)ch[1]<<16) + | ((opus_uint32)ch[2]<< 8) | (opus_uint32)ch[3]; +} + +#define check_encoder_option(decode_only, opt) do {if (decode_only) {fprintf(stderr, "option %s is only for encoding\n", opt); goto failure;}} while(0) + +static const int silk8_test[][4] = { + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960*3, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960*2, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 480, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960*3, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960*2, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_NARROWBAND, 480, 2} +}; + +static const int silk12_test[][4] = { + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960*3, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960*2, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 480, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960*3, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960*2, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 960, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_MEDIUMBAND, 480, 2} +}; + +static const int silk16_test[][4] = { + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960*3, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960*2, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 480, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960*3, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960*2, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_WIDEBAND, 480, 2} +}; + +static const int hybrid24_test[][4] = { + {MODE_SILK_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 960, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 480, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 960, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 480, 2} +}; + +static const int hybrid48_test[][4] = { + {MODE_SILK_ONLY, OPUS_BANDWIDTH_FULLBAND, 960, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_FULLBAND, 480, 1}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_FULLBAND, 960, 2}, + {MODE_SILK_ONLY, OPUS_BANDWIDTH_FULLBAND, 480, 2} +}; + +static const int celt_test[][4] = { + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 960, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 960, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960, 1}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 480, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 480, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 480, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 480, 1}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 240, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 240, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 240, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 240, 1}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 120, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 120, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 120, 1}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 120, 1}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 960, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 960, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 960, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 960, 2}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 480, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 480, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 480, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 480, 2}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 240, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 240, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 240, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 240, 2}, + + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 120, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_SUPERWIDEBAND, 120, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_WIDEBAND, 120, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_NARROWBAND, 120, 2}, + +}; + +static const int celt_hq_test[][4] = { + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 960, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 480, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 240, 2}, + {MODE_CELT_ONLY, OPUS_BANDWIDTH_FULLBAND, 120, 2}, +}; + +#if 0 /* This is a hack that replaces the normal encoder/decoder with the multistream version */ +#define OpusEncoder OpusMSEncoder +#define OpusDecoder OpusMSDecoder +#define opus_encode opus_multistream_encode +#define opus_decode opus_multistream_decode +#define opus_encoder_ctl opus_multistream_encoder_ctl +#define opus_decoder_ctl opus_multistream_decoder_ctl +#define opus_encoder_create ms_opus_encoder_create +#define opus_decoder_create ms_opus_decoder_create +#define opus_encoder_destroy opus_multistream_encoder_destroy +#define opus_decoder_destroy opus_multistream_decoder_destroy + +static OpusEncoder *ms_opus_encoder_create(opus_int32 Fs, int channels, int application, int *error) +{ + int streams, coupled_streams; + unsigned char mapping[256]; + return (OpusEncoder *)opus_multistream_surround_encoder_create(Fs, channels, 1, &streams, &coupled_streams, mapping, application, error); +} +static OpusDecoder *ms_opus_decoder_create(opus_int32 Fs, int channels, int *error) +{ + int streams; + int coupled_streams; + unsigned char mapping[256]={0,1}; + streams = 1; + coupled_streams = channels==2; + return (OpusDecoder *)opus_multistream_decoder_create(Fs, channels, streams, coupled_streams, mapping, error); +} +#endif + +int main(int argc, char *argv[]) +{ + int err; + char *inFile, *outFile; + FILE *fin=NULL; + FILE *fout=NULL; + OpusEncoder *enc=NULL; + OpusDecoder *dec=NULL; + int args; + int len[2]; + int frame_size, channels; + opus_int32 bitrate_bps=0; + unsigned char *data[2] = {NULL, NULL}; + unsigned char *fbytes=NULL; + opus_int32 sampling_rate; + int use_vbr; + int max_payload_bytes; + int complexity; + int use_inbandfec; + int use_dtx; + int forcechannels; + int cvbr = 0; + int packet_loss_perc; + opus_int32 count=0, count_act=0; + int k; + opus_int32 skip=0; + int stop=0; + short *in=NULL; + short *out=NULL; + int application=OPUS_APPLICATION_AUDIO; + double bits=0.0, bits_max=0.0, bits_act=0.0, bits2=0.0, nrg; + double tot_samples=0; + opus_uint64 tot_in, tot_out; + int bandwidth=OPUS_AUTO; + const char *bandwidth_string; + int lost = 0, lost_prev = 1; + int toggle = 0; + opus_uint32 enc_final_range[2]; + opus_uint32 dec_final_range; + int encode_only=0, decode_only=0; + int max_frame_size = 48000*2; + size_t num_read; + int curr_read=0; + int sweep_bps = 0; + int random_framesize=0, newsize=0, delayed_celt=0; + int sweep_max=0, sweep_min=0; + int random_fec=0; + const int (*mode_list)[4]=NULL; + int nb_modes_in_list=0; + int curr_mode=0; + int curr_mode_count=0; + int mode_switch_time = 48000; + int nb_encoded=0; + int remaining=0; + int variable_duration=OPUS_FRAMESIZE_ARG; + int delayed_decision=0; + int ret = EXIT_FAILURE; + + if (argc < 5 ) + { + print_usage( argv ); + goto failure; + } + + tot_in=tot_out=0; + fprintf(stderr, "%s\n", opus_get_version_string()); + + args = 1; + if (strcmp(argv[args], "-e")==0) + { + encode_only = 1; + args++; + } else if (strcmp(argv[args], "-d")==0) + { + decode_only = 1; + args++; + } + if (!decode_only && argc < 7 ) + { + print_usage( argv ); + goto failure; + } + + if (!decode_only) + { + if (strcmp(argv[args], "voip")==0) + application = OPUS_APPLICATION_VOIP; + else if (strcmp(argv[args], "restricted-lowdelay")==0) + application = OPUS_APPLICATION_RESTRICTED_LOWDELAY; + else if (strcmp(argv[args], "audio")!=0) { + fprintf(stderr, "unknown application: %s\n", argv[args]); + print_usage(argv); + goto failure; + } + args++; + } + sampling_rate = (opus_int32)atol(argv[args]); + args++; + + if (sampling_rate != 8000 && sampling_rate != 12000 + && sampling_rate != 16000 && sampling_rate != 24000 + && sampling_rate != 48000) + { + fprintf(stderr, "Supported sampling rates are 8000, 12000, " + "16000, 24000 and 48000.\n"); + goto failure; + } + frame_size = sampling_rate/50; + + channels = atoi(argv[args]); + args++; + + if (channels < 1 || channels > 2) + { + fprintf(stderr, "Opus_demo supports only 1 or 2 channels.\n"); + goto failure; + } + + if (!decode_only) + { + bitrate_bps = (opus_int32)atol(argv[args]); + args++; + } + + /* defaults: */ + use_vbr = 1; + max_payload_bytes = MAX_PACKET; + complexity = 10; + use_inbandfec = 0; + forcechannels = OPUS_AUTO; + use_dtx = 0; + packet_loss_perc = 0; + + while( args < argc - 2 ) { + /* process command line options */ + if( strcmp( argv[ args ], "-cbr" ) == 0 ) { + check_encoder_option(decode_only, "-cbr"); + use_vbr = 0; + args++; + } else if( strcmp( argv[ args ], "-bandwidth" ) == 0 ) { + check_encoder_option(decode_only, "-bandwidth"); + if (strcmp(argv[ args + 1 ], "NB")==0) + bandwidth = OPUS_BANDWIDTH_NARROWBAND; + else if (strcmp(argv[ args + 1 ], "MB")==0) + bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + else if (strcmp(argv[ args + 1 ], "WB")==0) + bandwidth = OPUS_BANDWIDTH_WIDEBAND; + else if (strcmp(argv[ args + 1 ], "SWB")==0) + bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + else if (strcmp(argv[ args + 1 ], "FB")==0) + bandwidth = OPUS_BANDWIDTH_FULLBAND; + else { + fprintf(stderr, "Unknown bandwidth %s. " + "Supported are NB, MB, WB, SWB, FB.\n", + argv[ args + 1 ]); + goto failure; + } + args += 2; + } else if( strcmp( argv[ args ], "-framesize" ) == 0 ) { + check_encoder_option(decode_only, "-framesize"); + if (strcmp(argv[ args + 1 ], "2.5")==0) + frame_size = sampling_rate/400; + else if (strcmp(argv[ args + 1 ], "5")==0) + frame_size = sampling_rate/200; + else if (strcmp(argv[ args + 1 ], "10")==0) + frame_size = sampling_rate/100; + else if (strcmp(argv[ args + 1 ], "20")==0) + frame_size = sampling_rate/50; + else if (strcmp(argv[ args + 1 ], "40")==0) + frame_size = sampling_rate/25; + else if (strcmp(argv[ args + 1 ], "60")==0) + frame_size = 3*sampling_rate/50; + else if (strcmp(argv[ args + 1 ], "80")==0) + frame_size = 4*sampling_rate/50; + else if (strcmp(argv[ args + 1 ], "100")==0) + frame_size = 5*sampling_rate/50; + else if (strcmp(argv[ args + 1 ], "120")==0) + frame_size = 6*sampling_rate/50; + else { + fprintf(stderr, "Unsupported frame size: %s ms. " + "Supported are 2.5, 5, 10, 20, 40, 60, 80, 100, 120.\n", + argv[ args + 1 ]); + goto failure; + } + args += 2; + } else if( strcmp( argv[ args ], "-max_payload" ) == 0 ) { + check_encoder_option(decode_only, "-max_payload"); + max_payload_bytes = atoi( argv[ args + 1 ] ); + args += 2; + } else if( strcmp( argv[ args ], "-complexity" ) == 0 ) { + check_encoder_option(decode_only, "-complexity"); + complexity = atoi( argv[ args + 1 ] ); + args += 2; + } else if( strcmp( argv[ args ], "-inbandfec" ) == 0 ) { + use_inbandfec = 1; + args++; + } else if( strcmp( argv[ args ], "-forcemono" ) == 0 ) { + check_encoder_option(decode_only, "-forcemono"); + forcechannels = 1; + args++; + } else if( strcmp( argv[ args ], "-cvbr" ) == 0 ) { + check_encoder_option(decode_only, "-cvbr"); + cvbr = 1; + args++; + } else if( strcmp( argv[ args ], "-delayed-decision" ) == 0 ) { + check_encoder_option(decode_only, "-delayed-decision"); + delayed_decision = 1; + args++; + } else if( strcmp( argv[ args ], "-dtx") == 0 ) { + check_encoder_option(decode_only, "-dtx"); + use_dtx = 1; + args++; + } else if( strcmp( argv[ args ], "-loss" ) == 0 ) { + packet_loss_perc = atoi( argv[ args + 1 ] ); + args += 2; + } else if( strcmp( argv[ args ], "-sweep" ) == 0 ) { + check_encoder_option(decode_only, "-sweep"); + sweep_bps = atoi( argv[ args + 1 ] ); + args += 2; + } else if( strcmp( argv[ args ], "-random_framesize" ) == 0 ) { + check_encoder_option(decode_only, "-random_framesize"); + random_framesize = 1; + args++; + } else if( strcmp( argv[ args ], "-sweep_max" ) == 0 ) { + check_encoder_option(decode_only, "-sweep_max"); + sweep_max = atoi( argv[ args + 1 ] ); + args += 2; + } else if( strcmp( argv[ args ], "-random_fec" ) == 0 ) { + check_encoder_option(decode_only, "-random_fec"); + random_fec = 1; + args++; + } else if( strcmp( argv[ args ], "-silk8k_test" ) == 0 ) { + check_encoder_option(decode_only, "-silk8k_test"); + mode_list = silk8_test; + nb_modes_in_list = 8; + args++; + } else if( strcmp( argv[ args ], "-silk12k_test" ) == 0 ) { + check_encoder_option(decode_only, "-silk12k_test"); + mode_list = silk12_test; + nb_modes_in_list = 8; + args++; + } else if( strcmp( argv[ args ], "-silk16k_test" ) == 0 ) { + check_encoder_option(decode_only, "-silk16k_test"); + mode_list = silk16_test; + nb_modes_in_list = 8; + args++; + } else if( strcmp( argv[ args ], "-hybrid24k_test" ) == 0 ) { + check_encoder_option(decode_only, "-hybrid24k_test"); + mode_list = hybrid24_test; + nb_modes_in_list = 4; + args++; + } else if( strcmp( argv[ args ], "-hybrid48k_test" ) == 0 ) { + check_encoder_option(decode_only, "-hybrid48k_test"); + mode_list = hybrid48_test; + nb_modes_in_list = 4; + args++; + } else if( strcmp( argv[ args ], "-celt_test" ) == 0 ) { + check_encoder_option(decode_only, "-celt_test"); + mode_list = celt_test; + nb_modes_in_list = 32; + args++; + } else if( strcmp( argv[ args ], "-celt_hq_test" ) == 0 ) { + check_encoder_option(decode_only, "-celt_hq_test"); + mode_list = celt_hq_test; + nb_modes_in_list = 4; + args++; + } else { + printf( "Error: unrecognized setting: %s\n\n", argv[ args ] ); + print_usage( argv ); + goto failure; + } + } + + if (sweep_max) + sweep_min = bitrate_bps; + + if (max_payload_bytes < 0 || max_payload_bytes > MAX_PACKET) + { + fprintf (stderr, "max_payload_bytes must be between 0 and %d\n", + MAX_PACKET); + goto failure; + } + + inFile = argv[argc-2]; + fin = fopen(inFile, "rb"); + if (!fin) + { + fprintf (stderr, "Could not open input file %s\n", argv[argc-2]); + goto failure; + } + if (mode_list) + { + int size; + fseek(fin, 0, SEEK_END); + size = ftell(fin); + fprintf(stderr, "File size is %d bytes\n", size); + fseek(fin, 0, SEEK_SET); + mode_switch_time = size/sizeof(short)/channels/nb_modes_in_list; + fprintf(stderr, "Switching mode every %d samples\n", mode_switch_time); + } + + outFile = argv[argc-1]; + fout = fopen(outFile, "wb+"); + if (!fout) + { + fprintf (stderr, "Could not open output file %s\n", argv[argc-1]); + goto failure; + } + + if (!decode_only) + { + enc = opus_encoder_create(sampling_rate, channels, application, &err); + if (err != OPUS_OK) + { + fprintf(stderr, "Cannot create encoder: %s\n", opus_strerror(err)); + goto failure; + } + opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate_bps)); + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(bandwidth)); + opus_encoder_ctl(enc, OPUS_SET_VBR(use_vbr)); + opus_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(cvbr)); + opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(complexity)); + opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(use_inbandfec)); + opus_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(forcechannels)); + opus_encoder_ctl(enc, OPUS_SET_DTX(use_dtx)); + opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(packet_loss_perc)); + + opus_encoder_ctl(enc, OPUS_GET_LOOKAHEAD(&skip)); + opus_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(16)); + opus_encoder_ctl(enc, OPUS_SET_EXPERT_FRAME_DURATION(variable_duration)); + } + if (!encode_only) + { + dec = opus_decoder_create(sampling_rate, channels, &err); + if (err != OPUS_OK) + { + fprintf(stderr, "Cannot create decoder: %s\n", opus_strerror(err)); + goto failure; + } + } + + + switch(bandwidth) + { + case OPUS_BANDWIDTH_NARROWBAND: + bandwidth_string = "narrowband"; + break; + case OPUS_BANDWIDTH_MEDIUMBAND: + bandwidth_string = "mediumband"; + break; + case OPUS_BANDWIDTH_WIDEBAND: + bandwidth_string = "wideband"; + break; + case OPUS_BANDWIDTH_SUPERWIDEBAND: + bandwidth_string = "superwideband"; + break; + case OPUS_BANDWIDTH_FULLBAND: + bandwidth_string = "fullband"; + break; + case OPUS_AUTO: + bandwidth_string = "auto bandwidth"; + break; + default: + bandwidth_string = "unknown"; + break; + } + + if (decode_only) + fprintf(stderr, "Decoding with %ld Hz output (%d channels)\n", + (long)sampling_rate, channels); + else + fprintf(stderr, "Encoding %ld Hz input at %.3f kb/s " + "in %s with %d-sample frames.\n", + (long)sampling_rate, bitrate_bps*0.001, + bandwidth_string, frame_size); + + in = (short*)malloc(max_frame_size*channels*sizeof(short)); + out = (short*)malloc(max_frame_size*channels*sizeof(short)); + /* We need to allocate for 16-bit PCM data, but we store it as unsigned char. */ + fbytes = (unsigned char*)malloc(max_frame_size*channels*sizeof(short)); + data[0] = (unsigned char*)calloc(max_payload_bytes,sizeof(unsigned char)); + if ( use_inbandfec ) { + data[1] = (unsigned char*)calloc(max_payload_bytes,sizeof(unsigned char)); + } + if(delayed_decision) + { + if (frame_size==sampling_rate/400) + variable_duration = OPUS_FRAMESIZE_2_5_MS; + else if (frame_size==sampling_rate/200) + variable_duration = OPUS_FRAMESIZE_5_MS; + else if (frame_size==sampling_rate/100) + variable_duration = OPUS_FRAMESIZE_10_MS; + else if (frame_size==sampling_rate/50) + variable_duration = OPUS_FRAMESIZE_20_MS; + else if (frame_size==sampling_rate/25) + variable_duration = OPUS_FRAMESIZE_40_MS; + else if (frame_size==3*sampling_rate/50) + variable_duration = OPUS_FRAMESIZE_60_MS; + else if (frame_size==4*sampling_rate/50) + variable_duration = OPUS_FRAMESIZE_80_MS; + else if (frame_size==5*sampling_rate/50) + variable_duration = OPUS_FRAMESIZE_100_MS; + else + variable_duration = OPUS_FRAMESIZE_120_MS; + opus_encoder_ctl(enc, OPUS_SET_EXPERT_FRAME_DURATION(variable_duration)); + frame_size = 2*48000; + } + while (!stop) + { + if (delayed_celt) + { + frame_size = newsize; + delayed_celt = 0; + } else if (random_framesize && rand()%20==0) + { + newsize = rand()%6; + switch(newsize) + { + case 0: newsize=sampling_rate/400; break; + case 1: newsize=sampling_rate/200; break; + case 2: newsize=sampling_rate/100; break; + case 3: newsize=sampling_rate/50; break; + case 4: newsize=sampling_rate/25; break; + case 5: newsize=3*sampling_rate/50; break; + } + while (newsize < sampling_rate/25 && bitrate_bps-abs(sweep_bps) <= 3*12*sampling_rate/newsize) + newsize*=2; + if (newsize < sampling_rate/100 && frame_size >= sampling_rate/100) + { + opus_encoder_ctl(enc, OPUS_SET_FORCE_MODE(MODE_CELT_ONLY)); + delayed_celt=1; + } else { + frame_size = newsize; + } + } + if (random_fec && rand()%30==0) + { + opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(rand()%4==0)); + } + if (decode_only) + { + unsigned char ch[4]; + num_read = fread(ch, 1, 4, fin); + if (num_read!=4) + break; + len[toggle] = char_to_int(ch); + if (len[toggle]>max_payload_bytes || len[toggle]<0) + { + fprintf(stderr, "Invalid payload length: %d\n",len[toggle]); + break; + } + num_read = fread(ch, 1, 4, fin); + if (num_read!=4) + break; + enc_final_range[toggle] = char_to_int(ch); + num_read = fread(data[toggle], 1, len[toggle], fin); + if (num_read!=(size_t)len[toggle]) + { + fprintf(stderr, "Ran out of input, " + "expecting %d bytes got %d\n", + len[toggle],(int)num_read); + break; + } + } else { + int i; + if (mode_list!=NULL) + { + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(mode_list[curr_mode][1])); + opus_encoder_ctl(enc, OPUS_SET_FORCE_MODE(mode_list[curr_mode][0])); + opus_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(mode_list[curr_mode][3])); + frame_size = mode_list[curr_mode][2]; + } + num_read = fread(fbytes, sizeof(short)*channels, frame_size-remaining, fin); + curr_read = (int)num_read; + tot_in += curr_read; + for(i=0;i sweep_max) + sweep_bps = -sweep_bps; + else if (bitrate_bps < sweep_min) + sweep_bps = -sweep_bps; + } + /* safety */ + if (bitrate_bps<1000) + bitrate_bps = 1000; + opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate_bps)); + } + opus_encoder_ctl(enc, OPUS_GET_FINAL_RANGE(&enc_final_range[toggle])); + if (len[toggle] < 0) + { + fprintf (stderr, "opus_encode() returned %d\n", len[toggle]); + goto failure; + } + curr_mode_count += frame_size; + if (curr_mode_count > mode_switch_time && curr_mode < nb_modes_in_list-1) + { + curr_mode++; + curr_mode_count = 0; + } + } + +#if 0 /* This is for testing the padding code, do not enable by default */ + if (len[toggle]<1275) + { + int new_len = len[toggle]+rand()%(max_payload_bytes-len[toggle]); + if ((err = opus_packet_pad(data[toggle], len[toggle], new_len)) != OPUS_OK) + { + fprintf(stderr, "padding failed: %s\n", opus_strerror(err)); + goto failure; + } + len[toggle] = new_len; + } +#endif + if (encode_only) + { + unsigned char int_field[4]; + int_to_char(len[toggle], int_field); + if (fwrite(int_field, 1, 4, fout) != 4) { + fprintf(stderr, "Error writing.\n"); + goto failure; + } + int_to_char(enc_final_range[toggle], int_field); + if (fwrite(int_field, 1, 4, fout) != 4) { + fprintf(stderr, "Error writing.\n"); + goto failure; + } + if (fwrite(data[toggle], 1, len[toggle], fout) != (unsigned)len[toggle]) { + fprintf(stderr, "Error writing.\n"); + goto failure; + } + tot_samples += nb_encoded; + } else { + opus_int32 output_samples; + lost = len[toggle]==0 || (packet_loss_perc>0 && rand()%100 < packet_loss_perc); + if (lost) + opus_decoder_ctl(dec, OPUS_GET_LAST_PACKET_DURATION(&output_samples)); + else + output_samples = max_frame_size; + if( count >= use_inbandfec ) { + /* delay by one packet when using in-band FEC */ + if( use_inbandfec ) { + if( lost_prev ) { + /* attempt to decode with in-band FEC from next packet */ + opus_decoder_ctl(dec, OPUS_GET_LAST_PACKET_DURATION(&output_samples)); + output_samples = opus_decode(dec, lost ? NULL : data[toggle], len[toggle], out, output_samples, 1); + } else { + /* regular decode */ + output_samples = max_frame_size; + output_samples = opus_decode(dec, data[1-toggle], len[1-toggle], out, output_samples, 0); + } + } else { + output_samples = opus_decode(dec, lost ? NULL : data[toggle], len[toggle], out, output_samples, 0); + } + if (output_samples>0) + { + if (!decode_only && tot_out + output_samples > tot_in) + { + stop=1; + output_samples = (opus_int32)(tot_in - tot_out); + } + if (output_samples>skip) { + int i; + for(i=0;i<(output_samples-skip)*channels;i++) + { + short s; + s=out[i+(skip*channels)]; + fbytes[2*i]=s&0xFF; + fbytes[2*i+1]=(s>>8)&0xFF; + } + if (fwrite(fbytes, sizeof(short)*channels, output_samples-skip, fout) != (unsigned)(output_samples-skip)){ + fprintf(stderr, "Error writing.\n"); + goto failure; + } + tot_out += output_samples-skip; + } + if (output_samples= use_inbandfec ) { + /* count bits */ + bits += len[toggle]*8; + bits_max = ( len[toggle]*8 > bits_max ) ? len[toggle]*8 : bits_max; + bits2 += len[toggle]*len[toggle]*64; + if (!decode_only) + { + nrg = 0.0; + for ( k = 0; k < frame_size * channels; k++ ) { + nrg += in[ k ] * (double)in[ k ]; + } + nrg /= frame_size * channels; + if( nrg > 1e5 ) { + bits_act += len[toggle]*8; + count_act++; + } + } + } + count++; + toggle = (toggle + use_inbandfec) & 1; + } + + if(decode_only && count > 0) + frame_size = (int)(tot_samples / count); + count -= use_inbandfec; + if (tot_samples >= 1 && count > 0 && frame_size) + { + /* Print out bitrate statistics */ + double var; + fprintf (stderr, "average bitrate: %7.3f kb/s\n", + 1e-3*bits*sampling_rate/tot_samples); + fprintf (stderr, "maximum bitrate: %7.3f kb/s\n", + 1e-3*bits_max*sampling_rate/frame_size); + if (!decode_only) + fprintf (stderr, "active bitrate: %7.3f kb/s\n", + 1e-3*bits_act*sampling_rate/(1e-15+frame_size*(double)count_act)); + var = bits2/count - bits*bits/(count*(double)count); + if (var < 0) + var = 0; + fprintf (stderr, "bitrate standard deviation: %7.3f kb/s\n", + 1e-3*sqrt(var)*sampling_rate/frame_size); + } else { + fprintf(stderr, "bitrate statistics are undefined\n"); + } + silk_TimerSave("opus_timing.txt"); + ret = EXIT_SUCCESS; +failure: + opus_encoder_destroy(enc); + opus_decoder_destroy(dec); + free(data[0]); + free(data[1]); + if (fin) + fclose(fin); + if (fout) + fclose(fout); + free(in); + free(out); + free(fbytes); + return ret; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_encoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_encoder.c new file mode 100644 index 000000000..321bb2bb1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_encoder.c @@ -0,0 +1,2773 @@ +/* Copyright (c) 2010-2011 Xiph.Org Foundation, Skype Limited + Written by Jean-Marc Valin and Koen Vos */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "celt.h" +#include "entenc.h" +#include "modes.h" +#include "API.h" +#include "stack_alloc.h" +#include "float_cast.h" +#include "opus.h" +#include "arch.h" +#include "pitch.h" +#include "opus_private.h" +#include "os_support.h" +#include "cpu_support.h" +#include "analysis.h" +#include "mathops.h" +#include "tuning_parameters.h" +#ifdef FIXED_POINT +#include "fixed/structs_FIX.h" +#else +#include "float/structs_FLP.h" +#endif + +#define MAX_ENCODER_BUFFER 480 + +#ifndef DISABLE_FLOAT_API +#define PSEUDO_SNR_THRESHOLD 316.23f /* 10^(25/10) */ +#endif + +typedef struct { + opus_val32 XX, XY, YY; + opus_val16 smoothed_width; + opus_val16 max_follower; +} StereoWidthState; + +struct OpusEncoder { + int celt_enc_offset; + int silk_enc_offset; + silk_EncControlStruct silk_mode; + int application; + int channels; + int delay_compensation; + int force_channels; + int signal_type; + int user_bandwidth; + int max_bandwidth; + int user_forced_mode; + int voice_ratio; + opus_int32 Fs; + int use_vbr; + int vbr_constraint; + int variable_duration; + opus_int32 bitrate_bps; + opus_int32 user_bitrate_bps; + int lsb_depth; + int encoder_buffer; + int lfe; + int arch; + int use_dtx; /* general DTX for both SILK and CELT */ +#ifndef DISABLE_FLOAT_API + TonalityAnalysisState analysis; +#endif + +#define OPUS_ENCODER_RESET_START stream_channels + int stream_channels; + opus_int16 hybrid_stereo_width_Q14; + opus_int32 variable_HP_smth2_Q15; + opus_val16 prev_HB_gain; + opus_val32 hp_mem[4]; + int mode; + int prev_mode; + int prev_channels; + int prev_framesize; + int bandwidth; + /* Bandwidth determined automatically from the rate (before any other adjustment) */ + int auto_bandwidth; + int silk_bw_switch; + /* Sampling rate (at the API level) */ + int first; + opus_val16 * energy_masking; + StereoWidthState width_mem; + opus_val16 delay_buffer[MAX_ENCODER_BUFFER*2]; +#ifndef DISABLE_FLOAT_API + int detected_bandwidth; + int nb_no_activity_ms_Q1; + opus_val32 peak_signal_energy; +#endif + int nonfinal_frame; /* current frame is not the final in a packet */ + opus_uint32 rangeFinal; +}; + +/* Transition tables for the voice and music. First column is the + middle (memoriless) threshold. The second column is the hysteresis + (difference with the middle) */ +static const opus_int32 mono_voice_bandwidth_thresholds[8] = { + 9000, 700, /* NB<->MB */ + 9000, 700, /* MB<->WB */ + 13500, 1000, /* WB<->SWB */ + 14000, 2000, /* SWB<->FB */ +}; +static const opus_int32 mono_music_bandwidth_thresholds[8] = { + 9000, 700, /* NB<->MB */ + 9000, 700, /* MB<->WB */ + 11000, 1000, /* WB<->SWB */ + 12000, 2000, /* SWB<->FB */ +}; +static const opus_int32 stereo_voice_bandwidth_thresholds[8] = { + 9000, 700, /* NB<->MB */ + 9000, 700, /* MB<->WB */ + 13500, 1000, /* WB<->SWB */ + 14000, 2000, /* SWB<->FB */ +}; +static const opus_int32 stereo_music_bandwidth_thresholds[8] = { + 9000, 700, /* NB<->MB */ + 9000, 700, /* MB<->WB */ + 11000, 1000, /* WB<->SWB */ + 12000, 2000, /* SWB<->FB */ +}; +/* Threshold bit-rates for switching between mono and stereo */ +static const opus_int32 stereo_voice_threshold = 19000; +static const opus_int32 stereo_music_threshold = 17000; + +/* Threshold bit-rate for switching between SILK/hybrid and CELT-only */ +static const opus_int32 mode_thresholds[2][2] = { + /* voice */ /* music */ + { 64000, 10000}, /* mono */ + { 44000, 10000}, /* stereo */ +}; + +static const opus_int32 fec_thresholds[] = { + 12000, 1000, /* NB */ + 14000, 1000, /* MB */ + 16000, 1000, /* WB */ + 20000, 1000, /* SWB */ + 22000, 1000, /* FB */ +}; + +int opus_encoder_get_size(int channels) +{ + int silkEncSizeBytes, celtEncSizeBytes; + int ret; + if (channels<1 || channels > 2) + return 0; + ret = silk_Get_Encoder_Size( &silkEncSizeBytes ); + if (ret) + return 0; + silkEncSizeBytes = align(silkEncSizeBytes); + celtEncSizeBytes = celt_encoder_get_size(channels); + return align(sizeof(OpusEncoder))+silkEncSizeBytes+celtEncSizeBytes; +} + +int opus_encoder_init(OpusEncoder* st, opus_int32 Fs, int channels, int application) +{ + void *silk_enc; + CELTEncoder *celt_enc; + int err; + int ret, silkEncSizeBytes; + + if((Fs!=48000&&Fs!=24000&&Fs!=16000&&Fs!=12000&&Fs!=8000)||(channels!=1&&channels!=2)|| + (application != OPUS_APPLICATION_VOIP && application != OPUS_APPLICATION_AUDIO + && application != OPUS_APPLICATION_RESTRICTED_LOWDELAY)) + return OPUS_BAD_ARG; + + OPUS_CLEAR((char*)st, opus_encoder_get_size(channels)); + /* Create SILK encoder */ + ret = silk_Get_Encoder_Size( &silkEncSizeBytes ); + if (ret) + return OPUS_BAD_ARG; + silkEncSizeBytes = align(silkEncSizeBytes); + st->silk_enc_offset = align(sizeof(OpusEncoder)); + st->celt_enc_offset = st->silk_enc_offset+silkEncSizeBytes; + silk_enc = (char*)st+st->silk_enc_offset; + celt_enc = (CELTEncoder*)((char*)st+st->celt_enc_offset); + + st->stream_channels = st->channels = channels; + + st->Fs = Fs; + + st->arch = opus_select_arch(); + + ret = silk_InitEncoder( silk_enc, st->arch, &st->silk_mode ); + if(ret)return OPUS_INTERNAL_ERROR; + + /* default SILK parameters */ + st->silk_mode.nChannelsAPI = channels; + st->silk_mode.nChannelsInternal = channels; + st->silk_mode.API_sampleRate = st->Fs; + st->silk_mode.maxInternalSampleRate = 16000; + st->silk_mode.minInternalSampleRate = 8000; + st->silk_mode.desiredInternalSampleRate = 16000; + st->silk_mode.payloadSize_ms = 20; + st->silk_mode.bitRate = 25000; + st->silk_mode.packetLossPercentage = 0; + st->silk_mode.complexity = 9; + st->silk_mode.useInBandFEC = 0; + st->silk_mode.useDTX = 0; + st->silk_mode.useCBR = 0; + st->silk_mode.reducedDependency = 0; + + /* Create CELT encoder */ + /* Initialize CELT encoder */ + err = celt_encoder_init(celt_enc, Fs, channels, st->arch); + if(err!=OPUS_OK)return OPUS_INTERNAL_ERROR; + + celt_encoder_ctl(celt_enc, CELT_SET_SIGNALLING(0)); + celt_encoder_ctl(celt_enc, OPUS_SET_COMPLEXITY(st->silk_mode.complexity)); + + st->use_vbr = 1; + /* Makes constrained VBR the default (safer for real-time use) */ + st->vbr_constraint = 1; + st->user_bitrate_bps = OPUS_AUTO; + st->bitrate_bps = 3000+Fs*channels; + st->application = application; + st->signal_type = OPUS_AUTO; + st->user_bandwidth = OPUS_AUTO; + st->max_bandwidth = OPUS_BANDWIDTH_FULLBAND; + st->force_channels = OPUS_AUTO; + st->user_forced_mode = OPUS_AUTO; + st->voice_ratio = -1; + st->encoder_buffer = st->Fs/100; + st->lsb_depth = 24; + st->variable_duration = OPUS_FRAMESIZE_ARG; + + /* Delay compensation of 4 ms (2.5 ms for SILK's extra look-ahead + + 1.5 ms for SILK resamplers and stereo prediction) */ + st->delay_compensation = st->Fs/250; + + st->hybrid_stereo_width_Q14 = 1 << 14; + st->prev_HB_gain = Q15ONE; + st->variable_HP_smth2_Q15 = silk_LSHIFT( silk_lin2log( VARIABLE_HP_MIN_CUTOFF_HZ ), 8 ); + st->first = 1; + st->mode = MODE_HYBRID; + st->bandwidth = OPUS_BANDWIDTH_FULLBAND; + +#ifndef DISABLE_FLOAT_API + tonality_analysis_init(&st->analysis, st->Fs); + st->analysis.application = st->application; +#endif + + return OPUS_OK; +} + +static unsigned char gen_toc(int mode, int framerate, int bandwidth, int channels) +{ + int period; + unsigned char toc; + period = 0; + while (framerate < 400) + { + framerate <<= 1; + period++; + } + if (mode == MODE_SILK_ONLY) + { + toc = (bandwidth-OPUS_BANDWIDTH_NARROWBAND)<<5; + toc |= (period-2)<<3; + } else if (mode == MODE_CELT_ONLY) + { + int tmp = bandwidth-OPUS_BANDWIDTH_MEDIUMBAND; + if (tmp < 0) + tmp = 0; + toc = 0x80; + toc |= tmp << 5; + toc |= period<<3; + } else /* Hybrid */ + { + toc = 0x60; + toc |= (bandwidth-OPUS_BANDWIDTH_SUPERWIDEBAND)<<4; + toc |= (period-2)<<3; + } + toc |= (channels==2)<<2; + return toc; +} + +#ifndef FIXED_POINT +static void silk_biquad_float( + const opus_val16 *in, /* I: Input signal */ + const opus_int32 *B_Q28, /* I: MA coefficients [3] */ + const opus_int32 *A_Q28, /* I: AR coefficients [2] */ + opus_val32 *S, /* I/O: State vector [2] */ + opus_val16 *out, /* O: Output signal */ + const opus_int32 len, /* I: Signal length (must be even) */ + int stride +) +{ + /* DIRECT FORM II TRANSPOSED (uses 2 element state vector) */ + opus_int k; + opus_val32 vout; + opus_val32 inval; + opus_val32 A[2], B[3]; + + A[0] = (opus_val32)(A_Q28[0] * (1.f/((opus_int32)1<<28))); + A[1] = (opus_val32)(A_Q28[1] * (1.f/((opus_int32)1<<28))); + B[0] = (opus_val32)(B_Q28[0] * (1.f/((opus_int32)1<<28))); + B[1] = (opus_val32)(B_Q28[1] * (1.f/((opus_int32)1<<28))); + B[2] = (opus_val32)(B_Q28[2] * (1.f/((opus_int32)1<<28))); + + /* Negate A_Q28 values and split in two parts */ + + for( k = 0; k < len; k++ ) { + /* S[ 0 ], S[ 1 ]: Q12 */ + inval = in[ k*stride ]; + vout = S[ 0 ] + B[0]*inval; + + S[ 0 ] = S[1] - vout*A[0] + B[1]*inval; + + S[ 1 ] = - vout*A[1] + B[2]*inval + VERY_SMALL; + + /* Scale back to Q0 and saturate */ + out[ k*stride ] = vout; + } +} +#endif + +static void hp_cutoff(const opus_val16 *in, opus_int32 cutoff_Hz, opus_val16 *out, opus_val32 *hp_mem, int len, int channels, opus_int32 Fs, int arch) +{ + opus_int32 B_Q28[ 3 ], A_Q28[ 2 ]; + opus_int32 Fc_Q19, r_Q28, r_Q22; + (void)arch; + + silk_assert( cutoff_Hz <= silk_int32_MAX / SILK_FIX_CONST( 1.5 * 3.14159 / 1000, 19 ) ); + Fc_Q19 = silk_DIV32_16( silk_SMULBB( SILK_FIX_CONST( 1.5 * 3.14159 / 1000, 19 ), cutoff_Hz ), Fs/1000 ); + silk_assert( Fc_Q19 > 0 && Fc_Q19 < 32768 ); + + r_Q28 = SILK_FIX_CONST( 1.0, 28 ) - silk_MUL( SILK_FIX_CONST( 0.92, 9 ), Fc_Q19 ); + + /* b = r * [ 1; -2; 1 ]; */ + /* a = [ 1; -2 * r * ( 1 - 0.5 * Fc^2 ); r^2 ]; */ + B_Q28[ 0 ] = r_Q28; + B_Q28[ 1 ] = silk_LSHIFT( -r_Q28, 1 ); + B_Q28[ 2 ] = r_Q28; + + /* -r * ( 2 - Fc * Fc ); */ + r_Q22 = silk_RSHIFT( r_Q28, 6 ); + A_Q28[ 0 ] = silk_SMULWW( r_Q22, silk_SMULWW( Fc_Q19, Fc_Q19 ) - SILK_FIX_CONST( 2.0, 22 ) ); + A_Q28[ 1 ] = silk_SMULWW( r_Q22, r_Q22 ); + +#ifdef FIXED_POINT + if( channels == 1 ) { + silk_biquad_alt_stride1( in, B_Q28, A_Q28, hp_mem, out, len ); + } else { + silk_biquad_alt_stride2( in, B_Q28, A_Q28, hp_mem, out, len, arch ); + } +#else + silk_biquad_float( in, B_Q28, A_Q28, hp_mem, out, len, channels ); + if( channels == 2 ) { + silk_biquad_float( in+1, B_Q28, A_Q28, hp_mem+2, out+1, len, channels ); + } +#endif +} + +#ifdef FIXED_POINT +static void dc_reject(const opus_val16 *in, opus_int32 cutoff_Hz, opus_val16 *out, opus_val32 *hp_mem, int len, int channels, opus_int32 Fs) +{ + int c, i; + int shift; + + /* Approximates -round(log2(6.3*cutoff_Hz/Fs)) */ + shift=celt_ilog2(Fs/(cutoff_Hz*4)); + for (c=0;cFs/400; + if (st->user_bitrate_bps==OPUS_AUTO) + return 60*st->Fs/frame_size + st->Fs*st->channels; + else if (st->user_bitrate_bps==OPUS_BITRATE_MAX) + return max_data_bytes*8*st->Fs/frame_size; + else + return st->user_bitrate_bps; +} + +#ifndef DISABLE_FLOAT_API +#ifdef FIXED_POINT +#define PCM2VAL(x) FLOAT2INT16(x) +#else +#define PCM2VAL(x) SCALEIN(x) +#endif + +void downmix_float(const void *_x, opus_val32 *y, int subframe, int offset, int c1, int c2, int C) +{ + const float *x; + int j; + + x = (const float *)_x; + for (j=0;j-1) + { + for (j=0;j-1) + { + for (j=0;j= OPUS_FRAMESIZE_2_5_MS && variable_duration <= OPUS_FRAMESIZE_120_MS) + { + if (variable_duration <= OPUS_FRAMESIZE_40_MS) + new_size = (Fs/400)<<(variable_duration-OPUS_FRAMESIZE_2_5_MS); + else + new_size = (variable_duration-OPUS_FRAMESIZE_2_5_MS-2)*Fs/50; + } + else + return -1; + if (new_size>frame_size) + return -1; + if (400*new_size!=Fs && 200*new_size!=Fs && 100*new_size!=Fs && + 50*new_size!=Fs && 25*new_size!=Fs && 50*new_size!=3*Fs && + 50*new_size!=4*Fs && 50*new_size!=5*Fs && 50*new_size!=6*Fs) + return -1; + return new_size; +} + +opus_val16 compute_stereo_width(const opus_val16 *pcm, int frame_size, opus_int32 Fs, StereoWidthState *mem) +{ + opus_val32 xx, xy, yy; + opus_val16 sqrt_xx, sqrt_yy; + opus_val16 qrrt_xx, qrrt_yy; + int frame_rate; + int i; + opus_val16 short_alpha; + + frame_rate = Fs/frame_size; + short_alpha = Q15ONE - MULT16_16(25, Q15ONE)/IMAX(50,frame_rate); + xx=xy=yy=0; + /* Unroll by 4. The frame size is always a multiple of 4 *except* for + 2.5 ms frames at 12 kHz. Since this setting is very rare (and very + stupid), we just discard the last two samples. */ + for (i=0;iXX += MULT16_32_Q15(short_alpha, xx-mem->XX); + mem->XY += MULT16_32_Q15(short_alpha, xy-mem->XY); + mem->YY += MULT16_32_Q15(short_alpha, yy-mem->YY); + mem->XX = MAX32(0, mem->XX); + mem->XY = MAX32(0, mem->XY); + mem->YY = MAX32(0, mem->YY); + if (MAX32(mem->XX, mem->YY)>QCONST16(8e-4f, 18)) + { + opus_val16 corr; + opus_val16 ldiff; + opus_val16 width; + sqrt_xx = celt_sqrt(mem->XX); + sqrt_yy = celt_sqrt(mem->YY); + qrrt_xx = celt_sqrt(sqrt_xx); + qrrt_yy = celt_sqrt(sqrt_yy); + /* Inter-channel correlation */ + mem->XY = MIN32(mem->XY, sqrt_xx*sqrt_yy); + corr = SHR32(frac_div32(mem->XY,EPSILON+MULT16_16(sqrt_xx,sqrt_yy)),16); + /* Approximate loudness difference */ + ldiff = MULT16_16(Q15ONE, ABS16(qrrt_xx-qrrt_yy))/(EPSILON+qrrt_xx+qrrt_yy); + width = MULT16_16_Q15(celt_sqrt(QCONST32(1.f,30)-MULT16_16(corr,corr)), ldiff); + /* Smoothing over one second */ + mem->smoothed_width += (width-mem->smoothed_width)/frame_rate; + /* Peak follower */ + mem->max_follower = MAX16(mem->max_follower-QCONST16(.02f,15)/frame_rate, mem->smoothed_width); + } + /*printf("%f %f %f %f %f ", corr/(float)Q15ONE, ldiff/(float)Q15ONE, width/(float)Q15ONE, mem->smoothed_width/(float)Q15ONE, mem->max_follower/(float)Q15ONE);*/ + return EXTRACT16(MIN32(Q15ONE, MULT16_16(20, mem->max_follower))); +} + +static int decide_fec(int useInBandFEC, int PacketLoss_perc, int last_fec, int mode, int *bandwidth, opus_int32 rate) +{ + int orig_bandwidth; + if (!useInBandFEC || PacketLoss_perc == 0 || mode == MODE_CELT_ONLY) + return 0; + orig_bandwidth = *bandwidth; + for (;;) + { + opus_int32 hysteresis; + opus_int32 LBRR_rate_thres_bps; + /* Compute threshold for using FEC at the current bandwidth setting */ + LBRR_rate_thres_bps = fec_thresholds[2*(*bandwidth - OPUS_BANDWIDTH_NARROWBAND)]; + hysteresis = fec_thresholds[2*(*bandwidth - OPUS_BANDWIDTH_NARROWBAND) + 1]; + if (last_fec == 1) LBRR_rate_thres_bps -= hysteresis; + if (last_fec == 0) LBRR_rate_thres_bps += hysteresis; + LBRR_rate_thres_bps = silk_SMULWB( silk_MUL( LBRR_rate_thres_bps, + 125 - silk_min( PacketLoss_perc, 25 ) ), SILK_FIX_CONST( 0.01, 16 ) ); + /* If loss <= 5%, we look at whether we have enough rate to enable FEC. + If loss > 5%, we decrease the bandwidth until we can enable FEC. */ + if (rate > LBRR_rate_thres_bps) + return 1; + else if (PacketLoss_perc <= 5) + return 0; + else if (*bandwidth > OPUS_BANDWIDTH_NARROWBAND) + (*bandwidth)--; + else + break; + } + /* Couldn't find any bandwidth to enable FEC, keep original bandwidth. */ + *bandwidth = orig_bandwidth; + return 0; +} + +static int compute_silk_rate_for_hybrid(int rate, int bandwidth, int frame20ms, int vbr, int fec, int channels) { + int entry; + int i; + int N; + int silk_rate; + static int rate_table[][5] = { + /* |total| |-------- SILK------------| + |-- No FEC -| |--- FEC ---| + 10ms 20ms 10ms 20ms */ + { 0, 0, 0, 0, 0}, + {12000, 10000, 10000, 11000, 11000}, + {16000, 13500, 13500, 15000, 15000}, + {20000, 16000, 16000, 18000, 18000}, + {24000, 18000, 18000, 21000, 21000}, + {32000, 22000, 22000, 28000, 28000}, + {64000, 38000, 38000, 50000, 50000} + }; + /* Do the allocation per-channel. */ + rate /= channels; + entry = 1 + frame20ms + 2*fec; + N = sizeof(rate_table)/sizeof(rate_table[0]); + for (i=1;i rate) break; + } + if (i == N) + { + silk_rate = rate_table[i-1][entry]; + /* For now, just give 50% of the extra bits to SILK. */ + silk_rate += (rate-rate_table[i-1][0])/2; + } else { + opus_int32 lo, hi, x0, x1; + lo = rate_table[i-1][entry]; + hi = rate_table[i][entry]; + x0 = rate_table[i-1][0]; + x1 = rate_table[i][0]; + silk_rate = (lo*(x1-rate) + hi*(rate-x0))/(x1-x0); + } + if (!vbr) + { + /* Tiny boost to SILK for CBR. We should probably tune this better. */ + silk_rate += 100; + } + if (bandwidth==OPUS_BANDWIDTH_SUPERWIDEBAND) + silk_rate += 300; + silk_rate *= channels; + /* Small adjustment for stereo (calibrated for 32 kb/s, haven't tried other bitrates). */ + if (channels == 2 && rate >= 12000) + silk_rate -= 1000; + return silk_rate; +} + +/* Returns the equivalent bitrate corresponding to 20 ms frames, + complexity 10 VBR operation. */ +static opus_int32 compute_equiv_rate(opus_int32 bitrate, int channels, + int frame_rate, int vbr, int mode, int complexity, int loss) +{ + opus_int32 equiv; + equiv = bitrate; + /* Take into account overhead from smaller frames. */ + if (frame_rate > 50) + equiv -= (40*channels+20)*(frame_rate - 50); + /* CBR is about a 8% penalty for both SILK and CELT. */ + if (!vbr) + equiv -= equiv/12; + /* Complexity makes about 10% difference (from 0 to 10) in general. */ + equiv = equiv * (90+complexity)/100; + if (mode == MODE_SILK_ONLY || mode == MODE_HYBRID) + { + /* SILK complexity 0-1 uses the non-delayed-decision NSQ, which + costs about 20%. */ + if (complexity<2) + equiv = equiv*4/5; + equiv -= equiv*loss/(6*loss + 10); + } else if (mode == MODE_CELT_ONLY) { + /* CELT complexity 0-4 doesn't have the pitch filter, which costs + about 10%. */ + if (complexity<5) + equiv = equiv*9/10; + } else { + /* Mode not known yet */ + /* Half the SILK loss*/ + equiv -= equiv*loss/(12*loss + 20); + } + return equiv; +} + +#ifndef DISABLE_FLOAT_API + +int is_digital_silence(const opus_val16* pcm, int frame_size, int channels, int lsb_depth) +{ + int silence = 0; + opus_val32 sample_max = 0; +#ifdef MLP_TRAINING + return 0; +#endif + sample_max = celt_maxabs16(pcm, frame_size*channels); + +#ifdef FIXED_POINT + silence = (sample_max == 0); + (void)lsb_depth; +#else + silence = (sample_max <= (opus_val16) 1 / (1 << lsb_depth)); +#endif + + return silence; +} + +#ifdef FIXED_POINT +static opus_val32 compute_frame_energy(const opus_val16 *pcm, int frame_size, int channels, int arch) +{ + int i; + opus_val32 sample_max; + int max_shift; + int shift; + opus_val32 energy = 0; + int len = frame_size*channels; + (void)arch; + /* Max amplitude in the signal */ + sample_max = celt_maxabs16(pcm, len); + + /* Compute the right shift required in the MAC to avoid an overflow */ + max_shift = celt_ilog2(len); + shift = IMAX(0, (celt_ilog2(sample_max) << 1) + max_shift - 28); + + /* Compute the energy */ + for (i=0; i NB_SPEECH_FRAMES_BEFORE_DTX*20*2) + { + if (*nb_no_activity_ms_Q1 <= (NB_SPEECH_FRAMES_BEFORE_DTX + MAX_CONSECUTIVE_DTX)*20*2) + /* Valid frame for DTX! */ + return 1; + else + (*nb_no_activity_ms_Q1) = NB_SPEECH_FRAMES_BEFORE_DTX*20*2; + } + } else + (*nb_no_activity_ms_Q1) = 0; + + return 0; +} + +#endif + +static opus_int32 encode_multiframe_packet(OpusEncoder *st, + const opus_val16 *pcm, + int nb_frames, + int frame_size, + unsigned char *data, + opus_int32 out_data_bytes, + int to_celt, + int lsb_depth, + int float_api) +{ + int i; + int ret = 0; + VARDECL(unsigned char, tmp_data); + int bak_mode, bak_bandwidth, bak_channels, bak_to_mono; + VARDECL(OpusRepacketizer, rp); + int max_header_bytes; + opus_int32 bytes_per_frame; + opus_int32 cbr_bytes; + opus_int32 repacketize_len; + int tmp_len; + ALLOC_STACK; + + /* Worst cases: + * 2 frames: Code 2 with different compressed sizes + * >2 frames: Code 3 VBR */ + max_header_bytes = nb_frames == 2 ? 3 : (2+(nb_frames-1)*2); + + if (st->use_vbr || st->user_bitrate_bps==OPUS_BITRATE_MAX) + repacketize_len = out_data_bytes; + else { + cbr_bytes = 3*st->bitrate_bps/(3*8*st->Fs/(frame_size*nb_frames)); + repacketize_len = IMIN(cbr_bytes, out_data_bytes); + } + bytes_per_frame = IMIN(1276, 1+(repacketize_len-max_header_bytes)/nb_frames); + + ALLOC(tmp_data, nb_frames*bytes_per_frame, unsigned char); + ALLOC(rp, 1, OpusRepacketizer); + opus_repacketizer_init(rp); + + bak_mode = st->user_forced_mode; + bak_bandwidth = st->user_bandwidth; + bak_channels = st->force_channels; + + st->user_forced_mode = st->mode; + st->user_bandwidth = st->bandwidth; + st->force_channels = st->stream_channels; + + bak_to_mono = st->silk_mode.toMono; + if (bak_to_mono) + st->force_channels = 1; + else + st->prev_channels = st->stream_channels; + + for (i=0;isilk_mode.toMono = 0; + st->nonfinal_frame = i<(nb_frames-1); + + /* When switching from SILK/Hybrid to CELT, only ask for a switch at the last frame */ + if (to_celt && i==nb_frames-1) + st->user_forced_mode = MODE_CELT_ONLY; + + tmp_len = opus_encode_native(st, pcm+i*(st->channels*frame_size), frame_size, + tmp_data+i*bytes_per_frame, bytes_per_frame, lsb_depth, NULL, 0, 0, 0, 0, + NULL, float_api); + + if (tmp_len<0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + + ret = opus_repacketizer_cat(rp, tmp_data+i*bytes_per_frame, tmp_len); + + if (ret<0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + } + + ret = opus_repacketizer_out_range_impl(rp, 0, nb_frames, data, repacketize_len, 0, !st->use_vbr); + + if (ret<0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + + /* Discard configs that were forced locally for the purpose of repacketization */ + st->user_forced_mode = bak_mode; + st->user_bandwidth = bak_bandwidth; + st->force_channels = bak_channels; + st->silk_mode.toMono = bak_to_mono; + + RESTORE_STACK; + return ret; +} + +static int compute_redundancy_bytes(opus_int32 max_data_bytes, opus_int32 bitrate_bps, int frame_rate, int channels) +{ + int redundancy_bytes_cap; + int redundancy_bytes; + opus_int32 redundancy_rate; + int base_bits; + opus_int32 available_bits; + base_bits = (40*channels+20); + + /* Equivalent rate for 5 ms frames. */ + redundancy_rate = bitrate_bps + base_bits*(200 - frame_rate); + /* For VBR, further increase the bitrate if we can afford it. It's pretty short + and we'll avoid artefacts. */ + redundancy_rate = 3*redundancy_rate/2; + redundancy_bytes = redundancy_rate/1600; + + /* Compute the max rate we can use given CBR or VBR with cap. */ + available_bits = max_data_bytes*8 - 2*base_bits; + redundancy_bytes_cap = (available_bits*240/(240+48000/frame_rate) + base_bits)/8; + redundancy_bytes = IMIN(redundancy_bytes, redundancy_bytes_cap); + /* It we can't get enough bits for redundancy to be worth it, rely on the decoder PLC. */ + if (redundancy_bytes > 4 + 8*channels) + redundancy_bytes = IMIN(257, redundancy_bytes); + else + redundancy_bytes = 0; + return redundancy_bytes; +} + +opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_size, + unsigned char *data, opus_int32 out_data_bytes, int lsb_depth, + const void *analysis_pcm, opus_int32 analysis_size, int c1, int c2, + int analysis_channels, downmix_func downmix, int float_api) +{ + void *silk_enc; + CELTEncoder *celt_enc; + int i; + int ret=0; + opus_int32 nBytes; + ec_enc enc; + int bytes_target; + int prefill=0; + int start_band = 0; + int redundancy = 0; + int redundancy_bytes = 0; /* Number of bytes to use for redundancy frame */ + int celt_to_silk = 0; + VARDECL(opus_val16, pcm_buf); + int nb_compr_bytes; + int to_celt = 0; + opus_uint32 redundant_rng = 0; + int cutoff_Hz, hp_freq_smth1; + int voice_est; /* Probability of voice in Q7 */ + opus_int32 equiv_rate; + int delay_compensation; + int frame_rate; + opus_int32 max_rate; /* Max bitrate we're allowed to use */ + int curr_bandwidth; + opus_val16 HB_gain; + opus_int32 max_data_bytes; /* Max number of bytes we're allowed to use */ + int total_buffer; + opus_val16 stereo_width; + const CELTMode *celt_mode; +#ifndef DISABLE_FLOAT_API + AnalysisInfo analysis_info; + int analysis_read_pos_bak=-1; + int analysis_read_subframe_bak=-1; + int is_silence = 0; +#endif + opus_int activity = VAD_NO_DECISION; + + VARDECL(opus_val16, tmp_prefill); + + ALLOC_STACK; + + max_data_bytes = IMIN(1276, out_data_bytes); + + st->rangeFinal = 0; + if (frame_size <= 0 || max_data_bytes <= 0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + /* Cannot encode 100 ms in 1 byte */ + if (max_data_bytes==1 && st->Fs==(frame_size*10)) + { + RESTORE_STACK; + return OPUS_BUFFER_TOO_SMALL; + } + + silk_enc = (char*)st+st->silk_enc_offset; + celt_enc = (CELTEncoder*)((char*)st+st->celt_enc_offset); + if (st->application == OPUS_APPLICATION_RESTRICTED_LOWDELAY) + delay_compensation = 0; + else + delay_compensation = st->delay_compensation; + + lsb_depth = IMIN(lsb_depth, st->lsb_depth); + + celt_encoder_ctl(celt_enc, CELT_GET_MODE(&celt_mode)); +#ifndef DISABLE_FLOAT_API + analysis_info.valid = 0; +#ifdef FIXED_POINT + if (st->silk_mode.complexity >= 10 && st->Fs>=16000) +#else + if (st->silk_mode.complexity >= 7 && st->Fs>=16000) +#endif + { + is_silence = is_digital_silence(pcm, frame_size, st->channels, lsb_depth); + analysis_read_pos_bak = st->analysis.read_pos; + analysis_read_subframe_bak = st->analysis.read_subframe; + run_analysis(&st->analysis, celt_mode, analysis_pcm, analysis_size, frame_size, + c1, c2, analysis_channels, st->Fs, + lsb_depth, downmix, &analysis_info); + + /* Track the peak signal energy */ + if (!is_silence && analysis_info.activity_probability > DTX_ACTIVITY_THRESHOLD) + st->peak_signal_energy = MAX32(MULT16_32_Q15(QCONST16(0.999f, 15), st->peak_signal_energy), + compute_frame_energy(pcm, frame_size, st->channels, st->arch)); + } else if (st->analysis.initialized) { + tonality_analysis_reset(&st->analysis); + } +#else + (void)analysis_pcm; + (void)analysis_size; + (void)c1; + (void)c2; + (void)analysis_channels; + (void)downmix; +#endif + +#ifndef DISABLE_FLOAT_API + /* Reset voice_ratio if this frame is not silent or if analysis is disabled. + * Otherwise, preserve voice_ratio from the last non-silent frame */ + if (!is_silence) + st->voice_ratio = -1; + + if (is_silence) + { + activity = !is_silence; + } else if (analysis_info.valid) + { + activity = analysis_info.activity_probability >= DTX_ACTIVITY_THRESHOLD; + if (!activity) + { + /* Mark as active if this noise frame is sufficiently loud */ + opus_val32 noise_energy = compute_frame_energy(pcm, frame_size, st->channels, st->arch); + activity = st->peak_signal_energy < (PSEUDO_SNR_THRESHOLD * noise_energy); + } + } + + st->detected_bandwidth = 0; + if (analysis_info.valid) + { + int analysis_bandwidth; + if (st->signal_type == OPUS_AUTO) + { + float prob; + if (st->prev_mode == 0) + prob = analysis_info.music_prob; + else if (st->prev_mode == MODE_CELT_ONLY) + prob = analysis_info.music_prob_max; + else + prob = analysis_info.music_prob_min; + st->voice_ratio = (int)floor(.5+100*(1-prob)); + } + + analysis_bandwidth = analysis_info.bandwidth; + if (analysis_bandwidth<=12) + st->detected_bandwidth = OPUS_BANDWIDTH_NARROWBAND; + else if (analysis_bandwidth<=14) + st->detected_bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + else if (analysis_bandwidth<=16) + st->detected_bandwidth = OPUS_BANDWIDTH_WIDEBAND; + else if (analysis_bandwidth<=18) + st->detected_bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + else + st->detected_bandwidth = OPUS_BANDWIDTH_FULLBAND; + } +#else + st->voice_ratio = -1; +#endif + + if (st->channels==2 && st->force_channels!=1) + stereo_width = compute_stereo_width(pcm, frame_size, st->Fs, &st->width_mem); + else + stereo_width = 0; + total_buffer = delay_compensation; + st->bitrate_bps = user_bitrate_to_bitrate(st, frame_size, max_data_bytes); + + frame_rate = st->Fs/frame_size; + if (!st->use_vbr) + { + int cbrBytes; + /* Multiply by 12 to make sure the division is exact. */ + int frame_rate12 = 12*st->Fs/frame_size; + /* We need to make sure that "int" values always fit in 16 bits. */ + cbrBytes = IMIN( (12*st->bitrate_bps/8 + frame_rate12/2)/frame_rate12, max_data_bytes); + st->bitrate_bps = cbrBytes*(opus_int32)frame_rate12*8/12; + /* Make sure we provide at least one byte to avoid failing. */ + max_data_bytes = IMAX(1, cbrBytes); + } + if (max_data_bytes<3 || st->bitrate_bps < 3*frame_rate*8 + || (frame_rate<50 && (max_data_bytes*frame_rate<300 || st->bitrate_bps < 2400))) + { + /*If the space is too low to do something useful, emit 'PLC' frames.*/ + int tocmode = st->mode; + int bw = st->bandwidth == 0 ? OPUS_BANDWIDTH_NARROWBAND : st->bandwidth; + int packet_code = 0; + int num_multiframes = 0; + + if (tocmode==0) + tocmode = MODE_SILK_ONLY; + if (frame_rate>100) + tocmode = MODE_CELT_ONLY; + /* 40 ms -> 2 x 20 ms if in CELT_ONLY or HYBRID mode */ + if (frame_rate==25 && tocmode!=MODE_SILK_ONLY) + { + frame_rate = 50; + packet_code = 1; + } + + /* >= 60 ms frames */ + if (frame_rate<=16) + { + /* 1 x 60 ms, 2 x 40 ms, 2 x 60 ms */ + if (out_data_bytes==1 || (tocmode==MODE_SILK_ONLY && frame_rate!=10)) + { + tocmode = MODE_SILK_ONLY; + + packet_code = frame_rate <= 12; + frame_rate = frame_rate == 12 ? 25 : 16; + } + else + { + num_multiframes = 50/frame_rate; + frame_rate = 50; + packet_code = 3; + } + } + + if(tocmode==MODE_SILK_ONLY&&bw>OPUS_BANDWIDTH_WIDEBAND) + bw=OPUS_BANDWIDTH_WIDEBAND; + else if (tocmode==MODE_CELT_ONLY&&bw==OPUS_BANDWIDTH_MEDIUMBAND) + bw=OPUS_BANDWIDTH_NARROWBAND; + else if (tocmode==MODE_HYBRID&&bw<=OPUS_BANDWIDTH_SUPERWIDEBAND) + bw=OPUS_BANDWIDTH_SUPERWIDEBAND; + + data[0] = gen_toc(tocmode, frame_rate, bw, st->stream_channels); + data[0] |= packet_code; + + ret = packet_code <= 1 ? 1 : 2; + + max_data_bytes = IMAX(max_data_bytes, ret); + + if (packet_code==3) + data[1] = num_multiframes; + + if (!st->use_vbr) + { + ret = opus_packet_pad(data, ret, max_data_bytes); + if (ret == OPUS_OK) + ret = max_data_bytes; + else + ret = OPUS_INTERNAL_ERROR; + } + RESTORE_STACK; + return ret; + } + max_rate = frame_rate*max_data_bytes*8; + + /* Equivalent 20-ms rate for mode/channel/bandwidth decisions */ + equiv_rate = compute_equiv_rate(st->bitrate_bps, st->channels, st->Fs/frame_size, + st->use_vbr, 0, st->silk_mode.complexity, st->silk_mode.packetLossPercentage); + + if (st->signal_type == OPUS_SIGNAL_VOICE) + voice_est = 127; + else if (st->signal_type == OPUS_SIGNAL_MUSIC) + voice_est = 0; + else if (st->voice_ratio >= 0) + { + voice_est = st->voice_ratio*327>>8; + /* For AUDIO, never be more than 90% confident of having speech */ + if (st->application == OPUS_APPLICATION_AUDIO) + voice_est = IMIN(voice_est, 115); + } else if (st->application == OPUS_APPLICATION_VOIP) + voice_est = 115; + else + voice_est = 48; + + if (st->force_channels!=OPUS_AUTO && st->channels == 2) + { + st->stream_channels = st->force_channels; + } else { +#ifdef FUZZING + /* Random mono/stereo decision */ + if (st->channels == 2 && (rand()&0x1F)==0) + st->stream_channels = 3-st->stream_channels; +#else + /* Rate-dependent mono-stereo decision */ + if (st->channels == 2) + { + opus_int32 stereo_threshold; + stereo_threshold = stereo_music_threshold + ((voice_est*voice_est*(stereo_voice_threshold-stereo_music_threshold))>>14); + if (st->stream_channels == 2) + stereo_threshold -= 1000; + else + stereo_threshold += 1000; + st->stream_channels = (equiv_rate > stereo_threshold) ? 2 : 1; + } else { + st->stream_channels = st->channels; + } +#endif + } + /* Update equivalent rate for channels decision. */ + equiv_rate = compute_equiv_rate(st->bitrate_bps, st->stream_channels, st->Fs/frame_size, + st->use_vbr, 0, st->silk_mode.complexity, st->silk_mode.packetLossPercentage); + + /* Allow SILK DTX if DTX is enabled but the generalized DTX cannot be used, + e.g. because of the complexity setting or sample rate. */ +#ifndef DISABLE_FLOAT_API + st->silk_mode.useDTX = st->use_dtx && !(analysis_info.valid || is_silence); +#else + st->silk_mode.useDTX = st->use_dtx; +#endif + + /* Mode selection depending on application and signal type */ + if (st->application == OPUS_APPLICATION_RESTRICTED_LOWDELAY) + { + st->mode = MODE_CELT_ONLY; + } else if (st->user_forced_mode == OPUS_AUTO) + { +#ifdef FUZZING + /* Random mode switching */ + if ((rand()&0xF)==0) + { + if ((rand()&0x1)==0) + st->mode = MODE_CELT_ONLY; + else + st->mode = MODE_SILK_ONLY; + } else { + if (st->prev_mode==MODE_CELT_ONLY) + st->mode = MODE_CELT_ONLY; + else + st->mode = MODE_SILK_ONLY; + } +#else + opus_int32 mode_voice, mode_music; + opus_int32 threshold; + + /* Interpolate based on stereo width */ + mode_voice = (opus_int32)(MULT16_32_Q15(Q15ONE-stereo_width,mode_thresholds[0][0]) + + MULT16_32_Q15(stereo_width,mode_thresholds[1][0])); + mode_music = (opus_int32)(MULT16_32_Q15(Q15ONE-stereo_width,mode_thresholds[1][1]) + + MULT16_32_Q15(stereo_width,mode_thresholds[1][1])); + /* Interpolate based on speech/music probability */ + threshold = mode_music + ((voice_est*voice_est*(mode_voice-mode_music))>>14); + /* Bias towards SILK for VoIP because of some useful features */ + if (st->application == OPUS_APPLICATION_VOIP) + threshold += 8000; + + /*printf("%f %d\n", stereo_width/(float)Q15ONE, threshold);*/ + /* Hysteresis */ + if (st->prev_mode == MODE_CELT_ONLY) + threshold -= 4000; + else if (st->prev_mode>0) + threshold += 4000; + + st->mode = (equiv_rate >= threshold) ? MODE_CELT_ONLY: MODE_SILK_ONLY; + + /* When FEC is enabled and there's enough packet loss, use SILK */ + if (st->silk_mode.useInBandFEC && st->silk_mode.packetLossPercentage > (128-voice_est)>>4) + st->mode = MODE_SILK_ONLY; + /* When encoding voice and DTX is enabled but the generalized DTX cannot be used, + use SILK in order to make use of its DTX. */ + if (st->silk_mode.useDTX && voice_est > 100) + st->mode = MODE_SILK_ONLY; +#endif + + /* If max_data_bytes represents less than 6 kb/s, switch to CELT-only mode */ + if (max_data_bytes < (frame_rate > 50 ? 9000 : 6000)*frame_size / (st->Fs * 8)) + st->mode = MODE_CELT_ONLY; + } else { + st->mode = st->user_forced_mode; + } + + /* Override the chosen mode to make sure we meet the requested frame size */ + if (st->mode != MODE_CELT_ONLY && frame_size < st->Fs/100) + st->mode = MODE_CELT_ONLY; + if (st->lfe) + st->mode = MODE_CELT_ONLY; + + if (st->prev_mode > 0 && + ((st->mode != MODE_CELT_ONLY && st->prev_mode == MODE_CELT_ONLY) || + (st->mode == MODE_CELT_ONLY && st->prev_mode != MODE_CELT_ONLY))) + { + redundancy = 1; + celt_to_silk = (st->mode != MODE_CELT_ONLY); + if (!celt_to_silk) + { + /* Switch to SILK/hybrid if frame size is 10 ms or more*/ + if (frame_size >= st->Fs/100) + { + st->mode = st->prev_mode; + to_celt = 1; + } else { + redundancy=0; + } + } + } + + /* When encoding multiframes, we can ask for a switch to CELT only in the last frame. This switch + * is processed above as the requested mode shouldn't interrupt stereo->mono transition. */ + if (st->stream_channels == 1 && st->prev_channels ==2 && st->silk_mode.toMono==0 + && st->mode != MODE_CELT_ONLY && st->prev_mode != MODE_CELT_ONLY) + { + /* Delay stereo->mono transition by two frames so that SILK can do a smooth downmix */ + st->silk_mode.toMono = 1; + st->stream_channels = 2; + } else { + st->silk_mode.toMono = 0; + } + + /* Update equivalent rate with mode decision. */ + equiv_rate = compute_equiv_rate(st->bitrate_bps, st->stream_channels, st->Fs/frame_size, + st->use_vbr, st->mode, st->silk_mode.complexity, st->silk_mode.packetLossPercentage); + + if (st->mode != MODE_CELT_ONLY && st->prev_mode == MODE_CELT_ONLY) + { + silk_EncControlStruct dummy; + silk_InitEncoder( silk_enc, st->arch, &dummy); + prefill=1; + } + + /* Automatic (rate-dependent) bandwidth selection */ + if (st->mode == MODE_CELT_ONLY || st->first || st->silk_mode.allowBandwidthSwitch) + { + const opus_int32 *voice_bandwidth_thresholds, *music_bandwidth_thresholds; + opus_int32 bandwidth_thresholds[8]; + int bandwidth = OPUS_BANDWIDTH_FULLBAND; + + if (st->channels==2 && st->force_channels!=1) + { + voice_bandwidth_thresholds = stereo_voice_bandwidth_thresholds; + music_bandwidth_thresholds = stereo_music_bandwidth_thresholds; + } else { + voice_bandwidth_thresholds = mono_voice_bandwidth_thresholds; + music_bandwidth_thresholds = mono_music_bandwidth_thresholds; + } + /* Interpolate bandwidth thresholds depending on voice estimation */ + for (i=0;i<8;i++) + { + bandwidth_thresholds[i] = music_bandwidth_thresholds[i] + + ((voice_est*voice_est*(voice_bandwidth_thresholds[i]-music_bandwidth_thresholds[i]))>>14); + } + do { + int threshold, hysteresis; + threshold = bandwidth_thresholds[2*(bandwidth-OPUS_BANDWIDTH_MEDIUMBAND)]; + hysteresis = bandwidth_thresholds[2*(bandwidth-OPUS_BANDWIDTH_MEDIUMBAND)+1]; + if (!st->first) + { + if (st->auto_bandwidth >= bandwidth) + threshold -= hysteresis; + else + threshold += hysteresis; + } + if (equiv_rate >= threshold) + break; + } while (--bandwidth>OPUS_BANDWIDTH_NARROWBAND); + /* We don't use mediumband anymore, except when explicitly requested or during + mode transitions. */ + if (bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) + bandwidth = OPUS_BANDWIDTH_WIDEBAND; + st->bandwidth = st->auto_bandwidth = bandwidth; + /* Prevents any transition to SWB/FB until the SILK layer has fully + switched to WB mode and turned the variable LP filter off */ + if (!st->first && st->mode != MODE_CELT_ONLY && !st->silk_mode.inWBmodeWithoutVariableLP && st->bandwidth > OPUS_BANDWIDTH_WIDEBAND) + st->bandwidth = OPUS_BANDWIDTH_WIDEBAND; + } + + if (st->bandwidth>st->max_bandwidth) + st->bandwidth = st->max_bandwidth; + + if (st->user_bandwidth != OPUS_AUTO) + st->bandwidth = st->user_bandwidth; + + /* This prevents us from using hybrid at unsafe CBR/max rates */ + if (st->mode != MODE_CELT_ONLY && max_rate < 15000) + { + st->bandwidth = IMIN(st->bandwidth, OPUS_BANDWIDTH_WIDEBAND); + } + + /* Prevents Opus from wasting bits on frequencies that are above + the Nyquist rate of the input signal */ + if (st->Fs <= 24000 && st->bandwidth > OPUS_BANDWIDTH_SUPERWIDEBAND) + st->bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + if (st->Fs <= 16000 && st->bandwidth > OPUS_BANDWIDTH_WIDEBAND) + st->bandwidth = OPUS_BANDWIDTH_WIDEBAND; + if (st->Fs <= 12000 && st->bandwidth > OPUS_BANDWIDTH_MEDIUMBAND) + st->bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + if (st->Fs <= 8000 && st->bandwidth > OPUS_BANDWIDTH_NARROWBAND) + st->bandwidth = OPUS_BANDWIDTH_NARROWBAND; +#ifndef DISABLE_FLOAT_API + /* Use detected bandwidth to reduce the encoded bandwidth. */ + if (st->detected_bandwidth && st->user_bandwidth == OPUS_AUTO) + { + int min_detected_bandwidth; + /* Makes bandwidth detection more conservative just in case the detector + gets it wrong when we could have coded a high bandwidth transparently. + When operating in SILK/hybrid mode, we don't go below wideband to avoid + more complicated switches that require redundancy. */ + if (equiv_rate <= 18000*st->stream_channels && st->mode == MODE_CELT_ONLY) + min_detected_bandwidth = OPUS_BANDWIDTH_NARROWBAND; + else if (equiv_rate <= 24000*st->stream_channels && st->mode == MODE_CELT_ONLY) + min_detected_bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + else if (equiv_rate <= 30000*st->stream_channels) + min_detected_bandwidth = OPUS_BANDWIDTH_WIDEBAND; + else if (equiv_rate <= 44000*st->stream_channels) + min_detected_bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + else + min_detected_bandwidth = OPUS_BANDWIDTH_FULLBAND; + + st->detected_bandwidth = IMAX(st->detected_bandwidth, min_detected_bandwidth); + st->bandwidth = IMIN(st->bandwidth, st->detected_bandwidth); + } +#endif + st->silk_mode.LBRR_coded = decide_fec(st->silk_mode.useInBandFEC, st->silk_mode.packetLossPercentage, + st->silk_mode.LBRR_coded, st->mode, &st->bandwidth, equiv_rate); + celt_encoder_ctl(celt_enc, OPUS_SET_LSB_DEPTH(lsb_depth)); + + /* CELT mode doesn't support mediumband, use wideband instead */ + if (st->mode == MODE_CELT_ONLY && st->bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) + st->bandwidth = OPUS_BANDWIDTH_WIDEBAND; + if (st->lfe) + st->bandwidth = OPUS_BANDWIDTH_NARROWBAND; + + curr_bandwidth = st->bandwidth; + + /* Chooses the appropriate mode for speech + *NEVER* switch to/from CELT-only mode here as this will invalidate some assumptions */ + if (st->mode == MODE_SILK_ONLY && curr_bandwidth > OPUS_BANDWIDTH_WIDEBAND) + st->mode = MODE_HYBRID; + if (st->mode == MODE_HYBRID && curr_bandwidth <= OPUS_BANDWIDTH_WIDEBAND) + st->mode = MODE_SILK_ONLY; + + /* Can't support higher than >60 ms frames, and >20 ms when in Hybrid or CELT-only modes */ + if ((frame_size > st->Fs/50 && (st->mode != MODE_SILK_ONLY)) || frame_size > 3*st->Fs/50) + { + int enc_frame_size; + int nb_frames; + + if (st->mode == MODE_SILK_ONLY) + { + if (frame_size == 2*st->Fs/25) /* 80 ms -> 2x 40 ms */ + enc_frame_size = st->Fs/25; + else if (frame_size == 3*st->Fs/25) /* 120 ms -> 2x 60 ms */ + enc_frame_size = 3*st->Fs/50; + else /* 100 ms -> 5x 20 ms */ + enc_frame_size = st->Fs/50; + } + else + enc_frame_size = st->Fs/50; + + nb_frames = frame_size/enc_frame_size; + +#ifndef DISABLE_FLOAT_API + if (analysis_read_pos_bak!= -1) + { + st->analysis.read_pos = analysis_read_pos_bak; + st->analysis.read_subframe = analysis_read_subframe_bak; + } +#endif + + ret = encode_multiframe_packet(st, pcm, nb_frames, enc_frame_size, data, + out_data_bytes, to_celt, lsb_depth, float_api); + + RESTORE_STACK; + return ret; + } + + /* For the first frame at a new SILK bandwidth */ + if (st->silk_bw_switch) + { + redundancy = 1; + celt_to_silk = 1; + st->silk_bw_switch = 0; + /* Do a prefill without reseting the sampling rate control. */ + prefill=2; + } + + /* If we decided to go with CELT, make sure redundancy is off, no matter what + we decided earlier. */ + if (st->mode == MODE_CELT_ONLY) + redundancy = 0; + + if (redundancy) + { + redundancy_bytes = compute_redundancy_bytes(max_data_bytes, st->bitrate_bps, frame_rate, st->stream_channels); + if (redundancy_bytes == 0) + redundancy = 0; + } + + /* printf("%d %d %d %d\n", st->bitrate_bps, st->stream_channels, st->mode, curr_bandwidth); */ + bytes_target = IMIN(max_data_bytes-redundancy_bytes, st->bitrate_bps * frame_size / (st->Fs * 8)) - 1; + + data += 1; + + ec_enc_init(&enc, data, max_data_bytes-1); + + ALLOC(pcm_buf, (total_buffer+frame_size)*st->channels, opus_val16); + OPUS_COPY(pcm_buf, &st->delay_buffer[(st->encoder_buffer-total_buffer)*st->channels], total_buffer*st->channels); + + if (st->mode == MODE_CELT_ONLY) + hp_freq_smth1 = silk_LSHIFT( silk_lin2log( VARIABLE_HP_MIN_CUTOFF_HZ ), 8 ); + else + hp_freq_smth1 = ((silk_encoder*)silk_enc)->state_Fxx[0].sCmn.variable_HP_smth1_Q15; + + st->variable_HP_smth2_Q15 = silk_SMLAWB( st->variable_HP_smth2_Q15, + hp_freq_smth1 - st->variable_HP_smth2_Q15, SILK_FIX_CONST( VARIABLE_HP_SMTH_COEF2, 16 ) ); + + /* convert from log scale to Hertz */ + cutoff_Hz = silk_log2lin( silk_RSHIFT( st->variable_HP_smth2_Q15, 8 ) ); + + if (st->application == OPUS_APPLICATION_VOIP) + { + hp_cutoff(pcm, cutoff_Hz, &pcm_buf[total_buffer*st->channels], st->hp_mem, frame_size, st->channels, st->Fs, st->arch); + } else { + dc_reject(pcm, 3, &pcm_buf[total_buffer*st->channels], st->hp_mem, frame_size, st->channels, st->Fs); + } +#ifndef FIXED_POINT + if (float_api) + { + opus_val32 sum; + sum = celt_inner_prod(&pcm_buf[total_buffer*st->channels], &pcm_buf[total_buffer*st->channels], frame_size*st->channels, st->arch); + /* This should filter out both NaNs and ridiculous signals that could + cause NaNs further down. */ + if (!(sum < 1e9f) || celt_isnan(sum)) + { + OPUS_CLEAR(&pcm_buf[total_buffer*st->channels], frame_size*st->channels); + st->hp_mem[0] = st->hp_mem[1] = st->hp_mem[2] = st->hp_mem[3] = 0; + } + } +#endif + + + /* SILK processing */ + HB_gain = Q15ONE; + if (st->mode != MODE_CELT_ONLY) + { + opus_int32 total_bitRate, celt_rate; +#ifdef FIXED_POINT + const opus_int16 *pcm_silk; +#else + VARDECL(opus_int16, pcm_silk); + ALLOC(pcm_silk, st->channels*frame_size, opus_int16); +#endif + + /* Distribute bits between SILK and CELT */ + total_bitRate = 8 * bytes_target * frame_rate; + if( st->mode == MODE_HYBRID ) { + /* Base rate for SILK */ + st->silk_mode.bitRate = compute_silk_rate_for_hybrid(total_bitRate, + curr_bandwidth, st->Fs == 50 * frame_size, st->use_vbr, st->silk_mode.LBRR_coded, + st->stream_channels); + if (!st->energy_masking) + { + /* Increasingly attenuate high band when it gets allocated fewer bits */ + celt_rate = total_bitRate - st->silk_mode.bitRate; + HB_gain = Q15ONE - SHR32(celt_exp2(-celt_rate * QCONST16(1.f/1024, 10)), 1); + } + } else { + /* SILK gets all bits */ + st->silk_mode.bitRate = total_bitRate; + } + + /* Surround masking for SILK */ + if (st->energy_masking && st->use_vbr && !st->lfe) + { + opus_val32 mask_sum=0; + opus_val16 masking_depth; + opus_int32 rate_offset; + int c; + int end = 17; + opus_int16 srate = 16000; + if (st->bandwidth == OPUS_BANDWIDTH_NARROWBAND) + { + end = 13; + srate = 8000; + } else if (st->bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) + { + end = 15; + srate = 12000; + } + for (c=0;cchannels;c++) + { + for(i=0;ienergy_masking[21*c+i], + QCONST16(.5f, DB_SHIFT)), -QCONST16(2.0f, DB_SHIFT)); + if (mask > 0) + mask = HALF16(mask); + mask_sum += mask; + } + } + /* Conservative rate reduction, we cut the masking in half */ + masking_depth = mask_sum / end*st->channels; + masking_depth += QCONST16(.2f, DB_SHIFT); + rate_offset = (opus_int32)PSHR32(MULT16_16(srate, masking_depth), DB_SHIFT); + rate_offset = MAX32(rate_offset, -2*st->silk_mode.bitRate/3); + /* Split the rate change between the SILK and CELT part for hybrid. */ + if (st->bandwidth==OPUS_BANDWIDTH_SUPERWIDEBAND || st->bandwidth==OPUS_BANDWIDTH_FULLBAND) + st->silk_mode.bitRate += 3*rate_offset/5; + else + st->silk_mode.bitRate += rate_offset; + } + + st->silk_mode.payloadSize_ms = 1000 * frame_size / st->Fs; + st->silk_mode.nChannelsAPI = st->channels; + st->silk_mode.nChannelsInternal = st->stream_channels; + if (curr_bandwidth == OPUS_BANDWIDTH_NARROWBAND) { + st->silk_mode.desiredInternalSampleRate = 8000; + } else if (curr_bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) { + st->silk_mode.desiredInternalSampleRate = 12000; + } else { + celt_assert( st->mode == MODE_HYBRID || curr_bandwidth == OPUS_BANDWIDTH_WIDEBAND ); + st->silk_mode.desiredInternalSampleRate = 16000; + } + if( st->mode == MODE_HYBRID ) { + /* Don't allow bandwidth reduction at lowest bitrates in hybrid mode */ + st->silk_mode.minInternalSampleRate = 16000; + } else { + st->silk_mode.minInternalSampleRate = 8000; + } + + st->silk_mode.maxInternalSampleRate = 16000; + if (st->mode == MODE_SILK_ONLY) + { + opus_int32 effective_max_rate = max_rate; + if (frame_rate > 50) + effective_max_rate = effective_max_rate*2/3; + if (effective_max_rate < 8000) + { + st->silk_mode.maxInternalSampleRate = 12000; + st->silk_mode.desiredInternalSampleRate = IMIN(12000, st->silk_mode.desiredInternalSampleRate); + } + if (effective_max_rate < 7000) + { + st->silk_mode.maxInternalSampleRate = 8000; + st->silk_mode.desiredInternalSampleRate = IMIN(8000, st->silk_mode.desiredInternalSampleRate); + } + } + + st->silk_mode.useCBR = !st->use_vbr; + + /* Call SILK encoder for the low band */ + + /* Max bits for SILK, counting ToC, redundancy bytes, and optionally redundancy. */ + st->silk_mode.maxBits = (max_data_bytes-1)*8; + if (redundancy && redundancy_bytes >= 2) + { + /* Counting 1 bit for redundancy position and 20 bits for flag+size (only for hybrid). */ + st->silk_mode.maxBits -= redundancy_bytes*8 + 1; + if (st->mode == MODE_HYBRID) + st->silk_mode.maxBits -= 20; + } + if (st->silk_mode.useCBR) + { + if (st->mode == MODE_HYBRID) + { + st->silk_mode.maxBits = IMIN(st->silk_mode.maxBits, st->silk_mode.bitRate * frame_size / st->Fs); + } + } else { + /* Constrained VBR. */ + if (st->mode == MODE_HYBRID) + { + /* Compute SILK bitrate corresponding to the max total bits available */ + opus_int32 maxBitRate = compute_silk_rate_for_hybrid(st->silk_mode.maxBits*st->Fs / frame_size, + curr_bandwidth, st->Fs == 50 * frame_size, st->use_vbr, st->silk_mode.LBRR_coded, + st->stream_channels); + st->silk_mode.maxBits = maxBitRate * frame_size / st->Fs; + } + } + + if (prefill) + { + opus_int32 zero=0; + int prefill_offset; + /* Use a smooth onset for the SILK prefill to avoid the encoder trying to encode + a discontinuity. The exact location is what we need to avoid leaving any "gap" + in the audio when mixing with the redundant CELT frame. Here we can afford to + overwrite st->delay_buffer because the only thing that uses it before it gets + rewritten is tmp_prefill[] and even then only the part after the ramp really + gets used (rather than sent to the encoder and discarded) */ + prefill_offset = st->channels*(st->encoder_buffer-st->delay_compensation-st->Fs/400); + gain_fade(st->delay_buffer+prefill_offset, st->delay_buffer+prefill_offset, + 0, Q15ONE, celt_mode->overlap, st->Fs/400, st->channels, celt_mode->window, st->Fs); + OPUS_CLEAR(st->delay_buffer, prefill_offset); +#ifdef FIXED_POINT + pcm_silk = st->delay_buffer; +#else + for (i=0;iencoder_buffer*st->channels;i++) + pcm_silk[i] = FLOAT2INT16(st->delay_buffer[i]); +#endif + silk_Encode( silk_enc, &st->silk_mode, pcm_silk, st->encoder_buffer, NULL, &zero, prefill, activity ); + /* Prevent a second switch in the real encode call. */ + st->silk_mode.opusCanSwitch = 0; + } + +#ifdef FIXED_POINT + pcm_silk = pcm_buf+total_buffer*st->channels; +#else + for (i=0;ichannels;i++) + pcm_silk[i] = FLOAT2INT16(pcm_buf[total_buffer*st->channels + i]); +#endif + ret = silk_Encode( silk_enc, &st->silk_mode, pcm_silk, frame_size, &enc, &nBytes, 0, activity ); + if( ret ) { + /*fprintf (stderr, "SILK encode error: %d\n", ret);*/ + /* Handle error */ + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + + /* Extract SILK internal bandwidth for signaling in first byte */ + if( st->mode == MODE_SILK_ONLY ) { + if( st->silk_mode.internalSampleRate == 8000 ) { + curr_bandwidth = OPUS_BANDWIDTH_NARROWBAND; + } else if( st->silk_mode.internalSampleRate == 12000 ) { + curr_bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + } else if( st->silk_mode.internalSampleRate == 16000 ) { + curr_bandwidth = OPUS_BANDWIDTH_WIDEBAND; + } + } else { + celt_assert( st->silk_mode.internalSampleRate == 16000 ); + } + + st->silk_mode.opusCanSwitch = st->silk_mode.switchReady && !st->nonfinal_frame; + + if (nBytes==0) + { + st->rangeFinal = 0; + data[-1] = gen_toc(st->mode, st->Fs/frame_size, curr_bandwidth, st->stream_channels); + RESTORE_STACK; + return 1; + } + + /* FIXME: How do we allocate the redundancy for CBR? */ + if (st->silk_mode.opusCanSwitch) + { + redundancy_bytes = compute_redundancy_bytes(max_data_bytes, st->bitrate_bps, frame_rate, st->stream_channels); + redundancy = (redundancy_bytes != 0); + celt_to_silk = 0; + st->silk_bw_switch = 1; + } + } + + /* CELT processing */ + { + int endband=21; + + switch(curr_bandwidth) + { + case OPUS_BANDWIDTH_NARROWBAND: + endband = 13; + break; + case OPUS_BANDWIDTH_MEDIUMBAND: + case OPUS_BANDWIDTH_WIDEBAND: + endband = 17; + break; + case OPUS_BANDWIDTH_SUPERWIDEBAND: + endband = 19; + break; + case OPUS_BANDWIDTH_FULLBAND: + endband = 21; + break; + } + celt_encoder_ctl(celt_enc, CELT_SET_END_BAND(endband)); + celt_encoder_ctl(celt_enc, CELT_SET_CHANNELS(st->stream_channels)); + } + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(OPUS_BITRATE_MAX)); + if (st->mode != MODE_SILK_ONLY) + { + opus_val32 celt_pred=2; + celt_encoder_ctl(celt_enc, OPUS_SET_VBR(0)); + /* We may still decide to disable prediction later */ + if (st->silk_mode.reducedDependency) + celt_pred = 0; + celt_encoder_ctl(celt_enc, CELT_SET_PREDICTION(celt_pred)); + + if (st->mode == MODE_HYBRID) + { + if( st->use_vbr ) { + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(st->bitrate_bps-st->silk_mode.bitRate)); + celt_encoder_ctl(celt_enc, OPUS_SET_VBR_CONSTRAINT(0)); + } + } else { + if (st->use_vbr) + { + celt_encoder_ctl(celt_enc, OPUS_SET_VBR(1)); + celt_encoder_ctl(celt_enc, OPUS_SET_VBR_CONSTRAINT(st->vbr_constraint)); + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(st->bitrate_bps)); + } + } + } + + ALLOC(tmp_prefill, st->channels*st->Fs/400, opus_val16); + if (st->mode != MODE_SILK_ONLY && st->mode != st->prev_mode && st->prev_mode > 0) + { + OPUS_COPY(tmp_prefill, &st->delay_buffer[(st->encoder_buffer-total_buffer-st->Fs/400)*st->channels], st->channels*st->Fs/400); + } + + if (st->channels*(st->encoder_buffer-(frame_size+total_buffer)) > 0) + { + OPUS_MOVE(st->delay_buffer, &st->delay_buffer[st->channels*frame_size], st->channels*(st->encoder_buffer-frame_size-total_buffer)); + OPUS_COPY(&st->delay_buffer[st->channels*(st->encoder_buffer-frame_size-total_buffer)], + &pcm_buf[0], + (frame_size+total_buffer)*st->channels); + } else { + OPUS_COPY(st->delay_buffer, &pcm_buf[(frame_size+total_buffer-st->encoder_buffer)*st->channels], st->encoder_buffer*st->channels); + } + /* gain_fade() and stereo_fade() need to be after the buffer copying + because we don't want any of this to affect the SILK part */ + if( st->prev_HB_gain < Q15ONE || HB_gain < Q15ONE ) { + gain_fade(pcm_buf, pcm_buf, + st->prev_HB_gain, HB_gain, celt_mode->overlap, frame_size, st->channels, celt_mode->window, st->Fs); + } + st->prev_HB_gain = HB_gain; + if (st->mode != MODE_HYBRID || st->stream_channels==1) + { + if (equiv_rate > 32000) + st->silk_mode.stereoWidth_Q14 = 16384; + else if (equiv_rate < 16000) + st->silk_mode.stereoWidth_Q14 = 0; + else + st->silk_mode.stereoWidth_Q14 = 16384 - 2048*(opus_int32)(32000-equiv_rate)/(equiv_rate-14000); + } + if( !st->energy_masking && st->channels == 2 ) { + /* Apply stereo width reduction (at low bitrates) */ + if( st->hybrid_stereo_width_Q14 < (1 << 14) || st->silk_mode.stereoWidth_Q14 < (1 << 14) ) { + opus_val16 g1, g2; + g1 = st->hybrid_stereo_width_Q14; + g2 = (opus_val16)(st->silk_mode.stereoWidth_Q14); +#ifdef FIXED_POINT + g1 = g1==16384 ? Q15ONE : SHL16(g1,1); + g2 = g2==16384 ? Q15ONE : SHL16(g2,1); +#else + g1 *= (1.f/16384); + g2 *= (1.f/16384); +#endif + stereo_fade(pcm_buf, pcm_buf, g1, g2, celt_mode->overlap, + frame_size, st->channels, celt_mode->window, st->Fs); + st->hybrid_stereo_width_Q14 = st->silk_mode.stereoWidth_Q14; + } + } + + if ( st->mode != MODE_CELT_ONLY && ec_tell(&enc)+17+20*(st->mode == MODE_HYBRID) <= 8*(max_data_bytes-1)) + { + /* For SILK mode, the redundancy is inferred from the length */ + if (st->mode == MODE_HYBRID) + ec_enc_bit_logp(&enc, redundancy, 12); + if (redundancy) + { + int max_redundancy; + ec_enc_bit_logp(&enc, celt_to_silk, 1); + if (st->mode == MODE_HYBRID) + { + /* Reserve the 8 bits needed for the redundancy length, + and at least a few bits for CELT if possible */ + max_redundancy = (max_data_bytes-1)-((ec_tell(&enc)+8+3+7)>>3); + } + else + max_redundancy = (max_data_bytes-1)-((ec_tell(&enc)+7)>>3); + /* Target the same bit-rate for redundancy as for the rest, + up to a max of 257 bytes */ + redundancy_bytes = IMIN(max_redundancy, redundancy_bytes); + redundancy_bytes = IMIN(257, IMAX(2, redundancy_bytes)); + if (st->mode == MODE_HYBRID) + ec_enc_uint(&enc, redundancy_bytes-2, 256); + } + } else { + redundancy = 0; + } + + if (!redundancy) + { + st->silk_bw_switch = 0; + redundancy_bytes = 0; + } + if (st->mode != MODE_CELT_ONLY)start_band=17; + + if (st->mode == MODE_SILK_ONLY) + { + ret = (ec_tell(&enc)+7)>>3; + ec_enc_done(&enc); + nb_compr_bytes = ret; + } else { + nb_compr_bytes = (max_data_bytes-1)-redundancy_bytes; + ec_enc_shrink(&enc, nb_compr_bytes); + } + +#ifndef DISABLE_FLOAT_API + if (redundancy || st->mode != MODE_SILK_ONLY) + celt_encoder_ctl(celt_enc, CELT_SET_ANALYSIS(&analysis_info)); +#endif + if (st->mode == MODE_HYBRID) { + SILKInfo info; + info.signalType = st->silk_mode.signalType; + info.offset = st->silk_mode.offset; + celt_encoder_ctl(celt_enc, CELT_SET_SILK_INFO(&info)); + } + + /* 5 ms redundant frame for CELT->SILK */ + if (redundancy && celt_to_silk) + { + int err; + celt_encoder_ctl(celt_enc, CELT_SET_START_BAND(0)); + celt_encoder_ctl(celt_enc, OPUS_SET_VBR(0)); + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(OPUS_BITRATE_MAX)); + err = celt_encode_with_ec(celt_enc, pcm_buf, st->Fs/200, data+nb_compr_bytes, redundancy_bytes, NULL); + if (err < 0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + celt_encoder_ctl(celt_enc, OPUS_GET_FINAL_RANGE(&redundant_rng)); + celt_encoder_ctl(celt_enc, OPUS_RESET_STATE); + } + + celt_encoder_ctl(celt_enc, CELT_SET_START_BAND(start_band)); + + if (st->mode != MODE_SILK_ONLY) + { + if (st->mode != st->prev_mode && st->prev_mode > 0) + { + unsigned char dummy[2]; + celt_encoder_ctl(celt_enc, OPUS_RESET_STATE); + + /* Prefilling */ + celt_encode_with_ec(celt_enc, tmp_prefill, st->Fs/400, dummy, 2, NULL); + celt_encoder_ctl(celt_enc, CELT_SET_PREDICTION(0)); + } + /* If false, we already busted the budget and we'll end up with a "PLC frame" */ + if (ec_tell(&enc) <= 8*nb_compr_bytes) + { + /* Set the bitrate again if it was overridden in the redundancy code above*/ + if (redundancy && celt_to_silk && st->mode==MODE_HYBRID && st->use_vbr) + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(st->bitrate_bps-st->silk_mode.bitRate)); + celt_encoder_ctl(celt_enc, OPUS_SET_VBR(st->use_vbr)); + ret = celt_encode_with_ec(celt_enc, pcm_buf, frame_size, NULL, nb_compr_bytes, &enc); + if (ret < 0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + /* Put CELT->SILK redundancy data in the right place. */ + if (redundancy && celt_to_silk && st->mode==MODE_HYBRID && st->use_vbr) + { + OPUS_MOVE(data+ret, data+nb_compr_bytes, redundancy_bytes); + nb_compr_bytes = nb_compr_bytes+redundancy_bytes; + } + } + } + + /* 5 ms redundant frame for SILK->CELT */ + if (redundancy && !celt_to_silk) + { + int err; + unsigned char dummy[2]; + int N2, N4; + N2 = st->Fs/200; + N4 = st->Fs/400; + + celt_encoder_ctl(celt_enc, OPUS_RESET_STATE); + celt_encoder_ctl(celt_enc, CELT_SET_START_BAND(0)); + celt_encoder_ctl(celt_enc, CELT_SET_PREDICTION(0)); + celt_encoder_ctl(celt_enc, OPUS_SET_VBR(0)); + celt_encoder_ctl(celt_enc, OPUS_SET_BITRATE(OPUS_BITRATE_MAX)); + + if (st->mode == MODE_HYBRID) + { + /* Shrink packet to what the encoder actually used. */ + nb_compr_bytes = ret; + ec_enc_shrink(&enc, nb_compr_bytes); + } + /* NOTE: We could speed this up slightly (at the expense of code size) by just adding a function that prefills the buffer */ + celt_encode_with_ec(celt_enc, pcm_buf+st->channels*(frame_size-N2-N4), N4, dummy, 2, NULL); + + err = celt_encode_with_ec(celt_enc, pcm_buf+st->channels*(frame_size-N2), N2, data+nb_compr_bytes, redundancy_bytes, NULL); + if (err < 0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + celt_encoder_ctl(celt_enc, OPUS_GET_FINAL_RANGE(&redundant_rng)); + } + + + + /* Signalling the mode in the first byte */ + data--; + data[0] = gen_toc(st->mode, st->Fs/frame_size, curr_bandwidth, st->stream_channels); + + st->rangeFinal = enc.rng ^ redundant_rng; + + if (to_celt) + st->prev_mode = MODE_CELT_ONLY; + else + st->prev_mode = st->mode; + st->prev_channels = st->stream_channels; + st->prev_framesize = frame_size; + + st->first = 0; + + /* DTX decision */ +#ifndef DISABLE_FLOAT_API + if (st->use_dtx && (analysis_info.valid || is_silence)) + { + if (decide_dtx_mode(activity, &st->nb_no_activity_ms_Q1, 2*1000*frame_size/st->Fs)) + { + st->rangeFinal = 0; + data[0] = gen_toc(st->mode, st->Fs/frame_size, curr_bandwidth, st->stream_channels); + RESTORE_STACK; + return 1; + } + } else { + st->nb_no_activity_ms_Q1 = 0; + } +#endif + + /* In the unlikely case that the SILK encoder busted its target, tell + the decoder to call the PLC */ + if (ec_tell(&enc) > (max_data_bytes-1)*8) + { + if (max_data_bytes < 2) + { + RESTORE_STACK; + return OPUS_BUFFER_TOO_SMALL; + } + data[1] = 0; + ret = 1; + st->rangeFinal = 0; + } else if (st->mode==MODE_SILK_ONLY&&!redundancy) + { + /*When in LPC only mode it's perfectly + reasonable to strip off trailing zero bytes as + the required range decoder behavior is to + fill these in. This can't be done when the MDCT + modes are used because the decoder needs to know + the actual length for allocation purposes.*/ + while(ret>2&&data[ret]==0)ret--; + } + /* Count ToC and redundancy */ + ret += 1+redundancy_bytes; + if (!st->use_vbr) + { + if (opus_packet_pad(data, ret, max_data_bytes) != OPUS_OK) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + ret = max_data_bytes; + } + RESTORE_STACK; + return ret; +} + +#ifdef FIXED_POINT + +#ifndef DISABLE_FLOAT_API +opus_int32 opus_encode_float(OpusEncoder *st, const float *pcm, int analysis_frame_size, + unsigned char *data, opus_int32 max_data_bytes) +{ + int i, ret; + int frame_size; + VARDECL(opus_int16, in); + ALLOC_STACK; + + frame_size = frame_size_select(analysis_frame_size, st->variable_duration, st->Fs); + if (frame_size <= 0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + ALLOC(in, frame_size*st->channels, opus_int16); + + for (i=0;ichannels;i++) + in[i] = FLOAT2INT16(pcm[i]); + ret = opus_encode_native(st, in, frame_size, data, max_data_bytes, 16, + pcm, analysis_frame_size, 0, -2, st->channels, downmix_float, 1); + RESTORE_STACK; + return ret; +} +#endif + +opus_int32 opus_encode(OpusEncoder *st, const opus_int16 *pcm, int analysis_frame_size, + unsigned char *data, opus_int32 out_data_bytes) +{ + int frame_size; + frame_size = frame_size_select(analysis_frame_size, st->variable_duration, st->Fs); + return opus_encode_native(st, pcm, frame_size, data, out_data_bytes, 16, + pcm, analysis_frame_size, 0, -2, st->channels, downmix_int, 0); +} + +#else +opus_int32 opus_encode(OpusEncoder *st, const opus_int16 *pcm, int analysis_frame_size, + unsigned char *data, opus_int32 max_data_bytes) +{ + int i, ret; + int frame_size; + VARDECL(float, in); + ALLOC_STACK; + + frame_size = frame_size_select(analysis_frame_size, st->variable_duration, st->Fs); + if (frame_size <= 0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + ALLOC(in, frame_size*st->channels, float); + + for (i=0;ichannels;i++) + in[i] = (1.0f/32768)*pcm[i]; + ret = opus_encode_native(st, in, frame_size, data, max_data_bytes, 16, + pcm, analysis_frame_size, 0, -2, st->channels, downmix_int, 0); + RESTORE_STACK; + return ret; +} +opus_int32 opus_encode_float(OpusEncoder *st, const float *pcm, int analysis_frame_size, + unsigned char *data, opus_int32 out_data_bytes) +{ + int frame_size; + frame_size = frame_size_select(analysis_frame_size, st->variable_duration, st->Fs); + return opus_encode_native(st, pcm, frame_size, data, out_data_bytes, 24, + pcm, analysis_frame_size, 0, -2, st->channels, downmix_float, 1); +} +#endif + + +int opus_encoder_ctl(OpusEncoder *st, int request, ...) +{ + int ret; + CELTEncoder *celt_enc; + va_list ap; + + ret = OPUS_OK; + va_start(ap, request); + + celt_enc = (CELTEncoder*)((char*)st+st->celt_enc_offset); + + switch (request) + { + case OPUS_SET_APPLICATION_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if ( (value != OPUS_APPLICATION_VOIP && value != OPUS_APPLICATION_AUDIO + && value != OPUS_APPLICATION_RESTRICTED_LOWDELAY) + || (!st->first && st->application != value)) + { + ret = OPUS_BAD_ARG; + break; + } + st->application = value; +#ifndef DISABLE_FLOAT_API + st->analysis.application = value; +#endif + } + break; + case OPUS_GET_APPLICATION_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->application; + } + break; + case OPUS_SET_BITRATE_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value != OPUS_AUTO && value != OPUS_BITRATE_MAX) + { + if (value <= 0) + goto bad_arg; + else if (value <= 500) + value = 500; + else if (value > (opus_int32)300000*st->channels) + value = (opus_int32)300000*st->channels; + } + st->user_bitrate_bps = value; + } + break; + case OPUS_GET_BITRATE_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = user_bitrate_to_bitrate(st, st->prev_framesize, 1276); + } + break; + case OPUS_SET_FORCE_CHANNELS_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if((value<1 || value>st->channels) && value != OPUS_AUTO) + { + goto bad_arg; + } + st->force_channels = value; + } + break; + case OPUS_GET_FORCE_CHANNELS_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->force_channels; + } + break; + case OPUS_SET_MAX_BANDWIDTH_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value < OPUS_BANDWIDTH_NARROWBAND || value > OPUS_BANDWIDTH_FULLBAND) + { + goto bad_arg; + } + st->max_bandwidth = value; + if (st->max_bandwidth == OPUS_BANDWIDTH_NARROWBAND) { + st->silk_mode.maxInternalSampleRate = 8000; + } else if (st->max_bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) { + st->silk_mode.maxInternalSampleRate = 12000; + } else { + st->silk_mode.maxInternalSampleRate = 16000; + } + } + break; + case OPUS_GET_MAX_BANDWIDTH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->max_bandwidth; + } + break; + case OPUS_SET_BANDWIDTH_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if ((value < OPUS_BANDWIDTH_NARROWBAND || value > OPUS_BANDWIDTH_FULLBAND) && value != OPUS_AUTO) + { + goto bad_arg; + } + st->user_bandwidth = value; + if (st->user_bandwidth == OPUS_BANDWIDTH_NARROWBAND) { + st->silk_mode.maxInternalSampleRate = 8000; + } else if (st->user_bandwidth == OPUS_BANDWIDTH_MEDIUMBAND) { + st->silk_mode.maxInternalSampleRate = 12000; + } else { + st->silk_mode.maxInternalSampleRate = 16000; + } + } + break; + case OPUS_GET_BANDWIDTH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->bandwidth; + } + break; + case OPUS_SET_DTX_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->use_dtx = value; + } + break; + case OPUS_GET_DTX_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->use_dtx; + } + break; + case OPUS_SET_COMPLEXITY_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>10) + { + goto bad_arg; + } + st->silk_mode.complexity = value; + celt_encoder_ctl(celt_enc, OPUS_SET_COMPLEXITY(value)); + } + break; + case OPUS_GET_COMPLEXITY_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->silk_mode.complexity; + } + break; + case OPUS_SET_INBAND_FEC_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->silk_mode.useInBandFEC = value; + } + break; + case OPUS_GET_INBAND_FEC_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->silk_mode.useInBandFEC; + } + break; + case OPUS_SET_PACKET_LOSS_PERC_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value < 0 || value > 100) + { + goto bad_arg; + } + st->silk_mode.packetLossPercentage = value; + celt_encoder_ctl(celt_enc, OPUS_SET_PACKET_LOSS_PERC(value)); + } + break; + case OPUS_GET_PACKET_LOSS_PERC_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->silk_mode.packetLossPercentage; + } + break; + case OPUS_SET_VBR_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->use_vbr = value; + st->silk_mode.useCBR = 1-value; + } + break; + case OPUS_GET_VBR_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->use_vbr; + } + break; + case OPUS_SET_VOICE_RATIO_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<-1 || value>100) + { + goto bad_arg; + } + st->voice_ratio = value; + } + break; + case OPUS_GET_VOICE_RATIO_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->voice_ratio; + } + break; + case OPUS_SET_VBR_CONSTRAINT_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + st->vbr_constraint = value; + } + break; + case OPUS_GET_VBR_CONSTRAINT_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->vbr_constraint; + } + break; + case OPUS_SET_SIGNAL_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value!=OPUS_AUTO && value!=OPUS_SIGNAL_VOICE && value!=OPUS_SIGNAL_MUSIC) + { + goto bad_arg; + } + st->signal_type = value; + } + break; + case OPUS_GET_SIGNAL_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->signal_type; + } + break; + case OPUS_GET_LOOKAHEAD_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->Fs/400; + if (st->application != OPUS_APPLICATION_RESTRICTED_LOWDELAY) + *value += st->delay_compensation; + } + break; + case OPUS_GET_SAMPLE_RATE_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->Fs; + } + break; + case OPUS_GET_FINAL_RANGE_REQUEST: + { + opus_uint32 *value = va_arg(ap, opus_uint32*); + if (!value) + { + goto bad_arg; + } + *value = st->rangeFinal; + } + break; + case OPUS_SET_LSB_DEPTH_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value<8 || value>24) + { + goto bad_arg; + } + st->lsb_depth=value; + } + break; + case OPUS_GET_LSB_DEPTH_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->lsb_depth; + } + break; + case OPUS_SET_EXPERT_FRAME_DURATION_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value != OPUS_FRAMESIZE_ARG && value != OPUS_FRAMESIZE_2_5_MS && + value != OPUS_FRAMESIZE_5_MS && value != OPUS_FRAMESIZE_10_MS && + value != OPUS_FRAMESIZE_20_MS && value != OPUS_FRAMESIZE_40_MS && + value != OPUS_FRAMESIZE_60_MS && value != OPUS_FRAMESIZE_80_MS && + value != OPUS_FRAMESIZE_100_MS && value != OPUS_FRAMESIZE_120_MS) + { + goto bad_arg; + } + st->variable_duration = value; + } + break; + case OPUS_GET_EXPERT_FRAME_DURATION_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->variable_duration; + } + break; + case OPUS_SET_PREDICTION_DISABLED_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if (value > 1 || value < 0) + goto bad_arg; + st->silk_mode.reducedDependency = value; + } + break; + case OPUS_GET_PREDICTION_DISABLED_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + goto bad_arg; + *value = st->silk_mode.reducedDependency; + } + break; + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>1) + { + goto bad_arg; + } + celt_encoder_ctl(celt_enc, OPUS_SET_PHASE_INVERSION_DISABLED(value)); + } + break; + case OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + celt_encoder_ctl(celt_enc, OPUS_GET_PHASE_INVERSION_DISABLED(value)); + } + break; + case OPUS_RESET_STATE: + { + void *silk_enc; + silk_EncControlStruct dummy; + char *start; + silk_enc = (char*)st+st->silk_enc_offset; +#ifndef DISABLE_FLOAT_API + tonality_analysis_reset(&st->analysis); +#endif + + start = (char*)&st->OPUS_ENCODER_RESET_START; + OPUS_CLEAR(start, sizeof(OpusEncoder) - (start - (char*)st)); + + celt_encoder_ctl(celt_enc, OPUS_RESET_STATE); + silk_InitEncoder( silk_enc, st->arch, &dummy ); + st->stream_channels = st->channels; + st->hybrid_stereo_width_Q14 = 1 << 14; + st->prev_HB_gain = Q15ONE; + st->first = 1; + st->mode = MODE_HYBRID; + st->bandwidth = OPUS_BANDWIDTH_FULLBAND; + st->variable_HP_smth2_Q15 = silk_LSHIFT( silk_lin2log( VARIABLE_HP_MIN_CUTOFF_HZ ), 8 ); + } + break; + case OPUS_SET_FORCE_MODE_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if ((value < MODE_SILK_ONLY || value > MODE_CELT_ONLY) && value != OPUS_AUTO) + { + goto bad_arg; + } + st->user_forced_mode = value; + } + break; + case OPUS_SET_LFE_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->lfe = value; + ret = celt_encoder_ctl(celt_enc, OPUS_SET_LFE(value)); + } + break; + case OPUS_SET_ENERGY_MASK_REQUEST: + { + opus_val16 *value = va_arg(ap, opus_val16*); + st->energy_masking = value; + ret = celt_encoder_ctl(celt_enc, OPUS_SET_ENERGY_MASK(value)); + } + break; + case OPUS_GET_IN_DTX_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + if (st->silk_mode.useDTX && (st->prev_mode == MODE_SILK_ONLY || st->prev_mode == MODE_HYBRID)) { + /* DTX determined by Silk. */ + silk_encoder *silk_enc = (silk_encoder*)(void *)((char*)st+st->silk_enc_offset); + *value = silk_enc->state_Fxx[0].sCmn.noSpeechCounter >= NB_SPEECH_FRAMES_BEFORE_DTX; + /* Stereo: check second channel unless only the middle channel was encoded. */ + if(*value == 1 && st->silk_mode.nChannelsInternal == 2 && silk_enc->prev_decode_only_middle == 0) { + *value = silk_enc->state_Fxx[1].sCmn.noSpeechCounter >= NB_SPEECH_FRAMES_BEFORE_DTX; + } + } +#ifndef DISABLE_FLOAT_API + else if (st->use_dtx) { + /* DTX determined by Opus. */ + *value = st->nb_no_activity_ms_Q1 >= NB_SPEECH_FRAMES_BEFORE_DTX*20*2; + } +#endif + else { + *value = 0; + } + } + break; + case CELT_GET_MODE_REQUEST: + { + const CELTMode ** value = va_arg(ap, const CELTMode**); + if (!value) + { + goto bad_arg; + } + ret = celt_encoder_ctl(celt_enc, CELT_GET_MODE(value)); + } + break; + default: + /* fprintf(stderr, "unknown opus_encoder_ctl() request: %d", request);*/ + ret = OPUS_UNIMPLEMENTED; + break; + } + va_end(ap); + return ret; +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +} + +void opus_encoder_destroy(OpusEncoder *st) +{ + opus_free(st); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream.c new file mode 100644 index 000000000..09c3639b7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream.c @@ -0,0 +1,92 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_multistream.h" +#include "opus.h" +#include "opus_private.h" +#include "stack_alloc.h" +#include +#include "float_cast.h" +#include "os_support.h" + + +int validate_layout(const ChannelLayout *layout) +{ + int i, max_channel; + + max_channel = layout->nb_streams+layout->nb_coupled_streams; + if (max_channel>255) + return 0; + for (i=0;inb_channels;i++) + { + if (layout->mapping[i] >= max_channel && layout->mapping[i] != 255) + return 0; + } + return 1; +} + + +int get_left_channel(const ChannelLayout *layout, int stream_id, int prev) +{ + int i; + i = (prev<0) ? 0 : prev+1; + for (;inb_channels;i++) + { + if (layout->mapping[i]==stream_id*2) + return i; + } + return -1; +} + +int get_right_channel(const ChannelLayout *layout, int stream_id, int prev) +{ + int i; + i = (prev<0) ? 0 : prev+1; + for (;inb_channels;i++) + { + if (layout->mapping[i]==stream_id*2+1) + return i; + } + return -1; +} + +int get_mono_channel(const ChannelLayout *layout, int stream_id, int prev) +{ + int i; + i = (prev<0) ? 0 : prev+1; + for (;inb_channels;i++) + { + if (layout->mapping[i]==stream_id+layout->nb_coupled_streams) + return i; + } + return -1; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_decoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_decoder.c new file mode 100644 index 000000000..a2837c354 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_decoder.c @@ -0,0 +1,552 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_multistream.h" +#include "opus.h" +#include "opus_private.h" +#include "stack_alloc.h" +#include +#include "float_cast.h" +#include "os_support.h" + +/* DECODER */ + +#if defined(ENABLE_HARDENING) || defined(ENABLE_ASSERTIONS) +static void validate_ms_decoder(OpusMSDecoder *st) +{ + validate_layout(&st->layout); +} +#define VALIDATE_MS_DECODER(st) validate_ms_decoder(st) +#else +#define VALIDATE_MS_DECODER(st) +#endif + + +opus_int32 opus_multistream_decoder_get_size(int nb_streams, int nb_coupled_streams) +{ + int coupled_size; + int mono_size; + + if(nb_streams<1||nb_coupled_streams>nb_streams||nb_coupled_streams<0)return 0; + coupled_size = opus_decoder_get_size(2); + mono_size = opus_decoder_get_size(1); + return align(sizeof(OpusMSDecoder)) + + nb_coupled_streams * align(coupled_size) + + (nb_streams-nb_coupled_streams) * align(mono_size); +} + +int opus_multistream_decoder_init( + OpusMSDecoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping +) +{ + int coupled_size; + int mono_size; + int i, ret; + char *ptr; + + if ((channels>255) || (channels<1) || (coupled_streams>streams) || + (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams)) + return OPUS_BAD_ARG; + + st->layout.nb_channels = channels; + st->layout.nb_streams = streams; + st->layout.nb_coupled_streams = coupled_streams; + + for (i=0;ilayout.nb_channels;i++) + st->layout.mapping[i] = mapping[i]; + if (!validate_layout(&st->layout)) + return OPUS_BAD_ARG; + + ptr = (char*)st + align(sizeof(OpusMSDecoder)); + coupled_size = opus_decoder_get_size(2); + mono_size = opus_decoder_get_size(1); + + for (i=0;ilayout.nb_coupled_streams;i++) + { + ret=opus_decoder_init((OpusDecoder*)ptr, Fs, 2); + if(ret!=OPUS_OK)return ret; + ptr += align(coupled_size); + } + for (;ilayout.nb_streams;i++) + { + ret=opus_decoder_init((OpusDecoder*)ptr, Fs, 1); + if(ret!=OPUS_OK)return ret; + ptr += align(mono_size); + } + return OPUS_OK; +} + + +OpusMSDecoder *opus_multistream_decoder_create( + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int *error +) +{ + int ret; + OpusMSDecoder *st; + if ((channels>255) || (channels<1) || (coupled_streams>streams) || + (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams)) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + st = (OpusMSDecoder *)opus_alloc(opus_multistream_decoder_get_size(streams, coupled_streams)); + if (st==NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + ret = opus_multistream_decoder_init(st, Fs, channels, streams, coupled_streams, mapping); + if (error) + *error = ret; + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + return st; +} + +static int opus_multistream_packet_validate(const unsigned char *data, + opus_int32 len, int nb_streams, opus_int32 Fs) +{ + int s; + int count; + unsigned char toc; + opus_int16 size[48]; + int samples=0; + opus_int32 packet_offset; + + for (s=0;slayout.nb_streams-1) + { + RESTORE_STACK; + return OPUS_INVALID_PACKET; + } + if (!do_plc) + { + int ret = opus_multistream_packet_validate(data, len, st->layout.nb_streams, Fs); + if (ret < 0) + { + RESTORE_STACK; + return ret; + } else if (ret > frame_size) + { + RESTORE_STACK; + return OPUS_BUFFER_TOO_SMALL; + } + } + for (s=0;slayout.nb_streams;s++) + { + OpusDecoder *dec; + opus_int32 packet_offset; + int ret; + + dec = (OpusDecoder*)ptr; + ptr += (s < st->layout.nb_coupled_streams) ? align(coupled_size) : align(mono_size); + + if (!do_plc && len<=0) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + packet_offset = 0; + ret = opus_decode_native(dec, data, len, buf, frame_size, decode_fec, s!=st->layout.nb_streams-1, &packet_offset, soft_clip); + if (!do_plc) + { + data += packet_offset; + len -= packet_offset; + } + if (ret <= 0) + { + RESTORE_STACK; + return ret; + } + frame_size = ret; + if (s < st->layout.nb_coupled_streams) + { + int chan, prev; + prev = -1; + /* Copy "left" audio to the channel(s) where it belongs */ + while ( (chan = get_left_channel(&st->layout, s, prev)) != -1) + { + (*copy_channel_out)(pcm, st->layout.nb_channels, chan, + buf, 2, frame_size, user_data); + prev = chan; + } + prev = -1; + /* Copy "right" audio to the channel(s) where it belongs */ + while ( (chan = get_right_channel(&st->layout, s, prev)) != -1) + { + (*copy_channel_out)(pcm, st->layout.nb_channels, chan, + buf+1, 2, frame_size, user_data); + prev = chan; + } + } else { + int chan, prev; + prev = -1; + /* Copy audio to the channel(s) where it belongs */ + while ( (chan = get_mono_channel(&st->layout, s, prev)) != -1) + { + (*copy_channel_out)(pcm, st->layout.nb_channels, chan, + buf, 1, frame_size, user_data); + prev = chan; + } + } + } + /* Handle muted channels */ + for (c=0;clayout.nb_channels;c++) + { + if (st->layout.mapping[c] == 255) + { + (*copy_channel_out)(pcm, st->layout.nb_channels, c, + NULL, 0, frame_size, user_data); + } + } + RESTORE_STACK; + return frame_size; +} + +#if !defined(DISABLE_FLOAT_API) +static void opus_copy_channel_out_float( + void *dst, + int dst_stride, + int dst_channel, + const opus_val16 *src, + int src_stride, + int frame_size, + void *user_data +) +{ + float *float_dst; + opus_int32 i; + (void)user_data; + float_dst = (float*)dst; + if (src != NULL) + { + for (i=0;ilayout.nb_streams;s++) + { + OpusDecoder *dec; + dec = (OpusDecoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_decoder_ctl(dec, request, &tmp); + if (ret != OPUS_OK) break; + *value ^= tmp; + } + } + break; + case OPUS_RESET_STATE: + { + int s; + for (s=0;slayout.nb_streams;s++) + { + OpusDecoder *dec; + + dec = (OpusDecoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_decoder_ctl(dec, OPUS_RESET_STATE); + if (ret != OPUS_OK) + break; + } + } + break; + case OPUS_MULTISTREAM_GET_DECODER_STATE_REQUEST: + { + int s; + opus_int32 stream_id; + OpusDecoder **value; + stream_id = va_arg(ap, opus_int32); + if (stream_id<0 || stream_id >= st->layout.nb_streams) + goto bad_arg; + value = va_arg(ap, OpusDecoder**); + if (!value) + { + goto bad_arg; + } + for (s=0;slayout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + } + *value = (OpusDecoder*)ptr; + } + break; + case OPUS_SET_GAIN_REQUEST: + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + int s; + /* This works for int32 params */ + opus_int32 value = va_arg(ap, opus_int32); + for (s=0;slayout.nb_streams;s++) + { + OpusDecoder *dec; + + dec = (OpusDecoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_decoder_ctl(dec, request, value); + if (ret != OPUS_OK) + break; + } + } + break; + default: + ret = OPUS_UNIMPLEMENTED; + break; + } + return ret; +bad_arg: + return OPUS_BAD_ARG; +} + +int opus_multistream_decoder_ctl(OpusMSDecoder *st, int request, ...) +{ + int ret; + va_list ap; + va_start(ap, request); + ret = opus_multistream_decoder_ctl_va_list(st, request, ap); + va_end(ap); + return ret; +} + +void opus_multistream_decoder_destroy(OpusMSDecoder *st) +{ + opus_free(st); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_encoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_encoder.c new file mode 100644 index 000000000..93204a14c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_multistream_encoder.c @@ -0,0 +1,1328 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus_multistream.h" +#include "opus.h" +#include "opus_private.h" +#include "stack_alloc.h" +#include +#include "float_cast.h" +#include "os_support.h" +#include "mathops.h" +#include "mdct.h" +#include "modes.h" +#include "bands.h" +#include "quant_bands.h" +#include "pitch.h" + +typedef struct { + int nb_streams; + int nb_coupled_streams; + unsigned char mapping[8]; +} VorbisLayout; + +/* Index is nb_channel-1*/ +static const VorbisLayout vorbis_mappings[8] = { + {1, 0, {0}}, /* 1: mono */ + {1, 1, {0, 1}}, /* 2: stereo */ + {2, 1, {0, 2, 1}}, /* 3: 1-d surround */ + {2, 2, {0, 1, 2, 3}}, /* 4: quadraphonic surround */ + {3, 2, {0, 4, 1, 2, 3}}, /* 5: 5-channel surround */ + {4, 2, {0, 4, 1, 2, 3, 5}}, /* 6: 5.1 surround */ + {4, 3, {0, 4, 1, 2, 3, 5, 6}}, /* 7: 6.1 surround */ + {5, 3, {0, 6, 1, 2, 3, 4, 5, 7}}, /* 8: 7.1 surround */ +}; + +static opus_val32 *ms_get_preemph_mem(OpusMSEncoder *st) +{ + int s; + char *ptr; + int coupled_size, mono_size; + + coupled_size = opus_encoder_get_size(2); + mono_size = opus_encoder_get_size(1); + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + for (s=0;slayout.nb_streams;s++) + { + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + } + /* void* cast avoids clang -Wcast-align warning */ + return (opus_val32*)(void*)(ptr+st->layout.nb_channels*120*sizeof(opus_val32)); +} + +static opus_val32 *ms_get_window_mem(OpusMSEncoder *st) +{ + int s; + char *ptr; + int coupled_size, mono_size; + + coupled_size = opus_encoder_get_size(2); + mono_size = opus_encoder_get_size(1); + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + for (s=0;slayout.nb_streams;s++) + { + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + } + /* void* cast avoids clang -Wcast-align warning */ + return (opus_val32*)(void*)ptr; +} + +static int validate_ambisonics(int nb_channels, int *nb_streams, int *nb_coupled_streams) +{ + int order_plus_one; + int acn_channels; + int nondiegetic_channels; + + if (nb_channels < 1 || nb_channels > 227) + return 0; + + order_plus_one = isqrt32(nb_channels); + acn_channels = order_plus_one * order_plus_one; + nondiegetic_channels = nb_channels - acn_channels; + + if (nondiegetic_channels != 0 && nondiegetic_channels != 2) + return 0; + + if (nb_streams) + *nb_streams = acn_channels + (nondiegetic_channels != 0); + if (nb_coupled_streams) + *nb_coupled_streams = nondiegetic_channels != 0; + return 1; +} + +static int validate_encoder_layout(const ChannelLayout *layout) +{ + int s; + for (s=0;snb_streams;s++) + { + if (s < layout->nb_coupled_streams) + { + if (get_left_channel(layout, s, -1)==-1) + return 0; + if (get_right_channel(layout, s, -1)==-1) + return 0; + } else { + if (get_mono_channel(layout, s, -1)==-1) + return 0; + } + } + return 1; +} + +static void channel_pos(int channels, int pos[8]) +{ + /* Position in the mix: 0 don't mix, 1: left, 2: center, 3:right */ + if (channels==4) + { + pos[0]=1; + pos[1]=3; + pos[2]=1; + pos[3]=3; + } else if (channels==3||channels==5||channels==6) + { + pos[0]=1; + pos[1]=2; + pos[2]=3; + pos[3]=1; + pos[4]=3; + pos[5]=0; + } else if (channels==7) + { + pos[0]=1; + pos[1]=2; + pos[2]=3; + pos[3]=1; + pos[4]=3; + pos[5]=2; + pos[6]=0; + } else if (channels==8) + { + pos[0]=1; + pos[1]=2; + pos[2]=3; + pos[3]=1; + pos[4]=3; + pos[5]=1; + pos[6]=3; + pos[7]=0; + } +} + +#if 1 +/* Computes a rough approximation of log2(2^a + 2^b) */ +static opus_val16 logSum(opus_val16 a, opus_val16 b) +{ + opus_val16 max; + opus_val32 diff; + opus_val16 frac; + static const opus_val16 diff_table[17] = { + QCONST16(0.5000000f, DB_SHIFT), QCONST16(0.2924813f, DB_SHIFT), QCONST16(0.1609640f, DB_SHIFT), QCONST16(0.0849625f, DB_SHIFT), + QCONST16(0.0437314f, DB_SHIFT), QCONST16(0.0221971f, DB_SHIFT), QCONST16(0.0111839f, DB_SHIFT), QCONST16(0.0056136f, DB_SHIFT), + QCONST16(0.0028123f, DB_SHIFT) + }; + int low; + if (a>b) + { + max = a; + diff = SUB32(EXTEND32(a),EXTEND32(b)); + } else { + max = b; + diff = SUB32(EXTEND32(b),EXTEND32(a)); + } + if (!(diff < QCONST16(8.f, DB_SHIFT))) /* inverted to catch NaNs */ + return max; +#ifdef FIXED_POINT + low = SHR32(diff, DB_SHIFT-1); + frac = SHL16(diff - SHL16(low, DB_SHIFT-1), 16-DB_SHIFT); +#else + low = (int)floor(2*diff); + frac = 2*diff - low; +#endif + return max + diff_table[low] + MULT16_16_Q15(frac, SUB16(diff_table[low+1], diff_table[low])); +} +#else +opus_val16 logSum(opus_val16 a, opus_val16 b) +{ + return log2(pow(4, a)+ pow(4, b))/2; +} +#endif + +void surround_analysis(const CELTMode *celt_mode, const void *pcm, opus_val16 *bandLogE, opus_val32 *mem, opus_val32 *preemph_mem, + int len, int overlap, int channels, int rate, opus_copy_channel_in_func copy_channel_in, int arch +) +{ + int c; + int i; + int LM; + int pos[8] = {0}; + int upsample; + int frame_size; + int freq_size; + opus_val16 channel_offset; + opus_val32 bandE[21]; + opus_val16 maskLogE[3][21]; + VARDECL(opus_val32, in); + VARDECL(opus_val16, x); + VARDECL(opus_val32, freq); + SAVE_STACK; + + upsample = resampling_factor(rate); + frame_size = len*upsample; + freq_size = IMIN(960, frame_size); + + /* LM = log2(frame_size / 120) */ + for (LM=0;LMmaxLM;LM++) + if (celt_mode->shortMdctSize<preemph, preemph_mem+c, 0); +#ifndef FIXED_POINT + { + opus_val32 sum; + sum = celt_inner_prod(in, in, frame_size+overlap, 0); + /* This should filter out both NaNs and ridiculous signals that could + cause NaNs further down. */ + if (!(sum < 1e18f) || celt_isnan(sum)) + { + OPUS_CLEAR(in, frame_size+overlap); + preemph_mem[c] = 0; + } + } +#endif + OPUS_CLEAR(bandE, 21); + for (frame=0;framemdct, in+960*frame, freq, celt_mode->window, + overlap, celt_mode->maxLM-LM, 1, arch); + if (upsample != 1) + { + int bound = freq_size/upsample; + for (i=0;i=0;i--) + bandLogE[21*c+i] = MAX16(bandLogE[21*c+i], bandLogE[21*c+i+1]-QCONST16(2.f, DB_SHIFT)); + if (pos[c]==1) + { + for (i=0;i<21;i++) + maskLogE[0][i] = logSum(maskLogE[0][i], bandLogE[21*c+i]); + } else if (pos[c]==3) + { + for (i=0;i<21;i++) + maskLogE[2][i] = logSum(maskLogE[2][i], bandLogE[21*c+i]); + } else if (pos[c]==2) + { + for (i=0;i<21;i++) + { + maskLogE[0][i] = logSum(maskLogE[0][i], bandLogE[21*c+i]-QCONST16(.5f, DB_SHIFT)); + maskLogE[2][i] = logSum(maskLogE[2][i], bandLogE[21*c+i]-QCONST16(.5f, DB_SHIFT)); + } + } +#if 0 + for (i=0;i<21;i++) + printf("%f ", bandLogE[21*c+i]); + float sum=0; + for (i=0;i<21;i++) + sum += bandLogE[21*c+i]; + printf("%f ", sum/21); +#endif + OPUS_COPY(mem+c*overlap, in+frame_size, overlap); + } + for (i=0;i<21;i++) + maskLogE[1][i] = MIN32(maskLogE[0][i],maskLogE[2][i]); + channel_offset = HALF16(celt_log2(QCONST32(2.f,14)/(channels-1))); + for (c=0;c<3;c++) + for (i=0;i<21;i++) + maskLogE[c][i] += channel_offset; +#if 0 + for (c=0;c<3;c++) + { + for (i=0;i<21;i++) + printf("%f ", maskLogE[c][i]); + } +#endif + for (c=0;cnb_streams||nb_coupled_streams<0)return 0; + coupled_size = opus_encoder_get_size(2); + mono_size = opus_encoder_get_size(1); + return align(sizeof(OpusMSEncoder)) + + nb_coupled_streams * align(coupled_size) + + (nb_streams-nb_coupled_streams) * align(mono_size); +} + +opus_int32 opus_multistream_surround_encoder_get_size(int channels, int mapping_family) +{ + int nb_streams; + int nb_coupled_streams; + opus_int32 size; + + if (mapping_family==0) + { + if (channels==1) + { + nb_streams=1; + nb_coupled_streams=0; + } else if (channels==2) + { + nb_streams=1; + nb_coupled_streams=1; + } else + return 0; + } else if (mapping_family==1 && channels<=8 && channels>=1) + { + nb_streams=vorbis_mappings[channels-1].nb_streams; + nb_coupled_streams=vorbis_mappings[channels-1].nb_coupled_streams; + } else if (mapping_family==255) + { + nb_streams=channels; + nb_coupled_streams=0; + } else if (mapping_family==2) + { + if (!validate_ambisonics(channels, &nb_streams, &nb_coupled_streams)) + return 0; + } else + return 0; + size = opus_multistream_encoder_get_size(nb_streams, nb_coupled_streams); + if (channels>2) + { + size += channels*(120*sizeof(opus_val32) + sizeof(opus_val32)); + } + return size; +} + +static int opus_multistream_encoder_init_impl( + OpusMSEncoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int application, + MappingType mapping_type +) +{ + int coupled_size; + int mono_size; + int i, ret; + char *ptr; + + if ((channels>255) || (channels<1) || (coupled_streams>streams) || + (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams)) + return OPUS_BAD_ARG; + + st->arch = opus_select_arch(); + st->layout.nb_channels = channels; + st->layout.nb_streams = streams; + st->layout.nb_coupled_streams = coupled_streams; + if (mapping_type != MAPPING_TYPE_SURROUND) + st->lfe_stream = -1; + st->bitrate_bps = OPUS_AUTO; + st->application = application; + st->variable_duration = OPUS_FRAMESIZE_ARG; + for (i=0;ilayout.nb_channels;i++) + st->layout.mapping[i] = mapping[i]; + if (!validate_layout(&st->layout)) + return OPUS_BAD_ARG; + if (mapping_type == MAPPING_TYPE_SURROUND && + !validate_encoder_layout(&st->layout)) + return OPUS_BAD_ARG; + if (mapping_type == MAPPING_TYPE_AMBISONICS && + !validate_ambisonics(st->layout.nb_channels, NULL, NULL)) + return OPUS_BAD_ARG; + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + coupled_size = opus_encoder_get_size(2); + mono_size = opus_encoder_get_size(1); + + for (i=0;ilayout.nb_coupled_streams;i++) + { + ret = opus_encoder_init((OpusEncoder*)ptr, Fs, 2, application); + if(ret!=OPUS_OK)return ret; + if (i==st->lfe_stream) + opus_encoder_ctl((OpusEncoder*)ptr, OPUS_SET_LFE(1)); + ptr += align(coupled_size); + } + for (;ilayout.nb_streams;i++) + { + ret = opus_encoder_init((OpusEncoder*)ptr, Fs, 1, application); + if (i==st->lfe_stream) + opus_encoder_ctl((OpusEncoder*)ptr, OPUS_SET_LFE(1)); + if(ret!=OPUS_OK)return ret; + ptr += align(mono_size); + } + if (mapping_type == MAPPING_TYPE_SURROUND) + { + OPUS_CLEAR(ms_get_preemph_mem(st), channels); + OPUS_CLEAR(ms_get_window_mem(st), channels*120); + } + st->mapping_type = mapping_type; + return OPUS_OK; +} + +int opus_multistream_encoder_init( + OpusMSEncoder *st, + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int application +) +{ + return opus_multistream_encoder_init_impl(st, Fs, channels, streams, + coupled_streams, mapping, + application, MAPPING_TYPE_NONE); +} + +int opus_multistream_surround_encoder_init( + OpusMSEncoder *st, + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + unsigned char *mapping, + int application +) +{ + MappingType mapping_type; + + if ((channels>255) || (channels<1)) + return OPUS_BAD_ARG; + st->lfe_stream = -1; + if (mapping_family==0) + { + if (channels==1) + { + *streams=1; + *coupled_streams=0; + mapping[0]=0; + } else if (channels==2) + { + *streams=1; + *coupled_streams=1; + mapping[0]=0; + mapping[1]=1; + } else + return OPUS_UNIMPLEMENTED; + } else if (mapping_family==1 && channels<=8 && channels>=1) + { + int i; + *streams=vorbis_mappings[channels-1].nb_streams; + *coupled_streams=vorbis_mappings[channels-1].nb_coupled_streams; + for (i=0;i=6) + st->lfe_stream = *streams-1; + } else if (mapping_family==255) + { + int i; + *streams=channels; + *coupled_streams=0; + for(i=0;i2 && mapping_family==1) { + mapping_type = MAPPING_TYPE_SURROUND; + } else if (mapping_family==2) + { + mapping_type = MAPPING_TYPE_AMBISONICS; + } else + { + mapping_type = MAPPING_TYPE_NONE; + } + return opus_multistream_encoder_init_impl(st, Fs, channels, *streams, + *coupled_streams, mapping, + application, mapping_type); +} + +OpusMSEncoder *opus_multistream_encoder_create( + opus_int32 Fs, + int channels, + int streams, + int coupled_streams, + const unsigned char *mapping, + int application, + int *error +) +{ + int ret; + OpusMSEncoder *st; + if ((channels>255) || (channels<1) || (coupled_streams>streams) || + (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams)) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + st = (OpusMSEncoder *)opus_alloc(opus_multistream_encoder_get_size(streams, coupled_streams)); + if (st==NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + ret = opus_multistream_encoder_init(st, Fs, channels, streams, coupled_streams, mapping, application); + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} + +OpusMSEncoder *opus_multistream_surround_encoder_create( + opus_int32 Fs, + int channels, + int mapping_family, + int *streams, + int *coupled_streams, + unsigned char *mapping, + int application, + int *error +) +{ + int ret; + opus_int32 size; + OpusMSEncoder *st; + if ((channels>255) || (channels<1)) + { + if (error) + *error = OPUS_BAD_ARG; + return NULL; + } + size = opus_multistream_surround_encoder_get_size(channels, mapping_family); + if (!size) + { + if (error) + *error = OPUS_UNIMPLEMENTED; + return NULL; + } + st = (OpusMSEncoder *)opus_alloc(size); + if (st==NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + ret = opus_multistream_surround_encoder_init(st, Fs, channels, mapping_family, streams, coupled_streams, mapping, application); + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} + +static void surround_rate_allocation( + OpusMSEncoder *st, + opus_int32 *rate, + int frame_size, + opus_int32 Fs + ) +{ + int i; + opus_int32 channel_rate; + int stream_offset; + int lfe_offset; + int coupled_ratio; /* Q8 */ + int lfe_ratio; /* Q8 */ + int nb_lfe; + int nb_uncoupled; + int nb_coupled; + int nb_normal; + opus_int32 channel_offset; + opus_int32 bitrate; + int total; + + nb_lfe = (st->lfe_stream!=-1); + nb_coupled = st->layout.nb_coupled_streams; + nb_uncoupled = st->layout.nb_streams-nb_coupled-nb_lfe; + nb_normal = 2*nb_coupled + nb_uncoupled; + + /* Give each non-LFE channel enough bits per channel for coding band energy. */ + channel_offset = 40*IMAX(50, Fs/frame_size); + + if (st->bitrate_bps==OPUS_AUTO) + { + bitrate = nb_normal*(channel_offset + Fs + 10000) + 8000*nb_lfe; + } else if (st->bitrate_bps==OPUS_BITRATE_MAX) + { + bitrate = nb_normal*300000 + nb_lfe*128000; + } else { + bitrate = st->bitrate_bps; + } + + /* Give LFE some basic stream_channel allocation but never exceed 1/20 of the + total rate for the non-energy part to avoid problems at really low rate. */ + lfe_offset = IMIN(bitrate/20, 3000) + 15*IMAX(50, Fs/frame_size); + + /* We give each stream (coupled or uncoupled) a starting bitrate. + This models the main saving of coupled channels over uncoupled. */ + stream_offset = (bitrate - channel_offset*nb_normal - lfe_offset*nb_lfe)/nb_normal/2; + stream_offset = IMAX(0, IMIN(20000, stream_offset)); + + /* Coupled streams get twice the mono rate after the offset is allocated. */ + coupled_ratio = 512; + /* Should depend on the bitrate, for now we assume LFE gets 1/8 the bits of mono */ + lfe_ratio = 32; + + total = (nb_uncoupled<<8) /* mono */ + + coupled_ratio*nb_coupled /* stereo */ + + nb_lfe*lfe_ratio; + channel_rate = 256*(opus_int64)(bitrate - lfe_offset*nb_lfe - stream_offset*(nb_coupled+nb_uncoupled) - channel_offset*nb_normal)/total; + + for (i=0;ilayout.nb_streams;i++) + { + if (ilayout.nb_coupled_streams) + rate[i] = 2*channel_offset + IMAX(0, stream_offset+(channel_rate*coupled_ratio>>8)); + else if (i!=st->lfe_stream) + rate[i] = channel_offset + IMAX(0, stream_offset + channel_rate); + else + rate[i] = IMAX(0, lfe_offset+(channel_rate*lfe_ratio>>8)); + } +} + +static void ambisonics_rate_allocation( + OpusMSEncoder *st, + opus_int32 *rate, + int frame_size, + opus_int32 Fs + ) +{ + int i; + opus_int32 total_rate; + opus_int32 per_stream_rate; + + const int nb_channels = st->layout.nb_streams + st->layout.nb_coupled_streams; + + if (st->bitrate_bps==OPUS_AUTO) + { + total_rate = (st->layout.nb_coupled_streams + st->layout.nb_streams) * + (Fs+60*Fs/frame_size) + st->layout.nb_streams * (opus_int32)15000; + } else if (st->bitrate_bps==OPUS_BITRATE_MAX) + { + total_rate = nb_channels * 320000; + } else + { + total_rate = st->bitrate_bps; + } + + /* Allocate equal number of bits to Ambisonic (uncoupled) and non-diegetic + * (coupled) streams */ + per_stream_rate = total_rate / st->layout.nb_streams; + for (i = 0; i < st->layout.nb_streams; i++) + { + rate[i] = per_stream_rate; + } +} + +static opus_int32 rate_allocation( + OpusMSEncoder *st, + opus_int32 *rate, + int frame_size + ) +{ + int i; + opus_int32 rate_sum=0; + opus_int32 Fs; + char *ptr; + + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + opus_encoder_ctl((OpusEncoder*)ptr, OPUS_GET_SAMPLE_RATE(&Fs)); + + if (st->mapping_type == MAPPING_TYPE_AMBISONICS) { + ambisonics_rate_allocation(st, rate, frame_size, Fs); + } else + { + surround_rate_allocation(st, rate, frame_size, Fs); + } + + for (i=0;ilayout.nb_streams;i++) + { + rate[i] = IMAX(rate[i], 500); + rate_sum += rate[i]; + } + return rate_sum; +} + +/* Max size in case the encoder decides to return six frames (6 x 20 ms = 120 ms) */ +#define MS_FRAME_TMP (6*1275+12) +int opus_multistream_encode_native +( + OpusMSEncoder *st, + opus_copy_channel_in_func copy_channel_in, + const void *pcm, + int analysis_frame_size, + unsigned char *data, + opus_int32 max_data_bytes, + int lsb_depth, + downmix_func downmix, + int float_api, + void *user_data +) +{ + opus_int32 Fs; + int coupled_size; + int mono_size; + int s; + char *ptr; + int tot_size; + VARDECL(opus_val16, buf); + VARDECL(opus_val16, bandSMR); + unsigned char tmp_data[MS_FRAME_TMP]; + OpusRepacketizer rp; + opus_int32 vbr; + const CELTMode *celt_mode; + opus_int32 bitrates[256]; + opus_val16 bandLogE[42]; + opus_val32 *mem = NULL; + opus_val32 *preemph_mem=NULL; + int frame_size; + opus_int32 rate_sum; + opus_int32 smallest_packet; + ALLOC_STACK; + + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + preemph_mem = ms_get_preemph_mem(st); + mem = ms_get_window_mem(st); + } + + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + opus_encoder_ctl((OpusEncoder*)ptr, OPUS_GET_SAMPLE_RATE(&Fs)); + opus_encoder_ctl((OpusEncoder*)ptr, OPUS_GET_VBR(&vbr)); + opus_encoder_ctl((OpusEncoder*)ptr, CELT_GET_MODE(&celt_mode)); + + frame_size = frame_size_select(analysis_frame_size, st->variable_duration, Fs); + if (frame_size <= 0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + /* Smallest packet the encoder can produce. */ + smallest_packet = st->layout.nb_streams*2-1; + /* 100 ms needs an extra byte per stream for the ToC. */ + if (Fs/frame_size == 10) + smallest_packet += st->layout.nb_streams; + if (max_data_bytes < smallest_packet) + { + RESTORE_STACK; + return OPUS_BUFFER_TOO_SMALL; + } + ALLOC(buf, 2*frame_size, opus_val16); + coupled_size = opus_encoder_get_size(2); + mono_size = opus_encoder_get_size(1); + + ALLOC(bandSMR, 21*st->layout.nb_channels, opus_val16); + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + surround_analysis(celt_mode, pcm, bandSMR, mem, preemph_mem, frame_size, 120, st->layout.nb_channels, Fs, copy_channel_in, st->arch); + } + + /* Compute bitrate allocation between streams (this could be a lot better) */ + rate_sum = rate_allocation(st, bitrates, frame_size); + + if (!vbr) + { + if (st->bitrate_bps == OPUS_AUTO) + { + max_data_bytes = IMIN(max_data_bytes, 3*rate_sum/(3*8*Fs/frame_size)); + } else if (st->bitrate_bps != OPUS_BITRATE_MAX) + { + max_data_bytes = IMIN(max_data_bytes, IMAX(smallest_packet, + 3*st->bitrate_bps/(3*8*Fs/frame_size))); + } + } + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + for (s=0;slayout.nb_streams;s++) + { + OpusEncoder *enc; + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrates[s])); + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + opus_int32 equiv_rate; + equiv_rate = st->bitrate_bps; + if (frame_size*50 < Fs) + equiv_rate -= 60*(Fs/frame_size - 50)*st->layout.nb_channels; + if (equiv_rate > 10000*st->layout.nb_channels) + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + else if (equiv_rate > 7000*st->layout.nb_channels) + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_SUPERWIDEBAND)); + else if (equiv_rate > 5000*st->layout.nb_channels) + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND)); + else + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + if (s < st->layout.nb_coupled_streams) + { + /* To preserve the spatial image, force stereo CELT on coupled streams */ + opus_encoder_ctl(enc, OPUS_SET_FORCE_MODE(MODE_CELT_ONLY)); + opus_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(2)); + } + } + else if (st->mapping_type == MAPPING_TYPE_AMBISONICS) { + opus_encoder_ctl(enc, OPUS_SET_FORCE_MODE(MODE_CELT_ONLY)); + } + } + + ptr = (char*)st + align(sizeof(OpusMSEncoder)); + /* Counting ToC */ + tot_size = 0; + for (s=0;slayout.nb_streams;s++) + { + OpusEncoder *enc; + int len; + int curr_max; + int c1, c2; + int ret; + + opus_repacketizer_init(&rp); + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + { + int i; + int left, right; + left = get_left_channel(&st->layout, s, -1); + right = get_right_channel(&st->layout, s, -1); + (*copy_channel_in)(buf, 2, + pcm, st->layout.nb_channels, left, frame_size, user_data); + (*copy_channel_in)(buf+1, 2, + pcm, st->layout.nb_channels, right, frame_size, user_data); + ptr += align(coupled_size); + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + for (i=0;i<21;i++) + { + bandLogE[i] = bandSMR[21*left+i]; + bandLogE[21+i] = bandSMR[21*right+i]; + } + } + c1 = left; + c2 = right; + } else { + int i; + int chan = get_mono_channel(&st->layout, s, -1); + (*copy_channel_in)(buf, 1, + pcm, st->layout.nb_channels, chan, frame_size, user_data); + ptr += align(mono_size); + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + for (i=0;i<21;i++) + bandLogE[i] = bandSMR[21*chan+i]; + } + c1 = chan; + c2 = -1; + } + if (st->mapping_type == MAPPING_TYPE_SURROUND) + opus_encoder_ctl(enc, OPUS_SET_ENERGY_MASK(bandLogE)); + /* number of bytes left (+Toc) */ + curr_max = max_data_bytes - tot_size; + /* Reserve one byte for the last stream and two for the others */ + curr_max -= IMAX(0,2*(st->layout.nb_streams-s-1)-1); + /* For 100 ms, reserve an extra byte per stream for the ToC */ + if (Fs/frame_size == 10) + curr_max -= st->layout.nb_streams-s-1; + curr_max = IMIN(curr_max,MS_FRAME_TMP); + /* Repacketizer will add one or two bytes for self-delimited frames */ + if (s != st->layout.nb_streams-1) curr_max -= curr_max>253 ? 2 : 1; + if (!vbr && s == st->layout.nb_streams-1) + opus_encoder_ctl(enc, OPUS_SET_BITRATE(curr_max*(8*Fs/frame_size))); + len = opus_encode_native(enc, buf, frame_size, tmp_data, curr_max, lsb_depth, + pcm, analysis_frame_size, c1, c2, st->layout.nb_channels, downmix, float_api); + if (len<0) + { + RESTORE_STACK; + return len; + } + /* We need to use the repacketizer to add the self-delimiting lengths + while taking into account the fact that the encoder can now return + more than one frame at a time (e.g. 60 ms CELT-only) */ + ret = opus_repacketizer_cat(&rp, tmp_data, len); + /* If the opus_repacketizer_cat() fails, then something's seriously wrong + with the encoder. */ + if (ret != OPUS_OK) + { + RESTORE_STACK; + return OPUS_INTERNAL_ERROR; + } + len = opus_repacketizer_out_range_impl(&rp, 0, opus_repacketizer_get_nb_frames(&rp), + data, max_data_bytes-tot_size, s != st->layout.nb_streams-1, !vbr && s == st->layout.nb_streams-1); + data += len; + tot_size += len; + } + /*printf("\n");*/ + RESTORE_STACK; + return tot_size; +} + +#if !defined(DISABLE_FLOAT_API) +static void opus_copy_channel_in_float( + opus_val16 *dst, + int dst_stride, + const void *src, + int src_stride, + int src_channel, + int frame_size, + void *user_data +) +{ + const float *float_src; + opus_int32 i; + (void)user_data; + float_src = (const float *)src; + for (i=0;ilayout.nb_channels, IMAX(500*st->layout.nb_channels, value)); + } + st->bitrate_bps = value; + } + break; + case OPUS_GET_BITRATE_REQUEST: + { + int s; + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = 0; + for (s=0;slayout.nb_streams;s++) + { + opus_int32 rate; + OpusEncoder *enc; + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + opus_encoder_ctl(enc, request, &rate); + *value += rate; + } + } + break; + case OPUS_GET_LSB_DEPTH_REQUEST: + case OPUS_GET_VBR_REQUEST: + case OPUS_GET_APPLICATION_REQUEST: + case OPUS_GET_BANDWIDTH_REQUEST: + case OPUS_GET_COMPLEXITY_REQUEST: + case OPUS_GET_PACKET_LOSS_PERC_REQUEST: + case OPUS_GET_DTX_REQUEST: + case OPUS_GET_VOICE_RATIO_REQUEST: + case OPUS_GET_VBR_CONSTRAINT_REQUEST: + case OPUS_GET_SIGNAL_REQUEST: + case OPUS_GET_LOOKAHEAD_REQUEST: + case OPUS_GET_SAMPLE_RATE_REQUEST: + case OPUS_GET_INBAND_FEC_REQUEST: + case OPUS_GET_FORCE_CHANNELS_REQUEST: + case OPUS_GET_PREDICTION_DISABLED_REQUEST: + case OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: + { + OpusEncoder *enc; + /* For int32* GET params, just query the first stream */ + opus_int32 *value = va_arg(ap, opus_int32*); + enc = (OpusEncoder*)ptr; + ret = opus_encoder_ctl(enc, request, value); + } + break; + case OPUS_GET_FINAL_RANGE_REQUEST: + { + int s; + opus_uint32 *value = va_arg(ap, opus_uint32*); + opus_uint32 tmp; + if (!value) + { + goto bad_arg; + } + *value=0; + for (s=0;slayout.nb_streams;s++) + { + OpusEncoder *enc; + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_encoder_ctl(enc, request, &tmp); + if (ret != OPUS_OK) break; + *value ^= tmp; + } + } + break; + case OPUS_SET_LSB_DEPTH_REQUEST: + case OPUS_SET_COMPLEXITY_REQUEST: + case OPUS_SET_VBR_REQUEST: + case OPUS_SET_VBR_CONSTRAINT_REQUEST: + case OPUS_SET_MAX_BANDWIDTH_REQUEST: + case OPUS_SET_BANDWIDTH_REQUEST: + case OPUS_SET_SIGNAL_REQUEST: + case OPUS_SET_APPLICATION_REQUEST: + case OPUS_SET_INBAND_FEC_REQUEST: + case OPUS_SET_PACKET_LOSS_PERC_REQUEST: + case OPUS_SET_DTX_REQUEST: + case OPUS_SET_FORCE_MODE_REQUEST: + case OPUS_SET_FORCE_CHANNELS_REQUEST: + case OPUS_SET_PREDICTION_DISABLED_REQUEST: + case OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: + { + int s; + /* This works for int32 params */ + opus_int32 value = va_arg(ap, opus_int32); + for (s=0;slayout.nb_streams;s++) + { + OpusEncoder *enc; + + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_encoder_ctl(enc, request, value); + if (ret != OPUS_OK) + break; + } + } + break; + case OPUS_MULTISTREAM_GET_ENCODER_STATE_REQUEST: + { + int s; + opus_int32 stream_id; + OpusEncoder **value; + stream_id = va_arg(ap, opus_int32); + if (stream_id<0 || stream_id >= st->layout.nb_streams) + goto bad_arg; + value = va_arg(ap, OpusEncoder**); + if (!value) + { + goto bad_arg; + } + for (s=0;slayout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + } + *value = (OpusEncoder*)ptr; + } + break; + case OPUS_SET_EXPERT_FRAME_DURATION_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + st->variable_duration = value; + } + break; + case OPUS_GET_EXPERT_FRAME_DURATION_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->variable_duration; + } + break; + case OPUS_RESET_STATE: + { + int s; + if (st->mapping_type == MAPPING_TYPE_SURROUND) + { + OPUS_CLEAR(ms_get_preemph_mem(st), st->layout.nb_channels); + OPUS_CLEAR(ms_get_window_mem(st), st->layout.nb_channels*120); + } + for (s=0;slayout.nb_streams;s++) + { + OpusEncoder *enc; + enc = (OpusEncoder*)ptr; + if (s < st->layout.nb_coupled_streams) + ptr += align(coupled_size); + else + ptr += align(mono_size); + ret = opus_encoder_ctl(enc, OPUS_RESET_STATE); + if (ret != OPUS_OK) + break; + } + } + break; + default: + ret = OPUS_UNIMPLEMENTED; + break; + } + return ret; +bad_arg: + return OPUS_BAD_ARG; +} + +int opus_multistream_encoder_ctl(OpusMSEncoder *st, int request, ...) +{ + int ret; + va_list ap; + va_start(ap, request); + ret = opus_multistream_encoder_ctl_va_list(st, request, ap); + va_end(ap); + return ret; +} + +void opus_multistream_encoder_destroy(OpusMSEncoder *st) +{ + opus_free(st); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_private.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_private.h new file mode 100644 index 000000000..5e2463f54 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_private.h @@ -0,0 +1,201 @@ +/* Copyright (c) 2012 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#ifndef OPUS_PRIVATE_H +#define OPUS_PRIVATE_H + +#include "arch.h" +#include "opus.h" +#include "celt.h" + +#include /* va_list */ +#include /* offsetof */ + +struct OpusRepacketizer { + unsigned char toc; + int nb_frames; + const unsigned char *frames[48]; + opus_int16 len[48]; + int framesize; +}; + +typedef struct ChannelLayout { + int nb_channels; + int nb_streams; + int nb_coupled_streams; + unsigned char mapping[256]; +} ChannelLayout; + +typedef enum { + MAPPING_TYPE_NONE, + MAPPING_TYPE_SURROUND, + MAPPING_TYPE_AMBISONICS +} MappingType; + +struct OpusMSEncoder { + ChannelLayout layout; + int arch; + int lfe_stream; + int application; + int variable_duration; + MappingType mapping_type; + opus_int32 bitrate_bps; + /* Encoder states go here */ + /* then opus_val32 window_mem[channels*120]; */ + /* then opus_val32 preemph_mem[channels]; */ +}; + +struct OpusMSDecoder { + ChannelLayout layout; + /* Decoder states go here */ +}; + +int opus_multistream_encoder_ctl_va_list(struct OpusMSEncoder *st, int request, + va_list ap); +int opus_multistream_decoder_ctl_va_list(struct OpusMSDecoder *st, int request, + va_list ap); + +int validate_layout(const ChannelLayout *layout); +int get_left_channel(const ChannelLayout *layout, int stream_id, int prev); +int get_right_channel(const ChannelLayout *layout, int stream_id, int prev); +int get_mono_channel(const ChannelLayout *layout, int stream_id, int prev); + +typedef void (*opus_copy_channel_in_func)( + opus_val16 *dst, + int dst_stride, + const void *src, + int src_stride, + int src_channel, + int frame_size, + void *user_data +); + +typedef void (*opus_copy_channel_out_func)( + void *dst, + int dst_stride, + int dst_channel, + const opus_val16 *src, + int src_stride, + int frame_size, + void *user_data +); + +#define MODE_SILK_ONLY 1000 +#define MODE_HYBRID 1001 +#define MODE_CELT_ONLY 1002 + +#define OPUS_SET_VOICE_RATIO_REQUEST 11018 +#define OPUS_GET_VOICE_RATIO_REQUEST 11019 + +/** Configures the encoder's expected percentage of voice + * opposed to music or other signals. + * + * @note This interface is currently more aspiration than actuality. It's + * ultimately expected to bias an automatic signal classifier, but it currently + * just shifts the static bitrate to mode mapping around a little bit. + * + * @param[in] x int: Voice percentage in the range 0-100, inclusive. + * @hideinitializer */ +#define OPUS_SET_VOICE_RATIO(x) OPUS_SET_VOICE_RATIO_REQUEST, __opus_check_int(x) +/** Gets the encoder's configured voice ratio value, @see OPUS_SET_VOICE_RATIO + * + * @param[out] x int*: Voice percentage in the range 0-100, inclusive. + * @hideinitializer */ +#define OPUS_GET_VOICE_RATIO(x) OPUS_GET_VOICE_RATIO_REQUEST, __opus_check_int_ptr(x) + + +#define OPUS_SET_FORCE_MODE_REQUEST 11002 +#define OPUS_SET_FORCE_MODE(x) OPUS_SET_FORCE_MODE_REQUEST, __opus_check_int(x) + +typedef void (*downmix_func)(const void *, opus_val32 *, int, int, int, int, int); +void downmix_float(const void *_x, opus_val32 *sub, int subframe, int offset, int c1, int c2, int C); +void downmix_int(const void *_x, opus_val32 *sub, int subframe, int offset, int c1, int c2, int C); +int is_digital_silence(const opus_val16* pcm, int frame_size, int channels, int lsb_depth); + +int encode_size(int size, unsigned char *data); + +opus_int32 frame_size_select(opus_int32 frame_size, int variable_duration, opus_int32 Fs); + +opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_size, + unsigned char *data, opus_int32 out_data_bytes, int lsb_depth, + const void *analysis_pcm, opus_int32 analysis_size, int c1, int c2, + int analysis_channels, downmix_func downmix, int float_api); + +int opus_decode_native(OpusDecoder *st, const unsigned char *data, opus_int32 len, + opus_val16 *pcm, int frame_size, int decode_fec, int self_delimited, + opus_int32 *packet_offset, int soft_clip); + +/* Make sure everything is properly aligned. */ +static OPUS_INLINE int align(int i) +{ + struct foo {char c; union { void* p; opus_int32 i; opus_val32 v; } u;}; + + unsigned int alignment = offsetof(struct foo, u); + + /* Optimizing compilers should optimize div and multiply into and + for all sensible alignment values. */ + return ((i + alignment - 1) / alignment) * alignment; +} + +int opus_packet_parse_impl(const unsigned char *data, opus_int32 len, + int self_delimited, unsigned char *out_toc, + const unsigned char *frames[48], opus_int16 size[48], + int *payload_offset, opus_int32 *packet_offset); + +opus_int32 opus_repacketizer_out_range_impl(OpusRepacketizer *rp, int begin, int end, + unsigned char *data, opus_int32 maxlen, int self_delimited, int pad); + +int pad_frame(unsigned char *data, opus_int32 len, opus_int32 new_len); + +int opus_multistream_encode_native +( + struct OpusMSEncoder *st, + opus_copy_channel_in_func copy_channel_in, + const void *pcm, + int analysis_frame_size, + unsigned char *data, + opus_int32 max_data_bytes, + int lsb_depth, + downmix_func downmix, + int float_api, + void *user_data +); + +int opus_multistream_decode_native( + struct OpusMSDecoder *st, + const unsigned char *data, + opus_int32 len, + void *pcm, + opus_copy_channel_out_func copy_channel_out, + int frame_size, + int decode_fec, + int soft_clip, + void *user_data +); + +#endif /* OPUS_PRIVATE_H */ diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_decoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_decoder.c new file mode 100644 index 000000000..c2e07d5bc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_decoder.c @@ -0,0 +1,258 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mathops.h" +#include "os_support.h" +#include "opus_private.h" +#include "opus_defines.h" +#include "opus_projection.h" +#include "opus_multistream.h" +#include "mapping_matrix.h" +#include "stack_alloc.h" + +struct OpusProjectionDecoder +{ + opus_int32 demixing_matrix_size_in_bytes; + /* Encoder states go here */ +}; + +#if !defined(DISABLE_FLOAT_API) +static void opus_projection_copy_channel_out_float( + void *dst, + int dst_stride, + int dst_channel, + const opus_val16 *src, + int src_stride, + int frame_size, + void *user_data) +{ + float *float_dst; + const MappingMatrix *matrix; + float_dst = (float *)dst; + matrix = (const MappingMatrix *)user_data; + + if (dst_channel == 0) + OPUS_CLEAR(float_dst, frame_size * dst_stride); + + if (src != NULL) + mapping_matrix_multiply_channel_out_float(matrix, src, dst_channel, + src_stride, float_dst, dst_stride, frame_size); +} +#endif + +static void opus_projection_copy_channel_out_short( + void *dst, + int dst_stride, + int dst_channel, + const opus_val16 *src, + int src_stride, + int frame_size, + void *user_data) +{ + opus_int16 *short_dst; + const MappingMatrix *matrix; + short_dst = (opus_int16 *)dst; + matrix = (const MappingMatrix *)user_data; + if (dst_channel == 0) + OPUS_CLEAR(short_dst, frame_size * dst_stride); + + if (src != NULL) + mapping_matrix_multiply_channel_out_short(matrix, src, dst_channel, + src_stride, short_dst, dst_stride, frame_size); +} + +static MappingMatrix *get_dec_demixing_matrix(OpusProjectionDecoder *st) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (MappingMatrix*)(void*)((char*)st + + align(sizeof(OpusProjectionDecoder))); +} + +static OpusMSDecoder *get_multistream_decoder(OpusProjectionDecoder *st) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (OpusMSDecoder*)(void*)((char*)st + + align(sizeof(OpusProjectionDecoder) + + st->demixing_matrix_size_in_bytes)); +} + +opus_int32 opus_projection_decoder_get_size(int channels, int streams, + int coupled_streams) +{ + opus_int32 matrix_size; + opus_int32 decoder_size; + + matrix_size = + mapping_matrix_get_size(streams + coupled_streams, channels); + if (!matrix_size) + return 0; + + decoder_size = opus_multistream_decoder_get_size(streams, coupled_streams); + if (!decoder_size) + return 0; + + return align(sizeof(OpusProjectionDecoder)) + matrix_size + decoder_size; +} + +int opus_projection_decoder_init(OpusProjectionDecoder *st, opus_int32 Fs, + int channels, int streams, int coupled_streams, + unsigned char *demixing_matrix, opus_int32 demixing_matrix_size) +{ + int nb_input_streams; + opus_int32 expected_matrix_size; + int i, ret; + unsigned char mapping[255]; + VARDECL(opus_int16, buf); + ALLOC_STACK; + + /* Verify supplied matrix size. */ + nb_input_streams = streams + coupled_streams; + expected_matrix_size = nb_input_streams * channels * sizeof(opus_int16); + if (expected_matrix_size != demixing_matrix_size) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + /* Convert demixing matrix input into internal format. */ + ALLOC(buf, nb_input_streams * channels, opus_int16); + for (i = 0; i < nb_input_streams * channels; i++) + { + int s = demixing_matrix[2*i + 1] << 8 | demixing_matrix[2*i]; + s = ((s & 0xFFFF) ^ 0x8000) - 0x8000; + buf[i] = (opus_int16)s; + } + + /* Assign demixing matrix. */ + st->demixing_matrix_size_in_bytes = + mapping_matrix_get_size(channels, nb_input_streams); + if (!st->demixing_matrix_size_in_bytes) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + mapping_matrix_init(get_dec_demixing_matrix(st), channels, nb_input_streams, 0, + buf, demixing_matrix_size); + + /* Set trivial mapping so each input channel pairs with a matrix column. */ + for (i = 0; i < channels; i++) + mapping[i] = i; + + ret = opus_multistream_decoder_init( + get_multistream_decoder(st), Fs, channels, streams, coupled_streams, mapping); + RESTORE_STACK; + return ret; +} + +OpusProjectionDecoder *opus_projection_decoder_create( + opus_int32 Fs, int channels, int streams, int coupled_streams, + unsigned char *demixing_matrix, opus_int32 demixing_matrix_size, int *error) +{ + int size; + int ret; + OpusProjectionDecoder *st; + + /* Allocate space for the projection decoder. */ + size = opus_projection_decoder_get_size(channels, streams, coupled_streams); + if (!size) { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + st = (OpusProjectionDecoder *)opus_alloc(size); + if (!st) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + + /* Initialize projection decoder with provided settings. */ + ret = opus_projection_decoder_init(st, Fs, channels, streams, coupled_streams, + demixing_matrix, demixing_matrix_size); + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} + +#ifdef FIXED_POINT +int opus_projection_decode(OpusProjectionDecoder *st, const unsigned char *data, + opus_int32 len, opus_int16 *pcm, int frame_size, + int decode_fec) +{ + return opus_multistream_decode_native(get_multistream_decoder(st), data, len, + pcm, opus_projection_copy_channel_out_short, frame_size, decode_fec, 0, + get_dec_demixing_matrix(st)); +} +#else +int opus_projection_decode(OpusProjectionDecoder *st, const unsigned char *data, + opus_int32 len, opus_int16 *pcm, int frame_size, + int decode_fec) +{ + return opus_multistream_decode_native(get_multistream_decoder(st), data, len, + pcm, opus_projection_copy_channel_out_short, frame_size, decode_fec, 1, + get_dec_demixing_matrix(st)); +} +#endif + +#ifndef DISABLE_FLOAT_API +int opus_projection_decode_float(OpusProjectionDecoder *st, const unsigned char *data, + opus_int32 len, float *pcm, int frame_size, int decode_fec) +{ + return opus_multistream_decode_native(get_multistream_decoder(st), data, len, + pcm, opus_projection_copy_channel_out_float, frame_size, decode_fec, 0, + get_dec_demixing_matrix(st)); +} +#endif + +int opus_projection_decoder_ctl(OpusProjectionDecoder *st, int request, ...) +{ + va_list ap; + int ret = OPUS_OK; + + va_start(ap, request); + ret = opus_multistream_decoder_ctl_va_list(get_multistream_decoder(st), + request, ap); + va_end(ap); + return ret; +} + +void opus_projection_decoder_destroy(OpusProjectionDecoder *st) +{ + opus_free(st); +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_encoder.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_encoder.c new file mode 100644 index 000000000..06fb2d252 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/opus_projection_encoder.c @@ -0,0 +1,468 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mathops.h" +#include "os_support.h" +#include "opus_private.h" +#include "opus_defines.h" +#include "opus_projection.h" +#include "opus_multistream.h" +#include "stack_alloc.h" +#include "mapping_matrix.h" + +struct OpusProjectionEncoder +{ + opus_int32 mixing_matrix_size_in_bytes; + opus_int32 demixing_matrix_size_in_bytes; + /* Encoder states go here */ +}; + +#if !defined(DISABLE_FLOAT_API) +static void opus_projection_copy_channel_in_float( + opus_val16 *dst, + int dst_stride, + const void *src, + int src_stride, + int src_channel, + int frame_size, + void *user_data +) +{ + mapping_matrix_multiply_channel_in_float((const MappingMatrix*)user_data, + (const float*)src, src_stride, dst, src_channel, dst_stride, frame_size); +} +#endif + +static void opus_projection_copy_channel_in_short( + opus_val16 *dst, + int dst_stride, + const void *src, + int src_stride, + int src_channel, + int frame_size, + void *user_data +) +{ + mapping_matrix_multiply_channel_in_short((const MappingMatrix*)user_data, + (const opus_int16*)src, src_stride, dst, src_channel, dst_stride, frame_size); +} + +static int get_order_plus_one_from_channels(int channels, int *order_plus_one) +{ + int order_plus_one_; + int acn_channels; + int nondiegetic_channels; + + /* Allowed numbers of channels: + * (1 + n)^2 + 2j, for n = 0...14 and j = 0 or 1. + */ + if (channels < 1 || channels > 227) + return OPUS_BAD_ARG; + + order_plus_one_ = isqrt32(channels); + acn_channels = order_plus_one_ * order_plus_one_; + nondiegetic_channels = channels - acn_channels; + if (nondiegetic_channels != 0 && nondiegetic_channels != 2) + return OPUS_BAD_ARG; + + if (order_plus_one) + *order_plus_one = order_plus_one_; + return OPUS_OK; +} + +static int get_streams_from_channels(int channels, int mapping_family, + int *streams, int *coupled_streams, + int *order_plus_one) +{ + if (mapping_family == 3) + { + if (get_order_plus_one_from_channels(channels, order_plus_one) != OPUS_OK) + return OPUS_BAD_ARG; + if (streams) + *streams = (channels + 1) / 2; + if (coupled_streams) + *coupled_streams = channels / 2; + return OPUS_OK; + } + return OPUS_BAD_ARG; +} + +static MappingMatrix *get_mixing_matrix(OpusProjectionEncoder *st) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (MappingMatrix *)(void*)((char*)st + + align(sizeof(OpusProjectionEncoder))); +} + +static MappingMatrix *get_enc_demixing_matrix(OpusProjectionEncoder *st) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (MappingMatrix *)(void*)((char*)st + + align(sizeof(OpusProjectionEncoder) + + st->mixing_matrix_size_in_bytes)); +} + +static OpusMSEncoder *get_multistream_encoder(OpusProjectionEncoder *st) +{ + /* void* cast avoids clang -Wcast-align warning */ + return (OpusMSEncoder *)(void*)((char*)st + + align(sizeof(OpusProjectionEncoder) + + st->mixing_matrix_size_in_bytes + + st->demixing_matrix_size_in_bytes)); +} + +opus_int32 opus_projection_ambisonics_encoder_get_size(int channels, + int mapping_family) +{ + int nb_streams; + int nb_coupled_streams; + int order_plus_one; + int mixing_matrix_rows, mixing_matrix_cols; + int demixing_matrix_rows, demixing_matrix_cols; + opus_int32 mixing_matrix_size, demixing_matrix_size; + opus_int32 encoder_size; + int ret; + + ret = get_streams_from_channels(channels, mapping_family, &nb_streams, + &nb_coupled_streams, &order_plus_one); + if (ret != OPUS_OK) + return 0; + + if (order_plus_one == 2) + { + mixing_matrix_rows = mapping_matrix_foa_mixing.rows; + mixing_matrix_cols = mapping_matrix_foa_mixing.cols; + demixing_matrix_rows = mapping_matrix_foa_demixing.rows; + demixing_matrix_cols = mapping_matrix_foa_demixing.cols; + } + else if (order_plus_one == 3) + { + mixing_matrix_rows = mapping_matrix_soa_mixing.rows; + mixing_matrix_cols = mapping_matrix_soa_mixing.cols; + demixing_matrix_rows = mapping_matrix_soa_demixing.rows; + demixing_matrix_cols = mapping_matrix_soa_demixing.cols; + } + else if (order_plus_one == 4) + { + mixing_matrix_rows = mapping_matrix_toa_mixing.rows; + mixing_matrix_cols = mapping_matrix_toa_mixing.cols; + demixing_matrix_rows = mapping_matrix_toa_demixing.rows; + demixing_matrix_cols = mapping_matrix_toa_demixing.cols; + } + else + return 0; + + mixing_matrix_size = + mapping_matrix_get_size(mixing_matrix_rows, mixing_matrix_cols); + if (!mixing_matrix_size) + return 0; + + demixing_matrix_size = + mapping_matrix_get_size(demixing_matrix_rows, demixing_matrix_cols); + if (!demixing_matrix_size) + return 0; + + encoder_size = + opus_multistream_encoder_get_size(nb_streams, nb_coupled_streams); + if (!encoder_size) + return 0; + + return align(sizeof(OpusProjectionEncoder)) + + mixing_matrix_size + demixing_matrix_size + encoder_size; +} + +int opus_projection_ambisonics_encoder_init(OpusProjectionEncoder *st, opus_int32 Fs, + int channels, int mapping_family, + int *streams, int *coupled_streams, + int application) +{ + MappingMatrix *mixing_matrix; + MappingMatrix *demixing_matrix; + OpusMSEncoder *ms_encoder; + int i; + int ret; + int order_plus_one; + unsigned char mapping[255]; + + if (streams == NULL || coupled_streams == NULL) { + return OPUS_BAD_ARG; + } + + if (get_streams_from_channels(channels, mapping_family, streams, + coupled_streams, &order_plus_one) != OPUS_OK) + return OPUS_BAD_ARG; + + if (mapping_family == 3) + { + /* Assign mixing matrix based on available pre-computed matrices. */ + mixing_matrix = get_mixing_matrix(st); + if (order_plus_one == 2) + { + mapping_matrix_init(mixing_matrix, mapping_matrix_foa_mixing.rows, + mapping_matrix_foa_mixing.cols, mapping_matrix_foa_mixing.gain, + mapping_matrix_foa_mixing_data, + sizeof(mapping_matrix_foa_mixing_data)); + } + else if (order_plus_one == 3) + { + mapping_matrix_init(mixing_matrix, mapping_matrix_soa_mixing.rows, + mapping_matrix_soa_mixing.cols, mapping_matrix_soa_mixing.gain, + mapping_matrix_soa_mixing_data, + sizeof(mapping_matrix_soa_mixing_data)); + } + else if (order_plus_one == 4) + { + mapping_matrix_init(mixing_matrix, mapping_matrix_toa_mixing.rows, + mapping_matrix_toa_mixing.cols, mapping_matrix_toa_mixing.gain, + mapping_matrix_toa_mixing_data, + sizeof(mapping_matrix_toa_mixing_data)); + } + else + return OPUS_BAD_ARG; + + st->mixing_matrix_size_in_bytes = mapping_matrix_get_size( + mixing_matrix->rows, mixing_matrix->cols); + if (!st->mixing_matrix_size_in_bytes) + return OPUS_BAD_ARG; + + /* Assign demixing matrix based on available pre-computed matrices. */ + demixing_matrix = get_enc_demixing_matrix(st); + if (order_plus_one == 2) + { + mapping_matrix_init(demixing_matrix, mapping_matrix_foa_demixing.rows, + mapping_matrix_foa_demixing.cols, mapping_matrix_foa_demixing.gain, + mapping_matrix_foa_demixing_data, + sizeof(mapping_matrix_foa_demixing_data)); + } + else if (order_plus_one == 3) + { + mapping_matrix_init(demixing_matrix, mapping_matrix_soa_demixing.rows, + mapping_matrix_soa_demixing.cols, mapping_matrix_soa_demixing.gain, + mapping_matrix_soa_demixing_data, + sizeof(mapping_matrix_soa_demixing_data)); + } + else if (order_plus_one == 4) + { + mapping_matrix_init(demixing_matrix, mapping_matrix_toa_demixing.rows, + mapping_matrix_toa_demixing.cols, mapping_matrix_toa_demixing.gain, + mapping_matrix_toa_demixing_data, + sizeof(mapping_matrix_toa_demixing_data)); + } + else + return OPUS_BAD_ARG; + + st->demixing_matrix_size_in_bytes = mapping_matrix_get_size( + demixing_matrix->rows, demixing_matrix->cols); + if (!st->demixing_matrix_size_in_bytes) + return OPUS_BAD_ARG; + } + else + return OPUS_UNIMPLEMENTED; + + /* Ensure matrices are large enough for desired coding scheme. */ + if (*streams + *coupled_streams > mixing_matrix->rows || + channels > mixing_matrix->cols || + channels > demixing_matrix->rows || + *streams + *coupled_streams > demixing_matrix->cols) + return OPUS_BAD_ARG; + + /* Set trivial mapping so each input channel pairs with a matrix column. */ + for (i = 0; i < channels; i++) + mapping[i] = i; + + /* Initialize multistream encoder with provided settings. */ + ms_encoder = get_multistream_encoder(st); + ret = opus_multistream_encoder_init(ms_encoder, Fs, channels, *streams, + *coupled_streams, mapping, application); + return ret; +} + +OpusProjectionEncoder *opus_projection_ambisonics_encoder_create( + opus_int32 Fs, int channels, int mapping_family, int *streams, + int *coupled_streams, int application, int *error) +{ + int size; + int ret; + OpusProjectionEncoder *st; + + /* Allocate space for the projection encoder. */ + size = opus_projection_ambisonics_encoder_get_size(channels, mapping_family); + if (!size) { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + st = (OpusProjectionEncoder *)opus_alloc(size); + if (!st) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + + /* Initialize projection encoder with provided settings. */ + ret = opus_projection_ambisonics_encoder_init(st, Fs, channels, + mapping_family, streams, coupled_streams, application); + if (ret != OPUS_OK) + { + opus_free(st); + st = NULL; + } + if (error) + *error = ret; + return st; +} + +int opus_projection_encode(OpusProjectionEncoder *st, const opus_int16 *pcm, + int frame_size, unsigned char *data, + opus_int32 max_data_bytes) +{ + return opus_multistream_encode_native(get_multistream_encoder(st), + opus_projection_copy_channel_in_short, pcm, frame_size, data, + max_data_bytes, 16, downmix_int, 0, get_mixing_matrix(st)); +} + +#ifndef DISABLE_FLOAT_API +#ifdef FIXED_POINT +int opus_projection_encode_float(OpusProjectionEncoder *st, const float *pcm, + int frame_size, unsigned char *data, + opus_int32 max_data_bytes) +{ + return opus_multistream_encode_native(get_multistream_encoder(st), + opus_projection_copy_channel_in_float, pcm, frame_size, data, + max_data_bytes, 16, downmix_float, 1, get_mixing_matrix(st)); +} +#else +int opus_projection_encode_float(OpusProjectionEncoder *st, const float *pcm, + int frame_size, unsigned char *data, + opus_int32 max_data_bytes) +{ + return opus_multistream_encode_native(get_multistream_encoder(st), + opus_projection_copy_channel_in_float, pcm, frame_size, data, + max_data_bytes, 24, downmix_float, 1, get_mixing_matrix(st)); +} +#endif +#endif + +void opus_projection_encoder_destroy(OpusProjectionEncoder *st) +{ + opus_free(st); +} + +int opus_projection_encoder_ctl(OpusProjectionEncoder *st, int request, ...) +{ + va_list ap; + MappingMatrix *demixing_matrix; + OpusMSEncoder *ms_encoder; + int ret = OPUS_OK; + + ms_encoder = get_multistream_encoder(st); + demixing_matrix = get_enc_demixing_matrix(st); + + va_start(ap, request); + switch(request) + { + case OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = + ms_encoder->layout.nb_channels * (ms_encoder->layout.nb_streams + + ms_encoder->layout.nb_coupled_streams) * sizeof(opus_int16); + } + break; + case OPUS_PROJECTION_GET_DEMIXING_MATRIX_GAIN_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = demixing_matrix->gain; + } + break; + case OPUS_PROJECTION_GET_DEMIXING_MATRIX_REQUEST: + { + int i, j, k, l; + int nb_input_streams; + int nb_output_streams; + unsigned char *external_char; + opus_int16 *internal_short; + opus_int32 external_size; + opus_int32 internal_size; + + /* (I/O is in relation to the decoder's perspective). */ + nb_input_streams = ms_encoder->layout.nb_streams + + ms_encoder->layout.nb_coupled_streams; + nb_output_streams = ms_encoder->layout.nb_channels; + + external_char = va_arg(ap, unsigned char *); + external_size = va_arg(ap, opus_int32); + if (!external_char) + { + goto bad_arg; + } + internal_short = mapping_matrix_get_data(demixing_matrix); + internal_size = nb_input_streams * nb_output_streams * sizeof(opus_int16); + if (external_size != internal_size) + { + goto bad_arg; + } + + /* Copy demixing matrix subset to output destination. */ + l = 0; + for (i = 0; i < nb_input_streams; i++) { + for (j = 0; j < nb_output_streams; j++) { + k = demixing_matrix->rows * i + j; + external_char[2*l] = (unsigned char)internal_short[k]; + external_char[2*l+1] = (unsigned char)(internal_short[k] >> 8); + l++; + } + } + } + break; + default: + { + ret = opus_multistream_encoder_ctl_va_list(ms_encoder, request, ap); + } + break; + } + va_end(ap); + return ret; + +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/repacketizer.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/repacketizer.c new file mode 100644 index 000000000..bda44a148 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/repacketizer.c @@ -0,0 +1,349 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "opus.h" +#include "opus_private.h" +#include "os_support.h" + + +int opus_repacketizer_get_size(void) +{ + return sizeof(OpusRepacketizer); +} + +OpusRepacketizer *opus_repacketizer_init(OpusRepacketizer *rp) +{ + rp->nb_frames = 0; + return rp; +} + +OpusRepacketizer *opus_repacketizer_create(void) +{ + OpusRepacketizer *rp; + rp=(OpusRepacketizer *)opus_alloc(opus_repacketizer_get_size()); + if(rp==NULL)return NULL; + return opus_repacketizer_init(rp); +} + +void opus_repacketizer_destroy(OpusRepacketizer *rp) +{ + opus_free(rp); +} + +static int opus_repacketizer_cat_impl(OpusRepacketizer *rp, const unsigned char *data, opus_int32 len, int self_delimited) +{ + unsigned char tmp_toc; + int curr_nb_frames,ret; + /* Set of check ToC */ + if (len<1) return OPUS_INVALID_PACKET; + if (rp->nb_frames == 0) + { + rp->toc = data[0]; + rp->framesize = opus_packet_get_samples_per_frame(data, 8000); + } else if ((rp->toc&0xFC) != (data[0]&0xFC)) + { + /*fprintf(stderr, "toc mismatch: 0x%x vs 0x%x\n", rp->toc, data[0]);*/ + return OPUS_INVALID_PACKET; + } + curr_nb_frames = opus_packet_get_nb_frames(data, len); + if(curr_nb_frames<1) return OPUS_INVALID_PACKET; + + /* Check the 120 ms maximum packet size */ + if ((curr_nb_frames+rp->nb_frames)*rp->framesize > 960) + { + return OPUS_INVALID_PACKET; + } + + ret=opus_packet_parse_impl(data, len, self_delimited, &tmp_toc, &rp->frames[rp->nb_frames], &rp->len[rp->nb_frames], NULL, NULL); + if(ret<1)return ret; + + rp->nb_frames += curr_nb_frames; + return OPUS_OK; +} + +int opus_repacketizer_cat(OpusRepacketizer *rp, const unsigned char *data, opus_int32 len) +{ + return opus_repacketizer_cat_impl(rp, data, len, 0); +} + +int opus_repacketizer_get_nb_frames(OpusRepacketizer *rp) +{ + return rp->nb_frames; +} + +opus_int32 opus_repacketizer_out_range_impl(OpusRepacketizer *rp, int begin, int end, + unsigned char *data, opus_int32 maxlen, int self_delimited, int pad) +{ + int i, count; + opus_int32 tot_size; + opus_int16 *len; + const unsigned char **frames; + unsigned char * ptr; + + if (begin<0 || begin>=end || end>rp->nb_frames) + { + /*fprintf(stderr, "%d %d %d\n", begin, end, rp->nb_frames);*/ + return OPUS_BAD_ARG; + } + count = end-begin; + + len = rp->len+begin; + frames = rp->frames+begin; + if (self_delimited) + tot_size = 1 + (len[count-1]>=252); + else + tot_size = 0; + + ptr = data; + if (count==1) + { + /* Code 0 */ + tot_size += len[0]+1; + if (tot_size > maxlen) + return OPUS_BUFFER_TOO_SMALL; + *ptr++ = rp->toc&0xFC; + } else if (count==2) + { + if (len[1] == len[0]) + { + /* Code 1 */ + tot_size += 2*len[0]+1; + if (tot_size > maxlen) + return OPUS_BUFFER_TOO_SMALL; + *ptr++ = (rp->toc&0xFC) | 0x1; + } else { + /* Code 2 */ + tot_size += len[0]+len[1]+2+(len[0]>=252); + if (tot_size > maxlen) + return OPUS_BUFFER_TOO_SMALL; + *ptr++ = (rp->toc&0xFC) | 0x2; + ptr += encode_size(len[0], ptr); + } + } + if (count > 2 || (pad && tot_size < maxlen)) + { + /* Code 3 */ + int vbr; + int pad_amount=0; + + /* Restart the process for the padding case */ + ptr = data; + if (self_delimited) + tot_size = 1 + (len[count-1]>=252); + else + tot_size = 0; + vbr = 0; + for (i=1;i=252) + len[i]; + tot_size += len[count-1]; + + if (tot_size > maxlen) + return OPUS_BUFFER_TOO_SMALL; + *ptr++ = (rp->toc&0xFC) | 0x3; + *ptr++ = count | 0x80; + } else { + tot_size += count*len[0]+2; + if (tot_size > maxlen) + return OPUS_BUFFER_TOO_SMALL; + *ptr++ = (rp->toc&0xFC) | 0x3; + *ptr++ = count; + } + pad_amount = pad ? (maxlen-tot_size) : 0; + if (pad_amount != 0) + { + int nb_255s; + data[1] |= 0x40; + nb_255s = (pad_amount-1)/255; + for (i=0;inb_frames, data, maxlen, 0, 0); +} + +int opus_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len) +{ + OpusRepacketizer rp; + opus_int32 ret; + if (len < 1) + return OPUS_BAD_ARG; + if (len==new_len) + return OPUS_OK; + else if (len > new_len) + return OPUS_BAD_ARG; + opus_repacketizer_init(&rp); + /* Moving payload to the end of the packet so we can do in-place padding */ + OPUS_MOVE(data+new_len-len, data, len); + ret = opus_repacketizer_cat(&rp, data+new_len-len, len); + if (ret != OPUS_OK) + return ret; + ret = opus_repacketizer_out_range_impl(&rp, 0, rp.nb_frames, data, new_len, 0, 1); + if (ret > 0) + return OPUS_OK; + else + return ret; +} + +opus_int32 opus_packet_unpad(unsigned char *data, opus_int32 len) +{ + OpusRepacketizer rp; + opus_int32 ret; + if (len < 1) + return OPUS_BAD_ARG; + opus_repacketizer_init(&rp); + ret = opus_repacketizer_cat(&rp, data, len); + if (ret < 0) + return ret; + ret = opus_repacketizer_out_range_impl(&rp, 0, rp.nb_frames, data, len, 0, 0); + celt_assert(ret > 0 && ret <= len); + return ret; +} + +int opus_multistream_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len, int nb_streams) +{ + int s; + int count; + unsigned char toc; + opus_int16 size[48]; + opus_int32 packet_offset; + opus_int32 amount; + + if (len < 1) + return OPUS_BAD_ARG; + if (len==new_len) + return OPUS_OK; + else if (len > new_len) + return OPUS_BAD_ARG; + amount = new_len - len; + /* Seek to last stream */ + for (s=0;s +#include +#include + +#define MAX_PACKETOUT 32000 + +void usage(char *argv0) +{ + fprintf(stderr, "usage: %s [options] input_file output_file\n", argv0); +} + +static void int_to_char(opus_uint32 i, unsigned char ch[4]) +{ + ch[0] = i>>24; + ch[1] = (i>>16)&0xFF; + ch[2] = (i>>8)&0xFF; + ch[3] = i&0xFF; +} + +static opus_uint32 char_to_int(unsigned char ch[4]) +{ + return ((opus_uint32)ch[0]<<24) | ((opus_uint32)ch[1]<<16) + | ((opus_uint32)ch[2]<< 8) | (opus_uint32)ch[3]; +} + +int main(int argc, char *argv[]) +{ + int i, eof=0; + FILE *fin, *fout; + unsigned char packets[48][1500]; + int len[48]; + int rng[48]; + OpusRepacketizer *rp; + unsigned char output_packet[MAX_PACKETOUT]; + int merge = 1, split=0; + + if (argc < 3) + { + usage(argv[0]); + return EXIT_FAILURE; + } + for (i=1;i48) + { + fprintf(stderr, "-merge parameter must be less than 48.\n"); + return EXIT_FAILURE; + } + i++; + } else if (strcmp(argv[i], "-split")==0) + split = 1; + else + { + fprintf(stderr, "Unknown option: %s\n", argv[i]); + usage(argv[0]); + return EXIT_FAILURE; + } + } + fin = fopen(argv[argc-2], "r"); + if(fin==NULL) + { + fprintf(stderr, "Error opening input file: %s\n", argv[argc-2]); + return EXIT_FAILURE; + } + fout = fopen(argv[argc-1], "w"); + if(fout==NULL) + { + fprintf(stderr, "Error opening output file: %s\n", argv[argc-1]); + fclose(fin); + return EXIT_FAILURE; + } + + rp = opus_repacketizer_create(); + while (!eof) + { + int err; + int nb_packets=merge; + opus_repacketizer_init(rp); + for (i=0;i1500 || len[i]<0) + { + if (feof(fin)) + { + eof = 1; + } else { + fprintf(stderr, "Invalid payload length\n"); + fclose(fin); + fclose(fout); + return EXIT_FAILURE; + } + break; + } + if (fread(ch, 1, 4, fin)!=4) + { + if (feof(fin)) + { + eof = 1; + } else { + fprintf(stderr, "Error reading.\n"); + fclose(fin); + fclose(fout); + return EXIT_FAILURE; + } + break; + } + rng[i] = char_to_int(ch); + if (fread(packets[i], len[i], 1, fin)!=1) { + if (feof(fin)) + { + eof = 1; + } else { + fprintf(stderr, "Error reading packet of %u bytes.\n", len[i]); + fclose(fin); + fclose(fout); + return EXIT_FAILURE; + } + break; + } + err = opus_repacketizer_cat(rp, packets[i], len[i]); + if (err!=OPUS_OK) + { + fprintf(stderr, "opus_repacketizer_cat() failed: %s\n", opus_strerror(err)); + break; + } + } + nb_packets = i; + + if (eof) + break; + + if (!split) + { + err = opus_repacketizer_out(rp, output_packet, MAX_PACKETOUT); + if (err>0) { + unsigned char int_field[4]; + int_to_char(err, int_field); + if(fwrite(int_field, 1, 4, fout)!=4){ + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + int_to_char(rng[nb_packets-1], int_field); + if (fwrite(int_field, 1, 4, fout)!=4) { + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + if (fwrite(output_packet, 1, err, fout)!=(unsigned)err) { + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + /*fprintf(stderr, "out len = %d\n", err);*/ + } else { + fprintf(stderr, "opus_repacketizer_out() failed: %s\n", opus_strerror(err)); + } + } else { + int nb_frames = opus_repacketizer_get_nb_frames(rp); + for (i=0;i0) { + unsigned char int_field[4]; + int_to_char(err, int_field); + if (fwrite(int_field, 1, 4, fout)!=4) { + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + if (i==nb_frames-1) + int_to_char(rng[nb_packets-1], int_field); + else + int_to_char(0, int_field); + if (fwrite(int_field, 1, 4, fout)!=4) { + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + if (fwrite(output_packet, 1, err, fout)!=(unsigned)err) { + fprintf(stderr, "Error writing.\n"); + return EXIT_FAILURE; + } + /*fprintf(stderr, "out len = %d\n", err);*/ + } else { + fprintf(stderr, "opus_repacketizer_out() failed: %s\n", opus_strerror(err)); + } + + } + } + } + + fclose(fin); + fclose(fout); + return EXIT_SUCCESS; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/tansig_table.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/tansig_table.h new file mode 100644 index 000000000..c76f844a7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/src/tansig_table.h @@ -0,0 +1,45 @@ +/* This file is auto-generated by gen_tables */ + +static const float tansig_table[201] = { +0.000000f, 0.039979f, 0.079830f, 0.119427f, 0.158649f, +0.197375f, 0.235496f, 0.272905f, 0.309507f, 0.345214f, +0.379949f, 0.413644f, 0.446244f, 0.477700f, 0.507977f, +0.537050f, 0.564900f, 0.591519f, 0.616909f, 0.641077f, +0.664037f, 0.685809f, 0.706419f, 0.725897f, 0.744277f, +0.761594f, 0.777888f, 0.793199f, 0.807569f, 0.821040f, +0.833655f, 0.845456f, 0.856485f, 0.866784f, 0.876393f, +0.885352f, 0.893698f, 0.901468f, 0.908698f, 0.915420f, +0.921669f, 0.927473f, 0.932862f, 0.937863f, 0.942503f, +0.946806f, 0.950795f, 0.954492f, 0.957917f, 0.961090f, +0.964028f, 0.966747f, 0.969265f, 0.971594f, 0.973749f, +0.975743f, 0.977587f, 0.979293f, 0.980869f, 0.982327f, +0.983675f, 0.984921f, 0.986072f, 0.987136f, 0.988119f, +0.989027f, 0.989867f, 0.990642f, 0.991359f, 0.992020f, +0.992631f, 0.993196f, 0.993718f, 0.994199f, 0.994644f, +0.995055f, 0.995434f, 0.995784f, 0.996108f, 0.996407f, +0.996682f, 0.996937f, 0.997172f, 0.997389f, 0.997590f, +0.997775f, 0.997946f, 0.998104f, 0.998249f, 0.998384f, +0.998508f, 0.998623f, 0.998728f, 0.998826f, 0.998916f, +0.999000f, 0.999076f, 0.999147f, 0.999213f, 0.999273f, +0.999329f, 0.999381f, 0.999428f, 0.999472f, 0.999513f, +0.999550f, 0.999585f, 0.999617f, 0.999646f, 0.999673f, +0.999699f, 0.999722f, 0.999743f, 0.999763f, 0.999781f, +0.999798f, 0.999813f, 0.999828f, 0.999841f, 0.999853f, +0.999865f, 0.999875f, 0.999885f, 0.999893f, 0.999902f, +0.999909f, 0.999916f, 0.999923f, 0.999929f, 0.999934f, +0.999939f, 0.999944f, 0.999948f, 0.999952f, 0.999956f, +0.999959f, 0.999962f, 0.999965f, 0.999968f, 0.999970f, +0.999973f, 0.999975f, 0.999977f, 0.999978f, 0.999980f, +0.999982f, 0.999983f, 0.999984f, 0.999986f, 0.999987f, +0.999988f, 0.999989f, 0.999990f, 0.999990f, 0.999991f, +0.999992f, 0.999992f, 0.999993f, 0.999994f, 0.999994f, +0.999994f, 0.999995f, 0.999995f, 0.999996f, 0.999996f, +0.999996f, 0.999997f, 0.999997f, 0.999997f, 0.999997f, +0.999997f, 0.999998f, 0.999998f, 0.999998f, 0.999998f, +0.999998f, 0.999998f, 0.999999f, 0.999999f, 0.999999f, +0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, +0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, +1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, +1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, +1.000000f, +}; diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/meson.build b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/meson.build new file mode 100644 index 000000000..5f3ac9df3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/meson.build @@ -0,0 +1,34 @@ +# Tests that link to libopus +opus_tests = [ + ['test_opus_api'], + ['test_opus_decode', [], 60], + ['test_opus_encode', 'opus_encode_regressions.c', 120], + ['test_opus_padding'], + ['test_opus_projection'], +] + +foreach t : opus_tests + test_name = t.get(0) + extra_srcs = t.get(1, []) + + test_kwargs = {} + if t.length() > 2 + test_kwargs += {'timeout': t[2]} + endif + + exe_kwargs = {} + # This test uses private symbols + if test_name == 'test_opus_projection' + exe_kwargs = { + 'link_with': [celt_lib, silk_lib], + 'objects': opus_lib.extract_all_objects(), + } + endif + + exe = executable(test_name, '@0@.c'.format(test_name), extra_srcs, + include_directories: opus_includes, + dependencies: [libm, opus_dep], + install: false, + kwargs: exe_kwargs) + test(test_name, exe, kwargs: test_kwargs) +endforeach diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.c new file mode 100644 index 000000000..ea6ec4fd1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.c @@ -0,0 +1,124 @@ +/* Copyright (c) 2017 Google Inc. */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "opus.h" +#include "opus_types.h" + +#define MAX_FRAME_SAMP 5760 +#define MAX_PACKET 1500 + +/* 4 bytes: packet length, 4 bytes: encoder final range */ +#define SETUP_BYTE_COUNT 8 + +#define MAX_DECODES 12 + +typedef struct { + int fs; + int channels; +} TocInfo; + +static void ParseToc(const uint8_t *toc, TocInfo *const info) { + const int samp_freqs[5] = {8000, 12000, 16000, 24000, 48000}; + const int bandwidth = opus_packet_get_bandwidth(toc); + + info->fs = samp_freqs[bandwidth - OPUS_BANDWIDTH_NARROWBAND]; + info->channels = opus_packet_get_nb_channels(toc); +} + +/* Treats the input data as concatenated packets encoded by opus_demo, + * structured as + * bytes 0..3: packet length + * bytes 4..7: encoder final range + * bytes 8+ : Opus packet, including ToC + */ +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + OpusDecoder *dec; + opus_int16 *pcm; + uint8_t *temp_data; + TocInfo toc; + int i = 0; + int err = OPUS_OK; + int num_decodes = 0; + + /* Not enough data to setup the decoder (+1 for the ToC) */ + if (size < SETUP_BYTE_COUNT + 1) { + return 0; + } + + /* Create decoder based on info from the first ToC available */ + ParseToc(&data[SETUP_BYTE_COUNT], &toc); + + dec = opus_decoder_create(toc.fs, toc.channels, &err); + if (err != OPUS_OK || dec == NULL) { + return 0; + } + + pcm = (opus_int16*) malloc(sizeof(*pcm) * MAX_FRAME_SAMP * toc.channels); + + while (i + SETUP_BYTE_COUNT < size && num_decodes++ < MAX_DECODES) { + int len, fec; + + len = (opus_uint32) data[i ] << 24 | + (opus_uint32) data[i + 1] << 16 | + (opus_uint32) data[i + 2] << 8 | + (opus_uint32) data[i + 3]; + if (len > MAX_PACKET || len < 0 || i + SETUP_BYTE_COUNT + len > size) { + break; + } + + /* Bytes 4..7 represent encoder final range, but are unused here. + * Instead, byte 4 is repurposed to determine if FEC is used. */ + fec = data[i + 4] & 1; + + if (len == 0) { + /* Lost packet */ + int frame_size; + opus_decoder_ctl(dec, OPUS_GET_LAST_PACKET_DURATION(&frame_size)); + (void) opus_decode(dec, NULL, len, pcm, frame_size, fec); + } else { + temp_data = (uint8_t*) malloc(len); + memcpy(temp_data, &data[i + SETUP_BYTE_COUNT], len); + + (void) opus_decode(dec, temp_data, len, pcm, MAX_FRAME_SAMP, fec); + + free(temp_data); + } + + i += SETUP_BYTE_COUNT + len; + } + + opus_decoder_destroy(dec); + free(pcm); + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.options b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.options new file mode 100644 index 000000000..e5ae71b93 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_decode_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 1000000 diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_encode_regressions.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_encode_regressions.c new file mode 100644 index 000000000..292347301 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/opus_encode_regressions.c @@ -0,0 +1,1035 @@ +/* Copyright (c) 2016 Mark Harris, Jean-Marc Valin */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include "opus_multistream.h" +#include "opus.h" +#include "test_opus_common.h" + + +static int celt_ec_internal_error(void) +{ + OpusMSEncoder *enc; + int err; + unsigned char data[2460]; + int streams; + int coupled_streams; + unsigned char mapping[1]; + + enc = opus_multistream_surround_encoder_create(16000, 1, 1, &streams, + &coupled_streams, mapping, OPUS_APPLICATION_VOIP, &err); + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(8)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(OPUS_AUTO)); + { + static const short pcm[320] = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1792, 1799, 1799, + 1799, 1799, 1799, 1799, 1799, 1799, 1799, 1799, 1799, + 1799, 1799, 1799, 1799, 1799, 0, 25600, 1799, 1799, + 1799, 1799, 1799, 1799, 1799, 1799, 1799, 1799, 1799, + 1799, 1799, 1799, 1799, 7, 0, 255, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 32767, -1, + 0, 0, 0, 100, 0, 16384, 0, 0, 0, + 0, 0, 0, 4, 0, 0, -256, 255, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0,-32768, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 128, 0, 0, 0, 0, + 0, 0, 0, 0, -256, 0, 0, 32, 0, + 0, 0, 0, 0, 0, 0, 4352, 4, 228, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 5632, 0, 0, + 0, 0,-32768, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 256, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + -3944, 10500, 4285, 10459, -6474, 10204, -6539, 11601, -6824, + 13385, -7142, 13872,-11553, 13670, -7725, 13463, -6887, 7874, + -5580, 12600, -4964, 12480, 3254, 11741, -4210, 9741, -3155, + 7558, -5468, 5431, -1073, 3641, -1304, 0, -1, 343, + 26, 0, 0, 150, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1799, 1799, 1799, 1799, 1799, -2553, + 7, 1792, 1799, 1799, 1799, 1799, 1799, 1799, 1799, + 1799, 1799, 1799, 1799, -9721 + }; + err = opus_multistream_encode(enc, pcm, 320, data, 2460); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(10)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(18)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(90)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(280130)); + { + static const short pcm[160] = + { + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9526, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, 25600, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510 + }; + err = opus_multistream_encode(enc, pcm, 160, data, 2460); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(10)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(18)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(90)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(280130)); + { + static const short pcm[160] = + { + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9494, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510 + }; + err = opus_multistream_encode(enc, pcm, 160, data, 2460); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(10)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(18)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(90)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(280130)); + { + static const short pcm[160] = + { + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9479, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, -9510, + -9510, -9510, -9510, -9510, -9510, -9510, -9510 + }; + err = opus_multistream_encode(enc, pcm, 160, data, 2460); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(10)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(18)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(90)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(280130)); + { + static const short pcm[160] = + { + -9510, -9510, 1799, 1799, 1799, 1799, 1799, 1799, 1799, + 1799, 1799, 1799, 1799, 1799, 1799, 1799, 1799, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + -256, 255, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 128, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 0, 0, 0, 0, 0, 0, 0, + 4352, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 148, 0, 0, 0, 0, + 5632 + }; + err = opus_multistream_encode(enc, pcm, 160, data, 2460); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(12)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(41)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(21425)); + { + static const short pcm[40] = + { + 10459, -6474, 10204, -6539, 11601, -6824, 13385, -7142, 13872, + -11553, 13670, -7725, 13463, -6887, 12482, -5580, 12600, -4964, + 12480, 3254, 11741, -4210, 9741, -3155, 7558, -5468, 5431, + -1073, 3641, -1304, 0, -1, 343, 26, 0, 0, + 0, 0, -256, 226 + }; + err = opus_multistream_encode(enc, pcm, 40, data, 2460); + assert(err > 0); + /* returns -3 */ + } + opus_multistream_encoder_destroy(enc); + return 0; +} + +static int mscbr_encode_fail10(void) +{ + OpusMSEncoder *enc; + int err; + unsigned char data[627300]; + static const unsigned char mapping[255] = + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101, + 102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118, + 119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152, + 153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169, + 170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186, + 187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203, + 204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220, + 221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237, + 238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 + }; + + enc = opus_multistream_encoder_create(8000, 255, 254, 1, mapping, + OPUS_APPLICATION_RESTRICTED_LOWDELAY, &err); + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(2)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(2)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(14)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(57)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(3642675)); + { + static const short pcm[20*255] = + { + 0 + }; + err = opus_multistream_encode(enc, pcm, 20, data, 627300); + assert(err > 0); + /* returns -1 */ + } + opus_multistream_encoder_destroy(enc); + return 0; +} + +static int mscbr_encode_fail(void) +{ + OpusMSEncoder *enc; + int err; + unsigned char data[472320]; + static const unsigned char mapping[192] = + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101, + 102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118, + 119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152, + 153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169, + 170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186, + 187,188,189,190,191 + }; + + enc = opus_multistream_encoder_create(8000, 192, 189, 3, mapping, + OPUS_APPLICATION_RESTRICTED_LOWDELAY, &err); + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_MEDIUMBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(8)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(15360)); + { + static const short pcm[20*192] = + { + 0 + }; + err = opus_multistream_encode(enc, pcm, 20, data, 472320); + assert(err > 0); + /* returns -1 */ + } + opus_multistream_encoder_destroy(enc); + return 0; +} + +static int surround_analysis_uninit(void) +{ + OpusMSEncoder *enc; + int err; + unsigned char data[7380]; + int streams; + int coupled_streams; + unsigned char mapping[3]; + + enc = opus_multistream_surround_encoder_create(24000, 3, 1, &streams, + &coupled_streams, mapping, OPUS_APPLICATION_AUDIO, &err); + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(8)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(84315)); + { + static const short pcm[960*3] = + { + -6896, 4901, -6158, 4120, -5164, 3631, -4442, 3153, -4070, + 3349, -4577, 4474, -5541, 5058, -6701, 3881, -7933, 1863, + -8041, 697, -6738,-31464, 14330,-12523, 4096, -6130, 29178, + -250,-21252, 10467, 16907, -3359, -6644, 31965, 14607,-21544, + -32497, 24020, 12557,-26926,-18421, -1842, 24587, 19659, 4878, + 10954, 23060, 8907,-10215,-16179, 31772,-11825,-15590,-23089, + 17173,-25903,-17387, 11733, 4884, 10204,-16476,-14367, 516, + 20453,-16898, 20967,-23813, -20, 22011,-17167, 9459, 32499, + -25855, -523, -3883, -390, -4206, 634, -3767, 2325, -2751, + 3115, -2392, 2746, -2173, 2317, -1147, 2326, 23142, 11314, + -15350,-24529, 3026, 6146, 2150, 2476, 1105, -830, 1775, + -3425, 3674,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + 4293,-14023, 3879,-15553, 3158,-16161, 2629, 18433,-12535, + -6645,-20735,-32763,-13824,-20992, 25859, 13052, -8731, 2292, + -3860, 24049, 10225,-19220, 10478,-22294, 22773, 28137, 13816, + 30953,-25863,-24598, 16888,-14612,-28942, 20974,-27397,-18944, + -18690, 20991,-16638, 5632,-14330, 28911,-25594, 17408, 29958, + -517,-20984, -1800, 11281, 9977,-21221,-14854, 23840, -9477, + 3362,-12805,-22493, 32507, 156, 16384, -1163, 2301, -1874, + 4600, -1748, 6950, 16557, 8192, -7372, -1033, -3278, 2806, + 20275, 3317, -717, 9792, -767, 9099, -613, 8362, 5027, + 7774, 2597, 8549, 5278, 8743, 9343, 6940, 13038, 4826, + 14086, 2964, 13215, 1355, 11596, 455, 9850, -519, 10680, + -2287, 12551, -3736, 13639, -4291, 13790, -2722, 14544, -866, + 15050, -304, 22833, -1196, 13520, -2063, 13051, -2317, 13066, + -2737, 13773, -2664, 14105, -3447, 13854, 24589, 24672, -5280, + 10388, -4933, 7543, -4149, 3654, -1552, 1726, 661, 57, + 2922, -751, 3917, 8419, 3840, -5218, 3435, 5540, -1073, + 4153, -6656, 1649, -769, -7276,-13072, 6380, -7948, 20717, + 18425, 17392, 14335,-18190, -1842, 24587, 19659, 11790, 10954, + 23060, 8907,-10215,-16179, 31772,-11825,-15590,-23101, 17173, + -25903,-17387, 11733, 4884, 10192,-16627,-14367, 516, 20453, + -16898, 20967,-23813, -20, 22011,-17167, 9468, 32499,-25607, + -523, -3883, -390, -4206, 634, -3767, 2325, -2751, 3115, + -2392, 2746, -2161, 2317, -1147, 2326, 23142, 11314,-15350, + -29137, 3026,-15056, -491,-15170, -386,-16015, -641,-16505, + -930,-16206, -717,-16175, -2839,-16374, -4558,-16237, -5207, + -15903, -6421, 6373, -1403, 5431, -1073, 3641, -1304, -4495, + -769, -7276, 2856, -7870, 3314, -8730, 3964,-10183, 4011, + -11135, 3421,-11727, 2966,-12360, 2818,-13472, 3660,-13805, + 5162,-13478, 6434,-12840, 7335,-12420, 6865,-12349, 5541, + -11965, 5530,-10820, 5132, -9197, 3367, -7745, 1223, -6910, + -433, -6211, -1711, -4958, -1025, -3755, -836, -3292, -1666, + -2661,-10755, 31472,-27906, 31471, 18690, 5617, 16649, 11253, + -22516,-17674,-31990, 3575,-31479, 5883, 26121, 12890, -6612, + 12228,-11634, 523, 26136,-21496, 20745,-15868, -4100,-24826, + 23282, 22798, 491, -1774, 15075,-27373,-13094, 6417,-29487, + 14608, 10185, 16143, 22211, -8436, 4288, -8694, 2375, 3023, + 486, 1455, 128, 202, 942, -923, 2068, -1233, -717, + -1042, -2167, 32255, -4632, 310, -4458, -3639, -5258, 2106, + -6857, 2681, -7497, 2765, -6601, 1945, -5219, 19154, -4877, + 619, -5719, -1928, -6208, -121, 593, 188, 1558, -4152, + 1648, 156, 1604, -3664, -6862, -2851, -5112, -3600, -3747, + -5081, -4428, -5592, 20974,-27397,-18944,-18690, 20991,-17406, + 5632,-14330, 28911, 15934, 15934, 15934, 15934, 15934, 15934, + 15934, 15934, 15934, 15934, 15934, 15934,-25594, 17408, 29958, + -7173,-16888, 9098, -613, 8362, 675, 7774, 2597, 8549, + 5278, 8743, 9375, 6940, 13038, 4826, 14598, 7721,-24308, + -29905,-19703,-17106,-16124, -3287,-26118,-19709,-10769, 24353, + 28648, 6946, -1363, 12485, -1187, 26074,-25055, 10004,-24798, + 7204, -4581, -9678, 1554, 10553, 3102, 12193, 2443, 11955, + 1213, 10689, -1293, 921, -4173, 10709, -6049, 8815, -7128, + 8147, -8308, 6847, -2977, 4920,-11447,-22426,-11794, 3514, + -10220, 3430, -7993, 1926, -7072, 327, -7569, -608, -7605, + 3695, -6271, -1579, -4877, -1419, -3103, -2197, 128, -3904, + 3760, -5401, 4906, -6051, 4250, -6272, 3492, -6343, 3197, + -6397, 4041, -6341, 6255, -6381, 7905, 16504, 0, -6144, + 8062, -5606, 8622, -5555, -9, -1, 7423, 0, 1, + 238, 5148, 1309, 4700, 2218, 4403, 2573, 3568, 28303, + 1758, 3454, -1247, 3434, -4912, 2862, -7844, 1718,-10095, + 369,-12631, 128, -3904, 3632, -5401, 4906, -6051, 4250, + -6272, 3492, -6343, 3197, -6397, 4041, -6341, 6255, -6381, + 7905, 16504, 0, -6144, 8062, -5606, 8622, -5555, 8439, + -3382, 7398, -1170, 6132, 238, 5148, 1309, 4700, 2218, + 4403, 2573, 3568, 2703, 1758, 3454, -1247, 3434, -4912, + 2862, -7844, 1718,-10095, 369,-12631, -259,-14632, 234, + -15056, -521,-15170, -386,-16015, -641,-16505, -930,-16206, + -1209,-16146, -2839,-16374, -4558,-16218, -5207,-15903, -6421, + -15615, -6925,-14871, -6149,-13759, -5233,-12844, 18313, -4357, + -5696, 2804, 12992,-22802, -6720, -9770, -7088, -8998, 14330, + -12523, 14843, -6130, 29178, -250,-27396, 10467, 16907, -3359, + -6644, 31965, 14607,-21544,-32497, 24020, 12557,-26926, -173, + -129, -6401, -130,-25089, -3841, -4916, -3048, 224, -237, + -3969, -189, -3529, -535, -3464,-14863,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14395,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907, 0, 32512,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907, 9925, -718, 9753, -767, + 9099, -613, 8362, 675, 7774, 2597, 8549, 5278, 8743, + 9375, 6940, 13038, 4826, 14598, 7721,-24308,-29905,-19703, + -17106,-16124, -3287,-26118,-19709, 0, 24353, 28648, 10274, + -11292,-29665,-16417, 24346, 14553, 18707, 26323, -4596,-17711, + 5133, 26328, 13,-31168, 24583, 18404,-28927,-24350, 19453, + 28642, 1019,-10777, -3079, 30188, -7686, 27635,-32521,-16384, + 12528, -6386, 10986, 23827,-25880,-32752,-23321, 14605, 32231, + 780,-13849, 15119, 28647, 4888, -7705, 30750, 64, 0, + 32488, 6687,-20758, 19745, -2070,-13792, -6414, 28188, -2821, + -4585, 7168, 7444, 23557,-21998, 13064, 3345, -4086,-28915, + -8694, 32262, 8461, 27387,-12275, 12012, 23563,-18719,-28410, + 29144,-22271, -562, -9986, -5434, 12288, 5573,-16642, 32448, + 29182, 32705,-30723, 24255,-19716, 18368, -4357, -5696, 2804, + 12992,-22802,-22080, -7701, -5183, 486, -3133, -5660, -1083, + 16871,-28726,-11029,-30259, -1209,-16146, -2839,-16374, -4558, + -16218,-10523, 20697, -9500, -1316, 5431, -1073, 3641, -1304, + 1649, -769, -7276, 2856, -7870, 3314, -8730, 3964,-10183, + 4011,-11135, 3421,-11727, 21398, 32767, -1, 32486, -1, + 6301,-13071, 6380, -7948, -1, 32767, 240, 14081, -5646, + 30973, -3598,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907, 32767,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907, 8901, 9375, 6940, 13038, 4826, 14598, 7721,-24308, + -29905,-19703,-17106,-16124, -3287,-26118,-19709,-10769, 24361, + 28648, 10274,-11292,-29665,-16417, 24346, 14580, 18707, 26323, + -4440,-17711, 5133, 26328,-14579,-31008, 24583, 18404, 28417, + -24350, 19453, 28642,-32513,-10777, -3079, 30188, -7686, 27635, + -32521,-16384,-20240, -6386, 10986, 23827,-25880,-32752,-23321, + 14605, 32231, 780,-13849, 15119, 28647, 4888, -7705,-15074, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907, 8192,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14897,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -15931,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907, 26121, 12890, 2604, + 12228,-11634, 12299, 5573,-16642, 32452, 29182, 32705,-30723, + 24255,-19716, 13248,-11779, -5696, 2804, 12992,-27666,-22080, + -7701, -5183, -6682,-31464, 14330,-12523, 14843, -6130, 29178, + -18,-27396, 10467, 16907, -3359, -6644, 31965, 14607,-21544, + -32497, 24020, 12557,-26926,-18421, 706, 24587, 19659, 4878, + 10954, 23060, 8907,-10215,-22579, 31772,-11825,-15590,-23089, + 17173,-25903,-17387, 3285, 4884, 10204,-16627,-14367, 516, + 20453,-16898, 20967,-23815, -20, 22011,-17167, 9468, 32499, + -25607, -523, -3883, -390, -4206, 634, -3767, 2325, -2751, + 3115, -2392, 2746, -2173, 2317, -1147, 2326, 23142, 11314, + -15130,-29137, 3026, 6146, 2150, 2476, 1105, -830, 1775, + -3425, 3674, -5287, 4609, -7175, 4922, -9579, 4556,-12007, + 4236,-14023, 3879,-15553, 3158,-16161, 2576, 18398,-12535, + -6645,-20735,-32763,-13824,-20992, 25859, 5372, 12040, 13307, + -4355,-30213, -9, -6019, 14061,-31487,-13842, 30449, 15083, + 14088, 31205,-18678,-12830, 14090,-26138,-25337,-11541, -3254, + 27628,-22270, 30953,-16136,-30745, 20991,-17406, 5632,-14330, + 28911,-25594, 17408,-20474, 13041, -8731, 2292, -3860, 24049, + 10225,-19220, 10478, -4374, -1199, 148, -330, -74, 593, + 188, 1558, -4152, 15984, 15934, 15934, 15934, 15934, 15934, + 15934, 15934, 15934, 15934, 15934, 15934, 1598, 156, 1604, + -1163, 2278,-30018,-25821,-21763,-23776, 24066, 9502, 25866, + -25055, 10004,-24798, 7204, -4581, -9678, 1554, 10553, 3102, + 12193, 2443, 11955, 1213, 10689, -1293, 921, -4173, 8661, + -6049, 8815,-21221,-14854, 23840, -9477, 8549, 5278, 8743, + 9375, 6940, 13038, 4826, 14598, 7721,-24308,-29905,-19703, + -17106,-16124, -3287,-26118,-19709,-10769, 24361, 28648, 10274, + -11292,-29665,-16417, 24346, 14580, 18707, 26323, -4410,-17711, + 5133, 26328,-14579,-31008, 24583, 18404, 28417,-24350, 19453, + 28642,-32513,-10777, -3079, 30188, -7686, 27635,-32521,-16384, + -20240, -6386, 10986, 23827,-25880,-32752,-23321, 14605, 32231, + 780,-13849, 15119, 28647, 4888, -7705, 30750, 64, 0, + 32488, 6687,-20758, 19745, -2070, -1, -1, 28, 256, + -4608, 7168, 7444, 23557,-21998, 13064, 3345, -4086,-28915, + -8594, 32262, 8461, 27387,-12275, 12012, 23563,-18719,-28410, + 29144,-22271,-32562,-16384, 12528, -6386, 10986, 23827,-25880, + -32752,-23321, 14605, 32231, 780,-13849, 15119, 28647, 4888, + -7705, 30750, 64, 0, 32488, 6687,-20758, 19745, -2070, + -13792, -6414, 28188, -2821, -4585, 7168, 7444, 23557,-21998, + 13064, 3345, -4086,-28915, -8694, 32262, 8461, 27387,-12275, + 12012, 23563,-18719,-28410, 29144,-22271, -562, -9986, -5434, + 12288, -2107,-16643, 32452, 29182, 32705,-30723, 24255,-19716, + 18368, -4357, -5696, 2804, 12992,-22802,-22080, -7701, -5183, + 486, -3133, -5660, -1083, 16871,-28726,-11029,-30259, -1209, + -16146, -2839,-16374, -4558,-16218,-10523, 20697, -9500, -1316, + 5431, -1073, 3641, -1304, 1649, -769, -7276, 2856, -7870, + 3314, -8730, 3964,-10183, 4011,-11135, 3421,-11727, 21398, + 32767, -1, 32486, -1, -99,-13072, 6380, -7948, 4864, + 32767, 17392, 14335, -5646, 30973, -3598,-10299,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14905,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-19771,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-16443,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-15931,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907, -1,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907, 7877, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, -994, -7276, 2856, -7870, + 3314, -8730, 3964,-10183, 4011,-11135, 3421,-11727, 21398, + 32767, -1, 32486, -1, -99,-13072, 6380, -7948, 4864, + 32767, 17392, 14335, -5646, 30973, -3598,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14905,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907, 197, 0,-14977,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907, 12838, 6653, 294, + -29699,-25821,-21763,-23776, 24066, 9502, 25866,-25055, 10004, + -24798, 7204, -4581, -9678, 1554, 10553, 3102, 12193, 2443, + 11955, 1213, 10689, -1293, 921, 179, 8448, -6049, 8815, + -7128, 8147, -8308, 6847, -9889, 4920,-11447, 3174,-11794, + 3514,-10220, 3430, 16384, 1926, -7072, 327, -7537, -608, + -7605, -1169, -6397, -1579, -4877, -1419, -3103, -2197, 128, + -3904, 3632, -5401, 4906, -6051, 4250, -6272, 3492, -6343, + 3197, -6397, 4041, -6341, 6255, -6381, 7905, 16504, 0, + -6144, 8062, -5606, 8622, -5555, 8439, -3382, 7398, -1170, + 6132, 238, 5148, 1309, 4700, 2218, 4403, 2573, 3568, + 2703, 1758, 3454, -1247, 3434, -4912, 2862, -7844, 1718, + -10095, 369,-12631, -259,-14632, 234,-15056, -491,-16194, + -386,-16015, -641,-16505, -930,-16206, -1209,-16146, -2839, + -16374, -4558,-16218, -5207,-15903, -6421,-15615, -6925,-14871, + -6149,-13759, -5233,-12844, 18313, -4357, -5696, 2804, 12992, + -22802, -6720, -9770, -7088, -8998, 14330,-12523, 14843, -6130, + 29178, -250,-27396, 10467, 16907, -3359, -6644, 31965, 14607, + -21544,-32497, 24020, 12557,-26926, -173, -129, -6401, -130, + -25089, -3816, -4916, -3048, -32, -1, -3969, 256, -3529, + -535, -3464,-14863,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -1209,-16146, -2839,-16374, -4558,-16218,-10523, 20697, -9500, + -1316, 5431, -1073, 3641, -1304, 1649, -769, -7276, 2856, + -7870, 3314, -8730, 3964,-10183, 4011,-11135, 3421,-11727, + 21398, 32767, -1, 32486, -1, 6301,-13071, 6380, -7948, + -1, 32767, 240, 14081, -5646, 30973, -3598,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + 32767,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907, 8901, 9375, 6940, + 13038, 4826, 14598, 7721,-24308,-29905,-19703,-17106,-16124, + -3287,-26118,-19709,-10769, 24361, 28648, 10274,-11292,-29665, + -16417, 24346, 14580, 18707, 26323, -4440,-17711, 5133, 26328, + -14579,-31008, 24583, 18404, 28417,-24350, 19453, 28642,-32513, + -10777, -3079, 30188, -7686, 27635,-32521,-16384,-20240, -6386, + 10986, 23827,-25880,-32752,-23321, 14605, 32231, 780,-13849, + 15119, 28647, 4888, -7705,-15074,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, 8192, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14897, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-15931,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907, 26121, 12890, 2604, 12228,-11634, 12299, 5573, + -16642, 32452, 29182, 32705,-30723, 24255,-19716, 13248,-11779, + -5696, 2804, 12992,-27666,-22080, -7701, -5183, -6682,-31464, + 14330,-12523, 14843, -6130, 29178, -18,-27396, 10467, 16907, + -3359, -6644, 31965, 14607,-21544,-32497, 24020, 12557,-26926, + -18421, 706, 24587, 19659, 4878, 10954, 23060, 8907,-10215, + -22579, 31772,-11825,-15590,-23089, 17173,-25903,-17387, 3285, + 4884, 10204,-16627,-14367, 516, 20453,-16898, 20967,-23815, + -20, 22011,-17167, 9468, 32499,-25607, -523, -3883, -390, + -4206, 634, -3767, 2325, -2751, 3115, -2392, 2746, -2173, + 2317, -1147, 2326, 23142, 11314,-15130,-29137, 3026, 6146, + 2150, 2476, 1105, -830, 1775, -3425, 3674, -5287, 4609, + -7175, 4922, -9579, 4556,-12007, 4236,-14023, 3879,-15553, + 3158,-16161, 2576, 18398,-12535, -6645,-20735,-32763,-13824, + -20992, 25859, 5372, 12040, 13307, -4355,-30213, -9, -6019 + }; + err = opus_multistream_encode(enc, pcm, 960, data, 7380); + assert(err > 0); + } + opus_multistream_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_VBR_CONSTRAINT(0)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PHASE_INVERSION_DISABLED(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_DTX(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_COMPLEXITY(6)); + opus_multistream_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); + opus_multistream_encoder_ctl(enc, OPUS_SET_LSB_DEPTH(9)); + opus_multistream_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_multistream_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(5)); + opus_multistream_encoder_ctl(enc, OPUS_SET_BITRATE(775410)); + { + static const short pcm[1440*3] = + { + 30449, 15083, 14088, 31205,-18678,-12830, 14090,-26138,-25337, + -11541, -3254, 27628,-22270, 30953,-16136,-30745, 20991,-17406, + 5632,-14330, 28911,-25594, 17408,-20474, 13041, -8731, 2292, + -3860, 24049, 10225,-19220, 10478, -4374, -1199, 148, -330, + -74, 593, 188, 1558, -4152, 15984, 15934, 15934, 15934, + 15934, 15934, 15934, 15934, 15934, 15934, 15934, 15934, 1598, + 156, 1604, -1163, 2278,-30018,-25821,-21763,-23776, 24066, + 9502, 25866,-25055, 10004,-24798, 7204, -4581, -9678, 1554, + 10553, 3102, 12193, 2443, 11955, 1213, 10689, -1293, 921, + -4173, 8661, -6049, 8815,-21221,-14854, 23840, -9477, 8549, + 5278, 8743, 9375, 6940, 13038, 4826, 14598, 7721,-24308, + -29905,-19703,-17106,-16124, -3287,-26118,-19709,-10769, 24361, + 28648, 10274,-11292,-29665,-16417, 24346, 14580, 18707, 26323, + -4410,-17711, 5133, 26328,-14579,-31008, 24583, 18404, 28417, + -24350, 19453, 28642,-32513,-10777, -3079, 30188, -7686, 27635, + -32521,-16384,-20240, -6386, 10986, 23827,-25880,-32752,-23321, + 14605, 32231, 780,-13849, 15119, 28647, 4888, -7705, 30750, + 64, 0, 32488, 6687,-20758, 19745, -2070, -1, -1, + 28, 256, -4608, 7168, 7444, 23557,-21998, 13064, 3345, + -4086,-28915, -8594, 32262, 8461, 27387,-12275, 12012, 23563, + -18719,-28410, 29144,-22271,-32562,-16384, 12528, -6386, 10986, + 23827,-25880,-32752,-23321, 14605, 32231, 780,-13849, 15119, + 28647, 4888, -7705, 30750, 64, 0, 32488, 6687,-20758, + 19745, -2070,-13792, -6414, 28188, -2821, -4585, 7168, 7444, + 23557,-21998, 13064, 3345, -4086,-28915, -8694, 32262, 8461, + -14853,-14907,-14907,-14907,-14907, 32767,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14891,-14907,-14907,-14907, + -14907,-14907, 8901, 9375, 6940, 13038, 4826, 14598, 7721, + -24308,-29905,-19703,-17106,-16124, -3287,-26118,-19709,-10769, + 24361, 28648, 10274,-11292,-29665,-16417, 24346, 14580, 18707, + 26323, -4440,-17711, 5133, 26328,-14579,-31008, 24583, 18404, + 28417,-24350, 19453, 28642,-32513,-10777, -3079, 30188, -7686, + 27635,-32521,-16384,-20240, -6386, 10986, 23827,-25880,-32752, + -23321, 14605, 32231, 780,-13849, 15119, 28647, 4888, -7705, + -15074,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907, 8192,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14897,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-15931,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907, 26121, 12890, + 2604, 12228,-11634, 12299, 5573,-16642, 32452, 29182, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, 7710, + 7710,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-10811,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14917,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14938,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907,-14907, + -14907,-14907,-14907,-14907, -571, -9986, -58, 12542,-18491, + 32507, 12838, 6653, 294, -1, 0,-19968, 18368, -4357, + -5696, 2804, 12998,-22802,-22080, -7701, -5183, 486, -3133, + -5660, -1083, 13799,-28726,-11029, 205,-14848, 32464, -1, + -129,-13072, 6380, -7948, 20717, 18425, 17392, 14335, -5646, + 30973, -3598, 7188, -3867, 3055, -4247, 5597, -4011,-26427, + -11,-30418, 7922, 2614, 237, -5839,-27413,-17624,-29716, + -13539, 239, 20991, 18164, -4082,-16647,-27386, 19458, 20224, + 4619, 19728, -7409,-18186,-25073, 27627,-23539, -7945,-31464, + 14330,-12523,-22021, -7701, -5183, 486, -3133, -5660, -1083, + 13799,-28726,-11029, 205,-14848, 32464, -1, -129,-13072, + 6380, -7948, 20717, 18425, 17392, 14093, -5646, 30973, -3598, + 7188, -3867, 3055, 3689, -5401, 4906, -6051, 4250, -6272, + 3492, -6343, 3197, -6397, 4041, -6341, 6255, -6381, 239, + 20991, 18164, -4082,-16647,-27386, 19458, 20224, 4619, 19728, + -7409,-18186,-25073, 27627,-23539, -7945,-31464, 14330,-12523, + 14843, -6130, 30202, -250,-28420, 10467, 16907, -3359, -6644, + 31965, 3343,-11727, 2966,-12616, 3064,-13472, 6732,-12349, + 5541,-11965, 5530,-10820, -1912, -3637, 32285, -4607, 310, + -32768, 0, -5258, 2106, -6857, 2681, -5449, -3606, -6717, + -5482, -3606, -1853, 4082, -7631, -9808, -1742, -2851, -5112, + 64, -868,-13546,-13365,-13365,-13365,-13365,-13365,-13365, + -13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365, + -13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365, + -13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365,-13365, + -13365,-13365,-13365,-13365,-13365,-13365,-13365, 7883, -2316, + 9086, -3944, 10500, 4285, 10459, -6474, 10204, -6539, 11601, + -6824, 13385, -7142, 13872, -7457, 13670, -7725, 13463, -6887, + 12482, -5580, 12600, -4964, 12480, 3254, 11741, -4210,-24819, + 23282, 22798, 491, -1774, -1073, 3641, -1304, 28928, -250, + -27396, 6657, -8961, 22524, 19987, 10231, 1791, 8947,-32763, + -26385,-31227, -792,-30461, 8926, 4866, 27863, 27756, 27756, + 27756, 27756, 27756, 27756, 27756, 27756, 5630,-11070,-16136, + 20671,-11530, 27328, 8179, 5059,-31503,-24379,-19472, 17863, + -29202, 22986, -23, 8909, 8422, 10450 + }; + err = opus_multistream_encode(enc, pcm, 1440, data, 7380); + /* reads uninitialized data at src/opus_multistream_encoder.c:293 */ + assert(err > 0); + } + opus_multistream_encoder_destroy(enc); + return 0; +} + +static int ec_enc_shrink_assert(void) +{ + OpusEncoder *enc; + int err; + int data_len; + unsigned char data[2000]; + static const short pcm1[960] = { 5140 }; + static const short pcm2[2880] = + { + -256,-12033, 0, -2817, 6912, 0, -5359, 5200, 3061, + 0, -2903, 5652, -1281,-24656,-14433,-24678, 32,-29793, + 2870, 0, 4096, 5120, 5140, -234,-20230,-24673,-24633, + -24673,-24705, 0,-32768,-25444,-25444, 0,-25444,-25444, + 156,-20480, -7948, -5920, -7968, -7968, 224, 0, 20480, + 11, 20496, 13, 20496, 11,-20480, 2292,-20240, 244, + 20480, 11, 20496, 11,-20480, 244,-20240, 7156, 20456, + -246,-20243, 244, 128, 244, 20480, 11, 20496, 11, + -20480, 244,-20256, 244, 20480, 256, 0, -246, 16609, + -176, 0, 29872, -4096, -2888, 516, 2896, 4096, 2896, + -20480, -3852, -2896, -1025,-31056,-14433, 244, 1792, -256, + -12033, 0, -2817, 0, 0, -5359, 5200, 3061, 16, + -2903, 5652, -1281,-24656,-14433,-24678, 32,-29793, 2870, + 0, 4096, 5120, 5140, -234,-20230,-24673,-24633,-24673, + -24705, 0,-32768,-25444,-25444, 0,-25444,-25444, 156, + -20480, -7973, -5920, -7968, -7968, 224, 0, 20480, 11, + 20496, 11, 20496, 11,-20480, 2292,-20213, 244, 20480, + 11, 20496, 11,-24698, -2873, 0, 7, -1, 208, + -256, 244, 0, 4352, 20715, -2796, 11,-22272, 5364, + -234,-20230,-24673,-25913, 8351,-24832, 13963, 11, 0, + 16, 5140, 5652, -1281,-24656,-14433,-24673, 32671, 159, + 0,-25472,-25444, 156,-25600,-25444,-25444, 0, -2896, + -7968, -7960, -7968, -7968, 0, 0, 2896, 4096, 2896, + 4096, 2896, 0, -2896, -4088, -2896, 0, 2896, 0, + -2896, -4096, -2896, 11, 2640, -4609, -2896,-32768, -3072, + 0, 2896, 4096, 2896, 0, -2896, -4096, -2896, 0, + 80, 1, 2816, 0, 20656, 255,-20480, 116,-18192 + }; + static const short pcm3[2880] = { 0 }; + + enc = opus_encoder_create(48000, 1, OPUS_APPLICATION_AUDIO, &err); + opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(10)); + opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(6)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(6000)); + data_len = opus_encode(enc, pcm1, 960, data, 2000); + assert(data_len > 0); + + opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + opus_encoder_ctl(enc, OPUS_SET_PREDICTION_DISABLED(1)); + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_SUPERWIDEBAND)); + opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(15600)); + data_len = opus_encode(enc, pcm2, 2880, data, 122); + assert(data_len > 0); + + opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(27000)); + data_len = opus_encode(enc, pcm3, 2880, data, 122); /* assertion failure */ + assert(data_len > 0); + + opus_encoder_destroy(enc); + return 0; +} + +static int ec_enc_shrink_assert2(void) +{ + OpusEncoder *enc; + int err; + int data_len; + unsigned char data[2000]; + + enc = opus_encoder_create(48000, 1, OPUS_APPLICATION_AUDIO, &err); + opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(6)); + opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(26)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(27000)); + { + static const short pcm[960] = { 0 }; + data_len = opus_encode(enc, pcm, 960, data, 2000); + assert(data_len > 0); + } + opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_MUSIC)); + { + static const short pcm[480] = + { + 32767, 32767, 0, 0, 32767, 32767, 0, 0, 32767, 32767, + -32768, -32768, 0, 0, -32768, -32768, 0, 0, -32768, -32768 + }; + data_len = opus_encode(enc, pcm, 480, data, 19); + assert(data_len > 0); + } + opus_encoder_destroy(enc); + return 0; +} + +static int silk_gain_assert(void) +{ + OpusEncoder *enc; + int err; + int data_len; + unsigned char data[1000]; + static const short pcm1[160] = { 0 }; + static const short pcm2[960] = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767 + }; + + enc = opus_encoder_create(8000, 1, OPUS_APPLICATION_AUDIO, &err); + opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(3)); + opus_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(6000)); + data_len = opus_encode(enc, pcm1, 160, data, 1000); + assert(data_len > 0); + + opus_encoder_ctl(enc, OPUS_SET_VBR(0)); + opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(0)); + opus_encoder_ctl(enc, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_MEDIUMBAND)); + opus_encoder_ctl(enc, OPUS_SET_BITRATE(2867)); + data_len = opus_encode(enc, pcm2, 960, data, 1000); + assert(data_len > 0); + + opus_encoder_destroy(enc); + return 0; +} + +void regression_test(void) +{ + fprintf(stderr, "Running simple tests for bugs that have been fixed previously\n"); + celt_ec_internal_error(); + mscbr_encode_fail10(); + mscbr_encode_fail(); + surround_analysis_uninit(); + ec_enc_shrink_assert(); + ec_enc_shrink_assert2(); + silk_gain_assert(); +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/run_vectors.sh b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/run_vectors.sh new file mode 100755 index 000000000..dcb76cf18 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/run_vectors.sh @@ -0,0 +1,143 @@ +#!/bin/sh + +# Copyright (c) 2011-2012 Jean-Marc Valin +# +# This file is extracted from RFC6716. Please see that RFC for additional +# information. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# - Neither the name of Internet Society, IETF or IETF Trust, nor the +# names of specific contributors, may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +rm -f logs_mono.txt logs_mono2.txt +rm -f logs_stereo.txt logs_stereo2.txt + +if [ "$#" -ne "3" ]; then + echo "usage: run_vectors.sh " + exit 1 +fi + +CMD_PATH=$1 +VECTOR_PATH=$2 +RATE=$3 + +: ${OPUS_DEMO:=$CMD_PATH/opus_demo} +: ${OPUS_COMPARE:=$CMD_PATH/opus_compare} + +if [ -d "$VECTOR_PATH" ]; then + echo "Test vectors found in $VECTOR_PATH" +else + echo "No test vectors found" + #Don't make the test fail here because the test vectors + #will be distributed separately + exit 0 +fi + +if [ ! -x "$OPUS_COMPARE" ]; then + echo "ERROR: Compare program not found: $OPUS_COMPARE" + exit 1 +fi + +if [ -x "$OPUS_DEMO" ]; then + echo "Decoding with $OPUS_DEMO" +else + echo "ERROR: Decoder not found: $OPUS_DEMO" + exit 1 +fi + +echo "==============" +echo "Testing mono" +echo "==============" +echo + +for file in 01 02 03 04 05 06 07 08 09 10 11 12 +do + if [ -e "$VECTOR_PATH/testvector$file.bit" ]; then + echo "Testing testvector$file" + else + echo "Bitstream file not found: testvector$file.bit" + fi + if "$OPUS_DEMO" -d "$RATE" 1 "$VECTOR_PATH/testvector$file.bit" tmp.out >> logs_mono.txt 2>&1; then + echo "successfully decoded" + else + echo "ERROR: decoding failed" + exit 1 + fi + "$OPUS_COMPARE" -r "$RATE" "$VECTOR_PATH/testvector${file}.dec" tmp.out >> logs_mono.txt 2>&1 + float_ret=$? + "$OPUS_COMPARE" -r "$RATE" "$VECTOR_PATH/testvector${file}m.dec" tmp.out >> logs_mono2.txt 2>&1 + float_ret2=$? + if [ "$float_ret" -eq "0" ] || [ "$float_ret2" -eq "0" ]; then + echo "output matches reference" + else + echo "ERROR: output does not match reference" + exit 1 + fi + echo +done + +echo "==============" +echo Testing stereo +echo "==============" +echo + +for file in 01 02 03 04 05 06 07 08 09 10 11 12 +do + if [ -e "$VECTOR_PATH/testvector$file.bit" ]; then + echo "Testing testvector$file" + else + echo "Bitstream file not found: testvector$file" + fi + if "$OPUS_DEMO" -d "$RATE" 2 "$VECTOR_PATH/testvector$file.bit" tmp.out >> logs_stereo.txt 2>&1; then + echo "successfully decoded" + else + echo "ERROR: decoding failed" + exit 1 + fi + "$OPUS_COMPARE" -s -r "$RATE" "$VECTOR_PATH/testvector${file}.dec" tmp.out >> logs_stereo.txt 2>&1 + float_ret=$? + "$OPUS_COMPARE" -s -r "$RATE" "$VECTOR_PATH/testvector${file}m.dec" tmp.out >> logs_stereo2.txt 2>&1 + float_ret2=$? + if [ "$float_ret" -eq "0" ] || [ "$float_ret2" -eq "0" ]; then + echo "output matches reference" + else + echo "ERROR: output does not match reference" + exit 1 + fi + echo +done + + + +echo "All tests have passed successfully" +mono1=`grep quality logs_mono.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` +mono2=`grep quality logs_mono2.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` +echo $mono1 $mono2 | awk '{if ($2 > $1) $1 = $2; print "Average mono quality is", $1, "%"}' + +stereo1=`grep quality logs_stereo.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` +stereo2=`grep quality logs_stereo2.txt | awk '{sum+=$4}END{if (NR == 12) sum /= 12; else sum = 0; print sum}'` +echo $stereo1 $stereo2 | awk '{if ($2 > $1) $1 = $2; print "Average stereo quality is", $1, "%"}' diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_api.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_api.c new file mode 100644 index 000000000..fb385c630 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_api.c @@ -0,0 +1,1904 @@ +/* Copyright (c) 2011-2013 Xiph.Org Foundation + Written by Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This tests the API presented by the libopus system. + It does not attempt to extensively exercise the codec internals. + The strategy here is to simply the API interface invariants: + That sane options are accepted, insane options are rejected, + and that nothing blows up. In particular we don't actually test + that settings are heeded by the codec (though we do check that + get after set returns a sane value when it should). Other + tests check the actual codec behavior. + In cases where its reasonable to do so we test exhaustively, + but its not reasonable to do so in all cases. + Although these tests are simple they found several library bugs + when they were initially developed. */ + +/* These tests are more sensitive if compiled with -DVALGRIND and + run inside valgrind. Malloc failure testing requires glibc. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include "arch.h" +#include "opus_multistream.h" +#include "opus.h" +#include "test_opus_common.h" + +#ifdef VALGRIND +#include +#define VG_UNDEF(x,y) VALGRIND_MAKE_MEM_UNDEFINED((x),(y)) +#define VG_CHECK(x,y) VALGRIND_CHECK_MEM_IS_DEFINED((x),(y)) +#else +#define VG_UNDEF(x,y) +#define VG_CHECK(x,y) +#endif + +#if defined(HAVE___MALLOC_HOOK) +#define MALLOC_FAIL +#include "os_support.h" +#include + +static const opus_int32 opus_apps[3] = {OPUS_APPLICATION_VOIP, + OPUS_APPLICATION_AUDIO,OPUS_APPLICATION_RESTRICTED_LOWDELAY}; + +void *malloc_hook(__attribute__((unused)) size_t size, + __attribute__((unused)) const void *caller) +{ + return 0; +} +#endif + +opus_int32 *null_int_ptr = (opus_int32 *)NULL; +opus_uint32 *null_uint_ptr = (opus_uint32 *)NULL; + +static const opus_int32 opus_rates[5] = {48000,24000,16000,12000,8000}; + +opus_int32 test_dec_api(void) +{ + opus_uint32 dec_final_range; + OpusDecoder *dec; + OpusDecoder *dec2; + opus_int32 i,j,cfgs; + unsigned char packet[1276]; +#ifndef DISABLE_FLOAT_API + float fbuf[960*2]; +#endif + short sbuf[960*2]; + int c,err; + + cfgs=0; + /*First test invalid configurations which should fail*/ + fprintf(stdout,"\n Decoder basic API tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); + for(c=0;c<4;c++) + { + i=opus_decoder_get_size(c); + if(((c==1||c==2)&&(i<=2048||i>1<<16))||((c!=1&&c!=2)&&i!=0))test_failed(); + fprintf(stdout," opus_decoder_get_size(%d)=%d ...............%s OK.\n",c,i,i>0?"":"...."); + cfgs++; + } + + /*Test with unsupported sample rates*/ + for(c=0;c<4;c++) + { + for(i=-7;i<=96000;i++) + { + int fs; + if((i==8000||i==12000||i==16000||i==24000||i==48000)&&(c==1||c==2))continue; + switch(i) + { + case(-5):fs=-8000;break; + case(-6):fs=INT32_MAX;break; + case(-7):fs=INT32_MIN;break; + default:fs=i; + } + err = OPUS_OK; + VG_UNDEF(&err,sizeof(err)); + dec = opus_decoder_create(fs, c, &err); + if(err!=OPUS_BAD_ARG || dec!=NULL)test_failed(); + cfgs++; + dec = opus_decoder_create(fs, c, 0); + if(dec!=NULL)test_failed(); + cfgs++; + dec=malloc(opus_decoder_get_size(2)); + if(dec==NULL)test_failed(); + err = opus_decoder_init(dec,fs,c); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + free(dec); + } + } + + VG_UNDEF(&err,sizeof(err)); + dec = opus_decoder_create(48000, 2, &err); + if(err!=OPUS_OK || dec==NULL)test_failed(); + VG_CHECK(dec,opus_decoder_get_size(2)); + cfgs++; + + fprintf(stdout," opus_decoder_create() ........................ OK.\n"); + fprintf(stdout," opus_decoder_init() .......................... OK.\n"); + + err=opus_decoder_ctl(dec, OPUS_GET_FINAL_RANGE(null_uint_ptr)); + if(err != OPUS_BAD_ARG)test_failed(); + VG_UNDEF(&dec_final_range,sizeof(dec_final_range)); + err=opus_decoder_ctl(dec, OPUS_GET_FINAL_RANGE(&dec_final_range)); + if(err!=OPUS_OK)test_failed(); + VG_CHECK(&dec_final_range,sizeof(dec_final_range)); + fprintf(stdout," OPUS_GET_FINAL_RANGE ......................... OK.\n"); + cfgs++; + + err=opus_decoder_ctl(dec,OPUS_UNIMPLEMENTED); + if(err!=OPUS_UNIMPLEMENTED)test_failed(); + fprintf(stdout," OPUS_UNIMPLEMENTED ........................... OK.\n"); + cfgs++; + + err=opus_decoder_ctl(dec, OPUS_GET_BANDWIDTH(null_int_ptr)); + if(err != OPUS_BAD_ARG)test_failed(); + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_BANDWIDTH(&i)); + if(err != OPUS_OK || i!=0)test_failed(); + fprintf(stdout," OPUS_GET_BANDWIDTH ........................... OK.\n"); + cfgs++; + + err=opus_decoder_ctl(dec, OPUS_GET_SAMPLE_RATE(null_int_ptr)); + if(err != OPUS_BAD_ARG)test_failed(); + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_SAMPLE_RATE(&i)); + if(err != OPUS_OK || i!=48000)test_failed(); + fprintf(stdout," OPUS_GET_SAMPLE_RATE ......................... OK.\n"); + cfgs++; + + /*GET_PITCH has different execution paths depending on the previously decoded frame.*/ + err=opus_decoder_ctl(dec, OPUS_GET_PITCH(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + VG_UNDEF(packet,sizeof(packet)); + packet[0]=63<<2;packet[1]=packet[2]=0; + if(opus_decode(dec, packet, 3, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + packet[0]=1; + if(opus_decode(dec, packet, 1, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_PITCH ............................... OK.\n"); + + err=opus_decoder_ctl(dec, OPUS_GET_LAST_PACKET_DURATION(null_int_ptr)); + if(err != OPUS_BAD_ARG)test_failed(); + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_LAST_PACKET_DURATION(&i)); + if(err != OPUS_OK || i!=960)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_LAST_PACKET_DURATION ................ OK.\n"); + + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_GAIN(&i)); + VG_CHECK(&i,sizeof(i)); + if(err != OPUS_OK || i!=0)test_failed(); + cfgs++; + err=opus_decoder_ctl(dec, OPUS_GET_GAIN(null_int_ptr)); + if(err != OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_decoder_ctl(dec, OPUS_SET_GAIN(-32769)); + if(err != OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_decoder_ctl(dec, OPUS_SET_GAIN(32768)); + if(err != OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_decoder_ctl(dec, OPUS_SET_GAIN(-15)); + if(err != OPUS_OK)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(dec, OPUS_GET_GAIN(&i)); + VG_CHECK(&i,sizeof(i)); + if(err != OPUS_OK || i!=-15)test_failed(); + cfgs++; + fprintf(stdout," OPUS_SET_GAIN ................................ OK.\n"); + fprintf(stdout," OPUS_GET_GAIN ................................ OK.\n"); + + /*Reset the decoder*/ + dec2=malloc(opus_decoder_get_size(2)); + memcpy(dec2,dec,opus_decoder_get_size(2)); + if(opus_decoder_ctl(dec, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + if(memcmp(dec2,dec,opus_decoder_get_size(2))==0)test_failed(); + free(dec2); + fprintf(stdout," OPUS_RESET_STATE ............................. OK.\n"); + cfgs++; + + VG_UNDEF(packet,sizeof(packet)); + packet[0]=0; + if(opus_decoder_get_nb_samples(dec,packet,1)!=480)test_failed(); + if(opus_packet_get_nb_samples(packet,1,48000)!=480)test_failed(); + if(opus_packet_get_nb_samples(packet,1,96000)!=960)test_failed(); + if(opus_packet_get_nb_samples(packet,1,32000)!=320)test_failed(); + if(opus_packet_get_nb_samples(packet,1,8000)!=80)test_failed(); + packet[0]=3; + if(opus_packet_get_nb_samples(packet,1,24000)!=OPUS_INVALID_PACKET)test_failed(); + packet[0]=(63<<2)|3; + packet[1]=63; + if(opus_packet_get_nb_samples(packet,0,24000)!=OPUS_BAD_ARG)test_failed(); + if(opus_packet_get_nb_samples(packet,2,48000)!=OPUS_INVALID_PACKET)test_failed(); + if(opus_decoder_get_nb_samples(dec,packet,2)!=OPUS_INVALID_PACKET)test_failed(); + fprintf(stdout," opus_{packet,decoder}_get_nb_samples() ....... OK.\n"); + cfgs+=9; + + if(OPUS_BAD_ARG!=opus_packet_get_nb_frames(packet,0))test_failed(); + for(i=0;i<256;i++) { + int l1res[4]={1,2,2,OPUS_INVALID_PACKET}; + packet[0]=i; + if(l1res[packet[0]&3]!=opus_packet_get_nb_frames(packet,1))test_failed(); + cfgs++; + for(j=0;j<256;j++) { + packet[1]=j; + if(((packet[0]&3)!=3?l1res[packet[0]&3]:packet[1]&63)!=opus_packet_get_nb_frames(packet,2))test_failed(); + cfgs++; + } + } + fprintf(stdout," opus_packet_get_nb_frames() .................. OK.\n"); + + for(i=0;i<256;i++) { + int bw; + packet[0]=i; + bw=packet[0]>>4; + bw=OPUS_BANDWIDTH_NARROWBAND+(((((bw&7)*9)&(63-(bw&8)))+2+12*((bw&8)!=0))>>4); + if(bw!=opus_packet_get_bandwidth(packet))test_failed(); + cfgs++; + } + fprintf(stdout," opus_packet_get_bandwidth() .................. OK.\n"); + + for(i=0;i<256;i++) { + int fp3s,rate; + packet[0]=i; + fp3s=packet[0]>>3; + fp3s=((((3-(fp3s&3))*13&119)+9)>>2)*((fp3s>13)*(3-((fp3s&3)==3))+1)*25; + for(rate=0;rate<5;rate++) { + if((opus_rates[rate]*3/fp3s)!=opus_packet_get_samples_per_frame(packet,opus_rates[rate]))test_failed(); + cfgs++; + } + } + fprintf(stdout," opus_packet_get_samples_per_frame() .......... OK.\n"); + + packet[0]=(63<<2)+3; + packet[1]=49; + for(j=2;j<51;j++)packet[j]=0; + VG_UNDEF(sbuf,sizeof(sbuf)); + if(opus_decode(dec, packet, 51, sbuf, 960, 0)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + packet[0]=(63<<2); + packet[1]=packet[2]=0; + if(opus_decode(dec, packet, -1, sbuf, 960, 0)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_decode(dec, packet, 3, sbuf, 60, 0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(opus_decode(dec, packet, 3, sbuf, 480, 0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(opus_decode(dec, packet, 3, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + fprintf(stdout," opus_decode() ................................ OK.\n"); +#ifndef DISABLE_FLOAT_API + VG_UNDEF(fbuf,sizeof(fbuf)); + if(opus_decode_float(dec, packet, 3, fbuf, 960, 0)!=960)test_failed(); + cfgs++; + fprintf(stdout," opus_decode_float() .......................... OK.\n"); +#endif + +#if 0 + /*These tests are disabled because the library crashes with null states*/ + if(opus_decoder_ctl(0,OPUS_RESET_STATE) !=OPUS_INVALID_STATE)test_failed(); + if(opus_decoder_init(0,48000,1) !=OPUS_INVALID_STATE)test_failed(); + if(opus_decode(0,packet,1,outbuf,2880,0) !=OPUS_INVALID_STATE)test_failed(); + if(opus_decode_float(0,packet,1,0,2880,0) !=OPUS_INVALID_STATE)test_failed(); + if(opus_decoder_get_nb_samples(0,packet,1) !=OPUS_INVALID_STATE)test_failed(); + if(opus_packet_get_nb_frames(NULL,1) !=OPUS_BAD_ARG)test_failed(); + if(opus_packet_get_bandwidth(NULL) !=OPUS_BAD_ARG)test_failed(); + if(opus_packet_get_samples_per_frame(NULL,48000)!=OPUS_BAD_ARG)test_failed(); +#endif + opus_decoder_destroy(dec); + cfgs++; + fprintf(stdout," All decoder interface tests passed\n"); + fprintf(stdout," (%6d API invocations)\n",cfgs); + return cfgs; +} + +opus_int32 test_msdec_api(void) +{ + opus_uint32 dec_final_range; + OpusMSDecoder *dec; + OpusDecoder *streamdec; + opus_int32 i,j,cfgs; + unsigned char packet[1276]; + unsigned char mapping[256]; +#ifndef DISABLE_FLOAT_API + float fbuf[960*2]; +#endif + short sbuf[960*2]; + int a,b,c,err; + + mapping[0]=0; + mapping[1]=1; + for(i=2;i<256;i++)VG_UNDEF(&mapping[i],sizeof(unsigned char)); + + cfgs=0; + /*First test invalid configurations which should fail*/ + fprintf(stdout,"\n Multistream decoder basic API tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); + for(a=-1;a<4;a++) + { + for(b=-1;b<4;b++) + { + i=opus_multistream_decoder_get_size(a,b); + if(((a>0&&b<=a&&b>=0)&&(i<=2048||i>((1<<16)*a)))||((a<1||b>a||b<0)&&i!=0))test_failed(); + fprintf(stdout," opus_multistream_decoder_get_size(%2d,%2d)=%d %sOK.\n",a,b,i,i>0?"":"... "); + cfgs++; + } + } + + /*Test with unsupported sample rates*/ + for(c=1;c<3;c++) + { + for(i=-7;i<=96000;i++) + { + int fs; + if((i==8000||i==12000||i==16000||i==24000||i==48000)&&(c==1||c==2))continue; + switch(i) + { + case(-5):fs=-8000;break; + case(-6):fs=INT32_MAX;break; + case(-7):fs=INT32_MIN;break; + default:fs=i; + } + err = OPUS_OK; + VG_UNDEF(&err,sizeof(err)); + dec = opus_multistream_decoder_create(fs, c, 1, c-1, mapping, &err); + if(err!=OPUS_BAD_ARG || dec!=NULL)test_failed(); + cfgs++; + dec = opus_multistream_decoder_create(fs, c, 1, c-1, mapping, 0); + if(dec!=NULL)test_failed(); + cfgs++; + dec=malloc(opus_multistream_decoder_get_size(1,1)); + if(dec==NULL)test_failed(); + err = opus_multistream_decoder_init(dec,fs,c,1,c-1, mapping); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + free(dec); + } + } + + for(c=0;c<2;c++) + { + int *ret_err; + ret_err = c?0:&err; + + mapping[0]=0; + mapping[1]=1; + for(i=2;i<256;i++)VG_UNDEF(&mapping[i],sizeof(unsigned char)); + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 2, 1, 0, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + mapping[0]=mapping[1]=0; + dec = opus_multistream_decoder_create(48000, 2, 1, 0, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_OK) || dec==NULL)test_failed(); + cfgs++; + opus_multistream_decoder_destroy(dec); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 1, 4, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_OK) || dec==NULL)test_failed(); + cfgs++; + + err = opus_multistream_decoder_init(dec,48000, 1, 0, 0, mapping); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + + err = opus_multistream_decoder_init(dec,48000, 1, 1, -1, mapping); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + + opus_multistream_decoder_destroy(dec); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 2, 1, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_OK) || dec==NULL)test_failed(); + cfgs++; + opus_multistream_decoder_destroy(dec); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 255, 255, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, -1, 1, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 0, 1, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 1, -1, 2, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 1, -1, -1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 256, 255, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + dec = opus_multistream_decoder_create(48000, 256, 255, 0, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + mapping[0]=255; + mapping[1]=1; + mapping[2]=2; + dec = opus_multistream_decoder_create(48000, 3, 2, 0, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + mapping[0]=0; + mapping[1]=0; + mapping[2]=0; + dec = opus_multistream_decoder_create(48000, 3, 2, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_OK) || dec==NULL)test_failed(); + cfgs++; + opus_multistream_decoder_destroy(dec); + cfgs++; + + VG_UNDEF(ret_err,sizeof(*ret_err)); + mapping[0]=0; + mapping[1]=255; + mapping[2]=1; + mapping[3]=2; + mapping[4]=3; + dec = opus_multistream_decoder_create(48001, 5, 4, 1, mapping, ret_err); + if(ret_err){VG_CHECK(ret_err,sizeof(*ret_err));} + if((ret_err && *ret_err!=OPUS_BAD_ARG) || dec!=NULL)test_failed(); + cfgs++; + } + + VG_UNDEF(&err,sizeof(err)); + mapping[0]=0; + mapping[1]=255; + mapping[2]=1; + mapping[3]=2; + dec = opus_multistream_decoder_create(48000, 4, 2, 1, mapping, &err); + VG_CHECK(&err,sizeof(err)); + if(err!=OPUS_OK || dec==NULL)test_failed(); + cfgs++; + + fprintf(stdout," opus_multistream_decoder_create() ............ OK.\n"); + fprintf(stdout," opus_multistream_decoder_init() .............. OK.\n"); + + VG_UNDEF(&dec_final_range,sizeof(dec_final_range)); + err=opus_multistream_decoder_ctl(dec, OPUS_GET_FINAL_RANGE(&dec_final_range)); + if(err!=OPUS_OK)test_failed(); + VG_CHECK(&dec_final_range,sizeof(dec_final_range)); + fprintf(stdout," OPUS_GET_FINAL_RANGE ......................... OK.\n"); + cfgs++; + + streamdec=0; + VG_UNDEF(&streamdec,sizeof(streamdec)); + err=opus_multistream_decoder_ctl(dec, OPUS_MULTISTREAM_GET_DECODER_STATE(-1,&streamdec)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_multistream_decoder_ctl(dec, OPUS_MULTISTREAM_GET_DECODER_STATE(1,&streamdec)); + if(err!=OPUS_OK||streamdec==NULL)test_failed(); + VG_CHECK(streamdec,opus_decoder_get_size(1)); + cfgs++; + err=opus_multistream_decoder_ctl(dec, OPUS_MULTISTREAM_GET_DECODER_STATE(2,&streamdec)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_multistream_decoder_ctl(dec, OPUS_MULTISTREAM_GET_DECODER_STATE(0,&streamdec)); + if(err!=OPUS_OK||streamdec==NULL)test_failed(); + VG_CHECK(streamdec,opus_decoder_get_size(1)); + fprintf(stdout," OPUS_MULTISTREAM_GET_DECODER_STATE ........... OK.\n"); + cfgs++; + + for(j=0;j<2;j++) + { + OpusDecoder *od; + err=opus_multistream_decoder_ctl(dec,OPUS_MULTISTREAM_GET_DECODER_STATE(j,&od)); + if(err != OPUS_OK)test_failed(); + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(od, OPUS_GET_GAIN(&i)); + VG_CHECK(&i,sizeof(i)); + if(err != OPUS_OK || i!=0)test_failed(); + cfgs++; + } + err=opus_multistream_decoder_ctl(dec,OPUS_SET_GAIN(15)); + if(err!=OPUS_OK)test_failed(); + fprintf(stdout," OPUS_SET_GAIN ................................ OK.\n"); + for(j=0;j<2;j++) + { + OpusDecoder *od; + err=opus_multistream_decoder_ctl(dec,OPUS_MULTISTREAM_GET_DECODER_STATE(j,&od)); + if(err != OPUS_OK)test_failed(); + VG_UNDEF(&i,sizeof(i)); + err=opus_decoder_ctl(od, OPUS_GET_GAIN(&i)); + VG_CHECK(&i,sizeof(i)); + if(err != OPUS_OK || i!=15)test_failed(); + cfgs++; + } + fprintf(stdout," OPUS_GET_GAIN ................................ OK.\n"); + + VG_UNDEF(&i,sizeof(i)); + err=opus_multistream_decoder_ctl(dec, OPUS_GET_BANDWIDTH(&i)); + if(err != OPUS_OK || i!=0)test_failed(); + fprintf(stdout," OPUS_GET_BANDWIDTH ........................... OK.\n"); + cfgs++; + + err=opus_multistream_decoder_ctl(dec,OPUS_UNIMPLEMENTED); + if(err!=OPUS_UNIMPLEMENTED)test_failed(); + fprintf(stdout," OPUS_UNIMPLEMENTED ........................... OK.\n"); + cfgs++; + +#if 0 + /*Currently unimplemented for multistream*/ + /*GET_PITCH has different execution paths depending on the previously decoded frame.*/ + err=opus_multistream_decoder_ctl(dec, OPUS_GET_PITCH(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_multistream_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + VG_UNDEF(packet,sizeof(packet)); + packet[0]=63<<2;packet[1]=packet[2]=0; + if(opus_multistream_decode(dec, packet, 3, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_multistream_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + packet[0]=1; + if(opus_multistream_decode(dec, packet, 1, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + err=opus_multistream_decoder_ctl(dec, OPUS_GET_PITCH(&i)); + if(err != OPUS_OK || i>0 || i<-1)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_PITCH ............................... OK.\n"); +#endif + + /*Reset the decoder*/ + if(opus_multistream_decoder_ctl(dec, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + fprintf(stdout," OPUS_RESET_STATE ............................. OK.\n"); + cfgs++; + + opus_multistream_decoder_destroy(dec); + cfgs++; + VG_UNDEF(&err,sizeof(err)); + dec = opus_multistream_decoder_create(48000, 2, 1, 1, mapping, &err); + if(err!=OPUS_OK || dec==NULL)test_failed(); + cfgs++; + + packet[0]=(63<<2)+3; + packet[1]=49; + for(j=2;j<51;j++)packet[j]=0; + VG_UNDEF(sbuf,sizeof(sbuf)); + if(opus_multistream_decode(dec, packet, 51, sbuf, 960, 0)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + packet[0]=(63<<2); + packet[1]=packet[2]=0; + if(opus_multistream_decode(dec, packet, -1, sbuf, 960, 0)!=OPUS_BAD_ARG){printf("%d\n",opus_multistream_decode(dec, packet, -1, sbuf, 960, 0));test_failed();} + cfgs++; + if(opus_multistream_decode(dec, packet, 3, sbuf, -960, 0)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_multistream_decode(dec, packet, 3, sbuf, 60, 0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(opus_multistream_decode(dec, packet, 3, sbuf, 480, 0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(opus_multistream_decode(dec, packet, 3, sbuf, 960, 0)!=960)test_failed(); + cfgs++; + fprintf(stdout," opus_multistream_decode() .................... OK.\n"); +#ifndef DISABLE_FLOAT_API + VG_UNDEF(fbuf,sizeof(fbuf)); + if(opus_multistream_decode_float(dec, packet, 3, fbuf, 960, 0)!=960)test_failed(); + cfgs++; + fprintf(stdout," opus_multistream_decode_float() .............. OK.\n"); +#endif + +#if 0 + /*These tests are disabled because the library crashes with null states*/ + if(opus_multistream_decoder_ctl(0,OPUS_RESET_STATE) !=OPUS_INVALID_STATE)test_failed(); + if(opus_multistream_decoder_init(0,48000,1) !=OPUS_INVALID_STATE)test_failed(); + if(opus_multistream_decode(0,packet,1,outbuf,2880,0) !=OPUS_INVALID_STATE)test_failed(); + if(opus_multistream_decode_float(0,packet,1,0,2880,0) !=OPUS_INVALID_STATE)test_failed(); + if(opus_multistream_decoder_get_nb_samples(0,packet,1) !=OPUS_INVALID_STATE)test_failed(); +#endif + opus_multistream_decoder_destroy(dec); + cfgs++; + fprintf(stdout," All multistream decoder interface tests passed\n"); + fprintf(stdout," (%6d API invocations)\n",cfgs); + return cfgs; +} + +#ifdef VALGRIND +#define UNDEFINE_FOR_PARSE toc=-1; \ + frames[0]=(unsigned char *)0; \ + frames[1]=(unsigned char *)0; \ + payload_offset=-1; \ + VG_UNDEF(&toc,sizeof(toc)); \ + VG_UNDEF(frames,sizeof(frames));\ + VG_UNDEF(&payload_offset,sizeof(payload_offset)); +#else +#define UNDEFINE_FOR_PARSE toc=-1; \ + frames[0]=(unsigned char *)0; \ + frames[1]=(unsigned char *)0; \ + payload_offset=-1; +#endif + +/* This test exercises the heck out of the libopus parser. + It is much larger than the parser itself in part because + it tries to hit a lot of corner cases that could never + fail with the libopus code, but might be problematic for + other implementations. */ +opus_int32 test_parse(void) +{ + opus_int32 i,j,jj,sz; + unsigned char packet[1276]; + opus_int32 cfgs,cfgs_total; + unsigned char toc; + const unsigned char *frames[48]; + short size[48]; + int payload_offset, ret; + fprintf(stdout,"\n Packet header parsing tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); + memset(packet,0,sizeof(char)*1276); + packet[0]=63<<2; + if(opus_packet_parse(packet,1,&toc,frames,0,&payload_offset)!=OPUS_BAD_ARG)test_failed(); + cfgs_total=cfgs=1; + /*code 0*/ + for(i=0;i<64;i++) + { + packet[0]=i<<2; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,4,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=1)test_failed(); + if(size[0]!=3)test_failed(); + if(frames[0]!=packet+1)test_failed(); + } + fprintf(stdout," code 0 (%2d cases) ............................ OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + /*code 1, two frames of the same size*/ + for(i=0;i<64;i++) + { + packet[0]=(i<<2)+1; + for(jj=0;jj<=1275*2+3;jj++) + { + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,jj,&toc,frames,size,&payload_offset); + cfgs++; + if((jj&1)==1 && jj<=2551) + { + /* Must pass if payload length even (packet length odd) and + size<=2551, must fail otherwise. */ + if(ret!=2)test_failed(); + if(size[0]!=size[1] || size[0]!=((jj-1)>>1))test_failed(); + if(frames[0]!=packet+1)test_failed(); + if(frames[1]!=frames[0]+size[0])test_failed(); + if((toc>>2)!=i)test_failed(); + } else if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + } + fprintf(stdout," code 1 (%6d cases) ........................ OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + /*code 2, length code overflow*/ + packet[0]=(i<<2)+2; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + packet[1]=252; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + for(j=0;j<1275;j++) + { + if(j<252)packet[1]=j; + else{packet[1]=252+(j&3);packet[2]=(j-252)>>2;} + /*Code 2, one too short*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,j+(j<252?2:3)-1,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*Code 2, one too long*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,j+(j<252?2:3)+1276,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*Code 2, second zero*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,j+(j<252?2:3),&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=2)test_failed(); + if(size[0]!=j||size[1]!=0)test_failed(); + if(frames[1]!=frames[0]+size[0])test_failed(); + if((toc>>2)!=i)test_failed(); + /*Code 2, normal*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,(j<<1)+4,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=2)test_failed(); + if(size[0]!=j||size[1]!=(j<<1)+3-j-(j<252?1:2))test_failed(); + if(frames[1]!=frames[0]+size[0])test_failed(); + if((toc>>2)!=i)test_failed(); + } + } + fprintf(stdout," code 2 (%6d cases) ........................ OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + packet[0]=(i<<2)+3; + /*code 3, length code overflow*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + fprintf(stdout," code 3 m-truncation (%2d cases) ............... OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + /*code 3, m is zero or 49-63*/ + packet[0]=(i<<2)+3; + for(jj=49;jj<=64;jj++) + { + packet[1]=0+(jj&63); /*CBR, no padding*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1275,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + packet[1]=128+(jj&63); /*VBR, no padding*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1275,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + packet[1]=64+(jj&63); /*CBR, padding*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1275,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + packet[1]=128+64+(jj&63); /*VBR, padding*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1275,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + } + fprintf(stdout," code 3 m=0,49-64 (%2d cases) ................ OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + packet[0]=(i<<2)+3; + /*code 3, m is one, cbr*/ + packet[1]=1; + for(j=0;j<1276;j++) + { + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,j+2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=1)test_failed(); + if(size[0]!=j)test_failed(); + if((toc>>2)!=i)test_failed(); + } + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1276+2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + fprintf(stdout," code 3 m=1 CBR (%2d cases) ................. OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + int frame_samp; + /*code 3, m>1 CBR*/ + packet[0]=(i<<2)+3; + frame_samp=opus_packet_get_samples_per_frame(packet,48000); + for(j=2;j<49;j++) + { + packet[1]=j; + for(sz=2;sz<((j+2)*1275);sz++) + { + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,sz,&toc,frames,size,&payload_offset); + cfgs++; + /*Must be <=120ms, must be evenly divisible, can't have frames>1275 bytes*/ + if(frame_samp*j<=5760 && (sz-2)%j==0 && (sz-2)/j<1276) + { + if(ret!=j)test_failed(); + for(jj=1;jj>2)!=i)test_failed(); + } else if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + } + /*Super jumbo packets*/ + packet[1]=5760/frame_samp; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,1275*packet[1]+2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=packet[1])test_failed(); + for(jj=0;jj>2)!=i)test_failed(); + } + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+1276,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + for(j=2;j<49;j++) + { + packet[1]=128+j; + /*Length code overflow*/ + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+j-2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + packet[2]=252; + packet[3]=0; + for(jj=4;jj<2+j;jj++)packet[jj]=0; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+j,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*One byte too short*/ + for(jj=2;jj<2+j;jj++)packet[jj]=0; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+j-2,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*One byte too short thanks to length coding*/ + packet[2]=252; + packet[3]=0; + for(jj=4;jj<2+j;jj++)packet[jj]=0; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+j+252-1,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*Most expensive way of coding zeros*/ + for(jj=2;jj<2+j;jj++)packet[jj]=0; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,2+j-1,&toc,frames,size,&payload_offset); + cfgs++; + if(frame_samp*j<=5760){ + if(ret!=j)test_failed(); + for(jj=0;jj>2)!=i)test_failed(); + } else if(ret!=OPUS_INVALID_PACKET)test_failed(); + /*Quasi-CBR use of mode 3*/ + for(sz=0;sz<8;sz++) + { + const int tsz[8]={50,201,403,700,1472,5110,20400,61298}; + int pos=0; + int as=(tsz[sz]+i-j-2)/j; + for(jj=0;jj>2;pos+=2;} + } + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,tsz[sz]+i,&toc,frames,size,&payload_offset); + cfgs++; + if(frame_samp*j<=5760 && as<1276 && (tsz[sz]+i-2-pos-as*(j-1))<1276){ + if(ret!=j)test_failed(); + for(jj=0;jj>2)!=i)test_failed(); + } else if(ret!=OPUS_INVALID_PACKET)test_failed(); + } + } + } + fprintf(stdout," code 3 m=1-48 VBR (%2d cases) ............. OK.\n",cfgs); + cfgs_total+=cfgs;cfgs=0; + + for(i=0;i<64;i++) + { + packet[0]=(i<<2)+3; + /*Padding*/ + packet[1]=128+1+64; + /*Overflow the length coding*/ + for(jj=2;jj<127;jj++)packet[jj]=255; + UNDEFINE_FOR_PARSE + ret=opus_packet_parse(packet,127,&toc,frames,size,&payload_offset); + cfgs++; + if(ret!=OPUS_INVALID_PACKET)test_failed(); + + for(sz=0;sz<4;sz++) + { + const int tsz[4]={0,72,512,1275}; + for(jj=sz;jj<65025;jj+=11) + { + int pos; + for(pos=0;pos>2)!=i)test_failed(); + } else if (ret!=OPUS_INVALID_PACKET)test_failed(); + } + } + } + fprintf(stdout," code 3 padding (%2d cases) ............... OK.\n",cfgs); + cfgs_total+=cfgs; + fprintf(stdout," opus_packet_parse ............................ OK.\n"); + fprintf(stdout," All packet parsing tests passed\n"); + fprintf(stdout," (%d API invocations)\n",cfgs_total); + return cfgs_total; +} + +/* This is a helper macro for the encoder tests. + The encoder api tests all have a pattern of set-must-fail, set-must-fail, + set-must-pass, get-and-compare, set-must-pass, get-and-compare. */ +#define CHECK_SETGET(setcall,getcall,badv,badv2,goodv,goodv2,sok,gok) \ + i=(badv);\ + if(opus_encoder_ctl(enc,setcall)==OPUS_OK)test_failed();\ + i=(badv2);\ + if(opus_encoder_ctl(enc,setcall)==OPUS_OK)test_failed();\ + j=i=(goodv);\ + if(opus_encoder_ctl(enc,setcall)!=OPUS_OK)test_failed();\ + i=-12345;\ + VG_UNDEF(&i,sizeof(i)); \ + err=opus_encoder_ctl(enc,getcall);\ + if(err!=OPUS_OK || i!=j)test_failed();\ + j=i=(goodv2);\ + if(opus_encoder_ctl(enc,setcall)!=OPUS_OK)test_failed();\ + fprintf(stdout,sok);\ + i=-12345;\ + VG_UNDEF(&i,sizeof(i)); \ + err=opus_encoder_ctl(enc,getcall);\ + if(err!=OPUS_OK || i!=j)test_failed();\ + fprintf(stdout,gok);\ + cfgs+=6; + +opus_int32 test_enc_api(void) +{ + opus_uint32 enc_final_range; + OpusEncoder *enc; + opus_int32 i,j; + unsigned char packet[1276]; +#ifndef DISABLE_FLOAT_API + float fbuf[960*2]; +#endif + short sbuf[960*2]; + int c,err,cfgs; + + cfgs=0; + /*First test invalid configurations which should fail*/ + fprintf(stdout,"\n Encoder basic API tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); + for(c=0;c<4;c++) + { + i=opus_encoder_get_size(c); + if(((c==1||c==2)&&(i<=2048||i>1<<17))||((c!=1&&c!=2)&&i!=0))test_failed(); + fprintf(stdout," opus_encoder_get_size(%d)=%d ...............%s OK.\n",c,i,i>0?"":"...."); + cfgs++; + } + + /*Test with unsupported sample rates, channel counts*/ + for(c=0;c<4;c++) + { + for(i=-7;i<=96000;i++) + { + int fs; + if((i==8000||i==12000||i==16000||i==24000||i==48000)&&(c==1||c==2))continue; + switch(i) + { + case(-5):fs=-8000;break; + case(-6):fs=INT32_MAX;break; + case(-7):fs=INT32_MIN;break; + default:fs=i; + } + err = OPUS_OK; + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(fs, c, OPUS_APPLICATION_VOIP, &err); + if(err!=OPUS_BAD_ARG || enc!=NULL)test_failed(); + cfgs++; + enc = opus_encoder_create(fs, c, OPUS_APPLICATION_VOIP, 0); + if(enc!=NULL)test_failed(); + cfgs++; + opus_encoder_destroy(enc); + enc=malloc(opus_encoder_get_size(2)); + if(enc==NULL)test_failed(); + err = opus_encoder_init(enc, fs, c, OPUS_APPLICATION_VOIP); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + free(enc); + } + } + + enc = opus_encoder_create(48000, 2, OPUS_AUTO, NULL); + if(enc!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(48000, 2, OPUS_AUTO, &err); + if(err!=OPUS_BAD_ARG || enc!=NULL)test_failed(); + cfgs++; + + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(48000, 2, OPUS_APPLICATION_VOIP, NULL); + if(enc==NULL)test_failed(); + opus_encoder_destroy(enc); + cfgs++; + + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(48000, 2, OPUS_APPLICATION_RESTRICTED_LOWDELAY, &err); + if(err!=OPUS_OK || enc==NULL)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_LOOKAHEAD(&i)); + if(err!=OPUS_OK || i<0 || i>32766)test_failed(); + cfgs++; + opus_encoder_destroy(enc); + + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &err); + if(err!=OPUS_OK || enc==NULL)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_LOOKAHEAD(&i)); + if(err!=OPUS_OK || i<0 || i>32766)test_failed(); + opus_encoder_destroy(enc); + cfgs++; + + VG_UNDEF(&err,sizeof(err)); + enc = opus_encoder_create(48000, 2, OPUS_APPLICATION_VOIP, &err); + if(err!=OPUS_OK || enc==NULL)test_failed(); + cfgs++; + + fprintf(stdout," opus_encoder_create() ........................ OK.\n"); + fprintf(stdout," opus_encoder_init() .......................... OK.\n"); + + i=-12345; + VG_UNDEF(&i,sizeof(i)); + err=opus_encoder_ctl(enc,OPUS_GET_LOOKAHEAD(&i)); + if(err!=OPUS_OK || i<0 || i>32766)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_LOOKAHEAD(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_LOOKAHEAD ........................... OK.\n"); + + err=opus_encoder_ctl(enc,OPUS_GET_SAMPLE_RATE(&i)); + if(err!=OPUS_OK || i!=48000)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_SAMPLE_RATE(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_SAMPLE_RATE ......................... OK.\n"); + + if(opus_encoder_ctl(enc,OPUS_UNIMPLEMENTED)!=OPUS_UNIMPLEMENTED)test_failed(); + fprintf(stdout," OPUS_UNIMPLEMENTED ........................... OK.\n"); + cfgs++; + + err=opus_encoder_ctl(enc,OPUS_GET_APPLICATION(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_APPLICATION(i),OPUS_GET_APPLICATION(&i),-1,OPUS_AUTO, + OPUS_APPLICATION_AUDIO,OPUS_APPLICATION_RESTRICTED_LOWDELAY, + " OPUS_SET_APPLICATION ......................... OK.\n", + " OPUS_GET_APPLICATION ......................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_BITRATE(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_encoder_ctl(enc,OPUS_SET_BITRATE(1073741832))!=OPUS_OK)test_failed(); + cfgs++; + VG_UNDEF(&i,sizeof(i)); + if(opus_encoder_ctl(enc,OPUS_GET_BITRATE(&i))!=OPUS_OK)test_failed(); + if(i>700000||i<256000)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_BITRATE(i),OPUS_GET_BITRATE(&i),-12345,0, + 500,256000, + " OPUS_SET_BITRATE ............................. OK.\n", + " OPUS_GET_BITRATE ............................. OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_FORCE_CHANNELS(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_FORCE_CHANNELS(i),OPUS_GET_FORCE_CHANNELS(&i),-1,3, + 1,OPUS_AUTO, + " OPUS_SET_FORCE_CHANNELS ...................... OK.\n", + " OPUS_GET_FORCE_CHANNELS ...................... OK.\n") + + i=-2; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))==OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_FULLBAND+1; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))==OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_NARROWBAND; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_FULLBAND; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_WIDEBAND; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_MEDIUMBAND; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + fprintf(stdout," OPUS_SET_BANDWIDTH ........................... OK.\n"); + /*We don't test if the bandwidth has actually changed. + because the change may be delayed until the encoder is advanced.*/ + i=-12345; + VG_UNDEF(&i,sizeof(i)); + err=opus_encoder_ctl(enc,OPUS_GET_BANDWIDTH(&i)); + if(err!=OPUS_OK || (i!=OPUS_BANDWIDTH_NARROWBAND&& + i!=OPUS_BANDWIDTH_MEDIUMBAND&&i!=OPUS_BANDWIDTH_WIDEBAND&& + i!=OPUS_BANDWIDTH_FULLBAND&&i!=OPUS_AUTO))test_failed(); + cfgs++; + if(opus_encoder_ctl(enc,OPUS_SET_BANDWIDTH(OPUS_AUTO))!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_BANDWIDTH(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_BANDWIDTH ........................... OK.\n"); + + i=-2; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))==OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_FULLBAND+1; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))==OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_NARROWBAND; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_FULLBAND; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_WIDEBAND; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + i=OPUS_BANDWIDTH_MEDIUMBAND; + if(opus_encoder_ctl(enc,OPUS_SET_MAX_BANDWIDTH(i))!=OPUS_OK)test_failed(); + cfgs++; + fprintf(stdout," OPUS_SET_MAX_BANDWIDTH ....................... OK.\n"); + /*We don't test if the bandwidth has actually changed. + because the change may be delayed until the encoder is advanced.*/ + i=-12345; + VG_UNDEF(&i,sizeof(i)); + err=opus_encoder_ctl(enc,OPUS_GET_MAX_BANDWIDTH(&i)); + if(err!=OPUS_OK || (i!=OPUS_BANDWIDTH_NARROWBAND&& + i!=OPUS_BANDWIDTH_MEDIUMBAND&&i!=OPUS_BANDWIDTH_WIDEBAND&& + i!=OPUS_BANDWIDTH_FULLBAND))test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_MAX_BANDWIDTH(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_MAX_BANDWIDTH ....................... OK.\n"); + + err=opus_encoder_ctl(enc,OPUS_GET_DTX(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_DTX(i),OPUS_GET_DTX(&i),-1,2, + 1,0, + " OPUS_SET_DTX ................................. OK.\n", + " OPUS_GET_DTX ................................. OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_COMPLEXITY(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_COMPLEXITY(i),OPUS_GET_COMPLEXITY(&i),-1,11, + 0,10, + " OPUS_SET_COMPLEXITY .......................... OK.\n", + " OPUS_GET_COMPLEXITY .......................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_INBAND_FEC(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_INBAND_FEC(i),OPUS_GET_INBAND_FEC(&i),-1,2, + 1,0, + " OPUS_SET_INBAND_FEC .......................... OK.\n", + " OPUS_GET_INBAND_FEC .......................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_PACKET_LOSS_PERC(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_PACKET_LOSS_PERC(i),OPUS_GET_PACKET_LOSS_PERC(&i),-1,101, + 100,0, + " OPUS_SET_PACKET_LOSS_PERC .................... OK.\n", + " OPUS_GET_PACKET_LOSS_PERC .................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_VBR(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_VBR(i),OPUS_GET_VBR(&i),-1,2, + 1,0, + " OPUS_SET_VBR ................................. OK.\n", + " OPUS_GET_VBR ................................. OK.\n") + +/* err=opus_encoder_ctl(enc,OPUS_GET_VOICE_RATIO(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_VOICE_RATIO(i),OPUS_GET_VOICE_RATIO(&i),-2,101, + 0,50, + " OPUS_SET_VOICE_RATIO ......................... OK.\n", + " OPUS_GET_VOICE_RATIO ......................... OK.\n")*/ + + err=opus_encoder_ctl(enc,OPUS_GET_VBR_CONSTRAINT(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_VBR_CONSTRAINT(i),OPUS_GET_VBR_CONSTRAINT(&i),-1,2, + 1,0, + " OPUS_SET_VBR_CONSTRAINT ...................... OK.\n", + " OPUS_GET_VBR_CONSTRAINT ...................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_SIGNAL(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_SIGNAL(i),OPUS_GET_SIGNAL(&i),-12345,0x7FFFFFFF, + OPUS_SIGNAL_MUSIC,OPUS_AUTO, + " OPUS_SET_SIGNAL .............................. OK.\n", + " OPUS_GET_SIGNAL .............................. OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_LSB_DEPTH(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_LSB_DEPTH(i),OPUS_GET_LSB_DEPTH(&i),7,25,16,24, + " OPUS_SET_LSB_DEPTH ........................... OK.\n", + " OPUS_GET_LSB_DEPTH ........................... OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_PREDICTION_DISABLED(&i)); + if(i!=0)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_GET_PREDICTION_DISABLED(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_PREDICTION_DISABLED(i),OPUS_GET_PREDICTION_DISABLED(&i),-1,2,1,0, + " OPUS_SET_PREDICTION_DISABLED ................. OK.\n", + " OPUS_GET_PREDICTION_DISABLED ................. OK.\n") + + err=opus_encoder_ctl(enc,OPUS_GET_EXPERT_FRAME_DURATION(null_int_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_2_5_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_5_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_10_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_20_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_40_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_60_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_80_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_100_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_120_MS)); + if(err!=OPUS_OK)test_failed(); + cfgs++; + CHECK_SETGET(OPUS_SET_EXPERT_FRAME_DURATION(i),OPUS_GET_EXPERT_FRAME_DURATION(&i),0,-1, + OPUS_FRAMESIZE_60_MS,OPUS_FRAMESIZE_ARG, + " OPUS_SET_EXPERT_FRAME_DURATION ............... OK.\n", + " OPUS_GET_EXPERT_FRAME_DURATION ............... OK.\n") + + /*OPUS_SET_FORCE_MODE is not tested here because it's not a public API, however the encoder tests use it*/ + + err=opus_encoder_ctl(enc,OPUS_GET_FINAL_RANGE(null_uint_ptr)); + if(err!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_encoder_ctl(enc,OPUS_GET_FINAL_RANGE(&enc_final_range))!=OPUS_OK)test_failed(); + cfgs++; + fprintf(stdout," OPUS_GET_FINAL_RANGE ......................... OK.\n"); + + /*Reset the encoder*/ + if(opus_encoder_ctl(enc, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + cfgs++; + fprintf(stdout," OPUS_RESET_STATE ............................. OK.\n"); + + memset(sbuf,0,sizeof(short)*2*960); + VG_UNDEF(packet,sizeof(packet)); + i=opus_encode(enc, sbuf, 960, packet, sizeof(packet)); + if(i<1 || (i>(opus_int32)sizeof(packet)))test_failed(); + VG_CHECK(packet,i); + cfgs++; + fprintf(stdout," opus_encode() ................................ OK.\n"); +#ifndef DISABLE_FLOAT_API + memset(fbuf,0,sizeof(float)*2*960); + VG_UNDEF(packet,sizeof(packet)); + i=opus_encode_float(enc, fbuf, 960, packet, sizeof(packet)); + if(i<1 || (i>(opus_int32)sizeof(packet)))test_failed(); + VG_CHECK(packet,i); + cfgs++; + fprintf(stdout," opus_encode_float() .......................... OK.\n"); +#endif + +#if 0 + /*These tests are disabled because the library crashes with null states*/ + if(opus_encoder_ctl(0,OPUS_RESET_STATE) !=OPUS_INVALID_STATE)test_failed(); + if(opus_encoder_init(0,48000,1,OPUS_APPLICATION_VOIP) !=OPUS_INVALID_STATE)test_failed(); + if(opus_encode(0,sbuf,960,packet,sizeof(packet)) !=OPUS_INVALID_STATE)test_failed(); + if(opus_encode_float(0,fbuf,960,packet,sizeof(packet))!=OPUS_INVALID_STATE)test_failed(); +#endif + opus_encoder_destroy(enc); + cfgs++; + fprintf(stdout," All encoder interface tests passed\n"); + fprintf(stdout," (%d API invocations)\n",cfgs); + return cfgs; +} + +#define max_out (1276*48+48*2+2) +int test_repacketizer_api(void) +{ + int ret,cfgs,i,j,k; + OpusRepacketizer *rp; + unsigned char *packet; + unsigned char *po; + cfgs=0; + fprintf(stdout,"\n Repacketizer tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); + + packet=malloc(max_out); + if(packet==NULL)test_failed(); + memset(packet,0,max_out); + po=malloc(max_out+256); + if(po==NULL)test_failed(); + + i=opus_repacketizer_get_size(); + if(i<=0)test_failed(); + cfgs++; + fprintf(stdout," opus_repacketizer_get_size()=%d ............. OK.\n",i); + + rp=malloc(i); + rp=opus_repacketizer_init(rp); + if(rp==NULL)test_failed(); + cfgs++; + free(rp); + fprintf(stdout," opus_repacketizer_init ....................... OK.\n"); + + rp=opus_repacketizer_create(); + if(rp==NULL)test_failed(); + cfgs++; + fprintf(stdout," opus_repacketizer_create ..................... OK.\n"); + + if(opus_repacketizer_get_nb_frames(rp)!=0)test_failed(); + cfgs++; + fprintf(stdout," opus_repacketizer_get_nb_frames .............. OK.\n"); + + /*Length overflows*/ + VG_UNDEF(packet,4); + if(opus_repacketizer_cat(rp,packet,0)!=OPUS_INVALID_PACKET)test_failed(); /* Zero len */ + cfgs++; + packet[0]=1; + if(opus_repacketizer_cat(rp,packet,2)!=OPUS_INVALID_PACKET)test_failed(); /* Odd payload code 1 */ + cfgs++; + packet[0]=2; + if(opus_repacketizer_cat(rp,packet,1)!=OPUS_INVALID_PACKET)test_failed(); /* Code 2 overflow one */ + cfgs++; + packet[0]=3; + if(opus_repacketizer_cat(rp,packet,1)!=OPUS_INVALID_PACKET)test_failed(); /* Code 3 no count */ + cfgs++; + packet[0]=2; + packet[1]=255; + if(opus_repacketizer_cat(rp,packet,2)!=OPUS_INVALID_PACKET)test_failed(); /* Code 2 overflow two */ + cfgs++; + packet[0]=2; + packet[1]=250; + if(opus_repacketizer_cat(rp,packet,251)!=OPUS_INVALID_PACKET)test_failed(); /* Code 2 overflow three */ + cfgs++; + packet[0]=3; + packet[1]=0; + if(opus_repacketizer_cat(rp,packet,2)!=OPUS_INVALID_PACKET)test_failed(); /* Code 3 m=0 */ + cfgs++; + packet[1]=49; + if(opus_repacketizer_cat(rp,packet,100)!=OPUS_INVALID_PACKET)test_failed(); /* Code 3 m=49 */ + cfgs++; + packet[0]=0; + if(opus_repacketizer_cat(rp,packet,3)!=OPUS_OK)test_failed(); + cfgs++; + packet[0]=1<<2; + if(opus_repacketizer_cat(rp,packet,3)!=OPUS_INVALID_PACKET)test_failed(); /* Change in TOC */ + cfgs++; + + /* Code 0,1,3 CBR -> Code 0,1,3 CBR */ + opus_repacketizer_init(rp); + for(j=0;j<32;j++) + { + /* TOC types, test half with stereo */ + int maxi; + packet[0]=((j<<1)+(j&1))<<2; + maxi=960/opus_packet_get_samples_per_frame(packet,8000); + for(i=1;i<=maxi;i++) + { + /* Number of CBR frames in the input packets */ + int maxp; + packet[0]=((j<<1)+(j&1))<<2; + if(i>1)packet[0]+=i==2?1:3; + packet[1]=i>2?i:0; + maxp=960/(i*opus_packet_get_samples_per_frame(packet,8000)); + for(k=0;k<=(1275+75);k+=3) + { + /*Payload size*/ + opus_int32 cnt,rcnt; + if(k%i!=0)continue; /* Only testing CBR here, payload must be a multiple of the count */ + for(cnt=0;cnt0) + { + ret=opus_repacketizer_cat(rp,packet,k+(i>2?2:1)); + if((cnt<=maxp&&k<=(1275*i))?ret!=OPUS_OK:ret!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + } + rcnt=k<=(1275*i)?(cnt0) + { + int len; + len=k*rcnt+((rcnt*i)>2?2:1); + if(ret!=len)test_failed(); + if((rcnt*i)<2&&(po[0]&3)!=0)test_failed(); /* Code 0 */ + if((rcnt*i)==2&&(po[0]&3)!=1)test_failed(); /* Code 1 */ + if((rcnt*i)>2&&(((po[0]&3)!=3)||(po[1]!=rcnt*i)))test_failed(); /* Code 3 CBR */ + cfgs++; + if(opus_repacketizer_out(rp,po,len)!=len)test_failed(); + cfgs++; + if(opus_packet_unpad(po,len)!=len)test_failed(); + cfgs++; + if(opus_packet_pad(po,len,len+1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_packet_pad(po,len+1,len+256)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_packet_unpad(po,len+256)!=len)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,len,1)!=len)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,len,len+1,1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,len+1,len+256,1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,len+256,1)!=len)test_failed(); + cfgs++; + if(opus_repacketizer_out(rp,po,len-1)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(len>1) + { + if(opus_repacketizer_out(rp,po,1)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + } + if(opus_repacketizer_out(rp,po,0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + } else if (ret!=OPUS_BAD_ARG)test_failed(); /* M must not be 0 */ + } + opus_repacketizer_init(rp); + } + } + } + + /*Change in input count code, CBR out*/ + opus_repacketizer_init(rp); + packet[0]=0; + if(opus_repacketizer_cat(rp,packet,5)!=OPUS_OK)test_failed(); + cfgs++; + packet[0]+=1; + if(opus_repacketizer_cat(rp,packet,9)!=OPUS_OK)test_failed(); + cfgs++; + i=opus_repacketizer_out(rp,po,max_out); + if((i!=(4+8+2))||((po[0]&3)!=3)||((po[1]&63)!=3)||((po[1]>>7)!=0))test_failed(); + cfgs++; + i=opus_repacketizer_out_range(rp,0,1,po,max_out); + if(i!=5||(po[0]&3)!=0)test_failed(); + cfgs++; + i=opus_repacketizer_out_range(rp,1,2,po,max_out); + if(i!=5||(po[0]&3)!=0)test_failed(); + cfgs++; + + /*Change in input count code, VBR out*/ + opus_repacketizer_init(rp); + packet[0]=1; + if(opus_repacketizer_cat(rp,packet,9)!=OPUS_OK)test_failed(); + cfgs++; + packet[0]=0; + if(opus_repacketizer_cat(rp,packet,3)!=OPUS_OK)test_failed(); + cfgs++; + i=opus_repacketizer_out(rp,po,max_out); + if((i!=(2+8+2+2))||((po[0]&3)!=3)||((po[1]&63)!=3)||((po[1]>>7)!=1))test_failed(); + cfgs++; + + /*VBR in, VBR out*/ + opus_repacketizer_init(rp); + packet[0]=2; + packet[1]=4; + if(opus_repacketizer_cat(rp,packet,8)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_repacketizer_cat(rp,packet,8)!=OPUS_OK)test_failed(); + cfgs++; + i=opus_repacketizer_out(rp,po,max_out); + if((i!=(2+1+1+1+4+2+4+2))||((po[0]&3)!=3)||((po[1]&63)!=4)||((po[1]>>7)!=1))test_failed(); + cfgs++; + + /*VBR in, CBR out*/ + opus_repacketizer_init(rp); + packet[0]=2; + packet[1]=4; + if(opus_repacketizer_cat(rp,packet,10)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_repacketizer_cat(rp,packet,10)!=OPUS_OK)test_failed(); + cfgs++; + i=opus_repacketizer_out(rp,po,max_out); + if((i!=(2+4+4+4+4))||((po[0]&3)!=3)||((po[1]&63)!=4)||((po[1]>>7)!=0))test_failed(); + cfgs++; + + /*Count 0 in, VBR out*/ + for(j=0;j<32;j++) + { + /* TOC types, test half with stereo */ + int maxi,sum,rcnt; + packet[0]=((j<<1)+(j&1))<<2; + maxi=960/opus_packet_get_samples_per_frame(packet,8000); + sum=0; + rcnt=0; + opus_repacketizer_init(rp); + for(i=1;i<=maxi+2;i++) + { + int len; + ret=opus_repacketizer_cat(rp,packet,i); + if(rcnt2&&(po[1]&63)!=rcnt)test_failed(); + if(rcnt==2&&(po[0]&3)!=2)test_failed(); + if(rcnt==1&&(po[0]&3)!=0)test_failed(); + cfgs++; + if(opus_repacketizer_out(rp,po,len)!=len)test_failed(); + cfgs++; + if(opus_packet_unpad(po,len)!=len)test_failed(); + cfgs++; + if(opus_packet_pad(po,len,len+1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_packet_pad(po,len+1,len+256)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_packet_unpad(po,len+256)!=len)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,len,1)!=len)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,len,len+1,1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,len+1,len+256,1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,len+256,1)!=len)test_failed(); + cfgs++; + if(opus_repacketizer_out(rp,po,len-1)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + if(len>1) + { + if(opus_repacketizer_out(rp,po,1)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + } + if(opus_repacketizer_out(rp,po,0)!=OPUS_BUFFER_TOO_SMALL)test_failed(); + cfgs++; + } + } + + po[0]='O'; + po[1]='p'; + if(opus_packet_pad(po,4,4)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,4,4,1)!=OPUS_OK)test_failed(); + cfgs++; + if(opus_packet_pad(po,4,5)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,4,5,1)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + if(opus_packet_pad(po,0,5)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,0,5,1)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_packet_unpad(po,0)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,0,1)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_packet_unpad(po,4)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + if(opus_multistream_packet_unpad(po,4,1)!=OPUS_INVALID_PACKET)test_failed(); + cfgs++; + po[0]=0; + po[1]=0; + po[2]=0; + if(opus_packet_pad(po,5,4)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + if(opus_multistream_packet_pad(po,5,4,1)!=OPUS_BAD_ARG)test_failed(); + cfgs++; + + fprintf(stdout," opus_repacketizer_cat ........................ OK.\n"); + fprintf(stdout," opus_repacketizer_out ........................ OK.\n"); + fprintf(stdout," opus_repacketizer_out_range .................. OK.\n"); + fprintf(stdout," opus_packet_pad .............................. OK.\n"); + fprintf(stdout," opus_packet_unpad ............................ OK.\n"); + fprintf(stdout," opus_multistream_packet_pad .................. OK.\n"); + fprintf(stdout," opus_multistream_packet_unpad ................ OK.\n"); + + opus_repacketizer_destroy(rp); + cfgs++; + free(packet); + free(po); + fprintf(stdout," All repacketizer tests passed\n"); + fprintf(stdout," (%7d API invocations)\n",cfgs); + + return cfgs; +} + +#ifdef MALLOC_FAIL +/* GLIBC 2.14 declares __malloc_hook as deprecated, generating a warning + * under GCC. However, this is the cleanest way to test malloc failure + * handling in our codebase, and the lack of thread safety isn't an + * issue here. We therefore disable the warning for this function. + */ +#if OPUS_GNUC_PREREQ(4,6) +/* Save the current warning settings */ +#pragma GCC diagnostic push +#endif +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +typedef void *(*mhook)(size_t __size, __const void *); +#endif + +int test_malloc_fail(void) +{ +#ifdef MALLOC_FAIL + OpusDecoder *dec; + OpusEncoder *enc; + OpusRepacketizer *rp; + unsigned char mapping[256] = {0,1}; + OpusMSDecoder *msdec; + OpusMSEncoder *msenc; + int rate,c,app,cfgs,err,useerr; + int *ep; + mhook orig_malloc; + cfgs=0; +#endif + fprintf(stdout,"\n malloc() failure tests\n"); + fprintf(stdout," ---------------------------------------------------\n"); +#ifdef MALLOC_FAIL + orig_malloc=__malloc_hook; + __malloc_hook=malloc_hook; + ep=(int *)opus_alloc(sizeof(int)); + if(ep!=NULL) + { + if(ep)free(ep); + __malloc_hook=orig_malloc; +#endif + fprintf(stdout," opus_decoder_create() ................... SKIPPED.\n"); + fprintf(stdout," opus_encoder_create() ................... SKIPPED.\n"); + fprintf(stdout," opus_repacketizer_create() .............. SKIPPED.\n"); + fprintf(stdout," opus_multistream_decoder_create() ....... SKIPPED.\n"); + fprintf(stdout," opus_multistream_encoder_create() ....... SKIPPED.\n"); + fprintf(stdout,"(Test only supported with GLIBC and without valgrind)\n"); + return 0; +#ifdef MALLOC_FAIL + } + for(useerr=0;useerr<2;useerr++) + { + ep=useerr?&err:0; + for(rate=0;rate<5;rate++) + { + for(c=1;c<3;c++) + { + err=1; + if(useerr) + { + VG_UNDEF(&err,sizeof(err)); + } + dec=opus_decoder_create(opus_rates[rate], c, ep); + if(dec!=NULL||(useerr&&err!=OPUS_ALLOC_FAIL)) + { + __malloc_hook=orig_malloc; + test_failed(); + } + cfgs++; + msdec=opus_multistream_decoder_create(opus_rates[rate], c, 1, c-1, mapping, ep); + if(msdec!=NULL||(useerr&&err!=OPUS_ALLOC_FAIL)) + { + __malloc_hook=orig_malloc; + test_failed(); + } + cfgs++; + for(app=0;app<3;app++) + { + if(useerr) + { + VG_UNDEF(&err,sizeof(err)); + } + enc=opus_encoder_create(opus_rates[rate], c, opus_apps[app],ep); + if(enc!=NULL||(useerr&&err!=OPUS_ALLOC_FAIL)) + { + __malloc_hook=orig_malloc; + test_failed(); + } + cfgs++; + msenc=opus_multistream_encoder_create(opus_rates[rate], c, 1, c-1, mapping, opus_apps[app],ep); + if(msenc!=NULL||(useerr&&err!=OPUS_ALLOC_FAIL)) + { + __malloc_hook=orig_malloc; + test_failed(); + } + cfgs++; + } + } + } + } + rp=opus_repacketizer_create(); + if(rp!=NULL) + { + __malloc_hook=orig_malloc; + test_failed(); + } + cfgs++; + __malloc_hook=orig_malloc; + fprintf(stdout," opus_decoder_create() ........................ OK.\n"); + fprintf(stdout," opus_encoder_create() ........................ OK.\n"); + fprintf(stdout," opus_repacketizer_create() ................... OK.\n"); + fprintf(stdout," opus_multistream_decoder_create() ............ OK.\n"); + fprintf(stdout," opus_multistream_encoder_create() ............ OK.\n"); + fprintf(stdout," All malloc failure tests passed\n"); + fprintf(stdout," (%2d API invocations)\n",cfgs); + return cfgs; +#endif +} + +#ifdef MALLOC_FAIL +#if __GNUC_PREREQ(4,6) +#pragma GCC diagnostic pop /* restore -Wdeprecated-declarations */ +#endif +#endif + +int main(int _argc, char **_argv) +{ + opus_int32 total; + const char * oversion; + if(_argc>1) + { + fprintf(stderr,"Usage: %s\n",_argv[0]); + return 1; + } + iseed=0; + + oversion=opus_get_version_string(); + if(!oversion)test_failed(); + fprintf(stderr,"Testing the %s API deterministically\n", oversion); + if(opus_strerror(-32768)==NULL)test_failed(); + if(opus_strerror(32767)==NULL)test_failed(); + if(strlen(opus_strerror(0))<1)test_failed(); + total=4; + + total+=test_dec_api(); + total+=test_msdec_api(); + total+=test_parse(); + total+=test_enc_api(); + total+=test_repacketizer_api(); + total+=test_malloc_fail(); + + fprintf(stderr,"\nAll API tests passed.\nThe libopus API was invoked %d times.\n",total); + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_common.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_common.h new file mode 100644 index 000000000..d96c7d84a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_common.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2011 Xiph.Org Foundation + Written by Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +static OPUS_INLINE void deb2_impl(unsigned char *_t,unsigned char **_p,int _k,int _x,int _y) +{ + int i; + if(_x>2){ + if(_y<3)for(i=0;i<_y;i++)*(--*_p)=_t[i+1]; + }else{ + _t[_x]=_t[_x-_y]; + deb2_impl(_t,_p,_k,_x+1,_y); + for(i=_t[_x-_y]+1;i<_k;i++){ + _t[_x]=i; + deb2_impl(_t,_p,_k,_x+1,_x); + } + } +} + +/*Generates a De Bruijn sequence (k,2) with length k^2*/ +static OPUS_INLINE void debruijn2(int _k, unsigned char *_res) +{ + unsigned char *p; + unsigned char *t; + t=malloc(sizeof(unsigned char)*_k*2); + memset(t,0,sizeof(unsigned char)*_k*2); + p=&_res[_k*_k]; + deb2_impl(t,&p,_k,1,1); + free(t); +} + +/*MWC RNG of George Marsaglia*/ +static opus_uint32 Rz, Rw; +static OPUS_INLINE opus_uint32 fast_rand(void) +{ + Rz=36969*(Rz&65535)+(Rz>>16); + Rw=18000*(Rw&65535)+(Rw>>16); + return (Rz<<16)+Rw; +} +static opus_uint32 iseed; + +#ifdef __GNUC__ +__attribute__((noreturn)) +#elif defined(_MSC_VER) +__declspec(noreturn) +#endif +static OPUS_INLINE void _test_failed(const char *file, int line) +{ + fprintf(stderr,"\n ***************************************************\n"); + fprintf(stderr," *** A fatal error was detected. ***\n"); + fprintf(stderr," ***************************************************\n"); + fprintf(stderr,"Please report this failure and include\n"); + fprintf(stderr,"'make check SEED=%u fails %s at line %d for %s'\n",iseed,file,line,opus_get_version_string()); + fprintf(stderr,"and any relevant details about your system.\n\n"); +#if defined(_MSC_VER) + _set_abort_behavior( 0, _WRITE_ABORT_MSG); +#endif + abort(); +} +#define test_failed() _test_failed(__FILE__, __LINE__); + +void regression_test(void); diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_decode.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_decode.c new file mode 100644 index 000000000..2d1e2d414 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_decode.c @@ -0,0 +1,463 @@ +/* Copyright (c) 2011-2013 Xiph.Org Foundation + Written by Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#else +#include +#define getpid _getpid +#endif +#include "opus.h" +#include "test_opus_common.h" + +#define MAX_PACKET (1500) +#define MAX_FRAME_SAMP (5760) + +int test_decoder_code0(int no_fuzz) +{ + static const opus_int32 fsv[5]={48000,24000,16000,12000,8000}; + int err,skip,plen; + int out_samples,fec; + int t; + opus_int32 i; + OpusDecoder *dec[5*2]; + opus_int32 decsize; + OpusDecoder *decbak; + opus_uint32 dec_final_range1,dec_final_range2,dec_final_acc; + unsigned char *packet; + unsigned char modes[4096]; + short *outbuf_int; + short *outbuf; + + dec_final_range1=dec_final_range2=2; + + packet=malloc(sizeof(unsigned char)*MAX_PACKET); + if(packet==NULL)test_failed(); + + outbuf_int=malloc(sizeof(short)*(MAX_FRAME_SAMP+16)*2); + for(i=0;i<(MAX_FRAME_SAMP+16)*2;i++)outbuf_int[i]=32749; + outbuf=&outbuf_int[8*2]; + + fprintf(stdout," Starting %d decoders...\n",5*2); + for(t=0;t<5*2;t++) + { + int fs=fsv[t>>1]; + int c=(t&1)+1; + err=OPUS_INTERNAL_ERROR; + dec[t] = opus_decoder_create(fs, c, &err); + if(err!=OPUS_OK || dec[t]==NULL)test_failed(); + fprintf(stdout," opus_decoder_create(%5d,%d) OK. Copy ",fs,c); + { + OpusDecoder *dec2; + /*The opus state structures contain no pointers and can be freely copied*/ + dec2=(OpusDecoder *)malloc(opus_decoder_get_size(c)); + if(dec2==NULL)test_failed(); + memcpy(dec2,dec[t],opus_decoder_get_size(c)); + memset(dec[t],255,opus_decoder_get_size(c)); + opus_decoder_destroy(dec[t]); + printf("OK.\n"); + dec[t]=dec2; + } + } + + decsize=opus_decoder_get_size(1); + decbak=(OpusDecoder *)malloc(decsize); + if(decbak==NULL)test_failed(); + + for(t=0;t<5*2;t++) + { + int factor=48000/fsv[t>>1]; + for(fec=0;fec<2;fec++) + { + opus_int32 dur; + /*Test PLC on a fresh decoder*/ + out_samples = opus_decode(dec[t], 0, 0, outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + if(opus_decoder_ctl(dec[t], OPUS_GET_LAST_PACKET_DURATION(&dur))!=OPUS_OK)test_failed(); + if(dur!=120/factor)test_failed(); + + /*Test on a size which isn't a multiple of 2.5ms*/ + out_samples = opus_decode(dec[t], 0, 0, outbuf, 120/factor+2, fec); + if(out_samples!=OPUS_BAD_ARG)test_failed(); + + /*Test null pointer input*/ + out_samples = opus_decode(dec[t], 0, -1, outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + out_samples = opus_decode(dec[t], 0, 1, outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + out_samples = opus_decode(dec[t], 0, 10, outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + out_samples = opus_decode(dec[t], 0, fast_rand(), outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + if(opus_decoder_ctl(dec[t], OPUS_GET_LAST_PACKET_DURATION(&dur))!=OPUS_OK)test_failed(); + if(dur!=120/factor)test_failed(); + + /*Zero lengths*/ + out_samples = opus_decode(dec[t], packet, 0, outbuf, 120/factor, fec); + if(out_samples!=120/factor)test_failed(); + + /*Zero buffer*/ + outbuf[0]=32749; + out_samples = opus_decode(dec[t], packet, 0, outbuf, 0, fec); + if(out_samples>0)test_failed(); +#if !defined(OPUS_BUILD) && (OPUS_GNUC_PREREQ(4, 6) || (defined(__clang_major__) && __clang_major__ >= 3)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnonnull" +#endif + out_samples = opus_decode(dec[t], packet, 0, 0, 0, fec); +#if !defined(OPUS_BUILD) && (OPUS_GNUC_PREREQ(4, 6) || (defined(__clang_major__) && __clang_major__ >= 3)) +#pragma GCC diagnostic pop +#endif + if(out_samples>0)test_failed(); + if(outbuf[0]!=32749)test_failed(); + + /*Invalid lengths*/ + out_samples = opus_decode(dec[t], packet, -1, outbuf, MAX_FRAME_SAMP, fec); + if(out_samples>=0)test_failed(); + out_samples = opus_decode(dec[t], packet, INT_MIN, outbuf, MAX_FRAME_SAMP, fec); + if(out_samples>=0)test_failed(); + out_samples = opus_decode(dec[t], packet, -1, outbuf, -1, fec); + if(out_samples>=0)test_failed(); + + /*Crazy FEC values*/ + out_samples = opus_decode(dec[t], packet, 1, outbuf, MAX_FRAME_SAMP, fec?-1:2); + if(out_samples>=0)test_failed(); + + /*Reset the decoder*/ + if(opus_decoder_ctl(dec[t], OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + } + } + fprintf(stdout," dec[all] initial frame PLC OK.\n"); + + /*Count code 0 tests*/ + for(i=0;i<64;i++) + { + opus_int32 dur; + int j,expected[5*2]; + packet[0]=i<<2; + packet[1]=255; + packet[2]=255; + err=opus_packet_get_nb_channels(packet); + if(err!=(i&1)+1)test_failed(); + + for(t=0;t<5*2;t++){ + expected[t]=opus_decoder_get_nb_samples(dec[t],packet,1); + if(expected[t]>2880)test_failed(); + } + + for(j=0;j<256;j++) + { + packet[1]=j; + for(t=0;t<5*2;t++) + { + out_samples = opus_decode(dec[t], packet, 3, outbuf, MAX_FRAME_SAMP, 0); + if(out_samples!=expected[t])test_failed(); + if(opus_decoder_ctl(dec[t], OPUS_GET_LAST_PACKET_DURATION(&dur))!=OPUS_OK)test_failed(); + if(dur!=out_samples)test_failed(); + opus_decoder_ctl(dec[t], OPUS_GET_FINAL_RANGE(&dec_final_range1)); + if(t==0)dec_final_range2=dec_final_range1; + else if(dec_final_range1!=dec_final_range2)test_failed(); + } + } + + for(t=0;t<5*2;t++){ + int factor=48000/fsv[t>>1]; + /* The PLC is run for 6 frames in order to get better PLC coverage. */ + for(j=0;j<6;j++) + { + out_samples = opus_decode(dec[t], 0, 0, outbuf, expected[t], 0); + if(out_samples!=expected[t])test_failed(); + if(opus_decoder_ctl(dec[t], OPUS_GET_LAST_PACKET_DURATION(&dur))!=OPUS_OK)test_failed(); + if(dur!=out_samples)test_failed(); + } + /* Run the PLC once at 2.5ms, as a simulation of someone trying to + do small drift corrections. */ + if(expected[t]!=120/factor) + { + out_samples = opus_decode(dec[t], 0, 0, outbuf, 120/factor, 0); + if(out_samples!=120/factor)test_failed(); + if(opus_decoder_ctl(dec[t], OPUS_GET_LAST_PACKET_DURATION(&dur))!=OPUS_OK)test_failed(); + if(dur!=out_samples)test_failed(); + } + out_samples = opus_decode(dec[t], packet, 2, outbuf, expected[t]-1, 0); + if(out_samples>0)test_failed(); + } + } + fprintf(stdout," dec[all] all 2-byte prefix for length 3 and PLC, all modes (64) OK.\n"); + + if(no_fuzz) + { + fprintf(stdout," Skipping many tests which fuzz the decoder as requested.\n"); + free(decbak); + for(t=0;t<5*2;t++)opus_decoder_destroy(dec[t]); + printf(" Decoders stopped.\n"); + + err=0; + for(i=0;i<8*2;i++)err|=outbuf_int[i]!=32749; + for(i=MAX_FRAME_SAMP*2;i<(MAX_FRAME_SAMP+8)*2;i++)err|=outbuf[i]!=32749; + if(err)test_failed(); + + free(outbuf_int); + free(packet); + return 0; + } + + { + /*We only test a subset of the modes here simply because the longer + durations end up taking a long time.*/ + static const int cmodes[4]={16,20,24,28}; + static const opus_uint32 cres[4]={116290185,2172123586u,2172123586u,2172123586u}; + static const opus_uint32 lres[3]={3285687739u,1481572662,694350475}; + static const int lmodes[3]={0,4,8}; + int mode=fast_rand()%4; + + packet[0]=cmodes[mode]<<3; + dec_final_acc=0; + t=fast_rand()%10; + + for(i=0;i<65536;i++) + { + int factor=48000/fsv[t>>1]; + packet[1]=i>>8; + packet[2]=i&255; + packet[3]=255; + out_samples = opus_decode(dec[t], packet, 4, outbuf, MAX_FRAME_SAMP, 0); + if(out_samples!=120/factor)test_failed(); + opus_decoder_ctl(dec[t], OPUS_GET_FINAL_RANGE(&dec_final_range1)); + dec_final_acc+=dec_final_range1; + } + if(dec_final_acc!=cres[mode])test_failed(); + fprintf(stdout," dec[%3d] all 3-byte prefix for length 4, mode %2d OK.\n",t,cmodes[mode]); + + mode=fast_rand()%3; + packet[0]=lmodes[mode]<<3; + dec_final_acc=0; + t=fast_rand()%10; + for(i=0;i<65536;i++) + { + int factor=48000/fsv[t>>1]; + packet[1]=i>>8; + packet[2]=i&255; + packet[3]=255; + out_samples = opus_decode(dec[t], packet, 4, outbuf, MAX_FRAME_SAMP, 0); + if(out_samples!=480/factor)test_failed(); + opus_decoder_ctl(dec[t], OPUS_GET_FINAL_RANGE(&dec_final_range1)); + dec_final_acc+=dec_final_range1; + } + if(dec_final_acc!=lres[mode])test_failed(); + fprintf(stdout," dec[%3d] all 3-byte prefix for length 4, mode %2d OK.\n",t,lmodes[mode]); + } + + skip=fast_rand()%7; + for(i=0;i<64;i++) + { + int j,expected[5*2]; + packet[0]=i<<2; + for(t=0;t<5*2;t++)expected[t]=opus_decoder_get_nb_samples(dec[t],packet,1); + for(j=2+skip;j<1275;j+=4) + { + int jj; + for(jj=0;jj1.f)test_failed(); + if(x[j]<-1.f)test_failed(); + } + } + for(i=1;i<9;i++) + { + for (j=0;j<1024;j++) + { + x[j]=(j&255)*(1/32.f)-4.f; + } + opus_pcm_soft_clip(x,1024/i,i,s); + for (j=0;j<(1024/i)*i;j++) + { + if(x[j]>1.f)test_failed(); + if(x[j]<-1.f)test_failed(); + } + } + opus_pcm_soft_clip(x,0,1,s); + opus_pcm_soft_clip(x,1,0,s); + opus_pcm_soft_clip(x,1,1,0); + opus_pcm_soft_clip(x,1,-1,s); + opus_pcm_soft_clip(x,-1,1,s); + opus_pcm_soft_clip(0,1,1,s); + printf("OK.\n"); +} +#endif + +int main(int _argc, char **_argv) +{ + const char * oversion; + const char * env_seed; + int env_used; + + if(_argc>2) + { + fprintf(stderr,"Usage: %s []\n",_argv[0]); + return 1; + } + + env_used=0; + env_seed=getenv("SEED"); + if(_argc>1)iseed=atoi(_argv[1]); + else if(env_seed) + { + iseed=atoi(env_seed); + env_used=1; + } + else iseed=(opus_uint32)time(NULL)^(((opus_uint32)getpid()&65535)<<16); + Rw=Rz=iseed; + + oversion=opus_get_version_string(); + if(!oversion)test_failed(); + fprintf(stderr,"Testing %s decoder. Random seed: %u (%.4X)\n", oversion, iseed, fast_rand() % 65535); + if(env_used)fprintf(stderr," Random seed set from the environment (SEED=%s).\n", env_seed); + + /*Setting TEST_OPUS_NOFUZZ tells the tool not to send garbage data + into the decoders. This is helpful because garbage data + may cause the decoders to clip, which angers CLANG IOC.*/ + test_decoder_code0(getenv("TEST_OPUS_NOFUZZ")!=NULL); +#ifndef DISABLE_FLOAT_API + test_soft_clip(); +#endif + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_encode.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_encode.c new file mode 100644 index 000000000..00795a1e2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_encode.c @@ -0,0 +1,703 @@ +/* Copyright (c) 2011-2013 Xiph.Org Foundation + Written by Gregory Maxwell */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#if (!defined WIN32 && !defined _WIN32) || defined(__MINGW32__) +#include +#else +#include +#define getpid _getpid +#endif +#include "opus_multistream.h" +#include "opus.h" +#include "../src/opus_private.h" +#include "test_opus_common.h" + +#define MAX_PACKET (1500) +#define SAMPLES (48000*30) +#define SSAMPLES (SAMPLES/3) +#define MAX_FRAME_SAMP (5760) +#define PI (3.141592653589793238462643f) +#define RAND_SAMPLE(a) (a[fast_rand() % sizeof(a)/sizeof(a[0])]) + +void generate_music(short *buf, opus_int32 len) +{ + opus_int32 a1,b1,a2,b2; + opus_int32 c1,c2,d1,d2; + opus_int32 i,j; + a1=b1=a2=b2=0; + c1=c2=d1=d2=0; + j=0; + /*60ms silence*/ + for(i=0;i<2880;i++)buf[i*2]=buf[i*2+1]=0; + for(i=2880;i>12)^((j>>10|j>>12)&26&j>>7)))&128)+128)<<15; + r=fast_rand();v1+=r&65535;v1-=r>>16; + r=fast_rand();v2+=r&65535;v2-=r>>16; + b1=v1-a1+((b1*61+32)>>6);a1=v1; + b2=v2-a2+((b2*61+32)>>6);a2=v2; + c1=(30*(c1+b1+d1)+32)>>6;d1=b1; + c2=(30*(c2+b2+d2)+32)>>6;d2=b2; + v1=(c1+128)>>8; + v2=(c2+128)>>8; + buf[i*2]=v1>32767?32767:(v1<-32768?-32768:v1); + buf[i*2+1]=v2>32767?32767:(v2<-32768?-32768:v2); + if(i%6==0)j++; + } +} + +#if 0 +static int save_ctr = 0; +static void int_to_char(opus_uint32 i, unsigned char ch[4]) +{ + ch[0] = i>>24; + ch[1] = (i>>16)&0xFF; + ch[2] = (i>>8)&0xFF; + ch[3] = i&0xFF; +} + +static OPUS_INLINE void save_packet(unsigned char* p, int len, opus_uint32 rng) +{ + FILE *fout; + unsigned char int_field[4]; + char name[256]; + snprintf(name,255,"test_opus_encode.%llu.%d.bit",(unsigned long long)iseed,save_ctr); + fprintf(stdout,"writing %d byte packet to %s\n",len,name); + fout=fopen(name, "wb+"); + if(fout==NULL)test_failed(); + int_to_char(len, int_field); + fwrite(int_field, 1, 4, fout); + int_to_char(rng, int_field); + fwrite(int_field, 1, 4, fout); + fwrite(p, 1, len, fout); + fclose(fout); + save_ctr++; +} +#endif + +int get_frame_size_enum(int frame_size, int sampling_rate) +{ + int frame_size_enum; + + if(frame_size==sampling_rate/400) + frame_size_enum = OPUS_FRAMESIZE_2_5_MS; + else if(frame_size==sampling_rate/200) + frame_size_enum = OPUS_FRAMESIZE_5_MS; + else if(frame_size==sampling_rate/100) + frame_size_enum = OPUS_FRAMESIZE_10_MS; + else if(frame_size==sampling_rate/50) + frame_size_enum = OPUS_FRAMESIZE_20_MS; + else if(frame_size==sampling_rate/25) + frame_size_enum = OPUS_FRAMESIZE_40_MS; + else if(frame_size==3*sampling_rate/50) + frame_size_enum = OPUS_FRAMESIZE_60_MS; + else if(frame_size==4*sampling_rate/50) + frame_size_enum = OPUS_FRAMESIZE_80_MS; + else if(frame_size==5*sampling_rate/50) + frame_size_enum = OPUS_FRAMESIZE_100_MS; + else if(frame_size==6*sampling_rate/50) + frame_size_enum = OPUS_FRAMESIZE_120_MS; + else + test_failed(); + + return frame_size_enum; +} + +int test_encode(OpusEncoder *enc, int channels, int frame_size, OpusDecoder *dec) +{ + int samp_count = 0; + opus_int16 *inbuf; + unsigned char packet[MAX_PACKET+257]; + int len; + opus_int16 *outbuf; + int out_samples; + int ret = 0; + + /* Generate input data */ + inbuf = (opus_int16*)malloc(sizeof(*inbuf)*SSAMPLES); + generate_music(inbuf, SSAMPLES/2); + + /* Allocate memory for output data */ + outbuf = (opus_int16*)malloc(sizeof(*outbuf)*MAX_FRAME_SAMP*3); + + /* Encode data, then decode for sanity check */ + do { + len = opus_encode(enc, &inbuf[samp_count*channels], frame_size, packet, MAX_PACKET); + if(len<0 || len>MAX_PACKET) { + fprintf(stderr,"opus_encode() returned %d\n",len); + ret = -1; + break; + } + + out_samples = opus_decode(dec, packet, len, outbuf, MAX_FRAME_SAMP, 0); + if(out_samples!=frame_size) { + fprintf(stderr,"opus_decode() returned %d\n",out_samples); + ret = -1; + break; + } + + samp_count += frame_size; + } while (samp_count < ((SSAMPLES/2)-MAX_FRAME_SAMP)); + + /* Clean up */ + free(inbuf); + free(outbuf); + return ret; +} + +void fuzz_encoder_settings(const int num_encoders, const int num_setting_changes) +{ + OpusEncoder *enc; + OpusDecoder *dec; + int i,j,err; + + /* Parameters to fuzz. Some values are duplicated to increase their probability of being tested. */ + int sampling_rates[5] = {8000, 12000, 16000, 24000, 48000}; + int channels[2] = {1, 2}; + int applications[3] = {OPUS_APPLICATION_AUDIO, OPUS_APPLICATION_VOIP, OPUS_APPLICATION_RESTRICTED_LOWDELAY}; + int bitrates[11] = {6000, 12000, 16000, 24000, 32000, 48000, 64000, 96000, 510000, OPUS_AUTO, OPUS_BITRATE_MAX}; + int force_channels[4] = {OPUS_AUTO, OPUS_AUTO, 1, 2}; + int use_vbr[3] = {0, 1, 1}; + int vbr_constraints[3] = {0, 1, 1}; + int complexities[11] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + int max_bandwidths[6] = {OPUS_BANDWIDTH_NARROWBAND, OPUS_BANDWIDTH_MEDIUMBAND, + OPUS_BANDWIDTH_WIDEBAND, OPUS_BANDWIDTH_SUPERWIDEBAND, + OPUS_BANDWIDTH_FULLBAND, OPUS_BANDWIDTH_FULLBAND}; + int signals[4] = {OPUS_AUTO, OPUS_AUTO, OPUS_SIGNAL_VOICE, OPUS_SIGNAL_MUSIC}; + int inband_fecs[3] = {0, 0, 1}; + int packet_loss_perc[4] = {0, 1, 2, 5}; + int lsb_depths[2] = {8, 24}; + int prediction_disabled[3] = {0, 0, 1}; + int use_dtx[2] = {0, 1}; + int frame_sizes_ms_x2[9] = {5, 10, 20, 40, 80, 120, 160, 200, 240}; /* x2 to avoid 2.5 ms */ + + for (i=0; i=64000?2:1)))!=OPUS_OK)test_failed(); + if(opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY((count>>2)%11))!=OPUS_OK)test_failed(); + if(opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC((fast_rand()&15)&(fast_rand()%15)))!=OPUS_OK)test_failed(); + bw=modes[j]==0?OPUS_BANDWIDTH_NARROWBAND+(fast_rand()%3): + modes[j]==1?OPUS_BANDWIDTH_SUPERWIDEBAND+(fast_rand()&1): + OPUS_BANDWIDTH_NARROWBAND+(fast_rand()%5); + if(modes[j]==2&&bw==OPUS_BANDWIDTH_MEDIUMBAND)bw+=3; + if(opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(bw))!=OPUS_OK)test_failed(); + len = opus_encode(enc, &inbuf[i<<1], frame_size, packet, MAX_PACKET); + if(len<0 || len>MAX_PACKET)test_failed(); + if(opus_encoder_ctl(enc, OPUS_GET_FINAL_RANGE(&enc_final_range))!=OPUS_OK)test_failed(); + if((fast_rand()&3)==0) + { + if(opus_packet_pad(packet,len,len+1)!=OPUS_OK)test_failed(); + len++; + } + if((fast_rand()&7)==0) + { + if(opus_packet_pad(packet,len,len+256)!=OPUS_OK)test_failed(); + len+=256; + } + if((fast_rand()&3)==0) + { + len=opus_packet_unpad(packet,len); + if(len<1)test_failed(); + } + out_samples = opus_decode(dec, packet, len, &outbuf[i<<1], MAX_FRAME_SAMP, 0); + if(out_samples!=frame_size)test_failed(); + if(opus_decoder_ctl(dec, OPUS_GET_FINAL_RANGE(&dec_final_range))!=OPUS_OK)test_failed(); + if(enc_final_range!=dec_final_range)test_failed(); + /*LBRR decode*/ + out_samples = opus_decode(dec_err[0], packet, len, out2buf, frame_size, (fast_rand()&3)!=0); + if(out_samples!=frame_size)test_failed(); + out_samples = opus_decode(dec_err[1], packet, (fast_rand()&3)==0?0:len, out2buf, MAX_FRAME_SAMP, (fast_rand()&7)!=0); + if(out_samples<120)test_failed(); + i+=frame_size; + count++; + }while(i<(SSAMPLES-MAX_FRAME_SAMP)); + fprintf(stdout," Mode %s FB encode %s, %6d bps OK.\n",mstrings[modes[j]],rc==0?" VBR":rc==1?"CVBR":" CBR",rate); + } + } + + if(opus_encoder_ctl(enc, OPUS_SET_FORCE_MODE(OPUS_AUTO))!=OPUS_OK)test_failed(); + if(opus_encoder_ctl(enc, OPUS_SET_FORCE_CHANNELS(OPUS_AUTO))!=OPUS_OK)test_failed(); + if(opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(0))!=OPUS_OK)test_failed(); + if(opus_encoder_ctl(enc, OPUS_SET_DTX(0))!=OPUS_OK)test_failed(); + + for(rc=0;rc<3;rc++) + { + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_VBR(rc<2))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_VBR_CONSTRAINT(rc==1))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_VBR_CONSTRAINT(rc==1))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_INBAND_FEC(rc==0))!=OPUS_OK)test_failed(); + for(j=0;j<16;j++) + { + int rate; + int modes[16]={0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2}; + int rates[16]={4000,12000,32000,8000,16000,32000,48000,88000,4000,12000,32000,8000,16000,32000,48000,88000}; + int frame[16]={160*1,160,80,160,160,80,40,20,160*1,160,80,160,160,80,40,20}; + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_INBAND_FEC(rc==0&&j==1))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_FORCE_MODE(MODE_SILK_ONLY+modes[j]))!=OPUS_OK)test_failed(); + rate=rates[j]+fast_rand()%rates[j]; + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_DTX(fast_rand()&1))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_BITRATE(rate))!=OPUS_OK)test_failed(); + count=i=0; + do { + int len,out_samples,frame_size,loss; + opus_int32 pred; + if(opus_multistream_encoder_ctl(MSenc, OPUS_GET_PREDICTION_DISABLED(&pred))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_PREDICTION_DISABLED((int)(fast_rand()&15)<(pred?11:4)))!=OPUS_OK)test_failed(); + frame_size=frame[j]; + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_COMPLEXITY((count>>2)%11))!=OPUS_OK)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_SET_PACKET_LOSS_PERC((fast_rand()&15)&(fast_rand()%15)))!=OPUS_OK)test_failed(); + if((fast_rand()&255)==0) + { + if(opus_multistream_encoder_ctl(MSenc, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + if(opus_multistream_decoder_ctl(MSdec, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + if((fast_rand()&3)!=0) + { + if(opus_multistream_decoder_ctl(MSdec_err, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + } + } + if((fast_rand()&255)==0) + { + if(opus_multistream_decoder_ctl(MSdec_err, OPUS_RESET_STATE)!=OPUS_OK)test_failed(); + } + len = opus_multistream_encode(MSenc, &inbuf[i<<1], frame_size, packet, MAX_PACKET); + if(len<0 || len>MAX_PACKET)test_failed(); + if(opus_multistream_encoder_ctl(MSenc, OPUS_GET_FINAL_RANGE(&enc_final_range))!=OPUS_OK)test_failed(); + if((fast_rand()&3)==0) + { + if(opus_multistream_packet_pad(packet,len,len+1,2)!=OPUS_OK)test_failed(); + len++; + } + if((fast_rand()&7)==0) + { + if(opus_multistream_packet_pad(packet,len,len+256,2)!=OPUS_OK)test_failed(); + len+=256; + } + if((fast_rand()&3)==0) + { + len=opus_multistream_packet_unpad(packet,len,2); + if(len<1)test_failed(); + } + out_samples = opus_multistream_decode(MSdec, packet, len, out2buf, MAX_FRAME_SAMP, 0); + if(out_samples!=frame_size*6)test_failed(); + if(opus_multistream_decoder_ctl(MSdec, OPUS_GET_FINAL_RANGE(&dec_final_range))!=OPUS_OK)test_failed(); + if(enc_final_range!=dec_final_range)test_failed(); + /*LBRR decode*/ + loss=(fast_rand()&63)==0; + out_samples = opus_multistream_decode(MSdec_err, packet, loss?0:len, out2buf, frame_size*6, (fast_rand()&3)!=0); + if(out_samples!=(frame_size*6))test_failed(); + i+=frame_size; + count++; + }while(i<(SSAMPLES/12-MAX_FRAME_SAMP)); + fprintf(stdout," Mode %s NB dual-mono MS encode %s, %6d bps OK.\n",mstrings[modes[j]],rc==0?" VBR":rc==1?"CVBR":" CBR",rate); + } + } + + bitrate_bps=512000; + fsize=fast_rand()%31; + fswitch=100; + + debruijn2(6,db62); + count=i=0; + do { + unsigned char toc; + const unsigned char *frames[48]; + short size[48]; + int payload_offset; + opus_uint32 dec_final_range2; + int jj,dec2; + int len,out_samples; + int frame_size=fsizes[db62[fsize]]; + opus_int32 offset=i%(SAMPLES-MAX_FRAME_SAMP); + + opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate_bps)); + + len = opus_encode(enc, &inbuf[offset<<1], frame_size, packet, MAX_PACKET); + if(len<0 || len>MAX_PACKET)test_failed(); + count++; + + opus_encoder_ctl(enc, OPUS_GET_FINAL_RANGE(&enc_final_range)); + + out_samples = opus_decode(dec, packet, len, &outbuf[offset<<1], MAX_FRAME_SAMP, 0); + if(out_samples!=frame_size)test_failed(); + + opus_decoder_ctl(dec, OPUS_GET_FINAL_RANGE(&dec_final_range)); + + /* compare final range encoder rng values of encoder and decoder */ + if(dec_final_range!=enc_final_range)test_failed(); + + /* We fuzz the packet, but take care not to only corrupt the payload + Corrupted headers are tested elsewhere and we need to actually run + the decoders in order to compare them. */ + if(opus_packet_parse(packet,len,&toc,frames,size,&payload_offset)<=0)test_failed(); + if((fast_rand()&1023)==0)len=0; + for(j=(opus_int32)(frames[0]-packet);j0?packet:NULL, len, out2buf, MAX_FRAME_SAMP, 0); + if(out_samples<0||out_samples>MAX_FRAME_SAMP)test_failed(); + if((len>0&&out_samples!=frame_size))test_failed(); /*FIXME use lastframe*/ + + opus_decoder_ctl(dec_err[0], OPUS_GET_FINAL_RANGE(&dec_final_range)); + + /*randomly select one of the decoders to compare with*/ + dec2=fast_rand()%9+1; + out_samples = opus_decode(dec_err[dec2], len>0?packet:NULL, len, out2buf, MAX_FRAME_SAMP, 0); + if(out_samples<0||out_samples>MAX_FRAME_SAMP)test_failed(); /*FIXME, use factor, lastframe for loss*/ + + opus_decoder_ctl(dec_err[dec2], OPUS_GET_FINAL_RANGE(&dec_final_range2)); + if(len>0&&dec_final_range!=dec_final_range2)test_failed(); + + fswitch--; + if(fswitch<1) + { + int new_size; + fsize=(fsize+1)%36; + new_size=fsizes[db62[fsize]]; + if(new_size==960||new_size==480)fswitch=2880/new_size*(fast_rand()%19+1); + else fswitch=(fast_rand()%(2880/new_size))+1; + } + bitrate_bps=((fast_rand()%508000+4000)+bitrate_bps)>>1; + i+=frame_size; + }while(i] [-fuzz ]\n",_argv[0]); +} + +int main(int _argc, char **_argv) +{ + int args=1; + char * strtol_str=NULL; + const char * oversion; + const char * env_seed; + int env_used; + int num_encoders_to_fuzz=5; + int num_setting_changes=40; + + env_used=0; + env_seed=getenv("SEED"); + if(_argc>1) + iseed=strtol(_argv[1], &strtol_str, 10); /* the first input argument might be the seed */ + if(strtol_str!=NULL && strtol_str[0]=='\0') /* iseed is a valid number */ + args++; + else if(env_seed) { + iseed=atoi(env_seed); + env_used=1; + } + else iseed=(opus_uint32)time(NULL)^(((opus_uint32)getpid()&65535)<<16); + Rw=Rz=iseed; + + while(args<_argc) + { + if(strcmp(_argv[args], "-fuzz")==0 && _argc==(args+3)) { + num_encoders_to_fuzz=strtol(_argv[args+1], &strtol_str, 10); + if(strtol_str[0]!='\0' || num_encoders_to_fuzz<=0) { + print_usage(_argv); + return EXIT_FAILURE; + } + num_setting_changes=strtol(_argv[args+2], &strtol_str, 10); + if(strtol_str[0]!='\0' || num_setting_changes<=0) { + print_usage(_argv); + return EXIT_FAILURE; + } + args+=3; + } + else { + print_usage(_argv); + return EXIT_FAILURE; + } + } + + oversion=opus_get_version_string(); + if(!oversion)test_failed(); + fprintf(stderr,"Testing %s encoder. Random seed: %u (%.4X)\n", oversion, iseed, fast_rand() % 65535); + if(env_used)fprintf(stderr," Random seed set from the environment (SEED=%s).\n", env_seed); + + regression_test(); + + /*Setting TEST_OPUS_NOFUZZ tells the tool not to send garbage data + into the decoders. This is helpful because garbage data + may cause the decoders to clip, which angers CLANG IOC.*/ + run_test1(getenv("TEST_OPUS_NOFUZZ")!=NULL); + + /* Fuzz encoder settings online */ + if(getenv("TEST_OPUS_NOFUZZ")==NULL) { + fprintf(stderr,"Running fuzz_encoder_settings with %d encoder(s) and %d setting change(s) each.\n", + num_encoders_to_fuzz, num_setting_changes); + fuzz_encoder_settings(num_encoders_to_fuzz, num_setting_changes); + } + + fprintf(stderr,"Tests completed successfully.\n"); + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_padding.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_padding.c new file mode 100644 index 000000000..c22e8f0db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_padding.c @@ -0,0 +1,93 @@ +/* Copyright (c) 2012 Xiph.Org Foundation + Written by Jüri Aedla and Ralph Giles */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Check for overflow in reading the padding length. + * http://lists.xiph.org/pipermail/opus/2012-November/001834.html + */ + +#include +#include +#include +#include "opus.h" +#include "test_opus_common.h" + +#define PACKETSIZE 16909318 +#define CHANNELS 2 +#define FRAMESIZE 5760 + +int test_overflow(void) +{ + OpusDecoder *decoder; + int result; + int error; + + unsigned char *in = malloc(PACKETSIZE); + opus_int16 *out = malloc(FRAMESIZE*CHANNELS*sizeof(*out)); + + fprintf(stderr, " Checking for padding overflow... "); + if (!in || !out) { + fprintf(stderr, "FAIL (out of memory)\n"); + return -1; + } + in[0] = 0xff; + in[1] = 0x41; + memset(in + 2, 0xff, PACKETSIZE - 3); + in[PACKETSIZE-1] = 0x0b; + + decoder = opus_decoder_create(48000, CHANNELS, &error); + result = opus_decode(decoder, in, PACKETSIZE, out, FRAMESIZE, 0); + opus_decoder_destroy(decoder); + + free(in); + free(out); + + if (result != OPUS_INVALID_PACKET) { + fprintf(stderr, "FAIL!\n"); + test_failed(); + } + + fprintf(stderr, "OK.\n"); + + return 1; +} + +int main(void) +{ + const char *oversion; + int tests = 0;; + + iseed = 0; + oversion = opus_get_version_string(); + if (!oversion) test_failed(); + fprintf(stderr, "Testing %s padding.\n", oversion); + + tests += test_overflow(); + + fprintf(stderr, "All padding tests passed.\n"); + + return 0; +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_projection.c b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_projection.c new file mode 100644 index 000000000..5f0d672c1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/tests/test_opus_projection.c @@ -0,0 +1,394 @@ +/* Copyright (c) 2017 Google Inc. + Written by Andrew Allen */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include "float_cast.h" +#include "opus.h" +#include "test_opus_common.h" +#include "opus_projection.h" +#include "mathops.h" +#include "../src/mapping_matrix.h" +#include "mathops.h" + +#define BUFFER_SIZE 960 +#define MAX_DATA_BYTES 32768 +#define MAX_FRAME_SAMPLES 5760 +#define ERROR_TOLERANCE 1 + +#define SIMPLE_MATRIX_SIZE 12 +#define SIMPLE_MATRIX_FRAME_SIZE 10 +#define SIMPLE_MATRIX_INPUT_SIZE 30 +#define SIMPLE_MATRIX_OUTPUT_SIZE 40 + +int assert_is_equal( + const opus_val16 *a, const opus_int16 *b, int size, opus_int16 tolerance) +{ + int i; + for (i = 0; i < size; i++) + { +#ifdef FIXED_POINT + opus_int16 val = a[i]; +#else + opus_int16 val = FLOAT2INT16(a[i]); +#endif + if (abs(val - b[i]) > tolerance) + return 1; + } + return 0; +} + +int assert_is_equal_short( + const opus_int16 *a, const opus_int16 *b, int size, opus_int16 tolerance) +{ + int i; + for (i = 0; i < size; i++) + if (abs(a[i] - b[i]) > tolerance) + return 1; + return 0; +} + +void test_simple_matrix(void) +{ + const MappingMatrix simple_matrix_params = {4, 3, 0}; + const opus_int16 simple_matrix_data[SIMPLE_MATRIX_SIZE] = {0, 32767, 0, 0, 32767, 0, 0, 0, 0, 0, 0, 32767}; + const opus_int16 input_int16[SIMPLE_MATRIX_INPUT_SIZE] = { + 32767, 0, -32768, 29491, -3277, -29491, 26214, -6554, -26214, 22938, -9830, + -22938, 19661, -13107, -19661, 16384, -16384, -16384, 13107, -19661, -13107, + 9830, -22938, -9830, 6554, -26214, -6554, 3277, -29491, -3277}; + const opus_int16 expected_output_int16[SIMPLE_MATRIX_OUTPUT_SIZE] = { + 0, 32767, 0, -32768, -3277, 29491, 0, -29491, -6554, 26214, 0, -26214, + -9830, 22938, 0, -22938, -13107, 19661, 0, -19661, -16384, 16384, 0, -16384, + -19661, 13107, 0, -13107, -22938, 9830, 0, -9830, -26214, 6554, 0, -6554, + -29491, 3277, 0, -3277}; + + int i, ret; + opus_int32 simple_matrix_size; + opus_val16 *input_val16; + opus_val16 *output_val16; + opus_int16 *output_int16; + MappingMatrix *simple_matrix; + + /* Allocate input/output buffers. */ + input_val16 = (opus_val16 *)opus_alloc(sizeof(opus_val16) * SIMPLE_MATRIX_INPUT_SIZE); + output_int16 = (opus_int16 *)opus_alloc(sizeof(opus_int16) * SIMPLE_MATRIX_OUTPUT_SIZE); + output_val16 = (opus_val16 *)opus_alloc(sizeof(opus_val16) * SIMPLE_MATRIX_OUTPUT_SIZE); + + /* Initialize matrix */ + simple_matrix_size = mapping_matrix_get_size(simple_matrix_params.rows, + simple_matrix_params.cols); + if (!simple_matrix_size) + test_failed(); + + simple_matrix = (MappingMatrix *)opus_alloc(simple_matrix_size); + mapping_matrix_init(simple_matrix, simple_matrix_params.rows, + simple_matrix_params.cols, simple_matrix_params.gain, simple_matrix_data, + sizeof(simple_matrix_data)); + + /* Copy inputs. */ + for (i = 0; i < SIMPLE_MATRIX_INPUT_SIZE; i++) + { +#ifdef FIXED_POINT + input_val16[i] = input_int16[i]; +#else + input_val16[i] = (1/32768.f)*input_int16[i]; +#endif + } + + /* _in_short */ + for (i = 0; i < SIMPLE_MATRIX_OUTPUT_SIZE; i++) + output_val16[i] = 0; + for (i = 0; i < simple_matrix->rows; i++) + { + mapping_matrix_multiply_channel_in_short(simple_matrix, + input_int16, simple_matrix->cols, &output_val16[i], i, + simple_matrix->rows, SIMPLE_MATRIX_FRAME_SIZE); + } + ret = assert_is_equal(output_val16, expected_output_int16, SIMPLE_MATRIX_OUTPUT_SIZE, ERROR_TOLERANCE); + if (ret) + test_failed(); + + /* _out_short */ + for (i = 0; i < SIMPLE_MATRIX_OUTPUT_SIZE; i++) + output_int16[i] = 0; + for (i = 0; i < simple_matrix->cols; i++) + { + mapping_matrix_multiply_channel_out_short(simple_matrix, + &input_val16[i], i, simple_matrix->cols, output_int16, + simple_matrix->rows, SIMPLE_MATRIX_FRAME_SIZE); + } + ret = assert_is_equal_short(output_int16, expected_output_int16, SIMPLE_MATRIX_OUTPUT_SIZE, ERROR_TOLERANCE); + if (ret) + test_failed(); + +#if !defined(DISABLE_FLOAT_API) && !defined(FIXED_POINT) + /* _in_float */ + for (i = 0; i < SIMPLE_MATRIX_OUTPUT_SIZE; i++) + output_val16[i] = 0; + for (i = 0; i < simple_matrix->rows; i++) + { + mapping_matrix_multiply_channel_in_float(simple_matrix, + input_val16, simple_matrix->cols, &output_val16[i], i, + simple_matrix->rows, SIMPLE_MATRIX_FRAME_SIZE); + } + ret = assert_is_equal(output_val16, expected_output_int16, SIMPLE_MATRIX_OUTPUT_SIZE, ERROR_TOLERANCE); + if (ret) + test_failed(); + + /* _out_float */ + for (i = 0; i < SIMPLE_MATRIX_OUTPUT_SIZE; i++) + output_val16[i] = 0; + for (i = 0; i < simple_matrix->cols; i++) + { + mapping_matrix_multiply_channel_out_float(simple_matrix, + &input_val16[i], i, simple_matrix->cols, output_val16, + simple_matrix->rows, SIMPLE_MATRIX_FRAME_SIZE); + } + ret = assert_is_equal(output_val16, expected_output_int16, SIMPLE_MATRIX_OUTPUT_SIZE, ERROR_TOLERANCE); + if (ret) + test_failed(); +#endif + + opus_free(input_val16); + opus_free(output_int16); + opus_free(output_val16); + opus_free(simple_matrix); +} + +void test_creation_arguments(const int channels, const int mapping_family) +{ + int streams; + int coupled_streams; + int enc_error; + int dec_error; + int ret; + OpusProjectionEncoder *st_enc = NULL; + OpusProjectionDecoder *st_dec = NULL; + + const opus_int32 Fs = 48000; + const int application = OPUS_APPLICATION_AUDIO; + + int order_plus_one = (int)floor(sqrt((float)channels)); + int nondiegetic_channels = channels - order_plus_one * order_plus_one; + + int is_channels_valid = 0; + int is_projection_valid = 0; + + st_enc = opus_projection_ambisonics_encoder_create(Fs, channels, + mapping_family, &streams, &coupled_streams, application, &enc_error); + if (st_enc != NULL) + { + opus_int32 matrix_size; + unsigned char *matrix; + + ret = opus_projection_encoder_ctl(st_enc, + OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE_REQUEST, &matrix_size); + if (ret != OPUS_OK || !matrix_size) + test_failed(); + + matrix = (unsigned char *)opus_alloc(matrix_size); + ret = opus_projection_encoder_ctl(st_enc, + OPUS_PROJECTION_GET_DEMIXING_MATRIX_REQUEST, matrix, matrix_size); + + opus_projection_encoder_destroy(st_enc); + + st_dec = opus_projection_decoder_create(Fs, channels, streams, + coupled_streams, matrix, matrix_size, &dec_error); + if (st_dec != NULL) + { + opus_projection_decoder_destroy(st_dec); + } + opus_free(matrix); + } + + is_channels_valid = (order_plus_one >= 2 && order_plus_one <= 4) && + (nondiegetic_channels == 0 || nondiegetic_channels == 2); + is_projection_valid = (enc_error == OPUS_OK && dec_error == OPUS_OK); + if (is_channels_valid ^ is_projection_valid) + { + fprintf(stderr, "Channels: %d, Family: %d\n", channels, mapping_family); + fprintf(stderr, "Order+1: %d, Non-diegetic Channels: %d\n", + order_plus_one, nondiegetic_channels); + fprintf(stderr, "Streams: %d, Coupled Streams: %d\n", + streams, coupled_streams); + test_failed(); + } +} + +void generate_music(short *buf, opus_int32 len, opus_int32 channels) +{ + opus_int32 i,j,k; + opus_int32 *a,*b,*c,*d; + a = (opus_int32 *)malloc(sizeof(opus_int32) * channels); + b = (opus_int32 *)malloc(sizeof(opus_int32) * channels); + c = (opus_int32 *)malloc(sizeof(opus_int32) * channels); + d = (opus_int32 *)malloc(sizeof(opus_int32) * channels); + memset(a, 0, sizeof(opus_int32) * channels); + memset(b, 0, sizeof(opus_int32) * channels); + memset(c, 0, sizeof(opus_int32) * channels); + memset(d, 0, sizeof(opus_int32) * channels); + j=0; + + for(i=0;i>12)^((j>>10|j>>12)&26&j>>7)))&128)+128)<<15; + r=fast_rand();v+=r&65535;v-=r>>16; + b[k]=v-a[k]+((b[k]*61+32)>>6);a[k]=v; + c[k]=(30*(c[k]+b[k]+d[k])+32)>>6;d[k]=b[k]; + v=(c[k]+128)>>8; + buf[i*channels+k]=v>32767?32767:(v<-32768?-32768:v); + if(i%6==0)j++; + } + } + + free(a); + free(b); + free(c); + free(d); +} + +void test_encode_decode(opus_int32 bitrate, opus_int32 channels, + const int mapping_family) +{ + const opus_int32 Fs = 48000; + const int application = OPUS_APPLICATION_AUDIO; + + OpusProjectionEncoder *st_enc; + OpusProjectionDecoder *st_dec; + int streams; + int coupled; + int error; + short *buffer_in; + short *buffer_out; + unsigned char data[MAX_DATA_BYTES] = { 0 }; + int len; + int out_samples; + opus_int32 matrix_size = 0; + unsigned char *matrix = NULL; + + buffer_in = (short *)malloc(sizeof(short) * BUFFER_SIZE * channels); + buffer_out = (short *)malloc(sizeof(short) * BUFFER_SIZE * channels); + + st_enc = opus_projection_ambisonics_encoder_create(Fs, channels, + mapping_family, &streams, &coupled, application, &error); + if (error != OPUS_OK) { + fprintf(stderr, + "Couldn\'t create encoder with %d channels and mapping family %d.\n", + channels, mapping_family); + free(buffer_in); + free(buffer_out); + test_failed(); + } + + error = opus_projection_encoder_ctl(st_enc, + OPUS_SET_BITRATE(bitrate * 1000 * (streams + coupled))); + if (error != OPUS_OK) + { + goto bad_cleanup; + } + + error = opus_projection_encoder_ctl(st_enc, + OPUS_PROJECTION_GET_DEMIXING_MATRIX_SIZE_REQUEST, &matrix_size); + if (error != OPUS_OK || !matrix_size) + { + goto bad_cleanup; + } + + matrix = (unsigned char *)opus_alloc(matrix_size); + error = opus_projection_encoder_ctl(st_enc, + OPUS_PROJECTION_GET_DEMIXING_MATRIX_REQUEST, matrix, matrix_size); + + st_dec = opus_projection_decoder_create(Fs, channels, streams, coupled, + matrix, matrix_size, &error); + opus_free(matrix); + + if (error != OPUS_OK) { + fprintf(stderr, + "Couldn\'t create decoder with %d channels, %d streams " + "and %d coupled streams.\n", channels, streams, coupled); + goto bad_cleanup; + } + + generate_music(buffer_in, BUFFER_SIZE, channels); + + len = opus_projection_encode( + st_enc, buffer_in, BUFFER_SIZE, data, MAX_DATA_BYTES); + if(len<0 || len>MAX_DATA_BYTES) { + fprintf(stderr,"opus_encode() returned %d\n", len); + goto bad_cleanup; + } + + out_samples = opus_projection_decode( + st_dec, data, len, buffer_out, MAX_FRAME_SAMPLES, 0); + if(out_samples!=BUFFER_SIZE) { + fprintf(stderr,"opus_decode() returned %d\n", out_samples); + goto bad_cleanup; + } + + opus_projection_decoder_destroy(st_dec); + opus_projection_encoder_destroy(st_enc); + free(buffer_in); + free(buffer_out); + return; +bad_cleanup: + free(buffer_in); + free(buffer_out); + test_failed(); +} + +int main(int _argc, char **_argv) +{ + unsigned int i; + + (void)_argc; + (void)_argv; + + /* Test simple matrix multiplication routines. */ + test_simple_matrix(); + + /* Test full range of channels in creation arguments. */ + for (i = 0; i < 255; i++) + test_creation_arguments(i, 3); + + /* Test encode/decode pipeline. */ + test_encode_decode(64 * 18, 18, 3); + + fprintf(stderr, "All projection tests passed.\n"); + return 0; +} + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_dump.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_dump.py new file mode 100755 index 000000000..c312088e8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_dump.py @@ -0,0 +1,66 @@ +#!/usr/bin/python + +from __future__ import print_function + +from keras.models import Sequential +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.models import load_model +from keras import backend as K +import sys + +import numpy as np + +def printVector(f, vector, name): + v = np.reshape(vector, (-1)); + #print('static const float ', name, '[', len(v), '] = \n', file=f) + f.write('static const opus_int8 {}[{}] = {{\n '.format(name, len(v))) + for i in range(0, len(v)): + f.write('{}'.format(max(-128,min(127,int(round(128*v[i])))))) + if (i!=len(v)-1): + f.write(',') + else: + break; + if (i%8==7): + f.write("\n ") + else: + f.write(" ") + #print(v, file=f) + f.write('\n};\n\n') + return; + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1) + + +#model = load_model(sys.argv[1], custom_objects={'binary_crossentrop2': binary_crossentrop2}) +main_input = Input(shape=(None, 25), name='main_input') +x = Dense(32, activation='tanh')(main_input) +x = GRU(24, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x) +x = Dense(2, activation='sigmoid')(x) +model = Model(inputs=main_input, outputs=x) +model.load_weights(sys.argv[1]) + +weights = model.get_weights() + +f = open(sys.argv[2], 'w') + +f.write('/*This file is automatically generated from a Keras model*/\n\n') +f.write('#ifdef HAVE_CONFIG_H\n#include "config.h"\n#endif\n\n#include "mlp.h"\n\n') + +printVector(f, weights[0], 'layer0_weights') +printVector(f, weights[1], 'layer0_bias') +printVector(f, weights[2], 'layer1_weights') +printVector(f, weights[3], 'layer1_recur_weights') +printVector(f, weights[4], 'layer1_bias') +printVector(f, weights[5], 'layer2_weights') +printVector(f, weights[6], 'layer2_bias') + +f.write('const DenseLayer layer0 = {\n layer0_bias,\n layer0_weights,\n 25, 32, 0\n};\n\n') +f.write('const GRULayer layer1 = {\n layer1_bias,\n layer1_weights,\n layer1_recur_weights,\n 32, 24\n};\n\n') +f.write('const DenseLayer layer2 = {\n layer2_bias,\n layer2_weights,\n 24, 2, 1\n};\n\n') + +f.close() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_train.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_train.py new file mode 100755 index 000000000..29bcb0346 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/rnn_train.py @@ -0,0 +1,177 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +from keras.models import Sequential +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.layers import CuDNNGRU +from keras.layers import SimpleRNN +from keras.layers import Dropout +from keras import losses +import h5py +from keras.optimizers import Adam + +from keras.constraints import Constraint +from keras import backend as K +import numpy as np + +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +config = tf.ConfigProto() +config.gpu_options.per_process_gpu_memory_fraction = 0.44 +set_session(tf.Session(config=config)) + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_true, y_pred), axis=-1) + +def binary_accuracy2(y_true, y_pred): + return K.mean(K.cast(K.equal(y_true, K.round(y_pred)), 'float32') + K.cast(K.equal(y_true, 0.5), 'float32'), axis=-1) + +def quant_model(model): + weights = model.get_weights() + for k in range(len(weights)): + weights[k] = np.maximum(-128, np.minimum(127, np.round(128*weights[k])*0.0078125)) + model.set_weights(weights) + +class WeightClip(Constraint): + '''Clips the weights incident to each hidden unit to be inside a range + ''' + def __init__(self, c=2): + self.c = c + + def __call__(self, p): + return K.clip(p, -self.c, self.c) + + def get_config(self): + return {'name': self.__class__.__name__, + 'c': self.c} + +reg = 0.000001 +constraint = WeightClip(.998) + +print('Build model...') + +main_input = Input(shape=(None, 25), name='main_input') +x = Dense(32, activation='tanh', kernel_constraint=constraint, bias_constraint=constraint)(main_input) +#x = CuDNNGRU(24, return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) +x = GRU(24, recurrent_activation='sigmoid', activation='tanh', return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) +x = Dense(2, activation='sigmoid', kernel_constraint=constraint, bias_constraint=constraint)(x) +model = Model(inputs=main_input, outputs=x) + +batch_size = 2048 + +print('Loading data...') +with h5py.File('features10b.h5', 'r') as hf: + all_data = hf['data'][:] +print('done.') + +window_size = 1500 + +nb_sequences = len(all_data)//window_size +print(nb_sequences, ' sequences') +x_train = all_data[:nb_sequences*window_size, :-2] +x_train = np.reshape(x_train, (nb_sequences, window_size, 25)) + +y_train = np.copy(all_data[:nb_sequences*window_size, -2:]) +y_train = np.reshape(y_train, (nb_sequences, window_size, 2)) + +print("Marking ignores") +for s in y_train: + for e in s: + if (e[1] >= 1): + break + e[0] = 0.5 + +all_data = 0; +x_train = x_train.astype('float32') +y_train = y_train.astype('float32') + +print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape) + +model.load_weights('newweights10a1b_ep206.hdf5') + +#weights = model.get_weights() +#for k in range(len(weights)): +# weights[k] = np.round(128*weights[k])*0.0078125 +#model.set_weights(weights) + +# try using different optimizers and different optimizer configs +model.compile(loss=binary_crossentrop2, + optimizer=Adam(0.0001), + metrics=[binary_accuracy2]) + +print('Train...') +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=10, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep10.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=50, initial_epoch=10) +model.save("newweights10a1c_ep50.hdf5") + +model.compile(loss=binary_crossentrop2, + optimizer=Adam(0.0001), + metrics=[binary_accuracy2]) + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=100, initial_epoch=50) +model.save("newweights10a1c_ep100.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=150, initial_epoch=100) +model.save("newweights10a1c_ep150.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=200, initial_epoch=150) +model.save("newweights10a1c_ep200.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=201, initial_epoch=200) +model.save("newweights10a1c_ep201.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=202, initial_epoch=201, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep202.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=203, initial_epoch=202, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep203.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=204, initial_epoch=203, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep204.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=205, initial_epoch=204, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep205.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=206, initial_epoch=205, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep206.hdf5") + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/txt2hdf5.py b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/txt2hdf5.py new file mode 100755 index 000000000..9c602877e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/training/txt2hdf5.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +from __future__ import print_function + +import numpy as np +import h5py +import sys + +data = np.loadtxt(sys.argv[1], dtype='float32') +h5f = h5py.File(sys.argv[2], 'w'); +h5f.create_dataset('data', data=data) +h5f.close() diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/common.props b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/common.props new file mode 100644 index 000000000..d83b7c2da --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/common.props @@ -0,0 +1,82 @@ + + + + + + $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\$(ProjectName)\ + Unicode + + + true + true + false + + + false + false + true + + + + Level3 + false + false + ..\..;..\..\include;..\..\silk;..\..\celt;..\..\win32;%(AdditionalIncludeDirectories) + HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + false + + + Console + + + true + Console + + + + + Guard + ProgramDatabase + NoExtensions + false + true + false + Disabled + false + false + Disabled + MultiThreadedDebug + MultiThreadedDebugDLL + true + false + + + true + + + + + false + None + true + true + false + Speed + Fast + Precise + true + true + true + MaxSpeed + MultiThreaded + MultiThreadedDLL + 16Bytes + + + false + + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.sln b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.sln new file mode 100644 index 000000000..abd7a4dda --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.sln @@ -0,0 +1,168 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "opus", "opus.vcxproj", "{219EC965-228A-1824-174D-96449D05F88A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "opus_demo", "opus_demo.vcxproj", "{016C739D-6389-43BF-8D88-24B2BF6F620F}" + ProjectSection(ProjectDependencies) = postProject + {219EC965-228A-1824-174D-96449D05F88A} = {219EC965-228A-1824-174D-96449D05F88A} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_opus_api", "test_opus_api.vcxproj", "{1D257A17-D254-42E5-82D6-1C87A6EC775A}" + ProjectSection(ProjectDependencies) = postProject + {219EC965-228A-1824-174D-96449D05F88A} = {219EC965-228A-1824-174D-96449D05F88A} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_opus_decode", "test_opus_decode.vcxproj", "{8578322A-1883-486B-B6FA-E0094B65C9F2}" + ProjectSection(ProjectDependencies) = postProject + {219EC965-228A-1824-174D-96449D05F88A} = {219EC965-228A-1824-174D-96449D05F88A} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_opus_encode", "test_opus_encode.vcxproj", "{84DAA768-1A38-4312-BB61-4C78BB59E5B8}" + ProjectSection(ProjectDependencies) = postProject + {219EC965-228A-1824-174D-96449D05F88A} = {219EC965-228A-1824-174D-96449D05F88A} + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + DebugDLL_fixed|Win32 = DebugDLL_fixed|Win32 + DebugDLL_fixed|x64 = DebugDLL_fixed|x64 + DebugDLL|Win32 = DebugDLL|Win32 + DebugDLL|x64 = DebugDLL|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + ReleaseDLL_fixed|Win32 = ReleaseDLL_fixed|Win32 + ReleaseDLL_fixed|x64 = ReleaseDLL_fixed|x64 + ReleaseDLL|Win32 = ReleaseDLL|Win32 + ReleaseDLL|x64 = ReleaseDLL|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {219EC965-228A-1824-174D-96449D05F88A}.Debug|Win32.ActiveCfg = Debug|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.Debug|Win32.Build.0 = Debug|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.Debug|x64.ActiveCfg = Debug|x64 + {219EC965-228A-1824-174D-96449D05F88A}.Debug|x64.Build.0 = Debug|x64 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL_fixed|Win32.ActiveCfg = DebugDLL_fixed|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL_fixed|Win32.Build.0 = DebugDLL_fixed|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL_fixed|x64.ActiveCfg = DebugDLL_fixed|x64 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL_fixed|x64.Build.0 = DebugDLL_fixed|x64 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL|Win32.ActiveCfg = DebugDLL|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL|Win32.Build.0 = DebugDLL|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL|x64.ActiveCfg = DebugDLL|x64 + {219EC965-228A-1824-174D-96449D05F88A}.DebugDLL|x64.Build.0 = DebugDLL|x64 + {219EC965-228A-1824-174D-96449D05F88A}.Release|Win32.ActiveCfg = Release|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.Release|Win32.Build.0 = Release|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.Release|x64.ActiveCfg = Release|x64 + {219EC965-228A-1824-174D-96449D05F88A}.Release|x64.Build.0 = Release|x64 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL_fixed|Win32.ActiveCfg = ReleaseDLL_fixed|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL_fixed|Win32.Build.0 = ReleaseDLL_fixed|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL_fixed|x64.ActiveCfg = ReleaseDLL_fixed|x64 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL_fixed|x64.Build.0 = ReleaseDLL_fixed|x64 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL|Win32.ActiveCfg = ReleaseDLL|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL|Win32.Build.0 = ReleaseDLL|Win32 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL|x64.ActiveCfg = ReleaseDLL|x64 + {219EC965-228A-1824-174D-96449D05F88A}.ReleaseDLL|x64.Build.0 = ReleaseDLL|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Debug|Win32.ActiveCfg = Debug|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Debug|Win32.Build.0 = Debug|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Debug|x64.ActiveCfg = Debug|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Debug|x64.Build.0 = Debug|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL_fixed|Win32.ActiveCfg = DebugDLL_fixed|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL_fixed|Win32.Build.0 = DebugDLL_fixed|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL_fixed|x64.ActiveCfg = DebugDLL_fixed|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL_fixed|x64.Build.0 = DebugDLL_fixed|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL|Win32.ActiveCfg = DebugDLL|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL|Win32.Build.0 = DebugDLL|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL|x64.ActiveCfg = DebugDLL|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.DebugDLL|x64.Build.0 = DebugDLL|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Release|Win32.ActiveCfg = Release|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Release|Win32.Build.0 = Release|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Release|x64.ActiveCfg = Release|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.Release|x64.Build.0 = Release|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL_fixed|Win32.ActiveCfg = ReleaseDLL_fixed|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL_fixed|Win32.Build.0 = ReleaseDLL_fixed|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL_fixed|x64.ActiveCfg = ReleaseDLL_fixed|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL_fixed|x64.Build.0 = ReleaseDLL_fixed|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL|Win32.ActiveCfg = ReleaseDLL|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL|Win32.Build.0 = ReleaseDLL|Win32 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL|x64.ActiveCfg = ReleaseDLL|x64 + {016C739D-6389-43BF-8D88-24B2BF6F620F}.ReleaseDLL|x64.Build.0 = ReleaseDLL|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Debug|Win32.ActiveCfg = Debug|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Debug|Win32.Build.0 = Debug|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Debug|x64.ActiveCfg = Debug|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Debug|x64.Build.0 = Debug|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL_fixed|Win32.ActiveCfg = DebugDLL_fixed|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL_fixed|Win32.Build.0 = DebugDLL_fixed|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL_fixed|x64.ActiveCfg = DebugDLL_fixed|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL_fixed|x64.Build.0 = DebugDLL_fixed|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL|Win32.ActiveCfg = DebugDLL|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL|Win32.Build.0 = DebugDLL|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL|x64.ActiveCfg = DebugDLL|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.DebugDLL|x64.Build.0 = DebugDLL|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Release|Win32.ActiveCfg = Release|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Release|Win32.Build.0 = Release|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Release|x64.ActiveCfg = Release|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.Release|x64.Build.0 = Release|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL_fixed|Win32.ActiveCfg = ReleaseDLL_fixed|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL_fixed|Win32.Build.0 = ReleaseDLL_fixed|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL_fixed|x64.ActiveCfg = ReleaseDLL_fixed|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL_fixed|x64.Build.0 = ReleaseDLL_fixed|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL|Win32.ActiveCfg = ReleaseDLL|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL|Win32.Build.0 = ReleaseDLL|Win32 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL|x64.ActiveCfg = ReleaseDLL|x64 + {1D257A17-D254-42E5-82D6-1C87A6EC775A}.ReleaseDLL|x64.Build.0 = ReleaseDLL|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Debug|Win32.ActiveCfg = Debug|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Debug|Win32.Build.0 = Debug|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Debug|x64.ActiveCfg = Debug|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Debug|x64.Build.0 = Debug|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL_fixed|Win32.ActiveCfg = DebugDLL_fixed|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL_fixed|Win32.Build.0 = DebugDLL_fixed|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL_fixed|x64.ActiveCfg = DebugDLL_fixed|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL_fixed|x64.Build.0 = DebugDLL_fixed|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL|Win32.ActiveCfg = DebugDLL|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL|Win32.Build.0 = DebugDLL|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL|x64.ActiveCfg = DebugDLL|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.DebugDLL|x64.Build.0 = DebugDLL|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Release|Win32.ActiveCfg = Release|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Release|Win32.Build.0 = Release|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Release|x64.ActiveCfg = Release|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.Release|x64.Build.0 = Release|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL_fixed|Win32.ActiveCfg = ReleaseDLL_fixed|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL_fixed|Win32.Build.0 = ReleaseDLL_fixed|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL_fixed|x64.ActiveCfg = ReleaseDLL_fixed|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL_fixed|x64.Build.0 = ReleaseDLL_fixed|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL|Win32.ActiveCfg = ReleaseDLL|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL|Win32.Build.0 = ReleaseDLL|Win32 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL|x64.ActiveCfg = ReleaseDLL|x64 + {8578322A-1883-486B-B6FA-E0094B65C9F2}.ReleaseDLL|x64.Build.0 = ReleaseDLL|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Debug|Win32.ActiveCfg = Debug|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Debug|Win32.Build.0 = Debug|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Debug|x64.ActiveCfg = Debug|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Debug|x64.Build.0 = Debug|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL_fixed|Win32.ActiveCfg = DebugDLL_fixed|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL_fixed|Win32.Build.0 = DebugDLL_fixed|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL_fixed|x64.ActiveCfg = DebugDLL_fixed|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL_fixed|x64.Build.0 = DebugDLL_fixed|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL|Win32.ActiveCfg = DebugDLL|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL|Win32.Build.0 = DebugDLL|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL|x64.ActiveCfg = DebugDLL|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.DebugDLL|x64.Build.0 = DebugDLL|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Release|Win32.ActiveCfg = Release|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Release|Win32.Build.0 = Release|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Release|x64.ActiveCfg = Release|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.Release|x64.Build.0 = Release|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL_fixed|Win32.ActiveCfg = ReleaseDLL_fixed|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL_fixed|Win32.Build.0 = ReleaseDLL_fixed|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL_fixed|x64.ActiveCfg = ReleaseDLL_fixed|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL_fixed|x64.Build.0 = ReleaseDLL_fixed|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL|Win32.ActiveCfg = ReleaseDLL|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL|Win32.Build.0 = ReleaseDLL|Win32 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL|x64.ActiveCfg = ReleaseDLL|x64 + {84DAA768-1A38-4312-BB61-4C78BB59E5B8}.ReleaseDLL|x64.Build.0 = ReleaseDLL|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj new file mode 100644 index 000000000..e7a17da56 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj @@ -0,0 +1,399 @@ + + + + + DebugDLL_fixed + Win32 + + + DebugDLL_fixed + x64 + + + DebugDLL + Win32 + + + DebugDLL + x64 + + + Debug + Win32 + + + Debug + x64 + + + ReleaseDLL_fixed + Win32 + + + ReleaseDLL_fixed + x64 + + + ReleaseDLL + Win32 + + + ReleaseDLL + x64 + + + Release + Win32 + + + Release + x64 + + + + Win32Proj + opus + {219EC965-228A-1824-174D-96449D05F88A} + + + + StaticLibrary + v140 + + + DynamicLibrary + v140 + + + DynamicLibrary + v140 + + + StaticLibrary + v140 + + + DynamicLibrary + v140 + + + DynamicLibrary + v140 + + + StaticLibrary + v140 + + + DynamicLibrary + v140 + + + DynamicLibrary + v140 + + + StaticLibrary + v140 + + + DynamicLibrary + v140 + + + DynamicLibrary + v140 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..\silk\fixed;..\..\silk\float;%(AdditionalIncludeDirectories) + DLL_EXPORT;%(PreprocessorDefinitions) + FIXED_POINT;%(PreprocessorDefinitions) + /arch:IA32 %(AdditionalOptions) + + + /ignore:4221 %(AdditionalOptions) + + + "$(ProjectDir)..\..\win32\genversion.bat" "$(ProjectDir)..\..\win32\version.h" PACKAGE_VERSION + Generating version.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4244;%(DisableSpecificWarnings) + + + + + + + + + + + + + + + false + + + false + + + true + + + + + + + true + + + true + + + false + + + + + + + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj.filters b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj.filters new file mode 100644 index 000000000..d59c7563e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus.vcxproj.filters @@ -0,0 +1,585 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj new file mode 100644 index 000000000..9860bf93f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj @@ -0,0 +1,171 @@ + + + + + DebugDLL_fixed + Win32 + + + DebugDLL_fixed + x64 + + + DebugDLL + Win32 + + + DebugDLL + x64 + + + Debug + Win32 + + + Debug + x64 + + + ReleaseDLL_fixed + Win32 + + + ReleaseDLL_fixed + x64 + + + ReleaseDLL + Win32 + + + ReleaseDLL + x64 + + + Release + Win32 + + + Release + x64 + + + + + {219ec965-228a-1824-174d-96449d05f88a} + + + + + + + {016C739D-6389-43BF-8D88-24B2BF6F620F} + Win32Proj + opus_demo + + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj.filters b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj.filters new file mode 100644 index 000000000..dbcc8ae92 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/opus_demo.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj new file mode 100644 index 000000000..c6abf88e5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj @@ -0,0 +1,171 @@ + + + + + DebugDLL_fixed + Win32 + + + DebugDLL_fixed + x64 + + + DebugDLL + Win32 + + + DebugDLL + x64 + + + Debug + Win32 + + + Debug + x64 + + + ReleaseDLL_fixed + Win32 + + + ReleaseDLL_fixed + x64 + + + ReleaseDLL + Win32 + + + ReleaseDLL + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + {219ec965-228a-1824-174d-96449d05f88a} + + + + {1D257A17-D254-42E5-82D6-1C87A6EC775A} + Win32Proj + test_opus_api + + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj.filters b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj.filters new file mode 100644 index 000000000..070c8ab01 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_api.vcxproj.filters @@ -0,0 +1,14 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj new file mode 100644 index 000000000..9db09b8da --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj @@ -0,0 +1,171 @@ + + + + + DebugDLL_fixed + Win32 + + + DebugDLL_fixed + x64 + + + DebugDLL + Win32 + + + DebugDLL + x64 + + + Debug + Win32 + + + Debug + x64 + + + ReleaseDLL_fixed + Win32 + + + ReleaseDLL_fixed + x64 + + + ReleaseDLL + Win32 + + + ReleaseDLL + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + {219ec965-228a-1824-174d-96449d05f88a} + + + + {8578322A-1883-486B-B6FA-E0094B65C9F2} + Win32Proj + test_opus_api + + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj.filters b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj.filters new file mode 100644 index 000000000..588637e83 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_decode.vcxproj.filters @@ -0,0 +1,14 @@ + + + + + {4a0dd677-931f-4728-afe5-b761149fc7eb} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj new file mode 100644 index 000000000..b10eaf21d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj @@ -0,0 +1,172 @@ + + + + + DebugDLL_fixed + Win32 + + + DebugDLL_fixed + x64 + + + DebugDLL + Win32 + + + DebugDLL + x64 + + + Debug + Win32 + + + Debug + x64 + + + ReleaseDLL_fixed + Win32 + + + ReleaseDLL_fixed + x64 + + + ReleaseDLL + Win32 + + + ReleaseDLL + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + {219ec965-228a-1824-174d-96449d05f88a} + + + + {84DAA768-1A38-4312-BB61-4C78BB59E5B8} + Win32Proj + test_opus_api + + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + Application + v140 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj.filters b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj.filters new file mode 100644 index 000000000..f04776380 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/VS2015/test_opus_encode.vcxproj.filters @@ -0,0 +1,17 @@ + + + + + {546c8d9a-103e-4f78-972b-b44e8d3c8aba} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/genversion.bat b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/genversion.bat new file mode 100644 index 000000000..aea557393 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/opus/win32/genversion.bat @@ -0,0 +1,37 @@ +@echo off + +setlocal enableextensions enabledelayedexpansion + +for /f %%v in ('cd "%~dp0.." ^&^& git status ^>NUL 2^>NUL ^&^& git describe --tags --match "v*" --dirty 2^>NUL') do set version=%%v + +if not "%version%"=="" set version=!version:~1! && goto :gotversion + +if exist "%~dp0..\package_version" goto :getversion + +echo Git cannot be found, nor can package_version. Generating unknown version. + +set version=unknown + +goto :gotversion + +:getversion + +for /f "delims== tokens=2" %%v in (%~dps0..\package_version) do set version=%%v +set version=!version:"=! + +:gotversion + +set version=!version: =! +set version_out=#define %~2 "%version%" + +echo %version_out%> "%~1_temp" + +echo n | comp "%~1_temp" "%~1" > NUL 2> NUL + +if not errorlevel 1 goto exit + +copy /y "%~1_temp" "%~1" + +:exit + +del "%~1_temp" diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/lib.rs b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/lib.rs new file mode 100644 index 000000000..8f145ed38 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/lib.rs @@ -0,0 +1,1251 @@ +/* automatically generated by rust-bindgen 0.58.1 */ + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + + +pub const OPUS_OK: i32 = 0; +pub const OPUS_BAD_ARG: i32 = -1; +pub const OPUS_BUFFER_TOO_SMALL: i32 = -2; +pub const OPUS_INTERNAL_ERROR: i32 = -3; +pub const OPUS_INVALID_PACKET: i32 = -4; +pub const OPUS_UNIMPLEMENTED: i32 = -5; +pub const OPUS_INVALID_STATE: i32 = -6; +pub const OPUS_ALLOC_FAIL: i32 = -7; +pub const OPUS_SET_APPLICATION_REQUEST: i32 = 4000; +pub const OPUS_GET_APPLICATION_REQUEST: i32 = 4001; +pub const OPUS_SET_BITRATE_REQUEST: i32 = 4002; +pub const OPUS_GET_BITRATE_REQUEST: i32 = 4003; +pub const OPUS_SET_MAX_BANDWIDTH_REQUEST: i32 = 4004; +pub const OPUS_GET_MAX_BANDWIDTH_REQUEST: i32 = 4005; +pub const OPUS_SET_VBR_REQUEST: i32 = 4006; +pub const OPUS_GET_VBR_REQUEST: i32 = 4007; +pub const OPUS_SET_BANDWIDTH_REQUEST: i32 = 4008; +pub const OPUS_GET_BANDWIDTH_REQUEST: i32 = 4009; +pub const OPUS_SET_COMPLEXITY_REQUEST: i32 = 4010; +pub const OPUS_GET_COMPLEXITY_REQUEST: i32 = 4011; +pub const OPUS_SET_INBAND_FEC_REQUEST: i32 = 4012; +pub const OPUS_GET_INBAND_FEC_REQUEST: i32 = 4013; +pub const OPUS_SET_PACKET_LOSS_PERC_REQUEST: i32 = 4014; +pub const OPUS_GET_PACKET_LOSS_PERC_REQUEST: i32 = 4015; +pub const OPUS_SET_DTX_REQUEST: i32 = 4016; +pub const OPUS_GET_DTX_REQUEST: i32 = 4017; +pub const OPUS_SET_VBR_CONSTRAINT_REQUEST: i32 = 4020; +pub const OPUS_GET_VBR_CONSTRAINT_REQUEST: i32 = 4021; +pub const OPUS_SET_FORCE_CHANNELS_REQUEST: i32 = 4022; +pub const OPUS_GET_FORCE_CHANNELS_REQUEST: i32 = 4023; +pub const OPUS_SET_SIGNAL_REQUEST: i32 = 4024; +pub const OPUS_GET_SIGNAL_REQUEST: i32 = 4025; +pub const OPUS_GET_LOOKAHEAD_REQUEST: i32 = 4027; +pub const OPUS_GET_SAMPLE_RATE_REQUEST: i32 = 4029; +pub const OPUS_GET_FINAL_RANGE_REQUEST: i32 = 4031; +pub const OPUS_GET_PITCH_REQUEST: i32 = 4033; +pub const OPUS_SET_GAIN_REQUEST: i32 = 4034; +pub const OPUS_GET_GAIN_REQUEST: i32 = 4045; +pub const OPUS_SET_LSB_DEPTH_REQUEST: i32 = 4036; +pub const OPUS_GET_LSB_DEPTH_REQUEST: i32 = 4037; +pub const OPUS_GET_LAST_PACKET_DURATION_REQUEST: i32 = 4039; +pub const OPUS_SET_EXPERT_FRAME_DURATION_REQUEST: i32 = 4040; +pub const OPUS_GET_EXPERT_FRAME_DURATION_REQUEST: i32 = 4041; +pub const OPUS_SET_PREDICTION_DISABLED_REQUEST: i32 = 4042; +pub const OPUS_GET_PREDICTION_DISABLED_REQUEST: i32 = 4043; +pub const OPUS_SET_PHASE_INVERSION_DISABLED_REQUEST: i32 = 4046; +pub const OPUS_GET_PHASE_INVERSION_DISABLED_REQUEST: i32 = 4047; +pub const OPUS_GET_IN_DTX_REQUEST: i32 = 4049; +pub const OPUS_AUTO: i32 = -1000; +pub const OPUS_BITRATE_MAX: i32 = -1; +pub const OPUS_APPLICATION_VOIP: i32 = 2048; +pub const OPUS_APPLICATION_AUDIO: i32 = 2049; +pub const OPUS_APPLICATION_RESTRICTED_LOWDELAY: i32 = 2051; +pub const OPUS_SIGNAL_VOICE: i32 = 3001; +pub const OPUS_SIGNAL_MUSIC: i32 = 3002; +pub const OPUS_BANDWIDTH_NARROWBAND: i32 = 1101; +pub const OPUS_BANDWIDTH_MEDIUMBAND: i32 = 1102; +pub const OPUS_BANDWIDTH_WIDEBAND: i32 = 1103; +pub const OPUS_BANDWIDTH_SUPERWIDEBAND: i32 = 1104; +pub const OPUS_BANDWIDTH_FULLBAND: i32 = 1105; +pub const OPUS_FRAMESIZE_ARG: i32 = 5000; +pub const OPUS_FRAMESIZE_2_5_MS: i32 = 5001; +pub const OPUS_FRAMESIZE_5_MS: i32 = 5002; +pub const OPUS_FRAMESIZE_10_MS: i32 = 5003; +pub const OPUS_FRAMESIZE_20_MS: i32 = 5004; +pub const OPUS_FRAMESIZE_40_MS: i32 = 5005; +pub const OPUS_FRAMESIZE_60_MS: i32 = 5006; +pub const OPUS_FRAMESIZE_80_MS: i32 = 5007; +pub const OPUS_FRAMESIZE_100_MS: i32 = 5008; +pub const OPUS_FRAMESIZE_120_MS: i32 = 5009; +pub const OPUS_RESET_STATE: i32 = 4028; +pub const OPUS_MULTISTREAM_GET_ENCODER_STATE_REQUEST: i32 = 5120; +pub const OPUS_MULTISTREAM_GET_DECODER_STATE_REQUEST: i32 = 5122; +pub type opus_int32 = ::std::os::raw::c_int; +pub type opus_uint32 = ::std::os::raw::c_uint; +pub type opus_int16 = ::std::os::raw::c_short; +pub type opus_uint16 = ::std::os::raw::c_ushort; +extern "C" { + #[doc = " Converts an opus error code into a human readable string."] + #[doc = ""] + #[doc = " @param[in] error int: Error number"] + #[doc = " @returns Error string"] + pub fn opus_strerror(error: ::std::os::raw::c_int) -> *const ::std::os::raw::c_char; +} +extern "C" { + #[doc = " Gets the libopus version string."] + #[doc = ""] + #[doc = " Applications may look for the substring \"-fixed\" in the version string to"] + #[doc = " determine whether they have a fixed-point or floating-point build at"] + #[doc = " runtime."] + #[doc = ""] + #[doc = " @returns Version string"] + pub fn opus_get_version_string() -> *const ::std::os::raw::c_char; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OpusEncoder { + _unused: [u8; 0], +} +extern "C" { + #[doc = " Gets the size of an OpusEncoder structure."] + #[doc = " @param[in] channels int: Number of channels."] + #[doc = " This must be 1 or 2."] + #[doc = " @returns The size in bytes."] + pub fn opus_encoder_get_size(channels: ::std::os::raw::c_int) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Allocates and initializes an encoder state."] + #[doc = " There are three coding modes:"] + #[doc = ""] + #[doc = " @ref OPUS_APPLICATION_VOIP gives best quality at a given bitrate for voice"] + #[doc = " signals. It enhances the input signal by high-pass filtering and"] + #[doc = " emphasizing formants and harmonics. Optionally it includes in-band"] + #[doc = " forward error correction to protect against packet loss. Use this"] + #[doc = " mode for typical VoIP applications. Because of the enhancement,"] + #[doc = " even at high bitrates the output may sound different from the input."] + #[doc = ""] + #[doc = " @ref OPUS_APPLICATION_AUDIO gives best quality at a given bitrate for most"] + #[doc = " non-voice signals like music. Use this mode for music and mixed"] + #[doc = " (music/voice) content, broadcast, and applications requiring less"] + #[doc = " than 15 ms of coding delay."] + #[doc = ""] + #[doc = " @ref OPUS_APPLICATION_RESTRICTED_LOWDELAY configures low-delay mode that"] + #[doc = " disables the speech-optimized mode in exchange for slightly reduced delay."] + #[doc = " This mode can only be set on an newly initialized or freshly reset encoder"] + #[doc = " because it changes the codec delay."] + #[doc = ""] + #[doc = " This is useful when the caller knows that the speech-optimized modes will not be needed (use with caution)."] + #[doc = " @param [in] Fs opus_int32: Sampling rate of input signal (Hz)"] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param [in] channels int: Number of channels (1 or 2) in input signal"] + #[doc = " @param [in] application int: Coding mode (@ref OPUS_APPLICATION_VOIP/@ref OPUS_APPLICATION_AUDIO/@ref OPUS_APPLICATION_RESTRICTED_LOWDELAY)"] + #[doc = " @param [out] error int*: @ref opus_errorcodes"] + #[doc = " @note Regardless of the sampling rate and number channels selected, the Opus encoder"] + #[doc = " can switch to a lower audio bandwidth or number of channels if the bitrate"] + #[doc = " selected is too low. This also means that it is safe to always use 48 kHz stereo input"] + #[doc = " and let the encoder optimize the encoding."] + pub fn opus_encoder_create( + Fs: opus_int32, + channels: ::std::os::raw::c_int, + application: ::std::os::raw::c_int, + error: *mut ::std::os::raw::c_int, + ) -> *mut OpusEncoder; +} +extern "C" { + #[doc = " Initializes a previously allocated encoder state"] + #[doc = " The memory pointed to by st must be at least the size returned by opus_encoder_get_size()."] + #[doc = " This is intended for applications which use their own allocator instead of malloc."] + #[doc = " @see opus_encoder_create(),opus_encoder_get_size()"] + #[doc = " To reset a previously initialized state, use the #OPUS_RESET_STATE CTL."] + #[doc = " @param [in] st OpusEncoder*: Encoder state"] + #[doc = " @param [in] Fs opus_int32: Sampling rate of input signal (Hz)"] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param [in] channels int: Number of channels (1 or 2) in input signal"] + #[doc = " @param [in] application int: Coding mode (OPUS_APPLICATION_VOIP/OPUS_APPLICATION_AUDIO/OPUS_APPLICATION_RESTRICTED_LOWDELAY)"] + #[doc = " @retval #OPUS_OK Success or @ref opus_errorcodes"] + pub fn opus_encoder_init( + st: *mut OpusEncoder, + Fs: opus_int32, + channels: ::std::os::raw::c_int, + application: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Encodes an Opus frame."] + #[doc = " @param [in] st OpusEncoder*: Encoder state"] + #[doc = " @param [in] pcm opus_int16*: Input signal (interleaved if 2 channels). length is frame_size*channels*sizeof(opus_int16)"] + #[doc = " @param [in] frame_size int: Number of samples per channel in the"] + #[doc = " input signal."] + #[doc = " This must be an Opus frame size for"] + #[doc = " the encoder's sampling rate."] + #[doc = " For example, at 48 kHz the permitted"] + #[doc = " values are 120, 240, 480, 960, 1920,"] + #[doc = " and 2880."] + #[doc = " Passing in a duration of less than"] + #[doc = " 10 ms (480 samples at 48 kHz) will"] + #[doc = " prevent the encoder from using the LPC"] + #[doc = " or hybrid modes."] + #[doc = " @param [out] data unsigned char*: Output payload."] + #[doc = " This must contain storage for at"] + #[doc = " least \\a max_data_bytes."] + #[doc = " @param [in] max_data_bytes opus_int32: Size of the allocated"] + #[doc = " memory for the output"] + #[doc = " payload. This may be"] + #[doc = " used to impose an upper limit on"] + #[doc = " the instant bitrate, but should"] + #[doc = " not be used as the only bitrate"] + #[doc = " control. Use #OPUS_SET_BITRATE to"] + #[doc = " control the bitrate."] + #[doc = " @returns The length of the encoded packet (in bytes) on success or a"] + #[doc = " negative error code (see @ref opus_errorcodes) on failure."] + pub fn opus_encode( + st: *mut OpusEncoder, + pcm: *const opus_int16, + frame_size: ::std::os::raw::c_int, + data: *mut ::std::os::raw::c_uchar, + max_data_bytes: opus_int32, + ) -> opus_int32; +} +extern "C" { + #[doc = " Encodes an Opus frame from floating point input."] + #[doc = " @param [in] st OpusEncoder*: Encoder state"] + #[doc = " @param [in] pcm float*: Input in float format (interleaved if 2 channels), with a normal range of +/-1.0."] + #[doc = " Samples with a range beyond +/-1.0 are supported but will"] + #[doc = " be clipped by decoders using the integer API and should"] + #[doc = " only be used if it is known that the far end supports"] + #[doc = " extended dynamic range."] + #[doc = " length is frame_size*channels*sizeof(float)"] + #[doc = " @param [in] frame_size int: Number of samples per channel in the"] + #[doc = " input signal."] + #[doc = " This must be an Opus frame size for"] + #[doc = " the encoder's sampling rate."] + #[doc = " For example, at 48 kHz the permitted"] + #[doc = " values are 120, 240, 480, 960, 1920,"] + #[doc = " and 2880."] + #[doc = " Passing in a duration of less than"] + #[doc = " 10 ms (480 samples at 48 kHz) will"] + #[doc = " prevent the encoder from using the LPC"] + #[doc = " or hybrid modes."] + #[doc = " @param [out] data unsigned char*: Output payload."] + #[doc = " This must contain storage for at"] + #[doc = " least \\a max_data_bytes."] + #[doc = " @param [in] max_data_bytes opus_int32: Size of the allocated"] + #[doc = " memory for the output"] + #[doc = " payload. This may be"] + #[doc = " used to impose an upper limit on"] + #[doc = " the instant bitrate, but should"] + #[doc = " not be used as the only bitrate"] + #[doc = " control. Use #OPUS_SET_BITRATE to"] + #[doc = " control the bitrate."] + #[doc = " @returns The length of the encoded packet (in bytes) on success or a"] + #[doc = " negative error code (see @ref opus_errorcodes) on failure."] + pub fn opus_encode_float( + st: *mut OpusEncoder, + pcm: *const f32, + frame_size: ::std::os::raw::c_int, + data: *mut ::std::os::raw::c_uchar, + max_data_bytes: opus_int32, + ) -> opus_int32; +} +extern "C" { + #[doc = " Frees an OpusEncoder allocated by opus_encoder_create()."] + #[doc = " @param[in] st OpusEncoder*: State to be freed."] + pub fn opus_encoder_destroy(st: *mut OpusEncoder); +} +extern "C" { + #[doc = " Perform a CTL function on an Opus encoder."] + #[doc = ""] + #[doc = " Generally the request and subsequent arguments are generated"] + #[doc = " by a convenience macro."] + #[doc = " @param st OpusEncoder*: Encoder state."] + #[doc = " @param request This and all remaining parameters should be replaced by one"] + #[doc = " of the convenience macros in @ref opus_genericctls or"] + #[doc = " @ref opus_encoderctls."] + #[doc = " @see opus_genericctls"] + #[doc = " @see opus_encoderctls"] + pub fn opus_encoder_ctl( + st: *mut OpusEncoder, + request: ::std::os::raw::c_int, + ... + ) -> ::std::os::raw::c_int; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OpusDecoder { + _unused: [u8; 0], +} +extern "C" { + #[doc = " Gets the size of an OpusDecoder structure."] + #[doc = " @param [in] channels int: Number of channels."] + #[doc = " This must be 1 or 2."] + #[doc = " @returns The size in bytes."] + pub fn opus_decoder_get_size(channels: ::std::os::raw::c_int) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Allocates and initializes a decoder state."] + #[doc = " @param [in] Fs opus_int32: Sample rate to decode at (Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param [in] channels int: Number of channels (1 or 2) to decode"] + #[doc = " @param [out] error int*: #OPUS_OK Success or @ref opus_errorcodes"] + #[doc = ""] + #[doc = " Internally Opus stores data at 48000 Hz, so that should be the default"] + #[doc = " value for Fs. However, the decoder can efficiently decode to buffers"] + #[doc = " at 8, 12, 16, and 24 kHz so if for some reason the caller cannot use"] + #[doc = " data at the full sample rate, or knows the compressed data doesn't"] + #[doc = " use the full frequency range, it can request decoding at a reduced"] + #[doc = " rate. Likewise, the decoder is capable of filling in either mono or"] + #[doc = " interleaved stereo pcm buffers, at the caller's request."] + pub fn opus_decoder_create( + Fs: opus_int32, + channels: ::std::os::raw::c_int, + error: *mut ::std::os::raw::c_int, + ) -> *mut OpusDecoder; +} +extern "C" { + #[doc = " Initializes a previously allocated decoder state."] + #[doc = " The state must be at least the size returned by opus_decoder_get_size()."] + #[doc = " This is intended for applications which use their own allocator instead of malloc. @see opus_decoder_create,opus_decoder_get_size"] + #[doc = " To reset a previously initialized state, use the #OPUS_RESET_STATE CTL."] + #[doc = " @param [in] st OpusDecoder*: Decoder state."] + #[doc = " @param [in] Fs opus_int32: Sampling rate to decode to (Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param [in] channels int: Number of channels (1 or 2) to decode"] + #[doc = " @retval #OPUS_OK Success or @ref opus_errorcodes"] + pub fn opus_decoder_init( + st: *mut OpusDecoder, + Fs: opus_int32, + channels: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Decode an Opus packet."] + #[doc = " @param [in] st OpusDecoder*: Decoder state"] + #[doc = " @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss"] + #[doc = " @param [in] len opus_int32: Number of bytes in payload*"] + #[doc = " @param [out] pcm opus_int16*: Output signal (interleaved if 2 channels). length"] + #[doc = " is frame_size*channels*sizeof(opus_int16)"] + #[doc = " @param [in] frame_size Number of samples per channel of available space in \\a pcm."] + #[doc = " If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will"] + #[doc = " not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1),"] + #[doc = " then frame_size needs to be exactly the duration of audio that is missing, otherwise the"] + #[doc = " decoder will not be in the optimal state to decode the next incoming packet. For the PLC and"] + #[doc = " FEC cases, frame_size must be a multiple of 2.5 ms."] + #[doc = " @param [in] decode_fec int: Flag (0 or 1) to request that any in-band forward error correction data be"] + #[doc = " decoded. If no such data is available, the frame is decoded as if it were lost."] + #[doc = " @returns Number of decoded samples or @ref opus_errorcodes"] + pub fn opus_decode( + st: *mut OpusDecoder, + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + pcm: *mut opus_int16, + frame_size: ::std::os::raw::c_int, + decode_fec: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Decode an Opus packet with floating point output."] + #[doc = " @param [in] st OpusDecoder*: Decoder state"] + #[doc = " @param [in] data char*: Input payload. Use a NULL pointer to indicate packet loss"] + #[doc = " @param [in] len opus_int32: Number of bytes in payload"] + #[doc = " @param [out] pcm float*: Output signal (interleaved if 2 channels). length"] + #[doc = " is frame_size*channels*sizeof(float)"] + #[doc = " @param [in] frame_size Number of samples per channel of available space in \\a pcm."] + #[doc = " If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will"] + #[doc = " not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1),"] + #[doc = " then frame_size needs to be exactly the duration of audio that is missing, otherwise the"] + #[doc = " decoder will not be in the optimal state to decode the next incoming packet. For the PLC and"] + #[doc = " FEC cases, frame_size must be a multiple of 2.5 ms."] + #[doc = " @param [in] decode_fec int: Flag (0 or 1) to request that any in-band forward error correction data be"] + #[doc = " decoded. If no such data is available the frame is decoded as if it were lost."] + #[doc = " @returns Number of decoded samples or @ref opus_errorcodes"] + pub fn opus_decode_float( + st: *mut OpusDecoder, + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + pcm: *mut f32, + frame_size: ::std::os::raw::c_int, + decode_fec: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Perform a CTL function on an Opus decoder."] + #[doc = ""] + #[doc = " Generally the request and subsequent arguments are generated"] + #[doc = " by a convenience macro."] + #[doc = " @param st OpusDecoder*: Decoder state."] + #[doc = " @param request This and all remaining parameters should be replaced by one"] + #[doc = " of the convenience macros in @ref opus_genericctls or"] + #[doc = " @ref opus_decoderctls."] + #[doc = " @see opus_genericctls"] + #[doc = " @see opus_decoderctls"] + pub fn opus_decoder_ctl( + st: *mut OpusDecoder, + request: ::std::os::raw::c_int, + ... + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Frees an OpusDecoder allocated by opus_decoder_create()."] + #[doc = " @param[in] st OpusDecoder*: State to be freed."] + pub fn opus_decoder_destroy(st: *mut OpusDecoder); +} +extern "C" { + #[doc = " Parse an opus packet into one or more frames."] + #[doc = " Opus_decode will perform this operation internally so most applications do"] + #[doc = " not need to use this function."] + #[doc = " This function does not copy the frames, the returned pointers are pointers into"] + #[doc = " the input packet."] + #[doc = " @param [in] data char*: Opus packet to be parsed"] + #[doc = " @param [in] len opus_int32: size of data"] + #[doc = " @param [out] out_toc char*: TOC pointer"] + #[doc = " @param [out] frames char*[48] encapsulated frames"] + #[doc = " @param [out] size opus_int16[48] sizes of the encapsulated frames"] + #[doc = " @param [out] payload_offset int*: returns the position of the payload within the packet (in bytes)"] + #[doc = " @returns number of frames"] + pub fn opus_packet_parse( + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + out_toc: *mut ::std::os::raw::c_uchar, + frames: *mut *const ::std::os::raw::c_uchar, + size: *mut opus_int16, + payload_offset: *mut ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the bandwidth of an Opus packet."] + #[doc = " @param [in] data char*: Opus packet"] + #[doc = " @retval OPUS_BANDWIDTH_NARROWBAND Narrowband (4kHz bandpass)"] + #[doc = " @retval OPUS_BANDWIDTH_MEDIUMBAND Mediumband (6kHz bandpass)"] + #[doc = " @retval OPUS_BANDWIDTH_WIDEBAND Wideband (8kHz bandpass)"] + #[doc = " @retval OPUS_BANDWIDTH_SUPERWIDEBAND Superwideband (12kHz bandpass)"] + #[doc = " @retval OPUS_BANDWIDTH_FULLBAND Fullband (20kHz bandpass)"] + #[doc = " @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type"] + pub fn opus_packet_get_bandwidth(data: *const ::std::os::raw::c_uchar) + -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the number of samples per frame from an Opus packet."] + #[doc = " @param [in] data char*: Opus packet."] + #[doc = " This must contain at least one byte of"] + #[doc = " data."] + #[doc = " @param [in] Fs opus_int32: Sampling rate in Hz."] + #[doc = " This must be a multiple of 400, or"] + #[doc = " inaccurate results will be returned."] + #[doc = " @returns Number of samples per frame."] + pub fn opus_packet_get_samples_per_frame( + data: *const ::std::os::raw::c_uchar, + Fs: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the number of channels from an Opus packet."] + #[doc = " @param [in] data char*: Opus packet"] + #[doc = " @returns Number of channels"] + #[doc = " @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type"] + pub fn opus_packet_get_nb_channels( + data: *const ::std::os::raw::c_uchar, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the number of frames in an Opus packet."] + #[doc = " @param [in] packet char*: Opus packet"] + #[doc = " @param [in] len opus_int32: Length of packet"] + #[doc = " @returns Number of frames"] + #[doc = " @retval OPUS_BAD_ARG Insufficient data was passed to the function"] + #[doc = " @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type"] + pub fn opus_packet_get_nb_frames( + packet: *const ::std::os::raw::c_uchar, + len: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the number of samples of an Opus packet."] + #[doc = " @param [in] packet char*: Opus packet"] + #[doc = " @param [in] len opus_int32: Length of packet"] + #[doc = " @param [in] Fs opus_int32: Sampling rate in Hz."] + #[doc = " This must be a multiple of 400, or"] + #[doc = " inaccurate results will be returned."] + #[doc = " @returns Number of samples"] + #[doc = " @retval OPUS_BAD_ARG Insufficient data was passed to the function"] + #[doc = " @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type"] + pub fn opus_packet_get_nb_samples( + packet: *const ::std::os::raw::c_uchar, + len: opus_int32, + Fs: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the number of samples of an Opus packet."] + #[doc = " @param [in] dec OpusDecoder*: Decoder state"] + #[doc = " @param [in] packet char*: Opus packet"] + #[doc = " @param [in] len opus_int32: Length of packet"] + #[doc = " @returns Number of samples"] + #[doc = " @retval OPUS_BAD_ARG Insufficient data was passed to the function"] + #[doc = " @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type"] + pub fn opus_decoder_get_nb_samples( + dec: *const OpusDecoder, + packet: *const ::std::os::raw::c_uchar, + len: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Applies soft-clipping to bring a float signal within the [-1,1] range. If"] + #[doc = " the signal is already in that range, nothing is done. If there are values"] + #[doc = " outside of [-1,1], then the signal is clipped as smoothly as possible to"] + #[doc = " both fit in the range and avoid creating excessive distortion in the"] + #[doc = " process."] + #[doc = " @param [in,out] pcm float*: Input PCM and modified PCM"] + #[doc = " @param [in] frame_size int Number of samples per channel to process"] + #[doc = " @param [in] channels int: Number of channels"] + #[doc = " @param [in,out] softclip_mem float*: State memory for the soft clipping process (one float per channel, initialized to zero)"] + pub fn opus_pcm_soft_clip( + pcm: *mut f32, + frame_size: ::std::os::raw::c_int, + channels: ::std::os::raw::c_int, + softclip_mem: *mut f32, + ); +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OpusRepacketizer { + _unused: [u8; 0], +} +extern "C" { + #[doc = " Gets the size of an OpusRepacketizer structure."] + #[doc = " @returns The size in bytes."] + pub fn opus_repacketizer_get_size() -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " (Re)initializes a previously allocated repacketizer state."] + #[doc = " The state must be at least the size returned by opus_repacketizer_get_size()."] + #[doc = " This can be used for applications which use their own allocator instead of"] + #[doc = " malloc()."] + #[doc = " It must also be called to reset the queue of packets waiting to be"] + #[doc = " repacketized, which is necessary if the maximum packet duration of 120 ms"] + #[doc = " is reached or if you wish to submit packets with a different Opus"] + #[doc = " configuration (coding mode, audio bandwidth, frame size, or channel count)."] + #[doc = " Failure to do so will prevent a new packet from being added with"] + #[doc = " opus_repacketizer_cat()."] + #[doc = " @see opus_repacketizer_create"] + #[doc = " @see opus_repacketizer_get_size"] + #[doc = " @see opus_repacketizer_cat"] + #[doc = " @param rp OpusRepacketizer*: The repacketizer state to"] + #[doc = " (re)initialize."] + #[doc = " @returns A pointer to the same repacketizer state that was passed in."] + pub fn opus_repacketizer_init(rp: *mut OpusRepacketizer) -> *mut OpusRepacketizer; +} +extern "C" { + #[doc = " Allocates memory and initializes the new repacketizer with"] + #[doc = " opus_repacketizer_init()."] + pub fn opus_repacketizer_create() -> *mut OpusRepacketizer; +} +extern "C" { + #[doc = " Frees an OpusRepacketizer allocated by"] + #[doc = " opus_repacketizer_create()."] + #[doc = " @param[in] rp OpusRepacketizer*: State to be freed."] + pub fn opus_repacketizer_destroy(rp: *mut OpusRepacketizer); +} +extern "C" { + #[doc = " Add a packet to the current repacketizer state."] + #[doc = " This packet must match the configuration of any packets already submitted"] + #[doc = " for repacketization since the last call to opus_repacketizer_init()."] + #[doc = " This means that it must have the same coding mode, audio bandwidth, frame"] + #[doc = " size, and channel count."] + #[doc = " This can be checked in advance by examining the top 6 bits of the first"] + #[doc = " byte of the packet, and ensuring they match the top 6 bits of the first"] + #[doc = " byte of any previously submitted packet."] + #[doc = " The total duration of audio in the repacketizer state also must not exceed"] + #[doc = " 120 ms, the maximum duration of a single packet, after adding this packet."] + #[doc = ""] + #[doc = " The contents of the current repacketizer state can be extracted into new"] + #[doc = " packets using opus_repacketizer_out() or opus_repacketizer_out_range()."] + #[doc = ""] + #[doc = " In order to add a packet with a different configuration or to add more"] + #[doc = " audio beyond 120 ms, you must clear the repacketizer state by calling"] + #[doc = " opus_repacketizer_init()."] + #[doc = " If a packet is too large to add to the current repacketizer state, no part"] + #[doc = " of it is added, even if it contains multiple frames, some of which might"] + #[doc = " fit."] + #[doc = " If you wish to be able to add parts of such packets, you should first use"] + #[doc = " another repacketizer to split the packet into pieces and add them"] + #[doc = " individually."] + #[doc = " @see opus_repacketizer_out_range"] + #[doc = " @see opus_repacketizer_out"] + #[doc = " @see opus_repacketizer_init"] + #[doc = " @param rp OpusRepacketizer*: The repacketizer state to which to"] + #[doc = " add the packet."] + #[doc = " @param[in] data const unsigned char*: The packet data."] + #[doc = " The application must ensure"] + #[doc = " this pointer remains valid"] + #[doc = " until the next call to"] + #[doc = " opus_repacketizer_init() or"] + #[doc = " opus_repacketizer_destroy()."] + #[doc = " @param len opus_int32: The number of bytes in the packet data."] + #[doc = " @returns An error code indicating whether or not the operation succeeded."] + #[doc = " @retval #OPUS_OK The packet's contents have been added to the repacketizer"] + #[doc = " state."] + #[doc = " @retval #OPUS_INVALID_PACKET The packet did not have a valid TOC sequence,"] + #[doc = " the packet's TOC sequence was not compatible"] + #[doc = " with previously submitted packets (because"] + #[doc = " the coding mode, audio bandwidth, frame size,"] + #[doc = " or channel count did not match), or adding"] + #[doc = " this packet would increase the total amount of"] + #[doc = " audio stored in the repacketizer state to more"] + #[doc = " than 120 ms."] + pub fn opus_repacketizer_cat( + rp: *mut OpusRepacketizer, + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Construct a new packet from data previously submitted to the repacketizer"] + #[doc = " state via opus_repacketizer_cat()."] + #[doc = " @param rp OpusRepacketizer*: The repacketizer state from which to"] + #[doc = " construct the new packet."] + #[doc = " @param begin int: The index of the first frame in the current"] + #[doc = " repacketizer state to include in the output."] + #[doc = " @param end int: One past the index of the last frame in the"] + #[doc = " current repacketizer state to include in the"] + #[doc = " output."] + #[doc = " @param[out] data const unsigned char*: The buffer in which to"] + #[doc = " store the output packet."] + #[doc = " @param maxlen opus_int32: The maximum number of bytes to store in"] + #[doc = " the output buffer. In order to guarantee"] + #[doc = " success, this should be at least"] + #[doc = " 1276 for a single frame,"] + #[doc = " or for multiple frames,"] + #[doc = " 1277*(end-begin)."] + #[doc = " However, 1*(end-begin) plus"] + #[doc = " the size of all packet data submitted to"] + #[doc = " the repacketizer since the last call to"] + #[doc = " opus_repacketizer_init() or"] + #[doc = " opus_repacketizer_create() is also"] + #[doc = " sufficient, and possibly much smaller."] + #[doc = " @returns The total size of the output packet on success, or an error code"] + #[doc = " on failure."] + #[doc = " @retval #OPUS_BAD_ARG [begin,end) was an invalid range of"] + #[doc = " frames (begin < 0, begin >= end, or end >"] + #[doc = " opus_repacketizer_get_nb_frames())."] + #[doc = " @retval #OPUS_BUFFER_TOO_SMALL \\a maxlen was insufficient to contain the"] + #[doc = " complete output packet."] + pub fn opus_repacketizer_out_range( + rp: *mut OpusRepacketizer, + begin: ::std::os::raw::c_int, + end: ::std::os::raw::c_int, + data: *mut ::std::os::raw::c_uchar, + maxlen: opus_int32, + ) -> opus_int32; +} +extern "C" { + #[doc = " Return the total number of frames contained in packet data submitted to"] + #[doc = " the repacketizer state so far via opus_repacketizer_cat() since the last"] + #[doc = " call to opus_repacketizer_init() or opus_repacketizer_create()."] + #[doc = " This defines the valid range of packets that can be extracted with"] + #[doc = " opus_repacketizer_out_range() or opus_repacketizer_out()."] + #[doc = " @param rp OpusRepacketizer*: The repacketizer state containing the"] + #[doc = " frames."] + #[doc = " @returns The total number of frames contained in the packet data submitted"] + #[doc = " to the repacketizer state."] + pub fn opus_repacketizer_get_nb_frames(rp: *mut OpusRepacketizer) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Construct a new packet from data previously submitted to the repacketizer"] + #[doc = " state via opus_repacketizer_cat()."] + #[doc = " This is a convenience routine that returns all the data submitted so far"] + #[doc = " in a single packet."] + #[doc = " It is equivalent to calling"] + #[doc = " @code"] + #[doc = " opus_repacketizer_out_range(rp, 0, opus_repacketizer_get_nb_frames(rp),"] + #[doc = " data, maxlen)"] + #[doc = " @endcode"] + #[doc = " @param rp OpusRepacketizer*: The repacketizer state from which to"] + #[doc = " construct the new packet."] + #[doc = " @param[out] data const unsigned char*: The buffer in which to"] + #[doc = " store the output packet."] + #[doc = " @param maxlen opus_int32: The maximum number of bytes to store in"] + #[doc = " the output buffer. In order to guarantee"] + #[doc = " success, this should be at least"] + #[doc = " 1277*opus_repacketizer_get_nb_frames(rp)."] + #[doc = " However,"] + #[doc = " 1*opus_repacketizer_get_nb_frames(rp)"] + #[doc = " plus the size of all packet data"] + #[doc = " submitted to the repacketizer since the"] + #[doc = " last call to opus_repacketizer_init() or"] + #[doc = " opus_repacketizer_create() is also"] + #[doc = " sufficient, and possibly much smaller."] + #[doc = " @returns The total size of the output packet on success, or an error code"] + #[doc = " on failure."] + #[doc = " @retval #OPUS_BUFFER_TOO_SMALL \\a maxlen was insufficient to contain the"] + #[doc = " complete output packet."] + pub fn opus_repacketizer_out( + rp: *mut OpusRepacketizer, + data: *mut ::std::os::raw::c_uchar, + maxlen: opus_int32, + ) -> opus_int32; +} +extern "C" { + #[doc = " Pads a given Opus packet to a larger size (possibly changing the TOC sequence)."] + #[doc = " @param[in,out] data const unsigned char*: The buffer containing the"] + #[doc = " packet to pad."] + #[doc = " @param len opus_int32: The size of the packet."] + #[doc = " This must be at least 1."] + #[doc = " @param new_len opus_int32: The desired size of the packet after padding."] + #[doc = " This must be at least as large as len."] + #[doc = " @returns an error code"] + #[doc = " @retval #OPUS_OK \\a on success."] + #[doc = " @retval #OPUS_BAD_ARG \\a len was less than 1 or new_len was less than len."] + #[doc = " @retval #OPUS_INVALID_PACKET \\a data did not contain a valid Opus packet."] + pub fn opus_packet_pad( + data: *mut ::std::os::raw::c_uchar, + len: opus_int32, + new_len: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Remove all padding from a given Opus packet and rewrite the TOC sequence to"] + #[doc = " minimize space usage."] + #[doc = " @param[in,out] data const unsigned char*: The buffer containing the"] + #[doc = " packet to strip."] + #[doc = " @param len opus_int32: The size of the packet."] + #[doc = " This must be at least 1."] + #[doc = " @returns The new size of the output packet on success, or an error code"] + #[doc = " on failure."] + #[doc = " @retval #OPUS_BAD_ARG \\a len was less than 1."] + #[doc = " @retval #OPUS_INVALID_PACKET \\a data did not contain a valid Opus packet."] + pub fn opus_packet_unpad(data: *mut ::std::os::raw::c_uchar, len: opus_int32) -> opus_int32; +} +extern "C" { + #[doc = " Pads a given Opus multi-stream packet to a larger size (possibly changing the TOC sequence)."] + #[doc = " @param[in,out] data const unsigned char*: The buffer containing the"] + #[doc = " packet to pad."] + #[doc = " @param len opus_int32: The size of the packet."] + #[doc = " This must be at least 1."] + #[doc = " @param new_len opus_int32: The desired size of the packet after padding."] + #[doc = " This must be at least 1."] + #[doc = " @param nb_streams opus_int32: The number of streams (not channels) in the packet."] + #[doc = " This must be at least as large as len."] + #[doc = " @returns an error code"] + #[doc = " @retval #OPUS_OK \\a on success."] + #[doc = " @retval #OPUS_BAD_ARG \\a len was less than 1."] + #[doc = " @retval #OPUS_INVALID_PACKET \\a data did not contain a valid Opus packet."] + pub fn opus_multistream_packet_pad( + data: *mut ::std::os::raw::c_uchar, + len: opus_int32, + new_len: opus_int32, + nb_streams: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Remove all padding from a given Opus multi-stream packet and rewrite the TOC sequence to"] + #[doc = " minimize space usage."] + #[doc = " @param[in,out] data const unsigned char*: The buffer containing the"] + #[doc = " packet to strip."] + #[doc = " @param len opus_int32: The size of the packet."] + #[doc = " This must be at least 1."] + #[doc = " @param nb_streams opus_int32: The number of streams (not channels) in the packet."] + #[doc = " This must be at least 1."] + #[doc = " @returns The new size of the output packet on success, or an error code"] + #[doc = " on failure."] + #[doc = " @retval #OPUS_BAD_ARG \\a len was less than 1 or new_len was less than len."] + #[doc = " @retval #OPUS_INVALID_PACKET \\a data did not contain a valid Opus packet."] + pub fn opus_multistream_packet_unpad( + data: *mut ::std::os::raw::c_uchar, + len: opus_int32, + nb_streams: ::std::os::raw::c_int, + ) -> opus_int32; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OpusMSEncoder { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OpusMSDecoder { + _unused: [u8; 0], +} +extern "C" { + #[doc = " Gets the size of an OpusMSEncoder structure."] + #[doc = " @param streams int: The total number of streams to encode from the"] + #[doc = " input."] + #[doc = " This must be no more than 255."] + #[doc = " @param coupled_streams int: Number of coupled (2 channel) streams"] + #[doc = " to encode."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " encoded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than 255."] + #[doc = " @returns The size in bytes on success, or a negative error code"] + #[doc = " (see @ref opus_errorcodes) on error."] + pub fn opus_multistream_encoder_get_size( + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + ) -> opus_int32; +} +extern "C" { + pub fn opus_multistream_surround_encoder_get_size( + channels: ::std::os::raw::c_int, + mapping_family: ::std::os::raw::c_int, + ) -> opus_int32; +} +extern "C" { + #[doc = " Allocates and initializes a multistream encoder state."] + #[doc = " Call opus_multistream_encoder_destroy() to release"] + #[doc = " this object when finished."] + #[doc = " @param Fs opus_int32: Sampling rate of the input signal (in Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param channels int: Number of channels in the input signal."] + #[doc = " This must be at most 255."] + #[doc = " It may be greater than the number of"] + #[doc = " coded channels (streams +"] + #[doc = " coupled_streams)."] + #[doc = " @param streams int: The total number of streams to encode from the"] + #[doc = " input."] + #[doc = " This must be no more than the number of channels."] + #[doc = " @param coupled_streams int: Number of coupled (2 channel) streams"] + #[doc = " to encode."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " encoded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than the number of input channels."] + #[doc = " @param[in] mapping const unsigned char[channels]: Mapping from"] + #[doc = " encoded channels to input channels, as described in"] + #[doc = " @ref opus_multistream. As an extra constraint, the"] + #[doc = " multistream encoder does not allow encoding coupled"] + #[doc = " streams for which one channel is unused since this"] + #[doc = " is never a good idea."] + #[doc = " @param application int: The target encoder application."] + #[doc = " This must be one of the following:"] + #[doc = "
"] + #[doc = "
#OPUS_APPLICATION_VOIP
"] + #[doc = "
Process signal for improved speech intelligibility.
"] + #[doc = "
#OPUS_APPLICATION_AUDIO
"] + #[doc = "
Favor faithfulness to the original input.
"] + #[doc = "
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
"] + #[doc = "
Configure the minimum possible coding delay by disabling certain modes"] + #[doc = " of operation.
"] + #[doc = "
"] + #[doc = " @param[out] error int *: Returns #OPUS_OK on success, or an error"] + #[doc = " code (see @ref opus_errorcodes) on"] + #[doc = " failure."] + pub fn opus_multistream_encoder_create( + Fs: opus_int32, + channels: ::std::os::raw::c_int, + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + mapping: *const ::std::os::raw::c_uchar, + application: ::std::os::raw::c_int, + error: *mut ::std::os::raw::c_int, + ) -> *mut OpusMSEncoder; +} +extern "C" { + pub fn opus_multistream_surround_encoder_create( + Fs: opus_int32, + channels: ::std::os::raw::c_int, + mapping_family: ::std::os::raw::c_int, + streams: *mut ::std::os::raw::c_int, + coupled_streams: *mut ::std::os::raw::c_int, + mapping: *mut ::std::os::raw::c_uchar, + application: ::std::os::raw::c_int, + error: *mut ::std::os::raw::c_int, + ) -> *mut OpusMSEncoder; +} +extern "C" { + #[doc = " Initialize a previously allocated multistream encoder state."] + #[doc = " The memory pointed to by \\a st must be at least the size returned by"] + #[doc = " opus_multistream_encoder_get_size()."] + #[doc = " This is intended for applications which use their own allocator instead of"] + #[doc = " malloc."] + #[doc = " To reset a previously initialized state, use the #OPUS_RESET_STATE CTL."] + #[doc = " @see opus_multistream_encoder_create"] + #[doc = " @see opus_multistream_encoder_get_size"] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state to initialize."] + #[doc = " @param Fs opus_int32: Sampling rate of the input signal (in Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param channels int: Number of channels in the input signal."] + #[doc = " This must be at most 255."] + #[doc = " It may be greater than the number of"] + #[doc = " coded channels (streams +"] + #[doc = " coupled_streams)."] + #[doc = " @param streams int: The total number of streams to encode from the"] + #[doc = " input."] + #[doc = " This must be no more than the number of channels."] + #[doc = " @param coupled_streams int: Number of coupled (2 channel) streams"] + #[doc = " to encode."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " encoded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than the number of input channels."] + #[doc = " @param[in] mapping const unsigned char[channels]: Mapping from"] + #[doc = " encoded channels to input channels, as described in"] + #[doc = " @ref opus_multistream. As an extra constraint, the"] + #[doc = " multistream encoder does not allow encoding coupled"] + #[doc = " streams for which one channel is unused since this"] + #[doc = " is never a good idea."] + #[doc = " @param application int: The target encoder application."] + #[doc = " This must be one of the following:"] + #[doc = "
"] + #[doc = "
#OPUS_APPLICATION_VOIP
"] + #[doc = "
Process signal for improved speech intelligibility.
"] + #[doc = "
#OPUS_APPLICATION_AUDIO
"] + #[doc = "
Favor faithfulness to the original input.
"] + #[doc = "
#OPUS_APPLICATION_RESTRICTED_LOWDELAY
"] + #[doc = "
Configure the minimum possible coding delay by disabling certain modes"] + #[doc = " of operation.
"] + #[doc = "
"] + #[doc = " @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes)"] + #[doc = " on failure."] + pub fn opus_multistream_encoder_init( + st: *mut OpusMSEncoder, + Fs: opus_int32, + channels: ::std::os::raw::c_int, + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + mapping: *const ::std::os::raw::c_uchar, + application: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + pub fn opus_multistream_surround_encoder_init( + st: *mut OpusMSEncoder, + Fs: opus_int32, + channels: ::std::os::raw::c_int, + mapping_family: ::std::os::raw::c_int, + streams: *mut ::std::os::raw::c_int, + coupled_streams: *mut ::std::os::raw::c_int, + mapping: *mut ::std::os::raw::c_uchar, + application: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Encodes a multistream Opus frame."] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state."] + #[doc = " @param[in] pcm const opus_int16*: The input signal as interleaved"] + #[doc = " samples."] + #[doc = " This must contain"] + #[doc = " frame_size*channels"] + #[doc = " samples."] + #[doc = " @param frame_size int: Number of samples per channel in the input"] + #[doc = " signal."] + #[doc = " This must be an Opus frame size for the"] + #[doc = " encoder's sampling rate."] + #[doc = " For example, at 48 kHz the permitted values"] + #[doc = " are 120, 240, 480, 960, 1920, and 2880."] + #[doc = " Passing in a duration of less than 10 ms"] + #[doc = " (480 samples at 48 kHz) will prevent the"] + #[doc = " encoder from using the LPC or hybrid modes."] + #[doc = " @param[out] data unsigned char*: Output payload."] + #[doc = " This must contain storage for at"] + #[doc = " least \\a max_data_bytes."] + #[doc = " @param [in] max_data_bytes opus_int32: Size of the allocated"] + #[doc = " memory for the output"] + #[doc = " payload. This may be"] + #[doc = " used to impose an upper limit on"] + #[doc = " the instant bitrate, but should"] + #[doc = " not be used as the only bitrate"] + #[doc = " control. Use #OPUS_SET_BITRATE to"] + #[doc = " control the bitrate."] + #[doc = " @returns The length of the encoded packet (in bytes) on success or a"] + #[doc = " negative error code (see @ref opus_errorcodes) on failure."] + pub fn opus_multistream_encode( + st: *mut OpusMSEncoder, + pcm: *const opus_int16, + frame_size: ::std::os::raw::c_int, + data: *mut ::std::os::raw::c_uchar, + max_data_bytes: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Encodes a multistream Opus frame from floating point input."] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state."] + #[doc = " @param[in] pcm const float*: The input signal as interleaved"] + #[doc = " samples with a normal range of"] + #[doc = " +/-1.0."] + #[doc = " Samples with a range beyond +/-1.0"] + #[doc = " are supported but will be clipped by"] + #[doc = " decoders using the integer API and"] + #[doc = " should only be used if it is known"] + #[doc = " that the far end supports extended"] + #[doc = " dynamic range."] + #[doc = " This must contain"] + #[doc = " frame_size*channels"] + #[doc = " samples."] + #[doc = " @param frame_size int: Number of samples per channel in the input"] + #[doc = " signal."] + #[doc = " This must be an Opus frame size for the"] + #[doc = " encoder's sampling rate."] + #[doc = " For example, at 48 kHz the permitted values"] + #[doc = " are 120, 240, 480, 960, 1920, and 2880."] + #[doc = " Passing in a duration of less than 10 ms"] + #[doc = " (480 samples at 48 kHz) will prevent the"] + #[doc = " encoder from using the LPC or hybrid modes."] + #[doc = " @param[out] data unsigned char*: Output payload."] + #[doc = " This must contain storage for at"] + #[doc = " least \\a max_data_bytes."] + #[doc = " @param [in] max_data_bytes opus_int32: Size of the allocated"] + #[doc = " memory for the output"] + #[doc = " payload. This may be"] + #[doc = " used to impose an upper limit on"] + #[doc = " the instant bitrate, but should"] + #[doc = " not be used as the only bitrate"] + #[doc = " control. Use #OPUS_SET_BITRATE to"] + #[doc = " control the bitrate."] + #[doc = " @returns The length of the encoded packet (in bytes) on success or a"] + #[doc = " negative error code (see @ref opus_errorcodes) on failure."] + pub fn opus_multistream_encode_float( + st: *mut OpusMSEncoder, + pcm: *const f32, + frame_size: ::std::os::raw::c_int, + data: *mut ::std::os::raw::c_uchar, + max_data_bytes: opus_int32, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Frees an OpusMSEncoder allocated by"] + #[doc = " opus_multistream_encoder_create()."] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state to be freed."] + pub fn opus_multistream_encoder_destroy(st: *mut OpusMSEncoder); +} +extern "C" { + #[doc = " Perform a CTL function on a multistream Opus encoder."] + #[doc = ""] + #[doc = " Generally the request and subsequent arguments are generated by a"] + #[doc = " convenience macro."] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state."] + #[doc = " @param request This and all remaining parameters should be replaced by one"] + #[doc = " of the convenience macros in @ref opus_genericctls,"] + #[doc = " @ref opus_encoderctls, or @ref opus_multistream_ctls."] + #[doc = " @see opus_genericctls"] + #[doc = " @see opus_encoderctls"] + #[doc = " @see opus_multistream_ctls"] + pub fn opus_multistream_encoder_ctl( + st: *mut OpusMSEncoder, + request: ::std::os::raw::c_int, + ... + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Gets the size of an OpusMSDecoder structure."] + #[doc = " @param streams int: The total number of streams coded in the"] + #[doc = " input."] + #[doc = " This must be no more than 255."] + #[doc = " @param coupled_streams int: Number streams to decode as coupled"] + #[doc = " (2 channel) streams."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " coded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than 255."] + #[doc = " @returns The size in bytes on success, or a negative error code"] + #[doc = " (see @ref opus_errorcodes) on error."] + pub fn opus_multistream_decoder_get_size( + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + ) -> opus_int32; +} +extern "C" { + #[doc = " Allocates and initializes a multistream decoder state."] + #[doc = " Call opus_multistream_decoder_destroy() to release"] + #[doc = " this object when finished."] + #[doc = " @param Fs opus_int32: Sampling rate to decode at (in Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param channels int: Number of channels to output."] + #[doc = " This must be at most 255."] + #[doc = " It may be different from the number of coded"] + #[doc = " channels (streams +"] + #[doc = " coupled_streams)."] + #[doc = " @param streams int: The total number of streams coded in the"] + #[doc = " input."] + #[doc = " This must be no more than 255."] + #[doc = " @param coupled_streams int: Number of streams to decode as coupled"] + #[doc = " (2 channel) streams."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " coded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than 255."] + #[doc = " @param[in] mapping const unsigned char[channels]: Mapping from"] + #[doc = " coded channels to output channels, as described in"] + #[doc = " @ref opus_multistream."] + #[doc = " @param[out] error int *: Returns #OPUS_OK on success, or an error"] + #[doc = " code (see @ref opus_errorcodes) on"] + #[doc = " failure."] + pub fn opus_multistream_decoder_create( + Fs: opus_int32, + channels: ::std::os::raw::c_int, + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + mapping: *const ::std::os::raw::c_uchar, + error: *mut ::std::os::raw::c_int, + ) -> *mut OpusMSDecoder; +} +extern "C" { + #[doc = " Intialize a previously allocated decoder state object."] + #[doc = " The memory pointed to by \\a st must be at least the size returned by"] + #[doc = " opus_multistream_encoder_get_size()."] + #[doc = " This is intended for applications which use their own allocator instead of"] + #[doc = " malloc."] + #[doc = " To reset a previously initialized state, use the #OPUS_RESET_STATE CTL."] + #[doc = " @see opus_multistream_decoder_create"] + #[doc = " @see opus_multistream_deocder_get_size"] + #[doc = " @param st OpusMSEncoder*: Multistream encoder state to initialize."] + #[doc = " @param Fs opus_int32: Sampling rate to decode at (in Hz)."] + #[doc = " This must be one of 8000, 12000, 16000,"] + #[doc = " 24000, or 48000."] + #[doc = " @param channels int: Number of channels to output."] + #[doc = " This must be at most 255."] + #[doc = " It may be different from the number of coded"] + #[doc = " channels (streams +"] + #[doc = " coupled_streams)."] + #[doc = " @param streams int: The total number of streams coded in the"] + #[doc = " input."] + #[doc = " This must be no more than 255."] + #[doc = " @param coupled_streams int: Number of streams to decode as coupled"] + #[doc = " (2 channel) streams."] + #[doc = " This must be no larger than the total"] + #[doc = " number of streams."] + #[doc = " Additionally, The total number of"] + #[doc = " coded channels (streams +"] + #[doc = " coupled_streams) must be no"] + #[doc = " more than 255."] + #[doc = " @param[in] mapping const unsigned char[channels]: Mapping from"] + #[doc = " coded channels to output channels, as described in"] + #[doc = " @ref opus_multistream."] + #[doc = " @returns #OPUS_OK on success, or an error code (see @ref opus_errorcodes)"] + #[doc = " on failure."] + pub fn opus_multistream_decoder_init( + st: *mut OpusMSDecoder, + Fs: opus_int32, + channels: ::std::os::raw::c_int, + streams: ::std::os::raw::c_int, + coupled_streams: ::std::os::raw::c_int, + mapping: *const ::std::os::raw::c_uchar, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Decode a multistream Opus packet."] + #[doc = " @param st OpusMSDecoder*: Multistream decoder state."] + #[doc = " @param[in] data const unsigned char*: Input payload."] + #[doc = " Use a NULL"] + #[doc = " pointer to indicate packet"] + #[doc = " loss."] + #[doc = " @param len opus_int32: Number of bytes in payload."] + #[doc = " @param[out] pcm opus_int16*: Output signal, with interleaved"] + #[doc = " samples."] + #[doc = " This must contain room for"] + #[doc = " frame_size*channels"] + #[doc = " samples."] + #[doc = " @param frame_size int: The number of samples per channel of"] + #[doc = " available space in \\a pcm."] + #[doc = " If this is less than the maximum packet duration"] + #[doc = " (120 ms; 5760 for 48kHz), this function will not be capable"] + #[doc = " of decoding some packets. In the case of PLC (data==NULL)"] + #[doc = " or FEC (decode_fec=1), then frame_size needs to be exactly"] + #[doc = " the duration of audio that is missing, otherwise the"] + #[doc = " decoder will not be in the optimal state to decode the"] + #[doc = " next incoming packet. For the PLC and FEC cases, frame_size"] + #[doc = " must be a multiple of 2.5 ms."] + #[doc = " @param decode_fec int: Flag (0 or 1) to request that any in-band"] + #[doc = " forward error correction data be decoded."] + #[doc = " If no such data is available, the frame is"] + #[doc = " decoded as if it were lost."] + #[doc = " @returns Number of samples decoded on success or a negative error code"] + #[doc = " (see @ref opus_errorcodes) on failure."] + pub fn opus_multistream_decode( + st: *mut OpusMSDecoder, + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + pcm: *mut opus_int16, + frame_size: ::std::os::raw::c_int, + decode_fec: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Decode a multistream Opus packet with floating point output."] + #[doc = " @param st OpusMSDecoder*: Multistream decoder state."] + #[doc = " @param[in] data const unsigned char*: Input payload."] + #[doc = " Use a NULL"] + #[doc = " pointer to indicate packet"] + #[doc = " loss."] + #[doc = " @param len opus_int32: Number of bytes in payload."] + #[doc = " @param[out] pcm opus_int16*: Output signal, with interleaved"] + #[doc = " samples."] + #[doc = " This must contain room for"] + #[doc = " frame_size*channels"] + #[doc = " samples."] + #[doc = " @param frame_size int: The number of samples per channel of"] + #[doc = " available space in \\a pcm."] + #[doc = " If this is less than the maximum packet duration"] + #[doc = " (120 ms; 5760 for 48kHz), this function will not be capable"] + #[doc = " of decoding some packets. In the case of PLC (data==NULL)"] + #[doc = " or FEC (decode_fec=1), then frame_size needs to be exactly"] + #[doc = " the duration of audio that is missing, otherwise the"] + #[doc = " decoder will not be in the optimal state to decode the"] + #[doc = " next incoming packet. For the PLC and FEC cases, frame_size"] + #[doc = " must be a multiple of 2.5 ms."] + #[doc = " @param decode_fec int: Flag (0 or 1) to request that any in-band"] + #[doc = " forward error correction data be decoded."] + #[doc = " If no such data is available, the frame is"] + #[doc = " decoded as if it were lost."] + #[doc = " @returns Number of samples decoded on success or a negative error code"] + #[doc = " (see @ref opus_errorcodes) on failure."] + pub fn opus_multistream_decode_float( + st: *mut OpusMSDecoder, + data: *const ::std::os::raw::c_uchar, + len: opus_int32, + pcm: *mut f32, + frame_size: ::std::os::raw::c_int, + decode_fec: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Perform a CTL function on a multistream Opus decoder."] + #[doc = ""] + #[doc = " Generally the request and subsequent arguments are generated by a"] + #[doc = " convenience macro."] + #[doc = " @param st OpusMSDecoder*: Multistream decoder state."] + #[doc = " @param request This and all remaining parameters should be replaced by one"] + #[doc = " of the convenience macros in @ref opus_genericctls,"] + #[doc = " @ref opus_decoderctls, or @ref opus_multistream_ctls."] + #[doc = " @see opus_genericctls"] + #[doc = " @see opus_decoderctls"] + #[doc = " @see opus_multistream_ctls"] + pub fn opus_multistream_decoder_ctl( + st: *mut OpusMSDecoder, + request: ::std::os::raw::c_int, + ... + ) -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " Frees an OpusMSDecoder allocated by"] + #[doc = " opus_multistream_decoder_create()."] + #[doc = " @param st OpusMSDecoder: Multistream decoder state to be freed."] + pub fn opus_multistream_decoder_destroy(st: *mut OpusMSDecoder); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn access_symbols() { + unsafe { + opus_get_version_string(); + opus_encoder_get_size(0); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/wrapper.h b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/wrapper.h new file mode 100644 index 000000000..a04539112 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/audiopus-sys/src/wrapper.h @@ -0,0 +1 @@ +#include "../opus/include/opus_multistream.h" diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/.cargo_vcs_info.json b/external/opennow/native/opennow-streamer/vendor/str0m/.cargo_vcs_info.json new file mode 100644 index 000000000..3bcd61bc0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "67820c52ff58df5350659063129e94d7d87225e4" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/CHANGELOG.md b/external/opennow/native/opennow-streamer/vendor/str0m/CHANGELOG.md new file mode 100644 index 000000000..7358000c0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/CHANGELOG.md @@ -0,0 +1,433 @@ +# Unreleased + +# 0.23.0 + + * Respond to RRTR with DLRR #1022 + * Use a local SSRC as the originator of outgoing RRTR reports #1027 + * Add `DirectApi::send_pli_feedback` to send a PLI with a chosen sender SSRC #1025 + +# 0.22.0 + + * Make audio and video RTCP report intervals configurable #1021 + * Stop padding media packets, no crypto need it #1023 + * Reset SCTP streams when closing data channels and safely reuse stream IDs #1010 + * Fix small padding invisible to the pacer #1020 + * Fix H.264 negotiation: accept a negotiable (downgradable) level and recognize the Constrained High profile #1016 + * Expose the latest send queue information on `StreamTx` #1005 + * Expose locally calculated interarrival jitter on `MediaIngressStats` #997 + * Fix VP9 non-flexible SS advertising a bogus R=0 GOF for single-layer streams #999 + * Fix AV1 depacketizer overflow on unsized OBU aggregation #1002 + * Support G722 audio codec (16 kHz codec with 8 kHz RTP clock per RFC 3551) #992 + * Support RFC 3389 Comfort Noise payloads #1013 + * Apply reliability parameters to a DCEP-receiving in-band channel #1004 + * Normalize `Rtc` shutdown and drain RTCP, SCTP, and DTLS close output consistently #1000 + * Fix an infinite loop when DTLS timeout handling fails #1017 + +# 0.21.0 + + * Upgrade to dimpl 0.7.0 (breaking) + * Handle SctpAssociation as Event::Closed #986 + * Reject SDP payload types > 127 instead of panicking when claiming PTs #983 + * Add RTCP PSFB FMT=15 (AppSpecificFeedback) send/receive via Direct API #981 + * Add H266/VVC RTP packetizer and depacketizer #971 + * Negotiate SCTP max message size for data channels #852 + * Expose the negotiated DTLS protocol version #979 + * Make MTU size configurable #967 + * Add optional `drv` feature for using public identity types as `drv::Input` #974 #975 + * Send DTLS `close_notify` on `Rtc::close()` #956 + * Fix `Simulcast::add_recv_layer` to push to recv instead of send #968 + * Accept any non-empty SDP session name (`s=`), not just `s=-` #969 + +# 0.20.0 + + * Update dimpl to 0.6.2 and sctp-proto to 0.9.1 + * Fix repeated rx lookup cleanup bug #965 + * Fix permissive vp8 depacketizer for invalid `l=1,t=0` cases (breaking) #963 + * Add VP8 descriptor parsing and `vp8_patch()` to `RtpWrite` #963 + * `write_rtp()` now takes a `RtpWrite` builder (breaking) #963 + * Add `abs-capture-time` writer support and fixed receiver-side (breaking) #964 + * Add stereo to opus format params #955 + * Use `Arc<[u8]>` as input/output of media data (breaking) #959 + +# is-0.9.1 + + * Expose ICE candidate pair RTT in CandidatePairStats #962 + +# 0.19.0 + + * Expose H265 some already referenced profile structs #948 + * Update to dimpl 0.6.1 + * Update openssl to 0.10.78 + +# is-0.9.0 + + * IceCandidate expose str0m/is-unique local property #951 + * Implement ICE role conflict resolution #950 + +# 0.18.1 + + * dtls: fix to resends during handshake #943 + * Add `SdpApi::stop_media()` to stop an existing m-line #941 + * Update chat example to use `SdpApi::stop_media()` when clients are removed #941 + * Accept recycled m-line indexes when the previous m-line was stopped #941 + * Always use ActPass when sending an SDP offer (RFC 8842 Section 5.2/5.5) #893 + * Fix frame drops in Firefox due missing RTX SSRC #940 + * Drop stale depacketizer state on stream pause and restore paused timestamp repair #929 + * Reduce bounds checks in STUN encode/decode #931 + +# 0.18.0 + + * Disable server cookie for all dimpl crypto impl #923 + * Implement SNAP and sctp-init #811 + * Implement wincrypto-dimpl feature #920 + * Implement openssl-dimpl feature #919 + * Update to dimpl 0.5.0 #916 + * Refactor out new crate "is" (ice agent) #912 + * Expose CSRC property in RTP Header #910 + * Fix jitter calculation in ReceiverReports #907 + * Preserve nomination on redundant pair replacement #908 + +# 0.17.0 + + * Update to dimpl 0.4.3 #902 + * Update ~dimpl to 0.4.2~ (see 902) and sctp-proto to 0.8.1 #892 + * Update ~dimpl to 0.3.0~ (see 902) #874 + * MSRV 1.85, Edition 2024 and bump deps #891 + * Reduce allocations in AV1 packetizer/depacketizer #889 + * Workaround DCEP panics due to reuse race condition #883 + * sprop-max-don-diff SDP parameter #885 + * Reduce number of RTP header extensions sent on every packet #880 + * Add RTP-level keyframe detection for VP8, VP9, AV1, H264 and H265 #869 + * VP9 switch packetizer to non-flexible mode (F=0) #868 + * Fix H265 SDP negotiation level handling #872 + * Avoid sending VLA on every packet #866 + * Add `abs-capture-time` RTP header extension #864 + * Adjust `IceAgent::ice_timeout` to return timeout of successful pair #875 + * Fix infinite loop on in-band data channel stream conflict #873 + * Add `Hash`, `PartialOrd` and `Ord` traits to `TransId` #905 + * Fix m-line ordering when using sdp `merge()` #887 + +# 0.16.2 + + * Fix hot timeout loop with null pacer #863 + * H265 to use is_irap() for keyframe detection #859 + * Fix probe estimator panic when probes accumulate #857 + * Improve H265 test coverage and organization #853 + * Add `DirectApi::invalidate_candidate()` #858 + +# 0.16.1 + + * Fix two AV1 packetizer panics #854 + * Fix ICE disconnections with Chrome v144+ when not using ICE Lite #846 + * Downgrade noisy BWE logs #847 + * Fix H265 PACI depacketizer panic on short payload #849 + +# 0.16.0 + + * Constant stream timeout check #845 + * H265 Packetizer/Depacketizer #822 + * Cap amount SCTP (data channels) buffer #831 + * Fix incorrectly purged ice-lite pairs #832 + * Provide start Instant in constructor (breaking) #843 + * Fix bug in dimpl resend logic (dimple 0.2.4) #842 + * Fix bug in dimpl HelloVerifyRequests (dimpl 0.2.3) #841 + * Stable RNG for tests with dimpl (dimpl 0.2.3) #841 + * BWE remove current_bitrate param (breaking) #838 + * BWE V2 (Probe clusters) #823 + +# 0.15.0 + + * MediaData::is_keyframe() helper #805 + * Handle tcptype ICE candidates #797 + * New API for building ICE Candidates (breaking) #797 + * AV1 Packetizer/Depacketizer #819 + * Surface Sender Info as Event (breaking) #820 + * Improve performance on apple crypto #818 #821 + * Crypto compliance tests #817 + * Fix HMAC P_hash implementation for Apple Crypto #824 + * Update Apple Crypto crate to use CryptoKit #813 + +# 0.14.3 + + * Bump dimpl 0.2.1 for SChannel compat #801 + * Reduce allocating in `str0m-aws-lc-rs` encrypt/decrypt #804 + +# 0.14.2 + + * Fix order of DTLS pending packet poll #796 + * Defer panic on missing crypto provider until Rtc init #792 + +# 0.14.1 + + * Configure sctp to not give up INIT/data retransmissions #791 + * update sctp-proto to 0.6.0 #791 + * wincrypto: queue data until handshake completes #787 + * openssl: queue data until handshake completes #787 + +# 0.14.0 + + * RTT as Duration #756 + * Revise Simulcast layer API #784 + * Fix max-bundle bug introduced in 0.13 #786 + * Reduce reallocation when depacketizing #782 + * Network emulation for tests #774 + * Handle SSRC 0 BWE probe #769 + * Apple Crypto backend #763 + * Move all crypto backends out of main crate #768 + * h264 profile/level matching comply with RFC 6184 #758 + * Constant time HMAC equality #753 + +# 0.13.0 + + * Simulcast send attributes RFC 8851 #780 + * Handle rejected m-lines (set port 0 etc) #772 + * Expose Opus packetizer #770 + * Handle incoming SSRC 0 BWE probe #769 + * New CryptoProvider API #760 + * Update H.264 profile and level matching logic #758 + * Use constant time equality in some key places #753 + * Fix off-by-one nack reports #748 + +# 0.12.0 + + * Bump str0m-wincrypto to 0.2.1 + * str0m-wincrypto crate: fix double-free #751 + * Represent NTP as SystemTime in SenderReports #735 + * Fix bug in PT remapping to unlocked PTs #734 + * Fix PT conflict when using Firefox #730 + * Rename Sample level API -> Frame level API #732 + * Add new non-openssl DTLS under `dimpl` feature #708 + * Expose packetizers under `unversioned` feature #718 + * Fix bug when TWCC sequence starts above 32768 #717 + +# 0.11.1 + + * stun: stricter string parsing #712 + * Tweak DirectAPI fingerprint() calls (breaking) #711 + * Restore disabling RtcConfig.fingerprint_verification #711 + * Fix panic width incorrect STUN packet #709 + +# 0.11.0 + + * Bump str0m-wincrypto to 0.2.0 + * Remove str0m/wincrypto use of thiserror #704 #706 + * Channel::config() to get data channel config #702 + * fix `Vp9Depacketizer` failing on high scalability modes #701 + * srtp: support (and prefer) AES_256_GCM #697 + * bwe: fix panic on history monotonicity assert in `TrendlineEstimator` #700 + * ice: fix panic on STUN Binding Indication #690 + +# 0.10.0 + + * Channel::buffered_amount/set_buffered_amount #688 + * Support PCMu/PCMa #685 + * Make STUN timers configurable on RtcConfig #674 + * ice: stricter srflx candidate checks #672 + * ice: Dont report Disconnected when no local candidates #670 + * ice: expose remote ICE credentials #675 + * ice: transition from `Disconnected -> Checking` on new candidates #676 + * Make ICE agent local preference pluggable #668 + * Simplified serde of SDP #664 + * SR/RR stats in Ingress/EgressStats #661 #662 + +# 0.9.0 + + * Wincrypto fail on drop #660 + * All logging on DEBUG and lower #658 + * Redact PII (ip addresses) in logs #656 + * Document behavior of MediaAdded #655 + * Fix growing feedback bug before ICE #654 + * Fix vp8 depacketize for small packets #653 + * Bust cache on config change #649 + * RTT in PeerStats #648 + * Fix off-by-one SDES size calculation #647 + * Increase ufrag size to 16 #646 + * `add_local_candidate` return candidate on success #650 + +# 0.8.0 + + * Add start of talkspurt for sample writer #645 + * Avoid overwriting acked SCTP association #643 + * Fix bug misparsing SDES #641 + * Prefer relayed candidates with matching IP versions #640 + * Add stats for the selected candidate pair #638 + * Optional integrity on STUN messages #632 + * Support for most of TURN's messages in STUN #631 + * Reset cached nack flag #629 + * Only reset deplayloader if SSRC did change #628 + * SSRC changes reset StreamRx state #627 + * Direct API reset_stream_tx() call #626 + * Handle very long media pauses #625 + +# 0.7.0 + + * Spoof raddr in srflx relay candidate #621 + * Support AV1-specific media format parameters #619 + * Support negative time deltas in BWE #615 + * Remove restriction on EC to use on ECDH exchange #616 + * Don't retain Receiver Report loss info when stats are disabled #614 + * Fix RIDs bug on updating non-simulcast media #612 + * Support writing VLA #603 + * Ensure BWE calculation exactly follows libWebRTC #608 + * Add EC-DSA custom certificate common names #607 + * Fix multiple bugs in TWCC #605 #606 #601 + * Support sending simulcast (breaking) #603 + * Wincrypto support #589 + +# 0.6.3 + + * Add warning log when exceeding max number of pairs #587 + * Add fuzz to Nightly CI #585 + * Add new loss based BWE controlled #579 #582 #583 + * Add new `vendored` feature flag affecting openssl #580 + * Make use of sha1 crate an optional feature #577 + * Dedupe remote ICE candidates #576 #578 + * Correctly handle per m-line Absolute Send Time #575 + * Correctly handle per m-line TWCC #573 + * Configure RTX ratio cap via `StreamTx::set_rtx_cache` #570 + * Match remote candidate of stun request by priority #569 + * Improve timeouts during DTLS handshake #565 + * Do not decrypt already received packets #554 + * Test for SRTP replay attack #555 + * refactor(ice): always use latest timing config #568 + * Fix RTX stops working after packet loss spike #566 + * Add Sub trait impl (back) to MediaTime #560 + * Make start of talkspurt information available for frame api #559 + * Do not disconnect whilst we still check new candidates #489 + * Ensure lexical ordering of SDP-formatted candidates follows priority #557 + * Limit TWCC iteration with packet status count #606 + * Dedupe acked packets from `TwccSendRegister::apply_report()` #601, #605 + * Align BWE ArrivalGroup calculation with libwebrtc implementation #608, #615 + * Support AV1-specific media format parameters #619 + +# 0.6.2 + + * Fix edge case breaking DTLS #531 + * Bump sctp-proto to be compatible with libdatachannel #558 + * Ensure not risk of ROC on initial sequence number #553 + * API for getting stream_id/track_id from a=msid #550 + * Use a=extmap-allow-mixed SDP attribute #551 + * Change the unix_time function to return libc::time_t (32-bit compat) #533 + * Fix bug using unreliable channels by default #548 + * New add_channel_with_config() for configured data channels #548 + +# 0.6.1 + * Force openssl to be >=0.10.66 #545 + * Fix bug when replacing redundant ice candidates #544 + * Add playout_delay builder function #543 + +# 0.6.0 + + * Doc updates + * IceAgent make timeout values configurable #537 + * Log more details when failing to decrypt SRTP #536 + * Remember max SeqNo per SSRC for reuse ROC #535 + * Only update NACK/TWCC registers after succesful SRTP decrypt #528 + * Fix bug when changing StreamRx SSRC #522 + * Simplify StreamRx lookup state cache #522 + * Fix bug in TWCC time delta #524 + * Make MediaTime nominator unsigned (breaking) #521 + * Provide reason for timeout #520 + * Reject ice-lite - ice-lite scenario #519 + * Fix bug in ice agent roles for ice-lite #519 + * DLTS cert serial number as random instead of sequential #518 + * IceAgent make timing advance (TA) configurable #515 + * Use sha1 crate instead of the deprecated sha-1 #512 + * Bump sctp-proto to 0.2.2 #511 + * Adjust logging levels to be less noisy #510 + * Fix crash when using VLA (or other) optional RTP exts with SDP #509 + * Re-add manually invalidated IceAgent candidates #508 + * New API to reset BWE state #506 + * Change parameter in BWE algo to match libwebrtc #506 + +# 0.5.1 + + * Expose STUN packet split_username() fn #505 + * IceAgent fix to not invalidate unrelated remote candidates #504 + * Expose ice credentials as configuration option #502 + * Bug fix in lazy NACK handling #501 + * Net structs Transmit/Receive serde Serializable #500 + * Payload matching for VP9 #499 + * IceAgent allow invalidating remote candidates #498 + * Avoid panic on too early DTLS traffic #495 + * `IceAgent::handle_packet` return false if packet not accepted #493 + * Don't panic on STUN requests for unknown NICs #493 + * Improve IceAgent `accepts_message` to avoid panic in some situations #488 + +# 0.5.0 + * Opus DTX support #492 + * RtcConfig does not generate the ICE creds #491 + * Accept all good remote address candidates, not just the nominated #487 + * Improve performance by only calling `init_time` once #479 + * Fix SCTP channel ID allocation bug when reusing channels #470 + * Fix bug causing nomination of ICE candidate pair that isn't better #463 + * Lower STUN_TIMEOUT for quicker connection checking #462 + * Fix bug making SCTP fail under packet loss #482 + * Add `base` parameter to `Candidate::server_reflexive` (breaking) #455 + * Refactor all OpenSSL (and other crypto code) to mod crypto #449 + * Fix bugs in RTX PT selection for BWE padding #454 + * Don't enable NACK timers unless there are stream to nack + * Fix bug in BWE trendline estimator + * Fix (unlikely) nack overflow error + * Speed up twcc register updates using max_seq() + * Parse `StunMessage` without allocations (and huge STUN parsing cleanup) + * Introduce top-level `ice` module having `ice::IceAgent` for standalone usage + * Remove `StunError::Other` because it was unused + * Optional parser for VLA (Video Layers Allocation) RTP header extension + * Chat example send PLI on RTC sequence interruption + * VP9 contiguity checks in depacketizer + * Improved VP9 support with parsing layer metadata + * Fix race in chat example on client disconnect + * MediaTime improve safety with Frequency newtype (breaking) + * Header extension abs_send_time is now an Instant + * Handle more optional a=candidate parameters + * Support REMB (receiver estimated maximum bitrate) feedback packets (breaking) + +# 0.4.1 + * Generated DTLS certificates set issuer/subject for compat with OBS/libdatachannel + +# 0.4.0 + * Allow SDP RID with `~` prefix (to indicate paused state) + * Fix problem with using RTX for audio + * Make IceCandidate serializable to SDP form (to/from_sdp_string) + * UserExtensionValues set_arc/get_arc to avoid cloning + * Provide correct Rid for repaired RTP headers + * Support 2-byte header extensions + * Remove a statistics performance bottleneck + * New ICE Candidate types `server_reflexive` and `relayed` + +# 0.3.0 + * Fix bad bug causing SCTP packets to not send + * Improve performance by reducing Event enum size + * SdpPendingOffer mergable into new OFFER + * Improved VP8 temporal layer handling + * Fix bug in discovering RTX channels due to NACK not being enabled + * Improve NACK sending with multiple fixes to receive register + * Fix bug where RTX channel would be allocated for audio m-lines + * Delay creation of DtlsCert to avoid unnecessary start-up time + * Writer take Into> to make it possible to avoid extra allocation (breaking) + * Refactor internal time handling (unix epoch translation) + * Fix bug in signaling media discontinuity + * User RTP header extensions + +# 0.2.0 + * Possible to disable DTLS fingerprint verification + * Manually set local ice credentials + * enable_raw_packets for debugging RTP/RTCP + * ICE restart + * SRTP: Implement AEAD_AES_128_GCM and use it by default + * Better FMTP matching of VP8 and H264 + * Fix incorrect handling of header extensions + * Fix incorrect handling of PT-codec assignment + * Dynamic SSRC via MID-only RTP headers + * Fix various undeflow and padding bugs + * VP8: parse out metata + * Clean separation of RtxCache from PacketizingBuffer. + * Major refactor of Media/Stream handling (we call it "kaboom") + * RTP Mode (directly using RTP packets) + * Bandwidth Estimation (BWE) + * Direct API for SDP-free control + * SDP Api to formalize SDP handling + +# 0.1.0 + * First published version diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml b/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml new file mode 100644 index 000000000..b715803f9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml @@ -0,0 +1,187 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85.0" +name = "str0m" +version = "0.23.0" +authors = [ + "Martin Algesten ", + "Hugo Tunius ", + "Davide Bertola ", +] +build = false +exclude = [ + "/cargo_deny.sh", + "/deny.toml", + "/run-fuzz.sh", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "WebRTC library in Sans-IO style" +readme = "README.md" +keywords = [ + "webrtc", + "streaming", + "video", + "audio", + "media", +] +categories = [ + "web-programming", + "multimedia", + "network-programming", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/algesten/str0m" + +[features] +_internal_dont_use_log_stats = [] +_internal_test_exports = [ + "str0m-proto/_internal_test_exports", + "is/_internal_test_exports", +] +apple-crypto = ["dep:str0m-apple-crypto"] +aws-lc-rs = ["dep:str0m-aws-lc-rs"] +default = [ + "aws-lc-rs", + "examples", +] +drv = ["dep:drv"] +examples = ["rouille/rustls"] +openssl = ["dep:str0m-openssl"] +openssl-dimpl = [ + "dep:str0m-openssl", + "str0m-openssl/dimpl", +] +pii = [ + "str0m-proto/pii", + "is/pii", +] +rust-crypto = ["dep:str0m-rust-crypto"] +unversioned = [] +vendored = ["str0m-openssl?/vendored"] +wincrypto = ["dep:str0m-wincrypto"] +wincrypto-dimpl = [ + "dep:str0m-wincrypto", + "str0m-wincrypto/dimpl", +] + +[lib] +name = "str0m" +path = "src/lib.rs" + +[dependencies.arrayvec] +version = "0.7.6" + +[dependencies.base64ct] +version = "1" + +[dependencies.combine] +version = "4.6.6" + +[dependencies.dimpl] +version = "0.7.1" +default-features = false + +[dependencies.drv] +version = "0.4.3" +optional = true + +[dependencies.fastrand] +version = "2.0.1" + +[dependencies.is] +version = "0.11.0" +default-features = false + +[dependencies.sctp-proto] +version = "0.10.3" + +[dependencies.serde] +version = "1.0.152" +features = ["derive"] + +[dependencies.str0m-aws-lc-rs] +version = "0.6.0" +optional = true + +[dependencies.str0m-openssl] +version = "0.6.0" +optional = true + +[dependencies.str0m-proto] +version = "0.7.0" +features = ["dtls"] +default-features = false + +[dependencies.str0m-rust-crypto] +version = "0.6.0" +optional = true + +[dependencies.subtle] +version = "2.0.0" + +[dependencies.time] +version = "0.3" + +[dependencies.tracing] +version = "0.1.37" + +[dev-dependencies.netem] +version = "0.4.0" +package = "str0m-netem" + +[dev-dependencies.pcap-file] +version = "2.0.0" + +[dev-dependencies.rand] +version = "0.9.0" + +[dev-dependencies.regex] +version = "1.11.1" + +[dev-dependencies.rouille] +version = "3.6.2" + +[dev-dependencies.serde_json] +version = "1.0" + +[dev-dependencies.str0m-rust-crypto] +version = "0.6.0" + +[dev-dependencies.systemstat] +version = "0.2.2" + +[dev-dependencies.tempfile] +version = "3" + +[dev-dependencies.tracing-subscriber] +version = "0.3.16" +features = [ + "env-filter", + "std", +] + +[dev-dependencies.url] +version = "2" + +[target.'cfg(target_vendor = "apple")'.dependencies.str0m-apple-crypto] +version = "0.6.0" +optional = true + +[target."cfg(windows)".dependencies.str0m-wincrypto] +version = "0.8.0" +optional = true diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml.orig b/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml.orig new file mode 100644 index 000000000..d20ea8f4e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/Cargo.toml.orig @@ -0,0 +1,184 @@ +[package] +name = "str0m" +version = "0.23.0" +authors = ["Martin Algesten ", "Hugo Tunius ", "Davide Bertola "] +description = "WebRTC library in Sans-IO style" +license = "MIT OR Apache-2.0" +repository = "https://github.com/algesten/str0m" +readme = "README.md" +keywords = ["webrtc", "streaming", "video", "audio", "media"] +categories = ["web-programming", "multimedia", "network-programming"] +edition = "2024" +exclude = ["/cargo_deny.sh", "/deny.toml", "/run-fuzz.sh"] + +# MSRV +rust-version = "1.85.0" + +# Notes on OpenSSL +# - Vendored (static link source build) ensures version consistency and easier +# to build but increases size. +# - Non-vendored (dynamic link) uses system libraries, reducing size but may +# cause compatibility issues. +# Choose based on security and footprint needs. +# For large deployments sharing OS images, OS updates can be easier for +# security patching than updating statically linked software. + +[workspace] +members = [ + "crates/proto", + "crates/is", + "crates/netem", + "crates/_str0m_test", + "crypto/aws-lc-rs", + "crypto/rust-crypto", + "crypto/openssl", + "crypto/apple-crypto", + "crypto/wincrypto" +] +exclude = ["crates/fuzz"] + +[workspace.dependencies] +# Internal crates +str0m = { version = "0.23.0", path = ".", default-features = false } +str0m-proto = { version = "0.7.0", path = "crates/proto", default-features = false } +is = { version = "0.11.0", path = "crates/is", default-features = false } +str0m-netem = { version = "0.4.0", path = "crates/netem" } +str0m-aws-lc-rs = { version = "0.6.0", path = "crypto/aws-lc-rs" } +str0m-rust-crypto = { version = "0.6.0", path = "crypto/rust-crypto" } +str0m-openssl = { version = "0.6.0", path = "crypto/openssl" } +str0m-apple-crypto = { version = "0.6.0", path = "crypto/apple-crypto" } +str0m-wincrypto = { version = "0.8.0", path = "crypto/wincrypto" } +_str0m_test = { path = "crates/_str0m_test" } + +# Core dependencies +tracing = "0.1.37" +fastrand = "2.0.1" +sctp-proto = "0.10.3" +combine = "4.6.6" +subtle = "2.0.0" +arrayvec = "0.7.6" +serde = { version = "1.0.152", features = ["derive"] } +dimpl = { version = "0.7.1", default-features = false } +drv = { version = "0.4.3" } +time = "0.3" +base64ct = "1" + +# Proto dependencies +openssl = "0.10.78" + +# IS dependencies +crc = "3" +hmac = "0.12.1" +sha1 = "0.10" + +# Crypto backend dependencies +aws-lc-rs = { version = "1", default-features = false, features = ["aws-lc-sys", "prebuilt-nasm"] } +aes-gcm = "0.10" +aes = "0.8" +ctr = "0.9" +p256 = { version = "0.13", features = ["ecdh"] } +sha2 = "0.10" +openssl-sys = "0.9.80" +libc = "0.2" +der = "0.7" +pkcs8 = { version = "0.10", features = ["pem"] } +sec1 = "0.7" +spki = "0.7" +x509-cert = { version = "0.2", default-features = false } + +# Dev/test dependencies +rouille = "3.6.2" +tempfile = "3" +url = "2" +serde_json = "1.0" +tracing-subscriber = { version = "0.3.16", features = ["env-filter", "std"] } +systemstat = "0.2.2" +pcap-file = "2.0.0" +regex = "1.11.1" +rand = "0.9.0" + +[features] +default = ["aws-lc-rs", "examples"] +aws-lc-rs = ["dep:str0m-aws-lc-rs"] +rust-crypto = ["dep:str0m-rust-crypto"] +openssl = ["dep:str0m-openssl"] +openssl-dimpl = ["dep:str0m-openssl", "str0m-openssl/dimpl"] +wincrypto = ["dep:str0m-wincrypto"] +wincrypto-dimpl = ["dep:str0m-wincrypto", "str0m-wincrypto/dimpl"] +apple-crypto = ["dep:str0m-apple-crypto"] +vendored = ["str0m-openssl?/vendored"] +unversioned = [] + +# Redacts personally identifiable information (PII) from logs debug and above +pii = ["str0m-proto/pii", "is/pii"] + +# The examples require TLS for the web server via rouille. +# We use rustls instead of ssl to avoid OpenSSL dependency issues on Windows. +examples = ["rouille/rustls"] + +_internal_dont_use_log_stats = [] +_internal_test_exports = ["str0m-proto/_internal_test_exports", "is/_internal_test_exports"] + +# Derives `drv::Input` (drv's memoization cache-key plumbing) on the public +# identity types — Mid, Rid, Ssrc, Pt, SessionId, SeqNo, Direction, MediaKind, +# Frequency, Codec, CodecSpec, FormatParams, VideoOrientation, and the H265 +# profile types — so downstream `#[drv::memo]` queries can take them by value. +# Off by default. +drv = ["dep:drv"] + +[dependencies] +tracing.workspace = true +fastrand.workspace = true +sctp-proto.workspace = true +combine.workspace = true +subtle.workspace = true +arrayvec.workspace = true +serde.workspace = true +time.workspace = true +base64ct.workspace = true + +# DTLS made for str0m (required dependency) +dimpl.workspace = true + +str0m-proto = { workspace = true, features = ["dtls"] } +is.workspace = true + +drv = { workspace = true, optional = true } + +# Crypto providers +str0m-aws-lc-rs = { workspace = true, optional = true } +str0m-rust-crypto = { workspace = true, optional = true } +str0m-openssl = { workspace = true, optional = true } + +[target.'cfg(target_vendor = "apple")'.dependencies] +str0m-apple-crypto = { workspace = true, optional = true } + +[target.'cfg(windows)'.dependencies] +# Windows Crypto (CNG + SChannel) +str0m-wincrypto = { workspace = true, optional = true } + +[dev-dependencies] +rouille.workspace = true +str0m-rust-crypto.workspace = true +tempfile.workspace = true +url.workspace = true +serde_json.workspace = true +tracing-subscriber.workspace = true +systemstat.workspace = true +pcap-file.workspace = true +regex.workspace = true +rand.workspace = true + +# Package rename not supported with workspace inheritance +netem = { package = "str0m-netem", version = "0.4.0", path = "crates/netem" } + +# dummy package that enables "_internal_test_exports" +_str0m_test = { path = "crates/_str0m_test" } + +[[example]] +name = "chat" +required-features = ["examples"] + +[[example]] +name = "http-post" +required-features = ["examples"] diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/LICENSE-MIT.txt b/external/opennow/native/opennow-streamer/vendor/str0m/LICENSE-MIT.txt new file mode 100644 index 000000000..235abe2ca --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/LICENSE-MIT.txt @@ -0,0 +1,7 @@ +Copyright 2022 Martin Algesten + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/OPENNOW-PATCH.md b/external/opennow/native/opennow-streamer/vendor/str0m/OPENNOW-PATCH.md new file mode 100644 index 000000000..9bad5234d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/OPENNOW-PATCH.md @@ -0,0 +1,52 @@ +# OpenNOW str0m patch + +This directory is the published `str0m` 0.23.0 crate with one upstream fix backported +and an additive direct-API payload registration method. +It replaces the crates.io package through `[patch.crates-io]` in +`native/opennow-streamer/Cargo.toml`. + +## Provenance + +- Source: `str0m-0.23.0.crate` from crates.io, SHA-256 + `840d11fdd44459d71bc7b3dd6796f55e9ac86c8e3fa8678098a8642f5701c5ff` (the checksum the + workspace lockfile recorded before this patch). Upstream commit for that release: + `67820c52ff58df5350659063129e94d7d87225e4` (see `.cargo_vcs_info.json`). +- License: `MIT OR Apache-2.0`, unchanged. `LICENSE-MIT.txt` is the crate's own file. +- Removed from the published crate because they are not built as a dependency: `tests/`, + `examples/`, `docs/`, `logo/`, editor/CI configuration, the crate's own `Cargo.lock`, and the + `[[test]]`/`[[example]]` targets that referenced them in `Cargo.toml`. `Cargo.toml.orig` is + the unmodified upstream manifest. + +## Change + +`src/packet/buffer_rx.rs` backports the depacketizer cache invalidation and its +`late_packet_in_front_of_waiting_frame` unit test from upstream commit +`67c6b4e7ccb25a989b65c835b575d6ecf881f84a` (first released in str0m 0.24.0). + +In 0.23.0, a frame waiting behind a sequence gap is depacketized ahead and cached under its +queue-index range. A late or retransmitted packet inserted in front of it shifts the queue +without invalidating that cache, so the repaired frame is emitted with the following frame's +payload and timestamp and is reported as contiguous. No other upstream change is included. + +`src/change/direct.rs` adds `DirectApi::register_payload_type` using the existing +codec configuration insertion path. OpenNOW reserves the RTC identity before SDP +negotiation, then registers negotiated microphone Opus and RED payload types before +starting DTLS. Unknown payload types retain their existing drop behavior. + +## Verification + +- `late_packet_in_front_of_waiting_frame` fails on unpatched 0.23.0 + (`Time not matching 4 != 3`) and passes with the patch. +- `opennow-streamer-transport`'s + `repairable_delta_loss_waits_for_retransmission_and_stays_contiguous` holds back one + original SRTP datagram, re-injects it, and requires all 20 frames with their sender + timestamps and payloads. +- `registered_microphone_payload_type_survives_direct_rtp_poll` checks dynamic + registration through a local ICE/DTLS exchange and verifies that an unregistered + payload type is still dropped. + +## Removal + +Delete this directory and its `[patch.crates-io]` entry when the workspace moves to +str0m 0.24.0 or later, which contains the cache fix, and an equivalent direct-API +payload registration API is available. Keep the transport regression tests. diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/README.md b/external/opennow/native/opennow-streamer/vendor/str0m/README.md new file mode 100644 index 000000000..6c931f774 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/README.md @@ -0,0 +1,656 @@ +# str0m + +str0m logo + +A Sans I/O WebRTC implementation in Rust. + +This is a [Sans I/O][sansio] implementation meaning the `Rtc` instance itself is not doing any network +talking. Furthermore it has no internal threads or async tasks. All operations are happening from the +calls of the public API. + +This is deliberately not a standard `RTCPeerConnection` API since that isn't a great fit for Rust. +See more details in below section. + +## Join us + +We are discussing str0m things on Discord. Join us using this [invitation link][discord]. + +silly clip showing video playing + +## Usage + +The [`chat`][x-chat] example shows how to connect multiple browsers +together and act as an SFU (Selective Forwarding Unit). The example +multiplexes all traffic over one server UDP socket and uses two threads +(one for the web server, and one for the SFU loop). + +### TLS + +For the browser to do WebRTC, all traffic must be under TLS. The +project ships with a self-signed certificate that is used for the +examples. The certificate is for hostname `str0m.test` since TLD .test +should never resolve to a real DNS name. + +``` +cargo run --example chat +``` + +The log should prompt you to connect a browser to https://10.0.0.103:3000 – this will +most likely cause a security warning that you must get the browser to accept. + +The [`http-post`][x-post] example roughly illustrates how to receive +media data from a browser client. The example is single threaded and +is a bit simpler than the chat. It is a good starting point to understand the API. + +``` +cargo run --example http-post +``` + +#### Real example + +To see how str0m is used in a real project, check out [BitWHIP][bitwhip] – +a CLI WebRTC Agent written in Rust. + +### Passive + +For passive connections, i.e. where the media and initial OFFER is +made by a remote peer, we need these steps to open the connection. + +```rust +// Instantiate a new Rtc instance. +let mut rtc = Rtc::new(Instant::now()); + +// Add some ICE candidate such as a locally bound UDP port. +let addr = "1.2.3.4:5000".parse().unwrap(); +let candidate = Candidate::host(addr, "udp").unwrap(); +rtc.add_local_candidate(candidate); + +// Accept an incoming offer from the remote peer +// and get the corresponding answer. +let offer = todo!(); +let answer = rtc.sdp_api().accept_offer(offer).unwrap(); + +// Forward the answer to the remote peer. + +// Go to _run loop_ +``` + +### Active + +Active connections means we are making the inital OFFER and waiting for a +remote ANSWER to start the connection. + +```rust +// Instantiate a new Rtc instance. +let mut rtc = Rtc::new(Instant::now()); + +// Add some ICE candidate such as a locally bound UDP port. +let addr = "1.2.3.4:5000".parse().unwrap(); +let candidate = Candidate::host(addr, "udp").unwrap(); +rtc.add_local_candidate(candidate); + +// Create a `SdpApi`. The change lets us make multiple changes +// before sending the offer. +let mut change = rtc.sdp_api(); + +// Do some change. A valid OFFER needs at least one "m-line" (media). +let mid = change.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + +// Get the offer. +let (offer, pending) = change.apply().unwrap(); + +// Forward the offer to the remote peer and await the answer. +// How to transfer this is outside the scope for this library. +let answer = todo!(); + +// Apply answer. +rtc.sdp_api().accept_answer(pending, answer).unwrap(); + +// Go to _run loop_ +``` + +### Run loop + +#### The single-mutation invariant + +str0m's API has one strict contract that the run loop is built around: + +> **Every mutation of an `Rtc` instance must be followed by a complete +> drain of `poll_output` until it returns `Output::Timeout`, before the +> next mutation on the same `Rtc`.** + +A "mutation" is anything that takes `&mut Rtc` (directly or through a +handle obtained from it). The common ones are: + +- `Rtc::handle_input` — feeding a network packet or a timeout +- `Writer::write` / `Writer::request_keyframe` — sending media +- `Channel::write` — sending data channel data +- `SdpApi::apply` / `DirectApi::*` — negotiation +- `Rtc::add_local_candidate` / `Rtc::add_remote_candidate` + +Always: **mutate → drain to `Timeout` → mutate → drain to `Timeout` → …** + +Doing two mutations back-to-back without draining in between, or +waiting on I/O while the engine still has output queued, leaves the +engine in an inconsistent state and produces wrong behavior. Mutations +issued from inside the drain loop (e.g. calling `Writer::write` in +response to an `Output::Event`) are fine — the drain loop naturally +continues calling `poll_output` afterward and so the invariant holds. + +#### Canonical shape + +Driving an `Rtc` forward follows the same six-step shape, regardless +of sync or async: + +1. **Wait** for one of: the next timeout firing, an incoming network + packet, or the application wanting to perform a mutation (e.g. + write media). +2. **Perform that ONE mutation** — feed `Input` to `handle_input`, or + call into a writer / channel / SDP API. +3. **Poll** `Rtc::poll_output`. +4. **Handle** the output: `Output::Transmit` is sent on the socket, + `Output::Event` is dispatched to the application, `Output::Timeout` + records the next deadline. +5. **Goto 3** until `poll_output` returns `Output::Timeout`. Only then + is the engine fully drained. +6. The returned timeout is what we wait on next — **goto 1**. + +```rust +// A UdpSocket we obtained _somehow_. +let socket: UdpSocket = todo!(); + +// Buffer for reading incoming UDP packets. +let mut buf = vec![0; 2000]; + +loop { + // === Steps 3-5: drain poll_output until we get the next timeout. === + let timeout = loop { + match rtc.poll_output().unwrap() { + // Step 5: a Timeout exits the drain loop. + Output::Timeout(t) => break t, + + // Step 4: transmit on the socket and keep draining. + // The destination IP comes from the ICE agent and may + // change during the session. + Output::Transmit(t) => { + socket.send_to(&t.contents, t.destination).unwrap(); + } + + // Step 4: hand the event to the application and keep draining. + // Events are mainly incoming media data from the remote peer, + // but also data channel data and statistics. + Output::Event(e) => { + if e == Event::IceConnectionStateChange(IceConnectionState::Disconnected) { + return; + } + // TODO: handle other events here, such as incoming media data. + } + } + }; + + // === Step 1: wait for ONE of: the timeout firing, an incoming + // packet, or application-side data. The example below uses a + // blocking UDP socket with a read timeout. With async you would + // `select!` over multiple futures; with application-side data you + // would also include a channel. + // + // set_read_timeout(Some(ZERO)) is not allowed, so clamp to >= 1ms. + let duration = (timeout - Instant::now()).max(Duration::from_millis(1)); + socket.set_read_timeout(Some(duration)).unwrap(); + buf.resize(2000, 0); + + // === Step 2: take ONE event and feed it as Input === + let input = match socket.recv_from(&mut buf) { + Ok((n, source)) => { + buf.truncate(n); + Input::Receive( + Instant::now(), + Receive { + proto: Protocol::Udp, + source, + destination: socket.local_addr().unwrap(), + contents: buf.as_slice().try_into().unwrap(), + }, + ) + } + + // The socket read timed out — feed Input::Timeout to advance + // the engine to the deadline. WouldBlock is the unix error, + // TimedOut is the windows error. + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => { + Input::Timeout(Instant::now()) + } + + Err(e) => { + eprintln!("Error: {:?}", e); + return; + } + }; + + rtc.handle_input(input).unwrap(); + + // === Step 6: back to the top of the outer loop (goto step 3). === +} +``` + +### Sending media data + +When creating the media, we can decide which codecs to support, and they +are negotiated with the remote side. Each codec corresponds to a +"payload type" (PT). To send media data we need to figure out which PT +to use when sending. + +```rust +// Obtain mid from Event::MediaAdded +let mid: Mid = todo!(); + +// Create a media writer for the mid. +let writer = rtc.writer(mid).unwrap(); + +// Get the payload type (pt) for the wanted codec. +let pt = writer.payload_params().nth(0).unwrap().pt(); + +// Write the data +let wallclock = todo!(); // Absolute time of the data +let media_time = todo!(); // Media time, in RTP time +let data: &[u8] = todo!(); // Actual data +writer.write(pt, wallclock, media_time, data).unwrap(); +``` + +`Writer::write` is a mutation, so the +[single-mutation invariant](#the-single-mutation-invariant) applies: +after writing, drain `Rtc::poll_output` to `Output::Timeout` before +the next mutation on this `Rtc`. + +### Media time, wallclock and local time + +str0m has three main concepts of time. "now", media time and wallclock. + +#### Now + +Some calls in str0m, such as `Rtc::handle_input` takes a `now` argument +that is a `std::time::Instant`. These calls "drive the time forward" in +the internal state. This is used for everything like deciding when +to produce various feedback reports (RTCP) to remote peers, to +bandwidth estimation (BWE) and statistics. + +Str0m has _no internal clock_ calls. I.e. str0m never calls +`Instant::now()` itself. All time is external input. That means it's +possible to construct test cases driving an `Rtc` instance faster +than realtime (see the [integration tests][intg]). + +#### Media time + +Each RTP header has a 32 bit number that str0m calls _media time_. +Media time is in some time base that is dependent on the codec, +however all codecs in str0m use 90_000Hz for video and 48_000Hz +for audio. + +For video the `MediaTime` type is `/90_000` str0m extends +the 32 bit number in the RTP header to 64 bit taking into account +"rollover". 64 bit is such a large number the user doesn't need to +think about rollovers. + +#### Wallclock + +With _wallclock_ str0m means the time a sample of media was produced +at an originating source. I.e. if we are talking into a microphone the +wallclock is the NTP time the sound is sampled. + +We can't know the exact wallclock for media from a remote peer since +not every device is synchronized with NTP. Every sender does +periodically produce a Sender Report (SR) that contains the peer's +idea of its wallclock, however this number can be very wrong compared to +"real" NTP time. + +Furthermore, not all remote devices will have a linear idea of +time passing that exactly matches the local time. A minute on the +remote peer might not be exactly one minute locally. + +These timestamps become important when handling simultaneous audio from +multiple peers. + +When writing media we need to provide str0m with an estimated wallclock. +The simplest strategy is to only trust local time and use arrival time +of the incoming UDP packet. Another simple strategy is to lock some +time T at the first UDP packet, and then offset each wallclock using +`MediaTime`, i.e. for video we could have `T + /90_000` + +A production worthy SFU probably needs an even more sophisticated +strategy weighing in all possible time sources to get a good estimate +of the remote wallclock for a packet. + +## Crypto backends + +str0m supports multiple crypto backends via feature flags. The default is `aws-lc-rs`. + +| Feature | Crate | DTLS | Platforms | +|-------------------|-----------------------|----------------------------------|-----------| +| `aws-lc-rs` | `str0m-aws-lc-rs` | dimpl + AWS-LC-RS | All | +| `rust-crypto` | `str0m-rust-crypto` | dimpl + RustCrypto | All | +| `openssl` | `str0m-openssl` | OpenSSL (DTLS 1.2 only) | All | +| `openssl-dimpl` | `str0m-openssl` | dimpl + OpenSSL crypto | All | +| `apple-crypto` | `str0m-apple-crypto` | dimpl + Apple CryptoKit | macOS/iOS | +| `wincrypto` | `str0m-wincrypto` | Windows SChannel (DTLS 1.2 only) | Windows | +| `wincrypto-dimpl` | `str0m-wincrypto` | dimpl + Windows CNG | Windows | + +If multiple backend features are enabled, str0m automatically selects the backend in this +priority order: `aws-lc-rs`, `rust-crypto`, `openssl-dimpl`, `openssl`, `apple-crypto` +(Apple platforms only), `wincrypt-dimpl` (Windows only), `wincrypto` (Windows only). + +If you disable the default features, you MUST explicitly configure an alternative +crypto backend either process-wide or per-instance. + +### Process-wide default + +For applications, the easiest is to set a process-wide default at startup. +Note that you can use any backend crate directly without enabling its feature flag: + +```rust +// Set process default (will panic if called twice) +// No need to enable the "rust-crypto" feature flag +str0m_rust_crypto::default_provider().install_process_default(); +``` + +### Crypto provider per Rtc instance + +```rust +use std::sync::Arc; +use std::time::Instant; +use str0m::Rtc; + +let rtc = Rtc::builder() + .set_crypto_provider(Arc::new(str0m_rust_crypto::default_provider())) + .build(Instant::now()); +``` + +## Project status + +Str0m was originally developed by Martin Algesten of +[Lookback][lookback]. We use str0m for a specific use case: str0m as a +server SFU (as opposed to peer-2-peer). That means we are heavily +testing and developing the parts needed for our use case. Str0m is +intended to be an all-purpose WebRTC library, which means it also +works for peer-2-peer, though that aspect has received less testing. + +Performance is very good, there have been some work the discover and +optimize bottlenecks. Such efforts are of course never ending with +diminishing returns. While there are no glaringly obvious performance +bottlenecks, more work is always welcome – both algorithmically and +allocation/cloning in hot paths etc. + +## Design + +Output from the `Rtc` instance can be grouped into three kinds. + +1. Events (such as receiving media or data channel data). +2. Network output. Data to be sent, typically from a UDP socket. +3. Timeouts. Indicates when the instance next expects a time input. + +Input to the `Rtc` instance is: + +1. User operations (such as sending media or data channel data). +2. Network input. Typically read from a UDP socket. +3. Timeouts. As obtained from the output above. + +The correct use can be seen in the above [Run loop](#run-loop) or in the +examples. + +Sans I/O is a pattern where we turn both network input/output as well +as time passing into external input to the API. This means str0m has +no internal threads, just an enormous state machine that is driven +forward by different kinds of input. + +### Frame or RTP level? + +Str0m defaults to the "frame level" which treats the RTP as an internal detail. The user +will thus mainly interact with: + +1. [`Event::MediaData`][evmed] to receive full frames (audio frames or video frames). +2. [`Writer::write`][writer] to write full frames. +3. [`Writer::request_keyframe`][reqkey] to request keyframes. + +#### Frame level + +All codecs such as h264, vp8, vp9 and opus outputs what we call +"Frames". A frame has a very specific meaning for video, but this +project uses it in a broader sense, where a frame is either a video +or audio time stamped chunk of encoded data that typically represents +a chunk of audio, or _one single frame for video_. + +Frames are not suitable to use directly in UDP (RTP) packets - for +one they are too big. Frames are therefore further chunked up by +codec specific payloaders into RTP packets. + +#### RTP mode + +Str0m also provides an RTP level API. This would be similar to many other +RTP libraries where the RTP packets themselves are the API surface +towards the user (when building an SFU one would often talk about "forwarding +RTP packets", while with str0m we can also "forward frames"). Using +this API requires a deeper knowledge of RTP and WebRTC. + +To enable RTP mode + +```rust +let rtc = Rtc::builder() + // Enable RTP mode for this Rtc instance. + // This disables `MediaEvent` and the `Writer::write` API. + .set_rtp_mode(true) + .build(Instant::now()); +``` + +RTP mode gives us some new API points. + +1. [`Event::RtpPacket`][rtppak] emitted for every incoming RTP packet. Empty packets for bandwidth + estimation are silently discarded. +2. [`StreamTx::write_rtp`][wrtrtp] to write outgoing RTP packets. +3. [`StreamRx::request_keyframe`][reqkey2] to request keyframes from remote. + +### NIC enumeration and TURN (and STUN) + +The [ICE RFC][ice] talks about "gathering ice candidates". This means +inspecting the local network interfaces and potentially binding UDP +sockets on each usable interface. Since str0m is Sans I/O, this part +is outside the scope of what str0m does. How the user figures out +local IP addresses, via config or via looking up local NICs is not +something str0m cares about. + +TURN is a way of obtaining IP addresses that can be used as fallback +in case direct connections fail. We consider TURN similar to +enumerating local network interfaces – it's a way of obtaining +sockets. + +All discovered candidates, be they local (NIC) or remote sockets +(TURN), are added to str0m and str0m will perform the task of ICE +agent, forming "candidate pairs" and figuring out the best connection +while the actual task of sending the network traffic is left to the +user. + +### The importance of `&mut self` + +Rust shines when we can eschew locks and heavily rely `&mut` for data +write access. Since str0m has no internal threads, we never have to +deal with shared data. Furthermore the the internals of the library is +organized such that we don't need multiple references to the same +entities. In str0m there are no `Rc`, `Mutex`, `mpsc`, `Arc`(*), or +other locks. + +This means all input to the lib can be modelled as +`handle_something(&mut self, something)`. + +(*) Ok. There is one `Arc` if you use Windows where we also require openssl. + +### Not a standard WebRTC "Peer Connection" API + +The library deliberately steps away from the "standard" WebRTC API as +seen in JavaScript and/or [webrtc-rs][webrtc-rs] (or [Pion][pion] in Go). +There are few reasons for this. + +First, in the standard API, events are callbacks, which are not a +great fit for Rust. Callbacks require some kind of reference +(ownership?) over the entity the callback is being dispatched +upon. I.e. if in Rust we want `pc.addEventListener(x)`, `x` needs +to be wholly owned by `pc`, or have some shared reference (like +`Arc`). Shared references means shared data, and to get mutable shared +data, we will need some kind of lock. i.e. `Arc>` +or similar. + +As an alternative we could turn all events into `mpsc` channels, but +listening to multiple channels is awkward without async. + +Second, in the standard API, entities like `RTCPeerConnection` and +`RTCRtpTransceiver`, are easily clonable and/or long lived +references. I.e. `pc.getTranscievers()` returns objects that can be +retained and owned by the caller. This pattern is fine for garbage +collected or reference counted languages, but not great with Rust. + +### Panics, Errors and unwraps + +Str0m adheres to [fail-fast][ff]. That means rather than brushing state +bugs under the carpet, it panics. We make a distinction between errors and +bugs. + +* Errors are as a result of incorrect or impossible to understand user input. +* Bugs are broken internal invariants (assumptions). + +If you scan the str0m code you find a few `unwrap()` (or `expect()`). These +will (should) always be accompanied by a code comment that explains why the +unwrap is okay. This is an internal invariant, a state assumption that +str0m is responsible for maintaining. + +We do not believe it's correct to change every `unwrap()`/`expect()` into +`unwrap_or_else()`, `if let Some(x) = x { ... }` etc, because doing so +brushes an actual problem (an incorrect assumption) under the carpet. Trying +to hobble along with an incorrect state would at best result in broken +behavior, at worst a security risk! + +Panics are our friends: *panic means bug* + +And also: str0m should *never* panic on any user input. If you encounter a panic, +please report it! + +#### Catching panics + +Panics should be incredibly rare, or we have a serious problem as a project. For an SFU, +it might not be ideal if str0m encounters a bug and brings the entire server down with it. + +For those who want an extra level of safety, we recommend looking at [`catch_unwind`][catch] +to safely discard a faulty `Rtc` instance. Since `Rtc` has no internal threads, locks or async +tasks, discarding the instance never risk poisoning locks or other issues that can happen +when catching a panic. + +### FAQ + +#### Features + +Below is a brief comparison of features between libWebRTC and str0m to help you determine +if str0m is suitable for your project. + +| Feature | str0m | libWebRTC | +| ------------------------ | ------------------ | ------------------ | +| Peer Connection API | :x: | :white_check_mark: | +| SDP | :white_check_mark: | :white_check_mark: | +| ICE | :white_check_mark: | :white_check_mark: | +| Data Channels | :white_check_mark: | :white_check_mark: | +| Send/Recv Reports | :white_check_mark: | :white_check_mark: | +| Transport Wide CC | :white_check_mark: | :white_check_mark: | +| Bandwidth Estimation | :white_check_mark: | :white_check_mark: | +| Simulcast | :white_check_mark: | :white_check_mark: | +| NACK | :white_check_mark: | :white_check_mark: | +| Packetize | :white_check_mark: | :white_check_mark: | +| Fixed Depacketize Buffer | :white_check_mark: | :white_check_mark: | +| Adaptive Jitter Buffer | :x: | :white_check_mark: | +| Video/audio capture | :x: | :white_check_mark: | +| Video/audio encode | :x: | :white_check_mark: | +| Video/audio decode | :x: | :white_check_mark: | +| Audio render | :x: | :white_check_mark: | +| Turn | :x: | :white_check_mark: | +| Network interface enum | :x: | :white_check_mark: | + +#### Platform Support + +Platforms str0m is compiled and tested on: + +| Platform | Compiled | Tested | +| ------------------------------ | ----------------- | ----------------- | +| `x86_64-pc-windows-msvc` | :white_check_mark:| :white_check_mark:| +| `x86_64-unknown-linux-gnu` | :white_check_mark:| :white_check_mark:| +| `x86_64-apple-darwin` | :white_check_mark:| :white_check_mark:| +| `aarch64-apple-darwin` | :white_check_mark:| :white_check_mark:| +| `aarch64-unknown-linux-gnu` | :white_check_mark:| :white_check_mark:| +| `aarch64-pc-windows-msvc` | :white_check_mark:| :white_check_mark:| +| `aarch64-apple-ios` | :white_check_mark:| :x: | +| `aarch64-linux-android` | :white_check_mark:| :x: | + +If your platform isn't listed but is supported by Rust, we'd love for you to give str0m a try and +share your experience. We greatly appreciate your feedback! + +#### Does str0m support IPv4, IPv6, UDP and TCP? + +Certainly! str0m fully support IPv4, IPv6, UDP and TCP protocols. + +#### Can I utilize str0m with any Rust async runtime? + +Absolutely! str0m is fully sync, ensuring that it integrates seamlessly with any Rust async +runtime you opt for. + +#### Can I create a client with str0m? + +Of course! You have the freedom to create a client with str0m. However, please note that some +common client features like media encoding, decoding, and capture are not included in str0m. But +don't let that stop you from building amazing applications! + +#### Can I use str0m in a media server? + +Yes! str0m excels as a server component with support for both RTP API and Frame API. You can +easily build that recording server or SFU you dreamt of in Rust! + +#### Can I deploy the chat example into production? + +While the chat example showcases how to use str0m's API, it's not intended for production use or +heavy load. Writing a full-featured SFU or MCU (Multipoint Control Unit) is a significant +undertaking, involving various design decisions based on production requirements. + +#### Discovered a bug? Here's how to share it with us + +We'd love to hear about it! Please submit an issue and consider joining our Discord community +to discuss further. For a seamless reporting experience, refer to this exemplary +bug report: . We appreciate your contribution +to making str0m better! + +#### I am allergic to SDP can you help me? + +Yes use the direct API! + +[sansio]: https://sans-io.readthedocs.io +[quinn]: https://github.com/quinn-rs/quinn +[pion]: https://github.com/pion/webrtc +[webrtc-rs]: https://github.com/webrtc-rs/webrtc +[discord]: https://discord.gg/e2CC8UYebP +[zulip]: https://str0m.zulipchat.com +[ice]: https://www.rfc-editor.org/rfc/rfc8445 +[lookback]: https://www.lookback.com +[x-post]: https://github.com/algesten/str0m/blob/main/examples/http-post.rs +[x-chat]: https://github.com/algesten/str0m/blob/main/examples/chat.rs +[intg]: https://github.com/algesten/str0m/blob/main/tests/unidirectional.rs#L12 +[ff]: https://en.wikipedia.org/wiki/Fail-fast +[catch]: https://doc.rust-lang.org/std/panic/fn.catch_unwind.html +[evmed]: https://docs.rs/str0m/*/str0m/enum.Event.html#variant.MediaData +[writer]: https://docs.rs/str0m/*/str0m/media/struct.Writer.html#method.write +[reqkey]: https://docs.rs/str0m/*/str0m/media/struct.Writer.html#method.request_keyframe +[rtppak]: https://docs.rs/str0m/*/str0m/enum.Event.html#variant.RtpPacket +[wrtrtp]: https://docs.rs/str0m/*/str0m/rtp/struct.StreamTx.html#method.write_rtp +[reqkey2]: https://docs.rs/str0m/*/str0m/rtp/struct.StreamRx.html#method.request_keyframe +[bitwhip]: https://github.com/bitwhip/bitwhip + +--- + +[str0m's current chat][discord] runs on Discord. + +--- + +[str0m's previous chat][zulip] ran on Zulip and is sponsored by Zulip for open source projects. +Zulip logo + +License: MIT OR Apache-2.0 diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/fuzz.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/fuzz.rs new file mode 100644 index 000000000..cbc5ef58f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/fuzz.rs @@ -0,0 +1,246 @@ +//! Exported fuzz targets to get them part of the compilation with feature `_internal_test_exports`. + +use std::collections::VecDeque; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::{Duration, Instant}; + +use crate::change::{SdpAnswer, SdpOffer}; +use crate::format::Codec; +use crate::net::{Protocol, Receive}; +use crate::packet::{CodecDepacketizer, CodecExtra, Depacketizer, DepacketizingBuffer, RtpMeta}; +use crate::rtp_::{Frequency, MediaTime, Rtcp, RtcpFb, RtpHeader}; +use crate::streams::register::ReceiverRegister; +use crate::streams::rtx_cache_buf::EvictingBuffer; +use crate::{Input, Output, Rtc}; + +use super::Rng; +#[allow(unused)] +use super::setup::{random_config, random_extmap}; + +pub fn rtx_buffer(data: &[u8]) { + if data.len() < 4 { + return; + } + + let buf_size = u16::from_be_bytes([data[0], data[1]]); + let max_age = data[2] as u64; + let max_size = data[3] as usize; + let mut buf = EvictingBuffer::new(buf_size as usize, Duration::from_secs(max_age), max_size); + let mut now = Instant::now(); + let mut pos = 0; + + for d in &data[4..] { + now += Duration::from_millis(*d as u64); + if d % 2 == 0 { + buf.maybe_evict(now) + } else { + pos += *d as u64; + buf.push(pos, now, d); + } + } +} + +pub fn rtp_header(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + let exts = random_extmap(&mut rng, 10)?; + let len = rng.usize(76)?; + RtpHeader::_parse(rng.slice(len)?, &exts); + Some(()) +} + +pub fn rtp_packet(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + let exts = random_extmap(&mut rng, 10)?; + // Maximum RTP packet size is typically around 1500 bytes (MTU) + let packet_len = rng.usize(1500)?; + let packet = rng.slice(packet_len)?; + + // Parse the header from the packet + let header = RtpHeader::_parse(packet, &exts)?; + + // Extract the payload (data after the header) + let _payload = packet.get(header.header_len..)?; + + Some(()) +} + +pub fn sdp_offer(data: &[u8]) -> Option<()> { + let str = std::str::from_utf8(data).ok()?; + let _ = SdpOffer::from_sdp_string(str); + Some(()) +} + +pub fn sdp_answer(data: &[u8]) -> Option<()> { + let str = std::str::from_utf8(data).ok()?; + let _ = SdpAnswer::from_sdp_string(str); + Some(()) +} + +pub fn depack(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + + let codec = match rng.u8(10)? { + 0 => Codec::Opus, + 1 => Codec::Vp8, + 2 => Codec::Vp9, + 3 => Codec::H264, + 4 => Codec::H265, + 5 => Codec::H266, + 6 => Codec::Av1, + 7 => Codec::PCMU, + 8 => Codec::PCMA, + 9 => Codec::G722, + 10 => Codec::CN, + _ => unreachable!(), + }; + + let mut depack = DepacketizingBuffer::new(codec.into(), rng.usize(300)?); + + let exts = random_extmap(&mut rng, 10)?; + + let start = Instant::now(); + + loop { + let do_push = rng.bool()?; + + if do_push { + let hlen = rng.usize(76)?; + let header = RtpHeader::_parse(rng.slice(hlen)?, &exts)?; + let meta = RtpMeta { + received: start + Duration::from_millis(rng.u64(10000)?), + time: MediaTime::new(rng.u64(u64::MAX)?, Frequency::MICROS), + seq_no: rng.u64(u64::MAX)?.into(), + header, + last_sender_info: None, + }; + let len = rng.usize(1200)?; + let data = rng.slice(len)?.to_vec(); + depack.push(meta, data); + } else { + depack.pop(); + } + } +} + +pub fn receive_register(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + let mut rr = ReceiverRegister::new(None); + let start = Instant::now(); + loop { + match rng.u8(2)? { + 0 => { + let seq = rng.u64(u64::MAX / 2)?; + let arrival = start + Duration::from_micros(rng.u64(u64::MAX / 100)?); + let rtp_time = rng.u32(u32::MAX / 2)?; + let clock_rate = rng.u32(u32::MAX / 2)?; + rr.update(seq.into(), arrival, rtp_time, clock_rate); + } + 1 => { + rr.nack_report(); + } + 2 => { + rr.reception_report(); + } + _ => unreachable!(), + } + } +} + +/// Fuzz the full `Rtc` receive path: instance init, datagram demux +/// (STUN/DTLS/RTP/RTCP), dispatch, and `poll_output` draining. +pub fn receive(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + + let base = Instant::now(); + let mut rtc = Rtc::new(base); + + let source = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 5000); + let destination = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(2, 2, 2, 2)), 5001); + + let mut millis: u64 = 0; + + loop { + // Drive time monotonically forward. + millis += rng.u64(50)?; + let now = base + Duration::from_millis(millis); + + if rng.bool()? { + // Feed a datagram through the full demux + dispatch path. + let len = rng.usize(2000)?; + let buf = rng.slice(len)?; + if let Ok(receive) = Receive::new(Protocol::Udp, source, destination, buf) { + let _ = rtc.handle_input(Input::Receive(now, receive)); + } + } else { + let _ = rtc.handle_input(Input::Timeout(now)); + } + + // Drain outputs (bounded to avoid spinning on a single input). + for _ in 0..1000 { + match rtc.poll_output() { + Ok(Output::Timeout(_)) => break, + Ok(_) => continue, + Err(_) => break, + } + } + + if !rtc.is_alive() { + return Some(()); + } + } +} + +/// Fuzz STUN message parsing (the ICE input path). +pub fn stun(data: &[u8]) -> Option<()> { + let _ = is::stun::StunMessage::parse(data); + Some(()) +} + +/// Fuzz RTCP compound-packet parsing and the feedback conversion. +pub fn rtcp(data: &[u8]) -> Option<()> { + let mut feedback = VecDeque::new(); + Rtcp::read_packet(data, &mut feedback); + // Exercise the feedback conversion as well. + let _ = RtcpFb::from_rtcp(feedback.drain(..)).count(); + Some(()) +} + +/// Fuzz each codec depacketizer directly (bypassing the buffer's gating), +/// exercising the raw `Depacketizer::depacketize` parsers. +pub fn depack_direct(data: &[u8]) -> Option<()> { + let mut rng = Rng::new(data); + + let codec = match rng.u8(10)? { + 0 => Codec::Opus, + 1 => Codec::Vp8, + 2 => Codec::Vp9, + 3 => Codec::H264, + 4 => Codec::H265, + 5 => Codec::H266, + 6 => Codec::Av1, + 7 => Codec::PCMU, + 8 => Codec::PCMA, + 9 => Codec::G722, + 10 => Codec::CN, + _ => unreachable!(), + }; + + let mut depack: CodecDepacketizer = codec.into(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + + loop { + let marker = rng.bool()?; + let len = rng.usize(1500)?; + let packet = rng.slice(len)?; + + let _ = depack.depacketize(packet, &mut out, &mut extra); + let _ = depack.is_partition_head(packet); + let _ = depack.is_partition_tail(marker, packet); + + // Keep memory bounded across iterations. + if out.len() > 4_000_000 { + out.clear(); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/mod.rs new file mode 100644 index 000000000..5fea7be0d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/mod.rs @@ -0,0 +1,48 @@ +//! Exported things with feature `_internal_test_exports`. + +use crate::Rtc; +use crate::format::PayloadParams; +use crate::media::Media; +use crate::media::Mid; +use crate::rtp::{ExtensionMap, RtpHeader}; +use is::IceCreds; + +pub mod fuzz; +mod rng; +use rng::Rng; + +mod setup; + +impl Rtc { + /// UNSTABLE: not public API! + pub fn _mids(&self) -> Vec { + self.session.medias.iter().map(Media::mid).collect() + } + + /// UNSTABLE: not public API! + pub fn _exts(&self) -> &ExtensionMap { + &self.session.exts + } + + /// UNSTABLE: not public API! + pub fn _local_ice_creds(&self) -> IceCreds { + self.ice.local_credentials().clone() + } +} + +impl RtpHeader { + /// UNSTABLE: not public API! + pub fn _parse(buf: &[u8], exts: &ExtensionMap) -> Option { + Self::parse(buf, exts) + } +} + +impl PayloadParams { + /// UNSTABLE: not public API! + pub fn _is_locked(&self) -> bool { + self.locked + } +} + +/// UNSTABLE: not public API! +pub use crate::bwe_::ProbeClusterConfig; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/rng.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/rng.rs new file mode 100644 index 000000000..c6d6aa87b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/rng.rs @@ -0,0 +1,55 @@ +pub struct Rng<'a>(&'a [u8], usize); + +macro_rules! rng_primary { + ($unit:tt, $above:tt) => { + pub fn $unit(&mut self, max: $unit) -> Option<$unit> { + const X: usize = std::mem::size_of::<$unit>(); + let bytes = self.array::()?; + let u = $unit::from_be_bytes(bytes); + Some(if max == $unit::MAX { + u + } else { + ((u as $above * max as $above) / $unit::MAX as $above) as $unit + }) + } + }; +} + +impl<'a> Rng<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self(data, 0) + } + + pub fn slice(&mut self, n: usize) -> Option<&[u8]> { + let start = self.1; + let end = self.1 + n; + + if end > self.0.len() { + // First that hits limit ends anymore data. + self.1 = self.0.len(); + return None; + } + + self.1 = end; + + Some(&self.0[start..end]) + } + + pub fn array(&mut self) -> Option<[u8; N]> { + let slice = self.slice(N)?; + + let mut arr = [0_u8; N]; + arr.copy_from_slice(slice); + + Some(arr) + } + + pub fn bool(&mut self) -> Option { + Some(self.array::<1>()?[0] < 128) + } + + rng_primary!(u8, u16); + rng_primary!(u32, u64); + rng_primary!(u64, u128); + rng_primary!(usize, u128); +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/setup.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/setup.rs new file mode 100644 index 000000000..f870030ad --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/_internal_test_exports/setup.rs @@ -0,0 +1,67 @@ +use std::time::Duration; + +use crate::Bitrate; +use crate::RtcConfig; +use crate::rtp::{Extension, ExtensionMap}; + +use super::Rng; + +pub fn random_extmap(rng: &mut Rng, to_set: usize) -> Option { + use Extension::*; + let mut e = ExtensionMap::empty(); + for _ in 0..to_set { + let id = rng.u8(13)? + 1; + let ext = match rng.u8(12)? { + 0 => AbsoluteSendTime, + 1 => AudioLevel, + 2 => TransmissionTimeOffset, + 3 => VideoOrientation, + 4 => TransportSequenceNumber, + 5 => PlayoutDelay, + 6 => VideoContentType, + 7 => VideoTiming, + 8 => RtpStreamId, + 9 => RepairedRtpStreamId, + 10 => RtpMid, + 11 => FrameMarking, + 12 => ColorSpace, + _ => unreachable!(), + }; + e.set(id, ext); + } + Some(e) +} + +#[allow(unused)] +pub fn random_config(rng: &mut Rng) -> Option { + let mut c = RtcConfig::new(); + c = c.set_extension_map(random_extmap(rng, 8)?); + c = c.set_ice_lite(rng.bool()?); + c = c.set_fingerprint_verification(rng.bool()?); + c = c.clear_codecs(); + c = c.enable_opus(rng.bool()?); + c = c.enable_h264(rng.bool()?); + c = c.enable_vp8(rng.bool()?); + c = c.enable_vp9(rng.bool()?); + c = c.enable_av1(rng.bool()?); + if rng.bool()? { + rng.bool(); // consume one + c = c.set_stats_interval(None); + } else { + let t = Duration::from_millis(rng.u64(10_000)?); + c = c.set_stats_interval(Some(t)); + } + if rng.bool()? { + rng.bool(); + c = c.enable_bwe(None); + } else { + c = c.enable_bwe(Some(Bitrate::bps(rng.u64(u64::MAX)?))); + } + c = c.set_reordering_size_audio(rng.usize(usize::MAX)?); + c = c.set_reordering_size_video(rng.usize(usize::MAX)?); + c = c.set_send_buffer_audio(rng.usize(usize::MAX)?.saturating_add(1)); // panics if set to 0 + c = c.set_send_buffer_video(rng.usize(usize::MAX)?); + c = c.set_rtp_mode(rng.bool()?); + c = c.enable_raw_packets(rng.bool()?); + Some(c) +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/acked_bitrate_estimator.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/acked_bitrate_estimator.rs new file mode 100644 index 000000000..d6aacef96 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/acked_bitrate_estimator.rs @@ -0,0 +1,253 @@ +use std::time::{Duration, Instant}; + +use crate::Bitrate; +use crate::rtp_::DataSize; + +const SMALL_SAMPLE_THRESHOLD: DataSize = DataSize::bytes(2000); +const SMALL_SAMPLE_UNCERTAINTY: f64 = 25.0; +const UNCERTAINTY: f64 = 10.0; +const ESTIMATE_FLOOR: Bitrate = Bitrate::kbps(40); + +pub struct AckedBitrateEstimator { + /// The initial window to use for the first estimate. + initial_window: Duration, + /// The window to use for subsequent estimates after achieving the initial estimate. + window: Duration, + /// The estimate of the acked bitrate. + estimate: Option, + /// The estimated variance. + estimate_var: f64, + /// The sum in the current window. + sum: DataSize, + /// The size of the current window. + current_window: Duration, + /// The last time the window was updated. + last_update: Option, +} + +impl AckedBitrateEstimator { + pub(super) fn new(initial_window: Duration, window: Duration) -> Self { + Self { + initial_window, + window, + estimate: None, + estimate_var: 50.0, + sum: DataSize::ZERO, + current_window: Duration::ZERO, + last_update: None, + } + } + + pub(super) fn update(&mut self, receive_time: Instant, packet_size: DataSize) { + let window = if self.estimate.is_none() { + // Use the initial, larger, window at first + self.initial_window + } else { + self.window + }; + let Some((sample_estimate, is_small_sample)) = + self.update_window(receive_time, packet_size, window) + else { + // No update + return; + }; + + let Some(estimate) = self.estimate else { + // This is the initial estimate, use it to initialize the estimate. + self.estimate = Some(sample_estimate); + return; + }; + + let scale = if is_small_sample && sample_estimate < estimate { + SMALL_SAMPLE_UNCERTAINTY + } else { + UNCERTAINTY + }; + + let sample_estimate_bps = sample_estimate.as_f64(); + let estimate_bps = estimate.as_f64(); + // Define the sample uncertainty as a function of how far away it is from the + // current estimate. With low values of uncertainty_symmetry_cap_ we add more + // uncertainty to increases than to decreases. For higher values we approach + // symmetry. + let sample_uncertainty = scale * (estimate_bps - sample_estimate_bps).abs() / estimate_bps; + let sample_var = sample_uncertainty.powf(2.0); + + // Update a bayesian estimate of the rate, weighting it lower if the sample + // uncertainty is large. + // The bitrate estimate uncertainty is increased with each update to model + // that the bitrate changes over time. + let pred_bitrate_estimate_var = self.estimate_var + 5.0; + let mut new_estimate = (sample_var * estimate_bps + + pred_bitrate_estimate_var * sample_estimate_bps) + / (sample_var + pred_bitrate_estimate_var); + + new_estimate = new_estimate.max(ESTIMATE_FLOOR.as_f64()); + self.estimate = Some(Bitrate::bps(new_estimate.ceil() as u64)); + self.estimate_var = + (sample_var * pred_bitrate_estimate_var) / (sample_var + pred_bitrate_estimate_var); + } + + pub(super) fn current_estimate(&self) -> Option { + self.estimate + } + + fn update_window( + &mut self, + receive_time: Instant, + packet_size: DataSize, + window: Duration, + ) -> Option<(Bitrate, bool)> { + let time_moved_back = Some(receive_time) < self.last_update; + if time_moved_back { + // Time moved backwards, reset state + self.sum = DataSize::ZERO; + self.current_window = Duration::ZERO; + self.last_update = Some(receive_time); + + return None; + } + + if let Some(last) = self.last_update { + self.current_window += receive_time - last; + if receive_time - last >= window { + // No update for a while, reset estimates. + self.sum = DataSize::ZERO; + let remaining = self.current_window.as_micros() as u64 % window.as_micros() as u64; + self.current_window = Duration::from_micros(remaining); + } + } + + self.last_update = Some(receive_time); + + let mut estimate = None; + + let mut is_small = false; + if self.current_window >= window { + is_small = self.sum < SMALL_SAMPLE_THRESHOLD; + estimate = Some(self.sum / window); + self.sum = DataSize::ZERO; + self.current_window -= window; + } + + self.sum += packet_size; + + estimate.map(|e| (e, is_small)) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_no_estimate_before_initial_window_has_passed() { + let now = Instant::now(); + let mut estimator = + AckedBitrateEstimator::new(Duration::from_millis(500), Duration::from_millis(150)); + + estimator.update(now, DataSize::bytes(950)); + estimator.update(now + Duration::from_millis(250), DataSize::bytes(381)); + estimator.update(now + Duration::from_millis(499), DataSize::bytes(1110)); + + assert!( + estimator.current_estimate().is_none(), + "AckedBitrateEstiamtor should produce no estimate before the initial window is reached" + ); + + estimator.update(now + Duration::from_millis(501), DataSize::bytes(1110)); + + let estimate = estimator.current_estimate().expect( + "After the first window of time has passed AckedBitrateEstimator should produce an estimate" + ); + + assert_eq!( + estimate.as_u64(), + 39056, + "AckedBitrateEstiamtor should produce the correct bitrate" + ); + } + + #[test] + fn test_correct_estimate_after_initial_window() { + let now = Instant::now(); + let mut estimator = + AckedBitrateEstimator::new(Duration::from_millis(500), Duration::from_millis(150)); + + estimator.update(now, DataSize::bytes(2500)); + estimator.update(now + Duration::from_millis(250), DataSize::bytes(1392)); + estimator.update(now + Duration::from_millis(499), DataSize::bytes(4021)); + estimator.update(now + Duration::from_millis(500), DataSize::bytes(0)); + + assert!( + estimator.current_estimate().is_some(), + "After the first window of time has passed AckedBitrateEstimator should produce an estimate" + ); + + estimator.update(now + Duration::from_millis(550), DataSize::bytes(271)); + estimator.update(now + Duration::from_millis(558), DataSize::bytes(813)); + estimator.update(now + Duration::from_millis(648), DataSize::bytes(731)); + // Will not be counted, part of next window + estimator.update(now + Duration::from_millis(651), DataSize::bytes(900)); + + let estimate = estimator.current_estimate().expect( + "After the first window of time has passed AckedBitrateEstimator should produce an estimate" + ); + + assert_eq!( + estimate.as_u64(), + 108320, + "AckedBitrateEstiamtor should produce the correct bitrate" + ); + } + + #[test] + fn test_long_gap_preserves_overflow_via_modulo_after_initial() { + let now = Instant::now(); + let window = Duration::from_millis(150); + let mut estimator = AckedBitrateEstimator::new(Duration::from_millis(500), window); + + // First, get past the initial 500ms window to establish an estimate + // Need to send enough data over 500ms to trigger the first estimate + for i in 0..50 { + estimator.update(now + Duration::from_millis(i * 10), DataSize::bytes(1000)); + } + estimator.update(now + Duration::from_millis(501), DataSize::bytes(1000)); + assert!( + estimator.current_estimate().is_some(), + "Should have initial estimate after 500ms" + ); + + // Now we're using the 150ms window + // Build up current_window to 100ms + estimator.update(now + Duration::from_millis(550), DataSize::bytes(1000)); + estimator.update(now + Duration::from_millis(650), DataSize::bytes(1000)); + // current_window is now 100ms + + // Long gap of 225ms (> 150ms window) + // current_window becomes 100 + 225 = 325ms + // After gap detection: current_window = 325ms % 150ms = 25ms (correct) + // or: current_window = 150ms % 150ms = 0ms (buggy) + estimator.update(now + Duration::from_millis(875), DataSize::bytes(1000)); + + // Add 50ms + estimator.update(now + Duration::from_millis(925), DataSize::bytes(1000)); + // Correct: current_window = 25 + 50 = 75ms + // Buggy: current_window = 0 + 50 = 50ms + + // Add 80ms + let estimate_before = estimator.current_estimate().unwrap(); + estimator.update(now + Duration::from_millis(1005), DataSize::bytes(1000)); + // Correct: current_window = 75 + 80 = 155ms (>= 150ms, triggers new sample) + // Buggy: current_window = 50 + 80 = 130ms (< 150ms, no new sample) + let estimate_after = estimator.current_estimate().unwrap(); + + // With correct modulo, estimate should change (new sample produced) + // With buggy modulo, estimate stays the same (no sample produced yet) + assert_ne!( + estimate_before.as_u64(), + estimate_after.as_u64(), + "Estimate should change after accumulating >= 150ms due to correct modulo overflow" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/budget.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/budget.rs new file mode 100644 index 000000000..0b4d4a3a2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/budget.rs @@ -0,0 +1,281 @@ +//! Interval budget tracker for ALR detection. +//! +//! Port of libWebRTC's IntervalBudget class from +//! `webrtc/modules/pacing/interval_budget.{h,cc}` + +use std::time::Duration; + +use crate::rtp_::{Bitrate, DataSize}; + +/// Leaky bucket budget tracker over a fixed time window. +/// +/// Used by ALR detector to handle bursty traffic (encoders, etc.) +/// over a 500ms window. Tracks how much "budget" we have left +/// based on target rate vs. actual send rate. +/// +/// This implements the same logic as libWebRTC's IntervalBudget class +/// to ensure ALR detection behaves identically when handling bursty +/// encoder traffic (keyframes, scene changes, temporal layers, etc.) +#[derive(Debug, Clone)] +pub struct IntervalBudget { + /// Target bitrate for budget calculation + target_rate: Bitrate, + /// Maximum bytes that can accumulate in the budget + max_bytes_in_budget: DataSize, + /// Current bytes remaining in budget (can be negative = debt) + bytes_remaining: DataSize, + /// Whether underuse can build up credit across intervals + can_build_up_underuse: bool, +} + +impl IntervalBudget { + /// Window size for budget tracking + const WINDOW: Duration = Duration::from_millis(500); + + /// Create a new IntervalBudget with the specified target rate. + pub fn new(target_rate: Bitrate, can_build_up_underuse: bool) -> Self { + let max_bytes_in_budget = Self::calculate_max_bytes(target_rate); + + Self { + target_rate, + max_bytes_in_budget, + bytes_remaining: DataSize::ZERO, + can_build_up_underuse, + } + } + + /// Update the target bitrate. + /// + /// Called when the BWE estimate changes. Clamps existing bytes_remaining + /// to the new max_bytes_in_budget range. + pub fn set_target_rate(&mut self, target_rate: Bitrate) { + self.target_rate = target_rate; + self.max_bytes_in_budget = Self::calculate_max_bytes(target_rate); + + // Clamp bytes_remaining to new budget limits + let max = self.max_bytes_in_budget; + let neg_max = max * -1i64; + self.bytes_remaining = DataSize::bytes( + self.bytes_remaining + .as_bytes_i64() + .clamp(neg_max, max.as_bytes_i64()), + ); + } + + /// Add budget based on elapsed time. + /// + /// Increases the budget by `target_rate * delta_time`. + /// Behavior depends on `can_build_up_underuse`: + /// - If true: underuse accumulates (for ALR detection) + /// - If false: budget resets each interval (for pacing) + pub fn increase_budget(&mut self, delta_time: Duration) { + let bytes = self.target_rate * delta_time; + + let max = self.max_bytes_in_budget; + if self.bytes_remaining.as_bytes_i64() < 0 || self.can_build_up_underuse { + // We overused last interval, compensate this interval + // OR we allow building up underuse credit + self.bytes_remaining = DataSize::bytes( + (self.bytes_remaining + bytes) + .as_bytes_i64() + .min(max.as_bytes_i64()), + ); + } else { + // If we underused last interval we can't use it this interval + self.bytes_remaining = DataSize::bytes(bytes.as_bytes_i64().min(max.as_bytes_i64())); + } + } + + /// Consume budget by the specified number of bytes. + /// + /// This represents sending data. The budget can go negative (debt). + pub fn use_budget(&mut self, bytes: DataSize) { + let max = self.max_bytes_in_budget; + let neg_max = max * -1i64; + self.bytes_remaining = + DataSize::bytes((self.bytes_remaining - bytes).as_bytes_i64().max(neg_max)); + } + + /// Get the current budget ratio. + /// + /// Returns a value in the range [-1.0, 1.0]: + /// - 1.0 = full budget (maximum underuse) + /// - 0.0 = neutral (sending exactly at target rate) + /// - -1.0 = maximum debt (maximum overuse) + /// + /// Used by ALR detector for hysteresis thresholds: + /// - Enter ALR when ratio > 0.80 (sustained low usage) + /// - Exit ALR when ratio < 0.50 (increased usage) + pub fn budget_ratio(&self) -> f64 { + let max = self.max_bytes_in_budget.as_bytes_i64(); + if max == 0 { + return 0.0; + } + self.bytes_remaining.as_bytes_i64() as f64 / max as f64 + } + + /// Calculate max_bytes_in_budget from target rate + fn calculate_max_bytes(target_rate: Bitrate) -> DataSize { + target_rate * Self::WINDOW + } + + #[cfg(test)] + pub fn target_rate(&self) -> Bitrate { + self.target_rate + } + + #[cfg(test)] + pub fn bytes_remaining(&self) -> i64 { + self.bytes_remaining.as_bytes_i64() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_initial_budget_is_zero() { + let budget = IntervalBudget::new(Bitrate::kbps(300), true); + assert_eq!(budget.bytes_remaining(), 0); + assert_eq!(budget.budget_ratio(), 0.0); + } + + #[test] + fn test_increase_budget_accumulates() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Add 100ms worth of budget: 300 kbps * 100ms = 3750 bytes + budget.increase_budget(Duration::from_millis(100)); + assert_eq!(budget.bytes_remaining(), 3750); + + // Budget ratio should be positive (underuse) + assert!(budget.budget_ratio() > 0.0); + } + + #[test] + fn test_use_budget_creates_debt() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Use 5000 bytes + budget.use_budget(DataSize::bytes(5000)); + assert_eq!(budget.bytes_remaining(), -5000); + + // Budget ratio should be negative (overuse) + assert!(budget.budget_ratio() < 0.0); + } + + #[test] + fn test_budget_ratio_clamped() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + let max_bytes = (Bitrate::kbps(300) * Duration::from_millis(500)).as_bytes_i64(); + + // Fill to max + budget.increase_budget(Duration::from_millis(500)); + assert_eq!(budget.budget_ratio(), 1.0); + + // Try to add more - should stay at 1.0 + budget.increase_budget(Duration::from_millis(100)); + assert_eq!(budget.budget_ratio(), 1.0); + + // Use all budget plus create max debt + budget.use_budget(DataSize::bytes(max_bytes * 2)); + assert_eq!(budget.budget_ratio(), -1.0); + } + + #[test] + fn test_can_build_up_underuse_false() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), false); + + // Add budget + budget.increase_budget(Duration::from_millis(100)); + let after_first = budget.bytes_remaining(); + assert!(after_first > 0); + + // Add more budget - should reset, not accumulate + budget.increase_budget(Duration::from_millis(100)); + assert_eq!(budget.bytes_remaining(), 3750); // Only the latest 100ms + + // Not the accumulated 7500 + assert_ne!(budget.bytes_remaining(), after_first + 3750); + } + + #[test] + fn test_can_build_up_underuse_true() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Add budget + budget.increase_budget(Duration::from_millis(100)); + let after_first = budget.bytes_remaining(); + + // Add more budget - should accumulate + budget.increase_budget(Duration::from_millis(100)); + assert_eq!(budget.bytes_remaining(), after_first + 3750); + } + + #[test] + fn test_debt_recovery() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Create debt + budget.use_budget(DataSize::bytes(5000)); + assert_eq!(budget.bytes_remaining(), -5000); + + // Add budget to recover from debt + budget.increase_budget(Duration::from_millis(200)); + let expected_recovery = 3750 * 2; // 200ms worth + assert_eq!(budget.bytes_remaining(), -5000 + expected_recovery); + } + + #[test] + fn test_set_target_rate_clamps_budget() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Build up budget at 300 kbps + budget.increase_budget(Duration::from_millis(500)); + let old_max = (Bitrate::kbps(300) * Duration::from_millis(500)).as_bytes_i64(); + assert_eq!(budget.bytes_remaining(), old_max); + + // Reduce target rate - budget should be clamped to new max + budget.set_target_rate(Bitrate::kbps(150)); + let new_max = (Bitrate::kbps(150) * Duration::from_millis(500)).as_bytes_i64(); + assert_eq!(budget.bytes_remaining(), new_max); + assert!(new_max < old_max); + } + + #[test] + fn test_alr_hysteresis_thresholds() { + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + + // Simulate ALR entry threshold (80% budget ratio) + let target_bytes = + (0.8 * (Bitrate::kbps(300) * Duration::from_millis(500)).as_bytes_i64() as f64) as i64; + + // Accumulate budget to 80% threshold + while budget.bytes_remaining() < target_bytes { + budget.increase_budget(Duration::from_millis(50)); + } + + assert!(budget.budget_ratio() >= 0.80); + + // Simulate ALR exit threshold (50% budget ratio) + // Use budget to bring ratio down + let use_amount = + (0.30 * (Bitrate::kbps(300) * Duration::from_millis(500)).as_bytes_i64() as f64) as i64; + budget.use_budget(DataSize::bytes(use_amount)); + + assert!(budget.budget_ratio() < 0.80); + assert!(budget.budget_ratio() > 0.40); // Should be around 50% + } + + #[test] + fn test_target_rate_getter() { + let budget = IntervalBudget::new(Bitrate::kbps(500), true); + assert_eq!(budget.target_rate(), Bitrate::kbps(500)); + + // After modifying target rate + let mut budget = IntervalBudget::new(Bitrate::kbps(300), true); + budget.set_target_rate(Bitrate::kbps(600)); + assert_eq!(budget.target_rate(), Bitrate::kbps(600)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/detector.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/detector.rs new file mode 100644 index 000000000..8313354e8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/detector.rs @@ -0,0 +1,344 @@ +use std::time::Instant; + +use super::budget::IntervalBudget; +use crate::rtp_::{Bitrate, DataSize}; + +/// Application Limited Region detector. +/// +/// Detects when we're sending significantly less than network capacity, +/// using IntervalBudget to handle bursty encoder traffic over 500ms windows. +/// +/// This is critical for real-world usage where encoders produce bursty traffic +/// (keyframes, scene changes, temporal layers, etc.) The IntervalBudget smooths +/// these bursts over a 500ms window to detect sustained application-limited state. +/// +/// ## ALR State Transitions +/// +/// - **Enter ALR**: When budget ratio > 0.80 (sustained low usage for 500ms) +/// - **Exit ALR**: When budget ratio < 0.50 (increased usage) +/// - The 30% hysteresis gap prevents rapid state flapping +#[derive(Debug)] +pub struct AlrDetector { + /// Budget tracker with 500ms window + budget: IntervalBudget, + /// Current ALR state + state: AlrState, + /// Last time we sent bytes + last_send_time: Option, + + // Configuration matching libWebRTC defaults + /// Target send rate as fraction of estimate (0.65 = 65%) + bandwidth_usage_ratio: f64, + /// Budget ratio threshold to enter ALR (0.80 = 80%) + start_budget_level_ratio: f64, + /// Budget ratio threshold to exit ALR (0.50 = 50%) + stop_budget_level_ratio: f64, +} + +/// ALR state for the detector. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AlrState { + /// Not currently in Application Limited Region + NotInAlr, + /// In Application Limited Region since the given instant + InAlr(Instant), +} + +impl AlrState { + fn alr_start_time(&self) -> Option { + match self { + AlrState::InAlr(start) => Some(*start), + AlrState::NotInAlr => None, + } + } +} + +impl AlrDetector { + /// Create a new ALR detector with libWebRTC default configuration. + pub fn new() -> Self { + Self { + // Start with zero bitrate, will be updated via set_estimated_bitrate() + // Use can_build_up_underuse=true for ALR detection + budget: IntervalBudget::new(Bitrate::ZERO, true), + state: AlrState::NotInAlr, + last_send_time: None, + + // libWebRTC defaults from alr_detector.cc:30-49 + bandwidth_usage_ratio: 0.65, + start_budget_level_ratio: 0.80, + stop_budget_level_ratio: 0.50, + } + } + + /// Update with bytes sent. + /// + /// Should be called for every media packet sent (excluding padding and probes). + /// This updates the IntervalBudget and checks for ALR state transitions. + pub fn on_bytes_sent(&mut self, bytes: DataSize, now: Instant) { + // First call - just record time + let Some(last) = self.last_send_time else { + self.last_send_time = Some(now); + return; + }; + + // saturating_duration_since protects against time going backwards + let delta = now.saturating_duration_since(last); + self.last_send_time = Some(now); + + // Update budget based on time and usage + self.budget.use_budget(bytes); + self.budget.increase_budget(delta); + + // Check for state transitions with hysteresis + let ratio = self.budget.budget_ratio(); + + match self.state { + AlrState::NotInAlr => { + if ratio > self.start_budget_level_ratio { + // Enter ALR: budget accumulated because we're not sending much + self.state = AlrState::InAlr(now); + trace!( + "ALR: Entered ALR state (ratio={:.3} > threshold={:.3})", + ratio, self.start_budget_level_ratio + ); + } + } + AlrState::InAlr(_) => { + if ratio < self.stop_budget_level_ratio { + // Exit ALR: budget depleted because we're sending more + self.state = AlrState::NotInAlr; + trace!( + "ALR: Exited ALR state (ratio={:.3} < threshold={:.3})", + ratio, self.stop_budget_level_ratio + ); + } + } + } + } + + /// Update the BWE estimate. + /// + /// This adjusts the IntervalBudget's target rate to 65% of the estimate. + /// Should be called whenever the bandwidth estimate changes. + pub fn set_estimated_bitrate(&mut self, estimate: Bitrate) { + let target = estimate * self.bandwidth_usage_ratio; + self.budget.set_target_rate(target); + } + + /// Get the time when ALR started, if currently in ALR. + /// + /// Returns `Some(Instant)` if in ALR, `None` otherwise. + /// Used by ProbeControl to determine if periodic ALR probes should be sent. + pub fn alr_start_time(&self) -> Option { + self.state.alr_start_time() + } + + #[cfg(test)] + pub fn budget_ratio(&self) -> f64 { + self.budget.budget_ratio() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn test_alr_not_detected_initially() { + let alr = AlrDetector::new(); + assert!(alr.alr_start_time().is_none()); + } + + #[test] + fn test_alr_enter_with_low_usage() { + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + // Set estimate to 1 Mbps (target will be 650 kbps) + alr.set_estimated_bitrate(Bitrate::mbps(1)); + + // Send very little data (way below target) + // Over 500ms, we should accumulate budget + for i in 0..50 { + let t = now + Duration::from_millis(i * 10); + // Send 100 bytes every 10ms = 80 kbps (way below 650 kbps target) + alr.on_bytes_sent(DataSize::bytes(100), t); + } + + // Should enter ALR + assert!(alr.alr_start_time().is_some()); + assert!(alr.budget_ratio() > 0.80); + } + + #[test] + fn test_alr_exit_with_high_usage() { + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + alr.set_estimated_bitrate(Bitrate::mbps(1)); + + // Enter ALR with low usage + for i in 0..50 { + let t = now + Duration::from_millis(i * 10); + alr.on_bytes_sent(DataSize::bytes(100), t); + } + assert!(alr.alr_start_time().is_some()); + + // Now send at high rate to exit ALR + let start_exit = now + Duration::from_millis(500); + for i in 0..50 { + let t = start_exit + Duration::from_millis(i * 10); + // Send 10,000 bytes every 10ms = 8 Mbps (way above 650 kbps target) + alr.on_bytes_sent(DataSize::bytes(10_000), t); + } + + // Should exit ALR + assert!(alr.alr_start_time().is_none()); + assert!(alr.budget_ratio() < 0.50); + } + + #[test] + fn test_alr_hysteresis() { + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + alr.set_estimated_bitrate(Bitrate::mbps(1)); + + // Send at exactly 65% (the target) - should not enter ALR + for i in 0..100 { + let t = now + Duration::from_millis(i * 10); + // 65% of 1 Mbps = 650 kbps = 812.5 bytes per 10ms + alr.on_bytes_sent(DataSize::bytes(812), t); + } + + // Budget ratio should be near 0, not trigger ALR entry (needs > 0.80) + assert!(alr.alr_start_time().is_none()); + let ratio = alr.budget_ratio(); + assert!(ratio < 0.80, "ratio={}", ratio); + assert!(ratio > -0.20, "ratio={}", ratio); // Some tolerance + } + + #[test] + fn test_alr_handles_bursts() { + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + alr.set_estimated_bitrate(Bitrate::mbps(1)); + + // Simulate bursty encoder: alternate between large and small frames + let mut time = now; + for i in 0..100 { + time = time + Duration::from_millis(33); // ~30fps + + if i % 10 == 0 { + // Keyframe: 10x normal size + alr.on_bytes_sent(DataSize::bytes(50_000), time); + } else { + // Normal frame: small + alr.on_bytes_sent(DataSize::bytes(5_000), time); + } + } + + // Average is about 9.5 KB per frame = 285 KB/s = 2.28 Mbps + // This is above the 650 kbps target, so should NOT be in ALR + // The IntervalBudget's 500ms window should smooth out the bursts + assert!(alr.alr_start_time().is_none()); + } + + #[test] + fn test_alr_estimate_change() { + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + // Start with 1 Mbps estimate (target = 650 kbps) + alr.set_estimated_bitrate(Bitrate::mbps(1)); + + // Send at 200 kbps (way below 650 kbps target) to accumulate budget quickly + for i in 0..100 { + let t = now + Duration::from_millis(i * 10); + alr.on_bytes_sent(DataSize::bytes(250), t); // 200 kbps + } + + // Should enter ALR after sending well below target + assert!(alr.alr_start_time().is_some()); + + // Now BWE drops to 300 kbps (target becomes 195 kbps) + alr.set_estimated_bitrate(Bitrate::kbps(300)); + + // Continue sending at 250 kbps - now above target (195 kbps) + let continue_time = now + Duration::from_millis(1000); + for i in 0..100 { + let t = continue_time + Duration::from_millis(i * 10); + alr.on_bytes_sent(DataSize::bytes(312), t); // 250 kbps + } + + // Should exit ALR (now sending above target) + assert!(alr.alr_start_time().is_none()); + } + + #[test] + fn test_alr_should_not_trigger_when_sending_at_target_rate() { + // This test verifies that ALR doesn't incorrectly trigger when sending + // at the target rate, even with small time advances between packets. + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + // Set estimate to 8.25 Mbps (target will be 5.36 Mbps at 65%) + let estimate = Bitrate::kbps(8250); + alr.set_estimated_bitrate(estimate); + + // Send at exactly 8.25 Mbps with realistic packet timing + // Packet: 1150 bytes + // Interval: 1150 bytes × 8 bits / 8,250,000 bps = 1.115ms + let packet_size = DataSize::bytes(1150); + let packet_interval = Duration::from_micros(1115); // 1.115ms + + // Send 100 packets over ~111ms + for i in 0..100 { + let t = now + packet_interval * i; + alr.on_bytes_sent(packet_size, t); + } + + // Should NOT be in ALR - we're sending at 8.25 Mbps, which is above + // the target threshold of 5.36 Mbps (65% of estimate) + assert!( + alr.alr_start_time().is_none(), + "ALR should not trigger when sending at target rate. Budget ratio: {:.3}", + alr.budget_ratio() + ); + } + + #[test] + fn test_alr_handles_spurious_small_time_advances() { + // This test verifies that ALR works correctly even when time advances + // in very small increments (like the test harness's forced 0.2ms advances). + let mut alr = AlrDetector::new(); + let now = Instant::now(); + + // Set estimate to 8.25 Mbps + let estimate = Bitrate::kbps(8250); + alr.set_estimated_bitrate(estimate); + + // Simulate sending 1150 byte packets at 8.25 Mbps + // But with spurious 0.2ms time advances between on_bytes_sent calls + let packet_size = DataSize::bytes(1150); + let spurious_advance = Duration::from_micros(200); // 0.2ms + + let mut time = now; + // Send packets - but accumulate proper time even if called frequently + for _ in 0..100 { + alr.on_bytes_sent(packet_size, time); + time += spurious_advance; // Small spurious advance + } + + // With 100 packets over 20ms (100 × 0.2ms), we've sent: + // 115,000 bytes in 20ms = 46 Mbps + // This is WAY above target, so ALR should NOT trigger + assert!( + alr.alr_start_time().is_none(), + "ALR should not trigger with spurious small advances when sending fast. Budget ratio: {:.3}", + alr.budget_ratio() + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/mod.rs new file mode 100644 index 000000000..d96335dc5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/alr_detector/mod.rs @@ -0,0 +1,6 @@ +//! Application Limited Region (ALR) detector for bandwidth estimation. + +mod budget; +mod detector; + +pub use detector::AlrDetector; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/api.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/api.rs new file mode 100644 index 000000000..67b53df69 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/api.rs @@ -0,0 +1,57 @@ +//! Bandwidth estimation. + +use crate::{Rtc, rtp_::Mid}; + +pub use crate::rtp_::Bitrate; + +#[derive(Debug, PartialEq)] +#[non_exhaustive] +/// Bandwidth estimation kind. +pub enum BweKind { + /// Transport wide congestion control. + Twcc(Bitrate), + /// REMB (Receiver Estimated Maximum Bitrate) + Remb(Mid, Bitrate), +} + +/// Access to the Bandwidth Estimate subsystem. +pub struct Bwe<'a>(pub(crate) &'a mut Rtc); + +impl<'a> Bwe<'a> { + /// Configure the desired bitrate. + /// + /// Configure the bandwidth estimation system with the desired bitrate. + /// **Note:** This only has an effect if BWE has been enabled via + /// [`RtcConfig::enable_bwe`][crate::RtcConfig::enable_bwe]. + /// + /// * `desired_bitrate` The bitrate you would like to eventually send at. The BWE system will try + /// to reach this bitrate by probing with padding packets. You should allocate your media bitrate + /// based on the estimated the BWE system produces via + /// [`Event::EgressBitrateEstimate`][crate::Event::EgressBitrateEstimate]. This rate might not + /// be reached if the network link cannot sustain the desired bitrate. + /// + /// ## Example + /// + /// Say you have three simulcast video tracks each with a high layer configured at 1.5Mbit/s. + /// You should then set the desired bitrate to 4.5Mbit/s (or slightly higher). If the network + /// link can sustain 4.5Mbit/s there will eventually be an + /// [`Event::EgressBitrateEstimate`][crate::Event::EgressBitrateEstimate] with this estimate. + pub fn set_desired_bitrate(&mut self, desired_bitrate: Bitrate) { + self.0.session.set_bwe_desired_bitrate(desired_bitrate); + } + + /// Reset the BWE with a new init_bitrate + /// + /// # Example + /// + /// This method is useful when you initially start with only an audio stream. In this case, + /// the BWE will report a very low estimated bitrate. + /// Later, when you start a video stream, the estimated bitrate will be affected by the previous + /// low bitrate, resulting in a very low estimated bitrate, which can cause poor video quality. + /// To avoid this, you need to warm up the video stream for a while then calling reset with a + /// provided init_bitrate. + /// + pub fn reset(&mut self, init_bitrate: Bitrate) { + self.0.session.reset_bwe(init_bitrate); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/arrival_group.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/arrival_group.rs new file mode 100644 index 000000000..21c18129a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/arrival_group.rs @@ -0,0 +1,437 @@ +use std::mem; +use std::time::{Duration, Instant}; + +use crate::rtp_::TwccSeq; + +use super::super::AckedPacket; +use super::super::time::{TimeDelta, Timestamp}; + +const BURST_TIME_INTERVAL: Duration = Duration::from_millis(5); +const SEND_TIME_GROUP_LENGTH: Duration = Duration::from_millis(5); +const MAX_BURST_DURATION: Duration = Duration::from_millis(100); + +#[derive(Debug, Default)] +pub struct ArrivalGroup { + first: Option<(TwccSeq, Instant, Instant)>, + last_seq_no: Option, + last_local_send_time: Option, + last_remote_recv_time: Option, + size: usize, +} + +impl ArrivalGroup { + /// Maybe add a packet to the group. + /// + /// Returns [`true`] if a new group needs to be created and [`false`] otherwise. + fn add_packet(&mut self, packet: &AckedPacket) -> bool { + match self.belongs_to_group(packet) { + Belongs::NewGroup => return true, + Belongs::Skipped => return false, + Belongs::Yes => {} + } + + if self.first.is_none() { + self.first = Some(( + packet.seq_no, + packet.local_send_time, + packet.remote_recv_time, + )); + } + + self.last_remote_recv_time = self + .last_remote_recv_time + .max(Some(packet.remote_recv_time)); + self.last_local_send_time = self.last_local_send_time.max(Some(packet.local_send_time)); + self.size += 1; + self.last_seq_no = self.last_seq_no.max(Some(packet.seq_no)); + + false + } + + fn belongs_to_group(&self, packet: &AckedPacket) -> Belongs { + let Some((_, first_local_send_time, first_remote_recv_time)) = self.first else { + // Start of the group + return Belongs::Yes; + }; + + let Some(first_send_delta) = packet + .local_send_time + .checked_duration_since(first_local_send_time) + else { + // Out of order + return Belongs::Skipped; + }; + + let send_time_delta = Timestamp::from(packet.local_send_time) - self.local_send_time(); + if send_time_delta == TimeDelta::ZERO { + return Belongs::Yes; + } + let arrival_time_delta = Timestamp::from(packet.remote_recv_time) - self.remote_recv_time(); + + let propagation_delta = arrival_time_delta - send_time_delta; + if propagation_delta < TimeDelta::ZERO + && arrival_time_delta <= BURST_TIME_INTERVAL + && packet.remote_recv_time - first_remote_recv_time < MAX_BURST_DURATION + { + Belongs::Yes + } else if first_send_delta > SEND_TIME_GROUP_LENGTH { + Belongs::NewGroup + } else { + Belongs::Yes + } + } + + /// Calculate the send time delta between self and a subsequent group. + fn departure_delta(&self, other: &Self) -> TimeDelta { + Timestamp::from(other.local_send_time()) - self.local_send_time() + } + + /// Calculate the remote receive time delta between self and a subsequent group. + fn arrival_delta(&self, other: &Self) -> TimeDelta { + Timestamp::from(other.remote_recv_time()) - self.remote_recv_time() + } + + /// The local send time i.e. departure time, for the group. + /// + /// Panics if the group doesn't have at least one packet. + fn local_send_time(&self) -> Instant { + self.last_local_send_time + .expect("local_send_time to only be called on non-empty groups") + } + + /// The remote receive time i.e. arrival time, for the group. + /// + /// Panics if the group doesn't have at least one packet. + fn remote_recv_time(&self) -> Instant { + self.last_remote_recv_time + .expect("remote_recv_time to only be called on non-empty groups") + } +} + +/// Whether a given packet is belongs to a group or not. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Belongs { + /// The packet is belongs to the group. + Yes, + /// The packet is does not belong to the group, a new group should be created. + NewGroup, + /// The packet was skipped and a decision wasn't made. + Skipped, +} + +impl Belongs { + #[cfg(test)] + fn new_group(&self) -> bool { + matches!(self, Self::NewGroup) + } +} + +#[derive(Debug, Default)] +pub struct ArrivalGroupAccumulator { + previous_group: Option, + current_group: ArrivalGroup, +} + +impl ArrivalGroupAccumulator { + /// + /// Accumulate a packet. + /// + /// If adding this packet produced a new delay delta it is returned. + pub fn accumulate_packet(&mut self, packet: &AckedPacket) -> Option { + let need_new_group = self.current_group.add_packet(packet); + + if !need_new_group { + return None; + } + + // Variation between previous group and current. + let arrival_delta = self.arrival_delta(); + let send_delta = self.send_delta(); + let last_remote_recv_time = self.current_group.remote_recv_time(); + + let current_group = mem::take(&mut self.current_group); + self.previous_group = Some(current_group); + + self.current_group.add_packet(packet); + + Some(InterGroupDelayDelta { + send_delta: send_delta?, + arrival_delta: arrival_delta?, + last_remote_recv_time, + }) + } + + fn arrival_delta(&self) -> Option { + self.previous_group + .as_ref() + .map(|prev| prev.arrival_delta(&self.current_group)) + } + + fn send_delta(&self) -> Option { + self.previous_group + .as_ref() + .map(|prev| prev.departure_delta(&self.current_group)) + } +} + +/// The calculate delay delta between two groups of packets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InterGroupDelayDelta { + /// The delta between the send times of the two groups i.e. delta between the last packet sent + /// in each group. + pub send_delta: TimeDelta, + /// The delta between the remote arrival times of the two groups. + pub arrival_delta: TimeDelta, + /// The reported receive time for the last packet in the first arrival group. + pub last_remote_recv_time: Instant, +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use crate::rtp_::DataSize; + + use super::{AckedPacket, ArrivalGroup, ArrivalGroupAccumulator, Belongs, TimeDelta}; + + #[test] + fn test_arrival_group_all_packets_belong_to_empty_group() { + let now = Instant::now(); + let group = ArrivalGroup::default(); + + assert_eq!( + group.belongs_to_group(&AckedPacket { + seq_no: 1.into(), + size: DataSize::ZERO, + local_send_time: now, + remote_recv_time: now + duration_us(10), + local_recv_time: now + duration_us(12), + }), + Belongs::Yes, + "Any packet should belong to an empty arrival group" + ); + } + + #[test] + fn test_arrival_group_all_packets_sent_within_burst_interval_belong() { + let now = Instant::now(); + #[allow(clippy::vec_init_then_push)] + let packets = { + let mut packets = vec![]; + + packets.push(AckedPacket { + seq_no: 0.into(), + size: DataSize::ZERO, + local_send_time: now, + remote_recv_time: now + duration_us(150), + local_recv_time: now + duration_us(200), + }); + + packets.push(AckedPacket { + seq_no: 1.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(50), + remote_recv_time: now + duration_us(225), + local_recv_time: now + duration_us(275), + }); + + packets.push(AckedPacket { + seq_no: 2.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(1005), + remote_recv_time: now + duration_us(1140), + local_recv_time: now + duration_us(1190), + }); + + packets.push(AckedPacket { + seq_no: 3.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(4995), + remote_recv_time: now + duration_us(5001), + local_recv_time: now + duration_us(5051), + }); + + // Should not belong + packets.push(AckedPacket { + seq_no: 4.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(5700), + remote_recv_time: now + duration_us(6000), + local_recv_time: now + duration_us(5750), + }); + + packets + }; + + let mut group = ArrivalGroup::default(); + + for p in packets { + let need_new_group = group.belongs_to_group(&p).new_group(); + if !need_new_group { + group.add_packet(&p); + } + } + + assert_eq!(group.size, 4, "Expected group to contain 4 packets"); + } + + #[test] + fn test_arrival_group_out_order_arrival_ignored() { + let now = Instant::now(); + #[allow(clippy::vec_init_then_push)] + let packets = { + let mut packets = vec![]; + + packets.push(AckedPacket { + seq_no: 0.into(), + size: DataSize::ZERO, + local_send_time: now, + remote_recv_time: now + duration_us(150), + local_recv_time: now + duration_us(200), + }); + + packets.push(AckedPacket { + seq_no: 1.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(50), + remote_recv_time: now + duration_us(225), + local_recv_time: now + duration_us(275), + }); + + packets.push(AckedPacket { + seq_no: 2.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(1005), + remote_recv_time: now + duration_us(1140), + local_recv_time: now + duration_us(1190), + }); + + packets.push(AckedPacket { + seq_no: 3.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(4995), + remote_recv_time: now + duration_us(5001), + local_recv_time: now + duration_us(5051), + }); + + // Should be skipped + packets.push(AckedPacket { + seq_no: 4.into(), + size: DataSize::ZERO, + local_send_time: now - duration_us(100), + remote_recv_time: now + duration_us(5000), + local_recv_time: now + duration_us(5050), + }); + + // Should not belong + packets.push(AckedPacket { + seq_no: 5.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(5700), + remote_recv_time: now + duration_us(6000), + local_recv_time: now + duration_us(6050), + }); + + packets + }; + + let mut group = ArrivalGroup::default(); + + for p in packets { + let need_new_group = group.belongs_to_group(&p).new_group(); + if !need_new_group { + group.add_packet(&p); + } + } + + assert_eq!(group.size, 4, "Expected group to contain 4 packets"); + } + + #[test] + fn test_arrival_group_arrival_membership() { + let now = Instant::now(); + #[allow(clippy::vec_init_then_push)] + let packets = { + let mut packets = vec![]; + + packets.push(AckedPacket { + seq_no: 0.into(), + size: DataSize::ZERO, + local_send_time: now, + remote_recv_time: now + duration_us(150), + local_recv_time: now + duration_us(200), + }); + + packets.push(AckedPacket { + seq_no: 1.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(50), + remote_recv_time: now + duration_us(225), + local_recv_time: now + duration_us(275), + }); + + packets.push(AckedPacket { + seq_no: 2.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(5152), + // Just less than 5ms inter arrival delta + remote_recv_time: now + duration_us(5224), + local_recv_time: now + duration_us(5274), + }); + + // Should not belong + packets.push(AckedPacket { + seq_no: 3.into(), + size: DataSize::ZERO, + local_send_time: now + duration_us(5700), + remote_recv_time: now + duration_us(6000), + local_recv_time: now + duration_us(6050), + }); + + packets + }; + + let mut group = ArrivalGroup::default(); + + for p in packets { + let need_new_group = group.belongs_to_group(&p).new_group(); + if !need_new_group { + group.add_packet(&p); + } + } + + assert_eq!(group.size, 3, "Expected group to contain 4 packets"); + } + + #[test] + fn group_reorder() { + let data = vec![ + ((Duration::from_millis(0), Duration::from_millis(0)), None), + ((Duration::from_millis(60), Duration::from_millis(5)), None), + ((Duration::from_millis(40), Duration::from_millis(10)), None), + ( + (Duration::from_millis(70), Duration::from_millis(20)), + Some((TimeDelta::from_millis(-20), TimeDelta::from_millis(5))), + ), + ]; + + let now = Instant::now(); + let mut aga = ArrivalGroupAccumulator::default(); + + for ((local_send_time, remote_recv_time), deltas) in data { + let group_delta = aga.accumulate_packet(&AckedPacket { + seq_no: Default::default(), + size: Default::default(), + local_send_time: now + local_send_time, + remote_recv_time: now + remote_recv_time, + local_recv_time: Instant::now(), // does not matter + }); + + assert_eq!(group_delta.map(|d| (d.send_delta, d.arrival_delta)), deltas); + } + } + + fn duration_us(us: u64) -> Duration { + Duration::from_micros(us) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/control.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/control.rs new file mode 100644 index 000000000..a3d7db606 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/control.rs @@ -0,0 +1,226 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use super::super::macros::{log_bitrate_estimate, log_delay_variation}; +use super::super::{AckedPacket, BandwidthUsage}; +use super::arrival_group::ArrivalGroupAccumulator; +use super::rate_control::RateControl; +use super::trendline::TrendlineEstimator; +use crate::rtp_::Bitrate; +use crate::util::{MovingAverage, already_happened}; + +const MAX_RTT_HISTORY_WINDOW: usize = 32; +const UPDATE_INTERVAL: Duration = Duration::from_millis(25); +/// The maximum time we keep updating our estimate without receiving a TWCC report. +const MAX_TWCC_GAP: Duration = Duration::from_millis(500); +/// RFC 6298: Exponentially Weighted Moving Average smoothing factor for RTT (alpha = 1/8) +const RTT_SMOOTHING_FACTOR: f64 = 0.125; + +/// Delay controller for googcc inspired BWE. +/// +/// This controller attempts to estimate the available send bandwidth by looking at the variations +/// in packet arrival times for groups of packets sent together. Broadly, if the delay variation is +/// increasing this indicates overuse. +pub struct DelayController { + arrival_group_accumulator: ArrivalGroupAccumulator, + trendline_estimator: TrendlineEstimator, + rate_control: RateControl, + /// Last estimate produced, unlike [`next_estimate`] this will always have a value after the + /// first estimate. + last_estimate: Option, + /// Smoothed RTT using EWMA (RFC 6298, alpha = 1/8). + smoothed_rtt: MovingAverage, + /// History of the max RTT derived for each TWCC report (kept for fallback). + max_rtt_history: VecDeque, + + /// The next time we should poll. + next_timeout: Instant, + /// The last time we ingested a TWCC report. + last_twcc_report: Instant, +} + +impl DelayController { + pub fn new(initial_bitrate: Bitrate) -> Self { + Self { + arrival_group_accumulator: ArrivalGroupAccumulator::default(), + trendline_estimator: TrendlineEstimator::new(20), + rate_control: RateControl::new(initial_bitrate, Bitrate::kbps(40), Bitrate::gbps(10)), + last_estimate: Some(initial_bitrate), + smoothed_rtt: MovingAverage::new(RTT_SMOOTHING_FACTOR), + max_rtt_history: VecDeque::default(), + next_timeout: already_happened(), + last_twcc_report: already_happened(), + } + } + + /// Record a packet from a TWCC report. + pub fn update( + &mut self, + acked: &[AckedPacket], + acked_bitrate: Option, + probe_bitrate: Option, + now: Instant, + ) -> Option { + let mut max_rtt = None; + + for acked_packet in acked { + max_rtt = max_rtt.max(Some(acked_packet.rtt())); + if let Some(delay_variation) = self + .arrival_group_accumulator + .accumulate_packet(acked_packet) + { + log_delay_variation!(delay_variation.arrival_delta); + + // Got a new delay variation, add it to the trendline. + // + // IMPORTANT: Match WebRTC's TrendlineEstimator time base. + // WebRTC calls Detect/UpdateThreshold with `arrival_time_ms` (remote receive time), + // not the local "time we processed this feedback". Using the remote receive time + // avoids threshold adaptation artifacts when many deltas are processed in one + // feedback batch (e.g. TWCC reports). + // + // Note: We use remote timestamps for relative timing only (computing time deltas + // between packets). Clock skew doesn't matter since we're measuring trends in + // delay variations, not absolute times. + self.trendline_estimator + .add_delay_observation(delay_variation, delay_variation.last_remote_recv_time); + } + } + + if let Some(rtt) = max_rtt { + self.update_rtt(rtt); + } + + let new_hypothesis = self.trendline_estimator.hypothesis(); + + self.update_estimate( + new_hypothesis, + acked_bitrate, + probe_bitrate, + self.get_smoothed_rtt(), + now, + ); + self.last_twcc_report = now; + + self.last_estimate + } + + pub fn poll_timeout(&self) -> Instant { + self.next_timeout + } + + pub fn handle_timeout(&mut self, acked_bitrate: Option, now: Instant) { + if !self.trendline_hypothesis_valid(now) { + // We haven't received a TWCC report in a while. The trendline hypothesis can + // no longer be considered valid. We need another TWCC report before we can update + // estimates. + let next_timeout_in = self + .get_smoothed_rtt() + .unwrap_or(MAX_TWCC_GAP) + .min(UPDATE_INTERVAL); + + // Set this even if we didn't update, otherwise we get stuck in a poll -> handle loop + // that starves the run loop. + self.next_timeout = now + next_timeout_in; + return; + } + + self.update_estimate( + self.trendline_estimator.hypothesis(), + acked_bitrate, + None, + self.get_smoothed_rtt(), + now, + ); + } + + /// Get the latest estimate. + pub fn last_estimate(&self) -> Option { + self.last_estimate + } + + /// Whether the delay-based detector currently signals overuse. + /// + /// This is useful for gating behaviors (like probing) that would otherwise + /// re-excite the system while we're already congested. + pub fn is_overusing(&self) -> bool { + self.trendline_estimator.hypothesis() == BandwidthUsage::Overuse + } + + /// Update smoothed RTT using EWMA (RFC 6298, alpha = 1/8). + fn update_rtt(&mut self, rtt: Duration) { + // Keep history as fallback in case smoothed RTT is not yet available + while self.max_rtt_history.len() >= MAX_RTT_HISTORY_WINDOW { + self.max_rtt_history.pop_front(); + } + self.max_rtt_history.push_back(rtt); + + // Update smoothed RTT using EWMA: smoothed = (7/8) * smoothed + (1/8) * sample + self.smoothed_rtt.update(rtt.as_secs_f64()); + } + + /// Get the current smoothed RTT, with fallback to mean of history if not yet available. + fn get_smoothed_rtt(&self) -> Option { + // Try smoothed RTT first (EWMA) + if let Some(avg_secs) = self.smoothed_rtt.get() { + return Some(Duration::from_secs_f64(avg_secs)); + } + + // Fallback to mean of history during initialization + if self.max_rtt_history.is_empty() { + return None; + } + + let sum = self + .max_rtt_history + .iter() + .fold(Duration::ZERO, |acc, rtt| acc + *rtt); + Some(sum / self.max_rtt_history.len() as u32) + } + + fn update_estimate( + &mut self, + hypothesis: BandwidthUsage, + observed_bitrate: Option, + probe_bitrate: Option, + mean_max_rtt: Option, + now: Instant, + ) { + // WebRTC's logic from delay_based_bwe.cc MaybeUpdateEstimate(): + // - If we have a probe result, apply it directly and skip delay-based updates + // - Otherwise, apply normal delay-based rate control + // + // This prevents probe results from being immediately overridden by delay-based + // decreases caused by the probe itself (probes cause temporary queuing delay). + + if let Some(probe_rate) = probe_bitrate { + // Apply probe result directly, bypassing delay-based updates + self.rate_control.set_probe_result(probe_rate, now); + let estimated_rate = self.rate_control.estimated_bitrate(); + log_bitrate_estimate!(estimated_rate.as_f64()); + self.last_estimate = Some(estimated_rate); + } else if let Some(observed_bitrate) = observed_bitrate { + // No probe result, apply normal delay-based rate control + self.rate_control + .update(hypothesis.into(), observed_bitrate, mean_max_rtt, now); + let estimated_rate = self.rate_control.estimated_bitrate(); + + log_bitrate_estimate!(estimated_rate.as_f64()); + self.last_estimate = Some(estimated_rate); + } + + // Set this even if we didn't update, otherwise we get stuck in a poll -> handle loop + // that starves the run loop. + self.next_timeout = now + UPDATE_INTERVAL; + } + + /// Whether the current trendline hypothesis is valid i.e. not too old. + fn trendline_hypothesis_valid(&self, now: Instant) -> bool { + now.duration_since(self.last_twcc_report) + <= self + .get_smoothed_rtt() + .map(|rtt| rtt * 2) + .unwrap_or(MAX_TWCC_GAP) + .min(UPDATE_INTERVAL * 2) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/mod.rs new file mode 100644 index 000000000..b301f73c2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/mod.rs @@ -0,0 +1,6 @@ +mod arrival_group; +mod control; +mod rate_control; +mod trendline; + +pub use control::DelayController; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/rate_control.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/rate_control.rs new file mode 100644 index 000000000..a94d070e8 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/rate_control.rs @@ -0,0 +1,524 @@ +use std::fmt; +use std::time::{Duration, Instant}; + +use super::super::macros::log_rate_control_applied_change; +use super::super::macros::log_rate_control_observed_bitrate; +use super::super::macros::log_rate_control_state; +use crate::rtp_::Bitrate; +use crate::util::MovingAverage; + +use super::super::BandwidthUsage; + +// Recommended values from https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02#section-5 +/// Smoothing factor applied to moving stats for observed bitrates when we are in the decreasing +/// state. +const OBSERVED_BIT_RATE_SMOOTHING_FACTOR: f64 = 0.95; +/// The ratio of current estimated bandwidth to use when decreasing the rate. +const BETA: f64 = 0.85; +/// The coefficient used for multiplicative rate increase. +const MULTIPLICATIVE_INCREASE_COEF: f64 = 1.08; +/// The maximal ratio of the observed bitrate that we allow estimating in a single increase. +const MAX_ESTIMATE_RATIO: f64 = 1.5; +/// Default backoff time added to RTT for response time calculation (kDefaultBackoffTimeInMs in WebRTC). +const DEFAULT_BACKOFF_TIME: Duration = Duration::from_millis(100); +/// Number of standard deviations below mean to reset observed bitrate average. +const OBSERVED_BITRATE_RESET_THRESHOLD_STD: f64 = 3.0; + +/// A type used to estimates a suitable send bitrate. +/// +/// Inputs to the rate controller are: +/// * The observed received bitrate(via TWCC feddback). +/// * RTT. +/// * Congestion estimates from the delay controller. +pub struct RateControl { + state: State, + + estimated_bitrate: Bitrate, + min_bitrate: Bitrate, + max_bitrate: Bitrate, + + /// The last observed bitrate calculated based on TWCC data. + last_observed_bitrate: Option, + /// The averaged observed bitrate when we have been in the decrease state. + averaged_observed_bitrate: MovingAverage, + /// The last time we updated the estimated bitrate. + last_estimate_update: Option, + // Last RTT estimate in micro-seconds + last_rtt: Option, +} + +impl RateControl { + pub fn new(start_bitrate: Bitrate, min_bitrate: Bitrate, max_bitrate: Bitrate) -> Self { + log_rate_control_state!(State::Increase as i8); + + Self { + state: State::Increase, + + estimated_bitrate: start_bitrate, + min_bitrate, + max_bitrate, + + last_observed_bitrate: None, + averaged_observed_bitrate: MovingAverage::new(OBSERVED_BIT_RATE_SMOOTHING_FACTOR), + last_estimate_update: None, + last_rtt: None, + } + } + + /// Update with input from the delay controller. + pub fn update( + &mut self, + signal: Signal, + observed_bitrate: Bitrate, + rtt: Option, + now: Instant, + ) { + self.last_observed_bitrate = Some(observed_bitrate); + if let Some(rtt) = rtt { + self.last_rtt = Some(rtt); + } + + self.state = self.state.transition(signal); + log_rate_control_observed_bitrate!( + observed_bitrate.as_f64(), + self.averaged_observed_bitrate + .get() + .map(|avg| avg.to_string()) + .unwrap_or_default() + ); + + match self.state { + State::Increase => { + self.increase(observed_bitrate, now); + } + State::Decrease => { + // Maintain observed bitrate statistics while we are in the decrease state. + // + // This must NOT be gated by time_to_reduce_further. That function is + // intended to gate *applying another reduction*, not collecting throughput stats. + self.update_observed_bitrate(observed_bitrate); + + // Only apply decrease if enough time has passed since last bitrate change + // or if throughput is critically low (< 50% of estimate) + if self.time_to_reduce_further(now, observed_bitrate) { + self.decrease(observed_bitrate, now); + } + } + State::Hold => { + // Do nothing + } + } + } + + fn update_observed_bitrate(&mut self, observed_bitrate: Bitrate) { + if self + .averaged_observed_bitrate + .lower_range(OBSERVED_BITRATE_RESET_THRESHOLD_STD) + .map(|lower| observed_bitrate.as_f64() < lower) + .unwrap_or(false) + { + self.averaged_observed_bitrate.reset(); + } + self.averaged_observed_bitrate + .update(observed_bitrate.as_f64()); + } + + /// Check if it's time to reduce the bitrate further. + /// + /// This implements WebRTC's TimeToReduceFurther logic which prevents + /// rapid successive decreases, especially after probe results. + /// + /// Returns true if: + /// 1. Enough time (1 RTT, clamped to 10-200ms) has passed since last change, OR + /// 2. Throughput is critically low (< 50% of current estimate) + fn time_to_reduce_further(&self, now: Instant, observed_bitrate: Bitrate) -> bool { + let Some(last_change) = self.last_estimate_update else { + return true; // No previous change, allow decrease + }; + + // WebRTC uses: clamp(rtt, 10ms, 200ms) + let rtt = self.last_rtt.unwrap_or(DEFAULT_BACKOFF_TIME); + let reduction_interval = rtt.clamp(Duration::from_millis(10), Duration::from_millis(200)); + + let time_since_change = now.saturating_duration_since(last_change); + + if time_since_change >= reduction_interval { + return true; + } + + // If throughput is critically low (< 50% of estimate), allow immediate decrease + let threshold = self.estimated_bitrate * 0.5; + if observed_bitrate < threshold { + return true; + } + + false + } + + /// The current estimated bitrate. + pub fn estimated_bitrate(&self) -> Bitrate { + self.estimated_bitrate + } + + /// Set a probe result indicating discovered capacity. + /// + /// When a probe succeeds, it means the network can handle at least this bitrate. + /// We use this to quickly increase our estimate without waiting for gradual ramp-up. + /// + /// Apply a probe result directly to the estimate. + /// + /// This matches WebRTC's behavior where probe results are accepted unconditionally + /// (subject only to min/max clamping), regardless of whether they're higher or lower + /// than the current estimate. The timestamp is updated to prevent the next regular + /// update from immediately overriding the probe result. + /// + /// WebRTC does NOT change the rate control state when applying a probe - the state + /// remains unchanged to avoid triggering unintended AIMD behavior. + pub fn set_probe_result(&mut self, probe_bitrate: Bitrate, now: Instant) { + // WebRTC calls SetEstimate() directly without filtering by current estimate + // or changing the rate control state. Accept the probe result unconditionally + // (update_estimate handles clamping to min/max bounds). + self.update_estimate(probe_bitrate, now); + + // Do NOT change self.state - keep current state to match WebRTC behavior. + // Changing state here could trigger unintended AIMD decrease/increase logic. + } + + fn increase(&mut self, observed_bitrate: Bitrate, now: Instant) { + // WebRTC limits increases to 1.5x observed throughput to avoid unlimited growth + // when we're already above what we're actually sending + // See: aimd_rate_control.cc line 251-252 + let increase_limit = observed_bitrate * 1.5 + Bitrate::kbps(10); + + if self.estimated_bitrate >= increase_limit { + // WebRTC updates time_last_bitrate_change_ even when skipping increase + // (see aimd_rate_control.cc line 281, which is outside the increase check) + // This prevents stale timestamps from allowing premature decreases + self.last_estimate_update = Some(now); + return; + } + + // Initialize timestamp if this is the first increase call, otherwise use existing value. + // Note: In practice, last_estimate_update is always Some here because either: + // 1) We returned early above and set it, or + // 2) A previous call to decrease() or increase() already set it + let last_estimate_update = self.last_estimate_update.unwrap_or(now); + if self.last_estimate_update.is_none() { + self.last_estimate_update = Some(now); + } + + if self + .averaged_observed_bitrate + .upper_range(3.0) + .map(|upper| observed_bitrate.as_f64() > upper) + .unwrap_or(false) + { + self.averaged_observed_bitrate.reset(); + } + + let since_last_update = now - last_estimate_update; + assert!(since_last_update >= Duration::ZERO); + let near_convergence = self.is_near_convergence(); + + let mut new_estimate = if near_convergence { + // Additive increase + log_rate_control_applied_change!("increase_additive"); + let response_time = self.last_rtt.unwrap_or(Duration::ZERO) + DEFAULT_BACKOFF_TIME; + + let alpha = + 0.5 * (since_last_update.as_secs_f64() / response_time.as_secs_f64()).min(1.0); + let expected_packet_size = self.estimated_packet_size(); + self.estimated_bitrate.as_f64() + (alpha * expected_packet_size).max(1000.0) + } else { + // Multiplicative increase + log_rate_control_applied_change!("increase_multiplicative"); + let eta = MULTIPLICATIVE_INCREASE_COEF.powf(since_last_update.as_secs_f64().min(1.0)); + let increase = ((eta - 1.0) * self.estimated_bitrate.as_f64()).max(1_000.0); + + self.estimated_bitrate.as_f64() + increase + }; + + // Cap at the increase limit (and observed bitrate ratio) + let max = observed_bitrate.as_f64() * MAX_ESTIMATE_RATIO; + new_estimate = max.min(new_estimate).min(increase_limit.as_f64()); + + self.update_estimate(new_estimate.into(), now); + } + + fn decrease(&mut self, observed_bitrate: Bitrate, now: Instant) { + log_rate_control_applied_change!("decrease"); + let mut new_estimate = observed_bitrate * BETA; + + if self.estimated_bitrate < new_estimate { + // Avoid increasing the bitrate on overuse + new_estimate = self.estimated_bitrate; + } + + #[allow(unused)] + if let Some(observed_average) = self.averaged_observed_bitrate.get() { + log_rate_control_observed_bitrate!( + observed_bitrate.as_u64(), + observed_average.round() as u64 + ); + } + // According to https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02#section-6 we + // should wait until this happens as consequence of the delay control, but libWebRTC does + // it immediately. + self.state = State::Hold; + log_rate_control_state!(self.state as i8); + self.update_estimate(new_estimate, now); + } + + fn is_near_convergence(&self) -> bool { + // Not near convergence until we have valid statistics + if !self.averaged_observed_bitrate.valid() { + return false; + } + let Some(last_observed_bitrate) = self.last_observed_bitrate else { + return false; + }; + + // Near convergence if the observed bandwidth is within 3 standard deviations of + // the moving average when we have been in the decrease state. + self.averaged_observed_bitrate + .within_std(last_observed_bitrate.as_f64(), 3.0) + } + + fn update_estimate(&mut self, bitrate: Bitrate, now: Instant) { + self.estimated_bitrate = bitrate.clamp(self.min_bitrate, self.max_bitrate); + self.last_estimate_update = Some(now); + } + + fn estimated_packet_size(&self) -> f64 { + // Assume 30 FPS video dominates the send rate + let bits_per_frame = self.estimated_bitrate.as_f64() / 30.0; + let packets_per_frame = (bits_per_frame / (1200.0 / 8.0)).ceil(); + + bits_per_frame / packets_per_frame + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Signal { + Overuse, + Underuse, + Normal, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +enum State { + Increase = 1, + Hold = 0, + #[default] + Decrease = -1, +} + +impl State { + fn transition(&self, signal: Signal) -> Self { + let new_state = match (self, signal) { + (_, Signal::Overuse) => Self::Decrease, + (_, Signal::Underuse) => Self::Hold, + // https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02#section-6 says to + // transition to Hold here, but libWebRTC stays in decrease. We will eventually + // transition to Hold on Underuse. + (Self::Decrease, Signal::Normal) => Self::Hold, + (Self::Hold | Self::Increase, Signal::Normal) => Self::Increase, + }; + + if new_state != *self { + log_rate_control_state!(new_state as i8); + } + + new_state + } +} + +impl From for Signal { + fn from(value: BandwidthUsage) -> Self { + match value { + BandwidthUsage::Overuse => Signal::Overuse, + BandwidthUsage::Normal => Signal::Normal, + BandwidthUsage::Underuse => Signal::Underuse, + } + } +} + +impl fmt::Display for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + State::Hold => write!(f, "hold"), + State::Increase => write!(f, "increase"), + State::Decrease => write!(f, "decrease"), + } + } +} + +impl fmt::Display for Signal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Signal::Overuse => write!(f, "overuse"), + Signal::Underuse => write!(f, "underuse"), + Signal::Normal => write!(f, "normal"), + } + } +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use super::{RateControl, Signal, State}; + + mod state { + use super::{Signal, State}; + + #[test] + fn test_state_transitions() { + // Tests based on the table in + // https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02#section-5 + + // Hold + let hold = State::Hold; + assert_eq!(hold.transition(Signal::Overuse), State::Decrease); + assert_eq!(hold.transition(Signal::Normal), State::Increase); + assert_eq!(hold.transition(Signal::Underuse), State::Hold); + + // Increase + let increase = State::Increase; + assert_eq!(increase.transition(Signal::Overuse), State::Decrease); + assert_eq!(increase.transition(Signal::Normal), State::Increase); + assert_eq!(increase.transition(Signal::Underuse), State::Hold); + + // Decrease + let decrease = State::Decrease; + assert_eq!(decrease.transition(Signal::Overuse), State::Decrease); + assert_eq!(decrease.transition(Signal::Normal), State::Hold); + assert_eq!(decrease.transition(Signal::Underuse), State::Hold); + } + } + + mod rate_controller { + use std::time::Instant; + + use super::{RateControl, Signal, duration_ms}; + + fn make_control(estimated_bitrate: u64) -> RateControl { + RateControl::new(estimated_bitrate.into(), 10_000.into(), 50_000_000.into()) + } + + #[test] + fn test_initial_estimate() { + let rate_controller = make_control(100_000); + + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 100_000); + } + + #[test] + fn test_normal_yields_multiplicative_increase() { + let now = Instant::now(); + let mut rate_controller = make_control(100_000); + // Seed last estimate value + rate_controller.update(Signal::Normal, 85_000.into(), None, now); + assert_eq!( + rate_controller.estimated_bitrate().as_u64(), + 101_000, + "Initial estimate should increase by the minimum(1Kbit/s)" + ); + + rate_controller.update(Signal::Normal, 95_000.into(), None, now + duration_ms(500)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 104_963); + + rate_controller.update(Signal::Normal, 97_000.into(), None, now + duration_ms(1000)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 109_081); + } + + #[test] + fn test_normal_to_under_use_yields_hold() { + let now = Instant::now(); + let mut rate_controller = make_control(100_000); + // Seed last estimate value + rate_controller.update(Signal::Normal, 85_000.into(), None, now); + assert_eq!( + rate_controller.estimated_bitrate().as_u64(), + 101_000, + "Initial estimate should increase by the minimum(1Kbit/s)" + ); + + // Should remain in increase and increase estimate + rate_controller.update(Signal::Normal, 95_000.into(), None, now + duration_ms(500)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 104_963); + + // Should transition to hold + rate_controller.update( + Signal::Underuse, + 97_000.into(), + None, + now + duration_ms(1000), + ); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 104_963); + + // Should remain in hold and not modify estimates + rate_controller.update( + Signal::Underuse, + 97_000.into(), + None, + now + duration_ms(2000), + ); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 104_963); + } + + #[test] + fn test_immediate_overuse() { + let now = Instant::now(); + let mut rate_controller = make_control(100_000); + // Seed last estimate value + rate_controller.update(Signal::Normal, 85_000.into(), None, now); + + rate_controller.update(Signal::Overuse, 90_000.into(), None, now + duration_ms(500)); + assert_eq!( + rate_controller.estimated_bitrate().as_u64(), + 76_500, + "When overuse is detected we should reduce the estimate to \ + 85% of the obeserved rate immediately" + ); + } + + #[test] + fn test_immediate_overuse_then_stable() { + let now = Instant::now(); + let mut rate_controller = make_control(100_000); + // Seed last estimate value + rate_controller.update(Signal::Normal, 85_000.into(), Some(duration_ms(80)), now); + + rate_controller.update(Signal::Overuse, 90_000.into(), None, now + duration_ms(500)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 76_500); + + rate_controller.update( + Signal::Overuse, + 75_000.into(), + None, + now + duration_ms(1000), + ); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 63_750); + + rate_controller.update(Signal::Normal, 60_000.into(), None, now + duration_ms(1500)); + // NB: This matches libWebRTC but diverges from the spec + assert_eq!( + rate_controller.estimated_bitrate().as_u64(), + 66_251, + "After adjusting on overuse we immediately return to increase on the next normal signal" + ); + + rate_controller.update(Signal::Normal, 60_000.into(), None, now + duration_ms(2500)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 71_552,); + + // NB: Additive increase because we are nearing convergence + rate_controller.update(Signal::Normal, 70_000.into(), None, now + duration_ms(3500)); + assert_eq!(rate_controller.estimated_bitrate().as_u64(), 72552); + } + } + + fn duration_ms(ms: u64) -> Duration { + Duration::from_millis(ms) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/trendline.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/trendline.rs new file mode 100644 index 000000000..cf9cc57e4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/delay/trendline.rs @@ -0,0 +1,454 @@ +use std::collections::VecDeque; +use std::ops::RangeInclusive; +use std::time::{Duration, Instant}; + +use super::super::macros::{log_trendline_estimate, log_trendline_modified_trend}; + +use super::super::BandwidthUsage; +use super::super::time::{TimeDelta, Timestamp}; +use super::arrival_group::InterGroupDelayDelta; + +const SMOOTHING_COEF: f64 = 0.9; +const OVER_USE_THRESHOLD_DEFAULT_MS: f64 = 12.5; +const OVER_USE_TIME_THRESHOLD: Duration = Duration::from_millis(10); +const MAX_ADOPT_OFFSET_MS: f64 = 15.0; +const THRESHOLD_GAIN: f64 = 4.0; + +const K_UP: f64 = 0.0087; +const K_DOWN: f64 = 0.039; + +const DELAY_COUNT_RANGE: RangeInclusive = 60..=1000; + +pub struct TrendlineEstimator { + /// The window size in packets + window_size: usize, + + /// The first instant we saw, used as zero point. + zero_time: Option, + + /// The history of observed delay variations. + history: VecDeque, + + /// The total number of observed delay variations. + num_delay_variations: usize, + + /// Accumulated delay. + accumulated_delay: f64, + + /// Last smoothed delay. + smoothed_delay: f64, + + /// The adaptive delay threshold. + delay_threshold: f64, + + /// Previous trend + previous_trend: f64, + + /// If we are overusing, this contains data about the overuse. + overuse: Option, + + /// The last time we updated the adaptive threshold. + last_threshold_update: Option, + + /// Our current hypothesis about the bandwidth usage. + hypothesis: BandwidthUsage, +} + +impl TrendlineEstimator { + pub fn new(window_size: usize) -> Self { + Self { + window_size, + zero_time: None, + history: VecDeque::default(), + num_delay_variations: 0, + accumulated_delay: 0.0, + smoothed_delay: 0.0, + delay_threshold: OVER_USE_THRESHOLD_DEFAULT_MS, + previous_trend: 0.0, + overuse: None, + last_threshold_update: None, + hypothesis: BandwidthUsage::Normal, + } + } + + pub fn add_delay_observation(&mut self, delay_variation: InterGroupDelayDelta, now: Instant) { + if self.history.is_empty() { + self.do_add_to_history(delay_variation, now); + return; + } + + self.do_add_to_history(delay_variation, now); + while self.history.len() > self.window_size { + let _ = self.history.pop_front(); + } + + if self.history.len() == self.window_size { + assert!( + self.history + .iter() + .zip(self.history.iter().skip(1)) + .fold(true, |acc, (a, b)| { + acc && a.remote_recv_time_ms <= b.remote_recv_time_ms + }), + "Out of order history {:?}", + self.history + ); + self.update_trendline(delay_variation, now); + } + } + + pub fn hypothesis(&self) -> BandwidthUsage { + self.hypothesis + } + + fn do_add_to_history(&mut self, variation: InterGroupDelayDelta, now: Instant) { + let last_remote_recv_time = Timestamp::from(variation.last_remote_recv_time); + + let zero_time = *self.zero_time.get_or_insert(last_remote_recv_time); + + let delay_delta = variation.arrival_delta.as_secs_f64() * 1000.0 + - variation.send_delta.as_secs_f64() * 1000.0; + + self.num_delay_variations += 1; + self.num_delay_variations = self.num_delay_variations.min(*DELAY_COUNT_RANGE.end()); + self.accumulated_delay += delay_delta; + self.smoothed_delay = + self.smoothed_delay * SMOOTHING_COEF + (1.0 - SMOOTHING_COEF) * self.accumulated_delay; + + let remote_recv_time = last_remote_recv_time - zero_time; + let timing = Timing { + at: now, + remote_recv_time_ms: remote_recv_time.as_secs_f64() * 1000.0, + smoothed_delay_ms: self.smoothed_delay, + }; + + let pos = self + .history + .iter() + .rev() + .position(|p| p.remote_recv_time_ms <= timing.remote_recv_time_ms) + .unwrap_or(self.history.len()); + + // we expect pos to be 0 more often than not + self.history.insert(self.history.len() - pos, timing); + } + + fn update_trendline(&mut self, variation: InterGroupDelayDelta, now: Instant) -> Option<()> { + let trend = self.linear_fit().unwrap_or(self.previous_trend); + trace!("Computed trend {:?}", trend); + log_trendline_estimate!(trend); + + self.detect(trend, variation, now); + + Some(()) + } + + fn linear_fit(&self) -> Option { + // Simple linear regression to compute slope. + assert!(self.history.len() > 2); + + let (sum_x, sum_y) = self.history.iter().fold((0.0, 0.0), |acc, t| { + (acc.0 + t.remote_recv_time_ms, acc.1 + t.smoothed_delay_ms) + }); + + let avg_x = sum_x / self.history.len() as f64; + let avg_y = sum_y / self.history.len() as f64; + + let (numerator, denomenator) = self.history.iter().fold((0.0, 0.0), |acc, t| { + let x = t.remote_recv_time_ms; + let y = t.smoothed_delay_ms; + + let numerator = acc.0 + (x - avg_x) * (y - avg_y); + let denomenator = acc.1 + (x - avg_x).powi(2); + + (numerator, denomenator) + }); + + if denomenator == 0.0 { + return None; + } + + Some(numerator / denomenator) + } + + fn detect(&mut self, trend: f64, variation: InterGroupDelayDelta, now: Instant) { + if self.num_delay_variations < 2 { + self.update_hypothesis(BandwidthUsage::Normal); + return; + } + + let modified_trend = self.num_delay_variations.min(*DELAY_COUNT_RANGE.start()) as f64 + * trend + * THRESHOLD_GAIN; + + log_trendline_modified_trend!(modified_trend, self.delay_threshold); + + if modified_trend > self.delay_threshold { + let overuse = match &mut self.overuse { + Some(o) => { + o.time_overusing += variation.send_delta; + o + } + None => { + let new_overuse = Overuse { + count: 0, + // Initialize the timer. Assume that we've been + // over-using half of the time since the previous + // sample. + time_overusing: variation.send_delta / 2, + }; + self.overuse = Some(new_overuse); + + self.overuse.as_mut().unwrap() + } + }; + + overuse.count += 1; + trace!( + timeoverusing = ?overuse.time_overusing, + trend, + previous_trend = self.previous_trend, + "Trendline Estimator: Maybe overusing" + ); + + if overuse.time_overusing > OVER_USE_TIME_THRESHOLD + && overuse.count > 1 + && trend > self.previous_trend + { + self.overuse = None; + + self.update_hypothesis(BandwidthUsage::Overuse); + } + } else if modified_trend < -self.delay_threshold { + self.overuse = None; + self.update_hypothesis(BandwidthUsage::Underuse); + } else { + self.overuse = None; + self.update_hypothesis(BandwidthUsage::Normal); + } + + self.previous_trend = trend; + self.update_threshold(modified_trend, now); + } + + fn update_threshold(&mut self, modified_trend: f64, now: Instant) { + if self.last_threshold_update.is_none() { + self.last_threshold_update = Some(now); + } + let abs_modified_trend = modified_trend.abs(); + + if abs_modified_trend > self.delay_threshold + MAX_ADOPT_OFFSET_MS { + // Avoid adapting the threshold to big latency spikes, caused e.g., + // by a sudden capacity drop. + self.last_threshold_update = Some(now); + return; + } + + let k = if abs_modified_trend < self.delay_threshold { + K_DOWN + } else { + K_UP + }; + let time_delta = now + .saturating_duration_since( + self.last_threshold_update + .expect("last_threshold_update must have been set"), + ) + .as_millis() as f64; + self.delay_threshold += + k * (abs_modified_trend - self.delay_threshold) * time_delta.min(100.0); + self.last_threshold_update = Some(now); + self.delay_threshold = self.delay_threshold.clamp(6.0, 600.0); + + trace!( + "Adaptive delay variation threshold changed to: {}", + self.delay_threshold + ); + } + + fn update_hypothesis(&mut self, new_hypothesis: BandwidthUsage) { + if self.hypothesis == new_hypothesis { + return; + } + + trace!("TrendLineEstimator: Setting hypothesis to {new_hypothesis}"); + self.hypothesis = new_hypothesis; + } +} + +#[derive(Debug)] +struct Timing { + #[allow(unused)] + at: Instant, + remote_recv_time_ms: f64, + smoothed_delay_ms: f64, +} + +struct Overuse { + count: usize, + time_overusing: TimeDelta, +} + +#[cfg(test)] +mod test { + use super::*; + use std::time::{Duration, Instant}; + + #[test] + fn test_window_size_limit() { + let now = Instant::now(); + let remote_recv_time_base = Instant::now(); + let mut estimator = TrendlineEstimator::new(20); + + estimator.add_delay_observation( + delay_variation(duration_ms(1), duration_ms(1), remote_recv_time_base), + now, + ); + + for _ in 0..25 { + estimator.add_delay_observation( + delay_variation( + duration_ms(11), + duration_ms(1), + remote_recv_time_base + duration_ms(350), + ), + now + duration_ms(500), + ); + } + + assert_eq!(estimator.history.len(), 20); + } + + #[test] + fn test_overuse() { + let now = Instant::now(); + let remote_recv_time_base = Instant::now(); + let mut estimator = TrendlineEstimator::new(20); + + for g in 0..5 { + for i in 0..5 { + estimator.add_delay_observation( + delay_variation( + duration_ms(1), + duration_ms(1), + remote_recv_time_base + Duration::from_micros(5_000 * g + i * 40), + ), + now + duration_ms(g * 100), + ); + } + } + + assert_eq!(estimator.hypothesis(), BandwidthUsage::Normal); + assert_eq!(estimator.history.len(), 20); + + estimator.add_delay_observation( + delay_variation( + duration_ms(17), + duration_ms(5), + remote_recv_time_base + Duration::from_micros(25_000), + ), + now + duration_ms(600), + ); + assert_eq!( + estimator.hypothesis(), + BandwidthUsage::Normal, + "After getting an initial increasing delay the hypothesis should remain at normal" + ); + + estimator.add_delay_observation( + delay_variation( + duration_ms(18), + duration_ms(5), + remote_recv_time_base + Duration::from_micros(25_140), + ), + now + duration_ms(600), + ); + assert_eq!( + estimator.hypothesis(), + BandwidthUsage::Normal, + "After getting an a second increasing delay the hypothesis should remain at normal \ + because we the time overusing threshold hasn't been reached yet" + ); + + estimator.add_delay_observation( + delay_variation( + duration_ms(22), + duration_ms(8), + remote_recv_time_base + Duration::from_micros(25_250), + ), + now + duration_ms(600), + ); + assert_eq!( + estimator.hypothesis(), + BandwidthUsage::Overuse, + "After getting a third increasing delay the hypothesis should move to over because \ + we have been overusing for more than 10ms" + ); + } + + fn duration_ms(ms: u64) -> Duration { + Duration::from_millis(ms) + } + + fn delay_variation( + recv_delta: Duration, + send_delta: Duration, + last_remote_recv_time: Instant, + ) -> InterGroupDelayDelta { + InterGroupDelayDelta { + send_delta: send_delta.into(), + arrival_delta: recv_delta.into(), + last_remote_recv_time, + } + } + + #[test] + fn test_history_insertion_regression_out_of_order() { + // Test for algesten/str0m#698 + // + // This test reproduces the scenario where a sample with a remote receive time + // earlier than all existing history entries is added. + // Prior to the fix, this appended the smallest (negative) element to the end + // causing the monotonicity assert to fail when the window is full. + + let now = Instant::now(); + let zero_time_base = Instant::now(); + let mut estimator = TrendlineEstimator::new(8); + + // Add a few normal monotonically increasing values + for offset_ms in [100, 200, 300, 400] { + estimator.add_delay_observation( + delay_variation( + duration_ms(10), + duration_ms(1), + zero_time_base + duration_ms(offset_ms), + ), + now, + ); + } + + // Add some out-of-order samples that are earlier than the zero_time + // (packet reordering is fun). + for offset_ms in [50, 100, 25, 300] { + estimator.add_delay_observation( + delay_variation( + duration_ms(10), + duration_ms(1), + zero_time_base - duration_ms(offset_ms), + ), + now, + ); + } + + // History should remain sorted. + let ordered = estimator + .history + .iter() + .zip(estimator.history.iter().skip(1)) + .all(|(a, b)| a.remote_recv_time_ms <= b.remote_recv_time_ms); + assert!( + ordered, + "history should remain sorted by remote_recv_time_ms" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/link_capacity_estimator.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/link_capacity_estimator.rs new file mode 100644 index 000000000..3544b6253 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/link_capacity_estimator.rs @@ -0,0 +1,181 @@ +use std::time::{Duration, Instant}; + +use crate::rtp_::Bitrate; + +/// Estimates link capacity based on successful probe results during ALR. +/// +/// This tracks the link's proven capacity from probes sent when the application +/// is in Application Limited Region (ALR), meaning the app is sending less than +/// network capacity. During ALR, probes can discover the true available bandwidth +/// without being constrained by application send rate. +/// +/// The estimator only accepts probe results obtained while in ALR, as these +/// represent genuine capacity measurements. Probes during non-ALR periods +/// may be artificially limited by application sending patterns. +/// +/// Capacity estimates decay over time (default 60s) since network conditions +/// can change, and old measurements become less reliable. +#[derive(Default)] +pub struct LinkCapacityEstimator { + /// Current estimate of link capacity, if available + capacity_estimate: Option, + + /// Time when the capacity estimate was last updated + last_estimate_time: Option, +} + +impl LinkCapacityEstimator { + /// Default duration before capacity estimate resets (60 seconds) + const DEFAULT_RESET_WINDOW: Duration = Duration::from_secs(60); + + /// Create a new LinkCapacityEstimator with default settings + pub fn new() -> Self { + Self::default() + } + + /// Update the capacity estimate from a successful probe result. + /// + /// This should only be called for probes that were sent during ALR. + /// The estimate is stored along with the current time for decay tracking. + pub fn update_from_probe(&mut self, probe_estimate: Bitrate, now: Instant) { + // Only accept valid probe estimates + if !probe_estimate.is_valid() { + return; + } + + // Update or set the capacity estimate + // If we already have an estimate, take the max (capacity shouldn't decrease + // from successful probes, only from decay/timeout) + let current = self.capacity_estimate.get_or_insert(probe_estimate); + *current = (*current).max(probe_estimate); + + self.last_estimate_time = Some(now); + + trace!( + "Link capacity estimate updated to {} from probe", + probe_estimate + ); + } + + /// Get the current capacity estimate, if available and not expired. + pub fn capacity_estimate(&self, now: Instant) -> Option { + let estimate = self.capacity_estimate?; + let last_time = self.last_estimate_time?; + + // Check if estimate has expired (defensive against clock skew) + if now.saturating_duration_since(last_time) > Self::DEFAULT_RESET_WINDOW { + trace!("Link capacity estimate expired"); + return None; + } + + Some(estimate) + } + + /// Reset the capacity estimate. + #[cfg(test)] + pub fn reset(&mut self) { + if self.capacity_estimate.is_some() { + trace!("Link capacity estimate reset"); + } + self.capacity_estimate = None; + self.last_estimate_time = None; + } + + /// Check if we currently have a valid capacity estimate + #[cfg(test)] + pub fn has_estimate(&self) -> bool { + self.capacity_estimate.is_some() && self.last_estimate_time.is_some() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn starts_with_no_estimate() { + let estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + + assert_eq!(estimator.capacity_estimate(now), None); + assert!(!estimator.has_estimate()); + } + + #[test] + fn stores_probe_result() { + let mut estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + let probe_result = Bitrate::mbps(10); + + estimator.update_from_probe(probe_result, now); + + assert_eq!(estimator.capacity_estimate(now), Some(probe_result)); + assert!(estimator.has_estimate()); + } + + #[test] + fn takes_maximum_of_multiple_probes() { + let mut estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + + estimator.update_from_probe(Bitrate::mbps(10), now); + estimator.update_from_probe(Bitrate::mbps(5), now); + + // Should keep the higher estimate + assert_eq!(estimator.capacity_estimate(now), Some(Bitrate::mbps(10))); + + estimator.update_from_probe(Bitrate::mbps(15), now); + + // Should update to higher estimate + assert_eq!(estimator.capacity_estimate(now), Some(Bitrate::mbps(15))); + } + + #[test] + fn estimate_expires_after_reset_window() { + let mut estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + + estimator.update_from_probe(Bitrate::mbps(10), now); + assert_eq!(estimator.capacity_estimate(now), Some(Bitrate::mbps(10))); + + // Check just before expiration + let almost_expired = now + Duration::from_secs(59); + assert_eq!( + estimator.capacity_estimate(almost_expired), + Some(Bitrate::mbps(10)) + ); + + // Check after expiration + let expired = now + Duration::from_secs(61); + assert_eq!(estimator.capacity_estimate(expired), None); + } + + #[test] + fn reset_clears_estimate() { + let mut estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + + estimator.update_from_probe(Bitrate::mbps(10), now); + assert!(estimator.has_estimate()); + + estimator.reset(); + + assert!(!estimator.has_estimate()); + assert_eq!(estimator.capacity_estimate(now), None); + } + + #[test] + fn ignores_invalid_probes() { + let mut estimator = LinkCapacityEstimator::new(); + let now = Instant::now(); + + // Set a valid estimate first + estimator.update_from_probe(Bitrate::mbps(10), now); + + // Try to update with invalid bitrate + estimator.update_from_probe(Bitrate::NEG_INFINITY, now); + + // Should keep the valid estimate + assert_eq!(estimator.capacity_estimate(now), Some(Bitrate::mbps(10))); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/loss_controller.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/loss_controller.rs new file mode 100644 index 000000000..b0b3a7c62 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/loss_controller.rs @@ -0,0 +1,1551 @@ +use std::cmp::max; +use std::cmp::min; +use std::time::{Duration, Instant}; + +use super::macros::log_inherent_loss; +use super::macros::log_loss_based_bitrate_estimate; +use super::macros::log_loss_bw_limit_in_window; +use crate::rtp_::TwccSendRecord; +use crate::{Bitrate, DataSize}; + +use super::time::{TimeDelta, Timestamp}; + +/// Loss controller based on libWebRTC's `LossBasedBweV2`. +/// +/// ## Overview +/// +/// The estimator attempts to estimate the inherent loss of the link using Maximum Likelihood +/// Estimation of an assumed Bernoulli distribution. This allows it to distinguish congestion +/// induced loss from this inherent loss. +/// +/// The controller integrates with ALR (Application Limited Region) detection and link capacity +/// tracking. When in ALR, it uses proven link capacity from successful ALR probes as an upper +/// bound on estimates, preventing overestimation in application-limited scenarios. When +/// transitioning into or out of ALR, the controller resets its observation window to avoid +/// mixing traffic patterns from different network utilization regimes. +/// +/// The estimate is bounded by the output of the delay-based estimator, meaning this controller +/// can only reduce estimates (acting as a safety cap), not increase them. +/// +/// +/// Ref: +/// * https://webrtc.googlesource.com/src/+/refs/heads/main/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc +/// * https://webrtc.googlesource.com/src/+/refs/heads/main/modules/congestion_controller/goog_cc/loss_based_bwe_v2.h +pub struct LossController { + /// Configuration for the controller. + config: Config, + + /// The current state of the controller. + state: LossControllerState, + + /// Staging ground for observations while they are being constructed. + partial_observation: PartialObservation, + + /// The last packet sent in the most recent observation. + last_send_time_most_recent_observation: Timestamp, + + // Observation window + /// Forever growing counter of observations. Observation::id derives from this. + num_observations: u64, + /// Window of observations. + observations: Box<[Observation]>, + /// Temporal weights, used to weight observations by recency. Same size as `observations`. + temporal_weights: Box<[f64]>, + /// Upper bound temporal weights, used to weight observations by recency. Same size as `observations`. + instant_upper_bound_temporal_weights: Box<[f64]>, + + /// Precomputed instantaneous upper bound on bandwidth estimate. + cached_instant_upper_bound: Option, + /// Last time we reduced the estimate. + last_time_estimate_reduced: Timestamp, + + /// When we started recovering after being loss limited last time. + /// While in this window the bandwidth estimate is bounded by `bandwidth_limit_in_current_window`. + recovering_after_loss_timestamp: Timestamp, + /// Upper bound on estimate while in recovery window. + bandwidth_limit_in_current_window: Bitrate, + + /// The current estimate + current_estimate: ChannelParameters, + + /// The min bitrate we will emit as an estimate. + min_bitrate: Bitrate, + /// The max bitrate we will emit as an estimate. + max_bitrate: Bitrate, + + /// The most recent acknowledged bitrate derived from TWCC. + acknowledged_bitrate: Bitrate, + + /// The most recent estimated bitrate from the delay based estimator. + delay_based_estimate: Bitrate, + + /// HOLD mechanism state - prevents immediate ramp-up after loss + last_hold_info: HoldInfo, + + /// ALR start time (None if not in ALR) + alr_start_time: Option, + + /// Link capacity estimate from probes during ALR + link_capacity_estimate: Option, + + /// Previous ALR state to detect transitions + was_in_alr: bool, +} + +/// State of the Loss Controller +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum LossControllerState { + /// LossController is in increasing state + Increasing, + /// LossController is in decreasing state + Decreasing, + /// LossController is in relaying the estimate of the delay controller + DelayBased, +} + +pub trait PacketResult { + /// When the packet was sent + fn local_send_time(&self) -> Instant; + /// Size of the packet payload + fn size(&self) -> DataSize; + + /// Whether this packet was lost or not. + fn lost(&self) -> bool; +} + +impl LossController { + pub fn new() -> LossController { + let config = Config::default(); + + let mut controller = LossController { + state: LossControllerState::DelayBased, + partial_observation: PartialObservation::new(), + last_send_time_most_recent_observation: Timestamp::DistantFuture, + observations: vec![Observation::DUMMY; config.observation_window_size] + .into_boxed_slice(), + num_observations: 0, + temporal_weights: vec![0_f64; config.observation_window_size].into_boxed_slice(), + instant_upper_bound_temporal_weights: vec![0_f64; config.observation_window_size] + .into_boxed_slice(), + cached_instant_upper_bound: None, + last_time_estimate_reduced: Timestamp::DistantPast, + recovering_after_loss_timestamp: Timestamp::DistantPast, + bandwidth_limit_in_current_window: Bitrate::MAX, + + current_estimate: ChannelParameters::new(config.initial_inherent_loss_estimate), + + min_bitrate: Bitrate::kbps(1), + max_bitrate: Bitrate::INFINITY, + + // review usage from here on after + acknowledged_bitrate: Bitrate::INFINITY, + delay_based_estimate: Bitrate::INFINITY, + + last_hold_info: HoldInfo::default(), + + alr_start_time: None, + link_capacity_estimate: None, + was_in_alr: false, + + config, + }; + + // Initialize weights + { + let this = &mut controller; + for i in 0..this.config.observation_window_size { + let val = f64::powi(this.config.temporal_weight_factor, i as i32); + this.temporal_weights[i] = val; + let val = f64::powi( + this.config.instant_upper_bound_temporal_weight_factor, + i as i32, + ); + this.instant_upper_bound_temporal_weights[i] = val; + } + }; + + controller + } + + /// Override the current bandwidth estimate. + pub fn set_bandwidth_estimate(&mut self, bandwidth_estimate: Bitrate) { + self.current_estimate.loss_limited_bandwidth = bandwidth_estimate; + } + + /// Update the acknowledged bitrate based on TWCC feedback. + pub fn set_acknowledged_bitrate(&mut self, acknowledged_bitrate: Bitrate) { + self.acknowledged_bitrate = acknowledged_bitrate; + } + + /// Set ALR start time from the ALR detector. + pub fn set_alr_start_time(&mut self, alr_start: Option) { + let was_in_alr = self.was_in_alr; + let is_in_alr = alr_start.is_some(); + + // Detect ALR state transition + if was_in_alr != is_in_alr { + // Reset observations on ALR transition to avoid mixing + // ALR and non-ALR traffic in the same observation window + self.reset_observations(); + trace!( + "LossController: ALR state changed (was: {}, now: {}), observations reset", + was_in_alr, is_in_alr + ); + } + + self.alr_start_time = alr_start; + self.was_in_alr = is_in_alr; + } + + /// Set link capacity estimate from successful ALR probes. + pub fn set_link_capacity_estimate(&mut self, capacity: Option) { + self.link_capacity_estimate = capacity; + } + + /// Check if currently in ALR state + fn is_in_alr(&self) -> bool { + self.alr_start_time.is_some() + } + + /// Reset all observations. + /// + /// Called when ALR state transitions to avoid mixing traffic patterns + /// from different network utilization regimes. + fn reset_observations(&mut self) { + self.partial_observation = PartialObservation::new(); + self.last_send_time_most_recent_observation = Timestamp::DistantFuture; + + // Clear observation window + for observation in self.observations.iter_mut() { + *observation = Observation::DUMMY; + } + + // Reset cached values that depend on observations + self.cached_instant_upper_bound = None; + } + + /// Update the estimate using TWCC feedback from the network. + /// After this [`loss_based_result`] returns the latest estimate. + pub fn update_bandwidth_estimate( + &mut self, + packet_results: &[impl PacketResult], + delay_based_estimated: Bitrate, + ) { + self.delay_based_estimate = delay_based_estimated; + + if packet_results.is_empty() { + debug!("packet results is empty"); + return; + } + + if !self.maybe_add_observation(packet_results) { + return; + } + + if !self.current_estimate.loss_limited_bandwidth.is_valid() { + warn!("estimator must be initialized before use"); + return; + } + + let mut best_candidate = self.current_estimate; + let mut objective_max = f64::MIN; + + for candidate in self.get_candidates().iter_mut() { + self.newtons_method_update(candidate); + + let candidate_objective = self.get_objective(candidate); + if candidate_objective > objective_max { + objective_max = candidate_objective; + best_candidate = *candidate; + } + } + + if best_candidate.loss_limited_bandwidth < self.current_estimate.loss_limited_bandwidth { + self.last_time_estimate_reduced = self.last_send_time_most_recent_observation; + } + + // do not increase the estimate if the average loss is greater than current inherent loss + if self.average_reported_loss_ratio() > best_candidate.inherent_loss + && self + .config + .not_increase_if_inherent_loss_less_than_average_loss + && self.current_estimate.loss_limited_bandwidth < best_candidate.loss_limited_bandwidth + { + best_candidate.loss_limited_bandwidth = self.current_estimate.loss_limited_bandwidth; + } + + if self.is_bandwidth_limited_due_to_loss() { + // Bound the estimate increase if: + // 1. The estimate has been increased for less than + // `delayed_increase_window` ago, and + // 2. The best candidate is greater than bandwidth_limit_in_current_window. + + if self.recovering_after_loss_timestamp.is_exact() + && self.recovering_after_loss_timestamp + self.config.delayed_increase_window + > self.last_send_time_most_recent_observation + && best_candidate.loss_limited_bandwidth > self.bandwidth_limit_in_current_window + { + best_candidate.loss_limited_bandwidth = self.bandwidth_limit_in_current_window; + } + + let increase_when_loss_limited = + self.is_estimate_increasing_when_loss_limited(best_candidate); + + if increase_when_loss_limited && self.acknowledged_bitrate.is_valid() { + // Choose rampup factor based on whether we're in HOLD region (WebRTC lines 270-281) + let rampup_factor = if self.last_hold_info.rate.is_valid() + && self.acknowledged_bitrate + < self.last_hold_info.rate * self.config.bandwidth_rampup_hold_threshold + { + self.config.bandwidth_rampup_upper_bound_factor_in_hold // 1.2 + } else { + self.config.bandwidth_rampup_upper_bound_factor // 1.5 + }; + + best_candidate.loss_limited_bandwidth = + self.current_estimate.loss_limited_bandwidth.max( + best_candidate + .loss_limited_bandwidth + .min(self.acknowledged_bitrate * rampup_factor), + ); + + // WebRTC lines 282-290: Ensure at least 1 bps increase when transitioning from Decreasing + if self.state == LossControllerState::Decreasing + && best_candidate.loss_limited_bandwidth + == self.current_estimate.loss_limited_bandwidth + { + best_candidate.loss_limited_bandwidth = + self.current_estimate.loss_limited_bandwidth + Bitrate::bps(1); + } + } + } + + let loss_limited_bandwidth = best_candidate.loss_limited_bandwidth; + + // HOLD check (WebRTC lines 321-334): If in Decreasing state and HOLD timer active, cap at HOLD rate + if self.state == LossControllerState::Decreasing + && self.last_hold_info.timestamp > self.last_send_time_most_recent_observation + && loss_limited_bandwidth < self.delay_based_estimate + { + // During HOLD period, cap estimate at HOLD rate + self.current_estimate = best_candidate; + self.current_estimate.loss_limited_bandwidth = + loss_limited_bandwidth.min(self.last_hold_info.rate); + log_inherent_loss!(self.current_estimate.inherent_loss); + log_loss_based_bitrate_estimate!(self.current_estimate.loss_limited_bandwidth.as_f64()); + return; + } + + // State transitions with HOLD mechanism (WebRTC lines 336-378) + let new_state = if self.is_estimate_increasing_when_loss_limited(best_candidate) + && loss_limited_bandwidth < delay_based_estimated + && loss_limited_bandwidth < self.max_bitrate + { + LossControllerState::Increasing + } else if loss_limited_bandwidth < self.delay_based_estimate + && loss_limited_bandwidth < self.max_bitrate + { + // Entering Decreasing state - set HOLD info + if self.state != LossControllerState::Decreasing + && self.config.hold_duration_factor > 0.0 + { + const MAX_HOLD_DURATION: Duration = Duration::from_secs(60); + self.last_hold_info = HoldInfo { + timestamp: self.last_send_time_most_recent_observation + + self.last_hold_info.duration, + duration: MAX_HOLD_DURATION.min(Duration::from_secs_f64( + self.last_hold_info.duration.as_secs_f64() + * self.config.hold_duration_factor, + )), + rate: loss_limited_bandwidth, + }; + } + LossControllerState::Decreasing + } else { + // Reset HOLD info when returning to DelayBased + self.last_hold_info = HoldInfo { + timestamp: Timestamp::DistantPast, + duration: Duration::from_millis(300), + rate: Bitrate::INFINITY, + }; + LossControllerState::DelayBased + }; + self.set_state(new_state); + + self.current_estimate = best_candidate; + log_inherent_loss!(self.current_estimate.inherent_loss); + log_loss_based_bitrate_estimate!(self.current_estimate.loss_limited_bandwidth.as_f64()); + + const CONGESTION_CONTROLLER_MIN_BITRATE: Bitrate = Bitrate::kbps(5); + const CONF_MAX_INCREASE_FACTOR: f64 = 1.3; + + if self.is_bandwidth_limited_due_to_loss() + && (!self.recovering_after_loss_timestamp.is_exact() + || self.recovering_after_loss_timestamp + self.config.delayed_increase_window + < self.last_send_time_most_recent_observation) + { + self.bandwidth_limit_in_current_window = CONGESTION_CONTROLLER_MIN_BITRATE + .max(loss_limited_bandwidth * CONF_MAX_INCREASE_FACTOR); + + self.recovering_after_loss_timestamp = self.last_send_time_most_recent_observation; + log_loss_bw_limit_in_window!(self.bandwidth_limit_in_current_window.as_f64()); + } + } + + // TODO: Determine if we want to integrate these two with the rest of the system. + #[cfg(test)] + pub fn set_max_bitrate(&mut self, max_bitrate: Bitrate) { + self.max_bitrate = max_bitrate; + } + + #[cfg(test)] + pub fn set_min_bitrate(&mut self, min_bitrate: Bitrate) { + self.min_bitrate = min_bitrate; + } + + pub fn loss_based_result(&self) -> LossBasedBweResult { + let mut result = LossBasedBweResult { + bandwidth_estimate: self.current_estimate.loss_limited_bandwidth.as_valid(), + state: self.state, + }; + + if self.num_observations == 0 { + return result; + } + + let Some(loss_limited_bandwidth) = self.current_estimate.loss_limited_bandwidth.as_valid() + else { + return result; + }; + let instant_upper_bound = self.get_instant_upper_bound(); + + if self.delay_based_estimate.is_valid() { + result.bandwidth_estimate = Some( + loss_limited_bandwidth + .min(self.delay_based_estimate) + .min(instant_upper_bound), + ) + } else { + result.bandwidth_estimate = Some(loss_limited_bandwidth.min(instant_upper_bound)) + } + + result + } + + fn maybe_add_observation(&mut self, packet_results: &[impl PacketResult]) -> bool { + let Some(summary) = PacketResultsSummary::from(packet_results) else { + return false; + }; + + let last_send_time = Timestamp::from(summary.last_send_time); + + self.partial_observation.update(summary); + + if !self.last_send_time_most_recent_observation.is_exact() { + self.last_send_time_most_recent_observation = last_send_time; + } + + let observation_duration = last_send_time - self.last_send_time_most_recent_observation; + + if observation_duration <= Duration::ZERO { + return false; + } + + // decide if we can accept the partial observation as complete + if observation_duration <= self.config.observation_duration_lower_bound { + return false; + } + + self.last_send_time_most_recent_observation = last_send_time; + + let observation = { + let id = self.num_observations; + self.num_observations += 1; + + Observation { + num_packets: self.partial_observation.num_packets, + size: self.partial_observation.size, + num_lost_packets: self.partial_observation.num_lost_packets, + lost_size: self.partial_observation.lost_size, + num_received_packets: self.partial_observation.num_packets + - self.partial_observation.num_lost_packets, + sending_rate: self.partial_observation.size / observation_duration, + id, + is_initialized: true, + } + }; + + // save our complete observation + self.observations[observation.id as usize % self.config.observation_window_size] = + observation; + + // renew the partial observation + self.partial_observation = PartialObservation::new(); + + // calculate upper bound + self.cached_instant_upper_bound = Some(self.calculate_instant_upper_bound()); + + true + } + + fn get_candidates(&self) -> Vec { + let mut bandwidths = vec![]; + + let current = self.current_estimate.loss_limited_bandwidth; + + for factor in self.config.candidate_factor.iter() { + bandwidths.push(factor * current.as_f64()); + } + + if self.delay_based_estimate.is_valid() + && self.config.append_delay_based_estimate_candidate + && self.delay_based_estimate > current + { + bandwidths.push(self.delay_based_estimate.as_f64()); + } + + let candidate_bandwidth_upper_bound = self.get_candidate_bandwidth_upper_bound().as_f64(); + + if self.config.append_acknowledged_rate_candidate && self.acknowledged_bitrate.is_valid() { + bandwidths.push( + (self.acknowledged_bitrate * self.config.bandwidth_backoff_lower_bound_factor) + .as_f64(), + ); + } + + if self.config.append_delay_based_estimate_candidate + && self.delay_based_estimate.is_valid() + && self.delay_based_estimate > current + { + bandwidths.push( + (self.delay_based_estimate * self.config.bandwidth_backoff_lower_bound_factor) + .as_f64(), + ); + } + + let mut candidates = Vec::with_capacity(bandwidths.len()); + + for bandwidth in bandwidths.iter_mut() { + let mut candidate = self.current_estimate; + candidate.loss_limited_bandwidth = if self.config.trendline_integration_enabled { + bandwidth.min(candidate_bandwidth_upper_bound).into() + } else { + bandwidth + .min( + self.current_estimate + .loss_limited_bandwidth + .as_f64() + .max(candidate_bandwidth_upper_bound), + ) + .into() + }; + candidate.inherent_loss = self.get_feasible_inherent_loss(&candidate); + candidates.push(candidate); + } + + candidates + } + + fn newtons_method_update(&self, channel_parameters: &mut ChannelParameters) { + if self.num_observations == 0 { + return; + } + + for _ in 0..self.config.newton_iterations { + let derivatives = self.get_derivatives(channel_parameters); + channel_parameters.inherent_loss -= + self.config.newton_step_size * (derivatives.0 / derivatives.1); + channel_parameters.inherent_loss = self.get_feasible_inherent_loss(channel_parameters); + } + } + + fn get_derivatives(&self, channel_prameters: &ChannelParameters) -> (f64, f64) { + let mut derivatives: (f64, f64) = (0.0, 0.0); + + for observation in self.observations.iter() { + if !observation.is_initialized { + continue; + } + + let loss_probability = self.get_loss_probability( + channel_prameters.inherent_loss, + channel_prameters.loss_limited_bandwidth, + observation.sending_rate, + ); + + let index = (self.num_observations - 1) - observation.id; + let temporal_weight = self.temporal_weights[index as usize]; + + if self.config.use_byte_loss_ratio { + derivatives.0 += temporal_weight + * ((observation.lost_size.as_kb() / loss_probability) + - ((observation.size - observation.lost_size).as_kb() + / (1.0 - loss_probability))); + + derivatives.1 -= temporal_weight + * ((observation.lost_size.as_kb() / f64::powi(loss_probability, 2)) + + ((observation.size - observation.lost_size).as_kb() + / f64::powi(1.0 - loss_probability, 2))); + } else { + derivatives.0 += temporal_weight + * ((observation.num_lost_packets as f64 / loss_probability) + - (observation.num_received_packets as f64 / (1.0 - loss_probability))); + + derivatives.1 -= temporal_weight + * ((observation.num_lost_packets as f64 / f64::powi(loss_probability, 2)) + + (observation.num_received_packets as f64 + / f64::powi(1.0 - loss_probability, 2))); + } + } + + // Clamp second derivative to safe value if invalid due to floating-point edge cases + // (infinity, denormals, extreme values from pathological TWCC feedback) + if !derivatives.1.is_sign_negative() || derivatives.1 == 0.0 || derivatives.1.is_nan() { + derivatives.1 = -1.0e-6; + debug!( + "Second derivative clamped to safe value due to invalid result: was {:?}", + derivatives.1 + ); + } + + derivatives + } + + fn get_loss_probability( + &self, + inherent_loss: f64, + loss_limited_bandwidth: Bitrate, + sending_rate: Bitrate, + ) -> f64 { + let inherent_loss = inherent_loss.clamp(0.0, 1.0); + + // maybe warn if sending rate or loss limited bandwidth are not finite + + let mut loss_probability = inherent_loss; + if sending_rate.is_valid() + && loss_limited_bandwidth.is_valid() + && sending_rate > loss_limited_bandwidth + { + loss_probability += (1.0 - inherent_loss) + * ((sending_rate - loss_limited_bandwidth).as_f64() / sending_rate.as_f64()); + } + + loss_probability.clamp(1.0e-6, 1.0 - 1.0e-6) + } + + fn get_objective(&self, candidate: &ChannelParameters) -> f64 { + let mut objective = 0.0; + let high_bandwidth_bias = self.get_high_bandwidth_bias(candidate.loss_limited_bandwidth); + + for observation in self.observations.iter() { + if !observation.is_initialized { + continue; + } + + let loss_probability = self.get_loss_probability( + candidate.inherent_loss, + candidate.loss_limited_bandwidth, + observation.sending_rate, + ); + + let index = (self.num_observations - 1) - observation.id; + let temporal_weight = self.temporal_weights[index as usize]; + + if self.config.use_byte_loss_ratio { + objective += temporal_weight + * ((observation.lost_size.as_kb() / 1000.0) * f64::ln(loss_probability) + + ((observation.size - observation.lost_size).as_kb() / 1000.0) + * f64::ln(1.0 - loss_probability)); + objective += + temporal_weight * high_bandwidth_bias * observation.size.as_kb() / 1000.0; + } else { + objective += temporal_weight + * (observation.num_lost_packets as f64 * f64::ln(loss_probability) + + (observation.num_received_packets as f64 + * f64::ln(1.0 - loss_probability))); + + objective += temporal_weight * high_bandwidth_bias * observation.num_packets as f64; + } + } + + objective + } + + fn is_estimate_increasing_when_loss_limited(&self, candidate: ChannelParameters) -> bool { + if !self.is_bandwidth_limited_due_to_loss() { + return false; + } + + let current = self.current_estimate.loss_limited_bandwidth; + let candidate = candidate.loss_limited_bandwidth; + + if current < candidate { + return true; + } + + current == candidate && self.state == LossControllerState::Increasing + } + + fn is_bandwidth_limited_due_to_loss(&self) -> bool { + self.state != LossControllerState::DelayBased + } + + fn get_candidate_bandwidth_upper_bound(&self) -> Bitrate { + let mut upper_bound = self.max_bitrate; + + // When in ALR and we have a link capacity estimate from probes, + // use it as the upper bound. This prevents estimating beyond proven capacity. + if self.is_in_alr() { + if let Some(capacity) = self.link_capacity_estimate { + if capacity.is_valid() { + upper_bound = upper_bound.min(capacity); + } + } + } + + if self.is_bandwidth_limited_due_to_loss() + && self.bandwidth_limit_in_current_window.is_valid() + { + upper_bound = self.bandwidth_limit_in_current_window.min(upper_bound); + } + + upper_bound = self.get_instant_upper_bound().min(upper_bound); + if self.delay_based_estimate.is_valid() { + upper_bound = upper_bound.min(self.delay_based_estimate); + } + + if !self.acknowledged_bitrate.is_valid() { + return upper_bound; + } + + if self.config.rampup_acceleration_max_factor > Duration::ZERO + && self.last_send_time_most_recent_observation.is_exact() + && self.last_time_estimate_reduced.is_exact() + { + let delta = (self.last_send_time_most_recent_observation + - self.last_time_estimate_reduced) + .max(TimeDelta::ZERO); + let time_since_bw_reduced = self + .config + .rampup_acceleration_maxout_time + .as_secs_f64() + .min(delta.as_secs_f64()); + + let rampup_acceleration = self.config.rampup_acceleration_max_factor.as_secs_f64() + * time_since_bw_reduced + / self.config.rampup_acceleration_maxout_time.as_secs_f64(); + + upper_bound = upper_bound + (self.acknowledged_bitrate * rampup_acceleration); + } + + upper_bound + } + + fn set_state(&mut self, state: LossControllerState) { + if state != self.state { + debug!( + "Changing loss controller state: {:?} -> {:?}", + self.state, state + ); + } + self.state = state; + } + + fn get_high_bandwidth_bias(&self, bandwidth: Bitrate) -> f64 { + if !bandwidth.is_valid() { + return 0.0; + } + + let average_reported_loss_ratio = self.average_reported_loss_ratio(); + + self.adjust_bias_factor( + average_reported_loss_ratio, + self.config.higher_bandwidth_bias_factor, + ) * bandwidth.as_f64() + + self.adjust_bias_factor( + average_reported_loss_ratio, + self.config.higher_log_bandwidth_bias_factor, + ) * f64::ln(1.0 + bandwidth.as_f64()) + } + + fn adjust_bias_factor(&self, loss_rate: f64, bias_factor: f64) -> f64 { + let diff = self.config.threshold_of_high_bandwidth_preference - loss_rate; + bias_factor * (diff / self.config.bandwidth_preference_smoothing_factor + diff.abs()) + } + + fn calculate_instant_upper_bound(&self) -> Bitrate { + // this requires someone to set the max bitrate from outside + let mut instant_limit = self.max_bitrate; + + let average_reported_loss_ratio = self.average_reported_loss_ratio(); + + if average_reported_loss_ratio > self.config.instant_upper_bound_loss_offset { + instant_limit = self.config.instant_upper_bound_bandwidth_balance + / (average_reported_loss_ratio - self.config.instant_upper_bound_loss_offset); + + if average_reported_loss_ratio > self.config.high_loss_rate_threshold { + let limit = self.config.bandwidth_cap_at_high_loss_rate + - self.config.slope_of_bwe_high_loss_function * average_reported_loss_ratio; + + instant_limit = limit.max(self.min_bitrate); + } + } + + instant_limit + } + + fn get_instant_upper_bound(&self) -> Bitrate { + self.cached_instant_upper_bound + .as_valid() + .unwrap_or(self.max_bitrate) + } + + fn average_reported_loss_ratio(&self) -> f64 { + let mut total = 0_f64; + let mut lost = 0_f64; + + for observation in self.observations.iter() { + if !observation.is_initialized { + continue; + } + + let index = (self.num_observations - 1) - observation.id; + + let instant_temporal_weight = self.instant_upper_bound_temporal_weights[index as usize]; + + if self.config.use_byte_loss_ratio { + total += instant_temporal_weight * observation.size.as_bytes_f64(); + lost += instant_temporal_weight * observation.lost_size.as_bytes_f64(); + } else { + total += instant_temporal_weight * observation.num_packets as f64; + lost += instant_temporal_weight * observation.num_lost_packets as f64; + } + } + + if total == 0_f64 { + return 0.0; + } + + lost / total + } + + fn get_feasible_inherent_loss(&self, channel_parameters: &ChannelParameters) -> f64 { + channel_parameters + .inherent_loss + .max(self.config.inherent_loss_lower_bound) + .min( + self.get_inherent_loss_upper_bound(Some(channel_parameters.loss_limited_bandwidth)), + ) + } + + fn get_inherent_loss_upper_bound(&self, bandwidth: Option) -> f64 { + let Some(bandwidth) = bandwidth else { + return 1.0; + }; + + if bandwidth == Bitrate::ZERO { + return 1.0; + } + + let inherent_loss_upper_bound = self.config.inherent_loss_upper_bound_offset + + self + .config + .inherent_loss_upper_bound_bandwidth_balance + .as_f64() + / bandwidth.as_f64(); + + inherent_loss_upper_bound.min(1.0) + } +} + +struct Config { + observation_window_size: usize, // minimum is 2 + observation_duration_lower_bound: Duration, + trendline_integration_enabled: bool, + temporal_weight_factor: f64, + instant_upper_bound_temporal_weight_factor: f64, + instant_upper_bound_loss_offset: f64, + instant_upper_bound_bandwidth_balance: Bitrate, + high_loss_rate_threshold: f64, + slope_of_bwe_high_loss_function: Bitrate, + bandwidth_cap_at_high_loss_rate: Bitrate, + initial_inherent_loss_estimate: f64, + inherent_loss_upper_bound_offset: f64, + inherent_loss_upper_bound_bandwidth_balance: Bitrate, + inherent_loss_lower_bound: f64, + newton_iterations: usize, + newton_step_size: f64, + not_increase_if_inherent_loss_less_than_average_loss: bool, + delayed_increase_window: Duration, + bandwidth_rampup_upper_bound_factor: f64, + candidate_factor: [f64; 3], + append_acknowledged_rate_candidate: bool, + append_delay_based_estimate_candidate: bool, + bandwidth_backoff_lower_bound_factor: f64, + rampup_acceleration_maxout_time: Duration, + rampup_acceleration_max_factor: Duration, + higher_bandwidth_bias_factor: f64, + higher_log_bandwidth_bias_factor: f64, + threshold_of_high_bandwidth_preference: f64, + bandwidth_preference_smoothing_factor: f64, + use_byte_loss_ratio: bool, + hold_duration_factor: f64, + bandwidth_rampup_hold_threshold: f64, + bandwidth_rampup_upper_bound_factor_in_hold: f64, +} + +#[derive(Debug)] +struct PacketResultsSummary { + num_packets: u64, + num_lost_packets: u64, + total_size: DataSize, + lost_size: DataSize, + first_send_time: Instant, + last_send_time: Instant, +} + +impl PacketResultsSummary { + pub fn new(first_send_time: Instant, last_send_time: Instant) -> PacketResultsSummary { + PacketResultsSummary { + num_packets: 0, + num_lost_packets: 0, + total_size: DataSize::ZERO, + lost_size: DataSize::ZERO, + last_send_time, + first_send_time, + } + } + + pub fn from(records: &[impl PacketResult]) -> Option { + let first = records.first()?; + + let mut summary = + PacketResultsSummary::new(first.local_send_time(), first.local_send_time()); + for record in records { + let lost: u64 = record.lost().into(); + let size = record.size(); + + summary.num_packets += 1; + summary.total_size += size; + summary.lost_size += size * lost; + summary.num_lost_packets += lost; + summary.first_send_time = min(summary.first_send_time, record.local_send_time()); + summary.last_send_time = max(summary.last_send_time, record.local_send_time()); + } + + Some(summary) + } +} + +#[derive(Debug, Clone, Copy)] +struct Observation { + num_packets: u64, + size: DataSize, + num_lost_packets: u64, + lost_size: DataSize, + num_received_packets: u64, + sending_rate: Bitrate, + id: u64, + is_initialized: bool, +} + +impl Observation { + pub const DUMMY: Self = Self { + num_packets: 0, + size: DataSize::ZERO, + num_lost_packets: 0, + lost_size: DataSize::ZERO, + num_received_packets: 0, + sending_rate: Bitrate::NEG_INFINITY, + id: 0, + is_initialized: false, + }; +} + +struct PartialObservation { + num_packets: u64, + num_lost_packets: u64, + size: DataSize, + lost_size: DataSize, +} + +impl PartialObservation { + pub fn new() -> PartialObservation { + PartialObservation { + num_packets: 0, + num_lost_packets: 0, + size: DataSize::ZERO, + lost_size: DataSize::ZERO, + } + } + + pub fn update(&mut self, summary: PacketResultsSummary) { + self.num_packets += summary.num_packets; + self.num_lost_packets += summary.num_lost_packets; + self.size += summary.total_size; + self.lost_size += summary.lost_size; + } +} + +/// An estimate derived from some candidate. +#[derive(Debug, Clone, Copy)] +struct ChannelParameters { + /// The estimated inherent loss + inherent_loss: f64, + /// The estimated bandwidth + loss_limited_bandwidth: Bitrate, +} + +impl ChannelParameters { + pub fn new(inherent_loss: f64) -> ChannelParameters { + ChannelParameters { + inherent_loss, + loss_limited_bandwidth: Bitrate::NEG_INFINITY, + } + } +} + +/// HOLD mechanism info - prevents immediate ramp-up after loss detection +#[derive(Debug, Clone, Copy)] +struct HoldInfo { + timestamp: Timestamp, + duration: Duration, + rate: Bitrate, +} + +impl Default for HoldInfo { + fn default() -> Self { + const INIT_HOLD_DURATION: Duration = Duration::from_millis(300); + Self { + timestamp: Timestamp::DistantPast, + duration: INIT_HOLD_DURATION, + rate: Bitrate::INFINITY, + } + } +} + +trait AsValid { + fn as_valid(&self) -> Option; +} + +impl AsValid for Option { + fn as_valid(&self) -> Option { + if let Some(bitrate) = self { + if bitrate.as_f64().is_finite() { + return Some(*bitrate); + } + } + None + } +} + +#[derive(Debug)] +pub struct LossBasedBweResult { + pub bandwidth_estimate: Option, + pub state: LossControllerState, +} + +impl Default for Config { + fn default() -> Self { + Self { + observation_window_size: 15, + observation_duration_lower_bound: Duration::from_millis(250), + trendline_integration_enabled: false, + temporal_weight_factor: 0.9, + instant_upper_bound_temporal_weight_factor: 0.9, + instant_upper_bound_loss_offset: 0.05, + instant_upper_bound_bandwidth_balance: Bitrate::kbps(100), + high_loss_rate_threshold: 1.0, + slope_of_bwe_high_loss_function: Bitrate::kbps(1000), + bandwidth_cap_at_high_loss_rate: Bitrate::kbps(500), + initial_inherent_loss_estimate: 0.01, + inherent_loss_upper_bound_offset: 0.05, + inherent_loss_upper_bound_bandwidth_balance: Bitrate::kbps(100), + inherent_loss_lower_bound: 1.0e-3, + newton_iterations: 1, + newton_step_size: 0.75, + not_increase_if_inherent_loss_less_than_average_loss: true, + delayed_increase_window: Duration::from_millis(300), + bandwidth_rampup_upper_bound_factor: 1.5, + candidate_factor: [1.02, 1.0, 0.95], + append_acknowledged_rate_candidate: true, + append_delay_based_estimate_candidate: true, + bandwidth_backoff_lower_bound_factor: 1.0, + rampup_acceleration_maxout_time: Duration::from_secs(60), + rampup_acceleration_max_factor: Duration::from_secs(60), + higher_bandwidth_bias_factor: 0.0002, + higher_log_bandwidth_bias_factor: 0.02, + threshold_of_high_bandwidth_preference: 0.2, + bandwidth_preference_smoothing_factor: 0.002, + use_byte_loss_ratio: true, + hold_duration_factor: 2.0, + bandwidth_rampup_hold_threshold: 1.3, + bandwidth_rampup_upper_bound_factor_in_hold: 1.2, + } + } +} + +impl PacketResult for &TwccSendRecord { + fn local_send_time(&self) -> Instant { + (*self).local_send_time() + } + + fn size(&self) -> DataSize { + (*self).size().into() + } + + fn lost(&self) -> bool { + (*self).remote_recv_time().is_none() + } +} + +#[cfg(test)] +mod test { + use std::time::Instant; + + use fastrand::Rng; + use systemstat::Duration; + + use super::{Bitrate, DataSize, LossBasedBweResult, LossController, LossControllerState}; + struct PacketResult { + local_send_time: Instant, + size: DataSize, + lost: bool, + } + + impl super::PacketResult for PacketResult { + fn local_send_time(&self) -> Instant { + self.local_send_time + } + + fn size(&self) -> DataSize { + self.size + } + + fn lost(&self) -> bool { + self.lost + } + } + + #[test] + fn no_loss() { + // Test no loss, estimate should be bounded by delay based estimate + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::from(50_000)); // 50 kbps + lbc.set_max_bitrate(Bitrate::from(1_000_000_000)); // 1 Gbps + + let acknowledged_bitrate = Bitrate::from(1_000_000); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::from(1_250_000)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()).num_packets(26); + + // A single observation at 1Mbps + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + + // A single observation at 1Mbps + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + assert_eq!( + bandwidth_estimate, + Some(Bitrate::bps(1_500_000)), + "Estimate should increase to delay based estimate, but not further" + ); + assert_eq!(state, LossControllerState::DelayBased); + } + + #[test] + fn stable_loss() { + // Test stable loss at 5% which should be ignored by the loss controller + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::from(50_000)); // 50 kbps + lbc.set_max_bitrate(Bitrate::from(1_000_000_000)); // 1 Gbps + + let acknowledged_bitrate = Bitrate::from(1_000_000); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::from(1_250_000)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()) + .with_loss(0.05) + .num_packets(26); + + // It takes a while for the maximum likelihood estimation to react to the inherent loss + // this is why we need quite a few observations before the estimate increases to the delay + // based bound + // 40 observations(10 seconds) at 1Mbps + for _ in 0..40 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + assert!( + state == LossControllerState::DelayBased || state == LossControllerState::Increasing, + "With stable inherent loss, should be in DelayBased or Increasing state, got {:?}", + state + ); + + // Note: The loss controller returns loss_limited_bandwidth even in DelayBased state. + // The BWE integration layer (SendSideBandwidthEstimator::last_estimate) is responsible + // for using the delay-based estimate when state is DelayBased. + // This matches WebRTC's LossBasedBweV2 behavior (see loss_based_bwe_v2.cc:379). + assert!( + bandwidth_estimate.is_some(), + "Loss controller should return an estimate even in DelayBased state" + ); + } + + #[test] + fn stable_loss_with_loss_spike() { + // Test stable loss at 5% which should be ignored by the loss controller, followed by a + // loss spike which should cause the estimate to dip + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::from(50_000)); // 50 kbps + lbc.set_max_bitrate(Bitrate::from(1_000_000_000)); // 1 Gbps + + let acknowledged_bitrate = Bitrate::from(1_000_000); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::from(1_250_000)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()) + .with_loss(0.05) + .num_packets(26); + + // It takes a while for the maximum likelihood estimation to react to the inherent loss + // this is why we need quite a few observations before the estimate increases to the delay + // based bound and is stable. + // 40 observations(10 seconds) at 1Mbps + for _ in 0..40 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + pkt_builder = pkt_builder.with_loss(0.9); + // Loss spike(1second at 90% loss) + for _ in 0..4 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + let estimate = bandwidth_estimate.expect("Should have an estimate"); + assert!( + estimate < Bitrate::bps(500_000), + "A loss spike should result in a reduced estimate, estimate was {estimate}" + ); + assert_eq!(state, LossControllerState::Decreasing); + } + + #[test] + fn loss_spike_recovery() { + // Test stable loss at 5% which should be ignored by the loss controller, followed by a + // loss spike which should cause the estimate to dip + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::from(50_000)); // 50 kbps + lbc.set_max_bitrate(Bitrate::from(1_000_000_000)); // 1 Gbps + + let acknowledged_bitrate = Bitrate::from(1_000_000); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::from(1_250_000)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()) + .with_loss(0.05) + .num_packets(26); + + // It takes a while for the maximum likelihood estimation to react to the inherent loss + // this is why we need quite a few observations before the estimate increases to the delay + // based bound and is stable. + // 40 observations(10 seconds) at 1Mbps + for _ in 0..40 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + // Loss spike + pkt_builder = pkt_builder.with_loss(0.9); + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + + pkt_builder = pkt_builder.with_loss(0.05); + // Set loss back to 5% and gradually ramp up the bitrate + for i in 0..40 { + pkt_builder = pkt_builder.num_packets(6 + i / 2); + + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + let LossBasedBweResult { state, .. } = lbc.loss_based_result(); + + // Note: The loss controller returns loss_limited_bandwidth even in DelayBased state. + // After recovery, it may be in Increasing or DelayBased state depending on dynamics. + // The BWE integration layer uses the delay-based estimate when state is DelayBased. + assert!( + state == LossControllerState::DelayBased || state == LossControllerState::Increasing, + "After recovery from loss spike, should be in DelayBased or Increasing state, got {:?}", + state + ); + } + + #[test] + fn stable_loss_gradual_overuse() { + // Test stable loss at 5% which should be ignored by the loss controller, followed by + // a gradual increase in loss as we overuse the capacity + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::from(50_000)); // 50 kbps + lbc.set_max_bitrate(Bitrate::from(1_000_000_000)); // 1 Gbps + + let acknowledged_bitrate = Bitrate::from(1_000_000); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::from(1_250_000)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()) + .with_loss(0.05) + .num_packets(26); + + // It takes a while for the maximum likelihood estimation to react to the inherent loss + // this is why we need quite a few observations before the estimate increases to the delay + // based bound + // 40 observations(10 seconds) at 1Mbps + for _ in 0..40 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + // Gradual increase + for inc in 0..10 { + pkt_builder = pkt_builder.with_loss(0.05 + (inc as f64 / 10.0)); + + for _ in 0..4 { + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + } + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + let estimate = bandwidth_estimate.expect("Should have an estimate"); + assert!( + estimate < Bitrate::bps(1_000_000), + "A gradual overuse should result in a lowered estimate" + ); + assert_eq!(state, LossControllerState::Decreasing); + } + + #[test] + fn test_loss_limited_window() { + let mut lbc = LossController::new(); + lbc.set_min_bitrate(Bitrate::kbps(50)); + lbc.set_max_bitrate(Bitrate::gbps(1)); + + let acknowledged_bitrate = Bitrate::mbps(1); // 1 Mbps + lbc.set_acknowledged_bitrate(acknowledged_bitrate); + lbc.set_bandwidth_estimate(Bitrate::kbps(1_250)); // 1.25Mbps + + let mut pkt_builder = PacketBuilder::new(Instant::now()).num_packets(25); + + { + // Initial observation with no loss + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + } + + let loss_limited = { + // loss spike observation at 50% + pkt_builder = pkt_builder.with_loss(0.5); + let result = pkt_builder.build_packets(); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + let estimate = bandwidth_estimate.expect("Should have an estimate"); + assert!( + estimate < Bitrate::kbps(600), + "A loss spike should've caused a significant drop in estimate, got {}", + estimate + ); + assert_eq!(state, LossControllerState::Decreasing); + + estimate + }; + + { + // Recovery observation at 0% loss + pkt_builder = pkt_builder.with_loss(0.0); + let result = pkt_builder.build_packets(); + // Lower acknowledged bitrate to simulate reacting to estimate due to spike + lbc.set_acknowledged_bitrate(Bitrate::kbps(300)); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + pkt_builder = pkt_builder.forward_time(Duration::from_millis(250)); + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + let estimate = bandwidth_estimate.expect("Should have an estimate"); + assert!( + estimate > loss_limited && estimate <= Bitrate::mbps(1), + "During the recovery window after a loss spike the estimate should increase, but be bounded. loss_limited={}, estimate={}, expected <= 1 Mbps", + loss_limited, + estimate + ); + assert_eq!(state, LossControllerState::Decreasing); + } + + { + // Another recovery observation at 0% loss, outside of the limit window + pkt_builder = pkt_builder.num_packets(80); + let result = pkt_builder.build_packets(); + lbc.set_acknowledged_bitrate(Bitrate::mbps(1)); + lbc.update_bandwidth_estimate(&result, Bitrate::bps(1_500_000)); + + let LossBasedBweResult { + bandwidth_estimate, + state, + } = lbc.loss_based_result(); + + let estimate = bandwidth_estimate.expect("Should have an estimate"); + assert!( + estimate == Bitrate::bps(1_000_000), + "Eventually the estimate should recover but still remain bounded until the average loss caused by spike ages out" + ); + assert_eq!(state, LossControllerState::Decreasing); + } + } + + struct PacketBuilder { + now: Instant, + rng: Rng, + loss_rate: f64, + send_distribution: LogNormalDistribution, + recv_distribution: LogNormalDistribution, + num_packets: u32, + packet_size: DataSize, + } + + impl PacketBuilder { + fn new(now: Instant) -> Self { + Self { + now, + rng: Rng::with_seed(34791910), + loss_rate: 0.0, + send_distribution: LogNormalDistribution { + mean: 0.05, + std_dev: 1.0, + }, + recv_distribution: LogNormalDistribution { + mean: 4.0, + std_dev: 10.0, + }, + num_packets: 10, + packet_size: DataSize::bytes(1200), + } + } + + fn forward_time(mut self, by: Duration) -> Self { + self.now += by; + self + } + + fn with_loss(mut self, loss_rate: f64) -> Self { + self.loss_rate = loss_rate; + self + } + + fn num_packets(mut self, packets: u32) -> Self { + self.num_packets = packets; + self + } + + fn build_packets(&mut self) -> Vec { + let mut last_send_time = self.now; + let mut last_recv_time = self.now; + let mut result: Vec = Vec::with_capacity(self.num_packets as usize); + + for _ in 0..self.num_packets { + let lost = self.rng.f64() <= self.loss_rate; + let first_send_time = last_send_time + + Duration::from_secs_f64( + self.send_distribution.sample(&mut self.rng) / 1000.0, + ); + let recv_time = last_recv_time + + Duration::from_secs_f64( + self.recv_distribution.sample(&mut self.rng) / 1000.0, + ); + + result.push(PacketResult { + local_send_time: first_send_time, + size: self.packet_size, + lost, + }); + + last_send_time = first_send_time; + if !lost { + last_recv_time = recv_time; + } + } + + result + } + } + + struct LogNormalDistribution { + mean: f64, + std_dev: f64, + } + + impl LogNormalDistribution { + fn sample(&self, rng: &mut Rng) -> f64 { + let normal = normal_distribution(rng); + let location = + (self.mean.powi(2) / (self.mean.powi(2) + self.std_dev.powi(2)).sqrt()).ln(); + let scale = (1.0 + (self.std_dev / self.mean).powi(2)).ln().sqrt(); + + (location + scale * normal).exp() + } + } + + fn normal_distribution(rng: &mut Rng) -> f64 { + let u1 = rng.f64(); + let u2 = rng.f64(); + + (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/macros.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/macros.rs new file mode 100644 index 000000000..8c3a1ba03 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/macros.rs @@ -0,0 +1,98 @@ +macro_rules! log_delay_variation { + ($($arg:expr),+) => { + crate::log_stat!("DELAY_VARIATION", $($arg),+); + } +} + +macro_rules! log_trendline_estimate { + ($($arg:expr),+) => { + crate::log_stat!("TRENDLINE_ESTIMATE", $($arg),+); + } +} + +macro_rules! log_trendline_modified_trend { + ($($arg:expr),+) => { + crate::log_stat!("TRENDLINE_MODIFIED_TREND", $($arg),+); + } +} + +macro_rules! log_bitrate_estimate { + ($($arg:expr),+) => { + crate::log_stat!("BITRATE_ESTIMATE", $($arg),+); + } +} + +macro_rules! log_loss_based_bitrate_estimate { + ($($arg:expr),+) => { + crate::log_stat!("LOSS_BITRATE_ESTIMATE", $($arg),+); + } +} + +macro_rules! log_rate_control_state { + ($($arg:expr),+) => { + crate::log_stat!("RATE_CONTROL_STATE", $($arg),+); + } +} + +macro_rules! log_rate_control_observed_bitrate { + ($($arg:expr),+) => { + crate::log_stat!("RATE_CONTROL_OBSERVED_BITRATE", $($arg),+); + } +} + +macro_rules! log_rate_control_applied_change { + ($($arg:expr),+) => { + crate::log_stat!("RATE_CONTROL_APPLIED_CHANGE", $($arg),+); + } +} + +macro_rules! log_pacer_media_debt { + ($($arg:expr),+) => { + crate::log_stat!("PACER_DEBT", $($arg),+, "media"); + } +} + +macro_rules! log_pacer_padding_debt { + ($($arg:expr),+) => { + crate::log_stat!("PACER_DEBT", $($arg),+, "padding"); + } +} + +macro_rules! log_inherent_loss { + ($($arg:expr),+) => { + crate::log_stat!("INHERENT_LOSS", $($arg),+); + } +} + +macro_rules! log_loss { + ($($arg:expr),+) => { + crate::log_stat!("LOSS", $($arg),+); + } +} + +macro_rules! log_loss_bw_limit_in_window { + ($($arg:expr),+) => { + crate::log_stat!("LOSS_BW_LIMIT_IN_WINDOW", $($arg),+); + } +} + +macro_rules! log_probe_bitrate_estimate { + ($($arg:expr),+) => { + crate::log_stat!("PROBE_BITRATE_ESTIMATE", $($arg),+); + } +} + +pub(crate) use log_bitrate_estimate; +pub(crate) use log_delay_variation; +pub(crate) use log_inherent_loss; +pub(crate) use log_loss; +pub(crate) use log_loss_based_bitrate_estimate; +pub(crate) use log_loss_bw_limit_in_window; +pub(crate) use log_pacer_media_debt; +pub(crate) use log_pacer_padding_debt; +pub(crate) use log_probe_bitrate_estimate; +pub(crate) use log_rate_control_applied_change; +pub(crate) use log_rate_control_observed_bitrate; +pub(crate) use log_rate_control_state; +pub(crate) use log_trendline_estimate; +pub(crate) use log_trendline_modified_trend; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/mod.rs new file mode 100644 index 000000000..2ec1dbfb4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/mod.rs @@ -0,0 +1,486 @@ +//! Google Congestion Control (GoogCC) Bandwidth Estimation based on TWCC feedback. +//! +//! This implementation is ported from libWebRTC's GoogCC and goes beyond the simplified +//! IETF draft () to include +//! WebRTC's production features: +//! +//! - Delay-based control (trendline estimator with AIMD rate control) +//! - Loss-based control (with inherent loss rate estimation) +//! - Probe controller with state machine and multi-stage probing strategy +//! - ALR (Application Limited Region) detection and periodic probing +//! - Link capacity estimation from ALR probes +//! +//! The probe controller in particular closely matches WebRTC's `ProbeController` behavior +//! and default constants, enabling compatible bandwidth discovery with WebRTC endpoints. + +use std::cmp::Ordering; +use std::fmt; +use std::time::{Duration, Instant}; + +use crate::Reason; +use crate::rtp_::{Bitrate, DataSize, TwccClusterId, TwccSendRecord, TwccSeq}; +use crate::util::Soonest; + +mod acked_bitrate_estimator; +mod alr_detector; +pub(crate) mod api; +mod delay; +mod link_capacity_estimator; +mod loss_controller; +mod macros; +mod probe; +mod smoother; +mod time; + +use acked_bitrate_estimator::AckedBitrateEstimator; +use alr_detector::AlrDetector; +use delay::DelayController; +use link_capacity_estimator::LinkCapacityEstimator; +use loss_controller::{LossController, LossControllerState}; +use macros::log_loss; +use smoother::EstimateSmoother; + +pub(crate) use macros::{log_pacer_media_debt, log_pacer_padding_debt}; +pub(crate) use probe::{BandwidthLimitedCause, ProbeEstimator}; +pub(crate) use probe::{ProbeClusterState, ProbeControl}; + +#[cfg(feature = "_internal_test_exports")] +pub use probe::ProbeClusterConfig; +#[cfg(not(feature = "_internal_test_exports"))] +pub(crate) use probe::ProbeClusterConfig; + +const INITIAL_BITRATE_WINDOW: Duration = Duration::from_millis(500); +const BITRATE_WINDOW: Duration = Duration::from_millis(150); +const STARTUP_PHASE: Duration = Duration::from_secs(2); + +pub struct Bwe { + bwe: SendSideBandwidthEstimator, + desired_bitrate: Bitrate, + smoother: EstimateSmoother, +} + +impl Bwe { + pub fn new(initial: Bitrate) -> Self { + let send_side_bwe = SendSideBandwidthEstimator::new(initial); + Bwe { + bwe: send_side_bwe, + desired_bitrate: Bitrate::ZERO, + smoother: EstimateSmoother::new(), + } + } + + pub fn handle_timeout(&mut self, now: Instant, do_probe: bool) -> Option { + let result = self.bwe.handle_timeout(self.desired_bitrate, do_probe, now); + if let Some(estimate) = self.bwe.last_estimate() { + self.smoother.record(now, estimate); + } + result + } + + pub fn start_probe(&mut self, config: ProbeClusterConfig, now: Instant) -> bool { + self.bwe.start_probe(config, now) + } + + pub fn end_probe(&mut self, now: Instant, cluster_id: TwccClusterId) { + self.bwe.end_probe(now, cluster_id); + } + + pub fn reset(&mut self, init_bitrate: Bitrate) { + self.bwe.reset(init_bitrate); + } + + pub fn update<'t>( + &mut self, + records: impl Iterator, + now: Instant, + ) { + self.bwe.update(records, now); + } + + pub fn poll_estimate(&mut self) -> Option { + self.smoother.poll() + } + + pub fn poll_timeout(&self) -> (Option, Reason) { + self.bwe.poll_timeout() + } + + pub fn last_estimate(&self) -> Option { + self.bwe.last_estimate() + } + + pub fn on_media_sent(&mut self, payload_size: DataSize, is_padding: bool, now: Instant) { + if !is_padding { + // Update ALR detector with media bytes sent + self.bwe.on_media_sent(payload_size, now); + } + } + + pub fn is_overusing(&self) -> bool { + self.bwe.is_overusing() + } + + pub fn set_desired_bitrate(&mut self, v: Bitrate) { + self.desired_bitrate = v; + } +} + +struct SendSideBandwidthEstimator { + delay_controller: DelayController, + loss_controller: LossController, + acked_bitrate_estimator: AckedBitrateEstimator, + probe_control: ProbeControl, + probe_estimator: ProbeEstimator, + started_at: Option, + alr_detector: AlrDetector, + link_capacity_estimator: LinkCapacityEstimator, + last_updated_estimate: Option, +} + +impl SendSideBandwidthEstimator { + pub fn new(initial_bitrate: Bitrate) -> Self { + let mut alr_detector = AlrDetector::new(); + alr_detector.set_estimated_bitrate(initial_bitrate); + + let mut loss_controller = LossController::new(); + loss_controller.set_bandwidth_estimate(initial_bitrate); + + Self { + delay_controller: DelayController::new(initial_bitrate), + loss_controller, + acked_bitrate_estimator: AckedBitrateEstimator::new( + INITIAL_BITRATE_WINDOW, + BITRATE_WINDOW, + ), + probe_control: ProbeControl::new(), + probe_estimator: ProbeEstimator::new(), + started_at: None, + alr_detector, + link_capacity_estimator: LinkCapacityEstimator::new(), + last_updated_estimate: None, + } + } + + /// Whether the delay-based detector currently signals overuse. + /// + /// This is useful for gating behaviors (like padding/probing) that would otherwise + /// re-excite the system while we're already congested. + pub fn is_overusing(&self) -> bool { + self.delay_controller.is_overusing() + } + + /// Update ALR detector with actual bytes sent. + /// + /// Should be called for media packets (not padding/probes). + /// This is typically called from the session's packet sending logic. + pub fn on_media_sent(&mut self, bytes: DataSize, now: Instant) { + self.alr_detector.on_bytes_sent(bytes, now); + } + + /// Record a packet from a TWCC report. + pub fn update<'t>(&mut self, records: impl Iterator, now: Instant) { + let _ = self.started_at.get_or_insert(now); + + let send_records: Vec<_> = records.collect(); + + // Feed records to probe estimator for analysis and process any new probe results + let mut latest_probe_result = None; + for (config, bitrate) in self.probe_estimator.update(send_records.iter().copied()) { + latest_probe_result = Some(bitrate); + + // Update link capacity estimator for every successful ALR probe, not just the latest. + // The estimator internally takes the max of all probe results, building up knowledge + // of proven link capacity. This differs from the delay controller, which only receives + // the latest probe result (matching WebRTC's FetchAndResetLastEstimatedBitrate behavior). + if config.is_alr_probe() { + self.link_capacity_estimator.update_from_probe(bitrate, now); + } + } + + let mut acked_packets = vec![]; + + let mut max_rtt = None; + let mut count = 0; + let mut lost = 0; + for record in send_records.iter() { + count += 1; + let Ok(acked_packet) = (*record).try_into() else { + lost += 1; + continue; + }; + acked_packets.push(acked_packet); + max_rtt = max_rtt.max(record.rtt()); + } + acked_packets.sort_by(AckedPacket::order_by_receive_time); + + for acked_packet in acked_packets.iter() { + self.acked_bitrate_estimator + .update(acked_packet.remote_recv_time, acked_packet.size); + } + + let acked_bitrate = self.acked_bitrate_estimator.current_estimate(); + + // Use the latest probe result from this update, if any + let probe_result = latest_probe_result; + + let is_probe_result = probe_result.is_some(); + + // Update delay controller with the latest probe result + let maybe_estimate = + self.delay_controller + .update(&acked_packets, acked_bitrate, probe_result, now); + + let Some(delay_estimate) = maybe_estimate else { + return; + }; + + let loss = if count == 0 { + 0.0 + } else { + lost as f64 / count as f64 + }; + log_loss!(loss); + + // During startup with no loss, use delay-based estimate directly + if in_startup_phase(self.started_at, now) && loss <= 0.001 { + self.loss_controller.set_bandwidth_estimate(delay_estimate); + return; + } + + // When probe succeeds, set bandwidth directly + if is_probe_result { + self.loss_controller.set_bandwidth_estimate(delay_estimate); + } + + if let Some(acked_bitrate) = acked_bitrate { + self.loss_controller.set_acknowledged_bitrate(acked_bitrate); + } + + // This corresponds to UpdateLossBasedEstimator + UpdateEstimate + self.loss_controller + .update_bandwidth_estimate(&send_records, delay_estimate); + + // Loss-based result is capped by delay_based_limit + let loss_result = self.loss_controller.loss_based_result(); + if let Some(loss_estimate) = loss_result.bandwidth_estimate { + if loss_estimate > delay_estimate { + // Loss controller produced higher estimate than delay controller + // Cap it at delay estimate (delay controller is the upper limit) + self.loss_controller.set_bandwidth_estimate(delay_estimate); + } + } + + // Feed the (possibly combined) estimate into subcomponents wanting it. + self.propagate_estimate(); + } + + pub fn poll_timeout(&self) -> (Option, Reason) { + let delay_timeout = Some(self.delay_controller.poll_timeout()); + let probe_timeout = Some(self.probe_control.poll_timeout()); + let probe_estimator_timeout = Some(self.probe_estimator.poll_timeout()); + (delay_timeout, Reason::BweDelayControl) + .soonest((probe_timeout, Reason::BweProbeControl)) + .soonest((probe_estimator_timeout, Reason::BweProbeEstimator)) + } + + /// Handle periodic timeout for BWE components. + pub fn handle_timeout( + &mut self, + desired_bitrate: Bitrate, + do_probe: bool, + now: Instant, + ) -> Option { + self.delay_controller + .handle_timeout(self.acked_bitrate_estimator.current_estimate(), now); + + // Update probe control with desired bitrate. + self.probe_control.set_desired_bitrate(desired_bitrate); + + // Get ALR state and forward to both probe control and loss controller + let alr_start_time = self.alr_detector.alr_start_time(); + if let Some(t) = alr_start_time { + self.probe_control.set_alr_start_time(t); + } else { + self.probe_control.set_alr_stop_time(now); + } + + self.loss_controller.set_alr_start_time(alr_start_time); + + // Get link capacity estimate and forward to loss controller. + let link_capacity = self.link_capacity_estimator.capacity_estimate(now); + self.loss_controller + .set_link_capacity_estimate(link_capacity); + + // Clean up expired probe cluster state + self.probe_estimator.handle_timeout(now); + + // Feed the current estimate into subcontrollers, if it changed. + self.propagate_estimate(); + + // If we can't probe, clear any pending/active probes + if !do_probe { + self.probe_estimator.clear_probes(); + } + + self.probe_control.enable(do_probe); + + // Timer-driven probe logic (WebRTC `Process()` equivalent). + self.probe_control.handle_timeout(now) + } + + fn propagate_estimate(&mut self) { + // Do we have a value? + let Some(estimate) = self.last_estimate() else { + return; + }; + // Did it change? + if self.last_updated_estimate == Some(estimate) { + return; + } + + let cause = self.bandwidth_limited_cause(); + + self.probe_control.set_estimated_bitrate(estimate, cause); + self.alr_detector.set_estimated_bitrate(estimate); + + // Don't update until this changes. + self.last_updated_estimate = Some(estimate); + } + + fn bandwidth_limited_cause(&self) -> BandwidthLimitedCause { + if self.delay_controller.is_overusing() { + return BandwidthLimitedCause::DelayBasedLimitedDelayIncreased; + } + + match self.loss_controller.loss_based_result().state { + LossControllerState::DelayBased => BandwidthLimitedCause::DelayBasedLimited, + LossControllerState::Increasing => BandwidthLimitedCause::LossLimitedBweIncreasing, + LossControllerState::Decreasing => BandwidthLimitedCause::LossLimitedBwe, + } + } + + /// Get the latest estimate. + pub fn last_estimate(&self) -> Option { + let delay_estimate = self.delay_controller.last_estimate(); + + let loss_result = self.loss_controller.loss_based_result(); + + // Only apply loss-based limiting when actively in a loss-limiting state + match loss_result.state { + LossControllerState::DelayBased => { + // Loss controller defers to delay-based estimate + delay_estimate + } + LossControllerState::Decreasing | LossControllerState::Increasing => { + // Loss controller is actively limiting or recovering + match (delay_estimate, loss_result.bandwidth_estimate) { + (Some(de), Some(le)) => Some(de.min(le)), + (None, le @ Some(_)) => le, + (de @ Some(_), None) => de, + (None, None) => None, + } + } + } + } + + /// Start analyzing a probe cluster. + /// + /// This should be called when the pacer starts sending a probe cluster, + /// to tell the estimator which cluster to watch for in TWCC feedback. + /// Returns `true` if the probe was started, `false` if rejected. + pub fn start_probe(&mut self, config: ProbeClusterConfig, now: Instant) -> bool { + self.probe_estimator.probe_start(config, now) + } + + /// End a probe cluster and mark it for cleanup. + /// + /// This should be called when the pacer finishes sending a probe cluster. + /// The estimator will continue collecting feedback for a cluster history period + /// to allow late-arriving TWCC reports to refine the estimate. + pub fn end_probe(&mut self, now: Instant, cluster_id: TwccClusterId) { + self.probe_estimator.end_probe(now, cluster_id); + } + + pub fn reset(&mut self, init_bitrate: Bitrate) { + *self = Self::new(init_bitrate); + } +} + +/// A RTP packet that has been sent and acknowledged by the receiver in a TWCC report. +#[derive(Debug, Copy, Clone)] +pub struct AckedPacket { + /// The TWCC sequence number + seq_no: TwccSeq, + /// The size of the packets in bytes. + size: DataSize, + /// When we sent the packet + local_send_time: Instant, + /// When the packet was received at the remote, note this Instant is only usable with other + /// instants of the same type i.e. those that represent a TWCC reported receive time for this + /// session. + remote_recv_time: Instant, + /// The local time when received confirmation that the other side received the seq i.e. when we + /// received the TWCC report for this packet. + local_recv_time: Instant, +} + +impl AckedPacket { + fn rtt(&self) -> Duration { + self.local_recv_time - self.local_send_time + } + + fn order_by_receive_time(lhs: &Self, rhs: &Self) -> Ordering { + if lhs.remote_recv_time != rhs.remote_recv_time { + lhs.remote_recv_time.cmp(&rhs.remote_recv_time) + } else if lhs.local_send_time != rhs.local_send_time { + lhs.local_send_time.cmp(&rhs.local_send_time) + } else { + lhs.seq_no.cmp(&rhs.seq_no) + } + } +} + +// NB: Extracted for lifetime reasons +fn in_startup_phase(started_at: Option, now: Instant) -> bool { + started_at + .map(|s| now.duration_since(s) <= STARTUP_PHASE) + .unwrap_or(false) +} + +impl TryFrom<&TwccSendRecord> for AckedPacket { + type Error = (); + + fn try_from(value: &TwccSendRecord) -> Result { + let Some(remote_recv_time) = value.remote_recv_time() else { + return Err(()); + }; + let Some(local_recv_time) = value.local_recv_time() else { + return Err(()); + }; + + Ok(Self { + seq_no: value.seq(), + size: value.size().into(), + local_send_time: value.local_send_time(), + remote_recv_time, + local_recv_time, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BandwidthUsage { + Overuse, + Normal, + Underuse, +} + +impl fmt::Display for BandwidthUsage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BandwidthUsage::Overuse => write!(f, "overuse"), + BandwidthUsage::Normal => write!(f, "normal"), + BandwidthUsage::Underuse => write!(f, "underuse"), + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/cluster.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/cluster.rs new file mode 100644 index 000000000..41de748ee --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/cluster.rs @@ -0,0 +1,431 @@ +//! Probe cluster data structures for bandwidth estimation. +//! +//! Probe clusters are short bursts of packets sent at specific bitrates to test +//! network capacity. + +use std::cmp; +use std::time::{Duration, Instant}; + +use crate::rtp_::{Bitrate, DataSize, TwccClusterId}; +use crate::util::{already_happened, not_happening}; + +const MAX_PADDING_PACKET_SIZE: DataSize = DataSize::bytes(240); + +/// Configuration for a probe cluster (the plan). +/// +/// This represents the immutable blueprint for a bandwidth probe: what bitrate +/// to test, for how long, and with what constraints. +#[derive(Debug, Clone, Copy)] +pub struct ProbeClusterConfig { + /// Unique identifier for this probe cluster + cluster: TwccClusterId, + + /// Target bitrate to probe at (e.g., 3 Mbps) + target_bitrate: Bitrate, + + /// How long to sustain the target bitrate (e.g., 15ms) + target_duration: Duration, + + /// Minimum number of packets to send (e.g., 5) + /// This ensures statistical validity even for short bursts. + min_packet_count: usize, + + /// Delta time between sent bursts of packets during probe. + /// + /// Mirrors WebRTC's `ProbeClusterConfig.min_probe_delta`. The pacer/probe sender must not + /// schedule probe packets more frequently than this. + min_probe_delta: Duration, + + /// The kind of probe this is. + kind: ProbeKind, +} + +/// Kind of probe +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ProbeKind { + Initial, + Exponential, + IncreaseAlr, + PeriodicAlr, + LargeDrop, + Stagnant, +} + +impl ProbeKind { + pub(crate) fn is_alr(&self) -> bool { + matches!(self, ProbeKind::PeriodicAlr | ProbeKind::IncreaseAlr) + } +} + +impl ProbeClusterConfig { + /// Create a new probe cluster configuration with standard defaults: + /// - 15ms duration (enough to get meaningful feedback without excessive delay) + /// - 5 minimum packets (statistical significance for BWE analysis) + pub fn new(cluster: TwccClusterId, target_bitrate: Bitrate, kind: ProbeKind) -> Self { + Self { + cluster, + target_bitrate, + // WebRTC defaults + target_duration: Duration::from_millis(15), + min_packet_count: 5, + // WebRTC default for general probing (not initial/network-state): 2ms. + min_probe_delta: Duration::from_millis(2), + kind, + } + } + + /// Set a custom target duration for this probe. + /// WebRTC uses 100ms for initial probes to allow time for media to start. + pub fn with_duration(mut self, duration: Duration) -> Self { + self.target_duration = duration; + self + } + + /// Set a custom minimum packet count for this probe. + pub fn with_min_packet_count(mut self, min_packet_count: usize) -> Self { + self.min_packet_count = min_packet_count; + self + } + + /// Set a custom minimum probe delta (spacing constraint) for this probe. + pub fn with_min_probe_delta(mut self, min_probe_delta: Duration) -> Self { + self.min_probe_delta = min_probe_delta; + self + } + + /// Check if this probe was created during ALR. + pub fn is_alr_probe(&self) -> bool { + self.kind.is_alr() + } + + /// Get the probe cluster ID. + pub fn cluster(&self) -> TwccClusterId { + self.cluster + } + + /// Get the target bitrate. + pub fn target_bitrate(&self) -> Bitrate { + self.target_bitrate + } + + /// Get the minimum packet count required for a valid probe. + pub fn min_packet_count(&self) -> usize { + self.min_packet_count + } + + /// Get the minimum probe delta (spacing constraint). + pub fn min_probe_delta(&self) -> Duration { + self.min_probe_delta + } + + /// Calculate the target bytes for this probe. + /// This is how much data we expect to send at target_bitrate for target_duration. + pub fn target_bytes(&self) -> DataSize { + self.target_bitrate * self.target_duration + } +} + +/// Runtime state of an active probe cluster (the execution). +/// +/// This tracks what's actually happening as we send probe packets: how much +/// we've sent, how many packets, and when we started. +#[derive(Debug)] +pub struct ProbeClusterState { + /// The immutable plan + config: ProbeClusterConfig, + + /// Total bytes sent so far in this probe + bytes_sent: DataSize, + + /// Total packets sent so far in this probe + packets_sent: usize, + + /// When the first packet was sent + /// None = probe hasn't started yet (still queued) + started_at: Option, + + /// When the last packet was sent + /// Used to calculate actual send duration (first to last packet) + last_packet_at: Option, + + /// When we last created a padding request + /// Used to prevent creating multiple padding packets for the same instant + last_padding_at: Instant, +} + +impl ProbeClusterState { + /// Create a new probe cluster state from a config. + pub fn new(config: ProbeClusterConfig) -> Self { + Self { + config, + bytes_sent: DataSize::ZERO, + packets_sent: 0, + started_at: None, + last_packet_at: None, + last_padding_at: already_happened(), + } + } + + /// Get the probe configuration. + pub fn config(&self) -> &ProbeClusterConfig { + &self.config + } + + /// Calculates the next probe time based on total bytes sent and target bitrate. + /// This naturally handles variable packet sizes (media packets can be much larger + /// than padding packets). Next packet can be sent when: + /// `now >= start_time + (bytes_sent / target_bitrate)` + pub fn next_probe_time(&self) -> Instant { + let Some(started_at) = self.started_at else { + return already_happened(); + }; + + if self.config.target_bitrate == Bitrate::ZERO { + return not_happening(); + } + + // Packet-level pacing: schedule based on bytes sent at the target bitrate. + // next_time = start_time + (bytes_sent / target_bitrate) + let send_duration = self.bytes_sent / self.config.target_bitrate; + let probe_time = started_at + send_duration; + + let min_burst_time = self.last_padding_at + self.config.min_probe_delta(); + probe_time.max(min_burst_time) + } + + /// Check if the probe cluster is complete. + /// + /// A probe is complete when BOTH conditions are met (matching WebRTC): + /// 1. Sent bytes >= target_bytes (target_bitrate * target_duration) + /// 2. Sent packets >= min_packet_count + /// + /// Note: WebRTC does NOT check duration for completion. Duration is only used + /// to calculate the target bytes threshold. This ensures probes complete even + /// when all packets are sent at the same instant (e.g., in tests or when the + /// pacer generates bursts faster than wall-clock time advances). + pub fn is_complete(&self, _now: Instant) -> bool { + if self.started_at.is_none() { + return false; // Not started yet + } + + // Match WebRTC's BitrateProber::ProbeSent() logic: + // if (sent_bytes >= probe_cluster_min_bytes && sent_probes >= probe_cluster_min_probes) + let bytes_met = self.bytes_sent >= self.config.target_bytes(); + let packets_met = self.packets_sent >= self.config.min_packet_count; + + bytes_met && packets_met + } + + /// Check if it's time to send the next probe packet. + /// + /// Returns `true` if `now >= next_probe_time()`, meaning we should send a packet now. + pub fn should_send_now(&self, now: Instant) -> bool { + let next_time = self.next_probe_time(); + let min_burst_time = self.last_padding_at + self.config.min_probe_delta(); + now >= next_time && now >= min_burst_time + } + + /// Calculate how much padding should be generated for the next probe packet. + /// + /// WebRTC probing uses **bursts** of packets, with a minimum delta between bursts + /// (`min_probe_delta`). In str0m, a `PaddingRequest` is expressed as a number of bytes + /// to queue (which may result in multiple RTP padding packets), so we request a burst + /// sized approximately as: + /// + /// `burst_bytes = target_bitrate * min_probe_delta` + /// + /// This avoids unintentionally capping probe throughput at `MAX_PADDING_PACKET_SIZE / + /// min_probe_delta` (e.g. 240 bytes / 2ms = 960 kbit/s), which would be far below the + /// intended probe bitrate. + /// + /// Returns `None` if it's not time to send yet, or if we've already created + /// a padding packet for this instant. + pub fn next_packet(&mut self, now: Instant) -> Option { + if self.started_at.is_none() { + self.started_at = Some(now); + } + + // Enforce min burst spacing (`min_probe_delta`) between successive padding requests. + // (The very first burst is not gated because `last_padding_at` starts at already_happened()). + if now < self.last_padding_at + self.config.min_probe_delta() { + return None; + } + + if now < self.next_probe_time() { + return None; + } + + // Check if we've already created a padding packet for this exact instant + // This prevents creating multiple padding packets when handle_timeout() is + // called multiple times with the same `now` value + if self.last_padding_at >= now { + return None; + } + + // Mark that we've created padding for this instant + self.last_padding_at = now; + + // It's time to send a new probe burst. + // Calculate recommended probe size as target_bitrate * min_probe_delta. + let min_delta = self.config.min_probe_delta(); + let recommended_probe_size = self.config.target_bitrate * min_delta; + + // Ensure we request at least one padding packet worth of bytes even if min_delta is 0. + let recommended_probe_size = DataSize::bytes(cmp::max( + recommended_probe_size.as_bytes_i64(), + MAX_PADDING_PACKET_SIZE.as_bytes_i64(), + )); + + // Calculate remaining bytes needed to complete the probe cluster. + let bytes_remaining = self.config.target_bytes().saturating_sub(self.bytes_sent); + + // Return the minimum of bytes_remaining and recommended_probe_size. + // When bytes_remaining is zero, this returns None (no more padding). + let request_bytes = cmp::min(bytes_remaining, recommended_probe_size); + + if request_bytes == DataSize::ZERO { + None + } else { + Some(request_bytes) + } + } + + /// Record a packet that was sent as part of this probe (media or padding). + /// + /// This updates the probe's tracking state so that `next_probe_time()` correctly + /// calculates when the next packet should be sent based on the target bitrate. + /// + /// Should be called by the pacer whenever ANY packet is sent during an active probe, + /// not just padding packets generated by `next_packet()`. + pub fn record_packet(&mut self, now: Instant, size: DataSize) { + // If a probe is active and we observe a media packet before any probe-generated padding, + // treat that as the start of the probe. This keeps timing semantics consistent and ensures + // `next_probe_time()` enforces `min_probe_delta` from the first packet onwards. + if self.started_at.is_none() { + self.started_at = Some(now); + } + + self.bytes_sent += size; + self.packets_sent += 1; + self.last_packet_at = Some(now); + } +} + +#[cfg(test)] +mod test { + use super::*; + + // Test helper to directly set state for testing + impl ProbeClusterState { + fn test_set_state(&mut self, bytes_sent: DataSize, packets_sent: usize, now: Instant) { + self.bytes_sent = bytes_sent; + self.packets_sent = packets_sent; + if self.started_at.is_none() { + self.started_at = Some(now); + } + self.last_packet_at = Some(now); + } + } + + #[test] + fn probe_cluster_not_complete_before_start() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + let state = ProbeClusterState::new(config); + + assert!(!state.is_complete(now)); + } + + #[test] + fn probe_cluster_not_complete_after_packets_but_not_bytes() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + // target_bytes = 3 Mbps * 15ms = 45,000 bits = 5,625 bytes + let mut state = ProbeClusterState::new(config); + + // Send 5 packets (meets packet count) but only small packets (doesn't meet bytes) + // 5 * 100 = 500 bytes, which is < 5,625 bytes + state.test_set_state(DataSize::bytes(500), 5, now); + + // Not complete: packets met but bytes not met + assert!(!state.is_complete(now)); + } + + #[test] + fn probe_cluster_not_complete_after_bytes_but_not_packets() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + // target_bytes = 3 Mbps * 15ms = 45,000 bits = 5,625 bytes + let mut state = ProbeClusterState::new(config); + + // Send 2 large packets (meets bytes) but not enough packets + // 2 * 3000 = 6000 bytes > 5,625 bytes, but only 2 packets < 5 packets + state.test_set_state(DataSize::bytes(6000), 2, now); + + // Not complete: bytes met but packets not met + assert!(!state.is_complete(now)); + } + + #[test] + fn probe_cluster_complete_when_both_criteria_met() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + // target_bytes = 3 Mbps * 15ms = 45,000 bits = 5,625 bytes + let mut state = ProbeClusterState::new(config); + + // Send 5 packets of 1200 bytes each = 6000 bytes + // Meets both: 6000 >= 5625 bytes AND 5 >= 5 packets + state.test_set_state(DataSize::bytes(6000), 5, now); + + // Complete: both bytes and packets met, even at same instant + assert!(state.is_complete(now)); + } + + #[test] + fn probe_cluster_complete_even_with_zero_duration() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + let mut state = ProbeClusterState::new(config); + + // Send all packets at exactly the same instant (duration = 0) + state.test_set_state(DataSize::bytes(6000), 5, now); + + // Should still complete (this is the bug fix - no duration requirement) + assert!(state.is_complete(now)); + } + + #[test] + fn probe_cluster_tracks_bytes_and_packets() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial); + let mut state = ProbeClusterState::new(config); + + state.test_set_state(DataSize::bytes(1200), 1, now); + assert_eq!(state.bytes_sent, DataSize::bytes(1200)); + assert_eq!(state.packets_sent, 1); + + state.test_set_state(DataSize::bytes(2200), 2, now + Duration::from_millis(1)); + assert_eq!(state.bytes_sent, DataSize::bytes(2200)); + assert_eq!(state.packets_sent, 2); + } + + #[test] + fn min_probe_delta_is_enforced_in_next_probe_time() { + let now = Instant::now(); + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(3), ProbeKind::Initial) + .with_min_probe_delta(Duration::from_millis(20)); + let mut state = ProbeClusterState::new(config); + assert_eq!(state.config().min_probe_delta(), Duration::from_millis(20)); + + // Start a burst at `now`, then verify that a second burst is blocked until >= now+20ms. + assert!(state.next_packet(now).is_some()); + assert!(state.next_packet(now + Duration::from_millis(19)).is_none()); + assert!(state.next_packet(now + Duration::from_millis(20)).is_some()); + let next = state.last_padding_at; + assert!( + next >= now + Duration::from_millis(20), + "next_probe_time {next:?} must be >= now + min_probe_delta" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/control.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/control.rs new file mode 100644 index 000000000..f8154e298 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/control.rs @@ -0,0 +1,1000 @@ +//! Bandwidth probing controller - decides when and how to probe network capacity. +//! +//! This module implements WebRTC's `ProbeController` state machine for discovering available +//! bandwidth through intentional bursts of packets at rates higher than current estimates. + +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use super::{ProbeClusterConfig, ProbeKind}; +use crate::rtp_::{Bitrate, TwccClusterId}; +use crate::util::{already_happened, not_happening}; + +// Port notes: +// This module ports WebRTC's `ProbeController` behavior from: +// `webrtc/modules/congestion_controller/goog_cc/probe_controller.cc` +// +// Key integration difference: WebRTC returns vectors of probe clusters, while str0m +// returns a single `ProbeClusterConfig` per `handle_timeout()` call. Configs are queued +// internally and `poll_timeout()` returns `already_happened()` until the queue is drained. + +/// WebRTC: `kMaxWaitingTimeForProbingResult`. +const MAX_WAITING_TIME_FOR_PROBING_RESULT: Duration = Duration::from_secs(1); + +/// WebRTC: `kBitrateDropThreshold`, `kBitrateDropTimeout`, `kProbeFractionAfterDrop`, +/// `kProbeUncertainty`, `kAlrEndedTimeout`, `kMinTimeBetweenAlrProbes`. +const BITRATE_DROP_THRESHOLD: f64 = 0.66; +const BITRATE_DROP_TIMEOUT: Duration = Duration::from_secs(5); +const PROBE_FRACTION_AFTER_DROP: f64 = 0.85; +const PROBE_UNCERTAINTY: f64 = 0.05; +const ALR_ENDED_TIMEOUT: Duration = Duration::from_secs(3); +const MIN_TIME_BETWEEN_ALR_PROBES: Duration = Duration::from_secs(5); + +/// WebRTC: inline `* 2` in probe_controller.cc InitiateProbing(). +/// Allows probing up to 2x max_bitrate to account for bursty streams. +const MAX_PROBE_BITRATE_FACTOR: f64 = 2.0; + +/// Minimum time between stagnant periodic probes to avoid excessive probing when at capacity. +const MIN_TIME_BETWEEN_STAGNANT_PROBES: Duration = Duration::from_secs(15); + +/// Threshold for considering an estimate change significant (5%). +const ESTIMATE_CHANGE_THRESHOLD: f64 = 0.05; + +/// Probe rate scale for stagnation probes (2× current estimate). +const STAGNANT_PROBE_SCALE: f64 = 2.0; + +/// WebRTC's `BandwidthLimitedCause` (subset used by probing gating). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BandwidthLimitedCause { + LossLimitedBweIncreasing, + LossLimitedBwe, + DelayBasedLimited, + DelayBasedLimitedDelayIncreased, +} + +pub struct ProbeControl { + config: Config, + next_timeout: Instant, + enabled: bool, + + desired_bitrate: Option, + prev_desired: Option, + + last_estimate: Option, + last_estimate_change: Option, + last_cause: BandwidthLimitedCause, + + prev_estimate: Option, + + alr_start: Option, + alr_stop: Option, + + last_probe: Option, + + large_drop: Option, + + last_stagnant: Option, + + next_cluster_id: TwccClusterId, + pending: VecDeque, + + scheduled_exponential: Option, + scheduled_periodic_alr: Option, + scheduled_stagnant: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct LastProbe { + when: Instant, + kind: ProbeKind, + further: Bitrate, + was_estimate: Option, +} + +struct LargeDrop { + when: Instant, + bitrate_before: Bitrate, +} + +impl Default for ProbeControl { + fn default() -> Self { + Self { + config: Config::default(), + enabled: false, + next_timeout: not_happening(), + desired_bitrate: None, + prev_desired: None, + last_estimate: None, + last_estimate_change: None, + last_cause: BandwidthLimitedCause::DelayBasedLimited, + prev_estimate: None, + alr_start: None, + alr_stop: None, + next_cluster_id: 0.into(), + last_probe: None, + large_drop: None, + last_stagnant: None, + pending: VecDeque::new(), + scheduled_exponential: None, + scheduled_periodic_alr: None, + scheduled_stagnant: None, + } + } +} + +impl ProbeControl { + pub fn new() -> Self { + Self::default() + } + + pub fn enable(&mut self, v: bool) { + if !self.enabled && v { + self.enabled = true; + self.request_immediate(); + } else if self.enabled && !v { + self.enabled = false; + self.pending.clear(); + self.last_estimate = None; + self.desired_bitrate = None; + self.last_estimate_change = None; + self.last_stagnant = None; + self.last_probe = None; + self.prev_estimate = None; + self.scheduled_exponential = None; + self.scheduled_periodic_alr = None; + self.scheduled_stagnant = None; + self.next_timeout = not_happening(); + } + } + + pub fn set_desired_bitrate(&mut self, v: Bitrate) { + // Don't accept Bitrate::ZERO as first ever value. + if self.desired_bitrate.is_none() && v.is_zero() { + return; + } + self.desired_bitrate = Some(v); + self.request_immediate(); + } + + pub fn set_estimated_bitrate(&mut self, v: Bitrate, cause: BandwidthLimitedCause) { + // Don't accept Bitrate::ZERO as first ever value. + if self.last_estimate.is_none() && v.is_zero() { + return; + } + + // Check if estimate changed significantly (>5%) or cause changed. + let dominated_by_last = self.last_estimate.is_some_and(|last| { + let upper = last * (1.0 + ESTIMATE_CHANGE_THRESHOLD); + let lower = last * (1.0 - ESTIMATE_CHANGE_THRESHOLD); + v <= upper && v >= lower + }); + + if dominated_by_last && self.last_cause == cause { + return; + } + + self.last_estimate = Some(v); + self.last_cause = cause; + self.request_immediate(); + } + + pub fn set_alr_start_time(&mut self, t: Instant) { + if self.alr_start.is_some() { + return; + } + self.alr_start = Some(t); + self.alr_stop = None; + self.request_immediate(); + } + + pub fn set_alr_stop_time(&mut self, t: Instant) { + if self.alr_start.is_none() || self.alr_stop.is_some() { + return; + } + self.alr_start = None; + self.alr_stop = Some(t); + self.request_immediate(); + } + + fn request_immediate(&mut self) { + self.next_timeout = already_happened(); + self.scheduled_exponential = None; + self.scheduled_periodic_alr = None; + self.scheduled_stagnant = None; + } + + pub fn poll_timeout(&self) -> Instant { + self.next_timeout + } + + pub fn handle_timeout(&mut self, now: Instant) -> Option { + // Spurious call before timeout is due - ignore. + if now < self.next_timeout { + return None; + } + + // Timeout fired - reset to not_happening until we compute the next one. + self.next_timeout = not_happening(); + + // Probing is disabled until first packet sent and padding queue exists. + if !self.enabled { + return None; + } + + // We need to have both desired AND last_estimate set to + // start considering probing. + let desired = self.desired_bitrate?; + let estimate = self.last_estimate?; + + // Return pending probes first. + if let Some(config) = self.pending.pop_front() { + // Schedule another. + self.request_immediate(); + return Some(config); + } + + // Can't probe in certain bandwidth-limited states. + if !self.can_probe(estimate) { + return None; + } + + // Try each probe type in order - only one fires per timeout. + let _ = self.maybe_initial(now, desired, estimate) + || self.maybe_exponential(now, desired, estimate) + || self.maybe_increase_alr(now, desired, estimate) + || self.maybe_large_drop(now, desired, estimate) + || self.maybe_periodic_alr(now, desired) + || self.maybe_stagnant(now, desired, estimate); + + self.update_estimate_change(now, estimate); + + // Update prev_estimate for next cycle (used by large drop and stagnation detection). + self.prev_estimate = Some(estimate); + + // Update timeout based on current state. + self.next_timeout = self.compute_next_timeout(now); + + if !self.pending.is_empty() { + self.request_immediate(); + } + + self.pending.pop_front() + } + + fn update_estimate_change(&mut self, now: Instant, estimate: Bitrate) { + // Track when estimate last changed significantly (>5%). + if let Some(prev) = self.prev_estimate { + if estimate != prev { + self.last_estimate_change = Some(now); + } + } + + // Initialize baseline if not set yet. + if self.last_estimate_change.is_none() { + self.last_estimate_change = Some(now); + } + } + + fn maybe_initial(&mut self, now: Instant, desired: Bitrate, estimate: Bitrate) -> bool { + // Initial probes only fire once at startup. + if self.last_probe.is_some() { + return false; + } + + // Queue 3× and 6× of estimate. + let p1 = estimate * self.config.first_exponential_probe_scale; + let p2 = estimate * self.config.second_exponential_probe_scale; + + self.queue_probe(p1, ProbeKind::Initial, desired, now); + self.queue_probe(p2, ProbeKind::Initial, desired, now); + true + } + + fn maybe_exponential(&mut self, now: Instant, desired: Bitrate, estimate: Bitrate) -> bool { + // Wait for pending probes to be dispatched first. + if !self.pending.is_empty() { + return false; + } + + // Need a previous probe to continue from. + let Some(last) = self.last_probe else { + return false; + }; + + // Estimate must exceed 70% of last probe rate to trigger further probing. + if estimate < last.further { + return false; + } + + let is_same = Some(estimate) == last.was_estimate; + let time_since = self.time_since_last_probe(now); + + // Don't re-probe at the same estimate; wait for new result or timeout. + if is_same && time_since < MAX_WAITING_TIME_FOR_PROBING_RESULT { + return false; + } + + let scale = self.last_cause.probe_scale(&self.config); + let target = estimate * scale; + + // Already probed at max rate; no point probing again. + let max = desired * MAX_PROBE_BITRATE_FACTOR; + if target >= max && last.further >= max * self.config.further_probe_threshold { + return false; + } + + self.queue_probe(target, ProbeKind::Exponential, desired, now); + + true + } + + fn maybe_increase_alr(&mut self, now: Instant, desired: Bitrate, estimate: Bitrate) -> bool { + // Don't interfere with initial probing phase. + if self.is_during_initial(now) { + return false; + } + + // Allocation probes only fire in ALR (application-limited region). + if !self.in_alr() { + return false; + } + + let prev = self.prev_desired; + self.prev_desired = Some(desired); + + // Need a previous desired value to compare against. + let Some(prev) = prev else { + return false; + }; + + // Only probe if desired increased + if desired <= prev { + return false; + } + + // No point probing if we already have enough bandwidth. + if desired <= estimate { + return false; + } + + // Allocation probes at 1× and 2× of desired, capped by 2× estimate + let current_bwe_limit = estimate * self.config.allocation_probe_limit_by_current_scale; + + let p1 = (desired * self.config.first_allocation_probe_scale).min(current_bwe_limit); + self.queue_probe(p1, ProbeKind::IncreaseAlr, desired, now); + + let p2 = desired * self.config.second_allocation_probe_scale; + if p2 <= current_bwe_limit && p2 > p1 { + self.queue_probe(p2, ProbeKind::IncreaseAlr, desired, now); + } + + true + } + + fn maybe_periodic_alr(&mut self, now: Instant, desired: Bitrate) -> bool { + // Don't interfere with initial probing phase. + if self.is_during_initial(now) { + return false; + } + + // Periodic probes only fire in ALR (application-limited region). + if !self.in_alr() { + return false; + } + + // Respect minimum interval between ALR probes. + if self.time_since_last_probe(now) < MIN_TIME_BETWEEN_ALR_PROBES { + return false; + } + + // Periodic ALR probe at 2× desired (capped by queue_probe to 2× desired anyway). + // Using desired rather than estimate allows discovering higher capacity when + // the app wants more bandwidth than currently estimated. + let target = desired * self.config.further_exponential_probe_scale; + self.queue_probe(target, ProbeKind::PeriodicAlr, desired, now); + true + } + + /// Probe when estimate has stagnated (no change for 15+ seconds) despite unmet demand. + /// + /// ## Why This Exists (str0m Addition) + /// + /// This probe type addresses a deadlock scenario in the BWE system where AIMD recovery + /// cannot make progress after network capacity is restored: + /// + /// **The Deadlock:** + /// 1. Network degrades from 5 Mbps → 1 Mbps, estimate drops to ~900 kbps + /// 2. Application reduces send rate to ~500 kbps (below estimate) + /// 3. Network recovers to 5 Mbps + /// 4. AIMD tries to increase but is capped at 1.5× observed throughput: + /// 500 kbps × 1.5 = 750 kbps maximum + /// 5. Sending at 500 kbps = 71% of estimate, which is above ALR threshold (65%) + /// 6. ALR never triggers → no periodic probing + /// 7. Large-drop probe requires ALR or recent ALR exit (see `maybe_large_drop`) + /// 8. System is stuck: estimate ~700 kbps on a 5 Mbps network + /// + /// **AIMD's 1.5× Cap (line 191 in rate_control.rs):** + /// `observed_bitrate * 1.5 + Bitrate::kbps(10)` + /// This prevents runaway growth beyond actual sending rate. It's conservative but + /// necessary - without it, the estimate could grow unbounded even when we're barely + /// sending anything. + /// + /// **ALR Detection Threshold:** + /// ALR triggers when sending < 65% of estimate consistently for 500ms with budget + /// accumulation > 80%. At 60-70% send rate, you're in the deadlock zone: too high + /// to trigger ALR, too low for AIMD to help much. + /// + /// **Loss Controller's 1.5× Cap:** + /// The loss controller also applies a 1.5× cap during recovery (line 303 in + /// loss_controller.rs), compounding the AIMD limitation. + /// + /// ## How This Differs from WebRTC + /// + /// WebRTC does not have stagnation-based probing. They rely on: + /// 1. Large-drop recovery probe (requires ALR or recent ALR exit) + /// 2. Rapid recovery field trial (`WebRTC-BweRapidRecoveryExperiment`) which removes + /// the ALR requirement from large-drop probes + /// + /// str0m adds stagnant probing as a complementary mechanism that: + /// - Catches deadlock regardless of whether a drop was detected + /// - Provides periodic escape from any stagnation scenario, not just post-drop + /// - Uses a conservative 15-second wait to avoid probing at convergence + /// - Rate-limited to once per 30 seconds to prevent oscillation + fn maybe_stagnant(&mut self, now: Instant, desired: Bitrate, estimate: Bitrate) -> bool { + // Don't interfere with initial probing phase. + if self.is_during_initial(now) { + return false; + } + + // Don't probe in ALR (periodic ALR handles that). + if self.in_alr() { + return false; + } + + let Some(last_change) = self.last_estimate_change else { + return false; + }; + + if now.saturating_duration_since(last_change) < MIN_TIME_BETWEEN_STAGNANT_PROBES { + return false; + } + + // Only if there's unmet demand. + if desired <= estimate { + return false; + } + + // Rate limit: at least 30 seconds between stagnation probes. + if let Some(last_probe) = self.last_stagnant { + if now.saturating_duration_since(last_probe) < MIN_TIME_BETWEEN_STAGNANT_PROBES { + return false; + } + } + + // Probe at 2× estimate (conservative, won't overwhelm if at capacity). + let probe_rate = estimate * STAGNANT_PROBE_SCALE; + self.queue_probe(probe_rate, ProbeKind::Stagnant, desired, now); + self.last_stagnant = Some(now); + + true + } + + fn maybe_large_drop(&mut self, now: Instant, desired: Bitrate, estimate: Bitrate) -> bool { + // Don't interfere with initial probing phase. + if self.is_during_initial(now) { + return false; + } + + // Detect large drops: estimate fell below 66% of previous. + if self.large_drop.is_none() { + if let Some(prev) = self.prev_estimate { + if estimate < prev * BITRATE_DROP_THRESHOLD { + self.large_drop = Some(LargeDrop { + when: now, + bitrate_before: prev, + }); + } + } + } + + // No large drop detected. + let Some(drop) = &self.large_drop else { + return false; + }; + + // Drop expires after 5 seconds. + if now.saturating_duration_since(drop.when) > BITRATE_DROP_TIMEOUT { + self.large_drop = None; + return false; + } + + // Large-drop probing requires ALR context (in ALR or recently exited). + if !self.in_alr() && !self.alr_ended_recently(now) { + return false; + } + + // Respect minimum interval between ALR probes. + if self.time_since_last_probe(now) < MIN_TIME_BETWEEN_ALR_PROBES { + return false; + } + + // Probe at 85% of pre-drop bitrate. + let target = drop.bitrate_before * PROBE_FRACTION_AFTER_DROP; + self.queue_probe(target, ProbeKind::LargeDrop, desired, now); + + self.large_drop = None; + true + } + + fn queue_probe(&mut self, bitrate: Bitrate, kind: ProbeKind, desired: Bitrate, now: Instant) { + // Cap at 2× desired bitrate. + let max = desired * MAX_PROBE_BITRATE_FACTOR; + let bitrate = bitrate.min(max); + + // No probe at too small values. + if bitrate < Bitrate::kbps(5) { + return; + } + + let cluster_id = self.next_cluster_id.inc(); + + let config = ProbeClusterConfig::new(cluster_id, bitrate, kind) + .with_min_packet_count(self.config.min_probe_packets_sent) + .with_duration(self.config.min_probe_duration) + .with_min_probe_delta(self.config.min_probe_delta); + + // Threshold for further exponential probing (probe_bitrate * 0.7). + let probe_further = bitrate * self.config.further_probe_threshold; + + self.pending.push_back(config); + self.last_probe = Some(LastProbe { + when: now, + kind, + further: probe_further, + was_estimate: self.last_estimate, + }); + } + + fn compute_next_timeout(&mut self, now: Instant) -> Instant { + // Exponential probing: wait for probe result before re-probing at same estimate. + // This handles the case where we sent a probe but haven't received updated estimate yet. + if let Some(last) = &self.last_probe { + if matches!(last.kind, ProbeKind::Initial | ProbeKind::Exponential) { + if self.scheduled_exponential.is_none() { + self.scheduled_exponential = Some(now + MAX_WAITING_TIME_FOR_PROBING_RESULT); + } + return self.scheduled_exponential.unwrap(); + } + } + + // ALR periodic probing + if self.in_alr() { + if self.scheduled_periodic_alr.is_none() { + self.scheduled_periodic_alr = Some(now + MIN_TIME_BETWEEN_ALR_PROBES); + } + return self.scheduled_periodic_alr.unwrap(); + } + + // Stagnant probing (only when not in ALR) + if !self.in_alr() { + if self.scheduled_stagnant.is_none() { + self.scheduled_stagnant = Some(now + MIN_TIME_BETWEEN_STAGNANT_PROBES); + } + return self.scheduled_stagnant.unwrap(); + } + + not_happening() + } + + fn can_probe(&self, estimate: Bitrate) -> bool { + // Infinite estimate indicates no valid measurement yet. + if estimate == Bitrate::INFINITY { + return false; + } + + // Only probe when delay-limited or loss-limited-but-increasing. + // Don't probe during active congestion (loss-limited, delay-increased). + matches!( + self.last_cause, + BandwidthLimitedCause::LossLimitedBweIncreasing + | BandwidthLimitedCause::DelayBasedLimited + ) + } + + fn in_alr(&self) -> bool { + self.alr_start.is_some() && self.alr_stop.is_none() + } + + fn alr_ended_recently(&self, now: Instant) -> bool { + self.alr_stop + .map(|stop| now.saturating_duration_since(stop) < ALR_ENDED_TIMEOUT) + .unwrap_or(false) + } + + fn is_during_initial(&self, now: Instant) -> bool { + let is_initial = matches!( + self.last_probe.map(|p| p.kind), + Some(ProbeKind::Initial) | Some(ProbeKind::Exponential) + ); + is_initial && self.time_since_last_probe(now) <= MAX_WAITING_TIME_FOR_PROBING_RESULT + } + + fn last_when(&self) -> Option { + self.last_probe.map(|p| p.when) + } + + fn time_since_last_probe(&self, now: Instant) -> Duration { + self.last_when() + .map(|t| now.saturating_duration_since(t)) + .unwrap_or(Duration::MAX) + } +} + +/// Configuration using WebRTC default constants (no field-trial plumbing). +#[derive(Debug, Clone, Copy)] +struct Config { + // Initial/exponential probing + first_exponential_probe_scale: f64, // p1 = 3.0 + second_exponential_probe_scale: f64, // p2 = 6.0 + further_exponential_probe_scale: f64, // step_size = 2.0 + further_probe_threshold: f64, // 0.7 + + // Allocation probing + first_allocation_probe_scale: f64, // 1.0 + second_allocation_probe_scale: f64, // 2.0 + allocation_probe_limit_by_current_scale: f64, // 2.0 + + // Probe cluster config defaults + min_probe_packets_sent: usize, // 5 + min_probe_duration: Duration, // 15ms + min_probe_delta: Duration, // 2ms + + // Gating / limits + loss_limited_probe_scale: f64, // 1.5 +} + +impl Default for Config { + fn default() -> Self { + Self { + first_exponential_probe_scale: 3.0, + second_exponential_probe_scale: 6.0, + further_exponential_probe_scale: 2.0, + further_probe_threshold: 0.7, + + first_allocation_probe_scale: 1.0, + second_allocation_probe_scale: 2.0, + allocation_probe_limit_by_current_scale: 2.0, + + min_probe_packets_sent: 5, + min_probe_duration: Duration::from_millis(15), + min_probe_delta: Duration::from_millis(2), + + loss_limited_probe_scale: 1.5, + } + } +} + +impl BandwidthLimitedCause { + /// Probe scale factor for exponential probing. + /// + /// When loss-limited but increasing, use a more conservative 1.575× (1.5 * 1.05). + /// Otherwise use the standard 2× scale. + fn probe_scale(&self, config: &Config) -> f64 { + match self { + BandwidthLimitedCause::LossLimitedBweIncreasing => { + config.loss_limited_probe_scale * (1.0 + PROBE_UNCERTAINTY) + } + _ => config.further_exponential_probe_scale, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn initial_exponential_probes_are_queued_and_emitted_one_per_tick() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(50)); + pc.set_estimated_bitrate(Bitrate::kbps(300), BandwidthLimitedCause::DelayBasedLimited); + + // First handle_timeout triggers initial probing and returns first probe. + let p1 = pc.handle_timeout(now).unwrap(); + + // poll_timeout returns already_happened while there are pending probes. + assert_eq!(pc.poll_timeout(), already_happened()); + + // Second handle_timeout returns the second queued probe. + let p2 = pc.handle_timeout(now).unwrap(); + + assert_eq!(p1.target_bitrate(), Bitrate::kbps(900)); + assert_eq!(p2.target_bitrate(), Bitrate::kbps(1800)); + assert_eq!(p1.min_packet_count(), 5); + assert_eq!(p1.min_probe_delta(), Duration::from_millis(2)); + assert!(!p1.is_alr_probe()); + + // Queue drained - no more probes. + assert!(pc.handle_timeout(now).is_none()); + } + + #[test] + fn further_probe_is_triggered_when_probe_result_is_high_enough() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.enable(true); + pc.set_desired_bitrate(Bitrate::mbps(50)); + pc.set_estimated_bitrate(Bitrate::mbps(1), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial two probes. + let _ = pc.handle_timeout(now).unwrap(); + let _ = pc.handle_timeout(now).unwrap(); + + // WebRTC rule: if measured bitrate > min_bitrate_to_probe_further, probe at 2x measured. + // min_bitrate_to_probe_further is 0.7 * last_probe_rate (6x start) = 4.2 Mbps. + pc.set_estimated_bitrate(Bitrate::mbps(5), BandwidthLimitedCause::DelayBasedLimited); + + let p = pc.handle_timeout(now + Duration::from_millis(10)).unwrap(); + assert_eq!(p.target_bitrate(), Bitrate::mbps(10)); + } + + #[test] + fn allocation_probe_is_triggered_in_alr_when_allocation_increases() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(1)); + pc.set_estimated_bitrate(Bitrate::mbps(1), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes. + let _ = pc.handle_timeout(now).unwrap(); + let _ = pc.handle_timeout(now).unwrap(); + + // Time out waiting for probing result -> probing complete. + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter ALR + pc.set_alr_start_time(now + Duration::from_secs(2)); + + // No probe yet - desired hasn't increased + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Increase desired bitrate while in ALR (desired > prev AND desired > estimate) + pc.set_desired_bitrate(Bitrate::mbps(4)); + + // Should trigger allocation probe: p1 = 4 Mbps * 1.0 = 4 Mbps, capped by 2× estimate = 2 Mbps + let p = pc.handle_timeout(now + Duration::from_secs(2)).unwrap(); + assert_eq!(p.target_bitrate(), Bitrate::mbps(2)); + } + + #[test] + fn handles_bitrate_infinity_without_panic() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(50)); + + // Should not panic with Infinity + pc.set_estimated_bitrate(Bitrate::INFINITY, BandwidthLimitedCause::DelayBasedLimited); + + // Verify behavior is reasonable (no probing with infinite estimate) + assert!(pc.handle_timeout(now).is_none()); + } + + #[test] + fn handles_clock_skew_gracefully() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(50)); + pc.set_estimated_bitrate(Bitrate::kbps(300), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Simulate time going backwards (clock skew) + let earlier = now - Duration::from_secs(5); + + // Should handle gracefully with saturating_duration_since + let _ = pc.handle_timeout(earlier); + + // Should still be able to continue normally + let _ = pc.handle_timeout(now + Duration::from_secs(1)); + } + + #[test] + fn handles_max_bitrate_zero() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + // Set max_bitrate to zero - this is rejected as first value to avoid + // creating probes with zero cap. + pc.set_desired_bitrate(Bitrate::ZERO); + pc.set_estimated_bitrate(Bitrate::kbps(300), BandwidthLimitedCause::DelayBasedLimited); + + // No probes should be created since desired was rejected. + let p1 = pc.handle_timeout(now); + assert!(p1.is_none(), "Should not create probes with zero desired"); + } + + #[test] + fn allocation_probe_fires_when_desired_increases_in_alr() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::kbps(500)); + pc.set_estimated_bitrate(Bitrate::kbps(500), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Timeout to reach probing complete + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter ALR + pc.set_alr_start_time(now + Duration::from_secs(3)); + + // No probe on ALR entry alone + assert!(pc.handle_timeout(now + Duration::from_secs(3)).is_none()); + + // Increase desired while in ALR (desired > prev AND desired > estimate) + pc.set_desired_bitrate(Bitrate::mbps(4)); + + // Should trigger allocation probe + let probe = pc.handle_timeout(now + Duration::from_secs(3)); + assert!( + probe.is_some(), + "Allocation probe should trigger when desired increases in ALR" + ); + } + + #[test] + fn large_drop_probing_after_alr_ended() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(5)); + pc.set_estimated_bitrate(Bitrate::mbps(5), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Timeout to probing complete + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter and exit ALR (large-drop works when ALR ended recently) + pc.set_alr_start_time(now + Duration::from_secs(2)); + pc.set_alr_stop_time(now + Duration::from_secs(3)); + + // Simulate large drop (to 60% of original = 3 Mbps, below 66% threshold) + pc.set_estimated_bitrate(Bitrate::mbps(3), BandwidthLimitedCause::DelayBasedLimited); + + // Check at now+5s (within 3s of ALR ending, so alr_ended_recently is true) + let later = now + Duration::from_secs(5); + + // Should trigger large-drop recovery probe at 85% of pre-drop rate (4.25 Mbps) + let p = pc.handle_timeout(later); + assert!(p.is_some(), "Large-drop recovery should schedule probe"); + if let Some(probe) = p { + // 85% of 5 Mbps = 4.25 Mbps + assert!(probe.target_bitrate() >= Bitrate::mbps(4)); + assert!(probe.target_bitrate() <= Bitrate::mbps(5)); + } + } + + #[test] + fn allocation_probe_requires_desired_increase_in_alr() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(5)); + pc.set_estimated_bitrate(Bitrate::mbps(1), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Timeout to probing complete + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter ALR with estimate < max_bitrate + pc.set_alr_start_time(now + Duration::from_secs(2)); + + // No allocation probe on ALR entry - need desired to increase + let probe = pc.handle_timeout(now + Duration::from_secs(2)); + assert!( + probe.is_none(), + "Should NOT trigger allocation probe on ALR entry alone" + ); + + // Increase desired while in ALR + pc.set_desired_bitrate(Bitrate::mbps(10)); + + // Now should trigger allocation probe + let probe = pc.handle_timeout(now + Duration::from_secs(2)); + assert!( + probe.is_some(), + "Should trigger allocation probe when desired increases in ALR" + ); + } + + #[test] + fn periodic_alr_probing() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(5)); + pc.set_estimated_bitrate(Bitrate::mbps(1), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Timeout to probing complete + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter ALR + pc.set_alr_start_time(now + Duration::from_secs(2)); + + // No immediate probe on ALR entry + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Wait 5 seconds for periodic probe (2s to complete initial + 5s = 7s) + let probe = pc.handle_timeout(now + Duration::from_secs(7)); + assert!( + probe.is_some(), + "Should trigger periodic ALR probe after 5 seconds in ALR" + ); + assert!(probe.unwrap().is_alr_probe()); + } + + #[test] + fn periodic_alr_probing_continues_even_when_estimate_reaches_max() { + let mut pc = ProbeControl::new(); + pc.enable(true); + let now = Instant::now(); + + pc.set_desired_bitrate(Bitrate::mbps(2)); + pc.set_estimated_bitrate(Bitrate::mbps(1), BandwidthLimitedCause::DelayBasedLimited); + + // Drain initial probes + let _ = pc.handle_timeout(now); + let _ = pc.handle_timeout(now); + + // Timeout to probing complete + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Enter ALR + pc.set_alr_start_time(now + Duration::from_secs(2)); + + // No immediate probe on ALR entry + assert!(pc.handle_timeout(now + Duration::from_secs(2)).is_none()); + + // Now increase estimate to match max_bitrate + pc.set_estimated_bitrate(Bitrate::mbps(2), BandwidthLimitedCause::DelayBasedLimited); + + // Wait 5 seconds - should still trigger periodic probe in ALR + // even though estimate >= max_bitrate, to maintain confidence in the estimate + let probe = pc.handle_timeout(now + Duration::from_secs(7)); + assert!( + probe.is_some(), + "Should continue periodic probing in ALR even when estimate >= max_bitrate" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/estimator.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/estimator.rs new file mode 100644 index 000000000..3279fa51c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/estimator.rs @@ -0,0 +1,696 @@ +use std::collections::VecDeque; +use std::fmt; +use std::time::{Duration, Instant}; + +use super::super::macros::log_probe_bitrate_estimate; +use super::ProbeClusterConfig; +use crate::rtp_::{Bitrate, DataSize, TwccClusterId, TwccSendRecord}; +use crate::util::not_happening; + +/// Minimum ratio of packets we need to receive for a valid probe (80%). +const MIN_RECEIVED_PROBES_RATIO: f64 = 0.80; + +/// Minimum ratio of bytes we need to receive for a valid probe (80%). +const MIN_RECEIVED_BYTES_RATIO: f64 = 0.80; + +/// Minimum packet count for a valid probe cluster (WebRTC's kMinClusterSize). +const MIN_CLUSTER_SIZE: usize = 4; + +/// The maximum valid duration between first and last probe packet on send/receive side. +/// Matches WebRTC's `kMaxProbeInterval` in `probe_bitrate_estimator.cc`. +const MAX_PROBE_INTERVAL: Duration = Duration::from_secs(1); + +/// The maximum |receive rate| / |send rate| ratio for a valid estimate. +/// Matches WebRTC's `kMaxValidRatio`. +const MAX_VALID_RATIO: f64 = 2.0; + +/// Minimum |receive rate| / |send rate| ratio to consider the link unsaturated. +/// Matches WebRTC's `kMinRatioForUnsaturatedLink`. +const MIN_RATIO_FOR_UNSATURATED_LINK: f64 = 0.9; + +/// Target utilization when we believe we've found the true capacity. +/// Matches WebRTC's `kTargetUtilizationFraction`. +const TARGET_UTILIZATION_FRACTION: f64 = 0.95; + +/// Maximum number of active probes before we trigger cleanup. +const MAX_ACTIVE_PROBES: usize = 20; + +/// Probes older than this are considered stale and will be removed when hitting the cap. +const STALE_PROBE_THRESHOLD: Duration = Duration::from_secs(5); + +/// Analyzes probe cluster results from TWCC feedback. +/// +/// This component takes packets tagged with a `TwccClusterId` and calculates the +/// achieved bitrate for each probe cluster. +/// +/// **Important:** This follows WebRTC's `ProbeBitrateEstimator` semantics: +/// only packets with a known remote receive timestamp are included in the probe +/// result. Probe packets reported as lost (no remote receive timestamp) are ignored. +#[derive(Debug)] +pub struct ProbeEstimator { + /// Active probe states (VecDeque for efficient front removal). + states: VecDeque, + + /// Clusters that were updated in the last call to `update`. + did_update: VecDeque, +} + +#[derive(Debug)] +struct ProbeEstimatorState { + /// Configuration of the active probe (targets for validation). + config: ProbeClusterConfig, + + /// When this probe was created. + created_at: Instant, + + /// When to erase this cluster's state (cluster history expiry). + finalize_at: Instant, + + /// First (earliest) send time among packets included in this probe. + first_send_time: Option, + /// Last (latest) send time among packets included in this probe. + last_send_time: Option, + /// Size of the packet with the last send time (excluded from send-rate calculation). + size_last_send: DataSize, + + /// First (earliest) receive time among packets included in this probe. + first_recv_time: Option, + /// Last (latest) receive time among packets included in this probe. + last_recv_time: Option, + /// Size of the packet with the first receive time (excluded from receive-rate calculation). + size_first_receive: DataSize, + + /// Total bytes for packets included in this probe (received packets only). + total_bytes: DataSize, + /// Number of packets included in this probe (received packets only). + packet_count: usize, +} + +impl ProbeEstimator { + pub fn new() -> Self { + Self { + states: VecDeque::new(), + did_update: VecDeque::with_capacity(10), + } + } + + /// Start analyzing a new probe cluster. + /// + /// Resets all accumulated state and begins watching for packets with the + /// given cluster ID. Returns `true` if the probe was started, `false` if + /// it was rejected due to too many active probes. + pub fn probe_start(&mut self, config: ProbeClusterConfig, now: Instant) -> bool { + // Under normal operation, we expect at most 2-4 active probes: + // - Initial exponential probing: 2 probes (3×, 6×) + // - Further probing: 1-2 additional probes + // - Allocation/recovery probing: 1-2 more + // Even with rapid probe sequences and 1-second cleanup delay, we shouldn't + // accumulate more than ~8 probes. If we hit 20, it indicates a bug in probe + // lifecycle management (missing end_probe() calls or handle_timeout() not running). + + // If we've hit the cap, try to clean up stale probes first. + if self.states.len() >= MAX_ACTIVE_PROBES { + let before_cleanup = self.states.len(); + self.states + .retain(|s| now.saturating_duration_since(s.created_at) < STALE_PROBE_THRESHOLD); + + let removed = before_cleanup - self.states.len(); + if removed > 0 { + debug!( + removed, + remaining = self.states.len(), + "Cleaned up stale probes" + ); + } + + // If still at cap after cleanup, reject the new probe. + if self.states.len() >= MAX_ACTIVE_PROBES { + debug!("Rejecting new probe: too many active probes with none stale"); + return false; + } + } + + self.states.push_back(ProbeEstimatorState::new(config, now)); + true + } + + /// Process TWCC feedback records. + /// + /// Only accumulates packets that match the active cluster ID. All other + /// packets are ignored. + pub fn update<'t>( + &mut self, + records: impl Iterator, + ) -> impl Iterator + '_ { + // Keep track of which clusters were updated in this call. + self.did_update.clear(); + + for record in records { + let Some(cluster) = record.cluster() else { + continue; + }; + + // Find the state for this cluster. + let maybe_state = self + .states + .iter_mut() + .find(|s| s.config.cluster() == cluster); + + let Some(state) = maybe_state else { + continue; + }; + + let did_update = state.update(record); + + if did_update { + // The correct behavior is that the _last updated_ + // is emitted last, so that the consumer of the returned + // iterator gets the latest probe result last. + self.did_update.retain(|c| *c != cluster); + self.did_update.push_back(cluster); + } + } + + self.did_update + .iter() + .filter_map(|cluster| self.states.iter().find(|s| s.config.cluster() == *cluster)) + .filter_map(|s| s.calculate_bitrate()) + } + + /// Mark the probe as ended. + /// + /// The probe will continue collecting feedback during a cluster history + /// period after the probe is finished. This period must be shorter than + /// the time between probe clusters to avoid overlap. + pub fn end_probe(&mut self, now: Instant, cluster_id: TwccClusterId) { + let maybe_state = self + .states + .iter_mut() + .find(|s| s.config.cluster() == cluster_id); + + let Some(state) = maybe_state else { + return; + }; + + state.end_probe(now); + } + + pub fn poll_timeout(&self) -> Instant { + self.states + .iter() + .map(|s| s.finalize_at) + .min() + .unwrap_or(not_happening()) + } + + /// Finalize probes that are ready. + pub fn handle_timeout(&mut self, now: Instant) { + self.states.retain(|s| { + let do_keep = now < s.finalize_at; + if do_keep { + return true; + } + + let result = s.do_calculate_bitrate(); + if let ProbeResult::Estimate(_) = result { + // Already logged in calculate_bitrate() during update(). + } else { + // Log the final rejection reason for the probe. + trace!(%result, "Probe result"); + } + + false + }); + } + + /// Clear all active probes. + /// + /// This should be called when probing is no longer possible. + pub fn clear_probes(&mut self) { + self.states.clear(); + } +} + +impl ProbeEstimatorState { + pub fn new(config: ProbeClusterConfig, now: Instant) -> Self { + Self { + config, + created_at: now, + finalize_at: not_happening(), + first_send_time: None, + last_send_time: None, + size_last_send: DataSize::ZERO, + first_recv_time: None, + last_recv_time: None, + size_first_receive: DataSize::ZERO, + total_bytes: DataSize::ZERO, + packet_count: 0, + } + } + + fn update(&mut self, record: &TwccSendRecord) -> bool { + // Only packets with a known remote receive time participate in probe estimation. + let Some(recv_time) = record.remote_recv_time() else { + return false; // lost/unreceived packet -> ignore for probe result + }; + + let packet_size = DataSize::from(record.size()); + let send_time = record.local_send_time(); + + // Track min/max send time among included packets. + let first = self.first_send_time.get_or_insert(send_time); + *first = (*first).min(send_time); + + let last = self.last_send_time.get_or_insert(send_time); + if send_time >= *last { + *last = send_time; + self.size_last_send = packet_size; + } + + // Track min/max receive time among included packets. + let first_recv = self.first_recv_time.get_or_insert(recv_time); + if recv_time <= *first_recv { + *first_recv = recv_time; + self.size_first_receive = packet_size; + } + + let last_recv = self.last_recv_time.get_or_insert(recv_time); + *last_recv = (*last_recv).max(recv_time); + + self.total_bytes += packet_size; + self.packet_count += 1; + + true + } + + fn calculate_bitrate(&self) -> Option<(ProbeClusterConfig, Bitrate)> { + let result = self.do_calculate_bitrate(); + + let ProbeResult::Estimate(bitrate) = result else { + return None; + }; + + // Log the estimates continuously during the probe. + trace!(%result, "Probe result"); + log_probe_bitrate_estimate!(bitrate.as_f64()); + + Some((self.config, bitrate)) + } + + /// Calculate the estimated bitrate for this probe cluster. + fn do_calculate_bitrate(&self) -> ProbeResult { + // WebRTC requires at least kMinClusterSize (4) packets received. + // We may send more, but packet loss can result in fewer received packets. + if self.packet_count < MIN_CLUSTER_SIZE { + return ProbeResult::ClusterTooSmall { + recv: self.packet_count, + limit: MIN_CLUSTER_SIZE, + }; + } + + // Also check we received enough of what was sent + let min_packets = + (self.config.min_packet_count() as f64 * MIN_RECEIVED_PROBES_RATIO) as usize; + let min_bytes = DataSize::bytes( + (self.config.target_bytes().as_bytes_usize() as f64 * MIN_RECEIVED_BYTES_RATIO) as i64, + ); + + if self.packet_count < min_packets { + return ProbeResult::InsufficientPackets { + recv: self.packet_count, + limit: min_packets, + }; + } + if self.total_bytes < min_bytes { + return ProbeResult::InsufficientBytes { + recv: self.total_bytes, + limit: min_bytes, + }; + } + + // Get timing bounds + let Some(first_send) = self.first_send_time else { + return ProbeResult::MissingTimingInfo; + }; + let Some(last_send) = self.last_send_time else { + return ProbeResult::MissingTimingInfo; + }; + let send_interval = last_send.saturating_duration_since(first_send); + + let Some(first_recv) = self.first_recv_time else { + return ProbeResult::MissingTimingInfo; + }; + let Some(last_recv) = self.last_recv_time else { + return ProbeResult::MissingTimingInfo; + }; + let recv_interval = last_recv.saturating_duration_since(first_recv); + + // Intervals must be positive and within bounds. + if send_interval.is_zero() { + return ProbeResult::SendIntervalInvalid { + interval: send_interval, + }; + } + if send_interval > MAX_PROBE_INTERVAL { + return ProbeResult::SendIntervalTooLong { + interval: send_interval, + }; + } + if recv_interval.is_zero() || recv_interval > MAX_PROBE_INTERVAL { + return ProbeResult::RecvIntervalInvalid { + interval: recv_interval, + }; + } + + // WebRTC boundary exclusions: + // - exclude the last sent packet size when computing send rate + // - exclude the first received packet size when computing receive rate + let send_size = self.total_bytes.saturating_sub(self.size_last_send); + let recv_size = self.total_bytes.saturating_sub(self.size_first_receive); + if send_size <= DataSize::ZERO || recv_size <= DataSize::ZERO { + return ProbeResult::InvalidDataSize; + } + + let recv_rate = recv_size / recv_interval; + let send_rate = send_size / send_interval; + + // WebRTC validation: reject if receive/send ratio is too high. + let ratio = recv_rate.as_f64() / send_rate.as_f64(); + if ratio > MAX_VALID_RATIO { + return ProbeResult::InvalidSendReceiveRatio { + ratio, + limit: MAX_VALID_RATIO, + }; + } + + // Match WebRTC semantics: + // - estimate is the min(send_rate, recv_rate) + // - if recv_rate is significantly lower than send_rate, assume saturation and + // return a conservative fraction of recv_rate. + let mut estimate = send_rate.min(recv_rate); + if recv_rate < send_rate * MIN_RATIO_FOR_UNSATURATED_LINK { + estimate = recv_rate * TARGET_UTILIZATION_FRACTION; + } + + ProbeResult::Estimate(estimate) + } + + fn end_probe(&mut self, now: Instant) { + self.finalize_at = now + Duration::from_secs(1); + } +} + +/// Result of a probe cluster estimation. +#[derive(Debug, Clone, Copy, PartialEq)] +enum ProbeResult { + /// Successfully estimated bitrate + Estimate(Bitrate), + /// Not enough packets in cluster (< 4) + ClusterTooSmall { recv: usize, limit: usize }, + /// Insufficient packets received (< 80% of sent) + InsufficientPackets { recv: usize, limit: usize }, + /// Insufficient bytes received (< 80% of sent) + InsufficientBytes { recv: DataSize, limit: DataSize }, + /// Send interval too long (> 1 second) + SendIntervalTooLong { interval: Duration }, + /// Send interval invalid (zero) + SendIntervalInvalid { interval: Duration }, + /// Receive interval invalid (zero or > 1 second) + RecvIntervalInvalid { interval: Duration }, + /// Invalid receive/send ratio (recv_rate / send_rate too high) + InvalidSendReceiveRatio { ratio: f64, limit: f64 }, + /// Calculated data size is zero + InvalidDataSize, + /// Missing timing information + MissingTimingInfo, +} + +impl fmt::Display for ProbeResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProbeResult::Estimate(bitrate) => write!(f, "estimate={}", bitrate), + ProbeResult::ClusterTooSmall { + recv: received, + limit: required, + } => { + write!(f, "cluster too small ({} < {})", received, required) + } + ProbeResult::InsufficientPackets { + recv: received, + limit: required, + } => { + write!(f, "insufficient packets ({} < {})", received, required) + } + ProbeResult::InsufficientBytes { + recv: received, + limit: required, + } => { + write!(f, "insufficient bytes ({} < {})", received, required) + } + ProbeResult::SendIntervalTooLong { interval } => { + write!(f, "send interval too long ({:?})", interval) + } + ProbeResult::SendIntervalInvalid { interval } => { + write!(f, "send interval invalid ({:?})", interval) + } + ProbeResult::RecvIntervalInvalid { interval } => { + write!(f, "recv interval invalid ({:?})", interval) + } + ProbeResult::InvalidSendReceiveRatio { ratio, limit } => { + write!(f, "invalid receive/send ratio ({ratio:.3} > {limit:.3})") + } + ProbeResult::InvalidDataSize => write!(f, "invalid data size"), + ProbeResult::MissingTimingInfo => write!(f, "missing timing info"), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::bwe_::probe::ProbeKind; + use crate::rtp_::{TwccPacketId, TwccSeq}; + + #[test] + fn probe_estimator_starts_with_no_active_probe() { + let estimator = ProbeEstimator::new(); + assert_eq!(estimator.poll_timeout(), not_happening()); + } + + #[test] + fn probe_estimator_lifecycle() { + let mut estimator = ProbeEstimator::new(); + let now = Instant::now(); + + // Start probe + let config = ProbeClusterConfig::new(1.into(), Bitrate::mbps(2), ProbeKind::Initial); + assert!(estimator.probe_start(config, now)); + assert!(estimator.states.len() == 1, "Should have one active probe"); + assert_eq!(estimator.poll_timeout(), not_happening()); + + // End probe with 1 second cluster history retention + estimator.end_probe(now, config.cluster()); + let timeout = estimator.poll_timeout(); + assert!( + timeout > now && timeout <= now + Duration::from_secs(1), + "Expected timeout between now and now+1s, got: {:?}", + timeout.duration_since(now) + ); + + // Handle timeout clears expired probes + estimator.handle_timeout(now + Duration::from_secs(1)); + assert!(estimator.states.is_empty(), "All probes should be cleared"); + assert_eq!(estimator.poll_timeout(), not_happening()); + } + + #[test] + fn lost_probe_packets_do_not_affect_estimate() { + let mut estimator = ProbeEstimator::new(); + let cluster: TwccClusterId = 7.into(); + let config = ProbeClusterConfig::new(cluster, Bitrate::mbps(2), ProbeKind::Initial); + + let base = Instant::now(); + let received = (0..5).map(|i| { + let seq: TwccSeq = (1000 + i).into(); + let pid = TwccPacketId::with_cluster(seq, cluster); + // send spaced 4ms, recv spaced 4ms (same ordering) + crate::rtp_::TwccSendRecord::test_new( + pid, + base + Duration::from_millis(i as u64 * 4), + 1200, + base + Duration::from_millis(i as u64 * 4 + 1), + Some(base + Duration::from_millis(i as u64 * 4 + 2)), + ) + }); + + // Add extra lost probe packets with later send times. These should not change the result. + let lost = (0..20).map(|i| { + let seq: TwccSeq = (2000 + i).into(); + let pid = TwccPacketId::with_cluster(seq, cluster); + crate::rtp_::TwccSendRecord::test_new( + pid, + base + Duration::from_millis(100 + i as u64), + 1200, + base + Duration::from_millis(150 + i as u64), + None, // lost + ) + }); + + // First run: only received packets + estimator.probe_start(config, base); + let recv_vec: Vec<_> = received.collect(); + let results: Vec<_> = estimator.update(recv_vec.iter()).collect(); + let estimate_only_received = results + .last() + .map(|(_, bitrate)| *bitrate) + .expect("expected a probe estimate"); + + // Second run: received + lost + let mut estimator2 = ProbeEstimator::new(); + estimator2.probe_start(config, base); + let mut all_vec = recv_vec; + all_vec.extend(lost); + let results: Vec<_> = estimator2.update(all_vec.iter()).collect(); + let estimate_with_lost = results + .last() + .map(|(_, bitrate)| *bitrate) + .expect("expected a probe estimate"); + + assert_eq!( + estimate_only_received, estimate_with_lost, + "lost packets must not change probe estimate" + ); + } + + #[test] + fn invalid_receive_send_ratio_is_rejected() { + let mut estimator = ProbeEstimator::new(); + let cluster: TwccClusterId = 9.into(); + let config = ProbeClusterConfig::new(cluster, Bitrate::mbps(2), ProbeKind::Initial); + + let base = Instant::now(); + // Make send times span 200ms, but receive times span only 1ms. + // This yields receive_rate >> send_rate. WebRTC would reject this via ratio check. + let records: Vec<_> = (0..5) + .map(|i| { + let seq: TwccSeq = (3000 + i).into(); + let pid = TwccPacketId::with_cluster(seq, cluster); + crate::rtp_::TwccSendRecord::test_new( + pid, + base + Duration::from_millis(i as u64 * 50), + 1200, + base + Duration::from_millis(250 + i as u64), + Some(base + Duration::from_millis(300 + (i as u64 % 2))), // ~0-1ms spread + ) + }) + .collect(); + + estimator.probe_start(config, base); + let results: Vec<_> = estimator.update(records.iter()).collect(); + + assert!( + results.is_empty(), + "probe should be rejected by ratio validation, got: {:?}", + results + ); + } + + #[test] + fn send_interval_zero_is_rejected() { + let mut estimator = ProbeEstimator::new(); + let cluster: TwccClusterId = 10.into(); + let config = ProbeClusterConfig::new(cluster, Bitrate::mbps(2), ProbeKind::Initial); + + let base = Instant::now(); + // All packets have the same send time -> send_interval == 0. + let records: Vec<_> = (0..5) + .map(|i| { + let seq: TwccSeq = (4000 + i).into(); + let pid = TwccPacketId::with_cluster(seq, cluster); + crate::rtp_::TwccSendRecord::test_new( + pid, + base, // identical send time for all + 1200, + base + Duration::from_millis(10 + i as u64), + Some(base + Duration::from_millis(20 + i as u64)), + ) + }) + .collect(); + + estimator.probe_start(config, base); + let results: Vec<_> = estimator.update(records.iter()).collect(); + + assert!( + results.is_empty(), + "send_interval == 0 should be rejected, got: {:?}", + results + ); + } + + #[test] + fn stale_probes_cleaned_up_at_capacity() { + let mut estimator = ProbeEstimator::new(); + let base = Instant::now(); + + // Fill up to MAX_ACTIVE_PROBES with probes created at `base` + for i in 0..MAX_ACTIVE_PROBES { + let config = + ProbeClusterConfig::new((i as u64).into(), Bitrate::mbps(2), ProbeKind::Initial); + assert!( + estimator.probe_start(config, base), + "should accept probe {i}" + ); + } + assert_eq!(estimator.states.len(), MAX_ACTIVE_PROBES); + + // Try to add another probe at the same time - should be rejected + let config = ProbeClusterConfig::new(100.into(), Bitrate::mbps(2), ProbeKind::Initial); + assert!( + !estimator.probe_start(config, base), + "should reject probe when at capacity with no stale probes" + ); + assert_eq!(estimator.states.len(), MAX_ACTIVE_PROBES); + + // Now try adding a probe 6 seconds later - all existing probes are stale + let later = base + Duration::from_secs(6); + let config = ProbeClusterConfig::new(101.into(), Bitrate::mbps(2), ProbeKind::Initial); + assert!( + estimator.probe_start(config, later), + "should accept probe after cleaning stale ones" + ); + // All old probes should be cleaned, leaving only the new one + assert_eq!(estimator.states.len(), 1); + } + + #[test] + fn non_stale_probes_preserved_during_cleanup() { + let mut estimator = ProbeEstimator::new(); + let base = Instant::now(); + + // Add some old probes + for i in 0..15 { + let config = + ProbeClusterConfig::new((i as u64).into(), Bitrate::mbps(2), ProbeKind::Initial); + estimator.probe_start(config, base); + } + + // Add some newer probes (3 seconds later, still within threshold) + let mid = base + Duration::from_secs(3); + for i in 15..MAX_ACTIVE_PROBES { + let config = + ProbeClusterConfig::new((i as u64).into(), Bitrate::mbps(2), ProbeKind::Initial); + estimator.probe_start(config, mid); + } + assert_eq!(estimator.states.len(), MAX_ACTIVE_PROBES); + + // Try to add at 6 seconds - old probes are stale, mid probes are not + let later = base + Duration::from_secs(6); + let config = ProbeClusterConfig::new(100.into(), Bitrate::mbps(2), ProbeKind::Initial); + assert!( + estimator.probe_start(config, later), + "should accept after cleaning only stale probes" + ); + // 15 old probes removed, 5 mid probes kept, 1 new probe added = 6 + assert_eq!(estimator.states.len(), 6); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/mod.rs new file mode 100644 index 000000000..621c38e87 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/probe/mod.rs @@ -0,0 +1,7 @@ +mod cluster; +mod control; +mod estimator; + +pub use cluster::{ProbeClusterConfig, ProbeClusterState, ProbeKind}; +pub use control::{BandwidthLimitedCause, ProbeControl}; +pub use estimator::ProbeEstimator; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/smoother.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/smoother.rs new file mode 100644 index 000000000..cfc4216d4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/smoother.rs @@ -0,0 +1,96 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use crate::rtp_::Bitrate; + +/// Time window for averaging estimate changes. +const ESTIMATE_WINDOW: Duration = Duration::from_secs(3); + +type TimeBitrate = (Instant, Bitrate); + +const TOLERANCE: f64 = 0.05; + +/// Smooths BWE estimates by averaging over a time window. +pub struct EstimateSmoother { + estimates: VecDeque, + maybe_emit: bool, + emitted: Option, +} + +impl EstimateSmoother { + pub fn new() -> Self { + Self { + estimates: VecDeque::new(), + maybe_emit: false, + emitted: None, + } + } + + /// Record a new estimate and update the smoothed average. + pub fn record(&mut self, now: Instant, estimate: Bitrate) { + // Did value change from previous? + let do_update = self.estimates.back().map(|b| b.1) != Some(estimate); + + if do_update { + self.maybe_emit = true; + self.estimates.push_back((now, estimate)); + } + + // Remove entries older than the window. + while let Some((time, _)) = self.estimates.front() { + if now.duration_since(*time) > ESTIMATE_WINDOW { + // Keep last entry + if self.estimates.len() == 1 { + break; + } + + self.maybe_emit = true; + self.estimates.pop_front(); + } else { + break; + } + } + } + + /// Poll for an estimate to emit. Returns Some only when there's a new value to emit. + pub fn poll(&mut self) -> Option { + if !self.maybe_emit { + return None; + } + + if self.estimates.is_empty() { + return None; + } + + let total: f64 = self.estimates.iter().map(|b| b.1.as_f64()).sum(); + let avg = total / self.estimates.len() as f64; + let rate: Bitrate = avg.into(); + + // This forces emitting if we have the first ever value, or a last + // where the rest of the window is gone (estimates stop coming). + let force = self.estimates.len() == 1 && self.emitted != Some(rate); + + // Emit if we deviate enough from previously emitted. + let deviate = if let Some(emitted) = self.emitted { + !in_tolerance(emitted, rate) + } else { + true + }; + + self.maybe_emit = false; + + // Are we not to emit? + if !force && !deviate { + return None; + } + + self.emitted = Some(rate); + Some(rate) + } +} + +fn in_tolerance(b1: Bitrate, b2: Bitrate) -> bool { + let min = b1 * (1.0 - TOLERANCE); + let max = b1 * (1.0 + TOLERANCE); + b2 > min && b2 < max +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/time.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/time.rs new file mode 100644 index 000000000..53c8296e4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/bwe/time.rs @@ -0,0 +1,760 @@ +use std::cmp::Ordering; +use std::fmt; +use std::ops::{Add, AddAssign, Div, Neg as _, Sub, SubAssign}; +use std::time::{Duration, Instant}; + +use crate::bwe::Bitrate; +use crate::rtp_::DataSize; + +/// Wrapper for [`Instant`] that provides additional time points in the past or future. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Timestamp { + /// A time in the past that already happened. + DistantPast, + + /// An exact instant. + Exact(Instant), + + /// A time in the future that will never happen. + DistantFuture, +} + +/// Wrapper for [`Duration`] that can be negative and provides a duration to a +/// distant future or past. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TimeDelta { + /// Time delta to some event in distant past that already happened. + NegativeInfinity, + + /// An exact negative duration. + Negative(Duration), + + /// An exact positive duration. + Positive(Duration), + + /// Time delta to some event in distant future that will never happen. + PositiveInfinity, +} + +impl TimeDelta { + pub(super) const ZERO: Self = Self::Positive(Duration::ZERO); + + /// Returns the number of seconds contained by this [`TimeDelta`] as `f64`. + pub fn as_secs_f64(&self) -> f64 { + match self { + Self::NegativeInfinity => f64::NEG_INFINITY, + Self::Negative(d) => d.as_secs_f64().neg(), + Self::Positive(d) => d.as_secs_f64(), + Self::PositiveInfinity => f64::INFINITY, + } + } +} + +#[cfg(test)] +impl TimeDelta { + /// Creates a [`TimeDelta`] from seconds. + pub const fn from_secs(secs: i64) -> TimeDelta { + if secs >= 0 { + Self::Positive(Duration::from_secs(secs as u64)) + } else { + Self::Negative(Duration::from_secs(-secs as u64)) + } + } + + /// Creates a [`TimeDelta`] from milliseconds. + pub const fn from_millis(millis: i64) -> Self { + if millis >= 0 { + Self::Positive(Duration::from_millis(millis as u64)) + } else { + Self::Negative(Duration::from_millis(-millis as u64)) + } + } +} + +impl Timestamp { + /// Indicates whether this [`Timestamp`] is [`Timestamp::Exact`]. + pub const fn is_exact(&self) -> bool { + matches!(self, Self::Exact(_)) + } +} + +impl Add for Timestamp { + type Output = Self; + + fn add(self, rhs: TimeDelta) -> Self::Output { + match (self, rhs) { + (Self::DistantFuture, _) | (_, TimeDelta::PositiveInfinity) => Self::DistantFuture, + (Self::DistantPast, _) | (_, TimeDelta::NegativeInfinity) => Self::DistantPast, + (Self::Exact(i), TimeDelta::Negative(d)) => Self::Exact(i - d), + (Self::Exact(i), TimeDelta::Positive(d)) => Self::Exact(i + d), + } + } +} + +impl Sub for Timestamp { + type Output = Self; + + fn sub(self, rhs: TimeDelta) -> Self::Output { + match (self, rhs) { + (Self::DistantFuture, _) | (_, TimeDelta::NegativeInfinity) => Self::DistantFuture, + (Self::DistantPast, _) | (_, TimeDelta::PositiveInfinity) => Self::DistantPast, + (Self::Exact(i), TimeDelta::Negative(d)) => Self::Exact(i + d), + (Self::Exact(i), TimeDelta::Positive(d)) => Self::Exact(i - d), + } + } +} + +impl Sub for Timestamp { + type Output = TimeDelta; + + fn sub(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::DistantFuture, _) | (_, Self::DistantPast) => TimeDelta::PositiveInfinity, + (Self::DistantPast, _) | (_, Self::DistantFuture) => TimeDelta::NegativeInfinity, + (Self::Exact(this), Self::Exact(that)) => match this.cmp(&that) { + Ordering::Less => TimeDelta::Negative(that - this), + Ordering::Equal => TimeDelta::ZERO, + Ordering::Greater => TimeDelta::Positive(this - that), + }, + } + } +} + +impl Add for Timestamp { + type Output = Self; + + fn add(self, rhs: Duration) -> Self::Output { + self + TimeDelta::from(rhs) + } +} + +impl Sub for Timestamp { + type Output = Self; + + fn sub(self, rhs: Duration) -> Self::Output { + self - TimeDelta::from(rhs) + } +} + +impl Sub for Timestamp { + type Output = TimeDelta; + + fn sub(self, rhs: Instant) -> Self::Output { + self.sub(Self::from(rhs)) + } +} + +impl SubAssign for Timestamp { + fn sub_assign(&mut self, rhs: TimeDelta) { + *self = *self - rhs; + } +} + +impl AddAssign for Timestamp { + fn add_assign(&mut self, rhs: TimeDelta) { + *self = *self + rhs; + } +} + +impl SubAssign for Timestamp { + fn sub_assign(&mut self, rhs: Duration) { + *self = *self - rhs; + } +} + +impl AddAssign for Timestamp { + fn add_assign(&mut self, rhs: Duration) { + *self = *self + rhs; + } +} + +impl PartialOrd for Timestamp { + fn partial_cmp(&self, other: &Self) -> Option { + Some(Self::cmp(self, other)) + } +} + +impl Ord for Timestamp { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (Self::DistantPast, Self::DistantPast) => Ordering::Equal, + (Self::DistantPast, _) => Ordering::Less, + (_, Self::DistantPast) => Ordering::Greater, + (Self::DistantFuture, Self::DistantFuture) => Ordering::Equal, + (Self::DistantFuture, _) => Ordering::Greater, + (_, Self::DistantFuture) => Ordering::Less, + (Self::Exact(v1), Self::Exact(v2)) => v1.cmp(v2), + } + } +} + +impl From for Timestamp { + fn from(value: Instant) -> Self { + Self::Exact(value) + } +} + +impl Add for TimeDelta { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::PositiveInfinity, _) | (_, Self::PositiveInfinity) => Self::PositiveInfinity, + (Self::NegativeInfinity, _) | (_, Self::NegativeInfinity) => Self::NegativeInfinity, + (Self::Negative(this), Self::Negative(that)) => Self::Negative(this + that), + (Self::Positive(this), Self::Positive(that)) => Self::Positive(this + that), + (Self::Positive(this), Self::Negative(that)) => match this.cmp(&that) { + Ordering::Less => Self::Negative(that - this), + Ordering::Equal => Self::ZERO, + Ordering::Greater => Self::Positive(this - that), + }, + (Self::Negative(this), Self::Positive(that)) => match this.cmp(&that) { + Ordering::Less => Self::Positive(that - this), + Ordering::Equal => Self::ZERO, + Ordering::Greater => Self::Negative(this - that), + }, + } + } +} + +impl Sub for TimeDelta { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::PositiveInfinity, _) | (_, Self::NegativeInfinity) => Self::PositiveInfinity, + (Self::NegativeInfinity, _) | (_, Self::PositiveInfinity) => Self::NegativeInfinity, + (Self::Positive(this), Self::Negative(that)) => Self::Positive(this + that), + (Self::Negative(this), Self::Positive(that)) => Self::Negative(this + that), + (Self::Positive(this), Self::Positive(that)) => match this.cmp(&that) { + Ordering::Less => Self::Negative(that - this), + Ordering::Equal => Self::ZERO, + Ordering::Greater => Self::Positive(this - that), + }, + (Self::Negative(this), Self::Negative(that)) => match this.cmp(&that) { + Ordering::Less => Self::Positive(that - this), + Ordering::Equal => Self::ZERO, + Ordering::Greater => Self::Negative(this - that), + }, + } + } +} + +impl PartialOrd for TimeDelta { + fn partial_cmp(&self, other: &Self) -> Option { + Some(Self::cmp(self, other)) + } +} + +impl Ord for TimeDelta { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (Self::NegativeInfinity, Self::NegativeInfinity) => Ordering::Equal, + (Self::NegativeInfinity, _) => Ordering::Less, + (_, Self::NegativeInfinity) => Ordering::Greater, + (Self::PositiveInfinity, Self::PositiveInfinity) => Ordering::Equal, + (Self::PositiveInfinity, _) => Ordering::Greater, + (_, Self::PositiveInfinity) => Ordering::Less, + (Self::Negative(_), Self::Positive(_)) => Ordering::Less, + (Self::Positive(_), Self::Negative(_)) => Ordering::Greater, + (Self::Positive(this), Self::Positive(that)) => this.cmp(that), + (Self::Negative(this), Self::Negative(that)) => that.cmp(this), + } + } +} + +impl PartialEq for TimeDelta { + fn eq(&self, other: &Duration) -> bool { + *self == Self::from(*other) + } +} + +impl PartialOrd for TimeDelta { + fn partial_cmp(&self, other: &Duration) -> Option { + Some(Self::cmp(self, &Self::from(*other))) + } +} + +impl SubAssign for TimeDelta { + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } +} + +impl AddAssign for TimeDelta { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + +impl Div for TimeDelta { + type Output = Self; + + #[inline] + fn div(self, rhs: u32) -> Self { + match self { + Self::NegativeInfinity | Self::PositiveInfinity => self, + Self::Negative(duration) => Self::Negative(duration / rhs), + Self::Positive(duration) => Self::Positive(duration / rhs), + } + } +} + +impl From for TimeDelta { + fn from(value: Duration) -> Self { + Self::Positive(value) + } +} + +impl Div for DataSize { + type Output = Bitrate; + + fn div(self, rhs: TimeDelta) -> Self::Output { + let bytes = self.as_bytes_f64(); + let s = rhs.as_secs_f64(); + + if s == 0.0 { + return Bitrate::ZERO; + } + + let bps = (bytes * 8.0) / s; + + bps.into() + } +} + +impl fmt::Display for TimeDelta { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TimeDelta::NegativeInfinity => write!(f, "-Inf"), + TimeDelta::Negative(v) => write!(f, "-{:.03}", v.as_secs_f32()), + TimeDelta::Positive(v) => write!(f, "{:.03}", v.as_secs_f32()), + TimeDelta::PositiveInfinity => write!(f, "+Inf"), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn instant_add_duration() { + let now = Instant::now(); + + assert_eq!( + Timestamp::Exact(now) + TimeDelta::from_secs(5), + Timestamp::from(now + Duration::from_secs(5)) + ); + assert_eq!( + Timestamp::Exact(now) + TimeDelta::from_secs(-5), + Timestamp::from(now - Duration::from_secs(5)) + ); + assert_eq!( + Timestamp::Exact(now) + TimeDelta::NegativeInfinity, + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::Exact(now) + TimeDelta::PositiveInfinity, + Timestamp::DistantFuture + ); + + assert_eq!( + Timestamp::DistantPast + TimeDelta::from_secs(5), + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::DistantPast + TimeDelta::from_secs(-5), + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::DistantPast + TimeDelta::NegativeInfinity, + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::DistantPast + TimeDelta::PositiveInfinity, + Timestamp::DistantFuture + ); + + assert_eq!( + Timestamp::DistantFuture + TimeDelta::from_secs(5), + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture + TimeDelta::from_secs(-5), + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture + TimeDelta::NegativeInfinity, + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture + TimeDelta::PositiveInfinity, + Timestamp::DistantFuture + ); + } + + #[test] + fn instant_sub_duration() { + let now = Instant::now(); + + assert_eq!( + Timestamp::Exact(now) - TimeDelta::from_secs(5), + Timestamp::from(now - Duration::from_secs(5)) + ); + assert_eq!( + Timestamp::Exact(now) - TimeDelta::from_secs(-5), + Timestamp::from(now + Duration::from_secs(5)) + ); + assert_eq!( + Timestamp::Exact(now) - TimeDelta::NegativeInfinity, + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::Exact(now) - TimeDelta::PositiveInfinity, + Timestamp::DistantPast + ); + + assert_eq!( + Timestamp::DistantPast - TimeDelta::from_secs(5), + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::DistantPast - TimeDelta::from_secs(-5), + Timestamp::DistantPast + ); + assert_eq!( + Timestamp::DistantPast - TimeDelta::NegativeInfinity, + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantPast - TimeDelta::PositiveInfinity, + Timestamp::DistantPast + ); + + assert_eq!( + Timestamp::DistantFuture - TimeDelta::from_secs(5), + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture - TimeDelta::from_secs(-5), + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture - TimeDelta::NegativeInfinity, + Timestamp::DistantFuture + ); + assert_eq!( + Timestamp::DistantFuture - TimeDelta::PositiveInfinity, + Timestamp::DistantFuture + ); + } + + #[test] + fn instant_sub_instant() { + let now = Instant::now(); + + assert_eq!( + Timestamp::Exact(now) - Timestamp::Exact(now), + TimeDelta::ZERO + ); + assert_eq!( + Timestamp::Exact(now) - Timestamp::Exact(now - Duration::from_secs(5)), + TimeDelta::from_secs(5) + ); + assert_eq!( + Timestamp::Exact(now) - Timestamp::Exact(now + Duration::from_secs(5)), + TimeDelta::from_secs(-5) + ); + assert_eq!( + Timestamp::Exact(now) - Timestamp::DistantPast, + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::Exact(now) - Timestamp::DistantFuture, + TimeDelta::NegativeInfinity + ); + + assert_eq!( + Timestamp::DistantPast - Timestamp::Exact(now), + TimeDelta::NegativeInfinity + ); + assert_eq!( + Timestamp::DistantPast - Timestamp::Exact(now - Duration::from_secs(5)), + TimeDelta::NegativeInfinity + ); + assert_eq!( + Timestamp::DistantPast - Timestamp::Exact(now + Duration::from_secs(5)), + TimeDelta::NegativeInfinity + ); + assert_eq!( + Timestamp::DistantPast - Timestamp::DistantPast, + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::DistantPast - Timestamp::DistantFuture, + TimeDelta::NegativeInfinity + ); + + assert_eq!( + Timestamp::DistantFuture - Timestamp::Exact(now), + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::DistantFuture - Timestamp::Exact(now - Duration::from_secs(5)), + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::DistantFuture - Timestamp::Exact(now + Duration::from_secs(5)), + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::DistantFuture - Timestamp::DistantPast, + TimeDelta::PositiveInfinity + ); + assert_eq!( + Timestamp::DistantFuture - Timestamp::DistantFuture, + TimeDelta::PositiveInfinity + ); + } + + #[test] + fn instant_ord() { + let now = Timestamp::Exact(Instant::now()); + let now_minus_1 = now - TimeDelta::from_secs(1); + let now_plus_1 = now + TimeDelta::from_secs(1); + + assert!(Timestamp::DistantFuture > now_plus_1); + assert!(Timestamp::DistantFuture > now_minus_1); + assert!(Timestamp::DistantFuture > Timestamp::DistantPast); + + assert!(now_plus_1 > now_minus_1); + assert!(now_plus_1 > Timestamp::DistantPast); + + assert!(now_minus_1 > Timestamp::DistantPast); + } + + #[test] + fn duration_ord() { + assert!(TimeDelta::PositiveInfinity > TimeDelta::from_secs(-2)); + assert!(TimeDelta::PositiveInfinity > TimeDelta::from_secs(2)); + assert!(TimeDelta::PositiveInfinity > TimeDelta::NegativeInfinity); + + assert!(TimeDelta::from_secs(2) > TimeDelta::from_secs(1)); + assert!(TimeDelta::from_secs(2) > TimeDelta::from_secs(-1)); + assert!(TimeDelta::from_secs(2) > TimeDelta::from_secs(-2)); + assert!(TimeDelta::from_secs(2) > TimeDelta::NegativeInfinity); + + assert!(TimeDelta::from_secs(1) > TimeDelta::from_secs(-1)); + assert!(TimeDelta::from_secs(1) > TimeDelta::from_secs(-2)); + assert!(TimeDelta::from_secs(1) > TimeDelta::NegativeInfinity); + + assert!(TimeDelta::from_secs(-1) > TimeDelta::from_secs(-2)); + assert!(TimeDelta::from_secs(-1) > TimeDelta::NegativeInfinity); + + assert!(TimeDelta::from_secs(-2) > TimeDelta::NegativeInfinity); + + assert_eq!(TimeDelta::from_secs(1), Duration::from_secs(1)); + assert!(TimeDelta::from_secs(2) > Duration::from_secs(1)); + assert!(TimeDelta::from_secs(1) < Duration::from_secs(2)); + assert!(TimeDelta::from_secs(-1) < Duration::ZERO); + assert!(TimeDelta::from_secs(-1) < Duration::from_secs(1)); + assert!(TimeDelta::PositiveInfinity > Duration::from_secs(2)); + assert!(TimeDelta::NegativeInfinity < Duration::from_secs(1)); + assert!(TimeDelta::NegativeInfinity < Duration::ZERO); + } + + #[test] + fn duration_add() { + assert_eq!( + TimeDelta::PositiveInfinity + TimeDelta::PositiveInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity + TimeDelta::NegativeInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity + TimeDelta::from_secs(-2), + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity + TimeDelta::from_secs(2), + TimeDelta::PositiveInfinity + ); + + assert_eq!( + TimeDelta::NegativeInfinity + TimeDelta::PositiveInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity + TimeDelta::NegativeInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity + TimeDelta::from_secs(-2), + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity + TimeDelta::from_secs(2), + TimeDelta::NegativeInfinity + ); + + assert_eq!( + TimeDelta::from_secs(1) + TimeDelta::PositiveInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::from_secs(1) + TimeDelta::NegativeInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::from_secs(1) + TimeDelta::from_secs(-1), + TimeDelta::ZERO + ); + assert_eq!( + TimeDelta::from_secs(1) + TimeDelta::from_secs(-2), + TimeDelta::from_secs(-1) + ); + assert_eq!( + TimeDelta::from_secs(1) + TimeDelta::from_secs(2), + TimeDelta::from_secs(3) + ); + + assert_eq!( + TimeDelta::from_secs(-1) + TimeDelta::PositiveInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::from_secs(-1) + TimeDelta::NegativeInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::from_secs(-1) + TimeDelta::from_secs(1), + TimeDelta::ZERO + ); + assert_eq!( + TimeDelta::from_secs(-1) + TimeDelta::from_secs(-2), + TimeDelta::from_secs(-3) + ); + assert_eq!( + TimeDelta::from_secs(-1) + TimeDelta::from_secs(2), + TimeDelta::from_secs(1) + ); + } + + #[test] + fn duration_sub() { + assert_eq!( + TimeDelta::PositiveInfinity - TimeDelta::PositiveInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity - TimeDelta::NegativeInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity - TimeDelta::from_secs(-2), + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::PositiveInfinity - TimeDelta::from_secs(2), + TimeDelta::PositiveInfinity + ); + + assert_eq!( + TimeDelta::NegativeInfinity - TimeDelta::NegativeInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity - TimeDelta::PositiveInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity - TimeDelta::from_secs(-2), + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::NegativeInfinity - TimeDelta::from_secs(2), + TimeDelta::NegativeInfinity + ); + + assert_eq!( + TimeDelta::from_secs(1) - TimeDelta::NegativeInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::from_secs(1) - TimeDelta::PositiveInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::from_secs(1) - TimeDelta::from_secs(1), + TimeDelta::ZERO + ); + assert_eq!( + TimeDelta::from_secs(1) - TimeDelta::from_secs(-1), + TimeDelta::from_secs(2) + ); + assert_eq!( + TimeDelta::from_secs(1) - TimeDelta::from_secs(2), + TimeDelta::from_secs(-1) + ); + + assert_eq!( + TimeDelta::from_secs(-1) - TimeDelta::NegativeInfinity, + TimeDelta::PositiveInfinity + ); + assert_eq!( + TimeDelta::from_secs(-1) - TimeDelta::PositiveInfinity, + TimeDelta::NegativeInfinity + ); + assert_eq!( + TimeDelta::from_secs(-1) - TimeDelta::from_secs(-1), + TimeDelta::ZERO + ); + assert_eq!( + TimeDelta::from_secs(-1) - TimeDelta::from_secs(1), + TimeDelta::from_secs(-2) + ); + assert_eq!( + TimeDelta::from_secs(-1) - TimeDelta::from_secs(2), + TimeDelta::from_secs(-3) + ); + } + + #[test] + fn super_instant_test() { + let mut past = Timestamp::DistantPast; + let mut future = Timestamp::DistantFuture; + let now = Timestamp::Exact(Instant::now()); + assert!(past < now); + assert!(now > past); + + assert!(now < future); + assert!(future > now); + + assert!(past < future); + assert!(future > past); + + assert!(now == now); + assert!(past == past); + assert!(future == future); + + assert!(now != past); + assert!(now != future); + assert!(future != past); + + assert!(past - Duration::from_secs(1) == past); + past -= Duration::from_secs(1); + assert!(past == past); + past += Duration::from_secs(1); + assert!(past == past); + + assert!(future + Duration::from_secs(1) == future); + assert!(future - Duration::from_secs(1) == future); + future -= Duration::from_secs(1); + assert!(future == future); + future += Duration::from_secs(1); + assert!(future == future); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/change/direct.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/direct.rs new file mode 100644 index 000000000..5ed6ee65b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/direct.rs @@ -0,0 +1,450 @@ +use std::sync::Arc; + +use crate::Candidate; +use crate::IceCreds; +use crate::Rtc; +use crate::RtcError; +use crate::channel::ChannelId; +use crate::crypto::Fingerprint; +use crate::crypto::dtls::ProtocolVersion; +use crate::format::{Codec, CodecConfig, FormatParams}; +use crate::media::{Media, MediaKind}; +use crate::rtp_::MidRid; +use crate::rtp_::{Frequency, Mid, Pt, Rid, Ssrc}; +use crate::sctp::{ChannelConfig, SctpInitData}; +use crate::streams::{DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP, StreamRx, StreamTx}; + +/// Direct change strategy. +/// +/// Makes immediate changes to the Rtc session without any SDP OFFER/ANSWER. This +/// is an alternative to [`Rtc::sdp_api()`] for use cases when you don’t want to use SDP +/// (or when you want to write RTP directly). +/// +/// To use the Direct API together with a browser client, you would need to make +/// the equivalent changes on the browser side by manually generating the correct +/// SDP OFFER/ANSWER to make the `RTCPeerConnection` match str0m's state. +/// +/// To change str0m's state through the Direct API followed by the SDP API produce +/// an SDP OFFER is not a supported use case. Either pick SDP API and let str0m handle +/// the OFFER/ANSWER or use Direct API and deal with SDP manually. Not both. +/// +///
This is a low level API. +/// +/// str0m normally guarantees that user input cannot cause panics. +/// However as an exception, the Direct API does allow the user to configure the +/// session in a way that is internally inconsistent. Such situations can +/// result in panics. +///
+pub struct DirectApi<'a> { + rtc: &'a mut Rtc, +} + +impl<'a> DirectApi<'a> { + /// Creates a new instance of the `DirectApi` struct with the specified `Rtc` instance. + /// + /// The `DirectApi` struct provides a high-level API for interacting with a WebRTC peer connection, + /// and the `Rtc` instance provides low-level access to the underlying WebRTC functionality. + pub fn new(rtc: &'a mut Rtc) -> Self { + DirectApi { rtc } + } + + /// Registers a negotiated RTP payload type before declaring its streams. + pub fn register_payload_type( + &mut self, + pt: Pt, + codec: Codec, + clock_rate: Frequency, + channels: Option, + format: FormatParams, + ) { + let params = self + .rtc + .session + .codec_config + .iter() + .filter(|params| params.pt() != pt && params.resend() != Some(pt)) + .copied() + .collect(); + self.rtc.session.codec_config = CodecConfig::new_from_payload_params(params); + self.rtc + .session + .codec_config + .add_config(pt, None, codec, clock_rate, channels, format); + } + + /// Sets the ICE controlling flag for this peer connection. + /// + /// If `controlling` is `true`, this peer connection is set as the ICE controlling agent, + /// meaning it will take the initiative to send connectivity checks and control the pace of + /// connectivity checks sent between two peers during the ICE session. + /// + /// If `controlling` is `false`, this peer connection is set as the ICE controlled agent, + /// meaning it will respond to connectivity checks sent by the controlling agent. + pub fn set_ice_controlling(&mut self, controlling: bool) { + self.rtc.ice.set_controlling(controlling); + } + + /// Returns a reference to the local ICE credentials used by this peer connection. + /// + /// The ICE credentials consist of the username and password used by the ICE agent during + /// the ICE session to authenticate and exchange connectivity checks with the remote peer. + pub fn local_ice_credentials(&self) -> IceCreds { + self.rtc.ice.local_credentials().clone() + } + + /// Sets the local ICE credentials. + pub fn set_local_ice_credentials(&mut self, local_ice_credentials: IceCreds) { + self.rtc.ice.set_local_credentials(local_ice_credentials); + } + + /// Sets the remote ICE credentials. + pub fn set_remote_ice_credentials(&mut self, remote_ice_credentials: IceCreds) { + self.rtc.ice.set_remote_credentials(remote_ice_credentials); + } + + /// Invalidate a candidate and remove it from the connection. + /// + /// This is done for host candidates disappearing due to changes in the network + /// interfaces like a WiFi disconnecting or changing IPs. + /// + /// It can also be used to invalidate _remote_ candidates, i.e. if the remote + /// has signalled us that they have invalidated one of their candidates. + /// + /// Returns `true` if the candidate was found and invalidated. + pub fn invalidate_candidate(&mut self, c: &Candidate) -> bool { + self.rtc.ice.invalidate_candidate(c) + } + + /// Returns a reference to the local DTLS fingerprint used by this peer connection. + /// + /// The DTLS fingerprint is a hash of the local SSL/TLS certificate used to authenticate the + /// peer connection and establish a secure communication channel between the peers. + pub fn local_dtls_fingerprint(&self) -> &Fingerprint { + self.rtc.dtls.local_fingerprint() + } + + /// Returns a reference to the remote DTLS fingerprint used by this peer connection. + pub fn remote_dtls_fingerprint(&self) -> Option<&Fingerprint> { + self.rtc.dtls.remote_fingerprint() + } + + /// Returns the negotiated DTLS protocol version. + /// + /// Call this after receiving [`crate::Event::Connected`] + /// to learn the negotiated DTLS version. Before the handshake completes, + /// this may return `None` + pub fn dtls_protocol_version(&self) -> Option { + self.rtc.dtls.protocol_version() + } + + /// Sets the remote DTLS fingerprint. + pub fn set_remote_fingerprint(&mut self, dtls_fingerprint: Fingerprint) { + self.rtc.remote_fingerprint = Some(dtls_fingerprint); + } + + /// Start the DTLS subsystem. + pub fn start_dtls(&mut self, active: bool) -> Result<(), RtcError> { + self.rtc.init_dtls(active) + } + + /// Start the SCTP over DTLS. + /// + /// When `client` is `true`, this side initiates the SCTP association as the + /// connecting party. + pub fn start_sctp(&mut self, client: bool) { + self.rtc + .try_init_sctp(client, None, None) + .expect("starting SCTP should be infallible") + } + + /// Start SCTP over DTLS using out-of-band SCTP INIT data. + /// + /// This is used for SNAP (SCTP Negotiation Acceleration Protocol), which + /// allows skipping the 4-way SCTP handshake entirely once both peers have + /// exchanged their SCTP INIT chunks out of band. + /// + /// The `sctp_init_data` must contain: + /// + /// 1. A local INIT chunk generated by calling + /// [`SctpInitData::local_init_chunk()`]. + /// 2. The remote INIT chunk set via [`SctpInitData::set_remote_init_chunk()`]. + /// + /// This method returns an error unless both the local and remote INIT + /// chunks are present. + /// + /// This method is the SNAP-specific alternative to [`Self::start_sctp()`]. + /// + /// # Example + /// ```ignore + /// use str0m::channel::SctpInitData; + /// + /// let mut init_data = SctpInitData::new(); + /// let local_init = init_data.local_init_chunk().unwrap(); + /// // ... exchange local_init via signaling, receive remote_init ... + /// init_data.set_remote_init_chunk(remote_init); + /// rtc.direct_api().start_sctp_with_snap(false, init_data)?; + /// ``` + pub fn start_sctp_with_snap( + &mut self, + client: bool, + sctp_init_data: SctpInitData, + ) -> Result<(), RtcError> { + self.rtc.try_init_sctp(client, Some(sctp_init_data), None) + } + + /// Create a new data channel. + pub fn create_data_channel(&mut self, config: ChannelConfig) -> ChannelId { + let id = self.rtc.chan.new_channel(&config); + self.rtc.chan.confirm(id, config); + id + } + + /// Close a data channel. + pub fn close_data_channel(&mut self, channel_id: ChannelId) { + self.rtc.chan.close_channel(channel_id, &mut self.rtc.sctp); + } + + /// Set whether to enable ice-lite. + pub fn set_ice_lite(&mut self, ice_lite: bool) { + self.rtc.ice.set_ice_lite(ice_lite); + } + + /// Enable twcc feedback. + pub fn enable_twcc_feedback(&mut self) { + self.rtc.session.enable_twcc_feedback() + } + + /// Generate a ssrc that is not already used in session + pub fn new_ssrc(&self) -> Ssrc { + self.rtc.session.streams.new_ssrc() + } + + /// Get the str0m `ChannelId` by an `sctp_stream_id`. + /// + /// This is useful when using out of band negotiated sctp stream id in + /// [`Self::create_data_channel()`] + pub fn channel_id_by_sctp_stream_id(&self, id: u16) -> Option { + self.rtc.chan.channel_id_by_stream_id(id) + } + + /// Get the `sctp_stream_id` from a str0m `ChannelId`. + /// + /// This is useful when using out of band negotiated sctp stream id in + /// [`Self::create_data_channel()`] + pub fn sctp_stream_id_by_channel_id(&self, id: ChannelId) -> Option { + self.rtc.chan.stream_id_by_channel_id(id) + } + + /// Create a new `Media`. + /// + /// All streams belong to a media identified by a `mid`. This creates the media without + /// doing any SDP dance. + pub fn declare_media(&mut self, mid: Mid, kind: MediaKind) -> &mut Media { + let max_index = self.rtc.session.medias.iter().map(|m| m.index()).max(); + + let next_index = if let Some(max_index) = max_index { + max_index + 1 + } else { + 0 + }; + + let exts = self.rtc.session.exts.cloned_with_type(kind.is_audio()); + let m = Media::from_direct_api(mid, next_index, kind, exts); + + self.rtc.session.medias.push(m); + self.rtc.session.medias.last_mut().unwrap() + } + + /// Remove `Media`. + /// + /// Removes media and all streams belong to a media identified by a `mid`. + pub fn remove_media(&mut self, mid: Mid) { + self.rtc.session.remove_media(mid); + } + + /// Allow incoming traffic from remote peer for the given SSRC. + /// + /// Can be called multiple times if the `rtx` is discovered later via RTP header extensions. + pub fn expect_stream_rx( + &mut self, + ssrc: Ssrc, + rtx: Option, + mid: Mid, + rid: Option, + ) -> &mut StreamRx { + let Some(_media) = self.rtc.session.media_by_mid(mid) else { + panic!("No media declared for mid: {}", mid); + }; + + // By default we do not suppress nacks, this has to be called explicitly by the user of direct API. + let suppress_nack = false; + + let midrid = MidRid(mid, rid); + + self.rtc + .session + .streams + .expect_stream_rx(ssrc, rtx, midrid, suppress_nack) + } + + /// Remove the receive stream for the given SSRC. + /// + /// Returns true if stream existed and was removed. + pub fn remove_stream_rx(&mut self, ssrc: Ssrc) -> bool { + self.rtc.session.streams.remove_stream_rx(ssrc) + } + + /// Obtain a receive stream. + /// + /// In RTP mode, the receive stream is used to signal keyframe requests. + /// + /// The stream must first be declared using [`DirectApi::expect_stream_rx`]. + pub fn stream_rx(&mut self, ssrc: &Ssrc) -> Option<&mut StreamRx> { + self.rtc.session.streams.stream_rx(ssrc) + } + + /// Obtain a recv stream by looking it up via mid/rid. + pub fn stream_rx_by_mid(&mut self, mid: Mid, rid: Option) -> Option<&mut StreamRx> { + let midrid = MidRid(mid, rid); + self.rtc.session.streams.stream_rx_by_midrid(midrid, true) + } + + /// Declare the intention to send data using the given SSRC. + /// + /// * The resend RTX is optional but necessary to do resends. str0m does not do + /// resends without RTX. + /// + /// Can be called multiple times without changing any internal state. However + /// the RTX value is only picked up the first ever time we see a new SSRC. + pub fn declare_stream_tx( + &mut self, + ssrc: Ssrc, + rtx: Option, + mid: Mid, + rid: Option, + ) -> &mut StreamTx { + let Some(media) = self.rtc.session.media_by_mid_mut(mid) else { + panic!("No media declared for mid: {}", mid); + }; + + let is_audio = media.kind().is_audio(); + + let midrid = MidRid(mid, rid); + + // If there is a RID tx, declare it so we an use it in Writer API + if let Some(rid) = rid { + media.add_to_rid_tx(rid); + } + + let stream = self + .rtc + .session + .streams + .declare_stream_tx(ssrc, rtx, midrid); + + let size = if is_audio { + self.rtc.session.send_buffer_audio + } else { + self.rtc.session.send_buffer_video + }; + + stream.set_rtx_cache(size, DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP); + + stream + } + + /// Remove the transmit stream for the given SSRC. + /// + /// Returns true if stream existed and was removed. + pub fn remove_stream_tx(&mut self, ssrc: Ssrc) -> bool { + self.rtc.session.streams.remove_stream_tx(ssrc) + } + + /// Obtain a send stream to write RTP data directly. + /// + /// The stream must first be declared using [`DirectApi::declare_stream_tx`]. + pub fn stream_tx(&mut self, ssrc: &Ssrc) -> Option<&mut StreamTx> { + self.rtc.session.streams.stream_tx(ssrc) + } + + /// Obtain a send stream by looking it up via mid/rid. + pub fn stream_tx_by_mid(&mut self, mid: Mid, rid: Option) -> Option<&mut StreamTx> { + let midrid = MidRid(mid, rid); + self.rtc.session.streams.stream_tx_by_midrid(midrid) + } + + /// Reset a transmit stream to use a new SSRC and optionally a new RTX SSRC. + /// + /// This changes the SSRC of an existing stream and resets all relevant state. + /// Use this when you need to change the SSRC of an existing stream without creating a new one. + /// + /// If the stream has an RTX SSRC, `new_rtx` must be provided. If the stream doesn't + /// have an RTX SSRC, `new_rtx` is ignored. + /// + /// Returns a reference to the updated stream or None if: + /// - No stream was found for the given mid/rid + /// - The new SSRC is the same as the current one (no change needed) + /// - The new RTX SSRC is the same as the current one (no change needed) + pub fn reset_stream_tx( + &mut self, + mid: Mid, + rid: Option, + new_ssrc: Ssrc, + new_rtx: Option, + ) -> Option<&mut StreamTx> { + let midrid = MidRid(mid, rid); + + // Find the stream by mid/rid + let stream = self.rtc.session.streams.stream_tx_by_midrid(midrid)?; + + // Don't change to the same SSRC + if stream.ssrc() == new_ssrc { + return None; + } + + // If the stream has an RTX SSRC, New RTX must be provided and differ. + // But it is allowed to start or turn off RTX. + if stream.rtx().is_some() && stream.rtx() == new_rtx { + return None; + } + + // Reset the stream with the new SSRC and RTX + stream.reset_ssrc(new_ssrc, new_rtx); + + // Return a reference to the updated stream + Some(stream) + } + + /// Send an application-specific feedback message (PSFB FMT=15, PT=206). + /// + /// The message is sent in its own standalone SRTCP compound datagram, prefixed + /// with a Receiver Report for RFC 3550 compliance. The `sender_ssrc` is used as + /// the RR's sender SSRC, which allows remote RTCP demuxers that route compound + /// packets by the first SSRC to deliver the feedback to the correct media session. + /// + /// The `payload` is opaque and application-defined. + pub fn send_app_specific_feedback( + &mut self, + sender_ssrc: Ssrc, + media_ssrc: Ssrc, + payload: impl Into>, + ) { + self.rtc + .session + .send_app_specific_feedback(sender_ssrc, media_ssrc, payload); + } + + /// Send a Picture Loss Indication (PLI, PT=206 FMT=1) for a media stream, + /// choosing the `sender_ssrc` of the outgoing RTCP feedback. + /// + /// The PLI is queued into the regular RTCP compound packet. The `sender_ssrc` + /// sets the feedback packet's sender SSRC. `media_ssrc` is the SSRC of the + /// stream a keyframe is requested for. + /// + /// Unlike requesting a keyframe on a local receive stream, this does not require a + /// receive stream for `media_ssrc` and lets the caller choose the sender SSRC, so it + /// can relay a keyframe request for a stream sourced by the remote peer. + pub fn send_pli_feedback(&mut self, sender_ssrc: Ssrc, media_ssrc: Ssrc) { + self.rtc.session.send_pli_feedback(sender_ssrc, media_ssrc); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/change/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/mod.rs new file mode 100644 index 000000000..d172b586b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/mod.rs @@ -0,0 +1,20 @@ +//! Ways to change the [`Rtc`][crate::Rtc] session. SDP or Direct. +//! +//! str0m has two main APIs for changing the WebRTC session. +//! +//! 1. SDP API. The common way to talk to browsers using SDP OFFER/ANSWER negotiations. +//! [`Rtc::sdp_api()`][crate::Rtc::sdp_api] +//! 2. Direct API. Makes changes directly to the session without any negotiation. +//! [`Rtc::direct_api()`][crate::Rtc::direct_api] +//! +//! ## Direct API +//! +//! The direct API is a lower level API which typically can't be mixed with the SDP API. If you make +//! changes directly to the session, the remote side would not be aware of them unless you construct +//! some "other way" keeping the two peers in sync. +mod sdp; +pub(crate) use sdp::AddMedia; +pub use sdp::{SdpAnswer, SdpApi, SdpOffer, SdpPendingOffer}; + +mod direct; +pub use direct::DirectApi; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/change/sdp.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/sdp.rs new file mode 100644 index 000000000..00a617020 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/change/sdp.rs @@ -0,0 +1,2699 @@ +//! Strategy that amends the [`Rtc`] via SDP OFFER/ANSWER negotiation. + +use std::fmt; +use std::ops::{Deref, DerefMut}; +use std::slice::Iter; + +use crate::Rtc; +use crate::RtcError; +use crate::channel::ChannelId; +use crate::crypto::Fingerprint; +use crate::format::CodecConfig; +use crate::format::PayloadParams; +use crate::media::{Media, Rids, Simulcast}; +use crate::packet::MediaKind; +use crate::rtp_::MidRid; +use crate::rtp_::Rid; +use crate::rtp_::{Direction, Extension, ExtensionMap, Mid, Pt, Ssrc}; +use crate::sctp::ChannelConfig; +use crate::sctp::RtcSctp; +use crate::sdp::{self, MediaAttribute, MediaLine, MediaType, Msid, Sdp}; +use crate::sdp::{Proto, SessionAttribute, Setup, SimulcastGroups}; +use crate::session::Session; +use crate::{Candidate, IceCreds}; +use str0m_proto::Id; + +pub use crate::sdp::{SdpAnswer, SdpOffer}; +use crate::streams::{DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP, Streams}; + +/// Changes to the Rtc via SDP Offer/Answer dance. +pub struct SdpApi<'a> { + rtc: &'a mut Rtc, + changes: Changes, +} + +impl<'a> SdpApi<'a> { + pub(crate) fn new(rtc: &'a mut Rtc) -> Self { + SdpApi { + rtc, + changes: Changes::default(), + } + } + + /// Accept an [`SdpOffer`] from the remote peer. If this call returns successfully, the + /// changes will have been made to the session. The resulting [`SdpAnswer`] should be + /// sent to the remote peer. + /// + /// Note. Pending changes from a previous non-completed [`SdpApi`][super::SdpApi] will be + /// considered rolled back when calling this function. + /// + /// The incoming SDP is validated in various ways which can cause this call to fail. + /// Example of such problems would be an SDP without any m-lines, missing `a=fingerprint` + /// or if `a=group` doesn't match the number of m-lines. + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// # use str0m::change::{SdpOffer}; + /// // obtain offer from remote peer. + /// let json_offer: &[u8] = todo!(); + /// let offer: SdpOffer = serde_json::from_slice(json_offer).unwrap(); + /// + /// let mut rtc = Rtc::new(Instant::now()); + /// let answer = rtc.sdp_api().accept_offer(offer).unwrap(); + /// + /// // send json_answer to remote peer. + /// let json_answer = serde_json::to_vec(&answer).unwrap(); + /// ``` + pub fn accept_offer(self, offer: SdpOffer) -> Result { + debug!("Accept offer"); + + // Invalidate any outstanding PendingOffer. + self.rtc.next_change_id(); + + if offer.media_lines.is_empty() { + return Err(RtcError::RemoteSdp("No m-lines in offer".into())); + } + + if self.rtc.ice.ice_lite() && offer.session.ice_lite() { + return Err(RtcError::RemoteSdp( + "Both peers being ICE-Lite not supported".into(), + )); + } + + add_ice_details(self.rtc, &offer, None)?; + + if self.rtc.remote_fingerprint.is_none() { + if let Some(f) = offer.fingerprint() { + self.rtc.remote_fingerprint = Some(f); + } else { + self.rtc.disconnect(); + return Err(RtcError::RemoteSdp("missing a=fingerprint".into())); + } + } + + if !self.rtc.dtls.is_inited() { + // The side that makes the first offer is the controlling side, unless they + // are ICE Lite, in which case the roles are reversed (see RFC 5245). + self.rtc.ice.set_controlling(offer.session.ice_lite()); + } + + // Ensure setup=active/passive is corresponding remote and init dtls. + init_dtls(self.rtc, &offer)?; + + let remote_max_message_size = extract_max_message_size(offer.media_lines.iter()); + + // Extract a=sctp-init from remote offer before apply_offer consumes it. + let remote_sctp_init = offer.sctp_init().map(|v| v.to_owned()); + + let has_snap = process_remote_sctp_init(&mut self.rtc.sctp, remote_sctp_init.as_deref())?; + + // Modify session with offer. + apply_offer(&mut self.rtc.session, offer)?; + + // Handle potentially new m=application line. + let client = self.rtc.dtls.is_active().expect("DTLS active to be set"); + + // Generate local sctp-init for the answer: + // When the remote included a=sctp-init, we reciprocate (§5.4). + if has_snap && !self.rtc.sctp.is_inited() && !self.rtc.sctp.ensure_local_snap_init() { + warn!("Failed to generate SNAP INIT chunk, degrading to non-SNAP"); + } + + if self.rtc.session.app().is_some() { + let init_data = self.rtc.sctp.build_snap_init_data(); + self.rtc + .try_init_sctp(client, init_data, remote_max_message_size)?; + } + + let params = AsSdpParams::new(self.rtc, None); + let sdp = as_sdp(&self.rtc.session, params); + + debug!("Create answer"); + Ok(sdp.into()) + } + + /// Accept an answer to a previously created [`SdpOffer`]. + /// + /// This function returns an [`RtcError::ChangesOutOfOrder`] if we have created and applied another + /// [`SdpApi`][super::SdpApi] before calling this. The same also happens if we use + /// [`SdpApi::accept_offer()`] before using this pending instance. + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// # use str0m::media::{MediaKind, Direction}; + /// # use str0m::change::SdpAnswer; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// let mid = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + /// let (offer, pending) = changes.apply().unwrap(); + /// + /// // send offer to remote peer, receive answer back + /// let answer: SdpAnswer = todo!(); + /// + /// rtc.sdp_api().accept_answer(pending, answer).unwrap(); + /// ``` + pub fn accept_answer( + self, + mut pending: SdpPendingOffer, + answer: SdpAnswer, + ) -> Result<(), RtcError> { + debug!("Accept answer"); + + // Ensure we don't use the wrong changes below. We must use that of pending. + drop(self.changes); + + if !self.rtc.is_correct_change_id(pending.change_id) { + return Err(RtcError::ChangesOutOfOrder); + } + + if self.rtc.ice.ice_lite() && answer.session.ice_lite() { + return Err(RtcError::RemoteSdp( + "Both peers being ICE-Lite not supported".into(), + )); + } + + add_ice_details(self.rtc, &answer, Some(&pending))?; + + // Ensure setup=active/passive is corresponding remote and init dtls. + init_dtls(self.rtc, &answer)?; + + if self.rtc.remote_fingerprint.is_none() { + if let Some(f) = answer.fingerprint() { + self.rtc.remote_fingerprint = Some(f); + } else { + self.rtc.disconnect(); + return Err(RtcError::RemoteSdp("missing a=fingerprint".into())); + } + } + + // Extract a=sctp-init from remote answer before apply_answer consumes it. + let remote_sctp_init = answer.sctp_init().map(|v| v.to_owned()); + + let expected_snap_answer = + !self.rtc.sctp.is_inited() && self.rtc.sctp.local_sctp_init_for_sdp().is_some(); + + // Validate or cache the remote value before mutating the session. This + // keeps a bad re-offer/re-answer from partially applying local state. + let has_snap = process_remote_sctp_init(&mut self.rtc.sctp, remote_sctp_init.as_deref())?; + + // Split out new channels, since that is not handled by the Session. + let new_channels = pending.changes.take_new_channels(); + + let remote_max_message_size = extract_max_message_size(answer.media_lines.iter()); + + // Modify session with answer + apply_answer(&mut self.rtc.session, pending.changes, answer)?; + + // Handle potentially new m=application line. + let client = self.rtc.dtls.is_active().expect("DTLS to be inited"); + + if expected_snap_answer && !has_snap { + debug!("Remote answer did not accept SNAP, falling back to regular SCTP handshake"); + self.rtc.sctp.disable_pending_snap(); + } + + if self.rtc.session.app().is_some() { + let init_data = self.rtc.sctp.build_snap_init_data(); + self.rtc + .try_init_sctp(client, init_data, remote_max_message_size)?; + } + + for (id, config) in new_channels { + self.rtc.chan.confirm(id, config); + } + + Ok(()) + } + + /// Test if any changes have been made. + /// + /// If changes have been made, nothing happens until we call [`SdpApi::apply()`]. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::{Rtc, media::MediaKind, media::Direction}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// assert!(!changes.has_changes()); + /// + /// let mid = changes.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + /// assert!(changes.has_changes()); + /// # } + /// ``` + pub fn has_changes(&self) -> bool { + !self.changes.0.is_empty() + } + + /// Add audio or video media and get the `mid` that will be used. + /// + /// Each call will result in a new m-line in the offer identified by the [`Mid`]. + /// + /// The mid is not valid to use until the SDP offer-answer dance is complete and + /// the mid been advertised via [`Event::MediaAdded`][crate::Event::MediaAdded]. + /// + /// * `stream_id` is used to synchronize media. It is `a=msid-semantic: WMS ` line in SDP. + /// * `track_id` is becomes both the track id in `a=msid ` as well as the + /// CNAME in the RTP SDES. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::{Rtc, media::MediaKind, media::Direction}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// + /// let mid = changes.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + /// # } + /// ``` + pub fn add_media( + &mut self, + kind: MediaKind, + dir: Direction, + stream_id: Option, + track_id: Option, + simulcast: Option, + ) -> Mid { + let mid = self.rtc.new_mid(); + + // https://www.rfc-editor.org/rfc/rfc8830 + // msid-id = 1*64token-char + fn is_token_char(c: &char) -> bool { + // token-char = %x21 / %x23-27 / %x2A-2B / %x2D-2E / %x30-39 + // / %x41-5A / %x5E-7E + let u = *c as u32; + u == 0x21 + || (0x23..=0x27).contains(&u) + || (0x2a..=0x2b).contains(&u) + || (0x2d..=0x2e).contains(&u) + || (0x30..=0x39).contains(&u) + || (0x41..=0x5a).contains(&u) + || (0x5e..0x7e).contains(&u) + } + + let stream_id = if let Some(stream_id) = stream_id { + stream_id.chars().filter(is_token_char).take(64).collect() + } else { + Id::<20>::random().to_string() + }; + + let track_id = if let Some(track_id) = track_id { + track_id.chars().filter(is_token_char).take(64).collect() + } else { + Id::<20>::random().to_string() + }; + + let mut ssrcs = Vec::new(); + + // Main SSRC, not counting RTX. + let main_ssrc_count = simulcast.as_ref().map(|s| s.send.len()).unwrap_or(1); + + for _ in 0..main_ssrc_count { + let rtx = kind.is_video().then(|| self.rtc.session.streams.new_ssrc()); + ssrcs.push((self.rtc.session.streams.new_ssrc(), rtx)); + } + + // TODO: let user configure stream/track name. + let msid = Msid { + stream_id, + track_id: track_id.clone(), + }; + + let add = AddMedia { + mid, + cname: track_id, + msid, + kind, + dir, + ssrcs, + simulcast, + + // Added later + pts: vec![], + exts: ExtensionMap::empty(), + index: 0, + }; + + self.changes.0.push(Change::AddMedia(add)); + mid + } + + /// Change the direction of an already existing media. + /// + /// All media have a direction. The media can be added by this side via + /// [`SdpApi::add_media()`] or by the remote peer. Either way, the direction + /// of the line can be changed at any time. + /// + /// It's possible to set the direction [`Direction::Inactive`] for media that + /// will not be used by the session anymore. + /// + /// If the direction is set for media that doesn't exist, or if the direction is + /// the same that's already set [`SdpApi::apply()`] not require a negotiation. + pub fn set_direction(&mut self, mid: Mid, dir: Direction) { + let changed = self.rtc.session.set_direction(mid, dir); + + if changed { + self.changes.0.push(Change::Direction(mid, dir)); + } + } + + /// Stop an already existing media. + /// + /// The next generated offer emits the m-line with port 0 and excludes + /// it from the BUNDLE group, per [RFC 8843] §7.5.3. The remote + /// transceiver transitions to the "stopped" state and the m-line slot + /// becomes eligible for recycling. + /// + /// Unlike [`SdpApi::set_direction()`] with [`Direction::Inactive`], + /// a stopped m-line cannot be reactivated. + /// + /// If the media doesn't exist, or is already stopped, [`SdpApi::apply()`] + /// will not require a negotiation. + /// + /// [RFC 8843]: https://datatracker.ietf.org/doc/html/rfc8843#section-7.5.3 + pub fn stop_media(&mut self, mid: Mid) { + let changed = self.rtc.session.stop_media(mid); + + if changed { + self.changes + .0 + .push(Change::Direction(mid, Direction::Inactive)); + } + } + + /// Add a new reliable ordered data channel and get the `id` that will be used. + /// + /// Use `add_channel_with_config` when unreliable or unordered data channels are preferred. + /// + /// The first ever data channel added to a WebRTC session results in a media + /// of a special "application" type in the SDP. The m-line is for a SCTP association over + /// DTLS, and all data channels are multiplexed over this single association. + /// + /// That means only the first ever `add_channel` will result in an [`SdpOffer`]. + /// Consecutive channels will be opened without needing a negotiation. + /// + /// The label is used to identify the data channel to the remote peer. This is mostly + /// useful when multiple channels are in use at the same time. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// + /// let cid = changes.add_channel("my special channel".to_string()); + /// # } + /// ``` + pub fn add_channel(&mut self, label: String) -> ChannelId { + self.add_channel_with_config(ChannelConfig { + label, + ..Default::default() + }) + } + + /// Add a new data channel with a given configuration and get the `id` that will be used. + /// + /// Refer to `add_channel` for more details. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::{channel::{ChannelConfig, Reliability}, Rtc}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// + /// let cid = changes.add_channel_with_config(ChannelConfig { + /// label: "my special channel".to_string(), + /// reliability: Reliability::MaxRetransmits{ retransmits: 0 }, + /// ordered: false, + /// ..Default::default() + /// }); + /// # } + /// ``` + pub fn add_channel_with_config(&mut self, config: ChannelConfig) -> ChannelId { + let has_media = self.rtc.session.app().is_some(); + let changes_contains_add_app = self.changes.contains_add_app(); + + if !has_media && !changes_contains_add_app { + let mid = self.rtc.new_mid(); + self.changes.0.push(Change::AddApp(mid)); + } + + let id = self.rtc.chan.new_channel(&config); + + self.changes.0.push(Change::AddChannel((id, config))); + + id + } + + /// Perform an ICE restart. + /// + /// Only one ICE restart can be pending at the time. Calling this repeatedly removes any other + /// pending ICE restart. + /// + /// The local ICE candidates can be kept as is, or be cleared out, in which case new ice + /// candidates must be added via [`Rtc::add_local_candidate`] before connectivity can be + /// re-established. + /// + /// Returns the new ICE credentials that will be used going forward. + pub fn ice_restart(&mut self, keep_local_candidates: bool) -> IceCreds { + self.changes + .retain(|c| !matches!(c, Change::IceRestart(_, _))); + + let new_creds = IceCreds::new(); + self.changes + .push(Change::IceRestart(new_creds.clone(), keep_local_candidates)); + + new_creds + } + + /// Attempt to apply the changes made. + /// + /// If this returns [`SdpOffer`], the caller the changes are + /// not happening straight away, and the caller is expected to do a negotiation with the remote + /// peer and apply the answer using [`SdpPendingOffer`]. + /// + /// In case this returns `None`, there either were no changes, or the changes could be applied + /// without doing a negotiation. Specifically for additional [`SdpApi::add_channel()`] + /// after the first, there is no negotiation needed. + /// + /// The [`SdpPendingOffer`] is valid until the next time we call this function, at which + /// point using it will raise an error. Using [`SdpApi::accept_offer()`] will also invalidate + /// the current [`SdpPendingOffer`]. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let changes = rtc.sdp_api(); + /// assert!(changes.apply().is_none()); + /// # } + /// ``` + pub fn apply(self) -> Option<(SdpOffer, SdpPendingOffer)> { + if self.changes.is_empty() { + return None; + } + + let change_id = self.rtc.next_change_id(); + + let requires_negotiation = self.changes.0.iter().any(requires_negotiation); + + if requires_negotiation { + let offer = create_offer(self.rtc, &self.changes); + let pending = SdpPendingOffer { + change_id, + changes: self.changes, + }; + debug!("Create offer"); + Some((offer, pending)) + } else { + debug!("Apply direct changes"); + apply_direct_changes(self.rtc, self.changes); + None + } + } + + /// Combines the modifications made in [`SdpApi`] with those in [`SdpPendingOffer`]. + /// + /// This function merges the changes present in [`SdpApi`] with the changes + /// in [`SdpPendingOffer`]. In result this [`SdpApi`] will incorporate modifications + /// from both the previous [`SdpPendingOffer`] and any newly added changes. + /// + /// ## Example + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::media::{Direction, MediaKind}; + /// # use str0m::Rtc; + /// let mut rtc = Rtc::new(Instant::now()); + /// let mut changes = rtc.sdp_api(); + /// changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + /// let (_offer, pending) = changes.apply().unwrap(); + /// + /// let mut changes = rtc.sdp_api(); + /// changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + /// changes.merge(pending); + /// + /// // This `SdpOffer` will have changes from the first `SdpPendingChanges` + /// // and new changes from `SdpApi` + /// let (_offer, pending) = changes.apply().unwrap(); + /// ``` + pub fn merge(&mut self, mut pending_offer: SdpPendingOffer) { + pending_offer.retain_relevant(self.rtc); + + // Prepend the original pending changes before the current SdpApi's own changes. + // + // AddMedia / AddApp entries from the pending offer carry already-allocated MIDs + // that the remote peer may have committed to specific m-line positions. They must + // appear before any new changes so that as_new_medias() assigns them the same + // indices as in the original offer. + pending_offer.changes.0.append(&mut self.changes.0); + self.changes.0 = pending_offer.changes.0; + } +} + +/// Pending offer from a previous [`Rtc::sdp_api()`] call. +/// +/// This allows us to accept a remote answer. No changes have been made to the session +/// before we call [`SdpApi::accept_answer()`], which means that rolling back a +/// change is as simple as dropping this instance. +/// +/// ```no_run +/// # use std::time::Instant; +/// # use str0m::Rtc; +/// # use str0m::media::{MediaKind, Direction}; +/// # use str0m::change::SdpAnswer; +/// let mut rtc = Rtc::new(Instant::now()); +/// +/// let mut changes = rtc.sdp_api(); +/// let mid = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); +/// let (offer, pending) = changes.apply().unwrap(); +/// +/// // send offer to remote peer, receive answer back +/// let answer: SdpAnswer = todo!(); +/// +/// rtc.sdp_api().accept_answer(pending, answer).unwrap(); +/// ``` +pub struct SdpPendingOffer { + change_id: usize, + changes: Changes, +} + +impl SdpPendingOffer { + /// Retains only the relevant changes in the `changes` vector based on the provided `Rtc` instance. + /// + /// This function filters the vector of `Change` instances stored in the current object and retains + /// only those changes that are considered relevant with respect to the provided `Rtc` instance. + fn retain_relevant(&mut self, rtc: &Rtc) { + fn is_relevant(rtc: &Rtc, c: &Change) -> bool { + match c { + Change::AddMedia(v) => rtc.media(v.mid).is_none(), + Change::AddApp(_) => rtc.session.app().is_none(), + Change::AddChannel(v) => rtc.chan.stream_id_by_channel_id(v.0).is_none(), + Change::Direction(m, d) => { + // If mid is missing, this is not relevant. + rtc.media(*m).map(|m| m.direction() != *d).unwrap_or(false) + } + Change::IceRestart(v, _) => rtc.ice.local_credentials() != v, + } + } + + self.changes.retain(|c| is_relevant(rtc, c)); + } +} + +#[derive(Default)] +pub(crate) struct Changes(pub Vec); + +impl Changes { + /// Details of the active ICE restart, if any. + /// + /// Returns the new local ICE credentials and the whether to keep local ICE candidates if an + /// ICE restart has been initiated in the offer, otherwise [`None`]. + fn ice_restart(&self) -> Option<(IceCreds, bool)> { + self.iter().find_map(|c| match c { + Change::IceRestart(creds, keep_local_candidates) => { + Some((creds.clone(), *keep_local_candidates)) + } + _ => None, + }) + } +} + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub(crate) enum Change { + AddMedia(AddMedia), + AddApp(Mid), + AddChannel((ChannelId, ChannelConfig)), + Direction(Mid, Direction), + IceRestart(IceCreds, bool), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AddMedia { + pub mid: Mid, + pub cname: String, + pub msid: Msid, + pub kind: MediaKind, + pub dir: Direction, + pub ssrcs: Vec<(Ssrc, Option)>, + pub simulcast: Option, + + // pts and index are filled in when creating the SDP OFFER. + // The default PT order is set by the Session (BUNDLE). + // TODO: We can make this configurable here too. + pub pts: Vec, + pub exts: ExtensionMap, + pub index: usize, +} + +impl Deref for Changes { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Changes { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +fn requires_negotiation(c: &Change) -> bool { + match c { + Change::IceRestart(_, _) => true, + Change::AddMedia(_) => true, + Change::AddApp(_) => true, + Change::AddChannel(_) => false, + Change::Direction(_, _) => true, + } +} + +fn apply_direct_changes(rtc: &mut Rtc, mut changes: Changes) { + // Split out new channels, since that is not handled by the Session. + let new_channels = changes.take_new_channels(); + + for (id, config) in new_channels { + rtc.chan.confirm(id, config); + } +} + +fn create_offer(rtc: &mut Rtc, changes: &Changes) -> SdpOffer { + if !rtc.dtls.is_inited() { + // The side that makes the first offer is the controlling side, unless they + // are ICE Lite, in which case the roles are reversed (see RFC 5245). + rtc.ice.set_controlling(!rtc.ice.ice_lite()); + } + + // Generate local sctp-init for SNAP if enabled and we have/will have an app m-line. + if rtc.sctp.snap_enabled() + && (rtc.session.app().is_some() || changes.contains_add_app()) + && !rtc.sctp.is_inited() + && !rtc.sctp.ensure_local_snap_init() + { + warn!("Failed to generate SNAP INIT chunk, degrading to non-SNAP"); + } + + let params = AsSdpParams::new(rtc, Some(changes)); + let sdp = as_sdp(&rtc.session, params); + + sdp.into() +} + +fn add_ice_details( + rtc: &mut Rtc, + sdp: &Sdp, + pending: Option<&SdpPendingOffer>, +) -> Result<(), RtcError> { + let Some(creds) = sdp.ice_creds() else { + return Err(RtcError::RemoteSdp("missing a=ice-ufrag/pwd".into())); + }; + + // If we are handling an **offer** from the remote, differing ICE credentials indicate an ICE + // restart initiated by the remote. + // + // If we are handling an **answer** from the remote, differing ICE credentials indicate an + // acceptance of an ICE restart we requested. + let ice_restart = match rtc.ice.remote_credentials() { + Some(v) => *v != creds, + None => false, + }; + if ice_restart { + let (new_local_creds, keep_local_candidates) = if let Some(pending) = pending { + // Since we have a pending, this is an answer to our offer. + pending.changes.ice_restart().ok_or_else(|| + // Answer contained changed remote creds, indicating an ice restart + // but since we have no pending ice-creds, we didn't initiate it + // Ice restart in an ANSWER breaks spec. + RtcError::RemoteSdp( + "Ice restart in answer without one in the preceeding offer".into(), + ))? + } else { + // The remote OFFER had an ice restart, and we need to respond with + // new credentials in the ANSWER. + (IceCreds::new(), true) + }; + + rtc.ice + .ice_restart(new_local_creds.clone(), keep_local_candidates); + } + + rtc.ice.set_remote_credentials(creds); + + for r in sdp.ice_candidates() { + rtc.ice.add_remote_candidate(r.clone()); + } + + Ok(()) +} + +fn init_dtls(rtc: &mut Rtc, remote_sdp: &Sdp) -> Result<(), RtcError> { + let setup = match remote_sdp.setup() { + Some(v) => match v { + // Remote being ActPass, we take Passive role. + Setup::ActPass => Setup::Passive, + _ => v.invert(), + }, + + None => { + warn!("Missing a=setup line"); + Setup::Passive + } + }; + + let active = setup == Setup::Active; + rtc.init_dtls(active)?; + + Ok(()) +} + +/// Shared logic for processing a remote `a=sctp-init` attribute from an offer or answer. +/// +/// Returns `true` if this is a new SNAP negotiation (remote init was accepted), +/// `false` otherwise. +/// +/// The remote init bytes are stored in `RtcSctp.snap_init` for §5.6 +/// re-offer validation. +/// +/// When the remote includes `a=sctp-init`, we always accept and reciprocate — +/// Section 5.4 of draft-hancke-tsvwg-snap says the answerer MAY include the +/// attribute, and doing so is beneficial. +/// +/// When an initial answer omits or rejects `a=sctp-init`, callers are expected +/// to fall back to a regular SCTP handshake. +fn process_remote_sctp_init( + sctp: &mut RtcSctp, + remote_init: Option<&str>, +) -> Result { + if let Some(remote_init_str) = remote_init { + if sctp.is_inited() { + // §5.6: SCTP already established — remote MUST re-send the + // same sctp-init value on subsequent offers/answers. + match sctp.snap_remote_init_string() { + Some(cached) if cached == remote_init_str => { + debug!("Remote re-sent expected a=sctp-init for established association"); + } + Some(_) => { + return Err(RtcError::RemoteSdp( + "Changed a=sctp-init for existing SCTP association".into(), + )); + } + None => { + // SCTP was established without SNAP but the remote is now + // sending a=sctp-init. We can't transition to SNAP + // mid-session, so just ignore it. + debug!("Ignoring a=sctp-init for non-SNAP established SCTP association"); + } + } + Ok(false) + } else if sctp.set_remote_snap_init_string(remote_init_str) { + Ok(true) + } else { + debug!("Ignoring malformed a=sctp-init"); + Ok(false) + } + } else if sctp.is_snap_established() { + Err(RtcError::RemoteSdp( + "Missing a=sctp-init for established SNAP SCTP association".into(), + )) + } else { + Ok(false) + } +} + +fn as_sdp(session: &Session, params: AsSdpParams) -> Sdp { + let (media_lines, mids, stream_ids) = { + let mut v = as_media_lines(session); + + let mut new_lines = vec![]; + + // When creating new m-lines from the pending changes, the m-line index starts from this. + let new_index_start = v.len(); + + // If there are additions in the pending changes, prepend them now. + if let Some(pending) = params.pending { + new_lines = pending + .as_new_medias(new_index_start, &session.codec_config, &session.exts) + .collect(); + } + + // Add potentially new m-lines to the existing ones. + v.extend(new_lines.iter().map(|n| n as &dyn AsSdpMediaLine)); + + // Turn into sdp::MediaLine (m-line). + let mut lines = v + .iter() + .map(|m| { + // Candidates should only be in the first BUNDLE mid + let include_candidates = m.index() == 0; + + let attrs = params.media_attributes(include_candidates); + + // Already made send stream SSRCs + let mut ssrcs = session.streams.ssrcs_tx(m.mid()); + + // Merged with pending stream SSRCs + if let Some(pending) = params.pending { + ssrcs.extend(pending.ssrcs_for_mid(m.mid())) + } + + let params: Vec<_> = session + .codec_config + .all_for_kind(m.kind()) + .cloned() + .collect(); + + m.as_media_line(attrs, &ssrcs, &session.exts, ¶ms) + }) + .collect::>(); + + // Add a=sctp-init to the application m-line if SNAP is configured. + if let Some(sctp_init) = ¶ms.local_sctp_init { + for line in &mut lines { + if line.typ.is_channel() { + line.attrs + .push(sdp::MediaAttribute::SctpInit(sctp_init.clone())); + } + } + } + + if let Some(pending) = params.pending { + pending.apply_to(&mut lines); + } + + // Mids go into the session part of the SDP. + // Rejected (disabled) m-lines must not be part of the BUNDLE group. + let mids = lines + .iter() + .filter(|l| !l.disabled) + .map(|l| l.mid()) + .collect(); + + let mut stream_ids = vec![]; + for msid in v.iter().filter_map(|v| v.msid()) { + if !stream_ids.contains(&msid.stream_id) { + stream_ids.push(msid.stream_id.clone()); + } + } + + (lines, mids, stream_ids) + }; + + // AllowMixedExts adds "a=extmap-allow-mixed" at session level to signal + // support for mixing one-byte and two-byte RTP header extensions. + // TODO: It would make sense to perform an actual negotiation, however + // just adding this line should work fine: + // https://github.com/meetecho/janus-gateway/blob/d2e74fdf9bb8aa7a39ed68ed28394afe1e0cd22d/src/sdp.c#L1519 + let mut attrs = vec![ + SessionAttribute::Group { + typ: "BUNDLE".into(), + mids, + }, + SessionAttribute::AllowMixedExts, + SessionAttribute::MsidSemantic { + semantic: "WMS".to_string(), + stream_ids, + }, + ]; + + if session.ice_lite { + attrs.push(SessionAttribute::IceLite); + } + + Sdp { + session: sdp::Session { + id: session.id(), + bw: None, + attrs, + }, + media_lines, + } +} + +fn apply_offer(session: &mut Session, offer: SdpOffer) -> Result<(), RtcError> { + offer.assert_consistency()?; + + update_session(session, &offer); + + let bundle_mids = offer.bundle_mids(); + let new_lines = sync_medias(session, &offer, true).map_err(RtcError::RemoteSdp)?; + + add_new_lines(session, &new_lines, true, bundle_mids).map_err(RtcError::RemoteSdp)?; + + ensure_stream_tx(session); + + Ok(()) +} + +fn apply_answer( + session: &mut Session, + pending: Changes, + answer: SdpAnswer, +) -> Result<(), RtcError> { + answer.assert_consistency()?; + + update_session(session, &answer); + + let bundle_mids = answer.bundle_mids(); + let new_lines = sync_medias(session, &answer, false).map_err(RtcError::RemoteSdp)?; + + // The new_lines from the answer must correspond to what we sent in the offer. + if let Some(err) = pending.ensure_correct_answer(&new_lines) { + return Err(RtcError::RemoteSdp(err)); + } + + add_new_lines(session, &new_lines, false, bundle_mids).map_err(RtcError::RemoteSdp)?; + + // Add all pending changes (since we pre-allocated SSRC communicated in the Offer). + add_pending_changes(session, pending); + + ensure_stream_tx(session); + + Ok(()) +} + +fn ensure_stream_tx(session: &mut Session) { + for media in &session.medias { + // Only make send streams when we have to. + if !media.direction().is_sending() { + continue; + } + + let mut rids: Vec> = vec![]; + + if let Some(sim) = media.simulcast() { + for layer in &*sim.send { + let rid: Rid = layer.restriction_id.0.as_str().into(); + rids.push(Some(rid)); + } + } else { + rids.push(None); + } + + // If any payload param has RTX, we need to prepare for RTX. This is because we always + // communicate a=ssrc lines, which need to be complete with main and RTX SSRC. + let has_rtx = session + .codec_config + .iter() + .filter(|p| media.remote_pts().contains(&p.pt)) + .any(|p| p.resend().is_some()); + + for rid in rids { + let midrid = MidRid(media.mid(), rid); + + // If we already have the stream, we don't make any new one. + let has_stream = session.streams.stream_tx_by_midrid(midrid).is_some(); + + if has_stream { + continue; + } + + let (ssrc, rtx) = if has_rtx { + let (ssrc, rtx) = session.streams.new_ssrc_pair(); + (ssrc, Some(rtx)) + } else { + let ssrc = session.streams.new_ssrc(); + (ssrc, None) + }; + + let stream = session.streams.declare_stream_tx(ssrc, rtx, midrid); + + // Configure cache size + let size = if media.kind().is_audio() { + session.send_buffer_audio + } else { + session.send_buffer_video + }; + + stream.set_rtx_cache(size, DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP); + } + } +} + +fn add_pending_changes(session: &mut Session, pending: Changes) { + // For pending AddMedia, we have outgoing SSRC communicated that needs to be added. + for change in pending.0 { + let add_media = match change { + Change::AddMedia(v) => v, + _ => continue, + }; + + let media = session + .medias + .iter_mut() + .find(|m| m.mid() == add_media.mid) + .expect("Media to be added for pending mid"); + + // the cname/msid has already been communicated in the offer, we need to kep + // it the same once the m-line is created. + media.set_cname(add_media.cname); + media.set_msid(add_media.msid); + + // If there are RIDs, the SSRC order matches that of the rid order. + let layers = add_media.simulcast.map(|x| x.send).unwrap_or(vec![]); + + for (i, (ssrc, rtx)) in add_media.ssrcs.into_iter().enumerate() { + let maybe_layer = layers.get(i).cloned(); + let midrid = MidRid(add_media.mid, maybe_layer.map(|layer| layer.rid)); + + let stream = session.streams.declare_stream_tx(ssrc, rtx, midrid); + + let size = if media.kind().is_audio() { + session.send_buffer_audio + } else { + session.send_buffer_video + }; + + stream.set_rtx_cache(size, DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP); + } + } +} + +/// Compares m-lines in Sdp with that already in the session. +/// +/// * Existing m-lines can apply changes (such as direction change). +/// * New m-lines are returned to the caller paired with the session +/// index they should occupy. The index is normally the next free slot, +/// but can be a recycled slot (RFC 8829 §5.2.2) when the remote has +/// replaced a previously disabled Media with a new mid. +fn sync_medias<'a>( + session: &mut Session, + sdp: &'a Sdp, + is_offer: bool, +) -> Result, String> { + let mut new_lines = Vec::with_capacity(sdp.media_lines.len()); + let bundle_mids = sdp.bundle_mids(); + + for (idx, m) in sdp.media_lines.iter().enumerate() { + // First, match existing m-lines. + match m.typ { + MediaType::Application => { + if let Some((_, index)) = session.app() { + if idx != *index { + return index_err(m.mid()); + } + continue; + } + } + MediaType::Audio | MediaType::Video => { + if let Some(media) = session.medias.iter_mut().find(|l| l.mid() == m.mid()) { + if idx != media.index() { + return index_err(m.mid()); + } + + update_media( + media, + m, + &session.codec_config, + &session.exts, + &mut session.streams, + bundle_mids, + ); + + continue; + } + + // Unknown mid at an index held by a disabled Media means + // the remote has recycled the slot (RFC 8829 §5.2.2). + // Recycling is only permitted in offers; an answer that + // rewires a mid at a stopped slot is malformed. + if let Some(pos) = session.medias.iter().position(|l| l.index() == idx) { + if !is_offer { + return Err(format!( + "Answer recycles stopped m-line (not permitted per RFC 8829 §5.2.2): {}", + m.mid() + )); + } + if !session.medias[pos].disabled() { + return index_err(m.mid()); + } + let retired_mid = session.medias[pos].mid(); + session.medias.swap_remove(pos); + session.streams.remove_streams_by_mid(retired_mid); + } + } + _ => { + continue; + } + } + + // Second, discover new m-lines. + new_lines.push((idx, m)); + } + + fn index_err(mid: Mid) -> Result { + Err(format!("Changed order for m-line with mid: {mid}")) + } + + Ok(new_lines) +} + +/// Adds new m-lines as found in an offer or answer. +/// +/// Each entry in `new_lines` pairs an m-line with the session index it +/// should occupy. +fn add_new_lines( + session: &mut Session, + new_lines: &[(usize, &MediaLine)], + is_offer: bool, + bundle_mids: Option<&[Mid]>, +) -> Result<(), String> { + for (idx, m) in new_lines { + let idx = *idx; + + if m.typ.is_media() { + let mut media = Media::from_remote_media_line(m, idx, is_offer); + + // For disabled (rejected) m-lines, don't fire open event + // and the direction will be set to Inactive by update_media. + // In max-bundle, port=0 with the MID in BUNDLE group is NOT rejected. + let is_in_bundle = bundle_mids + .map(|mids| mids.contains(&m.mid())) + .unwrap_or(false); + let is_rejected = m.disabled && !is_in_bundle; + media.need_open_event = is_offer && !is_rejected; + + // Match/remap remote params. + session + .codec_config + .update_params(&m.rtp_params(), m.direction()); + + // Remap the extension to that of the answer. + session.exts.remap(&m.extmaps()); + + update_media( + &mut media, + m, + &session.codec_config, + &session.exts, + &mut session.streams, + bundle_mids, + ); + + session.add_media(media); + } else if m.typ.is_channel() { + session.set_app(m.mid(), idx)?; + } else { + return Err(format!( + "New m-line is neither media nor channel: {}", + m.mid() + )); + } + } + + Ok(()) +} + +/// Update session level properties like +/// Extensions from offer or answer. +fn update_session(session: &mut Session, sdp: &Sdp) { + // Does any m-line contain a a=rtcp-fb:xx transport-cc? + let has_transport_cc = sdp + .media_lines + .iter() + .any(|m| m.rtp_params().iter().any(|p| p.fb_transport_cc)); + + // Is the session level sequence number enabled? + let has_twcc_header = session + .exts + .id_of(Extension::TransportSequenceNumber) + .is_some(); + + // Since twcc feedback is session wide we enable it if there are _any_ + // m-line with a a=rtcp-fb transport-cc parameter and the sequence number + // header is enabled. It can later be disabled for specific m-lines based + // on the extensions map. + if has_transport_cc && has_twcc_header { + session.enable_twcc_feedback(); + } +} + +/// Returns all media/channels as `AsMediaLine` trait. +fn as_media_lines(session: &Session) -> Vec<&dyn AsSdpMediaLine> { + let mut v = vec![]; + + if let Some(app) = session.app() { + v.push(app as &dyn AsSdpMediaLine); + } + v.extend(session.medias().iter().map(|m| m as &dyn AsSdpMediaLine)); + v.sort_by_key(|f| f.index()); + v +} + +fn update_media( + media: &mut Media, + m: &MediaLine, + config: &CodecConfig, + exts: &ExtensionMap, + streams: &mut Streams, + bundle_mids: Option<&[Mid]>, +) { + // If the m-line has port=0, it could mean: + // 1. The m-line is rejected (not in BUNDLE group) + // 2. The m-line is bundled with another m-line (max-bundle format, RFC 8843) + // + // In max-bundle, secondary m-lines use port=0 to indicate they share transport + // with the first m-line. This is NOT a rejection. + let is_in_bundle = bundle_mids + .map(|mids| mids.contains(&m.mid())) + .unwrap_or(false); + let is_rejected = m.disabled && !is_in_bundle; + + if is_rejected { + if !media.disabled() { + debug!( + "Mid ({}) is rejected (port=0, not in BUNDLE), setting to Inactive", + media.mid() + ); + } + media.mark_stopped(); + media.set_direction(Direction::Inactive); + return; + } + + // Direction changes + // + // All changes come from the other side, either via an incoming OFFER + // or a ANSWER from our OFFER. Either way, the direction is inverted to + // how we have it locally. + let new_dir = m.direction().invert(); + // + let change_direction_disallowed = !media.remote_created() + && media.direction() == Direction::Inactive + && new_dir == Direction::SendOnly; + + if change_direction_disallowed { + debug!( + "Ignore attempt to change inactive to recvonly by remote peer for locally created mid: {}", + media.mid() + ); + } else { + media.set_direction(new_dir); + } + + if new_dir.is_sending() { + // The other side has declared how it EXPECTING to receive. We must only send + // the RIDs declared in the answer. + let rids = m.rids(); + let rid_tx = if rids.is_empty() { + Rids::None + } else { + Rids::Specific(rids) + }; + media.set_rid_tx(rid_tx); + } + if new_dir.is_receiving() { + // The other side has declared what it proposes to send. We are accepting it. + let rids = m.rids(); + let rid_rx = if rids.is_empty() { + Rids::Any + } else { + Rids::Specific(rids) + }; + media.set_rid_rx(rid_rx); + } + + // Narrowing/ordering of of PT + let pts: Vec = m + .rtp_params() + .into_iter() + .filter_map(|p| config.sdp_match_remote(p, m.direction())) + .collect(); + media.set_remote_pts(pts); + + let mut remote_extmap = ExtensionMap::empty(); + for (id, ext) in m.extmaps().into_iter() { + // The remapping of extensions should already have happened, which + // means the ID are matching in the session to the remote. + + // Does the ID exist in session? + let in_session = match exts.lookup(id) { + Some(v) => v, + None => continue, + }; + + if in_session != ext { + // Don't set any extensions that aren't enabled in Session. + continue; + } + + // Use the Extension from session, since there might be a special + // serializer for cases like VLA. + remote_extmap.set(id, in_session.clone()); + } + media.set_remote_extmap(remote_extmap); + + // SSRC changes + // This will always be for ReceiverSource since any incoming a=ssrc line will be + // about the remote side's SSRC. + if !new_dir.is_receiving() { + return; + } + + // Simulcast configuration + if let Some(s) = m.simulcast() { + if s.is_munged { + warn!("Not supporting simulcast via munging SDP"); + } else if media.simulcast().is_none() { + // Invert before setting, since it has a recv and send config. + media.set_simulcast(s.invert()); + } + } + + // Only use pre-communicated SSRC if we are running without simulcast. + // We found a bug in FF where the order of the simulcast lines does not + // correspond to the order of the simulcast declarations. In this case + // it's better to fall back on mid/rid dynamic mapping. + if m.simulcast().is_some() { + return; + } + + let infos = m.ssrc_info(); + let main = infos.iter().filter(|i| i.repairs.is_none()); + + for i in main { + // TODO: If the remote is communicating _BOTH_ rid and a=ssrc this will fail. + debug!("Adding pre-communicated SSRC: {:?}", i); + let repair_ssrc = infos + .iter() + .find(|r| r.repairs == Some(i.ssrc)) + .map(|r| r.ssrc); + + // If remote communicated a main a=ssrc, but no RTX, we will not send nacks. + let midrid = MidRid(media.mid(), None); + let suppress_nack = repair_ssrc.is_none(); + streams.expect_stream_rx(i.ssrc, repair_ssrc, midrid, suppress_nack); + } +} + +fn extract_max_message_size(mut media_lines: Iter) -> Option { + if let Some(app_line) = media_lines.find(|m| m.typ.is_channel()) { + if let Some(max_size) = app_line.max_message_size() { + // RFC 8841 §6.1: a value of 0 means the peer can receive a message of + // any size, subject to local capacity. Represent that as the maximum. + let max_size = if max_size == 0 { + u32::MAX + } else { + u32::try_from(max_size).unwrap_or(u32::MAX) + }; + return Some(max_size); + } + } + + None +} + +trait AsSdpMediaLine { + fn mid(&self) -> Mid; + fn msid(&self) -> Option<&Msid>; + fn index(&self) -> usize; + fn kind(&self) -> MediaKind; + fn as_media_line( + &self, + attrs: Vec, + ssrcs_tx: &[(Ssrc, Option)], + exts: &ExtensionMap, + params: &[PayloadParams], + ) -> MediaLine; +} + +impl AsSdpMediaLine for (Mid, usize) { + fn mid(&self) -> Mid { + self.0 + } + fn msid(&self) -> Option<&Msid> { + None + } + fn index(&self) -> usize { + self.1 + } + fn kind(&self) -> MediaKind { + MediaKind::Audio // doesn't matter for App + } + fn as_media_line( + &self, + mut attrs: Vec, + _ssrcs_tx: &[(Ssrc, Option)], + _exts: &ExtensionMap, + _params: &[PayloadParams], + ) -> MediaLine { + attrs.push(MediaAttribute::Mid(self.0)); + attrs.push(MediaAttribute::SctpPort(5000)); + attrs.push(MediaAttribute::MaxMessageSize( + crate::sctp::LOCAL_MAX_MESSAGE_SIZE as usize, + )); + + MediaLine { + typ: sdp::MediaType::Application, + disabled: false, + proto: Proto::Sctp, + pts: vec![], + bw: None, + attrs, + } + } +} + +impl AsSdpMediaLine for Media { + fn mid(&self) -> Mid { + Media::mid(self) + } + fn msid(&self) -> Option<&Msid> { + Some(Media::msid(self)) + } + fn index(&self) -> usize { + Media::index(self) + } + fn kind(&self) -> MediaKind { + Media::kind(self) + } + fn as_media_line( + &self, + mut attrs: Vec, + ssrcs_tx: &[(Ssrc, Option)], + exts: &ExtensionMap, + params: &[PayloadParams], + ) -> MediaLine { + if self.app_tmp { + let app = (self.mid(), self.index()); + return app.as_media_line(attrs, ssrcs_tx, exts, params); + } + + attrs.push(MediaAttribute::Mid(self.mid())); + + let audio = self.kind() == MediaKind::Audio; + for (id, ext) in self.remote_extmap().iter_by_media_type(audio) { + attrs.push(MediaAttribute::ExtMap { + id, + ext: ext.clone(), + }); + } + + attrs.push(self.direction().into()); + attrs.push(MediaAttribute::Msid(self.msid().clone())); + attrs.push(MediaAttribute::RtcpMux); + + // The effective params start from the Session::codec_config to retain the + // user's configured preferred order, however they are narrowed only include + // those the remote peer wants. + let effective_params = params.iter().filter(|p| self.remote_pts().contains(&p.pt)); + + let mut pts = vec![]; + + for p in effective_params { + p.as_media_attrs(&mut attrs); + + // The pts that will be advertised in the SDP + pts.push(p.pt()); + if let Some(rtx) = p.resend() { + pts.push(rtx); + } + } + + if let Some(s) = self.simulcast() { + fn to_rids<'a>( + gs: &'a SimulcastGroups, + direction: &'static str, + ) -> impl Iterator + 'a { + gs.iter().map(move |layer| MediaAttribute::Rid { + id: layer.restriction_id.clone(), + direction, + pt: vec![], + restriction: layer.attributes.clone().unwrap_or_default(), + }) + } + attrs.extend(to_rids(&s.recv, "recv")); + attrs.extend(to_rids(&s.send, "send")); + attrs.push(MediaAttribute::Simulcast(s.clone())); + } + + // Outgoing SSRCs + let msid = format!("{} {}", self.msid().stream_id, self.msid().track_id); + for (ssrc, ssrc_rtx) in ssrcs_tx { + attrs.push(MediaAttribute::Ssrc { + ssrc: *ssrc, + attr: "cname".to_string(), + value: self.cname().to_string(), + }); + attrs.push(MediaAttribute::Ssrc { + ssrc: *ssrc, + attr: "msid".to_string(), + value: msid.clone(), + }); + if let Some(ssrc_rtx) = ssrc_rtx { + attrs.push(MediaAttribute::Ssrc { + ssrc: *ssrc_rtx, + attr: "cname".to_string(), + value: self.cname().to_string(), + }); + attrs.push(MediaAttribute::Ssrc { + ssrc: *ssrc_rtx, + attr: "msid".to_string(), + value: msid.clone(), + }); + } + } + + for (ssrc, ssrc_rtx) in ssrcs_tx { + if let Some(ssrc_rtx) = ssrc_rtx { + attrs.push(MediaAttribute::SsrcGroup { + semantics: "FID".to_string(), + ssrcs: vec![*ssrc, *ssrc_rtx], + }); + } + } + + MediaLine { + typ: self.kind().into(), + disabled: self.disabled(), + proto: Proto::Srtp, + pts, + bw: None, + attrs, + } + } +} + +impl From for MediaType { + fn from(value: MediaKind) -> Self { + match value { + MediaKind::Audio => MediaType::Audio, + MediaKind::Video => MediaType::Video, + } + } +} + +struct AsSdpParams<'a, 'b> { + pub candidates: Vec, + pub creds: IceCreds, + pub fingerprint: &'a Fingerprint, + pub setup: Setup, + pub pending: Option<&'b Changes>, + pub local_sctp_init: Option, +} + +impl<'a, 'b> AsSdpParams<'a, 'b> { + pub fn new(rtc: &'a Rtc, pending: Option<&'b Changes>) -> Self { + let (creds, candidates) = if let Some((new_creds, keep_local_candidates)) = + pending.and_then(|p| p.ice_restart()) + { + if keep_local_candidates { + // If we are performing an ICE restart and we are keeping the same + // candidates we need to use ufrag from the new ICE credentials + // in our offer. + let mut new_candidates = rtc.ice.local_candidates().collect::>(); + for c in &mut new_candidates { + c.set_ufrag(&new_creds.ufrag); + } + + (new_creds, new_candidates) + } else { + (new_creds, vec![]) + } + } else { + ( + rtc.ice.local_credentials().clone(), + rtc.ice.local_candidates().collect::>(), + ) + }; + + // RFC 8842 Section 5.2/5.5: Offerers MUST always use a=setup:actpass. + // Answerers use the negotiated role (active/passive). + // We distinguish offer vs answer by the presence of `pending` (Some = offer). + let setup = if pending.is_some() { + // This is an offer — always actpass per RFC 8842 + Setup::ActPass + } else { + // This is an answer — use the negotiated DTLS role + match rtc.dtls.is_active() { + Some(true) => Setup::Active, + Some(false) => Setup::Passive, + None => Setup::ActPass, + } + }; + + AsSdpParams { + candidates, + creds, + fingerprint: rtc.dtls.local_fingerprint(), + setup, + pending, + local_sctp_init: rtc.sctp.local_sctp_init_for_sdp(), + } + } + + fn media_attributes(&self, include_candidates: bool) -> Vec { + use MediaAttribute::*; + + let mut v = if include_candidates { + self.candidates + .iter() + .map(|c| Candidate(c.clone())) + .collect() + } else { + vec![] + }; + + v.push(IceUfrag(self.creds.ufrag.clone())); + v.push(IcePwd(self.creds.pass.clone())); + v.push(IceOptions("trickle".into())); + v.push(Fingerprint(self.fingerprint.clone())); + v.push(Setup(self.setup)); + + v + } +} + +impl fmt::Debug for SdpPendingOffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SdpPendingOffer").finish() + } +} + +impl Changes { + pub fn contains_add_app(&self) -> bool { + for i in 0..self.0.len() { + if matches!(&self.0[i], Change::AddApp(_)) { + return true; + } + } + false + } + + pub fn take_new_channels(&mut self) -> Vec<(ChannelId, ChannelConfig)> { + let mut v = vec![]; + + if self.0.is_empty() { + return v; + } + + for i in (0..self.0.len()).rev() { + if matches!(&self.0[i], Change::AddChannel(_)) { + if let Change::AddChannel(id) = self.0.remove(i) { + v.push(id); + } + } + } + + v + } + + /// Tests the given lines (from answer) corresponds to changes. + fn ensure_correct_answer(&self, lines: &[(usize, &MediaLine)]) -> Option { + if self.count_new_medias() != lines.len() { + return Some(format!( + "Differing m-line count in offer vs answer: {} != {}", + self.count_new_medias(), + lines.len() + )); + } + + 'next: for (_, l) in lines { + let mid = l.mid(); + + for m in &self.0 { + use Change::*; + match m { + AddMedia(v) if v.mid == mid => { + if !l.typ.is_media() { + return Some(format!( + "Answer m-line for mid ({}) is not of media type: {:?}", + mid, l.typ + )); + } + continue 'next; + } + AddApp(v) if *v == mid => { + if !l.typ.is_channel() { + return Some(format!( + "Answer m-line for mid ({}) is not a data channel: {:?}", + mid, l.typ + )); + } + continue 'next; + } + _ => {} + } + } + + return Some(format!("Mid in answer is not in offer: {mid}")); + } + + None + } + + fn count_new_medias(&self) -> usize { + self.0 + .iter() + .filter(|c| matches!(c, Change::AddMedia(_) | Change::AddApp(_))) + .count() + } + + pub fn as_new_medias<'a, 'b: 'a>( + &'a self, + index_start: usize, + config: &'b CodecConfig, + exts: &'b ExtensionMap, + ) -> impl Iterator + 'a { + // Use a separate counter that only advances for entries that actually produce + // an m-line (AddMedia / AddApp). Non-media entries (Direction, IceRestart, + // AddChannel) must not consume an index slot, otherwise the resulting m-line + // indices would be non-contiguous and out of sync with their array positions. + let mut media_idx = 0usize; + self.0.iter().filter_map(move |c| { + let result = c.as_new_media(index_start + media_idx, config, exts); + if result.is_some() { + media_idx += 1; + } + result + }) + } + + pub(crate) fn apply_to(&self, lines: &mut [MediaLine]) { + for change in &self.0 { + if let Change::Direction(mid, dir) = change { + if let Some(line) = lines.iter_mut().find(|l| l.mid() == *mid) { + if let Some(dir_pos) = line.attrs.iter().position(|a| a.is_direction()) { + line.attrs[dir_pos] = (*dir).into(); + } + } + } + } + } + + fn ssrcs_for_mid(&self, mid: Mid) -> &[(Ssrc, Option)] { + let maybe_add_media = self + .0 + .iter() + .filter_map(|c| { + if let Change::AddMedia(m) = c { + Some(m) + } else { + None + } + }) + .find(|m| m.mid == mid); + + let Some(m) = maybe_add_media else { + return &[]; + }; + + &m.ssrcs + } +} + +impl Change { + fn as_new_media( + &self, + index: usize, + config: &CodecConfig, + exts: &ExtensionMap, + ) -> Option { + use Change::*; + match self { + AddMedia(v) => { + // TODO can we avoid all this cloning? + let mut add = v.clone(); + add.pts = config.all_for_kind(v.kind).map(|p| p.pt()).collect(); + add.exts = exts.cloned_with_type(v.kind.is_audio()); + add.index = index; + + Some(Media::from_add_media(add)) + } + AddApp(mid) => Some(Media::from_app_tmp(*mid, index)), + _ => None, + } + } +} + +#[cfg(test)] +mod test { + use std::time::Instant; + + use sdp::RestrictionId; + use sdp::SimulcastLayer as SdpSimulcastLayer; + + use crate::format::Codec; + use crate::media::{Simulcast, SimulcastLayer}; + use crate::sdp::RtpMap; + + use super::*; + + fn resolve_pt(m_line: &MediaLine, needle: Pt) -> RtpMap { + m_line + .attrs + .iter() + .find_map(|attr| match attr { + MediaAttribute::RtpMap { pt, value } if *pt == needle => Some(*value), + _ => None, + }) + .unwrap_or_else(|| panic!("Expected to find RtpMap for {needle}")) + } + + fn count_lines(lines: &str, what: &str) -> usize { + lines.lines().filter(|l| l == &what).count() + } + + fn get_setup_from_media_line(line: &MediaLine) -> Setup { + line.setup().expect("Expected a=setup attribute in SDP") + } + + /// RFC 8842 §5.2: an offer MUST use a=setup:actpass regardless of DTLS role. + #[test] + fn offer_uses_actpass() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc = Rtc::new(now); + + let mut change = rtc.sdp_api(); + change.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + let (offer, _pending) = change.apply().unwrap(); + + let setup = get_setup_from_media_line(&offer.media_lines[0]); + assert_eq!( + setup, + Setup::ActPass, + "Offer must use a=setup:actpass per RFC 8842 §5.2" + ); + } + + /// RFC 8842 §5.5: the answerer uses the negotiated DTLS role (active or passive). + /// When the offerer uses actpass, the answerer picks passive (DTLS server) and the + /// offerer becomes active (DTLS client). The answerer's SDP must reflect passive. + #[test] + fn answer_uses_negotiated_dtls_role() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut offerer = Rtc::new(now); + let mut answerer = Rtc::new(now); + + // Offerer creates the offer. + let mut change = offerer.sdp_api(); + change.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); + let (offer, pending) = change.apply().unwrap(); + + // Verify the offer itself is actpass. + assert_eq!( + get_setup_from_media_line(&offer.media_lines[0]), + Setup::ActPass, + "Offer must use a=setup:actpass" + ); + + // Answerer accepts and generates the answer. + let answer = answerer.sdp_api().accept_offer(offer).unwrap(); + + // When the offerer sends actpass, the answerer takes the passive DTLS role + // (see init_dtls: ActPass from remote → local takes Passive). + // The answerer's SDP must reflect that with a=setup:passive. + assert_eq!( + get_setup_from_media_line(&answer.media_lines[0]), + Setup::Passive, + "Answerer's SDP must use a=setup:passive when responding to an actpass offer" + ); + + // Complete the exchange on the offerer side. + offerer.sdp_api().accept_answer(pending, answer).unwrap(); + + // Now generate a subsequent offer from the offerer (re-offer). + // Even after the DTLS role is settled (offerer is now passive), a new offer + // must still carry a=setup:actpass per RFC 8842 §5.2. + let mut change = offerer.sdp_api(); + change.add_media(MediaKind::Video, Direction::SendRecv, None, None, None); + let (reoffer, _) = change.apply().unwrap(); + + assert_eq!( + get_setup_from_media_line(&reoffer.media_lines[0]), + Setup::ActPass, + "Re-offer must still use a=setup:actpass even after DTLS role is settled" + ); + } + + #[test] + fn test_out_of_order_error() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + let mut change1 = rtc1.sdp_api(); + change1.add_channel("ch1".into()); + let (offer1, pending1) = change1.apply().unwrap(); + + let mut change2 = rtc2.sdp_api(); + change2.add_channel("ch2".into()); + let (offer2, _) = change2.apply().unwrap(); + + // invalidates pending1 + let _ = rtc1.sdp_api().accept_offer(offer2).unwrap(); + let answer2 = rtc2.sdp_api().accept_offer(offer1).unwrap(); + + let r = rtc1.sdp_api().accept_answer(pending1, answer2); + + assert!(matches!(r, Err(RtcError::ChangesOutOfOrder))); + } + + #[test] + fn sdp_api_merge_works() { + crate::init_crypto_default(); + + let mut rtc = Rtc::new(Instant::now()); + let mut changes = rtc.sdp_api(); + changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + let (offer, pending) = changes.apply().unwrap(); + + let mut changes = rtc.sdp_api(); + changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + changes.merge(pending); + let (new_offer, _) = changes.apply().unwrap(); + + // After merge(), the original AddMedia (audio) is prepended before the new AddMedia + // (video) to preserve its original m-line position. So audio is at index 0 and video at 1. + assert_eq!(offer.media_lines[0], new_offer.media_lines[0]); + assert_eq!(new_offer.media_lines[0].typ, MediaType::Audio); + assert_eq!(new_offer.media_lines[1].typ, MediaType::Video); + assert_eq!(new_offer.media_lines.len(), 2); + } + + // The indices assigned to Media objects by as_new_medias() must be contiguous starting + // from index_start. Only entries that produce m-lines (AddMedia / AddApp) should + // consume an index slot – non-media entries (AddChannel, Direction, IceRestart) must + // not advance the counter. + // + // We test this by building a Changes vector directly and inspecting the indices + // returned by as_new_medias(). + #[test] + fn as_new_medias_contiguous_indices_with_non_media_changes() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc = Rtc::new(now); + + // Build a Changes vector that interleaves non-media entries with media entries. + // The non-media AddChannel entry must not consume an index slot. + // + // Changes = [AddApp(mid0), AddChannel(id, cfg), AddMedia(audio)] + // + // index_start = 0 (fresh session) + // Expected indices: AddApp → 0, AddMedia(audio) → 1 + // Buggy indices: AddApp → 0, AddMedia(audio) → 2 + let mut changes = Changes::default(); + let mid_app = rtc.new_mid(); + changes.0.push(Change::AddApp(mid_app)); + let mid_audio = rtc.new_mid(); + let chan_id = rtc.chan.new_channel(&ChannelConfig { + label: "ch".into(), + ..Default::default() + }); + changes.0.push(Change::AddChannel(( + chan_id, + ChannelConfig { + label: "ch".into(), + ..Default::default() + }, + ))); + changes.0.push(Change::AddMedia(AddMedia { + mid: mid_audio, + cname: "test".into(), + msid: Msid { + stream_id: "stream".into(), + track_id: "track".into(), + }, + kind: MediaKind::Audio, + dir: Direction::SendOnly, + ssrcs: vec![], + simulcast: None, + pts: vec![], + exts: ExtensionMap::empty(), + index: 0, // will be overwritten by as_new_media() + })); + + let config = CodecConfig::new_with_defaults(); + let exts = ExtensionMap::standard(); + + let medias: Vec = changes.as_new_medias(0, &config, &exts).collect(); + + // Should produce two Media objects: one for AddApp, one for AddMedia(audio). + assert_eq!(medias.len(), 2, "Expected 2 media-producing entries"); + + // The first m-line is AddApp at index 0. + assert_eq!(medias[0].index(), 0, "AddApp should have index 0"); + + // The second m-line is Audio. It must get the contiguous index 1, not 2 (which + // would result from counting the non-media AddChannel entry as a slot). + assert_eq!( + medias[1].index(), + 1, + "AddMedia(audio) should have contiguous index 1, not 2" + ); + } + + // AddMedia/AddApp entries from a pending offer carry already-allocated MIDs whose + // m-line positions the remote peer may have committed to. When merging a pending offer + // into a new SdpApi, those original entries must be prepended so they keep their + // original lower-index positions rather than being displaced by new changes. + #[test] + fn sdp_api_merge_stale_media_keeps_original_position() { + crate::init_crypto_default(); + + let mut rtc = Rtc::new(Instant::now()); + + // First offer: add audio. Audio will be at index 0. + let mut changes = rtc.sdp_api(); + let _mid_audio = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + let (offer1, pending) = changes.apply().unwrap(); + + // The original offer has audio at position 0. + assert_eq!(offer1.media_lines[0].typ, MediaType::Audio); + + // Simulate glare: a new SdpApi adds video, then merges the pending offer + // containing the original audio AddMedia. + let mut changes = rtc.sdp_api(); + changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + changes.merge(pending); + let (offer2, _) = changes.apply().unwrap(); + + // The original audio must be prepended before the new video, keeping it at + // position 0 — the same position as in the original offer. + assert_eq!( + offer2.media_lines[0].typ, + MediaType::Audio, + "Original audio from pending should be at position 0 (same as original offer)" + ); + assert_eq!( + offer2.media_lines[1].typ, + MediaType::Video, + "New video added to current SdpApi should be at position 1" + ); + assert_eq!( + offer1.media_lines[0], offer2.media_lines[0], + "Audio m-line must be identical between original and merged offers" + ); + } + + // When a new SdpApi has its own AddMedia changes and also merges a pending offer that + // has AddMedia changes, the original pending entries must come before the new ones so + // their m-line positions are stable: + // 1. First offer: [AddMedia(audio), AddMedia(video)] → audio at 0, video at 1 + // 2. Glare – pending is saved + // 3. New SdpApi: [AddMedia(screen)] + // 4. After merge: [AddMedia(audio), AddMedia(video), AddMedia(screen)] + // → audio at 0, video at 1, screen at 2 (CORRECT) + #[test] + fn sdp_api_merge_with_direction_and_new_media_preserves_positions() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + // Establish one audio m-line so we can change its direction later. + let mut changes = rtc1.sdp_api(); + let mid_audio = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + let (offer0, pending0) = changes.apply().unwrap(); + let answer0 = rtc2.sdp_api().accept_offer(offer0).unwrap(); + rtc1.sdp_api().accept_answer(pending0, answer0).unwrap(); + // Session now: audio at index 0. + + // Create an offer with two new medias: video1 and video2. + // video1 → session position 1, video2 → session position 2. + let mut changes = rtc1.sdp_api(); + let mid_v1 = changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + let mid_v2 = changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + let (offer1, pending1) = changes.apply().unwrap(); + + assert_eq!(offer1.media_lines[1].mid(), mid_v1, "video1 at idx 1"); + assert_eq!(offer1.media_lines[2].mid(), mid_v2, "video2 at idx 2"); + + // Glare: create a new SdpApi that: + // (a) changes direction of existing audio m-line (non-media Change) + // (b) adds a new screen-share video m-line + // Then merge the original pending1 into it. + let mut changes = rtc1.sdp_api(); + changes.set_direction(mid_audio, Direction::Inactive); + let mid_screen = changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + changes.merge(pending1); + let (offer2, _) = changes.apply().unwrap(); + + // Session already has audio at index 0. + // The merged offer must have: + // position 0: audio (existing, unchanged) + // position 1: video1 (from pending — same position as offer1) + // position 2: video2 (from pending — same position as offer1) + // position 3: screen (newly added in the new SdpApi) + assert_eq!( + offer2.media_lines.len(), + 4, + "Merged offer should have 4 m-lines: audio + video1 + video2 + screen" + ); + + // Original video m-lines from the pending offer must keep their positions (1 and 2). + assert_eq!( + offer2.media_lines[1].mid(), + mid_v1, + "video1 from pending should remain at position 1" + ); + assert_eq!( + offer1.media_lines[1], offer2.media_lines[1], + "video1 m-line should be identical in both offers" + ); + assert_eq!( + offer2.media_lines[2].mid(), + mid_v2, + "video2 from pending should remain at position 2" + ); + assert_eq!( + offer1.media_lines[2], offer2.media_lines[2], + "video2 m-line should be identical in both offers" + ); + + // New screen share is at position 3, after the original pending m-lines. + assert_eq!( + offer2.media_lines[3].mid(), + mid_screen, + "New screen-share should be at position 3" + ); + } + + #[test] + fn test_rtp_payload_priority() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::builder() + .clear_codecs() + .enable_h264(true) + .enable_vp8(true) + .enable_vp9(true) + .build(now); + let mut rtc2 = Rtc::builder() + .clear_codecs() + .enable_vp8(true) + .enable_h264(true) + .build(now); + + let mut change1 = rtc1.sdp_api(); + change1.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + let (offer1, _) = change1.apply().unwrap(); + + let answer = rtc2.sdp_api().accept_offer(offer1).unwrap(); + assert_eq!( + answer.media_lines.len(), + 1, + "There should be one mline only" + ); + + let first_mline = &answer.media_lines[0]; + let first_pt = resolve_pt(first_mline, first_mline.pts[0]); + + assert_eq!( + first_pt.codec, + Codec::Vp8, + "The first PT returned should be the highest priority PT from the answer that is supported." + ); + + let vp9_unsupported = first_mline + .pts + .iter() + .any(|pt| resolve_pt(first_mline, *pt).codec == Codec::Vp9); + + assert!( + !vp9_unsupported, + "VP9 was not offered, so it should not be present in the answer" + ); + } + + #[test] + fn non_simulcast_rids() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + // Test initial media creation + let mid = { + let mut changes = rtc1.sdp_api(); + let mid = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + let (offer, pending) = changes.apply().unwrap(); + let answer = rtc2.sdp_api().accept_offer(offer).unwrap(); + rtc1.sdp_api().accept_answer(pending, answer).unwrap(); + + assert!(matches!(rtc1.media(mid).unwrap().rids_rx(), Rids::Any)); + assert!(matches!(rtc1.media(mid).unwrap().rids_tx(), Rids::None)); + assert!(matches!(rtc2.media(mid).unwrap().rids_rx(), Rids::Any)); + assert!(matches!(rtc2.media(mid).unwrap().rids_tx(), Rids::None)); + + mid + }; + + // Test later updates to that media + { + let mut changes = rtc1.sdp_api(); + changes.set_direction(mid, Direction::Inactive); + let (offer, pending) = changes.apply().unwrap(); + let answer = rtc2.sdp_api().accept_offer(offer).unwrap(); + rtc1.sdp_api().accept_answer(pending, answer).unwrap(); + + assert!(matches!(rtc1.media(mid).unwrap().rids_rx(), Rids::Any)); + assert!(matches!(rtc1.media(mid).unwrap().rids_tx(), Rids::None)); + assert!(matches!(rtc2.media(mid).unwrap().rids_rx(), Rids::Any)); + assert!(matches!(rtc2.media(mid).unwrap().rids_tx(), Rids::None)); + } + } + + #[test] + fn simulcast_ssrc_allocation() { + crate::init_crypto_default(); + + let mut rtc1 = Rtc::new(Instant::now()); + + let mut simulcast = Simulcast::new(); + + simulcast.add_send_layer(SimulcastLayer::new("h")); + simulcast.add_send_layer(SimulcastLayer::new("m")); + simulcast.add_send_layer(SimulcastLayer::new("l")); + + let mut change = rtc1.sdp_api(); + change.add_media( + MediaKind::Video, + Direction::SendOnly, + None, + None, + Some(simulcast), + ); + + let Change::AddMedia(am) = &change.changes[0] else { + panic!("Not AddMedia?!"); + }; + + // these should be organized in order: m, h, l + let pending_ssrcs = am.ssrcs.clone(); + assert_eq!(pending_ssrcs.len(), 3); + + for p in &pending_ssrcs { + assert!(p.1.is_some()); // all should have rtx + } + + let (offer, _) = change.apply().unwrap(); + let sdp = offer.into_inner(); + let line = &sdp.media_lines[0]; + + assert_eq!( + line.simulcast().unwrap().send, + SimulcastGroups(vec![ + SdpSimulcastLayer { + restriction_id: RestrictionId("h".into(), true), + attributes: None, + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("m".into(), true), + attributes: None, + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("l".into(), true), + attributes: None, + }, + ]) + ); + + // Each SSRC, both regular and RTX get their own a=ssrc line. + assert_eq!(line.ssrc_info().len(), pending_ssrcs.len() * 2); + + let fids: Vec<_> = line + .attrs + .iter() + .filter_map(|a| { + if let MediaAttribute::SsrcGroup { semantics, ssrcs } = a { + // We don't have any other semantics right now. + assert_eq!(semantics, "FID"); + assert_eq!(ssrcs.len(), 2); + Some((ssrcs[0], ssrcs[1])) + } else { + None + } + }) + .collect(); + + assert_eq!(fids.len(), pending_ssrcs.len()); + + for (a, b) in fids.iter().zip(pending_ssrcs.iter()) { + assert_eq!(a.0, b.0); + assert_eq!(Some(a.1), b.1); + } + + let line_string = line.to_string(); + + // The SDP offer should contain layers without any attributes + assert_eq!(count_lines(&line_string, "a=rid:h send"), 1); + assert_eq!(count_lines(&line_string, "a=rid:m send"), 1); + assert_eq!(count_lines(&line_string, "a=rid:l send"), 1); + } + + #[test] + fn simulcast_attributes() { + crate::init_crypto_default(); + + let mut rtc1 = Rtc::new(Instant::now()); + + let mut simulcast = Simulcast::new(); + + // High layer + simulcast.add_send_layer( + SimulcastLayer::new_with_attributes("high") + .max_width(1280) + .max_height(720) + .max_br(1100000) + .max_br(1300000) + .max_br(1500000) // the last one wins + .max_fps(30) + .build(), + ); + + // Medium layer + simulcast.add_send_layer( + SimulcastLayer::new_with_attributes("medium") + .max_width(640) + .max_height(360) + .max_br(600000) + // No max_fps + .build(), + ); + + // Low layer + simulcast.add_send_layer( + SimulcastLayer::new_with_attributes("low") + // No max_width + .max_height(180) + .max_br(200000) + .max_fps(15) + .build(), + ); + + // Custom attribute + simulcast.add_send_layer( + SimulcastLayer::new_with_attributes("custom") + .custom("foo", "bar") + .build(), + ); + + // No attributes + simulcast.add_send_layer(SimulcastLayer::new_with_attributes("no_attrs").build()); + + let mut change = rtc1.sdp_api(); + change.add_media( + MediaKind::Video, + Direction::SendOnly, + None, + None, + Some(simulcast), + ); + + let (offer, _) = change.apply().unwrap(); + let sdp = offer.into_inner(); + let line = &sdp.media_lines[0]; + + fn pairs_to_some_vec(pairs: &[(&str, &str)]) -> Option> { + Some( + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) + } + + assert_eq!( + line.simulcast().unwrap().send, + SimulcastGroups(vec![ + SdpSimulcastLayer { + restriction_id: RestrictionId("high".into(), true), + attributes: pairs_to_some_vec(&[ + ("max-width", "1280"), + ("max-height", "720"), + ("max-br", "1500000"), + ("max-fps", "30"), + ]), + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("medium".into(), true), + attributes: pairs_to_some_vec(&[ + ("max-width", "640"), + ("max-height", "360"), + ("max-br", "600000"), + ]), + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("low".into(), true), + attributes: pairs_to_some_vec(&[ + ("max-height", "180"), + ("max-br", "200000"), + ("max-fps", "15"), + ]), + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("custom".into(), true), + attributes: pairs_to_some_vec(&[("foo", "bar"),]), + }, + SdpSimulcastLayer { + restriction_id: RestrictionId("no_attrs".into(), true), + attributes: None, + }, + ]) + ); + + // The SDP offer should contain layers with our attributes + let line_string = line.to_string(); + assert_eq!( + count_lines( + &line_string, + "a=rid:high send max-width=1280;max-height=720;max-br=1500000;max-fps=30" + ), + 1 + ); + assert_eq!( + count_lines( + &line_string, + "a=rid:medium send max-width=640;max-height=360;max-br=600000" + ), + 1 + ); + assert_eq!( + count_lines( + &line_string, + "a=rid:low send max-height=180;max-br=200000;max-fps=15" + ), + 1 + ); + assert_eq!(count_lines(&line_string, "a=rid:custom send foo=bar"), 1); + // No space at the end + assert_eq!(count_lines(&line_string, "a=rid:no_attrs send"), 1); + } + + #[test] + fn test_local_max_message_size_advertised() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc = Rtc::new(now); + + // Create an offer with a data channel + let mut change = rtc.sdp_api(); + change.add_channel("test-channel".into()); + let (offer, _) = change.apply().unwrap(); + + // Find the application m-line in the offer + let app_line = offer + .media_lines + .iter() + .find(|m| m.typ.is_channel()) + .expect("should have application m-line"); + + // Verify that max-message-size attribute is present and matches LOCAL_MAX_MESSAGE_SIZE + let max_size = app_line + .max_message_size() + .expect("max-message-size attribute should be present"); + + assert_eq!( + max_size, + crate::sctp::LOCAL_MAX_MESSAGE_SIZE as usize, + "max-message-size should match LOCAL_MAX_MESSAGE_SIZE constant" + ); + + // Also verify it's in the SDP string output + let sdp_string = offer.to_sdp_string(); + let expected_line = format!("a=max-message-size:{}", crate::sctp::LOCAL_MAX_MESSAGE_SIZE); + assert!( + sdp_string.contains(&expected_line), + "SDP should contain max-message-size attribute with LOCAL_MAX_MESSAGE_SIZE value" + ); + } + + #[test] + fn test_remote_max_message_size_parsing() { + // Parse SDP with max-message-size attribute and verify value is extracted correctly + let sdp = "v=0\r\n\ + o=- 0 0 IN IP4 172.17.0.1\r\n\ + s=-\r\n\ + c=IN IP4 172.17.0.1\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + a=fingerprint:sha-256 B4:12:1C:7C:7D:ED:F1:FA:61:07:57:9C:29:BE:58:E3:BC:41:E7:13:8E:7D\ + :D3:9D:1F:94:6E:A5:23:46:94:23\r\n\ + m=application 9999 UDP/DTLS/SCTP webrtc-datachannel\r\n\ + a=mid:0\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword1234\r\n\ + a=setup:actpass\r\n\ + a=sctp-port:5000\r\n\ + a=max-message-size:131072\r\n\ + "; + + let offer = SdpOffer::from_sdp_string(sdp).expect("should parse"); + + // Find the application m-line + let app_line = offer + .media_lines + .iter() + .find(|m| m.typ.is_channel()) + .expect("should have application m-line"); + + assert_eq!( + app_line.max_message_size(), + Some(131072), + "max-message-size should be parsed as 131072" + ); + } + + #[test] + fn test_remote_max_message_size_applied() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + // Create an offer from rtc1 with a channel + let mut change1 = rtc1.sdp_api(); + change1.add_channel("test-channel".into()); + let (offer1, pending1) = change1.apply().unwrap(); + + // Get the offer SDP string and modify it to have a custom max-message-size + let custom_max_size1 = 98304u32; + let sdp_string = offer1.to_sdp_string(); + let modified_sdp = sdp_string.replace( + &format!("a=max-message-size:{}", crate::sctp::LOCAL_MAX_MESSAGE_SIZE), + &format!("a=max-message-size:{}", custom_max_size1), + ); + + let modified_offer = + SdpOffer::from_sdp_string(&modified_sdp).expect("modified SDP should parse"); + + let answer = rtc2.sdp_api().accept_offer(modified_offer).unwrap(); + + // Verify that rtc2's SCTP send limit is set to the custom value from rtc1's offer + assert_eq!( + rtc2.sctp.remote_max_message_size(), + custom_max_size1, + "rtc2 should have remote max message size set to rtc1's advertised value" + ); + + // Now verify the reverse: rtc1 accepts rtc2's answer and applies its max-message-size + let custom_max_size2 = 131072u32; + let sdp_string = answer.to_sdp_string(); + let modified_sdp = sdp_string.replace( + &format!("a=max-message-size:{}", crate::sctp::LOCAL_MAX_MESSAGE_SIZE), + &format!("a=max-message-size:{}", custom_max_size2), + ); + + let modified_answer = + SdpAnswer::from_sdp_string(&modified_sdp).expect("modified SDP should parse"); + + rtc1.sdp_api() + .accept_answer(pending1, modified_answer) + .unwrap(); + + assert_eq!( + rtc1.sctp.remote_max_message_size(), + custom_max_size2, + "rtc1 should have remote max message size set to rtc2's advertised value" + ); + } + + #[test] + fn test_remote_max_message_size_zero_means_unbounded() { + // RFC 8841 §6.1: a=max-message-size:0 means the peer can receive a message + // of any size, so we represent it as u32::MAX rather than passing 0 through + // (which would reject every non-empty outbound message in sctp-proto). + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + let mut change1 = rtc1.sdp_api(); + change1.add_channel("test-channel".into()); + let (offer1, _pending1) = change1.apply().unwrap(); + + let sdp_string = offer1.to_sdp_string(); + let modified_sdp = sdp_string.replace( + &format!("a=max-message-size:{}", crate::sctp::LOCAL_MAX_MESSAGE_SIZE), + "a=max-message-size:0", + ); + + let modified_offer = + SdpOffer::from_sdp_string(&modified_sdp).expect("modified SDP should parse"); + + rtc2.sdp_api().accept_offer(modified_offer).unwrap(); + + assert_eq!( + rtc2.sctp.remote_max_message_size(), + u32::MAX, + "a=max-message-size:0 should be treated as unbounded (u32::MAX)" + ); + } + + #[test] + fn test_remote_max_message_size_too_large_clamps_to_unbounded() { + crate::init_crypto_default(); + + let now = Instant::now(); + let mut rtc1 = Rtc::new(now); + let mut rtc2 = Rtc::new(now); + + let mut change1 = rtc1.sdp_api(); + change1.add_channel("test-channel".into()); + let (offer1, _pending1) = change1.apply().unwrap(); + + let sdp_string = offer1.to_sdp_string(); + let modified_sdp = sdp_string.replace( + &format!("a=max-message-size:{}", crate::sctp::LOCAL_MAX_MESSAGE_SIZE), + "a=max-message-size:4294967296", + ); + + let modified_offer = + SdpOffer::from_sdp_string(&modified_sdp).expect("modified SDP should parse"); + + rtc2.sdp_api().accept_offer(modified_offer).unwrap(); + + assert_eq!( + rtc2.sctp.remote_max_message_size(), + u32::MAX, + "a=max-message-size larger than u32::MAX should be treated as unbounded" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/channel.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/channel.rs new file mode 100644 index 000000000..e38fb3027 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/channel.rs @@ -0,0 +1,664 @@ +//! Data channel related types. + +use std::{fmt, str, time::Instant}; + +use crate::sctp::RtcSctp; +use crate::util::already_happened; +use crate::{Rtc, RtcError}; + +pub use crate::sctp::ChannelConfig; +pub use crate::sctp::Reliability; +pub use crate::sctp::SctpInitData; + +/// Identifier of a data channel. +/// +/// This is NOT the SCTP stream id. +// Deliberately not Deref or From to avoid this Id being created outside of this module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ChannelId(usize); + +/// Data channel data from remote peer. +/// +/// This is obtained via [`Event::ChannelData`][crate::Event::ChannelData]. +#[derive(PartialEq, Eq)] +pub struct ChannelData { + /// Identifier of the channel this data was sent on. + /// + /// The channel would have been previously announced via + /// [`Event::ChannelOpen`][crate::Event::ChannelOpen]. + pub id: ChannelId, + + /// Tells whether the sender sent this data as binary or text. + pub binary: bool, + + /// The actual data sent. If `binary` is false, this can be converted to text. + pub data: Vec, +} + +/// Channel for sending data to the remote peer. +/// +/// Get this handle from [`Rtc::channel()`][crate::Rtc::channel()]. +pub struct Channel<'a> { + sctp_stream_id: u16, + rtc: &'a mut Rtc, +} + +impl<'a> Channel<'a> { + pub(crate) fn new(sctp_stream_id: u16, rtc: &'a mut Rtc) -> Self { + Channel { + rtc, + sctp_stream_id, + } + } + + /// Write data to the remote peer and indicate whether it's text or binary. + /// + /// Returns true or false whether the buffer was accepted or not. + #[must_use = "Whether the buffer was accepted by the write()"] + pub fn write(&mut self, binary: bool, buf: &[u8]) -> Result { + // If it's not available, don't accept. + let available = self.rtc.sctp.available(); + if buf.len() > available { + return Ok(false); + } + + // Try write. + let written = self.rtc.sctp.write(self.sctp_stream_id, binary, buf)?; + + // Invariant: if available calculation is correct, we should have accepted. + assert_eq!( + written, + buf.len(), + "Data channel write() less than entire buffer" + ); + + Ok(true) + } + + /// Get the amount of buffered data. + /// + /// Returns 0 if the channel is closed or encountered some error. This is to + /// be similar to the [RTCPeerConnection equivalent][buff] + /// + /// [buff]: https://developer.mozilla.org/en-US/docs/Web/API/RTCDataChannel/bufferedAmount + pub fn buffered_amount(&mut self) -> usize { + self.rtc.sctp.buffered_amount(self.sctp_stream_id) + } + + /// Set the threshold to emit an + /// [`Event::ChannelBufferedAmountLow`][crate::Event::ChannelBufferedAmountLow] + /// + /// Setting this on a closed or broken channel does not show an error. This is + /// be similar to the [RTCPeerConnection equivalent][buff] + /// + /// [buff]: https://developer.mozilla.org/en-US/docs/Web/API/RTCDataChannel/bufferedAmountLowThreshold + pub fn set_buffered_amount_low_threshold(&mut self, threshold: usize) { + self.rtc + .sctp + .set_buffered_amount_low_threshold(self.sctp_stream_id, threshold); + } + + /// Get the channel config. + /// + /// The config is not available in every case depending on whether the channel was + /// negotiated in- or out of band. + /// + /// # In-band negotiation (the usual case) + /// + /// For (regular) in-band negotiation (DCEP, Data Channel Establishment Protocol), this + /// returns `None` until the DCEP handshake completes. The config is guaranteed to be + /// available when [`Event::ChannelOpen`][crate::Event::ChannelOpen] is emitted. + /// + /// # Out-of-band negotiation + /// + /// Returns `None` when the remote side created the data channel connection without using + /// DCEP. This is called out-of-band negotiation, where the remote peer opens a stream + /// but doesn't send the channel configuration through the DCEP protocol messages. + /// + /// For locally created out-of-band channels, the config is always available since it + /// was provided during channel creation. + /// + /// In str0m, DCEP is disabled by setting the `negotiated` field to `Some(stream_id)` in + /// [`ChannelConfig`]. This corresponds to the `negotiated: true` property in the + /// browser's [`createDataChannel()`][n] dictionary. + /// + /// [n]: https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/createDataChannel#negotiated + pub fn config(&self) -> Option<&ChannelConfig> { + self.rtc.sctp.config(self.sctp_stream_id) + } +} + +impl fmt::Debug for ChannelData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut ds = f.debug_struct("ChannelData"); + + ds.field("id", &self.id); + ds.field("binary", &self.binary); + + let len = &self.data.len(); + if self.binary { + ds.field("data", len); + } else { + match str::from_utf8(&self.data) { + Ok(s) => { + const MAX_LINE_WIDTH: usize = 79; + const REST_OF_LINE_WIDTH: usize = + "ChannelData { id: ChannelId(0), binary: false, data: \"\" }".len(); + const TUPLE_WIDTH: usize = "(xxx, ..)".len(); + const DATA_WIDTH: usize = MAX_LINE_WIDTH - REST_OF_LINE_WIDTH; + const PREFIX_WIDTH: usize = DATA_WIDTH - TUPLE_WIDTH; + if s.is_ascii() { + if len > &DATA_WIDTH { + let trunc: String = s.chars().take(PREFIX_WIDTH).collect(); + ds.field("data", &format_args!("({}, \"{}\"..)", len, trunc)); + } else { + ds.field("data", &s); + } + } else { + ds.field("data", len); + } + } + Err(e) => { + ds.field("data", &format_args!("{:?}", (len, &e))); + } + } + } + + ds.finish() + } +} + +#[derive(Debug, Default)] +pub(crate) struct ChannelHandler { + allocations: Vec, + next_channel_id: usize, + /// Stream IDs of closed channels whose reset handshake is still + /// outstanding, excluded from allocation until it completes. + closed_stream_ids: Vec, +} + +#[derive(Debug)] +struct ChannelAllocation { + id: ChannelId, + + /// Stream id, when it is known. This might be delayed awaiting sctp initialization to + /// know if we are client or server, or awaiting the reset handshake of a previous + /// incarnation of a negotiated id. + sctp_stream_id: Option, + + /// The out-of-band negotiated stream id the user asked for. Promoted to + /// `sctp_stream_id` once reset which was holding it completes. + negotiated_stream_id: Option, + + /// Holds the config until it is used in handle_timeout. + config: Option, +} + +impl ChannelHandler { + pub fn new_channel(&mut self, config: &ChannelConfig) -> ChannelId { + let id = self.next_channel_id(); + + // Out-of-band negotiated means the user names the stream id + // instead of us allocating one. We record it as a request in + // and leave `sctp_stream_id` unset. + let negotiated_stream_id = config.negotiated; + if let Some(sctp_stream_id) = negotiated_stream_id { + let exists = self.allocations.iter().any(|a| { + a.sctp_stream_id == Some(sctp_stream_id) + || a.negotiated_stream_id == Some(sctp_stream_id) + }); + assert!( + !exists, + "sctp_stream_id ({}) exists already", + sctp_stream_id + ); + } + + let alloc = ChannelAllocation { + id, + sctp_stream_id: None, + negotiated_stream_id, + // The config is none until we confirm we definitely want this channel. + config: None, + }; + + debug!("Allocate channel id: {:?}", id); + self.allocations.push(alloc); + + id + } + + pub fn confirm(&mut self, id: ChannelId, config: ChannelConfig) { + let a = self + .allocations + .iter_mut() + .find(|a| a.id == id) + .expect("Entry for issued channel id"); + a.config = Some(config); + } + + /// For translating sctp stream id to ChannelId. Any event out of sctp goes via this. + pub fn channel_id_by_stream_id(&self, sctp_stream_id: u16) -> Option { + self.allocations + .iter() + .find(|a| a.sctp_stream_id == Some(sctp_stream_id)) + .map(|a| a.id) + } + + /// Look up sctp stream id for channel id. + pub fn stream_id_by_channel_id(&self, id: ChannelId) -> Option { + self.allocations + .iter() + .find(|a| a.id == id) + .and_then(|a| a.sctp_stream_id) + } + + pub(crate) fn handle_timeout(&mut self, _now: Instant, sctp: &mut RtcSctp) { + if !sctp.is_inited() { + return; + } + + // Allocate sctp channel ids for ones that are missing. + self.do_allocations(sctp); + + // After do_allocations so we get a channel for any confirmed. + self.open_channels(sctp); + } + + /// Allocate next available `ChannelId`. + fn next_channel_id(&mut self) -> ChannelId { + let id = self.next_channel_id; + self.next_channel_id += 1; + + ChannelId(id) + } + + /// Whether a stream id is held back awaiting a reset completion. + fn is_held(&self, sctp_stream_id: u16) -> bool { + self.closed_stream_ids.contains(&sctp_stream_id) + } + + fn need_allocation(&self) -> bool { + self.allocations.iter().any(|a| { + a.sctp_stream_id.is_none() + // A negotiated id that is still held cannot be allocated yet. + && a.negotiated_stream_id.is_none_or(|want| !self.is_held(want)) + }) + } + + fn need_open(&self) -> bool { + self.allocations + .iter() + .any(|a| a.config.is_some() && a.sctp_stream_id.is_some()) + } + + // Do automatic allocations of sctp stream id. + fn do_allocations(&mut self, sctp: &RtcSctp) { + if !self.need_allocation() { + return; + } + + // RFC 8831 + // Unless otherwise defined or negotiated, the + // streams are picked based on the DTLS role (the client picks even + // stream identifiers, and the server picks odd stream identifiers). + let base = if sctp.is_client() { 0 } else { 1 }; + + let mut taken: Vec = self + .allocations + .iter() + .filter_map(|a| a.sctp_stream_id) + .chain(self.closed_stream_ids.iter().copied()) + .collect(); + + for a in &mut self.allocations { + if a.sctp_stream_id.is_some() { + continue; + } + + // Out-of-band negotiated. The user picked the id, all we do is wait until + // no closed generation of it is still awaiting its reset handshake. + if let Some(want) = a.negotiated_stream_id { + if taken.contains(&want) { + debug!("Negotiated stream id {} still held, retry later", want); + continue; + } + + debug!("Associate negotiated stream id {:?} => {}", a.id, want); + a.sctp_stream_id = Some(want); + taken.push(want); + continue; + } + + // We need to allocate + let mut proposed = base; + + while taken.contains(&proposed) { + proposed += 2 + } + + // Found the next free. + debug!("Associate stream id {:?} => {}", a.id, proposed); + a.sctp_stream_id = Some(proposed); + taken.push(proposed); + } + } + + // Actually open channels. + fn open_channels(&mut self, sctp: &mut RtcSctp) { + for a in &mut self.allocations { + // The stream id must be known before the config is taken. A negotiated + // channel waiting out a previous incarnation's reset has no id yet, and + // consuming its config here would leave nothing to open it with once the + // id is released. + let Some(sctp_stream_id) = a.sctp_stream_id else { + continue; + }; + let Some(config) = a.config.take() else { + continue; + }; + + debug!("Open stream for: {:?}", a.id); + sctp.open_stream(sctp_stream_id, config); + } + } + + pub fn poll_timeout(&self, sctp: &RtcSctp) -> Option { + if sctp.is_inited() && (self.need_allocation() || self.need_open()) { + Some(already_happened()) + } else { + None + } + } + + pub fn ensure_channel_id_for(&mut self, sctp_stream_id: u16) { + let exists = self + .allocations + .iter() + .any(|a| a.sctp_stream_id == Some(sctp_stream_id)); + + if !exists { + let id = self.next_channel_id(); + let alloc = ChannelAllocation { + id, + sctp_stream_id: Some(sctp_stream_id), + negotiated_stream_id: None, + config: None, + }; + self.allocations.push(alloc); + } + } + + // NB: Maybe this should still be &mut self or even `self` to prove singular ownership + pub fn close_channel(&self, id: ChannelId, sctp: &mut RtcSctp) { + if let Some(sctp_stream_id) = self + .allocations + .iter() + .find(|a| a.id == id) + .and_then(|s| s.sctp_stream_id) + { + sctp.close_stream(sctp_stream_id); + } + } + + /// The reset handshake for a held stream ID completed, release it for + /// reallocation immediately. + /// + /// Completions are reported per stream generation, so the same id can be held + /// more than once when resets overlap. Release a single generation, the id only + /// becomes reusable once every one of them has completed. + pub fn stream_reset_complete(&mut self, stream_id: u16) { + if let Some(pos) = self.closed_stream_ids.iter().position(|c| *c == stream_id) { + self.closed_stream_ids.remove(pos); + } + } + + /// The association is gone, so no reset can ever complete on it. + /// The held IDs belong to an association nobody will send on again. + pub fn association_lost(&mut self) { + self.closed_stream_ids.clear(); + } + + /// Remove a closed channel. + pub fn remove_channel(&mut self, id: ChannelId, reset_pending: bool) { + let stream_id = self + .allocations + .iter() + .find(|a| a.id == id) + .and_then(|a| a.sctp_stream_id); + + if let (true, Some(stream_id)) = (reset_pending, stream_id) { + self.closed_stream_ids.push(stream_id); + } + + self.allocations.retain(|a| a.id != id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn channel_id_allocation() { + let mut handler = ChannelHandler::default(); + + // allocate first channel, get unique id + assert_eq!(handler.new_channel(&Default::default()), ChannelId(0)); + + // allocate second channel, get unique id + assert_eq!(handler.new_channel(&Default::default()), ChannelId(1)); + + // free channel 0, allocate two more channels and verify that the + // new channels have unique IDs. + handler.remove_channel(ChannelId(0), true); + assert_eq!(handler.new_channel(&Default::default()), ChannelId(2)); + assert_eq!(handler.new_channel(&Default::default()), ChannelId(3)); + } + + #[test] + fn stream_id_held_until_reset_complete() { + let mut handler = ChannelHandler::default(); + + // Simulate two channels with known stream IDs (as if do_allocations ran + // for a client: even IDs 0, 2). + let id0 = handler.new_channel(&Default::default()); + let _id1 = handler.new_channel(&Default::default()); + // Manually set stream IDs as do_allocations would. + handler.allocations[0].sctp_stream_id = Some(0); + handler.allocations[1].sctp_stream_id = Some(2); + + // Close channel 0 (stream ID 0) with its reset outstanding. It should be held. + handler.remove_channel(id0, true); + assert_eq!(handler.closed_stream_ids, vec![0]); + + // Build the taken list as do_allocations does — stream 0 is held. + let _id2 = handler.new_channel(&Default::default()); + let taken: Vec = handler + .allocations + .iter() + .filter_map(|a| a.sctp_stream_id) + .chain(handler.closed_stream_ids.iter().copied()) + .collect(); + assert!(taken.contains(&0), "stream 0 should be held"); + assert!(taken.contains(&2), "stream 2 should be active"); + + // The completion signal is the only thing that releases it. + handler.stream_reset_complete(0); + assert!(handler.closed_stream_ids.is_empty()); + + let taken_after: Vec = handler + .allocations + .iter() + .filter_map(|a| a.sctp_stream_id) + .chain(handler.closed_stream_ids.iter().copied()) + .collect(); + assert!( + !taken_after.contains(&0), + "stream 0 should be available after reset completion" + ); + } + + #[test] + fn negotiated_stream_id_cannot_overlap_held_generation() { + let sctp = RtcSctp::new(1200); + let mut handler = ChannelHandler::default(); + let config = ChannelConfig { + negotiated: Some(4), + ..Default::default() + }; + + // The old generation was granted the id it asked for. + let old = handler.new_channel(&config); + handler.do_allocations(&sctp); + assert_eq!(handler.stream_id_by_channel_id(old), Some(4)); + + handler.remove_channel(old, true); + assert_eq!(handler.closed_stream_ids, vec![4]); + + // A replacement asking for the same id may be declared, but must stall + // without the id until the old reset completes. + let _replacement = handler.new_channel(&config); + handler.do_allocations(&sctp); + + let overlaps_held_generation = handler.allocations.iter().any(|allocation| { + handler + .closed_stream_ids + .iter() + .any(|closed| allocation.sctp_stream_id == Some(*closed)) + }); + + assert!( + !overlaps_held_generation, + "a negotiated ID must remain unavailable until its old reset completes" + ); + } + + #[test] + fn negotiated_stream_keeps_config_while_waiting_for_reset() { + let mut sctp = RtcSctp::new(1200); + let mut handler = ChannelHandler::default(); + let config = ChannelConfig { + label: "replacement".into(), + negotiated: Some(4), + ..Default::default() + }; + + handler.closed_stream_ids.push(4); + let replacement = handler.new_channel(&config); + handler.confirm(replacement, config); + + handler.do_allocations(&sctp); + handler.open_channels(&mut sctp); + + let allocation = handler + .allocations + .iter() + .find(|allocation| allocation.id == replacement) + .unwrap(); + assert!( + allocation.config.is_some(), + "a negotiated channel must retain its config while its stream id is held" + ); + + handler.stream_reset_complete(4); + handler.do_allocations(&sctp); + handler.open_channels(&mut sctp); + + assert_eq!( + sctp.config(4).map(|config| config.label.as_str()), + Some("replacement"), + "the negotiated channel must open once the old reset completes" + ); + } + + #[test] + fn negotiated_stream_waiting_for_reset_does_not_spin_timeout() { + let now = Instant::now(); + let mut sctp = RtcSctp::new(1200); + sctp.init(true, now, None, None).unwrap(); + + let mut handler = ChannelHandler::default(); + let config = ChannelConfig { + negotiated: Some(4), + ..Default::default() + }; + + handler.closed_stream_ids.push(4); + let replacement = handler.new_channel(&config); + handler.confirm(replacement, config); + handler.handle_timeout(now, &mut sctp); + + assert_eq!( + handler.poll_timeout(&sctp), + None, + "a channel that cannot open until network input arrives must not request an immediate timeout" + ); + } + + #[test] + fn one_reset_completion_releases_only_one_stream_generation() { + let mut handler = ChannelHandler::default(); + + // sctp-proto 0.10.3 reports reset completion per stream generation. + // Two outstanding generations of the same stream id therefore need two + // completion events before the id is reusable. + handler.closed_stream_ids.extend([4, 4]); + + handler.stream_reset_complete(4); + + assert_eq!( + handler.closed_stream_ids, + vec![4], + "the first completion must not release a newer pending generation" + ); + } + + #[test] + fn stream_id_remains_held_without_reset_completion() { + let sctp = RtcSctp::new(1200); + let mut handler = ChannelHandler::default(); + + let id = handler.new_channel(&Default::default()); + handler.allocations[0].sctp_stream_id = Some(0); + handler.remove_channel(id, true); + + // There is no timer that can release the id. + for _ in 0..100 { + handler.do_allocations(&sctp); + } + + assert_eq!( + handler.closed_stream_ids, + vec![0], + "elapsed time alone cannot make an SCTP stream ID safe to reuse" + ); + } + + #[test] + fn stream_id_not_held_when_no_reset_is_pending() { + let mut handler = ChannelHandler::default(); + + let id0 = handler.new_channel(&Default::default()); + handler.allocations[0].sctp_stream_id = Some(0); + + // The stream never made it into the association, so no reset was started and + // no completion will ever arrive. Holding the id would leak it forever. + handler.remove_channel(id0, false); + assert!(handler.closed_stream_ids.is_empty()); + } + + #[test] + fn association_lost_releases_held_stream_ids() { + let mut handler = ChannelHandler::default(); + + let id0 = handler.new_channel(&Default::default()); + handler.allocations[0].sctp_stream_id = Some(0); + handler.remove_channel(id0, true); + assert_eq!(handler.closed_stream_ids, vec![0]); + + // No reset can complete on a dead association, nothing holds the id back. + handler.association_lost(); + assert!(handler.closed_stream_ids.is_empty()); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/config.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/config.rs new file mode 100644 index 000000000..dc2cc9034 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/config.rs @@ -0,0 +1,847 @@ +use std::ops::RangeInclusive; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::Rtc; +use crate::config::DtlsCert; +use crate::crypto::CryptoProvider; +use crate::crypto::dtls::DtlsVersion; +use crate::format::CodecConfig; +use crate::format::Vp9PacketizerMode; +use crate::ice::IceCreds; +use crate::io::DATAGRAM_MTU_TARGET; +use crate::io::DATAGRAM_MTU_TARGET_MAX; +use crate::io::DATAGRAM_MTU_TARGET_MIN; +use crate::io::DATAGRAM_MTU_WARN; +use crate::rtp_::{Bitrate, Extension, ExtensionMap}; + +/// Customized config for creating an [`Rtc`] instance. +/// +/// ``` +/// # #[cfg(feature = "openssl")] { +/// use std::time::Instant; +/// use str0m::RtcConfig; +/// +/// let rtc = RtcConfig::new() +/// .set_ice_lite(true) +/// .build(Instant::now()); +/// # } +/// ``` +/// +/// Configs implement [`Clone`] to help create multiple `Rtc` instances. +#[derive(Debug, Clone)] +pub struct RtcConfig { + pub(crate) local_ice_credentials: Option, + pub(crate) crypto_provider: Option>, + pub(crate) dtls_cert: Option, + pub(crate) fingerprint_verification: bool, + pub(crate) ice_lite: bool, + pub(crate) initial_stun_rto: Option, + pub(crate) max_stun_rto: Option, + pub(crate) max_stun_retransmits: Option, + pub(crate) codec_config: CodecConfig, + pub(crate) exts: ExtensionMap, + pub(crate) stats_interval: Option, + pub(crate) intervals: RtcpReportIntervals, + pub(crate) bwe_config: Option, + pub(crate) reordering_size_audio: usize, + pub(crate) reordering_size_video: usize, + pub(crate) send_buffer_audio: usize, + pub(crate) send_buffer_video: usize, + pub(crate) rtp_mode: bool, + pub(crate) enable_raw_packets: bool, + pub(crate) dtls_version: DtlsVersion, + pub(crate) vp9_packetizer_mode: Vp9PacketizerMode, + pub(crate) snap_enabled: bool, + pub(crate) mtu: RangeInclusive, +} + +#[derive(Debug, Clone)] +pub(crate) struct BweConfig { + pub(crate) initial_bitrate: Bitrate, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct RtcpReportIntervals { + pub(crate) audio: Duration, + pub(crate) video: Duration, +} + +impl RtcpReportIntervals { + pub(crate) fn for_audio(self, audio: bool) -> Duration { + if audio { self.audio } else { self.video } + } +} + +impl RtcConfig { + /// Creates a new default config. + pub fn new() -> Self { + RtcConfig::default() + } + + /// Get the local ICE credentials, if set. + /// + /// If not specified, local credentials will be randomly generated when + /// building the [`Rtc`] instance. + pub fn local_ice_credentials(&self) -> &Option { + &self.local_ice_credentials + } + + /// Explicitly sets local ICE credentials. + pub fn set_local_ice_credentials(mut self, local_ice_credentials: IceCreds) -> Self { + self.local_ice_credentials = Some(local_ice_credentials); + self + } + + /// Set the crypto provider. + /// + /// This overrides what is set in [`crate::crypto::CryptoProvider::install_process_default()`]. + pub fn set_crypto_provider(mut self, p: Arc) -> Self { + self.crypto_provider = Some(p); + self + } + + /// The configured crypto provider, if explicitly set. + /// + /// Returns `None` if not explicitly set via [`Self::set_crypto_provider()`]. + /// When `None`, the process default will be checked when building the [`Rtc`] instance. + pub fn crypto_provider(&self) -> Option<&Arc> { + self.crypto_provider.as_ref() + } + + /// Returns the configured DTLS certificate, if set. + /// + /// If not set, a certificate will be generated automatically. + pub fn dtls_cert(&self) -> Option<&DtlsCert> { + self.dtls_cert.as_ref() + } + + /// Set a pregenerated DTLS certificate. + /// + /// If not set, a certificate will be generated automatically using + /// the configured crypto provider. + /// + /// ``` + /// # use str0m::RtcConfig; + /// # use str0m::crypto; + /// + /// let provider = crypto::from_feature_flags(); + /// let cert = provider.dtls_provider.generate_certificate().unwrap(); + /// let rtc_config = RtcConfig::default() + /// .set_dtls_cert(cert); + /// ``` + pub fn set_dtls_cert(mut self, cert: DtlsCert) -> Self { + self.dtls_cert = Some(cert); + self + } + + /// Toggle ice lite. Ice lite is a mode for WebRTC servers with public IP address. + /// An [`Rtc`] instance in ice lite mode will not make STUN binding requests, but only + /// answer to requests from the remote peer. + /// + /// See [ICE RFC][1] + /// + /// [1]: https://www.rfc-editor.org/rfc/rfc8445#page-13 + pub fn set_ice_lite(mut self, enabled: bool) -> Self { + self.ice_lite = enabled; + self + } + + /// Sets the initial STUN retransmission timeout (RTO). + /// + /// This is the initial wait time before a STUN request is retransmitted. + /// The timeout will double with each retry, starting from this value. + /// + /// Defaults to 250ms. + pub fn set_initial_stun_rto(&mut self, rto: Duration) { + self.initial_stun_rto = Some(rto); + } + + /// Sets the maximum STUN retransmission timeout for the ICE agent. + /// + /// This is the upper bound for how long to wait between retransmissions. + /// It also controls how often successful bindings are checked. + /// + /// Defaults to 3000ms. + pub fn set_max_stun_rto(&mut self, rto: Duration) { + self.max_stun_rto = Some(rto); + } + + /// Sets the maximum number of retransmits for STUN messages. + /// + /// Defaults to 9. + pub fn set_max_stun_retransmits(&mut self, num: usize) { + self.max_stun_retransmits = Some(num); + } + + /// Get fingerprint verification mode. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to true. + /// assert!(config.fingerprint_verification()); + /// ``` + pub fn fingerprint_verification(&self) -> bool { + self.fingerprint_verification + } + + /// Toggle certificate fingerprint verification. + /// + /// By default the certificate fingerprint is verified. + pub fn set_fingerprint_verification(mut self, enabled: bool) -> Self { + self.fingerprint_verification = enabled; + self + } + + /// Tells whether ice lite is enabled. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to false. + /// assert_eq!(config.ice_lite(), false); + /// # } + /// ``` + pub fn ice_lite(&self) -> bool { + self.ice_lite + } + + /// Lower level access to precise configuration of codecs (payload types). + pub fn codec_config(&mut self) -> &mut CodecConfig { + &mut self.codec_config + } + + /// Clear all configured codecs. + /// + /// ``` + /// # #[cfg(feature = "openssl")] { + /// # use std::time::Instant; + /// # use str0m::RtcConfig; + /// // For the session to use only OPUS and VP8. + /// let mut rtc = RtcConfig::default() + /// .clear_codecs() + /// .enable_opus(true) + /// .enable_vp8(true) + /// .build(Instant::now()); + /// # } + /// ``` + pub fn clear_codecs(mut self) -> Self { + self.codec_config.clear(); + self + } + + /// Enable opus audio codec. + /// + /// Enabled by default. + pub fn enable_opus(mut self, enabled: bool) -> Self { + self.codec_config.enable_opus(enabled); + self + } + + /// Enable PCM μ-law audio codec. + /// + /// This is 14-bit audio compressed to 8-bit as specified by G.711 + pub fn enable_pcmu(mut self, enabled: bool) -> Self { + self.codec_config.enable_pcmu(enabled); + self + } + + /// Enable PCM a-law audio codec. + /// + /// This is 13-bit audio compressed to 8-bit as specified by G.711 + pub fn enable_pcma(mut self, enabled: bool) -> Self { + self.codec_config.enable_pcma(enabled); + self + } + + /// Enable G722 audio codec. + /// + /// This is 16 kHz audio compressed to 64 kbit/s as specified by G.722. Note that + /// per RFC 3551 §4.5.2 the RTP clock rate on the wire is 8000 Hz even though the + /// codec samples at 16 kHz. str0m treats G722 as a 16 kHz codec in the API and + /// handles the 8 kHz RTP timestamp mapping internally. See + /// + pub fn enable_g722(mut self, enabled: bool) -> Self { + self.codec_config.enable_g722(enabled); + self + } + + /// Enable the RFC 3389 Comfort Noise payload at 8000 Hz. + /// + /// Disabled by default. Higher clock rates require a dynamic payload type + /// and can be added through [`RtcConfig::codec_config`]. + pub fn enable_comfort_noise(mut self, enabled: bool) -> Self { + self.codec_config.enable_comfort_noise(enabled); + self + } + + /// Enable VP8 video codec. + /// + /// Enabled by default. + pub fn enable_vp8(mut self, enabled: bool) -> Self { + self.codec_config.enable_vp8(enabled); + self + } + + /// Enable H264 video codec. + /// + /// Enabled by default. + pub fn enable_h264(mut self, enabled: bool) -> Self { + self.codec_config.enable_h264(enabled); + self + } + + /// Enable H265 video codec. + /// + /// Enabled by default. + pub fn enable_h265(mut self, enabled: bool) -> Self { + self.codec_config.enable_h265(enabled); + self + } + + /// Enable H266 (VVC) video codec. + /// + /// Disabled by default: no major browser (Chrome, Safari, Firefox) + /// currently supports H.266/VVC, so it is opt-in. + pub fn enable_h266(mut self, enabled: bool) -> Self { + self.codec_config.enable_h266(enabled); + self + } + + /// Enable VP9 video codec. + /// + /// Enabled by default. + pub fn enable_vp9(mut self, enabled: bool) -> Self { + self.codec_config.enable_vp9(enabled); + self + } + + /// Enable AV1 video codec. + /// + /// Enabled by default. + pub fn enable_av1(mut self, enabled: bool) -> Self { + self.codec_config.enable_av1(enabled); + self + } + + /// Configure the RTP extension mappings. + /// + /// The default extension map is + /// + /// ``` + /// # use str0m::rtp::{Extension, ExtensionMap}; + /// let exts = ExtensionMap::standard(); + /// + /// assert_eq!(exts.id_of(Extension::AudioLevel), Some(1)); + /// assert_eq!(exts.id_of(Extension::AbsoluteSendTime), Some(2)); + /// assert_eq!(exts.id_of(Extension::TransportSequenceNumber), Some(3)); + /// assert_eq!(exts.id_of(Extension::RtpMid), Some(4)); + /// assert_eq!(exts.id_of(Extension::RtpStreamId), Some(10)); + /// assert_eq!(exts.id_of(Extension::RepairedRtpStreamId), Some(11)); + /// assert_eq!(exts.id_of(Extension::VideoOrientation), Some(13)); + /// ``` + pub fn extension_map(&mut self) -> &mut ExtensionMap { + &mut self.exts + } + + /// Set the extension map replacing the existing. + pub fn set_extension_map(mut self, exts: ExtensionMap) -> Self { + self.exts = exts; + self + } + + /// Clear out the standard extension mappings. + pub fn clear_extension_map(mut self) -> Self { + self.exts.clear(); + + self + } + + /// Set an extension mapping on session level. + /// + /// The media level will be capped by the extension enabled on session level. + /// + /// The id must be 1-14 inclusive (1-indexed). + pub fn set_extension(mut self, id: u8, ext: Extension) -> Self { + self.exts.set(id, ext); + self + } + + /// Set the interval between statistics events. + /// + /// None turns off the stats events. + /// + /// This includes [`MediaEgressStats`][crate::stats::MediaEgressStats], + /// [`MediaIngressStats`][crate::stats::MediaIngressStats] + pub fn set_stats_interval(mut self, interval: Option) -> Self { + self.stats_interval = interval; + self + } + + /// The configured statistics interval. + /// + /// None means statistics are disabled. + /// + /// ``` + /// # use str0m::Rtc; + /// # use std::time::Duration; + /// let config = Rtc::builder(); + /// + /// // Defaults to None. + /// assert_eq!(config.stats_interval(), None); + /// ``` + pub fn stats_interval(&self) -> Option { + self.stats_interval + } + + /// Sets the interval between RTCP sender/receiver reports for audio streams. + /// + /// Defaults to 5 seconds. + /// + /// # Panics + /// + /// Panics if `interval` is zero. + pub fn set_rtcp_report_interval_audio(mut self, interval: Duration) -> Self { + assert!(!interval.is_zero()); + self.intervals.audio = interval; + self + } + + /// Returns the interval between RTCP sender/receiver reports for audio streams. + pub fn rtcp_report_interval_audio(&self) -> Duration { + self.intervals.audio + } + + /// Sets the interval between RTCP sender/receiver reports for video streams. + /// + /// Defaults to 1 second. + /// + /// # Panics + /// + /// Panics if `interval` is zero. + pub fn set_rtcp_report_interval_video(mut self, interval: Duration) -> Self { + assert!(!interval.is_zero()); + self.intervals.video = interval; + self + } + + /// Returns the interval between RTCP sender/receiver reports for video streams. + pub fn rtcp_report_interval_video(&self) -> Duration { + self.intervals.video + } + + /// Enables estimation of available bandwidth (BWE). + /// + /// None disables the BWE. This is an estimation of the send bandwidth, not receive. + /// + /// This includes setting the initial estimate to start with. + pub fn enable_bwe(mut self, initial_estimate: Option) -> Self { + match initial_estimate { + Some(b) => { + let conf = self.bwe_config.get_or_insert(BweConfig::new(b)); + conf.initial_bitrate = b; + } + None => { + self.bwe_config = None; + } + } + + self + } + + /// The initial bitrate as set by [`Self::enable_bwe()`]. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to None - BWE off. + /// assert_eq!(config.bwe_initial_bitrate(), None); + /// ``` + pub fn bwe_initial_bitrate(&self) -> Option { + self.bwe_config.as_ref().map(|c| c.initial_bitrate) + } + + /// Sets the number of packets held back for reordering audio packets. + /// + /// Str0m tries to deliver the frames in order. This number determines how many + /// packets to "wait" before releasing media + /// [`contiguous: false`][crate::media::MediaData::contiguous]. + /// + /// This setting is ignored in [RTP mode][`RtcConfig::set_rtp_mode()`] where RTP + /// packets can arrive out of order. + pub fn set_reordering_size_audio(mut self, size: usize) -> Self { + self.reordering_size_audio = size; + + self + } + + /// Returns the setting for audio reordering size. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to 15. + /// assert_eq!(config.reordering_size_audio(), 15); + /// ``` + /// + /// This setting is ignored in [RTP mode][`RtcConfig::set_rtp_mode()`] where RTP + /// packets can arrive out of order. + pub fn reordering_size_audio(&self) -> usize { + self.reordering_size_audio + } + + /// Sets the number of packets held back for reordering video packets. + /// + /// Str0m tries to deliver the frames in order. This number determines how many + /// packets to "wait" before releasing media with gaps. + /// + /// This must be at least as big as the number of packets the biggest keyframe + /// can be split over. + /// + /// WARNING: video is very different to audio. Setting this value too low will result in + /// missing video data. The 0 (as described for audio) is not relevant for video. + /// + /// Default: 30 + /// + /// This setting is ignored in [RTP mode][`RtcConfig::set_rtp_mode()`] where RTP + /// packets can arrive out of order. + pub fn set_reordering_size_video(mut self, size: usize) -> Self { + self.reordering_size_video = size; + + self + } + + /// Returns the setting for video reordering size. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to 30. + /// assert_eq!(config.reordering_size_video(), 30); + /// ``` + /// + /// This setting is ignored in [RTP mode][`RtcConfig::set_rtp_mode()`] where RTP + /// packets can arrive out of order. + pub fn reordering_size_video(&self) -> usize { + self.reordering_size_video + } + + /// Sets the buffer size for outgoing audio packets. + /// + /// This must be larger than 0. The value configures an internal ring buffer used as a temporary + /// holding space between calling [`Writer::write`][crate::media::Writer::write()] and + /// [`Rtc::poll_output`]. + /// + /// For audio one call to `write()` typically results in one RTP packet since the entire + /// payload fits in one. If you can guarantee that every `write()` is a single RTP packet, + /// and is always followed by a `poll_output()`, it might be possible to set this value to 1. + /// But that would give no margins for unexpected patterns. + /// + /// panics if set to 0. + pub fn set_send_buffer_audio(mut self, size: usize) -> Self { + assert!(size > 0); + self.send_buffer_audio = size; + self + } + + /// Returns the setting for audio resend size. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to 50. + /// assert_eq!(config.send_buffer_audio(), 50); + /// ``` + pub fn send_buffer_audio(&self) -> usize { + self.send_buffer_audio + } + + /// Sets the buffer size for outgoing video packets and resends. + /// + /// This must be larger than 0. The value configures an internal ring buffer that is both + /// used as a temporary holding space between calling + /// [`Writer::write`][crate::media::Writer::write()] and [`Rtc::poll_output`] as well as for + /// fulfilling resends. + /// + /// For video, this buffer is used for more than for audio. First, a call to `write()` often + /// results in multiple RTP packets since large frames don't fit in one payload. That means + /// the buffer must be at least as large to hold all those packets. Second, when the remote + /// requests resends (NACK), those are fulfilled from this buffer. Third, for Bandwidth + /// Estimation (BWE), when probing for available bandwidth, packets from this buffer are used + /// to do "spurious resends", i.e. we do resends for packets that were not asked for. + pub fn set_send_buffer_video(mut self, size: usize) -> Self { + self.send_buffer_video = size; + self + } + + /// Returns the setting for video resend size. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to 1000. + /// assert_eq!(config.send_buffer_video(), 1000); + /// ``` + pub fn send_buffer_video(&self) -> usize { + self.send_buffer_video + } + + /// Make the entire Rtc be in RTP mode. + /// + /// This means all media, read from [`RtpPacket`][crate::rtp::RtpPacket] and written to + /// [`StreamTx::write_rtp`][crate::rtp::StreamTx::write_rtp] are RTP packetized. + /// It bypasses all internal packetization/depacketization inside str0m. + /// + /// WARNING: This is a low level API and is not str0m's primary use case. + pub fn set_rtp_mode(mut self, enabled: bool) -> Self { + self.rtp_mode = enabled; + + self + } + + /// Checks if RTP mode is set. + /// + /// ``` + /// # use str0m::Rtc; + /// let config = Rtc::builder(); + /// + /// // Defaults to false. + /// assert_eq!(config.rtp_mode(), false); + /// ``` + pub fn rtp_mode(&self) -> bool { + self.rtp_mode + } + + /// Enable the [`Event::RawPacket`][crate::Event::RawPacket] event. + /// + /// This clones data, and is therefore expensive. + /// Should not be enabled outside of tests and troubleshooting. + pub fn enable_raw_packets(mut self, enabled: bool) -> Self { + self.enable_raw_packets = enabled; + self + } + + /// Enable SNAP (SCTP Negotiation Acceleration Protocol). + /// + /// When enabled, the `a=sctp-init` attribute is included in outgoing SDP + /// offers, allowing the SCTP association to skip the 4-way handshake + /// and go directly to established state. This saves up to two network + /// round-trip times when establishing data channels. + /// + /// If the remote answer does not accept SNAP, str0m logs a warning and + /// falls back to the regular SCTP handshake. + /// + /// When this is `false`, outgoing SDP offers will not include `a=sctp-init`. + /// However, incoming offers that contain `a=sctp-init` will still be + /// reciprocated in answers, since str0m always supports the extension at + /// the protocol level (per §5.4 of draft-hancke-tsvwg-snap). + /// + /// See [draft-hancke-tsvwg-snap](https://datatracker.ietf.org/doc/draft-hancke-tsvwg-snap/). + /// + /// Default: `false` + pub fn set_snap_enabled(mut self, enabled: bool) -> Self { + self.snap_enabled = enabled; + self + } + + /// Set which DTLS version to use. + /// + /// Defaults to [`DtlsVersion::Dtls12`]. + pub fn set_dtls_version(mut self, version: DtlsVersion) -> Self { + self.dtls_version = version; + self + } + + /// Get the configured DTLS version. + pub fn dtls_version(&self) -> DtlsVersion { + self.dtls_version + } + + /// Set the MTU as a `target..=warn` inclusive range. + /// + /// * `*range.start()` is the **target** MTU: the size str0m aims for when + /// fragmenting outgoing packets. Must be within + /// `[DATAGRAM_MTU_TARGET_MIN, DATAGRAM_MTU_TARGET_MAX]`. + /// * `*range.end()` is the **warn** threshold: packets whose length + /// exceeds the target trigger a warning. + /// + /// Set as `target..=target` to warn packets bigger than target. + /// Set as `target..=usize::MAX` to disable warning. + /// + /// Defaults to `DATAGRAM_MTU_TARGET..=DATAGRAM_MTU_WARN`. + pub fn set_mtu(mut self, range: RangeInclusive) -> Self { + let (start, end) = (*range.start(), *range.end()); + assert!( + DATAGRAM_MTU_TARGET_MIN <= start && start <= DATAGRAM_MTU_TARGET_MAX, + "mtu target {} out of bounds [{}, {}]", + start, + DATAGRAM_MTU_TARGET_MIN, + DATAGRAM_MTU_TARGET_MAX, + ); + assert!( + start <= end, + "mtu range start must be <= end, got {}..={}", + start, + end, + ); + self.mtu = range; + self + } + + /// Get the configured target MTU (the value str0m aims for). + pub fn mtu(&self) -> usize { + *self.mtu.start() + } + + /// Get the configured warn threshold above which outgoing packets + /// trigger a warning trace. + pub fn mtu_warn(&self) -> usize { + *self.mtu.end() + } + + /// Set the VP9 packetizer mode. + /// + /// VP9 supports two RTP payload descriptor formats: + /// + /// * [`Vp9PacketizerMode::NonFlexible`] (default) — 5-6 byte header with layer indices + /// and scalability structure. Compatible with all major browsers including Safari. + /// * [`Vp9PacketizerMode::Flexible`] — minimal 3-byte header. Simpler but may cause + /// issues with Safari which drops inter-frames. + pub fn set_vp9_packetizer_mode(mut self, mode: Vp9PacketizerMode) -> Self { + self.vp9_packetizer_mode = mode; + self + } + + /// Returns the configured VP9 packetizer mode. + /// + /// ``` + /// # use str0m::Rtc; + /// # use str0m::format::Vp9PacketizerMode; + /// let config = Rtc::builder(); + /// + /// // Defaults to NonFlexible. + /// assert_eq!(config.vp9_packetizer_mode(), Vp9PacketizerMode::NonFlexible); + /// ``` + pub fn vp9_packetizer_mode(&self) -> Vp9PacketizerMode { + self.vp9_packetizer_mode + } + + /// Create a [`Rtc`] from the configuration. + pub fn build(self, start: Instant) -> Rtc { + Rtc::new_from_config(self, start).expect("Failed to create Rtc from config") + } +} + +impl BweConfig { + fn new(initial_bitrate: Bitrate) -> Self { + Self { initial_bitrate } + } +} + +impl Default for RtcConfig { + fn default() -> Self { + Self { + local_ice_credentials: None, + crypto_provider: None, + dtls_cert: None, + fingerprint_verification: true, + ice_lite: false, + initial_stun_rto: None, + max_stun_rto: None, + max_stun_retransmits: None, + codec_config: CodecConfig::new_with_defaults(), + exts: ExtensionMap::standard(), + stats_interval: None, + intervals: RtcpReportIntervals { + audio: Duration::from_secs(5), + video: Duration::from_secs(1), + }, + bwe_config: None, + reordering_size_audio: 15, + reordering_size_video: 30, + send_buffer_audio: 50, + send_buffer_video: 1000, + rtp_mode: false, + enable_raw_packets: false, + dtls_version: DtlsVersion::Dtls12, + vp9_packetizer_mode: Vp9PacketizerMode::default(), + snap_enabled: false, + mtu: DATAGRAM_MTU_TARGET..=DATAGRAM_MTU_WARN, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enable_comfort_noise_updates_codec_config() { + let mut cfg = RtcConfig::default().enable_comfort_noise(true); + + assert!( + cfg.codec_config() + .params() + .iter() + .any(|p| p.spec().codec == crate::format::Codec::CN) + ); + } + + #[test] + fn default_mtu_is_datagram_mtu_target() { + let cfg = RtcConfig::default(); + assert_eq!(cfg.mtu(), DATAGRAM_MTU_TARGET); + assert_eq!(cfg.mtu_warn(), DATAGRAM_MTU_WARN); + } + + #[test] + fn set_mtu_round_trips() { + let cfg = RtcConfig::default().set_mtu(900..=1400); + assert_eq!(cfg.mtu(), 900); + assert_eq!(cfg.mtu_warn(), 1400); + + let cfg = RtcConfig::default().set_mtu(900..=usize::MAX); + assert_eq!(cfg.mtu(), 900); + assert_eq!(cfg.mtu_warn(), usize::MAX); + } + + #[test] + fn set_mtu_accepts_equal_bounds() { + let cfg = RtcConfig::default().set_mtu(1200..=1200); + assert_eq!(cfg.mtu(), 1200); + assert_eq!(cfg.mtu_warn(), 1200); + + let cfg = RtcConfig::default().set_mtu(DATAGRAM_MTU_TARGET_MIN..=DATAGRAM_MTU_TARGET_MIN); + assert_eq!(cfg.mtu(), DATAGRAM_MTU_TARGET_MIN); + + let cfg = RtcConfig::default().set_mtu(DATAGRAM_MTU_TARGET_MAX..=DATAGRAM_MTU_TARGET_MAX); + assert_eq!(cfg.mtu(), DATAGRAM_MTU_TARGET_MAX); + } + + #[test] + #[should_panic(expected = "mtu target")] + fn set_mtu_panics_on_target_below_min() { + let _ = RtcConfig::default().set_mtu((DATAGRAM_MTU_TARGET_MIN - 1)..=usize::MAX); + } + + #[test] + #[should_panic(expected = "mtu target")] + fn set_mtu_panics_on_target_above_max() { + let _ = RtcConfig::default().set_mtu((DATAGRAM_MTU_TARGET_MAX + 1)..=usize::MAX); + } + + #[test] + #[should_panic(expected = "mtu range start must be <= end")] + fn set_mtu_panics_on_inverted_range() { + let _ = RtcConfig::default().set_mtu(1300..=1299); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/finger.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/finger.rs new file mode 100644 index 000000000..d53995b7d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/finger.rs @@ -0,0 +1,89 @@ +use core::fmt; + +use subtle::ConstantTimeEq; + +/// Certificate fingerprint. +/// +/// DTLS uses self signed certificates, and the fingerprint is communicated via +/// SDP to let the remote peer verify who is connecting. +#[derive(Debug, Clone)] +pub struct Fingerprint { + /// Hash function used to produce the `bytes`. + /// + /// This is normally `sha-256`. + pub hash_func: String, + + /// Digest of the certificate by the algorithm in `hash_func`. + pub bytes: Vec, +} + +impl PartialEq for Fingerprint { + fn eq(&self, other: &Self) -> bool { + // Use constant-time comparison for the bytes to prevent timing attacks + self.hash_func == other.hash_func && self.bytes[..].ct_eq(&other.bytes[..]).into() + } +} + +impl Eq for Fingerprint {} + +impl std::hash::Hash for Fingerprint { + fn hash(&self, state: &mut H) { + self.hash_func.hash(state); + self.bytes.hash(state); + } +} + +// DO NOT CHANGE! +// This format is exactly what's needed in n SDP. +impl fmt::Display for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} ", self.hash_func)?; + for (i, b) in self.bytes.iter().enumerate() { + if i > 0 { + write!(f, ":")?; + } + write!(f, "{:02X}", b)?; + } + Ok(()) + } +} + +impl std::str::FromStr for Fingerprint { + type Err = String; + + fn from_str(hex_string: &str) -> Result { + let (hash_func, hex_with_colons) = hex_string + .split_once(' ') + .ok_or_else(|| "Failed to split once".to_owned())?; + + let mut bytes = Vec::new(); + for hex in hex_with_colons.split(':') { + let byte = u8::from_str_radix(hex, 16) + .map_err(|e| format!("Failed to parse fingerprint: {}", e))?; + bytes.push(byte); + } + + Ok(Self { + hash_func: hash_func.to_owned(), + bytes, + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn fingerprint_format() { + let f = Fingerprint { + hash_func: "foo".to_string(), + bytes: vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], + }; + + assert_eq!( + f.to_string(), + "foo 00:01:02:03:04:05:06:07:08:09:0A:0B:0C:0D:0E:0F:10:11" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/mod.rs new file mode 100644 index 000000000..5e735cc42 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/mod.rs @@ -0,0 +1,66 @@ +use str0m_proto::crypto as provider; + +pub use provider::{AeadAes128Gcm, AeadAes128GcmCipher, AeadAes256Gcm, AeadAes256GcmCipher}; +pub use provider::{Aes128CmSha1_80, Aes128CmSha1_80Cipher, CryptoProvider, CryptoSafe}; +pub use provider::{Sha1HmacProvider, Sha256Provider}; +pub use provider::{SrtpProvider, SupportedAeadAes128Gcm}; +pub use provider::{SupportedAeadAes256Gcm, SupportedAes128CmSha1_80}; + +/// DTLS related types and traits. +pub mod dtls { + pub use super::provider::dtls::{DtlsInstance, DtlsProvider}; + + pub use dimpl::DtlsCertificate as DtlsCert; + pub use dimpl::KeyingMaterial; + pub use dimpl::ProtocolVersion; + pub use dimpl::SrtpProfile; + // Note: dimpl::Error is renamed to DtlsImplError to avoid conflict with str0m's DtlsError + pub use super::provider::DtlsVersion; + pub use dimpl::{Error as DtlsImplError, Output as DtlsOutput}; +} + +#[cfg(any(test, feature = "_internal_test_exports"))] +#[allow(unused)] +pub(crate) fn test_default_provider() -> &'static CryptoProvider { + use std::sync::OnceLock; + static TEST_PROVIDER: OnceLock = OnceLock::new(); + TEST_PROVIDER.get_or_init(from_feature_flags) +} + +/// Create a crypto provider based on enabled feature flags. +/// +/// Priority order: aws-lc-rs, rust-crypto, openssl, wincrypto (Windows only) +/// +/// Note: For Apple platforms, use the separate `str0m-apple-crypto` crate +/// and call `str0m_apple_crypto::default_provider()` directly. +#[allow(unreachable_code, clippy::needless_return)] +pub fn from_feature_flags() -> CryptoProvider { + #[cfg(feature = "aws-lc-rs")] + return str0m_aws_lc_rs::default_provider(); + + #[cfg(feature = "rust-crypto")] + return str0m_rust_crypto::default_provider(); + + #[cfg(any(feature = "openssl", feature = "openssl-dimpl"))] + return str0m_openssl::default_provider(); + + #[cfg(all(feature = "apple-crypto", target_vendor = "apple"))] + return str0m_apple_crypto::default_provider(); + + #[cfg(all( + any(feature = "wincrypto", feature = "wincrypto-dimpl"), + target_os = "windows" + ))] + return str0m_wincrypto::default_provider(); + + panic!( + "No crypto provider available. Enable one of: aws-lc-rs, + rust-crypto, openssl, openssl-dimpl, wincrypto (Windows only), + wincrypto-dimpl (Windows only) or use str0m-apple-crypto crate" + ); +} + +mod finger; +pub use finger::Fingerprint; + +pub use str0m_proto::crypto::{CryptoError, DtlsError}; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/srtp.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/srtp.rs new file mode 100644 index 000000000..4e887e9a5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/crypto/srtp.rs @@ -0,0 +1,529 @@ +use std::fmt; + +use self::aes_128_cm_sha1_80::AesKey; + +use super::CryptoProvider; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SrtpProfile { + #[cfg(feature = "_internal_test_exports")] + PassThrough, + Aes128CmSha1_80, + AeadAes128Gcm, + AeadAes256Gcm, +} + +#[allow(dead_code)] +impl SrtpProfile { + // All the profiles we support, ordered from most preferred to least. + pub(crate) const ALL: &'static [SrtpProfile] = &[ + SrtpProfile::AeadAes256Gcm, + SrtpProfile::AeadAes128Gcm, + SrtpProfile::Aes128CmSha1_80, + ]; + + /// The length of keying material to extract from the DTLS session in bytes. + #[rustfmt::skip] + pub(crate) fn keying_material_len(&self) -> usize { + match self { + #[cfg(feature = "_internal_test_exports")] + SrtpProfile::PassThrough => 0, + // MASTER_KEY_LEN * 2 + MASTER_SALT * 2 + // TODO: This is a duplication of info that is held in srtp.rs, because we + // don't want a dependency in that direction. + SrtpProfile::Aes128CmSha1_80 => 16 * 2 + 14 * 2, + SrtpProfile::AeadAes128Gcm => 16 * 2 + 12 * 2, + SrtpProfile::AeadAes256Gcm => 32 * 2 + 12 * 2, + } + } +} + +pub enum SrtpCrypto { + #[cfg(any(feature = "openssl", feature = "openssl-dimpl"))] + OpenSsl(super::ossl::OsslSrtpCryptoImpl), + #[cfg(not(any(feature = "openssl", feature = "openssl-dimpl")))] + OpenSsl(DummySrtpCryptoImpl), + #[cfg(all(any(feature = "wincrypto", feature = "wincrypto-dimpl"), target_os = "windows"))] + WinCrypto(super::wincrypto::WinCryptoSrtpCryptoImpl), + #[cfg(not(all(any(feature = "wincrypto", feature = "wincrypto-dimpl"), target_os = "windows")))] + WinCrypto(DummySrtpCryptoImpl), + #[cfg(feature = "dimpl")] + AwsLc(super::aws_lc::AwsLcImpl), + #[cfg(not(feature = "dimpl"))] + AwsLc(DummySrtpCryptoImpl), +} + +#[allow(clippy::unit_arg)] +impl SrtpCrypto { + #[cfg(any(feature = "openssl", feature = "openssl-dimpl"))] + pub fn new_openssl() -> SrtpCrypto { + Self::OpenSsl(super::ossl::OsslSrtpCryptoImpl) + } + + #[cfg(not(any(feature = "openssl", feature = "openssl-dimpl")))] + pub fn new_openssl() -> SrtpCrypto { + Self::OpenSsl(DummySrtpCryptoImpl(CryptoProvider::OpenSsl)) + } + + #[cfg(all(any(feature = "wincrypto", feature = "wincrypto-dimpl"), target_os = "windows"))] + pub fn new_wincrypto() -> SrtpCrypto { + Self::WinCrypto(super::wincrypto::WinCryptoSrtpCryptoImpl) + } + + #[cfg(not(all(any(feature = "wincrypto", feature = "wincrypto-dimpl"), target_os = "windows")))] + pub fn new_wincrypto() -> SrtpCrypto { + Self::WinCrypto(DummySrtpCryptoImpl(CryptoProvider::WinCrypto)) + } + + #[cfg(feature = "dimpl")] + pub fn new_rust_crypto() -> SrtpCrypto { + Self::AwsLc(super::aws_lc::AwsLcImpl) + } + + #[cfg(not(feature = "dimpl"))] + pub fn new_rust_crypto() -> SrtpCrypto { + Self::AwsLc(DummySrtpCryptoImpl(CryptoProvider::Dimpl)) + } + + // TODO: Can we avoice dynamic dispatch in this signature? The parameters are: + // 1. As few "touch points" beteen rtp/srtp.rs and here as possible. + // 2. Clear contract towards the actual impl. + // 3. Choice of impl passed all the way from RtcConfig. + pub fn new_aes_128_cm_sha1_80( + &self, + key: AesKey, + encrypt: bool, + ) -> Box { + match self { + SrtpCrypto::OpenSsl(v) => Box::new(v.new_aes_128_cm_sha1_80(key, encrypt)), + SrtpCrypto::WinCrypto(v) => Box::new(v.new_aes_128_cm_sha1_80(key, encrypt)), + SrtpCrypto::AwsLc(v) => Box::new(v.new_aes_128_cm_sha1_80(key, encrypt)), + } + } + + pub fn new_aead_aes_128_gcm( + &self, + key: aead_aes_128_gcm::AeadKey, + encrypt: bool, + ) -> Box { + match self { + SrtpCrypto::OpenSsl(v) => Box::new(v.new_aead_aes_128_gcm(key, encrypt)), + SrtpCrypto::WinCrypto(v) => Box::new(v.new_aead_aes_128_gcm(key, encrypt)), + SrtpCrypto::AwsLc(v) => Box::new(v.new_aead_aes_128_gcm(key, encrypt)), + } + } + + pub fn new_aead_aes_256_gcm( + &self, + key: aead_aes_256_gcm::AeadKey, + encrypt: bool, + ) -> Box { + match self { + SrtpCrypto::OpenSsl(v) => Box::new(v.new_aead_aes_256_gcm(key, encrypt)), + SrtpCrypto::WinCrypto(v) => Box::new(v.new_aead_aes_256_gcm(key, encrypt)), + SrtpCrypto::AwsLc(v) => Box::new(v.new_aead_aes_256_gcm(key, encrypt)), + } + } + + pub fn srtp_aes_128_ecb_round(&self, key: &[u8], input: &[u8], output: &mut [u8]) { + match self { + SrtpCrypto::OpenSsl(v) => v.srtp_aes_128_ecb_round(key, input, output), + SrtpCrypto::WinCrypto(v) => v.srtp_aes_128_ecb_round(key, input, output), + SrtpCrypto::AwsLc(v) => v.srtp_aes_128_ecb_round(key, input, output), + } + } + + pub fn srtp_aes_256_ecb_round(&self, key: &[u8], input: &[u8], output: &mut [u8]) { + match self { + SrtpCrypto::OpenSsl(v) => v.srtp_aes_256_ecb_round(key, input, output), + SrtpCrypto::WinCrypto(v) => v.srtp_aes_256_ecb_round(key, input, output), + SrtpCrypto::AwsLc(v) => v.srtp_aes_256_ecb_round(key, input, output), + } + } +} + +pub trait SrtpCryptoImpl { + type Aes128CmSha1_80: aes_128_cm_sha1_80::CipherCtx; + type AeadAes128Gcm: aead_aes_128_gcm::CipherCtx; + type AeadAes256Gcm: aead_aes_256_gcm::CipherCtx; + + fn new_aes_128_cm_sha1_80(&self, key: AesKey, encrypt: bool) -> Self::Aes128CmSha1_80 { + ::new(key, encrypt) + } + + fn new_aead_aes_128_gcm( + &self, + key: aead_aes_128_gcm::AeadKey, + encrypt: bool, + ) -> Self::AeadAes128Gcm { + ::new(key, encrypt) + } + + fn new_aead_aes_256_gcm( + &self, + key: aead_aes_256_gcm::AeadKey, + encrypt: bool, + ) -> Self::AeadAes256Gcm { + ::new(key, encrypt) + } + + fn srtp_aes_128_ecb_round(&self, key: &[u8], input: &[u8], output: &mut [u8]); + + fn srtp_aes_256_ecb_round(&self, key: &[u8], input: &[u8], output: &mut [u8]); +} + +pub mod aes_128_cm_sha1_80 { + use std::panic::UnwindSafe; + + use subtle::ConstantTimeEq; + + use crate::crypto::CryptoError; + + pub const KEY_LEN: usize = 16; + pub const SALT_LEN: usize = 14; + pub const HMAC_KEY_LEN: usize = 20; + pub const HMAC_TAG_LEN: usize = 10; + pub type AesKey = [u8; 16]; + pub type RtpSalt = [u8; 14]; + pub type RtpIv = [u8; 16]; + + pub trait CipherCtx: UnwindSafe + Send + Sync { + fn new(key: AesKey, encrypt: bool) -> Self + where + Self: Sized; + + fn encrypt( + &mut self, + iv: &RtpIv, + input: &[u8], + output: &mut [u8], + ) -> Result<(), CryptoError>; + + fn decrypt( + &mut self, + iv: &RtpIv, + input: &[u8], + output: &mut [u8], + ) -> Result<(), CryptoError>; + } + + pub fn rtp_hmac(key: &[u8], buf: &mut [u8], srtp_index: u64, hmac_start: usize) { + let roc = (srtp_index >> 16) as u32; + let tag = crate::crypto::sha1_hmac(key, &[&buf[..hmac_start], &roc.to_be_bytes()]); + buf[hmac_start..(hmac_start + HMAC_TAG_LEN)].copy_from_slice(&tag[0..HMAC_TAG_LEN]); + } + + pub fn rtp_verify(key: &[u8], buf: &[u8], srtp_index: u64, cmp: &[u8]) -> bool { + let roc = (srtp_index >> 16) as u32; + let tag = crate::crypto::sha1_hmac(key, &[buf, &roc.to_be_bytes()]); + + tag[0..HMAC_TAG_LEN].ct_eq(cmp).into() + } + + pub fn rtp_iv(salt: RtpSalt, ssrc: u32, srtp_index: u64) -> RtpIv { + let mut iv = [0; 16]; + let ssrc_be = ssrc.to_be_bytes(); + let srtp_be = srtp_index.to_be_bytes(); + iv[4..8].copy_from_slice(&ssrc_be); + for i in 0..8 { + iv[i + 6] ^= srtp_be[i]; + } + for i in 0..14 { + iv[i] ^= salt[i]; + } + iv + } + + pub fn rtcp_hmac(key: &[u8], buf: &mut [u8], hmac_index: usize) { + let tag = crate::crypto::sha1_hmac(key, &[&buf[0..hmac_index]]); + + buf[hmac_index..(hmac_index + HMAC_TAG_LEN)].copy_from_slice(&tag[0..HMAC_TAG_LEN]); + } + + pub fn rtcp_verify(key: &[u8], buf: &[u8], cmp: &[u8]) -> bool { + let tag = crate::crypto::sha1_hmac(key, &[buf]); + + tag[0..HMAC_TAG_LEN].ct_eq(cmp).into() + } +} + +pub mod aead_aes_128_gcm { + use std::panic::UnwindSafe; + + use crate::crypto::CryptoError; + + pub const KEY_LEN: usize = 16; + pub const SALT_LEN: usize = 12; + pub const RTCP_AAD_LEN: usize = 12; + pub const TAG_LEN: usize = 16; + pub const IV_LEN: usize = 12; + pub type AeadKey = [u8; KEY_LEN]; + pub type RtpSalt = [u8; SALT_LEN]; + pub type RtpIv = [u8; SALT_LEN]; + + pub trait CipherCtx: UnwindSafe + Send + Sync { + fn new(key: AeadKey, encrypt: bool) -> Self + where + Self: Sized; + + fn encrypt( + &mut self, + iv: &[u8; IV_LEN], + aad: &[u8], + input: &[u8], + output: &mut [u8], + ) -> Result<(), CryptoError>; + + fn decrypt( + &mut self, + iv: &[u8; IV_LEN], + aads: &[&[u8]], + input: &[u8], + output: &mut [u8], + ) -> Result; + } + + pub fn rtp_iv(salt: RtpSalt, ssrc: u32, roc: u32, seq: u16) -> RtpIv { + // See: https://www.rfc-editor.org/rfc/rfc7714#section-8.1 + + // TODO: See if this is faster if rewritten for u128 + let mut iv = [0; SALT_LEN]; + + let ssrc_be = ssrc.to_be_bytes(); + let roc_be = roc.to_be_bytes(); + let seq_be = seq.to_be_bytes(); + + iv[2..6].copy_from_slice(&ssrc_be); + iv[6..10].copy_from_slice(&roc_be); + iv[10..12].copy_from_slice(&seq_be); + + // XOR with salt + for i in 0..SALT_LEN { + iv[i] ^= salt[i]; + } + + iv + } + + pub fn rtcp_iv(salt: RtpSalt, ssrc: u32, srtp_index: u32) -> RtpIv { + // See: https://www.rfc-editor.org/rfc/rfc7714#section-9.1 + // TODO: See if this is faster if rewritten for u128 + let mut iv = [0; SALT_LEN]; + + let ssrc_be = ssrc.to_be_bytes(); + let srtp_be = srtp_index.to_be_bytes(); + + iv[2..6].copy_from_slice(&ssrc_be); + iv[8..12].copy_from_slice(&srtp_be); + + // XOR with salt + for i in 0..SALT_LEN { + iv[i] ^= salt[i]; + } + + iv + } +} + +pub mod aead_aes_256_gcm { + use std::panic::UnwindSafe; + + use crate::crypto::CryptoError; + + pub const KEY_LEN: usize = 32; + pub const SALT_LEN: usize = 12; + pub const RTCP_AAD_LEN: usize = 12; + pub const TAG_LEN: usize = 16; + pub const IV_LEN: usize = 12; + pub type AeadKey = [u8; KEY_LEN]; + pub type RtpSalt = [u8; SALT_LEN]; + pub type RtpIv = [u8; SALT_LEN]; + + pub trait CipherCtx: UnwindSafe + Send + Sync { + fn new(key: AeadKey, encrypt: bool) -> Self + where + Self: Sized; + + fn encrypt( + &mut self, + iv: &[u8; IV_LEN], + aad: &[u8], + input: &[u8], + output: &mut [u8], + ) -> Result<(), CryptoError>; + + fn decrypt( + &mut self, + iv: &[u8; IV_LEN], + aads: &[&[u8]], + input: &[u8], + output: &mut [u8], + ) -> Result; + } + + pub fn rtp_iv(salt: RtpSalt, ssrc: u32, roc: u32, seq: u16) -> RtpIv { + // See: https://www.rfc-editor.org/rfc/rfc7714#section-8.1 + + // TODO: See if this is faster if rewritten for u128 + let mut iv = [0; SALT_LEN]; + + let ssrc_be = ssrc.to_be_bytes(); + let roc_be = roc.to_be_bytes(); + let seq_be = seq.to_be_bytes(); + + iv[2..6].copy_from_slice(&ssrc_be); + iv[6..10].copy_from_slice(&roc_be); + iv[10..12].copy_from_slice(&seq_be); + + // XOR with salt + for i in 0..SALT_LEN { + iv[i] ^= salt[i]; + } + + iv + } + + pub fn rtcp_iv(salt: RtpSalt, ssrc: u32, srtp_index: u32) -> RtpIv { + // See: https://www.rfc-editor.org/rfc/rfc7714#section-9.1 + // TODO: See if this is faster if rewritten for u128 + let mut iv = [0; SALT_LEN]; + + let ssrc_be = ssrc.to_be_bytes(); + let srtp_be = srtp_index.to_be_bytes(); + + iv[2..6].copy_from_slice(&ssrc_be); + iv[8..12].copy_from_slice(&srtp_be); + + // XOR with salt + for i in 0..SALT_LEN { + iv[i] ^= salt[i]; + } + + iv + } +} + +impl fmt::Display for SrtpProfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + #[cfg(feature = "_internal_test_exports")] + SrtpProfile::PassThrough => write!(f, "PassThrough"), + SrtpProfile::Aes128CmSha1_80 => write!(f, "SRTP_AES128_CM_SHA1_80"), + SrtpProfile::AeadAes128Gcm => write!(f, "SRTP_AEAD_AES_128_GCM"), + SrtpProfile::AeadAes256Gcm => write!(f, "SRTP_AEAD_AES_256_GCM"), + } + } +} + +pub struct DummySrtpCryptoImpl(CryptoProvider); + +impl SrtpCryptoImpl for DummySrtpCryptoImpl { + type Aes128CmSha1_80 = (); + type AeadAes128Gcm = (); + type AeadAes256Gcm = (); + + fn new_aes_128_cm_sha1_80(&self, _: AesKey, _: bool) -> Self::Aes128CmSha1_80 { + panic!("Must enable feature: {}", self.0) + } + + fn new_aead_aes_128_gcm(&self, _: aead_aes_128_gcm::AeadKey, _: bool) -> Self::AeadAes128Gcm { + panic!("Must enable feature: {}", self.0) + } + + fn new_aead_aes_256_gcm(&self, _: aead_aes_256_gcm::AeadKey, _: bool) -> Self::AeadAes256Gcm { + panic!("Must enable feature: {}", self.0) + } + + fn srtp_aes_128_ecb_round(&self, _: &[u8], _: &[u8], _: &mut [u8]) { + panic!("Must enable feature: {}", self.0) + } + + fn srtp_aes_256_ecb_round(&self, _: &[u8], _: &[u8], _: &mut [u8]) { + panic!("Must enable feature: {}", self.0) + } +} + +impl aes_128_cm_sha1_80::CipherCtx for () { + fn new(_: AesKey, _: bool) -> Self + where + Self: Sized, + { + unreachable!() + } + + fn encrypt( + &mut self, + _: &aes_128_cm_sha1_80::RtpIv, + _: &[u8], + _: &mut [u8], + ) -> Result<(), super::CryptoError> { + unreachable!() + } + + fn decrypt( + &mut self, + _: &aes_128_cm_sha1_80::RtpIv, + _: &[u8], + _: &mut [u8], + ) -> Result<(), super::CryptoError> { + unreachable!() + } +} + +impl aead_aes_128_gcm::CipherCtx for () { + fn new(_: aead_aes_128_gcm::AeadKey, _: bool) -> Self + where + Self: Sized, + { + unreachable!() + } + + fn encrypt( + &mut self, + _: &[u8; aead_aes_128_gcm::IV_LEN], + _: &[u8], + _: &[u8], + _: &mut [u8], + ) -> Result<(), super::CryptoError> { + unreachable!() + } + + fn decrypt( + &mut self, + _: &[u8; aead_aes_128_gcm::IV_LEN], + _: &[&[u8]], + _: &[u8], + _: &mut [u8], + ) -> Result { + unreachable!() + } +} + +impl aead_aes_256_gcm::CipherCtx for () { + fn new(_: aead_aes_256_gcm::AeadKey, _: bool) -> Self + where + Self: Sized, + { + unreachable!() + } + + fn encrypt( + &mut self, + _: &[u8; aead_aes_256_gcm::IV_LEN], + _: &[u8], + _: &[u8], + _: &mut [u8], + ) -> Result<(), super::CryptoError> { + unreachable!() + } + + fn decrypt( + &mut self, + _: &[u8; aead_aes_256_gcm::IV_LEN], + _: &[&[u8]], + _: &[u8], + _: &mut [u8], + ) -> Result { + unreachable!() + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/dtls.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/dtls.rs new file mode 100644 index 000000000..13da4f93a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/dtls.rs @@ -0,0 +1,187 @@ +use std::collections::VecDeque; +use std::io; +use std::ops::RangeInclusive; +use std::panic::{RefUnwindSafe, UnwindSafe}; +use std::time::Instant; + +use crate::crypto::Fingerprint; +use crate::crypto::Sha256Provider; +use crate::crypto::dtls::{DtlsCert, DtlsOutput, ProtocolVersion}; +use crate::crypto::dtls::{DtlsInstance, DtlsProvider, DtlsVersion}; +use crate::crypto::{CryptoError, DtlsError}; +use crate::io::DatagramSend; +use crate::util::already_happened; + +/// Encapsulation of DTLS. +/// +/// This is a thin wrapper around `DtlsInstance` that adds fingerprint tracking +/// and active/passive state management. The API follows dimpl's sans-IO pattern. +pub struct Dtls { + /// The underlying DTLS instance. + instance: Box, + + /// The fingerprint of the local certificate. + fingerprint: Fingerprint, + + /// Remote fingerprint (set when received via poll_output). + remote_fingerprint: Option, + + /// Whether set_active has been called. + active_state: Option, + + /// Packets to be sent. + pending_packets: VecDeque, + + /// Target MTU (start) and warn threshold (end). The target is forwarded to + /// the backend's fragmenter; outgoing records larger than the warn + /// threshold log a warning. + mtu: RangeInclusive, +} + +pub(crate) fn is_would_block(error: &DtlsError) -> bool { + match error { + DtlsError::Io(e) => e.kind() == io::ErrorKind::WouldBlock, + DtlsError::CryptoError(crypto_err) => match crypto_err { + CryptoError::Io(e) => e.kind() == io::ErrorKind::WouldBlock, + #[allow(unreachable_patterns)] + _ => false, + }, + } +} + +impl UnwindSafe for Dtls {} +impl RefUnwindSafe for Dtls {} + +impl Dtls { + /// Creates a new DTLS instance. + pub fn new( + cert: &DtlsCert, + dtls_provider: &dyn DtlsProvider, + sha256_provider: &dyn Sha256Provider, + now: Instant, + dtls_version: DtlsVersion, + mtu: RangeInclusive, + ) -> Result { + let instance = dtls_provider + .new_dtls(cert, now, dtls_version, Some(*mtu.start())) + .map_err(DtlsError::CryptoError)?; + + // Compute fingerprint from the certificate DER bytes + let fingerprint = Fingerprint { + hash_func: "sha-256".to_string(), + bytes: sha256_provider.sha256(&cert.certificate).to_vec(), + }; + + Ok(Self { + instance, + fingerprint, + remote_fingerprint: None, + active_state: None, + pending_packets: VecDeque::new(), + mtu, + }) + } + + /// Threshold above which an outgoing DTLS record triggers an MTU warning. + pub fn mtu_warn(&self) -> usize { + *self.mtu.end() + } + + /// Tells if this instance has been inited (set_active called). + pub fn is_inited(&self) -> bool { + self.active_state.is_some() + } + + /// Set whether this instance is active (client) or passive (server). + pub fn set_active(&mut self, active: bool) { + self.active_state = Some(active); + self.instance.set_active(active) + } + + /// If set_active was called, returns what was set. + pub fn is_active(&self) -> Option { + self.active_state + } + + /// The local certificate fingerprint. + pub fn local_fingerprint(&self) -> &Fingerprint { + &self.fingerprint + } + + /// Remote fingerprint, if received. + pub fn remote_fingerprint(&self) -> Option<&Fingerprint> { + self.remote_fingerprint.as_ref() + } + + /// The negotiated DTLS protocol version, or `None` before handshake completion. + pub fn protocol_version(&self) -> Option { + self.instance.protocol_version() + } + + pub fn is_closed(&self) -> bool { + self.pending_packets.is_empty() && self.instance.is_closed() + } + + /// Set the remote fingerprint (extracted from peer certificate). + pub fn set_remote_fingerprint(&mut self, fingerprint: Fingerprint) { + self.remote_fingerprint = Some(fingerprint); + } + + /// Poll for output from the DTLS instance. + pub fn poll_output<'a>(&mut self, buf: &'a mut [u8]) -> DtlsOutput<'a> { + let next = self.instance.poll_output(buf); + + if let DtlsOutput::Packet(packet) = next { + if packet.len() > self.mtu_warn() { + warn!("DTLS above MTU {}: {}", self.mtu_warn(), packet.len()); + } + self.pending_packets.push_back(packet.to_vec().into()); + + // Return timeout indicating we want another poll straight away + return DtlsOutput::Timeout(already_happened()); + } + + next + } + + pub fn poll_packet(&mut self) -> Option { + self.pending_packets.pop_front() + } + + /// Handle an incoming DTLS packet. + pub fn handle_receive(&mut self, packet: &[u8]) -> Result<(), DtlsError> { + if self.active_state.is_none() { + debug!("Ignoring DTLS datagram prior to DTLS start"); + return Ok(()); + } + + self.instance + .handle_packet(packet) + .map_err(|e| DtlsError::CryptoError(CryptoError::Other(format!("DTLS error: {}", e)))) + } + + /// Send application data over DTLS. + pub fn handle_input(&mut self, data: &[u8]) -> Result<(), DtlsError> { + self.instance.send_application_data(data).map_err(|e| { + if matches!(e, dimpl::Error::HandshakePending) { + DtlsError::Io(io::Error::new(io::ErrorKind::WouldBlock, e)) + } else { + DtlsError::CryptoError(CryptoError::Other(format!("DTLS error: {}", e))) + } + }) + } + + /// Handle a timeout event. + pub fn handle_timeout(&mut self, now: Instant) -> Result<(), DtlsError> { + self.instance + .handle_timeout(now) + .map_err(|e| DtlsError::CryptoError(CryptoError::Other(format!("DTLS error: {}", e)))) + } + + /// Initiate graceful shutdown by sending a close_notify alert. + pub fn close(&mut self) -> Result<(), DtlsError> { + self.instance + .close() + .map_err(|e| DtlsError::CryptoError(CryptoError::Other(format!("DTLS error: {}", e)))) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/error.rs new file mode 100644 index 000000000..ab0221fcb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/error.rs @@ -0,0 +1,182 @@ +//! Various error types. + +use std::error::Error; +use std::fmt; + +// Re-export all error types for convenience +pub use crate::crypto::DtlsError; +pub use crate::io::NetError; +pub use crate::packet::{PacketError, Vp8DescriptorError, Vp8PatchError}; +pub use crate::rtp_::RtpError; +pub use crate::sctp::ProtoError; +pub use crate::sctp::SctpError; +pub use crate::sdp::SdpError; +pub use is::IceError; + +use crate::{Direction, KeyframeRequestKind, Mid, Pt, Rid}; + +/// Errors for the whole Rtc engine. +#[derive(Debug)] +#[non_exhaustive] +pub enum RtcError { + /// Some problem with the remote SDP. + RemoteSdp(String), + + /// SDP errors. + Sdp(SdpError), + + /// RTP errors. + Rtp(RtpError), + + /// Other IO errors. + Io(std::io::Error), + + /// DTLS errors + Dtls(DtlsError), + + /// RTP packetization error + Packet(Mid, Pt, PacketError), + + /// The PT attempted to write to is not known. + UnknownPt(Pt), + + /// The Rid attempted to write is not known. + UnknownRid(Rid), + + /// If MediaWriter.write fails because we can't find an SSRC to use. + NoSenderSource, + + /// Using `write_rtp` for a stream with RTX without providing a rtx_pt. + ResendRequiresRtxPt, + + /// Direction does not allow sending of Media data. + NotSendingDirection(Direction), + + /// Direction does not allow receiving media data. + NotReceivingDirection, + + /// If MediaWriter.request_keyframe fails because we can't find an SSRC to use. + NoReceiverSource(Option), + + /// The keyframe request failed because the kind of request is not enabled + /// in the media. + FeedbackNotEnabled(KeyframeRequestKind), + + /// Parser errors from network packet parsing. + Net(NetError), + + /// ICE agent errors. + Ice(IceError), + + /// SCTP (data channel engine) errors. + Sctp(SctpError), + + /// [`crate::change::SdpApi`] was not done in a correct order. + /// + /// For [`crate::change::SdpApi`]: + /// + /// 1. We created an [`crate::change::SdpOffer`]. + /// 2. The remote side created an [`crate::change::SdpOffer`] at the same time. + /// 3. We applied the remote side [`crate::change::SdpApi::accept_offer`]. + /// 4. The we used the [`crate::change::SdpPendingOffer`] created in step 1. + ChangesOutOfOrder, + + /// The [`crate::media::Writer`] was used twice without doing [`crate::Rtc::poll_output`] in between. This + /// is an incorrect usage pattern of the str0m API. + WriteWithoutPoll, +} + +impl fmt::Display for RtcError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RtcError::RemoteSdp(msg) => write!(f, "remote sdp: {}", msg), + RtcError::Sdp(err) => write!(f, "{}", err), + RtcError::Rtp(err) => write!(f, "{}", err), + RtcError::Io(err) => write!(f, "{}", err), + RtcError::Dtls(err) => write!(f, "{}", err), + RtcError::Packet(mid, pt, err) => write!(f, "{} {} {}", mid, pt, err), + RtcError::UnknownPt(pt) => write!(f, "PT is unknown {}", pt), + RtcError::UnknownRid(rid) => write!(f, "RID is unknown {}", rid), + RtcError::NoSenderSource => write!(f, "No sender source"), + RtcError::ResendRequiresRtxPt => write!( + f, + "When outgoing stream has RTX, write_rtp must be called with rtp_pt set" + ), + RtcError::NotSendingDirection(dir) => { + write!(f, "Direction does not allow sending: {}", dir) + } + RtcError::NotReceivingDirection => write!(f, "Direction does not allow receiving"), + RtcError::NoReceiverSource(rid) => write!(f, "No receiver source (rid: {:?})", rid), + RtcError::FeedbackNotEnabled(kind) => { + write!(f, "Requested feedback is not enabled: {:?}", kind) + } + RtcError::Net(err) => write!(f, "{}", err), + RtcError::Ice(err) => write!(f, "{}", err), + RtcError::Sctp(err) => write!(f, "{}", err), + RtcError::ChangesOutOfOrder => write!(f, "Changes made out of order"), + RtcError::WriteWithoutPoll => write!( + f, + "Consecutive calls to write() without poll_output() in between" + ), + } + } +} + +impl Error for RtcError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + RtcError::Sdp(err) => Some(err), + RtcError::Rtp(err) => Some(err), + RtcError::Io(err) => Some(err), + RtcError::Dtls(err) => Some(err), + RtcError::Packet(_, _, err) => Some(err), + RtcError::Net(err) => Some(err), + RtcError::Ice(err) => Some(err), + RtcError::Sctp(err) => Some(err), + _ => None, + } + } +} + +// From implementations for error conversions +impl From for RtcError { + fn from(err: SdpError) -> Self { + RtcError::Sdp(err) + } +} + +impl From for RtcError { + fn from(err: RtpError) -> Self { + RtcError::Rtp(err) + } +} + +impl From for RtcError { + fn from(err: std::io::Error) -> Self { + RtcError::Io(err) + } +} + +impl From for RtcError { + fn from(err: DtlsError) -> Self { + RtcError::Dtls(err) + } +} + +impl From for RtcError { + fn from(err: NetError) -> Self { + RtcError::Net(err) + } +} + +impl From for RtcError { + fn from(err: IceError) -> Self { + RtcError::Ice(err) + } +} + +impl From for RtcError { + fn from(err: SctpError) -> Self { + RtcError::Sctp(err) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec.rs new file mode 100644 index 000000000..f76bd9a6a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec.rs @@ -0,0 +1,211 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +use crate::packet::MediaKind; +use crate::rtp_::Frequency; + +use super::format_params::FormatParams; + +/// Codec specification +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodecSpec { + /// The codec identifier. + pub codec: Codec, + + /// Clock rate of the codec. + pub clock_rate: Frequency, + + /// Number of audio channels (if any). + pub channels: Option, + + /// Codec specific format parameters. This might carry additional config for + /// things like h264. + pub format: FormatParams, +} + +impl CodecSpec { + /// The RTP clock rate used on the wire for this codec. + /// + /// For most codecs this is the same as [`CodecSpec::clock_rate`]. G722 is a + /// special case: although the codec samples audio at 16 kHz, its RTP timestamp + /// clock is 8000 Hz (to remain backwards compatible with RFC 1890, which + /// incorrectly used this value). str0m treats G722 as a 16 kHz codec everywhere + /// user facing, and only maps to 8 kHz when converting to and from RTP timestamps + /// (and in the SDP `a=rtpmap` line). + /// + /// See RFC 3551 §4.5.2 and + /// + pub(crate) fn rtp_clock_rate(&self) -> Frequency { + if self.codec == Codec::G722 { + Frequency::EIGHT_KHZ + } else { + self.clock_rate + } + } +} + +/// Known codecs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +#[allow(missing_docs)] +pub enum Codec { + Opus, + PCMU, + PCMA, + G722, + /// Comfort Noise payload, per RFC 3389. + CN, + H264, + // TODO show this when we support h265. + #[doc(hidden)] + H265, + /// H266/VVC. + H266, + Vp8, + Vp9, + // TODO show this when we support Av1. + #[doc(hidden)] + Av1, + /// Technically not a codec, but used in places where codecs go + /// in `a=rtpmap` lines. + #[doc(hidden)] + Rtx, + /// For RTP mode. No codec. + #[doc(hidden)] + Null, + #[doc(hidden)] + Unknown, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(Codec, CodecSpec); + +impl Codec { + /// Tells if codec is audio. + pub fn is_audio(&self) -> bool { + use Codec::*; + matches!(self, Opus | PCMU | PCMA | G722 | CN) + } + + /// Tells if codec is video. + pub fn is_video(&self) -> bool { + use Codec::*; + matches!(self, H266 | H265 | H264 | Vp8 | Vp9 | Av1) + } + + /// Audio/Video. + pub fn kind(&self) -> MediaKind { + if self.is_audio() { + MediaKind::Audio + } else { + MediaKind::Video + } + } +} + +impl<'a> From<&'a str> for Codec { + fn from(v: &'a str) -> Self { + let lc = v.to_ascii_lowercase(); + match &lc[..] { + "opus" => Codec::Opus, + "pcmu" => Codec::PCMU, + "pcma" => Codec::PCMA, + "g722" => Codec::G722, + "cn" => Codec::CN, + "h264" => Codec::H264, + "h265" => Codec::H265, + "h266" => Codec::H266, + "vp8" => Codec::Vp8, + "vp9" => Codec::Vp9, + "av1" => Codec::Av1, + "rtx" => Codec::Rtx, // resends + _ => Codec::Unknown, + } + } +} + +impl fmt::Display for Codec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Codec::Opus => write!(f, "opus"), + Codec::PCMU => write!(f, "PCMU"), + Codec::PCMA => write!(f, "PCMA"), + Codec::G722 => write!(f, "G722"), + Codec::CN => write!(f, "CN"), + Codec::H264 => write!(f, "H264"), + Codec::H265 => write!(f, "H265"), + Codec::H266 => write!(f, "H266"), + Codec::Vp8 => write!(f, "VP8"), + Codec::Vp9 => write!(f, "VP9"), + Codec::Av1 => write!(f, "AV1"), + Codec::Rtx => write!(f, "rtx"), + Codec::Null => write!(f, "null"), + Codec::Unknown => write!(f, "unknown"), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::format::FormatParams; + use crate::rtp_::{Frequency, MediaTime}; + + fn g722_spec() -> CodecSpec { + CodecSpec { + codec: Codec::G722, + clock_rate: Frequency::SIXTEEN_KHZ, + channels: None, + format: FormatParams::default(), + } + } + + #[test] + fn g722_is_audio_and_parses() { + assert!(Codec::G722.is_audio()); + assert!(!Codec::G722.is_video()); + assert_eq!(Codec::from("G722"), Codec::G722); + assert_eq!(Codec::from("g722"), Codec::G722); + assert_eq!(Codec::G722.to_string(), "G722"); + } + + #[test] + fn comfort_noise_is_audio_and_parses() { + assert!(Codec::CN.is_audio()); + assert!(!Codec::CN.is_video()); + assert_eq!(Codec::from("CN"), Codec::CN); + assert_eq!(Codec::from("cn"), Codec::CN); + assert_eq!(Codec::CN.to_string(), "CN"); + } + + #[test] + fn g722_rtp_clock_rate_is_8khz() { + let spec = g722_spec(); + // User facing clock rate is 16 kHz ... + assert_eq!(spec.clock_rate, Frequency::SIXTEEN_KHZ); + // ... but the RTP wire clock rate is 8 kHz (RFC 3551). + assert_eq!(spec.rtp_clock_rate(), Frequency::EIGHT_KHZ); + } + + #[test] + fn g722_16khz_media_time_halves_on_the_wire() { + // 20 ms of 16 kHz audio is 320 samples. On the wire the RTP timestamp must + // advance by 160 (8000 Hz clock), i.e. the 16 kHz media time is halved when + // converted to an RTP timestamp. + let spec = g722_spec(); + let media_time = MediaTime::new(320, spec.clock_rate); + let wire = media_time.rebase(spec.rtp_clock_rate()); + assert_eq!(wire.numer(), 160); + } + + #[test] + fn non_g722_rtp_clock_rate_equals_clock_rate() { + let spec = CodecSpec { + codec: Codec::Opus, + clock_rate: Frequency::FORTY_EIGHT_KHZ, + channels: Some(2), + format: FormatParams::default(), + }; + assert_eq!(spec.rtp_clock_rate(), Frequency::FORTY_EIGHT_KHZ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec_config.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec_config.rs new file mode 100644 index 000000000..3e6f6a889 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/codec_config.rs @@ -0,0 +1,1633 @@ +use std::collections::HashSet; +use std::ops::{Deref, DerefMut}; + +use crate::packet::H266ProfileTierLevel; +use crate::packet::MediaKind; +use crate::rtp_::Pt; +use crate::rtp_::{Direction, Frequency}; + +use super::codec::{Codec, CodecSpec}; +use super::format_params::FormatParams; +use super::payload_params::{Claimed, PREFERED_RANGES, PayloadParams}; + +/// Default payload type for PCMU (G.711 μ-law). +pub(crate) const PT_PCMU: Pt = Pt::new_with_value(0); + +/// Default payload type for PCMA (G.711 A-law). +pub(crate) const PT_PCMA: Pt = Pt::new_with_value(8); + +/// Default payload type for G722. +pub(crate) const PT_G722: Pt = Pt::new_with_value(9); + +/// Static payload type for Comfort Noise at 8000 Hz (RFC 3389). +pub(crate) const PT_COMFORT_NOISE: Pt = Pt::new_with_value(13); + +/// Default payload type for VP8. +pub(crate) const PT_VP8: Pt = Pt::new_with_value(96); + +/// Default payload type for VP8 RTX. +pub(crate) const PT_VP8_RTX: Pt = Pt::new_with_value(97); + +/// Default payload type for VP9 profile 0. +pub(crate) const PT_VP9: Pt = Pt::new_with_value(98); + +/// Default payload type for VP9 profile 0 RTX. +pub(crate) const PT_VP9_RTX: Pt = Pt::new_with_value(99); + +/// Default payload type for AV1. +pub(crate) const PT_AV1: Pt = Pt::new_with_value(45); + +/// Default payload type for AV1 RTX. +pub(crate) const PT_AV1_RTX: Pt = Pt::new_with_value(46); + +/// Default payload type for H265. +pub(crate) const PT_H265: Pt = Pt::new_with_value(102); + +/// Default payload type for H265 RTX. +pub(crate) const PT_H265_RTX: Pt = Pt::new_with_value(103); + +/// Default payload type for H266 (VVC). +pub(crate) const PT_H266: Pt = Pt::new_with_value(104); + +/// Default payload type for H266 RTX. +pub(crate) const PT_H266_RTX: Pt = Pt::new_with_value(105); + +/// Default payload type for Opus. +pub(crate) const PT_OPUS: Pt = Pt::new_with_value(111); + +/// Session config for all codecs. +#[derive(Debug, Clone, Default)] +pub struct CodecConfig { + params: Vec, +} + +impl CodecConfig { + /// Creates a new empty config. + pub fn empty() -> Self { + CodecConfig::default() + } + + /// Creates a new config from payload params + pub fn new_from_payload_params(payload_params: Vec) -> Self { + CodecConfig { + params: payload_params, + } + } + + /// Creates a new config with all default configurations enabled. + pub fn new_with_defaults() -> Self { + let mut c = Self::empty(); + c.enable_opus(true); + + c.enable_vp8(true); + c.enable_h264(true); + c.enable_h265(true); + // H266/VVC off by default: no browser supports it. + c.enable_h266(false); + c.enable_vp9(true); + c.enable_av1(true); + + c + } + + /// Returns a reference to the payload parameters. + pub fn params(&self) -> &[PayloadParams] { + &self.params + } + + /// Clear all configured configs. + pub fn clear(&mut self) { + self.params.clear(); + } + + /// Manually configure a payload type. + pub fn add_config( + &mut self, + pt: Pt, + resend: Option, + codec: Codec, + clock_rate: Frequency, + channels: Option, + format: FormatParams, + ) { + let (fb_transport_cc, fb_fir, fb_nack, fb_pli, fb_remb) = if codec.is_video() { + (true, true, true, true, true) + } else { + (true, false, false, false, false) + }; + + let p = PayloadParams { + pt, + spec: CodecSpec { + codec, + clock_rate, + channels, + format, + }, + resend, + fb_transport_cc, + fb_fir, + fb_nack, + fb_pli, + fb_remb, + locked: false, + }; + + self.params.push(p); + } + + fn add_static_config(&mut self, pt: Pt) { + let spec = CodecSpec::from_static_pt(pt).expect("known static payload type"); + self.add_config( + pt, + None, + spec.codec, + spec.clock_rate, + spec.channels, + spec.format, + ); + } + + /// Convenience for adding a h264 payload type. + pub fn add_h264( + &mut self, + pt: Pt, + resend: Option, + packetization_mode: bool, + profile_level_id: u32, + ) { + self.add_config( + pt, + resend, + Codec::H264, + Frequency::NINETY_KHZ, + None, + FormatParams { + level_asymmetry_allowed: Some(true), + packetization_mode: if packetization_mode { Some(1) } else { Some(0) }, + profile_level_id: Some(profile_level_id), + ..Default::default() + }, + ) + } + + /// Convenience for adding a h265 payload type. + pub fn add_h265( + &mut self, + pt: Pt, + resend: Option, + profile_id: u8, + tier_flag: u8, + level_id: u8, + ) { + use crate::packet::H265ProfileTierLevel; + + let ptl = H265ProfileTierLevel::new(profile_id, tier_flag, level_id) + .unwrap_or(H265ProfileTierLevel::FALLBACK); + + self.add_config( + pt, + resend, + Codec::H265, + Frequency::NINETY_KHZ, + None, + FormatParams { + h265_profile_tier_level: Some(ptl), + ..Default::default() + }, + ) + } + + /// Convenience for adding a PCM u-law payload type. + pub fn enable_pcmu(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::PCMU); + if !enabled { + return; + } + self.add_static_config(PT_PCMU); + } + + /// Convenience for adding a PCM a-law payload type. + pub fn enable_pcma(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::PCMA); + if !enabled { + return; + } + self.add_static_config(PT_PCMA); + } + + /// Convenience for adding a G722 payload type. + /// + /// G722 samples audio at 16 kHz, but per RFC 3551 §4.5.2 the RTP clock rate is + /// 8000 Hz. The codec is configured at 16 kHz here; str0m maps to 8 kHz for the + /// SDP `a=rtpmap` line and when converting to/from RTP timestamps. See + /// + pub fn enable_g722(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::G722); + if !enabled { + return; + } + self.add_static_config(PT_G722); + } + + /// Add the static Comfort Noise payload type at 8000 Hz. + /// + /// Higher clock rates require dynamic payload types and can be configured + /// with [`CodecConfig::add_config`]. + pub fn enable_comfort_noise(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::CN); + if !enabled { + return; + } + self.add_static_config(PT_COMFORT_NOISE); + } + + /// Add a default OPUS payload type. + pub fn enable_opus(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::Opus); + if !enabled { + return; + } + self.add_config( + PT_OPUS, + None, + Codec::Opus, + Frequency::FORTY_EIGHT_KHZ, + Some(2), + FormatParams { + min_p_time: Some(10), + use_inband_fec: Some(true), + ..Default::default() + }, + ) + } + + /// Add a default VP8 payload type. + pub fn enable_vp8(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::Vp8); + if !enabled { + return; + } + self.add_config( + PT_VP8, + Some(PT_VP8_RTX), + Codec::Vp8, + Frequency::NINETY_KHZ, + None, + FormatParams::default(), + ) + } + + /// Add a default H264 payload type. + pub fn enable_h264(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::H264); + if !enabled { + return; + } + const PARAMS: &[(u8, u8, bool, u32)] = &[ + (127, 121, true, 0x42001f), + (125, 107, false, 0x42001f), + (108, 109, true, 0x42e01f), + (124, 120, false, 0x42e01f), + (123, 119, true, 0x4d001f), + (35, 36, false, 0x4d001f), + (114, 115, true, 0x64001f), + ]; + + for p in PARAMS { + self.add_h264(p.0.into(), Some(p.1.into()), p.2, p.3) + } + } + + /// Add a default H265 payload type. + /// This enables H265 as a video codec (clock rate 90kHz) with default RTX payload type. + /// Note: H265 is still considered an experimental/hidden codec in parts of the public API, + /// but it is supported internally for packetization/depacketization. + pub fn enable_h265(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::H265); + if !enabled { + return; + } + + // Start with single default configuration + // Profile/tier/level negotiation will happen via SDP fmtp parameters + // + // Level 6.0 (level_id=180) — a high capability level well supported by modern browsers. + // Level-id formula per ITU-T H.265 Annex A: level_id = (a*10 + b) * 3 + // e.g. Level 6.0 → (6*10 + 0) * 3 = 180 + // + // Chromium H265Level enum definition (kLevel6 = 180): + // https://source.chromium.org/chromium/chromium/src/+/main: + // third_party/webrtc/api/video_codecs/h265_profile_tier_level.h + self.add_h265(PT_H265, Some(PT_H265_RTX), 1, 0, 180); // Main, Main tier, Level 6.0 + } + + /// Convenience for adding a h266 payload type. + pub fn add_h266( + &mut self, + pt: Pt, + resend: Option, + profile_id: u8, + tier_flag: u8, + level_id: u8, + ) { + let ptl = H266ProfileTierLevel::new(profile_id, tier_flag, level_id) + .unwrap_or(H266ProfileTierLevel::FALLBACK); + + self.add_config( + pt, + resend, + Codec::H266, + Frequency::NINETY_KHZ, + None, + FormatParams { + h266_profile_tier_level: Some(ptl), + ..Default::default() + }, + ) + } + + /// Add a default H266 (VVC) payload type. + /// + /// Enables H266 as a video codec (clock rate 90kHz) with a default RTX + /// payload type. Profile/tier/level negotiation happens via the SDP fmtp + /// parameters profile-id / tier-flag / level-id (RFC 9328 §7.2). + pub fn enable_h266(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::H266); + if !enabled { + return; + } + + // Single default configuration with a high capability level. + // + // Level 6.0 (level_id=96) — per ITU-T H.266 Annex A the level_id is + // 16*major + 3*minor, e.g. Level 6.0 → 16*6 = 96. + self.add_h266(PT_H266, Some(PT_H266_RTX), 1, 0, 96); // Main 10, Main tier, Level 6.0 + } + + /// Add a default VP9 payload type. + pub fn enable_vp9(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::Vp9); + if !enabled { + return; + } + self.add_config( + PT_VP9, + Some(PT_VP9_RTX), + Codec::Vp9, + Frequency::NINETY_KHZ, + None, + FormatParams { + profile_id: Some(0), + ..Default::default() + }, + ); + self.add_config( + 100.into(), + Some(101.into()), + Codec::Vp9, + Frequency::NINETY_KHZ, + None, + FormatParams { + profile_id: Some(2), + ..Default::default() + }, + ); + } + + /// Add a default AV1 payload type. + pub fn enable_av1(&mut self, enabled: bool) { + self.params.retain(|c| c.spec.codec != Codec::Av1); + if !enabled { + return; + } + self.add_config( + PT_AV1, + Some(PT_AV1_RTX), + Codec::Av1, + Frequency::NINETY_KHZ, + None, + FormatParams::default(), + ); + } + + /// Match the given parameters to the configured parameters. + /// + /// In a server scenario, a certain codec configuration might not have the same + /// payload type (PT) for two different peers. We will have incoming data with one + /// PT and need to match that against the PT of the outgoing. + /// + /// This call performs matching and if a match is found, returns the _local_ PT + /// that can be used for sending media. + pub fn match_params(&self, params: PayloadParams) -> Option<&PayloadParams> { + let c = self.params.iter().max_by_key(|p| p.match_score(¶ms))?; + c.match_score(¶ms)?; // avoid None, which isn't a match. + Some(c) + } + + /// When we get remote payload parameters, we need to match differently depending on direction. + pub(crate) fn sdp_match_remote( + &self, + remote_params: PayloadParams, + remote_dir: Direction, + ) -> Option { + // If we have no matching parameters locally, we can't accept the remote in any way. + let our_params = self.match_params(remote_params)?; + + if remote_dir.sdp_is_receiving() { + // The remote is talking about its own receive requirements. The PTs are not suggestions. + Some(remote_params.pt()) + } else { + // We can override the remote with our local config. We would have adjusted to + // the remote earlier if we can. + Some(our_params.pt()) + } + } + + /// Find a payload parameter using a finder function. + pub fn find(&self, mut f: impl FnMut(&PayloadParams) -> bool) -> Option<&PayloadParams> { + self.params.iter().find(move |p| f(p)) + } + + pub(crate) fn all_for_kind(&self, kind: MediaKind) -> impl Iterator { + self.params.iter().filter(move |params| { + if kind == MediaKind::Video { + params.spec.codec.is_video() + } else { + params.spec.codec.is_audio() + } + }) + } + + pub(crate) fn update_params(&mut self, remote_params: &[PayloadParams], remote_dir: Direction) { + // 0-128 of "claimed" PTs. I.e. PTs that we already allocated to something. + let mut claimed: [bool; 128] = [false; 128]; + + // Make a pass with all that are definitely confirmed by the remote, since these can't change. + for p in self.params.iter() { + if !p.locked { + continue; + } + + claimed.assert_claim_once(p.pt); + + if let Some(rtx) = p.resend { + claimed.assert_claim_once(rtx); + } + } + + // Collect all currently unlocked PTs since we will need to avoid them when reassigning. + let mut unlocked = self + .params + .iter() + .filter(|p| !p.locked) + .flat_map(|p| [Some(p.pt()), p.resend()]) + .flatten() + .collect::>(); + + // Now lock potential new parameters to remote. + // + // If the remote is doing `SendOnly`, we are receiving, so the PTs are suggestions, + // and we are allowed to ANSWER with our own allocations as overrides. + // If SendRecv or RecvOnly, the remote is talking about its own receiving capabilities + // and we are not allowed to change it in the ANSWER. + let local_is_controlling = !remote_dir.sdp_is_receiving(); + + for p in self.params.iter_mut() { + p.update_param( + remote_params, + &mut claimed, + local_is_controlling, + &mut unlocked, + ); + } + + // Make a pass to reassign unconfirmed payloads that have PT which are now claimed. + for p in self.params.iter_mut() { + if p.locked { + continue; + } + + if claimed.is_claimed(p.pt) { + let Some(pt) = claimed.find_unclaimed(PREFERED_RANGES, &unlocked) else { + // TODO: handle this gracefully. + panic!("Exhausted all PT ranges, inconsistent PayloadParam state"); + }; + + debug!("Reassigned PT {} => {}", p.pt, pt); + p.pt = pt; + + claimed.assert_claim_once(pt); + unlocked.remove(&pt); + } + + let Some(rtx) = p.resend else { + continue; + }; + + if claimed.is_claimed(rtx) { + let Some(rtx) = claimed.find_unclaimed(PREFERED_RANGES, &unlocked) else { + // TODO: handle this gracefully. + panic!("Exhausted all PT ranges, inconsistent PayloadParam state"); + }; + + debug!("Reassigned RTX PT {:?} => {:?}", p.resend, rtx); + p.resend = Some(rtx); + + claimed.assert_claim_once(rtx); + unlocked.remove(&rtx); + } + } + } + + pub(crate) fn has_pt(&self, pt: Pt) -> bool { + self.params.iter().any(|p| p.pt() == pt) + } +} + +impl Deref for CodecConfig { + type Target = [PayloadParams]; + + fn deref(&self) -> &Self::Target { + &self.params + } +} + +impl DerefMut for CodecConfig { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.params + } +} + +impl CodecSpec { + /// Get the standardized codec definition for a static RTP payload type. + pub(crate) fn from_static_pt(pt: Pt) -> Option { + let (codec, clock_rate) = if pt == PT_PCMU { + (Codec::PCMU, Frequency::EIGHT_KHZ) + } else if pt == PT_PCMA { + (Codec::PCMA, Frequency::EIGHT_KHZ) + } else if pt == PT_G722 { + // G722 is represented as 16 kHz internally. CodecSpec::rtp_clock_rate() + // maps it to the standardized 8 kHz RTP clock rate on the wire. + (Codec::G722, Frequency::SIXTEEN_KHZ) + } else if pt == PT_COMFORT_NOISE { + (Codec::CN, Frequency::EIGHT_KHZ) + } else { + return None; + }; + + Some(CodecSpec { + codec, + clock_rate, + channels: None, + format: FormatParams::default(), + }) + } +} + +#[cfg(test)] +mod test { + use crate::packet::MediaKind; + use crate::rtp_::Direction; + use crate::rtp_::Frequency; + + use super::*; + use crate::format::{CodecSpec, FormatParams}; + + #[test] + fn static_payload_types_have_canonical_codec_definitions() { + let definitions = [ + (PT_PCMU, Codec::PCMU, Frequency::EIGHT_KHZ), + (PT_PCMA, Codec::PCMA, Frequency::EIGHT_KHZ), + (PT_G722, Codec::G722, Frequency::SIXTEEN_KHZ), + (PT_COMFORT_NOISE, Codec::CN, Frequency::EIGHT_KHZ), + ]; + + for (pt, codec, clock_rate) in definitions { + let spec = CodecSpec::from_static_pt(pt).expect("static payload definition"); + assert_eq!(spec.codec, codec); + assert_eq!(spec.clock_rate, clock_rate); + assert_eq!(spec.channels, None); + assert_eq!(spec.format, FormatParams::default()); + } + + assert!(CodecSpec::from_static_pt(PT_OPUS).is_none()); + } + + #[test] + fn dynamic_comfort_noise_collision_preserves_configured_pt() { + let mut claimed = PayloadParams::new( + 96.into(), + None, + CodecSpec { + codec: Codec::Opus, + clock_rate: Frequency::FORTY_EIGHT_KHZ, + channels: Some(2), + format: FormatParams::default(), + }, + ); + claimed.locked = true; + + let local_cn = PayloadParams::new( + 97.into(), + None, + CodecSpec { + codec: Codec::CN, + clock_rate: Frequency::SIXTEEN_KHZ, + channels: None, + format: FormatParams::default(), + }, + ); + let remote_cn = PayloadParams::new( + 96.into(), + None, + CodecSpec { + codec: Codec::CN, + clock_rate: Frequency::SIXTEEN_KHZ, + channels: None, + format: FormatParams::default(), + }, + ); + + let mut config = CodecConfig::new_from_payload_params(vec![claimed, local_cn]); + config.update_params(&[remote_cn], Direction::SendOnly); + + let cn = config + .iter() + .find(|p| p.spec().codec == Codec::CN) + .expect("configured CN payload"); + assert_eq!( + *cn.pt(), + 97, + "dynamic CN must retain its configured payload type" + ); + } + + #[test] + fn collision_preserves_configured_primary_and_rtx_pts() { + let mut claimed = PayloadParams::new( + 96.into(), + Some(97.into()), + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + ); + claimed.locked = true; + + let vp8 = CodecSpec { + codec: Codec::Vp8, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }; + let local_vp8 = PayloadParams::new(110.into(), Some(111.into()), vp8); + let remote_vp8 = PayloadParams::new(96.into(), Some(97.into()), vp8); + + let mut config = CodecConfig::new_from_payload_params(vec![claimed, local_vp8]); + config.update_params(&[remote_vp8], Direction::SendOnly); + + let vp8 = config + .iter() + .find(|p| p.spec().codec == Codec::Vp8) + .expect("configured VP8 payload"); + assert_eq!((*vp8.pt(), vp8.resend().map(|pt| *pt)), (110, Some(111))); + } + + #[test] + fn enable_comfort_noise_uses_static_8khz_payload_type() { + let mut config = CodecConfig::empty(); + + config.enable_comfort_noise(true); + + let params = config + .params() + .iter() + .find(|p| p.spec().codec == Codec::CN) + .expect("Comfort Noise should be enabled"); + assert_eq!(params.pt(), Pt::new_with_value(13)); + assert_eq!(params.spec().clock_rate, Frequency::EIGHT_KHZ); + assert_eq!(params.spec().channels, None); + + config.enable_comfort_noise(false); + assert!(config.params().iter().all(|p| p.spec().codec != Codec::CN)); + } + + #[test] + fn comfort_noise_supports_dynamic_payload_types_at_higher_clock_rates() { + let mut config = CodecConfig::empty(); + + for (pt, clock_rate) in [(96, 16_000), (97, 24_000), (98, 32_000), (99, 48_000)] { + config.add_config( + pt.into(), + None, + Codec::CN, + Frequency::new(clock_rate).unwrap(), + None, + FormatParams::default(), + ); + } + + let configured: Vec<_> = config + .params() + .iter() + .map(|p| (*p.pt(), p.spec().clock_rate.get())) + .collect(); + assert_eq!( + configured, + vec![(96, 16_000), (97, 24_000), (98, 32_000), (99, 48_000)] + ); + } + + #[test] + fn test_pt_conflict_different_directions() { + // Simulates: + // 1. str0m OFFER sendonly H264 PT 108, RTX PT 109 + // 2. FF ANSWER recvonly (locks PT 109 as H264 RTX) + // 3. FF OFFER sendonly Opus PT 109 + // 4. Crash on PT 109 conflict + + let mut config = CodecConfig::empty(); + + // Initial local config: H264 with PT 108, RTX 109 and Opus with PT 111 + config.add_config( + 108.into(), + Some(109.into()), + Codec::H264, + Frequency::NINETY_KHZ, + None, + FormatParams { + packetization_mode: Some(1), + profile_level_id: Some(0x42e01f), + ..Default::default() + }, + ); + config.add_config( + 111.into(), + None, + Codec::Opus, + Frequency::FORTY_EIGHT_KHZ, + Some(2), + FormatParams::default(), + ); + + // Step 2: Remote answers recvonly with our H264 PT 108/109 + // This locks PT 109 as RTX + let remote_h264_params = vec![PayloadParams::new( + 108.into(), + Some(109.into()), + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + packetization_mode: Some(1), + profile_level_id: Some(0x42e01f), + ..Default::default() + }, + }, + )]; + + config.update_params(&remote_h264_params, Direction::RecvOnly); + + // Verify PT 109 is now locked as RTX for H264 + assert!( + config + .params() + .iter() + .any(|p| p.resend() == Some(109.into()) && p.locked) + ); + + // Step 3: Remote offers sendonly Opus PT 109 + // This should NOT crash - PT 109 should be remapped to 111 (Opus default) + let remote_opus_params = vec![PayloadParams::new( + 109.into(), + None, + CodecSpec { + codec: Codec::Opus, + clock_rate: Frequency::FORTY_EIGHT_KHZ, + channels: Some(2), + format: FormatParams::default(), + }, + )]; + + config.update_params(&remote_opus_params, Direction::SendOnly); + + // Verify the fix: Opus should be remapped to PT_OPUS (its default) + let opus_param = config + .params() + .iter() + .find(|p| p.spec().codec == Codec::Opus); + assert!(opus_param.is_some(), "Opus param should exist"); + assert_eq!( + opus_param.unwrap().pt(), + PT_OPUS, + "Opus should be remapped to PT_OPUS (default)" + ); + assert!(opus_param.unwrap().locked, "Opus param should be locked"); + + // Verify H264 RTX PT 109 is still locked + assert!( + config + .params() + .iter() + .any(|p| p.resend() == Some(109.into()) && p.locked) + ); + } + + #[test] + fn test_pt_conflict_same_mline_multiple_codecs() { + // Reproduces a bug we have had where reassigned PTs were conflicting with unlocked PTs. + // + // SCENARIO & ROOT CAUSE: + // This test reproduces the exact bug from sdp.json using str0m's default config. + // + // 1. str0m starts with DEFAULT codec configuration: + // - VP8 at PT 96/97 (unlocked) + // - H264 High (0x64001f) at PT 114/115 (unlocked) + // - Many other H264 profiles at various PTs (127, 125, 108, 124, 123, 35) + // + // 2. Chrome (peer A) offers video with H264 only - PTs: 103/104, 109/114, 117/118 + // Chrome's OFFER direction: sendonly (Chrome sends video to us) + // **CRITICAL**: Chrome uses PT 114 as RTX for H264 42e01f (PT 109) + // + // 3. str0m ANSWERs accepting Chrome's H264 offer: + // - Chrome's H264 PTs (103, 109, 117) are LOCKED + // - Chrome's RTX PTs (104, 114, 118) are LOCKED + // - VP8 at PT 96 remains UNLOCKED (Chrome didn't negotiate it) + // - H264 High at PT 114 CONFLICTS with Chrome's RTX usage of PT 114! + // + // 4. THE BUG (before fix): update_params reassigns H264 High from PT 114: + // - Searches for available PT starting from 96 + // - PT 96 appears "unclaimed" because it only checked LOCKED PTs + // - VP8 is at PT 96 but UNLOCKED, so wasn't checked! + // - H264 High gets reassigned to PT 96 + // - Now BOTH VP8 and H264 High are at PT 96! ❌ + // + // 5. Later, Firefox (peer B) joins - str0m creates NEW video m-line (mid KvE) + // When generating SDP, str0m collects ALL video codecs from codec_config + // Result (before fix): PT 96 appears TWICE in the same m-line! + // - Line 729 in sdp.json: a=rtpmap:96 VP8/90000 + // - Line 791 in sdp.json: a=rtpmap:96 H264/90000 profile-level-id=64001f + // + // EXPECTED: Each PT in an m-line must be unique + // THE FIX: Now checks BOTH locked and unlocked PTs when reassigning + + // Start with str0m's default codec configuration + // This includes VP8 at 96/97 and H264 High at 114/115 (among others) + let mut config = CodecConfig::new_with_defaults(); + + // Verify H264 High starts at its default PT 114 BEFORE Chrome's offer + let h264_high_before = config + .params() + .iter() + .find(|p| { + p.spec().codec == Codec::H264 && p.spec().format.profile_level_id == Some(0x64001f) + }) + .expect("H264 High should exist"); + assert_eq!( + h264_high_before.pt(), + 114.into(), + "H264 High should start at PT 114" + ); + + // Simulate Chrome's OFFER with H264 only (sendonly from Chrome's perspective) + // Chrome is sending video to us, so we're receiving + // Direction::SendOnly means the remote is sending (we receive) + let chrome_h264_params = vec![ + PayloadParams::new( + 103.into(), + Some(104.into()), + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + packetization_mode: Some(1), + profile_level_id: Some(0x42001f), + ..Default::default() + }, + }, + ), + PayloadParams::new( + 109.into(), + Some(114.into()), + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + packetization_mode: Some(1), + profile_level_id: Some(0x42e01f), + ..Default::default() + }, + }, + ), + PayloadParams::new( + 117.into(), + Some(118.into()), + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + packetization_mode: Some(1), + profile_level_id: Some(0x4d001f), + ..Default::default() + }, + }, + ), + ]; + + // Process Chrome's offer - this locks the H264 PTs that Chrome offered + // Chrome is SendOnly, so from our perspective we're receiving (recvonly) + config.update_params(&chrome_h264_params, Direction::SendOnly); + + // Verify: Chrome's H264 profiles are locked at their specific PTs + assert!( + config.params().iter().any(|p| p.pt() == 103.into() + && p.spec().codec == Codec::H264 + && p.spec().format.profile_level_id == Some(0x42001f) + && p.locked), + "H264 42001f should be locked at PT 103" + ); + assert!( + config.params().iter().any(|p| p.pt() == 109.into() + && p.spec().codec == Codec::H264 + && p.spec().format.profile_level_id == Some(0x42e01f) + && p.locked), + "H264 42e01f should be locked at PT 109" + ); + assert!( + config.params().iter().any(|p| p.pt() == 117.into() + && p.spec().codec == Codec::H264 + && p.spec().format.profile_level_id == Some(0x4d001f) + && p.locked), + "H264 4d001f should be locked at PT 117" + ); + + // Verify: VP8 at PT 96 is NOT locked (Chrome didn't negotiate it) + let vp8_param = config + .params() + .iter() + .find(|p| p.spec().codec == Codec::Vp8); + assert!(vp8_param.is_some(), "VP8 should still exist"); + assert_eq!(vp8_param.unwrap().pt(), 96.into(), "VP8 should be at PT 96"); + assert!(!vp8_param.unwrap().locked, "VP8 should NOT be locked"); + + // THE FIX VERIFICATION: + // H264 64001f was originally at PT 114, but Chrome used PT 114 for RTX + // update_params should reassign H264 64001f to a PT that avoids BOTH: + // - Locked PTs (103, 104, 109, 114, 117, 118) + // - Unlocked PTs already in use (96 for VP8, 97 for VP8 RTX) + let h264_high_param = config.params().iter().find(|p| { + p.spec().codec == Codec::H264 && p.spec().format.profile_level_id == Some(0x64001f) + }); + assert!(h264_high_param.is_some(), "H264 64001f should still exist"); + let h264_pt = h264_high_param.unwrap().pt(); + + // Should NOT be at PT 96 (VP8 is there) + assert_ne!( + h264_pt, + 96.into(), + "FIX WORKING: H264 64001f should NOT be at PT 96 (VP8 is there)" + ); + + // Should NOT be at any of the locked PTs + assert!( + ![103, 104, 109, 114, 117, 118].contains(&{ *h264_pt }), + "H264 64001f should not conflict with locked PTs" + ); + + assert!( + !h264_high_param.unwrap().locked, + "H264 64001f should NOT be locked (Chrome didn't offer it)" + ); + + // Now simulate creating a new video m-line (for Firefox's camera) + // This is where the bug manifests: we collect ALL video codecs from config + let all_video_params: Vec<_> = config.all_for_kind(MediaKind::Video).collect(); + + // Extract just the PTs (including RTX PTs) + let mut all_pts = Vec::new(); + for p in all_video_params { + all_pts.push(p.pt()); + if let Some(rtx) = p.resend() { + all_pts.push(rtx); + } + } + + // THE FIX: With the bug fixed, no PT should appear multiple times + // Check for duplicate PTs - this should now PASS + let mut seen = std::collections::HashSet::new(); + let mut duplicates = Vec::new(); + for pt in &all_pts { + if !seen.insert(pt) { + duplicates.push(*pt); + } + } + + assert!( + duplicates.is_empty(), + "FAILED: Found duplicate PTs in same m-line: {:?}\nAll PTs: {:?}\n + This creates invalid SDP with multiple a=rtpmap lines for the same PT!", + duplicates, + all_pts + ); + } + + // ── H.265 direction-based negotiation tests ────────────────────────── + + /// Helper: build a CodecConfig containing only H.265 at the given PTL. + fn h265_config(profile_id: u8, tier_flag: u8, level_id: u8) -> CodecConfig { + let mut config = CodecConfig::empty(); + config.add_h265( + 102.into(), + Some(103.into()), + profile_id, + tier_flag, + level_id, + ); + config + } + + /// Helper: build a single-element remote PayloadParams for H.265 with full PTL. + fn h265_remote_ptl(profile_id: u8, tier_flag: u8, level_id: u8) -> Vec { + use crate::packet::H265ProfileTierLevel; + vec![PayloadParams::new( + 102.into(), + Some(103.into()), + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h265_profile_tier_level: Some( + H265ProfileTierLevel::new(profile_id, tier_flag, level_id).unwrap(), + ), + ..Default::default() + }, + }, + )] + } + + /// Helper: build a single-element remote PayloadParams for H.265 with profile-id only. + fn h265_remote_profile_only(profile_id: u32) -> Vec { + vec![PayloadParams::new( + 102.into(), + Some(103.into()), + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + profile_id: Some(profile_id), + ..Default::default() + }, + }, + )] + } + + /// Helper: build a single-element remote PayloadParams for H.265 with no fmtp. + fn h265_remote_no_fmtp() -> Vec { + vec![PayloadParams::new( + 102.into(), + Some(103.into()), + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + )] + } + + /// Helper: extract the H.265 param from a config (panics if missing). + fn get_h265(config: &CodecConfig) -> &PayloadParams { + config + .params() + .iter() + .find(|p| p.spec().codec == Codec::H265) + .expect("H.265 param missing") + } + + // ── recvonly: remote sends to us ───────────────────────────────────── + + /// Remote sends at Level 5.2, we can receive at Level 6.0. + /// Negotiated level should narrow to min(6.0, 5.2) = 5.2. + #[test] + fn test_h265_recvonly_remote_level_lower() { + let mut config = h265_config(1, 0, 180); // Main, Main tier, Level 6.0 + let remote = h265_remote_ptl(1, 0, 156); // Level 5.2 + + // Direction::SendOnly = remote is sending (we receive) + config.update_params(&remote, Direction::SendOnly); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 156, + "Should narrow to min(180,156) = Level 5.2" + ); + assert_eq!(ptl.profile_id(), 1, "Profile should be preserved"); + assert_eq!(ptl.tier_flag(), 0, "Tier should be preserved"); + } + + /// Remote sends at Level 6.0, we can receive at Level 5.2. + /// Negotiated level should narrow to min(5.2, 6.0) = 5.2. + #[test] + fn test_h265_recvonly_local_level_lower() { + let mut config = h265_config(1, 0, 156); // Level 5.2 + let remote = h265_remote_ptl(1, 0, 180); // Level 6.0 + + config.update_params(&remote, Direction::SendOnly); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 156, + "Should narrow to min(156,180) = Level 5.2" + ); + } + + // ── sendrecv: both sides send and receive ──────────────────────────── + + /// Both sides sendrecv. Local Level 6.0, remote Level 5.2. + /// Effective level should narrow to min(6.0, 5.2) = 5.2. + #[test] + fn test_h265_sendrecv_remote_level_lower() { + let mut config = h265_config(1, 0, 180); // Level 6.0 + let remote = h265_remote_ptl(1, 0, 156); // Level 5.2 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 156, "sendrecv: should narrow to Level 5.2"); + } + + /// Both sides sendrecv. Local Level 5.2, remote Level 6.0. + /// Effective level should narrow to min(5.2, 6.0) = 5.2. + #[test] + fn test_h265_sendrecv_local_level_lower() { + let mut config = h265_config(1, 0, 156); // Level 5.2 + let remote = h265_remote_ptl(1, 0, 180); // Level 6.0 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 156, "sendrecv: should narrow to Level 5.2"); + } + + /// Same level on both sides. Should remain unchanged. + #[test] + fn test_h265_sendrecv_same_level() { + let mut config = h265_config(1, 0, 156); // Level 5.2 + let remote = h265_remote_ptl(1, 0, 156); // Level 5.2 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 156, "Same levels should stay at Level 5.2"); + } + + // ── sendonly: we send, remote receives ─────────────────────────────── + + /// Remote declares recvonly at Level 5.2, we offer sendonly at Level 6.0. + /// Negotiated level should narrow to 5.2 (receiver wins). + #[test] + fn test_h265_sendonly_receiver_level_lower() { + let mut config = h265_config(1, 0, 180); // Level 6.0 + let remote = h265_remote_ptl(1, 0, 156); // Level 5.2 + + // Direction::RecvOnly = remote is receiving (we send) + config.update_params(&remote, Direction::RecvOnly); + + let ptl = get_h265(&config) + .spec() + .format + .h265_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 156, + "sendonly: should narrow to receiver's Level 5.2" + ); + } + + /// Remote offers only profile-id=1, no tier/level. + /// Local should drop its full PTL and echo back profile-id only. + #[test] + fn test_h265_profile_only_remote_mirrors() { + let mut config = h265_config(1, 0, 180); // Full PTL + let remote = h265_remote_profile_only(1); // profile-id only + + config.update_params(&remote, Direction::SendRecv); + + let h265 = get_h265(&config); + assert!( + h265.spec().format.h265_profile_tier_level.is_none(), + "Full PTL should be cleared when remote only offers profile-id" + ); + assert_eq!( + h265.spec().format.profile_id, + Some(1), + "profile_id should mirror the remote's value" + ); + } + + // ── no-fmtp remote ────────────────────────────────────────────────── + + /// Remote has no H.265 fmtp at all. Local should clear both fields. + #[test] + fn test_h265_no_fmtp_remote_clears() { + let mut config = h265_config(1, 0, 180); // Full PTL + let remote = h265_remote_no_fmtp(); + + config.update_params(&remote, Direction::SendRecv); + + let h265 = get_h265(&config); + assert!( + h265.spec().format.h265_profile_tier_level.is_none(), + "PTL should be cleared when remote has no fmtp" + ); + assert!( + h265.spec().format.profile_id.is_none(), + "profile_id should be cleared when remote has no fmtp" + ); + } + + // ── profile_id field cleanup ───────────────────────────────────────── + + /// When remote has full PTL, the `profile_id` field must be None + /// to avoid duplicate serialization (profile-id appears in both + /// h265_profile_tier_level AND the VP9-style profile_id field). + #[test] + fn test_h265_profile_id_cleared_on_ptl_match() { + let mut config = CodecConfig::empty(); + // Artificially set both fields to simulate a misconfigured state + config.add_config( + 102.into(), + Some(103.into()), + Codec::H265, + Frequency::NINETY_KHZ, + None, + FormatParams { + h265_profile_tier_level: Some( + crate::packet::H265ProfileTierLevel::new(1, 0, 180).unwrap(), + ), + profile_id: Some(1), // should be cleared after negotiation + ..Default::default() + }, + ); + let remote = h265_remote_ptl(1, 0, 156); + + config.update_params(&remote, Direction::SendRecv); + + let h265 = get_h265(&config); + assert!( + h265.spec().format.profile_id.is_none(), + "profile_id field must be None when full PTL is present" + ); + assert!( + h265.spec().format.h265_profile_tier_level.is_some(), + "PTL should still be present" + ); + } + + // ── default level ──────────────────────────────────────────────────── + + /// `enable_h265` should produce Level 6.0 (level_id = 180). + #[test] + fn test_h265_default_level_is_6_0() { + let mut config = CodecConfig::empty(); + config.enable_h265(true); + + let h265 = get_h265(&config); + let ptl = h265.spec().format.h265_profile_tier_level.unwrap(); + assert_eq!( + ptl.level_id(), + 180, + "Default H.265 level should be 6.0 (180)" + ); + assert_eq!( + ptl.profile_id(), + 1, + "Default H.265 profile should be Main (1)" + ); + assert_eq!(ptl.tier_flag(), 0, "Default H.265 tier should be Main (0)"); + } + + // H.266 direction-based negotiation tests. + + /// Helper: build a CodecConfig containing only H.266 at the given PTL. + fn h266_config(profile_id: u8, tier_flag: u8, level_id: u8) -> CodecConfig { + let mut config = CodecConfig::empty(); + config.add_h266(PT_H266, Some(PT_H266_RTX), profile_id, tier_flag, level_id); + config + } + + /// Helper: build a single-element remote PayloadParams for H.266 with full PTL. + fn h266_remote_ptl(profile_id: u8, tier_flag: u8, level_id: u8) -> Vec { + vec![PayloadParams::new( + PT_H266, + Some(PT_H266_RTX), + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h266_profile_tier_level: Some( + H266ProfileTierLevel::new(profile_id, tier_flag, level_id).unwrap(), + ), + ..Default::default() + }, + }, + )] + } + + /// Helper: build a single-element remote PayloadParams for H.266 with profile-id only. + fn h266_remote_profile_only(profile_id: u32) -> Vec { + vec![PayloadParams::new( + PT_H266, + Some(PT_H266_RTX), + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + profile_id: Some(profile_id), + ..Default::default() + }, + }, + )] + } + + /// Helper: build a single-element remote PayloadParams for H.266 with no fmtp. + fn h266_remote_no_fmtp() -> Vec { + vec![PayloadParams::new( + PT_H266, + Some(PT_H266_RTX), + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + )] + } + + /// Helper: extract the H.266 param from a config (panics if missing). + fn get_h266(config: &CodecConfig) -> &PayloadParams { + config + .params() + .iter() + .find(|p| p.spec().codec == Codec::H266) + .expect("H.266 param missing") + } + + // recvonly: remote sends to us. + + /// Remote sends at Level 4.0, we can receive at Level 6.0. + /// Negotiated level should narrow to min(6.0, 4.0) = 4.0. + #[test] + fn test_h266_recvonly_remote_level_lower() { + let mut config = h266_config(1, 0, 96); // Main 10, Main tier, Level 6.0 + let remote = h266_remote_ptl(1, 0, 64); // Level 4.0 + + // Direction::SendOnly = remote is sending (we receive) + config.update_params(&remote, Direction::SendOnly); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 64, + "Should narrow to min(96,64) = Level 4.0" + ); + assert_eq!(ptl.profile_id(), 1, "Profile should be preserved"); + assert_eq!(ptl.tier_flag(), 0, "Tier should be preserved"); + } + + /// Remote sends at Level 6.0, we can receive at Level 4.0. + /// Negotiated level should narrow to min(4.0, 6.0) = 4.0. + #[test] + fn test_h266_recvonly_local_level_lower() { + let mut config = h266_config(1, 0, 64); // Level 4.0 + let remote = h266_remote_ptl(1, 0, 96); // Level 6.0 + + config.update_params(&remote, Direction::SendOnly); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 64, + "Should narrow to min(64,96) = Level 4.0" + ); + } + + // sendrecv: both sides send and receive. + + /// Both sides sendrecv. Local Level 6.0, remote Level 4.0. + /// Effective level should narrow to min(6.0, 4.0) = 4.0. + #[test] + fn test_h266_sendrecv_remote_level_lower() { + let mut config = h266_config(1, 0, 96); // Level 6.0 + let remote = h266_remote_ptl(1, 0, 64); // Level 4.0 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 64, "sendrecv: should narrow to Level 4.0"); + } + + /// Both sides sendrecv. Local Level 4.0, remote Level 6.0. + /// Effective level should narrow to min(4.0, 6.0) = 4.0. + #[test] + fn test_h266_sendrecv_local_level_lower() { + let mut config = h266_config(1, 0, 64); // Level 4.0 + let remote = h266_remote_ptl(1, 0, 96); // Level 6.0 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 64, "sendrecv: should narrow to Level 4.0"); + } + + /// Same level on both sides. Should remain unchanged. + #[test] + fn test_h266_sendrecv_same_level() { + let mut config = h266_config(1, 0, 64); // Level 4.0 + let remote = h266_remote_ptl(1, 0, 64); // Level 4.0 + + config.update_params(&remote, Direction::SendRecv); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!(ptl.level_id(), 64, "Same levels should stay at Level 4.0"); + } + + // sendonly: we send, remote receives. + + /// Remote declares recvonly at Level 4.0, we offer sendonly at Level 6.0. + /// Negotiated level should narrow to 4.0 (receiver wins). + #[test] + fn test_h266_sendonly_receiver_level_lower() { + let mut config = h266_config(1, 0, 96); // Level 6.0 + let remote = h266_remote_ptl(1, 0, 64); // Level 4.0 + + // Direction::RecvOnly = remote is receiving (we send) + config.update_params(&remote, Direction::RecvOnly); + + let ptl = get_h266(&config) + .spec() + .format + .h266_profile_tier_level + .unwrap(); + assert_eq!( + ptl.level_id(), + 64, + "sendonly: should narrow to receiver's Level 4.0" + ); + } + + /// Remote offers only profile-id=1, no tier/level. + /// Local should drop its full PTL and echo back profile-id only. + #[test] + fn test_h266_profile_only_remote_mirrors() { + let mut config = h266_config(1, 0, 96); // Full PTL + let remote = h266_remote_profile_only(1); // profile-id only + + config.update_params(&remote, Direction::SendRecv); + + let h266 = get_h266(&config); + assert!( + h266.spec().format.h266_profile_tier_level.is_none(), + "Full PTL should be cleared when remote only offers profile-id" + ); + assert_eq!( + h266.spec().format.profile_id, + Some(1), + "profile_id should mirror the remote's value" + ); + } + + // no-fmtp remote. + + /// Remote has no H.266 fmtp at all. Local should clear both fields. + #[test] + fn test_h266_no_fmtp_remote_clears() { + let mut config = h266_config(1, 0, 96); // Full PTL + let remote = h266_remote_no_fmtp(); + + config.update_params(&remote, Direction::SendRecv); + + let h266 = get_h266(&config); + assert!( + h266.spec().format.h266_profile_tier_level.is_none(), + "PTL should be cleared when remote has no fmtp" + ); + assert!( + h266.spec().format.profile_id.is_none(), + "profile_id should be cleared when remote has no fmtp" + ); + } + + // profile_id field cleanup. + + /// When remote has full PTL, the `profile_id` field must be None + /// to avoid duplicate serialization (profile-id appears in both + /// h266_profile_tier_level AND the VP9-style profile_id field). + #[test] + fn test_h266_profile_id_cleared_on_ptl_match() { + let mut config = CodecConfig::empty(); + // Artificially set both fields to simulate a misconfigured state + config.add_config( + PT_H266, + Some(PT_H266_RTX), + Codec::H266, + Frequency::NINETY_KHZ, + None, + FormatParams { + h266_profile_tier_level: Some(H266ProfileTierLevel::new(1, 0, 96).unwrap()), + profile_id: Some(1), // should be cleared after negotiation + ..Default::default() + }, + ); + let remote = h266_remote_ptl(1, 0, 64); + + config.update_params(&remote, Direction::SendRecv); + + let h266 = get_h266(&config); + assert!( + h266.spec().format.profile_id.is_none(), + "profile_id field must be None when full PTL is present" + ); + assert!( + h266.spec().format.h266_profile_tier_level.is_some(), + "PTL should still be present" + ); + } + + // H.266 default level. + + /// `enable_h266(true)` should add H.266 at Main 10 / Main tier / Level 6.0. + #[test] + fn test_h266_enable_default_level() { + let mut config = CodecConfig::empty(); + config.enable_h266(true); + + let h266 = config + .params() + .iter() + .find(|p| p.spec().codec == Codec::H266) + .expect("H.266 param missing"); + let ptl = h266.spec().format.h266_profile_tier_level.unwrap(); + assert_eq!( + ptl.profile_id(), + 1, + "Default H.266 profile should be Main 10 (1)" + ); + assert_eq!(ptl.tier_flag(), 0, "Default H.266 tier should be Main (0)"); + assert_eq!(ptl.level_id(), 96, "Default H.266 level should be 6.0 (96)"); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/format/format_params.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/format_params.rs new file mode 100644 index 000000000..b25b999e2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/format_params.rs @@ -0,0 +1,316 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +use crate::sdp::FormatParam; + +/// Codec specific format parameters. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct FormatParams { + /// Opus specific parameter. + /// + /// The minimum duration of media represented by a packet. + pub min_p_time: Option, + + /// Opus specific parameter. + /// + /// Decoder-side preference for stereo audio. Default is mono per + /// RFC 7587 Section 7.1: + pub stereo: Option, + + /// Opus specific parameter. + /// + /// Sender-side stereo hint for Opus. When set, it signals the intent to + /// transmit stereo from the sender, matching the symmetric m-line model. + pub sprop_stereo: Option, + + /// Opus specific parameter. + /// + /// Specifies that the decoder can do Opus in-band FEC + pub use_inband_fec: Option, + + /// Opus specific parameter. + /// + /// Specifies that the decoder prefers DTX (Discontinuous Transmission) such that + /// the packet rate is greatly lowered during periods of silence. + pub use_dtx: Option, + + /// Whether h264 sending media encoded at a different level in the offerer-to-answerer + /// direction than the level in the answerer-to-offerer direction, is allowed. + pub level_asymmetry_allowed: Option, + + /// What h264 packetization mode is used. + /// + /// * 0 - single nal. + /// * 1 - STAP-A, FU-A is allowed. Non-interleaved. + pub packetization_mode: Option, + + /// H264 profile level. + /// + /// * 42 00 1f - 4200=baseline (B) 1f=level 3.1 + /// * 42 e0 1f - 42e0=constrained baseline (CB) 1f=level 3.1 + /// * 4d 00 1f - 4d00=main (M) 1f=level 3.1 + /// * 64 00 1f - 6400=high (H) 1f=level 3.1 + pub profile_level_id: Option, + + /// VP9 profile id. + pub profile_id: Option, + + /// AV1 profile. + /// + /// Indicates the highest AV1 profile that may have been used to generate + /// the bitstream or that the receiver supports. The range of possible values + /// is identical to the seq_profile syntax element specified in AV1. If the + /// parameter is not present, it MUST be inferred to be 0 ("Main" profile). + /// + /// 0 8-bit or 10-bit 4:2:0 + /// 1 8-bit or 10-bit 4:4:4 + /// 2 8-bit or 10-bit 4:2:2 + /// 2 12-bit 4:2:0, 4:2:2, 4:4:4 + pub profile: Option, + + /// AV1 level-idx. + /// + /// Indicates the highest AV1 level that may have been used to generate the + /// bitstream or that the receiver supports. The range of possible values + /// is identical to the seq_level_idx syntax element specified in AV1. If + /// the parameter is not present, it MUST be inferred to be 5 (level 3.1). + pub level_idx: Option, + + /// AV1 tier. + /// + /// Indicates the highest tier that may have been used to generate the bitstream + /// or that the receiver supports. The range of possible values is identical + /// to the seq_tier syntax element specified in AV1. If the parameter is not + /// present, the tier MUST be inferred to be 0. + pub tier: Option, + + /// H.265/HEVC profile, tier, and level. + /// + /// Contains: + /// * profile: Main, Main10, Main Still Picture, etc. + /// * tier: Main or High. + /// * level: 3.1, 4.0, 5.0, etc. + /// + /// See ITU-T H.265 Annex A for complete definitions. + pub h265_profile_tier_level: Option, + + /// H.266/VVC profile, tier, and level. + /// + /// Contains: + /// * profile: Main 10, Main 10 4:4:4, etc. + /// * tier: Main or High. + /// * level: 3.1, 4.0, 5.0, etc. + /// + /// See ITU-T H.266 Annex A and RFC 9328 §7.2 for complete definitions. + pub h266_profile_tier_level: Option, + + /// H.265 sprop-max-don-diff parameter (RFC 7798 §7.1). + /// When > 0, DONL fields are included in H.265 RTP packets to support + /// out-of-order NAL unit decoding. + pub sprop_max_don_diff: Option, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(FormatParams); + +impl FormatParams { + /// Parse an fmtp line to create a FormatParams. + /// + /// Example `minptime=10;useinbandfec=1`. + pub fn parse_line(line: &str) -> Self { + let key_vals: Vec<_> = line + .split(';') + .filter_map(|pair| { + let mut kv = pair.split('='); + match (kv.next(), kv.next()) { + (Some(k), Some(v)) => Some((k.trim().to_string(), v.trim().to_string())), + _ => None, + } + }) + .collect(); + + let mut p = FormatParams::default(); + + for param in FormatParam::parse_pairs(key_vals) { + p.set_param(¶m); + } + + p + } + + pub(crate) fn set_param(&mut self, param: &FormatParam) { + use FormatParam::*; + match param { + MinPTime(v) => self.min_p_time = Some(*v), + Stereo(v) => self.stereo = Some(*v), + SpropStereo(v) => self.sprop_stereo = Some(*v), + UseInbandFec(v) => self.use_inband_fec = Some(*v), + UseDtx(v) => self.use_dtx = Some(*v), + LevelAsymmetryAllowed(v) => self.level_asymmetry_allowed = Some(*v), + PacketizationMode(v) => self.packetization_mode = Some(*v), + ProfileLevelId(v) => self.profile_level_id = Some(*v), + ProfileId(v) => self.profile_id = Some(*v), + Profile(v) => self.profile = Some(*v), + LevelIdx(v) => self.level_idx = Some(*v), + Tier(v) => self.tier = Some(*v), + H265ProfileTierLevel(v) => self.h265_profile_tier_level = Some(*v), + H266ProfileTierLevel(v) => self.h266_profile_tier_level = Some(*v), + SpropMaxDonDiff(v) => self.sprop_max_don_diff = Some(*v), + Apt(_) => {} + Unknown => {} + } + } + + pub(crate) fn to_format_param(self) -> Vec { + use FormatParam::*; + let mut r = Vec::with_capacity(5); + + if let Some(v) = self.min_p_time { + r.push(MinPTime(v)); + } + if let Some(v) = self.stereo { + r.push(Stereo(v)); + } + if let Some(v) = self.sprop_stereo { + r.push(SpropStereo(v)); + } + if let Some(v) = self.use_inband_fec { + r.push(UseInbandFec(v)); + } + if let Some(v) = self.use_dtx { + r.push(UseDtx(v)); + } + if let Some(v) = self.level_asymmetry_allowed { + r.push(LevelAsymmetryAllowed(v)); + } + if let Some(v) = self.packetization_mode { + r.push(PacketizationMode(v)); + } + if let Some(v) = self.profile_level_id { + r.push(ProfileLevelId(v)); + } + if let Some(v) = self.profile_id { + r.push(ProfileId(v)); + } + if let Some(v) = self.profile { + r.push(Profile(v)); + } + if let Some(v) = self.level_idx { + r.push(LevelIdx(v)); + } + if let Some(v) = self.tier { + r.push(Tier(v)); + } + // H.265 Profile/Tier/Level: Keep as composite, it serializes all three params + if let Some(ptl) = self.h265_profile_tier_level { + r.push(H265ProfileTierLevel(ptl)); + } + // H.266 Profile/Tier/Level: serialized as one composite (all three fmtp params). + if let Some(ptl) = self.h266_profile_tier_level { + r.push(H266ProfileTierLevel(ptl)); + } + if let Some(v) = self.sprop_max_don_diff { + r.push(SpropMaxDonDiff(v)); + } + + r + } +} + +impl fmt::Display for FormatParams { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = self + .to_format_param() + .into_iter() + .map(|f| f.to_string()) + .collect::>() + .join(";"); + write!(f, "{s}") + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn sprop_max_don_diff_roundtrip() { + // Test that sprop-max-don-diff parameter round-trips correctly + let f = FormatParams { + sprop_max_don_diff: Some(32), + ..Default::default() + }; + + // Serialize to string + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, "sprop-max-don-diff=32"); + + // Parse back + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.sprop_max_don_diff, Some(32)); + } + + #[test] + fn sprop_stereo_roundtrip() { + let f = FormatParams { + sprop_stereo: Some(true), + ..Default::default() + }; + + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, "sprop-stereo=1"); + + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.sprop_stereo, Some(true)); + } + + #[test] + fn h265_combined_params_roundtrip() { + use crate::packet::H265ProfileTierLevel; + + // Test H.265 with both profile/tier/level and sprop-max-don-diff + let ptl = H265ProfileTierLevel::new(1, 0, 93).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(ptl), + sprop_max_don_diff: Some(32), + ..Default::default() + }; + + // Serialize to string + let fmtp_str = f.to_string(); + assert!(fmtp_str.contains("profile-id=1")); + assert!(fmtp_str.contains("tier-flag=0")); + assert!(fmtp_str.contains("level-id=93")); + assert!(fmtp_str.contains("sprop-max-don-diff=32")); + + // Parse back + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.h265_profile_tier_level, Some(ptl)); + assert_eq!(parsed.sprop_max_don_diff, Some(32)); + } + + #[test] + fn h266_combined_params_roundtrip() { + use crate::packet::H266ProfileTierLevel; + + // Test H.266 with both profile/tier/level and sprop-max-don-diff + let ptl = H266ProfileTierLevel::new(1, 0, 51).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(ptl), + sprop_max_don_diff: Some(32), + ..Default::default() + }; + + // Serialize to string + let fmtp_str = f.to_string(); + assert!(fmtp_str.contains("profile-id=1")); + assert!(fmtp_str.contains("tier-flag=0")); + assert!(fmtp_str.contains("level-id=51")); + assert!(fmtp_str.contains("sprop-max-don-diff=32")); + + // Parse back + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.h266_profile_tier_level, Some(ptl)); + assert_eq!(parsed.sprop_max_don_diff, Some(32)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/format/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/mod.rs new file mode 100644 index 000000000..1ad4631f9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/mod.rs @@ -0,0 +1,29 @@ +//! Media formats and parameters + +// These really don't belong anywhere, but I guess they're kind of related +// to codecs etc. +pub use crate::packet::CodecExtra; +pub use crate::packet::H264CodecExtra; +pub use crate::packet::H265CodecExtra; +pub use crate::packet::H266CodecExtra; +pub use crate::packet::Vp8CodecExtra; +pub use crate::packet::Vp9CodecExtra; +pub use crate::packet::Vp9PacketizerMode; +pub use crate::packet::detect_av1_keyframe; +pub use crate::packet::detect_h264_keyframe; +pub use crate::packet::detect_h265_keyframe; +pub use crate::packet::detect_h266_keyframe; +pub use crate::packet::detect_vp8_keyframe; +pub use crate::packet::detect_vp9_keyframe; +pub use crate::packet::{H265Level, H265Profile, H265ProfileTierLevel, H265Tier}; +pub use crate::packet::{H266Level, H266Profile, H266ProfileTierLevel, H266Tier}; + +mod codec; +mod codec_config; +mod format_params; +mod payload_params; + +pub use codec::{Codec, CodecSpec}; +pub use codec_config::CodecConfig; +pub use format_params::FormatParams; +pub use payload_params::PayloadParams; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/format/payload_params.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/payload_params.rs new file mode 100644 index 000000000..7c4478bf1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/format/payload_params.rs @@ -0,0 +1,1882 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::ops::RangeInclusive; + +use crate::packet::H264ProfileLevel; +use crate::packet::H265ProfileTierLevel; +use crate::packet::H266ProfileTierLevel; +use crate::rtp_::Pt; + +use super::codec::{Codec, CodecSpec}; + +/// Preferred ranges for dynamic payload type allocation. +pub(crate) const PREFERED_RANGES: &[RangeInclusive] = &[ + // Payload identifiers 96–127 are used for payloads defined dynamically during a session. + 96..=127, + // "unassigned" ranged. note that RTCP packet type 207 (XR, Extended Reports) would be + // indistinguishable from RTP payload types 79 with the marker bit set + 80..=95, + 77..=78, + // reserved because RTCP packet types 200–204 would otherwise be indistinguishable + // from RTP payload types 72–76 + // 72..77, + // lol range. + 35..=71, +]; + +pub(crate) trait Claimed { + fn assert_claim_once(&mut self, pt: Pt); + fn is_claimed(&self, pt: Pt) -> bool; + fn find_unclaimed( + &self, + ranges: &[RangeInclusive], + unlocked: &HashSet, + ) -> Option; +} + +impl Claimed for [bool; 128] { + fn assert_claim_once(&mut self, pt: Pt) { + let idx = *pt as usize; + assert!(!self[idx], "Pt locked multiple times: {}", pt); + self[idx] = true; + } + fn is_claimed(&self, pt: Pt) -> bool { + let idx = *pt as usize; + self[idx] + } + fn find_unclaimed( + &self, + ranges: &[RangeInclusive], + unlocked: &HashSet, + ) -> Option { + for range in ranges { + for i in range.clone() { + if !self[i] && !unlocked.contains(&(i as u8).into()) { + let pt: Pt = (i as u8).into(); + return Some(pt); + } + } + } + + // Failed to find unclaimed PT. + None + } +} + +/// Group of parameters for a payload type (PT). +/// +/// In the SDP a payload type has a number of associated parameters. See example below: +/// +/// ```text +/// a=rtpmap:96 H264/90000 +/// a=rtcp-fb:96 goog-remb +/// a=rtcp-fb:96 transport-cc +/// a=rtcp-fb:96 ccm fir +/// a=rtcp-fb:96 nack +/// a=rtcp-fb:96 nack pli +/// a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +/// ``` +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct PayloadParams { + /// The payload type that groups these parameters. + pub(crate) pt: Pt, + + /// Whether these parameters are repairing some other set of parameters. + /// This is used to, via PT, separate RTX resend streams from the main stream. + pub(crate) resend: Option, + + /// The codec with settings for this group of parameters. + pub(crate) spec: CodecSpec, + + /// Whether the payload use the TWCC feedback mechanic. + pub(crate) fb_transport_cc: bool, + + /// Whether the payload uses NACK to request resends. + pub(crate) fb_nack: bool, + + /// Whether the payload uses the PLI (Picture Loss Indication) mechanic. + pub(crate) fb_pli: bool, + + /// Whether the payload uses the FIR (Full Intra Request) mechanic. + pub(crate) fb_fir: bool, + + /// Whether the payload uses the REMB (Receiver Estimated Maximum Bitrate) mechanic. + pub(crate) fb_remb: bool, + + /// Whether the payload is locked by negotiation or can still be debated. + /// + /// If we make an OFFER or ANSWER and the direction is sendrecv/recvonly, the parameters are locked + /// can't be further changed. If we make an OFFER for a sendonly, the parameters are only proposed + /// and don't lock. + pub(crate) locked: bool, +} + +// we don't want to compare "locked" +impl PartialEq for PayloadParams { + fn eq(&self, other: &Self) -> bool { + self.pt == other.pt + && self.resend == other.resend + && self.spec == other.spec + && self.fb_transport_cc == other.fb_transport_cc + && self.fb_nack == other.fb_nack + && self.fb_pli == other.fb_pli + && self.fb_fir == other.fb_fir + && self.fb_remb == other.fb_remb + } +} + +impl Eq for PayloadParams {} + +impl PayloadParams { + /// Maximum match score. + /// + /// This is returned for an exact `CodecSpec` match in `match_score()`. It is also used as the + /// starting point for codec-specific scoring in this file: + /// - Opus/H.264/H.265 decrement from a perfect match. + /// - VP9/AV1 return the max score when their relevant fmtp parameters match. + /// - VP8 currently only matches via the exact `CodecSpec` equality fast-path (no fuzzy scorer). + const EXACT_MATCH_SCORE: usize = 100; + + /// Score used for H.265 profile-only compatibility matches. + /// + /// Some endpoints (notably browsers) may signal H.265 using only `profile-id` without + /// tier/level information; when we can only validate profile equality we return this lower + /// score so full ProfileTierLevel matches win when available. + const PROFILE_ONLY_MATCH_SCORE: usize = 90; + + /// Creates new payload params. + /// + /// * `pt` is the payload type RTP mapping in the session. + /// * `resend` is the payload type used for (RTX) resend channel. + /// * `spec` configures details about the codec. + pub fn new(pt: Pt, resend: Option, spec: CodecSpec) -> Self { + let is_video = spec.codec.is_video(); + + PayloadParams { + pt, + resend, + + spec, + + // Both audio and video use TWCC + fb_transport_cc: true, + + // Only true for video. + fb_fir: is_video, + fb_nack: is_video, + fb_pli: is_video, + fb_remb: is_video, + + locked: false, + } + } + + /// Creates minimal payload params for SSRC 0 BWE probes. + /// + /// These probes don't carry real media, only padding for bandwidth estimation. + /// Uses 90kHz clock rate (video rate) and enables only transport_cc feedback. + pub(crate) fn new_probe(pt: Pt) -> Self { + use super::codec::Codec; + use super::format_params::FormatParams; + use crate::rtp_::Frequency; + + PayloadParams { + pt, + resend: None, + spec: CodecSpec { + codec: Codec::Null, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + fb_transport_cc: true, + fb_nack: false, + fb_pli: false, + fb_fir: false, + fb_remb: false, + locked: false, + } + } + + /// The payload type that groups these parameters. + pub fn pt(&self) -> Pt { + self.pt + } + + /// Whether these parameters are repairing some other set of parameters. + /// This is used to, via PT, separate RTX resend streams from the main stream. + pub fn resend(&self) -> Option { + self.resend + } + + /// The codec with settings for this group of parameters. + pub fn spec(&self) -> CodecSpec { + self.spec + } + + /// Sets whether the payload use the TWCC feedback mechanic. + pub fn set_fb_transport_cc(&mut self, fb_transport_cc: bool) { + self.fb_transport_cc = fb_transport_cc + } + + /// Whether the payload use the TWCC feedback mechanic. + pub fn fb_transport_cc(&self) -> bool { + self.fb_transport_cc + } + + /// Sets whether the payload uses NACK to request resends. + pub fn set_fb_nack(&mut self, fb_nack: bool) { + self.fb_nack = fb_nack + } + + /// Whether the payload uses NACK to request resends. + pub fn fb_nack(&self) -> bool { + self.fb_nack + } + + /// Set whether the payload uses the PLI (Picture Loss Indication) mechanic. + pub fn set_fb_pli(&mut self, fb_pli: bool) { + self.fb_pli = fb_pli + } + + /// Whether the payload uses the PLI (Picture Loss Indication) mechanic. + pub fn fb_pli(&self) -> bool { + self.fb_pli + } + + /// Set whether the payload uses the FIR (Full Intra Request) mechanic. + pub fn set_fb_fir(&mut self, fb_fir: bool) { + self.fb_fir = fb_fir + } + + /// Whether the payload uses the FIR (Full Intra Request) mechanic. + pub fn fb_fir(&self) -> bool { + self.fb_fir + } + + /// Set whether the payload uses the REMB (Receiver Estimated Maximum Bitrate) mechanic. + pub fn set_fb_remb(&mut self, fb_remb: bool) { + self.fb_remb = fb_remb + } + + /// Whether the payload uses the REMB (Receiver Estimated Maximum Bitrate) mechanic. + pub fn fb_remb(&self) -> bool { + self.fb_remb + } + + pub(crate) fn match_score(&self, o: &PayloadParams) -> Option { + // we don't want to compare PT + let c0 = self.spec; + let c1 = o.spec; + + if c0 == c1 { + // Exact match + return Some(Self::EXACT_MATCH_SCORE); + } + + // Attempt fuzzy matching, since we don't have an exact match + if c0.codec != c1.codec || (c0.codec == Codec::Unknown && c1.codec == Codec::Unknown) { + // Codecs must match + return None; + } + + if c0.clock_rate != c1.clock_rate { + // Clock rates must match + return None; + } + + if c0.channels != c1.channels { + // Channels must match + return None; + } + + if c0.codec.is_audio() && c0.codec == Codec::Opus { + return Some(Self::match_opus_score(c0, c1)); + } + + if c0.codec == Codec::H264 { + return Self::match_h264_score(c0, c1); + } + + if c0.codec == Codec::H265 { + return Self::match_h265_score(c0, c1); + } + + if c0.codec == Codec::H266 { + return Self::match_h266_score(c0, c1); + } + + if c0.codec == Codec::Vp9 { + return Self::match_vp9_score(c0, c1); + } + + if c0.codec == Codec::Av1 { + return Self::match_av1_score(c0, c1); + } + + // TODO: Fuzzy matching for any other audio codecs + // TODO: Fuzzy matching for video + + None + } + + fn match_opus_score(c0: CodecSpec, c1: CodecSpec) -> usize { + let mut score: usize = Self::EXACT_MATCH_SCORE; + + // If neither value is specified both sides should assume the default value, 3. + let either_p_time_specified = + c0.format.min_p_time.is_some() || c1.format.min_p_time.is_some(); + if either_p_time_specified && c0.format.min_p_time != c1.format.min_p_time { + score = score.saturating_sub(1); + } + + // If neither value is specified both sides should assume FEC is not used as this is + // the default. + let either_fec_specified = + c0.format.use_inband_fec.is_some() || c1.format.use_inband_fec.is_some(); + + if either_fec_specified && c0.format.use_inband_fec != c1.format.use_inband_fec { + score = score.saturating_sub(2); + } + + // If neither value is specified both sides should assume DTX is not used as this is + // the default. + let either_dtx_specified = c0.format.use_dtx.is_some() || c1.format.use_dtx.is_some(); + if either_dtx_specified && c0.format.use_dtx != c1.format.use_dtx { + score = score.saturating_sub(4); + } + + // If neither value is specified both sides should assume mono for stereo + // and the sprop-stereo hint defaults to false. + let c0_stereo = c0.format.stereo.unwrap_or(false); + let c1_stereo = c1.format.stereo.unwrap_or(false); + if c0_stereo != c1_stereo { + score = score.saturating_sub(8); + } + + let c0_sprop_stereo = c0.format.sprop_stereo.unwrap_or(false); + let c1_sprop_stereo = c1.format.sprop_stereo.unwrap_or(false); + if c0_sprop_stereo != c1_sprop_stereo { + score = score.saturating_sub(8); + } + + score + } + + fn match_vp9_score(c0: CodecSpec, c1: CodecSpec) -> Option { + // Default profile_id is 0. https://datatracker.ietf.org/doc/html/draft-ietf-payload-vp9-16#section-6 + let c0_profile_id = c0.format.profile_id.unwrap_or(0); + let c1_profile_id = c1.format.profile_id.unwrap_or(0); + + if c0_profile_id != c1_profile_id { + return None; + } + + Some(Self::EXACT_MATCH_SCORE) + } + + fn match_av1_score(c0: CodecSpec, c1: CodecSpec) -> Option { + // TODO: consider media direction for a proper less or equal matching + // The AV1 stream sent by either the offerer or the answerer MUST be + // encoded with a profile, level and tier, lesser or equal to the values + // of the level-idx, profile and tier declared in the SDP by the receiving + // agent. + // https://aomediacodec.github.io/av1-rtp-spec/#723-usage-with-the-sdp-offeranswer-model + + // Default values: profile = 0, level-idx = 5, tier = 0 + // https://aomediacodec.github.io/av1-rtp-spec/#72-sdp-parameters + let c0_profile = c0.format.profile.unwrap_or(0); + let c1_profile = c1.format.profile.unwrap_or(0); + if c0_profile != c1_profile { + return None; + } + + let c0_level_idx = c0.format.level_idx.unwrap_or(5); + let c1_level_idx = c1.format.level_idx.unwrap_or(5); + if c0_level_idx != c1_level_idx { + return None; + } + + let c0_tier = c0.format.tier.unwrap_or(0); + let c1_tier = c1.format.tier.unwrap_or(0); + if c0_tier != c1_tier { + return None; + } + + Some(Self::EXACT_MATCH_SCORE) + } + + pub(crate) fn match_h264_score(c0: CodecSpec, c1: CodecSpec) -> Option { + // Default packetization mode is 0. https://www.rfc-editor.org/rfc/rfc6184#section-6.2 + let c0_packetization_mode = c0.format.packetization_mode.unwrap_or(0); + let c1_packetization_mode = c1.format.packetization_mode.unwrap_or(0); + + if c0_packetization_mode != c1_packetization_mode { + return None; + } + + let c0_profile_level = c0 + .format + .profile_level_id + .map(|l| l.try_into().ok()) + .unwrap_or(Some(H264ProfileLevel::FALLBACK))?; + let c1_profile_level = c1 + .format + .profile_level_id + .map(|l| l.try_into().ok()) + .unwrap_or(Some(H264ProfileLevel::FALLBACK))?; + + // RFC 6184 Section 8.2.2: Profiles must match exactly. + if c0_profile_level.profile() != c1_profile_level.profile() { + return None; + } + + // RFC 6184 Section 8.2.2: the level is a negotiable (downgradable) + // parameter, so a level mismatch in either direction is allowed and + // only penalizes the match score rather than rejecting the payload. + // + // NOTE: this only affects *matching*. Unlike H.265/H.266, `update_param` + // does not narrow the answered H.264 level to `min(local, remote)`, so + // when `level-asymmetry-allowed` is absent we may answer with a level + // higher than the offer. See the note in `update_param`. + let level_difference: usize = c0_profile_level + .level() + .ordinal() + .abs_diff(c1_profile_level.level().ordinal()); + + Some(Self::EXACT_MATCH_SCORE.saturating_sub(level_difference)) + } + + /// Match H.265 codec specifications and return compatibility score. + /// + /// # Matching Rules — Full PTL (both sides have profile+tier+level) + /// + /// - **Profiles** must match exactly (no cross-profile compatibility) + /// - **Tiers** must match exactly (Main vs High tier are distinct) + /// - **Levels** penalize score by `|c0_level - c1_level|` but never reject. + /// Both H.264 and H.265 tolerate level mismatches in either direction at + /// match time. For H.265, `update_param` additionally narrows the + /// negotiated level to `min(local, remote)` afterward; H.264 does not. + /// + /// # Matching Rules — Profile-only (at least one side lacks tier+level) + /// + /// - **Profiles** must match; missing profile info falls back to + /// `FALLBACK.profile()` (not rejected). + /// - **Tiers and levels** are not checked (insufficient information). + /// - Returns a lower score (90) so full-PTL matches are preferred. + /// + /// # Returns + /// + /// - `Some(100)` for exact full-PTL match + /// - `Some(100 - level_gap)` for profile/tier match with level difference + /// - `Some(90)` for profile-only match (when tier/level unavailable) + /// - `None` for profile or tier mismatch + pub(crate) fn match_h265_score(c0: CodecSpec, c1: CodecSpec) -> Option { + match ( + c0.format.h265_profile_tier_level, + c1.format.h265_profile_tier_level, + ) { + (Some(c0_ptl), Some(c1_ptl)) => { + // Both have full PTL - strict matching + + // Profiles must match exactly. + // RFC 7798 §7.2.2: profile-id is a configuration parameter + // that MUST be used symmetrically in offer/answer. + // https://www.rfc-editor.org/rfc/rfc7798#section-7.2.2 + if c0_ptl.profile() != c1_ptl.profile() { + return None; + } + + // Tiers must match exactly. + // RFC 7798 §7.2.2: tier-flag is a configuration parameter + // that MUST be used symmetrically in offer/answer. + // https://www.rfc-editor.org/rfc/rfc7798#section-7.2.2 + if c0_ptl.tier() != c1_ptl.tier() { + return None; + } + + // Level difference penalizes score but never causes rejection. + // Actual level narrowing to min(local, remote) is done in update_param. + let level_difference: usize = + c0_ptl.level().ordinal().abs_diff(c1_ptl.level().ordinal()); + + // Pure scoring without artificial floor - let caller decide policy + Some(Self::EXACT_MATCH_SCORE.saturating_sub(level_difference)) + } + _ => { + // At least one side has only profile-id (not full PTL). + // Match on profile only, using FALLBACK for missing values. + + // Get profile from PTL if present, otherwise from profile_id field + let c0_profile = c0 + .format + .h265_profile_tier_level + .map(|ptl| ptl.profile().to_id()) + .or_else(|| c0.format.profile_id.map(|p| p as u8)) + .unwrap_or(H265ProfileTierLevel::FALLBACK.profile().to_id()); + + let c1_profile = c1 + .format + .h265_profile_tier_level + .map(|ptl| ptl.profile().to_id()) + .or_else(|| c1.format.profile_id.map(|p| p as u8)) + .unwrap_or(H265ProfileTierLevel::FALLBACK.profile().to_id()); + + // Profiles must match + if c0_profile != c1_profile { + return None; + } + + // When only profile is specified, we can't verify tier/level compatibility. + // Return a lower score to prefer exact PTL matches when available. + Some(Self::PROFILE_ONLY_MATCH_SCORE) + } + } + } + + pub(crate) fn match_h266_score(c0: CodecSpec, c1: CodecSpec) -> Option { + match ( + c0.format.h266_profile_tier_level, + c1.format.h266_profile_tier_level, + ) { + (Some(c0_ptl), Some(c1_ptl)) => { + // Both have full PTL - strict matching + + // Profiles must match exactly. + // RFC 9328 §7.2: profile-id is a configuration parameter + // that MUST be used symmetrically in offer/answer. + if c0_ptl.profile() != c1_ptl.profile() { + return None; + } + + // Tiers must match exactly. + // RFC 9328 §7.2: tier-flag is a configuration parameter + // that MUST be used symmetrically in offer/answer. + if c0_ptl.tier() != c1_ptl.tier() { + return None; + } + + // Level difference penalizes score but never causes rejection. + let level_difference: usize = + c0_ptl.level().ordinal().abs_diff(c1_ptl.level().ordinal()); + + Some(Self::EXACT_MATCH_SCORE.saturating_sub(level_difference)) + } + _ => { + // At least one side has no full PTL (e.g. a GStreamer offer + // with a bare rtpmap and no fmtp). Match on profile only, + // using FALLBACK for missing values. + let c0_profile = c0 + .format + .h266_profile_tier_level + .map(|ptl| ptl.profile().to_id()) + .or_else(|| c0.format.profile_id.map(|p| p as u8)) + .unwrap_or(H266ProfileTierLevel::FALLBACK.profile().to_id()); + + let c1_profile = c1 + .format + .h266_profile_tier_level + .map(|ptl| ptl.profile().to_id()) + .or_else(|| c1.format.profile_id.map(|p| p as u8)) + .unwrap_or(H266ProfileTierLevel::FALLBACK.profile().to_id()); + + // Profiles must match + if c0_profile != c1_profile { + return None; + } + + Some(Self::PROFILE_ONLY_MATCH_SCORE) + } + } + } + + pub(crate) fn update_param( + &mut self, + remote_pts: &[PayloadParams], + claimed: &mut [bool; 128], + local_is_controlling: bool, + unlocked: &mut HashSet, + ) { + let Some((first, _)) = remote_pts + .iter() + .filter_map(|p| self.match_score(p).map(|s| (p, s))) + .max_by_key(|(_, s)| *s) + else { + return; + }; + + // Mirror the remote's H.265 fmtp shape: echo back only the params they offered. + if self.spec.codec == Codec::H265 && first.spec.codec == Codec::H265 { + if let Some(remote_ptl) = first.spec.format.h265_profile_tier_level { + // Narrow level to min(local, remote) so the negotiated level + // never exceeds either side's capability. + let negotiated_ptl = + if let Some(local_ptl) = self.spec.format.h265_profile_tier_level { + remote_ptl.with_level(std::cmp::min(local_ptl.level(), remote_ptl.level())) + } else { + remote_ptl + }; + self.spec.format.h265_profile_tier_level = Some(negotiated_ptl); + // Avoid also serializing `profile-id` via the VP9 `profile_id` field. + self.spec.format.profile_id = None; + } else if let Some(profile_id) = first.spec.format.profile_id { + // Remote only offered `profile-id`. + self.spec.format.h265_profile_tier_level = None; + self.spec.format.profile_id = Some(profile_id); + } else { + // No remote H.265 fmtp, omit ours as well. + self.spec.format.h265_profile_tier_level = None; + self.spec.format.profile_id = None; + } + + // Adopt remote's sprop-max-don-diff so the depacketizer knows whether + // incoming packets contain DONL fields (RFC 7798 §7.1). + // The packetizer also uses this to decide whether to emit DONL. + if first.spec.format.sprop_max_don_diff.is_some() { + self.spec.format.sprop_max_don_diff = first.spec.format.sprop_max_don_diff; + } + } + + // Same negotiation for H.266 (RFC 9328 §7.2, offer/answer model §7.3.2). + if self.spec.codec == Codec::H266 && first.spec.codec == Codec::H266 { + if let Some(remote_ptl) = first.spec.format.h266_profile_tier_level { + // Narrow level to min(local, remote) so the negotiated level + // never exceeds either side's capability. + let negotiated_ptl = + if let Some(local_ptl) = self.spec.format.h266_profile_tier_level { + remote_ptl.with_level(std::cmp::min(local_ptl.level(), remote_ptl.level())) + } else { + remote_ptl + }; + self.spec.format.h266_profile_tier_level = Some(negotiated_ptl); + // Avoid also serializing `profile-id` via the VP9 `profile_id` field. + self.spec.format.profile_id = None; + } else if let Some(profile_id) = first.spec.format.profile_id { + // Remote only offered `profile-id`. + self.spec.format.h266_profile_tier_level = None; + self.spec.format.profile_id = Some(profile_id); + } else { + // No remote H.266 fmtp, omit ours as well. + self.spec.format.h266_profile_tier_level = None; + self.spec.format.profile_id = None; + } + + // Adopt remote's sprop-max-don-diff (RFC 9328 §7.2) — enables + // DONL on both packetizer and depacketizer. + if first.spec.format.sprop_max_don_diff.is_some() { + self.spec.format.sprop_max_don_diff = first.spec.format.sprop_max_don_diff; + } + } + + // KNOWN LIMITATION: H.264 has no equivalent level-narrowing pass here. + // `match_h264_score` treats the level as negotiable, but we keep our + // locally configured `profile-level-id` in the answer instead of + // narrowing it to `min(local, remote)`. Per RFC 6184 §8.2.2, when + // `level-asymmetry-allowed` is not set the answer should use the lower + // level. + + let mut remote_pt = first.pt; + let mut remote_rtx = first.resend; + + if self.locked { + // This can happen if the incoming PTs are suggestions (send-direction) rather than demanded + // (receive-direction). We only want to warn if we get receive direction changes. + if local_is_controlling { + return; + } + // Just verify it's still the same. We should validate this in apply_offer/answer instead + // of ever seeing this error message. + if self.pt != remote_pt { + warn!("Ignore remote PT change {} => {}", self.pt, remote_pt); + } + + if self.resend != remote_rtx { + warn!( + "Ignore remote PT RTX change {:?} => {:?}", + self.resend, remote_rtx + ); + } + } else { + // Before locking, check if the remote PT conflicts with an already locked PT. + // If we're receiving, we control what PTs to use, + // so we can remap the conflicting PT. + if local_is_controlling && claimed.is_claimed(remote_pt) { + // Prefer our configured PT, then fall back to any free PT. + let new_pt = (!claimed.is_claimed(self.pt)) + .then_some(self.pt) + .or_else(|| claimed.find_unclaimed(PREFERED_RANGES, unlocked)); + + if let Some(new_pt) = new_pt { + debug!( + "Remapped conflicting PT {} => {} for codec {:?}", + remote_pt, new_pt, self.spec.codec + ); + remote_pt = new_pt; + unlocked.remove(&new_pt); + } else { + panic!("Exhausted all PT ranges, inconsistent PayloadParam state"); + } + } + + // Lock down the PT + self.pt = remote_pt; + claimed.assert_claim_once(remote_pt); + + // Check if RTX PT also conflicts + if local_is_controlling { + if let Some(rtx) = remote_rtx { + if claimed.is_claimed(rtx) { + let new_rtx = self + .resend + .filter(|pt| !claimed.is_claimed(*pt)) + .or_else(|| claimed.find_unclaimed(PREFERED_RANGES, unlocked)); + + if let Some(new_rtx) = new_rtx { + debug!("Remapped conflicting RTX PT {:?} => {}", rtx, new_rtx); + remote_rtx = Some(new_rtx); + unlocked.remove(&new_rtx); + } else { + panic!("Exhausted all PT ranges, inconsistent PayloadParam state"); + } + } + } + } + + // Lock down the RTX PT + self.resend = remote_rtx; + if let Some(rtx) = remote_rtx { + claimed.assert_claim_once(rtx); + } + + // This is now locked. + self.locked = true; + } + } +} + +#[cfg(test)] +mod test { + use crate::rtp_::Frequency; + + use super::*; + use crate::format::{CodecSpec, FormatParams}; + + mod h264 { + use super::*; + + fn h264_codec_spec( + level_asymmetry_allowed: Option, + packetization_mode: Option, + profile_level_id: Option, + ) -> CodecSpec { + CodecSpec { + codec: Codec::H264, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + level_asymmetry_allowed, + packetization_mode, + profile_level_id, + ..Default::default() + }, + } + } + + #[test] + fn test_h264_profile_matching() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + must_match: bool, + msg: &'static str, + } + + let cases = [ + Case { + c0: h264_codec_spec(None, None, Some(0x42E01F)), + c1: h264_codec_spec(None, None, Some(0x4DA01F)), + must_match: true, + msg: "0x42A01F and 0x4DF01F should match, they are both constrained baseline subprofile", + }, + Case { + c0: h264_codec_spec(None, None, Some(0x42E01F)), + c1: h264_codec_spec(None, Some(1), Some(0x4DA01F)), + must_match: false, + msg: "0x42A01F and 0x4DF01F with differing packetization modes should not match", + }, + Case { + c0: h264_codec_spec(None, Some(0), Some(0x422000)), + c1: h264_codec_spec(None, None, Some(0x42B00A)), + must_match: true, + msg: "0x424000 and 0x42B00A should match because they are both the baseline subprofile \ + and the level idc of 0x42F01F will be adjusted to Level1B because the constraint \ + set 3 flag is set", + }, + ]; + + for Case { + c0, + c1, + must_match, + msg, + } in cases.into_iter() + { + let matched = PayloadParams::match_h264_score(c0, c1).is_some(); + assert_eq!(matched, must_match, "{msg}\nc0: {c0:#?}\nc1: {c1:#?}"); + } + } + + #[test] + fn test_h264_level_matching_rfc_compliant() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + expected: Option, + msg: &'static str, + } + + let cases = [ + // Test 1: Same profile, same level -> should match + Case { + c0: h264_codec_spec(None, None, Some(0x42e028)), // CB L4.0 + c1: h264_codec_spec(None, None, Some(0x42e028)), // CB L4.0 + expected: Some(PayloadParams::EXACT_MATCH_SCORE), + msg: "Same profile (CB) and same level (4.0) should match", + }, + // Test 2: Same profile, offered level lower -> should match (RFC 6184) + Case { + c0: h264_codec_spec(None, None, Some(0x42e028)), // CB L4.0 (configured) + c1: h264_codec_spec(None, None, Some(0x42e01f)), // CB L3.1 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 2), + msg: "Same profile (CB), offered level 3.1 < configured 4.0 should match per RFC 6184", + }, + // Test 3: Same profile, offered level higher -> should match (RFC 6184) + Case { + c0: h264_codec_spec(None, None, Some(0x42e01f)), // CB L3.1 (configured) + c1: h264_codec_spec(None, None, Some(0x42e028)), // CB L4.0 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 2), + msg: "Same profile (CB), offered level 4.0 > configured 3.1 should match, \ + the level is negotiable per RFC 6184", + }, + // Test 4: Different profiles -> should NOT match + Case { + c0: h264_codec_spec(None, None, Some(0x42e028)), // CB L4.0 + c1: h264_codec_spec(None, None, Some(0x4d0028)), // Main L4.0 + expected: None, + msg: "Different profiles (CB vs Main) should NOT match even with same level", + }, + // Test 5: Main profile, offered lower level -> should match + Case { + c0: h264_codec_spec(None, None, Some(0x4d002a)), // Main L4.2 (configured) + c1: h264_codec_spec(None, None, Some(0x4d0028)), // Main L4.0 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 2), + msg: "Same profile (Main), offered level 4.0 < configured 4.2 should match", + }, + // Test 6: High profile, multiple levels down -> should match + Case { + c0: h264_codec_spec(None, None, Some(0x640033)), // High L5.1 (configured) + c1: h264_codec_spec(None, None, Some(0x64001f)), // High L3.1 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 6), + msg: "Same profile (High), offered level 3.1 < configured 5.1 should match", + }, + // Test 7: Baseline (not CB) with Level1B as offered + Case { + c0: h264_codec_spec(None, None, Some(0x42001f)), // Baseline L3.1 + c1: h264_codec_spec(None, None, Some(0x420000)), // Baseline Level1B + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 8), + msg: "Same profile (Baseline), offered level 1B < configured 3.1 should match", + }, + // Test 8: Baseline (not CB) offered lower level > should match with (special Level1B case) + Case { + c0: h264_codec_spec(None, None, Some(0x420000)), // Baseline Level1B + c1: h264_codec_spec(None, None, Some(0x42000a)), // Baseline L1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "Same profile (Baseline), offered level 1 < configured 1B should match", + }, + // Test 9: Baseline (not CB) offered higher level -> should match (special Level1B case) + Case { + c0: h264_codec_spec(None, None, Some(0x42000a)), // Baseline L1 + c1: h264_codec_spec(None, None, Some(0x420000)), // Baseline Level1B + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "Same profile (Baseline), offered level 1B > configured 1 should match", + }, + // Test 10: Offered Constrained Baseline level 5.2 is higher than + // the default configured 3.1. Must match. + Case { + c0: h264_codec_spec(Some(true), Some(1), Some(0x42e01f)), // CB L3.1 (configured) + c1: h264_codec_spec(Some(true), Some(1), Some(0x42e034)), // CB L5.2 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 7), + msg: "Same profile (CB), offered level 5.2 > configured 3.1 should match", + }, + // Test 11: Constrained High (profile-iop 0x0C) must be recognized + // and match a configured Constrained High. + Case { + c0: h264_codec_spec(Some(true), Some(1), Some(0x640c1f)), // CH L3.1 (configured) + c1: h264_codec_spec(Some(true), Some(1), Some(0x640c34)), // CH L5.2 (offered) + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 7), + msg: "Same profile (Constrained High), offered level 5.2 should match", + }, + // Test 12: Constrained High is not the same profile as High. + Case { + c0: h264_codec_spec(None, None, Some(0x64001f)), // High L3.1 + c1: h264_codec_spec(None, None, Some(0x640c1f)), // Constrained High L3.1 + expected: None, + msg: "Different profiles (High vs Constrained High) should NOT match", + }, + ]; + + for Case { + c0, + c1, + expected, + msg, + } in cases.into_iter() + { + assert_eq!( + PayloadParams::match_h264_score(c0, c1), + expected, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + } + + mod h265 { + use super::*; + use crate::packet::Packetizer; + + fn h265_codec_spec(profile_id: u8, tier_flag: u8, level_id: u8) -> CodecSpec { + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h265_profile_tier_level: Some( + H265ProfileTierLevel::new(profile_id, tier_flag, level_id).unwrap(), + ), + ..Default::default() + }, + } + } + + fn h265_codec_spec_profile_only(profile_id: u32) -> CodecSpec { + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + profile_id: Some(profile_id), + ..Default::default() + }, + } + } + + /// Test basic H.265 profile/tier/level matching scenarios. + /// Verifies that full ProfileTierLevel parameters are matched correctly. + #[test] + fn test_profile_tier_level_matching() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + must_match: bool, + msg: &'static str, + } + + let cases = [ + // Same profile, tier, level -> should match + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 0, 93), + must_match: true, + msg: "Same profile (Main), tier (Main), level (3.1) should match", + }, + // Same profile and tier, offered level lower -> should match + Case { + c0: h265_codec_spec(1, 0, 120), // Main, Main tier, Level 4.0 + c1: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + must_match: true, + msg: "Same profile/tier, offered level 3.1 < configured 4.0 should match", + }, + // Same profile and tier, offered level higher -> should still match (with penalty) + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 0, 120), // Main, Main tier, Level 4.0 + must_match: true, + msg: "Same profile/tier, offered level 4.0 > configured 3.1 should match (penalized)", + }, + // Different profiles -> should NOT match + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(2, 0, 93), // Main10, Main tier, Level 3.1 + must_match: false, + msg: "Different profiles (Main vs Main10) should NOT match", + }, + // Different tiers -> should NOT match + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 1, 93), // Main, High tier, Level 3.1 + must_match: false, + msg: "Different tiers (Main vs High) should NOT match", + }, + // Main10 profile, same tier, lower level -> should match + Case { + c0: h265_codec_spec(2, 0, 153), // Main10, Main tier, Level 5.1 + c1: h265_codec_spec(2, 0, 120), // Main10, Main tier, Level 4.0 + must_match: true, + msg: "Main10 profile, offered level 4.0 < configured 5.1 should match", + }, + // High tier, multiple levels down -> should match + Case { + c0: h265_codec_spec(1, 1, 153), // Main, High tier, Level 5.1 + c1: h265_codec_spec(1, 1, 93), // Main, High tier, Level 3.1 + must_match: true, + msg: "High tier, offered level 3.1 < configured 5.1 should match", + }, + ]; + + for Case { + c0, + c1, + must_match, + msg, + } in cases.into_iter() + { + let matched = PayloadParams::match_h265_score(c0, c1).is_some(); + assert_eq!(matched, must_match, "{msg}\nc0: {c0:#?}\nc1: {c1:#?}"); + } + } + + /// Test H.265 level matching with exact scores. + /// Verifies that the score decrements based on level difference. + #[test] + fn test_level_matching_scores() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + expected: Option, + msg: &'static str, + } + + let cases = [ + // Exact match -> score 100 + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 0, 93), + expected: Some(PayloadParams::EXACT_MATCH_SCORE), + msg: "Exact match should return score 100", + }, + // One level down -> score 100 - 1 + Case { + c0: h265_codec_spec(1, 0, 120), // Main, Main tier, Level 4.0 + c1: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "One level difference should return score 99", + }, + // Two levels down -> score 100 - 2 + Case { + c0: h265_codec_spec(1, 0, 123), // Main, Main tier, Level 4.1 + c1: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 2), + msg: "Two level difference should return score 98", + }, + // Multiple levels down + Case { + c0: h265_codec_spec(1, 0, 153), // Main, Main tier, Level 5.1 + c1: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 4), + msg: "Four level difference (5.1 to 3.1) should return score 96", + }, + // Offered level higher -> still matches, penalized by 1 ordinal + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 0, 120), // Main, Main tier, Level 4.0 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "Offered level one higher than configured should return score 99", + }, + // Different profiles -> None + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(2, 0, 93), // Main10, Main tier, Level 3.1 + expected: None, + msg: "Different profiles should return None", + }, + // Different tiers -> None + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 1, 93), // Main, High tier, Level 3.1 + expected: None, + msg: "Different tiers should return None", + }, + ]; + + for Case { + c0, + c1, + expected, + msg, + } in cases.into_iter() + { + assert_eq!( + PayloadParams::match_h265_score(c0, c1), + expected, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test H.265 profile-only matching (for Chrome compatibility). + /// When only profile-id is provided without tier/level, match on profile only. + #[test] + fn test_profile_only_matching() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + expected: Option, + msg: &'static str, + } + + let cases = [ + // Both have profile-only, same profile -> score 90 + Case { + c0: h265_codec_spec_profile_only(1), // Main + c1: h265_codec_spec_profile_only(1), // Main + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Profile-only match (Main) should return score 90", + }, + // Both have profile-only, different profiles -> None + Case { + c0: h265_codec_spec_profile_only(1), // Main + c1: h265_codec_spec_profile_only(2), // Main10 + expected: None, + msg: "Profile-only with different profiles should return None", + }, + // One has full PTL, other has profile-only, same profile -> score 90 + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec_profile_only(1), // Main + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Mixed PTL and profile-only with matching profile should return score 90", + }, + // One has full PTL, other has profile-only, different profiles -> None + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec_profile_only(2), // Main10 + expected: None, + msg: "Mixed PTL and profile-only with different profiles should return None", + }, + // Reverse: profile-only vs full PTL, same profile -> score 90 + Case { + c0: h265_codec_spec_profile_only(1), // Main + c1: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Profile-only vs full PTL with matching profile should return score 90", + }, + ]; + + for Case { + c0, + c1, + expected, + msg, + } in cases.into_iter() + { + assert_eq!( + PayloadParams::match_h265_score(c0, c1), + expected, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test negative cases for H.265 matching to ensure proper rejection. + #[test] + fn test_negative_cases() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + msg: &'static str, + } + + let cases = [ + // Profile mismatch with full PTL + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(2, 0, 93), // Main10, Main tier, Level 3.1 + msg: "Profile mismatch (Main vs Main10) must not match", + }, + // Tier mismatch + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(1, 1, 93), // Main, High tier, Level 3.1 + msg: "Tier mismatch (Main vs High) must not match", + }, + // All three different (profile+tier mismatch rejects) + Case { + c0: h265_codec_spec(1, 0, 93), // Main, Main tier, Level 3.1 + c1: h265_codec_spec(2, 1, 153), // Main10, High tier, Level 5.1 + msg: "Profile and tier different must not match", + }, + // Profile-only mismatch + Case { + c0: h265_codec_spec_profile_only(1), // Main + c1: h265_codec_spec_profile_only(2), // Main10 + msg: "Profile-only mismatch must not match", + }, + // Mixed full PTL vs profile-only with different profiles + Case { + c0: h265_codec_spec(2, 0, 93), // Main10, Main tier, Level 3.1 + c1: h265_codec_spec_profile_only(1), // Main + msg: "Full PTL (Main10) vs profile-only (Main) must not match", + }, + ]; + + for Case { c0, c1, msg } in cases.into_iter() { + assert_eq!( + PayloadParams::match_h265_score(c0, c1), + None, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test that sprop-max-don-diff is adopted from remote SDP during negotiation, + /// and that packetizer/depacketizer are correctly initialized with DONL enabled. + #[test] + fn test_sprop_max_don_diff_negotiation_enables_donl() { + use crate::packet::{CodecDepacketizer, CodecPacketizer}; + + // Local config: no sprop-max-don-diff (default) + let local_spec = h265_codec_spec(1, 0, 93); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + assert_eq!( + local_params.spec.format.sprop_max_don_diff, None, + "Local should have no sprop-max-don-diff initially" + ); + + // Remote offers H.265 with sprop-max-don-diff=32 + let mut remote_spec = h265_codec_spec(1, 0, 93); + remote_spec.format.sprop_max_don_diff = Some(32); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + // Simulate SDP negotiation (update_param merges remote into local) + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param( + &[remote_params], + &mut claimed, + false, // remote is controlling (we're answering) + &mut unlocked, + ); + + // Verify sprop-max-don-diff was adopted from remote + assert_eq!( + local_params.spec.format.sprop_max_don_diff, + Some(32), + "sprop-max-don-diff should be adopted from remote after negotiation" + ); + + // Verify packetizer is initialized with DONL enabled + let donl_enabled = local_params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0; + assert!( + donl_enabled, + "DONL should be enabled when sprop-max-don-diff > 0" + ); + + let mut pack = CodecPacketizer::new( + local_params.spec.codec, + crate::format::Vp9PacketizerMode::default(), + ); + if let CodecPacketizer::H265(ref mut h265) = pack { + h265.with_donl(donl_enabled); + } + + // Packetize a NAL — should include DONL field (2 extra bytes) + let nalu = vec![0x02, 0x01, 0xAA, 0xBB]; + let packets = pack.packetize(1200, &nalu).unwrap(); + assert_eq!(packets.len(), 1); + assert_eq!( + packets[0].len(), + nalu.len() + 2, // +2 for DONL + "Packetized output should include 2-byte DONL field" + ); + + // Verify depacketizer is initialized with DONL enabled + let mut depack: CodecDepacketizer = local_params.spec.codec.into(); + if let CodecDepacketizer::H265(ref mut h265) = depack { + h265.with_donl(donl_enabled); + } + + // Depacketize the packet — should correctly strip DONL + use crate::packet::{CodecExtra, Depacketizer}; + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + if let CodecDepacketizer::H265(ref mut h265) = depack { + h265.depacketize(&packets[0], &mut out, &mut extra).unwrap(); + } + + // Output should be Annex-B: start code + original NAL + assert_eq!(&out[0..4], &[0x00, 0x00, 0x00, 0x01]); + assert_eq!( + &out[4..], + &nalu[..], + "Depacketized NAL should match original" + ); + } + + /// Test that sprop-max-don-diff=0 from remote does NOT enable DONL. + #[test] + fn test_sprop_max_don_diff_zero_disables_donl() { + let local_spec = h265_codec_spec(1, 0, 93); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + + // Remote offers with sprop-max-don-diff=0 (no reordering) + let mut remote_spec = h265_codec_spec(1, 0, 93); + remote_spec.format.sprop_max_don_diff = Some(0); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param(&[remote_params], &mut claimed, false, &mut unlocked); + + assert_eq!(local_params.spec.format.sprop_max_don_diff, Some(0)); + let donl_enabled = local_params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0; + assert!( + !donl_enabled, + "DONL should NOT be enabled when sprop-max-don-diff=0" + ); + + // Packetize without DONL — packet should be same size as NAL + let nalu = vec![0x02, 0x01, 0xAA, 0xBB]; + let mut pack = crate::packet::CodecPacketizer::new( + local_params.spec.codec, + crate::format::Vp9PacketizerMode::default(), + ); + // Don't call with_donl(true) since donl_enabled is false + let packets = pack.packetize(1200, &nalu).unwrap(); + assert_eq!( + packets[0].len(), + nalu.len(), + "No DONL field when sprop-max-don-diff=0" + ); + } + + /// Test that missing sprop-max-don-diff from remote preserves local value. + #[test] + fn test_sprop_max_don_diff_absent_preserves_local() { + // Local has sprop-max-don-diff=16 + let mut local_spec = h265_codec_spec(1, 0, 93); + local_spec.format.sprop_max_don_diff = Some(16); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + + // Remote offers without sprop-max-don-diff + let remote_spec = h265_codec_spec(1, 0, 93); + assert!(remote_spec.format.sprop_max_don_diff.is_none()); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param(&[remote_params], &mut claimed, false, &mut unlocked); + + // Local value should be preserved (remote didn't override) + assert_eq!( + local_params.spec.format.sprop_max_don_diff, + Some(16), + "Local sprop-max-don-diff should be preserved when remote doesn't specify it" + ); + } + } + + mod h266 { + use super::*; + use crate::packet::H266ProfileTierLevel; + use crate::packet::Packetizer; + + fn h266_codec_spec(profile_id: u8, tier_flag: u8, level_id: u8) -> CodecSpec { + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h266_profile_tier_level: Some( + H266ProfileTierLevel::new(profile_id, tier_flag, level_id).unwrap(), + ), + ..Default::default() + }, + } + } + + fn h266_codec_spec_profile_only(profile_id: u32) -> CodecSpec { + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + profile_id: Some(profile_id), + ..Default::default() + }, + } + } + + /// Test basic H.266 profile/tier/level matching scenarios. + /// Verifies that full ProfileTierLevel parameters are matched correctly. + #[test] + fn test_profile_tier_level_matching() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + must_match: bool, + msg: &'static str, + } + + let cases = [ + // Same profile, tier, level -> should match + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 0, 51), + must_match: true, + msg: "Same profile (Main), tier (Main), level (3.1) should match", + }, + // Same profile and tier, offered level lower -> should match + Case { + c0: h266_codec_spec(1, 0, 64), // Main, Main tier, Level 4.0 + c1: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + must_match: true, + msg: "Same profile/tier, offered level 3.1 < configured 4.0 should match", + }, + // Same profile and tier, offered level higher -> should still match (with penalty) + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 0, 64), // Main, Main tier, Level 4.0 + must_match: true, + msg: "Same profile/tier, offered level 4.0 > configured 3.1 should match (penalized)", + }, + // Different profiles -> should NOT match + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(33, 0, 51), // Main10, Main tier, Level 3.1 + must_match: false, + msg: "Different profiles (Main vs Main10) should NOT match", + }, + // Different tiers -> should NOT match + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 1, 51), // Main, High tier, Level 3.1 + must_match: false, + msg: "Different tiers (Main vs High) should NOT match", + }, + // Main10 profile, same tier, lower level -> should match + Case { + c0: h266_codec_spec(33, 0, 83), // Main10, Main tier, Level 5.1 + c1: h266_codec_spec(33, 0, 64), // Main10, Main tier, Level 4.0 + must_match: true, + msg: "Main10 profile, offered level 4.0 < configured 5.1 should match", + }, + // High tier, multiple levels down -> should match + Case { + c0: h266_codec_spec(1, 1, 83), // Main, High tier, Level 5.1 + c1: h266_codec_spec(1, 1, 51), // Main, High tier, Level 3.1 + must_match: true, + msg: "High tier, offered level 3.1 < configured 5.1 should match", + }, + ]; + + for Case { + c0, + c1, + must_match, + msg, + } in cases.into_iter() + { + let matched = PayloadParams::match_h266_score(c0, c1).is_some(); + assert_eq!(matched, must_match, "{msg}\nc0: {c0:#?}\nc1: {c1:#?}"); + } + } + + /// Test H.266 level matching with exact scores. + /// Verifies that the score decrements based on level difference. + #[test] + fn test_level_matching_scores() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + expected: Option, + msg: &'static str, + } + + let cases = [ + // Exact match -> score 100 + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 0, 51), + expected: Some(PayloadParams::EXACT_MATCH_SCORE), + msg: "Exact match should return score 100", + }, + // One level down -> score 100 - 1 + Case { + c0: h266_codec_spec(1, 0, 64), // Main, Main tier, Level 4.0 + c1: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "One level difference should return score 99", + }, + // Two levels down -> score 100 - 2 + Case { + c0: h266_codec_spec(1, 0, 67), // Main, Main tier, Level 4.1 + c1: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 2), + msg: "Two level difference should return score 98", + }, + // Multiple levels down + Case { + c0: h266_codec_spec(1, 0, 83), // Main, Main tier, Level 5.1 + c1: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 4), + msg: "Four level difference (5.1 to 3.1) should return score 96", + }, + // Offered level higher -> still matches, penalized by 1 ordinal + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 0, 64), // Main, Main tier, Level 4.0 + expected: Some(PayloadParams::EXACT_MATCH_SCORE - 1), + msg: "Offered level one higher than configured should return score 99", + }, + // Different profiles -> None + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(33, 0, 51), // Main10, Main tier, Level 3.1 + expected: None, + msg: "Different profiles should return None", + }, + // Different tiers -> None + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 1, 51), // Main, High tier, Level 3.1 + expected: None, + msg: "Different tiers should return None", + }, + ]; + + for Case { + c0, + c1, + expected, + msg, + } in cases.into_iter() + { + assert_eq!( + PayloadParams::match_h266_score(c0, c1), + expected, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test H.266 profile-only matching (for Chrome compatibility). + /// When only profile-id is provided without tier/level, match on profile only. + #[test] + fn test_profile_only_matching() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + expected: Option, + msg: &'static str, + } + + let cases = [ + // Both have profile-only, same profile -> score 90 + Case { + c0: h266_codec_spec_profile_only(1), // Main + c1: h266_codec_spec_profile_only(1), // Main + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Profile-only match (Main) should return score 90", + }, + // Both have profile-only, different profiles -> None + Case { + c0: h266_codec_spec_profile_only(1), // Main + c1: h266_codec_spec_profile_only(33), // Main10 + expected: None, + msg: "Profile-only with different profiles should return None", + }, + // One has full PTL, other has profile-only, same profile -> score 90 + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec_profile_only(1), // Main + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Mixed PTL and profile-only with matching profile should return score 90", + }, + // One has full PTL, other has profile-only, different profiles -> None + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec_profile_only(33), // Main10 + expected: None, + msg: "Mixed PTL and profile-only with different profiles should return None", + }, + // Reverse: profile-only vs full PTL, same profile -> score 90 + Case { + c0: h266_codec_spec_profile_only(1), // Main + c1: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + expected: Some(PayloadParams::PROFILE_ONLY_MATCH_SCORE), + msg: "Profile-only vs full PTL with matching profile should return score 90", + }, + ]; + + for Case { + c0, + c1, + expected, + msg, + } in cases.into_iter() + { + assert_eq!( + PayloadParams::match_h266_score(c0, c1), + expected, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test negative cases for H.266 matching to ensure proper rejection. + #[test] + fn test_negative_cases() { + struct Case { + c0: CodecSpec, + c1: CodecSpec, + msg: &'static str, + } + + let cases = [ + // Profile mismatch with full PTL + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(33, 0, 51), // Main10, Main tier, Level 3.1 + msg: "Profile mismatch (Main vs Main10) must not match", + }, + // Tier mismatch + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(1, 1, 51), // Main, High tier, Level 3.1 + msg: "Tier mismatch (Main vs High) must not match", + }, + // All three different (profile+tier mismatch rejects) + Case { + c0: h266_codec_spec(1, 0, 51), // Main, Main tier, Level 3.1 + c1: h266_codec_spec(33, 1, 83), // Main10, High tier, Level 5.1 + msg: "Profile and tier different must not match", + }, + // Profile-only mismatch + Case { + c0: h266_codec_spec_profile_only(1), // Main + c1: h266_codec_spec_profile_only(33), // Main10 + msg: "Profile-only mismatch must not match", + }, + // Mixed full PTL vs profile-only with different profiles + Case { + c0: h266_codec_spec(33, 0, 51), // Main10, Main tier, Level 3.1 + c1: h266_codec_spec_profile_only(1), // Main + msg: "Full PTL (Main10) vs profile-only (Main) must not match", + }, + ]; + + for Case { c0, c1, msg } in cases.into_iter() { + assert_eq!( + PayloadParams::match_h266_score(c0, c1), + None, + "{msg}\nc0: {c0:#?}\nc1: {c1:#?}" + ); + } + } + + /// Test that sprop-max-don-diff is adopted from remote SDP during negotiation, + /// and that packetizer/depacketizer are correctly initialized with DONL enabled. + #[test] + fn test_sprop_max_don_diff_negotiation_enables_donl() { + use crate::packet::{CodecDepacketizer, CodecPacketizer}; + + // Local config: no sprop-max-don-diff (default) + let local_spec = h266_codec_spec(1, 0, 51); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + assert_eq!( + local_params.spec.format.sprop_max_don_diff, None, + "Local should have no sprop-max-don-diff initially" + ); + + // Remote offers H.266 with sprop-max-don-diff=32 + let mut remote_spec = h266_codec_spec(1, 0, 51); + remote_spec.format.sprop_max_don_diff = Some(32); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + // Simulate SDP negotiation (update_param merges remote into local) + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param( + &[remote_params], + &mut claimed, + false, // remote is controlling (we're answering) + &mut unlocked, + ); + + // Verify sprop-max-don-diff was adopted from remote + assert_eq!( + local_params.spec.format.sprop_max_don_diff, + Some(32), + "sprop-max-don-diff should be adopted from remote after negotiation" + ); + + // Verify packetizer is initialized with DONL enabled + let donl_enabled = local_params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0; + assert!( + donl_enabled, + "DONL should be enabled when sprop-max-don-diff > 0" + ); + + let mut pack = CodecPacketizer::new( + local_params.spec.codec, + crate::format::Vp9PacketizerMode::default(), + ); + if let CodecPacketizer::H266(ref mut h266) = pack { + h266.with_donl(donl_enabled); + } + + // Packetize a NAL — should include DONL field (2 extra bytes) + let nalu = vec![0x00, 0x09, 0xAA, 0xBB]; + let packets = pack.packetize(1200, &nalu).unwrap(); + assert_eq!(packets.len(), 1); + assert_eq!( + packets[0].len(), + nalu.len() + 2, // +2 for DONL + "Packetized output should include 2-byte DONL field" + ); + + // Verify depacketizer is initialized with DONL enabled + let mut depack: CodecDepacketizer = local_params.spec.codec.into(); + if let CodecDepacketizer::H266(ref mut h266) = depack { + h266.with_donl(donl_enabled); + } + + // Depacketize the packet — should correctly strip DONL + use crate::packet::{CodecExtra, Depacketizer}; + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + if let CodecDepacketizer::H266(ref mut h266) = depack { + h266.depacketize(&packets[0], &mut out, &mut extra).unwrap(); + } + + // Output should be Annex-B: start code + original NAL + assert_eq!(&out[0..4], &[0x00, 0x00, 0x00, 0x01]); + assert_eq!( + &out[4..], + &nalu[..], + "Depacketized NAL should match original" + ); + } + + /// Test that missing sprop-max-don-diff from remote preserves local value. + #[test] + fn test_sprop_max_don_diff_absent_preserves_local() { + // Local has sprop-max-don-diff=16 + let mut local_spec = h266_codec_spec(1, 0, 51); + local_spec.format.sprop_max_don_diff = Some(16); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + + // Remote offers without sprop-max-don-diff + let remote_spec = h266_codec_spec(1, 0, 51); + assert!(remote_spec.format.sprop_max_don_diff.is_none()); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param(&[remote_params], &mut claimed, false, &mut unlocked); + + // Local value should be preserved (remote didn't override) + assert_eq!( + local_params.spec.format.sprop_max_don_diff, + Some(16), + "Local sprop-max-don-diff should be preserved when remote doesn't specify it" + ); + } + + /// Test that sprop-max-don-diff=0 from remote does NOT enable DONL. + #[test] + fn test_sprop_max_don_diff_zero_disables_donl() { + let local_spec = h266_codec_spec(1, 0, 51); + let mut local_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + local_spec, + ); + + // Remote offers with sprop-max-don-diff=0 (no reordering) + let mut remote_spec = h266_codec_spec(1, 0, 51); + remote_spec.format.sprop_max_don_diff = Some(0); + let remote_params = PayloadParams::new( + Pt::new_with_value(102), + Some(Pt::new_with_value(103)), + remote_spec, + ); + + let mut claimed = [false; 128]; + let mut unlocked = std::collections::HashSet::new(); + local_params.update_param(&[remote_params], &mut claimed, false, &mut unlocked); + + assert_eq!(local_params.spec.format.sprop_max_don_diff, Some(0)); + let donl_enabled = local_params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0; + assert!( + !donl_enabled, + "DONL should NOT be enabled when sprop-max-don-diff=0" + ); + + // Packetize without DONL — packet should be same size as NAL + let nalu = vec![0x00, 0x09, 0xAA, 0xBB]; + let mut pack = crate::packet::CodecPacketizer::new( + local_params.spec.codec, + crate::format::Vp9PacketizerMode::default(), + ); + // Don't enable DONL since donl_enabled is false + let packets = pack.packetize(1200, &nalu).unwrap(); + assert_eq!( + packets[0].len(), + nalu.len(), + "No DONL field when sprop-max-don-diff=0" + ); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/io/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/io/error.rs new file mode 100644 index 000000000..267ed4067 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/io/error.rs @@ -0,0 +1,45 @@ +use std::error::Error; +use std::fmt; +use std::io; + +pub use is::StunError; + +/// Errors from parsing network data. +#[derive(Debug)] +pub enum NetError { + /// Some STUN protocol error. + Stun(StunError), + + /// A wrapped IO error. + Io(io::Error), +} + +impl fmt::Display for NetError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + NetError::Stun(err) => write!(f, "{}", err), + NetError::Io(err) => write!(f, "{}", err), + } + } +} + +impl Error for NetError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + NetError::Stun(err) => Some(err), + NetError::Io(err) => Some(err), + } + } +} + +impl From for NetError { + fn from(err: StunError) -> Self { + NetError::Stun(err) + } +} + +impl From for NetError { + fn from(err: io::Error) -> Self { + NetError::Io(err) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/io/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/io/mod.rs new file mode 100644 index 000000000..4a5aa54c2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/io/mod.rs @@ -0,0 +1,182 @@ +#![allow(clippy::manual_range_contains)] +#![allow(clippy::new_without_default)] + +use std::convert::TryFrom; +use std::fmt; +use std::io; +use std::net::SocketAddr; + +use serde::{Deserialize, Serialize}; + +pub use is::stun::StunMessage; + +pub use str0m_proto::DATAGRAM_MTU_TARGET; +pub use str0m_proto::DATAGRAM_MTU_TARGET_MAX; +pub use str0m_proto::DATAGRAM_MTU_TARGET_MIN; +pub use str0m_proto::DATAGRAM_MTU_WARN; + +/// Max UDP packet size +pub(crate) const DATAGRAM_MAX_PACKET_SIZE: usize = 2000; + +/// Max expected RTP header over, with full extensions etc. +pub const MAX_RTP_OVERHEAD: usize = 80; + +/// Max expected DTLS record overhead (record header + explicit nonce + AEAD +/// auth tag), worst case across DTLS 1.2/1.3 and common AEAD ciphers +/// (AES-GCM, AES-CCM, CHACHA20-POLY1305). +pub const MAX_DTLS_OVERHEAD: usize = 48; + +mod error; +pub use self::error::NetError; +#[allow(unused_imports)] +pub use self::error::StunError; + +pub use str0m_proto::{DatagramSend, Protocol, TcpType, Transmit}; + +#[derive(Debug, Serialize, Deserialize)] +/// Received incoming data. +pub struct Receive<'a> { + /// The protocol the socket this received data originated from is using. + pub proto: Protocol, + + /// The socket this received data originated from. + pub source: SocketAddr, + + /// The destination ip of the datagram. + pub destination: SocketAddr, + + /// Parsed contents of the datagram. + #[serde(borrow)] + pub contents: DatagramRecv<'a>, +} + +impl<'a> Receive<'a> { + /// Creates a new instance by trying to parse the contents of `buf`. + pub fn new( + proto: Protocol, + source: SocketAddr, + destination: SocketAddr, + buf: &'a [u8], + ) -> Result { + let contents = DatagramRecv::try_from(buf)?; + Ok(Receive { + proto, + source, + destination, + contents, + }) + } +} + +/// Wrapper for a parsed payload to be received. +#[derive(Serialize, Deserialize)] +pub struct DatagramRecv<'a> { + #[serde(borrow)] + pub(crate) inner: DatagramRecvInner<'a>, +} + +#[allow(clippy::large_enum_variant)] // We purposely don't want to allocate. +#[derive(Serialize, Deserialize)] +pub(crate) enum DatagramRecvInner<'a> { + Stun(StunMessage<'a>), + Dtls(&'a [u8]), + Rtp(&'a [u8]), + Rtcp(&'a [u8]), +} + +impl<'a> TryFrom<&'a [u8]> for DatagramRecv<'a> { + type Error = NetError; + + fn try_from(value: &'a [u8]) -> Result { + use DatagramRecvInner::*; + + let kind = MultiplexKind::try_from(value)?; + + let inner = match kind { + MultiplexKind::Stun => Stun(StunMessage::parse(value)?), + MultiplexKind::Dtls => Dtls(value), + MultiplexKind::Rtp => Rtp(value), + MultiplexKind::Rtcp => Rtcp(value), + }; + + Ok(DatagramRecv { inner }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum MultiplexKind { + Stun, + Dtls, + Rtp, + Rtcp, +} + +impl<'a> TryFrom<&'a [u8]> for MultiplexKind { + type Error = io::Error; + + fn try_from(value: &'a [u8]) -> Result { + if value.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Empty datagram")); + } + + let byte0 = value[0]; + let len = value.len(); + + if byte0 < 2 && len >= 20 { + Ok(MultiplexKind::Stun) + } else if byte0 >= 20 && byte0 < 64 { + Ok(MultiplexKind::Dtls) + } else if byte0 >= 128 && byte0 < 192 && len > 2 { + let byte1 = value[1]; + let payload_type = byte1 & 0x7f; + + Ok(if payload_type < 64 { + // This is kinda novel, and probably breaks, but... + // we can use the < 64 pt as an escape hatch if we run out + // of dynamic numbers >= 96 + // https://bugs.chromium.org/p/webrtc/issues/detail?id=12194 + MultiplexKind::Rtp + } else if payload_type >= 64 && payload_type < 96 { + MultiplexKind::Rtcp + } else { + MultiplexKind::Rtp + }) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unknown datagram", + )) + } + } +} + +impl<'a> TryFrom<&'a Transmit> for Receive<'a> { + type Error = NetError; + + fn try_from(t: &'a Transmit) -> Result { + Ok(Receive { + proto: t.proto, + source: t.source, + destination: t.destination, + contents: DatagramRecv::try_from(&t.contents[..])?, + }) + } +} + +impl fmt::Debug for DatagramRecv<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + +impl fmt::Debug for DatagramRecvInner<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Stun(v) => f.debug_tuple("Stun").field(v).finish(), + Self::Dtls(v) => write!(f, "Dtls(len: {})", v.len()), + Self::Rtp(v) => write!(f, "Rtp(len: {})", v.len()), + Self::Rtcp(v) => write!(f, "Rtcp(len: {})", v.len()), + } + } + // +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/lib.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/lib.rs new file mode 100644 index 000000000..64c0d3014 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/lib.rs @@ -0,0 +1,2260 @@ +//! str0m logo +//! +//! A Sans I/O WebRTC implementation in Rust. +//! +//! This is a [Sans I/O][sansio] implementation meaning the `Rtc` instance itself is not doing any network +//! talking. Furthermore it has no internal threads or async tasks. All operations are happening from the +//! calls of the public API. +//! +//! This is deliberately not a standard `RTCPeerConnection` API since that isn't a great fit for Rust. +//! See more details in below section. +//! +//! # Join us +//! +//! We are discussing str0m things on Discord. Join us using this [invitation link][discord]. +//! +//! silly clip showing video playing +//! +//! # Usage +//! +//! The [`chat`][x-chat] example shows how to connect multiple browsers +//! together and act as an SFU (Selective Forwarding Unit). The example +//! multiplexes all traffic over one server UDP socket and uses two threads +//! (one for the web server, and one for the SFU loop). +//! +//! ## TLS +//! +//! For the browser to do WebRTC, all traffic must be under TLS. The +//! project ships with a self-signed certificate that is used for the +//! examples. The certificate is for hostname `str0m.test` since TLD .test +//! should never resolve to a real DNS name. +//! +//! ```text +//! cargo run --example chat +//! ``` +//! +//! The log should prompt you to connect a browser to https://10.0.0.103:3000 – this will +//! most likely cause a security warning that you must get the browser to accept. +//! +//! The [`http-post`][x-post] example roughly illustrates how to receive +//! media data from a browser client. The example is single threaded and +//! is a bit simpler than the chat. It is a good starting point to understand the API. +//! +//! ```text +//! cargo run --example http-post +//! ``` +//! +//! ### Real example +//! +//! To see how str0m is used in a real project, check out [BitWHIP][bitwhip] – +//! a CLI WebRTC Agent written in Rust. +//! +//! ## Passive +//! +//! For passive connections, i.e. where the media and initial OFFER is +//! made by a remote peer, we need these steps to open the connection. +//! +//! ```no_run +//! # use std::time::Instant; +//! # use str0m::{Rtc, Candidate}; +//! // Instantiate a new Rtc instance. +//! let mut rtc = Rtc::new(Instant::now()); +//! +//! // Add some ICE candidate such as a locally bound UDP port. +//! let addr = "1.2.3.4:5000".parse().unwrap(); +//! let candidate = Candidate::host(addr, "udp").unwrap(); +//! rtc.add_local_candidate(candidate); +//! +//! // Accept an incoming offer from the remote peer +//! // and get the corresponding answer. +//! let offer = todo!(); +//! let answer = rtc.sdp_api().accept_offer(offer).unwrap(); +//! +//! // Forward the answer to the remote peer. +//! +//! // Go to _run loop_ +//! ``` +//! +//! ## Active +//! +//! Active connections means we are making the inital OFFER and waiting for a +//! remote ANSWER to start the connection. +//! +//! ```no_run +//! # use std::time::Instant; +//! # use str0m::{Rtc, Candidate}; +//! # use str0m::media::{MediaKind, Direction}; +//! // Instantiate a new Rtc instance. +//! let mut rtc = Rtc::new(Instant::now()); +//! +//! // Add some ICE candidate such as a locally bound UDP port. +//! let addr = "1.2.3.4:5000".parse().unwrap(); +//! let candidate = Candidate::host(addr, "udp").unwrap(); +//! rtc.add_local_candidate(candidate); +//! +//! // Create a `SdpApi`. The change lets us make multiple changes +//! // before sending the offer. +//! let mut change = rtc.sdp_api(); +//! +//! // Do some change. A valid OFFER needs at least one "m-line" (media). +//! let mid = change.add_media(MediaKind::Audio, Direction::SendRecv, None, None, None); +//! +//! // Get the offer. +//! let (offer, pending) = change.apply().unwrap(); +//! +//! // Forward the offer to the remote peer and await the answer. +//! // How to transfer this is outside the scope for this library. +//! let answer = todo!(); +//! +//! // Apply answer. +//! rtc.sdp_api().accept_answer(pending, answer).unwrap(); +//! +//! // Go to _run loop_ +//! ``` +//! +//! ## Run loop +//! +//! ### The single-mutation invariant +//! +//! str0m's API has one strict contract that the run loop is built around: +//! +//! > **Every mutation of an `Rtc` instance must be followed by a complete +//! > drain of `poll_output` until it returns `Output::Timeout`, before the +//! > next mutation on the same `Rtc`.** +//! +//! A "mutation" is anything that takes `&mut Rtc` (directly or through a +//! handle obtained from it). The common ones are: +//! +//! - `Rtc::handle_input` — feeding a network packet or a timeout +//! - `Writer::write` / `Writer::request_keyframe` — sending media +//! - `Channel::write` — sending data channel data +//! - `SdpApi::apply` / `DirectApi::*` — negotiation +//! - `Rtc::add_local_candidate` / `Rtc::add_remote_candidate` +//! +//! Always: **mutate → drain to `Timeout` → mutate → drain to `Timeout` → …** +//! +//! Doing two mutations back-to-back without draining in between, or +//! waiting on I/O while the engine still has output queued, leaves the +//! engine in an inconsistent state and produces wrong behavior. Mutations +//! issued from inside the drain loop (e.g. calling `Writer::write` in +//! response to an `Output::Event`) are fine — the drain loop naturally +//! continues calling `poll_output` afterward and so the invariant holds. +//! +//! ### Canonical shape +//! +//! Driving an `Rtc` forward follows the same six-step shape, regardless +//! of sync or async: +//! +//! 1. **Wait** for one of: the next timeout firing, an incoming network +//! packet, or the application wanting to perform a mutation (e.g. +//! write media). +//! 2. **Perform that ONE mutation** — feed `Input` to `handle_input`, or +//! call into a writer / channel / SDP API. +//! 3. **Poll** `Rtc::poll_output`. +//! 4. **Handle** the output: `Output::Transmit` is sent on the socket, +//! `Output::Event` is dispatched to the application, `Output::Timeout` +//! records the next deadline. +//! 5. **Goto 3** until `poll_output` returns `Output::Timeout`. Only then +//! is the engine fully drained. +//! 6. The returned timeout is what we wait on next — **goto 1**. +//! +//! ```no_run +//! # use str0m::{Rtc, Output, IceConnectionState, Event, Input}; +//! # use str0m::net::{Receive, Protocol}; +//! # use std::io::ErrorKind; +//! # use std::net::UdpSocket; +//! # use std::time::{Duration, Instant}; +//! # let mut rtc = Rtc::new(Instant::now()); +//! // A UdpSocket we obtained _somehow_. +//! let socket: UdpSocket = todo!(); +//! +//! // Buffer for reading incoming UDP packets. +//! let mut buf = vec![0; 2000]; +//! +//! loop { +//! // === Steps 3-5: drain poll_output until we get the next timeout. === +//! let timeout = loop { +//! match rtc.poll_output().unwrap() { +//! // Step 5: a Timeout exits the drain loop. +//! Output::Timeout(t) => break t, +//! +//! // Step 4: transmit on the socket and keep draining. +//! // The destination IP comes from the ICE agent and may +//! // change during the session. +//! Output::Transmit(t) => { +//! socket.send_to(&t.contents, t.destination).unwrap(); +//! } +//! +//! // Step 4: hand the event to the application and keep draining. +//! // Events are mainly incoming media data from the remote peer, +//! // but also data channel data and statistics. +//! Output::Event(e) => { +//! if e == Event::IceConnectionStateChange(IceConnectionState::Disconnected) { +//! return; +//! } +//! // TODO: handle other events here, such as incoming media data. +//! } +//! } +//! }; +//! +//! // === Step 1: wait for ONE of: the timeout firing, an incoming +//! // packet, or application-side data. The example below uses a +//! // blocking UDP socket with a read timeout. With async you would +//! // `select!` over multiple futures; with application-side data you +//! // would also include a channel. +//! // +//! // set_read_timeout(Some(ZERO)) is not allowed, so clamp to >= 1ms. +//! let duration = (timeout - Instant::now()).max(Duration::from_millis(1)); +//! socket.set_read_timeout(Some(duration)).unwrap(); +//! buf.resize(2000, 0); +//! +//! // === Step 2: take ONE event and feed it as Input === +//! let input = match socket.recv_from(&mut buf) { +//! Ok((n, source)) => { +//! buf.truncate(n); +//! Input::Receive( +//! Instant::now(), +//! Receive { +//! proto: Protocol::Udp, +//! source, +//! destination: socket.local_addr().unwrap(), +//! contents: buf.as_slice().try_into().unwrap(), +//! }, +//! ) +//! } +//! +//! // The socket read timed out — feed Input::Timeout to advance +//! // the engine to the deadline. WouldBlock is the unix error, +//! // TimedOut is the windows error. +//! Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => { +//! Input::Timeout(Instant::now()) +//! } +//! +//! Err(e) => { +//! eprintln!("Error: {:?}", e); +//! return; +//! } +//! }; +//! +//! rtc.handle_input(input).unwrap(); +//! +//! // === Step 6: back to the top of the outer loop (goto step 3). === +//! } +//! ``` +//! +//! ## Sending media data +//! +//! When creating the media, we can decide which codecs to support, and they +//! are negotiated with the remote side. Each codec corresponds to a +//! "payload type" (PT). To send media data we need to figure out which PT +//! to use when sending. +//! +//! ```no_run +//! # use str0m::Rtc; +//! # use str0m::media::Mid; +//! # let rtc: Rtc = todo!(); +//! // Obtain mid from Event::MediaAdded +//! let mid: Mid = todo!(); +//! +//! // Create a media writer for the mid. +//! let writer = rtc.writer(mid).unwrap(); +//! +//! // Get the payload type (pt) for the wanted codec. +//! let pt = writer.payload_params().nth(0).unwrap().pt(); +//! +//! // Write the data +//! let wallclock = todo!(); // Absolute time of the data +//! let media_time = todo!(); // Media time, in RTP time +//! let data: &[u8] = todo!(); // Actual data +//! writer.write(pt, wallclock, media_time, data).unwrap(); +//! ``` +//! +//! `Writer::write` is a mutation, so the +//! [single-mutation invariant](#the-single-mutation-invariant) applies: +//! after writing, drain `Rtc::poll_output` to `Output::Timeout` before +//! the next mutation on this `Rtc`. +//! +//! ## Media time, wallclock and local time +//! +//! str0m has three main concepts of time. "now", media time and wallclock. +//! +//! ### Now +//! +//! Some calls in str0m, such as `Rtc::handle_input` takes a `now` argument +//! that is a `std::time::Instant`. These calls "drive the time forward" in +//! the internal state. This is used for everything like deciding when +//! to produce various feedback reports (RTCP) to remote peers, to +//! bandwidth estimation (BWE) and statistics. +//! +//! Str0m has _no internal clock_ calls. I.e. str0m never calls +//! `Instant::now()` itself. All time is external input. That means it's +//! possible to construct test cases driving an `Rtc` instance faster +//! than realtime (see the [integration tests][intg]). +//! +//! ### Media time +//! +//! Each RTP header has a 32 bit number that str0m calls _media time_. +//! Media time is in some time base that is dependent on the codec, +//! however all codecs in str0m use 90_000Hz for video and 48_000Hz +//! for audio. +//! +//! For video the `MediaTime` type is `/90_000` str0m extends +//! the 32 bit number in the RTP header to 64 bit taking into account +//! "rollover". 64 bit is such a large number the user doesn't need to +//! think about rollovers. +//! +//! ### Wallclock +//! +//! With _wallclock_ str0m means the time a sample of media was produced +//! at an originating source. I.e. if we are talking into a microphone the +//! wallclock is the NTP time the sound is sampled. +//! +//! We can't know the exact wallclock for media from a remote peer since +//! not every device is synchronized with NTP. Every sender does +//! periodically produce a Sender Report (SR) that contains the peer's +//! idea of its wallclock, however this number can be very wrong compared to +//! "real" NTP time. +//! +//! Furthermore, not all remote devices will have a linear idea of +//! time passing that exactly matches the local time. A minute on the +//! remote peer might not be exactly one minute locally. +//! +//! These timestamps become important when handling simultaneous audio from +//! multiple peers. +//! +//! When writing media we need to provide str0m with an estimated wallclock. +//! The simplest strategy is to only trust local time and use arrival time +//! of the incoming UDP packet. Another simple strategy is to lock some +//! time T at the first UDP packet, and then offset each wallclock using +//! `MediaTime`, i.e. for video we could have `T + /90_000` +//! +//! A production worthy SFU probably needs an even more sophisticated +//! strategy weighing in all possible time sources to get a good estimate +//! of the remote wallclock for a packet. +//! +//! # Crypto backends +//! +//! str0m supports multiple crypto backends via feature flags. The default is `aws-lc-rs`. +//! +//! | Feature | Crate | DTLS | Platforms | +//! |-------------------|-----------------------|----------------------------------|-----------| +//! | `aws-lc-rs` | `str0m-aws-lc-rs` | dimpl + AWS-LC-RS | All | +//! | `rust-crypto` | `str0m-rust-crypto` | dimpl + RustCrypto | All | +//! | `openssl` | `str0m-openssl` | OpenSSL (DTLS 1.2 only) | All | +//! | `openssl-dimpl` | `str0m-openssl` | dimpl + OpenSSL crypto | All | +//! | `apple-crypto` | `str0m-apple-crypto` | dimpl + Apple CryptoKit | macOS/iOS | +//! | `wincrypto` | `str0m-wincrypto` | Windows SChannel (DTLS 1.2 only) | Windows | +//! | `wincrypto-dimpl` | `str0m-wincrypto` | dimpl + Windows CNG | Windows | +//! +//! If multiple backend features are enabled, str0m automatically selects the backend in this +//! priority order: `aws-lc-rs`, `rust-crypto`, `openssl-dimpl`, `openssl`, `apple-crypto` +//! (Apple platforms only), `wincrypt-dimpl` (Windows only), `wincrypto` (Windows only). +//! +//! If you disable the default features, you MUST explicitly configure an alternative +//! crypto backend either process-wide or per-instance. +//! +//! ## Process-wide default +//! +//! For applications, the easiest is to set a process-wide default at startup. +//! Note that you can use any backend crate directly without enabling its feature flag: +//! +//! ```no_run +//! // Set process default (will panic if called twice) +//! // No need to enable the "rust-crypto" feature flag +//! str0m_rust_crypto::default_provider().install_process_default(); +//! ``` +//! +//! ## Crypto provider per Rtc instance +//! +//! ```no_run +//! use std::sync::Arc; +//! use std::time::Instant; +//! use str0m::Rtc; +//! +//! let rtc = Rtc::builder() +//! .set_crypto_provider(Arc::new(str0m_rust_crypto::default_provider())) +//! .build(Instant::now()); +//! ``` +//! +//! # Project status +//! +//! Str0m was originally developed by Martin Algesten of +//! [Lookback][lookback]. We use str0m for a specific use case: str0m as a +//! server SFU (as opposed to peer-2-peer). That means we are heavily +//! testing and developing the parts needed for our use case. Str0m is +//! intended to be an all-purpose WebRTC library, which means it also +//! works for peer-2-peer, though that aspect has received less testing. +//! +//! Performance is very good, there have been some work the discover and +//! optimize bottlenecks. Such efforts are of course never ending with +//! diminishing returns. While there are no glaringly obvious performance +//! bottlenecks, more work is always welcome – both algorithmically and +//! allocation/cloning in hot paths etc. +//! +//! # Design +//! +//! Output from the `Rtc` instance can be grouped into three kinds. +//! +//! 1. Events (such as receiving media or data channel data). +//! 2. Network output. Data to be sent, typically from a UDP socket. +//! 3. Timeouts. Indicates when the instance next expects a time input. +//! +//! Input to the `Rtc` instance is: +//! +//! 1. User operations (such as sending media or data channel data). +//! 2. Network input. Typically read from a UDP socket. +//! 3. Timeouts. As obtained from the output above. +//! +//! The correct use can be seen in the above [Run loop](#run-loop) or in the +//! examples. +//! +//! Sans I/O is a pattern where we turn both network input/output as well +//! as time passing into external input to the API. This means str0m has +//! no internal threads, just an enormous state machine that is driven +//! forward by different kinds of input. +//! +//! ## Frame or RTP level? +//! +//! Str0m defaults to the "frame level" which treats the RTP as an internal detail. The user +//! will thus mainly interact with: +//! +//! 1. [`Event::MediaData`][evmed] to receive full frames (audio frames or video frames). +//! 2. [`Writer::write`][writer] to write full frames. +//! 3. [`Writer::request_keyframe`][reqkey] to request keyframes. +//! +//! ### Frame level +//! +//! All codecs such as h264, vp8, vp9 and opus outputs what we call +//! "Frames". A frame has a very specific meaning for video, but this +//! project uses it in a broader sense, where a frame is either a video +//! or audio time stamped chunk of encoded data that typically represents +//! a chunk of audio, or _one single frame for video_. +//! +//! Frames are not suitable to use directly in UDP (RTP) packets - for +//! one they are too big. Frames are therefore further chunked up by +//! codec specific payloaders into RTP packets. +//! +//! ### RTP mode +//! +//! Str0m also provides an RTP level API. This would be similar to many other +//! RTP libraries where the RTP packets themselves are the API surface +//! towards the user (when building an SFU one would often talk about "forwarding +//! RTP packets", while with str0m we can also "forward frames"). Using +//! this API requires a deeper knowledge of RTP and WebRTC. +//! +//! To enable RTP mode +//! +//! ``` +//! # use std::time::Instant; +//! # use str0m::Rtc; +//! let rtc = Rtc::builder() +//! // Enable RTP mode for this Rtc instance. +//! // This disables `MediaEvent` and the `Writer::write` API. +//! .set_rtp_mode(true) +//! .build(Instant::now()); +//! ``` +//! +//! RTP mode gives us some new API points. +//! +//! 1. [`Event::RtpPacket`][rtppak] emitted for every incoming RTP packet. Empty packets for bandwidth +//! estimation are silently discarded. +//! 2. [`StreamTx::write_rtp`][wrtrtp] to write outgoing RTP packets. +//! 3. [`StreamRx::request_keyframe`][reqkey2] to request keyframes from remote. +//! +//! ## NIC enumeration and TURN (and STUN) +//! +//! The [ICE RFC][ice] talks about "gathering ice candidates". This means +//! inspecting the local network interfaces and potentially binding UDP +//! sockets on each usable interface. Since str0m is Sans I/O, this part +//! is outside the scope of what str0m does. How the user figures out +//! local IP addresses, via config or via looking up local NICs is not +//! something str0m cares about. +//! +//! TURN is a way of obtaining IP addresses that can be used as fallback +//! in case direct connections fail. We consider TURN similar to +//! enumerating local network interfaces – it's a way of obtaining +//! sockets. +//! +//! All discovered candidates, be they local (NIC) or remote sockets +//! (TURN), are added to str0m and str0m will perform the task of ICE +//! agent, forming "candidate pairs" and figuring out the best connection +//! while the actual task of sending the network traffic is left to the +//! user. +//! +//! ## The importance of `&mut self` +//! +//! Rust shines when we can eschew locks and heavily rely `&mut` for data +//! write access. Since str0m has no internal threads, we never have to +//! deal with shared data. Furthermore the the internals of the library is +//! organized such that we don't need multiple references to the same +//! entities. In str0m there are no `Rc`, `Mutex`, `mpsc`, `Arc`(*), or +//! other locks. +//! +//! This means all input to the lib can be modelled as +//! `handle_something(&mut self, something)`. +//! +//! (*) Ok. There is one `Arc` if you use Windows where we also require openssl. +//! +//! ## Not a standard WebRTC "Peer Connection" API +//! +//! The library deliberately steps away from the "standard" WebRTC API as +//! seen in JavaScript and/or [webrtc-rs][webrtc-rs] (or [Pion][pion] in Go). +//! There are few reasons for this. +//! +//! First, in the standard API, events are callbacks, which are not a +//! great fit for Rust. Callbacks require some kind of reference +//! (ownership?) over the entity the callback is being dispatched +//! upon. I.e. if in Rust we want `pc.addEventListener(x)`, `x` needs +//! to be wholly owned by `pc`, or have some shared reference (like +//! `Arc`). Shared references means shared data, and to get mutable shared +//! data, we will need some kind of lock. i.e. `Arc>` +//! or similar. +//! +//! As an alternative we could turn all events into `mpsc` channels, but +//! listening to multiple channels is awkward without async. +//! +//! Second, in the standard API, entities like `RTCPeerConnection` and +//! `RTCRtpTransceiver`, are easily clonable and/or long lived +//! references. I.e. `pc.getTranscievers()` returns objects that can be +//! retained and owned by the caller. This pattern is fine for garbage +//! collected or reference counted languages, but not great with Rust. +//! +//! ## Panics, Errors and unwraps +//! +//! Str0m adheres to [fail-fast][ff]. That means rather than brushing state +//! bugs under the carpet, it panics. We make a distinction between errors and +//! bugs. +//! +//! * Errors are as a result of incorrect or impossible to understand user input. +//! * Bugs are broken internal invariants (assumptions). +//! +//! If you scan the str0m code you find a few `unwrap()` (or `expect()`). These +//! will (should) always be accompanied by a code comment that explains why the +//! unwrap is okay. This is an internal invariant, a state assumption that +//! str0m is responsible for maintaining. +//! +//! We do not believe it's correct to change every `unwrap()`/`expect()` into +//! `unwrap_or_else()`, `if let Some(x) = x { ... }` etc, because doing so +//! brushes an actual problem (an incorrect assumption) under the carpet. Trying +//! to hobble along with an incorrect state would at best result in broken +//! behavior, at worst a security risk! +//! +//! Panics are our friends: *panic means bug* +//! +//! And also: str0m should *never* panic on any user input. If you encounter a panic, +//! please report it! +//! +//! ### Catching panics +//! +//! Panics should be incredibly rare, or we have a serious problem as a project. For an SFU, +//! it might not be ideal if str0m encounters a bug and brings the entire server down with it. +//! +//! For those who want an extra level of safety, we recommend looking at [`catch_unwind`][catch] +//! to safely discard a faulty `Rtc` instance. Since `Rtc` has no internal threads, locks or async +//! tasks, discarding the instance never risk poisoning locks or other issues that can happen +//! when catching a panic. +//! +//! ## FAQ +//! +//! ### Features +//! +//! Below is a brief comparison of features between libWebRTC and str0m to help you determine +//! if str0m is suitable for your project. +//! +//! | Feature | str0m | libWebRTC | +//! | ------------------------ | ------------------ | ------------------ | +//! | Peer Connection API | :x: | :white_check_mark: | +//! | SDP | :white_check_mark: | :white_check_mark: | +//! | ICE | :white_check_mark: | :white_check_mark: | +//! | Data Channels | :white_check_mark: | :white_check_mark: | +//! | Send/Recv Reports | :white_check_mark: | :white_check_mark: | +//! | Transport Wide CC | :white_check_mark: | :white_check_mark: | +//! | Bandwidth Estimation | :white_check_mark: | :white_check_mark: | +//! | Simulcast | :white_check_mark: | :white_check_mark: | +//! | NACK | :white_check_mark: | :white_check_mark: | +//! | Packetize | :white_check_mark: | :white_check_mark: | +//! | Fixed Depacketize Buffer | :white_check_mark: | :white_check_mark: | +//! | Adaptive Jitter Buffer | :x: | :white_check_mark: | +//! | Video/audio capture | :x: | :white_check_mark: | +//! | Video/audio encode | :x: | :white_check_mark: | +//! | Video/audio decode | :x: | :white_check_mark: | +//! | Audio render | :x: | :white_check_mark: | +//! | Turn | :x: | :white_check_mark: | +//! | Network interface enum | :x: | :white_check_mark: | +//! +//! ### Platform Support +//! +//! Platforms str0m is compiled and tested on: +//! +//! | Platform | Compiled | Tested | +//! | ------------------------------ | ----------------- | ----------------- | +//! | `x86_64-pc-windows-msvc` | :white_check_mark:| :white_check_mark:| +//! | `x86_64-unknown-linux-gnu` | :white_check_mark:| :white_check_mark:| +//! | `x86_64-apple-darwin` | :white_check_mark:| :white_check_mark:| +//! | `aarch64-apple-darwin` | :white_check_mark:| :white_check_mark:| +//! | `aarch64-unknown-linux-gnu` | :white_check_mark:| :white_check_mark:| +//! | `aarch64-pc-windows-msvc` | :white_check_mark:| :white_check_mark:| +//! | `aarch64-apple-ios` | :white_check_mark:| :x: | +//! | `aarch64-linux-android` | :white_check_mark:| :x: | +//! +//! If your platform isn't listed but is supported by Rust, we'd love for you to give str0m a try and +//! share your experience. We greatly appreciate your feedback! +//! +//! ### Does str0m support IPv4, IPv6, UDP and TCP? +//! +//! Certainly! str0m fully support IPv4, IPv6, UDP and TCP protocols. +//! +//! ### Can I utilize str0m with any Rust async runtime? +//! +//! Absolutely! str0m is fully sync, ensuring that it integrates seamlessly with any Rust async +//! runtime you opt for. +//! +//! ### Can I create a client with str0m? +//! +//! Of course! You have the freedom to create a client with str0m. However, please note that some +//! common client features like media encoding, decoding, and capture are not included in str0m. But +//! don't let that stop you from building amazing applications! +//! +//! ### Can I use str0m in a media server? +//! +//! Yes! str0m excels as a server component with support for both RTP API and Frame API. You can +//! easily build that recording server or SFU you dreamt of in Rust! +//! +//! ### Can I deploy the chat example into production? +//! +//! While the chat example showcases how to use str0m's API, it's not intended for production use or +//! heavy load. Writing a full-featured SFU or MCU (Multipoint Control Unit) is a significant +//! undertaking, involving various design decisions based on production requirements. +//! +//! ### Discovered a bug? Here's how to share it with us +//! +//! We'd love to hear about it! Please submit an issue and consider joining our Discord community +//! to discuss further. For a seamless reporting experience, refer to this exemplary +//! bug report: . We appreciate your contribution +//! to making str0m better! +//! +//! ### I am allergic to SDP can you help me? +//! +//! Yes use the direct API! +//! +//! [sansio]: https://sans-io.readthedocs.io +//! [quinn]: https://github.com/quinn-rs/quinn +//! [pion]: https://github.com/pion/webrtc +//! [webrtc-rs]: https://github.com/webrtc-rs/webrtc +//! [discord]: https://discord.gg/e2CC8UYebP +//! [zulip]: https://str0m.zulipchat.com +//! [ice]: https://www.rfc-editor.org/rfc/rfc8445 +//! [lookback]: https://www.lookback.com +//! [x-post]: https://github.com/algesten/str0m/blob/main/examples/http-post.rs +//! [x-chat]: https://github.com/algesten/str0m/blob/main/examples/chat.rs +//! [intg]: https://github.com/algesten/str0m/blob/main/tests/unidirectional.rs#L12 +//! [ff]: https://en.wikipedia.org/wiki/Fail-fast +//! [catch]: https://doc.rust-lang.org/std/panic/fn.catch_unwind.html +//! [evmed]: https://docs.rs/str0m/*/str0m/enum.Event.html#variant.MediaData +//! [writer]: https://docs.rs/str0m/*/str0m/media/struct.Writer.html#method.write +//! [reqkey]: https://docs.rs/str0m/*/str0m/media/struct.Writer.html#method.request_keyframe +//! [rtppak]: https://docs.rs/str0m/*/str0m/enum.Event.html#variant.RtpPacket +//! [wrtrtp]: https://docs.rs/str0m/*/str0m/rtp/struct.StreamTx.html#method.write_rtp +//! [reqkey2]: https://docs.rs/str0m/*/str0m/rtp/struct.StreamRx.html#method.request_keyframe +//! [bitwhip]: https://github.com/bitwhip/bitwhip + +#![forbid(unsafe_code)] +#![allow(clippy::new_without_default)] +#![allow(clippy::bool_to_int_with_if)] +#![allow(clippy::assertions_on_constants)] +#![allow(clippy::manual_range_contains)] +#![allow(clippy::get_first)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::precedence)] +#![allow(clippy::doc_overindented_list_items)] +#![allow(clippy::uninlined_format_args)] +#![allow(mismatched_lifetime_syntaxes)] +#![deny(clippy::needless_pass_by_ref_mut)] +#![deny(missing_docs)] + +#[macro_use] +extern crate tracing; + +use bwe::{Bwe, BweKind}; +use change::{DirectApi, SdpApi}; +use rtp::RawPacket; +use std::fmt; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Instant; +use str0m_proto::Pii; +use streams::RtpPacket; +use streams::StreamPaused; +use util::InstantExt; + +// Identity `drv::ToStatic` (`Static = Self`) for the `Copy` identity types. +// Used instead of `#[derive(drv::Input)]` because the derive's generated +// shadow type isn't `Eq`/`Hash`/`Borrow`, so it can't be a `HashMap` +// key inside a `#[drv::memo]` projection; `Static = Self` can. `eq_static` +// defers to each type's own `PartialEq`. Each module invokes this for its +// own types. +#[cfg(feature = "drv")] +macro_rules! drv_identity_copy { + ($($t:ty),* $(,)?) => { + $( + impl drv::ToStatic for $t { + type Static = $t; + fn to_static(&self) -> $t { *self } + fn eq_static(&self, other: &$t) -> bool { self == other } + } + )* + }; +} +#[cfg(feature = "drv")] +pub(crate) use drv_identity_copy; + +/// Cryptographic provider traits and implementations. +/// +/// This module provides the traits for pluggable cryptographic operations +/// used in DTLS, SRTP, and STUN. +pub mod crypto; +use crypto::Fingerprint; + +mod dtls; +use crate::crypto::dtls::DtlsOutput; +use crate::crypto::{CryptoProvider, DtlsError, from_feature_flags}; +use crate::dtls::is_would_block; +use dtls::Dtls; + +use is::IceAgent; +use is::IceAgentEvent; +pub use is::{Candidate, CandidateBuilder, CandidateKind, IceConnectionState, IceCreds}; + +#[path = "config.rs"] +mod config_mod; +pub use config_mod::RtcConfig; + +/// Default target MTU used when none is configured. +pub use io::DATAGRAM_MTU_TARGET; +/// Upper bound that the target endpoint of [`RtcConfig::set_mtu`] must satisfy. +pub use io::DATAGRAM_MTU_TARGET_MAX; +/// Lower bound that the target endpoint of [`RtcConfig::set_mtu`] must satisfy. +pub use io::DATAGRAM_MTU_TARGET_MIN; +/// Default warning threshold for over-sized packets. +pub use io::DATAGRAM_MTU_WARN; + +/// Additional configuration. +pub mod config { + pub use super::crypto::dtls::{DtlsCert, DtlsVersion, KeyingMaterial}; + pub use super::crypto::{CryptoProvider, Fingerprint}; +} + +/// Low level ICE access. +// The ICE API is not necessary to interact with directly for "regular" +// use of str0m. This is exported for other libraries that want to +// reuse str0m's ICE implementation. This is now in the `is` crate. +#[doc(hidden)] +pub mod ice { + pub use is::IceCreds; + pub use is::stun::{StunMessage, StunMessageBuilder, StunPacket, TransId}; + pub use is::{IceAgent, IceAgentEvent}; + pub use is::{LocalPreference, default_local_preference}; +} + +mod io; +use io::DatagramRecvInner; + +mod packet; + +#[path = "rtp/mod.rs"] +mod rtp_; +use rtp_::{Bitrate, DataSize}; + +/// Low level RTP access. +pub mod rtp { + /// Feedback for RTP. + pub mod rtcp { + pub use crate::rtp_::AppSpecificFeedback; + pub use crate::rtp_::{Descriptions, ExtendedReport, Fir, Goodbye, Nack, Pli}; + pub use crate::rtp_::{Dlrr, NackEntry, ReceptionReport, ReportBlock}; + pub use crate::rtp_::{FirEntry, ReceiverReport, SenderInfo, SenderReport, Twcc}; + pub use crate::rtp_::{ReportList, Rrtr, Rtcp, Sdes, SdesType}; + } + use self::rtcp::Rtcp; + + /// Video Layers Allocation RTP Header Extension + pub mod vla; + pub use crate::rtp_::{AbsCaptureTime, ExtensionValues, UserExtensionValues}; + pub use crate::rtp_::{Extension, ExtensionMap, ExtensionSerializer}; + + pub use crate::packet::{ + Vp8Descriptor, Vp8DescriptorError, Vp8Patch, Vp8PatchBuilder, Vp8PatchError, + }; + pub use crate::rtp_::{RtpHeader, SeqNo, Ssrc, VideoOrientation}; + pub use crate::streams::{ + RtpPacket, RtpWrite, StreamPaused, StreamRx, StreamTx, StreamTxQueueInfo, + }; + + /// Debug output of the unencrypted RTP and RTCP packets. + /// + /// Enable using [`RtcConfig::enable_raw_packets()`][crate::RtcConfig::enable_raw_packets]. + /// This clones data, and is therefore expensive. + /// Should not be enabled outside of tests and troubleshooting. + #[derive(Debug)] + pub enum RawPacket { + /// Sent RTCP. + RtcpTx(Rtcp), + /// Incoming RTCP. + RtcpRx(Rtcp), + /// Sent RTP. + RtpTx(RtpHeader, Vec), + /// Incoming RTP. + RtpRx(RtpHeader, Vec), + } +} + +pub(crate) mod pacer; + +#[path = "bwe/mod.rs"] +pub(crate) mod bwe_; + +/// Bandwidth estimation. +pub mod bwe { + pub use crate::bwe_::api::*; +} + +mod sctp; +use sctp::{RtcSctp, SctpEvent, SctpInitData}; + +mod sdp; + +pub mod format; +use format::CodecConfig; + +pub mod channel; +use channel::{Channel, ChannelData, ChannelHandler, ChannelId}; + +pub mod media; +use media::AppSpecificFeedback; +use media::SenderFeedback; +use media::{Direction, Media, Mid, Pt, Rid, Writer}; +use media::{KeyframeRequest, KeyframeRequestKind}; +use media::{MediaAdded, MediaChanged, MediaData}; + +pub mod change; + +mod util; +use util::{Soonest, not_happening}; + +mod session; +use session::Session; + +pub mod stats; + +use stats::{CandidatePairStats, CandidateStats, MediaEgressStats, MediaIngressStats}; +use stats::{PeerStats, Stats, StatsEvent, StatsSnapshot}; + +mod streams; + +pub mod error; + +/// Network related types to get socket data in/out of [`Rtc`]. +pub mod net { + pub use crate::io::{DatagramRecv, DatagramSend, Protocol, Receive, TcpType, Transmit}; +} + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub use error::RtcError; + +/// Instance that does WebRTC. Main struct of the entire library. +/// +/// ## Usage +/// +/// The canonical run loop is: perform exactly one mutation (typically +/// `handle_input` with a network packet or timeout), then drain +/// `poll_output` until it returns `Output::Timeout`, then wait for the +/// next event. **Every mutation must be followed by a full drain before +/// the next mutation.** See the [crate docs](crate#run-loop) for the full +/// explanation of the single-mutation invariant. +/// +/// ```no_run +/// # use std::time::Instant; +/// # use str0m::{Rtc, Output, Input}; +/// let mut rtc = Rtc::new(Instant::now()); +/// +/// loop { +/// // Drain poll_output until it returns the next timeout. +/// let timeout = loop { +/// match rtc.poll_output().unwrap() { +/// Output::Timeout(t) => break t, +/// Output::Transmit(t) => { +/// // TODO: Send t.contents to t.destination on a UDP socket. +/// } +/// Output::Event(e) => { +/// // TODO: Handle event. +/// } +/// } +/// }; +/// +/// // TODO: Wait for ONE of: `timeout` firing, an incoming network +/// // packet, or application-side data to send. Build an +/// // `Input` from whichever fires first. +/// let input: Input = todo!(); +/// +/// rtc.handle_input(input).unwrap(); +/// } +/// ``` +pub struct Rtc { + state: RtcState, + ice: IceAgent, + dtls: Dtls, + dtls_connected: bool, + dtls_buf: Vec, + next_dtls_timeout: Option, + sctp: RtcSctp, + chan: ChannelHandler, + stats: Option, + session: Session, + remote_fingerprint: Option, + remote_addrs: Vec, + send_addr: Option, + need_init_time: bool, + last_now: Instant, + peer_bytes_rx: u64, + peer_bytes_tx: u64, + change_counter: usize, + last_timeout_reason: Reason, + crypto_provider: Arc, + fingerprint_verification: bool, + close_dtls_started: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum RtcState { + Alive, + Closing, + Closed, +} + +struct SendAddr { + proto: net::Protocol, + source: SocketAddr, + destination: SocketAddr, +} + +/// Events produced by [`Rtc::poll_output()`]. +#[derive(Debug)] +#[non_exhaustive] +#[rustfmt::skip] +pub enum Event { + // =================== ICE related events =================== + + /// Emitted when we got ICE connection and established DTLS. + Connected, + + /// ICE connection state changes tells us whether the [`Rtc`] instance is + /// connected to the peer or not. + IceConnectionStateChange(IceConnectionState), + + // =================== Media related events ================== + + /// Upon detecting the remote side adding new media to the session. + /// + /// For locally added media, this event never fires. Thus it can be thought of as an + /// "SDP only" event. If the direct API is used on both sides, the declaration is local \ + /// to both sides and the event never fires. + /// + /// The [`Media`] instance is available via [`Rtc::media()`]. + MediaAdded(MediaAdded), + + /// Incoming media data sent by the remote peer. + MediaData(MediaData), + + /// Changes to the media may be emitted. + /// + ///. Currently only covers a change of direction. + MediaChanged(MediaChanged), + + // =================== Data channel related events =================== + + /// A data channel has opened. + /// + /// The string is the channel label which is set by the opening peer and can + /// be used to identify the purpose of the channel when there are more than one. + /// + /// The negotiation is to set up an SCTP association via DTLS. Subsequent data + /// channels reuse the same association. + /// + /// Upon this event, the [`Channel`] can be obtained via [`Rtc::channel()`]. + /// + /// For [`SdpApi`]: The first ever data channel results in an SDP + /// negotiation, and this events comes at the end of that. + ChannelOpen(ChannelId, String), + + /// Incoming data channel data from the remote peer. + ChannelData(ChannelData), + + /// A data channel has been closed. + ChannelClose(ChannelId), + + /// A data channel's buffered amount has dropped below the configured threshold. + ChannelBufferedAmountLow(ChannelId), + + // =================== Statistics and BWE related events =================== + + /// Statistics event for the Rtc instance + /// + /// Includes both media traffic (rtp payload) as well as all traffic + PeerStats(PeerStats), + + /// Aggregated statistics for each media (mid, rid) in the ingress direction + MediaIngressStats(MediaIngressStats), + + /// Aggregated statistics for each media (mid, rid) in the egress direction + MediaEgressStats(MediaEgressStats), + + /// A new estimate from the bandwidth estimation subsystem. + EgressBitrateEstimate(BweKind), + + // =================== RTP related events =================== + + /// Incoming keyframe request for media that we are sending to the remote peer. + /// + /// The request is either PLI (Picture Loss Indication) or FIR (Full Intra Request). + KeyframeRequest(KeyframeRequest), + + /// Whether an incoming encoded stream is paused. + /// + /// This means the stream has not received any data for some time (default 1.5 seconds). + StreamPaused(StreamPaused), + + /// Sender feedback for an incoming stream, derived from RTCP SR. + SenderFeedback(SenderFeedback), + + /// Incoming RTP data. + RtpPacket(RtpPacket), + + /// Incoming application-specific Payload-Specific Feedback (PSFB FMT=15, PT=206). + /// + /// Emitted when a non-REMB FMT=15 RTCP PSFB message is received. The payload + /// is opaque and application-defined (RFC 4585 Section 6.4). + AppSpecificFeedback(AppSpecificFeedback), + + /// The remote DTLS connection or SCTP association has closed. + /// + /// When this event is emitted, `Rtc` starts local shutdown signals and + /// remains alive only while [`Rtc::poll_output()`] drains pending local + /// close output such as RTCP BYE and the DTLS close_notify response. + Closed, + + /// Debug output of incoming and outgoing RTCP/RTP packets. + /// + /// Enable using [`RtcConfig::enable_raw_packets()`]. + /// This clones data, and is therefore expensive. + /// Should not be enabled outside of tests and troubleshooting. + RawPacket(Box), + + /// For internal testing only. + /// + /// The probe cluster config when a probe fires. + #[cfg(feature = "_internal_test_exports")] + Probe(crate::bwe_::ProbeClusterConfig), +} + +impl Event { + /// Reference to the [`RawPacket`] if this is indeed an `Event::RawPacket`. + pub fn as_raw_packet(&self) -> Option<&RawPacket> { + if let Self::RawPacket(boxed) = &self { + Some(&**boxed) + } else { + None + } + } +} + +/// Input as expected by [`Rtc::handle_input()`]. Either network data or a timeout. +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] // We purposely don't want to allocate. +pub enum Input<'a> { + /// A timeout without any network input. + Timeout(Instant), + /// Network input. The [`Instant`] is the time the packet was received from the socket - + /// not necessarily "now" (e.g. when UDP demultiplexing runs on a separate thread). It + /// drives time forward when more recent than the last "now" the instance has seen. + Receive(Instant, net::Receive<'a>), +} + +/// Output produced by [`Rtc::poll_output()`] +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum Output { + /// When the [`Rtc`] instance expects an [`Input::Timeout`]. + Timeout(Instant), + + /// Network data that is to be sent. + Transmit(net::Transmit), + + /// Some event such as media data arriving from the remote peer or connection events. + Event(Event), +} + +pub use crate::pacer::PacerReason; + +/// The reason for the next [`Output::Timeout`]. +/// +/// This enum is not considered stable API and may change in minor revisions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[non_exhaustive] +pub enum Reason { + /// No timeout scheduled. + /// + /// The timeout value is in the distant future. + #[default] + NotHappening, + + /// The DTLS subsystem. + /// + /// Only relevant during handshaking. + DTLS, + + /// The ICE agent. + /// + /// Includes checking candidate pairs and various cleanups. + Ice, + + /// The SCTP subsystem. + /// + /// Things like handling retransmissions and keep-alive checks. + Sctp, + + /// Data channels. + /// + /// Scheduled when we need to open allocations using SCTP. + Channel, + + /// Stats gathering (if enabled). + /// + /// Periodic gathering of statistics. + Stats, + + /// Regular RTP feedback. + /// + /// Receiver reports (RR) and sender reports (SR). + Feedback, + + /// Sending of RTP NACK. + /// + /// When missing packets are discovered, a NACK is scheduled. + Nack, + + /// Reporting of TWCC (if enabled). + /// + /// All incoming RTP packets are reported using TWCC. Enabled via SDP if both + /// sides support it. + Twcc, + + /// RTP streams not receiving data goes into a paused state. + /// + /// Whenever an RTP receive stream receives data, a new timeout is scheduled. + PauseCheck, + + /// Preprocessing of RTP packets to be sent. + /// + /// Housekeeping task in RTP send streams. + SendStream, + + /// Packetizing of media into RTP data (if used). + /// + /// Written media data needs packetizing. This is not used in RTP mode. + Packetize, + + /// Pacer doing things. + Pacer(PacerReason), + + /// The delay controller of the BWE subsystem. + BweDelayControl, + + /// The probe controller of the BWE subsystem. + BweProbeControl, + + /// The probe estimator of the BWE subsystem. + BweProbeEstimator, +} + +impl Rtc { + /// Creates a new instance with default settings. + /// + /// To configure the instance, use [`RtcConfig`]. + /// + /// ``` + /// use std::time::Instant; + /// use str0m::Rtc; + /// + /// let rtc = Rtc::new(Instant::now()); + /// ``` + pub fn new(start: Instant) -> Self { + let config = RtcConfig::default(); + Self::new_from_config(config, start).expect("Failed to create Rtc from default config") + } + + /// Creates a config builder that configures an [`Rtc`] instance. + /// + /// ``` + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// let rtc = Rtc::builder() + /// .set_ice_lite(true) + /// .build(Instant::now()); + /// ``` + pub fn builder() -> RtcConfig { + RtcConfig::new() + } + + pub(crate) fn new_from_config(config: RtcConfig, start: Instant) -> Result { + let crypto_provider = config + .crypto_provider + .clone() + // If crypto_provider is not set in config, check process default + .or_else(|| CryptoProvider::get_default().cloned().map(Arc::new)) + // Or fall back on feature flags + .or_else(|| Some(Arc::new(from_feature_flags()))) + // from_feature_flags panics already, so we should never see + // this expect message. + .expect("a crash earlier if no crypto provider was set"); + + let session = Session::new(&config); + + // Capture before any partial moves of `config` below. + let mtu = config.mtu.clone(); + + let local_creds = config.local_ice_credentials.unwrap_or_else(IceCreds::new); + let mut ice = IceAgent::with_hmac(local_creds, crypto_provider.sha1_hmac_provider); + if config.ice_lite { + ice.set_ice_lite(config.ice_lite); + } + + if let Some(initial_stun_rto) = config.initial_stun_rto { + ice.set_initial_stun_rto(initial_stun_rto); + } + + if let Some(max_stun_rto) = config.max_stun_rto { + ice.set_max_stun_rto(max_stun_rto); + } + + if let Some(max_stun_retransmits) = config.max_stun_retransmits { + ice.set_max_stun_retransmits(max_stun_retransmits); + } + + ice.set_mtu(mtu.clone()); + + let dtls_cert = config + .dtls_cert + .or_else(|| crypto_provider.dtls_provider.generate_certificate()) + .expect( + "No DTLS certificate provided and the crypto provider cannot generate one. \ + Either provide a certificate via RtcConfig::set_dtls_cert or use a \ + crypto provider that supports certificate generation.", + ); + + let mut sctp = RtcSctp::new(*mtu.start()); + if config.snap_enabled { + sctp.enable_snap(); + } + + Ok(Rtc { + state: RtcState::Alive, + ice, + dtls: Dtls::new( + &dtls_cert, + crypto_provider.dtls_provider, + crypto_provider.sha256_provider, + start, + config.dtls_version, + mtu, + ) + .expect("DTLS to init without problem"), + dtls_connected: false, + dtls_buf: vec![0; 2000], + next_dtls_timeout: None, + session, + sctp, + chan: ChannelHandler::default(), + stats: config.stats_interval.map(Stats::new), + remote_fingerprint: None, + remote_addrs: vec![], + send_addr: None, + need_init_time: true, + last_now: start, + peer_bytes_rx: 0, + peer_bytes_tx: 0, + change_counter: 0, + last_timeout_reason: Reason::NotHappening, + crypto_provider, + fingerprint_verification: config.fingerprint_verification, + close_dtls_started: false, + }) + } + + /// Tests if this instance is still working. + /// + /// Certain events will straight away disconnect the `Rtc` instance, such as + /// the DTLS fingerprint from the setup not matching that of the TLS negotiation + /// (since that would potentially indicate a MITM attack!). + /// + /// The instance can be manually disconnected using [`Rtc::disconnect()`]. + /// + /// During shutdown, this remains `true` until [`Rtc::poll_output()`] + /// has drained pending local close output. App-facing operations such as + /// [`Rtc::writer()`] and [`Rtc::channel()`] are unavailable during that + /// closing drain. + /// + /// ``` + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// assert!(rtc.is_alive()); + /// + /// rtc.disconnect(); + /// assert!(!rtc.is_alive()); + /// ``` + pub fn is_alive(&self) -> bool { + self.state != RtcState::Closed + } + + /// Force disconnects the instance making [`Rtc::is_alive()`] return `false`. + /// + /// This makes [`Rtc::poll_output`] and [`Rtc::handle_input`] go inert and not + /// produce anymore network output or events. + /// + /// ``` + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// rtc.disconnect(); + /// assert!(!rtc.is_alive()); + /// ``` + pub fn disconnect(&mut self) { + self.state = RtcState::Closed; + } + + /// Add a local ICE candidate. Local candidates are socket addresses the `Rtc` instance + /// use for communicating with the peer. + /// + /// If the candidate is accepted by the `Rtc` instance, it will return `Some` with a reference + /// to it. You should then signal this candidate to the remote peer. + /// + /// This library has no built-in discovery of local network addresses on the host + /// or NATed addresses via a STUN server or TURN server. The user of the library + /// is expected to add new local candidates as they are discovered. + /// + /// In WebRTC lingo, the `Rtc` instance is permanently in a mode of [Trickle Ice][1]. It's + /// however advisable to add at least one local candidate before starting the instance. + /// + /// ``` + /// # use std::time::Instant; + /// # use str0m::{Rtc, Candidate}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let a = "127.0.0.1:5000".parse().unwrap(); + /// let c = Candidate::host(a, "udp").unwrap(); + /// + /// rtc.add_local_candidate(c); + /// ``` + /// + /// [1]: https://www.rfc-editor.org/rfc/rfc8838.txt + pub fn add_local_candidate(&mut self, c: Candidate) -> Option<&Candidate> { + self.ice.add_local_candidate(c) + } + + /// Add a remote ICE candidate. Remote candidates are addresses of the peer. + /// + /// For [`SdpApi`]: Remote candidates are typically added via + /// receiving a remote [`SdpOffer`][change::SdpOffer] or [`SdpAnswer`][change::SdpAnswer]. + /// + /// However for the case of [Trickle Ice][1], this is the way to add remote candidates + /// that are "trickled" from the other side. + /// + /// ``` + /// # use std::time::Instant; + /// # use str0m::{Rtc, Candidate}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let a = "1.2.3.4:5000".parse().unwrap(); + /// let c = Candidate::host(a, "udp").unwrap(); + /// + /// rtc.add_remote_candidate(c); + /// ``` + /// + /// [1]: https://www.rfc-editor.org/rfc/rfc8838.txt + pub fn add_remote_candidate(&mut self, c: Candidate) { + self.ice.add_remote_candidate(c); + } + + /// Checks if we are connected. + /// + /// This tests if we have ICE connection, DTLS and the SRTP crypto derived contexts are up. + pub fn is_connected(&self) -> bool { + self.ice.state().is_connected() && self.dtls_connected && self.session.is_connected() + } + + /// Make changes to the Rtc session via SDP. + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// # use str0m::media::{MediaKind, Direction}; + /// # use str0m::change::SdpAnswer; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let mut changes = rtc.sdp_api(); + /// let mid_audio = changes.add_media(MediaKind::Audio, Direction::SendOnly, None, None, None); + /// let mid_video = changes.add_media(MediaKind::Video, Direction::SendOnly, None, None, None); + /// + /// let (offer, pending) = changes.apply().unwrap(); + /// let json = serde_json::to_vec(&offer).unwrap(); + /// + /// // Send json OFFER to remote peer. Receive an answer back. + /// let answer: SdpAnswer = todo!(); + /// + /// rtc.sdp_api().accept_answer(pending, answer).unwrap(); + /// ``` + pub fn sdp_api(&mut self) -> SdpApi { + SdpApi::new(self) + } + + /// Makes direct changes to the Rtc session. + /// + /// This is a low level API. For "normal" use via SDP, see [`Rtc::sdp_api()`]. + pub fn direct_api(&mut self) -> DirectApi { + DirectApi::new(self) + } + + /// Send outgoing media data (frames) or request keyframes. + /// + /// Returns `None` if the direction isn't sending (`sendrecv` or `sendonly`). + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// # use str0m::media::{MediaData, Mid}; + /// # use str0m::format::PayloadParams; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// // add candidates, do SDP negotiation + /// let mid: Mid = todo!(); // obtain mid from Event::MediaAdded. + /// + /// // Writer for this mid. + /// let writer = rtc.writer(mid).unwrap(); + /// + /// // Get incoming media data from another peer + /// let data: MediaData = todo!(); + /// + /// // Match incoming PT to an outgoing PT. + /// let pt = writer.match_params(data.params).unwrap(); + /// + /// writer.write(pt, data.network_time, data.time, data.data).unwrap(); + /// ``` + /// + /// This is a frame level API: For RTP level see [`DirectApi::stream_tx()`] + /// and [`DirectApi::stream_rx()`]. + /// + pub fn writer(&mut self, mid: Mid) -> Option { + if self.session.rtp_mode { + panic!("In rtp_mode use direct_api().stream_tx().write_rtp()"); + } + + if self.state != RtcState::Alive { + return None; + } + + // This does not catch potential RIDs required to send simulcast, but + // it's a good start. An error might arise later on RID mismatch. + self.session.media_by_mid_mut(mid)?; + + Some(Writer::new(&mut self.session, mid)) + } + + /// Currently configured media. + /// + /// Read only access. Changes are made via [`Rtc::sdp_api()`] or [`Rtc::direct_api()`]. + pub fn media(&self, mid: Mid) -> Option<&Media> { + self.session.media_by_mid(mid) + } + + fn init_dtls(&mut self, active: bool) -> Result<(), RtcError> { + if self.dtls.is_inited() { + return Ok(()); + } + + debug!("DTLS setup is: {:?}", active); + self.dtls.set_active(active); + + // Initialize the DTLS state (client or server) before any operations + // This ensures internal state like random (client) or last_now (server) is initialized + self.dtls.handle_timeout(self.last_now)?; + + if active { + // Drive handshake by sending an empty packet to trigger ClientHello + let _ = self.dtls.handle_receive(&[]); + } + + Ok(()) + } + + fn try_init_sctp( + &mut self, + client: bool, + sctp_init_data: Option, + remote_max_message_size: Option, + ) -> Result<(), RtcError> { + // If we got an m=application line, ensure we have negotiated the + // SCTP association with the other side. + if self.sctp.is_inited() { + return Ok(()); + } + + self.sctp.init( + client, + self.last_now, + sctp_init_data, + remote_max_message_size, + )?; + Ok(()) + } + + /// Creates a new Mid that is not in the session already. + pub(crate) fn new_mid(&self) -> Mid { + loop { + let mid = Mid::new(); + if !self.session.has_mid(mid) { + break mid; + } + } + } + + /// Poll the `Rtc` instance for output. Output can be three things, something to _Transmit_ + /// via a UDP socket (maybe via a TURN server). An _Event_, such as receiving media data, + /// or a _Timeout_. + /// + /// The user of the library is expected to continuously call this function and deal with + /// the output until it encounters an [`Output::Timeout`] at which point no further output + /// is produced (if polled again, it will result in just another timeout). + /// + /// After exhausting the `poll_output`, the function will only produce more output again + /// when one of two things happen: + /// + /// 1. The polled timeout is reached. + /// 2. New network input. + /// + /// See [`Rtc`] instance documentation for how this is expected to be used in a loop. + pub fn poll_output(&mut self) -> Result { + let o = self.do_poll_output()?; + + match &o { + Output::Event(e) => match e { + Event::ChannelData(_) + | Event::MediaData(_) + | Event::RtpPacket(_) + | Event::SenderFeedback(_) + | Event::MediaEgressStats(_) + | Event::MediaIngressStats(_) + | Event::PeerStats(_) + | Event::ChannelBufferedAmountLow(_) + | Event::EgressBitrateEstimate(_) + | Event::KeyframeRequest(_) => { + trace!("{:?}", e) + } + _ => debug!("{:?}", e), + }, + Output::Transmit(t) => { + self.peer_bytes_tx += t.contents.len() as u64; + trace!("OUT {:?}", t) + } + Output::Timeout(_t) => {} + } + + Ok(o) + } + + fn do_poll_output(&mut self) -> Result { + if self.state == RtcState::Closed { + self.last_timeout_reason = Reason::NotHappening; + return Ok(Output::Timeout(not_happening())); + } + + while let Some(e) = self.ice.poll_event() { + match e { + IceAgentEvent::IceRestart(_) => { + // + } + IceAgentEvent::IceConnectionStateChange(v) => { + return Ok(Output::Event(Event::IceConnectionStateChange(v))); + } + IceAgentEvent::DiscoveredRecv { proto, source } => { + debug!("ICE remote address: {:?}/{:?}", Pii(source), proto); + self.remote_addrs.push(source); + while self.remote_addrs.len() > 20 { + self.remote_addrs.remove(0); + } + } + IceAgentEvent::NominatedSend { + proto, + source, + destination, + } => { + debug!( + "ICE nominated send from: {:?} to: {:?} with protocol {:?}", + Pii(source), + Pii(destination), + proto, + ); + self.send_addr = Some(SendAddr { + proto, + source, + destination, + }); + } + } + } + + // Handle DTLS timeout before polling output so any retransmit packets + // queued by dimpl and the re-armed flight timer are picked up by the + // poll loop below in the same iteration. + if let Some(timeout) = self.next_dtls_timeout { + if timeout <= self.last_now { + self.next_dtls_timeout = None; + + if let Err(error) = self.dtls.handle_timeout(self.last_now) { + // A failed timer transition leaves DTLS unrecoverable. Close the + // RTC so continued polling cannot expose the same due timeout again. + self.disconnect(); + return Err(error.into()); + } + } + } + + // Poll DTLS output - collect packets, handle events + let mut just_connected = false; + loop { + match self.dtls.poll_output(&mut self.dtls_buf) { + DtlsOutput::Packet(_) => { + unreachable!("We don't expect DTLS packets here since we use poll_packet"); + } + DtlsOutput::Connected => { + if !self.dtls_connected { + debug!("DTLS connected"); + self.dtls_connected = true; + just_connected = true; + } + } + DtlsOutput::KeyingMaterial(km, profile) => { + use config::KeyingMaterial; + let km_bytes = km.as_ref().to_vec(); + debug!("DTLS set SRTP keying material and profile: {}", profile); + let active = self.dtls.is_active().expect("DTLS must be inited by now"); + self.session.set_keying_material( + KeyingMaterial::new(&km_bytes), + &self.crypto_provider, + profile, + active, + ); + } + DtlsOutput::PeerCert(der) => { + debug!("DTLS verify remote fingerprint"); + // Compute fingerprint from peer's DER certificate + let fingerprint = crate::crypto::Fingerprint { + hash_func: "sha-256".to_string(), + bytes: self.crypto_provider.sha256_provider.sha256(der).to_vec(), + }; + self.dtls.set_remote_fingerprint(fingerprint.clone()); + if let Some(expected) = &self.remote_fingerprint { + if !self.fingerprint_verification { + debug!("DTLS fingerprint verification disabled"); + } else if fingerprint != *expected { + self.disconnect(); + return Err(RtcError::RemoteSdp("remote fingerprint no match".into())); + } + } else { + self.disconnect(); + return Err(RtcError::RemoteSdp("no a=fingerprint before dtls".into())); + } + } + DtlsOutput::ApplicationData(data) => { + self.sctp.handle_input(self.last_now, data); + } + DtlsOutput::Timeout(t) => { + self.next_dtls_timeout = Some(t); + break; + } + DtlsOutput::CloseNotify => { + self.start_close()?; + return Ok(Output::Event(Event::Closed)); + } + other => { + return Err(RtcError::Dtls(DtlsError::Io(std::io::Error::other( + format!("Unexpected DTLS output: {other:?}"), + )))); + } + } + } + + if just_connected { + return Ok(Output::Event(Event::Connected)); + } + + while let Some(e) = self.sctp.poll() { + match e { + SctpEvent::Transmit { mut packets } => { + if let Some(v) = packets.front() { + if let Err(e) = self.dtls.handle_input(v) { + if is_would_block(&e) { + self.sctp.push_back_transmit(packets); + break; + } else if self.state == RtcState::Closing { + debug!( + "Dropping SCTP transmit while closing after DTLS error: {e}" + ); + packets.pop_front(); + if !packets.is_empty() { + self.sctp.push_back_transmit(packets); + } + return self.do_poll_output(); + } else { + return Err(e.into()); + } + } + + packets.pop_front(); + // If there are still packets, they are sent on next + // poll_output() + if !packets.is_empty() { + self.sctp.push_back_transmit(packets); + } + + // Run again since this would feed the DTLS subsystem + // to produce a packet now. + return self.do_poll_output(); + } + } + SctpEvent::Open { id, label } => { + self.chan.ensure_channel_id_for(id); + let id = self.chan.channel_id_by_stream_id(id).unwrap(); + return Ok(Output::Event(Event::ChannelOpen(id, label))); + } + SctpEvent::Close { + id: stream_id, + reset_pending, + } => { + let Some(channel_id) = self.chan.channel_id_by_stream_id(stream_id) else { + warn!("Drop ChannelClose event for id: {:?}", stream_id); + continue; + }; + // When a reset is outstanding, remove_channel holds the stream id + // back from reallocation until the handshake completes. + self.chan.remove_channel(channel_id, reset_pending); + return Ok(Output::Event(Event::ChannelClose(channel_id))); + } + SctpEvent::StreamResetComplete { id } => { + // The reset handshake completed, the stream id can be used again. + self.chan.stream_reset_complete(id); + continue; + } + SctpEvent::AssociationLost => { + self.chan.association_lost(); + self.start_close()?; + return Ok(Output::Event(Event::Closed)); + } + SctpEvent::Data { id, binary, data } => { + let Some(id) = self.chan.channel_id_by_stream_id(id) else { + warn!("Drop ChannelData event for id: {:?}", id); + continue; + }; + let cd = ChannelData { id, binary, data }; + return Ok(Output::Event(Event::ChannelData(cd))); + } + SctpEvent::BufferedAmountLow { id } => { + let Some(id) = self.chan.channel_id_by_stream_id(id) else { + warn!("Drop BufferedAmountLow for id: {:?}", id); + continue; + }; + return Ok(Output::Event(Event::ChannelBufferedAmountLow(id))); + } + } + } + + if let Some(ev) = self.session.poll_event() { + return Ok(Output::Event(ev)); + } + + // Some polling needs to bubble up errors. + if let Some(ev) = self.session.poll_event_fallible()? { + return Ok(Output::Event(ev)); + } + + if let Some(e) = self.stats.as_mut().and_then(|s| s.poll_output()) { + return Ok(match e { + StatsEvent::Peer(s) => Output::Event(Event::PeerStats(s)), + StatsEvent::MediaIngress(s) => Output::Event(Event::MediaIngressStats(s)), + StatsEvent::MediaEgress(s) => Output::Event(Event::MediaEgressStats(s)), + }); + } + + if let Some(v) = self.ice.poll_transmit() { + return Ok(Output::Transmit(v)); + } + + if let Some(send) = &self.send_addr { + // These can only be sent after we got an ICE connection. + let datagram = None + .or_else(|| self.dtls.poll_packet()) + .or_else(|| self.session.poll_datagram(self.last_now)); + + if let Some(contents) = datagram { + let t = net::Transmit { + proto: send.proto, + source: send.source, + destination: send.destination, + contents, + }; + return Ok(Output::Transmit(t)); + } + } else { + // Don't allow accumulated feedback to build up indefinitely + self.session.clear_feedback(); + } + + let stats_timeout = self.stats.as_mut().and_then(|s| s.poll_timeout()); + + let time_and_reason = (None, Reason::NotHappening) + .soonest((self.next_dtls_timeout, Reason::DTLS)) + .soonest((self.ice.poll_timeout(), Reason::Ice)) + .soonest(self.session.poll_timeout()) + .soonest((self.sctp.poll_timeout(), Reason::Sctp)) + .soonest((self.chan.poll_timeout(&self.sctp), Reason::Channel)) + .soonest((stats_timeout, Reason::Stats)); + + // trace!("poll_output timeout reason: {}", time_and_reason.1); + + let time = time_and_reason.0.unwrap_or_else(not_happening); + let reason = time_and_reason.1; + + // We want to guarantee time doesn't go backwards. + let next = if time < self.last_now { + self.last_now + } else { + time + }; + + if self.state == RtcState::Closing && self.close_drain_complete() { + self.state = RtcState::Closed; + self.last_timeout_reason = Reason::NotHappening; + return Ok(Output::Timeout(not_happening())); + } + + self.last_timeout_reason = reason; + Ok(Output::Timeout(next)) + } + + /// The reason for the last [`Output::Timeout`] + /// + /// This is updated when calling [`Rtc::poll_output()`] and the next output + /// is a timeout. + /// + /// ``` + /// # use str0m::{Rtc, Reason}; + /// # use std::time::Instant; + /// let rtc = Rtc::new(Instant::now()); + /// + /// // Before any call to poll_output(), the reason is the default. + /// assert_eq!(rtc.last_timeout_reason(), Reason::NotHappening); + /// ``` + pub fn last_timeout_reason(&self) -> Reason { + self.last_timeout_reason + } + + /// Check if this `Rtc` instance accepts the given input. This is used for demultiplexing + /// several `Rtc` instances over the same UDP server socket. + /// + /// [`Input::Timeout`] is always accepted. [`Input::Receive`] is tested against the nominated + /// ICE candidate. If that doesn't match and the incoming data is a STUN packet, the accept call + /// is delegated to the ICE agent which recognizes the remote peer from `a=ufrag`/`a=password` + /// credentials negotiated in the SDP. If that also doesn't match, all remote ICE candidates are + /// checked for a match. + /// + /// In a server setup, the server would try to find an `Rtc` instances using [`Rtc::accepts()`]. + /// The first found instance would be given the input via [`Rtc::handle_input()`]. + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::{Rtc, Input}; + /// // A vec holding the managed rtc instances. One instance per remote peer. + /// let now = Instant::now(); + /// let mut rtcs = vec![Rtc::new(now), Rtc::new(now), Rtc::new(now)]; + /// + /// // Configure instances with local ice candidates etc. + /// + /// loop { + /// // TODO poll_timeout() and handle the output. + /// + /// let input: Input = todo!(); // read network data from socket. + /// for rtc in &mut rtcs { + /// if rtc.accepts(&input) { + /// rtc.handle_input(input).unwrap(); + /// } + /// } + /// } + /// ``` + pub fn accepts(&self, input: &Input) -> bool { + let Input::Receive(_, r) = input else { + // always accept the Input::Timeout. + return true; + }; + + // Fast path: DTLS, RTP, and RTCP traffic coming in from the same socket address + // we've nominated for sending via the ICE agent. This is the typical case + if let Some(send_addr) = &self.send_addr { + if r.source == send_addr.destination { + return true; + } + } + + // STUN can use the ufrag/password to identify that a message belongs + // to this Rtc instance. + if let DatagramRecvInner::Stun(v) = &r.contents.inner { + return self.ice.accepts_message(v); + } + + // Slow path: Occasionally, traffic comes in on a socket address corresponding + // to a successful candidate pair other than the one we've currently nominated. + // This typically happens at the beginning of the connection + if self.ice.has_viable_remote_candidate(r.source) { + return true; + } + + false + } + + /// Provide input to this `Rtc` instance. Input is either a [`Input::Timeout`] for some + /// time that was previously obtained from [`Rtc::poll_output()`], or [`Input::Receive`] + /// for network data. + /// + /// Both the timeout and the network data contains a [`std::time::Instant`] which drives + /// time forward in the instance. For network data, the intention is to record the time + /// of receiving the network data as precise as possible. This time is used to calculate + /// things like jitter and bandwidth. + /// + /// It's always okay to call [`Rtc::handle_input()`] with a timeout, also before the + /// time obtained via [`Rtc::poll_output()`]. + /// + /// ```no_run + /// # use str0m::{Rtc, Input}; + /// # use std::time::Instant; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// loop { + /// let timeout: Instant = todo!(); // rtc.poll_output() until we get a timeout. + /// + /// let input: Input = todo!(); // wait for network data or timeout. + /// rtc.handle_input(input); + /// } + /// ``` + pub fn handle_input(&mut self, input: Input) -> Result<(), RtcError> { + if self.state == RtcState::Closed { + return Ok(()); + } + + match input { + Input::Timeout(now) => self.do_handle_timeout(now)?, + Input::Receive(recv_time, r) => { + self.do_handle_receive(recv_time, r)?; + self.do_handle_timeout(recv_time)?; + } + } + Ok(()) + } + + /// Starts local shutdown. + /// + /// The `Rtc` instance schedules RTCP BYE, requests SCTP shutdown, and sends + /// DTLS close_notify without waiting for remote replies. It enters a closing + /// drain state where app-facing operations stop, but [`Rtc::poll_output()`] + /// continues polling until pending local close output has drained. Once the + /// drain is complete, [`Rtc::is_alive()`] returns `false`. + pub fn close(&mut self) -> Result<(), RtcError> { + self.start_close() + } + + fn start_close(&mut self) -> Result<(), RtcError> { + if self.state == RtcState::Closed { + return Ok(()); + } + + if self.state != RtcState::Closing { + self.session.close_rtp(); + self.sctp.close()?; + self.state = RtcState::Closing; + } + + self.start_dtls_close() + } + + fn start_dtls_close(&mut self) -> Result<(), RtcError> { + if !self.close_dtls_started { + self.dtls.close()?; + self.close_dtls_started = true; + } + + Ok(()) + } + + fn close_drain_complete(&self) -> bool { + self.close_dtls_started && self.dtls.is_closed() && self.session.is_rtp_closed() + } + + fn init_time(&mut self, now: Instant) { + // The operation is somewhat expensive, hence we only do it once. + if !self.need_init_time { + return; + } + + // We assume this first "now" is a time 0 start point for calculating ntp/unix time offsets. + // This initializes the conversion of Instant -> NTP/Unix time. + let _ = now.to_unix_duration(); + + self.need_init_time = false; + } + + fn do_handle_timeout(&mut self, now: Instant) -> Result<(), RtcError> { + self.init_time(now); + + // Prevent time from going backwards. + if now < self.last_now { + return Ok(()); + } + + self.last_now = now; + self.ice.handle_timeout(now); + self.sctp.handle_timeout(now); + self.chan.handle_timeout(now, &mut self.sctp); + self.session.handle_timeout(now)?; + + if let Some(stats) = &mut self.stats { + if stats.wants_timeout(now) { + let mut snapshot = StatsSnapshot::new(now); + snapshot.peer_rx = self.peer_bytes_rx; + snapshot.peer_tx = self.peer_bytes_tx; + let current_round_trip_time = self.ice.nominated_pair_rtt(); + let total_round_trip_time = self.ice.nominated_pair_total_rtt().unwrap_or_default(); + let responses_received = self.ice.nominated_pair_responses_received().unwrap_or(0); + snapshot.selected_candidate_pair = + self.send_addr.as_ref().map(|s| CandidatePairStats { + protocol: s.proto, + local: CandidateStats { addr: s.source }, + remote: CandidateStats { + addr: s.destination, + }, + current_round_trip_time, + total_round_trip_time, + responses_received, + }); + self.session.visit_stats(now, &mut snapshot); + stats.do_handle_timeout(&mut snapshot); + } + } + + Ok(()) + } + + fn do_handle_receive(&mut self, recv_time: Instant, r: net::Receive) -> Result<(), RtcError> { + trace!("IN {:?}", r); + use DatagramRecvInner::*; + + let bytes_rx = match r.contents.inner { + // TODO: stun is already parsed (depacketized) here + Stun(_) => 0, + Dtls(v) | Rtp(v) | Rtcp(v) => v.len(), + }; + + self.peer_bytes_rx += bytes_rx as u64; + + match r.contents.inner { + Stun(stun) => { + let packet = is::stun::StunPacket { + proto: r.proto, + source: r.source, + destination: r.destination, + message: stun, + }; + self.ice.handle_packet(recv_time, packet); + } + Dtls(dtls) => self.dtls.handle_receive(dtls)?, + Rtp(rtp) => self.session.handle_rtp_receive(recv_time, rtp), + Rtcp(rtcp) => self.session.handle_rtcp_receive(recv_time, rtcp), + } + + Ok(()) + } + + /// Obtain handle for writing to a data channel. + /// + /// This is first available when a [`ChannelId`] is advertised via [`Event::ChannelOpen`]. + /// The function returns `None` also for IDs from [`SdpApi::add_channel()`]. + /// + /// Incoming channel data is via the [`Event::ChannelData`] event. + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::{Rtc, channel::ChannelId}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// let cid: ChannelId = todo!(); // obtain channel id from Event::ChannelOpen + /// let channel = rtc.channel(cid).unwrap(); + /// // TODO write data channel data. + /// ``` + pub fn channel(&mut self, id: ChannelId) -> Option> { + if self.state != RtcState::Alive { + return None; + } + + let sctp_stream_id = self.chan.stream_id_by_channel_id(id)?; + + if !self.sctp.is_open(sctp_stream_id) { + return None; + } + + Some(Channel::new(sctp_stream_id, self)) + } + + /// Configure the Bandwidth Estimate (BWE) subsystem. + /// + /// Only relevant if BWE was enabled in the [`RtcConfig::enable_bwe()`] + pub fn bwe(&mut self) -> Bwe { + Bwe(self) + } + + fn is_correct_change_id(&self, change_id: usize) -> bool { + self.change_counter == change_id + 1 + } + + fn next_change_id(&mut self) -> usize { + let n = self.change_counter; + self.change_counter += 1; + n + } + + /// The codec configs for sending/receiving data. + /// + /// The configurations can be set with [`RtcConfig`] before setting up the session, and they + /// might be further updated by SDP negotiation. + pub fn codec_config(&self) -> &CodecConfig { + &self.session.codec_config + } +} + +impl PartialEq for Event { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::IceConnectionStateChange(l0), Self::IceConnectionStateChange(r0)) => l0 == r0, + (Self::MediaAdded(m0), Self::MediaAdded(m1)) => m0 == m1, + (Self::MediaData(m1), Self::MediaData(m2)) => m1 == m2, + (Self::ChannelOpen(l0, l1), Self::ChannelOpen(r0, r1)) => l0 == r0 && l1 == r1, + (Self::ChannelData(l0), Self::ChannelData(r0)) => l0 == r0, + (Self::ChannelClose(l0), Self::ChannelClose(r0)) => l0 == r0, + _ => false, + } + } +} + +impl Eq for Event {} + +impl fmt::Debug for Rtc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Rtc").finish() + } +} + +/// Log a CSV like stat to stdout. +/// +/// ```ignore +/// log_stat!("MY_STAT", 1, "hello", 3); +/// ``` +/// +/// will result in the following being printed +/// +/// ```text +/// MY_STAT 1, hello, 3, {unix_timestamp_ms} +/// ```` +/// +/// These logs can be easily grepped for, parsed and graphed, or otherwise analyzed. +/// +/// This macro turns into a NO-OP if the `_internal_dont_use_log_stats` feature is not enabled +macro_rules! log_stat { + ($name:expr, $($arg:expr),+) => { + #[cfg(feature = "_internal_dont_use_log_stats")] + { + use std::time::SystemTime; + use std::io::{self, Write}; + + let now = SystemTime::now(); + let since_epoch = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + let unix_time_ms = since_epoch.as_millis(); + let mut lock = io::stdout().lock(); + write!(lock, "{} ", $name).expect("Failed to write to stdout"); + + $( + write!(lock, "{},", $arg).expect("Failed to write to stdout"); + )+ + writeln!(lock, "{}", unix_time_ms).expect("Failed to write to stdout"); + } + }; +} +pub(crate) use log_stat; + +#[cfg(test)] +#[doc(hidden)] +pub fn init_crypto_default() { + crate::crypto::from_feature_flags().install_process_default(); +} + +#[cfg(test)] +mod test { + use std::panic::UnwindSafe; + + use super::*; + + #[test] + fn rtc_is_send() { + fn is_send(_t: T) {} + fn is_sync(_t: T) {} + is_send(Rtc::new(Instant::now())); + is_sync(Rtc::new(Instant::now())); + } + + #[test] + fn rtc_is_unwind_safe() { + fn is_unwind_safe(_t: T) {} + is_unwind_safe(Rtc::new(Instant::now())); + } + + #[test] + fn event_is_reasonably_sized() { + let n = std::mem::size_of::(); + assert!(n < 490); // Increased to accommodate abs-capture-time fields in ExtensionValues + } +} + +#[cfg(feature = "_internal_test_exports")] +#[allow(missing_docs)] +pub mod _internal_test_exports; + +#[cfg(feature = "unversioned")] +pub mod unversioned { + //! This module provides functionality that is not versioned according to semver. + //! It may change in breaking ways between minor/patch releases, there are no guarantees. + //! USE AT YOUR OWN RISK. + //! + //! To use this module, enable the `unversioned` feature flag in your Cargo.toml. + + pub use super::packet::{ + Depacketizer, H264Depacketizer, H264Packetizer, OpusPacketizer, Packetizer, + Vp8Depacketizer, Vp8Packetizer, + }; +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/media/event.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/event.rs new file mode 100644 index 000000000..7e061f34f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/event.rs @@ -0,0 +1,370 @@ +use std::fmt; +use std::ops::RangeInclusive; +use std::sync::Arc; +use std::time::Instant; + +use crate::packet::MediaKind; +use crate::rtp_::{Direction, ExtensionValues, MediaTime, Mid, Pt, Rid, SenderInfo, SeqNo, Ssrc}; +use crate::sdp::SimulcastLayer as SdpSimulcastLayer; +use crate::sdp::{RestrictionId, Simulcast as SdpSimulcast, SimulcastGroups as SdpSimulcastGroups}; + +use super::PayloadParams; +use crate::format::CodecExtra; + +impl From<&SdpSimulcastLayer> for SimulcastLayer { + fn from(layer: &SdpSimulcastLayer) -> Self { + SimulcastLayer { + rid: Rid::from(layer.restriction_id.0.as_ref()), + attributes: layer.attributes.clone(), + } + } +} + +impl From<&SimulcastLayer> for SdpSimulcastLayer { + fn from(layer: &SimulcastLayer) -> Self { + SdpSimulcastLayer { + restriction_id: RestrictionId::new_active(layer.rid.to_string()), + attributes: layer.attributes.clone(), + } + } +} + +impl From for Simulcast { + fn from(s: SdpSimulcast) -> Self { + let send = s.send.iter().map(Into::into).collect(); + let recv = s.recv.iter().map(Into::into).collect(); + + Simulcast { send, recv } + } +} + +/// A new media appeared in an Rtc session. +/// +/// This event fires both for negotiations triggered by a remote or local offer. +/// +/// Does not fire for application media (data channel). +#[derive(Debug, PartialEq, Eq)] +pub struct MediaAdded { + /// Identifier of the new media. + pub mid: Mid, + + /// The kind of media carried. + pub kind: MediaKind, + + /// Current direction. + pub direction: Direction, + + /// If simulcast is configured, this holds the Rids. + /// + /// `a=simulcast:send h;l` + pub simulcast: Option, +} + +/// A change happening during an SDP re-negotiation. +/// +/// This event fires both for re-negotiations triggered by a remote or local offer. +/// +/// Does not fire for application media (data channel). +#[derive(Debug, PartialEq, Eq)] +pub struct MediaChanged { + /// Identifier of the media. + pub mid: Mid, + + /// Current direction. + pub direction: Direction, +} + +/// Simplified information about the simulcast config from SDP. +/// +/// The [full spec][1] covers many cases that are not used by simple simulcast. +/// +/// [1]: https://datatracker.ietf.org/doc/html/draft-ietf-mmusic-sdp-simulcast-14 +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Simulcast { + /// The layers used for sending simulcast. + pub send: Vec, + /// The layers used for receiving simulcast. + pub recv: Vec, +} + +impl Simulcast { + /// Create a new struct with no layers + pub fn new() -> Simulcast { + Simulcast { + send: vec![], + recv: vec![], + } + } +} + +impl Simulcast { + /// Add a send layer + pub fn add_send_layer(&mut self, layer: SimulcastLayer) { + self.send.push(layer); + } + + /// Add a receive layer + pub fn add_recv_layer(&mut self, layer: SimulcastLayer) { + self.recv.push(layer); + } +} + +/// A simulcast layer which has a RID and optional attributes +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct SimulcastLayer { + /// The layer's rid + pub rid: Rid, + /// The layer's attributes per RFC 8851 + pub attributes: Option>, +} + +impl SimulcastLayer { + /// Creates a new layer with the rid + pub fn new(rid: &str) -> SimulcastLayer { + SimulcastLayer { + rid: Rid::from(rid), + attributes: None, + } + } + + /// Create a new layer builder for setting attributes + pub fn new_with_attributes(rid: &str) -> SimulcastLayerBuilder { + SimulcastLayerBuilder { + rid: Rid::from(rid), + attributes: vec![], + } + } +} + +/// A builder which is used to populate layer attributes +pub struct SimulcastLayerBuilder { + rid: Rid, + // We could use a HashMap but that doesn't preserve order - which we need for tests (to validate the + // resulting SDP). BTreeMap seems like overkill for 4-6 keys, so does HashMap, actually. A simple + // vector of pairs is perfect: preserves order and is efficient enough. + attributes: Vec<(String, String)>, +} + +impl SimulcastLayerBuilder { + /// Maximum video width + pub fn max_width(mut self, max_width: u32) -> Self { + self.update_or_insert("max-width", max_width.to_string()); + self + } + + /// Maximum video height + pub fn max_height(mut self, max_height: u32) -> Self { + self.update_or_insert("max-height", max_height.to_string()); + self + } + + /// Maximum bitrate, in bits per second (not kilobits) + pub fn max_br(mut self, max_br: u32) -> Self { + self.update_or_insert("max-br", max_br.to_string()); + self + } + + /// Maximum frame rate, in frames per second, or 0 if none + pub fn max_fps(mut self, max_fps: u32) -> Self { + self.update_or_insert("max-fps", max_fps.to_string()); + self + } + + /// A custom attribute + pub fn custom(mut self, key: &str, value: &str) -> Self { + self.update_or_insert(key, value.to_string()); + self + } + + /// Build the layer + pub fn build(self) -> SimulcastLayer { + SimulcastLayer { + rid: self.rid, + attributes: if self.attributes.is_empty() { + None + } else { + Some(self.attributes) + }, + } + } + + fn update_or_insert(&mut self, key: &str, value: String) { + for (k, v) in &mut self.attributes { + if k == key { + *v = value; + return; + } + } + + self.attributes.push((key.to_string(), value)); + } +} + +impl Simulcast { + pub(crate) fn into_sdp(self) -> SdpSimulcast { + SdpSimulcast { + send: SdpSimulcastGroups(self.send.iter().map(Into::into).collect()), + recv: SdpSimulcastGroups(self.recv.iter().map(Into::into).collect()), + is_munged: false, + } + } +} + +/// Video or audio data from the remote peer. +/// +/// This is obtained via [`Event::MediaData`][crate::Event::MediaData]. +#[derive(PartialEq, Eq)] +pub struct MediaData { + /// Identifier of the media in the session this media belongs to. + pub mid: Mid, + + /// Payload type (PT) tells which negotiated codec is being used. Each media + /// can carry different codecs, the payload type can theoretically change + /// from one packet to the next. + pub pt: Pt, + + /// Rtp Stream Id (RID) identifies an RTP stream without referring to its + /// Synchronization Source (SSRC). + /// + /// This is a newer standard that is sometimes used in WebRTC to identify + /// a stream. Specifically when using Simulcast in Chrome. + pub rid: Option, + + /// Parameters for the codec. This is used to match incoming PT to outgoing PT. + pub params: PayloadParams, + + /// The RTP media time of this packet. Media time is described as a numerator/denominator + /// quantity. The numerator is the timestamp field from the RTP header, the denominator + /// depends on whether this is an audio or video packet. + /// + /// For audio the timebase is often 48kHz for video it is 90kHz. + pub time: MediaTime, + + /// The time of the [`Input::Receive`][crate::Input::Receive] of the first packet + /// that caused this MediaData. + /// + /// In simple SFU setups this can be used as wallclock for [`Writer::write`][crate::media::Writer]. + pub network_time: Instant, + + /// The (RTP) sequence numbers that made up this data. + pub seq_range: RangeInclusive, + + /// Whether the data is contiguous from the one just previously emitted. If this is false, + /// we got an interruption in RTP packets, and the data may or may not be usable in a decoder + /// without requesting a new keyframe. + /// + /// For audio this flag most likely doesn't matter. + pub contiguous: bool, + + /// The actual packet data a.k.a Frame. + /// + /// Bigger frames don't fit in one UDP packet, thus WebRTC RTP is chopping up codec + /// transmission units into smaller parts. + /// + /// This data is a full depayloaded Frame. + pub data: Arc<[u8]>, + + /// RTP header extensions for this media data. This is taken from the + /// first RTP header. + pub ext_vals: ExtensionValues, + + /// Additional codec specific information + pub codec_extra: CodecExtra, + + /// Sender information from the most recent Sender Report(SR). + /// + /// If no Sender Report(SR) has been received this is [`None`]. + pub last_sender_info: Option, + + /// First packet of a talkspurt, that is the first packet after a silence period during + /// which packets have not been transmitted contiguously. + /// + /// For audio only when dtx or silence suppression is enabled. + pub audio_start_of_talk_spurt: bool, +} + +impl MediaData { + /// Return true if MediaData is keyframe independently of Codec + pub fn is_keyframe(&self) -> bool { + match self.codec_extra { + CodecExtra::None => false, + CodecExtra::H264(h264_extra) => h264_extra.is_keyframe, + CodecExtra::H265(h265_extra) => h265_extra.is_keyframe, + CodecExtra::H266(h266_extra) => h266_extra.is_keyframe, + CodecExtra::Vp8(vp8_extra) => vp8_extra.is_keyframe, + CodecExtra::Vp9(vp9_extra) => vp9_extra.is_keyframe, + CodecExtra::Av1(av1_extra) => av1_extra.is_keyframe, + } + } +} + +/// Details for an incoming a keyframe request (PLI or FIR). +/// +/// This is obtained via the [`Event::KeyframeRequest`][crate::Event::KeyframeRequest]. +/// +/// Sending a keyframe request is done via [`Media::request_keyframe()`][crate::media::Media]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyframeRequest { + /// The media identifier this keyframe request is for. + pub mid: Mid, + + /// Rid the keyframe request is for. Relevant when doing simulcast. + pub rid: Option, + + /// The kind of keyframe request (PLI or FIR). + pub kind: KeyframeRequestKind, +} + +/// Type of keyframe request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyframeRequestKind { + /// Picture Loss Indication (PLI) is a less severe keyframe request that can be + /// automatically generated by an SFU or by the end peer. + Pli, + + /// Full Intra Request (FIR) is a more severe keyframe request that should only + /// be used when it's impossible for an end peer to show a video stream. + Fir, +} + +/// Incoming application-specific Payload-Specific Feedback (PSFB FMT=15, PT=206). +/// +/// Emitted when a non-REMB FMT=15 RTCP PSFB message is received. The payload +/// is opaque and application-defined (RFC 4585 Section 6.4). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppSpecificFeedback { + /// SSRC of the sender of this feedback message. + pub sender_ssrc: Ssrc, + /// SSRC of the media source this feedback relates to. + pub media_ssrc: Ssrc, + /// Application-dependent payload. + pub payload: Arc<[u8]>, +} + +/// Incoming feedback from a sender. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SenderFeedback { + /// The media identifier this feedback is for. + pub mid: Mid, + + /// Rid the feedback is for. Relevant when doing simulcast. + pub rid: Option, + + /// When the RTCP packet containing the included [`SenderInfo`] was received. + pub received_at: Instant, + + /// Information about the sender of this report. + pub sender_info: SenderInfo, +} + +impl fmt::Debug for MediaData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MediaData") + .field("mid", &self.mid) + .field("pt", &self.pt) + .field("rid", &self.rid) + .field("time", &self.time) + .field("len", &self.data.len()) + .finish() + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/media/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/mod.rs new file mode 100644 index 000000000..0d90824cd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/mod.rs @@ -0,0 +1,686 @@ +//! Media (audio/video) related content. + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use std::time::Instant; + +use crate::RtcError; +use crate::change::AddMedia; +use crate::format::CodecConfig; + +use crate::packet::{CodecDepacketizer, DepacketizingBuffer, Payloader, RtpMeta}; +use crate::rtp_::ExtensionMap; +use crate::rtp_::MidRid; +use crate::rtp_::SRTP_BLOCK_SIZE; +use crate::rtp_::SRTP_OVERHEAD; +use str0m_proto::Id; + +use crate::format::PayloadParams; +use crate::format::Vp9PacketizerMode; +use crate::sdp::Simulcast as SdpSimulcast; +use crate::sdp::{MediaLine, Msid}; +use crate::streams::{RtpPacket, Streams}; +use crate::util::already_happened; + +mod event; +pub use event::*; + +mod writer; +pub use writer::Writer; + +pub use crate::packet::MediaKind; +pub use crate::rtp_::{Direction, ExtensionValues, Frequency, MediaTime, Mid, Pt, Rid}; + +/// Mid used for SSRC 0 non-media BWE probes. +/// +/// libwebrtc sends bandwidth estimation probes on SSRC 0 when: +/// - Video m-line with RTX is negotiated +/// - `allow_probe_without_media` is enabled (Chrome default) +/// - No video media packets have been sent yet +/// +/// These probes carry `transport_cc` for TWCC feedback but no real media. +pub(crate) const MID_PROBE: Mid = Mid::from_array(*b"~]probe\0\0\0\0\0\0\0\0\0"); + +#[derive(Debug)] +/// Information about some configured media. +pub struct Media { + // ========================================= RTP level ========================================= + // + /// Identifier of this media. + /// + /// RTP level. + mid: Mid, + + /// Canonical name. + /// + /// RTP level. + cname: String, + + /// Rid that we are expecting to see on incoming RTP packets that map to this mid. + /// Once discovered, we make an entry in `stream_rx`. + /// + /// RTP level. + rids_rx: Rids, + + /// Rid that we can send using the [`Writer`]. + /// + /// RTP level. + rids_tx: Rids, + + // ========================================= SDP level ========================================= + // + /// The index of this media line in the Session::media Vec. + /// + /// SDP property. + index: usize, + + /// "Stream and track" identifiers. + /// + /// This is for _outgoing_ SDP. + /// + /// SDP property. + msid: Msid, + + /// Audio or video. + kind: MediaKind, + + /// Current media direction. + /// + /// Can be altered via negotiation. + /// + /// SDP property. + dir: Direction, + + /// Remote PTs negotiated for this media. + /// + /// This tells us both the desired priority order of payload types + /// as well as which PT the remote side wants (in case they are narrowed). + /// + /// These must have corresponding entries in Session::codec_config. + /// + /// SDP property. + /// + /// If this is empty, the m-line is disabled/rejected (port=0 in SDP). + remote_pts: Vec, + + /// Set when this m-line has been stopped via + /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or + /// rejected by the remote peer. Independent of `remote_pts` so that + /// an explicit stop preserves the negotiated PT list in SDP output + /// (the SDP grammar requires at least one fmt on a port=0 m-line). + stopped: bool, + + /// Remote extmaps negotiated for this media. + /// + /// The corresponding entries must exist in Session::codec_config. + /// + /// These are 1-indexed to be exactly like in the SDP. + remote_exts: ExtensionMap, + + /// [`true`] if this media was created by the remote peer, [`false`] if it was created by us. + remote_created: bool, + + /// Simulcast configuration, if set. + /// + /// SDP property. + simulcast: Option, + + // ========================================= Payloaders, etc ========================================= + // + /// Buffers of incoming RTP packets. These do reordering/jitter buffer and also + /// depayload from RTP to frames. + depayloaders: HashMap<(Pt, Option), DepacketizingBuffer>, + + /// Payloaders for outoing RTP packets. + payloaders: HashMap<(Pt, Option), Payloader>, + + /// Frames to payload. Should typically only be 0 or 1. + to_payload: VecDeque, + + pub(crate) need_open_event: bool, + pub(crate) need_changed_event: bool, + + /// When converting media lines to SDP, it's easier to represent the app m-line + /// as a Media. This field is true when we do that. No Session::medias will have + /// this set to true – they only exist temporarily. + pub(crate) app_tmp: bool, +} + +#[derive(Debug)] +/// Config value for [`Media::rids_rx()`] and [`Media::rids_tx()`] +pub enum Rids { + /// No rid is allowed. + None, + /// Any Rid is allowed. + /// + /// This is the default value for direct API. + Any, + /// These specific [`Rid`] are allowed. + /// + /// This is the default value for Simulcast configured via SDP. + Specific(Vec), +} + +impl Rids { + pub(crate) fn contains(&self, rid: Rid) -> bool { + match self { + Rids::None => false, + Rids::Any => true, + Rids::Specific(v) => v.contains(&rid), + } + } + + pub(crate) fn is_specific(&self) -> bool { + matches!(self, Rids::Specific(_)) + } + + fn add(&mut self, rid: Rid) { + match self { + Rids::None | Rids::Any => { + *self = Rids::Specific(vec![rid]); + } + Rids::Specific(vec) if !vec.contains(&rid) => vec.push(rid), + Rids::Specific(_) => {} + } + } +} + +#[derive(Debug)] +pub(crate) struct ToPayload { + pub pt: Pt, + pub rid: Option, + pub wallclock: Instant, + pub rtp_time: MediaTime, + pub start_of_talk_spurt: bool, + pub data: Arc<[u8]>, + pub ext_vals: ExtensionValues, +} + +impl Media { + /// Identifier of the media. + /// + /// RTP level. + pub fn mid(&self) -> Mid { + self.mid + } + + /// Canonical name. + /// + /// Persistent transport-level identifier for an RTP source. + /// + /// RTP level property. The value is sent in RTCP reports for `StreamTx`. Incoming + /// cnames can be found in [`StreamRx::cname`][crate::rtp::StreamRx::cname]. + pub fn cname(&self) -> &str { + &self.cname + } + + /// Add rid as one we are expecting to receive for this mid. + /// + /// This is used for situations where we don't know the SSRC upfront, such as not having + /// a=ssrc lines in an SDP. Adding a rid means we are dynamically discovering the SSRC from + /// a mid/rid combination in the RTP header extensions. + /// + /// RTP level. + pub fn expect_rid_rx(&mut self, rid: Rid) { + self.rids_rx.add(rid); + } + + /// Rids we are expecting to see on incoming RTP packets that map to this mid. + /// + /// By default this is set to [`Rids::Any`], which changes to [`Rids::Specific`] via SDP negotiation + /// that configures Simulcast where specific rids are expected. + /// + /// RTP level. + pub fn rids_rx(&self) -> &Rids { + &self.rids_rx + } + + /// Rids we are can send via the [`Writer`]. + /// + /// By default this is set to [`Rids::None`], which changes to [`Rids::Specific`] via SDP negotiation + /// that configures Simulcast where specific rids are expected. + /// + /// RTP level. + pub fn rids_tx(&self) -> &Rids { + &self.rids_tx + } + + pub(crate) fn index(&self) -> usize { + self.index + } + + pub(crate) fn msid(&self) -> &Msid { + &self.msid + } + + /// Identifier for the group this Media belongs to. + pub fn stream_id(&self) -> &str { + &self.msid().stream_id + } + + /// Identifier for this Media. Should be unique for the given stream id. + pub fn track_id(&self) -> &str { + &self.msid().track_id + } + + /// Whether this media is audio or video. + /// + /// SDP level property. + pub fn kind(&self) -> MediaKind { + self.kind + } + + /// Current direction. This can be changed using + /// [`SdpApi::set_direction()`][crate::SdpApi::set_direction()] followed by an SDP negotiation. + /// + /// To test whether it's possible to send media with the current direction, use + /// + /// ```no_run + /// # use str0m::media::Media; + /// let media: Media = todo!(); // Get hold of media row. + /// if media.direction().is_sending() { + /// // media.write(...); + /// } + /// ``` + /// + /// SDP level property. + pub fn direction(&self) -> Direction { + self.dir + } + + /// Whether this m-line is disabled/rejected (port=0 in SDP). + /// + /// An m-line is disabled if it has been stopped (via + /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or by the + /// remote peer), or if no codecs matched during negotiation. + /// + /// SDP level property. + pub fn disabled(&self) -> bool { + self.stopped || self.remote_pts.is_empty() + } + + /// Whether this m-line has been stopped. + /// + /// Unlike [`disabled`](Self::disabled) this does not include the "no + /// codecs matched" case - only explicit stop via + /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or a + /// port=0 m-line received from the remote peer. A stopped m-line + /// cannot be reactivated; its slot can however be recycled by a + /// subsequent new m-line (RFC 8829 §5.2.2). + pub fn stopped(&self) -> bool { + self.stopped + } + + pub(crate) fn mark_stopped(&mut self) { + self.stopped = true; + } + + pub(crate) fn simulcast(&self) -> Option<&SdpSimulcast> { + self.simulcast.as_ref() + } + + pub(crate) fn poll_sample( + &mut self, + params: &[PayloadParams], + ) -> Result, RtcError> { + for ((pt, rid), buf) in &mut self.depayloaders { + if let Some(r) = buf.pop() { + let dep = r.map_err(|e| RtcError::Packet(self.mid, *pt, e))?; + let Some(codec) = params.iter().find(|c| c.pt() == *pt) else { + return Ok(None); + }; + return Ok(Some(MediaData { + mid: self.mid, + pt: *pt, + rid: *rid, + params: *codec, + // The depacketized time is in the RTP wire clock rate. For the + // media (samples/frame) API we present it in the codec's nominal + // clock rate. These differ only for G722, which has a 16 kHz + // nominal rate but an 8 kHz RTP clock rate (RFC 3551 §4.5.2). For + // all other codecs this rebase is a no-op. See + // https://en.wikipedia.org/wiki/RTP_payload_formats#cite_note-55 + time: dep.time.rebase(codec.spec().clock_rate), + network_time: dep.first_network_time(), + seq_range: dep.seq_range(), + contiguous: dep.contiguous, + ext_vals: dep.ext_vals(), + codec_extra: dep.codec_extra, + last_sender_info: dep.first_sender_info(), + audio_start_of_talk_spurt: codec.spec().codec.is_audio() + && dep.start_of_talkspurt(), + data: dep.data.into(), + })); + } + } + Ok(None) + } + + pub(crate) fn depayload( + &mut self, + rid: Option, + packet: RtpPacket, + reordering_size_audio: usize, + reordering_size_video: usize, + params: &[PayloadParams], + ) { + if !self.dir.is_receiving() { + return; + } + + let pt = packet.header.payload_type; + + let key = (pt, rid); + + let exists = self.depayloaders.contains_key(&key); + + if !exists { + // This unwrap is ok, because the handle_input doesn't accept the RtpPacket for + // depayloading unless we have matched the PT to one in the session. + let params = params.iter().find(|p| p.pt == pt).unwrap(); + + let codec = params.spec.codec; + + // How many packets to hold back in the jitter buffer. + let hold_back = if codec.is_audio() { + reordering_size_audio + } else { + reordering_size_video + }; + + let mut depack: CodecDepacketizer = codec.into(); + + // Enable DONL for H.265 when sprop-max-don-diff > 0 (RFC 7798 §7.1) + if let CodecDepacketizer::H265(ref mut h265) = depack { + if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 { + h265.with_donl(true); + } + } + + // Enable DONL for H.266 when sprop-max-don-diff > 0 (RFC 9328 §7.2) + if let CodecDepacketizer::H266(ref mut h266) = depack { + if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 { + h266.with_donl(true); + } + } + + let buffer = DepacketizingBuffer::new(depack, hold_back); + + self.depayloaders.insert((pt, rid), buffer); + } + + let meta = RtpMeta { + received: packet.timestamp, + time: packet.time, + seq_no: packet.seq_no, + header: packet.header.clone(), + last_sender_info: packet.last_sender_info, + }; + + for ((other_pt, other_rid), buffer) in &mut self.depayloaders { + if *other_pt != pt && *other_rid == rid { + buffer.push_padding(meta.clone()); + } + } + + // The entry will be there by now. + let buffer = self.depayloaders.get_mut(&key).unwrap(); + + buffer.push(meta, packet.payload); + } + + pub(crate) fn set_cname(&mut self, cname: String) { + self.cname = cname; + } + + pub(crate) fn set_msid(&mut self, msid: Msid) { + self.msid = msid; + } + + pub(crate) fn set_direction(&mut self, new_dir: Direction) { + self.need_changed_event = self.dir != new_dir; + self.dir = new_dir; + } + + pub(crate) fn set_simulcast(&mut self, s: SdpSimulcast) { + debug!("Set simulcast: {:?}", s); + self.simulcast = Some(s); + } + + fn payloader_for( + &mut self, + pt: Pt, + rid: Option, + params: &[PayloadParams], + vp9_mode: Vp9PacketizerMode, + ) -> &mut Payloader { + self.payloaders.entry((pt, rid)).or_insert_with(|| { + // Unwrap is OK, the pt should be checked already when calling this function. + let params = params.iter().find(|p| p.pt == pt).unwrap(); + Payloader::new(params.spec, vp9_mode) + }) + } + + fn set_to_payload(&mut self, to_payload: ToPayload) -> Result<(), RtcError> { + if self.to_payload.len() > 100 { + return Err(RtcError::WriteWithoutPoll); + } + + self.to_payload.push_back(to_payload); + + Ok(()) + } + + pub(crate) fn poll_timeout(&self) -> Option { + if !self.to_payload.is_empty() { + Some(already_happened()) + } else { + None + } + } + + pub(crate) fn do_payload( + &mut self, + streams: &mut Streams, + params: &[PayloadParams], + vp9_mode: Vp9PacketizerMode, + mtu: usize, + ) -> Result<(), RtcError> { + let Some(to_payload) = self.to_payload.pop_front() else { + return Ok(()); + }; + + let ToPayload { pt, rid, .. } = &to_payload; + + let is_audio = self.kind.is_audio(); + + let midrid = MidRid(self.mid, *rid); + + let stream = streams.stream_tx_by_midrid(midrid); + + let Some(stream) = stream else { + return Err(RtcError::NoSenderSource); + }; + + let pt = *pt; + + let payloader = self.payloader_for(pt, *rid, params, vp9_mode); + + let rtp_size: usize = mtu - SRTP_OVERHEAD; + // align to SRTP block size to minimize padding needs + let aligned_mtu: usize = rtp_size - rtp_size % SRTP_BLOCK_SIZE; + + payloader + .push_sample(to_payload, aligned_mtu, is_audio, stream) + .map_err(|e| RtcError::Packet(self.mid, pt, e))?; + + Ok(()) + } + + pub(crate) fn set_remote_pts(&mut self, pts: Vec) { + // Have we already set PTs? + if !self.remote_pts.is_empty() { + return; + } + + // TODO: We should verify the remote peer doesn't suddenly change the PT + // order or removes/adds PTs that weren't there from the start. + debug!("Mid ({}) remote PT order is: {:?}", self.mid, pts); + self.remote_pts = pts; + } + + pub(crate) fn set_remote_extmap(&mut self, exts: ExtensionMap) { + self.remote_exts = exts; + } + + /// The remote PT (payload types) configured for this Media. + /// + /// These are negotiated with the remote peer and is the order the remote prefer them. + /// + /// I.e. these can be fewer than the `PayloadParams` configured for the `Rtc` instance, + /// and in a different order. + pub fn remote_pts(&self) -> &[Pt] { + &self.remote_pts + } + + /// The remote, agreed on, extension map, configured for this Media. + /// + /// For the SDP API, these are negotiated with the remote peer. + /// + /// For the Direct API, these are a clone of the session configured values narrowed by media + /// kind (audio/video). + pub fn remote_extmap(&self) -> &ExtensionMap { + &self.remote_exts + } + + pub(crate) fn remote_created(&self) -> bool { + self.remote_created + } + + pub(crate) fn first_pt_with_rtx(&self, config: &CodecConfig) -> Option { + config + .all_for_kind(self.kind) + // Only consider negotiated PTs + .filter(|p| self.remote_pts.contains(&p.pt)) + // Map to the first PT found in payload params with RTX + .find_map(|p| p.resend().map(|_| p.pt)) + } + + pub(crate) fn reset_depayloader(&mut self, payload_type: Pt, rid: Option) { + // Simply remove the depayloader, it will be re-created on the next RTP packet. + self.depayloaders.remove(&(payload_type, rid)); + } + + pub(crate) fn reset_depayloaders_for_rid(&mut self, rid: Option) { + self.depayloaders + .retain(|(_, existing_rid), _| *existing_rid != rid); + } + + pub(crate) fn set_rid_rx(&mut self, rids: Rids) { + self.rids_rx = rids; + } + + pub(crate) fn set_rid_tx(&mut self, rids: Rids) { + self.rids_tx = rids; + } + + pub(crate) fn add_to_rid_tx(&mut self, rid: Rid) { + self.rids_tx.add(rid) + } +} + +impl Default for Media { + fn default() -> Self { + Self { + mid: Mid::new(), + index: 0, + app_tmp: false, + cname: Id::<20>::random().to_string(), + msid: Msid::random(), + kind: MediaKind::Video, + remote_pts: vec![], + stopped: false, + remote_exts: ExtensionMap::empty(), + remote_created: false, + dir: Direction::SendRecv, + simulcast: None, + rids_rx: Rids::Any, + rids_tx: Rids::None, + payloaders: HashMap::new(), + depayloaders: HashMap::new(), + to_payload: VecDeque::default(), + need_open_event: true, + need_changed_event: false, + } + } +} + +impl Media { + pub(crate) fn from_remote_media_line( + l: &MediaLine, + index: usize, + remote_created: bool, + ) -> Self { + Media { + mid: l.mid(), + index, + // This is not reflected back, and thus added by add_pending_changes(). + // cname, + msid: l.msid().unwrap_or(Msid::random()), + kind: l.typ.clone().into(), + dir: if l.disabled { + Direction::Inactive + } else { + l.direction().invert() // remote direction is reverse. + }, + remote_created, + ..Default::default() + } + } + + // Going from AddMedia to Media for pending in a Change and are sent + // in the offer to the other side. + // + // from_add_media is only used when creating temporary Media to be + // included in the SDP. We don't want to make an _actual_ changes with this. + pub(crate) fn from_add_media(a: AddMedia) -> Self { + Media { + mid: a.mid, + index: a.index, + cname: a.cname, + msid: a.msid, + kind: a.kind, + dir: a.dir, + remote_pts: a.pts, + remote_exts: a.exts, + remote_created: false, + simulcast: a.simulcast.map(|s| s.into_sdp()), + ..Default::default() + } + } + + pub(crate) fn from_app_tmp(mid: Mid, index: usize) -> Media { + Media { + mid, + index, + app_tmp: true, + ..Default::default() + } + } + + pub(crate) fn from_direct_api( + mid: Mid, + index: usize, + kind: MediaKind, + exts: ExtensionMap, + ) -> Media { + Media { + mid, + index, + kind, + dir: Direction::SendRecv, + remote_exts: exts, + ..Default::default() + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/media/writer.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/writer.rs new file mode 100644 index 000000000..429d565f9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/media/writer.rs @@ -0,0 +1,249 @@ +use std::sync::Arc; +use std::time::Instant; + +use crate::RtcError; +use crate::format::PayloadParams; +use crate::rtp_::AbsCaptureTime; +use crate::rtp_::MidRid; +use crate::rtp_::VideoOrientation; +use crate::session::Session; + +use super::{ExtensionValues, KeyframeRequestKind, Media, MediaTime, Mid, Pt, Rid, ToPayload}; + +/// Writer of frame level data. +/// +/// Obtained via [`Rtc::writer`][crate::Rtc::writer]. +/// +/// This is the Frame Level API. For RTP level see +/// [`DirectApi::stream_tx`][crate::change::DirectApi::stream_tx]. +pub struct Writer<'a> { + session: &'a mut Session, + mid: Mid, + rid: Option, + start_of_talkspurt: Option, + ext_vals: ExtensionValues, +} + +impl<'a> Writer<'a> { + /// Create a new writer object. + /// + /// The `mid` parameter is required to have a corresponding media in `self.session`. + pub(crate) fn new(session: &'a mut Session, mid: Mid) -> Self { + Writer { + session, + mid, + rid: None, + start_of_talkspurt: None, + ext_vals: ExtensionValues::default(), + } + } + + /// Get the configured payload parameters for the `mid` this writer is for. + /// + /// For the [`Writer::write()`] call, the `pt` must be set correctly. + pub fn payload_params(&self) -> impl Iterator { + // This unwrap is OK due to the invariant of self.mid being resolvable + let media = self.session.media_by_mid(self.mid).unwrap(); + self.session + .codec_config + .params() + .iter() + .filter(|p| media.remote_pts().contains(&p.pt)) + } + + /// Match the given parameters to the configured parameters for this [`Media`]. + /// + /// In a server scenario, a certain codec configuration might not have the same + /// payload type (PT) for two different peers. We will have incoming data with one + /// PT and need to match that against the PT of the outgoing [`Media`]. + /// + /// This call performs matching and if a match is found, returns the _local_ PT + /// that can be used for sending media. + pub fn match_params(&self, params: PayloadParams) -> Option { + self.session + .codec_config + .match_params(params) + .map(|p| p.pt()) + } + + /// Add on an Rtp Stream Id. This is typically used to separate simulcast layers. + pub fn rid(mut self, rid: Rid) -> Self { + self.rid = Some(rid); + self + } + + /// Add on audio level and voice activity. These values are communicated in the same + /// RTP header extension, hence it makes sense setting both at the same time. + /// + /// Audio level is measured in negative decibel. 0 is max and a "normal" value might be -30. + pub fn audio_level(mut self, audio_level: i8, voice_activity: bool) -> Self { + self.ext_vals.audio_level = Some(audio_level); + self.ext_vals.voice_activity = Some(voice_activity); + self + } + + /// First packet of a talkspurt, that is the first packet after a silence period during + /// which packets have not been transmitted contiguously. + /// + /// For audio only when dtx or silence suppression is enabled. + /// This will set the marker bit in the RTP header. + pub fn start_of_talkspurt(mut self, start_of_talkspurt: bool) -> Self { + self.start_of_talkspurt = Some(start_of_talkspurt); + self + } + + /// Add video orientation. This can be used by a player on the receiver end to decide + /// whether the video requires to be rotated to show correctly. + pub fn video_orientation(mut self, o: VideoOrientation) -> Self { + self.ext_vals.video_orientation = Some(o); + self + } + + /// Set absolute capture time for this frame. + pub fn abs_capture_time(mut self, capture_time: AbsCaptureTime) -> Self { + self.ext_vals.abs_capture_time = Some(capture_time); + self + } + + /// Set the minimum and maximum playout delay values. This can be used by a player + /// on the receiver end to determine the size of the jitter buffer. + pub fn playout_delay(mut self, min: MediaTime, max: MediaTime) -> Self { + self.ext_vals.play_delay_min = Some(min); + self.ext_vals.play_delay_max = Some(max); + self + } + + /// Set a user extension value. + pub fn user_extension_value(mut self, val: T) -> Self { + self.ext_vals.user_values.set(val); + self + } + + /// Write media. + /// + /// This operation fails if the PT doesn't match a negotiated codec, or the RID (`None` or a value) + /// does not match anything negotiated. + /// + /// Regarding `wallclock` and `rtp_time`, the wallclock is the real world time that corresponds to + /// the `MediaTime`. For an SFU, this can be hard to know, since RTP packets typically only + /// contain the media time (RTP time). In the simplest SFU setup, the wallclock could simply + /// be the arrival time of the incoming RTP data (see + /// [`MediaData::network_time`][crate::media::MediaData]). For better synchronization the SFU + /// probably needs to weigh in clock drifts and data provided via the statistics. + /// + /// If you write media before `IceConnectionState` is `Connected` it will be dropped. + /// + /// Panics if [`RtcConfig::set_rtp_mode()`][crate::RtcConfig::set_rtp_mode] is `true`. + pub fn write( + self, + pt: Pt, + wallclock: Instant, + rtp_time: MediaTime, + data: impl Into>, + ) -> Result<(), RtcError> { + // This (indirect) unwrap is OK due to the invariant of self.mid being resolvable + let media = media_by_mid_mut(&mut self.session.medias, self.mid); + + if !self.session.codec_config.has_pt(pt) { + return Err(RtcError::UnknownPt(pt)); + } + + if let Some(rid) = self.rid { + if !media.rids_tx().contains(rid) { + return Err(RtcError::UnknownRid(rid)); + } + } + + let data: Arc<[u8]> = data.into(); + + trace!( + "write {:?} {:?} {:?} time: {:?} len: {}", + self.mid, + self.rid, + pt, + rtp_time, + data.len() + ); + + let to_payload = ToPayload { + pt, + rid: self.rid, + wallclock, + rtp_time, + data, + start_of_talk_spurt: self.start_of_talkspurt.unwrap_or(false), + ext_vals: self.ext_vals, + }; + + media.set_to_payload(to_payload)?; + + Ok(()) + } + + /// Test if the kind of keyframe request is possible. + /// + /// Sending a keyframe request requires the mechanic to be negotiated as a feedback mechanic + /// in the SDP offer/answer dance first. + /// + /// Specifically these SDP lines would enable FIR and PLI respectively (for payload type 96). + /// + /// ```text + /// a=rtcp-fb:96 ccm fir + /// a=rtcp-fb:96 nack pli + /// ``` + pub fn is_request_keyframe_possible(&self, kind: KeyframeRequestKind) -> bool { + self.session.is_request_keyframe_possible(kind) + } + + /// Request a keyframe from a remote peer sending media data. + /// + /// For SDP: This can fail if the kind of request (PLI or FIR), as specified by the + /// [`KeyframeRequestKind`], is not negotiated in the SDP answer/offer for this m-line. + /// + /// To ensure the call will not fail, use [`Writer::is_request_keyframe_possible()`] to + /// check whether the feedback mechanism is enabled. + /// + /// # Example + /// + /// ```no_run + /// # use std::time::Instant; + /// # use str0m::Rtc; + /// # use str0m::media::{Mid, KeyframeRequestKind}; + /// let mut rtc = Rtc::new(Instant::now()); + /// + /// // add candidates, do SDP negotiation + /// let mid: Mid = todo!(); // obtain mid from Event::MediaAdded. + /// + /// let writer = rtc.writer(mid).unwrap(); + /// + /// writer.request_keyframe(None, KeyframeRequestKind::Pli).unwrap(); + /// ``` + pub fn request_keyframe( + &mut self, + rid: Option, + kind: KeyframeRequestKind, + ) -> Result<(), RtcError> { + if !self.is_request_keyframe_possible(kind) { + return Err(RtcError::NotReceivingDirection); + } + + let midrid = MidRid(self.mid, rid); + + let stream = self + .session + .streams + .stream_rx_by_midrid(midrid, false) + .ok_or(RtcError::NoReceiverSource(rid))?; + + stream.request_keyframe(kind); + + Ok(()) + } +} + +/// Get a &mut Media in a slice for a `mid`. +/// +/// `mid` must be resolvable or panic will ensue. +fn media_by_mid_mut(medias: &mut [Media], mid: Mid) -> &mut Media { + medias.iter_mut().find(|m| m.mid() == mid).unwrap() +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/control.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/control.rs new file mode 100644 index 000000000..ea4a4f536 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/control.rs @@ -0,0 +1,91 @@ +use crate::rtp_::Bitrate; + +const PACING_FACTOR: f64 = 1.1; + +/// Target padding rate when media is active. This maintains NAT bindings, RTX state, +/// and allows ALR periodic probes to discover higher bandwidth. +const PADDING_TARGET: Bitrate = Bitrate::bps(50_000); + +pub(crate) struct PacingResult { + /// The bitrate at which the pacer may emit padding **when there is no media queued**. + /// + /// This value is intentionally an *absolute* target used by the pacer, not a “delta to add + /// on top of current media”. In practice the **effective padding** over time is the + /// difference between this target and the media actually sent, because padding is only used + /// to fill gaps (empty queues), not to continuously top-up while media is flowing. + pub padding_rate: Bitrate, + pub pacing_rate: Bitrate, +} + +/// Controls the pacing and padding rates. +pub(crate) struct PacerControl {} + +impl PacerControl { + pub fn new() -> Self { + Self {} + } + + pub fn calculate( + &self, + has_active_media: bool, + estimate: Bitrate, + is_overuse: bool, + ) -> PacingResult { + // ALR periodic probes handle bandwidth discovery, not continuous padding. + let padding_rate = if is_overuse { + // No padding during overuse + Bitrate::ZERO + } else if has_active_media { + // Pad to 50 kbps to maintain NAT bindings and RTX state + PADDING_TARGET + } else { + // No padding when no media is being sent + Bitrate::ZERO + }; + + // Set pacing rate to smooth out media transmission (burst avoidance). + // Must be at least the current BWE estimate * factor, but also high enough + // to allow the padding we want to send. + let min_pacing_rate = estimate * PACING_FACTOR; + let pacing_rate = min_pacing_rate.max(padding_rate); + + PacingResult { + padding_rate, + pacing_rate, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn padding_enabled_with_active_media() { + let c = PacerControl::new(); + let estimate = Bitrate::kbps(1_000); + + let r = c.calculate(true, estimate, false); + + assert_eq!(r.padding_rate, PADDING_TARGET); + } + + #[test] + fn no_padding_without_active_media() { + let c = PacerControl::new(); + let estimate = Bitrate::kbps(1_000); + + let r = c.calculate(false, estimate, false); + + assert_eq!(r.padding_rate, Bitrate::ZERO); + } + + #[test] + fn overuse_suppresses_padding() { + let c = PacerControl::new(); + let estimate = Bitrate::mbps(40); + + let r = c.calculate(true, estimate, true); + assert_eq!(r.padding_rate, Bitrate::ZERO); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/leaky.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/leaky.rs new file mode 100644 index 000000000..1e7597b53 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/leaky.rs @@ -0,0 +1,1370 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use super::Pacer; +use super::PaddingRequest; +use super::QueueState; +use crate::Reason; +use crate::bwe_::ProbeClusterConfig; +use crate::bwe_::ProbeClusterState; +use crate::bwe_::{log_pacer_media_debt, log_pacer_padding_debt}; +use crate::pacer::PacerReason; +use crate::rtp_::{Bitrate, DataSize, MidRid, TwccClusterId}; +use crate::util::Soonest; + +const MAX_BITRATE: Bitrate = Bitrate::gbps(10); +const MAX_DEBT_IN_TIME: Duration = Duration::from_millis(500); +const PADDING_BURST_INTERVAL: Duration = Duration::from_millis(5); +const PACING: Duration = Duration::from_millis(40); + +/// A leaky bucket pacer that can overshoot the target bitrate when required. +pub struct LeakyBucketPacer { + /// Pacing bitrate. + pacing_bitrate: Bitrate, + /// Adjusted pacing bitrate for when we need to drain queues. + adjusted_bitrate: Bitrate, + /// The bitrate at which to send padding packets when the pacing rate isn't being achieved. + padding_bitrate: Bitrate, + /// The last time we refreshed media debt and potentially adjusted the bitrate. + last_handle_time: Option, + /// The last time we indicated that a packet should be sent. + last_emitted: Option, + /// The next time we should send a queued packet. + next_poll_time: Option<(Instant, PacerReason)>, + /// The current media debt. + media_debt: DataSize, + /// The current padding debt. + padding_debt: DataSize, + /// The longest the average packet can spend in the queue before we force it to be drained. + queue_limit: Duration, + /// The queue states given by last handle_timeout. + queue_states: Vec, + /// The next return value for `poll_queue`` + next_poll_queue: Option, + /// Queue of probe clusters waiting to be executed + probe_queue: VecDeque, + /// Last completed probe cluster (to be consumed by check_probe_complete) + completed_probe: Option, + /// Gates poll_queue() until handle_timeout() is called after packet emission. + needs_timeout_before_next_poll: bool, + /// Caches whether we have any queue to send padding on (RTX). + has_padding_queue: bool, +} + +impl Pacer for LeakyBucketPacer { + fn set_pacing_rate(&mut self, pacing_bitrate: Bitrate) { + self.pacing_bitrate = pacing_bitrate; + + // bitrate will be updated on next handle_timeout(). + } + + fn set_padding_rate(&mut self, padding_bitrate: Bitrate) { + self.padding_bitrate = padding_bitrate; + + // bitrate will be updated on next handle_timeout(). + } + + fn poll_timeout(&self) -> (Option, Reason) { + let next_handle_time = self.last_handle_time.map(|lh| lh + PACING); + + let poll_at = self + .next_poll_time + .map(|(t, r)| (Some(t), Reason::Pacer(r))) + .unwrap_or((None, Reason::NotHappening)); + + (next_handle_time, Reason::Pacer(PacerReason::Handle)).soonest(poll_at) + } + + fn handle_timeout( + &mut self, + now: Instant, + iter: impl Iterator, + ) -> Option { + // Clear the gate when time advances + self.needs_timeout_before_next_poll = false; + + // Clear the poll time - it will be recalculated below if needed. + // This is important because if we return early (e.g., next_poll_queue is already set), + // we don't want the old Immediate timeout to keep firing. + self.next_poll_time = None; + + // This is called periodically and whenever packet is queued. + self.queue_states.clear(); + self.queue_states.extend(iter); + + let elapsed = self.update_handle_time_and_get_elapsed(now); + + self.clear_debt(elapsed); + self.maybe_update_adjusted_bitrate(now); + + if let Some(request) = self.maybe_create_padding_request(now) { + self.next_poll_queue = Some(request.midrid); + return Some(request); + } + + if self.next_poll_queue.is_some() { + return None; + } + + let (next_poll_time_and_reason, queue) = self.next_poll(now)?; + + if now < next_poll_time_and_reason.0 { + // We don't set this because between now and the neaxt poll, queue state can change such + // that we should poll a different queue i.e. media could be queued. + self.next_poll_queue = None; + } else { + self.next_poll_queue = queue.map(|q| q.midrid); + } + + self.next_poll_time = Some(next_poll_time_and_reason); + + None + } + + fn poll_queue(&mut self) -> Option<(MidRid, Option)> { + // GATE: Block if we need timeout first + if self.needs_timeout_before_next_poll { + return None; + } + + let next = self.next_poll_queue.take()?; + + // Mark that we need timeout before next poll + self.needs_timeout_before_next_poll = true; + self.request_immediate_timeout(); + + // Capture the cluster ID at poll time, before register_send() might clear it + let cluster_id = self.active_cluster(); + + Some((next, cluster_id)) + } + + fn register_send(&mut self, now: Instant, packet_size: DataSize, _from: MidRid) { + self.last_emitted = Some(now); + + self.media_debt += packet_size; + self.media_debt = self + .media_debt + .min(self.adjusted_bitrate * MAX_DEBT_IN_TIME); + log_pacer_media_debt!(self.media_debt.as_bytes_usize()); + self.add_padding_debt(packet_size); + + // Update active probe state to track this packet + // This ensures probe timing advances correctly even when sending media packets + if let Some(probe) = self.probe_queue.front_mut() { + probe.record_packet(now, packet_size); + } + + // Check if probe is complete and store it for later retrieval + if let Some(cluster_id) = self.check_probe_complete_internal(now) { + self.completed_probe = Some(cluster_id); + } + } + + fn has_padding_queue(&self) -> bool { + self.has_padding_queue + } +} + +impl LeakyBucketPacer { + pub fn new(initial_pacing_bitrate: Bitrate) -> Self { + const DEFAULT_QUEUE_LIMIT: Duration = Duration::from_secs(2); + + Self { + pacing_bitrate: initial_pacing_bitrate, + adjusted_bitrate: Bitrate::ZERO, + padding_bitrate: Bitrate::ZERO, + last_handle_time: None, + last_emitted: None, + next_poll_time: None, + media_debt: DataSize::ZERO, + padding_debt: DataSize::ZERO, + queue_limit: DEFAULT_QUEUE_LIMIT, + queue_states: vec![], + next_poll_queue: None, + probe_queue: VecDeque::new(), + completed_probe: None, + needs_timeout_before_next_poll: true, + has_padding_queue: false, + } + } + + /// Start executing a probe cluster. + /// + /// The pacer will pace at the probe's target bitrate and track packets sent. + /// Probes are queued and executed sequentially. + pub(crate) fn start_probe(&mut self, config: ProbeClusterConfig) { + trace!(?config, "Probe start"); + self.probe_queue.push_back(ProbeClusterState::new(config)); + } + + /// Get the cluster ID of the active probe, if any. + pub(crate) fn active_cluster(&self) -> Option { + self.probe_queue.front().map(|p| p.config().cluster()) + } + + /// Check if the active probe is complete and should be finished. + pub(crate) fn check_probe_complete(&mut self, now: Instant) -> Option { + // Check if we have a completed probe from a previous call + if let Some(cluster_id) = self.completed_probe.take() { + return Some(cluster_id); + } + + // Otherwise check if the active probe just completed + self.check_probe_complete_internal(now) + } + + /// Internal method to check if probe is complete (doesn't consume completed_probe) + fn check_probe_complete_internal(&mut self, now: Instant) -> Option { + let probe = self.probe_queue.front()?; + + if probe.is_complete(now) { + let cluster_id = probe.config().cluster(); + self.probe_queue.pop_front(); + return Some(cluster_id); + } + + None + } + + fn update_handle_time_and_get_elapsed(&mut self, now: Instant) -> Duration { + // Due the calling code this also happens when a packet is queued in any upstream queue. + let Some(previous_handle_time) = self.last_handle_time else { + self.last_handle_time = Some(now); + return Duration::ZERO; + }; + + let elapsed = now - previous_handle_time; + self.last_handle_time = Some(now); + + elapsed + } + + fn clear_debt(&mut self, elapsed: Duration) { + self.media_debt = self + .media_debt + .saturating_sub(self.adjusted_bitrate * elapsed); + self.padding_debt = self + .padding_debt + .saturating_sub(self.padding_bitrate * elapsed); + log_pacer_media_debt!(self.media_debt.as_bytes_usize()); + log_pacer_padding_debt!(self.padding_debt.as_bytes_usize()); + } + + fn next_poll(&self, now: Instant) -> Option<((Instant, PacerReason), Option<&QueueState>)> { + // If we have never sent before, do so immediately on an arbitrary non-empty queue. + if self.last_emitted.is_none() { + let mut queues = self + .queue_states + .iter() + .filter(|q| q.snapshot.packet_count > 0); + + return queues + .next() + .map(|q| ((now, PacerReason::FirstEver), Some(q))); + }; + + let unpaced = self + .queue_states + .iter() + .filter(|qs| qs.unpaced) + .filter_map(|qs| qs.snapshot.first_unsent.map(|t| (t, qs))) + .min_by_key(|(t, _)| *t); + + // Unpaced packets (such as audio by default) are sent immediately. + if let Some((queued_at, qs)) = unpaced { + return Some(((queued_at, PacerReason::Unpaced), Some(qs))); + } + + let non_empty_queue = { + let non_empty_queues = self + .queue_states + .iter() + .filter(|q| q.snapshot.packet_count > 0); + + // Send on the non-empty queue with the lowest priority that, was least recently + // sent on. + non_empty_queues.min_by_key(|q| (q.snapshot.priority, q.snapshot.last_emitted)) + }; + + if let Some(queue) = non_empty_queue { + if self.adjusted_bitrate > Bitrate::ZERO { + // Check if we're actively probing and should use probe-specific timing + let poll_at = if let Some(probe) = self.probe_queue.front() { + // During probe: use absolute time directly from probe state + (probe.next_probe_time(), PacerReason::Probe1) + } else { + // Normal pacing: use relative offset based on debt + let drain_debt_time = self.media_debt / self.adjusted_bitrate; + let next_send_offset = if drain_debt_time > PACING { + // If we have incurred too much debt we need to wait to let it clear out before sending + // again. + drain_debt_time + } else { + Duration::ZERO + }; + + let time = self + .last_handle_time + .map(|h| h + next_send_offset) + .unwrap_or(now); + + (time, PacerReason::Paced) + }; + + return Some((poll_at, Some(queue))); + } + } + + let any_queue_for_padding = self.queue_states.iter().any(|q| q.use_for_padding); + let padding_possible = self.padding_bitrate > Bitrate::ZERO && any_queue_for_padding; + + if !padding_possible { + return None; + } + + // If we're actively probing, use probe timing for padding + if let Some(probe) = self.probe_queue.front() { + let next_probe_time = probe.next_probe_time(); + // We explicitly don't return a queue to poll here. We need another call to + // handle_timeout to request the padding before we can poll the selected queue. + return Some(((next_probe_time, PacerReason::Probe2), None)); + } + + // If all queues are empty and we have a padding rate, wait until we have drained + // both the media debt and padding debt to send some padding. + let mut drain_debt_time = + (self.media_debt / self.adjusted_bitrate).max(self.padding_debt / self.padding_bitrate); + if drain_debt_time.is_zero() { + // Give the main loop some time to do something else e.g. queue media. + drain_debt_time = Duration::from_micros(1); + } + + let padding_at = self + .last_handle_time + .map(|h| h + drain_debt_time) + .unwrap_or(now); + + // We explicitly don't return a queue to poll here. We need another call to + // handle_timeout to request the padding before we can poll the selected queue. + Some(((padding_at, PacerReason::Padding), None)) + } + + fn maybe_update_adjusted_bitrate(&mut self, now: Instant) { + // Use probe's target bitrate if actively probing, otherwise use pacing bitrate + self.adjusted_bitrate = if let Some(probe) = self.probe_queue.front() { + probe.config().target_bitrate() + } else { + self.pacing_bitrate + }; + + let (queue_time, queued_packets, queue_size) = + self.queue_states + .iter() + .fold((Duration::ZERO, 0, DataSize::ZERO), |acc, q| { + ( + acc.0 + q.snapshot.total_queue_time(now), + acc.1 + q.snapshot.packet_count, + acc.2 + DataSize::from(q.snapshot.byte_size), + ) + }); + if queued_packets == 0 { + return; + } + + let avg_queue_time = queue_time / queued_packets; + + // The average time we want the packet in the queue to at most to wait to drain. + let target_queue_wait = + Duration::from_millis(1).max(self.queue_limit.saturating_sub(avg_queue_time)); + // Min data rate to drain what's currently in the queue. + let min_rate = queue_size / target_queue_wait; + if min_rate > self.adjusted_bitrate { + // Min rate exceeds our pacing rate, increase the rate to force drain the queue. + self.adjusted_bitrate = min_rate.clamp(Bitrate::ZERO, MAX_BITRATE); + } + } + + fn add_padding_debt(&mut self, size: DataSize) { + self.padding_debt += size; + self.padding_debt = self + .padding_debt + .min(self.padding_bitrate * MAX_DEBT_IN_TIME); + log_pacer_padding_debt!(self.padding_debt.as_bytes_usize()); + } + + /// Optimistically attempt to create a padding request. + /// + /// Returns `Some(PaddingRequest)` if padding is enabled and the current queue state + /// allows padding, otherwise returns `None`. + fn maybe_create_padding_request(&mut self, now: Instant) -> Option { + // Queues must be empty. + let all_queues_empty = self + .queue_states + .iter() + .all(|q| q.snapshot.packet_count == 0); + if !all_queues_empty { + return None; + } + + // We must have a queue that supports padding. + let maybe_queue = self + .queue_states + .iter() + .filter(|q| q.use_for_padding) + .max_by_key(|q| q.snapshot.last_emitted); + + // Save whether we have a valid padding queue. + self.has_padding_queue = maybe_queue.is_some(); + + if !self.has_padding_queue { + // No padding queue, no probes. + self.probe_queue.clear(); + } + + let queue = maybe_queue?; + + // Check for PROBE padding FIRST (bypasses debt checks) + // Active probes need padding to hit their target bitrate when there's insufficient media. + if let Some(probe) = self.probe_queue.front_mut() { + // Delegate probe timing and padding calculation to ProbeClusterState + if !probe.should_send_now(now) { + // Not time yet - wait until next_probe_time + return None; + } + + // Get recommended padding amount from ProbeClusterState + // This handles the calculation of how much padding is needed based on probe timing + let padding_size = probe.next_packet(now); + let Some(padding_size) = padding_size else { + // Probe says no padding needed (already sent enough for this interval) + return None; + }; + + return Some(PaddingRequest { + midrid: queue.midrid, + padding: padding_size.as_bytes_usize(), + }); + } + + // Normal padding: requires zero debt + if self.media_debt != DataSize::ZERO || self.padding_debt != DataSize::ZERO { + return None; + } + + if self.padding_bitrate == Bitrate::ZERO { + return None; + } + + // We can generate padding + let padding = (self.padding_bitrate * PADDING_BURST_INTERVAL).as_bytes_usize(); + + Some(PaddingRequest { + midrid: queue.midrid, + padding, + }) + } + + fn request_immediate_timeout(&mut self) { + // Request timeout at the next microsecond to ensure time advances between packets. + // We can't use already_happened() because that would cause the test harness to + // set a very old timestamp, and while lib.rs prevents last_now from going backwards, + // it doesn't force it to advance, so all packets would get the same timestamp. + const MINIMAL_DELTA: Duration = Duration::from_micros(1); + + let Some(time) = self.last_handle_time.map(|t| t + MINIMAL_DELTA) else { + self.next_poll_time = None; + return; + }; + + self.next_poll_time = Some((time, PacerReason::Immediate)); + } +} + +#[cfg(test)] +mod test { + use super::super::{QueuePriority, QueueSnapshot}; + use super::*; + use crate::rtp_::{DataSize, Mid, RtpHeader}; + use queue::{PacketKind, Queue, QueuedPacket}; + use std::time::{Duration, Instant}; + + #[test] + fn test_typical_behavior() { + let now = Instant::now(); + let mut queue = Queue::default(); + // 2,000 bits per second, 10 bytes per pacing interval(40ms) + let mut pacer = LeakyBucketPacer::new((10 * 200).into()); + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(1)); + + assert!( + pacer.poll_queue().is_none(), + "We initially attempt to poll any non-empty queue if we have never sent", + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 1, + 5, + PacketKind::Video, + now + duration_ms(21), + ); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(21), + "First packet should be released because we have no debt", + |packet| { + assert_eq!(packet.header.sequence_number, 1); + }, + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 2, + 8, + PacketKind::Video, + now + duration_ms(27), + ); + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 3, + 25, + PacketKind::Video, + now + duration_ms(28), + ); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(28), + "Second packet should be released because the debt is within tolerance", + |packet| { + assert_eq!(packet.header.sequence_number, 2); + }, + ); + + // We have incurred too much media debt so polling will now fail until the debt can be + // reduced. + assert!( + pacer.poll_queue().is_none(), + "Third packet should not be released because we have too much debt" + ); + + // Periodic timeout + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(41)); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(41), + "Third packet should be released because we have cleared debt as time moved forward", + |packet| { + assert_eq!(packet.header.sequence_number, 3); + }, + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 4, + 12, + PacketKind::Video, + now + duration_ms(45), + ); + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 5, + 25, + PacketKind::Video, + now + duration_ms(47), + ); + + // We have incurred too much media debt so polling will now fail until the debt can be + // reduced. + assert!( + pacer.poll_queue().is_none(), + "Fourth packet should not be released because we have too much debt" + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 6, + 100, + PacketKind::Audio, + now + duration_ms(52), + ); + + // Unpaced packets should be able to send even if we have too much media debt. + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(52), + "Sixth packet (audio) should be released despite too much media debt because \ + audio packets are not paced", + |packet| { + assert_eq!(packet.kind, PacketKind::Audio); + assert_eq!(packet.header.sequence_number, 6); + }, + ); + + // A lot of time passes, now the bitrate should be adjusted to force drain the queues to + // avoid packets being queued for too long. + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(2053)); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(2053), + "Fourth packet should be released after hitting the queue limit", + |packet| { + assert_eq!(packet.header.sequence_number, 4); + }, + ); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(2053), + "Fifth packet should be released after hitting the queue limit", + |packet| { + assert_eq!(packet.header.sequence_number, 5); + }, + ); + + assert!(queue.is_empty()); + } + + #[test] + fn test_queue_drain() { + let now = Instant::now(); + let mut queue = Queue::default(); + // 2,000 bits per second, 10 bytes per pacing interval(40ms) + let mut pacer = LeakyBucketPacer::new((10 * 200).into()); + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(1)); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 1, + 22, + PacketKind::Video, + now + duration_ms(21), + ); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(21), + "First packet should be released because we have no debt", + |packet| { + assert_eq!(packet.header.sequence_number, 1); + }, + ); + + // Time moves forward + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(41)); + + // Nothing happens for a while because there's nothing in the queues. + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 2, + 8, + PacketKind::Video, + // Debt will be just slightly above what can be drained in 40 ms + // after 66ms + now + duration_ms(66), + ); + + assert!( + pacer.poll_queue().is_none(), + "Second packet should not be released because there's too much debt" + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 3, + 5, + PacketKind::Video, + now + duration_ms(70), + ); + // Drain packet 2 + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(70), + "Second packet should be released because of the adjusted bitrate to drain the queue", + |packet| { + assert_eq!(packet.header.sequence_number, 2); + }, + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 4, + 1200, + PacketKind::Video, + now + duration_ms(71), + ); + + // Drain packet 3 + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(71), + "Third packet should be released because of the adjusted bitrate to drain the queue", + |packet| { + assert_eq!(packet.header.sequence_number, 3); + }, + ); + + // Drain packet 4 + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(71), + "Fourth packet should be released because of the adjusted bitrate to drain the queue", + |packet| { + assert_eq!(packet.header.sequence_number, 4); + }, + ); + + // Time moves forward + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(81)); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 5, + 40, + PacketKind::Video, + now + duration_ms(81), + ); + + assert!( + pacer.poll_queue().is_none(), + "Fifth packet shoud not be relaesed because there's too much debt" + ); + } + + #[test] + fn test_padding_fill_in() { + let now = Instant::now(); + let mut queue = Queue::default(); + let mut pacer = LeakyBucketPacer::new((10 * 200).into()); + // 2,000 bits per second, 10 bytes per pacing interval(40ms) with padding at 3,000 bits per + // second, 15 bytes per pacing interval(40ms) + pacer.set_pacing_rate((10 * 200).into()); + pacer.set_padding_rate((15 * 200).into()); + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(1)); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 1, + 22, + PacketKind::Video, + now + duration_ms(21), + ); + + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(21), + "First packet should be released because we have no debt", + |packet| { + assert_eq!(packet.header.sequence_number, 1); + }, + ); + + // Time moves forward + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(41)); + + // Nothing happens for a while because there's nothing in the queues. + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 2, + 8, + PacketKind::Video, + now + duration_ms(70), + ); + + // Drain packet 2 + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(70), + "Second packet should be released because of the adjusted bitrate to drain the queue", + |packet| { + assert_eq!(packet.header.sequence_number, 2); + }, + ); + + // Time moves forward, all debt is cleared out now + handle_timeout_noisy(&mut pacer, &mut queue, now + duration_ms(155)); + + // Drain padding packet + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(165), + "The queued padding packet should be drained", + |packet| { + assert_eq!(packet.size(), 2); + assert_eq!(packet.header.sequence_number, 0); + }, + ); + + enqueue_packet_noisy( + &mut pacer, + &mut queue, + 3, + 15, + PacketKind::Video, + now + duration_ms(165), + ); + + // Drain packet 3 + assert_poll_success( + &mut pacer, + &mut queue, + now + duration_ms(165), + "Third packet should be released because the sent padding doesn't \ + increase the media debt too much", + |packet| { + assert_eq!(packet.header.sequence_number, 3); + }, + ); + } + + #[test] + fn test_realistic() { + let config = RealisticTestConfig { + padding_rate: Bitrate::kbps(2500), + max_overshoot_factor: 0.05, + spike_probability: 3, + ..Default::default() + }; + let (media_rate, padding_rate, total_rate) = run_realistic_test(config); + let expected_padding = config.padding_rate - config.media_rate; + // Expect result to be within 2 standard deviations. + let upper_bound = + config.media_rate + expected_padding * (1.0 + config.max_overshoot_factor * 2.0) as f64; + let lower_bound = + config.media_rate + expected_padding * (1.0 - config.max_overshoot_factor * 2.0) as f64; + + assert!( + total_rate >= lower_bound && total_rate <= upper_bound, + "Expected reuslting total rate to be within expected bounds. \ + total_rate={total_rate}, media_rate={media_rate}, padding_rate={padding_rate}, \ + config={config:?}, lower_bound={lower_bound}, upper_bound={upper_bound}" + ); + } + + #[test] + fn test_queue_state_merge() { + let now = Instant::now(); + + let mut state = QueueState { + midrid: MidRid(Mid::from("001"), None), + unpaced: false, + use_for_padding: true, + snapshot: QueueSnapshot { + created_at: now, + byte_size: 10_usize, + packet_count: 1332, + total_queue_time_origin: duration_ms(1_000), + last_emitted: Some(now + duration_ms(500)), + first_unsent: None, + priority: QueuePriority::Media, + }, + }; + + let other = QueueState { + midrid: MidRid(Mid::from("002"), None), + unpaced: false, + use_for_padding: false, + snapshot: QueueSnapshot { + created_at: now, + byte_size: 30_usize, + packet_count: 5, + total_queue_time_origin: duration_ms(337), + last_emitted: None, + first_unsent: Some(now + duration_ms(19)), + priority: QueuePriority::Padding, + }, + }; + + state.snapshot.merge(&other.snapshot); + + assert_eq!(state.midrid.mid(), Mid::from("001")); + assert_eq!(state.snapshot.byte_size, 40_usize); + assert_eq!(state.snapshot.packet_count, 1337); + assert_eq!(state.snapshot.total_queue_time_origin, duration_ms(1337)); + + assert_eq!(state.snapshot.last_emitted, Some(now + duration_ms(500))); + assert_eq!(state.snapshot.first_unsent, Some(now + duration_ms(19))); + assert_eq!(state.snapshot.priority, QueuePriority::Media); + } + + #[test] + fn test_priority_ordering() { + assert!(QueuePriority::Media < QueuePriority::Padding); + assert!(QueuePriority::Media < QueuePriority::Empty); + assert!(QueuePriority::Padding < QueuePriority::Empty); + } + + fn assert_poll_success( + pacer: &mut impl Pacer, + queue: &mut Queue, + now: Instant, + msg: &str, + do_asserts: F, + ) -> Instant + where + F: Fn(QueuedPacket), + { + let (qid, _cluster_id) = pacer.poll_queue().expect(msg); + let packet = queue.next_packet().unwrap(); + let packet_size = packet.size(); + do_asserts(packet); + pacer.register_send(now, DataSize::from(packet_size), qid); + queue.register_send(qid, now); + + let timeout = pacer.poll_timeout().0; + // After gating, the pacer requests a timeout at now + 1µs to ensure time advances + const MINIMAL_DELTA: Duration = Duration::from_micros(1); + assert!( + timeout <= Some(now + MINIMAL_DELTA) && timeout.is_some(), + "After a successful send the pacer should return an immediate timeout" + ); + + // Simulate an immediate call to handle_timeout + handle_timeout_noisy(pacer, queue, now); + + timeout.unwrap() + } + + fn enqueue_packet_noisy( + pacer: &mut impl Pacer, + queue: &mut Queue, + seq_no: u16, + size: usize, + kind: PacketKind, + now: Instant, + ) { + let (header, payload_len, kind) = make_packet(seq_no, size, kind); + + let queued_packet = QueuedPacket { + queued_at: now, + header, + payload_len, + kind, + }; + queue.enqueue_packet(queued_packet); + + // Matches the queueing behavior when the pacer is used in real code. + // Each packet being queued causes time to move forward in the pacer and the queue. + handle_timeout_noisy(pacer, queue, now); + } + + fn handle_timeout_noisy(pacer: &mut impl Pacer, queue: &mut Queue, now: Instant) { + queue.update_average_queue_time(now); + if let Some(padding_request) = pacer.handle_timeout(now, queue.queue_state(now)) { + queue.generate_padding(padding_request.padding, now); + + let timeout = pacer.poll_timeout().0; + if timeout.map(|t| t <= now).unwrap_or(false) { + // Refresh queue state + pacer.handle_timeout(now, queue.queue_state(now)); + } + } + } + + fn duration_ms(ms: u64) -> Duration { + Duration::from_millis(ms) + } + + fn make_packet(seq_no: u16, size: usize, kind: PacketKind) -> (RtpHeader, usize, PacketKind) { + let header = RtpHeader { + sequence_number: seq_no, + ..Default::default() + }; + + (header, size, kind) + } + + #[derive(Debug, Clone, Copy)] + struct RealisticTestConfig { + media_rate: Bitrate, + padding_rate: Bitrate, + duration: Duration, + // Spike probability as a percentage + spike_probability: u8, + max_overshoot_factor: f32, + frame_pacing: Duration, + } + + impl Default for RealisticTestConfig { + fn default() -> Self { + RealisticTestConfig { + media_rate: Bitrate::kbps(250), + padding_rate: Bitrate::kbps(800), + duration: Duration::from_secs(10), + spike_probability: 0, + max_overshoot_factor: 0.25, + frame_pacing: Duration::from_millis(33), // ~30 FPS + } + } + } + + /// Run a realistic test of the pacer with simulated media. + /// + /// Returns the media rate, padding, rate, and total rate achieved by the test. + fn run_realistic_test(config: RealisticTestConfig) -> (Bitrate, Bitrate, Bitrate) { + let RealisticTestConfig { + media_rate, + padding_rate, + duration, + spike_probability, + max_overshoot_factor, + frame_pacing, + } = config; + + let base = Instant::now(); + let mut queue = Queue::default(); + let mut pacer = LeakyBucketPacer::new(media_rate); + pacer.set_pacing_rate(padding_rate); + pacer.set_padding_rate(padding_rate); + + let mut last_media_at = base - frame_pacing - Duration::from_millis(1); + let mut media_sent = DataSize::ZERO; + let mut padding_sent = DataSize::ZERO; + let mut elapsed = Duration::ZERO; + + let generate_padding = |queue: &mut Queue, now: Instant, request: PaddingRequest| { + let rand: f32 = fastrand::f32(); + let overshoot_factor: f32 = rand * max_overshoot_factor; + let final_size = ((request.padding as f32) * (1.0 + overshoot_factor).round()) as usize; + queue.generate_padding(final_size, now); + }; + + loop { + if elapsed > duration { + break; + } + + let timeout = { + if let Some((midrid, _cluster_id)) = pacer.poll_queue() { + let packet = queue + .next_packet() + .unwrap_or_else(|| panic!("Should have a packet for {:?}", midrid)); + queue.register_send(midrid, base + elapsed); + queue.update_average_queue_time(base + elapsed); + pacer.register_send( + base + elapsed, + DataSize::bytes(packet.payload_len as i64), + midrid, + ); + if packet.kind == PacketKind::Padding { + padding_sent += packet.payload_len.into(); + } else { + media_sent += packet.payload_len.into(); + } + continue; + } + + pacer.poll_timeout() + }; + + let sleep_until_poll = timeout + .0 + .map(|t| t.duration_since(base + elapsed)) + .unwrap_or(Duration::ZERO); + + let sleep_until_media = + frame_pacing.saturating_sub((base + elapsed).duration_since(last_media_at)); + + if sleep_until_poll < sleep_until_media { + elapsed += sleep_until_poll; + + queue.update_average_queue_time(base + elapsed); + if let Some(padding_request) = + pacer.handle_timeout(base + elapsed, queue.queue_state(base + elapsed)) + { + generate_padding(&mut queue, base + elapsed, padding_request); + } + continue; + } else { + elapsed += sleep_until_media; + } + + let large_overshoot = (fastrand::u8(..) % 100) >= (100 - spike_probability); + let mut to_add = if large_overshoot { + (media_rate * 2.5) * frame_pacing + } else { + media_rate * frame_pacing + }; + + while to_add > DataSize::ZERO { + let packet_size = to_add.min(DataSize::bytes(1100)); + let (header, size, kind) = + make_packet(0, packet_size.as_bytes_usize(), PacketKind::Video); + queue.enqueue_packet(QueuedPacket { + queued_at: base + elapsed, + header, + payload_len: size, + kind, + }); + to_add -= packet_size; + } + last_media_at = base + elapsed; + } + + let observed_media_rate = media_sent / duration; + let observed_padding_rate = padding_sent / duration; + let total_rate = (media_sent + padding_sent) / duration; + + (observed_media_rate, observed_padding_rate, total_rate) + } + + /// A packet queue for use in tests of the pacer. + mod queue { + use std::collections::VecDeque; + use std::time::{Duration, Instant}; + + use crate::rtp_::{DataSize, RtpHeader}; + + use super::*; + + // A packet queue + pub(super) struct Queue { + /// Queue for audio packets + audio_queue: Inner, + /// Queue for video packets + video_queue: Inner, + /// Queue for padding packets + padding_queue: Inner, + } + + pub(super) struct QueuedPacket { + pub(super) queued_at: Instant, + pub(super) header: RtpHeader, + pub(super) payload_len: usize, + pub(super) kind: PacketKind, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum PacketKind { + Audio, + Video, + Padding, + } + + impl Queue { + pub(super) fn is_empty(&self) -> bool { + self.audio_queue.is_empty() && self.video_queue.is_empty() + } + + pub(super) fn update_average_queue_time(&mut self, now: Instant) { + self.audio_queue.update_average_queue_time(now); + self.video_queue.update_average_queue_time(now); + } + + pub(super) fn enqueue_packet(&mut self, packet: QueuedPacket) { + let queue = self.queue_for_kind_mut(packet.kind); + queue.enqueue(packet); + } + + pub(super) fn next_packet(&mut self) -> Option { + if !self.audio_queue.is_empty() { + self.audio_queue.pop_packet() + } else if !self.video_queue.is_empty() { + self.video_queue.pop_packet() + } else { + self.padding_queue.pop_packet() + } + } + + pub(super) fn queue_state(&self, now: Instant) -> impl Iterator { + vec![ + self.audio_queue.queue_state(now), + self.video_queue.queue_state(now), + self.padding_queue.queue_state(now), + ] + .into_iter() + } + + pub(super) fn register_send(&mut self, midrid: MidRid, now: Instant) { + if self.video_queue.midrid == midrid { + self.video_queue.last_emitted = Some(now); + } else if self.audio_queue.midrid == midrid { + self.audio_queue.last_emitted = Some(now); + } else if self.padding_queue.midrid == midrid { + self.padding_queue.last_emitted = Some(now); + } else { + panic!( + "Attempted to register send on unknown queue with id {:?}", + midrid + ); + } + } + + pub(super) fn generate_padding(&mut self, mut pad_size: usize, now: Instant) { + while pad_size > 0 { + let final_packet_size = pad_size.min(1200); + let final_packet_size = DataSize::bytes(final_packet_size as i64); + let (header, payload_len, kind) = + make_packet(0, final_packet_size.as_bytes_usize(), PacketKind::Padding); + self.enqueue_packet(QueuedPacket { + queued_at: now, + header, + payload_len, + kind, + }); + self.update_average_queue_time(now); + + pad_size = pad_size.saturating_sub(final_packet_size.as_bytes_usize()); + } + } + + fn queue_for_kind_mut(&mut self, kind: PacketKind) -> &mut Inner { + match kind { + PacketKind::Audio => &mut self.audio_queue, + PacketKind::Video => &mut self.video_queue, + PacketKind::Padding => &mut self.padding_queue, + } + } + } + + impl Default for Queue { + fn default() -> Self { + Self { + audio_queue: Inner::new( + MidRid(Mid::from("001"), None), + true, + QueuePriority::Media, + ), + video_queue: Inner::new( + MidRid(Mid::from("002"), None), + false, + QueuePriority::Media, + ), + padding_queue: Inner::new( + MidRid(Mid::from("003"), None), + false, + QueuePriority::Padding, + ), + } + } + } + + impl QueuedPacket { + pub(super) fn size(&self) -> usize { + self.payload_len + } + } + + struct Inner { + midrid: MidRid, + last_emitted: Option, + queue: VecDeque, + packet_count: u32, + total_time_spent_queued: Duration, + last_update: Option, + is_audio: bool, + priority: QueuePriority, + } + + impl Inner { + fn new(midrid: MidRid, is_audio: bool, priority: QueuePriority) -> Self { + Self { + midrid, + last_emitted: None, + queue: VecDeque::default(), + packet_count: 0, + total_time_spent_queued: Duration::ZERO, + last_update: None, + is_audio, + priority, + } + } + + fn enqueue(&mut self, packet: QueuedPacket) { + self.queue.push_back(packet); + self.packet_count += 1; + } + + fn pop_packet(&mut self) -> Option { + let packet = self.queue.pop_front()?; + + let time_spent_queued = self + .last_update + .map(|last_update| last_update - packet.queued_at) + .unwrap_or(Duration::ZERO); + self.total_time_spent_queued = self + .total_time_spent_queued + .saturating_sub(time_spent_queued); + self.packet_count -= 1; + + Some(packet) + } + + fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + fn update_average_queue_time(&mut self, now: Instant) { + let Some(last_update) = self.last_update else { + self.last_update = Some(now); + return; + }; + + let elapsed = now - last_update; + self.total_time_spent_queued += elapsed * self.packet_count; + self.last_update = Some(now); + } + + fn queue_state(&self, now: Instant) -> QueueState { + QueueState { + midrid: self.midrid, + unpaced: self.is_audio, + use_for_padding: !self.is_audio && self.last_emitted.is_some(), + snapshot: QueueSnapshot { + created_at: now, + byte_size: self.queue.iter().map(QueuedPacket::size).sum(), + packet_count: self.packet_count, + total_queue_time_origin: self.total_time_spent_queued, + last_emitted: self.last_emitted, + first_unsent: self.queue.iter().next().map(|p| p.queued_at), + priority: self.priority, + }, + } + } + } + + use std::fmt; + + impl fmt::Display for PacketKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PacketKind::Audio => write!(f, "audio"), + PacketKind::Video => write!(f, "video"), + PacketKind::Padding => write!(f, "padding"), + } + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/mod.rs new file mode 100644 index 000000000..88f32b38d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/mod.rs @@ -0,0 +1,164 @@ +use std::time::Instant; + +use crate::Reason; +use crate::bwe_::ProbeClusterConfig; +use crate::rtp_::{Bitrate, DataSize, MidRid, TwccClusterId}; + +mod control; +pub(crate) use control::PacerControl; + +mod null; +use null::NullPacer; + +mod leaky; +use leaky::LeakyBucketPacer; + +mod queue; +pub(crate) use queue::{PaddingRequest, QueuePriority, QueueSnapshot, QueueState}; + +#[allow(clippy::large_enum_variant)] +pub(crate) enum PacerImpl { + Null(NullPacer), + LeakyBucket(LeakyBucketPacer), +} + +impl PacerImpl { + pub fn leaky_bucket(rate: Bitrate) -> PacerImpl { + PacerImpl::LeakyBucket(LeakyBucketPacer::new(rate)) + } + + pub fn null() -> PacerImpl { + PacerImpl::Null(NullPacer::default()) + } + + pub fn start_probe(&mut self, config: ProbeClusterConfig) { + match self { + PacerImpl::Null(_) => { + // NullPacer doesn't support probing + } + PacerImpl::LeakyBucket(v) => v.start_probe(config), + } + } + + pub fn check_probe_complete(&mut self, now: Instant) -> Option { + match self { + PacerImpl::Null(_) => None, + PacerImpl::LeakyBucket(v) => v.check_probe_complete(now), + } + } +} + +impl Pacer for PacerImpl { + fn set_pacing_rate(&mut self, pacing_bitrate: Bitrate) { + match self { + PacerImpl::Null(v) => v.set_pacing_rate(pacing_bitrate), + PacerImpl::LeakyBucket(v) => v.set_pacing_rate(pacing_bitrate), + } + } + + fn set_padding_rate(&mut self, padding_bitrate: Bitrate) { + match self { + PacerImpl::Null(v) => v.set_padding_rate(padding_bitrate), + PacerImpl::LeakyBucket(v) => v.set_padding_rate(padding_bitrate), + } + } + + fn poll_timeout(&self) -> (Option, Reason) { + match self { + PacerImpl::Null(v) => v.poll_timeout(), + PacerImpl::LeakyBucket(v) => v.poll_timeout(), + } + } + + fn handle_timeout( + &mut self, + now: Instant, + iter: impl Iterator, + ) -> Option { + match self { + PacerImpl::Null(v) => v.handle_timeout(now, iter), + PacerImpl::LeakyBucket(v) => v.handle_timeout(now, iter), + } + } + + fn poll_queue(&mut self) -> Option<(MidRid, Option)> { + match self { + PacerImpl::Null(v) => v.poll_queue(), + PacerImpl::LeakyBucket(v) => v.poll_queue(), + } + } + + fn register_send(&mut self, now: Instant, packet_size: DataSize, from: MidRid) { + match self { + PacerImpl::Null(v) => v.register_send(now, packet_size, from), + PacerImpl::LeakyBucket(v) => v.register_send(now, packet_size, from), + } + } + + fn has_padding_queue(&self) -> bool { + match self { + PacerImpl::Null(v) => v.has_padding_queue(), + PacerImpl::LeakyBucket(v) => v.has_padding_queue(), + } + } +} + +/// A packet Pacer. +/// +/// The pacer is responsible for ensuring correct pacing of packets onto the network at a given +/// bitrate. +pub trait Pacer { + /// Set the pacing bitrate. The pacing rate can be exceeded if required to drain excessively + /// long packet queues. + fn set_pacing_rate(&mut self, pacing_bitrate: Bitrate); + + /// Set the padding bitrate to send when there's no media to send + fn set_padding_rate(&mut self, padding_bitrate: Bitrate); + + /// Poll for a timeout. + fn poll_timeout(&self) -> (Option, Reason); + + /// Handle time moving forward, should be called periodically as indicated by [`Pacer::poll_timeout`]. + fn handle_timeout( + &mut self, + now: Instant, + iter: impl Iterator, + ) -> Option; + + /// Determines which mid to poll, if any. + /// + /// Returns the MidRid to poll and the probe cluster ID if this packet belongs to a probe. + fn poll_queue(&mut self) -> Option<(MidRid, Option)>; + + /// Register a packet having been sent. + /// + /// **MUST** be called each time [`Pacer::poll_queue`] produces a mid. + fn register_send(&mut self, now: Instant, packet_size: DataSize, from: MidRid); + + /// Whether we have a queue for padding. + fn has_padding_queue(&self) -> bool; +} + +/// The sub-reason for the [`Reason::Pacer`][crate::Reason::Pacer]. +/// +/// This enum is not considered stable API and may change in minor revisions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum PacerReason { + /// Handle to update pacer budgets. + Handle, + /// First ever call to handle_timeout(). + FirstEver, + /// Unpaced content such as audio. + Unpaced, + /// BWE probe cluster call 1. + Probe1, + /// BWE probe cluster call 2. + Probe2, + /// Regular paced content like video. + Paced, + /// Padding to inflate used bandwidth. + Padding, + /// Immediate timeout for state keeping reasons + Immediate, +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/null.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/null.rs new file mode 100644 index 000000000..4532da67f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/null.rs @@ -0,0 +1,84 @@ +use std::collections::HashMap; +use std::time::Instant; + +use super::Pacer; +use super::PaddingRequest; +use super::QueueState; +use crate::Reason; +use crate::pacer::PacerReason; +use crate::rtp_::{Bitrate, DataSize, MidRid, TwccClusterId}; + +/// A null pacer that doesn't pace. +#[derive(Debug)] +pub struct NullPacer { + last_sends: HashMap, + queue_states: Vec, + needs_timeout_before_next_poll: bool, +} + +impl Default for NullPacer { + fn default() -> Self { + Self { + last_sends: HashMap::default(), + queue_states: Vec::default(), + needs_timeout_before_next_poll: true, + } + } +} + +impl Pacer for NullPacer { + fn set_pacing_rate(&mut self, _padding_bitrate: Bitrate) { + // We don't care + } + + fn set_padding_rate(&mut self, _padding_bitrate: Bitrate) { + // We don't care + } + fn poll_timeout(&self) -> (Option, Reason) { + let time = if self.needs_timeout_before_next_poll { + self.last_sends.values().min().copied() + } else { + None + }; + + (time, Reason::Pacer(PacerReason::Handle)) + } + + fn handle_timeout( + &mut self, + _now: Instant, + iter: impl Iterator, + ) -> Option { + self.needs_timeout_before_next_poll = false; + self.queue_states.clear(); + self.queue_states.extend(iter); + + None + } + + fn poll_queue(&mut self) -> Option<(MidRid, Option)> { + let non_empty_queues = self + .queue_states + .iter() + .filter(|q| q.snapshot.packet_count > 0); + // Pick a queue using round robin, prioritize the least recently sent on queue. + let to_send_on = non_empty_queues.min_by_key(|q| self.last_sends.get(&q.midrid)); + + let result = to_send_on.map(|q| (q.midrid, None)); + + if result.is_some() { + self.needs_timeout_before_next_poll = true; + } + + result + } + + fn register_send(&mut self, now: Instant, _packet_size: DataSize, from: MidRid) { + let e = self.last_sends.entry(from).or_insert(now); + *e = now; + } + + fn has_padding_queue(&self) -> bool { + false + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/queue.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/queue.rs new file mode 100644 index 000000000..ad536406f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/pacer/queue.rs @@ -0,0 +1,108 @@ +use std::time::{Duration, Instant}; + +use crate::rtp_::MidRid; +use crate::util::not_happening; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct QueueSnapshot { + /// Time this snapshot was made + pub created_at: Instant, + /// The total byte size of the snapshot. + pub byte_size: usize, + /// The total number of packets in the queue. + /// NB: This is not a [`usize`] because it will later be used to divide a [`Duration`], for which + /// [`usize`] isn't implement. If the queues end up with 2^32 packets something has gone very wrong + /// in any case. + pub packet_count: u32, + /// Accumulation of all queue time at the time point `created_at`. To use this + /// Look at `total_queue_time(now)` which allows getting the queue time at a later Instant. + pub total_queue_time_origin: Duration, + /// Last time something was emitted from this queue. + pub last_emitted: Option, + /// Time the first unsent packet has spent in the queue. + pub first_unsent: Option, + /// The priority of the most important packet in the queue. + pub priority: QueuePriority, +} + +/// Priority for a given queue. +/// +/// When sorted, higher priority sorts first. +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum QueuePriority { + // Highest, priority for a queue that contains media. + Media = 0, + // Priority for a queue that only contains padding. + Padding = 1, + // Priority for an empty queue. + #[default] + Empty = 2, +} + +impl QueueSnapshot { + /// Update the priority if the snapshot is non-empty. + /// + /// Sets the priority to the provided priority if the queue is non-empty, otherwise empty. + pub fn update_priority(&mut self, priority: QueuePriority) { + if self.packet_count > 0 { + self.priority = priority; + } else { + self.priority = QueuePriority::Empty; + } + } +} + +impl Default for QueueSnapshot { + fn default() -> Self { + Self { + created_at: not_happening(), + byte_size: Default::default(), + packet_count: Default::default(), + total_queue_time_origin: Default::default(), + last_emitted: Default::default(), + first_unsent: Default::default(), + priority: QueuePriority::default(), + } + } +} + +/// The state of a single upstream queue. +/// The pacer manages packets across several upstream queues. +#[derive(Debug, Clone, Copy)] +pub struct QueueState { + pub midrid: MidRid, + pub unpaced: bool, + pub use_for_padding: bool, + pub snapshot: QueueSnapshot, +} + +/// A request to generate a specific amount of padding. +#[derive(Debug, Clone, Copy)] +pub struct PaddingRequest { + /// The Mid that should generate and queue the padding. + pub midrid: MidRid, + /// The amount of padding in bytes to generate. + pub padding: usize, +} + +impl QueueSnapshot { + /// Merge other into self. + pub fn merge(&mut self, other: &Self) { + self.created_at = self.created_at.min(other.created_at); + self.byte_size += other.byte_size; + self.packet_count += other.packet_count; + self.total_queue_time_origin += other.total_queue_time_origin; + self.last_emitted = self.last_emitted.max(other.last_emitted); + self.first_unsent = match (self.first_unsent, other.first_unsent) { + (None, None) => None, + (None, Some(v2)) => Some(v2), + (Some(v1), None) => Some(v1), + (Some(v1), Some(v2)) => Some(v1.min(v2)), + }; + self.priority = self.priority.min(other.priority); + } + + pub fn total_queue_time(&self, now: Instant) -> Duration { + self.total_queue_time_origin + self.packet_count * (now - self.created_at) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/LICENSE-APACHE.txt b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/LICENSE-APACHE.txt new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/LICENSE-APACHE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/av1.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/av1.rs new file mode 100644 index 000000000..69c7a3991 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/av1.rs @@ -0,0 +1,1278 @@ +use super::{BitRead, CodecExtra, Depacketizer, PacketError, Packetizer, encode_leb_u63}; + +const OBU_EXTENSION_PRESENT_MASK: u8 = 0b0000_0100; +const OBU_SIZE_PRESENT_MASK: u8 = 0b0000_0010; +const OBU_TYPE_MASK: u8 = 0b0111_1000; +const AGGREGATION_HEADER_SIZE: usize = 1; +const MAX_NUM_OBUS_TO_OMTI_SIZE: usize = 3; + +/// Detect whether an AV1 RTP payload contains a keyframe. +/// +/// Checks the N bit (new coded video sequence) in the AV1 aggregation header. +/// N=1 indicates the first packet of a keyframe (random access point). +/// +/// AV1 aggregation header layout: `Z|Y|W W|N|reserved` +/// - N (bit 3): 1 = new coded video sequence starts +pub fn detect_av1_keyframe(payload: &[u8]) -> bool { + if payload.is_empty() { + return false; + } + // N bit is bit 3 of the aggregation header + payload[0] & 0x08 != 0 +} + +/// AV1 information describing the depacketized / packetized data +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Av1CodecExtra { + /// Flag which indicates that within [`MediaData`], there is an individual frame + /// containing complete and independent visual information. This frame serves + /// as a reference point for other frames in the video sequence. + /// + /// [`MediaData`]: crate::media::MediaData + pub is_keyframe: bool, +} + +/// AV1 packetizer +#[derive(Default, Debug)] +pub struct Av1Packetizer { + packets: Vec, + obus: Vec, +} + +impl Av1Packetizer { + fn emit(&mut self, mtu: usize, payloads: &mut Vec>) { + if self.obus.is_empty() { + return; + } + + // Clear previous frame data + self.packets.clear(); + + // Reserve 1 byte for aggregation header + let max_payload_size = mtu - AGGREGATION_HEADER_SIZE; + let mut remaining_packet_size = max_payload_size; + + // Push as many obus as possible into each packet + let mut current_packet = Packet::new(0); + for (obu_idx, obu) in self.obus.iter().enumerate() { + let mut previous_obu_extra_size = self.extra_size_for_previous_obu(¤t_packet); + let min_required_size = if current_packet.num_obu_elements >= MAX_NUM_OBUS_TO_OMTI_SIZE + { + 2 + } else { + 1 + }; + + // Check if new packet is needed + if remaining_packet_size < previous_obu_extra_size + min_required_size { + self.packets.push(current_packet); + current_packet = Packet::new(obu_idx); + remaining_packet_size = max_payload_size; + previous_obu_extra_size = 0; + } + + current_packet.packet_size += previous_obu_extra_size; + current_packet.num_obu_elements += 1; + remaining_packet_size -= previous_obu_extra_size; + + let must_write_obu_element_size = + current_packet.num_obu_elements > MAX_NUM_OBUS_TO_OMTI_SIZE; + let mut required_bytes = obu.size; + if must_write_obu_element_size { + required_bytes += leb_128_size(obu.size); + } + + if required_bytes <= remaining_packet_size { + current_packet.last_obu_size = obu.size; + current_packet.packet_size += required_bytes; + remaining_packet_size -= required_bytes; + continue; + } + + // Fragment the obu + let max_first_fragment_size = match must_write_obu_element_size { + true => self.max_fragment_size(remaining_packet_size), + false => remaining_packet_size, + }; + + if max_first_fragment_size == 0 { + current_packet.num_obu_elements -= 1; + current_packet.packet_size -= previous_obu_extra_size; + } else { + current_packet.packet_size += max_first_fragment_size; + if must_write_obu_element_size { + current_packet.packet_size += leb_128_size(max_first_fragment_size); + } + current_packet.last_obu_size = max_first_fragment_size; + } + + // Middle fragments + let mut obu_offset = max_first_fragment_size; + while obu_offset + max_payload_size < obu.size { + self.packets.push(current_packet); + current_packet = Packet::new(obu_idx); + current_packet.num_obu_elements = 1; + current_packet.first_obu_offset = obu_offset; + current_packet.last_obu_size = max_payload_size; + current_packet.packet_size = max_payload_size; + + obu_offset += max_payload_size + } + + // Last fragment + let last_fragment_size = obu.size - obu_offset; + self.packets.push(current_packet); + current_packet = Packet::new(obu_idx); + current_packet.num_obu_elements = 1; + current_packet.first_obu_offset = obu_offset; + current_packet.last_obu_size = last_fragment_size; + current_packet.packet_size = last_fragment_size; + + remaining_packet_size = max_payload_size - last_fragment_size; + } + + self.packets.push(current_packet); + self.write_rtp_payloads(payloads) + } + + fn write_rtp_payloads(&self, payloads: &mut Vec>) { + for (i, packet) in self.packets.iter().enumerate() { + let is_first_packet = i == 0; + let mut rtp_payload: Vec = vec![0u8; AGGREGATION_HEADER_SIZE + packet.packet_size]; + let mut pos = 0; + + let header = self.aggregation_header(packet, &self.obus, is_first_packet); + rtp_payload[pos] = header; + pos += 1; + + let mut obu_offset = packet.first_obu_offset; + // Write all obu elements except the last one + for obu_idx in 0..(packet.num_obu_elements - 1) { + let obu = &self.obus[packet.first_obu + obu_idx]; + let obu_fragment_size = obu.size - obu_offset; + pos += encode_leb_u63(obu_fragment_size as u64, &mut rtp_payload[pos..]); + + if obu_offset == 0 { + rtp_payload[pos] = obu.header & !OBU_SIZE_PRESENT_MASK; + pos += 1; + } + + if obu_offset <= 1 && obu.has_extension() { + rtp_payload[pos] = obu.ext_header; + pos += 1; + } + + let payload_offset = + obu_offset.saturating_sub(if obu.has_extension() { 2 } else { 1 }); + let payload_size = obu.payload.len() - payload_offset; + + if !obu.payload.is_empty() && payload_size > 0 { + rtp_payload[pos..pos + payload_size].copy_from_slice( + &obu.payload[payload_offset..payload_offset + payload_size], + ); + } + + pos += payload_size; + obu_offset = 0; + } + + let last_obu = &self.obus[packet.first_obu + packet.num_obu_elements - 1]; + let mut obu_fragment_size = packet.last_obu_size; + if packet.num_obu_elements > MAX_NUM_OBUS_TO_OMTI_SIZE { + pos += encode_leb_u63(obu_fragment_size as u64, &mut rtp_payload[pos..]); + } + + if obu_offset == 0 && obu_fragment_size > 0 { + rtp_payload[pos] = last_obu.header & !OBU_SIZE_PRESENT_MASK; + pos += 1; + obu_fragment_size -= 1; + } + + if obu_offset <= 1 && last_obu.has_extension() && obu_fragment_size > 0 { + rtp_payload[pos] = last_obu.ext_header; + pos += 1; + obu_fragment_size -= 1; + } + + let payload_offset = + obu_offset.saturating_sub(if last_obu.has_extension() { 2 } else { 1 }); + rtp_payload[pos..pos + obu_fragment_size].copy_from_slice( + &last_obu.payload[payload_offset..payload_offset + obu_fragment_size], + ); + + payloads.push(rtp_payload); + } + } + + fn aggregation_header(&self, packet: &Packet, obus: &[Obu], is_first_packet: bool) -> u8 { + let mut agg_header = 0; + + // set Z flag: the first obu element is the continuation of the previous one + if packet.first_obu_offset > 0 { + agg_header |= 1 << 7; + } + + // set Y flag: the last obu element is continued in the next packet + let last_obu_offset = if packet.num_obu_elements == 1 { + packet.first_obu_offset + } else { + 0 + }; + let last_obu_is_fragment = (last_obu_offset + packet.last_obu_size) + < obus[packet.first_obu + packet.num_obu_elements - 1].size; + if last_obu_is_fragment { + agg_header |= 1 << 6; + } + + // set W field: small number of obu elements in the packet + if packet.num_obu_elements <= MAX_NUM_OBUS_TO_OMTI_SIZE { + agg_header |= packet.num_obu_elements << 4; + } + + // set N flag: + if let Some(ObuType::SequenceHeader) = obus[0].obu_type() { + if is_first_packet { + agg_header |= 1 << 3; + } + } + + agg_header as u8 + } + + /// Adding new OBU to the last packet would mean that the previous one is + /// no longer the final OBU element in that packet. Any OBU that is not the + /// last element must be prefixed with its length. `extra_size_for_previous_obu` + /// computes the number of bytes required to encode that length. + fn extra_size_for_previous_obu(&self, packet: &Packet) -> usize { + if packet.packet_size == 0 { + return 0; + } + + if packet.num_obu_elements > MAX_NUM_OBUS_TO_OMTI_SIZE { + return 0; + } + + leb_128_size(packet.last_obu_size) + } + + // Given the number of free bytes remaining in a packet, the function returns the largest + // OBU fragment size that will fit into the packet + // That is, FragmentSize + Leb128Size(FragmentSize) must not exceed remaining_bytes + fn max_fragment_size(&self, size: usize) -> usize { + if size <= 1 { + return 0; + } + + let mut idx = 1; + loop { + if size < (1 << 7 * idx) + 1 { + return size - idx; + } + idx += 1; + } + } +} + +impl Packetizer for Av1Packetizer { + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu <= AGGREGATION_HEADER_SIZE { + return Ok(vec![]); + } + + let mut payloads = vec![]; + parse_obus(payload, &mut self.obus)?; + self.emit(mtu, &mut payloads); + + Ok(payloads) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +/// AV1 Depacketizer +#[derive(Default, Debug)] +pub struct Av1Depacketizer { + /// current obu payload + obu_buffer: Vec, + + /// reusable buffer for parsed OBUs + parsed_obus: Vec, + + /// length of current obu + obu_length: usize, + + /// aggregation header Z bit + z: bool, + + /// aggregation header Y bit + y: bool, + + /// aggregation header N bit + n: bool, + + /// Number of obus in packet + obu_count: u8, +} + +impl Av1Depacketizer { + fn parse_aggregation_header(&mut self, agg_header: u8) { + // TODO: store these values as mask + self.z = agg_header & (1 << 7) != 0; + self.y = agg_header & (1 << 6) != 0; + self.obu_count = (agg_header & 0b0011_0000) >> 4; + self.n = agg_header & (1 << 3) != 0; + } +} + +impl Depacketizer for Av1Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + codec_extra: &mut super::CodecExtra, + ) -> Result<(), PacketError> { + if packet.is_empty() { + return Err(PacketError::ErrShortPacket); + } + + let mut reader = (packet, 0); + + self.parse_aggregation_header(packet[0]); + reader.skip_bytes(1); + + // if packet does not start with a continuation of an obu fragment + // from the previous packet new obu starts + if !self.z { + self.obu_length = 0; + self.obu_buffer.clear(); + } + + // set the key frame flag if N bit is set and clear the obu buffer + // as it cannot start with a fragment of the previous packet + let mut is_keyframe = matches!( + *codec_extra, + CodecExtra::Av1(Av1CodecExtra { is_keyframe: true }) + ); + if self.n { + is_keyframe = true; + self.obu_length = 0; + self.obu_buffer.clear(); + } + *codec_extra = CodecExtra::Av1(Av1CodecExtra { is_keyframe }); + + let mut obu_idx: usize = 0; + while reader.remaining_bits() > 0 { + let is_first_obu = obu_idx == 0; + let mut is_last_obu = self.obu_count != 0 && obu_idx == (self.obu_count as usize - 1); + + // Read the length of obu + let fragment_obu_length = if self.obu_count == 0 || !is_last_obu { + let len = reader + .get_leb128() + .ok_or(PacketError::ErrAv1CorruptedPacket)?; + + if self.obu_count == 0 && len == reader.remaining_bytes() { + is_last_obu = true; + } + + len + } else { + reader.remaining_bytes() + }; + + // Safety checks + if fragment_obu_length == 0 { + return Err(PacketError::ErrAv1CorruptedPacket); + } + if reader.byte_offset() > packet.len() + || fragment_obu_length > packet.len() - reader.byte_offset() + { + return Err(PacketError::ErrAv1CorruptedPacket); + } + + if is_first_obu && self.z { + // the previous fragment is lost, drop the buffer + if self.obu_buffer.is_empty() { + reader.skip_bytes(fragment_obu_length); + obu_idx = 1; + continue; + } + } + + let offset = reader.byte_offset(); + self.obu_buffer + .extend_from_slice(&packet[offset..offset + fragment_obu_length]); + reader.skip_bytes(fragment_obu_length); + + if is_last_obu && self.y { + self.obu_length += fragment_obu_length; + break; + } + + self.obu_length += fragment_obu_length; + + parse_obus(&self.obu_buffer, &mut self.parsed_obus)?; + let Some(obu) = self.parsed_obus.first_mut() else { + self.obu_length = 0; + self.obu_buffer.clear(); + obu_idx += 1; + continue; + }; + + // Write the obu payload to output + // set size flag + let size_flag_set = obu.header & OBU_SIZE_PRESENT_MASK != 0; + obu.header |= OBU_SIZE_PRESENT_MASK; + out.push(obu.header); + self.obu_length = self.obu_length.saturating_sub(1); + + // add extension if any + if obu.has_extension() { + out.push(obu.ext_header); + self.obu_length = self.obu_length.saturating_sub(1); + } + + // add size if not present + if !size_flag_set { + let mut temp_space = [0u8; 9]; + let bytes_written = encode_leb_u63(self.obu_length as u64, &mut temp_space[..]); + out.extend_from_slice(&temp_space[..bytes_written]); + } + + // finally payload + if self.obu_length > self.obu_buffer.len() { + return Err(PacketError::ErrAv1CorruptedPacket); + } + let start_idx = self.obu_buffer.len() - self.obu_length; + out.extend_from_slice(&self.obu_buffer[start_idx..start_idx + self.obu_length]); + + self.obu_length = 0; + self.obu_buffer.clear(); + + if is_last_obu { + self.obu_length = 0; + self.obu_buffer.clear(); + break; + } + + obu_idx += 1; + } + + Ok(()) + } + + fn is_partition_head(&self, packet: &[u8]) -> bool { + if packet.is_empty() { + return false; + } + + // Z bit in the aggregation header is set to 0 + packet[0] & (1 << 7) == 0 + } + + fn is_partition_tail(&self, marker: bool, _payload: &[u8]) -> bool { + marker + } +} + +#[repr(u8)] +enum ObuType { + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + TileGroup = 4, + MetaData = 5, + Frame = 6, + RedundantFrameHeader = 7, + TileList = 8, + Padding = 15, +} + +impl ObuType { + fn include_in_packetization(self) -> bool { + !matches!( + self, + Self::TemporalDelimiter | Self::TileList | Self::Padding + ) + } +} + +/// Represents an OBU (Open Bitstream Unit) with its header, extension, payload, and size. +#[derive(Default, Debug)] +struct Obu { + header: u8, + ext_header: u8, + payload: Vec, + size: usize, +} + +impl Obu { + fn has_extension(&self) -> bool { + (self.header & OBU_EXTENSION_PRESENT_MASK) != 0 + } + + fn has_size(&self) -> bool { + (self.header & OBU_SIZE_PRESENT_MASK) != 0 + } + + fn obu_type(&self) -> Option { + match (self.header & OBU_TYPE_MASK) >> 3 { + 1 => Some(ObuType::SequenceHeader), + 2 => Some(ObuType::TemporalDelimiter), + 3 => Some(ObuType::FrameHeader), + 4 => Some(ObuType::TileGroup), + 5 => Some(ObuType::MetaData), + 6 => Some(ObuType::Frame), + 7 => Some(ObuType::RedundantFrameHeader), + 8 => Some(ObuType::TileList), + 15 => Some(ObuType::Padding), + _ => None, + } + } +} + +/// The `Packet` struct holds information about the OBU (Open Bitstream Unit) and +/// related metadata necessary for constructing an RTP packet payload during packetization +#[derive(Debug)] +struct Packet { + first_obu: usize, + num_obu_elements: usize, + first_obu_offset: usize, + last_obu_size: usize, + packet_size: usize, +} + +impl Packet { + pub fn new(first_obu: usize) -> Self { + Packet { + first_obu, + num_obu_elements: 0, + first_obu_offset: 0, + last_obu_size: 0, + packet_size: 0, + } + } +} + +/// Returns the number of bytes required to encode a value using unsigned LEB128 encoding. +/// LEB128 uses variable-length encoding with 7 bits per byte, and this function computes +/// how many bytes are needed to represent the given `value`. +pub fn leb_128_size(mut value: usize) -> usize { + let mut size = 0; + + while value >= 0x80 { + size += 1; + value >>= 7; + } + + size + 1 +} + +/// OBU Header Structure (8 bits total): +/// +/// 0 1 2 3 4 5 6 7 +/// +-+-+-+-+-+-+-+-+ +/// |F| T |S|E|R| +/// +-+-+-+-+-+-+-+-+ +/// +/// Explanation of each bit: +/// - F (1 bit) - OBU Forbidden Bit: must be set to 0. +/// - T (4 bits) - OBU Type: This field specifies the type of data structure contained in the OBU payload. +/// - S (1 bit) - OBU Has Size Field: A flag indicating whether the obu_size syntax element will be present. +/// - E (1 bit) - OBU Extension Flag: A flag indicatin if the optional obu_extension_header is present. +/// - R (1 bit) - OBU Reserved Bits: must be set to 0. The value is ignored by a decoder. +fn parse_obus(payload: &[u8], parsed_obus: &mut Vec) -> Result<(), PacketError> { + let mut reader = (payload, 0); + let mut obu_idx = 0; + + while reader.remaining_bits() > 0 { + // Reuse existing Obu entry if available, otherwise grow the Vec + if obu_idx >= parsed_obus.len() { + parsed_obus.push(Obu::default()); + } + let obu = &mut parsed_obus[obu_idx]; + + obu.header = reader.get_u8().ok_or(PacketError::ErrAv1CorruptedPacket)?; + obu.ext_header = 0; + obu.size = 1; + + if obu.has_extension() { + obu.ext_header = reader.get_u8().ok_or(PacketError::ErrAv1CorruptedPacket)?; + obu.size += 1; + } + + // Reuse the existing payload Vec — clear preserves capacity + obu.payload.clear(); + + if obu.has_size() { + let obu_size = reader + .get_leb128() + .ok_or(PacketError::ErrAv1CorruptedPacket)?; + if obu_size > reader.remaining_bytes() { + return Err(PacketError::ErrAv1CorruptedPacket); + } + let bytes = reader + .get_bytes(obu_size) + .ok_or(PacketError::ErrAv1CorruptedPacket)?; + obu.payload.extend_from_slice(bytes); + } else { + obu.payload.extend_from_slice(reader.get_remaining()); + } + obu.size += obu.payload.len(); + + if let Some(obu_type) = obu.obu_type() { + if obu_type.include_in_packetization() { + obu_idx += 1; + } + } + } + + // Trim any leftover entries from previous calls + parsed_obus.truncate(obu_idx); + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + /// AV1 Packetizer tets + #[test] + fn packetize_one_frame_type_obu_without_size() { + let payload = &[0x30, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[0x10, 0x30, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]] + ); + } + + #[test] + fn packetize_one_frame_type_obu_without_size_with_extension() { + let payload = &[0x34, 0x28, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[0x10, 0x34, 0x28, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]] + ); + } + + #[test] + fn packetize_one_frame_type_obu_remove_size_field_without_extension() { + let payload = &[0x32, 0x07, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[0x10, 0x30, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11]] + ); + } + + #[test] + fn packetize_one_frame_type_obu_remove_size_field_with_extension() { + let payload = &[0x36, 0x28, 0x07, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[0x10, 0x34, 0x28, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]] + ); + } + + #[test] + fn omit_size_for_last_obu_when_three_obus_fits_into_the_packet() { + let payload = &[ + 0x0a, 0x06, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // sequence header + 0x2a, 0x04, 0x0b, 0x0c, 0x0d, 0x0e, // metadata + 0x32, 0x06, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, // frame + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[ + 0x38, 0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x05, 0x28, 0x0b, 0x0c, 0x0d, + 0x0e, 0x30, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a + ]] + ); + } + + #[test] + fn use_size_for_all_obus_when_four_obus_fit_into_the_packet() { + let payload = &[ + 0x0a, 0x06, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // sequence header + 0x2a, 0x04, 0x0b, 0x0c, 0x0d, 0x0e, // metadata + 0x1a, 0x03, 0x15, 0x16, 0x17, // frame + 0x22, 0x06, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, // tile group + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[ + 0x08, 0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x05, 0x28, 0x0b, 0x0c, 0x0d, + 0x0e, 0x04, 0x18, 0x15, 0x16, 0x17, 0x07, 0x20, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24 + ]] + ); + } + + #[test] + fn discards_temporal_delimiter_and_tile_list_obu() { + let payload = &[ + 0x12, 0x00, // temporal delimeter + 0x2a, 0x00, // metadata + 0x42, 0x06, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // tile list + 0x1a, 0x03, 0x15, 0x16, 0x17, // frame + 0x22, 0x06, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, // tile group + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(1200, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + [[ + 0x30, 0x01, 0x28, 0x04, 0x18, 0x15, 0x16, 0x17, 0x20, 0x1f, 0x20, 0x21, 0x22, 0x23, + 0x24 + ]] + ); + } + + #[test] + fn split_two_obus_into_two_packets_force_split_obu_header() { + let payload = &[ + 0x1e, 0x28, 0x01, 0x15, // frame + 0x26, 0x28, 0x04, 0x0b, 0x0c, 0x0d, 0x0e, // tile group + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(6, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + vec![ + vec![0x60, 0x03, 0x1c, 0x28, 0x15, 0x24], + vec![0x90, 0x28, 0x0b, 0x0c, 0x0d, 0x0e] + ] + ); + } + + #[test] + fn set_nbit_at_the_first_packet_of_coded_video_sequence() {} + + #[test] + fn split_single_obu_into_two_packets() { + let payload = &[ + 0x32, 0x09, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(8, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + vec![ + vec![0x50, 0x30, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10], + vec![0x90, 0x11, 0x12, 0x13] + ] + ); + } + + #[test] + fn split_single_obu_into_many_packets() { + let mut payload: Vec = vec![0x32, 0xB0, 0x09]; // obu header and leb128 encoded size + payload.extend(vec![27u8; 1200]); // obu payload + + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(100, &payload); + + assert!(result.is_ok()); + assert_eq!(result.unwrap().len(), 13 as usize); + } + + #[test] + fn split_two_obus_into_two_packets() { + let payload = &[ + 0x0a, 0x02, 0x0b, 0x0c, // sequence header + 0x32, 0x09, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, // frame + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(8, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + vec![ + vec![0x68, 0x03, 0x08, 0x0b, 0x0c, 0x30, 0x01, 0x02], + vec![0x90, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09] + ] + ); + } + + #[test] + fn split_single_obu_into_two_packets_because_of_mtu_limit() { + let payload = &[ + 0x32, 0x09, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + ]; + let mut packetizer = Av1Packetizer::default(); + + let result = packetizer.packetize(10, payload); + + assert!(result.is_ok()); + assert_eq!( + result.unwrap(), + vec![ + vec![0x50, 0x30, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12], + vec![0x90, 0x13] + ] + ); + } + + /// AV1 Depacketizer tests + #[test] + fn obu_payload_size_set_when_absent() { + let paylod = &[0x10, 0x30, 0x14, 0x1e, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(paylod, &mut out, &mut codec_extra); + + assert!(result.is_ok()); + assert_eq!(out, vec![0x32, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn obu_payload_size_set_when_present() { + let paylod = &[0x10, 0x32, 0x03, 0x14, 0x1e, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(paylod, &mut out, &mut codec_extra); + + assert!(result.is_ok()); + assert_eq!(out, vec![0x32, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn obu_payload_size_set_after_extension_when_absent() { + let paylod = &[0x10, 0x34, 0x48, 0x14, 0x1e, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(paylod, &mut out, &mut codec_extra); + + assert!(result.is_ok()); + assert_eq!(out, vec![0x36, 0x48, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn obu_payload_size_set_after_extension_when_present() { + let paylod = &[0x10, 0x36, 0x48, 0x03, 0x14, 0x1e, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(paylod, &mut out, &mut codec_extra); + + assert!(result.is_ok()); + assert_eq!(out, vec![0x36, 0x48, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn one_packet_with_two_obus() { + let paylod = &[0x20, 0x02, 0x08, 0x0a, 0x30, 0x14]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(paylod, &mut out, &mut codec_extra); + + assert!(result.is_ok()); + assert_eq!(out, vec![0x0a, 0x01, 0x0a, 0x32, 0x01, 0x14]); + } + + #[test] + fn one_obu_from_two_packets() { + let payload1 = &[0x50, 0x30, 0x14, 0x1e]; + let payload2 = &[0x90, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + + let result = depacketizer.depacketize(payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!(out, vec![0x32, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn two_packets_with_three_obus() { + let payload1 = &[0x60, 0x02, 0x08, 0x0a, 0x30, 0x14, 0x1e]; + let payload2 = &[0x90, 0x28]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + + let result = depacketizer.depacketize(payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!(out, vec![0x0a, 0x01, 0x0a, 0x32, 0x03, 0x14, 0x1e, 0x28]); + } + + #[test] + fn two_packets_with_many_obus_some_with_extensions() { + let payload1 = &[ + 0x40, 0x02, 0x08, 0x0a, 0x02, 0x28, 0x14, 0x04, 0x2c, 0x30, 0x14, 0x1e, 0x05, 0x34, + 0x30, 0x28, 0x32, 0x3c, + ]; + let payload2 = &[0x90, 0x46, 0x50, 0x5a]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + + let result = depacketizer.depacketize(payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!( + out, + vec![ + 0x0a, 0x01, 0x0a, 0x2a, 0x01, 0x14, 0x2e, 0x30, 0x02, 0x14, 0x1e, 0x36, 0x30, 0x06, + 0x28, 0x32, 0x3c, 0x46, 0x50, 0x5a + ] + ); + } + + #[test] + fn one_obu_from_many_packets() { + let payload1 = &[0x50, 0x30, 0x0b, 0x0c]; + let payload2 = &[0xd0, 0x0d, 0x0e]; + let payload3 = &[0xd0, 0x0f, 0x10, 0x11]; + let payload4 = &[0x90, 0x12]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload3, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload4, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!( + out, + vec![0x32, 0x08, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12] + ); + } + + #[test] + fn many_packets_with_border_aligned_obus() { + let payload1 = &[0x60, 0x03, 0x18, 0x0b, 0x0c, 0x20, 0x15, 0x16, 0x17]; + let payload2 = &[0x90, 0x18, 0x19, 0x1a, 0x1b]; + let payload3 = &[0x60, 0x03, 0x38, 0x0b, 0x0c, 0x20, 0x1f, 0x20]; + let payload4 = &[0x90, 0x21, 0x22, 0x23, 0x24]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + let result = depacketizer.depacketize(payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload3, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(payload4, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!( + out, + vec![ + 0x1a, 0x02, 0x0b, 0x0c, 0x22, 0x07, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x3a, + 0x02, 0x0b, 0x0c, 0x22, 0x06, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24 + ] + ); + } + + #[test] + fn one_packet_one_obu_with_payload_size_127_bytes() { + let mut payload = [0; 131]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + payload[0] = 0b0000_0000; // aggregation header + payload[1] = 0x80; // leb128 encoded size of 128 bytes + payload[2] = 0x01; // in two bytes + payload[3] = 0b0011_0000; // obu_header with size and extension bits unset. + payload[4 + 42] = 0x42; // random + + let result = depacketizer.depacketize(&payload, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!(out[0], 0b0011_0010); // size bit set + assert_eq!(out[1], 127); // obu payloa size, 1 byte is enough + assert_eq!(out[44], 0x42); // check random byte + } + + #[test] + fn two_packets_one_obu_with_payload_size_128_bytes() { + let mut payload1 = [0; 35]; + let mut payload2 = [0; 98]; + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + + payload1[0] = 0b0100_0000; // aggregation header + payload1[1] = 33; // payload size + payload1[2] = 0b0011_0000; // obu_header with size and extension bits unset. + payload1[3 + 10] = 0x10; // random + + payload2[0] = 0b1000_0000; + payload2[1] = 96; + payload2[2 + 20] = 0x20; + + let result = depacketizer.depacketize(&payload1, &mut out, &mut codec_extra); + assert!(result.is_ok()); + let result = depacketizer.depacketize(&payload2, &mut out, &mut codec_extra); + assert!(result.is_ok()); + assert_eq!(out[0], 0b0011_0010); + assert_eq!(out[1], 0x80); + assert_eq!(out[2], 0x01); + assert_eq!(out[3 + 10], 0x10); + assert_eq!(out[3 + 32 + 20], 0x20); + } + + // Regression tests for leb_128_size boundary values. + // The bug was `value > 0x80` instead of `value >= 0x80`. + // LEB128 encodes 7 bits per byte, so values >= 128 need 2 bytes, + // values >= 16384 need 3 bytes, etc. + + #[test] + fn leb128_size_boundary_at_128() { + // 127 fits in 1 byte (7 bits), 128 needs 2 bytes + assert_eq!(leb_128_size(0), 1); + assert_eq!(leb_128_size(1), 1); + assert_eq!(leb_128_size(127), 1); + assert_eq!(leb_128_size(128), 2); // was incorrectly 1 before fix + assert_eq!(leb_128_size(129), 2); + assert_eq!(leb_128_size(255), 2); + } + + #[test] + fn leb128_size_boundary_at_16384() { + // 16383 fits in 2 bytes (14 bits), 16384 needs 3 bytes + assert_eq!(leb_128_size(16383), 2); + assert_eq!(leb_128_size(16384), 3); // was incorrectly 2 before fix + assert_eq!(leb_128_size(16385), 3); + } + + #[test] + fn leb128_size_boundary_at_2097152() { + // 2097151 fits in 3 bytes (21 bits), 2097152 needs 4 bytes + assert_eq!(leb_128_size(2097151), 3); + assert_eq!(leb_128_size(2097152), 4); // was incorrectly 3 before fix + } + + // Regression test: packetize an OBU with exactly 128 bytes of payload. + // This triggers the leb_128_size boundary (128 needs 2-byte LEB128). + // Before the fix, leb_128_size(128) returned 1 instead of 2, causing + // the output buffer to be 1 byte too small → panic on slice indexing. + #[test] + fn packetize_obu_with_128_byte_payload_no_panic() { + // Build a raw OBU: header (0x32 = frame, size present) + // + LEB128 size (0x80, 0x01 = 128) + 128 bytes payload + let mut payload = Vec::with_capacity(3 + 128); + payload.push(0x32); // OBU header: frame type, size present + payload.push(0x80); // LEB128 size byte 1: 128 + payload.push(0x01); // LEB128 size byte 2 + payload.extend(vec![0xAB; 128]); // 128 bytes of payload + + let mut packetizer = Av1Packetizer::default(); + let result = packetizer.packetize(1200, &payload); + assert!(result.is_ok()); + let packets = result.unwrap(); + assert!(!packets.is_empty()); + } + + // Regression test: packetize an OBU that requires fragmentation across + // multiple RTP packets, with exactly 128 bytes of payload. + // This exercises both the leb_128_size fix (buffer allocation) and + // the payload_offset fix (correct data in continuation fragments). + #[test] + fn packetize_128_byte_obu_fragmented_correctly() { + // OBU: header + LEB128(128) + 128 bytes payload + let mut payload = Vec::with_capacity(3 + 128); + payload.push(0x32); // frame type, size present + payload.push(0x80); // LEB128: 128 + payload.push(0x01); + for i in 0u8..128 { + payload.push(i); // distinguishable payload bytes + } + + let mut packetizer = Av1Packetizer::default(); + // MTU of 50 forces fragmentation into multiple packets + let result = packetizer.packetize(50, &payload); + assert!(result.is_ok()); + let packets = result.unwrap(); + assert!( + packets.len() >= 3, + "128-byte OBU with MTU 50 should produce at least 3 packets" + ); + + // Verify round-trip: depacketize all packets and check payload integrity + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + for pkt in &packets { + let r = depacketizer.depacketize(pkt, &mut out, &mut codec_extra); + assert!(r.is_ok(), "depacketize failed: {:?}", r); + } + // Output should be: OBU header (with size bit) + LEB128 size + payload + // Find the payload portion and verify it matches + assert!( + out.len() >= 128, + "round-tripped output too short: {} bytes", + out.len() + ); + // The payload bytes should appear in order at the end + let payload_start = out.len() - 128; + for i in 0u8..128 { + assert_eq!( + out[payload_start + i as usize], + i, + "payload byte {} mismatch after round-trip", + i + ); + } + } + + // Regression test for the payload_offset bug in continuation fragments. + // When an OBU with extension header spans multiple packets, the second + // fragment used `obu_offset` instead of `payload_offset` to compute the + // remaining payload size, causing incorrect data or panics. + #[test] + fn packetize_obu_with_extension_fragmented_round_trip() { + // OBU with extension: header=0x36 (frame type + extension + size present), + // ext_header=0x10, LEB128 size, then payload + let mut raw = Vec::with_capacity(4 + 200); + raw.push(0x36); // OBU header: frame, extension present, size present + raw.push(0x10); // extension header + raw.push(0xC8); // LEB128 size: 200 + raw.push(0x01); + for i in 0u8..200 { + raw.push(i); + } + + let mut packetizer = Av1Packetizer::default(); + let result = packetizer.packetize(40, &raw); + assert!(result.is_ok()); + let packets = result.unwrap(); + assert!( + packets.len() >= 5, + "200-byte OBU with MTU 40 should produce multiple packets" + ); + + // Round-trip through depacketizer + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + let mut depacketizer = Av1Depacketizer::default(); + for pkt in &packets { + depacketizer + .depacketize(pkt, &mut out, &mut codec_extra) + .unwrap(); + } + + // Verify the 200-byte payload is intact + let payload_start = out.len() - 200; + for i in 0u8..200 { + assert_eq!( + out[payload_start + i as usize], + i, + "extension OBU payload byte {} corrupted after round-trip", + i + ); + } + } + + #[test] + fn test_detect_av1_keyframe() { + // Empty + assert!(!detect_av1_keyframe(&[])); + + // AV1 aggregation header: Z|Y|W W|N|reserved + // N bit is bit 3 (0x08) + + // N=1 → keyframe + assert!(detect_av1_keyframe(&[0x08])); + assert!(detect_av1_keyframe(&[0x18])); // Z=0,Y=0,W=01,N=1 + assert!(detect_av1_keyframe(&[0x78])); // Z=0,Y=1,W=11,N=1 + assert!(detect_av1_keyframe(&[0x88])); // Z=1,Y=0,W=00,N=1 + assert!(detect_av1_keyframe(&[0x0F])); // N=1, reserved bits set + + // N=0 → not a keyframe + assert!(!detect_av1_keyframe(&[0x00])); + assert!(!detect_av1_keyframe(&[0x10])); // W=01, N=0 + assert!(!detect_av1_keyframe(&[0x70])); // Y=1, W=11, N=0 + assert!(!detect_av1_keyframe(&[0xF0])); // Z=1, Y=1, W=11, N=0 + } + + #[test] + fn packetize_respects_mtu() { + // OBU header (frame-type, no size field) + payload bytes. + let mut payload = vec![0x30u8]; + payload.extend(std::iter::repeat(0xABu8).take(2000)); + for &mtu in &[100usize, 300, 600, 1200] { + let mut packetizer = Av1Packetizer::default(); + let pkts = packetizer + .packetize(mtu, &payload) + .expect("AV1 packetize ok"); + assert!(!pkts.is_empty(), "AV1 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "AV1 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/buffer_rx.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/buffer_rx.rs new file mode 100644 index 000000000..a04d29f6e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/buffer_rx.rs @@ -0,0 +1,1197 @@ +use std::collections::VecDeque; +use std::fmt; +use std::ops::{Range, RangeInclusive}; +use std::sync::Arc; +use std::time::Instant; + +use crate::rtp::vla::VideoLayersAllocation; +use crate::rtp_::{ExtensionValues, MediaTime, RtpHeader, SenderInfo, SeqNo}; + +use super::contiguity::Contiguity; +use super::contiguity_vp8::Vp8Contiguity; +use super::contiguity_vp9::Vp9Contiguity; +use super::{CodecDepacketizer, CodecExtra, Depacketizer, PacketError}; + +#[derive(Clone, PartialEq, Eq)] +/// Holds metadata incoming RTP data. +pub struct RtpMeta { + /// When this RTP packet was received. + pub received: Instant, + /// Media time translated from the RtpHeader time. + pub time: MediaTime, + /// Sequence number, extended from the RTPHeader. + pub seq_no: SeqNo, + /// The actual header. + pub header: RtpHeader, + /// Sender information from the most recent Sender Report(SR). + /// + /// If no Sender Report(SR) has been received this is [`None`]. + pub last_sender_info: Option, +} + +#[derive(Clone)] +pub struct Depacketized { + pub time: MediaTime, + pub contiguous: bool, + pub meta: Vec, + pub data: Vec, + pub codec_extra: CodecExtra, +} + +impl Depacketized { + pub fn first_network_time(&self) -> Instant { + self.meta + .iter() + .map(|m| m.received) + .min() + .expect("a depacketized to consist of at least one packet") + } + + pub fn first_sender_info(&self) -> Option { + self.meta + .iter() + .min_by_key(|m| m.received) + .map(|m| m.last_sender_info) + .expect("a depacketized to consist of at least one packet") + } + + pub fn seq_range(&self) -> RangeInclusive { + let first = self + .meta + .first() + .expect("a depacketized to consist of at least one packet") + .seq_no; + let last = self + .meta + .last() + .expect("a depacketized to consist of at least one packet") + .seq_no; + first..=last + } + + pub fn start_of_talkspurt(&self) -> bool { + self.meta + .first() + .expect("a depacketized to consist of at least one packet") + .header + .marker + } + + pub fn ext_vals(&self) -> ExtensionValues { + let last = &self + .meta + .last() + .expect("depacketized video frame must contain a trailing packet") + .header + .ext_vals; + + let first = &self + .meta + .first() + .expect("depacketized video frame must contain a leading packet") + .header + .ext_vals; + + // We use the extensions from the last packet because certain extensions, such as video + // orientation, are only added on the last packet to save bytes. + let mut merged = last.clone(); + + // str0m strictly attaches some fields to the first packet of a frame. + if let Some(first_val) = &first.abs_capture_time { + merged.abs_capture_time = Some(*first_val); + } + + if let Some(first_val) = first.user_values.get_arc::() { + merged.user_values.set_arc(first_val); + } + + merged + } +} + +#[derive(Debug)] +struct Entry { + meta: RtpMeta, + data: Arc<[u8]>, + head: bool, + tail: bool, +} + +#[derive(Debug)] +pub struct DepacketizingBuffer { + hold_back: usize, + depack: CodecDepacketizer, + queue: VecDeque, + segments: Vec<(usize, usize)>, + last_emitted: Option<(SeqNo, CodecExtra)>, + max_time: Option, + depack_cache: Option<(Range, Depacketized)>, + contiguity: Contiguity, +} + +impl DepacketizingBuffer { + pub(crate) fn new(depack: CodecDepacketizer, hold_back: usize) -> Self { + let contiguity = match depack { + CodecDepacketizer::Vp8(_) => Contiguity::Vp8(Vp8Contiguity::new()), + CodecDepacketizer::Vp9(_) => Contiguity::Vp9(Vp9Contiguity::new()), + CodecDepacketizer::H264(_) + | CodecDepacketizer::H265(_) + | CodecDepacketizer::H266(_) + | CodecDepacketizer::Av1(_) + | CodecDepacketizer::Boxed(_) + | CodecDepacketizer::Opus(_) + | CodecDepacketizer::ComfortNoise(_) + | CodecDepacketizer::G711(_) + | CodecDepacketizer::Null(_) => Contiguity::None, + }; + + DepacketizingBuffer { + hold_back, + depack, + queue: VecDeque::new(), + segments: Vec::new(), + last_emitted: None, + max_time: None, + depack_cache: None, + contiguity, + } + } + + pub fn push(&mut self, meta: RtpMeta, data: impl Into>) { + self.push_entry(meta, data.into(), None); + } + + pub(crate) fn push_padding(&mut self, meta: RtpMeta) { + self.push_entry(meta, Arc::from([]), Some((false, false))); + } + + fn push_entry(&mut self, meta: RtpMeta, data: Arc<[u8]>, partition: Option<(bool, bool)>) { + // We're not emitting frames in the wrong order. If we receive + // packets that are before the last emitted, we drop. + // + // As a special case, per popular demand, if hold_back is 0, we do emit + // out of order packets. + if let Some((last, _)) = self.last_emitted { + if meta.seq_no <= last && self.hold_back > 0 { + trace!("Drop before emitted: {} <= {}", meta.seq_no, last); + return; + } + } + + // Record that latest seen max time (used for extending time to u64). + self.max_time = Some(if let Some(m) = self.max_time { + m.max(meta.time) + } else { + meta.time + }); + + match self + .queue + .binary_search_by_key(&meta.seq_no, |r| r.meta.seq_no) + { + Ok(_) => { + // exact same seq_no found. ignore + trace!("Drop exactly same packet: {}", meta.seq_no); + } + Err(i) => { + let (head, tail) = partition.unwrap_or_else(|| { + ( + self.depack.is_partition_head(data.as_ref()), + self.depack + .is_partition_tail(meta.header.marker, data.as_ref()), + ) + }); + + // i is insertion point to maintain order + let entry = Entry { + meta, + data, + head, + tail, + }; + self.queue.insert(i, entry); + + // The depack cache is keyed by queue index. Inserting at or before the cached + // segment shifts it, so the cache would answer for the wrong packets. + if let Some((range, _)) = &self.depack_cache { + if i <= range.end { + self.depack_cache = None; + } + } + } + } + } + + pub fn pop(&mut self) -> Option> { + self.update_segments(); + + if self.segments.is_empty() { + self.discard_old_padding(); + return None; + } + + // println!( + // "{:?} {:?}", + // self.queue.iter().map(|e| e.meta.seq_no).collect::>(), + // self.segments + // ); + + let (start, stop) = *self.segments.first().expect("segment exists"); + + let seq = { + let last = self.queue.get(stop).expect("entry for stop index"); + last.meta.seq_no + }; + + // depack ahead, even if we may not emit right away + let mut dep = match self.depacketize(start, stop, seq) { + Ok(d) => d, + Err(e) => { + // this segment cannot be decoded correctly + // remove from the queue and return the error + self.last_emitted = Some((seq, CodecExtra::None)); + self.queue.drain(0..=stop); + return Some(Err(e)); + } + }; + + // If we have contiguity of seq numbers we emit right away, + // Otherwise, we wait for retransmissions up to `hold_back` frames + // and re-evaluate contiguity based on codec specific information + + let more_than_hold_back = self.segments.len() >= self.hold_back; + let contiguous_seq = self.is_following_last(start); + let wait_for_contiguity = !contiguous_seq && !more_than_hold_back; + + if wait_for_contiguity { + // if we are not sending, cache the depacked + self.depack_cache = Some((start..stop, dep)); + self.discard_old_padding(); + return None; + } + + let (can_emit, contiguous_codec) = self.contiguity.check(&dep.codec_extra, contiguous_seq); + dep.contiguous = contiguous_codec; + + let last = self + .queue + .get(stop) + .expect("entry for stop index") + .meta + .seq_no; + + // We're not going to emit frames in the incorrect order, there's no point in keeping + // stuff before the emitted range. + self.queue.drain(0..=stop); + + if !can_emit { + return None; + } + + self.last_emitted = Some((last, dep.codec_extra)); + + Some(Ok(dep)) + } + + fn discard_old_padding(&mut self) { + let original_len = self.queue.len(); + let is_padding = |entry: &Entry| entry.data.is_empty() && !entry.head && !entry.tail; + + if let Some((mut last, extra)) = self.last_emitted { + while self.queue.len() > self.hold_back { + let entry = self.queue.front().expect("queue exceeds hold back"); + if !is_padding(entry) { + break; + } + + if last.is_next(entry.meta.seq_no) { + last = entry.meta.seq_no; + } + self.queue.pop_front(); + } + + self.last_emitted = Some((last, extra)); + } + + // An incomplete frame can remain at the front while another payload type + // contributes synthetic padding indefinitely. Keep the frame, but retain + // only the newest padding needed for the reordering window. Padding removed + // from behind media cannot advance last_emitted across that media. + let mut excess_padding = self + .queue + .iter() + .filter(|entry| is_padding(entry)) + .count() + .saturating_sub(self.hold_back); + self.queue.retain(|entry| { + if excess_padding > 0 && is_padding(entry) { + excess_padding -= 1; + false + } else { + true + } + }); + + if self.queue.len() != original_len { + self.depack_cache = None; + } + } + + fn depacketize( + &mut self, + start: usize, + stop: usize, + _seq: SeqNo, + ) -> Result { + if let Some(cached) = self.depack_cache.take() { + if cached.0 == (start..stop) { + trace!("depack cache hit for segment start {}", start); + return Ok(cached.1); + } + } + + let packets_size = self.queue.range(start..=stop).map(|p| p.data.len()).sum(); + let mut data = self + .depack + .out_size_hint(packets_size) + .map(Vec::with_capacity) + .unwrap_or_else(Vec::new); + let mut codec_extra = CodecExtra::None; + + let time = self.queue.get(start).expect("first index exist").meta.time; + let mut meta = Vec::with_capacity(stop - start + 1); + + for entry in self.queue.range_mut(start..=stop) { + self.depack + .depacketize(entry.data.as_ref(), &mut data, &mut codec_extra)?; + meta.push(entry.meta.clone()); + } + + Ok(Depacketized { + time, + contiguous: true, // the caller taking ownership will modify this accordingly + meta, + data, + codec_extra, + }) + } + + fn update_segments(&mut self) -> Option<(usize, usize)> { + self.segments.clear(); + + #[derive(Clone, Copy)] + struct Start { + index: i64, + time: MediaTime, + offset: i64, + } + + let mut start: Option = None; + + for (index, entry) in self.queue.iter().enumerate() { + let index = index as i64; + let iseq = *entry.meta.seq_no as i64; + let expected_seq = start.map(|s| s.offset.saturating_add(index)); + + let is_expected_seq = expected_seq == Some(iseq); + let is_same_timestamp = start.map(|s| s.time) == Some(entry.meta.time); + let is_defacto_tail = is_expected_seq && !is_same_timestamp; + + if start.is_some() && is_defacto_tail { + // We found a segment that ended because the timestamp changed without + // a gap in the sequence number. The marker bit in the RTP packet is + // just indicative, this is the robust fallback. + let segment = (start.unwrap().index as usize, index as usize - 1); + self.segments.push(segment); + start = None; + } + + if start.is_some() && (!is_expected_seq || !is_same_timestamp) { + // Not contiguous. Start looking again. + start = None; + } + + // Each segment can have multiple is_partition_head() == true, record the first. + if start.is_none() && entry.head { + start = Some(Start { + index, + time: entry.meta.time, + offset: iseq.saturating_sub(index), + }); + } + + if start.is_some() && entry.tail { + // We found a contiguous sequence of packets ending with something from + // the packet (like the RTP marker bit) indicating it's the tail. + let segment = (start.unwrap().index as usize, index as usize); + self.segments.push(segment); + start = None; + } + } + + None + } + + fn is_following_last(&self, start: usize) -> bool { + let Some((last, _)) = self.last_emitted else { + // First time we emit something. + return true; + }; + + // track sequence numbers are sequential + let mut seq = last; + + // Expect all entries before start to be padding. + for entry in self.queue.range(0..start) { + if !seq.is_next(entry.meta.seq_no) { + // Not a sequence + return false; + } + // for next loop round. + seq = entry.meta.seq_no; + + let is_padding = entry.data.is_empty() && !entry.head && !entry.tail; + if !is_padding { + return false; + } + } + + let start_entry = self.queue.get(start).expect("entry for start index"); + + seq.is_next(start_entry.meta.seq_no) + } +} + +impl fmt::Debug for RtpMeta { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtpMeta") + .field("received", &self.received) + .field("time", &self.time) + .field("seq_no", &self.seq_no) + .field("header", &self.header) + .finish() + } +} + +impl fmt::Debug for Depacketized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Depacketized") + .field("time", &self.time) + .field("meta", &self.meta) + .field("data", &self.data.len()) + .finish() + } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant, SystemTime}; + + use super::*; + use crate::packet::vp9::Vp9Depacketizer; + use crate::rtp::UserExtensionValues; + use crate::rtp_::{AbsCaptureTime, Frequency, MediaTime, Pt, Ssrc, VideoOrientation}; + + #[test] + fn end_on_marker() { + test(&[ + // + (1, 1, &[1], &[]), + (2, 1, &[9], &[(1, &[1, 9])]), + ]) + } + + #[test] + fn late_packet_in_front_of_waiting_frame() { + // Seq 3 is late. Seq 4 arrives first and waits for contiguity (it is depacketized ahead + // and cached). When 3 then arrives it is inserted in front of 4, and both must come out + // as themselves: 3 must not be emitted with 4's data. + test_n( + 15, + &[ + (1, 1, &[1, 1, 9], &[(1, &[1, 1, 9])]), + (2, 2, &[1, 2, 9], &[(2, &[1, 2, 9])]), + (4, 4, &[1, 4, 9], &[]), + (3, 3, &[1, 3, 9], &[(3, &[1, 3, 9]), (4, &[1, 4, 9])]), + ], + ) + } + + #[test] + fn ext_vals_extracts_from_first_and_last_packet() { + let first_time = Instant::now(); + let abs_capture_time = AbsCaptureTime { + capture_time: SystemTime::UNIX_EPOCH + Duration::from_secs(1), + clock_offset: None, + }; + let mut vla = UserExtensionValues::default(); + vla.set(VideoLayersAllocation { + current_simulcast_stream_index: 1, + simulcast_streams: vec![], + }); + + let first_header = RtpHeader { + version: 2, + has_padding: false, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: Pt::new_with_value(98), + sequence_number: 1, + timestamp: 100, + ssrc: Ssrc::from(42), + csrc: [0; 15], + ext_vals: ExtensionValues { + abs_capture_time: Some(abs_capture_time), + user_values: vla.clone(), + ..Default::default() + }, + header_len: 0, + }; + + let last_header = RtpHeader { + ext_vals: ExtensionValues { + video_orientation: Some(VideoOrientation::Deg90), + ..Default::default() + }, + ..first_header.clone() + }; + + let time_value = 100_u64; + let dep = Depacketized { + time: MediaTime::new(time_value, Frequency::new(90000).unwrap()), + contiguous: true, + meta: vec![ + RtpMeta { + received: first_time, + time: MediaTime::new(time_value, Frequency::new(90000).unwrap()), + seq_no: SeqNo::from(1u64), + header: first_header, + last_sender_info: None, + }, + RtpMeta { + received: first_time + Duration::from_millis(1), + time: MediaTime::new(time_value, Frequency::new(90000).unwrap()), + seq_no: SeqNo::from(2u64), + header: last_header, + last_sender_info: None, + }, + ], + data: Vec::new(), + codec_extra: CodecExtra::None, + }; + + let merged = dep.ext_vals(); + assert_eq!( + merged.abs_capture_time.unwrap().capture_time, + abs_capture_time.capture_time + ); + assert_eq!( + merged + .user_values + .get::() + .unwrap() + .current_simulcast_stream_index, + vla.get::() + .unwrap() + .current_simulcast_stream_index, + ); + assert_eq!(merged.video_orientation, Some(VideoOrientation::Deg90)); + } + + #[test] + fn end_on_defacto() { + test(&[ + (1, 1, &[1], &[]), + (2, 1, &[2], &[]), + (3, 2, &[3], &[(1, &[1, 2])]), + ]) + } + + #[test] + fn skip_padding() { + test(&[ + (1, 1, &[1], &[]), + (2, 1, &[9], &[(1, &[1, 9])]), + (3, 1, &[], &[]), // padding! + (4, 2, &[1], &[]), + (5, 2, &[9], &[(2, &[1, 9])]), + ]) + } + + #[test] + fn gap_after_emit() { + test(&[ + (1, 1, &[1], &[]), + (2, 1, &[9], &[(1, &[1, 9])]), + // gap + (4, 2, &[1], &[]), + (5, 2, &[9], &[]), + ]) + } + + #[test] + fn gap_after_padding() { + test(&[ + (1, 1, &[1], &[]), + (2, 1, &[9], &[(1, &[1, 9])]), + (3, 1, &[], &[]), // padding! + // gap + (5, 2, &[1], &[]), + (6, 2, &[9], &[]), + ]) + } + + #[test] + fn single_packets() { + test(&[ + (1, 1, &[1, 9], &[(1, &[1, 9])]), + (2, 2, &[1, 9], &[(2, &[1, 9])]), + (3, 3, &[1, 9], &[(3, &[1, 9])]), + (4, 4, &[1, 9], &[(4, &[1, 9])]), + ]) + } + + #[test] + fn packets_out_of_order() { + test(&[ + (1, 1, &[1], &[]), + (2, 1, &[9], &[(1, &[1, 9])]), + (4, 2, &[9], &[]), + (3, 2, &[1], &[(2, &[1, 9])]), + ]) + } + + #[test] + fn packets_after_hold_out() { + test(&[ + (1, 1, &[1, 9], &[(1, &[1, 9])]), + (3, 3, &[1, 9], &[]), + (4, 4, &[1, 9], &[]), + (5, 5, &[1, 9], &[(3, &[1, 9]), (4, &[1, 9]), (5, &[1, 9])]), + ]) + } + + #[test] + fn packets_with_hold_0() { + test0(&[ + (1, 1, &[1, 9], &[(1, &[1, 9])]), + (3, 3, &[1, 9], &[(3, &[1, 9])]), + (4, 4, &[1, 9], &[(4, &[1, 9])]), + (5, 5, &[1, 9], &[(5, &[1, 9])]), + ]) + } + + #[test] + fn out_of_order_packets_with_hold_0() { + test0(&[ + (3, 1, &[1, 9], &[(1, &[1, 9])]), + (1, 3, &[1, 9], &[(3, &[1, 9])]), + (5, 4, &[1, 9], &[(4, &[1, 9])]), + (2, 5, &[1, 9], &[(5, &[1, 9])]), + ]) + } + + #[test] + fn padding_only_run_does_not_grow_the_queue_without_bound() { + let depack = CodecDepacketizer::Boxed(Box::new(TestDepack)); + let mut buf = DepacketizingBuffer::new(depack, 3); + let meta = |seq: u64| RtpMeta { + received: Instant::now(), + seq_no: seq.into(), + time: MediaTime::from_90khz(seq), + last_sender_info: None, + header: RtpHeader { + sequence_number: seq as u16, + timestamp: seq as u32, + ..Default::default() + }, + }; + + // Emit one packet for this PT, then model a long run on another PT. + buf.push(meta(1), vec![1, 9]); + assert!(buf.pop().is_some()); + + for seq in 2..=1_001 { + buf.push_padding(meta(seq)); + assert!(buf.pop().is_none()); + } + + assert!( + buf.queue.len() <= buf.hold_back, + "padding-only queue retained {} entries after a 1,000-packet run", + buf.queue.len() + ); + + // Returning to this PT after compaction must still be contiguous with the last packet. + buf.push(meta(1_002), vec![1, 9]); + let dep = buf.pop().expect("packet emitted").expect("valid packet"); + assert!(dep.contiguous); + } + + #[test] + fn padding_only_run_with_loss_stays_bounded_and_reports_discontinuity() { + let depack = CodecDepacketizer::Boxed(Box::new(TestDepack)); + let mut buf = DepacketizingBuffer::new(depack, 3); + let meta = |seq: u64| RtpMeta { + received: Instant::now(), + seq_no: seq.into(), + time: MediaTime::from_90khz(seq), + last_sender_info: None, + header: RtpHeader { + sequence_number: seq as u16, + timestamp: seq as u32, + ..Default::default() + }, + }; + + buf.push(meta(1), vec![1, 9]); + assert!(buf.pop().is_some()); + + // Sequence 2 is lost while another PT remains active. + for seq in 3..=1_002 { + buf.push_padding(meta(seq)); + assert!(buf.pop().is_none()); + } + + assert!(buf.queue.len() <= buf.hold_back); + + // A discontinuity waits for the normal hold-back before it is emitted. + buf.push(meta(1_003), vec![1, 9]); + assert!(buf.pop().is_none()); + buf.push(meta(1_004), vec![1, 9]); + assert!(buf.pop().is_none()); + buf.push(meta(1_005), vec![1, 9]); + let dep = buf.pop().expect("packet emitted").expect("valid packet"); + assert!(!dep.contiguous); + } + + #[test] + fn padding_after_incomplete_vp8_frame_does_not_grow_without_bound() { + let depack = CodecDepacketizer::Vp8(Default::default()); + let mut buf = DepacketizingBuffer::new(depack, 3); + let meta = |seq: u64, marker: bool| RtpMeta { + received: Instant::now(), + seq_no: seq.into(), + time: MediaTime::from_90khz(seq), + last_sender_info: None, + header: RtpHeader { + marker, + sequence_number: seq as u16, + timestamp: seq as u32, + ..Default::default() + }, + }; + + // Emit one complete VP8 frame for this PT. + buf.push(meta(1, true), [0x10, 0x00]); + assert!(buf.pop().is_some()); + + // The next frame's head is lost, leaving an S=0 fragment at the front. + buf.push(meta(2, false), [0x00]); + assert!(buf.pop().is_none()); + + // The sender switches PT. Every packet becomes synthetic padding here. + for seq in 3..=1_002 { + buf.push_padding(meta(seq, false)); + assert!(buf.pop().is_none()); + } + + assert!( + buf.queue.len() <= buf.hold_back + 1, + "incomplete VP8 frame retained {} entries after a 1,000-packet PT switch", + buf.queue.len() + ); + } + + #[test] + fn padding_after_waiting_vp8_frame_does_not_grow_without_bound() { + let depack = CodecDepacketizer::Vp8(Default::default()); + let mut buf = DepacketizingBuffer::new(depack, 3); + let meta = |seq: u64, marker: bool| RtpMeta { + received: Instant::now(), + seq_no: seq.into(), + time: MediaTime::from_90khz(seq), + last_sender_info: None, + header: RtpHeader { + marker, + sequence_number: seq as u16, + timestamp: seq as u32, + ..Default::default() + }, + }; + + buf.push(meta(1, true), [0x10, 0x00]); + assert!(buf.pop().is_some()); + + // Lose a frame head, then switch away long enough to compact its padding. + buf.push(meta(2, false), [0x00]); + assert!(buf.pop().is_none()); + for seq in 3..=1_002 { + buf.push_padding(meta(seq, false)); + assert!(buf.pop().is_none()); + } + + // One complete frame returns, but waits behind the orphan for hold-back. + buf.push(meta(1_003, true), [0x10, 0x00]); + assert!(buf.pop().is_none()); + + // Switching away again must remain bounded while that frame waits. + for seq in 1_004..=2_003 { + buf.push_padding(meta(seq, false)); + assert!(buf.pop().is_none()); + } + + assert!( + buf.queue.len() <= buf.hold_back + 2, + "waiting VP8 frame retained {} entries after a second 1,000-packet PT switch", + buf.queue.len() + ); + } + + #[test] + fn padding_after_initial_incomplete_vp8_frame_does_not_grow_without_bound() { + let depack = CodecDepacketizer::Vp8(Default::default()); + let mut buf = DepacketizingBuffer::new(depack, 3); + let meta = |seq: u64| RtpMeta { + received: Instant::now(), + seq_no: seq.into(), + time: MediaTime::from_90khz(seq), + last_sender_info: None, + header: RtpHeader { + sequence_number: seq as u16, + timestamp: seq as u32, + ..Default::default() + }, + }; + + // Start observing this PT after the head of a VP8 frame was lost. + buf.push(meta(1), [0x00]); + assert!(buf.pop().is_none()); + + // The sender switches PT before this depayloader has emitted anything. + for seq in 2..=1_001 { + buf.push_padding(meta(seq)); + assert!(buf.pop().is_none()); + } + + assert!( + buf.queue.len() <= buf.hold_back + 1, + "initial incomplete VP8 frame retained {} entries after a 1,000-packet PT switch", + buf.queue.len() + ); + } + + fn test( + v: &[( + u64, // seq + u64, // time + &[u8], // data + &[( + u64, // time + &[u8], // depacketized data + )], + )], + ) { + test_n(3, v) + } + + fn test0( + v: &[( + u64, // seq + u64, // time + &[u8], // data + &[( + u64, // time + &[u8], // depacketized data + )], + )], + ) { + test_n(0, v) + } + + fn test_n( + hold_back: usize, + v: &[( + u64, // seq + u64, // time + &[u8], // data + &[( + u64, // time + &[u8], // depacketized data + )], + )], + ) { + let depack = CodecDepacketizer::Boxed(Box::new(TestDepack)); + let mut buf = DepacketizingBuffer::new(depack, hold_back); + + let mut step = 1; + + for (seq, time, data, checks) in v { + let meta = RtpMeta { + received: Instant::now(), + seq_no: (*seq).into(), + time: MediaTime::from_90khz(*time), + last_sender_info: None, + header: RtpHeader { + sequence_number: *seq as u16, + timestamp: *time as u32, + ..Default::default() + }, + }; + + buf.push(meta, data.to_vec()); + + let mut depacks = vec![]; + while let Some(res) = buf.pop() { + let d = res.unwrap(); + depacks.push(d); + } + + assert_eq!( + depacks.len(), + checks.len(), + "Step {}: check count not matching {} != {}", + step, + depacks.len(), + checks.len() + ); + + let iter = depacks.into_iter().zip(checks.iter()); + + for (depack, (dtime, ddata)) in iter { + assert_eq!( + depack.time.numer(), + *dtime, + "Step {}: Time not matching {} != {}", + step, + depack.time.numer(), + *dtime + ); + + assert_eq!( + depack.data, *ddata, + "Step {}: Data not correct {:?} != {:?}", + step, depack.data, *ddata + ); + } + + step += 1; + } + } + + #[derive(Debug)] + struct TestDepack; + + impl Depacketizer for TestDepack { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + _: &mut CodecExtra, + ) -> Result<(), PacketError> { + out.extend_from_slice(packet); + Ok(()) + } + + fn is_partition_head(&self, packet: &[u8]) -> bool { + !packet.is_empty() && packet[0] == 1 + } + + fn is_partition_tail(&self, _marker: bool, packet: &[u8]) -> bool { + !packet.is_empty() && packet.contains(&9) + } + } + + #[test] + fn rtp_out_of_order() { + let construct_input = + |(time, seq, marker, cc, data): (u32, u16, bool, u16, Vec)| -> (RtpMeta, Vec) { + ( + RtpMeta { + received: Instant::now(), + time: MediaTime::new(time.into(), Frequency::new(90000).unwrap()), + seq_no: SeqNo::from(seq as u64), + header: RtpHeader { + version: 2, + has_padding: false, + has_extension: true, + csrc_count: 0, + marker, + payload_type: Pt::new_with_value(98), + sequence_number: seq, + timestamp: time, + ssrc: Ssrc::from(2930203832), + csrc: [0; 15], + ext_vals: ExtensionValues { + transport_cc: Some(cc), + ..Default::default() + }, + header_len: 28, + }, + last_sender_info: None, + }, + data, + ) + }; + + let inputs = [ + // PID: 23860 + ( + 821395241, // Timestamp + 8685, // SeqN + false, // Marker + 56, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([236, 221, 52, 80, 26, 10, 1, 1, 1, 1, 1, 1, 1, 1]), // Data + ), + // PID: 23860 + ( + 821395241, // Timestamp + 8686, // SeqN + true, // Marker + 57, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([237, 221, 52, 83, 26, 10, 2, 2, 2, 2, 2, 2, 2, 2]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8687, // SeqN + false, // Marker + 60, // Transport CC + // VP9 header--------+ + // | + // +------------------------+ + Vec::from([170, 221, 53, 16, 27, 56, 20, 0, 0, 0, 0, 0, 0, 0, 0]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8688, // SeqN + false, // Marker + 61, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([160, 221, 53, 16, 27, 20, 1, 1, 1, 1, 1, 1, 1, 1]), // Data + ), + // PID: 23861 + ( + 821398481, + 8689, + false, + 62, + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([164, 221, 53, 16, 27, 20, 2, 2, 2, 2, 2, 2, 2, 2]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8690, // SeqN + false, // Marker + 63, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([169, 221, 53, 19, 27, 20, 3, 3, 3, 3, 3, 3, 3, 3]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8691, // SeqN + false, // Marker + 64, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([161, 221, 53, 19, 27, 20, 4, 4, 4, 4, 4, 4, 4, 4]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8692, // SeqN + false, // Marker + 65, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([161, 221, 53, 19, 27, 20, 5, 5, 5, 5, 5, 5, 5, 5]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8693, // SeqN + false, // Marker + 66, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([161, 221, 53, 19, 27, 20, 6, 6, 6, 6, 6, 6, 6, 6]), // Data + ), + // PID: 23861 + ( + 821398481, // Timestamp + 8694, // SeqN + true, // Marker + 67, // Transport CC + // VP9 header--------+ + // | + // +--------------------+ + Vec::from([165, 221, 53, 19, 27, 20, 7, 7, 7, 7, 7, 7, 7, 7]), // Data + ), + ]; + + let mut buffer = + DepacketizingBuffer::new(CodecDepacketizer::Vp9(Vp9Depacketizer::default()), 30); + + for input in &inputs { + let (meta, data) = construct_input(input.clone()); + buffer.push(meta, data); + } + + let res0before = buffer.pop().unwrap().unwrap(); // Pop PID: 23860, `contiguous_seq == true`. + let res1before = buffer.pop().unwrap().unwrap(); // Pop PID: 23861, `contiguous_seq == true`. + + let mut buffer = + DepacketizingBuffer::new(CodecDepacketizer::Vp9(Vp9Depacketizer::default()), 30); + + for input in &inputs { + let (meta, data) = construct_input(input.clone()); + if meta.seq_no == SeqNo::from(8689) { + continue; // Skip RTP packet with seq_num=8689 vp9_payload=[20, 2, 2, 2, 2, 2, 2, 2, 2]. + } + buffer.push(meta.clone(), data.clone()); + } + + // Pop PID: 23860, `contiguous_seq == true`. + let res0after = buffer.pop().unwrap().unwrap(); + // Try to pop PID: 23861. `None` because `contiguous_seq == false` -- no seq_num=8689. + assert!(buffer.pop().is_none()); + assert!(buffer.pop().is_none()); // Ensure once again. + + for input in &inputs { + let (meta, data) = construct_input(input.clone()); + if meta.seq_no == SeqNo::from(8689) { + // Send RTP packet with seq_num=8689 vp9_payload=[20, 2, 2, 2, 2, 2, 2, 2, 2]. + buffer.push(meta.clone(), data.clone()); + break; + } + } + + let res1after = buffer.pop().unwrap().unwrap(); + + assert_eq!(res0before.data, res0after.data); + assert_eq!(res1before.data, res1after.data); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/comfort_noise.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/comfort_noise.rs new file mode 100644 index 000000000..827da6dac --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/comfort_noise.rs @@ -0,0 +1,89 @@ +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; + +#[derive(Debug)] +pub struct ComfortNoisePacketizer; + +#[derive(Debug)] +pub struct ComfortNoiseDepacketizer; + +impl Packetizer for ComfortNoisePacketizer { + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() { + return Ok(vec![]); + } + + if payload.len() > mtu { + return Err(PacketError::ErrPayloadTooLarge); + } + + Ok(vec![payload.to_vec()]) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, _last: bool) -> bool { + false + } +} + +impl Depacketizer for ComfortNoiseDepacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + _codec_extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + if packet.is_empty() { + return Err(PacketError::ErrShortPacket); + } + + out.extend_from_slice(packet); + Ok(()) + } + + fn is_partition_head(&self, _packet: &[u8]) -> bool { + true + } + + fn is_partition_tail(&self, _marker: bool, _packet: &[u8]) -> bool { + true + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn packetizer_rejects_payload_larger_than_mtu() { + let mut packetizer = ComfortNoisePacketizer; + + let result = packetizer.packetize(8, &[0; 9]); + + assert!( + result.is_err(), + "a CN payload cannot be fragmented and must not exceed the MTU" + ); + } + + #[test] + fn packetizer_does_not_emit_empty_cn_payload() { + let mut packetizer = ComfortNoisePacketizer; + + let packets = packetizer.packetize(1200, &[]).unwrap(); + + assert!(packets.is_empty(), "an empty input is not a CN payload"); + } + + #[test] + fn depacketizer_rejects_empty_cn_payload() { + let mut depacketizer = ComfortNoiseDepacketizer; + let mut output = Vec::new(); + + let result = depacketizer.depacketize(&[], &mut output, &mut CodecExtra::None); + + assert_eq!(result, Err(PacketError::ErrShortPacket)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity.rs new file mode 100644 index 000000000..06c857364 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity.rs @@ -0,0 +1,104 @@ +use super::CodecExtra; +use super::contiguity_vp8::Vp8Contiguity; +use super::contiguity_vp9::Vp9Contiguity; + +#[derive(Debug)] +pub enum Contiguity { + Vp8(Vp8Contiguity), + Vp9(Vp9Contiguity), + None, +} + +impl Contiguity { + pub fn check(&mut self, next: &CodecExtra, contiguous_seq: bool) -> (bool, bool) { + match (self, next) { + (Self::Vp8(contiguity), CodecExtra::Vp8(next)) => { + contiguity.check(next, contiguous_seq) + } + (Self::Vp9(contiguity), CodecExtra::Vp9(next)) => { + contiguity.check(next, contiguous_seq) + } + (Self::Vp8(_) | Self::Vp9(_) | Self::None, _) => (true, contiguous_seq), + } + } +} + +#[derive(Debug)] +pub(crate) struct FrameContiguityState { + /// last picture id of layer 0 that we allowed emitting + pub(crate) last_picture_id: Option, + + /// last picture id of any layer that we allowed emitting + pub(crate) last_tl0_picture_id: Option, +} + +impl FrameContiguityState { + /// Returns whether we can emit such frame and whether it's decodable with contiguity + pub(crate) fn next_frame( + &mut self, + picture_id: Option, + tl0_picture_id: Option, + layer_index: Option, + contiguous_seq: bool, + ) -> (bool, bool) { + let Some(picture_id) = picture_id else { + // picture id is not enabled or not progressing anyway + return (true, contiguous_seq); + }; + + let Some(tl0_picture_id) = tl0_picture_id else { + return (true, contiguous_seq); + }; + + let (Some(last_tl0_picture_id), Some(last_picture_id)) = + (self.last_tl0_picture_id, self.last_picture_id) + else { + self.last_tl0_picture_id = Some(tl0_picture_id); + self.last_picture_id = Some(picture_id); + return (true, true); + }; + + // discard older pictures if any + if picture_id <= last_picture_id { + return (false, true); + } + + if layer_index == Some(0) { + if tl0_picture_id == last_tl0_picture_id { + warn!( + "VP8 or VP9: 2 subsequent frames on layer zero must \ + have different tl0 picture id: encoding problem?" + ); + } + + // Frame on layer 0: always emit and report discontinuity if not subsequent + let emit = true; + // note: we use wrapping add because the encoder can restart if the + // camera is closed / reopened and here we only care about + // contiguity + let contiguous = tl0_picture_id == last_tl0_picture_id.wrapping_add(1); + + self.last_tl0_picture_id = Some(tl0_picture_id); + self.last_picture_id = Some(picture_id); + + return (emit, contiguous); + } + + // Frame on layer 1 or 2: only emit if they refer to the current layer 0 + // and they are subsequent + let emit = + tl0_picture_id == last_tl0_picture_id && picture_id == last_picture_id.wrapping_add(1); + + if emit { + self.last_picture_id = Some(picture_id); + if !contiguous_seq { + // this as happened in Safari + very lossy network (very rare) + debug!( + "VP8 or VP9: contiguous pictures implies contiguous seq numbers: encoding issue ?" + ) + } + } + + (emit, true) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp8.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp8.rs new file mode 100644 index 000000000..0caa12a2b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp8.rs @@ -0,0 +1,185 @@ +use super::Vp8CodecExtra; +use super::contiguity::FrameContiguityState; + +#[derive(Debug, Default)] +pub struct Vp8Contiguity { + /// last picture id of layer 0 that we allowed emitting + last_tl0_picture_id: Option, + /// last picture id of any layer that we allowed emitting + last_picture_id: Option, +} + +impl Vp8Contiguity { + pub fn new() -> Self { + Self::default() + } + + /// Called when depacketizing a new frame is ready to be emitted + /// + /// Returns whether we can emit such frame and whether it's decodable with contiguity + pub fn check(&mut self, next: &Vp8CodecExtra, contiguous_seq: bool) -> (bool, bool) { + let mut frame_state: FrameContiguityState = self.into(); + let res = frame_state.next_frame( + next.picture_id, + next.tl0_picture_id, + Some(next.layer_index.into()), + contiguous_seq, + ); + + *self = frame_state.into(); + res + } +} + +impl From<&mut Vp8Contiguity> for FrameContiguityState { + fn from(value: &mut Vp8Contiguity) -> Self { + Self { + last_picture_id: value.last_picture_id, + last_tl0_picture_id: value.last_tl0_picture_id, + } + } +} + +impl From for Vp8Contiguity { + fn from(value: FrameContiguityState) -> Self { + Self { + last_picture_id: value.last_picture_id, + last_tl0_picture_id: value.last_tl0_picture_id, + } + } +} + +#[cfg(test)] +mod test { + + use super::Vp8Contiguity; + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + const L1T2_LAYERS: &[u64] = &[0, 1, 0, 1]; + + #[test] + fn contiguous_l1t3() { + let mut contiguity = Vp8Contiguity::new(); + + for i in 0..100 { + let next = &crate::packet::Vp8CodecExtra { + discardable: false, + sync: true, + layer_index: L1T3_LAYERS[i as usize % 4] as u8, + picture_id: Some(i), + tl0_picture_id: Some(i / 4), + is_keyframe: false, + }; + + let res = contiguity.check(next, true); + assert_eq!(res, (true, true), "Failure at picture {} {:?}", i, next); + } + } + + #[test] + fn contiguous_l1t3_no_l2_contig_l0() { + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + + let mut contiguity = Vp8Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T3_LAYERS[i as usize % 4] as u8; + if layer_index == 2 { + continue; + } + + let next = &crate::packet::Vp8CodecExtra { + discardable: false, + sync: i % 8 == 0, + layer_index, + picture_id: Some(i), + tl0_picture_id: Some(i / 4), + is_keyframe: false, + }; + + let (emit, contiguous) = contiguity.check(next, true); + + // all layer 0 are contiguous therefore no discontinuity + assert_eq!(emit, next.layer_index == 0); + assert!(contiguous); + + if layer_index == 1 { + // frames on layer 1 are not emitted (could be improved tracking more deps) + // and we don't raise a discontinuity (stream still decodable) + assert!(!emit); + assert!(contiguous); + } + } + } + + #[test] + fn contiguous_l1t3_no_l2_discontig_l0() { + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + + let mut contiguity = Vp8Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T3_LAYERS[i as usize % 4] as u8; + if layer_index == 2 { + continue; + } + + let next = &crate::packet::Vp8CodecExtra { + discardable: false, + sync: i % 8 == 0, + layer_index, + picture_id: Some(i), + tl0_picture_id: Some(i), + is_keyframe: false, + }; + + let (emit, _) = contiguity.check(next, true); + + assert!(emit == (next.layer_index == 0)); + } + } + + #[test] + fn contiguous_l1t2() { + let mut contiguity = Vp8Contiguity::new(); + + for i in 0..100 { + let next = &crate::packet::Vp8CodecExtra { + discardable: false, + sync: true, + layer_index: L1T2_LAYERS[i as usize % 4] as u8, + picture_id: Some(i), + tl0_picture_id: Some(i / 2), + is_keyframe: false, + }; + + let res = contiguity.check(next, true); + assert_eq!(res, (true, true), "Failure at picture {} {:?}", i, next); + } + } + + #[test] + fn contiguous_l1t1_no_l1_contig_l0() { + let mut contiguity = Vp8Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T2_LAYERS[i as usize % 4] as u8; + if layer_index == 1 { + continue; + } + + let next = &crate::packet::Vp8CodecExtra { + discardable: false, + sync: i % 8 == 0, + layer_index, + picture_id: Some(i), + tl0_picture_id: Some(i / 2), + is_keyframe: false, + }; + + let (emit, _) = contiguity.check(next, true); + + // all layer 0 are contiguous therefore can be emitted + assert_eq!(emit, next.layer_index == 0); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp9.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp9.rs new file mode 100644 index 000000000..f9d312988 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/contiguity_vp9.rs @@ -0,0 +1,159 @@ +use super::Vp9CodecExtra; +use super::contiguity::FrameContiguityState; + +#[derive(Debug, Default)] +pub struct Vp9Contiguity { + /// last picture id of layer 0 that we allowed emitting + last_tl0_picture_id: Option, + + /// last picture id of any layer that we allowed emitting + last_picture_id: Option, +} + +impl Vp9Contiguity { + pub fn new() -> Self { + Self::default() + } + + /// Called when depacketizing a new frame is ready to be emitted + /// + /// Returns whether we can emit such frame and whether it's decodable with contiguity + pub fn check(&mut self, next: &Vp9CodecExtra, contiguous_seq: bool) -> (bool, bool) { + let mut frame_state: FrameContiguityState = self.into(); + let res = frame_state.next_frame( + Some(next.pid.into()), + next.tl0_picture_id.map(Into::into), + next.tid.map(Into::into), + contiguous_seq, + ); + + *self = frame_state.into(); + res + } +} + +impl From<&mut Vp9Contiguity> for FrameContiguityState { + fn from(value: &mut Vp9Contiguity) -> Self { + Self { + last_picture_id: value.last_picture_id.map(Into::into), + last_tl0_picture_id: value.last_tl0_picture_id.map(Into::into), + } + } +} + +impl From for Vp9Contiguity { + fn from(value: FrameContiguityState) -> Self { + Self { + last_picture_id: value.last_picture_id.and_then(|v| v.try_into().ok()), + last_tl0_picture_id: value.last_tl0_picture_id.and_then(|v| v.try_into().ok()), + } + } +} + +#[cfg(test)] +mod test { + use crate::packet::Vp9CodecExtra; + + use super::Vp9Contiguity; + + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + const L1T2_LAYERS: &[u64] = &[0, 1, 0, 1]; + + fn get_codec_extra(pid: u16, tid: u8, tl0_picture_id: u8) -> Vp9CodecExtra { + Vp9CodecExtra { + pid, + tid: Some(tid), + tl0_picture_id: Some(tl0_picture_id), + ..Default::default() + } + } + + #[test] + fn contiguous_l1t3() { + let mut contiguity = Vp9Contiguity::new(); + + for i in 0..100 { + let next = get_codec_extra(i, L1T3_LAYERS[i as usize % 4] as u8, (i / 4) as u8); + + let res = contiguity.check(&next, true); + assert_eq!(res, (true, true), "Failure at picture {} {:?}", i, next); + } + } + + #[test] + fn contiguous_l1t3_no_l2_contig_l0() { + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + + let mut contiguity = Vp9Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T3_LAYERS[i as usize % 4] as u8; + if layer_index == 2 { + continue; + } + + let next = get_codec_extra(i, layer_index, (i / 4) as u8); + let (emit, contiguous) = contiguity.check(&next, true); + + // all layer 0 are contiguous therefore no discontinuity + assert_eq!(emit, next.tid == Some(0)); + assert!(contiguous); + + if layer_index == 1 { + // frames on layer 1 are not emitted (could be improved tracking more deps) + // and we don't raise a discontinuity (stream still decodable) + assert!(!emit); + assert!(contiguous); + } + } + } + + #[test] + fn contiguous_l1t3_no_l2_discontig_l0() { + const L1T3_LAYERS: &[u64] = &[0, 2, 1, 2]; + + let mut contiguity = Vp9Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T3_LAYERS[i as usize % 4] as u8; + if layer_index == 2 { + continue; + } + + let next = get_codec_extra(i, layer_index, i as u8); + let (emit, _) = contiguity.check(&next, true); + + assert!(emit == (next.tid == Some(0))); + } + } + + #[test] + fn contiguous_l1t2() { + let mut contiguity = Vp9Contiguity::new(); + + for i in 0..100 { + let next = get_codec_extra(i, L1T2_LAYERS[i as usize % 4] as u8, (i / 2) as u8); + + let res = contiguity.check(&next, true); + assert_eq!(res, (true, true), "Failure at picture {} {:?}", i, next); + } + } + + #[test] + fn contiguous_l1t1_no_l1_contig_l0() { + let mut contiguity = Vp9Contiguity::new(); + + for i in 0..100 { + let layer_index = L1T2_LAYERS[i as usize % 4] as u8; + if layer_index == 1 { + continue; + } + + let next = get_codec_extra(i, layer_index, (i / 2) as u8); + let (emit, _) = contiguity.check(&next, true); + + // all layer 0 are contiguous therefore can be emitted + assert_eq!(emit, next.tid == Some(0)); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/error.rs new file mode 100644 index 000000000..716364d9d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/error.rs @@ -0,0 +1,52 @@ +use std::error::Error; +use std::fmt; + +/// Errors arising in packet- and depacketization. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +#[allow(missing_docs)] +pub enum PacketError { + ErrShortPacket, + ErrPayloadTooLarge, + ErrTooManySpatialLayers, + ErrTooManyPDiff, + ErrH265CorruptedPacket, + ErrH266CorruptedPacket, + ErrInvalidH265PacketType, + ErrH265PACIPHESTooLong, + StapASizeLargerThanBuffer(usize, usize), + NaluTypeIsNotHandled(u8), + ErrVP8CorruptedPacket, + ErrVP9CorruptedPacket, + ErrAv1CorruptedPacket, +} + +impl fmt::Display for PacketError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PacketError::ErrShortPacket => write!(f, "Packet is too short"), + PacketError::ErrPayloadTooLarge => write!(f, "Payload exceeds the MTU"), + PacketError::ErrTooManySpatialLayers => write!(f, "Too many spatial layers"), + PacketError::ErrTooManyPDiff => write!(f, "Too many P-Diff"), + PacketError::ErrH265CorruptedPacket => write!(f, "H265 corrupted packet"), + PacketError::ErrH266CorruptedPacket => write!(f, "H266 corrupted packet"), + PacketError::ErrInvalidH265PacketType => write!(f, "H265 invalid packet type"), + PacketError::ErrH265PACIPHESTooLong => { + write!(f, "H265 PACI PHES field exceeds maximum size of 31 bytes") + } + PacketError::StapASizeLargerThanBuffer(size, buffer) => write!( + f, + "H264 StapA size larger than buffer: {} > {}", + size, buffer + ), + PacketError::NaluTypeIsNotHandled(nalu_type) => { + write!(f, "H264 NALU type is not handled: {}", nalu_type) + } + PacketError::ErrVP8CorruptedPacket => write!(f, "VP8 corrupted packet"), + PacketError::ErrVP9CorruptedPacket => write!(f, "VP9 corrupted packet"), + PacketError::ErrAv1CorruptedPacket => write!(f, "AV1 corrupted packet"), + } + } +} + +impl Error for PacketError {} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/g7xx.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/g7xx.rs new file mode 100644 index 000000000..f1a138104 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/g7xx.rs @@ -0,0 +1,164 @@ +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; + +/// Packetizes G711 RTP packets. +pub type G711Packetizer = G7xxPacketizer; +/// Packetizes G722 RTP packets. +pub type G722Packetizer = G7xxPacketizer; + +/// Generic packetizer for G711 and G722 packets. +#[derive(Default, Debug, Copy, Clone)] +pub struct G7xxPacketizer; + +impl Packetizer for G7xxPacketizer { + /// Payload fragments an G7xx packet across one or more byte arrays + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu == 0 { + return Ok(vec![]); + } + + let mut payload_data_remaining = payload.len(); + let mut payload_data_index = 0; + let mut payloads = Vec::with_capacity(payload_data_remaining / mtu); + while payload_data_remaining > 0 { + let current_fragment_size = std::cmp::min(mtu, payload_data_remaining); + payloads.push( + payload[payload_data_index..payload_data_index + current_fragment_size].to_vec(), + ); + + payload_data_remaining -= current_fragment_size; + payload_data_index += current_fragment_size; + } + + Ok(payloads) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, _last: bool) -> bool { + false + } +} + +/// Depacketizer for G711 packets +#[derive(PartialEq, Eq, Debug, Default, Clone)] +pub struct G711Depacketizer; + +impl Depacketizer for G711Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + _: &mut CodecExtra, + ) -> Result<(), PacketError> { + if !packet.is_empty() { + out.extend_from_slice(packet); + } + + Ok(()) + } + + fn is_partition_head(&self, _payload: &[u8]) -> bool { + true + } + + fn is_partition_tail(&self, _marker: bool, _payload: &[u8]) -> bool { + true + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_g7xx_payload() -> Result<(), PacketError> { + let mut pck = G711Packetizer::default(); + + const TEST_LEN: usize = 10000; + const TEST_MTU: usize = 1500; + + //generate random 8-bit g722 samples + let samples: Vec = (0..TEST_LEN).map(|_| fastrand::u8(..)).collect(); + + //make a copy, for packetizer input + let mut samples_in = vec![0u8; TEST_LEN]; + samples_in.clone_from_slice(&samples); + let samples_in = samples_in.to_vec(); + + //split our samples into payloads + let payloads = pck.packetize(TEST_MTU, &samples_in)?; + + let outcnt = ((TEST_LEN as f64) / (TEST_MTU as f64)).ceil() as usize; + assert_eq!( + outcnt, + payloads.len(), + "Generated {} payloads instead of {}", + payloads.len(), + outcnt + ); + assert_eq!(&samples, &samples_in, "Modified input samples"); + + let samples_out = payloads.concat(); + assert_eq!(&samples_out, &samples_in, "Output samples don't match"); + + let empty = &[]; + let payload = &[0x90, 0x90, 0x90]; + + // Positive MTU, empty payload + let result = pck.packetize(1, empty)?; + assert!(result.is_empty(), "Generated payload should be empty"); + + // 0 MTU, small payload + let result = pck.packetize(0, payload)?; + assert_eq!(result.len(), 0, "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(10, payload)?; + assert_eq!(result.len(), 1, "Generated payload should be the 1"); + + Ok(()) + } + + #[test] + fn test_g7xx_unmarshal() -> Result<(), PacketError> { + let mut pck = G711Depacketizer; + let mut extra = CodecExtra::None; + + // Empty packet + let empty_bytes = &[]; + let mut out = Vec::new(); + let result = pck.depacketize(empty_bytes, &mut out, &mut extra); + assert!( + result.is_ok(), + "Result should not be err in case of empty packet" + ); + + // Normal packet + let raw_bytes: &[u8] = &[0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x90]; + let mut out = Vec::new(); + pck.depacketize(raw_bytes, &mut out, &mut extra)?; + assert_eq!(raw_bytes, &out, "Payload must be same"); + + Ok(()) + } + + #[test] + fn packetize_respects_mtu() -> Result<(), PacketError> { + let payload = vec![0xABu8; 2000]; + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = G711Packetizer::default(); + let pkts = pck.packetize(mtu, &payload)?; + assert!(!pkts.is_empty(), "G7xx produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "G7xx packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264.rs new file mode 100644 index 000000000..bf84e1e40 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264.rs @@ -0,0 +1,927 @@ +#![allow(clippy::all)] + +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; + +/// H264 information describing the depacketized / packetized data +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct H264CodecExtra { + /// Flag which indicates that within [`MediaData`], there is an individual frame + /// containing complete and independent visual information. This frame serves + /// as a reference point for other frames in the video sequence. + /// + /// [`MediaData`]: crate::media::MediaData + pub is_keyframe: bool, +} + +/// Packetizes H264 RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(Default, Debug, Clone)] +pub struct H264Packetizer { + sps_nalu: Option>, + pps_nalu: Option>, +} + +pub const STAPA_NALU_TYPE: u8 = 24; +pub const FUA_NALU_TYPE: u8 = 28; +pub const FUB_NALU_TYPE: u8 = 29; +pub const IDR_NALU_TYPE: u8 = 5; +pub const SPS_NALU_TYPE: u8 = 7; +pub const PPS_NALU_TYPE: u8 = 8; +pub const AUD_NALU_TYPE: u8 = 9; +pub const FILLER_NALU_TYPE: u8 = 12; + +pub const FUA_HEADER_SIZE: usize = 2; +pub const STAPA_HEADER_SIZE: usize = 1; +pub const STAPA_NALU_LENGTH_SIZE: usize = 2; + +pub const NALU_TYPE_BITMASK: u8 = 0x1F; +pub const NALU_REF_IDC_BITMASK: u8 = 0x60; +pub const FU_START_BITMASK: u8 = 0x80; +pub const FU_END_BITMASK: u8 = 0x40; + +pub const OUTPUT_STAP_AHEADER: u8 = 0x78; + +pub static ANNEXB_NALUSTART_CODE: &[u8] = &[0x00, 0x00, 0x00, 0x01]; + +/// Detect whether an H264 RTP payload contains a keyframe. +/// +/// Checks for IDR (Instantaneous Decoding Refresh) NAL units (type 5) +/// in the RTP payload. Handles single NAL units, STAP-A aggregation +/// packets, and FU-A fragmentation units. +/// +/// For FU-A packets, only the start fragment (S=1) is detected as a +/// keyframe since the NAL type is only reliably available there. +pub fn detect_h264_keyframe(payload: &[u8]) -> bool { + if payload.is_empty() { + return false; + } + + let nalu_type = payload[0] & NALU_TYPE_BITMASK; + + match nalu_type { + // Single NAL unit (types 1-23) + 1..=23 => nalu_type == IDR_NALU_TYPE, + + // STAP-A: check all aggregated NALUs + STAPA_NALU_TYPE => { + let mut offset = STAPA_HEADER_SIZE; + while offset + STAPA_NALU_LENGTH_SIZE <= payload.len() { + let nalu_size = ((payload[offset] as usize) << 8) | payload[offset + 1] as usize; + offset += STAPA_NALU_LENGTH_SIZE; + if offset + nalu_size > payload.len() { + break; + } + if let Some(&b0) = payload.get(offset) { + if b0 & NALU_TYPE_BITMASK == IDR_NALU_TYPE { + return true; + } + } + offset += nalu_size; + } + false + } + + // FU-A: check start fragment for IDR type + FUA_NALU_TYPE => { + if payload.len() < FUA_HEADER_SIZE { + return false; + } + let b1 = payload[1]; + // Only the start fragment (S=1) reliably carries the original NAL type + if b1 & FU_START_BITMASK == 0 { + return false; + } + b1 & NALU_TYPE_BITMASK == IDR_NALU_TYPE + } + + _ => false, + } +} + +impl H264Packetizer { + fn next_ind(nalu: &[u8], start: usize) -> (isize, isize) { + let mut zero_count = 0; + + for (i, &b) in nalu[start..].iter().enumerate() { + if b == 0 { + zero_count += 1; + continue; + } else if b == 1 && zero_count >= 2 { + return ((start + i - zero_count) as isize, zero_count as isize + 1); + } + zero_count = 0 + } + (-1, -1) + } + + fn emit(&mut self, nalu: &[u8], mtu: usize, payloads: &mut Vec>) { + if nalu.is_empty() { + return; + } + + let nalu_type = nalu[0] & NALU_TYPE_BITMASK; + let nalu_ref_idc = nalu[0] & NALU_REF_IDC_BITMASK; + + if nalu_type == AUD_NALU_TYPE || nalu_type == FILLER_NALU_TYPE { + return; + } else if nalu_type == SPS_NALU_TYPE { + self.sps_nalu = Some(nalu.to_vec()); + return; + } else if nalu_type == PPS_NALU_TYPE { + self.pps_nalu = Some(nalu.to_vec()); + return; + } else if let (Some(sps_nalu), Some(pps_nalu)) = (&self.sps_nalu, &self.pps_nalu) { + // Pack current NALU with SPS and PPS as STAP-A + let sps_len = (sps_nalu.len() as u16).to_be_bytes(); + let pps_len = (pps_nalu.len() as u16).to_be_bytes(); + + let mut stap_a_nalu = Vec::with_capacity(1 + 2 + sps_nalu.len() + 2 + pps_nalu.len()); + stap_a_nalu.push(OUTPUT_STAP_AHEADER); + stap_a_nalu.extend(sps_len); + stap_a_nalu.extend_from_slice(sps_nalu); + stap_a_nalu.extend(pps_len); + stap_a_nalu.extend_from_slice(pps_nalu); + if stap_a_nalu.len() <= mtu { + payloads.push(stap_a_nalu); + } + } + + if self.sps_nalu.is_some() && self.pps_nalu.is_some() { + self.sps_nalu = None; + self.pps_nalu = None; + } + + // Single NALU + if nalu.len() <= mtu { + payloads.push(nalu.to_vec()); + return; + } + + // FU-A + let max_fragment_size = mtu as isize - FUA_HEADER_SIZE as isize; + + // The FU payload consists of fragments of the payload of the fragmented + // NAL unit so that if the fragmentation unit payloads of consecutive + // FUs are sequentially concatenated, the payload of the fragmented NAL + // unit can be reconstructed. The NAL unit type octet of the fragmented + // NAL unit is not included as such in the fragmentation unit payload, + // but rather the information of the NAL unit type octet of the + // fragmented NAL unit is conveyed in the F and NRI fields of the FU + // indicator octet of the fragmentation unit and in the type field of + // the FU header. An FU payload MAY have any number of octets and MAY + // be empty. + + let nalu_data = nalu; + // According to the RFC, the first octet is skipped due to redundant information + let mut nalu_data_index = 1; + let nalu_data_length = nalu.len() as isize - nalu_data_index; + let mut nalu_data_remaining = nalu_data_length; + + if std::cmp::min(max_fragment_size, nalu_data_remaining) <= 0 { + return; + } + + while nalu_data_remaining > 0 { + let current_fragment_size = std::cmp::min(max_fragment_size, nalu_data_remaining); + //out: = make([]byte, fuaHeaderSize + currentFragmentSize) + let mut out = Vec::with_capacity(FUA_HEADER_SIZE + current_fragment_size as usize); + // +---------------+ + // |0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+ + // |F|NRI| Type | + // +---------------+ + let b0 = FUA_NALU_TYPE | nalu_ref_idc; + out.push(b0); + + // +---------------+ + //|0|1|2|3|4|5|6|7| + //+-+-+-+-+-+-+-+-+ + //|S|E|R| Type | + //+---------------+ + + let mut b1 = nalu_type; + if nalu_data_remaining == nalu_data_length { + // Set start bit + b1 |= 1 << 7; + } else if nalu_data_remaining - current_fragment_size == 0 { + // Set end bit + b1 |= 1 << 6; + } + out.push(b1); + + out.extend_from_slice( + &nalu_data + [nalu_data_index as usize..(nalu_data_index + current_fragment_size) as usize], + ); + payloads.push(out); + + nalu_data_remaining -= current_fragment_size; + nalu_data_index += current_fragment_size; + } + } +} + +impl Packetizer for H264Packetizer { + /// Payload fragments a H264 packet across one or more byte arrays + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu == 0 { + return Ok(vec![]); + } + + let mut payloads = vec![]; + + let (mut next_ind_start, mut next_ind_len) = H264Packetizer::next_ind(payload, 0); + if next_ind_start == -1 { + self.emit(payload, mtu, &mut payloads); + } else { + while next_ind_start != -1 { + let prev_start = (next_ind_start + next_ind_len) as usize; + let (next_ind_start2, next_ind_len2) = + H264Packetizer::next_ind(payload, prev_start); + next_ind_start = next_ind_start2; + next_ind_len = next_ind_len2; + if next_ind_start != -1 { + self.emit( + &payload[prev_start..next_ind_start as usize], + mtu, + &mut payloads, + ); + } else { + // Emit until end of stream, no end indicator found + self.emit(&payload[prev_start..], mtu, &mut payloads); + } + } + } + + Ok(payloads) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +/// Depacketizes H264 RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(PartialEq, Eq, Debug, Default, Clone)] +pub struct H264Depacketizer { + /// Whether to output in AVC format (length-prefixed NALUs) instead of Annex B format + pub is_avc: bool, + fua_buffer: Option>, +} + +impl Depacketizer for H264Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + // Roughly account for Annex B start codes or AVC length prefixes. + let estimated_packets = (packets_size / 1200).saturating_add(1); + Some(packets_size.saturating_add(4usize.saturating_mul(estimated_packets))) + } + + /// depacketize parses the passed byte slice and stores the result in the + /// H264Packet this method is called upon + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + if packet.len() == 0 { + return Err(PacketError::ErrShortPacket); + } + + // NALU Types + // https://tools.ietf.org/html/rfc6184#section-5.4 + let b0 = packet[0]; + let nalu_type = b0 & NALU_TYPE_BITMASK; + + match nalu_type { + t @ 1..=23 => { + let is_keyframe = if let CodecExtra::H264(e) = extra { + (t == IDR_NALU_TYPE) | e.is_keyframe + } else { + t == IDR_NALU_TYPE + }; + *extra = CodecExtra::H264(H264CodecExtra { is_keyframe }); + + if self.is_avc { + out.extend_from_slice(&(packet.len() as u32).to_be_bytes()); + } else { + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + } + out.extend_from_slice(packet); + Ok(()) + } + STAPA_NALU_TYPE => { + let mut curr_offset = STAPA_HEADER_SIZE; + while curr_offset + 1 < packet.len() { + let nalu_size = + ((packet[curr_offset] as usize) << 8) | packet[curr_offset + 1] as usize; + curr_offset += STAPA_NALU_LENGTH_SIZE; + + if curr_offset + nalu_size > packet.len() { + return Err(PacketError::StapASizeLargerThanBuffer( + nalu_size, + packet.len() - curr_offset, + )); + } + + let Some(b0) = packet.get(curr_offset) else { + continue; + }; + let t = b0 & NALU_TYPE_BITMASK; + let is_keyframe = if let CodecExtra::H264(e) = extra { + (t == IDR_NALU_TYPE) | e.is_keyframe + } else { + t == IDR_NALU_TYPE + }; + *extra = CodecExtra::H264(H264CodecExtra { is_keyframe }); + + if self.is_avc { + out.extend_from_slice(&(nalu_size as u32).to_be_bytes()); + } else { + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + } + out.extend_from_slice(&packet[curr_offset..curr_offset + nalu_size]); + curr_offset += nalu_size; + } + + Ok(()) + } + FUA_NALU_TYPE => { + if packet.len() < FUA_HEADER_SIZE as usize { + return Err(PacketError::ErrShortPacket); + } + + if self.fua_buffer.is_none() { + self.fua_buffer = Some(Vec::new()); + } + + if let Some(fua_buffer) = &mut self.fua_buffer { + fua_buffer.extend_from_slice(&packet[FUA_HEADER_SIZE as usize..]); + } + + let b1 = packet[1]; + if b1 & FU_END_BITMASK != 0 { + let nalu_ref_idc = b0 & NALU_REF_IDC_BITMASK; + let fragmented_nalu_type = b1 & NALU_TYPE_BITMASK; + + let is_keyframe = if let CodecExtra::H264(e) = extra { + (fragmented_nalu_type == IDR_NALU_TYPE) | e.is_keyframe + } else { + fragmented_nalu_type == IDR_NALU_TYPE + }; + *extra = CodecExtra::H264(H264CodecExtra { is_keyframe }); + + if let Some(fua_buffer) = self.fua_buffer.take() { + if self.is_avc { + out.extend_from_slice(&((fua_buffer.len() + 1) as u32).to_be_bytes()); + } else { + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + } + out.push(nalu_ref_idc | fragmented_nalu_type); + out.extend_from_slice(&fua_buffer); + } + + Ok(()) + } else { + Ok(()) + } + } + _ => Err(PacketError::NaluTypeIsNotHandled(nalu_type)), + } + } + + /// is_partition_head checks if this is the head of a packetized nalu stream. + fn is_partition_head(&self, packet: &[u8]) -> bool { + if packet.len() < 2 { + return false; + } + + if packet[0] & NALU_TYPE_BITMASK == FUA_NALU_TYPE + || packet[0] & NALU_TYPE_BITMASK == FUB_NALU_TYPE + { + (packet[1] & FU_START_BITMASK) != 0 + } else { + true + } + } + + fn is_partition_tail(&self, marker: bool, _packet: &[u8]) -> bool { + marker + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_h264_payload() -> Result<(), PacketError> { + let empty = &[]; + let small_payload = &[0x90, 0x90, 0x90]; + let multiple_payload = &[0x00, 0x00, 0x01, 0x90, 0x00, 0x00, 0x01, 0x90]; + let large_payload = &[ + 0x00, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, + ]; + let large_payload_packetized = vec![ + &[0x1c, 0x80, 0x01, 0x02, 0x03], + &[0x1c, 0x00, 0x04, 0x05, 0x06], + &[0x1c, 0x00, 0x07, 0x08, 0x09], + &[0x1c, 0x00, 0x10, 0x11, 0x12], + &[0x1c, 0x40, 0x13, 0x14, 0x15], + ]; + + let mut pck = H264Packetizer::default(); + + // Positive MTU, empty payload + let result = pck.packetize(1, empty)?; + assert!(result.is_empty(), "Generated payload should be empty"); + + // 0 MTU, small payload + let result = pck.packetize(0, small_payload)?; + assert_eq!(result.len(), 0, "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(1, small_payload)?; + assert_eq!(result.len(), 0, "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(5, small_payload)?; + assert_eq!(result.len(), 1, "Generated payload should be the 1"); + assert_eq!( + result[0].len(), + small_payload.len(), + "Generated payload should be the same size as original payload size" + ); + + // Multiple NALU in a single payload + let result = pck.packetize(5, multiple_payload)?; + assert_eq!(result.len(), 2, "2 nal units should be broken out"); + for i in 0..2 { + assert_eq!( + result[i].len(), + 1, + "Payload {} of 2 is packed incorrectly", + i + 1, + ); + } + + // Large Payload split across multiple RTP Packets + let result = pck.packetize(5, large_payload)?; + assert_eq!( + result, large_payload_packetized, + "FU-A packetization failed" + ); + + // Nalu type 9 or 12 + let small_payload2 = &[0x09, 0x00, 0x00]; + let result = pck.packetize(5, small_payload2)?; + assert_eq!(result.len(), 0, "Generated payload should be empty"); + + Ok(()) + } + + macro_rules! test_h264 { + ($name:tt, $is_avc:expr, $is_ok: expr, $payload:expr, $err:tt) => { + #[test] + fn $name() -> Result<(), PacketError> { + let mut pkt = H264Depacketizer::default(); + pkt.is_avc = $is_avc; + let mut extra = CodecExtra::None; + let mut out: Vec = Vec::new(); + let result = pkt.depacketize($payload, &mut out, &mut extra); + if $is_ok { + assert!(result.is_ok(), $err); + } else { + assert!(result.is_err(), $err); + } + Ok(()) + } + }; + } + + test_h264!( + nil_payload, + false, + false, + &[], + "Unmarshal did not fail on nil payload" + ); + test_h264!( + unit_delimiter, + false, + true, + &[0x09, 0x30], + "Unmarshal should accept minimal h.264 access unit delimiter" + ); + test_h264!( + end_of_sequence_nalu, + false, + true, + &[0x0A], + "Unmarshal should accept end of sequence NALU" + ); + test_h264!( + not_handled, + false, + false, + &[0xFF, 0x00, 0x00], + "Unmarshal accepted a packet with a NALU Type we don't handle" + ); + test_h264!( + incomplete_single_payload_multi_nalu, + false, + false, + &[ + 0x78, 0x00, 0x0f, 0x67, 0x42, 0xc0, 0x1f, 0x1a, 0x32, 0x35, 0x01, 0x40, 0x7a, 0x40, + 0x3c, 0x22, 0x11, + ], + "Unmarshal accepted a STAP-A packet with insufficient data" + ); + + #[test] + fn single_payload() -> Result<(), PacketError> { + let mut pkt = H264Depacketizer::default(); + let mut extra = CodecExtra::None; + let mut out: Vec = Vec::new(); + let single_payload = &[0x90, 0x90, 0x90]; + let _ = pkt.depacketize(single_payload, &mut out, &mut extra); + let single_payload_unmarshaled = &[0x00, 0x00, 0x00, 0x01, 0x90, 0x90, 0x90]; + assert_eq!( + out, single_payload_unmarshaled, + "Unmarshaling a single payload shouldn't modify the payload" + ); + Ok(()) + } + + #[test] + fn single_payload_avc() -> Result<(), PacketError> { + let mut pkt = H264Depacketizer::default(); + pkt.is_avc = true; + let mut extra = CodecExtra::None; + let mut out: Vec = Vec::new(); + let single_payload = &[0x90, 0x90, 0x90]; + let _ = pkt.depacketize(single_payload, &mut out, &mut extra); + let single_payload_unmarshaled_avc = &[0x00, 0x00, 0x00, 0x03, 0x90, 0x90, 0x90]; + assert_eq!( + out, single_payload_unmarshaled_avc, + "Unmarshaling a single payload into avc stream shouldn't modify the payload" + ); + Ok(()) + } + + #[test] + fn h264_large_out() -> Result<(), PacketError> { + let large_payload_packetized = vec![ + &[0x1c, 0x80, 0x01, 0x02, 0x03], + &[0x1c, 0x00, 0x04, 0x05, 0x06], + &[0x1c, 0x00, 0x07, 0x08, 0x09], + &[0x1c, 0x00, 0x10, 0x11, 0x12], + &[0x1c, 0x40, 0x13, 0x14, 0x15], + ]; + + let large_payload = &[ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + ]; + + let mut pkt = H264Depacketizer::default(); + let mut extra = CodecExtra::None; + + let mut large_out = Vec::new(); + for p in &large_payload_packetized { + pkt.depacketize(*p, &mut large_out, &mut extra)?; + } + assert_eq!( + large_out, large_payload, + "Failed to unmarshal a large payload" + ); + + Ok(()) + } + + #[test] + fn h264_large_out_avc() -> Result<(), PacketError> { + let large_payload_packetized = vec![ + &[0x1c, 0x80, 0x01, 0x02, 0x03], + &[0x1c, 0x00, 0x04, 0x05, 0x06], + &[0x1c, 0x00, 0x07, 0x08, 0x09], + &[0x1c, 0x00, 0x10, 0x11, 0x12], + &[0x1c, 0x40, 0x13, 0x14, 0x15], + ]; + + let large_payload_avc = &[ + 0x00, 0x00, 0x00, 0x10, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + ]; + + let mut avc_pkt = H264Depacketizer { + is_avc: true, + ..Default::default() + }; + + let mut extra = CodecExtra::None; + + let mut large_out_avc = Vec::new(); + for p in &large_payload_packetized { + avc_pkt.depacketize(*p, &mut large_out_avc, &mut extra)?; + } + assert_eq!( + large_out_avc, large_payload_avc, + "Failed to unmarshal a large payload into avc stream" + ); + + Ok(()) + } + + #[test] + fn single_payload_multi_nalu() -> Result<(), PacketError> { + let single_payload_multi_nalu = &[ + 0x78, 0x00, 0x0f, 0x67, 0x42, 0xc0, 0x1f, 0x1a, 0x32, 0x35, 0x01, 0x40, 0x7a, 0x40, + 0x3c, 0x22, 0x11, 0xa8, 0x00, 0x05, 0x68, 0x1a, 0x34, 0xe3, 0xc8, 0x00, + ]; + let single_payload_multi_nalu_unmarshaled = &[ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f, 0x1a, 0x32, 0x35, 0x01, 0x40, 0x7a, + 0x40, 0x3c, 0x22, 0x11, 0xa8, 0x00, 0x00, 0x00, 0x01, 0x68, 0x1a, 0x34, 0xe3, 0xc8, + ]; + + let mut pkt = H264Depacketizer::default(); + + let mut extra = CodecExtra::None; + + let mut out = Vec::new(); + pkt.depacketize(single_payload_multi_nalu, &mut out, &mut extra)?; + assert_eq!( + out, single_payload_multi_nalu_unmarshaled, + "Failed to unmarshal a single packet with multiple NALUs" + ); + + Ok(()) + } + + #[test] + fn single_payload_multi_nalu_avc() -> Result<(), PacketError> { + let single_payload_multi_nalu = &[ + 0x78, 0x00, 0x0f, 0x67, 0x42, 0xc0, 0x1f, 0x1a, 0x32, 0x35, 0x01, 0x40, 0x7a, 0x40, + 0x3c, 0x22, 0x11, 0xa8, 0x00, 0x05, 0x68, 0x1a, 0x34, 0xe3, 0xc8, 0x00, + ]; + let single_payload_multi_nalu_unmarshaled_avc = &[ + 0x00, 0x00, 0x00, 0x0f, 0x67, 0x42, 0xc0, 0x1f, 0x1a, 0x32, 0x35, 0x01, 0x40, 0x7a, + 0x40, 0x3c, 0x22, 0x11, 0xa8, 0x00, 0x00, 0x00, 0x05, 0x68, 0x1a, 0x34, 0xe3, 0xc8, + ]; + + let mut avc_pkt = H264Depacketizer::default(); + avc_pkt.is_avc = true; + + let mut extra = CodecExtra::None; + + let mut out = Vec::new(); + avc_pkt.depacketize(single_payload_multi_nalu, &mut out, &mut extra)?; + assert_eq!( + out, single_payload_multi_nalu_unmarshaled_avc, + "Failed to unmarshal a single packet with multiple NALUs into avc stream" + ); + + Ok(()) + } + + #[test] + fn test_h264_partition_head_checker_is_partition_head() -> Result<(), PacketError> { + let h264 = H264Depacketizer::default(); + let empty_nalu = &[]; + assert!( + !h264.is_partition_head(empty_nalu), + "empty nalu must not be a partition head" + ); + + let single_nalu = &[1, 0]; + assert!( + h264.is_partition_head(single_nalu), + "single nalu must be a partition head" + ); + + let stapa_nalu = &[STAPA_NALU_TYPE, 0]; + assert!( + h264.is_partition_head(stapa_nalu), + "stapa nalu must be a partition head" + ); + + let fua_start_nalu = &[FUA_NALU_TYPE, FU_START_BITMASK]; + assert!( + h264.is_partition_head(fua_start_nalu), + "fua start nalu must be a partition head" + ); + + let fua_end_nalu = &[FUA_NALU_TYPE, FU_END_BITMASK]; + assert!( + !h264.is_partition_head(fua_end_nalu), + "fua end nalu must not be a partition head" + ); + + let fub_start_nalu = &[FUB_NALU_TYPE, FU_START_BITMASK]; + assert!( + h264.is_partition_head(fub_start_nalu), + "fub start nalu must be a partition head" + ); + + let fub_end_nalu = &[FUB_NALU_TYPE, FU_END_BITMASK]; + assert!( + !h264.is_partition_head(fub_end_nalu), + "fub end nalu must not be a partition head" + ); + + Ok(()) + } + + #[test] + fn test_h264_packetizer_payload_sps_and_pps_handling() -> Result<(), PacketError> { + let mut pck = H264Packetizer::default(); + let expected: Vec<&[u8]> = vec![ + &[ + 0x78, 0x00, 0x03, 0x07, 0x00, 0x01, 0x00, 0x03, 0x08, 0x02, 0x03, + ], + &[0x05, 0x04, 0x05], + ]; + + // When packetizing SPS and PPS are emitted with following NALU + let res = pck.packetize(1500, &[0x07, 0x00, 0x01])?; + assert!(res.is_empty(), "Generated payload should be empty"); + + let res = pck.packetize(1500, &[0x08, 0x02, 0x03])?; + assert!(res.is_empty(), "Generated payload should be empty"); + + let actual = pck.packetize(1500, &[0x05, 0x04, 0x05])?; + assert_eq!(actual, expected, "SPS and PPS aren't packed together"); + + Ok(()) + } + + #[test] + fn test_h264_depacketizer_idr_handling() -> Result<(), PacketError> { + let mut pck = H264Depacketizer::default(); + let mut extra = CodecExtra::None; + let mut out = vec![]; + + // First byte is NALU type + let packet = [0x85]; + pck.depacketize(&packet, &mut out, &mut extra)?; + let CodecExtra::H264(e) = extra else { + panic!("Expected CodecExtra::H264"); + }; + assert!(e.is_keyframe); + + // First byte is STAPA NALU type + let packet = vec![ + vec![ + 120, 0, 15, 103, 66, 192, 21, 140, 141, 64, 160, 203, 207, 0, 240, 136, 70, 160, 0, + 4, 104, 206, 60, 128, 1, 20, 101, + ], + vec![0; 276], + ] + .into_iter() + .flatten() + .collect::>(); + pck.depacketize(packet.as_slice(), &mut out, &mut extra)?; + let CodecExtra::H264(e) = extra else { + panic!("Expected CodecExtra::H264"); + }; + assert!(e.is_keyframe); + + // First byte is FUA NALU type + let packet = [124, 69]; + pck.depacketize(&packet, &mut out, &mut extra)?; + let CodecExtra::H264(e) = extra else { + panic!("Expected CodecExtra::H264"); + }; + assert!(e.is_keyframe); + Ok(()) + } + + #[test] + fn parse_first_packet() { + const PACKET: &[u8] = &[ + 120, 000, 015, 103, 066, 192, 021, 140, 141, 064, 160, 203, 207, 000, 240, 136, 070, + 160, 000, 004, 104, 206, 060, 128, 000, 204, 101, 184, 000, 004, 000, 000, 005, 057, + 049, 064, 000, 064, 222, 078, 078, 078, 078, 078, 078, 078, 078, 078, 078, 078, 078, + 078, 078, 078, 078, 078, 078, 078, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, + 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, + 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 173, 223, 039, 125, 247, 223, + 125, 245, 215, 093, 117, 215, 093, 117, 214, 239, 174, 187, 235, 174, 186, 235, 174, + 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, + 174, 186, 235, 174, 183, 093, 117, 215, 093, 117, 215, 093, 117, 215, 093, 117, 215, + 093, 117, 215, 093, 117, 215, 092, 189, 117, 215, 093, 117, 215, 093, 117, 215, 093, + 117, 215, 093, 117, 215, 093, 117, 215, 093, 117, 214, 239, 190, 251, 239, 190, 186, + 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, 186, 235, 174, + 186, 235, 174, 186, 235, 175, 227, 255, 240, 247, 021, 223, 125, 247, 223, 125, 247, + 223, 125, 247, 223, 125, 247, 223, 125, 248, + ]; + + let mut pck = H264Depacketizer::default(); + let mut extra = CodecExtra::None; + let mut out = vec![]; + pck.depacketize(PACKET, &mut out, &mut extra).unwrap(); + } + + #[test] + fn test_out_of_bounds_access() { + const PACKET: &[u8] = &[STAPA_NALU_TYPE, 0x00, 0x00]; + + let mut pck = H264Depacketizer::default(); + let mut extra = CodecExtra::None; + let mut out = vec![]; + pck.depacketize(PACKET, &mut out, &mut extra).unwrap(); + } + + #[test] + fn test_detect_h264_keyframe() { + // Empty payload + assert!(!detect_h264_keyframe(&[])); + + // Single IDR NAL unit (type 5) + assert!(detect_h264_keyframe(&[0x65, 0x00, 0x00])); // 0x65 & 0x1F = 5 + + // Single non-IDR NAL unit (type 1 = coded slice) + assert!(!detect_h264_keyframe(&[0x41, 0x00, 0x00])); // 0x41 & 0x1F = 1 + + // SPS (type 7) - not a keyframe + assert!(!detect_h264_keyframe(&[0x67, 0x00, 0x00])); + + // PPS (type 8) - not a keyframe + assert!(!detect_h264_keyframe(&[0x68, 0x00, 0x00])); + + // STAP-A containing IDR + // Header: type 24 (STAP-A) + // NALU 1: 2 bytes, type 7 (SPS) + // NALU 2: 2 bytes, type 5 (IDR) + let stapa_with_idr = [ + 0x18, // STAP-A (24) + 0x00, 0x02, 0x67, 0xAA, // SPS: size=2, type=7 + 0x00, 0x02, 0x65, 0xBB, // IDR: size=2, type=5 + ]; + assert!(detect_h264_keyframe(&stapa_with_idr)); + + // STAP-A without IDR + let stapa_no_idr = [ + 0x18, // STAP-A (24) + 0x00, 0x02, 0x67, 0xAA, // SPS: size=2, type=7 + 0x00, 0x02, 0x68, 0xBB, // PPS: size=2, type=8 + ]; + assert!(!detect_h264_keyframe(&stapa_no_idr)); + + // FU-A start fragment with IDR type + let fua_start_idr = [ + 0x7C, // FU indicator: type=28 (FU-A) + 0x85, // FU header: S=1, type=5 (IDR) + 0x00, 0x00, + ]; + assert!(detect_h264_keyframe(&fua_start_idr)); + + // FU-A start fragment with non-IDR type + let fua_start_non_idr = [ + 0x7C, // FU indicator: type=28 (FU-A) + 0x81, // FU header: S=1, type=1 (non-IDR) + 0x00, 0x00, + ]; + assert!(!detect_h264_keyframe(&fua_start_non_idr)); + + // FU-A continuation fragment (S=0) - cannot detect + let fua_continuation = [ + 0x7C, // FU indicator: type=28 (FU-A) + 0x05, // FU header: S=0, type=5 (IDR) but S=0 + 0x00, 0x00, + ]; + assert!(!detect_h264_keyframe(&fua_continuation)); + + // FU-A too short + assert!(!detect_h264_keyframe(&[0x7C])); + } + + #[test] + fn packetize_respects_mtu() -> Result<(), PacketError> { + // NAL header byte (type=5, IDR slice) + payload. + let mut payload = vec![0x05u8]; + payload.extend(std::iter::repeat(0xABu8).take(2000)); + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = H264Packetizer::default(); + let pkts = pck.packetize(mtu, &payload)?; + assert!(!pkts.is_empty(), "H264 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "H264 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264_profile.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264_profile.rs new file mode 100644 index 000000000..f2357565b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h264_profile.rs @@ -0,0 +1,374 @@ +use crate::util::BitPattern; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct H264ProfileLevel { + profile: H264Profile, + level_idc: H264LevelIdc, +} + +impl H264ProfileLevel { + // TODO: The default should really be Baseline and Level1 + // according to the spec: https://tools.ietf.org/html/rfc6184#section-8.1. However, libWebRTC + // specifies Level3_1 to not break backwards compatibility and we copy them. + // When libWebRTC updates we are probably safe to do the same. + // + // See: https://webrtc.googlesource.com/src/+/refs/heads/main/api/video_codecs/h264_profile_level_id.cc#182 + pub(crate) const FALLBACK: Self = Self { + profile: H264Profile::Baseline, + level_idc: H264LevelIdc::Level3_1, + }; + + /// Different combinations of profile-iop and profile-idc match to the same profile. + /// See table 5 in https://www.rfc-editor.org/rfc/rfc6184#section-8.1 + /// + /// The first value in each tuple is the profile that is matched if the profile-idc and the + /// BitPattern matches a given fmtp line. + const PROFILES: &'static [(H264Profile, H264ProfileIdc, BitPattern)] = &[ + // Constrained Baseline + ( + H264Profile::ConstrainedBaseline, + H264ProfileIdc::X42, + BitPattern::new(*b"x1xx0000"), + ), + ( + H264Profile::ConstrainedBaseline, + H264ProfileIdc::X4D, + BitPattern::new(*b"1xxx0000"), + ), + ( + H264Profile::ConstrainedBaseline, + H264ProfileIdc::X58, + BitPattern::new(*b"11xx0000"), + ), + // Baseline + ( + H264Profile::Baseline, + H264ProfileIdc::X42, + BitPattern::new(*b"x0xx0000"), + ), + ( + H264Profile::Baseline, + H264ProfileIdc::X58, + BitPattern::new(*b"10xx0000"), + ), + // Main + ( + H264Profile::Main, + H264ProfileIdc::X4D, + BitPattern::new(*b"0x0x0000"), + ), + // Extended + ( + H264Profile::Extended, + H264ProfileIdc::X58, + BitPattern::new(*b"00xx0000"), + ), + // High(No constraints) + ( + H264Profile::High, + H264ProfileIdc::X64, + BitPattern::new(*b"00000000"), + ), + // Constrained High + ( + H264Profile::ConstrainedHigh, + H264ProfileIdc::X64, + BitPattern::new(*b"00001100"), + ), + ( + H264Profile::High10, + H264ProfileIdc::X6E, + BitPattern::new(*b"00000000"), + ), + ( + H264Profile::High422, + H264ProfileIdc::X7A, + BitPattern::new(*b"00000000"), + ), + ( + H264Profile::High444Predictive, + H264ProfileIdc::XF4, + BitPattern::new(*b"00000000"), + ), + // Intra profiles + ( + H264Profile::High10Intra, + H264ProfileIdc::X6E, + BitPattern::new(*b"00010000"), + ), + ( + H264Profile::High422Intra, + H264ProfileIdc::X7A, + BitPattern::new(*b"00010000"), + ), + ( + H264Profile::High444Intra, + H264ProfileIdc::XF4, + BitPattern::new(*b"00010000"), + ), + ( + H264Profile::CAVLC444Intra, + H264ProfileIdc::X2C, + BitPattern::new(*b"00010000"), + ), + ]; + + /// Construct a new H264ProfileLevel. + /// + /// Returns `Some(Self)` only if the provided parameters identify a valid profile. + fn new(profile_idc: H264ProfileIdc, profile_iop: u8, level_idc: H264LevelIdc) -> Option { + Self::PROFILES + .iter() + .find_map(|&(profile, expected_pidc, iop_pattern)| { + // Profile IDC must match + if expected_pidc != profile_idc { + return None; + } + // The profile-iop must match the pattern for the profile + if !iop_pattern.matches(profile_iop) { + return None; + } + + Some(Self { profile, level_idc }) + }) + } + + /// Returns the H.264 profile (Baseline, Main, High, etc.). + pub(crate) fn profile(&self) -> H264Profile { + self.profile + } + + /// Returns the H.264 level (Level 3.1, Level 4.0, etc.). + pub(crate) fn level(&self) -> H264LevelIdc { + self.level_idc + } +} + +impl TryFrom for H264ProfileLevel { + type Error = (); + + fn try_from(value: u32) -> Result { + const CONSTRAINT_SET3_FLAG: u8 = 0x10; + + let bytes = value.to_be_bytes(); + + let profile_idc = bytes[1].try_into()?; + let profile_iop = bytes[2]; + let mut profile_level = bytes[3].try_into()?; + + // When profile_idc is equal to 66, 77, or 88 (the Baseline, Main, or + // Extended profile), level_idc is equal to 11, and bit 4 + // (constraint_set3_flag) of the profile-iop byte is equal to 1, + // the default level is Level 1b. + if [ + H264ProfileIdc::X42, + H264ProfileIdc::X4D, + H264ProfileIdc::X58, + ] + .contains(&profile_idc) + && profile_level == H264LevelIdc::Level1 + { + profile_level = if (profile_iop & CONSTRAINT_SET3_FLAG) != 0 { + H264LevelIdc::Level1B + } else { + H264LevelIdc::Level1 + }; + } + + Self::new(profile_idc, profile_iop, profile_level).ok_or(()) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum H264Profile { + Baseline, + ConstrainedBaseline, + Main, + Extended, + High, + ConstrainedHigh, + High10, + High422, + High444Predictive, + High10Intra, + High422Intra, + High444Intra, + CAVLC444Intra, +} + +/// The various h264 profile_idc, not all of these have a name, +/// but they are a constrained portion of `u8`, hence an +/// enum to prevent using unspecified values. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u8)] +enum H264ProfileIdc { + X2C = 44_u8, + X42 = 66_u8, // B + X4D = 77_u8, // M + X58 = 88_u8, // E + X64 = 100_u8, + X6E = 110_u8, + X7A = 122_u8, + XF4 = 244_u8, +} + +impl TryFrom for H264ProfileIdc { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + x if (Self::X2C as u8) == x => Ok(Self::X2C), + x if (Self::X42 as u8) == x => Ok(Self::X42), + x if (Self::X4D as u8) == x => Ok(Self::X4D), + x if (Self::X58 as u8) == x => Ok(Self::X58), + x if (Self::X64 as u8) == x => Ok(Self::X64), + x if (Self::X6E as u8) == x => Ok(Self::X6E), + x if (Self::X7A as u8) == x => Ok(Self::X7A), + x if (Self::XF4 as u8) == x => Ok(Self::XF4), + _ => Err(()), + } + } +} + +// Per libWebRTC +// All values are equal to ten times the level number, except level 1b which is +// special. +// https://datatracker.ietf.org/doc/html/rfc6184#section-8.2.2 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +#[rustfmt::skip] +pub(crate) enum H264LevelIdc { + Level1B = 0_u8, + Level1 = 10_u8, + Level1_1 = 11_u8, + Level1_2 = 12_u8, + Level1_3 = 13_u8, + Level2 = 20_u8, + Level2_1 = 21_u8, + Level2_2 = 22_u8, + Level3 = 30_u8, + Level3_1 = 31_u8, + Level3_2 = 32_u8, + Level4 = 40_u8, + Level4_1 = 41_u8, + Level4_2 = 42_u8, + Level5 = 50_u8, + Level5_1 = 51_u8, + Level5_2 = 52_u8, +} + +impl H264LevelIdc { + /// Returns the ordinal position (0-16) representing capability order. + /// This takes into account that Level1B is between Level1 and Level1_1. + pub(crate) fn ordinal(self) -> usize { + match self { + Self::Level1 => 0, + Self::Level1B => 1, + Self::Level1_1 => 2, + Self::Level1_2 => 3, + Self::Level1_3 => 4, + Self::Level2 => 5, + Self::Level2_1 => 6, + Self::Level2_2 => 7, + Self::Level3 => 8, + Self::Level3_1 => 9, + Self::Level3_2 => 10, + Self::Level4 => 11, + Self::Level4_1 => 12, + Self::Level4_2 => 13, + Self::Level5 => 14, + Self::Level5_1 => 15, + Self::Level5_2 => 16, + } + } +} + +impl PartialOrd for H264LevelIdc { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for H264LevelIdc { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.ordinal().cmp(&other.ordinal()) + } +} + +impl TryFrom for H264LevelIdc { + type Error = (); + + #[rustfmt::skip] + fn try_from(value: u8) -> Result { + use H264LevelIdc::*; + + match value { + x if (Level1B as u8) == x => Ok(Level1B), + x if (Level1 as u8) == x => Ok(Level1), + x if (Level1_1 as u8) == x => Ok(Level1_1), + x if (Level1_2 as u8) == x => Ok(Level1_2), + x if (Level1_3 as u8) == x => Ok(Level1_3), + x if (Level2 as u8) == x => Ok(Level2), + x if (Level2_1 as u8) == x => Ok(Level2_1), + x if (Level2_2 as u8) == x => Ok(Level2_2), + x if (Level3 as u8) == x => Ok(Level3), + x if (Level3_1 as u8) == x => Ok(Level3_1), + x if (Level3_2 as u8) == x => Ok(Level3_2), + x if (Level4 as u8) == x => Ok(Level4), + x if (Level4_1 as u8) == x => Ok(Level4_1), + x if (Level4_2 as u8) == x => Ok(Level4_2), + x if (Level5 as u8) == x => Ok(Level5), + x if (Level5_1 as u8) == x => Ok(Level5_1), + x if (Level5_2 as u8) == x => Ok(Level5_2), + _ => Err(()), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_profile_level_parsing() { + struct Case { + profile_level_id: u32, + expected: Option<(H264Profile, H264LevelIdc)>, + msg: &'static str, + } + + let cases = [ + // Test 1: High profile (no constraint flags) -> should parse + Case { + profile_level_id: 0x64001f, + expected: Some((H264Profile::High, H264LevelIdc::Level3_1)), + msg: "0x64001f should parse as High level 3.1", + }, + // Test 2: Constrained High (constraint_set4_flag and + // constraint_set5_flag set) -> should parse + Case { + profile_level_id: 0x640c34, + expected: Some((H264Profile::ConstrainedHigh, H264LevelIdc::Level5_2)), + msg: "0x640c34 should parse as Constrained High level 5.2", + }, + // Test 3: Undefined profile-iop combination -> should NOT parse + Case { + profile_level_id: 0x64101f, + expected: None, + msg: "0x64101f should not parse, profile-iop 0x10 is not defined for profile-idc 0x64", + }, + ]; + + for Case { + profile_level_id, + expected, + msg, + } in cases.into_iter() + { + let parsed = H264ProfileLevel::try_from(profile_level_id) + .ok() + .map(|pl| (pl.profile(), pl.level())); + assert_eq!(parsed, expected, "{msg}"); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265.rs new file mode 100644 index 000000000..8ddb0a034 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265.rs @@ -0,0 +1,5203 @@ +#![allow(clippy::all)] +#![allow(unused)] + +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; +use arrayvec::ArrayVec; +use tracing::warn; + +/// H265 (HEVC) information describing the depacketized / packetized data. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct H265CodecExtra { + /// Flag which indicates that within [`MediaData`], there is an individual frame + /// containing complete and independent visual information. This frame serves + /// as a reference point for other frames in the video sequence. + /// + /// [`MediaData`]: crate::media::MediaData + pub is_keyframe: bool, +} + +/// +/// Network Abstraction Unit Header implementation +/// + +const H265NALU_HEADER_SIZE: usize = 2; +/// https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 +const H265NALU_AGGREGATION_PACKET_TYPE: u8 = 48; +/// https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 +const H265NALU_FRAGMENTATION_UNIT_TYPE: u8 = 49; +/// https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.4 +const H265NALU_PACI_PACKET_TYPE: u8 = 50; +/// Maximum PHES (Payload Header Extension Structure) size in PACI packets (RFC 7798 §4.4.4) +const H265PACI_MAX_PHES_SIZE: usize = 31; + +// HEVC NAL unit type values as defined by the H.265 / HEVC bitstream specification, +// ITU-T Rec. H.265 | ISO/IEC 23008-2, clause 7.4.2.2 ("NAL unit header semantics"), +// Table 7-1 (NAL unit type assignments). +const H265NALU_VPS_NALU_TYPE: u8 = 32; +const H265NALU_SPS_NALU_TYPE: u8 = 33; +const H265NALU_PPS_NALU_TYPE: u8 = 34; +const H265NALU_AUD_NALU_TYPE: u8 = 35; +const H265NALU_FILLER_NALU_TYPE: u8 = 38; + +// IRAP (Intra Random Access Point) NAL unit types - keyframes/random access points +// BLA (Broken Link Access) pictures. +const H265NALU_BLA_W_LP: u8 = 16; +const H265NALU_BLA_W_RADL: u8 = 17; +const H265NALU_BLA_N_LP: u8 = 18; +// IDR (Instantaneous Decoding Refresh) pictures. +const H265NALU_IDR_W_RADL: u8 = 19; +const H265NALU_IDR_N_LP: u8 = 20; +// CRA (Clean Random Access) picture. +const H265NALU_CRA_NUT: u8 = 21; + +pub static ANNEXB_NALUSTART_CODE: &[u8] = &[0x00, 0x00, 0x00, 0x01]; + +/// Detect whether an H265 (HEVC) RTP payload contains a keyframe. +/// +/// Checks for IRAP (Intra Random Access Point) NAL units in the RTP payload. +/// IRAP types include BLA (16-18), IDR (19-20), and CRA (21). +/// +/// Handles single NAL units, aggregation packets (AP, type 48), +/// and fragmentation units (FU, type 49). +/// +/// For FU packets, only the start fragment (S=1) is detected as a +/// keyframe since the original NAL type is in the FU header. +pub fn detect_h265_keyframe(payload: &[u8]) -> bool { + if payload.len() < H265NALU_HEADER_SIZE { + return false; + } + + let header = H265NALUHeader::new(payload[0], payload[1]); + let nalu_type = header.nalu_type(); + + match nalu_type { + // Single NAL unit (types 0-47) + 0..=47 => header.is_irap(), + + // Aggregation packet: check all aggregated NALUs + H265NALU_AGGREGATION_PACKET_TYPE => { + let mut offset = H265NALU_HEADER_SIZE; + while offset + 2 <= payload.len() { + let nalu_size = ((payload[offset] as usize) << 8) | payload[offset + 1] as usize; + offset += 2; + if offset + nalu_size > payload.len() || nalu_size < H265NALU_HEADER_SIZE { + break; + } + let inner = H265NALUHeader::new(payload[offset], payload[offset + 1]); + if inner.is_irap() { + return true; + } + offset += nalu_size; + } + false + } + + // Fragmentation unit: check FU header for original NAL type + H265NALU_FRAGMENTATION_UNIT_TYPE => { + // FU header is byte 2 (after 2-byte NAL header) + if payload.len() < H265NALU_HEADER_SIZE + 1 { + return false; + } + let fu_header = payload[H265NALU_HEADER_SIZE]; + // S bit (start fragment) is bit 7 + if fu_header & 0x80 == 0 { + return false; + } + // FU type is lower 6 bits + let fu_type = fu_header & 0x3F; + matches!( + fu_type, + H265NALU_BLA_W_LP + | H265NALU_BLA_W_RADL + | H265NALU_BLA_N_LP + | H265NALU_IDR_W_RADL + | H265NALU_IDR_N_LP + | H265NALU_CRA_NUT + ) + } + + _ => false, + } +} + +/// Packetizes H265 (HEVC) RTP payloads. +/// +/// This implements the packetization rules from RFC 7798. +/// +/// Supported output payload types: +/// - Single NAL Unit packets (one NAL unit per RTP payload) +/// - Fragmentation Units (FU, type 49) for NAL units larger than the MTU +/// - Aggregation Packets (AP, type 48) for parameter sets (VPS/SPS/PPS) +/// +/// The packetizer caches VPS, SPS, and PPS NAL units and emits them together +/// in a single Aggregation Packet (AP) immediately before the next non-parameter-set +/// NAL unit, as recommended by RFC 7798. +/// +/// ## Input format +/// +/// The input `payload` may be either: +/// - A single NAL unit (starting with the 2-byte HEVC NAL unit header), OR +/// - An Annex-B bytestream containing one or more NAL units separated by start codes +/// (`0x00 00 01` or `0x00 00 00 01`). +/// +/// Start codes are stripped from the output RTP payloads. + +/// Safe maximum RTP payload for real-world WebRTC (avoids IP fragmentation). +const MAX_PACKET_SIZE: usize = 1200; + +/// Minimum FU payload size - at least 1 byte of actual NAL data required. +const MIN_FU_PAYLOAD: usize = 1; + +/// Minimum MTU for H.265 fragmentation. +/// Calculated as: NALU header (2 bytes) + FU header (1 byte) + minimum payload (1 byte) = 4 bytes total. +/// Any MTU smaller than this cannot accommodate even a single fragmentation unit. +const MIN_MTU: usize = H265NALU_HEADER_SIZE + H265FRAGMENTATION_UNIT_HEADER_SIZE + MIN_FU_PAYLOAD; + +#[derive(Debug, Clone)] +pub struct H265Packetizer { + // Parameter sets: heap-allocated but set once per stream (cold path). + vps_nalu: Option>, + sps_nalu: Option>, + pps_nalu: Option>, + // Reusable packet buffer - heap allocated once, reused for zero-allocation hot path. + // This is reused on every fragment, avoiding N allocations per frame. + // Using Vec instead of ArrayVec since MAX_PACKET_SIZE (1200) is too large for stack. + pkt_buf: Vec, + // DONL (Decoding Order Number) tracking. + // When enabled (Some), tracks the 16-bit DONL value to include in RTP packets. + // DONL is used when sprop-max-don-diff > 0 (RFC 7798 §7.1). + // https://datatracker.ietf.org/doc/html/rfc7798#section-7.1 + donl: Option, +} + +impl Default for H265Packetizer { + fn default() -> Self { + Self { + vps_nalu: None, + sps_nalu: None, + pps_nalu: None, + // Pre-allocate to avoid reallocations during hot path + pkt_buf: Vec::with_capacity(MAX_PACKET_SIZE), + donl: None, + } + } +} + +impl H265Packetizer { + /// with_donl enables or disables DONL (Decoding Order Number) fields in RTP packets. + /// DONL should be enabled when `sprop-max-don-diff` > 0 in the SDP (RFC 7798 §7.1). + /// When enabled, DONL fields are included in Single NAL, FU, and AP packets. + pub fn with_donl(&mut self, value: bool) { + self.donl = if value { Some(0) } else { None }; + } + + /// Increments the DONL counter with wrapping at 65536 (RFC 7798 §7.1). + fn increment_donl(&mut self) { + if let Some(ref mut donl) = self.donl { + *donl = donl.wrapping_add(1); + } + } + + /// Increments the DONL counter by `n` with wrapping at 65536 (RFC 7798 §7.1). + /// Used when emitting an AP containing multiple NAL units, each consuming one DON value. + fn increment_donl_by(&mut self, n: u16) { + if let Some(ref mut donl) = self.donl { + *donl = donl.wrapping_add(n); + } + } + + /// Builds an Aggregation Packet (AP) from multiple NAL units into a reusable buffer. + /// + /// # Arguments + /// * `template_nalu` - A NAL unit to copy F, layer_id, and tid from. + /// * `nal_units` - Slice of NAL units to aggregate. + /// * `donl` - Optional DONL value for the first aggregation unit. + /// * `buf` - Reusable buffer to write the AP packet into (will be cleared). + /// * `max_size` - Maximum buffer size (MTU constraint). + /// + /// # Returns + /// `true` if AP was built successfully, `false` if it exceeded max_size. + fn build_ap_packet( + template_nalu: &[u8], + nal_units: &[&[u8]], + donl: Option, + buf: &mut Vec, + max_size: usize, + ) -> bool { + buf.clear(); + + // Build AP header (PayloadHdr) by copying F, layer_id, tid from template + // but setting Type=48. + const TYPE_MASK: u16 = 0b0111111 << 9; + let orig_u16 = u16::from_be_bytes([template_nalu[0], template_nalu[1]]); + let ap_u16 = (orig_u16 & !TYPE_MASK) | ((H265NALU_AGGREGATION_PACKET_TYPE as u16) << 9); + let ap_hdr = ap_u16.to_be_bytes(); + + buf.extend_from_slice(&ap_hdr); + + // Write DONL for first aggregation unit if present (RFC 7798 §4.4.2) + if let Some(donl_value) = donl { + buf.extend_from_slice(&donl_value.to_be_bytes()); + } + + // Append each NAL unit with its 16-bit size prefix. + // For 2nd and subsequent units, write DOND (1 byte) when DONL is enabled. + for (i, nal_unit) in nal_units.iter().enumerate() { + // Write DOND (Decoding Order Number Difference) for 2nd+ units. + // DOND is always 0 in our case since we emit in order. + if donl.is_some() && i > 0 { + buf.push(0); + } + buf.extend_from_slice(&(nal_unit.len() as u16).to_be_bytes()); + buf.extend_from_slice(nal_unit); + + // Check if we exceeded max size + if buf.len() > max_size { + return false; + } + } + + true + } + + /// Builds a PACI Packet (Type 50) wrapping an inner NAL unit with optional PHES into a reusable buffer. + /// + /// PACI packets allow wrapping another NAL unit with additional header information, + /// useful for temporal scalability and other extensions. + /// + /// # Arguments + /// * `inner_nalu` - The NAL unit to wrap (complete with 2-byte header) + /// * `phes` - Optional Payload Header Extension Structure (PHES) bytes + /// * `buf` - Reusable buffer to write the PACI packet into (will be cleared) + /// + /// # Returns + /// `Ok(())` if packet was built successfully, `Err(PacketError)` if PHES is too large (>31 bytes) + /// + /// Reference: RFC 7798 §4.4.4 + fn build_paci_packet( + inner_nalu: &[u8], + phes: &[u8], + buf: &mut Vec, + ) -> Result<(), PacketError> { + if phes.len() > H265PACI_MAX_PHES_SIZE { + return Err(PacketError::ErrH265PACIPHESTooLong); + } + + buf.clear(); + + // Extract F, layer_id, tid from inner NALU header + let inner_header = H265NALUHeader::new(inner_nalu[0], inner_nalu[1]); + let inner_type = inner_header.nalu_type(); + + // Build PACI PayloadHdr (Type=50) + let paci_payload_header = H265NALUHeader::new_with_type( + H265NALU_PACI_PACKET_TYPE, + inner_header.layer_id(), + inner_header.tid(), + ); + let paci_hdr = paci_payload_header.0.to_be_bytes(); + buf.extend_from_slice(&paci_hdr); + + // Build PACI header fields: A | cType | phssize | F0 F1 F2 | Y + // A = F bit from inner NALU + let a = if inner_header.f() { 1u16 << 15 } else { 0 }; + // cType = Type field from inner NALU + let ctype = (inner_type as u16) << 9; + // phssize = size of PHES (0-31) + let phssize = (phes.len() as u16) << 4; + // F0 = 1 if PHES contains TSCI (phes.len() >= 3) + let f0 = if phes.len() >= 3 { 1u16 << 3 } else { 0 }; + // F1, F2, Y reserved (must be 0) + let paci_fields = a | ctype | phssize | f0; + buf.extend_from_slice(&paci_fields.to_be_bytes()); + + // Append PHES if present + if !phes.is_empty() { + buf.extend_from_slice(phes); + } + + // Append inner NAL unit payload (without its 2-byte header) + if inner_nalu.len() > H265NALU_HEADER_SIZE { + buf.extend_from_slice(&inner_nalu[H265NALU_HEADER_SIZE..]); + } + + Ok(()) + } + + /// Finds the next Annex-B NAL unit start code in `payload`, starting at `start`. + /// + /// Detects `0x00 00 01` or `0x00 00 00 01` as defined by the HEVC Annex-B + /// byte stream format (ITU-T Rec. H.265 | ISO/IEC 23008-2, Annex B), + /// and returns `(start_index, start_code_len)` (length = 3 or 4). + /// + /// Returns `(-1, -1)` if no start code is found. + fn next_start_code(payload: &[u8], start: usize) -> (isize, isize) { + let mut zero_count = 0; + + for (i, &b) in payload[start..].iter().enumerate() { + if b == 0 { + zero_count += 1; + continue; + } else if b == 1 && zero_count >= 2 { + return ((start + i - zero_count) as isize, (zero_count as isize) + 1); + } + + zero_count = 0; + } + + (-1, -1) + } + + /// Packetization overview (RFC 7798): + /// + /// Input NAL units (Annex-B or raw) + /// ├─ Single NALU → RTP payload = NAL header + payload + /// ├─ Large NALU → Fragmentation Units (Type 49) + /// └─ VPS/SPS/PPS → Aggregation Packet (Type 48) + fn emit_nalu(&mut self, nalu: &[u8], mtu: usize, out: &mut Vec>) { + if mtu == 0 || nalu.len() < H265NALU_HEADER_SIZE { + return; + } + + // Parse the HEVC NAL unit header. + let original_hdr = H265NALUHeader::new(nalu[0], nalu[1]); + let original_type = original_hdr.nalu_type(); + + // Ignore AUD/filler. + if original_type == H265NALU_AUD_NALU_TYPE || original_type == H265NALU_FILLER_NALU_TYPE { + return; + } + + // Cache parameter sets; send them before the next non-parameter-set NALU. + // Option> for one-time heap allocation (cold path - happens once per stream). + match original_type { + H265NALU_VPS_NALU_TYPE => { + self.vps_nalu = Some(nalu.to_vec()); + return; + } + H265NALU_SPS_NALU_TYPE => { + self.sps_nalu = Some(nalu.to_vec()); + return; + } + H265NALU_PPS_NALU_TYPE => { + self.pps_nalu = Some(nalu.to_vec()); + return; + } + _ => {} + } + + // If we have cached VPS/SPS/PPS, emit an Aggregation Packet (AP, Type=48) + // immediately before the next non-parameter-set NAL unit, per RFC 7798 §4.4.2. + if let (Some(sps_nalu), Some(pps_nalu)) = (&self.sps_nalu, &self.pps_nalu) { + // Stack-allocate array for NAL unit slices (no heap allocation) + let mut nal_units_arr: [&[u8]; 3] = [&[], &[], &[]]; + let mut count = 0; + + if let Some(vps_nalu) = &self.vps_nalu { + nal_units_arr[count] = vps_nalu; + count += 1; + } + nal_units_arr[count] = sps_nalu; + count += 1; + nal_units_arr[count] = pps_nalu; + count += 1; + + let nal_units = &nal_units_arr[..count]; + + // Build the AP packet using reusable buffer. + // Returns false if AP exceeds MTU. + let ap_built = + Self::build_ap_packet(nalu, nal_units, self.donl, &mut self.pkt_buf, mtu); + + if ap_built { + // AP fits in MTU, emit it. + out.push(self.pkt_buf.clone()); + // Increment DONL by the number of NAL units in the AP. + // Each NAL unit inside the AP consumes its own DON value: + // DONL = N (first), DOND derives N+1 (second), N+2 (third), etc. + // So after emitting an AP with `count` NAL units, the next packet's + // DON should be N + count. (RFC 7798 §4.4.2) + self.increment_donl_by(count as u16); + } else { + // AP exceeds MTU. Fall back to emitting parameter sets as individual + // Single NAL Unit packets (RFC 7798 §4.4.1). + let vps = self.vps_nalu.take(); + let sps = self.sps_nalu.take(); + let pps = self.pps_nalu.take(); + + for nal_unit in [&vps, &sps, &pps].into_iter().flatten() { + if nal_unit.len() <= mtu { + if let Some(ref mut donl_value) = self.donl { + // Single NAL with DONL per RFC 7798 §4.4.1: + // [PayloadHdr(2B)] [DONL(2B)] [NAL_payload_data] + self.pkt_buf.clear(); + self.pkt_buf + .extend_from_slice(&nal_unit[..H265NALU_HEADER_SIZE]); + self.pkt_buf.extend_from_slice(&donl_value.to_be_bytes()); + self.pkt_buf + .extend_from_slice(&nal_unit[H265NALU_HEADER_SIZE..]); + out.push(self.pkt_buf.clone()); + *donl_value = donl_value.wrapping_add(1); + } else { + out.push(nal_unit.clone()); + } + } + // If parameter set is larger than MTU, we could fragment it as FU, but in practice + // VPS/SPS/PPS are typically small. Silently dropping oversized parameter sets + // is acceptable as a fallback. + } + } + + // Clear cache after emitting parameter sets (either as AP or individual packets). + // For the AP path, these are still Some; for the fallback path, already taken above. + self.vps_nalu = None; + self.sps_nalu = None; + self.pps_nalu = None; + } + + // Single NAL Unit packetization (RFC 7798 §4.4.1). + // https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.1 + if nalu.len() <= mtu { + // Write DONL field if enabled (2 bytes after NAL header) + if let Some(donl_value) = self.donl { + self.pkt_buf.clear(); + // Write NAL header (2 bytes), DONL (2 bytes), then payload + self.pkt_buf + .extend_from_slice(&nalu[..H265NALU_HEADER_SIZE]); + self.pkt_buf.extend_from_slice(&donl_value.to_be_bytes()); + self.pkt_buf + .extend_from_slice(&nalu[H265NALU_HEADER_SIZE..]); + out.push(self.pkt_buf.clone()); + self.increment_donl(); + } else { + out.push(nalu.to_vec()); + } + return; + } + + // Fragmentation Unit (FU) packetization (RFC 7798 §4.4.3). + // https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + const FU_OVERHEAD: usize = H265NALU_HEADER_SIZE + H265FRAGMENTATION_UNIT_HEADER_SIZE; + if mtu <= FU_OVERHEAD || nalu.len() <= H265NALU_HEADER_SIZE { + return; + } + + // Build FU indicator (Type=49) from original NAL header + const TYPE_MASK: u16 = 0b0111111 << 9; // bits 14..9 + let orig = u16::from_be_bytes([nalu[0], nalu[1]]); + let fu = (orig & !TYPE_MASK) | ((H265NALU_FRAGMENTATION_UNIT_TYPE as u16) << 9); + let fu_indicator = fu.to_be_bytes(); + + let payload = &nalu[H265NALU_HEADER_SIZE..]; + + let donl_overhead = if self.donl.is_some() { 2 } else { 0 }; + let donl_bytes = self.donl.map(u16::to_be_bytes); + + // Clamp to buffer capacity to avoid overflow + let effective_mtu = mtu.min(MAX_PACKET_SIZE); + + // Must have room for FU headers (+ optional DONL) and at least 1 byte of payload + if effective_mtu <= FU_OVERHEAD + donl_overhead { + return; + } + + let first_max = effective_mtu - FU_OVERHEAD - donl_overhead; + let max_fragment = effective_mtu - FU_OVERHEAD; + + let mut offset = 0; + + while offset < payload.len() { + let first = offset == 0; + let remaining = payload.len() - offset; + + let budget = if first { first_max } else { max_fragment }; + let take = remaining.min(budget); + debug_assert!(take > 0); + + let end = offset + take == payload.len(); + let fu_hdr = H265FragmentationUnitHeader::new(first, end, original_type); + + self.pkt_buf.clear(); + self.pkt_buf.extend_from_slice(&fu_indicator); + self.pkt_buf.push(fu_hdr.0); + if first { + if let Some(ref b) = donl_bytes { + self.pkt_buf.extend_from_slice(b); + } + } + self.pkt_buf + .extend_from_slice(&payload[offset..offset + take]); + + out.push(self.pkt_buf.clone()); + offset += take; + } + + // One DONL per NAL unit (not per fragment) + self.increment_donl(); + } +} + +impl Packetizer for H265Packetizer { + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() { + return Ok(vec![]); + } + + // Validate and log MTU issues. + let mtu = match mtu { + 0 => { + warn!("MTU is 0, cannot packetize H.265 - this indicates a programming bug"); + return Ok(vec![]); + } + mtu if mtu > MAX_PACKET_SIZE => { + warn!( + "MTU {} exceeds MAX_PACKET_SIZE {}, clamping to {}", + mtu, MAX_PACKET_SIZE, MAX_PACKET_SIZE + ); + MAX_PACKET_SIZE + } + mtu if mtu < MIN_MTU => { + warn!( + "MTU {} too small for H.265 fragmentation (min {}) - cannot fragment", + mtu, MIN_MTU + ); + return Ok(vec![]); + } + mtu => mtu, // Valid MTU, use as-is + }; + + // Pre-allocate with estimated capacity to avoid reallocations. + // Estimate: payload_size / (mtu - overhead) + extra for parameter sets. + let estimated_packets = payload + .len() + .checked_div(mtu.saturating_sub(3)) + .unwrap_or(1) + .saturating_add(4); + let mut packets = Vec::with_capacity(estimated_packets); + + // If no Annex-B start codes are present, treat as a single NAL unit. + let (mut next_start, mut next_len) = Self::next_start_code(payload, 0); + if next_start == -1 { + self.emit_nalu(payload, mtu, &mut packets); + return Ok(packets); + } + + // Walk Annex-B bytestream and emit NAL units between start codes. + while next_start != -1 { + let nalu_start = (next_start + next_len) as usize; + let (next_start2, next_len2) = Self::next_start_code(payload, nalu_start); + next_start = next_start2; + next_len = next_len2; + + if next_start != -1 { + let nalu_end = next_start as usize; + self.emit_nalu(&payload[nalu_start..nalu_end], mtu, &mut packets); + } else { + self.emit_nalu(&payload[nalu_start..], mtu, &mut packets); + } + } + + Ok(packets) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +/// H265NALUHeader is a H265 NAL Unit Header +/// https://datatracker.ietf.org/doc/html/rfc7798#section-1.1.4 +/// +/// ```text +/// +---------------+---------------+ +/// |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// |F| Type | layer_id | tid | +/// +-------------+-----------------+ +/// ``` +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub struct H265NALUHeader(pub u16); + +impl H265NALUHeader { + fn new(high_byte: u8, low_byte: u8) -> Self { + H265NALUHeader(((high_byte as u16) << 8) | low_byte as u16) + } + + /// f is the forbidden bit, should always be 0. + pub fn f(&self) -> bool { + (self.0 >> 15) != 0 + } + + /// nalu_type of NAL Unit. + pub fn nalu_type(&self) -> u8 { + // 01111110 00000000 + const MASK: u16 = 0b01111110 << 8; + ((self.0 & MASK) >> (8 + 1)) as u8 + } + + /// is_type_vcl_unit returns whether or not the NAL Unit type is a VCL NAL unit. + pub fn is_type_vcl_unit(&self) -> bool { + // Type is coded on 6 bits + const MSB_MASK: u8 = 0b00100000; + (self.nalu_type() & MSB_MASK) == 0 + } + + /// layer_id should always be 0 in non-3D HEVC context. + pub fn layer_id(&self) -> u8 { + // 00000001 11111000 + const MASK: u16 = (0b00000001 << 8) | 0b11111000; + ((self.0 & MASK) >> 3) as u8 + } + + /// tid is the temporal identifier of the NAL unit +1. + pub fn tid(&self) -> u8 { + const MASK: u16 = 0b00000111; + (self.0 & MASK) as u8 + } + + /// Creates a new H265NALUHeader with specified type, layer_id, and tid. + /// Used for building PACI packet headers. + pub fn new_with_type(nalu_type: u8, layer_id: u8, tid: u8) -> Self { + let header = ((nalu_type as u16) << 9) | ((layer_id as u16) << 3) | (tid as u16); + H265NALUHeader(header) + } + + /// is_aggregation_packet returns whether or not the packet is an Aggregation packet. + pub fn is_aggregation_packet(&self) -> bool { + self.nalu_type() == H265NALU_AGGREGATION_PACKET_TYPE + } + + /// is_fragmentation_unit returns whether or not the packet is a Fragmentation Unit packet. + pub fn is_fragmentation_unit(&self) -> bool { + self.nalu_type() == H265NALU_FRAGMENTATION_UNIT_TYPE + } + + /// is_paci_packet returns whether or not the packet is a PACI packet. + pub fn is_paci_packet(&self) -> bool { + self.nalu_type() == H265NALU_PACI_PACKET_TYPE + } + + /// is_idr_picture returns whether or not the NAL unit is an IDR picture. + pub fn is_idr_picture(&self) -> bool { + let typ = self.nalu_type(); + typ == H265NALU_IDR_W_RADL || typ == H265NALU_IDR_N_LP + } + + /// is_irap returns whether or not the NAL unit is an IRAP (Intra Random Access Point) picture. + /// IRAP pictures include BLA, IDR, and CRA pictures, which are all random access points / keyframes. + pub fn is_irap(&self) -> bool { + let typ = self.nalu_type(); + matches!( + typ, + H265NALU_BLA_W_LP + | H265NALU_BLA_W_RADL + | H265NALU_BLA_N_LP + | H265NALU_IDR_W_RADL + | H265NALU_IDR_N_LP + | H265NALU_CRA_NUT + ) + } +} + +/// +/// Single NAL Unit Packet implementation +/// +/// H265SingleNALUnitPacket represents a NALU packet, containing exactly one NAL unit. +/// +/// ```text +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | PayloadHdr | DONL (conditional) | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | | +/// | NAL unit payload data | +/// | | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | :...OPTIONAL RTP padding | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// ``` +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.1 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265SingleNALUnitPacket { + /// payload_header is the header of the H265 packet. + payload_header: H265NALUHeader, + /// donl is a 16-bit field, that may or may not be present. + donl: Option, + /// payload of the fragmentation unit. + payload: Vec, + + might_need_donl: bool, +} + +impl H265SingleNALUnitPacket { + /// with_donl can be called to specify whether or not DONL might be parsed. + /// DONL may need to be parsed if `sprop-max-don-diff` is greater than 0 on the RTP stream. + pub fn with_donl(&mut self, value: bool) { + self.might_need_donl = value; + } + + /// depacketize parses the passed byte slice and stores the result in the + /// H265SingleNALUnitPacket this method is called upon. + fn depacketize(&mut self, payload: &[u8]) -> Result<(), PacketError> { + if payload.len() <= H265NALU_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let payload_header = H265NALUHeader::new(payload[0], payload[1]); + if payload_header.f() { + return Err(PacketError::ErrH265CorruptedPacket); + } + if payload_header.is_fragmentation_unit() + || payload_header.is_paci_packet() + || payload_header.is_aggregation_packet() + { + return Err(PacketError::ErrInvalidH265PacketType); + } + + let mut payload = &payload[2..]; + + if self.might_need_donl { + // sizeof(uint16) + if payload.len() <= 2 { + return Err(PacketError::ErrShortPacket); + } + + let donl = ((payload[0] as u16) << 8) | (payload[1] as u16); + self.donl = Some(donl); + payload = &payload[2..]; + } + + self.payload_header = payload_header; + self.payload = payload.to_vec(); + + Ok(()) + } + + /// payload_header returns the NALU header of the packet. + pub fn payload_header(&self) -> H265NALUHeader { + self.payload_header + } + + /// donl returns the DONL of the packet. + pub fn donl(&self) -> Option { + self.donl + } + + /// payload returns the Fragmentation Unit packet payload. + pub fn payload(&self) -> &[u8] { + &self.payload + } +} + +/// +/// Aggregation Packets implementation +/// +/// H265AggregationUnitFirst represent the First Aggregation Unit in an AP. +/// +/// ```text +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// : DONL (conditional) | NALU size | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | NALU size | | +/// +-+-+-+-+-+-+-+-+ NAL unit | +/// | | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | : +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// ``` +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265AggregationUnitFirst { + donl: Option, + nal_unit_size: u16, + nal_unit: Vec, +} + +impl H265AggregationUnitFirst { + /// donl field, when present, specifies the value of the 16 least + /// significant bits of the decoding order number of the aggregated NAL + /// unit. + pub fn donl(&self) -> Option { + self.donl + } + + /// nalu_size represents the size, in bytes, of the nal_unit. + pub fn nalu_size(&self) -> u16 { + self.nal_unit_size + } + + /// nal_unit payload. + pub fn nal_unit(&self) -> &[u8] { + &self.nal_unit + } +} + +/// H265AggregationUnit represent the an Aggregation Unit in an AP, which is not the first one. +/// +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// : DOND (cond) | NALU size | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | | +/// | NAL unit | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | : +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265AggregationUnit { + dond: Option, + nal_unit_size: u16, + nal_unit: Vec, +} + +impl H265AggregationUnit { + /// dond field plus 1 specifies the difference between + /// the decoding order number values of the current aggregated NAL unit + /// and the preceding aggregated NAL unit in the same AP. + pub fn dond(&self) -> Option { + self.dond + } + + /// nalu_size represents the size, in bytes, of the nal_unit. + pub fn nalu_size(&self) -> u16 { + self.nal_unit_size + } + + /// nal_unit payload. + pub fn nal_unit(&self) -> &[u8] { + &self.nal_unit + } +} + +/// H265AggregationPacket represents an Aggregation packet. +/// +/// ```text +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | PayloadHdr (Type=48) | | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +/// | | +/// | two or more aggregation units | +/// | | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | :...OPTIONAL RTP padding | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// ``` +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265AggregationPacket { + first_unit: Option, + other_units: Vec, + + might_need_donl: bool, +} + +impl H265AggregationPacket { + /// with_donl can be called to specify whether or not DONL might be parsed. + /// DONL may need to be parsed if `sprop-max-don-diff` is greater than 0 on the RTP stream. + pub fn with_donl(&mut self, value: bool) { + self.might_need_donl = value; + } + + /// depacketize parses the passed byte slice and stores the result in the + /// H265AggregationPacket this method is called upon. + fn depacketize(&mut self, payload: &[u8]) -> Result<(), PacketError> { + if payload.len() <= H265NALU_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let payload_header = H265NALUHeader::new(payload[0], payload[1]); + if payload_header.f() { + return Err(PacketError::ErrH265CorruptedPacket); + } + if !payload_header.is_aggregation_packet() { + return Err(PacketError::ErrInvalidH265PacketType); + } + + // First parse the first aggregation unit + let mut payload = &payload[2..]; + let mut first_unit = H265AggregationUnitFirst::default(); + + if self.might_need_donl { + if payload.len() < 2 { + return Err(PacketError::ErrShortPacket); + } + + let donl = ((payload[0] as u16) << 8) | (payload[1] as u16); + first_unit.donl = Some(donl); + + payload = &payload[2..]; + } + if payload.len() < 2 { + return Err(PacketError::ErrShortPacket); + } + first_unit.nal_unit_size = ((payload[0] as u16) << 8) | (payload[1] as u16); + payload = &payload[2..]; + + if payload.len() < first_unit.nal_unit_size as usize { + return Err(PacketError::ErrShortPacket); + } + + first_unit.nal_unit = payload[..first_unit.nal_unit_size as usize].to_vec(); + payload = &payload[first_unit.nal_unit_size as usize..]; + + // Parse remaining Aggregation Units + let mut units = vec![]; //H265AggregationUnit + loop { + let mut unit = H265AggregationUnit::default(); + + if self.might_need_donl { + if payload.is_empty() { + break; + } + + let dond = payload[0]; + unit.dond = Some(dond); + + payload = &payload[1..]; + } + + if payload.len() < 2 { + break; + } + unit.nal_unit_size = ((payload[0] as u16) << 8) | (payload[1] as u16); + payload = &payload[2..]; + + if payload.len() < unit.nal_unit_size as usize { + break; + } + + unit.nal_unit = payload[..unit.nal_unit_size as usize].to_vec(); + payload = &payload[unit.nal_unit_size as usize..]; + + units.push(unit); + } + + // There need to be **at least** two Aggregation Units (first + another one) + if units.is_empty() { + return Err(PacketError::ErrShortPacket); + } + + self.first_unit = Some(first_unit); + self.other_units = units; + + Ok(()) + } + + /// first_unit returns the first Aggregated Unit of the packet. + pub fn first_unit(&self) -> Option<&H265AggregationUnitFirst> { + self.first_unit.as_ref() + } + + /// other_units returns the all the other Aggregated Unit of the packet (excluding the first one). + pub fn other_units(&self) -> &[H265AggregationUnit] { + self.other_units.as_slice() + } + + /// nal_units returns all NAL units in the aggregation packet. + pub fn nal_units(&self) -> Vec<&[u8]> { + let mut units = Vec::new(); + if let Some(first) = &self.first_unit { + units.push(first.nal_unit.as_slice()); + } + for unit in &self.other_units { + units.push(unit.nal_unit.as_slice()); + } + units + } +} + +/// +/// Fragmentation Unit implementation +/// + +const H265FRAGMENTATION_UNIT_HEADER_SIZE: usize = 1; + +/// H265FragmentationUnitHeader is a H265 FU Header +/// +/// ```text +/// +---------------+ +/// |0|1|2|3|4|5|6|7| +/// +-+-+-+-+-+-+-+-+ +/// |S|E| fu_type | +/// +---------------+ +/// ``` +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub struct H265FragmentationUnitHeader(pub u8); + +impl H265FragmentationUnitHeader { + /// new creates a new H265FragmentationUnitHeader. + /// + /// # Arguments + /// * `s` - Start bit: true if this is the first fragment + /// * `e` - End bit: true if this is the last fragment + /// * `fu_type` - The NAL unit type of the fragmented NAL unit (6 bits) + pub fn new(s: bool, e: bool, fu_type: u8) -> Self { + let mut header = fu_type & 0b0011_1111; // Mask to 6 bits + if s { + header |= 0b1000_0000; // Set S bit + } + if e { + header |= 0b0100_0000; // Set E bit + } + H265FragmentationUnitHeader(header) + } + + /// s represents the start of a fragmented NAL unit. + pub fn s(&self) -> bool { + const MASK: u8 = 0b10000000; + ((self.0 & MASK) >> 7) != 0 + } + + /// e represents the end of a fragmented NAL unit. + pub fn e(&self) -> bool { + const MASK: u8 = 0b01000000; + ((self.0 & MASK) >> 6) != 0 + } + + /// fu_type MUST be equal to the field Type of the fragmented NAL unit. + pub fn fu_type(&self) -> u8 { + const MASK: u8 = 0b00111111; + self.0 & MASK + } +} + +/// H265FragmentationUnitPacket represents a single Fragmentation Unit packet. +/// +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | PayloadHdr (Type=49) | FU header | DONL (cond) | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| +/// | DONL (cond) | | +/// |-+-+-+-+-+-+-+-+ | +/// | FU payload | +/// | | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | :...OPTIONAL RTP padding | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265FragmentationUnitPacket { + /// payload_header is the header of the H265 packet. + payload_header: H265NALUHeader, + /// fu_header is the header of the fragmentation unit + fu_header: H265FragmentationUnitHeader, + /// donl is a 16-bit field, that may or may not be present. + donl: Option, + /// payload of the fragmentation unit. + payload: Vec, + + might_need_donl: bool, +} + +impl H265FragmentationUnitPacket { + /// with_donl can be called to specify whether or not DONL might be parsed. + /// DONL may need to be parsed if `sprop-max-don-diff` is greater than 0 on the RTP stream. + pub fn with_donl(&mut self, value: bool) { + self.might_need_donl = value; + } + + /// depacketize parses the passed byte slice and stores the result in the + /// H265FragmentationUnitPacket this method is called upon. + fn depacketize(&mut self, payload: &[u8]) -> Result<(), PacketError> { + const TOTAL_HEADER_SIZE: usize = H265NALU_HEADER_SIZE + H265FRAGMENTATION_UNIT_HEADER_SIZE; + if payload.len() <= TOTAL_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let payload_header = H265NALUHeader::new(payload[0], payload[1]); + if payload_header.f() { + return Err(PacketError::ErrH265CorruptedPacket); + } + if !payload_header.is_fragmentation_unit() { + return Err(PacketError::ErrInvalidH265PacketType); + } + + let fu_header = H265FragmentationUnitHeader(payload[2]); + let mut payload = &payload[3..]; + + if fu_header.s() && self.might_need_donl { + if payload.len() <= 2 { + return Err(PacketError::ErrShortPacket); + } + + let donl = ((payload[0] as u16) << 8) | (payload[1] as u16); + self.donl = Some(donl); + payload = &payload[2..]; + } + + self.payload_header = payload_header; + self.fu_header = fu_header; + self.payload = payload.to_vec(); + + Ok(()) + } + + /// payload_header returns the NALU header of the packet. + pub fn payload_header(&self) -> H265NALUHeader { + self.payload_header + } + + /// fu_header returns the Fragmentation Unit Header of the packet. + pub fn fu_header(&self) -> H265FragmentationUnitHeader { + self.fu_header + } + + /// donl returns the DONL of the packet. + pub fn donl(&self) -> Option { + self.donl + } + + /// payload returns the Fragmentation Unit packet payload. + pub fn payload(&self) -> &[u8] { + &self.payload + } +} + +/// +/// PACI implementation +/// + +/// H265PACIPacket represents a single H265 PACI packet. +/// +/// ```text +/// 0 1 2 3 +/// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | PayloadHdr (Type=50) |A| cType | phssize |F0..2|Y| +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | payload Header Extension Structure (phes) | +/// |=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=| +/// | | +/// | PACI payload: NAL unit | +/// | . . . | +/// | | +/// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// | :...OPTIONAL RTP padding | +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// ``` +/// +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.4 +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265PACIPacket { + /// payload_header is the header of the H265 packet. + payload_header: H265NALUHeader, + + /// Field which holds value for `A`, `cType`, `phssize`, `F0`, `F1`, `F2` and `Y` fields. + paci_header_fields: u16, + + /// phes is a header extension, of byte length `phssize` + phes: Vec, + + /// payload contains NAL units & optional padding + payload: Vec, +} + +impl H265PACIPacket { + /// payload_header returns the NAL Unit Header. + pub fn payload_header(&self) -> H265NALUHeader { + self.payload_header + } + + /// a copies the F bit of the PACI payload NALU. + pub fn a(&self) -> bool { + const MASK: u16 = 0b10000000 << 8; + (self.paci_header_fields & MASK) != 0 + } + + /// ctype copies the Type field of the PACI payload NALU. + pub fn ctype(&self) -> u8 { + const MASK: u16 = 0b01111110 << 8; + ((self.paci_header_fields & MASK) >> (8 + 1)) as u8 + } + + /// phs_size indicates the size of the phes field. + pub fn phs_size(&self) -> u8 { + const MASK: u16 = (0b00000001 << 8) | 0b11110000; + ((self.paci_header_fields & MASK) >> 4) as u8 + } + + /// f0 indicates the presence of a Temporal Scalability support extension in the phes. + pub fn f0(&self) -> bool { + const MASK: u16 = 0b00001000; + (self.paci_header_fields & MASK) != 0 + } + + /// f1 must be zero, reserved for future extensions. + pub fn f1(&self) -> bool { + const MASK: u16 = 0b00000100; + (self.paci_header_fields & MASK) != 0 + } + + /// f2 must be zero, reserved for future extensions. + pub fn f2(&self) -> bool { + const MASK: u16 = 0b00000010; + (self.paci_header_fields & MASK) != 0 + } + + /// y must be zero, reserved for future extensions. + pub fn y(&self) -> bool { + const MASK: u16 = 0b00000001; + (self.paci_header_fields & MASK) != 0 + } + + /// phes contains header extensions. Its size is indicated by phssize. + pub fn phes(&self) -> &[u8] { + &self.phes + } + + /// payload is a single NALU or NALU-like struct, not including the first two octets (header). + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// tsci returns the Temporal Scalability Control Information extension, if present. + pub fn tsci(&self) -> Option { + if !self.f0() || self.phs_size() < 3 { + return None; + } + + Some(H265TSCI( + ((self.phes[0] as u32) << 16) | ((self.phes[1] as u32) << 8) | self.phes[0] as u32, + )) + } + + /// depacketize parses the passed byte slice and stores the result in the + /// H265PACIPacket this method is called upon. + fn depacketize(&mut self, payload: &[u8]) -> Result<(), PacketError> { + const TOTAL_HEADER_SIZE: usize = H265NALU_HEADER_SIZE + 2; + if payload.len() <= TOTAL_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let payload_header = H265NALUHeader::new(payload[0], payload[1]); + if payload_header.f() { + return Err(PacketError::ErrH265CorruptedPacket); + } + if !payload_header.is_paci_packet() { + return Err(PacketError::ErrInvalidH265PacketType); + } + + let paci_header_fields = ((payload[2] as u16) << 8) | (payload[3] as u16); + let mut payload = &payload[4..]; + + self.paci_header_fields = paci_header_fields; + let header_extension_size = self.phs_size(); + + if payload.len() < header_extension_size as usize + 1 { + self.paci_header_fields = 0; + return Err(PacketError::ErrShortPacket); + } + + self.payload_header = payload_header; + + if header_extension_size > 0 { + self.phes = payload[..header_extension_size as usize].to_vec(); + } + + payload = &payload[header_extension_size as usize..]; + self.payload = payload.to_vec(); + + Ok(()) + } + + /// Packetizes a PACI packet by wrapping an inner NAL unit with optional PHES. + /// + /// Uses the packetizer's reusable buffer for zero-allocation hot path. + /// + /// # Arguments + /// * `inner_nalu` - The NAL unit to wrap (complete with 2-byte header) + /// * `phes` - Optional Payload Header Extension Structure (PHES) bytes + /// * `buf` - Reusable buffer to write the packet (will be cleared) + /// + /// # Returns + /// Serialized PACI packet bytes, or error if PHES is too large + pub fn packetize( + inner_nalu: &[u8], + phes: &[u8], + buf: &mut Vec, + ) -> Result, PacketError> { + H265Packetizer::build_paci_packet(inner_nalu, phes, buf)?; + Ok(buf.clone()) + } +} + +/// +/// Temporal Scalability Control Information +/// + +/// H265TSCI is a Temporal Scalability Control Information header extension. +/// Reference: https://datatracker.ietf.org/doc/html/rfc7798#section-4.5 +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub struct H265TSCI(pub u32); + +impl H265TSCI { + /// tl0picidx see RFC7798 for more details. + pub fn tl0picidx(&self) -> u8 { + const M1: u32 = 0xFFFF0000; + const M2: u32 = 0xFF00; + ((((self.0 & M1) >> 16) & M2) >> 8) as u8 + } + + /// irap_pic_id see RFC7798 for more details. + pub fn irap_pic_id(&self) -> u8 { + const M1: u32 = 0xFFFF0000; + const M2: u32 = 0x00FF; + (((self.0 & M1) >> 16) & M2) as u8 + } + + /// s see RFC7798 for more details. + pub fn s(&self) -> bool { + const M1: u32 = 0xFF00; + const M2: u32 = 0b10000000; + (((self.0 & M1) >> 8) & M2) != 0 + } + + /// e see RFC7798 for more details. + pub fn e(&self) -> bool { + const M1: u32 = 0xFF00; + const M2: u32 = 0b01000000; + (((self.0 & M1) >> 8) & M2) != 0 + } + + /// res see RFC7798 for more details. + pub fn res(&self) -> u8 { + const M1: u32 = 0xFF00; + const M2: u32 = 0b00111111; + (((self.0 & M1) >> 8) & M2) as u8 + } +} + +/// +/// H265 Payload Enum +/// +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum H265Payload { + H265SingleNALUnitPacket(H265SingleNALUnitPacket), + H265FragmentationUnitPacket(H265FragmentationUnitPacket), + H265AggregationPacket(H265AggregationPacket), + H265PACIPacket(H265PACIPacket), +} + +impl Default for H265Payload { + fn default() -> Self { + H265Payload::H265SingleNALUnitPacket(H265SingleNALUnitPacket::default()) + } +} + +/// Depacketizes H265 RTP packets. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct H265Depacketizer { + payload: H265Payload, + might_need_donl: bool, + fu_buffer: Option>, +} + +impl H265Depacketizer { + /// with_donl can be called to specify whether or not DONL might be parsed. + /// DONL may need to be parsed if `sprop-max-don-diff` is greater than 0 on the RTP stream. + pub fn with_donl(&mut self, value: bool) { + self.might_need_donl = value; + } + + /// payload returns the populated payload. + /// Must be casted to one of: + /// - H265SingleNALUnitPacket + /// - H265FragmentationUnitPacket + /// - H265AggregationPacket + /// - H265PACIPacket + pub fn payload(&self) -> &H265Payload { + &self.payload + } +} + +impl Depacketizer for H265Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + // Roughly account for Annex B start codes + let estimated_packets = (packets_size / 1200).saturating_add(1); + Some(packets_size.saturating_add(4usize.saturating_mul(estimated_packets))) + } + + /// depacketize parses the passed byte slice and stores the result + /// in the H265Packet this method is called upon + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + codec_extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + if packet.len() <= H265NALU_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let header = H265NALUHeader::new(packet[0], packet[1]); + if header.f() { + return Err(PacketError::ErrH265CorruptedPacket); + } + + if header.is_paci_packet() { + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(packet)?; + + // Emit PACI payload with Annex-B start code + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + out.extend_from_slice(&decoded.payload()); + + // Check if this is a keyframe. + if decoded.payload().len() >= H265NALU_HEADER_SIZE { + let payload_hdr = H265NALUHeader::new(decoded.payload()[0], decoded.payload()[1]); + let is_keyframe = if let CodecExtra::H265(e) = codec_extra { + payload_hdr.is_irap() | e.is_keyframe + } else { + payload_hdr.is_irap() + }; + *codec_extra = CodecExtra::H265(H265CodecExtra { is_keyframe }); + } + + self.payload = H265Payload::H265PACIPacket(decoded); + } else if header.is_fragmentation_unit() { + let mut decoded = H265FragmentationUnitPacket::default(); + decoded.with_donl(self.might_need_donl); + + decoded.depacketize(packet)?; + + let fu_header = decoded.fu_header(); + + if fu_header.s() { + // Start of fragmented NAL unit. + // Reuse existing buffer to avoid allocation on every FU start. + match &mut self.fu_buffer { + Some(buf) => buf.clear(), + None => { + // First FU ever - allocate with typical max NAL size. + // 128KB covers most 4K frames. + self.fu_buffer = Some(Vec::with_capacity(128 * 1024)); + } + } + } + + if let Some(ref mut buf) = self.fu_buffer { + buf.extend_from_slice(&decoded.payload()); + } + + if fu_header.e() { + // End of fragmented NAL unit - reconstruct original NAL. + // Borrow buffer instead of take() to preserve allocation for reuse. + if let Some(ref fu_payload) = self.fu_buffer { + // Rebuild original NAL unit header from FU header. + const TYPE_MASK: u16 = 0b0111111 << 9; // bits 14..9 + let payload_hdr_u16 = u16::from_be_bytes([packet[0], packet[1]]); + let orig_type = fu_header.fu_type(); + let orig_hdr_u16 = (payload_hdr_u16 & !TYPE_MASK) | ((orig_type as u16) << 9); + let orig_hdr = orig_hdr_u16.to_be_bytes(); + let orig_hdr_obj = H265NALUHeader::new(orig_hdr[0], orig_hdr[1]); + + // Check if this is a keyframe. + let is_keyframe = if let CodecExtra::H265(e) = codec_extra { + orig_hdr_obj.is_irap() | e.is_keyframe + } else { + orig_hdr_obj.is_irap() + }; + *codec_extra = CodecExtra::H265(H265CodecExtra { is_keyframe }); + + // Emit Annex-B start code + original NAL header + payload. + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + out.extend_from_slice(&orig_hdr); + out.extend_from_slice(fu_payload); + } + // Note: We don't clear fu_buffer here - it will be cleared on next FU start. + // This preserves the allocation for reuse. + } + + self.payload = H265Payload::H265FragmentationUnitPacket(decoded); + } else if header.is_aggregation_packet() { + // Optimized AP parsing: write directly to output like H264 does. + // Parse inline without intermediate Vec allocations. + + let mut offset = H265NALU_HEADER_SIZE; + let mut is_first_unit = true; + let mut unit_count = 0; + + // Parse and emit NAL units in one pass (zero-copy approach) + while offset < packet.len() { + // Skip DONL/DOND if present + if self.might_need_donl { + if is_first_unit { + // First unit has DONL (2 bytes) + if offset + 2 > packet.len() { + break; + } + offset += 2; + } else { + // Subsequent units have DOND (1 byte) + if offset + 1 > packet.len() { + break; + } + offset += 1; + } + } + + // Read NAL unit size (2 bytes) + if offset + 2 > packet.len() { + break; + } + let nalu_size = ((packet[offset] as usize) << 8) | (packet[offset + 1] as usize); + offset += 2; + + // Validate NAL unit fits in packet + if offset + nalu_size > packet.len() { + break; + } + + let nalu = &packet[offset..offset + nalu_size]; + offset += nalu_size; + unit_count += 1; + + // Check if keyframe + if nalu.len() >= H265NALU_HEADER_SIZE { + let nalu_hdr = H265NALUHeader::new(nalu[0], nalu[1]); + let is_keyframe = if let CodecExtra::H265(e) = codec_extra { + nalu_hdr.is_irap() | e.is_keyframe + } else { + nalu_hdr.is_irap() + }; + *codec_extra = CodecExtra::H265(H265CodecExtra { is_keyframe }); + } + + // Write to output (zero allocation) + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + out.extend_from_slice(nalu); + + is_first_unit = false; + } + + // AP must have at least 2 units (RFC 7798) + if unit_count < 2 { + return Err(PacketError::ErrShortPacket); + } + + // Still parse into struct for payload() API compatibility. + // This allocates but is needed for the public API. + let mut decoded = H265AggregationPacket::default(); + decoded.with_donl(self.might_need_donl); + decoded.depacketize(packet)?; // Validate structure + + self.payload = H265Payload::H265AggregationPacket(decoded); + } else { + // Single NAL unit packet. + let mut decoded = H265SingleNALUnitPacket::default(); + decoded.with_donl(self.might_need_donl); + + decoded.depacketize(packet)?; + + // Check if this is a keyframe. + let is_keyframe = if let CodecExtra::H265(e) = codec_extra { + header.is_irap() | e.is_keyframe + } else { + header.is_irap() + }; + *codec_extra = CodecExtra::H265(H265CodecExtra { is_keyframe }); + + // Emit Annex-B start code + NAL header + payload (without DONL). + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + let hdr = decoded.payload_header(); + out.extend_from_slice(&[(hdr.0 >> 8) as u8, (hdr.0 & 0xFF) as u8]); + out.extend_from_slice(&decoded.payload()); + + self.payload = H265Payload::H265SingleNALUnitPacket(decoded); + } + + Ok(()) + } + + /// is_partition_head checks if this is the head of a packetized nalu stream. + fn is_partition_head(&self, payload: &[u8]) -> bool { + if payload.len() < H265NALU_HEADER_SIZE { + return false; + } + + let header = H265NALUHeader::new(payload[0], payload[1]); + + // If F bit is set, this is always a partition head (error case, but treated as head) + if header.f() { + return true; + } + + // Single NAL unit packets are always partition heads + if !header.is_fragmentation_unit() + && !header.is_aggregation_packet() + && !header.is_paci_packet() + { + return true; + } + + // Aggregation packets are partition heads + if header.is_aggregation_packet() { + return true; + } + + // PACI packets are partition heads + if header.is_paci_packet() { + return true; + } + + // For FU packets, only those with S (start) flag are partition heads + if header.is_fragmentation_unit() { + if payload.len() < H265NALU_HEADER_SIZE + 1 { + return false; + } + let fu_header = H265FragmentationUnitHeader(payload[2]); + return fu_header.s(); + } + + false + } + + fn is_partition_tail(&self, marker: bool, payload: &[u8]) -> bool { + if payload.len() < H265NALU_HEADER_SIZE { + return false; + } + + let header = H265NALUHeader::new(payload[0], payload[1]); + + // For FU packets, check E (end) flag + if header.is_fragmentation_unit() { + if payload.len() < H265NALU_HEADER_SIZE + 1 { + return false; + } + let fu_header = H265FragmentationUnitHeader(payload[2]); + return fu_header.e(); + } + + // For all other packet types, rely on RTP marker bit + marker + } +} + +#[cfg(test)] +mod test { + use super::*; + + type Result = std::result::Result; + + // ========== Shared Test Utilities ========== + + fn reconstruct_from_fu_packets(packets: &[Vec]) -> Vec { + // Reconstruct the original NAL unit from a sequence of FU packets. + // Assumes all packets are FU (type 49) and are consecutive. + // + // FU payloads are: + // [0..2) : PayloadHdr (type=49) + // [2] : FU header (S/E/type) + // [3..] : fragment bytes + const TYPE_MASK: u16 = 0b0111111 << 9; // bits 14..9 + + let mut out = Vec::new(); + let mut started = false; + + for pkt in packets { + assert!(pkt.len() >= 3); + let hdr = H265NALUHeader::new(pkt[0], pkt[1]); + assert!(hdr.is_fragmentation_unit()); + + let fu = H265FragmentationUnitHeader(pkt[2]); + + if fu.s() { + // Rebuild the original 2-byte NAL header by replacing Type with fu_type. + let fu_u16 = u16::from_be_bytes([pkt[0], pkt[1]]); + let orig_u16 = (fu_u16 & !TYPE_MASK) | ((fu.fu_type() as u16) << 9); + out.extend_from_slice(&orig_u16.to_be_bytes()); + started = true; + } + + assert!(started, "FU sequence must start with S=1"); + out.extend_from_slice(&pkt[3..]); + } + + out + } + + /// RFC 7798 bitfield correctness tests. + /// These are "unit tests" for header parsing and classification. + mod header_tests { + use super::*; + + /// Test H.265 NAL Unit header parsing and field extraction. + /// Verifies F bit, NAL type, layer_id, tid, and packet type detection (AP/FU/PACI). + #[test] + fn test_h265_nalu_header() -> Result<()> { + #[derive(Default)] + struct TestType { + raw_header: &'static [u8], + + fbit: bool, + typ: u8, + layer_id: u8, + tid: u8, + + is_ap: bool, + is_fu: bool, + is_paci: bool, + } + + let tests = vec![ + // fbit + TestType { + raw_header: &[0x80, 0x00], + typ: 0, + layer_id: 0, + tid: 0, + fbit: true, + ..Default::default() + }, + // VPS_NUT + TestType { + raw_header: &[0x40, 0x01], + typ: 32, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // SPS_NUT + TestType { + raw_header: &[0x42, 0x01], + typ: 33, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // PPS_NUT + TestType { + raw_header: &[0x44, 0x01], + typ: 34, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // PREFIX_SEI_NUT + TestType { + raw_header: &[0x4e, 0x01], + typ: 39, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // Fragmentation Unit + TestType { + raw_header: &[0x62, 0x01], + typ: H265NALU_FRAGMENTATION_UNIT_TYPE, + layer_id: 0, + tid: 1, + is_fu: true, + ..Default::default() + }, + // Aggregation Packet (Type 48) + TestType { + raw_header: &[0x60, 0x01], + typ: H265NALU_AGGREGATION_PACKET_TYPE, + layer_id: 0, + tid: 1, + is_ap: true, + ..Default::default() + }, + // PACI Packet (Type 50) + TestType { + raw_header: &[0x64, 0x01], + typ: H265NALU_PACI_PACKET_TYPE, + layer_id: 0, + tid: 1, + is_paci: true, + ..Default::default() + }, + ]; + + for cur in tests { + let header = H265NALUHeader::new(cur.raw_header[0], cur.raw_header[1]); + + assert_eq!(header.f(), cur.fbit, "invalid F bit"); + assert_eq!(header.nalu_type(), cur.typ, "invalid type"); + + // For any type < 32, NAL is a VLC NAL unit. + assert_eq!( + header.is_type_vcl_unit(), + (header.nalu_type() < 32), + "invalid IsTypeVCLUnit" + ); + assert_eq!( + header.is_aggregation_packet(), + cur.is_ap, + "invalid type (aggregation packet)" + ); + assert_eq!( + header.is_fragmentation_unit(), + cur.is_fu, + "invalid type (fragmentation unit)" + ); + assert_eq!(header.is_paci_packet(), cur.is_paci, "invalid type (PACI)"); + assert_eq!(header.layer_id(), cur.layer_id, "invalid layer_id"); + assert_eq!(header.tid(), cur.tid, "invalid tid"); + } + + Ok(()) + } + + /// Test IRAP (Intra Random Access Point) picture detection. + /// Verifies that BLA, IDR, and CRA NAL types are correctly identified as IRAP frames. + #[test] + fn test_h265_irap_detection() -> Result<()> { + // Test that is_irap() detects all IRAP types (BLA, IDR, CRA) + // BLA_W_LP (16) + let header = H265NALUHeader::new(0x20, 0x01); + assert!(header.is_irap(), "BLA_W_LP should be detected as IRAP"); + assert!(!header.is_idr_picture(), "BLA_W_LP is not an IDR"); + + // BLA_W_RADL (17) + let header = H265NALUHeader::new(0x22, 0x01); + assert!(header.is_irap(), "BLA_W_RADL should be detected as IRAP"); + assert!(!header.is_idr_picture(), "BLA_W_RADL is not an IDR"); + + // BLA_N_LP (18) + let header = H265NALUHeader::new(0x24, 0x01); + assert!(header.is_irap(), "BLA_N_LP should be detected as IRAP"); + assert!(!header.is_idr_picture(), "BLA_N_LP is not an IDR"); + + // IDR_W_RADL (19) + let header = H265NALUHeader::new(0x26, 0x01); + assert!(header.is_irap(), "IDR_W_RADL should be detected as IRAP"); + assert!(header.is_idr_picture(), "IDR_W_RADL is an IDR"); + + // IDR_N_LP (20) + let header = H265NALUHeader::new(0x28, 0x01); + assert!(header.is_irap(), "IDR_N_LP should be detected as IRAP"); + assert!(header.is_idr_picture(), "IDR_N_LP is an IDR"); + + // CRA_NUT (21) + let header = H265NALUHeader::new(0x2a, 0x01); + assert!(header.is_irap(), "CRA_NUT should be detected as IRAP"); + assert!(!header.is_idr_picture(), "CRA_NUT is not an IDR"); + + // TRAIL_R (1) - not an IRAP + let header = H265NALUHeader::new(0x02, 0x01); + assert!(!header.is_irap(), "TRAIL_R should not be detected as IRAP"); + assert!(!header.is_idr_picture(), "TRAIL_R is not an IDR"); + + // VPS (32) - not an IRAP + let header = H265NALUHeader::new(0x40, 0x01); + assert!(!header.is_irap(), "VPS should not be detected as IRAP"); + assert!(!header.is_idr_picture(), "VPS is not an IDR"); + + // RADL_R (6) – not IRAP + let header = H265NALUHeader::new(0x0c, 0x01); // 6 << 1 + assert!(!header.is_irap()); + + // RASL_R (9) – not IRAP + let header = H265NALUHeader::new(0x12, 0x01); // 9 << 1 + assert!(!header.is_irap()); + + // Prefix SEI (39) + let header = H265NALUHeader::new(0x4e, 0x01); // 39 << 1 + assert!(!header.is_irap()); + + Ok(()) + } + + /// Test Fragmentation Unit (FU) header parsing. + /// Verifies S (start), E (end) flags and fragmented NAL type extraction. + #[test] + fn test_h265_fu_header() -> Result<()> { + #[derive(Default)] + struct TestType { + header: H265FragmentationUnitHeader, + s: bool, + e: bool, + typ: u8, + } + + let tests = vec![ + // Start | IDR_W_RADL + TestType { + header: H265FragmentationUnitHeader(0x93), + s: true, + e: false, + typ: 19, + }, + // Continuation | IDR_W_RADL + TestType { + header: H265FragmentationUnitHeader(0x13), + s: false, + e: false, + typ: 19, + }, + // End | IDR_W_RADL + TestType { + header: H265FragmentationUnitHeader(0x53), + s: false, + e: true, + typ: 19, + }, + // Start | TRAIL_R + TestType { + header: H265FragmentationUnitHeader(0x81), + s: true, + e: false, + typ: 1, + }, + // Continuation | TRAIL_R + TestType { + header: H265FragmentationUnitHeader(0x01), + s: false, + e: false, + typ: 1, + }, + // End | TRAIL_R + TestType { + header: H265FragmentationUnitHeader(0x41), + s: false, + e: true, + typ: 1, + }, + // Invalid: S=1 and E=1 simultaneously (illegal per RFC 7798) + TestType { + header: H265FragmentationUnitHeader(0xD3), + s: true, + e: true, + typ: 19, + }, + // Illegal FU: VPS (type 32) must not be fragmented + TestType { + header: H265FragmentationUnitHeader(0xA0), + s: true, + e: false, + typ: 32, + }, + ]; + + for cur in tests { + assert_eq!(cur.header.s(), cur.s, "invalid s field"); + assert_eq!(cur.header.e(), cur.e, "invalid e field"); + assert_eq!(cur.header.fu_type(), cur.typ, "invalid FuType field"); + } + + Ok(()) + } + } // end header_tests + + /// Tests for RTP payload → NAL unit parsing. + /// Validates Single NAL, AP, FU, PACI, and TSCI packet formats. + mod parse_tests { + use super::*; + + /// Test Single NAL Unit packet depacketization. + /// Verifies parsing of single NAL packets with and without DONL, including error cases. + #[test] + fn test_h265_single_nalunit_packet() -> Result<()> { + #[derive(Default)] + struct TestType { + raw: &'static [u8], + with_donl: bool, + expected_packet: Option, + expected_err: Option, + } + + let tests = vec![ + TestType { + raw: &[], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // FBit enabled in H265NALUHeader + TestType { + raw: &[0x80, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrH265CorruptedPacket), + ..Default::default() + }, + // Type '49' in H265NALUHeader + TestType { + raw: &[0x62, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + // Type '50' in H265NALUHeader + TestType { + raw: &[0x64, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + TestType { + raw: &[0x01, 0x01, 0xab, 0xcd, 0xef], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x01, 0x01), + payload: vec![0xab, 0xcd, 0xef], + ..Default::default() + }), + ..Default::default() + }, + // DONL, payload too small + TestType { + raw: &[0x01, 0x01, 0x93, 0xaf], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + TestType { + raw: &[0x01, 0x01, 0xaa, 0xbb, 0xcc], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x01, 0x01), + donl: Some((0xaa << 8) | 0xbb), + payload: vec![0xcc], + ..Default::default() + }), + with_donl: true, + ..Default::default() + }, + // IDR_W_RADL (19) + TestType { + raw: &[0x26, 0x01, 0xde, 0xad, 0xbe, 0xef], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x26, 0x01), + payload: vec![0xde, 0xad, 0xbe, 0xef], + ..Default::default() + }), + ..Default::default() + }, + // CRA_NUT (21) + TestType { + raw: &[0x2a, 0x01, 0xaa, 0xbb], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x2a, 0x01), + payload: vec![0xaa, 0xbb], + ..Default::default() + }), + ..Default::default() + }, + // type=1, layer_id=3, tid=2 + TestType { + raw: &[0x01, 0x32, 0x99, 0x88], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x01, 0x32), + payload: vec![0x99, 0x88], + ..Default::default() + }), + ..Default::default() + }, + // IDR with DONL + TestType { + raw: &[0x26, 0x01, 0x12, 0x34, 0xaa, 0xbb], + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x26, 0x01), + donl: Some(0x1234), + payload: vec![0xaa, 0xbb], + ..Default::default() + }), + with_donl: true, + ..Default::default() + }, + TestType { + raw: &[0x26, 0x01, 0x12, 0x34, 0xaa, 0xbb], + with_donl: true, + expected_packet: Some(H265SingleNALUnitPacket { + payload_header: H265NALUHeader::new(0x26, 0x01), + donl: Some(0x1234), + payload: vec![0xaa, 0xbb], + ..Default::default() + }), + ..Default::default() + }, + ]; + + for cur in tests { + let mut parsed = H265SingleNALUnitPacket::default(); + if cur.with_donl { + parsed.with_donl(cur.with_donl); + } + + let result = parsed.depacketize(&cur.raw); + + if cur.expected_err.is_some() && result.is_ok() { + assert!(false, "should error"); + } else if cur.expected_err.is_none() && result.is_err() { + assert!(false, "should not error"); + } + + if let Some(expected_packet) = cur.expected_packet { + assert_eq!( + expected_packet.payload_header(), + parsed.payload_header(), + "invalid payload header" + ); + assert_eq!(expected_packet.donl(), parsed.donl(), "invalid DONL"); + + assert_eq!( + expected_packet.payload(), + parsed.payload(), + "invalid payload" + ); + } + } + + Ok(()) + } + + /// Test Aggregation Packet (AP) depacketization. + /// Verifies parsing of multiple NAL units in one packet with DONL/DOND fields. + #[test] + fn test_h265_aggregation_packet() -> Result<()> { + #[derive(Default)] + struct TestType { + raw: &'static [u8], + with_donl: bool, + expected_packet: Option, + expected_err: Option, + } + + let tests = vec![ + TestType { + raw: &[], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // FBit enabled in H265NALUHeader + TestType { + raw: &[0x80, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrH265CorruptedPacket), + ..Default::default() + }, + // Type '48' in H265NALUHeader + TestType { + raw: &[0xE0, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + // Small payload + TestType { + raw: &[0x60, 0x01, 0x00, 0x1], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // Small payload + TestType { + raw: &[0x60, 0x01, 0x00], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Small payload + TestType { + raw: &[0x60, 0x01, 0x00, 0x1], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Small payload + TestType { + raw: &[0x60, 0x01, 0x00, 0x01, 0x02], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Single Aggregation Unit + TestType { + raw: &[0x60, 0x01, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Incomplete second Aggregation Unit + TestType { + raw: &[ + 0x60, 0x01, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00, // DONL + 0x00, + ], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Incomplete second Aggregation Unit + TestType { + raw: &[ + 0x60, 0x01, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00, + // DONL, NAL Unit size (2 bytes) + 0x00, 0x55, 0x55, + ], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Valid Second Aggregation Unit + TestType { + raw: &[ + 0x60, 0x01, 0xcc, 0xdd, 0x00, 0x02, 0xff, 0xee, + // DONL, NAL Unit size (2 bytes), Payload + 0x77, 0x00, 0x01, 0xaa, + ], + with_donl: true, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: Some(0xccdd), + nal_unit_size: 2, + nal_unit: vec![0xff, 0xee], + }), + other_units: vec![H265AggregationUnit { + dond: Some(0x77), + nal_unit_size: 1, + nal_unit: vec![0xaa], + }], + might_need_donl: false, + }), + ..Default::default() + }, + // Valid AP WITHOUT DONL/DOND (with_donl = false) + // Requires: first unit + at least 1 other unit + TestType { + raw: &[ + 0x60, 0x01, // AP payload header (Type=48) + 0x00, 0x02, // first NALU size = 2 + 0x11, 0x22, // first NALU + 0x00, 0x01, // second NALU size = 1 + 0x33, // second NALU + ], + with_donl: false, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: None, + nal_unit_size: 2, + nal_unit: vec![0x11, 0x22], + }), + other_units: vec![H265AggregationUnit { + dond: None, + nal_unit_size: 1, + nal_unit: vec![0x33], + }], + might_need_donl: false, + }), + ..Default::default() + }, + // Valid AP WITH DONL + multiple other units (exercise DOND parsing twice) + // Includes DOND=0 and another DOND value + TestType { + raw: &[ + 0x60, 0x01, // AP payload header (Type=48) + 0x00, 0x10, // DONL = 0x0010 + 0x00, 0x01, // first NALU size = 1 + 0xaa, // first NALU + 0x00, // DOND for 2nd AU + 0x00, 0x01, // second NALU size = 1 + 0xbb, // second NALU + 0x05, // DOND for 3rd AU + 0x00, 0x02, // third NALU size = 2 + 0xcc, 0xdd, // third NALU + ], + with_donl: true, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: Some(0x0010), + nal_unit_size: 1, + nal_unit: vec![0xaa], + }), + other_units: vec![ + H265AggregationUnit { + dond: Some(0x00), + nal_unit_size: 1, + nal_unit: vec![0xbb], + }, + H265AggregationUnit { + dond: Some(0x05), + nal_unit_size: 2, + nal_unit: vec![0xcc, 0xdd], + }, + ], + might_need_donl: false, + }), + ..Default::default() + }, + // “Forgiving tail” behavior (with_donl=false): + // After parsing one valid other unit, an incomplete next unit causes a BREAK (not error). + TestType { + raw: &[ + 0x60, 0x01, // AP payload header (Type=48) + 0x00, 0x01, // first NALU size = 1 + 0x11, // first NALU + 0x00, 0x01, // second NALU size = 1 + 0x22, // second NALU + 0x99, // trailing junk: not enough bytes for next 2-byte size => loop breaks + ], + with_donl: false, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: None, + nal_unit_size: 1, + nal_unit: vec![0x11], + }), + other_units: vec![H265AggregationUnit { + dond: None, + nal_unit_size: 1, + nal_unit: vec![0x22], + }], + might_need_donl: false, + }), + ..Default::default() + }, + // "Forgiving tail" behavior (with_donl=true): + // Trailing single DOND byte is read, then payload.len()<2 => BREAK, but + // since we already parsed one unit, OK. + TestType { + raw: &[ + 0x60, 0x01, // AP payload header (Type=48) + 0x12, 0x34, // DONL + 0x00, 0x01, // first NALU size = 1 + 0xaa, // first NALU + 0x01, // DOND for 2nd AU + 0x00, 0x01, // second NALU size = 1 + 0xbb, // second NALU + 0x55, // trailing DOND only, no size => break (and still succeed) + ], + with_donl: true, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: Some(0x1234), + nal_unit_size: 1, + nal_unit: vec![0xaa], + }), + other_units: vec![H265AggregationUnit { + dond: Some(0x01), + nal_unit_size: 1, + nal_unit: vec![0xbb], + }], + might_need_donl: false, + }), + ..Default::default() + }, + // Wrong outer header type: FU (49) should be rejected by AP depacketizer + TestType { + raw: &[ + 0x62, 0x01, // Type=49 (FU), not AP + 0x00, 0x01, 0xaa, // extra bytes, just to avoid short-packet path + ], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + // Wrong outer header type: PACI (50) should be rejected by AP depacketizer + TestType { + raw: &[ + 0x64, 0x01, // Type=50 (PACI), not AP + 0x00, 0x01, 0xaa, + ], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + TestType { + raw: &[ + 0x60, 0x01, 0x12, 0x34, // DONL = 0x1234 + 0x00, 0x01, 0xaa, 0x05, // DOND = +5 + 0x00, 0x01, 0xbb, 0xFE, // DOND = -2 (wrap) + 0x00, 0x01, 0xcc, + ], + with_donl: true, + expected_packet: Some(H265AggregationPacket { + first_unit: Some(H265AggregationUnitFirst { + donl: Some(0x1234), + nal_unit_size: 1, + nal_unit: vec![0xaa], + }), + other_units: vec![ + H265AggregationUnit { + dond: Some(0x05), + nal_unit_size: 1, + nal_unit: vec![0xbb], + }, + H265AggregationUnit { + dond: Some(0xFE), + nal_unit_size: 1, + nal_unit: vec![0xcc], + }, + ], + might_need_donl: false, + }), + ..Default::default() + }, + ]; + + for cur in tests { + let mut parsed = H265AggregationPacket::default(); + if cur.with_donl { + parsed.with_donl(cur.with_donl); + } + + let result = parsed.depacketize(&cur.raw); + + if cur.expected_err.is_some() && result.is_ok() { + assert!(false, "should error"); + } else if cur.expected_err.is_none() && result.is_err() { + assert!(false, "should not error"); + } + + if let Some(expected_packet) = cur.expected_packet { + if let (Some(first_unit), Some(parsed_first_unit)) = + (expected_packet.first_unit(), parsed.first_unit()) + { + assert_eq!( + parsed_first_unit.nal_unit_size, first_unit.nal_unit_size, + "invalid first unit NALUSize" + ); + assert_eq!( + first_unit.donl(), + parsed_first_unit.donl(), + "invalid first unit DONL" + ); + assert_eq!( + first_unit.nal_unit(), + parsed_first_unit.nal_unit(), + "invalid first unit NalUnit" + ); + } + + assert_eq!( + expected_packet.other_units().len(), + parsed.other_units().len(), + "number of other units mismatch" + ); + + for ndx in 0..expected_packet.other_units().len() { + assert_eq!( + parsed.other_units()[ndx].nalu_size(), + expected_packet.other_units()[ndx].nalu_size(), + "invalid unit NALUSize" + ); + + assert_eq!( + expected_packet.other_units()[ndx].dond(), + parsed.other_units()[ndx].dond(), + "invalid unit DOND" + ); + + assert_eq!( + expected_packet.other_units()[ndx].nal_unit(), + parsed.other_units()[ndx].nal_unit(), + "invalid first unit NalUnit" + ); + } + + assert_eq!( + expected_packet.other_units(), + parsed.other_units(), + "invalid payload" + ); + } + } + + Ok(()) + } + + /// Test Fragmentation Unit (FU) packet depacketization. + /// Verifies parsing of fragmented large NAL units with and without DONL. + #[test] + fn test_h265_fragmentation_unit_packet() -> Result<()> { + #[derive(Default)] + struct TestType { + raw: &'static [u8], + with_donl: bool, + expected_fu: Option, + expected_err: Option, + } + let tests = vec![ + TestType { + raw: &[], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // FBit enabled in H265NALUHeader + TestType { + raw: &[0x80, 0x01, 0x93, 0xaf], + expected_err: Some(PacketError::ErrH265CorruptedPacket), + ..Default::default() + }, + // Type not '49' in H265NALUHeader + TestType { + raw: &[0x40, 0x01, 0x93, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93, 0xaf], + expected_fu: Some(H265FragmentationUnitPacket { + payload_header: H265NALUHeader::new(0x62, 0x01), + fu_header: H265FragmentationUnitHeader(0x93), + donl: None, + payload: vec![0xaf], + might_need_donl: false, + }), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93, 0xcc], + with_donl: true, + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93, 0xcc, 0xdd, 0xaf, 0x0d, 0x5a], + with_donl: true, + expected_fu: Some(H265FragmentationUnitPacket { + payload_header: H265NALUHeader::new(0x62, 0x01), + fu_header: H265FragmentationUnitHeader(0x93), + donl: Some((0xcc << 8) | 0xdd), + payload: vec![0xaf, 0x0d, 0x5a], + might_need_donl: false, + }), + ..Default::default() + }, + ]; + + for cur in tests { + let mut parsed = H265FragmentationUnitPacket::default(); + if cur.with_donl { + parsed.with_donl(cur.with_donl); + } + + let result = parsed.depacketize(&cur.raw); + + if cur.expected_err.is_some() && result.is_ok() { + assert!(false, "should error"); + } else if cur.expected_err.is_none() && result.is_err() { + assert!(false, "should not error"); + } + + if let Some(expected_fu) = &cur.expected_fu { + assert_eq!( + parsed.payload_header(), + expected_fu.payload_header(), + "invalid payload header" + ); + assert_eq!( + parsed.fu_header(), + expected_fu.fu_header(), + "invalid FU header" + ); + assert_eq!(parsed.donl(), expected_fu.donl(), "invalid DONL"); + assert_eq!(parsed.payload(), expected_fu.payload(), "invalid Payload"); + } + } + + Ok(()) + } + + /// Test TSCI (Temporal Scalability Control Information) field extraction. + /// Verifies TL0PICIDX, IrapPicID, S, E, and RES bit parsing. + #[test] + fn test_h265_temporal_scalability_control_information() -> Result<()> { + #[derive(Default)] + struct TestType { + value: H265TSCI, + expected_tl0picidx: u8, + expected_irap_pic_id: u8, + expected_s: bool, + expected_e: bool, + expected_res: u8, + } + + let tests = vec![ + TestType { + value: H265TSCI(((0xCA) << 24) | ((0xFE) << 16)), + expected_tl0picidx: 0xCA, + expected_irap_pic_id: 0xFE, + ..Default::default() + }, + TestType { + value: H265TSCI((1) << 15), + expected_s: true, + ..Default::default() + }, + TestType { + value: H265TSCI((1) << 14), + expected_e: true, + ..Default::default() + }, + TestType { + value: H265TSCI((0x0A) << 8), + expected_res: 0x0A, + ..Default::default() + }, + // Sets RES, and force sets S and E to 0. + TestType { + value: H265TSCI( + ((0xAA) << 8) & (u32::MAX ^ ((1) << 15)) & (u32::MAX ^ ((1) << 14)), + ), + expected_res: 0xAA & 0b00111111, + ..Default::default() + }, + ]; + + for cur in tests { + assert_eq!( + cur.value.tl0picidx(), + cur.expected_tl0picidx, + "invalid TL0PICIDX" + ); + assert_eq!( + cur.value.irap_pic_id(), + cur.expected_irap_pic_id, + "invalid IrapPicID" + ); + assert_eq!(cur.value.s(), cur.expected_s, "invalid S"); + assert_eq!(cur.value.e(), cur.expected_e, "invalid E"); + assert_eq!(cur.value.res(), cur.expected_res, "invalid RES"); + } + + Ok(()) + } + + /// Test PACI (Payload Content Information) packet depacketization. + /// Verifies parsing of PACI headers, PHES extensions, and TSCI data. + #[test] + fn test_h265_paci_packet() -> Result<()> { + #[derive(Default)] + struct TestType { + raw: &'static [u8], + expected_fu: Option, + expected_err: Option, + } + + let tests = vec![ + TestType { + raw: &[], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // FBit enabled in H265NALUHeader + TestType { + raw: &[0x80, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrH265CorruptedPacket), + ..Default::default() + }, + // Type not '50' in H265NALUHeader + TestType { + raw: &[0x40, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + // Invalid header extension size + TestType { + raw: &[0x64, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrInvalidH265PacketType), + ..Default::default() + }, + // No Header Extension + TestType { + raw: &[0x64, 0x01, 0x64, 0x00, 0xab, 0xcd, 0xef], + expected_fu: Some(H265PACIPacket { + payload_header: H265NALUHeader::new(0x64, 0x01), + paci_header_fields: ((0x64) << 8), + phes: vec![], + payload: vec![0xab, 0xcd, 0xef], + }), + ..Default::default() + }, + // Header Extension 1 byte + TestType { + raw: &[0x64, 0x01, 0x64, 0x10, 0xff, 0xab, 0xcd, 0xef], + expected_fu: Some(H265PACIPacket { + payload_header: H265NALUHeader::new(0x64, 0x01), + paci_header_fields: ((0x64) << 8) | (0x10), + phes: vec![0xff], + payload: vec![0xab, 0xcd, 0xef], + }), + ..Default::default() + }, + // Header Extension TSCI + TestType { + raw: &[ + 0x64, 0x01, 0x64, 0b00111000, 0xaa, 0xbb, 0x80, 0xab, 0xcd, 0xef, + ], + expected_fu: Some(H265PACIPacket { + payload_header: H265NALUHeader::new(0x64, 0x01), + paci_header_fields: ((0x64) << 8) | (0b00111000), + phes: vec![0xaa, 0xbb, 0x80], + payload: vec![0xab, 0xcd, 0xef], + }), + ..Default::default() + }, + ]; + + for cur in tests { + let mut parsed = H265PACIPacket::default(); + + let result = parsed.depacketize(&cur.raw); + + if cur.expected_err.is_some() && result.is_ok() { + assert!(false, "should error"); + } else if cur.expected_err.is_none() && result.is_err() { + assert!(false, "should not error"); + } + + if let Some(expected_fu) = &cur.expected_fu { + assert_eq!( + expected_fu.payload_header(), + parsed.payload_header(), + "invalid PayloadHeader" + ); + assert_eq!(expected_fu.a(), parsed.a(), "invalid A"); + assert_eq!(expected_fu.ctype(), parsed.ctype(), "invalid CType"); + assert_eq!(expected_fu.phs_size(), parsed.phs_size(), "invalid PHSsize"); + assert_eq!(expected_fu.f0(), parsed.f0(), "invalid F0"); + assert_eq!(expected_fu.f1(), parsed.f1(), "invalid F1"); + assert_eq!(expected_fu.f2(), parsed.f2(), "invalid F2"); + assert_eq!(expected_fu.y(), parsed.y(), "invalid Y"); + assert_eq!(expected_fu.phes(), parsed.phes(), "invalid PHES"); + assert_eq!(expected_fu.payload(), parsed.payload(), "invalid Payload"); + assert_eq!(expected_fu.tsci(), parsed.tsci(), "invalid TSCI"); + } + } + + Ok(()) + } + + /// Test PACI packet creation (packetization). + /// Verifies encoding of inner NAL units into PACI packets with various PHES sizes and flags. + #[test] + fn test_h265_paci_packetizer() -> Result<()> { + // Test 1: Basic PACI packet with no PHES + { + let inner_nalu = vec![0x26, 0x01, 0xab, 0xcd, 0xef]; // Type 19 (IDR_W_RADL) + let phes: &[u8] = &[]; + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, phes, &mut buf)?; + + // Verify packet structure + assert!(packet.len() >= 4, "PACI packet too short"); + + // Verify it's Type 50 + let header = H265NALUHeader::new(packet[0], packet[1]); + assert_eq!(header.nalu_type(), H265NALU_PACI_PACKET_TYPE); + + // Depacketize and verify + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + assert_eq!(decoded.ctype(), 19); // IDR_W_RADL + assert_eq!(decoded.phs_size(), 0); + assert!(!decoded.f0()); + assert_eq!(decoded.payload(), vec![0xab, 0xcd, 0xef]); + } + + // Test 2: PACI packet with 1-byte PHES + { + let inner_nalu = vec![0x02, 0x01, 0xff, 0xee, 0xdd]; + let phes = vec![0xaa]; + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, &phes, &mut buf)?; + + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + assert_eq!(decoded.ctype(), 1); + assert_eq!(decoded.phs_size(), 1); + assert!(!decoded.f0()); // F0 only set if PHES >= 3 bytes + assert_eq!(decoded.phes(), vec![0xaa]); + assert_eq!(decoded.payload(), vec![0xff, 0xee, 0xdd]); + } + + // Test 3: PACI packet with TSCI (3-byte PHES, F0=1) + { + let inner_nalu = vec![0x04, 0x01, 0x11, 0x22, 0x33]; + let phes = vec![0xca, 0xfe, 0x80]; // TSCI data + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, &phes, &mut buf)?; + + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + assert_eq!(decoded.ctype(), 2); + assert_eq!(decoded.phs_size(), 3); + assert!(decoded.f0()); // F0 set for TSCI + assert_eq!(decoded.phes(), vec![0xca, 0xfe, 0x80]); + assert_eq!(decoded.payload(), vec![0x11, 0x22, 0x33]); + } + + // Test 4: PACI with F bit set in inner NALU (A bit) + { + let inner_nalu = vec![0x80, 0x01, 0xaa]; // F bit set + let phes: &[u8] = &[]; + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, phes, &mut buf)?; + + let mut decoded = H265PACIPacket::default(); + // Depacketization will fail due to F bit in PACI header, but we check A bit is set + // Actually, the PACI payload header shouldn't have F bit set - only inner does + // Let's verify the PACI header fields have A bit set + let paci_fields = ((packet[2] as u16) << 8) | (packet[3] as u16); + let a_bit = (paci_fields & (1 << 15)) != 0; + assert!(a_bit, "A bit should be set when inner NALU has F bit"); + } + + // Test 5: PHES too long (>31 bytes) should error + { + let inner_nalu = vec![0x02, 0x01, 0xaa]; + let phes = vec![0u8; 32]; // 32 bytes - too long + let mut buf = Vec::new(); + + let result = H265PACIPacket::packetize(&inner_nalu, &phes, &mut buf); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), PacketError::ErrH265PACIPHESTooLong); + } + + // Test 6: Round-trip test with various layer_id and tid values + { + let inner_nalu = vec![0x12, 0xff, 0x12, 0x34]; // Type 9, layer_id=31, tid=7 + let phes = vec![0x11, 0x22]; + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, &phes, &mut buf)?; + + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + // Verify PACI header preserved layer_id and tid from inner NALU + assert_eq!(decoded.payload_header().layer_id(), 31); + assert_eq!(decoded.payload_header().tid(), 7); + assert_eq!(decoded.ctype(), 9); + assert_eq!(decoded.phs_size(), 2); + assert_eq!(decoded.phes(), vec![0x11, 0x22]); + assert_eq!(decoded.payload(), vec![0x12, 0x34]); + } + + // Test 7: Minimal payload (1 byte) in inner NALU + { + let inner_nalu = vec![0x02, 0x01, 0xaa]; // Header + 1 byte payload + let phes: &[u8] = &[]; + let mut buf = Vec::new(); + + let packet = H265PACIPacket::packetize(&inner_nalu, phes, &mut buf)?; + + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + assert_eq!(decoded.ctype(), 1); + assert_eq!(decoded.payload(), vec![0xaa]); + } + + Ok(()) + } + + /// Test PACI packet round-trip with TSCI extension. + /// Verifies that PACI packets with 3-byte TSCI data can be created and parsed correctly. + #[test] + fn test_h265_paci_roundtrip_with_tsci() -> Result<()> { + // Create a PACI packet with TSCI extension and verify all fields survive round-trip + let inner_nalu = vec![0x26, 0x01, 0xde, 0xad, 0xbe, 0xef]; // IDR_W_RADL + + // Build TSCI: TL0PICIDX=0xAB, IrapPicID=0xCD, S=1, E=0, RES=0x05 + let tsci_bytes = vec![ + 0xAB, // TL0PICIDX + 0xCD, // IrapPicID + 0x85, // S=1, E=0, RES=0x05 + ]; + + let mut buf = Vec::new(); + let packet = H265PACIPacket::packetize(&inner_nalu, &tsci_bytes, &mut buf)?; + + // Depacketize + let mut decoded = H265PACIPacket::default(); + decoded.depacketize(&packet)?; + + // Verify TSCI is present and correct + assert!(decoded.f0(), "F0 should be set for TSCI"); + let tsci = decoded.tsci().expect("TSCI should be present"); + + // Note: The TSCI constructor in the depacketizer has a bug (uses phes[0] three times) + // But we're testing that our packetizer creates valid packets + assert_eq!(decoded.phes(), tsci_bytes); + assert_eq!(decoded.ctype(), 19); // IDR_W_RADL type + assert_eq!(decoded.payload(), vec![0xde, 0xad, 0xbe, 0xef]); + + Ok(()) + } + + /// Test unified H265Depacketizer handling all packet types. + /// Verifies depacketizer correctly routes Single NAL, FU, AP, and PACI packets. + #[test] + fn test_h265_packet() -> Result<()> { + #[derive(Default)] + struct TestType { + raw: &'static [u8], + with_donl: bool, + expected_packet_type: Option, + expected_err: Option, + } + let tests = vec![ + TestType { + raw: &[], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x62, 0x01, 0x93], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x64, 0x01, 0x93, 0xaf], + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + TestType { + raw: &[0x01, 0x01], + with_donl: true, + expected_err: Some(PacketError::ErrShortPacket), + ..Default::default() + }, + // FBit enabled in H265NALUHeader + TestType { + raw: &[0x80, 0x01, 0x93, 0xaf, 0xaf, 0xaf, 0xaf], + expected_err: Some(PacketError::ErrH265CorruptedPacket), + ..Default::default() + }, + // Valid H265SingleNALUnitPacket + TestType { + raw: &[0x01, 0x01, 0xab, 0xcd, 0xef], + expected_packet_type: Some(H265Payload::H265SingleNALUnitPacket( + H265SingleNALUnitPacket::default(), + )), + ..Default::default() + }, + // Invalid H265SingleNALUnitPacket + TestType { + raw: &[0x01, 0x01, 0x93, 0xaf], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // Valid H265PACIPacket + TestType { + raw: &[ + 0x64, 0x01, 0x64, 0b00111000, 0xaa, 0xbb, 0x80, 0xab, 0xcd, 0xef, + ], + expected_packet_type: Some(H265Payload::H265PACIPacket( + H265PACIPacket::default(), + )), + ..Default::default() + }, + // Valid H265FragmentationUnitPacket + TestType { + raw: &[0x62, 0x01, 0x93, 0xcc, 0xdd, 0xaf, 0x0d, 0x5a], + expected_packet_type: Some(H265Payload::H265FragmentationUnitPacket( + H265FragmentationUnitPacket::default(), + )), + with_donl: true, + ..Default::default() + }, + // Valid H265AggregationPacket + TestType { + raw: &[ + 0x60, 0x01, 0xcc, 0xdd, 0x00, 0x02, 0xff, 0xee, 0x77, 0x00, 0x01, 0xaa, + ], + expected_packet_type: Some(H265Payload::H265AggregationPacket( + H265AggregationPacket::default(), + )), + with_donl: true, + ..Default::default() + }, + // Invalid H265AggregationPacket + TestType { + raw: &[0x60, 0x01, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00], + expected_err: Some(PacketError::ErrShortPacket), + with_donl: true, + ..Default::default() + }, + // IDR Single NAL + TestType { + raw: &[0x26, 0x01, 0xde, 0xad], + expected_packet_type: Some(H265Payload::H265SingleNALUnitPacket( + H265SingleNALUnitPacket::default(), + )), + ..Default::default() + }, + // FU start of IDR_W_RADL + TestType { + raw: &[0x62, 0x01, 0x93, 0xaa, 0xbb], + expected_packet_type: Some(H265Payload::H265FragmentationUnitPacket( + H265FragmentationUnitPacket::default(), + )), + ..Default::default() + }, + // AP containing IDR (with DONL) + TestType { + raw: &[ + 0x60, 0x01, // AP + 0x00, 0x10, // DONL + 0x00, 0x01, // size + 0x26, // IDR header byte + 0x00, // DOND + 0x00, 0x01, // size + 0x01, // TRAIL + ], + with_donl: true, + expected_packet_type: Some(H265Payload::H265AggregationPacket( + H265AggregationPacket::default(), + )), + ..Default::default() + }, + // FU with S=1 and E=1 (illegal but must still be routed as FU) + TestType { + raw: &[0x62, 0x01, 0xD3, 0xaa, 0xbb], + expected_packet_type: Some(H265Payload::H265FragmentationUnitPacket( + H265FragmentationUnitPacket::default(), + )), + ..Default::default() + }, + TestType { + raw: &[ + 0x64, 0x01, // PACI header + 0x64, // PHES + 0b00111000, // TSCI + 0x62, 0x01, // FU outer header + 0x93, // S=1, FuType=19 (IDR) + 0xaa, 0xbb, + ], + expected_packet_type: Some(H265Payload::H265PACIPacket( + H265PACIPacket::default(), + )), + ..Default::default() + }, + ]; + + for cur in tests { + let mut pck = H265Depacketizer::default(); + if cur.with_donl { + pck.with_donl(true); + } + + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let result = pck.depacketize(&cur.raw, &mut out, &mut extra); + + if cur.expected_err.is_some() && result.is_ok() { + assert!(false, "should error"); + } else if cur.expected_err.is_none() && result.is_err() { + assert!(false, "should not error"); + } + + if cur.expected_err.is_some() { + continue; + } + + if let Some(expected_packet_type) = &cur.expected_packet_type { + //TODO: assert_eq!(pck.packet(), expected_packet_type, "invalid packet type"); + let pck_packet = pck.payload(); + match (pck_packet, expected_packet_type) { + ( + &H265Payload::H265SingleNALUnitPacket(_), + &H265Payload::H265SingleNALUnitPacket(_), + ) => assert!(true), + ( + &H265Payload::H265FragmentationUnitPacket(_), + &H265Payload::H265FragmentationUnitPacket(_), + ) => assert!(true), + ( + &H265Payload::H265AggregationPacket(_), + &H265Payload::H265AggregationPacket(_), + ) => assert!(true), + (&H265Payload::H265PACIPacket(_), &H265Payload::H265PACIPacket(_)) => { + assert!(true) + } + _ => assert!(false), + }; + } + } + + Ok(()) + } + } // end parse_tests + + /// Tests for NAL unit → RTP payload emission. + /// Validates packetization behavior: single NAL, FU fragmentation, AP aggregation. + mod emit_tests { + use super::*; + + /// Test packetizer handling of small NAL units that fit in one packet. + /// Verifies single NAL units are passed through unchanged. + #[test] + fn test_h265_packetizer_single_nalu() -> Result<()> { + let mut p = H265Packetizer::default(); + + // A minimal "single NAL" (2-byte NAL header + payload). + // Use a non-parameter-set type so the packetizer doesn't cache it. + let nalu = b"\x02\x01\xaa\xbb\xcc".to_vec(); + + let out = p.packetize(1200, &nalu)?; + assert_eq!(out.len(), 1); + assert_eq!(out[0], nalu); + + Ok(()) + } + + /// Test packetizer splitting Annex-B byte stream into individual NAL units. + /// Verifies correct parsing of 00 00 01 and 00 00 00 01 start codes. + #[test] + fn test_h265_packetizer_annexb_split() -> Result<()> { + let mut p = H265Packetizer::default(); + + // Use non-parameter-set NALU types so they are emitted directly. + let nalu1 = b"\x02\x01\x11\x22\x33".to_vec(); + let nalu2 = b"\x26\x01\xaa\xbb\xcc\xdd".to_vec(); + + // Annex-B start codes around the NAL units. + let mut bytestream = Vec::new(); + bytestream.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + bytestream.extend_from_slice(&nalu1); + bytestream.extend_from_slice(&[0x00, 0x00, 0x01]); + bytestream.extend_from_slice(&nalu2); + + let out = p.packetize(1200, &bytestream)?; + assert_eq!(out.len(), 2); + assert_eq!(out[0], nalu1); + assert_eq!(out[1], nalu2); + + Ok(()) + } + + /// Test FU fragmentation preserves NAL unit payload exactly. + /// Verifies fragmented packets can be reassembled to original payload without corruption. + #[test] + fn test_h265_packetizer_fu_fragmentation_roundtrip_payload() -> Result<()> { + let mut p = H265Packetizer::default(); + + // Craft a NAL unit large enough to force FU fragmentation. + // Header 0x02 0x01 => F=0, type=1 (VCL), tid=1. + let mut nalu = vec![0x02, 0x01]; + nalu.extend((0..60).map(|i| i as u8)); + + // Force fragmentation: FU overhead is 3 bytes, so this yields multiple fragments. + let mtu = 20; + let out = p.packetize(mtu, &nalu)?; + assert!(out.len() > 1, "expected fragmentation"); + + // Validate each FU packet and reconstruct the original payload (sans 2-byte NAL header). + let orig_hdr = H265NALUHeader::new(nalu[0], nalu[1]); + let orig_type = orig_hdr.nalu_type(); + let orig_payload = &nalu[H265NALU_HEADER_SIZE..]; + + let mut reconstructed = Vec::new(); + + for (idx, pkt) in out.iter().enumerate() { + assert!(pkt.len() <= mtu); + assert!(pkt.len() >= H265NALU_HEADER_SIZE + H265FRAGMENTATION_UNIT_HEADER_SIZE); + + let hdr = H265NALUHeader::new(pkt[0], pkt[1]); + assert_eq!(hdr.nalu_type(), H265NALU_FRAGMENTATION_UNIT_TYPE); + assert!(!hdr.f()); + assert_eq!(hdr.layer_id(), orig_hdr.layer_id()); + assert_eq!(hdr.tid(), orig_hdr.tid()); + + let fu = H265FragmentationUnitHeader(pkt[2]); + assert_eq!(fu.fu_type(), orig_type); + if idx == 0 { + assert!(fu.s()); + assert!(!fu.e()); + } else if idx == out.len() - 1 { + assert!(!fu.s()); + assert!(fu.e()); + } else { + assert!(!fu.s()); + assert!(!fu.e()); + } + + reconstructed.extend_from_slice(&pkt[3..]); + } + + assert_eq!(reconstructed, orig_payload); + + Ok(()) + } + + /// Test packetizer does not fragment small NAL units. + /// Verifies NAL units smaller than MTU are emitted as single packets. + #[test] + fn test_h265_packetizer_single_nalu_no_fragment() -> Result<()> { + let mut p = H265Packetizer::default(); + + // A small NALU: 2-byte header + payload. + let nalu = [0x02, 0x01, 0xaa, 0xbb, 0xcc, 0xdd]; + let pkts = p.packetize(1200, &nalu)?; + + assert_eq!(pkts.len(), 1); + assert_eq!(pkts[0], nalu); + Ok(()) + } + + /// Test packetizer correctly splits multiple NAL units from Annex-B stream. + /// Verifies each NAL unit is extracted and emitted separately. + #[test] + fn test_h265_packetizer_annexb_splits_nalus() -> Result<()> { + let mut p = H265Packetizer::default(); + + // Use non-parameter-set NALU types so they are emitted directly. + let nalu1 = [0x02, 0x01, 0x11, 0x22, 0x33]; + let nalu2 = [0x26, 0x01, 0x44, 0x55]; + + let mut annexb = Vec::new(); + annexb.extend_from_slice(&[0x00, 0x00, 0x01]); + annexb.extend_from_slice(&nalu1); + annexb.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + annexb.extend_from_slice(&nalu2); + + let pkts = p.packetize(1200, &annexb)?; + + assert_eq!(pkts.len(), 2); + assert_eq!(pkts[0], nalu1); + assert_eq!(pkts[1], nalu2); + Ok(()) + } + + /// Test packetizer aggregates VPS/SPS/PPS parameter sets into AP packet. + /// Verifies parameter sets are cached and emitted together before first VCL NAL. + #[test] + fn test_h265_packetizer_emits_ap_for_vps_sps_pps() -> Result<()> { + let mut p = H265Packetizer::default(); + + // VPS (type 32), SPS (type 33), PPS (type 34), then a VCL NALU (type 1). + let vps = vec![0x40, 0x01, 0x01, 0x02]; + let sps = vec![0x42, 0x01, 0x03, 0x04, 0x05]; + let pps = vec![0x44, 0x01, 0x06]; + let vcl = vec![0x02, 0x01, 0xaa, 0xbb, 0xcc, 0xdd]; + + // Cache parameter sets (no output yet). + assert!(p.packetize(1200, &vps)?.is_empty()); + assert!(p.packetize(1200, &sps)?.is_empty()); + assert!(p.packetize(1200, &pps)?.is_empty()); + + // First non-parameter-set NALU should trigger AP emission. + let out = p.packetize(1200, &vcl)?; + assert_eq!(out.len(), 2); + + // Validate AP packet structure. + assert!(out[0].len() >= H265NALU_HEADER_SIZE + 2); + let ap_hdr = H265NALUHeader::new(out[0][0], out[0][1]); + assert_eq!(ap_hdr.nalu_type(), H265NALU_AGGREGATION_PACKET_TYPE); + + let mut off = H265NALU_HEADER_SIZE; + for expected in [&vps, &sps, &pps] { + let len = u16::from_be_bytes([out[0][off], out[0][off + 1]]) as usize; + off += 2; + assert_eq!(len, expected.len()); + assert_eq!(&out[0][off..off + len], expected.as_slice()); + off += len; + } + + // And the actual VCL NALU follows as a normal single-NALU packet. + assert_eq!(out[1], vcl); + + Ok(()) + } + + /// Test packetizer falls back to individual packets when AP exceeds MTU. + /// Verifies parameter sets are sent separately if aggregation would violate MTU limit. + #[test] + fn test_h265_packetizer_ap_exceeds_mtu_fallback() -> Result<()> { + let mut p = H265Packetizer::default(); + + // Create VPS, SPS, PPS parameter sets. + let vps = vec![0x40, 0x01, 0x01, 0x02]; + let sps = vec![0x42, 0x01, 0x03, 0x04, 0x05]; + let pps = vec![0x44, 0x01, 0x06]; + let vcl = vec![0x02, 0x01, 0xaa, 0xbb]; + + // Cache parameter sets (no output yet). + assert!(p.packetize(1200, &vps)?.is_empty()); + assert!(p.packetize(1200, &sps)?.is_empty()); + assert!(p.packetize(1200, &pps)?.is_empty()); + + // Set MTU to a value smaller than the AP size would be. + // AP overhead: 2 (AP header) + 2 (VPS size) + 4 (VPS) + 2 (SPS size) + // + 5 (SPS) + 2 (PPS size) + 3 (PPS) = 20 bytes + // So MTU=15 will be too small for the AP. + let small_mtu = 15; + let out = p.packetize(small_mtu, &vcl)?; + + // Should emit parameter sets as individual Single NAL packets, then the VCL NALU. + // Expected: VPS, SPS, PPS, VCL = 4 packets + assert_eq!( + out.len(), + 4, + "Expected 4 packets (VPS, SPS, PPS, VCL) when AP exceeds MTU" + ); + + // Verify each parameter set packet. + assert_eq!(out[0], vps, "First packet should be VPS"); + assert_eq!(out[1], sps, "Second packet should be SPS"); + assert_eq!(out[2], pps, "Third packet should be PPS"); + assert_eq!(out[3], vcl, "Fourth packet should be VCL"); + + Ok(()) + } + + /// Test full round-trip of packetizer + depacketizer for fragmented NAL units. + /// Verifies FU packets are correctly created and reassembled to Annex-B format. + #[test] + fn test_h265_fu_roundtrip_with_depacketizer() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let mut depacketizer = H265Depacketizer::default(); + + // Create a large NAL unit that will be fragmented. + // Type 1 (TRAIL_R), layer_id=0, tid=1 + let mut original_nalu = vec![0x02, 0x01]; + original_nalu.extend((0..200).map(|i| (i % 256) as u8)); + + // Force fragmentation with small MTU + let mtu = 50; + let packets = packetizer.packetize(mtu, &original_nalu)?; + + // Verify fragmentation occurred + assert!(packets.len() > 1, "Expected multiple FU packets"); + assert!( + packets.iter().all(|p| p.len() <= mtu), + "All packets should fit MTU" + ); + + // Depacketize each FU packet - they should accumulate in the depacketizer + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + + for (i, packet) in packets.iter().enumerate() { + output.clear(); + extra = CodecExtra::None; + + let result = depacketizer.depacketize(packet, &mut output, &mut extra); + + // Only the last fragment should produce output + if i < packets.len() - 1 { + assert!(result.is_ok()); + assert!( + output.is_empty(), + "Intermediate FU fragments should not produce output" + ); + } else { + assert!(result.is_ok()); + assert!( + !output.is_empty(), + "Final FU fragment should produce output" + ); + // Output should be the complete original NAL unit in Annex-B format + // Depacketizer prepends start code + let expected_output = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&original_nalu); + tmp + }; + assert_eq!( + output, expected_output, + "Depacketized NAL unit should match original with start code" + ); + } + } + + Ok(()) + } + + /// Test full round-trip of packetizer + depacketizer for aggregation packets. + /// Verifies AP packets are correctly created and depacketized to Annex-B format. + #[test] + fn test_h265_ap_roundtrip_with_depacketizer() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let mut depacketizer = H265Depacketizer::default(); + + // Create VPS, SPS, PPS parameter sets and a VCL NAL unit + let vps = vec![0x40, 0x01, 0xaa, 0xbb, 0xcc]; + let sps = vec![0x42, 0x01, 0xdd, 0xee, 0xff, 0x11, 0x22]; + let pps = vec![0x44, 0x01, 0x33, 0x44]; + let vcl = vec![0x26, 0x01, 0x55, 0x66, 0x77, 0x88]; // IDR_W_RADL + + // Cache parameter sets (no output yet) + assert!(packetizer.packetize(1200, &vps)?.is_empty()); + assert!(packetizer.packetize(1200, &sps)?.is_empty()); + assert!(packetizer.packetize(1200, &pps)?.is_empty()); + + // Emit AP + VCL + let packets = packetizer.packetize(1200, &vcl)?; + assert_eq!(packets.len(), 2, "Expected AP packet + VCL packet"); + + // Depacketize the AP packet + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + + depacketizer.depacketize(&packets[0], &mut output, &mut extra)?; + + // AP depacketization produces Annex-B format with start codes before each NAL + let mut offset = 0; + + // Check VPS (with start code) + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!( + &output[offset..offset + vps.len()], + &vps[..], + "VPS should match" + ); + offset += vps.len(); + + // Check SPS (with start code) + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!( + &output[offset..offset + sps.len()], + &sps[..], + "SPS should match" + ); + offset += sps.len(); + + // Check PPS (with start code) + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!( + &output[offset..offset + pps.len()], + &pps[..], + "PPS should match" + ); + offset += pps.len(); + + assert_eq!(offset, output.len(), "All AP payload should be consumed"); + + // Depacketize the VCL packet (also with start code) + output.clear(); + extra = CodecExtra::None; + depacketizer.depacketize(&packets[1], &mut output, &mut extra)?; + let expected_vcl = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&vcl); + tmp + }; + assert_eq!( + output, expected_vcl, + "VCL NAL unit should match original with start code" + ); + + Ok(()) + } + + /// Test full round-trip of packetizer + depacketizer for single NAL units. + /// Verifies small NAL units pass through correctly and are converted to Annex-B format. + #[test] + fn test_h265_single_nalu_roundtrip_with_depacketizer() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let mut depacketizer = H265Depacketizer::default(); + + // Create a small NAL unit that won't be fragmented + let original_nalu = vec![0x02, 0x01, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff]; + + // Packetize with large MTU (no fragmentation) + let packets = packetizer.packetize(1200, &original_nalu)?; + assert_eq!(packets.len(), 1, "Expected single packet"); + assert_eq!( + packets[0], original_nalu, + "Single NAL packet should be unchanged" + ); + + // Depacketize (output will have Annex-B start code) + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + depacketizer.depacketize(&packets[0], &mut output, &mut extra)?; + + let expected_output = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&original_nalu); + tmp + }; + assert_eq!( + output, expected_output, + "Depacketized NAL unit should match original with start code" + ); + + Ok(()) + } + + /// Test realistic sequence with multiple packet types (AP + single NAL + FU). + /// Verifies packetizer/depacketizer handle mixed packet types in one stream. + #[test] + fn test_h265_mixed_packet_types_roundtrip() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let mut depacketizer = H265Depacketizer::default(); + + // Test a realistic sequence: VPS, SPS, PPS, small VCL, large VCL + let vps = vec![0x40, 0x01, 0x01, 0x02, 0x03]; + let sps = vec![0x42, 0x01, 0x04, 0x05, 0x06, 0x07]; + let pps = vec![0x44, 0x01, 0x08, 0x09]; + let small_vcl = vec![0x02, 0x01, 0x0a, 0x0b, 0x0c]; + + // Large VCL that will be fragmented + let mut large_vcl = vec![0x26, 0x01]; // IDR_W_RADL + large_vcl.extend((0..150).map(|i| (i % 256) as u8)); + + // Cache parameter sets + assert!(packetizer.packetize(1200, &vps)?.is_empty()); + assert!(packetizer.packetize(1200, &sps)?.is_empty()); + assert!(packetizer.packetize(1200, &pps)?.is_empty()); + + // First VCL triggers AP emission + let packets1 = packetizer.packetize(1200, &small_vcl)?; + assert_eq!(packets1.len(), 2, "Expected AP + small VCL"); + + // Verify AP depacketization (Annex-B format with start codes) + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + depacketizer.depacketize(&packets1[0], &mut output, &mut extra)?; + + let mut offset = 0; + // VPS with start code + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + vps.len()], &vps[..]); + offset += vps.len(); + // SPS with start code + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + sps.len()], &sps[..]); + offset += sps.len(); + // PPS with start code + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + pps.len()], &pps[..]); + + // Verify small VCL (with start code) + output.clear(); + depacketizer.depacketize(&packets1[1], &mut output, &mut extra)?; + let expected_small_vcl = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&small_vcl); + tmp + }; + assert_eq!(output, expected_small_vcl); + + // Large VCL should be fragmented + let packets2 = packetizer.packetize(60, &large_vcl)?; + assert!(packets2.len() > 1, "Large VCL should be fragmented"); + + // Depacketize FU packets + for (i, packet) in packets2.iter().enumerate() { + output.clear(); + depacketizer.depacketize(packet, &mut output, &mut extra)?; + + if i < packets2.len() - 1 { + assert!( + output.is_empty(), + "Intermediate fragments shouldn't produce output" + ); + } else { + // Final fragment produces Annex-B output + let expected_large_vcl = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&large_vcl); + tmp + }; + assert_eq!( + output, expected_large_vcl, + "Final fragment should produce complete NAL with start code" + ); + } + } + + Ok(()) + } + + /// Test Annex-B input stream through packetizer and depacketizer. + /// Verifies complete round-trip preserves NAL units from Annex-B to RTP to Annex-B. + #[test] + fn test_h265_annexb_roundtrip_with_depacketizer() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let mut depacketizer = H265Depacketizer::default(); + + // Create Annex-B formatted input with multiple NAL units + let nalu1 = vec![0x40, 0x01, 0xaa, 0xbb]; // VPS + let nalu2 = vec![0x42, 0x01, 0xcc, 0xdd, 0xee]; // SPS + let nalu3 = vec![0x44, 0x01, 0xff]; // PPS + let nalu4 = vec![0x02, 0x01, 0x11, 0x22, 0x33]; // VCL + + // Build Annex-B bytestream + let mut annexb = Vec::new(); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&nalu1); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&nalu2); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&nalu3); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&nalu4); + + // Packetize the Annex-B stream + let packets = packetizer.packetize(1200, &annexb)?; + + // Should get AP (with VPS/SPS/PPS) + VCL + assert_eq!(packets.len(), 2, "Expected AP + VCL from Annex-B stream"); + + // Depacketize and verify (output is Annex-B format) + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + + // AP packet produces start code before each NAL + depacketizer.depacketize(&packets[0], &mut output, &mut extra)?; + let mut offset = 0; + // VPS + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + nalu1.len()], &nalu1[..]); + offset += nalu1.len(); + // SPS + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + nalu2.len()], &nalu2[..]); + offset += nalu2.len(); + // PPS + assert_eq!(&output[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&output[offset..offset + nalu3.len()], &nalu3[..]); + + // VCL packet (with start code) + output.clear(); + depacketizer.depacketize(&packets[1], &mut output, &mut extra)?; + let expected_nalu4 = { + let mut tmp = Vec::from(ANNEXB_NALUSTART_CODE); + tmp.extend_from_slice(&nalu4); + tmp + }; + assert_eq!(output, expected_nalu4); + + Ok(()) + } + + /// Test packetizer respects various MTU sizes during fragmentation. + /// Verifies all output packets fit within specified MTU limits. + #[test] + fn test_h265_mtu_variation() -> Result<()> { + // Test fragmentation at various MTU sizes to ensure correct packet splitting + let mut packetizer = H265Packetizer::default(); + + // Create a 2000-byte NAL unit + let mut large_nalu = vec![0x26, 0x01]; // NALU header (type 19 - IDR) + for i in 0..1998 { + large_nalu.push((i % 256) as u8); + } + + // Test various MTU sizes + let mtu_sizes = [100, 200, 500, 1000, MAX_PACKET_SIZE]; + + for mtu in mtu_sizes { + let packets = packetizer.packetize(mtu, &large_nalu)?; + + // Verify all packets fit within MTU + for packet in &packets { + assert!( + packet.len() <= mtu, + "Packet size {} exceeds MTU {} for MTU test", + packet.len(), + mtu + ); + } + + // Verify all packets are FU packets (type 49) + for packet in &packets { + let header = H265NALUHeader::new(packet[0], packet[1]); + assert_eq!( + header.nalu_type(), + H265NALU_FRAGMENTATION_UNIT_TYPE, + "Expected FU packet type for MTU {}", + mtu + ); + } + + // Verify at least one packet was created + assert!( + !packets.is_empty(), + "Expected at least one packet for MTU {}", + mtu + ); + } + + Ok(()) + } + + /// Test FU packet S (start) and E (end) flags are set correctly. + /// Verifies first packet has S=1, middle have S=0 E=0, last has E=1. + #[test] + fn test_h265_fragmentation_start_end_flags() -> Result<()> { + // Explicitly test S (start) and E (end) flags in FU headers + let mut packetizer = H265Packetizer::default(); + + // Create a NAL unit that will require fragmentation + let mut large_nalu = vec![0x26, 0x01]; // IDR NAL header + large_nalu.extend(vec![0xff; 200]); // 202 bytes total + + let packets = packetizer.packetize(100, &large_nalu)?; + + assert!( + packets.len() >= 2, + "Expected fragmentation into multiple packets" + ); + + // First packet should have S flag set + let first_fu_header = H265FragmentationUnitHeader(packets[0][2]); + assert!( + first_fu_header.s(), + "First FU packet should have S flag set" + ); + assert!( + !first_fu_header.e(), + "First FU packet should not have E flag set" + ); + + // Middle packets (if any) should have neither S nor E + for i in 1..packets.len() - 1 { + let mid_fu_header = H265FragmentationUnitHeader(packets[i][2]); + assert!( + !mid_fu_header.s(), + "Middle FU packet should not have S flag set" + ); + assert!( + !mid_fu_header.e(), + "Middle FU packet should not have E flag set" + ); + } + + // Last packet should have E flag set + let last_fu_header = H265FragmentationUnitHeader(packets[packets.len() - 1][2]); + assert!( + !last_fu_header.s(), + "Last FU packet should not have S flag set" + ); + assert!(last_fu_header.e(), "Last FU packet should have E flag set"); + + // All fragments should preserve the original NAL unit type + let original_type = 19; // IDR type + for packet in &packets { + let fu_header = H265FragmentationUnitHeader(packet[2]); + assert_eq!( + fu_header.fu_type(), + original_type, + "FU header should preserve original NAL type" + ); + } + + Ok(()) + } + + /// Test partition head detection for RTP packet reassembly. + /// Verifies single NAL, FU start, and AP packets are identified as partition heads. + #[test] + fn test_h265_is_partition_head() -> Result<()> { + let depacketizer = H265Depacketizer::default(); + + // Nil/empty should not be partition head + assert!( + !depacketizer.is_partition_head(&[]), + "Empty packet should not be partition head" + ); + + // Single NAL unit should be partition head + let single_nalu = vec![0x02, 0x01, 0xab, 0xcd, 0xef]; + assert!( + depacketizer.is_partition_head(&single_nalu), + "Single NAL unit should be partition head" + ); + + // Packet with F bit set should be partition head + let fbit_nalu = vec![0x80, 0x00, 0x00]; + assert!( + depacketizer.is_partition_head(&fbit_nalu), + "F-bit NAL unit should be partition head" + ); + + // FU start packet (S=1) should be partition head + let fu_start = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x93, // FU header: S=1, E=0, type=19 + ]; + assert!( + depacketizer.is_partition_head(&fu_start), + "FU start packet should be partition head" + ); + + // FU middle packet (S=0, E=0) should NOT be partition head + let fu_middle = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x13, // FU header: S=0, E=0, type=19 + ]; + assert!( + !depacketizer.is_partition_head(&fu_middle), + "FU middle packet should not be partition head" + ); + + // FU end packet (S=0, E=1) should NOT be partition head + let fu_end = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x53, // FU header: S=0, E=1, type=19 + ]; + assert!( + !depacketizer.is_partition_head(&fu_end), + "FU end packet should not be partition head" + ); + + // Aggregation packet should be partition head + let ap_packet = vec![ + 0x60, 0x01, // AP indicator (type 48) + 0x00, 0x04, // First NAL size + 0x40, 0x01, 0xaa, 0xbb, // VPS + ]; + assert!( + depacketizer.is_partition_head(&ap_packet), + "Aggregation packet should be partition head" + ); + + Ok(()) + } + + /// Test partition tail detection for RTP packet reassembly. + /// Verifies FU end packets and single NAL with marker are identified as partition tails. + #[test] + fn test_h265_is_partition_tail() -> Result<()> { + let depacketizer = H265Depacketizer::default(); + + // Nil/empty should not be partition tail + assert!( + !depacketizer.is_partition_tail(false, &[]), + "Empty packet should not be partition tail" + ); + + // Single NAL unit without marker should NOT be partition tail + let single_nalu = vec![0x02, 0x01, 0xab, 0xcd, 0xef]; + assert!( + !depacketizer.is_partition_tail(false, &single_nalu), + "Single NAL unit without marker should not be partition tail" + ); + + // Single NAL unit WITH marker should be partition tail + assert!( + depacketizer.is_partition_tail(true, &single_nalu), + "Single NAL unit with marker should be partition tail" + ); + + // F-bit packet without marker should NOT be partition tail + let fbit_nalu = vec![0x80, 0x00, 0x00]; + assert!( + !depacketizer.is_partition_tail(false, &fbit_nalu), + "F-bit NAL unit without marker should not be partition tail" + ); + + // FU start packet should NOT be partition tail + let fu_start = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x93, // FU header: S=1, E=0, type=19 + ]; + assert!( + !depacketizer.is_partition_tail(false, &fu_start), + "FU start packet should not be partition tail" + ); + + // FU middle packet should NOT be partition tail + let fu_middle = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x13, // FU header: S=0, E=0, type=19 + ]; + assert!( + !depacketizer.is_partition_tail(false, &fu_middle), + "FU middle packet should not be partition tail" + ); + + // FU end packet (E=1) should be partition tail + let fu_end = vec![ + 0x62, 0x01, // FU indicator (type 49) + 0x53, // FU header: S=0, E=1, type=19 + ]; + assert!( + depacketizer.is_partition_tail(false, &fu_end), + "FU end packet should be partition tail" + ); + + // Aggregation packet with marker should be partition tail + let ap_packet = vec![ + 0x60, 0x01, // AP indicator (type 48) + 0x00, 0x04, // First NAL size + 0x40, 0x01, 0xaa, 0xbb, // VPS + ]; + assert!( + depacketizer.is_partition_tail(true, &ap_packet), + "Aggregation packet with marker should be partition tail" + ); + + Ok(()) + } + + /// Test depacketization of manually constructed AP with multiple NAL units. + /// Verifies AP structure with multiple aggregated units works correctly. + #[test] + fn test_h265_multi_nalu_aggregation() -> Result<()> { + // Test aggregating multiple non-parameter-set NALUs + // Note: Current implementation only aggregates VPS/SPS/PPS automatically + // This test verifies the AP packet structure works for general NALUs + + let nalu1 = vec![0x02, 0x01, 0xff, 0xff, 0xff]; // VCL NAL + let nalu2 = vec![0x04, 0x01, 0xaa, 0xbb, 0xcc]; // Another VCL NAL + + // Manually build an AP packet + let mut ap_packet = vec![ + 0x60, 0x01, // AP header (type 48) + ]; + + // Add first NAL + ap_packet.extend_from_slice(&(nalu1.len() as u16).to_be_bytes()); + ap_packet.extend_from_slice(&nalu1); + + // Add second NAL + ap_packet.extend_from_slice(&(nalu2.len() as u16).to_be_bytes()); + ap_packet.extend_from_slice(&nalu2); + + // Depacketize and verify + let mut depacketizer = H265Depacketizer::default(); + let mut output = Vec::new(); + let mut extra = CodecExtra::None; + + depacketizer.depacketize(&ap_packet, &mut output, &mut extra)?; + + // Output should contain both NALUs with Annex-B start codes + let mut expected = Vec::new(); + expected.extend_from_slice(ANNEXB_NALUSTART_CODE); + expected.extend_from_slice(&nalu1); + expected.extend_from_slice(ANNEXB_NALUSTART_CODE); + expected.extend_from_slice(&nalu2); + + assert_eq!( + output, expected, + "Depacketized AP should contain both NALUs with start codes" + ); + + Ok(()) + } + + /// Test parameter set caching and AP emission on first VCL NAL. + /// Verifies VPS/SPS/PPS are cached, then emitted as AP when VCL arrives. + #[test] + fn test_h265_packetizer_aggregation_with_marker() -> Result<()> { + // Test that packetizer creates proper aggregation packets + let mut packetizer = H265Packetizer::default(); + + // Build Annex-B stream with VPS + SPS + PPS + let vps = vec![0x40, 0x01, 0xaa, 0xbb]; + let sps = vec![0x42, 0x01, 0xcc, 0xdd, 0xee]; + let pps = vec![0x44, 0x01, 0xff]; + + let mut annexb = Vec::new(); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&vps); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&sps); + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&pps); + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &annexb)?; + + // Should produce 0 packets (just caching parameter sets) + assert_eq!( + packets.len(), + 0, + "Parameter sets should be cached, not emitted immediately" + ); + + // Now send a VCL NAL to trigger AP emission + let mut vcl_annexb = Vec::new(); + vcl_annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + vcl_annexb.extend_from_slice(&[0x02, 0x01, 0x11, 0x22]); + + let vcl_packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl_annexb)?; + + // Should get AP + VCL + assert_eq!( + vcl_packets.len(), + 2, + "Expected AP with parameter sets + VCL packet" + ); + + // First packet should be AP (type 48) + let ap_header = H265NALUHeader::new(vcl_packets[0][0], vcl_packets[0][1]); + assert_eq!( + ap_header.nalu_type(), + H265NALU_AGGREGATION_PACKET_TYPE, + "First packet should be AP" + ); + + // Second packet should be single NAL (VCL type) + let vcl_header = H265NALUHeader::new(vcl_packets[1][0], vcl_packets[1][1]); + assert_eq!( + vcl_header.nalu_type(), + 1, + "Second packet should be VCL NAL (type 1)" + ); + + Ok(()) + } + #[test] + fn test_h265_packetizer_exact_fu_boundary_mtu() -> Result<()> { + // ------------------------------------------------------------------------- + // This test verifies the most dangerous MTU geometry for HEVC FU: + // + // MTU == FU_header_size + (NAL_payload / 2) + // + // which must produce exactly TWO fragments with correct S/E bits and no + // zero-length or extra FU packets. + // ------------------------------------------------------------------------- + + // Large IDR NAL (type = 19) + let nal_payload_size = 1400; + + // Build a fake IDR NALU: [NALU header (2 bytes)] + payload + let mut nalu = Vec::with_capacity(nal_payload_size + 2); + nalu.push(0x26); // nal_unit_type = 19 (IDR_W_RADL) + nalu.push(0x01); + nalu.extend(std::iter::repeat(0xaa).take(nal_payload_size)); + + // FU overhead: 2-byte NAL header + 1-byte FU header = 3 bytes + let fu_overhead = H265NALU_HEADER_SIZE + H265FRAGMENTATION_UNIT_HEADER_SIZE; + + // Force an exact split into two equal fragments + let mtu = fu_overhead + (nal_payload_size / 2); + + let mut packetizer = H265Packetizer::default(); + + // Packetize + let packets = packetizer.packetize(mtu, &nalu)?; + + // Must produce exactly two FU packets + assert_eq!( + packets.len(), + 2, + "exact-boundary FU must produce exactly 2 packets" + ); + + // ---- First packet: FU start ---- + { + let payload = &packets[0]; + let fu_hdr = H265FragmentationUnitHeader(payload[2]); + assert!(fu_hdr.s(), "first FU packet must have S=1"); + assert!(!fu_hdr.e(), "first FU packet must have E=0"); + assert_eq!(fu_hdr.fu_type(), 19, "FU must carry IDR type"); + } + + // ---- Second packet: FU end ---- + { + let payload = &packets[1]; + let fu_hdr = H265FragmentationUnitHeader(payload[2]); + assert!(!fu_hdr.s(), "last FU packet must have S=0"); + assert!(fu_hdr.e(), "last FU packet must have E=1"); + assert_eq!(fu_hdr.fu_type(), 19, "FU must carry IDR type"); + } + + // Reassemble payload to ensure no bytes were lost or duplicated + let mut reconstructed = Vec::new(); + for p in &packets { + // Skip: 2-byte outer NAL header + 1-byte FU header + reconstructed.extend_from_slice(&p[3..]); + } + + assert_eq!( + reconstructed.len(), + nal_payload_size, + "reassembled FU payload must exactly match original NAL size" + ); + + Ok(()) + } + } // end emit_tests + + /// Tests for Decoding Order Number (DONL/DOND) functionality. + /// Validates DONL increments, sequences, and interleaving support. + mod donl_tests { + use super::*; + + /// Test DONL field in single NAL unit packets. + /// Verifies DONL is added during packetization and removed during depacketization. + #[test] + fn test_h265_donl_single_nal_round_trip() -> Result<()> { + // Test DONL with single NAL unit packets + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); // Enable DONL + + let mut depacketizer = H265Depacketizer::default(); + depacketizer.with_donl(true); + + // Create a single NAL unit (type 1, VCL) + let nalu = vec![0x02, 0x01, 0xDE, 0xAD, 0xBE, 0xEF]; + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &nalu)?; + assert_eq!(packets.len(), 1, "Should produce 1 packet"); + + let packet = &packets[0]; + + // Verify DONL is present after NAL header + // Packet structure: [NAL_HDR (2)] [DONL (2)] [PAYLOAD] + assert!( + packet.len() >= 6, + "Packet should contain NAL header + DONL + payload" + ); + + // NAL header should match + assert_eq!(packet[0], 0x02); + assert_eq!(packet[1], 0x01); + + // DONL should be 0 for first packet + let donl = u16::from_be_bytes([packet[2], packet[3]]); + assert_eq!(donl, 0, "DONL should be 0 for first NAL"); + + // Payload should follow DONL + assert_eq!(packet[4], 0xDE); + assert_eq!(packet[5], 0xAD); + + // Depacketize and verify output (Annex-B format without DONL) + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + + // Output should be: [START_CODE (4)] [NAL_HDR (2)] [PAYLOAD] + assert_eq!( + out.len(), + 10, + "Annex-B output should be start code + NAL header + payload" + ); + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..6], &[0x02, 0x01]); + assert_eq!(&out[6..10], &[0xDE, 0xAD, 0xBE, 0xEF]); + + // Verify DONL was parsed + let payload = depacketizer.payload(); + if let H265Payload::H265SingleNALUnitPacket(pkt) = payload { + assert_eq!(pkt.donl(), Some(0), "DONL should be parsed as 0"); + } else { + panic!("Expected H265SingleNALUnitPacket"); + } + + Ok(()) + } + + /// Test DONL field in FU packets (only in first fragment). + /// Verifies DONL appears in FU start packet but not in middle/end fragments. + #[test] + fn test_h265_donl_fragmentation_round_trip() -> Result<()> { + // Test DONL with fragmentation units (FU) + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H265Depacketizer::default(); + depacketizer.with_donl(true); + + // Create a large NAL unit that will be fragmented + let mut nalu = vec![0x02, 0x01]; // NAL header (type 1) + nalu.extend(vec![0xAA; 3000]); // Large payload + + let packets = packetizer.packetize(1200, &nalu)?; + + // Should produce multiple FU packets + assert!( + packets.len() >= 3, + "Large NAL should be fragmented into multiple packets" + ); + + // Check first FU packet structure: [FU_HDR (2)] [FU_HEADER (1)] [DONL (2)] [PAYLOAD] + let first_packet = &packets[0]; + let fu_header = H265NALUHeader::new(first_packet[0], first_packet[1]); + assert_eq!( + fu_header.nalu_type(), + H265NALU_FRAGMENTATION_UNIT_TYPE, + "Should be FU packet" + ); + + // FU header at byte 2 + let fu_hdr = H265FragmentationUnitHeader(first_packet[2]); + assert!(fu_hdr.s(), "First FU should have S flag set"); + assert!(!fu_hdr.e(), "First FU should not have E flag"); + + // DONL should be at bytes 3-4 + let donl = u16::from_be_bytes([first_packet[3], first_packet[4]]); + assert_eq!(donl, 0, "DONL should be 0 for first NAL"); + + // Middle packets should NOT have DONL + if packets.len() > 2 { + let middle_packet = &packets[1]; + let middle_fu_hdr = H265FragmentationUnitHeader(middle_packet[2]); + assert!(!middle_fu_hdr.s(), "Middle FU should not have S flag"); + assert!(!middle_fu_hdr.e(), "Middle FU should not have E flag"); + // Payload starts right after FU header (no DONL) + } + + // Last packet should have E flag but no DONL + let last_packet = &packets[packets.len() - 1]; + let last_fu_hdr = H265FragmentationUnitHeader(last_packet[2]); + assert!(!last_fu_hdr.s(), "Last FU should not have S flag"); + assert!(last_fu_hdr.e(), "Last FU should have E flag set"); + + // Depacketize all fragments + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + + for packet in &packets { + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + } + + // Output should be complete NAL in Annex-B format + assert_eq!( + out.len(), + 4 + nalu.len(), + "Annex-B output should match original" + ); + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..], &nalu[..]); + + // Verify DONL was parsed from first FU + let payload = depacketizer.payload(); + if let H265Payload::H265FragmentationUnitPacket(pkt) = payload { + // Last FU packet won't have DONL, that's expected + // The DONL from the first packet was used internally + } else { + panic!("Expected H265FragmentationUnitPacket"); + } + + Ok(()) + } + + /// Test DONL/DOND fields in aggregation packets. + /// Verifies first aggregated NAL has DONL, subsequent NALs have DOND. + #[test] + fn test_h265_donl_aggregation_round_trip() -> Result<()> { + // Test DONL with aggregation packets (AP) + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H265Depacketizer::default(); + depacketizer.with_donl(true); + + // Create parameter sets to trigger aggregation + let mut annexb = Vec::new(); + + // VPS + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01, 0xFF, 0xFF]); + + // SPS + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&[0x42, 0x01, 0x01, 0x50, 0x00, 0x00]); + + // PPS + annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + annexb.extend_from_slice(&[0x44, 0x01, 0xC0, 0xF3, 0xC0, 0x02]); + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &annexb)?; + assert_eq!(packets.len(), 0, "Parameter sets should be cached"); + + // Send VCL NAL to trigger AP emission + let mut vcl_annexb = Vec::new(); + vcl_annexb.extend_from_slice(ANNEXB_NALUSTART_CODE); + vcl_annexb.extend_from_slice(&[0x02, 0x01, 0x11, 0x22, 0x33]); + + let vcl_packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl_annexb)?; + assert_eq!(vcl_packets.len(), 2, "Should produce AP + VCL"); + + let ap_packet = &vcl_packets[0]; + + // Verify AP packet structure: + // [AP_HDR (2)] [DONL (2)] [NALU_SIZE (2)] [NALU] [DOND (1)] [NALU_SIZE (2)] [NALU] ... + let ap_header = H265NALUHeader::new(ap_packet[0], ap_packet[1]); + assert_eq!(ap_header.nalu_type(), H265NALU_AGGREGATION_PACKET_TYPE); + + // DONL at bytes 2-3 + let donl = u16::from_be_bytes([ap_packet[2], ap_packet[3]]); + assert_eq!(donl, 0, "DONL should be 0 for first aggregated packet"); + + // First NAL size at bytes 4-5 + let first_size = u16::from_be_bytes([ap_packet[4], ap_packet[5]]); + assert_eq!(first_size, 6, "VPS size should be 6 bytes"); + + // After first NAL, there should be DOND (1 byte) before second NAL size + let first_nal_end = 6 + first_size as usize; + let dond1 = ap_packet[first_nal_end]; + assert_eq!(dond1, 0, "DOND should be 0 (same decoding order)"); + + // Depacketize AP + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(ap_packet, &mut out, &mut codec_extra)?; + + // Should output all 3 parameter sets in Annex-B format + // Each: [START_CODE (4)] [NAL] + // Note: Some parameter sets might be filtered/deduplicated, so check minimum + assert!( + out.len() >= 18, + "Should output at least 2 NALs with start codes" + ); + + // Verify first NAL (VPS) starts correctly + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..6], &[0x40, 0x01]); + + Ok(()) + } + + /// Test DONL counter increments for each NAL unit in transmission order. + /// Verifies DONL starts at 0 and increments by 1 for each subsequent NAL. + #[test] + fn test_h265_donl_increments_correctly() -> Result<()> { + // Verify DONL increments for each NAL unit + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + // Send 3 separate NAL units + let nalu1 = vec![0x02, 0x01, 0xAA]; + let nalu2 = vec![0x02, 0x01, 0xBB]; + let nalu3 = vec![0x02, 0x01, 0xCC]; + + let packets1 = packetizer.packetize(MAX_PACKET_SIZE, &nalu1)?; + let packets2 = packetizer.packetize(MAX_PACKET_SIZE, &nalu2)?; + let packets3 = packetizer.packetize(MAX_PACKET_SIZE, &nalu3)?; + + // Check DONL values + let donl1 = u16::from_be_bytes([packets1[0][2], packets1[0][3]]); + let donl2 = u16::from_be_bytes([packets2[0][2], packets2[0][3]]); + let donl3 = u16::from_be_bytes([packets3[0][2], packets3[0][3]]); + + assert_eq!(donl1, 0, "First DONL should be 0"); + assert_eq!(donl2, 1, "Second DONL should be 1"); + assert_eq!(donl3, 2, "Third DONL should be 2"); + + Ok(()) + } + + /// Test packetizer without DONL enabled produces standard packets. + /// Verifies no DONL fields are added when DONL is disabled (default behavior). + #[test] + fn test_h265_without_donl() -> Result<()> { + // Verify that without DONL enabled, no DONL fields are added + let mut packetizer = H265Packetizer::default(); + // Don't call with_donl(true) - DONL should be disabled by default + + let nalu = vec![0x02, 0x01, 0xDE, 0xAD, 0xBE, 0xEF]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &nalu)?; + + assert_eq!(packets.len(), 1); + let packet = &packets[0]; + + // Packet should be exactly the NAL unit (no DONL field) + assert_eq!( + packet.len(), + nalu.len(), + "Packet should not have DONL field" + ); + assert_eq!(packet, &nalu[..]); + + Ok(()) + } + + /// Test SDP-driven DONL enablement via sprop-max-don-diff. + /// Simulates the full flow: SDP fmtp → FormatParams → with_donl(true) → + /// packetize/depacketize round-trip for Single NAL, AP, and FU packet types. + #[test] + fn test_h265_sdp_driven_donl_all_packet_types() -> Result<()> { + use crate::format::FormatParams; + + // Step 1: Parse SDP fmtp line with sprop-max-don-diff > 0 + let fmtp = FormatParams::parse_line("sprop-max-don-diff=32"); + assert_eq!(fmtp.sprop_max_don_diff, Some(32)); + + // Step 2: Enable DONL on packetizer/depacketizer based on SDP (same as Payloader::new) + let donl_enabled = fmtp.sprop_max_don_diff.unwrap_or(0) > 0; + assert!(donl_enabled, "sprop-max-don-diff=32 should enable DONL"); + + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(donl_enabled); + + let mut depacketizer = H265Depacketizer::default(); + depacketizer.with_donl(donl_enabled); + + // --- Test 1: Single NAL Unit with DONL --- + let single_nalu = vec![0x02, 0x01, 0xDE, 0xAD, 0xBE, 0xEF]; // Type 1 (TRAIL_R) + let packets = packetizer.packetize(MAX_PACKET_SIZE, &single_nalu)?; + assert_eq!(packets.len(), 1, "Single NAL should produce 1 packet"); + + // Verify DONL=0 is present on wire: [NAL_HDR(2)] [DONL(2)] [payload] + let pkt = &packets[0]; + assert_eq!( + pkt.len(), + single_nalu.len() + 2, + "Packet should include 2-byte DONL" + ); + let donl = u16::from_be_bytes([pkt[2], pkt[3]]); + assert_eq!(donl, 0, "First NAL should have DONL=0"); + + // Depacketize and verify Annex-B output matches original + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + depacketizer.depacketize(pkt, &mut out, &mut extra)?; + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!( + &out[4..], + &single_nalu[..], + "Depacketized NAL should match original" + ); + + // --- Test 2: AP (VPS + SPS + PPS) with DONL --- + let vps = vec![0x40, 0x01, 0xAA, 0xBB, 0xCC]; + let sps = vec![0x42, 0x01, 0xDD, 0xEE, 0xFF, 0x11]; + let pps = vec![0x44, 0x01, 0x22, 0x33]; + let vcl = vec![0x26, 0x01, 0x44, 0x55, 0x66]; // IDR_W_RADL + + // Cache parameter sets (no output) + assert!(packetizer.packetize(MAX_PACKET_SIZE, &vps)?.is_empty()); + assert!(packetizer.packetize(MAX_PACKET_SIZE, &sps)?.is_empty()); + assert!(packetizer.packetize(MAX_PACKET_SIZE, &pps)?.is_empty()); + + // Trigger AP + VCL emission + let ap_packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl)?; + assert_eq!(ap_packets.len(), 2, "Should produce AP + VCL"); + + // Verify AP wire format: [AP_HDR(2)] [DONL(2)] [Size(2)] [NAL] [DOND(1)] [Size(2)] [NAL] ... + let ap = &ap_packets[0]; + let ap_hdr = H265NALUHeader::new(ap[0], ap[1]); + assert_eq!(ap_hdr.nalu_type(), H265NALU_AGGREGATION_PACKET_TYPE); + let ap_donl = u16::from_be_bytes([ap[2], ap[3]]); + assert_eq!( + ap_donl, 1, + "AP DONL should be 1 (after single NAL consumed DON=0)" + ); + + // Verify DOND bytes are present (0 = sequential order) + let first_size = u16::from_be_bytes([ap[4], ap[5]]) as usize; + let dond_offset = 6 + first_size; // after AP_HDR + DONL + size + first NAL + assert_eq!( + ap[dond_offset], 0, + "DOND should be 0 for sequential decoding" + ); + + // Depacketize AP and verify all 3 NALs appear in Annex-B output + out.clear(); + extra = CodecExtra::None; + depacketizer.depacketize(ap, &mut out, &mut extra)?; + + let mut offset = 0; + for expected in [&vps, &sps, &pps] { + assert_eq!(&out[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&out[offset..offset + expected.len()], &expected[..]); + offset += expected.len(); + } + assert_eq!(offset, out.len(), "All AP data should be consumed"); + + // Verify VCL has correct DONL (AP consumed 3 DONs: 1,2,3 → next is 4) + let vcl_pkt = &ap_packets[1]; + let vcl_donl = u16::from_be_bytes([vcl_pkt[2], vcl_pkt[3]]); + assert_eq!(vcl_donl, 4, "VCL DONL should be 4 (after AP with 3 NALs)"); + + // Depacketize VCL + out.clear(); + extra = CodecExtra::None; + depacketizer.depacketize(vcl_pkt, &mut out, &mut extra)?; + assert_eq!(&out[4..], &vcl[..], "VCL NAL should match original"); + + // --- Test 3: FU (large NAL fragmented) with DONL --- + let mut large_nalu = vec![0x02, 0x01]; // Type 1 + large_nalu.extend(vec![0xAA; 200]); + + let fu_packets = packetizer.packetize(100, &large_nalu)?; + assert!(fu_packets.len() > 1, "Large NAL should be fragmented"); + + // Verify DONL=5 is present only in first FU fragment (S=1) + let first_fu = &fu_packets[0]; + let fu_hdr_byte = H265FragmentationUnitHeader(first_fu[2]); + assert!(fu_hdr_byte.s(), "First FU should have S=1"); + let fu_donl = u16::from_be_bytes([first_fu[3], first_fu[4]]); + assert_eq!(fu_donl, 5, "FU DONL should be 5 (after VCL consumed DON=4)"); + + // Middle/end fragments should NOT have DONL (payload starts at byte 3) + let last_fu = &fu_packets[fu_packets.len() - 1]; + let last_fu_hdr = H265FragmentationUnitHeader(last_fu[2]); + assert!(last_fu_hdr.e(), "Last FU should have E=1"); + assert!(!last_fu_hdr.s(), "Last FU should not have S=1"); + // No DONL in last fragment — payload is at byte 3, not 5 + + // Depacketize all FU fragments + out.clear(); + extra = CodecExtra::None; + for pkt in &fu_packets { + depacketizer.depacketize(pkt, &mut out, &mut extra)?; + } + + // Verify reconstructed NAL matches original + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!( + &out[4..], + &large_nalu[..], + "Reassembled FU should match original" + ); + + // --- Verify DONL counter progression --- + // After all operations: DON 0 (single), 1-3 (AP with 3 NALs), 4 (VCL), 5 (FU) → next = 6 + let next_nalu = vec![0x02, 0x01, 0x77]; + let next_packets = packetizer.packetize(MAX_PACKET_SIZE, &next_nalu)?; + let next_donl = u16::from_be_bytes([next_packets[0][2], next_packets[0][3]]); + assert_eq!(next_donl, 6, "Next DONL should be 6 after full sequence"); + + Ok(()) + } + } // end donl_tests + + /// End-to-end round-trip validation. + /// Tests packetize → depacketize cycles for correctness. + mod roundtrip_tests { + use super::*; + + /// Test complete round-trip with DONL. + /// Verifies packetizer → depacketizer with DONL enabled produces identical output. + #[test] + fn test_h265_roundtrip_with_donl() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H265Depacketizer::default(); + depacketizer.with_donl(true); + + // Test 1: Single NAL with DONL + let single_nalu = vec![0x02, 0x01, 0xff, 0xff, 0xff]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &single_nalu)?; + + assert_eq!(packets.len(), 1, "Single NAL should produce 1 packet"); + + // Verify DONL is present (bytes 2-3 after NAL header) + let donl = u16::from_be_bytes([packets[0][2], packets[0][3]]); + assert_eq!(donl, 0, "First DONL should be 0"); + + // Depacketize and verify + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(&packets[0], &mut out, &mut codec_extra)?; + assert!(!out.is_empty(), "Should depacketize successfully"); + + // Test 2: Fragmented NAL with DONL + let mut large_nalu = vec![0x02, 0x01]; + for i in 0..512 { + large_nalu.push((i % 256) as u8); + } + + let fu_packets = packetizer.packetize(100, &large_nalu)?; + assert!(fu_packets.len() > 1, "Large NAL should fragment"); + + // First fragment should have DONL + let first_donl = u16::from_be_bytes([fu_packets[0][3], fu_packets[0][4]]); + assert_eq!(first_donl, 1, "Second NAL should have DONL=1"); + + // Reassemble all fragments + out.clear(); + codec_extra = CodecExtra::None; + for packet in &fu_packets { + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + } + + assert!(!out.is_empty(), "Should reassemble fragmented NAL"); + + // Verify reconstructed payload matches original (minus Annex B start codes) + // The output will have Annex B format with start codes + let has_start_code = out.len() > 4 && out[0..4] == [0x00, 0x00, 0x00, 0x01]; + assert!(has_start_code || out.len() > 0, "Should have valid output"); + + Ok(()) + } + + /// Test aggregation with exact DONL values. + /// Verifies contiguous and non-contiguous DONL sequences in AP packets. + #[test] + fn test_h265_aggregation_with_donl_sequences() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + // Send multiple parameter sets to build up cache + let vps = vec![0x40, 0x01, 0x00, 0x01, 0x02, 0x03]; + let sps = vec![0x42, 0x01, 0x00, 0x01, 0x02, 0x03]; + let pps = vec![0x44, 0x01, 0x00, 0x01, 0x02, 0x03]; + + packetizer.packetize(MAX_PACKET_SIZE, &vps)?; + packetizer.packetize(MAX_PACKET_SIZE, &sps)?; + packetizer.packetize(MAX_PACKET_SIZE, &pps)?; + + // Trigger with VCL NAL + let vcl = vec![0x02, 0x01, 0xAA, 0xBB]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl)?; + + // Should emit AP + VCL or individual packets with DONL + assert!(!packets.is_empty(), "Should produce packets"); + + // Each packet should have proper structure + for packet in &packets { + assert!(packet.len() >= 2, "Packet should have at least NAL header"); + } + + Ok(()) + } + } // end roundtrip_tests + + /// Edge case and buffer overflow prevention tests. + /// Validates MTU boundary conditions, zero-length inputs, and error handling. + mod regression_tests { + use super::*; + + /// Test packetizer with zero MTU returns empty output. + /// Verifies that invalid MTU (0) is handled gracefully without panicking. + #[test] + fn test_h265_zero_mtu() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let nalu = vec![0x02, 0x01, 0xAA, 0xBB, 0xCC]; + + let packets = packetizer.packetize(0, &nalu)?; + + // Zero MTU should result in no packets being created + assert!(packets.is_empty(), "Zero MTU should produce no packets"); + + Ok(()) + } + + /// Test packetizer with empty NAL unit returns empty output. + /// Verifies that empty input is handled gracefully without panicking. + #[test] + fn test_h265_empty_nalu() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + let nalu = vec![]; + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &nalu)?; + + // Empty NAL unit should result in no packets + assert!( + packets.is_empty(), + "Empty NAL unit should produce no packets" + ); + + Ok(()) + } + + /// Test packetizer with MTU smaller than FU overhead. + /// Verifies that MTU too small for fragmentation is handled gracefully. + #[test] + fn test_h265_mtu_smaller_than_fu_overhead() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create a large NAL that would require fragmentation + let mut large_nalu = vec![0x02, 0x01]; + large_nalu.extend(vec![0xAA; 200]); + + // FU overhead is 3 bytes (2 FU indicator + 1 FU header) + // MTU = 2 is smaller than overhead, should produce no packets + let packets = packetizer.packetize(2, &large_nalu)?; + + assert!( + packets.is_empty(), + "MTU smaller than FU overhead should produce no packets" + ); + + Ok(()) + } + + /// Test packetizer with MTU exactly equal to FU overhead. + /// Verifies edge case where MTU = overhead (no room for payload). + #[test] + fn test_h265_mtu_equals_fu_overhead() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create a large NAL that would require fragmentation + let mut large_nalu = vec![0x02, 0x01]; + large_nalu.extend(vec![0xBB; 200]); + + // FU overhead is 3 bytes, no room for payload + let packets = packetizer.packetize(3, &large_nalu)?; + + assert!( + packets.is_empty(), + "MTU equal to FU overhead (no payload room) should produce no packets" + ); + + Ok(()) + } + + /// Test packetizer with MTU = FU overhead + DONL (with DONL enabled). + /// Verifies that when DONL is enabled, first fragment needs extra space. + #[test] + fn test_h265_mtu_equals_fu_overhead_plus_donl() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + // Create a large NAL that would require fragmentation + let mut large_nalu = vec![0x02, 0x01]; + large_nalu.extend(vec![0xCC; 200]); + + // FU overhead (3) + DONL (2) = 5 bytes, no room for payload + let packets = packetizer.packetize(5, &large_nalu)?; + + assert!( + packets.is_empty(), + "MTU equal to FU overhead + DONL (no payload room) should produce no packets" + ); + + Ok(()) + } + + /// Test FU packetization with MTU larger than MAX_PACKET_SIZE. + /// Verifies that effective_mtu is clamped to MAX_PACKET_SIZE (1200) to prevent buffer overflow. + #[test] + fn test_h265_fu_mtu_exceeds_max_packet_size() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create a large NAL that requires fragmentation + let mut large_nalu = vec![0x02, 0x01]; + large_nalu.extend(vec![0xDD; 3000]); + + // Request MTU=2000, but should be clamped to MAX_PACKET_SIZE=1200 + let packets = packetizer.packetize(2000, &large_nalu)?; + + assert!(!packets.is_empty(), "Should produce FU packets"); + + // All packets should fit within MAX_PACKET_SIZE (1200) + for (i, packet) in packets.iter().enumerate() { + assert!( + packet.len() <= 1200, + "Packet {} size {} exceeds MAX_PACKET_SIZE (1200)", + i, + packet.len() + ); + } + + Ok(()) + } + + /// Test single NAL with DONL produces correct packet. + /// Verifies that large single NAL units with DONL are handled correctly. + #[test] + fn test_h265_single_nal_with_donl_large() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + packetizer.with_donl(true); + + // Create a large NAL that fits MTU + let mut large_nalu = vec![0x02, 0x01]; + large_nalu.extend(vec![0xEE; 1197]); // Total 1199 bytes + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &large_nalu)?; + + // With Vec buffer, large packets within MTU are produced successfully + assert_eq!(packets.len(), 1); + // Packet should have NAL header (2) + DONL (2) + payload (1197) + assert_eq!(packets[0].len(), 1199 + 2); // Original + DONL + + Ok(()) + } + + /// Test AP packet that would exceed MTU falls back to individual packets. + /// Verifies that when aggregation would violate MTU, parameter sets are sent separately. + #[test] + fn test_h265_ap_buffer_overflow_fallback() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create large parameter sets that would overflow when aggregated + let mut vps = vec![0x40, 0x01]; + vps.extend(vec![0xAA; 100]); + + let mut sps = vec![0x42, 0x01]; + sps.extend(vec![0xBB; 100]); + + let mut pps = vec![0x44, 0x01]; + pps.extend(vec![0xCC; 100]); + + let vcl = vec![0x02, 0x01, 0xDD]; + + // Cache parameter sets + assert!(packetizer.packetize(1200, &vps)?.is_empty()); + assert!(packetizer.packetize(1200, &sps)?.is_empty()); + assert!(packetizer.packetize(1200, &pps)?.is_empty()); + + // Trigger emission with MTU that's too small for AP + // AP overhead: 2 (header) + 3×2 (size fields) + 102+102+102 = 314 bytes + let small_mtu = 200; + let packets = packetizer.packetize(small_mtu, &vcl)?; + + // Should emit as individual packets (VPS, SPS, PPS, VCL) + assert_eq!(packets.len(), 4, "Should fall back to 4 individual packets"); + + // Verify all packets fit within MTU + for packet in &packets { + assert!( + packet.len() <= small_mtu, + "Fallback packet size {} exceeds MTU {}", + packet.len(), + small_mtu + ); + } + + Ok(()) + } + + /// Test FU packetization handles minimal MTU (overhead + 1 byte payload). + /// Verifies that fragmentation works even with smallest possible payload per packet. + #[test] + fn test_h265_fu_minimal_mtu() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create a NAL that requires fragmentation + let nalu = vec![0x02, 0x01, 0xAA, 0xBB, 0xCC, 0xDD]; + + // FU overhead = 3 bytes, MTU = 4 allows 1 byte payload per packet + let mtu = 4; + let packets = packetizer.packetize(mtu, &nalu)?; + + // Should create multiple packets, each with 1 byte of payload + assert!( + packets.len() >= 4, + "Should fragment into multiple packets with 1-byte payload" + ); + + // Verify all packets fit within MTU + for packet in &packets { + assert!(packet.len() <= mtu, "Packet exceeds MTU"); + } + + Ok(()) + } + + /// Test packetizer with NAL unit smaller than header size. + /// Verifies that malformed NAL (< 2 bytes) is handled gracefully. + #[test] + fn test_h265_nalu_smaller_than_header() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // NAL unit with only 1 byte (invalid, needs at least 2 for header) + let invalid_nalu = vec![0x02]; + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &invalid_nalu)?; + + // Should drop malformed NAL (< H265NALU_HEADER_SIZE) + assert!( + packets.is_empty(), + "Malformed NAL (< 2 bytes) should be dropped" + ); + + Ok(()) + } + } // end regression_tests + + /// Integration tests for complex scenarios. + /// Tests aggregation layouts, SE flag correctness, and real-world payloads. + mod integration_tests { + use super::*; + + /// Test depacketization of real H.265 RTP payloads from Wireshark captures. + /// Verifies practical compatibility with actual WebRTC H.265 streams. + /// This is an integration test because it exercises fragmentation, reassembly, and Annex-B output. + #[test] + fn test_h265_packet_real() -> Result<()> { + // Tests decoding of real H265 payloads extracted from a Wireshark dump. + let tests = vec![ + b"\x40\x01\x0c\x01\xff\xff\x01\x60\x00\x00\x03\x00\xb0\x00\x00\x03\x00\x00\x03\x00\x7b\xac\x09" + .to_vec(), + b"\x42\x01\x01\x01\x60\x00\x00\x03\x00\xb0\x00\x00\x03\x00\x00\x03\x00\x7b\xa0\x03\xc0\x80\x10\ + \xe5\x8d\xae\x49\x32\xf4\xdc\x04\x04\x04\x02".to_vec(), + b"\x44\x01\xc0\xf2\xf0\x3c\x90".to_vec(), + b"\x4e\x01\xe5\x04\x61\x0c\x00\x00\x80".to_vec(), + // Large test vector split across multiple lines for readability + [ + &b"\x62\x01\x93\xaf\x0d\x5a\xfe\x67\x77\x29\xc0\x74\xf3\x57\x4c\x16\x94\xaa"[..], + &b"\x7c\x2a\x64\x5f\xe9\xa5\xb7\x2a\xa3\x95\x9d\x94\xa7\xb4\xd3\xc4\x4a\xb1"[..], + &b"\xb7\x69\xca\xbe\x75\xc5\x64\xa8\x97\x4b\x8a\xbf\x7e\xf0\x0f\xc3\x22\x60"[..], + &b"\x67\xab\xae\x96\xd6\x99\xca\x7a\x8d\x35\x93\x1a\x67\x60\xe7\xbe\x7e\x13"[..], + &b"\x95\x3c\xe0\x11\xc1\xc1\xa7\x48\xef\xf7\x7b\xb0\xeb\x35\x49\x81\x4e\x4e"[..], + &b"\x54\xf7\x31\x6a\x38\xa1\xa7\x0c\xd6\xbe\x3b\x25\xba\x08\x19\x0b\x49\xfd"[..], + &b"\x90\xbb\x73\x7a\x45\x8c\xb9\x73\x43\x04\xc5\x5f\xda\x0f\xd5\x70\x4c\x11"[..], + &b"\xee\x72\xb8\x6a\xb4\x95\x62\x64\xb6\x23\x14\x7e\xdb\x0e\xa5\x0f\x86\x31"[..], + &b"\xe4\xd1\x64\x56\x43\xf6\xb7\xe7\x1b\x93\x4a\xeb\xd0\xa6\xe3\x1f\xce\xda"[..], + &b"\x15\x67\x05\xb6\x77\x36\x8b\x27\x5b\xc6\xf2\x95\xb8\x2b\xcc\x9b\x0a\x03"[..], + &b"\x05\xbe\xc3\xd3\x85\xf5\x69\xb6\x19\x1f\x63\x2d\x8b\x65\x9e\xc3\x9d\xd2"[..], + &b"\x44\xb3\x7c\x86\x3b\xea\xa8\x5d\x02\xe5\x40\x03\x20\x76\x48\xff\xf6\x2b"[..], + &b"\x0d\x18\xd6\x4d\x49\x70\x1a\x5e\xb2\x89\xca\xec\x71\x41\x79\x4e\x94\x17"[..], + &b"\x0c\x57\x51\x55\x14\x61\x40\x46\x4b\x3e\x17\xb2\xc8\xbd\x1c\x06\x13\x91"[..], + &b"\x72\xf8\xc8\xfc\x6f\xb0\x30\x9a\xec\x3b\xa6\xc9\x33\x0b\xa5\xe5\xf4\x65"[..], + &b"\x7a\x29\x8b\x76\x62\x81\x12\xaf\x20\x4c\xd9\x21\x23\x9e\xeb\xc9\x0e\x5b"[..], + &b"\x29\x35\x7f\x41\xcd\xce\xa1\xc4\xbe\x01\x30\xb9\x11\xc3\xb1\xe4\xce\x45"[..], + &b"\xd2\x5c\xb3\x1e\x69\x78\xba\xb1\x72\xe4\x88\x54\xd8\x5d\xd0\xa8\x3a\x74"[..], + &b"\xad\xe5\xc7\xc1\x59\x7c\x78\x15\x26\x37\x3d\x50\xae\xb3\xa4\x5b\x6c\x7d"[..], + &b"\x65\x66\x85\x4d\x16\x9a\x67\x74\xad\x55\x32\x3a\x84\x85\x0b\x6a\xeb\x24"[..], + &b"\x97\xb4\x20\x4d\xca\x41\x61\x7a\xd1\x7b\x60\xdb\x7f\xd5\x61\x22\xcf\xd1"[..], + &b"\x7e\x4c\xf3\x85\xfd\x13\x63\xe4\x9d\xed\xac\x13\x0a\xa0\x92\xb7\x34\xde"[..], + &b"\x65\x0f\xd9\x0f\x9b\xac\xe2\x47\xe8\x5c\xb3\x11\x8e\xc6\x08\x19\xd0\xb0"[..], + &b"\x85\x52\xc8\x5c\x1b\x08\x0a\xce\xc9\x6b\xa7\xef\x95\x2f\xd0\xb8\x63\xe5"[..], + &b"\x4c\xd4\xed\x6e\x87\xe9\xd4\x0a\xe6\x11\x44\x63\x00\x94\x18\xe9\x28\xba"[..], + &b"\xcf\x92\x43\x06\x59\xdd\x37\x4f\xd3\xef\x9d\x31\x5e\x9b\x48\xf9\x1f\x3e"[..], + &b"\x7b\x95\x3a\xbd\x1f\x71\x55\x0c\x06\xf9\x86\xf8\x3d\x39\x16\x50\xb3\x21"[..], + &b"\x11\x19\x6f\x70\xa9\x48\xe8\xbb\x0a\x11\x23\xf8\xab\xfe\x44\xe0\xbb\xe8"[..], + &b"\x64\xfa\x85\xe4\x02\x55\x88\x41\xc6\x30\x7f\x10\xad\x75\x02\x4b\xef\xe1"[..], + &b"\x0b\x06\x3c\x10\x49\x83\xf9\xd1\x3e\x3e\x67\x86\x4c\xf8\x9d\xde\x5a\xc4"[..], + &b"\xc8\xcf\xb6\xf4\xb0\xd3\x34\x58\xd4\x7b\x4d\xd3\x37\x63\xb2\x48\x8a\x7e"[..], + &b"\x20\x00\xde\xb4\x42\x8f\xda\xe9\x43\x9e\x0c\x16\xce\x79\xac\x2c\x70\xc1"[..], + &b"\x89\x05\x36\x62\x6e\xd9\xbc\xfb\x63\xc6\x79\x89\x3c\x90\x89\x2b\xd1\x8c"[..], + &b"\xe0\xc2\x54\xc7\xd6\xb4\xe8\x9e\x96\x55\x6e\x7b\xd5\x7f\xac\xd4\xa7\x1c"[..], + &b"\xa0\xdf\x01\x30\xad\xc0\x9f\x69\x06\x10\x43\x7f\xf4\x5d\x62\xa3\xea\x73"[..], + &b"\xf2\x14\x79\x19\x13\xea\x59\x14\x79\xa8\xe7\xce\xce\x44\x25\x13\x41\x18"[..], + &b"\x57\xdd\xce\xe4\xbe\xcc\x20\x80\x29\x71\x73\xa7\x7c\x86\x39\x76\xf4\xa7"[..], + &b"\x1c\x63\x24\x21\x93\x1e\xb5\x9a\x5c\x8a\x9e\xda\x8b\x9d\x88\x97\xfc\x98"[..], + &b"\x7d\x26\x74\x04\x1f\xa8\x10\x4f\x45\xcd\x46\xe8\x28\xe4\x8e\x59\x67\x63"[..], + &b"\x4a\xcf\x1e\xed\xdd\xbb\x79\x2f\x8d\x94\xab\xfc\xdb\xc5\x79\x1a\x4d\xcd"[..], + &b"\x53\x41\xdf\xd1\x7a\x8f\x46\x3e\x1f\x79\x88\xe3\xee\x9f\xc4\xc1\xe6\x2e"[..], + &b"\x89\x4d\x28\xc9\xca\x28\xc2\x0a\xc5\xc7\xf1\x22\xcd\xb3\x36\xfa\xe3\x7e"[..], + &b"\xa6\xcd\x95\x55\x5e\x0e\x1a\x75\x7f\x65\x27\xd3\x37\x4f\x23\xc5\xab\x49"[..], + &b"\x68\x4e\x02\xb5\xbf\xd7\x95\xc0\x78\x67\xbc\x1a\xe9\xae\x6f\x44\x58\x8a"[..], + &b"\xc2\xce\x42\x98\x4e\x77\xc7\x2a\xa0\xa7\x7d\xe4\x3b\xd1\x20\x82\x1a\xd3"[..], + &b"\xe2\xc7\x76\x5d\x06\x46\xb5\x24\xd7\xfb\x57\x63\x2b\x19\x51\x48\x65\x6d"[..], + &b"\xfb\xe0\x98\xd1\x14\x0e\x17\x64\x29\x34\x6f\x6e\x66\x9e\x8d\xc9\x89\x49"[..], + &b"\x69\xee\x74\xf3\x35\xe6\x8b\x67\x56\x95\x7f\x1b\xe9\xed\x8c\x0f\xe2\x19"[..], + &b"\x59\xbf\x03\x35\x55\x3c\x04\xbc\x40\x52\x90\x10\x08\xad\xa7\x65\xe0\x31"[..], + &b"\xcb\xcf\x3d\xd4\x62\x68\x01\x0d\xed\xf5\x28\x64\x2d\xaa\x7c\x99\x15\x8d"[..], + &b"\x70\x32\x53\xb8\x9d\x0a\x3c\xbf\x91\x02\x04\xd0\xee\x87\xce\x04\xcc\x3e"[..], + &b"\xa8\x20\xfd\x97\xdf\xbf\x4a\xbc\xfc\xc9\x7c\x77\x21\xcc\x23\x6f\x59\x38"[..], + &b"\xd8\xd9\xa0\x0e\xb1\x23\x4e\x04\x3f\x14\x9e\xcc\x05\x54\xab\x20\x69\xed"[..], + &b"\xa4\xd5\x1d\xb4\x1b\x52\xed\x6a\xea\xeb\x7f\xd1\xbc\xfd\x75\x20\xa0\x1c"[..], + &b"\x59\x8c\x5a\xa1\x2a\x70\x64\x11\xb1\x7b\xc1\x24\x80\x28\x51\x4c\x94\xa1"[..], + &b"\x95\x64\x72\xe8\x90\x67\x38\x74\x2b\xab\x38\x46\x12\x71\xce\x19\x98\x98"[..], + &b"\xf7\x89\xd4\xfe\x2f\x2a\xc5\x61\x20\xd0\xa4\x1a\x51\x3c\x82\xc8\x18\x31"[..], + &b"\x7a\x10\xe8\x1c\xc6\x95\x5a\xa0\x82\x88\xce\x8f\x4b\x47\x85\x7e\x89\x95"[..], + &b"\x95\x52\x1e\xac\xce\x45\x57\x61\x38\x97\x2b\x62\xa5\x14\x6f\xc3\xaa\x6c"[..], + &b"\x35\x83\xc9\xa3\x1e\x30\x89\xf4\xb1\xea\x4f\x39\xde\xde\xc7\x46\x5c\x0e"[..], + &b"\x85\x41\xec\x6a\xa4\xcb\xee\x70\x9c\x57\xd9\xf4\xa1\xc3\x9c\x2a\x0a\xf0"[..], + &b"\x5d\x58\xb0\xae\xd4\xdc\xc5\x6a\xa8\x34\xfa\x23\xef\xef\x08\x39\xc3\x3d"[..], + &b"\xea\x11\x6e\x6a\xe0\x1e\xd0\x52\xa8\xc3\x6e\xc9\x1c\xfc\xd0\x0c\x4c\xea"[..], + &b"\x0d\x82\xcb\xdd\x29\x1a\xc4\x4f\x6e\xa3\x4d\xcb\x7a\x38\x77\xe5\x15\x6e"[..], + &b"\xad\xfa\x9d\x2f\x02\xb6\x39\x84\x3a\x60\x8f\x71\x9f\x92\xe5\x24\x4f\xbd"[..], + &b"\x18\x49\xd5\xef\xbf\x70\xfb\xd1\x4c\x2e\xfc\x2f\x36\xf3\x00\x31\x2e\x90"[..], + &b"\x18\xcc\xf4\x71\xb9\xe4\xf9\xbe\xcb\x5e\xff\xf3\xe7\xf8\xca\x03\x60\x66"[..], + &b"\xb3\xc9\x5a\xf9\x74\x09\x02\x57\xb6\x90\x94\xfc\x41\x35\xdc\x35\x3f\x32"[..], + &b"\x7a\xa6\xa5\xcd\x8a\x8f\xc8\x3d\xc8\x81\xc3\xec\x37\x74\x86\x61\x41\x0d"[..], + &b"\xc5\xe2\xc8\x0c\x84\x2b\x3b\x71\x58\xde\x1b\xe3\x20\x65\x2e\x76\xf4\x98"[..], + &b"\xd8\xaa\x78\xe6\xeb\xb8\x85\x0d\xa0\xd0\xf5\x57\x64\x01\x58\x55\x82\xd5"[..], + &b"\x0f\x2d\x9c\x3e\x2a\xa0\x7e\xaf\x42\xf3\x37\xd1\xb3\xaf\xda\x5b\xa9\xda"[..], + &b"\xe3\x89\x5d\xf1\xca\xa5\x12\x3d\xe7\x91\x95\x53\x21\x72\xca\x7f\xf6\x79"[..], + &b"\x59\x21\xcf\x30\x18\xfb\x78\x55\x40\x59\xc3\xf9\xf1\xdd\x58\x44\x5e\x83"[..], + &b"\x11\x5c\x2d\x1d\x91\xf6\x01\x3d\x3f\xd4\x33\x81\x66\x6c\x40\x7a\x9d\x70"[..], + &b"\x10\x58\xe6\x53\xad\x85\x11\x99\x3e\x4b\xbc\x31\xc6\x78\x9d\x79\xc5\xde"[..], + &b"\x9f\x2e\x43\xfa\x76\x84\x2f\xfd\x28\x75\x12\x48\x25\xfd\x15\x8c\x29\x6a"[..], + &b"\x91\xa4\x63\xc0\xa2\x8c\x41\x3c\xf1\xb0\xf8\xdf\x66\xeb\xbd\x14\x88\xa9"[..], + &b"\x81\xa7\x35\xc4\x41\x40\x6c\x10\x3f\x09\xbd\xb5\xd3\x7a\xee\x4b\xd5\x86"[..], + &b"\xff\x36\x03\x6b\x78\xde"[..], + ].concat(), + b"\x62\x01\x53\x8a\xe9\x25\xe1\x06\x09\x8e\xba\x12\x74\x87\x09\x9a\x95\xe4\x86\x62\x2b\x4b\xf9\xa6\x2e\x7b\x35\x43\xf7\x39\x99\x0f\x3b\x6f\xfd\x1a\x6e\x23\x54\x70\xb5\x1d\x10\x1c\x63\x40\x96\x99\x41\xb6\x96\x0b\x70\x98\xec\x17\xb0\xaa\xdc\x4a\xab\xe8\x3b\xb7\x6b\x00\x1c\x5b\xc3\xe0\xa2\x8b\x7c\x17\xc8\x92\xc9\xb0\x92\xb6\x70\x84\x95\x30".to_vec(), + b"\x4e\x01\xe5\x04\x35\xac\x00\x00\x80".to_vec(), + b"\x62\x01\x41\xb0\x75\x5c\x27\x46\xef\x8a\xe7\x1d\x50\x38\xb2\x13\x33\xe0\x79\x35\x1b\xc2\xb5\x79\x73\xe7\xc2\x6f\xb9\x1a\x8c\x21\x0e\xa9\x54\x17\x6c\x41\xab\xc8\x16\x57\xec\x5e\xeb\x89\x3b\xa9\x90\x8c\xff\x4d\x46\x8b\xf0\xd9\xc0\xd0\x51\xcf\x8b\x88\xf1\x5f\x1e\x9e\xc1\xb9\x1f\xe3\x06\x45\x35\x8a\x47\xe8\x9a\xf2\x4f\x19\x4c\xf8\xce\x68\x1b\x63\x34\x11\x75\xea\xe5\xb1\x0f\x38\xcc\x05\x09\x8b\x3e\x2b\x88\x84\x9d\xc5\x03\xc3\xc0\x90\x32\xe2\x45\x69\xb1\xe5\xf7\x68\x6b\x16\x90\xa0\x40\xe6\x18\x74\xd8\x68\xf3\x34\x38\x99\xf2\x6c\xb7\x1a\x35\x21\xca\x52\x56\x4c\x7f\xb2\xa3\xd5\xb8\x40\x50\x48\x3e\xdc\xdf\x0b\xf5\x54\x5a\x15\x1a\xe2\xc3\xb4\x94\xda\x3f\xb5\x34\xa2\xca\xbc\x2f\xe0\xa4\xe5\x69\xf4\xbf\x62\x4d\x15\x21\x1b\x11\xfc\x39\xaa\x86\x74\x96\x63\xfd\x07\x53\x26\xf6\x34\x72\xeb\x14\x37\x98\x0d\xf4\x68\x91\x2c\x6b\x46\x83\x88\x82\x04\x8b\x9f\xb8\x32\x73\x75\x8b\xf9\xac\x71\x42\xd1\x2d\xb4\x28\x28\xf5\x78\xe0\x32\xf3\xe1\xfc\x43\x6b\xf9\x92\xf7\x48\xfe\x7f\xc0\x17\xbd\xfd\xba\x2f\x58\x6f\xee\x84\x03\x18\xce\xb0\x9d\x8d\xeb\x22\xf1\xfc\xb1\xcf\xff\x2f\xb2\x9f\x6c\xe5\xb4\x69\xdc\xdd\x20\x93\x00\x30\xad\x56\x04\x66\x7e\xa3\x3c\x18\x4b\x43\x66\x00\x27\x1e\x1c\x09\x11\xd8\xf4\x8a\x9e\xc5\x6a\x94\xe5\xae\x0b\x8a\xbe\x84\xda\xe5\x44\x7f\x38\x1c\xe7\xbb\x03\x19\x66\xe1\x5d\x1d\xc1\xbd\x3d\xc6\xb7\xe3\xff\x7f\x8e\xff\x1e\xf6\x9e\x6f\x58\x27\x74\x65\xef\x02\x5d\xa4\xde\x27\x7f\x51\xe3\x4b\x9e\x3f\x79\x83\xbd\x1b\x8f\x0d\x77\xfb\xbc\xc5\x9f\x15\xa7\x4e\x05\x8a\x24\x97\x66\xb2\x7c\xf6\xe1\x84\x54\xdb\x39\x5e\xf6\x1b\x8f\x05\x73\x1d\xb6\x8e\xd7\x09\x9a\xc5\x92\x80".to_vec(), + ]; + + for cur in tests { + let mut pck = H265Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let _ = pck.depacketize(&cur, &mut out, &mut extra)?; + } + + Ok(()) + } + + /// Test aggregated packets match exact payload layout. + /// Verifies AP packet structure: [AP_Header (2)] [Size1 (2)] [NAL1] [Size2 (2)] [NAL2]. + #[test] + fn test_h265_aggregated_exact_layout() -> Result<()> { + let mut packetizer = H265Packetizer::default(); + + // Create two identical simple NAL units + let header = H265NALUHeader::new(0x02, 0x01); // Type 1, layer_id=0, tid=1 + let payload = vec![0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; + + let mut nalu = vec![header.0 as u8, (header.0 >> 8) as u8]; + nalu.extend(&payload); + + // Cache NAL units (parameter sets or similar) + let vps = vec![0x40, 0x01, 0xAA]; + let sps = vec![0x42, 0x01, 0xBB]; + + packetizer.packetize(100, &vps)?; + packetizer.packetize(100, &sps)?; + + // Trigger aggregation + let packets = packetizer.packetize(100, &nalu)?; + + if packets.len() == 1 { + // If AP was created, verify structure + let ap = &packets[0]; + let ap_header = H265NALUHeader::new(ap[0], ap[1]); + + if ap_header.nalu_type() == H265NALU_AGGREGATION_PACKET_TYPE { + // AP structure verified + assert_eq!(ap_header.nalu_type(), H265NALU_AGGREGATION_PACKET_TYPE); + } + } + + Ok(()) + } + + /// Regression test for PACI packet with payload smaller than H265NALU_HEADER_SIZE. + /// Previously this would panic with "index out of bounds" when accessing payload[1]. + #[test] + fn test_paci_short_payload_no_panic() -> Result<()> { + // Create a PACI packet with a single-byte payload + // Type 50 = PACI packet + let paci_packet: Vec = vec![ + 0x64, 0x01, // PayloadHdr: Type=50 (PACI), TID=1 + 0x00, 0x00, // PACI header fields: A=0, cType=0, PHSsize=0 + 0xAB, // Single byte payload (less than H265NALU_HEADER_SIZE) + ]; + + let mut depacketizer = H265Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + + // This should NOT panic - the fix checks payload.len() >= H265NALU_HEADER_SIZE + let result = depacketizer.depacketize(&paci_packet, &mut out, &mut extra); + assert!(result.is_ok()); + + // Output should contain the PACI payload with Annex-B start code + assert!(out.starts_with(ANNEXB_NALUSTART_CODE)); + assert_eq!(out.len(), ANNEXB_NALUSTART_CODE.len() + 1); // start code + 1 byte payload + + Ok(()) + } + } // end integration_tests + + #[test] + fn test_detect_h265_keyframe() { + // Empty / too short payload + assert!(!detect_h265_keyframe(&[])); + assert!(!detect_h265_keyframe(&[0x00])); + + // Single IDR_W_RADL (type 19): nalu_type in bits [14:9] + // type 19 = 0b010011 → byte0 = 0b0_010011_0 = 0x26, byte1 = TID + let idr_w_radl = H265NALUHeader::new_with_type(H265NALU_IDR_W_RADL, 0, 1); + assert!(detect_h265_keyframe(&idr_w_radl.0.to_be_bytes())); + + // Single IDR_N_LP (type 20) + let idr_n_lp = H265NALUHeader::new_with_type(H265NALU_IDR_N_LP, 0, 1); + assert!(detect_h265_keyframe(&idr_n_lp.0.to_be_bytes())); + + // Single CRA (type 21) + let cra = H265NALUHeader::new_with_type(H265NALU_CRA_NUT, 0, 1); + assert!(detect_h265_keyframe(&cra.0.to_be_bytes())); + + // Single BLA_W_LP (type 16) + let bla = H265NALUHeader::new_with_type(H265NALU_BLA_W_LP, 0, 1); + assert!(detect_h265_keyframe(&bla.0.to_be_bytes())); + + // Single non-IRAP (type 1 = TRAIL_R) + let trail_r = H265NALUHeader::new_with_type(1, 0, 1); + assert!(!detect_h265_keyframe(&trail_r.0.to_be_bytes())); + + // Aggregation packet (type 48) with IDR inside + let ap_header = H265NALUHeader::new_with_type(H265NALU_AGGREGATION_PACKET_TYPE, 0, 1); + let idr_header = H265NALUHeader::new_with_type(H265NALU_IDR_W_RADL, 0, 1); + let idr_bytes = idr_header.0.to_be_bytes(); + let mut ap_with_idr = Vec::new(); + ap_with_idr.extend_from_slice(&ap_header.0.to_be_bytes()); // AP header + ap_with_idr.extend_from_slice(&[0x00, 0x03]); // NALU size = 3 + ap_with_idr.extend_from_slice(&idr_bytes); // IDR header + ap_with_idr.push(0x00); // payload byte + assert!(detect_h265_keyframe(&ap_with_idr)); + + // Aggregation packet without IRAP + let non_irap_header = H265NALUHeader::new_with_type(1, 0, 1); + let non_irap_bytes = non_irap_header.0.to_be_bytes(); + let mut ap_no_irap = Vec::new(); + ap_no_irap.extend_from_slice(&ap_header.0.to_be_bytes()); + ap_no_irap.extend_from_slice(&[0x00, 0x03]); + ap_no_irap.extend_from_slice(&non_irap_bytes); + ap_no_irap.push(0x00); + assert!(!detect_h265_keyframe(&ap_no_irap)); + + // FU start fragment with IDR type + let fu_header_bytes = H265NALUHeader::new_with_type(H265NALU_FRAGMENTATION_UNIT_TYPE, 0, 1); + let mut fu_start_idr = Vec::new(); + fu_start_idr.extend_from_slice(&fu_header_bytes.0.to_be_bytes()); + fu_start_idr.push(0x80 | H265NALU_IDR_W_RADL); // S=1, type=19 + fu_start_idr.extend_from_slice(&[0x00, 0x00]); + assert!(detect_h265_keyframe(&fu_start_idr)); + + // FU continuation fragment (S=0) - cannot detect + let mut fu_cont = Vec::new(); + fu_cont.extend_from_slice(&fu_header_bytes.0.to_be_bytes()); + fu_cont.push(H265NALU_IDR_W_RADL); // S=0, type=19 + fu_cont.extend_from_slice(&[0x00, 0x00]); + assert!(!detect_h265_keyframe(&fu_cont)); + + // FU too short (no FU header byte) + assert!(!detect_h265_keyframe(&fu_header_bytes.0.to_be_bytes())); + } + + #[test] + fn packetize_respects_mtu() -> Result<()> { + // 2-byte NAL header (non-parameter-set type) + payload. + let mut nalu = vec![0x02u8, 0x01]; + nalu.extend(std::iter::repeat(0xABu8).take(2000)); + for &mtu in &[100usize, 300, 600, 1200] { + let mut p = H265Packetizer::default(); + let pkts = p.packetize(mtu, &nalu)?; + assert!(!pkts.is_empty(), "H265 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "H265 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} // end test module diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265_profile.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265_profile.rs new file mode 100644 index 000000000..2527a06d0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h265_profile.rs @@ -0,0 +1,374 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// H.265/HEVC profile, tier, and level combination. +/// +/// Represents the three SDP fmtp parameters `profile-id`, `tier-flag`, and `level-id` +/// as defined in RFC 7798 §7.1 and ITU-T H.265 Annex A. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct H265ProfileTierLevel { + profile: H265Profile, + tier: H265Tier, + level: H265Level, +} + +impl H265ProfileTierLevel { + // RFC 7798 §7.1 default values when parameters are absent from SDP: + // profile-id=1 (Main), tier-flag=0 (Main tier), level-id=93 (Level 3.1). + // https://www.rfc-editor.org/rfc/rfc7798#section-7.1 + pub(crate) const FALLBACK: Self = Self { + profile: H265Profile::Main, + tier: H265Tier::Main, + level: H265Level::Level3_1, + }; + + /// Construct a new H265ProfileTierLevel from profile_id, tier_flag, and level_id. + /// + /// Returns `Some(Self)` only if the provided parameters identify valid values. + pub(crate) fn new(profile_id: u8, tier_flag: u8, level_id: u8) -> Option { + let profile = H265Profile::from_id(profile_id)?; + let tier = H265Tier::from_flag(tier_flag)?; + let level = H265Level::from_id(level_id)?; + + Some(Self { + profile, + tier, + level, + }) + } + + /// Parse H265ProfileTierLevel from SDP fmtp parameters. + /// + /// Expects keys: "profile-id", "tier-flag", "level-id" + /// Returns None if any required parameter is missing or invalid. + pub(crate) fn from_fmtp(params: &HashMap) -> Option { + let profile_id: u8 = params.get("profile-id")?.parse().ok()?; + let tier_flag: u8 = params.get("tier-flag")?.parse().ok()?; + let level_id: u8 = params.get("level-id")?.parse().ok()?; + + Self::new(profile_id, tier_flag, level_id) + } + + /// Returns the H.265 profile (Main, Main10, etc.). + pub fn profile(&self) -> H265Profile { + self.profile + } + + /// Returns the H.265 tier (Main or High). + pub fn tier(&self) -> H265Tier { + self.tier + } + + /// Returns the H.265 level (Level 3.1, Level 4.0, etc.). + pub fn level(&self) -> H265Level { + self.level + } + + /// Returns a copy with the level replaced. + pub fn with_level(self, level: H265Level) -> Self { + Self { level, ..self } + } + + /// Returns the numeric profile_id value for SDP serialization. + pub fn profile_id(&self) -> u8 { + self.profile.to_id() + } + + /// Returns the numeric tier_flag value for SDP serialization. + pub fn tier_flag(&self) -> u8 { + self.tier.to_flag() + } + + /// Returns the numeric level_id value for SDP serialization. + pub fn level_id(&self) -> u8 { + self.level.to_id() + } +} + +impl From<(u8, u8, u8)> for H265ProfileTierLevel { + fn from(value: (u8, u8, u8)) -> Self { + Self::new(value.0, value.1, value.2).unwrap_or(Self::FALLBACK) + } +} + +/// H.265 profile as defined in ITU-T H.265 Annex A. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum H265Profile { + /// Main profile (profile_id=1). + Main, + /// Main 10 profile (profile_id=2). + Main10, + /// Main Still Picture profile (profile_id=3). + MainStillPicture, + /// Format Range Extensions profile (profile_id=4). + FormatRangeExtensions, + /// High Throughput profile (profile_id=5). + HighThroughput, + /// Multiview Main profile (profile_id=6). + MultiviewMain, + /// Scalable Main profile (profile_id=7). + ScalableMain, + /// 3D Main profile (profile_id=8). + ThreeDMain, + /// Screen Content Coding profile (profile_id=9). + ScreenContentCoding, + /// Screen Content Coding Extensions profile (profile_id=10). + ScreenContentCodingExtensions, + /// High Throughput Screen Content Coding profile (profile_id=11). + HighThroughputScreenContentCoding, +} + +impl H265Profile { + /// Convert from H.265 profile_id value. + /// + /// See ITU-T H.265 (04/2013) Annex A for profile definitions. + fn from_id(profile_id: u8) -> Option { + match profile_id { + 1 => Some(Self::Main), + 2 => Some(Self::Main10), + 3 => Some(Self::MainStillPicture), + 4 => Some(Self::FormatRangeExtensions), + 5 => Some(Self::HighThroughput), + 6 => Some(Self::MultiviewMain), + 7 => Some(Self::ScalableMain), + 8 => Some(Self::ThreeDMain), + 9 => Some(Self::ScreenContentCoding), + 10 => Some(Self::ScreenContentCodingExtensions), + 11 => Some(Self::HighThroughputScreenContentCoding), + _ => None, + } + } + + /// Convert to H.265 profile_id value. + pub fn to_id(self) -> u8 { + match self { + Self::Main => 1, + Self::Main10 => 2, + Self::MainStillPicture => 3, + Self::FormatRangeExtensions => 4, + Self::HighThroughput => 5, + Self::MultiviewMain => 6, + Self::ScalableMain => 7, + Self::ThreeDMain => 8, + Self::ScreenContentCoding => 9, + Self::ScreenContentCodingExtensions => 10, + Self::HighThroughputScreenContentCoding => 11, + } + } +} + +/// H.265 tier (Main or High). +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum H265Tier { + /// Main tier (tier_flag=0). + Main, + /// High tier (tier_flag=1). + High, +} + +impl H265Tier { + /// Convert from H.265 tier_flag value. + /// + /// 0 = Main tier, 1 = High tier + fn from_flag(tier_flag: u8) -> Option { + match tier_flag { + 0 => Some(Self::Main), + 1 => Some(Self::High), + _ => None, + } + } + + /// Convert to H.265 tier_flag value. + pub fn to_flag(self) -> u8 { + match self { + Self::Main => 0, + Self::High => 1, + } + } +} + +/// H.265 level as defined in ITU-T H.265 Annex A. +/// +/// Level IDs are 30× the level number (e.g., Level 3.1 = 93). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[repr(u8)] +#[rustfmt::skip] +pub enum H265Level { + /// Level 1.0 (level_id=30). + Level1 = 30_u8, + /// Level 2.0 (level_id=60). + Level2 = 60_u8, + /// Level 2.1 (level_id=63). + Level2_1 = 63_u8, + /// Level 3.0 (level_id=90). + Level3 = 90_u8, + /// Level 3.1 (level_id=93). + Level3_1 = 93_u8, + /// Level 4.0 (level_id=120). + Level4 = 120_u8, + /// Level 4.1 (level_id=123). + Level4_1 = 123_u8, + /// Level 5.0 (level_id=150). + Level5 = 150_u8, + /// Level 5.1 (level_id=153). + Level5_1 = 153_u8, + /// Level 5.2 (level_id=156). + Level5_2 = 156_u8, + /// Level 6.0 (level_id=180). + Level6 = 180_u8, + /// Level 6.1 (level_id=183). + Level6_1 = 183_u8, + /// Level 6.2 (level_id=186). + Level6_2 = 186_u8, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(H265ProfileTierLevel, H265Profile, H265Tier, H265Level); + +impl H265Level { + /// Returns the ordinal position (0-12) representing capability order. + pub fn ordinal(self) -> usize { + match self { + Self::Level1 => 0, + Self::Level2 => 1, + Self::Level2_1 => 2, + Self::Level3 => 3, + Self::Level3_1 => 4, + Self::Level4 => 5, + Self::Level4_1 => 6, + Self::Level5 => 7, + Self::Level5_1 => 8, + Self::Level5_2 => 9, + Self::Level6 => 10, + Self::Level6_1 => 11, + Self::Level6_2 => 12, + } + } + + /// Convert from H.265 level_id value. + fn from_id(level_id: u8) -> Option { + use H265Level::*; + match level_id { + x if (Level1 as u8) == x => Some(Level1), + x if (Level2 as u8) == x => Some(Level2), + x if (Level2_1 as u8) == x => Some(Level2_1), + x if (Level3 as u8) == x => Some(Level3), + x if (Level3_1 as u8) == x => Some(Level3_1), + x if (Level4 as u8) == x => Some(Level4), + x if (Level4_1 as u8) == x => Some(Level4_1), + x if (Level5 as u8) == x => Some(Level5), + x if (Level5_1 as u8) == x => Some(Level5_1), + x if (Level5_2 as u8) == x => Some(Level5_2), + x if (Level6 as u8) == x => Some(Level6), + x if (Level6_1 as u8) == x => Some(Level6_1), + x if (Level6_2 as u8) == x => Some(Level6_2), + _ => None, + } + } + + /// Convert to H.265 level_id value. + pub fn to_id(self) -> u8 { + self as u8 + } +} + +impl PartialOrd for H265Level { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for H265Level { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.ordinal().cmp(&other.ordinal()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_h265_profile_tier_level_new() { + // Valid combination + let ptl = H265ProfileTierLevel::new(1, 0, 93); + assert!(ptl.is_some()); + let ptl = ptl.unwrap(); + assert_eq!(ptl.profile(), H265Profile::Main); + assert_eq!(ptl.tier(), H265Tier::Main); + assert_eq!(ptl.level(), H265Level::Level3_1); + + // Invalid profile_id + let ptl = H265ProfileTierLevel::new(99, 0, 93); + assert!(ptl.is_none()); + + // Invalid tier_flag + let ptl = H265ProfileTierLevel::new(1, 5, 93); + assert!(ptl.is_none()); + + // Invalid level_id + let ptl = H265ProfileTierLevel::new(1, 0, 255); + assert!(ptl.is_none()); + } + + #[test] + fn test_h265_profile_from_id() { + assert_eq!(H265Profile::from_id(1), Some(H265Profile::Main)); + assert_eq!(H265Profile::from_id(2), Some(H265Profile::Main10)); + assert_eq!(H265Profile::from_id(99), None); + } + + #[test] + fn test_h265_tier_from_flag() { + assert_eq!(H265Tier::from_flag(0), Some(H265Tier::Main)); + assert_eq!(H265Tier::from_flag(1), Some(H265Tier::High)); + assert_eq!(H265Tier::from_flag(2), None); + } + + #[test] + fn test_h265_level_from_id() { + assert_eq!(H265Level::from_id(30), Some(H265Level::Level1)); + assert_eq!(H265Level::from_id(93), Some(H265Level::Level3_1)); + assert_eq!(H265Level::from_id(186), Some(H265Level::Level6_2)); + assert_eq!(H265Level::from_id(255), None); + } + + #[test] + fn test_h265_level_ordering() { + assert!(H265Level::Level1 < H265Level::Level2); + assert!(H265Level::Level3_1 < H265Level::Level4); + assert!(H265Level::Level6_1 < H265Level::Level6_2); + assert_eq!(H265Level::Level3_1.ordinal(), 4); + } + + #[test] + fn test_h265_profile_tier_level_from_fmtp() { + let mut params = HashMap::new(); + params.insert("profile-id".to_string(), "1".to_string()); + params.insert("tier-flag".to_string(), "0".to_string()); + params.insert("level-id".to_string(), "93".to_string()); + + let ptl = H265ProfileTierLevel::from_fmtp(¶ms); + assert!(ptl.is_some()); + let ptl = ptl.unwrap(); + assert_eq!(ptl.profile(), H265Profile::Main); + assert_eq!(ptl.tier(), H265Tier::Main); + assert_eq!(ptl.level(), H265Level::Level3_1); + + // Missing parameter + let mut params = HashMap::new(); + params.insert("profile-id".to_string(), "1".to_string()); + params.insert("tier-flag".to_string(), "0".to_string()); + let ptl = H265ProfileTierLevel::from_fmtp(¶ms); + assert!(ptl.is_none()); + } + + #[test] + fn test_fallback_values() { + let fallback = H265ProfileTierLevel::FALLBACK; + assert_eq!(fallback.profile(), H265Profile::Main); + assert_eq!(fallback.tier(), H265Tier::Main); + assert_eq!(fallback.level(), H265Level::Level3_1); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266.rs new file mode 100644 index 000000000..f864ebfdb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266.rs @@ -0,0 +1,2580 @@ +//! H266 (VVC) RTP packetization per RFC 9328. +//! +//! Wire formats implemented (RFC 9328 §4.3): +//! - Single NAL unit packets (§4.3.1). +//! - Aggregation Packets, NAL type 28 (§4.3.2). +//! - Fragmentation Units, NAL type 29 (§4.3.3), FU header +//! `S(1) | E(1) | P(1) | FuType(5)`. +//! +//! The two-byte NAL unit header is `F(1) | Z(1) | LayerId(6) || Type(5) | +//! TID(3)` (§1.1.4) — the NAL type lives in the SECOND byte, upper 5 bits. +//! Parameter sets: VPS=14, SPS=15, PPS=16. Skipped on send: AUD=20, +//! Filler(FD)=25. IRAP (keyframe) NAL types: IDR_W_RADL=7, IDR_N_LP=8, CRA=9. +//! +//! Profile/tier/level fmtp negotiation (RFC 9328 §7.2) is implemented in +//! `h266_profile.rs` and the format layer. DONL (`sprop-max-don-diff`, +//! §7.2) is supported for send and receive via `with_donl`, wired from SDP +//! negotiation. + +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; +use tracing::warn; + +pub static ANNEXB_NALUSTART_CODE: &[u8] = &[0x00, 0x00, 0x00, 0x01]; + +/// H266 NAL unit header size (2 bytes). +const H266NALU_HEADER_SIZE: usize = 2; +/// FU header size (1 byte). +const H266FRAGMENTATION_UNIT_HEADER_SIZE: usize = 1; + +// H266 NAL unit types (ITU-T H.266 Table 5). +const H266NALU_IDR_W_RADL: u8 = 7; +const H266NALU_IDR_N_LP: u8 = 8; +const H266NALU_CRA: u8 = 9; +#[allow(dead_code)] // RFC 9328 NAL type table completeness; referenced by tests. +const H266NALU_GDR: u8 = 10; +const H266NALU_VPS_NALU_TYPE: u8 = 14; +const H266NALU_SPS_NALU_TYPE: u8 = 15; +const H266NALU_PPS_NALU_TYPE: u8 = 16; +const H266NALU_AUD_NALU_TYPE: u8 = 20; +const H266NALU_FILLER_NALU_TYPE: u8 = 25; // FD_NUT + +// RFC 9328 RTP packet types. +const H266NALU_AGGREGATION_PACKET_TYPE: u8 = 28; +const H266NALU_FRAGMENTATION_UNIT_TYPE: u8 = 29; + +const MAX_PACKET_SIZE: usize = 1200; +const MIN_FU_PAYLOAD: usize = 1; +const MIN_MTU: usize = H266NALU_HEADER_SIZE + H266FRAGMENTATION_UNIT_HEADER_SIZE + MIN_FU_PAYLOAD; + +/// Codec extra reported alongside depacketized H266 data. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct H266CodecExtra { + /// Whether the depacketized data contains an IRAP NAL (keyframe). + pub is_keyframe: bool, +} + +/// H266 NAL unit header (RFC 9328 §1.1.4). +/// +/// ```text +/// +---------------+---------------+ +/// |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| +/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/// |F|Z| LayerID | Type | TID | +/// +---------------+---------------+ +/// ``` +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub struct H266NALUHeader(pub u16); + +impl H266NALUHeader { + pub fn new(high_byte: u8, low_byte: u8) -> Self { + H266NALUHeader(((high_byte as u16) << 8) | low_byte as u16) + } + + /// Construct a header from NAL type, layer id and temporal id. + pub fn new_with_type(typ: u8, layer_id: u8, tid: u8) -> Self { + let b0 = layer_id & 0b0011_1111; + let b1 = ((typ & 0b1_1111) << 3) | (tid & 0b111); + Self::new(b0, b1) + } + + /// Forbidden zero bit. + pub fn f(&self) -> bool { + (self.0 >> 15) & 0b1 != 0 + } + + /// nuh_reserved_zero_bit (Z). + pub fn z(&self) -> bool { + (self.0 >> 14) & 0b1 != 0 + } + + /// nuh_layer_id (6 bits of the first byte). + pub fn layer_id(&self) -> u8 { + ((self.0 >> 8) & 0b0011_1111) as u8 + } + + /// NAL unit type — upper 5 bits of the SECOND byte. + pub fn nalu_type(&self) -> u8 { + ((self.0 >> 3) & 0b1_1111) as u8 + } + + /// nuh_temporal_id_plus1 — lower 3 bits of the second byte. + pub fn tid(&self) -> u8 { + (self.0 & 0b111) as u8 + } + + /// IRAP picture (keyframe): IDR_W_RADL, IDR_N_LP, CRA. + /// Note: GDR (gradual decoding refresh) is a recovery point but not an + /// instantaneous keyframe, so it is deliberately excluded. + pub fn is_irap(&self) -> bool { + matches!( + self.nalu_type(), + H266NALU_IDR_W_RADL | H266NALU_IDR_N_LP | H266NALU_CRA + ) + } + + pub fn is_aggregation_packet(&self) -> bool { + self.nalu_type() == H266NALU_AGGREGATION_PACKET_TYPE + } + + pub fn is_fragmentation_unit(&self) -> bool { + self.nalu_type() == H266NALU_FRAGMENTATION_UNIT_TYPE + } +} + +/// H266 FU header: `S(1) | E(1) | P(1) | FuType(5)` (RFC 9328 §4.3.3). +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub struct H266FragmentationUnitHeader(pub u8); + +impl H266FragmentationUnitHeader { + pub fn new(start: bool, end: bool, last_of_picture: bool, fu_type: u8) -> Self { + let mut b = fu_type & 0b1_1111; + if start { + b |= 0b1000_0000; + } + if end { + b |= 0b0100_0000; + } + if last_of_picture { + b |= 0b0010_0000; + } + H266FragmentationUnitHeader(b) + } + + /// Start of fragmented NAL unit. + pub fn s(&self) -> bool { + self.0 & 0b1000_0000 != 0 + } + + /// End of fragmented NAL unit. + pub fn e(&self) -> bool { + self.0 & 0b0100_0000 != 0 + } + + /// Last FU of the last VCL NAL of the picture. + #[allow(dead_code)] // RFC 9328 FU header accessor; exercised by tests. + pub fn p(&self) -> bool { + self.0 & 0b0010_0000 != 0 + } + + /// Original NAL unit type. + pub fn fu_type(&self) -> u8 { + self.0 & 0b1_1111 + } +} + +/// Detect whether an RTP payload contains an H266 keyframe (IRAP). +/// +/// Note: assumes payloads without DONL fields. When `sprop-max-don-diff > 0` +/// is negotiated, Aggregation Packet payloads carry a 16-bit DONL before the +/// first unit, which this scan does not account for (single NAL and FU +/// detection are unaffected — the inspected bytes precede the DONL). +pub fn detect_h266_keyframe(payload: &[u8]) -> bool { + if payload.len() < H266NALU_HEADER_SIZE { + return false; + } + + let header = H266NALUHeader::new(payload[0], payload[1]); + match header.nalu_type() { + H266NALU_AGGREGATION_PACKET_TYPE => { + // Check all aggregated NAL units. + let mut offset = H266NALU_HEADER_SIZE; + while offset + 2 <= payload.len() { + let nalu_size = ((payload[offset] as usize) << 8) | payload[offset + 1] as usize; + offset += 2; + if offset + nalu_size > payload.len() || nalu_size < H266NALU_HEADER_SIZE { + break; + } + let inner = H266NALUHeader::new(payload[offset], payload[offset + 1]); + if inner.is_irap() { + return true; + } + offset += nalu_size; + } + false + } + H266NALU_FRAGMENTATION_UNIT_TYPE => { + if payload.len() < H266NALU_HEADER_SIZE + 1 { + return false; + } + let fu = H266FragmentationUnitHeader(payload[2]); + let t = fu.fu_type(); + fu.s() && (H266NALU_IDR_W_RADL..=H266NALU_CRA).contains(&t) + } + _ => header.is_irap(), + } +} + +/// H266 packetizer (RFC 9328): Single NAL / AP (28) / FU (29). +#[derive(Debug)] +pub struct H266Packetizer { + /// Cached parameter sets, sent as an AP before the next non-param NAL. + vps_nalu: Option>, + sps_nalu: Option>, + pps_nalu: Option>, + /// DONL counter (RFC 9328 §4.3): present when sprop-max-don-diff > 0. + donl: Option, + /// Reusable packet build buffer. + pkt_buf: Vec, +} + +impl Default for H266Packetizer { + fn default() -> Self { + H266Packetizer { + vps_nalu: None, + sps_nalu: None, + pps_nalu: None, + donl: None, + pkt_buf: Vec::with_capacity(MAX_PACKET_SIZE), + } + } +} + +impl H266Packetizer { + /// Enable/disable DONL field emission (RFC 9328 §4.3, sprop-max-don-diff > 0). + pub fn with_donl(&mut self, value: bool) { + self.donl = if value { Some(0) } else { None }; + } + + fn increment_donl(&mut self) { + if let Some(ref mut donl) = self.donl { + *donl = donl.wrapping_add(1); + } + } + + fn increment_donl_by(&mut self, n: u16) { + if let Some(ref mut donl) = self.donl { + *donl = donl.wrapping_add(n); + } + } + + /// Build an AP (type 28) from NAL units into `pkt_buf`. + /// Returns false if the AP would exceed the MTU (or a unit cannot be + /// represented with a 16-bit length). + fn build_ap_packet( + nal_units: &[&[u8]], + donl: Option, + buf: &mut Vec, + mtu: usize, + ) -> bool { + // AP PayloadHdr (RFC 9328 §4.3.2): F MUST be 0 if the F bit of each + // aggregated NAL unit is 0 (OR otherwise); Z is reserved (0); + // LayerId and TID MUST be the lowest LayerId/TID of all the + // aggregated NAL units. + let mut f_bit = false; + let mut min_layer = 0b0011_1111u8; + let mut min_tid = 0b111u8; + for n in nal_units { + let h = H266NALUHeader::new(n[0], n[1]); + f_bit |= h.f(); + min_layer = min_layer.min(h.layer_id()); + min_tid = min_tid.min(h.tid()); + } + // TID is nuh_temporal_id_plus1: never emit 0. + let tid = min_tid.max(1); + + // Aggregation unit lengths are 16-bit (RFC 9328 §4.3.2). + if nal_units.iter().any(|n| n.len() > u16::MAX as usize) { + return false; + } + + // DONL (2 bytes) precedes the first aggregation unit when enabled + // (RFC 9328 §4.3.2). + let donl_overhead = if donl.is_some() { 2 } else { 0 }; + let total: usize = H266NALU_HEADER_SIZE + + donl_overhead + + nal_units.iter().map(|n| 2 + n.len()).sum::(); + if total > mtu { + return false; + } + + buf.clear(); + let ap_hdr = + H266NALUHeader::new_with_type(H266NALU_AGGREGATION_PACKET_TYPE, min_layer, tid); + let [b0, b1] = ap_hdr.0.to_be_bytes(); + buf.push(b0 | ((f_bit as u8) << 7)); // F | Z=0 | LayerId + buf.push(b1); + + // Write DONL for the first aggregation unit if present (RFC 9328 §4.3.2). + if let Some(donl_value) = donl { + buf.extend_from_slice(&donl_value.to_be_bytes()); + } + + for n in nal_units { + buf.extend_from_slice(&(n.len() as u16).to_be_bytes()); + buf.extend_from_slice(n); + } + true + } + + /// Find the next Annex-B start code at/after `start`. + /// Returns (offset, length) or (-1, -1). + fn next_start_code(payload: &[u8], start: usize) -> (isize, isize) { + let mut i = start; + while i + 3 <= payload.len() { + if payload[i] == 0 && payload[i + 1] == 0 { + if payload[i + 2] == 1 { + return (i as isize, 3); + } + if i + 4 <= payload.len() && payload[i + 2] == 0 && payload[i + 3] == 1 { + return (i as isize, 4); + } + } + i += 1; + } + (-1, -1) + } + + /// Emit one NAL unit as RTP payload(s): + /// param sets -> cached -> AP(28); small -> Single NAL; large -> FU(29). + fn emit_nalu(&mut self, nalu: &[u8], mtu: usize, out: &mut Vec>) { + if mtu == 0 || nalu.len() < H266NALU_HEADER_SIZE { + return; + } + + let original_hdr = H266NALUHeader::new(nalu[0], nalu[1]); + let original_type = original_hdr.nalu_type(); + + // Ignore AUD/filler. + if original_type == H266NALU_AUD_NALU_TYPE || original_type == H266NALU_FILLER_NALU_TYPE { + return; + } + + // NAL types 28 (AP) and 29 (FU) are reserved for RTP packetization + // (RFC 9328 §4.3) and must never appear in an input bitstream. + if original_type == H266NALU_AGGREGATION_PACKET_TYPE + || original_type == H266NALU_FRAGMENTATION_UNIT_TYPE + { + warn!( + "H266-PKT skipping reserved NAL type {} in input bitstream", + original_type + ); + return; + } + + // Cache parameter sets; send them as AP before the next other NALU. + match original_type { + H266NALU_VPS_NALU_TYPE => { + self.vps_nalu = Some(nalu.to_vec()); + return; + } + H266NALU_SPS_NALU_TYPE => { + self.sps_nalu = Some(nalu.to_vec()); + return; + } + H266NALU_PPS_NALU_TYPE => { + self.pps_nalu = Some(nalu.to_vec()); + return; + } + _ => {} + } + + // Emit cached parameter sets as an AP (or singles fallback). + // Note: VPS is optional in VVC streams — SPS+PPS is enough to emit. + if self.sps_nalu.is_some() && self.pps_nalu.is_some() { + let mut nal_units_arr: [&[u8]; 3] = [&[], &[], &[]]; + let mut count = 0; + if let Some(vps) = &self.vps_nalu { + nal_units_arr[count] = vps; + count += 1; + } + if let Some(sps) = &self.sps_nalu { + nal_units_arr[count] = sps; + count += 1; + } + if let Some(pps) = &self.pps_nalu { + nal_units_arr[count] = pps; + count += 1; + } + let nal_units = &nal_units_arr[..count]; + + if count >= 2 && Self::build_ap_packet(nal_units, self.donl, &mut self.pkt_buf, mtu) { + out.push(self.pkt_buf.clone()); + // One DONL per aggregated NAL unit. + self.increment_donl_by(count as u16); + } else { + // Fall back to individual Single NAL packets. + let fallback_donl_overhead = if self.donl.is_some() { 2 } else { 0 }; + for nal_unit in nal_units { + if nal_unit.len() + fallback_donl_overhead <= mtu { + if let Some(ref mut donl_value) = self.donl { + self.pkt_buf.clear(); + self.pkt_buf + .extend_from_slice(&nal_unit[..H266NALU_HEADER_SIZE]); + self.pkt_buf.extend_from_slice(&donl_value.to_be_bytes()); + self.pkt_buf + .extend_from_slice(&nal_unit[H266NALU_HEADER_SIZE..]); + out.push(self.pkt_buf.clone()); + *donl_value = donl_value.wrapping_add(1); + } else { + out.push(nal_unit.to_vec()); + } + } + } + } + + self.vps_nalu = None; + self.sps_nalu = None; + self.pps_nalu = None; + } + + // Single NAL unit packet (RFC 9328 §4.3.1). + // With DONL enabled, 2 extra bytes follow the NAL header. + let donl_overhead = if self.donl.is_some() { 2 } else { 0 }; + if nalu.len() + donl_overhead <= mtu { + if let Some(donl_value) = self.donl { + self.pkt_buf.clear(); + self.pkt_buf + .extend_from_slice(&nalu[..H266NALU_HEADER_SIZE]); + self.pkt_buf.extend_from_slice(&donl_value.to_be_bytes()); + self.pkt_buf + .extend_from_slice(&nalu[H266NALU_HEADER_SIZE..]); + out.push(self.pkt_buf.clone()); + self.increment_donl(); + } else { + out.push(nalu.to_vec()); + } + return; + } + + // Fragmentation Units (RFC 9328 §4.3.3). + const FU_OVERHEAD: usize = H266NALU_HEADER_SIZE + H266FRAGMENTATION_UNIT_HEADER_SIZE; + if nalu.len() <= H266NALU_HEADER_SIZE { + return; + } + + // FU payload header: copy of the original header with Type replaced + // by 29. Type lives in the SECOND byte, bits 7..3. + let fu_b0 = nalu[0]; + let fu_b1 = (H266NALU_FRAGMENTATION_UNIT_TYPE << 3) | (nalu[1] & 0b111); + + let payload = &nalu[H266NALU_HEADER_SIZE..]; + let effective_mtu = mtu.min(MAX_PACKET_SIZE); + // The first fragment also carries the DONL field when enabled + // (RFC 9328 §4.3.3). + if effective_mtu <= FU_OVERHEAD + donl_overhead { + return; + } + let first_max = effective_mtu - FU_OVERHEAD - donl_overhead; + let max_fragment = effective_mtu - FU_OVERHEAD; + let donl_bytes = self.donl.map(u16::to_be_bytes); + + let mut offset = 0; + while offset < payload.len() { + let first = offset == 0; + let remaining = payload.len() - offset; + let budget = if first { first_max } else { max_fragment }; + let take = remaining.min(budget); + let end = offset + take == payload.len(); + + // P bit (last FU of the last VCL NAL of a picture): RFC 9328 + // §4.3.3 only mandates P=0 when the FU is NOT that last + // fragment; always sending 0 is compliant. Not tracked here. + let fu_hdr = H266FragmentationUnitHeader::new(first, end, false, original_type); + + self.pkt_buf.clear(); + self.pkt_buf.push(fu_b0); + self.pkt_buf.push(fu_b1); + self.pkt_buf.push(fu_hdr.0); + if first { + if let Some(ref b) = donl_bytes { + self.pkt_buf.extend_from_slice(b); + } + } + self.pkt_buf + .extend_from_slice(&payload[offset..offset + take]); + out.push(self.pkt_buf.clone()); + offset += take; + } + + // One DONL per NAL unit (not per fragment). + self.increment_donl(); + } +} + +impl Packetizer for H266Packetizer { + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() { + return Ok(vec![]); + } + + let mtu = match mtu { + 0 => { + warn!("MTU is 0, cannot packetize H.266"); + return Ok(vec![]); + } + mtu if mtu > MAX_PACKET_SIZE => { + warn!( + "MTU {} exceeds MAX_PACKET_SIZE {}, clamping", + mtu, MAX_PACKET_SIZE + ); + MAX_PACKET_SIZE + } + mtu if mtu < MIN_MTU => { + warn!("MTU {} too small for H.266 fragmentation", mtu); + return Ok(vec![]); + } + mtu => mtu, + }; + + // Pre-allocate with estimated capacity to avoid reallocations + // in the hot path (one entry per expected RTP packet). + let estimated_packets = payload + .len() + .checked_div(mtu.saturating_sub(3)) + .unwrap_or(1) + .saturating_add(4); + let mut packets = Vec::with_capacity(estimated_packets); + + // No start codes: treat the whole payload as one NAL unit. + let (mut next_start, mut next_len) = Self::next_start_code(payload, 0); + if next_start == -1 { + self.emit_nalu(payload, mtu, &mut packets); + return Ok(packets); + } + + // Walk the Annex-B bytestream. + while next_start != -1 { + let nalu_start = (next_start + next_len) as usize; + let (s2, l2) = Self::next_start_code(payload, nalu_start); + next_start = s2; + next_len = l2; + + if next_start != -1 { + self.emit_nalu(&payload[nalu_start..next_start as usize], mtu, &mut packets); + } else { + self.emit_nalu(&payload[nalu_start..], mtu, &mut packets); + } + } + + Ok(packets) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +/// H266 depacketizer (RFC 9328): Single NAL / AP (28) / FU (29). +/// Emits Annex-B (start-code prefixed) NAL units. +#[derive(Debug, Default)] +pub struct H266Depacketizer { + /// Reassembly buffer for fragmentation units. + fu_buffer: Option>, + /// Whether DONL fields are present (sprop-max-don-diff > 0). + might_need_donl: bool, +} + +impl H266Depacketizer { + /// Enable/disable DONL field parsing (RFC 9328 §4.3, sprop-max-don-diff > 0). + pub fn with_donl(&mut self, value: bool) { + self.might_need_donl = value; + } +} + +impl Depacketizer for H266Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + let estimated_packets = (packets_size / 1200).saturating_add(1); + Some(packets_size.saturating_add(4usize.saturating_mul(estimated_packets))) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + codec_extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + if packet.len() <= H266NALU_HEADER_SIZE { + return Err(PacketError::ErrShortPacket); + } + + let header = H266NALUHeader::new(packet[0], packet[1]); + if header.f() { + return Err(PacketError::ErrH266CorruptedPacket); + } + if header.z() { + // nuh_reserved_zero_bit: required to be 0 (RFC 9328 §1.1.4, + // reserved for future extensions). Tolerated and ignored here. + warn!("H266-DEPKT NAL with reserved Z bit set (ignored)"); + } + + let mark_keyframe = |codec_extra: &mut CodecExtra, irap: bool| { + let is_keyframe = if let CodecExtra::H266(e) = codec_extra { + irap | e.is_keyframe + } else { + irap + }; + *codec_extra = CodecExtra::H266(H266CodecExtra { is_keyframe }); + }; + + if header.is_fragmentation_unit() { + if packet.len() < H266NALU_HEADER_SIZE + 1 { + return Err(PacketError::ErrShortPacket); + } + let fu_header = H266FragmentationUnitHeader(packet[2]); + + // The first FU of a series carries the DONL field when enabled + // (RFC 9328 §4.3.3) — skip it. + let payload_start = if fu_header.s() && self.might_need_donl { + if packet.len() < H266NALU_HEADER_SIZE + 1 + 2 { + return Err(PacketError::ErrShortPacket); + } + H266NALU_HEADER_SIZE + 1 + 2 + } else { + H266NALU_HEADER_SIZE + 1 + }; + let fu_payload = &packet[payload_start..]; + + if fu_header.s() { + match &mut self.fu_buffer { + Some(buf) => buf.clear(), + None => self.fu_buffer = Some(Vec::with_capacity(128 * 1024)), + } + } + + if let Some(ref mut buf) = self.fu_buffer { + buf.extend_from_slice(fu_payload); + } + + if fu_header.e() { + if let Some(ref buf) = self.fu_buffer { + // Rebuild the original NAL header: byte0 unchanged, + // byte1 = orig type in bits 7..3 + original TID. + let orig_type = fu_header.fu_type(); + let orig_b0 = packet[0]; + let orig_b1 = (orig_type << 3) | (packet[1] & 0b111); + let irap = matches!( + orig_type, + H266NALU_IDR_W_RADL | H266NALU_IDR_N_LP | H266NALU_CRA + ); + mark_keyframe(codec_extra, irap); + + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + out.push(orig_b0); + out.push(orig_b1); + out.extend_from_slice(buf); + } + } + Ok(()) + } else if header.is_aggregation_packet() { + let mut offset = H266NALU_HEADER_SIZE; + let mut unit_count = 0; + + // DONL precedes the first aggregation unit when enabled + // (RFC 9328 §4.3.2). + if self.might_need_donl { + if packet.len() < offset + 2 { + return Err(PacketError::ErrShortPacket); + } + offset += 2; + } + + // Remember where this AP's output started so a malformed + // packet doesn't leave partial NAL units in `out`. + let out_start = out.len(); + + while offset < packet.len() { + if offset + 2 > packet.len() { + out.truncate(out_start); + return Err(PacketError::ErrShortPacket); + } + let nalu_size = ((packet[offset] as usize) << 8) | (packet[offset + 1] as usize); + offset += 2; + if offset + nalu_size > packet.len() || nalu_size < H266NALU_HEADER_SIZE { + // Length field inconsistent with the remaining bytes. + out.truncate(out_start); + return Err(PacketError::ErrShortPacket); + } + let nalu = &packet[offset..offset + nalu_size]; + offset += nalu_size; + unit_count += 1; + + let inner = H266NALUHeader::new(nalu[0], nalu[1]); + + // NAL types 28 (AP) and 29 (FU) are RTP-only constructs and + // must never appear inside an aggregation packet + // (RFC 9328 §4.3.2). + if inner.is_aggregation_packet() || inner.is_fragmentation_unit() { + out.truncate(out_start); + return Err(PacketError::ErrH266CorruptedPacket); + } + + mark_keyframe(codec_extra, inner.is_irap()); + + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + out.extend_from_slice(nalu); + } + + if unit_count < 2 { + return Err(PacketError::ErrShortPacket); + } + + Ok(()) + } else { + // Single NAL unit packet. + mark_keyframe(codec_extra, header.is_irap()); + + out.extend_from_slice(ANNEXB_NALUSTART_CODE); + if self.might_need_donl { + // DONL (2 bytes) sits between the NAL header and the payload + // (RFC 9328 §4.3.1) — strip it. + if packet.len() < H266NALU_HEADER_SIZE + 2 + 1 { + return Err(PacketError::ErrShortPacket); + } + out.extend_from_slice(&packet[..H266NALU_HEADER_SIZE]); + out.extend_from_slice(&packet[H266NALU_HEADER_SIZE + 2..]); + } else { + out.extend_from_slice(packet); + } + Ok(()) + } + } + + fn is_partition_head(&self, payload: &[u8]) -> bool { + if payload.len() < H266NALU_HEADER_SIZE { + return false; + } + let header = H266NALUHeader::new(payload[0], payload[1]); + if header.f() { + return true; + } + // Single NAL and AP packets are partition heads. + if !header.is_fragmentation_unit() { + return true; + } + // FU: head only when the S bit is set. + if payload.len() < H266NALU_HEADER_SIZE + 1 { + return false; + } + H266FragmentationUnitHeader(payload[2]).s() + } + + fn is_partition_tail(&self, marker: bool, payload: &[u8]) -> bool { + if payload.len() < H266NALU_HEADER_SIZE { + return false; + } + let header = H266NALUHeader::new(payload[0], payload[1]); + if header.is_fragmentation_unit() { + if payload.len() < H266NALU_HEADER_SIZE + 1 { + return false; + } + return H266FragmentationUnitHeader(payload[2]).e(); + } + marker + } +} + +#[cfg(test)] +mod test { + use super::*; + + type Result = std::result::Result; + + // ========== Shared Test Utilities ========== + + /// Build a 2-byte H.266 NAL header: type in byte 1 bits 7..3. + fn hdr(typ: u8, layer: u8, tid: u8) -> [u8; 2] { + let h = H266NALUHeader::new_with_type(typ, layer, tid); + h.0.to_be_bytes() + } + + /// Annex-B encode a list of NAL units with 4-byte start codes. + fn annexb(nals: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for n in nals { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(n); + } + out + } + + /// Make a NAL unit of `len` total bytes with the given type. + fn make_nal(typ: u8, len: usize) -> Vec { + assert!(len >= 2); + let mut n = hdr(typ, 0, 1).to_vec(); + for i in 0..len - 2 { + n.push((i % 251) as u8); + } + n + } + + /// Make a NAL unit of `len` total bytes with type, layer id and tid. + fn make_nal_lt(typ: u8, len: usize, layer: u8, tid: u8) -> Vec { + assert!(len >= 2); + let mut n = hdr(typ, layer, tid).to_vec(); + for i in 0..len - 2 { + n.push((i % 251) as u8); + } + n + } + + /// Reconstruct the original NAL unit from a sequence of FU packets. + /// Assumes all packets are FU (type 29) and are consecutive. + fn reconstruct_from_fu_packets(packets: &[Vec]) -> Vec { + let mut out = Vec::new(); + let mut started = false; + + for pkt in packets { + assert!(pkt.len() >= 3); + let h = H266NALUHeader::new(pkt[0], pkt[1]); + assert!(h.is_fragmentation_unit()); + + let fu = H266FragmentationUnitHeader(pkt[2]); + + if fu.s() { + // Rebuild the original 2-byte NAL header: byte 0 unchanged, + // byte 1 = fu_type in bits 7..3 + original TID. + out.push(pkt[0]); + out.push((fu.fu_type() << 3) | (pkt[1] & 0b111)); + started = true; + } + + assert!(started, "FU sequence must start with S=1"); + out.extend_from_slice(&pkt[3..]); + } + + out + } + + /// Depacketize a list of RTP payloads, returning the Annex-B output + /// and whether any keyframe was flagged. + fn depacketize_all(packets: &[Vec]) -> Result<(Vec, bool)> { + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut keyframe = false; + for p in packets { + let mut extra = CodecExtra::None; + depack.depacketize(p, &mut out, &mut extra)?; + if let CodecExtra::H266(e) = extra { + keyframe |= e.is_keyframe; + } + } + Ok((out, keyframe)) + } + + /// RFC 9328 bitfield correctness tests. + mod header_tests { + use super::*; + + /// Test H.266 NAL Unit header parsing and field extraction. + /// Verifies F bit, NAL type, layer_id, tid, and packet type + /// detection (AP/FU). + #[test] + fn test_h266_nalu_header() -> Result<()> { + #[derive(Default)] + struct TestType { + raw_header: &'static [u8], + + fbit: bool, + typ: u8, + layer_id: u8, + tid: u8, + + is_ap: bool, + is_fu: bool, + } + + let tests = vec![ + // fbit (bit 7 of byte 0) + TestType { + raw_header: &[0x80, 0x00], + typ: 0, + layer_id: 0, + tid: 0, + fbit: true, + ..Default::default() + }, + // TRAIL_NUT (type 0), tid 1 + TestType { + raw_header: &[0x00, 0x01], + typ: 0, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // VPS_NUT (type 14) + TestType { + raw_header: &[0x00, 0x71], + typ: 14, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // SPS_NUT (type 15) + TestType { + raw_header: &[0x00, 0x79], + typ: 15, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // PPS_NUT (type 16) + TestType { + raw_header: &[0x00, 0x81], + typ: 16, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // PREFIX_SEI_NUT (type 23) + TestType { + raw_header: &[0x00, 0xB9], + typ: 23, + layer_id: 0, + tid: 1, + ..Default::default() + }, + // Fragmentation Unit (type 29) + TestType { + raw_header: &[0x00, 0xE9], + typ: H266NALU_FRAGMENTATION_UNIT_TYPE, + layer_id: 0, + tid: 1, + is_fu: true, + ..Default::default() + }, + // Aggregation Packet (type 28) + TestType { + raw_header: &[0x00, 0xE1], + typ: H266NALU_AGGREGATION_PACKET_TYPE, + layer_id: 0, + tid: 1, + is_ap: true, + ..Default::default() + }, + // layer_id = 5 (byte 0 low 6 bits), IDR_W_RADL, tid 1 + TestType { + raw_header: &[0x05, 0x39], + typ: 7, + layer_id: 5, + tid: 1, + ..Default::default() + }, + // tid = 3 + TestType { + raw_header: &[0x00, 0x0B], + typ: 1, + layer_id: 0, + tid: 3, + ..Default::default() + }, + ]; + + for (i, cur) in tests.iter().enumerate() { + let header = H266NALUHeader::new(cur.raw_header[0], cur.raw_header[1]); + + assert_eq!(header.f(), cur.fbit, "tc {i}: f bit"); + assert_eq!(header.nalu_type(), cur.typ, "tc {i}: type"); + assert_eq!(header.layer_id(), cur.layer_id, "tc {i}: layer_id"); + assert_eq!(header.tid(), cur.tid, "tc {i}: tid"); + assert_eq!(header.is_aggregation_packet(), cur.is_ap, "tc {i}: is_ap"); + assert_eq!(header.is_fragmentation_unit(), cur.is_fu, "tc {i}: is_fu"); + } + + Ok(()) + } + + /// Test IRAP detection across all NAL types. + /// H.266 IRAP: IDR_W_RADL(7), IDR_N_LP(8), CRA(9). GDR(10) is not. + #[test] + fn test_h266_irap_detection() -> Result<()> { + for typ in 0u8..=31 { + let header = H266NALUHeader::new_with_type(typ, 0, 1); + let expected = (7..=9).contains(&typ); + assert_eq!( + header.is_irap(), + expected, + "type {typ} irap should be {expected}" + ); + } + Ok(()) + } + + /// Test FU header bit parsing/serialization including the H.266 + /// specific P (last-FU-of-picture) bit. + #[test] + fn test_h266_fu_header() -> Result<()> { + struct TestType { + value: u8, + s: bool, + e: bool, + p: bool, + typ: u8, + } + + let tests = vec![ + // S=1, type 7 (IDR_W_RADL) + TestType { + value: 0x87, + s: true, + e: false, + p: false, + typ: 7, + }, + // E=1, type 7 + TestType { + value: 0x47, + s: false, + e: true, + p: false, + typ: 7, + }, + // P=1, type 7 + TestType { + value: 0x27, + s: false, + e: false, + p: true, + typ: 7, + }, + // E+P, type 0 (TRAIL) + TestType { + value: 0x60, + s: false, + e: true, + p: true, + typ: 0, + }, + // S, type 31 (max 5-bit type) + TestType { + value: 0x9F, + s: true, + e: false, + p: false, + typ: 31, + }, + // nothing set, type 9 (CRA) + TestType { + value: 0x09, + s: false, + e: false, + p: false, + typ: 9, + }, + ]; + + // F bit set rejects as corrupted. + { + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let res = depack.depacketize(&[0x80, 0x01, 0x93, 0xAF, 0xAF], &mut out, &mut extra); + assert!(matches!(res, Err(PacketError::ErrH266CorruptedPacket))); + } + + for (i, t) in tests.iter().enumerate() { + let fu = H266FragmentationUnitHeader(t.value); + assert_eq!(fu.s(), t.s, "tc {i}: s"); + assert_eq!(fu.e(), t.e, "tc {i}: e"); + assert_eq!(fu.p(), t.p, "tc {i}: p"); + assert_eq!(fu.fu_type(), t.typ, "tc {i}: type"); + + // round-trip via constructor (p only via raw value) + let built = H266FragmentationUnitHeader::new(t.s, t.e, t.p, t.typ); + assert_eq!(built.0, t.value, "tc {i}: build"); + } + + Ok(()) + } + } + + /// Depacketizer behavior tests, expressed against the depacketize() API. + mod depacketizer_tests { + use super::*; + + /// Valid/invalid single NAL unit packets. + #[test] + fn test_h266_single_nalunit_packet() -> Result<()> { + // Valid single NAL (TRAIL, type 0) -> Annex-B out. + let nal = make_nal(0, 5); + let (out, keyframe) = depacketize_all(&[nal.clone()])?; + assert_eq!(&out[..4], &[0, 0, 0, 1]); + assert_eq!(&out[4..], &nal[..]); + assert!(!keyframe); + + // IDR single NAL flags keyframe. + let idr = make_nal(H266NALU_IDR_W_RADL, 5); + let (out, keyframe) = depacketize_all(&[idr.clone()])?; + assert_eq!(&out[4..], &idr[..]); + assert!(keyframe); + + // Too short (header only, no payload beyond 2 bytes). + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let err = depack.depacketize(&hdr(0, 0, 1), &mut out, &mut extra); + assert_eq!(err, Err(PacketError::ErrShortPacket)); + + // Empty packet. + let err = depack.depacketize(&[], &mut out, &mut extra); + assert_eq!(err, Err(PacketError::ErrShortPacket)); + + // F bit set is rejected as corrupted. + let err = depack.depacketize(&[0x80, 0x01, 0xAA, 0xBB], &mut out, &mut extra); + assert!(matches!(err, Err(PacketError::ErrH266CorruptedPacket))); + + Ok(()) + } + + /// Valid/invalid aggregation packets. + #[test] + fn test_h266_aggregation_packet() -> Result<()> { + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 6); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 4); + + // Build AP: [AP hdr][len][sps][len][pps] + let mut ap = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + ap.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + ap.extend_from_slice(&sps); + ap.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + ap.extend_from_slice(&pps); + + let (out, keyframe) = depacketize_all(&[ap])?; + let expected = annexb(&[&sps, &pps]); + assert_eq!(out, expected); + assert!(!keyframe); + + // AP with a single unit violates RFC 9328 (>= 2 required). + let mut ap1 = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + ap1.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + ap1.extend_from_slice(&sps); + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let err = depack.depacketize(&ap1, &mut out, &mut extra); + assert_eq!(err, Err(PacketError::ErrShortPacket)); + + // Truncated size field: claims more bytes than present. + let mut bad = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + bad.extend_from_slice(&[0x00, 0x20]); // size 32 but nothing follows + let err = depack.depacketize(&bad, &mut out, &mut extra); + assert_eq!(err, Err(PacketError::ErrShortPacket)); + + // AP containing an IRAP flags keyframe. + let idr = make_nal(H266NALU_IDR_N_LP, 6); + let mut ap_idr = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + ap_idr.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + ap_idr.extend_from_slice(&sps); + ap_idr.extend_from_slice(&(idr.len() as u16).to_be_bytes()); + ap_idr.extend_from_slice(&idr); + let (_, keyframe) = depacketize_all(&[ap_idr])?; + assert!(keyframe); + + Ok(()) + } + + /// FU reassembly: start/middle/end, header reconstruction, orphan + /// fragments. + #[test] + fn test_h266_fragmentation_unit_packet() -> Result<()> { + // Original NAL: CRA (type 9), 8 bytes. + let orig = make_nal(H266NALU_CRA, 8); + let fu_hdr_bytes = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1); + + // Fragment payload (after the original 2-byte header). + let payload = &orig[2..]; + let (a, b) = payload.split_at(payload.len() / 2); + + let mut fu1 = fu_hdr_bytes.to_vec(); + fu1.push(H266FragmentationUnitHeader::new(true, false, false, H266NALU_CRA).0); + fu1.extend_from_slice(a); + + let mut fu2 = fu_hdr_bytes.to_vec(); + fu2.push(H266FragmentationUnitHeader::new(false, true, false, H266NALU_CRA).0); + fu2.extend_from_slice(b); + + let (out, keyframe) = depacketize_all(&[fu1, fu2])?; + assert_eq!(out, annexb(&[&orig])); + assert!(keyframe, "CRA via FU must flag keyframe"); + + // FU packet too short for an FU header. + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let err = depack.depacketize(&fu_hdr_bytes, &mut out, &mut extra); + assert_eq!(err, Err(PacketError::ErrShortPacket)); + + // Orphan middle/end fragment on a fresh depacketizer: silently + // consumed, no output. + let mut orphan = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1).to_vec(); + orphan.push(H266FragmentationUnitHeader::new(false, true, false, 0).0); + orphan.extend_from_slice(&[0xAA, 0xBB]); + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + depack.depacketize(&orphan, &mut out, &mut extra)?; + assert!(out.is_empty(), "orphan FU must not produce output"); + + Ok(()) + } + + /// General depacketizer error/dispatch table. + #[test] + fn test_h266_packet() -> Result<()> { + struct TestType { + raw: &'static [u8], + expect_err: bool, // all error cases expect ErrShortPacket + } + + let tests = vec![ + // empty + TestType { + raw: &[], + expect_err: true, + }, + // header only + TestType { + raw: &[0x00, 0x01], + expect_err: true, + }, + // valid single NAL (TRAIL) + TestType { + raw: &[0x00, 0x01, 0xAB, 0xCD, 0xEF], + expect_err: false, + }, + // FU too short (no FU header) + TestType { + raw: &[0x00, 0xE9], + expect_err: true, + }, + // valid FU start fragment (S=1, type 7) + TestType { + raw: &[0x00, 0xE9, 0x87, 0x11, 0x22], + expect_err: false, + }, + // AP with one unit only -> error + TestType { + raw: &[0x00, 0xE1, 0x00, 0x03, 0x00, 0x01, 0xAA], + expect_err: true, + }, + // valid AP with two units + TestType { + raw: &[ + 0x00, 0xE1, // AP header + 0x00, 0x03, 0x00, 0x01, 0xAA, // unit 1 (TRAIL) + 0x00, 0x03, 0x00, 0x01, 0xBB, // unit 2 (TRAIL) + ], + expect_err: false, + }, + ]; + + for (i, t) in tests.iter().enumerate() { + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let res = depack.depacketize(t.raw, &mut out, &mut extra); + if t.expect_err { + assert!( + matches!(res, Err(PacketError::ErrShortPacket)), + "tc {i} should error, got {res:?}" + ); + } else { + assert!(res.is_ok(), "tc {i} should succeed: {res:?}"); + } + } + + Ok(()) + } + } + + /// Packetizer behavior tests. + mod packetizer_tests { + use super::*; + + /// A payload without start codes is treated as one NAL unit. + #[test] + fn test_h266_packetizer_single_nalu() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 10); + let packets = pck.packetize(1200, &nal)?; + assert_eq!(packets.len(), 1); + assert_eq!(packets[0], nal); + Ok(()) + } + + /// Annex-B input is split into one packet per NAL. + #[test] + fn test_h266_packetizer_annexb_split() -> Result<()> { + let n1 = make_nal(0, 6); + let n2 = make_nal(1, 7); + // mix 4-byte and 3-byte start codes + let mut payload = Vec::new(); + payload.extend_from_slice(&[0, 0, 0, 1]); + payload.extend_from_slice(&n1); + payload.extend_from_slice(&[0, 0, 1]); + payload.extend_from_slice(&n2); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2); + assert_eq!(packets[0], n1); + assert_eq!(packets[1], n2); + Ok(()) + } + + /// NAL exactly == mtu stays a single packet (no fragmentation). + #[test] + fn test_h266_packetizer_single_nalu_no_fragment() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 100); + let packets = pck.packetize(100, &nal)?; + assert_eq!(packets.len(), 1); + assert_eq!(packets[0], nal); + Ok(()) + } + + /// Large NAL is fragmented into FUs and can be reconstructed. + #[test] + fn test_h266_packetizer_fu_fragmentation_roundtrip_payload() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(H266NALU_CRA, 3000); + let mtu = 200; + let packets = pck.packetize(mtu, &nal)?; + assert!(packets.len() > 1); + for p in &packets { + assert!(p.len() <= mtu); + let h = H266NALUHeader::new(p[0], p[1]); + assert!(h.is_fragmentation_unit()); + } + let rebuilt = reconstruct_from_fu_packets(&packets); + assert_eq!(rebuilt, nal); + Ok(()) + } + + /// VPS+SPS+PPS are aggregated into one AP before the next NAL. + #[test] + fn test_h266_packetizer_emits_ap_for_vps_sps_pps() -> Result<()> { + let vps = make_nal(H266NALU_VPS_NALU_TYPE, 8); + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 12); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 6); + let idr = make_nal(H266NALU_IDR_W_RADL, 20); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2, "AP + IDR single expected"); + + // packets[0] is the AP with 3 units. + let ap = &packets[0]; + let h = H266NALUHeader::new(ap[0], ap[1]); + assert!(h.is_aggregation_packet()); + let mut units = Vec::new(); + let mut off = 2; + while off + 2 <= ap.len() { + let sz = u16::from_be_bytes([ap[off], ap[off + 1]]) as usize; + off += 2; + units.push(ap[off..off + sz].to_vec()); + off += sz; + } + assert_eq!(units.len(), 3); + assert_eq!(units[0], vps); + assert_eq!(units[1], sps); + assert_eq!(units[2], pps); + + // packets[1] is the IDR single NAL. + assert_eq!(packets[1], idr); + Ok(()) + } + + /// When the AP would exceed the MTU, parameter sets fall back to + /// single NAL packets. + #[test] + fn test_h266_packetizer_ap_exceeds_mtu_fallback() -> Result<()> { + let mtu = 700; + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 600); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 600); + let idr = make_nal(H266NALU_IDR_W_RADL, 50); + let payload = annexb(&[&sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(mtu, &payload)?; + + // AP would be 2 + (2+600)*2 = 1206 > 700 -> fallback singles. + assert_eq!(packets.len(), 3); + assert_eq!(packets[0], sps); + assert_eq!(packets[1], pps); + assert_eq!(packets[2], idr); + Ok(()) + } + + /// AUD and filler NALs are dropped by the packetizer. + #[test] + fn test_h266_packetizer_drops_aud_and_filler() -> Result<()> { + let aud = make_nal(H266NALU_AUD_NALU_TYPE, 4); + let fd = make_nal(H266NALU_FILLER_NALU_TYPE, 10); + let trail = make_nal(0, 8); + let payload = annexb(&[&aud, &trail, &fd]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 1); + assert_eq!(packets[0], trail); + Ok(()) + } + + /// is_marker: last packet of the AU carries the marker. + #[test] + fn test_h266_packetizer_marker() -> Result<()> { + let mut pck = H266Packetizer::default(); + assert!(pck.is_marker(&[], None, true)); + assert!(!pck.is_marker(&[], None, false)); + Ok(()) + } + + /// FU S/E flags: first fragment S, last fragment E, middles neither. + /// The P bit is never set (RFC 9328 only mandates P=0 for + /// non-final fragments; always 0 is compliant). + #[test] + fn test_h266_fragmentation_start_end_flags() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 1000); + let packets = pck.packetize(100, &nal)?; + assert!(packets.len() >= 3); + + for (i, p) in packets.iter().enumerate() { + let fu = H266FragmentationUnitHeader(p[2]); + let is_first = i == 0; + let is_last = i == packets.len() - 1; + assert_eq!(fu.s(), is_first, "pkt {i} S"); + assert_eq!(fu.e(), is_last, "pkt {i} E"); + assert!(!fu.p(), "pkt {i} P never set"); + assert_eq!(fu.fu_type(), 0, "pkt {i} type"); + } + Ok(()) + } + + /// Exact boundary: len == mtu single; len == mtu+1 fragments, with + /// exact fragment sizes. + #[test] + fn test_h266_packetizer_exact_fu_boundary_mtu() -> Result<()> { + let mtu = 100; + + // len == mtu -> single + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, mtu); + let packets = pck.packetize(mtu, &nal)?; + assert_eq!(packets.len(), 1); + + // len == mtu + 1 -> FU; payload 99 bytes; max fragment = mtu - 3 = 97 + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, mtu + 1); + let packets = pck.packetize(mtu, &nal)?; + assert_eq!(packets.len(), 2); + assert_eq!(packets[0].len(), 3 + 97); + assert_eq!(packets[1].len(), 3 + 2); + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + Ok(()) + } + + /// MTU sweep roundtrip. + #[test] + fn test_h266_mtu_variation() -> Result<()> { + for mtu in [50usize, 128, 512, 1200] { + let mut pck = H266Packetizer::default(); + let nal = make_nal(H266NALU_IDR_N_LP, 1000); + let packets = pck.packetize(mtu, &nal)?; + for p in &packets { + assert!(p.len() <= mtu, "mtu {mtu}: packet {} too big", p.len()); + } + if nal.len() <= mtu { + assert_eq!(packets.len(), 1); + } else { + assert_eq!(reconstruct_from_fu_packets(&packets), nal, "mtu {mtu}"); + } + } + Ok(()) + } + + #[test] + fn packetize_respects_mtu() -> Result<()> { + // 2-byte NAL header (non-parameter-set type) + payload. + let nal = make_nal(0, 2002); + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = H266Packetizer::default(); + let pkts = pck.packetize(mtu, &nal)?; + assert!(!pkts.is_empty(), "H266 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "H266 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } + } + + /// Packetize → depacketize roundtrips. + mod roundtrip_tests { + use super::*; + + /// FU roundtrip. + #[test] + fn test_h266_fu_roundtrip_with_depacketizer() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(H266NALU_IDR_W_RADL, 2500); + let packets = pck.packetize(400, &nal)?; + assert!(packets.len() > 1); + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&nal])); + assert!(keyframe); + Ok(()) + } + + /// AP roundtrip. + #[test] + fn test_h266_ap_roundtrip_with_depacketizer() -> Result<()> { + let vps = make_nal(H266NALU_VPS_NALU_TYPE, 8); + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 12); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 6); + let idr = make_nal(H266NALU_IDR_N_LP, 40); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&vps, &sps, &pps, &idr])); + assert!(keyframe); + Ok(()) + } + + /// Single NAL roundtrip. + #[test] + fn test_h266_single_nalu_roundtrip_with_depacketizer() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 50); + let packets = pck.packetize(1200, &nal)?; + assert_eq!(packets.len(), 1); + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&nal])); + assert!(!keyframe); + Ok(()) + } + + /// Mixed AU: params (AP) + small (single) + large (FU) NALs. + #[test] + fn test_h266_mixed_packet_types_roundtrip() -> Result<()> { + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 10); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 6); + let small = make_nal(H266NALU_IDR_W_RADL, 100); + let large = make_nal(0, 1500); + let payload = annexb(&[&sps, &pps, &small, &large]); + + let mut pck = H266Packetizer::default(); + let mtu = 600; + let packets = pck.packetize(mtu, &payload)?; + + // AP (sps+pps) + 1 single + >=3 FUs + assert!(packets.len() >= 5, "got {}", packets.len()); + for p in &packets { + assert!(p.len() <= mtu); + } + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&sps, &pps, &small, &large])); + assert!(keyframe); + Ok(()) + } + + /// Multi-NAL Annex-B with mixed start-code lengths. + #[test] + fn test_h266_annexb_roundtrip_with_depacketizer() -> Result<()> { + let n1 = make_nal(1, 30); + let n2 = make_nal(2, 40); + let n3 = make_nal(0, 25); + + let mut payload = Vec::new(); + payload.extend_from_slice(&[0, 0, 1]); // 3-byte start code + payload.extend_from_slice(&n1); + payload.extend_from_slice(&[0, 0, 0, 1]); // 4-byte + payload.extend_from_slice(&n2); + payload.extend_from_slice(&[0, 0, 1]); + payload.extend_from_slice(&n3); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 3); + + let (out, _) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&n1, &n2, &n3])); + Ok(()) + } + } + + /// Partition head/tail classification. + mod partition_tests { + use super::*; + #[test] + fn test_h266_is_partition_head() -> Result<()> { + let depack = H266Depacketizer::default(); + + // Too short -> false. + assert!(!depack.is_partition_head(&[0x00])); + + // Single NAL -> head. + assert!(depack.is_partition_head(&make_nal(0, 5))); + + // AP -> head. + assert!(depack.is_partition_head(&hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1))); + + // FU with S=1 -> head. + let mut fu_s = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1).to_vec(); + fu_s.push(H266FragmentationUnitHeader::new(true, false, false, 0).0); + assert!(depack.is_partition_head(&fu_s)); + + // FU middle (no S) -> not head. + let mut fu_m = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1).to_vec(); + fu_m.push(H266FragmentationUnitHeader::new(false, false, false, 0).0); + assert!(!depack.is_partition_head(&fu_m)); + + // FU header missing -> not head. + assert!(!depack.is_partition_head(&hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1))); + + // F bit -> treated as head (error case). + assert!(depack.is_partition_head(&[0x80, 0x01, 0x00])); + + Ok(()) + } + #[test] + fn test_h266_is_partition_tail() -> Result<()> { + let depack = H266Depacketizer::default(); + + // Too short -> false even with marker. + assert!(!depack.is_partition_tail(true, &[0x00])); + + // Single NAL: marker decides. + let single = make_nal(0, 5); + assert!(depack.is_partition_tail(true, &single)); + assert!(!depack.is_partition_tail(false, &single)); + + // FU with E=1 -> tail regardless of marker. + let mut fu_e = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1).to_vec(); + fu_e.push(H266FragmentationUnitHeader::new(false, true, false, 0).0); + assert!(depack.is_partition_tail(false, &fu_e)); + + // FU without E -> not tail, marker ignored for FU. + let mut fu_s = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1).to_vec(); + fu_s.push(H266FragmentationUnitHeader::new(true, false, false, 0).0); + assert!(!depack.is_partition_tail(true, &fu_s)); + + // FU header missing -> false. + assert!(!depack.is_partition_tail(true, &hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1))); + + Ok(()) + } + } + + /// MTU and malformed-input edge cases. + mod edge_tests { + use super::*; + #[test] + fn test_h266_zero_mtu() -> Result<()> { + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(0, &make_nal(0, 10))?; + assert!(packets.is_empty()); + Ok(()) + } + #[test] + fn test_h266_empty_nalu() -> Result<()> { + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &[])?; + assert!(packets.is_empty()); + Ok(()) + } + + /// MTU below the minimum (header+fu+1) yields no packets. + #[test] + fn test_h266_mtu_smaller_than_fu_overhead() -> Result<()> { + let mut pck = H266Packetizer::default(); + // MIN_MTU is 4; mtu 3 == FU overhead exactly -> cannot carry payload. + let packets = pck.packetize(3, &make_nal(0, 100))?; + assert!(packets.is_empty()); + Ok(()) + } + + /// Minimal viable MTU fragments 1 byte at a time and round-trips. + #[test] + fn test_h266_fu_minimal_mtu() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 10); + let packets = pck.packetize(MIN_MTU, &nal)?; + // payload bytes = 8, one per fragment + assert_eq!(packets.len(), 8); + for p in &packets { + assert_eq!(p.len(), 4); + } + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + Ok(()) + } + + /// MTU above MAX_PACKET_SIZE is clamped. + #[test] + fn test_h266_fu_mtu_exceeds_max_packet_size() -> Result<()> { + let mut pck = H266Packetizer::default(); + let nal = make_nal(0, 3000); + let packets = pck.packetize(5000, &nal)?; + assert!(packets.len() > 1, "must fragment despite huge mtu"); + for p in &packets { + assert!(p.len() <= MAX_PACKET_SIZE); + } + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + Ok(()) + } + + /// A NAL smaller than the 2-byte header is skipped. + #[test] + fn test_h266_nalu_smaller_than_header() -> Result<()> { + let mut pck = H266Packetizer::default(); + // one-byte "NAL" between start codes + let mut payload = Vec::new(); + payload.extend_from_slice(&[0, 0, 0, 1]); + payload.push(0x00); + let packets = pck.packetize(1200, &payload)?; + assert!(packets.is_empty()); + Ok(()) + } + + /// Exact AP byte layout. + #[test] + fn test_h266_aggregated_exact_layout() -> Result<()> { + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 4); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 3); + let idr = make_nal(H266NALU_IDR_W_RADL, 5); + let payload = annexb(&[&sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2); + + // Expected AP: [0x00, (28<<3)|tid=1] [00 04 sps] [00 03 pps] + let mut expected = vec![0x00, (H266NALU_AGGREGATION_PACKET_TYPE << 3) | 1]; + expected.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + expected.extend_from_slice(&sps); + expected.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + expected.extend_from_slice(&pps); + assert_eq!(packets[0], expected); + Ok(()) + } + + /// Synthetic "real-world" AU shape: params + IDR slice, verified + /// end-to-end (structure synthesized for VVC). + #[test] + fn test_h266_packet_synthetic_au() -> Result<()> { + let sps = make_nal(H266NALU_SPS_NALU_TYPE, 45); + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 12); + let aps = make_nal(17, 30); // PREFIX_APS passes through as single + let idr = make_nal(H266NALU_IDR_N_LP, 800); + let payload = annexb(&[&sps, &pps, &aps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + + // AP(sps+pps) + APS single + IDR single + assert_eq!(packets.len(), 3); + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&sps, &pps, &aps, &idr])); + assert!(keyframe); + Ok(()) + } + } + + /// Keyframe detection on raw RTP payloads. + #[test] + fn test_detect_h266_keyframe() { + // Empty / too short payload + assert!(!detect_h266_keyframe(&[])); + assert!(!detect_h266_keyframe(&[0x00])); + + // Single IRAP NALs + for typ in [H266NALU_IDR_W_RADL, H266NALU_IDR_N_LP, H266NALU_CRA] { + let h = H266NALUHeader::new_with_type(typ, 0, 1); + assert!( + detect_h266_keyframe(&h.0.to_be_bytes()), + "type {typ} must be keyframe" + ); + } + + // GDR (type 10) is not treated as IRAP here. + let gdr = H266NALUHeader::new_with_type(H266NALU_GDR, 0, 1); + assert!(!detect_h266_keyframe(&gdr.0.to_be_bytes())); + + // TRAIL (type 0) is not a keyframe. + let trail = H266NALUHeader::new_with_type(0, 0, 1); + assert!(!detect_h266_keyframe(&trail.0.to_be_bytes())); + + // Aggregation packet with an IDR inside. + let ap = H266NALUHeader::new_with_type(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1); + let idr = H266NALUHeader::new_with_type(H266NALU_IDR_W_RADL, 0, 1); + let mut ap_with_idr = Vec::new(); + ap_with_idr.extend_from_slice(&ap.0.to_be_bytes()); + ap_with_idr.extend_from_slice(&[0x00, 0x03]); // unit size 3 + ap_with_idr.extend_from_slice(&idr.0.to_be_bytes()); + ap_with_idr.push(0x00); + assert!(detect_h266_keyframe(&ap_with_idr)); + + // Aggregation packet without IRAP. + let non_irap = H266NALUHeader::new_with_type(1, 0, 1); + let mut ap_no_irap = Vec::new(); + ap_no_irap.extend_from_slice(&ap.0.to_be_bytes()); + ap_no_irap.extend_from_slice(&[0x00, 0x03]); + ap_no_irap.extend_from_slice(&non_irap.0.to_be_bytes()); + ap_no_irap.push(0x00); + assert!(!detect_h266_keyframe(&ap_no_irap)); + + // FU start fragment with IDR type. + let fu = H266NALUHeader::new_with_type(H266NALU_FRAGMENTATION_UNIT_TYPE, 0, 1); + let mut fu_start_idr = Vec::new(); + fu_start_idr.extend_from_slice(&fu.0.to_be_bytes()); + fu_start_idr.push(0x80 | H266NALU_IDR_W_RADL); // S=1, type=7 + fu_start_idr.extend_from_slice(&[0x00, 0x00]); + assert!(detect_h266_keyframe(&fu_start_idr)); + + // FU continuation fragment (S=0) - cannot detect. + let mut fu_cont = Vec::new(); + fu_cont.extend_from_slice(&fu.0.to_be_bytes()); + fu_cont.push(H266NALU_IDR_W_RADL); // S=0 + fu_cont.extend_from_slice(&[0x00, 0x00]); + assert!(!detect_h266_keyframe(&fu_cont)); + + // FU too short (no FU header byte). + assert!(!detect_h266_keyframe(&fu.0.to_be_bytes())); + } + + /// DONL tests (RFC 9328 §4.3, sprop-max-don-diff > 0). + /// + mod donl_tests { + use super::*; + #[test] + fn test_h266_donl_single_nal_round_trip() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H266Depacketizer::default(); + depacketizer.with_donl(true); + + // Single NAL unit (TRAIL, type 1): [0x00, 0x09] + let nalu = vec![0x00, 0x09, 0xDE, 0xAD, 0xBE, 0xEF]; + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &nalu)?; + assert_eq!(packets.len(), 1, "Should produce 1 packet"); + + let packet = &packets[0]; + + // Packet structure: [NAL_HDR (2)] [DONL (2)] [PAYLOAD] + assert!(packet.len() >= 6); + assert_eq!(packet[0], 0x00); + assert_eq!(packet[1], 0x09); + + let donl = u16::from_be_bytes([packet[2], packet[3]]); + assert_eq!(donl, 0, "DONL should be 0 for first NAL"); + + assert_eq!(packet[4], 0xDE); + assert_eq!(packet[5], 0xAD); + + // Depacketize: Annex-B output without DONL. + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + + assert_eq!(out.len(), 10); + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..6], &[0x00, 0x09]); + assert_eq!(&out[6..10], &[0xDE, 0xAD, 0xBE, 0xEF]); + + Ok(()) + } + #[test] + fn test_h266_donl_fragmentation_round_trip() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H266Depacketizer::default(); + depacketizer.with_donl(true); + + let mut nalu = vec![0x00, 0x09]; // TRAIL (type 1) + nalu.extend(vec![0xAA; 3000]); + + let packets = packetizer.packetize(1200, &nalu)?; + assert!(packets.len() >= 3); + + // First FU: [FU_HDR (2)] [FU_HEADER (1)] [DONL (2)] [PAYLOAD] + let first_packet = &packets[0]; + let fu_header = H266NALUHeader::new(first_packet[0], first_packet[1]); + assert_eq!(fu_header.nalu_type(), H266NALU_FRAGMENTATION_UNIT_TYPE); + + let fu_hdr = H266FragmentationUnitHeader(first_packet[2]); + assert!(fu_hdr.s()); + assert!(!fu_hdr.e()); + + let donl = u16::from_be_bytes([first_packet[3], first_packet[4]]); + assert_eq!(donl, 0, "DONL should be 0 for first NAL"); + + // Middle fragments: no S/E, payload right after FU header (no DONL). + if packets.len() > 2 { + let middle_fu_hdr = H266FragmentationUnitHeader(packets[1][2]); + assert!(!middle_fu_hdr.s()); + assert!(!middle_fu_hdr.e()); + } + + let last_fu_hdr = H266FragmentationUnitHeader(packets[packets.len() - 1][2]); + assert!(!last_fu_hdr.s()); + assert!(last_fu_hdr.e()); + + // Depacketize all fragments. + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + for packet in &packets { + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + } + + assert_eq!(out.len(), 4 + nalu.len()); + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..], &nalu[..]); + + Ok(()) + } + #[test] + fn test_h266_donl_aggregation_round_trip() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H266Depacketizer::default(); + depacketizer.with_donl(true); + + let vps = vec![0x00, 0x71, 0x0C, 0x01, 0xFF, 0xFF]; // VPS (14) + let sps = vec![0x00, 0x79, 0x01, 0x50, 0x00, 0x00]; // SPS (15) + let pps = vec![0x00, 0x81, 0xC0, 0xF3, 0xC0, 0x02]; // PPS (16) + let payload = annexb(&[&vps, &sps, &pps]); + + let packets = packetizer.packetize(MAX_PACKET_SIZE, &payload)?; + assert_eq!(packets.len(), 0, "Parameter sets should be cached"); + + // VCL NAL triggers AP emission. + let vcl = vec![0x00, 0x09, 0x11, 0x22, 0x33]; + let vcl_packets = packetizer.packetize(MAX_PACKET_SIZE, &annexb(&[&vcl]))?; + assert_eq!(vcl_packets.len(), 2, "Should produce AP + VCL"); + + // AP structure (RFC 9328 §4.3.2): + // [AP_HDR (2)] [DONL (2)] [SIZE (2)] [NALU] [SIZE (2)] [NALU] ... + let ap_packet = &vcl_packets[0]; + let ap_header = H266NALUHeader::new(ap_packet[0], ap_packet[1]); + assert_eq!(ap_header.nalu_type(), H266NALU_AGGREGATION_PACKET_TYPE); + + let donl = u16::from_be_bytes([ap_packet[2], ap_packet[3]]); + assert_eq!(donl, 0, "DONL should be 0 for first aggregated unit"); + + let first_size = u16::from_be_bytes([ap_packet[4], ap_packet[5]]); + assert_eq!(first_size, 6, "VPS size should be 6 bytes"); + + // The second unit's 16-bit size follows the first unit + // immediately. + let first_nal_end = 6 + first_size as usize; + let second_size = + u16::from_be_bytes([ap_packet[first_nal_end], ap_packet[first_nal_end + 1]]); + assert_eq!(second_size, 6, "SPS size should be 6 bytes"); + + // Depacketize AP: all 3 parameter sets out, DONL stripped. + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(ap_packet, &mut out, &mut codec_extra)?; + + assert_eq!(out, annexb(&[&vps, &sps, &pps])); + + Ok(()) + } + #[test] + fn test_h266_donl_increments_correctly() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let nalu1 = vec![0x00, 0x09, 0xAA]; + let nalu2 = vec![0x00, 0x09, 0xBB]; + let nalu3 = vec![0x00, 0x09, 0xCC]; + + let packets1 = packetizer.packetize(MAX_PACKET_SIZE, &nalu1)?; + let packets2 = packetizer.packetize(MAX_PACKET_SIZE, &nalu2)?; + let packets3 = packetizer.packetize(MAX_PACKET_SIZE, &nalu3)?; + + let donl1 = u16::from_be_bytes([packets1[0][2], packets1[0][3]]); + let donl2 = u16::from_be_bytes([packets2[0][2], packets2[0][3]]); + let donl3 = u16::from_be_bytes([packets3[0][2], packets3[0][3]]); + + assert_eq!(donl1, 0); + assert_eq!(donl2, 1); + assert_eq!(donl3, 2); + + Ok(()) + } + #[test] + fn test_h266_without_donl() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + // DONL disabled by default. + + let nalu = vec![0x00, 0x09, 0xDE, 0xAD, 0xBE, 0xEF]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &nalu)?; + + assert_eq!(packets.len(), 1); + assert_eq!(packets[0].len(), nalu.len(), "No DONL field expected"); + assert_eq!(packets[0], nalu); + + Ok(()) + } + #[test] + fn test_h266_sdp_driven_donl_all_packet_types() -> Result<()> { + use crate::format::FormatParams; + + // SDP fmtp with sprop-max-don-diff > 0 enables DONL. + let fmtp = FormatParams::parse_line("sprop-max-don-diff=32"); + assert_eq!(fmtp.sprop_max_don_diff, Some(32)); + + let donl_enabled = fmtp.sprop_max_don_diff.unwrap_or(0) > 0; + assert!(donl_enabled); + + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(donl_enabled); + + let mut depacketizer = H266Depacketizer::default(); + depacketizer.with_donl(donl_enabled); + + // --- Test 1: Single NAL with DONL --- + let single_nalu = vec![0x00, 0x09, 0xDE, 0xAD, 0xBE, 0xEF]; // TRAIL + let packets = packetizer.packetize(MAX_PACKET_SIZE, &single_nalu)?; + assert_eq!(packets.len(), 1); + + let pkt = &packets[0]; + assert_eq!(pkt.len(), single_nalu.len() + 2, "2-byte DONL on wire"); + let donl = u16::from_be_bytes([pkt[2], pkt[3]]); + assert_eq!(donl, 0); + + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + depacketizer.depacketize(pkt, &mut out, &mut extra)?; + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..], &single_nalu[..]); + + // --- Test 2: AP (VPS + SPS + PPS) with DONL --- + let vps = vec![0x00, 0x71, 0xAA, 0xBB, 0xCC]; + let sps = vec![0x00, 0x79, 0xDD, 0xEE, 0xFF, 0x11]; + let pps = vec![0x00, 0x81, 0x22, 0x33]; + let vcl = vec![0x00, 0x39, 0x44, 0x55, 0x66]; // IDR_W_RADL (7) + + assert!(packetizer.packetize(MAX_PACKET_SIZE, &vps)?.is_empty()); + assert!(packetizer.packetize(MAX_PACKET_SIZE, &sps)?.is_empty()); + assert!(packetizer.packetize(MAX_PACKET_SIZE, &pps)?.is_empty()); + + let ap_packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl)?; + assert_eq!(ap_packets.len(), 2, "Should produce AP + VCL"); + + let ap = &ap_packets[0]; + let ap_hdr = H266NALUHeader::new(ap[0], ap[1]); + assert_eq!(ap_hdr.nalu_type(), H266NALU_AGGREGATION_PACKET_TYPE); + let ap_donl = u16::from_be_bytes([ap[2], ap[3]]); + assert_eq!(ap_donl, 1, "AP DONL should be 1 (single consumed DON=0)"); + + let first_size = u16::from_be_bytes([ap[4], ap[5]]) as usize; + // The next unit's 16-bit size follows the first unit directly. + let second_size_off = 6 + first_size; + let second_size = + u16::from_be_bytes([ap[second_size_off], ap[second_size_off + 1]]) as usize; + assert_eq!(second_size, 6, "SPS unit size expected"); + + out.clear(); + extra = CodecExtra::None; + depacketizer.depacketize(ap, &mut out, &mut extra)?; + + let mut offset = 0; + for expected in [&vps, &sps, &pps] { + assert_eq!(&out[offset..offset + 4], ANNEXB_NALUSTART_CODE); + offset += 4; + assert_eq!(&out[offset..offset + expected.len()], &expected[..]); + offset += expected.len(); + } + assert_eq!(offset, out.len()); + + // VCL DONL after AP consumed DONs 1..=3. + let vcl_pkt = &ap_packets[1]; + let vcl_donl = u16::from_be_bytes([vcl_pkt[2], vcl_pkt[3]]); + assert_eq!(vcl_donl, 4); + + out.clear(); + extra = CodecExtra::None; + depacketizer.depacketize(vcl_pkt, &mut out, &mut extra)?; + assert_eq!(&out[4..], &vcl[..]); + + // --- Test 3: FU with DONL --- + let mut large_nalu = vec![0x00, 0x09]; + large_nalu.extend(vec![0xAA; 200]); + + let fu_packets = packetizer.packetize(100, &large_nalu)?; + assert!(fu_packets.len() > 1); + + let first_fu = &fu_packets[0]; + let fu_hdr_byte = H266FragmentationUnitHeader(first_fu[2]); + assert!(fu_hdr_byte.s()); + let fu_donl = u16::from_be_bytes([first_fu[3], first_fu[4]]); + assert_eq!(fu_donl, 5, "FU DONL after VCL consumed DON=4"); + + let last_fu = &fu_packets[fu_packets.len() - 1]; + let last_fu_hdr = H266FragmentationUnitHeader(last_fu[2]); + assert!(last_fu_hdr.e()); + assert!(!last_fu_hdr.s()); + + out.clear(); + extra = CodecExtra::None; + for pkt in &fu_packets { + depacketizer.depacketize(pkt, &mut out, &mut extra)?; + } + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..], &large_nalu[..]); + + // --- DONL counter progression --- + // DON 0 (single), 1-3 (AP), 4 (VCL), 5 (FU) -> next = 6. + let next_nalu = vec![0x00, 0x09, 0x77]; + let next_packets = packetizer.packetize(MAX_PACKET_SIZE, &next_nalu)?; + let next_donl = u16::from_be_bytes([next_packets[0][2], next_packets[0][3]]); + assert_eq!(next_donl, 6); + + Ok(()) + } + #[test] + fn test_h266_roundtrip_with_donl() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let mut depacketizer = H266Depacketizer::default(); + depacketizer.with_donl(true); + + // Single NAL with DONL. + let single_nalu = vec![0x00, 0x09, 0xFF, 0xFF, 0xFF]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &single_nalu)?; + assert_eq!(packets.len(), 1); + let donl = u16::from_be_bytes([packets[0][2], packets[0][3]]); + assert_eq!(donl, 0); + + let mut out = Vec::new(); + let mut codec_extra = CodecExtra::None; + depacketizer.depacketize(&packets[0], &mut out, &mut codec_extra)?; + assert_eq!(&out[4..], &single_nalu[..]); + + // Fragmented NAL with DONL. + let mut large_nalu = vec![0x00, 0x09]; + for i in 0..512 { + large_nalu.push((i % 256) as u8); + } + + let fu_packets = packetizer.packetize(100, &large_nalu)?; + assert!(fu_packets.len() > 1); + + let first_donl = u16::from_be_bytes([fu_packets[0][3], fu_packets[0][4]]); + assert_eq!(first_donl, 1, "Second NAL should have DONL=1"); + + out.clear(); + codec_extra = CodecExtra::None; + for packet in &fu_packets { + depacketizer.depacketize(packet, &mut out, &mut codec_extra)?; + } + assert_eq!(&out[0..4], ANNEXB_NALUSTART_CODE); + assert_eq!(&out[4..], &large_nalu[..]); + + Ok(()) + } + #[test] + fn test_h266_aggregation_with_donl_sequences() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let vps = vec![0x00, 0x71, 0x00, 0x01, 0x02, 0x03]; + let sps = vec![0x00, 0x79, 0x00, 0x01, 0x02, 0x03]; + let pps = vec![0x00, 0x81, 0x00, 0x01, 0x02, 0x03]; + + packetizer.packetize(MAX_PACKET_SIZE, &vps)?; + packetizer.packetize(MAX_PACKET_SIZE, &sps)?; + packetizer.packetize(MAX_PACKET_SIZE, &pps)?; + + let vcl = vec![0x00, 0x09, 0xAA, 0xBB]; + let packets = packetizer.packetize(MAX_PACKET_SIZE, &vcl)?; + + assert!(!packets.is_empty()); + for packet in &packets { + assert!(packet.len() >= 2); + } + + Ok(()) + } + #[test] + fn test_h266_mtu_equals_fu_overhead_plus_donl() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + let mut large_nalu = vec![0x00, 0x09]; + large_nalu.extend(vec![0xCC; 200]); + + // FU overhead (3) + DONL (2) = 5 bytes, no room for payload. + let packets = packetizer.packetize(5, &large_nalu)?; + assert!(packets.is_empty()); + + Ok(()) + } + + /// Large single NAL with DONL: the DONL bytes count against the + /// MTU, so NAL + DONL == MTU is the single-NAL limit and one byte + /// more falls back to fragmentation. + #[test] + fn test_h266_single_nal_with_donl_large() -> Result<()> { + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + + // NAL + DONL == MTU exactly -> single packet. + let mut large_nalu = vec![0x00, 0x09]; + large_nalu.extend(vec![0xEE; 1196]); // total 1198 + 2 DONL = 1200 + let packets = packetizer.packetize(MAX_PACKET_SIZE, &large_nalu)?; + assert_eq!(packets.len(), 1); + assert_eq!(packets[0].len(), 1200); + + // One byte more -> must fragment (never exceed MTU on the wire). + let mut packetizer = H266Packetizer::default(); + packetizer.with_donl(true); + let mut larger_nalu = vec![0x00, 0x09]; + larger_nalu.extend(vec![0xEE; 1197]); // total 1199 + 2 DONL = 1201 + let packets = packetizer.packetize(MAX_PACKET_SIZE, &larger_nalu)?; + assert!(packets.len() > 1, "must fragment instead of exceeding MTU"); + for p in &packets { + assert!(p.len() <= MAX_PACKET_SIZE); + } + + Ok(()) + } + } + + /// Scalability-related tests: VVC temporal sublayers (TID) and + /// multilayer streams (nuh_layer_id). The RTP layer's responsibility + /// is to PRESERVE these fields through fragmentation/aggregation and + /// to derive AP header fields per RFC 9328 §4.3.2 (lowest LayerId/TID + /// of the aggregated units). Layer SELECTION (SFU dropping sublayers) + /// is application policy built on top of these fields. + mod svc_tests { + use super::*; + + /// FU fragmentation must preserve LayerId and TID end-to-end: + /// the FU payload header copies byte 0 (F/Z/LayerId) verbatim and + /// keeps the TID bits of byte 1; reassembly restores the original + /// two-byte header exactly. + #[test] + fn test_h266_fu_preserves_layer_and_tid() -> Result<()> { + // TRAIL (type 1) on layer 5, temporal sublayer tid=3. + let mut nal = hdr(1, 5, 3).to_vec(); + nal.extend(std::iter::repeat(0xAB).take(998)); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(100, &nal)?; + assert!(packets.len() > 1); + + for p in &packets { + let h = H266NALUHeader::new(p[0], p[1]); + assert!(h.is_fragmentation_unit()); + assert_eq!(h.layer_id(), 5, "FU header must keep LayerId"); + assert_eq!(h.tid(), 3, "FU header must keep TID"); + } + + // Reassembly restores the exact original header bytes. + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + + let (out, _) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&nal])); + Ok(()) + } + + /// The Z bit (nuh_reserved_zero_bit) and F bit live in byte 0, + /// which FU packets copy verbatim — a stream with Z set survives + /// the FU round-trip bit-exactly. + #[test] + fn test_h266_fu_preserves_z_bit() -> Result<()> { + let mut nal = hdr(1, 2, 1).to_vec(); + nal[0] |= 0b0100_0000; // set Z + nal.extend(std::iter::repeat(0xCD).take(500)); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(100, &nal)?; + assert!(packets.len() > 1); + + for p in &packets { + let h = H266NALUHeader::new(p[0], p[1]); + assert!(h.z(), "Z bit must be preserved in FU headers"); + } + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + Ok(()) + } + + /// RFC 9328 §4.3.2: the AP PayloadHdr LayerId and TID MUST be the + /// LOWEST LayerId/TID of all aggregated NAL units — and must not + /// leak from the NAL unit that triggers the AP emission. + #[test] + fn test_h266_ap_header_uses_lowest_layer_and_tid() -> Result<()> { + let vps = make_nal_lt(H266NALU_VPS_NALU_TYPE, 8, 2, 3); + let sps = make_nal_lt(H266NALU_SPS_NALU_TYPE, 8, 4, 1); + let pps = make_nal_lt(H266NALU_PPS_NALU_TYPE, 8, 0, 2); + // Trigger NAL with HIGH layer/tid — must not influence the AP. + let idr = make_nal_lt(H266NALU_IDR_W_RADL, 20, 9, 7); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2); + + let ap = H266NALUHeader::new(packets[0][0], packets[0][1]); + assert!(ap.is_aggregation_packet()); + assert_eq!( + ap.layer_id(), + 0, + "lowest LayerId of units (0), not the trigger's (9)" + ); + assert_eq!( + ap.tid(), + 1, + "lowest TID of units (1), not the trigger's (7)" + ); + assert!(!ap.f()); + Ok(()) + } + + /// Aggregated units inside the AP keep their own original headers — + /// depacketizing yields every unit bit-exact, mixed layers/TIDs + /// included. + #[test] + fn test_h266_ap_units_preserve_individual_headers() -> Result<()> { + let vps = make_nal_lt(H266NALU_VPS_NALU_TYPE, 8, 2, 3); + let sps = make_nal_lt(H266NALU_SPS_NALU_TYPE, 8, 4, 1); + let pps = make_nal_lt(H266NALU_PPS_NALU_TYPE, 8, 0, 2); + let idr = make_nal_lt(H266NALU_IDR_W_RADL, 20, 1, 1); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&vps, &sps, &pps, &idr])); + assert!(keyframe); + Ok(()) + } + + /// Keyframe detection is layer/TID-agnostic: an IRAP on a higher + /// layer or temporal sublayer is still a keyframe — as a single + /// NAL packet, inside an AP, and as the first FU fragment. + #[test] + fn test_h266_keyframe_detection_with_layers() -> Result<()> { + // Single NAL: CRA on layer 3, tid 2. + let cra = hdr(H266NALU_CRA, 3, 2); + assert!(detect_h266_keyframe(&cra)); + + // AP containing it. + let mut ap = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + ap.extend_from_slice(&[0x00, 0x03]); + ap.extend_from_slice(&cra); + ap.push(0x00); + assert!(detect_h266_keyframe(&ap)); + + // FU start fragment of it (FU header on layer 3, tid 2). + let mut fu = hdr(H266NALU_FRAGMENTATION_UNIT_TYPE, 3, 2).to_vec(); + fu.push(0x80 | H266NALU_CRA); // S=1 + fu.extend_from_slice(&[0x00, 0x00]); + assert!(detect_h266_keyframe(&fu)); + + // Non-IRAP on the same layer/tid is not a keyframe. + assert!(!detect_h266_keyframe(&hdr(1, 3, 2))); + Ok(()) + } + + /// A full mixed-AU round-trip with differing TIDs per NAL — + /// e.g. a temporal-base IDR plus a higher-sublayer TRAIL — comes + /// out bit-exact, so an SFU can still read layer_id()/tid() on the + /// receive side to implement sublayer dropping. + #[test] + fn test_h266_mixed_temporal_layers_roundtrip() -> Result<()> { + let idr = make_nal_lt(H266NALU_IDR_N_LP, 600, 0, 1); // base layer + let trail_t2 = make_nal_lt(1, 700, 0, 2); // sublayer 2 + let trail_t3 = make_nal_lt(1, 50, 0, 3); // sublayer 3 (small) + let payload = annexb(&[&idr, &trail_t2, &trail_t3]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(400, &payload)?; + + let (out, keyframe) = depacketize_all(&packets)?; + assert_eq!(out, annexb(&[&idr, &trail_t2, &trail_t3])); + assert!(keyframe); + + // Receive side can classify each NAL by TID again. + let mut tids = Vec::new(); + let mut off = 0; + while off + 6 <= out.len() { + assert_eq!(&out[off..off + 4], &[0, 0, 0, 1]); + let h = H266NALUHeader::new(out[off + 4], out[off + 5]); + tids.push(h.tid()); + // skip to next start code + let mut next = off + 4; + loop { + next += 1; + if next + 4 > out.len() { + next = out.len(); + break; + } + if out[next..next + 4] == [0, 0, 0, 1] { + break; + } + } + off = next; + } + assert_eq!(tids, vec![1, 2, 3]); + Ok(()) + } + } + + /// Adversarial and malformed-input hardening tests for the AP and FU + /// paths (header-bit derivation rules, nested RTP-only NAL types, + /// inconsistent length fields). + mod hardening_tests { + use super::*; + + /// the AP header F bit is the OR + /// of the aggregated units' F bits. + #[test] + fn test_h266_ap_header_f_bit_or() -> Result<()> { + let vps = make_nal(H266NALU_VPS_NALU_TYPE, 6); + let mut sps = make_nal(H266NALU_SPS_NALU_TYPE, 6); + sps[0] |= 0b1000_0000; // F set on one unit + let pps = make_nal(H266NALU_PPS_NALU_TYPE, 6); + let idr = make_nal(H266NALU_IDR_W_RADL, 10); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2); + + let ap = H266NALUHeader::new(packets[0][0], packets[0][1]); + assert!(ap.is_aggregation_packet()); + assert!(ap.f(), "AP header F bit must be OR of unit F bits"); + Ok(()) + } + + /// the Z bit of aggregated units + /// is ignored — the AP header Z is always 0. + #[test] + fn test_h266_ap_header_z_bit_ignored() -> Result<()> { + let mut vps = make_nal(H266NALU_VPS_NALU_TYPE, 6); + let mut sps = make_nal(H266NALU_SPS_NALU_TYPE, 6); + let mut pps = make_nal(H266NALU_PPS_NALU_TYPE, 6); + for n in [&mut vps, &mut sps, &mut pps] { + n[0] |= 0b0100_0000; // Z set on every unit + } + let idr = make_nal(H266NALU_IDR_W_RADL, 10); + let payload = annexb(&[&vps, &sps, &pps, &idr]); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(1200, &payload)?; + assert_eq!(packets.len(), 2); + + let ap = H266NALUHeader::new(packets[0][0], packets[0][1]); + assert!(ap.is_aggregation_packet()); + assert!(!ap.z(), "AP header Z bit must be 0 regardless of units"); + Ok(()) + } + + /// an AP containing a nested + /// AP (type 28) is corrupted. + #[test] + fn test_h266_ap_rejects_nested_ap() -> Result<()> { + let trail = make_nal(0, 4); + let nested = make_nal(H266NALU_AGGREGATION_PACKET_TYPE, 4); + + let mut ap = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + for unit in [&trail, &nested] { + ap.extend_from_slice(&(unit.len() as u16).to_be_bytes()); + ap.extend_from_slice(unit); + } + + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let res = depack.depacketize(&ap, &mut out, &mut extra); + assert!(matches!(res, Err(PacketError::ErrH266CorruptedPacket))); + assert!(out.is_empty(), "no partial output on corrupted AP"); + Ok(()) + } + + /// an AP containing a nested + /// FU (type 29) is corrupted. + #[test] + fn test_h266_ap_rejects_nested_fu() -> Result<()> { + let trail = make_nal(0, 4); + let nested = make_nal(H266NALU_FRAGMENTATION_UNIT_TYPE, 4); + + let mut ap = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + for unit in [&trail, &nested] { + ap.extend_from_slice(&(unit.len() as u16).to_be_bytes()); + ap.extend_from_slice(unit); + } + + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let res = depack.depacketize(&ap, &mut out, &mut extra); + assert!(matches!(res, Err(PacketError::ErrH266CorruptedPacket))); + assert!(out.is_empty()); + Ok(()) + } + + /// a length field larger than + /// the remaining payload errors — even after valid leading units, + /// and without leaking partial output. + #[test] + fn test_h266_ap_rejects_truncated_tail() -> Result<()> { + let a = make_nal(0, 4); + let b = make_nal(1, 4); + + let mut ap = hdr(H266NALU_AGGREGATION_PACKET_TYPE, 0, 1).to_vec(); + for unit in [&a, &b] { + ap.extend_from_slice(&(unit.len() as u16).to_be_bytes()); + ap.extend_from_slice(unit); + } + // third unit claims 0x00ff bytes but provides 1 + ap.extend_from_slice(&[0x00, 0xFF, 0x42]); + + let mut depack = H266Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + let res = depack.depacketize(&ap, &mut out, &mut extra); + assert!(matches!(res, Err(PacketError::ErrShortPacket))); + assert!( + out.is_empty(), + "valid leading units must not leak from a malformed AP" + ); + Ok(()) + } + + /// a NAL with + /// F, Z, LayerId and TID all set survives fragmentation and + /// reassembly bit-exact (packetizer side; the depacketizer rejects + /// F=1 packets by policy). + #[test] + fn test_h266_fu_preserves_all_header_flags() -> Result<()> { + let mut nal = hdr(1, 1, 1).to_vec(); + nal[0] |= 0b1100_0000; // F + Z + nal.extend(std::iter::repeat(0x5A).take(400)); + + let mut pck = H266Packetizer::default(); + let packets = pck.packetize(100, &nal)?; + assert!(packets.len() > 1); + + for p in &packets { + let h = H266NALUHeader::new(p[0], p[1]); + assert!(h.f() && h.z(), "F/Z must be copied into FU headers"); + assert_eq!(h.layer_id(), 1); + assert_eq!(h.tid(), 1); + } + assert_eq!(reconstruct_from_fu_packets(&packets), nal); + Ok(()) + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266_profile.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266_profile.rs new file mode 100644 index 000000000..eabc88a46 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/h266_profile.rs @@ -0,0 +1,405 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// H.266/VVC profile, tier, and level combination. +/// +/// Represents the three SDP fmtp parameters `profile-id`, `tier-flag`, and +/// `level-id` as defined in RFC 9328 §7.2 and ITU-T H.266 Annex A. +/// +/// Note: the parameter *names* are identical to H.265 (RFC 7798), but the +/// value spaces differ. In particular the level-id encodings are disjoint: +/// H.265 uses `30 × level` (e.g. 3.1 → 93) while H.266 uses +/// `16 × major + 3 × minor` (e.g. 3.1 → 51), which is what allows SDP +/// parsing to tell the two codecs' fmtp lines apart. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct H266ProfileTierLevel { + profile: H266Profile, + tier: H266Tier, + level: H266Level, +} + +impl H266ProfileTierLevel { + // Inferred values mandated by RFC 9328 §7.2 when parameters are + // absent from SDP: profile-id => 1 (Main 10), tier-flag => 0, + // level-id => 51 (Level 3.1). + pub(crate) const FALLBACK: Self = Self { + profile: H266Profile::Main10, + tier: H266Tier::Main, + level: H266Level::Level3_1, + }; + + /// Construct a new H266ProfileTierLevel from profile_id, tier_flag, and level_id. + /// + /// Returns `Some(Self)` only if the provided parameters identify valid values. + pub(crate) fn new(profile_id: u8, tier_flag: u8, level_id: u8) -> Option { + let profile = H266Profile::from_id(profile_id)?; + let tier = H266Tier::from_flag(tier_flag)?; + let level = H266Level::from_id(level_id)?; + + Some(Self { + profile, + tier, + level, + }) + } + + /// Parse H266ProfileTierLevel from SDP fmtp parameters. + /// + /// Expects keys: "profile-id", "tier-flag", "level-id" + /// Returns None if any required parameter is missing or invalid. + pub(crate) fn from_fmtp(params: &HashMap) -> Option { + let profile_id: u8 = params.get("profile-id")?.parse().ok()?; + let tier_flag: u8 = params.get("tier-flag")?.parse().ok()?; + let level_id: u8 = params.get("level-id")?.parse().ok()?; + + Self::new(profile_id, tier_flag, level_id) + } + + /// Returns the H.266 profile (Main 10, Main 10 4:4:4, etc.). + pub fn profile(&self) -> H266Profile { + self.profile + } + + /// Returns the H.266 tier (Main or High). + pub fn tier(&self) -> H266Tier { + self.tier + } + + /// Returns the H.266 level (Level 3.1, Level 4.0, etc.). + pub fn level(&self) -> H266Level { + self.level + } + + /// Returns a copy with the level replaced. + pub fn with_level(self, level: H266Level) -> Self { + Self { level, ..self } + } + + /// Returns the numeric profile_id value for SDP serialization. + pub fn profile_id(&self) -> u8 { + self.profile.to_id() + } + + /// Returns the numeric tier_flag value for SDP serialization. + pub fn tier_flag(&self) -> u8 { + self.tier.to_flag() + } + + /// Returns the numeric level_id value for SDP serialization. + pub fn level_id(&self) -> u8 { + self.level.to_id() + } +} + +impl From<(u8, u8, u8)> for H266ProfileTierLevel { + fn from(value: (u8, u8, u8)) -> Self { + Self::new(value.0, value.1, value.2).unwrap_or(Self::FALLBACK) + } +} + +/// H.266 profile as defined in ITU-T H.266 Annex A +/// (`general_profile_idc` values, version 1 profiles). +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum H266Profile { + /// Main 10 profile (profile_id=1). + Main10, + /// Multilayer Main 10 profile (profile_id=17). + MultilayerMain10, + /// Main 10 4:4:4 profile (profile_id=33). + Main10_444, + /// Multilayer Main 10 4:4:4 profile (profile_id=49). + MultilayerMain10_444, + /// Main 10 Still Picture profile (profile_id=65). + Main10StillPicture, + /// Multilayer Main 10 Still Picture profile (profile_id=81). + MultilayerMain10StillPicture, + /// Main 10 4:4:4 Still Picture profile (profile_id=97). + Main10_444StillPicture, + /// Multilayer Main 10 4:4:4 Still Picture profile (profile_id=113). + MultilayerMain10_444StillPicture, +} + +impl H266Profile { + /// Convert from H.266 profile_id (general_profile_idc) value. + /// + /// See ITU-T H.266 Annex A for profile definitions. + fn from_id(profile_id: u8) -> Option { + match profile_id { + 1 => Some(Self::Main10), + 17 => Some(Self::MultilayerMain10), + 33 => Some(Self::Main10_444), + 49 => Some(Self::MultilayerMain10_444), + 65 => Some(Self::Main10StillPicture), + 81 => Some(Self::MultilayerMain10StillPicture), + 97 => Some(Self::Main10_444StillPicture), + 113 => Some(Self::MultilayerMain10_444StillPicture), + _ => None, + } + } + + /// Convert to H.266 profile_id (general_profile_idc) value. + pub fn to_id(self) -> u8 { + match self { + Self::Main10 => 1, + Self::MultilayerMain10 => 17, + Self::Main10_444 => 33, + Self::MultilayerMain10_444 => 49, + Self::Main10StillPicture => 65, + Self::MultilayerMain10StillPicture => 81, + Self::Main10_444StillPicture => 97, + Self::MultilayerMain10_444StillPicture => 113, + } + } +} + +/// H.266 tier (Main or High). +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum H266Tier { + /// Main tier (tier_flag=0). + Main, + /// High tier (tier_flag=1). + High, +} + +impl H266Tier { + /// Convert from H.266 tier_flag value. + /// + /// 0 = Main tier, 1 = High tier + fn from_flag(tier_flag: u8) -> Option { + match tier_flag { + 0 => Some(Self::Main), + 1 => Some(Self::High), + _ => None, + } + } + + /// Convert to H.266 tier_flag value. + pub fn to_flag(self) -> u8 { + match self { + Self::Main => 0, + Self::High => 1, + } + } +} + +/// H.266 level as defined in ITU-T H.266 Annex A. +/// +/// Level IDs (`general_level_idc`) encode as `16 × major + 3 × minor` +/// (e.g., Level 3.1 = 16×3 + 3×1 = 51). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[repr(u8)] +#[rustfmt::skip] +pub enum H266Level { + /// Level 1.0 (level_id=16). + Level1 = 16_u8, + /// Level 2.0 (level_id=32). + Level2 = 32_u8, + /// Level 2.1 (level_id=35). + Level2_1 = 35_u8, + /// Level 3.0 (level_id=48). + Level3 = 48_u8, + /// Level 3.1 (level_id=51). + Level3_1 = 51_u8, + /// Level 4.0 (level_id=64). + Level4 = 64_u8, + /// Level 4.1 (level_id=67). + Level4_1 = 67_u8, + /// Level 5.0 (level_id=80). + Level5 = 80_u8, + /// Level 5.1 (level_id=83). + Level5_1 = 83_u8, + /// Level 5.2 (level_id=86). + Level5_2 = 86_u8, + /// Level 6.0 (level_id=96). + Level6 = 96_u8, + /// Level 6.1 (level_id=99). + Level6_1 = 99_u8, + /// Level 6.2 (level_id=102). + Level6_2 = 102_u8, + /// Level 6.3 (level_id=105). + Level6_3 = 105_u8, + /// Level 15.5 (level_id=255) — the special maximum level used by the + /// still-picture profiles (16*15 + 3*5 = 255). + Level15_5 = 255_u8, +} + +impl H266Level { + /// Returns the ordinal position (0-14) representing capability order. + pub fn ordinal(self) -> usize { + match self { + Self::Level1 => 0, + Self::Level2 => 1, + Self::Level2_1 => 2, + Self::Level3 => 3, + Self::Level3_1 => 4, + Self::Level4 => 5, + Self::Level4_1 => 6, + Self::Level5 => 7, + Self::Level5_1 => 8, + Self::Level5_2 => 9, + Self::Level6 => 10, + Self::Level6_1 => 11, + Self::Level6_2 => 12, + Self::Level6_3 => 13, + Self::Level15_5 => 14, + } + } + + /// Convert from H.266 level_id value. + fn from_id(level_id: u8) -> Option { + use H266Level::*; + match level_id { + x if (Level1 as u8) == x => Some(Level1), + x if (Level2 as u8) == x => Some(Level2), + x if (Level2_1 as u8) == x => Some(Level2_1), + x if (Level3 as u8) == x => Some(Level3), + x if (Level3_1 as u8) == x => Some(Level3_1), + x if (Level4 as u8) == x => Some(Level4), + x if (Level4_1 as u8) == x => Some(Level4_1), + x if (Level5 as u8) == x => Some(Level5), + x if (Level5_1 as u8) == x => Some(Level5_1), + x if (Level5_2 as u8) == x => Some(Level5_2), + x if (Level6 as u8) == x => Some(Level6), + x if (Level6_1 as u8) == x => Some(Level6_1), + x if (Level6_2 as u8) == x => Some(Level6_2), + x if (Level6_3 as u8) == x => Some(Level6_3), + x if (Level15_5 as u8) == x => Some(Level15_5), + _ => None, + } + } + + /// Convert to H.266 level_id value. + pub fn to_id(self) -> u8 { + self as u8 + } +} + +impl PartialOrd for H266Level { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for H266Level { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.ordinal().cmp(&other.ordinal()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_h266_profile_tier_level_new() { + // Valid combination + let ptl = H266ProfileTierLevel::new(1, 0, 51); + assert!(ptl.is_some()); + let ptl = ptl.unwrap(); + assert_eq!(ptl.profile(), H266Profile::Main10); + assert_eq!(ptl.tier(), H266Tier::Main); + assert_eq!(ptl.level(), H266Level::Level3_1); + + // Invalid profile_id + let ptl = H266ProfileTierLevel::new(99, 0, 51); + assert!(ptl.is_none()); + + // Invalid tier_flag + let ptl = H266ProfileTierLevel::new(1, 5, 51); + assert!(ptl.is_none()); + + // Invalid level_id (93 is an H.265 level encoding, not H.266) + let ptl = H266ProfileTierLevel::new(1, 0, 93); + assert!(ptl.is_none()); + } + + #[test] + fn test_h266_profile_from_id() { + assert_eq!(H266Profile::from_id(1), Some(H266Profile::Main10)); + assert_eq!(H266Profile::from_id(33), Some(H266Profile::Main10_444)); + assert_eq!( + H266Profile::from_id(65), + Some(H266Profile::Main10StillPicture) + ); + assert_eq!( + H266Profile::from_id(81), + Some(H266Profile::MultilayerMain10StillPicture) + ); + assert_eq!( + H266Profile::from_id(113), + Some(H266Profile::MultilayerMain10_444StillPicture) + ); + assert_eq!(H266Profile::from_id(2), None); + } + + #[test] + fn test_h266_tier_from_flag() { + assert_eq!(H266Tier::from_flag(0), Some(H266Tier::Main)); + assert_eq!(H266Tier::from_flag(1), Some(H266Tier::High)); + assert_eq!(H266Tier::from_flag(2), None); + } + + #[test] + fn test_h266_level_from_id() { + assert_eq!(H266Level::from_id(16), Some(H266Level::Level1)); + assert_eq!(H266Level::from_id(51), Some(H266Level::Level3_1)); + assert_eq!(H266Level::from_id(105), Some(H266Level::Level6_3)); + assert_eq!(H266Level::from_id(255), Some(H266Level::Level15_5)); + // H.265 encodings must NOT parse as H.266 levels. + assert_eq!(H266Level::from_id(93), None); + assert_eq!(H266Level::from_id(180), None); + } + + #[test] + fn test_h266_level_ordering() { + assert!(H266Level::Level1 < H266Level::Level2); + assert!(H266Level::Level3_1 < H266Level::Level4); + assert!(H266Level::Level6_2 < H266Level::Level6_3); + assert!(H266Level::Level6_3 < H266Level::Level15_5); + assert_eq!(H266Level::Level3_1.ordinal(), 4); + } + + #[test] + fn test_h266_profile_tier_level_from_fmtp() { + let mut params = HashMap::new(); + params.insert("profile-id".to_string(), "1".to_string()); + params.insert("tier-flag".to_string(), "0".to_string()); + params.insert("level-id".to_string(), "51".to_string()); + + let ptl = H266ProfileTierLevel::from_fmtp(¶ms); + assert!(ptl.is_some()); + let ptl = ptl.unwrap(); + assert_eq!(ptl.profile(), H266Profile::Main10); + assert_eq!(ptl.tier(), H266Tier::Main); + assert_eq!(ptl.level(), H266Level::Level3_1); + + // Missing parameter + let mut params = HashMap::new(); + params.insert("profile-id".to_string(), "1".to_string()); + params.insert("tier-flag".to_string(), "0".to_string()); + let ptl = H266ProfileTierLevel::from_fmtp(¶ms); + assert!(ptl.is_none()); + } + + #[test] + fn test_fallback_values() { + let fallback = H266ProfileTierLevel::FALLBACK; + assert_eq!(fallback.profile(), H266Profile::Main10); + assert_eq!(fallback.tier(), H266Tier::Main); + assert_eq!(fallback.level(), H266Level::Level3_1); + } + + #[test] + fn test_h265_h266_level_spaces_disjoint() { + // The disambiguation in SDP parsing relies on this property. + let h265_levels = [30, 60, 63, 90, 93, 120, 123, 150, 153, 156, 180, 183, 186]; + for id in h265_levels { + assert_eq!( + H266Level::from_id(id), + None, + "level-id {id} must not be valid H.266" + ); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/mod.rs new file mode 100644 index 000000000..88c518712 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/mod.rs @@ -0,0 +1,517 @@ +#![allow(clippy::type_complexity)] + +use std::fmt; +use std::panic::UnwindSafe; + +use crate::format::Codec; +use crate::sdp::MediaType; + +use crate::rtp::vla::encode_leb_u63; + +mod av1; +pub use av1::Av1CodecExtra; +pub use av1::detect_av1_keyframe; +use av1::{Av1Depacketizer, Av1Packetizer}; + +mod g7xx; +use g7xx::{G711Depacketizer, G711Packetizer, G722Packetizer}; + +mod h264; +pub use h264::H264CodecExtra; +pub use h264::detect_h264_keyframe; +pub use h264::{H264Depacketizer, H264Packetizer}; + +mod h264_profile; +pub(crate) use h264_profile::H264ProfileLevel; + +mod h265; +pub use h265::H265CodecExtra; +use h265::H265Depacketizer; +pub use h265::H265Packetizer; +pub use h265::detect_h265_keyframe; + +mod h265_profile; +pub use h265_profile::{H265Level, H265Profile, H265ProfileTierLevel, H265Tier}; + +mod h266; +pub use h266::H266CodecExtra; +use h266::H266Depacketizer; +pub use h266::H266Packetizer; +pub use h266::detect_h266_keyframe; + +mod h266_profile; +pub use h266_profile::{H266Level, H266Profile, H266ProfileTierLevel, H266Tier}; + +mod opus; +pub use opus::{OpusDepacketizer, OpusPacketizer}; + +mod vp8; +pub use vp8::Vp8CodecExtra; +pub use vp8::Vp8Descriptor; +pub use vp8::Vp8DescriptorError; +pub use vp8::Vp8Patch; +pub use vp8::Vp8PatchBuilder; +pub use vp8::Vp8PatchError; +pub use vp8::detect_vp8_keyframe; +pub use vp8::{Vp8Depacketizer, Vp8Packetizer}; + +mod vp9; +pub use vp9::Vp9CodecExtra; +pub use vp9::Vp9PacketizerMode; +pub use vp9::detect_vp9_keyframe; +use vp9::{Vp9Depacketizer, Vp9Packetizer}; + +mod null; +use null::{NullDepacketizer, NullPacketizer}; + +mod comfort_noise; +use comfort_noise::{ComfortNoiseDepacketizer, ComfortNoisePacketizer}; + +mod buffer_rx; +pub(crate) use buffer_rx::{DepacketizingBuffer, RtpMeta}; + +mod contiguity; +mod contiguity_vp8; +mod contiguity_vp9; +mod error; + +mod payload; +pub(crate) use payload::Payloader; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Types of media. +pub enum MediaKind { + /// Audio media. + Audio, + /// Video media. + Video, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(MediaKind); + +impl MediaKind { + /// Tests if this is `MediaKind::Audio` + pub fn is_audio(&self) -> bool { + *self == MediaKind::Audio + } + + /// Tests if this is `MediaKind::Video` + pub fn is_video(&self) -> bool { + *self == MediaKind::Video + } +} + +/// Packetizes some bytes for use as RTP packet. +/// +/// ## Unversioned API surface +/// +/// This trait is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +pub trait Packetizer: fmt::Debug { + /// Chunk the data up into RTP packets. + fn packetize(&mut self, mtu: usize, b: &[u8]) -> Result>, PacketError>; + + /// Tell if this is the last packet of a frame. + fn is_marker(&mut self, data: &[u8], previous: Option<&[u8]>, last: bool) -> bool; +} + +/// Codec specific information +/// +/// Contains additional codec specific information which are deemed useful for +/// managing and repackaging the frame +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum CodecExtra { + /// No extra information available + None, + /// Codec extra parameters for VP8. + Vp8(Vp8CodecExtra), + /// Codec extra parameters for VP9. + Vp9(Vp9CodecExtra), + /// Codec extra parameters for H264. + H264(H264CodecExtra), + /// Codec extra parameters for AV1, + Av1(Av1CodecExtra), + /// Codec extra parameters for H265. + H265(H265CodecExtra), + /// Codec extra parameters for H266 (VVC). Prototype. + H266(H266CodecExtra), +} + +/// Depacketizes an RTP payload. +/// +/// Removes any RTP specific data from the payload. +/// +/// ## Unversioned API surface +/// +/// This trait is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +pub trait Depacketizer: fmt::Debug { + /// Provide a size hint for the `out: &mut Vec` parameter in [`Depacketizer::depacketize`]. + /// The [`packets_size`] parameter is the sum of the size of all packets that will be processed in + /// subsequent calls to [`Depacketizer::depacketize`]. + /// This is used to allocate the vector with upfront capacity. + /// Return [`None`] if it cannot be determined. + fn out_size_hint(&self, packets_size: usize) -> Option; + + /// Unpack the RTP packet into a provided `Vec`. + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + codec_extra: &mut CodecExtra, + ) -> Result<(), PacketError>; + + /// Checks if the packet is at the beginning of a partition. + /// + /// Returns false if the result could not be determined. + fn is_partition_head(&self, packet: &[u8]) -> bool; + + /// Checks if the packet is at the end of a partition. + /// + /// Returns false if the result could not be determined. + fn is_partition_tail(&self, marker: bool, packet: &[u8]) -> bool; +} + +pub use error::PacketError; + +/// Helper to replace Bytes. Provides get_u8 and get_u16 over some buffer of bytes. +pub(crate) trait BitRead { + fn remaining_bits(&self) -> usize; + fn remaining_bytes(&self) -> usize; + fn byte_offset(&self) -> usize; + fn get_u8(&mut self) -> Option; + fn get_u16(&mut self) -> Option; + fn get_remaining(&mut self) -> &[u8]; + fn get_bytes(&mut self, count: usize) -> Option<&[u8]>; + fn get_leb128(&mut self) -> Option; + fn skip_bytes(&mut self, count: usize); +} + +impl BitRead for (&[u8], usize) { + #[inline(always)] + fn remaining_bits(&self) -> usize { + (self.0.len() * 8).saturating_sub(self.1) + } + + #[inline(always)] + fn remaining_bytes(&self) -> usize { + self.remaining_bits() / 8 + } + + #[inline(always)] + fn byte_offset(&self) -> usize { + self.1 / 8 + } + + #[inline(always)] + fn get_u8(&mut self) -> Option { + if self.remaining_bits() < 8 { + return None; + } + + let offs = self.1 / 8; + let shift = (self.1 % 8) as u32; + self.1 += 8; + + let mut n = self.0[offs]; + + if shift > 0 { + n <<= shift; + n |= self.0[offs + 1] >> (8 - shift) + } + + Some(n) + } + + fn get_u16(&mut self) -> Option { + if self.remaining_bits() < 16 { + return None; + } + Some(u16::from_be_bytes([self.get_u8()?, self.get_u8()?])) + } + + fn get_remaining(&mut self) -> &[u8] { + let offset = self.1 / 8; + self.1 = self.0.len() * 8; + &self.0[offset..] + } + + fn get_bytes(&mut self, count: usize) -> Option<&[u8]> { + let count_bits = count.saturating_mul(8); + if self.remaining_bits() < count_bits { + return None; + } + let offset = self.1 / 8; + self.1 += count_bits; + Some(&self.0[offset..offset + count]) + } + + fn get_leb128(&mut self) -> Option { + let mut temp_value: usize = 0; + + for i in (0..64).step_by(7) { + if self.remaining_bits() < 8 { + return None; + } + let byte = self.get_u8().unwrap(); + temp_value |= ((byte & 0x7f) as usize) << i; + + if byte < 0x80 { + return Some(temp_value); + } + } + + None + } + + fn skip_bytes(&mut self, count: usize) { + let bits = count.saturating_mul(8); + if self.remaining_bits() < bits { + return; + } + + self.1 += bits; + } +} + +#[derive(Debug)] +pub(crate) enum CodecPacketizer { + #[allow(unused)] + G711(G711Packetizer), + #[allow(unused)] + G722(G722Packetizer), + H264(H264Packetizer), + H265(H265Packetizer), + H266(H266Packetizer), + Opus(OpusPacketizer), + ComfortNoise(ComfortNoisePacketizer), + Vp8(Vp8Packetizer), + Vp9(Vp9Packetizer), + Av1(Av1Packetizer), + Null(NullPacketizer), + #[allow(unused)] + Boxed(Box), +} + +#[derive(Debug)] +pub(crate) enum CodecDepacketizer { + G711(G711Depacketizer), + H264(H264Depacketizer), + H265(H265Depacketizer), + H266(H266Depacketizer), + Opus(OpusDepacketizer), + ComfortNoise(ComfortNoiseDepacketizer), + Vp8(Vp8Depacketizer), + Vp9(Vp9Depacketizer), + Av1(Av1Depacketizer), + Null(NullDepacketizer), + #[allow(unused)] + Boxed(Box), +} + +impl CodecPacketizer { + pub(crate) fn new(codec: Codec, vp9_mode: Vp9PacketizerMode) -> Self { + match codec { + Codec::Opus => CodecPacketizer::Opus(OpusPacketizer), + Codec::PCMU => CodecPacketizer::G711(G711Packetizer::default()), + Codec::PCMA => CodecPacketizer::G711(G711Packetizer::default()), + Codec::G722 => CodecPacketizer::G722(G722Packetizer::default()), + Codec::CN => CodecPacketizer::ComfortNoise(ComfortNoisePacketizer), + Codec::H264 => CodecPacketizer::H264(H264Packetizer::default()), + Codec::H265 => CodecPacketizer::H265(H265Packetizer::default()), + Codec::H266 => CodecPacketizer::H266(H266Packetizer::default()), + Codec::Vp8 => CodecPacketizer::Vp8(Vp8Packetizer::default()), + Codec::Vp9 => CodecPacketizer::Vp9(Vp9Packetizer::with_mode(vp9_mode)), + Codec::Av1 => CodecPacketizer::Av1(Av1Packetizer::default()), + Codec::Null => CodecPacketizer::Null(NullPacketizer), + Codec::Rtx => panic!("Cant instantiate packetizer for RTX codec"), + Codec::Unknown => panic!("Cant instantiate packetizer for unknown codec"), + } + } +} + +impl From for CodecPacketizer { + fn from(c: Codec) -> Self { + CodecPacketizer::new(c, Vp9PacketizerMode::default()) + } +} + +impl From for CodecDepacketizer { + fn from(c: Codec) -> Self { + match c { + Codec::Opus => CodecDepacketizer::Opus(OpusDepacketizer), + Codec::PCMU => CodecDepacketizer::G711(G711Depacketizer), + Codec::PCMA => CodecDepacketizer::G711(G711Depacketizer), + Codec::G722 => CodecDepacketizer::G711(G711Depacketizer), + Codec::CN => CodecDepacketizer::ComfortNoise(ComfortNoiseDepacketizer), + Codec::H264 => CodecDepacketizer::H264(H264Depacketizer::default()), + Codec::H265 => CodecDepacketizer::H265(H265Depacketizer::default()), + Codec::H266 => CodecDepacketizer::H266(H266Depacketizer::default()), + Codec::Vp8 => CodecDepacketizer::Vp8(Vp8Depacketizer::default()), + Codec::Vp9 => CodecDepacketizer::Vp9(Vp9Depacketizer::default()), + Codec::Av1 => CodecDepacketizer::Av1(Av1Depacketizer::default()), + Codec::Null => CodecDepacketizer::Null(NullDepacketizer), + Codec::Rtx => panic!("Cant instantiate depacketizer for RTX codec"), + Codec::Unknown => panic!("Cant instantiate depacketizer for unknown codec"), + } + } +} + +impl Packetizer for CodecPacketizer { + fn packetize(&mut self, mtu: usize, b: &[u8]) -> Result>, PacketError> { + use CodecPacketizer::*; + match self { + G711(v) => v.packetize(mtu, b), + G722(v) => v.packetize(mtu, b), + H264(v) => v.packetize(mtu, b), + H265(v) => v.packetize(mtu, b), + H266(v) => v.packetize(mtu, b), + Opus(v) => v.packetize(mtu, b), + ComfortNoise(v) => v.packetize(mtu, b), + Vp8(v) => v.packetize(mtu, b), + Vp9(v) => v.packetize(mtu, b), + Av1(v) => v.packetize(mtu, b), + Null(v) => v.packetize(mtu, b), + Boxed(v) => v.packetize(mtu, b), + } + } + + fn is_marker(&mut self, data: &[u8], previous: Option<&[u8]>, last: bool) -> bool { + match self { + CodecPacketizer::G711(v) => v.is_marker(data, previous, last), + CodecPacketizer::G722(v) => v.is_marker(data, previous, last), + CodecPacketizer::Opus(v) => v.is_marker(data, previous, last), + CodecPacketizer::ComfortNoise(v) => v.is_marker(data, previous, last), + CodecPacketizer::H264(v) => v.is_marker(data, previous, last), + CodecPacketizer::H265(v) => v.is_marker(data, previous, last), + CodecPacketizer::H266(v) => v.is_marker(data, previous, last), + CodecPacketizer::Vp8(v) => v.is_marker(data, previous, last), + CodecPacketizer::Vp9(v) => v.is_marker(data, previous, last), + CodecPacketizer::Av1(v) => v.is_marker(data, previous, last), + CodecPacketizer::Null(v) => v.is_marker(data, previous, last), + CodecPacketizer::Boxed(v) => v.is_marker(data, previous, last), + } + } +} + +impl Depacketizer for CodecDepacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + use CodecDepacketizer::*; + match self { + H264(v) => v.out_size_hint(packets_size), + H265(v) => v.out_size_hint(packets_size), + H266(v) => v.out_size_hint(packets_size), + Opus(v) => v.out_size_hint(packets_size), + ComfortNoise(v) => v.out_size_hint(packets_size), + G711(v) => v.out_size_hint(packets_size), + Vp8(v) => v.out_size_hint(packets_size), + Vp9(v) => v.out_size_hint(packets_size), + Av1(v) => v.out_size_hint(packets_size), + Null(v) => v.out_size_hint(packets_size), + Boxed(v) => v.out_size_hint(packets_size), + } + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + use CodecDepacketizer::*; + match self { + H264(v) => v.depacketize(packet, out, extra), + H265(v) => v.depacketize(packet, out, extra), + H266(v) => v.depacketize(packet, out, extra), + Opus(v) => v.depacketize(packet, out, extra), + ComfortNoise(v) => v.depacketize(packet, out, extra), + G711(v) => v.depacketize(packet, out, extra), + Vp8(v) => v.depacketize(packet, out, extra), + Vp9(v) => v.depacketize(packet, out, extra), + Av1(v) => v.depacketize(packet, out, extra), + Null(v) => v.depacketize(packet, out, extra), + Boxed(v) => v.depacketize(packet, out, extra), + } + } + + fn is_partition_head(&self, packet: &[u8]) -> bool { + use CodecDepacketizer::*; + match self { + H264(v) => v.is_partition_head(packet), + H265(v) => v.is_partition_head(packet), + H266(v) => v.is_partition_head(packet), + Opus(v) => v.is_partition_head(packet), + ComfortNoise(v) => v.is_partition_head(packet), + G711(v) => v.is_partition_head(packet), + Vp8(v) => v.is_partition_head(packet), + Vp9(v) => v.is_partition_head(packet), + Av1(v) => v.is_partition_head(packet), + Null(v) => v.is_partition_head(packet), + Boxed(v) => v.is_partition_head(packet), + } + } + + fn is_partition_tail(&self, marker: bool, packet: &[u8]) -> bool { + use CodecDepacketizer::*; + match self { + H264(v) => v.is_partition_tail(marker, packet), + H265(v) => v.is_partition_tail(marker, packet), + H266(v) => v.is_partition_tail(marker, packet), + Opus(v) => v.is_partition_tail(marker, packet), + ComfortNoise(v) => v.is_partition_tail(marker, packet), + G711(v) => v.is_partition_tail(marker, packet), + Vp8(v) => v.is_partition_tail(marker, packet), + Vp9(v) => v.is_partition_tail(marker, packet), + Av1(v) => v.is_partition_tail(marker, packet), + Null(v) => v.is_partition_tail(marker, packet), + Boxed(v) => v.is_partition_tail(marker, packet), + } + } +} + +impl From for MediaKind { + fn from(v: MediaType) -> Self { + match v { + MediaType::Audio => MediaKind::Audio, + MediaType::Video => MediaKind::Video, + _ => panic!("Not MediaType::Audio or Video"), + } + } +} + +impl fmt::Display for MediaKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MediaKind::Audio => write!(f, "audio"), + MediaKind::Video => write!(f, "video"), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn comfort_noise_packetizer_preserves_payload_and_clears_marker() { + let payload = [42, 128, 64]; + let mut packetizer = CodecPacketizer::from(Codec::CN); + + assert_eq!( + packetizer.packetize(1200, &payload).unwrap(), + vec![payload.to_vec()] + ); + assert!(!packetizer.is_marker(&payload, None, true)); + + let mut depacketizer = CodecDepacketizer::from(Codec::CN); + let mut output = Vec::new(); + depacketizer + .depacketize(&payload, &mut output, &mut CodecExtra::None) + .unwrap(); + assert_eq!(output, payload); + assert!(depacketizer.is_partition_head(&payload)); + assert!(depacketizer.is_partition_tail(false, &payload)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/null.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/null.rs new file mode 100644 index 000000000..029a0677b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/null.rs @@ -0,0 +1,43 @@ +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; + +#[derive(Debug)] +pub struct NullPacketizer; + +#[derive(Debug)] +pub struct NullDepacketizer; + +impl Packetizer for NullPacketizer { + fn packetize(&mut self, _mtu: usize, b: &[u8]) -> Result>, PacketError> { + Ok(vec![b.to_vec()]) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, _last: bool) -> bool { + unreachable!("rtp_mode doesn't use is_marker") + } +} + +impl Depacketizer for NullDepacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + _codec_extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + out.extend_from_slice(packet); + Ok(()) + } + + fn is_partition_head(&self, _packet: &[u8]) -> bool { + // For rtp-mode since each packet is stand alone, it is both a partition head and tail. + true + } + + fn is_partition_tail(&self, _marker: bool, _packet: &[u8]) -> bool { + // For rtp-mode since each packet is stand alone, it is both a partition head and tail. + true + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/opus.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/opus.rs new file mode 100644 index 000000000..1135b4efb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/opus.rs @@ -0,0 +1,149 @@ +use super::{CodecExtra, Depacketizer, PacketError, Packetizer}; + +/// Packetizes Opus RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(Default, Debug, Copy, Clone)] +pub struct OpusPacketizer; + +impl Packetizer for OpusPacketizer { + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu == 0 { + return Ok(vec![]); + } + + let mut out = vec![]; + + let mut cur = 0; + while cur < payload.len() { + let min = mtu.min(payload.len() - cur); + out.push(payload[cur..(cur + min)].to_vec()); + cur += mtu; + } + + Ok(out) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, _last: bool) -> bool { + // TODO: dtx + false + } +} + +/// Depacketizes Opus RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(PartialEq, Eq, Debug, Default, Clone)] +pub struct OpusDepacketizer; + +impl Depacketizer for OpusDepacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + _: &mut CodecExtra, + ) -> Result<(), PacketError> { + if !packet.is_empty() { + out.extend_from_slice(packet); + } + + Ok(()) + } + + fn is_partition_head(&self, _payload: &[u8]) -> bool { + true + } + + fn is_partition_tail(&self, _marker: bool, _payload: &[u8]) -> bool { + true + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_opus_unmarshal() -> Result<(), PacketError> { + let mut pck = OpusDepacketizer; + let mut extra = CodecExtra::None; + + // Empty packet + let empty_bytes = &[]; + let mut out = Vec::new(); + let result = pck.depacketize(empty_bytes, &mut out, &mut extra); + assert!( + result.is_ok(), + "Result should not be err in case of empty packet" + ); + + // Normal packet + let raw_bytes: &[u8] = &[0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x90]; + let mut out = Vec::new(); + pck.depacketize(raw_bytes, &mut out, &mut extra)?; + assert_eq!(raw_bytes, &out, "Payload must be same"); + + Ok(()) + } + + #[test] + fn test_opus_payload() -> Result<(), PacketError> { + let mut pck = OpusPacketizer; + let empty = &[]; + let payload = &[0x90, 0x90, 0x90]; + + // Positive MTU, empty payload + let result = pck.packetize(1, empty)?; + assert!(result.is_empty(), "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(1, payload)?; + assert_eq!(result.len(), 3, "Generated payload should be 3"); + + // Positive MTU, small payload + let result = pck.packetize(2, payload)?; + assert_eq!(result.len(), 2, "Generated payload should be the 2"); + + Ok(()) + } + + #[test] + fn test_opus_is_partition_head() -> Result<(), PacketError> { + let opus = OpusDepacketizer; + //"NormalPacket" + assert!( + opus.is_partition_head(&[0x00, 0x00]), + "All OPUS RTP packet should be the head of a new partition" + ); + + Ok(()) + } + + #[test] + fn packetize_respects_mtu() -> Result<(), PacketError> { + let payload = vec![0xABu8; 2000]; + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = OpusPacketizer; + let pkts = pck.packetize(mtu, &payload)?; + assert!(!pkts.is_empty(), "Opus produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "Opus packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/payload.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/payload.rs new file mode 100644 index 000000000..4978f744d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/payload.rs @@ -0,0 +1,105 @@ +use crate::format::Vp9PacketizerMode; +use crate::format::{Codec, CodecSpec}; +use crate::media::ToPayload; +use crate::rtp::vla::VideoLayersAllocation; +use crate::rtp_::Frequency; +use crate::streams::{RtpWrite, StreamTx}; + +use super::PacketError; +use super::{CodecPacketizer, Packetizer}; + +#[derive(Debug)] +pub struct Payloader { + pack: CodecPacketizer, + clock_rate: Frequency, + allow_talkspurt_marker: bool, +} + +impl Payloader { + pub(crate) fn new(spec: CodecSpec, vp9_mode: Vp9PacketizerMode) -> Self { + let mut pack = CodecPacketizer::new(spec.codec, vp9_mode); + + // Enable DONL for H.265 when sprop-max-don-diff > 0 (RFC 7798 §7.1) + if let CodecPacketizer::H265(ref mut h265) = pack { + if spec.format.sprop_max_don_diff.unwrap_or(0) > 0 { + h265.with_donl(true); + } + } + + // Enable DONL for H.266 when sprop-max-don-diff > 0 (RFC 9328 §7.2) + if let CodecPacketizer::H266(ref mut h266) = pack { + if spec.format.sprop_max_don_diff.unwrap_or(0) > 0 { + h266.with_donl(true); + } + } + + Payloader { + pack, + clock_rate: spec.rtp_clock_rate(), + allow_talkspurt_marker: spec.codec != Codec::CN, + } + } + + pub(crate) fn push_sample( + &mut self, + to_payload: ToPayload, + mtu: usize, + is_audio: bool, + stream: &mut StreamTx, + ) -> Result<(), PacketError> { + let ToPayload { + pt, + wallclock, + rtp_time, + data, + start_of_talk_spurt, + ext_vals, + .. + } = to_payload; + + let chunks = self.pack.packetize(mtu, data.as_ref())?; + let len = chunks.len(); + + for (idx, data) in chunks.into_iter().enumerate() { + let last = idx == len - 1; + let first = idx == 0; + + let previous_data = stream.last_packet(); + let marker = self.pack.is_marker(data.as_slice(), previous_data, last) + || (is_audio && self.allow_talkspurt_marker && start_of_talk_spurt); + + let seq_no = stream.next_seq_no(); + + // TODO: delegate to self.pack to decide whether this packet is nackable. + let nackable = !is_audio; + + let mut pkt_ext_vals = ext_vals.clone(); + + if !first { + pkt_ext_vals.abs_capture_time = None; + pkt_ext_vals.user_values.remove::(); + } + + if !last { + pkt_ext_vals.video_orientation = None; + pkt_ext_vals.video_content_type = None; + pkt_ext_vals.video_timing = None; + } + + stream.write_rtp( + RtpWrite::new( + pt, + seq_no, + rtp_time.rebase(self.clock_rate).numer() as u32, + wallclock, + data, + ) + .marker(marker) + .ext_vals(pkt_ext_vals) + .nackable(nackable), + ); + } + + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp8.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp8.rs new file mode 100644 index 000000000..6c49c93db --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp8.rs @@ -0,0 +1,1264 @@ +use std::error::Error; +use std::fmt; + +use crate::rtp_::{extend_u7, extend_u8, extend_u15}; + +use super::{BitRead, CodecExtra, Depacketizer, PacketError, Packetizer}; + +pub const VP8_HEADER_SIZE: usize = 1; + +const PICTURE_ID_SHORT_MAX: u16 = 0x7f; // 7-bit PictureID, excluding the M bit +const PICTURE_ID_MAX: u16 = 0x7fff; // 15-bit PictureID, excluding the M bit +const PICTURE_ID_SHORT_MASK: u8 = PICTURE_ID_SHORT_MAX as u8; +const KEY_IDX_MASK: u8 = 0x1f; // 5-bit KEYIDX mask within the TID/Y/KEYIDX octet +const VP8_PATCH_BYTES: usize = 4; // two PictureID bytes, TL0PICIDX and KEYIDX + +/// Validated byte replacements for a VP8 payload descriptor. +/// +/// Build this from [`Vp8Descriptor::patch`] when the caller already parsed the +/// payload descriptor before fanout. The patch is fixed-size and never changes +/// payload length. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Vp8Patch { + len: u8, + offsets: [u8; VP8_PATCH_BYTES], + bytes: [u8; VP8_PATCH_BYTES], +} + +impl Vp8Patch { + fn new() -> Self { + Self { + len: 0, + offsets: [0; VP8_PATCH_BYTES], + bytes: [0; VP8_PATCH_BYTES], + } + } + + fn push(&mut self, offset: usize, byte: u8) { + let index = usize::from(self.len); + debug_assert!(index < self.offsets.len()); + debug_assert!(offset <= usize::from(u8::MAX)); + + self.offsets[index] = offset as u8; + self.bytes[index] = byte; + self.len += 1; + } + + /// Copy `payload` into `dst` while applying VP8 descriptor byte replacements. + /// + /// # Panics + /// + /// Panics if `dst` is not the same length as `payload`. + pub(crate) fn copy_to(&self, payload: &[u8], dst: &mut [u8]) { + assert_eq!(payload.len(), dst.len()); + + dst.copy_from_slice(payload); + + for i in 0..usize::from(self.len) { + dst[usize::from(self.offsets[i])] = self.bytes[i]; + } + } +} + +/// Errors returned when a VP8 payload descriptor cannot be parsed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Vp8DescriptorError { + /// The payload is too short to contain a valid VP8 payload descriptor. + ShortPayload, + /// The payload descriptor violates VP8 payload descriptor rules. + MalformedDescriptor, +} + +impl fmt::Display for Vp8DescriptorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ShortPayload => write!(f, "VP8 payload is too short"), + Self::MalformedDescriptor => write!(f, "VP8 payload descriptor is malformed"), + } + } +} + +impl Error for Vp8DescriptorError {} + +/// Errors returned when a VP8 payload descriptor patch cannot be built. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Vp8PatchError { + /// The payload descriptor does not contain a PictureID field. + PictureIdMissing, + /// The requested PictureID does not fit the existing descriptor width. + PictureIdTooLarge, + /// The payload descriptor does not contain a TL0PICIDX field. + Tl0PicIdxMissing, + /// The payload descriptor does not contain a KEYIDX field. + KeyIdxMissing, + /// The requested KEYIDX does not fit the five-bit KEYIDX field. + KeyIdxTooLarge, +} + +impl fmt::Display for Vp8PatchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::PictureIdMissing => write!(f, "VP8 payload descriptor has no PictureID"), + Self::PictureIdTooLarge => write!(f, "VP8 PictureID does not fit descriptor width"), + Self::Tl0PicIdxMissing => write!(f, "VP8 payload descriptor has no TL0PICIDX"), + Self::KeyIdxMissing => write!(f, "VP8 payload descriptor has no KEYIDX"), + Self::KeyIdxTooLarge => write!(f, "VP8 KEYIDX must fit in 5 bits"), + } + } +} + +impl Error for Vp8PatchError {} + +/// Builder for fixed-size VP8 payload descriptor patches. +#[derive(Clone, Copy, Debug)] +pub struct Vp8PatchBuilder { + descriptor: Vp8Descriptor, + picture_id: Option, + tl0_pic_idx: Option, + key_idx: Option, +} + +impl Vp8PatchBuilder { + /// Rewrite the VP8 PictureID field. + /// + /// The parsed descriptor decides whether the existing descriptor uses the + /// 7-bit or 15-bit PictureID representation. Rewriting preserves that + /// representation. + /// + /// [`Vp8PatchBuilder::build`] returns [`Vp8PatchError::PictureIdTooLarge`] + /// if the value does not fit the existing representation. + pub fn picture_id(mut self, picture_id: u16) -> Self { + self.picture_id = Some(picture_id); + self + } + + /// Rewrite the VP8 TL0PICIDX field. + /// + /// The rewrite is valid only when the payload descriptor already contains an + /// TL0PICIDX field. + pub fn tl0_pic_idx(mut self, tl0_pic_idx: u8) -> Self { + self.tl0_pic_idx = Some(tl0_pic_idx); + self + } + + /// Rewrite the VP8 KEYIDX field. + /// + /// Only the lower five KEYIDX bits are replaced. The TID and Y bits in the + /// same descriptor octet are preserved. + /// + /// [`Vp8PatchBuilder::build`] returns [`Vp8PatchError::KeyIdxTooLarge`] if + /// the value does not fit the five-bit KEYIDX field. + pub fn key_idx(mut self, key_idx: u8) -> Self { + self.key_idx = Some(key_idx); + self + } + + /// Build a validated fixed-size VP8 descriptor patch. + /// + /// # Errors + /// + /// Returns [`Vp8PatchError::PictureIdMissing`], + /// [`Vp8PatchError::Tl0PicIdxMissing`] or [`Vp8PatchError::KeyIdxMissing`] + /// when the requested rewrite targets a descriptor field that is absent. + /// Returns [`Vp8PatchError::PictureIdTooLarge`] or + /// [`Vp8PatchError::KeyIdxTooLarge`] when a requested rewrite value does + /// not fit the field representation. + pub fn build(self) -> Result { + let mut patch = Vp8Patch::new(); + + if let Some(picture_id) = self.picture_id { + let Some(parsed) = self.descriptor.picture_id else { + return Err(Vp8PatchError::PictureIdMissing); + }; + if picture_id > PICTURE_ID_MAX { + return Err(Vp8PatchError::PictureIdTooLarge); + } + if !parsed.is_15_bit && picture_id > PICTURE_ID_SHORT_MAX { + return Err(Vp8PatchError::PictureIdTooLarge); + } + if parsed.is_15_bit { + patch.push( + parsed.offset, + 0x80 | ((picture_id >> 8) as u8 & PICTURE_ID_SHORT_MASK), + ); + patch.push(parsed.offset + 1, picture_id as u8); + } else { + patch.push(parsed.offset, picture_id as u8); + } + } + + if let Some(tl0_pic_idx) = self.tl0_pic_idx { + let Some(parsed) = self.descriptor.tl0_pic_idx else { + return Err(Vp8PatchError::Tl0PicIdxMissing); + }; + patch.push(parsed.offset, tl0_pic_idx); + } + + if let Some(key_idx) = self.key_idx { + let Some(parsed) = self.descriptor.key_idx else { + return Err(Vp8PatchError::KeyIdxMissing); + }; + if key_idx > KEY_IDX_MASK { + return Err(Vp8PatchError::KeyIdxTooLarge); + } + patch.push(parsed.offset, (parsed.value & !KEY_IDX_MASK) | key_idx); + } + + Ok(patch) + } +} + +/// Parsed VP8 RTP payload descriptor. +/// +/// The descriptor stores VP8 fields plus private offsets needed to +/// build a validated [`Vp8Patch`]. It is valid only for the payload bytes parsed +/// by [`Vp8Descriptor::parse`] or a byte-identical copy of those bytes. +#[derive(Clone, Copy, Debug)] +pub struct Vp8Descriptor { + payload_offset: usize, + picture_id: Option, + tl0_pic_idx: Option, + key_idx: Option, +} + +#[derive(Clone, Copy, Debug)] +struct Vp8Field { + value: u8, + offset: usize, +} + +#[derive(Clone, Copy, Debug)] +struct Vp8PictureId { + value: u16, + offset: usize, + is_15_bit: bool, +} + +impl Vp8Descriptor { + /// Parse a VP8 RTP payload descriptor + /// + /// The input must be the RTP payload only. It must not include the RTP + /// header. + /// + /// # Errors + /// + /// Returns [`Vp8DescriptorError::ShortPayload`] when the payload is truncated. + /// Returns [`Vp8DescriptorError::MalformedDescriptor`] when the descriptor + /// has unsupported field combinations. + pub fn parse(payload: &[u8]) -> Result { + let Some(required) = payload.first() else { + return Err(Vp8DescriptorError::ShortPayload); + }; + + let mut descriptor = Self { + payload_offset: 1, + picture_id: None, + tl0_pic_idx: None, + key_idx: None, + }; + + if required & 0x80 != 0 { + let extension = read_vp8_descriptor_byte(payload, &mut descriptor.payload_offset)?; + let has_picture_id = extension & 0x80 != 0; + let has_tl0_pic_idx = extension & 0x40 != 0; + let has_tid = extension & 0x20 != 0; + let has_key_idx = extension & 0x10 != 0; + + if has_tl0_pic_idx && !has_tid { + return Err(Vp8DescriptorError::MalformedDescriptor); + } + + if has_picture_id { + let picture_id_offset = descriptor.payload_offset; + let picture_id = read_vp8_descriptor_byte(payload, &mut descriptor.payload_offset)?; + let is_15_bit = picture_id & 0x80 != 0; + let value = if is_15_bit { + let picture_id_low = + read_vp8_descriptor_byte(payload, &mut descriptor.payload_offset)?; + (u16::from(picture_id & PICTURE_ID_SHORT_MASK) << 8) | u16::from(picture_id_low) + } else { + u16::from(picture_id & PICTURE_ID_SHORT_MASK) + }; + descriptor.picture_id = Some(Vp8PictureId { + value, + offset: picture_id_offset, + is_15_bit, + }); + } + + if has_tl0_pic_idx { + let offset = descriptor.payload_offset; + let value = read_vp8_descriptor_byte(payload, &mut descriptor.payload_offset)?; + descriptor.tl0_pic_idx = Some(Vp8Field { value, offset }); + } + + if has_tid || has_key_idx { + let offset = descriptor.payload_offset; + let value = read_vp8_descriptor_byte(payload, &mut descriptor.payload_offset)?; + if has_key_idx { + descriptor.key_idx = Some(Vp8Field { value, offset }); + } + } + } + + if descriptor.payload_offset >= payload.len() { + return Err(Vp8DescriptorError::ShortPayload); + } + + Ok(descriptor) + } + + /// Returns the VP8 PictureID if present + pub const fn picture_id(&self) -> Option { + match self.picture_id { + Some(picture_id) => Some(picture_id.value), + None => None, + } + } + + /// Returns the VP8 TL0PICIDX if present + pub const fn tl0_pic_idx(&self) -> Option { + match self.tl0_pic_idx { + Some(tl0_pic_idx) => Some(tl0_pic_idx.value), + None => None, + } + } + + /// returns the VP8 KEYIDX if present + pub const fn key_idx(&self) -> Option { + match self.key_idx { + Some(key_idx) => Some(key_idx.value & KEY_IDX_MASK), + None => None, + } + } + + /// Returns whether this descriptor starts a VP8 keyframe. + /// + /// `payload` must be the same payload that produced this descriptor or a + /// byte-identical copy. A mismatched or shorter payload returns `false`. + pub fn starts_keyframe(&self, payload: &[u8]) -> bool { + let Some(required) = payload.first() else { + return false; + }; + + if required & 0x10 == 0 || required & 0x07 != 0 { + return false; + } + + payload + .get(self.payload_offset) + .is_some_and(|header| header & 0x01 == 0) + } + + /// Start building a validated fixed-size VP8 descriptor patch. + pub fn patch(&self) -> Vp8PatchBuilder { + Vp8PatchBuilder { + descriptor: *self, + picture_id: None, + tl0_pic_idx: None, + key_idx: None, + } + } +} + +fn read_vp8_descriptor_byte(payload: &[u8], offset: &mut usize) -> Result { + let byte = payload + .get(*offset) + .ok_or(Vp8DescriptorError::ShortPayload)?; + *offset += 1; + Ok(*byte) +} + +/// Vp8 information describing the depacketized / packetized data +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Vp8CodecExtra { + /// True if the frame can be discarded safely, without causing decoding problems + /// No other frames are encoded depending on this frame (non-reference frame) + pub discardable: bool, + /// True if this frame and subsequent ones on this layer depend only on tl0_pic_idx + pub sync: bool, + /// Index of the vp8 temporal layer. + pub layer_index: u8, + /// Extended picture id, if present + pub picture_id: Option, + /// Extended picture id of layer 0 frames, if present + pub tl0_picture_id: Option, + /// Flag which indicates that within [`MediaData`], there is an individual frame + /// containing complete and independent visual information. This frame serves + /// as a reference point for other frames in the video sequence. + /// + /// [`MediaData`]: crate::media::MediaData + pub is_keyframe: bool, +} + +/// Detect whether a VP8 RTP payload contains a keyframe. +/// +/// Parses the VP8 RTP payload descriptor (RFC 7741) to skip past the +/// variable-length header, then checks the P bit in the VP8 payload header. +/// P=0 means keyframe, P=1 means interframe. +/// +/// Returns `true` only for the first packet of a keyframe (S=1, PID=0). +pub fn detect_vp8_keyframe(payload: &[u8]) -> bool { + if payload.is_empty() { + return false; + } + let b0 = payload[0]; + let s = (b0 & 0x10) >> 4; // Start of VP8 partition + let pid = b0 & 0x07; // Partition index + // Only the first packet of a frame (S=1, PID=0) contains the payload header + if s != 1 || pid != 0 { + return false; + } + let x = (b0 & 0x80) >> 7; // Extension bit + let mut idx = 1; + if x == 1 { + if idx >= payload.len() { + return false; + } + let ext = payload[idx]; + idx += 1; + let i = (ext & 0x80) >> 7; // PictureID present + let l = (ext & 0x40) >> 6; // TL0PICIDX present + let t = (ext & 0x20) >> 5; // TID present + let k = (ext & 0x10) >> 4; // KEYIDX present + if i == 1 { + if idx >= payload.len() { + return false; + } + if payload[idx] & 0x80 != 0 { + idx += 2; // 16-bit PictureID + } else { + idx += 1; // 7-bit PictureID + } + } + if l == 1 { + idx += 1; // tl0picidx + } + if t == 1 || k == 1 { + idx += 1; // TID/KEYIDX + } + } + if idx >= payload.len() { + return false; + } + // VP8 Payload Header: P bit is bit 0 of the first byte + // P=0 → keyframe, P=1 → interframe + payload[idx] & 0x01 == 0 +} + +/// Packetizes VP8 RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(Default, Debug, Copy, Clone)] +pub struct Vp8Packetizer { + enable_picture_id: bool, + picture_id: u16, +} + +impl Packetizer for Vp8Packetizer { + /// Payload fragments a VP8 packet across one or more byte arrays + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu == 0 { + return Ok(vec![]); + } + + /* + * https://tools.ietf.org/html/rfc7741#section-4.2 + * + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |X|R|N|S|R| PID | (REQUIRED) + * +-+-+-+-+-+-+-+-+ + * X: |I|L|T|K| RSV | (OPTIONAL) + * +-+-+-+-+-+-+-+-+ + * I: |M| PictureID | (OPTIONAL) + * +-+-+-+-+-+-+-+-+ + * L: | tl0picidx | (OPTIONAL) + * +-+-+-+-+-+-+-+-+ + * T/K: |tid|Y| KEYIDX | (OPTIONAL) + * +-+-+-+-+-+-+-+-+ + * S: Start of VP8 partition. SHOULD be set to 1 when the first payload + * octet of the RTP packet is the beginning of a new VP8 partition, + * and MUST NOT be 1 otherwise. The S bit MUST be set to 1 for the + * first packet of each encoded frame. + */ + let using_header_size = if self.enable_picture_id { + if self.picture_id == 0 || self.picture_id < 128 { + VP8_HEADER_SIZE + 2 + } else { + VP8_HEADER_SIZE + 3 + } + } else { + VP8_HEADER_SIZE + }; + + let max_fragment_size = mtu as isize - using_header_size as isize; + let mut payload_data_remaining = payload.len() as isize; + let mut payload_data_index: usize = 0; + let mut payloads = vec![]; + + // Make sure the fragment/payload size is correct + if std::cmp::min(max_fragment_size, payload_data_remaining) <= 0 { + return Ok(payloads); + } + + let mut first = true; + while payload_data_remaining > 0 { + let current_fragment_size = + std::cmp::min(max_fragment_size, payload_data_remaining) as usize; + let mut out = Vec::with_capacity(using_header_size + current_fragment_size); + let mut buf = [0u8; 4]; + if first { + buf[0] = 0x10; + first = false; + } + + if self.enable_picture_id { + if using_header_size == VP8_HEADER_SIZE + 2 { + buf[0] |= 0x80; + buf[1] |= 0x80; + buf[2] |= (self.picture_id & u16::from(PICTURE_ID_SHORT_MASK)) as u8; + } else if using_header_size == VP8_HEADER_SIZE + 3 { + buf[0] |= 0x80; + buf[1] |= 0x80; + buf[2] |= + 0x80 | ((self.picture_id >> 8) & u16::from(PICTURE_ID_SHORT_MASK)) as u8; + buf[3] |= (self.picture_id & 0xFF) as u8; + } + } + + out.extend_from_slice(&buf[..using_header_size]); + + out.extend_from_slice( + &payload[payload_data_index..payload_data_index + current_fragment_size], + ); + payloads.push(out); + + payload_data_remaining -= current_fragment_size as isize; + payload_data_index += current_fragment_size; + } + + self.picture_id += 1; + self.picture_id &= PICTURE_ID_MAX; + + Ok(payloads) + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +/// Depacketizes VP8 RTP packets. +/// +/// ## Unversioned API surface +/// +/// This struct is not currently versioned according to semver rules. +/// Breaking changes may be made in minor or patch releases. +#[derive(PartialEq, Eq, Debug, Default, Clone)] +pub struct Vp8Depacketizer { + /// Required Header + /// extended controlbits present + pub x: u8, + /// when set to 1 this frame can be discarded + pub n: u8, + /// start of VP8 partition + pub s: u8, + /// partition index + pub pid: u8, + + /// Extended control bits + /// 1 if PictureID is present + pub i: u8, + /// 1 if tl0picidx is present + pub l: u8, + /// 1 if tid is present + pub t: u8, + /// 1 if KEYIDX is present + pub k: u8, + + /// Optional extension + /// 8 or 16 bits, picture ID + pub picture_id: u16, + /// extended picture id + pub extended_pid: Option, + + /// 8 bits temporal level zero index + pub tl0_pic_idx: u8, + /// extended version of picture_id of temporal layer 0 + pub extended_tl0_pic_idx: Option, + /// 2 bits temporal layer index + pub tid: u8, + /// 1 bit layer sync bit + pub y: u8, + /// 5 bits temporal key frame index + pub key_idx: u8, + + /// Inverse key frame flag. + /// + /// 0 if the current frame is a key frame. + pub p: u8, +} + +impl Depacketizer for Vp8Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + /// depacketize parses the passed byte slice and stores the result in the + /// VP8Packet this method is called upon + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + let payload_len = packet.len(); + // VP8 Payload Descriptor + // https://datatracker.ietf.org/doc/html/rfc7741#section-4.2 + // + // 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + // |X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED) + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + // X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + // I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + // L: | tl0picidx | (OPTIONAL) | PictureID | + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + //T/K:|tid|Y| KEYIDX | (OPTIONAL) L: | tl0picidx | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + // T/K:|tid|Y| KEYIDX | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ + + let mut reader = (packet, 0); + let mut payload_index = 0; + + let mut b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.x = (b & 0x80) >> 7; + self.n = (b & 0x20) >> 5; + self.s = (b & 0x10) >> 4; + self.pid = b & 0x07; + + let (i, l, t, k) = if self.x == 1 { + b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + let i = (b & 0x80) >> 7; + let l = (b & 0x40) >> 6; + let t = (b & 0x20) >> 5; + let k = (b & 0x10) >> 4; + + if l == 1 && t == 0 { + return Err(PacketError::ErrVP8CorruptedPacket); + } + + (i, l, t, k) + } else { + (0, 0, 0, 0) + }; + self.i = i; + self.l = l; + self.t = t; + self.k = k; + + if self.i == 1 { + b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + // PID present? + if b & 0x80 > 0 { + // M == 1, PID is 16bit + let x = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + self.picture_id = (((b & PICTURE_ID_SHORT_MASK) as u16) << 8) | (x as u16); + self.extended_pid = Some(extend_u15(self.extended_pid, self.picture_id)); + payload_index += 1; + } else { + self.picture_id = b as u16; + self.extended_pid = Some(extend_u7(self.extended_pid, b)); + } + } + + if payload_index >= payload_len { + return Err(PacketError::ErrShortPacket); + } + + if self.l == 1 { + self.tl0_pic_idx = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + self.extended_tl0_pic_idx = + Some(extend_u8(self.extended_tl0_pic_idx, self.tl0_pic_idx)); + payload_index += 1; + } + + if payload_index >= payload_len { + return Err(PacketError::ErrShortPacket); + } + + if self.t == 1 || self.k == 1 { + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + if self.t == 1 { + self.tid = b >> 6; + self.y = (b >> 5) & 0x1; + } + if self.k == 1 { + self.key_idx = b & KEY_IDX_MASK; + } + payload_index += 1; + } + + if payload_index >= packet.len() { + return Err(PacketError::ErrShortPacket); + } + + out.extend_from_slice(&packet[payload_index..]); + + // VP8 Payload Header + // https://datatracker.ietf.org/doc/html/rfc7741#section-4.3 + // + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |Size0|H| VER |P| + // +-+-+-+-+-+-+-+-+ + // | Size1 | + // +-+-+-+-+-+-+-+-+ + // | Size2 | + // +-+-+-+-+-+-+-+-+ + // | Octets 4..N of| + // | VP8 payload | + // : : + // +-+-+-+-+-+-+-+-+ + // | OPTIONAL RTP | + // | padding | + // : : + // +-+-+-+-+-+-+-+-+ + // + // The header is present only in packets that have the S bit equal + // to one and the PID equal to zero in the payload descriptor + self.p = if self.s == 1 && self.pid == 0 { + b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + b & 1 + } else { + 1 + }; + + let is_keyframe = if let CodecExtra::Vp8(e) = extra { + e.is_keyframe | (self.p == 0) + } else { + self.p == 0 + }; + + *extra = CodecExtra::Vp8(Vp8CodecExtra { + discardable: self.n == 1, + sync: self.y == 1, + layer_index: self.tid, + picture_id: if self.i == 1 { self.extended_pid } else { None }, + tl0_picture_id: if self.l == 1 { + self.extended_tl0_pic_idx + } else { + None + }, + is_keyframe, + }); + + let _ = payload_index; + + Ok(()) + } + + /// is_partition_head checks whether if this is a head of the VP8 partition + fn is_partition_head(&self, payload: &[u8]) -> bool { + if payload.is_empty() { + false + } else { + (payload[0] & 0x10) != 0 + } + } + + fn is_partition_tail(&self, marker: bool, _payload: &[u8]) -> bool { + marker + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_vp8_unmarshal() -> Result<(), PacketError> { + let mut pck = Vp8Depacketizer::default(); + let mut extra = CodecExtra::None; + + // Empty packet + let empty_bytes = &[]; + let mut payload = Vec::new(); + let result = pck.depacketize(empty_bytes, &mut payload, &mut extra); + assert!(result.is_err(), "Result should be err in case of error"); + + // Small Payload with single octet header + let small_bytes = &[0x00, 0x11, 0x22]; + let mut payload = Vec::new(); + pck.depacketize(small_bytes, &mut payload, &mut extra) + .expect("Small packet"); + assert_eq!(payload, [0x11, 0x22]); + + // Payload is header only + let small_bytes = &[0x00]; + let mut payload = Vec::new(); + let result = pck.depacketize(small_bytes, &mut payload, &mut extra); + assert!(result.is_err(), "Result should be err in case of error"); + + // Normal packet + let raw_bytes = &[0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x90]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("Normal packet"); + assert!(!payload.is_empty(), "Payload must be not empty"); + + // Header size, only X + let raw_bytes = &[0x80, 0x00, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("Only X"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 0, "I must be 0"); + assert_eq!(pck.l, 0, "L must be 0"); + assert_eq!(pck.t, 0, "T must be 0"); + assert_eq!(pck.k, 0, "K must be 0"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, X and I, PID 16bits + let raw_bytes = &[0x80, 0x80, 0x81, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("X and I, PID 16bits"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 1, "I must be 1"); + assert_eq!(pck.l, 0, "L must be 0"); + assert_eq!(pck.t, 0, "T must be 0"); + assert_eq!(pck.k, 0, "K must be 0"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, X and L without T + let raw_bytes = &[0x80, 0x40, 0x00, 0x00]; + let mut payload = Vec::new(); + assert_eq!( + pck.depacketize(raw_bytes, &mut payload, &mut extra), + Err(PacketError::ErrVP8CorruptedPacket) + ); + + let raw_bytes = &[0x00, 0x11, 0x22]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("valid packet after malformed descriptor"); + assert_eq!(payload, [0x11, 0x22]); + + // Header size, X and T + let raw_bytes = &[0x80, 0x20, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("X and T"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 0, "I must be 0"); + assert_eq!(pck.l, 0, "L must be 0"); + assert_eq!(pck.t, 1, "T must be 1"); + assert_eq!(pck.k, 0, "K must be 0"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, X and K + let raw_bytes = &[0x80, 0x10, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("X and K"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 0, "I must be 0"); + assert_eq!(pck.l, 0, "L must be 0"); + assert_eq!(pck.t, 0, "T must be 0"); + assert_eq!(pck.k, 1, "K must be 1"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, all flags and 8bit picture_id + let raw_bytes = &[0xff, 0xff, 0x00, 0x00, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("all flags and 8bit picture_id"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 1, "I must be 1"); + assert_eq!(pck.l, 1, "L must be 1"); + assert_eq!(pck.t, 1, "T must be 1"); + assert_eq!(pck.k, 1, "K must be 1"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, all flags and 16bit picture_id + let raw_bytes = &[0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("all flags and 16bit picture_id"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 1, "I must be 1"); + assert_eq!(pck.l, 1, "L must be 1"); + assert_eq!(pck.t, 1, "T must be 1"); + assert_eq!(pck.k, 1, "K must be 1"); + assert_eq!(pck.p, 1, "P must be 1"); + + // Header size, X, I and P + let raw_bytes = &[0x90, 0x80, 0x11, 0x10, 0x00, 0x00, 0x00]; + let mut payload = Vec::new(); + pck.depacketize(raw_bytes, &mut payload, &mut extra) + .expect("all flags and 16bit picture_id"); + assert!(!payload.is_empty(), "Payload must be not empty"); + assert_eq!(pck.x, 1, "X must be 1"); + assert_eq!(pck.i, 1, "I must be 1"); + assert_eq!(pck.l, 0, "L must be 0"); + assert_eq!(pck.t, 0, "T must be 0"); + assert_eq!(pck.k, 0, "K must be 0"); + assert_eq!(pck.p, 0, "P must be 0"); + + Ok(()) + } + + #[test] + fn test_vp8_payload() -> Result<(), PacketError> { + let tests: Vec<(&str, Vp8Packetizer, usize, Vec<&[u8]>, Vec>)> = vec![ + ( + "WithoutPictureID", + Vp8Packetizer::default(), + 2, + vec![&[0x90, 0x90, 0x90], &[0x91, 0x91]], + vec![ + vec![&[0x10, 0x90], &[0x00, 0x90], &[0x00, 0x90]], + vec![&[0x10, 0x91], &[0x00, 0x91]], + ], + ), + ( + "WithPictureID_1byte", + Vp8Packetizer { + enable_picture_id: true, + picture_id: 0x20, + }, + 5, + vec![&[0x90, 0x90, 0x90], &[0x91, 0x91]], + vec![ + vec![&[0x90, 0x80, 0x20, 0x90, 0x90], &[0x80, 0x80, 0x20, 0x90]], + vec![&[0x90, 0x80, 0x21, 0x91, 0x91]], + ], + ), + ( + "WithPictureID_2bytes", + Vp8Packetizer { + enable_picture_id: true, + picture_id: 0x120, + }, + 6, + vec![&[0x90, 0x90, 0x90], &[0x91, 0x91]], + vec![ + vec![ + &[0x90, 0x80, 0x81, 0x20, 0x90, 0x90], + &[0x80, 0x80, 0x81, 0x20, 0x90], + ], + vec![&[0x90, 0x80, 0x81, 0x21, 0x91, 0x91]], + ], + ), + ]; + + for (name, mut pck, mtu, payloads, expected) in tests { + for (i, payload) in payloads.iter().enumerate() { + let actual = pck.packetize(mtu, payload)?; + assert_eq!(expected[i], actual, "{name}: Generated packet[{i}] differs"); + } + } + + Ok(()) + } + + #[test] + fn test_vp8_payload_eror() -> Result<(), PacketError> { + let mut pck = Vp8Packetizer::default(); + let empty = &[]; + let payload = &[0x90, 0x90, 0x90]; + + // Positive MTU, empty payload + let result = pck.packetize(1, empty)?; + assert!(result.is_empty(), "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(1, payload)?; + assert_eq!(result.len(), 0, "Generated payload should be empty"); + + // Positive MTU, small payload + let result = pck.packetize(2, payload)?; + assert_eq!( + result.len(), + payload.len(), + "Generated payload should be the same size as original payload size" + ); + + Ok(()) + } + + #[test] + fn test_vp8_partition_head_checker_is_partition_head() -> Result<(), PacketError> { + let vp8 = Vp8Depacketizer::default(); + + //"SmallPacket" + assert!( + !vp8.is_partition_head(&[0x00]), + "Small packet should not be the head of a new partition" + ); + + //"SFlagON", + assert!( + vp8.is_partition_head(&[0x10, 0x00, 0x00, 0x00]), + "Packet with S flag should be the head of a new partition" + ); + + //"SFlagOFF" + assert!( + !vp8.is_partition_head(&[0x00, 0x00, 0x00, 0x00]), + "Packet without S flag should not be the head of a new partition" + ); + + Ok(()) + } + + fn rewrite_payload( + payload: &[u8], + patch: impl FnOnce(Vp8PatchBuilder) -> Vp8PatchBuilder, + ) -> Result, Vp8PatchError> { + let descriptor = Vp8Descriptor::parse(payload).expect("valid VP8 descriptor"); + let patch = patch(descriptor.patch()).build()?; + + let mut out = vec![0; payload.len()]; + patch.copy_to(payload, &mut out); + Ok(out) + } + + #[test] + fn test_vp8_descriptor_exposes_fields() { + let short = Vp8Descriptor::parse(&[0x90, 0x80, 0x42, 0x00]).unwrap(); + assert_eq!(short.picture_id(), Some(0x42)); + assert_eq!(short.tl0_pic_idx(), None); + assert_eq!(short.key_idx(), None); + + let full = Vp8Descriptor::parse(&[0x90, 0xf0, 0x92, 0x34, 0x55, 0xa7, 0x00]).unwrap(); + assert_eq!(full.picture_id(), Some(0x1234)); + assert_eq!(full.tl0_pic_idx(), Some(0x55)); + assert_eq!(full.key_idx(), Some(0x07)); + } + + #[test] + fn test_vp8_patch_updates_descriptor_fields() { + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x01, 0x90], |patch| { + patch.picture_id(u16::from(PICTURE_ID_SHORT_MASK)) + }) + .unwrap(), + [0x80, 0x80, PICTURE_ID_SHORT_MASK, 0x90] + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.picture_id(0x0234) + }) + .unwrap(), + [0x80, 0x80, 0x82, 0x34, 0x90] + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.picture_id(PICTURE_ID_MAX) + }) + .unwrap(), + [0x80, 0x80, 0xff, 0xff, 0x90] + ); + assert_eq!( + rewrite_payload(&[0x80, 0x60, 0x12, 0x00, 0x90], |patch| { + patch.tl0_pic_idx(0xab) + }) + .unwrap(), + [0x80, 0x60, 0xab, 0x00, 0x90] + ); + assert_eq!( + rewrite_payload(&[0x80, 0x30, 0xaa, 0x90], |patch| { + patch.key_idx(KEY_IDX_MASK) + }) + .unwrap(), + [0x80, 0x30, 0xbf, 0x90] + ); + } + + #[test] + fn test_vp8_patch_rejects_unsupported_descriptor_shapes() { + assert_eq!( + Vp8Descriptor::parse(&[0x80, 0x80]).unwrap_err(), + Vp8DescriptorError::ShortPayload + ); + assert_eq!( + rewrite_payload(&[0x00, 0x90], |patch| patch.picture_id(1)).unwrap_err(), + Vp8PatchError::PictureIdMissing + ); + assert_eq!( + rewrite_payload(&[0x00, 0x90], |patch| { + patch.picture_id(PICTURE_ID_MAX + 1) + }) + .unwrap_err(), + Vp8PatchError::PictureIdMissing + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.tl0_pic_idx(1) + }) + .unwrap_err(), + Vp8PatchError::Tl0PicIdxMissing + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.key_idx(1) + }) + .unwrap_err(), + Vp8PatchError::KeyIdxMissing + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.key_idx(KEY_IDX_MASK + 1) + }) + .unwrap_err(), + Vp8PatchError::KeyIdxMissing + ); + assert_eq!( + Vp8Descriptor::parse(&[0x80, 0x40, 0x12, 0x90]).unwrap_err(), + Vp8DescriptorError::MalformedDescriptor + ); + } + + #[test] + fn test_vp8_patch_rejects_impossible_values() { + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x01, 0x90], |patch| { + patch.picture_id(0x80) + }) + .unwrap_err(), + Vp8PatchError::PictureIdTooLarge + ); + assert_eq!( + rewrite_payload(&[0x80, 0x80, 0x81, 0x20, 0x90], |patch| { + patch.picture_id(PICTURE_ID_MAX + 1) + }) + .unwrap_err(), + Vp8PatchError::PictureIdTooLarge + ); + assert_eq!( + rewrite_payload(&[0x80, 0x10, 0x00, 0x90], |patch| { + patch.key_idx(KEY_IDX_MASK + 1) + }) + .unwrap_err(), + Vp8PatchError::KeyIdxTooLarge + ); + } + + #[test] + fn test_detect_vp8_keyframe() { + // Empty payload + assert!(!detect_vp8_keyframe(&[])); + + // Minimal keyframe: S=1, PID=0, no extensions, P=0 + // Byte 0: X=0, R=0, N=0, S=1, PID=0 → 0x10 + // Byte 1: VP8 payload header with P=0 (keyframe) → 0x00 + assert!(detect_vp8_keyframe(&[0x10, 0x00])); + + // Minimal interframe: S=1, PID=0, no extensions, P=1 + // Byte 1: VP8 payload header with P=1 → 0x01 + assert!(!detect_vp8_keyframe(&[0x10, 0x01])); + + // Not the first packet (S=0) — cannot detect keyframe + assert!(!detect_vp8_keyframe(&[0x00, 0x00])); + + // Continuation packet (PID != 0) + assert!(!detect_vp8_keyframe(&[0x11, 0x00])); + + // With extension (X=1), 7-bit PictureID, keyframe + // Byte 0: X=1, S=1, PID=0 → 0x90 + // Byte 1: I=1, L=0, T=0, K=0 → 0x80 + // Byte 2: 7-bit PictureID (M=0) → 0x42 + // Byte 3: VP8 payload header P=0 → 0x00 + assert!(detect_vp8_keyframe(&[0x90, 0x80, 0x42, 0x00])); + + // With extension, 7-bit PictureID, interframe + assert!(!detect_vp8_keyframe(&[0x90, 0x80, 0x42, 0x01])); + + // With extension, 16-bit PictureID (M=1), keyframe + // Byte 2: M=1 → 0x80 | PID_high + // Byte 3: PID_low + // Byte 4: VP8 payload header P=0 + assert!(detect_vp8_keyframe(&[0x90, 0x80, 0x80, 0x42, 0x00])); + + // With all extensions: I=1(16-bit), L=1, T=1 + // Byte 0: X=1, S=1 → 0x90 + // Byte 1: I=1, L=1, T=1 → 0xE0 + // Byte 2-3: 16-bit PictureID → 0x80, 0x42 + // Byte 4: TL0PICIDX + // Byte 5: TID/KEYIDX + // Byte 6: VP8 payload header P=0 + assert!(detect_vp8_keyframe(&[ + 0x90, 0xE0, 0x80, 0x42, 0x01, 0x00, 0x00 + ])); + + assert!(detect_vp8_keyframe(&[0x90, 0x40, 0x01, 0x00])); + + // Truncated: extension says PictureID but no bytes left + assert!(!detect_vp8_keyframe(&[0x90, 0x80])); + + // Truncated: header consumed all bytes + assert!(!detect_vp8_keyframe(&[0x90, 0x80, 0x42])); + } + + #[test] + fn test_vp8_descriptor_keyframe_detection_matches_helper() { + let payloads: &[&[u8]] = &[ + &[], + &[0x10, 0x00], + &[0x10, 0x01], + &[0x00, 0x00], + &[0x11, 0x00], + &[0x90, 0x80, 0x42, 0x00], + &[0x90, 0x80, 0x42, 0x01], + &[0x90, 0x80], + &[0x90, 0x80, 0x42], + ]; + + for payload in payloads { + let descriptor_starts_keyframe = Vp8Descriptor::parse(payload) + .map(|descriptor| descriptor.starts_keyframe(payload)) + .unwrap_or(false); + assert_eq!(descriptor_starts_keyframe, detect_vp8_keyframe(payload)); + } + } + + #[test] + fn packetize_respects_mtu() -> Result<(), PacketError> { + let payload = vec![0xABu8; 2000]; + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = Vp8Packetizer::default(); + let pkts = pck.packetize(mtu, &payload)?; + assert!(!pkts.is_empty(), "VP8 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "VP8 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp9.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp9.rs new file mode 100644 index 000000000..8ef546bad --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/packet/vp9.rs @@ -0,0 +1,1586 @@ +use super::{BitRead, CodecExtra, Depacketizer, PacketError, Packetizer}; + +use std::fmt; + +/// Max non-flexible VP9 descriptor size reserved for MTU headroom: +/// flags(1) + PID(2) + L(1) + tl0picidx(1) + ≤1 SS byte on keyframe first-packets; 8 is a safe bound. +const VP9_NON_FLEXIBLE_HEADER_SIZE: usize = 8; +/// VP9 RTP payload descriptor size for flexible mode: +/// flags(1) + 15-bit PID(2) = 3 +const VP9_FLEXIBLE_HEADER_SIZE: usize = 3; +const MAX_SPATIAL_LAYERS: usize = 3; +const MAX_VP9REF_PICS: usize = 3; + +/// VP9 packetizer mode controlling the RTP payload descriptor format. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Vp9PacketizerMode { + /// Flexible mode (F=1): minimal 3-byte header (flags + 15-bit PID). + /// Simpler but causes issues with Safari which drops inter-frames. + Flexible, + /// Non-flexible mode (F=0): 5-6 byte header with layer indices and + /// scalability structure. Compatible with all major browsers. + #[default] + NonFlexible, +} + +/// Vp9 information describing the depacketized/packetized data. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Vp9CodecExtra { + /// Map of the SVC layers. + /// + /// Index of each element corresponds to `spatial layer` index. + /// + /// Each element represents the end (not including) in [`MediaData::data`] + /// for `spatial layer` it belongs. + /// + /// The all `spatial layer`s (up to 3) in [`MediaData::data`] are sorted by + /// `spatial layer` index from lower to greater. So, the beginning of some + /// `spatial layer` is the end (including) of the previous `spatial layer` + /// (for the `spatial layer` with index 0 the beginning is 0). + /// + /// ```text + /// [MediaData::data]: + /// +-----+----------------------+---------------------------+ + /// | SL0 | SL1 | SL2 | + /// +-----+----------------------+---------------------------+ + /// | | | | + /// V V V V + /// 0 layers_scheme[0] layers_scheme[1] layers_scheme[2] + /// ``` + /// + /// **Note:** There is no syncronization between + /// [`Vp9CodecExtra::layers_scheme`] and [`MediaData::data`]! If you need + /// to keep [`Vp9CodecExtra::layers_scheme`] in actual state you have to + /// update the [`Vp9CodecExtra::layers_scheme`] on mutating the + /// [`MediaData::data`]. + /// + /// [`MediaData::data`]: crate::media::MediaData::data + /// + /// ## Example + /// + /// Say you have a VP9 track working in "L3T3" scalability mode and you + /// want to retranslate S1T2 layer. So you need to retranslate the all + /// spatial and temporal layers <= target layer. + /// + /// ```no_run + /// # use str0m::{format::CodecExtra, media::MediaData}; + /// # + /// // We want to receive S1T2 layer. + /// let (target_spatial, target_temporal) = (1u8, 2u8); + /// + /// let media_data: MediaData = todo!(); + /// + /// if let CodecExtra::Vp9(vp9_extra) = media_data.codec_extra { + /// // Try to get current `temporal layer`. If `None` there is no SVC. + /// if let Some(tid) = vp9_extra.tid { + /// if let (true, Some(end_of_target_layer)) = ( + /// // Check whether we need this `temporal layer`. + /// tid <= target_temporal, + /// // Check whether we have information about the right + /// // border of data for the target `spatial layer`. + /// vp9_extra.layers_scheme[target_spatial as usize], + /// ) { + /// // Keep only the bytes for spatial layers up to `target_spatial`. + /// let data = &media_data.data[..end_of_target_layer]; + /// } + /// } + /// } + /// ``` + pub layers_scheme: [Option; MAX_SPATIAL_LAYERS], + + /// Temporal layer id. + pub tid: Option, + + /// Map of the SVC layers widths. + /// + /// Specified for every spatial layer. + pub layers_widths: [Option; MAX_SPATIAL_LAYERS], + + /// Map of the SVC layers heights. + /// + /// Specified for every spatial layer. + pub layers_heights: [Option; MAX_SPATIAL_LAYERS], + + /// Extended picture id of layer 0 frames, if present + pub tl0_picture_id: Option, + + /// Picture ID. + pub pid: u16, + + /// Flag which indicates that within [`MediaData`], there is an individual frame + /// containing complete and independent visual information. This frame serves + /// as a reference point for other frames in the video sequence. + /// + /// [`MediaData`]: crate::media::MediaData + pub is_keyframe: bool, +} + +/// Detect whether a raw VP9 frame is a keyframe by inspecting the bitstream header. +/// +/// VP9 uncompressed header (profile 0/1): +/// `frame_marker(2) | profile_low(1) | profile_high(1) | show_existing(1) | frame_type(1)` +/// +/// VP9 uncompressed header (profile 2/3): +/// `frame_marker(2) | profile_low(1) | profile_high(1) | reserved(1) | show_existing(1) | frame_type(1)` +/// +/// - `frame_marker` must be `0b10` +/// - `frame_type`: 0 = KEY_FRAME, 1 = NON_KEY_FRAME +/// - `show_existing_frame` = 1 references a previously decoded frame (not a real keyframe) +/// +/// This function works on raw VP9 bitstream data (without RTP descriptor). +pub fn detect_vp9_keyframe_bitstream(payload: &[u8]) -> bool { + if payload.is_empty() { + return false; + } + let b = payload[0]; + // frame_marker must be 0b10 (bits 7-6) + if (b & 0xC0) != 0x80 { + return false; + } + // profile_high_bit is at bit 4 + let profile_high = (b >> 4) & 1; + let (show_existing_frame, frame_type) = if profile_high == 1 { + // Profile 2 or 3: reserved_zero at bit 3, show_existing at bit 2, frame_type at bit 1 + ((b >> 2) & 1, (b >> 1) & 1) + } else { + // Profile 0 or 1: show_existing at bit 3, frame_type at bit 2 + ((b >> 3) & 1, (b >> 2) & 1) + }; + // show_existing_frame references a previously decoded frame, not a real keyframe + if show_existing_frame != 0 { + return false; + } + // frame_type: 0 = KEY_FRAME + frame_type == 0 +} + +/// Detect whether a VP9 RTP payload contains a keyframe by inspecting the P bit. +/// +/// VP9 RTP descriptor byte 0: `I|P|L|F|B|E|V|Z` +/// - P=0: independently decodable frame (keyframe) +/// - P=1: inter-picture predicted frame +/// +/// Works with both flexible (F=1) and non-flexible (F=0) mode packets. +pub fn detect_vp9_keyframe(payload: &[u8]) -> bool { + if payload.is_empty() { + return false; + } + // P bit is bit 6 of byte 0. P=0 means keyframe. + (payload[0] & 0x40) == 0 +} + +/// Packetizes VP9 RTP packets. +/// +/// Supports both non-flexible mode (F=0) and flexible mode (F=1), controlled +/// by [`Vp9PacketizerMode`]. Defaults to non-flexible mode for broad browser +/// compatibility — Safari and older Chrome drop inter-frames with F=1. +#[derive(Default, Clone)] +pub struct Vp9Packetizer { + picture_id: u16, + /// Temporal layer 0 picture index — increments on every frame (since all + /// frames are packetized as TID=0). Used in L byte for non-flexible mode. + tl0picidx: u8, + initialized: bool, + mode: Vp9PacketizerMode, + #[cfg(test)] + initial_picture_id: u16, +} + +impl Vp9Packetizer { + /// Create a VP9 packetizer with the specified mode. + pub fn with_mode(mode: Vp9PacketizerMode) -> Self { + Self { + mode, + ..Default::default() + } + } +} + +impl fmt::Debug for Vp9Packetizer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Vp9Packetizer") + .field("picture_id", &self.picture_id) + .field("tl0picidx", &self.tl0picidx) + .field("initialized", &self.initialized) + .field("mode", &self.mode) + .finish() + } +} + +impl Packetizer for Vp9Packetizer { + /// Packetize a VP9 frame into one or more RTP packets. + fn packetize(&mut self, mtu: usize, payload: &[u8]) -> Result>, PacketError> { + if payload.is_empty() || mtu == 0 { + return Ok(vec![]); + } + + if !self.initialized { + #[cfg(test)] + { + self.picture_id = self.initial_picture_id; + } + #[cfg(not(test))] + { + use str0m_proto::NonCryptographicRng; + self.picture_id = NonCryptographicRng::u16() & 0x7FFF; + } + self.initialized = true; + } + + match self.mode { + Vp9PacketizerMode::NonFlexible => self.packetize_non_flexible(mtu, payload), + Vp9PacketizerMode::Flexible => self.packetize_flexible(mtu, payload), + } + } + + fn is_marker(&mut self, _data: &[u8], _previous: Option<&[u8]>, last: bool) -> bool { + last + } +} + +impl Vp9Packetizer { + /// Non-flexible mode (F=0) packetization. + /// + /// 5-8 byte header with layer indices and scalability structure. + /// Compatible with all major browsers including Safari. + /// + /// ```text + /// 0 1 2 3 4 5 6 7 + /// +-+-+-+-+-+-+-+-+ + /// |I|P|L|F|B|E|V|Z| (REQUIRED) + /// +-+-+-+-+-+-+-+-+ + /// I: |M| PICTURE ID | (RECOMMENDED) + /// +-+-+-+-+-+-+-+-+ + /// M: | EXTENDED PID | (RECOMMENDED) + /// +-+-+-+-+-+-+-+-+ + /// L: | tid |U| SID |D| (CONDITIONALLY RECOMMENDED) + /// +-+-+-+-+-+-+-+-+ + /// | tl0picidx | (CONDITIONALLY REQUIRED) + /// +-+-+-+-+-+-+-+-+ + /// V: | SS | + /// | .. | + /// +-+-+-+-+-+-+-+-+ + /// ``` + fn packetize_non_flexible( + &mut self, + mtu: usize, + payload: &[u8], + ) -> Result>, PacketError> { + // Detect keyframe from VP9 bitstream to set P flag correctly. + let is_keyframe = detect_vp9_keyframe_bitstream(payload); + + // tl0picidx increments on every TID=0 frame. Since we always emit TID=0, + // this increments on every frame per draft-ietf-payload-vp9, Section 6.3. + self.tl0picidx = self.tl0picidx.wrapping_add(1); + + // VP9_NON_FLEXIBLE_HEADER_SIZE is the max header (8 bytes, accounting for SS + // on keyframes). This ensures we never exceed MTU even on keyframe first-packets. + let max_fragment_size = mtu as isize - VP9_NON_FLEXIBLE_HEADER_SIZE as isize; + let mut payloads = vec![]; + let mut payload_data_remaining = payload.len(); + let mut payload_data_index = 0; + + if std::cmp::min(max_fragment_size, payload_data_remaining as isize) <= 0 { + return Ok(vec![]); + } + + while payload_data_remaining > 0 { + let current_fragment_size = + std::cmp::min(max_fragment_size as usize, payload_data_remaining); + let is_first = payload_data_index == 0; + let is_last = payload_data_remaining == current_fragment_size; + + // Non-keyframe: 5 bytes (flags + 15-bit PID + L + tl0picidx) + // Keyframe first packet: 6 bytes (+ 1 byte SS data, G=0) + let ss_size = if is_keyframe && is_first { 1 } else { 0 }; + let header_size = 5 + ss_size; + let mut out = Vec::with_capacity(header_size + current_fragment_size); + + // Byte 0: I|P|L|F|B|E|V|Z + let mut flags = 0xA0u8; // I=1, L=1 (F=0 implicit) + if !is_keyframe { + flags |= 0x40; // P=1 for inter-predicted frames + } + if is_first { + flags |= 0x08; // B=1 (beginning of frame) + } + if is_last { + flags |= 0x04; // E=1 (end of frame) + } + if is_keyframe && is_first { + flags |= 0x02; // V=1 (scalability structure present) + } + out.push(flags); + + // Bytes 1-2: 15-bit picture ID (M=1) + out.push((self.picture_id >> 8) as u8 | 0x80); + out.push((self.picture_id & 0xFF) as u8); + + // Bytes 3-4: Layer indices (L=1, non-flexible mode) + // TID(3)|U(1)|SID(3)|D(1) = TID=0, U=1, SID=0, D=0 + out.push(0x10); + // tl0picidx (required when F=0 and L=1) + out.push(self.tl0picidx); + + // SS byte on keyframe first packet: N_S=0, Y=0, G=0 (no GOF). A single-layer stream has no + // GOF to advertise; a G=1/R=0 GOF mis-signals references (see commit msg). This matches what + // libwebrtc emits for non-scalable VP9 (g_bit = num_frames_in_gof > 0). RFC 9628 §4.2. + if is_keyframe && is_first { + out.push(0x00); + } + + out.extend_from_slice( + &payload[payload_data_index..payload_data_index + current_fragment_size], + ); + + payloads.push(out); + + payload_data_remaining -= current_fragment_size; + payload_data_index += current_fragment_size; + } + + self.picture_id += 1; + self.picture_id &= 0x7FFF; + + Ok(payloads) + } + + /// Flexible mode (F=1) packetization. + /// + /// Minimal 3-byte header (flags + 15-bit PID). Simpler but may cause + /// issues with Safari which drops inter-frames. + /// + /// ```text + /// 0 1 2 3 4 5 6 7 + /// +-+-+-+-+-+-+-+-+ + /// |I|P|L|F|B|E|V|Z| (REQUIRED) + /// +-+-+-+-+-+-+-+-+ + /// I: |M| PICTURE ID | (RECOMMENDED) + /// +-+-+-+-+-+-+-+-+ + /// M: | EXTENDED PID | (RECOMMENDED) + /// +-+-+-+-+-+-+-+-+ + /// ``` + fn packetize_flexible( + &mut self, + mtu: usize, + payload: &[u8], + ) -> Result>, PacketError> { + let max_fragment_size = mtu as isize - VP9_FLEXIBLE_HEADER_SIZE as isize; + let mut payloads = vec![]; + let mut payload_data_remaining = payload.len(); + let mut payload_data_index = 0; + + if std::cmp::min(max_fragment_size, payload_data_remaining as isize) <= 0 { + return Ok(vec![]); + } + + while payload_data_remaining > 0 { + let current_fragment_size = + std::cmp::min(max_fragment_size as usize, payload_data_remaining); + let is_first = payload_data_index == 0; + let is_last = payload_data_remaining == current_fragment_size; + + let mut out = Vec::with_capacity(VP9_FLEXIBLE_HEADER_SIZE + current_fragment_size); + + // Byte 0: I|P|L|F|B|E|V|Z — I=1, F=1 + let mut flags = 0x90u8; // I=1, F=1 + if is_first { + flags |= 0x08; // B=1 + } + if is_last { + flags |= 0x04; // E=1 + } + out.push(flags); + + // Bytes 1-2: 15-bit picture ID (M=1) + out.push((self.picture_id >> 8) as u8 | 0x80); + out.push((self.picture_id & 0xFF) as u8); + + out.extend_from_slice( + &payload[payload_data_index..payload_data_index + current_fragment_size], + ); + + payloads.push(out); + + payload_data_remaining -= current_fragment_size; + payload_data_index += current_fragment_size; + } + + self.picture_id += 1; + self.picture_id &= 0x7FFF; + + Ok(payloads) + } +} + +/// Depacketizes VP9 RTP packets. +#[derive(PartialEq, Eq, Debug, Default, Clone)] +pub struct Vp9Depacketizer { + /// picture ID is present + pub i: bool, + /// inter-picture predicted frame. + pub p: bool, + /// layer indices present + pub l: bool, + /// flexible mode + pub f: bool, + /// start of frame. beginning of new vp9 frame + pub b: bool, + /// end of frame + pub e: bool, + /// scalability structure (SS) present + pub v: bool, + /// Not a reference frame for upper spatial layers + pub z: bool, + + /// Recommended headers + /// 7 or 16 bits, picture ID. + pub picture_id: u16, + + /// Conditionally recommended headers + /// Temporal layer ID + pub tid: u8, + /// Switching up point + pub u: bool, + /// Spatial layer ID + pub sid: u8, + /// Inter-layer dependency used + pub d: bool, + + /// Conditionally required headers + /// Reference index (F=1) + pub pdiff: [u8; MAX_VP9REF_PICS], + /// Temporal layer zero index (F=0) + pub tl0picidx: u8, + + /// Scalability structure headers + /// N_S + 1 indicates the number of spatial layers present in the VP9 stream + pub ns: u8, + /// Each spatial layer's frame resolution present + pub y: bool, + /// PG description present flag. + pub g: bool, + /// N_G indicates the number of pictures in a Picture Group (PG) + pub ng: u8, + pub width: [Option; MAX_SPATIAL_LAYERS], + pub height: [Option; MAX_SPATIAL_LAYERS], + /// Temporal layer ID of pictures in a Picture Group + pub pgtid: Vec, + /// Switching up point of pictures in a Picture Group + pub pgu: Vec, + /// Reference indices of pictures in a Picture Group + pub pgpdiff: Vec>, +} + +impl Depacketizer for Vp9Depacketizer { + fn out_size_hint(&self, packets_size: usize) -> Option { + Some(packets_size) + } + + /// depacketize parses the passed byte slice and stores the result + /// in the Vp9Packet this method is called upon + fn depacketize( + &mut self, + packet: &[u8], + out: &mut Vec, + extra: &mut CodecExtra, + ) -> Result<(), PacketError> { + if packet.is_empty() { + return Err(PacketError::ErrShortPacket); + } + + let mut reader = (packet, 0); + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + + self.i = (b & 0x80) != 0; + self.p = (b & 0x40) != 0; + self.l = (b & 0x20) != 0; + self.f = (b & 0x10) != 0; + self.b = (b & 0x08) != 0; + self.e = (b & 0x04) != 0; + self.v = (b & 0x02) != 0; + self.z = (b & 0x01) != 0; + + let mut payload_index = 1; + + if self.i { + payload_index = self.parse_picture_id(&mut reader, payload_index)?; + } + + if self.l { + payload_index = self.parse_layer_info(&mut reader, payload_index)?; + } + + if self.f && self.p { + payload_index = self.parse_ref_indices(&mut reader, payload_index)?; + } + + if self.v { + payload_index = self.parse_ssdata(&mut reader, payload_index)?; + } + + self.update_extra(extra, out.len(), packet.len(), payload_index)?; + + out.extend_from_slice(&packet[payload_index..]); + + Ok(()) + } + + /// is_partition_head checks whether if this is a head of the VP9 partition + fn is_partition_head(&self, payload: &[u8]) -> bool { + if payload.is_empty() { + false + } else { + (payload[0] & 0x08) != 0 + } + } + + fn is_partition_tail(&self, marker: bool, _payload: &[u8]) -> bool { + marker + } +} + +impl Vp9Depacketizer { + /// Updates provided [`CodecExtra`]. + /// __MUST__ be called after the all transformations of `payload_index`. + fn update_extra( + &self, + extra: &mut CodecExtra, + out_len: usize, + packet_len: usize, + payload_index: usize, + ) -> Result<(), PacketError> { + let mut vp9_extra = match extra { + CodecExtra::Vp9(e) => *e, + _ => Vp9CodecExtra::default(), + }; + + if self.l { + let new_stop = out_len + packet_len - payload_index; + + if let Some(stop) = vp9_extra.layers_scheme[self.sid as usize] { + if stop != out_len { + return Err(PacketError::ErrVP9CorruptedPacket); + } + } + + vp9_extra.layers_scheme[self.sid as usize] = Some(new_stop); + vp9_extra.tid = Some(self.tid); + + if !self.f { + vp9_extra.tl0_picture_id = Some(self.tl0picidx); + } + } + + vp9_extra.pid = self.picture_id; + vp9_extra + .layers_widths + .copy_from_slice(&self.width[..MAX_SPATIAL_LAYERS]); + vp9_extra + .layers_heights + .copy_from_slice(&self.height[..MAX_SPATIAL_LAYERS]); + vp9_extra.is_keyframe |= !self.p && (self.sid == 0 || !self.l) && self.b; + + *extra = CodecExtra::Vp9(vp9_extra); + + Ok(()) + } + + // Picture ID: + // + // +-+-+-+-+-+-+-+-+ + // I: |M| PICTURE ID | M:0 => picture id is 7 bits. + // +-+-+-+-+-+-+-+-+ M:1 => picture id is 15 bits. + // M: | EXTENDED PID | + // +-+-+-+-+-+-+-+-+ + // + fn parse_picture_id( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + // PID present? + if (b & 0x80) != 0 { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + // M == 1, PID is 15bit + let x = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + self.picture_id = (((b & 0x7f) as u16) << 8) | (x as u16); + payload_index += 1; + } else { + self.picture_id = (b & 0x7F) as u16; + } + + Ok(payload_index) + } + + fn parse_layer_info( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + payload_index = self.parse_layer_info_common(reader, payload_index)?; + + if self.f { + Ok(payload_index) + } else { + self.parse_layer_info_non_flexible_mode(reader, payload_index) + } + } + + // Layer indices (flexible mode): + // + // +-+-+-+-+-+-+-+-+ + // L: | T |U| S |D| + // +-+-+-+-+-+-+-+-+ + // + fn parse_layer_info_common( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.tid = b >> 5; + self.u = b & 0x10 != 0; + self.sid = (b >> 1) & 0x7; + self.d = b & 0x01 != 0; + + if self.sid as usize >= MAX_SPATIAL_LAYERS { + Err(PacketError::ErrTooManySpatialLayers) + } else { + Ok(payload_index) + } + } + + // Layer indices (non-flexible mode): + // + // +-+-+-+-+-+-+-+-+ + // L: | T |U| S |D| + // +-+-+-+-+-+-+-+-+ + // | tl0picidx | + // +-+-+-+-+-+-+-+-+ + // + fn parse_layer_info_non_flexible_mode( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + self.tl0picidx = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + Ok(payload_index) + } + + // Reference indices: + // + // +-+-+-+-+-+-+-+-+ P=1,F=1: At least one reference index + // P,F: | P_DIFF |N| up to 3 times has to be specified. + // +-+-+-+-+-+-+-+-+ N=1: An additional P_DIFF follows + // current P_DIFF. + // + fn parse_ref_indices( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + let mut b = 1u8; + let mut num_ref_pics = 0; + while (b & 0x1) != 0 { + if num_ref_pics == MAX_VP9REF_PICS { + return Err(PacketError::ErrTooManyPDiff); + } + + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.pdiff[num_ref_pics] = b >> 1; + num_ref_pics += 1; + } + + Ok(payload_index) + } + + // Scalability structure (SS): + // + // +-+-+-+-+-+-+-+-+ + // V: | N_S |Y|G|-|-|-| + // +-+-+-+-+-+-+-+-+ -| + // Y: | WIDTH | (OPTIONAL) . + // + + . + // | | (OPTIONAL) . + // +-+-+-+-+-+-+-+-+ . N_S + 1 times + // | HEIGHT | (OPTIONAL) . + // + + . + // | | (OPTIONAL) . + // +-+-+-+-+-+-+-+-+ -| + // G: | N_G | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ -| + // N_G: | T |U| R |-|-| (OPTIONAL) . + // +-+-+-+-+-+-+-+-+ -| . N_G times + // | P_DIFF | (OPTIONAL) . R times . + // +-+-+-+-+-+-+-+-+ -| -| + // + fn parse_ssdata( + &mut self, + reader: &mut dyn BitRead, + mut payload_index: usize, + ) -> Result { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.ns = b >> 5; + self.y = b & 0x10 != 0; + self.g = (b >> 1) & 0x7 != 0; + + let ns = (self.ns + 1) as usize; + self.ng = 0; + + if ns > MAX_SPATIAL_LAYERS { + return Err(PacketError::ErrVP9CorruptedPacket); + } + + if self.y { + if reader.remaining_bits() < 4 * ns { + return Err(PacketError::ErrShortPacket); + } + + for i in 0..ns { + self.width[i] = Some(reader.get_u16().ok_or(PacketError::ErrShortPacket)?); + self.height[i] = Some(reader.get_u16().ok_or(PacketError::ErrShortPacket)?); + } + payload_index += 4 * ns; + } + + if self.g { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + + self.ng = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + } + + for i in 0..self.ng as usize { + if reader.remaining_bits() == 0 { + return Err(PacketError::ErrShortPacket); + } + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.pgtid.push(b >> 5); + self.pgu.push(b & 0x10 != 0); + + let r = ((b >> 2) & 0x3) as usize; + if reader.remaining_bits() < r { + return Err(PacketError::ErrShortPacket); + } + + self.pgpdiff.push(vec![]); + for _ in 0..r { + let b = reader.get_u8().ok_or(PacketError::ErrShortPacket)?; + payload_index += 1; + + self.pgpdiff[i].push(b); + } + } + + Ok(payload_index) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_vp9_packet_unmarshal() -> Result<(), PacketError> { + let tests: Vec<(&str, &[u8], Vp9Depacketizer, &[u8], Option)> = vec![ + ( + "Empty", + &[], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "NonFlexible", + &[0x00, 0xAA], + Vp9Depacketizer::default(), + &[0xAA], + None, + ), + ( + "NonFlexiblePictureID", + &[0x80, 0x02, 0xAA], + Vp9Depacketizer { + i: true, + picture_id: 0x02, + ..Default::default() + }, + &[0xAA], + None, + ), + ( + "NonFlexiblePictureIDExt", + &[0x80, 0x81, 0xFF, 0xAA], + Vp9Depacketizer { + i: true, + picture_id: 0x01FF, + ..Default::default() + }, + &[0xAA], + None, + ), + ( + "NonFlexiblePictureIDExt_ShortPacket0", + &[0x80, 0x81], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "NonFlexiblePictureIDExt_ShortPacket1", + &[0x80], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "NonFlexibleLayerIndicePictureID", + &[0xA0, 0x02, 0x23, 0x01, 0xAA], + Vp9Depacketizer { + i: true, + l: true, + picture_id: 0x02, + tid: 0x01, + sid: 0x01, + d: true, + tl0picidx: 0x01, + ..Default::default() + }, + &[0xAA], + None, + ), + ( + "FlexibleLayerIndicePictureID", + &[0xB0, 0x02, 0x23, 0x01, 0xAA], + Vp9Depacketizer { + f: true, + i: true, + l: true, + picture_id: 0x02, + tid: 0x01, + sid: 0x01, + d: true, + ..Default::default() + }, + &[0x01, 0xAA], + None, + ), + ( + "NonFlexibleLayerIndicePictureID_ShortPacket0", + &[0xA0, 0x02, 0x23], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "NonFlexibleLayerIndicePictureID_ShortPacket1", + &[0xA0, 0x02], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "FlexiblePictureIDRefIndex", + &[0xD0, 0x02, 0x03, 0x04, 0xAA], + Vp9Depacketizer { + i: true, + p: true, + f: true, + picture_id: 0x02, + pdiff: [0x01, 0x02, 0], + ..Default::default() + }, + &[0xAA], + None, + ), + ( + "FlexiblePictureIDRefIndex_TooManyPDiff", + &[0xD0, 0x02, 0x03, 0x05, 0x07, 0x09, 0x10, 0xAA], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrTooManyPDiff), + ), + ( + "FlexiblePictureIDRefIndexNoPayload", + &[0xD0, 0x02, 0x03, 0x04], + Vp9Depacketizer { + i: true, + p: true, + f: true, + picture_id: 0x02, + pdiff: [0x01, 0x02, 0], + ..Default::default() + }, + &[], + None, + ), + ( + "FlexiblePictureIDRefIndex_ShortPacket0", + &[0xD0, 0x02, 0x03], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "FlexiblePictureIDRefIndex_ShortPacket1", + &[0xD0, 0x02], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "FlexiblePictureIDRefIndex_ShortPacket2", + &[0xD0], + Vp9Depacketizer::default(), + &[], + Some(PacketError::ErrShortPacket), + ), + ( + "ScalabilityStructureResolutionsNoPayload", + &[ + 0x0A, + (1 << 5) | (1 << 4), // NS:1 Y:1 G:0 + (640 >> 8) as u8, + (640 & 0xff) as u8, + (360 >> 8) as u8, + (360 & 0xff) as u8, + (1280 >> 8) as u8, + (1280 & 0xff) as u8, + (720 >> 8) as u8, + (720 & 0xff) as u8, + ], + Vp9Depacketizer { + b: true, + v: true, + ns: 1, + y: true, + g: false, + ng: 0, + width: { + let mut res = [None; MAX_SPATIAL_LAYERS]; + res[0] = Some(640); + res[1] = Some(1280); + + res + }, + height: { + let mut res = [None; MAX_SPATIAL_LAYERS]; + res[0] = Some(360); + res[1] = Some(720); + + res + }, + ..Default::default() + }, + &[], + None, + ), + ( + "ScalabilityStructureNoPayload", + &[ + 0x0A, + (1 << 5) | (1 << 3), // NS:1 Y:0 G:1 + 2, + (1 << 4), // T:0 U:1 R:0 - + (2 << 5) | (1 << 2), // T:2 U:0 R:1 - + 33, + ], + Vp9Depacketizer { + b: true, + v: true, + ns: 1, + y: false, + g: true, + ng: 2, + pgtid: vec![0, 2], + pgu: vec![true, false], + pgpdiff: vec![vec![], vec![33]], + ..Default::default() + }, + &[], + None, + ), + ( + "ScalabilityStructureNoPictureGroupNoPayload", + &[ + 0x0A, + 1 << 3, // NS:0 Y:0 G:1 + 0, // N_G=0, valid: single temporal layer / no fixed dependency info + ], + Vp9Depacketizer { + b: true, + v: true, + ns: 0, + y: false, + g: true, + ng: 0, + ..Default::default() + }, + &[], + None, + ), + ( + "ScalabilityStructureThreeLayers", + &[ + 0x0A, + (2 << 5) | (1 << 4), // NS:2 Y:1 G:0 + (480 >> 8) as u8, + (480 & 0xff) as u8, + (270 >> 8) as u8, + (270 & 0xff) as u8, + (960 >> 8) as u8, + (960 & 0xff) as u8, + (540 >> 8) as u8, + (540 & 0xff) as u8, + (1920 >> 8) as u8, + (1920 & 0xff) as u8, + (1080 >> 8) as u8, + (1080 & 0xff) as u8, + ], + Vp9Depacketizer { + b: true, + v: true, + ns: 2, + y: true, + g: false, + ng: 0, + width: [Some(480), Some(960), Some(1920)], + height: [Some(270), Some(540), Some(1080)], + ..Default::default() + }, + &[], + None, + ), + ( + "ParseRefIdx", + &[ + 0xD8, /* I:1 P:1 L:0 F:1 + * B:1 E:0 V:0 Z:0 */ + (0x80 | ((17 >> 8) & 0x7F)) as u8, // Two byte pictureID. + 17, // TL0PICIDX + (17 << 1) | 1, // P_DIFF N:1 + (18 << 1) | 1, // P_DIFF N:1 + (127 << 1) | 0, // P_DIFF N:0 + ], + Vp9Depacketizer { + i: true, + p: true, + f: true, + b: true, + picture_id: 17, + pdiff: [17, 18, 127], + ..Default::default() + }, + &[], + None, + ), + ]; + + for (name, b, pkt, expected, err) in tests { + let mut p = Vp9Depacketizer::default(); + + if let Some(expected) = err { + let mut payload = Vec::new(); + let mut extra = CodecExtra::None; + if let Err(actual) = p.depacketize(b, &mut payload, &mut extra) { + assert_eq!( + expected, actual, + "{name}: expected {expected}, but got {actual}" + ); + } else { + panic!("{name}: expected error, but got passed"); + } + } else { + let mut payload = Vec::new(); + let mut extra = CodecExtra::None; + p.depacketize(b, &mut payload, &mut extra)?; + assert_eq!(pkt, p, "{name}: expected {pkt:?}, but got {p:?}"); + assert_eq!(payload, expected); + } + } + + Ok(()) + } + + #[test] + fn test_vp9_packetizer_payload() -> Result<(), PacketError> { + let mut r0 = 8692; + let mut rands = vec![]; + for _ in 0..10 { + rands.push(vec![(r0 >> 8) as u8 | 0x80, (r0 & 0xFF) as u8]); + r0 += 1; + } + + // Non-flexible mode (F=0) header layout: + // Byte 0: I|P|L|F|B|E|V|Z (I=1, L=1 always set) + // Bytes 1-2: 15-bit PID (M=1) + // Byte 3: TID(3)|U(1)|SID(3)|D(1) = 0x10 (TID=0, U=1, SID=0, D=0) + // Byte 4: tl0picidx (wrapping u8, increments every frame) + // [Byte 5]: SS data on keyframe first packet only (N_S=0, Y=0, G=0) + // + // Flags byte (non-keyframe): I=1,P=1,L=1,F=0 = 0xE0 base + // + B=0x08, E=0x04 + // B+E: 0xEC, B only: 0xE8, E only: 0xE4, neither: 0xE0 + // + // Test payloads (0x01, 0x02, etc.) don't match VP9 bitstream keyframe + // pattern (frame_marker != 0b10), so they are treated as inter-frames (P=1). + // tl0picidx starts at 0 and increments by 1 per frame. + let tests: Vec<(&str, Vec>, usize, Vec>)> = vec![ + ("NilPayload", vec![vec![]], 100, vec![]), + ("SmallMTU", vec![vec![0x00, 0x00]], 1, vec![]), + ("NegativeMTU", vec![vec![0x00, 0x00]], 0, vec![]), + ( + // Inter-frame, single packet: B+E set, tl0picidx=1 + "OnePacket", + vec![vec![0x01, 0x02]], + 10, + vec![vec![0xEC, rands[0][0], rands[0][1], 0x10, 0x01, 0x01, 0x02]], + ), + ( + // Inter-frame, 2 packets: MTU=9 → max_frag=1, tl0picidx=1 + "TwoPackets", + vec![vec![0x01, 0x02]], + 9, + vec![ + vec![0xE8, rands[0][0], rands[0][1], 0x10, 0x01, 0x01], + vec![0xE4, rands[0][0], rands[0][1], 0x10, 0x01, 0x02], + ], + ), + ( + // Inter-frame, 3 packets: MTU=9 → max_frag=1, tl0picidx=1 + "ThreePackets", + vec![vec![0x01, 0x02, 0x03]], + 9, + vec![ + vec![0xE8, rands[0][0], rands[0][1], 0x10, 0x01, 0x01], + vec![0xE0, rands[0][0], rands[0][1], 0x10, 0x01, 0x02], + vec![0xE4, rands[0][0], rands[0][1], 0x10, 0x01, 0x03], + ], + ), + ( + // Two inter-frames: frame1 tl0picidx=1, frame2 tl0picidx=2 + "TwoFramesFourPackets", + vec![vec![0x01, 0x02, 0x03], vec![0x04]], + 10, + vec![ + vec![0xE8, rands[0][0], rands[0][1], 0x10, 0x01, 0x01, 0x02], + vec![0xE4, rands[0][0], rands[0][1], 0x10, 0x01, 0x03], + vec![0xEC, rands[1][0], rands[1][1], 0x10, 0x02, 0x04], + ], + ), + ]; + + for (name, bs, mtu, expected) in tests { + let mut pck = Vp9Packetizer { + initial_picture_id: 8692, + ..Default::default() + }; + + let mut actual = vec![]; + for b in &bs { + actual.extend(pck.packetize(mtu, b)?); + } + assert_eq!(expected, actual, "{name}: Payloaded packet"); + } + + //"PictureIDOverflow" + { + let mut pck = Vp9Packetizer { + initial_picture_id: 8692, + ..Default::default() + }; + let mut p_prev = Vp9Depacketizer::default(); + for i in 0..0x8000 { + let res = pck.packetize(9, &[0x01])?; + let mut p = Vp9Depacketizer::default(); + let mut payload = Vec::new(); + let mut extra = CodecExtra::None; + p.depacketize(&res[0], &mut payload, &mut extra)?; + + if i > 0 { + if p_prev.picture_id == 0x7FFF { + assert_eq!( + p.picture_id, 0, + "Picture ID next to 0x7FFF must be 0, got {}", + p.picture_id + ); + } else if p_prev.picture_id + 1 != p.picture_id { + panic!( + "Picture ID next must be incremented by 1: {} -> {}", + p_prev.picture_id, p.picture_id, + ); + } + } + + p_prev = p; + } + } + + Ok(()) + } + + #[test] + fn test_vp9_packetizer_keyframe() -> Result<(), PacketError> { + let mut pck = Vp9Packetizer { + initial_picture_id: 100, + ..Default::default() + }; + + // VP9 profile 0 keyframe bitstream: 0x82 = 0b10_00_0_0_10 + // frame_marker=10, profile=00, show_existing=0, frame_type=0 (KEY) + let keyframe = vec![0x82, 0xAA, 0xBB]; + let packets = pck.packetize(20, &keyframe)?; + + assert_eq!(packets.len(), 1); + let pkt = &packets[0]; + // Flags: I=1,P=0,L=1,F=0,B=1,E=1,V=1,Z=0 = 0xAE + assert_eq!(pkt[0], 0xAE, "Keyframe flags should be 0xAE"); + // PID + assert_eq!(pkt[1], (100 >> 8) as u8 | 0x80); + assert_eq!(pkt[2], 100 & 0xFF); + // L byte: TID=0, U=1, SID=0, D=0 + assert_eq!(pkt[3], 0x10); + // tl0picidx = 1 (first frame) + assert_eq!(pkt[4], 1); + // SS: N_S=0, Y=0, G=0 (no GOF) → 0x00 (1 byte) + assert_eq!(pkt[5], 0x00); + // Payload starts right after the 1-byte SS + assert_eq!(&pkt[6..], &keyframe[..]); + + // Parse-side: the emitted SS decodes as a single-layer, no-GOF structure (N_S=0, Y/G false). + let mut dep = Vp9Depacketizer::default(); + let mut out = Vec::new(); + let mut extra = CodecExtra::None; + dep.depacketize(pkt, &mut out, &mut extra)?; + assert!(dep.v && dep.ns == 0 && !dep.y && !dep.g && dep.ng == 0); + assert_eq!(&out[..], &keyframe[..]); + + // Now an inter-frame: 0x84 = 0b10_00_0_1_00 (frame_type=1) + let inter = vec![0x84, 0xCC]; + let packets = pck.packetize(20, &inter)?; + assert_eq!(packets.len(), 1); + let pkt = &packets[0]; + // Flags: I=1,P=1,L=1,F=0,B=1,E=1,V=0,Z=0 = 0xEC + assert_eq!(pkt[0], 0xEC, "Inter-frame flags should be 0xEC"); + // tl0picidx = 2 (second frame, both TID=0 so both increment) + assert_eq!(pkt[4], 2); + // No SS data — payload starts at byte 5 + assert_eq!(&pkt[5..], &inter[..]); + + Ok(()) + } + + #[test] + fn test_vp9_packetizer_fragmented_keyframe() -> Result<(), PacketError> { + let mut pck = Vp9Packetizer { + initial_picture_id: 50, + ..Default::default() + }; + + // VP9 profile 0 keyframe: 0x82 = frame_marker=10, profile=00, show_existing=0, frame_type=0 + // 10 bytes of payload, MTU=12 → max_frag = 12 - 8 = 4 (conservative reserve) + // First packet: 6-byte header (with 1-byte G=0 SS) + 4 bytes payload = 10 ≤ MTU + // Second packet: 5-byte header + 4 bytes payload = 9 ≤ MTU + // Third packet: 5-byte header + 2 bytes payload = 7 ≤ MTU + let keyframe = vec![0x82, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let packets = pck.packetize(12, &keyframe)?; + + assert_eq!( + packets.len(), + 3, + "10-byte keyframe at MTU=12 should produce 3 packets" + ); + + // Packet 1: B=1, E=0, V=1 (keyframe first) + // Flags: I=1,P=0,L=1,F=0,B=1,E=0,V=1,Z=0 = 0xAA + assert_eq!(packets[0][0], 0xAA); + assert_eq!(packets[0].len(), 10); // 6-byte header + 4-byte payload + assert_eq!(packets[0][3], 0x10); // L byte + assert_eq!(packets[0][4], 1); // tl0picidx + assert_eq!(packets[0][5], 0x00); // SS byte: N_S=0, Y=0, G=0 (no GOF) + assert_eq!(&packets[0][6..], &keyframe[0..4]); // payload + + // Packet 2: B=0, E=0, V=0 (middle) + // Flags: I=1,P=0,L=1,F=0,B=0,E=0,V=0,Z=0 = 0xA0 + assert_eq!(packets[1][0], 0xA0); + assert_eq!(packets[1].len(), 9); // 5-byte header + 4-byte payload + assert_eq!(packets[1][4], 1); // same tl0picidx + assert_eq!(&packets[1][5..], &keyframe[4..8]); + + // Packet 3: B=0, E=1, V=0 (last) + // Flags: I=1,P=0,L=1,F=0,B=0,E=1,V=0,Z=0 = 0xA4 + assert_eq!(packets[2][0], 0xA4); + assert_eq!(packets[2].len(), 7); // 5-byte header + 2-byte payload + assert_eq!(&packets[2][5..], &keyframe[8..10]); + + // Verify none exceed MTU + for (i, pkt) in packets.iter().enumerate() { + assert!( + pkt.len() <= 12, + "Packet {i} exceeds MTU: {} > 12", + pkt.len() + ); + } + + // Verify depacketizer round-trip: all 3 packets produce original payload + let mut full_payload = Vec::new(); + let mut extra = CodecExtra::None; + for pkt in &packets { + let mut depkt = Vp9Depacketizer::default(); + depkt.depacketize(pkt, &mut full_payload, &mut extra)?; + } + assert_eq!(full_payload, keyframe); + + Ok(()) + } + + #[test] + fn test_vp9_tl0picidx_wrapping() -> Result<(), PacketError> { + let mut pck = Vp9Packetizer { + initial_picture_id: 0, + tl0picidx: 254, // Start near wrap point + ..Default::default() + }; + + // Frame 1: tl0picidx wraps 254 → 255 + let packets = pck.packetize(20, &[0x01])?; + assert_eq!(packets[0][4], 255); + + // Frame 2: tl0picidx wraps 255 → 0 + let packets = pck.packetize(20, &[0x01])?; + assert_eq!(packets[0][4], 0); + + // Frame 3: 0 → 1 + let packets = pck.packetize(20, &[0x01])?; + assert_eq!(packets[0][4], 1); + + Ok(()) + } + + #[test] + fn test_vp9_packetizer_flexible_mode() -> Result<(), PacketError> { + let mut r0 = 8692; + let mut rands = vec![]; + for _ in 0..10 { + rands.push(vec![(r0 >> 8) as u8 | 0x80, (r0 & 0xFF) as u8]); + r0 += 1; + } + + // With VP9_FLEXIBLE_HEADER_SIZE=3, min productive MTU is 4. + // Flexible mode: 3 bytes header (flags + 15-bit PID) + // + // Flags byte: I|P|L|F|B|E|V|Z + // I=1(0x80), F=1(0x10), B=first(0x08), E=last(0x04) + // B+E: 0x80|0x10|0x08|0x04 = 0x9C + // B: 0x80|0x10|0x08 = 0x98 + // E: 0x80|0x10|0x04 = 0x94 + // none:0x80|0x10 = 0x90 + let tests: Vec<(&str, Vec>, usize, Vec>)> = vec![ + ("NilPayload", vec![vec![]], 100, vec![]), + ("SmallMTU", vec![vec![0x00, 0x00]], 1, vec![]), + ("NegativeMTU", vec![vec![0x00, 0x00]], 0, vec![]), + ( + "OnePacket", + vec![vec![0x01, 0x02]], + 10, + vec![vec![0x9C, rands[0][0], rands[0][1], 0x01, 0x02]], + ), + ( + // MTU=4 → max_frag=1 → 2 packets + "TwoPackets", + vec![vec![0x01, 0x02]], + 4, + vec![ + vec![0x98, rands[0][0], rands[0][1], 0x01], + vec![0x94, rands[0][0], rands[0][1], 0x02], + ], + ), + ( + // MTU=4 → max_frag=1 → 3 packets + "ThreePackets", + vec![vec![0x01, 0x02, 0x03]], + 4, + vec![ + vec![0x98, rands[0][0], rands[0][1], 0x01], + vec![0x90, rands[0][0], rands[0][1], 0x02], + vec![0x94, rands[0][0], rands[0][1], 0x03], + ], + ), + ( + // MTU=5 → max_frag=2 → frame1: 2 pkts, frame2: 1 pkt + "TwoFramesFourPackets", + vec![vec![0x01, 0x02, 0x03], vec![0x04]], + 5, + vec![ + vec![0x98, rands[0][0], rands[0][1], 0x01, 0x02], + vec![0x94, rands[0][0], rands[0][1], 0x03], + vec![0x9C, rands[1][0], rands[1][1], 0x04], + ], + ), + ]; + + for (name, bs, mtu, expected) in tests { + let mut pck = Vp9Packetizer { + initial_picture_id: 8692, + mode: Vp9PacketizerMode::Flexible, + ..Default::default() + }; + + let mut actual = vec![]; + for b in &bs { + actual.extend(pck.packetize(mtu, b)?); + } + assert_eq!(expected, actual, "{name}: Payloaded packet"); + } + + Ok(()) + } + + #[test] + fn test_vp9_packetizer_with_mode() { + let pck = Vp9Packetizer::with_mode(Vp9PacketizerMode::Flexible); + assert_eq!(pck.mode, Vp9PacketizerMode::Flexible); + + let pck = Vp9Packetizer::with_mode(Vp9PacketizerMode::NonFlexible); + assert_eq!(pck.mode, Vp9PacketizerMode::NonFlexible); + + let pck = Vp9Packetizer::default(); + assert_eq!(pck.mode, Vp9PacketizerMode::NonFlexible); + } + + #[test] + fn test_detect_vp9_keyframe_bitstream() { + // Empty payload + assert!(!detect_vp9_keyframe_bitstream(&[])); + + // Invalid frame_marker (not 0b10) + assert!(!detect_vp9_keyframe_bitstream(&[0x00])); // frame_marker=00 + assert!(!detect_vp9_keyframe_bitstream(&[0xC0])); // frame_marker=11 + assert!(!detect_vp9_keyframe_bitstream(&[0x40])); // frame_marker=01 + + // Profile 0 keyframe: 10_00_0_0_xx = 0x80..0x83 + assert!(detect_vp9_keyframe_bitstream(&[0x80])); + assert!(detect_vp9_keyframe_bitstream(&[0x82])); + + // Profile 0 inter-frame: 10_00_0_1_xx = 0x84..0x87 + assert!(!detect_vp9_keyframe_bitstream(&[0x84])); + assert!(!detect_vp9_keyframe_bitstream(&[0x86])); + + // Profile 0 show_existing_frame=1: 10_00_1_x_xx = 0x88..0x8F + assert!(!detect_vp9_keyframe_bitstream(&[0x88])); + assert!(!detect_vp9_keyframe_bitstream(&[0x8C])); + + // Profile 1 keyframe: 10_10_0_0_xx = 0xA0..0xA3 + assert!(detect_vp9_keyframe_bitstream(&[0xA0])); + + // Profile 1 inter-frame: 10_10_0_1_xx = 0xA4..0xA7 + assert!(!detect_vp9_keyframe_bitstream(&[0xA4])); + + // Profile 2 keyframe: 10_01_R_0_0_x + // 0x90 = 10_01_0_0_00 → profile_high=1, show_existing=0, frame_type=0 + assert!(detect_vp9_keyframe_bitstream(&[0x90])); + + // Profile 2 inter-frame: 10_01_0_0_1_0 = 0x92 + assert!(!detect_vp9_keyframe_bitstream(&[0x92])); + } + + #[test] + fn test_vp9_partition_head_checker_is_partition_head() -> Result<(), PacketError> { + let vp9 = Vp9Depacketizer::default(); + + //"SmallPacket" + assert!( + !vp9.is_partition_head(&[]), + "Small packet should not be the head of a new partition" + ); + + //"NormalPacket" + assert!( + vp9.is_partition_head(&[0x18, 0x00, 0x00]), + "VP9 RTP packet with B flag should be head of a new partition" + ); + assert!( + !vp9.is_partition_head(&[0x10, 0x00, 0x00]), + "VP9 RTP packet without B flag should not be head of a new partition" + ); + + Ok(()) + } + + #[test] + fn test_detect_vp9_keyframe() { + // Empty payload + assert!(!detect_vp9_keyframe(&[])); + + // VP9 RTP descriptor byte 0: I|P|L|F|B|E|V|Z + // P=0 (bit 6 clear) → keyframe + // P=1 (bit 6 set) → inter-frame + + // Keyframes (P=0) + assert!(detect_vp9_keyframe(&[0x80])); // I=1 + assert!(detect_vp9_keyframe(&[0xA0])); // I=1, L=1 + assert!(detect_vp9_keyframe(&[0xAE])); // I=1, L=1, B=1, E=1, V=1 + assert!(detect_vp9_keyframe(&[0x00])); // all flags clear + + // Inter-frames (P=1) + assert!(!detect_vp9_keyframe(&[0xE8])); // I=1, P=1, L=1, B=1 + assert!(!detect_vp9_keyframe(&[0xEC])); // I=1, P=1, L=1, B=1, E=1 + assert!(!detect_vp9_keyframe(&[0x40])); // P=1 only + assert!(!detect_vp9_keyframe(&[0xD5])); // I=1, P=1, F=1, E=1, Z=1 + } + + #[test] + fn packetize_respects_mtu() -> Result<(), PacketError> { + // VP9 profile 0 keyframe header followed by payload bytes. + let mut payload = vec![0x82u8]; + payload.extend(std::iter::repeat(0xABu8).take(2000)); + for &mtu in &[100usize, 300, 600, 1200] { + let mut pck = Vp9Packetizer { + initial_picture_id: 100, + ..Default::default() + }; + let pkts = pck.packetize(mtu, &payload)?; + assert!(!pkts.is_empty(), "VP9 produced no packets at mtu {mtu}"); + for (i, pkt) in pkts.iter().enumerate() { + assert!( + pkt.len() <= mtu, + "VP9 packet {i} size {} > mtu {mtu}", + pkt.len() + ); + } + } + Ok(()) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/dir.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/dir.rs new file mode 100644 index 000000000..43bb2a277 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/dir.rs @@ -0,0 +1,85 @@ +use std::fmt; + +/// Media direction. +/// +/// And also extmap direction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Direction { + /// Send only direction. + SendOnly, + /// Receive only direction. + RecvOnly, + /// Bi-directional. + SendRecv, + /// Disabled direction. + Inactive, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(Direction); + +impl Direction { + /// Change the direction to the opposite. + /// + /// `SendRecv` and `Inactive` are left unchanged. + pub fn invert(&self) -> Self { + match self { + Direction::SendOnly => Direction::RecvOnly, + Direction::RecvOnly => Direction::SendOnly, + _ => *self, + } + } + + /// Whether this direction is a sending direction. + pub fn is_sending(&self) -> bool { + matches!(self, Direction::SendOnly | Direction::SendRecv) + } + + /// Whether this direction is a receiving direction. + pub fn is_receiving(&self) -> bool { + matches!(self, Direction::RecvOnly | Direction::SendRecv) + } + + /// When negotiating SDP, in certain cases, we need to treat `Inactive` + /// as if it is receiving. + pub(crate) fn sdp_is_receiving(&self) -> bool { + // The spec says: + // + // > For streams marked as inactive in the answer, the list of media + // > formats is constructed based on the offer. If the offer was sendonly, + // > the list is constructed as if the answer were recvonly. + // + // https://datatracker.ietf.org/doc/html/rfc3264#section-6.1 + matches!( + self, + Direction::RecvOnly | Direction::SendRecv | Direction::Inactive + ) + } +} + +impl From<&str> for Direction { + fn from(v: &str) -> Self { + use Direction::*; + match v { + "sendonly" => SendOnly, + "recvonly" => RecvOnly, + "sendrecv" => SendRecv, + _ => Inactive, + } + } +} + +impl fmt::Display for Direction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + Direction::SendOnly => "sendonly", + Direction::RecvOnly => "recvonly", + Direction::SendRecv => "sendrecv", + Direction::Inactive => "inactive", + } + ) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/error.rs new file mode 100644 index 000000000..bb07e3cc6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/error.rs @@ -0,0 +1,49 @@ +use crate::crypto::CryptoError; +use std::error::Error; +use std::fmt; +use std::io; + +/// Errors from parsing and decrypting RTP. +#[derive(Debug)] +pub enum RtpError { + /// Error arising in the crypto + CryptoError(CryptoError), + + /// Other io error + Io(io::Error), + + /// Failed to parse RTP header. + ParseHeader, +} + +impl fmt::Display for RtpError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RtpError::CryptoError(err) => write!(f, "{}", err), + RtpError::Io(err) => write!(f, "{}", err), + RtpError::ParseHeader => write!(f, "Failed to parse RTP header"), + } + } +} + +impl Error for RtpError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + RtpError::CryptoError(err) => Some(err), + RtpError::Io(err) => Some(err), + _ => None, + } + } +} + +impl From for RtpError { + fn from(err: io::Error) -> Self { + RtpError::Io(err) + } +} + +impl From for RtpError { + fn from(err: CryptoError) -> Self { + RtpError::CryptoError(err) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/ext.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/ext.rs new file mode 100644 index 000000000..495825ba7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/ext.rs @@ -0,0 +1,1581 @@ +use std::any::Any; +use std::any::TypeId; +use std::collections::HashMap; +use std::fmt; +use std::fmt::Debug; +use std::hash::BuildHasherDefault; +use std::hash::Hasher; +use std::panic::UnwindSafe; +use std::str::from_utf8; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use crate::util::InstantExt; +use crate::util::SystemTimeExt; +use crate::util::already_happened; +use crate::util::epoch_to_beginning; + +use crate::rtp_::Frequency; + +use super::mtime::MediaTime; +use super::{Mid, Rid}; + +/// RTP header extensions. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum Extension { + /// + AbsoluteSendTime, + /// + AbsoluteCaptureTime, + /// + AudioLevel, + /// + /// + /// Use when a RTP packet is delayed by a send queue to indicate an offset in the "transmitter". + /// It effectively means we can set a timestamp offset exactly when the UDP packet leaves the + /// server. + TransmissionTimeOffset, + /// + VideoOrientation, + /// + TransportSequenceNumber, + /// + PlayoutDelay, + /// + VideoContentType, + /// + VideoTiming, + /// + /// + /// UTF8 encoded identifier for the RTP stream. Not the same as SSRC, this is is designed to + /// avoid running out of SSRC for very large sessions. + RtpStreamId, + /// + /// + /// UTF8 encoded identifier referencing another RTP stream's RtpStreamId. If we see + /// this extension type, we know the stream is a repair stream. + RepairedRtpStreamId, + /// + RtpMid, + /// + FrameMarking, + /// + ColorSpace, + + /// Not recognized URI, but it could still be user parseable. + #[doc(hidden)] + UnknownUri(String, Arc), +} + +// All header extensions must have a common "form", either using +// 1 byte for the (ID, len) or 2 bytes for the (ID, len). +// If one extension requires the two byte form +// (probably because of its size, but possibly because of ID), +// The form must be the two-byte variety for all of them. +#[repr(u16)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ExtensionsForm { + // See RFC 8285 Section 4.2 + // ID Range: 1..=14 + // Length Range: 1..=16 + OneByte = 0xBEDE, + // See RFC 8285 Section 4.3 + // ID Range: 1..=255 + // Length Range: 0..=255 + TwoByte = 0x1000, +} + +pub const MAX_ID_ONE_BYTE_FORM: u8 = 14; +// With the two byte form, it could be 255, but supporting larger values makes the ExtensionMap larger. +// So we support only up to this ID for now. +pub const MAX_ID: u8 = 16; + +impl ExtensionsForm { + pub(crate) fn as_u16(self) -> u16 { + self as u16 + } + + pub(crate) fn serialize(self) -> [u8; 2] { + // App bits set to 0 + self.as_u16().to_be_bytes() + } + + pub(crate) fn parse(bytes: [u8; 2]) -> Option { + let serialized = u16::from_be_bytes(bytes); + if serialized == ExtensionsForm::OneByte.as_u16() { + Some(ExtensionsForm::OneByte) + // Ignore the app bits + } else if (serialized & 0xFFF0) == ExtensionsForm::TwoByte.as_u16() { + Some(ExtensionsForm::TwoByte) + } else { + None + } + } +} + +// TODO: think this through. Is it unwind safe? +impl UnwindSafe for Extension {} + +/// Trait for parsing/writing user RTP header extensions. +pub trait ExtensionSerializer: Debug + Send + Sync + 'static { + /// Write the extension to the buffer of bytes. Must return the number + /// of bytes written. This can be 0 if the extension could not be serialized. + fn write_to(&self, buf: &mut [u8], ev: &ExtensionValues) -> usize; + + /// Parse a value and put it in the [`ExtensionValues::user_values`] field. + fn parse_value(&self, buf: &[u8], ev: &mut ExtensionValues) -> bool; + + /// Tell if this extension should be used for video media. + fn is_video(&self) -> bool; + + /// Tell if this extension should be used for audio media. + fn is_audio(&self) -> bool; + + /// When calling write_to, if the size of the written value may exceed 16 bytes, + /// or may be 0 bytes, the two byte header extension form must be used. + /// Otherwise, the one byte form may be used, which is usually the case. + fn requires_two_byte_form(&self, _ev: &ExtensionValues) -> bool { + false + } +} + +impl Extension { + fn requires_two_byte_form(&self, ev: &ExtensionValues) -> bool { + match self { + Extension::UnknownUri(_, serializer) => serializer.requires_two_byte_form(ev), + _ => false, + } + } +} + +/// This is a placeholder value for when the Extension URI are parsed in an SDP OFFER/ANSWER. +/// The trait write_to() and parse_value() should never be called (that would be a bug). +#[derive(Debug)] +struct SdpUnknownUri; + +impl ExtensionSerializer for SdpUnknownUri { + // If an unreachable happens, it's a bug. + fn write_to(&self, _buf: &mut [u8], _ev: &ExtensionValues) -> usize { + unreachable!("Incorrect ExtensionSerializer::write_to") + } + fn parse_value(&self, _buf: &[u8], _ev: &mut ExtensionValues) -> bool { + unreachable!("Incorrect ExtensionSerializer::parse_value") + } + fn is_video(&self) -> bool { + unreachable!("Incorrect ExtensionSerializer::is_video") + } + fn is_audio(&self) -> bool { + unreachable!("Incorrect ExtensionSerializer::is_audio") + } +} + +/// Mapping of extension URI to our enum +const EXT_URI: &[(Extension, &str)] = &[ + ( + Extension::AbsoluteSendTime, + "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", + ), + ( + Extension::AbsoluteCaptureTime, + "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time", + ), + ( + Extension::AudioLevel, + "urn:ietf:params:rtp-hdrext:ssrc-audio-level", + ), + ( + Extension::TransmissionTimeOffset, + "urn:ietf:params:rtp-hdrext:toffset", + ), + ( + Extension::VideoOrientation, // + "urn:3gpp:video-orientation", + ), + ( + Extension::TransportSequenceNumber, + "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", + ), + ( + Extension::PlayoutDelay, + "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay", + ), + ( + Extension::VideoContentType, + "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type", + ), + ( + Extension::VideoTiming, + "http://www.webrtc.org/experiments/rtp-hdrext/video-timing", + ), + ( + Extension::RtpStreamId, + "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id", + ), + ( + Extension::RepairedRtpStreamId, + "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id", + ), + ( + Extension::RtpMid, // + "urn:ietf:params:rtp-hdrext:sdes:mid", + ), + ( + Extension::FrameMarking, + "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07", + ), + ( + Extension::ColorSpace, + "http://www.webrtc.org/experiments/rtp-hdrext/color-space", + ), +]; + +impl Extension { + /// Parses an extension from a URI. This only happens for incoming SDP OFFER/ANSWER + /// while the corresponding Extension with a potential ExtensionSerializer is + /// in Rtc::session. + pub(crate) fn from_sdp_uri(uri: &str) -> Self { + for (t, spec) in EXT_URI.iter() { + if *spec == uri { + return t.clone(); + } + } + + Extension::UnknownUri(uri.to_string(), Arc::new(SdpUnknownUri)) + } + + /// Extension for a uri not handled by str0m itself. + pub fn with_serializer(uri: &str, s: impl ExtensionSerializer) -> Self { + Extension::UnknownUri(uri.to_string(), Arc::new(s)) + } + + /// Represents the extension as an URI. + pub fn as_uri(&self) -> &str { + for (t, spec) in EXT_URI.iter() { + if t == self { + return spec; + } + } + + if let Extension::UnknownUri(uri, _) = self { + return uri; + } + + "unknown" + } + + pub(crate) fn is_serialized(&self) -> bool { + if let Self::UnknownUri(_, s) = self { + // Check if this Arc contains the SdpUnknownUri. + let is_sdp = (s as &(dyn Any + 'static)) + .downcast_ref::() + .is_some(); + + // If it is the SdpUnknownUri, we are not serializing. If this happens, + // it's probably a bug. The only way to construct SdpUnknownUri is via SDP, + // but those values are only for Eq-comparison vs the values in Session. + // The SdpUnknownUri should not even try to be serialized. + if is_sdp { + panic!("is_serialized on SdpUnkownUri, this is a bug"); + } + } + true + } + + fn is_audio(&self) -> bool { + use Extension::*; + + if let UnknownUri(_, serializer) = self { + return serializer.is_audio(); + } + + matches!( + self, + RtpStreamId + | RepairedRtpStreamId + | RtpMid + | AbsoluteSendTime + | AbsoluteCaptureTime + | AudioLevel + | TransportSequenceNumber + | TransmissionTimeOffset + | PlayoutDelay + ) + } + + fn is_video(&self) -> bool { + use Extension::*; + + if let UnknownUri(_, serializer) = self { + return serializer.is_video(); + } + + matches!( + self, + RtpStreamId + | RepairedRtpStreamId + | RtpMid + | AbsoluteSendTime + | AbsoluteCaptureTime + | VideoOrientation + | TransportSequenceNumber + | TransmissionTimeOffset + | PlayoutDelay + | VideoContentType + | VideoTiming + | FrameMarking + | ColorSpace + ) + } +} + +// As of 2022-09-28, for audio google chrome offers these. +// "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level" +// "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time" +// "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" +// "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid" +// +// For video these. +// "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time" +// "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" +// "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid" +// "a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay" +// "a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type" +// "a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing" +// "a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space" +// "a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id" +// "a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id" +// "a=extmap:13 urn:3gpp:video-orientation" +// "a=extmap:14 urn:ietf:params:rtp-hdrext:toffset" + +/// Mapping between RTP extension id to what extension that is. +#[derive(Clone, PartialEq, Eq)] +pub struct ExtensionMap([Option; MAX_ID as usize]); // index 0 is extmap:1. + +#[derive(Debug, Clone, PartialEq, Eq)] +struct MapEntry { + ext: Extension, + locked: bool, +} + +impl ExtensionMap { + /// Create an empty map. + pub fn empty() -> Self { + ExtensionMap(std::array::from_fn(|_| None)) + } + + /// Creates a map with the "standard" mappings. + /// + /// The standard are taken from Chrome. + pub fn standard() -> Self { + let mut exts = Self::empty(); + + exts.set(1, Extension::AudioLevel); + exts.set(2, Extension::AbsoluteSendTime); + exts.set(3, Extension::TransportSequenceNumber); + exts.set(4, Extension::RtpMid); + // exts.set_mapping(&ExtMap::new(8, Extension::ColorSpace)); + exts.set(10, Extension::RtpStreamId); + exts.set(11, Extension::RepairedRtpStreamId); + exts.set(13, Extension::VideoOrientation); + + exts + } + + pub(crate) fn clear(&mut self) { + for i in &mut self.0 { + *i = None; + } + } + + /// Set a mapping for an extension. + /// + /// The id must be in 1..=MAX_ID (1-indexed). + pub fn set(&mut self, id: u8, ext: Extension) { + if id < 1 || id > MAX_ID { + debug!("Set RTP extension out of range 1-{}: {}", MAX_ID, id); + return; + } + let idx = id as usize - 1; + + let m = MapEntry { ext, locked: false }; + + self.0[idx] = Some(m); + } + + /// Look up the extension for the id. + /// + /// The id must be in 1..=MAX_ID (1-indexed). + pub fn lookup(&self, id: u8) -> Option<&Extension> { + if id >= 1 && id <= MAX_ID { + self.0[id as usize - 1].as_ref().map(|m| &m.ext) + } else { + debug!("Lookup RTP extension out of range 1-{}: {}", MAX_ID, id); + None + } + } + + /// Finds the id for an extension (if mapped). + /// + /// The returned id will be 1-based. + pub fn id_of(&self, e: Extension) -> Option { + self.0 + .iter() + .position(|x| x.as_ref().map(|e| &e.ext) == Some(&e)) + .map(|p| p as u8 + 1) + } + + /// Returns an iterator over the elements of the extension map + pub fn iter(&self) -> impl Iterator + '_ { + self.0 + .iter() + .enumerate() + .filter_map(|(i, e)| e.as_ref().map(|e| (i, e))) + .map(|(i, e)| ((i + 1) as u8, &e.ext)) + } + + /// Returns an iterator over the audio or video elements of the extension map + pub fn iter_by_media_type(&self, audio: bool) -> impl Iterator + '_ { + self.iter().filter(move |(_id, ext)| { + if audio { + ext.is_audio() + } else { + ext.is_video() + } + }) + } + + /// Returns an iterator over the audio elements of the extension map + #[allow(unused)] + pub fn iter_audio(&self) -> impl Iterator + '_ { + self.iter_by_media_type(true) + } + + /// Returns an iterator over the video elements of the extension map + #[allow(unused)] + pub fn iter_video(&self) -> impl Iterator + '_ { + self.iter_by_media_type(false) + } + + pub(crate) fn cloned_with_type(&self, audio: bool) -> Self { + let mut x = ExtensionMap::empty(); + for (id, ext) in self.iter_by_media_type(audio) { + x.set(id, ext.clone()); + } + x + } + + // https://tools.ietf.org/html/rfc5285 + pub(crate) fn parse( + &self, + mut buf: &[u8], + form: ExtensionsForm, + ext_vals: &mut ExtensionValues, + ) { + loop { + if buf.is_empty() { + return; + } + + if buf[0] == 0 { + // padding + buf = &buf[1..]; + continue; + } + + let (id, len) = match form { + ExtensionsForm::OneByte => { + let id = buf[0] >> 4; + let len = (buf[0] & 0xf) as usize + 1; + buf = &buf[1..]; + + if id == 15 { + // If the ID value 15 is + // encountered, its length field should be ignored, processing of the + // entire extension should terminate at that point, and only the + // extension elements present prior to the element with ID 15 + // considered. + return; + } + (id, len) + } + ExtensionsForm::TwoByte => { + if buf.len() < 2 { + trace!("Not enough ext header len: {} < {}", buf.len(), 2); + return; + } + let id = buf[0]; + let len = buf[1] as usize; + buf = &buf[2..]; + (id, len) + } + }; + + if buf.len() < len { + trace!("Not enough type ext len: {} < {}", buf.len(), len); + return; + } + + let ext_buf = &buf[..len]; + if let Some(ext) = self.lookup(id) { + ext.parse_value(ext_buf, ext_vals); + } + + buf = &buf[len..]; + } + } + + pub(crate) fn form(&self, ev: &ExtensionValues) -> ExtensionsForm { + if self + .iter() + .any(|(id, ext)| id > MAX_ID_ONE_BYTE_FORM || ext.requires_two_byte_form(ev)) + { + ExtensionsForm::TwoByte + } else { + ExtensionsForm::OneByte + } + } + + pub(crate) fn write_to( + &self, + ext_buf: &mut [u8], + ev: &ExtensionValues, + form: ExtensionsForm, + ) -> usize { + let orig_len = ext_buf.len(); + let mut b = ext_buf; + + for (idx, x) in self.0.iter().enumerate() { + if let Some(v) = x { + match form { + ExtensionsForm::OneByte => { + if let Some(n) = v.ext.write_to(&mut b[1..], ev) { + assert!(n <= 16); + assert!(n > 0); + b[0] = (idx as u8 + 1) << 4 | (n as u8 - 1); + b = &mut b[1 + n..]; + } + } + ExtensionsForm::TwoByte => { + if let Some(n) = v.ext.write_to(&mut b[2..], ev) { + b[0] = (idx + 1) as u8; + b[1] = n as u8; + b = &mut b[2 + n..]; + } + } + }; + } + } + + orig_len - b.len() + } + + pub(crate) fn remap(&mut self, remote_exts: &[(u8, &Extension)]) { + // Match remote numbers and lock down those we see for the first time. + for (id, ext) in remote_exts { + self.swap(*id, ext); + } + } + + fn swap(&mut self, id: u8, ext: &Extension) { + if id < 1 || id > MAX_ID { + return; + } + + // Mapping goes from 0 to 13. + let new_index = id as usize - 1; + + let Some(old_index) = self + .0 + .iter() + .enumerate() + .find(|(_, m)| m.as_ref().map(|m| &m.ext) == Some(ext)) + .map(|(i, _)| i) + else { + return; + }; + + // Unwrap OK because index is checking just above. + let old = self.0[old_index].as_mut().unwrap(); + + let is_change = new_index != old_index; + + // If either audio or video is locked, we got a previous extmap negotiation. + if is_change && old.locked { + warn!( + "Extmap locked by previous negotiation. Ignore change: {} -> {}", + old_index, new_index + ); + return; + } + + // Locking must be done regardless of whether there was an actual change. + old.locked = true; + + if !is_change { + return; + } + + self.0.swap(old_index, new_index); + } +} + +impl Extension { + pub(crate) fn write_to(&self, buf: &mut [u8], ev: &ExtensionValues) -> Option { + use Extension::*; + match self { + AbsoluteSendTime => { + // 24 bit fixed point 6 bits for seconds, 18 for the decimals. + // wraps around at 64 seconds. + + // We assume the Instant is absolute. + let time_abs = ev.abs_send_time?; + + // This should be a 64 second offset from unix epoch. + let dur = time_abs.to_unix_duration(); + + // Rebase to the 6.18 format. + let time_24 = MediaTime::from(dur) + .rebase(Frequency::FIXED_POINT_6_18) + .numer() as u32; + + buf[..3].copy_from_slice(&time_24.to_be_bytes()[1..]); + Some(3) + } + AbsoluteCaptureTime => { + let act = ev.abs_capture_time.as_ref()?; + let ntp_64 = act.capture_time.as_ntp_64(); + + if let Some(offset) = act.clock_offset { + if buf.len() < 16 { + return None; + } + buf[..8].copy_from_slice(&ntp_64.to_be_bytes()); + buf[8..16].copy_from_slice(&offset.to_be_bytes()); + Some(16) + } else { + if buf.len() < 8 { + return None; + } + buf[..8].copy_from_slice(&ntp_64.to_be_bytes()); + Some(8) + } + } + AudioLevel => { + let v1 = ev.audio_level?; + let v2 = ev.voice_activity?; + buf[0] = if v2 { 0x80 } else { 0 } | (-(0x7f & v1) as u8); + Some(1) + } + TransmissionTimeOffset => { + let v = ev.tx_time_offs?; + buf[..4].copy_from_slice(&v.to_be_bytes()); + Some(4) + } + VideoOrientation => { + let v = ev.video_orientation?; + buf[0] = v as u8; + Some(1) + } + TransportSequenceNumber => { + let v = ev.transport_cc?; + buf[..2].copy_from_slice(&v.to_be_bytes()); + Some(2) + } + PlayoutDelay => { + let v1 = ev.play_delay_min?.rebase(Frequency::HUNDREDTHS); + let v2 = ev.play_delay_max?.rebase(Frequency::HUNDREDTHS); + let min = (v1.numer() & 0xfff) as u32; + let max = (v2.numer() & 0xfff) as u32; + buf[0] = (min >> 4) as u8; + buf[1] = (min << 4) as u8 | (max >> 8) as u8; + buf[2] = max as u8; + Some(3) + } + VideoContentType => { + let v = ev.video_content_type?; + buf[0] = v; + Some(1) + } + VideoTiming => { + let v = ev.video_timing?; + buf[0] = v.flags; + buf[1..3].copy_from_slice(&v.encode_start.to_be_bytes()); + buf[3..5].copy_from_slice(&v.encode_finish.to_be_bytes()); + buf[5..7].copy_from_slice(&v.packetize_complete.to_be_bytes()); + buf[7..9].copy_from_slice(&v.last_left_pacer.to_be_bytes()); + // Reserved for network + buf[9..11].copy_from_slice(&0_u16.to_be_bytes()); + buf[11..13].copy_from_slice(&0_u16.to_be_bytes()); + Some(13) + } + RtpStreamId => { + let v = ev.rid?; + let l = v.len(); + buf[..l].copy_from_slice(v.as_bytes()); + Some(l) + } + RepairedRtpStreamId => { + let v = ev.rid_repair?; + let l = v.len(); + buf[..l].copy_from_slice(v.as_bytes()); + Some(l) + } + RtpMid => { + let v = ev.mid?; + let l = v.len(); + buf[..l].copy_from_slice(v.as_bytes()); + Some(l) + } + FrameMarking => { + let v = ev.frame_mark?; + buf[..4].copy_from_slice(&v.to_be_bytes()); + Some(4) + } + ColorSpace => { + // TODO HDR color space + None + } + UnknownUri(_, serializer) => { + let n = serializer.write_to(buf, ev); + + if n == 0 { None } else { Some(n) } + } + } + } + + pub(crate) fn parse_value(&self, buf: &[u8], ev: &mut ExtensionValues) -> Option<()> { + use Extension::*; + match self { + // 3 + AbsoluteSendTime => { + // 24 bit fixed point 6 bits for seconds, 18 for the decimals. + // wraps around at 64 seconds. + if buf.len() < 3 { + return None; + } + let time_24 = u32::from_be_bytes([0, buf[0], buf[1], buf[2]]); + + // Rebase to micros + let time_micros = MediaTime::from_fixed_point_6_18(time_24 as u64) + .rebase(Frequency::MICROS) + .numer(); + + // This should be the duration in 0-64 seconds from a fixed 64 second offset + // from UNIX EPOCH. For now, we must save this as offset from _something else_ and + // fix the correct value when we have the exact Instant::now() to relate it to. + let time_dur = Duration::from_micros(time_micros); + + let time_tmp = already_happened() + time_dur; + ev.abs_send_time = Some(time_tmp); + } + // 8 or 16 + AbsoluteCaptureTime => { + if buf.len() < 8 { + return None; + } + let ntp_64 = u64::from_be_bytes(buf[..8].try_into().unwrap()); + let clock_offset = + (buf.len() >= 16).then(|| i64::from_be_bytes(buf[8..16].try_into().unwrap())); + ev.abs_capture_time = Some(AbsCaptureTime { + capture_time: SystemTime::from_ntp_64(ntp_64), + clock_offset, + }); + } + // 1 + AudioLevel => { + if buf.is_empty() { + return None; + } + ev.audio_level = Some(-(0x7f & buf[0] as i8)); + ev.voice_activity = Some(buf[0] & 0x80 > 0); + } + // 3 + TransmissionTimeOffset => { + if buf.len() < 4 { + return None; + } + ev.tx_time_offs = Some(u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]])); + } + // 1 + VideoOrientation => { + if buf.is_empty() { + return None; + } + ev.video_orientation = Some(super::ext::VideoOrientation::from(buf[0] & 3)); + } + // 2 + TransportSequenceNumber => { + if buf.len() < 2 { + return None; + } + ev.transport_cc = Some(u16::from_be_bytes([buf[0], buf[1]])); + } + // 3 + PlayoutDelay => { + if buf.len() < 3 { + return None; + } + let min = (buf[0] as u32) << 4 | (buf[1] as u32) >> 4; + let max = ((buf[1] & 0xf) as u32) << 8 | buf[2] as u32; + ev.play_delay_min = Some(MediaTime::from_hundredths(min as u64)); + ev.play_delay_max = Some(MediaTime::from_hundredths(max as u64)); + } + // 1 + VideoContentType => { + if buf.is_empty() { + return None; + } + ev.video_content_type = Some(buf[0]); + } + // 13 + VideoTiming => { + if buf.len() < 9 { + return None; + } + ev.video_timing = Some(self::VideoTiming { + flags: buf[0], + encode_start: u16::from_be_bytes([buf[1], buf[2]]), + encode_finish: u16::from_be_bytes([buf[3], buf[4]]), + packetize_complete: u16::from_be_bytes([buf[5], buf[6]]), + last_left_pacer: u16::from_be_bytes([buf[7], buf[8]]), + // 9 - 10 // reserved for network + // 11 - 12 // reserved for network + }); + } + RtpStreamId => { + let s = from_utf8(buf).ok()?; + ev.rid = Some(s.into()); + } + RepairedRtpStreamId => { + let s = from_utf8(buf).ok()?; + ev.rid_repair = Some(s.into()); + } + RtpMid => { + let s = from_utf8(buf).ok()?; + ev.mid = Some(s.into()); + } + FrameMarking => { + if buf.len() < 4 { + return None; + } + ev.frame_mark = Some(u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]])); + } + ColorSpace => { + // TODO HDR color space + } + UnknownUri(_, serializer) => { + let success = serializer.parse_value(buf, ev); + if !success { + return None; + } + } + } + + Some(()) + } +} + +/// Values in an RTP header extension. +/// +/// This is metadata that is available also without decrypting the SRTP packets. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct ExtensionValues { + /// Audio level is measured in negative decibel. 0 is max and a "normal" value might be -30. + pub audio_level: Option, + + /// Indication that there is sound from a voice. + pub voice_activity: Option, + + /// Tell a receiver what rotation a video need to replay correctly. + pub video_orientation: Option, + + // The values below are considered internal until we have a reason to expose them. + // Generally we want to avoid expose experimental features unless there are strong + // reasons to do so. + #[doc(hidden)] + pub video_content_type: Option, // 0 = unspecified, 1 = screenshare + #[doc(hidden)] + pub tx_time_offs: Option, + #[doc(hidden)] + pub abs_send_time: Option, + #[doc(hidden)] + pub abs_capture_time: Option, + #[doc(hidden)] + pub transport_cc: Option, // (buf[0] << 8) | buf[1]; + #[doc(hidden)] + // https://webrtc.googlesource.com/src/+/refs/heads/master/docs/native-code/rtp-hdrext/playout-delay + pub play_delay_min: Option, + #[doc(hidden)] + pub play_delay_max: Option, + #[doc(hidden)] + pub video_timing: Option, + #[doc(hidden)] + pub rid: Option, + #[doc(hidden)] + pub rid_repair: Option, + #[doc(hidden)] + pub mid: Option, + #[doc(hidden)] + pub frame_mark: Option, + + /// User values for [`ExtensionSerializer`] to parse into and write from. + pub user_values: UserExtensionValues, +} +impl ExtensionValues { + pub(crate) fn update_absolute_send_time(&mut self, now: Instant) { + let Some(v) = self.abs_send_time else { + return; + }; + + // This should be 0-64 seconds, or we are not working with a newly parsed value. + let relative_64_secs = v - already_happened(); + assert!(relative_64_secs <= Duration::from_secs(64)); + + let now_since_epoch = now.to_unix_duration(); + + let closest_64 = now_since_epoch.saturating_sub(Duration::from_micros( + now_since_epoch.as_micros() as u64 % 64_000_000, + )); + + let since_beginning = closest_64.saturating_sub(epoch_to_beginning()); + + let mut offset = already_happened() + since_beginning; + + if offset + relative_64_secs > now { + offset -= Duration::from_secs(64); + } + + self.abs_send_time = Some(offset + relative_64_secs); + } +} + +/// Space for storing user extension values via [`ExtensionSerializer`]. +#[derive(Clone, Default)] +pub struct UserExtensionValues { + map: Option, +} + +// The "AnyMap" idea is borrowed from the http crate but replacing Box for Any. +type AnyMap = HashMap, BuildHasherDefault>; + +// No point in hashing the TypeId, since it is already unique. +#[derive(Default)] +struct IdHasher(u64); + +impl Hasher for IdHasher { + fn write(&mut self, _: &[u8]) { + unreachable!("TypeId calls write_u64"); + } + + #[inline] + fn write_u64(&mut self, id: u64) { + self.0 = id; + } + + #[inline] + fn finish(&self) -> u64 { + self.0 + } +} + +// TODO: I don't see a good way of comparing this. Is there one? +impl PartialEq for UserExtensionValues { + fn eq(&self, other: &Self) -> bool { + let (Some(m1), Some(m2)) = (&self.map, &other.map) else { + return self.map.is_none() == other.map.is_none(); + }; + + for k1 in m1.keys() { + if !m2.contains_key(k1) { + return false; + } + } + + for k2 in m2.keys() { + if !m1.contains_key(k2) { + return false; + } + } + + true + } +} + +impl Eq for UserExtensionValues {} + +impl UserExtensionValues { + /// Set a user extension value. + /// + /// This uses the type of the value as "key", i.e. it can only hold a single + /// per type. The user should make a wrapper type for the extension they want + /// to parse/write. + /// + /// ``` + /// # use str0m::rtp::ExtensionValues; + /// let mut exts = ExtensionValues::default(); + /// + /// #[derive(Debug, PartialEq, Eq)] + /// struct MySpecialType(u8); + /// + /// exts.user_values.set(MySpecialType(42)); + /// ``` + pub fn set(&mut self, val: T) { + // TODO: Consider simplifying to "self.set_arc(Arc::new(val))"; + self.map + .get_or_insert_with(HashMap::default) + .insert(TypeId::of::(), Arc::new(val)); + } + + /// Get a user extension value (by type). + /// ``` + /// # use str0m::rtp::ExtensionValues; + /// let mut exts = ExtensionValues::default(); + /// + /// #[derive(Debug, PartialEq, Eq)] + /// struct MySpecialType(u8); + /// + /// exts.user_values.set(MySpecialType(42)); + /// + /// let v = exts.user_values.get::(); + /// + /// assert_eq!(v, Some(&MySpecialType(42))); + /// ``` + pub fn get(&self) -> Option<&T> { + self.map + .as_ref() + .and_then(|map| map.get(&TypeId::of::())) + // unwrap here is OK because TypeId::of:: is guaranteed to be unique + .map(|boxed| (&**boxed as &(dyn Any + 'static)).downcast_ref().unwrap()) + } + + /// Like .set(), but takes an Arc, which can be nice to avoid cloning + /// large extension values. + pub fn set_arc(&mut self, val: Arc) { + self.map + .get_or_insert_with(HashMap::default) + .insert(TypeId::of::(), val); + } + + /// Like .get(), but clones and returns the Arc, which can be nice to + /// avoid cloning large extension values. + pub fn get_arc(&self) -> Option> { + self.map + .as_ref()? + .get(&TypeId::of::())? + .clone() + .downcast() + .ok() + } + + /// Remove a user extension value (by type). + pub fn remove(&mut self) { + if let Some(map) = &mut self.map { + map.remove(&TypeId::of::()); + } + } +} + +impl UnwindSafe for UserExtensionValues {} + +impl fmt::Debug for ExtensionValues { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ExtensionValues {{")?; + + if let Some(t) = self.mid { + write!(f, " mid: {t}")?; + } + if let Some(t) = self.rid { + write!(f, " rid: {t}")?; + } + if let Some(t) = self.rid_repair { + write!(f, " rid_repair: {t}")?; + } + if let Some(t) = self.abs_send_time { + write!(f, " abs_send_time: {:?}", t)?; + } + if let Some(t) = &self.abs_capture_time { + write!(f, " abs_capture_time: {:?}", t)?; + } + if let Some(t) = self.voice_activity { + write!(f, " voice_activity: {t}")?; + } + if let Some(t) = self.audio_level { + write!(f, " audio_level: {t}")?; + } + if let Some(t) = self.tx_time_offs { + write!(f, " tx_time_offs: {t}")?; + } + if let Some(t) = self.video_orientation { + write!(f, " video_orientation: {t:?}")?; + } + if let Some(t) = self.transport_cc { + write!(f, " transport_cc: {t}")?; + } + if let Some(t) = self.play_delay_min { + write!(f, " play_delay_min: {}", t.as_seconds())?; + } + if let Some(t) = self.play_delay_max { + write!(f, " play_delay_max: {}", t.as_seconds())?; + } + if let Some(t) = self.video_content_type { + write!(f, " video_content_type: {t}")?; + } + if let Some(t) = &self.video_timing { + write!(f, " video_timing: {t:?}")?; + } + if let Some(t) = &self.frame_mark { + write!(f, " frame_mark: {t}")?; + } + + write!(f, " }}")?; + Ok(()) + } +} + +/// Absolute capture time from the `abs-capture-time` RTP header extension. +/// +/// Contains the NTP wall-clock timestamp of original frame capture, and +/// optionally the sender's estimate of the offset between its own clock +/// and the capture system's clock. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AbsCaptureTime { + /// NTP wall-clock timestamp of original frame capture. + pub capture_time: SystemTime, + /// Sender's estimate of offset between its own NTP clock and the + /// capture system's NTP clock. Signed NTP fixed-point (Q32.32): + /// upper 32 bits whole seconds, lower 32 bits fractional. + pub clock_offset: Option, +} + +/// 2^32 as f64, for NTP Q32.32 fixed-point conversion. +const NTP_F32: f64 = 4_294_967_296.0; + +impl AbsCaptureTime { + /// Clock offset in seconds, if present. + /// + /// Per the spec: `Capture NTP Clock = Sender NTP Clock + offset`. + pub fn clock_offset_secs(&self) -> Option { + self.clock_offset.map(|v| v as f64 / NTP_F32) + } + + /// Set the clock offset in seconds. + pub fn set_clock_offset(&mut self, secs: f64) { + self.clock_offset = Some((secs * NTP_F32) as i64); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VideoTiming { + // 0x01 = extension is set due to timer. + // 0x02 - extension is set because the frame is larger than usual. + pub flags: u8, + pub encode_start: u16, + pub encode_finish: u16, + pub packetize_complete: u16, + pub last_left_pacer: u16, +} + +impl fmt::Display for Extension { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Extension::*; + write!( + f, + "{}", + match self { + AbsoluteSendTime => "abs-send-time", + AbsoluteCaptureTime => "abs-capture-time", + AudioLevel => "ssrc-audio-level", + TransmissionTimeOffset => "toffset", + VideoOrientation => "video-orientation", + TransportSequenceNumber => "transport-wide-cc", + PlayoutDelay => "playout-delay", + VideoContentType => "video-content-type", + VideoTiming => "video-timing", + RtpStreamId => "rtp-stream-id", + RepairedRtpStreamId => "repaired-rtp-stream-id", + RtpMid => "mid", + FrameMarking => "frame-marking07", + ColorSpace => "color-space", + UnknownUri(uri, _) => uri, + } + ) + } +} + +impl fmt::Debug for ExtensionMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Extensions(")?; + let joined = self + .0 + .iter() + .enumerate() + .filter_map(|(i, v)| v.as_ref().map(|v| (i + 1, v))) + .map(|(i, v)| format!("{}={}", i, v.ext)) + .collect::>() + .join(", "); + write!(f, "{joined}")?; + write!(f, ")")?; + Ok(()) + } +} + +/// How the video is rotated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoOrientation { + /// Not rotated. + Deg0 = 0, + /// 90 degress clockwise. + Deg90 = 3, + /// Upside down. + Deg180 = 2, + /// 90 degrees counter clockwise. + Deg270 = 1, +} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(VideoOrientation); + +impl From for VideoOrientation { + fn from(value: u8) -> Self { + match value { + 1 => Self::Deg270, + 2 => Self::Deg180, + 3 => Self::Deg90, + _ => Self::Deg0, + } + } +} + +impl PartialEq for Extension { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Extension::AbsoluteSendTime, Extension::AbsoluteSendTime) => true, + (Extension::AbsoluteCaptureTime, Extension::AbsoluteCaptureTime) => true, + (Extension::AudioLevel, Extension::AudioLevel) => true, + (Extension::TransmissionTimeOffset, Extension::TransmissionTimeOffset) => true, + (Extension::VideoOrientation, Extension::VideoOrientation) => true, + (Extension::TransportSequenceNumber, Extension::TransportSequenceNumber) => true, + (Extension::PlayoutDelay, Extension::PlayoutDelay) => true, + (Extension::VideoContentType, Extension::VideoContentType) => true, + (Extension::VideoTiming, Extension::VideoTiming) => true, + (Extension::RtpStreamId, Extension::RtpStreamId) => true, + (Extension::RepairedRtpStreamId, Extension::RepairedRtpStreamId) => true, + (Extension::RtpMid, Extension::RtpMid) => true, + (Extension::FrameMarking, Extension::FrameMarking) => true, + (Extension::ColorSpace, Extension::ColorSpace) => true, + (Extension::UnknownUri(uri1, _), Extension::UnknownUri(uri2, _)) => uri1 == uri2, + _ => false, + } + } +} + +impl Eq for Extension {} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn abs_send_time() { + let now = Instant::now() + Duration::from_secs(1000); + + let mut exts = ExtensionMap::empty(); + exts.set(4, Extension::AbsoluteSendTime); + let ev = ExtensionValues { + abs_send_time: Some(now), + ..Default::default() + }; + + let mut buf = vec![0_u8; 8]; + exts.write_to(&mut buf[..], &ev, ExtensionsForm::OneByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::OneByte, &mut ev2); + + // Let's pretend a 50 millisecond network latency. + ev2.update_absolute_send_time(now + Duration::from_millis(50)); + + let now2 = ev2.abs_send_time.unwrap(); + + let abs = if now > now2 { now - now2 } else { now2 - now }; + + assert!(abs < Duration::from_millis(1)); + } + + #[test] + fn abs_send_time_two_byte_form() { + let now = Instant::now() + Duration::from_secs(1000); + + let mut exts = ExtensionMap::empty(); + exts.set(16, Extension::AbsoluteSendTime); + let ev = ExtensionValues { + abs_send_time: Some(now), + ..Default::default() + }; + + let mut buf = vec![0_u8; 8]; + assert_eq!(ExtensionsForm::TwoByte, exts.form(&ev)); + exts.write_to(&mut buf[..], &ev, ExtensionsForm::TwoByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::TwoByte, &mut ev2); + + // Let's pretend a 50 millisecond network latency. + ev2.update_absolute_send_time(now + Duration::from_millis(50)); + + let now2 = ev2.abs_send_time.unwrap(); + + let abs = if now > now2 { now - now2 } else { now2 - now }; + + assert!(abs < Duration::from_millis(1)); + } + + #[test] + fn abs_capture_time_short_form() { + let now = SystemTime::now(); + + let mut exts = ExtensionMap::empty(); + exts.set(5, Extension::AbsoluteCaptureTime); + let ev = ExtensionValues { + abs_capture_time: Some(AbsCaptureTime { + capture_time: now, + clock_offset: None, + }), + ..Default::default() + }; + + let mut buf = vec![0_u8; 16]; + exts.write_to(&mut buf[..], &ev, ExtensionsForm::OneByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::OneByte, &mut ev2); + + let act = ev2.abs_capture_time.unwrap(); + + // Allow for small rounding errors in NTP conversion + let abs = if now > act.capture_time { + now.duration_since(act.capture_time) + } else { + act.capture_time.duration_since(now) + } + .unwrap(); + + assert!(abs < Duration::from_millis(1)); + assert_eq!(act.clock_offset, None); + } + + #[test] + fn abs_capture_time_extended_form() { + let now = SystemTime::now(); + let clock_offset: i64 = 123456789; + + let mut exts = ExtensionMap::empty(); + exts.set(5, Extension::AbsoluteCaptureTime); + let ev = ExtensionValues { + abs_capture_time: Some(AbsCaptureTime { + capture_time: now, + clock_offset: Some(clock_offset), + }), + ..Default::default() + }; + + let mut buf = vec![0_u8; 24]; + exts.write_to(&mut buf[..], &ev, ExtensionsForm::OneByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::OneByte, &mut ev2); + + let act = ev2.abs_capture_time.unwrap(); + + // Allow for small rounding errors in NTP conversion + let abs = if now > act.capture_time { + now.duration_since(act.capture_time) + } else { + act.capture_time.duration_since(now) + } + .unwrap(); + + assert!(abs < Duration::from_millis(1)); + assert_eq!(act.clock_offset, Some(clock_offset)); + } + + #[test] + fn abs_capture_time_two_byte_form() { + let now = SystemTime::now(); + let clock_offset: i64 = -987654321; + + let mut exts = ExtensionMap::empty(); + exts.set(16, Extension::AbsoluteCaptureTime); + let ev = ExtensionValues { + abs_capture_time: Some(AbsCaptureTime { + capture_time: now, + clock_offset: Some(clock_offset), + }), + ..Default::default() + }; + + // Should require two-byte form due to extension ID > 14 + assert_eq!(ExtensionsForm::TwoByte, exts.form(&ev)); + + let mut buf = vec![0_u8; 24]; + exts.write_to(&mut buf[..], &ev, ExtensionsForm::TwoByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::TwoByte, &mut ev2); + + let act = ev2.abs_capture_time.unwrap(); + + // Allow for small rounding errors in NTP conversion + let abs = if now > act.capture_time { + now.duration_since(act.capture_time) + } else { + act.capture_time.duration_since(now) + } + .unwrap(); + + assert!(abs < Duration::from_millis(1)); + assert_eq!(act.clock_offset, Some(clock_offset)); + } + + #[test] + fn playout_delay() { + let mut exts = ExtensionMap::empty(); + exts.set(2, Extension::PlayoutDelay); + let ev = ExtensionValues { + play_delay_min: Some(MediaTime::from_hundredths(100)), + play_delay_max: Some(MediaTime::from_hundredths(200)), + ..Default::default() + }; + + let mut buf = vec![0_u8; 8]; + exts.write_to(&mut buf[..], &ev, ExtensionsForm::OneByte); + + let mut ev2 = ExtensionValues::default(); + exts.parse(&buf, ExtensionsForm::OneByte, &mut ev2); + + assert_eq!(ev.play_delay_min, ev2.play_delay_min); + assert_eq!(ev.play_delay_max, ev2.play_delay_max); + } + + #[test] + fn remap_exts_audio() { + use Extension::*; + + let mut e1 = ExtensionMap::standard(); + let mut e2 = ExtensionMap::empty(); + e2.set(14, TransportSequenceNumber); + + println!("{:?}", e1.iter_video().collect::>()); + + e1.remap(&e2.iter_audio().collect::>()); + + // e1 should have adjusted the TransportSequenceNumber for audio + assert_eq!( + e1.iter_audio().collect::>(), + vec![ + (1, &AudioLevel), + (2, &AbsoluteSendTime), + (4, &RtpMid), + (10, &RtpStreamId), + (11, &RepairedRtpStreamId), + (14, &TransportSequenceNumber) + ] + ); + + // e1 should have adjusted the TransportSequenceNumber for vudeo + assert_eq!( + e1.iter_video().collect::>(), + vec![ + (2, &AbsoluteSendTime), + (4, &RtpMid), + (10, &RtpStreamId), + (11, &RepairedRtpStreamId), + (13, &VideoOrientation), + (14, &TransportSequenceNumber), + ] + ); + } + + #[test] + fn remap_exts_video() { + use Extension::*; + + let mut e1 = ExtensionMap::empty(); + e1.set(3, TransportSequenceNumber); + e1.set(4, VideoOrientation); + e1.set(5, VideoContentType); + let mut e2 = ExtensionMap::empty(); + e2.set(14, TransportSequenceNumber); + e2.set(12, VideoOrientation); + + e1.remap(&e2.iter_video().collect::>()); + + // e1 should have adjusted to e2. + assert_eq!( + e1.iter_video().collect::>(), + vec![ + (5, &VideoContentType), + (12, &VideoOrientation), + (14, &TransportSequenceNumber) + ] + ); + } + + #[test] + fn remap_exts_swaparoo() { + use Extension::*; + + let mut e1 = ExtensionMap::empty(); + e1.set(12, TransportSequenceNumber); + e1.set(14, VideoOrientation); + let mut e2 = ExtensionMap::empty(); + e2.set(14, TransportSequenceNumber); + e2.set(12, VideoOrientation); + + e1.remap(&e2.iter_video().collect::>()); + + // just make sure the logic isn't wrong for 12-14 -> 14-12 + assert_eq!( + e1.iter_video().collect::>(), + vec![(12, &VideoOrientation), (14, &TransportSequenceNumber)] + ); + } + + #[test] + fn remap_exts_illegal() { + use Extension::*; + + let mut e1 = ExtensionMap::empty(); + e1.set(12, TransportSequenceNumber); + e1.set(14, VideoOrientation); + + let mut e2 = ExtensionMap::empty(); + e2.set(14, TransportSequenceNumber); + e2.set(12, VideoOrientation); + + let mut e3 = ExtensionMap::empty(); + // Illegal change of already negotiated/locked number + e3.set(1, TransportSequenceNumber); + e3.set(12, AudioLevel); // change of type for existing. + + // First apply e2 + e1.remap(&e2.iter_video().collect::>()); + + println!("{:#?}", e1.0); + assert_eq!( + e1.iter_video().collect::>(), + vec![(12, &VideoOrientation), (14, &TransportSequenceNumber)] + ); + + // Now attempt e3 + e1.remap(&e3.iter_audio().collect::>()); + + println!("{:#?}", e1.0); + // At this point we should have not allowed the change, but remain as it was in first apply. + assert_eq!( + e1.iter_video().collect::>(), + vec![(12, &VideoOrientation), (14, &TransportSequenceNumber)] + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/header.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/header.rs new file mode 100644 index 000000000..4fe1a36d6 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/header.rs @@ -0,0 +1,926 @@ +#![allow(clippy::unusual_byte_groupings)] + +use super::ext::{ExtensionMap, ExtensionValues, ExtensionsForm}; +use super::{MAX_BLANK_PADDING_PAYLOAD_SIZE, Pt, SeqNo, Ssrc}; + +/// Parsed header from an RTP packet. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RtpHeader { + /// Always 2 + pub version: u8, + /// Whether the RTP packet has padding to be an equal of 4 bytes. + pub has_padding: bool, + /// RTP packet has "RTP header extensions". + pub has_extension: bool, + /// Number of contributing source identifiers. + pub csrc_count: usize, + /// For video, this marker signifies the end of a series of packets that + /// together form a single video frame. + /// For audio, it marks the beginning of a talkspurt, which is a burst of + /// audio packets. + pub marker: bool, + /// Type of payload being carried. What this correlates to is sent in the SDP. + pub payload_type: Pt, + /// Sequence number increasing by 1 for each RTP packet. + pub sequence_number: u16, + /// Timestamp in media time for the RTP packet. What the media time base is depends + /// on the codec. + pub timestamp: u32, + /// Sender source identifier. + pub ssrc: Ssrc, + /// Contributing source identifiers (CSRC). Up to 15 values per RFC 3550. + pub csrc: [u32; 15], + /// The extension values parsed using the mapping via SDP. + pub ext_vals: ExtensionValues, + /// Length of header. + pub header_len: usize, +} + +impl RtpHeader { + pub(crate) fn write_to(&self, buf: &mut [u8], exts: &ExtensionMap) -> usize { + assert!(self.csrc_count <= 15, "CSRC count must be <= 15"); + buf[0] = 0b10_0_0_0000 + | if self.has_padding { 1 << 5 } else { 0 } + | if self.has_extension { 1 << 4 } else { 0 } + | (self.csrc_count as u8); + + assert!(*self.payload_type <= 127); + buf[1] = *self.payload_type & 0b0111_1111 | if self.marker { 1 << 7 } else { 0 }; + + buf[2..4].copy_from_slice(&self.sequence_number.to_be_bytes()); + buf[4..8].copy_from_slice(&self.timestamp.to_be_bytes()); + buf[8..12].copy_from_slice(&self.ssrc.to_be_bytes()); + + let csrc_len = self.csrc_count * 4; + for i in 0..self.csrc_count { + let offset = 12 + i * 4; + buf[offset..offset + 4].copy_from_slice(&self.csrc[i].to_be_bytes()); + } + + let ext_start = 12 + csrc_len; + let exts_form = exts.form(&self.ext_vals); + buf[ext_start..ext_start + 2].copy_from_slice(&exts_form.serialize()); + + let ext_buf = &mut buf[ext_start + 4..]; + let mut ext_len = exts.write_to(ext_buf, &self.ext_vals, exts_form); + + let pad = 4 - ext_len % 4; + if pad < 4 { + ext_len += pad; + for i in 0..pad { + ext_buf[ext_len - i - 1] = 0; + } + } + + let bede_len = (ext_len / 4) as u16; + buf[ext_start + 2..ext_start + 4].copy_from_slice(&bede_len.to_be_bytes()); + + ext_start + 4 + ext_len + } + + fn do_pad(buf: &mut [u8], from: usize, pad: usize) { + assert!(pad <= 255); + + // set the padding bit + buf[0] |= 0b00_1_0_0000; + + for i in 0..pad { + buf[from + i] = 0; + } + + buf[from + pad - 1] = pad as u8; + } + + /// Write a packet consisting entirely of padding and write. + pub fn create_padding_packet( + buf: &mut [u8], + header_len: usize, + pad_len: u8, + block_size: usize, + ) -> usize { + if pad_len == 0 { + warn!("Not generating padding packet with zero length"); + return 0; + } + + let rounded_len = if pad_len as usize % block_size == 0 { + pad_len as usize + } else { + ((pad_len as usize / block_size) + 1) * block_size + } + .min(MAX_BLANK_PADDING_PAYLOAD_SIZE); + + Self::do_pad(buf, header_len, rounded_len); + + rounded_len + } + + /// Returns `buf` with any trailing SRTP padding removed, or `None` if the + /// trailing padding length is invalid. + pub(crate) fn unpad_payload(buf: &[u8]) -> Option<&[u8]> { + if buf.is_empty() { + return Some(buf); + } + let pad_len = buf[buf.len() - 1] as usize; + let unpadded_len = buf.len().checked_sub(pad_len)?; + Some(&buf[..unpadded_len]) + } + + pub(crate) fn parse(buf: &[u8], exts: &ExtensionMap) -> Option { + let orig_len = buf.len(); + if buf.len() < 12 { + trace!("RTP header too short < 12: {}", buf.len()); + return None; + } + + let version = (buf[0] & 0b1100_0000) >> 6; + if version != 2 { + trace!("RTP version is not 2"); + return None; + } + let has_padding = buf[0] & 0b0010_0000 > 0; + let has_extension = buf[0] & 0b0001_0000 > 0; + let csrc_count = (buf[0] & 0b0000_1111) as usize; + let marker = buf[1] & 0b1000_0000 > 0; + let payload_type = (buf[1] & 0b0111_1111).into(); + let sequence_number = u16::from_be_bytes([buf[2], buf[3]]); + + let timestamp = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + + let ssrc = u32::from_be_bytes([buf[8], buf[9], buf[10], buf[11]]); + + let buf: &[u8] = &buf[12..]; + + let csrc_len = 4 * csrc_count; + if buf.len() < csrc_len { + trace!("RTP header invalid, not enough csrc"); + return None; + } + + let mut csrc = [0_u32; 15]; + for (i, item) in csrc.iter_mut().enumerate().take(csrc_count) { + let offset = i * 4; + *item = u32::from_be_bytes([ + buf[offset], + buf[offset + 1], + buf[offset + 2], + buf[offset + 3], + ]); + } + + let buf: &[u8] = &buf[csrc_len..]; + + let mut ext = ExtensionValues { + ..Default::default() + }; + + let rest = if !has_extension { + buf + } else { + if buf.len() < 4 { + trace!("RTP bad header extension"); + return None; + } + + let Some(exts_form) = ExtensionsForm::parse([buf[0], buf[1]]) else { + trace!( + "Ignoring unknown RTP header extensions form: {:?}", + u16::from_be_bytes([buf[0], buf[1]]) + ); + return None; + }; + let ext_words = u16::from_be_bytes([buf[2], buf[3]]); + let ext_len = ext_words as usize * 4; + + let buf: &[u8] = &buf[4..]; + + if buf.len() < ext_len { + trace!("RTP ext len larger than header {} > {}", buf.len(), ext_len); + return None; + } + + exts.parse(&buf[..ext_len], exts_form, &mut ext); + &buf[ext_len..] + }; + + let header_len = orig_len - rest.len(); + + let ret = RtpHeader { + version, + has_padding, + has_extension, + csrc_count, + marker, + payload_type, + sequence_number, + timestamp, + ssrc: ssrc.into(), + csrc, + ext_vals: ext, + header_len, + }; + + Some(ret) + } + + /// For RTX the original sequence number is inserted before the RTP payload. + pub fn read_original_sequence_number(buf: &[u8], seq_no: &mut u16) -> usize { + if buf.len() < 2 { + // This is not an RTX packet, probably a padding packet + return 0; + } + *seq_no = u16::from_be_bytes([buf[0], buf[1]]); + 2 + } + + /// For RTX the original sequence number is inserted before the RTP payload. + pub fn write_original_sequence_number(buf: &mut [u8], seq_no: SeqNo) -> usize { + let seq_u16 = (*seq_no) as u16; + buf[0..2].copy_from_slice(&seq_u16.to_be_bytes()); + 2 + } + + /// Sequencer number of this RTP header given the previous number. + /// + /// The logic detects wrap-arounds of the 16-bit RTP sequence number. + pub fn sequence_number(&self, previous: Option) -> SeqNo { + let e_seq = extend_u16(previous.map(|v| *v), self.sequence_number); + e_seq.into() + } +} + +macro_rules! mk_extend { + ($id:ident, $t:ty, $bits:expr) => { + /// "extend" a less than 64 bit sequence number into a 64 bit by + /// using the knowledge of the previous such sequence number. + pub fn $id(prev_ext_seq: Option, seq: $t) -> u64 { + const MAX: u64 = 2_u64.pow($bits); // u16: 65_536; + const HALF: u64 = MAX / 2; // u16: 32_768 + const BITS: usize = $bits; + const VAL_MASK: u64 = ((1 << BITS) - 1); + const ROC_MASK: i64 = (u64::MAX >> BITS) as i64; + + // We define the index of the SRTP packet corresponding to a given + // ROC and RTP sequence number to be the 48-bit quantity + // i = 2^16 * ROC + SEQ. + // + // https://tools.ietf.org/html/rfc3711#appendix-A + // + + // Mask seq to the N bits we are considering. + // As an example, this allows us to handle the wrap-around of 7 bit + // values while storing them in u8 + let seq = seq as u64 & VAL_MASK; + + if prev_ext_seq.is_none() { + // No wrap-around so far. + return seq; + } + + let prev_index = prev_ext_seq.unwrap(); + let roc = (prev_index >> BITS) as i64; // how many wrap-arounds. + let prev_seq = prev_index & (MAX - 1); // u16: 0xffff + + let v = if prev_seq < HALF { + if seq > HALF + prev_seq { + (roc - 1) & ROC_MASK + } else { + roc + } + } else if prev_seq > seq + HALF { + (roc + 1) & ROC_MASK + } else { + roc + }; + + if v < 0 { + return 0; + } + + (v as u64) * MAX + seq + } + }; +} + +mk_extend!(extend_u8, u8, 8); +mk_extend!(extend_u16, u16, 16); +mk_extend!(extend_u32, u32, 32); + +// we 'host' 7 bits in u8 but we ignore the most significant one +mk_extend!(extend_u7, u8, 7); +mk_extend!(extend_u15, u16, 15); + +impl Default for RtpHeader { + fn default() -> Self { + Self { + version: 2, + has_padding: false, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 1.into(), + sequence_number: 0, + timestamp: 0, + ssrc: 0.into(), + csrc: [0; 15], + ext_vals: ExtensionValues::default(), + header_len: 16, + } + } +} + +#[cfg(test)] +mod test { + use crate::util::already_happened; + use crate::{io::DATAGRAM_MAX_PACKET_SIZE, rtp_::Extension}; + use std::time::Duration; + + use super::*; + + #[test] + fn extend_u16_wrap_around() { + assert_eq!(extend_u16(None, 0), 0); + assert_eq!(extend_u16(Some(0), 1), 1); + assert_eq!(extend_u16(Some(65_535), 0), 65_536); + assert_eq!(extend_u16(Some(65_500), 2), 65_538); + assert_eq!(extend_u16(Some(2), 1), 1); + assert_eq!(extend_u16(Some(65_538), 1), 65_537); + assert_eq!(extend_u16(Some(3), 3), 3); + assert_eq!(extend_u16(Some(65_500), 65_500), 65_500); + } + + #[test] + fn extend_u16_with_0_prev() { + // This tests going backwards from previous 0. This should wrap + // around "backwards" making a ridiculous number. + let seq = u16::MAX / 2 + 2; + let expected = u64::MAX - (u16::MAX - seq) as u64; + assert_eq!(extend_u16(Some(0), seq), expected); + } + + #[test] + fn extend_u32_wrap_around() { + const U32MAX: u64 = u32::MAX as u64 + 1; + assert_eq!(extend_u32(None, 0), 0); + assert_eq!(extend_u32(Some(0), 1), 1); + assert_eq!(extend_u32(Some(U32MAX - 1), 0), U32MAX); + assert_eq!(extend_u32(Some(U32MAX - 32), 2), U32MAX + 2); + assert_eq!(extend_u32(Some(2), 1), 1); + assert_eq!(extend_u32(Some(U32MAX + 2), 1), U32MAX + 1); + assert_eq!(extend_u32(Some(3), 3), 3); + assert_eq!( + extend_u32(Some(U32MAX - 32), (U32MAX - 32) as u32), + U32MAX - 32 + ); + } + + #[test] + fn extend_u32_with_0_prev() { + // This tests going backwards from previous 0. This should wrap + // around "backwards" making a ridiculous number. + let seq = u32::MAX / 2 + 2; + let expected = u64::MAX - (u32::MAX - seq) as u64; + assert_eq!(extend_u32(Some(0), seq), expected); + } + + #[test] + fn extend_u7_wrap_around() { + let max = 2_u64.pow(7) - 1; + let half = max / 2; + assert_eq!(extend_u7(None, 0), 0); + assert_eq!(extend_u7(None, 1), 1); + + // most significant bit gets ignored + assert_eq!(extend_u7(None, max as u8), max); + assert_eq!(extend_u7(None, (max + 1) as u8), 0); + + assert_eq!(extend_u7(Some(0), half as u8), half); + assert_eq!(extend_u7(Some(0), (half + 1) as u8), half + 1); + + let seq = max / 2 + 2; + let expected = u64::MAX - (max - seq); + assert_eq!(extend_u7(Some(0), seq as u8), expected); + } + + #[test] + fn extend_u15_wrap_around() { + let max = 2_u64.pow(15) - 1; + let half = max / 2; + assert_eq!(extend_u15(None, 0), 0); + assert_eq!(extend_u15(None, 1), 1); + + assert_eq!(extend_u15(None, max as u16), max); + assert_eq!(extend_u15(None, (max + 1) as u16), 0); + + assert_eq!(extend_u15(Some(0), half as u16), half); + assert_eq!(extend_u15(Some(0), (half + 1) as u16), half + 1); + + let seq = max / 2 + 2; + let expected = u64::MAX - (max - seq); + assert_eq!(extend_u15(Some(0), seq as u16), expected); + } + + #[test] + fn test_generate_one_length_padding_packet() { + let mut buf = vec![6; 255]; + RtpHeader::create_padding_packet(&mut buf, 10, 1, 16); + + let mut expected = vec![0; 16]; + expected[15] = 16; + assert_eq!(&buf[10..26], &expected); + } + + // version: 2, + // has_padding: false, + // has_extension: true, + // marker: false, + // payload_type: 1.into(), + // sequence_number: 0, + // timestamp: 0, + // ssrc: 0.into(), + // ext_vals: ExtensionValues::default(), + // header_len: 16, + + #[test] + fn test_write_rtp_headers() { + fn mk_header(seq: u16, ts: u32, level: i8, marker: bool, exts: &ExtensionMap) -> Vec { + let header = RtpHeader { + payload_type: 33.into(), + sequence_number: seq, + timestamp: ts, + ssrc: 44.into(), + marker, + ext_vals: ExtensionValues { + audio_level: Some(level), + voice_activity: Some(false), + ..Default::default() + }, + ..Default::default() + }; + let mut buf = vec![0; DATAGRAM_MAX_PACKET_SIZE]; + let n = header.write_to(&mut buf[..], exts); + buf.truncate(n); + + buf + } + + let mut exts = ExtensionMap::empty(); + exts.set(3, Extension::AudioLevel); + + let buf1 = mk_header(47_000, 10_000, -42, false, &exts); + let buf2 = mk_header(47_001, 12_000, -43, true, &exts); + let buf3 = mk_header(47_002, 14_000, -44, false, &exts); + + let p1 = &[ + 144, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, 0xBE, 0xDE, 0, 1, 48, 170, 0, 0, + ]; + let p2 = &[ + 144, 161, 183, 153, 0, 0, 46, 224, 0, 0, 0, 44, 0xBE, 0xDE, 0, 1, 48, 171, 0, 0, + ]; + let p3 = &[ + 144, 33, 183, 154, 0, 0, 54, 176, 0, 0, 0, 44, 0xBE, 0xDE, 0, 1, 48, 172, 0, 0, + ]; + + assert_eq!(&buf1, p1); + assert_eq!(&buf2, p2); + assert_eq!(&buf3, p3); + } + + #[test] + fn test_write_rtp_headers_two_byte_form() { + fn mk_header(seq: u16, ts: u32, level: i8, marker: bool, exts: &ExtensionMap) -> Vec { + let header = RtpHeader { + payload_type: 33.into(), + sequence_number: seq, + timestamp: ts, + ssrc: 44.into(), + marker, + ext_vals: ExtensionValues { + audio_level: Some(level), + voice_activity: Some(false), + ..Default::default() + }, + ..Default::default() + }; + let mut buf = vec![0; DATAGRAM_MAX_PACKET_SIZE]; + let n = header.write_to(&mut buf[..], exts); + buf.truncate(n); + + buf + } + + let mut exts = ExtensionMap::empty(); + // An ID larger than 14 forces the 2-byte header extension form + exts.set(15, Extension::AudioLevel); + + let buf1 = mk_header(47_000, 10_000, -42, false, &exts); + let buf2 = mk_header(47_001, 12_000, -43, true, &exts); + let buf3 = mk_header(47_002, 14_000, -44, false, &exts); + + let p1 = &[ + 144, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, 0x10, 0x00, 0, 1, 15, 1, 170, 0, + ]; + let p2 = &[ + 144, 161, 183, 153, 0, 0, 46, 224, 0, 0, 0, 44, 0x10, 0x00, 0, 1, 15, 1, 171, 0, + ]; + let p3 = &[ + 144, 33, 183, 154, 0, 0, 54, 176, 0, 0, 0, 44, 0x10, 0x00, 0, 1, 15, 1, 172, 0, + ]; + + assert_eq!(&buf1, p1); + assert_eq!(&buf2, p2); + assert_eq!(&buf3, p3); + } + + #[test] + fn test_parse_rtp_headers() { + let exts = ExtensionMap::standard(); + + let hb1 = [ + 176, 111, 183, 152, 0, 0, 39, 16, 46, 87, 21, 249, 0xBE, 0xDE, 0, 4, 16, 170, 34, 254, + 32, 106, 49, 0, 0, 66, 120, 89, 106, 0, 0, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 12, + ]; + + let hb2 = [ + 176, 111, 183, 153, 0, 0, 46, 224, 46, 87, 21, 249, 0xBE, 0xDE, 0, 4, 16, 171, 34, 254, + 134, 208, 49, 0, 2, 66, 120, 89, 106, 0, 0, 0, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 12, + ]; + let hb3 = [ + 176, 111, 183, 154, 0, 0, 54, 176, 46, 87, 21, 249, 0xBE, 0xDE, 0, 4, 16, 172, 34, 254, + 32, 106, 49, 0, 1, 66, 120, 89, 106, 0, 0, 0, 9, 10, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 12, + ]; + + let h1 = RtpHeader::parse(&hb1, &exts).unwrap(); + let abs1 = already_happened() + Duration::from_micros(63531654); + + assert_eq!( + h1, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47000, + timestamp: 10000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs1), + voice_activity: Some(true), + audio_level: Some(-42), + transport_cc: Some(0), + ..Default::default() + }, + header_len: 32 + } + ); + + let h2 = RtpHeader::parse(&hb2, &exts).unwrap(); + let abs2 = already_happened() + Duration::from_micros(63631652); + + assert_eq!( + h2, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47001, + timestamp: 12000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs2), + voice_activity: Some(true), + audio_level: Some(-43), + transport_cc: Some(2), + ..Default::default() + }, + header_len: 32 + } + ); + + let h3 = RtpHeader::parse(&hb3, &exts).unwrap(); + let abs3 = already_happened() + Duration::from_micros(63531654); + + assert_eq!( + h3, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47002, + timestamp: 14000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs3), + voice_activity: Some(true), + audio_level: Some(-44), + transport_cc: Some(1), + ..Default::default() + }, + header_len: 32 + } + ); + } + + #[test] + fn test_parse_rtp_headers_two_byte_form() { + let exts = ExtensionMap::standard(); + + let hb1 = [ + 176, 111, 183, 152, 0, 0, 39, 16, 46, 87, 21, 249, 0x10, 0x01, 0, 5, 1, 1, 170, 2, 3, + 254, 32, 106, 3, 2, 0, 0, 4, 3, 120, 89, 106, 0, 0, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 12, + ]; + + let hb2 = [ + 176, 111, 183, 153, 0, 0, 46, 224, 46, 87, 21, 249, 0x10, 0x02, 0, 5, 1, 1, 171, 2, 3, + 254, 134, 208, 3, 2, 0, 2, 4, 3, 120, 89, 106, 0, 0, 0, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 12, + ]; + let hb3 = [ + 176, 111, 183, 154, 0, 0, 54, 176, 46, 87, 21, 249, 0x10, 0x03, 0, 5, 1, 1, 172, 2, 3, + 254, 32, 106, 3, 2, 0, 1, 4, 3, 120, 89, 106, 0, 0, 0, 9, 10, 11, 12, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 12, + ]; + + let h1 = RtpHeader::parse(&hb1, &exts).unwrap(); + let abs1 = already_happened() + Duration::from_micros(63531654); + + assert_eq!( + h1, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47000, + timestamp: 10000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs1), + voice_activity: Some(true), + audio_level: Some(-42), + transport_cc: Some(0), + ..Default::default() + }, + header_len: 36 + } + ); + + let h2 = RtpHeader::parse(&hb2, &exts).unwrap(); + let abs2 = already_happened() + Duration::from_micros(63631652); + + assert_eq!( + h2, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47001, + timestamp: 12000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs2), + voice_activity: Some(true), + audio_level: Some(-43), + transport_cc: Some(2), + ..Default::default() + }, + header_len: 36 + } + ); + + let h3 = RtpHeader::parse(&hb3, &exts).unwrap(); + let abs3 = already_happened() + Duration::from_micros(63531654); + + assert_eq!( + h3, + RtpHeader { + version: 2, + has_padding: true, + has_extension: true, + csrc_count: 0, + marker: false, + payload_type: 111.into(), + sequence_number: 47002, + timestamp: 14000, + ssrc: 777459193.into(), + csrc: [0; 15], + ext_vals: ExtensionValues { + mid: Some("xYj".into()), + abs_send_time: Some(abs3), + voice_activity: Some(true), + audio_level: Some(-44), + transport_cc: Some(1), + ..Default::default() + }, + header_len: 36 + } + ); + } + + #[test] + fn test_write_rtp_headers_with_csrc() { + fn mk_header(csrc_count: usize, csrc: [u32; 15], exts: &ExtensionMap) -> Vec { + let header = RtpHeader { + csrc_count, + payload_type: 33.into(), + sequence_number: 47_000, + timestamp: 10_000, + ssrc: 44.into(), + csrc, + ext_vals: ExtensionValues { + audio_level: Some(-42), + voice_activity: Some(true), + ..Default::default() + }, + ..Default::default() + }; + let mut buf = vec![0; DATAGRAM_MAX_PACKET_SIZE]; + let n = header.write_to(&mut buf[..], exts); + buf.truncate(n); + buf + } + + let mut exts = ExtensionMap::empty(); + exts.set(3, Extension::AudioLevel); + + // 3 CSRC entries, one-byte extension form + let buf1 = mk_header( + 3, + [ + 100, + 0x12_34_56_78, + 0xFFFF_FFFF, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ], + &exts, + ); + let p1 = &[ + 147, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, // fixed header (CC=3) + 0, 0, 0, 100, 0x12, 0x34, 0x56, 0x78, 0xFF, 0xFF, 0xFF, 0xFF, // CSRC values + 0xBE, 0xDE, 0, 1, 48, 170, 0, 0, // extension + ]; + assert_eq!(&buf1, p1); + + // 15 CSRC entries (maximum) + let buf2 = mk_header( + 15, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + &exts, + ); + let p2 = &[ + 159, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, // fixed header (CC=15) + 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, // CSRC 1-4 + 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, // CSRC 5-8 + 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, // CSRC 9-12 + 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, // CSRC 13-15 + 0xBE, 0xDE, 0, 1, 48, 170, 0, 0, // extension + ]; + assert_eq!(&buf2, p2); + } + + #[test] + fn test_parse_rtp_headers_with_csrc() { + let mut exts = ExtensionMap::empty(); + exts.set(3, Extension::AudioLevel); + + // 3 CSRC entries with one-byte extension form (produced by test_write_rtp_headers_with_csrc) + let buf1 = [ + 147, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, 0, 0, 0, 100, 0x12, 0x34, 0x56, 0x78, + 0xFF, 0xFF, 0xFF, 0xFF, 0xBE, 0xDE, 0, 1, 48, 170, 0, 0, + ]; + + let h1 = RtpHeader::parse(&buf1, &exts).unwrap(); + + assert_eq!( + h1, + RtpHeader { + version: 2, + has_padding: false, + has_extension: true, + csrc_count: 3, + marker: false, + payload_type: 33.into(), + sequence_number: 47_000, + timestamp: 10_000, + ssrc: 44.into(), + csrc: [ + 100, + 0x12_34_56_78, + 0xFFFF_FFFF, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + ext_vals: ExtensionValues { + audio_level: Some(-42), + voice_activity: Some(true), + ..Default::default() + }, + header_len: 32, + } + ); + + // 15 CSRC entries (maximum) + let buf2 = [ + 159, 33, 183, 152, 0, 0, 39, 16, 0, 0, 0, 44, // fixed header (CC=15) + 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, // CSRC 1-4 + 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, // CSRC 5-8 + 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, // CSRC 9-12 + 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, // CSRC 13-15 + 0xBE, 0xDE, 0, 1, 48, 170, 0, 0, // extension + ]; + + let h2 = RtpHeader::parse(&buf2, &exts).unwrap(); + + assert_eq!( + h2, + RtpHeader { + version: 2, + has_padding: false, + has_extension: true, + csrc_count: 15, + marker: false, + payload_type: 33.into(), + sequence_number: 47_000, + timestamp: 10_000, + ssrc: 44.into(), + csrc: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ext_vals: ExtensionValues { + audio_level: Some(-42), + voice_activity: Some(true), + ..Default::default() + }, + header_len: 80, + } + ); + } + + #[test] + fn truncate_off_srtp_padding() { + let truncate = |payload: Vec| -> Result, ()> { + match RtpHeader::unpad_payload(&payload) { + Some(unpadded) => Ok(unpadded.to_vec()), + None => Err(()), + } + }; + + assert_eq!(Ok(vec![1, 2, 3, 4, 0]), truncate(vec![1, 2, 3, 4, 0])); + assert_eq!(Ok(vec![1, 2, 3, 4]), truncate(vec![1, 2, 3, 4, 1])); + assert_eq!(Ok(vec![1, 2, 3]), truncate(vec![1, 2, 3, 4, 2])); + assert_eq!(Ok(vec![1, 2]), truncate(vec![1, 2, 3, 4, 3])); + assert_eq!(Ok(vec![1]), truncate(vec![1, 2, 3, 4, 4])); + assert_eq!(Ok(vec![]), truncate(vec![1, 2, 3, 4, 5])); + assert_eq!(Err(()), truncate(vec![1, 2, 3, 4, 6])); + assert_eq!(Err(()), truncate(vec![1, 2, 3, 4, 255])); + assert_eq!(Ok(vec![0]), truncate(vec![0])); + assert_eq!(Ok(vec![]), truncate(vec![1])); + assert_eq!(Ok(vec![]), truncate(vec![])); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/id.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/id.rs new file mode 100644 index 000000000..327cd269f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/id.rs @@ -0,0 +1,374 @@ +use std::fmt; +use std::ops::Deref; +use std::str::from_utf8; + +use serde::{Deserialize, Serialize}; + +use str0m_proto::Id; +use str0m_proto::NonCryptographicRng; + +macro_rules! str_id { + ($id:ident, $name:literal, $num:tt, $new_len:tt) => { + impl $id { + /// Creates a new random id. + pub fn new() -> $id { + let mut arr = Id::<$num>::random().into_array(); + for i in $new_len..$num { + arr[i] = b' '; + } + $id(arr) + } + + /// Converts an array of bytes to an id. + pub const fn from_array(a: [u8; $num]) -> $id { + $id(a) + } + } + + impl fmt::Display for $id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = self; + write!(f, "{}", s) + } + } + + impl fmt::Debug for $id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = self; + write!(f, "{}({})", $name, s) + } + } + + impl Deref for $id { + type Target = str; + + fn deref(&self) -> &Self::Target { + from_utf8(&self.0).expect("ascii id").trim() + } + } + + impl<'a> From<&'a str> for $id { + fn from(v: &'a str) -> Self { + let v = v + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::(); + + let bytes = v.as_bytes(); + let bytes = &bytes[0..$num.min(bytes.len())]; + + // pad with space. + let mut array = [b' '; $num]; + + let max = bytes.len().min(array.len()); + (&mut array[0..max]).copy_from_slice(bytes); + + $id(array) + } + } + + impl Default for $id { + fn default() -> Self { + $id::new() + } + } + }; +} + +macro_rules! num_id { + ($id:ident, $t:tt) => { + impl $id { + /// Creates a new random id. + pub fn new() -> Self { + loop { + let v = NonCryptographicRng::$t(); + // At least Ssrc assigns special meaning to 0, + // but it's fine to avoid for the other numeric + // ids as well. + if v != 0 { + return $id(v); + } + } + } + } + + impl Deref for $id { + type Target = $t; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl From<$t> for $id { + fn from(v: $t) -> Self { + $id(v) + } + } + + impl fmt::Display for $id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } + } + }; +} + +/// Media identifier. +/// +/// In SDP this is found per m-line with the attribute `a=mid:`. +/// +/// When using Direct API we still need `Mid`, since they group individual +/// encoded streams. For example a simulcast of 3 layers would have +/// 3 incoming StreamRx, but since they belong to the same media, +/// the have the same `Mid`. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Mid([u8; 16]); +str_id!(Mid, "Mid", 16, 3); + +/// Identifier of a simulcast layer for an encoded stream. +/// +/// The abbreviation means "RTP Stream Id", which is a very confusing name, because +/// everything in RTP is a stream. People tend to just call it "rid". +/// +/// In SDP this is an optional value that will be seen in [`MediaData`][crate::media::MediaData] +/// if the remote peer is configured for simulcast. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Rid([u8; 8]); +str_id!(Rid, "Rid", 8, 3); + +/// Synchronization source. +/// +/// Uniquely identifies a sending source of data. Each video/audio stream would be associated +/// with at least one synchronization source. Multiple sources for the same stream happens +/// for RTX (resend) and simulcast. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Ssrc(u32); +num_id!(Ssrc, u32); + +impl Ssrc { + /// Returns true if this is the probe SSRC (0). + /// + /// libwebrtc uses SSRC 0 for bandwidth estimation probes sent before + /// video media starts. These probes require special handling. + pub fn is_probe(&self) -> bool { + self.0 == 0 + } +} + +/// Payload type. +/// +/// The payload type identifies which codec and format parameters a stream is sent with. +/// The mappings of Pt-Codec + parameters is negotiated in SDP OFFER/ANSWER. +/// +/// PTs in RTP headers are 7 bits. Values >=128 are not valid. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Pt(u8); +num_id!(Pt, u8); + +/// Identifier of an SDP session. +/// +/// This value is rarely interesting, but is part of the SDP OFFER and ANSWER. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SessionId(u64); +num_id!(SessionId, u64); + +/// Sequence number of an RTP packet. +/// +/// An RTP packet is identified by: SSRC + SeqNo. However in the RTP header the sequence number +/// is a `u16`, meaning the value quite quickly "rolls over". To uniquely identify a packet, +/// str0m keeps track of the roll overs and converts the `u16` to `u64` in this `SeqNo`. +/// +/// To get the RTP u16 value from a `SeqNo`, use `as_u16()` or cast it to u16. +/// +/// ``` +/// # use str0m::rtp::SeqNo; +/// let seq_no: SeqNo = 65_537.into(); +/// +/// // Use `as_u16()`. +/// let a = seq_no.as_u16(); +/// // Discard upper 48 bits to get RTP u16. +/// let b = *seq_no as u16; +/// +/// assert_eq!(a, 1); +/// assert_eq!(b, 1); +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SeqNo(u64); +num_id!(SeqNo, u64); + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(Mid, Rid, Ssrc, Pt, SessionId, SeqNo); + +/// TWCC-specific sequence number. +/// +/// Transport-Wide Congestion Control uses its own sequence number space, +/// separate from RTP sequence numbers. This type ensures TWCC sequences +/// cannot be confused with RTP SeqNo values. +/// +/// TWCC sequences are also u64 internally (tracking rollovers), though the +/// wire format uses u16. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, Default, +)] +pub struct TwccSeq(u64); +num_id!(TwccSeq, u64); + +impl SeqNo { + pub(crate) const MAX: SeqNo = SeqNo(u64::MAX); + + /// Check if the `other` sequence number is directly following this. + #[inline(always)] + pub fn is_next(&self, other: SeqNo) -> bool { + if **self >= *other { + return false; + } + *other - **self == 1 + } + + /// Increase (mutate) this sequence number and return the previous value. + #[inline(always)] + pub fn inc(&mut self) -> SeqNo { + let n = SeqNo(self.0); + self.0 += 1; + n + } + + #[inline(always)] + pub(crate) fn is_max(&self) -> bool { + self.0 == Self::MAX.0 + } + + /// The RTP header value (discarding the ROC). + /// + /// This is the same as discarding the top 48 bits by casting to a u16. + /// + /// ``` + /// # use str0m::rtp::SeqNo; + /// let seq_no: SeqNo = 65_537.into(); + /// + /// // Use `as_u16()`. + /// let a = seq_no.as_u16(); + /// + /// assert_eq!(a, 1); + /// ``` + #[inline(always)] + pub fn as_u16(&self) -> u16 { + self.0 as u16 + } + + /// Get the rollover counter (ROC) value. + /// + /// ``` + /// # use str0m::rtp::SeqNo; + /// // More than 2^16, thus rolled over. + /// let seq_no: SeqNo = 95_000.into(); + /// + /// assert_eq!(seq_no.roc(), 1); + /// + /// // Is the same as shifting 16 bits. + /// assert_eq!(seq_no.roc(), 95_000 >> 16); + /// ``` + #[inline(always)] + pub fn roc(&self) -> u64 { + self.0 >> 16 + } +} + +impl TwccSeq { + /// Check if the `other` sequence number is directly following this. + #[inline(always)] + pub fn is_next(&self, other: TwccSeq) -> bool { + if **self >= *other { + return false; + } + *other - **self == 1 + } + + /// Increase (mutate) this sequence number and return the previous value. + #[inline(always)] + pub fn inc(&mut self) -> TwccSeq { + let n = TwccSeq(self.0); + self.0 += 1; + n + } + + /// The TWCC wire format value (discarding the ROC). + /// + /// This is the same as discarding the top 48 bits by casting to a u16. + #[inline(always)] + pub fn as_u16(&self) -> u16 { + self.0 as u16 + } + + /// Get the rollover counter (ROC) value. + #[inline(always)] + pub fn roc(&self) -> u64 { + self.0 >> 16 + } +} + +impl Default for SeqNo { + fn default() -> Self { + // https://www.rfc-editor.org/rfc/rfc3550#page-13 + // The initial value of the sequence number SHOULD be random (unpredictable) + // to make known-plaintext attacks on encryption more difficult + // Upper half of range is avoided in order to prevent SRTP wraparound issues + // during startup. + // Sequence number 0 is avoided for historical reasons, presumably to avoid + // debugability or test usage conflicts. + // i.e the range is (1, 2^15-1) + Self((NonCryptographicRng::u16() % 32767 + 1) as u64) + } +} + +/// Probe cluster identifier for bandwidth estimation. +/// +/// Used to tag TWCC packets as belonging to a specific probe cluster, +/// enabling analysis of probe results when feedback arrives. +/// +/// Uses u64 to avoid wrap-around in long-running connections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +pub struct TwccClusterId(u64); +num_id!(TwccClusterId, u64); + +impl TwccClusterId { + /// Increase (mutate) this cluster ID and return the previous value. + #[inline(always)] + pub fn inc(&mut self) -> TwccClusterId { + let n = TwccClusterId(self.0); + self.0 = self.0.wrapping_add(1); + n + } +} + +impl Pt { + /// Create a PT with a specific value. + /// + /// PTs are 7 bit. Values with 8 bits are not valid in RTP headers. + pub const fn new_with_value(v: u8) -> Pt { + Pt(v) + } +} + +/// A combination of Mid/Rid +/// +/// In many cases they go hand-in-hand. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct MidRid(pub Mid, pub Option); + +impl MidRid { + #[inline(always)] + pub fn mid(&self) -> Mid { + self.0 + } + + #[inline(always)] + pub fn rid(&self) -> Option { + self.1 + } + + pub fn special_equals(&self, other: &MidRid) -> bool { + self.0 == other.0 && (self.1.is_none() || self.1 == other.1) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mod.rs new file mode 100644 index 000000000..22e36b07a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mod.rs @@ -0,0 +1,34 @@ +mod id; +pub(crate) use id::MidRid; +pub use id::{Mid, Pt, Rid, SeqNo, SessionId, Ssrc, TwccClusterId, TwccSeq}; + +mod ext; +pub use ext::{AbsCaptureTime, UserExtensionValues, VideoOrientation}; +pub use ext::{Extension, ExtensionMap, ExtensionSerializer, ExtensionValues}; + +mod dir; +pub use dir::Direction; + +mod mtime; +pub use mtime::Frequency; +pub use mtime::MediaTime; + +mod header; +pub use header::RtpHeader; +pub(crate) use header::{extend_u7, extend_u8, extend_u15, extend_u16, extend_u32}; + +mod srtp; +pub(crate) use srtp::SrtpContext; +pub(crate) use srtp::{SRTCP_OVERHEAD, SRTP_BLOCK_SIZE, SRTP_OVERHEAD}; + +mod rtcp; +pub use rtcp::*; + +pub use str0m_proto::{Bitrate, DataSize}; + +// Max in the RFC 3550 is 255 bytes, we limit it to be modulus 16 for SRTP and to match libWebRTC +pub const MAX_BLANK_PADDING_PAYLOAD_SIZE: usize = 240; + +/// Errors that can arise in RTP. +mod error; +pub use error::RtpError; diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mtime.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mtime.rs new file mode 100644 index 000000000..c26018231 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/mtime.rs @@ -0,0 +1,450 @@ +use std::cmp::Ordering; +use std::fmt::Display; +use std::num::NonZeroU32; +use std::ops::{Add, AddAssign, Sub, SubAssign}; +use std::str::FromStr; +use std::time::Duration; +use std::time::Instant; + +use serde::{Deserialize, Serialize}; + +/// Media timeline frequency as represented by a non-zero unsigned integer. +/// +/// The frequency can be found in the negotiated payload parameters for a +/// media writer. +/// +/// ```no_run +/// # use str0m::Rtc; +/// # use str0m::media::{Mid, MediaTime}; +/// # let rtc: Rtc = todo!(); +/// # +/// // Obtain mid from Event::MediaAdded +/// let mid: Mid = todo!(); +/// +/// // Create a media writer for the mid. +/// let writer = rtc.writer(mid).unwrap(); +/// +/// // Get the payload type (pt) for the wanted codec. +/// let params = writer.payload_params().nth(0).unwrap(); +/// +/// // Obtain the frequency for the selected codec. +/// let freq = params.spec().clock_rate; +/// +/// let mtime = MediaTime::new(2000, freq); +/// ``` +#[derive(Debug, Clone, Copy, Serialize)] +pub struct Frequency(NonZeroU32); + +impl Frequency { + /// Microseconds in a second. + pub const MICROS: Self = Self::make(1_000_000); + + /// 1 / 2 ^ 18 seconds in a second. + pub const FIXED_POINT_6_18: Frequency = Self::make(2u32.pow(18)); // 262_144 + + /// Cycles in a second of a 90 kHz signal. + pub const NINETY_KHZ: Frequency = Self::make(90_000); + + /// Cycles in a second of a 48 kHz signal. + pub const FORTY_EIGHT_KHZ: Frequency = Self::make(48_000); + + /// Cycles in a second of a 16000 Hz signal. + /// + /// This is the nominal clock rate for G722. G722 samples at 16 kHz but its RTP + /// clock rate is 8000 Hz (RFC 3551 §4.5.2). See + /// + pub const SIXTEEN_KHZ: Frequency = Self::make(16_000); + + /// Cycles in a second of a 8000 Hz signal. + pub const EIGHT_KHZ: Frequency = Self::make(8_000); + + /// Milliseconds in a second. + pub const MILLIS: Frequency = Self::make(1_000); + + /// Hundredths in a second. + pub const HUNDREDTHS: Frequency = Self::make(100); + + /// Seconds in a second. + pub const SECONDS: Frequency = Self::make(1); + + /// Private unconditional non-zero constructor for use with constants. + const fn make(v: u32) -> Frequency { + match NonZeroU32::new(v) { + Some(v) => Self(v), + None => panic!("assured non-zero value is zero"), + } + } + + /// Any non-zero u32 is a valid media timeline frequency. + pub const fn from_nonzero(v: NonZeroU32) -> Self { + Self(v) + } + + /// Every u32 is a valid media timeline frequency except zero. + pub fn new(v: u32) -> Option { + NonZeroU32::new(v).map(Self) + } + + /// The frequency as a u32. + pub const fn get(&self) -> u32 { + self.0.get() + } + + /// The frequency as a [`std::num::NonZeroU32`] i.e. including a positivity proof. + pub const fn nonzero(&self) -> NonZeroU32 { + self.0 + } +} + +impl PartialEq for Frequency { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl Eq for Frequency {} + +impl From for NonZeroU32 { + fn from(value: Frequency) -> Self { + value.0 + } +} + +impl From for Frequency { + fn from(value: NonZeroU32) -> Self { + Self(value) + } +} + +impl FromStr for Frequency { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + NonZeroU32::from_str(s).map(Self) + } +} + +impl Display for Frequency { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl<'de> Deserialize<'de> for Frequency { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + NonZeroU32::deserialize(deserializer).map(Self) + } +} + +/// Media timeline offset represented by a count (numerator) / frequency (denominator) in seconds. +/// +/// The numerator is typically the packet time of an Rtp header. The denominator is the clock +/// frequency of the media source (typically 90kHz for video and 48kHz for audio). The denominator +/// is guaranteed to be a positive integer while the numerator could be positive, negative, or zero. +/// +/// The frequency can be found in the negotiated payload parameters for a media writer. +/// +/// ```no_run +/// # use str0m::Rtc; +/// # use str0m::media::{Mid, MediaTime}; +/// # let rtc: Rtc = todo!(); +/// # +/// // Obtain mid from Event::MediaAdded +/// let mid: Mid = todo!(); +/// +/// // Create a media writer for the mid. +/// let writer = rtc.writer(mid).unwrap(); +/// +/// // Get the payload type (pt) for the wanted codec. +/// let params = writer.payload_params().nth(0).unwrap(); +/// +/// // Obtain the frequency for the selected codec. +/// let freq = params.spec().clock_rate; +/// +/// let mtime = MediaTime::new(2000, freq); +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct MediaTime(u64, Frequency); + +impl MediaTime { + /// The additive identity: 0/1. + pub const ZERO: MediaTime = MediaTime::from_secs(0); + + /// Construct a [`MediaTime`] from a guaranteed non-zero [`Frequency`]. + pub const fn new(numer: u64, denom: Frequency) -> Self { + Self(numer, denom) + } + + /// The numerator of the offset time. + #[inline(always)] + pub const fn numer(&self) -> u64 { + self.0 + } + + /// The denominator of the offset time. + #[inline(always)] + pub const fn denom(&self) -> u32 { + self.1.get() + } + + /// The [`Frequency`] of the offset time. + #[inline(always)] + pub const fn frequency(&self) -> Frequency { + self.1 + } + + /// Convenience constructor for numbers of microseconds (v/1_000_000). + #[inline(always)] + pub const fn from_micros(v: u64) -> MediaTime { + MediaTime(v, Frequency::MICROS) + } + + /// Convenience constructor for numbers of 24-bit 6.18 fixed point units (v/2^18). + #[inline(always)] + pub const fn from_fixed_point_6_18(v: u64) -> MediaTime { + MediaTime(v, Frequency::FIXED_POINT_6_18) + } + + /// Convenience constructor for numbers of 90kHz units (v/90_000). + #[inline(always)] + pub const fn from_90khz(v: u64) -> MediaTime { + MediaTime(v, Frequency::NINETY_KHZ) + } + + /// Convenience constructor for numbers of milliseconds (v/1000). + #[inline(always)] + pub const fn from_millis(v: u64) -> MediaTime { + MediaTime(v, Frequency::MILLIS) + } + + /// Convenience constructor for numbers of hundredths of seconds (v/100). + #[inline(always)] + pub const fn from_hundredths(v: u64) -> MediaTime { + MediaTime(v, Frequency::HUNDREDTHS) + } + + /// Convenience constructor for numbers of seconds (v/1). + #[inline(always)] + pub const fn from_secs(v: u64) -> MediaTime { + MediaTime(v, Frequency::SECONDS) + } + + /// Convenience constructor for floating point fractions of seconds as microsecond units. + #[inline(always)] + pub fn from_seconds(v: impl Into) -> MediaTime { + Self::from_micros((v.into() * 1_000_000.0_f64) as u64) + } + + /// A floating point fraction second representation. + #[inline(always)] + pub fn as_seconds(&self) -> f64 { + let denom: f64 = self.1.get().into(); + self.0 as f64 / denom + } + + /// A microsecond representation. + pub const fn as_micros(&self) -> u64 { + self.rebase(Frequency::MICROS).numer() + } + + /// Predicate for checking that the numerator is 0. + #[inline(always)] + pub const fn is_zero(&self) -> bool { + self.0 == 0 + } + + /// Convert this offset time to have a different denominator + /// (frequency). This conversion may lose precision and, after + /// arithmetic operations with other times of higher frequencies, + /// may have a higher frequency. + #[inline(always)] + pub const fn rebase(self, denom: Frequency) -> MediaTime { + if denom.get() == self.1.get() { + self + } else { + let numer = self.0 as i128 * denom.get() as i128 / self.1.get() as i128; + MediaTime::new(numer as u64, denom) + } + } + + #[inline(always)] + fn same_base(t0: MediaTime, t1: MediaTime) -> (MediaTime, MediaTime) { + let max = Frequency(t0.1.0.max(t1.1.0)); + (t0.rebase(max), t1.rebase(max)) + } + + /// Checked `MediaTime` subtraction. Returns [`None`] if the result + /// would be negative. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// use str0m::media::MediaTime; + /// + /// assert_eq!( + /// MediaTime::from_micros(10).checked_sub(MediaTime::from_micros(5)), + /// Some(MediaTime::from_micros(5)) + /// ); + /// assert_eq!( + /// MediaTime::from_micros(5).checked_sub(MediaTime::from_micros(10)), + /// None + /// ); + /// ``` + #[inline] + pub fn checked_sub(self, rhs: MediaTime) -> Option { + let (lhs, rhs) = MediaTime::same_base(self, rhs); + if lhs.0 < rhs.0 { + None + } else { + Some(MediaTime::new(lhs.0 - rhs.0, lhs.1)) + } + } + + /// Saturating `MediaTime` subtraction. Returns [`MediaTime::ZERO`] if the result + /// would be negative. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// use str0m::media::MediaTime; + /// + /// assert_eq!( + /// MediaTime::from_micros(10).saturating_sub(MediaTime::from_micros(5)), + /// MediaTime::from_micros(5) + /// ); + /// assert_eq!( + /// MediaTime::from_micros(5).saturating_sub(MediaTime::from_micros(10)), + /// MediaTime::ZERO + /// ); + /// ``` + #[inline] + pub fn saturating_sub(self, rhs: MediaTime) -> MediaTime { + match self.checked_sub(rhs) { + Some(v) => v, + None => MediaTime::ZERO, + } + } +} + +impl PartialEq for MediaTime { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + let (t0, t1) = MediaTime::same_base(*self, *other); + t0.0 == t1.0 + } +} +impl Eq for MediaTime {} + +#[cfg(feature = "drv")] +crate::drv_identity_copy!(Frequency, MediaTime); + +impl PartialOrd for MediaTime { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MediaTime { + #[inline(always)] + fn cmp(&self, other: &Self) -> Ordering { + let (t0, t1) = MediaTime::same_base(*self, *other); + if t0 == t1 { + Ordering::Equal + } else if t0.0 < t1.0 { + Ordering::Less + } else { + Ordering::Greater + } + } +} + +impl Add for MediaTime { + type Output = MediaTime; + + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + let (t0, t1) = MediaTime::same_base(self, rhs); + MediaTime::new(t0.0 + t1.0, t0.1) + } +} + +impl Sub for MediaTime { + type Output = MediaTime; + + fn sub(self, rhs: Self) -> Self::Output { + self.checked_sub(rhs) + .expect("overflow when subtracting MediaTime") + } +} + +impl SubAssign for MediaTime { + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } +} + +impl Sub for Instant { + type Output = Instant; + + fn sub(self, rhs: MediaTime) -> Self::Output { + self - Duration::from(rhs) + } +} + +impl SubAssign for Instant { + fn sub_assign(&mut self, rhs: MediaTime) { + *self = *self - rhs; + } +} + +impl AddAssign for MediaTime { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + +impl Add for Instant { + type Output = Instant; + + fn add(self, rhs: MediaTime) -> Self::Output { + self + Duration::from(rhs) + } +} + +impl AddAssign for Instant { + fn add_assign(&mut self, rhs: MediaTime) { + *self = *self + rhs; + } +} + +impl From for Duration { + fn from(value: MediaTime) -> Self { + Duration::from_micros(value.rebase(Frequency::MICROS).numer()) + } +} + +impl From for MediaTime { + fn from(v: Duration) -> Self { + MediaTime::new(v.as_micros() as u64, Frequency::MICROS) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn ts_rebase() { + let t1 = MediaTime::from_seconds(10.0); + let t2 = t1.rebase(Frequency::NINETY_KHZ); + assert_eq!(t2.numer(), 90_000 * 10); + assert_eq!(t2.denom(), 90_000); + + println!("{}", (10.0234_f64).fract()); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/bb.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/bb.rs new file mode 100644 index 000000000..6f560d0da --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/bb.rs @@ -0,0 +1,62 @@ +use super::{FeedbackMessageType, ReportList, RtcpHeader, RtcpPacket, RtcpType, Ssrc}; + +/// RTCP packet BY +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Goodbye { + /// The SSRC that are no longer in use. + pub reports: ReportList, +} + +impl RtcpPacket for Goodbye { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::Goodbye, + feedback_message_type: FeedbackMessageType::SourceCount(self.reports.len() as u8), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // each ssrc is one word + 1 + self.reports.len() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(&mut buf[..4]); + let buf = &mut buf[4..]; + for (i, s) in self.reports.iter().enumerate() { + buf[i * 4..(i + 1) * 4].copy_from_slice(&s.to_be_bytes()); + } + + self.length_words() * 4 + } +} + +impl<'a> TryFrom<(usize, &'a [u8])> for Goodbye { + type Error = &'static str; + + fn try_from((count, buf): (usize, &'a [u8])) -> Result { + if buf.len() < 4 { + return Err("Less than 4 bytes for Goodbye"); + } + + let mut reports = ReportList::new(); + let mut buf = buf; + + if buf.len() < count * 4 { + return Err("Less than count * 4 bytes for Goodbye"); + } + + let max = count.min(31); + + for _ in 0..max { + let ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + reports.push(ssrc); + buf = &buf[4..]; + } + + // TODO parse the optional reason string here. + + Ok(Goodbye { reports }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fir.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fir.rs new file mode 100644 index 000000000..80189367f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fir.rs @@ -0,0 +1,94 @@ +use super::list::private::WordSized; +use super::{FeedbackMessageType, PayloadType, ReportList, RtcpHeader, RtcpPacket, RtcpType, Ssrc}; + +/// Full Intra Request (FIR). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fir { + /// Sender of this feedback. Mostly irrelevant, but part of RTCP packets. + pub sender_ssrc: Ssrc, + /// The SSRC needing a full codec restart. + pub reports: ReportList, +} + +/// Entry reported needing a codec restart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FirEntry { + /// The SSRC needing a full codec restart. + pub ssrc: Ssrc, + /// Counter keeping track of which restart request this is. + pub seq_no: u8, +} + +impl RtcpPacket for Fir { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::PayloadSpecificFeedback, + feedback_message_type: FeedbackMessageType::PayloadFeedback( + PayloadType::FullIntraRequest, + ), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // header + // sender SSRC + // media SSRC (set to 0) + // reports * FirEntry: SSRC + seqNo + 1 + 1 + 1 + self.reports.len() * 2 + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(&mut buf[..4]); + + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + + let media_ssrc = self.reports.iter().next().map(|s| *s.ssrc).unwrap_or(0); + buf[8..12].copy_from_slice(&media_ssrc.to_be_bytes()); + + let mut buf = &mut buf[12..]; + for r in &self.reports { + buf[0..4].copy_from_slice(&r.ssrc.to_be_bytes()); + buf[4..8].copy_from_slice(&[r.seq_no, 0, 0, 0]); + buf = &mut buf[8..]; + } + + 4 + 4 + 4 + self.reports.len() * 8 + } +} + +impl WordSized for FirEntry { + fn word_size(&self) -> usize { + 2 + } +} + +impl<'a> TryFrom<&'a [u8]> for Fir { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 16 { + return Err("Fir less than 16 bytes"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + + let mut reports = ReportList::new(); + + let mut buf = &buf[8..]; + let count = buf.len() / 8; + let max = count.min(31); + + for _ in 0..max { + let ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let seq_no = buf[4]; + reports.push(FirEntry { ssrc, seq_no }); + buf = &buf[8..]; + } + + Ok(Fir { + sender_ssrc, + reports, + }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fmt.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fmt.rs new file mode 100644 index 000000000..00c0ada5c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/fmt.rs @@ -0,0 +1,132 @@ +/// Number of _something_ in the RTCP packet. +/// +/// PacketType determines how to interpret the count field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FeedbackMessageType { + /// When packet type SenderReport or ReceiverReport. + /// + /// The contained u8 is number of receiver reports. + ReceptionReport(u8), + + /// When packet type SourceDescription (SDES) or Goodbye + /// + /// The contained u8 is number of contained SDES or Goodbyes. + SourceCount(u8), + + /// When packet type ApplicationDefined + /// + /// The contained u8 is a subtype which is upp to the application. + Subtype(u8), + + /// When packet type is TransportLayerFeedback. + TransportFeedback(TransportType), + + /// When packet type is PayloadSpecificFeedback. + PayloadFeedback(PayloadType), + + /// When the packet type is ExtendedReport + NotUsed, +} + +impl FeedbackMessageType { + pub fn count(&self) -> u8 { + match self { + FeedbackMessageType::ReceptionReport(v) => *v, + FeedbackMessageType::SourceCount(v) => *v, + _ => panic!("Not a count"), + } + } +} + +impl From for u8 { + fn from(val: FeedbackMessageType) -> Self { + use FeedbackMessageType::*; + match val { + ReceptionReport(v) | SourceCount(v) | Subtype(v) => { + assert!(v <= 31, "rtcp fmt when count must be <= 31"); + v + } + TransportFeedback(v) => v as u8, + PayloadFeedback(v) => v as u8, + NotUsed => 0, + } + } +} + +/// Subtypes of [`FeedbackMessageType::TransportFeedback`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransportType { + /// Nack RTCP packet. + /// + /// Definition: + Nack = 1, + + /// Transportwide congestion control packet. + /// + /// Definition: + TransportWide = 15, +} + +impl TryFrom for TransportType { + type Error = &'static str; + + fn try_from(v: u8) -> Result { + use TransportType::*; + match v { + 1 => Ok(Nack), + 15 => Ok(TransportWide), + _ => { + trace!("Uknown TransportType: {}", v); + Err("Uknown TransportType") + } + } + } +} + +/// Subtypes of [`FeedbackMessageType::PayloadFeedback`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PayloadType { + /// PLI packet type. + /// + /// Definition: + PictureLossIndication = 1, + + /// SLI packet type. + /// + /// Definition: + SliceLossIndication = 2, + + /// RPSI packet type. + /// + /// Definition: + ReferencePictureSelectionIndication = 3, + + /// FIR packet type. + /// + /// Definition: + FullIntraRequest = 4, + + /// Application specific type. + /// + /// Definition: + ApplicationLayer = 15, +} + +impl TryFrom for PayloadType { + type Error = &'static str; + + fn try_from(v: u8) -> Result { + use PayloadType::*; + match v { + 1 => Ok(PictureLossIndication), + 2 => Ok(SliceLossIndication), + 3 => Ok(ReferencePictureSelectionIndication), + 4 => Ok(FullIntraRequest), + 15 => Ok(ApplicationLayer), + _ => { + trace!("Uknown PayloadType: {}", v); + Err("Uknown PayloadType") + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/header.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/header.rs new file mode 100644 index 000000000..0185b8d6d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/header.rs @@ -0,0 +1,198 @@ +#![allow(clippy::unusual_byte_groupings)] + +use super::{FeedbackMessageType, PayloadType, TransportType}; + +pub(crate) const LEN_HEADER: usize = 4; + +#[derive(Debug, PartialEq, Eq)] +pub struct RtcpHeader { + pub(crate) rtcp_type: RtcpType, + pub(crate) feedback_message_type: FeedbackMessageType, + pub(crate) words_less_one: u16, +} + +impl RtcpHeader { + /// Type of RTCP packet. This is further divided into subtypes by + /// `feedback_message_type`. + pub fn rtcp_type(&self) -> RtcpType { + self.rtcp_type + } + + /// Subtype of RTCP message. + pub fn feedback_message_type(&self) -> FeedbackMessageType { + self.feedback_message_type + } + + /// Number of reports stacked in this message. + /// + /// Depending `rtcp_type` and `feedback_message_type` multiple reports of the same type can be + /// stacked into one RTCP packet under the same header. + pub fn count(&self) -> usize { + match self.rtcp_type { + RtcpType::SenderReport + | RtcpType::ReceiverReport + | RtcpType::SourceDescription + | RtcpType::Goodbye => self.feedback_message_type.count() as usize, + RtcpType::ApplicationDefined => 1, + RtcpType::TransportLayerFeedback => { + let transport_type = match self.feedback_message_type { + FeedbackMessageType::TransportFeedback(v) => v, + _ => unreachable!(), + }; + + match transport_type { + TransportType::Nack => { + // [ssrc_sender, ssrc_media_source, fci, fci, ...] + let fci_length = self.length_words() * 4 - LEN_HEADER - 2 * 4; + + // each fci is one word: [pid, blp] + fci_length / 4 + } + TransportType::TransportWide => { + // TODO + 0 + } + } + } + RtcpType::PayloadSpecificFeedback => { + let payload_type = match self.feedback_message_type { + FeedbackMessageType::PayloadFeedback(v) => v, + _ => unreachable!(), + }; + + match payload_type { + PayloadType::PictureLossIndication => { + // PLI does not require parameters. Therefore, the length field MUST be + // 2, and there MUST NOT be any Feedback Control Information. + 1 + } + + PayloadType::FullIntraRequest => { + // [ssrc_sender, ssrc_media_source, fci, fci, ...] + let fci_length = self.length_words() * 4 - LEN_HEADER - 2 * 4; + + // each fci is two words: [ssrc, [seq_no, reserved]] + fci_length / 8 + } + + _ => { + // PayloadType::SliceLossIndication => {}, + // PayloadType::ReferencePictureSelectionIndication => {}, + // PayloadType::ApplicationLayer => {}, + 0 + } + } + } + RtcpType::ExtendedReport => 0, + } + } + + /// Length of entire RTCP packet (including header) in words (4 bytes). + pub fn length_words(&self) -> usize { + self.words_less_one as usize + 1 + } + + /// Write header to buffer. + pub(crate) fn write_to(&self, buf: &mut [u8]) -> usize { + let fmt: u8 = self.feedback_message_type.into(); + + buf[0] = 0b10_0_00000 | fmt; + buf[1] = self.rtcp_type as u8; + + buf[2..4].copy_from_slice(&self.words_less_one.to_be_bytes()); + + 4 + } +} + +/// Kind of RTCP packet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RtcpType { + /// RTCP_PT_SR + SenderReport = 200, + + /// RTCP_PT_RR + ReceiverReport = 201, + + /// RTCP_PT_SDES + SourceDescription = 202, + + /// RTCP_PT_BYE + Goodbye = 203, + + /// RTCP_PT_APP + ApplicationDefined = 204, + + /// RTCP_PT_RTPFB + // https://tools.ietf.org/html/rfc4585 + TransportLayerFeedback = 205, + + /// RTCP_PT_PSFB + // https://tools.ietf.org/html/rfc4585 + PayloadSpecificFeedback = 206, + + /// RTCP_PT_XR + ExtendedReport = 207, +} + +impl TryFrom for RtcpType { + type Error = &'static str; + + fn try_from(v: u8) -> Result { + use RtcpType::*; + match v { + 200 => Ok(SenderReport), // sr + 201 => Ok(ReceiverReport), // rr + 202 => Ok(SourceDescription), + 203 => Ok(Goodbye), + 204 => Ok(ApplicationDefined), + 205 => Ok(TransportLayerFeedback), + 206 => Ok(PayloadSpecificFeedback), + 207 => Ok(ExtendedReport), + _ => { + trace!("Unknown RtcpType: {}", v); + Err("Unknown RtcpType") + } + } + } +} + +impl<'a> TryFrom<&'a [u8]> for RtcpHeader { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 4 { + return Err("Need 4 bytes for RTCP header"); + } + + let version = (buf[0] & 0b11_0_00000) >> 6; + if version != 2 { + return Err("RTCP header version should be 2"); + } + + let fmt = buf[0] & 0b00_0_11111; + let rtcp_type: RtcpType = buf[1].try_into()?; + + let feedback_message_type = { + use FeedbackMessageType::*; + match rtcp_type { + RtcpType::SenderReport => ReceptionReport(fmt), + RtcpType::ReceiverReport => ReceptionReport(fmt), + RtcpType::SourceDescription => SourceCount(fmt), + RtcpType::Goodbye => SourceCount(fmt), + RtcpType::ApplicationDefined => Subtype(fmt), + RtcpType::TransportLayerFeedback => TransportFeedback(fmt.try_into()?), + RtcpType::PayloadSpecificFeedback => PayloadFeedback(fmt.try_into()?), + RtcpType::ExtendedReport => NotUsed, + } + }; + + let words_less_one = u16::from_be_bytes([buf[2], buf[3]]); + + Ok(RtcpHeader { + rtcp_type, + feedback_message_type, + words_less_one, + }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/list.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/list.rs new file mode 100644 index 000000000..19b222a29 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/list.rs @@ -0,0 +1,252 @@ +use std::fmt; +use std::ops::Index; + +/// List containing max 31 items. +#[derive(Clone, PartialEq, Eq)] +pub struct ReportList([Option; 31]); + +impl ReportList { + pub(crate) fn new() -> Self { + ReportList::default() + } + + /// Number of elements in the list. + pub fn len(&self) -> usize { + self.0.iter().position(|i| i.is_none()).unwrap_or(31) + } + + pub(crate) fn push(&mut self, v: T) { + let pos = self.len(); + self.0[pos] = Some(v); + } + + /// Get element at position. + pub fn get(&self, i: usize) -> Option<&T> { + self.0[i].as_ref() + } + + /// Iterator over the elements in the list. + pub fn iter(&self) -> impl Iterator { + self.into_iter() + } + + /// Tells if the list contains zero elements. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub(crate) fn lists_from_iter(iterator: impl IntoIterator) -> Vec { + let mut result = vec![]; + let mut current = Self::default(); + + for (i, item) in iterator.into_iter().enumerate() { + if i % 31 == 0 && i != 0 { + result.push(current); + current = Self::default(); + } + + current.push(item); + } + + if !current.is_empty() { + result.push(current); + } + + result + } + + pub(crate) fn is_full(&self) -> bool { + self.len() == 31 + } +} + +impl ReportList { + pub(crate) fn append_all_possible(&mut self, other: &mut Self, mut words_left: usize) -> usize { + // Position where we start inserting in self. + let pos = self.len(); + + // Max number we can move. + let max = (31 - pos).min(other.len()); + + // after this loop ends, i will hold the number of items moved. + let mut i = 0; + loop { + if i == max { + break; + } + + // first borrow item from other, to check the item size will fit. + let item = other.0[i].as_ref().unwrap(); + let item_size = item.word_size(); + + // can we fit one more item? + if words_left < item_size { + break; + } + + // it fits, move it. + self.0[pos + i] = other.0[i].take(); + + // reduce space left. + words_left -= item_size; + + i += 1; + } + + // shift down remaining in other + for j in i..31 { + other.0[j - i] = other.0[j].take(); + } + + // return number of appended items. + i + } +} + +pub(crate) mod private { + pub trait WordSized { + fn word_size(&self) -> usize; + } +} + +impl Index for ReportList { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + self.0[index].as_ref().unwrap() + } +} + +impl Default for ReportList { + fn default() -> Self { + // We don't want to require T: Copy. + ReportList([ + None, None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, + ]) + } +} + +impl<'a, T> IntoIterator for &'a ReportList { + type Item = &'a T; + type IntoIter = Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + Iter(self, 0) + } +} + +impl IntoIterator for ReportList { + type Item = T; + type IntoIter = IterOwned; + + fn into_iter(self) -> Self::IntoIter { + IterOwned(self, 0) + } +} + +pub struct Iter<'a, T>(&'a ReportList, usize); + +pub struct IterOwned(ReportList, usize); + +impl<'a, T> Iterator for Iter<'a, T> { + type Item = &'a T; + + fn next(&mut self) -> Option { + if self.1 > 30 { + return None; + } + + let n = self.0.0[self.1].as_ref(); + if n.is_some() { + self.1 += 1; + } + + n + } +} + +impl Iterator for IterOwned { + type Item = T; + + fn next(&mut self) -> Option { + if self.1 > 30 { + return None; + } + + let n = self.0.0[self.1].take(); + if n.is_some() { + self.1 += 1; + } + n + } +} + +impl From for ReportList { + fn from(t: T) -> Self { + let mut l = ReportList::default(); + l.push(t); + l + } +} + +impl fmt::Debug for ReportList { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + let len = self.len(); + for (i, s) in self.0.iter().filter_map(|f| f.as_ref()).enumerate() { + if i == len - 1 { + write!(f, "{s:?}")?; + } else { + write!(f, "{s:?},")?; + } + } + write!(f, "]") + } +} + +#[cfg(test)] +mod test { + use super::ReportList; + + #[test] + fn test_lists_from_iter() { + let lists = ReportList::lists_from_iter(0..66); + + assert_eq!(lists.len(), 3); + assert_eq!(lists[0].len(), 31); + assert_eq!(lists[1].len(), 31); + assert_eq!(lists[2].len(), 4); + } + + #[test] + fn test_max_length_iter() { + let list = { + let mut list = ReportList::new(); + for i in 1..=31 { + list.push(i); + } + + list + }; + let sum: u64 = list.iter().sum(); + + assert_eq!(sum, 496); + } + + #[test] + fn test_max_length_into_iter() { + let list = { + let mut list = ReportList::new(); + for i in 1..=31 { + list.push(i); + } + + list + }; + let sum: u64 = list.into_iter().sum(); + + assert_eq!(sum, 496); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/mod.rs new file mode 100644 index 000000000..1f598afee --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/mod.rs @@ -0,0 +1,726 @@ +#![allow(clippy::unusual_byte_groupings)] + +mod header; +use std::collections::VecDeque; + +pub use header::{RtcpHeader, RtcpType}; + +mod list; +pub use list::ReportList; +use list::private::WordSized; + +mod fmt; +pub use fmt::{FeedbackMessageType, PayloadType, TransportType}; + +mod sr; +pub use sr::{SenderInfo, SenderReport}; + +mod rr; +pub use rr::{ReceiverReport, ReceptionReport}; + +mod xr; +pub use xr::{Dlrr, DlrrItem, ExtendedReport, ReportBlock, Rrtr}; + +mod sdes; +pub use sdes::{Descriptions, Sdes, SdesType}; + +mod bb; +pub use bb::Goodbye; + +mod nack; +pub use nack::{Nack, NackEntry}; + +mod pli; +pub use pli::Pli; + +mod fir; +pub use fir::{Fir, FirEntry}; + +mod twcc; +pub use twcc::{Twcc, TwccPacketId, TwccRecvRegister, TwccSendRecord, TwccSendRegister}; + +mod rtcpfb; +pub use rtcpfb::RtcpFb; + +mod psfbapp; +mod remb; +pub use psfbapp::AppSpecificFeedback; +pub use remb::Remb; + +use super::SeqNo; +use super::Ssrc; +use super::extend_u16; + +pub trait RtcpPacket { + /// The... + fn header(&self) -> RtcpHeader; + + /// Length of entire RTCP packet (including header) in words (4 bytes). + fn length_words(&self) -> usize; + + /// Write this packet to the buffer. + /// + /// Panics if the buffer doesn't have capacity to hold length_words * 4 bytes. + fn write_to(&self, buf: &mut [u8]) -> usize; +} + +/// RTCP reports handled by str0m. +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Rtcp { + /// Sender report. Also known as SR. + SenderReport(SenderReport), + /// Receiver report. Also known as RR. + ReceiverReport(ReceiverReport), + /// Extended receiver report. Sometimes called XR. + /// + /// Always sent together with a receiver report. + ExtendedReport(ExtendedReport), + /// Description of Synchronization Sources (senders). + SourceDescription(Descriptions), + /// BYE. When a stream is over. + Goodbye(Goodbye), + /// Reports missing packets. + Nack(Nack), + /// Picture Loss Indiciation. When decoding a picture is not possible. + Pli(Pli), + /// Full Intra Request. Complete restart of a video decoder. + Fir(Fir), + /// Transport Wide Congestion Control. Feedback for every received RTP packet. + Twcc(Twcc), + /// Receiver Estimated Maximum Bitrate. Feedback to the sender about the maximum bitrate. + Remb(Remb), + /// Application-specific Payload-Specific Feedback (FMT=15, PT=206) that is not REMB. + /// Used for application-specific PSFB feedback (RFC 4585 Section 6.4). + AppSpecificFeedback(AppSpecificFeedback), +} + +impl Rtcp { + pub(crate) fn read_packet(buf: &[u8], feedback: &mut VecDeque) { + let mut buf = buf; + loop { + if buf.is_empty() { + break; + } + + let header: RtcpHeader = match buf.try_into() { + Ok(v) => v, + Err(e) => { + debug!("{}", e); + break; + } + }; + let has_padding = buf[0] & 0b00_1_00000 > 0; + let full_length = header.length_words() * 4; + + if full_length > buf.len() { + // this length is incorrect. + break; + } + + let unpadded_length = if has_padding { + let pad = buf[full_length - 1] as usize; + if full_length < pad { + debug!("buf.len() is less than padding: {} < {}", full_length, pad); + break; + } + full_length - pad + } else { + full_length + }; + + match (&buf[..unpadded_length]).try_into() { + Ok(v) => feedback.push_back(v), + Err(e) => debug!("{}", e), + } + + buf = &buf[full_length..]; + } + } + + pub(crate) fn write_packet( + feedback: &mut VecDeque, + buf: &mut [u8], + mut output: impl FnMut(Rtcp), + ) -> usize { + if feedback.is_empty() { + return 0; + } + + // Total length, in bytes, shrunk to be on the pad_to boundary. + let total_len = buf.len(); + + // Capacity in words + let word_capacity = total_len / 4; + + // Pack RTCP feedback packets. Merge together ones of the same type. + Rtcp::pack(feedback, word_capacity); + + let mut offset = 0; + while let Some(fb) = feedback.front() { + // Length of next item. + let item_len = fb.length_words() * 4; + + // Capacity left in the buffer. + let capacity = total_len - offset; + if capacity < item_len { + break; + } + + // We definitely can fit the next RTCP item. + let fb = feedback.pop_front().unwrap(); + let written = fb.write_to(&mut buf[offset..]); + + assert_eq!( + written, item_len, + "length_words equals write_to length: {fb:?}" + ); + + // When debugging we can pass an output to get the serialized packets. + output(fb); + + // Move offsets for the amount written. + offset += item_len; + } + + offset + } + + fn merge(&mut self, other: &mut Rtcp, words_left: usize) -> bool { + match (self, other) { + // Stack receiver reports into sender reports. + (Rtcp::SenderReport(sr), Rtcp::ReceiverReport(rr)) => { + let n = sr.reports.append_all_possible(&mut rr.reports, words_left); + n > 0 + } + + // Stack receiver reports. + (Rtcp::ReceiverReport(r1), Rtcp::ReceiverReport(r2)) => { + let n = r1.reports.append_all_possible(&mut r2.reports, words_left); + n > 0 + } + + // Stack source descriptions. + (Rtcp::SourceDescription(s1), Rtcp::SourceDescription(s2)) => { + let n = s1.reports.append_all_possible(&mut s2.reports, words_left); + n > 0 + } + + // Stack source descriptions. + (Rtcp::Goodbye(g1), Rtcp::Goodbye(g2)) => { + let n = g1.reports.append_all_possible(&mut g2.reports, words_left); + n > 0 + } + + // Stack Nack + (Rtcp::Nack(n1), Rtcp::Nack(n2)) if n1.ssrc == n2.ssrc => { + let n = n1.reports.append_all_possible(&mut n2.reports, words_left); + n > 0 + } + + // Stack source descriptions. + (Rtcp::Fir(f1), Rtcp::Fir(f2)) => { + let n = f1.reports.append_all_possible(&mut f2.reports, words_left); + n > 0 + } + + // No merge possible + _ => false, + } + } + + fn is_full(&self) -> bool { + match self { + Rtcp::SenderReport(v) => v.reports.is_full(), + Rtcp::ReceiverReport(v) => v.reports.is_full(), + Rtcp::ExtendedReport(_) => true, + Rtcp::SourceDescription(v) => v.reports.is_full(), + Rtcp::Goodbye(v) => v.reports.is_full(), + Rtcp::Nack(v) => v.reports.is_full(), + Rtcp::Pli(_) => true, + Rtcp::Fir(v) => v.reports.is_full(), + Rtcp::Twcc(_) => true, + Rtcp::Remb(_) => true, + Rtcp::AppSpecificFeedback(_) => true, + } + } + + /// If this RtcpFb contains no reports (anymore). This can happen after + /// merging reports together. + fn is_empty(&self) -> bool { + match self { + // A SenderReport always has, at least, the SenderInfo part. + Rtcp::SenderReport(_) => false, + // ReceiverReport can become empty. + Rtcp::ReceiverReport(v) => v.reports.is_empty(), + // ExtendedReport can become empty. + Rtcp::ExtendedReport(v) => v.blocks.is_empty(), + // SourceDescription can become empty. + Rtcp::SourceDescription(v) => v.reports.is_empty(), + // Goodbye can become empty, + Rtcp::Goodbye(v) => v.reports.is_empty(), + // Nack can become empty + Rtcp::Nack(v) => v.reports.is_empty(), + // Nack is never empty + Rtcp::Pli(_) => false, + // Fir can be merged to empty. + Rtcp::Fir(v) => v.reports.is_empty(), + // A twcc report is never empty. + Rtcp::Twcc(_) => false, + // A REMB report is never empty. + Rtcp::Remb(_) => false, + // An AppSpecificFeedback report is never empty. + Rtcp::AppSpecificFeedback(_) => false, + } + } + + fn pack(feedback: &mut VecDeque, mut word_capacity: usize) { + // Index into feedback of item we are to pack into. + let mut i = 0; + let len = feedback.len(); + + // Need at least on feedback to pack into, and one to take from. + if len < 2 { + return; + } + + // SenderReport/ReceiveReport first for SRTCP. + feedback.make_contiguous().sort_by_key(Self::order_no); + + 'outer: loop { + // If we reach last element, there is no more packing to do. + if i == len - 1 { + break; + } + + let (pack_into, pack_from) = feedback.make_contiguous().split_at_mut(i + 1); + let fb_a = pack_into.last_mut().unwrap(); + + // abort if fb_a won't fit in the spare capacity. + if word_capacity < fb_a.length_words() { + break 'outer; + } + + // if we manage to merge anything into fb_a. + let mut any_change = false; + + // fb_b goes from the item _after_ i + for fb_b in pack_from { + // if fb_a is full (or empty), we don't want to move any more elements into fb_a. + if fb_a.is_full() || fb_a.is_empty() { + break; + } + + // abort if fb_a won't fit in the spare capacity. + if word_capacity < fb_a.length_words() { + break 'outer; + } + + // amount of capacity (in words) left to fill. + let capacity = word_capacity - fb_a.length_words(); + + // attempt to merge some elements into fb_a from fb_b. + let did_merge = fb_a.merge(fb_b, capacity); + any_change |= did_merge; + } + + if !any_change { + word_capacity -= fb_a.length_words(); + i += 1; + } + } + + // Prune empty. + feedback.retain(|f| !f.is_empty()); + } + + fn order_no(&self) -> u8 { + use Rtcp::*; + match self { + // SenderReport/ReceiverReport first since they possibly contain + // the SSRC for the SRTCP encryption. + SenderReport(_) => 0, + ReceiverReport(_) => 1, + + SourceDescription(_) => 2, + Nack(_) => 3, + Pli(_) => 4, + Fir(_) => 5, + Twcc(_) => 6, + Remb(_) => 7, + AppSpecificFeedback(_) => 8, + ExtendedReport(_) => 10, + + // Goodbye last since they remove stuff. + Goodbye(_) => 11, + } + } +} + +impl RtcpPacket for Rtcp { + fn header(&self) -> RtcpHeader { + match self { + Rtcp::SenderReport(v) => v.header(), + Rtcp::ReceiverReport(v) => v.header(), + Rtcp::ExtendedReport(v) => v.header(), + Rtcp::SourceDescription(v) => v.header(), + Rtcp::Goodbye(v) => v.header(), + Rtcp::Nack(v) => v.header(), + Rtcp::Pli(v) => v.header(), + Rtcp::Fir(v) => v.header(), + Rtcp::Twcc(v) => v.header(), + Rtcp::Remb(v) => v.header(), + Rtcp::AppSpecificFeedback(v) => v.header(), + } + } + + fn length_words(&self) -> usize { + match self { + Rtcp::SenderReport(v) => v.length_words(), + Rtcp::ReceiverReport(v) => v.length_words(), + Rtcp::ExtendedReport(v) => v.length_words(), + Rtcp::SourceDescription(v) => v.length_words(), + Rtcp::Goodbye(v) => v.length_words(), + Rtcp::Nack(v) => v.length_words(), + Rtcp::Pli(v) => v.length_words(), + Rtcp::Fir(v) => v.length_words(), + Rtcp::Twcc(v) => v.length_words(), + Rtcp::Remb(v) => v.length_words(), + Rtcp::AppSpecificFeedback(v) => v.length_words(), + } + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + match self { + Rtcp::SenderReport(v) => v.write_to(buf), + Rtcp::ReceiverReport(v) => v.write_to(buf), + Rtcp::ExtendedReport(v) => v.write_to(buf), + Rtcp::SourceDescription(v) => v.write_to(buf), + Rtcp::Goodbye(v) => v.write_to(buf), + Rtcp::Nack(v) => v.write_to(buf), + Rtcp::Pli(v) => v.write_to(buf), + Rtcp::Fir(v) => v.write_to(buf), + Rtcp::Twcc(v) => v.write_to(buf), + Rtcp::Remb(v) => v.write_to(buf), + Rtcp::AppSpecificFeedback(v) => v.write_to(buf), + } + } +} + +impl<'a> TryFrom<&'a [u8]> for Rtcp { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + let header: RtcpHeader = buf.try_into()?; + + // By constraining the length, all subparsing can go + // until they exhaust the buffer length. This presupposes + // padding is removed from the input. + let buf = &buf[4..]; + + Ok(match header.rtcp_type() { + RtcpType::SenderReport => Rtcp::SenderReport(buf.try_into()?), + RtcpType::ReceiverReport => Rtcp::ReceiverReport(buf.try_into()?), + RtcpType::SourceDescription => Rtcp::SourceDescription(buf.try_into()?), + RtcpType::Goodbye => Rtcp::Goodbye((header.count(), buf).try_into()?), + RtcpType::ApplicationDefined => return Err("Ignore RTCP type: ApplicationDefined"), + RtcpType::TransportLayerFeedback => { + let tlfb = match header.feedback_message_type() { + FeedbackMessageType::TransportFeedback(v) => v, + _ => return Err("Expected TransportFeedback in FeedbackMessageType"), + }; + + match tlfb { + TransportType::Nack => Rtcp::Nack(buf.try_into()?), + TransportType::TransportWide => Rtcp::Twcc(buf.try_into()?), + } + } + RtcpType::PayloadSpecificFeedback => { + let plfb = match header.feedback_message_type() { + FeedbackMessageType::PayloadFeedback(v) => v, + _ => return Err("Expected PayloadFeedback in FeedbackMessageType"), + }; + + match plfb { + PayloadType::PictureLossIndication => Rtcp::Pli(buf.try_into()?), + PayloadType::SliceLossIndication => return Err("Ignore PayloadType type: SLI"), + PayloadType::ReferencePictureSelectionIndication => { + return Err("Ignore PayloadType type: RPSI"); + } + PayloadType::FullIntraRequest => Rtcp::Fir(buf.try_into()?), + PayloadType::ApplicationLayer => { + if header.rtcp_type() == RtcpType::PayloadSpecificFeedback { + if let Ok(remb) = Remb::try_from(buf) { + return Ok(Rtcp::Remb(remb)); + } + // Not REMB — parse as generic application-specific feedback + if let Ok(fb) = AppSpecificFeedback::try_from(buf) { + return Ok(Rtcp::AppSpecificFeedback(fb)); + } + } + return Err("Ignore PayloadType: ApplicationLayer"); + } + } + } + RtcpType::ExtendedReport => Rtcp::ExtendedReport(buf.try_into()?), + }) + } +} + +impl WordSized for Ssrc { + fn word_size(&self) -> usize { + 1 + } +} + +/// Pad up to the next word (4 byte) boundary. +fn pad_bytes_to_word(n: usize) -> usize { + let pad = 4 - n % 4; + if pad == 4 { n } else { n + pad } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, SystemTime}; + + use crate::rtp_::MediaTime; + + use super::twcc::{Delta, PacketChunk, PacketStatus}; + use super::*; + + #[test] + fn padding_of_rtcp() { + let mut queue = VecDeque::new(); + let mut twcc = Twcc { + sender_ssrc: 1.into(), + ssrc: 0.into(), + base_seq: 82, + status_count: 3, + reference_time: 25, + feedback_count: 17, + chunks: VecDeque::new(), + delta: VecDeque::new(), + }; + twcc.chunks + .push_back(PacketChunk::Run(PacketStatus::ReceivedSmallDelta, 3)); + twcc.delta.push_back(Delta::Small(0x7c)); + twcc.delta.push_back(Delta::Small(0x93)); + twcc.delta.push_back(Delta::Small(0x84)); + queue.push_back(Rtcp::Twcc(twcc)); + let mut buf = vec![0; 1500]; + let n = Rtcp::write_packet(&mut queue, &mut buf, |_| {}); + buf.truncate(n); + println!("{buf:02x?}"); + assert_eq!( + &buf, + &[ + // TWCC 0xaf got padding bit set + 0xaf, 0xcd, 0x00, 0x06, // + 0x00, 0x00, 0x00, 0x01, // sender SSRC + 0x00, 0x00, 0x00, 0x00, // media SSRC + 0x00, 0x52, // base seq + 0x00, 0x03, // status count + 0x00, 0x00, 0x19, // reference time + 0x11, // feedback count + 0x20, 0x03, // run of 3 + 0x7c, 0x93, 0x84, // three small delta + 0x00, 0x00, 0x03 // padding + ] + ); + } + + #[test] + fn pack_sr_4_rr() { + let now = SystemTime::now(); + let mut queue = VecDeque::new(); + queue.push_back(rr(3)); + queue.push_back(rr(4)); + queue.push_back(rr(5)); + queue.push_back(sr(1, now)); // should be sorted to front + + Rtcp::pack(&mut queue, 350); + + assert_eq!(queue.len(), 1); + + let sr = match queue.pop_front().unwrap() { + Rtcp::SenderReport(v) => v, + _ => unreachable!(), + }; + + assert_eq!(sr.reports.len(), 4); + let mut iter = sr.reports.iter(); + assert_eq!(iter.next().unwrap(), &report(2)); + assert_eq!(iter.next().unwrap(), &report(3)); + assert_eq!(iter.next().unwrap(), &report(4)); + assert_eq!(iter.next().unwrap(), &report(5)); + } + + #[test] + fn pack_4_rr() { + let mut queue = VecDeque::new(); + queue.push_back(rr(1)); + queue.push_back(rr(2)); + queue.push_back(rr(3)); + queue.push_back(rr(4)); + + Rtcp::pack(&mut queue, 350); + + assert_eq!(queue.len(), 1); + + let sr = match queue.pop_front().unwrap() { + Rtcp::ReceiverReport(v) => v, + _ => unreachable!(), + }; + + assert_eq!(sr.reports.len(), 4); + let mut iter = sr.reports.iter(); + assert_eq!(iter.next().unwrap(), &report(1)); + assert_eq!(iter.next().unwrap(), &report(2)); + assert_eq!(iter.next().unwrap(), &report(3)); + assert_eq!(iter.next().unwrap(), &report(4)); + } + + #[test] + fn roundtrip_sr_rr() { + let now = SystemTime::now(); + let mut feedback = VecDeque::new(); + feedback.push_back(sr(1, now)); + feedback.push_back(rr(3)); + feedback.push_back(rr(4)); + feedback.push_back(rr(5)); + + let mut buf = vec![0_u8; 1360]; + let n = Rtcp::write_packet(&mut feedback, &mut buf, |_| {}); + buf.truncate(n); + + let mut parsed = VecDeque::new(); + Rtcp::read_packet(&buf, &mut parsed); + + let Rtcp::SenderReport(s) = parsed.get(0).unwrap() else { + panic!("Not a SenderReport in Rtcp"); + }; + let now2 = s.sender_info.ntp_time; + + let mut compare = VecDeque::new(); + compare.push_back(sr(1, now2)); + compare.push_back(rr(3)); + compare.push_back(rr(4)); + compare.push_back(rr(5)); + Rtcp::pack(&mut compare, 1400); + + assert_eq!(parsed, compare); + + // Ensure ntp_time is not too far off. + let abs = abs_time_delta(now, now2); + assert!(abs < Duration::from_millis(1)); + } + + fn abs_time_delta(st1: SystemTime, st2: SystemTime) -> Duration { + let delta = if st1 > st2 { + st1.duration_since(st2) + } else { + st2.duration_since(st1) + }; + delta.expect("delta should be absolute") + } + + fn sr(ssrc: u32, ntp_time: SystemTime) -> Rtcp { + Rtcp::SenderReport(SenderReport { + sender_info: SenderInfo { + ssrc: ssrc.into(), + ntp_time, + rtp_time: MediaTime::from_secs(4), + sender_packet_count: 5, + sender_octet_count: 6, + }, + reports: report(2).into(), + }) + } + + fn rr(ssrc: u32) -> Rtcp { + Rtcp::ReceiverReport(ReceiverReport { + sender_ssrc: 42.into(), + reports: report(ssrc).into(), + }) + } + + fn report(ssrc: u32) -> ReceptionReport { + ReceptionReport { + ssrc: ssrc.into(), + fraction_lost: 3, + packets_lost: 1234, + max_seq: 4000, + jitter: 5, + last_sr_time: 12, + last_sr_delay: 1, + } + } + + // fn sdes(ssrc: u32) -> RtcpFb { + // RtcpFb::Sdes(Sdes { + // ssrc: ssrc.into(), + // values: vec![ + // (SdesType::NAME, "Martin".into()), + // (SdesType::TOOL, "str0m".into()), + // (SdesType::NOTE, "Writing things right here".into()), + // ], + // }) + // } + + // fn nack(ssrc: u32, pid: u16) -> RtcpFb { + // RtcpFb::Nack(Nack { + // ssrc: ssrc.into(), + // pid, + // blp: 0b1010_0101, + // }) + // } + + // fn gb(ssrc: u32) -> RtcpFb { + // RtcpFb::Goodbye(ssrc.into()) + // } + + #[test] + fn fuzz_failures() { + const TESTS: &[&[u8]] = &[ + // + &[133, 201, 0, 0], + &[191, 202, 54, 74], + &[166, 202, 0, 2, 218, 54, 214, 222, 160, 2, 146, 0, 251], + &[ + 151, 203, 0, 40, 88, 236, 217, 19, 82, 62, 73, 84, 112, 252, 69, 78, 38, 72, 43, 4, + 21, 136, 90, 29, 89, 70, 90, 196, 149, 168, 54, 1, 57, 16, 128, 8, 53, 172, 192, + 248, 175, 7, 92, 54, 82, 153, 179, 204, 181, 64, 94, 211, 67, 77, 110, 252, 181, + 18, 53, 48, 180, 179, 205, 234, 139, 61, 179, 54, 19, 120, 79, 119, 232, 208, 210, + 73, 78, 28, 242, 156, 242, 239, 19, 246, 183, 10, 49, 114, 216, 64, 105, 161, 50, + 99, 156, 113, 153, 90, 207, 53, 145, 96, 158, 198, 224, 114, 9, 20, 30, 156, 220, + 56, 151, 216, 164, 129, 156, 40, 85, 70, 189, 210, 146, 242, 242, 55, 70, 144, 113, + 9, 44, 74, 22, 123, 180, 153, 18, 88, 1, 185, 85, 227, 200, 62, 53, 142, 89, 28, + 37, 128, 223, 36, 248, 117, 26, 182, 173, 112, 42, 1, 2, 117, 203, 114, 179, + ], + &[ + 150, 202, 0, 54, 0, 149, 201, 0, 0, 138, 201, 0, 0, 152, 201, 0, 0, 151, 201, 0, 0, + 150, 201, 0, 0, 141, 201, 0, 0, 159, 201, 0, 0, 150, 201, 0, 0, 159, 201, 0, 0, + 134, 201, 0, 0, 143, 201, 0, 0, 162, 201, 0, 0, 166, 201, 0, 0, 177, 201, 0, 0, + 182, 201, 0, 0, 131, 201, 0, 0, 164, 201, 0, 0, 133, 201, 0, 0, 143, 201, 0, 0, + 174, 201, 0, 0, 186, 201, 0, 0, 165, 201, 0, 0, 173, 201, 0, 0, 186, 201, 0, 0, + 166, 201, 0, 0, 159, 201, 0, 0, 158, 201, 0, 0, 190, 201, 0, 0, 156, 201, 0, 0, + 147, 201, 0, 0, 169, 201, 0, 0, 135, 201, 0, 0, 148, 201, 0, 0, 132, 201, 0, 0, + 138, 201, 0, 0, 162, 201, 0, 0, 185, 201, 0, 0, 157, 201, 0, 0, 183, 201, 0, 0, + 145, 201, 0, 0, 130, 201, 0, 0, 183, 201, 0, 0, 152, 201, 0, 0, 153, 201, 0, 0, + 154, 201, 0, 0, 138, 201, 0, 0, 148, 201, 0, 0, 158, 201, 0, 0, 156, 201, 0, 0, + 181, 201, 0, 0, 173, 201, 0, 0, 171, 201, 0, 0, 169, 201, 0, 0, 167, 201, 41, 216, + ], + &[ + 143, 205, 0, 8, 143, 93, 208, 93, 201, 4, 131, 131, 131, 3, 0, 143, 1, 143, 0, 143, + 0, 80, 143, 231, 231, 0, 143, 181, 202, 0, 143, 236, 242, 0, 238, 21, + ], + ]; + + let mut parsed = VecDeque::new(); + + for t in TESTS { + parsed.clear(); + Rtcp::read_packet(t, &mut parsed); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/nack.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/nack.rs new file mode 100644 index 000000000..7091bd8b0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/nack.rs @@ -0,0 +1,156 @@ +use super::extend_u16; +use super::{FeedbackMessageType, ReportList, RtcpHeader, RtcpPacket, SeqNo}; +use super::{RtcpType, Ssrc, TransportType}; + +use super::list::private::WordSized; + +/// A NACK entry indiciating packets missing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Nack { + /// Sender of this feedback. Mostly irrelevant, but part of RTCP packets. + pub sender_ssrc: Ssrc, + /// The SSRC this nack reports missing packets for. + pub ssrc: Ssrc, + /// The missing nack. This can be multiple segments. + pub reports: ReportList, +} + +/// A range of sequence numbers missing. +#[allow(missing_docs)] +#[derive(Debug, PartialEq, Eq, Default, Clone, Copy)] +pub struct NackEntry { + pub pid: u16, + pub blp: u16, +} + +impl RtcpPacket for Nack { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::TransportLayerFeedback, + feedback_message_type: FeedbackMessageType::TransportFeedback(TransportType::Nack), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // header + // sender SSRC + // media SSRC + // 1 word per NackPair + 1 + 2 + self.reports.len() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(&mut buf[..4]); + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + buf[8..12].copy_from_slice(&self.ssrc.to_be_bytes()); + let mut buf = &mut buf[12..]; + for r in &self.reports { + buf[0..2].copy_from_slice(&r.pid.to_be_bytes()); + buf[2..4].copy_from_slice(&r.blp.to_be_bytes()); + buf = &mut buf[4..]; + } + self.length_words() * 4 + } +} + +impl WordSized for NackEntry { + fn word_size(&self) -> usize { + 1 + } +} + +impl<'a> TryFrom<&'a [u8]> for Nack { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 12 { + return Err("Nack less than 12 bytes"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]).into(); + + let mut reports = ReportList::new(); + + let mut buf = &buf[8..]; + let count = buf.len() / 4; + let max = count.min(31); + + for _ in 0..max { + let pid = u16::from_be_bytes([buf[0], buf[1]]); + let blp = u16::from_be_bytes([buf[2], buf[3]]); + reports.push(NackEntry { pid, blp }); + buf = &buf[4..]; + } + + Ok(Nack { + sender_ssrc, + ssrc, + reports, + }) + } +} + +impl NackEntry { + /// Iterator over sequence numbers missing. + /// + /// The given sequence number is used to interpret ROC. + pub fn into_iter(self, seq_no: SeqNo) -> impl Iterator { + NackEntryIterator(self, 0, seq_no) + } +} + +pub struct NackEntryIterator(NackEntry, u16, SeqNo); + +impl Iterator for NackEntryIterator { + type Item = SeqNo; + + fn next(&mut self) -> Option { + let seq_16 = if self.1 == 0 { + self.1 += 1; + self.0.pid + } else { + loop { + if self.1 >= 17 { + return None; + } + let i = self.1 - 1; + self.1 += 1; + if 1 << i & self.0.blp > 0 { + break self.0.pid.wrapping_add(self.1 - 1); + } + } + }; + let l = extend_u16(Some(*self.2), seq_16); + Some(l.into()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn nack_entry_iter() { + // 196_618 + let seq_no: SeqNo = (65_536_u64 * 3 + 10).into(); + + // 196_508 + let pid = (65_536_u32 - 100) as u16; + + println!("{seq_no:?} {pid:?}"); + + // 196_509, 196_512, 196_524 + let blp = 0b1000_0000_0000_1001; + + let entry = NackEntry { pid, blp }; + + let nacks: Vec<_> = entry.into_iter(seq_no).collect(); + + assert_eq!( + nacks, + vec![196508.into(), 196509.into(), 196512.into(), 196524.into()] + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/pli.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/pli.rs new file mode 100644 index 000000000..8b62037fb --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/pli.rs @@ -0,0 +1,52 @@ +use super::{FeedbackMessageType, PayloadType, RtcpHeader, RtcpPacket}; +use super::{RtcpType, Ssrc}; + +/// Picture loss indicator. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pli { + /// Sender of this feedback. Mostly irrelevant, but part of RTCP packets. + pub sender_ssrc: Ssrc, + /// The SSRC this picture loss indication is for. + pub ssrc: Ssrc, +} + +impl RtcpPacket for Pli { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::PayloadSpecificFeedback, + feedback_message_type: FeedbackMessageType::PayloadFeedback( + PayloadType::PictureLossIndication, + ), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // header + // sender SSRC + // media SSRC + 3 + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(&mut buf[..4]); + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + buf[8..12].copy_from_slice(&self.ssrc.to_be_bytes()); + 12 + } +} + +impl<'a> TryFrom<&'a [u8]> for Pli { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 8 { + return Err("Pli less than 8 bytes"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]).into(); + + Ok(Pli { sender_ssrc, ssrc }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/psfbapp.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/psfbapp.rs new file mode 100644 index 000000000..0547c1f9e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/psfbapp.rs @@ -0,0 +1,167 @@ +use std::sync::Arc; + +use crate::rtp::Ssrc; + +use super::RtcpType; +use super::{FeedbackMessageType, PayloadType, RtcpHeader, RtcpPacket}; + +/// Application-specific Payload-Specific Feedback message (PSFB FMT=15, PT=206). +/// +/// Generic container for FMT=15 RTCP PSFB messages that are not REMB. +/// Carries an opaque application-dependent payload after the standard +/// sender and media SSRC fields (RFC 4585 Section 6.4). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppSpecificFeedback { + /// SSRC of the sender of this feedback message. + pub sender_ssrc: Ssrc, + /// SSRC of the media source this feedback relates to. + pub media_ssrc: Ssrc, + /// Application-dependent payload (after sender_ssrc and media_ssrc). + pub payload: Arc<[u8]>, +} + +impl RtcpPacket for AppSpecificFeedback { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::PayloadSpecificFeedback, + feedback_message_type: FeedbackMessageType::PayloadFeedback( + PayloadType::ApplicationLayer, + ), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // 1 (RTCP header) + 2 (sender_ssrc + media_ssrc) + payload words + let payload_words = self.payload.len().div_ceil(4); + 1 + 2 + payload_words + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + let total_len = self.length_words() * 4; + assert!( + buf.len() >= total_len, + "AppSpecificFeedback::write_to: buffer too small ({} < {})", + buf.len(), + total_len, + ); + + self.header().write_to(&mut buf[..4]); + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + buf[8..12].copy_from_slice(&self.media_ssrc.to_be_bytes()); + + let payload_len = self.payload.len(); + buf[12..][..payload_len].copy_from_slice(&self.payload); + + // Pad to 4-byte boundary + let pad_len = payload_len.next_multiple_of(4) - payload_len; + buf[12 + payload_len..][..pad_len].fill(0); + + 12 + payload_len + pad_len + } +} + +impl<'a> TryFrom<&'a [u8]> for AppSpecificFeedback { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 8 { + return Err("AppSpecificFeedback less than 8 bytes"); + } + + let (ssrc_buf, payload) = buf.split_at(8); + let sender_ssrc = + u32::from_be_bytes([ssrc_buf[0], ssrc_buf[1], ssrc_buf[2], ssrc_buf[3]]).into(); + let media_ssrc = + u32::from_be_bytes([ssrc_buf[4], ssrc_buf[5], ssrc_buf[6], ssrc_buf[7]]).into(); + + Ok(AppSpecificFeedback { + sender_ssrc, + media_ssrc, + payload: payload.into(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_trip() { + let input = AppSpecificFeedback { + sender_ssrc: 1.into(), + media_ssrc: 2.into(), + payload: vec![0x01, 0x00, 0x00, 0x44, 0xAA, 0xBB, 0xCC, 0xDD].into(), + }; + + let mut buf = [0u8; 256]; + let written = input.write_to(&mut buf); + + // Parse back (skip 4-byte RTCP header) + let parsed = AppSpecificFeedback::try_from(&buf[4..written]).unwrap(); + assert_eq!(input.sender_ssrc, parsed.sender_ssrc); + assert_eq!(input.media_ssrc, parsed.media_ssrc); + assert_eq!(input.payload, parsed.payload); + } + + #[test] + fn test_header() { + let fb = AppSpecificFeedback { + sender_ssrc: 0.into(), + media_ssrc: 0.into(), + payload: vec![1, 2, 3, 4].into(), + }; + + let header = fb.header(); + assert_eq!(header.rtcp_type, RtcpType::PayloadSpecificFeedback); + } + + #[test] + fn test_non_aligned_payload() { + // Payload not aligned to 4 bytes — should still round-trip + let input = AppSpecificFeedback { + sender_ssrc: 100.into(), + media_ssrc: 200.into(), + payload: vec![0xAA, 0xBB, 0xCC].into(), // 3 bytes, needs 1 byte padding + }; + + let mut buf = [0u8; 256]; + let written = input.write_to(&mut buf); + + // Total should be 4 (header) + 4 (sender) + 4 (media) + 4 (padded payload) = 16 + assert_eq!(written, 16); + + let parsed = AppSpecificFeedback::try_from(&buf[4..written]).unwrap(); + assert_eq!(parsed.sender_ssrc, input.sender_ssrc); + assert_eq!(parsed.media_ssrc, input.media_ssrc); + // Parsed payload includes padding byte + assert_eq!(parsed.payload.len(), 4); + assert_eq!(&parsed.payload[..3], &[0xAA, 0xBB, 0xCC]); + assert_eq!(parsed.payload[3], 0); // padding + } + + #[test] + fn test_empty_payload() { + let input = AppSpecificFeedback { + sender_ssrc: 1.into(), + media_ssrc: 2.into(), + payload: vec![].into(), + }; + + let mut buf = [0u8; 256]; + let written = input.write_to(&mut buf); + assert_eq!(written, 12); // header + sender + media, no payload + + let parsed = AppSpecificFeedback::try_from(&buf[4..written]).unwrap(); + assert_eq!(parsed.sender_ssrc, input.sender_ssrc); + assert_eq!(parsed.media_ssrc, input.media_ssrc); + assert!(parsed.payload.is_empty()); + } + + #[test] + fn test_too_short_rejected() { + let buf = [0u8; 7]; // Less than 8 bytes + assert!(AppSpecificFeedback::try_from(&buf[..]).is_err()); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/remb.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/remb.rs new file mode 100644 index 000000000..a126d26bd --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/remb.rs @@ -0,0 +1,304 @@ +use crate::rtp::Ssrc; + +use super::RtcpType; +use super::{FeedbackMessageType, PayloadType, RtcpHeader, RtcpPacket}; + +const BITRATE_MAX: f32 = 2.417_842_4e24; //0x3FFFFp+63; +const MANTISSA_MAX: u32 = 0x7FFFFF; +const REMB_OFFSET: usize = 16; + +const UNIQUE_IDENTIFIER: [u8; 4] = *b"REMB"; + +/* + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |V=2|P| FMT=15 | PT=206 | length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of packet sender | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of media source | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Unique identifier 'R' 'E' 'M' 'B' | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Num SSRC | BR Exp | BR Mantissa | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC feedback | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | ... | +*/ + +#[derive(Debug, Clone)] +pub struct Remb { + /// SSRC of sender + pub sender_ssrc: Ssrc, + + /// SSRC of source, in Remb is default 0 + pub ssrc: Ssrc, + + /// Estimated maximum bitrate + pub bitrate: f32, + + /// SSRC entries which this packet applies to + pub ssrcs: Vec, +} + +impl Eq for Remb {} +impl PartialEq for Remb { + fn eq(&self, other: &Self) -> bool { + self.sender_ssrc == other.sender_ssrc + && (self.bitrate as u64) == (other.bitrate as u64) + && self.ssrcs == other.ssrcs + } +} + +impl RtcpPacket for Remb { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::PayloadSpecificFeedback, + feedback_message_type: FeedbackMessageType::PayloadFeedback( + PayloadType::ApplicationLayer, + ), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // header + // remb + // ssrcs + 1 + REMB_OFFSET / 4 + self.ssrcs.len() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + let mut exp = 0; + let mut bitrate = self.bitrate.clamp(0.0, BITRATE_MAX); + + while bitrate >= (1 << 18) as f32 { + bitrate /= 2.0; + exp += 1; + } + + let mantissa = bitrate.floor() as u32; + + self.header().write_to(&mut buf[..4]); + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + buf[8..12].copy_from_slice(&[0; 4]); + buf[12..16].copy_from_slice(&UNIQUE_IDENTIFIER); + buf[16] = self.ssrcs.len() as u8; + // We can't quite use the binary package because + // a) it's a uint24 and b) the exponent is only 6-bits + // Just trust me; this is big-endian encoding. + buf[17] = (exp << 2) as u8 | (mantissa >> 16) as u8; + buf[18] = (mantissa >> 8) as u8; + buf[19] = mantissa as u8; + + // Write the SSRCs at the very end. + for (index, ssrc) in self.ssrcs.iter().enumerate() { + let begin = 4 + REMB_OFFSET + index * 4; + let end = begin + 4; + buf[begin..end].copy_from_slice(&ssrc.to_be_bytes()); + } + + 4 + REMB_OFFSET + self.ssrcs.len() * 4 + } +} + +impl<'a> TryFrom<&'a [u8]> for Remb { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 16 { + return Err("Remb less than 16 bytes"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let media_ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + if media_ssrc != 0 { + return Err("Ssrc must be zero"); + } + + if buf[8] != UNIQUE_IDENTIFIER[0] + || buf[9] != UNIQUE_IDENTIFIER[1] + || buf[10] != UNIQUE_IDENTIFIER[2] + || buf[11] != UNIQUE_IDENTIFIER[3] + { + return Err("Missing remb identifier"); + } + + // The next byte is the number of SSRC entries at the end. + let ssrcs_len = buf[12] as usize; + + // Get the 6-bit exponent value. + let b17 = buf[13]; + let mut exp = (b17 as u64) >> 2; + exp += 127; // bias for IEEE754 + exp += 23; // IEEE754 biases the decimal to the left, abs-send-time biases it to the right + + // The remaining 2-bits plus the next 16-bits are the mantissa. + let b18 = buf[14]; + let b19 = buf[15]; + let mut mantissa = ((b17 & 3) as u32) << 16 | (b18 as u32) << 8 | b19 as u32; + + if mantissa != 0 { + // ieee754 requires an implicit leading bit + while (mantissa & (MANTISSA_MAX + 1)) == 0 { + exp -= 1; + mantissa *= 2; + } + } + + // bitrate = mantissa * 2^exp + let bitrate = f32::from_bits(((exp as u32) << 23) | (mantissa & MANTISSA_MAX)); + + let mut ssrcs = vec![]; + for i in 0..ssrcs_len { + let b_index = 16 + i * 4; + ssrcs.push(u32::from_be_bytes([ + buf[b_index], + buf[b_index + 1], + buf[b_index + 2], + buf[b_index + 3], + ])); + } + + Ok(Remb { + sender_ssrc, + ssrc: 0.into(), + ssrcs, + bitrate, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_receiver_estimated_maximum_bitrate_marshal() { + let input = Remb { + sender_ssrc: 1.into(), + ssrc: 0.into(), + bitrate: 8927168.0, + ssrcs: vec![1215622422], + }; + + let expected = [ + 143, 206, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 82, 69, 77, 66, 1, 26, 32, 223, 72, 116, 237, + 22, + ]; + + let mut output = [0; 1500]; + let len = input.write_to(&mut output); + assert_eq!(expected, output[0..len]); + } + + #[test] + fn test_receiver_estimated_maximum_bitrate_unmarshal() { + // Real data sent by Chrome while watching a 6Mb/s stream + let input = [ + 143, 206, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 82, 69, 77, 66, 1, 26, 32, 223, 72, 116, 237, + 22, + ]; + + // mantissa = []byte{26 & 3, 32, 223} = []byte{2, 32, 223} = 139487 + // exp = 26 >> 2 = 6 + // bitrate = 139487 * 2^6 = 139487 * 64 = 8927168 = 8.9 Mb/s + let expected = Remb { + sender_ssrc: 1.into(), + ssrc: 0.into(), + bitrate: 8927168.0, + ssrcs: vec![1215622422], + }; + + let packet = Remb::try_from(&input[4..]).unwrap(); + assert_eq!(expected, packet); + } + + #[test] + fn test_receiver_estimated_maximum_bitrate_truncate() { + let input = [ + 143, 206, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 82, 69, 77, 66, 1, 26, 32, 223, 72, 116, 237, + 22, + ]; + + // Make sure that we're interpreting the bitrate correctly. + // For the above example, we have: + + // mantissa = 139487 + // exp = 6 + // bitrate = 8927168 + + let mut packet = Remb::try_from(&input[4..]).unwrap(); + assert_eq!(8927168.0, packet.bitrate); + + // Just verify marshal produces the same input. + let mut output = [0; 1500]; + let output_len = packet.write_to(&mut output); + assert_eq!(input, output[0..output_len]); + + // If we subtract the bitrate by 1, we'll round down a lower mantissa + packet.bitrate -= 1.0; + + // bitrate = 8927167 + // mantissa = 139486 + // exp = 6 + + let output_len = packet.write_to(&mut output); + assert_ne!(input, output[0..output_len]); + let expected = [ + 143, 206, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 82, 69, 77, 66, 1, 26, 32, 222, 72, 116, 237, + 22, + ]; + assert_eq!(expected, output[0..output_len]); + + // Which if we actually unmarshal again, we'll find that it's actually decreased by 63 (which is exp) + // mantissa = 139486 + // exp = 6 + // bitrate = 8927104 + + let packet = Remb::try_from(&output[4..]).unwrap(); + assert_eq!(8927104.0, packet.bitrate); + } + + #[test] + fn test_receiver_estimated_maximum_bitrate_overflow() { + // Marshal a packet with the maximum possible bitrate. + let packet = Remb { + sender_ssrc: 0.into(), + ssrc: 0.into(), + bitrate: f32::MAX, + ssrcs: vec![], + }; + + // mantissa = 262143 = 0x3FFFF + // exp = 63 + + let expected = [ + 143, 206, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 82, 69, 77, 66, 0, 255, 255, 255, + ]; + + let mut output = [0; 1500]; + let output_len = packet.write_to(&mut output); + assert_eq!(expected, output[0..output_len]); + + // mantissa = 262143 + // exp = 63 + // bitrate = 0xFFFFC00000000000 + + let packet = Remb::try_from(&output[4..output_len]).unwrap(); + assert_eq!(f32::from_bits(0x67FFFFC0), packet.bitrate); + + // Make sure we marshal to the same result again. + let output_len = packet.write_to(&mut output); + assert_eq!(expected, output[0..output_len]); + + // Finally, try unmarshalling one number higher than we used to be able to handle. + let input = [ + 143, 206, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 82, 69, 77, 66, 0, 188, 0, 0, + ]; + let packet = Remb::try_from(&input[4..]).unwrap(); + assert_eq!(f32::from_bits(0x62800000), packet.bitrate); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rr.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rr.rs new file mode 100644 index 000000000..e65184161 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rr.rs @@ -0,0 +1,133 @@ +use super::Ssrc; +use super::list::private::WordSized; +use super::{FeedbackMessageType, ReportList, RtcpHeader, RtcpPacket, RtcpType}; + +/// A receiver report. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReceiverReport { + /// Sender of this feedback. Mostly irrelevant, but part of RTCP packets. + pub sender_ssrc: Ssrc, + /// The individual reports for received SSRC. + pub reports: ReportList, +} + +/// An individual report of reception. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReceptionReport { + pub ssrc: Ssrc, + pub fraction_lost: u8, + pub packets_lost: u32, // 24 bit + pub max_seq: u32, + pub jitter: u32, + pub last_sr_time: u32, + pub last_sr_delay: u32, +} + +impl RtcpPacket for ReceiverReport { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::ReceiverReport, + feedback_message_type: FeedbackMessageType::ReceptionReport(self.reports.len() as u8), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // * header: 1 + // * sender SSRC + // * reports: x 6 + 1 + 1 + 6 * self.reports.len() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(buf); + + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + + for (i, r) in self.reports.iter().enumerate() { + r.write_to(&mut buf[8 + i * 24..]); + } + + self.length_words() * 4 + } +} + +impl WordSized for ReceptionReport { + fn word_size(&self) -> usize { + 6 + } +} + +impl ReceptionReport { + pub(crate) fn write_to(&self, buf: &mut [u8]) { + buf[0..4].copy_from_slice(&self.ssrc.to_be_bytes()); + buf[4..8].copy_from_slice(&self.packets_lost.to_be_bytes()); + buf[4] = self.fraction_lost; + buf[8..12].copy_from_slice(&self.max_seq.to_be_bytes()); + buf[12..16].copy_from_slice(&self.jitter.to_be_bytes()); + buf[16..20].copy_from_slice(&self.last_sr_time.to_be_bytes()); + buf[20..24].copy_from_slice(&self.last_sr_delay.to_be_bytes()); + } +} + +impl<'a> TryFrom<&'a [u8]> for ReceiverReport { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 4 { + return Err("Less than 4 bytes for ReceiverReport"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + + let mut reports = ReportList::new(); + let mut buf = &buf[4..]; + + let count = buf.len() / 24; + + let max = count.min(31); + + for _ in 0..max { + let report = buf.try_into()?; + reports.push(report); + buf = &buf[24..]; + } + + Ok(ReceiverReport { + sender_ssrc, + reports, + }) + } +} + +impl<'a> TryFrom<&'a [u8]> for ReceptionReport { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 24 { + return Err("Less than 24 bytes for ReceptionReport"); + } + + // Receiver report shape is here + // https://www.rfc-editor.org/rfc/rfc3550#section-6.4.2 + + let ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let fraction_lost = buf[4]; + let packets_lost = u32::from_be_bytes([0, buf[5], buf[6], buf[7]]); + let max_seq = u32::from_be_bytes([buf[8], buf[9], buf[10], buf[11]]); + let jitter = u32::from_be_bytes([buf[12], buf[13], buf[14], buf[15]]); + let last_sr_time = u32::from_be_bytes([buf[16], buf[17], buf[18], buf[19]]); + let last_sr_delay = u32::from_be_bytes([buf[20], buf[21], buf[22], buf[23]]); + + Ok(ReceptionReport { + ssrc, + fraction_lost, + packets_lost, + max_seq, + jitter, + last_sr_time, + last_sr_delay, + }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rtcpfb.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rtcpfb.rs new file mode 100644 index 000000000..ad733093e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/rtcpfb.rs @@ -0,0 +1,175 @@ +use super::{AppSpecificFeedback, DlrrItem, FirEntry, NackEntry, ReceptionReport, Remb}; +use super::{ReportBlock, ReportList, Rrtr, Rtcp, Sdes, SenderInfo, Ssrc, Twcc}; + +/// Normalization of [`Rtcp`] so we can deal with one SSRC at a time. +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum RtcpFb { + SenderInfo(SenderInfo), // tx -> rx + ReceptionReport(ReceptionReport), // rx -> tx + DlrrItem(DlrrItem), // rx <- tx + Rrtr((Rrtr, Ssrc)), // rx -> tx + SourceDescription(Sdes), // tx -> rx + Goodbye(Ssrc), // tx -> rx + Nack(Ssrc, ReportList), // rx -> tx + Pli(Ssrc), // rx -> tx + Fir(FirEntry), // rx -> tx + Twcc(Twcc), // rx -> tx + Remb(Remb), // rx -> tx + AppSpecificFeedback(AppSpecificFeedback), // not stream-routed +} + +impl RtcpFb { + pub fn is_for_rx(&self) -> bool { + matches!( + self, + RtcpFb::SenderInfo(_) + | RtcpFb::SourceDescription(_) + | RtcpFb::Goodbye(_) + | RtcpFb::DlrrItem(_) + ) + } + + pub fn from_rtcp>(t: T) -> impl Iterator { + let mut q = Vec::new(); + let iter = t.into_iter(); + for pkt in iter { + match pkt { + Rtcp::SenderReport(v) => { + q.push(RtcpFb::SenderInfo(v.sender_info)); + q.extend(v.reports.into_iter().map(RtcpFb::ReceptionReport)); + } + Rtcp::ReceiverReport(v) => { + q.extend(v.reports.into_iter().map(RtcpFb::ReceptionReport)); + } + Rtcp::ExtendedReport(v) => { + for block in v.blocks { + match block { + ReportBlock::Rrtr(b) => q.push(RtcpFb::Rrtr((b, v.ssrc))), + ReportBlock::Dlrr(v) => { + q.extend(v.items.iter().map(|i| RtcpFb::DlrrItem(*i))) + } + } + } + } + Rtcp::SourceDescription(v) => { + q.extend(v.reports.into_iter().map(RtcpFb::SourceDescription)); + } + Rtcp::Goodbye(v) => { + q.extend(v.reports.into_iter().map(RtcpFb::Goodbye)); + } + Rtcp::Nack(v) => { + q.push(RtcpFb::Nack(v.ssrc, v.reports)); + } + Rtcp::Pli(v) => { + q.push(RtcpFb::Pli(v.ssrc)); + } + Rtcp::Fir(v) => { + q.extend(v.reports.into_iter().map(RtcpFb::Fir)); + } + Rtcp::Twcc(v) => { + q.push(RtcpFb::Twcc(v)); + } + Rtcp::Remb(v) => { + q.push(RtcpFb::Remb(v)); + } + Rtcp::AppSpecificFeedback(v) => { + q.push(RtcpFb::AppSpecificFeedback(v)); + } + } + } + q.into_iter() + } + + pub fn ssrc(&self) -> Ssrc { + match self { + RtcpFb::SenderInfo(v) => v.ssrc, + RtcpFb::ReceptionReport(v) => v.ssrc, + RtcpFb::DlrrItem(v) => v.ssrc, + RtcpFb::Rrtr((_, ssrc)) => *ssrc, + RtcpFb::SourceDescription(v) => v.ssrc, + RtcpFb::Goodbye(v) => *v, + RtcpFb::Nack(v, _) => *v, + RtcpFb::Pli(v) => *v, + RtcpFb::Fir(v) => v.ssrc, + RtcpFb::Twcc(v) => v.ssrc, + RtcpFb::Remb(v) => v.ssrcs.first().map(|ssrc| (*ssrc).into()).unwrap_or(v.ssrc), + RtcpFb::AppSpecificFeedback(v) => v.media_ssrc, + } + } +} + +#[cfg(test)] +mod tests { + use super::super::AppSpecificFeedback; + use super::*; + use std::collections::VecDeque; + + #[test] + fn app_specific_feedback_passes_through_rtcpfb() { + let fb = AppSpecificFeedback { + sender_ssrc: 100.into(), + media_ssrc: 200.into(), + payload: vec![0xDE, 0xAD, 0xBE, 0xEF].into(), + }; + + let rtcp_items = vec![Rtcp::AppSpecificFeedback(fb)]; + let result: Vec<_> = RtcpFb::from_rtcp(rtcp_items).collect(); + + assert_eq!(result.len(), 1); + assert!(matches!(&result[0], RtcpFb::AppSpecificFeedback(v) if v.media_ssrc == 200.into())); + } + + #[test] + fn app_specific_feedback_write_and_parse_round_trip() { + use crate::rtp_::RtcpPacket; + + let fb = AppSpecificFeedback { + sender_ssrc: 1001.into(), + media_ssrc: 2002.into(), + payload: vec![0x01, 0x00, 0x00, 0x44, 0xAA, 0xBB, 0xCC, 0xDD].into(), + }; + + // Write the RTCP packet + let rtcp = Rtcp::AppSpecificFeedback(fb.clone()); + let mut buf = [0u8; 256]; + let written = rtcp.write_to(&mut buf); + + // Parse it back via the Rtcp parser + let mut parsed = VecDeque::new(); + Rtcp::read_packet(&buf[..written], &mut parsed); + + assert_eq!(parsed.len(), 1); + match parsed.pop_front().unwrap() { + Rtcp::AppSpecificFeedback(parsed_fb) => { + assert_eq!(parsed_fb.sender_ssrc, fb.sender_ssrc); + assert_eq!(parsed_fb.media_ssrc, fb.media_ssrc); + assert_eq!(parsed_fb.payload, fb.payload); + } + _ => panic!("Expected AppSpecificFeedback"), + } + } + + #[test] + fn app_specific_feedback_not_confused_with_remb() { + use crate::rtp_::RtcpPacket; + + // A non-REMB FMT=15 payload should parse as AppSpecificFeedback, not Remb + let fb = AppSpecificFeedback { + sender_ssrc: 42.into(), + media_ssrc: 0.into(), + // Payload that does NOT start with "REMB" magic bytes + payload: vec![0x01, 0x02, 0x03, 0x04].into(), + }; + + let rtcp = Rtcp::AppSpecificFeedback(fb); + let mut buf = [0u8; 256]; + let written = rtcp.write_to(&mut buf); + + let mut parsed = VecDeque::new(); + Rtcp::read_packet(&buf[..written], &mut parsed); + + assert_eq!(parsed.len(), 1); + assert!(matches!(parsed[0], Rtcp::AppSpecificFeedback(_))); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sdes.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sdes.rs new file mode 100644 index 000000000..a764faa2e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sdes.rs @@ -0,0 +1,325 @@ +use std::str::from_utf8; + +use super::list::private::WordSized; +use super::{FeedbackMessageType, RtcpType, Ssrc}; +use super::{ReportList, RtcpHeader, RtcpPacket, pad_bytes_to_word}; + +/// Multiple source descriptions (SDES). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Descriptions { + /// The descriptions. + pub reports: Box>, +} + +/// A single source description (SDES). +#[allow(missing_docs)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sdes { + pub ssrc: Ssrc, + pub values: ReportList<(SdesType, String)>, +} + +/// Types of SDES values. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[allow(missing_docs)] +pub enum SdesType { + /// End of SDES list + END = 0, + /// Canonical name. + CNAME = 1, + /// User name + NAME = 2, + /// User's electronic mail address + EMAIL = 3, + /// User's phone number + PHONE = 4, + /// Geographic user location + LOC = 5, + /// Name of application or tool + TOOL = 6, + /// Notice about the source + NOTE = 7, + /// Private extensions + PRIV = 8, + /// Rtp stream ID. + RtpStreamId = 12, + /// Repaired rtp stream ID. + RepairedRtpStreamId = 13, + /// Who knows + Unknown, +} + +impl RtcpPacket for Descriptions { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::SourceDescription, + feedback_message_type: FeedbackMessageType::SourceCount(self.reports.len() as u8), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // * header: 1 + // * size-per-item * items + 1 + self.reports.iter().map(|r| r.word_size()).sum::() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(buf); + + let mut buf = &mut buf[4..]; + let mut tot = 4; + + for r in &*self.reports { + let n = r.write_to(buf); + buf = &mut buf[n..]; + tot += n; + } + + tot + } +} + +impl Sdes { + fn write_to(&self, buf: &mut [u8]) -> usize { + buf[..4].copy_from_slice(&self.ssrc.to_be_bytes()); + let mut tot = 4; + + let mut buf = &mut buf[4..]; + for (t, v) in &self.values { + let bytes = v.as_bytes(); + let len = bytes.len(); + + buf[0] = *t as u8; + buf[1] = len as u8; + + buf = &mut buf[2..]; + buf[..len].copy_from_slice(bytes); + + buf = &mut buf[len..]; + tot += 2 + len; + } + + buf[0] = SdesType::END as u8; + buf = &mut buf[1..]; + tot += 1; + + let pad = 4 - tot % 4; + if pad < 4 { + #[allow(clippy::needless_range_loop)] + for i in 0..pad { + buf[i] = SdesType::END as u8; + } + tot += pad; + } + + assert!(tot % 4 == 0, "Sdes is padded to word boundary"); + + tot + } +} + +impl WordSized for Sdes { + fn word_size(&self) -> usize { + let byte_size = 4 + + self + .values + .iter() + // 2 here for 2 byte encoding of type + length + .map(|(_, s)| 2 + s.len()) + .sum::() + + 1; // 1 here for the end byte + + let padded = pad_bytes_to_word(byte_size); + + padded / 4 + } +} + +impl From for SdesType { + fn from(v: u8) -> Self { + use SdesType::*; + match v { + 0 => END, + 1 => CNAME, + 2 => NAME, + 3 => EMAIL, + 4 => PHONE, + 5 => LOC, + 6 => TOOL, + 7 => NOTE, + 8 => PRIV, + 12 => RtpStreamId, + 13 => RepairedRtpStreamId, + _ => Unknown, + } + } +} + +impl<'a> TryFrom<&'a [u8]> for Descriptions { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + let mut reports = ReportList::new(); + + let mut buf = buf; + + loop { + if reports.len() == 31 { + break; + } + // For some reason FF sends us a full SDES and then [0,0,0,0] at the end. + // This can't be interpreted as SDES, so we just ignore it. + if buf.len() < 8 { + break; + } + let report: Sdes = buf.try_into()?; + + let len = report.word_size() * 4; + + reports.push(report); + + // word_size() pads up to a 32-bit boundary and may report more bytes + // than remain for a trailing or truncated chunk. Guard against slicing + // past the end (which would panic) and against a zero len that would + // otherwise spin forever. + if len == 0 || len >= buf.len() { + break; + } + + buf = &buf[len..]; + } + + Ok(Descriptions { + reports: Box::new(reports), + }) + } +} + +impl<'a> TryFrom<&'a [u8]> for Sdes { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 8 { + return Err("Less than 8 bytes for Sdes"); + } + + let ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let mut values = ReportList::new(); + + let mut buf = &buf[4..]; + let mut abs = 0; + + loop { + // SDES items are END-terminated rather than count-bounded, so a + // malformed chunk could contain more than the ReportList capacity. + // Stop at the cap rather than panicking on the 32nd push. + if values.is_full() { + break; + } + + // Per RFC 3550 Section 6.5: + // "The list of items in each chunk MUST be terminated by one or more null octets, + // the first of which is interpreted as an item type of end of list. + // No length octet follows the null item type octet, but additional null octets + // MUST be included if needed to pad until the next 32-bit boundary." + // + // This means we need a special case when there's only a single END byte (0x00) + // at the end of the buffer - it's a valid terminator despite not having a length byte. + if buf.len() == 1 && buf[0] == SdesType::END as u8 { + break; + } + + if buf.len() < 2 { + return Err("Less than 2 bytes for next Sdes value"); + } + + let stype: SdesType = buf[0].into(); + + if matches!(stype, SdesType::END) { + // The end of SDES. + + // Each chunk consists of an SSRC/CSRC identifier followed by a list of + // zero or more items, which carry information about the SSRC/CSRC. + // Each chunk starts on a 32-bit boundary. + // + // Items are contiguous, i.e., items are not individually padded to a + // 32-bit boundary. Text is not null terminated because some multi- + // octet encodings include null octets. + // + // No length octet follows the null item type octet, but additional null + // octets MUST be included if needed to pad until the next 32-bit + // boundary. + + let pad = 4 - abs % 4; + if pad < 4 && buf.len() < pad { + return Err("Not enough buf.len() for Sdes padding"); + } + + break; + } + + let len = buf[1] as usize; + + if buf.len() < 2 + len { + return Err("Not enough buf.len() for Sdes value"); + } + buf = &buf[2..]; + abs += 2; + + if let Ok(value) = from_utf8(&buf[..len]) { + values.push((stype, value.to_string())); + } else { + // failed to read as utf-8. skip. + } + + buf = &buf[len..]; + abs += len; + } + + Ok(Sdes { ssrc, values }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn computed_and_write_to_equal() { + let mut buf = vec![0; 1500]; + + for i in 1usize..=255 { + let mut r = ReportList::new(); + r.push(( + SdesType::CNAME, + String::from_utf8_lossy(&vec![b'a'; i][..]).to_string(), + )); + let sdes = Sdes { + ssrc: 1.into(), + values: r, + }; + assert_eq!(sdes.write_to(&mut buf), sdes.word_size() * 4); + } + } + + #[test] + fn cname_serialize_deserialize() { + let mut s1 = Sdes { + ssrc: 1.into(), + values: ReportList::new(), + }; + s1.values.push((SdesType::CNAME, "abc123".into())); + + let mut buf = vec![0; 50]; + let n = s1.write_to(&mut buf); + buf.truncate(n); + + println!("{buf:02x?}"); + + let s2: Sdes = buf.as_slice().try_into().unwrap(); + + assert_eq!(s1, s2); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sr.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sr.rs new file mode 100644 index 000000000..c0c2c6f0f --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/sr.rs @@ -0,0 +1,150 @@ +use std::time::SystemTime; + +use crate::rtp_::MediaTime; +use crate::util::SystemTimeExt; + +use super::{FeedbackMessageType, RtcpType, Ssrc}; +use super::{ReceptionReport, ReportList, RtcpHeader, RtcpPacket}; + +/// A report of packets sent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SenderReport { + /// Information about the sender of this report. + pub sender_info: SenderInfo, + /// A sender report is implicitly also a receiver report. This + /// might hold data that would otherwise come in a separate RR. + pub reports: ReportList, +} + +/// Information about a stream being sent. +/// +/// A subset of the information contained in Sender Reports(SR). +/// +/// See [RFC 3550 6.4.1](https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SenderInfo { + /// The SSRC of the SR originator. + pub ssrc: Ssrc, + /// The 64 bit NTP timestamp converted to a [`SystemTime`]. + pub ntp_time: SystemTime, + /// The RTP timestamp that corresponds to the same point in time as the NTP timestamp above. + pub rtp_time: MediaTime, + /// The total number of packets the sender had sent when this information was generated. + pub sender_packet_count: u32, + /// The total number of octets the sender had sent when this information was generated. + pub sender_octet_count: u32, +} + +impl RtcpPacket for SenderReport { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::SenderReport, + feedback_message_type: FeedbackMessageType::ReceptionReport(self.reports.len() as u8), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // * header: 1 + // * sender info: 6 + // * reports: x 6 + 1 + 6 + 6 * self.reports.len() + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + self.header().write_to(buf); + + self.sender_info.write_to(&mut buf[4..]); + + for (i, r) in self.reports.iter().enumerate() { + r.write_to(&mut buf[28 + i * 24..]); + } + + self.length_words() * 4 + } +} +impl SenderInfo { + fn write_to(&self, buf: &mut [u8]) { + // pub ssrc: Ssrc, + // pub ntp_time: MediaTime, + // pub rtp_time: u32, + // pub sender_packet_count: u32, + // pub sender_octet_count: u32, + buf[..4].copy_from_slice(&self.ssrc.to_be_bytes()); + + let mt = self.ntp_time.as_ntp_64(); + buf[4..12].copy_from_slice(&mt.to_be_bytes()); + + buf[12..16].copy_from_slice(&(self.rtp_time.numer() as u32).to_be_bytes()); + buf[16..20].copy_from_slice(&self.sender_packet_count.to_be_bytes()); + buf[20..24].copy_from_slice(&self.sender_octet_count.to_be_bytes()); + } +} + +impl<'a> TryFrom<&'a [u8]> for SenderReport { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + let sender_info = buf.try_into()?; + + let mut reports = ReportList::new(); + let mut buf = &buf[24..]; + + let count = buf.len() / 24; + + let max = count.min(31); + + for _ in 0..max { + let report = buf.try_into()?; + reports.push(report); + buf = &buf[24..]; + } + + Ok(SenderReport { + sender_info, + reports, + }) + } +} + +impl<'a> TryFrom<&'a [u8]> for SenderInfo { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 24 { + return Err("Less than 24 bytes for SenderInfo"); + } + + // Sender report shape is here + // https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1 + + let ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + + let ntp_time = u64::from_be_bytes([ + buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], + ]); + let ntp_time = SystemTime::from_ntp_64(ntp_time); + + // https://www.cs.columbia.edu/~hgs/rtp/faq.html#timestamp-computed + // For video, time clock rate is fixed at 90 kHz. The timestamps generated + // depend on whether the application can determine the frame number or not. + // If it can or it can be sure that it is transmitting every frame with a + // fixed frame rate, the timestamp is governed by the nominal frame rate. + // Thus, for a 30 f/s video, timestamps would increase by 3,000 for each + // frame, for a 25 f/s video by 3,600 for each frame. + let rtp_time = u32::from_be_bytes([buf[12], buf[13], buf[14], buf[15]]); + // The base (90kHz etc) is set higher up the stack (in StreamRx to be precise). + let rtp_time = MediaTime::from_secs(rtp_time as u64); + + let sender_packet_count = u32::from_be_bytes([buf[16], buf[17], buf[18], buf[19]]); + let sender_octet_count = u32::from_be_bytes([buf[20], buf[21], buf[22], buf[23]]); + + Ok(SenderInfo { + ssrc, + ntp_time, + rtp_time, + sender_packet_count, + sender_octet_count, + }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/twcc.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/twcc.rs new file mode 100644 index 000000000..2469390dc --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/twcc.rs @@ -0,0 +1,2324 @@ +use std::collections::VecDeque; +use std::collections::vec_deque; +use std::fmt; +use std::ops::RangeInclusive; +use std::time::{Duration, Instant}; + +use super::{FeedbackMessageType, RtcpHeader, RtcpPacket, extend_u16}; +use super::{RtcpType, Ssrc, TransportType}; + +use crate::rtp_::{TwccClusterId, TwccSeq}; + +/// Transport Wide Congestion Control. +/// +/// Sent in response to every RTP packet, but does ranges of packets to respond to. +#[derive(Clone, PartialEq, Eq)] +pub struct Twcc { + /// Sender of this feedback. Mostly irrelevant, but part of RTCP packets. + pub sender_ssrc: Ssrc, + /// The SSRC this report is for. + pub ssrc: Ssrc, + /// Start sequence number. + pub base_seq: u16, + /// Number of reported statuses. + pub status_count: u16, + /// Clock time this report was produced. Used for RTT measurement. + pub reference_time: u32, // 24 bit + /// Increasing counter for each TWCC. For deduping. + pub feedback_count: u8, // counter for each Twcc + /// Ranges received. + pub chunks: VecDeque, + /// Delta times for the ranges received. + pub delta: VecDeque, +} + +impl Twcc { + fn chunks_byte_len(&self) -> usize { + self.chunks.len() * 2 + } + + fn delta_byte_len(&self) -> usize { + self.delta.iter().map(|d| d.byte_len()).sum() + } + + /// Iterate over the reported sequences. + pub fn into_iter(self, time_zero: Instant, extend_from: TwccSeq) -> TwccIter { + let millis = self.reference_time as u64 * 64; + let time_base = time_zero + Duration::from_millis(millis); + let base_seq = extend_u16(Some(*extend_from), self.base_seq); + let last_seq = base_seq + self.status_count as u64; + + TwccIter { + base_seq, + last_seq, + time_base, + index: 0, + twcc: self, + } + } +} + +pub struct TwccIter { + base_seq: u64, + last_seq: u64, + time_base: Instant, + index: usize, + twcc: Twcc, +} + +impl Iterator for TwccIter { + type Item = (TwccSeq, PacketStatus, Option); + + fn next(&mut self) -> Option { + let seq: TwccSeq = (self.base_seq + self.index as u64).into(); + + if *seq == self.last_seq { + return None; + } + + let head = self.twcc.chunks.front()?; + + let (status, amount) = match head { + PacketChunk::Run(s, n) => { + use PacketStatus::*; + let status = match s { + NotReceived | Unknown => NotReceived, + ReceivedSmallDelta => ReceivedSmallDelta, + PacketStatus::ReceivedLargeOrNegativeDelta => ReceivedLargeOrNegativeDelta, + }; + (status, *n) + } + PacketChunk::VectorSingle(v, n) => { + let status = if 1 << (13 - self.index) & v > 0 { + PacketStatus::ReceivedSmallDelta + } else { + PacketStatus::NotReceived + }; + (status, *n) + } + PacketChunk::VectorDouble(v, n) => { + let e = ((v >> (12 - self.index * 2)) & 0b11) as u8; + let status = PacketStatus::from(e); + (status, *n) + } + }; + + let instant = match status { + PacketStatus::NotReceived => None, + PacketStatus::ReceivedSmallDelta => match self.twcc.delta.pop_front()? { + Delta::Small(v) => Some(self.time_base + Duration::from_micros(250 * v as u64)), + Delta::Large(_) => panic!("Incorrect large delta size"), + }, + PacketStatus::ReceivedLargeOrNegativeDelta => match self.twcc.delta.pop_front()? { + Delta::Small(_) => panic!("Incorrect small delta size"), + Delta::Large(v) => { + let dur = Duration::from_micros(250 * v.unsigned_abs() as u64); + Some(if v < 0 { + self.time_base.checked_sub(dur).unwrap() + } else { + self.time_base + dur + }) + } + }, + _ => unreachable!(), + }; + + if let Some(new_timebase) = instant { + self.time_base = new_timebase; + } + + self.index += 1; + if self.index == amount as usize { + self.twcc.chunks.pop_front(); + self.base_seq = *seq + 1; + self.index = 0; + } + + Some((seq, status, instant)) + } +} + +impl RtcpPacket for Twcc { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::TransportLayerFeedback, + feedback_message_type: FeedbackMessageType::TransportFeedback( + TransportType::TransportWide, + ), + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + // header: 1 + // sender ssrc: 1 + // ssrc: 1 + // base seq + packet status: 1 + // ref time + feedback count: 1 + // chunks byte len + delta byte len + padding + + let mut total = self.chunks_byte_len() + self.delta_byte_len(); + + let pad = 4 - total % 4; + if pad < 4 { + total += pad; + } + + assert!(total % 4 == 0); + + let total_words = total / 4; + + 5 + total_words + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + let len_start = buf.len(); + + let mut total = { + let buf = &mut buf[..]; + + self.header().write_to(buf); + buf[4..8].copy_from_slice(&self.sender_ssrc.to_be_bytes()); + buf[8..12].copy_from_slice(&self.ssrc.to_be_bytes()); + + buf[12..14].copy_from_slice(&self.base_seq.to_be_bytes()); + buf[14..16].copy_from_slice(&self.status_count.to_be_bytes()); + + let ref_time = self.reference_time.to_be_bytes(); + buf[16..19].copy_from_slice(&ref_time[1..4]); + buf[19] = self.feedback_count; + + let mut buf = &mut buf[20..]; + for p in &self.chunks { + p.write_to(buf); + buf = &mut buf[2..]; + } + + for d in &self.delta { + let n = d.write_to(buf); + buf = &mut buf[n..]; + } + + len_start - buf.len() + }; + + let pad = 4 - total % 4; + if pad < 4 { + for i in 0..pad { + buf[total + i] = 0; + } + buf[total + pad - 1] = pad as u8; + + total += pad; + // Toggle padding bit + buf[0] |= 0b00_1_00000; + } + + total + } +} + +#[derive(Debug)] +pub struct TwccRecvRegister { + // How many packets to keep when they are reported. This is to handle packets arriving out + // of order and where two consecutive calls to `build_report` needs to go "backwards" in + // base_seq. + keep_reported: usize, + + /// Queue of packets to form Twcc reports of. + /// + /// Once the queue has some content, we will always keep at least one entry to "remember" for the + /// next report. + queue: VecDeque, + + /// Index into queue from where we start reporting on next build_report(). + report_from: usize, + + /// Interims built in this for every build_report. + interims: VecDeque, + + /// The point in time we consider 0. All reported values are offset from this. Set to first + /// unreported packet in first `build_reported`. + /// + // https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01#page-5 + // reference time: 24 bits Signed integer indicating an absolute + // reference time in some (unknown) time base chosen by the + // sender of the feedback packets. + time_start: Option, + + /// Counter that increases by one for each report generated. + generated_reports: u64, + + /// Data to calculate received loss. + receive_window: ReceiveWindow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Receipt { + seq: TwccSeq, + time: Instant, +} + +impl TwccRecvRegister { + pub fn new(keep_reported: usize) -> Self { + TwccRecvRegister { + keep_reported, + queue: VecDeque::new(), + report_from: 0, + interims: VecDeque::new(), + time_start: None, + generated_reports: 0, + receive_window: ReceiveWindow::default(), + } + } + + /// Find the last sequence number, or `None` if the queue is empty. + pub fn max_seq(&self) -> Option { + // The highest seq must be the last since update_seq inserts values + // using a binary search. + self.queue.back().map(|r| r.seq) + } + + pub fn update_seq(&mut self, seq: TwccSeq, time: Instant) { + self.receive_window.record_seq(seq); + + match self.queue.binary_search_by_key(&seq, |r| r.seq) { + Ok(_) => { + // Exact same SeqNo found. This is an error where the sender potentially + // used the same twcc sequence number for two packets. Let's ignore it. + } + Err(idx) => { + if let Some(time_start) = self.time_start { + // If time goes back more than 8192 millis from the time point we've + // chosen as our 0, we can't represent that in the report. Let's just + // forget about it and hope for the best. + if time_start - time >= Duration::from_millis(8192) { + return; + } + } + + self.queue.insert(idx, Receipt { seq, time }); + + if idx < self.report_from { + self.report_from = idx; + } + } + } + } + + pub fn build_report(&mut self, max_byte_size: usize) -> Option { + if max_byte_size > 10_000 { + warn!("Refuse to build too large Twcc report"); + return None; + } + + // First unreported is the self.time_start relative offset of the next Twcc. + let first = self.queue.get(self.report_from); + let first = first?; + + // Set once on first ever built report. + if self.time_start.is_none() { + self.time_start = Some(first.time); + } + + let (base_seq, first_time) = (first.seq, first.time); + let time_start = self.time_start.expect("a start time"); + + // The difference between our Twcc reference time and the first ever report start time. + let first_time_rel = first_time - time_start; + + // The value is to be interpreted in multiples of 64ms. + let reference_time = (first_time_rel.as_micros() as u64 / 64_000) as u32; + + let mut twcc = Twcc { + sender_ssrc: 0.into(), + ssrc: 0.into(), + feedback_count: self.generated_reports as u8, + base_seq: *base_seq as u16, + reference_time, + status_count: 0, + chunks: VecDeque::new(), + delta: VecDeque::new(), + }; + + // Because reference time is in steps of 64ms, the first reported packet might have an + // offset (packet time resolution is 250us). This base_time is calculated backwards from + // reference time so that we can offset all packets from the "truncated" 64ms steps. + // The RFC says: + // The first recv delta in this packet is relative to the reference time. + let base_time = time_start + Duration::from_micros(reference_time as u64 * 64_000); + + // The ChunkInterim are helpers structures that hold the deltas between + // the registered receptions. + build_interims( + &self.queue, + self.report_from, + base_seq, + base_time, + &mut self.interims, + ); + let interims = &mut self.interims; + + // 20 bytes is the size of the fixed fields in Twcc. + let mut bytes_left = max_byte_size - 20; + + while !interims.is_empty() { + // 2 chunk + 2 large delta + 3 padding + const MIN_RUN_SIZE: usize = 2 + 2 + 3; + + if bytes_left < MIN_RUN_SIZE { + break; + } + + // Chose the packet chunk type that can fit the most interims. + let (mut chunk, max) = { + let first_status = interims.front().expect("at least one interim").status(); + + let c_run = PacketChunk::Run(first_status, 0); + let c_single = PacketChunk::VectorSingle(0, 0); + let c_double = PacketChunk::VectorDouble(0, 0); + + let max_run = c_run.append_max(interims.iter()); + let max_single = c_single.append_max(interims.iter()); + let max_double = c_double.append_max(interims.iter()); + + let max = max_run.max(max_single).max(max_double); + + // 2 chunk + 14 small delta + 3 padding + const MAX_SINGLE_SIZE: usize = 2 + 14 + 3; + // 2 chunk + 7 large delta + 3 padding + const MAX_DOUBLE_SIZE: usize = 2 + 14 + 3; + + if max == max_run { + (c_run, max_run) + } else if max == max_single && bytes_left >= MAX_SINGLE_SIZE { + (c_single, max_single) + } else if max == max_double && bytes_left >= MAX_DOUBLE_SIZE { + (c_double, max_double) + } else { + // fallback, since we can always do runs. + (c_run, max_run) + } + }; + + // we should _definitely_ be able to fit this many reported. + let mut todo = max; + + loop { + if bytes_left < MIN_RUN_SIZE { + break; + } + + if todo == 0 { + break; + } + + let i = match interims.front_mut() { + Some(v) => v, + None => break, + }; + + let appended = chunk.append(i); + assert!(appended > 0); + todo -= appended; + twcc.status_count += appended; + + if i.consume(appended) { + // it was fully consumed. + if matches!(i, ChunkInterim::Received(_, _)) { + self.report_from += 1; + } + + if let Some(delta) = i.delta() { + twcc.delta.push_back(delta); + bytes_left -= delta.byte_len(); + } + + // move on to next interim + interims.pop_front(); + } else { + // not fully consumed, then we must have run out of space in the chunk. + assert!(todo == 0); + } + } + + let free = chunk.free(); + if chunk.must_be_full() && free > 0 { + // this must be at the end where we can shift in missing + assert!(interims.is_empty()); + chunk.append(&ChunkInterim::Missing(free)); + } + + twcc.chunks.push_back(chunk); + bytes_left -= 2; + } + + // libWebRTC demands at least one chunk, or it will warn with + // "Buffer too small (16 bytes) to fit a FeedbackPacket. Minimum size = 18" + // (18 bytes here is not including the RTCP header). + if twcc.chunks.is_empty() { + return None; + } + + self.generated_reports += 1; + + // clean up + if self.report_from > self.keep_reported { + let to_remove = self.report_from - self.keep_reported; + self.queue.drain(..to_remove); + self.report_from -= to_remove; + } + + Some(twcc) + } + + pub fn has_unreported(&self) -> bool { + self.queue.len() > self.report_from + } + + /// Calculate the fraction of lost packets since the last call. + /// + /// To get periodic stats call this method at fixed intervals. + pub fn loss(&mut self) -> Option { + // Based on the algorithm described in + // [RFC 3550 Appendix A](https://www.rfc-editor.org/rfc/rfc3550#appendix-A.3), but instead + // of applying it to an individual RTP stream it's applied to the whole session using TWCC + // sequence numbers rather than RTP sequence numbers. + let max_seq = self.receive_window.max_seq?; + let base_seq = self.receive_window.base_seq?; + let expected = *max_seq - *base_seq + 1_u64; + + let expected_interval = expected - self.receive_window.expected_prior; + self.receive_window.expected_prior = expected; + + let received_interval = self.receive_window.received - self.receive_window.received_prior; + self.receive_window.received_prior = self.receive_window.received; + + let lost_interval = expected_interval.saturating_sub(received_interval); + + (expected_interval != 0).then_some(lost_interval as f32 / expected_interval as f32) + } +} + +/// Interims are deltas between `Receiption` which is an intermediary format before +/// we populate the Twcc report. +fn build_interims( + queue: &VecDeque, + report_from: usize, + base_seq: TwccSeq, + base_time: Instant, + interims: &mut VecDeque, +) { + interims.clear(); + let report_from = queue.iter().skip(report_from); + + let mut prev = (base_seq, base_time); + + for r in report_from { + let diff_seq = *r.seq - *prev.0; + + if diff_seq > 1 { + let mut todo = diff_seq - 1; + while todo > 0 { + // max 2^13 run length in each missing chunk + let n = todo.min(8192); + interims.push_back(ChunkInterim::Missing(n as u16)); + todo -= n; + } + } + + let diff_time = if r.time < prev.1 { + // negative + let dur = prev.1 - r.time; + -(dur.as_micros() as i64) + } else { + let dur = r.time - prev.1; + dur.as_micros() as i64 + }; + + let (status, time) = if diff_time < -8_192_000 || diff_time > 8_191_750 { + // This is too large to be representable in deltas. + // Abort, make a report of what we got, and start anew. + break; + } else if diff_time < 0 || diff_time > 63_750 { + let t = diff_time / 250; + assert!(t >= -32_768 && t <= 32_767); + (PacketStatus::ReceivedLargeOrNegativeDelta, t as i16) + } else { + let t = diff_time / 250; + assert!(t >= 0 && t <= 255); + (PacketStatus::ReceivedSmallDelta, t as i16) + }; + + interims.push_back(ChunkInterim::Received(status, time)); + prev = (r.seq, r.time); + } +} + +#[derive(Debug, Clone, Copy)] +enum ChunkInterim { + Missing(u16), // max 2^13 (one run length) + Received(PacketStatus, i16), +} + +#[derive(Debug, Default)] +struct ReceiveWindow { + /// The base seq num, set on the first receive. + base_seq: Option, + + /// The large seq num received. + max_seq: Option, + + /// The previous number of packets expected, used to calculate a delta for loss. + expected_prior: u64, + + /// The total number of packets received + received: u64, + + /// The previous number of packets received, used to calculate a delta for loss. + received_prior: u64, +} + +impl ReceiveWindow { + fn record_seq(&mut self, seq: TwccSeq) { + if self.base_seq.is_none() { + self.base_seq = Some(seq); + } + + self.received += 1; + self.max_seq = self.max_seq.max(Some(seq)); + } +} + +impl ChunkInterim { + fn status(&self) -> PacketStatus { + match self { + ChunkInterim::Missing(_) => PacketStatus::NotReceived, + ChunkInterim::Received(s, _) => *s, + } + } + + fn delta(&self) -> Option { + match self { + ChunkInterim::Missing(_) => None, + ChunkInterim::Received(s, d) => match *s { + PacketStatus::ReceivedSmallDelta => Some(Delta::Small(*d as u8)), + PacketStatus::ReceivedLargeOrNegativeDelta => Some(Delta::Large(*d)), + _ => unreachable!(), + }, + } + } + + fn consume(&mut self, n: u16) -> bool { + match self { + ChunkInterim::Missing(c) => { + *c -= n; + *c == 0 + } + ChunkInterim::Received(_, _) => { + assert!(n <= 1); + n == 1 + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PacketChunk { + Run(PacketStatus, u16), // 13 bit repeat + VectorSingle(u16, u16), + VectorDouble(u16, u16), +} + +impl PacketChunk { + fn append_max<'a>(&self, iter: impl Iterator) -> u16 { + let mut to_fill = *self; + + let mut reached_end = true; + + for i in iter { + if to_fill.free() == 0 { + reached_end = false; + break; + } + + // The status is not possible to add in this chunk. This could be + // a large delta in a single, or a mismatching run. + if !to_fill.can_append_status(i.status()) { + reached_end = false; + break; + } + + to_fill.append(i); + } + + // As a special case, single/double must be completely filled. However if + // we reached the end of the interims, we can shift in "missing" to make + // them full. + if to_fill.must_be_full() && to_fill.free() > 0 && !reached_end { + return 0; + } + + self.free() - to_fill.free() + } + + fn append(&mut self, i: &ChunkInterim) -> u16 { + use ChunkInterim::*; + use PacketChunk::*; + let free = self.free(); + match (self, i) { + (Run(s, n), Missing(c)) => { + if *s != PacketStatus::NotReceived { + return 0; + } + let max = free.min(*c); + *n += max; + max + } + (Run(s, n), Received(s2, _)) => { + if *s != *s2 { + return 0; + } + let max = free.min(1); + *n += max; + max + } + (VectorSingle(n, f), Missing(c)) => { + let max = free.min(*c); + *n <<= max; + *f += max; + max + } + (VectorSingle(n, f), Received(s2, _)) => { + if *s2 == PacketStatus::ReceivedLargeOrNegativeDelta { + return 0; + } + let max = free.min(1); + if max == 1 { + *n <<= 1; + *n |= 1; + *f += 1; + } + max + } + (VectorDouble(n, f), Missing(c)) => { + let max = free.min(*c); + *n <<= max * 2; + *f += max; + max + } + (VectorDouble(n, f), Received(s2, _)) => { + let max = free.min(1); + if max == 1 { + *n <<= 2; + *n |= *s2 as u16; + *f += 1; + } + max + } + } + } + + fn must_be_full(&self) -> bool { + match self { + PacketChunk::Run(_, _) => false, + PacketChunk::VectorSingle(_, _) => true, + PacketChunk::VectorDouble(_, _) => true, + } + } + + fn free(&self) -> u16 { + match self { + PacketChunk::Run(_, n) => 8192 - *n, + PacketChunk::VectorSingle(_, filled) => 14 - *filled, + PacketChunk::VectorDouble(_, filled) => 7 - *filled, + } + } + + fn write_to(&self, buf: &mut [u8]) { + let x = match self { + // 0 1 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |T| S | Run Length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // chunk type (T): 1 bit A zero identifies this as a run length chunk. + // packet status symbol (S): 2 bits The symbol repeated in this run. + // See above. + // run length (L): 13 bits An unsigned integer denoting the run length. + PacketChunk::Run(s, n) => { + let mut x = 0_u16; + x |= (*s as u16) << 13; + assert!(*n <= 8192); + x |= n; + x + } + + // Corrected according to email exchange at the bottom.. + // + // 0 1 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |T|S| symbol list | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // chunk type (T): 1 bit A one identifies this as a status vector + // chunk. + // symbol size (S): 1 bit A zero means this vector contains only + // "packet received" (1) and "packet not received" (0) + // symbols. This means we can compress each symbol to just + // one bit, 14 in total. A one means this vector contains + // the normal 2-bit symbols, 7 in total. + // symbol list: 14 bits A list of packet status symbols, 7 or 14 in + // total. + PacketChunk::VectorSingle(n, fill) => { + assert!(*fill == 14); + let mut x: u16 = 1 << 15; + assert!(*n <= 16384); + x |= *n; + x + } + PacketChunk::VectorDouble(n, fill) => { + assert!(*fill == 7); + let mut x: u16 = 1 << 15; + assert!(*n <= 16384); + x |= 1 << 14; + x |= *n; + x + } + }; + buf[..2].copy_from_slice(&x.to_be_bytes()); + } + + fn max_possible_status_count(&self) -> usize { + match self { + PacketChunk::Run(_, n) => *n as usize, + PacketChunk::VectorSingle(_, _) => 14, + PacketChunk::VectorDouble(_, _) => 7, + } + } + + fn can_append_status(&self, status: PacketStatus) -> bool { + match self { + PacketChunk::Run(s, _) => *s == status, + PacketChunk::VectorSingle(_, _) => status != PacketStatus::ReceivedLargeOrNegativeDelta, + PacketChunk::VectorDouble(_, _) => true, + } + } +} + +impl Delta { + fn write_to(&self, buf: &mut [u8]) -> usize { + match self { + Delta::Small(v) => { + buf[0] = *v; + 1 + } + Delta::Large(v) => { + buf[..2].copy_from_slice(&v.to_be_bytes()); + 2 + } + } + } + + fn byte_len(&self) -> usize { + match self { + Delta::Small(_) => 1, + Delta::Large(_) => 2, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PacketStatus { + NotReceived = 0b00, + ReceivedSmallDelta = 0b01, + ReceivedLargeOrNegativeDelta = 0b10, + Unknown = 0b11, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Delta { + Small(u8), + Large(i16), +} + +impl From for u8 { + fn from(val: PacketStatus) -> Self { + val as usize as u8 + } +} + +impl From for PacketStatus { + fn from(v: u8) -> Self { + match v { + 0b00 => Self::NotReceived, + 0b01 => Self::ReceivedSmallDelta, + 0b10 => Self::ReceivedLargeOrNegativeDelta, + _ => Self::Unknown, + } + } +} + +impl<'a> TryFrom<&'a [u8]> for Twcc { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 16 { + return Err("Less than 16 bytes for start of Twcc"); + } + + let sender_ssrc = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]).into(); + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]).into(); + let base_seq = u16::from_be_bytes([buf[8], buf[9]]); + let status_count = u16::from_be_bytes([buf[10], buf[11]]); + let reference_time = u32::from_be_bytes([0, buf[12], buf[13], buf[14]]); + let feedback_count = buf[15]; + + let mut twcc = Twcc { + sender_ssrc, + ssrc, + base_seq, + status_count, + reference_time, + feedback_count, + chunks: VecDeque::new(), + delta: VecDeque::new(), + }; + + let mut todo = status_count as isize; + let mut buf = &buf[16..]; + loop { + if todo <= 0 { + break; + } + + let chunk: PacketChunk = buf.try_into()?; + + todo -= chunk.max_possible_status_count() as isize; + + twcc.chunks.push_back(chunk); + buf = &buf[2..]; + } + + if twcc.chunks.is_empty() { + return Ok(twcc); + } + + fn read_delta_small( + buf: &[u8], + n: usize, + ) -> Result + '_, &'static str> { + if buf.len() < n { + return Err("Not enough buf for small deltas"); + } + Ok((0..n).map(|i| Delta::Small(buf[i]))) + } + + fn read_delta_large( + buf: &[u8], + n: usize, + ) -> Result + '_, &'static str> { + if buf.len() < n * 2 { + return Err("Not enough buf for large deltas"); + } + Ok((0..(n * 2)) + .step_by(2) + .map(|i| Delta::Large(i16::from_be_bytes([buf[i], buf[i + 1]])))) + } + + for c in &twcc.chunks { + match c { + PacketChunk::Run(PacketStatus::ReceivedSmallDelta, n) => { + let n = *n as usize; + twcc.delta.extend(read_delta_small(buf, n)?); + buf = &buf[n..]; + } + PacketChunk::Run(PacketStatus::ReceivedLargeOrNegativeDelta, n) => { + let n = *n as usize; + twcc.delta.extend(read_delta_large(buf, n)?); + buf = &buf[n..]; + } + PacketChunk::VectorSingle(v, _) => { + let n = v.count_ones() as usize; + twcc.delta.extend(read_delta_small(buf, n)?); + buf = &buf[n..]; + } + PacketChunk::VectorDouble(v, _) => { + for n in (0..=12).step_by(2) { + let x = (*v >> (12 - n)) & 0b11; + match PacketStatus::from(x as u8) { + PacketStatus::ReceivedSmallDelta => { + twcc.delta.extend(read_delta_small(buf, 1)?); + buf = &buf[1..]; + } + PacketStatus::ReceivedLargeOrNegativeDelta => { + twcc.delta.extend(read_delta_large(buf, 1)?); + buf = &buf[2..]; + } + _ => {} + } + } + } + _ => {} + } + } + + Ok(twcc) + } +} + +impl<'a> TryFrom<&'a [u8]> for PacketChunk { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 2 { + return Err("Less than 2 bytes for PacketChunk"); + } + + let x = u16::from_be_bytes([buf[0], buf[1]]); + + let is_vec = (x & 0b1000_0000_0000_0000) > 0; + + let p = if is_vec { + let is_double = (x & 0b0100_0000_0000_0000) > 0; + let n = x & 0b0011_1111_1111_1111; + if is_double { + PacketChunk::VectorDouble(n, 7) + } else { + PacketChunk::VectorSingle(n, 14) + } + } else { + let s: PacketStatus = ((x >> 13) as u8).into(); + let n = x & 0b0001_1111_1111_1111; + PacketChunk::Run(s, n) + }; + + Ok(p) + } +} + +#[derive(Debug)] +pub struct TwccSendRegister { + /// How many send records to keep. + keep: usize, + + /// Circular buffer of send records. + queue: VecDeque, + + /// 0 offset for remote time in Twcc structs. + time_zero: Option, + + /// Counter of invocations of apply_report. Used to identify + /// which TwccSendRecord resulted from each invocation. + apply_report_counter: u64, + + /// Last registered Twcc number. + last_registered: TwccSeq, +} + +impl<'a> IntoIterator for &'a TwccSendRegister { + type Item = &'a TwccSendRecord; + type IntoIter = vec_deque::Iter<'a, TwccSendRecord>; + + fn into_iter(self) -> Self::IntoIter { + self.queue.iter() + } +} + +/// Packet identification for TWCC tracking. +/// +/// Groups together the TWCC sequence number and optional probe cluster context. +#[derive(Debug, Clone, Copy)] +pub struct TwccPacketId { + /// TWCC sequence number + seq: TwccSeq, + /// Probe cluster this packet belongs to, if any + cluster: Option, +} + +impl TwccPacketId { + /// Create a packet ID for a regular media packet (no probe cluster). + pub fn new(seq: impl Into) -> Self { + Self { + seq: seq.into(), + cluster: None, + } + } + + /// Create a packet ID for a probe packet. + pub fn with_cluster(seq: impl Into, cluster: impl Into) -> Self { + Self { + seq: seq.into(), + cluster: Some(cluster.into()), + } + } + + /// Get the TWCC sequence number. + pub fn seq(&self) -> TwccSeq { + self.seq + } + + /// Get the probe cluster, if any. + pub fn cluster(&self) -> Option { + self.cluster + } +} + +/// Record for a send entry in twcc. +#[derive(Debug)] +pub struct TwccSendRecord { + /// Packet identification (sequence + optional probe cluster) + packet_id: TwccPacketId, + + /// The (local) time we sent the packet represented by seq. + local_send_time: Instant, + + /// Size in bytes of the payload sent. + size: u16, + + recv_report: Option, +} + +impl TwccSendRecord { + /// The twcc sequence number of the packet we sent. + pub fn seq(&self) -> TwccSeq { + self.packet_id.seq() + } + + /// The probe cluster this packet belongs to, if it's a probe packet. + pub fn cluster(&self) -> Option { + self.packet_id.cluster() + } + + /// The time we sent the packet. + pub fn local_send_time(&self) -> Instant { + self.local_send_time + } + + /// The time we received this TWCC record. [`None`] if no feedback has been received yet. + pub fn local_recv_time(&self) -> Option { + self.recv_report.as_ref().map(|r| r.local_recv_time) + } + + pub fn size(&self) -> usize { + self.size as usize + } + + /// The time indicated by the remote side for when they received the packet. + pub fn remote_recv_time(&self) -> Option { + self.recv_report.as_ref().and_then(|r| r.remote_recv_time) + } + + /// The rtt time between sending the packet and receiving the twcc report response. + pub fn rtt(&self) -> Option { + let recv_report = self.recv_report.as_ref()?; + Some(recv_report.local_recv_time - self.local_send_time) + } +} + +#[cfg(test)] +impl TwccSendRecord { + /// Test-only constructor to build TWCC send records with arbitrary receive status. + /// + /// This is used by unit tests for BWE/probing, allowing them to model received vs lost packets + /// without constructing full RTCP TWCC reports. + pub(crate) fn test_new( + packet_id: TwccPacketId, + local_send_time: Instant, + size: usize, + local_recv_time: Instant, + remote_recv_time: Option, + ) -> Self { + Self { + packet_id, + local_send_time, + size: size as u16, + recv_report: Some(TwccRecvReport { + local_recv_time, + remote_recv_time, + apply_report_counter: 0, + }), + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct TwccRecvReport { + /// The (local) time we received confirmation the other side received the seq. + local_recv_time: Instant, + + /// The remote time the other side received the seq. + remote_recv_time: Option, + + /// The invocation count of apply_report(). Used for filtering. + apply_report_counter: u64, +} + +impl TwccSendRegister { + pub fn new(keep: usize) -> Self { + TwccSendRegister { + keep, + queue: VecDeque::new(), + time_zero: None, + apply_report_counter: 0, + last_registered: 0.into(), + } + } + + pub fn register_seq(&mut self, packet_id: TwccPacketId, now: Instant, size: usize) { + self.last_registered = packet_id.seq(); + self.queue.push_back(TwccSendRecord { + packet_id, + local_send_time: now, + // In practice the max sizes is constrained by the MTU and will max out around 1200 + // bytes, hence this cast is fine. + size: size as u16, + // The recv report, derived from TWCC feedback later. + recv_report: None, + }); + while self.queue.len() > self.keep { + self.queue.pop_front(); + } + } + + /// Apply a TWCC RTCP report. + /// + /// Returns iterator over [`TwccSendRecord`]s included in the given [`Twcc`] + /// except for ones that was already acked and returned before. + pub fn apply_report( + &mut self, + twcc: Twcc, + now: Instant, + ) -> Option> { + if self.time_zero.is_none() { + self.time_zero = Some(now); + } + + self.apply_report_counter += 1; + let apply_report_counter = self.apply_report_counter; + + let time_zero = self.time_zero.unwrap(); + let head_seq = self.queue.front().map(|r| r.seq())?; + + let mut iter = twcc + .into_iter(time_zero, self.last_registered) + .skip_while(|(seq, _, _)| seq < &head_seq); + let (first_seq_no, _, first_instant) = iter.next()?; + + let mut iter2 = self + .queue + .iter_mut() + .skip_while(|r| *r.seq() < *first_seq_no); + let first_record = iter2.next()?; + + fn update( + now: Instant, + r: &mut TwccSendRecord, + seq: TwccSeq, + remote_recv_time: Option, + apply_report_counter: u64, + ) -> bool { + if r.seq() != seq { + return false; + } + + let apply_report_counter = if let Some(rr) = r.recv_report { + // This packed was already acked and handled before so carry + // over previous apply_report_counter, so it won't be included + // in the current apply_report() call result. + rr.remote_recv_time + .map(|_| rr.apply_report_counter) + .unwrap_or_else(|| apply_report_counter) + } else { + apply_report_counter + }; + + // Carry over remote recv time if this packet was acked before. + let remote_recv_time = r.remote_recv_time().or(remote_recv_time); + let recv_report = TwccRecvReport { + local_recv_time: now, + remote_recv_time, + apply_report_counter, + }; + r.recv_report = Some(recv_report); + + true + } + + if first_record.seq() != first_seq_no { + // Old report for which we no longer have any send records. + return None; + } + + let mut problematic_seq = None; + + if !update( + now, + first_record, + first_seq_no, + first_instant, + apply_report_counter, + ) { + problematic_seq = Some((first_record.seq(), first_seq_no)); + } + + let mut last_seq_no = first_seq_no; + for ((seq, _, instant), record) in iter.zip(iter2) { + if problematic_seq.is_some() { + break; + } + + if !update(now, record, seq, instant, apply_report_counter) { + problematic_seq = Some((record.seq(), seq)); + } + last_seq_no = seq; + } + + if let Some((record_seq, report_seq)) = problematic_seq { + let queue_tail: Vec<_> = self.queue.iter().rev().take(100).collect(); + panic!( + "Unexpected TWCC sequence when applying TWCC report. \ + Send Record Seq({record_seq}) does not match Report Seq({report_seq}).\ + \nLast 100 entires in queue: {queue_tail:?}." + ); + } + + let first_index = self + .queue + .binary_search_by_key(&first_seq_no, |r| r.seq()) + .expect("first_seq_no to be registered"); + + let range = first_seq_no..=last_seq_no; + + Some( + TwccSendRecordsIter { + range: range.clone(), + index: first_index, + current: first_seq_no, + queue: &self.queue, + } + // We only want the records that were registered in this invocation of + // apply_report_counter(). This is to not double count in the BWE, + // which is the consumer of this returned iterator. + .filter(move |s| { + s.recv_report + .map(|r| r.apply_report_counter == apply_report_counter) + .unwrap_or_default() + }), + ) + } + + /// Calculate the egress loss for given time window. + /// + /// **Note:** The register only keeps a limited number of records and using `duration` values + /// larger than ~1-2 seconds is liable to be inaccurate since some packets sent might have already + /// been evicted from the register. + pub fn loss(&self, duration: Duration, now: Instant) -> Option { + // Consider only packets in the span specified by the caller + let lower_bound = now - duration; + + let packets = self + .queue + .iter() + .rev() + // If there's ingress loss but no egress loss, there's a chance the TWCC reports + // themselves are lost. In this case considering packets that haven't been reported as + // lost will incorrectly conclude that there is in fact egress loss. + .filter(|s| s.recv_report.is_some()) + .take_while(|s| s.local_send_time >= lower_bound); + + let (total, lost) = packets.fold((0, 0), |(total, lost), s| { + let was_lost = s + .recv_report + .as_ref() + .map(|rr| rr.remote_recv_time.is_none()) + .unwrap_or(true); + + (total + 1, lost + u64::from(was_lost)) + }); + + if total == 0 { + return None; + } + + Some((lost as f32) / (total as f32)) + } + + /// Calculate the RTT for the most recently reported packet. + pub fn rtt(&self) -> Option { + self.queue.iter().rev().find_map(|s| s.rtt()) + } +} + +#[derive()] +struct TwccSendRecordsIter<'a> { + range: RangeInclusive, + current: TwccSeq, + index: usize, + queue: &'a VecDeque, +} + +impl<'a> Iterator for TwccSendRecordsIter<'a> { + type Item = &'a TwccSendRecord; + + fn next(&mut self) -> Option { + if self.current > *self.range.end() || self.current < *self.range.start() { + return None; + } + + let item = &self.queue[self.index]; + assert!(self.current == item.seq()); + self.current.inc(); + self.index += 1; + + Some(item) + } +} + +// Below is a clarification of the RFC draft from an email exchange with Erik Språng (one of the authors). +// +// > I'm trying to implement the draft spec +// > https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 +// > I found a number of errors/inconsistencies in the RFC, and wonder who I should address this to. +// > I think the RFC could benefit from another revision. +// > +// > There are three problems listed below. +// > +// > 1. There's a contradiction between section 3.1.1 and the example in 3.1.3. First the RFC tells +// > me 11 is Reserved, later it shows an example using 11 saying it is a run of packets received w/o +// > recv delta. Which one is right? +// > +// > Section 3.1.1 +// > ... +// > The status of a packet is described using a 2-bit symbol: +// > ... +// > 11 [Reserved] +// > +// > Section 3.1.3 +// > ... +// > Example 2: +// > +// > 0 1 +// > 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// > +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// > |0|1 1|0 0 0 0 0 0 0 0 1 1 0 0 0| +// > +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// > +// > +// > This is a run of the "packet received, w/o recv delta" status of +// > length 24. +// +// I believe this example is in error. Packets without receive deltas was a proposal for the v1 +// protocol but was dropped iirc. Note that there is a newer header extension available, which +// when negotiated allows send-side control over when feedback is generated - and that provides +// the option to omit all receive deltas from the feedback. +// +// > 2. In section 3.1.4 when using a 1-bit vector to indicate packet received or not received, +// > there's a contradiction between the the definition and the example. The definition says +// > "packet received" (0) and "packet not received" (1), while the example is the opposite way +// > around: 0 is packet not received. Which way around is it? +// > +// > symbol size (S): 1 bit A zero means this vector contains only +// > "packet received" (0) and "packet not received" (1) +// > symbols. This means we can compress each symbol to just +// > one bit, 14 in total. A one means this vector contains +// > the normal 2-bit symbols, 7 in total. +// > ... +// > Example 1: +// > +// > 0 1 +// > 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// > +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// > |1|0|0 1 1 1 1 1 0 0 0 1 1 1 0 0| +// > +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// > +// > This chunk contains, in order: +// > +// > 1x "packet not received" +// > +// > 5x "packet received" +// +// I believe the definition is wrong in this case. The intent is to just truncate the 2-bit values: +// +// 3.1.1. Packet Status Symbols +// +// The status of a packet is described using a 2-bit symbol: +// +// 00 Packet not received +// +// 01 Packet received, small delta +// +// So (0) for not received and (1) for received, small delta. +// This also matches what the libwebrtc source code does. +// +// > 3. In section 3.1.4 when using a 1-bit vector, the RFC doesn't say what a "packet received" in that +// > vector should be accompanied by in receive delta size. Is it an 8 bit delta or 16 bit +// > delta per "packet received"? +// +// Same as the question above, this is a truncation to (0) for not received and (1) for +// received, small delta. + +impl fmt::Debug for Twcc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Twcc") + .field("sender_ssrc", &self.sender_ssrc) + .field("ssrc", &self.ssrc) + .field("base_seq", &self.base_seq) + .field("status_count", &self.status_count) + .field("reference_time", &self.reference_time) + .field("feedback_count", &self.feedback_count) + .field("chunks", &self.chunks) + .field("delta", &self.delta.len()) + .finish() + } +} + +#[allow(clippy::assign_op_pattern)] +#[cfg(test)] +mod test { + use std::time::Duration; + + use super::*; + + use Delta::*; + use PacketChunk::*; + use PacketStatus::*; + + #[test] + fn register_write_parse_small_delta() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(12)); + reg.update_seq(12.into(), now + Duration::from_millis(23)); + reg.update_seq(13.into(), now + Duration::from_millis(43)); + + let report = reg.build_report(1000).unwrap(); + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = (&buf[..]).try_into().unwrap(); + let parsed: Twcc = (&buf[4..]).try_into().unwrap(); + + assert_eq!(header, report.header()); + assert_eq!(parsed, report); + } + + #[test] + fn register_write_parse_small_delta_missing() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(12)); + reg.update_seq(12.into(), now + Duration::from_millis(23)); + // 13 is not there + reg.update_seq(14.into(), now + Duration::from_millis(43)); + + let report = reg.build_report(1000).unwrap(); + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = (&buf[..]).try_into().unwrap(); + let parsed: Twcc = (&buf[4..]).try_into().unwrap(); + + assert_eq!(header, report.header()); + assert_eq!(parsed, report); + } + + #[test] + fn register_write_parse_large_delta() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(70)); + reg.update_seq(12.into(), now + Duration::from_millis(140)); + reg.update_seq(13.into(), now + Duration::from_millis(210)); + + let report = reg.build_report(1000).unwrap(); + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = (&buf[..]).try_into().unwrap(); + let parsed: Twcc = (&buf[4..]).try_into().unwrap(); + + assert_eq!(header, report.header()); + assert_eq!(parsed, report); + } + + #[test] + fn register_write_parse_mixed_delta() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(12)); + reg.update_seq(12.into(), now + Duration::from_millis(140)); + reg.update_seq(13.into(), now + Duration::from_millis(152)); + + let report = reg.build_report(1000).unwrap(); + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = (&buf[..]).try_into().unwrap(); + let parsed: Twcc = (&buf[4..]).try_into().unwrap(); + + assert_eq!(header, report.header()); + assert_eq!(parsed, report); + } + + #[test] + fn too_big_time_gap_requires_two_reports() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(12)); + reg.update_seq(12.into(), now + Duration::from_millis(9000)); + + let _ = reg.build_report(1000).unwrap(); + let report2 = reg.build_report(1000).unwrap(); + + // 9000 milliseconds is not possible to set as exact reference time which + // is in multiples of 64ms. 9000/64 = 140.625. + assert_eq!(report2.reference_time, 140); + + // 140 * 64 = 8960 + // So the first offset must be 40ms, i.e. 40_000us / 250us = 160 + assert_eq!(report2.delta[0], Small(160)); + } + + #[test] + fn report_padded_to_even_word() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + + let report = reg.build_report(1000).unwrap(); + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + + assert!(n % 4 == 0); + } + + #[test] + fn report_truncated_to_max_byte_size() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(11.into(), now + Duration::from_millis(12)); + reg.update_seq(12.into(), now + Duration::from_millis(140)); + reg.update_seq(13.into(), now + Duration::from_millis(152)); + + let report = reg.build_report(28).unwrap(); + + assert_eq!(report.status_count, 2); + assert_eq!(report.chunks, vec![Run(ReceivedSmallDelta, 2)]); + assert_eq!(report.delta, vec![Small(0), Small(48)]); + + let report = reg.build_report(28).unwrap(); + + assert_eq!(report.status_count, 2); + assert_eq!(report.chunks, vec![Run(ReceivedSmallDelta, 2)]); + assert_eq!(report.delta, vec![Small(48), Small(48)]); + } + + #[test] + fn truncated_counts_gaps_correctly() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + // gap + reg.update_seq(13.into(), now + Duration::from_millis(12)); + reg.update_seq(14.into(), now + Duration::from_millis(140)); + reg.update_seq(15.into(), now + Duration::from_millis(152)); + + let report = reg.build_report(32).unwrap(); + + assert_eq!(report.status_count, 4); + assert_eq!( + report.chunks, + vec![ + Run(ReceivedSmallDelta, 1), + Run(NotReceived, 2), + Run(ReceivedSmallDelta, 1) + ] + ); + assert_eq!(report.delta, vec![Small(0), Small(48)]); + } + + #[test] + fn run_max_is_8192() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(0.into(), now + Duration::from_millis(0)); + reg.update_seq(8194.into(), now + Duration::from_millis(10)); + + let report = reg.build_report(1000).unwrap(); + + assert_eq!(report.status_count, 8195); + assert_eq!( + report.chunks, + vec![ + VectorSingle(8192, 14), + Run(NotReceived, 8180), + Run(ReceivedSmallDelta, 1) + ] + ); + } + + #[test] + fn single_followed_by_missing() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(0)); + reg.update_seq(12.into(), now + Duration::from_millis(10)); + reg.update_seq(100.into(), now + Duration::from_millis(20)); + + let report = reg.build_report(2016).unwrap(); + + assert_eq!(report.status_count, 91); + assert_eq!( + report.chunks, + vec![ + VectorSingle(10240, 14), + Run(NotReceived, 76), + Run(ReceivedSmallDelta, 1) + ] + ); + assert_eq!(report.delta, vec![Small(0), Small(40), Small(40)]); + } + + #[test] + fn time_jump_small_back_for_second_report() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(8000)); + let _ = reg.build_report(2016).unwrap(); + + reg.update_seq(9.into(), now + Duration::from_millis(0)); + let report = reg.build_report(2016).unwrap(); + + assert_eq!(report.status_count, 2); + assert_eq!(report.chunks, vec![Run(ReceivedLargeOrNegativeDelta, 2)]); + assert_eq!(report.delta, vec![Large(-32000), Large(32000)]); + } + + #[test] + fn time_jump_large_back_for_second_report() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(9000)); + let _ = reg.build_report(2016).unwrap(); + + reg.update_seq(9.into(), now + Duration::from_millis(0)); + assert!(reg.build_report(2016).is_none()); + + assert_eq!(reg.queue.len(), 1); + } + + #[test] + fn empty_twcc() { + let twcc = Twcc { + sender_ssrc: 0.into(), + ssrc: 0.into(), + base_seq: 0, + status_count: 0, + reference_time: 0, + feedback_count: 0, + chunks: VecDeque::new(), + delta: VecDeque::new(), + }; + + let mut buf = vec![0_u8; 1500]; + let n = twcc.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = (&buf[..]).try_into().unwrap(); + let parsed: Twcc = (&buf[4..]).try_into().unwrap(); + + assert_eq!(header, twcc.header()); + assert_eq!(parsed, twcc); + } + + #[test] + fn negative_deltas() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + reg.update_seq(10.into(), now + Duration::from_millis(12)); + reg.update_seq(11.into(), now + Duration::from_millis(0)); + reg.update_seq(12.into(), now + Duration::from_millis(23)); + + let report = reg.build_report(1000).unwrap(); + + assert_eq!(report.status_count, 3); + assert_eq!(report.base_seq, 10); + assert_eq!(report.reference_time, 0); + assert_eq!(report.chunks, vec![VectorDouble(6400, 7)]); + assert_eq!(report.delta, vec![Small(0), Large(-48), Small(92)]); + + let base = reg.time_start.unwrap(); + + let mut iter = report.into_iter(base, 10.into()); + assert_eq!( + iter.next(), + Some(( + 10.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_millis(0)) + )) + ); + assert_eq!( + iter.next(), + Some(( + 11.into(), + PacketStatus::ReceivedLargeOrNegativeDelta, + Some(base.checked_sub(Duration::from_millis(12)).unwrap()) + )) + ); + assert_eq!( + iter.next(), + Some(( + 12.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_millis(11)) + )) + ); + } + + #[test] + fn twcc_fuzz_fail() { + let mut reg = TwccRecvRegister::new(100); + + let now = Instant::now(); + + // [Register(, ), Register(, ), Register(, ), BuildReport(43)] + + reg.update_seq(4542.into(), now + Duration::from_millis(2373281424)); + reg.update_seq(15918.into(), now + Duration::from_millis(2373862820)); + reg.update_seq(8405.into(), now + Duration::from_millis(2379074367)); + + let report = reg.build_report(43).unwrap(); + + let mut buf = vec![0_u8; 1500]; + let n = report.write_to(&mut buf[..]); + buf.truncate(n); + + let header: RtcpHeader = match (&buf[..]).try_into() { + Ok(v) => v, + Err(_) => return, + }; + let parsed: Twcc = match (&buf[4..]).try_into() { + Ok(v) => v, + Err(_) => return, + }; + + assert_eq!(header, report.header()); + assert_eq!(parsed, report); + } + + #[test] + fn twcc_large_time_delta_edges() { + let mut reg = TwccRecvRegister::new(100); + + // Stretch the boundaries of the i16 type used for deltas + let now = Instant::now(); + reg.update_seq(0.into(), now + Duration::from_micros(8_192_000)); + reg.update_seq(1.into(), now + Duration::from_micros(0)); + reg.update_seq(2.into(), now + Duration::from_micros(8_191_750)); + + let report = reg.build_report(1000).unwrap(); + + assert_eq!(report.status_count, 3); + assert_eq!(report.delta, vec![Small(0), Large(-32768), Large(32767)]); + } + + #[test] + fn twcc_crazy_negative_time_delta() { + let mut reg = TwccRecvRegister::new(100); + + // Deltas so big they wrap around the bounds of an i32 to become small again. + // These constants are chosen carefully to look normal when wrapped + let now = Instant::now(); + reg.update_seq(0.into(), now + Duration::from_micros(4_294_967_547)); // Wraps to -251 + reg.update_seq(1.into(), now + Duration::from_micros(0)); + + // The bogus value should be ignored + let report = reg.build_report(1000).unwrap(); + assert_eq!(report.status_count, 1); + assert_eq!(report.delta, vec![Small(0)]); + } + + #[test] + fn twcc_crazy_positive_time_delta() { + let mut reg = TwccRecvRegister::new(100); + + // Deltas so big they wrap around the bounds of an i32 to become small again. + // These constants are chosen carefully to look normal when wrapped + let now = Instant::now(); + reg.update_seq(0.into(), now + Duration::from_micros(0)); + reg.update_seq(1.into(), now + Duration::from_micros(4_294_967_547)); // Wraps to 251 + + // The bogus value should be ignored + let report = reg.build_report(1000).unwrap(); + assert_eq!(report.status_count, 1); + assert_eq!(report.delta, vec![Small(0)]); + } + + #[test] + fn test_send_register_apply_report_for_old_seq_numbers() { + let mut reg = TwccSendRegister::new(25); + let mut now = Instant::now(); + + for i in 0..50 { + reg.register_seq(TwccPacketId::new(i), now, 0); + now = now + Duration::from_micros(15); + } + + // At this point the front of the internal queue should be seq no 25. + // + // Set time zero base with empty packet + reg.apply_report( + Twcc { + sender_ssrc: Ssrc::new(), + ssrc: Ssrc::new(), + base_seq: 0, + status_count: 0, + reference_time: 0, + feedback_count: 0, + chunks: [].into(), + delta: [].into(), + }, + now, + ); + now = now + Duration::from_millis(35); + + let iter = reg.apply_report( + Twcc { + sender_ssrc: Ssrc::new(), + ssrc: Ssrc::new(), + base_seq: 20, + status_count: 8, + reference_time: 35, + feedback_count: 0, + chunks: [PacketChunk::Run(PacketStatus::ReceivedSmallDelta, 8)].into(), + delta: [ + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + ] + .into(), + }, + now, + ); + let iter = iter.unwrap(); + + for record in iter { + assert!( + record.recv_report.is_some(), + "Report should have recorded recv_report" + ); + } + } + + #[test] + fn test_twcc_iter_correct_deltas() { + let twcc = Twcc { + sender_ssrc: 0.into(), + ssrc: 0.into(), + base_seq: 1, + status_count: 12, + reference_time: 1337, + feedback_count: 0, + chunks: [ + PacketChunk::Run(PacketStatus::ReceivedSmallDelta, 2), + PacketChunk::VectorDouble(0b1101_0010_0101_0000, 7), + PacketChunk::Run(PacketStatus::ReceivedLargeOrNegativeDelta, 3), + ] + .into(), + delta: [ + // Run of length 2 + Delta::Small(10), + Delta::Small(15), + // Double status vector with 4 deltas + Delta::Small(7), + Delta::Large(280), + Delta::Small(3), + Delta::Small(13), + // Run of length 3 + Delta::Large(-37), + Delta::Large(32), + Delta::Large(89), + ] + .into(), + }; + + let now = Instant::now(); + let base = now + Duration::from_millis(1337 * 64); + let expected = vec![ + ( + 1.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_micros(10 * 250)), + ), + ( + 2.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_micros(25 * 250)), + ), + ( + 3.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_micros(32 * 250)), + ), + (4.into(), PacketStatus::NotReceived, None), + ( + 5.into(), + PacketStatus::ReceivedLargeOrNegativeDelta, + Some(base + Duration::from_micros(312 * 250)), + ), + ( + 6.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_micros(315 * 250)), + ), + ( + 7.into(), + PacketStatus::ReceivedSmallDelta, + Some(base + Duration::from_micros(328 * 250)), + ), + (8.into(), PacketStatus::NotReceived, None), + (9.into(), PacketStatus::NotReceived, None), + ( + 10.into(), + PacketStatus::ReceivedLargeOrNegativeDelta, + Some(base + Duration::from_micros(291 * 250)), + ), + ( + 11.into(), + PacketStatus::ReceivedLargeOrNegativeDelta, + Some(base + Duration::from_micros(323 * 250)), + ), + ( + 12.into(), + PacketStatus::ReceivedLargeOrNegativeDelta, + Some(base + Duration::from_micros(412 * 250)), + ), + ]; + + let result: Vec<_> = twcc.into_iter(now, 1.into()).collect(); + + assert_eq!(result, expected); + } + + #[test] + fn test_twcc_iter_limited_with_status_count() { + let status_count = 3; + + // [(1, NotReceived), (2, ReceivedSmallDelta), (3, ReceivedSmallDelta)] + let twcc_iter_count = Twcc { + sender_ssrc: 1.into(), + ssrc: 2.into(), + base_seq: 1, + status_count, + reference_time: 406753, + feedback_count: 1, + chunks: VecDeque::from(vec![VectorDouble(0b00_00_01_01_00_00_00_00, 7)]), + delta: VecDeque::from(vec![Small(236), Small(1)]), + } + .into_iter(Instant::now(), 1.into()) + .count(); + + assert_eq!(twcc_iter_count, status_count as usize); + } + + #[test] + fn test_twcc_register_send_records() { + let mut reg = TwccSendRegister::new(25); + let mut now = Instant::now(); + for i in 0..25 { + reg.register_seq(TwccPacketId::new(i), now, 0); + now = now + Duration::from_micros(15); + } + + let iter = reg + .apply_report( + Twcc { + sender_ssrc: Ssrc::new(), + ssrc: Ssrc::new(), + base_seq: 0, + status_count: 8, + reference_time: 35, + feedback_count: 0, + chunks: [PacketChunk::Run(PacketStatus::ReceivedSmallDelta, 8)].into(), + delta: [ + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + ] + .into(), + }, + now, + ) + .expect("apply_report to return Some(_)"); + + assert_eq!( + iter.map(|r| *r.seq()).collect::>(), + vec![0, 1, 2, 3, 4, 5, 6, 7] + ); + } + + #[test] + fn test_twcc_send_register_loss() { + let mut reg = TwccSendRegister::new(25); + let mut now = Instant::now(); + for i in 0..9 { + reg.register_seq(TwccPacketId::new(i), now, 0); + now = now + Duration::from_millis(15); + } + + now = now + Duration::from_millis(5); + #[allow(unused_must_use)] + reg.apply_report( + Twcc { + sender_ssrc: Ssrc::new(), + ssrc: Ssrc::new(), + base_seq: 0, + status_count: 9, + reference_time: 35, + feedback_count: 0, + chunks: [ + PacketChunk::VectorDouble(0b11_01_01_01_00_01_00_01, 7), + PacketChunk::Run(PacketStatus::ReceivedSmallDelta, 2), + ] + .into(), + delta: [ + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + Delta::Small(10), + ] + .into(), + }, + now, + ) + .expect("apply_report to return Some(_)"); + + now = now + Duration::from_millis(20); + let loss = reg + .loss(Duration::from_millis(150), now) + .expect("Should be able to calcualte loss"); + + let pct = (loss * 100.0).floor() as u32; + + assert_eq!( + pct, 25, + "The loss percentage should be 25 as 2 out of 8 packets are lost" + ); + } + + #[test] + fn test_twcc_recv_register_loss() { + let mut reg = TwccRecvRegister::new(25); + let mut now = Instant::now(); + + for i in 0..10 { + if i == 3 || i == 7 { + // simulate loss + continue; + } + reg.update_seq(i.into(), now); + now = now + Duration::from_millis(50); + } + + assert_eq!(reg.loss(), Some(2.0 / 10.0)); + + for i in 10..20 { + if i == 11 || i == 13 || i == 15 || i == 17 { + // simulate loss + continue; + } + reg.update_seq(i.into(), now); + now = now + Duration::from_millis(50); + } + + assert_eq!(reg.loss(), Some(4.0 / 10.0)); + } + + #[test] + fn no_acked_duplicates_when_reordered() { + let now = Instant::now(); + let mut twcc_gen = TwccRecvRegister::new(1000); + let mut twcc_handler = TwccSendRegister::new(1000); + + twcc_handler.register_seq(TwccPacketId::new(1), now + Duration::from_millis(1), 0); + twcc_handler.register_seq(TwccPacketId::new(2), now + Duration::from_millis(2), 0); + twcc_handler.register_seq(TwccPacketId::new(3), now + Duration::from_millis(3), 0); + twcc_handler.register_seq(TwccPacketId::new(4), now + Duration::from_millis(4), 0); + + { + let acked_packets = twcc_handler + .apply_report( + { + // 3rd packet is delayed + twcc_gen.update_seq(1.into(), now + Duration::from_millis(5)); + twcc_gen.update_seq(2.into(), now + Duration::from_millis(6)); + twcc_gen.update_seq(4.into(), now + Duration::from_millis(7)); + twcc_gen.build_report(10_000).unwrap() + }, + now + Duration::from_millis(8), + ) + .unwrap() + .filter_map(|sr| sr.remote_recv_time().map(|_| sr.seq().as_u16())) + .collect::>(); + + assert_eq!(acked_packets, [1, 2, 4]); + } + + twcc_handler.register_seq(TwccPacketId::new(5), now + Duration::from_millis(9), 0); + twcc_handler.register_seq(TwccPacketId::new(6), now + Duration::from_millis(10), 0); + twcc_handler.register_seq(TwccPacketId::new(7), now + Duration::from_millis(11), 0); + + { + let acked_packets = twcc_handler + .apply_report( + { + // So the receipt order is 1, 2, 4, 3, 7 + twcc_gen.update_seq(3.into(), now + Duration::from_millis(12)); + twcc_gen.update_seq(7.into(), now + Duration::from_millis(13)); + twcc_gen.build_report(10_000).unwrap() + }, + now + Duration::from_millis(14), + ) + .unwrap() + .filter_map(|sr| sr.remote_recv_time().map(|_| sr.seq().as_u16())) + .collect::>(); + + // 3 was delayed before and is acked now + // 4 is excluded since it was already returned from the previous call + // [5, 6] are delayed/lost + // 7 is acked in the last report + assert_eq!(acked_packets, [3, 7]); + } + } + + #[test] + fn test_twcc_recv_register_high_initial_sequence() { + // Test TWCC sequence numbers starting at high values (>= 32768) + let test_cases = vec![ + (32767, "just below half"), + (32768, "exactly half"), + (65434, "just below u16"), + (65535, "maximum u16"), + ]; + + for (transport_cc, description) in test_cases { + let mut twcc_rx_register = TwccRecvRegister::new(100); + let now = Instant::now(); + + // Empty register should return `None` + assert_eq!( + twcc_rx_register.max_seq(), + None, + "Empty register should return None for {}", + description + ); + + // This is what `session.rs` does: + let prev = twcc_rx_register.max_seq(); + let extended = extend_u16(prev.map(|s| *s), transport_cc); + twcc_rx_register.update_seq(extended.into(), now); + + assert_eq!( + twcc_rx_register.max_seq(), + Some(extended.into()), + "First packet for {}", + description + ); + + // Add a few more packets, simulating production code path + for i in 1..5 { + let raw_seq = ((transport_cc as u32 + i) % 65536) as u16; + let prev = twcc_rx_register.max_seq(); + let extended = extend_u16(prev.map(|s| *s), raw_seq); + twcc_rx_register + .update_seq(extended.into(), now + Duration::from_millis(i as u64 * 10)); + } + + // Verify wrap-around handling if we crossed the boundary + if transport_cc >= 65533 { + // Should have wrapped to extended values > 65536 + let max = twcc_rx_register.max_seq().unwrap(); + assert!( + *max > 65536, + "Should wrap correctly for {}: got {}", + description, + *max + ); + } + + // build_report should NOT panic or allocate huge memory (leading to OOM) + let report = twcc_rx_register.build_report(1000); + assert!(report.is_some(), "Should build report for {}", description); + + let report = report.unwrap(); + assert!( + report.status_count > 0, + "Should have packets for {}", + description + ); + assert!( + !report.chunks.is_empty(), + "Should have chunks for {}", + description + ); + + // Sanity check: should never have an absurdly large number of chunks + assert!( + report.chunks.len() < 10, + "Should not create excessive chunks for {}", + description + ); + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/xr.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/xr.rs new file mode 100644 index 000000000..fa72dfbe3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/rtcp/xr.rs @@ -0,0 +1,285 @@ +use std::time::SystemTime; + +use crate::util::SystemTimeExt; + +use super::{FeedbackMessageType, RtcpType, Ssrc}; +use super::{RtcpHeader, RtcpPacket}; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P|reserved | PT=XR=207 | length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | SSRC | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// : report blocks : +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +/// Extended receiver report (XR). +/// +/// RFC 3611: +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtendedReport { + /// The SSRC this report is for. + pub ssrc: Ssrc, + /// The blocks reported. + pub blocks: Vec, +} + +/// Parts of an extended report XR. +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(missing_docs)] +pub enum ReportBlock { + Rrtr(Rrtr), + Dlrr(Dlrr), +} + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | BT=4 | reserved | block length = 2 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | NTP timestamp, most significant word | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | NTP timestamp, least significant word | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +/// Receiver Reference Time Report Block. +/// +/// +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(missing_docs)] +pub struct Rrtr { + pub ntp_time: SystemTime, +} + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | BT=5 | reserved | block length | +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// | SSRC_1 (SSRC of first receiver) | sub- +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ block +// | last RR (LRR) | 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | delay since last RR (DLRR) | +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// | SSRC_2 (SSRC of second receiver) | sub- +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ block +// : ... : 2 +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + +/// DLRR Report Block +/// +/// +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(missing_docs)] +pub struct Dlrr { + pub items: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DlrrItem { + pub ssrc: Ssrc, + pub last_rr_time: u32, + pub last_rr_delay: u32, +} + +impl RtcpPacket for ExtendedReport { + fn header(&self) -> RtcpHeader { + RtcpHeader { + rtcp_type: RtcpType::ExtendedReport, + feedback_message_type: FeedbackMessageType::NotUsed, + words_less_one: (self.length_words() - 1) as u16, + } + } + + fn length_words(&self) -> usize { + let header = 1; + let ssrc = 1; + let blocks: usize = self.blocks.iter().map(|b| b.len() / 4).sum(); + header + ssrc + blocks + } + + fn write_to(&self, buf: &mut [u8]) -> usize { + let mut len = self.header().write_to(buf); + + buf[4..8].copy_from_slice(&self.ssrc.to_be_bytes()); + len += 4; + + for block in self.blocks.iter() { + len += match block { + ReportBlock::Rrtr(b) => b.write_to(&mut buf[len..]), + ReportBlock::Dlrr(b) => b.write_to(&mut buf[len..]), + }; + } + + len + } +} + +impl ReportBlock { + pub(crate) fn len(&self) -> usize { + match self { + Self::Rrtr(_) => Rrtr::len(), + Self::Dlrr(v) => v.len(), + } + } +} + +impl Rrtr { + fn write_to(&self, buf: &mut [u8]) -> usize { + // block type + buf[0] = 4_u8; + // reserved; + buf[1] = 0_u8; + // block length + buf[2..4].copy_from_slice(&2_u16.to_be_bytes()); + + // NTP timestamp + let mt = self.ntp_time.as_ntp_64(); + buf[4..12].copy_from_slice(&mt.to_be_bytes()); + + 12 + } + + fn len() -> usize { + 12 + } +} + +impl Dlrr { + fn write_to(&self, buf: &mut [u8]) -> usize { + // block type + buf[0] = 5_u8; + // reserved; + buf[1] = 0_u8; + // block length, in 32-bit words (RFC 3611 §3): each DLRR sub-block is 3 words + // (ssrc + last_rr + delay = 12 bytes). Must match the parser, which divides this + // field by 3 to recover the item count. + let len: u16 = self.items.len() as u16 * 3_u16; + buf[2..4].copy_from_slice(&len.to_be_bytes()); + + let mut buf = &mut buf[4..]; + + for item in self.items.iter() { + buf[0..4].copy_from_slice(&item.ssrc.to_be_bytes()); + buf[4..8].copy_from_slice(&item.last_rr_time.to_be_bytes()); + buf[8..12].copy_from_slice(&item.last_rr_delay.to_be_bytes()); + // advance past the full 12-byte sub-block + buf = &mut buf[12..]; + } + + self.len() + } + + fn len(&self) -> usize { + 4 + (self.items.len() * 4 * 3) + } +} + +impl<'a> TryFrom<&'a [u8]> for ExtendedReport { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 4 { + return Err("Less than 4 bytes for ExtendedReport"); + } + + let ssrc = u32::from_be_bytes(buf[..4].try_into().unwrap()).into(); + + let mut blocks: Vec = Vec::new(); + let mut buf = &buf[4..]; + + while let Ok(block) = buf.try_into() { + let block: ReportBlock = block; + let len = block.len(); + blocks.push(block); + // A block's reported len may exceed the remaining bytes for a + // truncated/malformed packet. Guard against slicing past the end + // (panic) and against a zero len that would spin forever. + if len == 0 || len >= buf.len() { + break; + } + buf = &buf[len..]; + } + + Ok(ExtendedReport { ssrc, blocks }) + } +} + +impl<'a> TryFrom<&'a [u8]> for ReportBlock { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.is_empty() { + return Err("not enough data"); + } + + let block_type: u8 = buf[0]; + match block_type { + 4 => { + let block = Rrtr::try_from(buf)?; + Ok(Self::Rrtr(block)) + } + 5 => { + let block = Dlrr::try_from(buf)?; + Ok(Self::Dlrr(block)) + } + _ => Err("unknown block type"), + } + } +} + +impl<'a> TryFrom<&'a [u8]> for Rrtr { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 12 { + return Err("Less than 12 bytes for Rrtr"); + } + + let ntp_time = u64::from_be_bytes(buf[4..4 + 8].try_into().unwrap()); + let ntp_time = SystemTime::from_ntp_64(ntp_time); + + Ok(Rrtr { ntp_time }) + } +} + +impl<'a> TryFrom<&'a [u8]> for Dlrr { + type Error = &'static str; + + fn try_from(buf: &'a [u8]) -> Result { + if buf.len() < 4 { + return Err("Less than 4 bytes for Dlrr"); + } + + let words_per_block = 3; + let blocks = u16::from_be_bytes(buf[2..4].try_into().unwrap()) / words_per_block; + + let mut items: Vec = Vec::with_capacity(blocks as usize); + + // move on after the header + let mut buf = &buf[4..]; + + for _ in 0..blocks { + // Each DLRR item is 12 bytes. The block count comes from the + // (attacker-controlled) length field, so stop if there aren't + // enough bytes left for another item. + if buf.len() < 12 { + break; + } + let ssrc = u32::from_be_bytes(buf[0..4].try_into().unwrap()).into(); + let last_rr_time = u32::from_be_bytes(buf[4..8].try_into().unwrap()); + let last_rr_delay = u32::from_be_bytes(buf[8..12].try_into().unwrap()); + items.push(DlrrItem { + ssrc, + last_rr_time, + last_rr_delay, + }); + buf = &buf[12..]; + } + + Ok(Dlrr { items }) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/srtp.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/srtp.rs new file mode 100644 index 000000000..327cd07b0 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/srtp.rs @@ -0,0 +1,1542 @@ +use std::fmt; +use std::panic::{RefUnwindSafe, UnwindSafe}; + +use crate::crypto::Aes128CmSha1_80Cipher; +use crate::crypto::dtls::{KeyingMaterial, SrtpProfile}; +use crate::crypto::{AeadAes128Gcm, AeadAes256Gcm, Aes128CmSha1_80}; +use crate::crypto::{AeadAes128GcmCipher, AeadAes256GcmCipher}; +use crate::crypto::{CryptoProvider, Sha1HmacProvider, SrtpProvider}; +use crate::io::DATAGRAM_MAX_PACKET_SIZE; + +use super::header::RtpHeader; + +// Common among various profiles(defined in RFC3711 Section 4.3) +const LABEL_RTP_AES: u8 = 0; +const LABEL_RTP_AUTHENTICATION_KEY: u8 = 1; +const LABEL_RTP_SALT: u8 = 2; + +const LABEL_RTCP_AES: u8 = 3; +const LABEL_RTCP_AUTHENTICATION_KEY: u8 = 4; +const LABEL_RTCP_SALT: u8 = 5; + +// header = 4 bytes +// ssrc = 4 bytes +// ssrtcp_index = 4 bytes +// tag = bytes +// TOTAL overhead for SRTCP = 12 + T bytes. +// However, each RTCP packet must be on a 4 byte boundary since length is +// given in number of 4 bytes - 1 (making 0 valid). + +pub const SRTP_BLOCK_SIZE: usize = 16; +const SRTCP_INDEX_LEN: usize = 4; +const MAX_TAG_LEN: usize = AeadAes256Gcm::TAG_LEN; +pub const SRTCP_OVERHEAD: usize = MAX_TAG_LEN + SRTCP_INDEX_LEN; +pub const SRTP_OVERHEAD: usize = MAX_TAG_LEN; + +// aws-lc-rs CTR mode requires output buffer to be input.len() + block_len - 1 +// This adds 15 bytes of padding that we truncate after encryption +const CTR_BUFFER_PADDING: usize = SRTP_BLOCK_SIZE - 1; + +/// Initial capacity for the RTP decryption scratch buffer. Sized to fit the +/// largest possible received datagram plus the CTR padding the AES-CM cipher +/// needs, so `unprotect_rtp` never has to grow the buffer in practice. +const RX_SCRATCH_CAPACITY: usize = DATAGRAM_MAX_PACKET_SIZE + CTR_BUFFER_PADDING; + +impl SrtpContext { + /// Create an SRTP context for the relevant profile using the provided keying material. + pub fn new( + crypto: &CryptoProvider, + profile: SrtpProfile, + mat: &KeyingMaterial, + left: bool, + ) -> Self { + let sha1_hmac_provider = crypto.sha1_hmac_provider; + + match profile { + SrtpProfile::AES128_CM_SHA1_80 => { + let key = + SrtpKey::<{ Aes128CmSha1_80::KEY_LEN }, { Aes128CmSha1_80::SALT_LEN }>::new( + mat, left, + ); + + let (rtp, rtcp) = Derived::aes_128_cm_sha1_80(crypto.srtp_provider, &key); + + SrtpContext { + rtp, + rtcp, + srtcp_index: 0, + sha1_hmac_provider, + rx_scratch: Vec::with_capacity(RX_SCRATCH_CAPACITY), + } + } + SrtpProfile::AEAD_AES_128_GCM => { + let key = SrtpKey::<{ AeadAes128Gcm::KEY_LEN }, { AeadAes128Gcm::SALT_LEN }>::new( + mat, left, + ); + + let (rtp, rtcp) = Derived::aead_aes_128_gcm(crypto.srtp_provider, &key); + + SrtpContext { + rtp, + rtcp, + srtcp_index: 0, + sha1_hmac_provider, + rx_scratch: Vec::with_capacity(RX_SCRATCH_CAPACITY), + } + } + SrtpProfile::AEAD_AES_256_GCM => { + let key = SrtpKey::<{ AeadAes256Gcm::KEY_LEN }, { AeadAes256Gcm::SALT_LEN }>::new( + mat, left, + ); + + let (rtp, rtcp) = Derived::aead_aes_256_gcm(crypto.srtp_provider, &key); + + SrtpContext { + rtp, + rtcp, + srtcp_index: 0, + sha1_hmac_provider, + rx_scratch: Vec::with_capacity(RX_SCRATCH_CAPACITY), + } + } + _ => panic!("Unexpected SRTP profile: {profile:?}"), + } + } + + #[cfg(test)] + fn new_aead_aes_128_gcm( + rtp_key: [u8; AeadAes128Gcm::KEY_LEN], + rtp_salt: [u8; AeadAes128Gcm::SALT_LEN], + rtcp_key: [u8; AeadAes128Gcm::KEY_LEN], + rtcp_salt: [u8; AeadAes128Gcm::SALT_LEN], + srtcp_index: u32, + ) -> Self { + let provider = crate::crypto::test_default_provider(); + + Self { + rtp: Derived::AeadAes128Gcm { + salt: rtp_salt, + enc: provider + .srtp_provider + .aead_aes_128_gcm() + .create_cipher(rtp_key, true), + dec: provider + .srtp_provider + .aead_aes_128_gcm() + .create_cipher(rtp_key, false), + }, + rtcp: Derived::AeadAes128Gcm { + salt: rtcp_salt, + enc: provider + .srtp_provider + .aead_aes_128_gcm() + .create_cipher(rtcp_key, true), + dec: provider + .srtp_provider + .aead_aes_128_gcm() + .create_cipher(rtcp_key, false), + }, + srtcp_index, + sha1_hmac_provider: provider.sha1_hmac_provider, + rx_scratch: Vec::with_capacity(RX_SCRATCH_CAPACITY), + } + } + + #[cfg(test)] + fn new_aead_aes_256_gcm( + rtp_key: [u8; AeadAes256Gcm::KEY_LEN], + rtp_salt: [u8; AeadAes256Gcm::SALT_LEN], + rtcp_key: [u8; AeadAes256Gcm::KEY_LEN], + rtcp_salt: [u8; AeadAes256Gcm::SALT_LEN], + srtcp_index: u32, + ) -> Self { + let provider = crate::crypto::test_default_provider(); + + Self { + rtp: Derived::AeadAes256Gcm { + salt: rtp_salt, + enc: provider + .srtp_provider + .aead_aes_256_gcm() + .create_cipher(rtp_key, true), + dec: provider + .srtp_provider + .aead_aes_256_gcm() + .create_cipher(rtp_key, false), + }, + rtcp: Derived::AeadAes256Gcm { + salt: rtcp_salt, + enc: provider + .srtp_provider + .aead_aes_256_gcm() + .create_cipher(rtcp_key, true), + dec: provider + .srtp_provider + .aead_aes_256_gcm() + .create_cipher(rtcp_key, false), + }, + srtcp_index, + sha1_hmac_provider: provider.sha1_hmac_provider, + rx_scratch: Vec::with_capacity(RX_SCRATCH_CAPACITY), + } + } +} + +#[derive(Debug)] +pub struct SrtpContext { + /// Encryption/decryption derived from srtp_key for RTP. + rtp: Derived, + /// Encryption/decryption derived from srtp_key for RTCP. + rtcp: Derived, + /// Counter for outgoing SRTCP packets. + srtcp_index: u32, + /// SHA1-HMAC provider for AES_128_CM_SHA1_80 profile. + sha1_hmac_provider: &'static dyn Sha1HmacProvider, + /// Reusable scratch buffer for RTP decryption output. Reused across calls + /// to `unprotect_rtp` so we don't allocate a fresh `Vec` per packet. + rx_scratch: Vec, +} + +/// SrtpContext contains cipher contexts that can't observe broken invariants after a panic. +impl UnwindSafe for SrtpContext {} +impl RefUnwindSafe for SrtpContext {} + +impl SrtpContext { + pub fn protect_rtp( + &mut self, + buf: &[u8], + header: &RtpHeader, + srtp_index: u64, // same as ext_seq + ) -> Vec { + // SRTP layout + // [header, [rtp, (padding + pad_count)], tag] + + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<+ + // |V=2|P|X| CC |M| PT | sequence number | | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | timestamp | | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | synchronization source (SSRC) identifier | | + // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ | + // | contributing source (CSRC) identifiers | | + // | .... | | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | RTP extension (OPTIONAL) | | + // +>+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | | payload ... | | + // | | +-------------------------------+ | + // | | | RTP padding | RTP pad count | | + // +>+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<+ + // | ~ SRTP MKI (OPTIONAL) ~ | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | : authentication tag (RECOMMENDED) : | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + // | | + // +- Encrypted Portion* Authenticated Portion ---+ + let hlen = header.header_len; + let input = &buf[hlen..]; + + match &mut self.rtp { + Derived::Aes128CmSha1_80 { key, salt, enc, .. } => { + // arbitrary body lengths are fine: the output buffer below is + // allocated with the CTR scratch the ciphers need and truncated + // after encryption, so no 16-byte alignment is required. + let iv = Aes128CmSha1_80::rtp_iv(*salt, *header.ssrc, srtp_index); + + // Allocate buffer with CTR padding (aws-lc-rs requirement) + // HMAC_TAG_LEN is 10, CTR_BUFFER_PADDING is 15, so we need 5 extra bytes + let padding_needed = + CTR_BUFFER_PADDING.saturating_sub(Aes128CmSha1_80::HMAC_TAG_LEN); + let mut output = + vec![0_u8; buf.len() + Aes128CmSha1_80::HMAC_TAG_LEN + padding_needed]; + + enc.encrypt(&iv, input, &mut output[hlen..]) + .expect("rtp encrypt"); + + output[..hlen].copy_from_slice(&buf[..hlen]); + + let hmac_start = buf.len(); + let sha1_hmac = |k: &[u8], p: &[&[u8]]| self.sha1_hmac_provider.sha1_hmac(k, p); + Aes128CmSha1_80::rtp_hmac(sha1_hmac, key, &mut output, srtp_index, hmac_start); + + output.truncate(buf.len() + Aes128CmSha1_80::HMAC_TAG_LEN); + output + } + Derived::AeadAes128Gcm { salt, enc, .. } => { + let roc = (srtp_index >> 16) as u32; + + let iv = AeadAes128Gcm::rtp_iv(*salt, *header.ssrc, roc, header.sequence_number); + let aad = &buf[..hlen]; + + // Input and output lengths for encryption: + // https://www.rfc-editor.org/rfc/rfc7714#section-5.2.1 + let mut output = vec![0_u8; buf.len() + AeadAes128Gcm::TAG_LEN]; + enc.encrypt(&iv, aad, input, &mut output[hlen..]) + .expect("rtp encrypt"); + + output[..hlen].copy_from_slice(aad); + + output + } + Derived::AeadAes256Gcm { salt, enc, .. } => { + let roc = (srtp_index >> 16) as u32; + + let iv = AeadAes256Gcm::rtp_iv(*salt, *header.ssrc, roc, header.sequence_number); + let aad = &buf[..hlen]; + + // Input and output lengths for encryption: + // https://www.rfc-editor.org/rfc/rfc7714#section-5.2.1 + let mut output = vec![0_u8; buf.len() + AeadAes256Gcm::TAG_LEN]; + enc.encrypt(&iv, aad, input, &mut output[hlen..]) + .expect("rtp encrypt"); + + output[..hlen].copy_from_slice(aad); + + output + } + } + } + + /// Decrypts an SRTP packet into the internal reusable scratch buffer and + /// returns a slice of the plaintext bytes. The returned slice borrows from + /// `self` until the next mutable use; the scratch buffer is reused across + /// calls to avoid per-packet allocation. + pub fn unprotect_rtp( + &mut self, + buf: &[u8], + header: &RtpHeader, + srtp_index: u64, // same as ext_seq + ) -> Option<&[u8]> { + match &mut self.rtp { + Derived::Aes128CmSha1_80 { key, salt, dec, .. } => { + if buf.len() < Aes128CmSha1_80::HMAC_TAG_LEN { + return None; + } + + let hmac_start = buf.len() - Aes128CmSha1_80::HMAC_TAG_LEN; + + let sha1_hmac = |k: &[u8], p: &[&[u8]]| self.sha1_hmac_provider.sha1_hmac(k, p); + if !Aes128CmSha1_80::rtp_verify( + sha1_hmac, + key, + &buf[..hmac_start], + srtp_index, + &buf[hmac_start..], + ) { + trace!("unprotect_rtp hmac verify fail"); + return None; + } + + let iv = Aes128CmSha1_80::rtp_iv(*salt, *header.ssrc, srtp_index); + + let input = &buf[header.header_len..hmac_start]; + // Sized with CTR padding (aws-lc-rs requirement). + self.rx_scratch.resize(input.len() + CTR_BUFFER_PADDING, 0); + + if let Err(e) = dec.decrypt(&iv, input, &mut self.rx_scratch) { + warn!( + "Failed to decrypt SRTP {} ({}): {}", + self.rtp.profile(), + error_details(header, srtp_index), + e + ); + return None; + }; + + Some(&self.rx_scratch[..input.len()]) + } + Derived::AeadAes128Gcm { salt, dec, .. } => { + if buf.len() < AeadAes128Gcm::TAG_LEN { + return None; + } + + let roc: u32 = (srtp_index >> 16) as u32; + let seq = header.sequence_number; + + let iv = AeadAes128Gcm::rtp_iv(*salt, *header.ssrc, roc, seq); + + let (aad, input) = buf.split_at(header.header_len); + // Input and output lengths for decryption: + // https://www.rfc-editor.org/rfc/rfc7714#section-5.2.2 + let out_len = input.len() - AeadAes128Gcm::TAG_LEN; + self.rx_scratch.resize(out_len, 0); + + match dec.decrypt(&iv, &[aad], input, &mut self.rx_scratch) { + Ok(v) => v, + Err(e) => { + warn!( + "Failed to decrypt SRTP {} ({}): {}", + self.rtp.profile(), + error_details(header, srtp_index), + e + ); + return None; + } + }; + + Some(&self.rx_scratch[..out_len]) + } + Derived::AeadAes256Gcm { salt, dec, .. } => { + if buf.len() < AeadAes256Gcm::TAG_LEN { + return None; + } + + let roc: u32 = (srtp_index >> 16) as u32; + let seq = header.sequence_number; + + let iv = AeadAes256Gcm::rtp_iv(*salt, *header.ssrc, roc, seq); + + let (aad, input) = buf.split_at(header.header_len); + // Input and output lengths for decryption: + // https://www.rfc-editor.org/rfc/rfc7714#section-5.2.2 + let out_len = input.len() - AeadAes256Gcm::TAG_LEN; + self.rx_scratch.resize(out_len, 0); + + match dec.decrypt(&iv, &[aad], input, &mut self.rx_scratch) { + Ok(v) => v, + Err(e) => { + warn!( + "Failed to decrypt SRTP {} ({}): {}", + self.rtp.profile(), + error_details(header, srtp_index), + e + ); + return None; + } + }; + + Some(&self.rx_scratch[..out_len]) + } + } + } + + pub fn protect_rtcp(&mut self, buf: &[u8]) -> Vec { + let srtcp_index = self.srtcp_index; + + // https://tools.ietf.org/html/rfc3711#page-15 + // The SRTCP index MUST be set to zero before the first SRTCP + // packet is sent, and MUST be incremented by one, + // modulo 2^31, after each SRTCP packet is sent. + self.srtcp_index = (self.srtcp_index + 1) % 2_u32.pow(31); + + // e is always encrypted, rest is 31 byte index. + let e_and_si = 0x8000_0000 | srtcp_index; + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + + if ssrc == 0 { + warn!("SSRC 0 does not make a good SRTCP IV"); + } + + match &mut self.rtcp { + Derived::Aes128CmSha1_80 { key, salt, enc, .. } => { + let iv = Aes128CmSha1_80::rtp_iv(*salt, ssrc, srtcp_index as u64); + + let final_size = buf.len() + SRTCP_INDEX_LEN + Aes128CmSha1_80::HMAC_TAG_LEN; + // Allocate buffer with CTR padding (aws-lc-rs requirement) + // SRTCP_INDEX_LEN + HMAC_TAG_LEN is 14, CTR_BUFFER_PADDING is 15, so we need 1 extra byte + let padding_needed = CTR_BUFFER_PADDING + .saturating_sub(SRTCP_INDEX_LEN + Aes128CmSha1_80::HMAC_TAG_LEN); + let mut output = vec![0_u8; final_size + padding_needed]; + + output[0..8].copy_from_slice(&buf[0..8]); + let input = &buf[8..]; + enc.encrypt(&iv, input, &mut output[8..]) + .expect("rtcp encrypt"); + + let to = &mut output[buf.len()..]; + to[0..4].copy_from_slice(&e_and_si.to_be_bytes()); + + let hmac_index = final_size - Aes128CmSha1_80::HMAC_TAG_LEN; + let sha1_hmac = |k: &[u8], p: &[&[u8]]| self.sha1_hmac_provider.sha1_hmac(k, p); + Aes128CmSha1_80::rtcp_hmac(sha1_hmac, key, &mut output, hmac_index); + + output.truncate(final_size); + output + } + Derived::AeadAes128Gcm { salt, enc, .. } => { + let iv = AeadAes128Gcm::rtcp_iv(*salt, ssrc, srtcp_index); + + let mut aad = [0; AeadAes128Gcm::RTCP_AAD_LEN]; + aad[..8].copy_from_slice(&buf[..8]); + aad[8..12].copy_from_slice(&e_and_si.to_be_bytes()); + + let mut output = vec![0_u8; buf.len() + SRTCP_INDEX_LEN + AeadAes128Gcm::TAG_LEN]; + output[0..8].copy_from_slice(&buf[0..8]); + let input = &buf[8..]; + + let enc_start = 8; + let enc_end = input.len() + 8 + AeadAes128Gcm::TAG_LEN; + let encout = &mut output[enc_start..enc_end]; + + enc.encrypt(&iv, &aad, input, encout).expect("rtcp encrypt"); + + let to = &mut output[enc_end..]; + to[0..4].copy_from_slice(&e_and_si.to_be_bytes()); + + output + } + Derived::AeadAes256Gcm { salt, enc, .. } => { + let iv = AeadAes256Gcm::rtcp_iv(*salt, ssrc, srtcp_index); + + let mut aad = [0; AeadAes256Gcm::RTCP_AAD_LEN]; + aad[..8].copy_from_slice(&buf[..8]); + aad[8..12].copy_from_slice(&e_and_si.to_be_bytes()); + + let mut output = vec![0_u8; buf.len() + SRTCP_INDEX_LEN + AeadAes256Gcm::TAG_LEN]; + output[0..8].copy_from_slice(&buf[0..8]); + let input = &buf[8..]; + + let enc_start = 8; + let enc_end = input.len() + 8 + AeadAes256Gcm::TAG_LEN; + let encout = &mut output[enc_start..enc_end]; + + enc.encrypt(&iv, &aad, input, encout).expect("rtcp encrypt"); + + let to = &mut output[enc_end..]; + to[0..4].copy_from_slice(&e_and_si.to_be_bytes()); + + output + } + } + } + + // SRTCP layout + // ["header", ssrc, payload, ["header", ssrc, payload], ...], ssrtcp_index, tag] + // + // |----------------------------------------------------------------------| + // authenticated + // + // |--------------------------------------| + // encrypted (aes) + pub fn unprotect_rtcp(&mut self, buf: &[u8]) -> Option> { + match &mut self.rtcp { + Derived::Aes128CmSha1_80 { key, salt, dec, .. } => { + if buf.len() < Aes128CmSha1_80::HMAC_TAG_LEN + SRTCP_INDEX_LEN { + return None; + } + + let hmac_start = buf.len() - Aes128CmSha1_80::HMAC_TAG_LEN; + + let sha1_hmac = |k: &[u8], p: &[&[u8]]| self.sha1_hmac_provider.sha1_hmac(k, p); + if !Aes128CmSha1_80::rtcp_verify( + sha1_hmac, + key, + &buf[..hmac_start], + &buf[hmac_start..], + ) { + trace!("unprotect_rtcp hmac verify fail"); + return None; + } + + let idx_start = hmac_start - SRTCP_INDEX_LEN; + + let srtcp_index_be = [ + buf[idx_start], + buf[idx_start + 1], + buf[idx_start + 2], + buf[idx_start + 3], + ]; + + // E-flag and SRTCP index. + let e_and_si = u32::from_be_bytes(srtcp_index_be); + + let is_encrypted = e_and_si & 0x8000_0000 > 0; + + if !is_encrypted { + // Non-encrypted we can just return + return Some(buf[0..idx_start].to_vec()); + } + + // The SRTCP index is a 31-bit counter for the SRTCP packet. + let srtcp_index = e_and_si & 0x7fff_ffff; + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + + let iv = Aes128CmSha1_80::rtp_iv(*salt, ssrc, srtcp_index as u64); + + // The Encrypted Portion of an SRTCP packet consists of the encryption + // of the RTCP payload of the equivalent compound RTCP packet, from the + // first RTCP packet, i.e., from the ninth (9) octet to the end of the + // compound packet. + let input = &buf[8..idx_start]; + // Allocate buffer with CTR padding (aws-lc-rs requirement) + let mut output = vec![0_u8; input.len() + 8 + CTR_BUFFER_PADDING]; + output[0..8].copy_from_slice(&buf[0..8]); + + if let Err(e) = dec.decrypt(&iv, input, &mut output[8..]) { + warn!("Failed to decrypt SRTCP {}: {}", self.rtcp.profile(), e); + return None; + } + + output.truncate(input.len() + 8); + Some(output) + } + Derived::AeadAes128Gcm { salt, dec, .. } => { + if buf.len() < SRTCP_INDEX_LEN + AeadAes128Gcm::TAG_LEN { + // Too short + return None; + } + + let idx_start = buf.len() - SRTCP_INDEX_LEN; + + // Assume no MKI + let e_and_si = u32::from_be_bytes( + buf[idx_start..buf.len()] + .try_into() + // This is ok because SRTCP_INDEX_LEN is 4 bytes and the buffer is at least + // that long. + .expect("SRTCP_INDEX_LEN to be 4"), + ); + let is_encrypted = e_and_si & 0x8000_0000 > 0; + + // The Encrypted Portion of an SRTCP packet consists of the encryption + // of the RTCP payload of the equivalent compound RTCP packet, from the + // first RTCP packet, i.e., from the ninth (9) octet to the end of the + // compound packet. + let input = if is_encrypted { + &buf[8..idx_start] + } else { + // No, encryption but we still pass the tag down to decrypt so it can verify + // it. + &buf[idx_start - AeadAes128Gcm::TAG_LEN..idx_start] + }; + + // The SRTCP index is a 31-bit counter for the SRTCP packet. + let srtcp_index = e_and_si & 0x7fff_ffff; + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + + let iv = AeadAes128Gcm::rtcp_iv(*salt, ssrc, srtcp_index); + // Declared out here for lifetime purposes, only used in the first branch of the if. + let mut encrypted_aad = [0; AeadAes128Gcm::RTCP_AAD_LEN]; + let mut aads: [&[u8]; 2] = [&[], &[]]; + + if is_encrypted { + encrypted_aad[0..8].copy_from_slice(&buf[0..8]); + encrypted_aad[8..12].copy_from_slice(&e_and_si.to_be_bytes()); + + aads[0] = encrypted_aad.as_slice(); + } else { + // The whole packet is AAD + aads[0] = &buf[0..idx_start - AeadAes128Gcm::TAG_LEN]; + aads[1] = &buf[idx_start..]; + }; + + let mut output = vec![0_u8; buf.len() - AeadAes128Gcm::TAG_LEN - SRTCP_INDEX_LEN]; + output[0..8].copy_from_slice(&buf[0..8]); + + let count = match dec.decrypt(&iv, &aads, input, &mut output[8..]) { + Ok(c) => c, + Err(e) => { + warn!("Failed to decrypt SRTCP {}: {}", self.rtcp.profile(), e); + return None; + } + }; + + if is_encrypted { + output.truncate(8 + count); + } else { + // decrypt didn't error, the data is authenticated. + output.copy_from_slice( + &buf[0..buf.len() - SRTCP_INDEX_LEN - AeadAes128Gcm::TAG_LEN], + ) + } + + Some(output) + } + Derived::AeadAes256Gcm { salt, dec, .. } => { + if buf.len() < SRTCP_INDEX_LEN + AeadAes256Gcm::TAG_LEN { + // Too short + return None; + } + + let idx_start = buf.len() - SRTCP_INDEX_LEN; + + // Assume no MKI + let e_and_si = u32::from_be_bytes( + buf[idx_start..buf.len()] + .try_into() + // This is ok because SRTCP_INDEX_LEN is 4 bytes and the buffer is at least + // that long. + .expect("SRTCP_INDEX_LEN to be 4"), + ); + let is_encrypted = e_and_si & 0x8000_0000 > 0; + + // The Encrypted Portion of an SRTCP packet consists of the encryption + // of the RTCP payload of the equivalent compound RTCP packet, from the + // first RTCP packet, i.e., from the ninth (9) octet to the end of the + // compound packet. + let input = if is_encrypted { + &buf[8..idx_start] + } else { + // No, encryption but we still pass the tag down to decrypt so it can verify + // it. + &buf[idx_start - AeadAes256Gcm::TAG_LEN..idx_start] + }; + + // The SRTCP index is a 31-bit counter for the SRTCP packet. + let srtcp_index = e_and_si & 0x7fff_ffff; + let ssrc = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + + let iv = AeadAes256Gcm::rtcp_iv(*salt, ssrc, srtcp_index); + // Declared out here for lifetime purposes, only used in the first branch of the if. + let mut encrypted_aad = [0; AeadAes256Gcm::RTCP_AAD_LEN]; + let mut aads: [&[u8]; 2] = [&[], &[]]; + + if is_encrypted { + encrypted_aad[0..8].copy_from_slice(&buf[0..8]); + encrypted_aad[8..12].copy_from_slice(&e_and_si.to_be_bytes()); + + aads[0] = encrypted_aad.as_slice(); + } else { + // The whole packet is AAD + aads[0] = &buf[0..idx_start - AeadAes256Gcm::TAG_LEN]; + aads[1] = &buf[idx_start..]; + }; + + let mut output = vec![0_u8; buf.len() - AeadAes256Gcm::TAG_LEN - SRTCP_INDEX_LEN]; + output[0..8].copy_from_slice(&buf[0..8]); + + let count = match dec.decrypt(&iv, &aads, input, &mut output[8..]) { + Ok(c) => c, + Err(e) => { + warn!("Failed to decrypt SRTCP {}: {}", self.rtcp.profile(), e); + return None; + } + }; + + if is_encrypted { + output.truncate(8 + count); + } else { + // decrypt didn't error, the data is authenticated. + output.copy_from_slice( + &buf[0..buf.len() - SRTCP_INDEX_LEN - AeadAes256Gcm::TAG_LEN], + ) + } + + Some(output) + } + } + } +} + +/// SrtpKeys created from DTLS SrtpKeyMaterial. +#[derive(Debug)] +struct SrtpKey { + master: [u8; ML], + salt: [u8; SL], +} + +impl SrtpKey { + pub fn new(mat: &KeyingMaterial, left: bool) -> Self { + // layout in SrtpKeyMaterial is [key_input, key_output, salt_input, salt_output] + + // Invariant + assert!( + mat.len() == ML * 2 + SL * 2, + "The KeyingMaterial provided to SrtpKey::new must be ML * 2 + SL * 2 in length" + ); + + // offset 0, offset 1 + let (o0, o1) = if left { (0, 0) } else { (ML, SL) }; + + let mut master = [0; ML]; + let mut salt = [0; SL]; + + master[0..ML].copy_from_slice(&mat[o0..(o0 + ML)]); + salt[0..SL].copy_from_slice(&mat[(ML + ML + o1)..(ML + ML + o1 + SL)]); + + SrtpKey { master, salt } + } + + fn derive(&self, crypto: &dyn SrtpProvider, label: u8, out: &mut [u8]) { + // AES-CM (128 or 256 bits) defined in RFC3711 + assert!(ML == 16 || ML == 32, "Only valid for 128 bit master keys"); + assert!(SL <= 14, "Only valid for 128 bit master keys"); + let mut i = 0; // index in out + + // input layout: [salt[SL] || label, round[2]] (|| is xor 7th byte) + let mut input = [0; 16]; + + input[0..SL].copy_from_slice(&self.salt[..]); + input[7] ^= label; + + let mut buf = [0; 16 + 16]; // output from each AES + let mut round: u16 = 0; // counter for each AES round + + // loop each AES round + loop { + if i == out.len() { + break; + } + + // splice in round at bottom of input + input[14..].copy_from_slice(&round.to_be_bytes()[..]); + + // default key derivation function, which uses AES-128 in Counter Mode + match ML { + 16 => crypto.srtp_aes_128_ecb_round(&self.master, &input[..], &mut buf[..]), + 32 => crypto.srtp_aes_256_ecb_round(&self.master, &input[..], &mut buf[..]), + _ => panic!("Only valid for 128 or 256 bit master keys"), + } + + // Copy to output. Even if we get 32 bytes of output with AES 128 ECB, we + // only use the first 16. That matches the tests in the RFC. + for j in buf.iter().take(16) { + if i == out.len() { + break; + } + out[i] = *j; + i += 1; + } + + round += 1; + } + } +} + +/// Encryption/decryption derived from the SrtpKey. +enum Derived { + Aes128CmSha1_80 { + key: [u8; 20], + salt: [u8; 14], + enc: Box, + dec: Box, + }, + AeadAes128Gcm { + salt: [u8; 12], + enc: Box, + dec: Box, + }, + AeadAes256Gcm { + salt: [u8; 12], + enc: Box, + dec: Box, + }, +} + +impl Derived { + fn aes_128_cm_sha1_80( + crypto: &dyn SrtpProvider, + srtp_key: &SrtpKey<{ Aes128CmSha1_80::KEY_LEN }, { Aes128CmSha1_80::SALT_LEN }>, + ) -> (Self, Self) { + // RTP AES Counter + let mut rtp_aes = [0; Aes128CmSha1_80::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTP_AES, &mut rtp_aes[..]); + + // RTP SHA1 HMAC + let rtp_hmac = { + let mut hmac = [0; Aes128CmSha1_80::HMAC_KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTP_AUTHENTICATION_KEY, &mut hmac[..]); + hmac + }; + + // RTP IV SALT + let mut rtp_salt = [0; Aes128CmSha1_80::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTP_SALT, &mut rtp_salt[..]); + + // RTCP AES Counter + let mut rtcp_aes = [0; Aes128CmSha1_80::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_AES, &mut rtcp_aes[..]); + + // RTCP SHA1 HMAC + let rtcp_hmac = { + let mut hmac = [0; Aes128CmSha1_80::HMAC_KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_AUTHENTICATION_KEY, &mut hmac[..]); + hmac + }; + + // RTCP IV SALT + let mut rtcp_salt = [0; Aes128CmSha1_80::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_SALT, &mut rtcp_salt[..]); + + let rtp = Derived::Aes128CmSha1_80 { + key: rtp_hmac, + salt: rtp_salt, + enc: crypto.aes_128_cm_sha1_80().create_cipher(rtp_aes, true), + dec: crypto.aes_128_cm_sha1_80().create_cipher(rtp_aes, false), + }; + + let rtcp = Derived::Aes128CmSha1_80 { + key: rtcp_hmac, + salt: rtcp_salt, + enc: crypto.aes_128_cm_sha1_80().create_cipher(rtcp_aes, true), + dec: crypto.aes_128_cm_sha1_80().create_cipher(rtcp_aes, false), + }; + + (rtp, rtcp) + } + + fn aead_aes_128_gcm( + crypto: &dyn SrtpProvider, + srtp_key: &SrtpKey<{ AeadAes128Gcm::KEY_LEN }, { AeadAes128Gcm::SALT_LEN }>, + ) -> (Derived, Derived) { + // RTP session key + let mut rtp_aes = [0; AeadAes128Gcm::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTP_AES, &mut rtp_aes[..]); + + // RTP session salt + let mut rtp_salt = [0; AeadAes128Gcm::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTP_SALT, &mut rtp_salt[..]); + + // RTCP session key + let mut rtcp_aes = [0; AeadAes128Gcm::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_AES, &mut rtcp_aes[..]); + + // RTCP session salt + let mut rtcp_salt = [0; AeadAes128Gcm::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_SALT, &mut rtcp_salt[..]); + + let rtp = Derived::AeadAes128Gcm { + salt: rtp_salt, + enc: crypto.aead_aes_128_gcm().create_cipher(rtp_aes, true), + dec: crypto.aead_aes_128_gcm().create_cipher(rtp_aes, false), + }; + + let rtcp = Derived::AeadAes128Gcm { + salt: rtcp_salt, + enc: crypto.aead_aes_128_gcm().create_cipher(rtcp_aes, true), + dec: crypto.aead_aes_128_gcm().create_cipher(rtcp_aes, false), + }; + + (rtp, rtcp) + } + + fn aead_aes_256_gcm( + crypto: &dyn SrtpProvider, + srtp_key: &SrtpKey<{ AeadAes256Gcm::KEY_LEN }, { AeadAes256Gcm::SALT_LEN }>, + ) -> (Derived, Derived) { + // RTP session key + let mut rtp_aes = [0; AeadAes256Gcm::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTP_AES, &mut rtp_aes[..]); + + // RTP session salt + let mut rtp_salt = [0; AeadAes256Gcm::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTP_SALT, &mut rtp_salt[..]); + + // RTCP session key + let mut rtcp_aes = [0; AeadAes256Gcm::KEY_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_AES, &mut rtcp_aes[..]); + + // RTCP session salt + let mut rtcp_salt = [0; AeadAes256Gcm::SALT_LEN]; + srtp_key.derive(crypto, LABEL_RTCP_SALT, &mut rtcp_salt[..]); + + let rtp = Derived::AeadAes256Gcm { + salt: rtp_salt, + enc: crypto.aead_aes_256_gcm().create_cipher(rtp_aes, true), + dec: crypto.aead_aes_256_gcm().create_cipher(rtp_aes, false), + }; + + let rtcp = Derived::AeadAes256Gcm { + salt: rtcp_salt, + enc: crypto.aead_aes_256_gcm().create_cipher(rtcp_aes, true), + dec: crypto.aead_aes_256_gcm().create_cipher(rtcp_aes, false), + }; + + (rtp, rtcp) + } + + fn profile(&self) -> SrtpProfile { + match self { + Derived::Aes128CmSha1_80 { .. } => SrtpProfile::AES128_CM_SHA1_80, + Derived::AeadAes128Gcm { .. } => SrtpProfile::AEAD_AES_128_GCM, + Derived::AeadAes256Gcm { .. } => SrtpProfile::AEAD_AES_256_GCM, + } + } +} + +impl fmt::Debug for Derived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Derived") + } +} + +fn error_details(header: &RtpHeader, srtp_index: u64) -> String { + format!( + "SSRC: {} seq_no: {} pt: {} mid: {:?} rid: {:?} rid_repair: {:?} srtp_index: {}", + header.ssrc, + header.sequence_number, + header.payload_type, + header.ext_vals.mid, + header.ext_vals.rid, + header.ext_vals.rid_repair, + srtp_index + ) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn derive_key() { + crate::init_crypto_default(); + let provider = crate::crypto::from_feature_flags(); + let crypto = provider.srtp_provider; + + // https://tools.ietf.org/html/rfc3711#appendix-B.3 + // + // Key Derivation Test Vectors. + + let master = [ + 0xE1, 0xF9, 0x7A, 0x0D, 0x3E, 0x01, 0x8B, 0xE0, // + 0xD6, 0x4F, 0xA3, 0x2C, 0x06, 0xDE, 0x41, 0x39, + ]; + + let salt = [ + 0x0E, 0xC6, 0x75, 0xAD, 0x49, 0x8A, 0xFE, // + 0xEB, 0xB6, 0x96, 0x0B, 0x3A, 0xAB, 0xE6, + ]; + + let sk = SrtpKey { master, salt }; + + // aes crypto key + let mut out = [0_u8; 16]; + sk.derive(crypto, 0, &mut out[..]); + + assert_eq!( + out, + [ + 0xC6, 0x1E, 0x7A, 0x93, 0x74, 0x4F, 0x39, 0xEE, // + 0x10, 0x73, 0x4A, 0xFE, 0x3F, 0xF7, 0xA0, 0x87 + ] + ); + + // hmac + let mut out = [0_u8; 20]; + sk.derive(crypto, 1, &mut out[..]); + + assert_eq!( + out, + [ + 0xCE, 0xBE, 0x32, 0x1F, 0x6F, 0xF7, 0x71, 0x6B, // + 0x6F, 0xD4, 0xAB, 0x49, 0xAF, 0x25, 0x6A, 0x15, // + 0x6D, 0x38, 0xBA, 0xA4 + ] + ); + + // salt + let mut out = [0_u8; 14]; + sk.derive(crypto, 2, &mut out[..]); + + assert_eq!( + out, + [ + 0x30, 0xCB, 0xBC, 0x08, 0x86, 0x3D, 0x8C, // + 0x85, 0xD4, 0x9D, 0xB3, 0x4A, 0x9A, 0xE1 + ] + ); + } + + mod test_aes128_cm_sha1_80 { + use super::*; + + const MAT: [u8; 60] = [ + 0x2C, 0xB0, 0x23, 0x46, 0xB4, 0x22, 0x76, 0xA6, 0x72, 0xCF, 0xD1, 0x43, 0xAE, 0xC2, + 0xD5, 0xEE, 0xDD, 0xDE, 0x55, 0xF0, 0xAD, 0x7B, 0xCA, 0xC2, 0x26, 0x66, 0xF1, 0xC6, + 0x38, 0x61, 0x73, 0xED, 0x6E, 0xB2, 0x5C, 0xB7, 0xD2, 0x6A, 0x61, 0xA1, 0xEE, 0x2C, + 0x21, 0x0A, 0xDA, 0xE7, 0x60, 0xAA, 0xA2, 0xFD, 0x67, 0xB6, 0x72, 0xC4, 0x1A, 0xED, + 0x10, 0x5F, 0x9D, 0x36, + ]; + + const SRTCP: &[u8] = &[ + // header + 0x80, 0xC8, 0x00, 0x06, // + // ssrc + 0x3C, 0xD7, 0xCC, 0x13, // + // encrypted payload + 0xB7, 0xC8, 0x31, 0xDC, 0xB7, 0x76, 0xCD, 0x8D, 0xC2, 0x6F, 0xDA, 0x1D, 0x9B, 0xFC, + 0x8E, 0xE6, 0x58, 0x9A, 0x1A, 0x8A, 0x49, 0x28, 0x9C, 0xAE, 0xB2, 0x64, 0x20, 0x0C, + 0x37, 0xD2, 0xD0, 0xA4, 0xAF, 0xAC, 0x63, 0x85, 0xFF, 0xC6, 0x0D, 0xEC, 0x7D, 0x06, + 0xD4, 0x87, 0x3D, 0xD3, 0xA8, 0xCC, // + // E flag and srtcp index (1) + 0x80, 0x00, 0x00, 0x01, // + // hmac + 0xB7, 0xBB, 0x52, 0x65, 0x21, 0xD1, 0xE7, 0x3C, 0x0F, 0xC0, + ]; + + const DECRYPTED_PAYLOAD: &[u8] = &[ + 0x80, 0xc8, 0x00, 0x06, 0x3c, 0xd7, 0xcc, 0x13, 0xe2, 0xee, 0x35, 0xc8, 0x60, 0x4e, + 0x61, 0x8c, 0x26, 0xf3, 0x27, 0x34, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x14, 0x07, + 0x81, 0xca, 0x00, 0x06, 0x3c, 0xd7, 0xcc, 0x13, 0x01, 0x10, 0x38, 0x6e, 0x46, 0x75, + 0x32, 0x68, 0x57, 0x66, 0x72, 0x4d, 0x44, 0x72, 0x47, 0x66, 0x34, 0x6f, 0x00, 0x00, + ]; + + #[test] + fn unprotect_rtcp() { + let key_mat = KeyingMaterial::new(&MAT); + let crypto = crate::crypto::test_default_provider(); + let mut ctx_rx = + SrtpContext::new(crypto, SrtpProfile::AES128_CM_SHA1_80, &key_mat, true); + ctx_rx.srtcp_index = 1; + + let decrypted = ctx_rx.unprotect_rtcp(SRTCP).unwrap(); + + assert_eq!(ctx_rx.srtcp_index, 1); + // check srtcp_index in incoming was indeed 1 + let srtcp_index = SRTCP.len() - Aes128CmSha1_80::HMAC_TAG_LEN - SRTCP_INDEX_LEN; + let e_and_i = &SRTCP[srtcp_index..(srtcp_index + 4)]; + assert_eq!(e_and_i, &0x8000_0001_u32.to_be_bytes()); + assert_eq!(decrypted, DECRYPTED_PAYLOAD); + + // Take us back to where we started. + let encrypted = ctx_rx.protect_rtcp(&decrypted); + assert_eq!(encrypted, SRTCP); + } + } + + mod test_aead_aes_128_gcm { + use crate::crypto::AeadAes128Gcm; + use crate::rtp_::ExtensionMap; + + use super::*; + + const TAG_LEN: usize = AeadAes128Gcm::TAG_LEN; + + mod rfc7714 { + // Test vectors from RFC7714 + + // Session Key (RTP and RTCP) + pub(super) const KEY: [u8; 16] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + ]; + + // Session Salt (RTP and RTCP) + pub(super) const SALT: [u8; 12] = [ + 0x51, 0x75, 0x69, 0x64, 0x20, 0x70, 0x72, 0x6f, 0x20, 0x71, 0x75, 0x6f, + ]; + + /// Full plaintext RTP packet. First 12 octets is the header + pub(super) const PLAINTEXT_RTP_PACKET: &[u8] = &[ + 0x80, 0x40, 0xf1, 0x7b, 0x80, 0x41, 0xf8, 0xd3, 0x55, 0x01, 0xa0, 0xb2, 0x47, 0x61, + 0x6c, 0x6c, 0x69, 0x61, 0x20, 0x65, 0x73, 0x74, 0x20, 0x6f, 0x6d, 0x6e, 0x69, 0x73, + 0x20, 0x64, 0x69, 0x76, 0x69, 0x73, 0x61, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x61, 0x72, + 0x74, 0x65, 0x73, 0x20, 0x74, 0x72, 0x65, 0x73, + ]; + + /// Full encrypted RTP packet. First 12 octets is the header. + pub(super) const PROTECTED_RTP_PACKET: &[u8] = &[ + 0x80, 0x40, 0xf1, 0x7b, 0x80, 0x41, 0xf8, 0xd3, 0x55, 0x01, 0xa0, 0xb2, 0xf2, 0x4d, + 0xe3, 0xa3, 0xfb, 0x34, 0xde, 0x6c, 0xac, 0xba, 0x86, 0x1c, 0x9d, 0x7e, 0x4b, 0xca, + 0xbe, 0x63, 0x3b, 0xd5, 0x0d, 0x29, 0x4e, 0x6f, 0x42, 0xa5, 0xf4, 0x7a, 0x51, 0xc7, + 0xd1, 0x9b, 0x36, 0xde, 0x3a, 0xdf, 0x88, 0x33, 0x89, 0x9d, 0x7f, 0x27, 0xbe, 0xb1, + 0x6a, 0x91, 0x52, 0xcf, 0x76, 0x5e, 0xe4, 0x39, 0x0c, 0xce, + ]; + + // Full plaintext RTCP packet + pub(super) const PLAINTEXT_RTCP_PACKET: &[u8] = &[ + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0x4e, 0x54, 0x50, 0x31, 0x4e, 0x54, + 0x50, 0x32, 0x52, 0x54, 0x50, 0x20, 0x00, 0x00, 0x04, 0x2a, 0x00, 0x00, 0xe9, 0x30, + 0x4c, 0x75, 0x6e, 0x61, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, + 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, + ]; + + /// Full encrypted RTCP packet + pub(super) const PROTECTED_RTCP_PACKET: &[u8] = &[ + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0x63, 0xe9, 0x48, 0x85, 0xdc, 0xda, + 0xb6, 0x7c, 0xa7, 0x27, 0xd7, 0x66, 0x2f, 0x6b, 0x7e, 0x99, 0x7f, 0xf5, 0xc0, 0xf7, + 0x6c, 0x06, 0xf3, 0x2d, 0xc6, 0x76, 0xa5, 0xf1, 0x73, 0x0d, 0x6f, 0xda, 0x4c, 0xe0, + 0x9b, 0x46, 0x86, 0x30, 0x3d, 0xed, 0x0b, 0xb9, 0x27, 0x5b, 0xc8, 0x4a, 0xa4, 0x58, + 0x96, 0xcf, 0x4d, 0x2f, 0xc5, 0xab, 0xf8, 0x72, 0x45, 0xd9, 0xea, 0xde, 0x80, 0x00, + 0x05, 0xd4, + ]; + + // A RTCP packet that hasn't been encrypted, only authenticated. + pub(super) const TAGGED_RTCP_PACKET: &[u8] = &[ + // RTCP Packet + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0x4e, 0x54, 0x50, 0x31, 0x4e, 0x54, + 0x50, 0x32, 0x52, 0x54, 0x50, 0x20, 0x00, 0x00, 0x04, 0x2a, 0x00, 0x00, 0xe9, 0x30, + 0x4c, 0x75, 0x6e, 0x61, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, + 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, // + // Tag + 0x84, 0x1d, 0xd9, 0x68, 0x3d, 0xd7, 0x8e, 0xc9, 0x2a, 0xe5, 0x87, 0x90, 0x12, 0x5f, + 0x62, 0xb3, // + // SRTCP Index + 0x00, 0x00, 0x05, 0xd4, + ]; + } + + #[test] + fn protect_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + + let header = + RtpHeader::parse(&rfc7714::PLAINTEXT_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let out = context.protect_rtp(rfc7714::PLAINTEXT_RTP_PACKET, &header, 0); + + assert_eq!( + out, + rfc7714::PROTECTED_RTP_PACKET, + "failed to encrypted packet.\n{:02x?}\n{:02x?}", + out, + &rfc7714::PLAINTEXT_RTP_PACKET + ); + } + + #[test] + fn unprotect_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + let header = + RtpHeader::parse(&rfc7714::PROTECTED_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + + let out = context + .unprotect_rtp(rfc7714::PROTECTED_RTP_PACKET, &header, 0) + .expect("decrypt rtp"); + + assert_eq!( + out, + &rfc7714::PLAINTEXT_RTP_PACKET[12..], + "failed to decrypt packet.\n{:02x?}\n{:02x?}", + out, + &rfc7714::PLAINTEXT_RTP_PACKET + ); + } + + #[test] + fn symmetry_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + + // First we encrypt + let header = + RtpHeader::parse(&rfc7714::PLAINTEXT_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let encrypted = context.protect_rtp(rfc7714::PLAINTEXT_RTP_PACKET, &header, 0); + + // Then we decrypt the resulting cipher text + let header = RtpHeader::parse(&encrypted[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let decrypted = context + .unprotect_rtp(&encrypted, &header, 0) + .expect("rtp unprotect"); + + // And verify we get the input back. + assert_eq!(decrypted, &rfc7714::PLAINTEXT_RTP_PACKET[12..]); + } + + #[test] + fn unprotect_rtp_should_fail_with_broken_tag_data() { + let mut context = make_rtp_context(); + + let header_buf = { + let mut buf = rfc7714::PROTECTED_RTP_PACKET[..12].to_vec(); + // Mess with part of the sequence number, since this makes up part of the + // authenticated additional data(AAD) the resulting authenticity tag should not + // match. + buf[3] ^= 0xFF; + + buf + }; + + let header = + RtpHeader::parse(&header_buf, &ExtensionMap::empty()).expect("header to parse"); + + let result = context.unprotect_rtp(rfc7714::PROTECTED_RTP_PACKET, &header, 0); + assert!( + result.is_none(), + "Should fail to decrypt a SRTP packet that has mismatched \ + authenicated additional data" + ); + } + + #[test] + fn unprotect_rtp_should_fail_with_broken_null_tag() { + let mut context = make_rtp_context(); + + let input = { + let mut input = rfc7714::PROTECTED_RTP_PACKET.to_vec(); + let len = input.len(); + input[len - TAG_LEN..].copy_from_slice(&[0; TAG_LEN]); + + input + }; + + let header = + RtpHeader::parse(&input[..12], &ExtensionMap::empty()).expect("header to parse"); + + let result = context.unprotect_rtp(&input, &header, 0); + assert!( + result.is_none(), + "Should fail to decrypt a SRTP packet with null tag" + ); + } + + #[test] + fn protect_rtcp_rfc_7714_test() { + let mut context = make_rtcp_context(); + + let out = context.protect_rtcp(rfc7714::PLAINTEXT_RTCP_PACKET); + + assert!( + out == rfc7714::PROTECTED_RTCP_PACKET, + "Expected encrypted and tagged RTCP packet:\n{:02x?}\nGot:\n{:02x?}", + rfc7714::PROTECTED_RTCP_PACKET, + out + ); + } + + #[test] + fn unprotect_rtcp_rfc_auth_only_7714_test() { + let mut context = make_rtcp_context(); + + let out = context + .unprotect_rtcp(rfc7714::TAGGED_RTCP_PACKET) + .expect("Unprotect RTCP"); + + assert_eq!(out, rfc7714::PLAINTEXT_RTCP_PACKET); + } + + fn make_rtp_context() -> SrtpContext { + crate::init_crypto_default(); + SrtpContext::new_aead_aes_128_gcm( + rfc7714::KEY, + rfc7714::SALT, + rfc7714::KEY, + rfc7714::SALT, + 0, + ) + } + + fn make_rtcp_context() -> SrtpContext { + crate::init_crypto_default(); + SrtpContext::new_aead_aes_128_gcm( + rfc7714::KEY, + rfc7714::SALT, + rfc7714::KEY, + rfc7714::SALT, + 0x000005d4, + ) + } + } + + mod test_aead_aes_256_gcm { + use crate::crypto::AeadAes256Gcm; + use crate::rtp_::ExtensionMap; + + use super::*; + + const TAG_LEN: usize = AeadAes256Gcm::TAG_LEN; + + mod rfc7714 { + // Test vectors from RFC7714 + + // Session Key (RTP and RTCP) + pub(super) const KEY: [u8; 32] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, // + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + ]; + + // Session Salt (RTP and RTCP) + pub(super) const SALT: [u8; 12] = [ + 0x51, 0x75, 0x69, 0x64, 0x20, 0x70, 0x72, 0x6f, 0x20, 0x71, 0x75, 0x6f, + ]; + + /// Full plaintext RTP packet. First 12 octets is the header + pub(super) const PLAINTEXT_RTP_PACKET: &[u8] = &[ + 0x80, 0x40, 0xf1, 0x7b, 0x80, 0x41, 0xf8, 0xd3, 0x55, 0x01, 0xa0, 0xb2, 0x47, 0x61, + 0x6c, 0x6c, 0x69, 0x61, 0x20, 0x65, 0x73, 0x74, 0x20, 0x6f, 0x6d, 0x6e, 0x69, 0x73, + 0x20, 0x64, 0x69, 0x76, 0x69, 0x73, 0x61, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x61, 0x72, + 0x74, 0x65, 0x73, 0x20, 0x74, 0x72, 0x65, 0x73, + ]; + + /// Full encrypted RTP packet. First 12 octets is the header. + pub(super) const PROTECTED_RTP_PACKET: &[u8] = &[ + 0x80, 0x40, 0xf1, 0x7b, 0x80, 0x41, 0xf8, 0xd3, 0x55, 0x01, 0xa0, 0xb2, 0x32, 0xb1, + 0xde, 0x78, 0xa8, 0x22, 0xfe, 0x12, 0xef, 0x9f, 0x78, 0xfa, 0x33, 0x2e, 0x33, 0xaa, + 0xb1, 0x80, 0x12, 0x38, 0x9a, 0x58, 0xe2, 0xf3, 0xb5, 0x0b, 0x2a, 0x02, 0x76, 0xff, + 0xae, 0x0f, 0x1b, 0xa6, 0x37, 0x99, 0xb8, 0x7b, 0x7a, 0xa3, 0xdb, 0x36, 0xdf, 0xff, + 0xd6, 0xb0, 0xf9, 0xbb, 0x78, 0x78, 0xd7, 0xa7, 0x6c, 0x13, + ]; + + // Full plaintext RTCP packet + pub(super) const PLAINTEXT_RTCP_PACKET: &[u8] = &[ + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0x4e, 0x54, 0x50, 0x31, 0x4e, 0x54, + 0x50, 0x32, 0x52, 0x54, 0x50, 0x20, 0x00, 0x00, 0x04, 0x2a, 0x00, 0x00, 0xe9, 0x30, + 0x4c, 0x75, 0x6e, 0x61, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, + 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, + ]; + + /// Full encrypted RTCP packet + pub(super) const PROTECTED_RTCP_PACKET: &[u8] = &[ + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0xd5, 0x0a, 0xe4, 0xd1, 0xf5, 0xce, + 0x5d, 0x30, 0x4b, 0xa2, 0x97, 0xe4, 0x7d, 0x47, 0x0c, 0x28, 0x2c, 0x3e, 0xce, 0x5d, + 0xbf, 0xfe, 0x0a, 0x50, 0xa2, 0xea, 0xa5, 0xc1, 0x11, 0x05, 0x55, 0xbe, 0x84, 0x15, + 0xf6, 0x58, 0xc6, 0x1d, 0xe0, 0x47, 0x6f, 0x1b, 0x6f, 0xad, 0x1d, 0x1e, 0xb3, 0x0c, + 0x44, 0x46, 0x83, 0x9f, 0x57, 0xff, 0x6f, 0x6c, 0xb2, 0x6a, 0xc3, 0xbe, 0x80, 0x00, + 0x05, 0xd4, + ]; + + // A RTCP packet that hasn't been encrypted, only authenticated. + pub(super) const TAGGED_RTCP_PACKET: &[u8] = &[ + // RTCP Packet + 0x81, 0xc8, 0x00, 0x0d, 0x4d, 0x61, 0x72, 0x73, 0x4e, 0x54, 0x50, 0x31, 0x4e, 0x54, + 0x50, 0x32, 0x52, 0x54, 0x50, 0x20, 0x00, 0x00, 0x04, 0x2a, 0x00, 0x00, 0xe9, 0x30, + 0x4c, 0x75, 0x6e, 0x61, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, + 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, // + // Tag + 0x91, 0xdb, 0x4a, 0xfb, 0xfe, 0xee, 0x5a, 0x97, 0x8f, 0xab, 0x43, 0x93, 0xed, 0x26, + 0x15, 0xfe, // + // SRTCP Index + 0x00, 0x00, 0x05, 0xd4, + ]; + } + + #[test] + fn protect_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + + let header = + RtpHeader::parse(&rfc7714::PLAINTEXT_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let out = context.protect_rtp(rfc7714::PLAINTEXT_RTP_PACKET, &header, 0); + + assert_eq!( + out, + rfc7714::PROTECTED_RTP_PACKET, + "failed to encrypted packet.\n{:02x?}\n{:02x?}", + out, + &rfc7714::PLAINTEXT_RTP_PACKET + ); + } + + #[test] + fn unprotect_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + let header = + RtpHeader::parse(&rfc7714::PROTECTED_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + + let out = context + .unprotect_rtp(rfc7714::PROTECTED_RTP_PACKET, &header, 0) + .expect("decrypt rtp"); + + assert_eq!( + out, + &rfc7714::PLAINTEXT_RTP_PACKET[12..], + "failed to decrypt packet.\n{:02x?}\n{:02x?}", + out, + &rfc7714::PLAINTEXT_RTP_PACKET + ); + } + + #[test] + fn symmetry_rtp_rfc_7714_test() { + let mut context = make_rtp_context(); + + // First we encrypt + let header = + RtpHeader::parse(&rfc7714::PLAINTEXT_RTP_PACKET[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let encrypted = context.protect_rtp(rfc7714::PLAINTEXT_RTP_PACKET, &header, 0); + + // Then we decrypt the resulting cipher text + let header = RtpHeader::parse(&encrypted[..12], &ExtensionMap::empty()) + .expect("header to parse"); + let decrypted = context + .unprotect_rtp(&encrypted, &header, 0) + .expect("rtp unprotect"); + + // And verify we get the input back. + assert_eq!(decrypted, &rfc7714::PLAINTEXT_RTP_PACKET[12..]); + } + + #[test] + fn unprotect_rtp_should_fail_with_broken_tag_data() { + let mut context = make_rtp_context(); + + let header_buf = { + let mut buf = rfc7714::PROTECTED_RTP_PACKET[..12].to_vec(); + // Mess with part of the sequence number, since this makes up part of the + // authenticated additional data(AAD) the resulting authenticity tag should not + // match. + buf[3] ^= 0xFF; + + buf + }; + + let header = + RtpHeader::parse(&header_buf, &ExtensionMap::empty()).expect("header to parse"); + + let result = context.unprotect_rtp(rfc7714::PROTECTED_RTP_PACKET, &header, 0); + assert!( + result.is_none(), + "Should fail to decrypt a SRTP packet that has mismatched \ + authenicated additional data" + ); + } + + #[test] + fn unprotect_rtp_should_fail_with_broken_null_tag() { + let mut context = make_rtp_context(); + + let input = { + let mut input = rfc7714::PROTECTED_RTP_PACKET.to_vec(); + let len = input.len(); + input[len - TAG_LEN..].copy_from_slice(&[0; TAG_LEN]); + + input + }; + + let header = + RtpHeader::parse(&input[..12], &ExtensionMap::empty()).expect("header to parse"); + + let result = context.unprotect_rtp(&input, &header, 0); + assert!( + result.is_none(), + "Should fail to decrypt a SRTP packet with null tag" + ); + } + + #[test] + fn protect_rtcp_rfc_7714_test() { + let mut context = make_rtcp_context(); + + let out = context.protect_rtcp(rfc7714::PLAINTEXT_RTCP_PACKET); + + assert!( + out == rfc7714::PROTECTED_RTCP_PACKET, + "Expected encrypted and tagged RTCP packet:\n{:02x?}\nGot:\n{:02x?}", + rfc7714::PROTECTED_RTCP_PACKET, + out + ); + } + + #[test] + fn unprotect_rtcp_rfc_auth_only_7714_test() { + let mut context = make_rtcp_context(); + + let out = context + .unprotect_rtcp(rfc7714::TAGGED_RTCP_PACKET) + .expect("Unprotect RTCP"); + + assert_eq!(out, rfc7714::PLAINTEXT_RTCP_PACKET); + } + + fn make_rtp_context() -> SrtpContext { + crate::init_crypto_default(); + SrtpContext::new_aead_aes_256_gcm( + rfc7714::KEY, + rfc7714::SALT, + rfc7714::KEY, + rfc7714::SALT, + 0, + ) + } + + fn make_rtcp_context() -> SrtpContext { + crate::init_crypto_default(); + SrtpContext::new_aead_aes_256_gcm( + rfc7714::KEY, + rfc7714::SALT, + rfc7714::KEY, + rfc7714::SALT, + 0x000005d4, + ) + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/vla.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/vla.rs new file mode 100644 index 000000000..ed4eb50d9 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/rtp/vla.rs @@ -0,0 +1,1165 @@ +use std::collections::VecDeque; + +use super::{ExtensionSerializer, ExtensionValues}; + +#[allow(dead_code)] +/// URI for the Video Layers Allocation RTP Header Extension +pub const URI: &str = "http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00"; + +/// Top-level "allocation" for the Video Layers Allocation RTP Header Extension +/// Contains allocations for many simulcast streams, which contain many spatial layers. +/// In practice, there are either many simulcast streams with 1 spatial layer each (simulcast) +/// or 1 simulcast stream with many spatial layers (SVC) +/// or 1 simulcast stream with 1 spatial layer (only temporal layers used). +/// But theoretically, you could have 5 simulcast streams with 4 spatial layers each. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct VideoLayersAllocation { + /// The index of the current simulcast stream. + /// AKA RTP stream index + /// Set to 0 when everything is inactive (the special case of the header extension being just 0). + /// Erroneously called "RID" in the spec. + pub current_simulcast_stream_index: u8, + + /// AKA RTP streams + /// Max size of this Vec: 5 + pub simulcast_streams: Vec, +} + +/// An allocation for a simulcast stream, which may contain up to 4 allocations for spatial layers. +/// There may be up to 5 of these per top-level allocation. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct SimulcastStreamAllocation { + /// May contains many spatial layers, or none. + /// Max size of this Vec: 4 + pub spatial_layers: Vec, +} + +/// An allocation for a spatial layer, which may contain up to 5 allocations for temporal layers. +/// There may be up to 4 per simulcast stream. +/// Also contains an optional resolution and framerate. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct SpatialLayerAllocation { + /// Contains many temporal layers, or none. + /// If empty, the spatial layer is not active. + /// Max size of this Vec: 5 + pub temporal_layers: Vec, + /// Contains an optional resolution and framerate + pub resolution_and_framerate: Option, +} + +/// An allocation for a temporal layer. There may be up to 5 per spatial layer. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct TemporalLayerAllocation { + /// Cumulative bitrate for this temporal layer and all below it within a spatial layer. + pub cumulative_kbps: u64, +} + +/// A resolution and a frame rate, tied together because that's how it's formed in +/// the header extension. Either they are both there or neither are there. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ResolutionAndFramerate { + /// Width in number of pixels + pub width: u16, + /// Height in number of pixels + pub height: u16, + /// Framerate in frames per second + pub framerate: u8, +} + +impl VideoLayersAllocation { + #[allow(dead_code)] + fn parse(buf: &[u8]) -> Option { + // First byte + let (&b0, after_b0) = buf.split_first()?; + if b0 == 0u8 && after_b0.is_empty() { + // Special case when everything is inactive. + return Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![], + }); + } + let current_simulcast_stream_index = read_bits(b0, 0..2); + // Maximum of 5 simulcast streams + let simulcast_stream_count = read_bits(b0, 2..4) + 1; + let shared_spatial_layer_bitmask = read_bits(b0, 4..8); + + // Spatial layer bitmasks, which can be either "shared" or not. + // If shared, each simulcast stream as the same spatial layers active. + // If not, each simulcast stream has its own 4 bits indicating which spatial layers are active. + // So, a maximum of 4 spatial layers per simulcast stream. + let (spatial_layer_active_bits, after_spatial_layer_bitmasks) = + if shared_spatial_layer_bitmask > 0 { + let shared_spatial_layer_active_bits = + read_lower_4bits_ignoring_leading_zeros(shared_spatial_layer_bitmask); + let spatial_layer_active_bits = + vec![shared_spatial_layer_active_bits; simulcast_stream_count as usize]; + let after_spatial_layer_bitmasks = after_b0; + (spatial_layer_active_bits, after_spatial_layer_bitmasks) + } else { + // 4 bits per simulcast stream + let (spatial_layer_bitmasks, after_spatial_layer_bitmasks) = + split_at(after_b0, div_round_up(simulcast_stream_count as usize, 2))?; + let spatial_layer_actives = spatial_layer_bitmasks + .iter() + .flat_map(|&byte| split_byte_in2(byte)) + .take(simulcast_stream_count as usize) + .map(read_lower_4bits_ignoring_leading_zeros) + .collect(); + (spatial_layer_actives, after_spatial_layer_bitmasks) + }; + // The number of active bits that are set across all simulcast streams, + // which is the number of active spatial layers across all simulcast streams. + // A maximum of 5x4 = 20 spatial layers across all simulcast streams. + let total_active_spatial_layer_count = spatial_layer_active_bits + .iter() + .flatten() + .filter(|&&active| active) + .count(); + + // Temporal layer counts + // 2 bits per spatial layer + // A maximum of 5 temporal layers per spatial layer. + // For a maximum of 5x4x5 = 100 temporal layers across all spatial layers + // across all simulcast streams. + let (temporal_layer_counts, after_temporal_layer_counts) = split_at( + after_spatial_layer_bitmasks, + div_round_up(total_active_spatial_layer_count, 4), + )?; + // Max size of this VecDeque: 20 + let mut temporal_layer_counts: VecDeque = temporal_layer_counts + .iter() + .flat_map(|&byte| split_byte_in4(byte)) + .map(|count_minus_1| count_minus_1 + 1) + .take(total_active_spatial_layer_count) + .collect(); + let total_temporal_layer_count = temporal_layer_counts.iter().sum(); + + // Temporal layer bitrates + let mut next_temporal_layer_bitrate = after_temporal_layer_counts; + // Max size of this VecDeque: 100 + let mut temporal_layer_cumulative_bitrates: VecDeque = (0..total_temporal_layer_count) + .map(|_temporal_layer_index| { + let (bitrate, after_temporal_layer_bitrate) = + parse_leb_u63(next_temporal_layer_bitrate); + next_temporal_layer_bitrate = after_temporal_layer_bitrate; + bitrate + }) + .collect(); + // libwebrtc fails to parse at a value of 1_000_000 kbps. We are a little more forgiving, + // but since we limit the LEB parse at 63 bits, we should be at least that strict. + if temporal_layer_cumulative_bitrates + .iter() + .any(|&kbps| kbps > (1u64 << 63)) + { + return None; + } + + // (Optional) resolutions and framerates + let mut next_resolution_and_framerate = next_temporal_layer_bitrate; + let mut resolutions_and_framerates = + (0..total_active_spatial_layer_count).filter_map(|_| { + let (resolution_and_framerate, after_resolution_and_framerate) = + split_at(next_resolution_and_framerate, 5)?; + next_resolution_and_framerate = after_resolution_and_framerate; + Some(ResolutionAndFramerate { + width: u16::from_be_bytes(resolution_and_framerate[0..2].try_into().unwrap()) + + 1, + height: u16::from_be_bytes(resolution_and_framerate[2..4].try_into().unwrap()) + + 1, + framerate: resolution_and_framerate[4], + }) + }); + + let simulcast_streams = spatial_layer_active_bits + .into_iter() + .map(|spatial_layer_actives| { + let spatial_layers = spatial_layer_actives + .into_iter() + .filter_map(|spatial_layer_active| { + let (temporal_layers, resolution_and_framerate) = if spatial_layer_active { + let temporal_layer_count = temporal_layer_counts.pop_front()?; + let temporal_layers = (0..temporal_layer_count) + .filter_map(|_temporal_layer_index| { + Some(TemporalLayerAllocation { + cumulative_kbps: temporal_layer_cumulative_bitrates + .pop_front()?, + }) + }) + .collect(); + let resolution_and_framerate = resolutions_and_framerates.next(); + (temporal_layers, resolution_and_framerate) + } else { + (vec![], None) + }; + Some(SpatialLayerAllocation { + temporal_layers, + resolution_and_framerate, + }) + }) + .collect(); + SimulcastStreamAllocation { spatial_layers } + }) + .collect(); + Some(VideoLayersAllocation { + current_simulcast_stream_index, + simulcast_streams, + }) + } +} + +/// Serializer of the Video Layers Allocation Header Extension +#[derive(Debug)] +pub struct Serializer; + +impl ExtensionSerializer for Serializer { + // +-+-+-+-+-+-+-+-+ + // |RID| NS| sl_bm | + // +-+-+-+-+-+-+-+-+ + // Spatial layer bitmask |sl0_bm |sl1_bm | + // up to 2 bytes |---------------| + // when sl_bm == 0 |sl2_bm |sl3_bm | + // +-+-+-+-+-+-+-+-+ + // Number of temporal layers |#tl|#tl|#tl|#tl| + // per spatial layer | | | | | + // +-+-+-+-+-+-+-+-+ + // Target bitrate in kpbs | | + // per temporal layer : ... : + // leb128 encoded | | + // +-+-+-+-+-+-+-+-+ + // Resolution and framerate | | + // 5 bytes per spatial layer + width-1 for + + // (optional) | rid=0, sid=0 | + // +---------------+ + // | | + // + height-1 for + + // | rid=0, sid=0 | + // +---------------+ + // | max framerate | + // +-+-+-+-+-+-+-+-+ + // : ... : + // +-+-+-+-+-+-+-+-+ + + fn write_to(&self, buf: &mut [u8], ev: &ExtensionValues) -> usize { + if let Some(vla) = ev.user_values.get::() { + let mut index = 0; + + buf[index] = 0; + + if vla.current_simulcast_stream_index == 0 && vla.simulcast_streams.is_empty() { + return index + 1; + } + + // RID: RTP stream index this allocation is sent on, numbered from 0. 2 bits. + buf[index] |= (vla.current_simulcast_stream_index & 0b11) << 6; + // NS: Number of RTP streams minus one. 2 bits, thus allowing up-to 4 RTP streams. + buf[index] |= ((vla.simulcast_streams.len() - 1) as u8 & 0b11) << 4; + + // sl_bm: BitMask of the active Spatial Layers when same for all RTP streams or 0 otherwise. + // 4 bits, thus allows up to 4 spatial layers per RTP streams. + let total_spatial_layers = vla.simulcast_streams.len(); + let spatial_layers = vla.simulcast_streams.iter().enumerate().fold( + [0u8; 4], + |mut spatial_layers, (stream_index, stream)| { + let sl_bm = stream.spatial_layers.iter().enumerate().fold( + 0u8, + |is_active, (layer_id, l)| { + is_active | if l.temporal_layers.is_empty() { 0 } else { 1 } << layer_id + }, + ); + + spatial_layers[stream_index] = sl_bm; + spatial_layers + }, + ); + + let shared_spatial_layer_bitmask = spatial_layers[..total_spatial_layers] + .iter() + .all(|i| *i == spatial_layers[0]); + + if shared_spatial_layer_bitmask { + buf[index] |= spatial_layers[0] & 0b1111; + } else { + // slX_bm: BitMask of the active Spatial Layers for RTP stream with index=X. + // When NS < 2, takes one byte, otherwise uses two bytes. Zero-padded to byte alignment. + for (stream_index, sl_bm) in + spatial_layers[..total_spatial_layers].iter().enumerate() + { + let shift = if stream_index % 2 == 0 { 4 } else { 0 }; + if shift == 4 { + index += 1; + buf[index] = 0; + } + buf[index + (stream_index / 2)] |= (sl_bm & 0b1111) << shift; + } + + // When writing 1 or 3 entries, skip the remaining nibble to be byte aligned + if total_spatial_layers % 2 != 0 { + buf[index] |= 0b1111; + } + } + + index += 1; + + // #tl: 2-bit value of number of temporal layers-1, thus allowing up-to 4 temporal layers. + // Values are stored in ascending order of spatial id. Zero-padded to byte alignment. + let mut tl_index = 0; + let mut wrote_temporal_layer_count = false; + for s in &vla.simulcast_streams { + for spatial in &s.spatial_layers { + if !spatial.temporal_layers.is_empty() { + wrote_temporal_layer_count = true; + let temporal_layer_count_minus_one = + (spatial.temporal_layers.len() - 1) as u8; + if tl_index % 4 == 0 { + if tl_index > 0 { + index += 1; + } + buf[index] = 0; + } + + buf[index] |= temporal_layer_count_minus_one << (6 - (tl_index % 4) * 2); + tl_index += 1; + } + } + } + + if wrote_temporal_layer_count { + index += 1; + } else { + buf[index] = 0; + index += 1; + return index; + } + + for s in &vla.simulcast_streams { + for spatial in &s.spatial_layers { + for temporal in &spatial.temporal_layers { + index += encode_leb_u63(temporal.cumulative_kbps, &mut buf[index..]); + } + } + } + + for s in &vla.simulcast_streams { + for spatial in &s.spatial_layers { + if let Some(r) = &spatial.resolution_and_framerate { + let width = (r.width - 1).to_be_bytes(); + let height = (r.height - 1).to_be_bytes(); + let framerate = r.framerate; + buf[index..index + 2].copy_from_slice(&width[..]); + index += 2; + buf[index..index + 2].copy_from_slice(&height[..]); + index += 2; + buf[index] = framerate; + index += 1; + } + } + } + + return index; + } + 0 + } + + fn parse_value(&self, buf: &[u8], ev: &mut ExtensionValues) -> bool { + let Some(vla) = VideoLayersAllocation::parse(buf) else { + return false; + }; + ev.user_values.set(vla); + true + } + + fn is_video(&self) -> bool { + true + } + + fn is_audio(&self) -> bool { + false + } + + fn requires_two_byte_form(&self, _ev: &ExtensionValues) -> bool { + true + } +} + +// See https://en.wikipedia.org/wiki/LEB128 +// Reads out at most 9 bytes (63 bits) unsigned +// returns (value, rest) +// libwebrtc reads out all 64 bits, but then fails the parse if the value +// is over 1_000_000 anyway, so reading 63 bits should be enough as long as +// we throw away the parse if it's above 1_000_000. +#[allow(dead_code)] +fn parse_leb_u63(bytes: &[u8]) -> (u64, &[u8]) { + let mut result = 0; + for (index, &byte) in bytes.iter().enumerate() { + let is_last = !read_bit(byte, 0); + let chunk = read_bits(byte, 1..8); + result |= (chunk as u64) << (7 * index); + if is_last || index == 8 { + return (result, &bytes[(index + 1)..]); + } + } + (0, bytes) +} + +/// Encodes leb128 +pub fn encode_leb_u63(mut value: u64, buf: &mut [u8]) -> usize { + let mut index = 0; + loop { + if value < 0x80 { + buf[index] = value as u8; + index += 1; + break; + } else { + buf[index] = ((value & 0x7f) | 0x80) as u8; + value >>= 7; + index += 1; + } + } + + index +} + +// If successful, the size of the left will be mid, +// and the size of the right while be buf.len()-mid. +#[allow(dead_code)] +fn split_at(buf: &[u8], mid: usize) -> Option<(&[u8], &[u8])> { + if mid > buf.len() { + return None; + } + Some(buf.split_at(mid)) +} + +#[allow(dead_code)] +fn div_round_up(top: usize, bottom: usize) -> usize { + if top == 0 { + 0 + } else { + ((top - 1) / bottom) + 1 + } +} + +// Into 2 chunks of 4 bits +#[allow(dead_code)] +fn split_byte_in2(byte: u8) -> [u8; 2] { + [read_bits(byte, 0..4), read_bits(byte, 4..8)] +} + +// Into 4 chunks of 2 bits +#[allow(dead_code)] +fn split_byte_in4(byte: u8) -> [u8; 4] { + [ + read_bits(byte, 0..2), + read_bits(byte, 2..4), + read_bits(byte, 4..6), + read_bits(byte, 6..8), + ] +} + +// Ignore top 4 bits and leading zeros. Then split into a Vecca +fn read_lower_4bits_ignoring_leading_zeros(bits: u8) -> Vec { + let mut count = 0; + let mut bools: Vec = (0..=3u8) + .map(|index| { + let bit = read_bit(bits, 7 - index); + if bit { + count = index + 1; + } + bit + }) + .collect(); + bools.truncate(count as usize); + bools +} + +#[allow(dead_code)] +fn read_bit(bits: u8, index: u8) -> bool { + read_bits(bits, index..(index + 1)) > 0 +} + +#[allow(dead_code)] +fn read_bits(bits: u8, range: std::ops::Range) -> u8 { + assert!(range.end <= 8); + (bits >> (8 - range.end)) & (0b1111_1111 >> (8 - range.len())) +} + +#[cfg(test)] +mod test { + use super::*; + + fn serialize(vla: Option<&VideoLayersAllocation>) -> Vec { + let Some(vla) = vla else { return Vec::new() }; + let mut buf: [u8; 100] = [0u8; 100]; + let mut ext_values: ExtensionValues = Default::default(); + ext_values.user_values.set(vla.clone()); + + let actual_size = Serializer {}.write_to(&mut buf, &ext_values); + + buf[..actual_size].to_vec() + } + + fn assert_ser_deser(bytes: &[u8], vla: Option) { + let vla_deserialized = VideoLayersAllocation::parse(bytes); + let vla_serialized = serialize(vla.as_ref()); + + assert_eq!(vla, vla_deserialized); + assert_eq!(bytes, vla_serialized); + } + + #[test] + fn test_read_bits() { + assert_eq!(read_bits(0b1100_0000, 0..2), 0b0000_0011); + assert_eq!(read_bits(0b1001_0101, 0..2), 0b0000_0010); + assert_eq!(read_bits(0b0110_1010, 0..2), 0b0000_0001); + assert_eq!(read_bits(0b0011_1111, 0..2), 0b0000_0000); + assert_eq!(read_bits(0b0011_0000, 2..4), 0b0000_0011); + assert_eq!(read_bits(0b0110_0101, 2..4), 0b0000_0010); + assert_eq!(read_bits(0b1001_1010, 2..4), 0b0000_0001); + assert_eq!(read_bits(0b1100_1111, 2..4), 0b0000_0000); + } + + #[test] + fn test_parse_leb_u63() { + let (value, rest) = parse_leb_u63(&[0b0000_0000, 5]); + assert_eq!(0, value); + assert_eq!(&[5], rest); + + let (value, rest) = parse_leb_u63(&[0b0000_0001, 5]); + assert_eq!(1, value); + assert_eq!(&[5], rest); + + let (value, rest) = parse_leb_u63(&[0b1000_0000, 0b0000_0001, 5]); + assert_eq!(128, value); + assert_eq!(&[5], rest); + + let (value, rest) = parse_leb_u63(&[0b1000_0000, 0b1000_0000, 0b0000_0001, 5]); + assert_eq!(16384, value); + assert_eq!(&[5], rest); + + let (value, rest) = parse_leb_u63(&[0b1000_0000, 0b1000_0000, 0b1000_0000, 0b0000_0001, 5]); + assert_eq!(2097152, value); + assert_eq!(&[5], rest); + + let (value, rest) = parse_leb_u63(&[ + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b0000_0001, + 5, + ]); + assert_eq!(72057594037927936, value); + assert_eq!(&[5], rest); + + // Too many bytes, so stop early. + let (value, rest) = parse_leb_u63(&[ + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0000, + 0b1000_0001, + 5, + ]); + assert_eq!(72057594037927936, value); + assert_eq!(&[5], rest); + } + + #[test] + fn test_parse_vla_empty_buffer() { + assert_ser_deser(&[], None); + } + + #[test] + fn test_parse_vla_empty() { + assert_ser_deser( + &[0b0000_0000], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![], + }), + ); + } + + #[test] + fn test_res() { + let vla = VideoLayersAllocation::parse(&[ + 17, 111, 7, 0, 15, 92, 4, 255, 2, 207, 30, 7, 127, 4, 55, 30, + ]); + assert!(vla.is_some()) + } + + #[test] + fn test_parse_vla_missing_spatial_layer_bitmasks() { + assert_eq!(VideoLayersAllocation::parse(&[0b0110_0000]), None); + } + + #[test] + fn test_parse_vla_1_simulcast_stream_with_no_active_layers() { + assert_ser_deser( + &[ + 0b0100_0000, + // 1 bitmask + 0b0000_0000, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 1, + simulcast_streams: vec![SimulcastStreamAllocation { + spatial_layers: vec![], + }], + }), + ); + } + + #[test] + fn test_parse_vla_3_simulcast_streams_with_no_active_layers() { + assert_eq!( + VideoLayersAllocation::parse(&[ + 0b0110_0000, + // 3 active spatial layer bitmasks, 4 bits each + 0b0000_0000, + 0b0000_1111, + ]), + Some(VideoLayersAllocation { + current_simulcast_stream_index: 1, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![], + }, + SimulcastStreamAllocation { + spatial_layers: vec![], + }, + SimulcastStreamAllocation { + spatial_layers: vec![], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_3_simulcast_streams_with_1_active_spatial_layers_and_2_temporal_layers() { + assert_ser_deser( + &[ + 0b0110_0001, + // 3 temporal layer counts (minus 1), 2 bits each + 0b0101_0100, + // 6 temporal layer bitrates + 0b0000_0001, + 0b0000_0010, + 0b0000_0100, + 0b0000_1000, + 0b0001_0000, + 0b0010_0000, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 1, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { cumulative_kbps: 1 }, + TemporalLayerAllocation { cumulative_kbps: 2 }, + ], + resolution_and_framerate: None, + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { cumulative_kbps: 4 }, + TemporalLayerAllocation { cumulative_kbps: 8 }, + ], + resolution_and_framerate: None, + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 16, + }, + TemporalLayerAllocation { + cumulative_kbps: 32, + }, + ], + resolution_and_framerate: None, + }], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_3_sim_1_spatial_2_temporal_with_resolutions() { + assert_ser_deser( + &[ + 0b0110_0001, + // 3 temporal layer counts (minus 1), 2 bits each + 0b0101_0100, + // 6 temporal layer bitrates + 100, + 101, + 110, + 111, + 120, + 121, + // 3 resolutions + framerates (5 bytes each) + // 320x180x15 + 1, + 63, + 0, + 179, + 15, + // 640x360x30 + 2, + 127, + 1, + 103, + 30, + // 1280x720x60 + 4, + 255, + 2, + 207, + 60, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 1, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: Some(ResolutionAndFramerate { + width: 320, + height: 180, + framerate: 15, + }), + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 110, + }, + TemporalLayerAllocation { + cumulative_kbps: 111, + }, + ], + resolution_and_framerate: Some(ResolutionAndFramerate { + width: 640, + height: 360, + framerate: 30, + }), + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 120, + }, + TemporalLayerAllocation { + cumulative_kbps: 121, + }, + ], + resolution_and_framerate: Some(ResolutionAndFramerate { + width: 1280, + height: 720, + framerate: 60, + }), + }], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_3_simulcast_streams_with_differing_active_spatial_layers_with_resolutions() { + assert_ser_deser( + &[ + 0b0010_0000, + // 3 active spatial layer bitmasks, 4 bits each; only the base layer is active + 0b0001_0000, + 0b0000_1111, + // 1 temporal layer counts (minus 1), 2 bits each + 0b0100_0000, + // 2 temporal layer bitrates + 100, + 101, + // 1 resolutions + framerates (5 bytes) + // 320x180x15 + 1, + 63, + 0, + 179, + 15, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: Some(ResolutionAndFramerate { + width: 320, + height: 180, + framerate: 15, + }), + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![], + }, + SimulcastStreamAllocation { + spatial_layers: vec![], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_1_simulcast_streams_with_3_spatial_layers() { + assert_ser_deser( + &[ + 0b0000_0111, + // 3 temporal layer counts (minus 1), 2 bits each + 0b0101_0100, + // 6 temporal layer bitrates + 100, + 101, + 110, + 111, + 120, + 121, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![SimulcastStreamAllocation { + spatial_layers: vec![ + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 110, + }, + TemporalLayerAllocation { + cumulative_kbps: 111, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 120, + }, + TemporalLayerAllocation { + cumulative_kbps: 121, + }, + ], + resolution_and_framerate: None, + }, + ], + }], + }), + ); + } + + #[test] + fn test_parse_vla_4_simulcast_streams_with_1_spatial_layer_each() { + assert_ser_deser( + &[ + 0b0011_0001, + // 4 temporal layer counts (minus 1), 2 bits each: all have 2 temporal layers + 0b0101_0101, + // 8 temporal layer bitrates + // 100, 101, 110, 111 are single-byte LEB128 + 100, + 101, + 110, + 111, + // 130 = LEB128 [0x82, 0x01], 131 = LEB128 [0x83, 0x01] + 0x82, + 0x01, + 0x83, + 0x01, + // 200 = LEB128 [0xC8, 0x01], 201 = LEB128 [0xC9, 0x01] + 0xC8, + 0x01, + 0xC9, + 0x01, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: None, + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 110, + }, + TemporalLayerAllocation { + cumulative_kbps: 111, + }, + ], + resolution_and_framerate: None, + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 130, + }, + TemporalLayerAllocation { + cumulative_kbps: 131, + }, + ], + resolution_and_framerate: None, + }], + }, + SimulcastStreamAllocation { + spatial_layers: vec![SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 200, + }, + TemporalLayerAllocation { + cumulative_kbps: 201, + }, + ], + resolution_and_framerate: None, + }], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_2_simulcast_streams_with_3_spatial_layers_each() { + assert_ser_deser( + &[ + 0b0001_0111, + // 6 temporal layer counts (minus 1), 2 bits each: all have 2 temporal layers + // First byte: 4 counts + 0b0101_0101, + // Second byte: 2 counts + zero padding + 0b0101_0000, + // 12 temporal layer bitrates + // Stream 1: 100, 101, 110, 111, 120, 121 are single-byte LEB128 + 100, + 101, + 110, + 111, + 120, + 121, + // Stream 2: values >= 128, each needs 2-byte LEB128 + // 200 = [0xC8, 0x01], 201 = [0xC9, 0x01] + 0xC8, + 0x01, + 0xC9, + 0x01, + // 210 = [0xD2, 0x01], 211 = [0xD3, 0x01] + 0xD2, + 0x01, + 0xD3, + 0x01, + // 220 = [0xDC, 0x01], 221 = [0xDD, 0x01] + 0xDC, + 0x01, + 0xDD, + 0x01, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![ + SimulcastStreamAllocation { + spatial_layers: vec![ + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 110, + }, + TemporalLayerAllocation { + cumulative_kbps: 111, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 120, + }, + TemporalLayerAllocation { + cumulative_kbps: 121, + }, + ], + resolution_and_framerate: None, + }, + ], + }, + SimulcastStreamAllocation { + spatial_layers: vec![ + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 200, + }, + TemporalLayerAllocation { + cumulative_kbps: 201, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 210, + }, + TemporalLayerAllocation { + cumulative_kbps: 211, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 220, + }, + TemporalLayerAllocation { + cumulative_kbps: 221, + }, + ], + resolution_and_framerate: None, + }, + ], + }, + ], + }), + ); + } + + #[test] + fn test_parse_vla_1_simulcast_streams_with_4_spatial_layers_1_inactive() { + assert_ser_deser( + &[ + 0b0000_1011, + // 3 temporal layer counts (minus 1), 2 bits each + 0b0101_0100, + // 6 temporal layer bitrates + 100, + 101, + 110, + 111, + 120, + 121, + ], + Some(VideoLayersAllocation { + current_simulcast_stream_index: 0, + simulcast_streams: vec![SimulcastStreamAllocation { + spatial_layers: vec![ + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 100, + }, + TemporalLayerAllocation { + cumulative_kbps: 101, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 110, + }, + TemporalLayerAllocation { + cumulative_kbps: 111, + }, + ], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![], + resolution_and_framerate: None, + }, + SpatialLayerAllocation { + temporal_layers: vec![ + TemporalLayerAllocation { + cumulative_kbps: 120, + }, + TemporalLayerAllocation { + cumulative_kbps: 121, + }, + ], + resolution_and_framerate: None, + }, + ], + }], + }), + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/dcep.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/dcep.rs new file mode 100644 index 000000000..18724578d --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/dcep.rs @@ -0,0 +1,122 @@ +use std::str::from_utf8; + +use sctp_proto::ReliabilityType; + +use super::SctpError as Error; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Message Type | Channel Type | Priority | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reliability Parameter | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Label Length | Protocol Length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// \ / +// | Label | +// / \ +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// \ / +// | Protocol | +// / \ +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +/// Data channel establishment protocol. +/// RFC 8832 +#[derive(Clone, Debug)] +pub struct DcepOpen { + pub unordered: bool, + pub channel_type: ReliabilityType, + pub priority: u16, + pub reliability_parameter: u32, + pub label: String, + pub protocol: String, +} + +impl TryFrom<&[u8]> for DcepOpen { + type Error = Error; + + fn try_from(buf: &[u8]) -> Result { + if buf.len() < 12 { + return Err(Error::DcepOpenTooSmall); + } + + if buf[0] != 0x03 { + return Err(Error::DcepIncorrectMessageType); + } + + let unordered = buf[1] & 0x80 > 0; + let channel_type: ReliabilityType = (buf[1] & 0x7f).into(); + let priority = u16::from_be_bytes([buf[2], buf[3]]); + let reliability_parameter = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); + let len_label = u16::from_be_bytes([buf[8], buf[9]]) as usize; + let len_proto = u16::from_be_bytes([buf[10], buf[11]]) as usize; + + if buf.len() < 12 + len_label + len_proto { + return Err(Error::DcepOpenTooSmall); + } + + let buf = &buf[12..]; + let label = from_utf8(&buf[0..len_label]) + .map_err(|_| Error::DcepBadUtf8)? + .to_string(); + + let buf = &buf[len_label..]; + let protocol = from_utf8(&buf[0..len_proto]) + .map_err(|_| Error::DcepBadUtf8)? + .to_string(); + + Ok(DcepOpen { + unordered, + channel_type, + priority, + reliability_parameter, + label, + protocol, + }) + } +} + +impl DcepOpen { + pub fn marshal_to(&self, buf: &mut [u8]) -> usize { + buf[0] = 0x03; + buf[1] = (self.channel_type as u8) | if self.unordered { 0x80 } else { 0x00 }; + buf[2..4].copy_from_slice(&self.priority.to_be_bytes()); + buf[4..8].copy_from_slice(&self.reliability_parameter.to_be_bytes()); + let bytes_label = self.label.as_bytes(); + let bytes_proto = self.protocol.as_bytes(); + buf[8..10].copy_from_slice(&(bytes_label.len() as u16).to_be_bytes()); + buf[10..12].copy_from_slice(&(bytes_proto.len() as u16).to_be_bytes()); + let buf = &mut buf[12..]; + buf[..bytes_label.len()].copy_from_slice(bytes_label); + let buf = &mut buf[bytes_label.len()..]; + buf[..bytes_proto.len()].copy_from_slice(bytes_proto); + 12 + bytes_label.len() + bytes_proto.len() + } +} + +pub struct DcepAck; + +impl TryFrom<&[u8]> for DcepAck { + type Error = Error; + + fn try_from(buf: &[u8]) -> Result { + if buf.is_empty() { + return Err(Error::DcepOpenTooSmall); + } + + if buf[0] != 0x02 { + return Err(Error::DcepIncorrectMessageType); + } + + Ok(DcepAck) + } +} + +impl DcepAck { + pub fn marshal_to(&self, buf: &mut [u8]) -> usize { + buf[0] = 0x02; + 1 + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/error.rs new file mode 100644 index 000000000..0cee3ad1e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/error.rs @@ -0,0 +1,55 @@ +use sctp_proto::Error as ProtoError; +use std::error::Error; +use std::fmt; + +/// Errors from the SCTP subsystem. +#[derive(Debug, Eq, Clone, PartialEq)] +pub enum SctpError { + /// Some protocol error as wrapped from the sctp_proto crate. + Proto(ProtoError), + + /// Stream was not ready and we tried to write. + WriteBeforeEstablished, + + /// The initial DCEP is not valid. + DcepOpenTooSmall, + + /// The initial DCEP is not the correct message type. + DcepIncorrectMessageType, + + /// The initial DCEP cant be read as utf-8. + DcepBadUtf8, + + /// Invalid SNAP token in SCTP INIT string. + InvalidSnap, +} + +impl fmt::Display for SctpError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SctpError::Proto(err) => write!(f, "{}", err), + SctpError::WriteBeforeEstablished => { + write!(f, "Write on a stream before it was established") + } + SctpError::DcepOpenTooSmall => write!(f, "DCEP open message too small"), + SctpError::DcepIncorrectMessageType => write!(f, "DCEP incorrect message type"), + SctpError::DcepBadUtf8 => write!(f, "DCEP bad UTF-8 string"), + SctpError::InvalidSnap => write!(f, "Invalid SNAP token"), + } + } +} + +impl Error for SctpError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + SctpError::Proto(err) => Some(err), + _ => None, + } + } +} + +impl From for SctpError { + fn from(err: ProtoError) -> Self { + SctpError::Proto(err) + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/mod.rs new file mode 100644 index 000000000..6df1d75a7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/mod.rs @@ -0,0 +1,1664 @@ +#![allow(clippy::new_without_default)] + +use std::collections::{HashSet, VecDeque}; +use std::fmt; +use std::net::SocketAddr; +use std::panic::UnwindSafe; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sctp_proto::{Association, AssociationHandle, DatagramEvent}; +use sctp_proto::{Endpoint, EndpointConfig, Stream, StreamEvent, Transmit}; +use sctp_proto::{Event, Payload, PayloadProtocolIdentifier, ServerConfig, TransportConfig}; + +use snap::{b64_encode, webrtc_transport_config}; + +pub use sctp_proto::Error as ProtoError; +use sctp_proto::ReliabilityType; + +mod snap; +pub use snap::SctpInitData; + +mod dcep; +use dcep::DcepAck; +use dcep::DcepOpen; + +mod error; +pub use error::SctpError; + +/// Bytes that can be buffered inside str0m across all streams. +const MAX_BUFFERED_ACROSS_STREAMS: usize = 128 * 1024; + +/// Maximum message size we advertise in SDP (what we can receive) +pub const LOCAL_MAX_MESSAGE_SIZE: u32 = 256 * 1024; + +/// Default max message size if remote doesn't advertise +pub const DEFAULT_REMOTE_MAX_MESSAGE_SIZE: u32 = 64 * 1024; + +pub(crate) struct RtcSctp { + state: RtcSctpState, + endpoint: Endpoint, + fake_addr: SocketAddr, + handle: AssociationHandle, + assoc: Option, + entries: Vec, + // Stream ids that still owe a `StreamEvent::ResetComplete`, + // thos ids needs to be held back from reuse. + reset_pending: HashSet, + // Completed resets awaiting delivery to the caller. + // Used to guarantee emission ordering, ResetComplete must + // be sent after Close. + reset_complete: VecDeque, + pushed_back_transmit: Option>>, + last_now: Instant, + client: bool, + remote_max_message_size: u32, + snap_enabled: bool, + snap_init: Option, + #[cfg(test)] + max_payload_size: usize, +} + +/// This is okay because there is no way for a user of Rtc to interact with the Sctp subsystem +/// in a way that would allow them to observe a potentially broken invariant when catching a panic. +impl UnwindSafe for RtcSctp {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RtcSctpState { + Uninited, + AwaitRemoteAssociation, + AwaitAssociationEstablished, + Established, +} + +impl RtcSctpState { + pub fn propagate_endpoint_to_assoc(&self) -> bool { + matches!( + self, + RtcSctpState::AwaitAssociationEstablished | RtcSctpState::Established + ) + } +} + +#[derive(Debug)] +struct StreamEntry { + /// Config as provided when opening the channel. This is None if we discover + /// the channel from the remote peer before getting a DcepOpen or local open_stream. + config: Option, + /// Current state + state: StreamEntryState, + /// Actual stream id. Negotiated or automatically allocated. + id: u16, + /// If we are to close this entry. + do_close: bool, + /// Deadline for retrying `open_stream` when it fails. + open_deadline: Option, + /// If the queued outgoing data drops below this threshold, Rtc is to emit an + /// event to the user. + buffered_threshold: BufferedThresholdConfig, +} + +#[derive(Debug)] +/// Tracks the `buffered_amount_low_threshold` for a stream. +/// +/// Lets us defer applying user changes to the underlying SCTP +/// stream until the next poll cycle, without first querying the current +/// configured value. +enum BufferedThresholdConfig { + /// No threshold has been set or it was cleared after an error. + Unconfigured, + /// A user-requested threshold to apply on the next poll. + Desired(usize), + /// The threshold value currently configured in the SCTP stream. + Configured(usize), +} + +impl BufferedThresholdConfig { + pub fn set(&mut self, v: usize) { + let is_change = match self { + BufferedThresholdConfig::Unconfigured => true, + BufferedThresholdConfig::Desired(w) if v != *w => true, + BufferedThresholdConfig::Configured(x) if v != *x => true, + _ => false, + }; + if is_change { + *self = BufferedThresholdConfig::Desired(v); + } + } +} + +pub(crate) enum SctpEvent { + Transmit { + packets: VecDeque>, + }, + Open { + id: u16, + label: String, + }, + Close { + id: u16, + /// Whether a reset handshake is outstanding for this stream id. + reset_pending: bool, + }, + /// The reset handshake for a closed stream completed and sctp-proto emitted + /// StreamEvent::ResetComplete. The stream id is now safe to be used again. + /// + /// Always reported after the `Close` for the same id. + StreamResetComplete { + id: u16, + }, + Data { + id: u16, + binary: bool, + data: Vec, + }, + BufferedAmountLow { + id: u16, + }, + AssociationLost, +} + +/// These are the possible paths: +/// ```text +/// local inited, in-band AwaitOpen -> AwaitDcepAck -> Open +/// local inited, out-of-band AwaitOpen -> Open +/// remote inited, in-band AwaitConfig -> (receive dcep) -> Open +/// remote inited, out-of-band AwaitConfig -> (open_stream) -> Open +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StreamEntryState { + /// A new stream declared locally, not discovered from remote. + AwaitOpen, + /// A new stream, discovered from remote. It can either be in-band or out-of band + /// We will either receive DcepOpen in-band, or a open_stream() call out-of-band. + AwaitConfig, + /// If we have sent DcepOpen and are waiting for the ack. + AwaitDcepAck, + /// Stream is open, ready to send data. + Open, + /// If some error occurs. + Closed, +} + +/// (Low level) configuration for a data channel. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ChannelConfig { + /// The label to use for the user to identify the channel. + pub label: String, + /// Whether channel is guaranteed ordered delivery of messages. + pub ordered: bool, + /// The reliability setting, which can allow to drop messages. + pub reliability: Reliability, + /// Whether channel is negotiated in-band (DCEP) or out-of-band. + /// None means in-band negotiated. Some(stream_id) means out-of-band. + pub negotiated: Option, + /// Protocol name. + /// + /// Defaults to "" + pub protocol: String, +} + +impl Default for ChannelConfig { + fn default() -> Self { + Self { + label: Default::default(), + ordered: true, + reliability: Default::default(), + negotiated: Default::default(), + protocol: Default::default(), + } + } +} + +/// Reliability setting of a data channel. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Reliability { + /// Packets are delivered in order, with retransmits. + #[default] + Reliable, + /// Packets delivered out of order with a max lifetime. + MaxPacketLifetime { + /// The lifetime of a packet in milliseconds. + lifetime: u16, + }, + /// Packets delivered out of order with a max number of retransmits. + MaxRetransmits { + /// Number of retransmits before giving up. + retransmits: u16, + }, +} + +impl StreamEntry { + fn set_state(&mut self, state: StreamEntryState) -> bool { + if self.state == state { + return false; + } + debug!("Stream {:?} -> {:?}", self.state, state); + self.state = state; + true + } + + #[must_use] + fn configure_reliability(&mut self, stream: &mut Stream) -> bool { + let dcep: DcepOpen = self.config.as_ref().expect("config to be set").into(); + + let ret = stream.set_reliability_params( + dcep.unordered, + dcep.channel_type, + dcep.reliability_parameter, + ); + + if let Err(e) = ret { + warn!( + "Failed to set reliability params on stream {}: {:?}", + self.id, e + ); + self.do_close = true; + return false; + } + + true + } +} + +/// SCTP framing per outgoing datagram: 12-byte common header + ~16-byte DATA +/// chunk header + some extra. +const SCTP_OVERHEAD: usize = 40; + +/// How long `open_stream` keeps retrying when blocked by a transient error +/// which could happen during reset handshake (RFC 6525). +/// Past this the peer is considered broken and the open gives up. +const STREAM_OPEN_TIMEOUT: Duration = Duration::from_secs(10); + +/// Backstop poll interval while an open is retrying. Retries are normally +/// driven by the network input that unblocks them, +/// either the reciprocal reset or RECONFIG-RESPONSE arriving. +/// This interval guarantees progress in case of absence of the latter messages. +const STREAM_OPEN_RETRY_INTERVAL: Duration = Duration::from_millis(500); + +/// Empirical: SCTP `max_payload_size` of 1200 has produced 1277-byte DTLS-wrapped +/// datagrams (77 bytes combined SCTP + DTLS overhead). +const _: () = assert!( + crate::io::MAX_DTLS_OVERHEAD + SCTP_OVERHEAD >= 80, + "MAX_DTLS_OVERHEAD + SCTP_OVERHEAD must cover observed 77-byte SCTP-over-DTLS overhead" +); + +impl RtcSctp { + pub fn new(mtu: usize) -> Self { + let mut config = EndpointConfig::default(); + let max_payload = mtu + .saturating_sub(crate::io::MAX_DTLS_OVERHEAD) + .saturating_sub(SCTP_OVERHEAD); + config.max_payload_size(max_payload as u32); + #[cfg(test)] + let max_payload_size = max_payload; + let mut server_config = ServerConfig::default(); + server_config.transport = webrtc_transport_config(); + let endpoint = Endpoint::new(Arc::new(config), Some(Arc::new(server_config))); + let fake_addr = "1.1.1.1:5000".parse().unwrap(); + + RtcSctp { + state: RtcSctpState::Uninited, + endpoint, + fake_addr, + handle: AssociationHandle(0), // temporary + assoc: None, + entries: vec![], + reset_pending: HashSet::new(), + reset_complete: VecDeque::new(), + pushed_back_transmit: None, + last_now: Instant::now(), // placeholder until init() + client: false, + remote_max_message_size: DEFAULT_REMOTE_MAX_MESSAGE_SIZE, + snap_enabled: false, + snap_init: None, + #[cfg(test)] + max_payload_size, + } + } + + #[cfg(test)] + pub(crate) fn max_payload_size(&self) -> usize { + self.max_payload_size + } + + pub fn is_inited(&self) -> bool { + self.state != RtcSctpState::Uninited + } + + pub fn is_closing(&self) -> bool { + self.assoc.as_ref().is_some_and(|a| a.is_closing()) + } + + pub fn is_closed(&self) -> bool { + self.assoc.as_ref().is_none_or(|a| a.is_closed()) + } + + pub fn init( + &mut self, + client: bool, + now: Instant, + sctp_init_data: Option, + remote_max_message_size: Option, + ) -> Result<(), SctpError> { + if self.state != RtcSctpState::Uninited { + return Err(SctpError::Proto(ProtoError::Other( + "SCTP already initialized".into(), + ))); + } + + self.client = client; + self.last_now = now; + + if let Some(max_msg_size) = remote_max_message_size { + self.remote_max_message_size = max_msg_size; + } + + if let Some(snap_data) = sctp_init_data { + // SNAP path: both local and remote INIT chunks must be present. + if snap_data.local_init.is_none() || snap_data.remote_init.is_none() { + return Err(SctpError::Proto(ProtoError::Other( + "SNAP requires both local and remote SCTP INIT chunks".into(), + ))); + } + + let config = snap_data.into_client_config(); + debug!( + "New {} association (out-of-band: true)", + if client { "local" } else { "server" }, + ); + let (handle, mut assoc) = self + .endpoint + .connect(config, self.fake_addr) + .map_err(|e| SctpError::Proto(ProtoError::Other(e.to_string())))?; + assoc.set_max_send_message_size(self.remote_max_message_size); + self.handle = handle; + self.assoc = Some(assoc); + + // With SNAP, both sides exchanged INIT chunks out-of-band. The + // sctp-proto association is already in established state (via + // `with_snap`). We set our state to Established immediately + // even though DTLS may not be connected yet. This is safe + // because the `dtls_connected` guard in `do_poll_output` + // prevents any SCTP packets from flowing until the DTLS + // handshake completes. + set_state(&mut self.state, RtcSctpState::Established); + } else if client { + // Normal client path: initiate the SCTP association. + let mut config = SctpInitData::default().into_client_config(); + + config.transport = Arc::new( + TransportConfig::default() + .with_max_init_retransmits(None) + .with_max_data_retransmits(None) + .with_max_receive_message_size(LOCAL_MAX_MESSAGE_SIZE) + .with_max_send_message_size(self.remote_max_message_size), + ); + + debug!("New local association (out-of-band: false)"); + let (handle, assoc) = self + .endpoint + .connect(config, self.fake_addr) + .map_err(|e| SctpError::Proto(ProtoError::Other(e.to_string())))?; + self.handle = handle; + self.assoc = Some(assoc); + set_state(&mut self.state, RtcSctpState::AwaitAssociationEstablished); + } else { + // Normal server path: wait for the remote to initiate. + set_state(&mut self.state, RtcSctpState::AwaitRemoteAssociation); + } + + Ok(()) + } + + pub fn is_client(&self) -> bool { + self.client + } + + /// Enable SNAP by pre-populating the init data. + pub fn enable_snap(&mut self) { + self.snap_enabled = true; + self.snap_init.get_or_insert_with(SctpInitData::new); + } + + /// Whether local offers should opt in to SNAP. + pub fn snap_enabled(&self) -> bool { + self.snap_enabled + } + + /// Ensure the local SNAP INIT chunk is generated. Returns `false` if + /// generation failed (degrades to non-SNAP). + pub fn ensure_local_snap_init(&mut self) -> bool { + let init_data = self.snap_init.get_or_insert_with(SctpInitData::new); + if init_data.local_init_chunk().is_err() { + self.snap_init = None; + false + } else { + true + } + } + + /// Discard pending SNAP negotiation state before SCTP starts. + /// + /// This preserves local opt-in for future offers. + pub fn disable_pending_snap(&mut self) { + if !self.is_inited() { + self.snap_init = None; + } + } + + /// Get the local INIT chunk as a base64 string for SDP, if applicable. + /// + /// Returns `None` when: + /// - SNAP is not active + /// - SCTP is established without SNAP (non-SNAP association) + pub fn local_sctp_init_for_sdp(&self) -> Option { + let d = self.snap_init.as_ref()?; + if self.is_inited() && d.remote_init.is_none() { + // Established non-SNAP association - MUST NOT inject sctp-init. + return None; + } + d.local_init.as_ref().map(|b| b64_encode(b)) + } + + /// Get the cached remote INIT string, if set. + pub fn snap_remote_init_string(&self) -> Option { + self.snap_init.as_ref().and_then(|d| d.remote_init_string()) + } + + /// Whether this is an established SNAP association (has remote init). + pub fn is_snap_established(&self) -> bool { + self.is_inited() + && self + .snap_init + .as_ref() + .and_then(|d| d.remote_init.as_ref()) + .is_some() + } + + /// Set the remote SNAP INIT from a base64 string. Returns `Ok(true)` if + /// accepted, `Ok(false)` on decode error (degrades to non-SNAP). + pub fn set_remote_snap_init_string(&mut self, value: &str) -> bool { + let init_data = self.snap_init.get_or_insert_with(SctpInitData::new); + match init_data.set_remote_init_string(value) { + Ok(()) => true, + Err(_) => { + self.disable_pending_snap(); + false + } + } + } + + /// Build a cloned `SctpInitData` for passing to `init()`, if both local + /// and remote INIT chunks are present. + pub fn build_snap_init_data(&self) -> Option { + let d = self.snap_init.as_ref()?; + if d.local_init.is_none() || d.remote_init.is_none() { + return None; + } + Some(d.clone()) + } + + /// Opens a new stream. + pub fn open_stream(&mut self, id: u16, config: ChannelConfig) { + // The channel might already have arrived via SCTP, and if it is negotiated out-of-band + // we are waiting for the configuration. + let entry = stream_entry( + &mut self.entries, + id, + StreamEntryState::AwaitOpen, + "open_stream", + ); + + let in_band = config.negotiated.is_none(); + + // Stream should not already have a config, we are either waiting for DcepOpen, or this is + // out-of-band configuration, in which case this call is setting the config. + if entry.config.is_some() { + warn!("Stream is already configured: {}", id); + entry.do_close = true; + return; + } else { + entry.config = Some(config); + } + + // If we are in AwaitConfig, the stream was discovered from the remote peer before + // we got to do open_stream. This means we _must_ be in the out-of-band track, + // since we shouldn't call open_stream on remotely started in-band. + if entry.state == StreamEntryState::AwaitConfig { + if in_band { + warn!("open_stream in-band negotiation for remote stream: {}", id); + entry.do_close = true; + } else { + // out-of-band where remote started. We can go to Open, but must configure the local + // stream for it first. + + // The association must be open since we don't get AwaitConfig state without + // polling from the remote peer. + let mut stream = self + .assoc + .as_mut() + .expect("association to be open") + .stream(entry.id) + .expect("stream of entry in AwaitConfig"); + + if !entry.configure_reliability(&mut stream) { + return; + } + + entry.set_state(StreamEntryState::Open); + } + } + } + + /// Close stream. + pub fn close_stream(&mut self, id: u16) { + let Some(entry) = self.entries.iter_mut().find(|v| v.id == id) else { + return; + }; + entry.do_close = true; + + // Explicitly close the sctp stream to allow re-use of the same id. + let _ = self.sctp_propagate_close(id); + } + + pub fn close(&mut self) -> Result<(), SctpError> { + let Some(assoc) = &mut self.assoc else { + return Ok(()); + }; + + if assoc.is_closing() || assoc.is_closed() { + return Ok(()); + } + + Ok(assoc.shutdown()?) + } + + pub fn is_open(&self, id: u16) -> bool { + if self.state != RtcSctpState::Established { + return false; + } + + let Some(rec) = self.entries.iter().find(|e| e.id == id) else { + return false; + }; + + rec.state == StreamEntryState::Open + } + + // TODO: fix sctp-proto so we don't need &mut here. + pub fn available(&mut self) -> usize { + let Some(assoc) = &mut self.assoc else { + return 0; + }; + + // The amount currently buffered. + let total: usize = self + .entries + .iter() + .filter_map(|e| { + assoc + .stream(e.id) + .ok() + .and_then(|s| s.buffered_amount().ok()) + }) + .sum(); + + MAX_BUFFERED_ACROSS_STREAMS - total + } + + pub fn write(&mut self, id: u16, binary: bool, buf: &[u8]) -> Result { + if self.state != RtcSctpState::Established || self.is_closing() || self.is_closed() { + return Err(SctpError::WriteBeforeEstablished); + } + + let assoc = self + .assoc + .as_mut() + .ok_or(SctpError::WriteBeforeEstablished)?; + + let rec = self + .entries + .iter() + .find(|e| e.id == id) + .expect("stream entry for write"); + + if rec.state != StreamEntryState::Open { + return Err(SctpError::WriteBeforeEstablished); + } + + let mut stream = assoc.stream(id)?; + + let ppi = if binary { + if buf.is_empty() { + PayloadProtocolIdentifier::BinaryEmpty + } else { + PayloadProtocolIdentifier::Binary + } + } else if buf.is_empty() { + PayloadProtocolIdentifier::StringEmpty + } else { + PayloadProtocolIdentifier::String + }; + + Ok(stream.write_with_ppi(buf, ppi)?) + } + + pub fn buffered_amount(&mut self, id: u16) -> usize { + let Some(assoc) = self.assoc.as_mut() else { + return 0; + }; + + let Ok(stream) = assoc.stream(id) else { + return 0; + }; + + stream.buffered_amount().unwrap_or(0) + } + + pub fn set_buffered_amount_low_threshold(&mut self, id: u16, threshold: usize) { + let entry = self + .entries + .iter_mut() + .find(|e| e.id == id) + .expect("stream entry for valid channel id"); + + // This update will be propagated on next poll. + entry.buffered_threshold.set(threshold); + } + + pub fn handle_input(&mut self, now: Instant, data: &[u8]) { + trace!("Handle input: {}", data.len()); + + // TODO, remove Bytes in sctp and just use &[u8]. + let data = data.to_vec().into(); + let r = self.endpoint.handle(now, self.fake_addr, None, None, data); + + let Some((handle, event)) = r else { + return; + }; + + match event { + DatagramEvent::NewAssociation(a) => { + // In slow or unreliable networks from browsers (use 3g or slow 4g) settings. + // The browser resends a new associations and str0m would override the previously + // acked association. Webrtc should use only 1 association. + if self.assoc.is_some() { + return; + } + debug!("New remote association"); + // Remote side initiated the association + self.assoc = Some(a); + self.handle = handle; + set_state(&mut self.state, RtcSctpState::AwaitAssociationEstablished); + } + DatagramEvent::AssociationEvent(event) => { + self.assoc + .as_mut() + .expect("association for event") + .handle_event(event); + } + } + } + + pub fn handle_timeout(&mut self, now: Instant) { + if self.state == RtcSctpState::Uninited { + // Need to call `init()` before any timeouts are accepted. + return; + } + + self.last_now = now; + + // Remove closed entries. + self.entries.retain(|e| e.state != StreamEntryState::Closed); + + let Some(assoc) = &mut self.assoc else { + return; + }; + + assoc.handle_timeout(now); + + // propagate events between endpoint and association. + while let Some(e) = assoc.poll_endpoint_event() { + if let Some(ae) = self.endpoint.handle_event(self.handle, e) { + assoc.handle_event(ae); + } + } + } + + pub fn poll(&mut self) -> Option { + let r = self.do_poll(); + + if let Some(r) = &r { + trace!("Poll {:?}", r); + } + + r + } + + pub fn do_poll(&mut self) -> Option { + if self.state == RtcSctpState::Uninited { + // Need to call `init()` before any polling starts. + return None; + } + + // Remove closed entries. handle_timeout() also does this, but the + // remote can reuse a stream id (its reset handshake completed) before + // our next timeout. + self.entries.retain(|e| e.state != StreamEntryState::Closed); + + if let Some(t) = self.pushed_back_transmit.take() { + return Some(SctpEvent::Transmit { packets: t }); + } + + while let Some(t) = self.poll_transmit() { + let Some(buf) = transmit_to_vec(t) else { + continue; + }; + + return Some(SctpEvent::Transmit { packets: buf }); + } + + // Don't progress to move data between association and endpoint until we have an + // association we want to drive forward. + if !self.state.propagate_endpoint_to_assoc() { + return None; + } + + let assoc = self.assoc.as_mut()?; + + while let Some(e) = assoc.poll() { + if let Event::Connected = e { + assoc.set_max_send_message_size(self.remote_max_message_size); + set_state(&mut self.state, RtcSctpState::Established); + return self.poll(); + } + + if let Event::AssociationLost { ref reason } = e { + debug!("Association lost, reason: {}", reason); + // No reset can complete on a dead association. + self.reset_pending.clear(); + self.reset_complete.clear(); + return Some(SctpEvent::AssociationLost); + } + + if let Event::Stream(se) = e { + match se { + StreamEvent::Readable { id } | StreamEvent::Writable { id } => { + stream_entry( + &mut self.entries, + id, + StreamEntryState::AwaitConfig, + "readable/writable", + ); + } + StreamEvent::Finished { id } | StreamEvent::Stopped { id, .. } => { + // sctp-proto unregistered it when a reset arrived. + // Id reuse is signalled separately by StreamEvent::ResetComplete. + // + // sctp-proto arms that completion here, so from now on a reset + // is outstanding for the id even if we never closed it locally. + self.reset_pending.insert(id); + + // Only a live entry has anything to drop. Closed entries and + // missing ones already went through Close, and an AwaitOpen + // entry is a new incarnation waiting on the same id, it must + // not be killed by the old pending teardown. + if let Some(entry) = self.entries.iter_mut().find(|e| e.id == id) { + if entry.state != StreamEntryState::Closed + && entry.state != StreamEntryState::AwaitOpen + { + debug!("Stream {} finished", id); + entry.do_close = true; + } + } + } + StreamEvent::ResetComplete { id } => { + // The reset handshake for this id has fully completed. + debug!("Stream {} reset complete", id); + self.reset_pending.remove(&id); + self.reset_complete.push_back(id); + } + StreamEvent::BufferedAmountLow { id } => { + return Some(SctpEvent::BufferedAmountLow { id }); + } + _ => {} + } + } + } + + // Must wait for association state to be established before opening streams. + if self.state != RtcSctpState::Established { + return None; + } + + for entry in &mut self.entries { + let want_open = entry.state == StreamEntryState::AwaitOpen; + + if want_open { + debug!("Open stream {}", entry.id); + match assoc.open_stream(entry.id, PayloadProtocolIdentifier::Unknown) { + Ok(mut s) => { + entry.open_deadline = None; + + if !entry.configure_reliability(&mut s) { + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + + let config = entry.config.as_ref().expect("config if AwaitOpen"); + let in_band = config.negotiated.is_none(); + + if in_band { + let dcep: DcepOpen = config.into(); + let mut buf = vec![0; 1500]; + let n = dcep.marshal_to(&mut buf); + buf.truncate(n); + + match s.write_with_ppi(&buf, PayloadProtocolIdentifier::Dcep) { + Ok(l) => { + assert!(n == l); + entry.set_state(StreamEntryState::AwaitDcepAck); + + // Start over with polling, since we might have caused + // some network traffic by writing the DcepOpen. + return self.do_poll(); + } + Err(e) => { + warn!( + "Failed to write DCEP open on stream {}: {:?}", + entry.id, e + ); + entry.do_close = true; + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + } + } + + // Continuing means we are opening the stream out-of-band. + } + Err( + e @ (ProtoError::ErrStreamAlreadyExist | ProtoError::ErrStreamResetPending), + ) => { + let config = entry.config.as_ref().expect("config if AwaitOpen"); + let in_band = config.negotiated.is_none(); + + // ErrStreamAlreadyExist has two causes: + // - the remote created it by sending on it first + // - a previous incarnation of the id is still registered, + // reset handshake hasn't finished + let stale_incarnation = matches!(e, ProtoError::ErrStreamAlreadyExist) + && assoc + .stream(entry.id) + .map(|s| !s.is_readable() && !s.is_writable()) + .unwrap_or(false); + + if in_band + || stale_incarnation + || matches!(e, ProtoError::ErrStreamResetPending) + { + // RFC 6525 reset handshake for a previous + // incarnation of this stream id hasn't finished yet + // + // - AlreadyExist clears when the remote's reciprocal reset arrives + // - ResetPending when the RECONFIG-RESPONSE arrives (silently) + // + // In both cases, stay in AwaitOpen and retry until past the deadline. + let deadline = *entry + .open_deadline + .get_or_insert(self.last_now + STREAM_OPEN_TIMEOUT); + + if self.last_now < deadline { + debug!("Stream {} open blocked ({:?}), will retry", entry.id, e); + continue; + } + + debug!("Opening stream {} failed after retries: {:?}", entry.id, e); + entry.do_close = true; + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + + // Continuing means we are adopting the live out-of-band stream the + // remote created. It skipped the Ok branch above, so reliability + // params haven't been applied to it yet. + let mut stream = assoc.stream(entry.id).expect("stream that exists"); + if !entry.configure_reliability(&mut stream) { + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + } + Err(e) => { + warn!("Opening stream {} failed: {:?}", entry.id, e); + entry.do_close = true; + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + } + + // Consider out-of-band stream open. + let config = entry.config.as_ref().expect("config if AwaitOpen"); + let in_band = config.negotiated.is_none(); + assert!(!in_band); + + let label = config.label.clone(); + entry.set_state(StreamEntryState::Open); + + return Some(SctpEvent::Open { + id: entry.id, + label, + }); + } + + if entry.do_close && entry.state != StreamEntryState::Closed { + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + + let mut stream = match assoc.stream(entry.id) { + Ok(v) => v, + Err(e) => { + // This is expected on browser refresh or similar abrupt shutdown. + debug!("Getting stream {} failed: {:?}", entry.id, e); + entry.do_close = true; + continue; + } + }; + + // Propagate the desired buffered threshold. + // The idea is to only do this if the user has changed the value for it without + // incurring the cost of looking up the currently confifured value. + if let BufferedThresholdConfig::Desired(x) = entry.buffered_threshold { + if let Err(e) = stream.set_buffered_amount_low_threshold(x) { + debug!("Setting buffered_amount_low_threshold failed: {:?}", e); + entry.do_close = true; + entry.buffered_threshold = BufferedThresholdConfig::Unconfigured; + continue; + } + + entry.buffered_threshold = BufferedThresholdConfig::Configured(x); + } + match stream_read_data(&mut stream) { + Ok(Some((buf, ppi))) => { + if ppi != PayloadProtocolIdentifier::Dcep { + // This is the normal path for incoming data. + let buf = ppi_adjust_buf(buf, ppi); + let binary = matches!( + ppi, + PayloadProtocolIdentifier::Binary + | PayloadProtocolIdentifier::BinaryEmpty + ); + return Some(SctpEvent::Data { + id: entry.id, + binary, + data: buf, + }); + } + + // It's Dcep, either a DcepOpen or DcepAck. + match entry.state { + // We are in AwaitConfig state which means we are either going to get it via + // the DcepOpen, or by an out-of-band configuration via open_stream. + // This indicates we are doing in-band. + StreamEntryState::AwaitConfig => { + let dcep: DcepOpen = match buf.as_slice().try_into() { + Ok(v) => v, + Err(e) => { + warn!("Failed to read incoming DCEP {}: {:?}", entry.id, e); + entry.do_close = true; + continue; + } + }; + + if entry.config.is_none() { + entry.config = Some((&dcep).into()); + } else { + warn!("Received DcepOpen for configured stream: {}", entry.id); + } + + // Apply DcepOpen's reliability parameters to the sctp-proto stream. + // Without this, the DCEP-receiving side of an in-band channel sends + // with stream defaults: ordered and fully reliable. + if !entry.configure_reliability(&mut stream) { + continue; + } + + let mut obuf = [0]; + DcepAck.marshal_to(&mut obuf); + match stream.write_with_ppi(&obuf, PayloadProtocolIdentifier::Dcep) { + Ok(l) => { + assert!(obuf.len() == l); + entry.set_state(StreamEntryState::Open); + + return Some(SctpEvent::Open { + id: entry.id, + label: dcep.label, + }); + } + Err(e) => { + warn!( + "Failed to write DCEP ack on stream {}: {:?}", + entry.id, e + ); + entry.do_close = true; + entry.set_state(StreamEntryState::Closed); + let stream_id = entry.id; + let reset_pending = self.sctp_propagate_close(stream_id); + return Some(SctpEvent::Close { + id: stream_id, + reset_pending, + }); + } + } + } + StreamEntryState::AwaitDcepAck => { + let res: Result = buf.as_slice().try_into(); + + if let Err(e) = res { + warn!("Failed to read incoming DCEP ACK {}: {:?}", entry.id, e); + entry.do_close = true; + continue; + } + + entry.set_state(StreamEntryState::Open); + let config = entry.config.as_ref().expect("config when DcepAck"); + + return Some(SctpEvent::Open { + id: entry.id, + label: config.label.clone(), + }); + } + _ => { + warn!( + "Stream {} in wrong state when receiving DCEP: {:?}", + entry.id, entry.state + ); + entry.do_close = true; + continue; + } + } + } + Ok(None) => continue, + Err(_) => entry.do_close = true, + } + } + + // Reset completions are reported last. Reaching here means the entry loop had + // no `Close` left to emit, so the close for a released id has been delivered. + if let Some(id) = self.reset_complete.pop_front() { + return Some(SctpEvent::StreamResetComplete { id }); + } + + None + } + + pub fn poll_timeout(&mut self) -> Option { + let assoc_timeout = self.assoc.as_mut().and_then(|a| a.poll_timeout()); + + // Wakeup backstop for entries whose open_stream() is being retried. + // + // Normally no wakeup is needed: a blocked open re-attempts on the + // next do_poll(), and the packet that unblocks it (the peer's + // RECONFIG-RESPONSE or reset) itself triggers handle_input() and that poll. + // + // Returning a wakeup while any entry retries guarantees the open + // either resolves or fails within STREAM_OPEN_TIMEOUT. + let retry_timeout = self + .entries + .iter() + .any(|e| e.state == StreamEntryState::AwaitOpen && e.open_deadline.is_some()) + .then(|| self.last_now + STREAM_OPEN_RETRY_INTERVAL); + + match (assoc_timeout, retry_timeout) { + (Some(a), Some(r)) => Some(a.min(r)), + (a, r) => a.or(r), + } + } + + pub fn push_back_transmit(&mut self, data: VecDeque>) { + trace!("Push back transmit: {}", data.len()); + assert!(self.pushed_back_transmit.is_none()); + self.pushed_back_transmit = Some(data); + } + + fn poll_transmit(&mut self) -> Option { + if let Some(t) = self.endpoint.poll_transmit() { + return Some(t); + } + + if let Some(t) = self.assoc.as_mut()?.poll_transmit(self.last_now) { + return Some(t); + } + + None + } + + pub fn config(&self, sctp_stream_id: u16) -> Option<&ChannelConfig> { + self.entries + .iter() + .find(|s| s.id == sctp_stream_id) + .and_then(|s| s.config.as_ref()) + } + + /// Close the sctp stream to allow re-use of the same id. + /// + /// Returns whether a reset handshake is outstanding for `stream_id`. + fn sctp_propagate_close(&mut self, stream_id: u16) -> bool { + let did_reset = match self.assoc.as_mut().map(|assoc| assoc.stream(stream_id)) { + // `close()` only fails on the reset, which needs an established + // association. Closing a channel while the association is shutting down + // therefore queues nothing and no completion will ever arrive. + Some(Ok(mut stream)) => stream.close().is_ok(), + // No stream to reset. + _ => false, + }; + + if did_reset { + self.reset_pending.insert(stream_id); + } + + self.reset_pending.contains(&stream_id) + } + + #[cfg(test)] + pub(crate) fn remote_max_message_size(&self) -> u32 { + self.remote_max_message_size + } +} + +fn transmit_to_vec(t: Transmit) -> Option>> { + let Payload::RawEncode(v) = t.payload else { + return None; + }; + + Some(v.into_iter().map(|b| b.to_vec()).collect()) +} + +fn set_state(current_state: &mut RtcSctpState, state: RtcSctpState) { + if *current_state != state { + debug!("{:?} => {:?}", current_state, state); + *current_state = state; + } +} + +fn stream_entry<'a>( + entries: &'a mut Vec, + id: u16, + initial_state: StreamEntryState, + reason: &'static str, +) -> &'a mut StreamEntry { + let idx = entries.iter().position(|v| v.id == id); + if let Some(idx) = idx { + entries.get_mut(idx).unwrap() + } else { + debug!("New stream {} ({:?}): {}", id, initial_state, reason); + let e = StreamEntry { + config: None, + state: initial_state, + id, + do_close: false, + open_deadline: None, + buffered_threshold: BufferedThresholdConfig::Unconfigured, + }; + entries.push(e); + entries.last_mut().unwrap() + } +} + +fn stream_read_data( + stream: &mut Stream, +) -> Result, PayloadProtocolIdentifier)>, SctpError> { + let Some(chunks) = stream.read()? else { + return Ok(None); + }; + + let n = chunks.len(); + let mut buf = vec![0; n]; + + let l = chunks.read(&mut buf)?; + assert!(l == n); + + use PayloadProtocolIdentifier::*; + match chunks.ppi { + Dcep | String | Binary => {} // keep as is + StringEmpty | BinaryEmpty => buf.clear(), + _ => { + return Err(SctpError::Proto(ProtoError::Other( + "Unknown PayloadProtocolIdentifier".into(), + ))); + } + } + + Ok(Some((buf, chunks.ppi))) +} + +fn ppi_adjust_buf(mut buf: Vec, ppi: PayloadProtocolIdentifier) -> Vec { + match ppi { + PayloadProtocolIdentifier::StringEmpty | PayloadProtocolIdentifier::BinaryEmpty => { + buf.clear(); + buf + } + _ => buf, + } +} + +impl fmt::Debug for SctpEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transmit { packets } => f + .debug_struct("Transmit") + .field("packets", &packets.len()) + .finish(), + Self::Open { id, label } => f + .debug_struct("Open") + .field("id", id) + .field("label", label) + .finish(), + Self::Close { id, reset_pending } => f + .debug_struct("Close") + .field("id", id) + .field("reset_pending", reset_pending) + .finish(), + Self::StreamResetComplete { id } => f + .debug_struct("StreamResetComplete") + .field("id", id) + .finish(), + Self::Data { id, binary, data } => f + .debug_struct("Data") + .field("id", id) + .field("binary", binary) + .field("data", &data.len()) + .finish(), + Self::BufferedAmountLow { id } => { + f.debug_struct("BufferedAmountLow").field("id", id).finish() + } + Self::AssociationLost => f.debug_struct("AssociationLost").finish(), + } + } +} + +impl From<&ChannelConfig> for DcepOpen { + fn from(v: &ChannelConfig) -> Self { + let (channel_type, reliability_parameter) = (&v.reliability).into(); + DcepOpen { + unordered: !v.ordered, + channel_type, + reliability_parameter, + priority: 0, + label: v.label.clone(), + protocol: v.protocol.clone(), + } + } +} + +impl From<&Reliability> for (ReliabilityType, u32) { + fn from(v: &Reliability) -> Self { + match v { + Reliability::Reliable => (ReliabilityType::Reliable, 0), + Reliability::MaxPacketLifetime { lifetime } => { + (ReliabilityType::Timed, *lifetime as u32) + } + Reliability::MaxRetransmits { retransmits } => { + (ReliabilityType::Rexmit, *retransmits as u32) + } + } + } +} + +impl From<&DcepOpen> for ChannelConfig { + fn from(v: &DcepOpen) -> Self { + ChannelConfig { + label: v.label.clone(), + ordered: !v.unordered, + reliability: (v.channel_type, v.reliability_parameter).into(), + negotiated: None, + protocol: v.protocol.clone(), + } + } +} + +impl From<(ReliabilityType, u32)> for Reliability { + fn from((r, p): (ReliabilityType, u32)) -> Self { + match r { + ReliabilityType::Reliable => Reliability::Reliable, + ReliabilityType::Rexmit => Reliability::MaxRetransmits { + retransmits: p as u16, + }, + ReliabilityType::Timed => Reliability::MaxPacketLifetime { lifetime: p as u16 }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use str0m_proto::DATAGRAM_MTU_TARGET; + + #[test] + fn partial_snap_init_requires_both_chunks() { + let now = Instant::now(); + let mut sctp = RtcSctp::new(DATAGRAM_MTU_TARGET); + let mut init_data = SctpInitData::new(); + + init_data.local_init_chunk().unwrap(); + + let err = sctp.init(true, now, Some(init_data), None).unwrap_err(); + assert!( + err.to_string() + .contains("SNAP requires both local and remote SCTP INIT chunks") + ); + } + + #[test] + fn malformed_remote_snap_does_not_disable_local_opt_in() { + let mut sctp = RtcSctp::new(DATAGRAM_MTU_TARGET); + sctp.enable_snap(); + + assert!(!sctp.set_remote_snap_init_string("!!!not-valid-base64!!!")); + assert!(sctp.snap_enabled()); + assert!(sctp.ensure_local_snap_init()); + assert!(sctp.local_sctp_init_for_sdp().is_some()); + } + + /// Helper to connect a client and server RtcSctp pair to Established state. + fn connect_client_server() -> (RtcSctp, RtcSctp) { + let now = Instant::now(); + let mut client = RtcSctp::new(DATAGRAM_MTU_TARGET); + let mut server = RtcSctp::new(DATAGRAM_MTU_TARGET); + + client.init(true, now, None, None).unwrap(); + server.init(false, now, None, None).unwrap(); + + // Exchange packets until both are Established. + for _ in 0..20 { + // Drain client transmits -> feed to server + while let Some(t) = client.poll_transmit() { + if let Some(bufs) = transmit_to_vec(t) { + for buf in bufs { + server.handle_input(now, &buf); + } + } + } + // Process server events + while let Some(e) = server.do_poll() { + if let SctpEvent::Transmit { packets } = e { + for buf in packets { + client.handle_input(now, &buf); + } + } + } + + // Drain server transmits -> feed to client + while let Some(t) = server.poll_transmit() { + if let Some(bufs) = transmit_to_vec(t) { + for buf in bufs { + client.handle_input(now, &buf); + } + } + } + // Process client events + while let Some(e) = client.do_poll() { + if let SctpEvent::Transmit { packets } = e { + for buf in packets { + server.handle_input(now, &buf); + } + } + } + + // Check if both established + if client.state == RtcSctpState::Established + && server.state == RtcSctpState::Established + { + break; + } + } + + assert_eq!(client.state, RtcSctpState::Established); + assert_eq!(server.state, RtcSctpState::Established); + + (client, server) + } + + /// Regression test: when `assoc.open_stream()` returns `ErrStreamAlreadyExist` + /// for an in-band (DCEP) data channel, the entry must eventually transition to + /// Closed and emit `SctpEvent::Close`. The error is transient (a reset + /// handshake could clear it), so the entry retries first — but bounded by + /// `STREAM_OPEN_TIMEOUT`, not the infinite loop this once was. + #[test] + fn err_stream_already_exist_in_band_returns_close() { + let (mut client, _server) = connect_client_server(); + + let stream_id: u16 = 0; + + // Pre-create the stream in the association so the next open_stream() with the + // same ID will return ErrStreamAlreadyExist. + let assoc = client.assoc.as_mut().unwrap(); + assoc + .open_stream(stream_id, PayloadProtocolIdentifier::Unknown) + .expect("first open_stream should succeed"); + + // Manually add an entry in AwaitOpen state with in-band config (negotiated: None). + // This simulates a locally-initiated in-band channel whose stream ID conflicts + // with one already opened by the remote peer. + client.entries.push(StreamEntry { + config: Some(ChannelConfig { + label: "test".to_string(), + ordered: true, + reliability: Reliability::Reliable, + negotiated: None, // in-band + protocol: String::new(), + }), + state: StreamEntryState::AwaitOpen, + id: stream_id, + do_close: false, + open_deadline: None, + buffered_threshold: BufferedThresholdConfig::Unconfigured, + }); + + // The first poll retries instead of failing (arms the deadline). + let event = client.do_poll(); + assert!( + event.is_none(), + "expected retry (no event) for stream {stream_id}, got {event:?}" + ); + + // Past the deadline the open gives up and closes. + let later = Instant::now() + STREAM_OPEN_TIMEOUT + Duration::from_secs(1); + client.handle_timeout(later); + let event = client.do_poll(); + + assert!( + matches!(&event, Some(SctpEvent::Close { id, .. }) if *id == stream_id), + "expected SctpEvent::Close for stream {stream_id}, got {event:?}" + ); + + // Verify entry transitioned to Closed. + let entry = client.entries.iter().find(|e| e.id == stream_id).unwrap(); + assert_eq!(entry.state, StreamEntryState::Closed); + } + + #[test] + fn negotiated_reuse_does_not_report_open_for_old_closed_stream() { + let (mut client, _server) = connect_client_server(); + let stream_id = 0; + + let assoc = client.assoc.as_mut().unwrap(); + let mut old = assoc + .open_stream(stream_id, PayloadProtocolIdentifier::Unknown) + .expect("old stream should open"); + old.close().expect("old stream should start closing"); + + client.entries.push(StreamEntry { + config: Some(ChannelConfig { + label: "replacement".to_string(), + ordered: true, + reliability: Reliability::Reliable, + negotiated: Some(stream_id), + protocol: String::new(), + }), + state: StreamEntryState::AwaitOpen, + id: stream_id, + do_close: false, + open_deadline: None, + buffered_threshold: BufferedThresholdConfig::Unconfigured, + }); + + for _ in 0..10 { + match client.do_poll() { + Some(SctpEvent::Open { id, .. }) if id == stream_id => { + panic!("replacement was reported open while the old stream still exists") + } + Some(_) => {} + None => break, + } + } + } + + #[test] + fn close_is_reported_before_reset_complete_when_events_are_batched() { + let now = Instant::now(); + let (mut client, mut server) = connect_client_server(); + let stream_id = 0; + let config = ChannelConfig { + label: "batched-close".to_string(), + negotiated: Some(stream_id), + ..Default::default() + }; + + client.open_stream(stream_id, config.clone()); + server.open_stream(stream_id, config); + assert!(matches!( + client.do_poll(), + Some(SctpEvent::Open { id, .. }) if id == stream_id + )); + assert!(matches!( + server.do_poll(), + Some(SctpEvent::Open { id, .. }) if id == stream_id + )); + + client.close_stream(stream_id); + + // Move all reset packets in both directions without polling client + // application events. This batches Finished and ResetComplete in the + // association event queue, which is valid for a sans-I/O caller. + for _ in 0..4 { + while let Some(transmit) = client.poll_transmit() { + for packet in transmit_to_vec(transmit).unwrap() { + server.handle_input(now, &packet); + } + } + while let Some(transmit) = server.poll_transmit() { + for packet in transmit_to_vec(transmit).unwrap() { + client.handle_input(now, &packet); + } + } + } + + for _ in 0..10 { + match client.do_poll() { + Some(SctpEvent::Close { id, .. }) if id == stream_id => return, + Some(SctpEvent::StreamResetComplete { id }) if id == stream_id => { + panic!("ResetComplete was reported before Close") + } + Some(_) => {} + None => break, + } + } + + panic!("stream close was not reported"); + } + + /// Regression test: the DCEP-receiving side of an in-band channel must apply + /// the DcepOpen reliability parameters (here: unordered) to its sctp-proto + /// stream, instead of sending with stream defaults (ordered, fully reliable). + #[test] + fn dcep_receiver_applies_reliability_params() { + let now = Instant::now(); + let (mut client, mut server) = connect_client_server(); + + // Drain `from`, feeding packets to `to` and returning the raw bytes. + let pump = |from: &mut RtcSctp, to: &mut RtcSctp| { + let mut wire = vec![]; + while let Some(e) = from.do_poll() { + if let SctpEvent::Transmit { packets } = e { + for p in packets { + to.handle_input(now, &p); + wire.extend(p); + } + } + } + wire + }; + + // Client opens an unordered in-band (DCEP) channel; once the handshake is + // pumped through, the server (DCEP receiver) sends data back. + client.open_stream( + 0, + ChannelConfig { + ordered: false, + ..Default::default() + }, + ); + pump(&mut client, &mut server); // DcepOpen + pump(&mut server, &mut client); // DcepAck, server side now open + let payload = b"from dcep receiver"; + server.write(0, true, payload).unwrap(); + + // The 16-byte DATA chunk header puts the flags byte 15 bytes before the + // payload it carries. 0x04 is the U (unordered) flag. + let wire = pump(&mut server, &mut client); + let pos = wire + .windows(payload.len()) + .position(|w| w == payload) + .unwrap(); + assert!( + wire[pos - 15] & 0x04 != 0, + "DCEP receiver should send unordered" + ); + } + + #[test] + fn max_payload_size_matches_mtu_minus_overhead() { + let overhead = crate::io::MAX_DTLS_OVERHEAD + SCTP_OVERHEAD; + + let default_sctp = RtcSctp::new(DATAGRAM_MTU_TARGET); + assert_eq!( + default_sctp.max_payload_size(), + DATAGRAM_MTU_TARGET - overhead + ); + + let small_sctp = RtcSctp::new(900); + assert_eq!(small_sctp.max_payload_size(), 900 - overhead); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/snap.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/snap.rs new file mode 100644 index 000000000..43fe3a640 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sctp/snap.rs @@ -0,0 +1,170 @@ +//! SNAP (SCTP Negotiation Acceleration Protocol) types. +//! +//! See [draft-hancke-tsvwg-snap](https://datatracker.ietf.org/doc/draft-hancke-tsvwg-snap/). + +use std::sync::Arc; + +use base64ct::{Base64, Encoding}; +use sctp_proto::{ClientConfig, TransportConfig, generate_snap_token}; + +use super::{LOCAL_MAX_MESSAGE_SIZE, SctpError as Error}; + +/// Build the WebRTC transport config with unlimited retransmits. +/// +/// For WebRTC, we never want to give up retransmitting init and data packets. +/// The connectivity is in ICE, and SCTP should not give up until ICE gives up. +pub(super) fn webrtc_transport_config() -> Arc { + Arc::new( + TransportConfig::default() + .with_max_init_retransmits(None) + .with_max_data_retransmits(None) + .with_max_receive_message_size(LOCAL_MAX_MESSAGE_SIZE), + ) +} + +/// Out-of-band SCTP INIT data for SNAP negotiation. +/// +/// Holds the local and remote SCTP INIT chunks exchanged via signaling +/// (e.g. SDP `a=sctp-init`). When both are present, the SCTP association +/// skips the 4-way handshake and goes directly to established state. +/// +/// Typical flow: +/// +/// 1. Create `SctpInitData`. +/// 2. Call [`Self::local_init_chunk()`] and signal the returned bytes. +/// 3. Receive the peer's INIT bytes over signaling. +/// 4. Call [`Self::set_remote_init_chunk()`]. +/// 5. Pass the populated value to +/// [`DirectApi::start_sctp_with_snap()`][crate::change::DirectApi::start_sctp_with_snap]. +/// +/// The local INIT chunk is lazily generated and cached on first access. +/// +/// # Example +/// ``` +/// use str0m::channel::SctpInitData; +/// +/// let mut data = SctpInitData::new(); +/// +/// // Get local INIT chunk to send to remote peer +/// let local_init = data.local_init_chunk().expect("valid chunk"); +/// +/// // Later, set the remote INIT chunk received from peer +/// // data.set_remote_init_chunk(remote_init_bytes); +/// ``` +#[derive(Debug, Clone)] +pub struct SctpInitData { + pub(crate) transport: Arc, + pub(crate) local_init: Option>, + pub(crate) remote_init: Option>, +} + +impl Default for SctpInitData { + fn default() -> Self { + SctpInitData { + transport: webrtc_transport_config(), + local_init: None, + remote_init: None, + } + } +} + +impl SctpInitData { + /// Creates new default SCTP INIT data. + /// + /// By default, max init and data retransmits are set to `None` (unlimited), + /// which is recommended for WebRTC where connectivity is managed by ICE. + pub fn new() -> Self { + Self::default() + } + + /// Get the local INIT chunk bytes for out-of-band signaling. + /// + /// This generates a fresh INIT chunk using the transport config parameters. + /// The result can be exchanged with the remote peer via a signaling channel, + /// allowing both sides to skip the SCTP 4-way handshake. + /// + /// The generated bytes are cached so subsequent calls return the same INIT. + /// + /// If you plan to call `start_sctp_with_snap()`, call this method first so + /// the local INIT is captured in the `SctpInitData` you pass in. + pub fn local_init_chunk(&mut self) -> Result, Error> { + if let Some(init) = &self.local_init { + return Ok(init.clone()); + } + let init = generate_snap_token(&self.transport)?.to_vec(); + self.local_init = Some(init.clone()); + Ok(init) + } + + /// Get the local INIT chunk as a base64-encoded string for out-of-band signaling. + /// + /// This generates a fresh INIT chunk using the transport config parameters. + /// The result can be exchanged with the remote peer via a signaling channel, + /// allowing both sides to skip the SCTP 4-way handshake. + /// + /// The generated bytes are cached so subsequent calls return the same string. + /// + /// If you plan to call `start_sctp_with_snap()`, call this method first so + /// the local INIT is captured in the `SctpInitData` you pass in. + pub fn local_init_string(&mut self) -> Result { + self.local_init_chunk().map(|c| b64_encode(&c)) + } + + /// Check if remote INIT chunk has been configured for out-of-band signaling. + pub fn has_remote_init_chunk(&self) -> bool { + self.remote_init.is_some() + } + + /// Get the remote INIT chunk as a base64-encoded string, if set. + pub fn remote_init_string(&self) -> Option { + self.remote_init.as_ref().map(|b| b64_encode(b)) + } + + /// Set the remote INIT chunk for out-of-band signaling. + /// + /// When both local and remote INIT chunks are exchanged via a signaling + /// channel, the SCTP association can skip the 4-way handshake and go + /// directly to established state. + /// + /// This must be called before starting SCTP with SNAP. + pub fn set_remote_init_chunk(&mut self, value: Vec) { + self.remote_init = Some(value); + } + + /// Set the remote INIT chunk from a base64-encoded string. + /// + /// Convenience method for signaling channels that exchange strings (e.g. SDP). + pub fn set_remote_init_string(&mut self, value: &str) -> Result<(), Error> { + let mut buf = vec![0u8; value.len()]; + let len = Base64::decode(value, &mut buf) + .map_err(|_| Error::InvalidSnap)? + .len(); + buf.truncate(len); + self.set_remote_init_chunk(buf); + Ok(()) + } + + /// Build a `ClientConfig` from this data, consuming it. + /// + /// Only used internally by [`super::RtcSctp::init()`] to feed sctp-proto. + pub(crate) fn into_client_config(self) -> ClientConfig { + let mut config = ClientConfig::new(); + config.transport = self.transport; + match (self.local_init, self.remote_init) { + (Some(local), Some(remote)) => { + config = config.with_snap(local.into(), remote.into()); + } + (Some(_), None) | (None, Some(_)) => { + unreachable!("SNAP requires both local and remote INIT chunks"); + } + (None, None) => {} + } + config + } +} + +pub(super) fn b64_encode(data: &[u8]) -> String { + let mut buf = vec![0u8; Base64::encoded_len(data)]; + let encoded = Base64::encode(data, &mut buf).expect("buffer sized correctly"); + encoded.to_string() +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/data.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/data.rs new file mode 100644 index 000000000..d19801657 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/data.rs @@ -0,0 +1,3720 @@ +#![allow(clippy::single_match)] + +use combine::EasyParser; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fmt::{self}; +use std::ops::Deref; + +use crate::crypto::Fingerprint; +use crate::format::Codec; +use crate::format::CodecSpec; +use crate::format::FormatParams; +use crate::format::PayloadParams; +use crate::packet::H265ProfileTierLevel; +use crate::packet::H266ProfileTierLevel; +use crate::rtp_::{Direction, Extension, Frequency, Mid, Pt, Rid, SessionId, Ssrc}; +use crate::{Candidate, IceCreds, VERSION}; +use str0m_proto::Id; + +use super::SdpError; +use super::parser::sdp_parser; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sdp { + pub session: Session, + pub media_lines: Vec, +} + +impl Sdp { + pub(crate) fn parse(input: &str) -> Result { + sdp_parser() + .easy_parse(input) + .map(|(sdp, _)| sdp) + .map_err(|e| SdpError::ParseError(e.to_string())) + } + + /// Get the MIDs listed in the BUNDLE group, if any. + pub(crate) fn bundle_mids(&self) -> Option<&[Mid]> { + self.session.attrs.iter().find_map(|a| { + if let SessionAttribute::Group { typ, mids } = a { + if typ == "BUNDLE" { + return Some(mids.as_slice()); + } + } + None + }) + } + + pub(crate) fn assert_consistency(&self) -> Result<(), SdpError> { + match self.do_assert_consistency() { + None => Ok(()), + Some(error) => Err(SdpError::Inconsistent(error)), + } + } + + pub(crate) fn fingerprint(&self) -> Option { + self.session + .fingerprint() + .or_else(|| self.media_lines.iter().find_map(|m| m.fingerprint())) + } + + pub(crate) fn ice_creds(&self) -> Option { + self.session + .ice_creds() + .or_else(|| self.media_lines.iter().find_map(|m| m.ice_creds())) + } + + pub(crate) fn ice_candidates(&self) -> impl Iterator { + let mut candidates: HashSet<&Candidate> = HashSet::new(); + + // Session level ice candidates. + candidates.extend(self.session.ice_candidates()); + + // Ice candidates. + for m in &self.media_lines { + candidates.extend(m.ice_candidates()); + } + + candidates.into_iter() + } + + /// Get the `a=sctp-init` value from the application m-line, if present. + /// + /// Returns the base64-encoded SCTP INIT value. + pub(crate) fn sctp_init(&self) -> Option<&str> { + self.media_lines + .iter() + .find(|m| m.typ.is_channel()) + .and_then(|m| m.sctp_init()) + } + + pub(crate) fn setup(&self) -> Option { + self.session + .setup() + .or_else(|| self.media_lines.iter().find_map(|m| m.setup())) + } + + fn do_assert_consistency(&self) -> Option { + // TODO: SDP assertions we need to make: + // 1. Ensure that every m-line has the same PT configuration for a codec. I.e. if FIR is enabled + // for PT 96 in one m-line it must be in all m-lines. + // 2. Compare with previous m-lines that remote isn't narrowing/expanding PT and/or extmaps. + + let group = self + .session + .attrs + .iter() + .find(|a| matches!(a, SessionAttribute::Group { .. })); + + if let Some(SessionAttribute::Group { mids, .. }) = group { + if !mids.len() == self.media_lines.len() { + return Some(format!( + "a=group mid count doesn't match m-line count {} != {}", + mids.len(), + self.media_lines.len() + )); + } + for (media, mid) in self.media_lines.iter().zip(mids.iter()) { + media.check_consistent()?; + let m = media.mid(); + if m != *mid { + return Some(format!("Mid order not matching a=group {m} != {mid}")); + } + } + } else { + return Some("Session attribute a=group missing".into()); + } + + None + } +} + +/// Session info, before the first m= line +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Session { + pub id: SessionId, + pub bw: Option, + pub attrs: Vec, +} + +/// Bandwidth from b= line +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Bandwidth { + pub typ: String, + pub val: String, +} + +impl Session { + pub fn setup(&self) -> Option { + let setup = self.attrs.iter().find_map(|m| { + if let SessionAttribute::Setup(v) = m { + Some(v) + } else { + None + } + })?; + + Some(*setup) + } + + pub fn ice_creds(&self) -> Option { + let ufrag = self.attrs.iter().find_map(|m| { + if let SessionAttribute::IceUfrag(v) = m { + Some(v) + } else { + None + } + })?; + + let pass = self.attrs.iter().find_map(|m| { + if let SessionAttribute::IcePwd(v) = m { + Some(v) + } else { + None + } + })?; + + Some(IceCreds { + ufrag: ufrag.to_string(), + pass: pass.to_string(), + }) + } + + pub fn fingerprint(&self) -> Option { + for a in &self.attrs { + if let SessionAttribute::Fingerprint(v) = a { + return Some(v.clone()); + } + } + None + } + + pub fn ice_lite(&self) -> bool { + self.attrs + .iter() + .any(|a| matches!(a, SessionAttribute::IceLite)) + } + + pub fn ice_candidates(&self) -> impl Iterator { + self.attrs.iter().filter_map(|a| { + if let SessionAttribute::Candidate(v) = a { + Some(v) + } else { + None + } + }) + } + + pub fn end_of_candidates(&self) -> bool { + self.attrs + .iter() + .any(|a| matches!(a, SessionAttribute::EndOfCandidates)) + } +} + +/// Attributes before the first m= line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SessionAttribute { + Group { + typ: String, // BUNDLE, LS etc + mids: Vec, // 0 1 2 3 + }, + MsidSemantic { + semantic: String, // WMS + stream_ids: Vec, + }, + AllowMixedExts, + IceLite, + IceUfrag(String), + IcePwd(String), + IceOptions(String), + Fingerprint(Fingerprint), + Setup(Setup), // active, passive, actpass, holdconn + Candidate(Candidate), + EndOfCandidates, + Unused(String), +} + +fn is_dir(a: &MediaAttribute) -> bool { + use MediaAttribute::*; + matches!(a, SendRecv | SendOnly | RecvOnly | Inactive) +} + +/// An m-line +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct MediaLine { + pub typ: MediaType, + pub disabled: bool, + pub proto: Proto, + pub pts: Vec, // payload types 96 97 125 107 from the m= line + pub bw: Option, + pub attrs: Vec, +} + +impl MediaLine { + pub fn mid(&self) -> Mid { + self.attrs + .iter() + .find_map(|a| { + if let MediaAttribute::Mid(m) = a { + Some(*m) + } else { + None + } + }) + // We should only use `mid()` once we're certain there is + // a mid line. This is checked by `check_consistent`. + .expect("missing a=mid") + } + + /// Get the sctp-init attribute value (base64-encoded string). + /// + /// Returns `None` if no `a=sctp-init` attribute is present. + pub fn sctp_init(&self) -> Option<&str> { + self.attrs.iter().find_map(|a| { + if let MediaAttribute::SctpInit(v) = a { + Some(v.as_str()) + } else { + None + } + }) + } + + pub fn msid(&self) -> Option { + self.attrs.iter().find_map(|a| { + if let MediaAttribute::Msid(m) = a { + Some(m.clone()) + } else { + None + } + }) + } + + pub fn direction(&self) -> Direction { + for a in &self.attrs { + match a { + MediaAttribute::SendRecv => return Direction::SendRecv, + MediaAttribute::SendOnly => return Direction::SendOnly, + MediaAttribute::RecvOnly => return Direction::RecvOnly, + MediaAttribute::Inactive => return Direction::Inactive, + _ => {} + } + } + // Should we error here? + Direction::Inactive + } + + pub fn set_direction(&mut self, dir: Direction) { + let idx = self + .attrs + .iter() + .position(is_dir) + .expect("m-line must have direction"); + self.attrs[idx] = dir.into(); + } + + pub fn rtp_params(&self) -> Vec { + let mut rtp_maps: Vec<_> = self + .attrs + .iter() + .filter_map(|a| { + if let MediaAttribute::RtpMap { pt, value: c } = a { + Some((*pt, *c)) + } else { + None + } + }) + .collect(); + + for pt in &self.pts { + if !rtp_maps.iter().any(|(mapped, _)| mapped == pt) { + if let Some(spec) = CodecSpec::from_static_pt(*pt) { + rtp_maps.push((*pt, spec.into())); + } + } + } + + let fmtps: Vec<_> = self + .attrs + .iter() + .filter_map(|a| { + if let MediaAttribute::Fmtp { pt, values } = a { + Some((pt, values)) + } else { + None + } + }) + .collect(); + + let fbs: Vec<_> = self + .attrs + .iter() + .filter_map(|a| { + if let MediaAttribute::RtcpFb { pt, value } = a { + Some((pt, value)) + } else { + None + } + }) + .collect(); + + let mut params: Vec<_> = rtp_maps + .iter() + .filter(|(_, c)| c.codec.is_audio() | c.codec.is_video()) + .map(|(pt, c)| PayloadParams::new(*pt, None, (*c).into())) + .collect(); + + for p in &mut params { + for (pt, values) in fmtps.iter() { + // find matching a=fmtp line, if it exists. + if **pt == p.pt { + for param in values.iter() { + p.spec.format.set_param(param); + } + } + + // find resend pt, if there is one. + for fp in values.iter() { + if let FormatParam::Apt(v) = fp { + if *v == p.pt { + // ensure this is a rtx + let is_rtx = rtp_maps + .iter() + .any(|(cpt, c)| cpt == *pt && c.codec == Codec::Rtx); + if is_rtx { + p.resend = Some(**pt); + } + } + } + } + } + + // rtcp feedback mechanisms + for (pt, value) in fbs.iter() { + if **pt == p.pt { + match &value[..] { + "goog-remb" => { + p.fb_remb = true; + } + "transport-cc" => { + p.fb_transport_cc = true; + } + "ccm fir" => { + p.fb_fir = true; + } + "nack" => { + p.fb_nack = true; + } + "nack pli" => { + p.fb_pli = true; + } + _ => { + // + } + } + } + } + } + + params + } + + pub fn check_consistent(&self) -> Option { + use MediaAttribute::*; + + let mid_count = self.attrs.iter().filter(|a| matches!(a, Mid(_))).count(); + + if mid_count == 0 { + return Some(format!( + "Media is missing a=mid: {} {}", + self.typ, self.proto + )); + } + + if mid_count > 1 { + return Some(format!( + "Media has more than one a=mid: {} {}", + self.typ, self.proto + )); + } + + let setup = self.attrs.iter().filter(|a| matches!(a, Setup(_))).count(); + + if setup > 1 { + return Some(format!( + "Expected 0 or 1 a=setup: line for mid: {}", + self.mid() + )); + } + + let dir_count = self.attrs.iter().filter(|a| is_dir(a)).count(); + + if self.proto == Proto::Srtp && dir_count != 1 { + return Some(format!( + "Expected exactly one of a=sendrecv, a=sendonly, a=recvonly, a=inactive for mid: {}", + self.mid() + )); + } + + if self.proto == Proto::Srtp && self.pts.is_empty() { + return Some(format!("Expected at least one PT for mid: {}", self.mid())); + } + + for m in &self.pts { + let rtp_count = self + .attrs + .iter() + .filter(|a| { + if let MediaAttribute::RtpMap { pt, value: _ } = a { + pt == m + } else { + false + } + }) + .count(); + if rtp_count == 0 && CodecSpec::from_static_pt(*m).is_none() { + return Some(format!("Missing a=rtp_map:{} for mid: {}", m, self.mid())); + } + if rtp_count > 1 { + return Some(format!( + "More than one a=rtp_map:{} for mid: {}", + m, + self.mid() + )); + } + } + None + } + + pub fn setup(&self) -> Option { + let setup = self.attrs.iter().find_map(|m| { + if let MediaAttribute::Setup(v) = m { + Some(v) + } else { + None + } + })?; + + Some(*setup) + } + + pub fn ice_creds(&self) -> Option { + let ufrag = self.attrs.iter().find_map(|m| { + if let MediaAttribute::IceUfrag(v) = m { + Some(v) + } else { + None + } + })?; + + let pass = self.attrs.iter().find_map(|m| { + if let MediaAttribute::IcePwd(v) = m { + Some(v) + } else { + None + } + })?; + + Some(IceCreds { + ufrag: ufrag.to_string(), + pass: pass.to_string(), + }) + } + pub fn fingerprint(&self) -> Option { + for a in &self.attrs { + if let MediaAttribute::Fingerprint(v) = a { + return Some(v.clone()); + } + } + None + } + + /// This hoovers the ice candidates from all m-lines, lots of dupes. + /// For WebRTC we don't expect different ice states per media line. + pub fn ice_candidates(&self) -> impl Iterator { + self.attrs.iter().filter_map(|a| { + if let MediaAttribute::Candidate(v) = a { + Some(v) + } else { + None + } + }) + } + + /// Any end-of-candidate in any m-line. + /// For WebRTC we don't expect different ice states per media line. + pub fn end_of_candidates(&self) -> bool { + self.attrs + .iter() + .any(|a| matches!(a, MediaAttribute::EndOfCandidates)) + } + + pub fn extmaps(&self) -> Vec<(u8, &Extension)> { + let mut ret = vec![]; + + for a in &self.attrs { + if let MediaAttribute::ExtMap { id, ext } = a { + ret.push((*id, ext)); + } + } + + ret + } + + pub fn rids(&self) -> Vec { + let mut ret = vec![]; + for a in &self.attrs { + if let MediaAttribute::Rid { id, .. } = a { + ret.push(id.0.as_str().into()) + } + } + ret + } + + pub fn simulcast(&self) -> Option { + let mut found = None; + + for a in &self.attrs { + if let MediaAttribute::Simulcast(s) = a { + found = Some(s.clone()); + } + if let MediaAttribute::Rid { + pt, restriction, .. + } = a + { + if !pt.is_empty() { + warn!("Not currently supporting PT via a=rid"); + } + if !restriction.is_empty() { + warn!("Not currently supporting restrictions via a=rid"); + } + } + } + + if found.is_some() { + return found; + } + + // Here we could handle munged SDPs and we used to, but we have dropped support for this. + + None + } + + pub fn ssrc_info(&self) -> Vec { + let mut v = vec![]; + + fn by_ssrc(v: &mut Vec, ssrc: Ssrc) -> &mut SsrcInfo { + if let Some(pos) = v.iter().position(|i| i.ssrc == ssrc) { + &mut v[pos] + } else { + v.push(SsrcInfo { + ssrc, + ..Default::default() + }); + v.last_mut().unwrap() + } + } + + for a in &self.attrs { + match a { + MediaAttribute::Ssrc { ssrc, attr, value } => { + let info = by_ssrc(&mut v, *ssrc); + + // a=ssrc:2147603131 cname:TbS1Ajv9obq6/63I + // a=ssrc:2147603131 msid:- 7a08dda6-518f-4027-b707-410a6d414176 + match attr.to_lowercase().as_str() { + "cname" => info.cname = Some(value.clone()), + "msid" => { + let mut iter = value.split(' '); + + fn trim_and_no_minus(s: &str) -> Option { + let s = s.trim(); + + if s == "-" { None } else { Some(s.into()) } + } + + if let Some(stream_id) = iter.next() { + info.stream_id = trim_and_no_minus(stream_id); + } + if let Some(track_id) = iter.next() { + info.track_id = trim_and_no_minus(track_id); + } + } + _ => {} + } + } + _ => {} + } + } + // Match this second to ensure we preserve order of a=ssrc. + for a in &self.attrs { + match a { + MediaAttribute::SsrcGroup { semantics, ssrcs } => { + if semantics.to_lowercase() != "fid" { + continue; + } + + // a=ssrc-group:FID 659652645 98148385 + // Should be two SSRC after FID. + if ssrcs.len() != 2 { + continue; + } + + let info = by_ssrc(&mut v, ssrcs[1]); + info.repairs = Some(ssrcs[0]); + } + _ => {} + } + } + + v + } + + /// Get the max-message-size attribute value, if present + pub fn max_message_size(&self) -> Option { + self.attrs.iter().find_map(|a| { + if let MediaAttribute::MaxMessageSize(size) = a { + Some(*size) + } else { + None + } + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SsrcInfo { + pub ssrc: Ssrc, + /// the other ssrc this ssrc is repairing + pub repairs: Option, + pub cname: Option, + pub stream_id: Option, + pub track_id: Option, +} + +impl Default for SsrcInfo { + fn default() -> Self { + Self { + ssrc: 0.into(), + repairs: None, + cname: None, + stream_id: None, + track_id: None, + } + } +} + +impl From for MediaAttribute { + fn from(v: Direction) -> Self { + match v { + Direction::SendOnly => MediaAttribute::SendOnly, + Direction::RecvOnly => MediaAttribute::RecvOnly, + Direction::SendRecv => MediaAttribute::SendRecv, + Direction::Inactive => MediaAttribute::Inactive, + } + } +} + +/// Identifier of an `a=rid` restriction. +/// +/// Defined in https://tools.ietf.org/html/draft-ietf-avtext-rid-09 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestrictionId(pub String, pub bool); + +impl RestrictionId { + pub fn new(rid: String, active: bool) -> Self { + RestrictionId(rid, active) + } + + pub fn new_active(rid: String) -> Self { + RestrictionId(rid, true) + } + + pub fn to_sdp(&self) -> String { + format!("{}{}", if self.1 { "" } else { "~" }, self.0) + } +} + +impl Default for RestrictionId { + fn default() -> Self { + RestrictionId("".to_string(), true) + } +} + +/// "audio", "video", "application" +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub enum MediaType { + #[default] + Audio, + Video, + Application, + // If a parsed SDP has a value we don't recognize, we stick it + // in here. We could consider making that a parse exception instead. + #[doc(hidden)] + Unknown(String), +} + +impl MediaType { + pub fn is_media(&self) -> bool { + matches!(self, MediaType::Audio | MediaType::Video) + } + + pub fn is_channel(&self) -> bool { + matches!(self, MediaType::Application) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Proto { + #[default] + Srtp, + Sctp, +} + +impl Proto { + pub fn proto_line(&self) -> &str { + match self { + Proto::Srtp => "UDP/TLS/RTP/SAVPF", + Proto::Sctp => "UDP/DTLS/SCTP", + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Setup { + #[default] + ActPass, + Active, + Passive, +} + +impl fmt::Display for Setup { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + Setup::ActPass => "actpass", + Setup::Active => "active", + Setup::Passive => "passive", + } + ) + } +} + +impl Setup { + pub fn setup_line(&self) -> &str { + match self { + Setup::ActPass => "actpass", + Setup::Active => "active", + Setup::Passive => "passive", + } + } + + pub fn compare_to_remote(&self, remote: Setup) -> Option { + use Setup::*; + match (self, remote) { + (ActPass, ActPass) => None, + (ActPass, Active) => Some(Passive), + (ActPass, Passive) => Some(Active), + (Active, ActPass) => Some(Active), + (Active, Active) => None, + (Active, Passive) => Some(Active), + (Passive, ActPass) => Some(Passive), + (Passive, Active) => Some(Passive), + (Passive, Passive) => None, + } + } + + pub fn invert(&self) -> Setup { + match self { + Setup::ActPass => Setup::ActPass, + Setup::Active => Setup::Passive, + Setup::Passive => Setup::Active, + } + } +} + +/// Attributes before the first m= line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MediaAttribute { + // The "a=rtcp" line MUST NOT be added if the most recent answer included an "a=rtcp-mux" line. + Rtcp(String), + IceUfrag(String), + IcePwd(String), + IceOptions(String), + Fingerprint(Fingerprint), + Setup(Setup), // active, passive, actpass, holdconn + Mid(Mid), // 0, 1, 2 + SctpPort(u16), + MaxMessageSize(usize), + /// a=sctp-init: + /// See draft-hancke-tsvwg-snap + SctpInit(String), + // a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level + // a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time + ExtMap { + id: u8, // 1-14 inclusive, + ext: Extension, + }, + RecvOnly, // a=recvonly + SendRecv, // a=sendrecv + SendOnly, // a=sendonly + Inactive, // a=inactive + // a=msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1 + // a=msid:- 78dde68-7055-4e20-bb37-433803dd1ed1 + Msid(Msid), + RtcpMux, // + RtcpMuxOnly, // only in offer, answer with a=rtcp-mux + // reduced size rtcp. remove this if not supported. + RtcpRsize, + Candidate(Candidate), + EndOfCandidates, + RtpMap { + pt: Pt, + value: RtpMap, + }, + // rtcp-fb RTCP feedback parameters, repeated + RtcpFb { + pt: Pt, // 111 + value: String, // nack, nack pli, ccm fir... + }, + // format parameters, seems to be one of these + Fmtp { + pt: Pt, // 111 + values: Vec, // minptime=10;useinbandfec=1 + }, + // a=rid: [pt=;]= + // a=rid:hi send pt=111,112;max-br=64000;max-height=360 + // https://tools.ietf.org/html/draft-ietf-mmusic-rid-15 + Rid { + id: RestrictionId, // + direction: &'static str, // send or recv + // No pt means the rid applies to all + pt: Vec, // 111, 112 (rtpmap no) + restriction: Vec<(String, String)>, + }, + // a=rid:hi send + // a=rid:lo send + // a=simulcast:send hi;lo + // https://tools.ietf.org/html/draft-ietf-mmusic-sdp-simulcast-14 + // a=simulcast: ;, [same] + Simulcast(Simulcast), + SsrcGroup { + semantics: String, // i.e. "FID" + ssrcs: Vec, // + }, + Ssrc { + ssrc: Ssrc, // synchronization source id + attr: String, + value: String, + }, + Unused(String), +} + +impl MediaAttribute { + pub fn is_direction(&self) -> bool { + use MediaAttribute::*; + matches!(self, RecvOnly | SendRecv | SendOnly | Inactive) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FormatParam { + /// The minimum duration of media represented by a packet. + /// + /// Default 3. Max 120. + MinPTime(u8), + + /// Decoder-side preference for stereo audio. Default is mono per + /// RFC 7587 Section 7.1: + Stereo(bool), + + /// Sender-side stereo hint for Opus. This is the symmetric m-line companion + /// to `stereo` and signals the sender's intention to transmit stereo. + SpropStereo(bool), + + /// Specifies that the decoder can do Opus in-band FEC + UseInbandFec(bool), + + /// Specifies that the decoder can do Opus DTX + UseDtx(bool), + + /// Whether h264 sending media encoded at a different level in the offerer-to-answerer + /// direction than the level in the answerer-to-offerer direction, is allowed. + LevelAsymmetryAllowed(bool), + + /// What h264 packetization mode is used. + /// + /// * 0 - single nal. + /// * 1 - STAP-A, FU-A is allowed. Non-interleaved. + PacketizationMode(u8), + + /// H264 profile level. + /// + /// * 42 00 1f - 4200=baseline (B) 1f=level 3.1 + /// * 42 e0 1f - 42e0=constrained baseline (CB) 1f=level 3.1 + /// * 4d 00 1f - 4d00=main (M) 1f=level 3.1 + /// * 64 00 1f - 6400=high (H) 1f=level 3.1 + ProfileLevelId(u32), + + /// VP9 profile id + ProfileId(u32), + + /// AV1 profile + Profile(u8), + + /// AV1 level-idx + LevelIdx(u8), + + /// AV1 tier + Tier(u8), + + /// H.265/HEVC profile, tier, and level. + H265ProfileTierLevel(crate::packet::H265ProfileTierLevel), + + /// H.266/VVC profile, tier, and level. + H266ProfileTierLevel(crate::packet::H266ProfileTierLevel), + + /// H.265 sprop-max-don-diff parameter (RFC 7798 §7.1). + /// When > 0, DONL fields are included in RTP packets to support + /// out-of-order NAL unit decoding. Valid range: 0–32767. + SpropMaxDonDiff(u16), + + /// RTX (resend) codecs, which PT it concerns. + Apt(Pt), + + /// Unrecognized fmtp. + Unknown, +} + +impl FormatParam { + pub fn parse(k: &str, v: &str) -> Self { + use FormatParam::*; + match k { + "minptime" => v.parse().map(MinPTime).ok(), + "useinbandfec" => Some(UseInbandFec(v == "1")), + "usedtx" => Some(UseDtx(v == "1")), + "stereo" => Some(Stereo(v == "1")), + "sprop-stereo" => Some(SpropStereo(v == "1")), + "level-asymmetry-allowed" => Some(LevelAsymmetryAllowed(v == "1")), + "packetization-mode" => v.parse().map(PacketizationMode).ok(), + "profile-level-id" => u32::from_str_radix(v, 16) + .or_else(|_| v.parse()) + .map(ProfileLevelId) + .ok(), + "profile-id" => v.parse().map(ProfileId).ok(), + "profile" => v.parse().map(Profile).ok(), + "level-idx" => v.parse().map(LevelIdx).ok(), + "tier" => v.parse().map(Tier).ok(), + "sprop-max-don-diff" => v + .parse::() + .ok() + .filter(|&v| v <= 32767) + .map(SpropMaxDonDiff), + "apt" => v.parse::().map(|v| Apt(Pt::from(v))).ok(), + _ => None, + } + .unwrap_or_else(|| { + trace!("Failed to parse FormatParam: {k}={v}"); + Unknown + }) + } + + /// Parse multiple format parameters from key-value pairs. + /// Handles the H.265/H.266 special case where three params combine into + /// one composite ProfileTierLevel. + pub fn parse_pairs(pairs: Vec<(String, String)>) -> Vec { + // Check if this looks like H.265/H.266 by presence of tier-flag or level-id. + // Both codecs use the same three fmtp keys (RFC 7798 / RFC 9328); + // their level-id value spaces are disjoint (H.265: 30×level, e.g. 93; + // H.266: 16×major + 3×minor, e.g. 51), which disambiguates them. + let is_ptl = pairs + .iter() + .any(|(k, _)| k == "tier-flag" || k == "level-id"); + + if is_ptl { + // Build composite ProfileTierLevel: H.265 first, else H.266. + let map: HashMap = pairs.into_iter().collect(); + + let mut result = vec![]; + if let Some(ptl) = H265ProfileTierLevel::from_fmtp(&map) { + result.push(FormatParam::H265ProfileTierLevel(ptl)); + } else if let Some(ptl) = H266ProfileTierLevel::from_fmtp(&map) { + result.push(FormatParam::H266ProfileTierLevel(ptl)); + } + + // Include non-PTL parameters. + for (k, v) in map.iter() { + if k != "profile-id" && k != "tier-flag" && k != "level-id" { + result.push(FormatParam::parse(k, v)); + } + } + + result + } else { + // Standard parsing for non-PTL codecs. + pairs + .into_iter() + .map(|(k, v)| FormatParam::parse(&k, &v)) + .collect() + } + } +} + +impl fmt::Display for FormatParam { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use FormatParam::*; + match self { + MinPTime(v) => write!(f, "minptime={v}"), + UseInbandFec(v) => write!(f, "useinbandfec={}", i32::from(*v)), + UseDtx(v) => write!(f, "usedtx={}", i32::from(*v)), + Stereo(v) => write!(f, "stereo={}", i32::from(*v)), + SpropStereo(v) => write!(f, "sprop-stereo={}", i32::from(*v)), + LevelAsymmetryAllowed(v) => { + write!(f, "level-asymmetry-allowed={}", i32::from(*v)) + } + PacketizationMode(v) => write!(f, "packetization-mode={}", *v), + ProfileLevelId(v) => write!(f, "profile-level-id={:06x}", *v), + ProfileId(v) => write!(f, "profile-id={}", *v), + Profile(v) => write!(f, "profile={}", v), + LevelIdx(v) => write!(f, "level-idx={}", *v), + Tier(v) => write!(f, "tier={}", v), + H265ProfileTierLevel(ptl) => { + write!( + f, + "profile-id={};tier-flag={};level-id={}", + ptl.profile_id(), + ptl.tier_flag(), + ptl.level_id() + ) + } + H266ProfileTierLevel(ptl) => { + write!( + f, + "profile-id={};tier-flag={};level-id={}", + ptl.profile_id(), + ptl.tier_flag(), + ptl.level_id() + ) + } + SpropMaxDonDiff(v) => write!(f, "sprop-max-don-diff={}", *v), + Apt(v) => write!(f, "apt={v}"), + Unknown => Ok(()), + } + } +} + +impl PayloadParams { + pub(crate) fn as_media_attrs(&self, attrs: &mut Vec) { + attrs.push(MediaAttribute::RtpMap { + pt: self.pt, + value: self.spec.into(), + }); + + if self.fb_transport_cc { + attrs.push(MediaAttribute::RtcpFb { + pt: self.pt, + value: "transport-cc".into(), + }); + } + if self.fb_remb { + attrs.push(MediaAttribute::RtcpFb { + pt: self.pt, + value: "goog-remb".into(), + }); + } + if self.fb_fir { + attrs.push(MediaAttribute::RtcpFb { + pt: self.pt, + value: "ccm fir".into(), + }); + } + if self.fb_nack { + attrs.push(MediaAttribute::RtcpFb { + pt: self.pt, + value: "nack".into(), + }); + } + if self.fb_pli { + attrs.push(MediaAttribute::RtcpFb { + pt: self.pt, + value: "nack pli".into(), + }); + } + + let fmtps = self.spec.format.to_format_param(); + if !fmtps.is_empty() { + attrs.push(MediaAttribute::Fmtp { + pt: self.pt, + values: fmtps, + }); + } + + if let Some(pt) = self.resend { + attrs.push(MediaAttribute::RtpMap { + pt, + value: RtpMap { + codec: Codec::Rtx, + clock_rate: self.spec.rtp_clock_rate(), + channels: None, + }, + }); + attrs.push(MediaAttribute::Fmtp { + pt, + values: vec![FormatParam::Apt(self.pt)], + }); + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RtpMap { + pub codec: Codec, + pub clock_rate: Frequency, + pub channels: Option, +} + +impl From for CodecSpec { + fn from(v: RtpMap) -> Self { + // G722 uses an 8000 Hz RTP clock rate in SDP (RFC 3551 §4.5.2), but str0m + // treats it as a 16 kHz mono codec everywhere user facing. Normalize on the way in. + // See https://en.wikipedia.org/wiki/RTP_payload_formats#cite_note-55 + let (clock_rate, channels) = if v.codec == Codec::G722 { + let channels = if v.channels == Some(1) { + None + } else { + v.channels + }; + (Frequency::SIXTEEN_KHZ, channels) + } else { + (v.clock_rate, v.channels) + }; + CodecSpec { + codec: v.codec, + clock_rate, + channels, + format: FormatParams::default(), + } + } +} + +impl From for RtpMap { + fn from(v: CodecSpec) -> Self { + RtpMap { + codec: v.codec, + // G722 is signalled with an 8000 Hz clock rate in SDP (RFC 3551 §4.5.2), + // even though it samples at 16 kHz. See + // https://en.wikipedia.org/wiki/RTP_payload_formats#cite_note-55 + clock_rate: v.rtp_clock_rate(), + channels: v.channels, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Simulcast { + pub send: SimulcastGroups, + pub recv: SimulcastGroups, + /// If this is created synthetically for a munged SDP. + pub is_munged: bool, +} + +impl Simulcast { + pub fn invert(self) -> Self { + Simulcast { + send: self.recv, + recv: self.send, + is_munged: self.is_munged, + } + } +} + +/// RID organization inside a=simulcast line. +/// +/// `a=simulcast send 2;3,4` would result in +/// `SimulcastGroups(vec![SimulcastLayer("2"), SimulcastLayer("3")])` +/// +/// The choice between 3 and 4 for the rid is currently ignored and the first option is always +/// selected. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SimulcastGroups(pub Vec); + +/// Represents a simulcast layer in a group +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SimulcastLayer { + /// The layer's rid + pub restriction_id: RestrictionId, + /// Optional attributes per RFC 8851 + pub attributes: Option>, +} + +impl Deref for SimulcastGroups { + type Target = [SimulcastLayer]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Msid { + pub stream_id: String, + pub track_id: String, +} + +impl Msid { + pub fn random() -> Self { + Msid { + stream_id: Id::<30>::random().to_string(), + track_id: Id::<30>::random().to_string(), + } + } +} + +impl fmt::Display for SimulcastGroups { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (idx, layer) in self.0.iter().enumerate() { + if idx + 1 == self.0.len() { + write!(f, "{}", layer.restriction_id.to_sdp())?; + } else { + write!(f, "{};", layer.restriction_id.to_sdp())?; + } + } + Ok(()) + } +} + +impl fmt::Display for Sdp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.session)?; + for m in &self.media_lines { + write!(f, "{m}")?; + } + Ok(()) + } +} + +impl fmt::Display for Session { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "v=0\r\n")?; + write!(f, "o=str0m-{} {} 2 IN IP4 0.0.0.0\r\n", VERSION, self.id)?; + write!(f, "s=-\r\n")?; + if let Some(bw) = &self.bw { + write!(f, "b={}:{}\r\n", bw.typ, bw.val)?; + } + write!(f, "t=0 0\r\n")?; + for a in &self.attrs { + write!(f, "{a}")?; + } + Ok(()) + } +} + +impl fmt::Display for SessionAttribute { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use SessionAttribute::*; + match self { + Group { typ, mids } => { + let mids: Vec<_> = mids.iter().map(|m| m.to_string()).collect(); + write!(f, "a=group:{} {}\r\n", typ, mids.join(" "))?; + } + AllowMixedExts => write!(f, "a=extmap-allow-mixed\r\n")?, + MsidSemantic { + semantic, + stream_ids, + } => { + write!( + f, + "a=msid-semantic: {} {}\r\n", + semantic, + stream_ids.join(" ") + )?; + } + IceLite => write!(f, "a=ice-lite\r\n")?, + IceUfrag(v) => write!(f, "a=ice-ufrag:{v}\r\n")?, + IcePwd(v) => write!(f, "a=ice-pwd:{v}\r\n")?, + IceOptions(v) => write!(f, "a=ice-options:{v}\r\n")?, + Fingerprint(v) => { + write!( + f, + "a=fingerprint:{} {}\r\n", + v.hash_func, + FingerprintFmt(&v.bytes) + )?; + } + Setup(v) => write!(f, "a=setup:{}\r\n", v.setup_line())?, + Candidate(c) => write!(f, "a={}\r\n", c.to_sdp_string())?, + EndOfCandidates => write!(f, "a=end-of-candidates\r\n")?, + Unused(v) => write!(f, "a={v}\r\n")?, + } + Ok(()) + } +} + +impl fmt::Display for MediaLine { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let port = if self.disabled { 0 } else { 9 }; + write!(f, "m={} {} {} ", self.typ, port, self.proto)?; + let len = self.pts.len(); + if self.typ.is_channel() { + write!(f, "webrtc-datachannel\r\n")?; + } else { + for (idx, m) in self.pts.iter().enumerate() { + if idx + 1 < len { + write!(f, "{m} ")?; + } else { + write!(f, "{m}")?; + } + } + write!(f, "\r\n")?; + } + write!(f, "c=IN IP4 0.0.0.0\r\n")?; + if let Some(bw) = &self.bw { + write!(f, "b={}:{}\r\n", bw.typ, bw.val)?; + } + for a in &self.attrs { + write!(f, "{a}")?; + } + Ok(()) + } +} + +impl fmt::Display for MediaType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MediaType::Audio => write!(f, "audio"), + MediaType::Video => write!(f, "video"), + MediaType::Application => write!(f, "application"), + MediaType::Unknown(v) => write!(f, "{v}"), + } + } +} + +impl fmt::Display for Proto { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.proto_line()) + } +} + +impl fmt::Display for MediaAttribute { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use MediaAttribute::*; + match self { + Rtcp(v) => write!(f, "a=rtcp:{v}\r\n")?, + IceUfrag(v) => write!(f, "a=ice-ufrag:{v}\r\n")?, + IcePwd(v) => write!(f, "a=ice-pwd:{v}\r\n")?, + IceOptions(v) => write!(f, "a=ice-options:{v}\r\n")?, + Fingerprint(v) => { + write!( + f, + "a=fingerprint:{} {}\r\n", + v.hash_func, + FingerprintFmt(&v.bytes) + )?; + } + Setup(v) => write!(f, "a=setup:{}\r\n", v.setup_line())?, + Mid(v) => write!(f, "a=mid:{v}\r\n")?, + SctpPort(v) => write!(f, "a=sctp-port:{v}\r\n")?, + MaxMessageSize(v) => write!(f, "a=max-message-size:{v}\r\n")?, + SctpInit(v) => write!(f, "a=sctp-init:{v}\r\n")?, + ExtMap { id, ext } => { + if !ext.is_serialized() { + return Ok(()); + } + write!(f, "a=extmap:{}", id)?; + // if let Some(d) = &e.direction { + // write!(f, "/{d}")?; + // } + write!(f, " {}", ext.as_uri())?; + // if let Some(e) = &e.ext { + // write!(f, " {}", e)?; + // } + write!(f, "\r\n")?; + } + RecvOnly => write!(f, "a=recvonly\r\n")?, + SendRecv => write!(f, "a=sendrecv\r\n")?, + SendOnly => write!(f, "a=sendonly\r\n")?, + Inactive => write!(f, "a=inactive\r\n")?, + // a=msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1 + // a=msid:- 78dde68-7055-4e20-bb37-433803dd1ed1 + Msid(v) => write!(f, "a=msid:{} {}\r\n", v.stream_id, v.track_id)?, + RtcpMux => write!(f, "a=rtcp-mux\r\n")?, + RtcpMuxOnly => write!(f, "a=rtcp-mux-only\r\n")?, + RtcpRsize => write!(f, "a=rtcp-rsize\r\n")?, + Candidate(c) => write!(f, "a={}\r\n", c.to_sdp_string())?, + EndOfCandidates => write!(f, "a=end-of-candidates\r\n")?, + RtpMap { pt, value: c } => { + write!(f, "a=rtpmap:{} {}/{}", pt, c.codec, c.clock_rate)?; + if let Some(e) = c.channels { + write!(f, "/{e}")?; + } + write!(f, "\r\n")?; + } + RtcpFb { pt, value } => write!(f, "a=rtcp-fb:{pt} {value}\r\n")?, + Fmtp { pt, values } => { + if values.is_empty() { + // Skip empty fmtp lines - they're invalid SDP + return Ok(()); + } + write!(f, "a=fmtp:{pt} ")?; + for (idx, v) in values.iter().enumerate() { + if idx + 1 < values.len() { + write!(f, "{v};")?; + } else { + write!(f, "{v}\r\n")?; + } + } + } + // a=rid:hi send pt=111,112;max-br=64000;max-height=360 + Rid { + id, + direction, + pt, + restriction, + } => { + write!(f, "a=rid:{} {}", id.0, direction)?; + for (idx, p) in pt.iter().enumerate() { + if idx == 0 { + write!(f, " pt=")?; + } + if idx + 1 == pt.len() { + write!(f, "{p}")?; + } else { + write!(f, "{p},")?; + } + } + for (idx, (k, v)) in restriction.iter().enumerate() { + if idx == 0 { + if pt.is_empty() { + write!(f, " ")?; + } else { + write!(f, ";")?; + } + } + if idx + 1 == restriction.len() { + write!(f, "{k}={v}")?; + } else { + write!(f, "{k}={v};")?; + } + } + write!(f, "\r\n")?; + } + // a=simulcast: ;, [same] + Simulcast(x) => { + let self::Simulcast { + send, + recv, + is_munged, + } = x; + assert!( + !(send.0.is_empty() && recv.0.is_empty()), + "Empty a=simulcast" + ); + if *is_munged { + // don't write + return Ok(()); + } + write!(f, "a=simulcast:")?; + if !send.0.is_empty() { + write!(f, "send {send}")?; + } + if !recv.0.is_empty() { + if !send.0.is_empty() { + write!(f, " ")?; + } + write!(f, "recv {recv}")?; + } + write!(f, "\r\n")?; + } + SsrcGroup { semantics, ssrcs } => { + write!(f, "a=ssrc-group:{semantics} ")?; + for (idx, ssrc) in ssrcs.iter().enumerate() { + if idx + 1 < ssrcs.len() { + write!(f, "{} ", **ssrc)?; + } else { + write!(f, "{}\r\n", **ssrc)?; + } + } + } + Ssrc { ssrc, attr, value } => { + write!(f, "a=ssrc:{} {}:{}\r\n", **ssrc, attr, value)?; + } + Unused(v) => write!(f, "a={v}\r\n")?, + } + Ok(()) + } +} + +pub struct FingerprintFmt<'a>(pub &'a [u8]); + +impl<'a> std::fmt::Display for FingerprintFmt<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let last = self.0.len() - 1; + for (idx, b) in self.0.iter().enumerate() { + if idx < last { + write!(f, "{b:02X}:")?; + } else { + write!(f, "{b:02X}")?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use crate::VERSION; + use crate::packet::H265ProfileTierLevel; + use crate::rtp_::{Extension, Frequency}; + + use super::*; + + /// Tests for general format parameter serialization and parsing. + /// These tests verify that format parameters can be correctly converted to/from strings. + mod format_params { + use super::*; + + #[test] + fn fmtp_param_to_string() { + let f = FormatParams { + min_p_time: Some(10), + stereo: Some(true), + use_inband_fec: Some(true), + ..Default::default() + }; + assert_eq!(f.to_string(), "minptime=10;stereo=1;useinbandfec=1"); + } + } + + /// Tests for SDP parsing, structure validation, and serialization. + /// Covers error handling, SDP generation, BUNDLE groups, and media line configuration. + mod sdp_parsing { + use super::*; + + #[test] + fn parse_error() { + let input = "v=0\r\n\ + o=mozilla...THIS_IS_SDPARTA-99.0 7710052215259647220 2 IN IP4 0.0.0.0\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=fingerprint:sha-256 A6:64:23:37:94:7E:4B:40:F6:62:86:8C:DD:09:D5:08:\ +7E:D4:0E:68:58:93:45:EC:99:F2:91:F7:19:72:E7:BB\r\n\ + a=group:BUNDLE 0 hxI i1X mxk B3D kNI nbB xIZ bKm Hkn\r\n\ + a=ice-options:trickle\r\n\ + a=msid-semantic:WMS *\r\n\ + m=audio 0 UDP/TLS/RTP/SAVPF 0\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=setup:actpass\r\n\ + a=mid:1\r\n\ + a=rtpmap:0 PCMU/8000\r\n\ + "; + + let sdp = Sdp::parse(input); + + match sdp { + Err(SdpError::ParseError(out)) => { + assert!(out.starts_with(&"Parse error at ".to_string())); + assert!(out.contains( + &"Expected exactly one of a=sendrecv, a=sendonly, a=recvonly, a=inactive for mid: 1" + .to_string() + )); + } + _ => panic!(), + } + } + + #[test] + fn write_sdp() { + let sdp = Sdp { + session: Session { + id: 5_058_682_828_002_148_772.into(), + bw: None, + attrs: vec![ + SessionAttribute::Group { + typ: "BUNDLE".into(), + mids: vec!["0".into()], + }, + SessionAttribute::Unused( + "msid-semantic: WMS 5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK".into(), + ), + ], + }, + media_lines: vec![ + MediaLine { + typ: MediaType::Audio, + disabled: false, + proto: Proto::Srtp, + pts: vec![111, 103, 104, 9, 0, 8, 106, 105, 13, 110, 112, 113, 126] + .into_iter() + .map(Pt::from) + .collect(), + bw: None, + attrs: vec![ + MediaAttribute::Rtcp("9 IN IP4 0.0.0.0".into()), + MediaAttribute::IceUfrag("S5hk".into()), + MediaAttribute::IcePwd("0zV/Yu3y8aDzbHgqWhnVQhqP".into()), + MediaAttribute::IceOptions("trickle".into()), + MediaAttribute::Fingerprint(Fingerprint { + hash_func: "sha-256".into(), + bytes: vec![ + 140, 100, 237, 3, 118, 208, 61, 180, 136, 8, 145, 100, 8, 128, + 168, 198, 90, 191, 139, 78, 56, 39, 150, 202, 8, 73, 37, 115, + 70, 96, 32, 220, + ], + }), + MediaAttribute::Setup(Setup::ActPass), + MediaAttribute::Mid("0".into()), + MediaAttribute::ExtMap{ id: 1, ext: Extension::AudioLevel }, + MediaAttribute::ExtMap{ id: 2, ext: Extension::AbsoluteSendTime }, + MediaAttribute::ExtMap{ id: 3, ext: Extension::TransportSequenceNumber }, + MediaAttribute::ExtMap{ id: 4, ext: Extension::RtpMid }, + MediaAttribute::ExtMap{ id: 5, ext: Extension::RtpStreamId }, + MediaAttribute::ExtMap{ id: 6, ext: Extension::RepairedRtpStreamId }, + MediaAttribute::SendRecv, + MediaAttribute::Msid(Msid { + stream_id: "5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK".into(), + track_id: "f78dde68-7055-4e20-bb37-433803dd1ed1".into(), + }), + MediaAttribute::RtcpMux, + MediaAttribute::RtpMap { + pt: 111.into(), + value: RtpMap { + codec: "opus".into(), + clock_rate: Frequency::FORTY_EIGHT_KHZ, + channels: Some(2), + }, + }, + MediaAttribute::RtcpFb { pt: 111.into(), value: "transport-cc".into() }, + MediaAttribute::Fmtp { + pt: 111.into(), + values: vec![FormatParam::MinPTime(10), FormatParam::UseInbandFec(true)], + }, + MediaAttribute::Ssrc { + ssrc: 3_948_621_874.into(), + attr: "cname".into(), + value: "xeXs3aE9AOBn00yJ".into(), + }, + MediaAttribute::Ssrc { + ssrc: 3_948_621_874.into(), + attr: "msid".into(), + value: "5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1" + .into(), + }, + MediaAttribute::Ssrc { + ssrc: 3_948_621_874.into(), + attr: "mslabel".into(), + value: "5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK".into(), + }, + MediaAttribute::Ssrc { + ssrc: 3_948_621_874.into(), + attr: "label".into(), + value: "f78dde68-7055-4e20-bb37-433803dd1ed1".into(), + }, + ], + }, + MediaLine { + typ: MediaType::Video, + disabled: false, + proto: Proto::Srtp, + pts: vec![45.into(), 46.into()], + bw: None, + attrs: vec![ + MediaAttribute::Rtcp("9 IN IP4 0.0.0.0".into()), + MediaAttribute::IceUfrag("S5hk".into()), + MediaAttribute::IcePwd("0zV/Yu3y8aDzbHgqWhnVQhqP".into()), + MediaAttribute::IceOptions("trickle".into()), + MediaAttribute::Fingerprint(Fingerprint { + hash_func: "sha-256".into(), + bytes: vec![ + 140, 100, 237, 3, 118, 208, 61, 180, 136, 8, 145, 100, 8, 128, + 168, 198, 90, 191, 139, 78, 56, 39, 150, 202, 8, 73, 37, 115, + 70, 96, 32, 220, + ], + }), + MediaAttribute::Setup(Setup::ActPass), + MediaAttribute::Mid("1".into()), + MediaAttribute::ExtMap{ id: 14, ext: Extension::TransmissionTimeOffset }, + MediaAttribute::ExtMap{ id: 2, ext: Extension::AbsoluteSendTime }, + MediaAttribute::ExtMap{ id: 13, ext: Extension::VideoOrientation }, + MediaAttribute::ExtMap{ id: 3, ext: Extension::TransportSequenceNumber }, + MediaAttribute::ExtMap{ id: 5, ext: Extension::PlayoutDelay }, + MediaAttribute::ExtMap{ id: 6, ext: Extension::VideoContentType }, + MediaAttribute::ExtMap{ id: 7, ext: Extension::VideoTiming }, + MediaAttribute::ExtMap{ id: 8, ext: Extension::ColorSpace }, + MediaAttribute::ExtMap{ id: 4, ext: Extension::RtpMid }, + MediaAttribute::ExtMap{ id: 10, ext: Extension::RtpStreamId }, + MediaAttribute::ExtMap{ id: 11, ext: Extension::RepairedRtpStreamId }, + MediaAttribute::SendRecv, + MediaAttribute::Msid(Msid { + stream_id: "-".into(), + track_id: "4018fd65-ac50-4861-89a4-1f2cc35bbb5e".into(), + }), + MediaAttribute::RtcpMux, + MediaAttribute::RtcpRsize, + MediaAttribute::RtpMap { + pt: 45.into(), + value: RtpMap { + codec: Codec::Av1, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + }, + }, + MediaAttribute::RtcpFb { pt: 45.into(), value: "goog-remb".into() }, + MediaAttribute::RtcpFb { pt: 45.into(), value: "transport-cc".into() }, + MediaAttribute::RtcpFb { pt: 45.into(), value: "ccm fir".into() }, + MediaAttribute::RtcpFb { pt: 45.into(), value: "nack".into() }, + MediaAttribute::RtcpFb { pt: 45.into(), value: "nack pli".into() }, + MediaAttribute::Fmtp { + pt: 45.into(), + values: vec![ + FormatParam::LevelIdx(5), + FormatParam::Profile(0), + FormatParam::Tier(0), + ], + }, + MediaAttribute::RtpMap { + pt: 46.into(), + value: RtpMap { + codec: Codec::Rtx, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + }, + }, + MediaAttribute::Fmtp { pt: 46.into(), values: vec![FormatParam::Apt(45.into())] } + ], + } + ], + }; + assert_eq!( + &format!("{sdp}"), + &format!( + "v=0\r\n\ + o=str0m-{VERSION} 5058682828002148772 2 IN IP4 0.0.0.0\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + a=msid-semantic: WMS 5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=rtcp:9 IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:S5hk\r\n\ + a=ice-pwd:0zV/Yu3y8aDzbHgqWhnVQhqP\r\n\ + a=ice-options:trickle\r\n\ + a=fingerprint:sha-256 8C:64:ED:03:76:D0:3D:B4:88:08:91:64:08:80:A8:C6:\ +5A:BF:8B:4E:38:27:96:CA:08:49:25:73:46:60:20:DC\r\n\ + a=setup:actpass\r\n\ + a=mid:0\r\n\ + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n\ + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n\ + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n\ + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n\ + a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n\ + a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n\ + a=sendrecv\r\n\ + a=msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + a=rtcp-mux\r\n\ + a=rtpmap:111 opus/48000/2\r\n\ + a=rtcp-fb:111 transport-cc\r\n\ + a=fmtp:111 minptime=10;useinbandfec=1\r\n\ + a=ssrc:3948621874 cname:xeXs3aE9AOBn00yJ\r\n\ + a=ssrc:3948621874 msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK \ +f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + a=ssrc:3948621874 mslabel:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK\r\n\ + a=ssrc:3948621874 label:f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 45 46\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=rtcp:9 IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:S5hk\r\n\ + a=ice-pwd:0zV/Yu3y8aDzbHgqWhnVQhqP\r\n\ + a=ice-options:trickle\r\n\ + a=fingerprint:sha-256 8C:64:ED:03:76:D0:3D:B4:88:08:91:64:08:80:A8:C6:\ +5A:BF:8B:4E:38:27:96:CA:08:49:25:73:46:60:20:DC\r\n\ + a=setup:actpass\r\n\ + a=mid:1\r\n\ + a=extmap:14 urn:ietf:params:rtp-hdrext:toffset\r\n\ + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n\ + a=extmap:13 urn:3gpp:video-orientation\r\n\ + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n\ + a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\n\ + a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\n\ + a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\n\ + a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\n\ + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n\ + a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n\ + a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n\ + a=sendrecv\r\n\ + a=msid:- 4018fd65-ac50-4861-89a4-1f2cc35bbb5e\r\n\ + a=rtcp-mux\r\n\ + a=rtcp-rsize\r\n\ + a=rtpmap:45 AV1/90000\r\n\ + a=rtcp-fb:45 goog-remb\r\n\ + a=rtcp-fb:45 transport-cc\r\n\ + a=rtcp-fb:45 ccm fir\r\n\ + a=rtcp-fb:45 nack\r\n\ + a=rtcp-fb:45 nack pli\r\n\ + a=fmtp:45 level-idx=5;profile=0;tier=0\r\n\ + a=rtpmap:46 rtx/90000\r\n\ + a=fmtp:46 apt=45\r\n\ + " + ) + ); + } + + #[test] + fn bundle_mids_returns_correct_mids() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0 1 2\r\n\ + a=msid-semantic:WMS *\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:111 opus/48000/2\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword1234\r\n\ + m=video 0 UDP/TLS/RTP/SAVPF 96\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:1\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 VP8/90000\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword1234\r\n\ + m=application 0 UDP/DTLS/SCTP webrtc-datachannel\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:2\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword1234\r\n\ + a=sctp-port:5000\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + + // bundle_mids should return all MIDs from the BUNDLE group + let mids = sdp.bundle_mids().expect("should have BUNDLE group"); + assert_eq!(mids.len(), 3); + assert_eq!(mids[0], "0".into()); + assert_eq!(mids[1], "1".into()); + assert_eq!(mids[2], "2".into()); + + // The m-lines with port=0 should have disabled=true at parsing level + // (interpretation of whether this means "rejected" happens later) + assert!(!sdp.media_lines[0].disabled, "audio has port=9"); + assert!(sdp.media_lines[1].disabled, "video has port=0"); + assert!(sdp.media_lines[2].disabled, "application has port=0"); + } + } + + /// Opus codec-specific tests. + /// These tests are mainly verify SDP format params. + mod opus_codec { + use super::*; + + #[test] + fn opus_fmtp_round_trip() { + let f = FormatParams { + use_dtx: Some(true), + stereo: Some(true), + sprop_stereo: Some(true), + ..Default::default() + }; + + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, "stereo=1;sprop-stereo=1;usedtx=1"); + + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed, f); + } + + #[test] + fn opus_default_to_mono() { + let f = FormatParams { + ..Default::default() + }; + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, ""); + + let parsed = FormatParams::parse_line(&fmtp_str); + assert!(parsed.stereo.is_none()); + assert_eq!(parsed.stereo.unwrap_or(false), false); + assert_eq!(parsed.sprop_stereo.unwrap_or(false), false); + } + } + + /// Core H.265 (HEVC) codec-specific tests. + /// Tests basic H.265 parameter handling including profile-tier-level parsing, + /// serialization, and format parameter combinations. + mod h265_codec { + use super::*; + + /// Test that H.265 format parameters can be serialized to a string and parsed back + /// without loss of information. Verifies the round-trip conversion works correctly. + #[test] + fn h265_fmtp_round_trip() { + // Test that H.265 fmtp params round-trip correctly + let ptl = H265ProfileTierLevel::new(1, 0, 93).unwrap(); // Main, Main tier, Level 3.1 + + let f = FormatParams { + h265_profile_tier_level: Some(ptl), + ..Default::default() + }; + + // Serialize to string + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, "profile-id=1;tier-flag=0;level-id=93"); + + // Parse back + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.h265_profile_tier_level, Some(ptl)); + } + + /// Test different combinations of H.265 profiles, tiers, and levels to ensure + /// they are all serialized correctly. Covers Main, Main 10, different tiers, and levels. + #[test] + fn h265_different_profiles() { + // Test Main profile (1) + let main = H265ProfileTierLevel::new(1, 0, 93).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(main), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=93"); + + // Test Main 10 profile (2) + let main10 = H265ProfileTierLevel::new(2, 0, 93).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(main10), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=2;tier-flag=0;level-id=93"); + + // Test different tier (High tier) + let high_tier = H265ProfileTierLevel::new(1, 1, 93).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(high_tier), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=1;level-id=93"); + + // Test different level (Level 5.1 = 153) + let level_5_1 = H265ProfileTierLevel::new(1, 0, 153).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(level_5_1), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=153"); + } + + /// Test parsing H.265 parameters from key-value pairs as they would appear in SDP. + /// Verifies that the three separate params (profile-id, tier-flag, level-id) are + /// correctly combined into a single H265ProfileTierLevel composite. + #[test] + fn h265_parse_from_pairs() { + // Test parsing from key-value pairs (as would come from SDP) + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have exactly one H265ProfileTierLevel param + let h265_params: Vec<_> = params + .iter() + .filter(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + .collect(); + assert_eq!(h265_params.len(), 1); + + if let FormatParam::H265ProfileTierLevel(ptl) = h265_params[0] { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 93); + } else { + panic!("Expected H265ProfileTierLevel"); + } + } + + /// Test that H.265 profile-tier-level params can coexist with other H.265-specific + /// parameters like sprop-max-don-diff without interfering with each other. + #[test] + fn h265_mixed_with_other_params() { + // Test H.265 params mixed with other format params + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ("sprop-max-don-diff".to_string(), "0".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have H265ProfileTierLevel and SpropMaxDonDiff + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::SpropMaxDonDiff(0))) + ); + } + + /// Test sprop-max-don-diff parsing, display, and round-trip for various values. + #[test] + fn sprop_max_don_diff_parse_and_display() { + // Parse standalone sprop-max-don-diff + let param = FormatParam::parse("sprop-max-don-diff", "32"); + assert!(matches!(param, FormatParam::SpropMaxDonDiff(32))); + + // Display format + assert_eq!(param.to_string(), "sprop-max-don-diff=32"); + + // Parse value 0 (DONL disabled) + let param0 = FormatParam::parse("sprop-max-don-diff", "0"); + assert!(matches!(param0, FormatParam::SpropMaxDonDiff(0))); + + // Parse max valid value (RFC 7798 §7.1: 0..32767) + let param_max = FormatParam::parse("sprop-max-don-diff", "32767"); + assert!(matches!(param_max, FormatParam::SpropMaxDonDiff(32767))); + + // Out-of-range value (32768) rejected per RFC 7798 §7.1 + let param_over = FormatParam::parse("sprop-max-don-diff", "32768"); + assert!(matches!(param_over, FormatParam::Unknown)); + + // Invalid value falls back to Unknown + let param_bad = FormatParam::parse("sprop-max-don-diff", "notanumber"); + assert!(matches!(param_bad, FormatParam::Unknown)); + } + + /// Test that sprop-max-don-diff > 0 round-trips through H.265 fmtp with PTL params. + #[test] + fn h265_sprop_max_don_diff_nonzero_with_ptl() { + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ("sprop-max-don-diff".to_string(), "32".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have both H265ProfileTierLevel and SpropMaxDonDiff(32) + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::SpropMaxDonDiff(32))) + ); + + // Verify PTL values are correct + if let Some(FormatParam::H265ProfileTierLevel(ptl)) = params + .iter() + .find(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 93); + } + } + } + + /// Tests ensuring H.265 parameters are not confused with other codecs. + /// Verifies that H.265's three-parameter format (profile-id, tier-flag, level-id) + /// is correctly distinguished from H.264's profile-level-id, VP9's profile-id, + /// and AV1's profile/tier/level-idx parameters. + mod no_confusion { + use super::*; + + /// Test that H.264's profile-level-id parameter (single hex value) is not confused + /// with H.265's three separate parameters. H.264 uses a different format entirely. + #[test] + fn h265_no_confusion_with_h264() { + // H.264 uses profile-level-id (single param) + let h264_pairs = vec![ + ("profile-level-id".to_string(), "42e01f".to_string()), + ("packetization-mode".to_string(), "1".to_string()), + ]; + + let h264_params = FormatParam::parse_pairs(h264_pairs); + + // Should NOT have H265ProfileTierLevel + assert!( + !h264_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Should have ProfileLevelId (H.264) + assert!( + h264_params + .iter() + .any(|p| matches!(p, FormatParam::ProfileLevelId(_))) + ); + assert!( + h264_params + .iter() + .any(|p| matches!(p, FormatParam::PacketizationMode(1))) + ); + } + + /// Test that VP9's profile-id parameter (standalone) is not confused with H.265's + /// profile-id which must be combined with tier-flag and level-id. + #[test] + fn h265_no_confusion_with_vp9() { + // VP9 uses profile-id (single param, not combined) + let vp9_pairs = vec![("profile-id".to_string(), "0".to_string())]; + + let vp9_params = FormatParam::parse_pairs(vp9_pairs); + + // Should NOT have H265ProfileTierLevel (missing tier-flag and level-id) + assert!( + !vp9_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Should have ProfileId (VP9) + assert!( + vp9_params + .iter() + .any(|p| matches!(p, FormatParam::ProfileId(0))) + ); + } + + /// Test that AV1's parameters (profile, tier, level-idx) use different naming than + /// H.265 (profile-id, tier-flag, level-id) and are not confused during parsing. + #[test] + fn h265_no_confusion_with_av1() { + // AV1 uses profile, level-idx, tier (but should not be confused with H.265) + let av1_pairs = vec![ + ("profile".to_string(), "0".to_string()), + ("level-idx".to_string(), "5".to_string()), + ("tier".to_string(), "0".to_string()), + ]; + + let av1_params = FormatParam::parse_pairs(av1_pairs); + + // Should NOT have H265ProfileTierLevel + // (AV1 uses different param names: profile vs profile-id, + // tier vs tier-flag, level-idx vs level-id) + assert!( + !av1_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Should have AV1-specific params + assert!( + av1_params + .iter() + .any(|p| matches!(p, FormatParam::Profile(0))) + ); + assert!( + av1_params + .iter() + .any(|p| matches!(p, FormatParam::LevelIdx(5))) + ); + assert!(av1_params.iter().any(|p| matches!(p, FormatParam::Tier(0)))); + } + } + + /// Additional H.265 parsing and validation tests. + /// Covers incomplete parameter handling and complete SDP integration scenarios. + mod h265_additional { + use super::*; + + /// Test that incomplete H.265 parameter sets do not create a composite ProfileTierLevel. + /// All three parameters (profile-id, tier-flag, level-id) must be present. + #[test] + fn h265_partial_params_no_composite() { + // If only some H.265 params are present, should not create composite + let partial_pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + // Missing level-id + ]; + + let params = FormatParam::parse_pairs(partial_pairs); + + // Should NOT have H265ProfileTierLevel (incomplete) + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + } + + /// Test H.265 parameters in a complete SDP offer/answer scenario. + /// Verifies that H.265 rtpmap and fmtp lines are correctly parsed along with RTX. + #[test] + fn h265_in_complete_sdp() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H265/90000\r\n\ + a=fmtp:96 profile-id=1;tier-flag=0;level-id=93\r\n\ + a=rtpmap:97 rtx/90000\r\n\ + a=fmtp:97 apt=96\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + + // Get RTP params + let params = sdp.media_lines[0].rtp_params(); + + // Find H.265 payload + let h265_payload = params + .iter() + .find(|p| p.spec.codec == Codec::H265) + .expect("should have H.265"); + + assert_eq!(h265_payload.pt, 96.into()); + assert_eq!(h265_payload.spec.clock_rate, Frequency::NINETY_KHZ); + + // Check H.265 params + let ptl = h265_payload + .spec + .format + .h265_profile_tier_level + .expect("should have H.265 PTL"); + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 93); + + // Check RTX + assert_eq!(h265_payload.resend, Some(97.into())); + } + } + + /// Multi-codec integration tests. + /// Verifies that multiple video codecs (H.264, H.265, VP8, VP9) can coexist + /// in the same SDP without parameter conflicts or interference. + mod multi_codec { + use super::*; + + /// Test that H.264, H.265, and other video codecs can coexist in the same SDP + /// without their format parameters interfering with each other. + #[test] + fn h265_and_h264_coexist() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97 98\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H264/90000\r\n\ + a=fmtp:96 profile-level-id=42e01f;packetization-mode=1\r\n\ + a=rtpmap:97 H265/90000\r\n\ + a=fmtp:97 profile-id=1;tier-flag=0;level-id=93\r\n\ + a=rtpmap:98 VP8/90000\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + // Check H.264 + let h264 = params + .iter() + .find(|p| p.spec.codec == Codec::H264) + .expect("should have H.264"); + assert_eq!(h264.pt, 96.into()); + assert_eq!(h264.spec.format.profile_level_id, Some(0x42e01f)); + assert_eq!(h264.spec.format.packetization_mode, Some(1)); + assert!(h264.spec.format.h265_profile_tier_level.is_none()); + + // Check H.265 + let h265 = params + .iter() + .find(|p| p.spec.codec == Codec::H265) + .expect("should have H.265"); + assert_eq!(h265.pt, 97.into()); + let ptl = h265 + .spec + .format + .h265_profile_tier_level + .expect("should have H.265 PTL"); + assert_eq!(ptl.profile_id(), 1); + assert!(h265.spec.format.profile_level_id.is_none()); + + // Check VP8 + let vp8 = params + .iter() + .find(|p| p.spec.codec == Codec::Vp8) + .expect("should have VP8"); + assert_eq!(vp8.pt, 98.into()); + assert!(vp8.spec.format.h265_profile_tier_level.is_none()); + assert!(vp8.spec.format.profile_level_id.is_none()); + } + } + + /// H.265 serialization and edge case tests. + /// Tests media attribute generation, empty parameter handling, extreme values, + /// parameter ordering independence, DONL parameters, and invalid value handling. + mod h265_serialization { + use super::*; + + /// Test that H.265 parameters are correctly serialized as media attributes, + /// including rtpmap, fmtp with ProfileTierLevel, rtcp-fb, and RTX payload. + #[test] + fn h265_serialization_in_media_attrs() { + let ptl = H265ProfileTierLevel::new(2, 1, 153).unwrap(); // Main 10, High tier, Level 5.1 + + let mut payload = PayloadParams::new( + 100.into(), + Some(101.into()), + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h265_profile_tier_level: Some(ptl), + ..Default::default() + }, + }, + ); + payload.fb_transport_cc = true; + payload.fb_nack = true; + payload.fb_pli = true; + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Should have rtpmap for H.265 + assert!(attrs.iter().any(|a| matches!(a, + MediaAttribute::RtpMap { pt, value } + if *pt == 100.into() && value.codec == Codec::H265 + ))); + + // Should have fmtp with H.265 params + let fmtp = attrs.iter().find(|a| { + matches!(a, + MediaAttribute::Fmtp { pt, .. } if *pt == 100.into() + ) + }); + assert!(fmtp.is_some()); + + if let Some(MediaAttribute::Fmtp { values, .. }) = fmtp { + assert!( + values + .iter() + .any(|v| matches!(v, FormatParam::H265ProfileTierLevel(p) + if p.profile_id() == 2 && p.tier_flag() == 1 && p.level_id() == 153 + )) + ); + } + + // Should have rtx mapping + assert!(attrs.iter().any(|a| matches!(a, + MediaAttribute::RtpMap { pt, value } + if *pt == 101.into() && value.codec == Codec::Rtx + ))); + } + + /// Test that empty format parameters do not generate an a=fmtp line in SDP, + /// avoiding invalid SDP syntax with empty fmtp attributes. + #[test] + fn h265_empty_fmtp_not_serialized() { + // Empty fmtp should not be written + let payload = PayloadParams::new( + 100.into(), + None, + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + ); + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Should NOT have fmtp line (it would be empty) + assert!( + !attrs + .iter() + .any(|a| matches!(a, MediaAttribute::Fmtp { .. })) + ); + } + + /// Test H.265 with extreme profile/tier/level values at boundaries. + /// Verifies parsing and serialization of edge case values. + #[test] + fn h265_extreme_values() { + // Test minimum level (Level 1.0 = 30) + let min_level = H265ProfileTierLevel::new(1, 0, 30).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(min_level), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=30"); + + // Test maximum level (Level 6.2 = 186) + let max_level = H265ProfileTierLevel::new(1, 0, 186).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(max_level), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=186"); + + // Test Main10 with High tier + let main10_high = H265ProfileTierLevel::new(2, 1, 93).unwrap(); + let f = FormatParams { + h265_profile_tier_level: Some(main10_high), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=2;tier-flag=1;level-id=93"); + } + + /// Test H.265 parameter ordering doesn't affect parsing. + /// SDP parameters can appear in any order. + #[test] + fn h265_parameter_order_independence() { + // Normal order + let normal_order = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + + // Reversed order + let reversed_order = vec![ + ("level-id".to_string(), "93".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("profile-id".to_string(), "1".to_string()), + ]; + + // Mixed order + let mixed_order = vec![ + ("tier-flag".to_string(), "0".to_string()), + ("profile-id".to_string(), "1".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + + let normal_params = FormatParam::parse_pairs(normal_order); + let reversed_params = FormatParam::parse_pairs(reversed_order); + let mixed_params = FormatParam::parse_pairs(mixed_order); + + // All should create H265ProfileTierLevel + assert!( + normal_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + assert!( + reversed_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + assert!( + mixed_params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // All should have the same values + for params in [&normal_params, &reversed_params, &mixed_params] { + if let Some(FormatParam::H265ProfileTierLevel(ptl)) = params + .iter() + .find(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 93); + } + } + } + + #[test] + fn h265_invalid_parameter_values() { + // Invalid profile-id + let invalid_profile = vec![ + ("profile-id".to_string(), "999".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Invalid tier-flag + let invalid_tier = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "5".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_tier); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Invalid level-id + let invalid_level = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "999".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_level); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + } + } + + /// H.265 integration with RTCP feedback mechanisms. + /// Tests H.265 behavior with multiple codecs and various RTCP feedback types + /// including NACK, PLI, FIR, and transport-cc. + mod h265_integration { + use super::*; + + /// Test H.265 in SDP with multiple video codecs and feedback mechanisms. + /// Verifies proper parsing of complex multi-codec scenarios. + #[test] + fn h265_multi_codec_with_feedback() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H265/90000\r\n\ + a=fmtp:96 profile-id=1;tier-flag=0;level-id=93\r\n\ + a=rtcp-fb:96 nack\r\n\ + a=rtcp-fb:96 nack pli\r\n\ + a=rtcp-fb:96 ccm fir\r\n\ + a=rtcp-fb:96 transport-cc\r\n\ + a=rtpmap:97 rtx/90000\r\n\ + a=fmtp:97 apt=96\r\n\ + a=rtpmap:98 VP9/90000\r\n\ + a=fmtp:98 profile-id=0\r\n\ + a=rtpmap:99 rtx/90000\r\n\ + a=fmtp:99 apt=98\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + // Check H.265 with all feedback mechanisms + let h265 = params + .iter() + .find(|p| p.spec.codec == Codec::H265) + .expect("should have H.265"); + assert_eq!(h265.pt, 96.into()); + assert!(h265.fb_nack); + assert!(h265.fb_pli); + assert!(h265.fb_fir); + assert!(h265.fb_transport_cc); + assert_eq!(h265.resend, Some(97.into())); + + // Verify H.265 PTL is correct + let ptl = h265 + .spec + .format + .h265_profile_tier_level + .expect("should have PTL"); + assert_eq!(ptl.profile_id(), 1); + + // Check VP9 is separate and correct + let vp9 = params + .iter() + .find(|p| p.spec.codec == Codec::Vp9) + .expect("should have VP9"); + assert_eq!(vp9.pt, 98.into()); + assert_eq!(vp9.spec.format.profile_id, Some(0)); + assert!(vp9.spec.format.h265_profile_tier_level.is_none()); + } + } + + /// Advanced H.265 functionality tests. + /// Covers SDP round-trip serialization, incomplete parameter permutations, + /// browser compatibility (Chrome profile-only mode), and validation of all + /// standard H.265 level values (1.0 through 6.2). + mod h265_advanced { + use super::*; + + /// Test H.265 SDP serialization maintains parameter integrity. + /// Ensures that serialized SDP can be parsed back with identical values. + #[test] + fn h265_sdp_serialization_round_trip() { + let ptl = H265ProfileTierLevel::new(2, 0, 153).unwrap(); + + let payload = PayloadParams::new( + 100.into(), + Some(101.into()), + CodecSpec { + codec: Codec::H265, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h265_profile_tier_level: Some(ptl), + ..Default::default() + }, + }, + ); + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Find the fmtp line + let fmtp = attrs + .iter() + .find_map(|a| match a { + MediaAttribute::Fmtp { pt, values } if *pt == 100.into() => Some(values), + _ => None, + }) + .expect("should have fmtp"); + + // Should contain H265ProfileTierLevel + assert!( + fmtp.iter().any( + |v| matches!(v, FormatParam::H265ProfileTierLevel(p) if p.profile_id() == 2) + ) + ); + } + + /// Test that only two of three H.265 params doesn't create PTL. + /// Each permutation of missing parameter should fail. + #[test] + fn h265_incomplete_params_all_permutations() { + // Missing level-id + let missing_level = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_level); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Missing tier-flag + let missing_tier = vec![ + ("profile-id".to_string(), "1".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_tier); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Missing profile-id + let missing_profile = vec![ + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "93".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + + // Only one parameter + let only_profile = vec![("profile-id".to_string(), "1".to_string())]; + let params = FormatParam::parse_pairs(only_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + ); + } + + /// Test H.265 with Chrome-style profile-only in complete SDP. + /// Verifies that browser compatibility mode works in real SDP. + #[test] + fn h265_chrome_profile_only_sdp() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H265/90000\r\n\ + a=fmtp:96 profile-id=1\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + let h265 = params + .iter() + .find(|p| p.spec.codec == Codec::H265) + .expect("should have H.265"); + + // Should NOT have full PTL + assert!(h265.spec.format.h265_profile_tier_level.is_none()); + + // Should have profile_id field instead + assert_eq!(h265.spec.format.profile_id, Some(1)); + } + + /// Test H.265 different level values in valid range. + /// Ensures all valid level IDs are properly handled. + #[test] + fn h265_all_valid_levels() { + let valid_levels = vec![ + 30, // Level 1.0 + 60, // Level 2.0 + 63, // Level 2.1 + 90, // Level 3.0 + 93, // Level 3.1 + 120, // Level 4.0 + 123, // Level 4.1 + 150, // Level 5.0 + 153, // Level 5.1 + 156, // Level 5.2 + 180, // Level 6.0 + 183, // Level 6.1 + 186, // Level 6.2 + ]; + + for level_id in valid_levels { + let ptl = H265ProfileTierLevel::new(1, 0, level_id); + assert!(ptl.is_some(), "Level ID {} should be valid", level_id); + + let f = FormatParams { + h265_profile_tier_level: ptl, + ..Default::default() + }; + + let serialized = f.to_string(); + assert!(serialized.contains(&format!("level-id={}", level_id))); + } + } + } + + /// Core H.266 (VVC) codec-specific tests, mirrored from h265_codec. + /// Tests basic H.266 parameter handling including profile-tier-level parsing, + /// serialization, and format parameter combinations. + mod h266_codec { + use super::*; + + /// Test that H.266 format parameters can be serialized to a string and parsed back + /// without loss of information. Verifies the round-trip conversion works correctly. + #[test] + fn h266_fmtp_round_trip() { + // Test that H.266 fmtp params round-trip correctly + let ptl = H266ProfileTierLevel::new(1, 0, 51).unwrap(); // Main, Main tier, Level 3.1 + + let f = FormatParams { + h266_profile_tier_level: Some(ptl), + ..Default::default() + }; + + // Serialize to string + let fmtp_str = f.to_string(); + assert_eq!(fmtp_str, "profile-id=1;tier-flag=0;level-id=51"); + + // Parse back + let parsed = FormatParams::parse_line(&fmtp_str); + assert_eq!(parsed.h266_profile_tier_level, Some(ptl)); + } + + /// Test different combinations of H.266 profiles, tiers, and levels to ensure + /// they are all serialized correctly. Covers Main, Main 10, different tiers, and levels. + #[test] + fn h266_different_profiles() { + // Test Main profile (1) + let main = H266ProfileTierLevel::new(1, 0, 51).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(main), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=51"); + + // Test Main 10 profile (2) + let main10 = H266ProfileTierLevel::new(33, 0, 51).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(main10), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=33;tier-flag=0;level-id=51"); + + // Test different tier (High tier) + let high_tier = H266ProfileTierLevel::new(1, 1, 51).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(high_tier), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=1;level-id=51"); + + // Test different level (Level 5.1 = 153) + let level_5_1 = H266ProfileTierLevel::new(1, 0, 83).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(level_5_1), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=83"); + } + + /// Test parsing H.266 parameters from key-value pairs as they would appear in SDP. + /// Verifies that the three separate params (profile-id, tier-flag, level-id) are + /// correctly combined into a single H266ProfileTierLevel composite. + #[test] + fn h266_parse_from_pairs() { + // Test parsing from key-value pairs (as would come from SDP) + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have exactly one H266ProfileTierLevel param + let h266_params: Vec<_> = params + .iter() + .filter(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + .collect(); + assert_eq!(h266_params.len(), 1); + + if let FormatParam::H266ProfileTierLevel(ptl) = h266_params[0] { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 51); + } else { + panic!("Expected H266ProfileTierLevel"); + } + } + + /// Test that H.266 profile-tier-level params can coexist with other H.266-specific + /// parameters like sprop-max-don-diff without interfering with each other. + #[test] + fn h266_mixed_with_other_params() { + // Test H.266 params mixed with other format params + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ("sprop-max-don-diff".to_string(), "0".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have H266ProfileTierLevel and SpropMaxDonDiff + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::SpropMaxDonDiff(0))) + ); + } + + /// Test sprop-max-don-diff parsing, display, and round-trip for various values. + #[test] + fn sprop_max_don_diff_parse_and_display() { + // Parse standalone sprop-max-don-diff + let param = FormatParam::parse("sprop-max-don-diff", "32"); + assert!(matches!(param, FormatParam::SpropMaxDonDiff(32))); + + // Display format + assert_eq!(param.to_string(), "sprop-max-don-diff=32"); + + // Parse value 0 (DONL disabled) + let param0 = FormatParam::parse("sprop-max-don-diff", "0"); + assert!(matches!(param0, FormatParam::SpropMaxDonDiff(0))); + + // Parse max valid value (RFC 9328 §7.2: 0..32767) + let param_max = FormatParam::parse("sprop-max-don-diff", "32767"); + assert!(matches!(param_max, FormatParam::SpropMaxDonDiff(32767))); + + // Out-of-range value (32768) rejected per RFC 9328 §7.2 + let param_over = FormatParam::parse("sprop-max-don-diff", "32768"); + assert!(matches!(param_over, FormatParam::Unknown)); + + // Invalid value falls back to Unknown + let param_bad = FormatParam::parse("sprop-max-don-diff", "notanumber"); + assert!(matches!(param_bad, FormatParam::Unknown)); + } + + /// Test that sprop-max-don-diff > 0 round-trips through H.266 fmtp with PTL params. + #[test] + fn h266_sprop_max_don_diff_nonzero_with_ptl() { + let pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ("sprop-max-don-diff".to_string(), "32".to_string()), + ]; + + let params = FormatParam::parse_pairs(pairs); + + // Should have both H266ProfileTierLevel and SpropMaxDonDiff(32) + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + assert!( + params + .iter() + .any(|p| matches!(p, FormatParam::SpropMaxDonDiff(32))) + ); + + // Verify PTL values are correct + if let Some(FormatParam::H266ProfileTierLevel(ptl)) = params + .iter() + .find(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 51); + } + } + } + + /// Tests ensuring H.266 parameters are not confused with other codecs. + /// Verifies that H.266's three-parameter format (profile-id, tier-flag, level-id) + /// is correctly distinguished from H.264's profile-level-id, VP9's profile-id, + /// and AV1's profile/tier/level-idx parameters. + mod no_confusion_h266 { + use super::*; + + /// Test that H.264's profile-level-id parameter (single hex value) is not confused + /// with H.266's three separate parameters. H.264 uses a different format entirely. + #[test] + fn h266_no_confusion_with_h264() { + // H.264 uses profile-level-id (single param) + let h264_pairs = vec![ + ("profile-level-id".to_string(), "42e01f".to_string()), + ("packetization-mode".to_string(), "1".to_string()), + ]; + + let h264_params = FormatParam::parse_pairs(h264_pairs); + + // Should NOT have H266ProfileTierLevel + assert!( + !h264_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Should have ProfileLevelId (H.264) + assert!( + h264_params + .iter() + .any(|p| matches!(p, FormatParam::ProfileLevelId(_))) + ); + assert!( + h264_params + .iter() + .any(|p| matches!(p, FormatParam::PacketizationMode(1))) + ); + } + + /// Test that VP9's profile-id parameter (standalone) is not confused with H.266's + /// profile-id which must be combined with tier-flag and level-id. + #[test] + fn h266_no_confusion_with_vp9() { + // VP9 uses profile-id (single param, not combined) + let vp9_pairs = vec![("profile-id".to_string(), "0".to_string())]; + + let vp9_params = FormatParam::parse_pairs(vp9_pairs); + + // Should NOT have H266ProfileTierLevel (missing tier-flag and level-id) + assert!( + !vp9_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Should have ProfileId (VP9) + assert!( + vp9_params + .iter() + .any(|p| matches!(p, FormatParam::ProfileId(0))) + ); + } + + /// Test that AV1's parameters (profile, tier, level-idx) use different naming than + /// H.266 (profile-id, tier-flag, level-id) and are not confused during parsing. + #[test] + fn h266_no_confusion_with_av1() { + // AV1 uses profile, level-idx, tier (but should not be confused with H.266) + let av1_pairs = vec![ + ("profile".to_string(), "0".to_string()), + ("level-idx".to_string(), "5".to_string()), + ("tier".to_string(), "0".to_string()), + ]; + + let av1_params = FormatParam::parse_pairs(av1_pairs); + + // Should NOT have H266ProfileTierLevel + // (AV1 uses different param names: profile vs profile-id, + // tier vs tier-flag, level-idx vs level-id) + assert!( + !av1_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Should have AV1-specific params + assert!( + av1_params + .iter() + .any(|p| matches!(p, FormatParam::Profile(0))) + ); + assert!( + av1_params + .iter() + .any(|p| matches!(p, FormatParam::LevelIdx(5))) + ); + assert!(av1_params.iter().any(|p| matches!(p, FormatParam::Tier(0)))); + } + } + + /// Additional H.266 parsing and validation tests. + /// Covers incomplete parameter handling and complete SDP integration scenarios. + mod h266_additional { + use super::*; + + /// Test that incomplete H.266 parameter sets do not create a composite ProfileTierLevel. + /// All three parameters (profile-id, tier-flag, level-id) must be present. + #[test] + fn h266_partial_params_no_composite() { + // If only some H.266 params are present, should not create composite + let partial_pairs = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + // Missing level-id + ]; + + let params = FormatParam::parse_pairs(partial_pairs); + + // Should NOT have H266ProfileTierLevel (incomplete) + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + } + + /// Test H.266 parameters in a complete SDP offer/answer scenario. + /// Verifies that H.266 rtpmap and fmtp lines are correctly parsed along with RTX. + #[test] + fn h266_in_complete_sdp() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H266/90000\r\n\ + a=fmtp:96 profile-id=1;tier-flag=0;level-id=51\r\n\ + a=rtpmap:97 rtx/90000\r\n\ + a=fmtp:97 apt=96\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + + // Get RTP params + let params = sdp.media_lines[0].rtp_params(); + + // Find H.266 payload + let h266_payload = params + .iter() + .find(|p| p.spec.codec == Codec::H266) + .expect("should have H.266"); + + assert_eq!(h266_payload.pt, 96.into()); + assert_eq!(h266_payload.spec.clock_rate, Frequency::NINETY_KHZ); + + // Check H.266 params + let ptl = h266_payload + .spec + .format + .h266_profile_tier_level + .expect("should have H.266 PTL"); + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 51); + + // Check RTX + assert_eq!(h266_payload.resend, Some(97.into())); + } + } + + /// Multi-codec integration tests. + /// Verifies that multiple video codecs (H.264, H.266, VP8, VP9) can coexist + /// in the same SDP without parameter conflicts or interference. + mod multi_codec_h266 { + use super::*; + + /// Test that H.264, H.266, and other video codecs can coexist in the same SDP + /// without their format parameters interfering with each other. + #[test] + fn h266_and_h264_coexist() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97 98\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H264/90000\r\n\ + a=fmtp:96 profile-level-id=42e01f;packetization-mode=1\r\n\ + a=rtpmap:97 H266/90000\r\n\ + a=fmtp:97 profile-id=1;tier-flag=0;level-id=51\r\n\ + a=rtpmap:98 VP8/90000\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + // Check H.264 + let h264 = params + .iter() + .find(|p| p.spec.codec == Codec::H264) + .expect("should have H.264"); + assert_eq!(h264.pt, 96.into()); + assert_eq!(h264.spec.format.profile_level_id, Some(0x42e01f)); + assert_eq!(h264.spec.format.packetization_mode, Some(1)); + assert!(h264.spec.format.h266_profile_tier_level.is_none()); + + // Check H.266 + let h266 = params + .iter() + .find(|p| p.spec.codec == Codec::H266) + .expect("should have H.266"); + assert_eq!(h266.pt, 97.into()); + let ptl = h266 + .spec + .format + .h266_profile_tier_level + .expect("should have H.266 PTL"); + assert_eq!(ptl.profile_id(), 1); + assert!(h266.spec.format.profile_level_id.is_none()); + + // Check VP8 + let vp8 = params + .iter() + .find(|p| p.spec.codec == Codec::Vp8) + .expect("should have VP8"); + assert_eq!(vp8.pt, 98.into()); + assert!(vp8.spec.format.h266_profile_tier_level.is_none()); + assert!(vp8.spec.format.profile_level_id.is_none()); + } + } + + /// H.266 serialization and edge case tests. + /// Tests media attribute generation, empty parameter handling, extreme values, + /// parameter ordering independence, DONL parameters, and invalid value handling. + mod h266_serialization { + use super::*; + + /// Test that H.266 parameters are correctly serialized as media attributes, + /// including rtpmap, fmtp with ProfileTierLevel, rtcp-fb, and RTX payload. + #[test] + fn h266_serialization_in_media_attrs() { + let ptl = H266ProfileTierLevel::new(33, 1, 83).unwrap(); // Main 10, High tier, Level 5.1 + + let mut payload = PayloadParams::new( + 100.into(), + Some(101.into()), + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h266_profile_tier_level: Some(ptl), + ..Default::default() + }, + }, + ); + payload.fb_transport_cc = true; + payload.fb_nack = true; + payload.fb_pli = true; + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Should have rtpmap for H.266 + assert!(attrs.iter().any(|a| matches!(a, + MediaAttribute::RtpMap { pt, value } + if *pt == 100.into() && value.codec == Codec::H266 + ))); + + // Should have fmtp with H.266 params + let fmtp = attrs.iter().find(|a| { + matches!(a, + MediaAttribute::Fmtp { pt, .. } if *pt == 100.into() + ) + }); + assert!(fmtp.is_some()); + + if let Some(MediaAttribute::Fmtp { values, .. }) = fmtp { + assert!( + values + .iter() + .any(|v| matches!(v, FormatParam::H266ProfileTierLevel(p) + if p.profile_id() == 33 && p.tier_flag() == 1 && p.level_id() == 83 + )) + ); + } + + // Should have rtx mapping + assert!(attrs.iter().any(|a| matches!(a, + MediaAttribute::RtpMap { pt, value } + if *pt == 101.into() && value.codec == Codec::Rtx + ))); + } + + /// Test that empty format parameters do not generate an a=fmtp line in SDP, + /// avoiding invalid SDP syntax with empty fmtp attributes. + #[test] + fn h266_empty_fmtp_not_serialized() { + // Empty fmtp should not be written + let payload = PayloadParams::new( + 100.into(), + None, + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams::default(), + }, + ); + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Should NOT have fmtp line (it would be empty) + assert!( + !attrs + .iter() + .any(|a| matches!(a, MediaAttribute::Fmtp { .. })) + ); + } + + /// Test H.266 with extreme profile/tier/level values at boundaries. + /// Verifies parsing and serialization of edge case values. + #[test] + fn h266_extreme_values() { + // Test minimum level (Level 1.0 = 30) + let min_level = H266ProfileTierLevel::new(1, 0, 16).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(min_level), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=16"); + + // Test maximum level (Level 6.2 = 186) + let max_level = H266ProfileTierLevel::new(1, 0, 102).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(max_level), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=1;tier-flag=0;level-id=102"); + + // Test Main10 with High tier + let main10_high = H266ProfileTierLevel::new(33, 1, 51).unwrap(); + let f = FormatParams { + h266_profile_tier_level: Some(main10_high), + ..Default::default() + }; + assert_eq!(f.to_string(), "profile-id=33;tier-flag=1;level-id=51"); + } + + /// Test H.266 parameter ordering doesn't affect parsing. + /// SDP parameters can appear in any order. + #[test] + fn h266_parameter_order_independence() { + // Normal order + let normal_order = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + + // Reversed order + let reversed_order = vec![ + ("level-id".to_string(), "51".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("profile-id".to_string(), "1".to_string()), + ]; + + // Mixed order + let mixed_order = vec![ + ("tier-flag".to_string(), "0".to_string()), + ("profile-id".to_string(), "1".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + + let normal_params = FormatParam::parse_pairs(normal_order); + let reversed_params = FormatParam::parse_pairs(reversed_order); + let mixed_params = FormatParam::parse_pairs(mixed_order); + + // All should create H266ProfileTierLevel + assert!( + normal_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + assert!( + reversed_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + assert!( + mixed_params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // All should have the same values + for params in [&normal_params, &reversed_params, &mixed_params] { + if let Some(FormatParam::H266ProfileTierLevel(ptl)) = params + .iter() + .find(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 51); + } + } + } + + #[test] + fn h266_invalid_parameter_values() { + // Invalid profile-id + let invalid_profile = vec![ + ("profile-id".to_string(), "999".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Invalid tier-flag + let invalid_tier = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "5".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_tier); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Invalid level-id + let invalid_level = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "999".to_string()), + ]; + let params = FormatParam::parse_pairs(invalid_level); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + } + } + + /// H.266 integration with RTCP feedback mechanisms. + /// Tests H.266 behavior with multiple codecs and various RTCP feedback types + /// including NACK, PLI, FIR, and transport-cc. + mod h266_integration { + use super::*; + + /// Test H.266 in SDP with multiple video codecs and feedback mechanisms. + /// Verifies proper parsing of complex multi-codec scenarios. + #[test] + fn h266_multi_codec_with_feedback() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H266/90000\r\n\ + a=fmtp:96 profile-id=1;tier-flag=0;level-id=51\r\n\ + a=rtcp-fb:96 nack\r\n\ + a=rtcp-fb:96 nack pli\r\n\ + a=rtcp-fb:96 ccm fir\r\n\ + a=rtcp-fb:96 transport-cc\r\n\ + a=rtpmap:97 rtx/90000\r\n\ + a=fmtp:97 apt=96\r\n\ + a=rtpmap:98 VP9/90000\r\n\ + a=fmtp:98 profile-id=0\r\n\ + a=rtpmap:99 rtx/90000\r\n\ + a=fmtp:99 apt=98\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + // Check H.266 with all feedback mechanisms + let h266 = params + .iter() + .find(|p| p.spec.codec == Codec::H266) + .expect("should have H.266"); + assert_eq!(h266.pt, 96.into()); + assert!(h266.fb_nack); + assert!(h266.fb_pli); + assert!(h266.fb_fir); + assert!(h266.fb_transport_cc); + assert_eq!(h266.resend, Some(97.into())); + + // Verify H.266 PTL is correct + let ptl = h266 + .spec + .format + .h266_profile_tier_level + .expect("should have PTL"); + assert_eq!(ptl.profile_id(), 1); + + // Check VP9 is separate and correct + let vp9 = params + .iter() + .find(|p| p.spec.codec == Codec::Vp9) + .expect("should have VP9"); + assert_eq!(vp9.pt, 98.into()); + assert_eq!(vp9.spec.format.profile_id, Some(0)); + assert!(vp9.spec.format.h266_profile_tier_level.is_none()); + } + } + + /// Advanced H.266 functionality tests. + /// Covers SDP round-trip serialization, incomplete parameter permutations, + /// browser compatibility (Chrome profile-only mode), and validation of all + /// standard H.266 level values (1.0 through 6.2). + mod h266_advanced { + use super::*; + + /// Test H.266 SDP serialization maintains parameter integrity. + /// Ensures that serialized SDP can be parsed back with identical values. + #[test] + fn h266_sdp_serialization_round_trip() { + let ptl = H266ProfileTierLevel::new(33, 0, 83).unwrap(); + + let payload = PayloadParams::new( + 100.into(), + Some(101.into()), + CodecSpec { + codec: Codec::H266, + clock_rate: Frequency::NINETY_KHZ, + channels: None, + format: FormatParams { + h266_profile_tier_level: Some(ptl), + ..Default::default() + }, + }, + ); + + let mut attrs = vec![]; + payload.as_media_attrs(&mut attrs); + + // Find the fmtp line + let fmtp = attrs + .iter() + .find_map(|a| match a { + MediaAttribute::Fmtp { pt, values } if *pt == 100.into() => Some(values), + _ => None, + }) + .expect("should have fmtp"); + + // Should contain H266ProfileTierLevel + assert!(fmtp.iter().any( + |v| matches!(v, FormatParam::H266ProfileTierLevel(p) if p.profile_id() == 33) + )); + } + + /// Test that only two of three H.266 params doesn't create PTL. + /// Each permutation of missing parameter should fail. + #[test] + fn h266_incomplete_params_all_permutations() { + // Missing level-id + let missing_level = vec![ + ("profile-id".to_string(), "1".to_string()), + ("tier-flag".to_string(), "0".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_level); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Missing tier-flag + let missing_tier = vec![ + ("profile-id".to_string(), "1".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_tier); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Missing profile-id + let missing_profile = vec![ + ("tier-flag".to_string(), "0".to_string()), + ("level-id".to_string(), "51".to_string()), + ]; + let params = FormatParam::parse_pairs(missing_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + + // Only one parameter + let only_profile = vec![("profile-id".to_string(), "1".to_string())]; + let params = FormatParam::parse_pairs(only_profile); + assert!( + !params + .iter() + .any(|p| matches!(p, FormatParam::H266ProfileTierLevel(_))) + ); + } + + /// Test H.266 with Chrome-style profile-only in complete SDP. + /// Verifies that browser compatibility mode works in real SDP. + #[test] + fn h266_chrome_profile_only_sdp() { + let input = "v=0\r\n\ + o=- 123456 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 96\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=mid:0\r\n\ + a=sendrecv\r\n\ + a=rtpmap:96 H266/90000\r\n\ + a=fmtp:96 profile-id=1\r\n\ + a=setup:actpass\r\n\ + a=ice-ufrag:test\r\n\ + a=ice-pwd:testpassword\r\n\ + "; + + let sdp = Sdp::parse(input).expect("should parse"); + let params = sdp.media_lines[0].rtp_params(); + + let h266 = params + .iter() + .find(|p| p.spec.codec == Codec::H266) + .expect("should have H.266"); + + // Should NOT have full PTL + assert!(h266.spec.format.h266_profile_tier_level.is_none()); + + // Should have profile_id field instead + assert_eq!(h266.spec.format.profile_id, Some(1)); + } + + /// Test H.266 different level values in valid range. + /// Ensures all valid level IDs are properly handled. + #[test] + fn h266_all_valid_levels() { + let valid_levels = vec![ + 16, // Level 1.0 + 32, // Level 2.0 + 35, // Level 2.1 + 48, // Level 3.0 + 51, // Level 3.1 + 64, // Level 4.0 + 67, // Level 4.1 + 80, // Level 5.0 + 83, // Level 5.1 + 86, // Level 5.2 + 96, // Level 6.0 + 99, // Level 6.1 + 102, // Level 6.2 + ]; + + for level_id in valid_levels { + let ptl = H266ProfileTierLevel::new(1, 0, level_id); + assert!(ptl.is_some(), "Level ID {} should be valid", level_id); + + let f = FormatParams { + h266_profile_tier_level: ptl, + ..Default::default() + }; + + let serialized = f.to_string(); + assert!(serialized.contains(&format!("level-id={}", level_id))); + } + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/error.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/error.rs new file mode 100644 index 000000000..71a0f7470 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/error.rs @@ -0,0 +1,21 @@ +use std::error::Error; +use std::fmt; + +/// Errors from parsing and serializing SDP. +#[derive(Debug)] +#[allow(missing_docs)] +pub enum SdpError { + ParseError(String), + Inconsistent(String), +} + +impl fmt::Display for SdpError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SdpError::ParseError(msg) => write!(f, "SDP parse: {}", msg), + SdpError::Inconsistent(msg) => write!(f, "SDP inconsistent: {}", msg), + } + } +} + +impl Error for SdpError {} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/mod.rs new file mode 100644 index 000000000..8645fd00e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/mod.rs @@ -0,0 +1,201 @@ +use std::fmt; +use std::ops::Deref; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; + +mod data; +#[cfg(test)] +pub(crate) use data::RtpMap; +pub(crate) use data::{FormatParam, Sdp, Session, SessionAttribute, Setup}; +pub(crate) use data::{MediaAttribute, MediaLine, MediaType, Msid, Proto}; +pub(crate) use data::{RestrictionId, Simulcast, SimulcastGroups, SimulcastLayer}; + +mod parser; + +mod error; +pub use error::SdpError; + +#[derive(Debug, PartialEq, Eq)] +/// SDP offer. Offers can be serialized via serde. +pub struct SdpOffer(Sdp); + +impl SdpOffer { + /// Takes the SDP string without any JSON wrapping and makes an `Offer`. + pub fn from_sdp_string(input: &str) -> Result { + let sdp = Sdp::parse(input)?; + Ok(SdpOffer(sdp)) + } + + /// Turns this offer into an SDP string, without any JSON wrapping. + pub fn to_sdp_string(&self) -> String { + self.0.to_string() + } + + #[cfg(test)] + pub(crate) fn into_inner(self) -> Sdp { + self.0 + } +} + +#[derive(Debug, PartialEq, Eq)] +/// SDP answer. Answers can be serialized via serde. +pub struct SdpAnswer(Sdp); + +impl SdpAnswer { + /// Takes the SDP string without any JSON wrapping and makes an `Answer`. + pub fn from_sdp_string(input: &str) -> Result { + let sdp = Sdp::parse(input)?; + Ok(SdpAnswer(sdp)) + } + + /// Turns this answer into an SDP string, without any JSON wrapping. + pub fn to_sdp_string(&self) -> String { + self.0.to_string() + } +} + +impl Deref for SdpOffer { + type Target = Sdp; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Deref for SdpAnswer { + type Target = Sdp; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for SdpOffer { + fn from(v: Sdp) -> Self { + SdpOffer(v) + } +} + +impl From for SdpAnswer { + fn from(v: Sdp) -> Self { + SdpAnswer(v) + } +} + +impl fmt::Display for SdpOffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", **self) + } +} + +impl fmt::Display for SdpAnswer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", **self) + } +} + +macro_rules! sdp_ser { + ($Struct:tt, $Name:literal, $LCName:literal) => { + impl Serialize for $Struct { + fn serialize(&self, s: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Data { + r#type: &'static str, + sdp: String, + } + + Data { + r#type: $LCName, + sdp: self.0.to_string(), + } + .serialize(s) + } + } + + impl<'de> Deserialize<'de> for $Struct { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Data { + r#type: String, + sdp: String, + } + + let data = Data::deserialize(d)?; + + if data.r#type != $LCName { + return Err(de::Error::custom(format!( + "Expected SDP type '{}', got '{}'", + $LCName, data.r#type + ))); + } + + let sdp = Sdp::parse(&data.sdp) + .map_err(|err| de::Error::custom(format!("Failed to parse SDP: {:?}", err)))?; + + Ok(Self(sdp)) + } + } + }; +} + +sdp_ser!(SdpOffer, "Offer", "offer"); +sdp_ser!(SdpAnswer, "Answer", "answer"); + +#[cfg(test)] +mod test { + use crate::VERSION; + use crate::rtp_::SessionId; + + use super::*; + + fn sdp() -> Sdp { + Sdp { + session: Session { + id: SessionId::from(123_u64), + attrs: vec![], + bw: None, + }, + media_lines: vec![], + } + } + + #[test] + fn serialize_deserialize_offer() { + let offer = SdpOffer(sdp()); + let json = serde_json::to_string(&offer).unwrap(); + + let expected = format!( + "{{\"type\":\"offer\",\"sdp\":\ + \"v=0\\r\\no=str0m-{VERSION} 123 2 \ + IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\n\"}}" + ); + assert_eq!(json, expected); + + let offer2: SdpOffer = serde_json::from_str(&json).unwrap(); + + assert_eq!(offer, offer2); + } + + #[test] + fn serialize_deserialize_answer() { + let answer = SdpAnswer(sdp()); + let json = serde_json::to_string(&answer).unwrap(); + + let expected = format!( + "{{\"type\":\"answer\",\"sdp\":\ + \"v=0\\r\\no=str0m-{VERSION} 123 2 \ + IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\n\"}}" + ); + assert_eq!(json, expected); + + let answer2: SdpAnswer = serde_json::from_str(&json).unwrap(); + + assert_eq!(answer, answer2); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/parser.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/parser.rs new file mode 100644 index 000000000..31770ce6b --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/sdp/parser.rs @@ -0,0 +1,1818 @@ +use std::net::{IpAddr, SocketAddr}; +use std::str::FromStr; + +use crate::crypto::Fingerprint; +use crate::io::TcpType; +use crate::rtp_::{Direction, Extension, Frequency, Mid, Pt, SessionId, Ssrc}; +use crate::{Candidate, CandidateKind}; +use combine::error::*; +use combine::parser::char::*; +use combine::parser::combinator::*; +use combine::stream::StreamErrorFor; +use combine::*; +use combine::{ParseError, Parser, Stream}; + +use super::data::*; + +/// Creates a parser of SDP +pub fn sdp_parser() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + (session_parser(), many::, _, _>(media_parser())).map(|(session, media)| Sdp { + session, + media_lines: media, + }) +} + +// /////////////////////////////////////////////////// Session description + +/// 1. First line must be v=0 +/// 2. The second SDP line MUST be an "o=" line The sess-id MUST be representable by a 64-bit signed +/// integer, and the initial value MUST be less than (2**62)-1 +/// 3. Third line is the session name. The "s=" field MUST NOT be empty and SHOULD contain +/// ISO 10646 characters (RFC 8866 section 5.3). +/// +/// Session is over when we find a "t=" line MUST be added, both and +/// SHOULD be set to zero, e.g. "t=0 0". +pub fn session_parser() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + ( + typed_line('v', token('0')), // v=0 + originator_line(), // o=- 6564425948916445306 2 IN IP4 127.0.0.1 + // The "s=" field MUST NOT be empty (RFC 8866 section 5.3) + typed_line('s', any_value()), + many::, _, _>(ignored_session_line()), + optional(bandwidth_line()), // b=CT:1234 + typed_line('t', string("0 0")), // t=0 0 + many::, _, _>(typed_line('r', any_value())), // r should never appear + // + many::, _, _>(session_attribute_line()), + ) + .map(|(_, id, _, _, bw, _, _, attrs)| Session { id, bw, attrs }) +} + +/// `o= ` +fn originator_line() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + let session_string = typed_line( + 'o', + // o=- 6564425948916445306 2 IN IP4 127.0.0.1 + ( + not_sp(), + token(' '), + many1::(digit()), + token(' '), + any_value(), + ) + .map(|(_, _, sess, _, _)| sess), + ); + from_str(session_string).map(|x: u64| -> SessionId { SessionId::from(x) }) +} + +/// `b=:` +fn bandwidth_line() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + typed_line('b', (many1(satisfy(|c| c != ':')), token(':'), any_value())) + .map(|(typ, _, val)| Bandwidth { typ, val }) +} + +/// An a= line that with value like: `a=:`. +fn attribute_line( + attribute: &'static str, + val: Pval, +) -> impl Parser +where + Input: Stream, + Pval: Parser, + Input::Error: ParseError, +{ + typed_line('a', (string(attribute), token(':'), val)).map(|(_, _, val)| val) +} + +/// An a= line that has no value like: `a=ice-lite`. +fn attribute_line_flag(attribute: &'static str) -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + typed_line('a', (string(attribute)).map(|_| ())) +} + +/// a=foo:bar lines belonging before the first m= line +fn session_attribute_line() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + // a=group:BUNDLE 1 2 + // a=group:LS 1 2 + let group = attribute_line( + "group", + ( + not_sp(), + token(' '), + sep_by1(not_sp().map(|m| Mid::from(m.as_str())), token(' ')), + ), + ) + .map(|(typ, _, mids)| SessionAttribute::Group { typ, mids }); + + // a=msid-semantic: WMS 9ce81ef6-c7cb-4da5-8f5f-3e7cc9b5f9b0 + let msid_semantic = attribute_line( + "msid-semantic", + ( + optional(token(' ')), + not_sp(), + token(' '), + sep_by1(not_sp(), token(' ')), + ), + ) + .map( + |(_, semantic, _, stream_ids)| SessionAttribute::MsidSemantic { + semantic, + stream_ids, + }, + ); + + // a=ice-lite + let ice_lite = attribute_line_flag("ice-lite").map(|_| SessionAttribute::IceLite); + + // a=ice-ufrag:IdNYTNL1fjvjyEzL + let ice_ufrag = attribute_line("ice-ufrag", any_value()).map(SessionAttribute::IceUfrag); + + // a=ice-pwd:4d64pT3T1xfwbZvi9fQKjoPb + let ice_pwd = attribute_line("ice-pwd", any_value()).map(SessionAttribute::IcePwd); + + // a=ice-options:trickle + let ice_opt = attribute_line("ice-options", any_value()).map(SessionAttribute::IceOptions); + + // a=fingerprint:sha-256 45:AD:5C:82:F8:BE:B5:2A:D1:74:A6:16:D0:50:CD:86:9C:97:9D:BD:06:8C:C9:85:C9:CD:AB:2B:A8:56:03:CD + // "sha-1" / "sha-224" / "sha-256" / + // "sha-384" / "sha-512" / + // "md5" / "md2" + let hex_byte = count_min_max(2, 2, hex_digit()).and_then(|x: String| { + u8::from_str_radix(&x, 16).map_err(StreamErrorFor::::message_format) + }); + let finger = attribute_line( + "fingerprint", + (not_sp(), token(' '), sep_by1(hex_byte, token(':'))), + ) + .map(|(hash_func, _, bytes)| SessionAttribute::Fingerprint(Fingerprint { hash_func, bytes })); + + let setup_val = choice(( + attempt(string("actpass").map(|_| Setup::ActPass)), + attempt(string("active").map(|_| Setup::Active)), + attempt(string("passive").map(|_| Setup::Passive)), + )); + + // a=setup:actpass + let setup = attribute_line("setup", setup_val).map(SessionAttribute::Setup); + + // a=candidate + let cand = candidate_attribute().map(SessionAttribute::Candidate); + + // a=end-of-candidates + let endof = attribute_line_flag("end-of-candidates").map(|_| SessionAttribute::EndOfCandidates); + + // tls-id + // identity + // extmap + let unused = typed_line('a', any_value()).map(SessionAttribute::Unused); + + choice(( + attempt(group), + attempt(msid_semantic), + attempt(ice_lite), + attempt(ice_ufrag), + attempt(ice_pwd), + attempt(ice_opt), + attempt(finger), + attempt(setup), + attempt(cand), + attempt(endof), + unused, + )) +} + +/// Parser for candidate, without attribute prefix (a=). +fn candidate() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + // Reference: https://datatracker.ietf.org/doc/html/rfc5245#section-15.1 + let port = || { + not_sp::().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }) + }; + + let ip_addr = || { + not_sp().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }) + }; + + let kind = choice(( + string("host").map(|_| CandidateKind::Host), + string("prflx").map(|_| CandidateKind::PeerReflexive), + string("srflx").map(|_| CandidateKind::ServerReflexive), + string("relay").map(|_| CandidateKind::Relayed), + )); + + ( + string("candidate:").and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + not_sp(), + token(' '), + not_sp().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + token(' '), + not_sp().and_then(|s| { + s.as_str().try_into().map_err(|_| { + StreamErrorFor::::message_format(format!("invalid protocol: {}", s)) + }) + }), + token(' '), + not_sp().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + token(' '), + ip_addr(), + token(' '), + port(), + string(" typ "), + kind, + optional(( + attempt(string(" raddr ")), + ip_addr(), + string(" rport "), + port(), + )), + optional(( + attempt(string(" tcptype ")), + not_sp().and_then(|s| { + TcpType::from_str(s.as_str()).map_err(StreamErrorFor::::message_format) + }), + )), + optional((attempt(string(" generation ")), not_sp())), + optional((attempt(string(" network-id ")), not_sp())), + optional((attempt(string(" ufrag ")), not_sp())), + optional((attempt(string(" network-cost ")), not_sp())), + ) + .map( + |( + _, + found, + _, + comp_id, + _, + proto, + _, + prio, + _, + addr, + _, + port, + _, + kind, + raddr, // (" raddr ", addr, " rport ", port) + tcptype, // (" tcptype ", tcptype) + _, // (" generation ", generation) + _, // (" network-id ", network_id) + ufrag, // (" ufrag ", ufrag) + _, // ("network-cost", network_cost) + )| { + Candidate::from_parts( + found, + comp_id, + proto, + prio, // remote candidates calculate prio on their side + SocketAddr::from((addr, port)), + kind, + raddr.map(|(_, addr, _, port)| SocketAddr::from((addr, port))), + tcptype.map(|(_, tcptype)| tcptype), + ufrag.map(|(_, u)| u), + ) + }, + ) +} + +/// Parser for a=candidate lines. +pub(crate) fn candidate_attribute() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + (string("a="), candidate(), line_end()).map(|(_, c, _)| c) +} + +/// Session line with a key we ignore (spec says we should validate them, but meh). +fn ignored_session_line() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + // TODO: there must be a better way + let ignored = choice(( + token('i'), + token('u'), + token('e'), + token('p'), + token('c'), + token('z'), + token('k'), + )); + line(ignored, any_value()).map(|_| ()) +} + +// /////////////////////////////////////////////////// Media description + +/// A m= section with attributes, until next m= or EOF +fn media_parser() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + ( + media_line(), + optional(typed_line('c', any_value())), // c=IN IP4 0.0.0.0 + optional(bandwidth_line()), // b=AS:2500 + many::, _, _>(media_attribute_line()), + ) + .and_then(|((typ, port, proto, pts), _, bw, attrs)| { + let m = MediaLine { + typ, + disabled: port == "0", + proto, + pts, + bw, + attrs, + }; + if let Some(err) = m.check_consistent() { + warn!("{:?}", err); + return Err(StreamErrorFor::::message_format(err)); + } + Ok(m) + }) +} + +/// The m= line +// dormammu: +// m=audio 9 UDP/TLS/RTP/SAVPF 111 +// m=video 9 UDP/TLS/RTP/SAVPF 96 97 125 107 100 101 +// m=application 9 DTLS/SCTP 5000 +// chrome: +// m=audio 64205 UDP/TLS/RTP/SAVPF 111 +// m=video 53151 UDP/TLS/RTP/SAVPF 96 97 125 107 100 101 +// m=application 54055 DTLS/SCTP 5000 +// +// newer chrome: +// m=application 9 UDP/DTLS/SCTP webrtc-datachannel +fn media_line() -> impl Parser)> +where + Input: Stream, + Input::Error: ParseError, +{ + let media_type = choice(( + attempt(string("audio").map(|_| MediaType::Audio)), + attempt(string("video").map(|_| MediaType::Video)), + attempt(string("application").map(|_| MediaType::Application)), + not_sp().map(MediaType::Unknown), + )); + + let proto_line = choice(( + attempt(string("UDP/TLS/RTP/SAVPF").map(|_| Proto::Srtp)), + attempt(string("DTLS/SCTP").map(|_| Proto::Sctp)), + attempt(string("UDP/DTLS/SCTP").map(|_| Proto::Sctp)), + )); + + let parse_pt = not_sp().and_then(|s| { + s.parse::() + .map(Pt::from) + .map_err(StreamErrorFor::::message_format) + }); + + // m= ... + // is: + // where either matches a=rtpmap: or a=sctpmap: + typed_line( + 'm', + ( + media_type, // type: audio, video etc. + token(' '), + not_sp(), // port: just set to 9 or something + token(' '), + proto_line, // proto: or + token(' '), + choice(( + attempt(sep_by(parse_pt, token(' '))), // + any_value().map(|_| vec![]), + )), + ), + ) + .map(|(typ, _, port, _, proto, _, pts)| (typ, port, proto, pts)) +} + +/// a=foo:bar lines belonging before the first m= line +fn media_attribute_line() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + // a=rtcp:9 IN IP4 0.0.0.0 + let rtcp = attribute_line("rtcp", any_value()).map(MediaAttribute::Rtcp); + + // a=ice-ufrag:IdNYTNL1fjvjyEzL + let ice_ufrag = attribute_line("ice-ufrag", any_value()).map(MediaAttribute::IceUfrag); + + // a=ice-pwd:4d64pT3T1xfwbZvi9fQKjoPb + let ice_pwd = attribute_line("ice-pwd", any_value()).map(MediaAttribute::IcePwd); + + // a=ice-options:trickle + let ice_opt = attribute_line("ice-options", any_value()).map(MediaAttribute::IceOptions); + + // a=fingerprint:sha-256 45:AD:5C:82:F8:BE:B5:2A:D1:74:A6:16:D0:50:CD:86:9C:97:9D:BD:06:8C:C9:85:C9:CD:AB:2B:A8:56:03:CD + // "sha-1" / "sha-224" / "sha-256" / + // "sha-384" / "sha-512" / + // "md5" / "md2" + let hex_byte = count_min_max(2, 2, hex_digit()).and_then(|x: String| { + u8::from_str_radix(&x, 16).map_err(StreamErrorFor::::message_format) + }); + let finger = attribute_line( + "fingerprint", + (not_sp(), token(' '), sep_by1(hex_byte, token(':'))), + ) + .map(|(hash_func, _, bytes)| MediaAttribute::Fingerprint(Fingerprint { hash_func, bytes })); + + let setup_val = choice(( + attempt(string("actpass").map(|_| Setup::ActPass)), + attempt(string("active").map(|_| Setup::Active)), + attempt(string("passive").map(|_| Setup::Passive)), + )); + + // a=setup:actpass + let setup = attribute_line("setup", setup_val).map(MediaAttribute::Setup); + + // a=mid:0 + let mid = attribute_line("mid", any_value()) + .map(|m| Mid::from(m.as_str())) + .map(MediaAttribute::Mid); + + let sctp_port = attribute_line( + "sctp-port", + not_sp::().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + ) + .map(MediaAttribute::SctpPort); + + let max_message_size = attribute_line( + "max-message-size", + not_sp::().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + ) + .map(MediaAttribute::MaxMessageSize); + + // a=sctp-init: + // See draft-hancke-tsvwg-snap (§4.2: base64 defined in RFC 4566, i.e. standard with padding) + // + // The base64 value is stored as-is; decoding happens when consumed. + let sctp_init = attribute_line("sctp-init", any_value()).map(MediaAttribute::SctpInit); + + // a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level + // a=extmap:["/"] + let extmap = attribute_line( + "extmap", + ( + many1::(satisfy(|c| c != '/' && c != ' ')).and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + optional((token('/'), not_sp().map(|d| Direction::from(&d[..])))), + token(' '), + not_sp().map(|uri| Extension::from_sdp_uri(&uri)), + optional((token(' '), any_value())), + ), + ) + .map(|(id, _dir_opt, _, ext, _ext_opt)| MediaAttribute::ExtMap { id, ext }); + + let direction = choice(( + attempt(attribute_line_flag("recvonly").map(|_| MediaAttribute::RecvOnly)), + attempt(attribute_line_flag("sendrecv").map(|_| MediaAttribute::SendRecv)), + attempt(attribute_line_flag("sendonly").map(|_| MediaAttribute::SendOnly)), + attempt(attribute_line_flag("inactive").map(|_| MediaAttribute::Inactive)), + )); + + // // a=msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1 + // // a=msid:- f78dde68-7055-4e20-bb37-433803dd1ed1 + let msid = attribute_line("msid", (not_sp(), token(' '), any_value())).map( + |(stream_id, _, track_id)| { + MediaAttribute::Msid(Msid { + stream_id, + track_id, + }) + }, + ); + + let rtcpmux = attribute_line_flag("rtcp-mux").map(|_| MediaAttribute::RtcpMux); + let rtcpmuxonly = attribute_line_flag("rtcp-mux-only").map(|_| MediaAttribute::RtcpMuxOnly); + let rtcprsize = attribute_line_flag("rtcp-rsize").map(|_| MediaAttribute::RtcpRsize); + + // a=candidate + let cand = candidate_attribute().map(MediaAttribute::Candidate); + + // a=end-of-candidates + let endof = attribute_line_flag("end-of-candidates").map(|_| MediaAttribute::EndOfCandidates); + + let pt = || { + not_sp().and_then(|s| { + let pt = s + .parse::() + .map_err(StreamErrorFor::::message_format)?; + // RTP payload type is 7-bit; reject out-of-range values rather than panicking later + // when PTs are claimed against a 128-entry table. + if pt > 127 { + return Err(StreamErrorFor::::message_format(format!( + "payload type out of range (0-127): {pt}" + ))); + } + Ok(Pt::from(pt)) + }) + }; + + // a=rtpmap:111 opus/48000/2 + let rtpmap = attribute_line( + "rtpmap", + ( + pt(), + token(' '), + many1::(satisfy(|c| c != '/' && c != '\r' && c != '\n')), + token('/'), + many1::(satisfy(|c| c != '/' && c != '\r' && c != '\n')).and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + optional(( + token('/'), + any_value().and_then(|s| { + s.parse::() + .map_err(StreamErrorFor::::message_format) + }), + )), // only audio has the last /2 (channels) + ), + ) + .map(|(pt, _, codec, _, clock_rate, opt_channels)| { + let channels = opt_channels.map(|(_, e)| e); + MediaAttribute::RtpMap { + pt, + value: RtpMap { + codec: codec.as_str().into(), + clock_rate, + channels, + }, + } + }); + + // a=rtcp-fb:111 transport-cc + // a=rtcp-fb:111 ccm fir + // a=rtcp-fb:111 nack + // a=rtcp-fb:111 nack pli + let rtcp_fb = attribute_line("rtcp-fb", (pt(), token(' '), any_value())) + .map(|(pt, _, value)| MediaAttribute::RtcpFb { pt, value }); + + let fmtp_param = sep_by1(key_val(), token(';')).map(FormatParam::parse_pairs); + + // a=fmtp:111 minptime=10; useinbandfec=1 + // a=fmtp:111 minptime=10;useinbandfec=1 + let fmtp1 = attribute_line("fmtp", (pt(), token(' '), fmtp_param)) + .map(|(pt, _, values)| MediaAttribute::Fmtp { pt, values }); + + // a=fmtp:101 0-15 + let fmtp2 = attribute_line("fmtp", (pt(), token(' '), not_sp())).map(|(pt, _, _value)| { + MediaAttribute::Fmtp { + pt, + values: vec![FormatParam::Unknown], + } + }); + + let fmtp = choice((attempt(fmtp1), attempt(fmtp2))); + + // a=rid: [pt=;]= + let rid = attribute_line( + "rid", + ( + name().map(RestrictionId::new_active), + token(' '), + choice((string("send"), string("recv"))), + optional(( + token(' '), + optional(( + string("pt="), + sep_by1::, _, _, _>(pt(), token(',')), + // TODO this is not really optional when there is + // a restriction part. It means we are incorrectly + // allowing this: a=rid:foo send pt=111max-br=64000 + optional(token(';')), + )), + sep_by::, _, _, _>(key_val(), token(';')), + )), + ), + ) + .map(|(id, _, direction, x)| { + let mut pt = vec![]; + let mut restriction = vec![]; + if let Some((_, ps, rs)) = x { + if let Some((_, ps, _)) = ps { + pt = ps; + } + restriction = rs; + } + + MediaAttribute::Rid { + id, + direction, + pt, + restriction, + } + }); + + let simul1 = |direction: &'static str| { + ( + string(direction), + token(' '), + sep_by1::>, _, _, _>( + sep_by1( + optional(token('~')) + .and(name()) + .map(|x| (x.1, x.0.is_none())), + token(','), + ), + token(';'), + ), + ) + }; + + // Parser guarantee that it's send => recv or recv => send. + let simul2 = choice(( + (simul1("send"), optional((token(' '), simul1("recv")))), + (simul1("recv"), optional((token(' '), simul1("send")))), + )); + + // a=simulcast: ;, [same] + let simulcast = attribute_line("simulcast", simul2).map(|(s1, maybe_s2)| { + let mut send = SimulcastGroups(vec![]); + let mut recv = SimulcastGroups(vec![]); + + fn to_simul(to: &mut SimulcastGroups, groups: Vec>) { + for group in groups { + // TODO: Properly support rid alternatives, for now we ignore any alternatives + // provided and use the first rid. + let first = group.into_iter().next(); + + if let Some(rid) = first.map(|(rid, active)| RestrictionId::new(rid, active)) { + to.0.push(SimulcastLayer { + restriction_id: rid, + attributes: None, + }); + } + } + } + + { + let to = if s1.0 == "send" { &mut send } else { &mut recv }; + to_simul(to, s1.2); + } + + if let Some(s2) = maybe_s2 { + let s2 = s2.1; + let to = if s2.0 == "send" { &mut send } else { &mut recv }; + to_simul(to, s2.2); + } + + MediaAttribute::Simulcast(Simulcast { + send, + recv, + is_munged: false, + }) + }); + + // a=ssrc-group:FID 1111 2222 + let ssrc_group = attribute_line( + "ssrc-group", + ( + not_sp(), + token(' '), + sep_by1( + not_sp().and_then(|s| { + s.parse::() + .map(Ssrc::from) + .map_err(StreamErrorFor::::message_format) + }), + token(' '), + ), + ), + ) + .map(|(semantics, _, ssrcs)| MediaAttribute::SsrcGroup { semantics, ssrcs }); + + // a=ssrc:3948621874 cname:xeXs3aE9AOBn00yJ + // a=ssrc:3948621874 msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1 + // a=ssrc:3948621874 mslabel:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK + // a=ssrc:3948621874 label:f78dde68-7055-4e20-bb37-433803dd1ed1 + let ssrc = attribute_line( + "ssrc", + ( + not_sp().and_then(|s| { + s.parse::() + .map(Ssrc::from) + .map_err(StreamErrorFor::::message_format) + }), + token(' '), + many1::(satisfy(|c| c != ':')), + token(':'), + any_value(), + ), + ) + .map(|(ssrc, _, attr, _, value)| MediaAttribute::Ssrc { ssrc, attr, value }); + + let unused = typed_line('a', any_value()).map(MediaAttribute::Unused); + + // Split into two choice() calls because combine's choice macro supports at most 25 parsers. + choice(( + attempt(choice(( + attempt(ice_ufrag), + attempt(ice_pwd), + attempt(ice_opt), + attempt(finger), + attempt(setup), + attempt(mid), + attempt(sctp_port), + attempt(max_message_size), + attempt(sctp_init), + attempt(extmap), + attempt(direction), + attempt(msid), + attempt(rtcp), + ))), + attempt(choice(( + attempt(rtcpmux), + attempt(rtcpmuxonly), + attempt(rtcprsize), + attempt(cand), + attempt(endof), + attempt(rtpmap), + attempt(rtcp_fb), + attempt(fmtp), + attempt(rid), + attempt(simulcast), + attempt(ssrc_group), + attempt(ssrc), + ))), + unused, + )) +} + +// /////////////////////////////////////////////////// Generic things below + +/// A specific line +fn typed_line(expected: char, val: Pval) -> impl Parser +where + Input: Stream, + Pval: Parser, + Input::Error: ParseError, +{ + line(token(expected), val) +} + +/// A line with some parser for value and parser for type. +fn line(typ: Ptyp, val: Pval) -> impl Parser +where + Ptyp: Parser, + Pval: Parser, + Input: Stream, + Input::Error: ParseError, +{ + attempt((typ, token('='), val, line_end())) + .map(|(_, _, value, _)| value) + .message("sdp line") +} + +/// alphanumeric name. +fn name() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + many1(satisfy(|c: char| c.is_alphanumeric())) +} + +/// Not SP, \r or \n +fn not_sp() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + many1(satisfy(|c| c != ' ' && c != '\r' && c != '\n')) +} + +/// Any value that isn't \r or \n. +fn any_value() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + many1(satisfy(|c| c != '\r' && c != '\n')) +} + +/// We discovered a stray \n in safari SDP. This line end handles \r\n, \n or EOF. +fn line_end() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + choice((crlf().map(|_| ()), newline().map(|_| ()), eof())) +} + +// minptime=10 +fn key_val() -> impl Parser +where + Input: Stream, + Input::Error: ParseError, +{ + ( + optional(spaces()), + many1(satisfy(|c| c != '=' && c != ' ' && c != '\r' && c != '\n')), + token('='), + many1(satisfy(|c| c != ';' && c != ' ' && c != '\r' && c != '\n')), + ) + .map(|(_, key, _, val)| (key, val)) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::format::Codec; + use crate::io::Protocol; + + #[test] + fn line_a() { + assert_eq!( + line(letter(), any_value()).parse("a=mid:0"), + Ok(("mid:0".to_string(), "")) + ) + } + + #[test] + fn line_end_crlf() { + assert_eq!(line_end().parse("\r\n"), Ok(((), ""))); + } + + #[test] + fn line_end_lf() { + assert_eq!(line_end().parse("\n"), Ok(((), ""))); + } + + #[test] + fn line_end_eof() { + assert_eq!(line_end().parse(""), Ok(((), ""))); + } + + #[test] + fn typed_line_v() { + assert_eq!(typed_line('v', token('0')).parse("v=0"), Ok(('0', ""))) + } + + #[test] + fn attribute_line_flag_foo() { + assert_eq!(attribute_line_flag("foo").parse("a=foo"), Ok(((), ""))) + } + + #[test] + fn attribute_line_foo_bar() { + assert_eq!( + attribute_line("foo", any_value()).parse("a=foo:bar"), + Ok(("bar".to_string(), "")) + ) + } + + #[test] + fn session_attribute_line_simple() { + let x = session_attribute_line().parse("a=ice-lite"); + assert_eq!(x, Ok((SessionAttribute::IceLite, ""))); + } + + #[test] + fn session_attribute_line_finger() { + let x = session_attribute_line().parse("a=fingerprint:sha-256 45:AD:5C:82:F8:BE"); + assert_eq!( + x, + Ok(( + SessionAttribute::Fingerprint(Fingerprint { + hash_func: "sha-256".to_string(), + bytes: vec![69, 173, 92, 130, 248, 190], + }), + "" + )) + ); + } + + #[test] + fn media_attribute_line_rid_simple() { + let x = media_attribute_line().parse("a=rid:lo send").unwrap(); + assert_eq!("a=rid:lo send\r\n", x.0.to_string()); + } + + #[test] + fn media_attribute_line_rid_pt() { + let x = media_attribute_line() + .parse("a=rid:lo send pt=99,100") + .unwrap(); + assert_eq!("a=rid:lo send pt=99,100\r\n", x.0.to_string()); + } + + #[test] + fn media_attribute_line_rid_restr() { + let x = media_attribute_line() + .parse("a=rid:lo send max-br=64000;max-height=360") + .unwrap(); + assert_eq!( + "a=rid:lo send max-br=64000;max-height=360\r\n", + x.0.to_string() + ); + } + + #[test] + fn media_attribute_line_rid_pt_restr() { + let x = media_attribute_line() + .parse("a=rid:lo send pt=99,100;max-br=64000;max-height=360") + .unwrap(); + assert_eq!( + "a=rid:lo send pt=99,100;max-br=64000;max-height=360\r\n", + x.0.to_string() + ); + } + + #[test] + fn media_attribute_line_simulcast() { + let x = media_attribute_line() + .parse("a=simulcast:send 3;4") + .unwrap(); + assert_eq!("a=simulcast:send 3;4\r\n", x.0.to_string()); + } + + #[test] + fn media_attribute_line_simulcast_alt() { + let x = media_attribute_line() + .parse("a=simulcast:send 2,3;4") + .unwrap(); + assert_eq!("a=simulcast:send 2;4\r\n", x.0.to_string()); + } + + #[test] + fn media_attribute_line_simulcast_send_recv() { + let x = media_attribute_line() + .parse("a=simulcast:send 2;3 recv 4") + .unwrap(); + assert_eq!("a=simulcast:send 2;3 recv 4\r\n", x.0.to_string()); + } + + #[test] + fn media_attribute_line_simulcast_recv_send() { + let x = media_attribute_line() + .parse("a=simulcast:recv 2;3 send 4") + .unwrap(); + assert_eq!("a=simulcast:send 4 recv 2;3\r\n", x.0.to_string()); + } + + #[test] + fn media_line_simple() { + let m = media_line().parse("m=audio 9 UDP/TLS/RTP/SAVPF 10\r\n"); + assert_eq!( + m, + Ok(( + (MediaType::Audio, "9".into(), Proto::Srtp, vec![10.into()],), + "" + )) + ); + } + + #[test] + fn rtpmap_payload_type_over_127_is_rejected() { + // RTP payload type is 7-bit. An out-of-range PT in a=rtpmap must not be accepted as an + // RtpMap; otherwise it becomes a PayloadParams that panics when claimed against a + // 128-entry table. It instead falls through to Unused and is ignored (the m-line then + // fails its own "missing a=rtpmap" check). 127 is the largest valid PT. + let bad = media_attribute_line().parse("a=rtpmap:200 opus/48000/2"); + assert!( + !matches!(bad, Ok((MediaAttribute::RtpMap { .. }, _))), + "out-of-range rtpmap PT must not parse as RtpMap: {bad:?}" + ); + assert!(matches!( + media_attribute_line().parse("a=rtpmap:127 opus/48000/2"), + Ok((MediaAttribute::RtpMap { .. }, _)) + )); + } + + #[test] + fn session_parser_simple() { + let sdp = "v=0\n\ + o=- 6564425948916445306 2 IN IP4 127.0.0.1\n\ + s=-\n\ + t=0 0\n\ + a=group:BUNDLE 0\n\ + a=msid-semantic: WMS\n\ + m=application 9 DTLS/SCTP 5000\n\ + "; + assert_eq!( + session_parser().parse(sdp), + Ok(( + Session { + id: 6_564_425_948_916_445_306.into(), + bw: None, + attrs: vec![ + SessionAttribute::Group { + typ: "BUNDLE".into(), + mids: vec!["0".into()], + }, + SessionAttribute::Unused("msid-semantic: WMS".into()) + ], + }, + "m=application 9 DTLS/SCTP 5000\n" + )) + ); + } + + #[test] + fn session_parser_named_session() { + // A session name other than "-". + let sdp = "v=0\r\n\ + o=publisher 4489045141692799359 2 IN IP4 127.0.0.1\r\n\ + s=NamedPublishSession\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0 1\r\n\ + a=extmap-allow-mixed\r\n\ + a=msid-semantic: WMS\r\n"; + assert_eq!( + session_parser().parse(sdp), + Ok(( + Session { + id: 4_489_045_141_692_799_359.into(), + bw: None, + attrs: vec![ + SessionAttribute::Group { + typ: "BUNDLE".into(), + mids: vec!["0".into(), "1".into()], + }, + SessionAttribute::Unused("extmap-allow-mixed".into()), + SessionAttribute::Unused("msid-semantic: WMS".into()), + ], + }, + "" + )) + ); + } + + #[test] + fn session_parser_space_session_name() { + // "If a session has no meaningful name, the value "s= " SHOULD be used + // (i.e., a single space as the session name)." (RFC 8866 section 5.3) + let sdp = "v=0\n\ + o=- 6564425948916445306 2 IN IP4 127.0.0.1\n\ + s= \n\ + t=0 0\n\ + "; + assert_eq!( + session_parser().parse(sdp), + Ok(( + Session { + id: 6_564_425_948_916_445_306.into(), + bw: None, + attrs: vec![], + }, + "" + )) + ); + } + + #[test] + fn parse_sdp_firefox() { + let sdp = "v=0\r\n\ + o=mozilla...THIS_IS_SDPARTA-83.0 7052848360639826063 0 IN IP4 0.0.0.0\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=fingerprint:sha-256 37:FC:96:B5:73:98:E6:F9:C5:0B:D9:EE:B1:F8:D0:01:07:2E:75:E8:6C:A4:32:A7:DC:63:99:5E:68:5C:BF:FB\r\n\ + a=ice-options:trickle\r\n\ + a=msid-semantic:WMS *\r\n"; + + assert_eq!( + sdp_parser().parse(sdp), + Ok(( + Sdp { + session: Session { + id: 7052848360639826063.into(), + bw: None, + attrs: vec![ + SessionAttribute::Fingerprint(Fingerprint { + hash_func: "sha-256".into(), + bytes: vec![ + 0x37, 0xFC, 0x96, 0xB5, 0x73, 0x98, 0xE6, 0xF9, 0xC5, 0x0B, + 0xD9, 0xEE, 0xB1, 0xF8, 0xD0, 0x01, 0x07, 0x2E, 0x75, 0xE8, + 0x6C, 0xA4, 0x32, 0xA7, 0xDC, 0x63, 0x99, 0x5E, 0x68, 0x5C, + 0xBF, 0xFB + ] + }), + SessionAttribute::IceOptions("trickle".to_string()), + SessionAttribute::MsidSemantic { + semantic: "WMS".into(), + stream_ids: vec!["*".into()] + }, + ] + }, + media_lines: vec![] + }, + "" + )) + ); + } + + #[test] + fn parse_offer_sdp_firefox() { + let sdp = "v=0\r\n\ + o=mozilla...THIS_IS_SDPARTA-84.0 9033133899747520364 1 IN IP4 0.0.0.0\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=fingerprint:sha-256 AE:DC:49:AE:CA:55:35:CB:4E:FA:FE:70:99:30:C0:14:C3:B8:06:80:1F:A9:DA:9A:7C:FB:B7:20:AB:83:60:45\r\n\ + a=group:BUNDLE 0\r\n\ + a=ice-options:trickle\r\n\ + a=msid-semantic:WMS *\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 109 9 0 8 101\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=sendonly\r\n\ + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n\ + a=extmap:2/recvonly urn:ietf:params:rtp-hdrext:csrc-audio-level\r\n\ + a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:mid\r\n\ + a=fmtp:109 maxplaybackrate=48000;stereo=1;useinbandfec=1\r\n\ + a=fmtp:101 0-15\r\n\ + a=ice-pwd:cd25258044061ec2ecc73378eb3dc6a3\r\n\ + a=ice-ufrag:c1e284ad\r\n\ + a=mid:0\r\n\ + a=msid:- {5c7f12e5-b4bd-7142-9a06-2885a2d1cb66}\r\n\ + a=rtcp-mux\r\n\ + a=rtpmap:109 opus/48000/2\r\n\ + a=rtpmap:9 G722/8000/1\r\n\ + a=rtpmap:0 PCMU/8000\r\n\ + a=rtpmap:8 PCMA/8000\r\n\ + a=rtpmap:101 telephone-event/8000/1\r\n\ + a=setup:actpass\r\n\ + a=ssrc:1481683531 cname:{326ec0d2-d1ae-974c-b1ad-aea85cdfa0ad}\r\n"; + + let parsed = sdp_parser().parse(sdp); + + assert!(parsed.is_ok()); + } + + #[test] + fn parse_offer_sdp_chrome() { + let sdp = "v=0\r\n\ + o=- 5058682828002148772 3 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + a=msid-semantic: WMS 5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=rtcp:9 IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:S5hk\r\n\ + a=ice-pwd:0zV/Yu3y8aDzbHgqWhnVQhqP\r\n\ + a=ice-options:trickle\r\n\ + a=fingerprint:sha-256 8C:64:ED:03:76:D0:3D:B4:88:08:91:64:08:80:A8:C6:5A:BF:8B:4E:38:27:96:CA:08:49:25:73:46:60:20:DC\r\n\ + a=setup:actpass\r\n\ + a=mid:0\r\n\ + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n\ + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n\ + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n\ + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n\ + a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n\ + a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n\ + a=sendrecv\r\n\ + a=msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + a=rtcp-mux\r\n\ + a=rtpmap:111 opus/48000/2\r\n\ + a=rtcp-fb:111 transport-cc\r\n\ + a=fmtp:111 minptime=10;useinbandfec=1\r\n\ + a=rtpmap:103 ISAC/16000\r\n\ + a=rtpmap:104 ISAC/32000\r\n\ + a=rtpmap:9 G722/8000\r\n\ + a=rtpmap:0 PCMU/8000\r\n\ + a=rtpmap:8 PCMA/8000\r\n\ + a=rtpmap:106 CN/32000\r\n\ + a=rtpmap:105 CN/16000\r\n\ + a=rtpmap:13 CN/8000\r\n\ + a=rtpmap:110 telephone-event/48000\r\n\ + a=rtpmap:112 telephone-event/32000\r\n\ + a=rtpmap:113 telephone-event/16000\r\n\ + a=rtpmap:126 telephone-event/8000\r\n\ + a=ssrc:3948621874 cname:xeXs3aE9AOBn00yJ\r\n\ + a=ssrc:3948621874 msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + a=ssrc:3948621874 mslabel:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK\r\n\ + a=ssrc:3948621874 label:f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\ + "; + + let parsed = sdp_parser().parse(sdp); + + assert!(parsed.is_ok()); + } + + #[test] + fn parse_offer_sdp_ffmpeg_whip() { + // ffmpeg -re \ + // -f lavfi -i "testsrc=size=640x480:rate=30" \ + // -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + // -c:v libx264 -profile:v baseline -pix_fmt yuv420p -g 60 -tune zerolatency \ + // -c:a libopus -ar 48000 -ac 2 \ + // -f whip "http://127.0.0.1:8080/whip" + let sdp = "v=0\r\n\ + o=FFmpeg 4489045141692799359 2 IN IP4 127.0.0.1\r\n\ + s=FFmpegPublishSession\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0 1\r\n\ + a=extmap-allow-mixed\r\n\ + a=msid-semantic: WMS\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:08a9c3f4\r\n\ + a=ice-pwd:a202d23be47d1decaa6f67d04d4e6c16\r\n\ + a=fingerprint:sha-256 21:69:7C:46:D3:DC:4D:D3:66:61:5C:C5:6A:0B:F7:36\ + :BD:91:A9:2A:FF:BC:1A:59:D9:38:38:C8:E2:A4:8D:AF\r\n\ + a=setup:passive\r\n\ + a=mid:0\r\n\ + a=sendonly\r\n\ + a=msid:FFmpeg audio\r\n\ + a=rtcp-mux\r\n\ + a=rtpmap:111 opus/48000/2\r\n\ + a=ssrc:3166090378 cname:FFmpeg\r\n\ + a=ssrc:3166090378 msid:FFmpeg audio\r\n\ + m=video 9 UDP/TLS/RTP/SAVPF 106 105\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:08a9c3f4\r\n\ + a=ice-pwd:a202d23be47d1decaa6f67d04d4e6c16\r\n\ + a=fingerprint:sha-256 21:69:7C:46:D3:DC:4D:D3:66:61:5C:C5:6A:0B:F7:36\ + :BD:91:A9:2A:FF:BC:1A:59:D9:38:38:C8:E2:A4:8D:AF\r\n\ + a=setup:passive\r\n\ + a=mid:1\r\n\ + a=sendonly\r\n\ + a=msid:FFmpeg video\r\n\ + a=rtcp-mux\r\n\ + a=rtcp-rsize\r\n\ + a=rtpmap:106 H264/90000\r\n\ + a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001e\r\n\ + a=rtcp-fb:106 nack\r\n\ + a=rtpmap:105 rtx/90000\r\n\ + a=fmtp:105 apt=106\r\n\ + a=ssrc-group:FID 3166090379 3166090380\r\n\ + a=ssrc:3166090379 cname:FFmpeg\r\n\ + a=ssrc:3166090379 msid:FFmpeg video\r\n\ + "; + + let (sdp, _) = sdp_parser().parse(sdp).unwrap(); + + assert_eq!(sdp.media_lines.len(), 2); + assert_eq!(sdp.media_lines[0].mid().to_string(), "0"); + assert_eq!(sdp.media_lines[1].mid().to_string(), "1"); + } + + #[test] + fn parse_safari_data_channel() { + let sdp = "v=0\r\n\ + o=- 4611516372927609806 2 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + a=extmap-allow-mixed\r\n\ + a=msid-semantic: WMS\r\n\ + m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=ice-ufrag:HhS+\r\n\ + a=ice-pwd:FhYTGhlAtKCe6KFIX8b+AThW\r\n\ + a=ice-options:trickle\r\n\ + a=fingerprint:sha-256 B4:12:1C:7C:7D:ED:F1:FA:61:07:57:9C:29:BE:58:E3:BC:41:E7:13:8E:7D:D3:9D:1F:94:6E:A5:23:46:94:23\r\n\ + a=setup:actpass\r\n\ + a=mid:0\r\n\ + a=sctp-port:5000\r\n\ + a=max-message-size:262144\r\n\ + "; + + let (sdp, _) = sdp_parser().parse(sdp).unwrap(); + + assert!(!sdp.media_lines.is_empty()); + assert_eq!(sdp.media_lines[0].mid().to_string(), "0"); + } + + #[test] + fn parse_minimal() { + let sdp = "v=0\r\n\ + o=- 5058682828002148772 3 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=rtcp:9 IN IP4 0.0.0.0\r\n\ + a=setup:actpass\r\n\ + a=inactive\r\n\ + a=mid:0\r\n\ + "; + + let parsed = sdp_parser().parse(sdp); + assert!(parsed.is_ok()); + } + + #[test] + fn bare_static_audio_pts_are_resolved() { + let sdp = "v=0\r\n\ + o=- 1 1 IN IP4 127.0.0.1\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=group:BUNDLE 0\r\n\ + m=audio 9 UDP/TLS/RTP/SAVPF 0 8 9 13\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=setup:actpass\r\n\ + a=sendrecv\r\n\ + a=mid:0\r\n\ + "; + + let (sdp, _) = sdp_parser() + .parse(sdp) + .expect("static PTs must not require an a=rtpmap line"); + sdp.assert_consistency() + .expect("bare static PTs are consistent"); + let params = sdp.media_lines[0].rtp_params(); + + let definitions: Vec<_> = params + .iter() + .map(|p| (*p.pt(), p.spec().codec, p.spec().clock_rate)) + .collect(); + assert_eq!( + definitions, + vec![ + (0, Codec::PCMU, Frequency::EIGHT_KHZ), + (8, Codec::PCMA, Frequency::EIGHT_KHZ), + (9, Codec::G722, Frequency::SIXTEEN_KHZ), + (13, Codec::CN, Frequency::EIGHT_KHZ), + ] + ); + } + + #[test] + fn parse_candidates() { + let a = "a=candidate:1 1 udp 1845494015 198.51.100.100 11100 typ srflx raddr 203.0.113.100 rport 10100 ufrag abc\r\n"; + + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.ufrag(), Some("abc")); + + let a = "a=candidate:1 1 udp 1845494015 198.51.100.100 11100 typ host ufrag abc\r\n"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.ufrag(), Some("abc")); + + let a = "a=candidate:3684617590 1 udp 2122260223 10.217.229.219 50028 typ host generation 0 network-id 1 network-cost 900"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "10.217.229.219:50028".parse().unwrap()); + + let a = "a=candidate:387183333 1 udp 1686052607 113.185.55.72 31267 typ srflx raddr 10.217.229.219 rport 50028 generation 0 network-id 1 network-cost 900"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "113.185.55.72:31267".parse().unwrap()); + + let a = "a=candidate:2501718406 1 tcp 1518280447 10.217.229.219 9 typ host tcptype active generation 0 network-id 1 network-cost 900"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "10.217.229.219:9".parse().unwrap()); + + let a = "a=candidate:387183333 1 udp 1686052607 113.185.55.72 41775 typ srflx raddr 10.217.229.219 rport 50028 generation 0 network-id 1 network-cost 900"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "113.185.55.72:41775".parse().unwrap()); + + let a = "a=candidate:3936339338 1 udp 2122265343 fd00:f8aa:3ff5:5914:90bf:3c5:3378:9d4b 59125 \ + typ host generation 0 network-id 4 network-cost 50"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!( + c.addr(), + "[fd00:f8aa:3ff5:5914:90bf:3c5:3378:9d4b]:59125" + .parse() + .unwrap() + ); + assert_eq!(c.proto(), Protocol::Udp); + + let a = "a=candidate:2113932030 1 tcp 2113932030 142.250.82.253 19305 \ + typ host tcptype passive generation 0"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "142.250.82.253:19305".parse().unwrap()); + assert_eq!(c.proto(), Protocol::Tcp); + assert_eq!(c.tcptype(), Some(TcpType::Passive)); + + let a = + "a=candidate:2113932029 1 ssltcp 2113932029 142.250.82.252 443 typ host generation 0"; + let (c, _) = candidate_attribute().parse(a).unwrap(); + assert_eq!(c.addr(), "142.250.82.252:443".parse().unwrap()); + assert_eq!(c.proto(), Protocol::SslTcp); + assert_eq!(c.tcptype(), None); + } + + #[test] + fn parse_firefox_missing_setup_on_mid1() { + let sdp = "v=0\r\n\ + o=mozilla...THIS_IS_SDPARTA-99.0 7710052215259647220 2 IN IP4 0.0.0.0\r\n\ + s=-\r\n\ + t=0 0\r\n\ + a=fingerprint:sha-256 A6:64:23:37:94:7E:4B:40:F6:62:86:8C:DD:09:D5:08:7E:D4:0E:68:58:93:45:EC:99:F2:91:F7:19:72:E7:BB\r\n\ + a=group:BUNDLE 0 hxI i1X mxk B3D kNI nbB xIZ bKm Hkn\r\n\ + a=ice-options:trickle\r\n\ + a=msid-semantic:WMS *\r\n\ + m=audio 0 UDP/TLS/RTP/SAVPF 0\r\n\ + c=IN IP4 0.0.0.0\r\n\ + a=inactive\r\n\ + a=mid:1\r\n\ + a=rtpmap:0 PCMU/8000\r\n\ + "; + + let (sdp, _) = sdp_parser().parse(sdp).unwrap(); + + // Firefox 'a=setup' attribute can be missing + // the "a=setup:" sdp attribute (which should be mandatory) + // - https://www.rfc-editor.org/rfc/rfc5763#section-5 + // - https://www.rfc-editor.org/rfc/rfc4145 + + assert!(sdp.media_lines[0].setup().is_none()); // must not crash + } + + #[test] + fn parse_no_media_c_line() { + let sdp = "v=0\r\n\ + o=- 0 0 IN IP4 172.17.0.1\r\n\ + s=-\r\n\ + c=IN IP4 172.17.0.1\r\n\ + t=0 0\r\n\ + m=application 9999 UDP/DTLS/SCTP webrtc-datachannel\r\n\ + a=mid:0\r\n\ + a=ice-options:ice2\r\n\ + a=ice-ufrag:libp2p+webrtc+v1/a75469cf670c4079f8c06af4a963c8a1:libp2p+webrtc+v1/a75469cf670c4079f8c06af4a963c8a1\r\n\ + a=ice-pwd:libp2p+webrtc+v1/a75469cf670c4079f8c06af4a963c8a1:libp2p+webrtc+v1/a75469cf670c4079f8c06af4a963c8a1\r\n\ + a=fingerprint:sha-256 FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF:FF\r\n\ + a=setup:actpass\r\n\ + a=sctp-port:5000\r\n\ + a=max-message-size:16384\r\n\ + "; + + let parsed = sdp_parser().parse(sdp); + println!("{:?}", parsed); + parsed.expect("to parse ok"); + } + + /// Test parsing an SDP offer containing H.265 with sprop-max-don-diff. + /// Verifies the fmtp line is correctly parsed into FormatParam::SpropMaxDonDiff + /// alongside H265ProfileTierLevel. + #[test] + fn parse_sdp_h265_with_sprop_max_don_diff() { + // Fingerprint split across lines to stay within 110 cols. + let sdp = concat!( + "v=0\r\n", + "o=- 1234567890 2 IN IP4 127.0.0.1\r\n", + "s=-\r\n", + "t=0 0\r\n", + "a=group:BUNDLE 0\r\n", + "a=fingerprint:sha-256 ", + "AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:", + "66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:", + "22:33:44:55:66:77:88:99\r\n", + "a=msid-semantic: WMS\r\n", + "m=video 9 UDP/TLS/RTP/SAVPF 102 103\r\n", + "c=IN IP4 0.0.0.0\r\n", + "a=rtcp:9 IN IP4 0.0.0.0\r\n", + "a=ice-ufrag:abcd\r\n", + "a=ice-pwd:abcdefghijklmnopqrstuv\r\n", + "a=ice-options:trickle\r\n", + "a=fingerprint:sha-256 ", + "AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:", + "66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:", + "22:33:44:55:66:77:88:99\r\n", + "a=setup:actpass\r\n", + "a=mid:0\r\n", + "a=sendrecv\r\n", + "a=rtpmap:102 H265/90000\r\n", + "a=fmtp:102 profile-id=1;tier-flag=0;", + "level-id=93;sprop-max-don-diff=32\r\n", + "a=rtpmap:103 rtx/90000\r\n", + "a=fmtp:103 apt=102\r\n", + ); + + let (sdp, _remaining) = sdp_parser().parse(sdp).expect("SDP should parse"); + + // Find the video media section + assert_eq!(sdp.media_lines.len(), 1); + let media = &sdp.media_lines[0]; + + // Find fmtp for PT 102 (H.265) + let fmtp_values: Vec<_> = media + .attrs + .iter() + .filter_map(|a| { + if let MediaAttribute::Fmtp { pt, values } = a { + if *pt == 102.into() { + return Some(values.clone()); + } + } + None + }) + .flatten() + .collect(); + + // Should contain H265ProfileTierLevel + assert!( + fmtp_values + .iter() + .any(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))), + "Should have H265ProfileTierLevel param" + ); + + // Should contain SpropMaxDonDiff(32) + assert!( + fmtp_values + .iter() + .any(|p| matches!(p, FormatParam::SpropMaxDonDiff(32))), + "Should have SpropMaxDonDiff(32) param, got: {:?}", + fmtp_values + ); + + // Verify PTL values + if let Some(FormatParam::H265ProfileTierLevel(ptl)) = fmtp_values + .iter() + .find(|p| matches!(p, FormatParam::H265ProfileTierLevel(_))) + { + assert_eq!(ptl.profile_id(), 1); + assert_eq!(ptl.tier_flag(), 0); + assert_eq!(ptl.level_id(), 93); + } else { + panic!("H265ProfileTierLevel not found"); + } + } +} + +// Safari addTransceiver('audio', {direction: 'sendonly'})) + +// v=0 +// o=- 1625652694357831865 2 IN IP4 127.0.0.1 +// s=- +// t=0 0 +// a=group:BUNDLE 0 +// a=msid-semantic: WMS +// m=audio 9 UDP/TLS/RTP/SAVPF 111 103 9 102 0 8 105 13 110 113 126 +// c=IN IP4 0.0.0.0\na=rtcp:9 IN IP4 0.0.0.0 +// a=ice-ufrag:CTNF +// a=ice-pwd:XbIkl+k8mwLz60TFwb9crzbz +// a=ice-options:trickle +// a=fingerprint:sha-256 77:8C:32:B8:DA:4E:78:64:C8:4A:2A:E2:1D:60:2A:83:6B:51:B8:D5:EE:5A:ED:75:4E:C6:98:2D:78:4D:94:D8 +// a=setup:actpass +// a=mid:0 +// a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level +// a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +// a=sendonly +// a=msid:- 39a7c3c3-ab8c-4b25-a47b-db52d89c2db1 +// a=rtcp-mux +// a=rtpmap:111 opus/48000/2 +// a=rtcp-fb:111 transport-cc +// a=fmtp:111 minptime=10;useinbandfec=1 +// a=rtpmap:103 ISAC/16000 +// a=rtpmap:9 G722/8000 +// a=rtpmap:102 ILBC/8000 +// a=rtpmap:0 PCMU/8000 +// a=rtpmap:8 PCMA/8000 +// a=rtpmap:105 CN/16000 +// a=rtpmap:13 CN/8000 +// a=rtpmap:110 telephone-event/48000 +// a=rtpmap:113 telephone-event/16000 +// a=rtpmap:126 telephone-event/8000 +// a=ssrc:457025658 cname:6OQ+jzZE+UnQgSUr +// a=ssrc:457025658 msid:- 39a7c3c3-ab8c-4b25-a47b-db52d89c2db1 +// a=ssrc:457025658 mslabel:- +// a=ssrc:457025658 label:39a7c3c3-ab8c-4b25-a47b-db52d89c2db1 + +// Chrome addTransceiver('audio', {direction: 'sendonly'})) + +// v=0 +// o=- 6740661649996974832 2 IN IP4 127.0.0.1 +// s=- +// t=0 0 +// a=group:BUNDLE 0 +// a=msid-semantic: WMS +// m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126 +// c=IN IP4 0.0.0.0 +// a=rtcp:9 IN IP4 0.0.0.0 +// a=ice-ufrag:Y28C +// a=ice-pwd:W6fHYINgGi9QF0BHdM/kchTW +// a=ice-options:trickle +// a=fingerprint:sha-256 E0:C0:2D:52:8D:FA:14:69:A0:A5:5D:63:E0:82:92:DB:37:38:D2:F3:12:D0:1F:4E:E5:6F:1A:F5:C3:97:6B:32 +// a=setup:actpass +// a=mid:0 +// a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level +// a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +// a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +// a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid +// a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +// a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +// a=sendonly +// a=msid:- 7a08dda6-518f-4027-b707-410a6d414176 +// a=rtcp-mux +// a=rtpmap:111 opus/48000/2 +// a=rtcp-fb:111 transport-cc +// a=fmtp:111 minptime=10;useinbandfec=1 +// a=rtpmap:103 ISAC/16000 +// a=rtpmap:104 ISAC/32000 +// a=rtpmap:9 G722/8000 +// a=rtpmap:0 PCMU/8000 +// a=rtpmap:8 PCMA/8000 +// a=rtpmap:106 CN/32000 +// a=rtpmap:105 CN/16000 +// a=rtpmap:13 CN/8000 +// a=rtpmap:110 telephone-event/48000 +// a=rtpmap:112 telephone-event/32000 +// a=rtpmap:113 telephone-event/16000 +// a=rtpmap:126 telephone-event/8000 +// a=ssrc:2147603131 cname:TbS1Ajv9obq6/63I +// a=ssrc:2147603131 msid:- 7a08dda6-518f-4027-b707-410a6d414176 +// a=ssrc:2147603131 mslabel:- +// a=ssrc:2147603131 label:7a08dda6-518f-4027-b707-410a6d414176 + +// Chrome add video +// pc.addTransceiver("video", { +// direction: 'sendonly', +// sendEncodings: [ +// { rid: 'hi' }, +// { rid: 'lo', scaleResolutionDownBy: 2 }, +// ] +// }) + +// v=0 +// o=- 2847298852000198709 3 IN IP4 127.0.0.1 +// s=- +// t=0 0 +// a=group:BUNDLE 0 +// a=extmap-allow-mixed +// a=msid-semantic: WMS +// m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 114 +// c=IN IP4 0.0.0.0 +// a=rtcp:9 IN IP4 0.0.0.0 +// a=ice-ufrag:numD +// a=ice-pwd:L/hHurqERpVUACiI8To+vXLv +// a=ice-options:trickle +// a=fingerprint:sha-256 A5:79:72:59:3D:32:74:9C:44:70:DE:39:15:C3:99:51:32:6E:0D:F0:60:DD:2F:31:90:E5:96:B4:1D:CA:48:E1 +// a=setup:actpass +// a=mid:0 +// a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +// a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +// a=extmap:3 urn:3gpp:video-orientation +// a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +// a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +// a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +// a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +// a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +// a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +// a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +// a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +// a=extmap:13 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +// a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 +// a=sendonly +// a=msid:- 7d27ad91-4770-4a4d-a053-2b06233753d2 +// a=rtcp-mux +// a=rtcp-rsize +// a=rtpmap:96 VP8/90000 +// a=rtcp-fb:96 goog-remb +// a=rtcp-fb:96 transport-cc +// a=rtcp-fb:96 ccm fir +// a=rtcp-fb:96 nack +// a=rtcp-fb:96 nack pli +// a=rtpmap:97 rtx/90000 +// a=fmtp:97 apt=96 +// a=rtpmap:102 H264/90000 +// a=rtcp-fb:102 goog-remb +// a=rtcp-fb:102 transport-cc +// a=rtcp-fb:102 ccm fir +// a=rtcp-fb:102 nack +// a=rtcp-fb:102 nack pli +// a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +// a=rtpmap:103 rtx/90000 +// a=fmtp:103 apt=102 +// a=rtpmap:104 H264/90000 +// a=rtcp-fb:104 goog-remb +// a=rtcp-fb:104 transport-cc +// a=rtcp-fb:104 ccm fir +// a=rtcp-fb:104 nack +// a=rtcp-fb:104 nack pli +// a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +// a=rtpmap:105 rtx/90000 +// a=fmtp:105 apt=104 +// a=rtpmap:106 H264/90000 +// a=rtcp-fb:106 goog-remb +// a=rtcp-fb:106 transport-cc +// a=rtcp-fb:106 ccm fir +// a=rtcp-fb:106 nack +// a=rtcp-fb:106 nack pli +// a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +// a=rtpmap:107 rtx/90000 +// a=fmtp:107 apt=106 +// a=rtpmap:108 H264/90000 +// a=rtcp-fb:108 goog-remb +// a=rtcp-fb:108 transport-cc +// a=rtcp-fb:108 ccm fir +// a=rtcp-fb:108 nack +// a=rtcp-fb:108 nack pli +// a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +// a=rtpmap:109 rtx/90000 +// a=fmtp:109 apt=108 +// a=rtpmap:127 H264/90000 +// a=rtcp-fb:127 goog-remb +// a=rtcp-fb:127 transport-cc +// a=rtcp-fb:127 ccm fir +// a=rtcp-fb:127 nack +// a=rtcp-fb:127 nack pli +// a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +// a=rtpmap:125 rtx/90000 +// a=fmtp:125 apt=127 +// a=rtpmap:39 H264/90000 +// a=rtcp-fb:39 goog-remb +// a=rtcp-fb:39 transport-cc +// a=rtcp-fb:39 ccm fir +// a=rtcp-fb:39 nack +// a=rtcp-fb:39 nack pli +// a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +// a=rtpmap:40 rtx/90000 +// a=fmtp:40 apt=39 +// a=rtpmap:45 AV1/90000 +// a=rtcp-fb:45 goog-remb +// a=rtcp-fb:45 transport-cc +// a=rtcp-fb:45 ccm fir +// a=rtcp-fb:45 nack +// a=rtcp-fb:45 nack pli +// a=fmtp:45 level-idx=5;profile=0;tier=0 +// a=rtpmap:46 rtx/90000 +// a=fmtp:46 apt=45 +// a=rtpmap:98 VP9/90000 +// a=rtcp-fb:98 goog-remb +// a=rtcp-fb:98 transport-cc +// a=rtcp-fb:98 ccm fir +// a=rtcp-fb:98 nack +// a=rtcp-fb:98 nack pli +// a=fmtp:98 profile-id=0 +// a=rtpmap:99 rtx/90000 +// a=fmtp:99 apt=98 +// a=rtpmap:100 VP9/90000 +// a=rtcp-fb:100 goog-remb +// a=rtcp-fb:100 transport-cc +// a=rtcp-fb:100 ccm fir +// a=rtcp-fb:100 nack +// a=rtcp-fb:100 nack pli +// a=fmtp:100 profile-id=2 +// a=rtpmap:101 rtx/90000 +// a=fmtp:101 apt=100 +// a=rtpmap:112 red/90000 +// a=rtpmap:113 rtx/90000 +// a=fmtp:113 apt=112 +// a=rtpmap:114 ulpfec/90000 +// a=rid:hi send +// a=rid:lo send +// a=simulcast:send hi;lo + +// DataChannel only in Safari +// v=0 +// o=- 4611516372927609806 2 IN IP4 127.0.0.1 +// s=- +// t=0 0 +// a=group:BUNDLE 0 +// a=extmap-allow-mixed +// a=msid-semantic: WMS +// m=application 9 UDP/DTLS/SCTP webrtc-datachannel +// c=IN IP4 0.0.0.0 +// a=ice-ufrag:HhS+ +// a=ice-pwd:FhYTGhlAtKCe6KFIX8b+AThW +// a=ice-options:trickle +// a=fingerprint:sha-256 B4:12:1C:7C:7D:ED:F1:FA:61:07:57:9C:29:BE:58:E3:BC:41:E7:13:8E:7D:D3:9D:1F:94:6E:A5:23:46:94:23 +// a=setup:actpass +// a=mid:0 +// a=sctp-port:5000 +// a=max-message-size:262144 diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/session.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/session.rs new file mode 100644 index 000000000..e1efa1ae3 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/session.rs @@ -0,0 +1,1329 @@ +use std::collections::{HashMap, VecDeque}; +use std::ops::RangeInclusive; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::Event; +use crate::bwe::BweKind; +use crate::bwe_::Bwe; +use crate::config::KeyingMaterial; +use crate::config_mod::RtcpReportIntervals; +use crate::crypto::CryptoProvider; +use crate::crypto::dtls::SrtpProfile; +use crate::format::CodecConfig; +use crate::format::PayloadParams; +use crate::format::Vp9PacketizerMode; +use crate::io::DatagramSend; +use crate::media::AppSpecificFeedback; +use crate::media::Media; +use crate::media::{KeyframeRequestKind, MID_PROBE}; +use crate::media::{MediaAdded, MediaChanged}; +use crate::pacer::PacerControl; +use crate::pacer::{Pacer, PacerImpl}; +use crate::rtp::{Extension, RawPacket}; +use crate::rtp_::Direction; +use crate::rtp_::MidRid; +use crate::rtp_::Pli; +use crate::rtp_::Pt; +use crate::rtp_::SRTCP_OVERHEAD; +use crate::rtp_::SeqNo; +use crate::rtp_::{Bitrate, ExtensionMap, Goodbye, Mid, ReportList, Rtcp, RtcpFb}; +use crate::rtp_::{Dlrr, DlrrItem, ExtendedReport, ReportBlock}; +use crate::rtp_::{RtpHeader, SessionId, TwccPacketId, extend_u16}; +use crate::rtp_::{SrtpContext, Ssrc}; +use crate::rtp_::{TwccRecvRegister, TwccSendRegister}; +use crate::stats::StatsSnapshot; +use crate::streams::{RtpPacket, StreamTimeoutConfig, Streams}; +use crate::util::{Soonest, SystemTimeExt, already_happened, not_happening}; +use crate::{Reason, net}; +use crate::{RtcConfig, RtcError}; + +/// Minimum time we delay between sending nacks. This should be +/// set high enough to not cause additional problems in very bad +/// network conditions. +const NACK_MIN_INTERVAL: Duration = Duration::from_millis(33); + +/// Delay between reports of TWCC. This is deliberately very low. +const TWCC_INTERVAL: Duration = Duration::from_millis(50); + +/// Maximum number of pending RRTR entries before oldest are dropped. +const MAX_PENDING_RRTRS: usize = 300; + +/// Maximum number of DLRR items to include in a single ExtendedReport. +const MAX_DLRR_PER_REPORT: usize = 50; + +pub(crate) struct Session { + id: SessionId, + + // These fields are pub to allow session_sdp.rs modify them. + // Notice the fields are maybe not in m-line index order since the app + // might be spliced in somewhere. + pub medias: Vec, + + // The actual RTP encoded streams. + pub streams: Streams, + + /// The app m-line. Spliced into medias above. + app: Option<(Mid, usize)>, + + reordering_size_audio: usize, + reordering_size_video: usize, + intervals: RtcpReportIntervals, + pub send_buffer_audio: usize, + pub send_buffer_video: usize, + + /// Extension mappings are _per BUNDLE_, but we can only have one a=group BUNDLE + /// in WebRTC (one ice connection), so they are effectively per session. + pub exts: ExtensionMap, + + // Configuration of how we are sending/receiving media. + pub codec_config: CodecConfig, + + // Payload params for SSRC 0 BWE probes. + // Lazy init when we see the PT. + probe_payload_params: Option, + + srtp_rx: Option, + srtp_tx: Option, + last_nack: Instant, + last_twcc: Instant, + twcc: u64, + twcc_rx_register: TwccRecvRegister, + twcc_tx_register: TwccSendRegister, + max_rx_seq_lookup: HashMap, + + bwe: Option, + + enable_twcc_feedback: bool, + + /// A pacer for sending RTP at specific rate. + pacer: PacerImpl, + pacer_control: PacerControl, + + // temporary buffer when getting the next (unencrypted) RTP packet from Media line. + poll_packet_buf: Vec, + + // Next packet for RtpPacket event. + pending_packet: Option, + + // Whether we sent a single outgoing RTP packet. + packet_first_sent: bool, + + // Total RTP payload bytes for media, including retransmissions. + media_bytes_rx: u64, + media_bytes_tx: u64, + + pub ice_lite: bool, + + /// Whether we are running in RTP-mode. + pub rtp_mode: bool, + + /// VP9 packetizer mode. + vp9_packetizer_mode: Vp9PacketizerMode, + + feedback_tx: VecDeque, + feedback_rx: VecDeque, + rtp_closing: bool, + + /// Pending RRTRs from remote SSRCs in FIFO order, deduplicated by SSRC. + /// A new RRTR for an already-pending SSRC updates its value in place. + /// Capped at 300 entries; at most 50 are consumed per DLRR report. + pending_rrtrs: VecDeque<(Ssrc, LastRrtr)>, + + raw_packets: Option>>, + + // Pending application-specific feedback (PSFB FMT=15) messages to emit as events. + pending_app_feedback: Option, + + /// Target MTU (start) and warn threshold (end). Buffer sizing uses the + /// target; oversized outgoing datagrams above the warn threshold log a warning. + mtu: RangeInclusive, + + #[cfg(feature = "_internal_test_exports")] + pending_probe: Option, +} + +/// Last RRTR (Receiver Reference Time Report, RFC 3611 §4.4) received from a remote SSRC. +/// Stored at session level so we can respond with a DLRR regardless of which local stream +/// the remote is observing. +#[derive(Debug, Clone, Copy)] +struct LastRrtr { + /// Middle 32 bits of the RRTR NTP timestamp (compact NTP / LRR). + lrr: u32, + /// When the RRTR arrived locally. + received_at: Instant, +} + +impl Session { + pub fn new(config: &RtcConfig) -> Self { + let mut id = SessionId::new(); + // Max 2^62 - 1: https://bugzilla.mozilla.org/show_bug.cgi?id=861895 + const MAX_ID: u64 = 2_u64.pow(62) - 1; + while *id > MAX_ID { + id = (*id >> 1).into(); + } + let (pacer, bwe) = if let Some(config) = &config.bwe_config { + let rate = config.initial_bitrate; + let pacer = PacerImpl::leaky_bucket(rate * 2.0); + let bwe = Bwe::new(rate); + (pacer, Some(bwe)) + } else { + (PacerImpl::null(), None) + }; + + let enable_stats = config.stats_interval.is_some(); + + Session { + id, + medias: vec![], + streams: Streams::new(enable_stats, *config.mtu.end()), + app: None, + reordering_size_audio: config.reordering_size_audio, + reordering_size_video: config.reordering_size_video, + intervals: config.intervals, + send_buffer_audio: config.send_buffer_audio, + send_buffer_video: config.send_buffer_video, + exts: config.exts.clone(), + + // Both sending and receiving starts from the configured codecs. + // These can then be changed in the SDP OFFER/ANSWER dance. + codec_config: config.codec_config.clone(), + probe_payload_params: None, + + srtp_rx: None, + srtp_tx: None, + last_nack: already_happened(), + last_twcc: already_happened(), + twcc: 0, + twcc_rx_register: TwccRecvRegister::new(100), + twcc_tx_register: TwccSendRegister::new(1000), + max_rx_seq_lookup: HashMap::new(), + bwe, + enable_twcc_feedback: false, + pacer, + pacer_control: PacerControl::new(), + poll_packet_buf: vec![0; 2000], + pending_packet: None, + packet_first_sent: false, + media_bytes_rx: 0, + media_bytes_tx: 0, + ice_lite: config.ice_lite, + rtp_mode: config.rtp_mode, + vp9_packetizer_mode: config.vp9_packetizer_mode, + feedback_tx: VecDeque::new(), + feedback_rx: VecDeque::new(), + rtp_closing: false, + pending_rrtrs: VecDeque::new(), + raw_packets: if config.enable_raw_packets { + Some(VecDeque::new()) + } else { + None + }, + pending_app_feedback: None, + mtu: config.mtu.clone(), + #[cfg(feature = "_internal_test_exports")] + pending_probe: None, + } + } + + fn mtu(&self) -> usize { + *self.mtu.start() + } + + fn mtu_warn(&self) -> usize { + *self.mtu.end() + } + + pub fn id(&self) -> SessionId { + self.id + } + + pub fn set_app(&mut self, mid: Mid, index: usize) -> Result<(), String> { + if let Some((mid_existing, index_existing)) = self.app { + if mid_existing != mid { + return Err(format!("App mid changed {} != {}", mid, mid_existing,)); + } + if index_existing != index { + return Err(format!("App index changed {} != {}", index, index_existing,)); + } + } else { + self.app = Some((mid, index)); + } + Ok(()) + } + + pub fn app(&self) -> &Option<(Mid, usize)> { + &self.app + } + + pub fn set_keying_material( + &mut self, + mat: KeyingMaterial, + crypto: &CryptoProvider, + srtp_profile: SrtpProfile, + active: bool, + ) { + // TODO: rename this to `initialise_srtp_context`? + // Whether we're active or passive determines if we use the left or right + // hand side of the key material to derive input/output. + let left = active; + + self.srtp_rx = Some(SrtpContext::new(crypto, srtp_profile, &mat, !left)); + self.srtp_tx = Some(SrtpContext::new(crypto, srtp_profile, &mat, left)); + } + + pub fn handle_timeout(&mut self, now: Instant) -> Result<(), RtcError> { + if self.rtp_closing { + return Ok(()); + } + + // Payload any waiting frames + self.do_payload()?; + + let sender_ssrc = self.streams.first_ssrc_local(); + + let do_nack = now >= self.nack_at().unwrap_or(not_happening()); + + self.streams.handle_timeout( + now, + sender_ssrc, + do_nack, + &self.medias, + StreamTimeoutConfig { + codecs: &self.codec_config, + intervals: self.intervals, + }, + &mut self.feedback_tx, + ); + + // Flush any pending RRTRs as a DLRR alongside the Sender Report compound packet. + // We piggyback on the SR cadence so the DLRR doesn't generate extra standalone + // RTCP traffic that could perturb bandwidth estimation. + // Consume at most 50 entries per report; the rest remain for subsequent SRs. + if !self.pending_rrtrs.is_empty() + && self + .feedback_tx + .iter() + .any(|r| matches!(r, Rtcp::SenderReport(_))) + { + let n = self.pending_rrtrs.len().min(MAX_DLRR_PER_REPORT); + let items: Vec = self + .pending_rrtrs + .drain(..n) + .map(|(ssrc, rrtr)| { + let delay = now.saturating_duration_since(rrtr.received_at); + let last_rr_delay = ((delay.as_micros() * 65_536) / 1_000_000) as u32; + DlrrItem { + ssrc, + last_rr_time: rrtr.lrr, + last_rr_delay, + } + }) + .collect(); + + self.feedback_tx + .push_back(Rtcp::ExtendedReport(ExtendedReport { + ssrc: sender_ssrc, + blocks: vec![ReportBlock::Dlrr(Dlrr { items })], + })); + } + + if do_nack { + self.last_nack = now; + } + + self.update_queue_state(now); + + if let Some(twcc_at) = self.twcc_at() { + if now >= twcc_at { + self.create_twcc_feedback(sender_ssrc, now); + } + } + + self.handle_timeout_bwe(now); + + Ok(()) + } + + fn handle_timeout_bwe(&mut self, now: Instant) { + let Some(bwe) = self.bwe.as_mut() else { + return; + }; + + // We can only run probes after first packet is sent and there + // are any queues that can handle padding requests. + let do_probe = self.packet_first_sent && self.pacer.has_padding_queue(); + + if let Some(probe_config) = bwe.handle_timeout(now, do_probe) { + // Only start the probe in the pacer if the estimator accepted it. + if bwe.start_probe(probe_config, now) { + #[cfg(feature = "_internal_test_exports")] + { + self.pending_probe = Some(probe_config); + } + self.pacer.start_probe(probe_config); + } + } + + // Check if active probe just completed + if let Some(cluster_id) = self.pacer.check_probe_complete(now) { + bwe.end_probe(now, cluster_id); + } + } + + fn update_queue_state(&mut self, now: Instant) { + let iter = self.streams.streams_tx().map(|m| m.queue_state(now)); + + let Some(padding_request) = self.pacer.handle_timeout(now, iter) else { + return; + }; + + let stream = self + .streams + .stream_tx_by_midrid(padding_request.midrid) + .expect("pacer to use an existing stream"); + + stream.generate_padding(padding_request.padding); + } + + fn create_twcc_feedback(&mut self, sender_ssrc: Ssrc, now: Instant) -> Option<()> { + self.last_twcc = now; + let mut twcc = self.twcc_rx_register.build_report(self.mtu() - 100)?; + + // These SSRC are on media level, but twcc is on session level, + // we fill in the first discovered media SSRC in each direction. + twcc.sender_ssrc = sender_ssrc; + twcc.ssrc = self.streams.first_ssrc_remote(); + + trace!("Created feedback TWCC: {:?}", twcc); + self.feedback_tx.push_front(Rtcp::Twcc(twcc)); + Some(()) + } + + /// Enqueue an application-specific feedback message (PSFB FMT=15, PT=206) for + /// transmission as part of the regular RTCP compound packet. + pub(crate) fn send_app_specific_feedback( + &mut self, + sender_ssrc: Ssrc, + media_ssrc: Ssrc, + payload: impl Into>, + ) { + use crate::rtp_::AppSpecificFeedback as RtcpAppFeedback; + let feedback = RtcpAppFeedback { + sender_ssrc, + media_ssrc, + payload: payload.into(), + }; + self.feedback_tx + .push_back(Rtcp::AppSpecificFeedback(feedback)); + } + + /// Enqueue a Picture Loss Indication (PLI, PT=206 FMT=1) for `media_ssrc`, to be + /// sent as part of the regular RTCP compound packet. The feedback packet's sender + /// SSRC is set to `sender_ssrc`. + pub(crate) fn send_pli_feedback(&mut self, sender_ssrc: Ssrc, media_ssrc: Ssrc) { + self.feedback_tx.push_back(Rtcp::Pli(Pli { + sender_ssrc, + ssrc: media_ssrc, + })); + } + + pub fn handle_rtp_receive(&mut self, now: Instant, message: &[u8]) { + let Some(header) = RtpHeader::parse(message, &self.exts) else { + trace!("Failed to parse RTP header"); + return; + }; + + self.handle_rtp(now, header, message); + } + + pub fn handle_rtcp_receive(&mut self, now: Instant, message: &[u8]) { + // According to spec, the outer enclosing SRTCP packet should always be a SR or RR, + // even if it's irrelevant and empty. + // In practice I'm not sure that is happening, because libWebRTC hates empty packets. + self.handle_rtcp(now, message); + } + + fn mid_and_ssrc_for_header(&mut self, now: Instant, header: &RtpHeader) -> Option<(Mid, Ssrc)> { + let ssrc_header = header.ssrc; + + if let Some(r) = self.streams.mid_ssrc_rx_by_ssrc_or_rtx(now, ssrc_header) { + return Some(r); + } + + // Attempt to dynamically map this header to some Media/ReceiveStream. + self.map_dynamic(header); + + // The dynamic mapping might have added an entry by now. + if let Some(r) = self.streams.mid_ssrc_rx_by_ssrc_or_rtx(now, ssrc_header) { + return Some(r); + } + + // SSRC 0 is used for non-media BWE probes from libwebrtc. + // These probes need TWCC feedback but don't carry actual media. + if ssrc_header.is_probe() { + return self.ensure_probe_stream(header.payload_type); + } + + None + } + + /// Creates the probe stream on-demand for handling SSRC 0 BWE probes. + /// + /// No Media is created since probes don't carry real media - they only need + /// SRTP decryption and TWCC feedback. + fn ensure_probe_stream(&mut self, pt: Pt) -> Option<(Mid, Ssrc)> { + let ssrc: Ssrc = 0.into(); + let midrid = MidRid(MID_PROBE, None); + + // Add PayloadParams for this PT if not already configured. + // Probes may use different PTs, so we add them as we see them. + self.probe_payload_params = Some(PayloadParams::new_probe(pt)); + + // Create the stream with NACK suppressed (probes don't need retransmission) + self.streams.expect_stream_rx(ssrc, None, midrid, true); + + Some((MID_PROBE, ssrc)) + } + + fn map_dynamic(&mut self, header: &RtpHeader) { + // There are two strategies for dynamically mapping SSRC. Both use the RTP "mid" + // header extension. + // A) Mid+Rid - used when doing simulcast. Rid points out which + // simulcast layer is in use. There is a separate header + // to indicate repair (RTX) stream. + // B) Mid+PT - when not doing simulcast, the PT identifies whether + // this is a repair stream. + + let Some(mid) = header.ext_vals.mid else { + return; + }; + let rid = header.ext_vals.rid.or(header.ext_vals.rid_repair); + + // The media the mid points out. Bail if the mid points to something + // we don't know about. + let Some(media) = self.medias.iter_mut().find(|m| m.mid() == mid) else { + return; + }; + + // Figure out which payload the PT maps to. Either main or RTX. + let maybe_payload = self + .codec_config + .iter() + .find(|p| p.pt() == header.payload_type || p.resend() == Some(header.payload_type)); + + // If we don't find it, bail out. + let Some(payload) = maybe_payload else { + return; + }; + + if let Some(rid) = rid { + // Case A - use the rid_repair header to identify RTX. + let is_main = header.ext_vals.rid.is_some(); + + let midrid = MidRid(mid, Some(rid)); + + self.streams + .map_dynamic_by_rid(header.ssrc, midrid, media, *payload, is_main); + } else { + // Case B - the payload type identifies RTX. + let is_main = payload.pt() == header.payload_type; + + let midrid = MidRid(mid, None); + + self.streams + .map_dynamic_by_pt(header.ssrc, midrid, media, *payload, is_main); + } + } + + pub(crate) fn handle_rtp(&mut self, now: Instant, mut header: RtpHeader, buf: &[u8]) { + // Rewrite absolute-send-time (if present) to be relative to now. + header.ext_vals.update_absolute_send_time(now); + + trace!("Handle RTP: {:?}", header); + + // The ssrc is the _main_ ssrc (no the rtx, that might be in the header). + let Some((mid, ssrc)) = self.mid_and_ssrc_for_header(now, &header) else { + debug!("No mid/SSRC for header: {:?}", header); + return; + }; + + let srtp = match self.srtp_rx.as_mut() { + Some(v) => v, + None => { + trace!("Rejecting SRTP while missing SrtpContext"); + return; + } + }; + + // mid_and_ssrc_for_header guarantees stream exists. + // Media may not exist for internal probe streams (SSRC 0). + let stream = self.streams.stream_rx(&ssrc).unwrap(); + + let maybe_params = if ssrc.is_probe() { + self.probe_payload_params.as_ref() + } else { + main_payload_params(&self.codec_config, header.payload_type) + }; + + let params = match maybe_params { + Some(p) => p, + None => { + trace!( + "No payload params could be found (main or RTX) for {:?}", + header.payload_type + ); + return; + } + }; + let clock_rate = params.spec().rtp_clock_rate(); + let pt = params.pt(); + let is_repair = pt != header.payload_type; + + let max_seq_lookup = make_max_seq_lookup(&self.max_rx_seq_lookup); + + // is_repair controls whether update is updating the main register or the RTX register. + // Either way we get a seq_no_outer which is used to decrypt the SRTP. + let mut seq_no = stream.extend_seq(&header, is_repair, max_seq_lookup); + + if !stream.is_new_packet(is_repair, seq_no) { + // Dupe packet. This could be a potential SRTP replay attack, which means + // we should not spend any CPU cycles towards decrypting it. + trace!( + "Ignoring dupe packet mid: {} seq_no: {} is_repair: {}", + mid, seq_no, is_repair + ); + return; + } + + // The decrypted plaintext borrows from the SRTP scratch buffer. We + // narrow it down to the actual payload via slicing only, then build + // the final `Arc<[u8]>` in a single allocation at the end. + let data = match srtp.unprotect_rtp(buf, &header, *seq_no) { + Some(d) => d, + None => { + trace!( + "Failed to unprotect SRTP for SSRC: {} pt: {} mid: {} \ + rid: {:?} seq_no: {} is_repair: {}", + header.ssrc, + pt, + stream.mid(), + stream.rid(), + seq_no, + is_repair + ); + return; + } + }; + + let data = if header.has_padding { + match RtpHeader::unpad_payload(data) { + Some(d) => d, + None => { + trace!("unpadding of unprotected payload failed"); + return; + } + } + } else { + data + }; + + if let Some(raw_packets) = &mut self.raw_packets { + raw_packets.push_back(Box::new(RawPacket::RtpRx(header.clone(), data.to_vec()))); + } + + // Mark as received for TWCC purposes + if let Some(transport_cc) = header.ext_vals.transport_cc { + let prev = self.twcc_rx_register.max_seq(); + let extended = extend_u16(prev.map(|s| *s), transport_cc); + self.twcc_rx_register.update_seq(extended.into(), now); + } + + // Store largest seen seq_no for the SSRC. This is used in case we get SSRC changes + // like A -> B -> A. When we go back to A, we must keep the ROC. + update_max_seq(&mut self.max_rx_seq_lookup, header.ssrc, seq_no); + + // Register reception in nack registers. + let receipt_outer = stream.update_register(now, &header, clock_rate, is_repair, seq_no); + + // RTX packets must be rewritten to be a normal packet. This only changes the + // the seq_no, however MediaTime might be different when interpreted against the + // the "main" register. + let (receipt, data) = if is_repair { + // Drop RTX packets that are just empty padding. The payload here + // is empty because we would have done RtpHeader::unpad_payload above. + // For unpausing, it's enough with the stream.update() already done above. + if data.is_empty() { + return; + } + + // Rewrite the header, and strip the resent seq_no prefix from the body. + let data = stream.un_rtx(&mut header, data, pt); + + let max_seq_lookup = make_max_seq_lookup(&self.max_rx_seq_lookup); + + // Header has changed, which means we extend a new seq_no. This time + // without is_repair since this is the wrapped resend. This is the + // extended number of the main stream. + seq_no = stream.extend_seq(&header, false, max_seq_lookup); + + // header.ssrc is changed by un_rtx() above to be main SSRC. + update_max_seq(&mut self.max_rx_seq_lookup, header.ssrc, seq_no); + + // Now update the "main" register with the repaired packet info. + let receipt = stream.update_register(now, &header, clock_rate, false, seq_no); + (receipt, data) + } else { + // This is not RTX, the outer seq and time is what we use. The first + // stream.update will have updated the main register. + (receipt_outer, data) + }; + + // Probe packets (SSRC 0) contain only padding, no real media to process + if ssrc.is_probe() { + return; + } + + self.media_bytes_rx += data.len() as u64; + + // One-shot conversion to Arc<[u8]>: a single allocation that copies + // only the trimmed payload bytes out of the SRTP scratch buffer. + let payload: Arc<[u8]> = Arc::from(data); + + let packet = stream.handle_rtp(now, header, payload, seq_no, receipt.time); + + if self.rtp_mode { + // In RTP mode, we store the packet temporarily here for the next poll_output(). + // However only if this is a packet not seen before. This filters out spurious + // resends for padding. + if receipt.is_new_packet { + self.pending_packet = Some(packet); + } + } else { + // In non-RTP mode, we let the Media use a Depayloader. + // unwrap is fine because mid_and_ssrc_for_header guarantees it. + let media = self.medias.iter_mut().find(|m| m.mid() == mid).unwrap(); + media.depayload( + stream.rid(), + packet, + self.reordering_size_audio, + self.reordering_size_video, + &self.codec_config, + ); + } + } + + fn handle_rtcp(&mut self, now: Instant, buf: &[u8]) -> Option<()> { + let srtp: &mut SrtpContext = self.srtp_rx.as_mut()?; + let unprotected = srtp.unprotect_rtcp(buf)?; + + Rtcp::read_packet(&unprotected, &mut self.feedback_rx); + let mut need_configure_pacer = false; + + if let Some(raw_packets) = &mut self.raw_packets { + for fb in &self.feedback_rx { + raw_packets.push_back(Box::new(RawPacket::RtcpRx(fb.clone()))); + } + } + + for fb in RtcpFb::from_rtcp(self.feedback_rx.drain(..)) { + if let RtcpFb::AppSpecificFeedback(asf) = fb { + self.pending_app_feedback = Some(AppSpecificFeedback { + sender_ssrc: asf.sender_ssrc, + media_ssrc: asf.media_ssrc, + payload: asf.payload, + }); + continue; + } + + if let RtcpFb::Twcc(twcc) = fb { + trace!("Handle TWCC: {:?}", twcc); + let maybe_records = self.twcc_tx_register.apply_report(twcc, now); + + if let (Some(maybe_records), Some(bwe)) = (maybe_records, &mut self.bwe) { + bwe.update(maybe_records, now); + } + need_configure_pacer = true; + + // The funky thing about TWCC reports is that they are never stapled + // together with other RTCP packet. If they were though, we want to + // handle more packets. + continue; + } + + if let RtcpFb::Rrtr((rrtr, ssrc)) = fb { + // DLRR responder (RFC 3611 §4.5): store the remote's RRTR so we can + // reply with a DLRR, letting the remote compute RTT. + let lrr = (rrtr.ntp_time.as_ntp_64() >> 16) as u32; + let entry = LastRrtr { + lrr, + received_at: now, + }; + + // Update in place if this SSRC is already pending (preserves queue order). + if let Some(existing) = self.pending_rrtrs.iter_mut().find(|(s, _)| *s == ssrc) { + existing.1 = entry; + } else { + // Cap at 300 entries; drop the oldest if full. + if self.pending_rrtrs.len() >= MAX_PENDING_RRTRS { + self.pending_rrtrs.pop_front(); + } + self.pending_rrtrs.push_back((ssrc, entry)); + } + continue; + } + + if fb.is_for_rx() { + let Some(stream) = self.streams.stream_rx(&fb.ssrc()) else { + continue; + }; + stream.handle_rtcp(now, fb); + } else { + let Some(stream) = self.streams.stream_tx(&fb.ssrc()) else { + continue; + }; + stream.handle_rtcp(now, fb); + } + } + + // Not in the above if due to lifetime issues, still okay because the method + // doesn't do anything when BWE isn't configured. + if need_configure_pacer { + self.configure_pacer(); + } + + Some(()) + } + + pub fn poll_event(&mut self) -> Option { + #[cfg(feature = "_internal_test_exports")] + { + if let Some(probe) = self.pending_probe.take() { + return Some(Event::Probe(probe)); + } + } + + if let Some(bitrate_estimate) = self.bwe.as_mut().and_then(|bwe| bwe.poll_estimate()) { + return Some(Event::EgressBitrateEstimate(BweKind::Twcc( + bitrate_estimate, + ))); + } + + // If we're not ready to flow media, don't send any events. + if !self.ready_for_srtp() { + return None; + } + + if let Some(raw_packets) = &mut self.raw_packets { + if let Some(p) = raw_packets.pop_front() { + return Some(Event::RawPacket(p)); + } + } + + if let Some(feedback) = self.pending_app_feedback.take() { + return Some(Event::AppSpecificFeedback(feedback)); + } + + // This must be before pending_packet.take() since we need to emit the unpaused event + // before the first packet causing the unpause. + if let Some(paused) = self.streams.poll_stream_paused() { + if paused.paused { + if let Some(media) = self.medias.iter_mut().find(|m| m.mid() == paused.mid) { + // Drop held partial depacketizer state so pre-pause fragments can't + // complete into stale MediaData after the stream resumes. + media.reset_depayloaders_for_rid(paused.rid); + } + } + return Some(Event::StreamPaused(paused)); + } + + if self.rtp_mode { + if let Some(packet) = self.pending_packet.take() { + return Some(Event::RtpPacket(packet)); + } + } + + if let Some(req) = self.streams.poll_keyframe_request() { + return Some(Event::KeyframeRequest(req)); + } + + if let Some(report) = self.streams.poll_sender_feedback() { + return Some(Event::SenderFeedback(report)); + } + + if let Some((mid, bitrate)) = self.streams.poll_remb_request() { + return Some(Event::EgressBitrateEstimate(BweKind::Remb(mid, bitrate))); + } + + for media in &mut self.medias { + if media.need_open_event { + media.need_open_event = false; + + return Some(Event::MediaAdded(MediaAdded { + mid: media.mid(), + kind: media.kind(), + direction: media.direction(), + simulcast: media.simulcast().map(|s| s.clone().into()), + })); + } + + if media.need_changed_event { + media.need_changed_event = false; + return Some(Event::MediaChanged(MediaChanged { + mid: media.mid(), + direction: media.direction(), + })); + } + } + + None + } + + pub fn poll_event_fallible(&mut self) -> Result, RtcError> { + // Not relevant in rtp_mode, where the packets are picked up by poll_event(). + if self.rtp_mode { + return Ok(None); + } + + for media in &mut self.medias { + if let Some(e) = media.poll_sample(&self.codec_config)? { + return Ok(Some(Event::MediaData(e))); + } + } + + Ok(None) + } + + fn ready_for_srtp(&self) -> bool { + self.srtp_rx.is_some() && self.srtp_tx.is_some() + } + + pub fn poll_datagram(&mut self, now: Instant) -> Option { + // Time must have progressed forward from start value. + if now == already_happened() { + return None; + } + + let x = if self.rtp_closing { + self.poll_feedback() + } else { + None.or_else(|| self.poll_feedback()) + .or_else(|| self.poll_packet(now)) + }; + + if let Some(x) = &x { + // In RTP mode we trust the API user feeds the RTP packet sizes they + // need for the MTU they are targeting. This warning is only for when + // str0m does the RTP packetization. + if !self.rtp_mode && x.len() > self.mtu_warn() { + warn!("RTP above MTU {}: {}", self.mtu_warn(), x.len()); + } + } + + x + } + + /// To be called in lieu of [`Self::poll_datagram`] when the owner is not in a position to transmit any + /// generated feedback, and thus such feedback should be dropped. + pub fn clear_feedback(&mut self) { + self.feedback_rx.clear(); + self.feedback_tx.clear(); + } + + pub fn close_rtp(&mut self) { + if self.rtp_closing { + return; + } + + self.rtp_closing = true; + self.clear_feedback(); + self.streams.reset_send_buffers(); + + for reports in ReportList::lists_from_iter(self.streams.local_sender_ssrcs()) { + self.feedback_tx + .push_back(Rtcp::Goodbye(Goodbye { reports })); + } + } + + pub fn is_rtp_closed(&self) -> bool { + self.rtp_closing && self.feedback_tx.is_empty() + } + + fn poll_feedback(&mut self) -> Option { + if self.feedback_tx.is_empty() { + return None; + } + + // Round to nearest multiple of 4 bytes. + let encryptable_mtu: usize = (self.mtu() - SRTCP_OVERHEAD) & !3; + assert!(encryptable_mtu % 4 == 0); + + let mut data = vec![0_u8; encryptable_mtu]; + + let mut raw_packets = self.raw_packets.as_mut(); + let output = move |fb| { + if let Some(raw_packets) = &mut raw_packets { + raw_packets.push_back(Box::new(RawPacket::RtcpTx(fb))); + } + }; + + let len = Rtcp::write_packet(&mut self.feedback_tx, &mut data, output); + + if len == 0 { + return None; + } + + data.truncate(len); + + let srtp = self.srtp_tx.as_mut()?; + let protected = srtp.protect_rtcp(&data); + + assert!( + protected.len() < self.mtu(), + "Encrypted SRTCP should be less than MTU" + ); + + Some(protected.into()) + } + + fn poll_packet(&mut self, now: Instant) -> Option { + let srtp_tx = self.srtp_tx.as_mut()?; + + // Figure out which, if any, queue to poll + // The cluster_id is captured by the pacer at poll time, before register_send() might clear it + let (midrid, cluster_id) = self.pacer.poll_queue()?; + let Some(media) = self.medias.iter().find(|m| m.mid() == midrid.mid()) else { + trace!("Pacer pointed to mid {} which has no media", midrid.mid()); + return None; + }; + + let buf = &mut self.poll_packet_buf; + let twcc_seq = self.twcc; + + let stream = self.streams.stream_tx_by_midrid(midrid)?; + + let params = &self.codec_config; + let exts = media.remote_extmap(); + + // TWCC might not be enabled for this m-line. Firefox do use TWCC, but not + // for audio. This is indiciated via the SDP. + let twcc_enabled = exts.id_of(Extension::TransportSequenceNumber).is_some(); + let twcc = twcc_enabled.then_some(&mut self.twcc); + + let receipt = stream.poll_packet(now, exts, twcc, params, buf)?; + + let PacketReceipt { + header, + seq_no, + is_padding, + payload_size, + } = receipt; + + trace!(payload_size, is_padding, "Poll RTP: {:?}", header); + + #[cfg(feature = "_internal_dont_use_log_stats")] + { + let kind = if is_padding { "padding" } else { "media" }; + + crate::log_stat!("PACKET_SENT", header.ssrc, payload_size, kind); + } + + self.pacer.register_send(now, payload_size.into(), midrid); + + if let Some(raw_packets) = &mut self.raw_packets { + raw_packets.push_back(Box::new(RawPacket::RtpTx(header.clone(), buf.clone()))); + } + + let protected = srtp_tx.protect_rtp(buf, &header, *seq_no); + + if twcc_enabled { + let packet_id = if let Some(cluster) = cluster_id { + TwccPacketId::with_cluster(twcc_seq, cluster) + } else { + TwccPacketId::new(twcc_seq) + }; + self.twcc_tx_register + .register_seq(packet_id, now, payload_size); + } + + // Update BWE subsystem + if let Some(bwe) = self.bwe.as_mut() { + bwe.on_media_sent(payload_size.into(), is_padding, now); + } + + if !is_padding && !header.ssrc.is_probe() { + self.media_bytes_tx += payload_size as u64; + } + + if !self.packet_first_sent { + self.packet_first_sent = true; + } + + Some(protected.into()) + } + + pub fn poll_timeout(&mut self) -> (Option, Reason) { + if self.rtp_closing { + return if self.feedback_tx.is_empty() { + (None, Reason::NotHappening) + } else { + (Some(already_happened()), Reason::Feedback) + }; + } + + let feedback_at = self.regular_feedback_at(); + let nack_at = self.nack_at(); + let twcc_at = self.twcc_at(); + let pacing_at = self.pacer.poll_timeout(); + let packetize_at = self.medias.iter().flat_map(|m| m.poll_timeout()).next(); + let paused_at = self.paused_at(); + let send_stream_at = self.streams.send_stream(); + + // Gives us built-in reason + let bwe_at = self + .bwe + .as_ref() + .map(|bwe| bwe.poll_timeout()) + // We should never see this reason. + .unwrap_or((None, Reason::BweDelayControl)); + + (feedback_at, Reason::Feedback) + .soonest((nack_at, Reason::Nack)) + .soonest((twcc_at, Reason::Twcc)) + .soonest(pacing_at) + .soonest((packetize_at, Reason::Packetize)) + .soonest((paused_at, Reason::PauseCheck)) + .soonest((send_stream_at, Reason::SendStream)) + .soonest(bwe_at) + } + + pub fn has_mid(&self, mid: Mid) -> bool { + self.medias.iter().any(|m| m.mid() == mid) + } + + fn regular_feedback_at(&self) -> Option { + self.streams.regular_feedback_at(self.intervals) + } + + fn paused_at(&self) -> Option { + self.streams.paused_at() + } + + fn nack_at(&mut self) -> Option { + if !self.streams.any_nack_enabled() { + return None; + } + + Some(self.last_nack + NACK_MIN_INTERVAL) + } + + fn twcc_at(&self) -> Option { + let is_receiving = self.streams.is_receiving(); + if is_receiving && self.enable_twcc_feedback && self.twcc_rx_register.has_unreported() { + Some(self.last_twcc + TWCC_INTERVAL) + } else { + None + } + } + + pub fn enable_twcc_feedback(&mut self) { + if !self.enable_twcc_feedback { + debug!("Enable TWCC feedback"); + self.enable_twcc_feedback = true; + } + } + + pub fn visit_stats(&mut self, now: Instant, snapshot: &mut StatsSnapshot) { + for stream in self.streams.streams_tx() { + stream.visit_stats(snapshot, now); + } + + for stream in self.streams.streams_rx() { + stream.visit_stats(snapshot, now); + } + + snapshot.tx = self.media_bytes_tx; + snapshot.rx = self.media_bytes_rx; + snapshot.bwe_tx = self.bwe.as_ref().and_then(|bwe| bwe.last_estimate()); + + snapshot.egress_loss_fraction = self.twcc_tx_register.loss(Duration::from_secs(1), now); + snapshot.rtt = self.twcc_tx_register.rtt(); + snapshot.ingress_loss_fraction = self.twcc_rx_register.loss(); + } + + pub fn set_bwe_desired_bitrate(&mut self, desired_bitrate: Bitrate) { + if let Some(bwe) = self.bwe.as_mut() { + bwe.set_desired_bitrate(desired_bitrate); + self.configure_pacer(); + } + } + + pub fn reset_bwe(&mut self, init_bitrate: Bitrate) { + if let Some(bwe) = self.bwe.as_mut() { + bwe.reset(init_bitrate); + } + } + + #[allow(dead_code)] + pub fn line_count(&self) -> usize { + self.medias.len() + if self.app.is_some() { 1 } else { 0 } + } + + pub fn add_media(&mut self, media: Media) { + self.medias.push(media); + } + + pub fn medias(&self) -> &[Media] { + &self.medias + } + + pub fn remove_media(&mut self, mid: Mid) { + self.medias.retain(|media| media.mid() != mid); + self.streams.remove_streams_by_mid(mid); + } + + fn configure_pacer(&mut self) { + let Some(bwe) = self.bwe.as_mut() else { + return; + }; + + let Some(current_estimate) = bwe.last_estimate() else { + // No estimate yet, no padding + return; + }; + let is_overuse = bwe.is_overusing(); + + let has_active_media = self.has_active_outgoing_media(); + + // Calculate pacing and padding rates + let result = self + .pacer_control + .calculate(has_active_media, current_estimate, is_overuse); + + self.pacer.set_padding_rate(result.padding_rate); + self.pacer.set_pacing_rate(result.pacing_rate); + } + + fn has_active_outgoing_media(&self) -> bool { + self.medias + .iter() + .any(|m| m.direction().is_sending() && !m.disabled()) + } + + pub fn media_by_mid(&self, mid: Mid) -> Option<&Media> { + self.medias.iter().find(|m| m.mid() == mid) + } + + pub fn media_by_mid_mut(&mut self, mid: Mid) -> Option<&mut Media> { + self.medias.iter_mut().find(|m| m.mid() == mid) + } + + fn do_payload(&mut self) -> Result<(), RtcError> { + let mtu = self.mtu(); + for m in &mut self.medias { + m.do_payload( + &mut self.streams, + &self.codec_config, + self.vp9_packetizer_mode, + mtu, + )?; + } + + Ok(()) + } + + pub fn set_direction(&mut self, mid: Mid, direction: Direction) -> bool { + let Some(media) = self.media_by_mid_mut(mid) else { + return false; + }; + let old_dir = media.direction(); + if old_dir == direction { + return false; + } + + media.set_direction(direction); + + if old_dir.is_sending() && !direction.is_sending() { + self.streams.reset_buffers_tx(mid); + } + + let max_seq_lookup = make_max_seq_lookup(&self.max_rx_seq_lookup); + if old_dir.is_receiving() && !direction.is_receiving() { + self.streams.reset_buffers_rx(mid, max_seq_lookup); + } + + true + } + + pub fn stop_media(&mut self, mid: Mid) -> bool { + let Some(media) = self.media_by_mid_mut(mid) else { + return false; + }; + if media.disabled() { + return false; + } + + media.mark_stopped(); + self.set_direction(mid, Direction::Inactive); + + true + } + + pub fn is_request_keyframe_possible(&self, kind: KeyframeRequestKind) -> bool { + // TODO: It's possible to have different set of feedback enabled for different + // payload types. I.e. we could have FIR enabled for H264, but not for VP8. + // We might want to make this check more fine grained by testing which PT is + // in "active use" right now. + self.codec_config.iter().any(|r| match kind { + KeyframeRequestKind::Pli => r.fb_pli, + KeyframeRequestKind::Fir => r.fb_fir, + }) + } + + /// Checks whether the SRTP contexts are up. + pub fn is_connected(&self) -> bool { + self.srtp_rx.is_some() && self.srtp_tx.is_some() + } +} + +pub struct PacketReceipt { + pub header: RtpHeader, + pub seq_no: SeqNo, + pub is_padding: bool, + pub payload_size: usize, +} + +/// Find the PayloadParams for the given Pt, either when the Pt is the main Pt for the Codec or +/// when it's the RTX Pt. +fn main_payload_params(c: &CodecConfig, pt: Pt) -> Option<&PayloadParams> { + c.iter().find(|p| p.pt == pt || p.resend == Some(pt)) +} + +fn make_max_seq_lookup(map: &HashMap) -> impl Fn(Ssrc) -> Option + '_ { + |ssrc| map.get(&ssrc).cloned() +} + +fn update_max_seq(map: &mut HashMap, ssrc: Ssrc, seq_no: SeqNo) { + let current = map.entry(ssrc).or_insert(seq_no); + if seq_no > *current { + *current = seq_no; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RtcConfig; + use crate::io::DATAGRAM_MTU_TARGET; + + #[test] + fn session_mtu_matches_config() { + let cfg = RtcConfig::default(); + let s = Session::new(&cfg); + assert_eq!(s.mtu(), DATAGRAM_MTU_TARGET); + + let cfg = RtcConfig::default().set_mtu(900..=1280); + let s = Session::new(&cfg); + assert_eq!(s.mtu(), 900); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/stats.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/stats.rs new file mode 100644 index 000000000..d75aa2281 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/stats.rs @@ -0,0 +1,337 @@ +//! Statistics events. + +use std::collections::{HashMap, VecDeque}; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; + +use crate::Bitrate; +use crate::rtp::SeqNo; +use crate::rtp_::{Mid, Rid}; +use crate::{io::Protocol, rtp_::MidRid}; + +pub(crate) struct Stats { + last_now: Option, + events: VecDeque, + interval: Duration, +} + +pub(crate) struct StatsSnapshot { + pub peer_tx: u64, + pub peer_rx: u64, + pub tx: u64, + pub rx: u64, + pub egress_loss_fraction: Option, + pub ingress_loss_fraction: Option, + pub rtt: Option, + pub ingress: HashMap, + pub egress: HashMap, + pub bwe_tx: Option, + pub selected_candidate_pair: Option, + timestamp: Instant, +} + +impl StatsSnapshot { + pub(crate) fn new(timestamp: Instant) -> StatsSnapshot { + StatsSnapshot { + peer_rx: 0, + peer_tx: 0, + tx: 0, + rx: 0, + egress_loss_fraction: None, + ingress_loss_fraction: None, + ingress: HashMap::new(), + egress: HashMap::new(), + rtt: None, + bwe_tx: None, + selected_candidate_pair: None, + timestamp, + } + } +} + +// Output events + +#[derive(Debug, Clone)] +pub(crate) enum StatsEvent { + Peer(PeerStats), + MediaEgress(MediaEgressStats), + MediaIngress(MediaIngressStats), +} + +/// Peer statistics in [`Event::PeerStats`][crate::Event::PeerStats]. +/// +/// This event is generated roughly every second +#[derive(Debug, Clone)] +pub struct PeerStats { + /// Total bytes transmitted. + pub peer_bytes_rx: u64, + /// Total bytes received. + pub peer_bytes_tx: u64, + /// Total bytes transmitted, only counting media traffic (rtp payload). + pub bytes_rx: u64, + /// Total bytes received, only counting media traffic (rtp payload). + pub bytes_tx: u64, + /// Timestamp when this event was generated. + pub timestamp: Instant, + /// The last egress bandwidth estimate from the BWE subsystem, if enabled. + pub bwe_tx: Option, + /// The egress loss over the last second. + pub egress_loss_fraction: Option, + /// The ingress loss since the last stats event. + pub ingress_loss_fraction: Option, + /// The most recent RTT since the last stats event. + pub rtt: Option, + /// The selected ICE candidate pair, if any. + pub selected_candidate_pair: Option, +} + +#[derive(Debug, Clone)] +/// ICE candidate pair statistics. +pub struct CandidatePairStats { + /// The selected protocol. + pub protocol: Protocol, + /// The local candidate. + pub local: CandidateStats, + /// The remote candidate. + pub remote: CandidateStats, + /// Round-trip time of the most recent STUN binding transaction on the + /// nominated pair. + /// + /// Spec equivalent of [`RTCIceCandidatePairStats.currentRoundTripTime`][1]. + /// + /// This is sourced from the ICE keepalive/consent traffic and is + /// available even on receive-only endpoints where RTP-based RTT + /// estimates aren't. + /// + /// [1]: https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-currentroundtriptime + pub current_round_trip_time: Option, + /// Sum of all RTTs measured from successful STUN binding transactions + /// on the nominated pair over its lifetime. Divide by + /// [`responses_received`][Self::responses_received] for the average. + /// + /// Spec equivalent of [`RTCIceCandidatePairStats.totalRoundTripTime`][1]. + /// + /// [1]: https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-totalroundtriptime + pub total_round_trip_time: Duration, + /// Total number of STUN binding responses received on the nominated + /// pair over its lifetime. + /// + /// Spec equivalent of [`RTCIceCandidatePairStats.responsesReceived`][1]. + /// + /// [1]: https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-responsesreceived + pub responses_received: u64, +} + +#[derive(Debug, Clone)] +/// ICE candidate statistics. +pub struct CandidateStats { + /// The address of the candidate. + pub addr: SocketAddr, +} + +/// Outgoing media statistics in [`Event::MediaEgressStats`][crate::Event::MediaEgressStats]. +/// +/// note: when simulcast is disabled, `rid` is `None` +#[derive(Debug, Clone)] +pub struct MediaEgressStats { + /// The identifier of the media these stats are for. + pub mid: Mid, + /// The Rid identifier in case of simulcast. + pub rid: Option, + /// Total bytes sent, including retransmissions. + /// + /// Spec equivalent to [`RTCSentRtpStreamStats.bytesSent`][1]. + /// + /// [1]: https://www.w3.org/TR/webrtc-stats/#dom-rtcsentrtpstreamstats-bytessent + pub bytes: u64, + /// Total number of rtp packets sent, including retransmissions + /// + /// Spec equivalent of [`RTCSentRtpStreamStats.packetsSent`][1]. + /// + /// [1]: https://www.w3.org/TR/webrtc-stats/#dom-rtcsentrtpstreamstats-packetssent + pub packets: u64, + /// Number of firs received. + pub firs: u64, + /// Number of plis received. + pub plis: u64, + /// Number of nacks received. + pub nacks: u64, + /// Round-trip-time extracted from the last RTCP receiver report. + pub rtt: Option, + /// Fraction of packets lost averaged from the RTCP receiver reports received. + /// `None` if no reports have been received since the last event + pub loss: Option, + /// Timestamp when this event was generated + pub timestamp: Instant, + /// Stats provided by the remote peer via ReceiverReports + pub remote: Option, +} + +/// Stats as reported by the remote side (via RTCP ReceiverReports). +#[derive(Debug, Clone)] +pub struct RemoteIngressStats { + /// The remotely calculated jitter. + pub jitter: u32, + /// The maximum extended sequence number received. + pub maximum_sequence_number: SeqNo, + /// The cumulative number of packets lost. + pub packets_lost: u64, +} + +/// Incoming media statistics in [`Event::MediaIngressStats`][crate::Event::MediaIngressStats]. +/// +/// note: when simulcast is disabled, `rid` is `None` +#[derive(Debug, Clone)] +pub struct MediaIngressStats { + /// The identifier of the media these stats are for. + pub mid: Mid, + /// The Rid identifier in case of simulcast. + pub rid: Option, + /// Total bytes received, including retransmissions. + pub bytes: u64, + /// Total number of rtp packets received, including retransmissions. + pub packets: u64, + /// Number of firs sent. + pub firs: u64, + /// Number of plis sent. + pub plis: u64, + /// Number of nacks sent. + pub nacks: u64, + /// Interarrival jitter, in RTP timestamp units, as reported in the last RTCP + /// receiver report we sent for this stream. + pub jitter: u32, + /// Round-trip-time extracted from the last RTCP XR DLRR report block. + pub rtt: Option, + /// Fraction of packets lost extracted from the last RTCP receiver report. + pub loss: Option, + /// Timestamp when this event was generated. + pub timestamp: Instant, + /// Stats provided by the remote peer via SenderReports + pub remote: Option, +} + +impl MediaIngressStats { + /// Merge `other` into `self`, mutating `self`. + /// + /// **Panics** if called with stats that don't have the same `(mid, rid)` pair. + pub(crate) fn merge_by_mid_rid(&mut self, other: &Self) { + assert!( + self.mid == other.mid, + "Cannot merge MediaIngressStats for different mids" + ); + assert!( + self.rid == other.rid, + "Cannot merge MediaIngressStats for different rids" + ); + let (jitter, rtt, loss) = if self.timestamp > other.timestamp { + (self.jitter, self.rtt, self.loss) + } else { + (other.jitter, other.rtt, other.loss) + }; + + *self = Self { + mid: self.mid, + rid: self.rid, + bytes: self.bytes + other.bytes, + packets: self.packets + other.packets, + firs: self.firs + other.firs, + plis: self.plis + other.plis, + nacks: self.nacks + other.nacks, + jitter, + rtt, + loss, + timestamp: self.timestamp.max(other.timestamp), + remote: match (&self.remote, &other.remote) { + (None, None) => None, + (Some(remote), None) => Some(remote.clone()), + (None, Some(other_remote)) => Some(other_remote.clone()), + (Some(remote), Some(other_remote)) => Some(RemoteEgressStats { + bytes: remote.bytes + other_remote.bytes, + packets: remote.packets + other_remote.packets, + }), + }, + }; + } +} + +/// Stats as reported by the remote side (via RTCP SenderReports). +#[derive(Debug, Clone)] +pub struct RemoteEgressStats { + /// Total bytes sent, including retransmissions. + pub bytes: u64, + /// Total number of rtp packets sent, including retransmissions. + pub packets: u64, +} + +impl Stats { + /// Create a new stats instance + /// + /// The internal state is market with the current `Instant::now()`. + /// This allows us to emit stats right away at the first upcoming timeout + pub fn new(interval: Duration) -> Stats { + Stats { + // by starting with the current time we can generate stats right on first timeout + last_now: None, + events: VecDeque::new(), + interval, + } + } + + /// Returns true if we want to handle the timeout + /// + /// The caller can use this to compute the snapshot only if needed, before calling \ + /// [`Stats::do_handle_timeout`] + pub fn wants_timeout(&mut self, now: Instant) -> bool { + let Some(last_now) = self.last_now else { + // Learn our first ever `now` + self.last_now = Some(now); + return false; + }; + + let min_step = last_now + self.interval; + now >= min_step + } + + /// Actually handles the timeout advancing the internal state and preparing the output + pub fn do_handle_timeout(&mut self, snapshot: &mut StatsSnapshot) { + // enqueue stats and timestamp them so they can be sent out + + let event = PeerStats { + peer_bytes_rx: snapshot.peer_rx, + peer_bytes_tx: snapshot.peer_tx, + bytes_rx: snapshot.rx, + bytes_tx: snapshot.tx, + timestamp: snapshot.timestamp, + bwe_tx: snapshot.bwe_tx, + egress_loss_fraction: snapshot.egress_loss_fraction, + ingress_loss_fraction: snapshot.ingress_loss_fraction, + rtt: snapshot.rtt, + selected_candidate_pair: snapshot.selected_candidate_pair.clone(), + }; + + self.events.push_back(StatsEvent::Peer(event)); + + for (_, event) in snapshot.ingress.drain() { + self.events.push_back(StatsEvent::MediaIngress(event)); + } + + for (_, event) in snapshot.egress.drain() { + self.events.push_back(StatsEvent::MediaEgress(event)); + } + + self.last_now = Some(snapshot.timestamp); + } + + /// Poll for the next time to call [`Stats::wants_timeout`] and [`Stats::do_handle_timeout`]. + /// + /// NOTE: we only need Option<_> to conform to .soonest() (see caller) + pub fn poll_timeout(&self) -> Option { + let last_now = self.last_now?; + Some(last_now + self.interval) + } + + /// Return any events ready for delivery + pub fn poll_output(&mut self) -> Option { + self.events.pop_front() + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/mod.rs new file mode 100644 index 000000000..a8d3b00c7 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/mod.rs @@ -0,0 +1,725 @@ +use std::collections::{HashMap, VecDeque}; +use std::fmt::{self}; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use crate::config_mod::RtcpReportIntervals; +use crate::format::CodecConfig; +use crate::format::PayloadParams; +use crate::media::{KeyframeRequest, Media, SenderFeedback}; +use crate::packet::Vp8Patch; +use crate::rtp_::MidRid; +use crate::rtp_::Ssrc; +use crate::rtp_::{Bitrate, Pt}; +use crate::rtp_::{MediaTime, SenderInfo}; +use crate::rtp_::{Mid, Rid, SeqNo}; +use crate::rtp_::{Rtcp, RtpHeader}; +use crate::util::already_happened; + +pub use self::receive::StreamRx; +pub use self::send::{RtpWrite, StreamTx, StreamTxQueueInfo}; + +mod receive; +pub(crate) mod register; +pub(crate) mod register_nack; +mod rtx_cache; +pub(crate) mod rtx_cache_buf; +mod send; +mod send_queue; +mod send_stats; + +pub(crate) use send::{DEFAULT_RTX_CACHE_DURATION, DEFAULT_RTX_RATIO_CAP}; + +pub(crate) struct StreamTimeoutConfig<'a> { + pub(crate) codecs: &'a CodecConfig, + pub(crate) intervals: RtcpReportIntervals, +} + +/// Packet of RTP data. +/// +/// As emitted by [`Event::RtpPacket`][crate::Event::RtpPacket] when using rtp mode. +#[derive(PartialEq, Eq)] +pub struct RtpPacket { + /// Extended sequence number to avoid having to deal with ROC. + pub seq_no: SeqNo, + + /// Extended RTP time in the clock frequency of the codec. To avoid dealing with ROC. + /// + /// For a newly scheduled outgoing packet, the clock_rate is not correctly set until + /// we do the poll_output(). + pub time: MediaTime, + + /// Parsed RTP header. + pub header: RtpHeader, + + /// RTP payload. This contains no header. + pub payload: Arc<[u8]>, + + vp8_patch: Option, + + /// str0m server timestamp. + /// + /// This timestamp has nothing to do with RTP itself. For outgoing packets, this is when + /// the packet was first handed over to str0m and enqueued in the outgoing send buffers. + /// For incoming packets it's the time we received the network packet. + pub timestamp: Instant, + + /// Sender information from the most recent Sender Report(SR). + /// + /// If no Sender Report(SR) has been received or this packet is being sent by str0m this is [`None`]. + pub last_sender_info: Option, + + /// Whether this packet can be nacked. + /// + /// This is often false for audio, but might also be false for discardable frames when + /// using temporal encoding as in a VP8 simulcast situation. + pub(crate) nackable: bool, +} + +/// Event when an encoded stream is considered paused/unpaused. +/// +/// This means the stream has not received any data for some time (default 1.5 seconds). +#[derive(Debug)] +pub struct StreamPaused { + /// The main SSRC of the encoded stream that paused. + pub ssrc: Ssrc, + + /// The mid the encoded stream belongs to. + pub mid: Mid, + + /// The rid, if the encoded stream has a rid. + pub rid: Option, + + /// Whether the stream is paused or not. + pub paused: bool, +} + +/// 255 is out of range for a real PT, which is 7 bit. +const BLANK_PACKET_DEFAULT_PT: Pt = Pt::new_with_value(255); + +impl RtpPacket { + fn blank() -> RtpPacket { + RtpPacket { + seq_no: 0.into(), + time: MediaTime::from_90khz(0), + header: RtpHeader { + payload_type: BLANK_PACKET_DEFAULT_PT, + ..Default::default() + }, + payload: Arc::default(), // This payload is never used. See RtpHeader::create_padding_packet + vp8_patch: None, + nackable: false, + last_sender_info: None, + timestamp: already_happened(), + } + } +} + +/// Holder of incoming/outgoing encoded streams. +/// +/// Each encoded stream is uniquely identified by an SSRC. The concept of mid/rid sits on the Media +/// level together with the ability to translate a mid/rid to an encoded stream. +#[derive(Debug)] +pub(crate) struct Streams { + /// All incoming encoded streams. + streams_rx: HashMap, + + /// Each incoming SSRC is mapped to a Mid/Ssrc. The Ssrc in the value is for the case + /// where the incoming SSRC is for an RTX and we want the "main". + rx_lookup: HashMap, + + /// Time we last cleaned up unused entries from source_keys_rx. + last_rx_lookup_cleanup: Instant, + + /// All outgoing encoded streams. + streams_tx: HashMap, + + /// Local SSRC used before we got any StreamTx. This is used for RTCP if we don't + /// have any reasonable value to use. + default_ssrc_tx: Ssrc, + + /// We need to report all RR/SR for a Mid together in one RTCP. This is a dynamic + /// list that we don't want to allocate on every handle_timeout. + mids_to_report: Vec, + + /// Whether nack reports are enabled. This is an optimization to avoid too frequent + /// Session::nack_at() when we don't need to send nacks. + any_nack_active: Option, + + /// Whether periodic statistics reports are expected to be generated. This informs us on + /// whether we should be holding onto data needed for those reports or not. + enable_stats: bool, + + /// Threshold above which an outgoing RTP packet triggers an MTU warning. Used as a + /// hard cap when selecting RTX cache entries for spurious padding. + mtu_warn: usize, +} + +/// Delay between cleaning up the RxLookup. +const RX_LOOKUP_CLEANUP_INTERVAL: Duration = Duration::from_millis(10_000); + +/// How old an RxLookup entry may be. +const RX_LOOKUP_EXPIRY: Duration = Duration::from_millis(30_000); + +#[derive(Debug)] +struct RxLookup { + mid: Mid, + main: Ssrc, + last_used: Instant, +} + +impl Streams { + pub(crate) fn new(enable_stats: bool, mtu_warn: usize) -> Self { + Self { + streams_rx: Default::default(), + rx_lookup: Default::default(), + last_rx_lookup_cleanup: already_happened(), + streams_tx: Default::default(), + default_ssrc_tx: 0.into(), // this will be changed + mids_to_report: Vec::with_capacity(10), + any_nack_active: None, + enable_stats, + mtu_warn, + } + } + + pub(crate) fn map_dynamic_by_rid( + &mut self, + ssrc: Ssrc, + midrid: MidRid, + media: &mut Media, + payload: PayloadParams, + is_main: bool, + ) { + // This is the point of the function. + let rid = midrid + .rid() + .expect("map_dynamic_by_rid to be called with Rid"); + + // Check if the mid/rid combo is not expected + if !media.rids_rx().contains(rid) { + trace!("Mid does not expect rid: {} {}", midrid.mid(), rid); + return; + } + + let maybe_stream = self.stream_rx_by_midrid(midrid, false); + + let (ssrc_main, rtx) = if is_main { + let maybe_rtx = maybe_stream.and_then(|s| s.rtx()); + (ssrc, maybe_rtx) + } else { + // This can bail if the main SSRC has not been discovered yet. + let Some(stream) = maybe_stream else { + return; + }; + (stream.ssrc(), Some(ssrc)) + }; + + self.map_dynamic_finish(midrid, ssrc_main, rtx, media, payload); + } + + pub(crate) fn map_dynamic_by_pt( + &mut self, + ssrc: Ssrc, + midrid: MidRid, + media: &mut Media, + payload: PayloadParams, + is_main: bool, + ) { + if media.rids_rx().is_specific() { + trace!( + "Media expects rid and RTP packet has only mid: {:?}", + media.rids_rx() + ); + return; + } + + let maybe_stream = self.stream_rx_by_midrid(midrid, false); + + let (ssrc_main, rtx) = if is_main { + let maybe_rtx = maybe_stream.and_then(|s| s.rtx()); + (ssrc, maybe_rtx) + } else { + // This can bail if the main SSRC has not been discovered yet. + let Some(stream) = maybe_stream else { + return; + }; + // The main is the SSRC in the stream already. The incoming is RTX. + let ssrc_main = stream.ssrc(); + (ssrc_main, Some(ssrc)) + }; + + self.map_dynamic_finish(midrid, ssrc_main, rtx, media, payload); + } + + #[allow(clippy::too_many_arguments)] + fn map_dynamic_finish( + &mut self, + midrid: MidRid, + ssrc_main: Ssrc, + rtx: Option, + media: &mut Media, + payload: PayloadParams, + ) { + let maybe_stream = self.stream_rx_by_midrid(midrid, false); + + if let Some(stream) = maybe_stream { + let ssrc_from = stream.ssrc(); + let rtx_from = stream.rtx(); + + // Handle changes in SSRC. + if ssrc_from != ssrc_main { + // We got a change in main SSRC for this stream. + let did_change = self.change_stream_rx_ssrc(ssrc_from, ssrc_main); + + // When the SSRCs changes the sequence number typically also does, the + // depayloader (if in use) relies on sequence numbers and will not handle a + // large jump correctly, reset it. + if did_change { + media.reset_depayloader(payload.pt(), midrid.rid()); + } + } + + // Handle changes in RTX + if let (Some(rtx_from), Some(rtx_to)) = (rtx_from, rtx) { + if rtx_from != rtx_to { + self.change_stream_rx_rtx(rtx_from, rtx_to); + } + } + } + + // If we don't have an RTX PT configured, we don't want NACK. + let suppress_nack = payload.resend.is_none(); + + // If stream already exists, this might only "fill in" the RTX. + self.expect_stream_rx(ssrc_main, rtx, midrid, suppress_nack); + } + + pub fn expect_stream_rx( + &mut self, + ssrc: Ssrc, + rtx: Option, + midrid: MidRid, + suppress_nack: bool, + ) -> &mut StreamRx { + // New stream might have enabled nacks. + self.any_nack_active = None; + + let stream = self + .streams_rx + .entry(ssrc) + .or_insert_with(|| StreamRx::new(ssrc, midrid, suppress_nack)); + + if let Some(rtx) = rtx { + stream.maybe_reset_rtx(rtx); + } + + stream + } + + pub fn remove_stream_rx(&mut self, ssrc: Ssrc) -> bool { + let stream = self.streams_rx.remove(&ssrc); + let existed = stream.is_some(); + + self.rx_lookup.retain(|k, l| *k != ssrc && l.main != ssrc); + + existed + } + + pub fn declare_stream_tx( + &mut self, + ssrc: Ssrc, + rtx: Option, + midrid: MidRid, + ) -> &mut StreamTx { + self.streams_tx + .entry(ssrc) + .or_insert_with(|| StreamTx::new(ssrc, rtx, midrid, self.enable_stats, self.mtu_warn)) + } + + pub fn remove_stream_tx(&mut self, ssrc: Ssrc) -> bool { + self.streams_tx.remove(&ssrc).is_some() + } + + pub(crate) fn local_sender_ssrcs(&self) -> Vec { + self.streams_tx + .values() + .flat_map(|s| std::iter::once(s.ssrc()).chain(s.rtx())) + .collect() + } + + pub(crate) fn reset_send_buffers(&mut self) { + for stream in self.streams_tx.values_mut() { + stream.reset_buffers(); + } + } + + pub fn stream_rx(&mut self, ssrc: &Ssrc) -> Option<&mut StreamRx> { + self.streams_rx.get_mut(ssrc) + } + + pub fn stream_tx(&mut self, ssrc: &Ssrc) -> Option<&mut StreamTx> { + self.streams_tx.get_mut(ssrc) + } + + /// Lookup the "main" SSRC and mid for a given SSRC(main or RTX). + pub(crate) fn mid_ssrc_rx_by_ssrc_or_rtx( + &mut self, + now: Instant, + ssrc: Ssrc, + ) -> Option<(Mid, Ssrc)> { + // A direct hit on SSRC is to prefer. The idea is that mid/rid are only sent + // for the initial x seconds and then we start using SSRC only instead. + if let Some(r) = self.rx_lookup.get_mut(&ssrc) { + r.last_used = now; + return Some((r.mid, r.main)); + } + + let maybe_stream = self.stream_rx_by_ssrc_or_rtx(ssrc); + if let Some(stream) = maybe_stream { + let mid = stream.mid(); + let ssrc_main = stream.ssrc(); + + self.rx_lookup.insert( + ssrc, + RxLookup { + mid, + main: ssrc_main, + last_used: now, + }, + ); + + return Some((mid, ssrc_main)); + } + + None + } + + pub(crate) fn regular_feedback_at(&self, i: RtcpReportIntervals) -> Option { + let r = self.streams_rx.values().map(|s| s.receiver_report_at(i)); + let s = self.streams_tx.values().map(|s| s.sender_report_at(i)); + r.chain(s).min() + } + + pub(crate) fn paused_at(&self) -> Option { + self.streams_rx.values().find_map(|s| s.paused_at()) + } + + pub(crate) fn send_stream(&self) -> Option { + if self.streams_tx.values().any(|s| s.need_timeout()) { + Some(already_happened()) + } else { + None + } + } + + pub(crate) fn is_receiving(&self) -> bool { + !self.streams_rx.is_empty() + } + + pub(crate) fn handle_timeout( + &mut self, + now: Instant, + sender_ssrc: Ssrc, + do_nack: bool, + medias: &[Media], + config: StreamTimeoutConfig<'_>, + feedback: &mut VecDeque, + ) { + let StreamTimeoutConfig { codecs, intervals } = config; + + self.mids_to_report.clear(); // Clear for checking StreamRx. + for stream in self.streams_rx.values() { + if stream.need_rr(now, intervals) { + self.mids_to_report.push(stream.mid()); + } + } + + for stream in self.streams_rx.values_mut() { + stream.maybe_create_keyframe_request(sender_ssrc, feedback); + stream.maybe_create_remb_request(sender_ssrc, feedback); + + // All StreamRx belonging to the same Mid are reported together. + if self.mids_to_report.contains(&stream.mid()) { + stream.create_rr_and_update(now, sender_ssrc, feedback); + } + + if do_nack { + stream.maybe_create_nack(sender_ssrc, feedback); + } + + stream.handle_timeout(now); + } + + self.mids_to_report.clear(); // start over for StreamTx. + for stream in self.streams_tx.values() { + if stream.need_sr(now, intervals) { + self.mids_to_report.push(stream.mid()); + } + } + + for stream in self.streams_tx.values_mut() { + let mid = stream.mid(); + + // All StreamTx belonging to the same Mid are reported together. + if self.mids_to_report.contains(&mid) { + stream.create_sr_and_update(now, feedback); + } + + // Finding the first (main) PT that also has RTX for the Media is expensive, + // this closure is run only when needed. + // The unwrap is okay because we cannot have StreamTx with a Mid without the corresponding Media. + let get_media = move || (medias.iter().find(|m| m.mid() == mid).unwrap(), codecs); + + stream.handle_timeout(now, get_media); + } + + if now > self.rx_lookup_at() { + self.rx_lookup + .retain(|_, l| now - l.last_used <= RX_LOOKUP_EXPIRY); + self.last_rx_lookup_cleanup = now; + } + } + + pub(crate) fn poll_keyframe_request(&mut self) -> Option { + self.streams_tx.values_mut().find_map(|s| { + let kind = s.poll_keyframe_request()?; + Some(KeyframeRequest { + mid: s.mid(), + rid: s.rid(), + kind, + }) + }) + } + + pub(crate) fn poll_sender_feedback(&mut self) -> Option { + self.streams_rx.values_mut().find_map(|s| { + let (sender_info, at) = s.poll_sender_info()?; + + Some(SenderFeedback { + mid: s.mid(), + rid: s.rid(), + received_at: at, + sender_info, + }) + }) + } + + pub(crate) fn poll_remb_request(&mut self) -> Option<(Mid, Bitrate)> { + self.streams_tx + .values_mut() + .find_map(|s| s.poll_remb_request().map(|b| (s.mid(), b))) + } + + pub(crate) fn poll_stream_paused(&mut self) -> Option { + self.streams_rx.values_mut().find_map(|s| s.poll_paused()) + } + + pub(crate) fn has_stream_rx(&self, ssrc: Ssrc) -> bool { + self.streams_rx.contains_key(&ssrc) + } + + pub(crate) fn has_stream_tx(&self, ssrc: Ssrc) -> bool { + self.streams_tx.contains_key(&ssrc) + } + + pub(crate) fn streams_rx(&mut self) -> impl Iterator { + self.streams_rx.values_mut() + } + + pub(crate) fn streams_tx(&mut self) -> impl Iterator { + self.streams_tx.values_mut() + } + + pub(crate) fn ssrcs_tx(&self, mid: Mid) -> Vec<(Ssrc, Option)> { + self.streams_tx + .values() + .filter(|s| s.mid() == mid) + .map(|s| (s.ssrc(), s.rtx())) + .collect() + } + + pub(crate) fn new_ssrc(&self) -> Ssrc { + loop { + // This new guarantees we never get SSRC 0 (reserved for BWE probes) + let ssrc = Ssrc::new(); + + let has_ssrc = self.has_stream_rx(ssrc) || self.has_stream_tx(ssrc); + + if has_ssrc { + continue; + } + + // Need to check RTX as well. + let has_rtx_rx = self.streams_rx.values().any(|s| s.rtx() == Some(ssrc)); + if has_rtx_rx { + continue; + } + + let has_rtx_tx = self.streams_tx.values().any(|s| s.rtx() == Some(ssrc)); + if has_rtx_tx { + continue; + } + + // Not used + break ssrc; + } + } + + pub fn new_ssrc_pair(&self) -> (Ssrc, Ssrc) { + let ssrc = self.new_ssrc(); + + let rtx = loop { + let proposed = self.new_ssrc(); + // Avoid clashing with just allocated main SSRC. + if proposed != ssrc { + break proposed; + } + }; + + (ssrc, rtx) + } + + pub(crate) fn first_ssrc_remote(&self) -> Ssrc { + *self.streams_rx.keys().next().unwrap_or(&0.into()) + } + + pub(crate) fn first_ssrc_local(&mut self) -> Ssrc { + if let Some(ssrc) = self.streams_tx.keys().next() { + // If there is some local Tx SSRC, use that. + *ssrc + } else { + // Fallback in case we don't have any Tx SSRC. + if *self.default_ssrc_tx == 0 { + // Do not use 0, allocate one that is not in session already. + self.default_ssrc_tx = self.new_ssrc(); + } + self.default_ssrc_tx + } + } + + pub(crate) fn stream_tx_by_midrid(&mut self, midrid: MidRid) -> Option<&mut StreamTx> { + self.streams_tx.values_mut().find(|s| s.is_midrid(midrid)) + } + + pub(crate) fn stream_rx_by_midrid( + &mut self, + midrid: MidRid, + reset_cached_nack_flag: bool, + ) -> Option<&mut StreamRx> { + if reset_cached_nack_flag { + // Invalidate nack_active since it's possible to manipulate the + // nack setting on the returned StreamRx. + self.any_nack_active = None; + } + + self.streams_rx.values_mut().find(|s| s.is_midrid(midrid)) + } + + pub(crate) fn remove_streams_by_mid(&mut self, mid: Mid) { + self.streams_tx.retain(|_, s| s.mid() != mid); + self.streams_rx.retain(|_, s| s.mid() != mid); + self.rx_lookup.retain(|_, v| v.mid != mid); + } + + /// An iterator over all the tx streams for a given mid. + pub(crate) fn streams_tx_by_mid(&mut self, mid: Mid) -> impl Iterator { + self.streams_tx.values_mut().filter(move |s| s.mid() == mid) + } + + /// An iterator over all the rx streams for a given mid. + pub(crate) fn streams_rx_by_mid(&mut self, mid: Mid) -> impl Iterator { + self.streams_rx.values_mut().filter(move |s| s.mid() == mid) + } + + pub(crate) fn reset_buffers_tx(&mut self, mid: Mid) { + for s in self.streams_tx_by_mid(mid) { + s.reset_buffers(); + } + } + + pub(crate) fn reset_buffers_rx( + &mut self, + mid: Mid, + max_seq_lookup: impl Fn(Ssrc) -> Option, + ) { + for s in self.streams_rx_by_mid(mid) { + s.reset_buffers(&max_seq_lookup); + } + } + + pub(crate) fn change_stream_rx_ssrc(&mut self, ssrc_from: Ssrc, ssrc_to: Ssrc) -> bool { + // This unwrap is OK, because we can't call change_stream_rx_ssrc without first + // knowing there is such a StreamRx. + let maybe_change = self.streams_rx.get_mut(&ssrc_from).unwrap(); + + // The StreamRx is allowed to not change the SSRC in case it is switching back + // to the previous SSRC. This is to avoid flapping in case RTP packets arrive + // out of order. + let did_change = maybe_change.change_ssrc(ssrc_to); + + if did_change { + // Unwrap is OK, see above. + let to_change = self.streams_rx.remove(&ssrc_from).unwrap(); + + // Reinsert under new SSRC key. + self.streams_rx.insert(ssrc_to, to_change); + + // Remove previous mappings for the SSRC + self.rx_lookup + .retain(|k, l| *k != ssrc_from && l.main != ssrc_from); + } + + did_change + } + + fn change_stream_rx_rtx(&mut self, rtx_from: Ssrc, rtx_to: Ssrc) { + // Invalidate since we might need to enable nacks now. + self.any_nack_active = None; + + // Remove the SSRC mapping + self.rx_lookup.remove(&rtx_from); + + let Some(to_change) = self + .streams_rx + .values_mut() + .find(|s| s.rtx() == Some(rtx_from)) + else { + // If there's no main stream associated with the RTX our job is done. + return; + }; + + to_change.maybe_reset_rtx(rtx_to); + } + + fn stream_rx_by_ssrc_or_rtx(&self, ssrc: Ssrc) -> Option<&StreamRx> { + self.streams_rx + .values() + .find(|s| s.ssrc() == ssrc || s.rtx() == Some(ssrc)) + } + + pub(crate) fn any_nack_enabled(&mut self) -> bool { + if self.any_nack_active.is_none() { + self.any_nack_active = Some(self.streams_rx.values().any(|s| s.nack_enabled())); + } + self.any_nack_active.unwrap() + } + + fn rx_lookup_at(&self) -> Instant { + self.last_rx_lookup_cleanup + RX_LOOKUP_CLEANUP_INTERVAL + } +} + +impl fmt::Debug for RtpPacket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtpPacket") + .field("seq_no", &self.seq_no) + .field("time", &self.time) + .field("header", &self.header) + .field("payload", &self.payload.len()) + .field("nackable", &self.nackable) + .field("timestamp", &self.timestamp) + .finish() + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/receive.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/receive.rs new file mode 100644 index 000000000..3ab3c5f42 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/receive.rs @@ -0,0 +1,911 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::config_mod::RtcpReportIntervals; +use crate::media::KeyframeRequestKind; +use crate::rtp_::MidRid; +use crate::rtp_::{Bitrate, DlrrItem, ExtendedReport, extend_u32}; +use crate::rtp_::{Fir, FirEntry, Frequency, MediaTime, Remb}; +use crate::rtp_::{Mid, Pli, Pt, ReceiverReport}; +use crate::rtp_::{ReportBlock, ReportList, Rid, Rrtr, Rtcp}; +use crate::rtp_::{RtcpFb, RtpHeader, SenderInfo, SeqNo}; +use crate::rtp_::{SdesType, Ssrc}; +use crate::stats::{MediaIngressStats, RemoteEgressStats, StatsSnapshot}; +use crate::util::{InstantExt, SystemTimeExt}; +use crate::util::{already_happened, calculate_rtt}; + +use super::RtpPacket; +use super::StreamPaused; +use super::register::ReceiverRegister; + +/// Incoming encoded stream. +/// +/// A stream is a primary SSRC + optional RTX SSRC. +/// +/// This is RTP level API. For frame level API see [`Rtc::writer`][crate::Rtc::writer]. +#[derive(Debug)] +pub struct StreamRx { + /// Unique idenfier of the remote stream. + /// + /// If the remote changes the SSRC, we will create a new stream, not change this id. + ssrc: Ssrc, + + /// Identifier of a resend (RTX) stream. This can be set later, once we discover it. + rtx: Option, + + /// Previous main SSRC. This is to ensure we never go "backwards" in terms + /// of changing SSRC (for FF). + previous_ssrc: Option, + + /// The Media mid/rid this stream belongs to. + midrid: MidRid, + + /// Incoming CNAME in Sdes reports. + cname: Option, + + /// Whether we explicitly want to supress NACK sending. This is normally done by not + /// setting an RTX, however this can be toggled off manually despite RTX being there. + /// + /// This is also set to true if the SDP negotiation disables RTX. + /// + /// Defaults to false. + suppress_nack: bool, + + /// Timestamp when we got some indication of remote using this stream. + last_used: Instant, + + /// Last seen pt and clock_rate in + last_clock_rate: Option<(Pt, Frequency)>, + + /// Last received sender info. + sender_info: Option, + + /// ROC to reset with on next incoming packet. + reset_roc: Option, + + /// Register of received packets. For NACK handling. + /// + /// Set on first ever packet. + register: Option, + + /// Register of received packets for RTX. + /// + /// Set on first ever RTXpacket. + register_rtx: Option, + + /// Last observed media time in an RTP packet. + last_time: Option, + + /// If we have a pending keyframe request to send. + pending_request_keyframe: Option, + + /// If we have a pending REMB request to send. + pending_request_remb: Option, + + /// Sequence number of the next FIR. + fir_seq_no: u8, + + /// Last time we produced regular feedback RR. + last_receiver_report: Instant, + + /// Statistics of incoming data. + stats: StreamRxStats, + + /// When we need to evaluate the paused state. + /// + /// now + pause_threshold + check_paused_at: Option, + + /// Whether we consider this StreamRx paused. + /// + /// A stream is considered paused if it has received no packets for some (configurable) duration. + /// This defaults to 1.5s. + paused: bool, + + /// Whether we need to emit a paused event for the current paused state. + need_paused_event: bool, + + /// The configured threshold before considering the lack of packets as going into paused. + pause_threshold: Duration, +} + +/// The last sender info we recieved. +#[derive(Debug)] +pub(crate) struct LastSenderInfo { + /// When this SenderInfo was received. + received_at: Instant, + /// The sender info itself. + info: SenderInfo, + /// Whether we have emitted it yet via `poll_sender_info` + emitted: bool, +} + +/// Holder of stats. +#[derive(Debug, Default)] +pub(crate) struct StreamRxStats { + /// count of bytes received, including retransmissions + bytes: u64, + /// count of packets received, including retransmissions + packets: u64, + /// count of FIR requests sent + firs: u64, + /// count of PLI requests sent + plis: u64, + /// count of NACKs sent + nacks: u64, + /// interarrival jitter (RTP timestamp units) from the last RR we sent + jitter: u32, + /// round trip time from the last DLRR, if any + rtt: Option, + /// fraction of packets lost from the last RR, if any + loss: Option, +} + +impl StreamRx { + pub(crate) fn new(ssrc: Ssrc, midrid: MidRid, suppress_nack: bool) -> Self { + debug!("Create StreamRx for SSRC: {}", ssrc); + + StreamRx { + ssrc, + rtx: None, + previous_ssrc: None, + midrid, + cname: None, + suppress_nack, + last_used: already_happened(), + last_clock_rate: None, + sender_info: None, + reset_roc: None, + register: None, + register_rtx: None, + last_time: None, + pending_request_keyframe: None, + pending_request_remb: None, + fir_seq_no: 0, + last_receiver_report: already_happened(), + stats: StreamRxStats::default(), + check_paused_at: None, + paused: true, + need_paused_event: false, + pause_threshold: Duration::from_millis(1500), + } + } + + /// The (primary) SSRC of this encoded stream. + pub fn ssrc(&self) -> Ssrc { + self.ssrc + } + + /// The resend (RTX) SSRC of this encoded stream. + pub fn rtx(&self) -> Option { + self.rtx + } + + /// Mid for this stream. + /// + /// In SDP this corresponds to m-line and "Media". + pub fn mid(&self) -> Mid { + self.midrid.mid() + } + + /// Rid for this stream. + /// + /// This is used to separate streams with the same [`Mid`] when using simulcast. + pub fn rid(&self) -> Option { + self.midrid.rid() + } + + /// CNAME as sent by remote peer in a Sdes. + /// + /// The value is None until we receive a first report with the value set. + pub fn cname(&self) -> Option<&str> { + self.cname.as_deref() + } + + /// Set threshold duration for emitting the paused event. + /// + /// This event is emitted when no packet have received for this duration. + pub fn set_pause_threshold(&mut self, t: Duration) { + self.pause_threshold = t; + } + + /// The last time we received a packet. + /// + /// Resets if the SSRC changes. + pub fn last_time(&self) -> Option { + self.last_time + } + + /// Request a keyframe for an incoming encoded stream. + /// + /// * SSRC the identifier of the remote encoded stream to request a keyframe for. + /// * kind PLI or FIR. + pub fn request_keyframe(&mut self, kind: KeyframeRequestKind) { + self.pending_request_keyframe = Some(kind); + } + + /// Request max recv bitrate for an incoming encoded stream. + /// + /// * bitrate Bitrate. + pub fn request_remb(&mut self, bitrate: Bitrate) { + self.pending_request_remb = Some(bitrate); + } + + /// Suppress NACK sending. + /// + /// Normally NACK is disabled by not having an RTX SSRC set. In some situations it might be + /// desirable to manually suppress NACK sending regardless of RTX setting. + pub fn suppress_nack(&mut self, suppress: bool) { + self.suppress_nack = suppress; + } + + fn is_audio(&self) -> bool { + self.rtx.is_none() // this is maybe not correct, but it's all we got. + } + + pub(crate) fn receiver_report_at(&self, intervals: RtcpReportIntervals) -> Instant { + self.last_receiver_report + intervals.for_audio(self.is_audio()) + } + + pub(crate) fn handle_rtcp(&mut self, now: Instant, fb: RtcpFb) { + use RtcpFb::*; + match fb { + SenderInfo(v) => { + self.set_sender_info(now, v); + } + SourceDescription(v) => { + for (sdes, st) in v.values { + if sdes == SdesType::CNAME { + if st.is_empty() { + // In simulcast, chrome doesn't send the SSRC lines, but + // expects us to infer that from rtp headers. It does + // however send the SourceDescription RTCP with an empty + // string CNAME. ¯\_(ツ)_/¯ + return; + } + + // Here we _could_ check CNAME here matches something. But + // CNAMEs are a bit unfashionable. + self.cname = Some(st); + return; + } + } + } + DlrrItem(v) => { + self.set_dlrr_item(now, v); + } + Goodbye(_v) => { + // We get Goodbye at weird times, like SDP renegotiation, which makes + // pausing on the BYE not a good idea. Chrome also reuses the SSRC it + // just sent BYE on. Very not helpful. + } + _ => {} + } + } + + fn set_sender_info(&mut self, now: Instant, mut info: SenderInfo) { + // Extend the incoming time given our knowledge of last time. + let extended = { + let prev = self + .sender_info + .as_ref() + .map(|last| last.info.rtp_time.numer()); + let r_u32 = info.rtp_time.numer() as u32; + extend_u32(prev, r_u32) + }; + + // The MediaTime has a base 1 after being parsed. At this point + // we know whether it's audio or video and set the base accordingly. + let clock_rate = self + .last_clock_rate + .map(|(_, r)| r) + .unwrap_or(Frequency::SECONDS); + + // Clock rate is that of the last received packet. + info.rtp_time = MediaTime::new(extended, clock_rate); + + self.sender_info = Some(LastSenderInfo { + received_at: now, + info, + emitted: false, + }); + } + + fn set_dlrr_item(&mut self, now: Instant, dlrr: DlrrItem) { + let ntp_time = now.to_ntp_duration(); + let rtt = calculate_rtt(ntp_time, dlrr.last_rr_delay, dlrr.last_rr_time); + self.stats.rtt = rtt; + } + + pub(crate) fn paused_at(&self) -> Option { + self.check_paused_at + } + + pub(crate) fn handle_timeout(&mut self, now: Instant) { + // No scheduled paused check? + if self.check_paused_at.is_none() { + return; + } + + // Not reached scheduled paused check? + if Some(now) < self.check_paused_at { + return; + } + + // Every update() schedules a paused check in the future. If we have reached that + // future we have implicitly also paused. + self.check_paused_at = None; + + self.paused = true; + self.need_paused_event = true; + } + + pub(crate) fn extend_seq( + &mut self, + header: &RtpHeader, + is_repair: bool, + max_seq_lookup: impl Fn(Ssrc) -> Option, + ) -> SeqNo { + // Select reference to register to use depending on RTX or not. The RTX has a separate + // sequence number series to the main register. + let register_ref = if is_repair { + &mut self.register_rtx + } else { + &mut self.register + }; + + let register = + register_ref.get_or_insert_with(|| ReceiverRegister::new(max_seq_lookup(header.ssrc))); + + // If the user has called `reset_seq_no`, this is the time to handle it, but only + // if the incoming packet is for main (not repair). + let mut reset_seq_no = None; + if !is_repair { + if let Some(reset_roc) = self.reset_roc.take() { + let s: SeqNo = (reset_roc << 16 | header.sequence_number as u64).into(); + reset_seq_no = Some(s); + } + } + + if let Some(reset_seq_no) = reset_seq_no { + reset_seq_no + } else { + header.sequence_number(register.max_seq()) + } + } + + pub(crate) fn is_new_packet(&self, is_repair: bool, seq_no: SeqNo) -> bool { + let register_ref = if is_repair { + self.register_rtx.as_ref() + } else { + self.register.as_ref() + }; + + // Unwrap is OK because we always call extend_seq() for the same is_repair flag beforehand + register_ref.unwrap().accepts(seq_no) + } + + pub(crate) fn update_register( + &mut self, + now: Instant, + header: &RtpHeader, + clock_rate: Frequency, + is_repair: bool, + seq_no: SeqNo, + ) -> RegisterUpdateReceipt { + self.last_used = now; + + let was_paused = self.paused; + if was_paused { + self.paused = false; + self.need_paused_event = true; + } + self.check_paused_at = Some(now + self.pause_threshold); + + let register_ref = if is_repair { + &mut self.register_rtx + } else { + &mut self.register + }; + + // Unwrap is OK because we always call extend_seq() for the same is_repair flag beforehand + let register = register_ref.as_mut().unwrap(); + + let is_new_packet = register.update(seq_no, now, header.timestamp, clock_rate.get()); + + // Get the previous time for comparison + let previous_time = self.last_time.map(|t| t.numer()); + + // Calculate the extended timestamp + let mut time_u32 = extend_u32(previous_time, header.timestamp); + + if was_paused && Some(time_u32) < previous_time { + // In 32-bit RTP timestamps, adding 2^31 (MAX/2) flips to the other half of timestamp space + // This forces extend_u32 to produce a value in the next cycle + const HALF_CYCLE: u32 = 1u32 << 31; + let adjusted_ts = header.timestamp.wrapping_add(HALF_CYCLE); + + // Recalculate extended timestamp with adjusted value + let adjusted_time_u32 = extend_u32(previous_time, adjusted_ts); + + // If this adjusted timestamp moves time forward, use it + if adjusted_time_u32 > previous_time.unwrap() { + time_u32 = adjusted_time_u32; + } else { + // Fallback + time_u32 = header.timestamp as u64; + } + } + + let time = MediaTime::new(time_u32, clock_rate); + + if !is_repair { + self.last_time = Some(time); + } + + RegisterUpdateReceipt { + time, + is_new_packet, + } + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn handle_rtp( + &mut self, + now: Instant, + header: RtpHeader, + payload: Arc<[u8]>, + seq_no: SeqNo, + time: MediaTime, + ) -> RtpPacket { + trace!("Handle RTP: {:?}", header); + + let need_clock_rate = self.last_clock_rate.map(|(pt, _)| pt) != Some(header.payload_type); + if need_clock_rate { + self.last_clock_rate = Some((header.payload_type, time.frequency())); + + // If we get an SR before the first packet, we update the potential clock rate. + if let Some(i) = &mut self.sender_info { + i.info.rtp_time = MediaTime::new(i.info.rtp_time.numer(), time.frequency()); + } + } + + let packet = RtpPacket { + seq_no, + time, + header, + payload, + vp8_patch: None, + nackable: false, + last_sender_info: self.sender_info.as_ref().map(|l| l.info), + timestamp: now, + }; + + self.stats.bytes += packet.payload.len() as u64; + self.stats.packets += 1; + + packet + } + + /// Rewrites the header to be the un-RTX'd original packet, and returns + /// `data` with the original sequence number prefix stripped. + pub(crate) fn un_rtx<'a>(&self, header: &mut RtpHeader, data: &'a [u8], pt: Pt) -> &'a [u8] { + let mut orig_seq_no_16 = 0; + + let n = RtpHeader::read_original_sequence_number(data, &mut orig_seq_no_16); + + trace!( + "Repaired seq no {} -> {}", + header.sequence_number, orig_seq_no_16 + ); + + header.sequence_number = orig_seq_no_16; + + header.ssrc = self.ssrc; + header.payload_type = pt; + header.ext_vals.rid = header.ext_vals.rid_repair.take(); + + &data[n..] + } + + pub(crate) fn maybe_create_keyframe_request( + &mut self, + sender_ssrc: Ssrc, + feedback: &mut VecDeque, + ) { + let Some(kind) = self.pending_request_keyframe.take() else { + return; + }; + + let ssrc = self.ssrc; + + match kind { + KeyframeRequestKind::Pli => { + self.stats.plis += 1; + feedback.push_back(Rtcp::Pli(Pli { sender_ssrc, ssrc })) + } + KeyframeRequestKind::Fir => { + self.stats.firs += 1; + feedback.push_back(Rtcp::Fir(Fir { + sender_ssrc, + reports: FirEntry { + ssrc, + seq_no: self.next_fir_seq_no(), + } + .into(), + })) + } + } + } + + pub(crate) fn maybe_create_remb_request( + &mut self, + sender_ssrc: Ssrc, + feedback: &mut VecDeque, + ) { + let Some(bitrate) = self.pending_request_remb.take() else { + return; + }; + + feedback.push_back(Rtcp::Remb(Remb { + sender_ssrc, + ssrc: 0.into(), + bitrate: bitrate.as_f64() as f32, + ssrcs: vec![*self.ssrc], + })) + } + + fn next_fir_seq_no(&mut self) -> u8 { + let x = self.fir_seq_no; + self.fir_seq_no = self.fir_seq_no.wrapping_add(1); + x + } + + pub(crate) fn need_rr(&self, now: Instant, intervals: RtcpReportIntervals) -> bool { + if self.ssrc.is_probe() { + return false; + } + + now >= self.receiver_report_at(intervals) + } + + pub(crate) fn create_rr_and_update( + &mut self, + now: Instant, + sender_ssrc: Ssrc, + feedback: &mut VecDeque, + ) { + let mut rr = self.create_receiver_report(now); + rr.sender_ssrc = sender_ssrc; + + if !rr.reports.is_empty() { + let report = &rr.reports[rr.reports.len() - 1]; + self.stats.update_loss(report.fraction_lost); + self.stats.jitter = report.jitter; + } + + let xr = self.create_extended_receiver_report(now, sender_ssrc); + + trace!( + "Created feedback RR/XR ({:?}): {:?} {:?}", + self.midrid, rr, xr + ); + feedback.push_back(Rtcp::ReceiverReport(rr)); + feedback.push_back(Rtcp::ExtendedReport(xr)); + + self.last_receiver_report = now; + } + + fn create_receiver_report(&mut self, now: Instant) -> ReceiverReport { + let Some(mut report) = self.register.as_mut().and_then(|r| r.reception_report()) else { + return ReceiverReport { + sender_ssrc: 0.into(), // set one level up + reports: ReportList::new(), + }; + }; + report.ssrc = self.ssrc; + + // The middle 32 bits out of 64 in the NTP timestamp (as explained in + // Section 4) received as part of the most recent RTCP sender report + // (SR) packet from source SSRC_n. If no SR has been received yet, + // the field is set to zero. + report.last_sr_time = { + let t64 = self + .sender_info + .as_ref() + .map_or(0u64, |l| l.info.ntp_time.as_ntp_64()); + + (t64 >> 16) as u32 + }; + + // The delay, expressed in units of 1/65_536 seconds, between + // receiving the last SR packet from source SSRC_n and sending this + // reception report block. If no SR packet has been received yet + // from SSRC_n, the DLSR field is set to zero. + report.last_sr_delay = if let Some(l) = self.sender_info.as_ref() { + let t = l.received_at; + let delay = now - t; + ((delay.as_micros() * 65_536) / 1_000_000) as u32 + } else { + 0 + }; + + ReceiverReport { + sender_ssrc: 0.into(), // set one level up + reports: report.into(), + } + } + + fn create_extended_receiver_report(&self, now: Instant, sender_ssrc: Ssrc) -> ExtendedReport { + // we only want to report our time to measure RTT, + // the source will answer with Dlrr feedback, allowing us to calculate RTT + let block = ReportBlock::Rrtr(Rrtr { + ntp_time: now.to_system_time(), + }); + ExtendedReport { + ssrc: sender_ssrc, + blocks: vec![block], + } + } + + pub(crate) fn nack_enabled(&self) -> bool { + // Deliberately don't look at RTX is_some() here, since when using dynamic SSRC, we might need + // to send NACK before discovering the remote RTX. + !self.suppress_nack + } + + pub(crate) fn maybe_create_nack( + &mut self, + sender_ssrc: Ssrc, + feedback: &mut VecDeque, + ) -> Option<()> { + if !self.nack_enabled() { + return None; + } + + let nacks = self.register.as_mut().and_then(|r| r.nack_report())?; + + for mut nack in nacks { + nack.sender_ssrc = sender_ssrc; + nack.ssrc = self.ssrc; + + trace!("Created feedback NACK: {:?}", nack); + feedback.push_back(Rtcp::Nack(nack)); + self.stats.nacks += 1; + } + + Some(()) + } + + pub(crate) fn visit_stats(&self, snapshot: &mut StatsSnapshot, now: Instant) { + if self.ssrc.is_probe() { + return; + } + + self.stats + .fill(snapshot, self.midrid, self.sender_info.as_ref(), now); + } + + pub(crate) fn poll_paused(&mut self) -> Option { + if self.ssrc.is_probe() { + return None; + } + + if !self.need_paused_event { + return None; + } + + self.need_paused_event = false; + + debug!( + "{} StreamRx with {:?} and SSRC: {}", + if self.paused { "Paused" } else { "Unpaused" }, + self.midrid, + self.ssrc + ); + + Some(StreamPaused { + ssrc: self.ssrc, + mid: self.midrid.mid(), + rid: self.midrid.rid(), + paused: self.paused, + }) + } + + /// Poll the most recent sender info and when it was received + pub(crate) fn poll_sender_info(&mut self) -> Option<(SenderInfo, Instant)> { + let i = self.sender_info.as_mut()?; + if i.emitted { + return None; + } + + i.emitted = true; + + Some((i.info, i.received_at)) + } + + pub(crate) fn reset_buffers(&mut self, max_seq_lookup: impl Fn(Ssrc) -> Option) { + if let Some(r) = &mut self.register { + r.clear(max_seq_lookup(self.ssrc)); + } + + if let Some(r) = &mut self.register_rtx { + r.clear(self.rtx.and_then(max_seq_lookup)); + } + self.pending_request_keyframe = None; + } + + #[must_use] + pub(crate) fn change_ssrc(&mut self, ssrc: Ssrc) -> bool { + // Avoid flapping + if ssrc == self.ssrc || Some(ssrc) == self.previous_ssrc { + return false; + } + + debug!( + "Change main SSRC: {} -> {} {:?}", + self.ssrc, ssrc, self.midrid + ); + + // Remember which was the previous in case a stray packet turns up + // so do we don't go "backwards". + self.previous_ssrc = Some(self.ssrc); + self.ssrc = ssrc; + + // Reset all SSRC-specific state + self.register = None; + self.last_time = None; + self.last_clock_rate = None; + self.sender_info = None; + self.last_receiver_report = already_happened(); + self.fir_seq_no = 0; + self.pending_request_keyframe = None; + self.pending_request_remb = None; + self.reset_roc = None; + + // Note: We don't reset the RTX register here, as the RTX SSRC is managed separately + // via maybe_reset_rtx() and is not directly tied to the main SSRC change. + + true + } + + pub(crate) fn maybe_reset_rtx(&mut self, rtx: Ssrc) { + if let Some(current) = self.rtx { + if current == rtx { + return; + } + + debug!( + "Change RTX SSRC {} -> {} for main SSRC: {} {:?}", + current, rtx, self.ssrc, self.midrid + ); + } else { + debug!("SSRC {} associated with RTX: {}", self.ssrc, rtx); + } + + self.rtx = Some(rtx); + self.register_rtx = None; + } + + /// Reset the current rollover counter (ROC). + /// + /// This is used in scenarios where we use a single sequence number across all + /// receivers of the same stream (as opposed to a sequence number unique per peer). + /// + /// [RFC3711](https://datatracker.ietf.org/doc/html/rfc3711#section-3.3.1): + /// + /// > Receivers joining an on-going session MUST be given the + /// > current ROC value using out-of-band signaling such as key-management + /// > signaling. Furthermore, the receiver SHALL initialize s_l to the RTP + /// > sequence number (SEQ) of the first observed SRTP packet (unless the + /// > initial value is provided by out of band signaling such as key + /// > management). + pub fn reset_roc(&mut self, roc: u64) { + self.register = None; + self.register_rtx = None; + self.reset_roc = Some(roc); + } + + pub(crate) fn is_midrid(&self, midrid: MidRid) -> bool { + midrid.special_equals(&self.midrid) + } +} + +impl StreamRxStats { + fn update_loss(&mut self, fraction_lost: u8) { + self.loss = Some(fraction_lost as f32 / u8::MAX as f32) + } + + pub(crate) fn fill( + &self, + snapshot: &mut StatsSnapshot, + midrid: MidRid, + sender_info: Option<&LastSenderInfo>, + now: Instant, + ) { + if self.bytes == 0 { + return; + } + + let stats = MediaIngressStats { + mid: midrid.mid(), + rid: midrid.rid(), + bytes: self.bytes, + packets: self.packets, + firs: self.firs, + plis: self.plis, + nacks: self.nacks, + jitter: self.jitter, + rtt: self.rtt, + loss: self.loss, + timestamp: now, + remote: sender_info.map(|l| RemoteEgressStats { + bytes: l.info.sender_octet_count as u64, + packets: l.info.sender_packet_count as u64, + }), + }; + + // Several SSRCs can back a given (mid, rid) tuple. For example, Firefox creates new SSRCs + // when a Transceiver transitions from send -> inactive -> send. In order to continue + // correctly reporting stats for this (mid, rid) pair we need to merge the stats across all + // the SSRCs that have been used. + snapshot + .ingress + .entry(midrid) + .and_modify(|s| s.merge_by_mid_rid(&stats)) + .or_insert(stats); + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct RegisterUpdateReceipt { + pub time: MediaTime, + pub is_new_packet: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paused_timestamp_repair_moves_time_forward() { + let now = already_happened(); + let mut stream = StreamRx::new(7.into(), MidRid("mid".into(), None), false); + let previous_time = 1; + stream.last_time = Some(MediaTime::new(previous_time, Frequency::NINETY_KHZ)); + stream.paused = true; + + let header = RtpHeader { + payload_type: Pt::new_with_value(96), + sequence_number: 42, + timestamp: 0, + ssrc: 7.into(), + ..Default::default() + }; + + let seq_no = stream.extend_seq(&header, false, |_| None); + let receipt = stream.update_register(now, &header, Frequency::NINETY_KHZ, false, seq_no); + + assert!( + receipt.time.numer() > previous_time, + "expected repaired media time to move forward" + ); + assert!(!stream.paused); + } + + #[test] + fn receiver_report_uses_supplied_intervals() { + let mut stream = StreamRx::new(7.into(), MidRid("mid".into(), None), false); + let now = Instant::now(); + stream.last_receiver_report = now; + + assert_eq!( + stream.receiver_report_at(RtcpReportIntervals { + audio: Duration::from_millis(750), + video: Duration::from_millis(500), + }), + now + Duration::from_millis(750) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register.rs new file mode 100644 index 000000000..c0b855c05 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register.rs @@ -0,0 +1,426 @@ +use std::time::Instant; + +use crate::rtp_::{Nack, ReceptionReport, SeqNo}; + +use super::register_nack::NackRegister; + +#[derive(Debug)] +pub struct ReceiverRegister { + nack: NackRegister, + + /// First sequence number received + first: Option, + + /// Number of packets received + count: u64, + + /// Previously received time point. + time_point_prior: Option, + + /// Expected at last reception report generation. + expected_prior: i64, + + /// Received at last reception report generation. + received_prior: i64, + + /// Interarrival jitter in **microseconds**. + /// + /// RTCP carries jitter in **RTP timestamp units** so use + /// [`ReceiverRegister::jitter_in_rtp_ts`] for the wire value. + jitter: f32, +} + +#[derive(Debug, Clone, Copy)] +struct TimePoint { + arrival: Instant, + rtp_time: u32, + clock_rate: u32, +} + +impl TimePoint { + fn is_same(&self, other: TimePoint) -> bool { + self.rtp_time == other.rtp_time + } + + fn delta(&self, other: TimePoint) -> f32 { + // See + // https://www.rfc-editor.org/rfc/rfc3550#appendix-A.8 + // + // rdur is often i 90kHz (for video) or 48kHz (for audio). we need + // a time unit of Duration, that is likely to give us an increase between + // 1 in rdur. milliseconds is thus "too coarse" + + // wrapping_sub to handle RTP time rollover + let rtp_diff = self.rtp_time.wrapping_sub(other.rtp_time) as i32; + let rdur = rtp_diff as f32 * 1_000_000.0 / self.clock_rate as f32; + + let tdur = (self.arrival - other.arrival).as_micros() as f32; + + let d = (tdur - rdur).abs(); + + trace!("Timepoint delta: {}", d); + + d + } +} + +impl ReceiverRegister { + pub fn new(max_seq_no: Option) -> Self { + ReceiverRegister { + nack: NackRegister::new(max_seq_no), + first: None, + count: 0, + time_point_prior: None, + expected_prior: 0, + received_prior: 0, + jitter: 0.0, + } + } + + pub fn accepts(&self, seq: SeqNo) -> bool { + self.nack.accepts(seq) + } + + pub fn update(&mut self, seq: SeqNo, arrival: Instant, rtp_time: u32, clock_rate: u32) -> bool { + if self.first.is_none() { + self.first = Some(seq); + } + + let new = self.nack.update(seq); + + if new { + self.count += 1; + } + + self.update_time(arrival, rtp_time, clock_rate); + + new + } + + /// Generates a NACK report + pub fn nack_report(&mut self) -> Option> { + self.nack.nack_reports() + } + + /// Create a new reception report. + /// + /// This modifies the state since fraction_lost is calculated + /// since the last call to this function. + pub fn reception_report(&mut self) -> Option { + let first = self.first?; + let last = self.max_seq()?; + + let expected = expected(first, last); + + Some(ReceptionReport { + ssrc: 0.into(), + fraction_lost: self.fraction_lost(expected, self.count as i64), + packets_lost: packets_lost(expected, self.count as i64), + max_seq: (*last % ((u32::MAX as u64) + 1_u64)) as u32, + jitter: self.jitter_in_rtp_ts(), + last_sr_time: 0, + last_sr_delay: 0, + }) + } + + pub fn max_seq(&self) -> Option { + self.nack.max_seq() + } + + pub fn clear(&mut self, max_seq_no: Option) { + self.nack = NackRegister::new(max_seq_no); + self.count = 0; + self.first = None; + self.time_point_prior = None; + self.expected_prior = 0; + self.received_prior = 0; + self.jitter = 0.0; + } + + fn update_time(&mut self, arrival: Instant, rtp_time: u32, clock_rate: u32) { + let tp = TimePoint { + arrival, + rtp_time, + clock_rate, + }; + + if let Some(prior) = self.time_point_prior { + if tp.is_same(prior) { + // rtp_time didn't move forward. this is quite normal + // when multiple rtp packets are needed for one keyframe. + + // https://www.cs.columbia.edu/~hgs/rtp/faq.html#jitter + // + // If several packets, say, within a video frame, bear the + // same timestamp, it is advisable to only use the first + // packet in a frame to compute the jitter. (This issue may + // be addressed in a future version of the specification.) + // Jitter is computed in timestamp units. For example, for + // an audio stream sampled at 8,000 Hz, the arrival time + // measured with the local clock is converted by multiplying + // the seconds by 8,000. + // + // Steve Casner wrote: + // + // For encodings such as MPEG that transmit data in a + // different order than it was sampled, this adds noise + // into the jitter calculation. I have heard handwavy + // arguments that this factor can be calculated out given + // that you know the shape of the noise, but my math + // isn't strong enough for that. + // + // In many of the cases that we care about, the jitter + // introduced by MPEG will be small enough that when the + // network jitter is of the same order we don't have a + // problem anyway. + // + // There is another problem for video in that all of the + // packets of a frame have the same timestamp because the + // whole frame is sampled at once. However, the + // dispersion in time of those packets really is all part + // of the network transfer process that the receiver must + // accommodate with its buffer. + // + // It has been suggested that jitter be calculated only + // on the first packet of a video frame, or only on "I" + // frames for MPEG. However, that may color the results + // also because those packets may see transit delays + // different than the following packets see. + // + // The main point to remember is that the primary + // function of the RTP timestamp is to represent the + // inherent notion of real time associated with the + // media. It also turns out to be useful for the jitter + // measure, but that is a secondary function. + // + // The jitter value is not expected to be useful as an + // absolute value. It is more useful as a means of + // comparing the reception quality at two receiver or + // comparing the reception quality 5 minutes ago to now. + + return; + } + + // update jitter. + let d = tp.delta(prior); + + self.jitter += (1.0 / 16.0) * (d - self.jitter); + } + + self.time_point_prior = Some(tp); + } + + // Calculations from here + // https://www.rfc-editor.org/rfc/rfc3550#appendix-A.3 + + /// Fraction lost since last call. + fn fraction_lost(&mut self, expected: i64, received: i64) -> u8 { + let expected_interval = expected - self.expected_prior; + self.expected_prior = expected; + + let received_interval = received - self.received_prior; + self.received_prior = received; + + let lost_interval = expected_interval - received_interval; + + let lost = if expected_interval == 0 || lost_interval == 0 { + 0 + } else { + (lost_interval << 8) / expected_interval + } as u8; + + trace!("Reception fraction lost: {}", lost); + + lost + } + + /// Jitter in RTP timestamp units. + fn jitter_in_rtp_ts(&self) -> u32 { + let Some(sample_rate) = self.time_point_prior.map(|tp| tp.clock_rate) else { + return 0; + }; + (self.jitter / 1_000_000.0 * sample_rate as f32).round() as u32 + } +} + +/// Absolute number of lost packets. +fn packets_lost(expected: i64, received: i64) -> u32 { + // Since this signed number is carried in 24 bits, it should be clamped + // at 0x7fffff for positive loss or 0x800000 for negative loss rather + // than wrapping around. + let lost_t = expected - received; + if lost_t > 0x7fffff { + 0x7fffff_u32 + } else if lost_t < -0x7fffff { + 0x8000000_u32 + } else { + lost_t as u32 + } +} + +fn expected(first: SeqNo, last: SeqNo) -> i64 { + let delta = (*last - *first) as i64; + delta.saturating_add(1) +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use crate::streams::register::{ReceiverRegister, expected, packets_lost}; + + #[test] + fn jitter_at_0() { + let mut r = ReceiverRegister::new(None); + + // 100 fps in clock rate 90kHz => 90_000/100 = 900 per frame + // 1/100 * 1_000_000 = 10_000 microseconds per frame. + + let start = Instant::now(); + let dur = Duration::from_micros(10_000); + + r.update_time(start + 4 * dur, 1234 + 4 * 900, 90_000); + r.update_time(start + 5 * dur, 1234 + 5 * 900, 90_000); + r.update_time(start + 6 * dur, 1234 + 6 * 900, 90_000); + r.update_time(start + 7 * dur, 1234 + 7 * 900, 90_000); + assert_eq!(r.jitter, 0.0); + } + + #[test] + fn jitter_at_20() { + let mut r = ReceiverRegister::new(None); + + // 100 fps in clock rate 90kHz => 90_000/100 = 900 per frame + // 1/100 * 1_000_000 = 10_000 microseconds per frame. + + let start = Instant::now(); + let dur = Duration::from_micros(10_000); + let off = Duration::from_micros(10); + + for i in 4..1000 { + let arrival = if i % 2 == 0 { + start + (i * dur).checked_sub(off).unwrap() + } else { + start + i * dur + off + }; + r.update((i as u64).into(), arrival, 1234 + i * 900, 90_000); + } + + // jitter should converge on 20.0 + assert!( + (20.0 - r.jitter).abs() < 0.01, + "Expected jitter to converge at 20.0, jitter was: {}", + r.jitter + ); + + // jitter is also present in reception report + let report = r.reception_report().expect("some report"); + // 90kHz is 11.1us ticks, so 20us jitter is 1.8 tick which equals 2 + // after rounding to int. + assert_eq!(report.jitter, 2); + assert_eq!(report.jitter, r.jitter_in_rtp_ts()); + } + + #[test] + fn expected_received_loss() { + let first = 14.into(); + let last = 17.into(); + let expected = expected(first, last); + assert_eq!(expected, 4); + // none of 4 was lost + assert_eq!(packets_lost(expected, 4), 0); + // one of 4 was lost:329 + assert_eq!(packets_lost(expected, 3), 1); + } + + #[test] + fn expected_overflow() { + let last = 0x7fff_ffff_ffff_ffff_u64.into(); + let first = 0_u64.into(); + let expected = expected(first, last); + assert_eq!(expected, i64::MAX); + } + + #[test] + fn receiver_report() { + let mut r = ReceiverRegister::new(None); + let now = Instant::now(); + let rtp_time = 0; + + // 50 % lost + for i in 10..14 { + r.update((i as u64).into(), now, rtp_time, 90_000); + } + r.update(19.into(), now, rtp_time, 90_000); + + let report = r.reception_report().expect("some report"); + assert_eq!(128, report.fraction_lost); + assert_eq!(5, report.packets_lost); + assert_eq!(19, report.max_seq); + assert_eq!(0, report.jitter); + } + + #[test] + fn simple_jitter_computation() { + // SimpleJitterComputation from receive_statistics_unittest.cc + const MS_PER_PACKET: u64 = 20; + const CODEC_SAMPLE_RATE: u32 = 48_000; + const SAMPLES_PER_PACKET: u32 = MS_PER_PACKET as u32 * CODEC_SAMPLE_RATE / 1_000; + const LATE_ARRIVAL_DELTA_MS: u64 = 100; + const LATE_DELTA_SAMPLES: u32 = LATE_ARRIVAL_DELTA_MS as u32 * CODEC_SAMPLE_RATE / 1_000; + + let mut clock = Instant::now(); + let mut r = ReceiverRegister::new(None); + + r.update_time(clock, 0, CODEC_SAMPLE_RATE); + clock += Duration::from_millis(MS_PER_PACKET + LATE_ARRIVAL_DELTA_MS); + r.update_time(clock, SAMPLES_PER_PACKET, CODEC_SAMPLE_RATE); + + assert_eq!(r.jitter_in_rtp_ts(), LATE_DELTA_SAMPLES / 16); + } + + #[test] + fn all_packets_have_same_frequency() { + // AllPacketsHaveSamePayloadTypeFrequency from receive_statistics_unittest.cc + let mut clock = Instant::now(); + let mut r = ReceiverRegister::new(None); + + r.update_time(clock, 1, 8_000); + clock += Duration::from_millis(50); + r.update_time(clock, 1 + 160, 8_000); + clock += Duration::from_millis(50); + r.update_time(clock, 1 + 160 + 160, 8_000); + + // packet1: no jitter calculation + // packet2: jitter = 0[jitter] + (abs(50[receive time ms] * + // 8[frequency KHz] - 160[timestamp diff]) * 16 - 0[jitter] + 8) + // / 16 = 240 + // packet3: jitter = 240[jitter] + (abs(50[receive time ms] * + // 8[frequency KHz] - 160[timestamp diff]) * 16 - 240[jitter] + 8) + // / 16 = 465 + // final jitter: 465 / 16 = 29 + assert_eq!(r.jitter_in_rtp_ts(), 29); + } + + #[test] + fn jitter_rtp_timestamp_rollover() { + // Same as jitter_same_frequency_three_packets but the timestamps are + // anchored so the u32 boundary falls between packets 1 and 2. + + let rtp_time_1 = u32::MAX - 79; + let rtp_time_2 = rtp_time_1.wrapping_add(160); + let rtp_time_3 = rtp_time_2.wrapping_add(160); + + let mut clock = Instant::now(); + let mut r = ReceiverRegister::new(None); + + r.update_time(clock, rtp_time_1, 8_000); + clock += Duration::from_millis(50); + r.update_time(clock, rtp_time_2, 8_000); + clock += Duration::from_millis(50); + r.update_time(clock, rtp_time_3, 8_000); + + // Same result as jitter_same_frequency_three_packets. + assert_eq!(r.jitter_in_rtp_ts(), 29); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register_nack.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register_nack.rs new file mode 100644 index 000000000..e2c211560 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/register_nack.rs @@ -0,0 +1,649 @@ +use std::ops::Range; + +use crate::rtp_::{Nack, NackEntry, ReportList, SeqNo}; + +/// Number of out of order packets we keep track of for reports +const MAX_MISORDER: u64 = 100; + +/// The max number of NACKs we perform for a single packet +const MAX_NACKS: u8 = 5; + +/// Circular buffer size +const BUFFER_SIZE: u64 = MAX_MISORDER + 1; + +#[derive(Debug)] +pub struct NackRegister { + /// Status of packets indexed by wrapping SeqNo. + packets: Vec, + + /// Range of seq numbers considered NACK reporting. + active: Option>, +} + +#[derive(Debug, Default, Clone, Copy)] +struct PacketStatus { + received: bool, + nack_count: u8, +} + +impl PacketStatus { + fn needs_nack(&self) -> bool { + !self.received && self.nack_count < MAX_NACKS + } + + fn mark_received(&mut self) -> bool { + let new = !self.received; + self.received = true; + new + } + + fn reset(&mut self) { + self.received = false; + self.nack_count = 0; + } +} + +struct NackIterator<'a> { + reg: &'a mut NackRegister, + next: u64, + end: u64, +} + +impl<'a> Iterator for NackIterator<'a> { + type Item = NackEntry; + + fn next(&mut self) -> Option { + self.next = + (self.next..=self.end).find(|s| self.reg.packet_mut((*s).into()).needs_nack())?; + + let mut entry = NackEntry { + pid: self.next as u16, + blp: 0, + }; + + self.reg.packet_mut(self.next.into()).nack_count += 1; + self.next += 1; + + for (i, s) in (self.next..self.end).take(16).enumerate() { + let packet = self.reg.packet_mut(s.into()); + if packet.needs_nack() { + self.reg.packet_mut(self.next.into()).nack_count += 1; + entry.blp |= 1 << i + } + self.next += 1; + } + + Some(entry) + } +} + +impl NackRegister { + /// Creates a new register. + /// + /// The max_seq_no is to provide a starting point for ROC calculations. + pub fn new(max_seq_no: Option) -> Self { + let mut n = NackRegister { + packets: vec![PacketStatus::default(); BUFFER_SIZE as usize], + active: None, + }; + + if let Some(seq) = max_seq_no { + n.init_with_seq(seq); + } + + n + } + + pub fn accepts(&self, seq: SeqNo) -> bool { + let Some(active) = self.active.clone() else { + // if we don't have initialized, we do want the first packet. + return true; + }; + + // behind the window + if seq < active.start { + return false; + } + + !self.packet(seq).received || seq > active.end + } + + pub fn update(&mut self, seq: SeqNo) -> bool { + let Some(active) = self.active.clone() else { + // automatically pick up the first seq number + self.init_with_seq(seq); + return true; + }; + + if seq < active.start { + // skip old seq numbers, report as not new + return false; + } + + let new = !self.packet_mut(seq).received || seq > active.end; + + let end = active.end.max(seq); + + let start: SeqNo = { + let min = end.saturating_sub(MAX_MISORDER); + let mut start = (*active.start).max(min); + while start < *end { + if !self.packet_mut(start.into()).received && start != *seq { + break; + } + start += 1; + } + start.into() + }; + + // reset packets that are rolling our of the nack window + for (i, s) in (*active.start..*start).enumerate() { + let p = self.packet_mut(s.into()); + if !p.received && s != *seq { + trace!("Seq no {} missing after {} attempts", s, p.nack_count); + } + self.packet_mut(s.into()).reset(); + + if i > self.packets.len() { + // we have reset all entries already + break; + } + } + + if (start..=end).contains(&seq) { + self.packet_mut(seq).mark_received(); + } + + self.active = Some(start..end); + + new + } + + fn init_with_seq(&mut self, seq: SeqNo) { + self.active = Some(seq..seq); + self.packet_mut(seq).mark_received(); + } + + pub fn max_seq(&self) -> Option { + self.active.as_ref().map(|a| a.end) + } + + /// Create a new nack report + /// + /// This modifies the state as it counts how many times packets have been nacked + pub fn nack_reports(&mut self) -> Option> { + let Range { start, end } = self.active.clone()?; + let start = (*start..=*end).find(|s| self.packet_mut((*s).into()).needs_nack())?; + + Some( + ReportList::lists_from_iter(NackIterator { + reg: self, + next: start, + end: *end, + }) + .into_iter() + .map(|reports| { + Nack { + sender_ssrc: 0.into(), + ssrc: 0.into(), // changed when sending + reports, + } + }), + ) + } + + fn as_index(&self, seq: SeqNo) -> usize { + (*seq % self.packets.len() as u64) as usize + } + + fn packet(&self, seq: SeqNo) -> &PacketStatus { + let index = self.as_index(seq); + &self.packets[index] + } + + fn packet_mut(&mut self, seq: SeqNo) -> &mut PacketStatus { + let index = self.as_index(seq); + &mut self.packets[index] + } +} + +#[cfg(test)] +mod test { + use std::ops::Range; + + use crate::streams::register_nack::MAX_MISORDER; + + use super::NackRegister; + + fn assert_update( + reg: &mut NackRegister, + seq: u64, + expect_new: bool, + expect_received: bool, + expect_active: Range, + ) { + assert_eq!( + reg.update(seq.into()), + expect_new, + "seq {} was expected to{} be new", + seq, + if expect_new { "" } else { " NOT" } + ); + let active = reg.active.clone().expect("nack range"); + assert_eq!( + reg.packet_mut(seq.into()).received, + expect_received, + "seq {} expected to{} be received in {:?}", + seq, + if expect_received { "" } else { " NOT" }, + active + ); + assert_eq!(active, expect_active.start.into()..expect_active.end.into()); + assert_not_dirty(reg); + } + + fn assert_not_dirty(reg: &NackRegister) { + // we should leave no dirty state outside of the nack window + let active = reg.active.clone().expect("nack range"); + let active = (*active.start..=*active.end) + .map(|seq| reg.as_index(seq.into())) + .collect::>(); + + for i in 0..reg.packets.len() { + if active.contains(&i) { + continue; + } + assert!( + !reg.packets[i].received && reg.packets[i].nack_count == 0, + "dirty state at index {} outside of nack window {:?}", + i, + active, + ); + } + } + + #[test] + fn active_window_sliding() { + let mut reg = NackRegister::new(None); + + assert!(reg.accepts(10.into())); + assert_update(&mut reg, 10, true, true, 10..10); + + // packet before window start is ignored + assert!(!reg.accepts(9.into())); + assert_update(&mut reg, 9, false, false, 10..10); + + // duped packet + assert!(!reg.accepts(10.into())); + assert_update(&mut reg, 10, false, true, 10..10); + + // future packets accepted, window not sliding + let next = 10 + MAX_MISORDER; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, true, 11..next); + let next = 11 + MAX_MISORDER; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, true, 11..next); + + // future packet accepted, sliding window + let next = 12 + MAX_MISORDER; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, true, 12..next); + + // older packet received within window + let next = 13; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, true, 12..(12 + MAX_MISORDER)); + + // do not want the same packet again + assert!(!reg.accepts(next.into())); + + // future packet accepted, sliding window start skips over received + let next = 13 + MAX_MISORDER; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, true, 14..next); + + // do not want the same packet again + assert!(!reg.accepts(next.into())); + + // older packet accepted, window star moves ahead + let next = 14; + assert!(reg.accepts(next.into())); + assert_update(&mut reg, next, true, false, 15..(13 + MAX_MISORDER)); + } + + #[test] + fn nack_report_none() { + let mut reg = NackRegister::new(None); + assert!(reg.nack_reports().is_none()); + + reg.update(110.into()); + assert!(reg.nack_reports().is_none()); + + reg.update(111.into()); + assert!(reg.nack_reports().is_none()); + } + + #[test] + fn nack_test_huge_seq_gap_no_hang() { + let mut reg = NackRegister::new(None); + + reg.update(0.into()); + reg.update(18446744073709551515.into()); + } + + #[test] + fn nack_report_one() { + let mut reg = NackRegister::new(None); + assert!(reg.nack_reports().is_none()); + + reg.update(110.into()); + assert!(reg.nack_reports().is_none()); + + reg.update(112.into()); + let report = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert!(report.len() == 1); + assert_eq!(report[0].reports.len(), 1); + assert_eq!(report[0].reports[0].pid, 111); + assert_eq!(report[0].reports[0].blp, 0); + } + + #[test] + fn nack_report_two() { + let mut reg = NackRegister::new(None); + assert!(reg.nack_reports().is_none()); + + reg.update(110.into()); + assert!(reg.nack_reports().is_none()); + + reg.update(113.into()); + let report = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert!(report.len() == 1); + assert_eq!(report[0].reports.len(), 1); + assert_eq!(report[0].reports[0].pid, 111); + assert_eq!(report[0].reports[0].blp, 0b1); + } + + #[test] + fn nack_report_with_hole() { + let mut reg = NackRegister::new(None); + + for i in &[100, 101, 103, 105, 106, 107, 108, 109, 110] { + reg.update((*i).into()); + } + + let report = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert!(report.len() == 1); + assert_eq!(report[0].reports.len(), 1); + assert_eq!(report[0].reports[0].pid, 102); + assert_eq!(report[0].reports[0].blp, 0b10); + } + + #[test] + fn nack_report_stop_at_17() { + let mut reg = NackRegister::new(None); + + let seq = &[ + 100, 101, 103, 104, 105, 106, 107, 108, 109, 110, // + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, // + 121, 122, 123, 125, + ]; + + for i in seq { + reg.update((*i).into()); + } + + let report = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert_eq!(report.len(), 1); + assert_eq!(report[0].reports.len(), 2); + assert_eq!(report[0].reports[0].pid, 102); + assert_eq!(report[0].reports[0].blp, 0); + } + + #[test] + fn nack_report_hole_at_17() { + let mut reg = NackRegister::new(None); + + let seq = &[ + 100, 101, 103, 104, 105, 106, 107, 108, 109, 110, // + 111, 112, 113, 114, 115, 116, 117, 119, 120, 121, // + 122, 123, 124, 125, 126, 127, 128, 129, + ]; + + for i in seq { + reg.update((*i).into()); + } + + let report = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert_eq!(report.len(), 1); + assert_eq!(report[0].reports.len(), 1); + assert_eq!(report[0].reports[0].pid, 102); + assert_eq!(report[0].reports[0].blp, 0b1000_0000_0000_0000); + } + + #[test] + fn nack_report_no_stop_all_there() { + let mut reg = NackRegister::new(None); + + let seq = &[ + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, // + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, // + 122, 123, 124, 125, 126, 127, 128, 129, + ]; + + for i in seq { + reg.update((*i).into()); + } + + assert!(reg.nack_reports().is_none()); + } + + #[test] + fn nack_report_rtx() { + let mut reg = NackRegister::new(None); + for i in &[ + 100, 101, 102, 103, 104, 105, // + ] { + reg.update((*i).into()); + } + assert!(reg.nack_reports().is_none()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 105); + + for i in &[ + 106, 108, 109, 110, 111, 112, 113, 114, 115, // + ] { + reg.update((*i).into()); + } + assert!(reg.nack_reports().is_some()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 107); + + reg.update(107.into()); // Got 107 via RTX + + let nacks = reg.nack_reports().map(Vec::from_iter); + assert!( + nacks.is_none(), + "Expected no NACKs to be generated after repairing the stream, got {nacks:?}" + ); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 115); + } + + #[test] + fn nack_report_rollover_rtx() { + // This test is checking that after rollover nacks are not skipped because of + // packet position that would remain marked as received from before the rollover + let mut reg = NackRegister::new(None); + for i in &[ + 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, + ] { + reg.update((*i).into()); + } + + reg.update(107.into()); // Got 107 via RTX + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 115); + + for i in 116..3106 { + reg.update(i.into()); + } + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 3105); + + for i in &[3106, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115] { + reg.update((*i).into()); // Missing at postion 107 again + } + + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 3107); + } + + #[test] + fn nack_report_rollover_rtx_with_seq_jump() { + let mut reg = NackRegister::new(None); + + // 2999 is missing + for i in 0..2999 { + reg.update(i.into()); + } + + // 3002 is missing + reg.update(3003.into()); + reg.update(3004.into()); + reg.update(3000.into()); + reg.update(3001.into()); + + let reports = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].reports[0].pid, 2999); + assert_eq!(reports[0].reports[0].blp, 4); + } + + #[test] + fn out_of_order_and_rollover() { + let mut reg = NackRegister::new(None); + + reg.update(2998.into()); + reg.update(2999.into()); + + // receive older packet + reg.update(2995.into()); + + // wrap + for i in 3000..5995 { + reg.update(i.into()); + } + + // 5995 is missing + + reg.update(5996.into()); + reg.update(5997.into()); + + let reports = reg.nack_reports().map(Vec::from_iter).expect("some report"); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].reports[0].pid, 5995); + } + + #[test] + fn nack_check_on_seq_rollover() { + let range = 65530..65541; + let missing = [65535_u64, 65536_u64, 65537_u64]; + let expected = [65535_u16, 0_u16, 1_u16]; + + for (missing, expected) in missing.iter().zip(expected.iter()) { + let mut seqs: Vec<_> = range.clone().collect(); + let mut reg = NackRegister::new(None); + + seqs.retain(|x| *x != *missing); + for i in seqs.as_slice() { + reg.update((*i).into()); + } + + let reports = reg.nack_reports().map(Vec::from_iter).expect("some report"); + let pid = reports[0].reports[0].pid; + assert_eq!(pid, *expected); + } + } + + #[test] + fn nack_check_forward_at_boundary() { + let mut reg = NackRegister::new(None); + for i in 2996..=3003 { + reg.update(i.into()); + } + + assert!(reg.nack_reports().is_none()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 3003); + + for i in 3004..=3008 { + reg.update(i.into()); + } + + let report = reg.nack_reports().map(Vec::from_iter); + assert!(report.is_none(), "Expected empty NACKs got {:?}", report); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 3008); + } + + #[test] + fn nack_check_forward_at_u16_boundary() { + let mut reg = NackRegister::new(None); + for i in 65500..=65534 { + reg.update(i.into()); + } + assert!(reg.nack_reports().is_none()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 65534); + + for i in 65536..=65566 { + reg.update(i.into()); + } + + assert!(reg.nack_reports().is_some()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 65535); + + for i in 65567..=65666 { + reg.update(i.into()); + } + + reg.update(65535.into()); + + assert!(reg.nack_reports().is_none()); + let active = reg.active.clone().expect("nack range"); + assert_eq!(*active.start, 65666); + } + + #[test] + fn nack_reports_on_boundaries() { + let mut reg = NackRegister::new(None); + + for i in 0..=20 { + // gap must be at least 17 packets to separate the reports. + if i == 2 || i == 19 { + continue; + } + reg.update(i.into()); + } + + let reports: Vec<_> = reg + .nack_reports() + .expect("should generate reports") + .flat_map(|nack| nack.reports) + .collect(); + + assert_eq!(reports.len(), 2, "Should have found two NACK entries"); + assert_eq!(reports[0].pid, 2, "First missing packet should be 2"); + assert_eq!( + reports[0].blp, 0, + "No missing packets in blp bits following 2" + ); + assert_eq!(reports[1].pid, 19, "Second missing packet should be 19"); + assert_eq!( + reports[1].blp, 0, + "No missing packets in blp bits following 19" + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache.rs new file mode 100644 index 000000000..a2dc315f5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache.rs @@ -0,0 +1,192 @@ +use std::time::Duration; +use std::time::Instant; + +use crate::io::DATAGRAM_MAX_PACKET_SIZE; +use crate::rtp_::SeqNo; + +use super::RtpPacket; +use super::rtx_cache_buf::EvictingBuffer; + +const RTX_CACHE_SIZE_QUANTIZER: usize = 25; +const RTX_CACHE_QUANTIZE_SLOTS: usize = DATAGRAM_MAX_PACKET_SIZE / RTX_CACHE_SIZE_QUANTIZER; + +#[derive(Debug)] +pub(crate) struct RtxCache { + // Data, new additions here probably need to be cleared in [`clear`]. + packet_by_seq_no: EvictingBuffer, + + // Technically we want [Option; X] to indicate the absence of + // a SeqNo. However We can half the storage space by using the sentinel + // values SeqNo::MAX to indicate None + seq_no_by_quantized_size: [SeqNo; RTX_CACHE_QUANTIZE_SLOTS], +} + +impl RtxCache { + pub fn new(max_packet_count: usize, max_packet_age: Duration) -> Self { + Self { + packet_by_seq_no: EvictingBuffer::new(10, max_packet_age, max_packet_count), + seq_no_by_quantized_size: [SeqNo::MAX; RTX_CACHE_QUANTIZE_SLOTS], + } + } + + pub fn cache_sent_packet(&mut self, packet: RtpPacket, now: Instant) { + assert!(packet.nackable); + let seq_no = packet.seq_no; + let quantized_size = packet.payload.len() / RTX_CACHE_SIZE_QUANTIZER; + self.packet_by_seq_no.push(*seq_no, now, packet); + self.seq_no_by_quantized_size[quantized_size] = seq_no; + self.remove_old_packets(now); + } + + pub fn last_cached_seq_no(&self) -> Option { + Some(self.packet_by_seq_no.last_position()?.into()) + } + + pub fn get_cached_packet_by_seq_no(&mut self, seq_no: SeqNo) -> Option<&mut RtpPacket> { + self.packet_by_seq_no.get_mut(*seq_no) + } + + pub fn get_cached_packet_smaller_than(&mut self, max_size: usize) -> Option<&mut RtpPacket> { + let quantized_size = max_size / RTX_CACHE_SIZE_QUANTIZER; + + let seq_no = self.seq_no_by_quantized_size[..quantized_size] + .iter() + .rev() + .filter(|seq_no| !seq_no.is_max()) + .find(|seq_no| self.packet_by_seq_no.contains(***seq_no))?; + + self.get_cached_packet_by_seq_no(*seq_no) + } + + fn remove_old_packets(&mut self, now: Instant) { + self.packet_by_seq_no.maybe_evict(now); + } + + pub(crate) fn last_packet(&self) -> Option<&[u8]> { + let packet = self.packet_by_seq_no.last()?; + Some(packet.payload.as_ref()) + } + + pub(crate) fn clear(&mut self) { + self.packet_by_seq_no.clear(); + self.seq_no_by_quantized_size = [SeqNo::MAX; RTX_CACHE_QUANTIZE_SLOTS]; + } +} + +#[cfg(test)] +mod test { + use crate::rtp_::MediaTime; + use crate::rtp_::RtpHeader; + + use super::*; + + fn after(now: Instant, millis: u64) -> Instant { + now + Duration::from_millis(millis) + } + + fn packet(now: Instant, seq_no: u64, millis: u64) -> RtpPacket { + RtpPacket { + header: RtpHeader::default(), + seq_no: seq_no.into(), + time: MediaTime::from_90khz(0), + payload: millis.to_be_bytes().into(), + vp8_patch: None, + timestamp: after(now, millis), + last_sender_info: None, + nackable: true, + } + } + + #[test] + fn rtx_cache_0_sized() { + let now = Instant::now(); + let mut rtx_cache = RtxCache::new(0, Duration::from_secs(3)); + rtx_cache.cache_sent_packet(packet(now, 1, 10), after(now, 10)); + assert_eq!( + Some(&mut packet(now, 1, 10)), + rtx_cache.get_cached_packet_by_seq_no(1.into()) + ); + assert_eq!( + Some(&mut packet(now, 1, 10)), + rtx_cache.get_cached_packet_smaller_than(1000) + ); + } + + #[test] + fn rtx_cache_0_duration() { + let now = Instant::now(); + let mut rtx_cache = RtxCache::new(10, Duration::from_secs(0)); + rtx_cache.cache_sent_packet(packet(now, 1, 10), after(now, 10)); + assert_eq!(None, rtx_cache.get_cached_packet_by_seq_no(1.into())); + assert_eq!(None, rtx_cache.get_cached_packet_smaller_than(1000)); + } + + #[test] + fn rtx_cache_1_sized() { + let now = Instant::now(); + let mut rtx_cache = RtxCache::new(1, Duration::from_secs(3)); + rtx_cache.cache_sent_packet(packet(now, 1, 10), after(now, 10)); + assert_eq!( + Some(&mut packet(now, 1, 10)), + rtx_cache.get_cached_packet_by_seq_no(1.into()) + ); + assert_eq!( + Some(&mut packet(now, 1, 10)), + rtx_cache.get_cached_packet_smaller_than(1000) + ); + assert_eq!( + Some(&mut packet(now, 1, 10)), + rtx_cache.get_cached_packet_smaller_than(25) + ); + assert_eq!(None, rtx_cache.get_cached_packet_smaller_than(24)); + rtx_cache.cache_sent_packet(packet(now, 2, 20), after(now, 20)); + assert_eq!( + Some(&mut packet(now, 2, 20)), + rtx_cache.get_cached_packet_by_seq_no(2.into()) + ); + } + + #[test] + fn rtx_cache_100_sized_backwards() { + let now = Instant::now(); + let mut rtx_cache = RtxCache::new(100, Duration::from_secs(3)); + + for i in 1..=200 { + let seq_no = 201 - i; + let pkt = packet(now, seq_no, (201 - i) * 10); + let now = after(now, i * 10); + rtx_cache.cache_sent_packet(pkt, now); + } + + assert_eq!( + Some(&mut packet(now, 200, 2000)), + rtx_cache.get_cached_packet_by_seq_no(200.into()) + ); + assert_eq!(None, rtx_cache.get_cached_packet_by_seq_no(100.into())); + assert_eq!(None, rtx_cache.get_cached_packet_by_seq_no(101.into())); + } + + #[test] + fn rtx_cache_eviction() { + let now = Instant::now(); + // Cache can take all entries + let mut rtx_cache = RtxCache::new(400, Duration::from_secs(1)); + for i in 1..=200 { + let seq_no = i; + let pkt = packet(now, seq_no, i * 10); + let now = after(now, i * 10); + rtx_cache.cache_sent_packet(pkt, now); + } + + assert_eq!(None, rtx_cache.get_cached_packet_by_seq_no(99.into())); + + assert_eq!( + Some(&mut packet(now, 100, 1000)), + rtx_cache.get_cached_packet_by_seq_no(100.into()) + ); + assert_eq!( + Some(&mut packet(now, 200, 2000)), + rtx_cache.get_cached_packet_by_seq_no(200.into()) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache_buf.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache_buf.rs new file mode 100644 index 000000000..e4b345a3a --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/rtx_cache_buf.rs @@ -0,0 +1,386 @@ +#![allow(missing_docs)] + +use std::mem; +use std::time::{Duration, Instant}; + +use crate::util::already_happened; + +/// Fixed size buffer that evicts the oldest entries based on time. +/// +/// The buffer is ringbuffer-esque in that all elements are inserted with a position that is modulo to an +/// insert index into the buffer. That makes lookups very fast since they are fixed offsets. +/// +/// The buffer evicts values based on time. If the size of the buffer is too small and would +/// overwrite an entry that has not been evicted due to age, the buffer grows. +#[derive(Debug)] +pub struct EvictingBuffer { + buf: Vec>>, + /// How long to keep entries for. + max_age: Duration, + /// The maximum size allowed to grow to. Once this sized is reached, new pushed + /// entries will overwrite older entries even if they haven't reached max_age. + max_size: usize, + /// Last inserted position + last_position: Option, + /// Next element to evict. + next_evict: Option, + // Last timeout when we evicted elements. + last_timeout: Instant, +} + +/// Entry in the buffer to keep track of position and time separately. +#[derive(Debug)] +struct Entry { + /// Position is offset modulus the buffer size. + position: u64, + /// Entry time. Used for eviction. + timestamp: Instant, + /// The value held at this entry. + value: T, +} + +// We don't want to require that T is Clone/Copy, which means we +// must do this instead of using the vec![] macro. +fn prepare_buf(len: usize) -> Vec> { + let mut buf = Vec::with_capacity(len); + for _ in 0..len { + buf.push(None); + } + buf +} + +impl EvictingBuffer { + /// Creates a new buffer with an initial size. + pub fn new(initial_size: usize, max_age: Duration, max_size: usize) -> Self { + Self { + buf: prepare_buf(initial_size), + max_age, + max_size, + last_position: None, + next_evict: None, + last_timeout: already_happened(), + } + } + + fn index_for_position(&self, position: u64) -> usize { + (position % self.buf.len() as u64) as usize + } + + #[inline(always)] + fn is_inert(&self) -> bool { + self.buf.is_empty() || self.max_age.is_zero() + } + + /// Push a new entry. + /// + /// Position is an increasing sequence number. The sequence can be out of order. + pub fn push(&mut self, position: u64, timestamp: Instant, value: T) { + if self.is_inert() { + return; + } + + if timestamp < self.last_timeout { + // Value is already considered evicted. + return; + } + + let next_evict = if let Some(v) = self.next_evict { + v + } else { + // First ever cached value sets the initial evict position. + self.next_evict = Some(position); + position + }; + + if position < next_evict { + // Do not cache values preceding evict position. + return; + } + + let mut index = self.index_for_position(position); + + if let Some(entry) = &self.buf[index] { + // If the position is exactly the same, we allow it since it's + // replacing the current T value. If position differs, we've + // wrapped around. + if entry.position != position { + // Make space to continue. + self.grow(); + index = self.index_for_position(position); + } + } + self.last_position = Some(position); + + self.buf[index] = Some(Entry { + position, + timestamp, + value, + }); + } + + /// Get the entry for the previously inserted position. + #[allow(unused)] + pub fn get(&self, position: u64) -> Option<&T> { + if self.is_inert() { + return None; + } + + let index = (position % self.buf.len() as u64) as usize; + if let Some(entry) = &self.buf[index] { + if entry.position == position { + return Some(&entry.value); + } + } + None + } + + pub fn get_mut(&mut self, position: u64) -> Option<&mut T> { + if self.is_inert() { + return None; + } + + let index = (position % self.buf.len() as u64) as usize; + if let Some(entry) = &mut self.buf[index] { + if entry.position == position { + return Some(&mut entry.value); + } + } + None + } + + pub fn maybe_evict(&mut self, now: Instant) { + if self.is_inert() { + return; + } + + if now < self.last_timeout { + // Time cannot go backwards. + return; + } + self.last_timeout = now; + + self.evict(now); + } + + fn evict(&mut self, now: Instant) { + let Some(start_position) = self.next_evict else { + // Before first element been pushed. + return; + }; + + let mut position = start_position; + let start_index = self.index_for_position(position); + + loop { + let index = self.index_for_position(position); + + if index == start_index && position > start_position { + // looped around without finding the end. Means there are no elements. + break; + } + + let Some(entry) = &self.buf[index] else { + // No entry means we might have a gap. We can't break on gaps because + // there might be entries to evict later. Skip the position and check + // until we find a timestamp. + position += 1; + continue; + }; + + let age = now.saturating_duration_since(entry.timestamp); + + if age > self.max_age { + // Evict. + self.buf[index] = None; + } else { + // We assume entries are roughly in time order (some jumble is allowed). + // Once we reach an element we should not evict, stop. + break; + } + + position += 1; + } + + self.next_evict = Some(position); + } + + fn grow(&mut self) { + if self.buf.len() >= self.max_size { + // No growing. + return; + } + + // This is the new sized buffer. We can make other strategies for growing. + let new_size = self + .max_size + .min((self.buf.len() * 133) / 100) + .max(self.buf.len() + 1); + + let old_buffer = mem::replace(&mut self.buf, prepare_buf(new_size)); + + // Move all entries over to the new buffer. Changing the buffer size might alter the + // index position. However max_position and next_evict are already containing positions + // not index, which means they stay correct. + for e in old_buffer.into_iter().flatten() { + let index = self.index_for_position(e.position); + self.buf[index] = Some(e); + } + } + + pub fn contains(&self, position: u64) -> bool { + if self.is_inert() { + return false; + } + + let index = self.index_for_position(position); + self.buf[index].is_some() + } + + pub fn last_position(&self) -> Option { + if self.is_inert() { + return None; + } + + self.last_position + } + + pub fn last(&self) -> Option<&T> { + if self.is_inert() { + return None; + } + + let last = self.last_position?; + self.get(last) + } + + pub fn clear(&mut self) { + for i in 0..self.buf.len() { + self.buf[i] = None; + } + self.last_position = None; + self.next_evict = None; + self.last_timeout = already_happened(); + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn push_and_get() { + let mut buf = EvictingBuffer::new(1, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(5, now, 'A'); + + assert_eq!(buf.index_for_position(5), buf.index_for_position(3)); + assert_eq!(buf.get(5), Some(&'A')); + assert_eq!(buf.get(3), None); // modulo to same index + } + + #[test] + fn push_over_capacity() { + let mut buf = EvictingBuffer::new(2, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(5, now + Duration::from_secs(0), 'A'); + buf.push(6, now + Duration::from_secs(1), 'B'); + buf.push(7, now + Duration::from_secs(2), 'C'); + + // The size is 2, we should have grown to accomodate. + assert_eq!(buf.get(5), Some(&'A')); + assert_eq!(buf.get(6), Some(&'B')); + assert_eq!(buf.get(7), Some(&'C')); + } + + #[test] + fn push_before_next_evict() { + let mut buf = EvictingBuffer::new(2, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(6, now + Duration::from_secs(0), 'B'); + assert_eq!(buf.next_evict, Some(6)); + + // Before the initial evict position, thus ignored. + buf.push(5, now + Duration::from_secs(1), 'A'); + + assert_eq!(buf.get(5), None); + } + + #[test] + fn evict_oldest() { + let mut buf = EvictingBuffer::new(2, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(5, now + Duration::from_secs(0), 'A'); + buf.push(6, now + Duration::from_secs(1), 'B'); + + // Nothing should go + buf.maybe_evict(now + Duration::from_secs(1)); + assert_eq!(buf.get(5), Some(&'A')); + assert_eq!(buf.get(6), Some(&'B')); + + // One entry gone. + buf.maybe_evict(now + Duration::from_secs(11)); + assert_eq!(buf.get(5), None); + assert_eq!(buf.get(6), Some(&'B')); + } + + #[test] + fn evict_with_gap() { + let mut buf = EvictingBuffer::new(4, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(5, now + Duration::from_secs(0), 'A'); + // GAP + buf.push(7, now + Duration::from_secs(2), 'C'); + buf.push(8, now + Duration::from_secs(3), 'D'); + + // Should evict A and C + buf.maybe_evict(now + Duration::from_secs(13)); + + assert_eq!(buf.get(5), None); + assert_eq!(buf.get(7), None); + assert_eq!(buf.get(8), Some(&'D')); + } + + #[test] + fn evict_all() { + let mut buf = EvictingBuffer::new(4, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(5, now + Duration::from_secs(0), 'A'); + buf.push(6, now + Duration::from_secs(1), 'B'); + + // Should evict A and B + buf.maybe_evict(now + Duration::from_secs(12)); + + assert_eq!(buf.get(5), None); + assert_eq!(buf.get(6), None); + } + + fn buffer_cmp(b: &EvictingBuffer) -> Vec> { + b.buf.iter().map(|e| e.as_ref().map(|v| v.value)).collect() + } + + #[test] + fn grow_and_reindex() { + let mut buf = EvictingBuffer::new(2, Duration::from_secs(10), 10); + let now = Instant::now(); + + buf.push(2, now + Duration::from_secs(0), 'A'); + buf.push(3, now + Duration::from_secs(1), 'B'); + + assert_eq!(buffer_cmp(&buf), &[Some('A'), Some('B')]); + + // overwrites 2, thus grows + buf.push(4, now + Duration::from_secs(2), 'C'); + + assert_eq!(buffer_cmp(&buf), &[Some('B'), Some('C'), Some('A')]); + + assert_eq!(buf.get(2), Some(&'A')); + assert_eq!(buf.get(3), Some(&'B')); + assert_eq!(buf.get(4), Some(&'C')); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send.rs new file mode 100644 index 000000000..5a828230e --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send.rs @@ -0,0 +1,1330 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use crate::config_mod::RtcpReportIntervals; +use crate::format::CodecConfig; +use crate::format::PayloadParams; +use crate::io::DATAGRAM_MAX_PACKET_SIZE; +use crate::io::MAX_RTP_OVERHEAD; +use crate::media::KeyframeRequestKind; +use crate::media::Media; +use crate::media::MediaKind; +use crate::pacer::QueuePriority; +use crate::pacer::QueueSnapshot; +use crate::pacer::QueueState; +use crate::packet::Vp8Patch; +use crate::rtp_::MidRid; +use crate::rtp_::{Bitrate, Descriptions, Extension, ExtensionMap, ExtensionValues, Frequency}; +use crate::rtp_::{MAX_BLANK_PADDING_PAYLOAD_SIZE, Sdes, SdesType}; +use crate::rtp_::{MediaTime, Mid, NackEntry, ReportList, Rtcp, RtpHeader}; +use crate::rtp_::{Pt, Rid, RtcpFb, SenderInfo, SenderReport, Ssrc}; +use crate::rtp_::{SRTP_BLOCK_SIZE, SeqNo}; +use crate::session::PacketReceipt; +use crate::stats::StatsSnapshot; +use crate::util::value_history::ValueHistory; +use crate::util::{InstantExt, already_happened, not_happening}; + +use super::RtpPacket; +use super::rtx_cache::RtxCache; +use super::send_queue::SendQueue; +use super::send_stats::StreamTxStats; + +/// The smallest size of padding for which we attempt to use a spurious resend. For padding +/// requests smaller than this we use blank packets instead. +const MIN_SPURIOUS_PADDING_SIZE: usize = 50; + +pub const DEFAULT_RTX_CACHE_DURATION: Duration = Duration::from_secs(3); + +pub const DEFAULT_RTX_RATIO_CAP: Option = Some(0.15f32); + +/// A recently sampled view of an outgoing stream's send queue. +/// +/// str0m refreshes this information as part of its regular pacer and bandwidth-estimation +/// processing. Reading it does not cause a new value to be computed. +pub trait StreamTxQueueInfo { + /// When this queue information was sampled. + /// + /// Compare this value between reads to determine whether str0m has computed a new sample. + fn created_at(&self) -> Instant; + + /// Total number of bytes in the queue. + fn byte_size(&self) -> usize; + + /// Total number of packets in the queue. + fn packet_count(&self) -> usize; + + /// When the first packet still in the queue was enqueued. + fn first_unsent(&self) -> Option; +} + +impl StreamTxQueueInfo for QueueSnapshot { + fn created_at(&self) -> Instant { + self.created_at + } + + fn byte_size(&self) -> usize { + self.byte_size + } + + fn packet_count(&self) -> usize { + self.packet_count as usize + } + + fn first_unsent(&self) -> Option { + self.first_unsent + } +} + +/// Outgoing encoded stream. +/// +/// A stream is a primary SSRC + optional RTX SSRC. +/// +/// This is RTP level API. For frame level API see [`Rtc::writer`][crate::Rtc::writer]. +#[derive(Debug)] +pub struct StreamTx { + /// Unique identifier of the remote encoded stream. + ssrc: Ssrc, + + /// Identifier of a resend (RTX) stream. If we are doing resends. + rtx: Option, + + /// The Media mid and rid this stream belongs to. + midrid: MidRid, + + /// Set on first handle_timeout. + kind: Option, + + /// Set on first handle_timeout. + cname: Option, + + /// The last main payload clock rate that was sent. + clock_rate: Option, + + /// If we are doing seq_no ourselves (when writing frame mode). + seq_no: SeqNo, + + /// If we are using RTX, this is the seq no counter. + seq_no_rtx: SeqNo, + + /// The last seq_no that we sent, either by increasing seq_no ourselves (media API), or by + /// direct RTP mode writing. + last_sent_seq_no: SeqNo, + + /// When we last sent something for this encoded stream, packet or RTCP. + last_used: Instant, + + /// Last written media + wallclock time. + rtp_and_wallclock: Option<(u32, Instant)>, + + /// Queue of packets to send. + /// + /// The packets here do not have correct sequence numbers, header extension values etc. + /// They must be updated when we are about to send. + send_queue: SendQueue, + + /// The queue information sampled during the latest pacer update. + queue_info: Option, + + /// Whether this sender is to be unpaced in BWE situations. + /// + /// Audio defaults to not being paced. + unpaced: Option, + + /// Scheduled resends due to NACK or spurious padding. + resends: VecDeque, + + /// Requested padding, that has not been turned into packets yet. + padding: usize, + + /// Dummy packet for resends. Used between poll_packet and poll_packet_padding + blank_packet: RtpPacket, + + /// Cache of sent packets to be able to answer to NACKs as well as + /// sending spurious resends as padding. + rtx_cache: RtxCache, + + /// Determines retransmitted bytes ratio value to clear queued resends. + rtx_ratio_cap: Option, + + /// Last time we produced a SR. + last_sender_report: Instant, + + /// If we have a pending incoming keyframe request. + pending_request_keyframe: Option, + + /// If we have a pending incoming remb request. + pending_request_remb: Option, + + /// Statistics of outgoing data. + /// + /// Stats are use to calculate the rtx ratio also when statistics events are disabled. + stats: StreamTxStats, + + // downsampled rtx ratio (value, last calculation) + rtx_ratio: (f32, Instant), + + // The _main_ PT to use for padding. This is main PT, since the poll_packet() loop + // figures out the param.resend() RTX PT using main. + pt_for_padding: Option, + + /// Whether a receiver report has been received for this SSRC, thus acknowledging + /// that the receiver has bound the Mid/Rid tuple to the SSRC and no longer + /// needs to be sent on every packet + remote_acked_ssrc: bool, + + // Same as `remote_acked_ssrc`, but for the RTX SSRC + remote_acked_rtx_ssrc: bool, + + /// MTU warn threshold; used to cap spurious-padding RTX cache lookups. + mtu_warn: usize, +} + +/// RTP packet data to enqueue on a direct RTP send stream. +/// +/// The payload is the RTP payload only, without the RTP header. +/// +/// Optional RTP fields default to the common unmarked non-nackable packet with +/// no header extension values, no CSRC entries and no VP8 rewrite. +#[derive(Debug)] +pub struct RtpWrite { + pt: Pt, + seq_no: SeqNo, + time: u32, + wallclock: Instant, + marker: bool, + ext_vals: ExtensionValues, + nackable: bool, + payload: Arc<[u8]>, + csrc_count: usize, + csrc: [u32; 15], + vp8_patch: Option, +} + +impl RtpWrite { + /// Create a direct RTP write command. + /// + /// The `payload` argument is expected to be only the RTP payload, not the + /// RTP packet header. + /// + /// * `pt` Payload type. Declared in the [`Media`][crate::media::Media] this + /// encoded stream belongs to. + /// * `seq_no` Sequence number to use for this packet. + /// * `time` Time in whatever the clock rate is for the media in question + /// (normally 90_000 for video and 48_000 for audio). + /// * `wallclock` Real world time that corresponds to the media time in the + /// RTP packet. For an SFU, this can be hard to know because + /// RTP packets typically only contain the media time (RTP + /// time). In the simplest SFU setup, the wallclock could + /// simply be the arrival time of the incoming RTP data. For + /// better synchronization the SFU probably needs to weigh in + /// clock drifts and data provided via the statistics, receiver + /// reports etc. + /// * `payload` RTP packet payload, without header. + /// + /// Optional fields default to an unmarked packet with no header extension + /// values, no CSRC entries, no VP8 patch and no NACK support. + /// [`RtpWrite::marker`] for video frame boundaries, + /// [`RtpWrite::ext_vals`] for RTP header extension values, + /// [`RtpWrite::nackable`] for packets that may answer incoming NACKs, + /// [`RtpWrite::csrc`] for contributing source identifiers, + /// [`RtpWrite::vp8_patch`] to apply a prevalidated VP8 payload descriptor patch during RTP serialization. + pub fn new( + pt: Pt, + seq_no: SeqNo, + time: u32, + wallclock: Instant, + payload: impl Into>, + ) -> Self { + Self { + pt, + seq_no, + time, + wallclock, + marker: false, + ext_vals: ExtensionValues::default(), + nackable: false, + payload: payload.into(), + csrc_count: 0, + csrc: [0; 15], + vp8_patch: None, + } + } + + /// Set the RTP marker bit. + pub fn marker(mut self, marker: bool) -> Self { + self.marker = marker; + self + } + + /// Set RTP header extension values. + pub fn ext_vals(mut self, ext_vals: ExtensionValues) -> Self { + self.ext_vals = ext_vals; + self + } + + /// Set whether this RTP packet should respond to incoming NACKs. + pub fn nackable(mut self, nackable: bool) -> Self { + self.nackable = nackable; + self + } + + /// Set contributing source identifiers. + /// + /// # Panics + /// + /// Panics if more than 15 entries are supplied. + pub fn csrc(mut self, csrc: &[u32]) -> Self { + assert!(csrc.len() <= 15, "CSRC count must be <= 15"); + + self.csrc = [0; 15]; + self.csrc[..csrc.len()].copy_from_slice(csrc); + self.csrc_count = csrc.len(); + self + } + + /// Apply a prevalidated VP8 payload descriptor patch during RTP serialization. + /// + /// Build the patch with [`Vp8Descriptor::patch`][crate::rtp::Vp8Descriptor::patch] + /// when the caller already parsed the VP8 payload descriptor before + /// constructing this write command. + pub fn vp8_patch(mut self, patch: Vp8Patch) -> Self { + self.vp8_patch = Some(patch); + self + } +} + +impl StreamTx { + pub(crate) fn new( + ssrc: Ssrc, + rtx: Option, + midrid: MidRid, + enable_stats: bool, + mtu_warn: usize, + ) -> Self { + debug!("Create StreamTx for SSRC: {}", ssrc); + + StreamTx { + ssrc, + rtx, + midrid, + kind: None, + cname: None, + clock_rate: None, + seq_no: SeqNo::default(), + seq_no_rtx: SeqNo::default(), + last_sent_seq_no: SeqNo::default(), + last_used: already_happened(), + rtp_and_wallclock: None, + send_queue: SendQueue::new(), + queue_info: None, + unpaced: None, + resends: VecDeque::new(), + padding: 0, + blank_packet: RtpPacket::blank(), + rtx_cache: RtxCache::new(2000, DEFAULT_RTX_CACHE_DURATION), + rtx_ratio_cap: DEFAULT_RTX_RATIO_CAP, + last_sender_report: already_happened(), + pending_request_keyframe: None, + pending_request_remb: None, + stats: StreamTxStats::new(enable_stats), + rtx_ratio: (0.0, already_happened()), + pt_for_padding: None, + remote_acked_ssrc: false, + remote_acked_rtx_ssrc: false, + mtu_warn, + } + } + + /// The (primary) SSRC of this encoded stream. + pub fn ssrc(&self) -> Ssrc { + self.ssrc + } + + /// The resend (RTX) SSRC of this encoded stream. + pub fn rtx(&self) -> Option { + self.rtx + } + + /// Mid for this stream. + /// + /// In SDP this corresponds to m-line and "Media". + pub fn mid(&self) -> Mid { + self.midrid.mid() + } + + /// Rid for this stream. + /// + /// This is used to separate streams with the same [`Mid`] when using simulcast. + pub fn rid(&self) -> Option { + self.midrid.rid() + } + + /// Information sampled during the latest pacer update for this stream's send queue. + /// + /// This is `None` until the pacer has observed the stream for the first time. + /// Calling this method does not refresh the information; use + /// [`StreamTxQueueInfo::created_at()`] to determine whether a new sample is available. + pub fn queue_info(&self) -> Option<&dyn StreamTxQueueInfo> { + self.queue_info + .as_ref() + .map(|info| info as &dyn StreamTxQueueInfo) + } + + /// Configure the RTX (resend) cache. + /// + /// This determines how old incoming NACKs we can reply to. + /// + /// `rtx_ratio_cap` determines when to clear queued resends because of too many resends, + /// i.e. if `tx_sum / (rtx_sum + tx_sum) > rtx_ratio_cap`. `None` disables this functionality + /// so all queued resends will be sent. + /// + /// The default is 1024 packets over 3 seconds and RTX cache drop ratio of 0.15. + pub fn set_rtx_cache( + &mut self, + max_packets: usize, + max_age: Duration, + rtx_ratio_cap: Option, + ) { + // Dump old cache to avoid having to deal with resizing logic inside the cache impl. + self.rtx_cache = RtxCache::new(max_packets, max_age); + if rtx_ratio_cap.is_some() { + self.stats + .bytes_transmitted + .get_or_insert_with(ValueHistory::default); + self.stats + .bytes_retransmitted + .get_or_insert_with(ValueHistory::default); + } else { + self.stats.bytes_transmitted = None; + self.stats.bytes_retransmitted = None; + } + self.rtx_ratio_cap = rtx_ratio_cap; + } + + /// Set whether this stream is unpaced or not. + /// + /// This is only relevant when BWE (Bandwidth Estimation) is enabled. By default, audio is unpaced + /// thus not held to a steady send rate by the Pacer. + /// + /// This overrides the default behavior. + pub fn set_unpaced(&mut self, unpaced: bool) { + self.unpaced = Some(unpaced); + } + + /// Write RTP packet to a send stream. + /// + /// The write command carries the RTP payload and optional RTP metadata. + pub fn write_rtp(&mut self, rtp: RtpWrite) { + let RtpWrite { + pt, + seq_no, + time, + wallclock, + marker, + ext_vals, + nackable, + payload, + csrc_count, + csrc, + vp8_patch, + } = rtp; + + let first_call = self.rtp_and_wallclock.is_none(); + + if first_call && seq_no.roc() > 0 { + // TODO: make it possible to supress this. + warn!( + "First SeqNo has non-zero ROC ({}), which needs out-of-band signalling \ + to remote peer", + seq_no.roc() + ); + } + + // This 1 in clock frequency will be fixed in poll_output. + let media_time = MediaTime::from_secs(time as u64); + self.rtp_and_wallclock = Some((time, wallclock)); + + let header = RtpHeader { + csrc_count, + sequence_number: *seq_no as u16, + marker, + payload_type: pt, + timestamp: time, + ssrc: self.ssrc, + csrc, + ext_vals, + ..Default::default() + }; + + let packet = RtpPacket { + seq_no, + time: media_time, + header, + payload, + vp8_patch, + nackable, + // The overall idea for str0m is to only drive time forward from handle_input. If we + // used a "now" argument to write_rtp(), we effectively get a second point that also need + // to move time forward _for all of Rtc_ – that's too complicated. + // + // Instead we set a future timestamp here. When time moves forward in the "regular way", + // in handle_timeout() we delegate to self.send_queue.handle_timeout() to mark the enqueued + // timestamp of all packets that are about to be sent. + timestamp: not_happening(), + + // This is only relevant for incoming RTP packets. + last_sender_info: None, + }; + + self.send_queue.push(packet); + } + + fn padding_enabled(&self) -> bool { + self.rtx.is_some() && self.pt_for_padding.is_some() + } + + pub(crate) fn poll_packet( + &mut self, + now: Instant, + exts: &ExtensionMap, + twcc: Option<&mut u64>, + params: &[PayloadParams], + buf: &mut Vec, + ) -> Option { + let mid = self.midrid.mid(); + let rid = self.midrid.rid(); + let ssrc_rtx = self.rtx; + let remote_acked_ssrc = self.remote_acked_ssrc; + let remote_acked_rtx_ssrc = self.remote_acked_rtx_ssrc; + + let (next, is_padding) = if let Some(next) = self.poll_packet_resend(now) { + (next, false) + } else if let Some(next) = self.poll_packet_regular(now) { + (next, false) + } else { + let next = self.poll_packet_padding(now)?; + (next, true) + }; + + let pop_send_queue = next.kind == NextPacketKind::Regular; + + // Need the header for the receipt and modifications + // TODO: Can we remove this? + let header_ref = &mut next.pkt.header; + + // + // BUNDLE requires that the receiver "bind" the received SSRC to the values + // in the MID and/or (R)RID header extensions if present. Therefore, the + // sender can reduce overhead by omitting these header extensions once it + // knows that the receiver has "bound" the SSRC. + // + // The algorithm here is fairly simple: Always attach a MID and/or RID (if + // configured) to the outgoing packets until an RTCP receiver report comes + // back for this SSRC. That feedback indicates the receiver must have + // received a packet with the SSRC and header extension(s), so the sender + // then stops attaching the MID and RID. + if !remote_acked_ssrc { + header_ref.ext_vals.mid = Some(mid); + header_ref.ext_vals.rid = rid; + } + + let pt_main = header_ref.payload_type; + + // The pt in next.pkt is the "main" pt. + let Some(param) = params.iter().find(|p| p.pt() == pt_main) else { + // PT does not exist in the connected media. + warn!("Media is missing PT ({}) used in RTP packet", pt_main); + + // Get rid of this packet we can't send. + if pop_send_queue { + self.send_queue.pop(now); + } + + return None; + }; + + let mut set_pt_for_padding = None; + let mut set_cr = None; + + let mut header = match next.kind { + NextPacketKind::Regular => { + let rtx_possible = param.resend().is_some(); + + if rtx_possible { + // Remember PT We want to set these directly on `self` here, but can't + // because we already have a mutable borrow. We are using pt_main + // since the above loop figuring out param needs to be correct also + // for the NextPacketKind::Blank case. + set_pt_for_padding = Some(pt_main); + } else { + // If the PT we're sending on doesn't have a corresponding RTX PT, + // the packet is de-facto not nackable. + // + // This blocks incoming NACK requests and thus ensures there are no + // entries in self.retries without a RTX PT. + next.pkt.nackable = false; + } + + let clock_rate = param.spec().rtp_clock_rate(); + set_cr = Some(clock_rate); + + // Modify the cached packet time. This is so write_rtp can use u32 media time without + // worrying about lengthening or the clock rate. + let time = MediaTime::new(next.pkt.time.numer(), clock_rate); + next.pkt.time = time; + + // Modify the original (and also cached) header value. + header_ref.ext_vals.rid_repair = None; + + header_ref.clone() + } + NextPacketKind::Resend(_) | NextPacketKind::Blank(_) => { + // * For the Resend case, we will not have accepted/cached the packet unless + // we have a RTX PT (see logic setting next.pkt.nackable above). + // * For the Blank case, we will only have produced blank packets if we + // got a "real" PTX RT, either via set_pt_for_padding above, or via + // the on_first_timeout() further down. + // Either way, unwrapping this optional _should_ be correct. + let pt_rtx = param.resend().expect("PT for resend or blank"); + + // Clone header to not change the original (cached) header. + let mut header = header_ref.clone(); + + // Update clone of header (to not change the cached value). + header.payload_type = pt_rtx; + header.ssrc = ssrc_rtx.expect("Should have RTX SSRC for resends"); + header.sequence_number = *next.seq_no as u16; + + header.ext_vals.rid = None; + header.ext_vals.rid_repair = rid; + + if !remote_acked_rtx_ssrc { + header.ext_vals.mid = Some(mid); + } + + header + } + }; + + // These need to match `Extension::is_supported()` so we are sending what we are + // declaring we support. + + // Absolute Send Time might not be enabled for this m-line. + if exts.id_of(Extension::AbsoluteSendTime).is_some() { + header.ext_vals.abs_send_time = Some(now); + } + + // TWCC might not be enabled for this m-line. + if let Some(twcc) = twcc { + header.ext_vals.transport_cc = Some(*twcc as u16); + *twcc += 1; + } + + buf.resize(DATAGRAM_MAX_PACKET_SIZE, 0); + + let header_len = header.write_to(buf, exts); + assert!(header_len % 4 == 0, "RTP header must be multiple of 4"); + header.header_len = header_len; + + let mut body_out = &mut buf[header_len..]; + + // For resends, the original seq_no is inserted before the payload. + let mut original_seq_len = 0; + if let NextPacketKind::Resend(orig_seq_no) = next.kind { + original_seq_len = RtpHeader::write_original_sequence_number(body_out, orig_seq_no); + body_out = &mut body_out[original_seq_len..]; + } + + let pkt = &next.pkt; + + let body_len = match next.kind { + NextPacketKind::Regular | NextPacketKind::Resend(_) => { + let body_len = pkt.payload.len(); + if let Some(patch) = pkt.vp8_patch.as_ref() { + patch.copy_to(pkt.payload.as_ref(), &mut body_out[..body_len]); + } else { + body_out[..body_len].copy_from_slice(pkt.payload.as_ref()); + } + + // media packets are sent unpadded: the SRTP ciphers handle + // arbitrary body lengths (the CTR scratch alignment is internal + // to protect_rtp), and RTP-level padding breaks SFUs that + // re-marshal forwarded packets without it while keeping the P + // bit (observed with LiveKit -> Chrome, str0m issue #1014) + body_len + original_seq_len + } + NextPacketKind::Blank(len) => { + let len = RtpHeader::create_padding_packet( + &mut buf[..], + header_len, + len, + SRTP_BLOCK_SIZE, + ); + + if len == 0 { + return None; + } + + len + } + }; + + buf.truncate(header_len + body_len); + + #[cfg(feature = "_internal_dont_use_log_stats")] + { + let queued_at = match next.kind { + NextPacketKind::Regular => Some(pkt.timestamp), + _ => { + // TODO: We don't have queued at stats for Resends or blank padding. + None + } + }; + + if let Some(delay) = queued_at.map(|i| now.duration_since(i)) { + crate::log_stat!("QUEUE_DELAY", header.ssrc, delay.as_secs_f64() * 1000.0); + } + } + + let seq_no = next.seq_no; + if next.kind == NextPacketKind::Regular { + self.last_sent_seq_no = seq_no; + } + + self.last_used = now; + + // Padding comes in two forms, "spurious resends" of sent packets where + // the remote side didn't ask for a resend. The other variant are blank + // packets, containing nothing but zeroes. Such packets must be sent from + // _some_ RTX PT. A good pick is the RTX for the PT last used to send + // regular media data. + // + // This is set here due to borrow checker. + if set_pt_for_padding.is_some() && self.pt_for_padding != set_pt_for_padding { + self.pt_for_padding = set_pt_for_padding; + } + + if set_cr.is_some() && self.clock_rate != set_cr { + self.clock_rate = set_cr; + } + + if pop_send_queue { + // poll_packet_regular leaves the packet in the head of the send_queue + let pkt = self + .send_queue + .pop(now) + .expect("head of send_queue to be there"); + if pkt.nackable { + self.rtx_cache.cache_sent_packet(pkt, now); + } + } + + Some(PacketReceipt { + header, + seq_no, + is_padding, + payload_size: body_len, + }) + } + + fn rtx_ratio_downsampled(&mut self, now: Instant) -> f32 { + assert!( + self.stats.bytes_transmitted.is_some(), + "rtx_ratio_cap must be enabled" + ); + assert!( + self.stats.bytes_retransmitted.is_some(), + "rtx_ratio_cap must be enabled" + ); + + let (value, ts) = self.rtx_ratio; + if now - ts < Duration::from_millis(50) { + // not worth re-evaluating, return the old value + return value; + } + + // bytes stats refer to the last second by default + self.stats + .bytes_transmitted + .as_mut() + .unwrap() + .purge_old(now); + self.stats + .bytes_retransmitted + .as_mut() + .unwrap() + .purge_old(now); + + let bytes_transmitted = self.stats.bytes_transmitted.as_mut().unwrap().sum(); + let bytes_retransmitted = self.stats.bytes_retransmitted.as_mut().unwrap().sum(); + let ratio = bytes_retransmitted as f32 / (bytes_retransmitted + bytes_transmitted) as f32; + let ratio = if ratio.is_finite() { ratio } else { 0_f32 }; + self.rtx_ratio = (ratio, now); + ratio + } + + fn poll_packet_resend(&mut self, now: Instant) -> Option> { + if let Some(ratio_cap) = self.rtx_ratio_cap { + let ratio = self.rtx_ratio_downsampled(now); + + // If we hit the cap, stop doing resends by clearing those we have queued. + if ratio > ratio_cap { + self.resends.clear(); + return None; + } + } + + let seq_no = loop { + let resend = self.resends.pop_front()?; + + let pkt = self.rtx_cache.get_cached_packet_by_seq_no(resend.seq_no); + + // The seq_no could simply be too old to exist in the buffer, in which + // case we will not do a resend. + let Some(pkt) = pkt else { + continue; + }; + + // Cached packets must be nackable. This is ensured before adding the + // entry to the self.rtx_cache. + assert!(pkt.nackable); + + break pkt.seq_no; + }; + + // Borrow checker gymnastics. + let pkt = self.rtx_cache.get_cached_packet_by_seq_no(seq_no).unwrap(); + + let len = pkt.payload.len() as u64; + self.stats.update_packet_counts(len, true); + if let Some(h) = &mut self.stats.bytes_retransmitted { + h.push(now, len); + } + + let seq_no = self.seq_no_rtx.inc(); + + let orig_seq_no = pkt.seq_no; + + Some(NextPacket { + kind: NextPacketKind::Resend(orig_seq_no), + seq_no, + pkt, + }) + } + + fn poll_packet_regular(&mut self, now: Instant) -> Option> { + // exit via ? here is ok since that means there is nothing to send. + // The packet remains in the head of the send queue until we + // finish poll_packet, at which point we move it to the cache. + let pkt = self.send_queue.peek()?; + + pkt.timestamp = now; + + let len = pkt.payload.len() as u64; + self.stats.update_packet_counts(len, false); + if let Some(h) = &mut self.stats.bytes_transmitted { + h.push(now, len) + } + + let seq_no = pkt.seq_no; + + Some(NextPacket { + kind: NextPacketKind::Regular, + seq_no, + pkt, + }) + } + + fn poll_packet_padding(&mut self, _now: Instant) -> Option { + if !self.padding_enabled() { + self.padding = 0; + return None; + } + + if self.padding == 0 { + return None; + } + + #[allow(clippy::unnecessary_operation)] + 'outer: { + if self.padding > MIN_SPURIOUS_PADDING_SIZE { + // Find a historic packet that is smaller than this max size. The max size + // is a headroom since we can accept slightly larger padding than asked for. + let max_size = (self.padding * 2).min(self.mtu_warn - MAX_RTP_OVERHEAD); + + let Some(pkt) = self.rtx_cache.get_cached_packet_smaller_than(max_size) else { + // Couldn't find spurious packet, try a blank packet instead. + break 'outer; + }; + + let orig_seq_no = pkt.seq_no; + let seq_no = self.seq_no_rtx.inc(); + + self.padding = self.padding.saturating_sub(pkt.payload.len()); + + return Some(NextPacket { + kind: NextPacketKind::Resend(orig_seq_no), + seq_no, + pkt, + }); + } + }; + + let seq_no = self.seq_no_rtx.inc(); + + let pkt = &mut self.blank_packet; + pkt.seq_no = seq_no; + // Unwrap here is correct because self.padding_enabled() above checks the we got the PT set. + pkt.header.payload_type = self.pt_for_padding.unwrap(); + + let len = self + .padding + .clamp(SRTP_BLOCK_SIZE, MAX_BLANK_PADDING_PAYLOAD_SIZE); + assert!(len <= 255); // should fit in a byte + + self.padding = self.padding.saturating_sub(len); + + Some(NextPacket { + kind: NextPacketKind::Blank(len as u8), + seq_no, + pkt, + }) + } + + fn is_audio(&self) -> bool { + self.kind.is_some_and(|kind| kind.is_audio()) + } + + pub(crate) fn sender_report_at(&self, intervals: RtcpReportIntervals) -> Instant { + if self.kind.is_none() { + // First handle_timeout sets the kind. No sender report until then. + return not_happening(); + } + self.last_sender_report + intervals.for_audio(self.is_audio()) + } + + pub(crate) fn poll_keyframe_request(&mut self) -> Option { + self.pending_request_keyframe.take() + } + + pub(crate) fn poll_remb_request(&mut self) -> Option { + self.pending_request_remb.take() + } + + pub(crate) fn handle_rtcp(&mut self, now: Instant, fb: RtcpFb) { + use RtcpFb::*; + match fb { + ReceptionReport(r) => { + if let Some(rtx_ssrc) = self.rtx { + if rtx_ssrc == r.ssrc { + // Receiver has bound MidRid to RTX SSRC + self.remote_acked_rtx_ssrc = true; + } + } else if r.ssrc == self.ssrc { + // Receiver has bound MidRid to SSRC + self.remote_acked_ssrc = true; + } + + self.stats.update_with_rr(now, self.last_sent_seq_no, r) + } + Nack(_, list) => { + self.stats.increase_nacks(); + let entries = list.into_iter(); + self.handle_nack(entries, now); + } + Pli(_) => { + self.stats.increase_plis(); + self.pending_request_keyframe = Some(KeyframeRequestKind::Pli); + } + Fir(_) => { + self.stats.increase_firs(); + self.pending_request_keyframe = Some(KeyframeRequestKind::Fir); + } + Remb(r) => { + self.pending_request_remb = Some(Bitrate::from(r.bitrate as f64)); + } + Twcc(_) => unreachable!("TWCC should be handled on session level"), + _ => {} + } + } + + pub(crate) fn handle_nack( + &mut self, + entries: impl Iterator, + now: Instant, + ) -> Option<()> { + // Turning NackEntry into SeqNo we need to know a SeqNo "close by" to lengthen the 16 bit + // sequence number into the 64 bit we have in SeqNo. + let seq_no = self.rtx_cache.last_cached_seq_no()?; + let iter = entries.flat_map(|n| n.into_iter(seq_no)); + + // Schedule all resends. They will be handled on next poll_packet + for seq_no in iter { + let Some(packet) = self.rtx_cache.get_cached_packet_by_seq_no(seq_no) else { + // Packet was not available in RTX cache, it has probably expired. + continue; + }; + + let resend = Resend { + seq_no, + queued_at: now, + payload_size: packet.payload.len(), + }; + self.resends.push_back(resend); + } + + Some(()) + } + + pub(crate) fn need_sr(&self, now: Instant, intervals: RtcpReportIntervals) -> bool { + now >= self.sender_report_at(intervals) + } + + pub(crate) fn create_sr_and_update(&mut self, now: Instant, feedback: &mut VecDeque) { + let sr = self.create_sender_report(now); + + trace!("Created feedback SR: {:?}", sr); + feedback.push_back(Rtcp::SenderReport(sr)); + + if let Some(ds) = self.create_sdes() { + feedback.push_back(Rtcp::SourceDescription(ds)); + } + + // Update timestamp to move time when next is created. + self.last_sender_report = now; + } + + fn create_sender_report(&self, now: Instant) -> SenderReport { + SenderReport { + sender_info: self.sender_info(now), + reports: ReportList::new(), + } + } + + fn create_sdes(&self) -> Option { + // CNAME is set on first handle_timeout. No SDES before that. + let cname = self.cname.as_ref()?; + let mut s = Sdes { + ssrc: self.ssrc, + values: ReportList::new(), + }; + s.values.push((SdesType::CNAME, cname.to_string())); + + let mut d = Descriptions { + reports: Box::new(ReportList::new()), + }; + d.reports.push(s); + + Some(d) + } + + fn sender_info(&self, now: Instant) -> SenderInfo { + let rtp_time = self.current_rtp_time(now).unwrap_or(MediaTime::ZERO); + + SenderInfo { + ssrc: self.ssrc, + ntp_time: now.to_system_time(), + rtp_time, + sender_packet_count: self.stats.packets as u32, + sender_octet_count: self.stats.bytes as u32, + } + } + + fn current_rtp_time(&self, now: Instant) -> Option { + // This is the RTP time and the wallclock from the last written media. + // We use that as an offset to current time (now), to calculate the + // current RTP time. + let (t_u32, w) = self.rtp_and_wallclock?; + + let clock_rate = self.clock_rate?; + let t = MediaTime::new(t_u32 as u64, clock_rate); + + // Wallclock needs to be in the past. + if w > now { + let delta = w - now; + debug!("write_rtp wallclock is in the future: {:?}", delta); + return None; + } + let offset = now - w; + + // This might be in the wrong base. + let rtp_time = t + offset.into(); + + Some(rtp_time.rebase(clock_rate)) + } + + pub(crate) fn next_seq_no(&mut self) -> SeqNo { + self.seq_no.inc() + } + + pub(crate) fn last_packet(&self) -> Option<&[u8]> { + if self.send_queue.is_empty() { + self.rtx_cache.last_packet() + } else { + self.send_queue.last().map(|q| q.payload.as_ref()) + } + } + + pub(crate) fn visit_stats(&mut self, snapshot: &mut StatsSnapshot, now: Instant) { + self.stats.fill(snapshot, self.midrid, now); + } + + pub(crate) fn queue_state(&mut self, now: Instant) -> QueueState { + // The unpaced flag is set to a default value on first handle_timeout. The + // default is to not pace audio. We unwrap default to "true" here to not + // apply any pacing until we know what kind of content we are sending. + let unpaced = self.unpaced.unwrap_or(true); + + // It's only possible to use this sender for padding if RTX is enabled and + // we know a PT to use for it. + let use_for_padding = self.padding_enabled(); + + let mut snapshot = self.send_queue.snapshot(now); + + if let Some(snapshot_resend) = self.queue_state_resend(now) { + snapshot.merge(&snapshot_resend); + } + + if let Some(snapshot_padding) = self.queue_state_padding(now) { + snapshot.merge(&snapshot_padding); + } + + self.queue_info = Some(snapshot); + + QueueState { + midrid: self.midrid, + unpaced, + use_for_padding, + snapshot, + } + } + + fn queue_state_resend(&self, now: Instant) -> Option { + if self.resends.is_empty() { + return None; + } + + // Outstanding resends + let mut snapshot = self + .resends + .iter() + .fold(QueueSnapshot::default(), |mut snapshot, r| { + snapshot.total_queue_time_origin += now.duration_since(r.queued_at); + snapshot.byte_size += r.payload_size; + snapshot.packet_count += 1; + snapshot.first_unsent = snapshot + .first_unsent + .map(|i| i.min(r.queued_at)) + .or(Some(r.queued_at)); + + snapshot + }); + snapshot.created_at = now; + snapshot.update_priority(QueuePriority::Media); + + Some(snapshot) + } + + fn queue_state_padding(&self, now: Instant) -> Option { + if self.padding == 0 { + return None; + } + + // TODO: Be more scientific about these factors. + const AVERAGE_PADDING_PACKET_SIZE: usize = 800; + const FAKE_PADDING_DURATION_MILLIS: usize = 5; + + let fake_packets = self.padding.div_ceil(AVERAGE_PADDING_PACKET_SIZE); + let fake_millis = fake_packets * FAKE_PADDING_DURATION_MILLIS; + let fake_duration = Duration::from_millis(fake_millis as u64); + + Some(QueueSnapshot { + created_at: now, + byte_size: self.padding, + packet_count: fake_packets as u32, + total_queue_time_origin: fake_duration, + priority: QueuePriority::Padding, + ..Default::default() + }) + } + + pub(crate) fn generate_padding(&mut self, padding: usize) { + if !self.padding_enabled() { + return; + } + self.padding += padding; + } + + pub(crate) fn need_timeout(&self) -> bool { + self.send_queue.need_timeout() + } + + pub(crate) fn handle_timeout<'a>( + &mut self, + now: Instant, + get_media: impl FnOnce() -> (&'a Media, &'a CodecConfig), + ) { + // If kind is None, this is the first time we ever get a handle_timeout. + if self.kind.is_none() { + let (media, config) = get_media(); + self.on_first_timeout(media, config); + } + + self.send_queue.handle_timeout(now); + } + + fn on_first_timeout(&mut self, media: &Media, config: &CodecConfig) { + // Always set on first timeout. + self.kind = Some(media.kind()); + self.cname = Some(media.cname().to_string()); + + // Set on first timeout, if not set already by configuration. + if self.unpaced.is_none() { + // Default audio to be unpaced. + self.unpaced = Some(media.kind().is_audio()); + } + + // To allow for sending padding on a newly created StreamTx, before any regular + // packet has been sent, we need any main PT that has associated RTX. This is + // later be overwritten when we send the first regular packet. + if self.rtx.is_some() && self.pt_for_padding.is_none() { + if let Some(pt) = media.first_pt_with_rtx(config) { + trace!( + "StreamTx {:?} PT {} before first regular packet", + self.midrid, pt + ); + self.pt_for_padding = Some(pt); + + // Setting the pt_for_rtx should enable RTX. + assert!(self.padding_enabled()); + } + } + } + + pub(crate) fn reset_buffers(&mut self) { + self.send_queue.clear(); + self.queue_info = None; + self.rtx_cache.clear(); + self.resends.clear(); + self.padding = 0; + } + + /// Reset this stream to use a new SSRC and optionally a new RTX SSRC. + /// + /// This updates the SSRCs and resets all relevant internal fields. + pub(crate) fn reset_ssrc(&mut self, new_ssrc: Ssrc, new_rtx: Option) { + // Update the SSRC and RTX + self.ssrc = new_ssrc; + self.rtx = new_rtx; + + // Reset sequence numbers + self.seq_no = SeqNo::default(); + self.seq_no_rtx = SeqNo::default(); + + // Reset timing related fields + self.last_used = already_happened(); + self.rtp_and_wallclock = None; + self.last_sender_report = already_happened(); + + // Reset blank packet's SSRC + self.blank_packet.header.ssrc = new_ssrc; + + // Clear any pending requests + self.pending_request_keyframe = None; + self.pending_request_remb = None; + + // Reset all statistics - preserve whether stats tracking is enabled + let stats_enabled = self.stats.bytes_transmitted.is_some(); + self.stats = StreamTxStats::new(stats_enabled); + self.rtx_ratio = (0.0, already_happened()); + self.remote_acked_ssrc = false; + + // Clear all buffers + self.reset_buffers(); + } + + pub(crate) fn is_midrid(&self, midrid: MidRid) -> bool { + midrid.special_equals(&self.midrid) + } +} + +struct NextPacket<'a> { + kind: NextPacketKind, + seq_no: SeqNo, + pkt: &'a mut RtpPacket, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NextPacketKind { + Regular, + Resend(SeqNo), + Blank(u8), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Resend { + seq_no: SeqNo, + queued_at: Instant, + payload_size: usize, +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn queue_info_is_cached_on_queue_state_update() { + let mut stream = StreamTx::new(42.into(), None, MidRid(Mid::from("0"), None), false, 1200); + + assert!(stream.queue_info().is_none()); + + let now = Instant::now(); + let first_unsent = now - Duration::from_millis(10); + stream.resends.push_back(Resend { + seq_no: 7.into(), + queued_at: first_unsent, + payload_size: 123, + }); + stream.queue_state(now); + + let info = stream.queue_info().expect("queue info after pacer update"); + assert_eq!(info.created_at(), now); + assert_eq!(info.byte_size(), 123); + assert_eq!(info.packet_count(), 1); + assert_eq!(info.first_unsent(), Some(first_unsent)); + + stream.reset_buffers(); + assert!(stream.queue_info().is_none()); + } + + #[test] + fn sub_average_padding_is_visible_to_the_pacer() { + let mut stream = StreamTx::new(42.into(), None, MidRid(Mid::from("0"), None), false, 1200); + stream.padding = 240; + + let snapshot = stream + .queue_state_padding(Instant::now()) + .expect("nonzero padding must produce a queue snapshot"); + + assert_eq!(snapshot.byte_size, 240); + assert_eq!(snapshot.packet_count, 1); + } + + #[test] + fn sender_report_uses_supplied_intervals() { + let mut stream = StreamTx::new(42.into(), None, MidRid(Mid::from("0"), None), false, 1200); + let now = Instant::now(); + stream.kind = Some(MediaKind::Audio); + stream.last_sender_report = now; + + assert_eq!( + stream.sender_report_at(RtcpReportIntervals { + audio: Duration::from_millis(750), + video: Duration::from_millis(500), + }), + now + Duration::from_millis(750) + ); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_queue.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_queue.rs new file mode 100644 index 000000000..94fb06caf --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_queue.rs @@ -0,0 +1,346 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use crate::pacer::{QueuePriority, QueueSnapshot}; +use crate::util::not_happening; + +use super::RtpPacket; + +#[derive(Debug)] +pub(crate) struct SendQueue { + queue: VecDeque, + total: TotalQueue, + last_emitted: Option, +} + +impl SendQueue { + pub fn new() -> Self { + Self { + queue: VecDeque::new(), + total: TotalQueue::default(), + last_emitted: None, + } + } + + pub fn push(&mut self, mut packet: RtpPacket) { + // Every incoming packet must be timestamped withe a handle_timeout. + // This sentinel value indicates it is needed. + packet.timestamp = not_happening(); + + self.queue.push_back(packet); + } + + pub fn handle_timeout(&mut self, now: Instant) { + for pkt in self.queue.iter_mut().rev() { + if pkt.timestamp != not_happening() { + // all enqueued packets are timestamped. + break; + } else { + pkt.timestamp = now; + self.total.increase(now, pkt.payload.len()); + } + } + } + + pub fn need_timeout(&self) -> bool { + // Packets are timestamped contiguously from the front, + // so checking the tail summarizes the entire queue's state. + self.queue + .back() + .map(|p| p.timestamp == not_happening()) + .unwrap_or_default() + } + + pub fn peek(&mut self) -> Option<&mut RtpPacket> { + let peeked = self.queue.front_mut()?; + if peeked.timestamp == not_happening() { + None + } else { + Some(peeked) + } + } + + pub fn pop(&mut self, now: Instant) -> Option { + // Don't release packets without a timestamp. + self.peek()?; + + // Unwrap is OK, because peek() above must have returned a value + // for us to be here. + let packet = self.queue.pop_front().unwrap(); + + // Must be timestamped + assert!(packet.timestamp != not_happening()); + + let queue_time = now - packet.timestamp; + self.total.decrease(now, packet.payload.len(), queue_time); + self.last_emitted = Some(now); + + Some(packet) + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + pub fn last(&self) -> Option<&RtpPacket> { + self.queue.back() + } + + pub(crate) fn snapshot(&mut self, now: Instant) -> QueueSnapshot { + self.total.move_time_forward(now); + + QueueSnapshot { + created_at: now, + byte_size: self.total.unsent_size, + packet_count: self.total.unsent_count as u32, + total_queue_time_origin: self.total.queue_time, + last_emitted: self.last_emitted, + first_unsent: self + .queue + .iter() + .find(|p| p.timestamp != not_happening()) + .map(|p| p.timestamp), + priority: if self.total.unsent_count > 0 { + QueuePriority::Media + } else { + QueuePriority::Empty + }, + } + } + + pub(crate) fn clear(&mut self) { + self.queue.clear(); + self.total.clear(); + self.last_emitted = None; + } +} + +// Total queue time in buffer. This lovely drawing explains how to add more time. +// +// -time---------------------------------------------------------> +// +// +--------------+ +// | | +// +--------------+ +// +---------+ Already +// | | queued +// +---------+ durations +// +-----+ +// | | +// +-----+ +// +-+ +// | | <----- Add next +// +-+ packet +// +// +// +// +--------------+--------+ +// | |@@@@@@@@| +// +--------------+--------+ +// +---------+--------+ The @ is +// | |@@@@@@@@| what's +// +---------+--------+ added +// +-----+--------+ +// | |@@@@@@@@| +// +-----+--------+ +// +-+ +// |@| +// +-+ +#[derive(Debug, Default)] +struct TotalQueue { + /// Number of unsent packets. + unsent_count: usize, + /// The data size (bytes) of the unsent packets. + unsent_size: usize, + // /// When we last added some value to `queue_time`. + // last: Option, + /// The total queue time of all the unsent packets. + queue_time: Duration, + // Timestamp of the last added packet. + last: Option, +} + +impl TotalQueue { + fn move_time_forward(&mut self, now: Instant) { + if let Some(last) = self.last { + assert!(self.unsent_count > 0); + let from_last = now - last; + self.queue_time += from_last * (self.unsent_count as u32); + self.last = Some(now); + } else { + assert!(self.unsent_count == 0); + assert!(self.unsent_size == 0); + assert!(self.queue_time == Duration::ZERO); + } + } + + fn increase(&mut self, now: Instant, size: usize) { + self.move_time_forward(now); + self.unsent_count += 1; + self.unsent_size += size; + self.last = Some(now); + } + + fn decrease(&mut self, now: Instant, size: usize, queue_time: Duration) { + self.move_time_forward(now); + + self.unsent_count -= 1; + self.unsent_size -= size; + + self.queue_time -= queue_time; + + if self.unsent_count == 0 { + assert!(self.unsent_size == 0); + self.queue_time = Duration::ZERO; + self.last = None; + } + } + + fn clear(&mut self) { + *self = Self::default(); + } +} + +#[cfg(test)] +mod test { + use crate::rtp_::MediaTime; + use crate::rtp_::RtpHeader; + + use super::*; + + #[test] + fn peek_pop_no_timestamp() { + let mut queue = SendQueue::new(); + + queue.push(RtpPacket { + seq_no: 0.into(), + time: MediaTime::from_90khz(10), + header: RtpHeader::default(), + payload: [].into(), + vp8_patch: None, + timestamp: Instant::now(), + last_sender_info: None, + nackable: true, + }); + + assert!(queue.peek().is_none()); + assert!(queue.pop(Instant::now()).is_none()); + assert!(queue.need_timeout()); + + let snapshot_at = Instant::now() + Duration::from_secs(3); + assert_eq!( + queue.snapshot(snapshot_at), + QueueSnapshot { + created_at: snapshot_at, + packet_count: 0, + byte_size: 0, + total_queue_time_origin: Duration::ZERO, + first_unsent: None, + priority: QueuePriority::Empty, + ..Default::default() + } + ); + } + + #[test] + fn peek_pop_after_timestamp() { + let mut queue = SendQueue::new(); + + let start = Instant::now(); + + queue.push(RtpPacket { + seq_no: 0.into(), + time: MediaTime::from_90khz(10), + header: RtpHeader::default(), + payload: [42, 42].into(), + vp8_patch: None, + timestamp: start, + last_sender_info: None, + nackable: true, + }); + + queue.handle_timeout(start); + + assert!(queue.peek().is_some()); + assert!(!queue.need_timeout()); + + let snapshot_at = start + Duration::from_secs(3); + assert_eq!( + queue.snapshot(snapshot_at), + QueueSnapshot { + created_at: snapshot_at, + packet_count: 1, + byte_size: 2, + total_queue_time_origin: Duration::from_secs(3), + first_unsent: Some(start), + priority: QueuePriority::Media, + ..Default::default() + } + ); + + assert!(queue.pop(Instant::now()).is_some()); + } + + #[test] + fn untimed_packets_are_contiguous_at_tail() { + let mut queue = SendQueue::new(); + let start = Instant::now(); + + queue.push(RtpPacket { + seq_no: 0.into(), + time: MediaTime::from_90khz(10), + header: RtpHeader::default(), + payload: [0; 10].into(), + vp8_patch: None, + timestamp: start, + last_sender_info: None, + nackable: true, + }); + queue.push(RtpPacket { + seq_no: 1.into(), + time: MediaTime::from_90khz(20), + header: RtpHeader::default(), + payload: [1; 10].into(), + vp8_patch: None, + timestamp: start, + last_sender_info: None, + nackable: true, + }); + queue.push(RtpPacket { + seq_no: 2.into(), + time: MediaTime::from_90khz(20), + header: RtpHeader::default(), + payload: [1; 10].into(), + vp8_patch: None, + timestamp: start, + last_sender_info: None, + nackable: true, + }); + + assert!( + queue.queue.iter().all(|q| q.timestamp == not_happening()), + "expect every new packet's timestamp to start with the sentinel value" + ); + assert!(queue.need_timeout()); + + let now = start + Duration::from_millis(10); + queue.handle_timeout(now); + + assert!( + queue.queue.iter().all(|q| q.timestamp != not_happening()), + "expect handle_timeout to have timestamped every packet" + ); + assert!(!queue.need_timeout()); + } + + #[test] + fn total_queue() { + let mut total_queue = TotalQueue::default(); + let now = Instant::now(); + total_queue.increase(now, 0); + total_queue.increase(now, 1); + total_queue.decrease(now, 1, Duration::ZERO); + // Doesn't panic + total_queue.move_time_forward(now + Duration::from_millis(1)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_stats.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_stats.rs new file mode 100644 index 000000000..c99b087e5 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/streams/send_stats.rs @@ -0,0 +1,190 @@ +use std::time::{Duration, Instant}; + +use crate::rtp::SeqNo; +use crate::rtp_::{ReceptionReport, extend_u32}; +use crate::stats::{MediaEgressStats, RemoteIngressStats, StatsSnapshot}; +use crate::util::value_history::ValueHistory; +use crate::util::{InstantExt, calculate_rtt}; + +use super::MidRid; + +/// Holder of stats. +#[derive(Debug)] +pub(crate) struct StreamTxStats { + /// count of bytes sent, including retransmissions + /// + pub bytes: u64, + /// count of retransmitted bytes alone + bytes_resent: u64, + /// count of packets sent, including retransmissions + /// + pub packets: u64, + /// count of retransmitted packets alone + packets_resent: u64, + /// count of FIR requests received + firs: u64, + /// count of PLI requests received + plis: u64, + /// count of NACKs received + nacks: u64, + /// round trip time + /// Can be null in case of missing or bad reports + rtt: Option, + /// losses collecter from RR (known packets, lost ratio) + losses: Losses, + /// The last reception report for the stream, if any. + /// + /// The SeqNo is the extended max_seq of the reception report, extended + /// using the last sent sequence number. + last_rr: Option<(SeqNo, ReceptionReport)>, + + /// `None` if `rtx_ratio_cap` is `None`. + pub bytes_transmitted: Option>, + + /// `None` if `rtx_ratio_cap` is `None`. + pub bytes_retransmitted: Option>, +} + +impl StreamTxStats { + pub fn new(enable_stats: bool) -> Self { + Self { + bytes: 0, + bytes_resent: 0, + packets: 0, + packets_resent: 0, + firs: 0, + plis: 0, + nacks: 0, + rtt: None, + losses: Losses::new(enable_stats), + last_rr: None, + bytes_transmitted: Some(Default::default()), + bytes_retransmitted: Some(Default::default()), + } + } + + pub fn update_packet_counts(&mut self, bytes: u64, is_resend: bool) { + self.packets += 1; + self.bytes += bytes; + if is_resend { + self.bytes_resent += bytes; + self.packets_resent += 1; + } + } + + pub fn increase_nacks(&mut self) { + self.nacks += 1; + } + + pub fn increase_plis(&mut self) { + self.plis += 1; + } + + pub fn increase_firs(&mut self) { + self.firs += 1; + } + + pub fn update_with_rr(&mut self, now: Instant, last_sent_seq_no: SeqNo, r: ReceptionReport) { + let ntp_time = now.to_ntp_duration(); + let rtt = calculate_rtt(ntp_time, r.last_sr_delay, r.last_sr_time); + self.rtt = rtt; + + // The last_sent_seq_no should be in the vicinity of the rr.max_seq. + let ext_seq = extend_u32(Some(*last_sent_seq_no), r.max_seq).into(); + + self.last_rr = Some((ext_seq, r)); + + self.losses + .push((*ext_seq, r.fraction_lost as f32 / u8::MAX as f32)); + } + + pub(crate) fn fill(&mut self, snapshot: &mut StatsSnapshot, midrid: MidRid, now: Instant) { + if self.bytes == 0 { + return; + } + + let loss = { + let mut value = 0_f32; + let mut total_weight = 0_u64; + + // average known RR losses weighted by their number of packets + for it in self.losses.iterator() { + let [prev, next] = it else { continue }; + let weight = next.0.saturating_sub(prev.0); + value += next.1 * weight as f32; + total_weight += weight; + } + + let result = value / total_weight as f32; + result.is_finite().then_some(result) + }; + + self.losses.clear_all_but_last(); + + snapshot.egress.insert( + midrid, + MediaEgressStats { + mid: midrid.mid(), + rid: midrid.rid(), + bytes: self.bytes, + packets: self.packets, + firs: self.firs, + plis: self.plis, + nacks: self.nacks, + rtt: self.rtt, + loss, + timestamp: now, + remote: self + .last_rr + .as_ref() + .map(|(seq_no, rr)| RemoteIngressStats { + jitter: rr.jitter, + maximum_sequence_number: *seq_no, + packets_lost: rr.packets_lost as u64, + }), + }, + ); + } +} + +/// Helper to avoid an unbounded vec if we are not enabling stats +#[derive(Debug)] +enum Losses { + Disabled, + Enabled(Vec<(u64, f32)>), +} + +impl Losses { + fn new(enabled: bool) -> Self { + if enabled { + Self::Enabled(vec![]) + } else { + Self::Disabled + } + } + + fn push(&mut self, value: (u64, f32)) { + let Self::Enabled(losses) = self else { + return; + }; + losses.push(value); + } + + fn iterator(&mut self) -> impl Iterator { + let Self::Enabled(losses) = self else { + return [].windows(2); + }; + + // just in case we received RRs out of order + losses.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + + losses.windows(2) + } + + fn clear_all_but_last(&mut self) { + let Self::Enabled(losses) = self else { + return; + }; + losses.drain(..losses.len().saturating_sub(1)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/util/average.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/average.rs new file mode 100644 index 000000000..f137b89d4 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/average.rs @@ -0,0 +1,80 @@ +/// Exponential moving average +#[derive(Debug)] +pub struct MovingAverage { + smoothing_factor: f64, + average: Option, + variance: f64, + std: f64, +} + +impl MovingAverage { + pub fn new(smoothing_factor: f64) -> Self { + Self { + smoothing_factor, + average: None, + variance: 0.0, + std: 0.0, + } + } + + pub fn within_std(&self, value: f64, num_std: f64) -> bool { + let Some(average) = self.average else { + return false; + }; + + let floor = average - self.std * num_std; + let ceil = average + self.std * num_std; + + floor <= value && value <= ceil + } + + pub fn upper_range(&self, num_std: f64) -> Option { + if self.std == 0.0 { + return None; + } + + self.average.map(|avg| avg + num_std * self.std) + } + + pub fn lower_range(&self, num_std: f64) -> Option { + if self.std == 0.0 { + return None; + } + + self.average.map(|avg| avg - num_std * self.std) + } + + pub fn update(&mut self, value: f64) { + let average = match self.average { + Some(average) => { + let delta = value - average; + let new_average = average + self.smoothing_factor * delta; + let new_variance = (1.0 - self.smoothing_factor) + * (self.variance + self.smoothing_factor * delta.powf(2.0)); + + self.variance = new_variance; + self.std = new_variance.sqrt(); + + new_average + } + None => value, + }; + + self.average = Some(average); + } + + /// Returns the current average value, or None if no values have been added yet. + pub fn get(&self) -> Option { + self.average + } + + pub fn valid(&self) -> bool { + self.average.is_some() + } + + pub fn reset(&mut self) { + self.average = None; + self.std = 0.0; + self.variance = 0.0; + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/util/bit_pattern.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/bit_pattern.rs new file mode 100644 index 000000000..c36b2b4f2 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/bit_pattern.rs @@ -0,0 +1,107 @@ +/// One byte bit pattern to allow matching against specific bits. +#[derive(Copy, Clone)] +pub(crate) struct BitPattern { + mask: u8, + masked_value: u8, +} + +impl BitPattern { + pub(crate) fn matches(&self, profile_iop: u8) -> bool { + ((profile_iop ^ self.masked_value) & self.mask) == 0x0 + } + + /// Create a bit pattern. Use `1` for bits that must be set, `0` for bits that must not be + /// set, and `x` for bits that can take any value. + /// + /// Panics if any other byte values than ASCII `1`, `0`, or `x` is used. + pub(crate) const fn new(pattern: [u8; 8]) -> Self { + const fn bit_to_mask_bit(pattern: [u8; 8], i: usize) -> u8 { + let bit = pattern[7 - i]; + match bit { + b'1' | b'0' => 0x1 << i, + b'x' => 0x0 << i, + _ => panic!("Invalid bit pattern in IOPPattern only ASCII 1, 0, and x are allowed"), + } + } + + const fn to_mask(pattern: [u8; 8]) -> u8 { + bit_to_mask_bit(pattern, 7) + | bit_to_mask_bit(pattern, 6) + | bit_to_mask_bit(pattern, 5) + | bit_to_mask_bit(pattern, 4) + | bit_to_mask_bit(pattern, 3) + | bit_to_mask_bit(pattern, 2) + | bit_to_mask_bit(pattern, 1) + | bit_to_mask_bit(pattern, 0) + } + + const fn bit_to_mask_value_bit(pattern: [u8; 8], i: usize) -> u8 { + let bit = pattern[7 - i]; + match bit { + b'1' => 0x1 << i, + b'x' | b'0' => 0x0 << i, + _ => panic!("Invalid bit pattern in IOPPattern only ASCII 1, 0, and x are allowed"), + } + } + + const fn to_mask_value(pattern: [u8; 8]) -> u8 { + bit_to_mask_value_bit(pattern, 7) + | bit_to_mask_value_bit(pattern, 6) + | bit_to_mask_value_bit(pattern, 5) + | bit_to_mask_value_bit(pattern, 4) + | bit_to_mask_value_bit(pattern, 3) + | bit_to_mask_value_bit(pattern, 2) + | bit_to_mask_value_bit(pattern, 1) + | bit_to_mask_value_bit(pattern, 0) + } + let mask = to_mask(pattern); + let masked_value = to_mask_value(pattern); + + Self { mask, masked_value } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_iop_pattern_matching() { + let all_any = BitPattern::new(*b"xxxxxxxx"); + for x in 0..255 { + assert!(all_any.matches(x)); + } + + let stripes = BitPattern::new(*b"10101010"); + assert!(stripes.matches(0b1010_1010)); + assert!(!stripes.matches(0b1011_1010)); + + let inverse_stripes = BitPattern::new(*b"01010101"); + assert!(inverse_stripes.matches(0b0101_0101)); + assert!(!inverse_stripes.matches(0b0111_0001)); + + let high_bits = BitPattern::new(*b"1101xxxx"); + assert!(high_bits.matches(0b1101_0101)); + assert!(!high_bits.matches(0b1001_0101)); + + let mid_bits = BitPattern::new(*b"xx0110xx"); + assert!(mid_bits.matches(0b0101_1001)); + assert!(!mid_bits.matches(0b1000_1001)); + + let only_ones = BitPattern::new(*b"11111111"); + assert!(only_ones.matches(0b1111_1111)); + assert!(!only_ones.matches(0b1110_1111)); + + let only_zeros = BitPattern::new(*b"00000000"); + assert!(only_zeros.matches(0b0000_0000)); + assert!(!only_zeros.matches(0b0000_0010)); + + let mixed_pattern = BitPattern::new(*b"1x0x1x01"); + assert!(mixed_pattern.matches(0b11011001)); + assert!(!mixed_pattern.matches(0b11011011)); + + let complex_pattern = BitPattern::new(*b"1xx01x0x"); + assert!(complex_pattern.matches(0b10001001)); + assert!(!complex_pattern.matches(0b10101010)); + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/util/mod.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/mod.rs new file mode 100644 index 000000000..884441093 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/mod.rs @@ -0,0 +1,87 @@ +use std::time::{Duration, Instant}; + +mod bit_pattern; + +pub(crate) use bit_pattern::BitPattern; + +pub(crate) mod value_history; + +mod time_tricks; +pub(crate) use time_tricks::{InstantExt, SystemTimeExt, not_happening}; +pub(crate) use time_tricks::{already_happened, epoch_to_beginning}; + +mod average; +pub(crate) use average::MovingAverage; + +pub(crate) trait Soonest { + fn soonest(self, other: Self) -> Self; +} + +impl Soonest for (Option, T) { + fn soonest(self, other: Self) -> Self { + match (self, other) { + ((Some(v1), s1), (Some(v2), s2)) => { + if v1 < v2 { + (Some(v1), s1) + } else { + (Some(v2), s2) + } + } + ((None, _), (None, _)) => (None, T::default()), + ((None, _), (v, s)) => (v, s), + ((v, s), (None, _)) => (v, s), + } + } +} + +/// Calculate the round trip time for a given peer as described in +/// [RFC3550 6.4.1](https://datatracker.ietf.org/doc/html/rfc3550#section-6.4.1). +/// +/// ## Params +/// - `ntp_time` the offset since 1900-01-01. +/// - `delay` the delay(`DLSR`) since last sender report expressed as fractions of a second in 32 bits. +/// - `last_report` the middle 32 bits of an NTP timestamp for the most recent sender report(LSR) +/// or Receiver Report(LRR). +pub(crate) fn calculate_rtt(ntp_time: Duration, delay: u32, last_report: u32) -> Option { + // [10 Nov 1995 11:33:25.125 UTC] [10 Nov 1995 11:33:36.5 UTC] + // n SR(n) A=b710:8000 (46864.500 s) + // ----------------------------------------------------------------> + // v ^ + // ntp_sec =0xb44db705 v ^ dlsr=0x0005:4000 ( 5.250s) + // ntp_frac=0x20000000 v ^ lsr =0xb705:2000 (46853.125s) + // (3024992005.125 s) v ^ + // r v ^ RR(n) + // ----------------------------------------------------------------> + // |<-DLSR->| + // (5.250 s) + // + // A 0xb710:8000 (46864.500 s) + // DLSR -0x0005:4000 ( 5.250 s) + // LSR -0xb705:2000 (46853.125 s) + // ------------------------------- + // delay 0x0006:2000 ( 6.125 s) + + // - we want the current middle 32 bits of an NTP timestamp for the current time. + // We treat the seconds separately to the fractions. + // [32 bit seconds].[32 bit fractions] + // [16 bit].[16 bit] + + // As per RFC delay is 0 in case no SR packet has been received yet. + if delay == 0 { + return None; + } + + let now_secs = ntp_time.as_secs(); + let now_fract_ns = ntp_time.subsec_nanos() as u64; + let now_fract = ((now_fract_ns * u32::MAX as u64) / 1_000_000_000) as u32; + + // Combine the final 2x16 bits together. + let now = (now_secs as u32) << 16 | (now_fract >> 16); + + let rtt = now.checked_sub(delay)?.checked_sub(last_report)?; + let rtt_seconds = rtt >> 16; + let rtt_fraction = (rtt & (u16::MAX as u32)) as f32 / (u16::MAX as u32) as f32; + + let nanos = (rtt_fraction * 1_000_000_000.0) as u32; + Some(Duration::new(rtt_seconds as u64, nanos)) +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/util/time_tricks.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/time_tricks.rs new file mode 100644 index 000000000..ac736491c --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/time_tricks.rs @@ -0,0 +1,210 @@ +use std::backtrace::Backtrace; +use std::sync::LazyLock; +use std::time::{Duration, Instant, SystemTime}; + +pub(crate) fn not_happening() -> Instant { + const YEARS_100: Duration = Duration::from_secs(60 * 60 * 24 * 365 * 100); + static FUTURE: LazyLock = LazyLock::new(|| Instant::now() + YEARS_100); + *FUTURE +} + +// The goal here is to make a constant "beginning of time" in both Instant and SystemTime +// that we can use as relative values for the rest of str0m. +// This is indeed a bit dodgy, but we want str0m's internal idea of time to be completely +// driven from the external API using `Instant`. What works against us is that Instant can't +// represent things like UNIX EPOCH (but SystemTime can). +static BEGINNING_OF_TIME: LazyLock<(Instant, SystemTime)> = LazyLock::new(|| { + // These two should be "frozen" the same instant. Hopefully they are not differing too much. + let now = Instant::now(); + let now_sys = SystemTime::now(); + + // Find an Instant in the past which is up to an hour back. + let beginning_of_time = { + let mut secs = 3600; + loop { + let dur = Duration::from_secs(secs); + if let Some(v) = now.checked_sub(dur) { + break v; + } + secs -= 1; + if secs == 0 { + panic!("Failed to find a beginning of time instant"); + } + } + }; + + // This might be less than 1 hour if the machine uptime is less. + let since_beginning_of_time = Instant::now() - beginning_of_time; + + let beginning_of_time_sys = now_sys - since_beginning_of_time; + + // This pair represents our "beginning of time" for the same moment. + (beginning_of_time, beginning_of_time_sys) +}); + +pub fn epoch_to_beginning() -> Duration { + BEGINNING_OF_TIME + .1 + .duration_since(SystemTime::UNIX_EPOCH) + .expect("beginning of time to be after epoch") +} + +pub(crate) fn already_happened() -> Instant { + BEGINNING_OF_TIME.0 +} + +pub trait InstantExt { + /// Convert an Instant to a Duration for unix time. + /// + /// First ever time must be "now". + /// + /// panics if `time` goes backwards, i.e. we use this for one Instant and then an earlier Instant. + fn to_unix_duration(&self) -> Duration; + + /// Convert an Instant to a Duration for ntp time. + fn to_ntp_duration(&self) -> Duration; + + /// Convert an Instant to a SystemTime for ntp time + fn to_system_time(&self) -> SystemTime; +} + +pub trait SystemTimeExt { + /// Convert an ntp_64 as seen in SR to a SystemTime. + fn from_ntp_64(v: u64) -> SystemTime; + + /// Convert a SystemTime to ntp_64. + fn as_ntp_64(&self) -> u64; +} + +// RTP spec "wallclock" uses NTP time, which starts at 1900-01-01. +// +// https://tools.ietf.org/html/rfc868 +// +// 365 days * 70 years + 17 leap year days +// (365 * 70 + 17) * 86400 = 2208988800 +const SECS_1900: u64 = 2_208_988_800; +const MICROS_1900: u64 = SECS_1900 * 1_000_000; + +/// 2^32 as float. +const F32: f64 = 4_294_967_296.0; + +impl InstantExt for Instant { + fn to_unix_duration(&self) -> Duration { + // This is a bit fishy. We "freeze" a moment in time for Instant and SystemTime, + // so we can make relative comparisons of Instant - Instant and translate that to + // SystemTime - unix epoch. Hopefully the error is quite small. + if *self < BEGINNING_OF_TIME.0 { + warn!("{}", Backtrace::force_capture()); + warn!("Time went backwards from beginning_of_time Instant"); + } + + let duration_since_time_0 = self.duration_since(BEGINNING_OF_TIME.0); + let system_time = BEGINNING_OF_TIME.1 + duration_since_time_0; + + system_time + .duration_since(SystemTime::UNIX_EPOCH) + .expect("clock to go forwards from unix epoch") + } + + fn to_ntp_duration(&self) -> Duration { + self.to_unix_duration() + Duration::from_micros(MICROS_1900) + } + + fn to_system_time(&self) -> SystemTime { + // This is a bit fishy. We "freeze" a moment in time for Instant and SystemTime, + // so we can make relative comparisons of Instant - Instant and translate that to + // SystemTime - unix epoch. Hopefully the error is quite small. + if *self < BEGINNING_OF_TIME.0 { + warn!("{}", Backtrace::force_capture()); + warn!("Time went backwards from beginning_of_time Instant"); + } + + let duration_since_time_0 = self.duration_since(BEGINNING_OF_TIME.0); + BEGINNING_OF_TIME.1 + duration_since_time_0 + } +} + +impl SystemTimeExt for SystemTime { + fn from_ntp_64(v: u64) -> SystemTime { + // https://tools.ietf.org/html/rfc3550#section-4 + // Wallclock time (absolute date and time) is represented using the + // timestamp format of the Network Time Protocol (NTP), which is in + // seconds relative to 0h UTC on 1 January 1900 [4]. The full + // resolution NTP timestamp is a 64-bit unsigned fixed-point number with + // the integer part in the first 32 bits and the fractional part in the + // last 32 bits. + let secs_ntp = (v as f64) / F32; + + // Shift to UNIX EPOCH + let secs_epoch = secs_ntp - SECS_1900 as f64; + + // Duration not allowed to be negative + let secs_dur = Duration::try_from_secs_f64(secs_epoch).unwrap_or(Duration::ZERO); + + // Time in SystemTime + SystemTime::UNIX_EPOCH + secs_dur + } + + fn as_ntp_64(&self) -> u64 { + let secs_epoch = if let Ok(value) = self.duration_since(SystemTime::UNIX_EPOCH) { + value.as_secs_f64() + SECS_1900 as f64 + } else { + 0.0 + }; + + (secs_epoch * F32) as u64 + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn not_happening_works() { + assert_eq!(not_happening(), not_happening()); + assert!(Instant::now() < not_happening()); + } + + #[test] + fn already_happened_works() { + assert_eq!(already_happened(), already_happened()); + assert!(Instant::now() > already_happened()); + } + + #[test] + fn already_happened_ne() { + assert_ne!(not_happening(), already_happened()) + } + + #[test] + fn ntp_64_from_to() { + let now = SystemTime::now(); + let ntp = now.as_ntp_64(); + let now2 = SystemTime::from_ntp_64(ntp); + let abs = if now > now2 { + now.duration_since(now2) + } else { + now2.duration_since(now) + }; + assert!(abs.unwrap() < Duration::from_millis(1)); + } + + #[test] + fn from_ntp_64_zero() { + let s = SystemTime::from_ntp_64(0); + assert_eq!( + s.duration_since(SystemTime::UNIX_EPOCH).unwrap(), + Duration::ZERO + ); + } + + #[test] + fn from_ntp_64_val1() { + let s = SystemTime::from_ntp_64((3971792775u64 << 32) | 4184015405u64); + match s.duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => assert_eq!(n.as_micros(), 1762803975974166), + Err(_) => panic!("Cannot calculate unix epoch"), + } + } +} diff --git a/external/opennow/native/opennow-streamer/vendor/str0m/src/util/value_history.rs b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/value_history.rs new file mode 100644 index 000000000..72e93a2d1 --- /dev/null +++ b/external/opennow/native/opennow-streamer/vendor/str0m/src/util/value_history.rs @@ -0,0 +1,109 @@ +use std::collections::VecDeque; +use std::iter::Sum; +use std::ops::{AddAssign, SubAssign}; +use std::time::{Duration, Instant}; + +/// Holds a history values of type T for up to a certain Duration, as well as the +/// cumulated (total) value. +#[derive(Debug)] +pub(crate) struct ValueHistory { + value: T, + history: VecDeque<(Instant, T)>, + max_time: Duration, +} + +const DEFAULT_VALUE_HISTORY_DURATION: Duration = Duration::from_secs(1); + +impl Default for ValueHistory { + fn default() -> Self { + Self { + value: Default::default(), + history: Default::default(), + max_time: DEFAULT_VALUE_HISTORY_DURATION, + } + } +} + +impl ValueHistory +where + T: Copy + AddAssign + SubAssign + Sum, +{ + /// Adds a timed value + /// Note: time should always monotonically increase in subsequent calls to add() + pub fn push(&mut self, t: Instant, v: T) { + self.value += v; + self.history.push_back((t, v)); + } + + /// Returns the sum of all values in the history up to max_time. Might + /// return stale value unless [`ValueHistory::purge_old`] is called before. + pub fn sum(&self) -> T { + self.value + } + + /// Recalculates sum purging values older than `now - max_time`. + pub fn purge_old(&mut self, now: Instant) { + while { + let Some(front_t) = self.history.front().map(|v| v.0) else { + return; + }; + now.duration_since(front_t) > self.max_time + } { + if let Some((_, v)) = self.history.pop_front() { + self.value -= v; + } + } + } +} + +#[allow(clippy::unchecked_duration_subtraction)] +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use super::ValueHistory; + + #[test] + fn with_value_test() { + let now = Instant::now(); + + let mut h = ValueHistory { + value: 11, + max_time: Duration::from_secs(1), + ..Default::default() + }; + + assert_eq!(h.sum(), 11); + h.purge_old(now); + assert_eq!(h.sum(), 11); + h.push(now - Duration::from_millis(1500), 22); + h.push(now - Duration::from_millis(500), 22); + assert_eq!(h.sum(), 11 + 22 + 22); + h.purge_old(now); + assert_eq!(h.sum(), 11 + 22); + h.push(now, 0); + assert_eq!(h.sum(), 11 + 22); + } + + #[test] + fn test() { + let now = Instant::now(); + let mut h = ValueHistory::default(); + + assert_eq!(h.sum(), 0); + h.push(now - Duration::from_millis(1500), 22); + assert_eq!(h.sum(), 22); + h.purge_old(now); + assert_eq!(h.sum(), 0); + h.push(now - Duration::from_millis(700), 22); + h.push(now - Duration::from_millis(500), 33); + assert_eq!(h.sum(), 22 + 33); + h.purge_old(now); + assert_eq!(h.sum(), 22 + 33); + + h.purge_old(now + Duration::from_millis(400)); + assert_eq!(h.sum(), 33); + h.purge_old(now + Duration::from_millis(600)); + assert_eq!(h.sum(), 0); + } +}